diff --git a/.travis.yml b/.travis.yml index 6520fddcb..7240c3d21 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,9 @@ language: python sudo: required python: - - "3" + - "3.6" + - "3.7" + - "3.8" install: - pip install tox codecov diff --git a/dolphin/alert_manager/alert_processor.py b/dolphin/alert_manager/alert_processor.py index 71b06410e..281e63fe4 100644 --- a/dolphin/alert_manager/alert_processor.py +++ b/dolphin/alert_manager/alert_processor.py @@ -47,13 +47,13 @@ def process_alert_info(self, alert): access_info = db.access_info_get_all(ctxt, filters=filters) except Exception: msg = "Access information could not be found with host %s." \ - % alert['transport_address'] + % alert['transport_address'] raise exception.AccessInfoNotFound(message=msg) # For given source ip, there should be unique access_info if len(access_info) != 1: msg = "Failed to get unique access information with host %s." \ - % alert['transport_address'] + % alert['transport_address'] raise exception.InvalidResults(message=msg) try: @@ -80,4 +80,3 @@ def process_alert_info(self, alert): def _export_alert_model(self, alert_model): """Exports the filled alert model to the export manager.""" LOG.info('Alert model to be exported: %s.', alert_model) - diff --git a/dolphin/alert_manager/constants.py b/dolphin/alert_manager/constants.py index f783a7dd2..365039a63 100644 --- a/dolphin/alert_manager/constants.py +++ b/dolphin/alert_manager/constants.py @@ -19,13 +19,13 @@ TRAP_RECEIVER_CLASS = 'dolphin.alert_manager.trap_receiver.TrapReceiver' # Temporary snmp community and user configurations -SNMP_COMMUNITY_STR='public' -SNMP_USM_USER='test1' -SNMP_V3_AUTHKEY='abcd123456' -SNMP_V3_PRIVKEY='abcd123456' -SNMP_V3_AUTH_PROTOCOL='usmHMACMD5AuthProtocol' -SNMP_V3_PRIV_PROTOCOL='usmDESPrivProtocol' -SNMP_ENGINE_ID='800000d30300000e112245' +SNMP_COMMUNITY_STR = 'public' +SNMP_USM_USER = 'test1' +SNMP_V3_AUTHKEY = 'abcd123456' +SNMP_V3_PRIVKEY = 'abcd123456' +SNMP_V3_AUTH_PROTOCOL = 'usmHMACMD5AuthProtocol' +SNMP_V3_PRIV_PROTOCOL = 'usmDESPrivProtocol' +SNMP_ENGINE_ID = '800000d30300000e112245' # Temporary mib lod dir. This mechanism to be changed later SNMP_MIB_PATH = '/usr/local/lib/python3.6/dist-packages/pysnmp/smi/mibs' diff --git a/dolphin/alert_manager/trap_receiver.py b/dolphin/alert_manager/trap_receiver.py index 692abe843..df0cade49 100644 --- a/dolphin/alert_manager/trap_receiver.py +++ b/dolphin/alert_manager/trap_receiver.py @@ -29,7 +29,7 @@ # Currently static mib file list is loaded # Mechanism to be changed to load all mib file -MIB_LOAD_LIST = ['SNMPv2-MIB','IF_MIB'] +MIB_LOAD_LIST = ['SNMPv2-MIB', 'IF_MIB'] class TrapReceiver(object): @@ -109,8 +109,8 @@ def _cb_fun(self, state_reference, context_engine_id, context_name, context_name.prettyPrint(), exec_context['securityModel'], exec_context['securityName'])) - var_binds = [rfc1902.ObjectType(rfc1902.ObjectIdentity(x[0]), x[1]) - .resolveWithMib(self.mib_view_controller) + var_binds = [rfc1902.ObjectType(rfc1902.ObjectIdentity(x[0]), + x[1]).resolveWithMib(self.mib_view_controller) for x in var_binds] alert = {} diff --git a/dolphin/api/api_utils.py b/dolphin/api/api_utils.py index b33ce3860..2a306bcf7 100644 --- a/dolphin/api/api_utils.py +++ b/dolphin/api/api_utils.py @@ -20,6 +20,7 @@ from oslo_utils import strutils from dolphin.common import constants +from dolphin.i18n import _ api_common_opts = [ cfg.IntOpt('api_max_limit', diff --git a/dolphin/api/common/wsgi.py b/dolphin/api/common/wsgi.py index 5b7b53605..32c5544c0 100644 --- a/dolphin/api/common/wsgi.py +++ b/dolphin/api/common/wsgi.py @@ -175,7 +175,7 @@ def best_match_content_type(self): content_type = self.accept.best_match(SUPPORTED_CONTENT_TYPES) self.environ['dolphin.best_content_type'] = (content_type or - 'application/json') + 'application/json') return self.environ['dolphin.best_content_type'] @@ -262,6 +262,7 @@ def decorator(func): func.wsgi_serializers = {} func.wsgi_serializers.update(serializers) return func + return decorator @@ -278,6 +279,7 @@ def decorator(func): func.wsgi_deserializers = {} func.wsgi_deserializers.update(deserializers) return func + return decorator @@ -292,6 +294,7 @@ def response(code): def decorator(func): func.wsgi_code = code return func + return decorator @@ -845,6 +848,7 @@ def action(name): def decorator(func): func.wsgi_action = name return func + return decorator diff --git a/dolphin/api/extensions.py b/dolphin/api/extensions.py index a445ead2e..16769060a 100644 --- a/dolphin/api/extensions.py +++ b/dolphin/api/extensions.py @@ -326,4 +326,3 @@ def load_standard_extensions(ext_mgr, logger, path, package, ext_list=None): # Update the list of directories we'll explore... dirnames[:] = subdirs - diff --git a/dolphin/api/schemas/alert.py b/dolphin/api/schemas/alert.py index 5118e2b9b..370995bb1 100644 --- a/dolphin/api/schemas/alert.py +++ b/dolphin/api/schemas/alert.py @@ -32,4 +32,3 @@ 'required': ['host', 'version'], 'additionalProperties': False, } - diff --git a/dolphin/api/v1/alert.py b/dolphin/api/v1/alert.py index 64e3ad688..187186247 100644 --- a/dolphin/api/v1/alert.py +++ b/dolphin/api/v1/alert.py @@ -52,7 +52,7 @@ def put(self, req, id, body): alert_source = db.alert_source_update(ctx, id, alert_source) else: alert_source = db.alert_source_create(ctx, alert_source) - except exception.StorageNotFound as e: + except exception.StorageNotFound: msg = (_("Alert source cannot be created or updated for a" " non-existing storage %s.") % id) raise exc.HTTPBadRequest(explanation=msg) diff --git a/dolphin/api/v1/pools.py b/dolphin/api/v1/pools.py index ec61e9a68..909a216ed 100644 --- a/dolphin/api/v1/pools.py +++ b/dolphin/api/v1/pools.py @@ -20,7 +20,6 @@ from dolphin.api import api_utils from dolphin.api.common import wsgi from dolphin.api.views import pools as pool_view -from dolphin.i18n import _ class PoolController(wsgi.Controller): @@ -53,7 +52,7 @@ def index(self, req): try: pools = db.pool_get_all(ctxt, marker, limit, sort_keys, sort_dirs, query_params, offset) - except exception.InvalidInput as e: + except exception.InvalidInput as e: raise exc.HTTPBadRequest(explanation=six.text_type(e)) return pool_view.build_pools(pools) diff --git a/dolphin/api/v1/storages.py b/dolphin/api/v1/storages.py index cd0d68bc9..236c5dc42 100644 --- a/dolphin/api/v1/storages.py +++ b/dolphin/api/v1/storages.py @@ -64,7 +64,6 @@ def _get_storages_search_options(self): def index(self, req): ctxt = req.environ['dolphin.context'] - supported_filters = ['name', 'vendor', 'model', 'status'] query_params = {} query_params.update(req.GET) # update options other than filters @@ -76,7 +75,7 @@ def index(self, req): try: storages = db.storage_get_all(context, marker, limit, sort_keys, sort_dirs, query_params, offset) - except exception.InvalidInput as e: + except exception.InvalidInput as e: raise exc.HTTPBadRequest(explanation=six.text_type(e)) return storage_view.build_storages(storages) diff --git a/dolphin/api/v1/volumes.py b/dolphin/api/v1/volumes.py index c2858b7ed..d7553ebe4 100644 --- a/dolphin/api/v1/volumes.py +++ b/dolphin/api/v1/volumes.py @@ -26,5 +26,3 @@ def show(self, req, id): def create_resource(): return wsgi.Resource(VolumeController()) - - diff --git a/dolphin/cmd/api.py b/dolphin/cmd/api.py index 0d0815876..22d5965ee 100644 --- a/dolphin/cmd/api.py +++ b/dolphin/cmd/api.py @@ -26,7 +26,7 @@ from oslo_config import cfg from oslo_log import log -from dolphin.common import config # Need to register global_opts +from dolphin.common import config # noqa from dolphin import service from dolphin import utils from dolphin import version @@ -47,7 +47,7 @@ def main(): launcher.launch_service(api_server, workers=api_server.workers or 1) launcher.launch_service(task_server) - #Launch alert manager service + # Launch alert manager service alert_manager = service.AlertMngrService() launcher.launch_service(alert_manager) diff --git a/dolphin/coordination.py b/dolphin/coordination.py index 8b4844b44..5aea55779 100644 --- a/dolphin/coordination.py +++ b/dolphin/coordination.py @@ -25,10 +25,8 @@ from dolphin import exception from dolphin.i18n import _ - LOG = log.getLogger(__name__) - coordination_opts = [ cfg.StrOpt('backend_url', default='redis://127.0.0.1:6379', @@ -117,6 +115,7 @@ class Lock(locking.Lock): Available field names are keys of lock_data. """ + def __init__(self, lock_name, lock_data=None, coordinator=None): super(Lock, self).__init__(six.text_type(id(self))) lock_data = lock_data or {} @@ -191,6 +190,7 @@ def foo(self, shr, snap): Available field names are: decorated function parameters and `f_name` as a decorated function name. """ + @decorator.decorator def _synchronized(f, *a, **k): call_args = inspect.getcallargs(f, *a, **k) @@ -198,6 +198,7 @@ def _synchronized(f, *a, **k): lock = Lock(lock_name, call_args, coordinator) with lock(blocking): LOG.info('Lock "%(name)s" acquired by "%(function)s".', - {'name': lock_name, 'function': f.__name__}) + {'name': lock_name, 'function': f.__name__}) return f(*a, **k) + return _synchronized diff --git a/dolphin/db/sqlalchemy/api.py b/dolphin/db/sqlalchemy/api.py index b6a9bd8c0..2d6b8bd09 100755 --- a/dolphin/db/sqlalchemy/api.py +++ b/dolphin/db/sqlalchemy/api.py @@ -47,9 +47,9 @@ def apply_sorting(model, query, sort_key, sort_dir): if sort_dir.lower() not in ('desc', 'asc'): - msg = ("Wrong sorting data provided: sort key is '%(sort_key)s' " - "and sort order is '%(sort_dir)s'.") % { - "sort_key": sort_key, "sort_dir": sort_dir} + msg = (("Wrong sorting data provided: sort key is '%(sort_key)s' " + "and sort order is '%(sort_dir)s'.") % + {"sort_key": sort_key, "sort_dir": sort_dir}) raise exception.InvalidInput(reason=msg) sort_attr = getattr(model, sort_key) @@ -181,7 +181,7 @@ def access_info_update(context, storage_id, values): def access_info_delete(context, storage_id): """Delete a storage access information.""" - _access_info_get_query(context).\ + _access_info_get_query(context). \ filter_by(storage_id=storage_id).delete() @@ -397,11 +397,7 @@ def volumes_update(context, volumes): def volume_get(context, volume_id): """Get a volume or raise an exception if it does not exist.""" - return _volume_get(context, vol_id) - - -def _volume_get_query(context, session=None): - return model_query(context, models.Volume, session=session) + return _volume_get(context, volume_id) def volume_get_all(context, marker=None, limit=None, sort_keys=None, @@ -412,8 +408,7 @@ def volume_get_all(context, marker=None, limit=None, sort_keys=None, # Generate the query query = _generate_paginate_query(context, session, models.Volume, marker, limit, sort_keys, sort_dirs, - filters, offset - ) + filters, offset) # No volume would match, return empty list if query is None: return [] @@ -682,7 +677,7 @@ def alert_source_delete(context, storage_id): def alert_source_get_all(context, marker=None, limit=None, sort_keys=None, - sort_dirs=None, filters=None, offset=None): + sort_dirs=None, filters=None, offset=None): session = get_session() with session.begin(): query = _generate_paginate_query(context, session, models.AlertSource, diff --git a/dolphin/db/sqlalchemy/models.py b/dolphin/db/sqlalchemy/models.py index bb13c4b36..10523081f 100755 --- a/dolphin/db/sqlalchemy/models.py +++ b/dolphin/db/sqlalchemy/models.py @@ -18,7 +18,6 @@ """ SQLAlchemy models for Dolphin data. """ -import json from oslo_config import cfg from oslo_db.sqlalchemy import models diff --git a/dolphin/drivers/api.py b/dolphin/drivers/api.py index cb1555081..ba021b6f1 100644 --- a/dolphin/drivers/api.py +++ b/dolphin/drivers/api.py @@ -70,12 +70,12 @@ def remove_storage(self, context, storage_id): def get_storage(self, context, storage_id): """Get storage device information from storage system""" - driver = self.driver_manager.get_driver(context, storage_id) + driver = self.driver_manager.get_driver(context, storage_id=storage_id) return driver.get_storage(context) def list_pools(self, context, storage_id): """List all storage pools from storage system.""" - driver = self.driver_manager.get_driver(context, storage_id) + driver = self.driver_manager.get_driver(context, storage_id=storage_id) return driver.list_pools(context) def list_volumes(self, context, storage_id): @@ -92,11 +92,9 @@ def remove_trap_config(self, context, storage_id, trap_config): def parse_alert(self, context, storage_id, alert): """Parse alert data got from snmp trap server.""" - driver = self.driver_manager.get_driver(context, **{'storage_id': storage_id}) + driver = self.driver_manager.get_driver(context, storage_id=storage_id) driver.parse_alert(context, alert) def clear_alert(self, context, storage_id, alert): """Clear alert from storage system.""" pass - - diff --git a/dolphin/drivers/dell_emc/vmax/client.py b/dolphin/drivers/dell_emc/vmax/client.py index c03c402d3..fd414eb08 100644 --- a/dolphin/drivers/dell_emc/vmax/client.py +++ b/dolphin/drivers/dell_emc/vmax/client.py @@ -12,21 +12,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json import PyU4V -from dolphin import exception -from dolphin.common import constants from oslo_log import log from oslo_utils import units +from dolphin import exception +from dolphin.common import constants + LOG = log.getLogger(__name__) -SUPPORTED_VERSION='90' +SUPPORTED_VERSION = '90' class VMAXClient(object): """ Client class for communicating with VMAX storage """ + def __init__(self): self.conn = None self.array_id = None @@ -39,8 +40,8 @@ def __del__(self): def init_connection(self, access_info): """ Given the access_info get a connection to VMAX storage """ - self.array_id = access_info.get('extra_attributes', {}).\ - get('array_id', None) + self.array_id = access_info.get('extra_attributes', {}). \ + get('array_id', None) if not self.array_id: raise exception.InvalidInput(reason='Input array_id is missing') diff --git a/dolphin/drivers/dell_emc/vmax/vmax.py b/dolphin/drivers/dell_emc/vmax/vmax.py index 50345ab5b..5ede9928b 100644 --- a/dolphin/drivers/dell_emc/vmax/vmax.py +++ b/dolphin/drivers/dell_emc/vmax/vmax.py @@ -33,19 +33,6 @@ def __init__(self, **kwargs): def _init_vmax(self, access_info): self.client.init_connection(access_info) - # Get the VMAX version to check connection - version = self.client.get_version() - - @staticmethod - def get_storage_registry(): - required_register_attributes = super.get_storage_registry() - extra_attributes = { - 'array_id': "The storage id in unisphere system.", - } - required_register_attributes['extra_attributes'] = extra_attributes - - return required_register_attributes - def get_storage(self, context): # Get the VMAX model diff --git a/dolphin/service.py b/dolphin/service.py index b403a213e..7b80ee3c8 100644 --- a/dolphin/service.py +++ b/dolphin/service.py @@ -20,20 +20,18 @@ import inspect import os import random -import socket +import oslo_messaging as messaging from oslo_config import cfg from oslo_log import log +from oslo_service import loopingcall from oslo_service import service from oslo_service import wsgi -from oslo_service import loopingcall from oslo_utils import importutils -import oslo_messaging as messaging from dolphin import context -from dolphin import exception -from dolphin import rpc from dolphin import coordination +from dolphin import rpc from dolphin.alert_manager import constants LOG = log.getLogger(__name__) @@ -218,6 +216,7 @@ def periodic_tasks(self, raise_on_error=False): ctxt = context.get_admin_context() self.manager.periodic_tasks(ctxt, raise_on_error=raise_on_error) + class AlertMngrService(service.Service): """Service object for triggering trap receiver functionalities. """ @@ -253,6 +252,7 @@ def stop(self): """Calls the shutdown flow of the service.""" self.manager.stop() + class WSGIService(service.ServiceBase): """Provides ability to launch API from a 'paste' configuration.""" diff --git a/dolphin/task_manager/manager.py b/dolphin/task_manager/manager.py index 1ea4e48f5..81752f194 100644 --- a/dolphin/task_manager/manager.py +++ b/dolphin/task_manager/manager.py @@ -63,7 +63,7 @@ def say_hello(self, context, request_spec=None, # report data to northbound platform base_exporter.dispatch_example_data(data) - except Exception as ex: + except Exception: pass def sync_storage_resource(self, context, storage_id, resource_task): diff --git a/dolphin/task_manager/tasks/task.py b/dolphin/task_manager/tasks/task.py index 75f8df6fa..23cdcb381 100644 --- a/dolphin/task_manager/tasks/task.py +++ b/dolphin/task_manager/tasks/task.py @@ -128,11 +128,6 @@ def remove(self): db.pool_delete_by_storage(self.context, self.storage_id) -class StorageVolumeTask(StorageResourceTask): - def sync(self): - pass - - class StorageVolumeTask(StorageResourceTask): def __init__(self, context, storage_id): super(StorageVolumeTask, self).__init__(context, storage_id) diff --git a/dolphin/tests/api/__init__.py b/dolphin/tests/unit/__init__.py similarity index 100% rename from dolphin/tests/api/__init__.py rename to dolphin/tests/unit/__init__.py diff --git a/dolphin/tests/api/v1/__init__.py b/dolphin/tests/unit/api/__init__.py similarity index 100% rename from dolphin/tests/api/v1/__init__.py rename to dolphin/tests/unit/api/__init__.py diff --git a/dolphin/tests/unit/api/v1/__init__.py b/dolphin/tests/unit/api/v1/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/dolphin/tests/api/v1/test_storages.py b/dolphin/tests/unit/api/v1/test_storages.py similarity index 100% rename from dolphin/tests/api/v1/test_storages.py rename to dolphin/tests/unit/api/v1/test_storages.py diff --git a/dolphin/utils.py b/dolphin/utils.py index 2abfa497a..ef629ad5b 100644 --- a/dolphin/utils.py +++ b/dolphin/utils.py @@ -27,7 +27,6 @@ import shutil import sys import tempfile -import time import threading from eventlet import pools @@ -481,6 +480,7 @@ def retry(exception, interval=1, retries=10, backoff_rate=2, :param backoff_sleep_max: Maximum number of seconds for the calculated backoff sleep. Use None if no maximum is needed. """ + def _retry_on_exception(e): return isinstance(e, exception) @@ -578,6 +578,7 @@ def wrapper(self, *args, **kwargs): driver_name = self.driver.__class__.__name__ raise exception.DriverNotInitialized(driver=driver_name) return func(self, *args, **kwargs) + return wrapper @@ -666,11 +667,13 @@ def notifications_enabled(conf): def if_notifications_enabled(function): """Calls decorated method only if notifications are enabled.""" + @functools.wraps(function) def wrapped(*args, **kwargs): if notifications_enabled(CONF): return function(*args, **kwargs) return DO_NOTHING + return wrapped diff --git a/pytest.ini b/pytest.ini index b759cd8f1..e52271e8b 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,4 +1,3 @@ [pytest] usefixtures = cache python_files = test_*.py __init__.py -addopts = -p no:warnings diff --git a/tox.ini b/tox.ini index 3a03d7752..ecffdb6f1 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,7 @@ skipsdist = True skip_missing_interpreters = True envlist = - py3 + py3, pep8 [testenv] @@ -20,5 +20,5 @@ deps = flake8 commands = flake8 {posargs:dolphin} [flake8] -ignore = E402 +ignore = E402,W503,W504 max-line-length = 120