Implements blueprint import-dynamic-stores.

Make glance more pluggable with regard to stores.

Change-Id: I7b264d1b047a321f7b60857bb73154f831b82a7b

Author: Joshua Harlow

etc/glance-api.conf

@@ -5,11 +5,20 @@ verbose = True
 # Show debugging output in logs (sets DEBUG log level output)
 debug = False
 
-# Which backend store should Glance use by default is not specified
-# in a request to add a new image to Glance? Default: 'file'
-# Available choices are 'file', 'swift', and 's3'
+# Which backend scheme should Glance use by default is not specified
+# in a request to add a new image to Glance? Known schemes are determined
+# by the known_stores option below.
+# Default: 'file'
 default_store = file
 
+# List of which store classes and store class locations are
+# currently known to glance at startup.
+known_stores = glance.store.filesystem.Store,
+               glance.store.http.Store,
+               glance.store.rbd.Store,
+               glance.store.s3.Store,
+               glance.store.swift.Store,
+
 # Address to bind the API server
 bind_host = 0.0.0.0
 

glance/api/v1/images.py

@@ -41,13 +41,8 @@
 from glance.common import wsgi
 from glance.common import utils
 from glance.openstack.common import cfg
-import glance.store
-import glance.store.filesystem
-import glance.store.http
-import glance.store.rbd
-import glance.store.s3
-import glance.store.swift
-from glance.store import (get_from_backend,
+from glance.store import (create_stores,
+                          get_from_backend,
                           get_size_from_backend,
                           schedule_delete_from_backend,
                           get_store_from_location,
@@ -67,6 +62,10 @@
 # as a BigInteger.
 IMAGE_SIZE_CAP = 1 << 50
 
+# Defined at module level due to _is_opt_registered
+# identity check (not equality).
+default_store_opt = cfg.StrOpt('default_store', default='file')
+
 
 class Controller(controller.BaseController):
     """
@@ -87,13 +86,11 @@ class Controller(controller.BaseController):
         DELETE /images/<ID> -- Delete the image with id <ID>
     """
 
-    default_store_opt = cfg.StrOpt('default_store', default='file')
-
     def __init__(self, conf):
         self.conf = conf
-        self.conf.register_opt(self.default_store_opt)
-        glance.store.create_stores(conf)
-        self.verify_store_or_exit(self.conf.default_store)
+        self.conf.register_opt(default_store_opt)
+        create_stores(self.conf)
+        self.verify_scheme_or_exit(self.conf.default_store)
         self.notifier = notifier.Notifier(conf)
         registry.configure_registry_client(conf)
         self.policy = policy.Enforcer(conf)
@@ -333,8 +330,8 @@ def _upload(self, req, image_meta):
         """
         Uploads the payload of the request to a backend store in
         Glance. If the `x-image-meta-store` header is set, Glance
-        will attempt to use that store, if not, Glance will use the
-        store set by the flag `default_store`.
+        will attempt to use that scheme; if not, Glance will use the
+        scheme set by the flag `default_store` to find the backing store.
 
         :param req: The WSGI/Webob Request object
         :param image_meta: Mapping of metadata about image
@@ -367,18 +364,18 @@ def _upload(self, req, image_meta):
                                "x-image-meta-size header"))
                 image_size = 0
 
-        store_name = req.headers.get('x-image-meta-store',
-                                     self.conf.default_store)
+        scheme = req.headers.get('x-image-meta-store',
+                                 self.conf.default_store)
 
-        store = self.get_store_or_400(req, store_name)
+        store = self.get_store_or_400(req, scheme)
 
         image_id = image_meta['id']
         logger.debug(_("Setting image %s to status 'saving'"), image_id)
         registry.update_image_metadata(req.context, image_id,
                                        {'status': 'saving'})
         try:
             logger.debug(_("Uploading image data for image %(image_id)s "
-                         "to %(store_name)s store"), locals())
+                           "to %(scheme)s store"), locals())
 
             if image_size > IMAGE_SIZE_CAP:
                 max_image_size = IMAGE_SIZE_CAP
@@ -750,41 +747,39 @@ def delete(self, req, id):
         else:
             self.notifier.info('image.delete', id)
 
-    def get_store_or_400(self, request, store_name):
+    def get_store_or_400(self, request, scheme):
         """
         Grabs the storage backend for the supplied store name
         or raises an HTTPBadRequest (400) response
 
         :param request: The WSGI/Webob Request object
-        :param store_name: The backend store name
+        :param scheme: The backend store scheme
 
         :raises HTTPNotFound if store does not exist
         """
         try:
-            return get_store_from_scheme(store_name)
+            return get_store_from_scheme(scheme)
         except exception.UnknownScheme:
-            msg = (_("Requested store %s not available on this Glance server")
-                   % store_name)
-            logger.error(msg)
+            msg = _("Store for scheme %s not found")
+            logger.error(msg % scheme)
             raise HTTPBadRequest(msg, request=request,
                                  content_type='text/plain')
 
-    def verify_store_or_exit(self, store_name):
+    def verify_scheme_or_exit(self, scheme):
         """
         Verifies availability of the storage backend for the
-        given store name or exits
+        given scheme or exits
 
-        :param store_name: The backend store name
+        :param scheme: The backend store scheme
         """
         try:
-            get_store_from_scheme(store_name)
+            get_store_from_scheme(scheme)
         except exception.UnknownScheme:
-            msg = (_("Default store %s not available on this Glance server\n")
-                   % store_name)
-            logger.error(msg)
+            msg = _("Store for scheme %s not found")
+            logger.error(msg % scheme)
             # message on stderr will only be visible if started directly via
             # bin/glance-api, as opposed to being daemonized by glance-control
-            sys.stderr.write(msg)
+            sys.stderr.write(msg % scheme)
             sys.exit(255)
 
 

glance/api/v2/image_data.py

@@ -20,11 +20,6 @@
 from glance.common import wsgi
 import glance.registry.db.api
 import glance.store
-import glance.store.filesystem
-import glance.store.http
-import glance.store.rbd
-import glance.store.s3
-import glance.store.swift
 
 
 class ImageDataController(base.Controller):

glance/store/__init__.py

@@ -28,9 +28,6 @@
 
 logger = logging.getLogger('glance.store')
 
-# Set of known store modules
-REGISTERED_STORE_MODULES = []
-
 # Set of store objects, constructed in create_stores()
 STORES = {}
 
@@ -128,50 +125,68 @@ def __len__(self):
         return self.size
 
 
-def register_store(store_module, schemes):
-    """
-    Registers a store module and a set of schemes
-    for which a particular URI request should be routed.
-
-    :param store_module: String representing the store module
-    :param schemes: List of strings representing schemes for
-                    which this store should be used in routing
-    """
+def _get_store_class(store_entry):
+    store_cls = None
     try:
-        utils.import_class(store_module + '.Store')
+        logger.debug("Attempting to import store %s", store_entry)
+        store_cls = utils.import_class(store_entry)
     except exception.NotFound:
-        raise BackendException('Unable to register store. Could not find '
-                               'a class named Store in module %s.'
-                               % store_module)
-    REGISTERED_STORE_MODULES.append(store_module)
-    scheme_map = {}
-    for scheme in schemes:
-        scheme_map[scheme] = store_module
-    location.register_scheme_map(scheme_map)
+        raise BackendException('Unable to load store. '
+                               'Could not find a class named %s.'
+                               % store_entry)
+    return store_cls
+
+
+known_stores_opt = cfg.ListOpt('known_stores',
+                               default=('glance.store.filesystem.Store',))
 
 
 def create_stores(conf):
     """
-    Construct the store objects with supplied configuration options
+    Registers all store modules and all schemes
+    from the given config. Duplicates are not re-registered.
     """
-    for store_module in REGISTERED_STORE_MODULES:
-        try:
-            store_class = utils.import_class(store_module + '.Store')
-        except exception.NotFound:
-            raise BackendException('Unable to create store. Could not find '
-                                   'a class named Store in module %s.'
-                                   % store_module)
-        STORES[store_module] = store_class(conf)
+    conf.register_opt(known_stores_opt)
+    store_count = 0
+    for store_entry in conf.known_stores:
+        store_entry = store_entry.strip()
+        if not store_entry:
+            continue
+        store_cls = _get_store_class(store_entry)
+        store_instance = store_cls(conf)
+        schemes = store_instance.get_schemes()
+        if not schemes:
+            raise BackendException('Unable to register store %s. '
+                                   'No schemes associated with it.'
+                                   % store_cls)
+        else:
+            if store_cls not in STORES:
+                logger.debug("Registering store %s with schemes %s",
+                         store_cls, schemes)
+                STORES[store_cls] = store_instance
+                scheme_map = {}
+                for scheme in schemes:
+                    loc_cls = store_instance.get_store_location_class()
+                    scheme_map[scheme] = {
+                        'store_class': store_cls,
+                        'location_class': loc_cls,
+                    }
+                location.register_scheme_map(scheme_map)
+                store_count += 1
+            else:
+                logger.debug("Store %s already registered", store_cls)
+    return store_count
 
 
 def get_store_from_scheme(scheme):
     """
     Given a scheme, return the appropriate store object
-    for handling that scheme
+    for handling that scheme.
     """
-    if scheme not in location.SCHEME_TO_STORE_MAP:
+    if scheme not in location.SCHEME_TO_CLS_MAP:
         raise exception.UnknownScheme(scheme=scheme)
-    return STORES[location.SCHEME_TO_STORE_MAP[scheme]]
+    scheme_info = location.SCHEME_TO_CLS_MAP[scheme]
+    return STORES[scheme_info['store_class']]
 
 
 def get_store_from_uri(uri):

glance/store/base.py

@@ -20,6 +20,7 @@
 import logging
 
 from glance.common import exception
+from glance.common import utils
 
 logger = logging.getLogger('glance.store.base')
 
@@ -35,7 +36,7 @@ def __init__(self, conf):
         :param conf: Optional dictionary of configuration options
         """
         self.conf = conf
-
+        self.store_location_class = None
         self.configure()
 
         try:
@@ -54,6 +55,22 @@ def configure(self):
         """
         pass
 
+    def get_schemes(self):
+        """
+        Returns a tuple of schemes which this store can handle.
+        """
+        raise NotImplementedError
+
+    def get_store_location_class(self):
+        """
+        Returns the store location class that is used by this store.
+        """
+        if not self.store_location_class:
+            class_name = "%s.StoreLocation" % (self.__module__)
+            logger.debug("Late loading location class %s", class_name)
+            self.store_location_class = utils.import_class(class_name)
+        return self.store_location_class
+
     def configure_add(self):
         """
         This is like `configure` except that it's specifically for

glance/store/filesystem.py

@@ -98,6 +98,9 @@ class Store(glance.store.base.Store):
 
     datadir_opt = cfg.StrOpt('filesystem_store_datadir')
 
+    def get_schemes(self):
+        return ('file', 'filesystem')
+
     def configure_add(self):
         """
         Configure the Store to use the stored configuration options
@@ -216,6 +219,3 @@ def add(self, image_id, image_file, image_size):
         logger.debug(_("Wrote %(bytes_written)d bytes to %(filepath)s with "
                      "checksum %(checksum_hex)s") % locals())
         return ('file://%s' % filepath, bytes_written, checksum_hex)
-
-
-glance.store.register_store(__name__, ['filesystem', 'file'])

glance/store/http.py

@@ -126,6 +126,9 @@ def another(self):
 
         return (ResponseIndexable(iterator, content_length), content_length)
 
+    def get_schemes(self):
+        return ('http', 'https')
+
     def get_size(self, location):
         """
         Takes a `glance.store.location.Location` object that indicates
@@ -155,6 +158,3 @@ def _get_conn_class(self, loc):
         """
         return {'http': httplib.HTTPConnection,
                 'https': httplib.HTTPSConnection}[loc.scheme]
-
-
-glance.store.register_store(__name__, ['http', 'https'])