Massive update

- Switched to using a config class instead of a dict.  Much
  cleaner but this touched almost every file.
- Lots of flake8 fixes
- Switched a large chunk of old strings to f-strings.

Author: scanon

imagegw/docker-compose.yml

@@ -1,40 +1,46 @@
-api:
-  image: imagegwapi
-  ports:
-   - 5555:8000
-  links:
-   - mongo
-  volumes:
-   - imagegw:/images
-   - munge:/var/run/munge/
-   - ./test/config:/config
-  command: api
-munge:
-  image: munge
-  volumes:
-   - munge:/var/run/munge/
-mongo:
-  image: mongo:3
-  volumes:
-    - /data/db
-  command: --smallfiles
-systema:
-  container_name: systema
-  image: shifter-test
-  extra_hosts:
-   - "api:172.17.0.1"
-   - "registry:172.17.0.1"
-  ports:
-   - "2222:22"
-  volumes:
-  - /images
-  - ./test/config/:/config
-  privileged: true
-  environment:
-  - ADDUSER=canon
-registry:
-  image: registry
-  ports:
-   - "5000:5000"
-  volumes:
-  - registry:/var/lib/registry
+services:
+  api:
+    image: imagegwapi
+    ports:
+     - 5555:8000
+    links:
+     - mongo
+    volumes:
+     - imagegw:/images
+     - munge:/var/run/munge/
+     - ./test/config:/config
+    command: api
+  munge:
+    image: munge
+    volumes:
+     - munge:/var/run/munge/
+     - ./test/config:/config
+    command: munge
+  mongo:
+    image: mongo:3
+    volumes:
+      - /data/db
+    command: --smallfiles
+  systema:
+    container_name: systema
+    image: shifter-test
+    extra_hosts:
+     - "api:172.17.0.1"
+     - "registry:172.17.0.1"
+    ports:
+     - "2222:22"
+    volumes:
+    - /images
+    - ./test/config/:/config
+    privileged: true
+    environment:
+    - ADDUSER=canon
+#  registry:
+#    image: registry
+#    ports:
+#     - "5000:5000"
+#    volumes:
+#    - registry:/var/lib/registry
+volumes:
+  imagegw:
+  munge:

imagegw/shifter_imagegw/api.py

@@ -21,49 +21,25 @@
 This module provides the REST API for the image gateway.
 """
 
-import json
-import os
 import logging
-import shifter_imagegw
 from shifter_imagegw.errors import AuthenticationError
 from shifter_imagegw.imagemngr import ImageMngr
 from contextlib import asynccontextmanager
 from fastapi import FastAPI, HTTPException, Request, Header, Query
 from fastapi.responses import JSONResponse, PlainTextResponse
 from pydantic import BaseModel
+from shifter_imagegw.config import Config
 
 
 mgr = None
-logger = logging.getLogger("fastapi.root")
+logger = logging.getLogger("imagegwapi")
 
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):
     global config
-    if 'GWCONFIG' in os.environ:
-        CONFIG_FILE = os.environ['GWCONFIG']
-    else:
-        CONFIG_FILE = f'{shifter_imagegw.CONFIG_PATH}/imagemanager.json'
-    # Configure logging
-    logger.debug('Initializing api image manager')
-
-    logger.info(f"initializing with {CONFIG_FILE}")
-    with open(CONFIG_FILE) as config_file:
-        config = json.load(config_file)
-        if 'LogLevel' in config:
-            LOG_STRING = config['LogLevel'].lower()
-            if LOG_STRING == 'debug':
-                logger.setLevel(logging.DEBUG)
-            elif LOG_STRING == 'info':
-                logger.setLevel(logging.INFO)
-            elif LOG_STRING == 'warn':
-                logger.setLevel(logging.WARN)
-            elif LOG_STRING == 'error':
-                logger.setLevel(logging.ERROR)
-            elif LOG_STRING == 'critical':
-                logger.setLevel(logging.CRITICAL)
-            else:
-                logger.critical('Unrecongnized Log Level specified')
+    config = Config()
+    logger.setLevel(config.LogLevel)
     global mgr
     mgr = ImageMngr(config, logname="fastapi.root")
     yield
@@ -264,10 +240,10 @@ async def doimport(system: str, imgtype: str, tag: str, data: ImportImage,
         # Convert to integers
         i['groupACL'] = list(map(lambda x: int(x),
                              data.allowed_gids.split(',')))
-    if 'ImportUsers' not in config:
+    if not config.ImportUsers:
         raise HTTPException(status_code=403, detail="User image import from file disabled.")
 
-    iusers = config['ImportUsers']
+    iusers = config.ImportUsers
     # If ImportUsers is None, no one can do this
     if iusers.lower() == "none":
         raise HTTPException(status_code=403, detail="User image import from file disabled.")

imagegw/shifter_imagegw/auth.py

@@ -23,32 +23,31 @@
 
 import json
 from shifter_imagegw import munge
+from shifter_imagegw.config import Config
 
 
 class Authentication(object):
     """
     Authentication Class to authenticate user requests
     """
 
-    def __init__(self, config):
+    def __init__(self, config: Config):
         """
         Initializes authenication handle.
         config is a dictionary.  It must define 'Authentication' and it must
         be a supported type (currently munge).
         Different auth mechanisms may require additional key value pairs
         """
-        if 'Authentication' not in config:
-            raise KeyError('Authentication not specified')
         self.sockets = dict()
-        if config['Authentication'] == "munge":
-            for system in config['Platforms']:
+        if config.Authentication == "munge":
+            for system in config.Platforms:
                 self.sockets[system] = \
-                        config['Platforms'][system]['mungeSocketPath']
+                        config.Platforms[system].mungeSocketPath
             self.type = 'munge'
-        elif config['Authentication'] == "mock":
+        elif config.Authentication == "mock":
             self.type = 'mock'
         else:
-            memo = 'Unsupported auth type %s' % (config['Authentication'])
+            memo = f'Unsupported auth type {config.Authentication}'
             raise NotImplementedError(memo)
 
     def _authenticate_munge(self, authstr, system=None):
@@ -75,7 +74,7 @@ def _authenticate_munge(self, authstr, system=None):
         message_json = response['MESSAGE']
         try:
             ret['tokens'] = json.loads(message_json)['authorized_locations']
-        except:
+        except Exception:
             pass
         return ret
 

imagegw/shifter_imagegw/config.py

@@ -0,0 +1,87 @@
+import logging
+import json
+import os
+from shifter_imagegw import CONFIG_PATH
+
+
+class Location():
+    def __init__(self, data: dict):
+        self.remotetype = data['remotetype']
+        self.authentication = data['authentication']
+        self.url = data.get('url')
+        self.sslcacert = None
+
+
+class Platform():
+    def __init__(self, data):
+        if data.get('accesstype') != 'local':
+            return None
+        self.mungeSocketPath = data.get('mungeSocketPath',
+                                        "/var/run/munge/munge.socket.2")
+        self.accesstype = "local"
+        self.admins = data.get('admins', ['root'])
+        self.imageDir = data['local'].get('imageDir', "/tmp")
+        self.policy_file = None
+
+
+class Config():
+    WorkerThreads = 2
+    LogLevel = "info"
+    DefaultImageLocation = "index.docker.io"
+    DefaultImageFormat = "squashfs"
+    PullUpdateTimeout = 300
+    ImageExpirationTimeout = "90:00:00:00"
+    CacheDirectory = "/tmp/imagegw/"
+    ExpandDirectory = "/tmp/imagegw/"
+    MongoDBURI = None
+    MongoDB = "Shifter"
+    Metrics = True
+    Authentication = "munge"
+    ImportUsers = None
+    Locations = {}
+    Platforms = {}
+    examiner = None
+    ConverterOptions = None
+
+    def __init__(self, data=None):
+        if data:
+            config = data
+        else:
+            if 'GWCONFIG' in os.environ:
+                CONFIG_FILE = os.environ['GWCONFIG']
+            else:
+                CONFIG_FILE = f'{CONFIG_PATH}/imagemanager.json'
+            # Configure logging
+            logging.info(f"initializing with {CONFIG_FILE}")
+            with open(CONFIG_FILE) as config_file:
+                config = json.load(config_file)
+        for attr in dir(self):
+            if attr[0] == "_":
+                continue
+            if attr in config:
+                val = config[attr]
+                if attr in ['WorkerThreads', 'PullUpdateTimeout']:
+                    val = int(config[attr])
+                setattr(self, attr, val)
+
+        for location in config['Locations']:
+            data = config['Locations'][location]
+            self.Locations[location] = Location(data)
+
+        for platform in config['Platforms']:
+            data = config['Platforms'][platform]
+            self.Platforms[platform] = Platform(data)
+
+        self.Locations = config['Locations']
+        self.Platofrms = config['Platforms']
+
+        loglevel = config.get('LogLevel', 'info').lower()
+        loglevel_map = {
+            'debug': logging.DEBUG,
+            'info': logging.INFO,
+            'warn': logging.WARNING,
+            'error': logging.ERROR,
+            'critical': logging.CRITICAL
+        }
+        self.LogLevel = loglevel_map[loglevel]
+        self.worker_threads = config.get('WorkerThreads')

imagegw/shifter_imagegw/converters.py

@@ -25,7 +25,6 @@
 
 import os
 import subprocess
-import shutil
 import tempfile
 from shifter_imagegw.util import program_exists, rmtree
 
@@ -43,21 +42,8 @@ def generate_cramfs_image(expand_path, image_path, options):
     """
     Creates a CramFS based image
     """
-    program_exists('mkfs.cramfs')
-    cmd = ["mkfs.cramfs", expand_path, image_path]
-    if options is not None:
-        cmd.extend(options)
-    ret = subprocess.call(cmd)
-    if ret != 0:
-        # error handling
-        pass
-    try:
-        rmtree(expand_path)
-    except:
-        # error handling
-        pass
-
-    return True
+    message = 'cramfs support is not supported'
+    raise NotImplementedError(message)
 
 
 def generate_squashfs_image(expand_path, image_path, options):
@@ -80,7 +66,7 @@ def generate_squashfs_image(expand_path, image_path, options):
         pass
     try:
         rmtree(expand_path)
-    except:
+    except Exception:
         pass
 
     return True
@@ -121,7 +107,7 @@ def convert(fmt, expand_path, image_path, options=None):
             success = True
         else:
             raise NotImplementedError("%s not a supported format" % fmt)
-    except:
+    except Exception:
         if os.path.exists(temp_path):
             os.unlink(temp_path)
         raise
@@ -130,7 +116,7 @@ def convert(fmt, expand_path, image_path, options=None):
         return False
     try:
         os.rename(temp_path, image_path)
-    except:
+    except Exception:
         return False
 
     # Some error must have occurred