Azure filestore logging

assemblyline/common/classification.py

@@ -342,9 +342,11 @@ def _get_c12n_groups(self, c12n: list[str], long_format: bool = True, get_dynami
 
         # Swap to long format if required
         if long_format:
-            return sorted(
-                [self.groups_map_stl.get(r, r) for r in g1_set]), sorted(
-                [self.subgroups_map_stl[r] for r in g2_set]), list(others)
+            return (
+                sorted([self.groups_map_stl.get(r, r) for r in g1_set]),
+                sorted([self.subgroups_map_stl[r] for r in g2_set]),
+                list(others)
+            )
         return sorted(list(g1_set)), sorted(list(g2_set)), list(others)
 
     @staticmethod

assemblyline/filestore/__init__.py

@@ -77,7 +77,8 @@ def create_transport(url, connection_attempts=None):
         sftp: private_key (string), private_key_pass (string), validate_host (bool)
         s3: aws_region (string), s3_bucket (string), use_ssl (bool), verify (bool)
         file: normalize (bool)
-        azure: access_key (string), tenant_id (string), client_id (string), client_secret (string), allow_directory_access (bool), use_default_credentials (bool)
+        azure: access_key (string), tenant_id (string), client_id (string), client_secret (string), 
+               allow_directory_access (bool), use_default_credentials (bool), initalize_container (bool)
 
     """
 
@@ -137,7 +138,7 @@ def create_transport(url, connection_attempts=None):
 
     elif scheme == 'azure':
         valid_str_keys = ['access_key', 'tenant_id', 'client_id', 'client_secret']
-        valid_bool_keys = ['allow_directory_access', 'use_default_credentials']
+        valid_bool_keys = ['allow_directory_access', 'use_default_credentials', 'initalize_container']
         extras = _get_extras(parse_qs(parsed.query), valid_str_keys=valid_str_keys, valid_bool_keys=valid_bool_keys)
 
         t = TransportAzure(base=base, host=host, connection_attempts=connection_attempts, **extras)

assemblyline/filestore/transport/azure.py

@@ -29,7 +29,8 @@
 class TransportAzure(Transport):
 
     def __init__(self, base=None, access_key=None, tenant_id=None, client_id=None, client_secret=None,
-                host=None, connection_attempts=None, allow_directory_access=False, use_default_credentials=False):
+                 host=None, connection_attempts=None, allow_directory_access=False, use_default_credentials=False,
+                 initalize_container=True):
         self.log = logging.getLogger('assemblyline.transport.azure')
         self.read_only = False
         self.connection_attempts: Optional[int] = connection_attempts
@@ -69,18 +70,19 @@ def __init__(self, base=None, access_key=None, tenant_id=None, client_id=None, c
         self.container_client = self.service_client.get_container_client(self.blob_container)
 
         # Init
-        try:
-            self.with_retries(self.container_client.get_container_properties)
-        except TransportException as e:
-            if not isinstance(e.cause, ResourceNotFoundError):
-                raise
+        if initalize_container:
             try:
-                self.with_retries(self.container_client.create_container)
-            except TransportException as error:
-                if not isinstance(error.cause, ResourceNotFoundError):
+                self.with_retries(self.container_client.get_container_properties)
+            except TransportException as e:
+                if not isinstance(e.cause, ResourceNotFoundError):
                     raise
-                self.log.info('Failed to create container, we\'re most likely in read only mode')
-                self.read_only = True
+                try:
+                    self.with_retries(self.container_client.create_container)
+                except TransportException as error:
+                    if not isinstance(error.cause, ResourceNotFoundError):
+                        raise
+                    self.log.info('Failed to create container, we\'re most likely in read only mode')
+                    self.read_only = True
 
         def azure_normalize(path):
             # flatten path to just the basename
@@ -115,7 +117,7 @@ def with_retries(self, func, *args, **kwargs):
                 raise
 
             except Exception as e:
-                self.log.warning(f"No connection to Azure transport "
+                self.log.warning(f"Could not run {func}, No connection to Azure transport "
                                  f"{os.path.join(self.endpoint_url, self.blob_container)}, retrying... "
                                  f"[{e.__class__.__name__}: {str(e)}]")
                 time.sleep(0.25)