diff --git a/assemblyline/common/classification.py b/assemblyline/common/classification.py index f9d2c138e..a99738bec 100644 --- a/assemblyline/common/classification.py +++ b/assemblyline/common/classification.py @@ -342,9 +342,11 @@ def _get_c12n_groups(self, c12n: list[str], long_format: bool = True, get_dynami # Swap to long format if required if long_format: - return sorted( - [self.groups_map_stl.get(r, r) for r in g1_set]), sorted( - [self.subgroups_map_stl[r] for r in g2_set]), list(others) + return ( + sorted([self.groups_map_stl.get(r, r) for r in g1_set]), + sorted([self.subgroups_map_stl[r] for r in g2_set]), + list(others) + ) return sorted(list(g1_set)), sorted(list(g2_set)), list(others) @staticmethod diff --git a/assemblyline/filestore/__init__.py b/assemblyline/filestore/__init__.py index 7da437b50..ce82a8c72 100644 --- a/assemblyline/filestore/__init__.py +++ b/assemblyline/filestore/__init__.py @@ -77,7 +77,8 @@ def create_transport(url, connection_attempts=None): sftp: private_key (string), private_key_pass (string), validate_host (bool) s3: aws_region (string), s3_bucket (string), use_ssl (bool), verify (bool) file: normalize (bool) - azure: access_key (string), tenant_id (string), client_id (string), client_secret (string), allow_directory_access (bool), use_default_credentials (bool) + azure: access_key (string), tenant_id (string), client_id (string), client_secret (string), + allow_directory_access (bool), use_default_credentials (bool), initalize_container (bool) """ @@ -137,7 +138,7 @@ def create_transport(url, connection_attempts=None): elif scheme == 'azure': valid_str_keys = ['access_key', 'tenant_id', 'client_id', 'client_secret'] - valid_bool_keys = ['allow_directory_access', 'use_default_credentials'] + valid_bool_keys = ['allow_directory_access', 'use_default_credentials', 'initalize_container'] extras = _get_extras(parse_qs(parsed.query), valid_str_keys=valid_str_keys, valid_bool_keys=valid_bool_keys) t = TransportAzure(base=base, host=host, connection_attempts=connection_attempts, **extras) diff --git a/assemblyline/filestore/transport/azure.py b/assemblyline/filestore/transport/azure.py index c48bd19dd..302b66aa1 100644 --- a/assemblyline/filestore/transport/azure.py +++ b/assemblyline/filestore/transport/azure.py @@ -29,7 +29,8 @@ class TransportAzure(Transport): def __init__(self, base=None, access_key=None, tenant_id=None, client_id=None, client_secret=None, - host=None, connection_attempts=None, allow_directory_access=False, use_default_credentials=False): + host=None, connection_attempts=None, allow_directory_access=False, use_default_credentials=False, + initalize_container=True): self.log = logging.getLogger('assemblyline.transport.azure') self.read_only = False self.connection_attempts: Optional[int] = connection_attempts @@ -69,18 +70,19 @@ def __init__(self, base=None, access_key=None, tenant_id=None, client_id=None, c self.container_client = self.service_client.get_container_client(self.blob_container) # Init - try: - self.with_retries(self.container_client.get_container_properties) - except TransportException as e: - if not isinstance(e.cause, ResourceNotFoundError): - raise + if initalize_container: try: - self.with_retries(self.container_client.create_container) - except TransportException as error: - if not isinstance(error.cause, ResourceNotFoundError): + self.with_retries(self.container_client.get_container_properties) + except TransportException as e: + if not isinstance(e.cause, ResourceNotFoundError): raise - self.log.info('Failed to create container, we\'re most likely in read only mode') - self.read_only = True + try: + self.with_retries(self.container_client.create_container) + except TransportException as error: + if not isinstance(error.cause, ResourceNotFoundError): + raise + self.log.info('Failed to create container, we\'re most likely in read only mode') + self.read_only = True def azure_normalize(path): # flatten path to just the basename @@ -115,7 +117,7 @@ def with_retries(self, func, *args, **kwargs): raise except Exception as e: - self.log.warning(f"No connection to Azure transport " + self.log.warning(f"Could not run {func}, No connection to Azure transport " f"{os.path.join(self.endpoint_url, self.blob_container)}, retrying... " f"[{e.__class__.__name__}: {str(e)}]") time.sleep(0.25)