feat: automated migration

Author: Powlinett

stream/microsoft-sentinel-intel/__metadata__/connector_config_schema.json

@@ -0,0 +1,134 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://www.filigran.io/connectors/microsoft-sentinel-intel_config.schema.json",
+  "type": "object",
+  "properties": {
+    "OPENCTI_URL": {
+      "description": "The base URL of the OpenCTI instance.",
+      "format": "uri",
+      "maxLength": 2083,
+      "minLength": 1,
+      "type": "string"
+    },
+    "OPENCTI_TOKEN": {
+      "description": "The API token to connect to OpenCTI.",
+      "type": "string"
+    },
+    "CONNECTOR_NAME": {
+      "default": "MicrosoftSentinelIntel",
+      "description": "The name of the connector.",
+      "type": "string"
+    },
+    "CONNECTOR_SCOPE": {
+      "description": "The scope of the connector, e.g. 'flashpoint'.",
+      "items": {
+        "type": "string"
+      },
+      "type": "array"
+    },
+    "CONNECTOR_LOG_LEVEL": {
+      "default": "error",
+      "description": "The minimum level of logs to display.",
+      "enum": [
+        "debug",
+        "info",
+        "warn",
+        "warning",
+        "error"
+      ],
+      "type": "string"
+    },
+    "CONNECTOR_TYPE": {
+      "const": "STREAM",
+      "default": "STREAM",
+      "type": "string"
+    },
+    "CONNECTOR_LIVE_STREAM_ID": {
+      "default": "live",
+      "description": "The ID of the live stream to connect to.",
+      "type": "string"
+    },
+    "CONNECTOR_LIVE_STREAM_LISTEN_DELETE": {
+      "default": true,
+      "description": "Whether to listen for delete events on the live stream.",
+      "type": "boolean"
+    },
+    "CONNECTOR_LIVE_STREAM_NO_DEPENDENCIES": {
+      "default": true,
+      "description": "Whether to ignore dependencies when processing events from the live stream.",
+      "type": "boolean"
+    },
+    "MICROSOFT_SENTINEL_INTEL_TENANT_ID": {
+      "description": "Your Azure App Tenant ID, see the screenshot to help you find this information.",
+      "type": "string"
+    },
+    "MICROSOFT_SENTINEL_INTEL_CLIENT_ID": {
+      "description": "Your Azure App Client ID, see the screenshot to help you find this information.",
+      "type": "string"
+    },
+    "MICROSOFT_SENTINEL_INTEL_CLIENT_SECRET": {
+      "description": "Your Azure App Client secret, See the screenshot to help you find this information.",
+      "format": "password",
+      "type": "string",
+      "writeOnly": true
+    },
+    "MICROSOFT_SENTINEL_INTEL_WORKSPACE_ID": {
+      "description": "Your Azure Workspace ID",
+      "type": "string"
+    },
+    "MICROSOFT_SENTINEL_INTEL_WORKSPACE_NAME": {
+      "description": "The name of the log analytics workspace",
+      "type": "string"
+    },
+    "MICROSOFT_SENTINEL_INTEL_SUBSCRIPTION_ID": {
+      "description": "The subscription id where the Log Analytics is",
+      "type": "string"
+    },
+    "MICROSOFT_SENTINEL_INTEL_RESOURCE_GROUP": {
+      "default": "default",
+      "deprecated": true,
+      "description": "The name of the resource group where the log analytics is",
+      "type": "string"
+    },
+    "MICROSOFT_SENTINEL_INTEL_SOURCE_SYSTEM": {
+      "default": "Opencti Stream Connector",
+      "description": "The name of the source system displayed in Microsoft Sentinel",
+      "type": "string"
+    },
+    "MICROSOFT_SENTINEL_INTEL_DELETE_EXTENSIONS": {
+      "default": true,
+      "description": "Delete the extensions in the stix bundle sent to the SIEM",
+      "type": "boolean"
+    },
+    "MICROSOFT_SENTINEL_INTEL_EXTRA_LABELS": {
+      "default": [],
+      "description": "Extra labels added to the bundle sent. String separated by comma",
+      "items": {
+        "type": "string"
+      },
+      "type": "array"
+    },
+    "MICROSOFT_SENTINEL_INTEL_WORKSPACE_API_VERSION": {
+      "default": "2024-02-01-preview",
+      "description": "API version of the Microsoft log analytics workspace interface",
+      "type": "string"
+    },
+    "MICROSOFT_SENTINEL_INTEL_MANAGEMENT_API_VERSION": {
+      "default": "2025-03-01",
+      "description": "API version of the Microsoft management interface",
+      "type": "string"
+    }
+  },
+  "required": [
+    "OPENCTI_URL",
+    "OPENCTI_TOKEN",
+    "CONNECTOR_SCOPE",
+    "MICROSOFT_SENTINEL_INTEL_TENANT_ID",
+    "MICROSOFT_SENTINEL_INTEL_CLIENT_ID",
+    "MICROSOFT_SENTINEL_INTEL_CLIENT_SECRET",
+    "MICROSOFT_SENTINEL_INTEL_WORKSPACE_ID",
+    "MICROSOFT_SENTINEL_INTEL_WORKSPACE_NAME",
+    "MICROSOFT_SENTINEL_INTEL_SUBSCRIPTION_ID"
+  ],
+  "additionalProperties": true
+}

stream/microsoft-sentinel-intel/docker-compose.yml

@@ -26,4 +26,4 @@ services:
 #      - MICROSOFT_SENTINEL_INTEL_EXTRA_LABELS=label1,label2,...
 #      - MICROSOFT_SENTINEL_INTEL_WORKSPACE_API_VERSION=2024-02-01-preview
 #      - MICROSOFT_SENTINEL_INTEL_MANAGEMENT_API_VERSION=2025-03-01
-    restart: unless-stopped
+    restart: unless-stopped

stream/microsoft-sentinel-intel/src/base_connector/config.py

@@ -21,27 +21,7 @@
     YamlConfigSettingsSource,
 )
 
-"""
-All the variables that have default values will override configuration from the OpenCTI helper.
-
-All the variables of this classes are customizable through:
-    - config.yml 
-    - .env
-    - environment variables.
-
-If a variable is set in 2 different places, the first one will be used in this order:
-    1. YAML file
-    2. .env file
-    3. Environment variables
-    4. Default value
-    
-WARNING:
-    The Environment variables in the .env or global environment must be set in the following format:
-    OPENCTI_<variable>
-    CONNECTOR_<variable>
-    
-    the split is made on the first occurrence of the "_" character.
-"""
+'\nAll the variables that have default values will override configuration from the OpenCTI helper.\n\nAll the variables of this classes are customizable through:\n    - config.yml \n    - .env\n    - environment variables.\n\nIf a variable is set in 2 different places, the first one will be used in this order:\n    1. YAML file\n    2. .env file\n    3. Environment variables\n    4. Default value\n    \nWARNING:\n    The Environment variables in the .env or global environment must be set in the following format:\n    OPENCTI_<variable>\n    CONNECTOR_<variable>\n    \n    the split is made on the first occurrence of the "_" character.\n'
 
 
 def environ_list_validator(value: str | list[str]) -> list[str]:
@@ -51,13 +31,13 @@ def environ_list_validator(value: str | list[str]) -> list[str]:
 
 
 def pycti_list_serializer(v: list[str], info: SerializationInfo) -> str | list[str]:
-    if isinstance(v, list) and info.context and info.context.get("mode") == "pycti":
-        return ",".join(v)  # [ "e1", "e2", "e3" ] -> "e1,e2,e3"
+    if isinstance(v, list) and info.context and (info.context.get("mode") == "pycti"):
+        return ",".join(v)
     return v
 
 
 ListFromString = Annotated[
-    list[str],  # Final type
+    list[str],
     BeforeValidator(environ_list_validator),
     PlainSerializer(pycti_list_serializer, when_used="json"),
 ]
@@ -76,21 +56,17 @@ class StreamConnectorConfig(BaseModel):
     type: Literal["STREAM"] = Field(default="STREAM")
     scope: ListFromString
     log_level: LogLevelType
-
     live_stream_id: str
     live_stream_listen_delete: bool = Field(default=True)
     live_stream_no_dependencies: bool = Field(default=True)
     live_stream_with_inferences: bool = Field(default=False)
     live_stream_recover_iso_date: str | None = Field(default=None)
     live_stream_start_timestamp: str | None = Field(default=None)
-
     listen_protocol: str = Field(default="AMQP")
     listen_protocol_uri: str = Field(default="http://127.0.0.1:7070")
     listen_protocol_path: str = Field(default="/api/callback")
     listen_protocol_ssl: bool = Field(default=False)
     listen_protocol_port: int = Field(default=7070)
-
-    # TODO: See what is specific to stream / external
     auto: bool = Field(default=False)
     expose_metrics: bool = Field(default=False)
     metrics_port: int = Field(default=9095)
@@ -99,7 +75,6 @@ class StreamConnectorConfig(BaseModel):
     validate_before_import: bool = Field(default=False)
     queue_protocol: str = Field(default="amqp")
     queue_threshold: int = Field(default=500)
-
     send_to_queue: bool = Field(default=True)
     send_to_directory: bool = Field(default=False)
     send_to_directory_path: str | None = Field(default=None)
@@ -128,8 +103,6 @@ def validate_live_stream_id(cls, v: str) -> str:
 class BaseConnectorSettings(abc.ABC, BaseSettings):
     opencti: _OpenCTIConfig
     connector: StreamConnectorConfig
-
-    # files needs to be at the same level as the module
     model_config = SettingsConfigDict(
         env_nested_delimiter="_", env_nested_max_split=1, enable_decoding=False
     )
@@ -160,9 +133,9 @@ def settings_customise_sources(
             3. Environment variables
             4. Default values
         """
-        if Path(settings_cls.model_config["yaml_file"] or "").is_file():  # type: ignore
+        if Path(settings_cls.model_config["yaml_file"] or "").is_file():
             return (YamlConfigSettingsSource(settings_cls),)
-        if Path(settings_cls.model_config["env_file"] or "").is_file():  # type: ignore
+        if Path(settings_cls.model_config["env_file"] or "").is_file():
             return (dotenv_settings,)
         return (env_settings,)
 

stream/microsoft-sentinel-intel/src/main.py

@@ -1,12 +1,10 @@
 import traceback
 
-from microsoft_sentinel_intel import Connector
+from microsoft_sentinel_intel import Connector, ConnectorSettings
 from microsoft_sentinel_intel.client import ConnectorClient
-from microsoft_sentinel_intel.config import ConnectorSettings
 from pycti import OpenCTIConnectorHelper
 
-
-def main() -> None:
+if __name__ == "__main__":
     """
     Entry point of the script
 
@@ -16,17 +14,13 @@ def main() -> None:
     - exit(1): effective way to terminate a Python program when an error is encountered.
     It signals to the operating system and any calling processes that the program did not complete successfully.
     """
-    config = ConnectorSettings()
-    helper = OpenCTIConnectorHelper(config=config.model_dump_pycti())
-    client = ConnectorClient(helper=helper, config=config)
-
-    connector = Connector(helper=helper, config=config, client=client)
-    connector.run()
-
-
-if __name__ == "__main__":
     try:
-        main()
+        settings = ConnectorSettings()
+        helper = OpenCTIConnectorHelper(config=settings.to_helper_config())
+        client = ConnectorClient(helper=helper, config=settings)
+
+        connector = Connector(config=settings, helper=helper, client=client)
+        connector.run()
     except Exception:
         traceback.print_exc()
         exit(1)

stream/microsoft-sentinel-intel/src/microsoft_sentinel_intel/__init__.py

@@ -1,9 +1,5 @@
 from microsoft_sentinel_intel.client import ConnectorClient
-from microsoft_sentinel_intel.config import ConnectorSettings
 from microsoft_sentinel_intel.connector import Connector
+from microsoft_sentinel_intel.settings import ConnectorSettings
 
-__all__ = [
-    "Connector",
-    "ConnectorClient",
-    "ConnectorSettings",
-]
+__all__ = ["Connector", "ConnectorClient", "ConnectorSettings"]

stream/microsoft-sentinel-intel/src/microsoft_sentinel_intel/connector.py

@@ -4,16 +4,14 @@
 
 from filigran_sseclient.sseclient import Event
 from microsoft_sentinel_intel.client import ConnectorClient
-from microsoft_sentinel_intel.config import ConnectorSettings
-from microsoft_sentinel_intel.errors import (
-    ConnectorError,
-    ConnectorWarning,
-)
+from microsoft_sentinel_intel.errors import ConnectorError, ConnectorWarning
+from microsoft_sentinel_intel.settings import ConnectorSettings
 from microsoft_sentinel_intel.utils import is_stix_indicator
 from pycti import OpenCTIConnectorHelper
 
 
 class Connector:
+
     def __init__(
         self,
         helper: OpenCTIConnectorHelper,
@@ -27,12 +25,10 @@ def __init__(
     def _prepare_stix_object(self, stix_object: dict) -> dict:
         if self.config.microsoft_sentinel_intel.delete_extensions:
             del stix_object["extensions"]
-
         if extra_labels := self.config.microsoft_sentinel_intel.extra_labels:
             stix_object["labels"] = list(
                 set(stix_object.get("labels", []) + extra_labels)
             )
-
         return stix_object
 
     def _process_event(self, event_type: str, stix_object: dict) -> None:
@@ -63,14 +59,12 @@ def _handle_event(self, event: Event):
                 message="[ERROR] Data cannot be parsed to JSON",
                 metadata={"message_data": event.data, "error": str(err)},
             ) from err
-
         if is_stix_indicator(data):
             self.helper.connector_logger.info(
                 message=f"[{event.event.upper()}] Processing message",
                 meta={"data": data, "event": event.event},
             )
             self._process_event(event_type=event.event, stix_object=data)
-
             self.helper.connector_logger.info(
                 message=f"[{event.event.upper()}] Indicator processed",
                 meta={"opencti_id": data["id"]},
@@ -96,15 +90,11 @@ def process_message(self, message: Event) -> None:
         except ConnectorWarning as err:
             self.helper.connector_logger.warning(message=err.message)
         except ConnectorError as err:
-            self.helper.connector_logger.error(
-                message=err.message,
-                meta=err.metadata,
-            )
+            self.helper.connector_logger.error(message=err.message, meta=err.metadata)
         except Exception as err:
             traceback.print_exc()
             self.helper.connector_logger.error(
-                message=f"Unexpected error: {err}",
-                meta={"error": str(err)},
+                message=f"Unexpected error: {err}", meta={"error": str(err)}
             )
 
     def run(self) -> None: