IDEV-2021: Adjust implementation for feeds products iterative response

.pre-commit-config.yaml

@@ -1,11 +1,11 @@
 repos:
 -   repo: https://github.com/phantomcyber/dev-cicd-tools
-    rev: v1.16
+    rev: v1.23
     hooks:
     -   id: org-hook
     -   id: package-app-dependencies
 -   repo: https://github.com/Yelp/detect-secrets
-    rev: v1.4.0
+    rev: v1.5.0
     hooks:
     -   id: detect-secrets
         args: ['--no-verify', '--exclude-files', '^domaintools_iris.json$']

domaintools_iris.json

@@ -12,7 +12,7 @@
     "product_vendor": "DomainTools",
     "product_name": "DomainTools Iris Investigate",
     "product_version_regex": ".*",
-    "min_phantom_version": "6.3.0",
+    "min_phantom_version": "6.3.1",
     "python_version": "3",
     "logo": "logo_domaintools_iris.svg",
     "logo_dark": "logo_domaintools_iris_dark.svg",
@@ -2115,20 +2115,25 @@
                     "data_type": "string",
                     "order": 0
                 },
-                "after": {
-                    "description": "A negative integer (in seconds) representing the start of the time window, relative to the current time in seconds, for which data will be provided.",
+                "before": {
+                    "description": "The end of the query window in seconds or in ISO8601 format, relative to the current time, inclusive.",
                     "data_type": "string",
                     "order": 1
                 },
+                "after": {
+                    "description": "The start of the query window in seconds in ISO8601 format, relative to the current time, inclusive.",
+                    "data_type": "string",
+                    "order": 2
+                },
                 "session_id": {
                     "description": "Serves as a unique identifier for the session. This parameter ensures that data retrieval begins from the latest timestamp recorded in the previous data pull.",
                     "data_type": "string",
-                    "order": 2
+                    "order": 3
                 },
                 "top": {
                     "description": "The number of results to return in the response payload. Primarily used for testing.",
                     "data_type": "string",
-                    "order": 3
+                    "order": 4
                 }
             },
             "render": {
@@ -2177,6 +2182,10 @@
                     "data_path": "action_result.parameter.after",
                     "data_type": "string"
                 },
+                {
+                    "data_path": "action_result.parameter.before",
+                    "data_type": "string"
+                },
                 {
                     "data_path": "action_result.parameter.domain",
                     "data_type": "string"
@@ -2218,20 +2227,25 @@
                     "data_type": "string",
                     "order": 0
                 },
-                "after": {
-                    "description": "A negative integer (in seconds) representing the start of the time window, relative to the current time in seconds, for which data will be provided.",
+                "before": {
+                    "description": "The end of the query window in seconds or in ISO8601 format, relative to the current time, inclusive.",
                     "data_type": "string",
                     "order": 1
                 },
+                "after": {
+                    "description": "The start of the query window in seconds in ISO8601 format, relative to the current time, inclusive.",
+                    "data_type": "string",
+                    "order": 2
+                },
                 "session_id": {
                     "description": "Serves as a unique identifier for the session. This parameter ensures that data retrieval begins from the latest timestamp recorded in the previous data pull.",
                     "data_type": "string",
-                    "order": 2
+                    "order": 3
                 },
                 "top": {
                     "description": "The number of results to return in the response payload. Primarily used for testing.",
                     "data_type": "string",
-                    "order": 3
+                    "order": 4
                 }
             },
             "render": {
@@ -2280,6 +2294,10 @@
                     "data_path": "action_result.parameter.after",
                     "data_type": "string"
                 },
+                {
+                    "data_path": "action_result.parameter.before",
+                    "data_type": "string"
+                },
                 {
                     "data_path": "action_result.parameter.domain",
                     "data_type": "string"
@@ -2314,43 +2332,47 @@
         "wheel": [
             {
                 "module": "anyio",
-                "input_file": "wheels/py3/anyio-3.6.1-py3-none-any.whl"
+                "input_file": "wheels/py3/anyio-4.8.0-py3-none-any.whl"
             },
             {
                 "module": "certifi",
-                "input_file": "wheels/py3/certifi-2022.6.15-py3-none-any.whl"
+                "input_file": "wheels/py3/certifi-2025.1.31-py3-none-any.whl"
             },
             {
                 "module": "charset_normalizer",
-                "input_file": "wheels/py3/charset_normalizer-2.0.12-py3-none-any.whl"
+                "input_file": "wheels/py3/charset_normalizer-3.4.1-py3-none-any.whl"
             },
             {
                 "module": "click",
-                "input_file": "wheels/py3/click-8.1.7-py3-none-any.whl"
+                "input_file": "wheels/py3/click-8.1.8-py3-none-any.whl"
+            },
+            {
+                "module": "exceptiongroup",
+                "input_file": "wheels/py3/exceptiongroup-1.2.2-py3-none-any.whl"
             },
             {
                 "module": "domaintools_api",
-                "input_file": "wheels/shared/domaintools_api-2.1.0-py2.py3-none-any.whl"
+                "input_file": "wheels/shared/domaintools_api-2.3.0-py2.py3-none-any.whl"
             },
             {
                 "module": "filelock",
-                "input_file": "wheels/py3/filelock-3.7.1-py3-none-any.whl"
+                "input_file": "wheels/py3/filelock-3.18.0-py3-none-any.whl"
             },
             {
                 "module": "h11",
-                "input_file": "wheels/py3/h11-0.12.0-py3-none-any.whl"
+                "input_file": "wheels/py3/h11-0.14.0-py3-none-any.whl"
             },
             {
                 "module": "httpcore",
-                "input_file": "wheels/py3/httpcore-0.15.0-py3-none-any.whl"
+                "input_file": "wheels/py3/httpcore-1.0.7-py3-none-any.whl"
             },
             {
                 "module": "httpx",
-                "input_file": "wheels/py3/httpx-0.23.0-py3-none-any.whl"
+                "input_file": "wheels/py3/httpx-0.28.1-py3-none-any.whl"
             },
             {
                 "module": "idna",
-                "input_file": "wheels/py3/idna-3.3-py3-none-any.whl"
+                "input_file": "wheels/py3/idna-3.10-py3-none-any.whl"
             },
             {
                 "module": "markdown_it_py",
@@ -2362,15 +2384,15 @@
             },
             {
                 "module": "pygments",
-                "input_file": "wheels/py3/pygments-2.18.0-py3-none-any.whl"
+                "input_file": "wheels/py3/pygments-2.19.1-py3-none-any.whl"
             },
             {
                 "module": "requests",
-                "input_file": "wheels/py3/requests-2.28.0-py3-none-any.whl"
+                "input_file": "wheels/py3/requests-2.32.3-py3-none-any.whl"
             },
             {
                 "module": "requests_file",
-                "input_file": "wheels/shared/requests_file-1.5.1-py2.py3-none-any.whl"
+                "input_file": "wheels/shared/requests_file-2.1.0-py2.py3-none-any.whl"
             },
             {
                 "module": "rfc3986",
@@ -2390,23 +2412,23 @@
             },
             {
                 "module": "sniffio",
-                "input_file": "wheels/py3/sniffio-1.2.0-py3-none-any.whl"
+                "input_file": "wheels/py3/sniffio-1.3.1-py3-none-any.whl"
             },
             {
                 "module": "tldextract",
-                "input_file": "wheels/py3/tldextract-3.4.4-py3-none-any.whl"
+                "input_file": "wheels/py3/tldextract-5.1.3-py3-none-any.whl"
             },
             {
                 "module": "typer",
-                "input_file": "wheels/py3/typer-0.13.0-py3-none-any.whl"
+                "input_file": "wheels/py3/typer-0.15.2-py3-none-any.whl"
             },
             {
                 "module": "typing_extensions",
                 "input_file": "wheels/py3/typing_extensions-4.12.2-py3-none-any.whl"
             },
             {
                 "module": "urllib3",
-                "input_file": "wheels/shared/urllib3-1.26.9-py2.py3-none-any.whl"
+                "input_file": "wheels/py3/urllib3-2.3.0-py3-none-any.whl"
             }
         ]
     },

domaintools_iris_connector.py

@@ -106,19 +106,27 @@ def _clean_empty_response(self, response):
         if response.get("domains") == []:
             del response["domains"]
 
-    def _parse_feeds_response(self, action_result, response_json):
-        rows = response_json.strip().split("\n")
-        data = []
-        for row in rows:
-            feed_result = json.loads(row)
-            data.append(
-                {
-                    "timestamp": feed_result.get("timestamp"),
-                    "domain": feed_result.get("domain"),
-                }
-            )
+    def _parse_feeds_response(self, service, action_result, feeds_results):
+        try:
+            for response in feeds_results.response():
+                data = []
+                rows = response.strip().split("\n")
+
+                for row in rows:
+                    if service in ("nod", "nad"):
+                        feed_result = json.loads(row)
+                        data.append(
+                            {
+                                "timestamp": feed_result.get("timestamp"),
+                                "domain": feed_result.get("domain"),
+                            }
+                        )
+
+                action_result.update_data(data)
+        except Exception as error:
+            action_result.add_data({})
+            return action_result.set_status(phantom.APP_ERROR, str(error))
 
-        action_result.update_data(data)
         return action_result.set_status(phantom.APP_SUCCESS)
 
     def _parse_response(self, action_result, response_json):
@@ -235,11 +243,11 @@ def _do_query(self, service, action_result, query_args=None):
                     response = service_api(**query_args, position=position)
 
                 try:
-                    response_json = response.data()
-
                     if self._is_feeds_service(service):
                         # Separate parsing of feeds product
-                        return self._parse_feeds_response(action_result, response_json)
+                        return self._parse_feeds_response(service, action_result, response)
+
+                    response_json = response.data()
 
                 except Exception as e:
                     return action_result.set_status(
@@ -871,9 +879,12 @@ def _nod_feed(self, param):
         if session_id:
             params["sessionID"] = session_id
 
-        self._do_query("nod", action_result, query_args=params)
+        ret_val = self._do_query("nod", action_result, query_args=params)
         self.save_progress("Completed nod_feed action.")
 
+        if not ret_val:
+            return action_result.get_data()
+
         return action_result.get_status()
 
     def _nad_feed(self, param):
@@ -885,8 +896,11 @@ def _nad_feed(self, param):
         if session_id:
             params["sessionID"] = session_id
 
-        self._do_query("nad", action_result, query_args=params)
-        self.save_progress("Completed nod_feed action.")
+        ret_val = self._do_query("nad", action_result, query_args=params)
+        self.save_progress("Completed nad_feed action.")
+
+        if not ret_val:
+            return action_result.get_data()
 
         return action_result.get_status()
 

exclude_files.txt

@@ -0,0 +1,12 @@
+.git*
+*.pyc
+.idea
+bin
+lib
+pyvenv.cfg
+build.sh
+venv
+splunk-soar-dev/
+domaintoolsiris.tgz
+domaintools_connector_old.py
+domaintools_old.json