[IDEV-1989_1990] feat(FeedDomainTools_v1.0.1): Add domaindiscovery and domainrdap feeds

Packs/FeedDomainTools/.secrets-ignore

@@ -0,0 +1 @@
+[email protected]

Packs/FeedDomainTools/Integrations/FeedDomainTools/FeedDomainTools.py

@@ -21,6 +21,8 @@ class DomainToolsClient(BaseClient):
 
     NOD_FEED = "nod"
     NAD_FEED = "nad"
+    DOMAINRDAP = "domainrdap"
+    DOMAINDISCOVERY = "domaindiscovery"
 
     FEED_URL = "/v1/feed"
     DOMAINTOOLS_API_BASE_URL = "https://api.domaintools.com"
@@ -153,7 +155,11 @@ def build_iterator(
 
                 timestamp = json_feed.get("timestamp", "")
                 indicator = json_feed.get("domain")
-                indicator_type = auto_detect_indicator_type(indicator)
+                indicator_type = "Domain"
+                # indicator_type = auto_detect_indicator_type(indicator)
+
+                # for `domainrdap` feed, we have more data to display including the parsed data.
+                parsed_record = json_feed.get("parsed_record", {})
 
                 if indicator and indicator_type:
                     yield {
@@ -162,6 +168,7 @@ def build_iterator(
                         "timestamp": timestamp,
                         "tags": ["DomainToolsFeeds", self.feed_type] + ud_tags,
                         "tlp_color": self.tlp_color,
+                        "parsed_record": parsed_record
                     }
 
                     limit_counter += 1
@@ -207,6 +214,7 @@ def fetch_indicators(
             timestamp_ = item.get("timestamp")
             tags_ = item.get("tags", [])
             tlp_color_ = item.get("tlp_color")
+            parsed_record_ = item.get("parsed_record")
 
             indicator_tags = ",".join(tags_).rstrip(",")
 
@@ -216,6 +224,9 @@ def fetch_indicators(
                 "timestamp": timestamp_,
             }
 
+            if parsed_record_:
+                raw_data["parsed_record"] = parsed_record_
+
             # Create indicator object for each value.
             indicator_obj = {
                 "value": value_,
@@ -302,22 +313,25 @@ def fetch_indicators_command(client: DomainToolsClient, params: dict[str, Any] =
 
     feed_type_ = params.get("feed_type", "ALL")
 
-    FEEDS_TO_PROCESS = {
-        client.NOD_FEED: {"top": top, "after": after, "session_id": session_id},
-        client.NAD_FEED: {"top": top, "after": after, "session_id": session_id},
-    }
+    FEEDS_TO_PROCESS = [
+        client.NOD_FEED,
+        client.NAD_FEED,
+        client.DOMAINRDAP,
+        client.DOMAINDISCOVERY
+    ]
+
+    dt_feed_kwargs = {"top": top, "after": after, "session_id": session_id}
 
     fetched_indicators = []
 
-    for feed_type, dt_feed_kwargs in FEEDS_TO_PROCESS.items():
+    for feed_type in FEEDS_TO_PROCESS:
         indicators = []
         if feed_type_ == "ALL":
             indicators = fetch_indicators(client, feed_type=feed_type, dt_feed_kwargs=dt_feed_kwargs)
-        if feed_type_ == feed_type.upper():
+        if feed_type_.upper() == feed_type.upper():
             indicators = fetch_indicators(client, feed_type=feed_type, dt_feed_kwargs=dt_feed_kwargs)
 
         fetched_indicators.extend(indicators)
-
     return fetched_indicators
 
 
@@ -334,6 +348,7 @@ def test_module(client: DomainToolsClient, args: dict[str, str], params: dict[st
     }
     try:
         next(client.build_iterator(dt_feed_kwargs=dt_feed_kwargs))
+        # fetch_indicators_command(client, params)
     except Exception as e:
         raise Exception(
             "Could not fetch DomainTools Feed\n"

Packs/FeedDomainTools/Integrations/FeedDomainTools/FeedDomainTools.yml

@@ -42,8 +42,10 @@ configuration:
   type: 15
   options:
   - ALL
-  - NOD
-  - NAD
+  - nod
+  - nad
+  - domainrdap
+  - domaindiscovery
   additionalinfo: The DomainTools feed type fo fetch. Defaults to 'ALL'.
   section: Collect
 - display: Fetch indicators
@@ -145,7 +147,14 @@ script:
     execution: false
     arguments:
     - name: feed_type
-      defaultValue: 'nod'
+      type: String
+      auto: PREDEFINED
+      predefined:
+      - "nod"
+      - "nad"
+      - "domainrdap"
+      - "domaindiscovery"
+      defaultValue: "nod"
       description: The DomainTools integration feed type to fetch.
       isArray: false
       default: false

Packs/FeedDomainTools/Integrations/FeedDomainTools/FeedDomainTools_test.py

@@ -78,7 +78,9 @@ def test_build_iterator_with_limit(self, mocker, dt_feeds_client):
             return_value=feed_mock_response.NOD_FEED_RESPONSE,
         )
 
-        indicators = list(dt_feeds_client.build_iterator(feed_type="nod", dt_feed_kwargs={"top": 5}))
+        indicators = list(
+            dt_feeds_client.build_iterator(feed_type="nod", dt_feed_kwargs={"top": 5})
+        )
         [indicator.get("value") for indicator in indicators]
 
         assert len(indicators) == 5
@@ -117,6 +119,8 @@ def test_conversion_feed_to_indicato_obj(mocker, dt_feeds_client):
     [
         "nod",
         "nad",
+        "domaindiscovery",
+        "domainrdap"
     ],
 )
 def test_get_indicators_command(mocker, dt_feeds_client, feed_type):
@@ -133,6 +137,8 @@ def test_get_indicators_command(mocker, dt_feeds_client, feed_type):
     mock_feed_response = {
         "nod": feed_mock_response.NOD_FEED_RESPONSE,
         "nad": feed_mock_response.NAD_FEED_RESPONSE,
+        "domaindiscovery": feed_mock_response.DOMAINDISCOVERY_RESPONSE,
+        "domainrdap": feed_mock_response.DOMAINRDAP_RESPONSE,
     }
 
     mocker.patch.object(
@@ -147,6 +153,8 @@ def test_get_indicators_command(mocker, dt_feeds_client, feed_type):
     expected_indicator_results = {
         "nod": feed_mock_response.NOD_PARSED_INDICATOR_RESPONSE,
         "nad": feed_mock_response.NAD_PARSED_INDICATOR_RESPONSE,
+        "domaindiscovery": feed_mock_response.DOMAINDISCOVERY_PARSED_INDICATOR_RESPONSE,
+        "domainrdap": feed_mock_response.DOMAINRDAP_PARSED_INDICATOR_RESPONSE
     }
 
     human_readable = tableToMarkdown(
@@ -155,6 +163,7 @@ def test_get_indicators_command(mocker, dt_feeds_client, feed_type):
         headers=["value", "type", "fields", "rawJSON"],
         removeNull=True,
     )
+
     assert results.readable_output == human_readable
 
 
@@ -168,15 +177,20 @@ def test_fetch_indicators_command(mocker, dt_feeds_client):
         - Create indicator objects list
 
     """
+
+    mock_return_value = (
+        feed_mock_response.NAD_FEED_RESPONSE
+        + feed_mock_response.NOD_FEED_RESPONSE
+        + feed_mock_response.DOMAINDISCOVERY_RESPONSE
+    )
     mocker.patch.object(
         dt_feeds_client,
         "_get_dt_feeds",
-        return_value=feed_mock_response.NAD_FEED_RESPONSE
-        + feed_mock_response.NOD_FEED_RESPONSE,
+        return_value=mock_return_value,
     )
-    results = fetch_indicators_command(dt_feeds_client, params={"top": "20"})
+    results = fetch_indicators_command(dt_feeds_client, params={"top": "2"})
 
-    assert len(results) == 40
+    assert len(results) == 8
 
 
 def test_calling_command_using_main(mocker, dt_feeds_client):
@@ -193,7 +207,9 @@ def test_calling_command_using_main(mocker, dt_feeds_client):
     mocker.patch.object(
         demisto,
         "params",
-        return_value={"credentials": {"identifier": "test_username", "password": "test_key"}},
+        return_value={
+            "credentials": {"identifier": "test_username", "password": "test_key"}
+        },
     )
     mocker.patch(
         "FeedDomainTools.DomainToolsClient._get_dt_feeds",