Enhancement: Added domainrisk and domainhotlist feeds

packages/ti_domaintools/_dev/build/docs/README.md

@@ -8,6 +8,8 @@ Summary of Available Feeds:
 - `Newly Observed Domains (NOD)`: Apex-level domains (e.g. example.com but not <www.example.com>) that we observe for the first time, and have not observed previously with our global DNS sensor network.
 - `Domain Discovery`: New domains as they are either discovered in domain registration information, observed by our global sensor network, or reported by trusted third parties.
 - `Domain RDAP`: Changes to global domain registration information, populated by the Registration Data Access Protocol (RDAP). Compliments the 5-Minute WHOIS Feed as registries and registrars switch from Whois to RDAP.
+- `Domain Risk`: Real-time updates to Domain Risk Scores for apex domains, regardless of observed traffic.
+- `Domain Hotlist`: Domains with high Domain Risk Scores that have also been active within 24 hours.
 
 With over 300,000 new domains observed daily, the feed empowers security teams to identify and block potentially malicious domains before they can be weaponized.
 Ideal for threat hunting, phishing prevention, and brand protection.
@@ -25,6 +27,8 @@ Log data streams collected by the DomainTools integration include the following
 - `Newly Active Domains (NAD)`
 - `Domain Discovery`
 - `Domain RDAP`
+- `Domain Risk`
+- `Domain Hotlist`
 
 ## Requirements
 
@@ -84,3 +88,25 @@ This data is collected via the [DomainTools Feeds API](https://docs.domaintools.
 {{event "domainrdap_feed"}}
 
 {{fields "domainrdap_feed"}}
+
+### Domain Risk Feed
+
+The `domainrisk_feed` data stream provides events from [DomainTools Domain Risk](https://www.domaintools.com/products/threat-intelligence-feeds/).
+This data is collected via the [DomainTools Feeds API](https://docs.domaintools.com/feeds/realtime/).
+
+#### Example
+
+{{event "domainrisk_feed"}}
+
+{{fields "domainrisk_feed"}}
+
+### Domain Hotlist Feed
+
+The `domainhotlist_feed` data stream provides events from [DomainTools Domain Hotlist](https://www.domaintools.com/products/threat-intelligence-feeds/).
+This data is collected via the [DomainTools Feeds API](https://docs.domaintools.com/feeds/realtime/).
+
+#### Example
+
+{{event "domainhotlist_feed"}}
+
+{{fields "domainhotlist_feed"}}

packages/ti_domaintools/_dev/deploy/docker/files/config.yml

@@ -26,3 +26,17 @@ rules:
       - status_code: 200
         body: |-
           {"timestamp":"2025-08-20T16:44:02Z","domain":"1xbet-ieon.lol","raw_record":{"first_request_timestamp":"2025-08-20T16:43:57Z","requests":[{"data":"{\"rdapConformance\":[\"icann_rdap_response_profile_1\",\"icann_rdap_response_profile_0\",\"icann_rdap_technical_implementation_guide_1\",\"icann_rdap_technical_implementation_guide_0\",\"rdap_level_0\"],\"lang\":\"en\",\"objectClassName\":\"domain\",\"handle\":\"D583238142-CNIC\",\"ldhName\":\"1xbet-ieon.lol\",\"nameservers\":[{\"objectClassName\":\"nameserver\",\"ldhName\":\"dns1.registrar-servers.com\",\"handle\":\"H46040-CNIC\",\"links\":[{\"title\":\"Authoritative URL for this resource\",\"rel\":\"self\",\"type\":\"application\\/rdap+json\",\"value\":\"https:\\/\\/rdap.centralnic.com\\/lol\\/domain\\/1xbet-ieon.lol\",\"href\":\"https:\\/\\/rdap.centralnic.com\\/lol\\/nameserver\\/dns1.registrar-servers.com\"}]},{\"objectClassName\":\"nameserver\",\"ldhName\":\"dns2.registrar-servers.com\",\"handle\":\"H46041-CNIC\",\"links\":[{\"title\":\"Authoritative URL for this resource\",\"rel\":\"self\",\"type\":\"application\\/rdap+json\",\"value\":\"https:\\/\\/rdap.centralnic.com\\/lol\\/domain\\/1xbet-ieon.lol\",\"href\":\"https:\\/\\/rdap.centralnic.com\\/lol\\/nameserver\\/dns2.registrar-servers.com\"}]}],\"secureDNS\":{\"delegationSigned\":false},\"entities\":[{\"objectClassName\":\"entity\",\"handle\":\"1068\",\"roles\":[\"registrar\"],\"vcardArray\":[\"vcard\",[[\"version\",[],\"text\",\"4.0\"],[\"fn\",[],\"text\",\"Namecheap\"]]],\"links\":[{\"title\":\"Authoritative URL for this resource\",\"rel\":\"self\",\"type\":\"application\\/rdap+json\",\"value\":\"https:\\/\\/rdap.centralnic.com\\/lol\\/domain\\/1xbet-ieon.lol\",\"href\":\"https:\\/\\/rdap.centralnic.com\\/lol\\/entity\\/1068\"},{\"title\":\"Registrar's Website\",\"rel\":\"about\",\"value\":\"https:\\/\\/rdap.namecheap.com\\/\",\"href\":\"https:\\/\\/namecheap.com\"}],\"entities\":[{\"objectClassName\":\"entity\",\"handle\":\"not applicable\",\"roles\":[\"abuse\"],\"vcardArray\":[\"vcard\",[[\"version\",[],\"text\",\"4.0\"],[\"fn\",[],\"text\",\"Abuse Contact\"],[\"org\",[],\"text\",\"Namecheap\"],[\"email\",[],\"text\",\"[email protected]\"],[\"tel\",{\"type\":\"voice\"},\"uri\",\"tel:+1.9854014545\"]]]}],\"publicIds\":[{\"type\":\"IANA Registrar ID\",\"identifier\":\"1068\"}]}],\"status\":[\"server transfer prohibited\",\"client transfer prohibited\",\"add period\"],\"port43\":\"whois.nic.lol\",\"events\":[{\"eventAction\":\"registration\",\"eventDate\":\"2025-08-19T14:50:37.0Z\"},{\"eventAction\":\"expiration\",\"eventDate\":\"2026-08-19T23:59:59.0Z\"},{\"eventAction\":\"last update of RDAP database\",\"eventDate\":\"2025-08-20T16:43:58.0Z\"},{\"eventAction\":\"last changed\",\"eventDate\":\"2025-08-19T14:50:42.0Z\"}],\"notices\":[{\"title\":\"Status Codes\",\"description\":[\"For more information on domain status codes, please visit https:\\/\\/icann.org\\/epp\"],\"links\":[{\"title\":\"More information on domain status codes\",\"rel\":\"glossary\",\"value\":\"https:\\/\\/rdap.centralnic.com\\/lol\\/domain\\/1xbet-ieon.lol\",\"href\":\"https:\\/\\/icann.org\\/epp\"}]},{\"title\":\"Terms of Use\",\"description\":[\"For more information on Whois status codes, please visit https:\\/\\/icann.org\\/epp\",\"\",\"\u003e\u003e\u003e IMPORTANT INFORMATION ABOUT THE DEPLOYMENT OF RDAP: please visit\",\"https:\\/\\/www.centralnicregistry.com\\/support\\/information\\/rdap \u003c\u003c\u003c\",\"\",\"The registration data available in this service is limited. Additional\",\"data may be available at https:\\/\\/lookup.icann.org\",\"\",\"The Whois and RDAP services are provided by CentralNic, and contain\",\"information pertaining to Internet domain names registered by our\",\"our customers. By using this service you are agreeing (1) not to use any\",\"information presented here for any purpose other than determining\",\"ownership of domain names, (2) not to store or reproduce this data in\",\"any way, (3) not to use any high-volume, automated, electronic processes\",\"to obtain data from this service. Abuse of this service is monitored and\",\"actions in contravention of these terms will result in being permanently\",\"blacklisted. All data is (c) CentralNic Ltd (https:\\/\\/www.centralnicregistry.com)\",\"\",\"Access to the Whois and RDAP services is rate limited. For more\",\"information, visit https:\\/\\/registrar-console.centralnicregistry.com\\/pub\\/whois_guidance.\"],\"links\":[{\"title\":\"Terms of Use\",\"rel\":\"terms-of-service\",\"value\":\"https:\\/\\/rdap.centralnic.com\\/lol\\/domain\\/1xbet-ieon.lol\",\"href\":\"https:\\/\\/www.centralnicregistry.com\\/\"}]},{\"title\":\"RDDS Inaccuracy Complaint Form\",\"description\":[\"URL of the ICANN RDDS Inaccuracy Complaint Form: https:\\/\\/icann.org\\/wicf\"],\"links\":[{\"title\":\"ICANN RDDS Inaccuracy Complaint Form\",\"rel\":\"help\",\"value\":\"https:\\/\\/rdap.centralnic.com\\/lol\\/domain\\/1xbet-ieon.lol\",\"href\":\"https:\\/\\/icann.org\\/wicf\"}]}],\"links\":[{\"title\":\"Authoritative URL for this resource\",\"rel\":\"self\",\"type\":\"application\\/rdap+json\",\"value\":\"https:\\/\\/rdap.centralnic.com\\/lol\\/domain\\/1xbet-ieon.lol\",\"href\":\"https:\\/\\/rdap.centralnic.com\\/lol\\/domain\\/1xbet-ieon.lol\"},{\"title\":\"RDAP Service Help\",\"rel\":\"help\",\"type\":\"text\\/html\",\"value\":\"https:\\/\\/rdap.centralnic.com\\/lol\\/domain\\/1xbet-ieon.lol\",\"href\":\"https:\\/\\/whois.nic.lol\\/rdap\"},{\"title\":\"XYZ.com, LLC\",\"rel\":\"related\",\"type\":\"text\\/html\",\"value\":\"https:\\/\\/rdap.centralnic.com\\/lol\\/domain\\/1xbet-ieon.lol\",\"href\":\"https:\\/\\/gen.xyz\\/\"},{\"title\":\"URL of Sponsoring Registrar's RDAP Record\",\"rel\":\"related\",\"type\":\"application\\/rdap+json\",\"value\":\"https:\\/\\/rdap.centralnic.com\\/lol\\/domain\\/1xbet-ieon.lol\",\"href\":\"https:\\/\\/rdap.namecheap.com\\/domain\\/1xbet-ieon.lol\"}]}","source_type":"registrar","timestamp":"2025-08-20T16:43:59Z","url":"https://rdap.centralnic.com/lol/domain/1xbet-ieon.lol"}]},"parsed_record":{"parsed_fields":{"conformance":["icann_rdap_response_profile_1","icann_rdap_response_profile_0","icann_rdap_technical_implementation_guide_1","icann_rdap_technical_implementation_guide_0","rdap_level_0"],"contacts":[],"creation_date":"2025-08-19T14:50:37+00:00","dnssec":{"signed":false},"domain":"1xbet-ieon.lol","domain_statuses":["server transfer prohibited","client transfer prohibited","add period"],"email_domains":["namecheap.com"],"emails":["[email protected]"],"expiration_date":"2026-08-19T23:59:59+00:00","handle":"D583238142-CNIC","last_changed_date":"2025-08-19T14:50:42+00:00","links":[{"href":"https://rdap.centralnic.com/lol/domain/1xbet-ieon.lol","rel":"self"},{"href":"https://whois.nic.lol/rdap","rel":"help"},{"href":"https://gen.xyz/","rel":"related"},{"href":"https://rdap.namecheap.com/domain/1xbet-ieon.lol","rel":"related"},{"href":"https://rdap.centralnic.com/lol/domain/1xbet-ieon.lol","rel":"self"},{"href":"https://whois.nic.lol/rdap","rel":"help"},{"href":"https://gen.xyz/","rel":"related"},{"href":"https://rdap.namecheap.com/domain/1xbet-ieon.lol","rel":"related"}],"nameservers":["dns1.registrar-servers.com","dns2.registrar-servers.com"],"registrar":{"contacts":[{"email":"[email protected]","handle":"not applicable","name":"Abuse Contact","org":"Namecheap","phone":"tel:+1.9854014545","roles":["abuse"]}],"iana_id":"1068","name":"Namecheap"},"unclassified_emails":[]},"registrar_request_url":"https://rdap.centralnic.com/lol/domain/1xbet-ieon.lol","registry_request_url":"https://rdap.centralnic.com/lol/domain/1xbet-ieon.lol"}}
+  - path: /v1/feed/domainrisk/
+    methods: [GET]
+    responses:
+      - status_code: 200
+        body: |-
+          {"timestamp":"2025-09-06T23:08:07Z","domain":"bathroom-remodeling-65908.bond","phishing_risk":99,"malware_risk":99,"spam_risk":77,"proximity_risk":100,"overall_risk":100}
+          {"timestamp":"2025-09-06T23:08:07Z","domain":"dental-implants-45730.bond","phishing_risk":99,"malware_risk":99,"spam_risk":66,"proximity_risk":100,"overall_risk":100}
+  - path: /v1/feed/domainhotlist/
+    methods: [GET]
+    responses:
+      - status_code: 200
+        body: |-
+          {"timestamp":"2025-09-06T23:00:08Z","domain":"axrszo1ibm.click","phishing_risk":99,"malware_risk":99,"spam_risk":99,"proximity_risk":100,"overall_risk":100,"expires":"2025-09-07T22:57:35Z"}
+          {"timestamp":"2025-09-06T22:35:37Z","domain":"tqnbs936.com","phishing_risk":99,"malware_risk":99,"spam_risk":99,"proximity_risk":100,"overall_risk":100,"expires":"2025-09-07T18:32:34Z"}

packages/ti_domaintools/data_stream/domainhotlist_feed/_dev/test/pipeline/test-event.log

@@ -0,0 +1,3 @@
+{"timestamp":"2025-09-06T23:00:08Z","domain":"axrszo1ibm.click","phishing_risk":99,"malware_risk":99,"spam_risk":99,"proximity_risk":100,"overall_risk":100,"expires":"2025-09-07T22:57:35Z"}
+{"timestamp":"2025-09-06T22:35:37Z","domain":"tqnbs936.com","phishing_risk":99,"malware_risk":99,"spam_risk":99,"proximity_risk":100,"overall_risk":100,"expires":"2025-09-07T18:32:34Z"}
+{"timestamp":"2025-09-06T22:36:07Z","domain":"trackers-fr-relais.com","phishing_risk":99,"malware_risk":99,"spam_risk":99,"proximity_risk":100,"overall_risk":100,"expires":"2025-09-07T22:33:47Z"}

packages/ti_domaintools/data_stream/domainhotlist_feed/_dev/test/pipeline/test-event.log-expected.json

@@ -0,0 +1,97 @@
+{
+    "expected": [
+        {
+            "domaintools": {
+                "domain": "axrszo1ibm.click",
+                "expires": "2025-09-07T22:57:35Z",
+                "malware_risk": 99,
+                "overall_risk": 100,
+                "phishing_risk": 99,
+                "proximity_risk": 100,
+                "spam_risk": 99,
+                "timestamp": "2025-09-06T23:00:08Z"
+            },
+            "ecs": {
+                "version": "8.17.0"
+            },
+            "event": {
+                "category": [
+                    "threat"
+                ],
+                "kind": "enrichment",
+                "original": "{\"timestamp\":\"2025-09-06T23:00:08Z\",\"domain\":\"axrszo1ibm.click\",\"phishing_risk\":99,\"malware_risk\":99,\"spam_risk\":99,\"proximity_risk\":100,\"overall_risk\":100,\"expires\":\"2025-09-07T22:57:35Z\"}",
+                "type": [
+                    "indicator"
+                ]
+            },
+            "threat": {
+                "indicator": {
+                    "name": "axrszo1ibm.click",
+                    "type": "domain-name"
+                }
+            }
+        },
+        {
+            "domaintools": {
+                "domain": "tqnbs936.com",
+                "expires": "2025-09-07T18:32:34Z",
+                "malware_risk": 99,
+                "overall_risk": 100,
+                "phishing_risk": 99,
+                "proximity_risk": 100,
+                "spam_risk": 99,
+                "timestamp": "2025-09-06T22:35:37Z"
+            },
+            "ecs": {
+                "version": "8.17.0"
+            },
+            "event": {
+                "category": [
+                    "threat"
+                ],
+                "kind": "enrichment",
+                "original": "{\"timestamp\":\"2025-09-06T22:35:37Z\",\"domain\":\"tqnbs936.com\",\"phishing_risk\":99,\"malware_risk\":99,\"spam_risk\":99,\"proximity_risk\":100,\"overall_risk\":100,\"expires\":\"2025-09-07T18:32:34Z\"}",
+                "type": [
+                    "indicator"
+                ]
+            },
+            "threat": {
+                "indicator": {
+                    "name": "tqnbs936.com",
+                    "type": "domain-name"
+                }
+            }
+        },
+        {
+            "domaintools": {
+                "domain": "trackers-fr-relais.com",
+                "expires": "2025-09-07T22:33:47Z",
+                "malware_risk": 99,
+                "overall_risk": 100,
+                "phishing_risk": 99,
+                "proximity_risk": 100,
+                "spam_risk": 99,
+                "timestamp": "2025-09-06T22:36:07Z"
+            },
+            "ecs": {
+                "version": "8.17.0"
+            },
+            "event": {
+                "category": [
+                    "threat"
+                ],
+                "kind": "enrichment",
+                "original": "{\"timestamp\":\"2025-09-06T22:36:07Z\",\"domain\":\"trackers-fr-relais.com\",\"phishing_risk\":99,\"malware_risk\":99,\"spam_risk\":99,\"proximity_risk\":100,\"overall_risk\":100,\"expires\":\"2025-09-07T22:33:47Z\"}",
+                "type": [
+                    "indicator"
+                ]
+            },
+            "threat": {
+                "indicator": {
+                    "name": "trackers-fr-relais.com",
+                    "type": "domain-name"
+                }
+            }
+        }
+    ]
+}

packages/ti_domaintools/data_stream/domainhotlist_feed/_dev/test/system/test-default-config.yml

@@ -0,0 +1,11 @@
+input: cel
+service: ti_domaintools
+vars:
+data_stream:
+  vars:
+    api_url: http://{{Hostname}}:{{Port}}/v1
+    interval: 10m
+    api_username: xxx
+    api_key: xxx
+assert:
+  hit_count: 2

packages/ti_domaintools/data_stream/domainhotlist_feed/agent/stream/cel.yml.hbs

@@ -0,0 +1,65 @@
+config_version: "2"
+interval: {{interval}}
+resource.tracer:
+  enabled: {{enable_request_tracer}}
+  filename: "../../logs/cel/http-request-trace-*.ndjson"
+  maxbackups: 5
+resource.url: {{api_url}}
+state:
+  api_username: {{api_username}}
+  api_key: {{api_key}}
+  session_id: {{session_id}}
+  app_name: elastic_feeds
+  app_partner: elastic
+  app_version: 1.1.0
+  top: {{top}}
+redact:
+  fields:
+    - api_key
+program: |
+  state.with(
+    request(
+      "GET",
+      state.url.trim_right("/") + "/feed/domainhotlist/?" + {
+        "api_username": [state.api_username],
+        "api_key": [state.api_key],
+        "sessionID": [state.session_id],
+        "app_name": [state.app_name],
+        "app_partner": [state.app_partner],
+        "app_version": [state.app_version],
+        "top": [string(state.top)],
+      }.format_query()
+    ).with(
+      {
+        "Header": {
+          "Accept": ["application/x-ndjson"],
+        },
+      }
+    ).do_request().as(resp, (resp.StatusCode == 200 || resp.StatusCode == 206) ?
+      {
+        "events": string(resp.Body).split("\n").map(e, e!="",
+          {
+            "message": e,
+          }
+        ),
+        "want_more": resp.StatusCode == 206
+      }
+    :
+      {
+        "events": {
+          "error": {
+            "code": string(resp.StatusCode),
+            "id": string(resp.Status),
+            "message": "GET: " +
+            (
+              (size(resp.Body) != 0) ?
+                string(resp.Body)
+              :
+                string(resp.Status) + " (" + string(resp.StatusCode) + ")"
+            ),
+          },
+        },
+        "want_more": false,
+      }
+    )
+  )

packages/ti_domaintools/data_stream/domainhotlist_feed/elasticsearch/ilm/default_policy.json

@@ -0,0 +1,23 @@
+{
+    "policy": {
+        "phases": {
+            "hot": {
+                "actions": {
+                    "rollover": {
+                        "max_age": "2d",
+                        "max_size": "50gb"
+                    },
+                    "set_priority": {
+                        "priority": 100
+                    }
+                }
+            },
+            "delete": {
+                "min_age": "3d",
+                "actions": {
+                    "delete": {}
+                }
+            }
+        }
+    }
+}

packages/ti_domaintools/data_stream/domainhotlist_feed/elasticsearch/ingest_pipeline/default.yml

@@ -0,0 +1,67 @@
+---
+description: Pipeline for processing domaindiscovery feed
+processors:
+  - set:
+      field: ecs.version
+      value: '8.17.0'
+
+  - terminate:
+      tag: data_collection_error
+      if: ctx.error?.message != null
+      description: error message set and no data to process.
+
+  - rename:
+      field: message
+      target_field: event.original
+      ignore_missing: true
+      if: ctx.event?.original == null
+  - remove:
+      field: message
+      tag: remove_message
+      ignore_missing: true
+      description: The `message` field is no longer required if the document has an `event.original` field.
+      if: ctx.event?.original != null
+
+  - json:
+      field: event.original
+      target_field: domaintools
+
+    ############################
+    # Generic indicator fields #
+    ############################
+
+  - set:
+      field: threat.indicator.type
+      value: domain-name
+  - set:
+      if: ctx.domaintools?.domain != null
+      field: threat.indicator.name
+      copy_from: domaintools.domain
+
+    ####################
+    # Event ECS fields #
+    ####################
+  - set:
+      field: event.kind
+      value: enrichment
+  - set:
+      field: event.category
+      value: ['threat']
+  - set:
+      field: event.type
+      value: ['indicator']
+
+on_failure:
+- set:
+    field: event.kind
+    value: pipeline_error
+- append:
+    field: tags
+    value: preserve_original_event
+    allow_duplicates: false
+- append:
+    field: error.message
+    value: >-
+      Processor '{{{ _ingest.on_failure_processor_type }}}'
+      {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+      {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}'

packages/ti_domaintools/data_stream/domainhotlist_feed/fields/base-fields.yml

@@ -0,0 +1,9 @@
+- name: data_stream.type
+  external: ecs
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  external: ecs
+- name: "@timestamp"
+  external: ecs

packages/ti_domaintools/data_stream/domainhotlist_feed/fields/ecs.yml

@@ -0,0 +1,30 @@
+- name: ecs.version
+  external: ecs
+- name: error.message
+  external: ecs
+- name: event.category
+  external: ecs
+- name: event.id
+  external: ecs
+- name: event.ingested
+  external: ecs
+- name: event.kind
+  external: ecs
+- name: event.type
+  external: ecs
+- name: threat.indicator.type
+  external: ecs
+- name: threat.indicator.name
+  external: ecs
+- name: threat.feed.description
+  type: constant_keyword
+  description: Display the feed description.
+  value: "Domains with high Domain Risk Scores that have also been active within 24 hours."
+- name: threat.feed.name
+  type: constant_keyword
+  description: Display friendly feed name.
+  value: "DomainTools domainhotlist"
+- name: threat.feed.reference
+  type: constant_keyword
+  description: Display the feed reference.
+  value: "https://docs.techdocs.ci.domaintools.cloud/feeds/realtime/userguide/"