From fa182c524ba1cb42220c02b0da72174a356f98e4 Mon Sep 17 00:00:00 2001 From: bluza Date: Tue, 20 May 2025 19:57:43 +0800 Subject: [PATCH 1/4] [IDEV-1964_1988] Implement nad and domaindiscovery feed; update readme docs --- .../ti_domaintools/_dev/build/docs/README.md | 58 +++- .../_dev/deploy/docker/files/config.yml | 14 + .../_dev/test/pipeline/test-event.json | 13 + .../pipeline/test-event.json-expected.json | 94 +++++++ .../_dev/test/system/test-default-config.yml | 11 + .../domaindiscovery/agent/stream/cel.yml.hbs | 63 +++++ .../elasticsearch/ilm/default_policy.json | 23 ++ .../elasticsearch/ingest_pipeline/default.yml | 59 ++++ .../domaindiscovery/fields/base-fields.yml | 8 + .../domaindiscovery/fields/ecs.yml | 24 ++ .../domaindiscovery/fields/fields.yml | 26 ++ .../fields/is-ioc-transform-source-true.yml | 4 + .../data_stream/domaindiscovery/lifecycle.yml | 1 + .../data_stream/domaindiscovery/manifest.yml | 58 ++++ .../domaindiscovery/sample_event.json | 77 ++++++ .../_dev/test/pipeline/test-event.json | 13 + .../pipeline/test-event.json-expected.json | 94 +++++++ .../_dev/test/system/test-default-config.yml | 11 + .../nad_feed/agent/stream/cel.yml.hbs | 63 +++++ .../elasticsearch/ilm/default_policy.json | 23 ++ .../elasticsearch/ingest_pipeline/default.yml | 60 ++++ .../nad_feed/fields/base-fields.yml | 8 + .../data_stream/nad_feed/fields/fields.yml | 26 ++ .../fields/is-ioc-transform-source-true.yml | 4 + .../data_stream/nad_feed/lifecycle.yml | 1 + .../data_stream/nad_feed/manifest.yml | 58 ++++ .../data_stream/nad_feed/sample_event.json | 77 ++++++ .../nod_feed/agent/stream/cel.yml.hbs | 5 +- .../elasticsearch/ingest_pipeline/default.yml | 10 +- .../data_stream/nod_feed/fields/fields.yml | 1 - .../data_stream/nod_feed/manifest.yml | 18 +- .../data_stream/nod_feed/sample_event.json | 1 + packages/ti_domaintools/docs/README.md | 259 +++++++++++++++++- .../fields/base-fields.yml | 16 ++ .../latest_domaindiscovery/fields/beats.yml | 11 + .../latest_domaindiscovery/fields/ecs.yml | 24 ++ .../latest_domaindiscovery/fields/fields.yml | 22 ++ .../fields/is-ioc-transform-source-false.yml | 4 + .../latest_domaindiscovery/manifest.yml | 18 ++ .../latest_domaindiscovery/transform.yml | 35 +++ .../latest_nad/fields/base-fields.yml | 16 ++ .../transform/latest_nad/fields/beats.yml | 11 + .../transform/latest_nad/fields/ecs.yml | 24 ++ .../transform/latest_nad/fields/fields.yml | 22 ++ .../fields/is-ioc-transform-source-false.yml | 4 + .../transform/latest_nad/manifest.yml | 18 ++ .../transform/latest_nad/transform.yml | 35 +++ .../transform/latest_nod/fields/fields.yml | 5 +- packages/ti_domaintools/manifest.yml | 10 +- 49 files changed, 1491 insertions(+), 49 deletions(-) create mode 100644 packages/ti_domaintools/data_stream/domaindiscovery/_dev/test/pipeline/test-event.json create mode 100644 packages/ti_domaintools/data_stream/domaindiscovery/_dev/test/pipeline/test-event.json-expected.json create mode 100644 packages/ti_domaintools/data_stream/domaindiscovery/_dev/test/system/test-default-config.yml create mode 100644 packages/ti_domaintools/data_stream/domaindiscovery/agent/stream/cel.yml.hbs create mode 100644 packages/ti_domaintools/data_stream/domaindiscovery/elasticsearch/ilm/default_policy.json create mode 100644 packages/ti_domaintools/data_stream/domaindiscovery/elasticsearch/ingest_pipeline/default.yml create mode 100644 packages/ti_domaintools/data_stream/domaindiscovery/fields/base-fields.yml create mode 100644 packages/ti_domaintools/data_stream/domaindiscovery/fields/ecs.yml create mode 100644 packages/ti_domaintools/data_stream/domaindiscovery/fields/fields.yml create mode 100644 packages/ti_domaintools/data_stream/domaindiscovery/fields/is-ioc-transform-source-true.yml create mode 100644 packages/ti_domaintools/data_stream/domaindiscovery/lifecycle.yml create mode 100644 packages/ti_domaintools/data_stream/domaindiscovery/manifest.yml create mode 100644 packages/ti_domaintools/data_stream/domaindiscovery/sample_event.json create mode 100644 packages/ti_domaintools/data_stream/nad_feed/_dev/test/pipeline/test-event.json create mode 100644 packages/ti_domaintools/data_stream/nad_feed/_dev/test/pipeline/test-event.json-expected.json create mode 100644 packages/ti_domaintools/data_stream/nad_feed/_dev/test/system/test-default-config.yml create mode 100644 packages/ti_domaintools/data_stream/nad_feed/agent/stream/cel.yml.hbs create mode 100644 packages/ti_domaintools/data_stream/nad_feed/elasticsearch/ilm/default_policy.json create mode 100644 packages/ti_domaintools/data_stream/nad_feed/elasticsearch/ingest_pipeline/default.yml create mode 100644 packages/ti_domaintools/data_stream/nad_feed/fields/base-fields.yml create mode 100644 packages/ti_domaintools/data_stream/nad_feed/fields/fields.yml create mode 100644 packages/ti_domaintools/data_stream/nad_feed/fields/is-ioc-transform-source-true.yml create mode 100644 packages/ti_domaintools/data_stream/nad_feed/lifecycle.yml create mode 100644 packages/ti_domaintools/data_stream/nad_feed/manifest.yml create mode 100644 packages/ti_domaintools/data_stream/nad_feed/sample_event.json create mode 100644 packages/ti_domaintools/elasticsearch/transform/latest_domaindiscovery/fields/base-fields.yml create mode 100644 packages/ti_domaintools/elasticsearch/transform/latest_domaindiscovery/fields/beats.yml create mode 100644 packages/ti_domaintools/elasticsearch/transform/latest_domaindiscovery/fields/ecs.yml create mode 100644 packages/ti_domaintools/elasticsearch/transform/latest_domaindiscovery/fields/fields.yml create mode 100644 packages/ti_domaintools/elasticsearch/transform/latest_domaindiscovery/fields/is-ioc-transform-source-false.yml create mode 100644 packages/ti_domaintools/elasticsearch/transform/latest_domaindiscovery/manifest.yml create mode 100644 packages/ti_domaintools/elasticsearch/transform/latest_domaindiscovery/transform.yml create mode 100644 packages/ti_domaintools/elasticsearch/transform/latest_nad/fields/base-fields.yml create mode 100644 packages/ti_domaintools/elasticsearch/transform/latest_nad/fields/beats.yml create mode 100644 packages/ti_domaintools/elasticsearch/transform/latest_nad/fields/ecs.yml create mode 100644 packages/ti_domaintools/elasticsearch/transform/latest_nad/fields/fields.yml create mode 100644 packages/ti_domaintools/elasticsearch/transform/latest_nad/fields/is-ioc-transform-source-false.yml create mode 100644 packages/ti_domaintools/elasticsearch/transform/latest_nad/manifest.yml create mode 100644 packages/ti_domaintools/elasticsearch/transform/latest_nad/transform.yml diff --git a/packages/ti_domaintools/_dev/build/docs/README.md b/packages/ti_domaintools/_dev/build/docs/README.md index 21da9b3ef80..81fe0083dfd 100644 --- a/packages/ti_domaintools/_dev/build/docs/README.md +++ b/packages/ti_domaintools/_dev/build/docs/README.md @@ -1,37 +1,45 @@ # DomainTools Real Time Unified Feeds -The DomainTools Real Time Unified Feeds integration allows you to monitor DomainTools Newly Observed Domains. -The DomainTools NOD Feed provides real-time access to newly registered and observed domains, enabling proactive threat detection and defense. +DomainTools Real-Time Threat Intelligence Feeds provide data on the different stages of the domain lifecycle: from first-observed in the wild, to newly re-activated after a period of quiet. Access current feed data in real-time or retrieve historical feed data through separate APIs. Some feeds also offer data for DNS firewalls in Response Policy Zone (RPZ) format. -With over 300,000 new domains observed daily, the feed empowers security teams to identify and block potentially malicious domains before they can be weaponized. -Ideal for threat hunting, phishing prevention, and brand protection, the NOD Feed delivers unparalleled visibility into emerging domain activity to stay ahead of evolving threats. +Summary of Available Feeds: -For example, if you wanted to monitor Newly Observed Domains (NOD) feed, you could ingest the DomainTools NOD feed. -Then you can reference domaintools.nod_feed when using visualizations or alerts. +- `Domain Discovery`: New domains as they are either discovered in domain registration information, observed by our global sensor network, or reported by trusted third parties. +- `Newly Active Domains (NAD)`: Apex-level domains (e.g. example.com but not ) that we observe based on the latest lifecycle of the domain. A domain may be seen either for the first time ever, or again after at least 10 days of inactivity (no observed resolutions in DNS). Populated with our global passive DNS (pDNS) sensor network. +- `Newly Observed Domains (NOD)`: Apex-level domains (e.g. example.com but not ) that we observe for the first time, and have not observed previously with our global DNS sensor network. + +With over 300,000 new domains observed daily, the feed empowers security teams to identify and block potentially malicious domains before they can be weaponized. +Ideal for threat hunting, phishing prevention, and brand protection. + +For example, if you wanted to monitor Newly Observed Domains (NOD) feed, you could ingest the DomainTools NOD feed. +Then you can reference ti_domaintools.nod_feed when using visualizations or alerts. ## Data streams -The DomainTools Real Time Unified Feeds integration collects one type of data streams: logs +The DomainTools Real Time Unified Feeds integration collects one type of data streams: **logs** + +Log data streams collected by the DomainTools integration include the following feeds: -Log data streams collected by the DomainTools integration include the Newly Observed Domains (NOD) feed: Apex-level domains (e.g. Example Domain but not www.example.com) that we observe for the first time, and have not observed previously. -Populated with our global DNS sensor network. +- `Domain Discovery` +- `Newly Observed Domains (NOD) feed` +- `Newly Active Domains (NAD)` ## Requirements -You need Elasticsearch for storing and searching your data and Kibana for visualizing and managing it. +You need Elasticsearch for storing and searching your data and Kibana for visualizing and managing it. You can use our hosted Elasticsearch Service on Elastic Cloud, which is recommended, or self-manage the Elastic Stack on your own hardware. -You will require a license to one or more DomainTools feeds, and API credentials. -Your required API credentials will vary with your authentication method, detailed below. +You will require a license to one or more DomainTools feeds, and API credentials. +Your required API credentials will vary with your authentication method, detailed below. -Obtain your API credentials from your group’s API administrator. +Obtain your API credentials from your group’s API administrator. API administrators can manage their API keys at research.domaintools.com, selecting the drop-down account menu and choosing API admin. ## Setup For step-by-step instructions on how to set up an integration, see the Getting started guide. -### Newly Observed Domains (NOD) Feed +### Newly Observed Domains (NOD) Feed The `nod_feed` data stream provides events from [DomainTools Newly Observed Domains Feed](https://www.domaintools.com/products/threat-intelligence-feeds/). This data is collected via the [DomainTools Real Time Feeds API](https://docs.domaintools.com/feeds/realtime/). @@ -42,3 +50,25 @@ This data is collected via the [DomainTools Real Time Feeds API](https://docs.do {{fields "nod_feed"}} + +### Newly Active Domains (NAD) Feed + +The `nod_feed` data stream provides events from [DomainTools Newly Active Domains Feed](https://www.domaintools.com/products/threat-intelligence-feeds/). +This data is collected via the [DomainTools Real Time Feeds API](https://docs.domaintools.com/feeds/realtime/). + +#### Example + +{{event "nad_feed"}} + +{{fields "nad_feed"}} + +### Domain Discovery Feed + +The `domaindiscovery feed` data stream provides events from [DomainTools Domain Discovery Feed](https://www.domaintools.com/products/threat-intelligence-feeds/). +This data is collected via the [DomainTools Real Time Feeds API](https://docs.domaintools.com/feeds/realtime/). + +#### Example + +{{event "domaindiscovery"}} + +{{fields "domaindiscovery"}} diff --git a/packages/ti_domaintools/_dev/deploy/docker/files/config.yml b/packages/ti_domaintools/_dev/deploy/docker/files/config.yml index ee46f1622f2..5f12d62bfdf 100644 --- a/packages/ti_domaintools/_dev/deploy/docker/files/config.yml +++ b/packages/ti_domaintools/_dev/deploy/docker/files/config.yml @@ -6,3 +6,17 @@ rules: body: |- {"timestamp":"2025-01-11T08:42:46Z","domain":"test1.com"} {"timestamp":"2025-01-11T08:42:46Z","domain":"test2.com"} + - path: /v1/feed/nad/ + methods: [GET] + responses: + - status_code: 200 + body: |- + {"timestamp":"2025-01-11T08:42:46Z","domain":"test3.com"} + {"timestamp":"2025-01-11T08:42:46Z","domain":"test4.com"} + - path: /v1/feed/domaindiscovery/ + methods: [GET] + responses: + - status_code: 200 + body: |- + {"timestamp":"2025-01-11T08:42:46Z","domain":"test5.com"} + {"timestamp":"2025-01-11T08:42:46Z","domain":"test6.com"} diff --git a/packages/ti_domaintools/data_stream/domaindiscovery/_dev/test/pipeline/test-event.json b/packages/ti_domaintools/data_stream/domaindiscovery/_dev/test/pipeline/test-event.json new file mode 100644 index 00000000000..75451670f59 --- /dev/null +++ b/packages/ti_domaintools/data_stream/domaindiscovery/_dev/test/pipeline/test-event.json @@ -0,0 +1,13 @@ +{ + "events": [ + { + "message": "{\"timestamp\":\"2025-01-11T08:42:46Z\",\"domain\":\"ccnitsolution.com\"}" + }, + { + "message": "{\"timestamp\":\"2025-01-11T08:42:46Z\",\"domain\":\"ccnitsolution2.com\"}" + }, + { + "message": "{\"timestamp\":\"2025-01-11T08:42:46Z\",\"domain\":\"ccnitsolution3.com\"}" + } + ] +} \ No newline at end of file diff --git a/packages/ti_domaintools/data_stream/domaindiscovery/_dev/test/pipeline/test-event.json-expected.json b/packages/ti_domaintools/data_stream/domaindiscovery/_dev/test/pipeline/test-event.json-expected.json new file mode 100644 index 00000000000..22563cc9f23 --- /dev/null +++ b/packages/ti_domaintools/data_stream/domaindiscovery/_dev/test/pipeline/test-event.json-expected.json @@ -0,0 +1,94 @@ +{ + "expected": [ + { + "domaintools": { + "domain": "ccnitsolution.com", + "timestamp": "2025-01-11T08:42:46Z" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "category": [ + "threat" + ], + "kind": "enrichment", + "type": [ + "indicator" + ] + }, + "message": "{\"timestamp\":\"2025-01-11T08:42:46Z\",\"domain\":\"ccnitsolution.com\"}", + "threat": { + "feed": { + "description": "Apex-level domains (e.g. example.com but not www.example.com) that we observe for the first time, and have not observed previously with our global DNS sensor network.", + "name": "DomainTools domaindiscovery", + "reference": "https://docs.techdocs.ci.domaintools.cloud/feeds/realtime/userguide/" + }, + "indicator": { + "name": "ccnitsolution.com", + "type": "domain-name" + } + } + }, + { + "domaintools": { + "domain": "ccnitsolution2.com", + "timestamp": "2025-01-11T08:42:46Z" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "category": [ + "threat" + ], + "kind": "enrichment", + "type": [ + "indicator" + ] + }, + "message": "{\"timestamp\":\"2025-01-11T08:42:46Z\",\"domain\":\"ccnitsolution2.com\"}", + "threat": { + "feed": { + "description": "Apex-level domains (e.g. example.com but not www.example.com) that we observe for the first time, and have not observed previously with our global DNS sensor network.", + "name": "DomainTools domaindiscovery", + "reference": "https://docs.techdocs.ci.domaintools.cloud/feeds/realtime/userguide/" + }, + "indicator": { + "name": "ccnitsolution2.com", + "type": "domain-name" + } + } + }, + { + "domaintools": { + "domain": "ccnitsolution3.com", + "timestamp": "2025-01-11T08:42:46Z" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "category": [ + "threat" + ], + "kind": "enrichment", + "type": [ + "indicator" + ] + }, + "message": "{\"timestamp\":\"2025-01-11T08:42:46Z\",\"domain\":\"ccnitsolution3.com\"}", + "threat": { + "feed": { + "description": "Apex-level domains (e.g. example.com but not www.example.com) that we observe for the first time, and have not observed previously with our global DNS sensor network.", + "name": "DomainTools domaindiscovery", + "reference": "https://docs.techdocs.ci.domaintools.cloud/feeds/realtime/userguide/" + }, + "indicator": { + "name": "ccnitsolution3.com", + "type": "domain-name" + } + } + } + ] +} \ No newline at end of file diff --git a/packages/ti_domaintools/data_stream/domaindiscovery/_dev/test/system/test-default-config.yml b/packages/ti_domaintools/data_stream/domaindiscovery/_dev/test/system/test-default-config.yml new file mode 100644 index 00000000000..84dfca9d63b --- /dev/null +++ b/packages/ti_domaintools/data_stream/domaindiscovery/_dev/test/system/test-default-config.yml @@ -0,0 +1,11 @@ +input: cel +service: domaintools +vars: +data_stream: + vars: + api_url: http://{{Hostname}}:{{Port}}/v1 + interval: 10m + api_username: xxx + api_key: xxx +assert: + hit_count: 2 diff --git a/packages/ti_domaintools/data_stream/domaindiscovery/agent/stream/cel.yml.hbs b/packages/ti_domaintools/data_stream/domaindiscovery/agent/stream/cel.yml.hbs new file mode 100644 index 00000000000..0d010317316 --- /dev/null +++ b/packages/ti_domaintools/data_stream/domaindiscovery/agent/stream/cel.yml.hbs @@ -0,0 +1,63 @@ +config_version: "2" +interval: {{interval}} +resource.tracer: + enabled: {{enable_request_tracer}} + filename: "../../logs/cel/http-request-trace-*.ndjson" + maxbackups: 5 +resource.url: {{api_url}} +state: + api_username: {{api_username}} + api_key: {{api_key}} + session_id: {{session_id}} + app_name: elastic_feeds + app_partner: elastic + app_version: 0.1.0 +redact: + fields: + - api_key +program: | + state.with( + request( + "GET", + state.url.trim_right("/") + "/feed/domaindiscovery/?" + { + "api_username": [state.api_username], + "api_key": [state.api_key], + "sessionID": [state.session_id], + "app_name": [state.app_name], + "app_partner": [state.app_partner], + "app_version": [state.app_version], + }.format_query() + ).with( + { + "Header": { + "Accept": ["application/x-ndjson"], + }, + } + ).do_request().as(resp, (resp.StatusCode == 200 || resp.StatusCode == 206) ? + { + "events": string(resp.Body).split("\n").filter(x,x!="").map(e, + { + "message": e, + } + ), + "want_more": resp.StatusCode == 206 + } + : + { + "events": { + "error": { + "code": string(resp.StatusCode), + "id": string(resp.Status), + "message": "GET:" + + ( + (size(resp.Body) != 0) ? + string(resp.Body) + : + string(resp.Status) + " (" + string(resp.StatusCode) + ")" + ), + }, + }, + "want_more": false, + } + ) + ) \ No newline at end of file diff --git a/packages/ti_domaintools/data_stream/domaindiscovery/elasticsearch/ilm/default_policy.json b/packages/ti_domaintools/data_stream/domaindiscovery/elasticsearch/ilm/default_policy.json new file mode 100644 index 00000000000..68d2c5e57a6 --- /dev/null +++ b/packages/ti_domaintools/data_stream/domaindiscovery/elasticsearch/ilm/default_policy.json @@ -0,0 +1,23 @@ +{ + "policy": { + "phases": { + "hot": { + "actions": { + "rollover": { + "max_age": "2d", + "max_size": "50gb" + }, + "set_priority": { + "priority": 100 + } + } + }, + "delete": { + "min_age": "3d", + "actions": { + "delete": {} + } + } + } + } +} \ No newline at end of file diff --git a/packages/ti_domaintools/data_stream/domaindiscovery/elasticsearch/ingest_pipeline/default.yml b/packages/ti_domaintools/data_stream/domaindiscovery/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..efded94147c --- /dev/null +++ b/packages/ti_domaintools/data_stream/domaindiscovery/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,59 @@ +--- +description: Pipeline for processing domaindiscovery feed +processors: +- json: + if: ctx?.message != null + field: message + target_field: domaintools +- set: + field: domaintools.feed + value: 'domaindiscovery' + + ############################ + # Generic indicator fields # + ############################ + +- set: + field: threat.indicator.type + value: domain-name +- set: + if: ctx.domaintools != null && ctx.domaintools?.domain != null + field: threat.indicator.name + copy_from: domaintools.domain + + ###################### + # Threat feed fields # + ###################### + +- set: + field: threat.feed.description + value: 'Apex-level domains (e.g. example.com but not www.example.com) that we observe for the first time, and have not observed previously with our global DNS sensor network.' +- set: + field: threat.feed.name + value: 'DomainTools domaindiscovery' +- set: + field: threat.feed.reference + value: https://docs.techdocs.ci.domaintools.cloud/feeds/realtime/userguide/ + + #################### + # Event ECS fields # + #################### + +- set: + field: ecs.version + value: '8.11.0' +- set: + field: event.kind + value: enrichment +- set: + field: event.category + value: ['threat'] +- set: + field: event.type + value: ['indicator'] + +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' + diff --git a/packages/ti_domaintools/data_stream/domaindiscovery/fields/base-fields.yml b/packages/ti_domaintools/data_stream/domaindiscovery/fields/base-fields.yml new file mode 100644 index 00000000000..0f3844069d4 --- /dev/null +++ b/packages/ti_domaintools/data_stream/domaindiscovery/fields/base-fields.yml @@ -0,0 +1,8 @@ +- name: data_stream.type + external: ecs +- name: data_stream.dataset + external: ecs +- name: data_stream.namespace + external: ecs +- name: "@timestamp" + external: ecs diff --git a/packages/ti_domaintools/data_stream/domaindiscovery/fields/ecs.yml b/packages/ti_domaintools/data_stream/domaindiscovery/fields/ecs.yml new file mode 100644 index 00000000000..a32072de31a --- /dev/null +++ b/packages/ti_domaintools/data_stream/domaindiscovery/fields/ecs.yml @@ -0,0 +1,24 @@ +- name: ecs.version + external: ecs +- name: error.message + external: ecs +- name: event.category + external: ecs +- name: event.id + external: ecs +- name: event.ingested + external: ecs +- name: event.kind + external: ecs +- name: event.type + external: ecs +- name: threat.indicator.type + external: ecs +- name: threat.indicator.name + external: ecs +- name: threat.feed.description + external: ecs +- name: threat.feed.name + external: ecs +- name: threat.feed.reference + external: ecs \ No newline at end of file diff --git a/packages/ti_domaintools/data_stream/domaindiscovery/fields/fields.yml b/packages/ti_domaintools/data_stream/domaindiscovery/fields/fields.yml new file mode 100644 index 00000000000..e24af4217e4 --- /dev/null +++ b/packages/ti_domaintools/data_stream/domaindiscovery/fields/fields.yml @@ -0,0 +1,26 @@ +- name: domaintools + type: group + fields: + - name: domain + type: keyword + description: > + The Domain. Apex-level domains (e.g. example.com but not www.example.com) that we observe for the first time, and have not observed previously with our global DNS sensor network. + + - name: feed + type: keyword + description: > + The feed type. + + - name: timestamp + type: date + description: > + Timestamp when the domain was added to the DomainTools feed, in ISO 8601 UTC form. + +- name: message + external: ecs + description: > + The feed from DomainTools Feed API. + +- name: input.type + type: keyword + description: Type of filebeat input. diff --git a/packages/ti_domaintools/data_stream/domaindiscovery/fields/is-ioc-transform-source-true.yml b/packages/ti_domaintools/data_stream/domaindiscovery/fields/is-ioc-transform-source-true.yml new file mode 100644 index 00000000000..0b58e653511 --- /dev/null +++ b/packages/ti_domaintools/data_stream/domaindiscovery/fields/is-ioc-transform-source-true.yml @@ -0,0 +1,4 @@ +- name: labels.is_ioc_transform_source + type: constant_keyword + value: "true" + description: Indicates whether an IOC is in the raw source data stream, or the in latest destination index. \ No newline at end of file diff --git a/packages/ti_domaintools/data_stream/domaindiscovery/lifecycle.yml b/packages/ti_domaintools/data_stream/domaindiscovery/lifecycle.yml new file mode 100644 index 00000000000..5a4af9095b7 --- /dev/null +++ b/packages/ti_domaintools/data_stream/domaindiscovery/lifecycle.yml @@ -0,0 +1 @@ +data_retention: "5d" diff --git a/packages/ti_domaintools/data_stream/domaindiscovery/manifest.yml b/packages/ti_domaintools/data_stream/domaindiscovery/manifest.yml new file mode 100644 index 00000000000..170b519db3a --- /dev/null +++ b/packages/ti_domaintools/data_stream/domaindiscovery/manifest.yml @@ -0,0 +1,58 @@ +title: "DomainTools Real Time Unified Feeds - domaindiscovery" +type: logs +ilm_policy: logs-ti_domaintools.domaindiscovery-default_policy +streams: + - input: cel + vars: + - name: api_url + type: text + title: DomainTools API URL + multi: false + required: true + show_user: true + default: https://api.domaintools.com/v1 + description: The URL of the DomainTools API. + - name: interval + type: text + title: Interval + multi: false + required: true + show_user: true + default: 10m + description: Interval at which the feed will be pulled. Supported units for this parameter are h/m/s. + - name: api_username + type: text + title: DomainTools API Username + multi: false + required: true + show_user: true + default: DomainTools API Username + description: DomainTools API Username + - name: api_key + type: password + title: DomainTools API Key + multi: false + required: true + show_user: true + secret: true + description: DomainTools API Key + - name: session_id + type: text + title: Session ID + multi: false + required: true + show_user: true + default: DomainToolsElasticSID + description: The Session ID to use in requesting feed. + - name: enable_request_tracer + type: bool + title: Enable request tracing + description: >- + The request tracer logs requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. Disabling the request tracer will delete any stored traces. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-cel.html#_resource_tracer_enable) for details. + default: false + multi: false + required: false + show_user: false + title: DomainTools Real Time Unified Feeds - domaindiscovery + description: Subscribe to DomainTools Real Time Unified Feeds + template_path: cel.yml.hbs diff --git a/packages/ti_domaintools/data_stream/domaindiscovery/sample_event.json b/packages/ti_domaintools/data_stream/domaindiscovery/sample_event.json new file mode 100644 index 00000000000..42c5bf6e47b --- /dev/null +++ b/packages/ti_domaintools/data_stream/domaindiscovery/sample_event.json @@ -0,0 +1,77 @@ +{ + "input": { + "type": "cel" + }, + "agent": { + "name": "docker-fleet-agent", + "id": "da8d0a37-2d46-4788-96bd-e9ee19e332ec", + "ephemeral_id": "d1cbe648-0a1d-48e8-a161-cd82403e623e", + "type": "filebeat", + "version": "8.15.3" + }, + "@timestamp": "2025-01-30T20:15:25.396Z", + "ecs": { + "version": "8.11.0" + }, + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "ti_domaintools.nod_feed" + }, + "host": { + "hostname": "docker-fleet-agent", + "os": { + "kernel": "6.10.11-linuxkit", + "codename": "focal", + "name": "Ubuntu", + "type": "linux", + "family": "debian", + "version": "20.04.6 LTS (Focal Fossa)", + "platform": "ubuntu" + }, + "containerized": false, + "ip": [ + "172.19.0.10" + ], + "name": "docker-fleet-agent", + "id": "cfae1e7244ae479b9b0968259c91b13a", + "mac": [ + "02-42-AC-13-00-0A" + ], + "architecture": "aarch64" + }, + "elastic_agent": { + "id": "da8d0a37-2d46-4788-96bd-e9ee19e332ec", + "version": "8.15.3", + "snapshot": false + }, + "domaintools": { + "domain": "tractorpoweredcoreaerator.com", + "timestamp": "2025-01-30T20:14:48Z", + "feed": "domaindiscovery" + }, + "threat": { + "indicator": { + "name": "tractorpoweredcoreaerator.com", + "type": "domain-name" + }, + "feed": { + "reference": "https://docs.techdocs.ci.domaintools.cloud/feeds/realtime/userguide/", + "name": "DomainTools domaindiscovery", + "description": "Apex-level domains (e.g. example.com but not www.example.com) that we observe for the first time, and have not observed previously with our global DNS sensor network." + } + }, + "message": "{\"timestamp\":\"2025-01-30T20:14:48Z\",\"domain\":\"tractorpoweredcoreaerator.com\"}", + "event": { + "agent_id_status": "verified", + "ingested": "2025-01-30T20:15:26Z", + "kind": "enrichment", + "category": [ + "threat" + ], + "type": [ + "indicator" + ], + "dataset": "ti_domaintools.domaindiscovery" + } +} \ No newline at end of file diff --git a/packages/ti_domaintools/data_stream/nad_feed/_dev/test/pipeline/test-event.json b/packages/ti_domaintools/data_stream/nad_feed/_dev/test/pipeline/test-event.json new file mode 100644 index 00000000000..75451670f59 --- /dev/null +++ b/packages/ti_domaintools/data_stream/nad_feed/_dev/test/pipeline/test-event.json @@ -0,0 +1,13 @@ +{ + "events": [ + { + "message": "{\"timestamp\":\"2025-01-11T08:42:46Z\",\"domain\":\"ccnitsolution.com\"}" + }, + { + "message": "{\"timestamp\":\"2025-01-11T08:42:46Z\",\"domain\":\"ccnitsolution2.com\"}" + }, + { + "message": "{\"timestamp\":\"2025-01-11T08:42:46Z\",\"domain\":\"ccnitsolution3.com\"}" + } + ] +} \ No newline at end of file diff --git a/packages/ti_domaintools/data_stream/nad_feed/_dev/test/pipeline/test-event.json-expected.json b/packages/ti_domaintools/data_stream/nad_feed/_dev/test/pipeline/test-event.json-expected.json new file mode 100644 index 00000000000..86ea0b52b63 --- /dev/null +++ b/packages/ti_domaintools/data_stream/nad_feed/_dev/test/pipeline/test-event.json-expected.json @@ -0,0 +1,94 @@ +{ + "expected": [ + { + "domaintools": { + "domain": "ccnitsolution.com", + "timestamp": "2025-01-11T08:42:46Z" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "category": [ + "threat" + ], + "kind": "enrichment", + "type": [ + "indicator" + ] + }, + "message": "{\"timestamp\":\"2025-01-11T08:42:46Z\",\"domain\":\"ccnitsolution.com\"}", + "threat": { + "feed": { + "description": "Apex-level domains (e.g. example.com but not www.example.com) that we observe for the first time, and have not observed previously with our global DNS sensor network.", + "name": "DomainTools NAD", + "reference": "https://docs.techdocs.ci.domaintools.cloud/feeds/realtime/userguide/" + }, + "indicator": { + "name": "ccnitsolution.com", + "type": "domain-name" + } + } + }, + { + "domaintools": { + "domain": "ccnitsolution2.com", + "timestamp": "2025-01-11T08:42:46Z" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "category": [ + "threat" + ], + "kind": "enrichment", + "type": [ + "indicator" + ] + }, + "message": "{\"timestamp\":\"2025-01-11T08:42:46Z\",\"domain\":\"ccnitsolution2.com\"}", + "threat": { + "feed": { + "description": "Apex-level domains (e.g. example.com but not www.example.com) that we observe for the first time, and have not observed previously with our global DNS sensor network.", + "name": "DomainTools NAD", + "reference": "https://docs.techdocs.ci.domaintools.cloud/feeds/realtime/userguide/" + }, + "indicator": { + "name": "ccnitsolution2.com", + "type": "domain-name" + } + } + }, + { + "domaintools": { + "domain": "ccnitsolution3.com", + "timestamp": "2025-01-11T08:42:46Z" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "category": [ + "threat" + ], + "kind": "enrichment", + "type": [ + "indicator" + ] + }, + "message": "{\"timestamp\":\"2025-01-11T08:42:46Z\",\"domain\":\"ccnitsolution3.com\"}", + "threat": { + "feed": { + "description": "Apex-level domains (e.g. example.com but not www.example.com) that we observe for the first time, and have not observed previously with our global DNS sensor network.", + "name": "DomainTools NAD", + "reference": "https://docs.techdocs.ci.domaintools.cloud/feeds/realtime/userguide/" + }, + "indicator": { + "name": "ccnitsolution3.com", + "type": "domain-name" + } + } + } + ] +} \ No newline at end of file diff --git a/packages/ti_domaintools/data_stream/nad_feed/_dev/test/system/test-default-config.yml b/packages/ti_domaintools/data_stream/nad_feed/_dev/test/system/test-default-config.yml new file mode 100644 index 00000000000..84dfca9d63b --- /dev/null +++ b/packages/ti_domaintools/data_stream/nad_feed/_dev/test/system/test-default-config.yml @@ -0,0 +1,11 @@ +input: cel +service: domaintools +vars: +data_stream: + vars: + api_url: http://{{Hostname}}:{{Port}}/v1 + interval: 10m + api_username: xxx + api_key: xxx +assert: + hit_count: 2 diff --git a/packages/ti_domaintools/data_stream/nad_feed/agent/stream/cel.yml.hbs b/packages/ti_domaintools/data_stream/nad_feed/agent/stream/cel.yml.hbs new file mode 100644 index 00000000000..fd4026c4d36 --- /dev/null +++ b/packages/ti_domaintools/data_stream/nad_feed/agent/stream/cel.yml.hbs @@ -0,0 +1,63 @@ +config_version: "2" +interval: {{interval}} +resource.tracer: + enabled: {{enable_request_tracer}} + filename: "../../logs/cel/http-request-trace-*.ndjson" + maxbackups: 5 +resource.url: {{api_url}} +state: + api_username: {{api_username}} + api_key: {{api_key}} + session_id: {{session_id}} + app_name: elastic_feeds + app_partner: elastic + app_version: 0.1.0 +redact: + fields: + - api_key +program: | + state.with( + request( + "GET", + state.url.trim_right("/") + "/feed/nad/?" + { + "api_username": [state.api_username], + "api_key": [state.api_key], + "sessionID": [state.session_id], + "app_name": [state.app_name], + "app_partner": [state.app_partner], + "app_version": [state.app_version], + }.format_query() + ).with( + { + "Header": { + "Accept": ["application/x-ndjson"], + }, + } + ).do_request().as(resp, (resp.StatusCode == 200 || resp.StatusCode == 206) ? + { + "events": string(resp.Body).split("\n").filter(x,x!="").map(e, + { + "message": e, + } + ), + "want_more": resp.StatusCode == 206 + } + : + { + "events": { + "error": { + "code": string(resp.StatusCode), + "id": string(resp.Status), + "message": "GET:" + + ( + (size(resp.Body) != 0) ? + string(resp.Body) + : + string(resp.Status) + " (" + string(resp.StatusCode) + ")" + ), + }, + }, + "want_more": false, + } + ) + ) \ No newline at end of file diff --git a/packages/ti_domaintools/data_stream/nad_feed/elasticsearch/ilm/default_policy.json b/packages/ti_domaintools/data_stream/nad_feed/elasticsearch/ilm/default_policy.json new file mode 100644 index 00000000000..68d2c5e57a6 --- /dev/null +++ b/packages/ti_domaintools/data_stream/nad_feed/elasticsearch/ilm/default_policy.json @@ -0,0 +1,23 @@ +{ + "policy": { + "phases": { + "hot": { + "actions": { + "rollover": { + "max_age": "2d", + "max_size": "50gb" + }, + "set_priority": { + "priority": 100 + } + } + }, + "delete": { + "min_age": "3d", + "actions": { + "delete": {} + } + } + } + } +} \ No newline at end of file diff --git a/packages/ti_domaintools/data_stream/nad_feed/elasticsearch/ingest_pipeline/default.yml b/packages/ti_domaintools/data_stream/nad_feed/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..83d6e077537 --- /dev/null +++ b/packages/ti_domaintools/data_stream/nad_feed/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,60 @@ +--- +description: Pipeline for processing nad feed +processors: +- json: + if: ctx?.message != null + field: message + target_field: domaintools +- set: + field: domaintools.feed + value: 'nad' + + + ############################ + # Generic indicator fields # + ############################ + +- set: + field: threat.indicator.type + value: domain-name +- set: + if: ctx.domaintools != null && ctx.domaintools?.domain != null + field: threat.indicator.name + copy_from: domaintools.domain + + ###################### + # Threat feed fields # + ###################### + +- set: + field: threat.feed.description + value: 'Apex-level domains (e.g. example.com but not www.example.com) that we observe for the first time, and have not observed previously with our global DNS sensor network.' +- set: + field: threat.feed.name + value: 'DomainTools NAD' +- set: + field: threat.feed.reference + value: https://docs.techdocs.ci.domaintools.cloud/feeds/realtime/userguide/ + + #################### + # Event ECS fields # + #################### + +- set: + field: ecs.version + value: '8.11.0' +- set: + field: event.kind + value: enrichment +- set: + field: event.category + value: ['threat'] +- set: + field: event.type + value: ['indicator'] + +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' + diff --git a/packages/ti_domaintools/data_stream/nad_feed/fields/base-fields.yml b/packages/ti_domaintools/data_stream/nad_feed/fields/base-fields.yml new file mode 100644 index 00000000000..0f3844069d4 --- /dev/null +++ b/packages/ti_domaintools/data_stream/nad_feed/fields/base-fields.yml @@ -0,0 +1,8 @@ +- name: data_stream.type + external: ecs +- name: data_stream.dataset + external: ecs +- name: data_stream.namespace + external: ecs +- name: "@timestamp" + external: ecs diff --git a/packages/ti_domaintools/data_stream/nad_feed/fields/fields.yml b/packages/ti_domaintools/data_stream/nad_feed/fields/fields.yml new file mode 100644 index 00000000000..8fa92f89ce0 --- /dev/null +++ b/packages/ti_domaintools/data_stream/nad_feed/fields/fields.yml @@ -0,0 +1,26 @@ +- name: domaintools + type: group + fields: + - name: domain + type: keyword + description: > + The Domain. Apex-level domains (e.g. example.com but not www.example.com) that we observe for the first time, and have not observed previously with our global DNS sensor network. + + - name: feed + type: keyword + description: > + The feed. + + - name: timestamp + type: date + description: > + Timestamp when the domain was added to the DomainTools feed, in ISO 8601 UTC form. + +- name: message + external: ecs + description: > + The feed. + +- name: input.type + type: keyword + description: Type of filebeat input. diff --git a/packages/ti_domaintools/data_stream/nad_feed/fields/is-ioc-transform-source-true.yml b/packages/ti_domaintools/data_stream/nad_feed/fields/is-ioc-transform-source-true.yml new file mode 100644 index 00000000000..b1d45027981 --- /dev/null +++ b/packages/ti_domaintools/data_stream/nad_feed/fields/is-ioc-transform-source-true.yml @@ -0,0 +1,4 @@ +- name: labels.is_ioc_transform_source + type: constant_keyword + value: "true" + description: Indicates whether an IOC is in the raw source data stream, or the in latest destination index. diff --git a/packages/ti_domaintools/data_stream/nad_feed/lifecycle.yml b/packages/ti_domaintools/data_stream/nad_feed/lifecycle.yml new file mode 100644 index 00000000000..5a4af9095b7 --- /dev/null +++ b/packages/ti_domaintools/data_stream/nad_feed/lifecycle.yml @@ -0,0 +1 @@ +data_retention: "5d" diff --git a/packages/ti_domaintools/data_stream/nad_feed/manifest.yml b/packages/ti_domaintools/data_stream/nad_feed/manifest.yml new file mode 100644 index 00000000000..3ab8c35b6d8 --- /dev/null +++ b/packages/ti_domaintools/data_stream/nad_feed/manifest.yml @@ -0,0 +1,58 @@ +title: "DomainTools Real Time Unified Feeds - NAD" +type: logs +ilm_policy: logs-ti_domaintools.nad_feed-default_policy +streams: + - input: cel + vars: + - name: api_url + type: text + title: DomainTools API URL + multi: false + required: true + show_user: true + default: https://api.domaintools.com/v1 + description: The URL of the DomainTools API. + - name: interval + type: text + title: Interval + multi: false + required: true + show_user: true + default: 10m + description: Interval at which the feed will be pulled. Supported units for this parameter are h/m/s. + - name: api_username + type: text + title: DomainTools API Username + multi: false + required: true + show_user: true + default: DomainTools API Username + description: DomainTools API Username + - name: api_key + type: password + title: DomainTools API Key + multi: false + required: true + show_user: true + secret: true + description: DomainTools API Key + - name: session_id + type: text + title: Session ID + multi: false + required: true + show_user: true + default: DomainToolsElasticSID + description: The Session ID to use in requesting feed. + - name: enable_request_tracer + type: bool + title: Enable request tracing + description: >- + The request tracer logs requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. Disabling the request tracer will delete any stored traces. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-cel.html#_resource_tracer_enable) for details. + default: false + multi: false + required: false + show_user: false + title: DomainTools Real Time Unified Feeds - NAD + description: Subscribe to DomainTools Real Time Unified Feeds + template_path: cel.yml.hbs diff --git a/packages/ti_domaintools/data_stream/nad_feed/sample_event.json b/packages/ti_domaintools/data_stream/nad_feed/sample_event.json new file mode 100644 index 00000000000..778f0a13e20 --- /dev/null +++ b/packages/ti_domaintools/data_stream/nad_feed/sample_event.json @@ -0,0 +1,77 @@ +{ + "input": { + "type": "cel" + }, + "agent": { + "name": "docker-fleet-agent", + "id": "da8d0a37-2d46-4788-96bd-e9ee19e332ec", + "ephemeral_id": "d1cbe648-0a1d-48e8-a161-cd82403e623e", + "type": "filebeat", + "version": "8.15.3" + }, + "@timestamp": "2025-01-30T20:15:25.396Z", + "ecs": { + "version": "8.11.0" + }, + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "ti_domaintools.nod_feed" + }, + "host": { + "hostname": "docker-fleet-agent", + "os": { + "kernel": "6.10.11-linuxkit", + "codename": "focal", + "name": "Ubuntu", + "type": "linux", + "family": "debian", + "version": "20.04.6 LTS (Focal Fossa)", + "platform": "ubuntu" + }, + "containerized": false, + "ip": [ + "172.19.0.10" + ], + "name": "docker-fleet-agent", + "id": "cfae1e7244ae479b9b0968259c91b13a", + "mac": [ + "02-42-AC-13-00-0A" + ], + "architecture": "aarch64" + }, + "elastic_agent": { + "id": "da8d0a37-2d46-4788-96bd-e9ee19e332ec", + "version": "8.15.3", + "snapshot": false + }, + "domaintools": { + "domain": "tractorpoweredcoreaerator.com", + "feed": "nad", + "timestamp": "2025-01-30T20:14:48Z" + }, + "threat": { + "indicator": { + "name": "tractorpoweredcoreaerator.com", + "type": "domain-name" + }, + "feed": { + "reference": "https://docs.techdocs.ci.domaintools.cloud/feeds/realtime/userguide/", + "name": "DomainTools NAD", + "description": "Apex-level domains (e.g. example.com but not www.example.com) that we observe for the first time, and have not observed previously with our global DNS sensor network." + } + }, + "message": "{\"timestamp\":\"2025-01-30T20:14:48Z\",\"domain\":\"tractorpoweredcoreaerator.com\"}", + "event": { + "agent_id_status": "verified", + "ingested": "2025-01-30T20:15:26Z", + "kind": "enrichment", + "category": [ + "threat" + ], + "type": [ + "indicator" + ], + "dataset": "ti_domaintools.nad_feed" + } +} \ No newline at end of file diff --git a/packages/ti_domaintools/data_stream/nod_feed/agent/stream/cel.yml.hbs b/packages/ti_domaintools/data_stream/nod_feed/agent/stream/cel.yml.hbs index caadb3970d2..1a1816ff247 100644 --- a/packages/ti_domaintools/data_stream/nod_feed/agent/stream/cel.yml.hbs +++ b/packages/ti_domaintools/data_stream/nod_feed/agent/stream/cel.yml.hbs @@ -8,7 +8,7 @@ resource.url: {{api_url}} state: api_username: {{api_username}} api_key: {{api_key}} - session_id: DomainToolsElasticSID + session_id: {{session_id}} app_name: elastic_feeds app_partner: elastic app_version: 0.1.0 @@ -33,13 +33,14 @@ program: | "Accept": ["application/x-ndjson"], }, } - ).do_request().as(resp, (resp.StatusCode == 200) ? + ).do_request().as(resp, (resp.StatusCode == 200 || resp.StatusCode == 206) ? { "events": string(resp.Body).split("\n").filter(x,x!="").map(e, { "message": e, } ), + "want_more": resp.StatusCode == 206 } : { diff --git a/packages/ti_domaintools/data_stream/nod_feed/elasticsearch/ingest_pipeline/default.yml b/packages/ti_domaintools/data_stream/nod_feed/elasticsearch/ingest_pipeline/default.yml index 8754ab49411..21c81c48e30 100644 --- a/packages/ti_domaintools/data_stream/nod_feed/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_domaintools/data_stream/nod_feed/elasticsearch/ingest_pipeline/default.yml @@ -1,13 +1,16 @@ --- description: Pipeline for processing nod feed processors: -- set: - field: domaintools.feed - value: 'nod' - json: + if: ctx?.message != null field: message target_field: domaintools +- set: + field: domaintools.feed + value: 'nod' + + ############################ # Generic indicator fields # ############################ @@ -16,6 +19,7 @@ processors: field: threat.indicator.type value: domain-name - set: + if: ctx.domaintools != null && ctx.domaintools?.domain != null field: threat.indicator.name copy_from: domaintools.domain diff --git a/packages/ti_domaintools/data_stream/nod_feed/fields/fields.yml b/packages/ti_domaintools/data_stream/nod_feed/fields/fields.yml index 494a628d94a..8fa92f89ce0 100644 --- a/packages/ti_domaintools/data_stream/nod_feed/fields/fields.yml +++ b/packages/ti_domaintools/data_stream/nod_feed/fields/fields.yml @@ -1,6 +1,5 @@ - name: domaintools type: group - fields: - name: domain type: keyword diff --git a/packages/ti_domaintools/data_stream/nod_feed/manifest.yml b/packages/ti_domaintools/data_stream/nod_feed/manifest.yml index 36f3c56a7cd..93b216d8843 100644 --- a/packages/ti_domaintools/data_stream/nod_feed/manifest.yml +++ b/packages/ti_domaintools/data_stream/nod_feed/manifest.yml @@ -1,4 +1,4 @@ -title: "DomainTools Real Time Unified Feeds" +title: "DomainTools Real Time Unified Feeds - NOD" type: logs ilm_policy: logs-ti_domaintools.nod_feed-default_policy streams: @@ -36,19 +36,23 @@ streams: show_user: true secret: true description: DomainTools API Key + - name: session_id + type: text + title: Session ID + multi: false + required: true + show_user: true + default: DomainToolsElasticSID + description: The Session ID to use in requesting feed. - name: enable_request_tracer type: bool title: Enable request tracing description: >- - The request tracer logs requests and responses to the agent's local file-system for debugging configurations. - Enabling this request tracing compromises security and should only be used for debugging. Disabling the request - tracer will delete any stored traces. - See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-cel.html#_resource_tracer_enable) - for details. + The request tracer logs requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. Disabling the request tracer will delete any stored traces. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-cel.html#_resource_tracer_enable) for details. default: false multi: false required: false show_user: false - title: DomainTools Real Time Unified Feeds + title: DomainTools Real Time Unified Feeds - NOD description: Subscribe to DomainTools Real Time Unified Feeds template_path: cel.yml.hbs diff --git a/packages/ti_domaintools/data_stream/nod_feed/sample_event.json b/packages/ti_domaintools/data_stream/nod_feed/sample_event.json index 1a93c4ba416..a96b9cfd845 100644 --- a/packages/ti_domaintools/data_stream/nod_feed/sample_event.json +++ b/packages/ti_domaintools/data_stream/nod_feed/sample_event.json @@ -47,6 +47,7 @@ }, "domaintools": { "domain": "tractorpoweredcoreaerator.com", + "feed": "nod", "timestamp": "2025-01-30T20:14:48Z" }, "threat": { diff --git a/packages/ti_domaintools/docs/README.md b/packages/ti_domaintools/docs/README.md index 39ed8315c65..9b08b05ec97 100644 --- a/packages/ti_domaintools/docs/README.md +++ b/packages/ti_domaintools/docs/README.md @@ -1,37 +1,45 @@ # DomainTools Real Time Unified Feeds -The DomainTools Real Time Unified Feeds integration allows you to monitor DomainTools Newly Observed Domains. -The DomainTools NOD Feed provides real-time access to newly registered and observed domains, enabling proactive threat detection and defense. +DomainTools Real-Time Threat Intelligence Feeds provide data on the different stages of the domain lifecycle: from first-observed in the wild, to newly re-activated after a period of quiet. Access current feed data in real-time or retrieve historical feed data through separate APIs. Some feeds also offer data for DNS firewalls in Response Policy Zone (RPZ) format. -With over 300,000 new domains observed daily, the feed empowers security teams to identify and block potentially malicious domains before they can be weaponized. -Ideal for threat hunting, phishing prevention, and brand protection, the NOD Feed delivers unparalleled visibility into emerging domain activity to stay ahead of evolving threats. +Summary of Available Feeds: -For example, if you wanted to monitor Newly Observed Domains (NOD) feed, you could ingest the DomainTools NOD feed. -Then you can reference domaintools.nod_feed when using visualizations or alerts. +- `Domain Discovery`: New domains as they are either discovered in domain registration information, observed by our global sensor network, or reported by trusted third parties. +- `Newly Active Domains (NAD)`: Apex-level domains (e.g. example.com but not ) that we observe based on the latest lifecycle of the domain. A domain may be seen either for the first time ever, or again after at least 10 days of inactivity (no observed resolutions in DNS). Populated with our global passive DNS (pDNS) sensor network. +- `Newly Observed Domains (NOD)`: Apex-level domains (e.g. example.com but not ) that we observe for the first time, and have not observed previously with our global DNS sensor network. + +With over 300,000 new domains observed daily, the feed empowers security teams to identify and block potentially malicious domains before they can be weaponized. +Ideal for threat hunting, phishing prevention, and brand protection. + +For example, if you wanted to monitor Newly Observed Domains (NOD) feed, you could ingest the DomainTools NOD feed. +Then you can reference ti_domaintools.nod_feed when using visualizations or alerts. ## Data streams -The DomainTools Real Time Unified Feeds integration collects one type of data streams: logs +The DomainTools Real Time Unified Feeds integration collects one type of data streams: **logs** -Log data streams collected by the DomainTools integration include the Newly Observed Domains (NOD) feed: Apex-level domains (e.g. Example Domain but not www.example.com) that we observe for the first time, and have not observed previously. -Populated with our global DNS sensor network. +Log data streams collected by the DomainTools integration include the following feeds: + +- `Domain Discovery` +- `Newly Observed Domains (NOD) feed` +- `Newly Active Domains (NAD)` ## Requirements -You need Elasticsearch for storing and searching your data and Kibana for visualizing and managing it. +You need Elasticsearch for storing and searching your data and Kibana for visualizing and managing it. You can use our hosted Elasticsearch Service on Elastic Cloud, which is recommended, or self-manage the Elastic Stack on your own hardware. -You will require a license to one or more DomainTools feeds, and API credentials. -Your required API credentials will vary with your authentication method, detailed below. +You will require a license to one or more DomainTools feeds, and API credentials. +Your required API credentials will vary with your authentication method, detailed below. -Obtain your API credentials from your group’s API administrator. +Obtain your API credentials from your group’s API administrator. API administrators can manage their API keys at research.domaintools.com, selecting the drop-down account menu and choosing API admin. ## Setup For step-by-step instructions on how to set up an integration, see the Getting started guide. -### Newly Observed Domains (NOD) Feed +### Newly Observed Domains (NOD) Feed The `nod_feed` data stream provides events from [DomainTools Newly Observed Domains Feed](https://www.domaintools.com/products/threat-intelligence-feeds/). This data is collected via the [DomainTools Real Time Feeds API](https://docs.domaintools.com/feeds/realtime/). @@ -90,6 +98,7 @@ An example event for `nod_feed` looks as following: }, "domaintools": { "domain": "tractorpoweredcoreaerator.com", + "feed": "nod", "timestamp": "2025-01-30T20:14:48Z" }, "threat": { @@ -135,3 +144,225 @@ An example event for `nod_feed` looks as following: | message | The feed. | match_only_text | + +### Newly Active Domains (NAD) Feed + +The `nod_feed` data stream provides events from [DomainTools Newly Active Domains Feed](https://www.domaintools.com/products/threat-intelligence-feeds/). +This data is collected via the [DomainTools Real Time Feeds API](https://docs.domaintools.com/feeds/realtime/). + +#### Example + +An example event for `nad_feed` looks as following: + +```json +{ + "input": { + "type": "cel" + }, + "agent": { + "name": "docker-fleet-agent", + "id": "da8d0a37-2d46-4788-96bd-e9ee19e332ec", + "ephemeral_id": "d1cbe648-0a1d-48e8-a161-cd82403e623e", + "type": "filebeat", + "version": "8.15.3" + }, + "@timestamp": "2025-01-30T20:15:25.396Z", + "ecs": { + "version": "8.11.0" + }, + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "ti_domaintools.nod_feed" + }, + "host": { + "hostname": "docker-fleet-agent", + "os": { + "kernel": "6.10.11-linuxkit", + "codename": "focal", + "name": "Ubuntu", + "type": "linux", + "family": "debian", + "version": "20.04.6 LTS (Focal Fossa)", + "platform": "ubuntu" + }, + "containerized": false, + "ip": [ + "172.19.0.10" + ], + "name": "docker-fleet-agent", + "id": "cfae1e7244ae479b9b0968259c91b13a", + "mac": [ + "02-42-AC-13-00-0A" + ], + "architecture": "aarch64" + }, + "elastic_agent": { + "id": "da8d0a37-2d46-4788-96bd-e9ee19e332ec", + "version": "8.15.3", + "snapshot": false + }, + "domaintools": { + "domain": "tractorpoweredcoreaerator.com", + "feed": "nad", + "timestamp": "2025-01-30T20:14:48Z" + }, + "threat": { + "indicator": { + "name": "tractorpoweredcoreaerator.com", + "type": "domain-name" + }, + "feed": { + "reference": "https://docs.techdocs.ci.domaintools.cloud/feeds/realtime/userguide/", + "name": "DomainTools NAD", + "description": "Apex-level domains (e.g. example.com but not www.example.com) that we observe for the first time, and have not observed previously with our global DNS sensor network." + } + }, + "message": "{\"timestamp\":\"2025-01-30T20:14:48Z\",\"domain\":\"tractorpoweredcoreaerator.com\"}", + "event": { + "agent_id_status": "verified", + "ingested": "2025-01-30T20:15:26Z", + "kind": "enrichment", + "category": [ + "threat" + ], + "type": [ + "indicator" + ], + "dataset": "ti_domaintools.nad_feed" + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | +| data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | +| domaintools.domain | The Domain. Apex-level domains (e.g. example.com but not www.example.com) that we observe for the first time, and have not observed previously with our global DNS sensor network. | keyword | +| domaintools.feed | The feed. | keyword | +| domaintools.timestamp | Timestamp when the domain was added to the DomainTools feed, in ISO 8601 UTC form. | date | +| input.type | Type of filebeat input. | keyword | +| labels.is_ioc_transform_source | Indicates whether an IOC is in the raw source data stream, or the in latest destination index. | constant_keyword | +| message | The feed. | match_only_text | + + +### Domain Discovery Feed + +The `domaindiscovery feed` data stream provides events from [DomainTools Domain Discovery Feed](https://www.domaintools.com/products/threat-intelligence-feeds/). +This data is collected via the [DomainTools Real Time Feeds API](https://docs.domaintools.com/feeds/realtime/). + +#### Example + +An example event for `domaindiscovery` looks as following: + +```json +{ + "input": { + "type": "cel" + }, + "agent": { + "name": "docker-fleet-agent", + "id": "da8d0a37-2d46-4788-96bd-e9ee19e332ec", + "ephemeral_id": "d1cbe648-0a1d-48e8-a161-cd82403e623e", + "type": "filebeat", + "version": "8.15.3" + }, + "@timestamp": "2025-01-30T20:15:25.396Z", + "ecs": { + "version": "8.11.0" + }, + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "ti_domaintools.nod_feed" + }, + "host": { + "hostname": "docker-fleet-agent", + "os": { + "kernel": "6.10.11-linuxkit", + "codename": "focal", + "name": "Ubuntu", + "type": "linux", + "family": "debian", + "version": "20.04.6 LTS (Focal Fossa)", + "platform": "ubuntu" + }, + "containerized": false, + "ip": [ + "172.19.0.10" + ], + "name": "docker-fleet-agent", + "id": "cfae1e7244ae479b9b0968259c91b13a", + "mac": [ + "02-42-AC-13-00-0A" + ], + "architecture": "aarch64" + }, + "elastic_agent": { + "id": "da8d0a37-2d46-4788-96bd-e9ee19e332ec", + "version": "8.15.3", + "snapshot": false + }, + "domaintools": { + "domain": "tractorpoweredcoreaerator.com", + "timestamp": "2025-01-30T20:14:48Z", + "feed": "domaindiscovery" + }, + "threat": { + "indicator": { + "name": "tractorpoweredcoreaerator.com", + "type": "domain-name" + }, + "feed": { + "reference": "https://docs.techdocs.ci.domaintools.cloud/feeds/realtime/userguide/", + "name": "DomainTools domaindiscovery", + "description": "Apex-level domains (e.g. example.com but not www.example.com) that we observe for the first time, and have not observed previously with our global DNS sensor network." + } + }, + "message": "{\"timestamp\":\"2025-01-30T20:14:48Z\",\"domain\":\"tractorpoweredcoreaerator.com\"}", + "event": { + "agent_id_status": "verified", + "ingested": "2025-01-30T20:15:26Z", + "kind": "enrichment", + "category": [ + "threat" + ], + "type": [ + "indicator" + ], + "dataset": "ti_domaintools.domaindiscovery" + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | +| data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | +| domaintools.domain | The Domain. Apex-level domains (e.g. example.com but not www.example.com) that we observe for the first time, and have not observed previously with our global DNS sensor network. | keyword | +| domaintools.feed | The feed type. | keyword | +| domaintools.timestamp | Timestamp when the domain was added to the DomainTools feed, in ISO 8601 UTC form. | date | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| error.message | Error message. | match_only_text | +| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | +| event.id | Unique ID to describe the event. | keyword | +| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | +| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | +| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | +| input.type | Type of filebeat input. | keyword | +| labels.is_ioc_transform_source | Indicates whether an IOC is in the raw source data stream, or the in latest destination index. | constant_keyword | +| message | The feed from DomainTools Feed API. | match_only_text | +| threat.feed.description | Description of the threat feed in a UI friendly format. | keyword | +| threat.feed.name | The name of the threat feed in UI friendly format. | keyword | +| threat.feed.reference | Reference information for the threat feed in a UI friendly format. | keyword | +| threat.indicator.name | The display name indicator in an UI friendly format URL, IP address, email address, registry key, port number, hash value, or other relevant name can serve as the display name. | keyword | +| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. | keyword | + diff --git a/packages/ti_domaintools/elasticsearch/transform/latest_domaindiscovery/fields/base-fields.yml b/packages/ti_domaintools/elasticsearch/transform/latest_domaindiscovery/fields/base-fields.yml new file mode 100644 index 00000000000..be7bccb662c --- /dev/null +++ b/packages/ti_domaintools/elasticsearch/transform/latest_domaindiscovery/fields/base-fields.yml @@ -0,0 +1,16 @@ +- name: data_stream.type + external: ecs +- name: data_stream.dataset + external: ecs +- name: data_stream.namespace + external: ecs +- name: event.module + type: constant_keyword + external: ecs + value: ti_domaintools +- name: event.dataset + type: constant_keyword + external: ecs + value: ti_domaintools.domaindiscovery +- name: "@timestamp" + external: ecs diff --git a/packages/ti_domaintools/elasticsearch/transform/latest_domaindiscovery/fields/beats.yml b/packages/ti_domaintools/elasticsearch/transform/latest_domaindiscovery/fields/beats.yml new file mode 100644 index 00000000000..b13d5cc96f4 --- /dev/null +++ b/packages/ti_domaintools/elasticsearch/transform/latest_domaindiscovery/fields/beats.yml @@ -0,0 +1,11 @@ +- name: input.type + type: keyword + description: Type of Filebeat input. +- name: log.flags + type: keyword + description: Flags for the log file. +- name: log.offset + type: long + description: Offset of the entry in the log file. +- name: log.file.path + external: ecs diff --git a/packages/ti_domaintools/elasticsearch/transform/latest_domaindiscovery/fields/ecs.yml b/packages/ti_domaintools/elasticsearch/transform/latest_domaindiscovery/fields/ecs.yml new file mode 100644 index 00000000000..664ff10ce6a --- /dev/null +++ b/packages/ti_domaintools/elasticsearch/transform/latest_domaindiscovery/fields/ecs.yml @@ -0,0 +1,24 @@ +- name: ecs.version + external: ecs +- name: error.message + external: ecs +- name: event.category + external: ecs +- name: event.id + external: ecs +- name: event.ingested + external: ecs +- name: event.kind + external: ecs +- name: event.type + external: ecs +- name: threat.indicator.type + external: ecs +- name: threat.indicator.name + external: ecs +- name: threat.feed.description + external: ecs +- name: threat.feed.name + external: ecs +- name: threat.feed.reference + external: ecs diff --git a/packages/ti_domaintools/elasticsearch/transform/latest_domaindiscovery/fields/fields.yml b/packages/ti_domaintools/elasticsearch/transform/latest_domaindiscovery/fields/fields.yml new file mode 100644 index 00000000000..eb84d086d13 --- /dev/null +++ b/packages/ti_domaintools/elasticsearch/transform/latest_domaindiscovery/fields/fields.yml @@ -0,0 +1,22 @@ +- name: domaintools + type: group + fields: + - name: domain + type: keyword + description: > + The Domain. Apex-level domains (e.g. example.com but not www.example.com) that we observe for the first time, and have not observed previously with our global DNS sensor network. + + - name: feed + type: keyword + description: > + The feed. + + - name: timestamp + type: date + description: > + Timestamp when the domain was added to the DomainTools feed, in ISO 8601 UTC form. + +- name: message + external: ecs + description: >- + The feed. diff --git a/packages/ti_domaintools/elasticsearch/transform/latest_domaindiscovery/fields/is-ioc-transform-source-false.yml b/packages/ti_domaintools/elasticsearch/transform/latest_domaindiscovery/fields/is-ioc-transform-source-false.yml new file mode 100644 index 00000000000..15524f7a6cc --- /dev/null +++ b/packages/ti_domaintools/elasticsearch/transform/latest_domaindiscovery/fields/is-ioc-transform-source-false.yml @@ -0,0 +1,4 @@ +- name: labels.is_ioc_transform_source + type: constant_keyword + value: "false" + description: Indicates whether an IOC is in the raw source data stream, or the in latest destination index. diff --git a/packages/ti_domaintools/elasticsearch/transform/latest_domaindiscovery/manifest.yml b/packages/ti_domaintools/elasticsearch/transform/latest_domaindiscovery/manifest.yml new file mode 100644 index 00000000000..f5296fd0c0a --- /dev/null +++ b/packages/ti_domaintools/elasticsearch/transform/latest_domaindiscovery/manifest.yml @@ -0,0 +1,18 @@ +start: true +destination_index_template: + settings: + index: + sort: + field: + - "@timestamp" + order: + - desc + mappings: + dynamic: true + dynamic_templates: + - strings_as_keyword: + match_mapping_type: string + mapping: + ignore_above: 1024 + type: keyword + date_detection: false diff --git a/packages/ti_domaintools/elasticsearch/transform/latest_domaindiscovery/transform.yml b/packages/ti_domaintools/elasticsearch/transform/latest_domaindiscovery/transform.yml new file mode 100644 index 00000000000..db6a0df9d95 --- /dev/null +++ b/packages/ti_domaintools/elasticsearch/transform/latest_domaindiscovery/transform.yml @@ -0,0 +1,35 @@ +# Use of "*" to use all namespaces defined. +source: + index: + - "logs-ti_domaintools.domaindiscovery_feed-*" +# The version suffix on the dest.index should be incremented if a breaking change +# is made to the index mapping. You must also bump the fleet_transform_version +# for any change to this transform configuration to take effect. The old destination +# index is not automatically removed. We are dependent on https://github.com/elastic/package-spec/issues/523 to give +# us that ability in order to prevent having duplicate IoC data and prevent query +# time field type conflicts. +dest: + index: "logs-ti_domaintools_latest.domaindiscovery_feed-2" + aliases: + - alias: "logs-ti_domaintools_latest.domaindiscovery_feed" + move_on_creation: true +latest: + unique_key: + - domaintools.domain + sort: event.ingested +description: Latest Newly Domain discovery data +frequency: 30s +sync: + time: + field: event.ingested + # Updated to 120s because of refresh delay in Serverless. With default 60s, sometimes transform wouldn't process all documents. + delay: 120s +retention_policy: + time: + field: event.ingested + max_age: 7d +_meta: + managed: true + # Bump this version to delete, reinstall, and restart the transform during package. + # Version bump is needed if there is any code change in transform. + fleet_transform_version: 0.2.0 diff --git a/packages/ti_domaintools/elasticsearch/transform/latest_nad/fields/base-fields.yml b/packages/ti_domaintools/elasticsearch/transform/latest_nad/fields/base-fields.yml new file mode 100644 index 00000000000..a2bcd004393 --- /dev/null +++ b/packages/ti_domaintools/elasticsearch/transform/latest_nad/fields/base-fields.yml @@ -0,0 +1,16 @@ +- name: data_stream.type + external: ecs +- name: data_stream.dataset + external: ecs +- name: data_stream.namespace + external: ecs +- name: event.module + type: constant_keyword + external: ecs + value: ti_domaintools +- name: event.dataset + type: constant_keyword + external: ecs + value: ti_domaintools.nad_feed +- name: "@timestamp" + external: ecs diff --git a/packages/ti_domaintools/elasticsearch/transform/latest_nad/fields/beats.yml b/packages/ti_domaintools/elasticsearch/transform/latest_nad/fields/beats.yml new file mode 100644 index 00000000000..b13d5cc96f4 --- /dev/null +++ b/packages/ti_domaintools/elasticsearch/transform/latest_nad/fields/beats.yml @@ -0,0 +1,11 @@ +- name: input.type + type: keyword + description: Type of Filebeat input. +- name: log.flags + type: keyword + description: Flags for the log file. +- name: log.offset + type: long + description: Offset of the entry in the log file. +- name: log.file.path + external: ecs diff --git a/packages/ti_domaintools/elasticsearch/transform/latest_nad/fields/ecs.yml b/packages/ti_domaintools/elasticsearch/transform/latest_nad/fields/ecs.yml new file mode 100644 index 00000000000..664ff10ce6a --- /dev/null +++ b/packages/ti_domaintools/elasticsearch/transform/latest_nad/fields/ecs.yml @@ -0,0 +1,24 @@ +- name: ecs.version + external: ecs +- name: error.message + external: ecs +- name: event.category + external: ecs +- name: event.id + external: ecs +- name: event.ingested + external: ecs +- name: event.kind + external: ecs +- name: event.type + external: ecs +- name: threat.indicator.type + external: ecs +- name: threat.indicator.name + external: ecs +- name: threat.feed.description + external: ecs +- name: threat.feed.name + external: ecs +- name: threat.feed.reference + external: ecs diff --git a/packages/ti_domaintools/elasticsearch/transform/latest_nad/fields/fields.yml b/packages/ti_domaintools/elasticsearch/transform/latest_nad/fields/fields.yml new file mode 100644 index 00000000000..eb84d086d13 --- /dev/null +++ b/packages/ti_domaintools/elasticsearch/transform/latest_nad/fields/fields.yml @@ -0,0 +1,22 @@ +- name: domaintools + type: group + fields: + - name: domain + type: keyword + description: > + The Domain. Apex-level domains (e.g. example.com but not www.example.com) that we observe for the first time, and have not observed previously with our global DNS sensor network. + + - name: feed + type: keyword + description: > + The feed. + + - name: timestamp + type: date + description: > + Timestamp when the domain was added to the DomainTools feed, in ISO 8601 UTC form. + +- name: message + external: ecs + description: >- + The feed. diff --git a/packages/ti_domaintools/elasticsearch/transform/latest_nad/fields/is-ioc-transform-source-false.yml b/packages/ti_domaintools/elasticsearch/transform/latest_nad/fields/is-ioc-transform-source-false.yml new file mode 100644 index 00000000000..15524f7a6cc --- /dev/null +++ b/packages/ti_domaintools/elasticsearch/transform/latest_nad/fields/is-ioc-transform-source-false.yml @@ -0,0 +1,4 @@ +- name: labels.is_ioc_transform_source + type: constant_keyword + value: "false" + description: Indicates whether an IOC is in the raw source data stream, or the in latest destination index. diff --git a/packages/ti_domaintools/elasticsearch/transform/latest_nad/manifest.yml b/packages/ti_domaintools/elasticsearch/transform/latest_nad/manifest.yml new file mode 100644 index 00000000000..f5296fd0c0a --- /dev/null +++ b/packages/ti_domaintools/elasticsearch/transform/latest_nad/manifest.yml @@ -0,0 +1,18 @@ +start: true +destination_index_template: + settings: + index: + sort: + field: + - "@timestamp" + order: + - desc + mappings: + dynamic: true + dynamic_templates: + - strings_as_keyword: + match_mapping_type: string + mapping: + ignore_above: 1024 + type: keyword + date_detection: false diff --git a/packages/ti_domaintools/elasticsearch/transform/latest_nad/transform.yml b/packages/ti_domaintools/elasticsearch/transform/latest_nad/transform.yml new file mode 100644 index 00000000000..e8b8d382bac --- /dev/null +++ b/packages/ti_domaintools/elasticsearch/transform/latest_nad/transform.yml @@ -0,0 +1,35 @@ +# Use of "*" to use all namespaces defined. +source: + index: + - "logs-ti_domaintools.nad_feed-*" +# The version suffix on the dest.index should be incremented if a breaking change +# is made to the index mapping. You must also bump the fleet_transform_version +# for any change to this transform configuration to take effect. The old destination +# index is not automatically removed. We are dependent on https://github.com/elastic/package-spec/issues/523 to give +# us that ability in order to prevent having duplicate IoC data and prevent query +# time field type conflicts. +dest: + index: "logs-ti_domaintools_latest.nad_feed-2" + aliases: + - alias: "logs-ti_domaintools_latest.nad_feed" + move_on_creation: true +latest: + unique_key: + - domaintools.domain + sort: event.ingested +description: Latest Newly Active Domains data +frequency: 30s +sync: + time: + field: event.ingested + # Updated to 120s because of refresh delay in Serverless. With default 60s, sometimes transform wouldn't process all documents. + delay: 120s +retention_policy: + time: + field: event.ingested + max_age: 7d +_meta: + managed: true + # Bump this version to delete, reinstall, and restart the transform during package. + # Version bump is needed if there is any code change in transform. + fleet_transform_version: 0.2.0 diff --git a/packages/ti_domaintools/elasticsearch/transform/latest_nod/fields/fields.yml b/packages/ti_domaintools/elasticsearch/transform/latest_nod/fields/fields.yml index 515775b2f2c..eb84d086d13 100644 --- a/packages/ti_domaintools/elasticsearch/transform/latest_nod/fields/fields.yml +++ b/packages/ti_domaintools/elasticsearch/transform/latest_nod/fields/fields.yml @@ -1,6 +1,5 @@ - name: domaintools type: group - fields: - name: domain type: keyword @@ -19,5 +18,5 @@ - name: message external: ecs - description: > - The feed. \ No newline at end of file + description: >- + The feed. diff --git a/packages/ti_domaintools/manifest.yml b/packages/ti_domaintools/manifest.yml index 4c534191aef..6b162184c94 100644 --- a/packages/ti_domaintools/manifest.yml +++ b/packages/ti_domaintools/manifest.yml @@ -4,7 +4,7 @@ title: "DomainTools Real Time Unified Feeds" version: "0.3.2" source: license: "Elastic-2.0" -description: "The DomainTools NOD Feed provides real-time access to newly registered and observed domains, enabling proactive threat detection and defense." +description: "DomainTools Real-Time Threat Intelligence Feeds provide data on the different stages of the domain lifecycle: from first-observed in the wild, to newly re-activated after a period of quiet." type: integration categories: - security @@ -21,12 +21,12 @@ icons: type: image/svg+xml policy_templates: - name: domaintools - title: DomainTools Newly Observed Domains - description: "DomainTools Newly Observed Domains provides real-time access to newly registered and observed domains, enabling proactive threat detection and defense" + title: DomainTools Real Time Unified Feeds + description: "The DomainTools Feed provides real-time access to newly registered and observed domains, enabling proactive threat detection and defense." inputs: - type: cel - title: "DomainTools Newly Observed Domains" - description: "DomainTools Newly Observed Domains provides real-time access to newly registered and observed domains, enabling proactive threat detection and defense" + title: "Collect DomainTools Real Time Unified Feeds" + description: "The DomainTools Feed provides real-time access to newly registered and observed domains, enabling proactive threat detection and defense." owner: github: elastic/security-service-integrations type: partner From 4fd3a7771b27fd8751535b91aaa15141e70410ee Mon Sep 17 00:00:00 2001 From: bluza Date: Tue, 20 May 2025 21:51:46 +0800 Subject: [PATCH 2/4] Update the app naming to DomainTools Feeds --- packages/ti_domaintools/_dev/build/docs/README.md | 13 ++++++------- .../data_stream/domaindiscovery/manifest.yml | 6 +++--- .../data_stream/nad_feed/manifest.yml | 6 +++--- .../data_stream/nod_feed/manifest.yml | 6 +++--- packages/ti_domaintools/docs/README.md | 13 ++++++------- packages/ti_domaintools/manifest.yml | 8 ++++---- 6 files changed, 25 insertions(+), 27 deletions(-) diff --git a/packages/ti_domaintools/_dev/build/docs/README.md b/packages/ti_domaintools/_dev/build/docs/README.md index 81fe0083dfd..31c9d35a187 100644 --- a/packages/ti_domaintools/_dev/build/docs/README.md +++ b/packages/ti_domaintools/_dev/build/docs/README.md @@ -1,6 +1,6 @@ -# DomainTools Real Time Unified Feeds +# DomainTools Feeds -DomainTools Real-Time Threat Intelligence Feeds provide data on the different stages of the domain lifecycle: from first-observed in the wild, to newly re-activated after a period of quiet. Access current feed data in real-time or retrieve historical feed data through separate APIs. Some feeds also offer data for DNS firewalls in Response Policy Zone (RPZ) format. +DomainTools Feeds provide data on the different stages of the domain lifecycle: from first-observed in the wild, to newly re-activated after a period of quiet. Access current feed data in real-time or retrieve historical feed data through separate APIs. Some feeds also offer data for DNS firewalls in Response Policy Zone (RPZ) format. Summary of Available Feeds: @@ -16,7 +16,7 @@ Then you can reference ti_domaintools.nod_feed when using visualizations or aler ## Data streams -The DomainTools Real Time Unified Feeds integration collects one type of data streams: **logs** +The DomainTools Feeds integration collects one type of data streams: **logs** Log data streams collected by the DomainTools integration include the following feeds: @@ -42,7 +42,7 @@ For step-by-step instructions on how to set up an integration, see the Getting s ### Newly Observed Domains (NOD) Feed The `nod_feed` data stream provides events from [DomainTools Newly Observed Domains Feed](https://www.domaintools.com/products/threat-intelligence-feeds/). -This data is collected via the [DomainTools Real Time Feeds API](https://docs.domaintools.com/feeds/realtime/). +This data is collected via the [DomainTools Feeds API](https://docs.domaintools.com/feeds/realtime/). #### Example @@ -50,11 +50,10 @@ This data is collected via the [DomainTools Real Time Feeds API](https://docs.do {{fields "nod_feed"}} - ### Newly Active Domains (NAD) Feed The `nod_feed` data stream provides events from [DomainTools Newly Active Domains Feed](https://www.domaintools.com/products/threat-intelligence-feeds/). -This data is collected via the [DomainTools Real Time Feeds API](https://docs.domaintools.com/feeds/realtime/). +This data is collected via the [DomainTools Feeds API](https://docs.domaintools.com/feeds/realtime/). #### Example @@ -65,7 +64,7 @@ This data is collected via the [DomainTools Real Time Feeds API](https://docs.do ### Domain Discovery Feed The `domaindiscovery feed` data stream provides events from [DomainTools Domain Discovery Feed](https://www.domaintools.com/products/threat-intelligence-feeds/). -This data is collected via the [DomainTools Real Time Feeds API](https://docs.domaintools.com/feeds/realtime/). +This data is collected via the [DomainTools Feeds API](https://docs.domaintools.com/feeds/realtime/). #### Example diff --git a/packages/ti_domaintools/data_stream/domaindiscovery/manifest.yml b/packages/ti_domaintools/data_stream/domaindiscovery/manifest.yml index 170b519db3a..dd835aea6ce 100644 --- a/packages/ti_domaintools/data_stream/domaindiscovery/manifest.yml +++ b/packages/ti_domaintools/data_stream/domaindiscovery/manifest.yml @@ -1,4 +1,4 @@ -title: "DomainTools Real Time Unified Feeds - domaindiscovery" +title: "DomainTools Domain Discovery Feeds" type: logs ilm_policy: logs-ti_domaintools.domaindiscovery-default_policy streams: @@ -53,6 +53,6 @@ streams: multi: false required: false show_user: false - title: DomainTools Real Time Unified Feeds - domaindiscovery - description: Subscribe to DomainTools Real Time Unified Feeds + title: DomainTools Domain Discovery Feeds + description: Subscribe to DomainTools Domain Discovery Feeds template_path: cel.yml.hbs diff --git a/packages/ti_domaintools/data_stream/nad_feed/manifest.yml b/packages/ti_domaintools/data_stream/nad_feed/manifest.yml index 3ab8c35b6d8..f5fe84218b6 100644 --- a/packages/ti_domaintools/data_stream/nad_feed/manifest.yml +++ b/packages/ti_domaintools/data_stream/nad_feed/manifest.yml @@ -1,4 +1,4 @@ -title: "DomainTools Real Time Unified Feeds - NAD" +title: "DomainTools Newly Active Domains Feeds" type: logs ilm_policy: logs-ti_domaintools.nad_feed-default_policy streams: @@ -53,6 +53,6 @@ streams: multi: false required: false show_user: false - title: DomainTools Real Time Unified Feeds - NAD - description: Subscribe to DomainTools Real Time Unified Feeds + title: DomainTools Newly Active Domains Feeds + description: Subscribe to DomainTools Newly Active Domains Feeds template_path: cel.yml.hbs diff --git a/packages/ti_domaintools/data_stream/nod_feed/manifest.yml b/packages/ti_domaintools/data_stream/nod_feed/manifest.yml index 93b216d8843..e82726b2468 100644 --- a/packages/ti_domaintools/data_stream/nod_feed/manifest.yml +++ b/packages/ti_domaintools/data_stream/nod_feed/manifest.yml @@ -1,4 +1,4 @@ -title: "DomainTools Real Time Unified Feeds - NOD" +title: "DomainTools Newly Observed Domains Feeds" type: logs ilm_policy: logs-ti_domaintools.nod_feed-default_policy streams: @@ -53,6 +53,6 @@ streams: multi: false required: false show_user: false - title: DomainTools Real Time Unified Feeds - NOD - description: Subscribe to DomainTools Real Time Unified Feeds + title: DomainTools Newly Observed Domains Feeds + description: Subscribe to DomainTools Newly Observed Domains Feeds template_path: cel.yml.hbs diff --git a/packages/ti_domaintools/docs/README.md b/packages/ti_domaintools/docs/README.md index 9b08b05ec97..378b2811198 100644 --- a/packages/ti_domaintools/docs/README.md +++ b/packages/ti_domaintools/docs/README.md @@ -1,6 +1,6 @@ -# DomainTools Real Time Unified Feeds +# DomainTools Feeds -DomainTools Real-Time Threat Intelligence Feeds provide data on the different stages of the domain lifecycle: from first-observed in the wild, to newly re-activated after a period of quiet. Access current feed data in real-time or retrieve historical feed data through separate APIs. Some feeds also offer data for DNS firewalls in Response Policy Zone (RPZ) format. +DomainTools Feeds provide data on the different stages of the domain lifecycle: from first-observed in the wild, to newly re-activated after a period of quiet. Access current feed data in real-time or retrieve historical feed data through separate APIs. Some feeds also offer data for DNS firewalls in Response Policy Zone (RPZ) format. Summary of Available Feeds: @@ -16,7 +16,7 @@ Then you can reference ti_domaintools.nod_feed when using visualizations or aler ## Data streams -The DomainTools Real Time Unified Feeds integration collects one type of data streams: **logs** +The DomainTools Feeds integration collects one type of data streams: **logs** Log data streams collected by the DomainTools integration include the following feeds: @@ -42,7 +42,7 @@ For step-by-step instructions on how to set up an integration, see the Getting s ### Newly Observed Domains (NOD) Feed The `nod_feed` data stream provides events from [DomainTools Newly Observed Domains Feed](https://www.domaintools.com/products/threat-intelligence-feeds/). -This data is collected via the [DomainTools Real Time Feeds API](https://docs.domaintools.com/feeds/realtime/). +This data is collected via the [DomainTools Feeds API](https://docs.domaintools.com/feeds/realtime/). #### Example @@ -144,11 +144,10 @@ An example event for `nod_feed` looks as following: | message | The feed. | match_only_text | - ### Newly Active Domains (NAD) Feed The `nod_feed` data stream provides events from [DomainTools Newly Active Domains Feed](https://www.domaintools.com/products/threat-intelligence-feeds/). -This data is collected via the [DomainTools Real Time Feeds API](https://docs.domaintools.com/feeds/realtime/). +This data is collected via the [DomainTools Feeds API](https://docs.domaintools.com/feeds/realtime/). #### Example @@ -253,7 +252,7 @@ An example event for `nad_feed` looks as following: ### Domain Discovery Feed The `domaindiscovery feed` data stream provides events from [DomainTools Domain Discovery Feed](https://www.domaintools.com/products/threat-intelligence-feeds/). -This data is collected via the [DomainTools Real Time Feeds API](https://docs.domaintools.com/feeds/realtime/). +This data is collected via the [DomainTools Feeds API](https://docs.domaintools.com/feeds/realtime/). #### Example diff --git a/packages/ti_domaintools/manifest.yml b/packages/ti_domaintools/manifest.yml index 6b162184c94..fe7db836dcd 100644 --- a/packages/ti_domaintools/manifest.yml +++ b/packages/ti_domaintools/manifest.yml @@ -1,10 +1,10 @@ format_version: 3.3.0 name: ti_domaintools -title: "DomainTools Real Time Unified Feeds" +title: "DomainTools Feeds" version: "0.3.2" source: license: "Elastic-2.0" -description: "DomainTools Real-Time Threat Intelligence Feeds provide data on the different stages of the domain lifecycle: from first-observed in the wild, to newly re-activated after a period of quiet." +description: "DomainTools Feeds provide data on the different stages of the domain lifecycle: from first-observed in the wild, to newly re-activated after a period of quiet." type: integration categories: - security @@ -21,11 +21,11 @@ icons: type: image/svg+xml policy_templates: - name: domaintools - title: DomainTools Real Time Unified Feeds + title: DomainTools Feeds description: "The DomainTools Feed provides real-time access to newly registered and observed domains, enabling proactive threat detection and defense." inputs: - type: cel - title: "Collect DomainTools Real Time Unified Feeds" + title: "Collect DomainTools Feeds" description: "The DomainTools Feed provides real-time access to newly registered and observed domains, enabling proactive threat detection and defense." owner: github: elastic/security-service-integrations From b58446c874567d179b29ceab6779c6b869babd86 Mon Sep 17 00:00:00 2001 From: bluza Date: Tue, 20 May 2025 21:59:25 +0800 Subject: [PATCH 3/4] Update manifest file --- .../ti_domaintools/data_stream/domaindiscovery/manifest.yml | 6 +++--- packages/ti_domaintools/data_stream/nad_feed/manifest.yml | 6 +++--- packages/ti_domaintools/data_stream/nod_feed/manifest.yml | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/ti_domaintools/data_stream/domaindiscovery/manifest.yml b/packages/ti_domaintools/data_stream/domaindiscovery/manifest.yml index dd835aea6ce..094e755c2dd 100644 --- a/packages/ti_domaintools/data_stream/domaindiscovery/manifest.yml +++ b/packages/ti_domaintools/data_stream/domaindiscovery/manifest.yml @@ -1,4 +1,4 @@ -title: "DomainTools Domain Discovery Feeds" +title: "DomainTools Domain Discovery Feed" type: logs ilm_policy: logs-ti_domaintools.domaindiscovery-default_policy streams: @@ -53,6 +53,6 @@ streams: multi: false required: false show_user: false - title: DomainTools Domain Discovery Feeds - description: Subscribe to DomainTools Domain Discovery Feeds + title: DomainTools Domain Discovery Feed + description: Subscribe to DomainTools Domain Discovery Feed template_path: cel.yml.hbs diff --git a/packages/ti_domaintools/data_stream/nad_feed/manifest.yml b/packages/ti_domaintools/data_stream/nad_feed/manifest.yml index f5fe84218b6..346421d9116 100644 --- a/packages/ti_domaintools/data_stream/nad_feed/manifest.yml +++ b/packages/ti_domaintools/data_stream/nad_feed/manifest.yml @@ -1,4 +1,4 @@ -title: "DomainTools Newly Active Domains Feeds" +title: "DomainTools Newly Active Domains Feed" type: logs ilm_policy: logs-ti_domaintools.nad_feed-default_policy streams: @@ -53,6 +53,6 @@ streams: multi: false required: false show_user: false - title: DomainTools Newly Active Domains Feeds - description: Subscribe to DomainTools Newly Active Domains Feeds + title: DomainTools Newly Active Domains Feed + description: Subscribe to DomainTools Newly Active Domains Feed template_path: cel.yml.hbs diff --git a/packages/ti_domaintools/data_stream/nod_feed/manifest.yml b/packages/ti_domaintools/data_stream/nod_feed/manifest.yml index e82726b2468..baf13d2cf2c 100644 --- a/packages/ti_domaintools/data_stream/nod_feed/manifest.yml +++ b/packages/ti_domaintools/data_stream/nod_feed/manifest.yml @@ -1,4 +1,4 @@ -title: "DomainTools Newly Observed Domains Feeds" +title: "DomainTools Newly Observed Domains Feed" type: logs ilm_policy: logs-ti_domaintools.nod_feed-default_policy streams: @@ -53,6 +53,6 @@ streams: multi: false required: false show_user: false - title: DomainTools Newly Observed Domains Feeds - description: Subscribe to DomainTools Newly Observed Domains Feeds + title: DomainTools Newly Observed Domains Feed + description: Subscribe to DomainTools Newly Observed Domains Feed template_path: cel.yml.hbs From a2d34e8e963c510f41d2a0aa6590032d4d077a54 Mon Sep 17 00:00:00 2001 From: bluza Date: Tue, 20 May 2025 22:01:49 +0800 Subject: [PATCH 4/4] Update README --- packages/ti_domaintools/_dev/build/docs/README.md | 2 +- packages/ti_domaintools/docs/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/ti_domaintools/_dev/build/docs/README.md b/packages/ti_domaintools/_dev/build/docs/README.md index 31c9d35a187..5faa3e17554 100644 --- a/packages/ti_domaintools/_dev/build/docs/README.md +++ b/packages/ti_domaintools/_dev/build/docs/README.md @@ -21,7 +21,7 @@ The DomainTools Feeds integration collects one type of data streams: **logs** Log data streams collected by the DomainTools integration include the following feeds: - `Domain Discovery` -- `Newly Observed Domains (NOD) feed` +- `Newly Observed Domains (NOD)` - `Newly Active Domains (NAD)` ## Requirements diff --git a/packages/ti_domaintools/docs/README.md b/packages/ti_domaintools/docs/README.md index 378b2811198..709bcc485bc 100644 --- a/packages/ti_domaintools/docs/README.md +++ b/packages/ti_domaintools/docs/README.md @@ -21,7 +21,7 @@ The DomainTools Feeds integration collects one type of data streams: **logs** Log data streams collected by the DomainTools integration include the following feeds: - `Domain Discovery` -- `Newly Observed Domains (NOD) feed` +- `Newly Observed Domains (NOD)` - `Newly Active Domains (NAD)` ## Requirements