diff --git a/.generator/schemas/v1/openapi.yaml b/.generator/schemas/v1/openapi.yaml index 4ce7a3163783..bc9ff110b078 100644 --- a/.generator/schemas/v1/openapi.yaml +++ b/.generator/schemas/v1/openapi.yaml @@ -5163,6 +5163,143 @@ components: type: string x-enum-varnames: - ARITHMETIC_PROCESSOR + LogsArrayProcessor: + description: 'A processor for extracting, aggregating, or transforming values + from JSON arrays within your logs. + + Supported operations are: + + - Select value from matching element + + - Compute array length + + - Append a value to an array' + properties: + is_enabled: + default: false + description: Whether or not the processor is enabled. + type: boolean + name: + description: Name of the processor. + type: string + operation: + $ref: '#/components/schemas/LogsArrayProcessorOperation' + type: + $ref: '#/components/schemas/LogsArrayProcessorType' + required: + - operation + - type + type: object + LogsArrayProcessorOperation: + description: Configuration of the array processor operation to perform. + oneOf: + - $ref: '#/components/schemas/LogsArrayProcessorOperationAppend' + - $ref: '#/components/schemas/LogsArrayProcessorOperationLength' + - $ref: '#/components/schemas/LogsArrayProcessorOperationSelect' + LogsArrayProcessorOperationAppend: + description: Operation that appends a value to a target array attribute. + properties: + preserve_source: + default: true + description: Remove or preserve the remapped source element. + type: boolean + source: + description: Attribute path containing the value to append. + example: network.client.ip + type: string + target: + description: Attribute path of the array to append to. + example: sourceIps + type: string + type: + $ref: '#/components/schemas/LogsArrayProcessorOperationAppendType' + required: + - type + - source + - target + type: object + LogsArrayProcessorOperationAppendType: + description: Operation type. + enum: + - append + example: append + type: string + x-enum-varnames: + - APPEND + LogsArrayProcessorOperationLength: + description: Operation that computes the length of a `source` array and stores + the result in the `target` attribute. + properties: + source: + description: Attribute path of the array to measure. + example: tags + type: string + target: + description: Attribute that receives the computed length. + example: tagCount + type: string + type: + $ref: '#/components/schemas/LogsArrayProcessorOperationLengthType' + required: + - type + - source + - target + type: object + LogsArrayProcessorOperationLengthType: + description: Operation type. + enum: + - length + example: length + type: string + x-enum-varnames: + - LENGTH + LogsArrayProcessorOperationSelect: + description: Operation that finds an object in a `source` array using a `filter`, + and then extracts a specific value into the `target` attribute. + properties: + filter: + description: Filter condition expressed as `key:value` used to find the + matching element. + example: name:Referrer + type: string + source: + description: Attribute path of the array to search into. + example: httpRequest.headers + type: string + target: + description: Attribute that receives the extracted value. + example: referrer + type: string + type: + $ref: '#/components/schemas/LogsArrayProcessorOperationSelectType' + value_to_extract: + description: Key of the value to extract from the matching element. + example: value + type: string + required: + - type + - source + - target + - filter + - value_to_extract + type: object + LogsArrayProcessorOperationSelectType: + description: Operation type. + enum: + - select + example: select + type: string + x-enum-varnames: + - SELECT + LogsArrayProcessorType: + default: array-processor + description: Type of logs array processor. + enum: + - array-processor + example: array-processor + type: string + x-enum-varnames: + - ARRAY_PROCESSOR LogsAttributeRemapper: description: 'The remapper processor remaps any source attribute(s) or tag to another target attribute or tag. @@ -6060,6 +6197,7 @@ components: - $ref: '#/components/schemas/ReferenceTableLogsLookupProcessor' - $ref: '#/components/schemas/LogsTraceRemapper' - $ref: '#/components/schemas/LogsSpanRemapper' + - $ref: '#/components/schemas/LogsArrayProcessor' LogsQueryCompute: description: Define computation for a log query. properties: diff --git a/cassettes/v1/Logs-Pipelines_3794283453/Create-a-pipeline-with-Array-Processor-Append-Operation-returns-OK-response_2598677476/frozen.json b/cassettes/v1/Logs-Pipelines_3794283453/Create-a-pipeline-with-Array-Processor-Append-Operation-returns-OK-response_2598677476/frozen.json new file mode 100644 index 000000000000..d4c5c734cade --- /dev/null +++ b/cassettes/v1/Logs-Pipelines_3794283453/Create-a-pipeline-with-Array-Processor-Append-Operation-returns-OK-response_2598677476/frozen.json @@ -0,0 +1 @@ +"2025-06-30T15:45:40.994Z" diff --git a/cassettes/v1/Logs-Pipelines_3794283453/Create-a-pipeline-with-Array-Processor-Append-Operation-returns-OK-response_2598677476/recording.har b/cassettes/v1/Logs-Pipelines_3794283453/Create-a-pipeline-with-Array-Processor-Append-Operation-returns-OK-response_2598677476/recording.har new file mode 100644 index 000000000000..813884543239 --- /dev/null +++ b/cassettes/v1/Logs-Pipelines_3794283453/Create-a-pipeline-with-Array-Processor-Append-Operation-returns-OK-response_2598677476/recording.har @@ -0,0 +1,110 @@ +{ + "log": { + "_recordingName": "Logs Pipelines/Create a pipeline with Array Processor Append Operation returns \"OK\" response", + "creator": { + "comment": "persister:fs", + "name": "Polly.JS", + "version": "6.0.5" + }, + "entries": [ + { + "_id": "69ee9de755f2e0b61cbbe11dbe56dc06", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 247, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "accept", + "value": "application/json" + }, + { + "_fromType": "array", + "name": "content-type", + "value": "application/json" + } + ], + "headersSize": 574, + "httpVersion": "HTTP/1.1", + "method": "POST", + "postData": { + "mimeType": "application/json", + "params": [], + "text": "{\"filter\":{\"query\":\"source:python\"},\"name\":\"testPipelineArrayAppend\",\"processors\":[{\"is_enabled\":true,\"name\":\"append_ip_to_array\",\"operation\":{\"source\":\"network.client.ip\",\"target\":\"sourceIps\",\"type\":\"append\"},\"type\":\"array-processor\"}],\"tags\":[]}" + }, + "queryString": [], + "url": "https://api.datadoghq.com/api/v1/logs/config/pipelines" + }, + "response": { + "bodySize": 359, + "content": { + "mimeType": "application/json", + "size": 359, + "text": "{\"id\":\"s_cPqdnkQVaU6PwbPPt2ZQ\",\"type\":\"pipeline\",\"name\":\"testPipelineArrayAppend\",\"is_enabled\":false,\"is_read_only\":false,\"filter\":{\"query\":\"source:python\"},\"processors\":[{\"name\":\"append_ip_to_array\",\"is_enabled\":true,\"operation\":{\"source\":\"network.client.ip\",\"target\":\"sourceIps\",\"preserve_source\":true,\"type\":\"append\"},\"type\":\"array-processor\"}],\"tags\":[]}\n" + }, + "cookies": [], + "headers": [ + { + "name": "content-type", + "value": "application/json" + } + ], + "headersSize": 702, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2025-06-30T15:45:41.001Z", + "time": 398 + }, + { + "_id": "2d231c72b566a1746dfa7066c5e6b025", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "accept", + "value": "*/*" + } + ], + "headersSize": 533, + "httpVersion": "HTTP/1.1", + "method": "DELETE", + "queryString": [], + "url": "https://api.datadoghq.com/api/v1/logs/config/pipelines/s_cPqdnkQVaU6PwbPPt2ZQ" + }, + "response": { + "bodySize": 3, + "content": { + "mimeType": "application/json", + "size": 3, + "text": "{}\n" + }, + "cookies": [], + "headers": [ + { + "name": "content-type", + "value": "application/json" + } + ], + "headersSize": 677, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2025-06-30T15:45:41.406Z", + "time": 432 + } + ], + "pages": [], + "version": "1.2" + } +} diff --git a/cassettes/v1/Logs-Pipelines_3794283453/Create-a-pipeline-with-Array-Processor-Append-Operation-with-preserve_source-false-return_3111560159/frozen.json b/cassettes/v1/Logs-Pipelines_3794283453/Create-a-pipeline-with-Array-Processor-Append-Operation-with-preserve_source-false-return_3111560159/frozen.json new file mode 100644 index 000000000000..31abe7fa748b --- /dev/null +++ b/cassettes/v1/Logs-Pipelines_3794283453/Create-a-pipeline-with-Array-Processor-Append-Operation-with-preserve_source-false-return_3111560159/frozen.json @@ -0,0 +1 @@ +"2025-06-30T15:45:41.844Z" diff --git a/cassettes/v1/Logs-Pipelines_3794283453/Create-a-pipeline-with-Array-Processor-Append-Operation-with-preserve_source-false-return_3111560159/recording.har b/cassettes/v1/Logs-Pipelines_3794283453/Create-a-pipeline-with-Array-Processor-Append-Operation-with-preserve_source-false-return_3111560159/recording.har new file mode 100644 index 000000000000..1bf653d80b2e --- /dev/null +++ b/cassettes/v1/Logs-Pipelines_3794283453/Create-a-pipeline-with-Array-Processor-Append-Operation-with-preserve_source-false-return_3111560159/recording.har @@ -0,0 +1,110 @@ +{ + "log": { + "_recordingName": "Logs Pipelines/Create a pipeline with Array Processor Append Operation with preserve_source false returns \"OK\" response", + "creator": { + "comment": "persister:fs", + "name": "Polly.JS", + "version": "6.0.5" + }, + "entries": [ + { + "_id": "c5d4e470e0dfe8eb6dfd55ebe4d07d64", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 290, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "accept", + "value": "application/json" + }, + { + "_fromType": "array", + "name": "content-type", + "value": "application/json" + } + ], + "headersSize": 574, + "httpVersion": "HTTP/1.1", + "method": "POST", + "postData": { + "mimeType": "application/json", + "params": [], + "text": "{\"filter\":{\"query\":\"source:python\"},\"name\":\"testPipelineArrayAppendNoPreserve\",\"processors\":[{\"is_enabled\":true,\"name\":\"append_ip_and_remove_source\",\"operation\":{\"preserve_source\":false,\"source\":\"network.client.ip\",\"target\":\"sourceIps\",\"type\":\"append\"},\"type\":\"array-processor\"}],\"tags\":[]}" + }, + "queryString": [], + "url": "https://api.datadoghq.com/api/v1/logs/config/pipelines" + }, + "response": { + "bodySize": 379, + "content": { + "mimeType": "application/json", + "size": 379, + "text": "{\"id\":\"B91fO94kQnCeZ4bOoZQOWg\",\"type\":\"pipeline\",\"name\":\"testPipelineArrayAppendNoPreserve\",\"is_enabled\":false,\"is_read_only\":false,\"filter\":{\"query\":\"source:python\"},\"processors\":[{\"name\":\"append_ip_and_remove_source\",\"is_enabled\":true,\"operation\":{\"source\":\"network.client.ip\",\"target\":\"sourceIps\",\"preserve_source\":false,\"type\":\"append\"},\"type\":\"array-processor\"}],\"tags\":[]}\n" + }, + "cookies": [], + "headers": [ + { + "name": "content-type", + "value": "application/json" + } + ], + "headersSize": 702, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2025-06-30T15:45:41.847Z", + "time": 409 + }, + { + "_id": "b152c85178b71b06e1b12177b9404b61", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "accept", + "value": "*/*" + } + ], + "headersSize": 533, + "httpVersion": "HTTP/1.1", + "method": "DELETE", + "queryString": [], + "url": "https://api.datadoghq.com/api/v1/logs/config/pipelines/B91fO94kQnCeZ4bOoZQOWg" + }, + "response": { + "bodySize": 3, + "content": { + "mimeType": "application/json", + "size": 3, + "text": "{}\n" + }, + "cookies": [], + "headers": [ + { + "name": "content-type", + "value": "application/json" + } + ], + "headersSize": 677, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2025-06-30T15:45:42.260Z", + "time": 391 + } + ], + "pages": [], + "version": "1.2" + } +} diff --git a/cassettes/v1/Logs-Pipelines_3794283453/Create-a-pipeline-with-Array-Processor-Append-Operation-with-preserve_source-true-returns-_558178392/frozen.json b/cassettes/v1/Logs-Pipelines_3794283453/Create-a-pipeline-with-Array-Processor-Append-Operation-with-preserve_source-true-returns-_558178392/frozen.json new file mode 100644 index 000000000000..836eec861769 --- /dev/null +++ b/cassettes/v1/Logs-Pipelines_3794283453/Create-a-pipeline-with-Array-Processor-Append-Operation-with-preserve_source-true-returns-_558178392/frozen.json @@ -0,0 +1 @@ +"2025-06-30T15:45:42.655Z" diff --git a/cassettes/v1/Logs-Pipelines_3794283453/Create-a-pipeline-with-Array-Processor-Append-Operation-with-preserve_source-true-returns-_558178392/recording.har b/cassettes/v1/Logs-Pipelines_3794283453/Create-a-pipeline-with-Array-Processor-Append-Operation-with-preserve_source-true-returns-_558178392/recording.har new file mode 100644 index 000000000000..a00e6f9e1a6b --- /dev/null +++ b/cassettes/v1/Logs-Pipelines_3794283453/Create-a-pipeline-with-Array-Processor-Append-Operation-with-preserve_source-true-returns-_558178392/recording.har @@ -0,0 +1,110 @@ +{ + "log": { + "_recordingName": "Logs Pipelines/Create a pipeline with Array Processor Append Operation with preserve_source true returns \"OK\" response", + "creator": { + "comment": "persister:fs", + "name": "Polly.JS", + "version": "6.0.5" + }, + "entries": [ + { + "_id": "795c16a2871f6268c906d718e9084f14", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 285, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "accept", + "value": "application/json" + }, + { + "_fromType": "array", + "name": "content-type", + "value": "application/json" + } + ], + "headersSize": 574, + "httpVersion": "HTTP/1.1", + "method": "POST", + "postData": { + "mimeType": "application/json", + "params": [], + "text": "{\"filter\":{\"query\":\"source:python\"},\"name\":\"testPipelineArrayAppendPreserve\",\"processors\":[{\"is_enabled\":true,\"name\":\"append_ip_and_keep_source\",\"operation\":{\"preserve_source\":true,\"source\":\"network.client.ip\",\"target\":\"sourceIps\",\"type\":\"append\"},\"type\":\"array-processor\"}],\"tags\":[]}" + }, + "queryString": [], + "url": "https://api.datadoghq.com/api/v1/logs/config/pipelines" + }, + "response": { + "bodySize": 374, + "content": { + "mimeType": "application/json", + "size": 374, + "text": "{\"id\":\"VX29vifpTjOKtlFWDp2-gA\",\"type\":\"pipeline\",\"name\":\"testPipelineArrayAppendPreserve\",\"is_enabled\":false,\"is_read_only\":false,\"filter\":{\"query\":\"source:python\"},\"processors\":[{\"name\":\"append_ip_and_keep_source\",\"is_enabled\":true,\"operation\":{\"source\":\"network.client.ip\",\"target\":\"sourceIps\",\"preserve_source\":true,\"type\":\"append\"},\"type\":\"array-processor\"}],\"tags\":[]}\n" + }, + "cookies": [], + "headers": [ + { + "name": "content-type", + "value": "application/json" + } + ], + "headersSize": 702, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2025-06-30T15:45:42.658Z", + "time": 390 + }, + { + "_id": "fc7504d0455783289f98c10d9c4e8c12", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "accept", + "value": "*/*" + } + ], + "headersSize": 533, + "httpVersion": "HTTP/1.1", + "method": "DELETE", + "queryString": [], + "url": "https://api.datadoghq.com/api/v1/logs/config/pipelines/VX29vifpTjOKtlFWDp2-gA" + }, + "response": { + "bodySize": 3, + "content": { + "mimeType": "application/json", + "size": 3, + "text": "{}\n" + }, + "cookies": [], + "headers": [ + { + "name": "content-type", + "value": "application/json" + } + ], + "headersSize": 677, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2025-06-30T15:45:43.071Z", + "time": 400 + } + ], + "pages": [], + "version": "1.2" + } +} diff --git a/cassettes/v1/Logs-Pipelines_3794283453/Create-a-pipeline-with-Array-Processor-Length-Operation-returns-OK-response_85115628/frozen.json b/cassettes/v1/Logs-Pipelines_3794283453/Create-a-pipeline-with-Array-Processor-Length-Operation-returns-OK-response_85115628/frozen.json new file mode 100644 index 000000000000..922a49904b76 --- /dev/null +++ b/cassettes/v1/Logs-Pipelines_3794283453/Create-a-pipeline-with-Array-Processor-Length-Operation-returns-OK-response_85115628/frozen.json @@ -0,0 +1 @@ +"2025-06-30T15:45:43.474Z" diff --git a/cassettes/v1/Logs-Pipelines_3794283453/Create-a-pipeline-with-Array-Processor-Length-Operation-returns-OK-response_85115628/recording.har b/cassettes/v1/Logs-Pipelines_3794283453/Create-a-pipeline-with-Array-Processor-Length-Operation-returns-OK-response_85115628/recording.har new file mode 100644 index 000000000000..836d55ca39e9 --- /dev/null +++ b/cassettes/v1/Logs-Pipelines_3794283453/Create-a-pipeline-with-Array-Processor-Length-Operation-returns-OK-response_85115628/recording.har @@ -0,0 +1,110 @@ +{ + "log": { + "_recordingName": "Logs Pipelines/Create a pipeline with Array Processor Length Operation returns \"OK\" response", + "creator": { + "comment": "persister:fs", + "name": "Polly.JS", + "version": "6.0.5" + }, + "entries": [ + { + "_id": "670259822f8c625d85b43218a993ca7d", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 225, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "accept", + "value": "application/json" + }, + { + "_fromType": "array", + "name": "content-type", + "value": "application/json" + } + ], + "headersSize": 574, + "httpVersion": "HTTP/1.1", + "method": "POST", + "postData": { + "mimeType": "application/json", + "params": [], + "text": "{\"filter\":{\"query\":\"source:python\"},\"name\":\"testPipelineArrayLength\",\"processors\":[{\"is_enabled\":true,\"name\":\"count_tags\",\"operation\":{\"source\":\"tags\",\"target\":\"tagCount\",\"type\":\"length\"},\"type\":\"array-processor\"}],\"tags\":[]}" + }, + "queryString": [], + "url": "https://api.datadoghq.com/api/v1/logs/config/pipelines" + }, + "response": { + "bodySize": 314, + "content": { + "mimeType": "application/json", + "size": 314, + "text": "{\"id\":\"MB-HZA9rRlKRHb-2LpYBxw\",\"type\":\"pipeline\",\"name\":\"testPipelineArrayLength\",\"is_enabled\":false,\"is_read_only\":false,\"filter\":{\"query\":\"source:python\"},\"processors\":[{\"name\":\"count_tags\",\"is_enabled\":true,\"operation\":{\"source\":\"tags\",\"target\":\"tagCount\",\"type\":\"length\"},\"type\":\"array-processor\"}],\"tags\":[]}\n" + }, + "cookies": [], + "headers": [ + { + "name": "content-type", + "value": "application/json" + } + ], + "headersSize": 702, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2025-06-30T15:45:43.477Z", + "time": 374 + }, + { + "_id": "f36ba1d2bc57574e54ed23c513caf164", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "accept", + "value": "*/*" + } + ], + "headersSize": 533, + "httpVersion": "HTTP/1.1", + "method": "DELETE", + "queryString": [], + "url": "https://api.datadoghq.com/api/v1/logs/config/pipelines/MB-HZA9rRlKRHb-2LpYBxw" + }, + "response": { + "bodySize": 3, + "content": { + "mimeType": "application/json", + "size": 3, + "text": "{}\n" + }, + "cookies": [], + "headers": [ + { + "name": "content-type", + "value": "application/json" + } + ], + "headersSize": 677, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2025-06-30T15:45:43.855Z", + "time": 382 + } + ], + "pages": [], + "version": "1.2" + } +} diff --git a/cassettes/v1/Logs-Pipelines_3794283453/Create-a-pipeline-with-Array-Processor-Select-Operation-returns-OK-response_405829604/frozen.json b/cassettes/v1/Logs-Pipelines_3794283453/Create-a-pipeline-with-Array-Processor-Select-Operation-returns-OK-response_405829604/frozen.json new file mode 100644 index 000000000000..2158ab1b75ce --- /dev/null +++ b/cassettes/v1/Logs-Pipelines_3794283453/Create-a-pipeline-with-Array-Processor-Select-Operation-returns-OK-response_405829604/frozen.json @@ -0,0 +1 @@ +"2025-06-30T15:45:44.240Z" diff --git a/cassettes/v1/Logs-Pipelines_3794283453/Create-a-pipeline-with-Array-Processor-Select-Operation-returns-OK-response_405829604/recording.har b/cassettes/v1/Logs-Pipelines_3794283453/Create-a-pipeline-with-Array-Processor-Select-Operation-returns-OK-response_405829604/recording.har new file mode 100644 index 000000000000..5f6947d6b90a --- /dev/null +++ b/cassettes/v1/Logs-Pipelines_3794283453/Create-a-pipeline-with-Array-Processor-Select-Operation-returns-OK-response_405829604/recording.har @@ -0,0 +1,110 @@ +{ + "log": { + "_recordingName": "Logs Pipelines/Create a pipeline with Array Processor Select Operation returns \"OK\" response", + "creator": { + "comment": "persister:fs", + "name": "Polly.JS", + "version": "6.0.5" + }, + "entries": [ + { + "_id": "154a281c9947b2224fa90ce0c43a8d0a", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 298, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "accept", + "value": "application/json" + }, + { + "_fromType": "array", + "name": "content-type", + "value": "application/json" + } + ], + "headersSize": 574, + "httpVersion": "HTTP/1.1", + "method": "POST", + "postData": { + "mimeType": "application/json", + "params": [], + "text": "{\"filter\":{\"query\":\"source:python\"},\"name\":\"testPipelineArraySelect\",\"processors\":[{\"is_enabled\":true,\"name\":\"extract_referrer\",\"operation\":{\"filter\":\"name:Referrer\",\"source\":\"httpRequest.headers\",\"target\":\"referrer\",\"type\":\"select\",\"value_to_extract\":\"value\"},\"type\":\"array-processor\"}],\"tags\":[]}" + }, + "queryString": [], + "url": "https://api.datadoghq.com/api/v1/logs/config/pipelines" + }, + "response": { + "bodySize": 387, + "content": { + "mimeType": "application/json", + "size": 387, + "text": "{\"id\":\"e3TVPUCYQ7a37CTTJB1HcA\",\"type\":\"pipeline\",\"name\":\"testPipelineArraySelect\",\"is_enabled\":false,\"is_read_only\":false,\"filter\":{\"query\":\"source:python\"},\"processors\":[{\"name\":\"extract_referrer\",\"is_enabled\":true,\"operation\":{\"source\":\"httpRequest.headers\",\"target\":\"referrer\",\"filter\":\"name:Referrer\",\"value_to_extract\":\"value\",\"type\":\"select\"},\"type\":\"array-processor\"}],\"tags\":[]}\n" + }, + "cookies": [], + "headers": [ + { + "name": "content-type", + "value": "application/json" + } + ], + "headersSize": 702, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2025-06-30T15:45:44.243Z", + "time": 413 + }, + { + "_id": "effbea81ffb83c0000bd7f72dd5460a9", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "accept", + "value": "*/*" + } + ], + "headersSize": 533, + "httpVersion": "HTTP/1.1", + "method": "DELETE", + "queryString": [], + "url": "https://api.datadoghq.com/api/v1/logs/config/pipelines/e3TVPUCYQ7a37CTTJB1HcA" + }, + "response": { + "bodySize": 3, + "content": { + "mimeType": "application/json", + "size": 3, + "text": "{}\n" + }, + "cookies": [], + "headers": [ + { + "name": "content-type", + "value": "application/json" + } + ], + "headersSize": 677, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2025-06-30T15:45:44.661Z", + "time": 387 + } + ], + "pages": [], + "version": "1.2" + } +} diff --git a/features/v1/logs_pipelines.feature b/features/v1/logs_pipelines.feature index 6a895252efa9..bf817fda07cc 100644 --- a/features/v1/logs_pipelines.feature +++ b/features/v1/logs_pipelines.feature @@ -35,6 +35,41 @@ Feature: Logs Pipelines When the request is sent Then the response status is 200 OK + @team:DataDog/event-platform-experience + Scenario: Create a pipeline with Array Processor Append Operation returns "OK" response + Given new "CreateLogsPipeline" request + And body with value {"filter": {"query": "source:python"}, "name": "testPipelineArrayAppend", "processors": [{"type": "array-processor", "is_enabled": true, "name": "append_ip_to_array", "operation": {"type": "append", "source": "network.client.ip", "target": "sourceIps"}}], "tags": []} + When the request is sent + Then the response status is 200 OK + + @team:DataDog/event-platform-experience + Scenario: Create a pipeline with Array Processor Append Operation with preserve_source false returns "OK" response + Given new "CreateLogsPipeline" request + And body with value {"filter": {"query": "source:python"}, "name": "testPipelineArrayAppendNoPreserve", "processors": [{"type": "array-processor", "is_enabled": true, "name": "append_ip_and_remove_source", "operation": {"type": "append", "source": "network.client.ip", "target": "sourceIps", "preserve_source": false}}], "tags": []} + When the request is sent + Then the response status is 200 OK + + @team:DataDog/event-platform-experience + Scenario: Create a pipeline with Array Processor Append Operation with preserve_source true returns "OK" response + Given new "CreateLogsPipeline" request + And body with value {"filter": {"query": "source:python"}, "name": "testPipelineArrayAppendPreserve", "processors": [{"type": "array-processor", "is_enabled": true, "name": "append_ip_and_keep_source", "operation": {"type": "append", "source": "network.client.ip", "target": "sourceIps", "preserve_source": true}}], "tags": []} + When the request is sent + Then the response status is 200 OK + + @team:DataDog/event-platform-experience + Scenario: Create a pipeline with Array Processor Length Operation returns "OK" response + Given new "CreateLogsPipeline" request + And body with value {"filter": {"query": "source:python"}, "name": "testPipelineArrayLength", "processors": [{"type": "array-processor", "is_enabled": true, "name": "count_tags", "operation": {"type": "length", "source": "tags", "target": "tagCount"}}], "tags": []} + When the request is sent + Then the response status is 200 OK + + @team:DataDog/event-platform-experience + Scenario: Create a pipeline with Array Processor Select Operation returns "OK" response + Given new "CreateLogsPipeline" request + And body with value {"filter": {"query": "source:python"}, "name": "testPipelineArraySelect", "processors": [{"type": "array-processor", "is_enabled": true, "name": "extract_referrer", "operation": {"type": "select", "source": "httpRequest.headers", "target": "referrer", "filter": "name:Referrer", "value_to_extract": "value"}}], "tags": []} + When the request is sent + Then the response status is 200 OK + @team:DataDog/event-platform-experience Scenario: Create a pipeline with Span Id Remapper returns "OK" response Given new "CreateLogsPipeline" request diff --git a/services/logs_pipelines/src/v1/index.ts b/services/logs_pipelines/src/v1/index.ts index 347553da22d1..13d633df19b0 100644 --- a/services/logs_pipelines/src/v1/index.ts +++ b/services/logs_pipelines/src/v1/index.ts @@ -12,6 +12,15 @@ export { LogsAPIError } from "./models/LogsAPIError"; export { LogsAPIErrorResponse } from "./models/LogsAPIErrorResponse"; export { LogsArithmeticProcessor } from "./models/LogsArithmeticProcessor"; export { LogsArithmeticProcessorType } from "./models/LogsArithmeticProcessorType"; +export { LogsArrayProcessor } from "./models/LogsArrayProcessor"; +export { LogsArrayProcessorOperation } from "./models/LogsArrayProcessorOperation"; +export { LogsArrayProcessorOperationAppend } from "./models/LogsArrayProcessorOperationAppend"; +export { LogsArrayProcessorOperationAppendType } from "./models/LogsArrayProcessorOperationAppendType"; +export { LogsArrayProcessorOperationLength } from "./models/LogsArrayProcessorOperationLength"; +export { LogsArrayProcessorOperationLengthType } from "./models/LogsArrayProcessorOperationLengthType"; +export { LogsArrayProcessorOperationSelect } from "./models/LogsArrayProcessorOperationSelect"; +export { LogsArrayProcessorOperationSelectType } from "./models/LogsArrayProcessorOperationSelectType"; +export { LogsArrayProcessorType } from "./models/LogsArrayProcessorType"; export { LogsAttributeRemapper } from "./models/LogsAttributeRemapper"; export { LogsAttributeRemapperType } from "./models/LogsAttributeRemapperType"; export { LogsCategoryProcessor } from "./models/LogsCategoryProcessor"; diff --git a/services/logs_pipelines/src/v1/models/LogsArrayProcessor.ts b/services/logs_pipelines/src/v1/models/LogsArrayProcessor.ts new file mode 100644 index 000000000000..d3419966f6d3 --- /dev/null +++ b/services/logs_pipelines/src/v1/models/LogsArrayProcessor.ts @@ -0,0 +1,77 @@ +import { AttributeTypeMap } from "@datadog/datadog-api-client"; + +import { LogsArrayProcessorOperation } from "./LogsArrayProcessorOperation"; +import { LogsArrayProcessorType } from "./LogsArrayProcessorType"; + +/** + * A processor for extracting, aggregating, or transforming values from JSON arrays within your logs. + * Supported operations are: + * - Select value from matching element + * - Compute array length + * - Append a value to an array + */ +export class LogsArrayProcessor { + /** + * Whether or not the processor is enabled. + */ + "isEnabled"?: boolean; + /** + * Name of the processor. + */ + "name"?: string; + /** + * Configuration of the array processor operation to perform. + */ + "operation": LogsArrayProcessorOperation; + /** + * Type of logs array processor. + */ + "type": LogsArrayProcessorType; + /** + * A container for additional, undeclared properties. + * This is a holder for any undeclared properties as specified with + * the 'additionalProperties' keyword in the OAS document. + */ + "additionalProperties"?: { [key: string]: any }; + /** + * @ignore + */ + "_unparsed"?: boolean; + + /** + * @ignore + */ + static readonly attributeTypeMap: AttributeTypeMap = { + isEnabled: { + baseName: "is_enabled", + type: "boolean", + }, + name: { + baseName: "name", + type: "string", + }, + operation: { + baseName: "operation", + type: "LogsArrayProcessorOperation", + required: true, + }, + type: { + baseName: "type", + type: "LogsArrayProcessorType", + required: true, + }, + additionalProperties: { + baseName: "additionalProperties", + type: "{ [key: string]: any; }", + }, + }; + + /** + * @ignore + */ + static getAttributeTypeMap(): AttributeTypeMap { + return LogsArrayProcessor.attributeTypeMap; + } + + public constructor() {} +} diff --git a/services/logs_pipelines/src/v1/models/LogsArrayProcessorOperation.ts b/services/logs_pipelines/src/v1/models/LogsArrayProcessorOperation.ts new file mode 100644 index 000000000000..a3b0c81917d8 --- /dev/null +++ b/services/logs_pipelines/src/v1/models/LogsArrayProcessorOperation.ts @@ -0,0 +1,14 @@ +import { UnparsedObject } from "@datadog/datadog-api-client"; + +import { LogsArrayProcessorOperationAppend } from "./LogsArrayProcessorOperationAppend"; +import { LogsArrayProcessorOperationLength } from "./LogsArrayProcessorOperationLength"; +import { LogsArrayProcessorOperationSelect } from "./LogsArrayProcessorOperationSelect"; + +/** + * Configuration of the array processor operation to perform. + */ +export type LogsArrayProcessorOperation = + | LogsArrayProcessorOperationAppend + | LogsArrayProcessorOperationLength + | LogsArrayProcessorOperationSelect + | UnparsedObject; diff --git a/services/logs_pipelines/src/v1/models/LogsArrayProcessorOperationAppend.ts b/services/logs_pipelines/src/v1/models/LogsArrayProcessorOperationAppend.ts new file mode 100644 index 000000000000..6168d2bcef98 --- /dev/null +++ b/services/logs_pipelines/src/v1/models/LogsArrayProcessorOperationAppend.ts @@ -0,0 +1,73 @@ +import { AttributeTypeMap } from "@datadog/datadog-api-client"; + +import { LogsArrayProcessorOperationAppendType } from "./LogsArrayProcessorOperationAppendType"; + +/** + * Operation that appends a value to a target array attribute. + */ +export class LogsArrayProcessorOperationAppend { + /** + * Remove or preserve the remapped source element. + */ + "preserveSource"?: boolean; + /** + * Attribute path containing the value to append. + */ + "source": string; + /** + * Attribute path of the array to append to. + */ + "target": string; + /** + * Operation type. + */ + "type": LogsArrayProcessorOperationAppendType; + /** + * A container for additional, undeclared properties. + * This is a holder for any undeclared properties as specified with + * the 'additionalProperties' keyword in the OAS document. + */ + "additionalProperties"?: { [key: string]: any }; + /** + * @ignore + */ + "_unparsed"?: boolean; + + /** + * @ignore + */ + static readonly attributeTypeMap: AttributeTypeMap = { + preserveSource: { + baseName: "preserve_source", + type: "boolean", + }, + source: { + baseName: "source", + type: "string", + required: true, + }, + target: { + baseName: "target", + type: "string", + required: true, + }, + type: { + baseName: "type", + type: "LogsArrayProcessorOperationAppendType", + required: true, + }, + additionalProperties: { + baseName: "additionalProperties", + type: "{ [key: string]: any; }", + }, + }; + + /** + * @ignore + */ + static getAttributeTypeMap(): AttributeTypeMap { + return LogsArrayProcessorOperationAppend.attributeTypeMap; + } + + public constructor() {} +} diff --git a/services/logs_pipelines/src/v1/models/LogsArrayProcessorOperationAppendType.ts b/services/logs_pipelines/src/v1/models/LogsArrayProcessorOperationAppendType.ts new file mode 100644 index 000000000000..de66518ada74 --- /dev/null +++ b/services/logs_pipelines/src/v1/models/LogsArrayProcessorOperationAppendType.ts @@ -0,0 +1,9 @@ +import { UnparsedObject } from "@datadog/datadog-api-client"; + +/** + * Operation type. + */ +export type LogsArrayProcessorOperationAppendType = + | typeof APPEND + | UnparsedObject; +export const APPEND = "append"; diff --git a/services/logs_pipelines/src/v1/models/LogsArrayProcessorOperationLength.ts b/services/logs_pipelines/src/v1/models/LogsArrayProcessorOperationLength.ts new file mode 100644 index 000000000000..082eeed5e7b4 --- /dev/null +++ b/services/logs_pipelines/src/v1/models/LogsArrayProcessorOperationLength.ts @@ -0,0 +1,65 @@ +import { AttributeTypeMap } from "@datadog/datadog-api-client"; + +import { LogsArrayProcessorOperationLengthType } from "./LogsArrayProcessorOperationLengthType"; + +/** + * Operation that computes the length of a `source` array and stores the result in the `target` attribute. + */ +export class LogsArrayProcessorOperationLength { + /** + * Attribute path of the array to measure. + */ + "source": string; + /** + * Attribute that receives the computed length. + */ + "target": string; + /** + * Operation type. + */ + "type": LogsArrayProcessorOperationLengthType; + /** + * A container for additional, undeclared properties. + * This is a holder for any undeclared properties as specified with + * the 'additionalProperties' keyword in the OAS document. + */ + "additionalProperties"?: { [key: string]: any }; + /** + * @ignore + */ + "_unparsed"?: boolean; + + /** + * @ignore + */ + static readonly attributeTypeMap: AttributeTypeMap = { + source: { + baseName: "source", + type: "string", + required: true, + }, + target: { + baseName: "target", + type: "string", + required: true, + }, + type: { + baseName: "type", + type: "LogsArrayProcessorOperationLengthType", + required: true, + }, + additionalProperties: { + baseName: "additionalProperties", + type: "{ [key: string]: any; }", + }, + }; + + /** + * @ignore + */ + static getAttributeTypeMap(): AttributeTypeMap { + return LogsArrayProcessorOperationLength.attributeTypeMap; + } + + public constructor() {} +} diff --git a/services/logs_pipelines/src/v1/models/LogsArrayProcessorOperationLengthType.ts b/services/logs_pipelines/src/v1/models/LogsArrayProcessorOperationLengthType.ts new file mode 100644 index 000000000000..9cab8021fcbf --- /dev/null +++ b/services/logs_pipelines/src/v1/models/LogsArrayProcessorOperationLengthType.ts @@ -0,0 +1,9 @@ +import { UnparsedObject } from "@datadog/datadog-api-client"; + +/** + * Operation type. + */ +export type LogsArrayProcessorOperationLengthType = + | typeof LENGTH + | UnparsedObject; +export const LENGTH = "length"; diff --git a/services/logs_pipelines/src/v1/models/LogsArrayProcessorOperationSelect.ts b/services/logs_pipelines/src/v1/models/LogsArrayProcessorOperationSelect.ts new file mode 100644 index 000000000000..4f628fab2753 --- /dev/null +++ b/services/logs_pipelines/src/v1/models/LogsArrayProcessorOperationSelect.ts @@ -0,0 +1,83 @@ +import { AttributeTypeMap } from "@datadog/datadog-api-client"; + +import { LogsArrayProcessorOperationSelectType } from "./LogsArrayProcessorOperationSelectType"; + +/** + * Operation that finds an object in a `source` array using a `filter`, and then extracts a specific value into the `target` attribute. + */ +export class LogsArrayProcessorOperationSelect { + /** + * Filter condition expressed as `key:value` used to find the matching element. + */ + "filter": string; + /** + * Attribute path of the array to search into. + */ + "source": string; + /** + * Attribute that receives the extracted value. + */ + "target": string; + /** + * Operation type. + */ + "type": LogsArrayProcessorOperationSelectType; + /** + * Key of the value to extract from the matching element. + */ + "valueToExtract": string; + /** + * A container for additional, undeclared properties. + * This is a holder for any undeclared properties as specified with + * the 'additionalProperties' keyword in the OAS document. + */ + "additionalProperties"?: { [key: string]: any }; + /** + * @ignore + */ + "_unparsed"?: boolean; + + /** + * @ignore + */ + static readonly attributeTypeMap: AttributeTypeMap = { + filter: { + baseName: "filter", + type: "string", + required: true, + }, + source: { + baseName: "source", + type: "string", + required: true, + }, + target: { + baseName: "target", + type: "string", + required: true, + }, + type: { + baseName: "type", + type: "LogsArrayProcessorOperationSelectType", + required: true, + }, + valueToExtract: { + baseName: "value_to_extract", + type: "string", + required: true, + }, + additionalProperties: { + baseName: "additionalProperties", + type: "{ [key: string]: any; }", + }, + }; + + /** + * @ignore + */ + static getAttributeTypeMap(): AttributeTypeMap { + return LogsArrayProcessorOperationSelect.attributeTypeMap; + } + + public constructor() {} +} diff --git a/services/logs_pipelines/src/v1/models/LogsArrayProcessorOperationSelectType.ts b/services/logs_pipelines/src/v1/models/LogsArrayProcessorOperationSelectType.ts new file mode 100644 index 000000000000..29485a23a86f --- /dev/null +++ b/services/logs_pipelines/src/v1/models/LogsArrayProcessorOperationSelectType.ts @@ -0,0 +1,9 @@ +import { UnparsedObject } from "@datadog/datadog-api-client"; + +/** + * Operation type. + */ +export type LogsArrayProcessorOperationSelectType = + | typeof SELECT + | UnparsedObject; +export const SELECT = "select"; diff --git a/services/logs_pipelines/src/v1/models/LogsArrayProcessorType.ts b/services/logs_pipelines/src/v1/models/LogsArrayProcessorType.ts new file mode 100644 index 000000000000..71261e4df9f3 --- /dev/null +++ b/services/logs_pipelines/src/v1/models/LogsArrayProcessorType.ts @@ -0,0 +1,7 @@ +import { UnparsedObject } from "@datadog/datadog-api-client"; + +/** + * Type of logs array processor. + */ +export type LogsArrayProcessorType = typeof ARRAY_PROCESSOR | UnparsedObject; +export const ARRAY_PROCESSOR = "array-processor"; diff --git a/services/logs_pipelines/src/v1/models/LogsProcessor.ts b/services/logs_pipelines/src/v1/models/LogsProcessor.ts index 4c9eeaacb82a..2bb93deb0a44 100644 --- a/services/logs_pipelines/src/v1/models/LogsProcessor.ts +++ b/services/logs_pipelines/src/v1/models/LogsProcessor.ts @@ -1,6 +1,7 @@ import { UnparsedObject } from "@datadog/datadog-api-client"; import { LogsArithmeticProcessor } from "./LogsArithmeticProcessor"; +import { LogsArrayProcessor } from "./LogsArrayProcessor"; import { LogsAttributeRemapper } from "./LogsAttributeRemapper"; import { LogsCategoryProcessor } from "./LogsCategoryProcessor"; import { LogsDateRemapper } from "./LogsDateRemapper"; @@ -39,4 +40,5 @@ export type LogsProcessor = | ReferenceTableLogsLookupProcessor | LogsTraceRemapper | LogsSpanRemapper + | LogsArrayProcessor | UnparsedObject; diff --git a/services/logs_pipelines/src/v1/models/TypingInfo.ts b/services/logs_pipelines/src/v1/models/TypingInfo.ts index 9ba6b78a3c4e..5ade339001e4 100644 --- a/services/logs_pipelines/src/v1/models/TypingInfo.ts +++ b/services/logs_pipelines/src/v1/models/TypingInfo.ts @@ -4,6 +4,10 @@ import { APIErrorResponse } from "./APIErrorResponse"; import { LogsAPIError } from "./LogsAPIError"; import { LogsAPIErrorResponse } from "./LogsAPIErrorResponse"; import { LogsArithmeticProcessor } from "./LogsArithmeticProcessor"; +import { LogsArrayProcessor } from "./LogsArrayProcessor"; +import { LogsArrayProcessorOperationAppend } from "./LogsArrayProcessorOperationAppend"; +import { LogsArrayProcessorOperationLength } from "./LogsArrayProcessorOperationLength"; +import { LogsArrayProcessorOperationSelect } from "./LogsArrayProcessorOperationSelect"; import { LogsAttributeRemapper } from "./LogsAttributeRemapper"; import { LogsCategoryProcessor } from "./LogsCategoryProcessor"; import { LogsCategoryProcessorCategory } from "./LogsCategoryProcessorCategory"; @@ -29,6 +33,10 @@ import { ReferenceTableLogsLookupProcessor } from "./ReferenceTableLogsLookupPro export const TypingInfo: ModelTypingInfo = { enumsMap: { LogsArithmeticProcessorType: ["arithmetic-processor"], + LogsArrayProcessorOperationAppendType: ["append"], + LogsArrayProcessorOperationLengthType: ["length"], + LogsArrayProcessorOperationSelectType: ["select"], + LogsArrayProcessorType: ["array-processor"], LogsAttributeRemapperType: ["attribute-remapper"], LogsCategoryProcessorType: ["category-processor"], LogsDateRemapperType: ["date-remapper"], @@ -47,6 +55,11 @@ export const TypingInfo: ModelTypingInfo = { TargetFormatType: ["auto", "string", "integer", "double"], }, oneOfMap: { + LogsArrayProcessorOperation: [ + "LogsArrayProcessorOperationAppend", + "LogsArrayProcessorOperationLength", + "LogsArrayProcessorOperationSelect", + ], LogsProcessor: [ "LogsGrokParser", "LogsDateRemapper", @@ -65,6 +78,7 @@ export const TypingInfo: ModelTypingInfo = { "ReferenceTableLogsLookupProcessor", "LogsTraceRemapper", "LogsSpanRemapper", + "LogsArrayProcessor", ], }, typeMap: { @@ -72,6 +86,10 @@ export const TypingInfo: ModelTypingInfo = { LogsAPIError: LogsAPIError, LogsAPIErrorResponse: LogsAPIErrorResponse, LogsArithmeticProcessor: LogsArithmeticProcessor, + LogsArrayProcessor: LogsArrayProcessor, + LogsArrayProcessorOperationAppend: LogsArrayProcessorOperationAppend, + LogsArrayProcessorOperationLength: LogsArrayProcessorOperationLength, + LogsArrayProcessorOperationSelect: LogsArrayProcessorOperationSelect, LogsAttributeRemapper: LogsAttributeRemapper, LogsCategoryProcessor: LogsCategoryProcessor, LogsCategoryProcessorCategory: LogsCategoryProcessorCategory,