Add support for Array Processor in Logs Pipelines

.generator/schemas/v1/openapi.yaml

@@ -5163,6 +5163,143 @@ components:
       type: string
       x-enum-varnames:
       - ARITHMETIC_PROCESSOR
+    LogsArrayProcessor:
+      description: 'A processor for extracting, aggregating, or transforming values
+        from JSON arrays within your logs.
+
+        Supported operations are:
+
+        - Select value from matching element
+
+        - Compute array length
+
+        - Append a value to an array'
+      properties:
+        is_enabled:
+          default: false
+          description: Whether or not the processor is enabled.
+          type: boolean
+        name:
+          description: Name of the processor.
+          type: string
+        operation:
+          $ref: '#/components/schemas/LogsArrayProcessorOperation'
+        type:
+          $ref: '#/components/schemas/LogsArrayProcessorType'
+      required:
+      - operation
+      - type
+      type: object
+    LogsArrayProcessorOperation:
+      description: Configuration of the array processor operation to perform.
+      oneOf:
+      - $ref: '#/components/schemas/LogsArrayProcessorOperationAppend'
+      - $ref: '#/components/schemas/LogsArrayProcessorOperationLength'
+      - $ref: '#/components/schemas/LogsArrayProcessorOperationSelect'
+    LogsArrayProcessorOperationAppend:
+      description: Operation that appends a value to a target array attribute.
+      properties:
+        preserve_source:
+          default: true
+          description: Remove or preserve the remapped source element.
+          type: boolean
+        source:
+          description: Attribute path containing the value to append.
+          example: network.client.ip
+          type: string
+        target:
+          description: Attribute path of the array to append to.
+          example: sourceIps
+          type: string
+        type:
+          $ref: '#/components/schemas/LogsArrayProcessorOperationAppendType'
+      required:
+      - type
+      - source
+      - target
+      type: object
+    LogsArrayProcessorOperationAppendType:
+      description: Operation type.
+      enum:
+      - append
+      example: append
+      type: string
+      x-enum-varnames:
+      - APPEND
+    LogsArrayProcessorOperationLength:
+      description: Operation that computes the length of a `source` array and stores
+        the result in the `target` attribute.
+      properties:
+        source:
+          description: Attribute path of the array to measure.
+          example: tags
+          type: string
+        target:
+          description: Attribute that receives the computed length.
+          example: tagCount
+          type: string
+        type:
+          $ref: '#/components/schemas/LogsArrayProcessorOperationLengthType'
+      required:
+      - type
+      - source
+      - target
+      type: object
+    LogsArrayProcessorOperationLengthType:
+      description: Operation type.
+      enum:
+      - length
+      example: length
+      type: string
+      x-enum-varnames:
+      - LENGTH
+    LogsArrayProcessorOperationSelect:
+      description: Operation that finds an object in a `source` array using a `filter`,
+        and then extracts a specific value into the `target` attribute.
+      properties:
+        filter:
+          description: Filter condition expressed as `key:value` used to find the
+            matching element.
+          example: name:Referrer
+          type: string
+        source:
+          description: Attribute path of the array to search into.
+          example: httpRequest.headers
+          type: string
+        target:
+          description: Attribute that receives the extracted value.
+          example: referrer
+          type: string
+        type:
+          $ref: '#/components/schemas/LogsArrayProcessorOperationSelectType'
+        value_to_extract:
+          description: Key of the value to extract from the matching element.
+          example: value
+          type: string
+      required:
+      - type
+      - source
+      - target
+      - filter
+      - value_to_extract
+      type: object
+    LogsArrayProcessorOperationSelectType:
+      description: Operation type.
+      enum:
+      - select
+      example: select
+      type: string
+      x-enum-varnames:
+      - SELECT
+    LogsArrayProcessorType:
+      default: array-processor
+      description: Type of logs array processor.
+      enum:
+      - array-processor
+      example: array-processor
+      type: string
+      x-enum-varnames:
+      - ARRAY_PROCESSOR
     LogsAttributeRemapper:
       description: 'The remapper processor remaps any source attribute(s) or tag to
         another target attribute or tag.
@@ -6060,6 +6197,7 @@ components:
       - $ref: '#/components/schemas/ReferenceTableLogsLookupProcessor'
       - $ref: '#/components/schemas/LogsTraceRemapper'
       - $ref: '#/components/schemas/LogsSpanRemapper'
+      - $ref: '#/components/schemas/LogsArrayProcessor'
     LogsQueryCompute:
       description: Define computation for a log query.
       properties:

cassettes/v1/Logs-Pipelines_3794283453/Create-a-pipeline-with-Array-Processor-Append-Operation-returns-OK-response_2598677476/frozen.json

@@ -0,0 +1 @@
+"2025-06-30T15:45:40.994Z"

cassettes/v1/Logs-Pipelines_3794283453/Create-a-pipeline-with-Array-Processor-Append-Operation-returns-OK-response_2598677476/recording.har

@@ -0,0 +1,110 @@
+{
+  "log": {
+    "_recordingName": "Logs Pipelines/Create a pipeline with Array Processor Append Operation returns \"OK\" response",
+    "creator": {
+      "comment": "persister:fs",
+      "name": "Polly.JS",
+      "version": "6.0.5"
+    },
+    "entries": [
+      {
+        "_id": "69ee9de755f2e0b61cbbe11dbe56dc06",
+        "_order": 0,
+        "cache": {},
+        "request": {
+          "bodySize": 247,
+          "cookies": [],
+          "headers": [
+            {
+              "_fromType": "array",
+              "name": "accept",
+              "value": "application/json"
+            },
+            {
+              "_fromType": "array",
+              "name": "content-type",
+              "value": "application/json"
+            }
+          ],
+          "headersSize": 574,
+          "httpVersion": "HTTP/1.1",
+          "method": "POST",
+          "postData": {
+            "mimeType": "application/json",
+            "params": [],
+            "text": "{\"filter\":{\"query\":\"source:python\"},\"name\":\"testPipelineArrayAppend\",\"processors\":[{\"is_enabled\":true,\"name\":\"append_ip_to_array\",\"operation\":{\"source\":\"network.client.ip\",\"target\":\"sourceIps\",\"type\":\"append\"},\"type\":\"array-processor\"}],\"tags\":[]}"
+          },
+          "queryString": [],
+          "url": "https://api.datadoghq.com/api/v1/logs/config/pipelines"
+        },
+        "response": {
+          "bodySize": 359,
+          "content": {
+            "mimeType": "application/json",
+            "size": 359,
+            "text": "{\"id\":\"s_cPqdnkQVaU6PwbPPt2ZQ\",\"type\":\"pipeline\",\"name\":\"testPipelineArrayAppend\",\"is_enabled\":false,\"is_read_only\":false,\"filter\":{\"query\":\"source:python\"},\"processors\":[{\"name\":\"append_ip_to_array\",\"is_enabled\":true,\"operation\":{\"source\":\"network.client.ip\",\"target\":\"sourceIps\",\"preserve_source\":true,\"type\":\"append\"},\"type\":\"array-processor\"}],\"tags\":[]}\n"
+          },
+          "cookies": [],
+          "headers": [
+            {
+              "name": "content-type",
+              "value": "application/json"
+            }
+          ],
+          "headersSize": 702,
+          "httpVersion": "HTTP/1.1",
+          "redirectURL": "",
+          "status": 200,
+          "statusText": "OK"
+        },
+        "startedDateTime": "2025-06-30T15:45:41.001Z",
+        "time": 398
+      },
+      {
+        "_id": "2d231c72b566a1746dfa7066c5e6b025",
+        "_order": 0,
+        "cache": {},
+        "request": {
+          "bodySize": 0,
+          "cookies": [],
+          "headers": [
+            {
+              "_fromType": "array",
+              "name": "accept",
+              "value": "*/*"
+            }
+          ],
+          "headersSize": 533,
+          "httpVersion": "HTTP/1.1",
+          "method": "DELETE",
+          "queryString": [],
+          "url": "https://api.datadoghq.com/api/v1/logs/config/pipelines/s_cPqdnkQVaU6PwbPPt2ZQ"
+        },
+        "response": {
+          "bodySize": 3,
+          "content": {
+            "mimeType": "application/json",
+            "size": 3,
+            "text": "{}\n"
+          },
+          "cookies": [],
+          "headers": [
+            {
+              "name": "content-type",
+              "value": "application/json"
+            }
+          ],
+          "headersSize": 677,
+          "httpVersion": "HTTP/1.1",
+          "redirectURL": "",
+          "status": 200,
+          "statusText": "OK"
+        },
+        "startedDateTime": "2025-06-30T15:45:41.406Z",
+        "time": 432
+      }
+    ],
+    "pages": [],
+    "version": "1.2"
+  }
+}

cassettes/v1/Logs-Pipelines_3794283453/Create-a-pipeline-with-Array-Processor-Append-Operation-with-preserve_source-false-return_3111560159/frozen.json

@@ -0,0 +1 @@
+"2025-06-30T15:45:41.844Z"

cassettes/v1/Logs-Pipelines_3794283453/Create-a-pipeline-with-Array-Processor-Append-Operation-with-preserve_source-false-return_3111560159/recording.har

@@ -0,0 +1,110 @@
+{
+  "log": {
+    "_recordingName": "Logs Pipelines/Create a pipeline with Array Processor Append Operation with preserve_source false returns \"OK\" response",
+    "creator": {
+      "comment": "persister:fs",
+      "name": "Polly.JS",
+      "version": "6.0.5"
+    },
+    "entries": [
+      {
+        "_id": "c5d4e470e0dfe8eb6dfd55ebe4d07d64",
+        "_order": 0,
+        "cache": {},
+        "request": {
+          "bodySize": 290,
+          "cookies": [],
+          "headers": [
+            {
+              "_fromType": "array",
+              "name": "accept",
+              "value": "application/json"
+            },
+            {
+              "_fromType": "array",
+              "name": "content-type",
+              "value": "application/json"
+            }
+          ],
+          "headersSize": 574,
+          "httpVersion": "HTTP/1.1",
+          "method": "POST",
+          "postData": {
+            "mimeType": "application/json",
+            "params": [],
+            "text": "{\"filter\":{\"query\":\"source:python\"},\"name\":\"testPipelineArrayAppendNoPreserve\",\"processors\":[{\"is_enabled\":true,\"name\":\"append_ip_and_remove_source\",\"operation\":{\"preserve_source\":false,\"source\":\"network.client.ip\",\"target\":\"sourceIps\",\"type\":\"append\"},\"type\":\"array-processor\"}],\"tags\":[]}"
+          },
+          "queryString": [],
+          "url": "https://api.datadoghq.com/api/v1/logs/config/pipelines"
+        },
+        "response": {
+          "bodySize": 379,
+          "content": {
+            "mimeType": "application/json",
+            "size": 379,
+            "text": "{\"id\":\"B91fO94kQnCeZ4bOoZQOWg\",\"type\":\"pipeline\",\"name\":\"testPipelineArrayAppendNoPreserve\",\"is_enabled\":false,\"is_read_only\":false,\"filter\":{\"query\":\"source:python\"},\"processors\":[{\"name\":\"append_ip_and_remove_source\",\"is_enabled\":true,\"operation\":{\"source\":\"network.client.ip\",\"target\":\"sourceIps\",\"preserve_source\":false,\"type\":\"append\"},\"type\":\"array-processor\"}],\"tags\":[]}\n"
+          },
+          "cookies": [],
+          "headers": [
+            {
+              "name": "content-type",
+              "value": "application/json"
+            }
+          ],
+          "headersSize": 702,
+          "httpVersion": "HTTP/1.1",
+          "redirectURL": "",
+          "status": 200,
+          "statusText": "OK"
+        },
+        "startedDateTime": "2025-06-30T15:45:41.847Z",
+        "time": 409
+      },
+      {
+        "_id": "b152c85178b71b06e1b12177b9404b61",
+        "_order": 0,
+        "cache": {},
+        "request": {
+          "bodySize": 0,
+          "cookies": [],
+          "headers": [
+            {
+              "_fromType": "array",
+              "name": "accept",
+              "value": "*/*"
+            }
+          ],
+          "headersSize": 533,
+          "httpVersion": "HTTP/1.1",
+          "method": "DELETE",
+          "queryString": [],
+          "url": "https://api.datadoghq.com/api/v1/logs/config/pipelines/B91fO94kQnCeZ4bOoZQOWg"
+        },
+        "response": {
+          "bodySize": 3,
+          "content": {
+            "mimeType": "application/json",
+            "size": 3,
+            "text": "{}\n"
+          },
+          "cookies": [],
+          "headers": [
+            {
+              "name": "content-type",
+              "value": "application/json"
+            }
+          ],
+          "headersSize": 677,
+          "httpVersion": "HTTP/1.1",
+          "redirectURL": "",
+          "status": 200,
+          "statusText": "OK"
+        },
+        "startedDateTime": "2025-06-30T15:45:42.260Z",
+        "time": 391
+      }
+    ],
+    "pages": [],
+    "version": "1.2"
+  }
+}

cassettes/v1/Logs-Pipelines_3794283453/Create-a-pipeline-with-Array-Processor-Append-Operation-with-preserve_source-true-returns-_558178392/frozen.json

@@ -0,0 +1 @@
+"2025-06-30T15:45:42.655Z"