diff --git a/.apigentools-info b/.apigentools-info index 75056fa5871f4..ee75c36c6a595 100644 --- a/.apigentools-info +++ b/.apigentools-info @@ -4,13 +4,13 @@ "spec_versions": { "v1": { "apigentools_version": "1.6.6", - "regenerated": "2025-08-22 18:46:40.838944", - "spec_repo_commit": "d58cb844" + "regenerated": "2025-08-25 08:42:50.989273", + "spec_repo_commit": "205813dc" }, "v2": { "apigentools_version": "1.6.6", - "regenerated": "2025-08-22 18:46:51.222340", - "spec_repo_commit": "d58cb844" + "regenerated": "2025-08-25 08:43:00.967505", + "spec_repo_commit": "205813dc" } } } \ No newline at end of file diff --git a/config/_default/menus/api.en.yaml b/config/_default/menus/api.en.yaml index a67f503c1379e..76d6c59ee6e84 100644 --- a/config/_default/menus/api.en.yaml +++ b/config/_default/menus/api.en.yaml @@ -9356,6 +9356,23 @@ menu: - ListCatalogEntity unstable: [] order: 1 + - name: Spa + url: /api/latest/spa/ + identifier: spa + generated: true + - name: Get SPA Recommendations + url: '#get-spa-recommendations' + identifier: spa-get-spa-recommendations + parent: spa + generated: true + params: + versions: + - v2 + operationids: + - GetSPARecommendations + unstable: + - v2 + order: 1 - name: Spans url: /api/latest/spans/ identifier: spans diff --git a/content/en/api/latest/spa/_index.md b/content/en/api/latest/spa/_index.md new file mode 100644 index 0000000000000..86272bbd6ceb4 --- /dev/null +++ b/content/en/api/latest/spa/_index.md @@ -0,0 +1,3 @@ +--- +title: Spa +--- diff --git a/content/en/api/v2/spa/_index.md b/content/en/api/v2/spa/_index.md new file mode 100644 index 0000000000000..6e402b8f910d2 --- /dev/null +++ b/content/en/api/v2/spa/_index.md @@ -0,0 +1,4 @@ +--- +title: Spa +headless: true +--- diff --git a/content/en/api/v2/spa/examples.json b/content/en/api/v2/spa/examples.json new file mode 100644 index 0000000000000..673468fe5c534 --- /dev/null +++ b/content/en/api/v2/spa/examples.json @@ -0,0 +1,72 @@ +{ + "GetSPARecommendations": { + "responses": { + "200": { + "json": { + "data": { + "attributes": { + "driver": { + "estimation": { + "cpu": { + "max": "integer", + "p75": "integer", + "p95": "integer" + }, + "ephemeral_storage": "integer", + "heap": "integer", + "memory": "integer", + "overhead": "integer" + } + }, + "executor": { + "estimation": { + "cpu": { + "max": "integer", + "p75": "integer", + "p95": "integer" + }, + "ephemeral_storage": "integer", + "heap": "integer", + "memory": "integer", + "overhead": "integer" + } + } + }, + "id": "string", + "type": "recommendation" + } + }, + "html": "
\n
\n
\n
\n

data [required]

\n
\n

object

\n

JSON:API resource object for SPA Recommendation. Includes type, optional ID, and resource attributes with structured recommendations.

\n
\n
\n
\n
\n
\n

attributes [required]

\n
\n

object

\n

Attributes of the SPA Recommendation resource. Contains recommendations for both driver and executor components.

\n
\n
\n
\n
\n
\n

driver [required]

\n
\n

object

\n

Resource recommendation for a single Spark component (driver or executor). Contains estimation data used to patch Spark job specs.

\n
\n
\n
\n
\n
\n

estimation [required]

\n
\n

object

\n

Recommended resource values for a Spark driver or executor, derived from recent real usage metrics. Used by SPA to propose more efficient pod sizing.

\n
\n
\n
\n
\n
\n

cpu

\n
\n

object

\n

CPU usage statistics derived from historical Spark job metrics. Provides multiple estimates so users can choose between conservative and cost-saving risk profiles.

\n
\n
\n
\n
\n
\n

max

\n
\n

int64

\n

Maximum CPU usage observed for the job, expressed in millicores. This represents the upper bound of usage.

\n
\n \n
\n
\n
\n
\n
\n

p75

\n
\n

int64

\n

75th percentile of CPU usage (millicores). Represents a cost-saving configuration while covering most workloads.

\n
\n \n
\n
\n
\n
\n
\n

p95

\n
\n

int64

\n

95th percentile of CPU usage (millicores). Balances performance and cost, providing a safer margin than p75.

\n
\n \n
\n
\n
\n
\n
\n
\n
\n

ephemeral_storage

\n
\n

int64

\n

Recommended ephemeral storage allocation (in MiB). Derived from job temporary storage patterns.

\n
\n \n
\n
\n
\n
\n
\n

heap

\n
\n

int64

\n

Recommended JVM heap size (in MiB).

\n
\n \n
\n
\n
\n
\n
\n

memory

\n
\n

int64

\n

Recommended total memory allocation (in MiB). Includes both heap and overhead.

\n
\n \n
\n
\n
\n
\n
\n

overhead

\n
\n

int64

\n

Recommended JVM overhead (in MiB). Computed as total memory - heap.

\n
\n \n
\n
\n
\n
\n
\n
\n
\n
\n
\n

executor [required]

\n
\n

object

\n

Resource recommendation for a single Spark component (driver or executor). Contains estimation data used to patch Spark job specs.

\n
\n
\n
\n
\n
\n

estimation [required]

\n
\n

object

\n

Recommended resource values for a Spark driver or executor, derived from recent real usage metrics. Used by SPA to propose more efficient pod sizing.

\n
\n
\n
\n
\n
\n

cpu

\n
\n

object

\n

CPU usage statistics derived from historical Spark job metrics. Provides multiple estimates so users can choose between conservative and cost-saving risk profiles.

\n
\n
\n
\n
\n
\n

max

\n
\n

int64

\n

Maximum CPU usage observed for the job, expressed in millicores. This represents the upper bound of usage.

\n
\n \n
\n
\n
\n
\n
\n

p75

\n
\n

int64

\n

75th percentile of CPU usage (millicores). Represents a cost-saving configuration while covering most workloads.

\n
\n \n
\n
\n
\n
\n
\n

p95

\n
\n

int64

\n

95th percentile of CPU usage (millicores). Balances performance and cost, providing a safer margin than p75.

\n
\n \n
\n
\n
\n
\n
\n
\n
\n

ephemeral_storage

\n
\n

int64

\n

Recommended ephemeral storage allocation (in MiB). Derived from job temporary storage patterns.

\n
\n \n
\n
\n
\n
\n
\n

heap

\n
\n

int64

\n

Recommended JVM heap size (in MiB).

\n
\n \n
\n
\n
\n
\n
\n

memory

\n
\n

int64

\n

Recommended total memory allocation (in MiB). Includes both heap and overhead.

\n
\n \n
\n
\n
\n
\n
\n

overhead

\n
\n

int64

\n

Recommended JVM overhead (in MiB). Computed as total memory - heap.

\n
\n \n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n

id

\n
\n

string

\n

Resource identifier for the recommendation. Optional in responses.

\n
\n \n
\n
\n
\n
\n
\n

type [required]

\n
\n

enum

\n

JSON:API resource type for Spark Pod Autosizing recommendations. Identifies the Recommendation resource returned by SPA. \nAllowed enum values: recommendation

default: recommendation

\n
\n \n
\n
\n
\n
" + }, + "400": { + "json": { + "errors": [ + "Bad Request" + ] + }, + "html": "
\n
\n
\n
\n

errors [required]

\n
\n

[string]

\n

A list of errors.

\n
\n \n
\n
" + }, + "403": { + "json": { + "errors": [ + "Bad Request" + ] + }, + "html": "
\n
\n
\n
\n

errors [required]

\n
\n

[string]

\n

A list of errors.

\n
\n \n
\n
" + }, + "429": { + "json": { + "errors": [ + "Bad Request" + ] + }, + "html": "
\n
\n
\n
\n

errors [required]

\n
\n

[string]

\n

A list of errors.

\n
\n \n
\n
" + } + }, + "request": { + "json_curl": {}, + "json": {}, + "html": "" + } + } +} \ No newline at end of file diff --git a/data/api/v2/full_spec.yaml b/data/api/v2/full_spec.yaml index a0cc9a943d046..f58b86ac0a8a4 100644 --- a/data/api/v2/full_spec.yaml +++ b/data/api/v2/full_spec.yaml @@ -9345,6 +9345,15 @@ components: that evaluates to a boolean. example: ${true} type: string + ComponentRecommendation: + description: Resource recommendation for a single Spark component (driver or + executor). Contains estimation data used to patch Spark job specs. + properties: + estimation: + $ref: '#/components/schemas/Estimation' + required: + - estimation + type: object ComponentType: description: The UI component type. enum: @@ -10551,6 +10560,28 @@ components: type: string x-enum-varnames: - COST_BY_ORG + Cpu: + description: CPU usage statistics derived from historical Spark job metrics. + Provides multiple estimates so users can choose between conservative and cost-saving + risk profiles. + properties: + max: + description: Maximum CPU usage observed for the job, expressed in millicores. + This represents the upper bound of usage. + format: int64 + type: integer + p75: + description: 75th percentile of CPU usage (millicores). Represents a cost-saving + configuration while covering most workloads. + format: int64 + type: integer + p95: + description: 95th percentile of CPU usage (millicores). Balances performance + and cost, providing a safer margin than p75. + format: int64 + type: integer + type: object + x-model-simple-name: SpaCpu CreateActionConnectionRequest: description: Request used to create an action connection. properties: @@ -15817,6 +15848,33 @@ components: type: string x-enum-varnames: - ESCALATION_POLICY_STEPS + Estimation: + description: Recommended resource values for a Spark driver or executor, derived + from recent real usage metrics. Used by SPA to propose more efficient pod + sizing. + properties: + cpu: + $ref: '#/components/schemas/Cpu' + ephemeral_storage: + description: Recommended ephemeral storage allocation (in MiB). Derived + from job temporary storage patterns. + format: int64 + type: integer + heap: + description: Recommended JVM heap size (in MiB). + format: int64 + type: integer + memory: + description: Recommended total memory allocation (in MiB). Includes both + heap and overhead. + format: int64 + type: integer + overhead: + description: Recommended JVM overhead (in MiB). Computed as total memory + - heap. + format: int64 + type: integer + type: object Event: description: The metadata associated with a request. properties: @@ -33289,6 +33347,52 @@ components: x-enum-varnames: - ANY - ALL + RecommendationAttributes: + description: Attributes of the SPA Recommendation resource. Contains recommendations + for both driver and executor components. + properties: + driver: + $ref: '#/components/schemas/ComponentRecommendation' + executor: + $ref: '#/components/schemas/ComponentRecommendation' + required: + - driver + - executor + type: object + RecommendationData: + description: JSON:API resource object for SPA Recommendation. Includes type, + optional ID, and resource attributes with structured recommendations. + properties: + attributes: + $ref: '#/components/schemas/RecommendationAttributes' + id: + description: Resource identifier for the recommendation. Optional in responses. + type: string + type: + $ref: '#/components/schemas/RecommendationType' + required: + - type + - attributes + type: object + RecommendationDocument: + description: JSON:API document containing a single Recommendation resource. + Returned by SPA when the Spark Gateway requests recommendations. + properties: + data: + $ref: '#/components/schemas/RecommendationData' + required: + - data + type: object + RecommendationType: + default: recommendation + description: JSON:API resource type for Spark Pod Autosizing recommendations. + Identifies the Recommendation resource returned by SPA. + enum: + - recommendation + example: recommendation + type: string + x-enum-varnames: + - RECOMMENDATION RegisterAppKeyResponse: description: The response object after creating an app key registration. properties: @@ -69937,6 +70041,72 @@ paths: x-unstable: '**Note**: This feature is in private beta. To request access, use the request access form in the [Service Level Objectives](https://docs.datadoghq.com/service_management/service_level_objectives/#slo-csv-export) docs.' + /api/v2/spa/recommendations/{service}/{shard}: + get: + description: Retrieve resource recommendations for a Spark job. The caller (Spark + Gateway or DJM UI) provides a service name and shard identifier, and SPA returns + structured recommendations for driver and executor resources. + operationId: GetSPARecommendations + parameters: + - description: The shard tag for a spark job, which differentiates jobs within + the same service that have different resource needs + in: path + name: shard + required: true + schema: + type: string + - description: The service name for a spark job + in: path + name: service + required: true + schema: + type: string + responses: + '200': + content: + application/json: + example: + data: + attributes: + driver: + estimation: + cpu: + max: 1500 + p75: 1000 + p95: 1200 + ephemeral_storage: 896 + heap: 6144 + memory: 7168 + overhead: 1024 + executor: + estimation: + cpu: + max: 2000 + p75: 1200 + p95: 1500 + ephemeral_storage: 512 + heap: 3072 + memory: 4096 + overhead: 1024 + id: dedupeactivecontexts:adp_dedupeactivecontexts_org2 + type: recommendation + schema: + $ref: '#/components/schemas/RecommendationDocument' + description: OK + '400': + $ref: '#/components/responses/BadRequestResponse' + '403': + $ref: '#/components/responses/NotAuthorizedResponse' + '429': + $ref: '#/components/responses/TooManyRequestsResponse' + summary: Get SPA Recommendations + tags: + - Spa + x-menu-order: 1 + x-undo: + type: safe + x-unstable: '**Note**: This endpoint is in public beta and may change in the + future. It is not yet recommended for production use.' /api/v2/spans/analytics/aggregate: post: description: 'The API endpoint to aggregate spans into buckets and compute metrics @@ -73677,6 +73847,9 @@ tags: externalDocs: url: https://docs.datadoghq.com/service_catalog/service_definitions#metadata-schema-v30-beta name: Software Catalog +- description: SPA (Spark Pod Autosizing) API. Provides resource recommendations and + cost insights to help optimize Spark job configurations. + name: Spa - description: Search and aggregate your spans from your Datadog platform over HTTP. name: Spans - description: Manage configuration of [span-based metrics](https://app.datadoghq.com/apm/traces/generate-metrics) diff --git a/data/api/v2/full_spec_deref.json b/data/api/v2/full_spec_deref.json index 235c44e6d7b33..6239c1b28cdc8 100644 --- a/data/api/v2/full_spec_deref.json +++ b/data/api/v2/full_spec_deref.json @@ -57215,6 +57215,63 @@ } ] }, + "ComponentRecommendation": { + "description": "Resource recommendation for a single Spark component (driver or executor). Contains estimation data used to patch Spark job specs.", + "properties": { + "estimation": { + "description": "Recommended resource values for a Spark driver or executor, derived from recent real usage metrics. Used by SPA to propose more efficient pod sizing.", + "properties": { + "cpu": { + "description": "CPU usage statistics derived from historical Spark job metrics. Provides multiple estimates so users can choose between conservative and cost-saving risk profiles.", + "properties": { + "max": { + "description": "Maximum CPU usage observed for the job, expressed in millicores. This represents the upper bound of usage.", + "format": "int64", + "type": "integer" + }, + "p75": { + "description": "75th percentile of CPU usage (millicores). Represents a cost-saving configuration while covering most workloads.", + "format": "int64", + "type": "integer" + }, + "p95": { + "description": "95th percentile of CPU usage (millicores). Balances performance and cost, providing a safer margin than p75.", + "format": "int64", + "type": "integer" + } + }, + "type": "object", + "x-model-simple-name": "SpaCpu" + }, + "ephemeral_storage": { + "description": "Recommended ephemeral storage allocation (in MiB). Derived from job temporary storage patterns.", + "format": "int64", + "type": "integer" + }, + "heap": { + "description": "Recommended JVM heap size (in MiB).", + "format": "int64", + "type": "integer" + }, + "memory": { + "description": "Recommended total memory allocation (in MiB). Includes both heap and overhead.", + "format": "int64", + "type": "integer" + }, + "overhead": { + "description": "Recommended JVM overhead (in MiB). Computed as total memory - heap.", + "format": "int64", + "type": "integer" + } + }, + "type": "object" + } + }, + "required": [ + "estimation" + ], + "type": "object" + }, "ComponentType": { "description": "The UI component type.", "enum": [ @@ -61764,6 +61821,28 @@ "COST_BY_ORG" ] }, + "Cpu": { + "description": "CPU usage statistics derived from historical Spark job metrics. Provides multiple estimates so users can choose between conservative and cost-saving risk profiles.", + "properties": { + "max": { + "description": "Maximum CPU usage observed for the job, expressed in millicores. This represents the upper bound of usage.", + "format": "int64", + "type": "integer" + }, + "p75": { + "description": "75th percentile of CPU usage (millicores). Represents a cost-saving configuration while covering most workloads.", + "format": "int64", + "type": "integer" + }, + "p95": { + "description": "95th percentile of CPU usage (millicores). Balances performance and cost, providing a safer margin than p75.", + "format": "int64", + "type": "integer" + } + }, + "type": "object", + "x-model-simple-name": "SpaCpu" + }, "CreateActionConnectionRequest": { "description": "Request used to create an action connection.", "properties": { @@ -99633,6 +99712,54 @@ "ESCALATION_POLICY_STEPS" ] }, + "Estimation": { + "description": "Recommended resource values for a Spark driver or executor, derived from recent real usage metrics. Used by SPA to propose more efficient pod sizing.", + "properties": { + "cpu": { + "description": "CPU usage statistics derived from historical Spark job metrics. Provides multiple estimates so users can choose between conservative and cost-saving risk profiles.", + "properties": { + "max": { + "description": "Maximum CPU usage observed for the job, expressed in millicores. This represents the upper bound of usage.", + "format": "int64", + "type": "integer" + }, + "p75": { + "description": "75th percentile of CPU usage (millicores). Represents a cost-saving configuration while covering most workloads.", + "format": "int64", + "type": "integer" + }, + "p95": { + "description": "95th percentile of CPU usage (millicores). Balances performance and cost, providing a safer margin than p75.", + "format": "int64", + "type": "integer" + } + }, + "type": "object", + "x-model-simple-name": "SpaCpu" + }, + "ephemeral_storage": { + "description": "Recommended ephemeral storage allocation (in MiB). Derived from job temporary storage patterns.", + "format": "int64", + "type": "integer" + }, + "heap": { + "description": "Recommended JVM heap size (in MiB).", + "format": "int64", + "type": "integer" + }, + "memory": { + "description": "Recommended total memory allocation (in MiB). Includes both heap and overhead.", + "format": "int64", + "type": "integer" + }, + "overhead": { + "description": "Recommended JVM overhead (in MiB). Computed as total memory - heap.", + "format": "int64", + "type": "integer" + } + }, + "type": "object" + }, "Event": { "description": "The metadata associated with a request.", "properties": { @@ -240248,6 +240375,451 @@ "ALL" ] }, + "RecommendationAttributes": { + "description": "Attributes of the SPA Recommendation resource. Contains recommendations for both driver and executor components.", + "properties": { + "driver": { + "description": "Resource recommendation for a single Spark component (driver or executor). Contains estimation data used to patch Spark job specs.", + "properties": { + "estimation": { + "description": "Recommended resource values for a Spark driver or executor, derived from recent real usage metrics. Used by SPA to propose more efficient pod sizing.", + "properties": { + "cpu": { + "description": "CPU usage statistics derived from historical Spark job metrics. Provides multiple estimates so users can choose between conservative and cost-saving risk profiles.", + "properties": { + "max": { + "description": "Maximum CPU usage observed for the job, expressed in millicores. This represents the upper bound of usage.", + "format": "int64", + "type": "integer" + }, + "p75": { + "description": "75th percentile of CPU usage (millicores). Represents a cost-saving configuration while covering most workloads.", + "format": "int64", + "type": "integer" + }, + "p95": { + "description": "95th percentile of CPU usage (millicores). Balances performance and cost, providing a safer margin than p75.", + "format": "int64", + "type": "integer" + } + }, + "type": "object", + "x-model-simple-name": "SpaCpu" + }, + "ephemeral_storage": { + "description": "Recommended ephemeral storage allocation (in MiB). Derived from job temporary storage patterns.", + "format": "int64", + "type": "integer" + }, + "heap": { + "description": "Recommended JVM heap size (in MiB).", + "format": "int64", + "type": "integer" + }, + "memory": { + "description": "Recommended total memory allocation (in MiB). Includes both heap and overhead.", + "format": "int64", + "type": "integer" + }, + "overhead": { + "description": "Recommended JVM overhead (in MiB). Computed as total memory - heap.", + "format": "int64", + "type": "integer" + } + }, + "type": "object" + } + }, + "required": [ + "estimation" + ], + "type": "object" + }, + "executor": { + "description": "Resource recommendation for a single Spark component (driver or executor). Contains estimation data used to patch Spark job specs.", + "properties": { + "estimation": { + "description": "Recommended resource values for a Spark driver or executor, derived from recent real usage metrics. Used by SPA to propose more efficient pod sizing.", + "properties": { + "cpu": { + "description": "CPU usage statistics derived from historical Spark job metrics. Provides multiple estimates so users can choose between conservative and cost-saving risk profiles.", + "properties": { + "max": { + "description": "Maximum CPU usage observed for the job, expressed in millicores. This represents the upper bound of usage.", + "format": "int64", + "type": "integer" + }, + "p75": { + "description": "75th percentile of CPU usage (millicores). Represents a cost-saving configuration while covering most workloads.", + "format": "int64", + "type": "integer" + }, + "p95": { + "description": "95th percentile of CPU usage (millicores). Balances performance and cost, providing a safer margin than p75.", + "format": "int64", + "type": "integer" + } + }, + "type": "object", + "x-model-simple-name": "SpaCpu" + }, + "ephemeral_storage": { + "description": "Recommended ephemeral storage allocation (in MiB). Derived from job temporary storage patterns.", + "format": "int64", + "type": "integer" + }, + "heap": { + "description": "Recommended JVM heap size (in MiB).", + "format": "int64", + "type": "integer" + }, + "memory": { + "description": "Recommended total memory allocation (in MiB). Includes both heap and overhead.", + "format": "int64", + "type": "integer" + }, + "overhead": { + "description": "Recommended JVM overhead (in MiB). Computed as total memory - heap.", + "format": "int64", + "type": "integer" + } + }, + "type": "object" + } + }, + "required": [ + "estimation" + ], + "type": "object" + } + }, + "required": [ + "driver", + "executor" + ], + "type": "object" + }, + "RecommendationData": { + "description": "JSON:API resource object for SPA Recommendation. Includes type, optional ID, and resource attributes with structured recommendations.", + "properties": { + "attributes": { + "description": "Attributes of the SPA Recommendation resource. Contains recommendations for both driver and executor components.", + "properties": { + "driver": { + "description": "Resource recommendation for a single Spark component (driver or executor). Contains estimation data used to patch Spark job specs.", + "properties": { + "estimation": { + "description": "Recommended resource values for a Spark driver or executor, derived from recent real usage metrics. Used by SPA to propose more efficient pod sizing.", + "properties": { + "cpu": { + "description": "CPU usage statistics derived from historical Spark job metrics. Provides multiple estimates so users can choose between conservative and cost-saving risk profiles.", + "properties": { + "max": { + "description": "Maximum CPU usage observed for the job, expressed in millicores. This represents the upper bound of usage.", + "format": "int64", + "type": "integer" + }, + "p75": { + "description": "75th percentile of CPU usage (millicores). Represents a cost-saving configuration while covering most workloads.", + "format": "int64", + "type": "integer" + }, + "p95": { + "description": "95th percentile of CPU usage (millicores). Balances performance and cost, providing a safer margin than p75.", + "format": "int64", + "type": "integer" + } + }, + "type": "object", + "x-model-simple-name": "SpaCpu" + }, + "ephemeral_storage": { + "description": "Recommended ephemeral storage allocation (in MiB). Derived from job temporary storage patterns.", + "format": "int64", + "type": "integer" + }, + "heap": { + "description": "Recommended JVM heap size (in MiB).", + "format": "int64", + "type": "integer" + }, + "memory": { + "description": "Recommended total memory allocation (in MiB). Includes both heap and overhead.", + "format": "int64", + "type": "integer" + }, + "overhead": { + "description": "Recommended JVM overhead (in MiB). Computed as total memory - heap.", + "format": "int64", + "type": "integer" + } + }, + "type": "object" + } + }, + "required": [ + "estimation" + ], + "type": "object" + }, + "executor": { + "description": "Resource recommendation for a single Spark component (driver or executor). Contains estimation data used to patch Spark job specs.", + "properties": { + "estimation": { + "description": "Recommended resource values for a Spark driver or executor, derived from recent real usage metrics. Used by SPA to propose more efficient pod sizing.", + "properties": { + "cpu": { + "description": "CPU usage statistics derived from historical Spark job metrics. Provides multiple estimates so users can choose between conservative and cost-saving risk profiles.", + "properties": { + "max": { + "description": "Maximum CPU usage observed for the job, expressed in millicores. This represents the upper bound of usage.", + "format": "int64", + "type": "integer" + }, + "p75": { + "description": "75th percentile of CPU usage (millicores). Represents a cost-saving configuration while covering most workloads.", + "format": "int64", + "type": "integer" + }, + "p95": { + "description": "95th percentile of CPU usage (millicores). Balances performance and cost, providing a safer margin than p75.", + "format": "int64", + "type": "integer" + } + }, + "type": "object", + "x-model-simple-name": "SpaCpu" + }, + "ephemeral_storage": { + "description": "Recommended ephemeral storage allocation (in MiB). Derived from job temporary storage patterns.", + "format": "int64", + "type": "integer" + }, + "heap": { + "description": "Recommended JVM heap size (in MiB).", + "format": "int64", + "type": "integer" + }, + "memory": { + "description": "Recommended total memory allocation (in MiB). Includes both heap and overhead.", + "format": "int64", + "type": "integer" + }, + "overhead": { + "description": "Recommended JVM overhead (in MiB). Computed as total memory - heap.", + "format": "int64", + "type": "integer" + } + }, + "type": "object" + } + }, + "required": [ + "estimation" + ], + "type": "object" + } + }, + "required": [ + "driver", + "executor" + ], + "type": "object" + }, + "id": { + "description": "Resource identifier for the recommendation. Optional in responses.", + "type": "string" + }, + "type": { + "default": "recommendation", + "description": "JSON:API resource type for Spark Pod Autosizing recommendations. Identifies the Recommendation resource returned by SPA.", + "enum": [ + "recommendation" + ], + "example": "recommendation", + "type": "string", + "x-enum-varnames": [ + "RECOMMENDATION" + ] + } + }, + "required": [ + "type", + "attributes" + ], + "type": "object" + }, + "RecommendationDocument": { + "description": "JSON:API document containing a single Recommendation resource. Returned by SPA when the Spark Gateway requests recommendations.", + "properties": { + "data": { + "description": "JSON:API resource object for SPA Recommendation. Includes type, optional ID, and resource attributes with structured recommendations.", + "properties": { + "attributes": { + "description": "Attributes of the SPA Recommendation resource. Contains recommendations for both driver and executor components.", + "properties": { + "driver": { + "description": "Resource recommendation for a single Spark component (driver or executor). Contains estimation data used to patch Spark job specs.", + "properties": { + "estimation": { + "description": "Recommended resource values for a Spark driver or executor, derived from recent real usage metrics. Used by SPA to propose more efficient pod sizing.", + "properties": { + "cpu": { + "description": "CPU usage statistics derived from historical Spark job metrics. Provides multiple estimates so users can choose between conservative and cost-saving risk profiles.", + "properties": { + "max": { + "description": "Maximum CPU usage observed for the job, expressed in millicores. This represents the upper bound of usage.", + "format": "int64", + "type": "integer" + }, + "p75": { + "description": "75th percentile of CPU usage (millicores). Represents a cost-saving configuration while covering most workloads.", + "format": "int64", + "type": "integer" + }, + "p95": { + "description": "95th percentile of CPU usage (millicores). Balances performance and cost, providing a safer margin than p75.", + "format": "int64", + "type": "integer" + } + }, + "type": "object", + "x-model-simple-name": "SpaCpu" + }, + "ephemeral_storage": { + "description": "Recommended ephemeral storage allocation (in MiB). Derived from job temporary storage patterns.", + "format": "int64", + "type": "integer" + }, + "heap": { + "description": "Recommended JVM heap size (in MiB).", + "format": "int64", + "type": "integer" + }, + "memory": { + "description": "Recommended total memory allocation (in MiB). Includes both heap and overhead.", + "format": "int64", + "type": "integer" + }, + "overhead": { + "description": "Recommended JVM overhead (in MiB). Computed as total memory - heap.", + "format": "int64", + "type": "integer" + } + }, + "type": "object" + } + }, + "required": [ + "estimation" + ], + "type": "object" + }, + "executor": { + "description": "Resource recommendation for a single Spark component (driver or executor). Contains estimation data used to patch Spark job specs.", + "properties": { + "estimation": { + "description": "Recommended resource values for a Spark driver or executor, derived from recent real usage metrics. Used by SPA to propose more efficient pod sizing.", + "properties": { + "cpu": { + "description": "CPU usage statistics derived from historical Spark job metrics. Provides multiple estimates so users can choose between conservative and cost-saving risk profiles.", + "properties": { + "max": { + "description": "Maximum CPU usage observed for the job, expressed in millicores. This represents the upper bound of usage.", + "format": "int64", + "type": "integer" + }, + "p75": { + "description": "75th percentile of CPU usage (millicores). Represents a cost-saving configuration while covering most workloads.", + "format": "int64", + "type": "integer" + }, + "p95": { + "description": "95th percentile of CPU usage (millicores). Balances performance and cost, providing a safer margin than p75.", + "format": "int64", + "type": "integer" + } + }, + "type": "object", + "x-model-simple-name": "SpaCpu" + }, + "ephemeral_storage": { + "description": "Recommended ephemeral storage allocation (in MiB). Derived from job temporary storage patterns.", + "format": "int64", + "type": "integer" + }, + "heap": { + "description": "Recommended JVM heap size (in MiB).", + "format": "int64", + "type": "integer" + }, + "memory": { + "description": "Recommended total memory allocation (in MiB). Includes both heap and overhead.", + "format": "int64", + "type": "integer" + }, + "overhead": { + "description": "Recommended JVM overhead (in MiB). Computed as total memory - heap.", + "format": "int64", + "type": "integer" + } + }, + "type": "object" + } + }, + "required": [ + "estimation" + ], + "type": "object" + } + }, + "required": [ + "driver", + "executor" + ], + "type": "object" + }, + "id": { + "description": "Resource identifier for the recommendation. Optional in responses.", + "type": "string" + }, + "type": { + "default": "recommendation", + "description": "JSON:API resource type for Spark Pod Autosizing recommendations. Identifies the Recommendation resource returned by SPA.", + "enum": [ + "recommendation" + ], + "example": "recommendation", + "type": "string", + "x-enum-varnames": [ + "RECOMMENDATION" + ] + } + }, + "required": [ + "type", + "attributes" + ], + "type": "object" + } + }, + "required": [ + "data" + ], + "type": "object" + }, + "RecommendationType": { + "default": "recommendation", + "description": "JSON:API resource type for Spark Pod Autosizing recommendations. Identifies the Recommendation resource returned by SPA.", + "enum": [ + "recommendation" + ], + "example": "recommendation", + "type": "string", + "x-enum-varnames": [ + "RECOMMENDATION" + ] + }, "RegisterAppKeyResponse": { "description": "The response object after creating an app key registration.", "properties": { @@ -605706,7 +606278,186 @@ } } }, - "description": "Forbidden" + "description": "Forbidden" + }, + "429": { + "content": { + "application/json": { + "schema": { + "description": "API error response.", + "properties": { + "errors": { + "description": "A list of errors.", + "example": [ + "Bad Request" + ], + "items": { + "description": "A list of items.", + "example": "Bad Request", + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "errors" + ], + "type": "object" + } + } + }, + "description": "Too many requests" + } + }, + "security": [ + { + "apiKeyAuth": [], + "appKeyAuth": [] + }, + { + "AuthZ": [ + "slos_read" + ] + } + ], + "summary": "Create a new SLO report", + "tags": [ + "Service Level Objectives" + ], + "x-codegen-request-body-name": "body", + "x-given": { + "report": { + "parameters": [ + { + "name": "body", + "value": "{\n \"data\": {\n \"attributes\": {\n \"query\": \"slo_type:metric \\\"SLO Reporting Test\\\"\",\n \"interval\": \"monthly\",\n \"from_ts\": {{ timestamp(\"now - 21d\") }},\n \"to_ts\": {{ timestamp(\"now\") }}\n }\n }\n}" + } + ], + "step": "there is a valid \"report\" in the system" + } + }, + "x-menu-order": 101, + "x-permission": { + "operator": "OR", + "permissions": [ + "slos_read" + ] + }, + "x-undo": { + "type": "safe" + }, + "x-unstable": "**Note**: This feature is in private beta. To request access, use the request access form in the [Service Level Objectives](https://docs.datadoghq.com/service_management/service_level_objectives/#slo-csv-export) docs." + } + }, + "/api/v2/slo/report/{report_id}/download": { + "get": { + "description": "Download an SLO report. This can only be performed after the report job has completed.\n\nReports are not guaranteed to exist indefinitely. Datadog recommends that you download the report as soon as it is available.", + "operationId": "GetSLOReport", + "parameters": [ + { + "description": "The ID of the report job.", + "in": "path", + "name": "report_id", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "text/csv": { + "schema": { + "type": "string" + } + } + }, + "description": "OK" + }, + "400": { + "content": { + "application/json": { + "schema": { + "description": "API error response.", + "properties": { + "errors": { + "description": "A list of errors.", + "example": [ + "Bad Request" + ], + "items": { + "description": "A list of items.", + "example": "Bad Request", + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "errors" + ], + "type": "object" + } + } + }, + "description": "Bad Request" + }, + "403": { + "content": { + "application/json": { + "schema": { + "description": "API error response.", + "properties": { + "errors": { + "description": "A list of errors.", + "example": [ + "Bad Request" + ], + "items": { + "description": "A list of items.", + "example": "Bad Request", + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "errors" + ], + "type": "object" + } + } + }, + "description": "Forbidden" + }, + "404": { + "content": { + "application/json": { + "schema": { + "description": "API error response.", + "properties": { + "errors": { + "description": "A list of errors.", + "example": [ + "Bad Request" + ], + "items": { + "description": "A list of items.", + "example": "Bad Request", + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "errors" + ], + "type": "object" + } + } + }, + "description": "Not Found" }, "429": { "content": { @@ -605748,39 +606499,21 @@ ] } ], - "summary": "Create a new SLO report", + "summary": "Get SLO report", "tags": [ "Service Level Objectives" ], - "x-codegen-request-body-name": "body", - "x-given": { - "report": { - "parameters": [ - { - "name": "body", - "value": "{\n \"data\": {\n \"attributes\": {\n \"query\": \"slo_type:metric \\\"SLO Reporting Test\\\"\",\n \"interval\": \"monthly\",\n \"from_ts\": {{ timestamp(\"now - 21d\") }},\n \"to_ts\": {{ timestamp(\"now\") }}\n }\n }\n}" - } - ], - "step": "there is a valid \"report\" in the system" - } - }, - "x-menu-order": 101, - "x-permission": { - "operator": "OR", - "permissions": [ - "slos_read" - ] - }, + "x-menu-order": 103, "x-undo": { "type": "safe" }, "x-unstable": "**Note**: This feature is in private beta. To request access, use the request access form in the [Service Level Objectives](https://docs.datadoghq.com/service_management/service_level_objectives/#slo-csv-export) docs." } }, - "/api/v2/slo/report/{report_id}/download": { + "/api/v2/slo/report/{report_id}/status": { "get": { - "description": "Download an SLO report. This can only be performed after the report job has completed.\n\nReports are not guaranteed to exist indefinitely. Datadog recommends that you download the report as soon as it is available.", - "operationId": "GetSLOReport", + "description": "Get the status of the SLO report job.", + "operationId": "GetSLOReportJobStatus", "parameters": [ { "description": "The ID of the report job.", @@ -605795,9 +606528,51 @@ "responses": { "200": { "content": { - "text/csv": { + "application/json": { "schema": { - "type": "string" + "description": "The SLO report status response.", + "properties": { + "data": { + "description": "The data portion of the SLO report status response.", + "properties": { + "attributes": { + "description": "The attributes portion of the SLO report status response.", + "properties": { + "status": { + "description": "The status of the SLO report job.", + "enum": [ + "in_progress", + "completed", + "completed_with_errors", + "failed" + ], + "example": "completed", + "type": "string", + "x-enum-varnames": [ + "IN_PROGRESS", + "COMPLETED", + "COMPLETED_WITH_ERRORS", + "FAILED" + ] + } + }, + "type": "object" + }, + "id": { + "description": "The ID of the report job.", + "example": "dc8d92aa-e0af-11ee-af21-1feeaccaa3a3", + "type": "string" + }, + "type": { + "description": "The type of ID.", + "example": "report_id", + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" } } }, @@ -605927,26 +606702,35 @@ ] } ], - "summary": "Get SLO report", + "summary": "Get SLO report status", "tags": [ "Service Level Objectives" ], - "x-menu-order": 103, + "x-menu-order": 102, "x-undo": { "type": "safe" }, "x-unstable": "**Note**: This feature is in private beta. To request access, use the request access form in the [Service Level Objectives](https://docs.datadoghq.com/service_management/service_level_objectives/#slo-csv-export) docs." } }, - "/api/v2/slo/report/{report_id}/status": { + "/api/v2/spa/recommendations/{service}/{shard}": { "get": { - "description": "Get the status of the SLO report job.", - "operationId": "GetSLOReportJobStatus", + "description": "Retrieve resource recommendations for a Spark job. The caller (Spark Gateway or DJM UI) provides a service name and shard identifier, and SPA returns structured recommendations for driver and executor resources.", + "operationId": "GetSPARecommendations", "parameters": [ { - "description": "The ID of the report job.", + "description": "The shard tag for a spark job, which differentiates jobs within the same service that have different resource needs", "in": "path", - "name": "report_id", + "name": "shard", + "required": true, + "schema": { + "type": "string" + } + }, + { + "description": "The service name for a spark job", + "in": "path", + "name": "service", "required": true, "schema": { "type": "string" @@ -605957,49 +606741,197 @@ "200": { "content": { "application/json": { + "example": { + "data": { + "attributes": { + "driver": { + "estimation": { + "cpu": { + "max": 1500, + "p75": 1000, + "p95": 1200 + }, + "ephemeral_storage": 896, + "heap": 6144, + "memory": 7168, + "overhead": 1024 + } + }, + "executor": { + "estimation": { + "cpu": { + "max": 2000, + "p75": 1200, + "p95": 1500 + }, + "ephemeral_storage": 512, + "heap": 3072, + "memory": 4096, + "overhead": 1024 + } + } + }, + "id": "dedupeactivecontexts:adp_dedupeactivecontexts_org2", + "type": "recommendation" + } + }, "schema": { - "description": "The SLO report status response.", + "description": "JSON:API document containing a single Recommendation resource. Returned by SPA when the Spark Gateway requests recommendations.", "properties": { "data": { - "description": "The data portion of the SLO report status response.", + "description": "JSON:API resource object for SPA Recommendation. Includes type, optional ID, and resource attributes with structured recommendations.", "properties": { "attributes": { - "description": "The attributes portion of the SLO report status response.", + "description": "Attributes of the SPA Recommendation resource. Contains recommendations for both driver and executor components.", "properties": { - "status": { - "description": "The status of the SLO report job.", - "enum": [ - "in_progress", - "completed", - "completed_with_errors", - "failed" + "driver": { + "description": "Resource recommendation for a single Spark component (driver or executor). Contains estimation data used to patch Spark job specs.", + "properties": { + "estimation": { + "description": "Recommended resource values for a Spark driver or executor, derived from recent real usage metrics. Used by SPA to propose more efficient pod sizing.", + "properties": { + "cpu": { + "description": "CPU usage statistics derived from historical Spark job metrics. Provides multiple estimates so users can choose between conservative and cost-saving risk profiles.", + "properties": { + "max": { + "description": "Maximum CPU usage observed for the job, expressed in millicores. This represents the upper bound of usage.", + "format": "int64", + "type": "integer" + }, + "p75": { + "description": "75th percentile of CPU usage (millicores). Represents a cost-saving configuration while covering most workloads.", + "format": "int64", + "type": "integer" + }, + "p95": { + "description": "95th percentile of CPU usage (millicores). Balances performance and cost, providing a safer margin than p75.", + "format": "int64", + "type": "integer" + } + }, + "type": "object", + "x-model-simple-name": "SpaCpu" + }, + "ephemeral_storage": { + "description": "Recommended ephemeral storage allocation (in MiB). Derived from job temporary storage patterns.", + "format": "int64", + "type": "integer" + }, + "heap": { + "description": "Recommended JVM heap size (in MiB).", + "format": "int64", + "type": "integer" + }, + "memory": { + "description": "Recommended total memory allocation (in MiB). Includes both heap and overhead.", + "format": "int64", + "type": "integer" + }, + "overhead": { + "description": "Recommended JVM overhead (in MiB). Computed as total memory - heap.", + "format": "int64", + "type": "integer" + } + }, + "type": "object" + } + }, + "required": [ + "estimation" ], - "example": "completed", - "type": "string", - "x-enum-varnames": [ - "IN_PROGRESS", - "COMPLETED", - "COMPLETED_WITH_ERRORS", - "FAILED" - ] + "type": "object" + }, + "executor": { + "description": "Resource recommendation for a single Spark component (driver or executor). Contains estimation data used to patch Spark job specs.", + "properties": { + "estimation": { + "description": "Recommended resource values for a Spark driver or executor, derived from recent real usage metrics. Used by SPA to propose more efficient pod sizing.", + "properties": { + "cpu": { + "description": "CPU usage statistics derived from historical Spark job metrics. Provides multiple estimates so users can choose between conservative and cost-saving risk profiles.", + "properties": { + "max": { + "description": "Maximum CPU usage observed for the job, expressed in millicores. This represents the upper bound of usage.", + "format": "int64", + "type": "integer" + }, + "p75": { + "description": "75th percentile of CPU usage (millicores). Represents a cost-saving configuration while covering most workloads.", + "format": "int64", + "type": "integer" + }, + "p95": { + "description": "95th percentile of CPU usage (millicores). Balances performance and cost, providing a safer margin than p75.", + "format": "int64", + "type": "integer" + } + }, + "type": "object", + "x-model-simple-name": "SpaCpu" + }, + "ephemeral_storage": { + "description": "Recommended ephemeral storage allocation (in MiB). Derived from job temporary storage patterns.", + "format": "int64", + "type": "integer" + }, + "heap": { + "description": "Recommended JVM heap size (in MiB).", + "format": "int64", + "type": "integer" + }, + "memory": { + "description": "Recommended total memory allocation (in MiB). Includes both heap and overhead.", + "format": "int64", + "type": "integer" + }, + "overhead": { + "description": "Recommended JVM overhead (in MiB). Computed as total memory - heap.", + "format": "int64", + "type": "integer" + } + }, + "type": "object" + } + }, + "required": [ + "estimation" + ], + "type": "object" } }, + "required": [ + "driver", + "executor" + ], "type": "object" }, "id": { - "description": "The ID of the report job.", - "example": "dc8d92aa-e0af-11ee-af21-1feeaccaa3a3", + "description": "Resource identifier for the recommendation. Optional in responses.", "type": "string" }, "type": { - "description": "The type of ID.", - "example": "report_id", - "type": "string" + "default": "recommendation", + "description": "JSON:API resource type for Spark Pod Autosizing recommendations. Identifies the Recommendation resource returned by SPA.", + "enum": [ + "recommendation" + ], + "example": "recommendation", + "type": "string", + "x-enum-varnames": [ + "RECOMMENDATION" + ] } }, + "required": [ + "type", + "attributes" + ], "type": "object" } }, + "required": [ + "data" + ], "type": "object" } } @@ -606060,35 +606992,7 @@ } } }, - "description": "Forbidden" - }, - "404": { - "content": { - "application/json": { - "schema": { - "description": "API error response.", - "properties": { - "errors": { - "description": "A list of errors.", - "example": [ - "Bad Request" - ], - "items": { - "description": "A list of items.", - "example": "Bad Request", - "type": "string" - }, - "type": "array" - } - }, - "required": [ - "errors" - ], - "type": "object" - } - } - }, - "description": "Not Found" + "description": "Not Authorized" }, "429": { "content": { @@ -606119,26 +607023,15 @@ "description": "Too many requests" } }, - "security": [ - { - "apiKeyAuth": [], - "appKeyAuth": [] - }, - { - "AuthZ": [ - "slos_read" - ] - } - ], - "summary": "Get SLO report status", + "summary": "Get SPA Recommendations", "tags": [ - "Service Level Objectives" + "Spa" ], - "x-menu-order": 102, + "x-menu-order": 1, "x-undo": { "type": "safe" }, - "x-unstable": "**Note**: This feature is in private beta. To request access, use the request access form in the [Service Level Objectives](https://docs.datadoghq.com/service_management/service_level_objectives/#slo-csv-export) docs." + "x-unstable": "**Note**: This endpoint is in public beta and may change in the future. It is not yet recommended for production use." } }, "/api/v2/spans/analytics/aggregate": { @@ -635654,6 +636547,10 @@ }, "name": "Software Catalog" }, + { + "description": "SPA (Spark Pod Autosizing) API. Provides resource recommendations and cost insights to help optimize Spark job configurations.", + "name": "Spa" + }, { "description": "Search and aggregate your spans from your Datadog platform over HTTP.", "name": "Spans" diff --git a/data/api/v2/translate_actions.json b/data/api/v2/translate_actions.json index ad8670310718c..f74bc197fbf2e 100644 --- a/data/api/v2/translate_actions.json +++ b/data/api/v2/translate_actions.json @@ -2427,6 +2427,10 @@ "description": "Get the status of the SLO report job.", "summary": "Get SLO report status" }, + "GetSPARecommendations": { + "description": "Retrieve resource recommendations for a Spark job. The caller (Spark Gateway or DJM UI) provides a service name and shard identifier, and SPA returns structured recommendations for driver and executor resources.", + "summary": "Get SPA Recommendations" + }, "AggregateSpans": { "description": "The API endpoint to aggregate spans into buckets and compute metrics and timeseries.\nThis endpoint is rate limited to `300` requests per hour.", "summary": "Aggregate spans", diff --git a/data/api/v2/translate_tags.json b/data/api/v2/translate_tags.json index ab1ed845edd0e..adad7a8cf28d5 100644 --- a/data/api/v2/translate_tags.json +++ b/data/api/v2/translate_tags.json @@ -259,6 +259,10 @@ "name": "Software Catalog", "description": "API to create, update, retrieve, and delete Software Catalog entities." }, + "spa": { + "name": "Spa", + "description": "SPA (Spark Pod Autosizing) API. Provides resource recommendations and cost insights to help optimize Spark job configurations." + }, "spans": { "name": "Spans", "description": "Search and aggregate your spans from your Datadog platform over HTTP." diff --git a/static/resources/json/full_spec_v2.json b/static/resources/json/full_spec_v2.json index 235c44e6d7b33..6239c1b28cdc8 100644 --- a/static/resources/json/full_spec_v2.json +++ b/static/resources/json/full_spec_v2.json @@ -57215,6 +57215,63 @@ } ] }, + "ComponentRecommendation": { + "description": "Resource recommendation for a single Spark component (driver or executor). Contains estimation data used to patch Spark job specs.", + "properties": { + "estimation": { + "description": "Recommended resource values for a Spark driver or executor, derived from recent real usage metrics. Used by SPA to propose more efficient pod sizing.", + "properties": { + "cpu": { + "description": "CPU usage statistics derived from historical Spark job metrics. Provides multiple estimates so users can choose between conservative and cost-saving risk profiles.", + "properties": { + "max": { + "description": "Maximum CPU usage observed for the job, expressed in millicores. This represents the upper bound of usage.", + "format": "int64", + "type": "integer" + }, + "p75": { + "description": "75th percentile of CPU usage (millicores). Represents a cost-saving configuration while covering most workloads.", + "format": "int64", + "type": "integer" + }, + "p95": { + "description": "95th percentile of CPU usage (millicores). Balances performance and cost, providing a safer margin than p75.", + "format": "int64", + "type": "integer" + } + }, + "type": "object", + "x-model-simple-name": "SpaCpu" + }, + "ephemeral_storage": { + "description": "Recommended ephemeral storage allocation (in MiB). Derived from job temporary storage patterns.", + "format": "int64", + "type": "integer" + }, + "heap": { + "description": "Recommended JVM heap size (in MiB).", + "format": "int64", + "type": "integer" + }, + "memory": { + "description": "Recommended total memory allocation (in MiB). Includes both heap and overhead.", + "format": "int64", + "type": "integer" + }, + "overhead": { + "description": "Recommended JVM overhead (in MiB). Computed as total memory - heap.", + "format": "int64", + "type": "integer" + } + }, + "type": "object" + } + }, + "required": [ + "estimation" + ], + "type": "object" + }, "ComponentType": { "description": "The UI component type.", "enum": [ @@ -61764,6 +61821,28 @@ "COST_BY_ORG" ] }, + "Cpu": { + "description": "CPU usage statistics derived from historical Spark job metrics. Provides multiple estimates so users can choose between conservative and cost-saving risk profiles.", + "properties": { + "max": { + "description": "Maximum CPU usage observed for the job, expressed in millicores. This represents the upper bound of usage.", + "format": "int64", + "type": "integer" + }, + "p75": { + "description": "75th percentile of CPU usage (millicores). Represents a cost-saving configuration while covering most workloads.", + "format": "int64", + "type": "integer" + }, + "p95": { + "description": "95th percentile of CPU usage (millicores). Balances performance and cost, providing a safer margin than p75.", + "format": "int64", + "type": "integer" + } + }, + "type": "object", + "x-model-simple-name": "SpaCpu" + }, "CreateActionConnectionRequest": { "description": "Request used to create an action connection.", "properties": { @@ -99633,6 +99712,54 @@ "ESCALATION_POLICY_STEPS" ] }, + "Estimation": { + "description": "Recommended resource values for a Spark driver or executor, derived from recent real usage metrics. Used by SPA to propose more efficient pod sizing.", + "properties": { + "cpu": { + "description": "CPU usage statistics derived from historical Spark job metrics. Provides multiple estimates so users can choose between conservative and cost-saving risk profiles.", + "properties": { + "max": { + "description": "Maximum CPU usage observed for the job, expressed in millicores. This represents the upper bound of usage.", + "format": "int64", + "type": "integer" + }, + "p75": { + "description": "75th percentile of CPU usage (millicores). Represents a cost-saving configuration while covering most workloads.", + "format": "int64", + "type": "integer" + }, + "p95": { + "description": "95th percentile of CPU usage (millicores). Balances performance and cost, providing a safer margin than p75.", + "format": "int64", + "type": "integer" + } + }, + "type": "object", + "x-model-simple-name": "SpaCpu" + }, + "ephemeral_storage": { + "description": "Recommended ephemeral storage allocation (in MiB). Derived from job temporary storage patterns.", + "format": "int64", + "type": "integer" + }, + "heap": { + "description": "Recommended JVM heap size (in MiB).", + "format": "int64", + "type": "integer" + }, + "memory": { + "description": "Recommended total memory allocation (in MiB). Includes both heap and overhead.", + "format": "int64", + "type": "integer" + }, + "overhead": { + "description": "Recommended JVM overhead (in MiB). Computed as total memory - heap.", + "format": "int64", + "type": "integer" + } + }, + "type": "object" + }, "Event": { "description": "The metadata associated with a request.", "properties": { @@ -240248,6 +240375,451 @@ "ALL" ] }, + "RecommendationAttributes": { + "description": "Attributes of the SPA Recommendation resource. Contains recommendations for both driver and executor components.", + "properties": { + "driver": { + "description": "Resource recommendation for a single Spark component (driver or executor). Contains estimation data used to patch Spark job specs.", + "properties": { + "estimation": { + "description": "Recommended resource values for a Spark driver or executor, derived from recent real usage metrics. Used by SPA to propose more efficient pod sizing.", + "properties": { + "cpu": { + "description": "CPU usage statistics derived from historical Spark job metrics. Provides multiple estimates so users can choose between conservative and cost-saving risk profiles.", + "properties": { + "max": { + "description": "Maximum CPU usage observed for the job, expressed in millicores. This represents the upper bound of usage.", + "format": "int64", + "type": "integer" + }, + "p75": { + "description": "75th percentile of CPU usage (millicores). Represents a cost-saving configuration while covering most workloads.", + "format": "int64", + "type": "integer" + }, + "p95": { + "description": "95th percentile of CPU usage (millicores). Balances performance and cost, providing a safer margin than p75.", + "format": "int64", + "type": "integer" + } + }, + "type": "object", + "x-model-simple-name": "SpaCpu" + }, + "ephemeral_storage": { + "description": "Recommended ephemeral storage allocation (in MiB). Derived from job temporary storage patterns.", + "format": "int64", + "type": "integer" + }, + "heap": { + "description": "Recommended JVM heap size (in MiB).", + "format": "int64", + "type": "integer" + }, + "memory": { + "description": "Recommended total memory allocation (in MiB). Includes both heap and overhead.", + "format": "int64", + "type": "integer" + }, + "overhead": { + "description": "Recommended JVM overhead (in MiB). Computed as total memory - heap.", + "format": "int64", + "type": "integer" + } + }, + "type": "object" + } + }, + "required": [ + "estimation" + ], + "type": "object" + }, + "executor": { + "description": "Resource recommendation for a single Spark component (driver or executor). Contains estimation data used to patch Spark job specs.", + "properties": { + "estimation": { + "description": "Recommended resource values for a Spark driver or executor, derived from recent real usage metrics. Used by SPA to propose more efficient pod sizing.", + "properties": { + "cpu": { + "description": "CPU usage statistics derived from historical Spark job metrics. Provides multiple estimates so users can choose between conservative and cost-saving risk profiles.", + "properties": { + "max": { + "description": "Maximum CPU usage observed for the job, expressed in millicores. This represents the upper bound of usage.", + "format": "int64", + "type": "integer" + }, + "p75": { + "description": "75th percentile of CPU usage (millicores). Represents a cost-saving configuration while covering most workloads.", + "format": "int64", + "type": "integer" + }, + "p95": { + "description": "95th percentile of CPU usage (millicores). Balances performance and cost, providing a safer margin than p75.", + "format": "int64", + "type": "integer" + } + }, + "type": "object", + "x-model-simple-name": "SpaCpu" + }, + "ephemeral_storage": { + "description": "Recommended ephemeral storage allocation (in MiB). Derived from job temporary storage patterns.", + "format": "int64", + "type": "integer" + }, + "heap": { + "description": "Recommended JVM heap size (in MiB).", + "format": "int64", + "type": "integer" + }, + "memory": { + "description": "Recommended total memory allocation (in MiB). Includes both heap and overhead.", + "format": "int64", + "type": "integer" + }, + "overhead": { + "description": "Recommended JVM overhead (in MiB). Computed as total memory - heap.", + "format": "int64", + "type": "integer" + } + }, + "type": "object" + } + }, + "required": [ + "estimation" + ], + "type": "object" + } + }, + "required": [ + "driver", + "executor" + ], + "type": "object" + }, + "RecommendationData": { + "description": "JSON:API resource object for SPA Recommendation. Includes type, optional ID, and resource attributes with structured recommendations.", + "properties": { + "attributes": { + "description": "Attributes of the SPA Recommendation resource. Contains recommendations for both driver and executor components.", + "properties": { + "driver": { + "description": "Resource recommendation for a single Spark component (driver or executor). Contains estimation data used to patch Spark job specs.", + "properties": { + "estimation": { + "description": "Recommended resource values for a Spark driver or executor, derived from recent real usage metrics. Used by SPA to propose more efficient pod sizing.", + "properties": { + "cpu": { + "description": "CPU usage statistics derived from historical Spark job metrics. Provides multiple estimates so users can choose between conservative and cost-saving risk profiles.", + "properties": { + "max": { + "description": "Maximum CPU usage observed for the job, expressed in millicores. This represents the upper bound of usage.", + "format": "int64", + "type": "integer" + }, + "p75": { + "description": "75th percentile of CPU usage (millicores). Represents a cost-saving configuration while covering most workloads.", + "format": "int64", + "type": "integer" + }, + "p95": { + "description": "95th percentile of CPU usage (millicores). Balances performance and cost, providing a safer margin than p75.", + "format": "int64", + "type": "integer" + } + }, + "type": "object", + "x-model-simple-name": "SpaCpu" + }, + "ephemeral_storage": { + "description": "Recommended ephemeral storage allocation (in MiB). Derived from job temporary storage patterns.", + "format": "int64", + "type": "integer" + }, + "heap": { + "description": "Recommended JVM heap size (in MiB).", + "format": "int64", + "type": "integer" + }, + "memory": { + "description": "Recommended total memory allocation (in MiB). Includes both heap and overhead.", + "format": "int64", + "type": "integer" + }, + "overhead": { + "description": "Recommended JVM overhead (in MiB). Computed as total memory - heap.", + "format": "int64", + "type": "integer" + } + }, + "type": "object" + } + }, + "required": [ + "estimation" + ], + "type": "object" + }, + "executor": { + "description": "Resource recommendation for a single Spark component (driver or executor). Contains estimation data used to patch Spark job specs.", + "properties": { + "estimation": { + "description": "Recommended resource values for a Spark driver or executor, derived from recent real usage metrics. Used by SPA to propose more efficient pod sizing.", + "properties": { + "cpu": { + "description": "CPU usage statistics derived from historical Spark job metrics. Provides multiple estimates so users can choose between conservative and cost-saving risk profiles.", + "properties": { + "max": { + "description": "Maximum CPU usage observed for the job, expressed in millicores. This represents the upper bound of usage.", + "format": "int64", + "type": "integer" + }, + "p75": { + "description": "75th percentile of CPU usage (millicores). Represents a cost-saving configuration while covering most workloads.", + "format": "int64", + "type": "integer" + }, + "p95": { + "description": "95th percentile of CPU usage (millicores). Balances performance and cost, providing a safer margin than p75.", + "format": "int64", + "type": "integer" + } + }, + "type": "object", + "x-model-simple-name": "SpaCpu" + }, + "ephemeral_storage": { + "description": "Recommended ephemeral storage allocation (in MiB). Derived from job temporary storage patterns.", + "format": "int64", + "type": "integer" + }, + "heap": { + "description": "Recommended JVM heap size (in MiB).", + "format": "int64", + "type": "integer" + }, + "memory": { + "description": "Recommended total memory allocation (in MiB). Includes both heap and overhead.", + "format": "int64", + "type": "integer" + }, + "overhead": { + "description": "Recommended JVM overhead (in MiB). Computed as total memory - heap.", + "format": "int64", + "type": "integer" + } + }, + "type": "object" + } + }, + "required": [ + "estimation" + ], + "type": "object" + } + }, + "required": [ + "driver", + "executor" + ], + "type": "object" + }, + "id": { + "description": "Resource identifier for the recommendation. Optional in responses.", + "type": "string" + }, + "type": { + "default": "recommendation", + "description": "JSON:API resource type for Spark Pod Autosizing recommendations. Identifies the Recommendation resource returned by SPA.", + "enum": [ + "recommendation" + ], + "example": "recommendation", + "type": "string", + "x-enum-varnames": [ + "RECOMMENDATION" + ] + } + }, + "required": [ + "type", + "attributes" + ], + "type": "object" + }, + "RecommendationDocument": { + "description": "JSON:API document containing a single Recommendation resource. Returned by SPA when the Spark Gateway requests recommendations.", + "properties": { + "data": { + "description": "JSON:API resource object for SPA Recommendation. Includes type, optional ID, and resource attributes with structured recommendations.", + "properties": { + "attributes": { + "description": "Attributes of the SPA Recommendation resource. Contains recommendations for both driver and executor components.", + "properties": { + "driver": { + "description": "Resource recommendation for a single Spark component (driver or executor). Contains estimation data used to patch Spark job specs.", + "properties": { + "estimation": { + "description": "Recommended resource values for a Spark driver or executor, derived from recent real usage metrics. Used by SPA to propose more efficient pod sizing.", + "properties": { + "cpu": { + "description": "CPU usage statistics derived from historical Spark job metrics. Provides multiple estimates so users can choose between conservative and cost-saving risk profiles.", + "properties": { + "max": { + "description": "Maximum CPU usage observed for the job, expressed in millicores. This represents the upper bound of usage.", + "format": "int64", + "type": "integer" + }, + "p75": { + "description": "75th percentile of CPU usage (millicores). Represents a cost-saving configuration while covering most workloads.", + "format": "int64", + "type": "integer" + }, + "p95": { + "description": "95th percentile of CPU usage (millicores). Balances performance and cost, providing a safer margin than p75.", + "format": "int64", + "type": "integer" + } + }, + "type": "object", + "x-model-simple-name": "SpaCpu" + }, + "ephemeral_storage": { + "description": "Recommended ephemeral storage allocation (in MiB). Derived from job temporary storage patterns.", + "format": "int64", + "type": "integer" + }, + "heap": { + "description": "Recommended JVM heap size (in MiB).", + "format": "int64", + "type": "integer" + }, + "memory": { + "description": "Recommended total memory allocation (in MiB). Includes both heap and overhead.", + "format": "int64", + "type": "integer" + }, + "overhead": { + "description": "Recommended JVM overhead (in MiB). Computed as total memory - heap.", + "format": "int64", + "type": "integer" + } + }, + "type": "object" + } + }, + "required": [ + "estimation" + ], + "type": "object" + }, + "executor": { + "description": "Resource recommendation for a single Spark component (driver or executor). Contains estimation data used to patch Spark job specs.", + "properties": { + "estimation": { + "description": "Recommended resource values for a Spark driver or executor, derived from recent real usage metrics. Used by SPA to propose more efficient pod sizing.", + "properties": { + "cpu": { + "description": "CPU usage statistics derived from historical Spark job metrics. Provides multiple estimates so users can choose between conservative and cost-saving risk profiles.", + "properties": { + "max": { + "description": "Maximum CPU usage observed for the job, expressed in millicores. This represents the upper bound of usage.", + "format": "int64", + "type": "integer" + }, + "p75": { + "description": "75th percentile of CPU usage (millicores). Represents a cost-saving configuration while covering most workloads.", + "format": "int64", + "type": "integer" + }, + "p95": { + "description": "95th percentile of CPU usage (millicores). Balances performance and cost, providing a safer margin than p75.", + "format": "int64", + "type": "integer" + } + }, + "type": "object", + "x-model-simple-name": "SpaCpu" + }, + "ephemeral_storage": { + "description": "Recommended ephemeral storage allocation (in MiB). Derived from job temporary storage patterns.", + "format": "int64", + "type": "integer" + }, + "heap": { + "description": "Recommended JVM heap size (in MiB).", + "format": "int64", + "type": "integer" + }, + "memory": { + "description": "Recommended total memory allocation (in MiB). Includes both heap and overhead.", + "format": "int64", + "type": "integer" + }, + "overhead": { + "description": "Recommended JVM overhead (in MiB). Computed as total memory - heap.", + "format": "int64", + "type": "integer" + } + }, + "type": "object" + } + }, + "required": [ + "estimation" + ], + "type": "object" + } + }, + "required": [ + "driver", + "executor" + ], + "type": "object" + }, + "id": { + "description": "Resource identifier for the recommendation. Optional in responses.", + "type": "string" + }, + "type": { + "default": "recommendation", + "description": "JSON:API resource type for Spark Pod Autosizing recommendations. Identifies the Recommendation resource returned by SPA.", + "enum": [ + "recommendation" + ], + "example": "recommendation", + "type": "string", + "x-enum-varnames": [ + "RECOMMENDATION" + ] + } + }, + "required": [ + "type", + "attributes" + ], + "type": "object" + } + }, + "required": [ + "data" + ], + "type": "object" + }, + "RecommendationType": { + "default": "recommendation", + "description": "JSON:API resource type for Spark Pod Autosizing recommendations. Identifies the Recommendation resource returned by SPA.", + "enum": [ + "recommendation" + ], + "example": "recommendation", + "type": "string", + "x-enum-varnames": [ + "RECOMMENDATION" + ] + }, "RegisterAppKeyResponse": { "description": "The response object after creating an app key registration.", "properties": { @@ -605706,7 +606278,186 @@ } } }, - "description": "Forbidden" + "description": "Forbidden" + }, + "429": { + "content": { + "application/json": { + "schema": { + "description": "API error response.", + "properties": { + "errors": { + "description": "A list of errors.", + "example": [ + "Bad Request" + ], + "items": { + "description": "A list of items.", + "example": "Bad Request", + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "errors" + ], + "type": "object" + } + } + }, + "description": "Too many requests" + } + }, + "security": [ + { + "apiKeyAuth": [], + "appKeyAuth": [] + }, + { + "AuthZ": [ + "slos_read" + ] + } + ], + "summary": "Create a new SLO report", + "tags": [ + "Service Level Objectives" + ], + "x-codegen-request-body-name": "body", + "x-given": { + "report": { + "parameters": [ + { + "name": "body", + "value": "{\n \"data\": {\n \"attributes\": {\n \"query\": \"slo_type:metric \\\"SLO Reporting Test\\\"\",\n \"interval\": \"monthly\",\n \"from_ts\": {{ timestamp(\"now - 21d\") }},\n \"to_ts\": {{ timestamp(\"now\") }}\n }\n }\n}" + } + ], + "step": "there is a valid \"report\" in the system" + } + }, + "x-menu-order": 101, + "x-permission": { + "operator": "OR", + "permissions": [ + "slos_read" + ] + }, + "x-undo": { + "type": "safe" + }, + "x-unstable": "**Note**: This feature is in private beta. To request access, use the request access form in the [Service Level Objectives](https://docs.datadoghq.com/service_management/service_level_objectives/#slo-csv-export) docs." + } + }, + "/api/v2/slo/report/{report_id}/download": { + "get": { + "description": "Download an SLO report. This can only be performed after the report job has completed.\n\nReports are not guaranteed to exist indefinitely. Datadog recommends that you download the report as soon as it is available.", + "operationId": "GetSLOReport", + "parameters": [ + { + "description": "The ID of the report job.", + "in": "path", + "name": "report_id", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "text/csv": { + "schema": { + "type": "string" + } + } + }, + "description": "OK" + }, + "400": { + "content": { + "application/json": { + "schema": { + "description": "API error response.", + "properties": { + "errors": { + "description": "A list of errors.", + "example": [ + "Bad Request" + ], + "items": { + "description": "A list of items.", + "example": "Bad Request", + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "errors" + ], + "type": "object" + } + } + }, + "description": "Bad Request" + }, + "403": { + "content": { + "application/json": { + "schema": { + "description": "API error response.", + "properties": { + "errors": { + "description": "A list of errors.", + "example": [ + "Bad Request" + ], + "items": { + "description": "A list of items.", + "example": "Bad Request", + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "errors" + ], + "type": "object" + } + } + }, + "description": "Forbidden" + }, + "404": { + "content": { + "application/json": { + "schema": { + "description": "API error response.", + "properties": { + "errors": { + "description": "A list of errors.", + "example": [ + "Bad Request" + ], + "items": { + "description": "A list of items.", + "example": "Bad Request", + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "errors" + ], + "type": "object" + } + } + }, + "description": "Not Found" }, "429": { "content": { @@ -605748,39 +606499,21 @@ ] } ], - "summary": "Create a new SLO report", + "summary": "Get SLO report", "tags": [ "Service Level Objectives" ], - "x-codegen-request-body-name": "body", - "x-given": { - "report": { - "parameters": [ - { - "name": "body", - "value": "{\n \"data\": {\n \"attributes\": {\n \"query\": \"slo_type:metric \\\"SLO Reporting Test\\\"\",\n \"interval\": \"monthly\",\n \"from_ts\": {{ timestamp(\"now - 21d\") }},\n \"to_ts\": {{ timestamp(\"now\") }}\n }\n }\n}" - } - ], - "step": "there is a valid \"report\" in the system" - } - }, - "x-menu-order": 101, - "x-permission": { - "operator": "OR", - "permissions": [ - "slos_read" - ] - }, + "x-menu-order": 103, "x-undo": { "type": "safe" }, "x-unstable": "**Note**: This feature is in private beta. To request access, use the request access form in the [Service Level Objectives](https://docs.datadoghq.com/service_management/service_level_objectives/#slo-csv-export) docs." } }, - "/api/v2/slo/report/{report_id}/download": { + "/api/v2/slo/report/{report_id}/status": { "get": { - "description": "Download an SLO report. This can only be performed after the report job has completed.\n\nReports are not guaranteed to exist indefinitely. Datadog recommends that you download the report as soon as it is available.", - "operationId": "GetSLOReport", + "description": "Get the status of the SLO report job.", + "operationId": "GetSLOReportJobStatus", "parameters": [ { "description": "The ID of the report job.", @@ -605795,9 +606528,51 @@ "responses": { "200": { "content": { - "text/csv": { + "application/json": { "schema": { - "type": "string" + "description": "The SLO report status response.", + "properties": { + "data": { + "description": "The data portion of the SLO report status response.", + "properties": { + "attributes": { + "description": "The attributes portion of the SLO report status response.", + "properties": { + "status": { + "description": "The status of the SLO report job.", + "enum": [ + "in_progress", + "completed", + "completed_with_errors", + "failed" + ], + "example": "completed", + "type": "string", + "x-enum-varnames": [ + "IN_PROGRESS", + "COMPLETED", + "COMPLETED_WITH_ERRORS", + "FAILED" + ] + } + }, + "type": "object" + }, + "id": { + "description": "The ID of the report job.", + "example": "dc8d92aa-e0af-11ee-af21-1feeaccaa3a3", + "type": "string" + }, + "type": { + "description": "The type of ID.", + "example": "report_id", + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" } } }, @@ -605927,26 +606702,35 @@ ] } ], - "summary": "Get SLO report", + "summary": "Get SLO report status", "tags": [ "Service Level Objectives" ], - "x-menu-order": 103, + "x-menu-order": 102, "x-undo": { "type": "safe" }, "x-unstable": "**Note**: This feature is in private beta. To request access, use the request access form in the [Service Level Objectives](https://docs.datadoghq.com/service_management/service_level_objectives/#slo-csv-export) docs." } }, - "/api/v2/slo/report/{report_id}/status": { + "/api/v2/spa/recommendations/{service}/{shard}": { "get": { - "description": "Get the status of the SLO report job.", - "operationId": "GetSLOReportJobStatus", + "description": "Retrieve resource recommendations for a Spark job. The caller (Spark Gateway or DJM UI) provides a service name and shard identifier, and SPA returns structured recommendations for driver and executor resources.", + "operationId": "GetSPARecommendations", "parameters": [ { - "description": "The ID of the report job.", + "description": "The shard tag for a spark job, which differentiates jobs within the same service that have different resource needs", "in": "path", - "name": "report_id", + "name": "shard", + "required": true, + "schema": { + "type": "string" + } + }, + { + "description": "The service name for a spark job", + "in": "path", + "name": "service", "required": true, "schema": { "type": "string" @@ -605957,49 +606741,197 @@ "200": { "content": { "application/json": { + "example": { + "data": { + "attributes": { + "driver": { + "estimation": { + "cpu": { + "max": 1500, + "p75": 1000, + "p95": 1200 + }, + "ephemeral_storage": 896, + "heap": 6144, + "memory": 7168, + "overhead": 1024 + } + }, + "executor": { + "estimation": { + "cpu": { + "max": 2000, + "p75": 1200, + "p95": 1500 + }, + "ephemeral_storage": 512, + "heap": 3072, + "memory": 4096, + "overhead": 1024 + } + } + }, + "id": "dedupeactivecontexts:adp_dedupeactivecontexts_org2", + "type": "recommendation" + } + }, "schema": { - "description": "The SLO report status response.", + "description": "JSON:API document containing a single Recommendation resource. Returned by SPA when the Spark Gateway requests recommendations.", "properties": { "data": { - "description": "The data portion of the SLO report status response.", + "description": "JSON:API resource object for SPA Recommendation. Includes type, optional ID, and resource attributes with structured recommendations.", "properties": { "attributes": { - "description": "The attributes portion of the SLO report status response.", + "description": "Attributes of the SPA Recommendation resource. Contains recommendations for both driver and executor components.", "properties": { - "status": { - "description": "The status of the SLO report job.", - "enum": [ - "in_progress", - "completed", - "completed_with_errors", - "failed" + "driver": { + "description": "Resource recommendation for a single Spark component (driver or executor). Contains estimation data used to patch Spark job specs.", + "properties": { + "estimation": { + "description": "Recommended resource values for a Spark driver or executor, derived from recent real usage metrics. Used by SPA to propose more efficient pod sizing.", + "properties": { + "cpu": { + "description": "CPU usage statistics derived from historical Spark job metrics. Provides multiple estimates so users can choose between conservative and cost-saving risk profiles.", + "properties": { + "max": { + "description": "Maximum CPU usage observed for the job, expressed in millicores. This represents the upper bound of usage.", + "format": "int64", + "type": "integer" + }, + "p75": { + "description": "75th percentile of CPU usage (millicores). Represents a cost-saving configuration while covering most workloads.", + "format": "int64", + "type": "integer" + }, + "p95": { + "description": "95th percentile of CPU usage (millicores). Balances performance and cost, providing a safer margin than p75.", + "format": "int64", + "type": "integer" + } + }, + "type": "object", + "x-model-simple-name": "SpaCpu" + }, + "ephemeral_storage": { + "description": "Recommended ephemeral storage allocation (in MiB). Derived from job temporary storage patterns.", + "format": "int64", + "type": "integer" + }, + "heap": { + "description": "Recommended JVM heap size (in MiB).", + "format": "int64", + "type": "integer" + }, + "memory": { + "description": "Recommended total memory allocation (in MiB). Includes both heap and overhead.", + "format": "int64", + "type": "integer" + }, + "overhead": { + "description": "Recommended JVM overhead (in MiB). Computed as total memory - heap.", + "format": "int64", + "type": "integer" + } + }, + "type": "object" + } + }, + "required": [ + "estimation" ], - "example": "completed", - "type": "string", - "x-enum-varnames": [ - "IN_PROGRESS", - "COMPLETED", - "COMPLETED_WITH_ERRORS", - "FAILED" - ] + "type": "object" + }, + "executor": { + "description": "Resource recommendation for a single Spark component (driver or executor). Contains estimation data used to patch Spark job specs.", + "properties": { + "estimation": { + "description": "Recommended resource values for a Spark driver or executor, derived from recent real usage metrics. Used by SPA to propose more efficient pod sizing.", + "properties": { + "cpu": { + "description": "CPU usage statistics derived from historical Spark job metrics. Provides multiple estimates so users can choose between conservative and cost-saving risk profiles.", + "properties": { + "max": { + "description": "Maximum CPU usage observed for the job, expressed in millicores. This represents the upper bound of usage.", + "format": "int64", + "type": "integer" + }, + "p75": { + "description": "75th percentile of CPU usage (millicores). Represents a cost-saving configuration while covering most workloads.", + "format": "int64", + "type": "integer" + }, + "p95": { + "description": "95th percentile of CPU usage (millicores). Balances performance and cost, providing a safer margin than p75.", + "format": "int64", + "type": "integer" + } + }, + "type": "object", + "x-model-simple-name": "SpaCpu" + }, + "ephemeral_storage": { + "description": "Recommended ephemeral storage allocation (in MiB). Derived from job temporary storage patterns.", + "format": "int64", + "type": "integer" + }, + "heap": { + "description": "Recommended JVM heap size (in MiB).", + "format": "int64", + "type": "integer" + }, + "memory": { + "description": "Recommended total memory allocation (in MiB). Includes both heap and overhead.", + "format": "int64", + "type": "integer" + }, + "overhead": { + "description": "Recommended JVM overhead (in MiB). Computed as total memory - heap.", + "format": "int64", + "type": "integer" + } + }, + "type": "object" + } + }, + "required": [ + "estimation" + ], + "type": "object" } }, + "required": [ + "driver", + "executor" + ], "type": "object" }, "id": { - "description": "The ID of the report job.", - "example": "dc8d92aa-e0af-11ee-af21-1feeaccaa3a3", + "description": "Resource identifier for the recommendation. Optional in responses.", "type": "string" }, "type": { - "description": "The type of ID.", - "example": "report_id", - "type": "string" + "default": "recommendation", + "description": "JSON:API resource type for Spark Pod Autosizing recommendations. Identifies the Recommendation resource returned by SPA.", + "enum": [ + "recommendation" + ], + "example": "recommendation", + "type": "string", + "x-enum-varnames": [ + "RECOMMENDATION" + ] } }, + "required": [ + "type", + "attributes" + ], "type": "object" } }, + "required": [ + "data" + ], "type": "object" } } @@ -606060,35 +606992,7 @@ } } }, - "description": "Forbidden" - }, - "404": { - "content": { - "application/json": { - "schema": { - "description": "API error response.", - "properties": { - "errors": { - "description": "A list of errors.", - "example": [ - "Bad Request" - ], - "items": { - "description": "A list of items.", - "example": "Bad Request", - "type": "string" - }, - "type": "array" - } - }, - "required": [ - "errors" - ], - "type": "object" - } - } - }, - "description": "Not Found" + "description": "Not Authorized" }, "429": { "content": { @@ -606119,26 +607023,15 @@ "description": "Too many requests" } }, - "security": [ - { - "apiKeyAuth": [], - "appKeyAuth": [] - }, - { - "AuthZ": [ - "slos_read" - ] - } - ], - "summary": "Get SLO report status", + "summary": "Get SPA Recommendations", "tags": [ - "Service Level Objectives" + "Spa" ], - "x-menu-order": 102, + "x-menu-order": 1, "x-undo": { "type": "safe" }, - "x-unstable": "**Note**: This feature is in private beta. To request access, use the request access form in the [Service Level Objectives](https://docs.datadoghq.com/service_management/service_level_objectives/#slo-csv-export) docs." + "x-unstable": "**Note**: This endpoint is in public beta and may change in the future. It is not yet recommended for production use." } }, "/api/v2/spans/analytics/aggregate": { @@ -635654,6 +636547,10 @@ }, "name": "Software Catalog" }, + { + "description": "SPA (Spark Pod Autosizing) API. Provides resource recommendations and cost insights to help optimize Spark job configurations.", + "name": "Spa" + }, { "description": "Search and aggregate your spans from your Datadog platform over HTTP.", "name": "Spans"