diff --git a/hugo/config/_default/menus/main.en.yaml b/hugo/config/_default/menus/main.en.yaml index 9e01add5783..0a0d61d1dcd 100644 --- a/hugo/config/_default/menus/main.en.yaml +++ b/hugo/config/_default/menus/main.en.yaml @@ -5669,16 +5669,56 @@ menu: parent: llm_obs_monitoring identifier: llm_obs_patterns weight: 404 + - name: Evaluations + url: llm_observability/investigate/evaluations/ + parent: llm_obs_monitoring + identifier: llm_obs_evaluations + weight: 405 + - name: Custom LLM-as-a-Judge + url: llm_observability/investigate/evaluations/llm_as_a_judge_evaluations + parent: llm_obs_evaluations + identifier: llm_obs_custom_llm_as_a_judge_evaluations + weight: 40501 + - name: Language Mismatch + url: llm_observability/investigate/evaluations/language_mismatch + parent: llm_obs_evaluations + identifier: llm_obs_language_mismatch + weight: 40502 + - name: Sensitive Data Scanner + url: llm_observability/investigate/evaluations/managed_evaluations/security_and_safety_evaluations + parent: llm_obs_evaluations + identifier: llm_obs_sensitive_data_scanner + weight: 40503 + - name: End-User Feedback + url: llm_observability/investigate/evaluations/end_user_feedback + parent: llm_obs_evaluations + identifier: llm_obs_end_user_feedback + weight: 40504 + - name: External Evaluations + url: llm_observability/investigate/evaluations/external_evaluations + parent: llm_obs_evaluations + identifier: llm_obs_external_evaluations + weight: 40505 + - name: Evaluation Developer Guide + url: llm_observability/investigate/evaluations/evaluation_developer_guide + parent: llm_obs_evaluations + identifier: llm_obs_evaluation_developer_guide + weight: 40506 + - name: Compatibility + url: llm_observability/investigate/evaluations/compatibility + parent: llm_obs_evaluations + identifier: llm_obs_evaluation_compatibility + weight: 40507 - name: Annotation Queues url: llm_observability/investigate/annotation_queues parent: llm_obs_monitoring identifier: llm_obs_annotation_queues - weight: 405 + weight: 406 - name: Export API url: llm_observability/investigate/export_api parent: llm_obs_monitoring identifier: llm_obs_evaluations_export_api - weight: 406 + weight: 407 - name: Improve url: llm_observability/improve/ parent: llm_obs @@ -5729,51 +5769,16 @@ menu: parent: llm_obs identifier: llm_obs_configure weight: 6 - - name: Evaluations - url: llm_observability/configure/evaluations/ - parent: llm_obs_configure - identifier: llm_obs_evaluations - weight: 601 - - name: Custom LLM-as-a-Judge - url: llm_observability/configure/evaluations/llm_as_a_judge_evaluations - parent: llm_obs_evaluations - identifier: llm_obs_custom_llm_as_a_judge_evaluations - weight: 60101 - - name: Language Mismatch - url: llm_observability/configure/evaluations/language_mismatch - parent: llm_obs_evaluations - identifier: llm_obs_language_mismatch - weight: 60102 - - name: Sensitive Data Scanner - url: llm_observability/configure/evaluations/managed_evaluations/security_and_safety_evaluations - parent: llm_obs_evaluations - identifier: llm_obs_sensitive_data_scanner - weight: 60103 - - name: End-User Feedback - url: llm_observability/configure/evaluations/end_user_feedback - parent: llm_obs_evaluations - identifier: llm_obs_end_user_feedback - weight: 60104 - - name: Compatibility - url: llm_observability/configure/evaluations/compatibility - parent: llm_obs_evaluations - identifier: llm_obs_evaluation_compatibility - weight: 60106 - - name: External Evaluations - url: llm_observability/configure/evaluations/external_evaluations - parent: llm_obs_evaluations - identifier: llm_obs_external_evaluations - weight: 60107 - name: Prompt Management url: llm_observability/configure/prompt_management parent: llm_obs_configure identifier: llm_obs_prompt_management - weight: 602 + weight: 601 - name: Automation Rules url: llm_observability/configure/automation_rules parent: llm_obs_configure identifier: llm_obs_monitoring_automation_rules - weight: 603 + weight: 602 - name: Data Privacy, Security, and RBAC url: llm_observability/data_privacy_security_and_rbac parent: llm_obs diff --git a/hugo/content/en/llm_observability/_index.md b/hugo/content/en/llm_observability/_index.md index 22d37a09909..6191897765b 100644 --- a/hugo/content/en/llm_observability/_index.md +++ b/hugo/content/en/llm_observability/_index.md @@ -139,5 +139,5 @@ See the [Setup documentation][5] for instructions on instrumenting your LLM appl [6]: /llm_observability/quickstart [7]: https://app.datadoghq.com/dash/integration/llm_operational_insights [8]: /llm_observability/setup/auto_instrumentation -[9]: /llm_observability/configure/evaluations/managed_evaluations +[9]: /llm_observability/investigate/evaluations/managed_evaluations [10]: /llm_observability/investigate/patterns diff --git a/hugo/content/en/llm_observability/build_with_ai/claude_code_skills.md b/hugo/content/en/llm_observability/build_with_ai/claude_code_skills.md index 6e36547dd05..5ea51a0734e 100644 --- a/hugo/content/en/llm_observability/build_with_ai/claude_code_skills.md +++ b/hugo/content/en/llm_observability/build_with_ai/claude_code_skills.md @@ -4,13 +4,13 @@ aliases: - /llm_observability/guide/claude_code_skills/ description: Use Datadog's Claude Code skills to classify sessions, diagnose failures, compare experiments, generate Python experiment code, and bootstrap evaluators against your live production data. further_reading: - - link: '/llm_observability/configure/evaluations/' + - link: '/llm_observability/investigate/evaluations/' tag: 'Documentation' text: 'Agent Observability Evaluations' - link: '/llm_observability/improve/experiments/' tag: 'Documentation' text: 'LLM Experiments' - - link: '/llm_observability/guide/evaluation_developer_guide' + - link: '/llm_observability/investigate/evaluations/evaluation_developer_guide' tag: 'Guide' text: 'Evaluation Developer Guide: Build custom evaluators' - link: "https://www.datadoghq.com/blog/bits-evals/" @@ -302,7 +302,7 @@ If you are new to evaluating an LLM application, the recommended flow is: [1]: https://claude.ai/code [2]: /llm_observability/setup/ [3]: /llm_observability/improve/experiments/ -[4]: /llm_observability/guide/evaluation_developer_guide +[4]: /llm_observability/investigate/evaluations/evaluation_developer_guide [5]: https://datadoghq.atlassian.net/wiki/spaces/BITSAI/pages/5226692942/pup+CLI [6]: https://github.com/datadog-labs/agent-skills [7]: https://github.com/DataDog/llm-observability/tree/main/experiments/notebooks diff --git a/hugo/content/en/llm_observability/configure/_index.md b/hugo/content/en/llm_observability/configure/_index.md index acb0cc17d04..834b98bbd78 100644 --- a/hugo/content/en/llm_observability/configure/_index.md +++ b/hugo/content/en/llm_observability/configure/_index.md @@ -1,12 +1,11 @@ --- title: Configure Agent Observability -description: Configure evaluations, prompt management, and automation rules for Agent Observability. +description: Configure prompt management and automation rules for Agent Observability. --- -Configure the Agent Observability capabilities that Datadog runs for you, including evaluations, automation rules, and prompt management. Datadog hosts and scales them, so you do not operate your own evaluation, routing, or prompt-serving infrastructure. +Configure the Agent Observability capabilities that Datadog runs for you, including automation rules and prompt management. Datadog hosts and scales them, so you do not operate your own routing or prompt-serving infrastructure. {{< whatsnext desc="Configuration options:" >}} - {{< nextlink href="/llm_observability/configure/evaluations/" >}}Evaluations{{< /nextlink >}} {{< nextlink href="/llm_observability/configure/prompt_management" >}}Prompt Management{{< /nextlink >}} {{< nextlink href="/llm_observability/configure/automation_rules" >}}Automation Rules{{< /nextlink >}} {{< /whatsnext >}} diff --git a/hugo/content/en/llm_observability/guide/_index.md b/hugo/content/en/llm_observability/guide/_index.md index 5396a5d9c56..0987cb5b77b 100644 --- a/hugo/content/en/llm_observability/guide/_index.md +++ b/hugo/content/en/llm_observability/guide/_index.md @@ -12,12 +12,11 @@ cascade: {{< nextlink href="/llm_observability/quickstart/" >}}Trace an LLM Application{{< /nextlink >}} {{< nextlink href="/llm_observability/guide/trace-an-llm-application-in-aws-lambda" >}}Trace an LLM Application in AWS Lambda{{< /nextlink >}} {{< nextlink href="/llm_observability/guide/monitor_proxy_services" >}}Trace Proxy and Gateway Services{{< /nextlink >}} - {{< nextlink href="/llm_observability/configure/evaluations/" >}}Evaluations{{< /nextlink >}} {{< nextlink href="/llm_observability/instrument/agent_observability_and_apm" >}}Using Agent Observability and APM{{< /nextlink >}} {{< nextlink href="/llm_observability/guide/monitor_mcp_client" >}}Monitor MCP Clients{{< /nextlink >}} {{< nextlink href="/llm_observability/guide/crewai_guide" >}}Using the Datadog-CrewAI integration for Agent Observability{{< /nextlink >}} {{< nextlink href="/llm_observability/guide/nextjs_guide" >}}Instrument a Next.js Application for Agent Observability{{< /nextlink >}} {{< nextlink href="/llm_observability/guide/agent_monitoring" >}}Agent Monitoring{{< /nextlink >}} - {{< nextlink href="/llm_observability/configure/evaluations/evaluation_developer_guide" >}}Evaluation Developer Guide: Build custom evaluators{{< /nextlink >}} + {{< nextlink href="/llm_observability/investigate/evaluations/evaluation_developer_guide" >}}Evaluation Developer Guide: Build custom evaluators{{< /nextlink >}} {{< nextlink href="/llm_observability/build_with_ai/claude_code_skills" >}}Analyze LLM Applications with Claude Code Skills{{< /nextlink >}} {{< /whatsnext >}} diff --git a/hugo/content/en/llm_observability/improve/experiments/setup.md b/hugo/content/en/llm_observability/improve/experiments/setup.md index ef1a85e833b..b59a77754ec 100644 --- a/hugo/content/en/llm_observability/improve/experiments/setup.md +++ b/hugo/content/en/llm_observability/improve/experiments/setup.md @@ -347,7 +347,7 @@ Note: LLM Experiments traces are retained for 90 days. [1]: /llm_observability/improve/datasets [2]: /llm_observability/instrument/custom_instrumentation?tab=decorators#trace-an-llm-application [3]: /llm_observability/instrument/auto_instrumentation?tab=python -[4]: /llm_observability/guide/evaluation_developer_guide +[4]: /llm_observability/investigate/evaluations/evaluation_developer_guide [5]: /llm_observability/instrument/agent_observability_and_apm/ [6]: /llm_observability/instrument/otel_instrumentation diff --git a/hugo/content/en/llm_observability/instrument/api.md b/hugo/content/en/llm_observability/instrument/api.md index 7cb8340647b..d8275086e97 100644 --- a/hugo/content/en/llm_observability/instrument/api.md +++ b/hugo/content/en/llm_observability/instrument/api.md @@ -374,7 +374,7 @@ The name can be up to 193 characters long and may not contain contiguous or trai ## Evaluations API -
For comprehensive examples and guidance on building custom evaluators, see the Evaluation Developer Guide.
+
For comprehensive examples and guidance on building custom evaluators, see the Evaluation Developer Guide.
Use this endpoint to send evaluations and end-user feedback to Datadog. Evaluations can be associated with spans, traces, or sessions. End-user feedback can be associated with spans, traces, sessions, or a customer-defined feedback join key. @@ -702,5 +702,5 @@ For feedback events, provide exactly one of `span_id`, `trace_id`, `session_id`, [1]: /llm_observability/setup/sdk/ [2]: /llm_observability/quickstart/terms/ [3]: /getting_started/tagging/ -[4]: /llm_observability/configure/evaluations/end_user_feedback +[4]: /llm_observability/investigate/evaluations/end_user_feedback [5]: /llm_observability/instrument/sdk/?tab=python#enriching-spans diff --git a/hugo/content/en/llm_observability/instrument/otel_instrumentation.md b/hugo/content/en/llm_observability/instrument/otel_instrumentation.md index a8d92303249..68f3a4a123d 100644 --- a/hugo/content/en/llm_observability/instrument/otel_instrumentation.md +++ b/hugo/content/en/llm_observability/instrument/otel_instrumentation.md @@ -19,7 +19,7 @@ Agent Observability supports ingesting OpenTelemetry traces that follow either t ### Evaluations -To send [external evaluations directly to the API](/llm_observability/configure/evaluations/external_evaluations#submitting-external-evaluations-with-the-api) for OpenTelemetry spans, include the `source:otel` tag in the evaluation. When referencing spans, provide `span_id` and `trace_id` as decimal strings. OpenTelemetry uses hexadecimal IDs natively, so convert them to decimal before submitting evaluations. For example, use Python's `int(hex_span_id, 16)` to convert a hex span ID to its decimal equivalent. +To send [external evaluations directly to the API](/llm_observability/investigate/evaluations/external_evaluations#submitting-external-evaluations-with-the-api) for OpenTelemetry spans, include the `source:otel` tag in the evaluation. When referencing spans, provide `span_id` and `trace_id` as decimal strings. OpenTelemetry uses hexadecimal IDs natively, so convert them to decimal before submitting evaluations. For example, use Python's `int(hex_span_id, 16)` to convert a hex span ID to its decimal equivalent. ### Prompt Tracking @@ -944,7 +944,7 @@ with tracer.start_as_current_span("my-span") as span: [3]: https://app.datadoghq.com/llm/traces [4]: /help/ [5]: https://pypi.org/project/strands-agents/ -[6]: /llm_observability/configure/evaluations/external_evaluations +[6]: /llm_observability/investigate/evaluations/external_evaluations [7]: https://strandsagents.com/latest/ [8]: /account_management/rbac/data_access/ [9]: https://opentelemetry.io/docs/concepts/signals/traces/#span-links diff --git a/hugo/content/en/llm_observability/instrument/sdk.md b/hugo/content/en/llm_observability/instrument/sdk.md index 12a8fdfcd66..50261dea7ee 100644 --- a/hugo/content/en/llm_observability/instrument/sdk.md +++ b/hugo/content/en/llm_observability/instrument/sdk.md @@ -1959,8 +1959,8 @@ Supported keys: - `template` (str): Template string with placeholders (for example, `"Translate {{text}} to {{lang}}"`). - `chat_template` (List[Message]): Multi-message template form. Provide a list of `{ "role": "", "content": "