sensei: improve azure-observability triggers and add unit/trigger/e2e tests

plugin/skills/azure-observability/SKILL.md

@@ -1,9 +1,16 @@
 ---
 name: azure-observability
 description: >-
-  Azure Observability Services including Azure Monitor, Application Insights, Log Analytics, Alerts, and Workbooks. Provides metrics, APM, distributed tracing, KQL queries, and interactive reports.
-  USE FOR: Azure Monitor, Application Insights, Log Analytics, Alerts, Workbooks, metrics, APM, distributed tracing, KQL queries, interactive reports, observability, monitoring dashboards.
-  DO NOT USE FOR: instrumenting apps with App Insights SDK (use appinsights-instrumentation), querying Kusto/ADX clusters (use azure-kusto), cost analysis (use azure-cost-optimization).
+  Monitor Azure applications and infrastructure using Azure Monitor, Application
+  Insights, Log Analytics, Alerts, and Workbooks. Query metrics, analyze logs
+  with KQL, configure alert rules, set up dashboards, and troubleshoot
+  performance.
+  USE FOR: set up Azure Monitor, configure Application Insights, query logs with KQL,
+  create alert rules, build dashboards, analyze metrics, view traces, list/query Log Analytics
+  workspaces, show workspace tables, workspace logs.
+  DO NOT USE FOR: instrumenting apps with App Insights SDK code (use
+  appinsights-instrumentation), querying Kusto/ADX clusters (use azure-kusto),
+  cost analysis (use azure-cost-optimization).
 ---
 
 # Azure Observability Services
@@ -22,18 +29,57 @@ description: >-
 
 When Azure MCP is enabled:
 
-### Monitor
-- `azure__monitor` with command `monitor_metrics_query` - Query metrics
-- `azure__monitor` with command `monitor_logs_query` - Query logs with KQL
-
-### Application Insights
-- `azure__applicationinsights` with command `applicationinsights_component_list` - List App Insights resources
-
-### Log Analytics
-- `azure__kusto` with command `kusto_cluster_list` - List clusters
-- `azure__kusto` with command `kusto_query` - Execute KQL queries
-
-**If Azure MCP is not enabled:** Run `/azure:setup` or enable via `/mcp`.
+### Required: Azure Subscription Context
+
+**BEFORE calling any monitor MCP command:**
+
+1. **If user provided workspace name** → Use Azure Resource Graph to find it and extract subscription:
+   ```yaml
+   azure_resources-query_azure_resource_graph
+     arg_intent: "find log analytics workspace named <workspace-name> and return its subscription id"
+     useDefaultSubscriptionFilter: false
+   ```
+   Extract subscription ID from the resource ID (format: `/subscriptions/{subscription-id}/resourceGroups/...`)
+
+2. **If no workspace name** → Check if subscription ID is available from prior conversation
+
+3. **If still no subscription** → Search azure.yaml or .azure/.env files for AZURE_SUBSCRIPTION_ID
+
+4. **If still nothing** → Ask user: "Which Azure subscription should I use?"
+
+**Monitor:**
+- `azure__monitor` with command `monitor_workspace_list` — List Log Analytics workspaces
+- `azure__monitor` with command `monitor_workspace_log_query` — Query logs across entire workspace with KQL
+- `azure__monitor` with command `monitor_resource_log_query` — Query logs for specific resource with KQL
+- `azure__monitor` with command `monitor_table_list` — List tables in workspace
+- `azure__monitor` with command `monitor_table_type_list` — List table types
+- `azure__monitor` with command `monitor_metrics_query` — Query metrics for resources
+- `azure__monitor` with command `monitor_metrics_definitions` — List available metric definitions
+- `azure__monitor` with command `monitor_activitylog_list` — List activity logs for resources
+- `azure__monitor` with command `monitor_healthmodels_entity_get` — Get health status of entity
+- `azure__monitor` with command `monitor_webtests_get` — Get or list web tests
+- `azure__monitor` with command `monitor_webtests_createorupdate` — Create/update availability web tests
+
+**Application Insights:**
+- `azure__applicationinsights` with command `applicationinsights_recommendation_list` — List code optimization recommendations from Profiler
+
+**Azure Data Explorer:**
+
+Note: For Log Analytics workspace queries, use `azure__monitor` commands above.
+- `azure__kusto` with command `kusto_cluster_list` — List ADX clusters
+- `azure__kusto` with command `kusto_cluster_get` — Get cluster details
+- `azure__kusto` with command `kusto_database_list` — List databases
+- `azure__kusto` with command `kusto_table_list` — List tables
+- `azure__kusto` with command `kusto_table_schema` — Get table schema
+- `azure__kusto` with command `kusto_sample` — Sample table data
+- `azure__kusto` with command `kusto_query` — Execute KQL queries on ADX
+
+**Workbooks:**
+- `azure__workbooks` with command `workbooks_list` — List workbooks in resource group
+- `azure__workbooks` with command `workbooks_show` — Get workbook details
+- `azure__workbooks` with command `workbooks_create` — Create new workbook
+- `azure__workbooks` with command `workbooks_update` — Update existing workbook
+- `azure__workbooks` with command `workbooks_delete` — Delete workbook (soft delete, 90-day retention)
 
 ## CLI Reference
 

tests/azure-observability/__snapshots__/triggers.test.ts.snap

@@ -0,0 +1,101 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`azure-observability - Trigger Tests Trigger Keywords Snapshot skill description triggers match snapshot 1`] = `
+{
+  "description": "Monitor Azure applications and infrastructure using Azure Monitor, Application Insights, Log Analytics, Alerts, and Workbooks. Query metrics, analyze logs with KQL, configure alert rules, set up dashboards, and troubleshoot performance. USE FOR: monitor my app, set up Azure Monitor, configure Application Insights, query logs with KQL, create alert rules, build monitoring dashboards, analyze metrics, view distributed traces, set up observability, check application performance. DO NOT USE FOR: instrumenting apps with App Insights SDK code (use appinsights-instrumentation), querying Kusto/ADX clusters (use azure-kusto), cost analysis (use azure-cost-optimization).",
+  "extractedKeywords": [
+    "alert",
+    "alerts",
+    "analysis",
+    "analytics",
+    "analyze",
+    "appinsights-instrumentation",
+    "application",
+    "applications",
+    "apps",
+    "azure",
+    "azure-cost-optimization",
+    "azure-kusto",
+    "build",
+    "check",
+    "cli",
+    "clusters",
+    "code",
+    "configure",
+    "cost",
+    "create",
+    "dashboards",
+    "distributed",
+    "infrastructure",
+    "insights",
+    "instrumenting",
+    "kusto",
+    "logs",
+    "mcp",
+    "metrics",
+    "monitor",
+    "monitoring",
+    "observability",
+    "performance",
+    "query",
+    "querying",
+    "rules",
+    "security",
+    "traces",
+    "troubleshoot",
+    "using",
+    "view",
+    "with",
+    "workbooks",
+  ],
+  "name": "azure-observability",
+}
+`;
+
+exports[`azure-observability - Trigger Tests Trigger Keywords Snapshot skill keywords match snapshot 1`] = `
+[
+  "alert",
+  "alerts",
+  "analysis",
+  "analytics",
+  "analyze",
+  "appinsights-instrumentation",
+  "application",
+  "applications",
+  "apps",
+  "azure",
+  "azure-cost-optimization",
+  "azure-kusto",
+  "build",
+  "check",
+  "cli",
+  "clusters",
+  "code",
+  "configure",
+  "cost",
+  "create",
+  "dashboards",
+  "distributed",
+  "infrastructure",
+  "insights",
+  "instrumenting",
+  "kusto",
+  "logs",
+  "mcp",
+  "metrics",
+  "monitor",
+  "monitoring",
+  "observability",
+  "performance",
+  "query",
+  "querying",
+  "rules",
+  "security",
+  "traces",
+  "troubleshoot",
+  "using",
+  "view",
+  "with",
+  "workbooks",
+]
+`;

tests/azure-observability/integration.test.ts

@@ -1,18 +1,20 @@
 /**
  * Integration Tests for azure-observability
- * 
- * Tests skill behavior with a real Copilot agent session.
- * Runs prompts multiple times to measure skill invocation rate.
- * 
+ *
+ * End-to-end tests that verify the skill responds correctly to real Azure
+ * Monitor, Application Insights, and Log Analytics scenarios.
+ *
  * Prerequisites:
  * 1. npm install -g @github/copilot-cli
  * 2. Run `copilot` and authenticate
+ * 3. az login (for Azure Monitor/Log Analytics queries)
  */
 
 import {
   useAgentRunner,
   shouldSkipIntegrationTests,
-  getIntegrationSkipReason
+  getIntegrationSkipReason,
+  doesAssistantMessageIncludeKeyword
 } from "../utils/agent-runner";
 import { softCheckSkill } from "../utils/evaluate";
 
@@ -70,4 +72,158 @@ describeIntegration(`${SKILL_NAME}_ - Integration Tests`, () => {
       }
     });
   });
+
+  // Azure Monitor metrics scenarios
+  describe("azure-monitor-metrics", () => {
+    test("response mentions Azure Monitor for metrics query", async () => {
+      try {
+        const agentMetadata = await agent.run({
+          prompt: "Show me CPU and memory metrics for my Azure resources over the last hour"
+        });
+
+        const mentionsMonitor =
+          doesAssistantMessageIncludeKeyword(agentMetadata, "Azure Monitor") ||
+          doesAssistantMessageIncludeKeyword(agentMetadata, "az monitor metrics") ||
+          doesAssistantMessageIncludeKeyword(agentMetadata, "monitor_metrics_query");
+        expect(mentionsMonitor).toBe(true);
+      } catch (e: unknown) {
+        if (e instanceof Error && e.message?.includes("Failed to load @github/copilot-sdk")) {
+          console.log("⏭️  SDK not loadable, skipping test");
+          return;
+        }
+        throw e;
+      }
+    });
+
+    test("response provides CLI or MCP command for querying metrics", async () => {
+      try {
+        const agentMetadata = await agent.run({
+          prompt: "What CLI command can I use to query CPU metrics for my App Service?"
+        });
+
+        const mentionsMetricsCommand =
+          doesAssistantMessageIncludeKeyword(agentMetadata, "az monitor") ||
+          doesAssistantMessageIncludeKeyword(agentMetadata, "metrics list") ||
+          doesAssistantMessageIncludeKeyword(agentMetadata, "metric");
+        expect(mentionsMetricsCommand).toBe(true);
+      } catch (e: unknown) {
+        if (e instanceof Error && e.message?.includes("Failed to load @github/copilot-sdk")) {
+          console.log("⏭️  SDK not loadable, skipping test");
+          return;
+        }
+        throw e;
+      }
+    });
+  });
+
+  // Log Analytics / KQL scenarios
+  describe("log-analytics-kql", () => {
+    test("response provides KQL query for log analysis", async () => {
+      try {
+        const agentMetadata = await agent.run({
+          prompt: "Write a KQL query to find application exceptions in the last 24 hours"
+        });
+
+        const mentionsKql =
+          doesAssistantMessageIncludeKeyword(agentMetadata, "KQL") ||
+          doesAssistantMessageIncludeKeyword(agentMetadata, "AppExceptions") ||
+          doesAssistantMessageIncludeKeyword(agentMetadata, "TimeGenerated") ||
+          doesAssistantMessageIncludeKeyword(agentMetadata, "Log Analytics");
+        expect(mentionsKql).toBe(true);
+      } catch (e: unknown) {
+        if (e instanceof Error && e.message?.includes("Failed to load @github/copilot-sdk")) {
+          console.log("⏭️  SDK not loadable, skipping test");
+          return;
+        }
+        throw e;
+      }
+    });
+
+    test("response mentions Log Analytics workspace for log queries", async () => {
+      try {
+        const agentMetadata = await agent.run({
+          prompt: "How do I query failed HTTP requests in my Log Analytics workspace?"
+        });
+
+        const mentionsLogAnalytics =
+          doesAssistantMessageIncludeKeyword(agentMetadata, "Log Analytics") ||
+          doesAssistantMessageIncludeKeyword(agentMetadata, "log-analytics") ||
+          doesAssistantMessageIncludeKeyword(agentMetadata, "workspace");
+        expect(mentionsLogAnalytics).toBe(true);
+      } catch (e: unknown) {
+        if (e instanceof Error && e.message?.includes("Failed to load @github/copilot-sdk")) {
+          console.log("⏭️  SDK not loadable, skipping test");
+          return;
+        }
+        throw e;
+      }
+    });
+  });
+
+  // Application Insights scenarios
+  describe("application-insights", () => {
+    test("response explains Application Insights for APM", async () => {
+      try {
+        const agentMetadata = await agent.run({
+          prompt: "How do I view slow requests and dependencies in Application Insights?"
+        });
+
+        const mentionsAppInsights =
+          doesAssistantMessageIncludeKeyword(agentMetadata, "Application Insights") ||
+          doesAssistantMessageIncludeKeyword(agentMetadata, "AppRequests") ||
+          doesAssistantMessageIncludeKeyword(agentMetadata, "DurationMs");
+        expect(mentionsAppInsights).toBe(true);
+      } catch (e: unknown) {
+        if (e instanceof Error && e.message?.includes("Failed to load @github/copilot-sdk")) {
+          console.log("⏭️  SDK not loadable, skipping test");
+          return;
+        }
+        throw e;
+      }
+    });
+
+    test("response mentions distributed tracing for Application Insights", async () => {
+      try {
+        const agentMetadata = await agent.run({
+          prompt: "Show me distributed traces for my microservices in Application Insights"
+        });
+
+        const mentionsTracing =
+          doesAssistantMessageIncludeKeyword(agentMetadata, "Application Insights") ||
+          doesAssistantMessageIncludeKeyword(agentMetadata, "distributed tracing") ||
+          doesAssistantMessageIncludeKeyword(agentMetadata, "trace");
+        expect(mentionsTracing).toBe(true);
+      } catch (e: unknown) {
+        if (e instanceof Error && e.message?.includes("Failed to load @github/copilot-sdk")) {
+          console.log("⏭️  SDK not loadable, skipping test");
+          return;
+        }
+        throw e;
+      }
+    });
+  });
+
+  // Alert configuration scenarios
+  describe("alert-configuration", () => {
+    test("response provides guidance for creating alert rules", async () => {
+      try {
+        const agentMetadata = await agent.run({
+          prompt: "Create an Azure Monitor alert when my app's error rate exceeds 5%"
+        });
+
+        const mentionsAlerts =
+          doesAssistantMessageIncludeKeyword(agentMetadata, "alert") ||
+          doesAssistantMessageIncludeKeyword(agentMetadata, "action group") ||
+          doesAssistantMessageIncludeKeyword(agentMetadata, "az monitor metrics alert");
+        expect(mentionsAlerts).toBe(true);
+      } catch (e: unknown) {
+        if (e instanceof Error && e.message?.includes("Failed to load @github/copilot-sdk")) {
+          console.log("⏭️  SDK not loadable, skipping test");
+          return;
+        }
+        throw e;
+      }
+    });
+  });
 });
+