sensei: improve azure-observability triggers and add unit/trigger/e2e tests

plugin/skills/azure-observability/SKILL.md

@@ -1,9 +1,17 @@
 ---
 name: azure-observability
 description: >-
-  Azure Observability Services including Azure Monitor, Application Insights, Log Analytics, Alerts, and Workbooks. Provides metrics, APM, distributed tracing, KQL queries, and interactive reports.
-  USE FOR: Azure Monitor, Application Insights, Log Analytics, Alerts, Workbooks, metrics, APM, distributed tracing, KQL queries, interactive reports, observability, monitoring dashboards.
-  DO NOT USE FOR: instrumenting apps with App Insights SDK (use appinsights-instrumentation), querying Kusto/ADX clusters (use azure-kusto), cost analysis (use azure-cost-optimization).
+  Monitor Azure applications and infrastructure using Azure Monitor, Application
+  Insights, Log Analytics, Alerts, and Workbooks. Query metrics, analyze logs
+  with KQL, configure alert rules, set up dashboards, and troubleshoot
+  performance.
+  USE FOR: monitor my app, set up Azure Monitor, configure Application Insights,
+  query logs with KQL, create alert rules, build monitoring dashboards, analyze
+  metrics, view distributed traces, set up observability, check application
+  performance.
+  DO NOT USE FOR: instrumenting apps with App Insights SDK code (use
+  appinsights-instrumentation), querying Kusto/ADX clusters (use azure-kusto),
+  cost analysis (use azure-cost-optimization).
 ---
 
 # Azure Observability Services

tests/azure-observability/__snapshots__/triggers.test.ts.snap

@@ -0,0 +1,101 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`azure-observability - Trigger Tests Trigger Keywords Snapshot skill description triggers match snapshot 1`] = `
+{
+  "description": "Monitor Azure applications and infrastructure using Azure Monitor, Application Insights, Log Analytics, Alerts, and Workbooks. Query metrics, analyze logs with KQL, configure alert rules, set up dashboards, and troubleshoot performance. USE FOR: monitor my app, set up Azure Monitor, configure Application Insights, query logs with KQL, create alert rules, build monitoring dashboards, analyze metrics, view distributed traces, set up observability, check application performance. DO NOT USE FOR: instrumenting apps with App Insights SDK code (use appinsights-instrumentation), querying Kusto/ADX clusters (use azure-kusto), cost analysis (use azure-cost-optimization).",
+  "extractedKeywords": [
+    "alert",
+    "alerts",
+    "analysis",
+    "analytics",
+    "analyze",
+    "appinsights-instrumentation",
+    "application",
+    "applications",
+    "apps",
+    "azure",
+    "azure-cost-optimization",
+    "azure-kusto",
+    "build",
+    "check",
+    "cli",
+    "clusters",
+    "code",
+    "configure",
+    "cost",
+    "create",
+    "dashboards",
+    "distributed",
+    "infrastructure",
+    "insights",
+    "instrumenting",
+    "kusto",
+    "logs",
+    "mcp",
+    "metrics",
+    "monitor",
+    "monitoring",
+    "observability",
+    "performance",
+    "query",
+    "querying",
+    "rules",
+    "security",
+    "traces",
+    "troubleshoot",
+    "using",
+    "view",
+    "with",
+    "workbooks",
+  ],
+  "name": "azure-observability",
+}
+`;
+
+exports[`azure-observability - Trigger Tests Trigger Keywords Snapshot skill keywords match snapshot 1`] = `
+[
+  "alert",
+  "alerts",
+  "analysis",
+  "analytics",
+  "analyze",
+  "appinsights-instrumentation",
+  "application",
+  "applications",
+  "apps",
+  "azure",
+  "azure-cost-optimization",
+  "azure-kusto",
+  "build",
+  "check",
+  "cli",
+  "clusters",
+  "code",
+  "configure",
+  "cost",
+  "create",
+  "dashboards",
+  "distributed",
+  "infrastructure",
+  "insights",
+  "instrumenting",
+  "kusto",
+  "logs",
+  "mcp",
+  "metrics",
+  "monitor",
+  "monitoring",
+  "observability",
+  "performance",
+  "query",
+  "querying",
+  "rules",
+  "security",
+  "traces",
+  "troubleshoot",
+  "using",
+  "view",
+  "with",
+  "workbooks",
+]
+`;

tests/azure-observability/integration.test.ts

@@ -1,18 +1,20 @@
 /**
  * Integration Tests for azure-observability
- * 
- * Tests skill behavior with a real Copilot agent session.
- * Runs prompts multiple times to measure skill invocation rate.
- * 
+ *
+ * End-to-end tests that verify the skill responds correctly to real Azure
+ * Monitor, Application Insights, and Log Analytics scenarios.
+ *
  * Prerequisites:
  * 1. npm install -g @github/copilot-cli
  * 2. Run `copilot` and authenticate
+ * 3. az login (for Azure Monitor/Log Analytics queries)
  */
 
 import {
   useAgentRunner,
   shouldSkipIntegrationTests,
-  getIntegrationSkipReason
+  getIntegrationSkipReason,
+  doesAssistantMessageIncludeKeyword
 } from "../utils/agent-runner";
 import { softCheckSkill } from "../utils/evaluate";
 
@@ -70,4 +72,158 @@ describeIntegration(`${SKILL_NAME}_ - Integration Tests`, () => {
       }
     });
   });
+
+  // Azure Monitor metrics scenarios
+  describe("azure-monitor-metrics", () => {
+    test("response mentions Azure Monitor for metrics query", async () => {
+      try {
+        const agentMetadata = await agent.run({
+          prompt: "Show me CPU and memory metrics for my Azure resources over the last hour"
+        });
+
+        const mentionsMonitor =
+          doesAssistantMessageIncludeKeyword(agentMetadata, "Azure Monitor") ||
+          doesAssistantMessageIncludeKeyword(agentMetadata, "az monitor metrics") ||
+          doesAssistantMessageIncludeKeyword(agentMetadata, "monitor_metrics_query");
+        expect(mentionsMonitor).toBe(true);
+      } catch (e: unknown) {
+        if (e instanceof Error && e.message?.includes("Failed to load @github/copilot-sdk")) {
+          console.log("⏭️  SDK not loadable, skipping test");
+          return;
+        }
+        throw e;
+      }
+    });
+
+    test("response provides CLI or MCP command for querying metrics", async () => {
+      try {
+        const agentMetadata = await agent.run({
+          prompt: "What CLI command can I use to query CPU metrics for my App Service?"
+        });
+
+        const mentionsMetricsCommand =
+          doesAssistantMessageIncludeKeyword(agentMetadata, "az monitor") ||
+          doesAssistantMessageIncludeKeyword(agentMetadata, "metrics list") ||
+          doesAssistantMessageIncludeKeyword(agentMetadata, "metric");
+        expect(mentionsMetricsCommand).toBe(true);
+      } catch (e: unknown) {
+        if (e instanceof Error && e.message?.includes("Failed to load @github/copilot-sdk")) {
+          console.log("⏭️  SDK not loadable, skipping test");
+          return;
+        }
+        throw e;
+      }
+    });
+  });
+
+  // Log Analytics / KQL scenarios
+  describe("log-analytics-kql", () => {
+    test("response provides KQL query for log analysis", async () => {
+      try {
+        const agentMetadata = await agent.run({
+          prompt: "Write a KQL query to find application exceptions in the last 24 hours"
+        });
+
+        const mentionsKql =
+          doesAssistantMessageIncludeKeyword(agentMetadata, "KQL") ||
+          doesAssistantMessageIncludeKeyword(agentMetadata, "AppExceptions") ||
+          doesAssistantMessageIncludeKeyword(agentMetadata, "TimeGenerated") ||
+          doesAssistantMessageIncludeKeyword(agentMetadata, "Log Analytics");
+        expect(mentionsKql).toBe(true);
+      } catch (e: unknown) {
+        if (e instanceof Error && e.message?.includes("Failed to load @github/copilot-sdk")) {
+          console.log("⏭️  SDK not loadable, skipping test");
+          return;
+        }
+        throw e;
+      }
+    });
+
+    test("response mentions Log Analytics workspace for log queries", async () => {
+      try {
+        const agentMetadata = await agent.run({
+          prompt: "How do I query failed HTTP requests in my Log Analytics workspace?"
+        });
+
+        const mentionsLogAnalytics =
+          doesAssistantMessageIncludeKeyword(agentMetadata, "Log Analytics") ||
+          doesAssistantMessageIncludeKeyword(agentMetadata, "log-analytics") ||
+          doesAssistantMessageIncludeKeyword(agentMetadata, "workspace");
+        expect(mentionsLogAnalytics).toBe(true);
+      } catch (e: unknown) {
+        if (e instanceof Error && e.message?.includes("Failed to load @github/copilot-sdk")) {
+          console.log("⏭️  SDK not loadable, skipping test");
+          return;
+        }
+        throw e;
+      }
+    });
+  });
+
+  // Application Insights scenarios
+  describe("application-insights", () => {
+    test("response explains Application Insights for APM", async () => {
+      try {
+        const agentMetadata = await agent.run({
+          prompt: "How do I view slow requests and dependencies in Application Insights?"
+        });
+
+        const mentionsAppInsights =
+          doesAssistantMessageIncludeKeyword(agentMetadata, "Application Insights") ||
+          doesAssistantMessageIncludeKeyword(agentMetadata, "AppRequests") ||
+          doesAssistantMessageIncludeKeyword(agentMetadata, "DurationMs");
+        expect(mentionsAppInsights).toBe(true);
+      } catch (e: unknown) {
+        if (e instanceof Error && e.message?.includes("Failed to load @github/copilot-sdk")) {
+          console.log("⏭️  SDK not loadable, skipping test");
+          return;
+        }
+        throw e;
+      }
+    });
+
+    test("response mentions distributed tracing for Application Insights", async () => {
+      try {
+        const agentMetadata = await agent.run({
+          prompt: "Show me distributed traces for my microservices in Application Insights"
+        });
+
+        const mentionsTracing =
+          doesAssistantMessageIncludeKeyword(agentMetadata, "Application Insights") ||
+          doesAssistantMessageIncludeKeyword(agentMetadata, "distributed tracing") ||
+          doesAssistantMessageIncludeKeyword(agentMetadata, "trace");
+        expect(mentionsTracing).toBe(true);
+      } catch (e: unknown) {
+        if (e instanceof Error && e.message?.includes("Failed to load @github/copilot-sdk")) {
+          console.log("⏭️  SDK not loadable, skipping test");
+          return;
+        }
+        throw e;
+      }
+    });
+  });
+
+  // Alert configuration scenarios
+  describe("alert-configuration", () => {
+    test("response provides guidance for creating alert rules", async () => {
+      try {
+        const agentMetadata = await agent.run({
+          prompt: "Create an Azure Monitor alert when my app's error rate exceeds 5%"
+        });
+
+        const mentionsAlerts =
+          doesAssistantMessageIncludeKeyword(agentMetadata, "alert") ||
+          doesAssistantMessageIncludeKeyword(agentMetadata, "action group") ||
+          doesAssistantMessageIncludeKeyword(agentMetadata, "az monitor metrics alert");
+        expect(mentionsAlerts).toBe(true);
+      } catch (e: unknown) {
+        if (e instanceof Error && e.message?.includes("Failed to load @github/copilot-sdk")) {
+          console.log("⏭️  SDK not loadable, skipping test");
+          return;
+        }
+        throw e;
+      }
+    });
+  });
 });
+