diff --git a/plugin/skills/azure-cost-forecast/SKILL.md b/plugin/skills/azure-cost-forecast/SKILL.md new file mode 100644 index 00000000..06f96f94 --- /dev/null +++ b/plugin/skills/azure-cost-forecast/SKILL.md @@ -0,0 +1,202 @@ +--- +name: azure-cost-forecast +description: "Forecast future Azure costs using the Cost Management Forecast API. Builds and executes forecast requests with proper time-period guardrails, training-data validation, and response interpretation. WHEN: forecast Azure costs, predict spending, projected costs, estimate bill, future Azure costs, cost projection, budget forecast, end of month costs, how much will I spend. DO NOT USE FOR: querying historical costs (use azure-cost-query), reducing costs (use azure-cost-optimization)." +license: MIT +metadata: + author: Microsoft + version: "1.0.0" +--- + +# Azure Cost Forecast Skill + +Construct and execute Azure Cost Management Forecast API requests to project future Azure spending with actual-cost context. + +## Quick Reference + +| Property | Value | +|----------|-------| +| **API Endpoint** | `POST {scope}/providers/Microsoft.CostManagement/forecast?api-version=2023-11-01` | +| **MCP Tools** | `azure__documentation`, `azure__extension_cli_generate`, `azure__get_azure_bestpractices` | +| **CLI** | `az rest` | +| **Best For** | Future cost projections, budget planning, end-of-month estimates | +| **Related Skills** | azure-cost-query, azure-cost-optimization | + +## When to Use This Skill + +- Forecast future Azure spending for subscriptions, resource groups, or billing accounts +- Project end-of-month or end-of-year costs +- Get daily or monthly cost projections +- Include actual costs alongside forecast data for context +- Estimate future budget impact + +> โš ๏ธ **Warning:** If the user wants **historical** cost data, use azure-cost-query. If they want to **reduce** costs, use azure-cost-optimization. + +## Key Differences from Query API + +| Aspect | Query API | Forecast API | +|--------|-----------|--------------| +| Purpose | Historical cost data | Projected future costs | +| Time period | Past dates only | Must include future dates | +| Grouping | โœ… Up to 2 dimensions | โŒ **Not supported** | +| `includeActualCost` | N/A | Include historical alongside forecast | +| Response columns | Cost, Date, Currency | Cost, Date, **CostStatus**, Currency | +| Max response rows | 5,000/page | 40 rows recommended | +| Timeframe | Multiple presets + Custom | Typically `Custom` only | + +## MCP Tools + +| Tool | Purpose | Required | +|------|---------|----------| +| `azure__documentation` | Look up Forecast API parameters and reference | Optional | +| `azure__extension_cli_generate` | Generate `az rest` commands for forecast requests | Optional | +| `azure__get_azure_bestpractices` | Get cost management best practices | Optional | + +> ๐Ÿ’ก **Tip:** Prefer Azure MCP tools over direct CLI commands where possible. + +## Workflow + +### Step 1: Determine Scope + +Same scope URL patterns as the Query API: + +| Scope | URL Pattern | +|-------|-------------| +| Subscription | `/subscriptions/` | +| Resource Group | `/subscriptions//resourceGroups/` | +| Management Group | `/providers/Microsoft.Management/managementGroups/` | +| Billing Account | `/providers/Microsoft.Billing/billingAccounts/` | + +> ๐Ÿ’ก **Tip:** These are scope paths only โ€” not complete URLs. Combine with the API endpoint and version: `{scope}/providers/Microsoft.CostManagement/forecast?api-version=2023-11-01` + +### Step 2: Choose Report Type + +`ActualCost` is most common for forecasting. `AmortizedCost` for reservation/savings plan projections. + +### Step 3: Set Time Period + +> โš ๏ธ **Warning:** The `to` date **MUST** be in the future. The API returns an error for entirely past date ranges. + +- Set `timeframe` to `Custom` and provide `timePeriod` with `from` and `to` dates +- `from` can be in the past โ€” shows actual costs up to today, then forecast to `to` +- Minimum 28 days of historical cost data required for forecast to work +- Maximum forecast period: 10 years + +> **Full rules:** [Guardrails Reference](./references/guardrails.md) + +### Step 4: Configure Dataset + +- **Granularity**: `Daily` or `Monthly` recommended +- **Aggregation**: Typically `Sum` of `Cost` +- See [Request Body Schema](./references/request-body-schema.md) for full schema + +> โš ๏ธ **Warning:** Grouping is **NOT supported** for forecast. If the user requests a grouped forecast, inform them this is an API limitation and suggest using `azure-cost-query` for grouped historical data instead. + +### Step 5: Set Forecast-Specific Options + +| Field | Default | Description | +|-------|---------|-------------| +| `includeActualCost` | `true` | Include historical actual costs alongside forecast | +| `includeFreshPartialCost` | `true` | Include partial cost data for recent days. **Requires `includeActualCost: true`** | + +### Step 6: Construct and Execute + +**Create forecast query file:** + +Create `temp/cost-forecast.json` with: +```json +{ + "type": "ActualCost", + "timeframe": "Custom", + "timePeriod": { + "from": "", + "to": "" + }, + "dataset": { + "granularity": "Daily", + "aggregation": { + "totalCost": { "name": "Cost", "function": "Sum" } + }, + "sorting": [{ "direction": "Ascending", "name": "UsageDate" }] + }, + "includeActualCost": true, + "includeFreshPartialCost": true +} +``` + +**Execute forecast query:** +```powershell +# Create temp folder +New-Item -ItemType Directory -Path "temp" -Force + +# Query using REST API +az rest --method post ` + --url "https://management.azure.com/subscriptions//providers/Microsoft.CostManagement/forecast?api-version=2023-11-01" ` + --body '@temp/cost-forecast.json' +``` + +### Step 7: Interpret Response + +The response includes a `CostStatus` column: + +| CostStatus | Meaning | +|------------|---------| +| `Actual` | Historical actual cost (when `includeActualCost: true`) | +| `Forecast` | Projected future cost | + +> ๐Ÿ’ก **Tip:** "Forecast is unavailable for the specified time period" is not an error โ€” it means the scope has insufficient historical data. Suggest using `azure-cost-query` for available data. + +## Key Guardrails + +| Rule | Constraint | +|------|-----------| +| `to` date | Must be in the future | +| Grouping | โŒ Not supported | +| Min training data | 28 days of historical cost data | +| Max forecast period | 10 years | +| Response row limit | 40 rows recommended | +| `includeFreshPartialCost` | Requires `includeActualCost: true` | +| Monthly + includeActualCost | Requires explicit `timePeriod` | + +> **Full details:** [Guardrails Reference](./references/guardrails.md) + +## Error Handling + +| Status | Error | Remediation | +|--------|-------|-------------| +| 400 | Can't forecast on the past | Ensure `to` date is in the future. | +| 400 | Missing dataset | Add required `dataset` field. | +| 400 | Invalid dependency | Set `includeActualCost: true` when using `includeFreshPartialCost`. | +| 403 | Forbidden | Needs **Cost Management Reader** role on scope. | +| 424 | Bad training data | Insufficient history; falls back to actual costs if available. | +| 429 | Rate limited | Retry after the `x-ms-ratelimit-microsoft.costmanagement-qpu-retry-after` header value. **Max 3 retries.** | +| 503 | Service unavailable | Check [Azure Status](https://status.azure.com). | + +> **Full details:** [Error Handling Reference](./references/error-handling.md) + +## Examples + +### Forecast Rest of Current Month (Daily) + +```json +{ + "type": "ActualCost", + "timeframe": "Custom", + "timePeriod": { + "from": "", + "to": "" + }, + "dataset": { + "granularity": "Daily", + "aggregation": { + "totalCost": { "name": "Cost", "function": "Sum" } + }, + "sorting": [{ "direction": "Ascending", "name": "UsageDate" }] + }, + "includeActualCost": true, + "includeFreshPartialCost": true +} +``` + +> ๐Ÿ’ก **Tip:** Set `from` to the first of the month to see actual costs so far alongside the forecast for remaining days. + +More examples: [references/examples.md](./references/examples.md) diff --git a/plugin/skills/azure-cost-forecast/references/error-handling.md b/plugin/skills/azure-cost-forecast/references/error-handling.md new file mode 100644 index 00000000..86e9b0ce --- /dev/null +++ b/plugin/skills/azure-cost-forecast/references/error-handling.md @@ -0,0 +1,48 @@ +# Forecast API Error Handling + +## HTTP Status Codes + +| Status | Error | Cause | Remediation | +|---|---|---|---| +| 400 | Bad Request | Invalid request body, missing `dataset`, past-only dates, invalid field dependency combinations | Check request body structure; ensure `to` date is in the future; verify `includeActualCost`/`includeFreshPartialCost` dependency | +| 401 | Unauthorized | Authentication failure โ€” missing or expired token | Re-authenticate with `az login` or refresh the access token | +| 403 | Forbidden | Insufficient permissions on the scope | Ensure the identity has **Cost Management Reader** role (or higher) on the target scope | +| 404 | Not Found | Invalid scope URL โ€” subscription, resource group, or billing account not found | Verify the scope URL path and resource IDs are correct | +| 424 | Failed Dependency | Bad training data โ€” forecast model cannot compute predictions | Falls back to actual costs if `includeActualCost=true`; otherwise suggest using **azure-cost-query** for historical data | +| 429 | Too Many Requests | Rate limited โ€” QPU quota exceeded | Read `x-ms-ratelimit-microsoft.costmanagement-qpu-retry-after` header and wait before retrying | +| 503 | Service Unavailable | Temporary service issue | Check [Azure Status](https://status.azure.com) for service health. | + +## Validation Error Reference + +| Error Code | Description | Fix | +|---|---|---| +| `EmptyForecastRequestBody` | Request body is empty or null | Provide a complete request body with `type`, `timeframe`, `timePeriod`, and `dataset` | +| `InvalidForecastRequestBody` | Request body has invalid JSON structure | Check JSON syntax โ€” verify braces, commas, and field names | +| `DontContainsDataSet` | The `dataset` field is missing from the request body | Add the `dataset` object with `granularity` and `aggregation` | +| `DontContainsValidTimeRangeWhileContainsPeriod` | `timePeriod` is present but `from` or `to` is invalid | Ensure both `from` and `to` are valid ISO 8601 datetime strings | +| `DontContainsValidTimeRangeWhileMonthlyAndIncludeCost` | Monthly granularity with `includeActualCost=true` but missing valid `timePeriod` | Add explicit `timePeriod` with valid `from` and `to` dates | +| `DontContainIncludeActualCostWhileIncludeFreshPartialCost` | `includeFreshPartialCost=true` without `includeActualCost=true` | Set `includeActualCost=true` or set `includeFreshPartialCost=false` | +| `CantForecastOnThePast` | Both `from` and `to` dates are in the past | Ensure the `to` date is in the future | + +## Forecast-Specific Scenarios + +| Scenario | Response | Action | +|---|---|---| +| "Forecast is unavailable for the specified time period" | Valid response with null/empty rows | Not an error โ€” insufficient history (< 28 days). Suggest using **azure-cost-query** for available historical data. | +| "Can't forecast on the past" | 400 error with `CantForecastOnThePast` | Ensure the `to` date is in the future. | +| Bad training data | 424 Failed Dependency | If `includeActualCost=true`, the response falls back to actual cost data only. Otherwise, suggest using **azure-cost-query** for historical data. | +| Parsing exception | 400 Bad Request | Check JSON format โ€” validate braces, quotes, commas, and field types. | + +## Retry Strategy + +| Status | Retry? | Strategy | +|---|---|---| +| 429 | โœ… Yes | Wait for duration specified in `x-ms-ratelimit-microsoft.costmanagement-qpu-retry-after` header. **Maximum 3 retries.** | +| 400 | โŒ No | Fix the request body based on the validation error code | +| 401 | โŒ No | Re-authenticate โ€” the token is missing or expired | +| 403 | โŒ No | Grant **Cost Management Reader** role on the target scope | +| 404 | โŒ No | Fix the scope URL โ€” verify subscription, resource group, or billing account IDs | +| 424 | โŒ No | Training data issue โ€” retrying will not help. Fall back to actual costs or use **azure-cost-query** | +| 503 | โŒ No | Do not retry. Check [Azure Status](https://status.azure.com) for service health. | + +> โš ๏ธ **Warning:** Do not retry any errors except 429. All other errors indicate issues that must be fixed before re-attempting the request. diff --git a/plugin/skills/azure-cost-forecast/references/examples.md b/plugin/skills/azure-cost-forecast/references/examples.md new file mode 100644 index 00000000..4c153ade --- /dev/null +++ b/plugin/skills/azure-cost-forecast/references/examples.md @@ -0,0 +1,125 @@ +# Forecast API Examples + +Common forecast patterns with request bodies. Use the [SKILL.md workflow](../SKILL.md) to construct and execute the `az rest` command. + +## 1. Forecast Rest of Current Month (Daily) + +```json +{ + "type": "ActualCost", + "timeframe": "Custom", + "timePeriod": { + "from": "", + "to": "" + }, + "dataset": { + "granularity": "Daily", + "aggregation": { + "totalCost": { "name": "Cost", "function": "Sum" } + }, + "sorting": [ + { "direction": "Ascending", "name": "UsageDate" } + ] + }, + "includeActualCost": true, + "includeFreshPartialCost": true +} +``` + +> ๐Ÿ’ก **Tip:** Set `from` to the first of the month โ€” the response contains `Actual` rows up to today and `Forecast` rows for remaining days. + +--- + +## 2. Forecast Next 3 Months (Monthly) + +```json +{ + "type": "ActualCost", + "timeframe": "Custom", + "timePeriod": { + "from": "", + "to": "<3-months-out>" + }, + "dataset": { + "granularity": "Monthly", + "aggregation": { + "totalCost": { "name": "Cost", "function": "Sum" } + }, + "sorting": [ + { "direction": "Ascending", "name": "BillingMonth" } + ] + }, + "includeActualCost": true, + "includeFreshPartialCost": true +} +``` + +> ๐Ÿ’ก **Tip:** Monthly granularity uses the `BillingMonth` column in the response. + +--- + +## 3. Forecast for Resource Group Scope + +```json +{ + "type": "ActualCost", + "timeframe": "Custom", + "timePeriod": { + "from": "", + "to": "" + }, + "dataset": { + "granularity": "Daily", + "aggregation": { + "totalCost": { "name": "Cost", "function": "Sum" } + }, + "sorting": [ + { "direction": "Ascending", "name": "UsageDate" } + ] + }, + "includeActualCost": true, + "includeFreshPartialCost": true +} +``` + +> ๐Ÿ’ก **Tip:** Scope is set at the URL level. Use the resource group scope URL to limit the forecast. + +--- + +## 4. Forecast for Billing Account Scope + +```json +{ + "type": "ActualCost", + "timeframe": "Custom", + "timePeriod": { + "from": "", + "to": "" + }, + "dataset": { + "granularity": "Monthly", + "aggregation": { + "totalCost": { "name": "Cost", "function": "Sum" } + }, + "sorting": [ + { "direction": "Ascending", "name": "BillingMonth" } + ] + }, + "includeActualCost": true, + "includeFreshPartialCost": true +} +``` + +> ๐Ÿ’ก **Tip:** Use URL pattern `/providers/Microsoft.Billing/billingAccounts//...`. Monthly granularity recommended for billing account forecasts. + +--- + +## Scope URL Reference + +| Scope | URL Pattern | +|---|---| +| Subscription | `/subscriptions//providers/Microsoft.CostManagement/forecast` | +| Resource Group | `/subscriptions//resourceGroups//providers/Microsoft.CostManagement/forecast` | +| Billing Account | `/providers/Microsoft.Billing/billingAccounts//providers/Microsoft.CostManagement/forecast` | + +> ๐Ÿ’ก **Tip:** These are path-only patterns โ€” not complete URLs. Append `?api-version=2023-11-01` when constructing the full request URL. diff --git a/plugin/skills/azure-cost-forecast/references/guardrails.md b/plugin/skills/azure-cost-forecast/references/guardrails.md new file mode 100644 index 00000000..e64686dc --- /dev/null +++ b/plugin/skills/azure-cost-forecast/references/guardrails.md @@ -0,0 +1,78 @@ +# Forecast API Guardrails + +Detailed guardrails derived from CCM-LUX getForecastData and CCM-UX-MIDDLEWARE Forecaster. + +## Time Period Validation + +| Rule | Detail | +|---|---| +| `to` date must be in the future | `numberOfDaysToForecast` must be > 0. Entirely past date ranges return a `CantForecastOnThePast` error. | +| `from` can be in the past | When `from` is in the past, the response includes actual costs from `from` to today and forecast costs from today to `to`. | +| Both dates must be valid | When `timePeriod` is present, both `from` and `to` must be valid parseable ISO 8601 datetime strings. | +| Monthly + includeActualCost | Monthly granularity with `includeActualCost=true` requires an explicit `timePeriod` with valid `from` and `to` dates. Omitting it produces `DontContainsValidTimeRangeWhileMonthlyAndIncludeCost`. | +| Maximum forecast period | 10 years maximum forecast window. | + +> โš ๏ธ **Warning:** If both `from` and `to` are in the past, the API returns `CantForecastOnThePast`. At least the `to` date must be in the future. + +## Training Data Requirements + +| Requirement | Value | +|---|---| +| Minimum historical data | 4 weeks (28 days) of cost data | +| Preferred training window | Up to 3 months of history | +| Late arrival tolerance | 2 days for billing data to arrive | +| New subscriptions (< 28 days) | Forecast unavailable | + +> โš ๏ธ **Warning:** New subscriptions with fewer than 28 days of cost history cannot generate forecasts. Suggest using **azure-cost-query** to retrieve available historical data instead. + +## Grouping Restriction + +| Aspect | Detail | +|---|---| +| Grouping support | โŒ **Not supported** | +| API limitation | This is a hard limitation of the Forecast API. The `grouping` field is not accepted in the request body. | +| Workaround | If the user requests a grouped forecast (e.g., forecast by resource group or service), inform them that grouping is not supported for forecasts. Suggest querying historical data with grouping using **azure-cost-query** instead. | + +> โš ๏ธ **Warning:** Even when using **azure-cost-query** for grouped historical data, `ResourceId` grouping is only supported at subscription scope and below. It is not supported at billing account, management group, or higher scopes. + +## Response Row Limit + +| Constraint | Detail | +|---|---| +| Maximum rows | 40 rows per forecast response | +| Daily example | 30 actual days + 30 forecast days = 60 rows โ†’ **exceeds limit** | +| Recommendation | For daily granularity, keep forecast period to ~2โ€“3 weeks | +| Longer periods | Use monthly granularity to stay within the row limit | + +> ๐Ÿ’ก **Tip:** If the user needs a daily forecast for more than 2โ€“3 weeks, consider splitting the request into smaller time windows or switching to monthly granularity. + +## includeActualCost / includeFreshPartialCost + +| Field | Default | Dependency | +|---|---|---| +| `includeActualCost` | `true` | None | +| `includeFreshPartialCost` | `true` | **Requires `includeActualCost=true`** | + +> โš ๏ธ **Warning:** Setting `includeFreshPartialCost=true` without `includeActualCost=true` produces validation error `DontContainIncludeActualCostWhileIncludeFreshPartialCost`. Always set both fields explicitly. + +## Forecast Availability + +The API returns "Forecast is unavailable for the specified time period" when: + +| Condition | Detail | +|---|---| +| Null/empty response rows | Response has no data rows | +| Insufficient training data | Scope has fewer than 28 days of cost history | +| No cost history | Scope has never had any cost data | + +> โš ๏ธ **Warning:** This is **not an error** โ€” it is a valid response indicating the forecast model cannot generate predictions. Do not retry. Instead, suggest using **azure-cost-query** to retrieve whatever historical data is available. + +## Rate Limiting + +| Header | Description | +|---|---| +| `x-ms-ratelimit-microsoft.costmanagement-qpu-retry-after` | Seconds to wait before retrying (QPU-based) | +| `x-ms-ratelimit-microsoft.costmanagement-entity-retry-after` | Seconds to wait for entity-level throttle | +| `x-ms-ratelimit-microsoft.costmanagement-tenant-retry-after` | Seconds to wait for tenant-level throttle | + +The Forecast API uses the same QPU-based rate limiting as the Query API. When a 429 response is received, read the retry-after headers and wait before retrying. diff --git a/plugin/skills/azure-cost-forecast/references/request-body-schema.md b/plugin/skills/azure-cost-forecast/references/request-body-schema.md new file mode 100644 index 00000000..73386184 --- /dev/null +++ b/plugin/skills/azure-cost-forecast/references/request-body-schema.md @@ -0,0 +1,111 @@ +# Forecast API Request Body Schema + +## Complete JSON Schema + +```json +{ + "type": "ActualCost", + "timeframe": "Custom", + "timePeriod": { + "from": "2024-01-01T00:00:00Z", + "to": "2024-03-31T00:00:00Z" + }, + "dataset": { + "granularity": "Daily", + "aggregation": { + "totalCost": { + "name": "Cost", + "function": "Sum" + } + }, + "sorting": [ + { + "direction": "Ascending", + "name": "UsageDate" + } + ], + "filter": { + "dimensions": { + "name": "ResourceGroupName", + "operator": "In", + "values": ["my-resource-group"] + } + } + }, + "includeActualCost": true, + "includeFreshPartialCost": true +} +``` + +## Field Reference + +| Field | Type | Required | Values | Description | +|---|---|---|---|---| +| `type` | string | โœ… | `ActualCost`, `AmortizedCost`, `Usage` | Cost type for the forecast | +| `timeframe` | string | โœ… | `Custom` | Must be `Custom` for forecast requests | +| `timePeriod` | object | โœ… | โ€” | Start and end dates for the forecast window | +| `timePeriod.from` | string | โœ… | ISO 8601 datetime | Start date; can be in the past to include actuals | +| `timePeriod.to` | string | โœ… | ISO 8601 datetime | End date; **must be in the future** for forecast | +| `dataset` | object | โœ… | โ€” | Dataset configuration for the forecast | +| `dataset.granularity` | string | โœ… | `Daily`, `Monthly` | Time granularity of forecast results | +| `dataset.aggregation` | object | โœ… | โ€” | Aggregation functions to apply | +| `dataset.aggregation.totalCost.name` | string | โœ… | `Cost` | Column name to aggregate | +| `dataset.aggregation.totalCost.function` | string | โœ… | `Sum` | Aggregation function | +| `dataset.sorting` | array | Optional | โ€” | Sort order for results | +| `dataset.sorting[].direction` | string | Optional | `Ascending`, `Descending` | Sort direction | +| `dataset.sorting[].name` | string | Optional | `UsageDate` | Column to sort by | +| `dataset.filter` | object | Optional | โ€” | Filter expression (dimensions/tags) | +| `includeActualCost` | boolean | Optional | `true`, `false` | Include historical actual costs alongside forecast. Default: `true` | +| `includeFreshPartialCost` | boolean | Optional | `true`, `false` | Include partial cost data for recent days. Default: `true`. **Requires `includeActualCost=true`** | + +## Forecast-Specific Fields + +### `includeActualCost` + +- **Type:** boolean +- **Default:** `true` +- When `true`, the response includes historical actual cost rows from the `from` date up to today, alongside projected forecast rows from today to the `to` date. +- When `false`, only forecast (projected) rows are returned. + +### `includeFreshPartialCost` + +- **Type:** boolean +- **Default:** `true` +- When `true`, includes partial (incomplete) cost data for the most recent days where billing data is still arriving. +- โš ๏ธ **Requires `includeActualCost=true`.** Setting `includeFreshPartialCost=true` without `includeActualCost=true` produces a validation error (`DontContainIncludeActualCostWhileIncludeFreshPartialCost`). + +## Response Structure + +### Response Columns + +| Column | Type | Description | +|---|---|---| +| `Cost` | Number | The cost amount (actual or forecasted) | +| `UsageDate` / `BillingMonth` | Datetime | The date for the cost row | +| `CostStatus` | String | Indicates whether the row is historical or projected | +| `Currency` | String | Currency code (e.g., `USD`, `EUR`) | + +### CostStatus Values + +| Value | Meaning | +|---|---| +| `Actual` | Historical cost data (already incurred) | +| `Forecast` | Projected future cost (model prediction) | + +### Granularity and Date Column Mapping + +| Granularity | Date Column | +|---|---| +| `Daily` | `UsageDate` | +| `Monthly` | `BillingMonth` | + +## Key Differences from Query API Request Body + +| Aspect | Forecast API | Query API | +|---|---|---| +| Grouping | โŒ Not supported | โœ… Supported via `grouping` field | +| `timeframe` | Typically `Custom` only | Supports `Custom`, `MonthToDate`, `BillingMonthToDate`, etc. | +| `includeActualCost` | โœ… Forecast-specific field | โŒ Not applicable | +| `includeFreshPartialCost` | โœ… Forecast-specific field | โŒ Not applicable | +| Response `CostStatus` column | โœ… Distinguishes `Actual` vs `Forecast` rows | โŒ Not present | +| `to` date | Must be in the future | Can be any valid past/present date | diff --git a/plugin/skills/azure-cost-query/SKILL.md b/plugin/skills/azure-cost-query/SKILL.md new file mode 100644 index 00000000..5b49bb92 --- /dev/null +++ b/plugin/skills/azure-cost-query/SKILL.md @@ -0,0 +1,201 @@ +--- +name: azure-cost-query +description: "Query and analyze historical Azure cost data using the Cost Management Query API. Construct API requests to break down costs by service, resource, location, or tag across subscriptions, resource groups, and billing accounts. WHEN: query Azure costs, cost breakdown by service, Azure spending, cost by resource group, actual cost, amortized cost, cost trends, top cost drivers, cost by subscription, how much did I spend, show my Azure bill, cost by tag, monthly cost summary. DO NOT USE FOR: forecasting future costs (use azure-cost-forecast), reducing costs or optimization recommendations (use azure-cost-optimization)." +license: MIT +metadata: + author: Microsoft + version: "1.0.0" +--- + +# Azure Cost Query Skill + +Query historical Azure cost data using the Cost Management Query API. Supports cost breakdowns by service, resource, location, tag, and other dimensions across any Azure scope. + +## Quick Reference + +| Property | Value | +|----------|-------| +| **API Endpoint** | `POST {scope}/providers/Microsoft.CostManagement/query?api-version=2023-11-01` | +| **MCP Tools** | `azure__documentation`, `azure__extension_cli_generate`, `azure__get_azure_bestpractices` | +| **CLI** | `az rest` | +| **Best For** | Historical cost analysis, cost breakdowns, spend tracking | +| **Required Role** | Cost Management Reader on target scope | + +## When to Use This Skill + +Use this skill when the user asks to: + +- Query historical cost data for subscriptions, resource groups, or billing accounts +- Break down costs by service, resource, location, or tag +- Analyze cost trends over time (daily or monthly granularity) +- Filter costs by specific dimensions (e.g., service name, charge type) +- Compare actual vs. amortized costs +- Identify top cost drivers across Azure resources +- View a cost summary for a specific time period + +> โš ๏ธ **Warning:** Do **not** use this skill for cost forecasting (use `azure-cost-forecast`) or cost optimization recommendations (use `azure-cost-optimization`). + +## MCP Tools + +| Tool | Description | When to Use | +|------|-------------|-------------| +| `azure__documentation` | Search Azure documentation | Research Cost Management API parameters and options | +| `azure__extension_cli_generate` | Generate Azure CLI commands | Construct `az rest` commands for cost queries | +| `azure__get_azure_bestpractices` | Get Azure best practices | Inform query design with cost management best practices | + +> ๐Ÿ’ก **Tip:** Prefer MCP tools over direct CLI commands. Use `az rest` only when MCP tools don't cover the specific operation. + +## Workflow + +### Step 1: Determine Scope + +Identify the Azure scope for the cost query. The scope defines the cost boundary. + +| Scope | URL Pattern | +|-------|-------------| +| Subscription | `/subscriptions/` | +| Resource Group | `/subscriptions//resourceGroups/` | +| Management Group | `/providers/Microsoft.Management/managementGroups/` | +| Billing Account | `/providers/Microsoft.Billing/billingAccounts/` | +| Billing Profile | `/providers/Microsoft.Billing/billingAccounts//billingProfiles/` | + +> ๐Ÿ’ก **Tip:** These are scope paths only โ€” not complete URLs. Combine with the API endpoint and version: `{scope}/providers/Microsoft.CostManagement/query?api-version=2023-11-01` + +### Step 2: Choose Report Type + +| Type | Description | +|------|-------------| +| `ActualCost` | Actual billed costs including purchases | +| `AmortizedCost` | Reservation/savings plan costs spread across usage period | +| `Usage` | Usage-based cost data | + +### Step 3: Set Timeframe + +Use a preset timeframe (e.g., `MonthToDate`, `TheLastMonth`, `TheLastYear`) or `Custom` with a `timePeriod` object. + +> โš ๏ธ **Warning:** Key time period guardrails: +> - **Daily granularity**: max **31 days** +> - **Monthly/None granularity**: max **12 months** +> - `Custom` timeframe **requires** a `timePeriod` object with `from` and `to` dates +> - Future dates are not allowed for historical queries +> +> See [guardrails.md](./references/guardrails.md) for the complete set of validation rules. + +### Step 4: Configure Dataset + +Define granularity, aggregation, grouping, filtering, and sorting in the `dataset` object. + +- **Granularity**: `None`, `Daily`, or `Monthly` +- **Aggregation**: Use `Sum` on `Cost` or `PreTaxCost` for total cost +- **Grouping**: Up to **2** `GroupBy` dimensions (e.g., `ServiceName`, `ResourceGroupName`) +- **Filtering**: Use `dimensions` filters with `In`, `Equal`, or `Contains` operators +- **Sorting**: Order results by cost or dimension columns + +> ๐Ÿ’ก **Tip:** Not all dimensions are available at every scope. See [dimensions-by-scope.md](./references/dimensions-by-scope.md) for the availability matrix. + +For the full request body schema, see [request-body-schema.md](./references/request-body-schema.md). + +### Step 5: Construct and Execute the API Call + +Use `az rest` to call the Cost Management Query API. + +**Create cost query file:** + +Create `temp/cost-query.json` with: +```json +{ + "type": "ActualCost", + "timeframe": "MonthToDate", + "dataset": { + "granularity": "None", + "aggregation": { + "totalCost": { + "name": "Cost", + "function": "Sum" + } + }, + "grouping": [ + { + "type": "Dimension", + "name": "ServiceName" + } + ] + } +} +``` + +**Execute cost query:** +```powershell +# Create temp folder +New-Item -ItemType Directory -Path "temp" -Force + +# Query using REST API (more reliable than az costmanagement query) +az rest --method post ` + --url "/providers/Microsoft.CostManagement/query?api-version=2023-11-01" ` + --body '@temp/cost-query.json' +``` + +### Step 6: Handle Pagination and Errors + +- The API returns a maximum of **5,000 rows** per page (default: 1,000). +- If `nextLink` is present in the response, follow it to retrieve additional pages. +- Handle rate limiting (HTTP 429) by respecting `Retry-After` headers. + +See [error-handling.md](./references/error-handling.md) for the full error reference. + +## Key Guardrails + +| Rule | Constraint | +|------|-----------| +| Daily granularity max range | 31 days | +| Monthly/None granularity max range | 12 months | +| Absolute API max range | 37 months | +| Max GroupBy dimensions | 2 | +| ResourceId grouping scope | Subscription and resource group only โ€” not supported at billing account, management group, or higher scopes | +| Max rows per page | 5,000 | +| Custom timeframe | Requires `timePeriod` with `from`/`to` | +| Filter AND/OR | Must have at least 2 expressions | + +See [guardrails.md](./references/guardrails.md) for the complete guardrails reference. + +## Error Handling + +| HTTP Status | Error | Remediation | +|-------------|-------|-------------| +| 400 | Invalid request body | Check schema, date ranges, and dimension compatibility. | +| 401 | Unauthorized | Verify authentication (`az login`). | +| 403 | Forbidden | Ensure Cost Management Reader role on scope. | +| 404 | Scope not found | Verify scope URL and resource IDs. | +| 429 | Too many requests | Retry after the `x-ms-ratelimit-microsoft.costmanagement-qpu-retry-after` header value. **Max 3 retries.** | +| 503 | Service unavailable | Check [Azure Status](https://status.azure.com). | + +See [error-handling.md](./references/error-handling.md) for detailed error handling including rate limit headers and retry strategies. + +## Examples + +**Cost by service for the current month:** + +```powershell +az rest --method post ` + --url "/subscriptions//providers/Microsoft.CostManagement/query?api-version=2023-11-01" ` + --body '{ + "type": "ActualCost", + "timeframe": "MonthToDate", + "dataset": { + "granularity": "None", + "aggregation": { + "totalCost": { "name": "Cost", "function": "Sum" } + }, + "grouping": [ + { "type": "Dimension", "name": "ServiceName" } + ] + } + }' +``` + +For more examples including daily trends, tag-based filtering, and multi-dimension queries, see [examples.md](./references/examples.md). + +## Related Skills + +- **azure-cost-forecast** โ€” Forecast future Azure costs +- **azure-cost-optimization** โ€” Identify cost savings and optimization recommendations diff --git a/plugin/skills/azure-cost-query/references/dimensions-by-scope.md b/plugin/skills/azure-cost-query/references/dimensions-by-scope.md new file mode 100644 index 00000000..000a49f9 --- /dev/null +++ b/plugin/skills/azure-cost-query/references/dimensions-by-scope.md @@ -0,0 +1,180 @@ +# Dimensions by Scope + +Dimension availability matrix for the Cost Management Query API, organized by scope type and agreement type. + +## Scope URL Patterns + +| Scope | URL Pattern | +|-------|-------------| +| Subscription | `/subscriptions/` | +| Resource Group | `/subscriptions//resourceGroups/` | +| Management Group | `/providers/Microsoft.Management/managementGroups/` | +| Billing Account | `/providers/Microsoft.Billing/billingAccounts/` | +| Billing Profile | `/providers/Microsoft.Billing/billingAccounts//billingProfiles/` | +| Invoice Section | `/providers/Microsoft.Billing/billingAccounts//billingProfiles//invoiceSections/` | +| Department (EA) | `/providers/Microsoft.Billing/billingAccounts//departments/` | +| Enrollment Account (EA) | `/providers/Microsoft.Billing/billingAccounts//enrollmentAccounts/` | +| Customer (Partner) | `/providers/Microsoft.Billing/billingAccounts//customers/` | + +## Common Dimensions + +The following dimensions are available across most scopes: + +> โš ๏ธ **Warning:** `ResourceId` grouping is **only supported at subscription and resource group scopes**. At higher scopes (billing account, management group, billing profile, etc.), use `ServiceName`, `SubscriptionName`, or another supported dimension instead. See [guardrails.md](guardrails.md) for the full scope restriction table. + +| Dimension | Description | +|-----------|-------------| +| `ResourceGroupName` | Resource group containing the resource. | +| `ResourceId` | Full Azure resource ID. | +| `ResourceLocation` | Azure region where the resource is deployed. | +| `ServiceName` | Azure service name (e.g., Virtual Machines, Storage). | +| `ServiceFamily` | Service family grouping (e.g., Compute, Storage, Networking). | +| `ServiceTier` | Service tier or SKU tier (e.g., Standard, Premium). | +| `MeterCategory` | Top-level meter classification. | +| `MeterSubCategory` | Meter sub-classification. | +| `Meter` | Specific meter name. | +| `ChargeType` | Type of charge (e.g., Usage, Purchase, Refund). | +| `PublisherType` | Publisher type (e.g., Azure, Marketplace, AWS). | +| `PricingModel` | Pricing model (e.g., OnDemand, Reservation, SavingsPlan, Spot). | +| `SubscriptionName` | Subscription display name. | +| `SubscriptionId` | Subscription GUID. | +| `TagKey` | Azure resource tag key (use with `TagKey` column type in grouping). | +| `Product` | Product name from the price sheet. | +| `BenefitName` | Reservation or savings plan name. | +| `BillingPeriod` | Billing period identifier. | + +## Scope-Specific Dimensions + +Additional dimensions available only at certain scopes: + +| Scope | Additional Dimensions | +|-------|-----------------------| +| Subscription | _(common dimensions only)_ | +| Resource Group | _(common dimensions only)_ | +| Management Group | `DepartmentName`, `EnrollmentAccountName` | +| Billing Account | `BillingProfileName`, `DepartmentName`, `EnrollmentAccountName`, `InvoiceSectionName`, `Customer` | +| Billing Profile | `InvoiceSectionName`, `Customer` | +| Invoice Section | _(common dimensions only)_ | +| Department (EA) | `EnrollmentAccountName` | +| Enrollment Account (EA) | _(common dimensions only)_ | +| Customer (Partner) | _(common dimensions only)_ | + +## Agreement Type Dimension Sets + +Available dimensions vary by agreement type. Only dimensions listed for your agreement type are valid in GroupBy and Filter expressions. + +### EA (Enterprise Agreement) + +| Dimension | Available | +|-----------|-----------| +| `ResourceGroupName` | โœ… | +| `ResourceId` | โœ… | +| `SubscriptionName` | โœ… | +| `Product` | โœ… | +| `ResourceLocation` | โœ… | +| `ServiceName` | โœ… | +| `ServiceFamily` | โœ… | +| `TagKey` | โœ… | +| `MeterSubCategory` | โœ… | +| `PublisherType` | โœ… | +| `PricingModel` | โœ… | +| `ChargeType` | โœ… | +| `ServiceTier` | โœ… | +| `BenefitName` | โœ… | +| `BillingProfileName` | โœ… | +| `DepartmentName` | โœ… | +| `EnrollmentAccountName` | โœ… | +| `BillingPeriod` | โœ… | + +### MCA (Microsoft Customer Agreement) + +| Dimension | Available | +|-----------|-----------| +| `ResourceGroupName` | โœ… | +| `ResourceId` | โœ… | +| `SubscriptionName` | โœ… | +| `Product` | โœ… | +| `ResourceLocation` | โœ… | +| `ServiceName` | โœ… | +| `ServiceFamily` | โœ… | +| `TagKey` | โœ… | +| `MeterSubCategory` | โœ… | +| `PublisherType` | โœ… | +| `PricingModel` | โœ… | +| `ChargeType` | โœ… | +| `ServiceTier` | โœ… | +| `BenefitName` | โœ… | +| `BillingProfileName` | โœ… | +| `InvoiceSectionName` | โœ… | + +### MOSP (Microsoft Online Services Program / Pay-As-You-Go) + +| Dimension | Available | +|-----------|-----------| +| `ResourceGroupName` | โœ… | +| `ResourceId` | โœ… | +| `SubscriptionName` | โœ… | +| `Product` | โœ… | +| `ResourceLocation` | โœ… | +| `ServiceName` | โœ… | +| `ServiceFamily` | โœ… | +| `TagKey` | โœ… | +| `MeterSubCategory` | โœ… | +| `PublisherType` | โœ… | +| `PricingModel` | โœ… | +| `ChargeType` | โœ… | +| `ServiceTier` | โœ… | +| `BenefitName` | โœ… | + +### Comparison Summary + +| Dimension | EA | MCA | MOSP | +|-----------|----|----|------| +| `ResourceGroupName` | โœ… | โœ… | โœ… | +| `ResourceId` | โœ… | โœ… | โœ… | +| `SubscriptionName` | โœ… | โœ… | โœ… | +| `Product` | โœ… | โœ… | โœ… | +| `ResourceLocation` | โœ… | โœ… | โœ… | +| `ServiceName` | โœ… | โœ… | โœ… | +| `ServiceFamily` | โœ… | โœ… | โœ… | +| `TagKey` | โœ… | โœ… | โœ… | +| `MeterSubCategory` | โœ… | โœ… | โœ… | +| `PublisherType` | โœ… | โœ… | โœ… | +| `PricingModel` | โœ… | โœ… | โœ… | +| `ChargeType` | โœ… | โœ… | โœ… | +| `ServiceTier` | โœ… | โœ… | โœ… | +| `BenefitName` | โœ… | โœ… | โœ… | +| `BillingProfileName` | โœ… | โœ… | โŒ | +| `DepartmentName` | โœ… | โŒ | โŒ | +| `EnrollmentAccountName` | โœ… | โŒ | โŒ | +| `InvoiceSectionName` | โŒ | โœ… | โŒ | +| `BillingPeriod` | โœ… | โŒ | โŒ | + +## Scope Resolution Priority + +When multiple scope identifiers are available in context, use the following priority order (highest first): + +| Priority | Scope | Notes | +|----------|-------|-------| +| 1 | Management Group | Broadest organizational scope. | +| 2 | Resource Group | Narrowest resource scope. | +| 3 | Subscription | Default scope for most queries. | +| 4 | Billing Profile + Invoice Section | MCA billing hierarchy. | +| 5 | Billing Profile | MCA billing scope. | +| 6 | Department | EA organizational unit. | +| 7 | Enrollment Account | EA enrollment scope. | +| 8 | Customer | Partner/CSP customer scope. | +| 9 | Invoice Section | Standalone invoice section. | +| 10 | Billing Account | Top-level billing scope. | + +> ๐Ÿ’ก **Tip:** When a user provides both a subscription and a resource group, prefer the Resource Group scope (priority 2) over Subscription (priority 3) for more targeted results. + +## Required Context Variables + +The following context variables are needed to resolve scope and validate dimensions: + +| Variable | Description | Used For | +|----------|-------------|----------| +| `AgreementType` | The agreement type (`EA`, `MCA`, `MOSP`). | Determines valid dimension set. | +| `AccountType` | Account type within the agreement. | Refines scope-specific behavior. | +| `CallScopeId` | The fully qualified scope URL for the API call. | Builds the request URL path. | diff --git a/plugin/skills/azure-cost-query/references/error-handling.md b/plugin/skills/azure-cost-query/references/error-handling.md new file mode 100644 index 00000000..b63ad21c --- /dev/null +++ b/plugin/skills/azure-cost-query/references/error-handling.md @@ -0,0 +1,67 @@ +# Cost Query Error Handling + +Detailed error handling reference for the Cost Management Query API. + +## HTTP Status Codes + +| Status | Error | Cause | Remediation | +|--------|-------|-------|-------------| +| 400 | `BadRequest` | Invalid request body, unsupported dimension, date range exceeds limits, malformed filter expression. | Validate request body against [request-body-schema.md](request-body-schema.md). Check dimension compatibility in [dimensions-by-scope.md](dimensions-by-scope.md). | +| 401 | `Unauthorized` | Missing or expired authentication token. | Re-authenticate with `az login` or refresh the bearer token. | +| 403 | `Forbidden` | Insufficient permissions on the target scope. User lacks Cost Management Reader or equivalent role. | Assign `Cost Management Reader` or `Cost Management Contributor` role on the scope. | +| 404 | `NotFound` | Scope does not exist, subscription not found, or resource group does not exist. | Verify the scope URL. Confirm the subscription ID and resource group name are correct. | +| 429 | `TooManyRequests` | Rate limit exceeded. QPU, entity, or tenant throttling triggered. | Retry after the duration specified in the `x-ms-ratelimit-microsoft.costmanagement-qpu-retry-after` header. | +| 503 | `ServiceUnavailable` | Cost Management service is temporarily unavailable. | Check [Azure Status](https://status.azure.com) for service health. | + +## Common Error Scenarios + +| Error Message / Scenario | Cause | Remediation | +|--------------------------|-------|-------------| +| "Agreement type X does not support Y scope" | Scope type is incompatible with the account's agreement type. | Use a compatible scope. EA accounts cannot use Invoice Section scope; MOSP accounts cannot use Department scope. | +| "Dimension Z is not valid for scope" | The requested dimension is not available for the current scope and agreement type combination. | Check [dimensions-by-scope.md](dimensions-by-scope.md) for valid dimensions. | +| "SubscriptionName filter without SubscriptionId" | EA + Management Group scope: filtering by `SubscriptionName` without also filtering by `SubscriptionId`. | Add a `SubscriptionId` filter alongside the `SubscriptionName` filter. See [guardrails.md](guardrails.md). | +| Date range exceeds granularity limit | `Daily` range > 31 days or `Monthly`/`None` range > 12 months. | System auto-truncates `from` date. To avoid silent truncation, ensure range is within limits. | +| Date range exceeds absolute limit (37 months) | `from` to `to` spans more than 37 months. | Reduce the date range to 37 months or less. Split into multiple queries if needed. | +| "Request body is null or invalid" | Missing or malformed JSON in the request body. | Validate JSON syntax. Ensure `type`, `timeframe`, and `dataset` fields are present. | +| Invalid filter structure | `And`/`Or` has fewer than 2 child expressions, or `Not` has more than 1. | Ensure `And`/`Or` contain 2+ children. Use `Not` with exactly 1 child. For single conditions, use the filter directly without a logical wrapper. | +| "The query usage is not supported for the scope" | The query type (e.g., `AmortizedCost`) is not supported at the given scope. | Try a different scope or query type. Not all scopes support all report types. | +| `BillingSubscriptionNotFound` | The subscription ID in the scope URL is invalid or not associated with the billing account. | Verify the subscription ID exists and is active. Check that it belongs to the expected billing account. | + +## Retry Strategy + +| Status | Retry? | Strategy | +|--------|--------|----------| +| 429 | โœ… Yes | Wait for the duration specified in the `x-ms-ratelimit-microsoft.costmanagement-qpu-retry-after` response header, then retry. **Maximum 3 retries.** | +| 400 | โŒ No | Fix the request. Review error message for specific field or validation issue. | +| 401 | โŒ No | Re-authenticate. Token has expired or is missing. | +| 403 | โŒ No | Fix permissions. Request appropriate RBAC role assignment on the scope. | +| 404 | โŒ No | Fix the scope URL. Verify resource exists. | +| 503 | โŒ No | Do not retry. Check [Azure Status](https://status.azure.com) for service health. | +| 5xx (other) | โŒ No | Do not retry. Investigate the error and check service health. | + +> โš ๏ธ **Warning:** Do not retry any errors except 429. All other errors indicate issues that must be fixed before re-attempting the request. + +## Error Response Structure + +All error responses follow a consistent JSON structure: + +```json +{ + "error": { + "code": "", + "message": "", + "details": [ + { + "code": "", + "message": "" + } + ] + } +} +``` + +| Field | Description | +|-------|-------------| +| `error.code` | Machine-readable error code (e.g., `BadRequest`, `BillingSubscriptionNotFound`). | +| `error.message` | Human-readable description of the error. | +| `error.details` | Optional array of additional detail objects with more specific error information. | diff --git a/plugin/skills/azure-cost-query/references/examples.md b/plugin/skills/azure-cost-query/references/examples.md new file mode 100644 index 00000000..03d96e37 --- /dev/null +++ b/plugin/skills/azure-cost-query/references/examples.md @@ -0,0 +1,127 @@ +# Cost Management Query Examples + +Common query patterns with request bodies. Use the [SKILL.md workflow](../SKILL.md) to construct and execute the `az rest` command. + +## 1. Monthly Cost by Service + +```json +{ + "type": "ActualCost", + "timeframe": "MonthToDate", + "dataset": { + "granularity": "None", + "aggregation": { + "totalCost": { "name": "Cost", "function": "Sum" } + }, + "grouping": [ + { "type": "Dimension", "name": "ServiceName" } + ], + "sorting": [ + { "direction": "Descending", "name": "Cost" } + ] + } +} +``` + +--- + +## 2. Daily Cost Trend (Last 30 Days) + +```json +{ + "type": "ActualCost", + "timeframe": "Custom", + "timePeriod": { + "from": "2024-01-01T00:00:00Z", + "to": "2024-01-31T23:59:59Z" + }, + "dataset": { + "granularity": "Daily", + "aggregation": { + "totalCost": { "name": "Cost", "function": "Sum" } + } + } +} +``` + +> โš ๏ธ **Warning:** Daily granularity supports a maximum of 31 days. + +--- + +## 3. Cost by Resource Group with Tag Filter + +```json +{ + "type": "ActualCost", + "timeframe": "MonthToDate", + "dataset": { + "granularity": "None", + "aggregation": { + "totalCost": { "name": "Cost", "function": "Sum" } + }, + "grouping": [ + { "type": "Dimension", "name": "ResourceGroupName" } + ], + "filter": { + "Tags": { + "Name": "Environment", + "Operator": "In", + "Values": ["production", "staging"] + } + }, + "sorting": [ + { "direction": "Descending", "name": "Cost" } + ] + } +} +``` + +--- + +## 4. Amortized Cost for Reservation Analysis + +```json +{ + "type": "AmortizedCost", + "timeframe": "TheLastMonth", + "dataset": { + "granularity": "None", + "aggregation": { + "totalCost": { "name": "Cost", "function": "Sum" } + }, + "grouping": [ + { "type": "Dimension", "name": "BenefitName" } + ], + "sorting": [ + { "direction": "Descending", "name": "Cost" } + ] + } +} +``` + +> ๐Ÿ’ก **Tip:** `AmortizedCost` spreads reservation purchases across the term for accurate daily/monthly effective cost. + +--- + +## 5. Top 10 Most Expensive Resources + +```json +{ + "type": "ActualCost", + "timeframe": "MonthToDate", + "dataset": { + "granularity": "None", + "aggregation": { + "totalCost": { "name": "Cost", "function": "Sum" } + }, + "grouping": [ + { "type": "Dimension", "name": "ResourceId" } + ], + "sorting": [ + { "direction": "Descending", "name": "Cost" } + ] + } +} +``` + +> ๐Ÿ’ก **Tip:** Append `&$top=10` to the URL to limit results: `...query?api-version=2023-11-01&$top=10` diff --git a/plugin/skills/azure-cost-query/references/guardrails.md b/plugin/skills/azure-cost-query/references/guardrails.md new file mode 100644 index 00000000..86706880 --- /dev/null +++ b/plugin/skills/azure-cost-query/references/guardrails.md @@ -0,0 +1,164 @@ +# Cost Query Guardrails + +Detailed validation rules and guardrails for the Cost Management Query API. The system applies these automatically, but understanding them helps avoid unexpected query modifications or errors. + +## Time Period Validation + +### Default Behavior + +| Scenario | System Behavior | +|----------|----------------| +| No time period specified | Defaults to current month start โ†’ today. | +| `from` is after `to` | System silently swaps `from` and `to`. | + +### Future Date Handling + +| Scenario | System Behavior | +|----------|----------------| +| Both `from` and `to` are in the future | Entire period shifted to the equivalent period last year. | +| Only `to` is in the future | `to` is adjusted to today's date. | + +> โš ๏ธ **Warning:** Future date shifting happens silently. The response data will cover the adjusted period, not the originally requested dates. + +### Granularity-Based Range Limits + +| Granularity | Maximum Range | Truncation Behavior | +|-------------|---------------|---------------------| +| `Daily` | 31 days | `from` truncated to `to - 1 month + 1 day`. | +| `Monthly` | 12 months | `from` truncated to `to - 12 months + 1 day`. | +| `None` | 12 months | `from` truncated to `to - 12 months + 1 day`. | + +> โš ๏ธ **Warning:** The absolute API limit is **37 months**. Requests exceeding 37 months return HTTP 400 regardless of granularity. + +### Minimum Start Date + +| Constraint | Value | +|------------|-------| +| Earliest allowed `from` date | May 1, 2014 | + +### GroupBy Interaction with Time Period + +| Combination | System Behavior | +|-------------|----------------| +| GroupBy + Daily granularity | Time period adjusted to the last day of the requested range. | +| GroupBy + Monthly granularity | Time period adjusted to the last month of the requested range. | + +> ๐Ÿ’ก **Tip:** When using GroupBy with Daily granularity over a multi-day range, the system may return data only for the last day. For full daily breakdown with grouping, ensure the range is within the 31-day limit. + +## ResourceId Scope Restriction + +> โš ๏ธ **Warning:** Grouping by `ResourceId` is **only supported at subscription scope and below** (subscription, resource group). It is NOT supported at higher scopes. + +| Scope | ResourceId GroupBy | +|-------|--------------------| +| Subscription | โœ… Supported | +| Resource Group | โœ… Supported | +| Billing Account | โŒ Not supported | +| Management Group | โŒ Not supported | +| Billing Profile | โŒ Not supported | +| Department (EA) | โŒ Not supported | +| Enrollment Account (EA) | โŒ Not supported | +| Invoice Section (MCA) | โŒ Not supported | +| Customer (Partner) | โŒ Not supported | + +When the user requests a cost breakdown by resource at a billing account or management group scope, use `ServiceName`, `SubscriptionName`, or another supported dimension instead. If per-resource detail is needed, narrow the scope to a specific subscription first. + +## Dataset Validation + +### GroupBy Constraints + +| Rule | Limit | Error Behavior | +|------|-------|----------------| +| Maximum GroupBy dimensions | 2 | Validation error if more than 2 specified. | +| Duplicate columns in GroupBy | Not allowed | Validation error on duplicate column names. | +| Same column in Aggregation and GroupBy | Not allowed | Validation error if a column appears in both. | + +### Aggregation Constraints + +| Rule | Details | +|------|---------| +| Standard queries aggregation function | Only `Sum` is allowed. | +| `Date` in aggregation with granularity | Not allowed. Cannot aggregate on `Date` when granularity is `Daily` or `Monthly`. | + +### Filter Constraints + +| Rule | Details | +|------|---------| +| `And` operator | Must have 2 or more child expressions. | +| `Or` operator | Must have 2 or more child expressions. | +| `Not` operator | Must have exactly 1 child expression. | + +> โš ๏ธ **Warning:** A filter with a single child in `And` or `Or` will fail validation. Wrap single-condition filters directly without a logical operator, or use `Not` for negation. + +## Scope & Dimension Compatibility + +Dimensions must be valid for the intersection of the agreement type **and** scope type. + +| Agreement Type | Unique Dimensions | Reference | +|----------------|-------------------|-----------| +| EA | `DepartmentName`, `EnrollmentAccountName`, `BillingPeriod` | See [dimensions-by-scope.md](dimensions-by-scope.md) | +| MCA | `InvoiceSectionName` | See [dimensions-by-scope.md](dimensions-by-scope.md) | +| MOSP | _(common dimensions only)_ | See [dimensions-by-scope.md](dimensions-by-scope.md) | + +| Validation | Error Behavior | +|------------|----------------| +| Dimension not valid for agreement type | `BillingSubscriptionNotFound` or dimension validation error. | +| Dimension not valid for scope type | `BadRequest` with invalid dimension message. | + +> โš ๏ธ **Warning:** Using an EA-only dimension (e.g., `DepartmentName`) on a MOSP subscription will return a validation error. Always verify the agreement type before selecting dimensions. + +## EA + Management Group Special Case + +| Scenario | Result | +|----------|--------| +| Filter by `SubscriptionName` without `SubscriptionId` at Management Group scope | Error returned. | +| Error message | _"To view cost data, the subscription ID is needed. Select Subscriptions to find the ID for your subscription, and then ask your question again."_ | + +**Remediation:** When filtering by subscription name at Management Group scope under EA, always include a `SubscriptionId` filter alongside the `SubscriptionName` filter. + +```json +{ + "And": [ + { + "Dimensions": { + "Name": "SubscriptionId", + "Operator": "In", + "Values": [""] + } + }, + { + "Dimensions": { + "Name": "SubscriptionName", + "Operator": "In", + "Values": ["My Subscription"] + } + } + ] +} +``` + +## Rate Limiting + +### QPU-Based Throttling + +| Tier | Description | +|------|-------------| +| Premium | Higher QPU allocation (EA, MCA enterprise). | +| Non-premium | Lower QPU allocation (MOSP, trial). | + +### Rate Limit Headers + +| Header | Description | +|--------|-------------| +| `x-ms-ratelimit-microsoft.costmanagement-qpu-retry-after` | Seconds to wait before retrying (QPU limit). | +| `x-ms-ratelimit-microsoft.costmanagement-entity-retry-after` | Seconds to wait before retrying (entity limit). | +| `x-ms-ratelimit-microsoft.costmanagement-tenant-retry-after` | Seconds to wait before retrying (tenant limit). | + +### Pagination + +| Parameter | Default | Maximum | +|-----------|---------|---------| +| Page size | 1,000 rows | 5,000 rows | +| Pagination | Use `nextLink` from response to fetch subsequent pages. | โ€” | + +> ๐Ÿ’ก **Tip:** For large result sets, always check the `nextLink` field in the response. If present, make additional requests to retrieve all pages. diff --git a/plugin/skills/azure-cost-query/references/request-body-schema.md b/plugin/skills/azure-cost-query/references/request-body-schema.md new file mode 100644 index 00000000..7b4a85a4 --- /dev/null +++ b/plugin/skills/azure-cost-query/references/request-body-schema.md @@ -0,0 +1,170 @@ +# Cost Management Query API โ€” Request Body Schema + +Schema for the [Cost Management Query API](https://learn.microsoft.com/en-us/rest/api/cost-management/query/usage). + +## Request Body Structure + +```json +{ + "type": "", + "timeframe": "", + "timePeriod": { "from": "2024-01-01T00:00:00Z", "to": "2024-01-31T23:59:59Z" }, + "dataset": { + "granularity": "", + "aggregation": { "": { "name": "", "function": "" } }, + "grouping": [{ "type": "", "name": "" }], + "filter": { "" }, + "sorting": [{ "direction": "", "name": "" }] + } +} +``` + +## Top-Level Fields + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `type` | string | Yes | `ActualCost`, `AmortizedCost`, or `Usage` | +| `timeframe` | string | Yes | Predefined or `Custom` time window | +| `timePeriod` | object | Conditional | Required when `timeframe` is `Custom`. Contains `from`/`to` ISO 8601 dates. | +| `dataset` | object | Yes | Defines granularity, aggregation, grouping, filtering, sorting | + +### Timeframe Values + +`WeekToDate` ยท `MonthToDate` ยท `BillingMonthToDate` ยท `YearToDate` ยท `TheLastWeek` ยท `TheLastMonth` ยท `TheLastBillingMonth` ยท `TheLastYear` ยท `TheLast7Days` ยท `TheLast3Months` ยท `Custom` + +## Dataset Fields + +### Granularity + +| Value | Max Range | Description | +|-------|-----------|-------------| +| `None` | 12 months | Aggregated total, no date breakdown | +| `Daily` | 31 days | Day-by-day breakdown | +| `Monthly` | 12 months | Month-by-month breakdown | + +### Aggregation + +```json +"aggregation": { "totalCost": { "name": "Cost", "function": "Sum" } } +``` + +| Field | Required | Description | +|-------|----------|-------------| +| `` (key) | Yes | Output column alias (e.g., `totalCost`) | +| `name` | Yes | Source column: `Cost`, `PreTaxCost`, or `UsageQuantity` | +| `function` | Yes | `Sum` (only supported function for cost queries) | + +### Grouping + +```json +"grouping": [ + { "type": "Dimension", "name": "ServiceName" }, + { "type": "TagKey", "name": "Environment" } +] +``` + +- `type`: `Dimension` (built-in) or `TagKey` (resource tag) +- Maximum **2** GroupBy dimensions per query. No duplicates. + +### Filter + +Filter expressions restrict which cost records are included. Filters support logical operators (`And`, `Or`, `Not`) and comparison operators on dimensions or tags. + +#### Filter Expression Structure + +```json +"filter": { + "And": [ + { + "Dimensions": { + "Name": "ResourceGroupName", + "Operator": "In", + "Values": ["rg-prod", "rg-staging"] + } + }, + { + "Not": { + "Tags": { + "Name": "Environment", + "Operator": "Equal", + "Values": ["dev"] + } + } + } + ] +} +``` + +#### Logical Operators + +| Operator | Description | Children | +|----------|-------------|----------| +| `And` | All child expressions must match. | 2 or more expressions. | +| `Or` | Any child expression must match. | 2 or more expressions. | +| `Not` | Negates a single child expression. | Exactly 1 expression. | + +> โš ๏ธ **Warning:** `And` and `Or` must contain at least 2 child expressions. `Not` must contain exactly 1. + +#### Comparison Operators (ComparisonOperator Enum) + +| Operator | Description | Example | +|----------|-------------|---------| +| `In` | Value is in the provided list. Supports multiple values. | `"Values": ["vm", "storage"]` | +| `Equal` | Exact match against a single value. | `"Values": ["production"]` | +| `Contains` | String contains the specified substring. | `"Values": ["prod"]` | +| `LessThan` | Numeric less-than comparison. | `"Values": ["100"]` | +| `GreaterThan` | Numeric greater-than comparison. | `"Values": ["0"]` | +| `NotEqual` | Value does not match the specified value. | `"Values": ["dev"]` | + +#### Filter Target Types + +| Target | Description | +|--------|-------------| +| `Dimensions` | Filter on built-in dimensions (e.g., `ResourceGroupName`, `ServiceName`). | +| `Tags` | Filter on Azure resource tags (e.g., `Environment`, `CostCenter`). | + +### Sorting + +```json +"sorting": [ + { "direction": "Descending", "name": "Cost" } +] +``` + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `direction` | string | Yes | `Ascending` or `Descending`. | +| `name` | string | Yes | Column name to sort by (must be present in aggregation or grouping). | + +## Response Structure + +```json +{ + "id": "", + "name": "", + "type": "Microsoft.CostManagement/query", + "properties": { + "nextLink": "", + "columns": [ + { "name": "Cost", "type": "Number" }, + { "name": "ServiceName", "type": "String" }, + { "name": "UsageDate", "type": "Number" }, + { "name": "Currency", "type": "String" } + ], + "rows": [ + [123.45, "Virtual Machines", 20240115, "USD"], + [67.89, "Storage", 20240115, "USD"] + ] + } +} +``` + +| Field | Type | Description | +|-------|------|-------------| +| `columns` | array | Array of column definitions with `name` and `type`. | +| `columns[].name` | string | Column name. | +| `columns[].type` | string | Data type: `Number` or `String`. | +| `rows` | array | Array of row arrays. Values ordered to match `columns`. | +| `nextLink` | string | URL for next page of results, or `null` if no more pages. | + +> ๐Ÿ’ก **Tip:** `UsageDate` is returned as a number in `YYYYMMDD` format (e.g., `20240115`) when granularity is `Daily` or `Monthly`. diff --git a/tests/azure-cost-forecast/__snapshots__/triggers.test.ts.snap b/tests/azure-cost-forecast/__snapshots__/triggers.test.ts.snap new file mode 100644 index 00000000..09254010 --- /dev/null +++ b/tests/azure-cost-forecast/__snapshots__/triggers.test.ts.snap @@ -0,0 +1,91 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`azure-cost-forecast - Trigger Tests Trigger Keywords Snapshot skill description triggers match snapshot 1`] = ` +{ + "description": "Forecast future Azure costs using the Cost Management Forecast API. Builds and executes forecast requests with proper time-period guardrails, training-data validation, and response interpretation. WHEN: forecast Azure costs, predict spending, projected costs, estimate bill, future Azure costs, cost projection, budget forecast, end of month costs, how much will I spend. DO NOT USE FOR: querying historical costs (use azure-cost-query), reducing costs (use azure-cost-optimization).", + "extractedKeywords": [ + "azure", + "azure-cost-optimization", + "azure-cost-query", + "bill", + "budget", + "builds", + "cli", + "cost", + "costs", + "estimate", + "executes", + "forecast", + "function", + "future", + "guardrails", + "historical", + "interpretation", + "management", + "mcp", + "month", + "much", + "predict", + "projected", + "projection", + "proper", + "querying", + "reducing", + "requests", + "response", + "spend", + "spending", + "time-period", + "training-data", + "using", + "validation", + "when", + "will", + "with", + ], + "name": "azure-cost-forecast", +} +`; + +exports[`azure-cost-forecast - Trigger Tests Trigger Keywords Snapshot skill keywords match snapshot 1`] = ` +[ + "azure", + "azure-cost-optimization", + "azure-cost-query", + "bill", + "budget", + "builds", + "cli", + "cost", + "costs", + "estimate", + "executes", + "forecast", + "function", + "future", + "guardrails", + "historical", + "interpretation", + "management", + "mcp", + "month", + "much", + "predict", + "projected", + "projection", + "proper", + "querying", + "reducing", + "requests", + "response", + "spend", + "spending", + "time-period", + "training-data", + "using", + "validation", + "when", + "will", + "with", +] +`; diff --git a/tests/azure-cost-forecast/fixtures/sample.json b/tests/azure-cost-forecast/fixtures/sample.json new file mode 100644 index 00000000..234a345a --- /dev/null +++ b/tests/azure-cost-forecast/fixtures/sample.json @@ -0,0 +1,38 @@ +{ + "samplePrompts": { + "shouldTrigger": [ + "What will my Azure costs be next month?", + "Forecast my Azure spending for the rest of the quarter", + "Predict my subscription costs for the next 90 days", + "Show me projected costs for this billing period", + "Estimate my Azure bill for next month" + ], + "shouldNotTrigger": [ + "What were my Azure costs last month?", + "Show me cost breakdown by service", + "How do I reduce my Azure spending?", + "Deploy a new VM to Azure", + "Set up an AWS budget" + ] + }, + "sampleInput": { + "description": "Sample forecast request body", + "type": "ActualCost", + "timeframe": "Custom", + "timePeriod": { + "from": "2026-03-01T00:00:00Z", + "to": "2026-03-31T23:59:59Z" + }, + "dataset": { + "granularity": "Daily", + "aggregation": { + "totalCost": { "name": "Cost", "function": "Sum" } + } + }, + "includeActualCost": true, + "includeFreshPartialCost": true + }, + "expectedOutput": { + "description": "Expected response contains columns with CostStatus and rows" + } +} diff --git a/tests/azure-cost-forecast/integration.test.ts b/tests/azure-cost-forecast/integration.test.ts new file mode 100644 index 00000000..8febd02e --- /dev/null +++ b/tests/azure-cost-forecast/integration.test.ts @@ -0,0 +1,128 @@ +/** + * Integration Tests for azure-cost-forecast + * + * Tests skill behavior with a real Copilot agent session. + * Runs prompts multiple times to measure skill invocation rate. + * + * Prerequisites: + * 1. npm install -g @github/copilot-cli + * 2. Run `copilot` and authenticate + */ + +import { + useAgentRunner, + shouldSkipIntegrationTests, + getIntegrationSkipReason, + doesAssistantMessageIncludeKeyword +} from "../utils/agent-runner"; +import { softCheckSkill } from "../utils/evaluate"; + +const SKILL_NAME = "azure-cost-forecast"; +const RUNS_PER_PROMPT = 5; + +// Check if integration tests should be skipped at module level +const skipTests = shouldSkipIntegrationTests(); +const skipReason = getIntegrationSkipReason(); + +if (skipTests && skipReason) { + console.log(`โญ๏ธ Skipping integration tests: ${skipReason}`); +} + +const describeIntegration = skipTests ? describe.skip : describe; + +describeIntegration(`${SKILL_NAME}_ - Integration Tests`, () => { + const agent = useAgentRunner(); + + describe("skill-invocation", () => { + test("invokes azure-cost-forecast skill for future cost prompt", async () => { + for (let i = 0; i < RUNS_PER_PROMPT; i++) { + try { + const agentMetadata = await agent.run({ + prompt: "What will my Azure costs be next month?" + }); + + softCheckSkill(agentMetadata, SKILL_NAME); + } catch (e: unknown) { + if (e instanceof Error && e.message?.includes("Failed to load @github/copilot-sdk")) { + console.log("โญ๏ธ SDK not loadable, skipping test"); + return; + } + throw e; + } + } + }); + + test("invokes azure-cost-forecast skill for quarterly forecast prompt", async () => { + for (let i = 0; i < RUNS_PER_PROMPT; i++) { + try { + const agentMetadata = await agent.run({ + prompt: "Forecast my Azure spending for the rest of the quarter" + }); + + softCheckSkill(agentMetadata, SKILL_NAME); + } catch (e: unknown) { + if (e instanceof Error && e.message?.includes("Failed to load @github/copilot-sdk")) { + console.log("โญ๏ธ SDK not loadable, skipping test"); + return; + } + throw e; + } + } + }); + + test("invokes skill for subscription cost prediction prompt", async () => { + for (let i = 0; i < RUNS_PER_PROMPT; i++) { + try { + const agentMetadata = await agent.run({ + prompt: "Predict my subscription costs for the next 90 days" + }); + + softCheckSkill(agentMetadata, SKILL_NAME); + } catch (e: unknown) { + if (e instanceof Error && e.message?.includes("Failed to load @github/copilot-sdk")) { + console.log("โญ๏ธ SDK not loadable, skipping test"); + return; + } + throw e; + } + } + }); + }); + + test("response contains forecast-related keywords", async () => { + try { + const agentMetadata = await agent.run({ + prompt: "What will my Azure costs be next month?" + }); + + const hasForecast = doesAssistantMessageIncludeKeyword(agentMetadata, "forecast") || + doesAssistantMessageIncludeKeyword(agentMetadata, "projected") || + doesAssistantMessageIncludeKeyword(agentMetadata, "estimate"); + expect(hasForecast).toBe(true); + } catch (e: unknown) { + if (e instanceof Error && e.message?.includes("Failed to load @github/copilot-sdk")) { + console.log("โญ๏ธ SDK not loadable, skipping test"); + return; + } + throw e; + } + }); + + test("response mentions Cost Management for forecast", async () => { + try { + const agentMetadata = await agent.run({ + prompt: "Forecast my Azure spending for next quarter" + }); + + const mentionsCostManagement = doesAssistantMessageIncludeKeyword(agentMetadata, "Cost Management") || + doesAssistantMessageIncludeKeyword(agentMetadata, "az costmanagement"); + expect(mentionsCostManagement).toBe(true); + } catch (e: unknown) { + if (e instanceof Error && e.message?.includes("Failed to load @github/copilot-sdk")) { + console.log("โญ๏ธ SDK not loadable, skipping test"); + return; + } + throw e; + } + }); +}); diff --git a/tests/azure-cost-forecast/triggers.test.ts b/tests/azure-cost-forecast/triggers.test.ts new file mode 100644 index 00000000..86f114ec --- /dev/null +++ b/tests/azure-cost-forecast/triggers.test.ts @@ -0,0 +1,99 @@ +/** + * Trigger Tests for azure-cost-forecast + * + * Tests that verify the skill triggers on appropriate prompts + * and does NOT trigger on unrelated prompts. + */ + +import { TriggerMatcher } from "../utils/trigger-matcher"; +import { loadSkill, LoadedSkill } from "../utils/skill-loader"; + +const SKILL_NAME = "azure-cost-forecast"; + +describe(`${SKILL_NAME} - Trigger Tests`, () => { + let triggerMatcher: TriggerMatcher; + let skill: LoadedSkill; + + beforeAll(async () => { + skill = await loadSkill(SKILL_NAME); + triggerMatcher = new TriggerMatcher(skill); + }); + + describe("Should Trigger", () => { + const shouldTriggerPrompts: string[] = [ + "What will my Azure costs be next month?", + "Forecast my Azure spending for the rest of the quarter", + "Predict my subscription costs for the next 90 days", + "Show me projected costs for this billing period", + "Estimate my Azure bill for next month", + "How much will I spend on Azure by end of year?", + "Show my forecast for Azure costs going forward", + ]; + + test.each(shouldTriggerPrompts)( + 'triggers on: "%s"', + (prompt) => { + const result = triggerMatcher.shouldTrigger(prompt); + expect(result.triggered).toBe(true); + } + ); + }); + + describe("Should NOT Trigger", () => { + const shouldNotTriggerPrompts: string[] = [ + // Query skill (should not trigger cost-forecast) + "Analyze actual and amortized cost trends by service", + "What is my amortized cost trend this year?", + // Optimization skill (should not trigger cost-forecast) + "Find orphaned resources and rightsize VMs", + "Reduce waste and optimize cloud expenses", + // Deployment (different skill) + "Deploy a new VM to Azure", + // Wrong cloud provider + "Set up an AWS budget", + // Unrelated + "Write a Python script", + ]; + + test.each(shouldNotTriggerPrompts)( + 'does not trigger on: "%s"', + (prompt) => { + const result = triggerMatcher.shouldTrigger(prompt); + expect(result.triggered).toBe(false); + } + ); + }); + + describe("Trigger Keywords Snapshot", () => { + test("skill keywords match snapshot", () => { + expect(triggerMatcher.getKeywords()).toMatchSnapshot(); + }); + + test("skill description triggers match snapshot", () => { + expect({ + name: skill.metadata.name, + description: skill.metadata.description, + extractedKeywords: triggerMatcher.getKeywords() + }).toMatchSnapshot(); + }); + }); + + describe("Edge Cases", () => { + test("handles empty prompt", () => { + const result = triggerMatcher.shouldTrigger(""); + expect(result.triggered).toBe(false); + }); + + test("handles very long prompt", () => { + const longPrompt = "Azure cost forecast spending ".repeat(500); + const result = triggerMatcher.shouldTrigger(longPrompt); + expect(typeof result.triggered).toBe("boolean"); + }); + + test("is case insensitive", () => { + const result1 = triggerMatcher.shouldTrigger("FORECAST AZURE COSTS"); + const result2 = triggerMatcher.shouldTrigger("forecast azure costs"); + expect(result1.triggered).toBe(result2.triggered); + }); + }); +}); diff --git a/tests/azure-cost-forecast/unit.test.ts b/tests/azure-cost-forecast/unit.test.ts new file mode 100644 index 00000000..df0e58e3 --- /dev/null +++ b/tests/azure-cost-forecast/unit.test.ts @@ -0,0 +1,154 @@ +/** + * Unit Tests for azure-cost-forecast + * + * Tests isolated skill logic and validation rules. + */ + +import { readFileSync } from "node:fs"; +import { loadSkill, LoadedSkill } from "../utils/skill-loader"; + +const SKILL_NAME = "azure-cost-forecast"; + +describe(`${SKILL_NAME} - Unit Tests`, () => { + let skill: LoadedSkill; + + beforeAll(async () => { + skill = await loadSkill(SKILL_NAME); + }); + + describe("Skill Metadata", () => { + test("has valid SKILL.md with required fields", () => { + expect(skill.metadata).toBeDefined(); + expect(skill.metadata.name).toBe(SKILL_NAME); + expect(skill.metadata.description).toBeDefined(); + expect(skill.metadata.description.length).toBeGreaterThan(10); + }); + + test("description is concise and actionable", () => { + expect(skill.metadata.description.length).toBeGreaterThan(50); + expect(skill.metadata.description.length).toBeLessThan(1000); + }); + + test("description contains forecast-related keywords", () => { + const description = skill.metadata.description.toLowerCase(); + expect(description).toMatch(/forecast|predict|project|estimate|future/); + }); + + test("description mentions key use cases", () => { + const description = skill.metadata.description.toLowerCase(); + expect(description).toContain("when:"); + expect(description).toMatch(/forecast|predict|projected|estimate/); + }); + + test("description clarifies what NOT to use it for", () => { + const description = skill.metadata.description.toLowerCase(); + expect(description).toMatch(/do not\s+use for/); + expect(description).toMatch(/historical.*azure-cost-query|optimization.*azure-cost-optimization/); + }); + }); + + describe("Skill Content", () => { + test("has substantive content", () => { + expect(skill.content).toBeDefined(); + expect(skill.content.length).toBeGreaterThan(100); + }); + + test("contains Quick Reference section", () => { + expect(skill.content).toMatch(/## Quick Reference/i); + }); + + test("contains When to Use section", () => { + expect(skill.content).toMatch(/## When to Use/i); + }); + + test("contains MCP Tools section", () => { + expect(skill.content).toMatch(/## MCP Tools/i); + }); + + test("contains Workflow section", () => { + expect(skill.content).toMatch(/## Workflow/i); + }); + + test("contains Error Handling section", () => { + expect(skill.content).toMatch(/## Error Handling/i); + }); + + test("contains Guardrails section", () => { + expect(skill.content).toMatch(/Guardrails/i); + }); + }); + + describe("MCP Tool References", () => { + test("references azure__documentation tool", () => { + expect(skill.content).toContain("azure__documentation"); + }); + + test("references azure__extension_cli_generate tool", () => { + expect(skill.content).toContain("azure__extension_cli_generate"); + }); + + test("references azure__get_azure_bestpractices tool", () => { + expect(skill.content).toContain("azure__get_azure_bestpractices"); + }); + }); + + describe("Forecast API References", () => { + test("references forecast API endpoint or resource type", () => { + expect(skill.content).toMatch(/Microsoft\.CostManagement\/forecast|forecast\s+API/i); + }); + }); + + describe("Forecast Guardrails", () => { + test("mentions to-date must be in the future", () => { + const content = skill.content.toLowerCase(); + expect(content).toMatch(/future|must be in the future/); + }); + + test("mentions grouping not supported", () => { + const content = skill.content.toLowerCase(); + expect(content).toContain("grouping"); + expect(content).toMatch(/not supported/); + }); + + test("mentions includeActualCost field", () => { + expect(skill.content).toContain("includeActualCost"); + }); + + test("mentions minimum training data requirement", () => { + const content = skill.content.toLowerCase(); + expect(content).toMatch(/training data|28 days/); + }); + }); + + describe("Response Type References", () => { + test("references CostStatus or response types", () => { + expect(skill.content).toMatch(/CostStatus|("Actual".*"Forecast"|Actual.*Forecast)/); + }); + }); + + describe("Frontmatter Formatting", () => { + test("frontmatter has no tabs", () => { + const raw = readFileSync(skill.filePath, "utf-8"); + const frontmatter = raw.split("---")[1]; + expect(frontmatter).not.toMatch(/\t/); + }); + + test("frontmatter keys are only supported attributes", () => { + const raw = readFileSync(skill.filePath, "utf-8"); + const frontmatter = raw.split("---")[1]; + const supported = ["name", "description", "compatibility", "license", "metadata", + "argument-hint", "disable-model-invocation", "user-invokable"]; + const keys = frontmatter.split("\n") + .filter((l: string) => /^[a-z][\w-]*\s*:/.test(l)) + .map((l: string) => l.split(":")[0].trim()); + for (const key of keys) { + expect(supported).toContain(key); + } + }); + + test("WHEN clause is inside description", () => { + const description = skill.metadata.description; + expect(description).toContain("WHEN:"); + }); + }); +}); diff --git a/tests/azure-cost-query/__snapshots__/triggers.test.ts.snap b/tests/azure-cost-query/__snapshots__/triggers.test.ts.snap new file mode 100644 index 00000000..bc367932 --- /dev/null +++ b/tests/azure-cost-query/__snapshots__/triggers.test.ts.snap @@ -0,0 +1,111 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`azure-cost-query - Trigger Tests Trigger Keywords Snapshot skill description triggers match snapshot 1`] = ` +{ + "description": "Query and analyze historical Azure cost data using the Cost Management Query API. Construct API requests to break down costs by service, resource, location, or tag across subscriptions, resource groups, and billing accounts. WHEN: query Azure costs, cost breakdown by service, Azure spending, cost by resource group, actual cost, amortized cost, cost trends, top cost drivers, cost by subscription, how much did I spend, show my Azure bill, cost by tag, monthly cost summary. DO NOT USE FOR: forecasting future costs (use azure-cost-forecast), reducing costs or optimization recommendations (use azure-cost-optimization).", + "extractedKeywords": [ + "accounts", + "across", + "actual", + "amortized", + "analyze", + "authentication", + "azure", + "azure-cost-forecast", + "azure-cost-optimization", + "bill", + "billing", + "break", + "breakdown", + "cli", + "construct", + "cost", + "costs", + "data", + "down", + "drivers", + "forecasting", + "function", + "future", + "group", + "groups", + "historical", + "location", + "management", + "mcp", + "monthly", + "much", + "optimization", + "query", + "recommendations", + "reducing", + "requests", + "resource", + "service", + "show", + "spend", + "spending", + "subscription", + "subscriptions", + "summary", + "trends", + "using", + "validation", + "when", + ], + "name": "azure-cost-query", +} +`; + +exports[`azure-cost-query - Trigger Tests Trigger Keywords Snapshot skill keywords match snapshot 1`] = ` +[ + "accounts", + "across", + "actual", + "amortized", + "analyze", + "authentication", + "azure", + "azure-cost-forecast", + "azure-cost-optimization", + "bill", + "billing", + "break", + "breakdown", + "cli", + "construct", + "cost", + "costs", + "data", + "down", + "drivers", + "forecasting", + "function", + "future", + "group", + "groups", + "historical", + "location", + "management", + "mcp", + "monthly", + "much", + "optimization", + "query", + "recommendations", + "reducing", + "requests", + "resource", + "service", + "show", + "spend", + "spending", + "subscription", + "subscriptions", + "summary", + "trends", + "using", + "validation", + "when", +] +`; diff --git a/tests/azure-cost-query/fixtures/sample.json b/tests/azure-cost-query/fixtures/sample.json new file mode 100644 index 00000000..7dbb80c7 --- /dev/null +++ b/tests/azure-cost-query/fixtures/sample.json @@ -0,0 +1,35 @@ +{ + "samplePrompts": { + "shouldTrigger": [ + "What are my Azure costs this month?", + "Show me cost breakdown by service", + "Query Azure spending for the last 30 days", + "How much did I spend on storage last month?", + "Break down costs by resource group" + ], + "shouldNotTrigger": [ + "What will my Azure costs be next month?", + "How do I reduce my Azure spending?", + "Deploy a new VM to Azure", + "Set up an AWS budget", + "Write a Python script" + ] + }, + "sampleInput": { + "description": "Sample cost query request body", + "type": "ActualCost", + "timeframe": "MonthToDate", + "dataset": { + "granularity": "None", + "aggregation": { + "totalCost": { "name": "Cost", "function": "Sum" } + }, + "grouping": [ + { "type": "Dimension", "name": "ServiceName" } + ] + } + }, + "expectedOutput": { + "description": "Expected response contains columns and rows arrays" + } +} diff --git a/tests/azure-cost-query/integration.test.ts b/tests/azure-cost-query/integration.test.ts new file mode 100644 index 00000000..dd677915 --- /dev/null +++ b/tests/azure-cost-query/integration.test.ts @@ -0,0 +1,110 @@ +/** + * Integration Tests for azure-cost-query + * + * Tests skill behavior with a real Copilot agent session. + * Runs prompts multiple times to measure skill invocation rate. + * + * Prerequisites: + * 1. npm install -g @github/copilot-cli + * 2. Run `copilot` and authenticate + */ + +import { + useAgentRunner, + shouldSkipIntegrationTests, + getIntegrationSkipReason, + doesAssistantMessageIncludeKeyword +} from "../utils/agent-runner"; +import { softCheckSkill } from "../utils/evaluate"; + +const SKILL_NAME = "azure-cost-query"; +const RUNS_PER_PROMPT = 5; + +// Check if integration tests should be skipped at module level +const skipTests = shouldSkipIntegrationTests(); +const skipReason = getIntegrationSkipReason(); + +if (skipTests && skipReason) { + console.log(`โญ๏ธ Skipping integration tests: ${skipReason}`); +} + +const describeIntegration = skipTests ? describe.skip : describe; + +describeIntegration(`${SKILL_NAME}_ - Integration Tests`, () => { + const agent = useAgentRunner(); + + describe("skill-invocation", () => { + test("invokes azure-cost-query skill for monthly cost prompt", async () => { + for (let i = 0; i < RUNS_PER_PROMPT; i++) { + try { + const agentMetadata = await agent.run({ + prompt: "What are my Azure costs this month?" + }); + + softCheckSkill(agentMetadata, SKILL_NAME); + } catch (e: unknown) { + if (e instanceof Error && e.message?.includes("Failed to load @github/copilot-sdk")) { + console.log("โญ๏ธ SDK not loadable, skipping test"); + return; + } + throw e; + } + } + }); + + test("invokes azure-cost-query skill for cost breakdown prompt", async () => { + for (let i = 0; i < RUNS_PER_PROMPT; i++) { + try { + const agentMetadata = await agent.run({ + prompt: "Show me cost breakdown by service for my subscription" + }); + + softCheckSkill(agentMetadata, SKILL_NAME); + } catch (e: unknown) { + if (e instanceof Error && e.message?.includes("Failed to load @github/copilot-sdk")) { + console.log("โญ๏ธ SDK not loadable, skipping test"); + return; + } + throw e; + } + } + }); + + test("invokes skill for resource group cost query", async () => { + for (let i = 0; i < RUNS_PER_PROMPT; i++) { + try { + const agentMetadata = await agent.run({ + prompt: "Break down costs by resource group for the last 30 days" + }); + + softCheckSkill(agentMetadata, SKILL_NAME); + } catch (e: unknown) { + if (e instanceof Error && e.message?.includes("Failed to load @github/copilot-sdk")) { + console.log("โญ๏ธ SDK not loadable, skipping test"); + return; + } + throw e; + } + } + }); + }); + + test("response mentions Cost Management for cost query", async () => { + try { + const agentMetadata = await agent.run({ + prompt: "What are my Azure costs this month?" + }); + + const mentionsCostManagement = doesAssistantMessageIncludeKeyword(agentMetadata, "Cost Management") || + doesAssistantMessageIncludeKeyword(agentMetadata, "cost") || + doesAssistantMessageIncludeKeyword(agentMetadata, "query"); + expect(mentionsCostManagement).toBe(true); + } catch (e: unknown) { + if (e instanceof Error && e.message?.includes("Failed to load @github/copilot-sdk")) { + console.log("โญ๏ธ SDK not loadable, skipping test"); + return; + } + throw e; + } + }); +}); diff --git a/tests/azure-cost-query/triggers.test.ts b/tests/azure-cost-query/triggers.test.ts new file mode 100644 index 00000000..ebdd905a --- /dev/null +++ b/tests/azure-cost-query/triggers.test.ts @@ -0,0 +1,101 @@ +/** + * Trigger Tests for azure-cost-query + * + * Tests that verify the skill triggers on appropriate prompts + * and does NOT trigger on unrelated prompts. + */ + +import { TriggerMatcher } from "../utils/trigger-matcher"; +import { loadSkill, LoadedSkill } from "../utils/skill-loader"; + +const SKILL_NAME = "azure-cost-query"; + +describe(`${SKILL_NAME} - Trigger Tests`, () => { + let triggerMatcher: TriggerMatcher; + let skill: LoadedSkill; + + beforeAll(async () => { + skill = await loadSkill(SKILL_NAME); + triggerMatcher = new TriggerMatcher(skill); + }); + + describe("Should Trigger", () => { + const shouldTriggerPrompts: string[] = [ + // Direct cost query requests + "What are my Azure costs this month?", + "Show me cost breakdown by service for my subscription", + "Query Azure spending for the last 30 days", + "How much did I spend on storage last month?", + "Show me a cost breakdown by resource group", + "Show me actual vs amortized cost for my subscription", + "What are my top cost drivers in Azure?", + ]; + + test.each(shouldTriggerPrompts)( + 'triggers on: "%s"', + (prompt) => { + const result = triggerMatcher.shouldTrigger(prompt); + expect(result.triggered).toBe(true); + } + ); + }); + + describe("Should NOT Trigger", () => { + const shouldNotTriggerPrompts: string[] = [ + // Forecast skill (should not trigger cost-query) + "Predict the budget for next quarter", + "What will the projected budget look like next quarter?", + // Optimization skill (should not trigger cost-query) + "Find orphaned resources and rightsize VMs", + "Reduce waste and optimize cloud expenses", + // Deployment (different skill) + "Deploy a new VM to Azure", + // Wrong cloud provider + "Set up an AWS budget", + // Unrelated + "Write a Python script", + "Help me write a poem", + ]; + + test.each(shouldNotTriggerPrompts)( + 'does not trigger on: "%s"', + (prompt) => { + const result = triggerMatcher.shouldTrigger(prompt); + expect(result.triggered).toBe(false); + } + ); + }); + + describe("Trigger Keywords Snapshot", () => { + test("skill keywords match snapshot", () => { + expect(triggerMatcher.getKeywords()).toMatchSnapshot(); + }); + + test("skill description triggers match snapshot", () => { + expect({ + name: skill.metadata.name, + description: skill.metadata.description, + extractedKeywords: triggerMatcher.getKeywords() + }).toMatchSnapshot(); + }); + }); + + describe("Edge Cases", () => { + test("handles empty prompt", () => { + const result = triggerMatcher.shouldTrigger(""); + expect(result.triggered).toBe(false); + }); + + test("handles very long prompt", () => { + const longPrompt = "Azure cost query breakdown ".repeat(500); + const result = triggerMatcher.shouldTrigger(longPrompt); + expect(typeof result.triggered).toBe("boolean"); + }); + + test("is case insensitive", () => { + const result1 = triggerMatcher.shouldTrigger("AZURE COST BREAKDOWN"); + const result2 = triggerMatcher.shouldTrigger("azure cost breakdown"); + expect(result1.triggered).toBe(result2.triggered); + }); + }); +}); diff --git a/tests/azure-cost-query/unit.test.ts b/tests/azure-cost-query/unit.test.ts new file mode 100644 index 00000000..7dceb940 --- /dev/null +++ b/tests/azure-cost-query/unit.test.ts @@ -0,0 +1,130 @@ +/** + * Unit Tests for azure-cost-query + * + * Tests isolated skill logic and validation rules. + */ + +import { readFileSync } from "node:fs"; +import { loadSkill, LoadedSkill } from "../utils/skill-loader"; + +const SKILL_NAME = "azure-cost-query"; + +describe(`${SKILL_NAME} - Unit Tests`, () => { + let skill: LoadedSkill; + + beforeAll(async () => { + skill = await loadSkill(SKILL_NAME); + }); + + describe("Skill Metadata", () => { + test("has valid SKILL.md with required fields", () => { + expect(skill.metadata).toBeDefined(); + expect(skill.metadata.name).toBe(SKILL_NAME); + expect(skill.metadata.description).toBeDefined(); + expect(skill.metadata.description.length).toBeGreaterThan(10); + }); + + test("description is concise and actionable", () => { + expect(skill.metadata.description.length).toBeGreaterThan(50); + expect(skill.metadata.description.length).toBeLessThan(1000); + }); + + test("description contains cost-query-related keywords", () => { + const description = skill.metadata.description.toLowerCase(); + expect(description).toMatch(/cost|query|spend|breakdown|actual|amortized/); + }); + + test("description mentions key use cases", () => { + const description = skill.metadata.description.toLowerCase(); + expect(description).toContain("when:"); + expect(description).toMatch(/query|breakdown|spending/); + }); + + test("description clarifies what NOT to use it for", () => { + const description = skill.metadata.description.toLowerCase(); + expect(description).toMatch(/do not\s+use for/); + expect(description).toMatch(/forecasting.*azure-cost-forecast|optimization.*azure-cost-optimization/); + }); + }); + + describe("Skill Content", () => { + test("has substantive content", () => { + expect(skill.content).toBeDefined(); + expect(skill.content.length).toBeGreaterThan(100); + }); + + test("contains Quick Reference section", () => { + expect(skill.content).toMatch(/## Quick Reference/i); + }); + + test("contains When to Use section", () => { + expect(skill.content).toMatch(/## When to Use/i); + }); + + test("contains MCP Tools section", () => { + expect(skill.content).toMatch(/## MCP Tools/i); + }); + + test("contains Workflow section", () => { + expect(skill.content).toMatch(/## Workflow/i); + }); + + test("contains Error Handling section", () => { + expect(skill.content).toMatch(/## Error Handling/i); + }); + + test("contains Guardrails section", () => { + expect(skill.content).toMatch(/Guardrails/i); + }); + + test("references MCP tools", () => { + expect(skill.content).toContain("azure__documentation"); + expect(skill.content).toContain("azure__extension_cli_generate"); + expect(skill.content).toContain("azure__get_azure_bestpractices"); + }); + + test("references Cost Management API endpoint", () => { + expect(skill.content).toMatch(/Microsoft\.CostManagement\/query/); + }); + + test("mentions key guardrails", () => { + const content = skill.content.toLowerCase(); + expect(content).toMatch(/granularity/); + expect(content).toMatch(/date range/); + expect(content).toMatch(/groupby/i); + }); + + test("references scopes", () => { + const content = skill.content.toLowerCase(); + expect(content).toMatch(/subscription/); + expect(content).toMatch(/resource group/); + expect(content).toMatch(/billing account/); + }); + }); + + describe("Frontmatter Formatting", () => { + test("frontmatter has no tabs", () => { + const raw = readFileSync(skill.filePath, "utf-8"); + const frontmatter = raw.split("---")[1]; + expect(frontmatter).not.toMatch(/\t/); + }); + + test("frontmatter keys are only supported attributes", () => { + const raw = readFileSync(skill.filePath, "utf-8"); + const frontmatter = raw.split("---")[1]; + const supported = ["name", "description", "compatibility", "license", "metadata", + "argument-hint", "disable-model-invocation", "user-invokable"]; + const keys = frontmatter.split("\n") + .filter((l: string) => /^[a-z][\w-]*\s*:/.test(l)) + .map((l: string) => l.split(":")[0].trim()); + for (const key of keys) { + expect(supported).toContain(key); + } + }); + + test("WHEN clause is inside description", () => { + const description = skill.metadata.description; + expect(description).toContain("WHEN:"); + }); + }); +}); diff --git a/tests/skills.json b/tests/skills.json index f74e0290..8481e6d4 100644 --- a/tests/skills.json +++ b/tests/skills.json @@ -6,7 +6,9 @@ "azure-cloud-migrate", "azure-compliance", "azure-compute", + "azure-cost-forecast", "azure-cost-optimization", + "azure-cost-query", "azure-deploy", "azure-diagnostics", "azure-hosted-copilot-sdk", @@ -25,6 +27,6 @@ "integrationTestSchedule": { "0 5 * * 2-6": "microsoft-foundry", "0 8 * * 2-6": "azure-deploy", - "0 12 * * 2-6": "appinsights-instrumentation,azure-ai,azure-aigateway,azure-cloud-migrate,azure-compliance,azure-compute,azure-cost-optimization,azure-diagnostics,azure-hosted-copilot-sdk,azure-kusto,azure-messaging,azure-prepare,azure-quotas,azure-rbac,azure-resource-lookup,azure-resource-visualizer,azure-storage,azure-validate,entra-app-registration" + "0 12 * * 2-6": "appinsights-instrumentation,azure-ai,azure-aigateway,azure-cloud-migrate,azure-compliance,azure-compute,azure-cost-forecast,azure-cost-optimization,azure-cost-query,azure-diagnostics,azure-hosted-copilot-sdk,azure-kusto,azure-messaging,azure-prepare,azure-quotas,azure-rbac,azure-resource-lookup,azure-resource-visualizer,azure-storage,azure-validate,entra-app-registration" } } \ No newline at end of file