diff --git a/.github/actions/mcp-eval/badges/action.yaml b/.github/actions/mcp-eval/badges/action.yaml new file mode 100644 index 0000000..9152531 --- /dev/null +++ b/.github/actions/mcp-eval/badges/action.yaml @@ -0,0 +1,101 @@ +name: "Generate MCP-Eval Badges" +description: "Generate badges from MCP-Eval JSON report using shields.io" +branding: + icon: award + color: green +inputs: + report-path: + description: "Path to the MCP-Eval JSON report file" + required: true + output-dir: + description: "Directory to write generated badge files (optional, for caching)" + default: "mcpeval-reports/badges" + required: false + format: + description: "Output format: svg, endpoint, or both (default: both)" + default: "both" + required: false + tests-label: + description: "Label text for the tests badge" + default: "mcp-tests" + required: false + coverage-label: + description: "Label text for the coverage badge" + default: "mcp-cov" + required: false + upload-artifacts: + description: "Upload badges as workflow artifacts" + default: "false" + required: false + artifact-name: + description: "Name for the uploaded badge artifacts" + default: "mcpeval-badges" + required: false +outputs: + tests-badge-path: + description: "Path to the generated tests badge SVG (if output-dir is set)" + value: ${{ steps.generate.outputs.tests_badge_path }} + coverage-badge-path: + description: "Path to the generated coverage badge SVG (if output-dir is set)" + value: ${{ steps.generate.outputs.coverage_badge_path }} +runs: + using: "composite" + steps: + - name: Generate badges + id: generate + shell: bash + run: | + set -euo pipefail + + # Check if local script exists, otherwise fetch from upstream + if [ -f "scripts/generate_badges.py" ]; then + echo "Using local badge generation script" + SCRIPT_PATH="scripts/generate_badges.py" + else + echo "Fetching badge generation script from upstream" + # Create a temporary directory for the mcp-eval script + mkdir -p .mcp-eval-action + cd .mcp-eval-action + + # Initialize git and configure sparse checkout + git init + git remote add origin https://github.com/lastmile-ai/mcp-eval.git + git config core.sparseCheckout true + + # Configure sparse checkout to only get the script we need + echo "scripts/generate_badges.py" >> .git/info/sparse-checkout + + # Fetch and checkout the specific file (pinned to a stable commit) + # TODO: Update this to a specific tag/release when available + git fetch --depth=1 origin main + git checkout origin/main + + # Move back to the workspace root + cd .. + SCRIPT_PATH=".mcp-eval-action/scripts/generate_badges.py" + fi + + # Run the badge generation script with uv + uv run "$SCRIPT_PATH" \ + --report "${{ inputs.report-path }}" \ + --outdir "${{ inputs.output-dir }}" \ + --label-tests "${{ inputs.tests-label }}" \ + --label-cov "${{ inputs.coverage-label }}" \ + --format "${{ inputs.format }}" + + # Set output paths if badges were generated + if [ -n "${{ inputs.output-dir }}" ]; then + if [ -f "${{ inputs.output-dir }}/tests.svg" ] && [ -f "${{ inputs.output-dir }}/coverage.svg" ]; then + echo "tests_badge_path=$(realpath ${{ inputs.output-dir }}/tests.svg)" >> $GITHUB_OUTPUT + echo "coverage_badge_path=$(realpath ${{ inputs.output-dir }}/coverage.svg)" >> $GITHUB_OUTPUT + fi + fi + + - name: Upload badge artifacts + if: ${{ inputs.upload-artifacts == 'true' && inputs.output-dir != '' }} + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.artifact-name }} + path: ${{ inputs.output-dir }} + retention-days: 14 + if-no-files-found: warn \ No newline at end of file diff --git a/.github/workflows/mcpeval-reusable.yml b/.github/workflows/mcpeval-reusable.yml index 85c8387..ad0613d 100644 --- a/.github/workflows/mcpeval-reusable.yml +++ b/.github/workflows/mcpeval-reusable.yml @@ -43,6 +43,10 @@ on: required: false type: boolean default: false + deploy-pages-branch: + required: false + type: string + default: 'refs/heads/main' secrets: ANTHROPIC_API_KEY: required: false @@ -53,6 +57,9 @@ jobs: run: name: Run MCP-Eval runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write outputs: json: ${{ steps.mcpeval.outputs.results-json-path }} md: ${{ steps.mcpeval.outputs.results-md-path }} @@ -63,7 +70,7 @@ jobs: - name: Run MCP-Eval id: mcpeval - uses: ./.github/actions/mcp-eval/run + uses: lastmile-ai/mcp-eval/.github/actions/mcp-eval/run@main with: python-version: ${{ inputs.python-version }} working-directory: ${{ inputs.working-directory }} @@ -78,21 +85,49 @@ jobs: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - - name: Generate badges - run: | - set -euo pipefail - uv run scripts/generate_badges.py --report "${{ steps.mcpeval.outputs.results-json-path }}" --outdir mcpeval-reports/badges - - - name: Upload badge artifacts - uses: actions/upload-artifact@v4 + - name: Generate and upload badges + uses: lastmile-ai/mcp-eval/.github/actions/mcp-eval/badges@main with: - name: mcpeval-badges - path: mcpeval-reports/badges + report-path: ${{ steps.mcpeval.outputs.results-json-path }} + output-dir: badges + format: 'both' + upload-artifacts: 'true' + artifact-name: mcpeval-badges + + # Post the Markdown report as a sticky PR comment for easy review + - name: Comment PR with MCP-Eval report + if: ${{ github.event_name == 'pull_request' }} + uses: actions/github-script@v7 + env: + REPORT_PATH: ${{ steps.mcpeval.outputs.results-md-path }} + with: + script: | + const fs = require('fs'); + const path = process.env.REPORT_PATH; + let body = '\n'; + body += '## MCP-Eval Report\n\n'; + try { + const content = fs.readFileSync(path, 'utf8'); + body += content; + } catch (e) { + body += '_No report found at ' + path + '_\n'; + } + const { owner, repo } = context.repo; + const issue_number = context.issue.number; + + // Find existing sticky comment + const { data: comments } = await github.rest.issues.listComments({ owner, repo, issue_number, per_page: 100 }); + const previous = comments.find(c => c.user.type === 'Bot' && c.body.startsWith('')); + if (previous) { + await github.rest.issues.updateComment({ owner, repo, comment_id: previous.id, body }); + } else { + await github.rest.issues.createComment({ owner, repo, issue_number, body }); + } pages: - name: Publish Report to Pages + name: Publish report and badges to GitHub Pages needs: run - if: ${{ inputs.deploy-pages }} + if: ${{ inputs.deploy-pages && github.event_name == 'push' && github.ref == inputs.deploy-pages-branch }} runs-on: ubuntu-latest permissions: pages: write @@ -101,8 +136,20 @@ jobs: environment: name: github-pages url: ${{ steps.deployment.outputs.page_url }} + concurrency: + group: "pages" + cancel-in-progress: false steps: - - name: Download artifacts + - name: Checkout + uses: actions/checkout@v4 + + - name: Download badge artifacts + uses: actions/download-artifact@v4 + with: + name: mcpeval-badges + path: badges/ + + - name: Download report artifacts uses: actions/download-artifact@v4 with: name: ${{ inputs.artifact-name }} @@ -112,6 +159,13 @@ jobs: run: | set -euo pipefail mkdir -p site + + # Copy badges to site + if [[ -d badges ]]; then + cp -r badges site/ + fi + + # Copy HTML report as index.html if [[ -f "mcpeval-artifacts/${{ inputs.reports-dir }}/${{ inputs.html-report }}" ]]; then cp mcpeval-artifacts/${{ inputs.reports-dir }}/${{ inputs.html-report }} site/index.html else @@ -119,6 +173,9 @@ jobs: if [[ -n "$file" ]]; then cp "$file" site/index.html; else echo '

No report available

' > site/index.html; fi fi + - name: Setup Pages + uses: actions/configure-pages@v5 + - name: Upload Pages artifact uses: actions/upload-pages-artifact@v3 with: diff --git a/.github/workflows/mcpeval.yml b/.github/workflows/mcpeval.yml index d1c6eba..b50e689 100644 --- a/.github/workflows/mcpeval.yml +++ b/.github/workflows/mcpeval.yml @@ -10,129 +10,15 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true -permissions: - contents: read - pull-requests: write - jobs: - tests: - name: Run MCP-Eval - runs-on: ubuntu-latest - defaults: - run: - shell: bash - steps: - - name: Checkout - uses: actions/checkout@v4 - - # Tip: set your LLM provider keys in repo/org secrets - # Settings discovery follows mcp-agent/mcpeval configs in your repo. - - name: Run MCP-Eval (uv) - id: mcpeval - uses: ./.github/actions/mcp-eval/run - with: - python-version: "3.11" - working-directory: . - run-args: "-v" - tests: tests/ - reports-dir: mcpeval-reports - json-report: mcpeval-results.json - markdown-report: mcpeval-results.md - html-report: mcpeval-results.html - artifact-name: mcpeval-artifacts - pr-comment: "true" - set-summary: "true" - upload-artifacts: "true" - env: - # Provide at least one provider key; both are supported - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - - - name: Generate badges - run: | - set -euo pipefail - uv run scripts/generate_badges.py --report "${{ steps.mcpeval.outputs.results-json-path }}" --outdir mcpeval-reports/badges - - - name: Upload badge artifacts - uses: actions/upload-artifact@v4 - with: - name: mcpeval-badges - path: mcpeval-reports/badges - - # Post the Markdown report as a sticky PR comment for easy review - - name: Comment PR with MCP-Eval report - if: ${{ github.event_name == 'pull_request' }} - uses: actions/github-script@v7 - env: - REPORT_PATH: ${{ steps.mcpeval.outputs.results-md-path }} - with: - script: | - const fs = require('fs'); - const path = process.env.REPORT_PATH; - let body = '\n'; - body += '## MCP-Eval Report\n\n'; - try { - const content = fs.readFileSync(path, 'utf8'); - body += content; - } catch (e) { - body += '_No report found at ' + path + '_\n'; - } - const { owner, repo } = context.repo; - const issue_number = context.issue.number; - - // Find existing sticky comment - const { data: comments } = await github.rest.issues.listComments({ owner, repo, issue_number, per_page: 100 }); - const previous = comments.find(c => c.user.type === 'Bot' && c.body.startsWith('')); - if (previous) { - await github.rest.issues.updateComment({ owner, repo, comment_id: previous.id, body }); - } else { - await github.rest.issues.createComment({ owner, repo, issue_number, body }); - } - - # Optional: Publish the HTML report to GitHub Pages on main/master pushes - pages: - name: Publish Report to Pages - needs: tests - if: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master') }} - runs-on: ubuntu-latest + call-mcpeval: + uses: lastmile-ai/mcp-eval/.github/workflows/mcpeval-reuseable.yml@main + with: + deploy-pages: true + tests: examples/mcp_server_fetch/tests/test_simple_decorator.py permissions: + contents: read pages: write id-token: write - contents: read - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - steps: - - name: Download artifacts - uses: actions/download-artifact@v4 - with: - name: mcpeval-artifacts - path: ./mcpeval-artifacts - - - name: Prepare site - run: | - set -euo pipefail - mkdir -p site - # Prefer the configured HTML filename; fallback to first HTML we find - if [[ -f "mcpeval-artifacts/mcpeval-reports/mcpeval-results.html" ]]; then - cp mcpeval-artifacts/mcpeval-reports/mcpeval-results.html site/index.html - else - file=$(find mcpeval-artifacts -name "*.html" | head -n 1 || true) - if [[ -n "$file" ]]; then cp "$file" site/index.html; else echo '

No report available

' > site/index.html; fi - fi - # Include badges if available - if [[ -d "mcpeval-artifacts/mcpeval-reports/badges" ]]; then - mkdir -p site/badges - cp -r mcpeval-artifacts/mcpeval-reports/badges/*.svg site/badges/ || true - fi - - - name: Upload Pages artifact - uses: actions/upload-pages-artifact@v3 - with: - path: ./site - - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@v4 - - + pull-requests: write + secrets: inherit \ No newline at end of file diff --git a/docs/ci-cd.mdx b/docs/ci-cd.mdx index 7f0a92b..3874a21 100644 --- a/docs/ci-cd.mdx +++ b/docs/ci-cd.mdx @@ -3,50 +3,98 @@ title: "CI/CD" description: "Run mcp-eval in GitHub Actions, publish artifacts, post PR comments, and add badges." sidebarTitle: "CI/CD" icon: "truck-fast" -keywords: ["github actions","artifacts","pages","badges"] +keywords: ["github actions", "artifacts", "pages", "badges"] --- ### Run action -Use the uv‑based action to install, run, and upload artifacts. +The recommended approach is to use the reusable workflow which handles all the setup, testing, and deployment. For the complete workflow configuration, visit [mcpeval.yml](https://github.com/lastmile-ai/mcp-eval/blob/main/.github/workflows/mcpeval.yml). ```yaml +name: MCP-Eval CI + +on: + push: + branches: [main, master, trunk] + workflow_dispatch: + jobs: - tests: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: lastmile-ai/mcp-eval/.github/actions/mcp-eval/run@v1 - with: - python-version: '3.11' - tests: tests/ - run-args: '-v --max-concurrency 4' - pr-comment: 'true' - set-summary: 'true' - upload-artifacts: 'true' - env: - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + call-mcpeval: + uses: lastmile-ai/mcp-eval/.github/workflows/mcpeval-reusable.yml + with: + deploy-pages: true + permissions: + contents: read + pages: write + id-token: write + pull-requests: write + secrets: inherit +``` + +Alternatively, you can directly use the action in your workflow: + +```yaml +steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Run MCP-Eval (uv) + id: mcpeval + uses: lastmile-ai/mcp-eval/.github/actions/mcp-eval/run + with: + python-version: "3.11" + working-directory: . + run-args: "-v" + tests: tests/ + reports-dir: mcpeval-reports + json-report: mcpeval-results.json + markdown-report: mcpeval-results.md + html-report: mcpeval-results.html + artifact-name: mcpeval-artifacts + pr-comment: "true" + set-summary: "true" + upload-artifacts: "true" + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + + - name: Generate and upload badges + uses: lastmile-ai/mcp-eval/.github/actions/mcp-eval/badges + with: + report-path: ${{ steps.mcpeval.outputs.results-json-path }} + output-dir: badges + format: "both" + upload-artifacts: "true" + artifact-name: mcpeval-badges ``` Sources: + - Action: [action.yaml](https://github.com/lastmile-ai/mcp-eval/blob/main/.github/actions/mcp-eval/run/action.yaml) - README: [run/README.md](https://github.com/lastmile-ai/mcp-eval/blob/main/.github/actions/mcp-eval/run/README.md) - Workflows: [mcpeval.yml](https://github.com/lastmile-ai/mcp-eval/blob/main/.github/workflows/mcpeval.yml), [mcpeval-reusable.yml](https://github.com/lastmile-ai/mcp-eval/blob/main/.github/workflows/mcpeval-reusable.yml) -### Publish HTML via Pages +### Publish HTML and Badges via Pages + +The workflow automatically deploys both the HTML report and badges to GitHub Pages when pushing to main/master branches. After deployment, badges are accessible at `https://.github.io//badges/`. + +To enable GitHub Pages deployment: -Enable the Pages deploy job in the provided workflow. +1. Enable GitHub Pages in your repository settings +2. The workflow will automatically deploy on pushes to main/master +3. Badges and reports will be available at your Pages URL ### Badges -Artifacts include badges under `mcpeval-reports/badges`. Embed in README: +After deployment to GitHub Pages, reference badges using your Pages URL: ```markdown -![MCP Tests](mcpeval-reports/badges/tests.svg) -![MCP Tool Coverage](mcpeval-reports/badges/coverage.svg) +[![mcp-tests](https://img.shields.io/endpoint?url=https://YOUR_USERNAME.github.io/YOUR_REPO/badges/mcp-tests.json&cacheSeconds=300)](https://YOUR_USERNAME.github.io/YOUR_REPO/) +[![mcp-cov](https://img.shields.io/endpoint?url=https://YOUR_USERNAME.github.io/YOUR_REPO/badges/mcp-cov.json&cacheSeconds=300)](https://YOUR_USERNAME.github.io/YOUR_REPO/) ``` -{/* TODO: Add screenshots of the PR comment, the summary, and the published HTML report on Pages. */} - +For example, +[![mcp-tests](https://img.shields.io/endpoint?url=https://lastmile-ai.github.io/mcp-eval/badges/mcp-tests.json&cacheSeconds=300)](https://lastmile-ai.github.io/mcp-eval/) +[![mcp-cov](https://img.shields.io/endpoint?url=https://lastmile-ai.github.io/mcp-eval/badges/mcp-cov.json&cacheSeconds=300)](https://lastmile-ai.github.io/mcp-eval/) +{/* TODO: Add screenshots of the PR comment, the summary, and the published HTML report on Pages. */} diff --git a/docs/common-workflows.mdx b/docs/common-workflows.mdx index acf2dce..4641d1d 100644 --- a/docs/common-workflows.mdx +++ b/docs/common-workflows.mdx @@ -297,75 +297,67 @@ Ensure your agent follows the optimal execution path: - Create `.github/workflows/mcp-eval.yml`: + Create `.github/workflows/mcp-eval.yml` using the reusable workflow: ```yaml - name: mcp-eval Tests + name: MCP-Eval CI on: - pull_request: push: - branches: [main] + branches: [main, master, trunk] + pull_request: + workflow_dispatch: + + # Cancel redundant runs on the same ref + concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: - test: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - - name: Install dependencies - run: | - pip install mcpevals - # Or using uv (faster!): - # uv add mcpevals - # Or from your repo: - # pip install -e . - - - name: Run mcp-eval tests - env: - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - run: | - mcp-eval run tests/ \ - --json test-reports/results.json \ - --markdown test-reports/results.md \ - --html test-reports/index.html - - - name: Upload test reports - if: always() - uses: actions/upload-artifact@v3 - with: - name: mcp-eval-reports - path: test-reports/ - - - name: Comment PR with results - if: github.event_name == 'pull_request' - uses: actions/github-script@v6 - with: - script: | - const fs = require('fs'); - const markdown = fs.readFileSync('test-reports/results.md', 'utf8'); - - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: `## mcp-eval Test Results\n\n${markdown}` - }); + call-mcpeval: + uses: lastmile-ai/mcp-eval/.github/workflows/mcpeval-reusable.yml + with: + deploy-pages: true + # Optional: customize test configuration + # python-version: '3.11' + # tests: 'tests/' + # run-args: '-v --max-concurrency 4' + permissions: + contents: read + pages: write + id-token: write + pull-requests: write + secrets: inherit ``` + + This reusable workflow automatically: + - Runs tests and generates reports + - Posts PR comments with results + - Uploads artifacts + - Deploys badges and HTML reports to GitHub Pages (on main branch) - - In your README.md: + + In your repository settings: + 1. Go to Settings → Pages + 2. Source: Deploy from a branch + 3. Branch: gh-pages (created automatically by the workflow) + 4. Save the settings + + Your badges and reports will be available at: + - Badges: `https://YOUR_USERNAME.github.io/YOUR_REPO/badges/` + - Report: `https://YOUR_USERNAME.github.io/YOUR_REPO/` + + + + After deploying to GitHub Pages, you may add badges to your README.md to show users your mcp-eval test and coverage status: ```markdown - ![mcp-eval Tests](https://github.com/YOUR_ORG/YOUR_REPO/actions/workflows/mcp-eval.yml/badge.svg) + [![mcp-tests](https://img.shields.io/endpoint?url=https://YOUR_USERNAME.github.io/YOUR_REPO/badges/mcp-tests.json&cacheSeconds=300)](https://YOUR_USERNAME.github.io/YOUR_REPO/) + [![mcp-cov](https://img.shields.io/endpoint?url=https://YOUR_USERNAME.github.io/YOUR_REPO/badges/mcp-cov.json&cacheSeconds=300)](https://YOUR_USERNAME.github.io/YOUR_REPO/) ``` + + These badges will automatically update after each push to main. diff --git a/docs/examples.mdx b/docs/examples.mdx index 8bef707..d3539c8 100644 --- a/docs/examples.mdx +++ b/docs/examples.mdx @@ -938,43 +938,35 @@ mcp-eval run examples/ \ ### CI/CD integration ```yaml -# .github/workflows/test.yml -name: Run mcp-eval Tests - -on: [push, pull_request] +name: mcp-eval PR Tests +on: + pull_request: + branches: [ "main" ] jobs: - test: + tests: + permissions: + contents: read + pull-requests: write + issues: write runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Setup Python - uses: actions/setup-python@v4 + - name: Checkout + uses: actions/checkout@v4 + - name: Run MCP-Eval + id: mcpeval + uses: lastmile-ai/mcp-eval/.github/actions/mcp-eval/run with: python-version: '3.11' - - - name: Install dependencies - run: | - # We recommend using uv: - # uv add mcpevals - pip install mcpevals - pip install -r requirements.txt - - - name: Run tests + tests: tests/ + run-args: '-v --max-concurrency 4' + pr-comment: 'true' + set-summary: 'true' + upload-artifacts: 'true' + commit-reports: 'true' env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - run: | - mcp-eval run examples/ \ - --html test-results/report.html \ - --junit test-results/junit.xml - - - name: Upload results - uses: actions/upload-artifact@v3 - with: - name: test-results - path: test-results/ + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} ``` {/* TODO: Screenshot of CI/CD test results in GitHub Actions */} diff --git a/examples/mcp_server_fetch/tests/test_simple_decorator.py b/examples/mcp_server_fetch/tests/test_simple_decorator.py new file mode 100644 index 0000000..5daf24d --- /dev/null +++ b/examples/mcp_server_fetch/tests/test_simple_decorator.py @@ -0,0 +1,18 @@ +from mcp_eval import task, setup, Expect +from mcp_eval.session import TestAgent, TestSession + + +@setup +def configure_decorator_tests(): + pass + + +@task("basic_website_fetch") +async def basic_website_fetch(agent: TestAgent, session: TestSession): + await agent.generate_str( + "Please fetch the content from https://example.com and tell me what you find" + ) + await session.assert_that(Expect.tools.was_called("fetch", min_times=1)) + await session.assert_that( + Expect.tools.called_with("fetch", {"url": "https://example.com"}) + ) diff --git a/scripts/generate_badges.py b/scripts/generate_badges.py index 0caa4fd..d5a3242 100644 --- a/scripts/generate_badges.py +++ b/scripts/generate_badges.py @@ -114,7 +114,7 @@ def make_badge(label: str, value: str, color: str) -> str: right_w = _measure_text(value) total_w = left_w + right_w # Construct an SVG similar to shields style - svg = f''' + svg = f""" @@ -133,7 +133,7 @@ def make_badge(label: str, value: str, color: str) -> str: {value} {value} -''' +""" return svg @@ -142,6 +142,13 @@ def write_text(path: Path, text: str) -> None: path.write_text(text, encoding="utf-8") +def write_endpoint_json(path: Path, label: str, message: str, color: str) -> None: + """Write a Shields.io endpoint JSON file.""" + data = {"schemaVersion": 1, "label": label, "message": message, "color": color} + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, indent=2), encoding="utf-8") + + def cli( report: str = typer.Option( ..., @@ -151,7 +158,7 @@ def cli( outdir: str = typer.Option( "mcpeval-reports/badges", "--outdir", - help="Output directory for generated SVG badges", + help="Output directory for generated badges", ), label_tests: str = typer.Option( "mcp-tests", "--label-tests", help="Label for tests badge" @@ -159,16 +166,39 @@ def cli( label_cov: str = typer.Option( "mcp-cov", "--label-cov", help="Label for coverage badge" ), + format: str = typer.Option( + "both", + "--format", + help="Output format: svg, endpoint, or both (default: both)", + ), ): report_path = Path(report) outdir_path = Path(outdir) + # Validate format option + if format not in ["svg", "endpoint", "both"]: + typer.echo(f"Invalid format: {format}. Must be 'svg', 'endpoint', or 'both'") + raise typer.Exit(1) + try: report_obj = load_report(report_path) except Exception: outdir_path.mkdir(parents=True, exist_ok=True) - write_text(outdir_path / "tests.svg", make_badge(label_tests, "0/0", "#9f9f9f")) - write_text(outdir_path / "coverage.svg", make_badge(label_cov, "0%", "#9f9f9f")) + # Generate fallback badges for errors + if format in ["svg", "both"]: + write_text( + outdir_path / "tests.svg", make_badge(label_tests, "0/0", "#9f9f9f") + ) + write_text( + outdir_path / "coverage.svg", make_badge(label_cov, "0%", "#9f9f9f") + ) + if format in ["endpoint", "both"]: + write_endpoint_json( + outdir_path / "mcp-tests.json", label_tests, "0/0", "#9f9f9f" + ) + write_endpoint_json( + outdir_path / "mcp-cov.json", label_cov, "0%", "#9f9f9f" + ) return passed, total, rate = compute_pass_fail(report_obj) @@ -179,12 +209,23 @@ def cli( cov_value = f"{int(round(cov_pct))}%" cov_color = _color_for_percentage(cov_pct) - write_text( - outdir_path / "tests.svg", make_badge(label_tests, tests_value, tests_color) - ) - write_text( - outdir_path / "coverage.svg", make_badge(label_cov, cov_value, cov_color) - ) + # Generate SVG badges + if format in ["svg", "both"]: + write_text( + outdir_path / "tests.svg", make_badge(label_tests, tests_value, tests_color) + ) + write_text( + outdir_path / "coverage.svg", make_badge(label_cov, cov_value, cov_color) + ) + + # Generate Shields endpoint JSON files + if format in ["endpoint", "both"]: + write_endpoint_json( + outdir_path / "mcp-tests.json", label_tests, tests_value, tests_color + ) + write_endpoint_json( + outdir_path / "mcp-cov.json", label_cov, cov_value, cov_color + ) if __name__ == "__main__":