diff --git a/.github/workflows/guide-draft.yml b/.github/workflows/guide-draft.yml index 011db35..ea380c3 100644 --- a/.github/workflows/guide-draft.yml +++ b/.github/workflows/guide-draft.yml @@ -113,6 +113,17 @@ jobs: echo "CURSOR_API_KEY secret is not set" > "${RUNNER_TEMP}/failure_reason.txt" exit 1 fi + # Fold issue comments into distill context so clarifications on the + # ticket (not only the body) reach --notes on retry. + comments=$(gh api "repos/${GH_REPO}/issues/${ISSUE_NUMBER}/comments" \ + --jq '[.[].body] | join("\n\n---\n\n")' \ + 2>/dev/null || echo "") + if [ -n "${comments}" ]; then + export ISSUE_BODY="${ISSUE_BODY:-} + +## Issue thread (for clarifications) +${comments}" + fi set +e npm run resolve-issue -- --output "${RUNNER_TEMP}/resolved.json" code=$? @@ -149,7 +160,7 @@ jobs: echo "**Candidates:** ${candidates}" fi echo - echo "Edit the issue (or reply clarifying which MCP server), then re-add \`guide:draft\`." + echo "Reply on this issue (or edit the body) clarifying which MCP server, then re-add \`guide:draft\`." } > "${RUNNER_TEMP}/failure_reason.txt" exit 1 fi @@ -214,10 +225,30 @@ jobs: npm run draft-guide -- "${SLUG}" "${args[@]}" code=$? set -e - if [ "$code" -ne 0 ]; then - echo "draft-guide exited ${code} (see workflow logs). Converged drafts still open a PR; unconverged/blocked/failed fail the job." > "${RUNNER_TEMP}/failure_reason.txt" - exit "$code" + # Prefer the newest run record for this slug (written even on unconverged). + record="" + if compgen -G "${GITHUB_WORKSPACE}/retro/runs/*-${SLUG}.json" > /dev/null; then + record=$(ls -t "${GITHUB_WORKSPACE}"/retro/runs/*-"${SLUG}".json | head -n1) + echo "record=${record}" >> "$GITHUB_OUTPUT" + cp "$record" "${RUNNER_TEMP}/run-record.json" + fi + # CLI: 0 = converged, 2 = unconverged/blocked/failed guide status, + # 1 = hard failure. Exit 2 still wrote files — open a PR for humans. + if [ "$code" -eq 0 ]; then + echo "outcome=converged" >> "$GITHUB_OUTPUT" + exit 0 fi + if [ "$code" -eq 2 ]; then + if [ -d "${GITHUB_WORKSPACE}/guides/${SLUG}" ]; then + echo "outcome=unconverged" >> "$GITHUB_OUTPUT" + echo "draft-guide exited 2 (unconverged/blocked/failed). Opening a draft PR with whatever was written for human review." > "${RUNNER_TEMP}/failure_reason.txt" + exit 0 + fi + echo "draft-guide exited 2 and guides/${SLUG}/ is missing" > "${RUNNER_TEMP}/failure_reason.txt" + exit 1 + fi + echo "draft-guide exited ${code} (hard failure; see workflow logs)" > "${RUNNER_TEMP}/failure_reason.txt" + exit "$code" - name: Commit and push if: steps.preflight.outputs.refused != 'true' && success() @@ -245,9 +276,13 @@ jobs: BRANCH: ${{ steps.branch.outputs.name }} SLUG: ${{ steps.distill.outputs.slug }} PROVIDER: ${{ steps.distill.outputs.provider }} + OUTCOME: ${{ steps.draft.outputs.outcome }} run: | set -euo pipefail title="Draft guide: ${PROVIDER} (#${ISSUE_NUMBER})" + if [ "${OUTCOME}" = "unconverged" ]; then + title="Draft guide (unconverged): ${PROVIDER} (#${ISSUE_NUMBER})" + fi title="${title:0:256}" body_file="${RUNNER_TEMP}/pr-body.md" { @@ -255,22 +290,57 @@ jobs: echo echo "Factory draft of \`guides/${SLUG}/\` via \`draft-guide\` (Cursor SDK)." echo + if [ "${OUTCOME}" = "unconverged" ]; then + echo "**Pipeline status:** unconverged (reviewers still had blockers after max rounds)." + echo "See the issue comment **Pipeline review** for unresolved blockers and open questions." + echo "Do not merge until those are settled." + echo + fi echo "Human review still required — agents never commit; this Action did." } > "$body_file" + # Append a short blocker summary into the PR body when present. + if [ -f "${RUNNER_TEMP}/run-record.json" ]; then + echo >> "$body_file" + bash "${GITHUB_WORKSPACE}/scripts/ci/format-pipeline-review.sh" \ + "${RUNNER_TEMP}/run-record.json" "" \ + "${GITHUB_WORKSPACE}/guides/${SLUG}" >> "$body_file" + fi pr_url=$(gh pr create --draft --base main --head "$BRANCH" --title "$title" --body-file "$body_file" | tail -n1) echo "pr_url=$pr_url" >> "$GITHUB_OUTPUT" - - name: Comment PR URL + - name: Comment pipeline review on issue if: steps.preflight.outputs.refused != 'true' && success() env: PR_URL: ${{ steps.open_pr.outputs.pr_url }} + OUTCOME: ${{ steps.draft.outputs.outcome }} + SLUG: ${{ steps.distill.outputs.slug }} run: | - gh issue comment "$ISSUE_NUMBER" --body "Draft PR opened: ${PR_URL}" + set -euo pipefail + body_file="${RUNNER_TEMP}/pipeline-review-comment.md" + if [ -f "${RUNNER_TEMP}/run-record.json" ]; then + bash "${GITHUB_WORKSPACE}/scripts/ci/format-pipeline-review.sh" \ + "${RUNNER_TEMP}/run-record.json" "${PR_URL}" \ + "${GITHUB_WORKSPACE}/guides/${SLUG}" > "$body_file" + else + { + echo "## Pipeline review" + echo + if [ "${OUTCOME}" = "unconverged" ]; then + echo "Draft PR opened (pipeline **unconverged**): ${PR_URL}" + else + echo "Draft PR opened: ${PR_URL}" + fi + echo + echo "_No run record found to summarize blockers / open questions._" + } > "$body_file" + fi + gh issue comment "$ISSUE_NUMBER" --body-file "$body_file" - name: Mark blocked on failure if: steps.preflight.outputs.refused != 'true' && failure() env: RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + SLUG: ${{ steps.distill.outputs.slug }} run: | reason="(no reason file written; check workflow logs)" if [ -f "${RUNNER_TEMP}/failure_reason.txt" ]; then @@ -278,14 +348,29 @@ jobs: fi body_file="${RUNNER_TEMP}/failure-comment.md" { - echo "\`guide:draft\` run failed." + echo "\`guide:draft\` run failed (no PR opened)." echo echo "$reason" echo echo "**Workflow run:** ${RUN_URL}" echo - echo "Re-add \`guide:draft\` to retry after clarifying or fixing." } > "$body_file" + # If a run record exists from a partial draft, surface blockers here too. + if [ -f "${RUNNER_TEMP}/run-record.json" ]; then + echo >> "$body_file" + bash "${GITHUB_WORKSPACE}/scripts/ci/format-pipeline-review.sh" \ + "${RUNNER_TEMP}/run-record.json" "" \ + "${GITHUB_WORKSPACE}/guides/${SLUG}" >> "$body_file" || true + elif [ -n "${SLUG}" ] && compgen -G "${GITHUB_WORKSPACE}/retro/runs/*-${SLUG}.json" > /dev/null; then + record=$(ls -t "${GITHUB_WORKSPACE}"/retro/runs/*-"${SLUG}".json | head -n1) + echo >> "$body_file" + bash "${GITHUB_WORKSPACE}/scripts/ci/format-pipeline-review.sh" \ + "$record" "" "${GITHUB_WORKSPACE}/guides/${SLUG}" >> "$body_file" || true + else + { + echo "Reply on this issue with clarifications, then re-add \`guide:draft\`." + } >> "$body_file" + fi gh issue edit "$ISSUE_NUMBER" --add-label "guide:blocked" || true gh issue comment "$ISSUE_NUMBER" --body-file "$body_file" diff --git a/docs/agents/guide-factory.md b/docs/agents/guide-factory.md index 3fadfde..0b63f6a 100644 --- a/docs/agents/guide-factory.md +++ b/docs/agents/guide-factory.md @@ -7,6 +7,7 @@ commits, pushes, and opens the PR. Workflow: [`.github/workflows/guide-draft.yml`](../../.github/workflows/guide-draft.yml). Distill CLI: `npm run resolve-issue` in `scripts/cursor-sdk/`. +Review comment formatter: [`scripts/ci/format-pipeline-review.sh`](../../scripts/ci/format-pipeline-review.sh). ## How to file an issue @@ -16,13 +17,29 @@ Distill CLI: `npm run resolve-issue` in `scripts/cursor-sdk/`. - `Draft a BigQuery MCP setup guide` - `We need HubSpot — prefer OAuth docs at https://…` 2. Add the label `guide:draft`. -3. Wait for the Action. On success you get a draft PR (`Closes #`). - On clarification or failure you get `guide:blocked` and a comment; edit the - issue and re-add `guide:draft` to retry. +3. Wait for the Action. You get: + - a **Resolved as …** comment (distill intent), + - a **Pipeline review** comment (unresolved blockers, open questions, nits), + - a draft PR (`Closes #`) when files were written — including + **unconverged** runs (PR title/body say so). Persona defaults to `it-admin` unless the distill step confidently finds a known id under `docs/personas/`. +## Where to put clarifications + +When the **Pipeline review** comment asks for a call (exact UI labels, whether +to drop a recovery branch, etc.): + +1. **Reply on the issue** with the answers (preferred — easy to skim in the + thread), and/or +2. **Edit the issue body** with the same facts. + +Then re-add `guide:draft`. Distill re-reads the **body and the full comment +thread** into `--notes` for the next run. You do **not** need to paste the +whole guide into the ticket — answer the open questions / blockers listed in +the review comment. + ## Labels The workflow **creates these labels automatically** if missing. You can still @@ -32,7 +49,7 @@ create them by hand for triage before the first run: | --- | --- | | `guide:draft` | Trigger — removed as soon as the job accepts the work | | `guide:in-progress` | Set while distill + pipeline run; always cleared in `always()` | -| `guide:blocked` | Set on preflight refusal, distill clarification, or pipeline failure; cleared when a new successful accept starts | +| `guide:blocked` | Set on preflight refusal, distill clarification, or hard failure; cleared when a new successful accept starts | Suggested colors (optional): draft = blue, in-progress = yellow, blocked = red. @@ -51,18 +68,22 @@ pushing with full permissions matters. 1. **Preflight** — refuse if an open collaborator PR already `Closes #N`. 2. **Labels** — remove `guide:draft` + `guide:blocked`, add `guide:in-progress`. -3. **Distill** — light Cursor agent (`composer-2.5` / `CURSOR_MODEL_LIGHT`) - reads title+body (+ existing `guides/*` slugs) and writes structured JSON - (`slug`, `provider`, `persona`, `notes`) or `needs_clarification`. -4. **Comment** — on `ok`, comment a short “Resolved as `slug` …” summary. -5. **Draft** — `npm run draft-guide -- --overwrite [--notes …] [--persona …]`. +3. **Distill** — light Cursor agent reads title + body + issue comments (+ + existing `guides/*` slugs) → structured JSON or `needs_clarification`. +4. **Comment** — “Resolved as `slug` …” summary. +5. **Draft** — `npm run draft-guide -- --overwrite [--notes …]`. 6. **PR** — commit `guides//` + matching `retro/runs/*-.json`, - push `guide/issue--`, open a **draft** PR. -7. **Failure** — `guide:blocked` + comment with reason and Actions run URL. -8. **Always** — remove `guide:in-progress`. - -CLI exit `2` (unconverged / blocked / failed) fails the job so the blocked -path runs. Only converged drafts open a PR for human review. + push `guide/issue--`, open a **draft** PR (also on unconverged + when files exist). +7. **Comment** — **Pipeline review** on the issue (blockers / open questions / + nits + PR link). Same summary is appended to the PR body. +8. **Hard failure** — `guide:blocked` + comment (includes review summary when + a run record exists). +9. **Always** — remove `guide:in-progress`. + +CLI exit `0` (converged) and exit `2` (unconverged / blocked / failed guide +status with files on disk) both open a draft PR. Hard failures (exit `1`, +missing `guides//`) take the blocked path with no PR. ## What v1 does not do diff --git a/scripts/ci/format-pipeline-review.sh b/scripts/ci/format-pipeline-review.sh new file mode 100755 index 0000000..566cbc4 --- /dev/null +++ b/scripts/ci/format-pipeline-review.sh @@ -0,0 +1,184 @@ +#!/usr/bin/env bash +# Format a draft-guide run record as a human-readable GitHub issue comment. +# Usage: format-pipeline-review.sh [pr-url] [guides/ dir] +set -euo pipefail + +record="${1:?run record json path required}" +pr_url="${2:-}" +guide_dir="${3:-}" + +if [ ! -f "$record" ]; then + echo "No run record at $record" >&2 + exit 1 +fi + +status=$(jq -r '.status // "unknown"' "$record") +rounds=$(jq -r '.rounds // "?"' "$record") +slug=$(jq -r '.slug // "?"' "$record") + +# Prefer guide dir from arg, else infer from slug next to typical checkout layout. +if [ -z "$guide_dir" ] && [ -n "$slug" ] && [ -d "guides/${slug}" ]; then + guide_dir="guides/${slug}" +fi +setup_md="" +if [ -n "$guide_dir" ] && [ -f "${guide_dir}/setup.md" ]; then + setup_md="${guide_dir}/setup.md" +fi + +plain_dimension() { + case "$1" in + fidelity) echo "Fact check failed — setup and research disagree (or research is missing the fact)." ;; + achievability) echo "A cold reader would get stuck — a click, field, or next step is not named clearly enough." ;; + voice) echo "Tone / persona mismatch." ;; + formatting) echo "Guide structure / formatting rule broken." ;; + concision) echo "Extra prose the reader does not need." ;; + *) echo "$1" ;; + esac +} + +plain_target() { + case "$1" in + research) echo "Needs a fact in \`research.md\` (or drop the step that depends on it)." ;; + setup) echo "Needs a clearer step in \`setup.md\` (the fact may already be in research)." ;; + meta) echo "Needs a fix in \`meta.yaml\`." ;; + *) echo "Target: \`$1\`" ;; + esac +} + +# Extract the H3 section whose closing anchor matches #foo from setup.md. +quote_section() { + local md="$1" anchor="$2" + [ -f "$md" ] || return 0 + [ -n "$anchor" ] || return 0 + # anchor like #copy-client-credentials + local id="${anchor#\#}" + python3 - "$md" "$id" <<'PY' 2>/dev/null || true +import sys +from pathlib import Path +md = Path(sys.argv[1]).read_text(encoding="utf-8") +needle = "{#" + sys.argv[2] + "}" +lines = md.splitlines() +start = None +for i, line in enumerate(lines): + if needle in line and line.lstrip().startswith("#"): + start = i + break +if start is None: + sys.exit(0) +# collect until next H2/H3 +out = [lines[start]] +for line in lines[start + 1 :]: + if line.startswith("## ") or (line.startswith("### ") and "{#" in line): + break + out.append(line) +# trim trailing blanks; cap length +while out and not out[-1].strip(): + out.pop() +text = "\n".join(out).strip() +if len(text) > 900: + text = text[:900].rstrip() + "\n…" +print(text) +PY +} + +extract_anchor() { + # Prefer first #kebab-case token in where + echo "$1" | grep -oE '#[a-z0-9-]+' | head -n1 || true +} + +echo "## Pipeline review (\`${slug}\`)" +echo +case "$status" in + converged) + echo "**Outcome:** Reviewers passed after ${rounds} round(s). Still skim the open questions below before merging." + ;; + unconverged) + echo "**Outcome:** Did **not** fully converge after ${rounds} review round(s). The draft may still be useful — decide on each item below, then reply and re-run." + ;; + *) + echo "**Outcome:** \`${status}\` after ${rounds} review round(s)." + ;; +esac +if [ -n "$pr_url" ]; then + echo + echo "**Draft PR:** ${pr_url}" +fi +echo + +unresolved_n=$(jq '(.unresolved // []) | length' "$record") +if [ "$unresolved_n" -gt 0 ]; then + echo "### Decisions needed (${unresolved_n})" + echo + i=0 + while IFS= read -r row; do + i=$((i + 1)) + dim=$(echo "$row" | jq -r '.dimension // "?"') + target=$(echo "$row" | jq -r '.target // "?"') + where=$(echo "$row" | jq -r '.where // "?"') + problem=$(echo "$row" | jq -r '.problem // ""') + suggestion=$(echo "$row" | jq -r '.suggestion // ""') + anchor=$(extract_anchor "$where") + + echo "#### ${i}. $(plain_dimension "$dim")" + echo + echo "- **Where in the guide:** \`${where}\`" + if [ -n "$anchor" ]; then + echo "- **Section anchor:** \`${anchor}\`" + fi + echo "- **What's wrong:** ${problem}" + echo "- **What would unblock it:** ${suggestion}" + echo "- **$(plain_target "$target")**" + echo + if [ -n "$setup_md" ] && [ -n "$anchor" ]; then + quote=$(quote_section "$setup_md" "$anchor" || true) + if [ -n "${quote}" ]; then + echo "
Current guide text for this section" + echo + echo '```markdown' + echo "$quote" + echo '```' + echo + echo "
" + echo + fi + fi + echo "**Reply with one of:**" + echo "- \`Decision ${i}: verified — …\` (paste the exact button / field / nav labels)" + echo "- \`Decision ${i}: drop this branch\` (remove the recovery/optional path until we can verify it)" + echo "- \`Decision ${i}: hedge — …\` (keep a softer “if you see X, ask your admin” line instead of exact clicks)" + echo + done < <(jq -c '(.unresolved // [])[]' "$record") +fi + +oq_n=$(jq '(.open_questions // []) | length' "$record") +if [ "$oq_n" -gt 0 ]; then + echo "### Open questions (${oq_n})" + echo + echo "Research could not prove these from public docs. Check the boxes by replying with answers, or say “unknown / omit”." + echo + jq -r '(.open_questions // [])[] | "- [ ] \(.)"' "$record" + echo +fi + +nits_n=$(jq '(.nits // []) | length' "$record") +if [ "$nits_n" -gt 0 ] && [ "$nits_n" -le 8 ]; then + echo "### Polish nits (${nits_n}) — optional" + echo + jq -r ' + (.nits // [])[] | + "- `\(.where // "?")`: \(.problem // "") → \(.suggestion // "—")" + ' "$record" + echo +elif [ "$nits_n" -gt 8 ]; then + echo "### Polish nits" + echo + echo "_${nits_n} optional nits — see the run record in the PR if you care._" + echo +fi + +echo "### How to retry" +echo +echo "1. Reply on **this issue** using the \`Decision N: …\` lines above (and answer open questions)." +echo "2. Re-add the \`guide:draft\` label. Distill reads the issue body **and** comments into pipeline notes." +echo +echo "_Source: \`$(basename "$record")\`_"