diff --git a/factory/scripts/container-entrypoint.sh b/factory/scripts/container-entrypoint.sh index bf108e6..49b834e 100755 --- a/factory/scripts/container-entrypoint.sh +++ b/factory/scripts/container-entrypoint.sh @@ -13,18 +13,48 @@ test -r "$INPUT_ROOT/catalog.json" test -r "$REPO_ROOT/factory/coordinator.md" rm -rf "$WORKSPACE_ROOT" mkdir -p "$WORKSPACE_ROOT/.factory" "$KIT_HOME" "$EXPORT_ROOT" -rm -rf "$EXPORT_ROOT/guide" "$EXPORT_ROOT/run-report.json" +rm -rf "$EXPORT_ROOT/guide" "$EXPORT_ROOT/run-report.json" "$EXPORT_ROOT/kit-error-summary.json" cp -a "$REPO_ROOT/." "$WORKSPACE_ROOT/" rm -rf "$WORKSPACE_ROOT/.git" export HOME="$KIT_HOME" KIT_BIN=${KIT_BIN:-kit} -"$KIT_BIN" prompt \ +if ! "$KIT_BIN" prompt \ --root "$WORKSPACE_ROOT" \ --provider openrouter \ --model "$KIT_MODEL" \ --reasoning-effort "$KIT_REASONING_EFFORT" \ --mcp-config "$WORKSPACE_ROOT/factory/mcp/exa.json" \ - "$(cat "$WORKSPACE_ROOT/factory/coordinator.md")" + "$(cat "$WORKSPACE_ROOT/factory/coordinator.md")"; then + error_files=() + while IFS= read -r error_file; do + error_files+=("$error_file") + done < <(find "$KIT_HOME/errors" -type f -name '*.json' -print 2>/dev/null || true) + if (( ${#error_files[@]} == 0 )); then + jq -n '{schema_version:1,records:[]}' >"$EXPORT_ROOT/kit-error-summary.json" + else + jq -s ' + {schema_version:1,records:[.[] | { + schema_version,kind,code, + diagnostics:(if (.diagnostics | type) == "object" then { + stage:.diagnostics.stage, + retryable:.diagnostics.retryable, + attempt:.diagnostics.attempt, + response_request_id:(.diagnostics.response_request_id // null), + reqwest:{ + timeout:.diagnostics.reqwest.timeout, + connect:.diagnostics.reqwest.connect, + request:.diagnostics.reqwest.request, + body:.diagnostics.reqwest.body, + decode:.diagnostics.reqwest.decode + }, + source_chain_unknown:.diagnostics.source_chain_unknown, + source_chain_truncated:.diagnostics.source_chain_truncated + } else null end) + }]} + ' "${error_files[@]}" >"$EXPORT_ROOT/kit-error-summary.json" + fi + exit 1 +fi test -s "$WORKSPACE_ROOT/.factory/run-report.json" "$REPORT_VALIDATOR" "$WORKSPACE_ROOT/.factory/run-report.json" outcome="$(jq -r '.outcome' "$WORKSPACE_ROOT/.factory/run-report.json")" diff --git a/factory/scripts/run-kit.sh b/factory/scripts/run-kit.sh index efc69b8..c51d265 100755 --- a/factory/scripts/run-kit.sh +++ b/factory/scripts/run-kit.sh @@ -21,7 +21,7 @@ issue_json="$(realpath "$issue_json")" catalog_json="$(realpath "$catalog_json")" mkdir -p "$export_dir" export_dir="$(realpath "$export_dir")" -rm -rf "$export_dir/guide" "$export_dir/run-report.json" +rm -rf "$export_dir/guide" "$export_dir/run-report.json" "$export_dir/kit-error-summary.json" source_snapshot="$(mktemp -d "${TMPDIR:-/tmp}/mcp-setup-docs-source.XXXXXX")" source_snapshot="$(realpath "$source_snapshot")" @@ -43,7 +43,7 @@ tar -cf - --exclude-from="$ROOT/.dockerignore" -C "$ROOT" . \ --tag "$KIT_IMAGE" \ "$ROOT" -"$FACTORY_DOCKER" run --rm \ +if ! "$FACTORY_DOCKER" run --rm \ --env OPENROUTER_API_KEY \ --env "KIT_MODEL=$KIT_MODEL" \ --env "KIT_REASONING_EFFORT=$KIT_REASONING_EFFORT" \ @@ -51,4 +51,34 @@ tar -cf - --exclude-from="$ROOT/.dockerignore" -C "$ROOT" . \ --volume "$issue_json:/input/issue.json:ro" \ --volume "$catalog_json:/input/catalog.json:ro" \ --volume "$export_dir:/export" \ - "$KIT_IMAGE" + "$KIT_IMAGE"; then + if [[ -f "$export_dir/kit-error-summary.json" ]] && jq -e ' + (keys | sort) == (["records","schema_version"] | sort) + and .schema_version == 1 + and (.records | type) == "array" + and all(.records[]; + (keys | sort) == (["code","diagnostics","kind","schema_version"] | sort) + and .schema_version == 2 + and (.kind | type) == "string" and (.kind | test("^[a-z0-9_]{1,64}$")) + and (.code | type) == "string" and (.code | test("^[a-z0-9_]{1,64}$")) + and (.diagnostics == null or ( + (.diagnostics | type) == "object" + and (.diagnostics | keys | sort) == (["attempt","reqwest","response_request_id","retryable","source_chain_truncated","source_chain_unknown","stage"] | sort) + and (.diagnostics.stage | IN("request","stream")) + and (.diagnostics.retryable | type) == "boolean" + and (.diagnostics.attempt | type) == "number" and (.diagnostics.attempt | floor) == .diagnostics.attempt + and (.diagnostics.attempt >= 1 and .diagnostics.attempt <= 1000) + and (.diagnostics.response_request_id == null or ( + (.diagnostics.response_request_id | type) == "string" + and (.diagnostics.response_request_id | test("^[A-Za-z0-9_.:-]{1,128}$")))) + and (.diagnostics.reqwest | keys | sort) == (["body","connect","decode","request","timeout"] | sort) + and all(.diagnostics.reqwest[]; type == "boolean") + and (.diagnostics.source_chain_unknown | type) == "boolean" + and (.diagnostics.source_chain_truncated | type) == "boolean" + ))) + ' "$export_dir/kit-error-summary.json" >/dev/null; then + printf 'factory: Kit failure diagnostic: ' >&2 + jq -c . "$export_dir/kit-error-summary.json" >&2 + fi + exit 1 +fi diff --git a/factory/tests/test-container.sh b/factory/tests/test-container.sh index 5c806a2..36f6a45 100755 --- a/factory/tests/test-container.sh +++ b/factory/tests/test-container.sh @@ -105,6 +105,39 @@ test_run_kit_does_not_forward_github_credentials() { ! grep -qE 'GH_TOKEN|SSH_AUTH_SOCK' "$TMP/docker.args" } +test_run_kit_reports_safe_failure_diagnostics() { + export OPENROUTER_API_KEY=or-test FACTORY_DOCKER="$TMP/bin/docker" + # shellcheck disable=SC2016 + make_fake docker '[[ "${1:-}" == build ]] && exit 0; mkdir -p "$TMP/error-run-export"; printf '"'"'{"schema_version":1,"records":[{"schema_version":2,"kind":"provider","code":"provider_error","diagnostics":null}]}\n'"'"' >"$TMP/error-run-export/kit-error-summary.json"; exit 1' + printf '{}\n' >"$TMP/issue.json"; printf '{}\n' >"$TMP/catalog.json" + if "$ROOT/factory/scripts/run-kit.sh" "$TMP/issue.json" "$TMP/catalog.json" "$TMP/error-run-export" \ + >"$TMP/out" 2>"$TMP/err"; then + fail 'run-kit accepted failed container' + fi + assert_contains '"code":"provider_error"' "$(cat "$TMP/err")" + + # shellcheck disable=SC2016 + make_fake docker '[[ "${1:-}" == build ]] && exit 0; mkdir -p "$TMP/error-run-export"; printf '"'"'{"schema_version":1,"records":[{"schema_version":2,"kind":"provider","code":"provider_error","diagnostics":{"stage":"request","retryable":false,"attempt":1,"response_request_id":null,"reqwest":{"timeout":false,"connect":false,"request":false,"body":false,"decode":false},"source_chain_unknown":false,"source_chain_truncated":false,"provider_body":"sensitive-nested-diagnostic"}}]}\n'"'"' >"$TMP/error-run-export/kit-error-summary.json"; exit 1' + if "$ROOT/factory/scripts/run-kit.sh" "$TMP/issue.json" "$TMP/catalog.json" "$TMP/error-run-export" \ + >"$TMP/out" 2>"$TMP/err"; then + fail 'run-kit accepted malicious diagnostics' + fi + if grep -q 'sensitive-nested-diagnostic' "$TMP/err"; then + fail 'run-kit logged unvalidated nested diagnostics' + fi + + printf '%s\n' 'sensitive-stale-diagnostic' >"$TMP/error-run-export/kit-error-summary.json" + make_fake docker 'exit 1' + if "$ROOT/factory/scripts/run-kit.sh" "$TMP/issue.json" "$TMP/catalog.json" "$TMP/error-run-export" \ + >"$TMP/out" 2>"$TMP/err"; then + fail 'run-kit accepted failed build' + fi + test ! -e "$TMP/error-run-export/kit-error-summary.json" + if grep -q 'sensitive-stale-diagnostic' "$TMP/err"; then + fail 'run-kit logged stale diagnostics' + fi +} + test_run_kit_uses_only_allowed_mounts() { export OPENROUTER_API_KEY=or-test export FACTORY_DOCKER="$TMP/bin/docker" @@ -281,6 +314,46 @@ MOCK done } +test_entrypoint_exports_safe_kit_failure_diagnostics() { + local repo input workspace export_root fake_kit + repo="$TMP/error-repo"; input="$TMP/error-input" + workspace="$TMP/error-workspace"; export_root="$TMP/error-export" + fake_kit="$TMP/bin/error-kit" + mkdir -p "$repo/factory" "$input" "$TMP/bin" + printf 'assignment\n' >"$repo/factory/coordinator.md" + printf '{}\n' >"$input/issue.json"; printf '{}\n' >"$input/catalog.json" + cat >"$fake_kit" <<'MOCK' +#!/usr/bin/env bash +mkdir -p "$HOME/errors/s-test" +cat >"$HOME/errors/s-test/e-test.json" <<'JSON' +{"schema_version":2,"event_id":"e-test","occurred_at_ms":1,"kit_version":"0.1.98","session_id":"s-test","surface":"prompt","kind":"provider","code":"provider_error","message":"sensitive-provider-body","prompt":{"code":"sensitive-code","provider":"sensitive-provider"},"url":"https://sensitive.example","diagnostics":null} +JSON +cat >"$HOME/errors/s-test/e-transport.json" <<'JSON' +{"schema_version":2,"event_id":"e-transport","occurred_at_ms":2,"kit_version":"0.1.98","session_id":"s-test","surface":"prompt","kind":"provider","code":"request_transport","message":"sensitive-transport-message","diagnostics":{"stage":"request","retryable":true,"attempt":2,"response_request_id":"req_safe-123","reqwest":{"timeout":false,"connect":true,"request":true,"body":false,"decode":false},"source_chain":[{"code":"sensitive-source-code","provider":"sensitive-source-provider"}],"source_chain_unknown":true,"source_chain_truncated":false}} +JSON +exit 1 +MOCK + chmod +x "$fake_kit" + if FACTORY_REPO_ROOT="$repo" FACTORY_INPUT_ROOT="$input" \ + FACTORY_WORKSPACE_ROOT="$workspace" FACTORY_EXPORT_ROOT="$export_root" \ + FACTORY_KIT_HOME="$TMP/error-home" KIT_BIN="$fake_kit" \ + FACTORY_REPORT_VALIDATOR="$ROOT/factory/scripts/validate-report.sh" \ + KIT_MODEL=openai/gpt-5.6-sol KIT_REASONING_EFFORT=high \ + "$ROOT/factory/scripts/container-entrypoint.sh" >/dev/null 2>&1; then + fail 'entrypoint accepted failed Kit process' + fi + jq -e ' + (.records | length) == 2 + and any(.records[]; .kind == "provider" and .code == "provider_error" and .diagnostics == null) + and any(.records[]; + .code == "request_transport" + and .diagnostics == {stage:"request",retryable:true,attempt:2,response_request_id:"req_safe-123",reqwest:{timeout:false,connect:true,request:true,body:false,decode:false},source_chain_unknown:true,source_chain_truncated:false}) + ' "$export_root/kit-error-summary.json" >/dev/null + if grep -Eq 'sensitive-provider-body|sensitive-code|sensitive-provider|sensitive-transport-message|sensitive-source|sensitive\.example' "$export_root/kit-error-summary.json"; then + fail 'Kit failure diagnostics leaked unsafe fields' + fi +} + test_local_draft_parsing_and_secret_boundary() { local bin log tmpdir issue_path bin="$TMP/local-bin"; log="$TMP/local.log"; tmpdir="$TMP/local tmp" @@ -380,10 +453,12 @@ test_dockerfile_builds_static_linter_without_go_in_final_image test_docker_context_excludes_credentials_and_keeps_build_inputs test_release_archive_layout_and_checksum test_run_kit_does_not_forward_github_credentials +test_run_kit_reports_safe_failure_diagnostics test_run_kit_uses_only_allowed_mounts test_run_kit_source_snapshot_applies_dockerignore test_entrypoint_exports_only_selected_guide_with_mocked_kit test_entrypoint_rejects_invalid_report +test_entrypoint_exports_safe_kit_failure_diagnostics test_local_draft_parsing_and_secret_boundary test_local_draft_rejects_invalid_arguments_and_slug_mismatch test_opt_in_final_image