From a0b7addd737215df4b5de31efc35637c34918822 Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Fri, 7 Aug 2026 20:07:47 +0200 Subject: [PATCH 01/37] go: update Orchestrion integration to v1.12.0 --- .bazelignore | 1 + CHANGELOG.md | 4 + MODULE.bazel.lock | 2 +- README.md | 26 +- docs/Configuration_Reference.md | 5 +- docs/Installation_Reference.md | 10 +- docs/Language_Onboarding.md | 8 +- docs/Troubleshooting.md | 2 +- docs/go_orchestrion_bazel_deep_dive.md | 2 +- docs/internal_monorepo_go_rollout_guide.md | 2 +- examples/README.md | 8 +- examples/single_service/src/go-project/go.mod | 130 ++++--- examples/single_service/src/go-project/go.sum | 352 ++++++++---------- modules/go/tests/test_workspace_helpers.bzl | 10 +- modules/go/tools/dd_topt_go_bootstrap/main.go | 4 +- .../tools/dd_topt_go_bootstrap/main_test.go | 136 +++---- modules/go/tools/onboardingpins/pins.go | 4 +- modules/go/tools/onboardingpins/pins_test.go | 4 +- .../go/private/orchestrion/extensions.bzl | 37 +- .../go/tools/builders/orchestrion_version.go | 2 +- .../go/private/orchestrion/extensions.bzl | 37 +- .../go/tools/builders/orchestrion_version.go | 2 +- .../go/private/orchestrion/extensions.bzl | 37 +- .../go/tools/builders/orchestrion_version.go | 2 +- .../v0_60_0/base/0001-full-delta.patch | 45 +-- .../v0_61_1/base/0001-full-delta.patch | 45 +-- .../v0_62_0/base/0001-full-delta.patch | 45 +-- .../references/bzlmod-onboarding.md | 3 +- .../references/workspace-onboarding.md | 5 +- tools/dev/verify_rules_go_profiles.py | 8 +- .../fixtures/orchestrion_pin_graph/go.mod | 4 +- .../fixtures/orchestrion_pin_graph/go.sum | 8 +- .../missing_module/go.mod | 4 +- .../missing_module/go.sum | 8 +- .../integration/run_bzlmod_go_integration.sh | 6 +- .../integration/run_mock_server_tests.sh | 64 ++-- .../run_workspace_go_integration.sh | 6 +- .../python/test_rules_go_profile_tools.py | 4 +- 38 files changed, 480 insertions(+), 602 deletions(-) diff --git a/.bazelignore b/.bazelignore index 7c8a658c..81c3c714 100644 --- a/.bazelignore +++ b/.bazelignore @@ -11,6 +11,7 @@ bazel-out bazel-testlogs third_party/rgo/v0_60_0/base third_party/rgo/v0_61_1/base +third_party/rgo/v0_62_0/base tools/tests/rules_go_variant_regressions modules/go/bazel-bin modules/go/bazel-out diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ef36051..124254fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,10 @@ versioning. Orchestrion integration and public consumer patch profile. ### Changed +- Updated the maintained `rules_go` Orchestrion integration to Orchestrion + `v1.12.0` and `dd-trace-go/v2` `v2.9.1`, preserving upstream test-variant + resolution while retaining the Bazel-specific resolver compatibility layer. + Orchestrion `v1.12.0` requires Go `1.25.0` or newer. - The public Go Bzlmod extension now defaults `enabled_by_env` to `True`, so omitting `--config=test-optimization` disables metadata sync and Orchestrion together while the named config enables both. diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index c14c2795..dac5f63a 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -766,7 +766,7 @@ }, "@@rules_go+//go:extensions.bzl%orchestrion": { "general": { - "bzlTransitiveDigest": "k6TuFeNdo4I01jk91s4xBDhniwJJf3syb0kGpA/q7QA=", + "bzlTransitiveDigest": "7qy3MxlPmbSSl3MDmdiHrZuoYkvewN+gUmuzc/uJvzo=", "usagesDigest": "9kKEG/hjK4fnk42l4jAKDLhx9xExyCYAS8r8NlbPLMw=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, diff --git a/README.md b/README.md index 9ddebbc1..a39bcb13 100644 --- a/README.md +++ b/README.md @@ -286,8 +286,8 @@ RTO_ARCHIVE_TYPE="tar.gz" RULES_GO_UPSTREAM="v0_60_0" RULES_GO_VARIANT="base" RULES_GO_STRIP_PREFIX="third_party/rgo/v0_60_0/base" -DD_TRACE_GO_VERSION="v2.9.0" -ORCHESTRION_VERSION="v1.9.0" +DD_TRACE_GO_VERSION="v2.9.1" +ORCHESTRION_VERSION="v1.12.0" ``` The archive URL, SHA256, and prefix are tied to the repository commit. @@ -431,14 +431,16 @@ bazel_dep(name = "rules_go", version = "0.60.0") Then run the Datadog bootstrap helper once from the workspace that owns your Go module. `--dd-trace-go-version` is optional; if you omit it, the default is -`v2.9.0`. +`v2.9.1`. Orchestrion `v1.12.0` requires Go `1.25.0` or newer, so the +`--runtime-version` value and registered Bazel Go toolchain must satisfy that +minimum. ```bash bazel run @datadog-rules-test-optimization-go//:dd_topt_go_bootstrap -- \ --guided \ --service go-service \ --runtime-version 1.25.0 \ - --dd-trace-go-version v2.9.0 \ + --dd-trace-go-version v2.9.1 \ --write-bazelrc ``` @@ -449,7 +451,7 @@ bazel run @datadog-rules-test-optimization-go//:dd_topt_go_bootstrap -- \ --guided \ --service go-service \ --runtime-version 1.25.0 \ - --dd-trace-go-version v2.9.0 \ + --dd-trace-go-version v2.9.1 \ --go-module-dir path/to/go-module \ --write-bazelrc ``` @@ -1768,7 +1770,7 @@ warmed host Go module cache. Test payloads still use the Bazel file-output contract: the tracer writes JSON files under `TEST_UNDECLARED_OUTPUTS_DIR`, and the uploader enriches those JSON files with repository and Bazel metadata. Pass `--dd-trace-go-version ` if you want a non-default tracer version; -otherwise the default is `v2.9.0`. +otherwise the default is `v2.9.1`. ```bash bazel run @datadog-rules-test-optimization-go//:dd_topt_go_bootstrap -- \ @@ -1808,7 +1810,7 @@ use_repo(test_optimization_go_sdk, "test_optimization_go_sdk") orchestrion = use_extension("@rules_go//go:extensions.bzl", "orchestrion") orchestrion.from_source( - version = "v1.9.0", + version = "v1.12.0", dd_trace_go_pin_files = [ "@//:go.mod", "@//:go.sum", @@ -1836,11 +1838,11 @@ use_repo(test_optimization_go_sdk, "test_optimization_go_sdk") orchestrion = use_extension("@rules_go//go:extensions.bzl", "orchestrion") orchestrion.from_source( - version = "v1.9.0", + version = "v1.12.0", dd_trace_go_versions = { - "github.com/DataDog/dd-trace-go/v2": "v2.9.0", - "github.com/DataDog/dd-trace-go/contrib/net/http/v2": "v2.9.0", - "github.com/DataDog/dd-trace-go/contrib/log/slog/v2": "v2.9.0", + "github.com/DataDog/dd-trace-go/v2": "v2.9.1", + "github.com/DataDog/dd-trace-go/contrib/net/http/v2": "v2.9.1", + "github.com/DataDog/dd-trace-go/contrib/log/slog/v2": "v2.9.1", }, go_sdk_root = "@test_optimization_go_sdk//:ROOT", go_sdk_version = "", @@ -1853,7 +1855,7 @@ with explicit Bazel flags in the script itself. There is no special repo-root `--config=hermetic` shortcut for this flow. If all three selection settings are omitted, the legacy default is still -`v2.9.0`. Manual setups must keep the local Go module pins on the same effective +`v2.9.1`. Manual setups must keep the local Go module pins on the same effective versions, or the build will stop with a mismatch error. Do not combine `dd_trace_go_pin_files`, `dd_trace_go_version`, or `dd_trace_go_versions` in the same `orchestrion.from_source(...)` call. diff --git a/docs/Configuration_Reference.md b/docs/Configuration_Reference.md index cf448fa5..95d4830f 100644 --- a/docs/Configuration_Reference.md +++ b/docs/Configuration_Reference.md @@ -16,7 +16,7 @@ Guided Go bootstrap accepts: | Flag | Default | Description | |------|---------|-------------| -| `--dd-trace-go-version` | `v2.9.0` | Go tracer query for bootstrap. Accepts a tag, pseudo-version, branch, or commit SHA and persists the exact resolved versions Bazel should use | +| `--dd-trace-go-version` | `v2.9.1` | Go tracer query for bootstrap. Accepts a tag, pseudo-version, branch, or commit SHA and persists the exact resolved versions Bazel should use | | `--go-binary` | `go` | Go binary used for bootstrap module graph synchronization. Set this to a pinned SDK path named `go` or `go.exe` when the repository must match Bazel's Go SDK; do not include shell syntax or arguments | | `--go-mod-sync` | `targeted` | Local Go module synchronization strategy: `targeted` updates and verifies only Orchestrion tool packages, `tidy` also runs `go mod tidy`, and `off` skips Go module commands | @@ -99,7 +99,7 @@ Manual Orchestrion wiring in `MODULE.bazel` accepts: | Setting | Default | Description | |---------|---------|-------------| | `orchestrion.from_source(..., dd_trace_go_pin_files = ["@//:go.mod", "@//:go.sum"])` | none | Preferred consumer mode. Derives exact direct and transitive supported tracer versions from one checked-in `go.mod` and `go.sum` using the Bazel-managed Go SDK and `-mod=readonly` | -| `orchestrion.from_source(..., dd_trace_go_version = "...")` | legacy `v2.9.0` when no selection mode is set | Explicit shared-version escape hatch that Bazel validates against the target Go module | +| `orchestrion.from_source(..., dd_trace_go_version = "...")` | legacy `v2.9.1` when no selection mode is set | Explicit shared-version escape hatch that Bazel validates against the target Go module | | `orchestrion.from_source(..., dd_trace_go_versions = {...})` | none | Exact canonical per-module tracer versions that Bazel validates against the target Go module for `github.com/DataDog/dd-trace-go/v2`, `github.com/DataDog/dd-trace-go/contrib/net/http/v2`, and `github.com/DataDog/dd-trace-go/contrib/log/slog/v2` | | `orchestrion.from_source(..., go_sdk_root = "@repo//:ROOT")` | none | Bazel-managed Go SDK root used to build Orchestrion instead of discovering a host `go` binary | | `orchestrion.from_source(..., go_sdk_version = "...")` | none | Exact version of `go_sdk_root`; enables bootstrap-cache lookup before SDK materialization and is verified after materialization on a miss | @@ -114,6 +114,7 @@ Notes: - Guided bootstrap declares `go_sdk_root` and `go_sdk_version` from `--runtime-version`. Manual wiring must set both together and keep the version equal to the registered Go toolchain and Test Optimization `runtime_version`. +- Orchestrion `v1.12.0` requires Go `1.25.0` or newer. - Bootstrap repins the local Go module to the same effective versions. - Bootstrap uses targeted module sync by default and does not run `go mod tidy` unless `--go-mod-sync=tidy` is selected. diff --git a/docs/Installation_Reference.md b/docs/Installation_Reference.md index 1bea65f3..c9570ec1 100644 --- a/docs/Installation_Reference.md +++ b/docs/Installation_Reference.md @@ -126,8 +126,8 @@ RTO_ARCHIVE_TYPE="tar.gz" RULES_GO_UPSTREAM="v0_60_0" RULES_GO_VARIANT="base" RULES_GO_STRIP_PREFIX="third_party/rgo/v0_60_0/base" -DD_TRACE_GO_VERSION="v2.9.0" -ORCHESTRION_VERSION="v1.9.0" +DD_TRACE_GO_VERSION="v2.9.1" +ORCHESTRION_VERSION="v1.12.0" ``` The archive URL, SHA256, and prefix are tied to the repository commit. @@ -237,7 +237,7 @@ bazel run @datadog-rules-test-optimization-go//:dd_topt_go_bootstrap -- \ --guided \ --service go-service \ --runtime-version 1.25.0 \ - --dd-trace-go-version v2.9.0 \ + --dd-trace-go-version v2.9.1 \ --write-bazelrc ``` @@ -248,13 +248,13 @@ bazel run @datadog-rules-test-optimization-go//:dd_topt_go_bootstrap -- \ --guided \ --service go-service \ --runtime-version 1.25.0 \ - --dd-trace-go-version v2.9.0 \ + --dd-trace-go-version v2.9.1 \ --go-module-dir path/to/go-module \ --write-bazelrc ``` `--dd-trace-go-version` is optional. If omitted, the workspace uses the default -`v2.9.0`. It accepts a tag, pseudo-version, +`v2.9.1`. It accepts a tag, pseudo-version, branch, or commit SHA. Bootstrap resolves that input to exact tracer versions, keeps the local Go module pins on those same versions, and prevents Bazel and the Go module from silently drifting apart. diff --git a/docs/Language_Onboarding.md b/docs/Language_Onboarding.md index c7a70fa0..b6cddecf 100644 --- a/docs/Language_Onboarding.md +++ b/docs/Language_Onboarding.md @@ -229,7 +229,7 @@ bazel run @datadog-rules-test-optimization-go//:dd_topt_go_bootstrap -- \ --guided \ --service go-service \ --runtime-version 1.25.0 \ - --dd-trace-go-version v2.9.0 \ + --dd-trace-go-version v2.9.1 \ --write-bazelrc ``` @@ -245,7 +245,7 @@ missing. With `--write-bazelrc`, it also writes the managed `test-optimization` config used by the command examples above. `--dd-trace-go-version` is optional. If omitted, the default is -`v2.9.0`. It accepts a tag, pseudo-version, +`v2.9.1`. It accepts a tag, pseudo-version, branch, or commit SHA. Bootstrap resolves that input to the exact tracer versions Bazel will use, repins the local Go module to match, and later builds fail fast if the workspace setting and local pins no longer match. @@ -369,7 +369,7 @@ datadog_go_test_optimization_workspace_repositories( load("@datadog-rules-test-optimization-go//:topt_go_orchestrion_repository.bzl", "dd_topt_go_orchestrion_tool_repo") dd_topt_go_orchestrion_tool_repo( - version = "v1.9.0", + version = "v1.12.0", dd_trace_go_pin_files = [ "@//:go.mod", "@//:go.sum", @@ -435,7 +435,7 @@ bazel run @datadog-rules-test-optimization-go//:dd_topt_go_bootstrap -- \ --rules-go-repo-name "" \ --rules-go-upstream v0_60_0 \ --rules-go-variant base \ - --dd-trace-go-version v2.9.0 \ + --dd-trace-go-version v2.9.1 \ --write-bazelrc \ --write-orchestrion-files \ --write-wrapper-template \ diff --git a/docs/Troubleshooting.md b/docs/Troubleshooting.md index 85aeaebd..7c8689fe 100644 --- a/docs/Troubleshooting.md +++ b/docs/Troubleshooting.md @@ -876,7 +876,7 @@ module version. either explicit version field. 5. **If you omitted every selection mode**, remember the legacy default is - `v2.9.0`. + `v2.9.1`. The build fails on purpose here. It is preventing Bazel from injecting one set of tracer versions while the local Go module still resolves another. diff --git a/docs/go_orchestrion_bazel_deep_dive.md b/docs/go_orchestrion_bazel_deep_dive.md index cf7b5809..762639b7 100644 --- a/docs/go_orchestrion_bazel_deep_dive.md +++ b/docs/go_orchestrion_bazel_deep_dive.md @@ -224,7 +224,7 @@ their repository placement and local wrapper policy under owner review while still using the same Orchestrion-enabled `rules_go` variants. If no tracer setting is present, the default is still -`v2.9.0`. Bootstrap keeps the local Go module +`v2.9.1`. Bootstrap keeps the local Go module on the same effective versions, and the Bazel build now fails fast if the workspace setting and the local Go module pins drift apart. diff --git a/docs/internal_monorepo_go_rollout_guide.md b/docs/internal_monorepo_go_rollout_guide.md index 0d97852b..7d9087de 100644 --- a/docs/internal_monorepo_go_rollout_guide.md +++ b/docs/internal_monorepo_go_rollout_guide.md @@ -115,7 +115,7 @@ bazel run @datadog-rules-test-optimization-go//:dd_topt_go_bootstrap -- \ --rules-go-repo-name "" \ --rules-go-upstream v0_60_0 \ --rules-go-variant base \ - --dd-trace-go-version v2.9.0 \ + --dd-trace-go-version v2.9.1 \ --write-bazelrc \ --write-orchestrion-files \ --write-wrapper-template \ diff --git a/examples/README.md b/examples/README.md index 116c2b8c..f2ac9afc 100644 --- a/examples/README.md +++ b/examples/README.md @@ -224,11 +224,11 @@ bazel run @datadog-rules-test-optimization-go//:dd_topt_go_bootstrap -- \ --service go-service \ --sync-repo-name test_optimization_data_go \ --runtime-version 1.25.0 \ - --dd-trace-go-version v2.9.0 + --dd-trace-go-version v2.9.1 ``` `--dd-trace-go-version` is optional. If omitted, bootstrap uses the default -`v2.9.0`. It accepts a tag, +`v2.9.1`. It accepts a tag, pseudo-version, branch, or commit SHA. Bootstrap resolves that input to exact versions and repins the local Go module to match what Bazel will use. @@ -559,11 +559,11 @@ Bootstrap once after adding the Go module files: ```bash bazel run @datadog-rules-test-optimization-go//:dd_topt_go_bootstrap -- \ --go-module-dir src/go-project \ - --dd-trace-go-version v2.9.0 + --dd-trace-go-version v2.9.1 ``` As in the single-service flow, `--dd-trace-go-version` is optional and defaults -to `v2.9.0`. It may resolve to one shared tracer version or to separate exact +to `v2.9.1`. It may resolve to one shared tracer version or to separate exact versions for the traced Go modules when you pass a branch or commit SHA. This multi-service path stays on the lower-level/manual API. Guided bootstrap is diff --git a/examples/single_service/src/go-project/go.mod b/examples/single_service/src/go-project/go.mod index 360e1264..e725c4ea 100644 --- a/examples/single_service/src/go-project/go.mod +++ b/examples/single_service/src/go-project/go.mod @@ -9,78 +9,74 @@ module example.com/single-service-go-project go 1.25.0 require ( - github.com/DataDog/dd-trace-go/contrib/log/slog/v2 v2.9.0 - github.com/DataDog/dd-trace-go/contrib/net/http/v2 v2.9.0 - github.com/DataDog/dd-trace-go/v2 v2.9.0 - github.com/DataDog/orchestrion v1.10.0 + github.com/DataDog/dd-trace-go/contrib/log/slog/v2 v2.9.1 + github.com/DataDog/dd-trace-go/contrib/net/http/v2 v2.9.1 + github.com/DataDog/dd-trace-go/v2 v2.9.1 + github.com/DataDog/orchestrion v1.12.0 ) require ( - github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.77.0 // indirect - github.com/DataDog/datadog-agent/pkg/obfuscate v0.77.0 // indirect - github.com/DataDog/datadog-agent/pkg/opentelemetry-mapping-go/otlp/attributes v0.77.0 // indirect - github.com/DataDog/datadog-agent/pkg/proto v0.77.0 // indirect - github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.77.0 // indirect - github.com/DataDog/datadog-agent/pkg/template v0.77.0 // indirect - github.com/DataDog/datadog-agent/pkg/trace v0.77.0 // indirect - github.com/DataDog/datadog-agent/pkg/trace/log v0.77.0 // indirect - github.com/DataDog/datadog-agent/pkg/trace/otel v0.77.0 // indirect - github.com/DataDog/datadog-agent/pkg/trace/stats v0.77.0 // indirect - github.com/DataDog/datadog-agent/pkg/trace/traceutil v0.77.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/log v0.77.0 // indirect - github.com/DataDog/datadog-agent/pkg/util/scrubber v0.77.0 // indirect - github.com/DataDog/datadog-agent/pkg/version v0.77.0 // indirect - github.com/DataDog/datadog-go/v5 v5.8.3 // indirect - github.com/DataDog/go-libddwaf/v4 v4.9.0 // indirect + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.81.2 // indirect + github.com/DataDog/datadog-agent/pkg/obfuscate v0.81.2 // indirect + github.com/DataDog/datadog-agent/pkg/opentelemetry-mapping-go/otlp/attributes v0.81.2 // indirect + github.com/DataDog/datadog-agent/pkg/proto v0.81.2 // indirect + github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.81.2 // indirect + github.com/DataDog/datadog-agent/pkg/trace v0.81.2 // indirect + github.com/DataDog/datadog-agent/pkg/trace/log v0.81.2 // indirect + github.com/DataDog/datadog-agent/pkg/trace/stats v0.81.2 // indirect + github.com/DataDog/datadog-agent/pkg/trace/traceutil v0.81.2 // indirect + github.com/DataDog/datadog-go/v5 v5.9.0 // indirect + github.com/DataDog/go-libddwaf/v4 v4.10.0 // indirect github.com/DataDog/go-runtime-metrics-internal v0.0.4-0.20260217080614-b0f4edc38a6d // indirect - github.com/DataDog/go-sqllexer v0.1.13 // indirect + github.com/DataDog/go-sqllexer v0.2.4 // indirect github.com/DataDog/go-tuf v1.1.1-0.5.2 // indirect github.com/DataDog/sketches-go v1.4.8 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect - github.com/antithesishq/antithesis-sdk-go v0.7.0 // indirect + github.com/antithesishq/antithesis-sdk-go v0.7.2 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/charmbracelet/colorprofile v0.3.2 // indirect + github.com/charmbracelet/colorprofile v0.4.3 // indirect github.com/charmbracelet/lipgloss v1.1.0 // indirect - github.com/charmbracelet/x/ansi v0.10.2 // indirect - github.com/charmbracelet/x/cellbuf v0.0.13 // indirect + github.com/charmbracelet/x/ansi v0.11.7 // indirect + github.com/charmbracelet/x/cellbuf v0.0.15 // indirect github.com/charmbracelet/x/term v0.2.2 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect + github.com/clipperhouse/displaywidth v0.11.0 // indirect github.com/clipperhouse/uax29/v2 v2.7.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect - github.com/dave/dst v0.27.3 // indirect + github.com/dave/dst v0.27.4 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/ebitengine/purego v0.10.0 // indirect - github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/ebitengine/purego v0.10.2 // indirect + github.com/fsnotify/fsnotify v1.10.1 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/goccy/go-yaml v1.19.2 // indirect github.com/google/go-tpm v0.9.8 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/hashicorp/go-version v1.8.0 // indirect + github.com/hashicorp/go-version v1.9.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.18.6 // indirect - github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/klauspost/compress v1.19.1 // indirect + github.com/klauspost/cpuid/v2 v2.4.0 // indirect github.com/linkdata/deadlock v0.5.5 // indirect - github.com/lucasb-eyer/go-colorful v1.3.0 // indirect - github.com/lufia/plan9stats v0.0.0-20260216142805-b3301c5f2a88 // indirect - github.com/mattn/go-colorable v0.1.14 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect - github.com/mattn/go-runewidth v0.0.21 // indirect + github.com/lucasb-eyer/go-colorful v1.4.0 // indirect + github.com/lufia/plan9stats v0.0.0-20260627054121-477a66015f15 // indirect + github.com/mattn/go-colorable v0.1.15 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect + github.com/mattn/go-runewidth v0.0.27 // indirect github.com/minio/highwayhash v1.0.4 // indirect github.com/minio/simdjson-go v0.4.5 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/muesli/termenv v0.16.0 // indirect - github.com/nats-io/jwt/v2 v2.8.1 // indirect - github.com/nats-io/nats-server/v2 v2.12.6 // indirect - github.com/nats-io/nats.go v1.49.0 // indirect - github.com/nats-io/nkeys v0.4.15 // indirect + github.com/nats-io/jwt/v2 v2.8.2 // indirect + github.com/nats-io/nats-server/v2 v2.14.3 // indirect + github.com/nats-io/nats.go v1.52.0 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/outcaste-io/ristretto v0.2.3 // indirect - github.com/petermattis/goid v0.0.0-20260226131333-17d1149c6ac6 // indirect + github.com/petermattis/goid v0.0.0-20260725062400-500c67a39b75 // indirect github.com/philhofer/fwd v1.2.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect @@ -89,14 +85,14 @@ require ( github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/puzpuzpuz/xsync/v3 v3.5.1 // indirect github.com/rivo/uniseg v0.4.7 // indirect - github.com/rs/zerolog v1.34.0 // indirect + github.com/rs/zerolog v1.35.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/secure-systems-lab/go-securesystemslib v0.10.0 // indirect - github.com/shirou/gopsutil/v4 v4.26.2 // indirect + github.com/secure-systems-lab/go-securesystemslib v0.11.0 // indirect + github.com/shirou/gopsutil/v4 v4.26.6 // indirect github.com/stretchr/testify v1.11.1 // indirect - github.com/tinylib/msgp v1.6.3 // indirect - github.com/tklauser/go-sysconf v0.3.16 // indirect - github.com/tklauser/numcpus v0.11.0 // indirect + github.com/tinylib/msgp v1.6.4 // indirect + github.com/tklauser/go-sysconf v0.4.0 // indirect + github.com/tklauser/numcpus v0.12.0 // indirect github.com/trailofbits/go-mutexasserts v0.0.0-20250514102930-c1f3d2e37561 // indirect github.com/urfave/cli/v2 v2.27.7 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect @@ -105,29 +101,29 @@ require ( github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect - go.opentelemetry.io/collector/component v1.51.1-0.20260205185216-81bc641f26c0 // indirect - go.opentelemetry.io/collector/featuregate v1.51.1-0.20260205185216-81bc641f26c0 // indirect - go.opentelemetry.io/collector/pdata v1.51.1-0.20260205185216-81bc641f26c0 // indirect - go.opentelemetry.io/collector/pdata/pprofile v0.145.1-0.20260205185216-81bc641f26c0 // indirect - go.opentelemetry.io/otel v1.41.0 // indirect - go.opentelemetry.io/otel/metric v1.41.0 // indirect - go.opentelemetry.io/otel/trace v1.41.0 // indirect - go.opentelemetry.io/proto/otlp v1.9.0 // indirect + go.opentelemetry.io/collector/component v1.63.0 // indirect + go.opentelemetry.io/collector/featuregate v1.63.0 // indirect + go.opentelemetry.io/collector/pdata v1.63.0 // indirect + go.opentelemetry.io/collector/pdata/pprofile v0.157.0 // indirect + go.opentelemetry.io/otel v1.44.0 // indirect + go.opentelemetry.io/otel/metric v1.44.0 // indirect + go.opentelemetry.io/otel/trace v1.44.0 // indirect + go.opentelemetry.io/proto/otlp v1.11.0 // indirect go.uber.org/atomic v1.11.0 // indirect + go.uber.org/mock v0.6.0 // indirect go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.27.1 // indirect - go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/crypto v0.53.0 // indirect - golang.org/x/exp v0.0.0-20260209203927-2842357ff358 // indirect - golang.org/x/mod v0.34.0 // indirect - golang.org/x/sync v0.20.0 // indirect - golang.org/x/sys v0.46.0 // indirect - golang.org/x/term v0.44.0 // indirect + go.uber.org/zap v1.28.0 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/crypto v0.54.0 // indirect + golang.org/x/exp v0.0.0-20260718201538-764159d718ef // indirect + golang.org/x/mod v0.38.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/term v0.45.0 // indirect golang.org/x/time v0.15.0 // indirect - golang.org/x/tools v0.43.0 // indirect + golang.org/x/tools v0.48.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 // indirect - google.golang.org/protobuf v1.36.11 // indirect - gopkg.in/ini.v1 v1.67.1 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260724162435-b2f20204f0df // indirect + google.golang.org/protobuf v1.36.12-0.20260116114154-8c4c4ae446ca // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/examples/single_service/src/go-project/go.sum b/examples/single_service/src/go-project/go.sum index 3c83a35f..596ce7b6 100644 --- a/examples/single_service/src/go-project/go.sum +++ b/examples/single_service/src/go-project/go.sum @@ -1,56 +1,46 @@ -github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.77.0 h1:Lu/HEo5svx/UwE7XWh8vOrEHCrVRsein9X1N0jGK5bo= -github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.77.0/go.mod h1:+Ty3r23MjcmMSkr8JbFeqA3utgtc1wxsZ0KaQ9CzoWA= -github.com/DataDog/datadog-agent/pkg/obfuscate v0.77.0 h1:mrHaNnDAIOFAVYhCqDpkenUtbadswHN68ZlG5krv40o= -github.com/DataDog/datadog-agent/pkg/obfuscate v0.77.0/go.mod h1:E6RGAcEOr/d8wsV5/khYHvaHkijWex6dfNEvsBIgR7A= -github.com/DataDog/datadog-agent/pkg/opentelemetry-mapping-go/otlp/attributes v0.77.0 h1:g1d9d1CfG54WjXgvkysTFL9yjXexWeDbYssQaf1PG6c= -github.com/DataDog/datadog-agent/pkg/opentelemetry-mapping-go/otlp/attributes v0.77.0/go.mod h1:N/AB9VGpVwHxCcyX+7GLNYMfnTZvn65vp5cHb5Ed0ow= -github.com/DataDog/datadog-agent/pkg/proto v0.77.0 h1:21nDAKD+LdxZz2pMsLAQTZ+w9Z4JqecjKpt8xY1b7Ig= -github.com/DataDog/datadog-agent/pkg/proto v0.77.0/go.mod h1:g2QYJe1CheZdssiDQpSYWra9hORkh+S3WO8aOqDNLkg= -github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.77.0 h1:SxFGFN/Dd/uREaUTxuVTi0R7fRABzvUtu32YOXcjf6c= -github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.77.0/go.mod h1:TpW5ZwsQTrlRBPjtZH5/OFwpmOqxV/v2i9BiF4Xfcac= -github.com/DataDog/datadog-agent/pkg/template v0.77.0 h1:sUbTCoQyU9kXMc6/aDd4YTP2pe9PlNkgySM11ydMEbE= -github.com/DataDog/datadog-agent/pkg/template v0.77.0/go.mod h1:ZUjICHSlN0of0cmWrYk9Pof0DV0eqHSpTUK1NTnN26Y= -github.com/DataDog/datadog-agent/pkg/trace v0.77.0 h1:B7M6IW0sd60XnLfeP6HEneeR4lRnaNtI9bboU4R1cV0= -github.com/DataDog/datadog-agent/pkg/trace v0.77.0/go.mod h1:+7zMNPjHTDidiphECirrpq5jLK09S9kmLtGRv7di29Q= -github.com/DataDog/datadog-agent/pkg/trace/log v0.77.0 h1:2VY1byEA2XnYVg7+eLQSTgX2f76ZCf/AwpCBJDXCiDc= -github.com/DataDog/datadog-agent/pkg/trace/log v0.77.0/go.mod h1:thnxBOGfMU9uRlFUClXud6J7DdI8qWtElELSds5jqts= -github.com/DataDog/datadog-agent/pkg/trace/otel v0.77.0 h1:9zSto72E+wSETaKs47Yiq5D9du9H71IqWgbDJzGpzSs= -github.com/DataDog/datadog-agent/pkg/trace/otel v0.77.0/go.mod h1:IxBidgqUt8aBrKYq4VKynBHWYZYNoflk+0+m7w+lfbI= -github.com/DataDog/datadog-agent/pkg/trace/stats v0.77.0 h1:InA6JO5R8TFAUcKRxsdmIF1hJpVZqVO5Aqux+Iw/2V0= -github.com/DataDog/datadog-agent/pkg/trace/stats v0.77.0/go.mod h1:iZVotmInV8qaU6Q5h+tsKk4CBYupDcOgTzonzLlMi0k= -github.com/DataDog/datadog-agent/pkg/trace/traceutil v0.77.0 h1:b/2+uA/cG2xEV0LzgwnxloMFWe5sdJa3xtaQhSGN0+s= -github.com/DataDog/datadog-agent/pkg/trace/traceutil v0.77.0/go.mod h1:csT+8o3GOUjhKPs/GqWMb5Zh4iQpuZ/HZQ4Z5ls8Sak= -github.com/DataDog/datadog-agent/pkg/util/log v0.77.0 h1:YFa+8kIg2qQZca9zvowtwCPdHDhGMcTIF+PMIQsLSRs= -github.com/DataDog/datadog-agent/pkg/util/log v0.77.0/go.mod h1:DFK2U5RcB8/BcObgmVEZ4VxqXUi2t7y2svLTtJwQqeo= -github.com/DataDog/datadog-agent/pkg/util/scrubber v0.77.0 h1:dd0W9e39rv0R3DSgnaurVnQ43/jX/juqQPwLpGAJgFs= -github.com/DataDog/datadog-agent/pkg/util/scrubber v0.77.0/go.mod h1:nkhevws2pJvoXSGhjc8wuTbptNQ9ECRBjwVr4hSvoq0= -github.com/DataDog/datadog-agent/pkg/version v0.77.0 h1:fxpMWuoaRHS5vHzCNHftvJ6wdQrGhEmuozjjl8wZG5k= -github.com/DataDog/datadog-agent/pkg/version v0.77.0/go.mod h1:h9eJjfeTHlYYv+kzq6n3rQ07qXGirdCCacn1Ryu4TFQ= -github.com/DataDog/datadog-go/v5 v5.8.3 h1:s58CUJ9s8lezjhTNJO/SxkPBv2qZjS3ktpRSqGF5n0s= -github.com/DataDog/datadog-go/v5 v5.8.3/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= -github.com/DataDog/dd-trace-go/contrib/log/slog/v2 v2.9.0 h1:o5PABRmFQQ1uJcog3PnNF9+182EODnjHB6fjGTFkOIs= -github.com/DataDog/dd-trace-go/contrib/log/slog/v2 v2.9.0/go.mod h1:+wuXa6KiqwWqg3J29gSFcRqu4hxTIJ2twz7BhjZ933Q= -github.com/DataDog/dd-trace-go/contrib/net/http/v2 v2.9.0 h1:bkNoThs8Y2i1pt4eoSWq2QhQ84qAoSZRtuHyQgXbDUE= -github.com/DataDog/dd-trace-go/contrib/net/http/v2 v2.9.0/go.mod h1:IublECGcvP3j2VkSyyfD3vhC/QcbwWigkQyzzQY15eY= -github.com/DataDog/dd-trace-go/v2 v2.9.0 h1:J/EsZ7nPqkf3Pa56AAre306ylYMhtzz6oylmchqc6JA= -github.com/DataDog/dd-trace-go/v2 v2.9.0/go.mod h1:SdMkCESSBc2knx56Xol2pO7jhMDPi7MxyNj6vRYMW48= -github.com/DataDog/go-libddwaf/v4 v4.9.0 h1:a788e37iuH7sR9uIYHkulvTnp2FkXTiZ3yY/kuaHgZE= -github.com/DataDog/go-libddwaf/v4 v4.9.0/go.mod h1:/AZqP6zw3qGJK5mLrA0PkfK3UQDk1zCI2fUNCt4xftE= +github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.81.2 h1:qPnh8RP+tenxoBg28JWG6NnhjpEyIYJm7TdT/Hq5jMY= +github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.81.2/go.mod h1:JTXCHrV/ERkPhtL/fjxD6EUpwimWNc1NE0F2VkYHrKQ= +github.com/DataDog/datadog-agent/pkg/obfuscate v0.81.2 h1:r52XSdzhFUaogBQMh503fTOaJ2WqxXN3yO6S5VocLOk= +github.com/DataDog/datadog-agent/pkg/obfuscate v0.81.2/go.mod h1:wauJ14cAQu8cxwvdel8Qav0oZXJ6kVo0rv2NfwzT65I= +github.com/DataDog/datadog-agent/pkg/opentelemetry-mapping-go/otlp/attributes v0.81.2 h1:31CEJR13+i61vFgvARBUM9m5j2zc7AFEpT1v4bdUTPQ= +github.com/DataDog/datadog-agent/pkg/opentelemetry-mapping-go/otlp/attributes v0.81.2/go.mod h1:6ZZS8xxV9GhPrr6Y5Joi7UOsrTaFJCyFHypFTOVC+50= +github.com/DataDog/datadog-agent/pkg/proto v0.81.2 h1:COucpP+KZcXQuP3WhDaEr60HFkUAiufS3d3MQZC1OmE= +github.com/DataDog/datadog-agent/pkg/proto v0.81.2/go.mod h1:6KiJkkYXi8EizP4kohrzyPWM/wd1yvrvq/ATmixXjL4= +github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.81.2 h1:quh/FpATHUTQVazp6MhNujHOCU2x1hZ3fHMgzmDy8+k= +github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.81.2/go.mod h1:FWe3w1K4kpFZ4se5Mfi2InI9OK0YqNRaQtDdSWIusW8= +github.com/DataDog/datadog-agent/pkg/trace v0.81.2 h1:QokGjgy8rWSllH/2fJoBQNOXHlIsMRrzhW+4Kaxq9ME= +github.com/DataDog/datadog-agent/pkg/trace v0.81.2/go.mod h1:l4jGEuwXdGEMhKlEDdHPkPmdsEpEw1a37T0aHsVy0FM= +github.com/DataDog/datadog-agent/pkg/trace/log v0.81.2 h1:0Lloz6yAUuO5ZNCYuQeaf/dr3IyMXJ/kQLCeJkXVMuQ= +github.com/DataDog/datadog-agent/pkg/trace/log v0.81.2/go.mod h1:tv6ZFPuup37eRzI4xjd3nbdYsoHZqjm7z4jCeEJXUPs= +github.com/DataDog/datadog-agent/pkg/trace/stats v0.81.2 h1:F2kf3CXGd+PRHnxoPHQcTV42wlqL5o0SOrp1Lfou/wU= +github.com/DataDog/datadog-agent/pkg/trace/stats v0.81.2/go.mod h1:cHb2PaQ2m7GC8DWPaW6KtoeBddHthd4CcMopbPBBiWM= +github.com/DataDog/datadog-agent/pkg/trace/traceutil v0.81.2 h1:f9CK7SxhYorNvRvYDTY8o/V2mY2JulEd5AG5Qim9ipY= +github.com/DataDog/datadog-agent/pkg/trace/traceutil v0.81.2/go.mod h1:q4C0jud8BszL2GcEt0n1j0T1Uas98H/1kDHDuZvpgRM= +github.com/DataDog/datadog-go/v5 v5.9.0 h1:0rhs5wBov9Iz+xLXLk4maaReHvOANM1ijSm2IKWtKFs= +github.com/DataDog/datadog-go/v5 v5.9.0/go.mod h1:2SBt8zJu6r7sRQHZFMQ8oCukWTKj0ymwulmNgQzJ1JM= +github.com/DataDog/dd-trace-go/contrib/log/slog/v2 v2.9.1 h1:rsnXTMMo/wF4nRHjdZbpaTuODsm+/iaWm30xIVab0B0= +github.com/DataDog/dd-trace-go/contrib/log/slog/v2 v2.9.1/go.mod h1:rriKtwRyNttBX1/LPuLfPzDvL1iwy9PmlPXLzIP6zGI= +github.com/DataDog/dd-trace-go/contrib/net/http/v2 v2.9.1 h1:+t6JGRRPXlp+jcYnB80rGvkLQYBKiOygnhXpio/AvEc= +github.com/DataDog/dd-trace-go/contrib/net/http/v2 v2.9.1/go.mod h1:bzdF5n+saPrL5RiYNeWLDvFbWgqjawlZhOxhbXDMJMw= +github.com/DataDog/dd-trace-go/v2 v2.9.1 h1:N2aqlWS0nAG5o+ETVyvz3gtboZbfemaD8Q/VumStGRY= +github.com/DataDog/dd-trace-go/v2 v2.9.1/go.mod h1:SdMkCESSBc2knx56Xol2pO7jhMDPi7MxyNj6vRYMW48= +github.com/DataDog/go-libddwaf/v4 v4.10.0 h1:e1kqR5yqqttLRuFXHb8FrrgLfMWTx5LnZICMwQwTDYM= +github.com/DataDog/go-libddwaf/v4 v4.10.0/go.mod h1:/AZqP6zw3qGJK5mLrA0PkfK3UQDk1zCI2fUNCt4xftE= github.com/DataDog/go-runtime-metrics-internal v0.0.4-0.20260217080614-b0f4edc38a6d h1:cH9Bm0tJ8FEQbA4FRi0iRm7Zr/5Lata/Or31c+Dth0E= github.com/DataDog/go-runtime-metrics-internal v0.0.4-0.20260217080614-b0f4edc38a6d/go.mod h1:yDuvU+Ak1TKwgd4K8DNcpJmUrrK8ONLkBMGNAppmBRk= -github.com/DataDog/go-sqllexer v0.1.13 h1:HhT2G21y7SDZYQx9i1b+3Sy/CHhESHet/YKMSm06XcE= -github.com/DataDog/go-sqllexer v0.1.13/go.mod h1:vOw7Ia7z+z6nl3zGZlLIZe0vQlPtCPR906WIPBJadxc= +github.com/DataDog/go-sqllexer v0.2.4 h1:Ah9v8g5KX8vNV7q3t6YoA3fW4F/1OvZh/JbyfPcaySA= +github.com/DataDog/go-sqllexer v0.2.4/go.mod h1:3xTFXBU69vUikYpESggScvC0RKYA7ZIdVrIkLwUOWdE= github.com/DataDog/go-tuf v1.1.1-0.5.2 h1:YWvghV4ZvrQsPcUw8IOUMSDpqc3W5ruOIC+KJxPknv0= github.com/DataDog/go-tuf v1.1.1-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= -github.com/DataDog/orchestrion v1.10.0 h1:BIdnXuu652cL6KmZvN3kTnEuATOnJU6I7hSxprLEE/M= -github.com/DataDog/orchestrion v1.10.0/go.mod h1:nOMG/SAcsXeyUDwlIpNl3iSnDVO9rz6zv0Ed87D+UFQ= +github.com/DataDog/orchestrion v1.12.0 h1:GMdYBXkn3QbaU1mla2b7FN0uF9vJ2UJzrUH1X254Hc0= +github.com/DataDog/orchestrion v1.12.0/go.mod h1:Xg4S/8UP7LpmmeSR/pqhO7vKUpiohSJcU8VCDPkpBpA= github.com/DataDog/sketches-go v1.4.8 h1:pFk9BNn+Rzv8IMIoPUttoOpOr3bJOqU3P6EP5wK+Lv8= github.com/DataDog/sketches-go v1.4.8/go.mod h1:a/wjRUqzqtGS8qRHRPDCs4EAQfmvPDZGDlMIF5mxXOE= github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= -github.com/antithesishq/antithesis-sdk-go v0.7.0 h1:uWDG8BqLD1lI2ps38WDz2vXflrTX2+vLX0SvZtztJtE= -github.com/antithesishq/antithesis-sdk-go v0.7.0/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/antithesishq/antithesis-sdk-go v0.7.2 h1:oEEedg1Xgi8drRjqB0f9tfjhLoInE0IYZfZ6zAhQUbY= +github.com/antithesishq/antithesis-sdk-go v0.7.2/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb h1:m935MPodAbYS46DG4pJSv7WO+VECIWUQ7OJYSoTrMh4= @@ -60,25 +50,26 @@ github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F9 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/charmbracelet/colorprofile v0.3.2 h1:9J27WdztfJQVAQKX2WOlSSRB+5gaKqqITmrvb1uTIiI= -github.com/charmbracelet/colorprofile v0.3.2/go.mod h1:mTD5XzNeWHj8oqHb+S1bssQb7vIHbepiebQ2kPKVKbI= +github.com/charmbracelet/colorprofile v0.4.3 h1:QPa1IWkYI+AOB+fE+mg/5/4HRMZcaXex9t5KX76i20Q= +github.com/charmbracelet/colorprofile v0.4.3/go.mod h1:/zT4BhpD5aGFpqQQqw7a+VtHCzu+zrQtt1zhMt9mR4Q= github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY= github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30= -github.com/charmbracelet/x/ansi v0.10.2 h1:ith2ArZS0CJG30cIUfID1LXN7ZFXRCww6RUvAPA+Pzw= -github.com/charmbracelet/x/ansi v0.10.2/go.mod h1:HbLdJjQH4UH4AqA2HpRWuWNluRE6zxJH/yteYEYCFa8= -github.com/charmbracelet/x/cellbuf v0.0.13 h1:/KBBKHuVRbq1lYx5BzEHBAFBP8VcQzJejZ/IA3iR28k= -github.com/charmbracelet/x/cellbuf v0.0.13/go.mod h1:xe0nKWGd3eJgtqZRaN9RjMtK7xUYchjzPr7q6kcvCCs= +github.com/charmbracelet/x/ansi v0.11.7 h1:kzv1kJvjg2S3r9KHo8hDdHFQLEqn4RBCb39dAYC84jI= +github.com/charmbracelet/x/ansi v0.11.7/go.mod h1:9qGpnAVYz+8ACONkZBUWPtL7lulP9No6p1epAihUZwQ= +github.com/charmbracelet/x/cellbuf v0.0.15 h1:ur3pZy0o6z/R7EylET877CBxaiE1Sp1GMxoFPAIztPI= +github.com/charmbracelet/x/cellbuf v0.0.15/go.mod h1:J1YVbR7MUuEGIFPCaaZ96KDl5NoS0DAWkskup+mOY+Q= github.com/charmbracelet/x/term v0.2.2 h1:xVRT/S2ZcKdhhOuSP4t5cLi5o+JxklsoEObBSgfgZRk= github.com/charmbracelet/x/term v0.2.2/go.mod h1:kF8CY5RddLWrsgVwpw4kAa6TESp6EB5y3uxGLeCqzAI= github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 h1:kHaBemcxl8o/pQ5VM1c8PVE1PubbNx3mjUr09OqWGCs= github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575/go.mod h1:9d6lWj8KzO/fd/NrVaLscBKmPigpZpn5YawRPw+e3Yo= +github.com/clipperhouse/displaywidth v0.11.0 h1:lBc6kY44VFw+TDx4I8opi/EtL9m20WSEFgwIwO+UVM8= +github.com/clipperhouse/displaywidth v0.11.0/go.mod h1:bkrFNkf81G8HyVqmKGxsPufD3JhNl3dSqnGhOoSD/o0= github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= -github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo= github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= -github.com/dave/dst v0.27.3 h1:P1HPoMza3cMEquVf9kKy8yXsFirry4zEnWOdYPOoIzY= -github.com/dave/dst v0.27.3/go.mod h1:jHh6EOibnHgcUW3WjKHisiooEkYwqpHLBSX1iOBhEyc= +github.com/dave/dst v0.27.4 h1:d+EVnOZmphH+lUEXq9rit4GjsFSKJ3AhfRWf7eobTps= +github.com/dave/dst v0.27.4/go.mod h1:jHh6EOibnHgcUW3WjKHisiooEkYwqpHLBSX1iOBhEyc= github.com/dave/jennifer v1.7.1 h1:B4jJJDHelWcDhlRQxWeo0Npa/pYKBLrirAQoTN45txo= github.com/dave/jennifer v1.7.1/go.mod h1:nXbxhEmQfOZhWml3D1cDK5M1FLnMSozpbFN/m3RmGZc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -91,10 +82,10 @@ github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da/go.mod h1:SqUrOPUn github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU= -github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= -github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= -github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/ebitengine/purego v0.10.2 h1:W809HbnvzAxgdm+aOvlSekrM16wGCdT/e76+9tS7gzE= +github.com/ebitengine/purego v0.10.2/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho= +github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -104,12 +95,7 @@ github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= -github.com/golang/mock v1.7.0-rc.1 h1:YojYx61/OLFsiv6Rw1Z96LpldJIy31o+UHmwAUMJ6/U= -github.com/golang/mock v1.7.0-rc.1/go.mod h1:s42URUywIqd+OcERslBJvOjepvNymP31m3q8d/GkuRs= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= @@ -121,14 +107,14 @@ github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/hashicorp/go-version v1.8.0 h1:KAkNb1HAiZd1ukkxDFGmokVZe1Xy9HG6NUp+bPle2i4= -github.com/hashicorp/go-version v1.8.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go-version v1.9.0 h1:CeOIz6k+LoN3qX9Z0tyQrPtiB1DFYRPfCIBtaXPSCnA= +github.com/hashicorp/go-version v1.9.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= -github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= -github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= -github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk= +github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= +github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -137,19 +123,16 @@ github.com/liamg/memoryfs v1.6.0 h1:jAFec2HI1PgMTem5gR7UT8zi9u4BfG5jorCRlLH06W8= github.com/liamg/memoryfs v1.6.0/go.mod h1:z7mfqXFQS8eSeBBsFjYLlxYRMRyiPktytvYCYTb3BSk= github.com/linkdata/deadlock v0.5.5 h1:d6O+rzEqasSfamGDA8u7bjtaq7hOX8Ha4Zn36Wxrkvo= github.com/linkdata/deadlock v0.5.5/go.mod h1:tXb28stzAD3trzEEK0UJWC+rZKuobCoPktPYzebb1u0= -github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag= -github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= -github.com/lufia/plan9stats v0.0.0-20260216142805-b3301c5f2a88 h1:PTw+yKnXcOFCR6+8hHTyWBeQ/P4Nb7dd4/0ohEcWQuM= -github.com/lufia/plan9stats v0.0.0-20260216142805-b3301c5f2a88/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= -github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= -github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= -github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-runewidth v0.0.21 h1:jJKAZiQH+2mIinzCJIaIG9Be1+0NR+5sz/lYEEjdM8w= -github.com/mattn/go-runewidth v0.0.21/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/lucasb-eyer/go-colorful v1.4.0 h1:UtrWVfLdarDgc44HcS7pYloGHJUjHV/4FwW4TvVgFr4= +github.com/lucasb-eyer/go-colorful v1.4.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/lufia/plan9stats v0.0.0-20260627054121-477a66015f15 h1:YkjVPl/YH5XlJ+/NiwzJtPYXXKRcyjmEUhsDci6YK3c= +github.com/lufia/plan9stats v0.0.0-20260627054121-477a66015f15/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= +github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY= +github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/mattn/go-runewidth v0.0.27 h1:Feg/Oou5zI/wnpgDF6omIU0OokC9GxLC/WRknhVlIR0= +github.com/mattn/go-runewidth v0.0.27/go.mod h1:3qAiGCV4Koz/yuveO58qUefmUTRm8r0IGEXZ9jeHp/8= github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= github.com/minio/simdjson-go v0.4.5 h1:r4IQwjRGmWCQ2VeMc7fGiilu1z5du0gJ/I/FsKwgo5A= @@ -162,20 +145,16 @@ github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFd github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc= github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= -github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= -github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= -github.com/nats-io/nats-server/v2 v2.12.6 h1:Egbx9Vl7Ch8wTtpXPGqbehkZ+IncKqShUxvrt1+Enc8= -github.com/nats-io/nats-server/v2 v2.12.6/go.mod h1:4HPlrvtmSO3yd7KcElDNMx9kv5EBJBnJJzQPptXlheo= -github.com/nats-io/nats.go v1.49.0 h1:yh/WvY59gXqYpgl33ZI+XoVPKyut/IcEaqtsiuTJpoE= -github.com/nats-io/nats.go v1.49.0/go.mod h1:fDCn3mN5cY8HooHwE2ukiLb4p4G4ImmzvXyJt+tGwdw= -github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4= -github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs= +github.com/nats-io/jwt/v2 v2.8.2 h1:XXRgB60MSTnqsRwejQurVDs/hcv2dkt+86GjI+I/bMc= +github.com/nats-io/jwt/v2 v2.8.2/go.mod h1:Ag/56sq9OblL4JgdYufDd16Egb17Kr/8WwwuO/forVc= +github.com/nats-io/nats-server/v2 v2.14.3 h1:+xjydPt7rkit67G+04TN0mcO2n+8nveZE7tK/PPV53A= +github.com/nats-io/nats-server/v2 v2.14.3/go.mod h1:5IlCtBzfwyzQzPMjmoJ9W2/LKmnJRtNyuOs/OT+NHDY= +github.com/nats-io/nats.go v1.52.0 h1:n3avV4VBsCgsdwh71TppsTwtv+QdPs7ntSKM8qJLGsc= +github.com/nats-io/nats.go v1.52.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/open-telemetry/opentelemetry-collector-contrib/pkg/sampling v0.145.0 h1:7rdLY2Ewa1WVnjMfJTEKwQ5uPDHYeA1tqNPNROi957U= -github.com/open-telemetry/opentelemetry-collector-contrib/pkg/sampling v0.145.0/go.mod h1:jYlQAaJO4ZyJAW2jcKAbjN+nt5BRCyu49mlZv4Rui7U= -github.com/open-telemetry/opentelemetry-collector-contrib/processor/probabilisticsamplerprocessor v0.145.0 h1:12mxn+8YLeAjMZ1kLGulBcvHrdhRNUmxLVIDnaLkJbQ= -github.com/open-telemetry/opentelemetry-collector-contrib/processor/probabilisticsamplerprocessor v0.145.0/go.mod h1:V87HYJpfmvCeQ6Cjy3Q4xylxfCn2wVSS80wvv5ECc0s= github.com/otiai10/copy v1.14.1 h1:5/7E6qsUMBaH5AnQ0sSLzzTg1oTECmcCmT6lvF45Na8= github.com/otiai10/copy v1.14.1/go.mod h1:oQwrEDDOci3IM8dJF0d8+jnbfPDllW6vUjNc3DoZm9I= github.com/otiai10/mint v1.6.3 h1:87qsV/aw1F5as1eH1zS/yqHY85ANKVMgkDrf9rcxbQs= @@ -183,8 +162,8 @@ github.com/otiai10/mint v1.6.3/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAq github.com/outcaste-io/ristretto v0.2.3 h1:AK4zt/fJ76kjlYObOeNwh4T3asEuaCmp26pOvUOL9w0= github.com/outcaste-io/ristretto v0.2.3/go.mod h1:W8HywhmtlopSB1jeMg3JtdIhf+DYkLAr0VN/s4+MHac= github.com/petermattis/goid v0.0.0-20250813065127-a731cc31b4fe/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= -github.com/petermattis/goid v0.0.0-20260226131333-17d1149c6ac6 h1:rh2lKw/P/EqHa724vYH2+VVQ1YnW4u6EOXl0PMAovZE= -github.com/petermattis/goid v0.0.0-20260226131333-17d1149c6ac6/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= +github.com/petermattis/goid v0.0.0-20260725062400-500c67a39b75 h1:VmZ6mKVkxavKEhEy4ZYyV7BwBYBFBP0TwIqmLk84fpU= +github.com/petermattis/goid v0.0.0-20260725062400-500c67a39b75/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -206,50 +185,48 @@ github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= -github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= -github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= -github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= +github.com/rs/zerolog v1.35.1 h1:m7xQeoiLIiV0BCEY4Hs+j2NG4Gp2o2KPKmhnnLiazKI= +github.com/rs/zerolog v1.35.1/go.mod h1:EjML9kdfa/RMA7h/6z6pYmq1ykOuA8/mjWaEvGI+jcw= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= -github.com/secure-systems-lab/go-securesystemslib v0.10.0 h1:l+H5ErcW0PAehBNrBxoGv1jjNpGYdZ9RcheFkB2WI14= -github.com/secure-systems-lab/go-securesystemslib v0.10.0/go.mod h1:MRKONWmRoFzPNQ9USRF9i1mc7MvAVvF1LlW8X5VWDvk= +github.com/secure-systems-lab/go-securesystemslib v0.11.0 h1:iuCR9kcMFD4QurdKrGvPLoKZLv9YvwPYVr0473BdtFs= +github.com/secure-systems-lab/go-securesystemslib v0.11.0/go.mod h1:+PMOTjUGwHj2vcZ+TFKlb1tXRbrdWE1LYDT5i9JC80Q= github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shirou/gopsutil/v4 v4.26.2 h1:X8i6sicvUFih4BmYIGT1m2wwgw2VG9YgrDTi7cIRGUI= -github.com/shirou/gopsutil/v4 v4.26.2/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= +github.com/shirou/gopsutil/v4 v4.26.6 h1:Mzr/npDtQC/xpeEuQKHZt8Zo9CmPvhTj8nkR8w5TLDs= +github.com/shirou/gopsutil/v4 v4.26.6/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/tinylib/msgp v1.6.3 h1:bCSxiTz386UTgyT1i0MSCvdbWjVW+8sG3PjkGsZQt4s= -github.com/tinylib/msgp v1.6.3/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= -github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA= -github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI= -github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw= -github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ= +github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ= +github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= +github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU= +github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI= +github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4= +github.com/tklauser/numcpus v0.12.0/go.mod h1:ABHeXzJnr/qqwguhClkZKT1/8VABcYrsyUiUGobwWJg= github.com/trailofbits/go-mutexasserts v0.0.0-20250514102930-c1f3d2e37561 h1:qqa3P9AtNn6RMe90l/lxd3eJWnIRxjI4eb5Rx8xqCLA= github.com/trailofbits/go-mutexasserts v0.0.0-20250514102930-c1f3d2e37561/go.mod h1:GA3+Mq3kt3tYAfM0WZCu7ofy+GW9PuGysHfhr+6JX7s= github.com/urfave/cli/v2 v2.27.7 h1:bH59vdhbjLv3LAvIu6gd0usJHgoTTPhCFib8qqOwXYU= github.com/urfave/cli/v2 v2.27.7/go.mod h1:CyNAG/xg+iAOg0N4MPGZqVmv2rCoP267496AOXUZjA4= -github.com/vmihailenco/msgpack/v4 v4.3.13 h1:A2wsiTbvp63ilDaWmsk2wjx6xZdxQOvpiNlKBGKKXKI= -github.com/vmihailenco/msgpack/v4 v4.3.13/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= -github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc= -github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= +github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8= +github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= +github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= +github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= @@ -266,87 +243,67 @@ github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/collector/component v1.51.1-0.20260205185216-81bc641f26c0 h1:ZSlXxE90IY0Cl53RTqzyEgRgRPLTeTNBdGhaTmvj9eY= -go.opentelemetry.io/collector/component v1.51.1-0.20260205185216-81bc641f26c0/go.mod h1:944C7vEIdk13Pn1fBbyaU8C1qKf2XC0jRBlc69NAsRY= -go.opentelemetry.io/collector/component/componentstatus v0.145.0 h1:EwUZfSaagdpRXnlrb0TqReJXXW2p9HWBU5YiIeXPCAE= -go.opentelemetry.io/collector/component/componentstatus v0.145.0/go.mod h1:OiYb8rT4FtSJPFSGCKYvOaajdueDUTJZncixGrmy5aM= -go.opentelemetry.io/collector/component/componenttest v0.145.1-0.20260205185216-81bc641f26c0 h1:+VCK6wX/WN170dcaWJweRAkxpmAEyVucfrUV13NwUlY= -go.opentelemetry.io/collector/component/componenttest v0.145.1-0.20260205185216-81bc641f26c0/go.mod h1:U2wUjKMGwgqM49/q8ORkzzYzSWY2m6zpG/e606eK1wc= -go.opentelemetry.io/collector/consumer v1.51.1-0.20260205185216-81bc641f26c0 h1:WNkJ1bKnRVAEJtBm1bwEkoLG2x7GyANc3/OnErZJ338= -go.opentelemetry.io/collector/consumer v1.51.1-0.20260205185216-81bc641f26c0/go.mod h1:Erk6qdfVj+24QTrGCpurcrF+qdUlHkb4dgMy5wJxLvY= -go.opentelemetry.io/collector/consumer/consumertest v0.145.1-0.20260205185216-81bc641f26c0 h1:FHyDIlTbqt0Y6tDI9EbI3hr9uWthwkeLY7uGF1jZYqQ= -go.opentelemetry.io/collector/consumer/consumertest v0.145.1-0.20260205185216-81bc641f26c0/go.mod h1:IFc/FeaIHQClb8KK0aVn0tFDNMc+/MmfQ+aBT1cJNeo= -go.opentelemetry.io/collector/consumer/xconsumer v0.145.1-0.20260205185216-81bc641f26c0 h1:zg2Jqfy7n7o/LEmLsXB4sFhxWtOEMFCKRyQUFLFUS9M= -go.opentelemetry.io/collector/consumer/xconsumer v0.145.1-0.20260205185216-81bc641f26c0/go.mod h1:SryDCLP2ZaFeZJtA2CSksJ0XvjH8k3LmlfXvy/kC7Wc= -go.opentelemetry.io/collector/featuregate v1.51.1-0.20260205185216-81bc641f26c0 h1:fOXhfT2xKqNhfalTXaT/Wic9EBRK8+9ZH0y8phReQS4= -go.opentelemetry.io/collector/featuregate v1.51.1-0.20260205185216-81bc641f26c0/go.mod h1:/1bclXgP91pISaEeNulRxzzmzMTm4I5Xih2SnI4HRSo= -go.opentelemetry.io/collector/internal/componentalias v0.145.1-0.20260205185216-81bc641f26c0 h1:s4/vCxeIxgQpuWmX1AK1DRbZmEdmNBq925EKES8ebiI= -go.opentelemetry.io/collector/internal/componentalias v0.145.1-0.20260205185216-81bc641f26c0/go.mod h1:Z0TtMbzaMp2qhj1dw4toya8toyQzqoTF46/WhJXplVw= -go.opentelemetry.io/collector/internal/testutil v0.145.0 h1:H/KL0GH3kGqSMKxZvnQ0B0CulfO9xdTg4DZf28uV7fY= -go.opentelemetry.io/collector/internal/testutil v0.145.0/go.mod h1:YAD9EAkwh/l5asZNbEBEUCqEjoL1OKMjAMoPjPqH76c= -go.opentelemetry.io/collector/pdata v1.51.1-0.20260205185216-81bc641f26c0 h1:8tgf9W3aW3vFabyVxPNHKsaoyUytudfVOQbqZI9xBHQ= -go.opentelemetry.io/collector/pdata v1.51.1-0.20260205185216-81bc641f26c0/go.mod h1:GoX1bjKDR++mgFKdT7Hynv9+mdgQ1DDXbjs7/Ww209Q= -go.opentelemetry.io/collector/pdata/pprofile v0.145.1-0.20260205185216-81bc641f26c0 h1:MJcnK8txYZlqHZyfZ1rVf66kt5/kEveIdR6KX//BY1Y= -go.opentelemetry.io/collector/pdata/pprofile v0.145.1-0.20260205185216-81bc641f26c0/go.mod h1:a60GC7wQPhLAixWzKbbP51QLwwc+J0Cmp4SurOlhGUk= -go.opentelemetry.io/collector/pdata/testdata v0.145.0 h1:iFsxsCMtE3lnAc/5kZbhZHpRv1OMmM+O5ry46xdQHbg= -go.opentelemetry.io/collector/pdata/testdata v0.145.0/go.mod h1:0y2ERArdzqmYdJHdKLKue+AUubSEGlwK49F+23+Mbic= -go.opentelemetry.io/collector/pipeline v1.51.1-0.20260205185216-81bc641f26c0 h1:1KP5gXGF9qN1mEzJupZDUQVIND35qe/0Hy6Cptvdk0s= -go.opentelemetry.io/collector/pipeline v1.51.1-0.20260205185216-81bc641f26c0/go.mod h1:xUrAqiebzYbrgxyoXSkk6/Y3oi5Sy3im2iCA51LwUAI= -go.opentelemetry.io/collector/processor v1.51.0 h1:PKpCzkLQmqaW08TOVh/zM0qx07Ihq+DR5J/OBkPiL9o= -go.opentelemetry.io/collector/processor v1.51.0/go.mod h1:rtIPFS+EFRAkG+CSwtjxs2IsIkuZStObvALeueD02XI= -go.opentelemetry.io/collector/processor/processorhelper v0.145.0 h1:vXdv6lHz20Tm3ZEsg0i6jPZJBQgy9kzk/PuqWhHWiiM= -go.opentelemetry.io/collector/processor/processorhelper v0.145.0/go.mod h1:3Ecpe5jHRHGf24EvJHeJ/ekK/a1DLByyq0CSUxjjURg= -go.opentelemetry.io/collector/processor/processortest v0.145.0 h1:RDGBmyZnHk7XVK/EdLt/8iPWj+QLStbbVi1nFTNR01s= -go.opentelemetry.io/collector/processor/processortest v0.145.0/go.mod h1:WAvxAzSojkdoZB915Z1lsVHCPDJBb2fepjJBjenrzjg= -go.opentelemetry.io/collector/processor/xprocessor v0.145.0 h1:DaIE7MxRlg0OL1o2P0GQZtmZeExAmVso3qWv8S0RLps= -go.opentelemetry.io/collector/processor/xprocessor v0.145.0/go.mod h1:kUwRyKBU/kjCmXodd+0z7CpvcP0A9G9/QL+MaJt4U2o= -go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c= -go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE= -go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ= -go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps= -go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8= -go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE= -go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw= -go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg= -go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0= -go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis= -go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= -go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= -go.opentelemetry.io/proto/slim/otlp v1.9.0 h1:fPVMv8tP3TrsqlkH1HWYUpbCY9cAIemx184VGkS6vlE= -go.opentelemetry.io/proto/slim/otlp v1.9.0/go.mod h1:xXdeJJ90Gqyll+orzUkY4bOd2HECo5JofeoLpymVqdI= -go.opentelemetry.io/proto/slim/otlp/collector/profiles/v1development v0.2.0 h1:o13nadWDNkH/quoDomDUClnQBpdQQ2Qqv0lQBjIXjE8= -go.opentelemetry.io/proto/slim/otlp/collector/profiles/v1development v0.2.0/go.mod h1:Gyb6Xe7FTi/6xBHwMmngGoHqL0w29Y4eW8TGFzpefGA= -go.opentelemetry.io/proto/slim/otlp/profiles/v1development v0.2.0 h1:EiUYvtwu6PMrMHVjcPfnsG3v+ajPkbUeH+IL93+QYyk= -go.opentelemetry.io/proto/slim/otlp/profiles/v1development v0.2.0/go.mod h1:mUUHKFiN2SST3AhJ8XhJxEoeVW12oqfXog0Bo8W3Ec4= +go.opentelemetry.io/collector/component v1.63.0 h1:l98ZCxfCTt/O6dYB0JVKKtewaFLe/a6N2qQe61Tbf2o= +go.opentelemetry.io/collector/component v1.63.0/go.mod h1:yLGMmT7jUiqvuGvkqlfR1CBi0dRkSV67tq22I08ZMPk= +go.opentelemetry.io/collector/component/componenttest v0.154.0 h1:uH06tUatG4S45A/f3sFENMMAMzWURmgxKK3MAbVZAUI= +go.opentelemetry.io/collector/component/componenttest v0.154.0/go.mod h1:SQ1JRosjFAZ7kN2yNHNcNakOliqrP0QxglKcYyUrUpQ= +go.opentelemetry.io/collector/featuregate v1.63.0 h1:6EWX1C5AtmIh8hFH97DwK6R7R8Jk3fTLxAUfZPXGutY= +go.opentelemetry.io/collector/featuregate v1.63.0/go.mod h1:4ga1QBMPEejXXmpyJS8lmaRpknJ3Lb9Bvk6e420bUFU= +go.opentelemetry.io/collector/internal/testutil v0.157.0 h1:plojUQwFC5l1ex9KUDaLmCFY/mTxEmf3zrlP7M23IEw= +go.opentelemetry.io/collector/internal/testutil v0.157.0/go.mod h1:Jkjs6rkqs973LqgZ0Fe3zrokQRKULYXPIf4HuqStiEE= +go.opentelemetry.io/collector/pdata v1.63.0 h1:fY2xSG2MnyoBwA4GUhzoogGZMuNS0qHpCoODqaKwiVQ= +go.opentelemetry.io/collector/pdata v1.63.0/go.mod h1:jzozYYhQEkTQ/CCbCBNC+hYUeju9S2J8HIqIDHdxZWk= +go.opentelemetry.io/collector/pdata/pprofile v0.157.0 h1:YRTPhwWzdG0pfJmb8p/qpQm1EdX+JfV20qzwG3ypDqI= +go.opentelemetry.io/collector/pdata/pprofile v0.157.0/go.mod h1:kwy/ufNUBkw8PsFPQnAqCvD12OpGU8h9A1cz5S7xS6g= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk= +go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E= +go.opentelemetry.io/proto/slim/otlp v1.10.0 h1:iR97Vs/ZDR+y9TfuP9b1XBtdPWeC+OMslIBmhcLU7jM= +go.opentelemetry.io/proto/slim/otlp v1.10.0/go.mod h1:lV9250stpjYLPNA5viFabIgP2QlUGRT1GdTgAf8SIUk= +go.opentelemetry.io/proto/slim/otlp/collector/profiles/v1development v0.3.0 h1:RUF5rO0hAlgiJt1fzQVzcVs3vZVNHIcMLgOgG4rWNcQ= +go.opentelemetry.io/proto/slim/otlp/collector/profiles/v1development v0.3.0/go.mod h1:I89cynRj8y+383o7tEQVg2SVA6SRgDVIouWPUVXjx0U= +go.opentelemetry.io/proto/slim/otlp/profiles/v1development v0.3.0 h1:CQvJSldHRUN6Z8jsUeYv8J0lXRvygALXIzsmAeCcZE0= +go.opentelemetry.io/proto/slim/otlp/profiles/v1development v0.3.0/go.mod h1:xSQ+mEfJe/GjK1LXEyVOoSI1N9JV9ZI923X5kup43W4= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= +go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= -go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= -go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo= +go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= -golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= -golang.org/x/exp v0.0.0-20260209203927-2842357ff358 h1:kpfSV7uLwKJbFSEgNhWzGSL47NDSF/5pYYQw1V0ub6c= -golang.org/x/exp v0.0.0-20260209203927-2842357ff358/go.mod h1:R3t0oliuryB5eenPWl3rrQxwnNM3WTwnsRZZiXLAAW8= +golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= +golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= +golang.org/x/exp v0.0.0-20260718201538-764159d718ef h1:LkZ48HFgy/TvhTI0bcWkjgFkgLyKUwcTbDjS0DUjw+A= +golang.org/x/exp v0.0.0-20260718201538-764159d718ef/go.mod h1:EdfpwwqSu+0Li0mzskwHU6FWDV3t9Q+RZDo3QMUtL3Q= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= -golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= +golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= +golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= -golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= +golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -357,47 +314,38 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= -golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= -golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= +golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= -golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s= -golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0= +golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= +golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= -google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= -google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 h1:mWPCjDEyshlQYzBpMNHaEof6UX1PmHcaUODUywQ0uac= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= -google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= -google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260724162435-b2f20204f0df h1:O3ig1i5WDDzsVzRp+cCdgelT9vXnlnOFdlEeFtL4HCc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260724162435-b2f20204f0df/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/protobuf v1.36.12-0.20260116114154-8c4c4ae446ca h1:/ro7D0tSP+jEnQPzy9e1r5L6mAcEShGlE5kFsShX5O8= +google.golang.org/protobuf v1.36.12-0.20260116114154-8c4c4ae446ca/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/ini.v1 v1.67.1 h1:tVBILHy0R6e4wkYOn3XmiITt/hEVH4TFMYvAX2Ytz6k= -gopkg.in/ini.v1 v1.67.1/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= -k8s.io/apimachinery v0.35.1 h1:yxO6gV555P1YV0SANtnTjXYfiivaTPvCTKX6w6qdDsU= -k8s.io/apimachinery v0.35.1/go.mod h1:jQCgFZFR1F4Ik7hvr2g84RTJSZegBc8yHgFWKn//hns= diff --git a/modules/go/tests/test_workspace_helpers.bzl b/modules/go/tests/test_workspace_helpers.bzl index 43077a5f..d5745adf 100644 --- a/modules/go/tests/test_workspace_helpers.bzl +++ b/modules/go/tests/test_workspace_helpers.bzl @@ -121,8 +121,8 @@ def _go_workspace_specs_default_to_config_gated_test(ctx): def _orchestrion_call_spec_test(ctx): env = unittest.begin(ctx) call = build_orchestrion_repo_call_for_tests( - dd_trace_go_version = "v2.9.0", - version = "v1.9.0", + dd_trace_go_version = "v2.9.1", + version = "v1.12.0", go_sdk_root = "@go_sdk//:ROOT", go_sdk_version = "1.25.0", log_timing = True, @@ -131,11 +131,11 @@ def _orchestrion_call_spec_test(ctx): env, { "name": "rules_go_orchestrion_tool", - "dd_trace_go_version": "v2.9.0", + "dd_trace_go_version": "v2.9.1", "dd_trace_go_versions": {}, "dd_trace_go_pin_files": [], "enabled_by_env": True, - "version": "v1.9.0", + "version": "v1.12.0", "go_sdk_root": "@go_sdk//:ROOT", "go_sdk_version": "1.25.0", "log_timing": True, @@ -152,7 +152,7 @@ def _orchestrion_pin_file_call_spec_test(ctx): "@//:go.mod", "@//:go.sum", ], - version = "v1.9.0", + version = "v1.12.0", go_sdk_root = "@go_sdk//:ROOT", go_sdk_version = "1.25.0", ) diff --git a/modules/go/tools/dd_topt_go_bootstrap/main.go b/modules/go/tools/dd_topt_go_bootstrap/main.go index c9f9ce20..b9ea7c47 100644 --- a/modules/go/tools/dd_topt_go_bootstrap/main.go +++ b/modules/go/tools/dd_topt_go_bootstrap/main.go @@ -31,8 +31,8 @@ const ( defaultDatadogFetch = "git" defaultRulesGoFetch = "git" defaultRulesGoRepoName = "io_bazel_rules_go" - defaultOrchestrionVersion = "v1.9.0" - defaultDDTraceGoVersion = "v2.9.0" + defaultOrchestrionVersion = "v1.12.0" + defaultDDTraceGoVersion = "v2.9.1" defaultSyncRepoName = "test_optimization_data" defaultDoctorTargetName = "dd_test_optimization_doctor" defaultUploaderTargetName = "dd_upload_payloads" diff --git a/modules/go/tools/dd_topt_go_bootstrap/main_test.go b/modules/go/tools/dd_topt_go_bootstrap/main_test.go index 9161da61..188cc108 100644 --- a/modules/go/tools/dd_topt_go_bootstrap/main_test.go +++ b/modules/go/tools/dd_topt_go_bootstrap/main_test.go @@ -80,7 +80,7 @@ func TestReplaceManagedSectionAppendsWhenMissing(t *testing.T) { func TestManagedModuleBlockIncludesRulesGoExtension(t *testing.T) { cfg := config{ - orchestrionVersion: "v1.9.0", + orchestrionVersion: "v1.12.0", ddTraceGoVersion: "v2.5.0", runtimeVersion: "1.25.0", rulesGoRemote: "https://github.com/example/repo.git", @@ -108,7 +108,7 @@ func TestManagedModuleBlockIncludesRulesGoExtension(t *testing.T) { if !strings.Contains(got, `orchestrion.from_source(`) { t.Fatalf("expected orchestrion extension call in managed block:\n%s", got) } - if !strings.Contains(got, `version = "v1.9.0"`) { + if !strings.Contains(got, `version = "v1.12.0"`) { t.Fatalf("expected orchestrion version in managed block:\n%s", got) } if strings.Contains(got, `enabled_by_env`) { @@ -124,7 +124,7 @@ func TestManagedModuleBlockIncludesRulesGoExtension(t *testing.T) { func TestManagedModuleBlockWithoutRuntimeVersionPreservesLegacyShape(t *testing.T) { cfg := config{ - orchestrionVersion: "v1.9.0", + orchestrionVersion: "v1.12.0", ddTraceGoVersion: "v2.5.0", rulesGoRemote: "https://github.com/example/repo.git", rulesGoCommit: "deadbeef", @@ -139,7 +139,7 @@ func TestManagedModuleBlockWithoutRuntimeVersionPreservesLegacyShape(t *testing. func TestManagedModuleBlockCanSelectBaseRulesGoVariant(t *testing.T) { cfg := config{ - orchestrionVersion: "v1.9.0", + orchestrionVersion: "v1.12.0", ddTraceGoVersion: "v2.5.0", rulesGoRemote: "https://github.com/example/repo.git", rulesGoCommit: "deadbeef", @@ -177,8 +177,8 @@ func TestWorkspaceSnippetSupportsMixedFetchModes(t *testing.T) { rtoArchiveSHA256: strings.Repeat("0", 64), rtoArchivePrefix: "rules_test_optimization-published-sha", rtoArchiveType: "tar.gz", - orchestrionVersion: "v1.9.0", - ddTraceGoVersion: "v2.9.0", + orchestrionVersion: "v1.12.0", + ddTraceGoVersion: "v2.9.1", runtimeVersion: "1.25.0", } got, err := workspaceSnippet(cfg) @@ -195,7 +195,7 @@ func TestWorkspaceSnippetSupportsMixedFetchModes(t *testing.T) { `rules_go_upstream = "v0_60_0"`, `rules_go_variant = "base"`, `go_orchestrion_tool_repo(`, - `dd_trace_go_version = "v2.9.0"`, + `dd_trace_go_version = "v2.9.1"`, `go_sdk_root = "@go_sdk//:ROOT"`, `go_sdk_version = "1.25.0"`, `go_register_toolchains(version = "1.25.0")`, @@ -219,8 +219,8 @@ func TestWorkspaceSnippetFallsBackToRulesGoCommit(t *testing.T) { rulesGoFetch: "git", rulesGoRepoName: "io_bazel_rules_go", rulesGoVariant: "base", - orchestrionVersion: "v1.9.0", - ddTraceGoVersion: "v2.9.0", + orchestrionVersion: "v1.12.0", + ddTraceGoVersion: "v2.9.1", } got, err := workspaceSnippet(cfg) if err != nil { @@ -239,8 +239,8 @@ func TestWorkspaceSnippetDoesNotRequireModuleFiles(t *testing.T) { rulesGoFetch: "git", rulesGoRepoName: "io_bazel_rules_go", rulesGoVariant: "base", - orchestrionVersion: "v1.9.0", - ddTraceGoVersion: "v2.9.0", + orchestrionVersion: "v1.12.0", + ddTraceGoVersion: "v2.9.1", } if _, err := workspaceSnippet(cfg); err != nil { t.Fatalf("workspaceSnippet should not inspect MODULE.bazel or go.mod: %v", err) @@ -256,8 +256,8 @@ func TestWorkspaceModeSnippetIncludesSyncAndBaseVariant(t *testing.T) { rulesGoFetch: "git", rulesGoRepoName: "io_bazel_rules_go", rulesGoVariant: "base", - orchestrionVersion: "v1.9.0", - ddTraceGoVersion: "v2.9.0", + orchestrionVersion: "v1.12.0", + ddTraceGoVersion: "v2.9.1", syncRepoName: "test_optimization_data_worker", service: "worker", runtimeVersion: "1.25.9", @@ -1361,7 +1361,7 @@ git_override( func TestManagedModuleBlockIncludesPerModuleVersions(t *testing.T) { cfg := config{ - orchestrionVersion: "v1.9.0", + orchestrionVersion: "v1.12.0", ddTraceGoVersions: map[string]string{ "github.com/DataDog/dd-trace-go/v2": "v2.7.0-rc.4", "github.com/DataDog/dd-trace-go/contrib/net/http/v2": "v2.8.0-dev.0.20260316165907-0cdd3b7576b7", @@ -1440,8 +1440,8 @@ git_override( cfg := config{ moduleFile: moduleFile, - orchestrionVersion: "v1.9.0", - ddTraceGoVersion: "v2.9.0", + orchestrionVersion: "v1.12.0", + ddTraceGoVersion: "v2.9.1", rulesGoRemote: defaultRulesGoRemote, rulesGoVariant: defaultRulesGoVariant, rulesGoCommitSet: false, @@ -1479,8 +1479,8 @@ git_override( orchestrion = use_extension("@rules_go//go:extensions.bzl", "orchestrion") orchestrion.from_source( - version = "v1.9.0", - dd_trace_go_version = "v2.9.0", + version = "v1.12.0", + dd_trace_go_version = "v2.9.1", ) use_repo(orchestrion, "rules_go_orchestrion_tool") # END Datadog Go Orchestrion bootstrap @@ -1491,8 +1491,8 @@ use_repo(orchestrion, "rules_go_orchestrion_tool") cfg := config{ moduleFile: moduleFile, - orchestrionVersion: "v1.9.0", - ddTraceGoVersion: "v2.9.0", + orchestrionVersion: "v1.12.0", + ddTraceGoVersion: "v2.9.1", rulesGoRemote: defaultRulesGoRemote, rulesGoVariant: "base", ddTraceGoVersionSet: true, @@ -1529,8 +1529,8 @@ git_override( orchestrion = use_extension("@rules_go//go:extensions.bzl", "orchestrion") orchestrion.from_source( - version = "v1.9.0", - dd_trace_go_version = "v2.9.0", + version = "v1.12.0", + dd_trace_go_version = "v2.9.1", ) use_repo(orchestrion, "rules_go_orchestrion_tool") # END Datadog Go Orchestrion bootstrap @@ -1541,8 +1541,8 @@ use_repo(orchestrion, "rules_go_orchestrion_tool") cfg := config{ moduleFile: moduleFile, - orchestrionVersion: "v1.9.0", - ddTraceGoVersion: "v2.9.0", + orchestrionVersion: "v1.12.0", + ddTraceGoVersion: "v2.9.1", rulesGoRemote: defaultRulesGoRemote, ddTraceGoVersionSet: true, } @@ -1575,8 +1575,8 @@ func TestPatchModuleFileRequiresRulesGoCommitWhenNoPublishedSourceExists(t *test cfg := config{ moduleFile: moduleFile, - orchestrionVersion: "v1.9.0", - ddTraceGoVersion: "v2.9.0", + orchestrionVersion: "v1.12.0", + ddTraceGoVersion: "v2.9.1", rulesGoRemote: defaultRulesGoRemote, rulesGoVariant: defaultRulesGoVariant, ddTraceGoVersionSet: true, @@ -1669,8 +1669,8 @@ func TestWriteOrchestrionToolFileWritesManagedImports(t *testing.T) { func TestBootstrapSyncCommandsTargetedModeAvoidsGoModTidy(t *testing.T) { cfg := config{ - orchestrionVersion: "v1.9.0", - ddTraceGoVersion: "v2.9.0", + orchestrionVersion: "v1.12.0", + ddTraceGoVersion: "v2.9.1", goModSync: "targeted", } @@ -1678,7 +1678,7 @@ func TestBootstrapSyncCommandsTargetedModeAvoidsGoModTidy(t *testing.T) { if len(got) < 3 { t.Fatalf("bootstrapSyncCommands returned too few commands: %#v", got) } - if strings.Join(got[0], " ") != "mod edit -require=github.com/DataDog/orchestrion@v1.9.0" { + if strings.Join(got[0], " ") != "mod edit -require=github.com/DataDog/orchestrion@v1.12.0" { t.Fatalf("first bootstrap sync command=%q, want orchestrion version pin", strings.Join(got[0], " ")) } joined := strings.Join(flattenCommands(got), "\n") @@ -1700,10 +1700,10 @@ func TestBootstrapSyncCommandsTargetedModeAvoidsGoModTidy(t *testing.T) { } } expectedDownloads := []string{ - "github.com/DataDog/orchestrion@v1.9.0", - "github.com/DataDog/dd-trace-go/v2@v2.9.0", - "github.com/DataDog/dd-trace-go/contrib/net/http/v2@v2.9.0", - "github.com/DataDog/dd-trace-go/contrib/log/slog/v2@v2.9.0", + "github.com/DataDog/orchestrion@v1.12.0", + "github.com/DataDog/dd-trace-go/v2@v2.9.1", + "github.com/DataDog/dd-trace-go/contrib/net/http/v2@v2.9.1", + "github.com/DataDog/dd-trace-go/contrib/log/slog/v2@v2.9.1", } for _, moduleVersion := range expectedDownloads { want := "mod download " + moduleVersion @@ -1720,8 +1720,8 @@ func TestBootstrapSyncCommandsTargetedModeAvoidsGoModTidy(t *testing.T) { func TestBootstrapSyncCommandsDefaultsToTargetedMode(t *testing.T) { cfg := config{ - orchestrionVersion: "v1.9.0", - ddTraceGoVersion: "v2.9.0", + orchestrionVersion: "v1.12.0", + ddTraceGoVersion: "v2.9.1", } joined := strings.Join(flattenCommands(bootstrapSyncCommands(cfg)), "\n") @@ -1735,8 +1735,8 @@ func TestBootstrapSyncCommandsDefaultsToTargetedMode(t *testing.T) { func TestBootstrapSyncCommandsTidyModeKeepsExplicitGoModTidy(t *testing.T) { cfg := config{ - orchestrionVersion: "v1.9.0", - ddTraceGoVersion: "v2.9.0", + orchestrionVersion: "v1.12.0", + ddTraceGoVersion: "v2.9.1", goModSync: "tidy", } @@ -1748,8 +1748,8 @@ func TestBootstrapSyncCommandsTidyModeKeepsExplicitGoModTidy(t *testing.T) { func TestBootstrapSyncCommandsOffModeSkipsGoCommands(t *testing.T) { cfg := config{ - orchestrionVersion: "v1.9.0", - ddTraceGoVersion: "v2.9.0", + orchestrionVersion: "v1.12.0", + ddTraceGoVersion: "v2.9.1", goModSync: "off", } @@ -1800,8 +1800,8 @@ exit 0 goBinary: goPath, goModuleDir: dir, goModSync: "targeted", - orchestrionVersion: "v1.9.0", - ddTraceGoVersion: "v2.9.0", + orchestrionVersion: "v1.12.0", + ddTraceGoVersion: "v2.9.1", ddTraceGoVersions: nil, ddTraceGoVersionSet: true, } @@ -1840,8 +1840,8 @@ esac goBinary: goPath, goModuleDir: dir, goModSync: "targeted", - orchestrionVersion: "v1.9.0", - ddTraceGoVersion: "v2.9.0", + orchestrionVersion: "v1.12.0", + ddTraceGoVersion: "v2.9.1", ddTraceGoVersionSet: true, } @@ -2218,9 +2218,9 @@ exit 0 goBinary: goPath, goModuleDir: t.TempDir(), ddTraceGoVersions: map[string]string{ - "github.com/DataDog/dd-trace-go/v2": "v2.9.0", - "github.com/DataDog/dd-trace-go/contrib/net/http/v2": "v2.9.0", - "github.com/DataDog/dd-trace-go/contrib/log/slog/v2": "v2.9.0", + "github.com/DataDog/dd-trace-go/v2": "v2.9.1", + "github.com/DataDog/dd-trace-go/contrib/net/http/v2": "v2.9.1", + "github.com/DataDog/dd-trace-go/contrib/log/slog/v2": "v2.9.1", }, } if err := warmOrchestrionModuleCache(cfg); err != nil { @@ -2384,7 +2384,7 @@ func TestEnsureBootstrapCanManageTracerConfigRejectsManualTracerConfig(t *testin content := `module(name = "example") orchestrion = use_extension("@rules_go//go:extensions.bzl", "orchestrion") orchestrion.from_source( - version = "v1.9.0", + version = "v1.12.0", dd_trace_go_versions = { "github.com/DataDog/dd-trace-go/v2": "v2.7.0-rc.4", "github.com/DataDog/dd-trace-go/contrib/net/http/v2": "v2.8.0-dev.0.20260316165907-0cdd3b7576b7", @@ -2409,7 +2409,7 @@ git_override( orchestrion = use_extension("@rules_go//go:extensions.bzl", "orchestrion") orchestrion.from_source( - version = "v1.9.0", + version = "v1.12.0", dd_trace_go_versions = { "github.com/DataDog/dd-trace-go/v2": "v2.7.0-rc.4", "github.com/DataDog/dd-trace-go/contrib/net/http/v2": "v2.8.0-dev.0.20260316165907-0cdd3b7576b7", @@ -2436,17 +2436,17 @@ func TestParseGoRepositoryDeclarations(t *testing.T) { go_repository( name = "com_github_datadog_orchestrion", importpath = "github.com/DataDog/orchestrion", - version = "v1.9.0", + version = "v1.12.0", ) go_repository( name = 'com_github_datadog_dd_trace_go_v2', importpath = 'github.com/DataDog/dd-trace-go/v2', - version = 'v2.9.0', + version = 'v2.9.1', ) ` got := parseGoRepositoryDeclarations(content) - if got["github.com/DataDog/orchestrion"].version != "v1.9.0" { + if got["github.com/DataDog/orchestrion"].version != "v1.12.0" { t.Fatalf("unexpected orchestrion declaration: %#v", got) } if got["github.com/DataDog/dd-trace-go/v2"].name != "com_github_datadog_dd_trace_go_v2" { @@ -2465,11 +2465,11 @@ func TestParseGoRepositoryDeclarationsIgnoresCommentedBlocks(t *testing.T) { go_repository( name = "com_github_datadog_orchestrion", importpath = "github.com/DataDog/orchestrion", # active declaration - version = "v1.9.0", + version = "v1.12.0", ) ` got := parseGoRepositoryDeclarations(content) - if got["github.com/DataDog/orchestrion"].version != "v1.9.0" { + if got["github.com/DataDog/orchestrion"].version != "v1.12.0" { t.Fatalf("commented declaration should be ignored: %#v", got) } } @@ -2477,10 +2477,10 @@ go_repository( func TestCheckGoRepositoriesAcceptsMatchingVersions(t *testing.T) { dir := t.TempDir() writeRepositoriesFile(t, filepath.Join(dir, "repositories.bzl"), map[string]string{ - "github.com/DataDog/orchestrion": "v1.9.0", - "github.com/DataDog/dd-trace-go/v2": "v2.9.0", - "github.com/DataDog/dd-trace-go/contrib/net/http/v2": "v2.9.0", - "github.com/DataDog/dd-trace-go/contrib/log/slog/v2": "v2.9.0", + "github.com/DataDog/orchestrion": "v1.12.0", + "github.com/DataDog/dd-trace-go/v2": "v2.9.1", + "github.com/DataDog/dd-trace-go/contrib/net/http/v2": "v2.9.1", + "github.com/DataDog/dd-trace-go/contrib/log/slog/v2": "v2.9.1", }) cfg := goRepositoryDiagnosticsTestConfig(dir) if err := checkGoRepositories(cfg, false); err != nil { @@ -2492,9 +2492,9 @@ func TestCheckGoRepositoriesRejectsStaleVersionWithActionableMessage(t *testing. dir := t.TempDir() writeRepositoriesFile(t, filepath.Join(dir, "repositories.bzl"), map[string]string{ "github.com/DataDog/orchestrion": "v1.8.0", - "github.com/DataDog/dd-trace-go/v2": "v2.9.0", - "github.com/DataDog/dd-trace-go/contrib/net/http/v2": "v2.9.0", - "github.com/DataDog/dd-trace-go/contrib/log/slog/v2": "v2.9.0", + "github.com/DataDog/dd-trace-go/v2": "v2.9.1", + "github.com/DataDog/dd-trace-go/contrib/net/http/v2": "v2.9.1", + "github.com/DataDog/dd-trace-go/contrib/log/slog/v2": "v2.9.1", }) cfg := goRepositoryDiagnosticsTestConfig(dir) err := checkGoRepositories(cfg, false) @@ -2505,7 +2505,7 @@ func TestCheckGoRepositoriesRejectsStaleVersionWithActionableMessage(t *testing. for _, want := range []string{ "github.com/DataDog/orchestrion", "v1.8.0", - "v1.9.0", + "v1.12.0", "--go-repositories-refresh-command", "go_repository(", } { @@ -2518,7 +2518,7 @@ func TestCheckGoRepositoriesRejectsStaleVersionWithActionableMessage(t *testing. func TestCheckGoRepositoriesRejectsMissingModule(t *testing.T) { dir := t.TempDir() writeRepositoriesFile(t, filepath.Join(dir, "repositories.bzl"), map[string]string{ - "github.com/DataDog/orchestrion": "v1.9.0", + "github.com/DataDog/orchestrion": "v1.12.0", }) cfg := goRepositoryDiagnosticsTestConfig(dir) err := checkGoRepositories(cfg, false) @@ -2545,10 +2545,10 @@ func TestCheckGoRepositoriesRefreshHookRepairsStaleFile(t *testing.T) { refreshPath := filepath.Join(dir, "refresh.sh") if err := os.WriteFile(refreshPath, []byte(`#!/bin/sh cat > repositories.bzl <<'EOF' -go_repository(name = "com_github_datadog_orchestrion", importpath = "github.com/DataDog/orchestrion", version = "v1.9.0") -go_repository(name = "com_github_datadog_dd_trace_go_v2", importpath = "github.com/DataDog/dd-trace-go/v2", version = "v2.9.0") -go_repository(name = "com_github_datadog_dd_trace_go_contrib_net_http_v2", importpath = "github.com/DataDog/dd-trace-go/contrib/net/http/v2", version = "v2.9.0") -go_repository(name = "com_github_datadog_dd_trace_go_contrib_log_slog_v2", importpath = "github.com/DataDog/dd-trace-go/contrib/log/slog/v2", version = "v2.9.0") +go_repository(name = "com_github_datadog_orchestrion", importpath = "github.com/DataDog/orchestrion", version = "v1.12.0") +go_repository(name = "com_github_datadog_dd_trace_go_v2", importpath = "github.com/DataDog/dd-trace-go/v2", version = "v2.9.1") +go_repository(name = "com_github_datadog_dd_trace_go_contrib_net_http_v2", importpath = "github.com/DataDog/dd-trace-go/contrib/net/http/v2", version = "v2.9.1") +go_repository(name = "com_github_datadog_dd_trace_go_contrib_log_slog_v2", importpath = "github.com/DataDog/dd-trace-go/contrib/log/slog/v2", version = "v2.9.1") EOF `), 0o755); err != nil { t.Fatalf("write refresh hook: %v", err) @@ -2592,8 +2592,8 @@ func goRepositoryDiagnosticsTestConfig(dir string) config { workspaceDir: dir, goRepositoriesFile: "repositories.bzl", checkGoRepositories: true, - orchestrionVersion: "v1.9.0", - ddTraceGoVersion: "v2.9.0", + orchestrionVersion: "v1.12.0", + ddTraceGoVersion: "v2.9.1", ddTraceGoVersions: nil, ddTraceGoVersionSet: true, } diff --git a/modules/go/tools/onboardingpins/pins.go b/modules/go/tools/onboardingpins/pins.go index c9b350e4..aa254c91 100644 --- a/modules/go/tools/onboardingpins/pins.go +++ b/modules/go/tools/onboardingpins/pins.go @@ -29,9 +29,9 @@ const ( // DefaultArchiveType is the archive type used by GitHub codeload tarballs. DefaultArchiveType = "tar.gz" // DefaultDDTraceGoVersion is the Go tracer version that supports the Bazel JSON payload contract. - DefaultDDTraceGoVersion = "v2.9.0" + DefaultDDTraceGoVersion = "v2.9.1" // DefaultOrchestrionVersion is the Orchestrion version validated by the Go onboarding fixtures. - DefaultOrchestrionVersion = "v1.9.0" + DefaultOrchestrionVersion = "v1.12.0" // DefaultMainRef is the remote ref that published pins must be reachable from. DefaultMainRef = "origin/main" diff --git a/modules/go/tools/onboardingpins/pins_test.go b/modules/go/tools/onboardingpins/pins_test.go index 3a7f133c..915c1bb6 100644 --- a/modules/go/tools/onboardingpins/pins_test.go +++ b/modules/go/tools/onboardingpins/pins_test.go @@ -414,6 +414,6 @@ func ExampleFormatShell() { // RULES_GO_UPSTREAM="v0_60_0" // RULES_GO_VARIANT="base" // RULES_GO_STRIP_PREFIX="third_party/rgo/v0_60_0/base" - // DD_TRACE_GO_VERSION="v2.9.0" - // ORCHESTRION_VERSION="v1.9.0" + // DD_TRACE_GO_VERSION="v2.9.1" + // ORCHESTRION_VERSION="v1.12.0" } diff --git a/third_party/rgo/v0_60_0/base/go/private/orchestrion/extensions.bzl b/third_party/rgo/v0_60_0/base/go/private/orchestrion/extensions.bzl index db1f149f..a56e5681 100644 --- a/third_party/rgo/v0_60_0/base/go/private/orchestrion/extensions.bzl +++ b/third_party/rgo/v0_60_0/base/go/private/orchestrion/extensions.bzl @@ -14,7 +14,7 @@ """Module extension for configuring orchestrion in rules_go.""" -DEFAULT_DD_TRACE_GO_VERSION = "v2.9.0" +DEFAULT_DD_TRACE_GO_VERSION = "v2.9.1" ORCHESTRION_BOOTSTRAP_CACHE_ABI = "v3" ORCHESTRION_SEED_GO_MOD_VERSION = "1.21" @@ -1372,34 +1372,21 @@ def _orchestrion_build_impl(ctx): ctx.report_progress("rules_go_orchestrion: patching orchestrion source") patch_start_ms = _probe_now_ms(ctx) - # The upstream package resolver recursively re-runs `go list` under - # `-toolexec=orchestrion toolexec`, which causes woven dependency lookups to - # fail under Bazel's sandbox even when plain `go list -mod=mod` succeeds in - # the same environment. Mirror the safer non-recursive pattern used by other - # loader paths in Orchestrion and disable toolexec for dependency resolution. + # Keep Orchestrion's recursive toolexec resolver: v1.12 uses it to build the + # correct dependency variants for generated test mains. Only adapt its + # temporary directory to Bazel's execroot layout. resolve_path = "internal/jobserver/pkgs/resolve.go" resolve_src = ctx.read(resolve_path) - resolve_old = """\n\t\tbuildFlags := append(\n\t\t\tgoFlags.Slice(),\n\t\t\tfmt.Sprintf(\"-toolexec=%q toolexec\", binpath.Orchestrion),\n\t\t)\n""" - resolve_new = """\n\t\tbuildFlags := append(goFlags.Slice(), \"-toolexec=\")\n""" - if resolve_old not in resolve_src: - fail("Could not patch Orchestrion resolver in %s" % resolve_path) - resolve_src = resolve_src.replace(resolve_old, resolve_new) - resolve_src = resolve_src.replace('\n\t"github.com/DataDog/orchestrion/internal/binpath"', "") - resolve_imports_old = """\t\"fmt\"\n\t\"os\"\n\t\"slices\"\n""" - resolve_imports_new = """\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"slices\"\n""" + resolve_imports_old = """\t\"maps\"\n\t\"os\"\n\t\"slices\"\n""" + resolve_imports_new = """\t\"maps\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"slices\"\n""" if resolve_imports_old not in resolve_src: fail("Could not patch Orchestrion resolver imports in %s" % resolve_path) resolve_src = resolve_src.replace(resolve_imports_old, resolve_imports_new, 1) - resolve_tempdir_old = """\t\tif req.TempDir != \"\" {\n\t\t\t// Make sure the directory exists (go blindly assumes that...)\n\t\t\tif err := os.MkdirAll(req.TempDir, 0o755); err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"creating temporary directory %q: %w\", req.TempDir, err)\n\t\t\t}\n\t\t\tenv = append(env, fmt.Sprintf(\"%s=%s\", envVarGotmpdir, req.TempDir))\n\t\t}\n""" - resolve_tempdir_new = """\t\tif req.TempDir != \"\" {\n\t\t\tabsTempDir, absErr := filepath.Abs(req.TempDir)\n\t\t\tif absErr != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"absolutizing temporary directory %q: %w\", req.TempDir, absErr)\n\t\t\t}\n\t\t\t// Make sure the directory exists (go blindly assumes that...)\n\t\t\tif err := os.MkdirAll(absTempDir, 0o755); err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"creating temporary directory %q: %w\", absTempDir, err)\n\t\t\t}\n\t\t\tfor _, name := range []string{\"external\", \"bazel-out\"} {\n\t\t\t\tsrcPath := filepath.Join(req.Dir, name)\n\t\t\t\tif !filepath.IsAbs(srcPath) {\n\t\t\t\t\tif srcPath, absErr = filepath.Abs(srcPath); absErr != nil {\n\t\t\t\t\t\treturn nil, fmt.Errorf(\"absolutizing compatibility path %q: %w\", srcPath, absErr)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif _, statErr := os.Stat(srcPath); statErr != nil {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tdstPath := filepath.Join(absTempDir, name)\n\t\t\t\tif _, statErr := os.Lstat(dstPath); statErr == nil {\n\t\t\t\t\tcontinue\n\t\t\t\t} else if !os.IsNotExist(statErr) {\n\t\t\t\t\treturn nil, fmt.Errorf(\"stat temporary compatibility path %q: %w\", dstPath, statErr)\n\t\t\t\t}\n\t\t\t\tlinkTarget, relErr := filepath.Rel(absTempDir, srcPath)\n\t\t\t\tif relErr != nil {\n\t\t\t\t\tif linkTarget, absErr = filepath.Abs(srcPath); absErr != nil {\n\t\t\t\t\t\treturn nil, fmt.Errorf(\"compute temporary compatibility path for %q: %w\", name, relErr)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif linkErr := os.Symlink(linkTarget, dstPath); linkErr != nil {\n\t\t\t\t\treturn nil, fmt.Errorf(\"create temporary compatibility symlink %q -> %q: %w\", dstPath, linkTarget, linkErr)\n\t\t\t\t}\n\t\t\t\tif log.GetLevel() <= zerolog.TraceLevel {\n\t\t\t\t\tlog.Trace().Str(\"dst\", dstPath).Str(\"target\", linkTarget).Msg(\"pkgs.Resolve created temp compatibility symlink\")\n\t\t\t\t}\n\t\t\t}\n\t\t\tenv = append(env, fmt.Sprintf(\"%s=%s\", envVarGotmpdir, absTempDir))\n\t\t}\n""" + resolve_tempdir_old = """\tif req.TempDir != \"\" {\n\t\tif err := os.MkdirAll(req.TempDir, 0o755); err != nil {\n\t\t\treturn resolvedPackageSet{}, fmt.Errorf(\"creating temporary directory %q: %w\", req.TempDir, err)\n\t\t}\n\t}\n\tenv := resolveEnvironment(ctx, req)\n""" + resolve_tempdir_new = """\tif req.TempDir != \"\" {\n\t\tabsTempDir, absErr := filepath.Abs(req.TempDir)\n\t\tif absErr != nil {\n\t\t\treturn resolvedPackageSet{}, fmt.Errorf(\"absolutizing temporary directory %q: %w\", req.TempDir, absErr)\n\t\t}\n\t\tif err := os.MkdirAll(absTempDir, 0o755); err != nil {\n\t\t\treturn resolvedPackageSet{}, fmt.Errorf(\"creating temporary directory %q: %w\", absTempDir, err)\n\t\t}\n\t\tfor _, name := range []string{\"external\", \"bazel-out\"} {\n\t\t\tsrcPath := filepath.Join(req.Dir, name)\n\t\t\tif !filepath.IsAbs(srcPath) {\n\t\t\t\tif srcPath, absErr = filepath.Abs(srcPath); absErr != nil {\n\t\t\t\t\treturn resolvedPackageSet{}, fmt.Errorf(\"absolutizing compatibility path %q: %w\", srcPath, absErr)\n\t\t\t\t}\n\t\t\t}\n\t\t\tif _, statErr := os.Stat(srcPath); statErr != nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tdstPath := filepath.Join(absTempDir, name)\n\t\t\tif _, statErr := os.Lstat(dstPath); statErr == nil {\n\t\t\t\tcontinue\n\t\t\t} else if !os.IsNotExist(statErr) {\n\t\t\t\treturn resolvedPackageSet{}, fmt.Errorf(\"stat temporary compatibility path %q: %w\", dstPath, statErr)\n\t\t\t}\n\t\t\tlinkTarget, relErr := filepath.Rel(absTempDir, srcPath)\n\t\t\tif relErr != nil {\n\t\t\t\tlinkTarget = srcPath\n\t\t\t}\n\t\t\tif linkErr := os.Symlink(linkTarget, dstPath); linkErr != nil {\n\t\t\t\treturn resolvedPackageSet{}, fmt.Errorf(\"create temporary compatibility symlink %q -> %q: %w\", dstPath, linkTarget, linkErr)\n\t\t\t}\n\t\t\tlog.Trace().Str(\"dst\", dstPath).Str(\"target\", linkTarget).Msg(\"pkgs.Resolve created temp compatibility symlink\")\n\t\t}\n\t\treq.TempDir = absTempDir\n\t}\n\tenv := resolveEnvironment(ctx, req)\n""" if resolve_tempdir_old not in resolve_src: fail("Could not patch Orchestrion resolver tempdir block in %s" % resolve_path) resolve_src = resolve_src.replace(resolve_tempdir_old, resolve_tempdir_new, 1) - resolve_merge_old = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error {\n\tif pkg.PkgPath == \"\" || pkg.PkgPath == \"unsafe\" || r[pkg.PkgPath] != \"\" {\n\t\t// Ignore the \"unsafe\" package (no archive file, ever), packages with an empty import path\n\t\t// (standard library), and those already present in the map (already processed previously).\n\t\treturn nil\n\t}\n""" - resolve_merge_new = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error {\n\tif pkg.PkgPath == \"\" || pkg.PkgPath == \"unsafe\" || r[pkg.PkgPath] != \"\" {\n\t\t// Ignore the \"unsafe\" package (no archive file, ever), packages with an empty import path\n\t\t// (standard library), and those already present in the map (already processed previously).\n\t\treturn nil\n\t}\n""" - if resolve_merge_old not in resolve_src: - fail("Could not patch Orchestrion resolver mergeFrom in %s" % resolve_path) - resolve_src = resolve_src.replace(resolve_merge_old, resolve_merge_new, 1) ctx.file(resolve_path, resolve_src) # Compile-proxy / archive metadata patches for the synthetic testmain flow. @@ -1460,16 +1447,16 @@ def _orchestrion_build_impl(ctx): # run under Bazel's import-path context instead of synthetic package names. onlink_path = "internal/toolexec/aspect/onlink.go" onlink_src = ctx.read(onlink_path) - onlink_old = """\t\t\tdeps, err := resolvePackageFiles(ctx, depPath, cmd.WorkDir)\n""" - onlink_new = """\t\t\tprevImportPath, hadImportPath := os.LookupEnv(\"TOOLEXEC_IMPORTPATH\")\n\t\t\tif err := os.Setenv(\"TOOLEXEC_IMPORTPATH\", w.ImportPath); err != nil {\n\t\t\t\treturn fmt.Errorf(\"setting TOOLEXEC_IMPORTPATH=%q: %w\", w.ImportPath, err)\n\t\t\t}\n\t\t\tdeps, err := resolvePackageFiles(ctx, depPath, \".\")\n\t\t\tif hadImportPath {\n\t\t\t\t_ = os.Setenv(\"TOOLEXEC_IMPORTPATH\", prevImportPath)\n\t\t\t} else {\n\t\t\t\t_ = os.Unsetenv(\"TOOLEXEC_IMPORTPATH\")\n\t\t\t}\n""" + onlink_old = """\t\t\tdeps, err := resolvePackageFilesForTest(ctx, depPath, testVariantFor, cmd.WorkDir)\n""" + onlink_new = """\t\t\tprevImportPath, hadImportPath := os.LookupEnv(\"TOOLEXEC_IMPORTPATH\")\n\t\t\tif err := os.Setenv(\"TOOLEXEC_IMPORTPATH\", w.ImportPath); err != nil {\n\t\t\t\treturn fmt.Errorf(\"setting TOOLEXEC_IMPORTPATH=%q: %w\", w.ImportPath, err)\n\t\t\t}\n\t\t\tdeps, err := resolvePackageFilesForTest(ctx, depPath, testVariantFor, \".\")\n\t\t\tif hadImportPath {\n\t\t\t\t_ = os.Setenv(\"TOOLEXEC_IMPORTPATH\", prevImportPath)\n\t\t\t} else {\n\t\t\t\t_ = os.Unsetenv(\"TOOLEXEC_IMPORTPATH\")\n\t\t\t}\n""" if onlink_old not in onlink_src: fail("Could not patch Orchestrion onlink resolver context in %s" % onlink_path) ctx.file(onlink_path, onlink_src.replace(onlink_old, onlink_new, 1)) oncompile_main_path = "internal/toolexec/aspect/oncompile-main.go" oncompile_main_src = ctx.read(oncompile_main_path) - oncompile_main_old = """\t\tdeps, err := resolvePackageFiles(ctx, linkDepPath, cmd.WorkDir)\n""" - oncompile_main_new = """\t\tprevImportPath, hadImportPath := os.LookupEnv(\"TOOLEXEC_IMPORTPATH\")\n\t\tif err := os.Setenv(\"TOOLEXEC_IMPORTPATH\", \"main\"); err != nil {\n\t\t\treturn fmt.Errorf(\"setting TOOLEXEC_IMPORTPATH for synthetic main: %w\", err)\n\t\t}\n\t\tfor archiveImportPath, archivePath := range reg.PackageFile {\n\t\t\tld, ldErr := linkdeps.FromArchive(ctx, archivePath)\n\t\t\tif ldErr != nil || !ld.Contains(linkDepPath) {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif err := os.Setenv(\"TOOLEXEC_IMPORTPATH\", archiveImportPath); err != nil {\n\t\t\t\treturn fmt.Errorf(\"setting TOOLEXEC_IMPORTPATH=%q: %w\", archiveImportPath, err)\n\t\t\t}\n\t\t\tbreak\n\t\t}\n\t\tdeps, err := resolvePackageFiles(ctx, linkDepPath, \".\")\n\t\tif hadImportPath {\n\t\t\t_ = os.Setenv(\"TOOLEXEC_IMPORTPATH\", prevImportPath)\n\t\t} else {\n\t\t\t_ = os.Unsetenv(\"TOOLEXEC_IMPORTPATH\")\n\t\t}\n""" + oncompile_main_old = """\t\tdeps, err := resolvePackageFilesForTest(ctx, item.path, testVariantFor, cmd.WorkDir)\n""" + oncompile_main_new = """\t\tprevImportPath, hadImportPath := os.LookupEnv(\"TOOLEXEC_IMPORTPATH\")\n\t\tresolverImportPath := item.parent\n\t\tif resolverImportPath == \"\" {\n\t\t\tresolverImportPath = \"main\"\n\t\t}\n\t\tif err := os.Setenv(\"TOOLEXEC_IMPORTPATH\", resolverImportPath); err != nil {\n\t\t\treturn fmt.Errorf(\"setting TOOLEXEC_IMPORTPATH=%q: %w\", resolverImportPath, err)\n\t\t}\n\t\tdeps, err := resolvePackageFilesForTest(ctx, item.path, testVariantFor, \".\")\n\t\tif hadImportPath {\n\t\t\t_ = os.Setenv(\"TOOLEXEC_IMPORTPATH\", prevImportPath)\n\t\t} else {\n\t\t\t_ = os.Unsetenv(\"TOOLEXEC_IMPORTPATH\")\n\t\t}\n""" if oncompile_main_old not in oncompile_main_src: fail("Could not patch Orchestrion oncompile-main resolver context in %s" % oncompile_main_path) ctx.file(oncompile_main_path, oncompile_main_src.replace(oncompile_main_old, oncompile_main_new, 1)) diff --git a/third_party/rgo/v0_60_0/base/go/tools/builders/orchestrion_version.go b/third_party/rgo/v0_60_0/base/go/tools/builders/orchestrion_version.go index d59f9b42..0469ab53 100644 --- a/third_party/rgo/v0_60_0/base/go/tools/builders/orchestrion_version.go +++ b/third_party/rgo/v0_60_0/base/go/tools/builders/orchestrion_version.go @@ -13,7 +13,7 @@ import ( const ( rulesGoOrchestrionVersionFileEnvVar = "RULES_GO_ORCHESTRION_VERSION_FILE" - defaultDDTraceGoVersion = "v2.9.0" + defaultDDTraceGoVersion = "v2.9.1" orchestrionSyntheticGoModVersion = "1.21" ) diff --git a/third_party/rgo/v0_61_1/base/go/private/orchestrion/extensions.bzl b/third_party/rgo/v0_61_1/base/go/private/orchestrion/extensions.bzl index db1f149f..a56e5681 100644 --- a/third_party/rgo/v0_61_1/base/go/private/orchestrion/extensions.bzl +++ b/third_party/rgo/v0_61_1/base/go/private/orchestrion/extensions.bzl @@ -14,7 +14,7 @@ """Module extension for configuring orchestrion in rules_go.""" -DEFAULT_DD_TRACE_GO_VERSION = "v2.9.0" +DEFAULT_DD_TRACE_GO_VERSION = "v2.9.1" ORCHESTRION_BOOTSTRAP_CACHE_ABI = "v3" ORCHESTRION_SEED_GO_MOD_VERSION = "1.21" @@ -1372,34 +1372,21 @@ def _orchestrion_build_impl(ctx): ctx.report_progress("rules_go_orchestrion: patching orchestrion source") patch_start_ms = _probe_now_ms(ctx) - # The upstream package resolver recursively re-runs `go list` under - # `-toolexec=orchestrion toolexec`, which causes woven dependency lookups to - # fail under Bazel's sandbox even when plain `go list -mod=mod` succeeds in - # the same environment. Mirror the safer non-recursive pattern used by other - # loader paths in Orchestrion and disable toolexec for dependency resolution. + # Keep Orchestrion's recursive toolexec resolver: v1.12 uses it to build the + # correct dependency variants for generated test mains. Only adapt its + # temporary directory to Bazel's execroot layout. resolve_path = "internal/jobserver/pkgs/resolve.go" resolve_src = ctx.read(resolve_path) - resolve_old = """\n\t\tbuildFlags := append(\n\t\t\tgoFlags.Slice(),\n\t\t\tfmt.Sprintf(\"-toolexec=%q toolexec\", binpath.Orchestrion),\n\t\t)\n""" - resolve_new = """\n\t\tbuildFlags := append(goFlags.Slice(), \"-toolexec=\")\n""" - if resolve_old not in resolve_src: - fail("Could not patch Orchestrion resolver in %s" % resolve_path) - resolve_src = resolve_src.replace(resolve_old, resolve_new) - resolve_src = resolve_src.replace('\n\t"github.com/DataDog/orchestrion/internal/binpath"', "") - resolve_imports_old = """\t\"fmt\"\n\t\"os\"\n\t\"slices\"\n""" - resolve_imports_new = """\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"slices\"\n""" + resolve_imports_old = """\t\"maps\"\n\t\"os\"\n\t\"slices\"\n""" + resolve_imports_new = """\t\"maps\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"slices\"\n""" if resolve_imports_old not in resolve_src: fail("Could not patch Orchestrion resolver imports in %s" % resolve_path) resolve_src = resolve_src.replace(resolve_imports_old, resolve_imports_new, 1) - resolve_tempdir_old = """\t\tif req.TempDir != \"\" {\n\t\t\t// Make sure the directory exists (go blindly assumes that...)\n\t\t\tif err := os.MkdirAll(req.TempDir, 0o755); err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"creating temporary directory %q: %w\", req.TempDir, err)\n\t\t\t}\n\t\t\tenv = append(env, fmt.Sprintf(\"%s=%s\", envVarGotmpdir, req.TempDir))\n\t\t}\n""" - resolve_tempdir_new = """\t\tif req.TempDir != \"\" {\n\t\t\tabsTempDir, absErr := filepath.Abs(req.TempDir)\n\t\t\tif absErr != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"absolutizing temporary directory %q: %w\", req.TempDir, absErr)\n\t\t\t}\n\t\t\t// Make sure the directory exists (go blindly assumes that...)\n\t\t\tif err := os.MkdirAll(absTempDir, 0o755); err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"creating temporary directory %q: %w\", absTempDir, err)\n\t\t\t}\n\t\t\tfor _, name := range []string{\"external\", \"bazel-out\"} {\n\t\t\t\tsrcPath := filepath.Join(req.Dir, name)\n\t\t\t\tif !filepath.IsAbs(srcPath) {\n\t\t\t\t\tif srcPath, absErr = filepath.Abs(srcPath); absErr != nil {\n\t\t\t\t\t\treturn nil, fmt.Errorf(\"absolutizing compatibility path %q: %w\", srcPath, absErr)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif _, statErr := os.Stat(srcPath); statErr != nil {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tdstPath := filepath.Join(absTempDir, name)\n\t\t\t\tif _, statErr := os.Lstat(dstPath); statErr == nil {\n\t\t\t\t\tcontinue\n\t\t\t\t} else if !os.IsNotExist(statErr) {\n\t\t\t\t\treturn nil, fmt.Errorf(\"stat temporary compatibility path %q: %w\", dstPath, statErr)\n\t\t\t\t}\n\t\t\t\tlinkTarget, relErr := filepath.Rel(absTempDir, srcPath)\n\t\t\t\tif relErr != nil {\n\t\t\t\t\tif linkTarget, absErr = filepath.Abs(srcPath); absErr != nil {\n\t\t\t\t\t\treturn nil, fmt.Errorf(\"compute temporary compatibility path for %q: %w\", name, relErr)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif linkErr := os.Symlink(linkTarget, dstPath); linkErr != nil {\n\t\t\t\t\treturn nil, fmt.Errorf(\"create temporary compatibility symlink %q -> %q: %w\", dstPath, linkTarget, linkErr)\n\t\t\t\t}\n\t\t\t\tif log.GetLevel() <= zerolog.TraceLevel {\n\t\t\t\t\tlog.Trace().Str(\"dst\", dstPath).Str(\"target\", linkTarget).Msg(\"pkgs.Resolve created temp compatibility symlink\")\n\t\t\t\t}\n\t\t\t}\n\t\t\tenv = append(env, fmt.Sprintf(\"%s=%s\", envVarGotmpdir, absTempDir))\n\t\t}\n""" + resolve_tempdir_old = """\tif req.TempDir != \"\" {\n\t\tif err := os.MkdirAll(req.TempDir, 0o755); err != nil {\n\t\t\treturn resolvedPackageSet{}, fmt.Errorf(\"creating temporary directory %q: %w\", req.TempDir, err)\n\t\t}\n\t}\n\tenv := resolveEnvironment(ctx, req)\n""" + resolve_tempdir_new = """\tif req.TempDir != \"\" {\n\t\tabsTempDir, absErr := filepath.Abs(req.TempDir)\n\t\tif absErr != nil {\n\t\t\treturn resolvedPackageSet{}, fmt.Errorf(\"absolutizing temporary directory %q: %w\", req.TempDir, absErr)\n\t\t}\n\t\tif err := os.MkdirAll(absTempDir, 0o755); err != nil {\n\t\t\treturn resolvedPackageSet{}, fmt.Errorf(\"creating temporary directory %q: %w\", absTempDir, err)\n\t\t}\n\t\tfor _, name := range []string{\"external\", \"bazel-out\"} {\n\t\t\tsrcPath := filepath.Join(req.Dir, name)\n\t\t\tif !filepath.IsAbs(srcPath) {\n\t\t\t\tif srcPath, absErr = filepath.Abs(srcPath); absErr != nil {\n\t\t\t\t\treturn resolvedPackageSet{}, fmt.Errorf(\"absolutizing compatibility path %q: %w\", srcPath, absErr)\n\t\t\t\t}\n\t\t\t}\n\t\t\tif _, statErr := os.Stat(srcPath); statErr != nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tdstPath := filepath.Join(absTempDir, name)\n\t\t\tif _, statErr := os.Lstat(dstPath); statErr == nil {\n\t\t\t\tcontinue\n\t\t\t} else if !os.IsNotExist(statErr) {\n\t\t\t\treturn resolvedPackageSet{}, fmt.Errorf(\"stat temporary compatibility path %q: %w\", dstPath, statErr)\n\t\t\t}\n\t\t\tlinkTarget, relErr := filepath.Rel(absTempDir, srcPath)\n\t\t\tif relErr != nil {\n\t\t\t\tlinkTarget = srcPath\n\t\t\t}\n\t\t\tif linkErr := os.Symlink(linkTarget, dstPath); linkErr != nil {\n\t\t\t\treturn resolvedPackageSet{}, fmt.Errorf(\"create temporary compatibility symlink %q -> %q: %w\", dstPath, linkTarget, linkErr)\n\t\t\t}\n\t\t\tlog.Trace().Str(\"dst\", dstPath).Str(\"target\", linkTarget).Msg(\"pkgs.Resolve created temp compatibility symlink\")\n\t\t}\n\t\treq.TempDir = absTempDir\n\t}\n\tenv := resolveEnvironment(ctx, req)\n""" if resolve_tempdir_old not in resolve_src: fail("Could not patch Orchestrion resolver tempdir block in %s" % resolve_path) resolve_src = resolve_src.replace(resolve_tempdir_old, resolve_tempdir_new, 1) - resolve_merge_old = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error {\n\tif pkg.PkgPath == \"\" || pkg.PkgPath == \"unsafe\" || r[pkg.PkgPath] != \"\" {\n\t\t// Ignore the \"unsafe\" package (no archive file, ever), packages with an empty import path\n\t\t// (standard library), and those already present in the map (already processed previously).\n\t\treturn nil\n\t}\n""" - resolve_merge_new = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error {\n\tif pkg.PkgPath == \"\" || pkg.PkgPath == \"unsafe\" || r[pkg.PkgPath] != \"\" {\n\t\t// Ignore the \"unsafe\" package (no archive file, ever), packages with an empty import path\n\t\t// (standard library), and those already present in the map (already processed previously).\n\t\treturn nil\n\t}\n""" - if resolve_merge_old not in resolve_src: - fail("Could not patch Orchestrion resolver mergeFrom in %s" % resolve_path) - resolve_src = resolve_src.replace(resolve_merge_old, resolve_merge_new, 1) ctx.file(resolve_path, resolve_src) # Compile-proxy / archive metadata patches for the synthetic testmain flow. @@ -1460,16 +1447,16 @@ def _orchestrion_build_impl(ctx): # run under Bazel's import-path context instead of synthetic package names. onlink_path = "internal/toolexec/aspect/onlink.go" onlink_src = ctx.read(onlink_path) - onlink_old = """\t\t\tdeps, err := resolvePackageFiles(ctx, depPath, cmd.WorkDir)\n""" - onlink_new = """\t\t\tprevImportPath, hadImportPath := os.LookupEnv(\"TOOLEXEC_IMPORTPATH\")\n\t\t\tif err := os.Setenv(\"TOOLEXEC_IMPORTPATH\", w.ImportPath); err != nil {\n\t\t\t\treturn fmt.Errorf(\"setting TOOLEXEC_IMPORTPATH=%q: %w\", w.ImportPath, err)\n\t\t\t}\n\t\t\tdeps, err := resolvePackageFiles(ctx, depPath, \".\")\n\t\t\tif hadImportPath {\n\t\t\t\t_ = os.Setenv(\"TOOLEXEC_IMPORTPATH\", prevImportPath)\n\t\t\t} else {\n\t\t\t\t_ = os.Unsetenv(\"TOOLEXEC_IMPORTPATH\")\n\t\t\t}\n""" + onlink_old = """\t\t\tdeps, err := resolvePackageFilesForTest(ctx, depPath, testVariantFor, cmd.WorkDir)\n""" + onlink_new = """\t\t\tprevImportPath, hadImportPath := os.LookupEnv(\"TOOLEXEC_IMPORTPATH\")\n\t\t\tif err := os.Setenv(\"TOOLEXEC_IMPORTPATH\", w.ImportPath); err != nil {\n\t\t\t\treturn fmt.Errorf(\"setting TOOLEXEC_IMPORTPATH=%q: %w\", w.ImportPath, err)\n\t\t\t}\n\t\t\tdeps, err := resolvePackageFilesForTest(ctx, depPath, testVariantFor, \".\")\n\t\t\tif hadImportPath {\n\t\t\t\t_ = os.Setenv(\"TOOLEXEC_IMPORTPATH\", prevImportPath)\n\t\t\t} else {\n\t\t\t\t_ = os.Unsetenv(\"TOOLEXEC_IMPORTPATH\")\n\t\t\t}\n""" if onlink_old not in onlink_src: fail("Could not patch Orchestrion onlink resolver context in %s" % onlink_path) ctx.file(onlink_path, onlink_src.replace(onlink_old, onlink_new, 1)) oncompile_main_path = "internal/toolexec/aspect/oncompile-main.go" oncompile_main_src = ctx.read(oncompile_main_path) - oncompile_main_old = """\t\tdeps, err := resolvePackageFiles(ctx, linkDepPath, cmd.WorkDir)\n""" - oncompile_main_new = """\t\tprevImportPath, hadImportPath := os.LookupEnv(\"TOOLEXEC_IMPORTPATH\")\n\t\tif err := os.Setenv(\"TOOLEXEC_IMPORTPATH\", \"main\"); err != nil {\n\t\t\treturn fmt.Errorf(\"setting TOOLEXEC_IMPORTPATH for synthetic main: %w\", err)\n\t\t}\n\t\tfor archiveImportPath, archivePath := range reg.PackageFile {\n\t\t\tld, ldErr := linkdeps.FromArchive(ctx, archivePath)\n\t\t\tif ldErr != nil || !ld.Contains(linkDepPath) {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif err := os.Setenv(\"TOOLEXEC_IMPORTPATH\", archiveImportPath); err != nil {\n\t\t\t\treturn fmt.Errorf(\"setting TOOLEXEC_IMPORTPATH=%q: %w\", archiveImportPath, err)\n\t\t\t}\n\t\t\tbreak\n\t\t}\n\t\tdeps, err := resolvePackageFiles(ctx, linkDepPath, \".\")\n\t\tif hadImportPath {\n\t\t\t_ = os.Setenv(\"TOOLEXEC_IMPORTPATH\", prevImportPath)\n\t\t} else {\n\t\t\t_ = os.Unsetenv(\"TOOLEXEC_IMPORTPATH\")\n\t\t}\n""" + oncompile_main_old = """\t\tdeps, err := resolvePackageFilesForTest(ctx, item.path, testVariantFor, cmd.WorkDir)\n""" + oncompile_main_new = """\t\tprevImportPath, hadImportPath := os.LookupEnv(\"TOOLEXEC_IMPORTPATH\")\n\t\tresolverImportPath := item.parent\n\t\tif resolverImportPath == \"\" {\n\t\t\tresolverImportPath = \"main\"\n\t\t}\n\t\tif err := os.Setenv(\"TOOLEXEC_IMPORTPATH\", resolverImportPath); err != nil {\n\t\t\treturn fmt.Errorf(\"setting TOOLEXEC_IMPORTPATH=%q: %w\", resolverImportPath, err)\n\t\t}\n\t\tdeps, err := resolvePackageFilesForTest(ctx, item.path, testVariantFor, \".\")\n\t\tif hadImportPath {\n\t\t\t_ = os.Setenv(\"TOOLEXEC_IMPORTPATH\", prevImportPath)\n\t\t} else {\n\t\t\t_ = os.Unsetenv(\"TOOLEXEC_IMPORTPATH\")\n\t\t}\n""" if oncompile_main_old not in oncompile_main_src: fail("Could not patch Orchestrion oncompile-main resolver context in %s" % oncompile_main_path) ctx.file(oncompile_main_path, oncompile_main_src.replace(oncompile_main_old, oncompile_main_new, 1)) diff --git a/third_party/rgo/v0_61_1/base/go/tools/builders/orchestrion_version.go b/third_party/rgo/v0_61_1/base/go/tools/builders/orchestrion_version.go index d59f9b42..0469ab53 100644 --- a/third_party/rgo/v0_61_1/base/go/tools/builders/orchestrion_version.go +++ b/third_party/rgo/v0_61_1/base/go/tools/builders/orchestrion_version.go @@ -13,7 +13,7 @@ import ( const ( rulesGoOrchestrionVersionFileEnvVar = "RULES_GO_ORCHESTRION_VERSION_FILE" - defaultDDTraceGoVersion = "v2.9.0" + defaultDDTraceGoVersion = "v2.9.1" orchestrionSyntheticGoModVersion = "1.21" ) diff --git a/third_party/rgo/v0_62_0/base/go/private/orchestrion/extensions.bzl b/third_party/rgo/v0_62_0/base/go/private/orchestrion/extensions.bzl index db1f149f..a56e5681 100644 --- a/third_party/rgo/v0_62_0/base/go/private/orchestrion/extensions.bzl +++ b/third_party/rgo/v0_62_0/base/go/private/orchestrion/extensions.bzl @@ -14,7 +14,7 @@ """Module extension for configuring orchestrion in rules_go.""" -DEFAULT_DD_TRACE_GO_VERSION = "v2.9.0" +DEFAULT_DD_TRACE_GO_VERSION = "v2.9.1" ORCHESTRION_BOOTSTRAP_CACHE_ABI = "v3" ORCHESTRION_SEED_GO_MOD_VERSION = "1.21" @@ -1372,34 +1372,21 @@ def _orchestrion_build_impl(ctx): ctx.report_progress("rules_go_orchestrion: patching orchestrion source") patch_start_ms = _probe_now_ms(ctx) - # The upstream package resolver recursively re-runs `go list` under - # `-toolexec=orchestrion toolexec`, which causes woven dependency lookups to - # fail under Bazel's sandbox even when plain `go list -mod=mod` succeeds in - # the same environment. Mirror the safer non-recursive pattern used by other - # loader paths in Orchestrion and disable toolexec for dependency resolution. + # Keep Orchestrion's recursive toolexec resolver: v1.12 uses it to build the + # correct dependency variants for generated test mains. Only adapt its + # temporary directory to Bazel's execroot layout. resolve_path = "internal/jobserver/pkgs/resolve.go" resolve_src = ctx.read(resolve_path) - resolve_old = """\n\t\tbuildFlags := append(\n\t\t\tgoFlags.Slice(),\n\t\t\tfmt.Sprintf(\"-toolexec=%q toolexec\", binpath.Orchestrion),\n\t\t)\n""" - resolve_new = """\n\t\tbuildFlags := append(goFlags.Slice(), \"-toolexec=\")\n""" - if resolve_old not in resolve_src: - fail("Could not patch Orchestrion resolver in %s" % resolve_path) - resolve_src = resolve_src.replace(resolve_old, resolve_new) - resolve_src = resolve_src.replace('\n\t"github.com/DataDog/orchestrion/internal/binpath"', "") - resolve_imports_old = """\t\"fmt\"\n\t\"os\"\n\t\"slices\"\n""" - resolve_imports_new = """\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"slices\"\n""" + resolve_imports_old = """\t\"maps\"\n\t\"os\"\n\t\"slices\"\n""" + resolve_imports_new = """\t\"maps\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"slices\"\n""" if resolve_imports_old not in resolve_src: fail("Could not patch Orchestrion resolver imports in %s" % resolve_path) resolve_src = resolve_src.replace(resolve_imports_old, resolve_imports_new, 1) - resolve_tempdir_old = """\t\tif req.TempDir != \"\" {\n\t\t\t// Make sure the directory exists (go blindly assumes that...)\n\t\t\tif err := os.MkdirAll(req.TempDir, 0o755); err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"creating temporary directory %q: %w\", req.TempDir, err)\n\t\t\t}\n\t\t\tenv = append(env, fmt.Sprintf(\"%s=%s\", envVarGotmpdir, req.TempDir))\n\t\t}\n""" - resolve_tempdir_new = """\t\tif req.TempDir != \"\" {\n\t\t\tabsTempDir, absErr := filepath.Abs(req.TempDir)\n\t\t\tif absErr != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"absolutizing temporary directory %q: %w\", req.TempDir, absErr)\n\t\t\t}\n\t\t\t// Make sure the directory exists (go blindly assumes that...)\n\t\t\tif err := os.MkdirAll(absTempDir, 0o755); err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"creating temporary directory %q: %w\", absTempDir, err)\n\t\t\t}\n\t\t\tfor _, name := range []string{\"external\", \"bazel-out\"} {\n\t\t\t\tsrcPath := filepath.Join(req.Dir, name)\n\t\t\t\tif !filepath.IsAbs(srcPath) {\n\t\t\t\t\tif srcPath, absErr = filepath.Abs(srcPath); absErr != nil {\n\t\t\t\t\t\treturn nil, fmt.Errorf(\"absolutizing compatibility path %q: %w\", srcPath, absErr)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif _, statErr := os.Stat(srcPath); statErr != nil {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tdstPath := filepath.Join(absTempDir, name)\n\t\t\t\tif _, statErr := os.Lstat(dstPath); statErr == nil {\n\t\t\t\t\tcontinue\n\t\t\t\t} else if !os.IsNotExist(statErr) {\n\t\t\t\t\treturn nil, fmt.Errorf(\"stat temporary compatibility path %q: %w\", dstPath, statErr)\n\t\t\t\t}\n\t\t\t\tlinkTarget, relErr := filepath.Rel(absTempDir, srcPath)\n\t\t\t\tif relErr != nil {\n\t\t\t\t\tif linkTarget, absErr = filepath.Abs(srcPath); absErr != nil {\n\t\t\t\t\t\treturn nil, fmt.Errorf(\"compute temporary compatibility path for %q: %w\", name, relErr)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif linkErr := os.Symlink(linkTarget, dstPath); linkErr != nil {\n\t\t\t\t\treturn nil, fmt.Errorf(\"create temporary compatibility symlink %q -> %q: %w\", dstPath, linkTarget, linkErr)\n\t\t\t\t}\n\t\t\t\tif log.GetLevel() <= zerolog.TraceLevel {\n\t\t\t\t\tlog.Trace().Str(\"dst\", dstPath).Str(\"target\", linkTarget).Msg(\"pkgs.Resolve created temp compatibility symlink\")\n\t\t\t\t}\n\t\t\t}\n\t\t\tenv = append(env, fmt.Sprintf(\"%s=%s\", envVarGotmpdir, absTempDir))\n\t\t}\n""" + resolve_tempdir_old = """\tif req.TempDir != \"\" {\n\t\tif err := os.MkdirAll(req.TempDir, 0o755); err != nil {\n\t\t\treturn resolvedPackageSet{}, fmt.Errorf(\"creating temporary directory %q: %w\", req.TempDir, err)\n\t\t}\n\t}\n\tenv := resolveEnvironment(ctx, req)\n""" + resolve_tempdir_new = """\tif req.TempDir != \"\" {\n\t\tabsTempDir, absErr := filepath.Abs(req.TempDir)\n\t\tif absErr != nil {\n\t\t\treturn resolvedPackageSet{}, fmt.Errorf(\"absolutizing temporary directory %q: %w\", req.TempDir, absErr)\n\t\t}\n\t\tif err := os.MkdirAll(absTempDir, 0o755); err != nil {\n\t\t\treturn resolvedPackageSet{}, fmt.Errorf(\"creating temporary directory %q: %w\", absTempDir, err)\n\t\t}\n\t\tfor _, name := range []string{\"external\", \"bazel-out\"} {\n\t\t\tsrcPath := filepath.Join(req.Dir, name)\n\t\t\tif !filepath.IsAbs(srcPath) {\n\t\t\t\tif srcPath, absErr = filepath.Abs(srcPath); absErr != nil {\n\t\t\t\t\treturn resolvedPackageSet{}, fmt.Errorf(\"absolutizing compatibility path %q: %w\", srcPath, absErr)\n\t\t\t\t}\n\t\t\t}\n\t\t\tif _, statErr := os.Stat(srcPath); statErr != nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tdstPath := filepath.Join(absTempDir, name)\n\t\t\tif _, statErr := os.Lstat(dstPath); statErr == nil {\n\t\t\t\tcontinue\n\t\t\t} else if !os.IsNotExist(statErr) {\n\t\t\t\treturn resolvedPackageSet{}, fmt.Errorf(\"stat temporary compatibility path %q: %w\", dstPath, statErr)\n\t\t\t}\n\t\t\tlinkTarget, relErr := filepath.Rel(absTempDir, srcPath)\n\t\t\tif relErr != nil {\n\t\t\t\tlinkTarget = srcPath\n\t\t\t}\n\t\t\tif linkErr := os.Symlink(linkTarget, dstPath); linkErr != nil {\n\t\t\t\treturn resolvedPackageSet{}, fmt.Errorf(\"create temporary compatibility symlink %q -> %q: %w\", dstPath, linkTarget, linkErr)\n\t\t\t}\n\t\t\tlog.Trace().Str(\"dst\", dstPath).Str(\"target\", linkTarget).Msg(\"pkgs.Resolve created temp compatibility symlink\")\n\t\t}\n\t\treq.TempDir = absTempDir\n\t}\n\tenv := resolveEnvironment(ctx, req)\n""" if resolve_tempdir_old not in resolve_src: fail("Could not patch Orchestrion resolver tempdir block in %s" % resolve_path) resolve_src = resolve_src.replace(resolve_tempdir_old, resolve_tempdir_new, 1) - resolve_merge_old = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error {\n\tif pkg.PkgPath == \"\" || pkg.PkgPath == \"unsafe\" || r[pkg.PkgPath] != \"\" {\n\t\t// Ignore the \"unsafe\" package (no archive file, ever), packages with an empty import path\n\t\t// (standard library), and those already present in the map (already processed previously).\n\t\treturn nil\n\t}\n""" - resolve_merge_new = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error {\n\tif pkg.PkgPath == \"\" || pkg.PkgPath == \"unsafe\" || r[pkg.PkgPath] != \"\" {\n\t\t// Ignore the \"unsafe\" package (no archive file, ever), packages with an empty import path\n\t\t// (standard library), and those already present in the map (already processed previously).\n\t\treturn nil\n\t}\n""" - if resolve_merge_old not in resolve_src: - fail("Could not patch Orchestrion resolver mergeFrom in %s" % resolve_path) - resolve_src = resolve_src.replace(resolve_merge_old, resolve_merge_new, 1) ctx.file(resolve_path, resolve_src) # Compile-proxy / archive metadata patches for the synthetic testmain flow. @@ -1460,16 +1447,16 @@ def _orchestrion_build_impl(ctx): # run under Bazel's import-path context instead of synthetic package names. onlink_path = "internal/toolexec/aspect/onlink.go" onlink_src = ctx.read(onlink_path) - onlink_old = """\t\t\tdeps, err := resolvePackageFiles(ctx, depPath, cmd.WorkDir)\n""" - onlink_new = """\t\t\tprevImportPath, hadImportPath := os.LookupEnv(\"TOOLEXEC_IMPORTPATH\")\n\t\t\tif err := os.Setenv(\"TOOLEXEC_IMPORTPATH\", w.ImportPath); err != nil {\n\t\t\t\treturn fmt.Errorf(\"setting TOOLEXEC_IMPORTPATH=%q: %w\", w.ImportPath, err)\n\t\t\t}\n\t\t\tdeps, err := resolvePackageFiles(ctx, depPath, \".\")\n\t\t\tif hadImportPath {\n\t\t\t\t_ = os.Setenv(\"TOOLEXEC_IMPORTPATH\", prevImportPath)\n\t\t\t} else {\n\t\t\t\t_ = os.Unsetenv(\"TOOLEXEC_IMPORTPATH\")\n\t\t\t}\n""" + onlink_old = """\t\t\tdeps, err := resolvePackageFilesForTest(ctx, depPath, testVariantFor, cmd.WorkDir)\n""" + onlink_new = """\t\t\tprevImportPath, hadImportPath := os.LookupEnv(\"TOOLEXEC_IMPORTPATH\")\n\t\t\tif err := os.Setenv(\"TOOLEXEC_IMPORTPATH\", w.ImportPath); err != nil {\n\t\t\t\treturn fmt.Errorf(\"setting TOOLEXEC_IMPORTPATH=%q: %w\", w.ImportPath, err)\n\t\t\t}\n\t\t\tdeps, err := resolvePackageFilesForTest(ctx, depPath, testVariantFor, \".\")\n\t\t\tif hadImportPath {\n\t\t\t\t_ = os.Setenv(\"TOOLEXEC_IMPORTPATH\", prevImportPath)\n\t\t\t} else {\n\t\t\t\t_ = os.Unsetenv(\"TOOLEXEC_IMPORTPATH\")\n\t\t\t}\n""" if onlink_old not in onlink_src: fail("Could not patch Orchestrion onlink resolver context in %s" % onlink_path) ctx.file(onlink_path, onlink_src.replace(onlink_old, onlink_new, 1)) oncompile_main_path = "internal/toolexec/aspect/oncompile-main.go" oncompile_main_src = ctx.read(oncompile_main_path) - oncompile_main_old = """\t\tdeps, err := resolvePackageFiles(ctx, linkDepPath, cmd.WorkDir)\n""" - oncompile_main_new = """\t\tprevImportPath, hadImportPath := os.LookupEnv(\"TOOLEXEC_IMPORTPATH\")\n\t\tif err := os.Setenv(\"TOOLEXEC_IMPORTPATH\", \"main\"); err != nil {\n\t\t\treturn fmt.Errorf(\"setting TOOLEXEC_IMPORTPATH for synthetic main: %w\", err)\n\t\t}\n\t\tfor archiveImportPath, archivePath := range reg.PackageFile {\n\t\t\tld, ldErr := linkdeps.FromArchive(ctx, archivePath)\n\t\t\tif ldErr != nil || !ld.Contains(linkDepPath) {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif err := os.Setenv(\"TOOLEXEC_IMPORTPATH\", archiveImportPath); err != nil {\n\t\t\t\treturn fmt.Errorf(\"setting TOOLEXEC_IMPORTPATH=%q: %w\", archiveImportPath, err)\n\t\t\t}\n\t\t\tbreak\n\t\t}\n\t\tdeps, err := resolvePackageFiles(ctx, linkDepPath, \".\")\n\t\tif hadImportPath {\n\t\t\t_ = os.Setenv(\"TOOLEXEC_IMPORTPATH\", prevImportPath)\n\t\t} else {\n\t\t\t_ = os.Unsetenv(\"TOOLEXEC_IMPORTPATH\")\n\t\t}\n""" + oncompile_main_old = """\t\tdeps, err := resolvePackageFilesForTest(ctx, item.path, testVariantFor, cmd.WorkDir)\n""" + oncompile_main_new = """\t\tprevImportPath, hadImportPath := os.LookupEnv(\"TOOLEXEC_IMPORTPATH\")\n\t\tresolverImportPath := item.parent\n\t\tif resolverImportPath == \"\" {\n\t\t\tresolverImportPath = \"main\"\n\t\t}\n\t\tif err := os.Setenv(\"TOOLEXEC_IMPORTPATH\", resolverImportPath); err != nil {\n\t\t\treturn fmt.Errorf(\"setting TOOLEXEC_IMPORTPATH=%q: %w\", resolverImportPath, err)\n\t\t}\n\t\tdeps, err := resolvePackageFilesForTest(ctx, item.path, testVariantFor, \".\")\n\t\tif hadImportPath {\n\t\t\t_ = os.Setenv(\"TOOLEXEC_IMPORTPATH\", prevImportPath)\n\t\t} else {\n\t\t\t_ = os.Unsetenv(\"TOOLEXEC_IMPORTPATH\")\n\t\t}\n""" if oncompile_main_old not in oncompile_main_src: fail("Could not patch Orchestrion oncompile-main resolver context in %s" % oncompile_main_path) ctx.file(oncompile_main_path, oncompile_main_src.replace(oncompile_main_old, oncompile_main_new, 1)) diff --git a/third_party/rgo/v0_62_0/base/go/tools/builders/orchestrion_version.go b/third_party/rgo/v0_62_0/base/go/tools/builders/orchestrion_version.go index d59f9b42..0469ab53 100644 --- a/third_party/rgo/v0_62_0/base/go/tools/builders/orchestrion_version.go +++ b/third_party/rgo/v0_62_0/base/go/tools/builders/orchestrion_version.go @@ -13,7 +13,7 @@ import ( const ( rulesGoOrchestrionVersionFileEnvVar = "RULES_GO_ORCHESTRION_VERSION_FILE" - defaultDDTraceGoVersion = "v2.9.0" + defaultDDTraceGoVersion = "v2.9.1" orchestrionSyntheticGoModVersion = "1.21" ) diff --git a/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch index 613a47fa..f29b0d00 100644 --- a/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch @@ -2091,10 +2091,10 @@ index 0000000..31a01c3 +) diff --git a/go/private/orchestrion/extensions.bzl b/go/private/orchestrion/extensions.bzl new file mode 100644 -index 0000000..db1f149 +index 0000000..a56e568 --- /dev/null +++ b/go/private/orchestrion/extensions.bzl -@@ -0,0 +1,1732 @@ +@@ -0,0 +1,1719 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); @@ -2111,7 +2111,7 @@ index 0000000..db1f149 + +"""Module extension for configuring orchestrion in rules_go.""" + -+DEFAULT_DD_TRACE_GO_VERSION = "v2.9.0" ++DEFAULT_DD_TRACE_GO_VERSION = "v2.9.1" +ORCHESTRION_BOOTSTRAP_CACHE_ABI = "v3" +ORCHESTRION_SEED_GO_MOD_VERSION = "1.21" + @@ -3469,34 +3469,21 @@ index 0000000..db1f149 + ctx.report_progress("rules_go_orchestrion: patching orchestrion source") + patch_start_ms = _probe_now_ms(ctx) + -+ # The upstream package resolver recursively re-runs `go list` under -+ # `-toolexec=orchestrion toolexec`, which causes woven dependency lookups to -+ # fail under Bazel's sandbox even when plain `go list -mod=mod` succeeds in -+ # the same environment. Mirror the safer non-recursive pattern used by other -+ # loader paths in Orchestrion and disable toolexec for dependency resolution. ++ # Keep Orchestrion's recursive toolexec resolver: v1.12 uses it to build the ++ # correct dependency variants for generated test mains. Only adapt its ++ # temporary directory to Bazel's execroot layout. + resolve_path = "internal/jobserver/pkgs/resolve.go" + resolve_src = ctx.read(resolve_path) -+ resolve_old = """\n\t\tbuildFlags := append(\n\t\t\tgoFlags.Slice(),\n\t\t\tfmt.Sprintf(\"-toolexec=%q toolexec\", binpath.Orchestrion),\n\t\t)\n""" -+ resolve_new = """\n\t\tbuildFlags := append(goFlags.Slice(), \"-toolexec=\")\n""" -+ if resolve_old not in resolve_src: -+ fail("Could not patch Orchestrion resolver in %s" % resolve_path) -+ resolve_src = resolve_src.replace(resolve_old, resolve_new) -+ resolve_src = resolve_src.replace('\n\t"github.com/DataDog/orchestrion/internal/binpath"', "") -+ resolve_imports_old = """\t\"fmt\"\n\t\"os\"\n\t\"slices\"\n""" -+ resolve_imports_new = """\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"slices\"\n""" ++ resolve_imports_old = """\t\"maps\"\n\t\"os\"\n\t\"slices\"\n""" ++ resolve_imports_new = """\t\"maps\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"slices\"\n""" + if resolve_imports_old not in resolve_src: + fail("Could not patch Orchestrion resolver imports in %s" % resolve_path) + resolve_src = resolve_src.replace(resolve_imports_old, resolve_imports_new, 1) -+ resolve_tempdir_old = """\t\tif req.TempDir != \"\" {\n\t\t\t// Make sure the directory exists (go blindly assumes that...)\n\t\t\tif err := os.MkdirAll(req.TempDir, 0o755); err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"creating temporary directory %q: %w\", req.TempDir, err)\n\t\t\t}\n\t\t\tenv = append(env, fmt.Sprintf(\"%s=%s\", envVarGotmpdir, req.TempDir))\n\t\t}\n""" -+ resolve_tempdir_new = """\t\tif req.TempDir != \"\" {\n\t\t\tabsTempDir, absErr := filepath.Abs(req.TempDir)\n\t\t\tif absErr != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"absolutizing temporary directory %q: %w\", req.TempDir, absErr)\n\t\t\t}\n\t\t\t// Make sure the directory exists (go blindly assumes that...)\n\t\t\tif err := os.MkdirAll(absTempDir, 0o755); err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"creating temporary directory %q: %w\", absTempDir, err)\n\t\t\t}\n\t\t\tfor _, name := range []string{\"external\", \"bazel-out\"} {\n\t\t\t\tsrcPath := filepath.Join(req.Dir, name)\n\t\t\t\tif !filepath.IsAbs(srcPath) {\n\t\t\t\t\tif srcPath, absErr = filepath.Abs(srcPath); absErr != nil {\n\t\t\t\t\t\treturn nil, fmt.Errorf(\"absolutizing compatibility path %q: %w\", srcPath, absErr)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif _, statErr := os.Stat(srcPath); statErr != nil {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tdstPath := filepath.Join(absTempDir, name)\n\t\t\t\tif _, statErr := os.Lstat(dstPath); statErr == nil {\n\t\t\t\t\tcontinue\n\t\t\t\t} else if !os.IsNotExist(statErr) {\n\t\t\t\t\treturn nil, fmt.Errorf(\"stat temporary compatibility path %q: %w\", dstPath, statErr)\n\t\t\t\t}\n\t\t\t\tlinkTarget, relErr := filepath.Rel(absTempDir, srcPath)\n\t\t\t\tif relErr != nil {\n\t\t\t\t\tif linkTarget, absErr = filepath.Abs(srcPath); absErr != nil {\n\t\t\t\t\t\treturn nil, fmt.Errorf(\"compute temporary compatibility path for %q: %w\", name, relErr)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif linkErr := os.Symlink(linkTarget, dstPath); linkErr != nil {\n\t\t\t\t\treturn nil, fmt.Errorf(\"create temporary compatibility symlink %q -> %q: %w\", dstPath, linkTarget, linkErr)\n\t\t\t\t}\n\t\t\t\tif log.GetLevel() <= zerolog.TraceLevel {\n\t\t\t\t\tlog.Trace().Str(\"dst\", dstPath).Str(\"target\", linkTarget).Msg(\"pkgs.Resolve created temp compatibility symlink\")\n\t\t\t\t}\n\t\t\t}\n\t\t\tenv = append(env, fmt.Sprintf(\"%s=%s\", envVarGotmpdir, absTempDir))\n\t\t}\n""" ++ resolve_tempdir_old = """\tif req.TempDir != \"\" {\n\t\tif err := os.MkdirAll(req.TempDir, 0o755); err != nil {\n\t\t\treturn resolvedPackageSet{}, fmt.Errorf(\"creating temporary directory %q: %w\", req.TempDir, err)\n\t\t}\n\t}\n\tenv := resolveEnvironment(ctx, req)\n""" ++ resolve_tempdir_new = """\tif req.TempDir != \"\" {\n\t\tabsTempDir, absErr := filepath.Abs(req.TempDir)\n\t\tif absErr != nil {\n\t\t\treturn resolvedPackageSet{}, fmt.Errorf(\"absolutizing temporary directory %q: %w\", req.TempDir, absErr)\n\t\t}\n\t\tif err := os.MkdirAll(absTempDir, 0o755); err != nil {\n\t\t\treturn resolvedPackageSet{}, fmt.Errorf(\"creating temporary directory %q: %w\", absTempDir, err)\n\t\t}\n\t\tfor _, name := range []string{\"external\", \"bazel-out\"} {\n\t\t\tsrcPath := filepath.Join(req.Dir, name)\n\t\t\tif !filepath.IsAbs(srcPath) {\n\t\t\t\tif srcPath, absErr = filepath.Abs(srcPath); absErr != nil {\n\t\t\t\t\treturn resolvedPackageSet{}, fmt.Errorf(\"absolutizing compatibility path %q: %w\", srcPath, absErr)\n\t\t\t\t}\n\t\t\t}\n\t\t\tif _, statErr := os.Stat(srcPath); statErr != nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tdstPath := filepath.Join(absTempDir, name)\n\t\t\tif _, statErr := os.Lstat(dstPath); statErr == nil {\n\t\t\t\tcontinue\n\t\t\t} else if !os.IsNotExist(statErr) {\n\t\t\t\treturn resolvedPackageSet{}, fmt.Errorf(\"stat temporary compatibility path %q: %w\", dstPath, statErr)\n\t\t\t}\n\t\t\tlinkTarget, relErr := filepath.Rel(absTempDir, srcPath)\n\t\t\tif relErr != nil {\n\t\t\t\tlinkTarget = srcPath\n\t\t\t}\n\t\t\tif linkErr := os.Symlink(linkTarget, dstPath); linkErr != nil {\n\t\t\t\treturn resolvedPackageSet{}, fmt.Errorf(\"create temporary compatibility symlink %q -> %q: %w\", dstPath, linkTarget, linkErr)\n\t\t\t}\n\t\t\tlog.Trace().Str(\"dst\", dstPath).Str(\"target\", linkTarget).Msg(\"pkgs.Resolve created temp compatibility symlink\")\n\t\t}\n\t\treq.TempDir = absTempDir\n\t}\n\tenv := resolveEnvironment(ctx, req)\n""" + if resolve_tempdir_old not in resolve_src: + fail("Could not patch Orchestrion resolver tempdir block in %s" % resolve_path) + resolve_src = resolve_src.replace(resolve_tempdir_old, resolve_tempdir_new, 1) -+ resolve_merge_old = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error {\n\tif pkg.PkgPath == \"\" || pkg.PkgPath == \"unsafe\" || r[pkg.PkgPath] != \"\" {\n\t\t// Ignore the \"unsafe\" package (no archive file, ever), packages with an empty import path\n\t\t// (standard library), and those already present in the map (already processed previously).\n\t\treturn nil\n\t}\n""" -+ resolve_merge_new = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error {\n\tif pkg.PkgPath == \"\" || pkg.PkgPath == \"unsafe\" || r[pkg.PkgPath] != \"\" {\n\t\t// Ignore the \"unsafe\" package (no archive file, ever), packages with an empty import path\n\t\t// (standard library), and those already present in the map (already processed previously).\n\t\treturn nil\n\t}\n""" -+ if resolve_merge_old not in resolve_src: -+ fail("Could not patch Orchestrion resolver mergeFrom in %s" % resolve_path) -+ resolve_src = resolve_src.replace(resolve_merge_old, resolve_merge_new, 1) + ctx.file(resolve_path, resolve_src) + + # Compile-proxy / archive metadata patches for the synthetic testmain flow. @@ -3557,16 +3544,16 @@ index 0000000..db1f149 + # run under Bazel's import-path context instead of synthetic package names. + onlink_path = "internal/toolexec/aspect/onlink.go" + onlink_src = ctx.read(onlink_path) -+ onlink_old = """\t\t\tdeps, err := resolvePackageFiles(ctx, depPath, cmd.WorkDir)\n""" -+ onlink_new = """\t\t\tprevImportPath, hadImportPath := os.LookupEnv(\"TOOLEXEC_IMPORTPATH\")\n\t\t\tif err := os.Setenv(\"TOOLEXEC_IMPORTPATH\", w.ImportPath); err != nil {\n\t\t\t\treturn fmt.Errorf(\"setting TOOLEXEC_IMPORTPATH=%q: %w\", w.ImportPath, err)\n\t\t\t}\n\t\t\tdeps, err := resolvePackageFiles(ctx, depPath, \".\")\n\t\t\tif hadImportPath {\n\t\t\t\t_ = os.Setenv(\"TOOLEXEC_IMPORTPATH\", prevImportPath)\n\t\t\t} else {\n\t\t\t\t_ = os.Unsetenv(\"TOOLEXEC_IMPORTPATH\")\n\t\t\t}\n""" ++ onlink_old = """\t\t\tdeps, err := resolvePackageFilesForTest(ctx, depPath, testVariantFor, cmd.WorkDir)\n""" ++ onlink_new = """\t\t\tprevImportPath, hadImportPath := os.LookupEnv(\"TOOLEXEC_IMPORTPATH\")\n\t\t\tif err := os.Setenv(\"TOOLEXEC_IMPORTPATH\", w.ImportPath); err != nil {\n\t\t\t\treturn fmt.Errorf(\"setting TOOLEXEC_IMPORTPATH=%q: %w\", w.ImportPath, err)\n\t\t\t}\n\t\t\tdeps, err := resolvePackageFilesForTest(ctx, depPath, testVariantFor, \".\")\n\t\t\tif hadImportPath {\n\t\t\t\t_ = os.Setenv(\"TOOLEXEC_IMPORTPATH\", prevImportPath)\n\t\t\t} else {\n\t\t\t\t_ = os.Unsetenv(\"TOOLEXEC_IMPORTPATH\")\n\t\t\t}\n""" + if onlink_old not in onlink_src: + fail("Could not patch Orchestrion onlink resolver context in %s" % onlink_path) + ctx.file(onlink_path, onlink_src.replace(onlink_old, onlink_new, 1)) + + oncompile_main_path = "internal/toolexec/aspect/oncompile-main.go" + oncompile_main_src = ctx.read(oncompile_main_path) -+ oncompile_main_old = """\t\tdeps, err := resolvePackageFiles(ctx, linkDepPath, cmd.WorkDir)\n""" -+ oncompile_main_new = """\t\tprevImportPath, hadImportPath := os.LookupEnv(\"TOOLEXEC_IMPORTPATH\")\n\t\tif err := os.Setenv(\"TOOLEXEC_IMPORTPATH\", \"main\"); err != nil {\n\t\t\treturn fmt.Errorf(\"setting TOOLEXEC_IMPORTPATH for synthetic main: %w\", err)\n\t\t}\n\t\tfor archiveImportPath, archivePath := range reg.PackageFile {\n\t\t\tld, ldErr := linkdeps.FromArchive(ctx, archivePath)\n\t\t\tif ldErr != nil || !ld.Contains(linkDepPath) {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif err := os.Setenv(\"TOOLEXEC_IMPORTPATH\", archiveImportPath); err != nil {\n\t\t\t\treturn fmt.Errorf(\"setting TOOLEXEC_IMPORTPATH=%q: %w\", archiveImportPath, err)\n\t\t\t}\n\t\t\tbreak\n\t\t}\n\t\tdeps, err := resolvePackageFiles(ctx, linkDepPath, \".\")\n\t\tif hadImportPath {\n\t\t\t_ = os.Setenv(\"TOOLEXEC_IMPORTPATH\", prevImportPath)\n\t\t} else {\n\t\t\t_ = os.Unsetenv(\"TOOLEXEC_IMPORTPATH\")\n\t\t}\n""" ++ oncompile_main_old = """\t\tdeps, err := resolvePackageFilesForTest(ctx, item.path, testVariantFor, cmd.WorkDir)\n""" ++ oncompile_main_new = """\t\tprevImportPath, hadImportPath := os.LookupEnv(\"TOOLEXEC_IMPORTPATH\")\n\t\tresolverImportPath := item.parent\n\t\tif resolverImportPath == \"\" {\n\t\t\tresolverImportPath = \"main\"\n\t\t}\n\t\tif err := os.Setenv(\"TOOLEXEC_IMPORTPATH\", resolverImportPath); err != nil {\n\t\t\treturn fmt.Errorf(\"setting TOOLEXEC_IMPORTPATH=%q: %w\", resolverImportPath, err)\n\t\t}\n\t\tdeps, err := resolvePackageFilesForTest(ctx, item.path, testVariantFor, \".\")\n\t\tif hadImportPath {\n\t\t\t_ = os.Setenv(\"TOOLEXEC_IMPORTPATH\", prevImportPath)\n\t\t} else {\n\t\t\t_ = os.Unsetenv(\"TOOLEXEC_IMPORTPATH\")\n\t\t}\n""" + if oncompile_main_old not in oncompile_main_src: + fail("Could not patch Orchestrion oncompile-main resolver context in %s" % oncompile_main_path) + ctx.file(oncompile_main_path, oncompile_main_src.replace(oncompile_main_old, oncompile_main_new, 1)) @@ -14174,7 +14161,7 @@ index 0000000..828fec6 +} diff --git a/go/tools/builders/orchestrion_version.go b/go/tools/builders/orchestrion_version.go new file mode 100644 -index 0000000..d59f9b4 +index 0000000..0469ab5 --- /dev/null +++ b/go/tools/builders/orchestrion_version.go @@ -0,0 +1,425 @@ @@ -14193,7 +14180,7 @@ index 0000000..d59f9b4 + +const ( + rulesGoOrchestrionVersionFileEnvVar = "RULES_GO_ORCHESTRION_VERSION_FILE" -+ defaultDDTraceGoVersion = "v2.9.0" ++ defaultDDTraceGoVersion = "v2.9.1" + orchestrionSyntheticGoModVersion = "1.21" +) + diff --git a/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch index 6e5c51ee..b668a4f6 100644 --- a/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch @@ -2084,10 +2084,10 @@ index 0000000..177eeaf +) diff --git a/go/private/orchestrion/extensions.bzl b/go/private/orchestrion/extensions.bzl new file mode 100644 -index 0000000..db1f149 +index 0000000..a56e568 --- /dev/null +++ b/go/private/orchestrion/extensions.bzl -@@ -0,0 +1,1732 @@ +@@ -0,0 +1,1719 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); @@ -2104,7 +2104,7 @@ index 0000000..db1f149 + +"""Module extension for configuring orchestrion in rules_go.""" + -+DEFAULT_DD_TRACE_GO_VERSION = "v2.9.0" ++DEFAULT_DD_TRACE_GO_VERSION = "v2.9.1" +ORCHESTRION_BOOTSTRAP_CACHE_ABI = "v3" +ORCHESTRION_SEED_GO_MOD_VERSION = "1.21" + @@ -3462,34 +3462,21 @@ index 0000000..db1f149 + ctx.report_progress("rules_go_orchestrion: patching orchestrion source") + patch_start_ms = _probe_now_ms(ctx) + -+ # The upstream package resolver recursively re-runs `go list` under -+ # `-toolexec=orchestrion toolexec`, which causes woven dependency lookups to -+ # fail under Bazel's sandbox even when plain `go list -mod=mod` succeeds in -+ # the same environment. Mirror the safer non-recursive pattern used by other -+ # loader paths in Orchestrion and disable toolexec for dependency resolution. ++ # Keep Orchestrion's recursive toolexec resolver: v1.12 uses it to build the ++ # correct dependency variants for generated test mains. Only adapt its ++ # temporary directory to Bazel's execroot layout. + resolve_path = "internal/jobserver/pkgs/resolve.go" + resolve_src = ctx.read(resolve_path) -+ resolve_old = """\n\t\tbuildFlags := append(\n\t\t\tgoFlags.Slice(),\n\t\t\tfmt.Sprintf(\"-toolexec=%q toolexec\", binpath.Orchestrion),\n\t\t)\n""" -+ resolve_new = """\n\t\tbuildFlags := append(goFlags.Slice(), \"-toolexec=\")\n""" -+ if resolve_old not in resolve_src: -+ fail("Could not patch Orchestrion resolver in %s" % resolve_path) -+ resolve_src = resolve_src.replace(resolve_old, resolve_new) -+ resolve_src = resolve_src.replace('\n\t"github.com/DataDog/orchestrion/internal/binpath"', "") -+ resolve_imports_old = """\t\"fmt\"\n\t\"os\"\n\t\"slices\"\n""" -+ resolve_imports_new = """\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"slices\"\n""" ++ resolve_imports_old = """\t\"maps\"\n\t\"os\"\n\t\"slices\"\n""" ++ resolve_imports_new = """\t\"maps\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"slices\"\n""" + if resolve_imports_old not in resolve_src: + fail("Could not patch Orchestrion resolver imports in %s" % resolve_path) + resolve_src = resolve_src.replace(resolve_imports_old, resolve_imports_new, 1) -+ resolve_tempdir_old = """\t\tif req.TempDir != \"\" {\n\t\t\t// Make sure the directory exists (go blindly assumes that...)\n\t\t\tif err := os.MkdirAll(req.TempDir, 0o755); err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"creating temporary directory %q: %w\", req.TempDir, err)\n\t\t\t}\n\t\t\tenv = append(env, fmt.Sprintf(\"%s=%s\", envVarGotmpdir, req.TempDir))\n\t\t}\n""" -+ resolve_tempdir_new = """\t\tif req.TempDir != \"\" {\n\t\t\tabsTempDir, absErr := filepath.Abs(req.TempDir)\n\t\t\tif absErr != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"absolutizing temporary directory %q: %w\", req.TempDir, absErr)\n\t\t\t}\n\t\t\t// Make sure the directory exists (go blindly assumes that...)\n\t\t\tif err := os.MkdirAll(absTempDir, 0o755); err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"creating temporary directory %q: %w\", absTempDir, err)\n\t\t\t}\n\t\t\tfor _, name := range []string{\"external\", \"bazel-out\"} {\n\t\t\t\tsrcPath := filepath.Join(req.Dir, name)\n\t\t\t\tif !filepath.IsAbs(srcPath) {\n\t\t\t\t\tif srcPath, absErr = filepath.Abs(srcPath); absErr != nil {\n\t\t\t\t\t\treturn nil, fmt.Errorf(\"absolutizing compatibility path %q: %w\", srcPath, absErr)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif _, statErr := os.Stat(srcPath); statErr != nil {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tdstPath := filepath.Join(absTempDir, name)\n\t\t\t\tif _, statErr := os.Lstat(dstPath); statErr == nil {\n\t\t\t\t\tcontinue\n\t\t\t\t} else if !os.IsNotExist(statErr) {\n\t\t\t\t\treturn nil, fmt.Errorf(\"stat temporary compatibility path %q: %w\", dstPath, statErr)\n\t\t\t\t}\n\t\t\t\tlinkTarget, relErr := filepath.Rel(absTempDir, srcPath)\n\t\t\t\tif relErr != nil {\n\t\t\t\t\tif linkTarget, absErr = filepath.Abs(srcPath); absErr != nil {\n\t\t\t\t\t\treturn nil, fmt.Errorf(\"compute temporary compatibility path for %q: %w\", name, relErr)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif linkErr := os.Symlink(linkTarget, dstPath); linkErr != nil {\n\t\t\t\t\treturn nil, fmt.Errorf(\"create temporary compatibility symlink %q -> %q: %w\", dstPath, linkTarget, linkErr)\n\t\t\t\t}\n\t\t\t\tif log.GetLevel() <= zerolog.TraceLevel {\n\t\t\t\t\tlog.Trace().Str(\"dst\", dstPath).Str(\"target\", linkTarget).Msg(\"pkgs.Resolve created temp compatibility symlink\")\n\t\t\t\t}\n\t\t\t}\n\t\t\tenv = append(env, fmt.Sprintf(\"%s=%s\", envVarGotmpdir, absTempDir))\n\t\t}\n""" ++ resolve_tempdir_old = """\tif req.TempDir != \"\" {\n\t\tif err := os.MkdirAll(req.TempDir, 0o755); err != nil {\n\t\t\treturn resolvedPackageSet{}, fmt.Errorf(\"creating temporary directory %q: %w\", req.TempDir, err)\n\t\t}\n\t}\n\tenv := resolveEnvironment(ctx, req)\n""" ++ resolve_tempdir_new = """\tif req.TempDir != \"\" {\n\t\tabsTempDir, absErr := filepath.Abs(req.TempDir)\n\t\tif absErr != nil {\n\t\t\treturn resolvedPackageSet{}, fmt.Errorf(\"absolutizing temporary directory %q: %w\", req.TempDir, absErr)\n\t\t}\n\t\tif err := os.MkdirAll(absTempDir, 0o755); err != nil {\n\t\t\treturn resolvedPackageSet{}, fmt.Errorf(\"creating temporary directory %q: %w\", absTempDir, err)\n\t\t}\n\t\tfor _, name := range []string{\"external\", \"bazel-out\"} {\n\t\t\tsrcPath := filepath.Join(req.Dir, name)\n\t\t\tif !filepath.IsAbs(srcPath) {\n\t\t\t\tif srcPath, absErr = filepath.Abs(srcPath); absErr != nil {\n\t\t\t\t\treturn resolvedPackageSet{}, fmt.Errorf(\"absolutizing compatibility path %q: %w\", srcPath, absErr)\n\t\t\t\t}\n\t\t\t}\n\t\t\tif _, statErr := os.Stat(srcPath); statErr != nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tdstPath := filepath.Join(absTempDir, name)\n\t\t\tif _, statErr := os.Lstat(dstPath); statErr == nil {\n\t\t\t\tcontinue\n\t\t\t} else if !os.IsNotExist(statErr) {\n\t\t\t\treturn resolvedPackageSet{}, fmt.Errorf(\"stat temporary compatibility path %q: %w\", dstPath, statErr)\n\t\t\t}\n\t\t\tlinkTarget, relErr := filepath.Rel(absTempDir, srcPath)\n\t\t\tif relErr != nil {\n\t\t\t\tlinkTarget = srcPath\n\t\t\t}\n\t\t\tif linkErr := os.Symlink(linkTarget, dstPath); linkErr != nil {\n\t\t\t\treturn resolvedPackageSet{}, fmt.Errorf(\"create temporary compatibility symlink %q -> %q: %w\", dstPath, linkTarget, linkErr)\n\t\t\t}\n\t\t\tlog.Trace().Str(\"dst\", dstPath).Str(\"target\", linkTarget).Msg(\"pkgs.Resolve created temp compatibility symlink\")\n\t\t}\n\t\treq.TempDir = absTempDir\n\t}\n\tenv := resolveEnvironment(ctx, req)\n""" + if resolve_tempdir_old not in resolve_src: + fail("Could not patch Orchestrion resolver tempdir block in %s" % resolve_path) + resolve_src = resolve_src.replace(resolve_tempdir_old, resolve_tempdir_new, 1) -+ resolve_merge_old = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error {\n\tif pkg.PkgPath == \"\" || pkg.PkgPath == \"unsafe\" || r[pkg.PkgPath] != \"\" {\n\t\t// Ignore the \"unsafe\" package (no archive file, ever), packages with an empty import path\n\t\t// (standard library), and those already present in the map (already processed previously).\n\t\treturn nil\n\t}\n""" -+ resolve_merge_new = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error {\n\tif pkg.PkgPath == \"\" || pkg.PkgPath == \"unsafe\" || r[pkg.PkgPath] != \"\" {\n\t\t// Ignore the \"unsafe\" package (no archive file, ever), packages with an empty import path\n\t\t// (standard library), and those already present in the map (already processed previously).\n\t\treturn nil\n\t}\n""" -+ if resolve_merge_old not in resolve_src: -+ fail("Could not patch Orchestrion resolver mergeFrom in %s" % resolve_path) -+ resolve_src = resolve_src.replace(resolve_merge_old, resolve_merge_new, 1) + ctx.file(resolve_path, resolve_src) + + # Compile-proxy / archive metadata patches for the synthetic testmain flow. @@ -3550,16 +3537,16 @@ index 0000000..db1f149 + # run under Bazel's import-path context instead of synthetic package names. + onlink_path = "internal/toolexec/aspect/onlink.go" + onlink_src = ctx.read(onlink_path) -+ onlink_old = """\t\t\tdeps, err := resolvePackageFiles(ctx, depPath, cmd.WorkDir)\n""" -+ onlink_new = """\t\t\tprevImportPath, hadImportPath := os.LookupEnv(\"TOOLEXEC_IMPORTPATH\")\n\t\t\tif err := os.Setenv(\"TOOLEXEC_IMPORTPATH\", w.ImportPath); err != nil {\n\t\t\t\treturn fmt.Errorf(\"setting TOOLEXEC_IMPORTPATH=%q: %w\", w.ImportPath, err)\n\t\t\t}\n\t\t\tdeps, err := resolvePackageFiles(ctx, depPath, \".\")\n\t\t\tif hadImportPath {\n\t\t\t\t_ = os.Setenv(\"TOOLEXEC_IMPORTPATH\", prevImportPath)\n\t\t\t} else {\n\t\t\t\t_ = os.Unsetenv(\"TOOLEXEC_IMPORTPATH\")\n\t\t\t}\n""" ++ onlink_old = """\t\t\tdeps, err := resolvePackageFilesForTest(ctx, depPath, testVariantFor, cmd.WorkDir)\n""" ++ onlink_new = """\t\t\tprevImportPath, hadImportPath := os.LookupEnv(\"TOOLEXEC_IMPORTPATH\")\n\t\t\tif err := os.Setenv(\"TOOLEXEC_IMPORTPATH\", w.ImportPath); err != nil {\n\t\t\t\treturn fmt.Errorf(\"setting TOOLEXEC_IMPORTPATH=%q: %w\", w.ImportPath, err)\n\t\t\t}\n\t\t\tdeps, err := resolvePackageFilesForTest(ctx, depPath, testVariantFor, \".\")\n\t\t\tif hadImportPath {\n\t\t\t\t_ = os.Setenv(\"TOOLEXEC_IMPORTPATH\", prevImportPath)\n\t\t\t} else {\n\t\t\t\t_ = os.Unsetenv(\"TOOLEXEC_IMPORTPATH\")\n\t\t\t}\n""" + if onlink_old not in onlink_src: + fail("Could not patch Orchestrion onlink resolver context in %s" % onlink_path) + ctx.file(onlink_path, onlink_src.replace(onlink_old, onlink_new, 1)) + + oncompile_main_path = "internal/toolexec/aspect/oncompile-main.go" + oncompile_main_src = ctx.read(oncompile_main_path) -+ oncompile_main_old = """\t\tdeps, err := resolvePackageFiles(ctx, linkDepPath, cmd.WorkDir)\n""" -+ oncompile_main_new = """\t\tprevImportPath, hadImportPath := os.LookupEnv(\"TOOLEXEC_IMPORTPATH\")\n\t\tif err := os.Setenv(\"TOOLEXEC_IMPORTPATH\", \"main\"); err != nil {\n\t\t\treturn fmt.Errorf(\"setting TOOLEXEC_IMPORTPATH for synthetic main: %w\", err)\n\t\t}\n\t\tfor archiveImportPath, archivePath := range reg.PackageFile {\n\t\t\tld, ldErr := linkdeps.FromArchive(ctx, archivePath)\n\t\t\tif ldErr != nil || !ld.Contains(linkDepPath) {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif err := os.Setenv(\"TOOLEXEC_IMPORTPATH\", archiveImportPath); err != nil {\n\t\t\t\treturn fmt.Errorf(\"setting TOOLEXEC_IMPORTPATH=%q: %w\", archiveImportPath, err)\n\t\t\t}\n\t\t\tbreak\n\t\t}\n\t\tdeps, err := resolvePackageFiles(ctx, linkDepPath, \".\")\n\t\tif hadImportPath {\n\t\t\t_ = os.Setenv(\"TOOLEXEC_IMPORTPATH\", prevImportPath)\n\t\t} else {\n\t\t\t_ = os.Unsetenv(\"TOOLEXEC_IMPORTPATH\")\n\t\t}\n""" ++ oncompile_main_old = """\t\tdeps, err := resolvePackageFilesForTest(ctx, item.path, testVariantFor, cmd.WorkDir)\n""" ++ oncompile_main_new = """\t\tprevImportPath, hadImportPath := os.LookupEnv(\"TOOLEXEC_IMPORTPATH\")\n\t\tresolverImportPath := item.parent\n\t\tif resolverImportPath == \"\" {\n\t\t\tresolverImportPath = \"main\"\n\t\t}\n\t\tif err := os.Setenv(\"TOOLEXEC_IMPORTPATH\", resolverImportPath); err != nil {\n\t\t\treturn fmt.Errorf(\"setting TOOLEXEC_IMPORTPATH=%q: %w\", resolverImportPath, err)\n\t\t}\n\t\tdeps, err := resolvePackageFilesForTest(ctx, item.path, testVariantFor, \".\")\n\t\tif hadImportPath {\n\t\t\t_ = os.Setenv(\"TOOLEXEC_IMPORTPATH\", prevImportPath)\n\t\t} else {\n\t\t\t_ = os.Unsetenv(\"TOOLEXEC_IMPORTPATH\")\n\t\t}\n""" + if oncompile_main_old not in oncompile_main_src: + fail("Could not patch Orchestrion oncompile-main resolver context in %s" % oncompile_main_path) + ctx.file(oncompile_main_path, oncompile_main_src.replace(oncompile_main_old, oncompile_main_new, 1)) @@ -14164,7 +14151,7 @@ index 0000000..828fec6 +} diff --git a/go/tools/builders/orchestrion_version.go b/go/tools/builders/orchestrion_version.go new file mode 100644 -index 0000000..d59f9b4 +index 0000000..0469ab5 --- /dev/null +++ b/go/tools/builders/orchestrion_version.go @@ -0,0 +1,425 @@ @@ -14183,7 +14170,7 @@ index 0000000..d59f9b4 + +const ( + rulesGoOrchestrionVersionFileEnvVar = "RULES_GO_ORCHESTRION_VERSION_FILE" -+ defaultDDTraceGoVersion = "v2.9.0" ++ defaultDDTraceGoVersion = "v2.9.1" + orchestrionSyntheticGoModVersion = "1.21" +) + diff --git a/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch index 03781e4e..62dd8307 100644 --- a/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch @@ -2088,10 +2088,10 @@ index 0000000..177eeaf +) diff --git a/go/private/orchestrion/extensions.bzl b/go/private/orchestrion/extensions.bzl new file mode 100644 -index 0000000..db1f149 +index 0000000..a56e568 --- /dev/null +++ b/go/private/orchestrion/extensions.bzl -@@ -0,0 +1,1732 @@ +@@ -0,0 +1,1719 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); @@ -2108,7 +2108,7 @@ index 0000000..db1f149 + +"""Module extension for configuring orchestrion in rules_go.""" + -+DEFAULT_DD_TRACE_GO_VERSION = "v2.9.0" ++DEFAULT_DD_TRACE_GO_VERSION = "v2.9.1" +ORCHESTRION_BOOTSTRAP_CACHE_ABI = "v3" +ORCHESTRION_SEED_GO_MOD_VERSION = "1.21" + @@ -3466,34 +3466,21 @@ index 0000000..db1f149 + ctx.report_progress("rules_go_orchestrion: patching orchestrion source") + patch_start_ms = _probe_now_ms(ctx) + -+ # The upstream package resolver recursively re-runs `go list` under -+ # `-toolexec=orchestrion toolexec`, which causes woven dependency lookups to -+ # fail under Bazel's sandbox even when plain `go list -mod=mod` succeeds in -+ # the same environment. Mirror the safer non-recursive pattern used by other -+ # loader paths in Orchestrion and disable toolexec for dependency resolution. ++ # Keep Orchestrion's recursive toolexec resolver: v1.12 uses it to build the ++ # correct dependency variants for generated test mains. Only adapt its ++ # temporary directory to Bazel's execroot layout. + resolve_path = "internal/jobserver/pkgs/resolve.go" + resolve_src = ctx.read(resolve_path) -+ resolve_old = """\n\t\tbuildFlags := append(\n\t\t\tgoFlags.Slice(),\n\t\t\tfmt.Sprintf(\"-toolexec=%q toolexec\", binpath.Orchestrion),\n\t\t)\n""" -+ resolve_new = """\n\t\tbuildFlags := append(goFlags.Slice(), \"-toolexec=\")\n""" -+ if resolve_old not in resolve_src: -+ fail("Could not patch Orchestrion resolver in %s" % resolve_path) -+ resolve_src = resolve_src.replace(resolve_old, resolve_new) -+ resolve_src = resolve_src.replace('\n\t"github.com/DataDog/orchestrion/internal/binpath"', "") -+ resolve_imports_old = """\t\"fmt\"\n\t\"os\"\n\t\"slices\"\n""" -+ resolve_imports_new = """\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"slices\"\n""" ++ resolve_imports_old = """\t\"maps\"\n\t\"os\"\n\t\"slices\"\n""" ++ resolve_imports_new = """\t\"maps\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"slices\"\n""" + if resolve_imports_old not in resolve_src: + fail("Could not patch Orchestrion resolver imports in %s" % resolve_path) + resolve_src = resolve_src.replace(resolve_imports_old, resolve_imports_new, 1) -+ resolve_tempdir_old = """\t\tif req.TempDir != \"\" {\n\t\t\t// Make sure the directory exists (go blindly assumes that...)\n\t\t\tif err := os.MkdirAll(req.TempDir, 0o755); err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"creating temporary directory %q: %w\", req.TempDir, err)\n\t\t\t}\n\t\t\tenv = append(env, fmt.Sprintf(\"%s=%s\", envVarGotmpdir, req.TempDir))\n\t\t}\n""" -+ resolve_tempdir_new = """\t\tif req.TempDir != \"\" {\n\t\t\tabsTempDir, absErr := filepath.Abs(req.TempDir)\n\t\t\tif absErr != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"absolutizing temporary directory %q: %w\", req.TempDir, absErr)\n\t\t\t}\n\t\t\t// Make sure the directory exists (go blindly assumes that...)\n\t\t\tif err := os.MkdirAll(absTempDir, 0o755); err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"creating temporary directory %q: %w\", absTempDir, err)\n\t\t\t}\n\t\t\tfor _, name := range []string{\"external\", \"bazel-out\"} {\n\t\t\t\tsrcPath := filepath.Join(req.Dir, name)\n\t\t\t\tif !filepath.IsAbs(srcPath) {\n\t\t\t\t\tif srcPath, absErr = filepath.Abs(srcPath); absErr != nil {\n\t\t\t\t\t\treturn nil, fmt.Errorf(\"absolutizing compatibility path %q: %w\", srcPath, absErr)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif _, statErr := os.Stat(srcPath); statErr != nil {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tdstPath := filepath.Join(absTempDir, name)\n\t\t\t\tif _, statErr := os.Lstat(dstPath); statErr == nil {\n\t\t\t\t\tcontinue\n\t\t\t\t} else if !os.IsNotExist(statErr) {\n\t\t\t\t\treturn nil, fmt.Errorf(\"stat temporary compatibility path %q: %w\", dstPath, statErr)\n\t\t\t\t}\n\t\t\t\tlinkTarget, relErr := filepath.Rel(absTempDir, srcPath)\n\t\t\t\tif relErr != nil {\n\t\t\t\t\tif linkTarget, absErr = filepath.Abs(srcPath); absErr != nil {\n\t\t\t\t\t\treturn nil, fmt.Errorf(\"compute temporary compatibility path for %q: %w\", name, relErr)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif linkErr := os.Symlink(linkTarget, dstPath); linkErr != nil {\n\t\t\t\t\treturn nil, fmt.Errorf(\"create temporary compatibility symlink %q -> %q: %w\", dstPath, linkTarget, linkErr)\n\t\t\t\t}\n\t\t\t\tif log.GetLevel() <= zerolog.TraceLevel {\n\t\t\t\t\tlog.Trace().Str(\"dst\", dstPath).Str(\"target\", linkTarget).Msg(\"pkgs.Resolve created temp compatibility symlink\")\n\t\t\t\t}\n\t\t\t}\n\t\t\tenv = append(env, fmt.Sprintf(\"%s=%s\", envVarGotmpdir, absTempDir))\n\t\t}\n""" ++ resolve_tempdir_old = """\tif req.TempDir != \"\" {\n\t\tif err := os.MkdirAll(req.TempDir, 0o755); err != nil {\n\t\t\treturn resolvedPackageSet{}, fmt.Errorf(\"creating temporary directory %q: %w\", req.TempDir, err)\n\t\t}\n\t}\n\tenv := resolveEnvironment(ctx, req)\n""" ++ resolve_tempdir_new = """\tif req.TempDir != \"\" {\n\t\tabsTempDir, absErr := filepath.Abs(req.TempDir)\n\t\tif absErr != nil {\n\t\t\treturn resolvedPackageSet{}, fmt.Errorf(\"absolutizing temporary directory %q: %w\", req.TempDir, absErr)\n\t\t}\n\t\tif err := os.MkdirAll(absTempDir, 0o755); err != nil {\n\t\t\treturn resolvedPackageSet{}, fmt.Errorf(\"creating temporary directory %q: %w\", absTempDir, err)\n\t\t}\n\t\tfor _, name := range []string{\"external\", \"bazel-out\"} {\n\t\t\tsrcPath := filepath.Join(req.Dir, name)\n\t\t\tif !filepath.IsAbs(srcPath) {\n\t\t\t\tif srcPath, absErr = filepath.Abs(srcPath); absErr != nil {\n\t\t\t\t\treturn resolvedPackageSet{}, fmt.Errorf(\"absolutizing compatibility path %q: %w\", srcPath, absErr)\n\t\t\t\t}\n\t\t\t}\n\t\t\tif _, statErr := os.Stat(srcPath); statErr != nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tdstPath := filepath.Join(absTempDir, name)\n\t\t\tif _, statErr := os.Lstat(dstPath); statErr == nil {\n\t\t\t\tcontinue\n\t\t\t} else if !os.IsNotExist(statErr) {\n\t\t\t\treturn resolvedPackageSet{}, fmt.Errorf(\"stat temporary compatibility path %q: %w\", dstPath, statErr)\n\t\t\t}\n\t\t\tlinkTarget, relErr := filepath.Rel(absTempDir, srcPath)\n\t\t\tif relErr != nil {\n\t\t\t\tlinkTarget = srcPath\n\t\t\t}\n\t\t\tif linkErr := os.Symlink(linkTarget, dstPath); linkErr != nil {\n\t\t\t\treturn resolvedPackageSet{}, fmt.Errorf(\"create temporary compatibility symlink %q -> %q: %w\", dstPath, linkTarget, linkErr)\n\t\t\t}\n\t\t\tlog.Trace().Str(\"dst\", dstPath).Str(\"target\", linkTarget).Msg(\"pkgs.Resolve created temp compatibility symlink\")\n\t\t}\n\t\treq.TempDir = absTempDir\n\t}\n\tenv := resolveEnvironment(ctx, req)\n""" + if resolve_tempdir_old not in resolve_src: + fail("Could not patch Orchestrion resolver tempdir block in %s" % resolve_path) + resolve_src = resolve_src.replace(resolve_tempdir_old, resolve_tempdir_new, 1) -+ resolve_merge_old = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error {\n\tif pkg.PkgPath == \"\" || pkg.PkgPath == \"unsafe\" || r[pkg.PkgPath] != \"\" {\n\t\t// Ignore the \"unsafe\" package (no archive file, ever), packages with an empty import path\n\t\t// (standard library), and those already present in the map (already processed previously).\n\t\treturn nil\n\t}\n""" -+ resolve_merge_new = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error {\n\tif pkg.PkgPath == \"\" || pkg.PkgPath == \"unsafe\" || r[pkg.PkgPath] != \"\" {\n\t\t// Ignore the \"unsafe\" package (no archive file, ever), packages with an empty import path\n\t\t// (standard library), and those already present in the map (already processed previously).\n\t\treturn nil\n\t}\n""" -+ if resolve_merge_old not in resolve_src: -+ fail("Could not patch Orchestrion resolver mergeFrom in %s" % resolve_path) -+ resolve_src = resolve_src.replace(resolve_merge_old, resolve_merge_new, 1) + ctx.file(resolve_path, resolve_src) + + # Compile-proxy / archive metadata patches for the synthetic testmain flow. @@ -3554,16 +3541,16 @@ index 0000000..db1f149 + # run under Bazel's import-path context instead of synthetic package names. + onlink_path = "internal/toolexec/aspect/onlink.go" + onlink_src = ctx.read(onlink_path) -+ onlink_old = """\t\t\tdeps, err := resolvePackageFiles(ctx, depPath, cmd.WorkDir)\n""" -+ onlink_new = """\t\t\tprevImportPath, hadImportPath := os.LookupEnv(\"TOOLEXEC_IMPORTPATH\")\n\t\t\tif err := os.Setenv(\"TOOLEXEC_IMPORTPATH\", w.ImportPath); err != nil {\n\t\t\t\treturn fmt.Errorf(\"setting TOOLEXEC_IMPORTPATH=%q: %w\", w.ImportPath, err)\n\t\t\t}\n\t\t\tdeps, err := resolvePackageFiles(ctx, depPath, \".\")\n\t\t\tif hadImportPath {\n\t\t\t\t_ = os.Setenv(\"TOOLEXEC_IMPORTPATH\", prevImportPath)\n\t\t\t} else {\n\t\t\t\t_ = os.Unsetenv(\"TOOLEXEC_IMPORTPATH\")\n\t\t\t}\n""" ++ onlink_old = """\t\t\tdeps, err := resolvePackageFilesForTest(ctx, depPath, testVariantFor, cmd.WorkDir)\n""" ++ onlink_new = """\t\t\tprevImportPath, hadImportPath := os.LookupEnv(\"TOOLEXEC_IMPORTPATH\")\n\t\t\tif err := os.Setenv(\"TOOLEXEC_IMPORTPATH\", w.ImportPath); err != nil {\n\t\t\t\treturn fmt.Errorf(\"setting TOOLEXEC_IMPORTPATH=%q: %w\", w.ImportPath, err)\n\t\t\t}\n\t\t\tdeps, err := resolvePackageFilesForTest(ctx, depPath, testVariantFor, \".\")\n\t\t\tif hadImportPath {\n\t\t\t\t_ = os.Setenv(\"TOOLEXEC_IMPORTPATH\", prevImportPath)\n\t\t\t} else {\n\t\t\t\t_ = os.Unsetenv(\"TOOLEXEC_IMPORTPATH\")\n\t\t\t}\n""" + if onlink_old not in onlink_src: + fail("Could not patch Orchestrion onlink resolver context in %s" % onlink_path) + ctx.file(onlink_path, onlink_src.replace(onlink_old, onlink_new, 1)) + + oncompile_main_path = "internal/toolexec/aspect/oncompile-main.go" + oncompile_main_src = ctx.read(oncompile_main_path) -+ oncompile_main_old = """\t\tdeps, err := resolvePackageFiles(ctx, linkDepPath, cmd.WorkDir)\n""" -+ oncompile_main_new = """\t\tprevImportPath, hadImportPath := os.LookupEnv(\"TOOLEXEC_IMPORTPATH\")\n\t\tif err := os.Setenv(\"TOOLEXEC_IMPORTPATH\", \"main\"); err != nil {\n\t\t\treturn fmt.Errorf(\"setting TOOLEXEC_IMPORTPATH for synthetic main: %w\", err)\n\t\t}\n\t\tfor archiveImportPath, archivePath := range reg.PackageFile {\n\t\t\tld, ldErr := linkdeps.FromArchive(ctx, archivePath)\n\t\t\tif ldErr != nil || !ld.Contains(linkDepPath) {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif err := os.Setenv(\"TOOLEXEC_IMPORTPATH\", archiveImportPath); err != nil {\n\t\t\t\treturn fmt.Errorf(\"setting TOOLEXEC_IMPORTPATH=%q: %w\", archiveImportPath, err)\n\t\t\t}\n\t\t\tbreak\n\t\t}\n\t\tdeps, err := resolvePackageFiles(ctx, linkDepPath, \".\")\n\t\tif hadImportPath {\n\t\t\t_ = os.Setenv(\"TOOLEXEC_IMPORTPATH\", prevImportPath)\n\t\t} else {\n\t\t\t_ = os.Unsetenv(\"TOOLEXEC_IMPORTPATH\")\n\t\t}\n""" ++ oncompile_main_old = """\t\tdeps, err := resolvePackageFilesForTest(ctx, item.path, testVariantFor, cmd.WorkDir)\n""" ++ oncompile_main_new = """\t\tprevImportPath, hadImportPath := os.LookupEnv(\"TOOLEXEC_IMPORTPATH\")\n\t\tresolverImportPath := item.parent\n\t\tif resolverImportPath == \"\" {\n\t\t\tresolverImportPath = \"main\"\n\t\t}\n\t\tif err := os.Setenv(\"TOOLEXEC_IMPORTPATH\", resolverImportPath); err != nil {\n\t\t\treturn fmt.Errorf(\"setting TOOLEXEC_IMPORTPATH=%q: %w\", resolverImportPath, err)\n\t\t}\n\t\tdeps, err := resolvePackageFilesForTest(ctx, item.path, testVariantFor, \".\")\n\t\tif hadImportPath {\n\t\t\t_ = os.Setenv(\"TOOLEXEC_IMPORTPATH\", prevImportPath)\n\t\t} else {\n\t\t\t_ = os.Unsetenv(\"TOOLEXEC_IMPORTPATH\")\n\t\t}\n""" + if oncompile_main_old not in oncompile_main_src: + fail("Could not patch Orchestrion oncompile-main resolver context in %s" % oncompile_main_path) + ctx.file(oncompile_main_path, oncompile_main_src.replace(oncompile_main_old, oncompile_main_new, 1)) @@ -14168,7 +14155,7 @@ index 0000000..828fec6 +} diff --git a/go/tools/builders/orchestrion_version.go b/go/tools/builders/orchestrion_version.go new file mode 100644 -index 0000000..d59f9b4 +index 0000000..0469ab5 --- /dev/null +++ b/go/tools/builders/orchestrion_version.go @@ -0,0 +1,425 @@ @@ -14187,7 +14174,7 @@ index 0000000..d59f9b4 + +const ( + rulesGoOrchestrionVersionFileEnvVar = "RULES_GO_ORCHESTRION_VERSION_FILE" -+ defaultDDTraceGoVersion = "v2.9.0" ++ defaultDDTraceGoVersion = "v2.9.1" + orchestrionSyntheticGoModVersion = "1.21" +) + diff --git a/tools/agent-skills/go-test-optimization-onboarding/references/bzlmod-onboarding.md b/tools/agent-skills/go-test-optimization-onboarding/references/bzlmod-onboarding.md index e71897b2..6fd5bf8b 100644 --- a/tools/agent-skills/go-test-optimization-onboarding/references/bzlmod-onboarding.md +++ b/tools/agent-skills/go-test-optimization-onboarding/references/bzlmod-onboarding.md @@ -76,7 +76,7 @@ use_repo(test_optimization_go_sdk, "test_optimization_go_sdk") orchestrion = use_extension("@rules_go//go:extensions.bzl", "orchestrion") orchestrion.from_source( - version = "v1.9.0", + version = "v1.12.0", dd_trace_go_pin_files = [ "@//:go.mod", "@//:go.sum", @@ -90,6 +90,7 @@ use_repo(orchestrion, "rules_go_orchestrion_tool") Guided bootstrap writes this SDK declaration from `--runtime-version`. Orchestrion uses the Bazel-managed SDK on cache misses, while a compatible bootstrap cache hit can be restored before the SDK repository is materialized. +For Orchestrion `v1.12.0`, use Go `1.25.0` or newer. Do not add SDK or Orchestrion settings to individual service or test targets. Export the root `go.mod` and `go.sum` labels from their BUILD package. The pin-file mode uses this Bazel-managed SDK with `-mod=readonly` to derive direct diff --git a/tools/agent-skills/go-test-optimization-onboarding/references/workspace-onboarding.md b/tools/agent-skills/go-test-optimization-onboarding/references/workspace-onboarding.md index 6e0707b4..bb1bf3bd 100644 --- a/tools/agent-skills/go-test-optimization-onboarding/references/workspace-onboarding.md +++ b/tools/agent-skills/go-test-optimization-onboarding/references/workspace-onboarding.md @@ -270,7 +270,7 @@ dd_topt_go_orchestrion_tool_repo( ], go_sdk_root = "@go_sdk//:ROOT", go_sdk_version = "", - version = "v1.9.0", + version = "v1.12.0", ) # Call the repository's existing go_rules_dependencies() wiring after the @@ -286,6 +286,9 @@ dd_topt_go_workspace_sync_repositories( ) ``` +For Orchestrion `v1.12.0`, `go_sdk_version` and the registered Go toolchain +must be Go `1.25.0` or newer. + The public Go helper is config-gated by default. Do not add a second enable attribute to each repository or test target. diff --git a/tools/dev/verify_rules_go_profiles.py b/tools/dev/verify_rules_go_profiles.py index 9c4ae93c..30ae8ab5 100644 --- a/tools/dev/verify_rules_go_profiles.py +++ b/tools/dev/verify_rules_go_profiles.py @@ -62,8 +62,8 @@ def verify_profiles( run_functional_smoke: bool = True, bazel: Path = REPO_ROOT / "bazelw", go_version: str = "1.25.0", - orchestrion_version: str = "v1.9.0", - dd_trace_go_version: str = "v2.9.0", + orchestrion_version: str = "v1.12.0", + dd_trace_go_version: str = "v2.9.1", ) -> None: """Generate and validate one profile patch for selected upstreams.""" registry = load_registry(registry_path) @@ -483,8 +483,8 @@ def main(argv: list[str] | None = None) -> int: ) parser.add_argument("--bazel", type=Path, default=REPO_ROOT / "bazelw") parser.add_argument("--go-version", default="1.25.0") - parser.add_argument("--orchestrion-version", default="v1.9.0") - parser.add_argument("--dd-trace-go-version", default="v2.9.0") + parser.add_argument("--orchestrion-version", default="v1.12.0") + parser.add_argument("--dd-trace-go-version", default="v2.9.1") args = parser.parse_args(argv) try: if args.output_dir: diff --git a/tools/tests/integration/fixtures/orchestrion_pin_graph/go.mod b/tools/tests/integration/fixtures/orchestrion_pin_graph/go.mod index cd4694a1..2f5eae48 100644 --- a/tools/tests/integration/fixtures/orchestrion_pin_graph/go.mod +++ b/tools/tests/integration/fixtures/orchestrion_pin_graph/go.mod @@ -4,7 +4,7 @@ go 1.25.0 require ( github.com/DataDog/dd-trace-go/contrib/net/http/v2 v2.9.1-rc.3 - github.com/DataDog/dd-trace-go/v2 v2.9.1-rc.3 - github.com/DataDog/orchestrion v1.9.0 + github.com/DataDog/dd-trace-go/v2 v2.9.1 + github.com/DataDog/orchestrion v1.12.0 gopkg.in/DataDog/dd-trace-go.v1 v1.74.8 ) diff --git a/tools/tests/integration/fixtures/orchestrion_pin_graph/go.sum b/tools/tests/integration/fixtures/orchestrion_pin_graph/go.sum index b5b58172..b1cc86d5 100644 --- a/tools/tests/integration/fixtures/orchestrion_pin_graph/go.sum +++ b/tools/tests/integration/fixtures/orchestrion_pin_graph/go.sum @@ -2,9 +2,9 @@ github.com/DataDog/dd-trace-go/contrib/log/slog/v2 v2.3.0 h1:4b6P8HD3y0JaiXSTOAT github.com/DataDog/dd-trace-go/contrib/log/slog/v2 v2.3.0/go.mod h1:Ps0MUVRg2EhbFMXgyar4gzrtQ8QCUXwwvf4VfrvGOck= github.com/DataDog/dd-trace-go/contrib/net/http/v2 v2.9.1-rc.3 h1:Sa2Xl/25nkVZtsDRMMA0NtJavWEE+4GRc0fPoLznHZI= github.com/DataDog/dd-trace-go/contrib/net/http/v2 v2.9.1-rc.3/go.mod h1:L0HbDaTdLRK59szho4l7X7YsZ2OkSXgmskvfiOTPLSI= -github.com/DataDog/dd-trace-go/v2 v2.9.1-rc.3 h1:3pOaLc2C2JN0EdQV6Ea0uz724aJ7ECP3u5VGHoJnk2U= -github.com/DataDog/dd-trace-go/v2 v2.9.1-rc.3/go.mod h1:SdMkCESSBc2knx56Xol2pO7jhMDPi7MxyNj6vRYMW48= -github.com/DataDog/orchestrion v1.9.0 h1:TmjQfgaIMZDnGAmNXHIw5P7R+q4hOEJN5B/S24IqbKA= -github.com/DataDog/orchestrion v1.9.0/go.mod h1:FvbdNvK2PY3YnEIw0MHqdBELhZ0P7nUpWaJB3TgUtNE= +github.com/DataDog/dd-trace-go/v2 v2.9.1 h1:N2aqlWS0nAG5o+ETVyvz3gtboZbfemaD8Q/VumStGRY= +github.com/DataDog/dd-trace-go/v2 v2.9.1/go.mod h1:SdMkCESSBc2knx56Xol2pO7jhMDPi7MxyNj6vRYMW48= +github.com/DataDog/orchestrion v1.12.0 h1:GMdYBXkn3QbaU1mla2b7FN0uF9vJ2UJzrUH1X254Hc0= +github.com/DataDog/orchestrion v1.12.0/go.mod h1:Xg4S/8UP7LpmmeSR/pqhO7vKUpiohSJcU8VCDPkpBpA= gopkg.in/DataDog/dd-trace-go.v1 v1.74.8 h1:h96ji92t9eXbPvSWhJ+lrPWetHiQNYlt48JKRO09NFA= gopkg.in/DataDog/dd-trace-go.v1 v1.74.8/go.mod h1:LpHbtHsCZBlm1HWrlVOUQcEXwMWZnU6yMvmtd1GvSDI= diff --git a/tools/tests/integration/fixtures/orchestrion_pin_graph/missing_module/go.mod b/tools/tests/integration/fixtures/orchestrion_pin_graph/missing_module/go.mod index 4829f2ee..a4f3479c 100644 --- a/tools/tests/integration/fixtures/orchestrion_pin_graph/missing_module/go.mod +++ b/tools/tests/integration/fixtures/orchestrion_pin_graph/missing_module/go.mod @@ -4,6 +4,6 @@ go 1.25.0 require ( github.com/DataDog/dd-trace-go/contrib/net/http/v2 v2.9.1-rc.3 - github.com/DataDog/dd-trace-go/v2 v2.9.1-rc.3 - github.com/DataDog/orchestrion v1.9.0 + github.com/DataDog/dd-trace-go/v2 v2.9.1 + github.com/DataDog/orchestrion v1.12.0 ) diff --git a/tools/tests/integration/fixtures/orchestrion_pin_graph/missing_module/go.sum b/tools/tests/integration/fixtures/orchestrion_pin_graph/missing_module/go.sum index 1e7e5ec8..70f897e3 100644 --- a/tools/tests/integration/fixtures/orchestrion_pin_graph/missing_module/go.sum +++ b/tools/tests/integration/fixtures/orchestrion_pin_graph/missing_module/go.sum @@ -1,6 +1,6 @@ github.com/DataDog/dd-trace-go/contrib/net/http/v2 v2.9.1-rc.3 h1:Sa2Xl/25nkVZtsDRMMA0NtJavWEE+4GRc0fPoLznHZI= github.com/DataDog/dd-trace-go/contrib/net/http/v2 v2.9.1-rc.3/go.mod h1:L0HbDaTdLRK59szho4l7X7YsZ2OkSXgmskvfiOTPLSI= -github.com/DataDog/dd-trace-go/v2 v2.9.1-rc.3 h1:3pOaLc2C2JN0EdQV6Ea0uz724aJ7ECP3u5VGHoJnk2U= -github.com/DataDog/dd-trace-go/v2 v2.9.1-rc.3/go.mod h1:SdMkCESSBc2knx56Xol2pO7jhMDPi7MxyNj6vRYMW48= -github.com/DataDog/orchestrion v1.9.0 h1:TmjQfgaIMZDnGAmNXHIw5P7R+q4hOEJN5B/S24IqbKA= -github.com/DataDog/orchestrion v1.9.0/go.mod h1:FvbdNvK2PY3YnEIw0MHqdBELhZ0P7nUpWaJB3TgUtNE= +github.com/DataDog/dd-trace-go/v2 v2.9.1 h1:N2aqlWS0nAG5o+ETVyvz3gtboZbfemaD8Q/VumStGRY= +github.com/DataDog/dd-trace-go/v2 v2.9.1/go.mod h1:SdMkCESSBc2knx56Xol2pO7jhMDPi7MxyNj6vRYMW48= +github.com/DataDog/orchestrion v1.12.0 h1:GMdYBXkn3QbaU1mla2b7FN0uF9vJ2UJzrUH1X254Hc0= +github.com/DataDog/orchestrion v1.12.0/go.mod h1:Xg4S/8UP7LpmmeSR/pqhO7vKUpiohSJcU8VCDPkpBpA= diff --git a/tools/tests/integration/run_bzlmod_go_integration.sh b/tools/tests/integration/run_bzlmod_go_integration.sh index 1a4382d5..ab5e3f9b 100755 --- a/tools/tests/integration/run_bzlmod_go_integration.sh +++ b/tools/tests/integration/run_bzlmod_go_integration.sh @@ -41,7 +41,7 @@ BAZEL_VERSION="${BAZEL_VERSION:-$(tr -d '[:space:]' < "$REPO_ROOT/.bazelversion" # release downloaded SDKs, extracted repos, and sandbox outputs during cleanup. BAZEL_OUTPUT_USER_ROOT="${BAZEL_OUTPUT_USER_ROOT:-$TMP_ROOT/bazel_output_user_root}" GO_VERSION="${GO_VERSION:-1.25.0}" -ORCHESTRION_VERSION="${ORCHESTRION_VERSION:-v1.9.0}" +ORCHESTRION_VERSION="${ORCHESTRION_VERSION:-v1.12.0}" ORCHESTRION_MODE="${ORCHESTRION_MODE:-general}" ORCHESTRION_DISABLED_SENTINEL="${ORCHESTRION_DISABLED_SENTINEL:-0}" ORCHESTRION_DISABLED_SENTINEL_VERSION="v0.0.0-rto-disabled-fetch-sentinel" @@ -52,8 +52,8 @@ CONFIG_TRANSITION_ONLY="${CONFIG_TRANSITION_ONLY:-$WINDOWS_CONFIG_TRANSITION_ONL FORBID_HOST_GO="${FORBID_HOST_GO:-0}" EXPECTED_ORCHESTRION_CACHE_PHASE="${EXPECTED_ORCHESTRION_CACHE_PHASE:-}" HOST_GO_SENTINEL_LOG="" -DD_TRACE_GO_VERSION="${DD_TRACE_GO_VERSION:-v2.9.0}" -PIN_ROOT_VERSION="v2.9.1-rc.3" +DD_TRACE_GO_VERSION="${DD_TRACE_GO_VERSION:-v2.9.1}" +PIN_ROOT_VERSION="v2.9.1" PIN_HTTP_VERSION="v2.9.1-rc.3" PIN_SLOG_VERSION="v2.3.0" SERVICE_NAME="${SERVICE_NAME:-bzlmod-go-service}" diff --git a/tools/tests/integration/run_mock_server_tests.sh b/tools/tests/integration/run_mock_server_tests.sh index c9ad60f3..7d993381 100755 --- a/tools/tests/integration/run_mock_server_tests.sh +++ b/tools/tests/integration/run_mock_server_tests.sh @@ -40,7 +40,7 @@ SNAPSHOT_DIR="$REPO_ROOT/tools/tests/integration/snapshots" PYTHON="${PYTHON:-python3}" # Keep the mock-server harness aligned with the supported Orchestrion version # under test instead of relying on the old hardcoded bootstrap tag. -ORCHESTRION_VERSION="${ORCHESTRION_VERSION:-v1.9.0}" +ORCHESTRION_VERSION="${ORCHESTRION_VERSION:-v1.12.0}" export ORCHESTRION_VERSION GO_VERSION="${GO_VERSION:-1.25.0}" RULES_GO_UPSTREAM="${RULES_GO_UPSTREAM:-default}" @@ -3188,7 +3188,7 @@ cat > "$BOOT_WS/bin/go" <<'FAKE_GO_EOF' #!/bin/sh set -eu -ORCH_VERSION="${ORCHESTRION_VERSION:-v1.9.0}" +ORCH_VERSION="${ORCHESTRION_VERSION:-v1.12.0}" # The plain bootstrap scenario still validates file edits, but deterministic # proxy generation now resolves real modules during repository bootstrap. Keep @@ -3272,9 +3272,9 @@ fi if [ "${1:-}" = "mod" ] && [ "${2:-}" = "download" ]; then case "${3:-}" in github.com/DataDog/orchestrion@${ORCH_VERSION}|\ - github.com/DataDog/dd-trace-go/v2@v2.9.0|\ - github.com/DataDog/dd-trace-go/contrib/net/http/v2@v2.9.0|\ - github.com/DataDog/dd-trace-go/contrib/log/slog/v2@v2.9.0) + github.com/DataDog/dd-trace-go/v2@v2.9.1|\ + github.com/DataDog/dd-trace-go/contrib/net/http/v2@v2.9.1|\ + github.com/DataDog/dd-trace-go/contrib/log/slog/v2@v2.9.1) run_real_go "$@" exit 0 ;; @@ -3295,9 +3295,9 @@ if [ "${1:-}" = "mod" ] && [ "${2:-}" = "edit" ]; then ensure_require "github.com/DataDog/orchestrion" "${ORCH_VERSION}" exit 0 ;; - -require=github.com/DataDog/dd-trace-go/v2@v2.9.0|\ - -require=github.com/DataDog/dd-trace-go/contrib/net/http/v2@v2.9.0|\ - -require=github.com/DataDog/dd-trace-go/contrib/log/slog/v2@v2.9.0) + -require=github.com/DataDog/dd-trace-go/v2@v2.9.1|\ + -require=github.com/DataDog/dd-trace-go/contrib/net/http/v2@v2.9.1|\ + -require=github.com/DataDog/dd-trace-go/contrib/log/slog/v2@v2.9.1) module_and_version="${3#-require=}" module_path="${module_and_version%@*}" version="${module_and_version##*@}" @@ -3307,8 +3307,8 @@ if [ "${1:-}" = "mod" ] && [ "${2:-}" = "edit" ]; then esac fi -if [ "${1:-}" = "get" ] && [ "${2:-}" = "github.com/DataDog/dd-trace-go/v2/orchestrion@v2.9.0" ]; then - ensure_require "github.com/DataDog/dd-trace-go/v2" "v2.9.0" +if [ "${1:-}" = "get" ] && [ "${2:-}" = "github.com/DataDog/dd-trace-go/v2/orchestrion@v2.9.1" ]; then + ensure_require "github.com/DataDog/dd-trace-go/v2" "v2.9.1" exit 0 fi @@ -3322,7 +3322,7 @@ if [ "${1:-}" = "list" ] && [ "${2:-}" = "-m" ] && [ "${3:-}" = "-f" ] && [ "${4 github.com/DataDog/dd-trace-go/v2|\ github.com/DataDog/dd-trace-go/contrib/net/http/v2|\ github.com/DataDog/dd-trace-go/contrib/log/slog/v2) - printf 'v2.9.0\n' + printf 'v2.9.1\n' exit 0 ;; esac @@ -3330,10 +3330,10 @@ fi if [ "${1:-}" = "list" ] && [ "${2:-}" = "-m" ] && [ "${3:-}" = "-json" ]; then case "${4:-}" in - github.com/DataDog/dd-trace-go/v2@v2.9.0|\ - github.com/DataDog/dd-trace-go/contrib/net/http/v2@v2.9.0|\ - github.com/DataDog/dd-trace-go/contrib/log/slog/v2@v2.9.0) - printf '{"Version":"v2.9.0"}\n' + github.com/DataDog/dd-trace-go/v2@v2.9.1|\ + github.com/DataDog/dd-trace-go/contrib/net/http/v2@v2.9.1|\ + github.com/DataDog/dd-trace-go/contrib/log/slog/v2@v2.9.1) + printf '{"Version":"v2.9.1"}\n' exit 0 ;; esac @@ -3348,7 +3348,7 @@ if [ "${1:-}" = "list" ] && [ "${2:-}" = "-mod=mod" ] && [ "${3:-}" = "-m" ] && github.com/DataDog/dd-trace-go/v2|\ github.com/DataDog/dd-trace-go/contrib/net/http/v2|\ github.com/DataDog/dd-trace-go/contrib/log/slog/v2) - printf '{"Version":"v2.9.0"}\n' + printf '{"Version":"v2.9.1"}\n' exit 0 ;; esac @@ -3522,7 +3522,7 @@ cat > "$GUIDED_BOOT_WS/bin/go" <<'FAKE_GO_GUIDED_EOF' #!/bin/sh set -eu -ORCH_VERSION="${ORCHESTRION_VERSION:-v1.9.0}" +ORCH_VERSION="${ORCHESTRION_VERSION:-v1.12.0}" # The guided bootstrap scenario later builds a real Go test, so the fake Go # tool delegates the download-heavy paths to the host Go binary using temporary @@ -3605,9 +3605,9 @@ fi if [ "${1:-}" = "mod" ] && [ "${2:-}" = "download" ]; then case "${3:-}" in github.com/DataDog/orchestrion@${ORCH_VERSION}|\ - github.com/DataDog/dd-trace-go/v2@v2.9.0|\ - github.com/DataDog/dd-trace-go/contrib/net/http/v2@v2.9.0|\ - github.com/DataDog/dd-trace-go/contrib/log/slog/v2@v2.9.0) + github.com/DataDog/dd-trace-go/v2@v2.9.1|\ + github.com/DataDog/dd-trace-go/contrib/net/http/v2@v2.9.1|\ + github.com/DataDog/dd-trace-go/contrib/log/slog/v2@v2.9.1) run_real_go "$@" exit 0 ;; @@ -3620,9 +3620,9 @@ if [ "${1:-}" = "mod" ] && [ "${2:-}" = "edit" ]; then ensure_require "github.com/DataDog/orchestrion" "${ORCH_VERSION}" exit 0 ;; - -require=github.com/DataDog/dd-trace-go/v2@v2.9.0|\ - -require=github.com/DataDog/dd-trace-go/contrib/net/http/v2@v2.9.0|\ - -require=github.com/DataDog/dd-trace-go/contrib/log/slog/v2@v2.9.0) + -require=github.com/DataDog/dd-trace-go/v2@v2.9.1|\ + -require=github.com/DataDog/dd-trace-go/contrib/net/http/v2@v2.9.1|\ + -require=github.com/DataDog/dd-trace-go/contrib/log/slog/v2@v2.9.1) module_and_version="${3#-require=}" module_path="${module_and_version%@*}" version="${module_and_version##*@}" @@ -3632,8 +3632,8 @@ if [ "${1:-}" = "mod" ] && [ "${2:-}" = "edit" ]; then esac fi -if [ "${1:-}" = "get" ] && [ "${2:-}" = "github.com/DataDog/dd-trace-go/v2/orchestrion@v2.9.0" ]; then - ensure_require "github.com/DataDog/dd-trace-go/v2" "v2.9.0" +if [ "${1:-}" = "get" ] && [ "${2:-}" = "github.com/DataDog/dd-trace-go/v2/orchestrion@v2.9.1" ]; then + ensure_require "github.com/DataDog/dd-trace-go/v2" "v2.9.1" exit 0 fi @@ -3647,7 +3647,7 @@ if [ "${1:-}" = "list" ] && [ "${2:-}" = "-m" ] && [ "${3:-}" = "-f" ] && [ "${4 github.com/DataDog/dd-trace-go/v2|\ github.com/DataDog/dd-trace-go/contrib/net/http/v2|\ github.com/DataDog/dd-trace-go/contrib/log/slog/v2) - printf 'v2.9.0\n' + printf 'v2.9.1\n' exit 0 ;; esac @@ -3655,10 +3655,10 @@ fi if [ "${1:-}" = "list" ] && [ "${2:-}" = "-m" ] && [ "${3:-}" = "-json" ]; then case "${4:-}" in - github.com/DataDog/dd-trace-go/v2@v2.9.0|\ - github.com/DataDog/dd-trace-go/contrib/net/http/v2@v2.9.0|\ - github.com/DataDog/dd-trace-go/contrib/log/slog/v2@v2.9.0) - printf '{"Version":"v2.9.0"}\n' + github.com/DataDog/dd-trace-go/v2@v2.9.1|\ + github.com/DataDog/dd-trace-go/contrib/net/http/v2@v2.9.1|\ + github.com/DataDog/dd-trace-go/contrib/log/slog/v2@v2.9.1) + printf '{"Version":"v2.9.1"}\n' exit 0 ;; esac @@ -3673,7 +3673,7 @@ if [ "${1:-}" = "list" ] && [ "${2:-}" = "-mod=mod" ] && [ "${3:-}" = "-m" ] && github.com/DataDog/dd-trace-go/v2|\ github.com/DataDog/dd-trace-go/contrib/net/http/v2|\ github.com/DataDog/dd-trace-go/contrib/log/slog/v2) - printf '{"Version":"v2.9.0"}\n' + printf '{"Version":"v2.9.1"}\n' exit 0 ;; esac @@ -4032,7 +4032,7 @@ if ! grep -q 'orchestrion_mode = "test_optimization"' "$GUIDED_BOOT_WS/tools/bui exit 1 fi -# The release tracer pinned by default in this fixture is v2.9.0. The generated +# The release tracer pinned by default in this fixture is v2.9.1. The generated # wrapper above must still default to test_optimization mode, but that release # does not include the unreleased Go testing Orchestrion package needed to build # a real instrumented test binary. Run the raw target for this temporary runtime diff --git a/tools/tests/integration/run_workspace_go_integration.sh b/tools/tests/integration/run_workspace_go_integration.sh index b82a250a..c7f51cae 100755 --- a/tools/tests/integration/run_workspace_go_integration.sh +++ b/tools/tests/integration/run_workspace_go_integration.sh @@ -44,7 +44,7 @@ BAZEL_VERSION="${BAZEL_VERSION:-$(tr -d '[:space:]' < "$REPO_ROOT/.bazelversion" # release downloaded SDKs, extracted repos, and sandbox outputs during cleanup. BAZEL_OUTPUT_USER_ROOT="${BAZEL_OUTPUT_USER_ROOT:-$TMP_ROOT/bazel_output_user_root}" GO_VERSION="${GO_VERSION:-1.25.0}" -ORCHESTRION_VERSION="${ORCHESTRION_VERSION:-v1.9.0}" +ORCHESTRION_VERSION="${ORCHESTRION_VERSION:-v1.12.0}" ORCHESTRION_MODE="${ORCHESTRION_MODE:-general}" ORCHESTRION_DISABLED_SENTINEL="${ORCHESTRION_DISABLED_SENTINEL:-0}" ORCHESTRION_DISABLED_SENTINEL_VERSION="v0.0.0-rto-disabled-fetch-sentinel" @@ -57,8 +57,8 @@ EXPECTED_ORCHESTRION_CACHE_PHASE="${EXPECTED_ORCHESTRION_CACHE_PHASE:-}" HOST_GO_SENTINEL_LOG="" # Keep this aligned with the bootstrap helper's published default tracer pin so # the WORKSPACE harness validates the same public Go path the docs describe. -DD_TRACE_GO_VERSION="${DD_TRACE_GO_VERSION:-v2.9.0}" -PIN_ROOT_VERSION="v2.9.1-rc.3" +DD_TRACE_GO_VERSION="${DD_TRACE_GO_VERSION:-v2.9.1}" +PIN_ROOT_VERSION="v2.9.1" PIN_HTTP_VERSION="v2.9.1-rc.3" PIN_SLOG_VERSION="v2.3.0" SERVICE_NAME="${SERVICE_NAME:-workspace-go-service}" diff --git a/tools/tests/python/test_rules_go_profile_tools.py b/tools/tests/python/test_rules_go_profile_tools.py index fd59576b..42c75b67 100644 --- a/tools/tests/python/test_rules_go_profile_tools.py +++ b/tools/tests/python/test_rules_go_profile_tools.py @@ -397,8 +397,8 @@ def test_smoke_workspace_wires_hermetic_go_sdk_into_orchestrion(self) -> None: workspace=workspace, rules_go_root=root / "rules_go", go_version="1.25.0", - orchestrion_version="v1.9.0", - dd_trace_go_version="v2.9.0", + orchestrion_version="v1.12.0", + dd_trace_go_version="v2.9.1", ) workspace_text = (workspace / "WORKSPACE").read_text(encoding="utf-8") From 73172ce3bf6268068b277beb4a39abd7c39e714f Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Fri, 7 Aug 2026 22:44:59 +0200 Subject: [PATCH 02/37] go: preserve exact tracer pins in offline proxy --- .../go/private/orchestrion/extensions.bzl | 20 ++++++++ .../starlark/orchestrion_extension_tests.bzl | 20 ++++++++ .../go/private/orchestrion/extensions.bzl | 20 ++++++++ .../starlark/orchestrion_extension_tests.bzl | 20 ++++++++ .../go/private/orchestrion/extensions.bzl | 20 ++++++++ .../starlark/orchestrion_extension_tests.bzl | 20 ++++++++ .../v0_60_0/base/0001-full-delta.patch | 48 +++++++++++++++++-- .../v0_61_1/base/0001-full-delta.patch | 48 +++++++++++++++++-- .../v0_62_0/base/0001-full-delta.patch | 48 +++++++++++++++++-- .../fixtures/orchestrion_pin_graph/go.mod | 3 +- .../fixtures/orchestrion_pin_graph/go.sum | 6 +-- .../integration/run_bzlmod_go_integration.sh | 2 +- .../run_workspace_go_integration.sh | 2 +- 13 files changed, 257 insertions(+), 20 deletions(-) diff --git a/third_party/rgo/v0_60_0/base/go/private/orchestrion/extensions.bzl b/third_party/rgo/v0_60_0/base/go/private/orchestrion/extensions.bzl index a56e5681..5b757a9f 100644 --- a/third_party/rgo/v0_60_0/base/go/private/orchestrion/extensions.bzl +++ b/third_party/rgo/v0_60_0/base/go/private/orchestrion/extensions.bzl @@ -764,6 +764,12 @@ def _module_proxy_resolved_modules_json(resolved_versions): entries.append(' "%s": "%s"' % (module_path, resolved_versions[module_path])) return "{\n%s\n}\n" % ",\n".join(entries) +def _module_proxy_exact_dd_trace_go_queries(version_map): + return [ + "%s@%s" % (module_path, version_map[module_path]) + for module_path in _DD_TRACE_GO_MODULES + ] + def _split_go_list_json_objects(output, error_prefix): objects = [] current = [] @@ -1038,6 +1044,19 @@ def _write_orchestrion_module_proxy(ctx, go_path, version, version_map): "Failed to download Orchestrion module proxy graph", ) + _ctx_execute_or_fail( + ctx, + [ + str(go_path), + "-C", + seed_dir, + "mod", + "download", + ] + _module_proxy_exact_dd_trace_go_queries(version_map), + seed_env, + "Failed to download configured dd-trace-go module versions", + ) + seed_go_sum_path = seed_dir + "/go.sum" if not ctx.path(seed_go_sum_path).exists: fail("Failed to generate Orchestrion module proxy seed go.sum in %s" % seed_dir) @@ -1218,6 +1237,7 @@ orchestrion_extension_test_helpers = struct( go_module_fetch_env = _go_module_fetch_env, host_path_is_writable = _host_path_is_writable, module_proxy_resolved_modules_json = _module_proxy_resolved_modules_json, + module_proxy_exact_dd_trace_go_queries = _module_proxy_exact_dd_trace_go_queries, module_proxy_seed_go_mod = _module_proxy_seed_go_mod, normalize_host_goarch = _normalize_host_goarch, normalize_host_goos = _normalize_host_goos, diff --git a/third_party/rgo/v0_60_0/base/tests/core/starlark/orchestrion_extension_tests.bzl b/third_party/rgo/v0_60_0/base/tests/core/starlark/orchestrion_extension_tests.bzl index 48b94524..ec4826f1 100644 --- a/third_party/rgo/v0_60_0/base/tests/core/starlark/orchestrion_extension_tests.bzl +++ b/third_party/rgo/v0_60_0/base/tests/core/starlark/orchestrion_extension_tests.bzl @@ -122,6 +122,25 @@ def _module_proxy_resolved_modules_json_test(ctx): module_proxy_resolved_modules_json_test = unittest.make(_module_proxy_resolved_modules_json_test) +def _module_proxy_exact_dd_trace_go_queries_test(ctx): + env = unittest.begin(ctx) + + queries = orchestrion_extension_test_helpers.module_proxy_exact_dd_trace_go_queries({ + "github.com/DataDog/dd-trace-go/v2": "v2.9.1-rc.3", + "github.com/DataDog/dd-trace-go/contrib/net/http/v2": "v2.9.1-rc.3", + "github.com/DataDog/dd-trace-go/contrib/log/slog/v2": "v2.3.0", + }) + + asserts.equals(env, [ + "github.com/DataDog/dd-trace-go/v2@v2.9.1-rc.3", + "github.com/DataDog/dd-trace-go/contrib/net/http/v2@v2.9.1-rc.3", + "github.com/DataDog/dd-trace-go/contrib/log/slog/v2@v2.3.0", + ], queries) + + return unittest.end(env) + +module_proxy_exact_dd_trace_go_queries_test = unittest.make(_module_proxy_exact_dd_trace_go_queries_test) + def _parse_certutil_sha256_test(ctx): env = unittest.begin(ctx) @@ -283,6 +302,7 @@ def orchestrion_extension_test_suite(): git_env_test, go_module_fetch_env_test, host_platform_normalization_test, + module_proxy_exact_dd_trace_go_queries_test, module_proxy_resolved_modules_json_test, module_proxy_seed_go_mod_test, parse_certutil_sha256_test, diff --git a/third_party/rgo/v0_61_1/base/go/private/orchestrion/extensions.bzl b/third_party/rgo/v0_61_1/base/go/private/orchestrion/extensions.bzl index a56e5681..5b757a9f 100644 --- a/third_party/rgo/v0_61_1/base/go/private/orchestrion/extensions.bzl +++ b/third_party/rgo/v0_61_1/base/go/private/orchestrion/extensions.bzl @@ -764,6 +764,12 @@ def _module_proxy_resolved_modules_json(resolved_versions): entries.append(' "%s": "%s"' % (module_path, resolved_versions[module_path])) return "{\n%s\n}\n" % ",\n".join(entries) +def _module_proxy_exact_dd_trace_go_queries(version_map): + return [ + "%s@%s" % (module_path, version_map[module_path]) + for module_path in _DD_TRACE_GO_MODULES + ] + def _split_go_list_json_objects(output, error_prefix): objects = [] current = [] @@ -1038,6 +1044,19 @@ def _write_orchestrion_module_proxy(ctx, go_path, version, version_map): "Failed to download Orchestrion module proxy graph", ) + _ctx_execute_or_fail( + ctx, + [ + str(go_path), + "-C", + seed_dir, + "mod", + "download", + ] + _module_proxy_exact_dd_trace_go_queries(version_map), + seed_env, + "Failed to download configured dd-trace-go module versions", + ) + seed_go_sum_path = seed_dir + "/go.sum" if not ctx.path(seed_go_sum_path).exists: fail("Failed to generate Orchestrion module proxy seed go.sum in %s" % seed_dir) @@ -1218,6 +1237,7 @@ orchestrion_extension_test_helpers = struct( go_module_fetch_env = _go_module_fetch_env, host_path_is_writable = _host_path_is_writable, module_proxy_resolved_modules_json = _module_proxy_resolved_modules_json, + module_proxy_exact_dd_trace_go_queries = _module_proxy_exact_dd_trace_go_queries, module_proxy_seed_go_mod = _module_proxy_seed_go_mod, normalize_host_goarch = _normalize_host_goarch, normalize_host_goos = _normalize_host_goos, diff --git a/third_party/rgo/v0_61_1/base/tests/core/starlark/orchestrion_extension_tests.bzl b/third_party/rgo/v0_61_1/base/tests/core/starlark/orchestrion_extension_tests.bzl index 48b94524..ec4826f1 100644 --- a/third_party/rgo/v0_61_1/base/tests/core/starlark/orchestrion_extension_tests.bzl +++ b/third_party/rgo/v0_61_1/base/tests/core/starlark/orchestrion_extension_tests.bzl @@ -122,6 +122,25 @@ def _module_proxy_resolved_modules_json_test(ctx): module_proxy_resolved_modules_json_test = unittest.make(_module_proxy_resolved_modules_json_test) +def _module_proxy_exact_dd_trace_go_queries_test(ctx): + env = unittest.begin(ctx) + + queries = orchestrion_extension_test_helpers.module_proxy_exact_dd_trace_go_queries({ + "github.com/DataDog/dd-trace-go/v2": "v2.9.1-rc.3", + "github.com/DataDog/dd-trace-go/contrib/net/http/v2": "v2.9.1-rc.3", + "github.com/DataDog/dd-trace-go/contrib/log/slog/v2": "v2.3.0", + }) + + asserts.equals(env, [ + "github.com/DataDog/dd-trace-go/v2@v2.9.1-rc.3", + "github.com/DataDog/dd-trace-go/contrib/net/http/v2@v2.9.1-rc.3", + "github.com/DataDog/dd-trace-go/contrib/log/slog/v2@v2.3.0", + ], queries) + + return unittest.end(env) + +module_proxy_exact_dd_trace_go_queries_test = unittest.make(_module_proxy_exact_dd_trace_go_queries_test) + def _parse_certutil_sha256_test(ctx): env = unittest.begin(ctx) @@ -283,6 +302,7 @@ def orchestrion_extension_test_suite(): git_env_test, go_module_fetch_env_test, host_platform_normalization_test, + module_proxy_exact_dd_trace_go_queries_test, module_proxy_resolved_modules_json_test, module_proxy_seed_go_mod_test, parse_certutil_sha256_test, diff --git a/third_party/rgo/v0_62_0/base/go/private/orchestrion/extensions.bzl b/third_party/rgo/v0_62_0/base/go/private/orchestrion/extensions.bzl index a56e5681..5b757a9f 100644 --- a/third_party/rgo/v0_62_0/base/go/private/orchestrion/extensions.bzl +++ b/third_party/rgo/v0_62_0/base/go/private/orchestrion/extensions.bzl @@ -764,6 +764,12 @@ def _module_proxy_resolved_modules_json(resolved_versions): entries.append(' "%s": "%s"' % (module_path, resolved_versions[module_path])) return "{\n%s\n}\n" % ",\n".join(entries) +def _module_proxy_exact_dd_trace_go_queries(version_map): + return [ + "%s@%s" % (module_path, version_map[module_path]) + for module_path in _DD_TRACE_GO_MODULES + ] + def _split_go_list_json_objects(output, error_prefix): objects = [] current = [] @@ -1038,6 +1044,19 @@ def _write_orchestrion_module_proxy(ctx, go_path, version, version_map): "Failed to download Orchestrion module proxy graph", ) + _ctx_execute_or_fail( + ctx, + [ + str(go_path), + "-C", + seed_dir, + "mod", + "download", + ] + _module_proxy_exact_dd_trace_go_queries(version_map), + seed_env, + "Failed to download configured dd-trace-go module versions", + ) + seed_go_sum_path = seed_dir + "/go.sum" if not ctx.path(seed_go_sum_path).exists: fail("Failed to generate Orchestrion module proxy seed go.sum in %s" % seed_dir) @@ -1218,6 +1237,7 @@ orchestrion_extension_test_helpers = struct( go_module_fetch_env = _go_module_fetch_env, host_path_is_writable = _host_path_is_writable, module_proxy_resolved_modules_json = _module_proxy_resolved_modules_json, + module_proxy_exact_dd_trace_go_queries = _module_proxy_exact_dd_trace_go_queries, module_proxy_seed_go_mod = _module_proxy_seed_go_mod, normalize_host_goarch = _normalize_host_goarch, normalize_host_goos = _normalize_host_goos, diff --git a/third_party/rgo/v0_62_0/base/tests/core/starlark/orchestrion_extension_tests.bzl b/third_party/rgo/v0_62_0/base/tests/core/starlark/orchestrion_extension_tests.bzl index 48b94524..ec4826f1 100644 --- a/third_party/rgo/v0_62_0/base/tests/core/starlark/orchestrion_extension_tests.bzl +++ b/third_party/rgo/v0_62_0/base/tests/core/starlark/orchestrion_extension_tests.bzl @@ -122,6 +122,25 @@ def _module_proxy_resolved_modules_json_test(ctx): module_proxy_resolved_modules_json_test = unittest.make(_module_proxy_resolved_modules_json_test) +def _module_proxy_exact_dd_trace_go_queries_test(ctx): + env = unittest.begin(ctx) + + queries = orchestrion_extension_test_helpers.module_proxy_exact_dd_trace_go_queries({ + "github.com/DataDog/dd-trace-go/v2": "v2.9.1-rc.3", + "github.com/DataDog/dd-trace-go/contrib/net/http/v2": "v2.9.1-rc.3", + "github.com/DataDog/dd-trace-go/contrib/log/slog/v2": "v2.3.0", + }) + + asserts.equals(env, [ + "github.com/DataDog/dd-trace-go/v2@v2.9.1-rc.3", + "github.com/DataDog/dd-trace-go/contrib/net/http/v2@v2.9.1-rc.3", + "github.com/DataDog/dd-trace-go/contrib/log/slog/v2@v2.3.0", + ], queries) + + return unittest.end(env) + +module_proxy_exact_dd_trace_go_queries_test = unittest.make(_module_proxy_exact_dd_trace_go_queries_test) + def _parse_certutil_sha256_test(ctx): env = unittest.begin(ctx) @@ -283,6 +302,7 @@ def orchestrion_extension_test_suite(): git_env_test, go_module_fetch_env_test, host_platform_normalization_test, + module_proxy_exact_dd_trace_go_queries_test, module_proxy_resolved_modules_json_test, module_proxy_seed_go_mod_test, parse_certutil_sha256_test, diff --git a/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch index f29b0d00..22355f51 100644 --- a/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch @@ -2091,10 +2091,10 @@ index 0000000..31a01c3 +) diff --git a/go/private/orchestrion/extensions.bzl b/go/private/orchestrion/extensions.bzl new file mode 100644 -index 0000000..a56e568 +index 0000000..5b757a9 --- /dev/null +++ b/go/private/orchestrion/extensions.bzl -@@ -0,0 +1,1719 @@ +@@ -0,0 +1,1739 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); @@ -2861,6 +2861,12 @@ index 0000000..a56e568 + entries.append(' "%s": "%s"' % (module_path, resolved_versions[module_path])) + return "{\n%s\n}\n" % ",\n".join(entries) + ++def _module_proxy_exact_dd_trace_go_queries(version_map): ++ return [ ++ "%s@%s" % (module_path, version_map[module_path]) ++ for module_path in _DD_TRACE_GO_MODULES ++ ] ++ +def _split_go_list_json_objects(output, error_prefix): + objects = [] + current = [] @@ -3135,6 +3141,19 @@ index 0000000..a56e568 + "Failed to download Orchestrion module proxy graph", + ) + ++ _ctx_execute_or_fail( ++ ctx, ++ [ ++ str(go_path), ++ "-C", ++ seed_dir, ++ "mod", ++ "download", ++ ] + _module_proxy_exact_dd_trace_go_queries(version_map), ++ seed_env, ++ "Failed to download configured dd-trace-go module versions", ++ ) ++ + seed_go_sum_path = seed_dir + "/go.sum" + if not ctx.path(seed_go_sum_path).exists: + fail("Failed to generate Orchestrion module proxy seed go.sum in %s" % seed_dir) @@ -3315,6 +3334,7 @@ index 0000000..a56e568 + go_module_fetch_env = _go_module_fetch_env, + host_path_is_writable = _host_path_is_writable, + module_proxy_resolved_modules_json = _module_proxy_resolved_modules_json, ++ module_proxy_exact_dd_trace_go_queries = _module_proxy_exact_dd_trace_go_queries, + module_proxy_seed_go_mod = _module_proxy_seed_go_mod, + normalize_host_goarch = _normalize_host_goarch, + normalize_host_goos = _normalize_host_goos, @@ -16497,10 +16517,10 @@ index a1cd2ea..406e52f 100644 ) diff --git a/tests/core/starlark/orchestrion_extension_tests.bzl b/tests/core/starlark/orchestrion_extension_tests.bzl new file mode 100644 -index 0000000..48b9452 +index 0000000..ec4826f --- /dev/null +++ b/tests/core/starlark/orchestrion_extension_tests.bzl -@@ -0,0 +1,290 @@ +@@ -0,0 +1,310 @@ +load("@bazel_skylib//lib:unittest.bzl", "asserts", "unittest") +load("//go/private/orchestrion:extensions.bzl", "orchestrion_extension_test_helpers") + @@ -16625,6 +16645,25 @@ index 0000000..48b9452 + +module_proxy_resolved_modules_json_test = unittest.make(_module_proxy_resolved_modules_json_test) + ++def _module_proxy_exact_dd_trace_go_queries_test(ctx): ++ env = unittest.begin(ctx) ++ ++ queries = orchestrion_extension_test_helpers.module_proxy_exact_dd_trace_go_queries({ ++ "github.com/DataDog/dd-trace-go/v2": "v2.9.1-rc.3", ++ "github.com/DataDog/dd-trace-go/contrib/net/http/v2": "v2.9.1-rc.3", ++ "github.com/DataDog/dd-trace-go/contrib/log/slog/v2": "v2.3.0", ++ }) ++ ++ asserts.equals(env, [ ++ "github.com/DataDog/dd-trace-go/v2@v2.9.1-rc.3", ++ "github.com/DataDog/dd-trace-go/contrib/net/http/v2@v2.9.1-rc.3", ++ "github.com/DataDog/dd-trace-go/contrib/log/slog/v2@v2.3.0", ++ ], queries) ++ ++ return unittest.end(env) ++ ++module_proxy_exact_dd_trace_go_queries_test = unittest.make(_module_proxy_exact_dd_trace_go_queries_test) ++ +def _parse_certutil_sha256_test(ctx): + env = unittest.begin(ctx) + @@ -16786,6 +16825,7 @@ index 0000000..48b9452 + git_env_test, + go_module_fetch_env_test, + host_platform_normalization_test, ++ module_proxy_exact_dd_trace_go_queries_test, + module_proxy_resolved_modules_json_test, + module_proxy_seed_go_mod_test, + parse_certutil_sha256_test, diff --git a/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch index b668a4f6..cf1b01fe 100644 --- a/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch @@ -2084,10 +2084,10 @@ index 0000000..177eeaf +) diff --git a/go/private/orchestrion/extensions.bzl b/go/private/orchestrion/extensions.bzl new file mode 100644 -index 0000000..a56e568 +index 0000000..5b757a9 --- /dev/null +++ b/go/private/orchestrion/extensions.bzl -@@ -0,0 +1,1719 @@ +@@ -0,0 +1,1739 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); @@ -2854,6 +2854,12 @@ index 0000000..a56e568 + entries.append(' "%s": "%s"' % (module_path, resolved_versions[module_path])) + return "{\n%s\n}\n" % ",\n".join(entries) + ++def _module_proxy_exact_dd_trace_go_queries(version_map): ++ return [ ++ "%s@%s" % (module_path, version_map[module_path]) ++ for module_path in _DD_TRACE_GO_MODULES ++ ] ++ +def _split_go_list_json_objects(output, error_prefix): + objects = [] + current = [] @@ -3128,6 +3134,19 @@ index 0000000..a56e568 + "Failed to download Orchestrion module proxy graph", + ) + ++ _ctx_execute_or_fail( ++ ctx, ++ [ ++ str(go_path), ++ "-C", ++ seed_dir, ++ "mod", ++ "download", ++ ] + _module_proxy_exact_dd_trace_go_queries(version_map), ++ seed_env, ++ "Failed to download configured dd-trace-go module versions", ++ ) ++ + seed_go_sum_path = seed_dir + "/go.sum" + if not ctx.path(seed_go_sum_path).exists: + fail("Failed to generate Orchestrion module proxy seed go.sum in %s" % seed_dir) @@ -3308,6 +3327,7 @@ index 0000000..a56e568 + go_module_fetch_env = _go_module_fetch_env, + host_path_is_writable = _host_path_is_writable, + module_proxy_resolved_modules_json = _module_proxy_resolved_modules_json, ++ module_proxy_exact_dd_trace_go_queries = _module_proxy_exact_dd_trace_go_queries, + module_proxy_seed_go_mod = _module_proxy_seed_go_mod, + normalize_host_goarch = _normalize_host_goarch, + normalize_host_goos = _normalize_host_goos, @@ -16487,10 +16507,10 @@ index a1cd2ea..406e52f 100644 ) diff --git a/tests/core/starlark/orchestrion_extension_tests.bzl b/tests/core/starlark/orchestrion_extension_tests.bzl new file mode 100644 -index 0000000..48b9452 +index 0000000..ec4826f --- /dev/null +++ b/tests/core/starlark/orchestrion_extension_tests.bzl -@@ -0,0 +1,290 @@ +@@ -0,0 +1,310 @@ +load("@bazel_skylib//lib:unittest.bzl", "asserts", "unittest") +load("//go/private/orchestrion:extensions.bzl", "orchestrion_extension_test_helpers") + @@ -16615,6 +16635,25 @@ index 0000000..48b9452 + +module_proxy_resolved_modules_json_test = unittest.make(_module_proxy_resolved_modules_json_test) + ++def _module_proxy_exact_dd_trace_go_queries_test(ctx): ++ env = unittest.begin(ctx) ++ ++ queries = orchestrion_extension_test_helpers.module_proxy_exact_dd_trace_go_queries({ ++ "github.com/DataDog/dd-trace-go/v2": "v2.9.1-rc.3", ++ "github.com/DataDog/dd-trace-go/contrib/net/http/v2": "v2.9.1-rc.3", ++ "github.com/DataDog/dd-trace-go/contrib/log/slog/v2": "v2.3.0", ++ }) ++ ++ asserts.equals(env, [ ++ "github.com/DataDog/dd-trace-go/v2@v2.9.1-rc.3", ++ "github.com/DataDog/dd-trace-go/contrib/net/http/v2@v2.9.1-rc.3", ++ "github.com/DataDog/dd-trace-go/contrib/log/slog/v2@v2.3.0", ++ ], queries) ++ ++ return unittest.end(env) ++ ++module_proxy_exact_dd_trace_go_queries_test = unittest.make(_module_proxy_exact_dd_trace_go_queries_test) ++ +def _parse_certutil_sha256_test(ctx): + env = unittest.begin(ctx) + @@ -16776,6 +16815,7 @@ index 0000000..48b9452 + git_env_test, + go_module_fetch_env_test, + host_platform_normalization_test, ++ module_proxy_exact_dd_trace_go_queries_test, + module_proxy_resolved_modules_json_test, + module_proxy_seed_go_mod_test, + parse_certutil_sha256_test, diff --git a/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch index 62dd8307..7e373ad7 100644 --- a/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch @@ -2088,10 +2088,10 @@ index 0000000..177eeaf +) diff --git a/go/private/orchestrion/extensions.bzl b/go/private/orchestrion/extensions.bzl new file mode 100644 -index 0000000..a56e568 +index 0000000..5b757a9 --- /dev/null +++ b/go/private/orchestrion/extensions.bzl -@@ -0,0 +1,1719 @@ +@@ -0,0 +1,1739 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); @@ -2858,6 +2858,12 @@ index 0000000..a56e568 + entries.append(' "%s": "%s"' % (module_path, resolved_versions[module_path])) + return "{\n%s\n}\n" % ",\n".join(entries) + ++def _module_proxy_exact_dd_trace_go_queries(version_map): ++ return [ ++ "%s@%s" % (module_path, version_map[module_path]) ++ for module_path in _DD_TRACE_GO_MODULES ++ ] ++ +def _split_go_list_json_objects(output, error_prefix): + objects = [] + current = [] @@ -3132,6 +3138,19 @@ index 0000000..a56e568 + "Failed to download Orchestrion module proxy graph", + ) + ++ _ctx_execute_or_fail( ++ ctx, ++ [ ++ str(go_path), ++ "-C", ++ seed_dir, ++ "mod", ++ "download", ++ ] + _module_proxy_exact_dd_trace_go_queries(version_map), ++ seed_env, ++ "Failed to download configured dd-trace-go module versions", ++ ) ++ + seed_go_sum_path = seed_dir + "/go.sum" + if not ctx.path(seed_go_sum_path).exists: + fail("Failed to generate Orchestrion module proxy seed go.sum in %s" % seed_dir) @@ -3312,6 +3331,7 @@ index 0000000..a56e568 + go_module_fetch_env = _go_module_fetch_env, + host_path_is_writable = _host_path_is_writable, + module_proxy_resolved_modules_json = _module_proxy_resolved_modules_json, ++ module_proxy_exact_dd_trace_go_queries = _module_proxy_exact_dd_trace_go_queries, + module_proxy_seed_go_mod = _module_proxy_seed_go_mod, + normalize_host_goarch = _normalize_host_goarch, + normalize_host_goos = _normalize_host_goos, @@ -16491,10 +16511,10 @@ index a1cd2ea..406e52f 100644 ) diff --git a/tests/core/starlark/orchestrion_extension_tests.bzl b/tests/core/starlark/orchestrion_extension_tests.bzl new file mode 100644 -index 0000000..48b9452 +index 0000000..ec4826f --- /dev/null +++ b/tests/core/starlark/orchestrion_extension_tests.bzl -@@ -0,0 +1,290 @@ +@@ -0,0 +1,310 @@ +load("@bazel_skylib//lib:unittest.bzl", "asserts", "unittest") +load("//go/private/orchestrion:extensions.bzl", "orchestrion_extension_test_helpers") + @@ -16619,6 +16639,25 @@ index 0000000..48b9452 + +module_proxy_resolved_modules_json_test = unittest.make(_module_proxy_resolved_modules_json_test) + ++def _module_proxy_exact_dd_trace_go_queries_test(ctx): ++ env = unittest.begin(ctx) ++ ++ queries = orchestrion_extension_test_helpers.module_proxy_exact_dd_trace_go_queries({ ++ "github.com/DataDog/dd-trace-go/v2": "v2.9.1-rc.3", ++ "github.com/DataDog/dd-trace-go/contrib/net/http/v2": "v2.9.1-rc.3", ++ "github.com/DataDog/dd-trace-go/contrib/log/slog/v2": "v2.3.0", ++ }) ++ ++ asserts.equals(env, [ ++ "github.com/DataDog/dd-trace-go/v2@v2.9.1-rc.3", ++ "github.com/DataDog/dd-trace-go/contrib/net/http/v2@v2.9.1-rc.3", ++ "github.com/DataDog/dd-trace-go/contrib/log/slog/v2@v2.3.0", ++ ], queries) ++ ++ return unittest.end(env) ++ ++module_proxy_exact_dd_trace_go_queries_test = unittest.make(_module_proxy_exact_dd_trace_go_queries_test) ++ +def _parse_certutil_sha256_test(ctx): + env = unittest.begin(ctx) + @@ -16780,6 +16819,7 @@ index 0000000..48b9452 + git_env_test, + go_module_fetch_env_test, + host_platform_normalization_test, ++ module_proxy_exact_dd_trace_go_queries_test, + module_proxy_resolved_modules_json_test, + module_proxy_seed_go_mod_test, + parse_certutil_sha256_test, diff --git a/tools/tests/integration/fixtures/orchestrion_pin_graph/go.mod b/tools/tests/integration/fixtures/orchestrion_pin_graph/go.mod index 2f5eae48..183dc814 100644 --- a/tools/tests/integration/fixtures/orchestrion_pin_graph/go.mod +++ b/tools/tests/integration/fixtures/orchestrion_pin_graph/go.mod @@ -4,7 +4,6 @@ go 1.25.0 require ( github.com/DataDog/dd-trace-go/contrib/net/http/v2 v2.9.1-rc.3 - github.com/DataDog/dd-trace-go/v2 v2.9.1 - github.com/DataDog/orchestrion v1.12.0 + github.com/DataDog/dd-trace-go/v2 v2.9.1-rc.3 gopkg.in/DataDog/dd-trace-go.v1 v1.74.8 ) diff --git a/tools/tests/integration/fixtures/orchestrion_pin_graph/go.sum b/tools/tests/integration/fixtures/orchestrion_pin_graph/go.sum index b1cc86d5..a92ab559 100644 --- a/tools/tests/integration/fixtures/orchestrion_pin_graph/go.sum +++ b/tools/tests/integration/fixtures/orchestrion_pin_graph/go.sum @@ -2,9 +2,7 @@ github.com/DataDog/dd-trace-go/contrib/log/slog/v2 v2.3.0 h1:4b6P8HD3y0JaiXSTOAT github.com/DataDog/dd-trace-go/contrib/log/slog/v2 v2.3.0/go.mod h1:Ps0MUVRg2EhbFMXgyar4gzrtQ8QCUXwwvf4VfrvGOck= github.com/DataDog/dd-trace-go/contrib/net/http/v2 v2.9.1-rc.3 h1:Sa2Xl/25nkVZtsDRMMA0NtJavWEE+4GRc0fPoLznHZI= github.com/DataDog/dd-trace-go/contrib/net/http/v2 v2.9.1-rc.3/go.mod h1:L0HbDaTdLRK59szho4l7X7YsZ2OkSXgmskvfiOTPLSI= -github.com/DataDog/dd-trace-go/v2 v2.9.1 h1:N2aqlWS0nAG5o+ETVyvz3gtboZbfemaD8Q/VumStGRY= -github.com/DataDog/dd-trace-go/v2 v2.9.1/go.mod h1:SdMkCESSBc2knx56Xol2pO7jhMDPi7MxyNj6vRYMW48= -github.com/DataDog/orchestrion v1.12.0 h1:GMdYBXkn3QbaU1mla2b7FN0uF9vJ2UJzrUH1X254Hc0= -github.com/DataDog/orchestrion v1.12.0/go.mod h1:Xg4S/8UP7LpmmeSR/pqhO7vKUpiohSJcU8VCDPkpBpA= +github.com/DataDog/dd-trace-go/v2 v2.9.1-rc.3 h1:3pOaLc2C2JN0EdQV6Ea0uz724aJ7ECP3u5VGHoJnk2U= +github.com/DataDog/dd-trace-go/v2 v2.9.1-rc.3/go.mod h1:SdMkCESSBc2knx56Xol2pO7jhMDPi7MxyNj6vRYMW48= gopkg.in/DataDog/dd-trace-go.v1 v1.74.8 h1:h96ji92t9eXbPvSWhJ+lrPWetHiQNYlt48JKRO09NFA= gopkg.in/DataDog/dd-trace-go.v1 v1.74.8/go.mod h1:LpHbtHsCZBlm1HWrlVOUQcEXwMWZnU6yMvmtd1GvSDI= diff --git a/tools/tests/integration/run_bzlmod_go_integration.sh b/tools/tests/integration/run_bzlmod_go_integration.sh index ab5e3f9b..9e0201c1 100755 --- a/tools/tests/integration/run_bzlmod_go_integration.sh +++ b/tools/tests/integration/run_bzlmod_go_integration.sh @@ -53,7 +53,7 @@ FORBID_HOST_GO="${FORBID_HOST_GO:-0}" EXPECTED_ORCHESTRION_CACHE_PHASE="${EXPECTED_ORCHESTRION_CACHE_PHASE:-}" HOST_GO_SENTINEL_LOG="" DD_TRACE_GO_VERSION="${DD_TRACE_GO_VERSION:-v2.9.1}" -PIN_ROOT_VERSION="v2.9.1" +PIN_ROOT_VERSION="v2.9.1-rc.3" PIN_HTTP_VERSION="v2.9.1-rc.3" PIN_SLOG_VERSION="v2.3.0" SERVICE_NAME="${SERVICE_NAME:-bzlmod-go-service}" diff --git a/tools/tests/integration/run_workspace_go_integration.sh b/tools/tests/integration/run_workspace_go_integration.sh index c7f51cae..57114f86 100755 --- a/tools/tests/integration/run_workspace_go_integration.sh +++ b/tools/tests/integration/run_workspace_go_integration.sh @@ -58,7 +58,7 @@ HOST_GO_SENTINEL_LOG="" # Keep this aligned with the bootstrap helper's published default tracer pin so # the WORKSPACE harness validates the same public Go path the docs describe. DD_TRACE_GO_VERSION="${DD_TRACE_GO_VERSION:-v2.9.1}" -PIN_ROOT_VERSION="v2.9.1" +PIN_ROOT_VERSION="v2.9.1-rc.3" PIN_HTTP_VERSION="v2.9.1-rc.3" PIN_SLOG_VERSION="v2.3.0" SERVICE_NAME="${SERVICE_NAME:-workspace-go-service}" From a283283db9af34f35d24a1f0aacc0b32d40b01e6 Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Fri, 7 Aug 2026 23:43:47 +0200 Subject: [PATCH 03/37] fix: invalidate incomplete Orchestrion bootstrap caches --- .../rgo/v0_60_0/base/go/private/orchestrion/extensions.bzl | 2 +- .../rgo/v0_61_1/base/go/private/orchestrion/extensions.bzl | 2 +- .../rgo/v0_62_0/base/go/private/orchestrion/extensions.bzl | 2 +- .../patches/v0_60_0/base/0001-full-delta.patch | 4 ++-- .../patches/v0_61_1/base/0001-full-delta.patch | 4 ++-- .../patches/v0_62_0/base/0001-full-delta.patch | 4 ++-- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/third_party/rgo/v0_60_0/base/go/private/orchestrion/extensions.bzl b/third_party/rgo/v0_60_0/base/go/private/orchestrion/extensions.bzl index 5b757a9f..dc4e4830 100644 --- a/third_party/rgo/v0_60_0/base/go/private/orchestrion/extensions.bzl +++ b/third_party/rgo/v0_60_0/base/go/private/orchestrion/extensions.bzl @@ -15,7 +15,7 @@ """Module extension for configuring orchestrion in rules_go.""" DEFAULT_DD_TRACE_GO_VERSION = "v2.9.1" -ORCHESTRION_BOOTSTRAP_CACHE_ABI = "v3" +ORCHESTRION_BOOTSTRAP_CACHE_ABI = "v4" ORCHESTRION_SEED_GO_MOD_VERSION = "1.21" # Bump this identifier whenever the in-repo Orchestrion patch block changes in diff --git a/third_party/rgo/v0_61_1/base/go/private/orchestrion/extensions.bzl b/third_party/rgo/v0_61_1/base/go/private/orchestrion/extensions.bzl index 5b757a9f..dc4e4830 100644 --- a/third_party/rgo/v0_61_1/base/go/private/orchestrion/extensions.bzl +++ b/third_party/rgo/v0_61_1/base/go/private/orchestrion/extensions.bzl @@ -15,7 +15,7 @@ """Module extension for configuring orchestrion in rules_go.""" DEFAULT_DD_TRACE_GO_VERSION = "v2.9.1" -ORCHESTRION_BOOTSTRAP_CACHE_ABI = "v3" +ORCHESTRION_BOOTSTRAP_CACHE_ABI = "v4" ORCHESTRION_SEED_GO_MOD_VERSION = "1.21" # Bump this identifier whenever the in-repo Orchestrion patch block changes in diff --git a/third_party/rgo/v0_62_0/base/go/private/orchestrion/extensions.bzl b/third_party/rgo/v0_62_0/base/go/private/orchestrion/extensions.bzl index 5b757a9f..dc4e4830 100644 --- a/third_party/rgo/v0_62_0/base/go/private/orchestrion/extensions.bzl +++ b/third_party/rgo/v0_62_0/base/go/private/orchestrion/extensions.bzl @@ -15,7 +15,7 @@ """Module extension for configuring orchestrion in rules_go.""" DEFAULT_DD_TRACE_GO_VERSION = "v2.9.1" -ORCHESTRION_BOOTSTRAP_CACHE_ABI = "v3" +ORCHESTRION_BOOTSTRAP_CACHE_ABI = "v4" ORCHESTRION_SEED_GO_MOD_VERSION = "1.21" # Bump this identifier whenever the in-repo Orchestrion patch block changes in diff --git a/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch index 22355f51..615dd150 100644 --- a/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch @@ -2091,7 +2091,7 @@ index 0000000..31a01c3 +) diff --git a/go/private/orchestrion/extensions.bzl b/go/private/orchestrion/extensions.bzl new file mode 100644 -index 0000000..5b757a9 +index 0000000..dc4e483 --- /dev/null +++ b/go/private/orchestrion/extensions.bzl @@ -0,0 +1,1739 @@ @@ -2112,7 +2112,7 @@ index 0000000..5b757a9 +"""Module extension for configuring orchestrion in rules_go.""" + +DEFAULT_DD_TRACE_GO_VERSION = "v2.9.1" -+ORCHESTRION_BOOTSTRAP_CACHE_ABI = "v3" ++ORCHESTRION_BOOTSTRAP_CACHE_ABI = "v4" +ORCHESTRION_SEED_GO_MOD_VERSION = "1.21" + +# Bump this identifier whenever the in-repo Orchestrion patch block changes in diff --git a/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch index cf1b01fe..d2d3ccd9 100644 --- a/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch @@ -2084,7 +2084,7 @@ index 0000000..177eeaf +) diff --git a/go/private/orchestrion/extensions.bzl b/go/private/orchestrion/extensions.bzl new file mode 100644 -index 0000000..5b757a9 +index 0000000..dc4e483 --- /dev/null +++ b/go/private/orchestrion/extensions.bzl @@ -0,0 +1,1739 @@ @@ -2105,7 +2105,7 @@ index 0000000..5b757a9 +"""Module extension for configuring orchestrion in rules_go.""" + +DEFAULT_DD_TRACE_GO_VERSION = "v2.9.1" -+ORCHESTRION_BOOTSTRAP_CACHE_ABI = "v3" ++ORCHESTRION_BOOTSTRAP_CACHE_ABI = "v4" +ORCHESTRION_SEED_GO_MOD_VERSION = "1.21" + +# Bump this identifier whenever the in-repo Orchestrion patch block changes in diff --git a/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch index 7e373ad7..2542f8e4 100644 --- a/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch @@ -2088,7 +2088,7 @@ index 0000000..177eeaf +) diff --git a/go/private/orchestrion/extensions.bzl b/go/private/orchestrion/extensions.bzl new file mode 100644 -index 0000000..5b757a9 +index 0000000..dc4e483 --- /dev/null +++ b/go/private/orchestrion/extensions.bzl @@ -0,0 +1,1739 @@ @@ -2109,7 +2109,7 @@ index 0000000..5b757a9 +"""Module extension for configuring orchestrion in rules_go.""" + +DEFAULT_DD_TRACE_GO_VERSION = "v2.9.1" -+ORCHESTRION_BOOTSTRAP_CACHE_ABI = "v3" ++ORCHESTRION_BOOTSTRAP_CACHE_ABI = "v4" +ORCHESTRION_SEED_GO_MOD_VERSION = "1.21" + +# Bump this identifier whenever the in-repo Orchestrion patch block changes in From d719c47e34213a853728e8761cf929dd5349b9ca Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Sat, 8 Aug 2026 00:53:33 +0200 Subject: [PATCH 04/37] go: wait for active Orchestrion cache owners --- .../go/tools/builders/orchestrion_cache.go | 3 ++ .../tools/builders/orchestrion_cache_test.go | 18 ++++++++++++ .../go/tools/builders/orchestrion_cache.go | 3 ++ .../tools/builders/orchestrion_cache_test.go | 18 ++++++++++++ .../go/tools/builders/orchestrion_cache.go | 3 ++ .../tools/builders/orchestrion_cache_test.go | 18 ++++++++++++ .../v0_60_0/base/0001-full-delta.patch | 29 ++++++++++++++++--- .../v0_61_1/base/0001-full-delta.patch | 29 ++++++++++++++++--- .../v0_62_0/base/0001-full-delta.patch | 29 ++++++++++++++++--- 9 files changed, 138 insertions(+), 12 deletions(-) diff --git a/third_party/rgo/v0_60_0/base/go/tools/builders/orchestrion_cache.go b/third_party/rgo/v0_60_0/base/go/tools/builders/orchestrion_cache.go index e4768cfc..746ac80c 100644 --- a/third_party/rgo/v0_60_0/base/go/tools/builders/orchestrion_cache.go +++ b/third_party/rgo/v0_60_0/base/go/tools/builders/orchestrion_cache.go @@ -102,6 +102,9 @@ func acquireCacheLockWithTimings(lockDir string, timeout, staleAfter, pollInterv } continue } + if !stale { + continue + } return nil, fmt.Errorf("timeout acquiring cache lock %s", lockDir) } } diff --git a/third_party/rgo/v0_60_0/base/go/tools/builders/orchestrion_cache_test.go b/third_party/rgo/v0_60_0/base/go/tools/builders/orchestrion_cache_test.go index 4d2d0164..5bc0af2e 100644 --- a/third_party/rgo/v0_60_0/base/go/tools/builders/orchestrion_cache_test.go +++ b/third_party/rgo/v0_60_0/base/go/tools/builders/orchestrion_cache_test.go @@ -84,6 +84,24 @@ func TestAcquireCacheLockReplacesStaleLock(t *testing.T) { } } +func TestAcquireCacheLockWaitsForActiveOwnerPastTimeout(t *testing.T) { + lockDir := filepath.Join(t.TempDir(), "cache.lock") + releaseOwner, err := tryAcquireCacheLock(lockDir) + if err != nil { + t.Fatalf("acquire owner lock: %v", err) + } + go func() { + time.Sleep(50 * time.Millisecond) + releaseOwner() + }() + + releaseWaiter, err := acquireCacheLockWithTimings(lockDir, 20*time.Millisecond, time.Minute, 5*time.Millisecond) + if err != nil { + t.Fatalf("wait for active owner: %v", err) + } + releaseWaiter() +} + func TestWriteFileAtomically(t *testing.T) { path := filepath.Join(t.TempDir(), "nested", "manifest.json") if err := writeFileAtomically(path, []byte("payload\n"), 0o644); err != nil { diff --git a/third_party/rgo/v0_61_1/base/go/tools/builders/orchestrion_cache.go b/third_party/rgo/v0_61_1/base/go/tools/builders/orchestrion_cache.go index e4768cfc..746ac80c 100644 --- a/third_party/rgo/v0_61_1/base/go/tools/builders/orchestrion_cache.go +++ b/third_party/rgo/v0_61_1/base/go/tools/builders/orchestrion_cache.go @@ -102,6 +102,9 @@ func acquireCacheLockWithTimings(lockDir string, timeout, staleAfter, pollInterv } continue } + if !stale { + continue + } return nil, fmt.Errorf("timeout acquiring cache lock %s", lockDir) } } diff --git a/third_party/rgo/v0_61_1/base/go/tools/builders/orchestrion_cache_test.go b/third_party/rgo/v0_61_1/base/go/tools/builders/orchestrion_cache_test.go index 4d2d0164..5bc0af2e 100644 --- a/third_party/rgo/v0_61_1/base/go/tools/builders/orchestrion_cache_test.go +++ b/third_party/rgo/v0_61_1/base/go/tools/builders/orchestrion_cache_test.go @@ -84,6 +84,24 @@ func TestAcquireCacheLockReplacesStaleLock(t *testing.T) { } } +func TestAcquireCacheLockWaitsForActiveOwnerPastTimeout(t *testing.T) { + lockDir := filepath.Join(t.TempDir(), "cache.lock") + releaseOwner, err := tryAcquireCacheLock(lockDir) + if err != nil { + t.Fatalf("acquire owner lock: %v", err) + } + go func() { + time.Sleep(50 * time.Millisecond) + releaseOwner() + }() + + releaseWaiter, err := acquireCacheLockWithTimings(lockDir, 20*time.Millisecond, time.Minute, 5*time.Millisecond) + if err != nil { + t.Fatalf("wait for active owner: %v", err) + } + releaseWaiter() +} + func TestWriteFileAtomically(t *testing.T) { path := filepath.Join(t.TempDir(), "nested", "manifest.json") if err := writeFileAtomically(path, []byte("payload\n"), 0o644); err != nil { diff --git a/third_party/rgo/v0_62_0/base/go/tools/builders/orchestrion_cache.go b/third_party/rgo/v0_62_0/base/go/tools/builders/orchestrion_cache.go index e4768cfc..746ac80c 100644 --- a/third_party/rgo/v0_62_0/base/go/tools/builders/orchestrion_cache.go +++ b/third_party/rgo/v0_62_0/base/go/tools/builders/orchestrion_cache.go @@ -102,6 +102,9 @@ func acquireCacheLockWithTimings(lockDir string, timeout, staleAfter, pollInterv } continue } + if !stale { + continue + } return nil, fmt.Errorf("timeout acquiring cache lock %s", lockDir) } } diff --git a/third_party/rgo/v0_62_0/base/go/tools/builders/orchestrion_cache_test.go b/third_party/rgo/v0_62_0/base/go/tools/builders/orchestrion_cache_test.go index 4d2d0164..5bc0af2e 100644 --- a/third_party/rgo/v0_62_0/base/go/tools/builders/orchestrion_cache_test.go +++ b/third_party/rgo/v0_62_0/base/go/tools/builders/orchestrion_cache_test.go @@ -84,6 +84,24 @@ func TestAcquireCacheLockReplacesStaleLock(t *testing.T) { } } +func TestAcquireCacheLockWaitsForActiveOwnerPastTimeout(t *testing.T) { + lockDir := filepath.Join(t.TempDir(), "cache.lock") + releaseOwner, err := tryAcquireCacheLock(lockDir) + if err != nil { + t.Fatalf("acquire owner lock: %v", err) + } + go func() { + time.Sleep(50 * time.Millisecond) + releaseOwner() + }() + + releaseWaiter, err := acquireCacheLockWithTimings(lockDir, 20*time.Millisecond, time.Minute, 5*time.Millisecond) + if err != nil { + t.Fatalf("wait for active owner: %v", err) + } + releaseWaiter() +} + func TestWriteFileAtomically(t *testing.T) { path := filepath.Join(t.TempDir(), "nested", "manifest.json") if err := writeFileAtomically(path, []byte("payload\n"), 0o644); err != nil { diff --git a/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch index 615dd150..702f2d9e 100644 --- a/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch @@ -12795,10 +12795,10 @@ index 0000000..93f9e48 +} diff --git a/go/tools/builders/orchestrion_cache.go b/go/tools/builders/orchestrion_cache.go new file mode 100644 -index 0000000..e4768cf +index 0000000..746ac80 --- /dev/null +++ b/go/tools/builders/orchestrion_cache.go -@@ -0,0 +1,273 @@ +@@ -0,0 +1,276 @@ +package main + +import ( @@ -12903,6 +12903,9 @@ index 0000000..e4768cf + } + continue + } ++ if !stale { ++ continue ++ } + return nil, fmt.Errorf("timeout acquiring cache lock %s", lockDir) + } +} @@ -13074,10 +13077,10 @@ index 0000000..e4768cf +} diff --git a/go/tools/builders/orchestrion_cache_test.go b/go/tools/builders/orchestrion_cache_test.go new file mode 100644 -index 0000000..4d2d016 +index 0000000..5bc0af2 --- /dev/null +++ b/go/tools/builders/orchestrion_cache_test.go -@@ -0,0 +1,99 @@ +@@ -0,0 +1,117 @@ +package main + +import ( @@ -13164,6 +13167,24 @@ index 0000000..4d2d016 + } +} + ++func TestAcquireCacheLockWaitsForActiveOwnerPastTimeout(t *testing.T) { ++ lockDir := filepath.Join(t.TempDir(), "cache.lock") ++ releaseOwner, err := tryAcquireCacheLock(lockDir) ++ if err != nil { ++ t.Fatalf("acquire owner lock: %v", err) ++ } ++ go func() { ++ time.Sleep(50 * time.Millisecond) ++ releaseOwner() ++ }() ++ ++ releaseWaiter, err := acquireCacheLockWithTimings(lockDir, 20*time.Millisecond, time.Minute, 5*time.Millisecond) ++ if err != nil { ++ t.Fatalf("wait for active owner: %v", err) ++ } ++ releaseWaiter() ++} ++ +func TestWriteFileAtomically(t *testing.T) { + path := filepath.Join(t.TempDir(), "nested", "manifest.json") + if err := writeFileAtomically(path, []byte("payload\n"), 0o644); err != nil { diff --git a/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch index d2d3ccd9..589bd52e 100644 --- a/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch @@ -12785,10 +12785,10 @@ index 0000000..93f9e48 +} diff --git a/go/tools/builders/orchestrion_cache.go b/go/tools/builders/orchestrion_cache.go new file mode 100644 -index 0000000..e4768cf +index 0000000..746ac80 --- /dev/null +++ b/go/tools/builders/orchestrion_cache.go -@@ -0,0 +1,273 @@ +@@ -0,0 +1,276 @@ +package main + +import ( @@ -12893,6 +12893,9 @@ index 0000000..e4768cf + } + continue + } ++ if !stale { ++ continue ++ } + return nil, fmt.Errorf("timeout acquiring cache lock %s", lockDir) + } +} @@ -13064,10 +13067,10 @@ index 0000000..e4768cf +} diff --git a/go/tools/builders/orchestrion_cache_test.go b/go/tools/builders/orchestrion_cache_test.go new file mode 100644 -index 0000000..4d2d016 +index 0000000..5bc0af2 --- /dev/null +++ b/go/tools/builders/orchestrion_cache_test.go -@@ -0,0 +1,99 @@ +@@ -0,0 +1,117 @@ +package main + +import ( @@ -13154,6 +13157,24 @@ index 0000000..4d2d016 + } +} + ++func TestAcquireCacheLockWaitsForActiveOwnerPastTimeout(t *testing.T) { ++ lockDir := filepath.Join(t.TempDir(), "cache.lock") ++ releaseOwner, err := tryAcquireCacheLock(lockDir) ++ if err != nil { ++ t.Fatalf("acquire owner lock: %v", err) ++ } ++ go func() { ++ time.Sleep(50 * time.Millisecond) ++ releaseOwner() ++ }() ++ ++ releaseWaiter, err := acquireCacheLockWithTimings(lockDir, 20*time.Millisecond, time.Minute, 5*time.Millisecond) ++ if err != nil { ++ t.Fatalf("wait for active owner: %v", err) ++ } ++ releaseWaiter() ++} ++ +func TestWriteFileAtomically(t *testing.T) { + path := filepath.Join(t.TempDir(), "nested", "manifest.json") + if err := writeFileAtomically(path, []byte("payload\n"), 0o644); err != nil { diff --git a/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch index 2542f8e4..f6c44dd2 100644 --- a/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch @@ -12789,10 +12789,10 @@ index 0000000..93f9e48 +} diff --git a/go/tools/builders/orchestrion_cache.go b/go/tools/builders/orchestrion_cache.go new file mode 100644 -index 0000000..e4768cf +index 0000000..746ac80 --- /dev/null +++ b/go/tools/builders/orchestrion_cache.go -@@ -0,0 +1,273 @@ +@@ -0,0 +1,276 @@ +package main + +import ( @@ -12897,6 +12897,9 @@ index 0000000..e4768cf + } + continue + } ++ if !stale { ++ continue ++ } + return nil, fmt.Errorf("timeout acquiring cache lock %s", lockDir) + } +} @@ -13068,10 +13071,10 @@ index 0000000..e4768cf +} diff --git a/go/tools/builders/orchestrion_cache_test.go b/go/tools/builders/orchestrion_cache_test.go new file mode 100644 -index 0000000..4d2d016 +index 0000000..5bc0af2 --- /dev/null +++ b/go/tools/builders/orchestrion_cache_test.go -@@ -0,0 +1,99 @@ +@@ -0,0 +1,117 @@ +package main + +import ( @@ -13158,6 +13161,24 @@ index 0000000..4d2d016 + } +} + ++func TestAcquireCacheLockWaitsForActiveOwnerPastTimeout(t *testing.T) { ++ lockDir := filepath.Join(t.TempDir(), "cache.lock") ++ releaseOwner, err := tryAcquireCacheLock(lockDir) ++ if err != nil { ++ t.Fatalf("acquire owner lock: %v", err) ++ } ++ go func() { ++ time.Sleep(50 * time.Millisecond) ++ releaseOwner() ++ }() ++ ++ releaseWaiter, err := acquireCacheLockWithTimings(lockDir, 20*time.Millisecond, time.Minute, 5*time.Millisecond) ++ if err != nil { ++ t.Fatalf("wait for active owner: %v", err) ++ } ++ releaseWaiter() ++} ++ +func TestWriteFileAtomically(t *testing.T) { + path := filepath.Join(t.TempDir(), "nested", "manifest.json") + if err := writeFileAtomically(path, []byte("payload\n"), 0o644); err != nil { From df41e6568491990d5be3d258f3070652f8749346 Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Sat, 8 Aug 2026 02:15:33 +0200 Subject: [PATCH 05/37] build: refresh Orchestrion extension lock --- MODULE.bazel.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index dac5f63a..f84d579b 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -766,7 +766,7 @@ }, "@@rules_go+//go:extensions.bzl%orchestrion": { "general": { - "bzlTransitiveDigest": "7qy3MxlPmbSSl3MDmdiHrZuoYkvewN+gUmuzc/uJvzo=", + "bzlTransitiveDigest": "ddXasJ1gcaFD8IfNKQVFG/bfJZSinEMN9y7/lUDv4gI=", "usagesDigest": "9kKEG/hjK4fnk42l4jAKDLhx9xExyCYAS8r8NlbPLMw=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, From 3222be11d9f2fe4811f5240a41d95783cf505cbd Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Sat, 8 Aug 2026 10:07:11 +0200 Subject: [PATCH 06/37] uploader: preserve valid payloads after partial failures --- AGENTS.md | 26 ++- README.md | 16 +- docs/Configuration_Reference.md | 8 +- docs/Initial_documentation.md | 4 +- docs/Installation_Reference.md | 8 +- docs/Language_Onboarding.md | 2 +- docs/RFC.md | 4 +- docs/Troubleshooting.md | 5 +- docs/Uploader_Reference.md | 15 +- docs/internal_monorepo_go_rollout_guide.md | 5 +- examples/README.md | 10 +- modules/go/tools/dd_topt_go_bootstrap/main.go | 28 ++-- .../tools/dd_topt_go_bootstrap/main_test.go | 57 +++++++ .../go-test-optimization-onboarding/SKILL.md | 4 +- .../references/validation-checklist.md | 20 +-- .../SKILL.md | 2 +- .../references/validation-checklist.md | 23 +-- .../SKILL.md | 2 +- .../references/validation-checklist.md | 23 +-- tools/core/uploader_bash_runtime.sh.tpl | 11 +- .../core/uploader_powershell_runtime.ps1.tpl | 10 +- .../run_test_optimization_ci.ps1 | 19 +-- .../run_test_optimization_ci.sh | 28 ++-- tools/tests/python/test_python_tools.py | 155 +++++++++++++++++- 24 files changed, 331 insertions(+), 154 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 3c2bb5cc..aa5fd3bf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -95,21 +95,14 @@ The sync rule creates `@test_optimization_data//` containing: ```bash # Tests write payloads to TEST_UNDECLARED_OUTPUTS_DIR automatically # Bazel collects them to bazel-testlogs//test.outputs/ - ./bazelw test //... || test_status=$?; test_status=${test_status:-0} - ./bazelw run //:dd_test_optimization_doctor || doctor_status=$?; doctor_status=${doctor_status:-0} - if [ "$doctor_status" -ne 0 ]; then - if [ "$test_status" -ne 0 ]; then exit "$test_status"; fi - exit "$doctor_status" - fi - ./bazelw run //:dd_upload_payloads -- --dry-run --validate-enrichment || dry_run_status=$?; dry_run_status=${dry_run_status:-0} - if [ "$dry_run_status" -ne 0 ]; then - if [ "$test_status" -ne 0 ]; then exit "$test_status"; fi - exit "$dry_run_status" - fi - DD_API_KEY="$DD_API_KEY" DD_SITE="$DD_SITE" ./bazelw run //:dd_upload_payloads - upload_status=$? - if [ "$test_status" -ne 0 ]; then exit "$test_status"; fi - exit "$upload_status" + test_status=0; doctor_status=0; dry_run_status=0; upload_status=0 + ./bazelw test //... || test_status=$? + ./bazelw run //:dd_test_optimization_doctor || doctor_status=$? + ./bazelw run //:dd_upload_payloads -- --dry-run --validate-enrichment || dry_run_status=$? + DD_API_KEY="$DD_API_KEY" DD_SITE="$DD_SITE" ./bazelw run //:dd_upload_payloads || upload_status=$? + for status in "$test_status" "$doctor_status" "$dry_run_status" "$upload_status"; do + if [ "$status" -ne 0 ]; then exit "$status"; fi + done ``` - Force refetch of test optimization data: ```bash @@ -164,7 +157,8 @@ The sync rule creates `@test_optimization_data//` containing: - Bazel automatically collects these to `bazel-testlogs///test.outputs/`. - In consumer workspaces, run `./bazelw run //:dd_test_optimization_doctor` after tests complete, then run `./bazelw run //:dd_upload_payloads -- --dry-run --validate-enrichment`, then upload with `./bazelw run //:dd_upload_payloads`. - Do not run the real upload if doctor or dry-run enrichment validation fails. + When upload is authorized, process every available fresh valid payload even if + tests, doctor, or dry-run fail, and preserve the earliest failure as the job result. - For Go, route the repository's central `dd_go_test` wrapper through `dd_topt_go_test`; `--config=test-optimization` is the only user-facing enable switch. diff --git a/README.md b/README.md index a39bcb13..ca671857 100644 --- a/README.md +++ b/README.md @@ -377,7 +377,7 @@ tools/test_optimization/run_test_optimization_ci.sh \ --support-bundle .topt/reports/dd-test-optimization-support.zip \ //... -# Add --upload only when the real upload should run after doctor and dry-run pass. +# Add --upload to send every available fresh valid payload after validation attempts. DD_API_KEY="$DD_API_KEY" DD_SITE="$DD_SITE" \ tools/test_optimization/run_test_optimization_ci.sh \ --report-dir .topt/reports \ @@ -395,7 +395,7 @@ DD_API_KEY="$DD_API_KEY" DD_SITE="$DD_SITE" \ -SupportBundle .topt\reports\dd-test-optimization-support.zip ` //... -# Add -Upload only when the real upload should run after doctor and dry-run pass. +# Add -Upload to send every available fresh valid payload after validation attempts. $env:DD_API_KEY = "" $env:DD_SITE = "datadoghq.com" .\tools\test_optimization\run_test_optimization_ci.ps1 ` @@ -1426,7 +1426,7 @@ tools/test_optimization/run_test_optimization_ci.sh \ --support-bundle .topt/reports/dd-test-optimization-support.zip \ //... -# Add --upload only when the real upload should run after doctor and dry-run pass. +# Add --upload to send every available fresh valid payload after validation attempts. DD_API_KEY="$DD_API_KEY" DD_SITE="$DD_SITE" \ tools/test_optimization/run_test_optimization_ci.sh \ --report-dir .topt/reports \ @@ -1442,7 +1442,7 @@ DD_API_KEY="$DD_API_KEY" DD_SITE="$DD_SITE" \ -SupportBundle .topt\reports\dd-test-optimization-support.zip ` //... -# Add -Upload only when the real upload should run after doctor and dry-run pass. +# Add -Upload to send every available fresh valid payload after validation attempts. $env:DD_API_KEY = "" $env:DD_SITE = "datadoghq.com" .\tools\test_optimization\run_test_optimization_ci.ps1 ` @@ -1452,10 +1452,10 @@ $env:DD_SITE = "datadoghq.com" //... ``` -**IMPORTANT**: Always preserve the test exit code. Upload failed-test payloads -when doctor and dry-run enrichment pass, but do not run the real upload after a -doctor or dry-run failure. Using plain `;` causes CI to report success even when -tests fail. +**IMPORTANT**: Always preserve the test exit code. When upload is enabled, the +wrapper runs doctor and dry-run for diagnostics, then uploads every available +fresh valid payload even if tests or either validation step failed. The earliest +test, doctor, dry-run, or upload failure remains the job result. ### Important runtime requirements diff --git a/docs/Configuration_Reference.md b/docs/Configuration_Reference.md index 95d4830f..37f9a19a 100644 --- a/docs/Configuration_Reference.md +++ b/docs/Configuration_Reference.md @@ -305,8 +305,8 @@ Rule: `dd_payload_uploader(...)` | `filter_prefix` | bool | `False` | Only upload files matching `span_events_*.json` or `coverage_*.json` | | `gzip_payloads` | bool | `False` | Gzip test payloads before upload | | `data` | label_list | `[]` | Data files to include (for example, `context.json` for enrichment) | -| `expected_targets` | string_list | `[]` | Optional exact local labels expected in the matching BEP. Fresh and cached results jointly satisfy coverage; only fresh outputs are inspected or uploaded | -| `expected_targets_file` | label | unset | Optional schema-v1 exact-target file. Static and file inputs must match when both are non-empty | +| `expected_targets` | string_list | `[]` | Optional exact local labels expected in the matching BEP. Fresh and cached results jointly satisfy coverage; missing results are reported while other fresh outputs continue to upload | +| `expected_targets_file` | label | unset | Optional schema-v1 exact-target file. Static and file inputs must match when both are non-empty; missing results do not block other fresh uploads | ## Doctor rule attributes @@ -377,8 +377,8 @@ workspace root package. | `sync_repo_name` | string | `"test_optimization_data"` | Repository exposing `:test_optimization_context` | | `doctor_name` | string | `"dd_test_optimization_doctor"` | Generated doctor target name | | `uploader_name` | string | `"dd_upload_payloads"` | Generated uploader target name | -| `expected_targets` | string_list | `[]` | Strict labels passed to both doctor and uploader. List only instrumented runtime test targets that emit payloads | -| `expected_targets_file` | label or `None` | `None` | Generated exact-target JSON file forwarded to both doctor and uploader for manifest-driven invocations | +| `expected_targets` | string_list | `[]` | Exact labels passed to both tools. Doctor validates them strictly; uploader reports missing results while continuing with other fresh payloads | +| `expected_targets_file` | label or `None` | `None` | Generated exact-target JSON file forwarded to both tools with the same doctor/uploader semantics for manifest-driven invocations | | `context_data` | label_list or `None` | `["@//:test_optimization_context"]` | Explicit context data labels when the default sync repo label is not enough | | `doctor_kwargs` | dict or `None` | `{}` | Extra attrs for `dd_test_optimization_doctor`; cannot override `name`, `data`, `expected_targets`, or `expected_targets_file` | | `uploader_kwargs` | dict or `None` | `{}` | Extra attrs for `dd_payload_uploader`; cannot override `name`, `data`, `expected_targets`, or `expected_targets_file` | diff --git a/docs/Initial_documentation.md b/docs/Initial_documentation.md index e88fc31e..a161804d 100644 --- a/docs/Initial_documentation.md +++ b/docs/Initial_documentation.md @@ -70,8 +70,8 @@ The steps are: as `//tools/test_optimization`. Usage: run `bazel test`, then the doctor target, then the uploader with `--dry-run --validate-enrichment`, then the real uploader target. Preserve - the test exit code, but do not run the real upload if doctor or dry-run - enrichment validation fails. + the earliest failure, but still let the real uploader process every available + fresh valid payload when upload is enabled. 4. **Language macros (optional)**: Thin wrappers (for Go/Python/Java/NodeJS/.NET/Ruby) set up the right runfiles/env so test code can read the synced files and write payloads to `TEST_UNDECLARED_OUTPUTS_DIR`. diff --git a/docs/Installation_Reference.md b/docs/Installation_Reference.md index c9570ec1..2c075739 100644 --- a/docs/Installation_Reference.md +++ b/docs/Installation_Reference.md @@ -441,7 +441,7 @@ tools/test_optimization/run_test_optimization_ci.sh \ --support-bundle .topt/reports/dd-test-optimization-support.zip \ //... -# Add --upload only when the real upload should run after doctor and dry-run pass. +# Add --upload to send every available fresh valid payload after validation attempts. DD_API_KEY="$DD_API_KEY" DD_SITE="$DD_SITE" \ tools/test_optimization/run_test_optimization_ci.sh \ --report-dir .topt/reports \ @@ -458,7 +458,7 @@ DD_API_KEY="$DD_API_KEY" DD_SITE="$DD_SITE" \ -SupportBundle .topt\reports\dd-test-optimization-support.zip ` //... -# Add -Upload only when the real upload should run after doctor and dry-run pass. +# Add -Upload to send every available fresh valid payload after validation attempts. $env:DD_API_KEY = "" $env:DD_SITE = "datadoghq.com" .\tools\test_optimization\run_test_optimization_ci.ps1 ` @@ -521,7 +521,9 @@ python3 tools/test_optimization/render_report_summary.py \ --output .topt/reports/upload-diagnostics.md ``` -Do not run the real uploader if the doctor or dry-run enrichment step fails. +When upload is enabled, run the real uploader after the doctor and dry-run +attempts even if either validation failed. It processes every available fresh +valid payload while the wrapper preserves the earlier failure as the job result. For manual Go extension wiring, set `module_path` to the Go module path from `go.mod`: diff --git a/docs/Language_Onboarding.md b/docs/Language_Onboarding.md index b6cddecf..bad6a666 100644 --- a/docs/Language_Onboarding.md +++ b/docs/Language_Onboarding.md @@ -111,7 +111,7 @@ tools/test_optimization/run_test_optimization_ci.sh \ --support-bundle .topt/reports/dd-test-optimization-support.zip \ //... -# Add --upload only when the real upload should run after doctor and dry-run pass. +# Add --upload to send every available fresh valid payload after validation attempts. DD_API_KEY="$DD_API_KEY" DD_SITE="$DD_SITE" \ tools/test_optimization/run_test_optimization_ci.sh \ --doctor-target //tools/test_optimization:dd_test_optimization_doctor \ diff --git a/docs/RFC.md b/docs/RFC.md index 5917b21a..2c29754c 100644 --- a/docs/RFC.md +++ b/docs/RFC.md @@ -142,8 +142,8 @@ At a high level, the proposal moves all network‑dependent metadata fetching ou - The uploader discovers all `test.outputs/` directories in `bazel-testlogs/`, waits for filesystem quiescence, enriches test payloads with `context.json` (if present), uploads via agentless (`DD_API_KEY`, `DD_SITE`) or an EVP proxy (`DD_TEST_OPTIMIZATION_AGENT_URL`), and deletes successfully uploaded files. - Usage: run `bazel test`, then `//:dd_test_optimization_doctor`, then `//:dd_upload_payloads -- --dry-run --validate-enrichment`, then the real - `//:dd_upload_payloads` target. Preserve the test exit code, but do not run - the real upload if doctor or dry-run enrichment validation fails. + `//:dd_upload_payloads` target. Preserve the earliest failure while still + uploading every available fresh valid payload. - [Multi‑service monorepos](../tools/core/test_optimization_multi_sync.bzl): diff --git a/docs/Troubleshooting.md b/docs/Troubleshooting.md index 7c8689fe..65f3f95d 100644 --- a/docs/Troubleshooting.md +++ b/docs/Troubleshooting.md @@ -995,7 +995,10 @@ long command sequence, local controls, disk checks, and an explicit upload step. ./tools/test_optimization/validate_go_pilot.sh --no-upload ``` -3. Upload only after tests and doctor pass: +3. When credentials and upload authorization are available, run the upload even + if the validation-only attempt reported a test, doctor, or dry-run failure. + The script uploads every available fresh valid payload and still returns the + earliest failure: ```bash DD_API_KEY="$DD_API_KEY" DD_SITE="$DD_SITE" \ ./tools/test_optimization/validate_go_pilot.sh --upload diff --git a/docs/Uploader_Reference.md b/docs/Uploader_Reference.md index a36ab44e..8769f18f 100644 --- a/docs/Uploader_Reference.md +++ b/docs/Uploader_Reference.md @@ -68,7 +68,7 @@ tools/test_optimization/run_test_optimization_ci.sh \ # With --upload it also writes uploader-upload-report.json. --support-bundle # adds dd-test-optimization-support.zip for escalation. -# Add --upload only when the real upload should run after doctor and dry-run pass. +# Add --upload to send every available fresh valid payload after validation attempts. DD_API_KEY="$DD_API_KEY" DD_SITE="$DD_SITE" \ tools/test_optimization/run_test_optimization_ci.sh \ --doctor-target //tools/test_optimization:dd_test_optimization_doctor \ @@ -94,7 +94,7 @@ DD_API_KEY="$DD_API_KEY" DD_SITE="$DD_SITE" \ # With -Upload it also writes uploader-upload-report.json. -SupportBundle # adds dd-test-optimization-support.zip for escalation. -# Add -Upload only when the real upload should run after doctor and dry-run pass. +# Add -Upload to send every available fresh valid payload after validation attempts. $env:DD_API_KEY = "" $env:DD_SITE = "datadoghq.com" .\tools\test_optimization\run_test_optimization_ci.ps1 ` @@ -106,9 +106,9 @@ $env:DD_SITE = "datadoghq.com" //... ``` -Always preserve all statuses. Test failures should win, doctor failures should -stop the real upload while still preserving an earlier test failure, and -uploader failures must still fail the job when tests and validations passed. +Always preserve all statuses. Test failures win, followed by doctor, dry-run, +and uploader failures. When upload is enabled, validation failures do not block +the uploader from processing other fresh valid payloads. Dry-run enrichment validation: @@ -727,8 +727,9 @@ A cached Bazel test does not produce a fresh payload for the current invocation. With exact expected targets configured on both doctor and uploader, fresh and cached BEP results jointly satisfy invocation coverage. Only fresh outputs are validated or uploaded; an all-cached invocation is a successful -no-op. Every fresh expected output must independently contain a handled -payload, so one valid sibling output cannot hide an empty one. +no-op. A missing expected result is reported without blocking other fresh +payloads. Every fresh expected output that exists must independently contain a +handled payload, so one valid sibling output cannot hide an empty one. ### Advanced: reuse an already-fetched context file diff --git a/docs/internal_monorepo_go_rollout_guide.md b/docs/internal_monorepo_go_rollout_guide.md index 7d9087de..57869dbc 100644 --- a/docs/internal_monorepo_go_rollout_guide.md +++ b/docs/internal_monorepo_go_rollout_guide.md @@ -165,8 +165,9 @@ DD_API_KEY="$DD_API_KEY" DD_SITE="$DD_SITE" bazel run --config=test-optimization bazel shutdown ``` -Do not run the real uploader if the doctor or dry-run enrichment step fails. -Upload failed-test payloads only when those validation steps pass. +Run the real uploader after doctor and dry-run attempts even when an earlier +phase fails. It uploads available fresh valid failed-test payloads while the +wrapper preserves the earlier failure as the job result. The doctor must see JSON payloads, Bazel target metadata, Git metadata, and only valid Go payload-selection states. `module`, `module_override`, and diff --git a/examples/README.md b/examples/README.md index f2ac9afc..cb0d35ea 100644 --- a/examples/README.md +++ b/examples/README.md @@ -408,7 +408,7 @@ tools/test_optimization/run_test_optimization_ci.sh \ --support-bundle .topt/reports/dd-test-optimization-support.zip \ //... -# Add --upload only when the real upload should run after doctor and dry-run pass. +# Add --upload to send every available fresh valid payload after validation attempts. DD_API_KEY="$DD_API_KEY" DD_SITE="$DD_SITE" \ tools/test_optimization/run_test_optimization_ci.sh \ --config test-optimization \ @@ -428,7 +428,7 @@ DD_API_KEY="$DD_API_KEY" DD_SITE="$DD_SITE" \ -SupportBundle .topt\reports\dd-test-optimization-support.zip ` //... -# Add -Upload only when the real upload should run after doctor and dry-run pass. +# Add -Upload to send every available fresh valid payload after validation attempts. $env:DD_API_KEY = "" $env:DD_SITE = "datadoghq.com" .\tools\test_optimization\run_test_optimization_ci.ps1 ` @@ -440,9 +440,9 @@ $env:DD_SITE = "datadoghq.com" ``` Notes: -- The wrapper preserves test failures, blocks upload success when doctor - or dry-run enrichment fails, and still fails on uploader errors when the - earlier steps passed. +- The wrapper preserves test and validation failures while still uploading every + available fresh valid payload when upload is enabled. Uploader errors fail the + job when no earlier phase already failed. - For first-pass support after tests have run, the doctor `--support-bundle=.topt/reports/dd-test-optimization-support.zip` option creates a doctor-only bundle without vendoring the wrapper helper directory. diff --git a/modules/go/tools/dd_topt_go_bootstrap/main.go b/modules/go/tools/dd_topt_go_bootstrap/main.go index b9ea7c47..110893ea 100644 --- a/modules/go/tools/dd_topt_go_bootstrap/main.go +++ b/modules/go/tools/dd_topt_go_bootstrap/main.go @@ -1505,7 +1505,9 @@ Runs the Datadog Go Test Optimization validation flow: sync -> controls -> instrumented tests -> doctor -> dry-run uploader -> optional upload Upload is disabled by default. Pass --upload only when local Datadog -credentials are already available in the environment. +credentials are already available in the environment. When enabled, every +available fresh valid payload is uploaded even if an earlier phase failed; the +earliest failure remains the script result. Set DD_TEST_OPTIMIZATION_REPORT_DIR to persist doctor/uploader reports in CI. EOF @@ -1677,40 +1679,36 @@ if ((${#INSTRUMENTED_TARGETS[@]} > 0)); then done fi -if (( test_status != 0 )); then - warn "one or more tests failed; skipping doctor and upload" - exit "${test_status}" -fi +final_status=${test_status} check_disk run_step "doctor ${DOCTOR_TARGET}" "${BAZEL}" run "${RUN_FLAGS[@]}" "${DOCTOR_TARGET}" -- "${BEP_JSON_ARGS[@]}" "${BEP_RUN_ARGS[@]}" "--report-json=${DOCTOR_REPORT_JSON}" doctor_status=$? -if (( doctor_status != 0 )); then - warn "doctor failed; skipping upload" - log_report_dir - exit "${doctor_status}" +if (( doctor_status != 0 && final_status == 0 )); then + final_status=${doctor_status} fi check_disk run_step "dry-run upload ${UPLOAD_TARGET}" "${BAZEL}" run "${RUN_FLAGS[@]}" "${UPLOAD_TARGET}" -- "${BEP_JSON_ARGS[@]}" "${BEP_RUN_ARGS[@]}" "--report-json=${UPLOADER_DRY_RUN_REPORT_JSON}" --dry-run --validate-enrichment dry_run_status=$? -if (( dry_run_status != 0 )); then - warn "dry-run uploader failed; skipping upload" - log_report_dir - exit "${dry_run_status}" +if (( dry_run_status != 0 && final_status == 0 )); then + final_status=${dry_run_status} fi if (( upload == 0 )); then log "upload skipped; rerun with --upload to run ${UPLOAD_TARGET}" log_report_dir - exit 0 + exit "${final_status}" fi check_disk run_step "upload ${UPLOAD_TARGET}" "${BAZEL}" run "${RUN_FLAGS[@]}" "${UPLOAD_TARGET}" -- "${BEP_JSON_ARGS[@]}" "${BEP_RUN_ARGS[@]}" "--report-json=${UPLOADER_UPLOAD_REPORT_JSON}" upload_status=$? +if (( upload_status != 0 && final_status == 0 )); then + final_status=${upload_status} +fi log_report_dir -exit "${upload_status}" +exit "${final_status}" `) return buf.String(), nil } diff --git a/modules/go/tools/dd_topt_go_bootstrap/main_test.go b/modules/go/tools/dd_topt_go_bootstrap/main_test.go index 188cc108..f97e6147 100644 --- a/modules/go/tools/dd_topt_go_bootstrap/main_test.go +++ b/modules/go/tools/dd_topt_go_bootstrap/main_test.go @@ -1009,6 +1009,63 @@ set -euo pipefail } } +func TestValidationScriptUploadsAfterPartialFailures(t *testing.T) { + dir := t.TempDir() + fakeBazel := filepath.Join(dir, "bazel") + logPath := filepath.Join(dir, "bazel.log") + fakeBazelScript := `#!/usr/bin/env bash +printf '%s\n' "$*" >> "$BAZEL_LOG" +if [[ "${1:-}" == "test" ]]; then exit 7; fi +if [[ "${1:-}" == "run" && "$*" == *"//:dd_test_optimization_doctor"* ]]; then exit 8; fi +if [[ "${1:-}" == "run" && "$*" == *"//:dd_upload_payloads"*"--dry-run"* ]]; then exit 9; fi +if [[ "${1:-}" == "run" && "$*" == *"//:dd_upload_payloads"* ]]; then exit 10; fi +exit 0 +` + if err := os.WriteFile(fakeBazel, []byte(fakeBazelScript), 0o755); err != nil { + t.Fatalf("write fake bazel: %v", err) + } + + script, err := validationScript(config{ + printValidationScript: true, + bazelCommand: fakeBazel, + bazelConfig: "test-optimization", + syncRepoName: defaultSyncRepoName, + validationDoctorTarget: "//:dd_test_optimization_doctor", + validationUploadTarget: "//:dd_upload_payloads", + expectedTargets: []string{"//pkg:go_default_test"}, + minFreeDiskGB: defaultMinFreeDiskGB, + }) + if err != nil { + t.Fatalf("validationScript error: %v", err) + } + scriptPath := filepath.Join(dir, "validate.sh") + if err := os.WriteFile(scriptPath, []byte(script), 0o755); err != nil { + t.Fatalf("write validation script: %v", err) + } + + cmd := exec.Command("bash", scriptPath, "--upload") + cmd.Dir = dir + cmd.Env = append(os.Environ(), "BAZEL_LOG="+logPath) + output, err := cmd.CombinedOutput() + exitErr, ok := err.(*exec.ExitError) + if !ok || exitErr.ExitCode() != 7 { + t.Fatalf("validation script error=%v, want test exit 7\n%s", err, output) + } + + logBytes, err := os.ReadFile(logPath) + if err != nil { + t.Fatalf("read fake bazel log: %v", err) + } + logText := string(logBytes) + if got := strings.Count(logText, "run --config=test-optimization //:dd_upload_payloads"); got != 2 { + t.Fatalf("uploader command count=%d, want dry-run and real upload:\n%s", got, logText) + } + if !strings.Contains(logText, "--dry-run --validate-enrichment") || + !strings.Contains(logText, "uploader-upload-report.json") { + t.Fatalf("validation script did not run both uploader phases:\n%s", logText) + } +} + func TestValidationScriptUsesBepArtifactSourceForZippedOutputs(t *testing.T) { dir := t.TempDir() fakeBazel := filepath.Join(dir, "bazel") diff --git a/tools/agent-skills/go-test-optimization-onboarding/SKILL.md b/tools/agent-skills/go-test-optimization-onboarding/SKILL.md index 0a8e9a42..15314ee7 100644 --- a/tools/agent-skills/go-test-optimization-onboarding/SKILL.md +++ b/tools/agent-skills/go-test-optimization-onboarding/SKILL.md @@ -136,7 +136,7 @@ Every successful Go onboarding should end with these pieces: Prefer the wrapper support bundle for full CI escalation; use the doctor-only support bundle for the simplest initial customer request. Keep individual reports for local inspection and manual fallback flows. -- Real upload happens only after tests, doctor, and dry-run enrichment pass. +- Real upload processes available fresh valid payloads after doctor and dry-run attempts, while preserving any earlier failure. For automatic managed Go/Python monorepos, the universal shape has these additional constraints: @@ -189,7 +189,7 @@ Go wrapper, treat it as a consumer-specific integration: - Validate with fresh `bazel-testlogs//test.outputs/`, inspect `bazel_target_metadata.json` for `bazel.go.orchestrion.mode = "test_optimization"` on Go targets, then run - the doctor and uploader dry-run before any real upload. + the doctor and uploader dry-run before the real upload attempt. ## Branch And PR Hygiene diff --git a/tools/agent-skills/go-test-optimization-onboarding/references/validation-checklist.md b/tools/agent-skills/go-test-optimization-onboarding/references/validation-checklist.md index b71a34c1..11fdb625 100644 --- a/tools/agent-skills/go-test-optimization-onboarding/references/validation-checklist.md +++ b/tools/agent-skills/go-test-optimization-onboarding/references/validation-checklist.md @@ -227,8 +227,8 @@ artifact_staging_dir="$(mktemp -d "${TMPDIR:-/tmp}/dd-topt-artifacts.XXXXXX")" report_dir="${REPORT_DIR:-.topt/reports}" mkdir -p "$report_dir" +test_status=0; doctor_status=0; dry_run_status=0; upload_status=0 bazel test --config=test-optimization --build_event_json_file="$bep_json" //path/to:pilot_test || test_status=$? -test_status=${test_status:-0} bazel run --config=test-optimization //:dd_test_optimization_doctor -- \ --bep-json="$bep_json" \ @@ -237,11 +237,6 @@ bazel run --config=test-optimization //:dd_test_optimization_docto --artifact-source=bep \ --artifact-staging-dir="$artifact_staging_dir" \ --report-json="$report_dir/doctor-report.json" || doctor_status=$? -doctor_status=${doctor_status:-0} -if [ "$doctor_status" -ne 0 ]; then - if [ "$test_status" -ne 0 ]; then exit "$test_status"; fi - exit "$doctor_status" -fi bazel run --config=test-optimization //:dd_upload_payloads -- \ --bep-json="$bep_json" \ @@ -252,11 +247,6 @@ bazel run --config=test-optimization //:dd_upload_payloads -- \ --dry-run \ --validate-enrichment \ --report-json="$report_dir/uploader-dry-run-report.json" || dry_run_status=$? -dry_run_status=${dry_run_status:-0} -if [ "$dry_run_status" -ne 0 ]; then - if [ "$test_status" -ne 0 ]; then exit "$test_status"; fi - exit "$dry_run_status" -fi DD_API_KEY="$DD_API_KEY" DD_SITE="$DD_SITE" \ bazel run --config=test-optimization //:dd_upload_payloads -- \ @@ -265,11 +255,11 @@ DD_API_KEY="$DD_API_KEY" DD_SITE="$DD_SITE" \ --freshness-mode=required \ --artifact-source=bep \ --artifact-staging-dir="$artifact_staging_dir" \ - --report-json="$report_dir/uploader-upload-report.json" -upload_status=$? + --report-json="$report_dir/uploader-upload-report.json" || upload_status=$? -if [ "$test_status" -ne 0 ]; then exit "$test_status"; fi -exit "$upload_status" +for status in "$test_status" "$doctor_status" "$dry_run_status" "$upload_status"; do + if [ "$status" -ne 0 ]; then exit "$status"; fi +done ``` Do not run the real upload unless credentials are intentionally available and diff --git a/tools/agent-skills/java-test-optimization-onboarding/SKILL.md b/tools/agent-skills/java-test-optimization-onboarding/SKILL.md index 48e44e0a..a73e0e1a 100644 --- a/tools/agent-skills/java-test-optimization-onboarding/SKILL.md +++ b/tools/agent-skills/java-test-optimization-onboarding/SKILL.md @@ -132,7 +132,7 @@ Every successful Java onboarding should end with these pieces: - `FETCH_SALT` is used only for a separate, explicit `bazel sync --only= --repo_env=FETCH_SALT="$(date +%s)"` refresh, never as part of normal test, doctor, or uploader commands. -- Real upload happens only after tests, doctor, and dry-run enrichment pass. +- Real upload processes available fresh valid payloads after doctor and dry-run attempts, while preserving any earlier failure. Use the consumer's existing Bazel entrypoint in all commands. Do not switch a repository from `bzl` or `bazelw` to raw `bazel` just because examples use the diff --git a/tools/agent-skills/java-test-optimization-onboarding/references/validation-checklist.md b/tools/agent-skills/java-test-optimization-onboarding/references/validation-checklist.md index 308cab9a..ad62aebd 100644 --- a/tools/agent-skills/java-test-optimization-onboarding/references/validation-checklist.md +++ b/tools/agent-skills/java-test-optimization-onboarding/references/validation-checklist.md @@ -116,8 +116,8 @@ artifact_staging_dir="$(mktemp -d "${TMPDIR:-/tmp}/dd-topt-artifacts.XXXXXX")" report_dir="${REPORT_DIR:-.topt/reports}" mkdir -p "$report_dir" +test_status=0; doctor_status=0; dry_run_status=0; upload_status=0 bazel test --config=test-optimization --build_event_json_file="$bep_json" //path/to:java_test || test_status=$? -test_status=${test_status:-0} bazel run --config=test-optimization //tools/test_optimization:dd_test_optimization_doctor -- \ --bep-json="$bep_json" \ @@ -126,11 +126,6 @@ bazel run --config=test-optimization //tools/test_optimization:dd_test_optimizat --artifact-source=bep \ --artifact-staging-dir="$artifact_staging_dir" \ --report-json="$report_dir/doctor-report.json" || doctor_status=$? -doctor_status=${doctor_status:-0} -if [ "$doctor_status" -ne 0 ]; then - if [ "$test_status" -ne 0 ]; then exit "$test_status"; fi - exit "$doctor_status" -fi bazel run --config=test-optimization //tools/test_optimization:dd_upload_payloads -- \ --bep-json="$bep_json" \ @@ -141,11 +136,6 @@ bazel run --config=test-optimization //tools/test_optimization:dd_upload_payload --dry-run \ --validate-enrichment \ --report-json="$report_dir/uploader-dry-run-report.json" || dry_run_status=$? -dry_run_status=${dry_run_status:-0} -if [ "$dry_run_status" -ne 0 ]; then - if [ "$test_status" -ne 0 ]; then exit "$test_status"; fi - exit "$dry_run_status" -fi DD_API_KEY="$DD_API_KEY" DD_SITE="$DD_SITE" \ bazel run --config=test-optimization //tools/test_optimization:dd_upload_payloads -- \ @@ -154,11 +144,11 @@ DD_API_KEY="$DD_API_KEY" DD_SITE="$DD_SITE" \ --freshness-mode=required \ --artifact-source=bep \ --artifact-staging-dir="$artifact_staging_dir" \ - --report-json="$report_dir/uploader-upload-report.json" -upload_status=$? + --report-json="$report_dir/uploader-upload-report.json" || upload_status=$? -if [ "$test_status" -ne 0 ]; then exit "$test_status"; fi -exit "$upload_status" +for status in "$test_status" "$doctor_status" "$dry_run_status" "$upload_status"; do + if [ "$status" -ne 0 ]; then exit "$status"; fi +done ``` Do not run the real upload unless credentials are intentionally available and @@ -176,7 +166,8 @@ After tests: tags. - The doctor passes. - Dry-run enrichment passes. -- Real upload sends data only after local validation succeeds. +- Real upload processes every available fresh valid payload after validation + attempts; any earlier validation failure still fails the workflow. - Datadog shows Git metadata, Bazel metadata, and the expected test service. Do not list build-only or analysis-only targets in doctor `expected_targets`; diff --git a/tools/agent-skills/python-test-optimization-onboarding/SKILL.md b/tools/agent-skills/python-test-optimization-onboarding/SKILL.md index 8773b745..a8ca61fd 100644 --- a/tools/agent-skills/python-test-optimization-onboarding/SKILL.md +++ b/tools/agent-skills/python-test-optimization-onboarding/SKILL.md @@ -128,7 +128,7 @@ Every successful Python onboarding should end with these pieces: - `FETCH_SALT` is used only for a separate, explicit `bazel sync --config=test-optimization --only= --repo_env=FETCH_SALT="$(date +%s)"` refresh, never as part of normal test, doctor, or uploader commands. -- Real upload happens only after tests, doctor, and dry-run enrichment pass. +- Real upload processes available fresh valid payloads after doctor and dry-run attempts, while preserving any earlier failure. For automatic managed Go/Python monorepos: diff --git a/tools/agent-skills/python-test-optimization-onboarding/references/validation-checklist.md b/tools/agent-skills/python-test-optimization-onboarding/references/validation-checklist.md index 16ca77f5..65a0a53b 100644 --- a/tools/agent-skills/python-test-optimization-onboarding/references/validation-checklist.md +++ b/tools/agent-skills/python-test-optimization-onboarding/references/validation-checklist.md @@ -188,8 +188,8 @@ artifact_staging_dir="$(mktemp -d "${TMPDIR:-/tmp}/dd-topt-artifacts.XXXXXX")" report_dir="${REPORT_DIR:-.topt/reports}" mkdir -p "$report_dir" +test_status=0; doctor_status=0; dry_run_status=0; upload_status=0 bazel test --config=test-optimization --build_event_json_file="$bep_json" //path/to:python_test || test_status=$? -test_status=${test_status:-0} bazel run --config=test-optimization //tools/test_optimization:dd_test_optimization_doctor -- \ --bep-json="$bep_json" \ @@ -198,11 +198,6 @@ bazel run --config=test-optimization //tools/test_optimization:dd_test_optimizat --artifact-source=bep \ --artifact-staging-dir="$artifact_staging_dir" \ --report-json="$report_dir/doctor-report.json" || doctor_status=$? -doctor_status=${doctor_status:-0} -if [ "$doctor_status" -ne 0 ]; then - if [ "$test_status" -ne 0 ]; then exit "$test_status"; fi - exit "$doctor_status" -fi bazel run --config=test-optimization //tools/test_optimization:dd_upload_payloads -- \ --bep-json="$bep_json" \ @@ -213,11 +208,6 @@ bazel run --config=test-optimization //tools/test_optimization:dd_upload_payload --dry-run \ --validate-enrichment \ --report-json="$report_dir/uploader-dry-run-report.json" || dry_run_status=$? -dry_run_status=${dry_run_status:-0} -if [ "$dry_run_status" -ne 0 ]; then - if [ "$test_status" -ne 0 ]; then exit "$test_status"; fi - exit "$dry_run_status" -fi DD_API_KEY="$DD_API_KEY" DD_SITE="$DD_SITE" \ bazel run --config=test-optimization //tools/test_optimization:dd_upload_payloads -- \ @@ -226,11 +216,11 @@ DD_API_KEY="$DD_API_KEY" DD_SITE="$DD_SITE" \ --freshness-mode=required \ --artifact-source=bep \ --artifact-staging-dir="$artifact_staging_dir" \ - --report-json="$report_dir/uploader-upload-report.json" -upload_status=$? + --report-json="$report_dir/uploader-upload-report.json" || upload_status=$? -if [ "$test_status" -ne 0 ]; then exit "$test_status"; fi -exit "$upload_status" +for status in "$test_status" "$doctor_status" "$dry_run_status" "$upload_status"; do + if [ "$status" -ne 0 ]; then exit "$status"; fi +done ``` Do not run the real upload unless credentials are intentionally available and @@ -244,7 +234,8 @@ After tests: - `bazel_target_metadata.json` exists for instrumented runtime tests. - The doctor passes. - Dry-run enrichment passes. -- Real upload sends data only after local validation succeeds. +- Real upload processes every available fresh valid payload after validation + attempts; any earlier validation failure still fails the workflow. - Datadog shows Git metadata, Bazel metadata, and the expected test service. Do not list build-only or analysis-only targets in doctor `expected_targets`; diff --git a/tools/core/uploader_bash_runtime.sh.tpl b/tools/core/uploader_bash_runtime.sh.tpl index 53bbb3fd..1165a8fe 100644 --- a/tools/core/uploader_bash_runtime.sh.tpl +++ b/tools/core/uploader_bash_runtime.sh.tpl @@ -3240,7 +3240,7 @@ validate_expected_target_coverage() { (( EXPECTED_TARGETS_CONFIGURED == 1 )) || return 0 [[ "$FRESHNESS_SELECTED_SOURCE" == "bep" ]] || return 0 - local label + local label missing_count=0 while IFS= read -r label; do [[ -n "$label" ]] || continue if grep -Fq "$label"$'\t' "$FRESHNESS_ELIGIBLE_OUTPUTS_FILE" 2>/dev/null || @@ -3251,12 +3251,15 @@ validate_expected_target_coverage() { continue fi if grep -Fxq "$label" "$FRESHNESS_MISSING_OUTPUT_LABELS_FILE" 2>/dev/null; then - log "error: expected target output is neither fresh nor exclusively cached in BEP: $label (the fresh TestResult did not contain a mappable test.outputs reference)" + log "warning: expected target output is neither fresh nor exclusively cached in BEP: $label (the fresh TestResult did not contain a mappable test.outputs reference); continuing with other fresh outputs" else - log "error: expected target output is neither fresh nor exclusively cached in BEP: $label (no TestResult matched this target)" + log "warning: expected target output is neither fresh nor exclusively cached in BEP: $label (no TestResult matched this target); continuing with other fresh outputs" fi - exit 2 + ((++missing_count)) done <"$EXPECTED_TARGETS_RESOLVED_FILE" + if (( missing_count > 0 )); then + log "warning: $missing_count expected target(s) produced no current uploadable output; available fresh payloads will still be processed" + fi } validate_bep_remote_only_outputs() { diff --git a/tools/core/uploader_powershell_runtime.ps1.tpl b/tools/core/uploader_powershell_runtime.ps1.tpl index 310aad3d..af7f2cee 100644 --- a/tools/core/uploader_powershell_runtime.ps1.tpl +++ b/tools/core/uploader_powershell_runtime.ps1.tpl @@ -3223,6 +3223,7 @@ function Merge-StagedBepFreshness { function Assert-ExpectedTargetCoverage { if (-not $script:ExpectedTargetsConfigured -or $script:FreshnessSelectedSource -ne "bep") { return } + $missingCount = 0 foreach ($label in $script:ExpectedTargets) { $hasFresh = @($script:FreshnessEligibleOutputs | Where-Object { $_.StartsWith("$label`t", [System.StringComparison]::Ordinal) }).Count -gt 0 $hasCached = @($script:FreshnessCachedOutputs | Where-Object { $_.StartsWith("$label`t", [System.StringComparison]::Ordinal) }).Count -gt 0 @@ -3230,11 +3231,14 @@ function Assert-ExpectedTargetCoverage { $hasRemote = @($script:FreshnessRemoteOnlyOutputs | Where-Object { $_.Label -eq $label }).Count -gt 0 if ($hasRemote) { continue } if ($script:FreshnessMissingOutputLabels.Contains($label)) { - Log "error: expected target output is neither fresh nor exclusively cached in BEP: $label (the fresh TestResult did not contain a mappable test.outputs reference)" + Log "warning: expected target output is neither fresh nor exclusively cached in BEP: $label (the fresh TestResult did not contain a mappable test.outputs reference); continuing with other fresh outputs" } else { - Log "error: expected target output is neither fresh nor exclusively cached in BEP: $label (no TestResult matched this target)" + Log "warning: expected target output is neither fresh nor exclusively cached in BEP: $label (no TestResult matched this target); continuing with other fresh outputs" } - exit 2 + $missingCount++ + } + if ($missingCount -gt 0) { + Log "warning: $missingCount expected target(s) produced no current uploadable output; available fresh payloads will still be processed" } } diff --git a/tools/test_optimization/run_test_optimization_ci.ps1 b/tools/test_optimization/run_test_optimization_ci.ps1 index c060426a..3df3a933 100644 --- a/tools/test_optimization/run_test_optimization_ci.ps1 +++ b/tools/test_optimization/run_test_optimization_ci.ps1 @@ -236,19 +236,16 @@ try { $finalStatus = $doctorStatus } - $dryRunStatus = 0 - if ($doctorStatus -eq 0) { - $dryRunRuntimeArgs = $runtimeArgs - if (-not [string]::IsNullOrWhiteSpace($UploaderReportJson)) { - $dryRunRuntimeArgs += "--report-json=$UploaderReportJson" - } - $dryRunStatus = Invoke-BazelCommand -Args (@("run", "--config=$Config", $UploadTarget, "--") + $dryRunRuntimeArgs + @("--dry-run", "--validate-enrichment")) - if ($dryRunStatus -ne 0 -and $finalStatus -eq 0) { - $finalStatus = $dryRunStatus - } + $dryRunRuntimeArgs = $runtimeArgs + if (-not [string]::IsNullOrWhiteSpace($UploaderReportJson)) { + $dryRunRuntimeArgs += "--report-json=$UploaderReportJson" + } + $dryRunStatus = Invoke-BazelCommand -Args (@("run", "--config=$Config", $UploadTarget, "--") + $dryRunRuntimeArgs + @("--dry-run", "--validate-enrichment")) + if ($dryRunStatus -ne 0 -and $finalStatus -eq 0) { + $finalStatus = $dryRunStatus } - if ($doctorStatus -eq 0 -and $dryRunStatus -eq 0 -and $Upload.IsPresent) { + if ($Upload.IsPresent) { $uploadRuntimeArgs = $runtimeArgs if (-not [string]::IsNullOrWhiteSpace($UploadReportJson)) { $uploadRuntimeArgs += "--report-json=$UploadReportJson" diff --git a/tools/test_optimization/run_test_optimization_ci.sh b/tools/test_optimization/run_test_optimization_ci.sh index 0f8f42a9..6bca2b5f 100755 --- a/tools/test_optimization/run_test_optimization_ci.sh +++ b/tools/test_optimization/run_test_optimization_ci.sh @@ -43,7 +43,7 @@ Options: --support-bundle-collector PATH Collector script path. Defaults to create_support_bundle.py beside this wrapper. --test-flag FLAG Extra flag passed to every bazel test invocation. - --upload Run the real upload after dry-run enrichment validation. + --upload Upload every available fresh valid payload after validation attempts. --no-upload Skip the real upload. This is the default. --keep-tmp Keep generated BEP and artifact-staging files. -h, --help Show this help. @@ -394,24 +394,20 @@ else fi fi -if [[ "$doctor_status" -eq 0 ]]; then - dry_run_runtime_args=("${runtime_args[@]}") - if [[ -n "$UPLOADER_REPORT_JSON" ]]; then - dry_run_runtime_args+=("--report-json=$UPLOADER_REPORT_JSON") - fi - if run_bazel run "--config=$BAZEL_CONFIG" "$UPLOAD_TARGET" -- "${dry_run_runtime_args[@]}" --dry-run --validate-enrichment; then - dry_run_status=0 - else - dry_run_status=$? - if [[ "$final_status" -eq 0 ]]; then - final_status="$dry_run_status" - fi - fi -else +dry_run_runtime_args=("${runtime_args[@]}") +if [[ -n "$UPLOADER_REPORT_JSON" ]]; then + dry_run_runtime_args+=("--report-json=$UPLOADER_REPORT_JSON") +fi +if run_bazel run "--config=$BAZEL_CONFIG" "$UPLOAD_TARGET" -- "${dry_run_runtime_args[@]}" --dry-run --validate-enrichment; then dry_run_status=0 +else + dry_run_status=$? + if [[ "$final_status" -eq 0 ]]; then + final_status="$dry_run_status" + fi fi -if [[ "$doctor_status" -eq 0 && "$dry_run_status" -eq 0 && "$DO_UPLOAD" -eq 1 ]]; then +if [[ "$DO_UPLOAD" -eq 1 ]]; then upload_runtime_args=("${runtime_args[@]}") if [[ -n "$UPLOAD_REPORT_JSON" ]]; then upload_runtime_args+=("--report-json=$UPLOAD_REPORT_JSON") diff --git a/tools/tests/python/test_python_tools.py b/tools/tests/python/test_python_tools.py index 6d6638ac..86541002 100644 --- a/tools/tests/python/test_python_tools.py +++ b/tools/tests/python/test_python_tools.py @@ -546,6 +546,51 @@ def test_bash_wrapper_preserves_failed_test_status(self) -> None: normalized_log_text, ) + def test_bash_wrapper_uploads_after_partial_validation_failures(self) -> None: + """Validate upload still processes fresh payloads after validation failures.""" + bash = _require_functional_bash(self) + wrapper = _runfile("tools/test_optimization/run_test_optimization_ci.sh") + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + fake_bazel = root / "bazel" + log_path = root / "bazel.log" + fake_bazel.write_text( + f"""#!/usr/bin/env bash +printf '%s\\n' "$*" >> {str(log_path)!r} +if [[ "$1" == "test" ]]; then exit 0; fi +if [[ "$1" == "run" && "$3" == "//:dd_test_optimization_doctor" ]]; then exit 11; fi +if [[ "$1" == "run" && "$*" == *"--dry-run"* ]]; then exit 12; fi +if [[ "$1" == "run" && "$3" == "//:dd_upload_payloads" ]]; then exit 0; fi +exit 99 +""", + encoding="utf-8", + ) + fake_bazel.chmod(0o755) + tmpdir = root / "tmp" + tmpdir.mkdir() + env = os.environ.copy() + env["BAZEL"] = str(fake_bazel) + env["DD_TEST_OPTIMIZATION_TMPDIR"] = str(tmpdir) + + result = subprocess.run( + [bash, str(wrapper), "--upload", "//pkg:target"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + check=False, + ) + + self.assertEqual(11, result.returncode, result.stderr) + upload_commands = [ + line + for line in log_path.read_text(encoding="utf-8").splitlines() + if "//:dd_upload_payloads" in line + ] + self.assertEqual(2, len(upload_commands), upload_commands) + self.assertIn("--dry-run --validate-enrichment", upload_commands[0]) + self.assertNotIn("--dry-run", upload_commands[1]) + def test_bash_wrapper_support_bundle_preserves_failed_test_status(self) -> None: """Validate Bash support bundle path preserves the original test status.""" if os.name == "nt": @@ -844,6 +889,64 @@ def test_powershell_wrapper_reaches_uploader_when_bazel_writes_stdout(self) -> N self.assertIn("//:dd_upload_payloads", log_text) self.assertIn("--dry-run --validate-enrichment", log_text) + def test_powershell_wrapper_uploads_after_partial_validation_failures(self) -> None: + """Validate PowerShell uploads fresh payloads after validation failures.""" + pwsh = _require_command(self, "pwsh", "pwsh is required for PowerShell wrapper smoke") + wrapper = _runfile("tools/test_optimization/run_test_optimization_ci.ps1") + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + fake_bazel = root / "bazel.ps1" + log_path = root / "bazel.log" + fake_bazel.write_text( + f""" +param([Parameter(ValueFromRemainingArguments = $true)][string[]]$BazelArgs) +$Command = $BazelArgs -join ' ' +Add-Content -LiteralPath {str(log_path)!r} -Value $Command +if ($BazelArgs[0] -eq 'test') {{ exit 0 }} +if ($Command -like '*//:dd_test_optimization_doctor*') {{ exit 11 }} +if ($Command -like '*//:dd_upload_payloads*--dry-run*') {{ exit 12 }} +if ($Command -like '*//:dd_upload_payloads*') {{ exit 0 }} +exit 99 +""", + encoding="utf-8", + ) + fake_bazel.chmod(0o755) + tmpdir = root / "tmp" + tmpdir.mkdir() + env = os.environ.copy() + env["DD_TEST_OPTIMIZATION_TMPDIR"] = str(tmpdir) + + result = subprocess.run( + [ + pwsh, + "-NoLogo", + "-NoProfile", + "-ExecutionPolicy", + "Bypass", + "-File", + str(wrapper), + "-Bazel", + str(fake_bazel), + "-Upload", + "//pkg:target", + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + check=False, + ) + + self.assertEqual(11, result.returncode, result.stderr + result.stdout) + upload_commands = [ + line + for line in log_path.read_text(encoding="utf-8").splitlines() + if "//:dd_upload_payloads" in line + ] + self.assertEqual(2, len(upload_commands), upload_commands) + self.assertIn("--dry-run --validate-enrichment", upload_commands[0]) + self.assertNotIn("--dry-run", upload_commands[1]) + def test_powershell_wrapper_support_bundle_preserves_failed_test_status(self) -> None: """Validate PowerShell support bundle path preserves the original test status.""" pwsh = _require_command(self, "pwsh", "pwsh is required for PowerShell support bundle smoke") @@ -6134,7 +6237,7 @@ def _write_mixed_fresh_output_fixture(root: Path) -> tuple[Path, Path]: expected_targets.write_text( json.dumps({ "schema_version": 1, - "targets": ["//pkg:empty", "//pkg:valid"], + "targets": ["//pkg:empty", "//pkg:missing", "//pkg:valid"], }), encoding="utf-8", ) @@ -6227,8 +6330,8 @@ def _assert_uploader_report_success(self, report_path: Path, bep_path: Path, sta self.assertEqual(0, report["payloads"]["telemetry"]["processed"]) self.assertEqual(0, report["upload_failures"]) - def test_generated_uploaders_reject_any_fresh_expected_output_without_payloads(self) -> None: - """Validate one valid output cannot hide another empty expected output.""" + def test_generated_uploaders_process_valid_payloads_before_reporting_partial_outputs(self) -> None: + """Validate missing or empty outputs do not hide another valid expected output.""" _require_command(self, "jq", "jq is required for Bash BEP freshness parsing") bash = _require_functional_bash(self) if os.name == "nt": @@ -6285,6 +6388,13 @@ def test_generated_uploaders_reject_any_fresh_expected_output_without_payloads(s ] for name, command in invocations: with self.subTest(runtime=name): + expected_targets.write_text( + json.dumps({ + "schema_version": 1, + "targets": ["//pkg:empty", "//pkg:missing", "//pkg:valid"], + }), + encoding="utf-8", + ) report = root / f"{name.lower()}-report.json" result = subprocess.run( [ @@ -6307,11 +6417,50 @@ def test_generated_uploaders_reject_any_fresh_expected_output_without_payloads(s ) output = result.stdout + result.stderr self.assertNotEqual(0, result.returncode, output) + self.assertIn( + "no TestResult matched this target); continuing with other fresh outputs", + output, + ) self.assertIn("fresh expected test output produced no uploadable payloads", output) self.assertIn("//pkg:empty", output) report_doc = json.loads(report.read_text(encoding="utf-8")) self.assertEqual("fail", report_doc["status"]) self.assertEqual("upload_failed_unknown", report_doc["result"]["reason_code"]) + self.assertEqual(1, report_doc["payloads"]["tests"]["processed"]) + + expected_targets.write_text( + json.dumps({ + "schema_version": 1, + "targets": ["//pkg:missing", "//pkg:valid"], + }), + encoding="utf-8", + ) + partial_report = root / f"{name.lower()}-missing-report.json" + partial_result = subprocess.run( + [ + *command, + "--bep-json", + str(bep), + "--freshness-source=bep", + "--freshness-mode=required", + "--report-json", + str(partial_report), + "--dry-run", + ], + cwd=root, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=30, + check=False, + ) + partial_output = partial_result.stdout + partial_result.stderr + self.assertEqual(0, partial_result.returncode, partial_output) + self.assertIn("continuing with other fresh outputs", partial_output) + partial_report_doc = json.loads(partial_report.read_text(encoding="utf-8")) + self.assertEqual("ok", partial_report_doc["status"]) + self.assertEqual(1, partial_report_doc["payloads"]["tests"]["processed"]) def test_generated_uploaders_accept_empty_expected_target_set(self) -> None: """Validate an invocation without optimized targets ignores unrelated BEP rows.""" From 7837867e4ec9e328dd9e2bc50b7ede54cf9e38e2 Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Sun, 23 Aug 2026 14:09:43 +0200 Subject: [PATCH 07/37] go: guard Orchestrion resolver package cycles --- .../go/private/orchestrion/extensions.bzl | 23 +++++++- .../starlark/orchestrion_extension_tests.bzl | 22 ++++++++ .../go/private/orchestrion/extensions.bzl | 23 +++++++- .../starlark/orchestrion_extension_tests.bzl | 22 ++++++++ .../go/private/orchestrion/extensions.bzl | 23 +++++++- .../starlark/orchestrion_extension_tests.bzl | 22 ++++++++ .../v0_60_0/base/0001-full-delta.patch | 53 +++++++++++++++++-- .../v0_61_1/base/0001-full-delta.patch | 53 +++++++++++++++++-- .../v0_62_0/base/0001-full-delta.patch | 53 +++++++++++++++++-- 9 files changed, 276 insertions(+), 18 deletions(-) diff --git a/third_party/rgo/v0_60_0/base/go/private/orchestrion/extensions.bzl b/third_party/rgo/v0_60_0/base/go/private/orchestrion/extensions.bzl index dc4e4830..9ec04e69 100644 --- a/third_party/rgo/v0_60_0/base/go/private/orchestrion/extensions.bzl +++ b/third_party/rgo/v0_60_0/base/go/private/orchestrion/extensions.bzl @@ -20,7 +20,7 @@ ORCHESTRION_SEED_GO_MOD_VERSION = "1.21" # Bump this identifier whenever the in-repo Orchestrion patch block changes in # a way that should invalidate previously cached bootstrap binaries. -ORCHESTRION_PATCHSET_ID = "20260427-go-tool-transition" +ORCHESTRION_PATCHSET_ID = "20260823-resolver-cycle-guard" _DD_TRACE_GO_MODULES = [ "github.com/DataDog/dd-trace-go/v2", "github.com/DataDog/dd-trace-go/contrib/net/http/v2", @@ -1225,6 +1225,25 @@ def _restore_bootstrap_cache(ctx, paths, version, version_map, go_identity, bina ctx.file("BUILD.bazel", _orchestrion_build_file(binary_name)) return True +def _patch_resolver_cycle_guard(source): + header = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error { + if pkg.PkgPath == "" || pkg.PkgPath == "unsafe" || r[pkg.PkgPath].ExportFile != "" {""" + guarded_header = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error { + return r.mergeFromVisited(pkg, make(map[*packages.Package]struct{})) +} + +func (r ResolveResponse) mergeFromVisited(pkg *packages.Package, visited map[*packages.Package]struct{}) error { + // packages.Load can return cycles whose nodes do not have ExportFile yet. + if _, ok := visited[pkg]; ok { + return nil + } + visited[pkg] = struct{}{} + if pkg.PkgPath == "" || pkg.PkgPath == "unsafe" || r[pkg.PkgPath].ExportFile != "" {""" + recursive_call = "r.mergeFrom(dep)" + if header not in source or recursive_call not in source: + fail("Could not patch Orchestrion resolver cycle guard") + return source.replace(header, guarded_header, 1).replace(recursive_call, "r.mergeFromVisited(dep, visited)", 1) + orchestrion_extension_test_helpers = struct( bootstrap_cache_key = _bootstrap_cache_key, bootstrap_manifest_content = _bootstrap_manifest_content, @@ -1243,6 +1262,7 @@ orchestrion_extension_test_helpers = struct( normalize_host_goos = _normalize_host_goos, parse_certutil_sha256 = _parse_certutil_sha256, powershell_single_quoted_literal = _powershell_single_quoted_literal, + patch_resolver_cycle_guard = _patch_resolver_cycle_guard, ) _TEST_OPTIMIZATION_ENABLED_ENV = "DD_TEST_OPTIMIZATION_ENABLED" @@ -1407,6 +1427,7 @@ def _orchestrion_build_impl(ctx): if resolve_tempdir_old not in resolve_src: fail("Could not patch Orchestrion resolver tempdir block in %s" % resolve_path) resolve_src = resolve_src.replace(resolve_tempdir_old, resolve_tempdir_new, 1) + resolve_src = _patch_resolver_cycle_guard(resolve_src) ctx.file(resolve_path, resolve_src) # Compile-proxy / archive metadata patches for the synthetic testmain flow. diff --git a/third_party/rgo/v0_60_0/base/tests/core/starlark/orchestrion_extension_tests.bzl b/third_party/rgo/v0_60_0/base/tests/core/starlark/orchestrion_extension_tests.bzl index ec4826f1..9d605a80 100644 --- a/third_party/rgo/v0_60_0/base/tests/core/starlark/orchestrion_extension_tests.bzl +++ b/third_party/rgo/v0_60_0/base/tests/core/starlark/orchestrion_extension_tests.bzl @@ -290,6 +290,27 @@ def _declared_dd_trace_go_versions_test(ctx): declared_dd_trace_go_versions_test = unittest.make(_declared_dd_trace_go_versions_test) +def _resolver_cycle_guard_patch_test(ctx): + env = unittest.begin(ctx) + source = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error { + if pkg.PkgPath == "" || pkg.PkgPath == "unsafe" || r[pkg.PkgPath].ExportFile != "" { + } + for _, dep := range pkg.Imports { + errs = errors.Join(errs, r.mergeFrom(dep)) + } +} +""" + + patched = orchestrion_extension_test_helpers.patch_resolver_cycle_guard(source) + asserts.true(env, "make(map[*packages.Package]struct{})" in patched) + asserts.true(env, "if _, ok := visited[pkg]; ok" in patched) + asserts.true(env, "r.mergeFromVisited(dep, visited)" in patched) + asserts.false(env, "r.mergeFrom(dep)" in patched) + + return unittest.end(env) + +resolver_cycle_guard_patch_test = unittest.make(_resolver_cycle_guard_patch_test) + def orchestrion_extension_test_suite(): unittest.suite( "orchestrion_extension_tests", @@ -307,4 +328,5 @@ def orchestrion_extension_test_suite(): module_proxy_seed_go_mod_test, parse_certutil_sha256_test, powershell_single_quoted_literal_test, + resolver_cycle_guard_patch_test, ) diff --git a/third_party/rgo/v0_61_1/base/go/private/orchestrion/extensions.bzl b/third_party/rgo/v0_61_1/base/go/private/orchestrion/extensions.bzl index dc4e4830..9ec04e69 100644 --- a/third_party/rgo/v0_61_1/base/go/private/orchestrion/extensions.bzl +++ b/third_party/rgo/v0_61_1/base/go/private/orchestrion/extensions.bzl @@ -20,7 +20,7 @@ ORCHESTRION_SEED_GO_MOD_VERSION = "1.21" # Bump this identifier whenever the in-repo Orchestrion patch block changes in # a way that should invalidate previously cached bootstrap binaries. -ORCHESTRION_PATCHSET_ID = "20260427-go-tool-transition" +ORCHESTRION_PATCHSET_ID = "20260823-resolver-cycle-guard" _DD_TRACE_GO_MODULES = [ "github.com/DataDog/dd-trace-go/v2", "github.com/DataDog/dd-trace-go/contrib/net/http/v2", @@ -1225,6 +1225,25 @@ def _restore_bootstrap_cache(ctx, paths, version, version_map, go_identity, bina ctx.file("BUILD.bazel", _orchestrion_build_file(binary_name)) return True +def _patch_resolver_cycle_guard(source): + header = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error { + if pkg.PkgPath == "" || pkg.PkgPath == "unsafe" || r[pkg.PkgPath].ExportFile != "" {""" + guarded_header = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error { + return r.mergeFromVisited(pkg, make(map[*packages.Package]struct{})) +} + +func (r ResolveResponse) mergeFromVisited(pkg *packages.Package, visited map[*packages.Package]struct{}) error { + // packages.Load can return cycles whose nodes do not have ExportFile yet. + if _, ok := visited[pkg]; ok { + return nil + } + visited[pkg] = struct{}{} + if pkg.PkgPath == "" || pkg.PkgPath == "unsafe" || r[pkg.PkgPath].ExportFile != "" {""" + recursive_call = "r.mergeFrom(dep)" + if header not in source or recursive_call not in source: + fail("Could not patch Orchestrion resolver cycle guard") + return source.replace(header, guarded_header, 1).replace(recursive_call, "r.mergeFromVisited(dep, visited)", 1) + orchestrion_extension_test_helpers = struct( bootstrap_cache_key = _bootstrap_cache_key, bootstrap_manifest_content = _bootstrap_manifest_content, @@ -1243,6 +1262,7 @@ orchestrion_extension_test_helpers = struct( normalize_host_goos = _normalize_host_goos, parse_certutil_sha256 = _parse_certutil_sha256, powershell_single_quoted_literal = _powershell_single_quoted_literal, + patch_resolver_cycle_guard = _patch_resolver_cycle_guard, ) _TEST_OPTIMIZATION_ENABLED_ENV = "DD_TEST_OPTIMIZATION_ENABLED" @@ -1407,6 +1427,7 @@ def _orchestrion_build_impl(ctx): if resolve_tempdir_old not in resolve_src: fail("Could not patch Orchestrion resolver tempdir block in %s" % resolve_path) resolve_src = resolve_src.replace(resolve_tempdir_old, resolve_tempdir_new, 1) + resolve_src = _patch_resolver_cycle_guard(resolve_src) ctx.file(resolve_path, resolve_src) # Compile-proxy / archive metadata patches for the synthetic testmain flow. diff --git a/third_party/rgo/v0_61_1/base/tests/core/starlark/orchestrion_extension_tests.bzl b/third_party/rgo/v0_61_1/base/tests/core/starlark/orchestrion_extension_tests.bzl index ec4826f1..9d605a80 100644 --- a/third_party/rgo/v0_61_1/base/tests/core/starlark/orchestrion_extension_tests.bzl +++ b/third_party/rgo/v0_61_1/base/tests/core/starlark/orchestrion_extension_tests.bzl @@ -290,6 +290,27 @@ def _declared_dd_trace_go_versions_test(ctx): declared_dd_trace_go_versions_test = unittest.make(_declared_dd_trace_go_versions_test) +def _resolver_cycle_guard_patch_test(ctx): + env = unittest.begin(ctx) + source = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error { + if pkg.PkgPath == "" || pkg.PkgPath == "unsafe" || r[pkg.PkgPath].ExportFile != "" { + } + for _, dep := range pkg.Imports { + errs = errors.Join(errs, r.mergeFrom(dep)) + } +} +""" + + patched = orchestrion_extension_test_helpers.patch_resolver_cycle_guard(source) + asserts.true(env, "make(map[*packages.Package]struct{})" in patched) + asserts.true(env, "if _, ok := visited[pkg]; ok" in patched) + asserts.true(env, "r.mergeFromVisited(dep, visited)" in patched) + asserts.false(env, "r.mergeFrom(dep)" in patched) + + return unittest.end(env) + +resolver_cycle_guard_patch_test = unittest.make(_resolver_cycle_guard_patch_test) + def orchestrion_extension_test_suite(): unittest.suite( "orchestrion_extension_tests", @@ -307,4 +328,5 @@ def orchestrion_extension_test_suite(): module_proxy_seed_go_mod_test, parse_certutil_sha256_test, powershell_single_quoted_literal_test, + resolver_cycle_guard_patch_test, ) diff --git a/third_party/rgo/v0_62_0/base/go/private/orchestrion/extensions.bzl b/third_party/rgo/v0_62_0/base/go/private/orchestrion/extensions.bzl index dc4e4830..9ec04e69 100644 --- a/third_party/rgo/v0_62_0/base/go/private/orchestrion/extensions.bzl +++ b/third_party/rgo/v0_62_0/base/go/private/orchestrion/extensions.bzl @@ -20,7 +20,7 @@ ORCHESTRION_SEED_GO_MOD_VERSION = "1.21" # Bump this identifier whenever the in-repo Orchestrion patch block changes in # a way that should invalidate previously cached bootstrap binaries. -ORCHESTRION_PATCHSET_ID = "20260427-go-tool-transition" +ORCHESTRION_PATCHSET_ID = "20260823-resolver-cycle-guard" _DD_TRACE_GO_MODULES = [ "github.com/DataDog/dd-trace-go/v2", "github.com/DataDog/dd-trace-go/contrib/net/http/v2", @@ -1225,6 +1225,25 @@ def _restore_bootstrap_cache(ctx, paths, version, version_map, go_identity, bina ctx.file("BUILD.bazel", _orchestrion_build_file(binary_name)) return True +def _patch_resolver_cycle_guard(source): + header = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error { + if pkg.PkgPath == "" || pkg.PkgPath == "unsafe" || r[pkg.PkgPath].ExportFile != "" {""" + guarded_header = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error { + return r.mergeFromVisited(pkg, make(map[*packages.Package]struct{})) +} + +func (r ResolveResponse) mergeFromVisited(pkg *packages.Package, visited map[*packages.Package]struct{}) error { + // packages.Load can return cycles whose nodes do not have ExportFile yet. + if _, ok := visited[pkg]; ok { + return nil + } + visited[pkg] = struct{}{} + if pkg.PkgPath == "" || pkg.PkgPath == "unsafe" || r[pkg.PkgPath].ExportFile != "" {""" + recursive_call = "r.mergeFrom(dep)" + if header not in source or recursive_call not in source: + fail("Could not patch Orchestrion resolver cycle guard") + return source.replace(header, guarded_header, 1).replace(recursive_call, "r.mergeFromVisited(dep, visited)", 1) + orchestrion_extension_test_helpers = struct( bootstrap_cache_key = _bootstrap_cache_key, bootstrap_manifest_content = _bootstrap_manifest_content, @@ -1243,6 +1262,7 @@ orchestrion_extension_test_helpers = struct( normalize_host_goos = _normalize_host_goos, parse_certutil_sha256 = _parse_certutil_sha256, powershell_single_quoted_literal = _powershell_single_quoted_literal, + patch_resolver_cycle_guard = _patch_resolver_cycle_guard, ) _TEST_OPTIMIZATION_ENABLED_ENV = "DD_TEST_OPTIMIZATION_ENABLED" @@ -1407,6 +1427,7 @@ def _orchestrion_build_impl(ctx): if resolve_tempdir_old not in resolve_src: fail("Could not patch Orchestrion resolver tempdir block in %s" % resolve_path) resolve_src = resolve_src.replace(resolve_tempdir_old, resolve_tempdir_new, 1) + resolve_src = _patch_resolver_cycle_guard(resolve_src) ctx.file(resolve_path, resolve_src) # Compile-proxy / archive metadata patches for the synthetic testmain flow. diff --git a/third_party/rgo/v0_62_0/base/tests/core/starlark/orchestrion_extension_tests.bzl b/third_party/rgo/v0_62_0/base/tests/core/starlark/orchestrion_extension_tests.bzl index ec4826f1..9d605a80 100644 --- a/third_party/rgo/v0_62_0/base/tests/core/starlark/orchestrion_extension_tests.bzl +++ b/third_party/rgo/v0_62_0/base/tests/core/starlark/orchestrion_extension_tests.bzl @@ -290,6 +290,27 @@ def _declared_dd_trace_go_versions_test(ctx): declared_dd_trace_go_versions_test = unittest.make(_declared_dd_trace_go_versions_test) +def _resolver_cycle_guard_patch_test(ctx): + env = unittest.begin(ctx) + source = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error { + if pkg.PkgPath == "" || pkg.PkgPath == "unsafe" || r[pkg.PkgPath].ExportFile != "" { + } + for _, dep := range pkg.Imports { + errs = errors.Join(errs, r.mergeFrom(dep)) + } +} +""" + + patched = orchestrion_extension_test_helpers.patch_resolver_cycle_guard(source) + asserts.true(env, "make(map[*packages.Package]struct{})" in patched) + asserts.true(env, "if _, ok := visited[pkg]; ok" in patched) + asserts.true(env, "r.mergeFromVisited(dep, visited)" in patched) + asserts.false(env, "r.mergeFrom(dep)" in patched) + + return unittest.end(env) + +resolver_cycle_guard_patch_test = unittest.make(_resolver_cycle_guard_patch_test) + def orchestrion_extension_test_suite(): unittest.suite( "orchestrion_extension_tests", @@ -307,4 +328,5 @@ def orchestrion_extension_test_suite(): module_proxy_seed_go_mod_test, parse_certutil_sha256_test, powershell_single_quoted_literal_test, + resolver_cycle_guard_patch_test, ) diff --git a/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch index 702f2d9e..590f993a 100644 --- a/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch @@ -2091,10 +2091,10 @@ index 0000000..31a01c3 +) diff --git a/go/private/orchestrion/extensions.bzl b/go/private/orchestrion/extensions.bzl new file mode 100644 -index 0000000..dc4e483 +index 0000000..9ec04e6 --- /dev/null +++ b/go/private/orchestrion/extensions.bzl -@@ -0,0 +1,1739 @@ +@@ -0,0 +1,1760 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); @@ -2117,7 +2117,7 @@ index 0000000..dc4e483 + +# Bump this identifier whenever the in-repo Orchestrion patch block changes in +# a way that should invalidate previously cached bootstrap binaries. -+ORCHESTRION_PATCHSET_ID = "20260427-go-tool-transition" ++ORCHESTRION_PATCHSET_ID = "20260823-resolver-cycle-guard" +_DD_TRACE_GO_MODULES = [ + "github.com/DataDog/dd-trace-go/v2", + "github.com/DataDog/dd-trace-go/contrib/net/http/v2", @@ -3322,6 +3322,25 @@ index 0000000..dc4e483 + ctx.file("BUILD.bazel", _orchestrion_build_file(binary_name)) + return True + ++def _patch_resolver_cycle_guard(source): ++ header = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error { ++ if pkg.PkgPath == "" || pkg.PkgPath == "unsafe" || r[pkg.PkgPath].ExportFile != "" {""" ++ guarded_header = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error { ++ return r.mergeFromVisited(pkg, make(map[*packages.Package]struct{})) ++} ++ ++func (r ResolveResponse) mergeFromVisited(pkg *packages.Package, visited map[*packages.Package]struct{}) error { ++ // packages.Load can return cycles whose nodes do not have ExportFile yet. ++ if _, ok := visited[pkg]; ok { ++ return nil ++ } ++ visited[pkg] = struct{}{} ++ if pkg.PkgPath == "" || pkg.PkgPath == "unsafe" || r[pkg.PkgPath].ExportFile != "" {""" ++ recursive_call = "r.mergeFrom(dep)" ++ if header not in source or recursive_call not in source: ++ fail("Could not patch Orchestrion resolver cycle guard") ++ return source.replace(header, guarded_header, 1).replace(recursive_call, "r.mergeFromVisited(dep, visited)", 1) ++ +orchestrion_extension_test_helpers = struct( + bootstrap_cache_key = _bootstrap_cache_key, + bootstrap_manifest_content = _bootstrap_manifest_content, @@ -3340,6 +3359,7 @@ index 0000000..dc4e483 + normalize_host_goos = _normalize_host_goos, + parse_certutil_sha256 = _parse_certutil_sha256, + powershell_single_quoted_literal = _powershell_single_quoted_literal, ++ patch_resolver_cycle_guard = _patch_resolver_cycle_guard, +) + +_TEST_OPTIMIZATION_ENABLED_ENV = "DD_TEST_OPTIMIZATION_ENABLED" @@ -3504,6 +3524,7 @@ index 0000000..dc4e483 + if resolve_tempdir_old not in resolve_src: + fail("Could not patch Orchestrion resolver tempdir block in %s" % resolve_path) + resolve_src = resolve_src.replace(resolve_tempdir_old, resolve_tempdir_new, 1) ++ resolve_src = _patch_resolver_cycle_guard(resolve_src) + ctx.file(resolve_path, resolve_src) + + # Compile-proxy / archive metadata patches for the synthetic testmain flow. @@ -16538,10 +16559,10 @@ index a1cd2ea..406e52f 100644 ) diff --git a/tests/core/starlark/orchestrion_extension_tests.bzl b/tests/core/starlark/orchestrion_extension_tests.bzl new file mode 100644 -index 0000000..ec4826f +index 0000000..9d605a8 --- /dev/null +++ b/tests/core/starlark/orchestrion_extension_tests.bzl -@@ -0,0 +1,310 @@ +@@ -0,0 +1,332 @@ +load("@bazel_skylib//lib:unittest.bzl", "asserts", "unittest") +load("//go/private/orchestrion:extensions.bzl", "orchestrion_extension_test_helpers") + @@ -16834,6 +16855,27 @@ index 0000000..ec4826f + +declared_dd_trace_go_versions_test = unittest.make(_declared_dd_trace_go_versions_test) + ++def _resolver_cycle_guard_patch_test(ctx): ++ env = unittest.begin(ctx) ++ source = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error { ++ if pkg.PkgPath == "" || pkg.PkgPath == "unsafe" || r[pkg.PkgPath].ExportFile != "" { ++ } ++ for _, dep := range pkg.Imports { ++ errs = errors.Join(errs, r.mergeFrom(dep)) ++ } ++} ++""" ++ ++ patched = orchestrion_extension_test_helpers.patch_resolver_cycle_guard(source) ++ asserts.true(env, "make(map[*packages.Package]struct{})" in patched) ++ asserts.true(env, "if _, ok := visited[pkg]; ok" in patched) ++ asserts.true(env, "r.mergeFromVisited(dep, visited)" in patched) ++ asserts.false(env, "r.mergeFrom(dep)" in patched) ++ ++ return unittest.end(env) ++ ++resolver_cycle_guard_patch_test = unittest.make(_resolver_cycle_guard_patch_test) ++ +def orchestrion_extension_test_suite(): + unittest.suite( + "orchestrion_extension_tests", @@ -16851,4 +16893,5 @@ index 0000000..ec4826f + module_proxy_seed_go_mod_test, + parse_certutil_sha256_test, + powershell_single_quoted_literal_test, ++ resolver_cycle_guard_patch_test, + ) diff --git a/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch index 589bd52e..b8cac5fa 100644 --- a/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch @@ -2084,10 +2084,10 @@ index 0000000..177eeaf +) diff --git a/go/private/orchestrion/extensions.bzl b/go/private/orchestrion/extensions.bzl new file mode 100644 -index 0000000..dc4e483 +index 0000000..9ec04e6 --- /dev/null +++ b/go/private/orchestrion/extensions.bzl -@@ -0,0 +1,1739 @@ +@@ -0,0 +1,1760 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); @@ -2110,7 +2110,7 @@ index 0000000..dc4e483 + +# Bump this identifier whenever the in-repo Orchestrion patch block changes in +# a way that should invalidate previously cached bootstrap binaries. -+ORCHESTRION_PATCHSET_ID = "20260427-go-tool-transition" ++ORCHESTRION_PATCHSET_ID = "20260823-resolver-cycle-guard" +_DD_TRACE_GO_MODULES = [ + "github.com/DataDog/dd-trace-go/v2", + "github.com/DataDog/dd-trace-go/contrib/net/http/v2", @@ -3315,6 +3315,25 @@ index 0000000..dc4e483 + ctx.file("BUILD.bazel", _orchestrion_build_file(binary_name)) + return True + ++def _patch_resolver_cycle_guard(source): ++ header = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error { ++ if pkg.PkgPath == "" || pkg.PkgPath == "unsafe" || r[pkg.PkgPath].ExportFile != "" {""" ++ guarded_header = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error { ++ return r.mergeFromVisited(pkg, make(map[*packages.Package]struct{})) ++} ++ ++func (r ResolveResponse) mergeFromVisited(pkg *packages.Package, visited map[*packages.Package]struct{}) error { ++ // packages.Load can return cycles whose nodes do not have ExportFile yet. ++ if _, ok := visited[pkg]; ok { ++ return nil ++ } ++ visited[pkg] = struct{}{} ++ if pkg.PkgPath == "" || pkg.PkgPath == "unsafe" || r[pkg.PkgPath].ExportFile != "" {""" ++ recursive_call = "r.mergeFrom(dep)" ++ if header not in source or recursive_call not in source: ++ fail("Could not patch Orchestrion resolver cycle guard") ++ return source.replace(header, guarded_header, 1).replace(recursive_call, "r.mergeFromVisited(dep, visited)", 1) ++ +orchestrion_extension_test_helpers = struct( + bootstrap_cache_key = _bootstrap_cache_key, + bootstrap_manifest_content = _bootstrap_manifest_content, @@ -3333,6 +3352,7 @@ index 0000000..dc4e483 + normalize_host_goos = _normalize_host_goos, + parse_certutil_sha256 = _parse_certutil_sha256, + powershell_single_quoted_literal = _powershell_single_quoted_literal, ++ patch_resolver_cycle_guard = _patch_resolver_cycle_guard, +) + +_TEST_OPTIMIZATION_ENABLED_ENV = "DD_TEST_OPTIMIZATION_ENABLED" @@ -3497,6 +3517,7 @@ index 0000000..dc4e483 + if resolve_tempdir_old not in resolve_src: + fail("Could not patch Orchestrion resolver tempdir block in %s" % resolve_path) + resolve_src = resolve_src.replace(resolve_tempdir_old, resolve_tempdir_new, 1) ++ resolve_src = _patch_resolver_cycle_guard(resolve_src) + ctx.file(resolve_path, resolve_src) + + # Compile-proxy / archive metadata patches for the synthetic testmain flow. @@ -16528,10 +16549,10 @@ index a1cd2ea..406e52f 100644 ) diff --git a/tests/core/starlark/orchestrion_extension_tests.bzl b/tests/core/starlark/orchestrion_extension_tests.bzl new file mode 100644 -index 0000000..ec4826f +index 0000000..9d605a8 --- /dev/null +++ b/tests/core/starlark/orchestrion_extension_tests.bzl -@@ -0,0 +1,310 @@ +@@ -0,0 +1,332 @@ +load("@bazel_skylib//lib:unittest.bzl", "asserts", "unittest") +load("//go/private/orchestrion:extensions.bzl", "orchestrion_extension_test_helpers") + @@ -16824,6 +16845,27 @@ index 0000000..ec4826f + +declared_dd_trace_go_versions_test = unittest.make(_declared_dd_trace_go_versions_test) + ++def _resolver_cycle_guard_patch_test(ctx): ++ env = unittest.begin(ctx) ++ source = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error { ++ if pkg.PkgPath == "" || pkg.PkgPath == "unsafe" || r[pkg.PkgPath].ExportFile != "" { ++ } ++ for _, dep := range pkg.Imports { ++ errs = errors.Join(errs, r.mergeFrom(dep)) ++ } ++} ++""" ++ ++ patched = orchestrion_extension_test_helpers.patch_resolver_cycle_guard(source) ++ asserts.true(env, "make(map[*packages.Package]struct{})" in patched) ++ asserts.true(env, "if _, ok := visited[pkg]; ok" in patched) ++ asserts.true(env, "r.mergeFromVisited(dep, visited)" in patched) ++ asserts.false(env, "r.mergeFrom(dep)" in patched) ++ ++ return unittest.end(env) ++ ++resolver_cycle_guard_patch_test = unittest.make(_resolver_cycle_guard_patch_test) ++ +def orchestrion_extension_test_suite(): + unittest.suite( + "orchestrion_extension_tests", @@ -16841,4 +16883,5 @@ index 0000000..ec4826f + module_proxy_seed_go_mod_test, + parse_certutil_sha256_test, + powershell_single_quoted_literal_test, ++ resolver_cycle_guard_patch_test, + ) diff --git a/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch index f6c44dd2..cbbd654e 100644 --- a/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch @@ -2088,10 +2088,10 @@ index 0000000..177eeaf +) diff --git a/go/private/orchestrion/extensions.bzl b/go/private/orchestrion/extensions.bzl new file mode 100644 -index 0000000..dc4e483 +index 0000000..9ec04e6 --- /dev/null +++ b/go/private/orchestrion/extensions.bzl -@@ -0,0 +1,1739 @@ +@@ -0,0 +1,1760 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); @@ -2114,7 +2114,7 @@ index 0000000..dc4e483 + +# Bump this identifier whenever the in-repo Orchestrion patch block changes in +# a way that should invalidate previously cached bootstrap binaries. -+ORCHESTRION_PATCHSET_ID = "20260427-go-tool-transition" ++ORCHESTRION_PATCHSET_ID = "20260823-resolver-cycle-guard" +_DD_TRACE_GO_MODULES = [ + "github.com/DataDog/dd-trace-go/v2", + "github.com/DataDog/dd-trace-go/contrib/net/http/v2", @@ -3319,6 +3319,25 @@ index 0000000..dc4e483 + ctx.file("BUILD.bazel", _orchestrion_build_file(binary_name)) + return True + ++def _patch_resolver_cycle_guard(source): ++ header = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error { ++ if pkg.PkgPath == "" || pkg.PkgPath == "unsafe" || r[pkg.PkgPath].ExportFile != "" {""" ++ guarded_header = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error { ++ return r.mergeFromVisited(pkg, make(map[*packages.Package]struct{})) ++} ++ ++func (r ResolveResponse) mergeFromVisited(pkg *packages.Package, visited map[*packages.Package]struct{}) error { ++ // packages.Load can return cycles whose nodes do not have ExportFile yet. ++ if _, ok := visited[pkg]; ok { ++ return nil ++ } ++ visited[pkg] = struct{}{} ++ if pkg.PkgPath == "" || pkg.PkgPath == "unsafe" || r[pkg.PkgPath].ExportFile != "" {""" ++ recursive_call = "r.mergeFrom(dep)" ++ if header not in source or recursive_call not in source: ++ fail("Could not patch Orchestrion resolver cycle guard") ++ return source.replace(header, guarded_header, 1).replace(recursive_call, "r.mergeFromVisited(dep, visited)", 1) ++ +orchestrion_extension_test_helpers = struct( + bootstrap_cache_key = _bootstrap_cache_key, + bootstrap_manifest_content = _bootstrap_manifest_content, @@ -3337,6 +3356,7 @@ index 0000000..dc4e483 + normalize_host_goos = _normalize_host_goos, + parse_certutil_sha256 = _parse_certutil_sha256, + powershell_single_quoted_literal = _powershell_single_quoted_literal, ++ patch_resolver_cycle_guard = _patch_resolver_cycle_guard, +) + +_TEST_OPTIMIZATION_ENABLED_ENV = "DD_TEST_OPTIMIZATION_ENABLED" @@ -3501,6 +3521,7 @@ index 0000000..dc4e483 + if resolve_tempdir_old not in resolve_src: + fail("Could not patch Orchestrion resolver tempdir block in %s" % resolve_path) + resolve_src = resolve_src.replace(resolve_tempdir_old, resolve_tempdir_new, 1) ++ resolve_src = _patch_resolver_cycle_guard(resolve_src) + ctx.file(resolve_path, resolve_src) + + # Compile-proxy / archive metadata patches for the synthetic testmain flow. @@ -16532,10 +16553,10 @@ index a1cd2ea..406e52f 100644 ) diff --git a/tests/core/starlark/orchestrion_extension_tests.bzl b/tests/core/starlark/orchestrion_extension_tests.bzl new file mode 100644 -index 0000000..ec4826f +index 0000000..9d605a8 --- /dev/null +++ b/tests/core/starlark/orchestrion_extension_tests.bzl -@@ -0,0 +1,310 @@ +@@ -0,0 +1,332 @@ +load("@bazel_skylib//lib:unittest.bzl", "asserts", "unittest") +load("//go/private/orchestrion:extensions.bzl", "orchestrion_extension_test_helpers") + @@ -16828,6 +16849,27 @@ index 0000000..ec4826f + +declared_dd_trace_go_versions_test = unittest.make(_declared_dd_trace_go_versions_test) + ++def _resolver_cycle_guard_patch_test(ctx): ++ env = unittest.begin(ctx) ++ source = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error { ++ if pkg.PkgPath == "" || pkg.PkgPath == "unsafe" || r[pkg.PkgPath].ExportFile != "" { ++ } ++ for _, dep := range pkg.Imports { ++ errs = errors.Join(errs, r.mergeFrom(dep)) ++ } ++} ++""" ++ ++ patched = orchestrion_extension_test_helpers.patch_resolver_cycle_guard(source) ++ asserts.true(env, "make(map[*packages.Package]struct{})" in patched) ++ asserts.true(env, "if _, ok := visited[pkg]; ok" in patched) ++ asserts.true(env, "r.mergeFromVisited(dep, visited)" in patched) ++ asserts.false(env, "r.mergeFrom(dep)" in patched) ++ ++ return unittest.end(env) ++ ++resolver_cycle_guard_patch_test = unittest.make(_resolver_cycle_guard_patch_test) ++ +def orchestrion_extension_test_suite(): + unittest.suite( + "orchestrion_extension_tests", @@ -16845,4 +16887,5 @@ index 0000000..ec4826f + module_proxy_seed_go_mod_test, + parse_certutil_sha256_test, + powershell_single_quoted_literal_test, ++ resolver_cycle_guard_patch_test, + ) From 5fc4e4b9130541f8e6e0b7d17f5ebe62aabb85ec Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Sun, 23 Aug 2026 17:53:05 +0200 Subject: [PATCH 08/37] go: expose SDK headers to instrumented links --- .../rgo/v0_60_0/base/go/private/actions/link.bzl | 6 ++++-- .../rgo/v0_61_1/base/go/private/actions/link.bzl | 6 ++++-- .../rgo/v0_62_0/base/go/private/actions/link.bzl | 6 ++++-- .../patches/v0_60_0/base/0001-full-delta.patch | 12 +++++++----- .../patches/v0_61_1/base/0001-full-delta.patch | 12 +++++++----- .../patches/v0_62_0/base/0001-full-delta.patch | 12 +++++++----- 6 files changed, 33 insertions(+), 21 deletions(-) diff --git a/third_party/rgo/v0_60_0/base/go/private/actions/link.bzl b/third_party/rgo/v0_60_0/base/go/private/actions/link.bzl index 2cc2212c..0d59e086 100644 --- a/third_party/rgo/v0_60_0/base/go/private/actions/link.bzl +++ b/third_party/rgo/v0_60_0/base/go/private/actions/link.bzl @@ -295,8 +295,10 @@ def emit_link( # Orchestrion needs the go binary to run `go env GOMOD` inputs_direct.append(go.sdk.go) - # The toolexec path may resolve woven dependencies during linking too, - # so keep the SDK source tree available in sandboxed executions. + # The toolexec path may resolve and compile woven dependencies during + # linking, so keep the SDK source tree and assembly headers available + # in sandboxed executions. + inputs_transitive.append(go.sdk.headers) inputs_transitive.append(go.sdk.srcs) if getattr(go, "orchestrion_module_proxy_files", None): inputs_transitive.append(go.orchestrion_module_proxy_files) diff --git a/third_party/rgo/v0_61_1/base/go/private/actions/link.bzl b/third_party/rgo/v0_61_1/base/go/private/actions/link.bzl index 25834540..57dab7bf 100644 --- a/third_party/rgo/v0_61_1/base/go/private/actions/link.bzl +++ b/third_party/rgo/v0_61_1/base/go/private/actions/link.bzl @@ -296,8 +296,10 @@ def emit_link( # Orchestrion needs the go binary to run `go env GOMOD` inputs_direct.append(go.sdk.go) - # The toolexec path may resolve woven dependencies during linking too, - # so keep the SDK source tree available in sandboxed executions. + # The toolexec path may resolve and compile woven dependencies during + # linking, so keep the SDK source tree and assembly headers available + # in sandboxed executions. + inputs_transitive.append(go.sdk.headers) inputs_transitive.append(go.sdk.srcs) if getattr(go, "orchestrion_module_proxy_files", None): inputs_transitive.append(go.orchestrion_module_proxy_files) diff --git a/third_party/rgo/v0_62_0/base/go/private/actions/link.bzl b/third_party/rgo/v0_62_0/base/go/private/actions/link.bzl index e84f2048..ba934667 100644 --- a/third_party/rgo/v0_62_0/base/go/private/actions/link.bzl +++ b/third_party/rgo/v0_62_0/base/go/private/actions/link.bzl @@ -298,8 +298,10 @@ def emit_link( # Orchestrion needs the go binary to run `go env GOMOD` inputs_direct.append(go.sdk.go) - # The toolexec path may resolve woven dependencies during linking too, - # so keep the SDK source tree available in sandboxed executions. + # The toolexec path may resolve and compile woven dependencies during + # linking, so keep the SDK source tree and assembly headers available + # in sandboxed executions. + inputs_transitive.append(go.sdk.headers) inputs_transitive.append(go.sdk.srcs) if getattr(go, "orchestrion_module_proxy_files", None): inputs_transitive.append(go.orchestrion_module_proxy_files) diff --git a/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch index 590f993a..77345f27 100644 --- a/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch @@ -1419,7 +1419,7 @@ index a339eef..7525a1e 100644 inputs = depset(inputs_direct, transitive = inputs_transitive), outputs = outputs, diff --git a/go/private/actions/link.bzl b/go/private/actions/link.bzl -index 7d8ff5f..2cc2212 100644 +index 7d8ff5f..0d59e08 100644 --- a/go/private/actions/link.bzl +++ b/go/private/actions/link.bzl @@ -33,10 +33,88 @@ load( @@ -1539,7 +1539,7 @@ index 7d8ff5f..2cc2212 100644 inputs_transitive = [ archive.libs, archive.cgo_deps, -@@ -186,6 +276,46 @@ def emit_link( +@@ -186,6 +276,48 @@ def emit_link( go.sdk.tools, go.stdlib.libs, ] @@ -1562,8 +1562,10 @@ index 7d8ff5f..2cc2212 100644 + # Orchestrion needs the go binary to run `go env GOMOD` + inputs_direct.append(go.sdk.go) + -+ # The toolexec path may resolve woven dependencies during linking too, -+ # so keep the SDK source tree available in sandboxed executions. ++ # The toolexec path may resolve and compile woven dependencies during ++ # linking, so keep the SDK source tree and assembly headers available ++ # in sandboxed executions. ++ inputs_transitive.append(go.sdk.headers) + inputs_transitive.append(go.sdk.srcs) + if getattr(go, "orchestrion_module_proxy_files", None): + inputs_transitive.append(go.orchestrion_module_proxy_files) @@ -1586,7 +1588,7 @@ index 7d8ff5f..2cc2212 100644 inputs = depset(direct = inputs_direct, transitive = inputs_transitive) go.actions.run( -@@ -194,7 +324,13 @@ def emit_link( +@@ -194,7 +326,13 @@ def emit_link( mnemonic = "GoLink", executable = go.toolchain._builder, arguments = [builder_args, "--", tool_args], diff --git a/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch index b8cac5fa..40746e5f 100644 --- a/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch @@ -1403,7 +1403,7 @@ index 9c067b2..1b2ab71 100644 if ldflags: arguments.append(ldflags) diff --git a/go/private/actions/link.bzl b/go/private/actions/link.bzl -index 61f076f..2583454 100644 +index 61f076f..57dab7b 100644 --- a/go/private/actions/link.bzl +++ b/go/private/actions/link.bzl @@ -33,10 +33,88 @@ load( @@ -1523,7 +1523,7 @@ index 61f076f..2583454 100644 inputs_transitive = [ archive.libs, archive.cgo_deps, -@@ -187,6 +277,46 @@ def emit_link( +@@ -187,6 +277,48 @@ def emit_link( go.sdk.tools, go.stdlib.libs, ] @@ -1546,8 +1546,10 @@ index 61f076f..2583454 100644 + # Orchestrion needs the go binary to run `go env GOMOD` + inputs_direct.append(go.sdk.go) + -+ # The toolexec path may resolve woven dependencies during linking too, -+ # so keep the SDK source tree available in sandboxed executions. ++ # The toolexec path may resolve and compile woven dependencies during ++ # linking, so keep the SDK source tree and assembly headers available ++ # in sandboxed executions. ++ inputs_transitive.append(go.sdk.headers) + inputs_transitive.append(go.sdk.srcs) + if getattr(go, "orchestrion_module_proxy_files", None): + inputs_transitive.append(go.orchestrion_module_proxy_files) @@ -1570,7 +1572,7 @@ index 61f076f..2583454 100644 inputs = depset(direct = inputs_direct, transitive = inputs_transitive) go.actions.run( -@@ -195,7 +325,13 @@ def emit_link( +@@ -195,7 +327,13 @@ def emit_link( mnemonic = "GoLink", executable = go.toolchain._builder, arguments = [builder_args, "--", tool_args], diff --git a/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch index cbbd654e..30c60495 100644 --- a/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch @@ -1403,7 +1403,7 @@ index 9c067b2..1b2ab71 100644 if ldflags: arguments.append(ldflags) diff --git a/go/private/actions/link.bzl b/go/private/actions/link.bzl -index c249ca7..e84f204 100644 +index c249ca7..ba93466 100644 --- a/go/private/actions/link.bzl +++ b/go/private/actions/link.bzl @@ -33,10 +33,88 @@ load( @@ -1523,7 +1523,7 @@ index c249ca7..e84f204 100644 inputs_transitive = [ archive.libs, archive.cgo_deps, -@@ -189,6 +279,46 @@ def emit_link( +@@ -189,6 +279,48 @@ def emit_link( go.sdk.tools, go.stdlib.libs, ] @@ -1546,8 +1546,10 @@ index c249ca7..e84f204 100644 + # Orchestrion needs the go binary to run `go env GOMOD` + inputs_direct.append(go.sdk.go) + -+ # The toolexec path may resolve woven dependencies during linking too, -+ # so keep the SDK source tree available in sandboxed executions. ++ # The toolexec path may resolve and compile woven dependencies during ++ # linking, so keep the SDK source tree and assembly headers available ++ # in sandboxed executions. ++ inputs_transitive.append(go.sdk.headers) + inputs_transitive.append(go.sdk.srcs) + if getattr(go, "orchestrion_module_proxy_files", None): + inputs_transitive.append(go.orchestrion_module_proxy_files) @@ -1570,7 +1572,7 @@ index c249ca7..e84f204 100644 inputs = depset(direct = inputs_direct, transitive = inputs_transitive) go.actions.run( -@@ -197,7 +327,13 @@ def emit_link( +@@ -197,7 +329,13 @@ def emit_link( mnemonic = "GoLink", executable = go.toolchain._builder, arguments = [builder_args, "--", tool_args], From 4be1debcc735d9a2fbc109b676c8dc7d2284ffc8 Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Sun, 23 Aug 2026 19:30:47 +0200 Subject: [PATCH 09/37] go: make resolver cycle patch line-ending safe --- .../go/private/orchestrion/extensions.bzl | 31 +++++---- .../starlark/orchestrion_extension_tests.bzl | 30 +++++---- .../go/private/orchestrion/extensions.bzl | 31 +++++---- .../starlark/orchestrion_extension_tests.bzl | 30 +++++---- .../go/private/orchestrion/extensions.bzl | 31 +++++---- .../starlark/orchestrion_extension_tests.bzl | 30 +++++---- .../v0_60_0/base/0001-full-delta.patch | 67 ++++++++++--------- .../v0_61_1/base/0001-full-delta.patch | 67 ++++++++++--------- .../v0_62_0/base/0001-full-delta.patch | 67 ++++++++++--------- 9 files changed, 213 insertions(+), 171 deletions(-) diff --git a/third_party/rgo/v0_60_0/base/go/private/orchestrion/extensions.bzl b/third_party/rgo/v0_60_0/base/go/private/orchestrion/extensions.bzl index 9ec04e69..4487054c 100644 --- a/third_party/rgo/v0_60_0/base/go/private/orchestrion/extensions.bzl +++ b/third_party/rgo/v0_60_0/base/go/private/orchestrion/extensions.bzl @@ -1226,19 +1226,24 @@ def _restore_bootstrap_cache(ctx, paths, version, version_map, go_identity, bina return True def _patch_resolver_cycle_guard(source): - header = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error { - if pkg.PkgPath == "" || pkg.PkgPath == "unsafe" || r[pkg.PkgPath].ExportFile != "" {""" - guarded_header = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error { - return r.mergeFromVisited(pkg, make(map[*packages.Package]struct{})) -} - -func (r ResolveResponse) mergeFromVisited(pkg *packages.Package, visited map[*packages.Package]struct{}) error { - // packages.Load can return cycles whose nodes do not have ExportFile yet. - if _, ok := visited[pkg]; ok { - return nil - } - visited[pkg] = struct{}{} - if pkg.PkgPath == "" || pkg.PkgPath == "unsafe" || r[pkg.PkgPath].ExportFile != "" {""" + line_ending = "\r\n" if "\r\n" in source else "\n" + header = line_ending.join([ + "func (r ResolveResponse) mergeFrom(pkg *packages.Package) error {", + "\tif pkg.PkgPath == \"\" || pkg.PkgPath == \"unsafe\" || r[pkg.PkgPath].ExportFile != \"\" {", + ]) + guarded_header = line_ending.join([ + "func (r ResolveResponse) mergeFrom(pkg *packages.Package) error {", + "\treturn r.mergeFromVisited(pkg, make(map[*packages.Package]struct{}))", + "}", + "", + "func (r ResolveResponse) mergeFromVisited(pkg *packages.Package, visited map[*packages.Package]struct{}) error {", + "\t// packages.Load can return cycles whose nodes do not have ExportFile yet.", + "\tif _, ok := visited[pkg]; ok {", + "\t\treturn nil", + "\t}", + "\tvisited[pkg] = struct{}{}", + "\tif pkg.PkgPath == \"\" || pkg.PkgPath == \"unsafe\" || r[pkg.PkgPath].ExportFile != \"\" {", + ]) recursive_call = "r.mergeFrom(dep)" if header not in source or recursive_call not in source: fail("Could not patch Orchestrion resolver cycle guard") diff --git a/third_party/rgo/v0_60_0/base/tests/core/starlark/orchestrion_extension_tests.bzl b/third_party/rgo/v0_60_0/base/tests/core/starlark/orchestrion_extension_tests.bzl index 9d605a80..884076ac 100644 --- a/third_party/rgo/v0_60_0/base/tests/core/starlark/orchestrion_extension_tests.bzl +++ b/third_party/rgo/v0_60_0/base/tests/core/starlark/orchestrion_extension_tests.bzl @@ -292,20 +292,22 @@ declared_dd_trace_go_versions_test = unittest.make(_declared_dd_trace_go_version def _resolver_cycle_guard_patch_test(ctx): env = unittest.begin(ctx) - source = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error { - if pkg.PkgPath == "" || pkg.PkgPath == "unsafe" || r[pkg.PkgPath].ExportFile != "" { - } - for _, dep := range pkg.Imports { - errs = errors.Join(errs, r.mergeFrom(dep)) - } -} -""" - - patched = orchestrion_extension_test_helpers.patch_resolver_cycle_guard(source) - asserts.true(env, "make(map[*packages.Package]struct{})" in patched) - asserts.true(env, "if _, ok := visited[pkg]; ok" in patched) - asserts.true(env, "r.mergeFromVisited(dep, visited)" in patched) - asserts.false(env, "r.mergeFrom(dep)" in patched) + source_lines = [ + "func (r ResolveResponse) mergeFrom(pkg *packages.Package) error {", + "\tif pkg.PkgPath == \"\" || pkg.PkgPath == \"unsafe\" || r[pkg.PkgPath].ExportFile != \"\" {", + "\t}", + "\tfor _, dep := range pkg.Imports {", + "\t\terrs = errors.Join(errs, r.mergeFrom(dep))", + "\t}", + "}", + ] + + for line_ending in ["\n", "\r\n"]: + patched = orchestrion_extension_test_helpers.patch_resolver_cycle_guard(line_ending.join(source_lines) + line_ending) + asserts.true(env, ("return r.mergeFromVisited(pkg, make(map[*packages.Package]struct{}))" + line_ending + "}") in patched) + asserts.true(env, "if _, ok := visited[pkg]; ok" in patched) + asserts.true(env, "r.mergeFromVisited(dep, visited)" in patched) + asserts.false(env, "r.mergeFrom(dep)" in patched) return unittest.end(env) diff --git a/third_party/rgo/v0_61_1/base/go/private/orchestrion/extensions.bzl b/third_party/rgo/v0_61_1/base/go/private/orchestrion/extensions.bzl index 9ec04e69..4487054c 100644 --- a/third_party/rgo/v0_61_1/base/go/private/orchestrion/extensions.bzl +++ b/third_party/rgo/v0_61_1/base/go/private/orchestrion/extensions.bzl @@ -1226,19 +1226,24 @@ def _restore_bootstrap_cache(ctx, paths, version, version_map, go_identity, bina return True def _patch_resolver_cycle_guard(source): - header = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error { - if pkg.PkgPath == "" || pkg.PkgPath == "unsafe" || r[pkg.PkgPath].ExportFile != "" {""" - guarded_header = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error { - return r.mergeFromVisited(pkg, make(map[*packages.Package]struct{})) -} - -func (r ResolveResponse) mergeFromVisited(pkg *packages.Package, visited map[*packages.Package]struct{}) error { - // packages.Load can return cycles whose nodes do not have ExportFile yet. - if _, ok := visited[pkg]; ok { - return nil - } - visited[pkg] = struct{}{} - if pkg.PkgPath == "" || pkg.PkgPath == "unsafe" || r[pkg.PkgPath].ExportFile != "" {""" + line_ending = "\r\n" if "\r\n" in source else "\n" + header = line_ending.join([ + "func (r ResolveResponse) mergeFrom(pkg *packages.Package) error {", + "\tif pkg.PkgPath == \"\" || pkg.PkgPath == \"unsafe\" || r[pkg.PkgPath].ExportFile != \"\" {", + ]) + guarded_header = line_ending.join([ + "func (r ResolveResponse) mergeFrom(pkg *packages.Package) error {", + "\treturn r.mergeFromVisited(pkg, make(map[*packages.Package]struct{}))", + "}", + "", + "func (r ResolveResponse) mergeFromVisited(pkg *packages.Package, visited map[*packages.Package]struct{}) error {", + "\t// packages.Load can return cycles whose nodes do not have ExportFile yet.", + "\tif _, ok := visited[pkg]; ok {", + "\t\treturn nil", + "\t}", + "\tvisited[pkg] = struct{}{}", + "\tif pkg.PkgPath == \"\" || pkg.PkgPath == \"unsafe\" || r[pkg.PkgPath].ExportFile != \"\" {", + ]) recursive_call = "r.mergeFrom(dep)" if header not in source or recursive_call not in source: fail("Could not patch Orchestrion resolver cycle guard") diff --git a/third_party/rgo/v0_61_1/base/tests/core/starlark/orchestrion_extension_tests.bzl b/third_party/rgo/v0_61_1/base/tests/core/starlark/orchestrion_extension_tests.bzl index 9d605a80..884076ac 100644 --- a/third_party/rgo/v0_61_1/base/tests/core/starlark/orchestrion_extension_tests.bzl +++ b/third_party/rgo/v0_61_1/base/tests/core/starlark/orchestrion_extension_tests.bzl @@ -292,20 +292,22 @@ declared_dd_trace_go_versions_test = unittest.make(_declared_dd_trace_go_version def _resolver_cycle_guard_patch_test(ctx): env = unittest.begin(ctx) - source = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error { - if pkg.PkgPath == "" || pkg.PkgPath == "unsafe" || r[pkg.PkgPath].ExportFile != "" { - } - for _, dep := range pkg.Imports { - errs = errors.Join(errs, r.mergeFrom(dep)) - } -} -""" - - patched = orchestrion_extension_test_helpers.patch_resolver_cycle_guard(source) - asserts.true(env, "make(map[*packages.Package]struct{})" in patched) - asserts.true(env, "if _, ok := visited[pkg]; ok" in patched) - asserts.true(env, "r.mergeFromVisited(dep, visited)" in patched) - asserts.false(env, "r.mergeFrom(dep)" in patched) + source_lines = [ + "func (r ResolveResponse) mergeFrom(pkg *packages.Package) error {", + "\tif pkg.PkgPath == \"\" || pkg.PkgPath == \"unsafe\" || r[pkg.PkgPath].ExportFile != \"\" {", + "\t}", + "\tfor _, dep := range pkg.Imports {", + "\t\terrs = errors.Join(errs, r.mergeFrom(dep))", + "\t}", + "}", + ] + + for line_ending in ["\n", "\r\n"]: + patched = orchestrion_extension_test_helpers.patch_resolver_cycle_guard(line_ending.join(source_lines) + line_ending) + asserts.true(env, ("return r.mergeFromVisited(pkg, make(map[*packages.Package]struct{}))" + line_ending + "}") in patched) + asserts.true(env, "if _, ok := visited[pkg]; ok" in patched) + asserts.true(env, "r.mergeFromVisited(dep, visited)" in patched) + asserts.false(env, "r.mergeFrom(dep)" in patched) return unittest.end(env) diff --git a/third_party/rgo/v0_62_0/base/go/private/orchestrion/extensions.bzl b/third_party/rgo/v0_62_0/base/go/private/orchestrion/extensions.bzl index 9ec04e69..4487054c 100644 --- a/third_party/rgo/v0_62_0/base/go/private/orchestrion/extensions.bzl +++ b/third_party/rgo/v0_62_0/base/go/private/orchestrion/extensions.bzl @@ -1226,19 +1226,24 @@ def _restore_bootstrap_cache(ctx, paths, version, version_map, go_identity, bina return True def _patch_resolver_cycle_guard(source): - header = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error { - if pkg.PkgPath == "" || pkg.PkgPath == "unsafe" || r[pkg.PkgPath].ExportFile != "" {""" - guarded_header = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error { - return r.mergeFromVisited(pkg, make(map[*packages.Package]struct{})) -} - -func (r ResolveResponse) mergeFromVisited(pkg *packages.Package, visited map[*packages.Package]struct{}) error { - // packages.Load can return cycles whose nodes do not have ExportFile yet. - if _, ok := visited[pkg]; ok { - return nil - } - visited[pkg] = struct{}{} - if pkg.PkgPath == "" || pkg.PkgPath == "unsafe" || r[pkg.PkgPath].ExportFile != "" {""" + line_ending = "\r\n" if "\r\n" in source else "\n" + header = line_ending.join([ + "func (r ResolveResponse) mergeFrom(pkg *packages.Package) error {", + "\tif pkg.PkgPath == \"\" || pkg.PkgPath == \"unsafe\" || r[pkg.PkgPath].ExportFile != \"\" {", + ]) + guarded_header = line_ending.join([ + "func (r ResolveResponse) mergeFrom(pkg *packages.Package) error {", + "\treturn r.mergeFromVisited(pkg, make(map[*packages.Package]struct{}))", + "}", + "", + "func (r ResolveResponse) mergeFromVisited(pkg *packages.Package, visited map[*packages.Package]struct{}) error {", + "\t// packages.Load can return cycles whose nodes do not have ExportFile yet.", + "\tif _, ok := visited[pkg]; ok {", + "\t\treturn nil", + "\t}", + "\tvisited[pkg] = struct{}{}", + "\tif pkg.PkgPath == \"\" || pkg.PkgPath == \"unsafe\" || r[pkg.PkgPath].ExportFile != \"\" {", + ]) recursive_call = "r.mergeFrom(dep)" if header not in source or recursive_call not in source: fail("Could not patch Orchestrion resolver cycle guard") diff --git a/third_party/rgo/v0_62_0/base/tests/core/starlark/orchestrion_extension_tests.bzl b/third_party/rgo/v0_62_0/base/tests/core/starlark/orchestrion_extension_tests.bzl index 9d605a80..884076ac 100644 --- a/third_party/rgo/v0_62_0/base/tests/core/starlark/orchestrion_extension_tests.bzl +++ b/third_party/rgo/v0_62_0/base/tests/core/starlark/orchestrion_extension_tests.bzl @@ -292,20 +292,22 @@ declared_dd_trace_go_versions_test = unittest.make(_declared_dd_trace_go_version def _resolver_cycle_guard_patch_test(ctx): env = unittest.begin(ctx) - source = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error { - if pkg.PkgPath == "" || pkg.PkgPath == "unsafe" || r[pkg.PkgPath].ExportFile != "" { - } - for _, dep := range pkg.Imports { - errs = errors.Join(errs, r.mergeFrom(dep)) - } -} -""" - - patched = orchestrion_extension_test_helpers.patch_resolver_cycle_guard(source) - asserts.true(env, "make(map[*packages.Package]struct{})" in patched) - asserts.true(env, "if _, ok := visited[pkg]; ok" in patched) - asserts.true(env, "r.mergeFromVisited(dep, visited)" in patched) - asserts.false(env, "r.mergeFrom(dep)" in patched) + source_lines = [ + "func (r ResolveResponse) mergeFrom(pkg *packages.Package) error {", + "\tif pkg.PkgPath == \"\" || pkg.PkgPath == \"unsafe\" || r[pkg.PkgPath].ExportFile != \"\" {", + "\t}", + "\tfor _, dep := range pkg.Imports {", + "\t\terrs = errors.Join(errs, r.mergeFrom(dep))", + "\t}", + "}", + ] + + for line_ending in ["\n", "\r\n"]: + patched = orchestrion_extension_test_helpers.patch_resolver_cycle_guard(line_ending.join(source_lines) + line_ending) + asserts.true(env, ("return r.mergeFromVisited(pkg, make(map[*packages.Package]struct{}))" + line_ending + "}") in patched) + asserts.true(env, "if _, ok := visited[pkg]; ok" in patched) + asserts.true(env, "r.mergeFromVisited(dep, visited)" in patched) + asserts.false(env, "r.mergeFrom(dep)" in patched) return unittest.end(env) diff --git a/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch index 77345f27..77e47f14 100644 --- a/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch @@ -2093,10 +2093,10 @@ index 0000000..31a01c3 +) diff --git a/go/private/orchestrion/extensions.bzl b/go/private/orchestrion/extensions.bzl new file mode 100644 -index 0000000..9ec04e6 +index 0000000..4487054 --- /dev/null +++ b/go/private/orchestrion/extensions.bzl -@@ -0,0 +1,1760 @@ +@@ -0,0 +1,1765 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); @@ -3325,19 +3325,24 @@ index 0000000..9ec04e6 + return True + +def _patch_resolver_cycle_guard(source): -+ header = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error { -+ if pkg.PkgPath == "" || pkg.PkgPath == "unsafe" || r[pkg.PkgPath].ExportFile != "" {""" -+ guarded_header = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error { -+ return r.mergeFromVisited(pkg, make(map[*packages.Package]struct{})) -+} -+ -+func (r ResolveResponse) mergeFromVisited(pkg *packages.Package, visited map[*packages.Package]struct{}) error { -+ // packages.Load can return cycles whose nodes do not have ExportFile yet. -+ if _, ok := visited[pkg]; ok { -+ return nil -+ } -+ visited[pkg] = struct{}{} -+ if pkg.PkgPath == "" || pkg.PkgPath == "unsafe" || r[pkg.PkgPath].ExportFile != "" {""" ++ line_ending = "\r\n" if "\r\n" in source else "\n" ++ header = line_ending.join([ ++ "func (r ResolveResponse) mergeFrom(pkg *packages.Package) error {", ++ "\tif pkg.PkgPath == \"\" || pkg.PkgPath == \"unsafe\" || r[pkg.PkgPath].ExportFile != \"\" {", ++ ]) ++ guarded_header = line_ending.join([ ++ "func (r ResolveResponse) mergeFrom(pkg *packages.Package) error {", ++ "\treturn r.mergeFromVisited(pkg, make(map[*packages.Package]struct{}))", ++ "}", ++ "", ++ "func (r ResolveResponse) mergeFromVisited(pkg *packages.Package, visited map[*packages.Package]struct{}) error {", ++ "\t// packages.Load can return cycles whose nodes do not have ExportFile yet.", ++ "\tif _, ok := visited[pkg]; ok {", ++ "\t\treturn nil", ++ "\t}", ++ "\tvisited[pkg] = struct{}{}", ++ "\tif pkg.PkgPath == \"\" || pkg.PkgPath == \"unsafe\" || r[pkg.PkgPath].ExportFile != \"\" {", ++ ]) + recursive_call = "r.mergeFrom(dep)" + if header not in source or recursive_call not in source: + fail("Could not patch Orchestrion resolver cycle guard") @@ -16561,10 +16566,10 @@ index a1cd2ea..406e52f 100644 ) diff --git a/tests/core/starlark/orchestrion_extension_tests.bzl b/tests/core/starlark/orchestrion_extension_tests.bzl new file mode 100644 -index 0000000..9d605a8 +index 0000000..884076a --- /dev/null +++ b/tests/core/starlark/orchestrion_extension_tests.bzl -@@ -0,0 +1,332 @@ +@@ -0,0 +1,334 @@ +load("@bazel_skylib//lib:unittest.bzl", "asserts", "unittest") +load("//go/private/orchestrion:extensions.bzl", "orchestrion_extension_test_helpers") + @@ -16859,20 +16864,22 @@ index 0000000..9d605a8 + +def _resolver_cycle_guard_patch_test(ctx): + env = unittest.begin(ctx) -+ source = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error { -+ if pkg.PkgPath == "" || pkg.PkgPath == "unsafe" || r[pkg.PkgPath].ExportFile != "" { -+ } -+ for _, dep := range pkg.Imports { -+ errs = errors.Join(errs, r.mergeFrom(dep)) -+ } -+} -+""" ++ source_lines = [ ++ "func (r ResolveResponse) mergeFrom(pkg *packages.Package) error {", ++ "\tif pkg.PkgPath == \"\" || pkg.PkgPath == \"unsafe\" || r[pkg.PkgPath].ExportFile != \"\" {", ++ "\t}", ++ "\tfor _, dep := range pkg.Imports {", ++ "\t\terrs = errors.Join(errs, r.mergeFrom(dep))", ++ "\t}", ++ "}", ++ ] + -+ patched = orchestrion_extension_test_helpers.patch_resolver_cycle_guard(source) -+ asserts.true(env, "make(map[*packages.Package]struct{})" in patched) -+ asserts.true(env, "if _, ok := visited[pkg]; ok" in patched) -+ asserts.true(env, "r.mergeFromVisited(dep, visited)" in patched) -+ asserts.false(env, "r.mergeFrom(dep)" in patched) ++ for line_ending in ["\n", "\r\n"]: ++ patched = orchestrion_extension_test_helpers.patch_resolver_cycle_guard(line_ending.join(source_lines) + line_ending) ++ asserts.true(env, ("return r.mergeFromVisited(pkg, make(map[*packages.Package]struct{}))" + line_ending + "}") in patched) ++ asserts.true(env, "if _, ok := visited[pkg]; ok" in patched) ++ asserts.true(env, "r.mergeFromVisited(dep, visited)" in patched) ++ asserts.false(env, "r.mergeFrom(dep)" in patched) + + return unittest.end(env) + diff --git a/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch index 40746e5f..3f8ecc3a 100644 --- a/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch @@ -2086,10 +2086,10 @@ index 0000000..177eeaf +) diff --git a/go/private/orchestrion/extensions.bzl b/go/private/orchestrion/extensions.bzl new file mode 100644 -index 0000000..9ec04e6 +index 0000000..4487054 --- /dev/null +++ b/go/private/orchestrion/extensions.bzl -@@ -0,0 +1,1760 @@ +@@ -0,0 +1,1765 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); @@ -3318,19 +3318,24 @@ index 0000000..9ec04e6 + return True + +def _patch_resolver_cycle_guard(source): -+ header = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error { -+ if pkg.PkgPath == "" || pkg.PkgPath == "unsafe" || r[pkg.PkgPath].ExportFile != "" {""" -+ guarded_header = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error { -+ return r.mergeFromVisited(pkg, make(map[*packages.Package]struct{})) -+} -+ -+func (r ResolveResponse) mergeFromVisited(pkg *packages.Package, visited map[*packages.Package]struct{}) error { -+ // packages.Load can return cycles whose nodes do not have ExportFile yet. -+ if _, ok := visited[pkg]; ok { -+ return nil -+ } -+ visited[pkg] = struct{}{} -+ if pkg.PkgPath == "" || pkg.PkgPath == "unsafe" || r[pkg.PkgPath].ExportFile != "" {""" ++ line_ending = "\r\n" if "\r\n" in source else "\n" ++ header = line_ending.join([ ++ "func (r ResolveResponse) mergeFrom(pkg *packages.Package) error {", ++ "\tif pkg.PkgPath == \"\" || pkg.PkgPath == \"unsafe\" || r[pkg.PkgPath].ExportFile != \"\" {", ++ ]) ++ guarded_header = line_ending.join([ ++ "func (r ResolveResponse) mergeFrom(pkg *packages.Package) error {", ++ "\treturn r.mergeFromVisited(pkg, make(map[*packages.Package]struct{}))", ++ "}", ++ "", ++ "func (r ResolveResponse) mergeFromVisited(pkg *packages.Package, visited map[*packages.Package]struct{}) error {", ++ "\t// packages.Load can return cycles whose nodes do not have ExportFile yet.", ++ "\tif _, ok := visited[pkg]; ok {", ++ "\t\treturn nil", ++ "\t}", ++ "\tvisited[pkg] = struct{}{}", ++ "\tif pkg.PkgPath == \"\" || pkg.PkgPath == \"unsafe\" || r[pkg.PkgPath].ExportFile != \"\" {", ++ ]) + recursive_call = "r.mergeFrom(dep)" + if header not in source or recursive_call not in source: + fail("Could not patch Orchestrion resolver cycle guard") @@ -16551,10 +16556,10 @@ index a1cd2ea..406e52f 100644 ) diff --git a/tests/core/starlark/orchestrion_extension_tests.bzl b/tests/core/starlark/orchestrion_extension_tests.bzl new file mode 100644 -index 0000000..9d605a8 +index 0000000..884076a --- /dev/null +++ b/tests/core/starlark/orchestrion_extension_tests.bzl -@@ -0,0 +1,332 @@ +@@ -0,0 +1,334 @@ +load("@bazel_skylib//lib:unittest.bzl", "asserts", "unittest") +load("//go/private/orchestrion:extensions.bzl", "orchestrion_extension_test_helpers") + @@ -16849,20 +16854,22 @@ index 0000000..9d605a8 + +def _resolver_cycle_guard_patch_test(ctx): + env = unittest.begin(ctx) -+ source = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error { -+ if pkg.PkgPath == "" || pkg.PkgPath == "unsafe" || r[pkg.PkgPath].ExportFile != "" { -+ } -+ for _, dep := range pkg.Imports { -+ errs = errors.Join(errs, r.mergeFrom(dep)) -+ } -+} -+""" ++ source_lines = [ ++ "func (r ResolveResponse) mergeFrom(pkg *packages.Package) error {", ++ "\tif pkg.PkgPath == \"\" || pkg.PkgPath == \"unsafe\" || r[pkg.PkgPath].ExportFile != \"\" {", ++ "\t}", ++ "\tfor _, dep := range pkg.Imports {", ++ "\t\terrs = errors.Join(errs, r.mergeFrom(dep))", ++ "\t}", ++ "}", ++ ] + -+ patched = orchestrion_extension_test_helpers.patch_resolver_cycle_guard(source) -+ asserts.true(env, "make(map[*packages.Package]struct{})" in patched) -+ asserts.true(env, "if _, ok := visited[pkg]; ok" in patched) -+ asserts.true(env, "r.mergeFromVisited(dep, visited)" in patched) -+ asserts.false(env, "r.mergeFrom(dep)" in patched) ++ for line_ending in ["\n", "\r\n"]: ++ patched = orchestrion_extension_test_helpers.patch_resolver_cycle_guard(line_ending.join(source_lines) + line_ending) ++ asserts.true(env, ("return r.mergeFromVisited(pkg, make(map[*packages.Package]struct{}))" + line_ending + "}") in patched) ++ asserts.true(env, "if _, ok := visited[pkg]; ok" in patched) ++ asserts.true(env, "r.mergeFromVisited(dep, visited)" in patched) ++ asserts.false(env, "r.mergeFrom(dep)" in patched) + + return unittest.end(env) + diff --git a/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch index 30c60495..44a732d1 100644 --- a/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch @@ -2090,10 +2090,10 @@ index 0000000..177eeaf +) diff --git a/go/private/orchestrion/extensions.bzl b/go/private/orchestrion/extensions.bzl new file mode 100644 -index 0000000..9ec04e6 +index 0000000..4487054 --- /dev/null +++ b/go/private/orchestrion/extensions.bzl -@@ -0,0 +1,1760 @@ +@@ -0,0 +1,1765 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); @@ -3322,19 +3322,24 @@ index 0000000..9ec04e6 + return True + +def _patch_resolver_cycle_guard(source): -+ header = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error { -+ if pkg.PkgPath == "" || pkg.PkgPath == "unsafe" || r[pkg.PkgPath].ExportFile != "" {""" -+ guarded_header = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error { -+ return r.mergeFromVisited(pkg, make(map[*packages.Package]struct{})) -+} -+ -+func (r ResolveResponse) mergeFromVisited(pkg *packages.Package, visited map[*packages.Package]struct{}) error { -+ // packages.Load can return cycles whose nodes do not have ExportFile yet. -+ if _, ok := visited[pkg]; ok { -+ return nil -+ } -+ visited[pkg] = struct{}{} -+ if pkg.PkgPath == "" || pkg.PkgPath == "unsafe" || r[pkg.PkgPath].ExportFile != "" {""" ++ line_ending = "\r\n" if "\r\n" in source else "\n" ++ header = line_ending.join([ ++ "func (r ResolveResponse) mergeFrom(pkg *packages.Package) error {", ++ "\tif pkg.PkgPath == \"\" || pkg.PkgPath == \"unsafe\" || r[pkg.PkgPath].ExportFile != \"\" {", ++ ]) ++ guarded_header = line_ending.join([ ++ "func (r ResolveResponse) mergeFrom(pkg *packages.Package) error {", ++ "\treturn r.mergeFromVisited(pkg, make(map[*packages.Package]struct{}))", ++ "}", ++ "", ++ "func (r ResolveResponse) mergeFromVisited(pkg *packages.Package, visited map[*packages.Package]struct{}) error {", ++ "\t// packages.Load can return cycles whose nodes do not have ExportFile yet.", ++ "\tif _, ok := visited[pkg]; ok {", ++ "\t\treturn nil", ++ "\t}", ++ "\tvisited[pkg] = struct{}{}", ++ "\tif pkg.PkgPath == \"\" || pkg.PkgPath == \"unsafe\" || r[pkg.PkgPath].ExportFile != \"\" {", ++ ]) + recursive_call = "r.mergeFrom(dep)" + if header not in source or recursive_call not in source: + fail("Could not patch Orchestrion resolver cycle guard") @@ -16555,10 +16560,10 @@ index a1cd2ea..406e52f 100644 ) diff --git a/tests/core/starlark/orchestrion_extension_tests.bzl b/tests/core/starlark/orchestrion_extension_tests.bzl new file mode 100644 -index 0000000..9d605a8 +index 0000000..884076a --- /dev/null +++ b/tests/core/starlark/orchestrion_extension_tests.bzl -@@ -0,0 +1,332 @@ +@@ -0,0 +1,334 @@ +load("@bazel_skylib//lib:unittest.bzl", "asserts", "unittest") +load("//go/private/orchestrion:extensions.bzl", "orchestrion_extension_test_helpers") + @@ -16853,20 +16858,22 @@ index 0000000..9d605a8 + +def _resolver_cycle_guard_patch_test(ctx): + env = unittest.begin(ctx) -+ source = """func (r ResolveResponse) mergeFrom(pkg *packages.Package) error { -+ if pkg.PkgPath == "" || pkg.PkgPath == "unsafe" || r[pkg.PkgPath].ExportFile != "" { -+ } -+ for _, dep := range pkg.Imports { -+ errs = errors.Join(errs, r.mergeFrom(dep)) -+ } -+} -+""" ++ source_lines = [ ++ "func (r ResolveResponse) mergeFrom(pkg *packages.Package) error {", ++ "\tif pkg.PkgPath == \"\" || pkg.PkgPath == \"unsafe\" || r[pkg.PkgPath].ExportFile != \"\" {", ++ "\t}", ++ "\tfor _, dep := range pkg.Imports {", ++ "\t\terrs = errors.Join(errs, r.mergeFrom(dep))", ++ "\t}", ++ "}", ++ ] + -+ patched = orchestrion_extension_test_helpers.patch_resolver_cycle_guard(source) -+ asserts.true(env, "make(map[*packages.Package]struct{})" in patched) -+ asserts.true(env, "if _, ok := visited[pkg]; ok" in patched) -+ asserts.true(env, "r.mergeFromVisited(dep, visited)" in patched) -+ asserts.false(env, "r.mergeFrom(dep)" in patched) ++ for line_ending in ["\n", "\r\n"]: ++ patched = orchestrion_extension_test_helpers.patch_resolver_cycle_guard(line_ending.join(source_lines) + line_ending) ++ asserts.true(env, ("return r.mergeFromVisited(pkg, make(map[*packages.Package]struct{}))" + line_ending + "}") in patched) ++ asserts.true(env, "if _, ok := visited[pkg]; ok" in patched) ++ asserts.true(env, "r.mergeFromVisited(dep, visited)" in patched) ++ asserts.false(env, "r.mergeFrom(dep)" in patched) + + return unittest.end(env) + From cf728d3a59065426824d87ddbe13f8ce5bb2e721 Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Mon, 24 Aug 2026 15:34:50 +0200 Subject: [PATCH 10/37] go: isolate runtime data tools from Orchestrion --- .../v0_60_0/base/go/private/actions/link.bzl | 2 + .../base/go/private/rules/transition.bzl | 15 ++++++- .../v0_61_1/base/go/private/actions/link.bzl | 2 + .../base/go/private/rules/transition.bzl | 15 ++++++- .../v0_62_0/base/go/private/actions/link.bzl | 2 + .../base/go/private/rules/transition.bzl | 15 ++++++- .../v0_60_0/base/0001-full-delta.patch | 42 +++++++++++++++---- .../v0_61_1/base/0001-full-delta.patch | 42 +++++++++++++++---- .../v0_62_0/base/0001-full-delta.patch | 42 +++++++++++++++---- .../integration/run_bzlmod_go_integration.sh | 21 +++++++++- .../run_workspace_go_integration.sh | 21 +++++++++- 11 files changed, 190 insertions(+), 29 deletions(-) diff --git a/third_party/rgo/v0_60_0/base/go/private/actions/link.bzl b/third_party/rgo/v0_60_0/base/go/private/actions/link.bzl index 0d59e086..ca639965 100644 --- a/third_party/rgo/v0_60_0/base/go/private/actions/link.bzl +++ b/third_party/rgo/v0_60_0/base/go/private/actions/link.bzl @@ -98,6 +98,8 @@ def _orchestrion_pin_file_inputs(go, allowed_pin_files): def _orchestrion_enabled_for_link(go, synthetic_testmain_manifest): if not go.orchestrion: return False + if getattr(go, "orchestrion_mode", "") == _ORCHESTRION_MODE_TEST_OPTIMIZATION: + return False # Synthetic testmain compile already produced the Datadog helper packagefile # manifest that final link needs. Keep that final test-binary link on the diff --git a/third_party/rgo/v0_60_0/base/go/private/rules/transition.bzl b/third_party/rgo/v0_60_0/base/go/private/rules/transition.bzl index 15ef2a87..071746f6 100644 --- a/third_party/rgo/v0_60_0/base/go/private/rules/transition.bzl +++ b/third_party/rgo/v0_60_0/base/go/private/rules/transition.bzl @@ -424,12 +424,23 @@ def _non_go_transition_impl(settings, _attr): # and cause incorrect Go setting values. new_settings[original_key] = "" + # Runtime tools and files reached through non-Go attributes such as data + # are separate executables, not part of the selected instrumented target. + new_settings[_ORCHESTRION_ENABLED_SETTING] = False + new_settings[_ORCHESTRION_MODE_SETTING] = _ORCHESTRION_MODE_GENERAL + return new_settings non_go_transition = transition( implementation = _non_go_transition_impl, - inputs = TRANSITIONED_GO_SETTING_KEYS + _SETTING_KEY_TO_ORIGINAL_SETTING_KEY.values(), - outputs = TRANSITIONED_GO_SETTING_KEYS + _SETTING_KEY_TO_ORIGINAL_SETTING_KEY.values(), + inputs = TRANSITIONED_GO_SETTING_KEYS + _SETTING_KEY_TO_ORIGINAL_SETTING_KEY.values() + [ + _ORCHESTRION_ENABLED_SETTING, + _ORCHESTRION_MODE_SETTING, + ], + outputs = TRANSITIONED_GO_SETTING_KEYS + _SETTING_KEY_TO_ORIGINAL_SETTING_KEY.values() + [ + _ORCHESTRION_ENABLED_SETTING, + _ORCHESTRION_MODE_SETTING, + ], ) def _check_ternary(name, value): diff --git a/third_party/rgo/v0_61_1/base/go/private/actions/link.bzl b/third_party/rgo/v0_61_1/base/go/private/actions/link.bzl index 57dab7bf..3e2b1788 100644 --- a/third_party/rgo/v0_61_1/base/go/private/actions/link.bzl +++ b/third_party/rgo/v0_61_1/base/go/private/actions/link.bzl @@ -98,6 +98,8 @@ def _orchestrion_pin_file_inputs(go, allowed_pin_files): def _orchestrion_enabled_for_link(go, synthetic_testmain_manifest): if not go.orchestrion: return False + if getattr(go, "orchestrion_mode", "") == _ORCHESTRION_MODE_TEST_OPTIMIZATION: + return False # Synthetic testmain compile already produced the Datadog helper packagefile # manifest that final link needs. Keep that final test-binary link on the diff --git a/third_party/rgo/v0_61_1/base/go/private/rules/transition.bzl b/third_party/rgo/v0_61_1/base/go/private/rules/transition.bzl index 4cb09534..6751455a 100644 --- a/third_party/rgo/v0_61_1/base/go/private/rules/transition.bzl +++ b/third_party/rgo/v0_61_1/base/go/private/rules/transition.bzl @@ -424,12 +424,23 @@ def _non_go_transition_impl(settings, _attr): # and cause incorrect Go setting values. new_settings[original_key] = "" + # Runtime tools and files reached through non-Go attributes such as data + # are separate executables, not part of the selected instrumented target. + new_settings[_ORCHESTRION_ENABLED_SETTING] = False + new_settings[_ORCHESTRION_MODE_SETTING] = _ORCHESTRION_MODE_GENERAL + return new_settings non_go_transition = transition( implementation = _non_go_transition_impl, - inputs = TRANSITIONED_GO_SETTING_KEYS + _SETTING_KEY_TO_ORIGINAL_SETTING_KEY.values(), - outputs = TRANSITIONED_GO_SETTING_KEYS + _SETTING_KEY_TO_ORIGINAL_SETTING_KEY.values(), + inputs = TRANSITIONED_GO_SETTING_KEYS + _SETTING_KEY_TO_ORIGINAL_SETTING_KEY.values() + [ + _ORCHESTRION_ENABLED_SETTING, + _ORCHESTRION_MODE_SETTING, + ], + outputs = TRANSITIONED_GO_SETTING_KEYS + _SETTING_KEY_TO_ORIGINAL_SETTING_KEY.values() + [ + _ORCHESTRION_ENABLED_SETTING, + _ORCHESTRION_MODE_SETTING, + ], ) def _check_ternary(name, value): diff --git a/third_party/rgo/v0_62_0/base/go/private/actions/link.bzl b/third_party/rgo/v0_62_0/base/go/private/actions/link.bzl index ba934667..d2d15d48 100644 --- a/third_party/rgo/v0_62_0/base/go/private/actions/link.bzl +++ b/third_party/rgo/v0_62_0/base/go/private/actions/link.bzl @@ -98,6 +98,8 @@ def _orchestrion_pin_file_inputs(go, allowed_pin_files): def _orchestrion_enabled_for_link(go, synthetic_testmain_manifest): if not go.orchestrion: return False + if getattr(go, "orchestrion_mode", "") == _ORCHESTRION_MODE_TEST_OPTIMIZATION: + return False # Synthetic testmain compile already produced the Datadog helper packagefile # manifest that final link needs. Keep that final test-binary link on the diff --git a/third_party/rgo/v0_62_0/base/go/private/rules/transition.bzl b/third_party/rgo/v0_62_0/base/go/private/rules/transition.bzl index 4cb09534..6751455a 100644 --- a/third_party/rgo/v0_62_0/base/go/private/rules/transition.bzl +++ b/third_party/rgo/v0_62_0/base/go/private/rules/transition.bzl @@ -424,12 +424,23 @@ def _non_go_transition_impl(settings, _attr): # and cause incorrect Go setting values. new_settings[original_key] = "" + # Runtime tools and files reached through non-Go attributes such as data + # are separate executables, not part of the selected instrumented target. + new_settings[_ORCHESTRION_ENABLED_SETTING] = False + new_settings[_ORCHESTRION_MODE_SETTING] = _ORCHESTRION_MODE_GENERAL + return new_settings non_go_transition = transition( implementation = _non_go_transition_impl, - inputs = TRANSITIONED_GO_SETTING_KEYS + _SETTING_KEY_TO_ORIGINAL_SETTING_KEY.values(), - outputs = TRANSITIONED_GO_SETTING_KEYS + _SETTING_KEY_TO_ORIGINAL_SETTING_KEY.values(), + inputs = TRANSITIONED_GO_SETTING_KEYS + _SETTING_KEY_TO_ORIGINAL_SETTING_KEY.values() + [ + _ORCHESTRION_ENABLED_SETTING, + _ORCHESTRION_MODE_SETTING, + ], + outputs = TRANSITIONED_GO_SETTING_KEYS + _SETTING_KEY_TO_ORIGINAL_SETTING_KEY.values() + [ + _ORCHESTRION_ENABLED_SETTING, + _ORCHESTRION_MODE_SETTING, + ], ) def _check_ternary(name, value): diff --git a/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch index 77e47f14..b5e99598 100644 --- a/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch @@ -1419,10 +1419,10 @@ index a339eef..7525a1e 100644 inputs = depset(inputs_direct, transitive = inputs_transitive), outputs = outputs, diff --git a/go/private/actions/link.bzl b/go/private/actions/link.bzl -index 7d8ff5f..0d59e08 100644 +index 7d8ff5f..ca63996 100644 --- a/go/private/actions/link.bzl +++ b/go/private/actions/link.bzl -@@ -33,10 +33,88 @@ load( +@@ -33,10 +33,90 @@ load( "//go/private:rpath.bzl", "rpath", ) @@ -1491,6 +1491,8 @@ index 7d8ff5f..0d59e08 100644 +def _orchestrion_enabled_for_link(go, synthetic_testmain_manifest): + if not go.orchestrion: + return False ++ if getattr(go, "orchestrion_mode", "") == _ORCHESTRION_MODE_TEST_OPTIMIZATION: ++ return False + + # Synthetic testmain compile already produced the Datadog helper packagefile + # manifest that final link needs. Keep that final test-binary link on the @@ -1511,7 +1513,7 @@ index 7d8ff5f..0d59e08 100644 def emit_link( go, archive = None, -@@ -167,6 +245,12 @@ def emit_link( +@@ -167,6 +247,12 @@ def emit_link( builder_args.add("-o", executable) builder_args.add("-main", archive.data.file) builder_args.add("-p", archive.data.importmap) @@ -1524,7 +1526,7 @@ index 7d8ff5f..0d59e08 100644 tool_args.add_all(gc_linkopts) tool_args.add_all(go.toolchain.flags.link) -@@ -177,8 +261,14 @@ def emit_link( +@@ -177,8 +263,14 @@ def emit_link( tool_args.add_joined("-extldflags", extldflags, join_with = " ") inputs_direct = stamp_inputs + [go.sdk.package_list] @@ -1539,7 +1541,7 @@ index 7d8ff5f..0d59e08 100644 inputs_transitive = [ archive.libs, archive.cgo_deps, -@@ -186,6 +276,48 @@ def emit_link( +@@ -186,6 +278,48 @@ def emit_link( go.sdk.tools, go.stdlib.libs, ] @@ -1588,7 +1590,7 @@ index 7d8ff5f..0d59e08 100644 inputs = depset(direct = inputs_direct, transitive = inputs_transitive) go.actions.run( -@@ -194,7 +326,13 @@ def emit_link( +@@ -194,7 +328,13 @@ def emit_link( mnemonic = "GoLink", executable = go.toolchain._builder, arguments = [builder_args, "--", tool_args], @@ -4097,7 +4099,7 @@ index 911a924..9f73288 100644 test_deps.append(go.coverdata) test_go_info = new_go_info( diff --git a/go/private/rules/transition.bzl b/go/private/rules/transition.bzl -index b31a29a..15ef2a8 100644 +index b31a29a..071746f 100644 --- a/go/private/rules/transition.bzl +++ b/go/private/rules/transition.bzl @@ -161,6 +161,10 @@ go_transition = transition( @@ -4183,6 +4185,32 @@ index b31a29a..15ef2a8 100644 """ return dict(settings, **_reset_transition_dict) +@@ -408,12 +424,23 @@ def _non_go_transition_impl(settings, _attr): + # and cause incorrect Go setting values. + new_settings[original_key] = "" + ++ # Runtime tools and files reached through non-Go attributes such as data ++ # are separate executables, not part of the selected instrumented target. ++ new_settings[_ORCHESTRION_ENABLED_SETTING] = False ++ new_settings[_ORCHESTRION_MODE_SETTING] = _ORCHESTRION_MODE_GENERAL ++ + return new_settings + + non_go_transition = transition( + implementation = _non_go_transition_impl, +- inputs = TRANSITIONED_GO_SETTING_KEYS + _SETTING_KEY_TO_ORIGINAL_SETTING_KEY.values(), +- outputs = TRANSITIONED_GO_SETTING_KEYS + _SETTING_KEY_TO_ORIGINAL_SETTING_KEY.values(), ++ inputs = TRANSITIONED_GO_SETTING_KEYS + _SETTING_KEY_TO_ORIGINAL_SETTING_KEY.values() + [ ++ _ORCHESTRION_ENABLED_SETTING, ++ _ORCHESTRION_MODE_SETTING, ++ ], ++ outputs = TRANSITIONED_GO_SETTING_KEYS + _SETTING_KEY_TO_ORIGINAL_SETTING_KEY.values() + [ ++ _ORCHESTRION_ENABLED_SETTING, ++ _ORCHESTRION_MODE_SETTING, ++ ], + ) + + def _check_ternary(name, value): diff --git a/go/tools/builders/BUILD.bazel b/go/tools/builders/BUILD.bazel index 42b26a2..6f57dc4 100644 --- a/go/tools/builders/BUILD.bazel diff --git a/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch index 3f8ecc3a..aa56263e 100644 --- a/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch @@ -1403,10 +1403,10 @@ index 9c067b2..1b2ab71 100644 if ldflags: arguments.append(ldflags) diff --git a/go/private/actions/link.bzl b/go/private/actions/link.bzl -index 61f076f..57dab7b 100644 +index 61f076f..3e2b178 100644 --- a/go/private/actions/link.bzl +++ b/go/private/actions/link.bzl -@@ -33,10 +33,88 @@ load( +@@ -33,10 +33,90 @@ load( "//go/private:rpath.bzl", "rpath", ) @@ -1475,6 +1475,8 @@ index 61f076f..57dab7b 100644 +def _orchestrion_enabled_for_link(go, synthetic_testmain_manifest): + if not go.orchestrion: + return False ++ if getattr(go, "orchestrion_mode", "") == _ORCHESTRION_MODE_TEST_OPTIMIZATION: ++ return False + + # Synthetic testmain compile already produced the Datadog helper packagefile + # manifest that final link needs. Keep that final test-binary link on the @@ -1495,7 +1497,7 @@ index 61f076f..57dab7b 100644 def emit_link( go, archive = None, -@@ -167,6 +245,12 @@ def emit_link( +@@ -167,6 +247,12 @@ def emit_link( builder_args.add("-o", executable) builder_args.add("-main", archive.data.file) builder_args.add("-p", archive.data.importmap) @@ -1508,7 +1510,7 @@ index 61f076f..57dab7b 100644 tool_args.add_all(gc_linkopts) tool_args.add_all(go.toolchain.flags.link) -@@ -177,8 +261,14 @@ def emit_link( +@@ -177,8 +263,14 @@ def emit_link( tool_args.add_joined("-extldflags", extldflags, join_with = " ") inputs_direct = stamp_inputs + [go.sdk.package_list] @@ -1523,7 +1525,7 @@ index 61f076f..57dab7b 100644 inputs_transitive = [ archive.libs, archive.cgo_deps, -@@ -187,6 +277,48 @@ def emit_link( +@@ -187,6 +279,48 @@ def emit_link( go.sdk.tools, go.stdlib.libs, ] @@ -1572,7 +1574,7 @@ index 61f076f..57dab7b 100644 inputs = depset(direct = inputs_direct, transitive = inputs_transitive) go.actions.run( -@@ -195,7 +327,13 @@ def emit_link( +@@ -195,7 +329,13 @@ def emit_link( mnemonic = "GoLink", executable = go.toolchain._builder, arguments = [builder_args, "--", tool_args], @@ -4090,7 +4092,7 @@ index d65b5ca..4bdf9de 100644 test_deps.append(go.coverdata) test_go_info = new_go_info( diff --git a/go/private/rules/transition.bzl b/go/private/rules/transition.bzl -index 809ede6..4cb0953 100644 +index 809ede6..6751455 100644 --- a/go/private/rules/transition.bzl +++ b/go/private/rules/transition.bzl @@ -161,18 +161,48 @@ go_transition = transition( @@ -4179,6 +4181,32 @@ index 809ede6..4cb0953 100644 """ return dict(settings, **_reset_transition_dict) +@@ -388,12 +424,23 @@ def _non_go_transition_impl(settings, _attr): + # and cause incorrect Go setting values. + new_settings[original_key] = "" + ++ # Runtime tools and files reached through non-Go attributes such as data ++ # are separate executables, not part of the selected instrumented target. ++ new_settings[_ORCHESTRION_ENABLED_SETTING] = False ++ new_settings[_ORCHESTRION_MODE_SETTING] = _ORCHESTRION_MODE_GENERAL ++ + return new_settings + + non_go_transition = transition( + implementation = _non_go_transition_impl, +- inputs = TRANSITIONED_GO_SETTING_KEYS + _SETTING_KEY_TO_ORIGINAL_SETTING_KEY.values(), +- outputs = TRANSITIONED_GO_SETTING_KEYS + _SETTING_KEY_TO_ORIGINAL_SETTING_KEY.values(), ++ inputs = TRANSITIONED_GO_SETTING_KEYS + _SETTING_KEY_TO_ORIGINAL_SETTING_KEY.values() + [ ++ _ORCHESTRION_ENABLED_SETTING, ++ _ORCHESTRION_MODE_SETTING, ++ ], ++ outputs = TRANSITIONED_GO_SETTING_KEYS + _SETTING_KEY_TO_ORIGINAL_SETTING_KEY.values() + [ ++ _ORCHESTRION_ENABLED_SETTING, ++ _ORCHESTRION_MODE_SETTING, ++ ], + ) + + def _check_ternary(name, value): diff --git a/go/tools/builders/BUILD.bazel b/go/tools/builders/BUILD.bazel index ae68ab5..9cb7118 100644 --- a/go/tools/builders/BUILD.bazel diff --git a/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch index 44a732d1..ae8f0d31 100644 --- a/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch @@ -1403,10 +1403,10 @@ index 9c067b2..1b2ab71 100644 if ldflags: arguments.append(ldflags) diff --git a/go/private/actions/link.bzl b/go/private/actions/link.bzl -index c249ca7..ba93466 100644 +index c249ca7..d2d15d4 100644 --- a/go/private/actions/link.bzl +++ b/go/private/actions/link.bzl -@@ -33,10 +33,88 @@ load( +@@ -33,10 +33,90 @@ load( "//go/private:rpath.bzl", "rpath", ) @@ -1475,6 +1475,8 @@ index c249ca7..ba93466 100644 +def _orchestrion_enabled_for_link(go, synthetic_testmain_manifest): + if not go.orchestrion: + return False ++ if getattr(go, "orchestrion_mode", "") == _ORCHESTRION_MODE_TEST_OPTIMIZATION: ++ return False + + # Synthetic testmain compile already produced the Datadog helper packagefile + # manifest that final link needs. Keep that final test-binary link on the @@ -1495,7 +1497,7 @@ index c249ca7..ba93466 100644 def emit_link( go, archive = None, -@@ -168,6 +246,12 @@ def emit_link( +@@ -168,6 +248,12 @@ def emit_link( builder_args.add("-o", executable) builder_args.add("-main", archive.data.file) builder_args.add("-p", archive.data.importmap) @@ -1508,7 +1510,7 @@ index c249ca7..ba93466 100644 tool_args.add_all(gc_linkopts) tool_args.add_all(go.toolchain.flags.link) -@@ -178,8 +262,14 @@ def emit_link( +@@ -178,8 +264,14 @@ def emit_link( tool_args.add_joined("-extldflags", extldflags, join_with = " ") inputs_direct = stamp_inputs + [go.sdk.package_list] @@ -1523,7 +1525,7 @@ index c249ca7..ba93466 100644 inputs_transitive = [ archive.libs, archive.cgo_deps, -@@ -189,6 +279,48 @@ def emit_link( +@@ -189,6 +281,48 @@ def emit_link( go.sdk.tools, go.stdlib.libs, ] @@ -1572,7 +1574,7 @@ index c249ca7..ba93466 100644 inputs = depset(direct = inputs_direct, transitive = inputs_transitive) go.actions.run( -@@ -197,7 +329,13 @@ def emit_link( +@@ -197,7 +331,13 @@ def emit_link( mnemonic = "GoLink", executable = go.toolchain._builder, arguments = [builder_args, "--", tool_args], @@ -4094,7 +4096,7 @@ index 3214c71..66728df 100644 test_deps.append(go.coverdata) test_go_info = new_go_info( diff --git a/go/private/rules/transition.bzl b/go/private/rules/transition.bzl -index 809ede6..4cb0953 100644 +index 809ede6..6751455 100644 --- a/go/private/rules/transition.bzl +++ b/go/private/rules/transition.bzl @@ -161,18 +161,48 @@ go_transition = transition( @@ -4183,6 +4185,32 @@ index 809ede6..4cb0953 100644 """ return dict(settings, **_reset_transition_dict) +@@ -388,12 +424,23 @@ def _non_go_transition_impl(settings, _attr): + # and cause incorrect Go setting values. + new_settings[original_key] = "" + ++ # Runtime tools and files reached through non-Go attributes such as data ++ # are separate executables, not part of the selected instrumented target. ++ new_settings[_ORCHESTRION_ENABLED_SETTING] = False ++ new_settings[_ORCHESTRION_MODE_SETTING] = _ORCHESTRION_MODE_GENERAL ++ + return new_settings + + non_go_transition = transition( + implementation = _non_go_transition_impl, +- inputs = TRANSITIONED_GO_SETTING_KEYS + _SETTING_KEY_TO_ORIGINAL_SETTING_KEY.values(), +- outputs = TRANSITIONED_GO_SETTING_KEYS + _SETTING_KEY_TO_ORIGINAL_SETTING_KEY.values(), ++ inputs = TRANSITIONED_GO_SETTING_KEYS + _SETTING_KEY_TO_ORIGINAL_SETTING_KEY.values() + [ ++ _ORCHESTRION_ENABLED_SETTING, ++ _ORCHESTRION_MODE_SETTING, ++ ], ++ outputs = TRANSITIONED_GO_SETTING_KEYS + _SETTING_KEY_TO_ORIGINAL_SETTING_KEY.values() + [ ++ _ORCHESTRION_ENABLED_SETTING, ++ _ORCHESTRION_MODE_SETTING, ++ ], + ) + + def _check_ternary(name, value): diff --git a/go/tools/builders/BUILD.bazel b/go/tools/builders/BUILD.bazel index a8834a1..64996d3 100644 --- a/go/tools/builders/BUILD.bazel diff --git a/tools/tests/integration/run_bzlmod_go_integration.sh b/tools/tests/integration/run_bzlmod_go_integration.sh index 9e0201c1..c02463b8 100755 --- a/tools/tests/integration/run_bzlmod_go_integration.sh +++ b/tools/tests/integration/run_bzlmod_go_integration.sh @@ -709,6 +709,12 @@ go_binary( importpath = "${MODULE_IMPORTPATH}/fixture_tool", ) +go_binary( + name = "fixture_testing_tool", + srcs = ["fixture_testing_tool.go"], + importpath = "${MODULE_IMPORTPATH}/fixture_testing_tool", +) + go_reset_target( name = "fixture_tool_reset", dep = ":fixture_tool", @@ -720,7 +726,10 @@ dd_go_test( "hello_external_test.go", "hello_test.go", ], - data = [":fixture_tool_reset"], + data = [ + ":fixture_testing_tool", + ":fixture_tool_reset", + ], embed = [":hello_lib"], orchestrion_pin_files = [ "//:go.mod", @@ -749,6 +758,16 @@ EOF cat > "$ws_dir/app/fixture_tool.go" <<'EOF' package main +func main() {} +EOF + + cat > "$ws_dir/app/fixture_testing_tool.go" <<'EOF' +package main + +import "testing" + +var _ = testing.Short + func main() {} EOF diff --git a/tools/tests/integration/run_workspace_go_integration.sh b/tools/tests/integration/run_workspace_go_integration.sh index 57114f86..60d6ee01 100755 --- a/tools/tests/integration/run_workspace_go_integration.sh +++ b/tools/tests/integration/run_workspace_go_integration.sh @@ -694,6 +694,12 @@ go_binary( importpath = "${MODULE_IMPORTPATH}/fixture_tool", ) +go_binary( + name = "fixture_testing_tool", + srcs = ["fixture_testing_tool.go"], + importpath = "${MODULE_IMPORTPATH}/fixture_testing_tool", +) + go_reset_target( name = "fixture_tool_reset", dep = ":fixture_tool", @@ -705,7 +711,10 @@ dd_go_test( "hello_external_test.go", "hello_test.go", ], - data = [":fixture_tool_reset"], + data = [ + ":fixture_testing_tool", + ":fixture_tool_reset", + ], embed = [":hello_lib"], ) @@ -727,6 +736,16 @@ EOF cat > "$ws_dir/app/fixture_tool.go" <<'EOF' package main +func main() {} +EOF + + cat > "$ws_dir/app/fixture_testing_tool.go" <<'EOF' +package main + +import "testing" + +var _ = testing.Short + func main() {} EOF From ba0c3dd263069d8482ba5f4aa2cd1698904a2f6a Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Mon, 24 Aug 2026 15:34:57 +0200 Subject: [PATCH 11/37] uploader: batch CODEOWNERS enrichment --- tools/core/uploader_bash_runtime.sh.tpl | 155 +++++++++++++++++------- tools/tests/python/test_python_tools.py | 8 ++ 2 files changed, 121 insertions(+), 42 deletions(-) diff --git a/tools/core/uploader_bash_runtime.sh.tpl b/tools/core/uploader_bash_runtime.sh.tpl index 1165a8fe..c7e8943e 100644 --- a/tools/core/uploader_bash_runtime.sh.tpl +++ b/tools/core/uploader_bash_runtime.sh.tpl @@ -2671,61 +2671,132 @@ inject_codeowners_tags() { init_codeowners (( CODEOWNERS_ENABLED == 1 )) || return 0 - local events_len idx event_type has_existing source_path owners_json tmp_payload - # Skip gracefully on malformed payload shapes; uploader remains best-effort. - events_len=$(jq '.events | if type=="array" then length else 0 end' "$payload_file" 2>/dev/null || echo 0) - if ! [[ "$events_len" =~ ^[0-9]+$ ]]; then + local event_rows unique_sources owners_by_source assignments stats tmp_payload + event_rows=$(mktemp "$TMP_PAYLOAD_DIR/codeowners_events.XXXXXX" 2>/dev/null || true) + unique_sources=$(mktemp "$TMP_PAYLOAD_DIR/codeowners_sources.XXXXXX" 2>/dev/null || true) + owners_by_source=$(mktemp "$TMP_PAYLOAD_DIR/codeowners_owners.XXXXXX" 2>/dev/null || true) + assignments=$(mktemp "$TMP_PAYLOAD_DIR/codeowners_assignments.XXXXXX" 2>/dev/null || true) + stats=$(mktemp "$TMP_PAYLOAD_DIR/codeowners_stats.XXXXXX" 2>/dev/null || true) + if [[ -z "$event_rows" || -z "$unique_sources" || -z "$owners_by_source" || -z "$assignments" || -z "$stats" ]]; then + [[ -n "$event_rows" ]] && rm -f "$event_rows" 2>/dev/null || true + [[ -n "$unique_sources" ]] && rm -f "$unique_sources" 2>/dev/null || true + [[ -n "$owners_by_source" ]] && rm -f "$owners_by_source" 2>/dev/null || true + [[ -n "$assignments" ]] && rm -f "$assignments" 2>/dev/null || true + [[ -n "$stats" ]] && rm -f "$stats" 2>/dev/null || true + ((++CO_EVENTS_SKIPPED_ERRORS)) + [[ "$DEBUG" == "1" ]] && dbg "codeowners: skip internal error creating batch files" return 0 fi - for ((idx = 0; idx < events_len; idx++)); do - event_type=$(jq -r --argjson idx "$idx" '.events[$idx].type // ""' "$payload_file" 2>/dev/null || true) + # Extract the relevant event state once. URI encoding keeps tabs and newlines + # in source paths from interfering with the tab-separated batch format. + if ! jq -r ' + def source_path: + (.content.meta["test.source.file"] + // .content.meta["test.source.path"] + // .content.meta["source.file"] + // .content.meta["source.path"] + // .content.source.file + // .content.source.path + // "") + | tostring; + .events + | if type == "array" then to_entries[] else empty end + | .key as $idx + | .value as $event # CODEOWNERS remains scoped to non-span lifecycle/test events. Span-form Go # events still receive context and Bazel tags before this CODEOWNERS pass. - [[ "$event_type" == "span" ]] && continue - ((++CO_EVENTS_SCANNED)) - - has_existing=$(jq -r --argjson idx "$idx" 'if (.events[$idx].content.meta | type) == "object" and (.events[$idx].content.meta | has("test.codeowners")) then "1" else "0" end' "$payload_file" 2>/dev/null || echo "0") - if [[ "$has_existing" == "1" ]]; then - ((++CO_EVENTS_SKIPPED_EXISTING)) - [[ "$DEBUG" == "1" ]] && dbg "codeowners: skip existing tag at event[$idx]" - continue - fi - - source_path=$(jq -r --argjson idx "$idx" '.events[$idx].content.meta["test.source.file"] // .events[$idx].content.meta["test.source.path"] // .events[$idx].content.meta["source.file"] // .events[$idx].content.meta["source.path"] // .events[$idx].content.source.file // .events[$idx].content.source.path // ""' "$payload_file" 2>/dev/null || true) - if [[ -z "$source_path" ]]; then - ((++CO_EVENTS_SKIPPED_MISSING_SOURCE)) - [[ "$DEBUG" == "1" ]] && dbg "codeowners: skip missing source at event[$idx]" - continue - fi + | select(($event.type // "") != "span") + | if (($event.content.meta | type) == "object" and ($event.content.meta | has("test.codeowners"))) then + [$idx, "existing"] + else + ($event | source_path) as $source + | if $source == "" then + [$idx, "missing"] + else + [$idx, "source", ($source | @uri)] + end + end + | @tsv + ' "$payload_file" > "$event_rows" 2>/dev/null; then + rm -f "$event_rows" "$unique_sources" "$owners_by_source" "$assignments" "$stats" 2>/dev/null || true + return 0 + fi + awk -F '\t' '$2 == "source" { print $3 }' "$event_rows" | LC_ALL=C sort -u > "$unique_sources" + local encoded_source source_path owners_json + while IFS= read -r encoded_source; do + [[ -n "$encoded_source" ]] || continue + source_path=$(decode_percent_path "$encoded_source") owners_json=$(resolve_codeowners_json_for_source "$source_path") - if [[ -z "$owners_json" ]]; then - ((++CO_EVENTS_SKIPPED_UNMATCHED)) - [[ "$DEBUG" == "1" ]] && dbg "codeowners: skip unmatched source '$source_path' at event[$idx]" - continue - fi + printf '%s\t%s\n' "$encoded_source" "$owners_json" >> "$owners_by_source" + [[ "$DEBUG" == "1" ]] && dbg "codeowners: resolved source '$source_path' owners='${owners_json:-}'" + done < "$unique_sources" + + # Join all events to the per-source ownership cache in one process. This + # avoids both a jq invocation and a linear Bash cache scan per event. + if ! awk -F '\t' \ + -v owners_file="$owners_by_source" \ + -v assignments_file="$assignments" \ + -v stats_file="$stats" ' + FILENAME == owners_file { + owners[$1] = $2 + next + } + { + scanned++ + if ($2 == "existing") { + existing++ + } else if ($2 == "missing") { + missing++ + } else if ($2 == "source" && owners[$3] != "") { + print $1 "\t" owners[$3] > assignments_file + enriched++ + } else { + unmatched++ + } + } + END { + print scanned + 0 "\t" enriched + 0 "\t" existing + 0 "\t" missing + 0 "\t" unmatched + 0 > stats_file + } + ' "$owners_by_source" "$event_rows"; then + rm -f "$event_rows" "$unique_sources" "$owners_by_source" "$assignments" "$stats" 2>/dev/null || true + ((++CO_EVENTS_SKIPPED_ERRORS)) + [[ "$DEBUG" == "1" ]] && dbg "codeowners: skip internal error joining batch assignments" + return 0 + fi + + local batch_scanned pending_enriched batch_existing batch_missing batch_unmatched + IFS=$'\t' read -r batch_scanned pending_enriched batch_existing batch_missing batch_unmatched < "$stats" + CO_EVENTS_SCANNED=$((CO_EVENTS_SCANNED + batch_scanned)) + CO_EVENTS_SKIPPED_EXISTING=$((CO_EVENTS_SKIPPED_EXISTING + batch_existing)) + CO_EVENTS_SKIPPED_MISSING_SOURCE=$((CO_EVENTS_SKIPPED_MISSING_SOURCE + batch_missing)) + CO_EVENTS_SKIPPED_UNMATCHED=$((CO_EVENTS_SKIPPED_UNMATCHED + batch_unmatched)) + if (( pending_enriched > 0 )); then tmp_payload=$(mktemp "$TMP_PAYLOAD_DIR/codeowners_payload.XXXXXX" 2>/dev/null || true) - if [[ -z "$tmp_payload" ]]; then - ((++CO_EVENTS_SKIPPED_ERRORS)) - [[ "$DEBUG" == "1" ]] && dbg "codeowners: skip internal error creating temp payload at event[$idx]" - continue - fi - if jq --arg owners "$owners_json" --argjson idx "$idx" ' - .events[$idx].content = (.events[$idx].content // {}) - | .events[$idx].content.meta = ((.events[$idx].content.meta // {}) | .["test.codeowners"] = $owners) + if [[ -n "$tmp_payload" ]] && jq --rawfile assignments "$assignments" ' + ($assignments + | split("\n") + | map(select(length > 0) | split("\t") | {(.[0]): .[1]}) + | add // {}) as $owners_by_index + | reduce ($owners_by_index | to_entries[]) as $entry (.; + ($entry.key | tonumber) as $idx + | .events[$idx].content = (.events[$idx].content // {}) + | .events[$idx].content.meta = ((.events[$idx].content.meta // {}) | .["test.codeowners"] = $entry.value) + ) ' "$payload_file" > "$tmp_payload"; then - # Atomic replacement prevents partially-written payload files. + # One atomic replacement avoids rewriting the full payload per event. mv "$tmp_payload" "$payload_file" - ((++CO_EVENTS_ENRICHED)) - [[ "$DEBUG" == "1" ]] && dbg "codeowners: assigned owners '$owners_json' at event[$idx]" + CO_EVENTS_ENRICHED=$((CO_EVENTS_ENRICHED + pending_enriched)) else - rm -f "$tmp_payload" 2>/dev/null || true - ((++CO_EVENTS_SKIPPED_ERRORS)) - [[ "$DEBUG" == "1" ]] && dbg "codeowners: skip jq update failure at event[$idx]" + [[ -n "$tmp_payload" ]] && rm -f "$tmp_payload" 2>/dev/null || true + CO_EVENTS_SKIPPED_ERRORS=$((CO_EVENTS_SKIPPED_ERRORS + pending_enriched)) + [[ "$DEBUG" == "1" ]] && dbg "codeowners: skip batch jq update failure for $pending_enriched event(s)" fi - done + fi + + rm -f "$event_rows" "$unique_sources" "$owners_by_source" "$assignments" "$stats" 2>/dev/null || true if [[ "$DEBUG" == "1" ]]; then dbg "codeowners: scanned=$CO_EVENTS_SCANNED enriched=$CO_EVENTS_ENRICHED skipped_existing=$CO_EVENTS_SKIPPED_EXISTING skipped_missing_source=$CO_EVENTS_SKIPPED_MISSING_SOURCE skipped_unmatched=$CO_EVENTS_SKIPPED_UNMATCHED skipped_errors=$CO_EVENTS_SKIPPED_ERRORS" diff --git a/tools/tests/python/test_python_tools.py b/tools/tests/python/test_python_tools.py index 86541002..37c241bd 100644 --- a/tools/tests/python/test_python_tools.py +++ b/tools/tests/python/test_python_tools.py @@ -7621,6 +7621,14 @@ def test_uploader_enriches_span_event_payloads(self) -> None: self.assertIn("events still receive context and Bazel tags before this CODEOWNERS pass", powershell_text) self.assertNotIn('[string]$eventType -eq "span"', powershell_text) + def test_bash_uploader_batches_codeowners_payload_updates(self) -> None: + """Validate CODEOWNERS enrichment rewrites each payload only once.""" + bash_text = _runfile("tools/core/uploader_bash_runtime.sh.tpl").read_text(encoding="utf-8") + + self.assertIn('jq --rawfile assignments "$assignments"', bash_text) + self.assertIn("One atomic replacement avoids rewriting the full payload per event", bash_text) + self.assertNotIn('jq --arg owners "$owners_json" --argjson idx "$idx"', bash_text) + def test_uploader_skips_empty_test_payload_placeholders(self) -> None: """Validate empty JSON placeholders are not uploaded as test payloads.""" bash_text = _runfile("tools/core/uploader_bash_runtime.sh.tpl").read_text(encoding="utf-8") From a8c8def793a725720577b6aeccb75661fa0daf52 Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Tue, 25 Aug 2026 19:49:49 +0200 Subject: [PATCH 12/37] go: mirror rules_go inferred test importpaths --- AGENTS.md | 4 +- MODULE.bazel.lock | 2 +- README.md | 4 +- docs/Initial_documentation.md | 8 +-- docs/RFC.md | 8 +-- modules/go/tests/BUILD.bazel | 23 ++++--- modules/go/tests/selector_fixture.go | 7 +++ modules/go/tests/test_macro.bzl | 16 +++++ modules/go/tests/test_payloads_selector.bzl | 66 +++++++-------------- modules/go/topt_go_infer.bzl | 25 +++----- modules/go/topt_go_test.bzl | 43 +++++++------- 11 files changed, 105 insertions(+), 101 deletions(-) create mode 100644 modules/go/tests/selector_fixture.go diff --git a/AGENTS.md b/AGENTS.md index aa5fd3bf..2c9c7c99 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -184,8 +184,8 @@ The sync rule creates `@test_optimization_data//` containing: ) ``` - In the repository's central Go wrapper: load `dd_topt_go_test` and the generated `topt_data`, then delegate every public `dd_go_test(...)` call to the Datadog macro. Do not create separate plain and optimized onboarding paths. -- Import path inference (preferred): add a `go_library` and set `embed = [":"]` in the central wrapper call. The macro reads rules_go's provider to compute the same `importpath` `go_test` uses and selects the matching per-module payload group. Inferred misses use the core bundle. When synchronized metadata exposes module groups, an explicit `importpath` or `module_label_override` that does not match one fails analysis; when no groups exist, the canonical full bundle remains valid. -- Fallback (no embed): if neither `embed` nor explicit `importpath` is provided, the macro computes `/` using the exported `topt_data["runtimes"]["go"]["module_path"]`. In this fallback mode only, it consults `topt_data["runtimes"]["go"]["module_included"]` as a coarse gate before attempting per‑module selection. +- Import path inference (preferred): add a `go_library` with an explicit `importpath` and set `embed = [":"]` in the central wrapper call. The macro reads rules_go's provider to compute the same `importpath` `go_test` uses and selects the matching per-module payload group. Inferred misses use the core bundle. When synchronized metadata exposes module groups, an explicit `importpath` or `module_label_override` that does not match one fails analysis; when no groups exist, the canonical full bundle remains valid. +- Fallback (no embed): if neither `embed` nor explicit `importpath` is provided, the macro computes the label-derived importpath that `rules_go` assigns to the hidden raw `go_test` target. In this fallback mode only, it consults `topt_data["runtimes"]["go"]["module_included"]` as a coarse gate before attempting per-module selection. - Tests can read `DD_TEST_OPTIMIZATION_MANIFEST_FILE` to resolve the manifest directory (via `filepath.Dir()`) and access synced payloads. - For Python/Java/NodeJS/.NET/Ruby companions, follow the corresponding quickstart sections in `README.md` (`Bzlmod + Python companion`, `Bzlmod + Java companion`, `Bzlmod + NodeJS companion`, `Bzlmod + .NET companion`, `Bzlmod + Ruby companion`). diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index f84d579b..5e951a00 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -766,7 +766,7 @@ }, "@@rules_go+//go:extensions.bzl%orchestrion": { "general": { - "bzlTransitiveDigest": "ddXasJ1gcaFD8IfNKQVFG/bfJZSinEMN9y7/lUDv4gI=", + "bzlTransitiveDigest": "3GekHiacfoRyFYc/4lHKb65uhKwiU9DswEVKvmrzfdY=", "usagesDigest": "9kKEG/hjK4fnk42l4jAKDLhx9xExyCYAS8r8NlbPLMw=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, diff --git a/README.md b/README.md index ca671857..e6d7bf7a 100644 --- a/README.md +++ b/README.md @@ -1981,8 +1981,8 @@ The macro auto-selects the correct per-module payloads by inferring the Go packa - Precedence: 1) `importpath` explicitly set on your `go_test` invocation (if provided in kwargs) - 2) Inference via `embed = [":"]` by reading `GoArchive.importpath` from rules_go (recommended) - 3) Fallback: `/` where the Go module path comes from the synced repo's exported `topt_data["runtimes"]["go"]["module_path"]` + 2) Inference via `embed = [":"]` when that library has an explicit rules_go importpath (recommended) + 3) Fallback: the label-derived importpath that `rules_go` assigns to the hidden raw `go_test` target When automatic per-module selection is close but not exact (for example, custom import path layouts), use `module_label_override` to pin the expected sanitized diff --git a/docs/Initial_documentation.md b/docs/Initial_documentation.md index a161804d..8fe2545c 100644 --- a/docs/Initial_documentation.md +++ b/docs/Initial_documentation.md @@ -147,18 +147,18 @@ whose virtual context keys preserve exact per-payload enrichment. The `dd_topt_go_test` macro automatically selects the correct per‑module payloads by inferring the Go package `importpath` using `rules_go` providers, mirroring how `go_test` computes it. -- Preferred: add a `go_library` and set `embed = [":"]` in your `dd_topt_go_test` call. The macro reads `GoArchive`/`GoInfo` from `@rules_go//go:def.bzl` via a Starlark aspect walking `embed`. +- Preferred: add a `go_library` with an explicit `importpath` and set `embed = [":"]` in your `dd_topt_go_test` call. The macro reads `GoArchive`/`GoInfo` from `@rules_go//go:def.bzl` via a Starlark aspect. - Precedence for determining importpath: 1) `importpath` explicitly set on the `go_test` invocation (if provided via kwargs) - 2) Provider‑based inference via `embed` - 3) Fallback to `/`, where the module path is exported by the sync repo in `topt_data["runtimes"]["go"]["module_path"]` + 2) Provider-based inference via `embed` when the library importpath is explicit + 3) Fallback to the label-derived importpath that `rules_go` assigns to the hidden raw `go_test` target - Per‑module selection: - When synchronized metadata exposes module groups, explicit `importpath` or `module_label_override` values must match one or analysis fails. When no module groups exist, the canonical full bundle remains valid. - Provider-based inference via `embed` attempts per-module selection and may fall back to the canonical full bundle on a miss. - - When using (3), the macro consults + - When neither `importpath` nor `embed` is supplied, the macro consults `topt_data["runtimes"]["go"]["module_included"]` as a coarse gate; if false, it uses the full bundle. diff --git a/docs/RFC.md b/docs/RFC.md index 2c29754c..490747f4 100644 --- a/docs/RFC.md +++ b/docs/RFC.md @@ -283,10 +283,10 @@ Language Macros - Configure payloads to write to `TEST_UNDECLARED_OUTPUTS_DIR` automatically. - Surface reasonable defaults and allow overrides. - Note: Macros no longer create per-test uploaders. Users create ONE uploader target per workspace. -- Go importpath inference: - - A Starlark aspect walks `embed` on the `go_test` target and reads `GoArchive.importpath` from rules_go providers, mirroring how `go_test` computes it. - - A small rule uses the inferred importpath to pick the matching `:module_` filegroup from the synced repo and exposes it in runfiles; the macro sets `DD_TEST_OPTIMIZATION_MANIFEST_FILE` to `$(rlocationpath )` using `topt_data["manifest_path"]`, so custom `out_dir` values are supported. - - Precedence: (1) explicit `importpath` kwarg on the `go_test`; (2) provider‑based inference via `embed`; (3) fallback to `/`. +- Go importpath inference: + - A Starlark aspect reads an embedded library's explicit `GoArchive.importpath` from rules_go providers, mirroring how `go_test` computes it. + - A small rule uses the inferred importpath to pick the matching `:module_` filegroup from the synced repo and exposes it in runfiles; the macro sets `DD_TEST_OPTIMIZATION_MANIFEST_FILE` to `$(rlocationpath )` using `topt_data["manifest_path"]`, so custom `out_dir` values are supported. + - Precedence: (1) explicit `importpath` kwarg on the `go_test`; (2) an explicit provider importpath via `embed`; (3) the label-derived importpath that `rules_go` assigns to the hidden raw `go_test` target. - When synchronized metadata exposes module groups, explicit `importpath` and `module_label_override` values must match one or analysis fails. Provider-based or derived inference, and metadata with no module groups, may use the canonical full bundle. The exported `topt_data["runtimes"]["go"]["module_included"]` flag is consulted only in fallback mode. - Module dependency: the Go companion module (`datadog-rules-test-optimization-go`) declares `bazel_dep("rules_go", )` to make provider loads visible under Bzlmod; it does not configure toolchains. Consumers must still configure `rules_go` and the Go SDK in their own `MODULE.bazel`. diff --git a/modules/go/tests/BUILD.bazel b/modules/go/tests/BUILD.bazel index acec191c..3e91f4ed 100644 --- a/modules/go/tests/BUILD.bazel +++ b/modules/go/tests/BUILD.bazel @@ -27,6 +27,7 @@ load( "go_macro_explicit_service_wiring_test", "go_macro_general_mode_linker_flags_target", "go_macro_general_mode_linker_flags_wiring_test", + "go_macro_inferred_importpath_metadata_test", "go_macro_multi_service_target", "go_macro_multi_service_wiring_test", "go_macro_orchestrion_enablement_mismatch_failure_test", @@ -83,8 +84,6 @@ load( ) load( ":test_payloads_selector.bzl", - "selector_deps_precedence_target", - "selector_deps_precedence_test", "selector_embed_precedence_target", "selector_embed_precedence_test", "selector_empty_importpath_fallback_target", @@ -98,6 +97,8 @@ load( "selector_fallback_test", "selector_include_disabled_target", "selector_include_disabled_test", + "selector_inferred_embed_uses_fallback_target", + "selector_inferred_embed_uses_fallback_test", "selector_no_match_fallback_target", "selector_no_match_fallback_test", "selector_omits_flaky_tests_target", @@ -293,14 +294,14 @@ selector_embed_precedence_test( target_under_test = ":selector_embed_precedence_target", ) -selector_deps_precedence_target( - name = "selector_deps_precedence_target", +selector_inferred_embed_uses_fallback_target( + name = "selector_inferred_embed_uses_fallback_target", tags = ["manual"], ) -selector_deps_precedence_test( - name = "selector_deps_precedence_test", - target_under_test = ":selector_deps_precedence_target", +selector_inferred_embed_uses_fallback_test( + name = "selector_inferred_embed_uses_fallback_test", + target_under_test = ":selector_inferred_embed_uses_fallback_target", ) selector_fallback_target( @@ -396,6 +397,11 @@ go_macro_single_service_wiring_test( target_under_test = ":go_macro_single_service_target__raw_go_test", ) +go_macro_inferred_importpath_metadata_test( + name = "go_macro_inferred_importpath_metadata_test", + target_under_test = ":go_macro_single_service_target_topt_bazel_metadata", +) + go_macro_public_wrapper_test( name = "go_macro_public_wrapper_test", target_under_test = ":go_macro_single_service_target", @@ -785,6 +791,7 @@ test_suite( ":go_macro_dynamic_manifest_wiring_test", ":go_macro_env_none_wiring_test", ":go_macro_explicit_service_wiring_test", + ":go_macro_inferred_importpath_metadata_test", ":go_macro_multi_service_wiring_test", ":go_macro_orchestrion_enablement_mismatch_failure_test", ":go_macro_orchestrion_pin_files_provider_test", @@ -824,7 +831,6 @@ test_suite( ":resolve_topt_service_key_prefers_exact_then_sanitized_test", ":resolve_topt_service_key_unknown_failure_test", ":select_module_group_name_test", - ":selector_deps_precedence_test", ":selector_embed_precedence_test", ":selector_empty_importpath_fallback_test", ":selector_explicit_miss_failure_test", @@ -832,6 +838,7 @@ test_suite( ":selector_explicit_precedence_test", ":selector_fallback_test", ":selector_include_disabled_test", + ":selector_inferred_embed_uses_fallback_test", ":selector_no_match_fallback_test", ":selector_omits_flaky_tests_test", ":selector_override_miss_failure_test", diff --git a/modules/go/tests/selector_fixture.go b/modules/go/tests/selector_fixture.go new file mode 100644 index 00000000..27c73fa3 --- /dev/null +++ b/modules/go/tests/selector_fixture.go @@ -0,0 +1,7 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under +// the Apache 2.0 License. +// +// This product includes software developed at Datadog +// (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +package selectorfixture diff --git a/modules/go/tests/test_macro.bzl b/modules/go/tests/test_macro.bzl index 5ba12a78..ecc1c0a7 100644 --- a/modules/go/tests/test_macro.bzl +++ b/modules/go/tests/test_macro.bzl @@ -549,6 +549,19 @@ def _go_macro_single_service_wiring_test_impl(ctx): asserts.true(env, captured.rundir.endswith("tests")) return analysistest.end(env) +def _go_macro_inferred_importpath_metadata_test_impl(ctx): + """Assert fallback metadata mirrors the hidden rules_go test label.""" + env = analysistest.begin(ctx) + target = analysistest.target_under_test(env) + metadata = target[ToptGoBazelMetadataInfo].metadata + asserts.equals( + env, + "modules/go/tests/go_macro_single_service_target__raw_go_test", + metadata["bazel.go.importpath"], + ) + asserts.equals(env, "fallback", metadata["bazel.go.importpath_source"]) + return analysistest.end(env) + def _go_macro_disabled_raw_wiring_test_impl(ctx): """Assert disabled metadata forwards caller kwargs to one raw public test.""" env = analysistest.begin(ctx) @@ -1050,6 +1063,9 @@ def _orch_wrapper_materialized_actual_windows_test_impl(ctx): go_macro_single_service_wiring_test = analysistest.make( _go_macro_single_service_wiring_test_impl, ) +go_macro_inferred_importpath_metadata_test = analysistest.make( + _go_macro_inferred_importpath_metadata_test_impl, +) go_macro_disabled_raw_wiring_test = analysistest.make( _go_macro_disabled_raw_wiring_test_impl, ) diff --git a/modules/go/tests/test_payloads_selector.bzl b/modules/go/tests/test_payloads_selector.bzl index 1e43b653..395eb2b4 100644 --- a/modules/go/tests/test_payloads_selector.bzl +++ b/modules/go/tests/test_payloads_selector.bzl @@ -8,12 +8,13 @@ load("@bazel_skylib//lib:unittest.bzl", "analysistest", "asserts") load("@datadog-rules-test-optimization-go//:topt_go_infer.bzl", "topt_go_payloads_selector") +load("@rules_go//go:def.bzl", "go_library") _COMMON_MODULE_GROUPS = [ ":module_example_com_explicit_pkg", ":module_example_com_embed_pkg", - ":module_example_com_deps_pkg", ":module_example_com_fallback_pkg", + ":module_modules_go_tests_inferred_embed_library", ":module_custom_override", ] @@ -52,20 +53,6 @@ _cache_payload = rule( implementation = _cache_payload_impl, ) -def _embed_source_impl(_ctx): - """Implement embed source impl behavior.""" - return [] - -_embed_source = rule( - implementation = _embed_source_impl, - attrs = { - # Keep attribute names aligned with what _importpath_aspect inspects. - "importpath": attr.string(), - "embed": attr.label_list(), - "deps": attr.label_list(), - }, -) - def selector_payload_fixture_targets(): """Shared marker and embed fixtures used by selector tests.""" _payload_marker( @@ -84,14 +71,14 @@ def selector_payload_fixture_targets(): name = "module_example_com_embed_pkg", marker = "module:embed", ) - _payload_marker( - name = "module_example_com_deps_pkg", - marker = "module:deps", - ) _payload_marker( name = "module_example_com_fallback_pkg", marker = "module:fallback", ) + _payload_marker( + name = "module_modules_go_tests_inferred_embed_library", + marker = "module:inferred-embed", + ) _payload_marker( name = "module_custom_override", marker = "module:override", @@ -99,21 +86,14 @@ def selector_payload_fixture_targets(): _cache_payload( name = "module_example_com_cache_pkg", ) - _embed_source( - name = "embed_leaf", + go_library( + name = "explicit_embed_library", + srcs = ["selector_fixture.go"], importpath = "example.com/embed/pkg", ) - _embed_source( - name = "embed_wrapper", - embed = [":embed_leaf"], - ) - _embed_source( - name = "deps_leaf", - importpath = "example.com/deps/pkg", - ) - _embed_source( - name = "deps_wrapper", - deps = [":deps_leaf"], + go_library( + name = "inferred_embed_library", + srcs = ["selector_fixture.go"], ) def selector_explicit_precedence_target( @@ -125,7 +105,7 @@ def selector_explicit_precedence_target( topt_go_payloads_selector( name = name, explicit_importpath = "example.com/explicit/pkg", - embeds = [":embed_wrapper"], + embeds = [":explicit_embed_library"], fallback_importpath = "example.com/fallback/pkg", full_files = ":full_payload", module_group_names = module_group_names or [], @@ -138,7 +118,7 @@ def selector_embed_precedence_target(name, tags = None): """embed-derived importpath wins when explicit_importpath is unset.""" topt_go_payloads_selector( name = name, - embeds = [":embed_wrapper"], + embeds = [":explicit_embed_library"], fallback_importpath = "example.com/fallback/pkg", full_files = ":full_payload", module_groups = _COMMON_MODULE_GROUPS, @@ -146,11 +126,11 @@ def selector_embed_precedence_target(name, tags = None): tags = tags, ) -def selector_deps_precedence_target(name, tags = None): - """deps-traversal importpath is used when embed chain has deps-only path.""" +def selector_inferred_embed_uses_fallback_target(name, tags = None): + """An inferred embed importpath does not override the go_test fallback.""" topt_go_payloads_selector( name = name, - embeds = [":deps_wrapper"], + embeds = [":inferred_embed_library"], fallback_importpath = "example.com/fallback/pkg", full_files = ":full_payload", module_groups = _COMMON_MODULE_GROUPS, @@ -200,7 +180,7 @@ def selector_include_disabled_target(name, tags = None): topt_go_payloads_selector( name = name, explicit_importpath = "example.com/explicit/pkg", - embeds = [":embed_wrapper"], + embeds = [":explicit_embed_library"], fallback_importpath = "example.com/fallback/pkg", full_files = ":full_payload", module_groups = _COMMON_MODULE_GROUPS, @@ -319,11 +299,11 @@ def _selector_embed_precedence_test_impl(ctx): _assert_selected(env, target, "module_example_com_embed_pkg") return analysistest.end(env) -def _selector_deps_precedence_test_impl(ctx): - """Implement selector deps precedence test impl behavior.""" +def _selector_inferred_embed_uses_fallback_test_impl(ctx): + """Assert rules_go-style inferred embeds use the target fallback.""" env = analysistest.begin(ctx) target = analysistest.target_under_test(env) - _assert_selected(env, target, "module_example_com_deps_pkg") + _assert_selected(env, target, "module_example_com_fallback_pkg") return analysistest.end(env) def _selector_fallback_test_impl(ctx): @@ -397,8 +377,8 @@ selector_explicit_namespaced_test = analysistest.make( selector_embed_precedence_test = analysistest.make( _selector_embed_precedence_test_impl, ) -selector_deps_precedence_test = analysistest.make( - _selector_deps_precedence_test_impl, +selector_inferred_embed_uses_fallback_test = analysistest.make( + _selector_inferred_embed_uses_fallback_test_impl, ) selector_fallback_test = analysistest.make( _selector_fallback_test_impl, diff --git a/modules/go/topt_go_infer.bzl b/modules/go/topt_go_infer.bzl index 42407336..7213aaa1 100644 --- a/modules/go/topt_go_infer.bzl +++ b/modules/go/topt_go_infer.bzl @@ -7,8 +7,8 @@ """Starlark helpers to infer Go importpath via rules_go providers. This file provides: -- An aspect that walks the `embed` attribute to find a go_library's - computed importpath (matching rules_go's logic). +- An aspect that reads an embedded go_library's explicit importpath + (matching rules_go's logic). - A rule that uses that aspect result to choose the correct per-module filegroup from the synced repository and expose those files as runfiles. @@ -132,14 +132,17 @@ def _importpath_aspect_impl(target, ctx): - If this target provides GoInfo (formerly GoLibrary), read its importpath. - Else, if this target provides GoArchive, read its importpath. - Else, if rule has an explicit importpath attribute, use it. - - Else, traverse children via `embed` and propagate first discovered value. + - Else, report no explicit embed importpath so the caller can use the + label-derived go_test fallback. """ - # Prefer GoInfo provider (rules_go v0.51+, replaces GoLibrary) + # rules_go only inherits an embedded library's importpath when that path + # was explicit. Inferred embed paths are ignored and the go_test label is + # used instead. if GoInfo in target: info = target[GoInfo] ip = getattr(info, "importpath", None) - if type(ip) == type("") and ip: + if getattr(info, "pathtype", None) == "explicit" and type(ip) == type("") and ip: return [ToptGoImportpathInfo(importpath = ip)] # Fallback: GoArchive may carry importpath @@ -152,7 +155,7 @@ def _importpath_aspect_impl(target, ctx): ip = getattr(arch.source, "importpath", None) if (not ip) and hasattr(arch, "library"): ip = getattr(arch.library, "importpath", None) - if type(ip) == type("") and ip: + if getattr(arch, "pathtype", None) == "explicit" and type(ip) == type("") and ip: return [ToptGoImportpathInfo(importpath = ip)] # Explicit attribute on some go_* rules @@ -161,21 +164,11 @@ def _importpath_aspect_impl(target, ctx): if type(ip) == type("") and ip: return [ToptGoImportpathInfo(importpath = ip)] - # Propagate from transitive deps. - # Returning the first non-empty importpath preserves deterministic behavior. - for attr_name in ["embed", "deps"]: - for dep in getattr(ctx.rule.attr, attr_name, []): - if ToptGoImportpathInfo in dep: - ip = dep[ToptGoImportpathInfo].importpath - if type(ip) == type("") and ip: - return [ToptGoImportpathInfo(importpath = ip)] - # No information found at this node return [] _importpath_aspect = aspect( implementation = _importpath_aspect_impl, - attr_aspects = ["embed", "deps"], ) def _topt_go_payloads_selector_impl(ctx): diff --git a/modules/go/topt_go_test.bzl b/modules/go/topt_go_test.bzl index e4f5aea9..f5d34428 100644 --- a/modules/go/topt_go_test.bzl +++ b/modules/go/topt_go_test.bzl @@ -18,9 +18,9 @@ Notes: --remote_download_regex=.*test[.]outputs.*, and --zip_undeclared_test_outputs are set, and run doctor/uploader with BEP freshness plus artifact staging. -- Import path inference mirrors rules_go behavior by walking `embed` via - an aspect and reading the GoArchive provider; when unavailable, falls - back to go_module_path + Bazel package path. +- Import path inference mirrors rules_go behavior by reading an explicit + `embed` importpath via an aspect; when unavailable, it falls back to the + hidden raw go_test label's inferred importpath. - Create ONE uploader target per workspace (see dd_payload_uploader in test_optimization_uploader.bzl) and run it via `bazel run` after tests. @@ -245,6 +245,19 @@ def _label_dedupe_key(value): return label +def _rules_go_inferred_importpath(package_name, target_name): + """Mirror rules_go's label-based fallback importpath calculation.""" + importpath = package_name + if not importpath.endswith(target_name): + importpath += "/" + target_name + + vendor_prefix = "/vendor/" + if importpath.rfind(vendor_prefix) != -1: + importpath = importpath[len(vendor_prefix) + importpath.rfind(vendor_prefix):] + if importpath.startswith("/"): + importpath = importpath[1:] + return importpath + def dd_topt_go_test( name, # Required: pass the exported `modules` dict from @//:export.bzl @@ -287,7 +300,7 @@ def dd_topt_go_test( name: Test target name. topt_data: Either the single-service dict exported by @//:export.bzl, or the aggregator mapping (topt_data_by_service) exported by the multi-service repo. - Used to derive the repo alias, go_module_path, and whether to include per-module files. + Used to derive the repo alias and whether to include per-module files. go_test_rule: Optional override for the underlying rules_go go_test rule. Defaults to `go_test` from `@rules_go//go:def.bzl`. This hook is for tests and low-level experiments; repository policy wrappers should stay @@ -454,23 +467,12 @@ def dd_topt_go_test( macro_name = "dd_topt_go_test", ) - # Fallback importpath when providers are unavailable: go_module_path + Bazel package + # Mirror rules_go's fallback for the actual hidden go_test label. rules_go + # only inherits an embed importpath when that path is explicit; otherwise + # it derives the importpath from the go_test package and target name. pkg_path = native.package_name() - fallback_importpath = None - if explicit_importpath: - # Explicit importpath is authoritative and mirrors rules_go semantics. - fallback_importpath = explicit_importpath - else: - _mp = (_go.get("module_path") if _is_dict(_go) else None) or None - if _mp: - # Build module-relative fallback importpath: "/". - # Trim trailing slash to avoid accidental double separators. - base = _mp[:-1] if _mp.endswith("/") else _mp - fallback_importpath = (base + "/" + pkg_path) if pkg_path else base - else: - # Last resort: package path only, still sufficient for best-effort - # module label matching when repo metadata is incomplete. - fallback_importpath = pkg_path + raw_name = name + "__raw_go_test" + fallback_importpath = explicit_importpath or _rules_go_inferred_importpath(pkg_path, raw_name) selector_name = name + "_topt_payloads" metadata_name = name + "_topt_bazel_metadata" @@ -602,7 +604,6 @@ def dd_topt_go_test( _TEST_BINARY_LINKER_OPTIMIZATION_GC_LINKOPTS, ) - raw_name = name + "__raw_go_test" user_tags = wrapper_kwargs.get("tags") kwargs["tags"] = (user_tags or []) + ["manual"] kwargs["visibility"] = ["//visibility:private"] From 3f4cb7478caa1a0e52b92facf2b267d28445f924 Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Tue, 25 Aug 2026 20:05:49 +0200 Subject: [PATCH 13/37] test: make importpath assertion module-relative --- modules/go/tests/test_macro.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/go/tests/test_macro.bzl b/modules/go/tests/test_macro.bzl index ecc1c0a7..a35352a1 100644 --- a/modules/go/tests/test_macro.bzl +++ b/modules/go/tests/test_macro.bzl @@ -556,7 +556,7 @@ def _go_macro_inferred_importpath_metadata_test_impl(ctx): metadata = target[ToptGoBazelMetadataInfo].metadata asserts.equals( env, - "modules/go/tests/go_macro_single_service_target__raw_go_test", + target.label.package + "/go_macro_single_service_target__raw_go_test", metadata["bazel.go.importpath"], ) asserts.equals(env, "fallback", metadata["bazel.go.importpath_source"]) From 99974aee531f8f01000e35417287da9d745db397 Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Tue, 25 Aug 2026 20:56:42 +0200 Subject: [PATCH 14/37] runfiles: preserve external repository aliases --- modules/dotnet/topt_dotnet_infer.bzl | 6 +++- modules/go/topt_go_infer.bzl | 6 +++- modules/java/topt_java_infer.bzl | 6 +++- modules/nodejs/topt_nodejs_infer.bzl | 6 +++- modules/python/tests/BUILD.bazel | 13 ++++++++ .../python/tests/test_payloads_selector.bzl | 32 +++++++++++++++++++ modules/python/topt_py_infer.bzl | 6 +++- modules/ruby/topt_ruby_infer.bzl | 6 +++- tools/core/topt_selection_utils.bzl | 18 ++++++++--- 9 files changed, 88 insertions(+), 11 deletions(-) diff --git a/modules/dotnet/topt_dotnet_infer.bzl b/modules/dotnet/topt_dotnet_infer.bzl index da8fb1c7..ef33f494 100644 --- a/modules/dotnet/topt_dotnet_infer.bzl +++ b/modules/dotnet/topt_dotnet_infer.bzl @@ -144,7 +144,11 @@ def _topt_dotnet_payloads_selector_impl(ctx): ) return [DefaultInfo( files = depset(payload.files), - runfiles = ctx.runfiles(files = payload.files, symlinks = payload.symlinks), + runfiles = ctx.runfiles( + files = payload.files, + root_symlinks = payload.root_symlinks, + symlinks = payload.symlinks, + ), )] topt_dotnet_payloads_selector = rule( diff --git a/modules/go/topt_go_infer.bzl b/modules/go/topt_go_infer.bzl index 7213aaa1..8a918d5b 100644 --- a/modules/go/topt_go_infer.bzl +++ b/modules/go/topt_go_infer.bzl @@ -196,7 +196,11 @@ def _topt_go_payloads_selector_impl(ctx): ) return [DefaultInfo( files = depset(payload.files), - runfiles = ctx.runfiles(files = payload.files, symlinks = payload.symlinks), + runfiles = ctx.runfiles( + files = payload.files, + root_symlinks = payload.root_symlinks, + symlinks = payload.symlinks, + ), )] def _topt_go_bazel_metadata_impl(ctx): diff --git a/modules/java/topt_java_infer.bzl b/modules/java/topt_java_infer.bzl index faed69f7..0679b7f3 100644 --- a/modules/java/topt_java_infer.bzl +++ b/modules/java/topt_java_infer.bzl @@ -160,7 +160,11 @@ def _topt_java_payloads_selector_impl(ctx): ) return [DefaultInfo( files = depset(payload.files), - runfiles = ctx.runfiles(files = payload.files, symlinks = payload.symlinks), + runfiles = ctx.runfiles( + files = payload.files, + root_symlinks = payload.root_symlinks, + symlinks = payload.symlinks, + ), )] topt_java_payloads_selector = rule( diff --git a/modules/nodejs/topt_nodejs_infer.bzl b/modules/nodejs/topt_nodejs_infer.bzl index 59303016..831184c8 100644 --- a/modules/nodejs/topt_nodejs_infer.bzl +++ b/modules/nodejs/topt_nodejs_infer.bzl @@ -144,7 +144,11 @@ def _topt_nodejs_payloads_selector_impl(ctx): ) return [DefaultInfo( files = depset(payload.files), - runfiles = ctx.runfiles(files = payload.files, symlinks = payload.symlinks), + runfiles = ctx.runfiles( + files = payload.files, + root_symlinks = payload.root_symlinks, + symlinks = payload.symlinks, + ), )] topt_nodejs_payloads_selector = rule( diff --git a/modules/python/tests/BUILD.bazel b/modules/python/tests/BUILD.bazel index 75a79515..6187fb39 100644 --- a/modules/python/tests/BUILD.bazel +++ b/modules/python/tests/BUILD.bazel @@ -76,6 +76,8 @@ load( "selector_explicit_namespaced_test", "selector_explicit_precedence_target", "selector_explicit_precedence_test", + "selector_external_module_runfiles_target", + "selector_external_module_runfiles_test", "selector_fallback_target", "selector_fallback_test", "selector_imports_precedence_target", @@ -193,6 +195,16 @@ selector_omits_flaky_tests_test( target_under_test = ":selector_omits_flaky_tests_target", ) +selector_external_module_runfiles_target( + name = "selector_external_module_runfiles_target", + tags = ["manual"], +) + +selector_external_module_runfiles_test( + name = "selector_external_module_runfiles_test", + target_under_test = ":selector_external_module_runfiles_target", +) + selector_explicit_precedence_target( name = "selector_explicit_precedence_target", tags = ["manual"], @@ -702,6 +714,7 @@ test_suite( ":selector_explicit_miss_failure_test", ":selector_explicit_namespaced_test", ":selector_explicit_precedence_test", + ":selector_external_module_runfiles_test", ":selector_fallback_test", ":selector_imports_precedence_test", ":selector_include_disabled_test", diff --git a/modules/python/tests/test_payloads_selector.bzl b/modules/python/tests/test_payloads_selector.bzl index 61b86ba6..3f34c54d 100644 --- a/modules/python/tests/test_payloads_selector.bzl +++ b/modules/python/tests/test_payloads_selector.bzl @@ -286,6 +286,21 @@ def selector_omits_flaky_tests_target(name, tags = None): tags = tags, ) +def selector_external_module_runfiles_target(name, tags = None): + topt_py_payloads_selector( + name = name, + explicit_identifier = "example/python/pkg", + imports = [], + deps = [], + attribute_candidates = [], + fallback_identifier = "example/python/pkg", + full_files = "@test_optimization_data_python//:test_optimization_files", + module_group_names = ["module_example_python_pkg"], + module_groups = ["@test_optimization_data_python//:module_example_python_pkg"], + include_per_module = True, + tags = tags, + ) + def _has_fragment(items, fragment): for item in items: if fragment in item: @@ -403,6 +418,20 @@ def _selector_omits_flaky_tests_test_impl(ctx): asserts.false(env, _has_suffix(symlink_paths, "/flaky_tests.json"), "unexpected flaky_tests.json symlink: %s" % symlink_paths) return analysistest.end(env) +def _selector_external_module_runfiles_test_impl(ctx): + env = analysistest.begin(ctx) + target = analysistest.target_under_test(env) + symlink_paths = [s.path for s in target[DefaultInfo].default_runfiles.symlinks.to_list()] + root_symlink_paths = [s.path for s in target[DefaultInfo].default_runfiles.root_symlinks.to_list()] + _assert_core_cache_symlinks(env, root_symlink_paths) + asserts.equals(env, [], symlink_paths) + asserts.false( + env, + _has_suffix(root_symlink_paths, "/flaky_tests.json"), + "unexpected flaky_tests.json root symlink: %s" % root_symlink_paths, + ) + return analysistest.end(env) + selector_explicit_precedence_test = analysistest.make( _selector_explicit_precedence_test_impl, ) @@ -444,3 +473,6 @@ selector_override_miss_failure_test = analysistest.make( selector_omits_flaky_tests_test = analysistest.make( _selector_omits_flaky_tests_test_impl, ) +selector_external_module_runfiles_test = analysistest.make( + _selector_external_module_runfiles_test_impl, +) diff --git a/modules/python/topt_py_infer.bzl b/modules/python/topt_py_infer.bzl index f9bf0bd3..96f52f70 100644 --- a/modules/python/topt_py_infer.bzl +++ b/modules/python/topt_py_infer.bzl @@ -161,7 +161,11 @@ def _topt_py_payloads_selector_impl(ctx): ) return [DefaultInfo( files = depset(payload.files), - runfiles = ctx.runfiles(files = payload.files, symlinks = payload.symlinks), + runfiles = ctx.runfiles( + files = payload.files, + root_symlinks = payload.root_symlinks, + symlinks = payload.symlinks, + ), )] topt_py_payloads_selector = rule( diff --git a/modules/ruby/topt_ruby_infer.bzl b/modules/ruby/topt_ruby_infer.bzl index 81bfb383..5707fc52 100644 --- a/modules/ruby/topt_ruby_infer.bzl +++ b/modules/ruby/topt_ruby_infer.bzl @@ -144,7 +144,11 @@ def _topt_ruby_payloads_selector_impl(ctx): ) return [DefaultInfo( files = depset(payload.files), - runfiles = ctx.runfiles(files = payload.files, symlinks = payload.symlinks), + runfiles = ctx.runfiles( + files = payload.files, + root_symlinks = payload.root_symlinks, + symlinks = payload.symlinks, + ), )] topt_ruby_payloads_selector = rule( diff --git a/tools/core/topt_selection_utils.bzl b/tools/core/topt_selection_utils.bzl index 1be830ed..6460cc60 100644 --- a/tools/core/topt_selection_utils.bzl +++ b/tools/core/topt_selection_utils.bzl @@ -71,7 +71,7 @@ def select_module_group_name( return "" def selected_payload_runfiles(files, include_flaky_tests = False): - """Return selected payload files and canonical cache/http symlinks.""" + """Return selected payload files and canonical cache/http runfiles aliases.""" selected_files = [] settings_file = None known_tests_file = None @@ -93,12 +93,13 @@ def selected_payload_runfiles(files, include_flaky_tests = False): flaky_tests_file = file if settings_file == None: - return struct(files = selected_files, symlinks = {}) + return struct(files = selected_files, root_symlinks = {}, symlinks = {}) cache_http_dir = "/".join(settings_file.short_path.split("/")[:-1]) if not cache_http_dir: - return struct(files = selected_files, symlinks = {}) + return struct(files = selected_files, root_symlinks = {}, symlinks = {}) + root_symlinks = {} symlinks = {} def maybe_add(filename, file): @@ -106,10 +107,17 @@ def selected_payload_runfiles(files, include_flaky_tests = False): return canonical_path = cache_http_dir + "/" + filename if file.short_path != canonical_path: - symlinks[canonical_path] = file + if canonical_path.startswith("../"): + root_symlinks[canonical_path[3:]] = file + else: + symlinks[canonical_path] = file maybe_add("known_tests.json", known_tests_file) maybe_add("test_management.json", test_management_file) if include_flaky_tests: maybe_add("flaky_tests.json", flaky_tests_file) - return struct(files = selected_files, symlinks = symlinks) + return struct( + files = selected_files, + root_symlinks = root_symlinks, + symlinks = symlinks, + ) From 15c9687646fe90e41306027f0a896585122e5aa2 Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Tue, 25 Aug 2026 21:18:52 +0200 Subject: [PATCH 15/37] python: materialize selected metadata runfiles --- modules/python/tests/BUILD.bazel | 21 +++++++++ ..._external_module_physical_runfiles_test.py | 31 ++++++++++++ modules/python/tests/test_macro.bzl | 28 +++++------ .../python/tests/test_payloads_selector.bzl | 30 ++++++------ modules/python/topt_py_infer.bzl | 47 +++++++++++++++++-- modules/python/topt_py_test.bzl | 15 +++--- 6 files changed, 131 insertions(+), 41 deletions(-) create mode 100644 modules/python/tests/selector_external_module_physical_runfiles_test.py diff --git a/modules/python/tests/BUILD.bazel b/modules/python/tests/BUILD.bazel index 6187fb39..2b52a07f 100644 --- a/modules/python/tests/BUILD.bazel +++ b/modules/python/tests/BUILD.bazel @@ -4,6 +4,7 @@ # This product includes software developed at Datadog # (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. +load("@rules_python//python:py_test.bzl", "py_test") load( ":test_macro.bzl", "py_macro_consumer_runner_capture_test", @@ -205,6 +206,25 @@ selector_external_module_runfiles_test( target_under_test = ":selector_external_module_runfiles_target", ) +filegroup( + name = "selector_external_module_manifest", + srcs = [":selector_external_module_runfiles_target"], + output_group = "selected_manifest", +) + +py_test( + name = "selector_external_module_physical_runfiles_test", + srcs = ["selector_external_module_physical_runfiles_test.py"], + data = [ + ":selector_external_module_manifest", + ":selector_external_module_runfiles_target", + ], + env = { + "DD_TEST_OPTIMIZATION_MANIFEST_FILE": "$(rlocationpath :selector_external_module_manifest)", + }, + deps = ["@rules_python//python/runfiles"], +) + selector_explicit_precedence_target( name = "selector_explicit_precedence_target", tags = ["manual"], @@ -714,6 +734,7 @@ test_suite( ":selector_explicit_miss_failure_test", ":selector_explicit_namespaced_test", ":selector_explicit_precedence_test", + ":selector_external_module_physical_runfiles_test", ":selector_external_module_runfiles_test", ":selector_fallback_test", ":selector_imports_precedence_test", diff --git a/modules/python/tests/selector_external_module_physical_runfiles_test.py b/modules/python/tests/selector_external_module_physical_runfiles_test.py new file mode 100644 index 00000000..64fed708 --- /dev/null +++ b/modules/python/tests/selector_external_module_physical_runfiles_test.py @@ -0,0 +1,31 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +import os +import unittest +from pathlib import Path + +from python.runfiles import runfiles + + +class PhysicalRunfilesTest(unittest.TestCase): + def test_selected_metadata_is_materialized_beside_manifest(self) -> None: + runfiles_resolver = runfiles.Create() + self.assertIsNotNone(runfiles_resolver) + manifest_location = runfiles_resolver.Rlocation( + os.environ["DD_TEST_OPTIMIZATION_MANIFEST_FILE"], + ) + self.assertIsNotNone(manifest_location) + manifest = Path(manifest_location) + known_tests = manifest.parent / "cache" / "http" / "known_tests.json" + + self.assertTrue(manifest.is_file(), manifest) + self.assertTrue(known_tests.is_file(), known_tests) + self.assertIn("module:example_python_pkg", known_tests.read_text()) + + +if __name__ == "__main__": + unittest.main() diff --git a/modules/python/tests/test_macro.bzl b/modules/python/tests/test_macro.bzl index d31eb3bc..9838a274 100644 --- a/modules/python/tests/test_macro.bzl +++ b/modules/python/tests/test_macro.bzl @@ -616,7 +616,7 @@ def _py_macro_consumer_runner_wiring_test_impl(ctx): # Test Optimization wiring is present. asserts.true(env, _has_label_suffix(captured.data_labels, ":py_macro_consumer_runner_target_topt_payloads")) - asserts.true(env, _has_label_suffix(captured.data_labels, ":.testoptimization/manifest.txt")) + asserts.true(env, _has_label_suffix(captured.data_labels, ":py_macro_consumer_runner_target_topt_payloads_manifest")) manifest_env = captured.env.get("DD_TEST_OPTIMIZATION_MANIFEST_FILE") asserts.true(env, manifest_env != None) asserts.true(env, "rlocationpath" in manifest_env) @@ -653,7 +653,7 @@ def _py_macro_consumer_runner_capture_test_impl(ctx): # Data has selector and manifest. asserts.true(env, _has_label_suffix(captured.data_labels, ":py_macro_consumer_runner_with_capture_target_topt_payloads")) - asserts.true(env, _has_label_suffix(captured.data_labels, ":.testoptimization/manifest.txt")) + asserts.true(env, _has_label_suffix(captured.data_labels, ":py_macro_consumer_runner_with_capture_target_topt_payloads_manifest")) # imports is empty — not synthesized from package path. asserts.equals(env, [], captured.imports) @@ -696,7 +696,7 @@ def _py_macro_consumer_runner_kwargs_test_impl(ctx): asserts.equals(env, ["pytest", "ddtrace"], captured.dd_requirements) asserts.true(env, _has_label_suffix(captured.data_labels, ":test_macro.bzl")) asserts.true(env, _has_label_suffix(captured.data_labels, ":py_macro_consumer_runner_kwargs_target_topt_payloads")) - asserts.true(env, _has_label_suffix(captured.data_labels, ":.testoptimization/manifest.txt")) + asserts.true(env, _has_label_suffix(captured.data_labels, ":py_macro_consumer_runner_kwargs_target_topt_payloads_manifest")) asserts.equals(env, "--ddtrace", captured.env.get("PYTEST_ADDOPTS")) asserts.true(env, "manual" in captured.tags) asserts.true(env, "consumer_tag" in captured.tags) @@ -867,15 +867,13 @@ def _py_macro_single_service_wiring_test_impl(ctx): captured = target[ToptPyMacroCaptureInfo] asserts.true(env, _has_label_suffix(captured.data_labels, ":py_macro_single_service_target_topt_payloads")) + asserts.true(env, _has_label_suffix(captured.data_labels, ":py_macro_single_service_target_topt_payloads_manifest")) asserts.true(env, _has_label_suffix(captured.data_labels, ":test_macro.bzl")) - asserts.true(env, _has_fragment(captured.data_labels, "test_optimization_data")) - asserts.true(env, _has_label_suffix(captured.data_labels, ":.testoptimization/manifest.txt")) manifest_env = captured.env.get("DD_TEST_OPTIMIZATION_MANIFEST_FILE") asserts.true(env, manifest_env != None) asserts.true(env, "rlocationpath" in manifest_env) - asserts.true(env, "test_optimization_data" in manifest_env) - asserts.true(env, ".testoptimization/manifest.txt" in manifest_env) + asserts.true(env, "py_macro_single_service_target_topt_payloads_manifest" in manifest_env) asserts.equals( env, "py_macro_single_service_target_topt_bazel_metadata.json", @@ -893,8 +891,7 @@ def _py_macro_multi_service_wiring_test_impl(ctx): captured = target[ToptPyMacroCaptureInfo] asserts.true(env, _has_label_suffix(captured.data_labels, ":py_macro_multi_service_target_topt_payloads")) - asserts.true(env, _has_fragment(captured.data_labels, "test_optimization_data")) - asserts.true(env, _has_label_suffix(captured.data_labels, ":.testoptimization/manifest.txt")) + asserts.true(env, _has_label_suffix(captured.data_labels, ":py_macro_multi_service_target_topt_payloads_manifest")) asserts.equals( env, "py_macro_multi_service_target_topt_bazel_metadata.json", @@ -909,7 +906,7 @@ def _py_macro_dynamic_manifest_wiring_test_impl(ctx): env = analysistest.begin(ctx) captured = analysistest.target_under_test(env)[ToptPyMacroCaptureInfo] asserts.true(env, _has_label_suffix(captured.data_labels, ":py_macro_dynamic_manifest_target_topt_payloads")) - asserts.true(env, _has_label_suffix(captured.data_labels, ":test_macro.bzl")) + asserts.true(env, _has_label_suffix(captured.data_labels, ":py_macro_dynamic_manifest_target_topt_payloads_manifest")) asserts.false(env, _has_fragment(captured.data_labels, "virtual_dynamic_repo_that_must_not_resolve")) asserts.equals(env, "dynamic-python-service", captured.env.get("DD_SERVICE")) return analysistest.end(env) @@ -1001,14 +998,15 @@ def _py_macro_public_wrapper_test_impl(ctx): def _py_macro_fallback_payloads_test_impl(ctx): env = analysistest.begin(ctx) - target = analysistest.target_under_test(env) - paths = [file.short_path for file in target[DefaultInfo].files.to_list()] + input_paths = [] + for action in analysistest.target_actions(env): + input_paths.extend([file.short_path for file in action.inputs.to_list()]) package_suffix = ctx.label.package.replace("/", "_") expected_module_name = "module_example_python_%s" % package_suffix expected_module_known_tests = False full_bundle_known_tests = False - for path in paths: + for path in input_paths: if path.endswith("/.testoptimization/%s/known_tests.json" % expected_module_name): expected_module_known_tests = True if path.endswith("/.testoptimization/cache/http/known_tests.json"): @@ -1017,12 +1015,12 @@ def _py_macro_fallback_payloads_test_impl(ctx): asserts.true( env, expected_module_known_tests, - msg = "macro-generated selector must expose %s: %s" % (expected_module_name, paths), + msg = "macro-generated selector must select %s: %s" % (expected_module_name, input_paths), ) asserts.false( env, full_bundle_known_tests, - msg = "macro-generated selector must not fall back to the full known-tests bundle: %s" % paths, + msg = "macro-generated selector must not read the full known-tests bundle: %s" % input_paths, ) return analysistest.end(env) diff --git a/modules/python/tests/test_payloads_selector.bzl b/modules/python/tests/test_payloads_selector.bzl index 3f34c54d..344c1dd9 100644 --- a/modules/python/tests/test_payloads_selector.bzl +++ b/modules/python/tests/test_payloads_selector.bzl @@ -313,16 +313,16 @@ def _has_suffix(items, suffix): return True return False -def _assert_core_cache_symlinks(env, symlink_paths): +def _assert_core_cache_paths(env, paths): asserts.true( env, - _has_fragment(symlink_paths, "/.testoptimization/cache/http/known_tests.json"), - "expected canonical known_tests.json symlink in paths: %s" % symlink_paths, + _has_fragment(paths, "/.testoptimization/cache/http/known_tests.json"), + "expected canonical known_tests.json in paths: %s" % paths, ) asserts.true( env, - _has_fragment(symlink_paths, "/.testoptimization/cache/http/test_management.json"), - "expected canonical test_management.json symlink in paths: %s" % symlink_paths, + _has_fragment(paths, "/.testoptimization/cache/http/test_management.json"), + "expected canonical test_management.json in paths: %s" % paths, ) def _assert_selected(env, target, expected_fragment): @@ -409,26 +409,26 @@ def _selector_override_miss_failure_test_impl(ctx): def _selector_omits_flaky_tests_test_impl(ctx): env = analysistest.begin(ctx) target = analysistest.target_under_test(env) - files = [f.basename for f in target[DefaultInfo].files.to_list()] - runfiles = [f.basename for f in target[DefaultInfo].default_runfiles.files.to_list()] - symlink_paths = [s.path for s in target[DefaultInfo].default_runfiles.symlinks.to_list()] - _assert_core_cache_symlinks(env, symlink_paths) - asserts.false(env, _has_fragment(files, "flaky_tests.json"), "unexpected flaky_tests.json in files: %s" % files) - asserts.false(env, _has_fragment(runfiles, "flaky_tests.json"), "unexpected flaky_tests.json in runfiles: %s" % runfiles) - asserts.false(env, _has_suffix(symlink_paths, "/flaky_tests.json"), "unexpected flaky_tests.json symlink: %s" % symlink_paths) + file_paths = [f.short_path for f in target[DefaultInfo].files.to_list()] + runfile_paths = [f.short_path for f in target[DefaultInfo].default_runfiles.files.to_list()] + _assert_core_cache_paths(env, file_paths) + asserts.false(env, _has_fragment(file_paths, "flaky_tests.json"), "unexpected flaky_tests.json in files: %s" % file_paths) + asserts.false(env, _has_fragment(runfile_paths, "flaky_tests.json"), "unexpected flaky_tests.json in runfiles: %s" % runfile_paths) return analysistest.end(env) def _selector_external_module_runfiles_test_impl(ctx): env = analysistest.begin(ctx) target = analysistest.target_under_test(env) + file_paths = [f.short_path for f in target[DefaultInfo].files.to_list()] symlink_paths = [s.path for s in target[DefaultInfo].default_runfiles.symlinks.to_list()] root_symlink_paths = [s.path for s in target[DefaultInfo].default_runfiles.root_symlinks.to_list()] - _assert_core_cache_symlinks(env, root_symlink_paths) + _assert_core_cache_paths(env, file_paths) asserts.equals(env, [], symlink_paths) + asserts.equals(env, [], root_symlink_paths) asserts.false( env, - _has_suffix(root_symlink_paths, "/flaky_tests.json"), - "unexpected flaky_tests.json root symlink: %s" % root_symlink_paths, + _has_suffix(file_paths, "/flaky_tests.json"), + "unexpected flaky_tests.json in files: %s" % file_paths, ) return analysistest.end(env) diff --git a/modules/python/topt_py_infer.bzl b/modules/python/topt_py_infer.bzl index 96f52f70..ff285a6a 100644 --- a/modules/python/topt_py_infer.bzl +++ b/modules/python/topt_py_infer.bzl @@ -93,6 +93,39 @@ def _select_from_candidates(candidates, module_group_names, include_per_module, return selected_name return "" +def _materialize_selected_payloads(ctx, files): + """Materialize the selected metadata under one physical manifest root.""" + sources_by_basename = {file.basename: file for file in files} + manifest_source = sources_by_basename.get("manifest.txt") + if manifest_source == None: + return struct(files = files, manifest = None) + + output_paths = { + "manifest.txt": ".testoptimization/manifest.txt", + "settings.json": ".testoptimization/cache/http/settings.json", + "known_tests.json": ".testoptimization/cache/http/known_tests.json", + "test_management.json": ".testoptimization/cache/http/test_management.json", + "flaky_tests.json": ".testoptimization/cache/http/flaky_tests.json", + } + outputs = [] + replaced_paths = {} + manifest = None + for basename, relative_path in output_paths.items(): + source = sources_by_basename.get(basename) + if source == None: + continue + output = ctx.actions.declare_file(ctx.label.name + "/" + relative_path) + ctx.actions.symlink(output = output, target_file = source) + outputs.append(output) + replaced_paths[source.path] = True + if basename == "manifest.txt": + manifest = output + + return struct( + files = outputs + [file for file in files if not replaced_paths.get(file.path)], + manifest = manifest, + ) + def _topt_py_payloads_selector_impl(ctx): module_group_names = ctx.attr.module_group_names if module_group_names: @@ -159,14 +192,18 @@ def _topt_py_payloads_selector_impl(ctx): src_default.files.to_list(), include_flaky_tests = False, ) - return [DefaultInfo( - files = depset(payload.files), + materialized = _materialize_selected_payloads(ctx, payload.files) + providers = [DefaultInfo( + files = depset(materialized.files), runfiles = ctx.runfiles( - files = payload.files, - root_symlinks = payload.root_symlinks, - symlinks = payload.symlinks, + files = materialized.files, ), )] + if materialized.manifest != None: + providers.append(OutputGroupInfo( + selected_manifest = depset([materialized.manifest]), + )) + return providers topt_py_payloads_selector = rule( implementation = _topt_py_payloads_selector_impl, diff --git a/modules/python/topt_py_test.bzl b/modules/python/topt_py_test.bzl index 6d414387..73822f2f 100644 --- a/modules/python/topt_py_test.bzl +++ b/modules/python/topt_py_test.bzl @@ -23,7 +23,6 @@ load( "merge_user_env", "normalize_user_data", "resolve_files_label", - "resolve_manifest_label", "resolve_module_group_names", "resolve_module_labels", "resolve_topt_service_key", @@ -293,6 +292,7 @@ def dd_topt_py_test( ) selector_name = name + "_topt_payloads" + selector_manifest_name = selector_name + "_manifest" metadata_name = name + "_topt_bazel_metadata" topt_py_payloads_selector( name = selector_name, @@ -309,6 +309,11 @@ def dd_topt_py_test( importpath = importpath_candidate if importpath_candidate != None else "", module_path = module_path_candidate if module_path_candidate != None else "", ) + native.filegroup( + name = selector_manifest_name, + srcs = [":" + selector_name], + output_group = "selected_manifest", + ) pkg_path = native.package_name() topt_bazel_metadata( @@ -345,14 +350,12 @@ def dd_topt_py_test( user_env = dict(user_env) user_env["PYTEST_ADDOPTS"] = _existing_pytest_addopts + " --ddtrace" - data = _append_data_dependencies(data, [":" + selector_name]) - - manifest_label = resolve_manifest_label(_svc, sync_repo_name, macro_name = "dd_topt_py_test") - data = _append_data_dependencies(data, [manifest_label]) + selector_manifest_label = ":" + selector_manifest_name + data = _append_data_dependencies(data, [":" + selector_name, selector_manifest_label]) env = _merge_user_env( user_env, { - "DD_TEST_OPTIMIZATION_MANIFEST_FILE": "$(rlocationpath %s)" % manifest_label, + "DD_TEST_OPTIMIZATION_MANIFEST_FILE": "$(rlocationpath %s)" % selector_manifest_label, "DD_TEST_OPTIMIZATION_PAYLOADS_IN_FILES": "true", "DD_TEST_OPTIMIZATION_BAZEL_TARGET_METADATA_BASENAME": metadata_name + ".json", }, From e969a8b727bec6fef061e28863279c8f2b0d87d3 Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Tue, 25 Aug 2026 21:29:45 +0200 Subject: [PATCH 16/37] tests: share Python selector stub repository --- modules/python/tests/test_payloads_selector.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/python/tests/test_payloads_selector.bzl b/modules/python/tests/test_payloads_selector.bzl index 344c1dd9..e9d464d3 100644 --- a/modules/python/tests/test_payloads_selector.bzl +++ b/modules/python/tests/test_payloads_selector.bzl @@ -294,9 +294,9 @@ def selector_external_module_runfiles_target(name, tags = None): deps = [], attribute_candidates = [], fallback_identifier = "example/python/pkg", - full_files = "@test_optimization_data_python//:test_optimization_files", + full_files = "@test_optimization_data//:test_optimization_files", module_group_names = ["module_example_python_pkg"], - module_groups = ["@test_optimization_data_python//:module_example_python_pkg"], + module_groups = ["@test_optimization_data//:module_example_python_pkg"], include_per_module = True, tags = tags, ) From 15539c5e6e0e7ab5cd9ef29db3b8f4ff5d670c5f Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Wed, 26 Aug 2026 08:49:00 +0200 Subject: [PATCH 17/37] uploader: split and validate outbound test payloads --- README.md | 40 ++- docs/Configuration_Reference.md | 2 +- docs/Uploader_Reference.md | 35 +- tools/core/uploader_bash_runtime.sh.tpl | 234 ++++++++++-- .../core/uploader_powershell_runtime.ps1.tpl | 183 ++++++++-- tools/tests/python/test_python_tools.py | 340 +++++++++++++++++- 6 files changed, 754 insertions(+), 80 deletions(-) diff --git a/README.md b/README.md index e6d7bf7a..61f61dda 100644 --- a/README.md +++ b/README.md @@ -1306,7 +1306,10 @@ For a generic wrapper pattern, see [Other languages (without companion macro)](# - **Windows**: `powershell.exe` (Windows PowerShell 5.1+ or PowerShell 7+); the uploader uses .NET `HttpClient` and is intentionally PowerShell-only (no Git Bash dependency) Optional tooling: -- **jq** (Linux/macOS) - Used to enrich test payloads with `context.json`. If missing, uploads proceed without enrichment. +- **jq** (Linux/macOS) - Used to enrich test payloads with `context.json` and + split oversized test payloads. If missing, uploads proceed without enrichment; + payloads up to the 5,000,000-byte intake limit can still be sent intact, but + larger payloads fail because their `events` array cannot be partitioned. - **python3** - Used for uploader payload schema validation and Unix telemetry metadata extraction. If missing, schema validation is skipped and telemetry files fail individually with a warning. ### Contract gate checklist @@ -1411,6 +1414,15 @@ The uploader is a normal Bazel rule (not a test) that runs via `bazel run` after 4. Then run the uploader via `bazel run` 5. The uploader discovers all `test.outputs/` directories, waits for quiescence, uploads, and deletes files +Before transport, the uploader enriches each test payload and splits any body +larger than 4,500,000 bytes along its `events` array. Parts preserve event order +and the original top-level envelope, are compressed independently when gzip is +enabled, and use independent retries. The original payload is deleted only +after every part uploads successfully. A terminal upload failure reports the +HTTP status, a bounded response body, and uncompressed/compressed/transmitted +sizes without requiring debug mode. A single event above the 5,000,000-byte +intake limit is rejected because it cannot be split safely. + Telemetry-specific notes: - Telemetry files must contain one raw top-level tracer telemetry request body per file. - Telemetry uploads are reconstructed from the raw body plus the uploader mode. @@ -1658,17 +1670,30 @@ pre-existing optional schema and telemetry helpers. Bash BEP freshness parsing still requires `jq` whenever BEP freshness validation is enabled in the Bash uploader path. -### Enrichment dry-run +### Enrichment validation Payload files written by tests are intentionally raw Bazel outputs. They do not need to contain every Git, Bazel, CODEOWNERS, or runtime tag before upload. The uploader enriches test payloads at `bazel run` time using `context.json` plus `bazel_target_metadata.json`. -Use dry-run mode when you want to prove that the final outbound body would be -properly enriched without uploading data or deleting local payload files. When -invoking the uploader manually, use the same `$bep_json` path generated by the -matching `bazel test --build_event_json_file=...` invocation: +Pass `--validate-enrichment` during a normal upload to validate the exact +enriched body before it is split and sent. This enriches and processes each +source payload once; a validation failure skips that source payload while the +uploader continues processing other fresh payloads: + +```bash +DD_API_KEY="$DD_API_KEY" DD_SITE="$DD_SITE" \ +bazel run --config=test-optimization //:dd_upload_payloads -- \ + --bep-json="$bep_json" \ + --freshness-source=bep \ + --freshness-mode=required \ + --validate-enrichment +``` + +Add `--dry-run` when you want the same validation without uploading data or +deleting local payload files. In both modes, use the same `$bep_json` path +generated by the matching `bazel test --build_event_json_file=...` invocation: ```bash bazel run --config=test-optimization //:dd_upload_payloads -- \ @@ -1679,7 +1704,7 @@ bazel run --config=test-optimization //:dd_upload_payloads -- \ --validate-enrichment ``` -By default this validates that the enriched test payload has +By default `--validate-enrichment` requires the enriched test payload to have `git.repository_url`, `git.commit.sha`, `bazel.target`, and `bazel.package`. Add repeatable `--expected-enriched-tag=` arguments when a repository needs extra tags validated before upload. Go rollouts that need to prove @@ -1692,6 +1717,7 @@ For complete uploader details, use [`docs/Uploader_Reference.md`](docs/Uploader_ - uploader target attributes and optional environment variables - agentless vs EVP credential modes and endpoint behavior - retry/reliability semantics and exit codes +- automatic test-payload splitting and failure-size diagnostics - metadata enrichment (`context.json`, CODEOWNERS) and schema validation ## Convenience macro: dd_topt_go_test diff --git a/docs/Configuration_Reference.md b/docs/Configuration_Reference.md index 37f9a19a..8d903cb0 100644 --- a/docs/Configuration_Reference.md +++ b/docs/Configuration_Reference.md @@ -642,7 +642,7 @@ Uploader CLI flags: | Flag | Purpose | |------|---------| | `--dry-run` | Enrich and validate discovered payloads without uploading or deleting files | -| `--validate-enrichment` | In dry-run mode, require key Git and Bazel tags to exist after enrichment | +| `--validate-enrichment` | Require key Git and Bazel tags to exist after enrichment, before either validation-only completion or upload | | `--expected-enriched-tag=` | Add a required enriched tag; repeatable. Defaults cover `git.repository_url`, `git.commit.sha`, `bazel.target`, and `bazel.package`. Add `bazel.go.payload_selection` explicitly when a Go rollout must prove per-module selection | | `--bep-json=` | BEP JSON file from the matching Bazel test invocation; repeat for multiple invocations | | `--freshness-source=` | Freshness source: `auto`, `bep`, or `execution_log` | diff --git a/docs/Uploader_Reference.md b/docs/Uploader_Reference.md index 8769f18f..c1031f02 100644 --- a/docs/Uploader_Reference.md +++ b/docs/Uploader_Reference.md @@ -110,17 +110,21 @@ Always preserve all statuses. Test failures win, followed by doctor, dry-run, and uploader failures. When upload is enabled, validation failures do not block the uploader from processing other fresh valid payloads. -Dry-run enrichment validation: +Enrichment validation: - The CI wrappers always run `--dry-run --validate-enrichment` before a real - upload. Manual invocations should pass the matching `--bep-json=` - together with `--freshness-source=bep --freshness-mode=required - --artifact-source=bep`. + upload. A normal uploader invocation can instead pass + `--validate-enrichment` without `--dry-run` to validate each enriched source + payload once, immediately before splitting and upload. Manual invocations + should pass the matching `--bep-json=` together with + `--freshness-source=bep --freshness-mode=required --artifact-source=bep`. - Dry-run mode does not upload data, does not require `DD_API_KEY` in agentless mode, and does not delete payload files. - Raw files under `bazel-testlogs` are not expected to contain every final tag. - The dry-run validates the enriched outbound body after merging `context.json` - and `bazel_target_metadata.json`. + Validation inspects the enriched outbound body after merging `context.json` + and `bazel_target_metadata.json`. During a normal upload, a validation failure + prevents that source payload from being sent but does not suppress other + fresh payloads. - Empty JSON placeholders such as `{}` under `payloads/tests/` are skipped. They have no uploadable `events[]` and are not valid Test Optimization test payloads. @@ -267,7 +271,7 @@ bazel run --config=test-optimization //:dd_upload_payloads |----------|---------|---------| | `DD_TEST_OPTIMIZATION_KEEP_PAYLOADS` | `0` | Set to `1` to retain payloads after successful upload (for debugging/re-upload) | | `DD_TEST_OPTIMIZATION_FILTER_PREFIX` | `0` | `0` uploads all payload files; set to `1` to only upload `span_events_*.json` or `coverage_*.json` | -| `DD_TEST_OPTIMIZATION_DEBUG` | `0` | Set to `1` to enable verbose upload logging (HTTP codes, response bodies, startTime stats, and key runfile/CODEOWNERS resolution hits) | +| `DD_TEST_OPTIMIZATION_DEBUG` | `0` | Set to `1` to enable verbose attempt, success, startTime, and runfile/CODEOWNERS resolution logging. Terminal test-upload failures always report the HTTP status, a bounded response body, and payload sizes. | | `DD_TEST_OPTIMIZATION_GZIP` | `0` | Set to `1` to gzip test payloads before upload (adds `Content-Encoding: gzip`) | | `DD_TEST_OPTIMIZATION_MAX_WAIT_SEC` | `300` | Override max wait time for slow filesystems (NFS, network drives); set to `0` to skip waiting when no payloads are present | | `DD_TEST_OPTIMIZATION_QUIESCENT_SEC` | `10` | Override quiescence wait time | @@ -659,6 +663,23 @@ payload discovery/quiescence before proceeding. attempts - Both transient errors (connection issues) and HTTP errors (4xx/5xx) trigger retries +- After enrichment, test payloads larger than 4,500,000 bytes are split by + their top-level `events` array before compression and transport. Each part + preserves the original top-level envelope and remains at or below the split + target when possible. This leaves headroom below the 5,000,000-byte intake + limit. +- A single event between the split target and the intake limit is sent intact. + A single event above the intake limit cannot be split and fails without an + upload attempt. +- On Unix, splitting requires `jq`. Without it, payloads at or below the intake + limit retain the existing unsplit upload path; larger payloads fail locally. + PowerShell performs the same split with its built-in JSON support. +- Split parts are uploaded in event order and retry independently. A failed + part does not prevent the uploader from attempting the remaining parts. The + source payload is reported as failed and retained unless every part succeeds. +- Terminal test-upload failures always log the HTTP status, up to 2,000 + characters of the response body, and the uncompressed, compressed, and + transmitted byte counts. Response logging does not require debug mode. - Behavior is consistent across Linux/macOS (bash/curl) and Windows (PowerShell-only runtime path; no Git Bash requirement) diff --git a/tools/core/uploader_bash_runtime.sh.tpl b/tools/core/uploader_bash_runtime.sh.tpl index c7e8943e..24a6e220 100644 --- a/tools/core/uploader_bash_runtime.sh.tpl +++ b/tools/core/uploader_bash_runtime.sh.tpl @@ -600,6 +600,9 @@ KEEP_PAYLOADS=$(normalize_bool "${DD_TEST_OPTIMIZATION_KEEP_PAYLOADS:-__DDTPL_KE FILTER_PREFIX=$(normalize_bool "${DD_TEST_OPTIMIZATION_FILTER_PREFIX:-__DDTPL_FILTER_PREFIX__}") DEBUG=$(normalize_bool "${DD_TEST_OPTIMIZATION_DEBUG:-__DDTPL_DEBUG__}") GZIP_PAYLOADS=$(normalize_bool "${DD_TEST_OPTIMIZATION_GZIP:-__DDTPL_GZIP_PAYLOADS__}") +TEST_PAYLOAD_SPLIT_TARGET_BYTES=4500000 +TEST_PAYLOAD_MAX_BYTES=5000000 +UPLOAD_RESPONSE_LOG_BYTES=2000 RULES_VERSION="__DDTPL_RULES_VERSION__" RUNTIME_ID=$(generate_uuid) # Reuse one uploader-local session fallback for telemetry files that do not @@ -678,11 +681,11 @@ DEFAULT_EXPECTED_ENRICHED_TAGS=( print_usage() { cat <<'EOF' -Usage: dd_upload_payloads [--dry-run [--validate-enrichment] [--expected-enriched-tag=TAG ...]] +Usage: dd_upload_payloads [--dry-run] [--validate-enrichment] [--expected-enriched-tag=TAG ...] Options: --dry-run Enrich and validate payloads without uploading or deleting files. - --validate-enrichment In dry-run mode, require key context and Bazel tags after enrichment. + --validate-enrichment Require key context and Bazel tags after enrichment, before upload. --expected-enriched-tag TAG Add one required enriched tag; repeatable. Defaults to git and Bazel tags. --bep-json PATH BEP JSON file from the matching bazel test invocation; repeatable. --freshness-source SOURCE Cache-safety source: auto, bep, execution_log. Default: auto. @@ -889,11 +892,6 @@ if (( FRESHNESS_DISABLED_EXPLICIT == 1 )); then EXECUTION_LOG_MODE="disabled" fi -if (( VALIDATE_ENRICHMENT == 1 && DRY_RUN == 0 )); then - log "error: --validate-enrichment requires --dry-run" - exit 2 -fi - FRESHNESS_MODE="$(echo "$FRESHNESS_MODE" | tr '[:upper:]' '[:lower:]')" FRESHNESS_SOURCE="$(echo "$FRESHNESS_SOURCE" | tr '[:upper:]' '[:lower:]')" ARTIFACT_SOURCE="$(echo "$ARTIFACT_SOURCE" | tr '[:upper:]' '[:lower:]')" @@ -962,6 +960,10 @@ CURL_RETRY_FLAGS=(__DDTPL_CURL_RETRY_FLAGS__) if curl --help all 2>/dev/null | grep -q -- '--retry-all-errors'; then CURL_RETRY_FLAGS+=(--retry-all-errors) fi +CURL_FAIL_FLAG=(-f) +if curl --help all 2>/dev/null | grep -q -- '--fail-with-body'; then + CURL_FAIL_FLAG=(--fail-with-body) +fi dbg "curl retry flags: ${CURL_RETRY_FLAGS[*]}" # Acquire exclusive lock to prevent concurrent uploaders @@ -4756,7 +4758,11 @@ validate_enriched_payload_tags() { log "error: enriched test payload for '$source_file' is missing expected tag(s): ${missing[*]}" return 1 fi - log "dry-run validated enriched test payload: $source_file" + if (( DRY_RUN == 1 )); then + log "dry-run validated enriched test payload: $source_file" + else + log "validated enriched test payload: $source_file" + fi return 0 } @@ -4778,54 +4784,161 @@ dry_run_single_test() { rm -f "$body" 2>/dev/null || true return 1 fi - rm -f "$body" 2>/dev/null || true + if ! prepare_test_payload_parts "$body" "$file"; then + cleanup_prepared_test_payloads + return 1 + fi + if (( ${#PREPARED_TEST_PAYLOADS[@]} > 1 )); then + log "dry-run would split test payload '$file' into ${#PREPARED_TEST_PAYLOADS[@]} parts" + fi + cleanup_prepared_test_payloads return 0 } -# Handle upload single test behavior. -upload_single_test() { - local file="$1" - local body resp payload_file gz http rc - # Use a temp file to avoid collisions when multiple uploads run in parallel. - body="$(mktemp "$TMP_PAYLOAD_DIR/test_payload.XXXXXX" 2>/dev/null || true)" - if [[ -z "$body" ]]; then - dbg "upload_single_test: failed to create temp file" +PREPARED_TEST_PAYLOADS=() +PREPARED_TEST_TEMP_FILES=() + +test_payload_size_bytes() { + wc -c <"$1" | tr -d '[:space:]' +} + +track_prepared_test_temp_file() { + PREPARED_TEST_TEMP_FILES+=("$1") +} + +cleanup_prepared_test_payloads() { + local path + for path in "${PREPARED_TEST_TEMP_FILES[@]}"; do + [[ -n "$path" ]] && rm -f "$path" 2>/dev/null || true + done + PREPARED_TEST_PAYLOADS=() + PREPARED_TEST_TEMP_FILES=() +} + +split_test_payload_part() { + local payload="$1" + local source_file="$2" + local size event_count midpoint left right + size="$(test_payload_size_bytes "$payload")" + if (( size <= TEST_PAYLOAD_SPLIT_TARGET_BYTES )); then + PREPARED_TEST_PAYLOADS+=("$payload") + return 0 + fi + + if ! event_count="$(jq -r '.events | if type == "array" then length else -1 end' "$payload" 2>/dev/null)" || + [[ ! "$event_count" =~ ^[0-9]+$ ]] || (( event_count < 1 )); then + log "error: oversized test payload cannot be split because its events array is invalid: $source_file" return 1 fi - enrich_with_context "$file" "$body" - validate_payload "$body" - build_common_headers "$body" - dbg "upload_single_test: posting '$file' (body '$body')" - if [[ "$DEBUG" == "1" ]]; then - local gzip_note="" - if [[ "$GZIP_PAYLOADS" == "1" ]]; then - gzip_note="; Content-Encoding=gzip" + if (( event_count == 1 )); then + if (( size <= TEST_PAYLOAD_MAX_BYTES )); then + log "warning: single-event test payload exceeds the split target but remains within the intake limit: source='$source_file' uncompressed_bytes=$size" + PREPARED_TEST_PAYLOADS+=("$payload") + return 0 fi - echo "[dd-uploader][dbg] payload content (enriched) for '$file':" >&2 - cat "$body" >&2 - echo "" >&2 - log_start_time_stats "$body" - dbg "headers: Content-Type=application/json${gzip_note}" + log "error: single_event_too_large: source='$source_file' uncompressed_bytes=$size max_bytes=$TEST_PAYLOAD_MAX_BYTES" + return 1 fi + midpoint=$((event_count / 2)) + left="$(mktemp "$TMP_PAYLOAD_DIR/test_payload_part.XXXXXX" 2>/dev/null || true)" + right="$(mktemp "$TMP_PAYLOAD_DIR/test_payload_part.XXXXXX" 2>/dev/null || true)" + if [[ -z "$left" || -z "$right" ]]; then + [[ -n "$left" ]] && rm -f "$left" 2>/dev/null || true + [[ -n "$right" ]] && rm -f "$right" 2>/dev/null || true + log "error: failed to create temporary files while splitting test payload: $source_file" + return 1 + fi + track_prepared_test_temp_file "$left" + track_prepared_test_temp_file "$right" + if ! jq -c --argjson midpoint "$midpoint" '.events = .events[0:$midpoint]' "$payload" >"$left" || + ! jq -c --argjson midpoint "$midpoint" '.events = .events[$midpoint:]' "$payload" >"$right"; then + log "error: failed to split oversized test payload: $source_file" + return 1 + fi + split_test_payload_part "$left" "$source_file" && split_test_payload_part "$right" "$source_file" +} + +prepare_test_payload_parts() { + local body="$1" + local source_file="$2" + local size compact + PREPARED_TEST_PAYLOADS=() + PREPARED_TEST_TEMP_FILES=("$body") + size="$(test_payload_size_bytes "$body")" + if (( size <= TEST_PAYLOAD_SPLIT_TARGET_BYTES )); then + PREPARED_TEST_PAYLOADS=("$body") + return 0 + fi + if (( JQ_AVAILABLE == 0 )); then + if (( size <= TEST_PAYLOAD_MAX_BYTES )); then + log "warning: test payload exceeds the split target but jq is unavailable; sending within the intake limit: source='$source_file' uncompressed_bytes=$size" + PREPARED_TEST_PAYLOADS=("$body") + return 0 + fi + log "error: oversized test payload requires jq for event splitting: source='$source_file' uncompressed_bytes=$size" + return 1 + fi + + compact="$(mktemp "$TMP_PAYLOAD_DIR/test_payload_compact.XXXXXX" 2>/dev/null || true)" + if [[ -z "$compact" ]]; then + log "error: failed to create temporary file while preparing test payload: $source_file" + return 1 + fi + track_prepared_test_temp_file "$compact" + if ! jq -c '.' "$body" >"$compact"; then + if (( size <= TEST_PAYLOAD_MAX_BYTES )); then + log "warning: oversized test payload could not be compacted; sending within the intake limit: source='$source_file' uncompressed_bytes=$size" + PREPARED_TEST_PAYLOADS=("$body") + return 0 + fi + log "error: oversized test payload is not valid JSON and cannot be split: $source_file" + return 1 + fi + if ! split_test_payload_part "$compact" "$source_file"; then + return 1 + fi + log "split test payload: source='$source_file' uncompressed_bytes=$size parts=${#PREPARED_TEST_PAYLOADS[@]} target_bytes=$TEST_PAYLOAD_SPLIT_TARGET_BYTES" + return 0 +} + +bounded_upload_response() { + local response_file="$1" + head -c "$UPLOAD_RESPONSE_LOG_BYTES" "$response_file" 2>/dev/null | tr '\r\n' ' ' +} + +send_test_payload_part() { + local source_file="$1" + local body="$2" + local part_index="$3" + local part_count="$4" + local resp payload_file gz http rc uncompressed_bytes transmitted_bytes compressed_bytes encoding response_bytes response_text truncated + build_common_headers "$body" + uncompressed_bytes="$(test_payload_size_bytes "$body")" + payload_file="$body" gz="" + compressed_bytes="none" + encoding="identity" if [[ "$GZIP_PAYLOADS" == "1" ]]; then # Compress enriched payload, but gracefully fall back to plain JSON if # gzip is unavailable/fails on the host. gz="$body.gz" if gzip -c "$body" > "$gz"; then payload_file="$gz" + compressed_bytes="$(test_payload_size_bytes "$gz")" + encoding="gzip" else log "warning: gzip failed; sending uncompressed payload" gz="" fi fi + transmitted_bytes="$(test_payload_size_bytes "$payload_file")" resp="$(mktemp "$TMP_PAYLOAD_DIR/test_resp.XXXXXX" 2>/dev/null || true)" if [[ -z "$resp" ]]; then - dbg "upload_single_test: failed to create response temp file" - rm -f "$body" "$gz" 2>/dev/null || true + log "error: failed to create response temp file for test payload: $source_file" + rm -f "$gz" 2>/dev/null || true return 1 fi local ce_hdr=() @@ -4844,14 +4957,14 @@ upload_single_test() { fi fi if (( AGENTLESS == 1 )); then - if http=$(curl_agentless -f -sS --connect-timeout 10 --max-time 60 "${CURL_RETRY_FLAGS[@]}" \ + if http=$(curl_agentless "${CURL_FAIL_FLAG[@]}" -sS --connect-timeout 10 --max-time 60 "${CURL_RETRY_FLAGS[@]}" \ -X POST "${TEST_URL}" "${COMMON_HDRS[@]}" "${ce_hdr[@]+${ce_hdr[@]}}" -H "Content-Type: application/json" --data-binary @"${payload_file}" -o "$resp" -w "%{http_code}"); then rc=0 else rc=$? fi else - if http=$(curl -f -sS --connect-timeout 10 --max-time 60 "${CURL_RETRY_FLAGS[@]}" \ + if http=$(curl "${CURL_FAIL_FLAG[@]}" -sS --connect-timeout 10 --max-time 60 "${CURL_RETRY_FLAGS[@]}" \ -X POST "${TEST_URL}" "${COMMON_HDRS[@]}" "${TEST_EVP[@]}" "${ce_hdr[@]+${ce_hdr[@]}}" -H "Content-Type: application/json" --data-binary @"${payload_file}" -o "$resp" -w "%{http_code}"); then rc=0 else @@ -4859,13 +4972,20 @@ upload_single_test() { fi fi http="${http:-000}" - if [[ "$DEBUG" == "1" || $rc -ne 0 || "$http" -lt 200 || "$http" -ge 300 ]]; then - dbg "upload_single_test: HTTP $http (rc=$rc)" + if [[ "$DEBUG" == "1" ]]; then + dbg "upload_single_test: HTTP $http (rc=$rc; part=$part_index/$part_count; uncompressed_bytes=$uncompressed_bytes; transmitted_bytes=$transmitted_bytes; encoding=$encoding)" + fi + if [[ $rc -ne 0 || "$http" -lt 200 || "$http" -ge 300 ]]; then + response_bytes="$(test_payload_size_bytes "$resp")" + response_text="" + truncated="false" if [[ -s "$resp" ]]; then - dbg "upload_single_test response: $(head -c 2000 "$resp")" + response_text="$(bounded_upload_response "$resp")" + (( response_bytes > UPLOAD_RESPONSE_LOG_BYTES )) && truncated="true" fi + log "upload failed: source='$source_file' part=$part_index/$part_count http=$http curl_rc=$rc encoding=$encoding uncompressed_bytes=$uncompressed_bytes compressed_bytes=$compressed_bytes transmitted_bytes=$transmitted_bytes response_bytes=$response_bytes response_truncated=$truncated response_body='$response_text'" fi - rm -f "$resp" "$body" "$gz" 2>/dev/null || true + rm -f "$resp" "$gz" 2>/dev/null || true # Cleanup happens before return to avoid temp-file buildup on retries/runs. if [[ $rc -ne 0 || "$http" -lt 200 || "$http" -ge 300 ]]; then return 1 @@ -4873,6 +4993,44 @@ upload_single_test() { return 0 } +# Enrich one source payload, split it when necessary, and send every prepared +# part independently so one failed part does not suppress the remaining data. +upload_single_test() { + local file="$1" + local body part part_index=0 part_count failed=0 + body="$(mktemp "$TMP_PAYLOAD_DIR/test_payload.XXXXXX" 2>/dev/null || true)" + if [[ -z "$body" ]]; then + log "error: failed to create temporary test payload: $file" + return 1 + fi + enrich_with_context "$file" "$body" + validate_payload "$body" + dbg "upload_single_test: posting '$file' (body '$body')" + if [[ "$DEBUG" == "1" ]]; then + echo "[dd-uploader][dbg] payload content (enriched) for '$file':" >&2 + cat "$body" >&2 + echo "" >&2 + log_start_time_stats "$body" + fi + if ! validate_enriched_payload_tags "$body" "$file"; then + rm -f "$body" 2>/dev/null || true + return 1 + fi + if ! prepare_test_payload_parts "$body" "$file"; then + cleanup_prepared_test_payloads + return 1 + fi + part_count=${#PREPARED_TEST_PAYLOADS[@]} + for part in "${PREPARED_TEST_PAYLOADS[@]}"; do + ((++part_index)) + if ! send_test_payload_part "$file" "$part" "$part_index" "$part_count"; then + failed=1 + fi + done + cleanup_prepared_test_payloads + (( failed == 0 )) +} + # Handle upload single coverage behavior. upload_single_coverage() { local file="$1" diff --git a/tools/core/uploader_powershell_runtime.ps1.tpl b/tools/core/uploader_powershell_runtime.ps1.tpl index af7f2cee..fd7cd62d 100644 --- a/tools/core/uploader_powershell_runtime.ps1.tpl +++ b/tools/core/uploader_powershell_runtime.ps1.tpl @@ -644,6 +644,9 @@ $KeepPayloads = if ($env:DD_TEST_OPTIMIZATION_KEEP_PAYLOADS) { Normalize-Bool $e $FilterPrefix = if ($env:DD_TEST_OPTIMIZATION_FILTER_PREFIX) { Normalize-Bool $env:DD_TEST_OPTIMIZATION_FILTER_PREFIX } else { Normalize-Bool "__DDTPL_FILTER_PREFIX__" } $Debug = if ($env:DD_TEST_OPTIMIZATION_DEBUG) { Normalize-Bool $env:DD_TEST_OPTIMIZATION_DEBUG } else { Normalize-Bool "__DDTPL_DEBUG__" } $GzipPayloads = if ($env:DD_TEST_OPTIMIZATION_GZIP) { Normalize-Bool $env:DD_TEST_OPTIMIZATION_GZIP } else { Normalize-Bool "__DDTPL_GZIP_PAYLOADS__" } +$script:TestPayloadSplitTargetBytes = 4500000 +$script:TestPayloadMaxBytes = 5000000 +$script:UploadResponseLogChars = 2000 # Now that $Debug is set, update the script-level debug mode for Dbg function $script:DebugMode = $Debug @@ -725,11 +728,11 @@ $DefaultExpectedEnrichedTags = @( ) function Show-Usage { - Write-Host "Usage: dd_upload_payloads [--dry-run [--validate-enrichment] [--expected-enriched-tag=TAG ...]]" + Write-Host "Usage: dd_upload_payloads [--dry-run] [--validate-enrichment] [--expected-enriched-tag=TAG ...]" Write-Host "" Write-Host "Options:" Write-Host " --dry-run Enrich and validate payloads without uploading or deleting files." - Write-Host " --validate-enrichment In dry-run mode, require key context and Bazel tags after enrichment." + Write-Host " --validate-enrichment Require key context and Bazel tags after enrichment, before upload." Write-Host " --expected-enriched-tag TAG Add one required enriched tag; repeatable. Defaults to git and Bazel tags." Write-Host " --bep-json PATH BEP JSON file from the matching bazel test invocation; repeatable." Write-Host " --freshness-source SOURCE Cache-safety source: auto, bep, execution_log. Default: auto." @@ -939,10 +942,6 @@ for ($i = 0; $i -lt $args.Count; $i++) { exit 2 } -if ($ValidateEnrichment -and -not $DryRun) { - Log "error: --validate-enrichment requires --dry-run" - exit 2 -} if ($FreshnessDisabledExplicit) { $FreshnessMode = "disabled" $ExecutionLogMode = "disabled" @@ -3773,9 +3772,30 @@ $script:ReportTelemetryProcessed = 0 $script:ReportTelemetryFailed = 0 $script:ReportTelemetrySkipped = 0 -function Send-PostJson([string]$url, [hashtable]$headers, [string]$file) { +function Format-BoundedUploadResponse([string]$Body) { + if ([string]::IsNullOrEmpty($Body)) { + return [pscustomobject]@{ Text = ''; Bytes = 0; Truncated = $false } + } + $responseBytes = [System.Text.Encoding]::UTF8.GetByteCount($Body) + $singleLine = $Body.Replace("`r", ' ').Replace("`n", ' ') + $truncated = $singleLine.Length -gt $script:UploadResponseLogChars + if ($truncated) { + $singleLine = $singleLine.Substring(0, $script:UploadResponseLogChars) + } + return [pscustomobject]@{ Text = $singleLine; Bytes = $responseBytes; Truncated = $truncated } +} + +function Send-PostJson( + [string]$url, + [hashtable]$headers, + [string]$file, + [string]$SourcePath = $file, + [int]$PartIndex = 1, + [int]$PartCount = 1 +) { $maxRetries = 3 $retryDelay = 2 + $uncompressedBytes = (Get-Item -LiteralPath $file -ErrorAction Stop).Length if (-not (Ensure-HttpClientTypes)) { Log "upload failed: System.Net.Http.HttpClient unavailable in this PowerShell runtime" return [bool]$false @@ -3805,10 +3825,16 @@ function Send-PostJson([string]$url, [hashtable]$headers, [string]$file) { $content = New-Object System.Net.Http.ByteArrayContent -ArgumentList (, $compressed) $content.Headers.ContentType = 'application/json' $null = $content.Headers.ContentEncoding.Add('gzip') + $compressedBytes = $compressed.Length + $transmittedBytes = $compressedBytes + $encoding = 'gzip' Dbg "Send-PostJson: Content-Type=application/json; Content-Encoding=gzip (bytes=$($compressed.Length))" } else { $content = New-Object System.Net.Http.StringContent([IO.File]::ReadAllText($file, [System.Text.Encoding]::UTF8)) $content.Headers.ContentType = 'application/json' + $compressedBytes = 'none' + $transmittedBytes = $uncompressedBytes + $encoding = 'identity' Dbg "Send-PostJson: Content-Type=application/json" } $resp = $client.PostAsync($url, $content).GetAwaiter().GetResult() @@ -3822,15 +3848,18 @@ function Send-PostJson([string]$url, [hashtable]$headers, [string]$file) { $body = $resp.Content.ReadAsStringAsync().GetAwaiter().GetResult() Dbg "Send-PostJson: HTTP $([int]$resp.StatusCode) on attempt $attempt" if ($attempt -eq $maxRetries) { - # Emit user-facing failure only after retry budget is exhausted. - Log "upload failed: HTTP $([int]$resp.StatusCode) $body" + $bounded = Format-BoundedUploadResponse $body + Log "upload failed: source='$SourcePath' part=$PartIndex/$PartCount http=$([int]$resp.StatusCode) encoding=$encoding uncompressed_bytes=$uncompressedBytes compressed_bytes=$compressedBytes transmitted_bytes=$transmittedBytes response_bytes=$($bounded.Bytes) response_truncated=$($bounded.Truncated.ToString().ToLowerInvariant()) response_body='$($bounded.Text)'" return [bool]$false } } } catch { Dbg "Send-PostJson: Exception on attempt $attempt - $_" if ($attempt -eq $maxRetries) { - Log "upload failed: $_" + $encoding = if ($script:GzipPayloads) { 'gzip' } else { 'identity' } + $compressedBytes = if ($script:GzipPayloads -and $null -ne $compressed) { $compressed.Length } else { 'none' } + $transmittedBytes = if ($script:GzipPayloads -and $null -ne $compressed) { $compressed.Length } else { $uncompressedBytes } + Log "upload failed: source='$SourcePath' part=$PartIndex/$PartCount http=000 encoding=$encoding uncompressed_bytes=$uncompressedBytes compressed_bytes=$compressedBytes transmitted_bytes=$transmittedBytes response_bytes=0 response_truncated=false response_body='' exception='$_'" return [bool]$false } } finally { @@ -3843,6 +3872,80 @@ function Send-PostJson([string]$url, [hashtable]$headers, [string]$file) { return [bool]$false } +$script:PreparedTestPayloads = [System.Collections.Generic.List[string]]::new() +$script:PreparedTestTempFiles = [System.Collections.Generic.List[string]]::new() + +function Clear-PreparedTestPayloads { + foreach ($path in @($script:PreparedTestTempFiles.ToArray())) { + Remove-Item -LiteralPath $path -Force -ErrorAction SilentlyContinue + } + $script:PreparedTestPayloads.Clear() + $script:PreparedTestTempFiles.Clear() +} + +function Write-TestPayloadPart($Payload, [object[]]$Events) { + $path = Join-Path $script:TmpPayloadDir ("test_payload_part_" + [System.Guid]::NewGuid().ToString("N") + ".json") + $Payload.events = @($Events) + Write-Utf8NoBomFile -Path $path -Content (($Payload | ConvertTo-Json -Depth 100 -Compress) + "`n") + $script:PreparedTestTempFiles.Add($path) | Out-Null + return $path +} + +function Split-TestPayloadEvents($Payload, [object[]]$Events, [string]$SourcePath) { + $path = Write-TestPayloadPart $Payload $Events + $size = (Get-Item -LiteralPath $path -ErrorAction Stop).Length + if ($size -le $script:TestPayloadSplitTargetBytes) { + $script:PreparedTestPayloads.Add($path) | Out-Null + return [bool]$true + } + if ($Events.Count -eq 1) { + if ($size -le $script:TestPayloadMaxBytes) { + Log "warning: single-event test payload exceeds the split target but remains within the intake limit: source='$SourcePath' uncompressed_bytes=$size" + $script:PreparedTestPayloads.Add($path) | Out-Null + return [bool]$true + } + Log "error: single_event_too_large: source='$SourcePath' uncompressed_bytes=$size max_bytes=$($script:TestPayloadMaxBytes)" + return [bool]$false + } + + $midpoint = [int][Math]::Floor($Events.Count / 2) + $left = @($Events[0..($midpoint - 1)]) + $right = @($Events[$midpoint..($Events.Count - 1)]) + return [bool]((Split-TestPayloadEvents $Payload $left $SourcePath) -and (Split-TestPayloadEvents $Payload $right $SourcePath)) +} + +function Prepare-TestPayloadParts([string]$BodyPath, [string]$SourcePath) { + $script:PreparedTestPayloads.Clear() + $script:PreparedTestTempFiles.Clear() + $script:PreparedTestTempFiles.Add($BodyPath) | Out-Null + $size = (Get-Item -LiteralPath $BodyPath -ErrorAction Stop).Length + if ($size -le $script:TestPayloadSplitTargetBytes) { + $script:PreparedTestPayloads.Add($BodyPath) | Out-Null + return [bool]$true + } + try { + $payload = Get-Content -LiteralPath $BodyPath -Raw -Encoding UTF8 | ConvertFrom-Json -ErrorAction Stop + } catch { + if ($size -le $script:TestPayloadMaxBytes) { + Log "warning: oversized test payload is not valid JSON; sending within the intake limit: source='$SourcePath' uncompressed_bytes=$size" + $script:PreparedTestPayloads.Add($BodyPath) | Out-Null + return [bool]$true + } + Log "error: oversized test payload is not valid JSON and cannot be split: $SourcePath" + return [bool]$false + } + $events = @(Get-MapValue $payload 'events') + if ($events.Count -lt 1) { + Log "error: oversized test payload cannot be split because its events array is invalid: $SourcePath" + return [bool]$false + } + if (-not (Split-TestPayloadEvents $payload $events $SourcePath)) { + return [bool]$false + } + Log "split test payload: source='$SourcePath' uncompressed_bytes=$size parts=$($script:PreparedTestPayloads.Count) target_bytes=$($script:TestPayloadSplitTargetBytes)" + return [bool]$true +} + function Get-TelemetryHeaders([string]$FilePath) { try { $payloadObj = Get-Content -LiteralPath $FilePath -Raw -Encoding UTF8 | ConvertFrom-Json -ErrorAction Stop @@ -4471,26 +4574,40 @@ function Upload-SingleTest([string]$FilePath) { $body = Join-Path $script:TmpPayloadDir ("test_payload_" + [System.Guid]::NewGuid().ToString("N") + ".json") Merge-With-Context $FilePath $body Validate-Payload $body - $hdrs = Get-CommonHeaders $body - if (-not $Agentless) { $hdrs['X-Datadog-EVP-Subdomain'] = 'citestcycle-intake' } Dbg "Upload-SingleTest: posting '$FilePath' (body '$body')" if ($script:DebugMode) { Write-Host "[dd-uploader][dbg] payload content (enriched) for '$FilePath':" Write-Host (Get-Content -LiteralPath $body -Raw) - Dbg "request: POST $TestUrl" - Dbg-Headers "common" $hdrs Log-StartTimeStats $body } - # Native command / .NET call paths can emit incidental pipeline items. - # Consume the stream and treat the final emitted value as the boolean result. - $resultStream = @(Send-PostJson $TestUrl $hdrs $body) - $result = $false - if ($resultStream.Count -gt 0) { - $result = [bool]$resultStream[-1] + if (-not (Test-EnrichedPayloadTags $body $FilePath)) { + Remove-Item -LiteralPath $body -Force -ErrorAction SilentlyContinue + return [bool]$false } - # Enriched temp payload is always ephemeral. - Remove-Item -LiteralPath $body -Force -ErrorAction SilentlyContinue - return [bool]$result + if (-not (Prepare-TestPayloadParts $body $FilePath)) { + Clear-PreparedTestPayloads + return [bool]$false + } + $partCount = $script:PreparedTestPayloads.Count + $failed = $false + try { + for ($index = 0; $index -lt $partCount; $index++) { + $part = $script:PreparedTestPayloads[$index] + $hdrs = Get-CommonHeaders $part + if (-not $Agentless) { $hdrs['X-Datadog-EVP-Subdomain'] = 'citestcycle-intake' } + if ($script:DebugMode) { + Dbg "request: POST $TestUrl (part=$($index + 1)/$partCount)" + Dbg-Headers "common" $hdrs + } + $resultStream = @(Send-PostJson $TestUrl $hdrs $part $FilePath ($index + 1) $partCount) + if ($resultStream.Count -eq 0 -or -not [bool]$resultStream[-1]) { + $failed = $true + } + } + } finally { + Clear-PreparedTestPayloads + } + return [bool](-not $failed) } function Get-ExpectedEnrichedTags { @@ -4512,7 +4629,7 @@ function Test-EventHasEnrichedTag($EventObj, [string]$Tag) { function Test-EnrichedPayloadTags([string]$BodyPath, [string]$SourcePath) { if (-not $script:ValidateEnrichment) { return [bool]$true } - $payload = Read-JsonObjectFile $BodyPath "dry-run could not parse enriched test payload '$SourcePath'" + $payload = Read-JsonObjectFile $BodyPath "could not parse enriched test payload '$SourcePath'" if (-not $payload) { return [bool]$false } $events = @(Get-MapValue $payload 'events') $missing = New-Object System.Collections.Generic.List[string] @@ -4534,7 +4651,11 @@ function Test-EnrichedPayloadTags([string]$BodyPath, [string]$SourcePath) { Log "error: enriched test payload for '$SourcePath' is missing expected tag(s): $($missing -join ', ')" return [bool]$false } - Log "dry-run validated enriched test payload: $SourcePath" + if ($script:DryRun) { + Log "dry-run validated enriched test payload: $SourcePath" + } else { + Log "validated enriched test payload: $SourcePath" + } return [bool]$true } @@ -4558,8 +4679,18 @@ function DryRun-SingleTest([string]$FilePath) { Write-Output $item } } - return [bool]$validated + if (-not $validated) { + return [bool]$false + } + if (-not (Prepare-TestPayloadParts $body $FilePath)) { + return [bool]$false + } + if ($script:PreparedTestPayloads.Count -gt 1) { + Log "dry-run would split test payload '$FilePath' into $($script:PreparedTestPayloads.Count) parts" + } + return [bool]$true } finally { + Clear-PreparedTestPayloads Remove-Item -LiteralPath $body -Force -ErrorAction SilentlyContinue } } diff --git a/tools/tests/python/test_python_tools.py b/tools/tests/python/test_python_tools.py index 37c241bd..70145bc7 100644 --- a/tools/tests/python/test_python_tools.py +++ b/tools/tests/python/test_python_tools.py @@ -12,6 +12,7 @@ import ast import contextlib import functools +import gzip import http.server import importlib.util import io @@ -177,6 +178,38 @@ def _serve_handler(handler: type[http.server.BaseHTTPRequestHandler]) -> Iterato thread.join(timeout=5) +@contextlib.contextmanager +def _serve_test_uploads(responder): + """Record decoded citestcycle requests and return responder-selected statuses.""" + records: list[dict[str, object]] = [] + + class Handler(QuietBaseHTTPRequestHandler): + def do_POST(self) -> None: # noqa: N802 - stdlib handler API + length = int(self.headers.get("Content-Length", "0")) + wire_body = self.rfile.read(length) + encoding = self.headers.get("Content-Encoding", "").lower() + decoded_body = gzip.decompress(wire_body) if "gzip" in encoding else wire_body + payload = json.loads(decoded_body.decode("utf-8")) + status, response = responder(payload, records) + response_body = json.dumps(response, separators=(",", ":")).encode("utf-8") + records.append({ + "decoded_bytes": len(decoded_body), + "encoding": encoding, + "headers": {key.lower(): value for key, value in self.headers.items()}, + "payload": payload, + "status": status, + "wire_bytes": len(wire_body), + }) + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(response_body))) + self.end_headers() + self.wfile.write(response_body) + + with _serve_handler(Handler) as base_url: + yield base_url, records + + def _render_uploader_runtime_template( rel_path: str, *, @@ -185,6 +218,7 @@ def _render_uploader_runtime_template( expected_targets_path: str = "", expected_targets_file_path: str = "", fail_on_error: bool = False, + curl_retry_flags: str = "--retry 3 --retry-delay 2 --retry-connrefused", ) -> str: """Render uploader runtime template placeholders for direct unit tests.""" text = _runfile(rel_path).read_text(encoding="utf-8") @@ -194,7 +228,7 @@ def _render_uploader_runtime_template( "context_json_rloc": "", "context_manifest_path": "", "context_manifest_rloc": "", - "curl_retry_flags": "--retry 3 --retry-delay 2 --retry-connrefused", + "curl_retry_flags": curl_retry_flags, "debug": "false", "doctor_runtime_rloc": doctor_runtime_rloc, "expected_targets_file_path": expected_targets_file_path, @@ -6298,6 +6332,310 @@ def _generated_uploader_smoke_env(root: Path, runfiles_dir: Path) -> dict[str, s env.pop("RUNFILES_MANIFEST_FILE", None) return env + def _run_local_test_upload( + self, + runtime: str, + payload: dict[str, object], + responder, + *, + extra_args: tuple[str, ...] = (), + gzip_enabled: bool, + keep_payloads: bool = True, + ) -> tuple[subprocess.CompletedProcess[str], list[dict[str, object]], bool]: + """Run one generated uploader against a recording local test intake.""" + bash = _require_functional_bash(self) + pwsh = _require_command(self, "pwsh", "pwsh is required for generated uploader parity") + _require_command(self, "jq", "jq is required for oversized Bash payload splitting") + + root = Path(tempfile.mkdtemp()) + try: + payload_dir = root / "bazel-testlogs" / "pkg" / "test" / "test.outputs" / "payloads" / "tests" + payload_dir.mkdir(parents=True) + source = payload_dir / "span_events_generated.json" + source.write_text(json.dumps(payload, separators=(",", ":")), encoding="utf-8") + runfiles_dir = root / "empty.runfiles" + runfiles_dir.mkdir() + + if runtime == "Bash": + generated = root / "generated_uploader.sh" + generated.write_text( + _render_uploader_runtime_template( + "tools/core/uploader_bash_runtime.sh.tpl", + fail_on_error=True, + curl_retry_flags="--retry 1 --retry-delay 0 --retry-connrefused", + ), + encoding="utf-8", + ) + generated.chmod(0o755) + command = [bash, str(generated), *extra_args] + elif runtime == "PowerShell": + generated = root / "generated_uploader.ps1" + generated.write_text( + _render_uploader_runtime_template( + "tools/core/uploader_powershell_runtime.ps1.tpl", + fail_on_error=True, + ), + encoding="utf-8", + ) + command = [pwsh, "-NoLogo", "-NoProfile", "-File", str(generated), *extra_args] + else: + raise AssertionError(f"unknown uploader runtime: {runtime}") + + env = os.environ.copy() + env.update({ + "BUILD_WORKSPACE_DIRECTORY": str(root), + "DD_TEST_OPTIMIZATION_DEBUG": "0", + "DD_TEST_OPTIMIZATION_GZIP": "1" if gzip_enabled else "0", + "DD_TEST_OPTIMIZATION_KEEP_PAYLOADS": "1" if keep_payloads else "0", + "DD_TEST_OPTIMIZATION_MAX_WAIT_SEC": "0", + "DD_TEST_OPTIMIZATION_QUIESCENT_SEC": "0", + "RUNFILES_DIR": str(runfiles_dir), + "TESTLOGS_DIR": str(root / "bazel-testlogs"), + }) + env.pop("DD_API_KEY", None) + env.pop("RUNFILES_MANIFEST_FILE", None) + + with _serve_test_uploads(responder) as (base_url, records): + env["DD_TEST_OPTIMIZATION_AGENT_URL"] = base_url + result = subprocess.run( + command, + cwd=root, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=45, + check=False, + ) + return result, list(records), source.is_file() + finally: + _cleanup_tempdir_with_windows_retry(root) + + @staticmethod + def _large_test_payload(*, event_count: int, blob_bytes: int) -> dict[str, object]: + """Build an oversized payload without checking a large fixture into the repo.""" + return { + "custom_envelope": {"preserved": True}, + "events": [ + { + "content": { + "meta": {"blob": "x" * blob_bytes, "event.id": f"event-{index}"}, + "metrics": {}, + "resource": f"pkg.test.{index}", + }, + "type": "test", + } + for index in range(event_count) + ], + "metadata": {"*": {"language": "python", "library_version": "test"}}, + "version": 1, + } + + def test_generated_uploaders_log_failed_test_upload_details(self) -> None: + """Expose bounded response bodies and wire sizes without debug mode.""" + payload = self._large_test_payload(event_count=1, blob_bytes=64) + + def reject(_payload, _records): + return 413, {"error": "payload too large"} + + for runtime in ("Bash", "PowerShell"): + for gzip_enabled in (False, True): + with self.subTest(runtime=runtime, gzip=gzip_enabled): + result, records, source_exists = self._run_local_test_upload( + runtime, + payload, + reject, + gzip_enabled=gzip_enabled, + ) + output = result.stdout + result.stderr + self.assertNotEqual(0, result.returncode, output) + self.assertTrue(source_exists) + self.assertGreaterEqual(len(records), 1) + self.assertIn("http=413", output) + self.assertIn("uncompressed_bytes=", output) + self.assertIn("transmitted_bytes=", output) + self.assertIn("response_body='{\"error\":\"payload too large\"}'", output) + self.assertIn( + "encoding=gzip" if gzip_enabled else "encoding=identity", + output, + ) + self.assertIn( + "compressed_bytes=" if gzip_enabled else "compressed_bytes=none", + output, + ) + + def test_generated_uploaders_split_large_test_payloads_and_retry_parts(self) -> None: + """Split before transport while preserving order, envelope, gzip, and retry isolation.""" + payload = self._large_test_payload(event_count=4, blob_bytes=1_300_000) + + def retry_second_part(candidate, records): + event_ids = [event["content"]["meta"]["event.id"] for event in candidate["events"]] + prior_attempts = sum( + 1 + for record in records + if [event["content"]["meta"]["event.id"] for event in record["payload"]["events"]] + == event_ids + ) + if event_ids == ["event-2", "event-3"] and prior_attempts == 0: + return 503, {"error": "retry this part"} + return 200, {} + + for runtime in ("Bash", "PowerShell"): + for gzip_enabled in (False, True): + with self.subTest(runtime=runtime, gzip=gzip_enabled): + result, records, source_exists = self._run_local_test_upload( + runtime, + payload, + retry_second_part, + gzip_enabled=gzip_enabled, + keep_payloads=False, + ) + output = result.stdout + result.stderr + self.assertEqual(0, result.returncode, output) + self.assertFalse(source_exists) + self.assertIn("parts=2", output) + successful = [record for record in records if record["status"] == 200] + self.assertEqual(2, len(successful), records) + event_ids = [ + event["content"]["meta"]["event.id"] + for record in successful + for event in record["payload"]["events"] + ] + self.assertEqual([f"event-{index}" for index in range(4)], event_ids) + for record in records: + self.assertLessEqual(record["decoded_bytes"], 4_500_000) + self.assertEqual({"preserved": True}, record["payload"]["custom_envelope"]) + self.assertEqual(1, record["payload"]["version"]) + self.assertEqual("gzip" if gzip_enabled else "", record["encoding"]) + first_part_attempts = [ + record + for record in records + if record["payload"]["events"][0]["content"]["meta"]["event.id"] == "event-0" + ] + self.assertEqual(1, len(first_part_attempts), records) + + def test_generated_uploaders_dry_run_validates_split_without_uploading(self) -> None: + """Use the real split preparation during dry-run without sending data.""" + payload = self._large_test_payload(event_count=4, blob_bytes=1_300_000) + + def accept(_payload, _records): + return 200, {} + + for runtime in ("Bash", "PowerShell"): + with self.subTest(runtime=runtime): + result, records, source_exists = self._run_local_test_upload( + runtime, + payload, + accept, + extra_args=("--dry-run",), + gzip_enabled=True, + ) + output = result.stdout + result.stderr + self.assertEqual(0, result.returncode, output) + self.assertTrue(source_exists) + self.assertEqual([], records) + self.assertIn("dry-run would split test payload", output) + self.assertIn("into 2 parts", output) + + def test_generated_uploaders_validate_enrichment_during_upload(self) -> None: + """Validate the enriched outbound body without a separate dry-run pass.""" + payload = self._large_test_payload(event_count=1, blob_bytes=64) + + def accept(_payload, _records): + return 200, {} + + for runtime in ("Bash", "PowerShell"): + with self.subTest(runtime=runtime, result="valid"): + result, records, source_exists = self._run_local_test_upload( + runtime, + payload, + accept, + extra_args=("--validate-enrichment", "--expected-enriched-tag=event.id"), + gzip_enabled=True, + keep_payloads=False, + ) + output = result.stdout + result.stderr + self.assertEqual(0, result.returncode, output) + self.assertFalse(source_exists) + self.assertEqual(1, len(records), records) + self.assertIn("validated enriched test payload", output) + self.assertNotIn("dry-run validated enriched test payload", output) + + with self.subTest(runtime=runtime, result="missing-tag"): + result, records, source_exists = self._run_local_test_upload( + runtime, + payload, + accept, + extra_args=("--validate-enrichment", "--expected-enriched-tag=missing.tag"), + gzip_enabled=True, + keep_payloads=False, + ) + output = result.stdout + result.stderr + self.assertNotEqual(0, result.returncode, output) + self.assertTrue(source_exists) + self.assertEqual([], records) + self.assertIn("missing expected tag(s): missing.tag", output) + + def test_generated_uploaders_continue_after_one_split_part_fails(self) -> None: + """Keep successful parts uploaded while reporting aggregate source failure.""" + payload = self._large_test_payload(event_count=4, blob_bytes=1_300_000) + + def reject_second_part(candidate, _records): + first_id = candidate["events"][0]["content"]["meta"]["event.id"] + if first_id == "event-2": + return 503, {"error": "persistent part failure"} + return 200, {} + + for runtime in ("Bash", "PowerShell"): + with self.subTest(runtime=runtime): + result, records, source_exists = self._run_local_test_upload( + runtime, + payload, + reject_second_part, + gzip_enabled=True, + keep_payloads=False, + ) + output = result.stdout + result.stderr + self.assertNotEqual(0, result.returncode, output) + self.assertTrue(source_exists) + first_part = [ + record + for record in records + if record["payload"]["events"][0]["content"]["meta"]["event.id"] == "event-0" + ] + failed_part = [ + record + for record in records + if record["payload"]["events"][0]["content"]["meta"]["event.id"] == "event-2" + ] + self.assertEqual(1, len(first_part), records) + self.assertEqual(200, first_part[0]["status"]) + self.assertGreaterEqual(len(failed_part), 2, records) + self.assertTrue(all(record["status"] == 503 for record in failed_part)) + self.assertIn("part=2/2", output) + self.assertIn("persistent part failure", output) + + def test_generated_uploaders_reject_unsplittable_single_event(self) -> None: + """Reject one event above the hard intake limit without making a request.""" + payload = self._large_test_payload(event_count=1, blob_bytes=5_100_000) + + def accept(_payload, _records): + return 200, {} + + for runtime in ("Bash", "PowerShell"): + with self.subTest(runtime=runtime): + result, records, source_exists = self._run_local_test_upload( + runtime, + payload, + accept, + gzip_enabled=False, + ) + output = result.stdout + result.stderr + self.assertNotEqual(0, result.returncode, output) + self.assertTrue(source_exists) + self.assertEqual([], records) + self.assertIn("single_event_too_large", output) + def _assert_uploader_report_success(self, report_path: Path, bep_path: Path, staging_dir: Path) -> None: """Validate a successful generated uploader machine-readable report.""" report = json.loads(report_path.read_text(encoding="utf-8")) From e743cb8b3cd4e19f3d1eeed3853c0081bfe614f9 Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Wed, 26 Aug 2026 09:01:11 +0200 Subject: [PATCH 18/37] uploader: remove payloads from read-only output trees --- tools/core/uploader_bash_runtime.sh.tpl | 1 + tools/core/uploader_powershell_runtime.ps1.tpl | 3 +++ tools/tests/python/test_python_tools.py | 4 ++++ 3 files changed, 8 insertions(+) diff --git a/tools/core/uploader_bash_runtime.sh.tpl b/tools/core/uploader_bash_runtime.sh.tpl index 24a6e220..4dc29046 100644 --- a/tools/core/uploader_bash_runtime.sh.tpl +++ b/tools/core/uploader_bash_runtime.sh.tpl @@ -3972,6 +3972,7 @@ cleanup_file() { # Some runfiles can be read-only; best-effort cleanup keeps uploads resilient. if ! rm -f "$file" 2>/dev/null; then chmod u+w "$file" 2>/dev/null || true + chmod u+w "$(dirname "$file")" 2>/dev/null || true rm -f "$file" 2>/dev/null || true fi else diff --git a/tools/core/uploader_powershell_runtime.ps1.tpl b/tools/core/uploader_powershell_runtime.ps1.tpl index fd7cd62d..6c3dff34 100644 --- a/tools/core/uploader_powershell_runtime.ps1.tpl +++ b/tools/core/uploader_powershell_runtime.ps1.tpl @@ -3753,6 +3753,9 @@ function Remove-PayloadFile([string]$FilePath) { $item.IsReadOnly = $false } } catch {} + if (-not $IsWindows) { + & chmod u+w -- (Split-Path -Parent $FilePath) 2>$null + } Remove-Item -LiteralPath $FilePath -Force -ErrorAction SilentlyContinue } } else { diff --git a/tools/tests/python/test_python_tools.py b/tools/tests/python/test_python_tools.py index 70145bc7..32fec2de 100644 --- a/tools/tests/python/test_python_tools.py +++ b/tools/tests/python/test_python_tools.py @@ -6341,6 +6341,7 @@ def _run_local_test_upload( extra_args: tuple[str, ...] = (), gzip_enabled: bool, keep_payloads: bool = True, + read_only_parent: bool = False, ) -> tuple[subprocess.CompletedProcess[str], list[dict[str, object]], bool]: """Run one generated uploader against a recording local test intake.""" bash = _require_functional_bash(self) @@ -6353,6 +6354,8 @@ def _run_local_test_upload( payload_dir.mkdir(parents=True) source = payload_dir / "span_events_generated.json" source.write_text(json.dumps(payload, separators=(",", ":")), encoding="utf-8") + if read_only_parent: + payload_dir.chmod(0o555) runfiles_dir = root / "empty.runfiles" runfiles_dir.mkdir() @@ -6553,6 +6556,7 @@ def accept(_payload, _records): extra_args=("--validate-enrichment", "--expected-enriched-tag=event.id"), gzip_enabled=True, keep_payloads=False, + read_only_parent=True, ) output = result.stdout + result.stderr self.assertEqual(0, result.returncode, output) From be7487e4e4a446e189d298fc1f27069c46bd92e5 Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Wed, 26 Aug 2026 09:14:18 +0200 Subject: [PATCH 19/37] tests: isolate uploader transport fixtures --- tools/tests/python/test_python_tools.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tools/tests/python/test_python_tools.py b/tools/tests/python/test_python_tools.py index 32fec2de..f83f00a4 100644 --- a/tools/tests/python/test_python_tools.py +++ b/tools/tests/python/test_python_tools.py @@ -6370,7 +6370,7 @@ def _run_local_test_upload( encoding="utf-8", ) generated.chmod(0o755) - command = [bash, str(generated), *extra_args] + command = [bash, str(generated), "--allow-cached-payload-uploads", *extra_args] elif runtime == "PowerShell": generated = root / "generated_uploader.ps1" generated.write_text( @@ -6380,7 +6380,15 @@ def _run_local_test_upload( ), encoding="utf-8", ) - command = [pwsh, "-NoLogo", "-NoProfile", "-File", str(generated), *extra_args] + command = [ + pwsh, + "-NoLogo", + "-NoProfile", + "-File", + str(generated), + "--allow-cached-payload-uploads", + *extra_args, + ] else: raise AssertionError(f"unknown uploader runtime: {runtime}") @@ -6412,6 +6420,8 @@ def _run_local_test_upload( ) return result, list(records), source.is_file() finally: + if read_only_parent and payload_dir.exists(): + payload_dir.chmod(0o755) _cleanup_tempdir_with_windows_retry(root) @staticmethod From bf470d7c54211689e5fc3611406dbaa18462537e Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Wed, 26 Aug 2026 10:42:38 +0200 Subject: [PATCH 20/37] go: add Test Optimization stdlib warmup target --- README.md | 21 ++++++++++++ modules/go/BUILD.bazel | 1 + modules/go/tests/BUILD.bazel | 8 +++++ modules/go/tests/test_macro.bzl | 16 +++++++++ modules/go/topt_go_stdlib.bzl | 60 +++++++++++++++++++++++++++++++++ 5 files changed, 106 insertions(+) create mode 100644 modules/go/topt_go_stdlib.bzl diff --git a/README.md b/README.md index 61f61dda..57e2ecb6 100644 --- a/README.md +++ b/README.md @@ -530,6 +530,27 @@ Go setup. Manual Go callsites should set `orchestrion_mode = "test_optimization"` for standard Go `testing`; the default `general` mode preserves broader generic Orchestrion behavior. +Large consumers can warm the matching instrumented standard library in a +dedicated cache-writing invocation: + +```bzl +load("@datadog-rules-test-optimization-go//:topt_go_stdlib.bzl", "dd_topt_go_stdlib_warmup") + +dd_topt_go_stdlib_warmup(name = "go_stdlib_warmup") +``` + +```bash +bazel build //tools/test_optimization:go_stdlib_warmup \ + --config=test-optimization \ + --config=remote-cache-write +``` + +The named config supplies the same Orchestrion enablement used by +`dd_topt_go_test`; the warmup target selects `test_optimization` mode and +materializes both the instrumented stdlib and its Go build cache. The consumer +remains responsible for restricting remote-cache writes to a trusted CI +invocation. + ### Bzlmod + Python companion (`dd_topt_py_test`) Configure the Python sync extension with `enabled_by_env = True` and put diff --git a/modules/go/BUILD.bazel b/modules/go/BUILD.bazel index d736d37e..a6776139 100644 --- a/modules/go/BUILD.bazel +++ b/modules/go/BUILD.bazel @@ -12,6 +12,7 @@ exports_files( "topt_go_infer.bzl", "topt_go_extension.bzl", "topt_go_orchestrion_repository.bzl", + "topt_go_stdlib.bzl", "topt_go_workspace.bzl", ], visibility = ["//visibility:public"], diff --git a/modules/go/tests/BUILD.bazel b/modules/go/tests/BUILD.bazel index 3e91f4ed..3e338af5 100644 --- a/modules/go/tests/BUILD.bazel +++ b/modules/go/tests/BUILD.bazel @@ -72,6 +72,7 @@ load( "resolve_topt_service_key_missing_target_rule", "resolve_topt_service_key_unknown_failure_test", "resolve_topt_service_key_unknown_target_rule", + "stdlib_warmup_transition_selects_test_optimization_test", "validate_orchestrion_mode_invalid_failure_test", "validate_orchestrion_mode_invalid_target_rule", "validate_orchestrion_mode_test", @@ -776,6 +777,12 @@ orch_transition_forwards_mode_test( timeout = "short", ) +stdlib_warmup_transition_selects_test_optimization_test( + name = "stdlib_warmup_transition_selects_test_optimization_test", + size = "small", + timeout = "short", +) + test_suite( name = "tests", tests = [ @@ -844,6 +851,7 @@ test_suite( ":selector_override_miss_failure_test", ":selector_override_test", ":service_mapping_entries_filters_non_service_test", + ":stdlib_warmup_transition_selects_test_optimization_test", ":validate_orchestrion_mode_invalid_failure_test", ":validate_orchestrion_mode_test", ":validate_test_optimization_pin_files_missing_go_mod_failure_test", diff --git a/modules/go/tests/test_macro.bzl b/modules/go/tests/test_macro.bzl index a35352a1..a9db551b 100644 --- a/modules/go/tests/test_macro.bzl +++ b/modules/go/tests/test_macro.bzl @@ -28,6 +28,10 @@ load( "select_wrapper_output_name_for_tests", "windows_wrapper_content_for_tests", ) +load( + "@datadog-rules-test-optimization-go//:topt_go_stdlib.bzl", + "stdlib_warmup_transition_impl_for_tests", +) load( "@datadog-rules-test-optimization-go//:topt_go_test.bzl", "dd_topt_go_test", @@ -1024,6 +1028,15 @@ def _orch_transition_forwards_mode_test_impl(ctx): asserts.false(env, "@rules_go//go/private/orchestrion:enabled" in result) return unittest.end(env) +def _stdlib_warmup_transition_selects_test_optimization_test_impl(ctx): + """Assert the cache-warm target selects the exact instrumented stdlib.""" + env = unittest.begin(ctx) + result = stdlib_warmup_transition_impl_for_tests(None, None) + asserts.equals(env, 1, len(result)) + asserts.equals(env, "test_optimization", result["@rules_go//go/private/orchestrion:mode"]) + asserts.false(env, "@rules_go//go/private/orchestrion:enabled" in result) + return unittest.end(env) + def _orch_wrapper_materialized_actual_non_windows_test_impl(ctx): """Assert the wrapper target ships transitioned inputs as siblings.""" env = analysistest.begin(ctx) @@ -1206,6 +1219,9 @@ validate_orchestrion_mode_test = unittest.make( orch_transition_forwards_mode_test = unittest.make( _orch_transition_forwards_mode_test_impl, ) +stdlib_warmup_transition_selects_test_optimization_test = unittest.make( + _stdlib_warmup_transition_selects_test_optimization_test_impl, +) orch_wrapper_materialized_actual_non_windows_test = analysistest.make( _orch_wrapper_materialized_actual_non_windows_test_impl, ) diff --git a/modules/go/topt_go_stdlib.bzl b/modules/go/topt_go_stdlib.bzl new file mode 100644 index 00000000..7db5dc32 --- /dev/null +++ b/modules/go/topt_go_stdlib.bzl @@ -0,0 +1,60 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Build target for warming the Test Optimization Go standard library.""" + +load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") +load("@rules_go//go/private:providers.bzl", "GoStdLib") + +_ORCHESTRION_MODE_SETTING = "@rules_go//go/private/orchestrion:mode" +_ORCHESTRION_MODE_TEST_OPTIMIZATION = "test_optimization" + +def _stdlib_warmup_transition_impl(_settings, _attr): + return { + _ORCHESTRION_MODE_SETTING: _ORCHESTRION_MODE_TEST_OPTIMIZATION, + } + +stdlib_warmup_transition_impl_for_tests = _stdlib_warmup_transition_impl + +_stdlib_warmup_transition = transition( + implementation = _stdlib_warmup_transition_impl, + inputs = [], + outputs = [ + _ORCHESTRION_MODE_SETTING, + ], +) + +def _first_target(dep): + if type(dep) == "list": + if not dep: + fail("dd_topt_go_stdlib_warmup: stdlib transition produced no targets") + return dep[0] + return dep + +def _dd_topt_go_stdlib_warmup_impl(ctx): + if not ctx.attr._orchestrion_enabled[BuildSettingInfo].value: + fail("dd_topt_go_stdlib_warmup requires the consumer's Test Optimization Bazel config") + stdlib = _first_target(ctx.attr._stdlib)[GoStdLib] + return [DefaultInfo(files = depset(transitive = [stdlib.libs, stdlib.cache_dir]))] + +dd_topt_go_stdlib_warmup = rule( + implementation = _dd_topt_go_stdlib_warmup_impl, + attrs = { + "_stdlib": attr.label( + default = "@rules_go//:stdlib", + cfg = _stdlib_warmup_transition, + providers = [GoStdLib], + ), + "_orchestrion_enabled": attr.label( + default = "@rules_go//go/private/orchestrion:enabled", + providers = [BuildSettingInfo], + ), + "_allowlist_function_transition": attr.label( + default = "@bazel_tools//tools/allowlists/function_transition_allowlist", + ), + }, + doc = "Builds the Orchestrion-instrumented Go standard library used by Test Optimization.", +) From f9c1453b3467a08f0aad13a38e3030f8cd7534da Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Wed, 26 Aug 2026 11:36:28 +0200 Subject: [PATCH 21/37] go: make disabled stdlib warmup a no-op --- README.md | 2 ++ modules/go/tests/BUILD.bazel | 16 ++++++++++++++++ modules/go/tests/test_macro.bzl | 10 ++++++++++ modules/go/topt_go_stdlib.bzl | 2 +- 4 files changed, 29 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 57e2ecb6..c88f3c9d 100644 --- a/README.md +++ b/README.md @@ -550,6 +550,8 @@ The named config supplies the same Orchestrion enablement used by materializes both the instrumented stdlib and its Go build cache. The consumer remains responsible for restricting remote-cache writes to a trusted CI invocation. +Without the named config, the target is a no-op so ordinary broad builds can +analyze its package without enabling Test Optimization. ### Bzlmod + Python companion (`dd_topt_py_test`) diff --git a/modules/go/tests/BUILD.bazel b/modules/go/tests/BUILD.bazel index 3e338af5..cf34749b 100644 --- a/modules/go/tests/BUILD.bazel +++ b/modules/go/tests/BUILD.bazel @@ -72,6 +72,7 @@ load( "resolve_topt_service_key_missing_target_rule", "resolve_topt_service_key_unknown_failure_test", "resolve_topt_service_key_unknown_target_rule", + "stdlib_warmup_disabled_noop_test", "stdlib_warmup_transition_selects_test_optimization_test", "validate_orchestrion_mode_invalid_failure_test", "validate_orchestrion_mode_invalid_target_rule", @@ -83,6 +84,10 @@ load( "wrapper_output_name_target_rule", "wrapper_output_name_windows_test", ) +load( + "@datadog-rules-test-optimization-go//:topt_go_stdlib.bzl", + "dd_topt_go_stdlib_warmup", +) load( ":test_payloads_selector.bzl", "selector_embed_precedence_target", @@ -783,6 +788,16 @@ stdlib_warmup_transition_selects_test_optimization_test( timeout = "short", ) +dd_topt_go_stdlib_warmup( + name = "stdlib_warmup_disabled_target", + tags = ["manual"], +) + +stdlib_warmup_disabled_noop_test( + name = "stdlib_warmup_disabled_noop_test", + target_under_test = ":stdlib_warmup_disabled_target", +) + test_suite( name = "tests", tests = [ @@ -851,6 +866,7 @@ test_suite( ":selector_override_miss_failure_test", ":selector_override_test", ":service_mapping_entries_filters_non_service_test", + ":stdlib_warmup_disabled_noop_test", ":stdlib_warmup_transition_selects_test_optimization_test", ":validate_orchestrion_mode_invalid_failure_test", ":validate_orchestrion_mode_test", diff --git a/modules/go/tests/test_macro.bzl b/modules/go/tests/test_macro.bzl index a9db551b..59110f8a 100644 --- a/modules/go/tests/test_macro.bzl +++ b/modules/go/tests/test_macro.bzl @@ -1037,6 +1037,13 @@ def _stdlib_warmup_transition_selects_test_optimization_test_impl(ctx): asserts.false(env, "@rules_go//go/private/orchestrion:enabled" in result) return unittest.end(env) +def _stdlib_warmup_disabled_noop_test_impl(ctx): + """Assert broad builds can analyze the warmup target without enablement.""" + env = analysistest.begin(ctx) + target = analysistest.target_under_test(env) + asserts.equals(env, 0, len(target[DefaultInfo].files.to_list())) + return analysistest.end(env) + def _orch_wrapper_materialized_actual_non_windows_test_impl(ctx): """Assert the wrapper target ships transitioned inputs as siblings.""" env = analysistest.begin(ctx) @@ -1222,6 +1229,9 @@ orch_transition_forwards_mode_test = unittest.make( stdlib_warmup_transition_selects_test_optimization_test = unittest.make( _stdlib_warmup_transition_selects_test_optimization_test_impl, ) +stdlib_warmup_disabled_noop_test = analysistest.make( + _stdlib_warmup_disabled_noop_test_impl, +) orch_wrapper_materialized_actual_non_windows_test = analysistest.make( _orch_wrapper_materialized_actual_non_windows_test_impl, ) diff --git a/modules/go/topt_go_stdlib.bzl b/modules/go/topt_go_stdlib.bzl index 7db5dc32..5f051b89 100644 --- a/modules/go/topt_go_stdlib.bzl +++ b/modules/go/topt_go_stdlib.bzl @@ -36,7 +36,7 @@ def _first_target(dep): def _dd_topt_go_stdlib_warmup_impl(ctx): if not ctx.attr._orchestrion_enabled[BuildSettingInfo].value: - fail("dd_topt_go_stdlib_warmup requires the consumer's Test Optimization Bazel config") + return [DefaultInfo()] stdlib = _first_target(ctx.attr._stdlib)[GoStdLib] return [DefaultInfo(files = depset(transitive = [stdlib.libs, stdlib.cache_dir]))] From e18db80304122f522c87c8fd6e8db555ee2d47c2 Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Wed, 26 Aug 2026 12:42:36 +0200 Subject: [PATCH 22/37] go: match go_test transitions in stdlib warmup --- README.md | 5 ++-- modules/go/topt_go_stdlib.bzl | 43 ++++++++++++++++++++++++++++++----- 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index c88f3c9d..a6807874 100644 --- a/README.md +++ b/README.md @@ -546,8 +546,9 @@ bazel build //tools/test_optimization:go_stdlib_warmup \ ``` The named config supplies the same Orchestrion enablement used by -`dd_topt_go_test`; the warmup target selects `test_optimization` mode and -materializes both the instrumented stdlib and its Go build cache. The consumer +`dd_topt_go_test`; the warmup target applies the same Go configuration +transitions, selects `test_optimization` mode, and materializes both the +instrumented stdlib and its Go build cache. The consumer remains responsible for restricting remote-cache writes to a trusted CI invocation. Without the named config, the target is a no-op so ordinary broad builds can diff --git a/modules/go/topt_go_stdlib.bzl b/modules/go/topt_go_stdlib.bzl index 5f051b89..10149f34 100644 --- a/modules/go/topt_go_stdlib.bzl +++ b/modules/go/topt_go_stdlib.bzl @@ -8,6 +8,7 @@ load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") load("@rules_go//go/private:providers.bzl", "GoStdLib") +load("@rules_go//go/private/rules:transition.bzl", "go_transition") _ORCHESTRION_MODE_SETTING = "@rules_go//go/private/orchestrion:mode" _ORCHESTRION_MODE_TEST_OPTIMIZATION = "test_optimization" @@ -34,19 +35,35 @@ def _first_target(dep): return dep[0] return dep +def _go_transition_stdlib_warmup_impl(ctx): + stdlib = ctx.attr._stdlib[GoStdLib] + return [DefaultInfo(files = depset(transitive = [stdlib.libs, stdlib.cache_dir]))] + +_go_transition_stdlib_warmup = rule( + implementation = _go_transition_stdlib_warmup_impl, + attrs = { + "_stdlib": attr.label( + default = "@rules_go//:stdlib", + providers = [GoStdLib], + ), + "_allowlist_function_transition": attr.label( + default = "@bazel_tools//tools/allowlists/function_transition_allowlist", + ), + }, + cfg = go_transition, +) + def _dd_topt_go_stdlib_warmup_impl(ctx): if not ctx.attr._orchestrion_enabled[BuildSettingInfo].value: return [DefaultInfo()] - stdlib = _first_target(ctx.attr._stdlib)[GoStdLib] - return [DefaultInfo(files = depset(transitive = [stdlib.libs, stdlib.cache_dir]))] + return [DefaultInfo(files = _first_target(ctx.attr.actual)[DefaultInfo].files)] -dd_topt_go_stdlib_warmup = rule( +_dd_topt_go_stdlib_warmup = rule( implementation = _dd_topt_go_stdlib_warmup_impl, attrs = { - "_stdlib": attr.label( - default = "@rules_go//:stdlib", + "actual": attr.label( + mandatory = True, cfg = _stdlib_warmup_transition, - providers = [GoStdLib], ), "_orchestrion_enabled": attr.label( default = "@rules_go//go/private/orchestrion:enabled", @@ -58,3 +75,17 @@ dd_topt_go_stdlib_warmup = rule( }, doc = "Builds the Orchestrion-instrumented Go standard library used by Test Optimization.", ) + +def dd_topt_go_stdlib_warmup(name, **kwargs): + """Creates a warmup target with the same Go transitions as a real go_test.""" + transitioned_name = name + "__go_transition" + _go_transition_stdlib_warmup( + name = transitioned_name, + tags = ["manual"], + visibility = ["//visibility:private"], + ) + _dd_topt_go_stdlib_warmup( + name = name, + actual = ":" + transitioned_name, + **kwargs + ) From 564723ae6e8beefb2da7bfc8c4a51651cdd61ed1 Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Wed, 26 Aug 2026 16:37:24 +0200 Subject: [PATCH 23/37] Address remaining review feedback --- README.md | 23 ++- docs/Initial_documentation.md | 8 +- docs/Installation_Reference.md | 14 +- docs/RFC.md | 15 +- docs/Uploader_Reference.md | 21 ++- examples/common/runtests_common.sh | 27 +--- modules/go/tools/dd_topt_go_bootstrap/main.go | 87 +++++++++-- .../tools/dd_topt_go_bootstrap/main_test.go | 80 ++++++++-- modules/go/tools/onboardingpins/pins.go | 6 +- modules/go/tools/onboardingpins/pins_test.go | 5 +- .../python/tools/dd_topt_py_bootstrap/main.py | 21 +-- .../tools/dd_topt_py_bootstrap/main_test.py | 15 +- .../go/tools/builders/orchestrion_cache.go | 82 +++++++++- .../tools/builders/orchestrion_cache_test.go | 42 ++++- .../go/tools/builders/orchestrion_cache.go | 82 +++++++++- .../tools/builders/orchestrion_cache_test.go | 42 ++++- .../go/tools/builders/orchestrion_cache.go | 82 +++++++++- .../tools/builders/orchestrion_cache_test.go | 42 ++++- .../v0_60_0/base/0001-full-delta.patch | 132 ++++++++++++++-- .../v0_61_1/base/0001-full-delta.patch | 132 ++++++++++++++-- .../v0_62_0/base/0001-full-delta.patch | 132 ++++++++++++++-- .../go-test-optimization-onboarding/SKILL.md | 7 +- .../references/validation-checklist.md | 17 +-- .../SKILL.md | 4 +- .../references/validation-checklist.md | 17 +-- .../SKILL.md | 4 +- .../references/validation-checklist.md | 17 +-- tools/core/bep_artifact_stage_helper.py | 28 +++- tools/core/uploader_bash_runtime.sh.tpl | 55 ++++--- .../core/uploader_powershell_runtime.ps1.tpl | 42 +++-- .../run_test_optimization_ci.ps1 | 35 +++-- .../run_test_optimization_ci.sh | 49 +++--- .../integration/run_mock_server_tests.sh | 11 +- tools/tests/python/test_python_tools.py | 143 ++++++++++++++++-- 34 files changed, 1212 insertions(+), 307 deletions(-) diff --git a/README.md b/README.md index a6807874..81e31d14 100644 --- a/README.md +++ b/README.md @@ -286,8 +286,8 @@ RTO_ARCHIVE_TYPE="tar.gz" RULES_GO_UPSTREAM="v0_60_0" RULES_GO_VARIANT="base" RULES_GO_STRIP_PREFIX="third_party/rgo/v0_60_0/base" -DD_TRACE_GO_VERSION="v2.9.1" -ORCHESTRION_VERSION="v1.12.0" +DD_TRACE_GO_VERSION="v2.9.0" +ORCHESTRION_VERSION="v1.9.0" ``` The archive URL, SHA256, and prefix are tied to the repository commit. @@ -1261,11 +1261,11 @@ the fact that the repo still owns the actual refresh command. Bootstrap can also generate an operator-owned validation script for large repositories. The script repeats the RFC flow without hiding Bazel behavior: -`sync -> controls -> instrumented tests -> doctor -> dry-run uploader -> optional upload`. -Upload is disabled unless the operator passes `--upload`. Set -`DD_TEST_OPTIMIZATION_REPORT_DIR` to keep its `doctor-report.json`, -`uploader-dry-run-report.json`, and optional `uploader-upload-report.json` -outside the script's temporary directory. +`sync -> controls -> instrumented tests -> doctor -> validated uploader`. +Without `--upload`, the uploader runs once in dry-run mode. With `--upload`, it +runs once as a real upload and validates enrichment in that same pass. Set +`DD_TEST_OPTIMIZATION_REPORT_DIR` to keep `doctor-report.json` and +`uploader-report.json` outside the script's temporary directory. ```bash bazel run @datadog-rules-test-optimization-go//:dd_topt_go_bootstrap -- \ @@ -1596,10 +1596,9 @@ tools/test_optimization/run_test_optimization_ci.sh \ //... ``` -The wrapper writes `.topt/reports/doctor-report.json` and -`.topt/reports/uploader-dry-run-report.json`. When `--upload` is also used, it -writes the real upload result to `.topt/reports/uploader-upload-report.json` -instead of overwriting the dry-run report. Reports include a `result` block with +The wrapper writes `.topt/reports/doctor-report.json` plus exactly one uploader +report: `uploader-dry-run-report.json` without `--upload`, or +`uploader-upload-report.json` with it. Reports include a `result` block with `status`, `reason_code`, `reason`, `next_steps`, and the exact counts needed to answer why payloads were or were not uploaded: expected/seen targets, BEP fresh/cached/remote-only outputs, artifact staging, discovered payloads, @@ -1609,7 +1608,7 @@ For support tickets, prefer `.topt/reports/dd-test-optimization-support.zip`. For the simplest customer ask, run the doctor with `--support-bundle=` and attach the resulting doctor-only bundle. In CI, prefer the wrapper `--support-bundle=` because it also -includes uploader dry-run and optional upload reports. Bundles are redacted and +includes the selected uploader report. Bundles are redacted and bounded by default and include `summary.md`, selected BEP summaries, effective flags, runtime metadata, and a redaction manifest. They do not include raw payloads, raw CI logs, raw environment variables, or raw BEP files. diff --git a/docs/Initial_documentation.md b/docs/Initial_documentation.md index 8fe2545c..ab91a933 100644 --- a/docs/Initial_documentation.md +++ b/docs/Initial_documentation.md @@ -68,10 +68,10 @@ The steps are: Bazel targets do not have to live in the root package. Small repositories can use root labels; large monorepos should use a lightweight package such as `//tools/test_optimization`. - Usage: run `bazel test`, then the doctor target, then the uploader with - `--dry-run --validate-enrichment`, then the real uploader target. Preserve - the earliest failure, but still let the real uploader process every available - fresh valid payload when upload is enabled. + Usage: run `bazel test`, then the doctor target, then one uploader pass with + `--validate-enrichment`; add `--dry-run` only when upload is disabled. + Preserve the earliest failure while still processing every available fresh + valid payload. 4. **Language macros (optional)**: Thin wrappers (for Go/Python/Java/NodeJS/.NET/Ruby) set up the right runfiles/env so test code can read the synced files and write payloads to `TEST_UNDECLARED_OUTPUTS_DIR`. diff --git a/docs/Installation_Reference.md b/docs/Installation_Reference.md index 2c075739..824dd55c 100644 --- a/docs/Installation_Reference.md +++ b/docs/Installation_Reference.md @@ -371,12 +371,12 @@ bazel run @datadog-rules-test-optimization-go//:dd_topt_go_bootstrap -- \ ``` The generated script runs -`sync -> controls -> instrumented tests -> doctor -> dry-run uploader -> optional upload`. +`sync -> controls -> instrumented tests -> doctor -> validated uploader`. It captures one BEP JSON file per Bazel test invocation and passes those files to doctor/uploader with `--freshness-source=bep --freshness-mode=required`. It -always runs the uploader dry-run with enrichment validation after a successful -doctor, and uploads only when called with `--upload`; the default is -`--no-upload`. It does not delete caches, print secrets, proxy payloads, or pass +runs the uploader exactly once with enrichment validation: dry-run by default, +or real upload when called with `--upload`. It does not delete caches, print +secrets, proxy payloads, or pass `DD_GIT_*` through `--test_env`. The generated test config uses `--zip_undeclared_test_outputs`, and the script passes `--artifact-source=bep` to doctor/uploader so local `outputs.zip` carriers are extracted through BEP @@ -385,9 +385,9 @@ carriers, doctor/uploader can stage them natively with `--remote-artifacts=download` or `required`; bytestream/CAS/custom-auth providers still need `--bep-artifact-downloader`. Set `DD_TEST_OPTIMIZATION_REPORT_DIR` to choose where the -generated script writes `doctor-report.json`, `uploader-dry-run-report.json`, -and, when upload is enabled, `uploader-upload-report.json`; otherwise it writes -those reports under its per-run temporary directory and logs that path. +generated script writes `doctor-report.json` and `uploader-report.json` under +that directory; otherwise it writes them under its per-run temporary directory +and logs that path. ### Go Bazel config diff --git a/docs/RFC.md b/docs/RFC.md index 490747f4..d6b83f55 100644 --- a/docs/RFC.md +++ b/docs/RFC.md @@ -140,10 +140,10 @@ At a high level, the proposal moves all network‑dependent metadata fetching ou - A single workspace-level uploader target (normal rule, not test) runs via `bazel run` after tests complete. - The uploader discovers all `test.outputs/` directories in `bazel-testlogs/`, waits for filesystem quiescence, enriches test payloads with `context.json` (if present), uploads via agentless (`DD_API_KEY`, `DD_SITE`) or an EVP proxy (`DD_TEST_OPTIMIZATION_AGENT_URL`), and deletes successfully uploaded files. - - Usage: run `bazel test`, then `//:dd_test_optimization_doctor`, then - `//:dd_upload_payloads -- --dry-run --validate-enrichment`, then the real - `//:dd_upload_payloads` target. Preserve the earliest failure while still - uploading every available fresh valid payload. + - Usage: run `bazel test`, then `//:dd_test_optimization_doctor`, then one + `//:dd_upload_payloads -- --validate-enrichment` pass. Add `--dry-run` when + upload is disabled. Preserve the earliest failure while still processing + every available fresh valid payload. - [Multi‑service monorepos](../tools/core/test_optimization_multi_sync.bzl): @@ -265,12 +265,7 @@ Runtime Uploader if [ "$test_status" -ne 0 ]; then exit "$test_status"; fi exit "$doctor_status" fi - bazel run --config=test-optimization //:dd_upload_payloads -- --dry-run --validate-enrichment || dry_run_status=$?; dry_run_status=${dry_run_status:-0} - if [ "$dry_run_status" -ne 0 ]; then - if [ "$test_status" -ne 0 ]; then exit "$test_status"; fi - exit "$dry_run_status" - fi - DD_API_KEY="$DD_API_KEY" DD_SITE="$DD_SITE" bazel run --config=test-optimization //:dd_upload_payloads + DD_API_KEY="$DD_API_KEY" DD_SITE="$DD_SITE" bazel run --config=test-optimization //:dd_upload_payloads -- --validate-enrichment upload_status=$? if [ "$test_status" -ne 0 ]; then exit "$test_status"; fi exit "$upload_status" diff --git a/docs/Uploader_Reference.md b/docs/Uploader_Reference.md index c1031f02..9800196a 100644 --- a/docs/Uploader_Reference.md +++ b/docs/Uploader_Reference.md @@ -106,16 +106,16 @@ $env:DD_SITE = "datadoghq.com" //... ``` -Always preserve all statuses. Test failures win, followed by doctor, dry-run, -and uploader failures. When upload is enabled, validation failures do not block -the uploader from processing other fresh valid payloads. +Always preserve all statuses. Test failures win, followed by doctor and uploader +failures. When upload is enabled, enrichment validation happens during the real +upload and does not block processing other fresh valid payloads. Enrichment validation: -- The CI wrappers always run `--dry-run --validate-enrichment` before a real - upload. A normal uploader invocation can instead pass - `--validate-enrichment` without `--dry-run` to validate each enriched source - payload once, immediately before splitting and upload. Manual invocations +- The CI wrappers run the uploader exactly once. Without upload they pass + `--dry-run --validate-enrichment`; with upload they pass + `--validate-enrichment` without `--dry-run`, validating each enriched source + payload once immediately before splitting and upload. Manual invocations should pass the matching `--bep-json=` together with `--freshness-source=bep --freshness-mode=required --artifact-source=bep`. - Dry-run mode does not upload data, does not require `DD_API_KEY` in @@ -327,10 +327,9 @@ tools/test_optimization/run_test_optimization_ci.sh \ //... ``` -The wrapper writes `.topt/reports/doctor-report.json` and -`.topt/reports/uploader-dry-run-report.json`. If upload is enabled, it writes -`.topt/reports/uploader-upload-report.json` for the real upload so the dry-run -report is preserved. +The wrapper writes `.topt/reports/doctor-report.json` plus exactly one uploader +report: `.topt/reports/uploader-dry-run-report.json` without upload, or +`.topt/reports/uploader-upload-report.json` with upload. For first-pass support, the doctor can create a doctor-only bundle without the wrapper: diff --git a/examples/common/runtests_common.sh b/examples/common/runtests_common.sh index f7a46e1a..c94c382f 100644 --- a/examples/common/runtests_common.sh +++ b/examples/common/runtests_common.sh @@ -16,8 +16,7 @@ run_example_runtests() { local bazelw local test_status=0 local doctor_status=0 - local dry_run_status=0 - local upload_status=0 + local uploader_status=0 local tmp_root local artifact_staging_dir bazelw="${script_dir}/../../bazelw" @@ -67,28 +66,16 @@ run_example_runtests() { echo "--- validating payloads" run_cmd "${bazelw}" run --config=test-optimization //:dd_test_optimization_doctor -- "${bep_args[@]}" || doctor_status=$? - if [[ "$doctor_status" -ne 0 ]]; then - if [[ "$test_status" -ne 0 ]]; then - return "$test_status" - fi - return "$doctor_status" - fi - - echo "--- validating upload enrichment" - run_cmd "${bazelw}" run --config=test-optimization //:dd_upload_payloads -- "${bep_args[@]}" --dry-run --validate-enrichment || dry_run_status=$? - if [[ "$dry_run_status" -ne 0 ]]; then - if [[ "$test_status" -ne 0 ]]; then - return "$test_status" - fi - return "$dry_run_status" - fi - echo "--- uploading payloads" + echo "--- validating and uploading payloads" # Requires DD_API_KEY and DD_SITE environment variables. - DD_API_KEY="${DD_API_KEY:-}" DD_SITE="${DD_SITE:-datadoghq.com}" run_cmd "${bazelw}" run --config=test-optimization //:dd_upload_payloads -- "${bep_args[@]}" || upload_status=$? + DD_API_KEY="${DD_API_KEY:-}" DD_SITE="${DD_SITE:-datadoghq.com}" run_cmd "${bazelw}" run --config=test-optimization //:dd_upload_payloads -- "${bep_args[@]}" --validate-enrichment || uploader_status=$? if [[ "$test_status" -ne 0 ]]; then return "$test_status" fi - return "$upload_status" + if [[ "$doctor_status" -ne 0 ]]; then + return "$doctor_status" + fi + return "$uploader_status" } diff --git a/modules/go/tools/dd_topt_go_bootstrap/main.go b/modules/go/tools/dd_topt_go_bootstrap/main.go index 110893ea..9527c030 100644 --- a/modules/go/tools/dd_topt_go_bootstrap/main.go +++ b/modules/go/tools/dd_topt_go_bootstrap/main.go @@ -18,6 +18,7 @@ import ( "path/filepath" "regexp" "runtime" + "strconv" "strings" "github.com/DataDog/rules_test_optimization/modules/go/tools/onboardingpins" @@ -414,6 +415,9 @@ func run(cfg config) error { if err := validateBootstrapMode(cfg); err != nil { return err } + if err := validateGoRuntimeCompatibility(cfg); err != nil { + return err + } if err := validateValidationScriptConfig(cfg); err != nil { return err } @@ -662,6 +666,61 @@ func validateBootstrapMode(cfg config) error { return nil } +// validateGoRuntimeCompatibility rejects managed SDK versions that cannot +// build the configured Orchestrion release before bootstrap writes any files. +func validateGoRuntimeCompatibility(cfg config) error { + runtimeVersion := strings.TrimSpace(cfg.runtimeVersion) + if runtimeVersion == "" { + return nil + } + orchestrionVersion, err := parseVersionTriplet(cfg.orchestrionVersion) + if err != nil { + return fmt.Errorf("invalid Orchestrion version %q: %w", cfg.orchestrionVersion, err) + } + if compareVersionTriplets(orchestrionVersion, [3]int{1, 12, 0}) < 0 { + return nil + } + goVersion, err := parseVersionTriplet(runtimeVersion) + if err != nil { + return fmt.Errorf("invalid Go runtime version %q: %w", runtimeVersion, err) + } + if compareVersionTriplets(goVersion, [3]int{1, 25, 0}) < 0 { + return fmt.Errorf("Orchestrion %s requires Go 1.25.0 or newer; got --runtime-version=%s", cfg.orchestrionVersion, runtimeVersion) + } + return nil +} + +func parseVersionTriplet(value string) ([3]int, error) { + match := regexp.MustCompile(`^[vV]?(\d+)\.(\d+)(?:\.(\d+))?(?:[-+].*)?$`).FindStringSubmatch(strings.TrimSpace(value)) + if match == nil { + return [3]int{}, errors.New("expected a semantic version such as 1.25.0") + } + var parsed [3]int + for i := range parsed { + if match[i+1] == "" { + continue + } + component, err := strconv.Atoi(match[i+1]) + if err != nil { + return [3]int{}, fmt.Errorf("parse numeric component: %w", err) + } + parsed[i] = component + } + return parsed, nil +} + +func compareVersionTriplets(left, right [3]int) int { + for i := range left { + if left[i] < right[i] { + return -1 + } + if left[i] > right[i] { + return 1 + } + } + return 0 +} + // runWorkspaceMode writes only local scaffolding files for WORKSPACE consumers. // It never edits WORKSPACE itself; operators can pair it with the printed // repository snippet and place that snippet where their monorepo expects it. @@ -1474,8 +1533,7 @@ func validationScript(cfg config) (string, error) { buf.WriteString("ARTIFACT_STAGING_DIR=\"\"\n") buf.WriteString("REPORT_DIR=\"${DD_TEST_OPTIMIZATION_REPORT_DIR:-}\"\n") buf.WriteString("DOCTOR_REPORT_JSON=\"\"\n") - buf.WriteString("UPLOADER_DRY_RUN_REPORT_JSON=\"\"\n") - buf.WriteString("UPLOADER_UPLOAD_REPORT_JSON=\"\"\n") + buf.WriteString("UPLOADER_REPORT_JSON=\"\"\n") fmt.Fprintf(&buf, "MIN_FREE_DISK_GB=%d\n", cfg.minFreeDiskGB) fmt.Fprintf(&buf, "LARGE_MONOREPO=%s\n", shellBool(cfg.largeMonorepo)) fmt.Fprintf(&buf, "SHUTDOWN_BAZEL_ON_EXIT=%s\n", shellBool(cfg.shutdownBazelOnExit)) @@ -1502,7 +1560,7 @@ usage() { Usage: validate_go_pilot.sh [--upload|--no-upload] Runs the Datadog Go Test Optimization validation flow: - sync -> controls -> instrumented tests -> doctor -> dry-run uploader -> optional upload + sync -> controls -> instrumented tests -> doctor -> validated uploader Upload is disabled by default. Pass --upload only when local Datadog credentials are already available in the environment. When enabled, every @@ -1562,8 +1620,7 @@ prepare_bep_files() { fi mkdir -p "${BEP_JSON_DIR}" "${ARTIFACT_STAGING_DIR}" "${REPORT_DIR}" DOCTOR_REPORT_JSON="${REPORT_DIR}/doctor-report.json" - UPLOADER_DRY_RUN_REPORT_JSON="${REPORT_DIR}/uploader-dry-run-report.json" - UPLOADER_UPLOAD_REPORT_JSON="${REPORT_DIR}/uploader-upload-report.json" + UPLOADER_REPORT_JSON="${REPORT_DIR}/uploader-report.json" BEP_RUN_ARGS+=("--artifact-staging-dir=${ARTIFACT_STAGING_DIR}") } @@ -1689,23 +1746,21 @@ if (( doctor_status != 0 && final_status == 0 )); then fi check_disk -run_step "dry-run upload ${UPLOAD_TARGET}" "${BAZEL}" run "${RUN_FLAGS[@]}" "${UPLOAD_TARGET}" -- "${BEP_JSON_ARGS[@]}" "${BEP_RUN_ARGS[@]}" "--report-json=${UPLOADER_DRY_RUN_REPORT_JSON}" --dry-run --validate-enrichment -dry_run_status=$? -if (( dry_run_status != 0 && final_status == 0 )); then - final_status=${dry_run_status} -fi - if (( upload == 0 )); then + run_step "dry-run upload ${UPLOAD_TARGET}" "${BAZEL}" run "${RUN_FLAGS[@]}" "${UPLOAD_TARGET}" -- "${BEP_JSON_ARGS[@]}" "${BEP_RUN_ARGS[@]}" "--report-json=${UPLOADER_REPORT_JSON}" --dry-run --validate-enrichment + uploader_status=$? + if (( uploader_status != 0 && final_status == 0 )); then + final_status=${uploader_status} + fi log "upload skipped; rerun with --upload to run ${UPLOAD_TARGET}" log_report_dir exit "${final_status}" fi -check_disk -run_step "upload ${UPLOAD_TARGET}" "${BAZEL}" run "${RUN_FLAGS[@]}" "${UPLOAD_TARGET}" -- "${BEP_JSON_ARGS[@]}" "${BEP_RUN_ARGS[@]}" "--report-json=${UPLOADER_UPLOAD_REPORT_JSON}" -upload_status=$? -if (( upload_status != 0 && final_status == 0 )); then - final_status=${upload_status} +run_step "upload ${UPLOAD_TARGET}" "${BAZEL}" run "${RUN_FLAGS[@]}" "${UPLOAD_TARGET}" -- "${BEP_JSON_ARGS[@]}" "${BEP_RUN_ARGS[@]}" "--report-json=${UPLOADER_REPORT_JSON}" --validate-enrichment +uploader_status=$? +if (( uploader_status != 0 && final_status == 0 )); then + final_status=${uploader_status} fi log_report_dir exit "${final_status}" diff --git a/modules/go/tools/dd_topt_go_bootstrap/main_test.go b/modules/go/tools/dd_topt_go_bootstrap/main_test.go index f97e6147..0020d3b0 100644 --- a/modules/go/tools/dd_topt_go_bootstrap/main_test.go +++ b/modules/go/tools/dd_topt_go_bootstrap/main_test.go @@ -164,6 +164,65 @@ func TestValidateRulesGoVariantRejectsCompleteVariant(t *testing.T) { } } +func TestValidateGoRuntimeCompatibility(t *testing.T) { + tests := []struct { + name string + orchestrion string + runtime string + wantError bool + }{ + {name: "minimum supported", orchestrion: "v1.12.0", runtime: "1.25.0"}, + {name: "newer Go", orchestrion: "v1.12.1", runtime: "1.26"}, + {name: "older Orchestrion", orchestrion: "v1.11.0", runtime: "1.24.9"}, + {name: "runtime omitted", orchestrion: "v1.12.0"}, + {name: "runtime too old", orchestrion: "v1.12.0", runtime: "1.24.9", wantError: true}, + {name: "malformed runtime", orchestrion: "v1.12.0", runtime: "latest", wantError: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := validateGoRuntimeCompatibility(config{ + orchestrionVersion: test.orchestrion, + runtimeVersion: test.runtime, + }) + if (err != nil) != test.wantError { + t.Fatalf("validateGoRuntimeCompatibility() error=%v, wantError=%v", err, test.wantError) + } + }) + } +} + +func TestRunRejectsUnsupportedGoRuntimeBeforeWriting(t *testing.T) { + dir := t.TempDir() + modulePath := filepath.Join(dir, "MODULE.bazel") + const originalModule = "module(name = \"example\")\n" + if err := os.WriteFile(modulePath, []byte(originalModule), 0o644); err != nil { + t.Fatalf("write MODULE.bazel: %v", err) + } + err := run(config{ + workspaceDir: dir, + guided: true, + service: "example", + runtimeVersion: "1.24.9", + orchestrionVersion: "v1.12.0", + rulesGoUpstream: "v0_60_0", + rulesGoVariant: "base", + datadogFetch: defaultDatadogFetch, + rulesGoFetch: defaultRulesGoFetch, + goBinary: defaultGoBinary, + goModSync: defaultGoModSync, + }) + if err == nil || !strings.Contains(err.Error(), "requires Go 1.25.0 or newer") { + t.Fatalf("run() error=%v, want minimum Go version error", err) + } + got, readErr := os.ReadFile(modulePath) + if readErr != nil { + t.Fatalf("read MODULE.bazel: %v", readErr) + } + if string(got) != originalModule { + t.Fatalf("MODULE.bazel changed before runtime validation:\n%s", got) + } +} + func TestWorkspaceSnippetSupportsMixedFetchModes(t *testing.T) { cfg := config{ rulesGoRemote: "https://github.com/example/repo.git", @@ -740,11 +799,10 @@ func TestValidationScriptUsesConfiguredFlowAndUploadOptIn(t *testing.T) { `--artifact-source=bep`, `--artifact-staging-dir=${ARTIFACT_STAGING_DIR}`, `doctor-report.json`, - `uploader-dry-run-report.json`, - `uploader-upload-report.json`, + `uploader-report.json`, `--report-json`, `mktemp -d "${tmp_parent%/}/dd-go-topt.XXXXXX"`, - `sync -> controls -> instrumented tests -> doctor -> dry-run uploader -> optional upload`, + `sync -> controls -> instrumented tests -> doctor -> validated uploader`, `validate ordinary no-config bootstrap`, `validate explicit disabled precedence`, `query "@${SYNC_REPO}//:test_optimization_files"`, @@ -978,8 +1036,7 @@ set -euo pipefail "test --config=test-optimization --build_event_json_file=" + twoBEPPath + " //pkg:two_test", "run --config=test-optimization //:dd_test_optimization_doctor -- --bep-json=" + controlBEPPath + " --bep-json=" + oneBEPPath + " --bep-json=" + twoBEPPath + " --freshness-source=bep --freshness-mode=required --artifact-source=bep --artifact-staging-dir=", "run --config=test-optimization //:dd_upload_payloads -- --bep-json=" + controlBEPPath + " --bep-json=" + oneBEPPath + " --bep-json=" + twoBEPPath + " --freshness-source=bep --freshness-mode=required --artifact-source=bep --artifact-staging-dir=", - "uploader-dry-run-report.json --dry-run --validate-enrichment", - "uploader-upload-report.json", + "uploader-report.json --validate-enrichment", } { if !strings.Contains(logText, want) { t.Fatalf("fake bazel log missing %q:\n%s\nscript output:\n%s", want, logText, output) @@ -1057,12 +1114,12 @@ exit 0 t.Fatalf("read fake bazel log: %v", err) } logText := string(logBytes) - if got := strings.Count(logText, "run --config=test-optimization //:dd_upload_payloads"); got != 2 { - t.Fatalf("uploader command count=%d, want dry-run and real upload:\n%s", got, logText) + if got := strings.Count(logText, "run --config=test-optimization //:dd_upload_payloads"); got != 1 { + t.Fatalf("uploader command count=%d, want one validated real upload:\n%s", got, logText) } - if !strings.Contains(logText, "--dry-run --validate-enrichment") || - !strings.Contains(logText, "uploader-upload-report.json") { - t.Fatalf("validation script did not run both uploader phases:\n%s", logText) + if strings.Contains(logText, "--dry-run") || + !strings.Contains(logText, "uploader-report.json --validate-enrichment") { + t.Fatalf("validation script did not run one validated real upload:\n%s", logText) } } @@ -1110,8 +1167,7 @@ func TestValidationScriptUsesBepArtifactSourceForZippedOutputs(t *testing.T) { "run --config=test-optimization //:dd_test_optimization_doctor -- --bep-json=", "--freshness-source=bep --freshness-mode=required --artifact-source=bep", "run --config=test-optimization //:dd_upload_payloads -- --bep-json=", - "uploader-dry-run-report.json --dry-run --validate-enrichment", - "uploader-upload-report.json", + "uploader-report.json --validate-enrichment", } { if !strings.Contains(logText, want) { t.Fatalf("fake bazel log missing %q:\n%s\nscript output:\n%s", want, logText, output) diff --git a/modules/go/tools/onboardingpins/pins.go b/modules/go/tools/onboardingpins/pins.go index aa254c91..66589649 100644 --- a/modules/go/tools/onboardingpins/pins.go +++ b/modules/go/tools/onboardingpins/pins.go @@ -204,9 +204,9 @@ func FormatMarkdownSummary(pins Pins) string { buf.WriteString("## Recommended Flow\n\n") buf.WriteString("1. Wire the published repositories using the tuple above.\n") buf.WriteString("2. Run tests with the Test Optimization Bazel config so JSON payloads are downloaded locally.\n") - buf.WriteString("3. Run the doctor target, then the uploader dry-run with enrichment validation.\n") - buf.WriteString("4. Upload only after tests, doctor, and dry-run pass, with credentials available only to `bazel run`.\n") - buf.WriteString("5. In CI, set `DD_TEST_OPTIMIZATION_REPORT_DIR` or wrapper `--report-dir` so `doctor-report.json`, `uploader-dry-run-report.json`, and optional `uploader-upload-report.json` are archived separately.\n") + buf.WriteString("3. Run the doctor target, then one uploader pass with enrichment validation.\n") + buf.WriteString("4. Enable real upload only with credentials available to `bazel run`; otherwise keep that uploader pass in dry-run mode.\n") + buf.WriteString("5. In CI, set `DD_TEST_OPTIMIZATION_REPORT_DIR` or wrapper `--report-dir` so `doctor-report.json` and the selected uploader report are archived.\n") return buf.String() } diff --git a/modules/go/tools/onboardingpins/pins_test.go b/modules/go/tools/onboardingpins/pins_test.go index 915c1bb6..b2a65eda 100644 --- a/modules/go/tools/onboardingpins/pins_test.go +++ b/modules/go/tools/onboardingpins/pins_test.go @@ -298,10 +298,9 @@ func TestSummaryContainsCurrentCommitOnly(t *testing.T) { t.Fatalf("summary contains stale commit %s:\n%s", staleCommit, got) } for _, want := range []string{ - "uploader dry-run with enrichment validation", + "one uploader pass with enrichment validation", "DD_TEST_OPTIMIZATION_REPORT_DIR", - "uploader-dry-run-report.json", - "uploader-upload-report.json", + "selected uploader report", } { if !strings.Contains(got, want) { t.Fatalf("summary missing %q:\n%s", want, got) diff --git a/modules/python/tools/dd_topt_py_bootstrap/main.py b/modules/python/tools/dd_topt_py_bootstrap/main.py index b2be07a9..a06a43f0 100644 --- a/modules/python/tools/dd_topt_py_bootstrap/main.py +++ b/modules/python/tools/dd_topt_py_bootstrap/main.py @@ -405,7 +405,7 @@ def _test_targets_for_commands(args: argparse.Namespace) -> tuple[list[str], str def render_command_snippet(args: argparse.Namespace) -> str: - """Render the normal test, doctor, dry-run, and upload command flow.""" + """Render the normal test, doctor, and validated upload command flow.""" targets, comment = _test_targets_for_commands(args) target_args = " ".join(targets) label_prefix = f"//{args.test_optimization_package}" if args.test_optimization_package else "//" @@ -413,8 +413,7 @@ def render_command_snippet(args: argparse.Namespace) -> str: f""" {comment}test_status=0 doctor_status=0 - dry_run_status=0 - upload_status=0 + uploader_status=0 export DD_TEST_OPTIMIZATION_BEP_JSON=.topt/bazel-bep.json export DD_TEST_OPTIMIZATION_FRESHNESS_SOURCE=bep export DD_TEST_OPTIMIZATION_FRESHNESS_MODE=required @@ -426,20 +425,10 @@ def render_command_snippet(args: argparse.Namespace) -> str: {args.bazel_command} test --config={args.bazelrc_config} --build_event_json_file="$DD_TEST_OPTIMIZATION_BEP_JSON" {target_args} || test_status=$? {args.bazel_command} run --config={args.bazelrc_config} {label_prefix}:{args.doctor_name} || doctor_status=$? - if [ "$doctor_status" -ne 0 ]; then - if [ "$test_status" -ne 0 ]; then exit "$test_status"; fi - exit "$doctor_status" - fi - - {args.bazel_command} run --config={args.bazelrc_config} {label_prefix}:{args.uploader_name} -- --dry-run --validate-enrichment || dry_run_status=$? - if [ "$dry_run_status" -ne 0 ]; then - if [ "$test_status" -ne 0 ]; then exit "$test_status"; fi - exit "$dry_run_status" - fi - - DD_API_KEY="$DD_API_KEY" DD_SITE="$DD_SITE" {args.bazel_command} run --config={args.bazelrc_config} {label_prefix}:{args.uploader_name} || upload_status=$? + DD_API_KEY="$DD_API_KEY" DD_SITE="$DD_SITE" {args.bazel_command} run --config={args.bazelrc_config} {label_prefix}:{args.uploader_name} -- --validate-enrichment || uploader_status=$? if [ "$test_status" -ne 0 ]; then exit "$test_status"; fi - exit "$upload_status" + if [ "$doctor_status" -ne 0 ]; then exit "$doctor_status"; fi + exit "$uploader_status" """ ).strip() + "\n" diff --git a/modules/python/tools/dd_topt_py_bootstrap/main_test.py b/modules/python/tools/dd_topt_py_bootstrap/main_test.py index 7d0f6364..9842567d 100644 --- a/modules/python/tools/dd_topt_py_bootstrap/main_test.py +++ b/modules/python/tools/dd_topt_py_bootstrap/main_test.py @@ -143,18 +143,15 @@ def test_command_snippet_honors_bazel_command_and_test_targets(self) -> None: self.assertIn("--build_event_json_file=\"$DD_TEST_OPTIMIZATION_BEP_JSON\" //app:explicit", snippet) self.assertNotIn("//app:expected", snippet) - def test_command_snippet_gates_real_upload_on_validation(self) -> None: - """Generated commands do not run the real upload after validation failures.""" + def test_command_snippet_runs_one_validated_upload_after_doctor(self) -> None: + """Generated commands preserve failures while running one validated upload.""" args = _args("--expected-target=//app:expected") snippet = main.render_command_snippet(args) self.assertIn("doctor_status=0", snippet) - self.assertIn('if [ "$doctor_status" -ne 0 ]; then', snippet) - self.assertIn("dry_run_status=0", snippet) - self.assertIn('if [ "$dry_run_status" -ne 0 ]; then', snippet) - self.assertLess( - snippet.index("--dry-run --validate-enrichment || dry_run_status=$?"), - snippet.index('DD_API_KEY="$DD_API_KEY"'), - ) + self.assertIn("uploader_status=0", snippet) + self.assertIn('-- --validate-enrichment || uploader_status=$?', snippet) + self.assertNotIn("--dry-run", snippet) + self.assertLess(snippet.index(":dd_test_optimization_doctor"), snippet.index(":dd_upload_payloads")) def test_command_snippet_falls_back_to_expected_targets(self) -> None: """Expected targets are reused for commands when test targets are omitted.""" diff --git a/third_party/rgo/v0_60_0/base/go/tools/builders/orchestrion_cache.go b/third_party/rgo/v0_60_0/base/go/tools/builders/orchestrion_cache.go index 746ac80c..5cccefd7 100644 --- a/third_party/rgo/v0_60_0/base/go/tools/builders/orchestrion_cache.go +++ b/third_party/rgo/v0_60_0/base/go/tools/builders/orchestrion_cache.go @@ -1,7 +1,9 @@ package main import ( + "crypto/rand" "crypto/sha256" + "encoding/hex" "encoding/json" "fmt" "os" @@ -31,6 +33,7 @@ const ( cacheManifestFileName = "manifest.json" cacheReadyFileName = "ready" + cacheLockOwnerPrefix = "owner-" cacheLockPollInterval = 200 * time.Millisecond cacheLockTimeout = 60 * time.Second @@ -71,7 +74,7 @@ func acquireCacheLock(lockDir string, timeout, staleAfter time.Duration) (func() func acquireCacheLockWithTimings(lockDir string, timeout, staleAfter, pollInterval time.Duration) (func(), error) { retryAfterStaleRemoval := true for { - release, err := tryAcquireCacheLock(lockDir) + release, err := tryAcquireCacheLock(lockDir, staleAfter) if err == nil { return release, nil } @@ -82,7 +85,7 @@ func acquireCacheLockWithTimings(lockDir string, timeout, staleAfter, pollInterv deadline := time.Now().Add(timeout) for time.Now().Before(deadline) { time.Sleep(pollInterval) - release, err = tryAcquireCacheLock(lockDir) + release, err = tryAcquireCacheLock(lockDir, staleAfter) if err == nil { return release, nil } @@ -97,7 +100,7 @@ func acquireCacheLockWithTimings(lockDir string, timeout, staleAfter, pollInterv } if stale && retryAfterStaleRemoval { retryAfterStaleRemoval = false - if err := os.RemoveAll(lockDir); err != nil && !os.IsNotExist(err) { + if err := removeStaleCacheLock(lockDir); err != nil && !os.IsNotExist(err) { return nil, fmt.Errorf("remove stale cache lock %s: %w", lockDir, err) } continue @@ -109,18 +112,87 @@ func acquireCacheLockWithTimings(lockDir string, timeout, staleAfter, pollInterv } } -func tryAcquireCacheLock(lockDir string) (func(), error) { +func tryAcquireCacheLock(lockDir string, staleAfter time.Duration) (func(), error) { if err := os.MkdirAll(filepath.Dir(lockDir), 0o755); err != nil { return nil, err } if err := os.Mkdir(lockDir, 0o755); err != nil { return nil, err } + token, err := cacheLockToken() + if err != nil { + _ = os.Remove(lockDir) + return nil, err + } + ownerPath := filepath.Join(lockDir, cacheLockOwnerPrefix+token) + if err := os.WriteFile(ownerPath, []byte(token+"\n"), 0o600); err != nil { + _ = os.Remove(lockDir) + return nil, err + } + stopHeartbeat := make(chan struct{}) + heartbeatDone := make(chan struct{}) + go maintainCacheLockHeartbeat(lockDir, ownerPath, staleAfter, stopHeartbeat, heartbeatDone) + released := false return func() { - _ = os.RemoveAll(lockDir) + if released { + return + } + released = true + close(stopHeartbeat) + <-heartbeatDone + if err := os.Remove(ownerPath); err != nil { + return + } + _ = os.Remove(lockDir) }, nil } +func cacheLockToken() (string, error) { + var data [16]byte + if _, err := rand.Read(data[:]); err != nil { + return "", err + } + return hex.EncodeToString(data[:]), nil +} + +func maintainCacheLockHeartbeat(lockDir, ownerPath string, staleAfter time.Duration, stop <-chan struct{}, done chan<- struct{}) { + defer close(done) + interval := staleAfter / 4 + if interval < time.Millisecond { + interval = time.Millisecond + } + if interval > time.Minute { + interval = time.Minute + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-stop: + return + case now := <-ticker.C: + if err := os.Chtimes(ownerPath, now, now); err != nil { + return + } + if err := os.Chtimes(lockDir, now, now); err != nil { + return + } + } + } +} + +func removeStaleCacheLock(lockDir string) error { + token, err := cacheLockToken() + if err != nil { + return err + } + staleDir := lockDir + ".stale-" + token + if err := os.Rename(lockDir, staleDir); err != nil { + return err + } + return os.RemoveAll(staleDir) +} + func cacheLockIsStale(lockDir string, staleAfter time.Duration) (bool, error) { info, err := os.Stat(lockDir) if err != nil { diff --git a/third_party/rgo/v0_60_0/base/go/tools/builders/orchestrion_cache_test.go b/third_party/rgo/v0_60_0/base/go/tools/builders/orchestrion_cache_test.go index 5bc0af2e..4e7f2bcf 100644 --- a/third_party/rgo/v0_60_0/base/go/tools/builders/orchestrion_cache_test.go +++ b/third_party/rgo/v0_60_0/base/go/tools/builders/orchestrion_cache_test.go @@ -86,7 +86,7 @@ func TestAcquireCacheLockReplacesStaleLock(t *testing.T) { func TestAcquireCacheLockWaitsForActiveOwnerPastTimeout(t *testing.T) { lockDir := filepath.Join(t.TempDir(), "cache.lock") - releaseOwner, err := tryAcquireCacheLock(lockDir) + releaseOwner, err := tryAcquireCacheLock(lockDir, time.Minute) if err != nil { t.Fatalf("acquire owner lock: %v", err) } @@ -102,6 +102,46 @@ func TestAcquireCacheLockWaitsForActiveOwnerPastTimeout(t *testing.T) { releaseWaiter() } +func TestAcquireCacheLockDoesNotStealLiveOwnerPastStaleThreshold(t *testing.T) { + lockDir := filepath.Join(t.TempDir(), "cache.lock") + releaseOwner, err := tryAcquireCacheLock(lockDir, 20*time.Millisecond) + if err != nil { + t.Fatalf("acquire owner lock: %v", err) + } + go func() { + time.Sleep(80 * time.Millisecond) + releaseOwner() + }() + + releaseWaiter, err := acquireCacheLockWithTimings(lockDir, 15*time.Millisecond, 20*time.Millisecond, 2*time.Millisecond) + if err != nil { + t.Fatalf("wait for live owner: %v", err) + } + releaseWaiter() +} + +func TestOldCacheLockReleaseDoesNotRemoveReplacementOwner(t *testing.T) { + root := t.TempDir() + lockDir := filepath.Join(root, "cache.lock") + releaseOld, err := tryAcquireCacheLock(lockDir, time.Minute) + if err != nil { + t.Fatalf("acquire old lock: %v", err) + } + abandonedDir := filepath.Join(root, "abandoned.lock") + if err := os.Rename(lockDir, abandonedDir); err != nil { + t.Fatalf("rename old lock: %v", err) + } + releaseReplacement, err := tryAcquireCacheLock(lockDir, time.Minute) + if err != nil { + t.Fatalf("acquire replacement lock: %v", err) + } + releaseOld() + if _, err := os.Stat(lockDir); err != nil { + t.Fatalf("old release removed replacement lock: %v", err) + } + releaseReplacement() +} + func TestWriteFileAtomically(t *testing.T) { path := filepath.Join(t.TempDir(), "nested", "manifest.json") if err := writeFileAtomically(path, []byte("payload\n"), 0o644); err != nil { diff --git a/third_party/rgo/v0_61_1/base/go/tools/builders/orchestrion_cache.go b/third_party/rgo/v0_61_1/base/go/tools/builders/orchestrion_cache.go index 746ac80c..5cccefd7 100644 --- a/third_party/rgo/v0_61_1/base/go/tools/builders/orchestrion_cache.go +++ b/third_party/rgo/v0_61_1/base/go/tools/builders/orchestrion_cache.go @@ -1,7 +1,9 @@ package main import ( + "crypto/rand" "crypto/sha256" + "encoding/hex" "encoding/json" "fmt" "os" @@ -31,6 +33,7 @@ const ( cacheManifestFileName = "manifest.json" cacheReadyFileName = "ready" + cacheLockOwnerPrefix = "owner-" cacheLockPollInterval = 200 * time.Millisecond cacheLockTimeout = 60 * time.Second @@ -71,7 +74,7 @@ func acquireCacheLock(lockDir string, timeout, staleAfter time.Duration) (func() func acquireCacheLockWithTimings(lockDir string, timeout, staleAfter, pollInterval time.Duration) (func(), error) { retryAfterStaleRemoval := true for { - release, err := tryAcquireCacheLock(lockDir) + release, err := tryAcquireCacheLock(lockDir, staleAfter) if err == nil { return release, nil } @@ -82,7 +85,7 @@ func acquireCacheLockWithTimings(lockDir string, timeout, staleAfter, pollInterv deadline := time.Now().Add(timeout) for time.Now().Before(deadline) { time.Sleep(pollInterval) - release, err = tryAcquireCacheLock(lockDir) + release, err = tryAcquireCacheLock(lockDir, staleAfter) if err == nil { return release, nil } @@ -97,7 +100,7 @@ func acquireCacheLockWithTimings(lockDir string, timeout, staleAfter, pollInterv } if stale && retryAfterStaleRemoval { retryAfterStaleRemoval = false - if err := os.RemoveAll(lockDir); err != nil && !os.IsNotExist(err) { + if err := removeStaleCacheLock(lockDir); err != nil && !os.IsNotExist(err) { return nil, fmt.Errorf("remove stale cache lock %s: %w", lockDir, err) } continue @@ -109,18 +112,87 @@ func acquireCacheLockWithTimings(lockDir string, timeout, staleAfter, pollInterv } } -func tryAcquireCacheLock(lockDir string) (func(), error) { +func tryAcquireCacheLock(lockDir string, staleAfter time.Duration) (func(), error) { if err := os.MkdirAll(filepath.Dir(lockDir), 0o755); err != nil { return nil, err } if err := os.Mkdir(lockDir, 0o755); err != nil { return nil, err } + token, err := cacheLockToken() + if err != nil { + _ = os.Remove(lockDir) + return nil, err + } + ownerPath := filepath.Join(lockDir, cacheLockOwnerPrefix+token) + if err := os.WriteFile(ownerPath, []byte(token+"\n"), 0o600); err != nil { + _ = os.Remove(lockDir) + return nil, err + } + stopHeartbeat := make(chan struct{}) + heartbeatDone := make(chan struct{}) + go maintainCacheLockHeartbeat(lockDir, ownerPath, staleAfter, stopHeartbeat, heartbeatDone) + released := false return func() { - _ = os.RemoveAll(lockDir) + if released { + return + } + released = true + close(stopHeartbeat) + <-heartbeatDone + if err := os.Remove(ownerPath); err != nil { + return + } + _ = os.Remove(lockDir) }, nil } +func cacheLockToken() (string, error) { + var data [16]byte + if _, err := rand.Read(data[:]); err != nil { + return "", err + } + return hex.EncodeToString(data[:]), nil +} + +func maintainCacheLockHeartbeat(lockDir, ownerPath string, staleAfter time.Duration, stop <-chan struct{}, done chan<- struct{}) { + defer close(done) + interval := staleAfter / 4 + if interval < time.Millisecond { + interval = time.Millisecond + } + if interval > time.Minute { + interval = time.Minute + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-stop: + return + case now := <-ticker.C: + if err := os.Chtimes(ownerPath, now, now); err != nil { + return + } + if err := os.Chtimes(lockDir, now, now); err != nil { + return + } + } + } +} + +func removeStaleCacheLock(lockDir string) error { + token, err := cacheLockToken() + if err != nil { + return err + } + staleDir := lockDir + ".stale-" + token + if err := os.Rename(lockDir, staleDir); err != nil { + return err + } + return os.RemoveAll(staleDir) +} + func cacheLockIsStale(lockDir string, staleAfter time.Duration) (bool, error) { info, err := os.Stat(lockDir) if err != nil { diff --git a/third_party/rgo/v0_61_1/base/go/tools/builders/orchestrion_cache_test.go b/third_party/rgo/v0_61_1/base/go/tools/builders/orchestrion_cache_test.go index 5bc0af2e..4e7f2bcf 100644 --- a/third_party/rgo/v0_61_1/base/go/tools/builders/orchestrion_cache_test.go +++ b/third_party/rgo/v0_61_1/base/go/tools/builders/orchestrion_cache_test.go @@ -86,7 +86,7 @@ func TestAcquireCacheLockReplacesStaleLock(t *testing.T) { func TestAcquireCacheLockWaitsForActiveOwnerPastTimeout(t *testing.T) { lockDir := filepath.Join(t.TempDir(), "cache.lock") - releaseOwner, err := tryAcquireCacheLock(lockDir) + releaseOwner, err := tryAcquireCacheLock(lockDir, time.Minute) if err != nil { t.Fatalf("acquire owner lock: %v", err) } @@ -102,6 +102,46 @@ func TestAcquireCacheLockWaitsForActiveOwnerPastTimeout(t *testing.T) { releaseWaiter() } +func TestAcquireCacheLockDoesNotStealLiveOwnerPastStaleThreshold(t *testing.T) { + lockDir := filepath.Join(t.TempDir(), "cache.lock") + releaseOwner, err := tryAcquireCacheLock(lockDir, 20*time.Millisecond) + if err != nil { + t.Fatalf("acquire owner lock: %v", err) + } + go func() { + time.Sleep(80 * time.Millisecond) + releaseOwner() + }() + + releaseWaiter, err := acquireCacheLockWithTimings(lockDir, 15*time.Millisecond, 20*time.Millisecond, 2*time.Millisecond) + if err != nil { + t.Fatalf("wait for live owner: %v", err) + } + releaseWaiter() +} + +func TestOldCacheLockReleaseDoesNotRemoveReplacementOwner(t *testing.T) { + root := t.TempDir() + lockDir := filepath.Join(root, "cache.lock") + releaseOld, err := tryAcquireCacheLock(lockDir, time.Minute) + if err != nil { + t.Fatalf("acquire old lock: %v", err) + } + abandonedDir := filepath.Join(root, "abandoned.lock") + if err := os.Rename(lockDir, abandonedDir); err != nil { + t.Fatalf("rename old lock: %v", err) + } + releaseReplacement, err := tryAcquireCacheLock(lockDir, time.Minute) + if err != nil { + t.Fatalf("acquire replacement lock: %v", err) + } + releaseOld() + if _, err := os.Stat(lockDir); err != nil { + t.Fatalf("old release removed replacement lock: %v", err) + } + releaseReplacement() +} + func TestWriteFileAtomically(t *testing.T) { path := filepath.Join(t.TempDir(), "nested", "manifest.json") if err := writeFileAtomically(path, []byte("payload\n"), 0o644); err != nil { diff --git a/third_party/rgo/v0_62_0/base/go/tools/builders/orchestrion_cache.go b/third_party/rgo/v0_62_0/base/go/tools/builders/orchestrion_cache.go index 746ac80c..5cccefd7 100644 --- a/third_party/rgo/v0_62_0/base/go/tools/builders/orchestrion_cache.go +++ b/third_party/rgo/v0_62_0/base/go/tools/builders/orchestrion_cache.go @@ -1,7 +1,9 @@ package main import ( + "crypto/rand" "crypto/sha256" + "encoding/hex" "encoding/json" "fmt" "os" @@ -31,6 +33,7 @@ const ( cacheManifestFileName = "manifest.json" cacheReadyFileName = "ready" + cacheLockOwnerPrefix = "owner-" cacheLockPollInterval = 200 * time.Millisecond cacheLockTimeout = 60 * time.Second @@ -71,7 +74,7 @@ func acquireCacheLock(lockDir string, timeout, staleAfter time.Duration) (func() func acquireCacheLockWithTimings(lockDir string, timeout, staleAfter, pollInterval time.Duration) (func(), error) { retryAfterStaleRemoval := true for { - release, err := tryAcquireCacheLock(lockDir) + release, err := tryAcquireCacheLock(lockDir, staleAfter) if err == nil { return release, nil } @@ -82,7 +85,7 @@ func acquireCacheLockWithTimings(lockDir string, timeout, staleAfter, pollInterv deadline := time.Now().Add(timeout) for time.Now().Before(deadline) { time.Sleep(pollInterval) - release, err = tryAcquireCacheLock(lockDir) + release, err = tryAcquireCacheLock(lockDir, staleAfter) if err == nil { return release, nil } @@ -97,7 +100,7 @@ func acquireCacheLockWithTimings(lockDir string, timeout, staleAfter, pollInterv } if stale && retryAfterStaleRemoval { retryAfterStaleRemoval = false - if err := os.RemoveAll(lockDir); err != nil && !os.IsNotExist(err) { + if err := removeStaleCacheLock(lockDir); err != nil && !os.IsNotExist(err) { return nil, fmt.Errorf("remove stale cache lock %s: %w", lockDir, err) } continue @@ -109,18 +112,87 @@ func acquireCacheLockWithTimings(lockDir string, timeout, staleAfter, pollInterv } } -func tryAcquireCacheLock(lockDir string) (func(), error) { +func tryAcquireCacheLock(lockDir string, staleAfter time.Duration) (func(), error) { if err := os.MkdirAll(filepath.Dir(lockDir), 0o755); err != nil { return nil, err } if err := os.Mkdir(lockDir, 0o755); err != nil { return nil, err } + token, err := cacheLockToken() + if err != nil { + _ = os.Remove(lockDir) + return nil, err + } + ownerPath := filepath.Join(lockDir, cacheLockOwnerPrefix+token) + if err := os.WriteFile(ownerPath, []byte(token+"\n"), 0o600); err != nil { + _ = os.Remove(lockDir) + return nil, err + } + stopHeartbeat := make(chan struct{}) + heartbeatDone := make(chan struct{}) + go maintainCacheLockHeartbeat(lockDir, ownerPath, staleAfter, stopHeartbeat, heartbeatDone) + released := false return func() { - _ = os.RemoveAll(lockDir) + if released { + return + } + released = true + close(stopHeartbeat) + <-heartbeatDone + if err := os.Remove(ownerPath); err != nil { + return + } + _ = os.Remove(lockDir) }, nil } +func cacheLockToken() (string, error) { + var data [16]byte + if _, err := rand.Read(data[:]); err != nil { + return "", err + } + return hex.EncodeToString(data[:]), nil +} + +func maintainCacheLockHeartbeat(lockDir, ownerPath string, staleAfter time.Duration, stop <-chan struct{}, done chan<- struct{}) { + defer close(done) + interval := staleAfter / 4 + if interval < time.Millisecond { + interval = time.Millisecond + } + if interval > time.Minute { + interval = time.Minute + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-stop: + return + case now := <-ticker.C: + if err := os.Chtimes(ownerPath, now, now); err != nil { + return + } + if err := os.Chtimes(lockDir, now, now); err != nil { + return + } + } + } +} + +func removeStaleCacheLock(lockDir string) error { + token, err := cacheLockToken() + if err != nil { + return err + } + staleDir := lockDir + ".stale-" + token + if err := os.Rename(lockDir, staleDir); err != nil { + return err + } + return os.RemoveAll(staleDir) +} + func cacheLockIsStale(lockDir string, staleAfter time.Duration) (bool, error) { info, err := os.Stat(lockDir) if err != nil { diff --git a/third_party/rgo/v0_62_0/base/go/tools/builders/orchestrion_cache_test.go b/third_party/rgo/v0_62_0/base/go/tools/builders/orchestrion_cache_test.go index 5bc0af2e..4e7f2bcf 100644 --- a/third_party/rgo/v0_62_0/base/go/tools/builders/orchestrion_cache_test.go +++ b/third_party/rgo/v0_62_0/base/go/tools/builders/orchestrion_cache_test.go @@ -86,7 +86,7 @@ func TestAcquireCacheLockReplacesStaleLock(t *testing.T) { func TestAcquireCacheLockWaitsForActiveOwnerPastTimeout(t *testing.T) { lockDir := filepath.Join(t.TempDir(), "cache.lock") - releaseOwner, err := tryAcquireCacheLock(lockDir) + releaseOwner, err := tryAcquireCacheLock(lockDir, time.Minute) if err != nil { t.Fatalf("acquire owner lock: %v", err) } @@ -102,6 +102,46 @@ func TestAcquireCacheLockWaitsForActiveOwnerPastTimeout(t *testing.T) { releaseWaiter() } +func TestAcquireCacheLockDoesNotStealLiveOwnerPastStaleThreshold(t *testing.T) { + lockDir := filepath.Join(t.TempDir(), "cache.lock") + releaseOwner, err := tryAcquireCacheLock(lockDir, 20*time.Millisecond) + if err != nil { + t.Fatalf("acquire owner lock: %v", err) + } + go func() { + time.Sleep(80 * time.Millisecond) + releaseOwner() + }() + + releaseWaiter, err := acquireCacheLockWithTimings(lockDir, 15*time.Millisecond, 20*time.Millisecond, 2*time.Millisecond) + if err != nil { + t.Fatalf("wait for live owner: %v", err) + } + releaseWaiter() +} + +func TestOldCacheLockReleaseDoesNotRemoveReplacementOwner(t *testing.T) { + root := t.TempDir() + lockDir := filepath.Join(root, "cache.lock") + releaseOld, err := tryAcquireCacheLock(lockDir, time.Minute) + if err != nil { + t.Fatalf("acquire old lock: %v", err) + } + abandonedDir := filepath.Join(root, "abandoned.lock") + if err := os.Rename(lockDir, abandonedDir); err != nil { + t.Fatalf("rename old lock: %v", err) + } + releaseReplacement, err := tryAcquireCacheLock(lockDir, time.Minute) + if err != nil { + t.Fatalf("acquire replacement lock: %v", err) + } + releaseOld() + if _, err := os.Stat(lockDir); err != nil { + t.Fatalf("old release removed replacement lock: %v", err) + } + releaseReplacement() +} + func TestWriteFileAtomically(t *testing.T) { path := filepath.Join(t.TempDir(), "nested", "manifest.json") if err := writeFileAtomically(path, []byte("payload\n"), 0o644); err != nil { diff --git a/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch index b5e99598..0025eb8f 100644 --- a/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch @@ -12851,14 +12851,16 @@ index 0000000..93f9e48 +} diff --git a/go/tools/builders/orchestrion_cache.go b/go/tools/builders/orchestrion_cache.go new file mode 100644 -index 0000000..746ac80 +index 0000000..5cccefd --- /dev/null +++ b/go/tools/builders/orchestrion_cache.go -@@ -0,0 +1,276 @@ +@@ -0,0 +1,348 @@ +package main + +import ( ++ "crypto/rand" + "crypto/sha256" ++ "encoding/hex" + "encoding/json" + "fmt" + "os" @@ -12888,6 +12890,7 @@ index 0000000..746ac80 + + cacheManifestFileName = "manifest.json" + cacheReadyFileName = "ready" ++ cacheLockOwnerPrefix = "owner-" + + cacheLockPollInterval = 200 * time.Millisecond + cacheLockTimeout = 60 * time.Second @@ -12928,7 +12931,7 @@ index 0000000..746ac80 +func acquireCacheLockWithTimings(lockDir string, timeout, staleAfter, pollInterval time.Duration) (func(), error) { + retryAfterStaleRemoval := true + for { -+ release, err := tryAcquireCacheLock(lockDir) ++ release, err := tryAcquireCacheLock(lockDir, staleAfter) + if err == nil { + return release, nil + } @@ -12939,7 +12942,7 @@ index 0000000..746ac80 + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + time.Sleep(pollInterval) -+ release, err = tryAcquireCacheLock(lockDir) ++ release, err = tryAcquireCacheLock(lockDir, staleAfter) + if err == nil { + return release, nil + } @@ -12954,7 +12957,7 @@ index 0000000..746ac80 + } + if stale && retryAfterStaleRemoval { + retryAfterStaleRemoval = false -+ if err := os.RemoveAll(lockDir); err != nil && !os.IsNotExist(err) { ++ if err := removeStaleCacheLock(lockDir); err != nil && !os.IsNotExist(err) { + return nil, fmt.Errorf("remove stale cache lock %s: %w", lockDir, err) + } + continue @@ -12966,18 +12969,87 @@ index 0000000..746ac80 + } +} + -+func tryAcquireCacheLock(lockDir string) (func(), error) { ++func tryAcquireCacheLock(lockDir string, staleAfter time.Duration) (func(), error) { + if err := os.MkdirAll(filepath.Dir(lockDir), 0o755); err != nil { + return nil, err + } + if err := os.Mkdir(lockDir, 0o755); err != nil { + return nil, err + } ++ token, err := cacheLockToken() ++ if err != nil { ++ _ = os.Remove(lockDir) ++ return nil, err ++ } ++ ownerPath := filepath.Join(lockDir, cacheLockOwnerPrefix+token) ++ if err := os.WriteFile(ownerPath, []byte(token+"\n"), 0o600); err != nil { ++ _ = os.Remove(lockDir) ++ return nil, err ++ } ++ stopHeartbeat := make(chan struct{}) ++ heartbeatDone := make(chan struct{}) ++ go maintainCacheLockHeartbeat(lockDir, ownerPath, staleAfter, stopHeartbeat, heartbeatDone) ++ released := false + return func() { -+ _ = os.RemoveAll(lockDir) ++ if released { ++ return ++ } ++ released = true ++ close(stopHeartbeat) ++ <-heartbeatDone ++ if err := os.Remove(ownerPath); err != nil { ++ return ++ } ++ _ = os.Remove(lockDir) + }, nil +} + ++func cacheLockToken() (string, error) { ++ var data [16]byte ++ if _, err := rand.Read(data[:]); err != nil { ++ return "", err ++ } ++ return hex.EncodeToString(data[:]), nil ++} ++ ++func maintainCacheLockHeartbeat(lockDir, ownerPath string, staleAfter time.Duration, stop <-chan struct{}, done chan<- struct{}) { ++ defer close(done) ++ interval := staleAfter / 4 ++ if interval < time.Millisecond { ++ interval = time.Millisecond ++ } ++ if interval > time.Minute { ++ interval = time.Minute ++ } ++ ticker := time.NewTicker(interval) ++ defer ticker.Stop() ++ for { ++ select { ++ case <-stop: ++ return ++ case now := <-ticker.C: ++ if err := os.Chtimes(ownerPath, now, now); err != nil { ++ return ++ } ++ if err := os.Chtimes(lockDir, now, now); err != nil { ++ return ++ } ++ } ++ } ++} ++ ++func removeStaleCacheLock(lockDir string) error { ++ token, err := cacheLockToken() ++ if err != nil { ++ return err ++ } ++ staleDir := lockDir + ".stale-" + token ++ if err := os.Rename(lockDir, staleDir); err != nil { ++ return err ++ } ++ return os.RemoveAll(staleDir) ++} ++ +func cacheLockIsStale(lockDir string, staleAfter time.Duration) (bool, error) { + info, err := os.Stat(lockDir) + if err != nil { @@ -13133,10 +13205,10 @@ index 0000000..746ac80 +} diff --git a/go/tools/builders/orchestrion_cache_test.go b/go/tools/builders/orchestrion_cache_test.go new file mode 100644 -index 0000000..5bc0af2 +index 0000000..4e7f2bc --- /dev/null +++ b/go/tools/builders/orchestrion_cache_test.go -@@ -0,0 +1,117 @@ +@@ -0,0 +1,157 @@ +package main + +import ( @@ -13225,7 +13297,7 @@ index 0000000..5bc0af2 + +func TestAcquireCacheLockWaitsForActiveOwnerPastTimeout(t *testing.T) { + lockDir := filepath.Join(t.TempDir(), "cache.lock") -+ releaseOwner, err := tryAcquireCacheLock(lockDir) ++ releaseOwner, err := tryAcquireCacheLock(lockDir, time.Minute) + if err != nil { + t.Fatalf("acquire owner lock: %v", err) + } @@ -13241,6 +13313,46 @@ index 0000000..5bc0af2 + releaseWaiter() +} + ++func TestAcquireCacheLockDoesNotStealLiveOwnerPastStaleThreshold(t *testing.T) { ++ lockDir := filepath.Join(t.TempDir(), "cache.lock") ++ releaseOwner, err := tryAcquireCacheLock(lockDir, 20*time.Millisecond) ++ if err != nil { ++ t.Fatalf("acquire owner lock: %v", err) ++ } ++ go func() { ++ time.Sleep(80 * time.Millisecond) ++ releaseOwner() ++ }() ++ ++ releaseWaiter, err := acquireCacheLockWithTimings(lockDir, 15*time.Millisecond, 20*time.Millisecond, 2*time.Millisecond) ++ if err != nil { ++ t.Fatalf("wait for live owner: %v", err) ++ } ++ releaseWaiter() ++} ++ ++func TestOldCacheLockReleaseDoesNotRemoveReplacementOwner(t *testing.T) { ++ root := t.TempDir() ++ lockDir := filepath.Join(root, "cache.lock") ++ releaseOld, err := tryAcquireCacheLock(lockDir, time.Minute) ++ if err != nil { ++ t.Fatalf("acquire old lock: %v", err) ++ } ++ abandonedDir := filepath.Join(root, "abandoned.lock") ++ if err := os.Rename(lockDir, abandonedDir); err != nil { ++ t.Fatalf("rename old lock: %v", err) ++ } ++ releaseReplacement, err := tryAcquireCacheLock(lockDir, time.Minute) ++ if err != nil { ++ t.Fatalf("acquire replacement lock: %v", err) ++ } ++ releaseOld() ++ if _, err := os.Stat(lockDir); err != nil { ++ t.Fatalf("old release removed replacement lock: %v", err) ++ } ++ releaseReplacement() ++} ++ +func TestWriteFileAtomically(t *testing.T) { + path := filepath.Join(t.TempDir(), "nested", "manifest.json") + if err := writeFileAtomically(path, []byte("payload\n"), 0o644); err != nil { diff --git a/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch index aa56263e..7aa6cecf 100644 --- a/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch @@ -12841,14 +12841,16 @@ index 0000000..93f9e48 +} diff --git a/go/tools/builders/orchestrion_cache.go b/go/tools/builders/orchestrion_cache.go new file mode 100644 -index 0000000..746ac80 +index 0000000..5cccefd --- /dev/null +++ b/go/tools/builders/orchestrion_cache.go -@@ -0,0 +1,276 @@ +@@ -0,0 +1,348 @@ +package main + +import ( ++ "crypto/rand" + "crypto/sha256" ++ "encoding/hex" + "encoding/json" + "fmt" + "os" @@ -12878,6 +12880,7 @@ index 0000000..746ac80 + + cacheManifestFileName = "manifest.json" + cacheReadyFileName = "ready" ++ cacheLockOwnerPrefix = "owner-" + + cacheLockPollInterval = 200 * time.Millisecond + cacheLockTimeout = 60 * time.Second @@ -12918,7 +12921,7 @@ index 0000000..746ac80 +func acquireCacheLockWithTimings(lockDir string, timeout, staleAfter, pollInterval time.Duration) (func(), error) { + retryAfterStaleRemoval := true + for { -+ release, err := tryAcquireCacheLock(lockDir) ++ release, err := tryAcquireCacheLock(lockDir, staleAfter) + if err == nil { + return release, nil + } @@ -12929,7 +12932,7 @@ index 0000000..746ac80 + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + time.Sleep(pollInterval) -+ release, err = tryAcquireCacheLock(lockDir) ++ release, err = tryAcquireCacheLock(lockDir, staleAfter) + if err == nil { + return release, nil + } @@ -12944,7 +12947,7 @@ index 0000000..746ac80 + } + if stale && retryAfterStaleRemoval { + retryAfterStaleRemoval = false -+ if err := os.RemoveAll(lockDir); err != nil && !os.IsNotExist(err) { ++ if err := removeStaleCacheLock(lockDir); err != nil && !os.IsNotExist(err) { + return nil, fmt.Errorf("remove stale cache lock %s: %w", lockDir, err) + } + continue @@ -12956,18 +12959,87 @@ index 0000000..746ac80 + } +} + -+func tryAcquireCacheLock(lockDir string) (func(), error) { ++func tryAcquireCacheLock(lockDir string, staleAfter time.Duration) (func(), error) { + if err := os.MkdirAll(filepath.Dir(lockDir), 0o755); err != nil { + return nil, err + } + if err := os.Mkdir(lockDir, 0o755); err != nil { + return nil, err + } ++ token, err := cacheLockToken() ++ if err != nil { ++ _ = os.Remove(lockDir) ++ return nil, err ++ } ++ ownerPath := filepath.Join(lockDir, cacheLockOwnerPrefix+token) ++ if err := os.WriteFile(ownerPath, []byte(token+"\n"), 0o600); err != nil { ++ _ = os.Remove(lockDir) ++ return nil, err ++ } ++ stopHeartbeat := make(chan struct{}) ++ heartbeatDone := make(chan struct{}) ++ go maintainCacheLockHeartbeat(lockDir, ownerPath, staleAfter, stopHeartbeat, heartbeatDone) ++ released := false + return func() { -+ _ = os.RemoveAll(lockDir) ++ if released { ++ return ++ } ++ released = true ++ close(stopHeartbeat) ++ <-heartbeatDone ++ if err := os.Remove(ownerPath); err != nil { ++ return ++ } ++ _ = os.Remove(lockDir) + }, nil +} + ++func cacheLockToken() (string, error) { ++ var data [16]byte ++ if _, err := rand.Read(data[:]); err != nil { ++ return "", err ++ } ++ return hex.EncodeToString(data[:]), nil ++} ++ ++func maintainCacheLockHeartbeat(lockDir, ownerPath string, staleAfter time.Duration, stop <-chan struct{}, done chan<- struct{}) { ++ defer close(done) ++ interval := staleAfter / 4 ++ if interval < time.Millisecond { ++ interval = time.Millisecond ++ } ++ if interval > time.Minute { ++ interval = time.Minute ++ } ++ ticker := time.NewTicker(interval) ++ defer ticker.Stop() ++ for { ++ select { ++ case <-stop: ++ return ++ case now := <-ticker.C: ++ if err := os.Chtimes(ownerPath, now, now); err != nil { ++ return ++ } ++ if err := os.Chtimes(lockDir, now, now); err != nil { ++ return ++ } ++ } ++ } ++} ++ ++func removeStaleCacheLock(lockDir string) error { ++ token, err := cacheLockToken() ++ if err != nil { ++ return err ++ } ++ staleDir := lockDir + ".stale-" + token ++ if err := os.Rename(lockDir, staleDir); err != nil { ++ return err ++ } ++ return os.RemoveAll(staleDir) ++} ++ +func cacheLockIsStale(lockDir string, staleAfter time.Duration) (bool, error) { + info, err := os.Stat(lockDir) + if err != nil { @@ -13123,10 +13195,10 @@ index 0000000..746ac80 +} diff --git a/go/tools/builders/orchestrion_cache_test.go b/go/tools/builders/orchestrion_cache_test.go new file mode 100644 -index 0000000..5bc0af2 +index 0000000..4e7f2bc --- /dev/null +++ b/go/tools/builders/orchestrion_cache_test.go -@@ -0,0 +1,117 @@ +@@ -0,0 +1,157 @@ +package main + +import ( @@ -13215,7 +13287,7 @@ index 0000000..5bc0af2 + +func TestAcquireCacheLockWaitsForActiveOwnerPastTimeout(t *testing.T) { + lockDir := filepath.Join(t.TempDir(), "cache.lock") -+ releaseOwner, err := tryAcquireCacheLock(lockDir) ++ releaseOwner, err := tryAcquireCacheLock(lockDir, time.Minute) + if err != nil { + t.Fatalf("acquire owner lock: %v", err) + } @@ -13231,6 +13303,46 @@ index 0000000..5bc0af2 + releaseWaiter() +} + ++func TestAcquireCacheLockDoesNotStealLiveOwnerPastStaleThreshold(t *testing.T) { ++ lockDir := filepath.Join(t.TempDir(), "cache.lock") ++ releaseOwner, err := tryAcquireCacheLock(lockDir, 20*time.Millisecond) ++ if err != nil { ++ t.Fatalf("acquire owner lock: %v", err) ++ } ++ go func() { ++ time.Sleep(80 * time.Millisecond) ++ releaseOwner() ++ }() ++ ++ releaseWaiter, err := acquireCacheLockWithTimings(lockDir, 15*time.Millisecond, 20*time.Millisecond, 2*time.Millisecond) ++ if err != nil { ++ t.Fatalf("wait for live owner: %v", err) ++ } ++ releaseWaiter() ++} ++ ++func TestOldCacheLockReleaseDoesNotRemoveReplacementOwner(t *testing.T) { ++ root := t.TempDir() ++ lockDir := filepath.Join(root, "cache.lock") ++ releaseOld, err := tryAcquireCacheLock(lockDir, time.Minute) ++ if err != nil { ++ t.Fatalf("acquire old lock: %v", err) ++ } ++ abandonedDir := filepath.Join(root, "abandoned.lock") ++ if err := os.Rename(lockDir, abandonedDir); err != nil { ++ t.Fatalf("rename old lock: %v", err) ++ } ++ releaseReplacement, err := tryAcquireCacheLock(lockDir, time.Minute) ++ if err != nil { ++ t.Fatalf("acquire replacement lock: %v", err) ++ } ++ releaseOld() ++ if _, err := os.Stat(lockDir); err != nil { ++ t.Fatalf("old release removed replacement lock: %v", err) ++ } ++ releaseReplacement() ++} ++ +func TestWriteFileAtomically(t *testing.T) { + path := filepath.Join(t.TempDir(), "nested", "manifest.json") + if err := writeFileAtomically(path, []byte("payload\n"), 0o644); err != nil { diff --git a/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch index ae8f0d31..7b73deda 100644 --- a/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch @@ -12845,14 +12845,16 @@ index 0000000..93f9e48 +} diff --git a/go/tools/builders/orchestrion_cache.go b/go/tools/builders/orchestrion_cache.go new file mode 100644 -index 0000000..746ac80 +index 0000000..5cccefd --- /dev/null +++ b/go/tools/builders/orchestrion_cache.go -@@ -0,0 +1,276 @@ +@@ -0,0 +1,348 @@ +package main + +import ( ++ "crypto/rand" + "crypto/sha256" ++ "encoding/hex" + "encoding/json" + "fmt" + "os" @@ -12882,6 +12884,7 @@ index 0000000..746ac80 + + cacheManifestFileName = "manifest.json" + cacheReadyFileName = "ready" ++ cacheLockOwnerPrefix = "owner-" + + cacheLockPollInterval = 200 * time.Millisecond + cacheLockTimeout = 60 * time.Second @@ -12922,7 +12925,7 @@ index 0000000..746ac80 +func acquireCacheLockWithTimings(lockDir string, timeout, staleAfter, pollInterval time.Duration) (func(), error) { + retryAfterStaleRemoval := true + for { -+ release, err := tryAcquireCacheLock(lockDir) ++ release, err := tryAcquireCacheLock(lockDir, staleAfter) + if err == nil { + return release, nil + } @@ -12933,7 +12936,7 @@ index 0000000..746ac80 + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + time.Sleep(pollInterval) -+ release, err = tryAcquireCacheLock(lockDir) ++ release, err = tryAcquireCacheLock(lockDir, staleAfter) + if err == nil { + return release, nil + } @@ -12948,7 +12951,7 @@ index 0000000..746ac80 + } + if stale && retryAfterStaleRemoval { + retryAfterStaleRemoval = false -+ if err := os.RemoveAll(lockDir); err != nil && !os.IsNotExist(err) { ++ if err := removeStaleCacheLock(lockDir); err != nil && !os.IsNotExist(err) { + return nil, fmt.Errorf("remove stale cache lock %s: %w", lockDir, err) + } + continue @@ -12960,18 +12963,87 @@ index 0000000..746ac80 + } +} + -+func tryAcquireCacheLock(lockDir string) (func(), error) { ++func tryAcquireCacheLock(lockDir string, staleAfter time.Duration) (func(), error) { + if err := os.MkdirAll(filepath.Dir(lockDir), 0o755); err != nil { + return nil, err + } + if err := os.Mkdir(lockDir, 0o755); err != nil { + return nil, err + } ++ token, err := cacheLockToken() ++ if err != nil { ++ _ = os.Remove(lockDir) ++ return nil, err ++ } ++ ownerPath := filepath.Join(lockDir, cacheLockOwnerPrefix+token) ++ if err := os.WriteFile(ownerPath, []byte(token+"\n"), 0o600); err != nil { ++ _ = os.Remove(lockDir) ++ return nil, err ++ } ++ stopHeartbeat := make(chan struct{}) ++ heartbeatDone := make(chan struct{}) ++ go maintainCacheLockHeartbeat(lockDir, ownerPath, staleAfter, stopHeartbeat, heartbeatDone) ++ released := false + return func() { -+ _ = os.RemoveAll(lockDir) ++ if released { ++ return ++ } ++ released = true ++ close(stopHeartbeat) ++ <-heartbeatDone ++ if err := os.Remove(ownerPath); err != nil { ++ return ++ } ++ _ = os.Remove(lockDir) + }, nil +} + ++func cacheLockToken() (string, error) { ++ var data [16]byte ++ if _, err := rand.Read(data[:]); err != nil { ++ return "", err ++ } ++ return hex.EncodeToString(data[:]), nil ++} ++ ++func maintainCacheLockHeartbeat(lockDir, ownerPath string, staleAfter time.Duration, stop <-chan struct{}, done chan<- struct{}) { ++ defer close(done) ++ interval := staleAfter / 4 ++ if interval < time.Millisecond { ++ interval = time.Millisecond ++ } ++ if interval > time.Minute { ++ interval = time.Minute ++ } ++ ticker := time.NewTicker(interval) ++ defer ticker.Stop() ++ for { ++ select { ++ case <-stop: ++ return ++ case now := <-ticker.C: ++ if err := os.Chtimes(ownerPath, now, now); err != nil { ++ return ++ } ++ if err := os.Chtimes(lockDir, now, now); err != nil { ++ return ++ } ++ } ++ } ++} ++ ++func removeStaleCacheLock(lockDir string) error { ++ token, err := cacheLockToken() ++ if err != nil { ++ return err ++ } ++ staleDir := lockDir + ".stale-" + token ++ if err := os.Rename(lockDir, staleDir); err != nil { ++ return err ++ } ++ return os.RemoveAll(staleDir) ++} ++ +func cacheLockIsStale(lockDir string, staleAfter time.Duration) (bool, error) { + info, err := os.Stat(lockDir) + if err != nil { @@ -13127,10 +13199,10 @@ index 0000000..746ac80 +} diff --git a/go/tools/builders/orchestrion_cache_test.go b/go/tools/builders/orchestrion_cache_test.go new file mode 100644 -index 0000000..5bc0af2 +index 0000000..4e7f2bc --- /dev/null +++ b/go/tools/builders/orchestrion_cache_test.go -@@ -0,0 +1,117 @@ +@@ -0,0 +1,157 @@ +package main + +import ( @@ -13219,7 +13291,7 @@ index 0000000..5bc0af2 + +func TestAcquireCacheLockWaitsForActiveOwnerPastTimeout(t *testing.T) { + lockDir := filepath.Join(t.TempDir(), "cache.lock") -+ releaseOwner, err := tryAcquireCacheLock(lockDir) ++ releaseOwner, err := tryAcquireCacheLock(lockDir, time.Minute) + if err != nil { + t.Fatalf("acquire owner lock: %v", err) + } @@ -13235,6 +13307,46 @@ index 0000000..5bc0af2 + releaseWaiter() +} + ++func TestAcquireCacheLockDoesNotStealLiveOwnerPastStaleThreshold(t *testing.T) { ++ lockDir := filepath.Join(t.TempDir(), "cache.lock") ++ releaseOwner, err := tryAcquireCacheLock(lockDir, 20*time.Millisecond) ++ if err != nil { ++ t.Fatalf("acquire owner lock: %v", err) ++ } ++ go func() { ++ time.Sleep(80 * time.Millisecond) ++ releaseOwner() ++ }() ++ ++ releaseWaiter, err := acquireCacheLockWithTimings(lockDir, 15*time.Millisecond, 20*time.Millisecond, 2*time.Millisecond) ++ if err != nil { ++ t.Fatalf("wait for live owner: %v", err) ++ } ++ releaseWaiter() ++} ++ ++func TestOldCacheLockReleaseDoesNotRemoveReplacementOwner(t *testing.T) { ++ root := t.TempDir() ++ lockDir := filepath.Join(root, "cache.lock") ++ releaseOld, err := tryAcquireCacheLock(lockDir, time.Minute) ++ if err != nil { ++ t.Fatalf("acquire old lock: %v", err) ++ } ++ abandonedDir := filepath.Join(root, "abandoned.lock") ++ if err := os.Rename(lockDir, abandonedDir); err != nil { ++ t.Fatalf("rename old lock: %v", err) ++ } ++ releaseReplacement, err := tryAcquireCacheLock(lockDir, time.Minute) ++ if err != nil { ++ t.Fatalf("acquire replacement lock: %v", err) ++ } ++ releaseOld() ++ if _, err := os.Stat(lockDir); err != nil { ++ t.Fatalf("old release removed replacement lock: %v", err) ++ } ++ releaseReplacement() ++} ++ +func TestWriteFileAtomically(t *testing.T) { + path := filepath.Join(t.TempDir(), "nested", "manifest.json") + if err := writeFileAtomically(path, []byte("payload\n"), 0o644); err != nil { diff --git a/tools/agent-skills/go-test-optimization-onboarding/SKILL.md b/tools/agent-skills/go-test-optimization-onboarding/SKILL.md index 15314ee7..067e75e4 100644 --- a/tools/agent-skills/go-test-optimization-onboarding/SKILL.md +++ b/tools/agent-skills/go-test-optimization-onboarding/SKILL.md @@ -130,8 +130,8 @@ Every successful Go onboarding should end with these pieces: flags and required BEP freshness/artifact flags. Use `DD_TEST_OPTIMIZATION_*` environment variables only for single-invocation manual flows where one BEP file is sufficient. -- CI wrappers write `doctor-report.json`, `uploader-dry-run-report.json`, - optional `uploader-upload-report.json`, and, when configured, +- CI wrappers write `doctor-report.json`, one selected uploader report + (`uploader-dry-run-report.json` or `uploader-upload-report.json`), and `dd-test-optimization-support.zip` under a per-job report directory. Prefer the wrapper support bundle for full CI escalation; use the doctor-only support bundle for the simplest initial customer request. Keep individual @@ -189,7 +189,8 @@ Go wrapper, treat it as a consumer-specific integration: - Validate with fresh `bazel-testlogs//test.outputs/`, inspect `bazel_target_metadata.json` for `bazel.go.orchestrion.mode = "test_optimization"` on Go targets, then run - the doctor and uploader dry-run before the real upload attempt. + the doctor and one enrichment-validating uploader pass; use dry-run only + when real upload is disabled. ## Branch And PR Hygiene diff --git a/tools/agent-skills/go-test-optimization-onboarding/references/validation-checklist.md b/tools/agent-skills/go-test-optimization-onboarding/references/validation-checklist.md index 11fdb625..58bb192e 100644 --- a/tools/agent-skills/go-test-optimization-onboarding/references/validation-checklist.md +++ b/tools/agent-skills/go-test-optimization-onboarding/references/validation-checklist.md @@ -227,7 +227,7 @@ artifact_staging_dir="$(mktemp -d "${TMPDIR:-/tmp}/dd-topt-artifacts.XXXXXX")" report_dir="${REPORT_DIR:-.topt/reports}" mkdir -p "$report_dir" -test_status=0; doctor_status=0; dry_run_status=0; upload_status=0 +test_status=0; doctor_status=0; uploader_status=0 bazel test --config=test-optimization --build_event_json_file="$bep_json" //path/to:pilot_test || test_status=$? bazel run --config=test-optimization //:dd_test_optimization_doctor -- \ @@ -238,16 +238,6 @@ bazel run --config=test-optimization //:dd_test_optimization_docto --artifact-staging-dir="$artifact_staging_dir" \ --report-json="$report_dir/doctor-report.json" || doctor_status=$? -bazel run --config=test-optimization //:dd_upload_payloads -- \ - --bep-json="$bep_json" \ - --freshness-source=bep \ - --freshness-mode=required \ - --artifact-source=bep \ - --artifact-staging-dir="$artifact_staging_dir" \ - --dry-run \ - --validate-enrichment \ - --report-json="$report_dir/uploader-dry-run-report.json" || dry_run_status=$? - DD_API_KEY="$DD_API_KEY" DD_SITE="$DD_SITE" \ bazel run --config=test-optimization //:dd_upload_payloads -- \ --bep-json="$bep_json" \ @@ -255,9 +245,10 @@ DD_API_KEY="$DD_API_KEY" DD_SITE="$DD_SITE" \ --freshness-mode=required \ --artifact-source=bep \ --artifact-staging-dir="$artifact_staging_dir" \ - --report-json="$report_dir/uploader-upload-report.json" || upload_status=$? + --validate-enrichment \ + --report-json="$report_dir/uploader-upload-report.json" || uploader_status=$? -for status in "$test_status" "$doctor_status" "$dry_run_status" "$upload_status"; do +for status in "$test_status" "$doctor_status" "$uploader_status"; do if [ "$status" -ne 0 ]; then exit "$status"; fi done ``` diff --git a/tools/agent-skills/java-test-optimization-onboarding/SKILL.md b/tools/agent-skills/java-test-optimization-onboarding/SKILL.md index a73e0e1a..8d9612ab 100644 --- a/tools/agent-skills/java-test-optimization-onboarding/SKILL.md +++ b/tools/agent-skills/java-test-optimization-onboarding/SKILL.md @@ -123,8 +123,8 @@ Every successful Java onboarding should end with these pieces: flags and required BEP freshness/artifact flags. Use `DD_TEST_OPTIMIZATION_*` environment variables only for single-invocation manual flows where one BEP file is sufficient. -- CI wrappers write `doctor-report.json`, `uploader-dry-run-report.json`, - optional `uploader-upload-report.json`, and, when configured, +- CI wrappers write `doctor-report.json`, one selected uploader report + (`uploader-dry-run-report.json` or `uploader-upload-report.json`), and `dd-test-optimization-support.zip` under a per-job report directory. Prefer the wrapper support bundle for full CI escalation; use the doctor-only support bundle for the simplest initial customer request. Keep individual diff --git a/tools/agent-skills/java-test-optimization-onboarding/references/validation-checklist.md b/tools/agent-skills/java-test-optimization-onboarding/references/validation-checklist.md index ad62aebd..5fc23ab7 100644 --- a/tools/agent-skills/java-test-optimization-onboarding/references/validation-checklist.md +++ b/tools/agent-skills/java-test-optimization-onboarding/references/validation-checklist.md @@ -116,7 +116,7 @@ artifact_staging_dir="$(mktemp -d "${TMPDIR:-/tmp}/dd-topt-artifacts.XXXXXX")" report_dir="${REPORT_DIR:-.topt/reports}" mkdir -p "$report_dir" -test_status=0; doctor_status=0; dry_run_status=0; upload_status=0 +test_status=0; doctor_status=0; uploader_status=0 bazel test --config=test-optimization --build_event_json_file="$bep_json" //path/to:java_test || test_status=$? bazel run --config=test-optimization //tools/test_optimization:dd_test_optimization_doctor -- \ @@ -127,16 +127,6 @@ bazel run --config=test-optimization //tools/test_optimization:dd_test_optimizat --artifact-staging-dir="$artifact_staging_dir" \ --report-json="$report_dir/doctor-report.json" || doctor_status=$? -bazel run --config=test-optimization //tools/test_optimization:dd_upload_payloads -- \ - --bep-json="$bep_json" \ - --freshness-source=bep \ - --freshness-mode=required \ - --artifact-source=bep \ - --artifact-staging-dir="$artifact_staging_dir" \ - --dry-run \ - --validate-enrichment \ - --report-json="$report_dir/uploader-dry-run-report.json" || dry_run_status=$? - DD_API_KEY="$DD_API_KEY" DD_SITE="$DD_SITE" \ bazel run --config=test-optimization //tools/test_optimization:dd_upload_payloads -- \ --bep-json="$bep_json" \ @@ -144,9 +134,10 @@ DD_API_KEY="$DD_API_KEY" DD_SITE="$DD_SITE" \ --freshness-mode=required \ --artifact-source=bep \ --artifact-staging-dir="$artifact_staging_dir" \ - --report-json="$report_dir/uploader-upload-report.json" || upload_status=$? + --validate-enrichment \ + --report-json="$report_dir/uploader-upload-report.json" || uploader_status=$? -for status in "$test_status" "$doctor_status" "$dry_run_status" "$upload_status"; do +for status in "$test_status" "$doctor_status" "$uploader_status"; do if [ "$status" -ne 0 ]; then exit "$status"; fi done ``` diff --git a/tools/agent-skills/python-test-optimization-onboarding/SKILL.md b/tools/agent-skills/python-test-optimization-onboarding/SKILL.md index a8ca61fd..849e0f73 100644 --- a/tools/agent-skills/python-test-optimization-onboarding/SKILL.md +++ b/tools/agent-skills/python-test-optimization-onboarding/SKILL.md @@ -119,8 +119,8 @@ Every successful Python onboarding should end with these pieces: flags and required BEP freshness/artifact flags. Use `DD_TEST_OPTIMIZATION_*` environment variables only for single-invocation manual flows where one BEP file is sufficient. -- CI wrappers write `doctor-report.json`, `uploader-dry-run-report.json`, - optional `uploader-upload-report.json`, and, when configured, +- CI wrappers write `doctor-report.json`, one selected uploader report + (`uploader-dry-run-report.json` or `uploader-upload-report.json`), and `dd-test-optimization-support.zip` under a per-job report directory. Prefer the wrapper support bundle for full CI escalation; use the doctor-only support bundle for the simplest initial customer request. Keep individual diff --git a/tools/agent-skills/python-test-optimization-onboarding/references/validation-checklist.md b/tools/agent-skills/python-test-optimization-onboarding/references/validation-checklist.md index 65a0a53b..a89f55c0 100644 --- a/tools/agent-skills/python-test-optimization-onboarding/references/validation-checklist.md +++ b/tools/agent-skills/python-test-optimization-onboarding/references/validation-checklist.md @@ -188,7 +188,7 @@ artifact_staging_dir="$(mktemp -d "${TMPDIR:-/tmp}/dd-topt-artifacts.XXXXXX")" report_dir="${REPORT_DIR:-.topt/reports}" mkdir -p "$report_dir" -test_status=0; doctor_status=0; dry_run_status=0; upload_status=0 +test_status=0; doctor_status=0; uploader_status=0 bazel test --config=test-optimization --build_event_json_file="$bep_json" //path/to:python_test || test_status=$? bazel run --config=test-optimization //tools/test_optimization:dd_test_optimization_doctor -- \ @@ -199,16 +199,6 @@ bazel run --config=test-optimization //tools/test_optimization:dd_test_optimizat --artifact-staging-dir="$artifact_staging_dir" \ --report-json="$report_dir/doctor-report.json" || doctor_status=$? -bazel run --config=test-optimization //tools/test_optimization:dd_upload_payloads -- \ - --bep-json="$bep_json" \ - --freshness-source=bep \ - --freshness-mode=required \ - --artifact-source=bep \ - --artifact-staging-dir="$artifact_staging_dir" \ - --dry-run \ - --validate-enrichment \ - --report-json="$report_dir/uploader-dry-run-report.json" || dry_run_status=$? - DD_API_KEY="$DD_API_KEY" DD_SITE="$DD_SITE" \ bazel run --config=test-optimization //tools/test_optimization:dd_upload_payloads -- \ --bep-json="$bep_json" \ @@ -216,9 +206,10 @@ DD_API_KEY="$DD_API_KEY" DD_SITE="$DD_SITE" \ --freshness-mode=required \ --artifact-source=bep \ --artifact-staging-dir="$artifact_staging_dir" \ - --report-json="$report_dir/uploader-upload-report.json" || upload_status=$? + --validate-enrichment \ + --report-json="$report_dir/uploader-upload-report.json" || uploader_status=$? -for status in "$test_status" "$doctor_status" "$dry_run_status" "$upload_status"; do +for status in "$test_status" "$doctor_status" "$uploader_status"; do if [ "$status" -ne 0 ]; then exit "$status"; fi done ``` diff --git a/tools/core/bep_artifact_stage_helper.py b/tools/core/bep_artifact_stage_helper.py index 50297d87..ab8f8ad3 100644 --- a/tools/core/bep_artifact_stage_helper.py +++ b/tools/core/bep_artifact_stage_helper.py @@ -11,6 +11,7 @@ import argparse import importlib.util +import json from pathlib import Path import sys from types import ModuleType @@ -34,6 +35,28 @@ def _load_doctor_runtime(path: str) -> ModuleType: return module +def _valid_bep_paths(paths: list[str]) -> list[Path]: + valid: list[Path] = [] + for value in paths: + path = Path(value) + if not path.is_file(): + print(f"[dd-test-optimization] error: BEP JSON not found: {path}; continuing with other BEP files", file=sys.stderr) + continue + try: + with path.open("r", encoding="utf-8-sig") as handle: + for line_number, line in enumerate(handle, start=1): + if line.strip(): + json.loads(line) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + print( + f"[dd-test-optimization] error: failed to parse BEP JSON {path}: {exc}; continuing with other BEP files", + file=sys.stderr, + ) + continue + valid.append(path) + return valid + + def main(argv: list[str]) -> int: parser = argparse.ArgumentParser() parser.add_argument("--doctor-runtime", required=True) @@ -54,7 +77,10 @@ def main(argv: list[str]) -> int: workspace = doctor._workspace_root() staging_base = Path(args.staging_dir) - freshness = doctor._parse_bep_freshness([Path(path) for path in args.bep_json], unavailable_is_error=True) + valid_bep_paths = _valid_bep_paths(args.bep_json) + if not valid_bep_paths: + return 0 + freshness = doctor._parse_bep_freshness(valid_bep_paths, unavailable_is_error=True) if freshness is None: return 0 selected_outputs = sorted( diff --git a/tools/core/uploader_bash_runtime.sh.tpl b/tools/core/uploader_bash_runtime.sh.tpl index 4dc29046..60dd4f80 100644 --- a/tools/core/uploader_bash_runtime.sh.tpl +++ b/tools/core/uploader_bash_runtime.sh.tpl @@ -658,6 +658,7 @@ DEFAULT_EXECUTION_LOG_JSON=".topt/bazel-execution-log.json" FRESHNESS_ELIGIBILITY_ENABLED=0 FRESHNESS_SELECTED_SOURCE="none" FRESHNESS_ELIGIBLE_LABELS_FILE="" +REMOTE_ONLY_OUTPUTS_VALIDATED=0 FRESHNESS_ELIGIBLE_OUTPUTS_FILE="" FRESHNESS_CACHED_OUTPUTS_FILE="" FRESHNESS_SKIPPED_OUTPUTS_FILE="" @@ -1502,8 +1503,8 @@ stage_bep_artifacts() { for bep_json in "${BEP_JSON_FILES[@]+"${BEP_JSON_FILES[@]}"}"; do resolved_bep_json="$(resolve_runtime_file_path "$bep_json")" if [[ -z "$resolved_bep_json" || ! -f "$resolved_bep_json" ]]; then - log "error: BEP JSON not found for artifact staging: $bep_json" - exit 2 + log "error: BEP JSON not found for artifact staging: $bep_json; continuing with other BEP files" + continue fi resolved_bep_files+=("$resolved_bep_json") done @@ -3074,15 +3075,16 @@ prepare_bep_eligibility() { : >"$FRESHNESS_REMOTE_ONLY_OUTPUTS_FILE" : >"$FRESHNESS_MISSING_OUTPUT_LABELS_FILE" - local bep_json resolved_bep tmp_records tmp_remote tmp_missing + local bep_json resolved_bep tmp_records tmp_remote tmp_missing bep_valid for bep_json in "${BEP_JSON_FILES[@]}"; do resolved_bep="$(resolve_runtime_file_path "$bep_json")" if [[ -z "$resolved_bep" || ! -f "$resolved_bep" ]]; then if optional_bep_unavailable "BEP JSON not found: $bep_json"; then return 0 fi - log "error: BEP JSON not found: $bep_json" - exit 2 + log "error: BEP JSON not found: $bep_json; continuing with other BEP files" + ((++UPLOAD_FAILURES)) + continue fi tmp_records="$(mktemp "$TMP_PAYLOAD_DIR/bep_records.XXXXXX" 2>/dev/null || true)" tmp_remote="$(mktemp "$TMP_PAYLOAD_DIR/bep_remote.XXXXXX" 2>/dev/null || true)" @@ -3092,6 +3094,7 @@ prepare_bep_eligibility() { exit 2 fi + bep_valid=1 if ! jq -r "$bep_test_output_key_jq $is_remote_only_bep_reference_jq"' def field($obj; $camel; $snake): ($obj[$camel] // $obj[$snake]); @@ -3138,13 +3141,11 @@ prepare_bep_eligibility() { if optional_bep_unavailable "failed to parse BEP JSON: $resolved_bep"; then return 0 fi - log "error: failed to parse BEP JSON: $resolved_bep" - exit 2 + log "error: failed to parse BEP JSON: $resolved_bep; continuing with other BEP files" + bep_valid=0 fi - awk -F '\t' '$3 == "eligible" { print $1 "\t" $2 }' "$tmp_records" >>"$FRESHNESS_ELIGIBLE_OUTPUTS_FILE" - awk -F '\t' '$3 == "cached" { print $1 "\t" $2 }' "$tmp_records" >>"$FRESHNESS_CACHED_OUTPUTS_FILE" - if ! jq -r "$bep_test_output_key_jq $is_remote_only_bep_reference_jq"' + if (( bep_valid == 1 )) && ! jq -r "$bep_test_output_key_jq $is_remote_only_bep_reference_jq"' def field($obj; $camel; $snake): ($obj[$camel] // $obj[$snake]); def candidates($output): @@ -3193,12 +3194,11 @@ prepare_bep_eligibility() { if optional_bep_unavailable "failed to parse BEP remote-only outputs: $resolved_bep"; then return 0 fi - log "error: failed to parse BEP remote-only outputs: $resolved_bep" - exit 2 + log "error: failed to parse BEP remote-only outputs: $resolved_bep; continuing with other BEP files" + bep_valid=0 fi - cat "$tmp_remote" >>"$FRESHNESS_REMOTE_ONLY_OUTPUTS_FILE" - if ! jq -r "$bep_test_output_key_jq $is_remote_only_bep_reference_jq"' + if (( bep_valid == 1 )) && ! jq -r "$bep_test_output_key_jq $is_remote_only_bep_reference_jq"' def field($obj; $camel; $snake): ($obj[$camel] // $obj[$snake]); def candidates($output): @@ -3244,9 +3244,16 @@ prepare_bep_eligibility() { if optional_bep_unavailable "failed to parse BEP missing output mappings: $resolved_bep"; then return 0 fi - log "error: failed to parse BEP missing output mappings: $resolved_bep" - exit 2 + log "error: failed to parse BEP missing output mappings: $resolved_bep; continuing with other BEP files" + bep_valid=0 fi + if (( bep_valid == 0 )); then + ((++UPLOAD_FAILURES)) + continue + fi + awk -F '\t' '$3 == "eligible" { print $1 "\t" $2 }' "$tmp_records" >>"$FRESHNESS_ELIGIBLE_OUTPUTS_FILE" + awk -F '\t' '$3 == "cached" { print $1 "\t" $2 }' "$tmp_records" >>"$FRESHNESS_CACHED_OUTPUTS_FILE" + cat "$tmp_remote" >>"$FRESHNESS_REMOTE_ONLY_OUTPUTS_FILE" cat "$tmp_missing" >>"$FRESHNESS_MISSING_OUTPUT_LABELS_FILE" done @@ -3332,10 +3339,15 @@ validate_expected_target_coverage() { done <"$EXPECTED_TARGETS_RESOLVED_FILE" if (( missing_count > 0 )); then log "warning: $missing_count expected target(s) produced no current uploadable output; available fresh payloads will still be processed" + UPLOAD_FAILURES=$((UPLOAD_FAILURES + missing_count)) fi } validate_bep_remote_only_outputs() { + if (( REMOTE_ONLY_OUTPUTS_VALIDATED == 1 )); then + return 0 + fi + REMOTE_ONLY_OUTPUTS_VALIDATED=1 if [[ "$FRESHNESS_SELECTED_SOURCE" != "bep" ]]; then return 0 fi @@ -3345,8 +3357,11 @@ validate_bep_remote_only_outputs() { first_artifact="$(awk -F '\t' 'NR == 1 { print $3 }' "$FRESHNESS_REMOTE_ONLY_OUTPUTS_FILE")" first_artifact_display="$(display_artifact_reference "$first_artifact")" if [[ "$FRESHNESS_MODE" == "required" || "$REMOTE_ARTIFACTS" == "required" ]]; then - log "error: BEP references remote-only test outputs for ${first_label:-}, but local test.outputs was not found: ${first_artifact_display:-}. Rerun with --remote_download_minimal --remote_download_regex=.*test[.]outputs.* or configure a BEP artifact fetcher. If the test run used --zip_undeclared_test_outputs, rerun the uploader with --artifact-source=bep." - exit 2 + local remote_count + remote_count="$(wc -l <"$FRESHNESS_REMOTE_ONLY_OUTPUTS_FILE" | tr -d ' ')" + log "error: BEP references remote-only test outputs for ${first_label:-}, but local test.outputs was not found: ${first_artifact_display:-}. Those outputs will be skipped while other fresh payloads are processed. Rerun with --remote_download_minimal --remote_download_regex=.*test[.]outputs.* or configure a BEP artifact fetcher. If the test run used --zip_undeclared_test_outputs, rerun the uploader with --artifact-source=bep." + UPLOAD_FAILURES=$((UPLOAD_FAILURES + remote_count)) + return 0 fi if [[ "$REMOTE_ARTIFACTS" == "download" ]]; then log "warning: BEP references remote-only test outputs for ${first_label:-} and they were not materialized: ${first_artifact_display:-}; skipping those outputs." @@ -4712,8 +4727,8 @@ PY return 0 } -# Track upload failures globally -UPLOAD_FAILURES=0 +# Track per-payload upload report counters. UPLOAD_FAILURES is initialized +# before BEP preparation so partial-input failures remain part of the result. REPORT_TESTS_PROCESSED=0 REPORT_TESTS_FAILED=0 REPORT_TESTS_SKIPPED=0 diff --git a/tools/core/uploader_powershell_runtime.ps1.tpl b/tools/core/uploader_powershell_runtime.ps1.tpl index 6c3dff34..75a6ce76 100644 --- a/tools/core/uploader_powershell_runtime.ps1.tpl +++ b/tools/core/uploader_powershell_runtime.ps1.tpl @@ -712,6 +712,7 @@ $script:FreshnessSkippedOutputs = [System.Collections.Generic.HashSet[string]]:: $script:FreshnessRemoteOnlyOutputs = New-Object System.Collections.Generic.List[object] $script:FreshnessMissingOutputLabels = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) $script:FreshnessSkipWasWritten = $false +$script:RemoteOnlyOutputsValidated = $false $script:ExpectedTargetsConfigured = $false $script:ExpectedTargets = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) $script:HandledFreshOutputs = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) @@ -1390,8 +1391,8 @@ function Stage-BepArtifacts { foreach ($bepJson in @($script:BepJsonFiles)) { $resolvedBepJson = Resolve-RuntimeFilePath $bepJson if ([string]::IsNullOrWhiteSpace($resolvedBepJson) -or -not (Test-Path -LiteralPath $resolvedBepJson -PathType Leaf)) { - Log "error: BEP JSON not found for artifact staging: $bepJson" - exit 2 + Log "error: BEP JSON not found for artifact staging: $bepJson; continuing with other BEP files" + continue } $resolvedBepJsonFiles.Add($resolvedBepJson) | Out-Null } @@ -2947,9 +2948,15 @@ function Initialize-BepEligibility { if (Use-OptionalBepUnavailable "BEP JSON not found: $bepJson") { return } - Log "error: BEP JSON not found: $bepJson" - exit 2 + Log "error: BEP JSON not found: $bepJson; continuing with other BEP files" + $script:UploadFailures++ + continue } + $eligibleLabelsBefore = @($script:FreshnessEligibleLabels) + $eligibleOutputsBefore = @($script:FreshnessEligibleOutputs) + $cachedOutputsBefore = @($script:FreshnessCachedOutputs) + $remoteOutputsBefore = @($script:FreshnessRemoteOnlyOutputs.ToArray()) + $missingLabelsBefore = @($script:FreshnessMissingOutputLabels) try { foreach ($event in @(Get-JsonStreamObjects $resolvedBep)) { $eventId = Get-MapValue $event 'id' @@ -3038,11 +3045,22 @@ function Initialize-BepEligibility { } } } catch { + $script:FreshnessEligibleLabels.Clear() + foreach ($entry in $eligibleLabelsBefore) { $script:FreshnessEligibleLabels.Add([string]$entry) | Out-Null } + $script:FreshnessEligibleOutputs.Clear() + foreach ($entry in $eligibleOutputsBefore) { $script:FreshnessEligibleOutputs.Add([string]$entry) | Out-Null } + $script:FreshnessCachedOutputs.Clear() + foreach ($entry in $cachedOutputsBefore) { $script:FreshnessCachedOutputs.Add([string]$entry) | Out-Null } + $script:FreshnessRemoteOnlyOutputs.Clear() + foreach ($entry in $remoteOutputsBefore) { $script:FreshnessRemoteOnlyOutputs.Add($entry) | Out-Null } + $script:FreshnessMissingOutputLabels.Clear() + foreach ($entry in $missingLabelsBefore) { $script:FreshnessMissingOutputLabels.Add([string]$entry) | Out-Null } if (Use-OptionalBepUnavailable "failed to parse BEP JSON: $resolvedBep ($($_.Exception.Message))") { return } - Log "error: failed to parse BEP JSON: $resolvedBep ($($_.Exception.Message))" - exit 2 + Log "error: failed to parse BEP JSON: $resolvedBep ($($_.Exception.Message)); continuing with other BEP files" + $script:UploadFailures++ + continue } } @@ -3238,6 +3256,7 @@ function Assert-ExpectedTargetCoverage { } if ($missingCount -gt 0) { Log "warning: $missingCount expected target(s) produced no current uploadable output; available fresh payloads will still be processed" + $script:UploadFailures += $missingCount } } @@ -3248,13 +3267,16 @@ function Write-ExecutionSkipOnce([string]$OutputsDir, [string]$Reason) { } function Assert-NoRequiredRemoteOnlyBepOutputs { + if ($script:RemoteOnlyOutputsValidated) { return } + $script:RemoteOnlyOutputsValidated = $true if ($script:FreshnessSelectedSource -ne "bep") { return } if ($script:FreshnessRemoteOnlyOutputs.Count -gt 0) { $first = $script:FreshnessRemoteOnlyOutputs[0] $firstArtifact = Format-ArtifactReferenceForLog $first.Artifact if ($script:FreshnessMode -eq "required" -or $script:RemoteArtifacts -eq "required") { - Log "error: BEP references remote-only test outputs for $($first.Label), but local test.outputs was not found: $firstArtifact. Rerun with --remote_download_minimal --remote_download_regex=.*test[.]outputs.* or configure a BEP artifact fetcher. If the test run used --zip_undeclared_test_outputs, rerun the uploader with --artifact-source=bep." - exit 2 + Log "error: BEP references remote-only test outputs for $($first.Label), but local test.outputs was not found: $firstArtifact. Those outputs will be skipped while other fresh payloads are processed. Rerun with --remote_download_minimal --remote_download_regex=.*test[.]outputs.* or configure a BEP artifact fetcher. If the test run used --zip_undeclared_test_outputs, rerun the uploader with --artifact-source=bep." + $script:UploadFailures += $script:FreshnessRemoteOnlyOutputs.Count + return } if ($script:RemoteArtifacts -eq "download") { Log "warning: BEP references remote-only test outputs for $($first.Label): $firstArtifact; unmaterialized outputs will be skipped." @@ -3763,8 +3785,8 @@ function Remove-PayloadFile([string]$FilePath) { } } -# Track upload failures globally -$script:UploadFailures = 0 +# Track per-payload upload report counters. UploadFailures is initialized +# before BEP preparation so partial-input failures remain part of the result. $script:ReportTestsProcessed = 0 $script:ReportTestsFailed = 0 $script:ReportTestsSkipped = 0 diff --git a/tools/test_optimization/run_test_optimization_ci.ps1 b/tools/test_optimization/run_test_optimization_ci.ps1 index 3df3a933..2b773a85 100644 --- a/tools/test_optimization/run_test_optimization_ci.ps1 +++ b/tools/test_optimization/run_test_optimization_ci.ps1 @@ -86,9 +86,15 @@ if (-not [string]::IsNullOrWhiteSpace($ReportDir)) { $DoctorReportJson = Join-Path $ReportDir "doctor-report.json" } if ([string]::IsNullOrWhiteSpace($UploaderReportJson)) { - $UploaderReportJson = Join-Path $ReportDir "uploader-dry-run-report.json" + if ($Upload.IsPresent) { + $UploaderReportJson = Join-Path $ReportDir "uploader-upload-report.json" + } else { + $UploaderReportJson = Join-Path $ReportDir "uploader-dry-run-report.json" + } + } + if ($Upload.IsPresent) { + $UploadReportJson = $UploaderReportJson } - $UploadReportJson = Join-Path $ReportDir "uploader-upload-report.json" } $keepGeneratedFiles = $KeepTmp.IsPresent -or $env:DD_TEST_OPTIMIZATION_KEEP_TMP -eq "1" @@ -236,24 +242,21 @@ try { $finalStatus = $doctorStatus } - $dryRunRuntimeArgs = $runtimeArgs - if (-not [string]::IsNullOrWhiteSpace($UploaderReportJson)) { - $dryRunRuntimeArgs += "--report-json=$UploaderReportJson" - } - $dryRunStatus = Invoke-BazelCommand -Args (@("run", "--config=$Config", $UploadTarget, "--") + $dryRunRuntimeArgs + @("--dry-run", "--validate-enrichment")) - if ($dryRunStatus -ne 0 -and $finalStatus -eq 0) { - $finalStatus = $dryRunStatus - } - + $uploaderRuntimeArgs = $runtimeArgs if ($Upload.IsPresent) { - $uploadRuntimeArgs = $runtimeArgs if (-not [string]::IsNullOrWhiteSpace($UploadReportJson)) { - $uploadRuntimeArgs += "--report-json=$UploadReportJson" + $uploaderRuntimeArgs += "--report-json=$UploadReportJson" } - $uploadStatus = Invoke-BazelCommand -Args (@("run", "--config=$Config", $UploadTarget, "--") + $uploadRuntimeArgs) - if ($uploadStatus -ne 0 -and $finalStatus -eq 0) { - $finalStatus = $uploadStatus + $uploaderRuntimeArgs += "--validate-enrichment" + } else { + if (-not [string]::IsNullOrWhiteSpace($UploaderReportJson)) { + $uploaderRuntimeArgs += "--report-json=$UploaderReportJson" } + $uploaderRuntimeArgs += @("--dry-run", "--validate-enrichment") + } + $uploaderStatus = Invoke-BazelCommand -Args (@("run", "--config=$Config", $UploadTarget, "--") + $uploaderRuntimeArgs) + if ($uploaderStatus -ne 0 -and $finalStatus -eq 0) { + $finalStatus = $uploaderStatus } exit $finalStatus diff --git a/tools/test_optimization/run_test_optimization_ci.sh b/tools/test_optimization/run_test_optimization_ci.sh index 6bca2b5f..d9a2ccb7 100755 --- a/tools/test_optimization/run_test_optimization_ci.sh +++ b/tools/test_optimization/run_test_optimization_ci.sh @@ -36,7 +36,7 @@ Options: --doctor-report-json PATH Write the doctor machine-readable report to PATH. --uploader-report-json PATH - Write the dry-run uploader machine-readable report to PATH. + Write the uploader machine-readable report to PATH. --report-dir PATH Write doctor/uploader reports under PATH. --upload-target LABEL Uploader target. Defaults to //:dd_upload_payloads. --support-bundle PATH Write a redacted support diagnostics zip to PATH. @@ -186,9 +186,15 @@ if [[ -n "$REPORT_DIR" ]]; then DOCTOR_REPORT_JSON="$REPORT_DIR/doctor-report.json" fi if [[ -z "$UPLOADER_REPORT_JSON" ]]; then - UPLOADER_REPORT_JSON="$REPORT_DIR/uploader-dry-run-report.json" + if [[ "$DO_UPLOAD" -eq 1 ]]; then + UPLOADER_REPORT_JSON="$REPORT_DIR/uploader-upload-report.json" + else + UPLOADER_REPORT_JSON="$REPORT_DIR/uploader-dry-run-report.json" + fi + fi + if [[ "$DO_UPLOAD" -eq 1 ]]; then + UPLOAD_REPORT_JSON="$UPLOADER_REPORT_JSON" fi - UPLOAD_REPORT_JSON="$REPORT_DIR/uploader-upload-report.json" fi command_manifest_json="$tmp_root/support-command-manifest.json" @@ -394,31 +400,24 @@ else fi fi -dry_run_runtime_args=("${runtime_args[@]}") -if [[ -n "$UPLOADER_REPORT_JSON" ]]; then - dry_run_runtime_args+=("--report-json=$UPLOADER_REPORT_JSON") -fi -if run_bazel run "--config=$BAZEL_CONFIG" "$UPLOAD_TARGET" -- "${dry_run_runtime_args[@]}" --dry-run --validate-enrichment; then - dry_run_status=0 -else - dry_run_status=$? - if [[ "$final_status" -eq 0 ]]; then - final_status="$dry_run_status" - fi -fi - +uploader_runtime_args=("${runtime_args[@]}") if [[ "$DO_UPLOAD" -eq 1 ]]; then - upload_runtime_args=("${runtime_args[@]}") if [[ -n "$UPLOAD_REPORT_JSON" ]]; then - upload_runtime_args+=("--report-json=$UPLOAD_REPORT_JSON") + uploader_runtime_args+=("--report-json=$UPLOAD_REPORT_JSON") fi - if run_bazel run "--config=$BAZEL_CONFIG" "$UPLOAD_TARGET" -- "${upload_runtime_args[@]}"; then - : - else - upload_status=$? - if [[ "$final_status" -eq 0 ]]; then - final_status="$upload_status" - fi + uploader_runtime_args+=(--validate-enrichment) +else + if [[ -n "$UPLOADER_REPORT_JSON" ]]; then + uploader_runtime_args+=("--report-json=$UPLOADER_REPORT_JSON") + fi + uploader_runtime_args+=(--dry-run --validate-enrichment) +fi +if run_bazel run "--config=$BAZEL_CONFIG" "$UPLOAD_TARGET" -- "${uploader_runtime_args[@]}"; then + : +else + uploader_status=$? + if [[ "$final_status" -eq 0 ]]; then + final_status="$uploader_status" fi fi diff --git a/tools/tests/integration/run_mock_server_tests.sh b/tools/tests/integration/run_mock_server_tests.sh index 7d993381..0c41ad23 100755 --- a/tools/tests/integration/run_mock_server_tests.sh +++ b/tools/tests/integration/run_mock_server_tests.sh @@ -4032,13 +4032,6 @@ if ! grep -q 'orchestrion_mode = "test_optimization"' "$GUIDED_BOOT_WS/tools/bui exit 1 fi -# The release tracer pinned by default in this fixture is v2.9.1. The generated -# wrapper above must still default to test_optimization mode, but that release -# does not include the unreleased Go testing Orchestrion package needed to build -# a real instrumented test binary. Run the raw target for this temporary runtime -# smoke so the integration still validates generated Bazel wiring, staged -# sources, metadata shape, doctor behavior, and payload discovery without -# depending on unreleased tracer internals. GUIDED_BEP_JSON="$GUIDED_BOOT_WS/.topt/guided-bootstrap.bep.json" mkdir -p "$(dirname "$GUIDED_BEP_JSON")" rm -f "$GUIDED_BEP_JSON" @@ -4046,7 +4039,7 @@ rm -f "$GUIDED_BEP_JSON" cd "$GUIDED_BOOT_WS" "$BAZEL" "${BAZEL_FLAGS[@]}" test --config=test-optimization \ --build_event_json_file="$GUIDED_BEP_JSON" \ - //src/go-project:hello_test__raw_go_test \ + //src/go-project:hello_test \ "${BAZEL_TEST_FLAGS[@]}" \ "${REPO_ENVS[@]}" ) @@ -4071,7 +4064,7 @@ import sys import zipfile from pathlib import Path -root = Path(os.environ["GUIDED_TESTLOGS_DIR"]) / "src" / "go-project" / "hello_test__raw_go_test" +root = Path(os.environ["GUIDED_TESTLOGS_DIR"]) / "src" / "go-project" / "hello_test" direct_path = root / "test.outputs" / "bazel_target_metadata.json" payload = None source = "" diff --git a/tools/tests/python/test_python_tools.py b/tools/tests/python/test_python_tools.py index f83f00a4..40e1d8de 100644 --- a/tools/tests/python/test_python_tools.py +++ b/tools/tests/python/test_python_tools.py @@ -621,9 +621,9 @@ def test_bash_wrapper_uploads_after_partial_validation_failures(self) -> None: for line in log_path.read_text(encoding="utf-8").splitlines() if "//:dd_upload_payloads" in line ] - self.assertEqual(2, len(upload_commands), upload_commands) - self.assertIn("--dry-run --validate-enrichment", upload_commands[0]) - self.assertNotIn("--dry-run", upload_commands[1]) + self.assertEqual(1, len(upload_commands), upload_commands) + self.assertIn("--validate-enrichment", upload_commands[0]) + self.assertNotIn("--dry-run", upload_commands[0]) def test_bash_wrapper_support_bundle_preserves_failed_test_status(self) -> None: """Validate Bash support bundle path preserves the original test status.""" @@ -977,9 +977,9 @@ def test_powershell_wrapper_uploads_after_partial_validation_failures(self) -> N for line in log_path.read_text(encoding="utf-8").splitlines() if "//:dd_upload_payloads" in line ] - self.assertEqual(2, len(upload_commands), upload_commands) - self.assertIn("--dry-run --validate-enrichment", upload_commands[0]) - self.assertNotIn("--dry-run", upload_commands[1]) + self.assertEqual(1, len(upload_commands), upload_commands) + self.assertIn("--validate-enrichment", upload_commands[0]) + self.assertNotIn("--dry-run", upload_commands[0]) def test_powershell_wrapper_support_bundle_preserves_failed_test_status(self) -> None: """Validate PowerShell support bundle path preserves the original test status.""" @@ -4667,6 +4667,54 @@ def test_bep_artifact_stage_helper_emits_stable_tsv_protocol(self) -> None: self.assertTrue(any(line.startswith("staged\t//pkg:target\tpkg/target/test.outputs\t") for line in stdout_lines)) self.assertEqual("", result.stderr) + def test_bep_artifact_stage_helper_skips_invalid_sibling_bep(self) -> None: + """Validate one malformed BEP does not prevent staging a valid sibling.""" + helper = _runfile("tools/core/bep_artifact_stage_helper.py") + doctor_runtime = _runfile("tools/core/test_optimization_doctor.py") + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source_output = self._write_doctor_output(root / "external-artifacts", "module", "//pkg:target") + valid_bep = root / "valid.bep.json" + self._write_bep( + valid_bep, + [{ + "id": {"testResult": {"label": "//pkg:target", "run": 1, "shard": 1, "attempt": 1}}, + "testResult": { + "status": "PASSED", + "testActionOutput": [{ + "name": "test.outputs", + "uri": source_output.as_uri(), + "pathPrefix": ["bazel-out", "k8-fastbuild", "testlogs", "pkg", "target"], + }], + }, + }], + ) + invalid_bep = root / "invalid.bep.json" + invalid_bep.write_text("{not-json}\n", encoding="utf-8") + result = subprocess.run( + [ + sys.executable, + str(helper), + "--doctor-runtime", + str(doctor_runtime), + "--staging-dir", + str(root / ".topt" / "bep-artifacts"), + "--remote-artifacts=download", + "--artifact-source=bep", + str(invalid_bep), + str(valid_bep), + ], + cwd=root, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + self.assertEqual(0, result.returncode, result.stderr) + self.assertIn("failed to parse BEP JSON", result.stderr) + self.assertIn("selected\t//pkg:target\tpkg/target/test.outputs", result.stdout) + def test_bep_artifact_stage_helper_selects_remote_without_downloader_in_download_mode(self) -> None: """Validate helper suppresses stale local fallback even when remote staging skips.""" helper = _runfile("tools/core/bep_artifact_stage_helper.py") @@ -6808,12 +6856,89 @@ def test_generated_uploaders_process_valid_payloads_before_reporting_partial_out check=False, ) partial_output = partial_result.stdout + partial_result.stderr - self.assertEqual(0, partial_result.returncode, partial_output) + self.assertNotEqual(0, partial_result.returncode, partial_output) self.assertIn("continuing with other fresh outputs", partial_output) partial_report_doc = json.loads(partial_report.read_text(encoding="utf-8")) - self.assertEqual("ok", partial_report_doc["status"]) + self.assertEqual("fail", partial_report_doc["status"]) self.assertEqual(1, partial_report_doc["payloads"]["tests"]["processed"]) + def test_generated_uploaders_process_valid_bep_siblings_before_reporting_missing_or_invalid_bep(self) -> None: + """Validate bad BEP inputs fail the result only after valid siblings are processed.""" + _require_command(self, "jq", "jq is required for Bash BEP freshness parsing") + bash = _require_functional_bash(self) + if os.name == "nt": + self.skipTest("generated PowerShell uploader execution smoke is covered on non-Windows") + pwsh = _require_command(self, "pwsh", "pwsh is required for generated PowerShell execution") + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + valid_bep, expected_targets = self._write_mixed_fresh_output_fixture(root) + expected_targets.write_text( + json.dumps({"schema_version": 1, "targets": ["//pkg:valid"]}), + encoding="utf-8", + ) + missing_bep = root / "missing.bep.json" + invalid_bep = root / "invalid.bep.json" + invalid_bep.write_text("{not-json}\n", encoding="utf-8") + runfiles_dir = self._write_non_sibling_runtime_runfiles(root) + env = self._generated_uploader_smoke_env(root, runfiles_dir) + + generated_bash = root / "generated_uploader.sh" + generated_bash.write_text( + _render_uploader_runtime_template( + "tools/core/uploader_bash_runtime.sh.tpl", + doctor_runtime_rloc=_NON_SIBLING_DOCTOR_RUNTIME_RLOC, + expected_targets_file_path=str(expected_targets), + fail_on_error=True, + ), + encoding="utf-8", + ) + generated_bash.chmod(0o755) + generated_powershell = root / "generated_uploader.ps1" + generated_powershell.write_text( + _render_uploader_runtime_template( + "tools/core/uploader_powershell_runtime.ps1.tpl", + doctor_runtime_rloc=_NON_SIBLING_DOCTOR_RUNTIME_RLOC, + expected_targets_file_path=str(expected_targets), + fail_on_error=True, + ), + encoding="utf-8", + ) + + for name, command in ( + ("Bash", [bash, str(generated_bash)]), + ("PowerShell", [pwsh, "-NoLogo", "-NoProfile", "-File", str(generated_powershell)]), + ): + with self.subTest(runtime=name): + report = root / f"{name.lower()}-bad-bep-report.json" + result = subprocess.run( + [ + *command, + "--bep-json", str(missing_bep), + "--bep-json", str(invalid_bep), + "--bep-json", str(valid_bep), + "--freshness-source=bep", + "--freshness-mode=required", + "--artifact-source=bep", + "--report-json", str(report), + "--dry-run", + ], + cwd=root, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=30, + check=False, + ) + output = result.stdout + result.stderr + self.assertEqual(1, result.returncode, output) + self.assertIn("BEP JSON not found", output) + self.assertIn("failed to parse BEP JSON", output) + report_doc = json.loads(report.read_text(encoding="utf-8")) + self.assertEqual(1, report_doc["payloads"]["tests"]["processed"]) + self.assertGreaterEqual(report_doc["upload_failures"], 2) + def test_generated_uploaders_accept_empty_expected_target_set(self) -> None: """Validate an invocation without optimized targets ignores unrelated BEP rows.""" _require_command(self, "jq", "jq is required for Bash BEP freshness parsing") @@ -7330,7 +7455,7 @@ def test_generated_uploaders_require_configured_remote_artifacts(self) -> None: for runtime, result in (("Bash", bash_result), ("PowerShell", powershell_result)): output = result.stdout + result.stderr - self.assertEqual(2, result.returncode, f"{runtime} output:\n{output}") + self.assertEqual(1, result.returncode, f"{runtime} output:\n{output}") self.assertIn("BEP references remote-only test outputs", output) self.assertIn("local test.outputs was not found", output) From 8bf79cd6a20426f28f262c21fe29637ca2f92dee Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Wed, 26 Aug 2026 16:37:43 +0200 Subject: [PATCH 24/37] docs: plan deterministic stdlib cache publication --- .../rules_go_stdlib_cache_determinism_plan.md | 802 ++++++++++++++++++ 1 file changed, 802 insertions(+) create mode 100644 docs/rules_go_stdlib_cache_determinism_plan.md diff --git a/docs/rules_go_stdlib_cache_determinism_plan.md b/docs/rules_go_stdlib_cache_determinism_plan.md new file mode 100644 index 00000000..9a502221 --- /dev/null +++ b/docs/rules_go_stdlib_cache_determinism_plan.md @@ -0,0 +1,802 @@ + + +# Deterministic rules_go Standard-Library Cache Plan + +## Status + +Draft implementation plan for the Orchestrion v1.12.0 update. This document +describes the intended implementation and validation sequence; the cache fix is +not implemented merely by adding this plan. + +The work starts in `rules_test_optimization`, is validated through +`rules_test_optimization_tests`, and is adopted by `dd-source` only after a +published rules commit is available. The source changes must cover every +maintained `rules_go` profile: + +- `v0_60_0` +- `v0_61_1` +- `v0_62_0` + +It must preserve both supported Go execution modes: + +- ordinary Go builds and tests without Test Optimization; +- Go builds and tests with Orchestrion in `test_optimization` mode. + +Generic Orchestrion behavior must remain compatible as well, even though the +cross-repository end-to-end acceptance matrix is focused on the two consumer +modes above. + +## Objective + +Make the `GoStdlib` action output deterministic without losing the woven +standard-library archives required by Test Optimization. + +The implementation must separate two concepts that currently overlap: + +1. a writable, action-private Go build cache used by live `go install` and + `go list -export -deps` subprocesses; and +2. the Bazel-declared `-cacheout` TreeArtifact consumed by later Orchestrion + compile and link actions. + +No Go subprocess may use the declared TreeArtifact as its live `GOCACHE`. +Instead, Test Optimization must explicitly publish only the deterministic +archive data required by downstream actions. Ordinary Go builds must still +create the declared TreeArtifact, but leave it empty. + +## Problem Statement + +The Go build cache is not a reproducible Bazel action output. In addition to +archive data entries (`*-d`), the Go tool writes action index entries (`*-a`) +whose bytes include a wall-clock timestamp. Re-running the same command with +identical sources can therefore produce different cache bytes. + +The current `GoStdlib` integration allows that mutable cache format to become a +Bazel-declared output in two places: + +1. `stdlib.go` selects `-cacheout` as `GOCACHE` when the flag is present. +2. `env_orchestrion.go:newBufferedCommand` overwrites a subprocess's `GOCACHE` + with `env.stdlibCache`, which is the declared `-cacheout` directory. + +There is a third indirect writer. During publication, +`syncPersistedOrchestrionExportsToCache` calls +`resolveCacheStdlibExportsAt` for both the declared cache and the current +environment cache. That resolver executes `go list -export -deps` against the +selected cache root. Running it against the declared root creates the same +non-deterministic Go cache metadata even if the original stdlib build used a +private cache. + +The downstream `dd-source` patch +`third_party/rules_go/0019-Keep-stdlib-GOCACHE-off-the-declared-cacheout-output.patch` +correctly identifies the first writer and moves the primary stdlib build to a +private cache. Porting that patch literally is insufficient for this repository +because the two Orchestrion-specific writers would remain. + +## Terminology and Data Flow + +This plan uses these names consistently: + +- **scratch cache**: the private, writable `GOCACHE` rooted at + `/.gocache`. It may contain any files produced by Go and is + deleted when the `GoStdlib` action finishes. +- **declared cache**: the directory supplied by Bazel through `-cacheout` and + recorded as `env.stdlibCache`. It is a TreeArtifact and must contain only + deterministic published files. +- **persisted stdlib exports**: the woven archives already copied under the + replicated GOROOT output and recorded by the existing persisted-export + manifest. +- **published cache exports**: the selected woven archives projected into the + declared cache using their Go cache data-entry paths, plus + `.orchestrion_stdlib_cache_manifest`. +- **consumer cache**: a later action's writable cache. It can be seeded from the + declared cache, but it is never the declared cache itself. + +The intended flow is: + +```text +GoStdlib action + | + +-- live go install / go list + | GOCACHE=/.gocache + | produces mutable *-a and data *-d entries + | + +-- Orchestrion persistence + | produces woven archives under the replicated GOROOT + | + +-- deterministic publisher + reads archive locations from the scratch cache + copies selected woven bytes to equivalent *-d paths in -cacheout + writes a sorted .orchestrion_stdlib_cache_manifest + +Later compile/link action + | + +-- RULES_GO_ORCHESTRION_STDLIB_CACHE= + +-- GOCACHE= + +-- seedWovenStdlibCache copies/hard-links declared archives into GOCACHE +``` + +## Required Invariants + +The implementation is complete only if all of the following remain true. + +### Cache ownership + +- `GOCACHE` and `env.stdlibCache` never refer to the same directory. +- Every `go install`, `go list`, or Orchestrion subprocess receives a writable + scratch or consumer cache through `GOCACHE`. +- `RULES_GO_ORCHESTRION_STDLIB_CACHE` may point to `env.stdlibCache`, but treats + it as a read-only source of published archives. +- The scratch cache is removed at the end of the stdlib action on success and + failure. +- Bazel's declared cache directory exists whenever `-cacheout` is provided, + including ordinary non-Orchestrion builds. + +### Declared output contents + +- Without Orchestrion, the declared cache is empty. +- With Test Optimization, the declared cache contains only: + - selected woven archive data entries at stable relative `*-d` paths; and + - `.orchestrion_stdlib_cache_manifest`. +- The declared cache contains no `*-a` action index entries, `trim.txt`, Go + cache README files, temporary files, lock files, or unrelated cache-prefix + siblings. +- Manifest records are sorted deterministically and contain relative paths. +- Two isolated executions with identical inputs produce the same set of paths + and identical bytes for every declared output file. + +### Runtime correctness + +- Ordinary Go targets compile, link, and test without Test Optimization. +- Test Optimization still weaves the required stdlib closure. +- Synthetic test-main and final link actions can find the published woven + archives. +- A real Go test emits Test Optimization test and telemetry payloads. A green + build without payloads is a failure, not successful validation. +- Existing partial-result, doctor, enrichment-validation, and upload behavior + remains unchanged; this fix is confined to stdlib cache production and + consumption. + +## Scope + +### In scope + +- The base `rules_go` trees for all three registered upstreams. +- Builder unit tests covering cache ownership, publication, manifest contents, + path safety, and command environments. +- Generated public patch profiles, metadata, and changed-files reports. +- The generated-profile functional smoke in + `tools/dev/verify_rules_go_profiles.py`. +- Consumer validation in `rules_test_optimization_tests` for every maintained + upstream in both disabled/plain and Test Optimization modes. +- Adoption in `dd-source`, including removal of its temporary `0019` overlay, + after the rules change has been published and pinned. + +### Out of scope + +- Replacing the Go build cache implementation. +- Persisting or remotely sharing the entire Go `GOCACHE`. +- Changing the public Test Optimization configuration or onboarding API. +- Adding another user-facing Orchestrion mode. +- Parallelizing doctor or payload upload work. +- Changing which stdlib packages are woven, except where required to preserve + the existing closure while changing cache storage. +- Refactoring unrelated Orchestrion cache, jobserver, resolver, uploader, or + payload code. + +## Design Decisions + +### D-1: Fix the base trees, then regenerate + +The three directories under `third_party/rgo/*/base` are the editable source of +truth. Generated files under `third_party/rules_go_orchestrion/patches` must not +be hand-edited. After the base implementations and tests agree, use the +repository generators once to refresh all derived artifacts. + +This is especially important on the current PR branch because generated patch +files already contain pending review fixes. Regenerating once at the end avoids +overlapping generated churn and preserves those changes. + +### D-2: Keep the current Go-cache-shaped archive layout + +Published archives will retain their cache-relative `*-d` paths. Existing +manifest readers and cache-seeding logic already understand this format, so it +is the smallest compatible change. A new custom flat archive layout would +increase the diff and require more consumer rewrites without improving the +determinism guarantee. + +### D-3: Resolve layout only against writable scratch + +`go list -export -deps` remains the authority for locating Go cache archive +paths, but it runs only against the scratch cache during stdlib publication. +The resulting path for each package is made relative to the scratch root and +projected into the declared root. The resolver must never run with the declared +root as `GOCACHE`. + +### D-4: Publish an allowlist, not a copied cache tree + +The publisher copies the existing woven archive bytes for the selected package +set. It does not recursively copy a Go cache directory or a two-character cache +prefix. This prevents an adjacent `*-a`, metadata file, or unrelated data entry +from entering the declared TreeArtifact. + +### D-5: Treat path containment as a correctness boundary + +For every projected destination: + +1. compute the source archive's path relative to the absolute scratch root; +2. reject absolute relative results; +3. reject `..` and any path whose cleaned form escapes the root; +4. join the validated relative path to the declared root; +5. verify the resulting destination is still contained by the declared root. + +The implementation must use `filepath` semantics so the same validation works +on Linux, macOS, and Windows. + +### D-6: Prove determinism from bytes, not filesystem timestamps + +The verifier compares a canonical inventory of relative path, file type, size, +and SHA-256 content digest. Filesystem mtimes are not part of the TreeArtifact's +semantic content and must not be used to create a false failure. Symlinks, if +any unexpectedly appear, must be reported rather than silently followed. + +## Implementation Plan + +### S-1: Introduce explicit cache setup in `stdlib.go` + +Apply the same logical change in: + +- `third_party/rgo/v0_60_0/base/go/tools/builders/stdlib.go` +- `third_party/rgo/v0_61_1/base/go/tools/builders/stdlib.go` +- `third_party/rgo/v0_62_0/base/go/tools/builders/stdlib.go` + +In `stdlib`, immediately after the replicated GOROOT is selected: + +1. Always define `cachePath` as `filepath.Join(output, ".gocache")`. +2. If `-cacheout` is non-empty, store `abs(*cacheOut)` in + `goenv.stdlibCache`; do not assign it to `cachePath`. +3. Set process `GOCACHE` to the scratch `cachePath`. +4. Create the scratch directory and register unconditional deferred removal. +5. If `goenv.stdlibCache` is non-empty, create that directory separately so + the declared TreeArtifact exists even when the plain path publishes no + files. +6. Remove `shouldRemoveStdlibCache`; cache ownership no longer depends on + Orchestrion or on the presence of `-cacheout`. + +Errors must identify whether scratch preparation or declared-output +preparation failed and include the affected path. + +Expected behavior after S-1 alone: + +- plain stdlib builds no longer write live cache data into `-cacheout`; +- Orchestrion has access to both roots through separate fields/environment; +- publication is not yet safe until S-2 and S-3 are complete. + +### S-2: Stop Orchestrion commands from overriding `GOCACHE` + +Apply the same logical change to each version's +`go/tools/builders/env_orchestrion.go`. + +Change `env.newBufferedCommand` so it: + +- preserves the `GOCACHE` already present in `os.Environ()`; +- sets `RULES_GO_ORCHESTRION_STDLIB_CACHE` when `env.stdlibCache` exists and is + a directory; +- never assigns `env.stdlibCache` to `GOCACHE`. + +Update the function comment to describe the declared cache as an archive +source, not a cache override. Do not duplicate scratch-cache creation here: +stdlib setup and the existing command-environment helpers already own writable +cache selection. + +Audit all direct uses of `env.stdlibCache` in the builder package. Classify each +one as either: + +- a read of the declared manifest/archive source; or +- an accidental attempt to use it as a writable Go cache. + +Only the latter are changed. In particular, preserve manifest readers and +`seedWovenStdlibCache`, which intentionally copy declared archives into a +different writable cache. + +### S-3: Make stdlib publication one-way and deterministic + +Refactor `syncPersistedOrchestrionExportsToCache` in each version's +`stdlib.go`. + +The new algorithm is: + +1. Return early when there is no environment, no persisted export, no selected + root, or no declared cache. +2. Read the current absolute scratch cache from `GOCACHE`. Reject an empty + value or an alias with the declared cache. +3. Create the declared cache directory without changing process `GOCACHE`. +4. Call `resolveCacheStdlibExportsAt(goenv, roots, scratchCache)` exactly once. +5. Sort the resolved package names. +6. For each selected package: + - require a persisted woven source archive; + - require a resolved scratch data-entry path; + - derive and validate its cache-relative destination; + - require the destination to be a data entry rather than an action index; + - copy the persisted woven archive bytes to that destination under the + declared cache; + - append `package=relative/path` to the manifest. +7. Write `.orchestrion_stdlib_cache_manifest` atomically after all archives + have been copied successfully. + +Prefer a small helper for safe path projection, for example a function that +takes `(scratchRoot, declaredRoot, scratchArchive)` and returns a validated +destination and relative manifest path. Keep it private to the builder package. + +The publisher must not: + +- build a `candidateCaches` list; +- temporarily set `GOCACHE` to the declared cache; +- call `resolveCacheStdlibExportsAt` against the declared cache; +- copy a whole Go cache prefix directory into the declared cache; +- include a manifest record for a package whose archive was not published. + +If later code in the same stdlib action needs the woven archive in scratch, it +may replace the corresponding scratch `*-d` entry with the woven bytes as a +separate operation. That mutable update remains inside scratch and is not a +substitute for explicit publication. + +### S-4: Preserve downstream manifest and seeding behavior + +Review, and change only if a test proves it necessary: + +- `readStdlibCacheManifest` +- `readAllStdlibCacheManifest` +- `currentWovenStdlibCacheKey` +- `seedWovenStdlibCache` +- `resolveCacheStdlibExportsAt` +- importcfg rewrite helpers that consume `goenv.stdlibCache` + +The desired downstream contract is: + +- manifest reads never invoke Go or mutate the declared cache; +- `seedWovenStdlibCache` resolves destinations against its writable + `cacheRoot`, then copies or hard-links source bytes from the declared cache; +- a consumer that receives `RULES_GO_ORCHESTRION_STDLIB_CACHE` can populate its + own `GOCACHE` without modifying the TreeArtifact; +- generic Orchestrion and Test Optimization continue to use their existing + package selection logic. + +Do not make `resolveCacheStdlibExportsAt` globally read-only. It is still valid +for ephemeral writable caches used by consumers. The caller, not the resolver, +must enforce that the declared cache is never passed as its writable root. + +### S-5: Add builder unit tests + +The existing builder test target includes `stdlib_test.go` in +`@rules_go//go/tools/builders:orchestrion_test`. Add the tests to the base trees +and keep the test sources identical across all three versions where the +production sources are identical. + +#### Cache setup and ownership tests + +- Replace `TestShouldRemoveStdlibCache` with tests for the new cache-selection + helper or directly test the smallest extracted setup seam. +- With no `-cacheout`, verify scratch is `/.gocache` and scheduled for + removal. +- With `-cacheout`, verify scratch remains `/.gocache`, declared cache + is stored separately, both directories exist, and the paths do not alias. +- Cover relative inputs normalized to absolute paths. + +Avoid testing deferred filesystem cleanup through the complete `stdlib` +command if a small ownership helper can prove the same contract more reliably. + +#### Command environment tests + +Add a focused test for `newBufferedCommand`: + +- set a known private `GOCACHE` in the process environment; +- set `env.stdlibCache` to a distinct existing directory; +- construct the command without executing it; +- assert its `GOCACHE` is still the private value; +- assert `RULES_GO_ORCHESTRION_STDLIB_CACHE` is the declared value; +- verify an absent or invalid declared directory does not replace `GOCACHE`. + +#### Publisher determinism tests + +Construct two fake scratch cache roots with: + +- identical selected `*-d` relative paths; +- identical persisted woven archive bytes; +- different `*-a` contents that model Go's timestamped index records; +- different unrelated metadata or prefix siblings. + +Publish each into a fresh declared directory and assert: + +- the canonical path-and-content inventories are identical; +- the expected `*-d` archives contain the woven bytes; +- no `*-a`, metadata, temporary, or unrelated sibling file is present; +- the manifest contains only successfully published packages; +- records and relative paths are identical and sorted. + +Where invoking the real resolver would make the unit test depend on a host Go +cache, factor the copying/projection portion into a deterministic helper and +test it with an explicit package-to-scratch-path map. Existing integration +tests will cover the real resolver. + +#### Negative tests + +- Reject a resolved path outside the scratch root. +- Reject an absolute or `..`-escaping manifest projection. +- Reject an action-index (`*-a`) source where a data entry is required. +- Return a useful error when a selected package has no persisted woven archive. +- Do not leave a complete-looking manifest after a failed publication. + +#### Existing regression tests + +Keep existing tests for: + +- persisted stdlib export installation; +- manifest reads; +- seeding a writable cache; +- importcfg rewrites; +- the standard-library package closure in `general` and + `test_optimization` modes. + +### S-6: Apply the implementation to every maintained upstream + +At the start of implementation, compare the relevant source files across the +three bases. They are currently byte-identical for `stdlib.go`, +`stdlib_test.go`, `importcfg.go`, and `env_orchestrion.go`, so the preferred +workflow is: + +1. implement and review the change in `v0_60_0`; +2. copy the exact logical change to `v0_61_1` and `v0_62_0`; +3. compare the resulting files or focused diffs across versions; +4. retain a version-specific difference only when the upstream base requires + it and document the reason in the diff. + +Do not assume future equality: re-run the comparison immediately before +propagating the change. + +### S-7: Extend generated-profile functional verification + +Extend the existing functional smoke in +`tools/dev/verify_rules_go_profiles.py`; do not create a parallel profile +verifier. + +For each registered upstream, the verifier already materializes a pristine +upstream tree, applies the generated public patch, creates a temporary WORKSPACE +consumer, executes a Test Optimization test, and inspects `aquery`. Add a +determinism phase to that same temporary consumer. + +#### Plain mode + +1. Build or test the Go target without the Orchestrion flags. +2. Identify the `GoStdlib` action and its declared `gocache` TreeArtifact. +3. Record a canonical inventory of the output. +4. Force a second execution with identical source inputs but without reusing + the first `GoStdlib` action result. +5. Record the second inventory and compare it with the first. +6. Assert both declared cache inventories are empty. + +#### Test Optimization mode + +1. Run with: + + ```text + --@io_bazel_rules_go//go/private/orchestrion:enabled=true + --@io_bazel_rules_go//go/private/orchestrion:mode=test_optimization + ``` + +2. Capture the `GoStdlib` declared cache inventory. +3. Force the same action to execute again in an isolated output state. +4. Compare every relative path and file digest. +5. Require a non-empty sorted manifest and its referenced archives. +6. Reject `*-a`, `trim.txt`, cache metadata, symlinks, and unmanifested files. +7. Preserve the existing `aquery` assertions proving that the generated patch + actually enabled Test Optimization. +8. Preserve the real test execution proving that the woven stdlib can be + consumed. + +The implementation may use two isolated Bazel output roots or an explicit +action-cache invalidation mechanism. It must not compare one execution with a +cache hit from that same execution. Disable remote and disk cache reuse for this +specific determinism check so both inventories come from executed actions. + +Use `aquery --output=jsonproto` or another structured Bazel output to locate +the action outputs. Do not depend on a hard-coded configuration hash in a +`bazel-out` path. + +#### Functional payload assertion + +The generated-profile smoke currently proves the test and action shape. If it +cannot observe Test Optimization payloads directly, add the smallest fixture +assertion that the test created the expected test and telemetry output files. +Keep full doctor/uploader validation in the consumer repository, where those +targets already exist. + +### S-8: Regenerate public profiles and metadata + +After all base files and focused tests are stable, regenerate and inspect the +derived outputs: + +```bash +python3 tools/dev/diff_rules_go_fork.py --all --write-report +python3 tools/dev/generate_rules_go_fork_maps.py --check +python3 tools/dev/materialize_rules_go_fork.py check --all +python3 tools/dev/verify_rules_go_profiles.py \ + --public-denylist tools/dev/private_leak_public_denylist.txt +python3 tools/dev/check_release_archive_contents.py +``` + +Expected generated changes include the matching +`third_party/rules_go_orchestrion/patches//base/0001-full-delta.patch` +files and any changed-files or metadata output owned by the generator. Inspect +every generated diff to ensure it contains the cache fix and the already +intended PR changes, without private paths or unrelated churn. + +If a generator modifies a base source file, stop and understand why before +continuing; the base source implementation must remain the reviewed source of +truth. + +## Validation Matrix + +The minimum acceptance matrix is: + +| rules_go | Plain Go | Test Optimization | Declared-cache bytes | Payload proof | +| --- | --- | --- | --- | --- | +| v0.60.0 | required | required | two isolated runs | required | +| v0.61.1 | required | required | two isolated runs | required | +| v0.62.0 | required | required | two isolated runs | required | + +For every row: + +- plain Go must compile and test without enabling Orchestrion; +- Test Optimization must compile, link, execute, and emit payloads; +- the declared cache must satisfy the file allowlist; +- two independently executed `GoStdlib` actions must produce identical + canonical inventories; +- a subsequent unchanged consumer run must be eligible for a Bazel cache hit. + +### Focused repository validation + +Before the full repository suite, run the smallest relevant builder and Python +tooling tests. Resolve exact labels with `bazel query` if necessary rather than +guessing a target that is not visible from the root module. + +Expected focused coverage includes: + +```bash +./bazelw test @rules_go//go/tools/builders:orchestrion_test \ + --noexperimental_split_xml_generation +./bazelw test @rules_go//go/tools/builders:importcfg_test \ + --noexperimental_split_xml_generation +./bazelw test //tools/tests/python:python_tools_test \ + --noexperimental_split_xml_generation +``` + +Then run: + +```bash +./bazelw test //... --noexperimental_split_xml_generation +``` + +On macOS, use the repository's documented Bazel wrapper and macOS validation +procedure. Do not introduce a source-level workaround for a local Bazel test +runner issue. + +### Consumer validation in `rules_test_optimization_tests` + +Before publishing, validate the local rule checkout through the sibling +consumer repository. Enable the existing local overrides for: + +- `datadog-rules-test-optimization`; +- `datadog-rules-test-optimization-go`; +- the selected `rules_go` base tree. + +Exercise all three upstreams using the existing fixture support: + +```bash +for rules_go_upstream in v0_60_0 v0_61_1 v0_62_0; do + RULES_GO_UPSTREAM="$rules_go_upstream" RTO_LOCAL_ARCHIVE=1 \ + ./fixtures/bzlmod-go/runtests +done +``` + +Use the repository's documented WORKSPACE and hermetic variants as required by +the canonical matrix. For each upstream, validate both phases explicitly: + +1. disabled/plain run without `--config=test-optimization`; +2. enabled run with `--config=test-optimization`; +3. doctor; +4. uploader enrichment validation using the current one-pass uploader + contract; +5. actual upload only where the CI workflow is already authorized to upload; +6. a second unchanged run to observe Bazel cache reuse. + +The local run must use Go 1.25.0, reset long-lived Bazel state when diagnosing a +discrepancy, and clear the stable Orchestrion cache when the test specifically +needs a cold Orchestrion execution. Do not clear caches between the first and +second runs whose purpose is to prove cache reuse. + +## Cross-Repository Rollout + +### R-1: Publish `rules_test_optimization` + +Only after the complete rules and consumer matrices pass: + +1. inspect the final diff and generated artifacts; +2. commit only the intended PR #214 files; +3. push the current PR branch; +4. record the exact remote commit SHA; +5. allow PR CI to verify Linux, macOS, and Windows behavior. + +Publishing is a separate authorized phase. Implementation and local validation +do not themselves authorize a commit or push. + +### R-2: Repin `rules_test_optimization_tests` + +Update PR #108 to the exact published rules SHA using its maintained refresh +tooling, including every generated fixture pin and hash. Run the full +multi-version consumer matrix from the pinned commit, not from a local override, +before considering the repin complete. + +### R-3: Adopt in `dd-source` + +After PR #108 proves the published artifact: + +1. update dd-source's rules/Test Optimization pin to the exact rules SHA; +2. regenerate dd-source's composed `rules_go` patch so the deterministic cache + fix is already present in its generated rules patch; +3. remove + `third_party/rules_go/0019-Keep-stdlib-GOCACHE-off-the-declared-cacheout-output.patch`; +4. remove the corresponding patch-list entry from `WORKSPACE`; +5. remove the `0019` section from `third_party/rules_go/README.md`; +6. verify there is no second application of the same source hunk. + +Validate dd-source in both modes: + +- a representative ordinary Go target without Test Optimization; +- the Test Optimization pilot target with + `--config=test-optimization`; +- the existing Reprise determinism check for `GoStdlib` outputs; +- the Test Optimization load job, including a fresh run and an unchanged + cached run; +- the existing Bazel cache hydration flow after the generated patch and pin are + final. + +For the Test Optimization run, require test and telemetry payload counts and +successful doctor/enrichment/upload stages. A Reprise success from a plain Go +target alone does not prove the Orchestrion path. + +## Failure Handling and Rollback + +### During Rule implementation + +- If plain mode is deterministic but Test Optimization is not, inspect the + declared tree for a remaining writer before changing the output comparison. +- If the declared tree is deterministic but payloads disappear, revert the + consumer-side cache-read change and inspect manifest/seeding behavior. Do not + accept build-only success. +- If only one upstream fails, compare its pristine upstream source and + generated patch with the other bases before adding a version-specific branch. +- If Windows path tests fail, fix path normalization and containment logic; do + not disable the negative test on Windows. + +### During downstream adoption + +- Keep dd-source's `0019` until the new rules commit has been successfully + pinned and its composed patch is verified to contain the replacement fix. +- If the repinned dd-source patch cannot apply, restore the previous exact rules + pin and retain `0019`; do not partially combine old and new hunks. +- If CI exposes a regression after publication, roll back by exact pin rather + than adding another overlay before determining whether the failure is in + Rule generation, consumer integration, or dd-source composition. + +## Risks and Mitigations + +### Silent loss of instrumentation + +**Risk:** compile and link succeed while downstream actions read plain stdlib +archives. + +**Mitigation:** require real payload emission plus doctor/enrichment validation, +not only builder unit tests or `aquery` output. + +### A hidden declared-cache writer remains + +**Risk:** another helper passes `env.stdlibCache` as `GOCACHE` and reintroduces +timestamped index files. + +**Mitigation:** audit every `env.stdlibCache`, `GOCACHE`, and +`resolveCacheStdlibExportsAt` call site; assert the output allowlist in both unit +and generated-profile tests. + +### Accidental cache-prefix copying + +**Risk:** copying siblings of a selected data archive includes an action index +or unrelated package entry. + +**Mitigation:** publish only explicit selected archive paths and verify that +every output except the manifest is referenced by the manifest. + +### Path escape or platform-specific path handling + +**Risk:** relative projection produces a destination outside the declared root, +especially with Windows volume or separator semantics. + +**Mitigation:** centralized `filepath`-based containment validation with Linux, +macOS, and Windows CI coverage and explicit negative unit tests. + +### False determinism from Bazel cache reuse + +**Risk:** the verifier compares a newly executed action with a Bazel cache hit +of the same output. + +**Mitigation:** use isolated output/action-cache state for both runs and record +evidence that `GoStdlib` executed twice. + +### Excessive stdlib preparation time + +**Risk:** resolving exports once per package or per destination cache increases +the already expensive preparation phase. + +**Mitigation:** resolve the complete selected package set once against scratch, +sort once, and copy once. Add no new `go list` invocation per package. + +### Generated patch drift on the active PR + +**Risk:** regenerating while review fixes are pending loses or duplicates +changes already present in generated profiles. + +**Mitigation:** modify base sources first, keep existing user changes intact, +then run the canonical generators once and inspect the combined generated diff. + +## Completion Criteria + +The work is complete only when: + +- all three base profiles implement the same cache ownership contract; +- no live Go or Orchestrion subprocess receives the declared cache as + `GOCACHE`; +- plain Go declared cache outputs are empty and reproducible; +- Test Optimization declared cache outputs contain only deterministic + manifested archive data; +- two isolated executions per mode and upstream have identical path-and-byte + inventories; +- builder unit tests, profile materialization, profile verification, release + archive checks, and the full rules suite pass; +- local and published consumer fixtures pass for `v0.60.0`, `v0.61.1`, and + `v0.62.0`; +- real Test Optimization tests emit valid test and telemetry payloads; +- an unchanged second consumer run demonstrates Bazel cache reuse; +- PR #108 is pinned to the final rules SHA and passes its CI matrix; +- dd-source is pinned to that same rules version, passes both plain and Test + Optimization validation, and no longer carries the standalone `0019` patch; +- no temporary overrides, scratch artifacts, or unrelated generated changes + remain in any of the three repositories. + +## Execution Checklist + +- [ ] Confirm the active Rule branch and preserve all existing review changes. +- [ ] Recompare relevant builder sources across all three bases. +- [ ] Implement private scratch and separate declared-cache setup. +- [ ] Remove the Orchestrion `GOCACHE=env.stdlibCache` override. +- [ ] Implement safe deterministic archive publication. +- [ ] Audit every declared-cache read and write call site. +- [ ] Add cache ownership and command-environment tests. +- [ ] Add deterministic publisher and path-safety tests. +- [ ] Run focused builder tests. +- [ ] Propagate and compare the implementation across all upstreams. +- [ ] Extend generated-profile plain and Test Optimization determinism smoke. +- [ ] Regenerate profiles, metadata, and changed-files reports. +- [ ] Run materialization, verifier, release, and full-suite checks. +- [ ] Validate local consumer fixtures across all versions and both modes. +- [ ] Publish the Rule commit when explicitly authorized. +- [ ] Repin and validate PR #108 from the published commit. +- [ ] Repin dd-source and regenerate its composed rules_go patch. +- [ ] Remove dd-source `0019` and its WORKSPACE/README references. +- [ ] Validate dd-source plain, Test Optimization, Reprise, load, and cache + hydration flows. +- [ ] Inspect final diffs and confirm all temporary local wiring is removed. + From 74ef9cea464d6f010b1c71649cea241e0d4b1e12 Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Wed, 26 Aug 2026 17:04:20 +0200 Subject: [PATCH 25/37] Fix final review findings --- modules/go/tests/BUILD.bazel | 9 ++++++++- modules/go/tests/test_macro.bzl | 12 ++++++++++++ modules/go/topt_go_stdlib.bzl | 9 +++++++++ tools/core/uploader_bash_runtime.sh.tpl | 13 ++++++++----- tools/core/uploader_powershell_runtime.ps1.tpl | 16 +++++++++++----- tools/tests/python/test_python_tools.py | 12 +++++++----- 6 files changed, 55 insertions(+), 16 deletions(-) diff --git a/modules/go/tests/BUILD.bazel b/modules/go/tests/BUILD.bazel index cf34749b..4cabc375 100644 --- a/modules/go/tests/BUILD.bazel +++ b/modules/go/tests/BUILD.bazel @@ -73,6 +73,7 @@ load( "resolve_topt_service_key_unknown_failure_test", "resolve_topt_service_key_unknown_target_rule", "stdlib_warmup_disabled_noop_test", + "stdlib_warmup_tags_test", "stdlib_warmup_transition_selects_test_optimization_test", "validate_orchestrion_mode_invalid_failure_test", "validate_orchestrion_mode_invalid_target_rule", @@ -788,9 +789,14 @@ stdlib_warmup_transition_selects_test_optimization_test( timeout = "short", ) +stdlib_warmup_tags_test( + name = "stdlib_warmup_tags_test", + size = "small", + timeout = "short", +) + dd_topt_go_stdlib_warmup( name = "stdlib_warmup_disabled_target", - tags = ["manual"], ) stdlib_warmup_disabled_noop_test( @@ -867,6 +873,7 @@ test_suite( ":selector_override_test", ":service_mapping_entries_filters_non_service_test", ":stdlib_warmup_disabled_noop_test", + ":stdlib_warmup_tags_test", ":stdlib_warmup_transition_selects_test_optimization_test", ":validate_orchestrion_mode_invalid_failure_test", ":validate_orchestrion_mode_test", diff --git a/modules/go/tests/test_macro.bzl b/modules/go/tests/test_macro.bzl index 59110f8a..3f39fbec 100644 --- a/modules/go/tests/test_macro.bzl +++ b/modules/go/tests/test_macro.bzl @@ -30,6 +30,7 @@ load( ) load( "@datadog-rules-test-optimization-go//:topt_go_stdlib.bzl", + "stdlib_warmup_tags_for_tests", "stdlib_warmup_transition_impl_for_tests", ) load( @@ -1037,6 +1038,14 @@ def _stdlib_warmup_transition_selects_test_optimization_test_impl(ctx): asserts.false(env, "@rules_go//go/private/orchestrion:enabled" in result) return unittest.end(env) +def _stdlib_warmup_tags_test_impl(ctx): + """Assert warmup targets remain excluded from wildcard builds.""" + env = unittest.begin(ctx) + asserts.equals(env, ["manual"], stdlib_warmup_tags_for_tests(None)) + asserts.equals(env, ["custom", "manual"], stdlib_warmup_tags_for_tests(["custom"])) + asserts.equals(env, ["custom", "manual"], stdlib_warmup_tags_for_tests(["custom", "manual"])) + return unittest.end(env) + def _stdlib_warmup_disabled_noop_test_impl(ctx): """Assert broad builds can analyze the warmup target without enablement.""" env = analysistest.begin(ctx) @@ -1229,6 +1238,9 @@ orch_transition_forwards_mode_test = unittest.make( stdlib_warmup_transition_selects_test_optimization_test = unittest.make( _stdlib_warmup_transition_selects_test_optimization_test_impl, ) +stdlib_warmup_tags_test = unittest.make( + _stdlib_warmup_tags_test_impl, +) stdlib_warmup_disabled_noop_test = analysistest.make( _stdlib_warmup_disabled_noop_test_impl, ) diff --git a/modules/go/topt_go_stdlib.bzl b/modules/go/topt_go_stdlib.bzl index 10149f34..b1ab5d13 100644 --- a/modules/go/topt_go_stdlib.bzl +++ b/modules/go/topt_go_stdlib.bzl @@ -20,6 +20,14 @@ def _stdlib_warmup_transition_impl(_settings, _attr): stdlib_warmup_transition_impl_for_tests = _stdlib_warmup_transition_impl +def _with_manual_tag(tags): + tags = list(tags or []) + if "manual" not in tags: + tags.append("manual") + return tags + +stdlib_warmup_tags_for_tests = _with_manual_tag + _stdlib_warmup_transition = transition( implementation = _stdlib_warmup_transition_impl, inputs = [], @@ -84,6 +92,7 @@ def dd_topt_go_stdlib_warmup(name, **kwargs): tags = ["manual"], visibility = ["//visibility:private"], ) + kwargs["tags"] = _with_manual_tag(kwargs.get("tags")) _dd_topt_go_stdlib_warmup( name = name, actual = ":" + transitioned_name, diff --git a/tools/core/uploader_bash_runtime.sh.tpl b/tools/core/uploader_bash_runtime.sh.tpl index 60dd4f80..5372cd2e 100644 --- a/tools/core/uploader_bash_runtime.sh.tpl +++ b/tools/core/uploader_bash_runtime.sh.tpl @@ -616,6 +616,7 @@ REPORT_REASON_CODE="running" REPORT_REASON="Uploader is still running." REPORT_NEXT_STEPS=() REPORT_UPLOAD_ATTEMPTED=0 +REPORT_UPLOAD_FAILED=0 REPORT_PAYLOADS_DISCOVERED_TESTS=0 REPORT_PAYLOADS_DISCOVERED_COVERAGE=0 REPORT_PAYLOADS_DISCOVERED_TELEMETRY=0 @@ -1216,7 +1217,7 @@ classify_uploader_result() { "Use the BEP from the exact matching bazel test invocation and verify each expected target is fresh or exclusively cached." return fi - if (( exit_code != 0 && DRY_RUN == 0 && REPORT_UPLOAD_ATTEMPTED > 0 && ${UPLOAD_FAILURES:-0} > 0 )); then + if (( exit_code != 0 && DRY_RUN == 0 && REPORT_UPLOAD_FAILED > 0 )); then set_report_result "upload_failed_http" \ "One or more payload uploads failed." \ "Check HTTP status diagnostics and Datadog credentials/site configuration." @@ -4972,6 +4973,7 @@ send_test_payload_part() { dbg "header[content-encoding]: Content-Encoding: gzip" fi fi + REPORT_UPLOAD_ATTEMPTED=1 if (( AGENTLESS == 1 )); then if http=$(curl_agentless "${CURL_FAIL_FLAG[@]}" -sS --connect-timeout 10 --max-time 60 "${CURL_RETRY_FLAGS[@]}" \ -X POST "${TEST_URL}" "${COMMON_HDRS[@]}" "${ce_hdr[@]+${ce_hdr[@]}}" -H "Content-Type: application/json" --data-binary @"${payload_file}" -o "$resp" -w "%{http_code}"); then @@ -4992,6 +4994,7 @@ send_test_payload_part() { dbg "upload_single_test: HTTP $http (rc=$rc; part=$part_index/$part_count; uncompressed_bytes=$uncompressed_bytes; transmitted_bytes=$transmitted_bytes; encoding=$encoding)" fi if [[ $rc -ne 0 || "$http" -lt 200 || "$http" -ge 300 ]]; then + REPORT_UPLOAD_FAILED=1 response_bytes="$(test_payload_size_bytes "$resp")" response_text="" truncated="false" @@ -5078,6 +5081,7 @@ upload_single_coverage() { fi dbg "headers: multipart/form-data (event + coveragex=${coverage_content_type})" fi + REPORT_UPLOAD_ATTEMPTED=1 if (( AGENTLESS == 1 )); then if http=$(curl_agentless -f -sS --connect-timeout 10 --max-time 60 "${CURL_RETRY_FLAGS[@]}" \ -X POST "${COV_URL}" "${COMMON_HDRS[@]}" \ @@ -5106,6 +5110,7 @@ upload_single_coverage() { fi rm -f "$resp" "$eventjson" 2>/dev/null || true if [[ $rc -ne 0 || "$http" -lt 200 || "$http" -ge 300 ]]; then + REPORT_UPLOAD_FAILED=1 return 1 fi return 0 @@ -5152,6 +5157,7 @@ upload_single_telemetry() { rm -f "$meta_file" 2>/dev/null || true return 1 fi + REPORT_UPLOAD_ATTEMPTED=1 if (( AGENTLESS == 1 )); then if http=$(curl_agentless -f -sS --connect-timeout 10 --max-time 60 "${CURL_RETRY_FLAGS[@]}" \ -X POST "${TELEMETRY_URL}" "${TELEMETRY_HDRS[@]}" -H "Content-Type: application/json" --data-binary @"${upload_body}" -o "$resp" -w "%{http_code}"); then @@ -5176,6 +5182,7 @@ upload_single_telemetry() { fi rm -f "$resp" "$meta_file" "$provider_body" 2>/dev/null || true if [[ $rc -ne 0 || "$http" -lt 200 || "$http" -ge 300 ]]; then + REPORT_UPLOAD_FAILED=1 return 1 fi return 0 @@ -5232,7 +5239,6 @@ upload_all_tests() { fi continue fi - REPORT_UPLOAD_ATTEMPTED=1 if upload_single_test "$f"; then log "uploaded test payload: $f" mark_fresh_output_handled "$outputs_dir" @@ -5291,7 +5297,6 @@ upload_all_coverage() { ((++REPORT_COVERAGE_PROCESSED)) continue fi - REPORT_UPLOAD_ATTEMPTED=1 if upload_single_coverage "$f"; then log "uploaded coverage payload: $f" mark_fresh_output_handled "$outputs_dir" @@ -5352,7 +5357,6 @@ upload_all_telemetry() { ((++REPORT_TELEMETRY_PROCESSED)) continue fi - REPORT_UPLOAD_ATTEMPTED=1 if upload_single_telemetry "$f" "${replacement_body:-$f}"; then log "uploaded telemetry payload: $f" mark_fresh_output_handled "$outputs_dir" @@ -5379,7 +5383,6 @@ upload_all_telemetry() { ((++REPORT_TELEMETRY_PROCESSED)) continue fi - REPORT_UPLOAD_ATTEMPTED=1 if upload_single_telemetry "$anchor_path" "$synthetic_body"; then log "uploaded telemetry payload: $anchor_path" ((++total)) diff --git a/tools/core/uploader_powershell_runtime.ps1.tpl b/tools/core/uploader_powershell_runtime.ps1.tpl index 75a6ce76..23abb53a 100644 --- a/tools/core/uploader_powershell_runtime.ps1.tpl +++ b/tools/core/uploader_powershell_runtime.ps1.tpl @@ -661,6 +661,7 @@ $script:ReportReasonCode = "running" $script:ReportReason = "Uploader is still running." $script:ReportNextSteps = [System.Collections.Generic.List[string]]::new() $script:ReportUploadAttempted = $false +$script:ReportUploadFailed = $false $script:ReportPayloadsDiscoveredTests = 0 $script:ReportPayloadsDiscoveredCoverage = 0 $script:ReportPayloadsDiscoveredTelemetry = 0 @@ -1132,7 +1133,7 @@ function Set-ClassifiedUploaderResult([int]$ExitCode) { @("Use the BEP from the exact matching bazel test invocation and verify each expected target is fresh or exclusively cached.") return } - if ($ExitCode -ne 0 -and -not $script:DryRun -and $script:ReportUploadAttempted -and $script:UploadFailures -gt 0) { + if ($ExitCode -ne 0 -and -not $script:DryRun -and $script:ReportUploadFailed) { Set-ReportResult "upload_failed_http" ` "One or more payload uploads failed." ` @("Check HTTP status diagnostics and Datadog credentials/site configuration.") @@ -3862,6 +3863,7 @@ function Send-PostJson( $encoding = 'identity' Dbg "Send-PostJson: Content-Type=application/json" } + $script:ReportUploadAttempted = $true $resp = $client.PostAsync($url, $content).GetAwaiter().GetResult() if ($resp.IsSuccessStatusCode) { if ($script:DebugMode) { @@ -3873,6 +3875,7 @@ function Send-PostJson( $body = $resp.Content.ReadAsStringAsync().GetAwaiter().GetResult() Dbg "Send-PostJson: HTTP $([int]$resp.StatusCode) on attempt $attempt" if ($attempt -eq $maxRetries) { + $script:ReportUploadFailed = $true $bounded = Format-BoundedUploadResponse $body Log "upload failed: source='$SourcePath' part=$PartIndex/$PartCount http=$([int]$resp.StatusCode) encoding=$encoding uncompressed_bytes=$uncompressedBytes compressed_bytes=$compressedBytes transmitted_bytes=$transmittedBytes response_bytes=$($bounded.Bytes) response_truncated=$($bounded.Truncated.ToString().ToLowerInvariant()) response_body='$($bounded.Text)'" return [bool]$false @@ -3881,6 +3884,7 @@ function Send-PostJson( } catch { Dbg "Send-PostJson: Exception on attempt $attempt - $_" if ($attempt -eq $maxRetries) { + $script:ReportUploadFailed = $true $encoding = if ($script:GzipPayloads) { 'gzip' } else { 'identity' } $compressedBytes = if ($script:GzipPayloads -and $null -ne $compressed) { $compressed.Length } else { 'none' } $transmittedBytes = if ($script:GzipPayloads -and $null -ne $compressed) { $compressed.Length } else { $uncompressedBytes } @@ -4566,6 +4570,7 @@ function Send-PostRawJson([string]$url, [hashtable]$headers, [string]$file) { $content = New-Object System.Net.Http.ByteArrayContent -ArgumentList (, $bytes) $content.Headers.ContentType = 'application/json' Dbg "Send-PostRawJson: Content-Type=application/json" + $script:ReportUploadAttempted = $true $resp = $client.PostAsync($url, $content).GetAwaiter().GetResult() if ($resp.IsSuccessStatusCode) { if ($script:DebugMode) { @@ -4577,6 +4582,7 @@ function Send-PostRawJson([string]$url, [hashtable]$headers, [string]$file) { $body = $resp.Content.ReadAsStringAsync().GetAwaiter().GetResult() Dbg "Send-PostRawJson: HTTP $([int]$resp.StatusCode) on attempt $attempt" if ($attempt -eq $maxRetries) { + $script:ReportUploadFailed = $true Log "upload failed: HTTP $([int]$resp.StatusCode) $body" return [bool]$false } @@ -4584,6 +4590,7 @@ function Send-PostRawJson([string]$url, [hashtable]$headers, [string]$file) { } catch { Dbg "Send-PostRawJson: Exception on attempt $attempt - $_" if ($attempt -eq $maxRetries) { + $script:ReportUploadFailed = $true Log "upload failed: $_" return [bool]$false } @@ -4766,6 +4773,7 @@ function Upload-SingleCoverage([string]$FilePath) { $covContent.Headers.ContentType = $coverageContentType $content.Add($covContent, 'coveragex', $coverageFileName) Dbg "Upload-SingleCoverage: posting '$FilePath' (attempt $attempt/$maxRetries; Content-Type=multipart/form-data; coveragex=$coverageContentType)" + $script:ReportUploadAttempted = $true $resp = $client.PostAsync($CovUrl, $content).GetAwaiter().GetResult() if ($resp.IsSuccessStatusCode) { $uploaded = $true @@ -4777,6 +4785,7 @@ function Upload-SingleCoverage([string]$FilePath) { $respBody = $resp.Content.ReadAsStringAsync().GetAwaiter().GetResult() Dbg "Upload-SingleCoverage: HTTP $([int]$resp.StatusCode) on attempt $attempt" if ($attempt -eq $maxRetries) { + $script:ReportUploadFailed = $true # Only emit user-facing error after final retry to avoid # noisy logs for transient first-attempt failures. Log "coverage upload failed: HTTP $([int]$resp.StatusCode) $respBody" @@ -4785,6 +4794,7 @@ function Upload-SingleCoverage([string]$FilePath) { } catch { Dbg "Upload-SingleCoverage: Exception on attempt $attempt - $_" if ($attempt -eq $maxRetries) { + $script:ReportUploadFailed = $true Log "coverage upload failed: $_" } } finally { @@ -4915,7 +4925,6 @@ function Upload-AllTests { } continue } - $script:ReportUploadAttempted = $true $uploadedResult = @(Upload-SingleTest $f.FullName) $uploaded = $false if ($uploadedResult.Count -gt 0) { @@ -4969,7 +4978,6 @@ function Upload-AllCoverage { $script:ReportCoverageProcessed++ continue } - $script:ReportUploadAttempted = $true $uploadedResult = @(Upload-SingleCoverage $f.FullName) $uploaded = $false if ($uploadedResult.Count -gt 0) { @@ -5024,7 +5032,6 @@ function Upload-AllTelemetry { $script:ReportTelemetryProcessed++ continue } - $script:ReportUploadAttempted = $true $uploadedResult = @(Upload-SingleTelemetry $f.FullName $bodyPath) $uploaded = $false if ($uploadedResult.Count -gt 0) { @@ -5057,7 +5064,6 @@ function Upload-AllTelemetry { $script:ReportTelemetryProcessed++ continue } - $script:ReportUploadAttempted = $true $uploadedResult = @(Upload-SingleTelemetry $entry.AnchorPath $entry.BodyPath) $uploaded = $false if ($uploadedResult.Count -gt 0) { diff --git a/tools/tests/python/test_python_tools.py b/tools/tests/python/test_python_tools.py index 40e1d8de..00024382 100644 --- a/tools/tests/python/test_python_tools.py +++ b/tools/tests/python/test_python_tools.py @@ -7036,8 +7036,8 @@ def _assert_uploader_report_failure(self, report_path: Path, bep_path: Path) -> self.assertEqual("fail", report["result"]["status"]) self.assertEqual("payload_enrichment_failed", report["result"]["reason_code"]) self.assertFalse(report["upload"]["attempted"]) - self.assertTrue(report["upload"]["dry_run"]) - self.assertTrue(report["config"]["dry_run"]) + self.assertFalse(report["upload"]["dry_run"]) + self.assertFalse(report["config"]["dry_run"]) self.assertTrue(report["config"]["validate_enrichment"]) self.assertEqual([str(bep_path)], report["bep"]["files"]) self.assertEqual(1, report["payloads"]["test_outputs_dirs"]) @@ -7706,7 +7706,7 @@ def test_generated_bash_uploader_executes_bep_staging_runfiles(self) -> None: def test_generated_bash_uploader_writes_failure_report(self) -> None: """Validate generated Bash uploader writes a report for controlled upload failures.""" - _require_command(self, "jq", "jq is required for Bash dry-run enrichment validation") + _require_command(self, "jq", "jq is required for Bash enrichment validation") bash = _require_functional_bash(self) with tempfile.TemporaryDirectory() as tmp: @@ -7726,6 +7726,8 @@ def test_generated_bash_uploader_writes_failure_report(self) -> None: generated_bash.chmod(0o755) env = self._generated_uploader_smoke_env(root, runfiles_dir) env["DD_TEST_OPTIMIZATION_BEP_JSON"] = str(bep) + env["DD_API_KEY"] = "test-api-key" + env["DD_TEST_OPTIMIZATION_AGENTLESS_URL"] = "http://127.0.0.1:9" report = root / "uploader-report.json" env["DD_TEST_OPTIMIZATION_UPLOADER_REPORT_JSON"] = str(report) result = subprocess.run( @@ -7740,7 +7742,6 @@ def test_generated_bash_uploader_writes_failure_report(self) -> None: "--remote-artifacts=download", "--artifact-staging-dir", str(root / ".topt" / "bep-artifacts"), - "--dry-run", "--validate-enrichment", "--expected-enriched-tag=missing.required.tag", ], @@ -7886,6 +7887,8 @@ def test_generated_powershell_uploader_writes_failure_report(self) -> None: ) env = self._generated_uploader_smoke_env(root, runfiles_dir) env["DD_TEST_OPTIMIZATION_BEP_JSON"] = str(bep) + env["DD_API_KEY"] = "test-api-key" + env["DD_TEST_OPTIMIZATION_AGENTLESS_URL"] = "http://127.0.0.1:9" report = root / "uploader-report.json" env["DD_TEST_OPTIMIZATION_UPLOADER_REPORT_JSON"] = str(report) result = subprocess.run( @@ -7903,7 +7906,6 @@ def test_generated_powershell_uploader_writes_failure_report(self) -> None: "--remote-artifacts=download", "--artifact-staging-dir", str(root / ".topt" / "bep-artifacts"), - "--dry-run", "--validate-enrichment", "--expected-enriched-tag=missing.required.tag", ], From e852e47d3d213f18b05a7b73a1d802dcb238e47b Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Wed, 26 Aug 2026 17:21:02 +0200 Subject: [PATCH 26/37] docs: align validated uploader guidance --- AGENTS.md | 13 +++++++------ README.md | 17 ++++++++--------- docs/Configuration_Reference.md | 10 +++++----- docs/Initial_documentation.md | 6 +++--- docs/Installation_Reference.md | 21 +++++++++++---------- docs/Language_Onboarding.md | 8 ++++---- docs/Troubleshooting.md | 4 ++-- docs/Uploader_Reference.md | 10 +++++----- docs/internal_monorepo_go_rollout_guide.md | 11 +++++------ 9 files changed, 50 insertions(+), 50 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 2c9c7c99..a0bd027c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -95,12 +95,11 @@ The sync rule creates `@test_optimization_data//` containing: ```bash # Tests write payloads to TEST_UNDECLARED_OUTPUTS_DIR automatically # Bazel collects them to bazel-testlogs//test.outputs/ - test_status=0; doctor_status=0; dry_run_status=0; upload_status=0 + test_status=0; doctor_status=0; uploader_status=0 ./bazelw test //... || test_status=$? ./bazelw run //:dd_test_optimization_doctor || doctor_status=$? - ./bazelw run //:dd_upload_payloads -- --dry-run --validate-enrichment || dry_run_status=$? - DD_API_KEY="$DD_API_KEY" DD_SITE="$DD_SITE" ./bazelw run //:dd_upload_payloads || upload_status=$? - for status in "$test_status" "$doctor_status" "$dry_run_status" "$upload_status"; do + DD_API_KEY="$DD_API_KEY" DD_SITE="$DD_SITE" ./bazelw run //:dd_upload_payloads -- --validate-enrichment || uploader_status=$? + for status in "$test_status" "$doctor_status" "$uploader_status"; do if [ "$status" -ne 0 ]; then exit "$status"; fi done ``` @@ -156,9 +155,11 @@ The sync rule creates `@test_optimization_data//` containing: - Tests write payloads to `$TEST_UNDECLARED_OUTPUTS_DIR/payloads/{tests,coverage}` (Bazel's built-in writable directory). - Bazel automatically collects these to `bazel-testlogs///test.outputs/`. - In consumer workspaces, run `./bazelw run //:dd_test_optimization_doctor` - after tests complete, then run `./bazelw run //:dd_upload_payloads -- --dry-run --validate-enrichment`, then upload with `./bazelw run //:dd_upload_payloads`. + after tests complete, then run the uploader once with `--validate-enrichment`. + Add `--dry-run` to that invocation only when upload is disabled. When upload is authorized, process every available fresh valid payload even if - tests, doctor, or dry-run fail, and preserve the earliest failure as the job result. + tests or doctor fail, and preserve the earliest test, doctor, or uploader + failure as the job result. - For Go, route the repository's central `dd_go_test` wrapper through `dd_topt_go_test`; `--config=test-optimization` is the only user-facing enable switch. diff --git a/README.md b/README.md index 81e31d14..702ea61a 100644 --- a/README.md +++ b/README.md @@ -57,10 +57,10 @@ A managed command owns two Bazel phases behind one user-facing entrypoint: 3. run metadata sync and the exact selected tests with `--config=test-optimization`; 4. run the workspace doctor against the generated exact-target list, then run - uploader dry-run and optional upload. + one validated uploader pass: dry-run without upload, or real upload. The command reuses one manifest path and one resolved metadata snapshot for -test, doctor, dry-run, and upload. A later command invocation creates a new +test, doctor, and uploader. A later command invocation creates a new manifest path and fetches current backend state once. When the selected settings and module payloads are unchanged, those stable test inputs remain byte-identical and normal Bazel test-result cache hits are preserved. @@ -80,8 +80,7 @@ flowchart LR M --> R[One aggregate metadata repository] R --> T[Exact selected tests] T --> D[Doctor with exact target set] - D --> V[Uploader dry-run] - V --> X[Optional upload] + D --> V[Validated uploader] ``` See [Installation Reference](docs/Installation_Reference.md#manifest-driven-managed-gopython-monorepos), @@ -237,7 +236,7 @@ automatically. No per-target disable attribute is required. `--build_event_json_file` in `.bazelrc`; CI wrappers should create a fresh BEP file path for each Bazel test invocation so parallel jobs cannot overwrite each other. -5. Configure doctor, enrichment dry-run, and uploader with the matching BEP +5. Configure doctor and the validated uploader with the matching BEP files and BEP artifact staging through wrapper arguments: - `--bep-json=` once per Bazel test invocation - `--freshness-source=bep` @@ -1489,9 +1488,9 @@ $env:DD_SITE = "datadoghq.com" ``` **IMPORTANT**: Always preserve the test exit code. When upload is enabled, the -wrapper runs doctor and dry-run for diagnostics, then uploads every available -fresh valid payload even if tests or either validation step failed. The earliest -test, doctor, dry-run, or upload failure remains the job result. +wrapper runs the doctor, then validates enrichment and uploads every available +fresh valid payload in one uploader pass even if tests or doctor failed. The +earliest test, doctor, or uploader failure remains the job result. ### Important runtime requirements @@ -1999,7 +1998,7 @@ dd_topt_go_test( `embedsrcs`. When enabled, it changes the default `rundir` to `.` only if you did not already set `rundir`. An explicit `rundir` still wins unchanged. -Then run tests, doctor, enrichment dry-run, and upload with the same BEP-required +Then run tests, doctor, and one validated uploader pass with the same BEP-required flow shown in [Basic usage](#basic-usage): let the wrapper generate a fresh BEP file path for each `bazel test` invocation, pass those paths to doctor/uploader via repeatable `--bep-json`, and keep diff --git a/docs/Configuration_Reference.md b/docs/Configuration_Reference.md index 8d903cb0..87cb4521 100644 --- a/docs/Configuration_Reference.md +++ b/docs/Configuration_Reference.md @@ -261,7 +261,7 @@ be added to a user's `.bazelrc` or set in ordinary jobs. Enabled resolution requires it to name a valid schema-v1 manifest; disabled resolution ignores it. One managed command invocation must keep that exact manifest path and -environment value for test, doctor, uploader dry-run, and optional upload. +environment value for test, doctor, and the validated uploader. Those phases therefore share one resolved repository snapshot. A later command invocation owns a new temporary manifest path and performs one new fetch round. Equivalent backend settings and module payloads remain byte-identical test @@ -624,7 +624,7 @@ The doctor and/or uploader runtimes read these variables at `bazel run` time: | `DD_TEST_OPTIMIZATION_FRESHNESS_MODE` | Freshness mode: `auto`, `required`, `optional`, or `disabled` | | `DD_TEST_OPTIMIZATION_DOCTOR_REPORT_JSON` | Optional path for the doctor machine-readable diagnostic report | | `DD_TEST_OPTIMIZATION_UPLOADER_REPORT_JSON` | Optional path for the uploader machine-readable diagnostic report | -| `DD_TEST_OPTIMIZATION_REPORT_DIR` | Optional wrapper/report-script directory. CI wrappers write `doctor-report.json`, `uploader-dry-run-report.json`, and, when upload is enabled, `uploader-upload-report.json` under this directory unless explicit report paths override the doctor or dry-run uploader path | +| `DD_TEST_OPTIMIZATION_REPORT_DIR` | Optional wrapper/report-script directory. CI wrappers write `doctor-report.json` plus `uploader-dry-run-report.json` without upload or `uploader-upload-report.json` with upload unless explicit report paths override them | | `DD_TEST_OPTIMIZATION_SUPPORT_BUNDLE` | Optional doctor or wrapper path for the redacted support diagnostics zip | | `DD_TEST_OPTIMIZATION_SUPPORT_BUNDLE_COLLECTOR` | Optional override for the support bundle collector script. Doctor targets provide this through runfiles; wrappers default to `create_support_bundle.py` beside the wrapper | | `DD_TEST_OPTIMIZATION_PYTHON` | Optional Python interpreter used by wrapper support-bundle generation and helper scripts before falling back to `PYTHON`, `python3`, and `python` | @@ -676,9 +676,9 @@ Wrapper report options: | Option / variable | Purpose | |-------------------|---------| -| `--report-dir=` / `-ReportDir ` / `DD_TEST_OPTIMIZATION_REPORT_DIR` | Recommended CI artifact directory. Bash and PowerShell wrappers write separate doctor, dry-run uploader, and upload reports under this directory | +| `--report-dir=` / `-ReportDir ` / `DD_TEST_OPTIMIZATION_REPORT_DIR` | Recommended CI artifact directory. Bash and PowerShell wrappers write a doctor report plus exactly one uploader report under this directory | | `--doctor-report-json=` / `-DoctorReportJson ` / `DD_TEST_OPTIMIZATION_DOCTOR_REPORT_JSON` | Override only the wrapper doctor report path | -| `--uploader-report-json=` / `-UploaderReportJson ` / `DD_TEST_OPTIMIZATION_UPLOADER_REPORT_JSON` | Override only the wrapper dry-run uploader report path. Real upload still uses `/uploader-upload-report.json` when `--report-dir` is set | +| `--uploader-report-json=` / `-UploaderReportJson ` / `DD_TEST_OPTIMIZATION_UPLOADER_REPORT_JSON` | Override the selected wrapper uploader report path, for either dry-run or real upload | | `--support-bundle=` / `-SupportBundle ` / `DD_TEST_OPTIMIZATION_SUPPORT_BUNDLE` | Write a redacted support diagnostics zip containing reports, selected BEP summaries, command metadata, runtime metadata, and a Markdown summary | | `--support-bundle-collector=` / `-SupportBundleCollector ` / `DD_TEST_OPTIMIZATION_SUPPORT_BUNDLE_COLLECTOR` | Override the support bundle collector path. Defaults to `create_support_bundle.py` beside the wrapper | @@ -687,7 +687,7 @@ so customers do not need to copy helper scripts just to create a first-pass diagnostics package. The wrapper support bundle still requires the full `tools/test_optimization/` helper directory or an explicit `DD_TEST_OPTIMIZATION_SUPPORT_BUNDLE_COLLECTOR` path. Both modes are opt-in and -do not change the doctor, test, dry-run, upload, or final CI exit status. +do not change the doctor, test, uploader, or final CI exit status. Bundle generation failures are reported as warnings. | Bundle file | Purpose | diff --git a/docs/Initial_documentation.md b/docs/Initial_documentation.md index ab91a933..da90add2 100644 --- a/docs/Initial_documentation.md +++ b/docs/Initial_documentation.md @@ -119,7 +119,7 @@ flowchart TB TX --> D[Doctor exact-target validation] TY --> D TZ --> D - D --> U[Uploader dry-run and optional upload] + D --> U[Validated uploader] end ``` @@ -129,8 +129,8 @@ because repository rules cannot discover the final analyzed test set. The temporary manifest is therefore an internal handoff between those phases, not user-maintained configuration. -Within one managed command, test, doctor, uploader dry-run, and optional upload -reuse the exact same manifest path and external-repository snapshot. A later +Within one managed command, test, doctor, and the validated uploader reuse the +exact same manifest path and external-repository snapshot. A later command uses a new temporary path and fetches current backend state once. Stable settings and per-module payload files are the test action inputs, so unchanged backend responses preserve normal Bazel test-result caching. diff --git a/docs/Installation_Reference.md b/docs/Installation_Reference.md index 824dd55c..5990b6aa 100644 --- a/docs/Installation_Reference.md +++ b/docs/Installation_Reference.md @@ -126,8 +126,8 @@ RTO_ARCHIVE_TYPE="tar.gz" RULES_GO_UPSTREAM="v0_60_0" RULES_GO_VARIANT="base" RULES_GO_STRIP_PREFIX="third_party/rgo/v0_60_0/base" -DD_TRACE_GO_VERSION="v2.9.1" -ORCHESTRION_VERSION="v1.12.0" +DD_TRACE_GO_VERSION="v2.9.0" +ORCHESTRION_VERSION="v1.9.0" ``` The archive URL, SHA256, and prefix are tied to the repository commit. @@ -471,9 +471,9 @@ $env:DD_SITE = "datadoghq.com" During rollout debugging, prefer `--report-dir ` plus `--support-bundle ` on the CI wrapper, or set `DD_TEST_OPTIMIZATION_REPORT_DIR` and `DD_TEST_OPTIMIZATION_SUPPORT_BUNDLE`. -The wrapper writes separate -`doctor-report.json`, `uploader-dry-run-report.json`, and, when `--upload` is -enabled, `uploader-upload-report.json` files. When support bundle output is +The wrapper writes `doctor-report.json` plus exactly one uploader report: +`uploader-dry-run-report.json` without `--upload`, or +`uploader-upload-report.json` with it. When support bundle output is configured, it also writes `dd-test-optimization-support.zip` with redacted reports, selected BEP summaries, effective wrapper flags, runtime metadata, and `summary.md`. Use `--doctor-report-json` or @@ -521,9 +521,10 @@ python3 tools/test_optimization/render_report_summary.py \ --output .topt/reports/upload-diagnostics.md ``` -When upload is enabled, run the real uploader after the doctor and dry-run -attempts even if either validation failed. It processes every available fresh -valid payload while the wrapper preserves the earlier failure as the job result. +When upload is enabled, the wrapper runs the real uploader once after the doctor +and validates enrichment in that same pass. It processes every available fresh +valid payload even if tests or doctor failed while preserving the earliest +failure as the job result. For manual Go extension wiring, set `module_path` to the Go module path from `go.mod`: @@ -728,8 +729,8 @@ creating the manifest or its environment handoff manually. When and emits stable disabled stubs. When enabled, a missing or invalid manifest fails before any metadata HTTP request. -The command must reuse one manifest path for test, doctor, uploader dry-run, -and optional upload so all phases resolve the same metadata snapshot. The next +The command must reuse one manifest path for test, doctor, and the validated +uploader so all phases resolve the same metadata snapshot. The next command invocation creates a new temporary manifest path and fetches current backend state once. Equivalent selected settings/module files remain stable test action inputs, preserving normal Bazel test-result cache hits; variable diff --git a/docs/Language_Onboarding.md b/docs/Language_Onboarding.md index bad6a666..b828e003 100644 --- a/docs/Language_Onboarding.md +++ b/docs/Language_Onboarding.md @@ -136,7 +136,7 @@ The consumer command: 3. derives a service and runtime context from each full label; 4. writes a private invocation-scoped manifest; 5. runs the exact labels with `--config=test-optimization`, then doctor and - uploader dry-run. + one validated uploader pass. The Rule does not discover affected tests and does not prescribe a repository's service grammar. A common consumer policy is to derive an application service @@ -177,8 +177,8 @@ sequenceDiagram Runner->>Bazel: sync and test exact labels with config Bazel->>Sync: materialize selected Go/Python contexts Sync-->>Bazel: narrow per-context/module inputs - Runner->>Post: doctor exact targets, then dry-run - Post-->>User: validation result and optional upload + Runner->>Post: doctor exact targets, then validated uploader + Post-->>User: validation or upload result end ``` @@ -876,7 +876,7 @@ bazel run @datadog-rules-test-optimization-python//tools/dd_topt_py_bootstrap:dd The default output excludes `FETCH_SALT`. If you intentionally need fresh metadata, print the separate force-refresh command with `--print-refresh-snippet`, run that sync command once, then return to the normal -test, doctor, dry-run, and upload flow without `FETCH_SALT`. +test, doctor, and validated uploader flow without `FETCH_SALT`. ### Multi-service diff --git a/docs/Troubleshooting.md b/docs/Troubleshooting.md index 65f3f95d..0e27dc12 100644 --- a/docs/Troubleshooting.md +++ b/docs/Troubleshooting.md @@ -95,7 +95,7 @@ Use this escalation ladder: | Situation | Ask for | Why | | --- | --- | --- | | First customer response after tests already ran | Doctor `--support-bundle=` | Smallest command; built into the doctor target; no helper scripts required | -| CI failure where upload, enrichment, or dry-run behavior matters | Wrapper `--report-dir= --support-bundle=` | Includes doctor, uploader dry-run, optional upload report, BEP summaries, and effective wrapper flags | +| CI failure where upload, enrichment, or dry-run behavior matters | Wrapper `--report-dir= --support-bundle=` | Includes doctor, the selected uploader report, BEP summaries, and effective wrapper flags | | Repository cannot run the wrapper or doctor bundle | Raw `--report-json` files plus manual `create_support_bundle.py` output | Fallback only; raw reports may include internal paths until the redacted zip is created | Ask the customer to attach the zip, not screenshots of terminal output. If the @@ -286,7 +286,7 @@ repository rule receives exact targets and runtime contexts. ### Doctor or uploader triggers another metadata fetch One managed command invocation must pass the same temporary manifest path to -test, doctor, uploader dry-run, and optional upload. If the request log shows +test, doctor, and the validated uploader. If the request log shows another fetch during a post-test phase, check that the command did not create a new temporary directory or change `DD_TEST_OPTIMIZATION_SERVICES_MANIFEST` between child Bazel processes. diff --git a/docs/Uploader_Reference.md b/docs/Uploader_Reference.md index 9800196a..ffb78719 100644 --- a/docs/Uploader_Reference.md +++ b/docs/Uploader_Reference.md @@ -283,7 +283,7 @@ bazel run --config=test-optimization //:dd_upload_payloads | `DD_TEST_OPTIMIZATION_FRESHNESS_MODE` | `auto` | Cache-safety mode: `auto`, `required`, `optional`, or `disabled`. In CI, `auto` fails closed when no freshness source is available. | | `DD_TEST_OPTIMIZATION_DOCTOR_REPORT_JSON` | unset | Optional path for the doctor machine-readable diagnostic report. Equivalent to passing `--report-json=` after the doctor target's `--` separator. | | `DD_TEST_OPTIMIZATION_UPLOADER_REPORT_JSON` | unset | Optional path for the uploader machine-readable diagnostic report. Equivalent to passing `--report-json=` after the uploader target's `--` separator. | -| `DD_TEST_OPTIMIZATION_REPORT_DIR` | unset | Optional wrapper report directory. CI wrappers write `doctor-report.json`, `uploader-dry-run-report.json`, and optional `uploader-upload-report.json` under this directory. | +| `DD_TEST_OPTIMIZATION_REPORT_DIR` | unset | Optional wrapper report directory. CI wrappers write `doctor-report.json` plus `uploader-dry-run-report.json` without upload or `uploader-upload-report.json` with upload. | | `DD_TEST_OPTIMIZATION_EXECUTION_LOG_MODE` | `auto` | Legacy alias for freshness mode when `DD_TEST_OPTIMIZATION_FRESHNESS_MODE` is unset. | | `DD_TEST_OPTIMIZATION_EXECUTION_LOG_JSON` | unset | Optional explicit legacy execution-log fallback path. Prefer BEP for new CI integrations. The uploader does not auto-discover `.topt/bazel-execution-log.json` because stale execution-log files can authorize stale local outputs. | | `DD_TEST_OPTIMIZATION_ARTIFACT_SOURCE` | `local` | Artifact source for `test.outputs` materialization: `local`, `bep`, or `auto`. `local` preserves existing discovery. `bep` requires an explicit BEP JSON file and stages BEP-referenced artifacts. | @@ -427,10 +427,10 @@ common no-upload cases such as `bep_output_remote_only_without_downloader`, `no_payload_json_found`, `payload_enrichment_failed`, and `upload_skipped_dry_run`. -The CI wrapper writes a separate dry-run uploader report when `--report-dir` is -used. If only `DD_TEST_OPTIMIZATION_UPLOADER_REPORT_JSON` or -`--uploader-report-json` is configured and `--report-dir` is not set, that -single path is used by the dry-run invocation only. +The CI wrapper writes exactly one uploader report when `--report-dir` is used: +`uploader-dry-run-report.json` without upload, or `uploader-upload-report.json` +with upload. `DD_TEST_OPTIMIZATION_UPLOADER_REPORT_JSON` and +`--uploader-report-json` override that selected report path. Example: diff --git a/docs/internal_monorepo_go_rollout_guide.md b/docs/internal_monorepo_go_rollout_guide.md index 57869dbc..5a199e12 100644 --- a/docs/internal_monorepo_go_rollout_guide.md +++ b/docs/internal_monorepo_go_rollout_guide.md @@ -160,21 +160,20 @@ bazel test --config=test-optimization bazel test --config=test-optimization bazel test --config=test-optimization bazel run --config=test-optimization //tools/test_optimization:dd_test_optimization_doctor -bazel run --config=test-optimization //tools/test_optimization:dd_upload_payloads -- --dry-run --validate-enrichment -DD_API_KEY="$DD_API_KEY" DD_SITE="$DD_SITE" bazel run --config=test-optimization //tools/test_optimization:dd_upload_payloads +DD_API_KEY="$DD_API_KEY" DD_SITE="$DD_SITE" bazel run --config=test-optimization //tools/test_optimization:dd_upload_payloads -- --validate-enrichment bazel shutdown ``` -Run the real uploader after doctor and dry-run attempts even when an earlier -phase fails. It uploads available fresh valid failed-test payloads while the -wrapper preserves the earlier failure as the job result. +Run the real uploader after the doctor even when an earlier phase fails. It +validates enrichment and uploads available fresh valid failed-test payloads in +one pass while the wrapper preserves the earlier failure as the job result. The doctor must see JSON payloads, Bazel target metadata, Git metadata, and only valid Go payload-selection states. `module`, `module_override`, and `full_bundle_disabled` are valid. `full_bundle_no_match` is a rollout blocker unless the target was explicitly configured to allow it. -The dry-run enrichment step is the local proof that tags expected in Datadog are +Upload-time enrichment validation proves that tags expected in Datadog are present in the final upload body. Raw payload files on disk are intentionally not the final enriched body. From 83e661437f87390c060aa4a91b441015332bfed7 Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Wed, 26 Aug 2026 19:11:07 +0200 Subject: [PATCH 27/37] rules_go: make stdlib cache outputs deterministic --- .../base/go/tools/builders/env_orchestrion.go | 4 +- .../base/go/tools/builders/importcfg.go | 81 ++- .../base/go/tools/builders/importcfg_test.go | 41 ++ .../v0_60_0/base/go/tools/builders/stdlib.go | 220 +++--- .../base/go/tools/builders/stdlib_test.go | 286 +++++++- .../base/go/tools/builders/env_orchestrion.go | 4 +- .../base/go/tools/builders/importcfg.go | 81 ++- .../base/go/tools/builders/importcfg_test.go | 41 ++ .../v0_61_1/base/go/tools/builders/stdlib.go | 220 +++--- .../base/go/tools/builders/stdlib_test.go | 286 +++++++- .../base/go/tools/builders/env_orchestrion.go | 4 +- .../base/go/tools/builders/importcfg.go | 81 ++- .../base/go/tools/builders/importcfg_test.go | 41 ++ .../v0_62_0/base/go/tools/builders/stdlib.go | 220 +++--- .../base/go/tools/builders/stdlib_test.go | 286 +++++++- .../v0_60_0/base/0001-full-delta.patch | 661 +++++++++++++----- .../v0_61_1/base/0001-full-delta.patch | 661 +++++++++++++----- .../v0_62_0/base/0001-full-delta.patch | 661 +++++++++++++----- tools/dev/verify_rules_go_profiles.py | 205 +++++- .../python/test_rules_go_profile_tools.py | 52 ++ 20 files changed, 3063 insertions(+), 1073 deletions(-) diff --git a/third_party/rgo/v0_60_0/base/go/tools/builders/env_orchestrion.go b/third_party/rgo/v0_60_0/base/go/tools/builders/env_orchestrion.go index fbc392cb..73c7992c 100644 --- a/third_party/rgo/v0_60_0/base/go/tools/builders/env_orchestrion.go +++ b/third_party/rgo/v0_60_0/base/go/tools/builders/env_orchestrion.go @@ -59,7 +59,8 @@ func (e *env) runCommandWithJobserver(args []string, jobserver *orchestrionJobse } // newBufferedCommand creates a subprocess command wired to the shared builder -// buffer and applies the stdlib cache override needed by Orchestrion actions. +// buffer and advertises the declared stdlib archive source without replacing +// the subprocess's writable GOCACHE. func (e *env) newBufferedCommand(args []string, buf *bytes.Buffer) *exec.Cmd { cmd := exec.Command(args[0], args[1:]...) cmd.Stdout = buf @@ -67,7 +68,6 @@ func (e *env) newBufferedCommand(args []string, buf *bytes.Buffer) *exec.Cmd { cmd.Env = os.Environ() if e.stdlibCache != "" { if info, err := os.Stat(e.stdlibCache); err == nil && info.IsDir() { - cmd.Env = setEnv(cmd.Env, "GOCACHE", e.stdlibCache) cmd.Env = setEnv(cmd.Env, orchestrionStdlibCacheEnvVar, e.stdlibCache) } } diff --git a/third_party/rgo/v0_60_0/base/go/tools/builders/importcfg.go b/third_party/rgo/v0_60_0/base/go/tools/builders/importcfg.go index 3f0e2030..4c859ae4 100644 --- a/third_party/rgo/v0_60_0/base/go/tools/builders/importcfg.go +++ b/third_party/rgo/v0_60_0/base/go/tools/builders/importcfg.go @@ -1304,29 +1304,17 @@ func resolveInstrumentedStdlibExports(goenv *env, packages []string) (map[string baseEnv = setEnv(baseEnv, "GOROOT", goenv.goroot) baseEnv = setEnv(baseEnv, "GO111MODULE", "off") baseEnv = setEnv(baseEnv, "GOWORK", "off") - orchestrionCachePath := goenv.stdlibCache - if orchestrionCachePath == "" { - orchestrionCachePath = filepath.Join(goenv.goroot, ".gocache") - } - cachePath := "" - if info, err := os.Stat(orchestrionCachePath); err == nil && info.IsDir() { - cachePath = orchestrionCachePath - } - if cachePath == "" { - cachePath = getEnv(baseEnv, "GOCACHE") - } - if cachePath == "" { - if goenv.stdlibCache != "" { - cachePath = goenv.stdlibCache - } else { - cachePath = filepath.Join(goenv.goroot, ".gocache") - } - } - cachePath = abs(cachePath) - if err := os.MkdirAll(cachePath, 0o755); err != nil { + cachePath, err := writableStdlibCacheRoot(goenv, baseEnv) + if err != nil { return nil, fmt.Errorf("prepare stdlib export cache: %w", err) } + if err := seedWovenStdlibCache(goenv, cachePath); err != nil { + return nil, fmt.Errorf("seed stdlib export cache: %w", err) + } baseEnv = setEnv(baseEnv, "GOCACHE", cachePath) + if goenv.stdlibCache != "" { + baseEnv = setEnv(baseEnv, orchestrionStdlibCacheEnvVar, goenv.stdlibCache) + } if getEnv(baseEnv, "HOME") == "" { homePath := filepath.Join(goenv.goroot, ".home") if err := os.MkdirAll(homePath, 0o755); err != nil { @@ -1366,9 +1354,18 @@ func resolveInstrumentedStdlibExports(goenv *env, packages []string) (map[string } func resolveCacheStdlibExports(goenv *env, packages []string) (map[string]string, error) { - cacheRoot := "" - if goenv != nil { - cacheRoot = goenv.stdlibCache + if goenv != nil && goenv.stdlibCache != "" { + exports, err := readStdlibCacheManifest(goenv.stdlibCache, packages) + if err != nil { + return nil, err + } + if len(exports) > 0 { + return exports, nil + } + } + cacheRoot, err := writableStdlibCacheRoot(goenv, os.Environ()) + if err != nil { + return nil, err } return resolveCacheStdlibExportsAt(goenv, packages, cacheRoot) } @@ -1397,6 +1394,9 @@ func resolveCacheStdlibExportsAt(goenv *env, packages []string, cacheRoot string return manifestExports, nil } } + if goenv.stdlibCache != "" && sameFilePath(cacheRoot, goenv.stdlibCache) { + return nil, fmt.Errorf("refusing to use declared stdlib cache %s as writable GOCACHE", abs(goenv.stdlibCache)) + } goenv.goroot = abs(goenv.goroot) goenv.sdk = abs(goenv.sdk) if info, err := os.Stat(goenv.goroot); err != nil || !info.IsDir() { @@ -1480,6 +1480,24 @@ func cacheStdlibGoListBaseEnv(goenv *env, cachePath string, environ []string) [] return env } +func writableStdlibCacheRoot(goenv *env, environ []string) (string, error) { + cacheRoot := strings.TrimSpace(getEnv(environ, "GOCACHE")) + if cacheRoot == "" { + if goenv == nil || strings.TrimSpace(goenv.goroot) == "" { + return "", fmt.Errorf("writable stdlib GOCACHE is empty") + } + cacheRoot = filepath.Join(abs(goenv.goroot), ".gocache") + } + cacheRoot = abs(cacheRoot) + if goenv != nil && goenv.stdlibCache != "" && sameFilePath(cacheRoot, goenv.stdlibCache) { + return "", fmt.Errorf("writable stdlib GOCACHE %s aliases declared cache", cacheRoot) + } + if err := os.MkdirAll(cacheRoot, 0o755); err != nil { + return "", err + } + return cacheRoot, nil +} + func readStdlibCacheManifest(cacheRoot string, packages []string) (map[string]string, error) { if cacheRoot == "" { return nil, nil @@ -1874,18 +1892,17 @@ func resolveStdlibExportsForPackage(goenv *env, pkg string) (map[string]string, cmd.Env = setEnv(cmd.Env, "GOROOT", goenv.goroot) cmd.Env = setEnv(cmd.Env, "GO111MODULE", "on") cmd.Env = setEnv(cmd.Env, "GOWORK", "off") - cachePath := goenv.stdlibCache - if cachePath == "" { - cachePath = getEnv(cmd.Env, "GOCACHE") - } - if cachePath == "" { - cachePath = filepath.Join(goenv.goroot, ".gocache") - } - cachePath = abs(cachePath) - if err := os.MkdirAll(cachePath, 0o755); err != nil { + cachePath, err := writableStdlibCacheRoot(goenv, cmd.Env) + if err != nil { return nil, fmt.Errorf("prepare stdlib closure cache: %w", err) } + if err := seedWovenStdlibCache(goenv, cachePath); err != nil { + return nil, fmt.Errorf("seed stdlib closure cache: %w", err) + } cmd.Env = setEnv(cmd.Env, "GOCACHE", cachePath) + if goenv.stdlibCache != "" { + cmd.Env = setEnv(cmd.Env, orchestrionStdlibCacheEnvVar, goenv.stdlibCache) + } normalizedEnv, err := normalizeGoActionCacheEnv(cmd.Env) if err != nil { return nil, fmt.Errorf("prepare stdlib closure action cache env: %w", err) diff --git a/third_party/rgo/v0_60_0/base/go/tools/builders/importcfg_test.go b/third_party/rgo/v0_60_0/base/go/tools/builders/importcfg_test.go index 3724594d..35a1fa4a 100644 --- a/third_party/rgo/v0_60_0/base/go/tools/builders/importcfg_test.go +++ b/third_party/rgo/v0_60_0/base/go/tools/builders/importcfg_test.go @@ -563,6 +563,47 @@ func TestCacheStdlibGoListBaseEnvUsesSDKGoRoot(t *testing.T) { } } +func TestWritableStdlibCacheRootUsesPrivateCache(t *testing.T) { + privateCache := filepath.Join(t.TempDir(), "private") + declaredCache := filepath.Join(t.TempDir(), "declared") + cacheRoot, err := writableStdlibCacheRoot(&env{ + goroot: t.TempDir(), + stdlibCache: declaredCache, + }, []string{"GOCACHE=" + privateCache}) + if err != nil { + t.Fatalf("writableStdlibCacheRoot error: %v", err) + } + if cacheRoot != abs(privateCache) { + t.Fatalf("writable cache = %q, want %q", cacheRoot, abs(privateCache)) + } + if info, err := os.Stat(cacheRoot); err != nil || !info.IsDir() { + t.Fatalf("writable cache was not created: %v", err) + } +} + +func TestWritableStdlibCacheRootRejectsDeclaredCache(t *testing.T) { + declaredCache := filepath.Join(t.TempDir(), "declared") + _, err := writableStdlibCacheRoot(&env{ + goroot: t.TempDir(), + stdlibCache: declaredCache, + }, []string{"GOCACHE=" + declaredCache}) + if err == nil || !strings.Contains(err.Error(), "aliases declared cache") { + t.Fatalf("writableStdlibCacheRoot error = %v", err) + } +} + +func TestResolveCacheStdlibExportsRejectsDeclaredCacheAsWritable(t *testing.T) { + declaredCache := t.TempDir() + _, err := resolveCacheStdlibExportsAt(&env{ + goroot: "fake-goroot", + sdk: "fake-sdk", + stdlibCache: declaredCache, + }, []string{"fmt"}, declaredCache) + if err == nil || !strings.Contains(err.Error(), "refusing to use declared stdlib cache") { + t.Fatalf("resolveCacheStdlibExportsAt error = %v", err) + } +} + func TestModuleExportCacheManifestRoundTrip(t *testing.T) { root := t.TempDir() paths := orchestrionCachePaths(root, "module-exports", "abc123") diff --git a/third_party/rgo/v0_60_0/base/go/tools/builders/stdlib.go b/third_party/rgo/v0_60_0/base/go/tools/builders/stdlib.go index 08b29926..9b5c36c7 100644 --- a/third_party/rgo/v0_60_0/base/go/tools/builders/stdlib.go +++ b/third_party/rgo/v0_60_0/base/go/tools/builders/stdlib.go @@ -96,20 +96,15 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) os.Setenv("GOROOT", output) goenv.goroot = output - // Create a temporary cache directory. "go build" requires this starting - // in Go 1.12. - cachePath := filepath.Join(output, ".gocache") - if *cacheOut != "" { - cachePath = abs(*cacheOut) - goenv.stdlibCache = cachePath + // Live Go commands use an action-private scratch cache. The separately + // declared cache output is populated only with deterministic woven archives. + cachePath, declaredCache, cleanupCache, err := prepareStdlibCaches(output, *cacheOut) + if err != nil { + return err } + defer cleanupCache() + goenv.stdlibCache = declaredCache os.Setenv("GOCACHE", cachePath) - if err := os.MkdirAll(cachePath, 0o755); err != nil { - return fmt.Errorf("prepare stdlib gocache at %s: %w", cachePath, err) - } - if shouldRemoveStdlibCache(*orchestrion, *cacheOut) { - defer os.RemoveAll(cachePath) - } // Disable modules for the plain stdlib build command. When Orchestrion is // enabled we flip this back on later after preparing a synthetic module that @@ -356,12 +351,33 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) return nil } -// shouldRemoveStdlibCache reports whether the builder owns the cache directory -// as scratch space. A non-empty -cacheout value is a Bazel-declared TreeArtifact -// output, so it must remain present even for plain non-Orchestrion stdlib -// actions. -func shouldRemoveStdlibCache(orchestrionPath, cacheOut string) bool { - return strings.TrimSpace(orchestrionPath) == "" && strings.TrimSpace(cacheOut) == "" +func prepareStdlibCaches(output, cacheOut string) (scratchCache, declaredCache string, cleanup func(), err error) { + scratchCache = filepath.Join(abs(output), ".gocache") + cleanup = func() { + _ = os.RemoveAll(scratchCache) + } + if err := os.MkdirAll(scratchCache, 0o755); err != nil { + return "", "", func() {}, fmt.Errorf("prepare stdlib scratch gocache at %s: %w", scratchCache, err) + } + if strings.TrimSpace(cacheOut) == "" { + return scratchCache, "", cleanup, nil + } + + declaredCache = abs(cacheOut) + if sameFilePath(scratchCache, declaredCache) { + cleanup() + return "", "", func() {}, fmt.Errorf("stdlib scratch gocache aliases declared cache output at %s", declaredCache) + } + if err := os.MkdirAll(declaredCache, 0o755); err != nil { + cleanup() + return "", "", func() {}, fmt.Errorf("prepare declared stdlib cache output at %s: %w", declaredCache, err) + } + return scratchCache, declaredCache, cleanup, nil +} + +func sameFilePath(left, right string) bool { + rel, err := filepath.Rel(abs(left), abs(right)) + return err == nil && rel == "." } func mergeGoDebugSetting(existing, setting string) string { @@ -497,111 +513,101 @@ func syncPersistedOrchestrionExportsToCache(goenv *env, exports map[string]strin defer func() { span.End(err) }() - if goenv == nil || len(exports) == 0 { + if goenv == nil || len(exports) == 0 || len(roots) == 0 || strings.TrimSpace(goenv.stdlibCache) == "" { return nil } - // We have two cache families to keep consistent: - // 1. the Bazel-declared stdlib cache consumed by later compile/link actions - // 2. the shared Datadog/Orchestrion cache used by internal `go list -export` - // dependency resolution when woven deps are injected. - // - // If only one is populated, compile and link can observe different archive - // fingerprints for stdlib packages like log/fmt/flag. Populate both from the - // same woven persisted exports. - candidateCaches := []string{} - seenCaches := map[string]struct{}{} - addCache := func(path string) { - path = strings.TrimSpace(path) - if path == "" { - return - } - path = abs(path) - if _, ok := seenCaches[path]; ok { - return - } - seenCaches[path] = struct{}{} - candidateCaches = append(candidateCaches, path) + scratchCache := strings.TrimSpace(os.Getenv("GOCACHE")) + if scratchCache == "" { + return fmt.Errorf("publish stdlib cache exports: GOCACHE scratch path is empty") } - - addCache(goenv.stdlibCache) - if envCache := strings.TrimSpace(os.Getenv("GOCACHE")); envCache != "" { - addCache(envCache) + scratchCache = abs(scratchCache) + declaredCache := abs(goenv.stdlibCache) + if sameFilePath(scratchCache, declaredCache) { + return fmt.Errorf("publish stdlib cache exports: scratch cache %s aliases declared cache %s", scratchCache, declaredCache) } - if len(candidateCaches) == 0 { - addCache(filepath.Join(abs(goenv.goroot), ".gocache")) + if err := os.MkdirAll(declaredCache, 0o755); err != nil { + return fmt.Errorf("prepare declared stdlib cache exports at %s: %w", declaredCache, err) } + if verbose { + fmt.Fprintf(os.Stderr, "stdlib: resolving cache exports against scratch GOCACHE=%s\n", scratchCache) + } + cacheExports, err := resolveCacheStdlibExportsAt(goenv, roots, scratchCache) + if err != nil { + return err + } + return publishPersistedOrchestrionExportsToCache(exports, cacheExports, scratchCache, declaredCache, verbose) +} - prevCachePath, hadPrevCachePath := os.LookupEnv("GOCACHE") - defer func() { - if hadPrevCachePath { - _ = os.Setenv("GOCACHE", prevCachePath) - } else { - _ = os.Unsetenv("GOCACHE") - } - }() - - for _, cachePath := range candidateCaches { - if err := os.MkdirAll(cachePath, 0o755); err != nil { - return fmt.Errorf("prepare stdlib cache exports at %s: %w", cachePath, err) - } - if err := os.Setenv("GOCACHE", cachePath); err != nil { - return fmt.Errorf("set stdlib cache exports path %s: %w", cachePath, err) - } - if verbose { - fmt.Fprintf(os.Stderr, "stdlib: resolving cache-family exports against GOCACHE=%s\n", cachePath) +func publishPersistedOrchestrionExportsToCache(exports, cacheExports map[string]string, scratchCache, declaredCache string, verbose bool) error { + manifestPath := filepath.Join(abs(declaredCache), orchestrionStdlibCacheManifestName) + if err := os.Remove(manifestPath); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("remove stale stdlib cache manifest at %s: %w", manifestPath, err) + } + packages := make([]string, 0, len(cacheExports)) + for pkg := range cacheExports { + packages = append(packages, pkg) + } + sort.Strings(packages) + var manifest strings.Builder + for _, pkg := range packages { + src := strings.TrimSpace(exports[pkg]) + if src == "" { + return fmt.Errorf("missing persisted stdlib archive for cache package %s", pkg) } - - cacheExports, err := resolveCacheStdlibExportsAt(goenv, roots, cachePath) + dst, relDst, err := projectStdlibCacheArchive(scratchCache, declaredCache, cacheExports[pkg]) if err != nil { + return fmt.Errorf("project stdlib cache archive for %s: %w", pkg, err) + } + if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { return err } - packages := make([]string, 0, len(cacheExports)) - for pkg := range cacheExports { - packages = append(packages, pkg) - } - sort.Strings(packages) - var manifest strings.Builder - for _, pkg := range packages { - src := exports[pkg] - dst, ok := cacheExports[pkg] - if !ok || dst == "" { - continue - } - if strings.TrimSpace(src) == "" { - return fmt.Errorf("missing persisted stdlib archive for cache package %s", pkg) - } - if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { - return err - } - if err := copyArchiveFile(src, dst); err != nil { - return fmt.Errorf("copy persisted stdlib archive %s -> cache %s: %w", src, dst, err) - } - if verbose { - fmt.Fprintf(os.Stderr, "stdlib: synced persisted orchestrion export %s -> cache %s\n", src, dst) - } - relDst := dst - if rel, err := filepath.Rel(cachePath, dst); err == nil { - relDst = rel - } - manifest.WriteString(pkg) - manifest.WriteString("=") - manifest.WriteString(relDst) - manifest.WriteString("\n") - } - if manifest.Len() > 0 { - manifestPath := filepath.Join(cachePath, orchestrionStdlibCacheManifestName) - if err := os.WriteFile(manifestPath, []byte(manifest.String()), 0o644); err != nil { - return fmt.Errorf("write stdlib cache manifest at %s: %w", manifestPath, err) - } - if verbose { - fmt.Fprintf(os.Stderr, "stdlib: wrote stdlib cache manifest %s\n", manifestPath) - } + if err := copyArchiveFile(src, dst); err != nil { + return fmt.Errorf("copy persisted stdlib archive %s -> declared cache %s: %w", src, dst, err) + } + manifest.WriteString(pkg) + manifest.WriteString("=") + manifest.WriteString(relDst) + manifest.WriteString("\n") + if verbose { + fmt.Fprintf(os.Stderr, "stdlib: published persisted orchestrion export %s -> declared cache %s\n", src, dst) } } + if manifest.Len() == 0 { + return nil + } + if err := writeFileAtomically(manifestPath, []byte(manifest.String()), 0o644); err != nil { + return fmt.Errorf("write stdlib cache manifest at %s: %w", manifestPath, err) + } + if verbose { + fmt.Fprintf(os.Stderr, "stdlib: wrote stdlib cache manifest %s\n", manifestPath) + } return nil } +func projectStdlibCacheArchive(scratchCache, declaredCache, scratchArchive string) (destination, relative string, err error) { + scratchRoot := abs(scratchCache) + declaredRoot := abs(declaredCache) + archive := abs(scratchArchive) + rel, err := filepath.Rel(scratchRoot, archive) + if err != nil { + return "", "", err + } + rel = filepath.Clean(rel) + if rel == "." || filepath.IsAbs(rel) || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return "", "", fmt.Errorf("cache archive %s escapes scratch root %s", archive, scratchRoot) + } + if !strings.HasSuffix(filepath.Base(rel), "-d") { + return "", "", fmt.Errorf("cache archive %s is not a Go cache data entry", archive) + } + destination = filepath.Join(declaredRoot, rel) + destinationRel, err := filepath.Rel(declaredRoot, destination) + if err != nil || filepath.Clean(destinationRel) != rel { + return "", "", fmt.Errorf("cache archive destination %s escapes declared root %s", destination, declaredRoot) + } + return destination, filepath.ToSlash(rel), nil +} + func ensureSyntheticOrchestrionToolGo(verbose bool, orchestrionMode string) (func(), error) { const toolFile = "orchestrion.tool.go" if _, err := os.Stat(toolFile); err == nil { diff --git a/third_party/rgo/v0_60_0/base/go/tools/builders/stdlib_test.go b/third_party/rgo/v0_60_0/base/go/tools/builders/stdlib_test.go index b92f214b..7ca400cd 100644 --- a/third_party/rgo/v0_60_0/base/go/tools/builders/stdlib_test.go +++ b/third_party/rgo/v0_60_0/base/go/tools/builders/stdlib_test.go @@ -1,8 +1,12 @@ package main import ( + "bytes" + "crypto/sha256" + "fmt" "os" "path/filepath" + "reflect" "strings" "testing" ) @@ -116,48 +120,223 @@ func TestEnsureImportableStdlibModulePathRewritesAndRestores(t *testing.T) { } } -func TestShouldRemoveStdlibCache(t *testing.T) { - t.Parallel() +func TestPrepareStdlibCachesWithoutDeclaredOutput(t *testing.T) { + output := filepath.Join(t.TempDir(), "stdlib") + scratch, declared, cleanup, err := prepareStdlibCaches(output, "") + if err != nil { + t.Fatalf("prepareStdlibCaches error: %v", err) + } + if want := filepath.Join(abs(output), ".gocache"); scratch != want { + t.Fatalf("scratch cache = %q, want %q", scratch, want) + } + if declared != "" { + t.Fatalf("declared cache = %q, want empty", declared) + } + if info, err := os.Stat(scratch); err != nil || !info.IsDir() { + t.Fatalf("scratch cache was not created: %v", err) + } + cleanup() + if _, err := os.Stat(scratch); !os.IsNotExist(err) { + t.Fatalf("scratch cache still exists after cleanup: %v", err) + } +} + +func TestPrepareStdlibCachesSeparatesDeclaredOutput(t *testing.T) { + root := t.TempDir() + output := filepath.Join(root, "stdlib") + cacheOut := filepath.Join(root, "declared") + scratch, declared, cleanup, err := prepareStdlibCaches(output, cacheOut) + if err != nil { + t.Fatalf("prepareStdlibCaches error: %v", err) + } + if sameFilePath(scratch, declared) { + t.Fatalf("scratch cache %q aliases declared cache %q", scratch, declared) + } + for name, path := range map[string]string{"scratch": scratch, "declared": declared} { + if info, err := os.Stat(path); err != nil || !info.IsDir() { + t.Fatalf("%s cache was not created: %v", name, err) + } + } + cleanup() + if _, err := os.Stat(scratch); !os.IsNotExist(err) { + t.Fatalf("scratch cache still exists after cleanup: %v", err) + } + if info, err := os.Stat(declared); err != nil || !info.IsDir() { + t.Fatalf("declared cache should remain after scratch cleanup: %v", err) + } +} + +func TestPrepareStdlibCachesNormalizesRelativeInputs(t *testing.T) { + root := t.TempDir() + previousWD, err := os.Getwd() + if err != nil { + t.Fatalf("getwd: %v", err) + } + if err := os.Chdir(root); err != nil { + t.Fatalf("chdir: %v", err) + } + defer func() { _ = os.Chdir(previousWD) }() + normalizedRoot, err := os.Getwd() + if err != nil { + t.Fatalf("get normalized workdir: %v", err) + } + + scratch, declared, cleanup, err := prepareStdlibCaches("stdlib", "declared") + if err != nil { + t.Fatalf("prepareStdlibCaches error: %v", err) + } + defer cleanup() + if !filepath.IsAbs(scratch) || !filepath.IsAbs(declared) { + t.Fatalf("cache paths are not absolute: scratch=%q declared=%q", scratch, declared) + } + if scratch != filepath.Join(normalizedRoot, "stdlib", ".gocache") { + t.Fatalf("scratch cache = %q", scratch) + } + if declared != filepath.Join(normalizedRoot, "declared") { + t.Fatalf("declared cache = %q", declared) + } +} + +func TestPrepareStdlibCachesRejectsAliasedOutput(t *testing.T) { + output := filepath.Join(t.TempDir(), "stdlib") + alias := filepath.Join(output, ".gocache") + if _, _, _, err := prepareStdlibCaches(output, alias); err == nil || !strings.Contains(err.Error(), "aliases declared cache") { + t.Fatalf("prepareStdlibCaches alias error = %v", err) + } + if _, err := os.Stat(alias); !os.IsNotExist(err) { + t.Fatalf("aliased scratch cache still exists after failure: %v", err) + } +} + +func TestNewBufferedCommandPreservesWritableGoCache(t *testing.T) { + scratch := filepath.Join(t.TempDir(), "scratch") + declared := filepath.Join(t.TempDir(), "declared") + if err := os.MkdirAll(declared, 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("GOCACHE", scratch) + + cmd := (&env{stdlibCache: declared}).newBufferedCommand([]string{"go", "version"}, &bytes.Buffer{}) + if got := getEnv(cmd.Env, "GOCACHE"); got != scratch { + t.Fatalf("GOCACHE = %q, want writable cache %q", got, scratch) + } + if got := getEnv(cmd.Env, orchestrionStdlibCacheEnvVar); got != declared { + t.Fatalf("%s = %q, want %q", orchestrionStdlibCacheEnvVar, got, declared) + } + + for _, invalid := range []string{"", filepath.Join(t.TempDir(), "missing")} { + cmd = (&env{stdlibCache: invalid}).newBufferedCommand([]string{"go", "version"}, &bytes.Buffer{}) + if got := getEnv(cmd.Env, "GOCACHE"); got != scratch { + t.Fatalf("invalid declared cache %q replaced GOCACHE with %q", invalid, got) + } + if got := getEnv(cmd.Env, orchestrionStdlibCacheEnvVar); got != "" { + t.Fatalf("invalid declared cache %q set %s=%q", invalid, orchestrionStdlibCacheEnvVar, got) + } + } +} +func TestPublishPersistedOrchestrionExportsIsDeterministic(t *testing.T) { + archives := map[string][]byte{ + "fmt": []byte("woven fmt archive"), + "log": []byte("woven log archive"), + } + cachePaths := map[string]string{ + "fmt": filepath.Join("11", "fmt-d"), + "log": filepath.Join("aa", "log-d"), + } + + inventories := make([]map[string]string, 0, 2) + for run := 0; run < 2; run++ { + scratch := filepath.Join(t.TempDir(), "scratch") + declared := filepath.Join(t.TempDir(), "declared") + persisted := filepath.Join(t.TempDir(), "persisted") + if err := os.MkdirAll(scratch, 0o755); err != nil { + t.Fatal(err) + } + exports := make(map[string]string, len(archives)) + resolved := make(map[string]string, len(cachePaths)) + for pkg, data := range archives { + src := filepath.Join(persisted, pkg+".a") + writeTestFile(t, src, data) + exports[pkg] = src + resolved[pkg] = filepath.Join(scratch, cachePaths[pkg]) + writeTestFile(t, resolved[pkg], []byte(fmt.Sprintf("unwoven run %d", run))) + } + writeTestFile(t, filepath.Join(scratch, "11", "fmt-a"), []byte(fmt.Sprintf("timestamp %d", run))) + writeTestFile(t, filepath.Join(scratch, "trim.txt"), []byte(fmt.Sprintf("trim %d", run))) + writeTestFile(t, filepath.Join(scratch, "unrelated", "entry-d"), []byte(fmt.Sprintf("unrelated %d", run))) + + if err := publishPersistedOrchestrionExportsToCache(exports, resolved, scratch, declared, false); err != nil { + t.Fatalf("publish run %d: %v", run, err) + } + inventory := testTreeInventory(t, declared) + inventories = append(inventories, inventory) + if len(inventory) != len(archives)+1 { + t.Fatalf("declared inventory has unexpected entries: %v", inventory) + } + for pkg, data := range archives { + path := filepath.ToSlash(cachePaths[pkg]) + if got := inventory[path]; got != testDigest(data) { + t.Fatalf("published %s digest = %q, want %q", path, got, testDigest(data)) + } + } + manifest, err := os.ReadFile(filepath.Join(declared, orchestrionStdlibCacheManifestName)) + if err != nil { + t.Fatal(err) + } + wantManifest := "fmt=11/fmt-d\nlog=aa/log-d\n" + if string(manifest) != wantManifest { + t.Fatalf("manifest = %q, want %q", string(manifest), wantManifest) + } + } + if !reflect.DeepEqual(inventories[0], inventories[1]) { + t.Fatalf("declared cache inventories differ:\nrun 1: %v\nrun 2: %v", inventories[0], inventories[1]) + } +} + +func TestProjectStdlibCacheArchiveRejectsInvalidPaths(t *testing.T) { + root := t.TempDir() + scratch := filepath.Join(root, "scratch") + declared := filepath.Join(root, "declared") tests := []struct { - name string - orchestrionPath string - cacheOut string - want bool + name string + archive string + want string }{ - { - name: "plain internal cache", - want: true, - }, - { - name: "plain declared cache output", - cacheOut: "bazel-out/bin/external/rules_go/stdlib_/gocache", - want: false, - }, - { - name: "orchestrion internal cache", - orchestrionPath: "external/rules_go_orchestrion_tool/orchestrion", - want: false, - }, - { - name: "orchestrion declared cache output", - orchestrionPath: "external/rules_go_orchestrion_tool/orchestrion", - cacheOut: "bazel-out/bin/external/rules_go/stdlib_/gocache", - want: false, - }, + {name: "outside scratch", archive: filepath.Join(root, "outside", "entry-d"), want: "escapes scratch root"}, + {name: "action index", archive: filepath.Join(scratch, "aa", "entry-a"), want: "not a Go cache data entry"}, } - for _, tt := range tests { - tt := tt t.Run(tt.name, func(t *testing.T) { - t.Parallel() - if got := shouldRemoveStdlibCache(tt.orchestrionPath, tt.cacheOut); got != tt.want { - t.Fatalf("shouldRemoveStdlibCache(%q, %q) = %v, want %v", tt.orchestrionPath, tt.cacheOut, got, tt.want) + if _, _, err := projectStdlibCacheArchive(scratch, declared, tt.archive); err == nil || !strings.Contains(err.Error(), tt.want) { + t.Fatalf("projectStdlibCacheArchive error = %v, want %q", err, tt.want) } }) } } +func TestPublishPersistedOrchestrionExportsFailureLeavesNoManifest(t *testing.T) { + root := t.TempDir() + scratch := filepath.Join(root, "scratch") + declared := filepath.Join(root, "declared") + manifest := filepath.Join(declared, orchestrionStdlibCacheManifestName) + writeTestFile(t, manifest, []byte("stale=stale-d\n")) + + err := publishPersistedOrchestrionExportsToCache( + map[string]string{}, + map[string]string{"fmt": filepath.Join(scratch, "aa", "fmt-d")}, + scratch, + declared, + false, + ) + if err == nil || !strings.Contains(err.Error(), "missing persisted stdlib archive for cache package fmt") { + t.Fatalf("publish error = %v", err) + } + if _, err := os.Stat(manifest); !os.IsNotExist(err) { + t.Fatalf("manifest exists after failed publication: %v", err) + } +} + func TestMergeGoDebugSettingPreservesExistingFlags(t *testing.T) { tests := []struct { name string @@ -238,3 +417,48 @@ func TestPersistOrchestrionStdlibExportsUsesGoToolInstallDir(t *testing.T) { } } } + +func writeTestFile(t *testing.T, path string, data []byte) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("mkdir %s: %v", filepath.Dir(path), err) + } + if err := os.WriteFile(path, data, 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } +} + +func testTreeInventory(t *testing.T, root string) map[string]string { + t.Helper() + inventory := make(map[string]string) + err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if path == root || entry.IsDir() { + return nil + } + if entry.Type()&os.ModeSymlink != 0 { + return fmt.Errorf("unexpected symlink %s", path) + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + rel, err := filepath.Rel(root, path) + if err != nil { + return err + } + inventory[filepath.ToSlash(rel)] = testDigest(data) + return nil + }) + if err != nil { + t.Fatalf("inventory %s: %v", root, err) + } + return inventory +} + +func testDigest(data []byte) string { + sum := sha256.Sum256(data) + return fmt.Sprintf("%x", sum[:]) +} diff --git a/third_party/rgo/v0_61_1/base/go/tools/builders/env_orchestrion.go b/third_party/rgo/v0_61_1/base/go/tools/builders/env_orchestrion.go index fbc392cb..73c7992c 100644 --- a/third_party/rgo/v0_61_1/base/go/tools/builders/env_orchestrion.go +++ b/third_party/rgo/v0_61_1/base/go/tools/builders/env_orchestrion.go @@ -59,7 +59,8 @@ func (e *env) runCommandWithJobserver(args []string, jobserver *orchestrionJobse } // newBufferedCommand creates a subprocess command wired to the shared builder -// buffer and applies the stdlib cache override needed by Orchestrion actions. +// buffer and advertises the declared stdlib archive source without replacing +// the subprocess's writable GOCACHE. func (e *env) newBufferedCommand(args []string, buf *bytes.Buffer) *exec.Cmd { cmd := exec.Command(args[0], args[1:]...) cmd.Stdout = buf @@ -67,7 +68,6 @@ func (e *env) newBufferedCommand(args []string, buf *bytes.Buffer) *exec.Cmd { cmd.Env = os.Environ() if e.stdlibCache != "" { if info, err := os.Stat(e.stdlibCache); err == nil && info.IsDir() { - cmd.Env = setEnv(cmd.Env, "GOCACHE", e.stdlibCache) cmd.Env = setEnv(cmd.Env, orchestrionStdlibCacheEnvVar, e.stdlibCache) } } diff --git a/third_party/rgo/v0_61_1/base/go/tools/builders/importcfg.go b/third_party/rgo/v0_61_1/base/go/tools/builders/importcfg.go index 3f0e2030..4c859ae4 100644 --- a/third_party/rgo/v0_61_1/base/go/tools/builders/importcfg.go +++ b/third_party/rgo/v0_61_1/base/go/tools/builders/importcfg.go @@ -1304,29 +1304,17 @@ func resolveInstrumentedStdlibExports(goenv *env, packages []string) (map[string baseEnv = setEnv(baseEnv, "GOROOT", goenv.goroot) baseEnv = setEnv(baseEnv, "GO111MODULE", "off") baseEnv = setEnv(baseEnv, "GOWORK", "off") - orchestrionCachePath := goenv.stdlibCache - if orchestrionCachePath == "" { - orchestrionCachePath = filepath.Join(goenv.goroot, ".gocache") - } - cachePath := "" - if info, err := os.Stat(orchestrionCachePath); err == nil && info.IsDir() { - cachePath = orchestrionCachePath - } - if cachePath == "" { - cachePath = getEnv(baseEnv, "GOCACHE") - } - if cachePath == "" { - if goenv.stdlibCache != "" { - cachePath = goenv.stdlibCache - } else { - cachePath = filepath.Join(goenv.goroot, ".gocache") - } - } - cachePath = abs(cachePath) - if err := os.MkdirAll(cachePath, 0o755); err != nil { + cachePath, err := writableStdlibCacheRoot(goenv, baseEnv) + if err != nil { return nil, fmt.Errorf("prepare stdlib export cache: %w", err) } + if err := seedWovenStdlibCache(goenv, cachePath); err != nil { + return nil, fmt.Errorf("seed stdlib export cache: %w", err) + } baseEnv = setEnv(baseEnv, "GOCACHE", cachePath) + if goenv.stdlibCache != "" { + baseEnv = setEnv(baseEnv, orchestrionStdlibCacheEnvVar, goenv.stdlibCache) + } if getEnv(baseEnv, "HOME") == "" { homePath := filepath.Join(goenv.goroot, ".home") if err := os.MkdirAll(homePath, 0o755); err != nil { @@ -1366,9 +1354,18 @@ func resolveInstrumentedStdlibExports(goenv *env, packages []string) (map[string } func resolveCacheStdlibExports(goenv *env, packages []string) (map[string]string, error) { - cacheRoot := "" - if goenv != nil { - cacheRoot = goenv.stdlibCache + if goenv != nil && goenv.stdlibCache != "" { + exports, err := readStdlibCacheManifest(goenv.stdlibCache, packages) + if err != nil { + return nil, err + } + if len(exports) > 0 { + return exports, nil + } + } + cacheRoot, err := writableStdlibCacheRoot(goenv, os.Environ()) + if err != nil { + return nil, err } return resolveCacheStdlibExportsAt(goenv, packages, cacheRoot) } @@ -1397,6 +1394,9 @@ func resolveCacheStdlibExportsAt(goenv *env, packages []string, cacheRoot string return manifestExports, nil } } + if goenv.stdlibCache != "" && sameFilePath(cacheRoot, goenv.stdlibCache) { + return nil, fmt.Errorf("refusing to use declared stdlib cache %s as writable GOCACHE", abs(goenv.stdlibCache)) + } goenv.goroot = abs(goenv.goroot) goenv.sdk = abs(goenv.sdk) if info, err := os.Stat(goenv.goroot); err != nil || !info.IsDir() { @@ -1480,6 +1480,24 @@ func cacheStdlibGoListBaseEnv(goenv *env, cachePath string, environ []string) [] return env } +func writableStdlibCacheRoot(goenv *env, environ []string) (string, error) { + cacheRoot := strings.TrimSpace(getEnv(environ, "GOCACHE")) + if cacheRoot == "" { + if goenv == nil || strings.TrimSpace(goenv.goroot) == "" { + return "", fmt.Errorf("writable stdlib GOCACHE is empty") + } + cacheRoot = filepath.Join(abs(goenv.goroot), ".gocache") + } + cacheRoot = abs(cacheRoot) + if goenv != nil && goenv.stdlibCache != "" && sameFilePath(cacheRoot, goenv.stdlibCache) { + return "", fmt.Errorf("writable stdlib GOCACHE %s aliases declared cache", cacheRoot) + } + if err := os.MkdirAll(cacheRoot, 0o755); err != nil { + return "", err + } + return cacheRoot, nil +} + func readStdlibCacheManifest(cacheRoot string, packages []string) (map[string]string, error) { if cacheRoot == "" { return nil, nil @@ -1874,18 +1892,17 @@ func resolveStdlibExportsForPackage(goenv *env, pkg string) (map[string]string, cmd.Env = setEnv(cmd.Env, "GOROOT", goenv.goroot) cmd.Env = setEnv(cmd.Env, "GO111MODULE", "on") cmd.Env = setEnv(cmd.Env, "GOWORK", "off") - cachePath := goenv.stdlibCache - if cachePath == "" { - cachePath = getEnv(cmd.Env, "GOCACHE") - } - if cachePath == "" { - cachePath = filepath.Join(goenv.goroot, ".gocache") - } - cachePath = abs(cachePath) - if err := os.MkdirAll(cachePath, 0o755); err != nil { + cachePath, err := writableStdlibCacheRoot(goenv, cmd.Env) + if err != nil { return nil, fmt.Errorf("prepare stdlib closure cache: %w", err) } + if err := seedWovenStdlibCache(goenv, cachePath); err != nil { + return nil, fmt.Errorf("seed stdlib closure cache: %w", err) + } cmd.Env = setEnv(cmd.Env, "GOCACHE", cachePath) + if goenv.stdlibCache != "" { + cmd.Env = setEnv(cmd.Env, orchestrionStdlibCacheEnvVar, goenv.stdlibCache) + } normalizedEnv, err := normalizeGoActionCacheEnv(cmd.Env) if err != nil { return nil, fmt.Errorf("prepare stdlib closure action cache env: %w", err) diff --git a/third_party/rgo/v0_61_1/base/go/tools/builders/importcfg_test.go b/third_party/rgo/v0_61_1/base/go/tools/builders/importcfg_test.go index 3724594d..35a1fa4a 100644 --- a/third_party/rgo/v0_61_1/base/go/tools/builders/importcfg_test.go +++ b/third_party/rgo/v0_61_1/base/go/tools/builders/importcfg_test.go @@ -563,6 +563,47 @@ func TestCacheStdlibGoListBaseEnvUsesSDKGoRoot(t *testing.T) { } } +func TestWritableStdlibCacheRootUsesPrivateCache(t *testing.T) { + privateCache := filepath.Join(t.TempDir(), "private") + declaredCache := filepath.Join(t.TempDir(), "declared") + cacheRoot, err := writableStdlibCacheRoot(&env{ + goroot: t.TempDir(), + stdlibCache: declaredCache, + }, []string{"GOCACHE=" + privateCache}) + if err != nil { + t.Fatalf("writableStdlibCacheRoot error: %v", err) + } + if cacheRoot != abs(privateCache) { + t.Fatalf("writable cache = %q, want %q", cacheRoot, abs(privateCache)) + } + if info, err := os.Stat(cacheRoot); err != nil || !info.IsDir() { + t.Fatalf("writable cache was not created: %v", err) + } +} + +func TestWritableStdlibCacheRootRejectsDeclaredCache(t *testing.T) { + declaredCache := filepath.Join(t.TempDir(), "declared") + _, err := writableStdlibCacheRoot(&env{ + goroot: t.TempDir(), + stdlibCache: declaredCache, + }, []string{"GOCACHE=" + declaredCache}) + if err == nil || !strings.Contains(err.Error(), "aliases declared cache") { + t.Fatalf("writableStdlibCacheRoot error = %v", err) + } +} + +func TestResolveCacheStdlibExportsRejectsDeclaredCacheAsWritable(t *testing.T) { + declaredCache := t.TempDir() + _, err := resolveCacheStdlibExportsAt(&env{ + goroot: "fake-goroot", + sdk: "fake-sdk", + stdlibCache: declaredCache, + }, []string{"fmt"}, declaredCache) + if err == nil || !strings.Contains(err.Error(), "refusing to use declared stdlib cache") { + t.Fatalf("resolveCacheStdlibExportsAt error = %v", err) + } +} + func TestModuleExportCacheManifestRoundTrip(t *testing.T) { root := t.TempDir() paths := orchestrionCachePaths(root, "module-exports", "abc123") diff --git a/third_party/rgo/v0_61_1/base/go/tools/builders/stdlib.go b/third_party/rgo/v0_61_1/base/go/tools/builders/stdlib.go index 08b29926..9b5c36c7 100644 --- a/third_party/rgo/v0_61_1/base/go/tools/builders/stdlib.go +++ b/third_party/rgo/v0_61_1/base/go/tools/builders/stdlib.go @@ -96,20 +96,15 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) os.Setenv("GOROOT", output) goenv.goroot = output - // Create a temporary cache directory. "go build" requires this starting - // in Go 1.12. - cachePath := filepath.Join(output, ".gocache") - if *cacheOut != "" { - cachePath = abs(*cacheOut) - goenv.stdlibCache = cachePath + // Live Go commands use an action-private scratch cache. The separately + // declared cache output is populated only with deterministic woven archives. + cachePath, declaredCache, cleanupCache, err := prepareStdlibCaches(output, *cacheOut) + if err != nil { + return err } + defer cleanupCache() + goenv.stdlibCache = declaredCache os.Setenv("GOCACHE", cachePath) - if err := os.MkdirAll(cachePath, 0o755); err != nil { - return fmt.Errorf("prepare stdlib gocache at %s: %w", cachePath, err) - } - if shouldRemoveStdlibCache(*orchestrion, *cacheOut) { - defer os.RemoveAll(cachePath) - } // Disable modules for the plain stdlib build command. When Orchestrion is // enabled we flip this back on later after preparing a synthetic module that @@ -356,12 +351,33 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) return nil } -// shouldRemoveStdlibCache reports whether the builder owns the cache directory -// as scratch space. A non-empty -cacheout value is a Bazel-declared TreeArtifact -// output, so it must remain present even for plain non-Orchestrion stdlib -// actions. -func shouldRemoveStdlibCache(orchestrionPath, cacheOut string) bool { - return strings.TrimSpace(orchestrionPath) == "" && strings.TrimSpace(cacheOut) == "" +func prepareStdlibCaches(output, cacheOut string) (scratchCache, declaredCache string, cleanup func(), err error) { + scratchCache = filepath.Join(abs(output), ".gocache") + cleanup = func() { + _ = os.RemoveAll(scratchCache) + } + if err := os.MkdirAll(scratchCache, 0o755); err != nil { + return "", "", func() {}, fmt.Errorf("prepare stdlib scratch gocache at %s: %w", scratchCache, err) + } + if strings.TrimSpace(cacheOut) == "" { + return scratchCache, "", cleanup, nil + } + + declaredCache = abs(cacheOut) + if sameFilePath(scratchCache, declaredCache) { + cleanup() + return "", "", func() {}, fmt.Errorf("stdlib scratch gocache aliases declared cache output at %s", declaredCache) + } + if err := os.MkdirAll(declaredCache, 0o755); err != nil { + cleanup() + return "", "", func() {}, fmt.Errorf("prepare declared stdlib cache output at %s: %w", declaredCache, err) + } + return scratchCache, declaredCache, cleanup, nil +} + +func sameFilePath(left, right string) bool { + rel, err := filepath.Rel(abs(left), abs(right)) + return err == nil && rel == "." } func mergeGoDebugSetting(existing, setting string) string { @@ -497,111 +513,101 @@ func syncPersistedOrchestrionExportsToCache(goenv *env, exports map[string]strin defer func() { span.End(err) }() - if goenv == nil || len(exports) == 0 { + if goenv == nil || len(exports) == 0 || len(roots) == 0 || strings.TrimSpace(goenv.stdlibCache) == "" { return nil } - // We have two cache families to keep consistent: - // 1. the Bazel-declared stdlib cache consumed by later compile/link actions - // 2. the shared Datadog/Orchestrion cache used by internal `go list -export` - // dependency resolution when woven deps are injected. - // - // If only one is populated, compile and link can observe different archive - // fingerprints for stdlib packages like log/fmt/flag. Populate both from the - // same woven persisted exports. - candidateCaches := []string{} - seenCaches := map[string]struct{}{} - addCache := func(path string) { - path = strings.TrimSpace(path) - if path == "" { - return - } - path = abs(path) - if _, ok := seenCaches[path]; ok { - return - } - seenCaches[path] = struct{}{} - candidateCaches = append(candidateCaches, path) + scratchCache := strings.TrimSpace(os.Getenv("GOCACHE")) + if scratchCache == "" { + return fmt.Errorf("publish stdlib cache exports: GOCACHE scratch path is empty") } - - addCache(goenv.stdlibCache) - if envCache := strings.TrimSpace(os.Getenv("GOCACHE")); envCache != "" { - addCache(envCache) + scratchCache = abs(scratchCache) + declaredCache := abs(goenv.stdlibCache) + if sameFilePath(scratchCache, declaredCache) { + return fmt.Errorf("publish stdlib cache exports: scratch cache %s aliases declared cache %s", scratchCache, declaredCache) } - if len(candidateCaches) == 0 { - addCache(filepath.Join(abs(goenv.goroot), ".gocache")) + if err := os.MkdirAll(declaredCache, 0o755); err != nil { + return fmt.Errorf("prepare declared stdlib cache exports at %s: %w", declaredCache, err) } + if verbose { + fmt.Fprintf(os.Stderr, "stdlib: resolving cache exports against scratch GOCACHE=%s\n", scratchCache) + } + cacheExports, err := resolveCacheStdlibExportsAt(goenv, roots, scratchCache) + if err != nil { + return err + } + return publishPersistedOrchestrionExportsToCache(exports, cacheExports, scratchCache, declaredCache, verbose) +} - prevCachePath, hadPrevCachePath := os.LookupEnv("GOCACHE") - defer func() { - if hadPrevCachePath { - _ = os.Setenv("GOCACHE", prevCachePath) - } else { - _ = os.Unsetenv("GOCACHE") - } - }() - - for _, cachePath := range candidateCaches { - if err := os.MkdirAll(cachePath, 0o755); err != nil { - return fmt.Errorf("prepare stdlib cache exports at %s: %w", cachePath, err) - } - if err := os.Setenv("GOCACHE", cachePath); err != nil { - return fmt.Errorf("set stdlib cache exports path %s: %w", cachePath, err) - } - if verbose { - fmt.Fprintf(os.Stderr, "stdlib: resolving cache-family exports against GOCACHE=%s\n", cachePath) +func publishPersistedOrchestrionExportsToCache(exports, cacheExports map[string]string, scratchCache, declaredCache string, verbose bool) error { + manifestPath := filepath.Join(abs(declaredCache), orchestrionStdlibCacheManifestName) + if err := os.Remove(manifestPath); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("remove stale stdlib cache manifest at %s: %w", manifestPath, err) + } + packages := make([]string, 0, len(cacheExports)) + for pkg := range cacheExports { + packages = append(packages, pkg) + } + sort.Strings(packages) + var manifest strings.Builder + for _, pkg := range packages { + src := strings.TrimSpace(exports[pkg]) + if src == "" { + return fmt.Errorf("missing persisted stdlib archive for cache package %s", pkg) } - - cacheExports, err := resolveCacheStdlibExportsAt(goenv, roots, cachePath) + dst, relDst, err := projectStdlibCacheArchive(scratchCache, declaredCache, cacheExports[pkg]) if err != nil { + return fmt.Errorf("project stdlib cache archive for %s: %w", pkg, err) + } + if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { return err } - packages := make([]string, 0, len(cacheExports)) - for pkg := range cacheExports { - packages = append(packages, pkg) - } - sort.Strings(packages) - var manifest strings.Builder - for _, pkg := range packages { - src := exports[pkg] - dst, ok := cacheExports[pkg] - if !ok || dst == "" { - continue - } - if strings.TrimSpace(src) == "" { - return fmt.Errorf("missing persisted stdlib archive for cache package %s", pkg) - } - if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { - return err - } - if err := copyArchiveFile(src, dst); err != nil { - return fmt.Errorf("copy persisted stdlib archive %s -> cache %s: %w", src, dst, err) - } - if verbose { - fmt.Fprintf(os.Stderr, "stdlib: synced persisted orchestrion export %s -> cache %s\n", src, dst) - } - relDst := dst - if rel, err := filepath.Rel(cachePath, dst); err == nil { - relDst = rel - } - manifest.WriteString(pkg) - manifest.WriteString("=") - manifest.WriteString(relDst) - manifest.WriteString("\n") - } - if manifest.Len() > 0 { - manifestPath := filepath.Join(cachePath, orchestrionStdlibCacheManifestName) - if err := os.WriteFile(manifestPath, []byte(manifest.String()), 0o644); err != nil { - return fmt.Errorf("write stdlib cache manifest at %s: %w", manifestPath, err) - } - if verbose { - fmt.Fprintf(os.Stderr, "stdlib: wrote stdlib cache manifest %s\n", manifestPath) - } + if err := copyArchiveFile(src, dst); err != nil { + return fmt.Errorf("copy persisted stdlib archive %s -> declared cache %s: %w", src, dst, err) + } + manifest.WriteString(pkg) + manifest.WriteString("=") + manifest.WriteString(relDst) + manifest.WriteString("\n") + if verbose { + fmt.Fprintf(os.Stderr, "stdlib: published persisted orchestrion export %s -> declared cache %s\n", src, dst) } } + if manifest.Len() == 0 { + return nil + } + if err := writeFileAtomically(manifestPath, []byte(manifest.String()), 0o644); err != nil { + return fmt.Errorf("write stdlib cache manifest at %s: %w", manifestPath, err) + } + if verbose { + fmt.Fprintf(os.Stderr, "stdlib: wrote stdlib cache manifest %s\n", manifestPath) + } return nil } +func projectStdlibCacheArchive(scratchCache, declaredCache, scratchArchive string) (destination, relative string, err error) { + scratchRoot := abs(scratchCache) + declaredRoot := abs(declaredCache) + archive := abs(scratchArchive) + rel, err := filepath.Rel(scratchRoot, archive) + if err != nil { + return "", "", err + } + rel = filepath.Clean(rel) + if rel == "." || filepath.IsAbs(rel) || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return "", "", fmt.Errorf("cache archive %s escapes scratch root %s", archive, scratchRoot) + } + if !strings.HasSuffix(filepath.Base(rel), "-d") { + return "", "", fmt.Errorf("cache archive %s is not a Go cache data entry", archive) + } + destination = filepath.Join(declaredRoot, rel) + destinationRel, err := filepath.Rel(declaredRoot, destination) + if err != nil || filepath.Clean(destinationRel) != rel { + return "", "", fmt.Errorf("cache archive destination %s escapes declared root %s", destination, declaredRoot) + } + return destination, filepath.ToSlash(rel), nil +} + func ensureSyntheticOrchestrionToolGo(verbose bool, orchestrionMode string) (func(), error) { const toolFile = "orchestrion.tool.go" if _, err := os.Stat(toolFile); err == nil { diff --git a/third_party/rgo/v0_61_1/base/go/tools/builders/stdlib_test.go b/third_party/rgo/v0_61_1/base/go/tools/builders/stdlib_test.go index b92f214b..7ca400cd 100644 --- a/third_party/rgo/v0_61_1/base/go/tools/builders/stdlib_test.go +++ b/third_party/rgo/v0_61_1/base/go/tools/builders/stdlib_test.go @@ -1,8 +1,12 @@ package main import ( + "bytes" + "crypto/sha256" + "fmt" "os" "path/filepath" + "reflect" "strings" "testing" ) @@ -116,48 +120,223 @@ func TestEnsureImportableStdlibModulePathRewritesAndRestores(t *testing.T) { } } -func TestShouldRemoveStdlibCache(t *testing.T) { - t.Parallel() +func TestPrepareStdlibCachesWithoutDeclaredOutput(t *testing.T) { + output := filepath.Join(t.TempDir(), "stdlib") + scratch, declared, cleanup, err := prepareStdlibCaches(output, "") + if err != nil { + t.Fatalf("prepareStdlibCaches error: %v", err) + } + if want := filepath.Join(abs(output), ".gocache"); scratch != want { + t.Fatalf("scratch cache = %q, want %q", scratch, want) + } + if declared != "" { + t.Fatalf("declared cache = %q, want empty", declared) + } + if info, err := os.Stat(scratch); err != nil || !info.IsDir() { + t.Fatalf("scratch cache was not created: %v", err) + } + cleanup() + if _, err := os.Stat(scratch); !os.IsNotExist(err) { + t.Fatalf("scratch cache still exists after cleanup: %v", err) + } +} + +func TestPrepareStdlibCachesSeparatesDeclaredOutput(t *testing.T) { + root := t.TempDir() + output := filepath.Join(root, "stdlib") + cacheOut := filepath.Join(root, "declared") + scratch, declared, cleanup, err := prepareStdlibCaches(output, cacheOut) + if err != nil { + t.Fatalf("prepareStdlibCaches error: %v", err) + } + if sameFilePath(scratch, declared) { + t.Fatalf("scratch cache %q aliases declared cache %q", scratch, declared) + } + for name, path := range map[string]string{"scratch": scratch, "declared": declared} { + if info, err := os.Stat(path); err != nil || !info.IsDir() { + t.Fatalf("%s cache was not created: %v", name, err) + } + } + cleanup() + if _, err := os.Stat(scratch); !os.IsNotExist(err) { + t.Fatalf("scratch cache still exists after cleanup: %v", err) + } + if info, err := os.Stat(declared); err != nil || !info.IsDir() { + t.Fatalf("declared cache should remain after scratch cleanup: %v", err) + } +} + +func TestPrepareStdlibCachesNormalizesRelativeInputs(t *testing.T) { + root := t.TempDir() + previousWD, err := os.Getwd() + if err != nil { + t.Fatalf("getwd: %v", err) + } + if err := os.Chdir(root); err != nil { + t.Fatalf("chdir: %v", err) + } + defer func() { _ = os.Chdir(previousWD) }() + normalizedRoot, err := os.Getwd() + if err != nil { + t.Fatalf("get normalized workdir: %v", err) + } + + scratch, declared, cleanup, err := prepareStdlibCaches("stdlib", "declared") + if err != nil { + t.Fatalf("prepareStdlibCaches error: %v", err) + } + defer cleanup() + if !filepath.IsAbs(scratch) || !filepath.IsAbs(declared) { + t.Fatalf("cache paths are not absolute: scratch=%q declared=%q", scratch, declared) + } + if scratch != filepath.Join(normalizedRoot, "stdlib", ".gocache") { + t.Fatalf("scratch cache = %q", scratch) + } + if declared != filepath.Join(normalizedRoot, "declared") { + t.Fatalf("declared cache = %q", declared) + } +} + +func TestPrepareStdlibCachesRejectsAliasedOutput(t *testing.T) { + output := filepath.Join(t.TempDir(), "stdlib") + alias := filepath.Join(output, ".gocache") + if _, _, _, err := prepareStdlibCaches(output, alias); err == nil || !strings.Contains(err.Error(), "aliases declared cache") { + t.Fatalf("prepareStdlibCaches alias error = %v", err) + } + if _, err := os.Stat(alias); !os.IsNotExist(err) { + t.Fatalf("aliased scratch cache still exists after failure: %v", err) + } +} + +func TestNewBufferedCommandPreservesWritableGoCache(t *testing.T) { + scratch := filepath.Join(t.TempDir(), "scratch") + declared := filepath.Join(t.TempDir(), "declared") + if err := os.MkdirAll(declared, 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("GOCACHE", scratch) + + cmd := (&env{stdlibCache: declared}).newBufferedCommand([]string{"go", "version"}, &bytes.Buffer{}) + if got := getEnv(cmd.Env, "GOCACHE"); got != scratch { + t.Fatalf("GOCACHE = %q, want writable cache %q", got, scratch) + } + if got := getEnv(cmd.Env, orchestrionStdlibCacheEnvVar); got != declared { + t.Fatalf("%s = %q, want %q", orchestrionStdlibCacheEnvVar, got, declared) + } + + for _, invalid := range []string{"", filepath.Join(t.TempDir(), "missing")} { + cmd = (&env{stdlibCache: invalid}).newBufferedCommand([]string{"go", "version"}, &bytes.Buffer{}) + if got := getEnv(cmd.Env, "GOCACHE"); got != scratch { + t.Fatalf("invalid declared cache %q replaced GOCACHE with %q", invalid, got) + } + if got := getEnv(cmd.Env, orchestrionStdlibCacheEnvVar); got != "" { + t.Fatalf("invalid declared cache %q set %s=%q", invalid, orchestrionStdlibCacheEnvVar, got) + } + } +} +func TestPublishPersistedOrchestrionExportsIsDeterministic(t *testing.T) { + archives := map[string][]byte{ + "fmt": []byte("woven fmt archive"), + "log": []byte("woven log archive"), + } + cachePaths := map[string]string{ + "fmt": filepath.Join("11", "fmt-d"), + "log": filepath.Join("aa", "log-d"), + } + + inventories := make([]map[string]string, 0, 2) + for run := 0; run < 2; run++ { + scratch := filepath.Join(t.TempDir(), "scratch") + declared := filepath.Join(t.TempDir(), "declared") + persisted := filepath.Join(t.TempDir(), "persisted") + if err := os.MkdirAll(scratch, 0o755); err != nil { + t.Fatal(err) + } + exports := make(map[string]string, len(archives)) + resolved := make(map[string]string, len(cachePaths)) + for pkg, data := range archives { + src := filepath.Join(persisted, pkg+".a") + writeTestFile(t, src, data) + exports[pkg] = src + resolved[pkg] = filepath.Join(scratch, cachePaths[pkg]) + writeTestFile(t, resolved[pkg], []byte(fmt.Sprintf("unwoven run %d", run))) + } + writeTestFile(t, filepath.Join(scratch, "11", "fmt-a"), []byte(fmt.Sprintf("timestamp %d", run))) + writeTestFile(t, filepath.Join(scratch, "trim.txt"), []byte(fmt.Sprintf("trim %d", run))) + writeTestFile(t, filepath.Join(scratch, "unrelated", "entry-d"), []byte(fmt.Sprintf("unrelated %d", run))) + + if err := publishPersistedOrchestrionExportsToCache(exports, resolved, scratch, declared, false); err != nil { + t.Fatalf("publish run %d: %v", run, err) + } + inventory := testTreeInventory(t, declared) + inventories = append(inventories, inventory) + if len(inventory) != len(archives)+1 { + t.Fatalf("declared inventory has unexpected entries: %v", inventory) + } + for pkg, data := range archives { + path := filepath.ToSlash(cachePaths[pkg]) + if got := inventory[path]; got != testDigest(data) { + t.Fatalf("published %s digest = %q, want %q", path, got, testDigest(data)) + } + } + manifest, err := os.ReadFile(filepath.Join(declared, orchestrionStdlibCacheManifestName)) + if err != nil { + t.Fatal(err) + } + wantManifest := "fmt=11/fmt-d\nlog=aa/log-d\n" + if string(manifest) != wantManifest { + t.Fatalf("manifest = %q, want %q", string(manifest), wantManifest) + } + } + if !reflect.DeepEqual(inventories[0], inventories[1]) { + t.Fatalf("declared cache inventories differ:\nrun 1: %v\nrun 2: %v", inventories[0], inventories[1]) + } +} + +func TestProjectStdlibCacheArchiveRejectsInvalidPaths(t *testing.T) { + root := t.TempDir() + scratch := filepath.Join(root, "scratch") + declared := filepath.Join(root, "declared") tests := []struct { - name string - orchestrionPath string - cacheOut string - want bool + name string + archive string + want string }{ - { - name: "plain internal cache", - want: true, - }, - { - name: "plain declared cache output", - cacheOut: "bazel-out/bin/external/rules_go/stdlib_/gocache", - want: false, - }, - { - name: "orchestrion internal cache", - orchestrionPath: "external/rules_go_orchestrion_tool/orchestrion", - want: false, - }, - { - name: "orchestrion declared cache output", - orchestrionPath: "external/rules_go_orchestrion_tool/orchestrion", - cacheOut: "bazel-out/bin/external/rules_go/stdlib_/gocache", - want: false, - }, + {name: "outside scratch", archive: filepath.Join(root, "outside", "entry-d"), want: "escapes scratch root"}, + {name: "action index", archive: filepath.Join(scratch, "aa", "entry-a"), want: "not a Go cache data entry"}, } - for _, tt := range tests { - tt := tt t.Run(tt.name, func(t *testing.T) { - t.Parallel() - if got := shouldRemoveStdlibCache(tt.orchestrionPath, tt.cacheOut); got != tt.want { - t.Fatalf("shouldRemoveStdlibCache(%q, %q) = %v, want %v", tt.orchestrionPath, tt.cacheOut, got, tt.want) + if _, _, err := projectStdlibCacheArchive(scratch, declared, tt.archive); err == nil || !strings.Contains(err.Error(), tt.want) { + t.Fatalf("projectStdlibCacheArchive error = %v, want %q", err, tt.want) } }) } } +func TestPublishPersistedOrchestrionExportsFailureLeavesNoManifest(t *testing.T) { + root := t.TempDir() + scratch := filepath.Join(root, "scratch") + declared := filepath.Join(root, "declared") + manifest := filepath.Join(declared, orchestrionStdlibCacheManifestName) + writeTestFile(t, manifest, []byte("stale=stale-d\n")) + + err := publishPersistedOrchestrionExportsToCache( + map[string]string{}, + map[string]string{"fmt": filepath.Join(scratch, "aa", "fmt-d")}, + scratch, + declared, + false, + ) + if err == nil || !strings.Contains(err.Error(), "missing persisted stdlib archive for cache package fmt") { + t.Fatalf("publish error = %v", err) + } + if _, err := os.Stat(manifest); !os.IsNotExist(err) { + t.Fatalf("manifest exists after failed publication: %v", err) + } +} + func TestMergeGoDebugSettingPreservesExistingFlags(t *testing.T) { tests := []struct { name string @@ -238,3 +417,48 @@ func TestPersistOrchestrionStdlibExportsUsesGoToolInstallDir(t *testing.T) { } } } + +func writeTestFile(t *testing.T, path string, data []byte) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("mkdir %s: %v", filepath.Dir(path), err) + } + if err := os.WriteFile(path, data, 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } +} + +func testTreeInventory(t *testing.T, root string) map[string]string { + t.Helper() + inventory := make(map[string]string) + err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if path == root || entry.IsDir() { + return nil + } + if entry.Type()&os.ModeSymlink != 0 { + return fmt.Errorf("unexpected symlink %s", path) + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + rel, err := filepath.Rel(root, path) + if err != nil { + return err + } + inventory[filepath.ToSlash(rel)] = testDigest(data) + return nil + }) + if err != nil { + t.Fatalf("inventory %s: %v", root, err) + } + return inventory +} + +func testDigest(data []byte) string { + sum := sha256.Sum256(data) + return fmt.Sprintf("%x", sum[:]) +} diff --git a/third_party/rgo/v0_62_0/base/go/tools/builders/env_orchestrion.go b/third_party/rgo/v0_62_0/base/go/tools/builders/env_orchestrion.go index fbc392cb..73c7992c 100644 --- a/third_party/rgo/v0_62_0/base/go/tools/builders/env_orchestrion.go +++ b/third_party/rgo/v0_62_0/base/go/tools/builders/env_orchestrion.go @@ -59,7 +59,8 @@ func (e *env) runCommandWithJobserver(args []string, jobserver *orchestrionJobse } // newBufferedCommand creates a subprocess command wired to the shared builder -// buffer and applies the stdlib cache override needed by Orchestrion actions. +// buffer and advertises the declared stdlib archive source without replacing +// the subprocess's writable GOCACHE. func (e *env) newBufferedCommand(args []string, buf *bytes.Buffer) *exec.Cmd { cmd := exec.Command(args[0], args[1:]...) cmd.Stdout = buf @@ -67,7 +68,6 @@ func (e *env) newBufferedCommand(args []string, buf *bytes.Buffer) *exec.Cmd { cmd.Env = os.Environ() if e.stdlibCache != "" { if info, err := os.Stat(e.stdlibCache); err == nil && info.IsDir() { - cmd.Env = setEnv(cmd.Env, "GOCACHE", e.stdlibCache) cmd.Env = setEnv(cmd.Env, orchestrionStdlibCacheEnvVar, e.stdlibCache) } } diff --git a/third_party/rgo/v0_62_0/base/go/tools/builders/importcfg.go b/third_party/rgo/v0_62_0/base/go/tools/builders/importcfg.go index 3f0e2030..4c859ae4 100644 --- a/third_party/rgo/v0_62_0/base/go/tools/builders/importcfg.go +++ b/third_party/rgo/v0_62_0/base/go/tools/builders/importcfg.go @@ -1304,29 +1304,17 @@ func resolveInstrumentedStdlibExports(goenv *env, packages []string) (map[string baseEnv = setEnv(baseEnv, "GOROOT", goenv.goroot) baseEnv = setEnv(baseEnv, "GO111MODULE", "off") baseEnv = setEnv(baseEnv, "GOWORK", "off") - orchestrionCachePath := goenv.stdlibCache - if orchestrionCachePath == "" { - orchestrionCachePath = filepath.Join(goenv.goroot, ".gocache") - } - cachePath := "" - if info, err := os.Stat(orchestrionCachePath); err == nil && info.IsDir() { - cachePath = orchestrionCachePath - } - if cachePath == "" { - cachePath = getEnv(baseEnv, "GOCACHE") - } - if cachePath == "" { - if goenv.stdlibCache != "" { - cachePath = goenv.stdlibCache - } else { - cachePath = filepath.Join(goenv.goroot, ".gocache") - } - } - cachePath = abs(cachePath) - if err := os.MkdirAll(cachePath, 0o755); err != nil { + cachePath, err := writableStdlibCacheRoot(goenv, baseEnv) + if err != nil { return nil, fmt.Errorf("prepare stdlib export cache: %w", err) } + if err := seedWovenStdlibCache(goenv, cachePath); err != nil { + return nil, fmt.Errorf("seed stdlib export cache: %w", err) + } baseEnv = setEnv(baseEnv, "GOCACHE", cachePath) + if goenv.stdlibCache != "" { + baseEnv = setEnv(baseEnv, orchestrionStdlibCacheEnvVar, goenv.stdlibCache) + } if getEnv(baseEnv, "HOME") == "" { homePath := filepath.Join(goenv.goroot, ".home") if err := os.MkdirAll(homePath, 0o755); err != nil { @@ -1366,9 +1354,18 @@ func resolveInstrumentedStdlibExports(goenv *env, packages []string) (map[string } func resolveCacheStdlibExports(goenv *env, packages []string) (map[string]string, error) { - cacheRoot := "" - if goenv != nil { - cacheRoot = goenv.stdlibCache + if goenv != nil && goenv.stdlibCache != "" { + exports, err := readStdlibCacheManifest(goenv.stdlibCache, packages) + if err != nil { + return nil, err + } + if len(exports) > 0 { + return exports, nil + } + } + cacheRoot, err := writableStdlibCacheRoot(goenv, os.Environ()) + if err != nil { + return nil, err } return resolveCacheStdlibExportsAt(goenv, packages, cacheRoot) } @@ -1397,6 +1394,9 @@ func resolveCacheStdlibExportsAt(goenv *env, packages []string, cacheRoot string return manifestExports, nil } } + if goenv.stdlibCache != "" && sameFilePath(cacheRoot, goenv.stdlibCache) { + return nil, fmt.Errorf("refusing to use declared stdlib cache %s as writable GOCACHE", abs(goenv.stdlibCache)) + } goenv.goroot = abs(goenv.goroot) goenv.sdk = abs(goenv.sdk) if info, err := os.Stat(goenv.goroot); err != nil || !info.IsDir() { @@ -1480,6 +1480,24 @@ func cacheStdlibGoListBaseEnv(goenv *env, cachePath string, environ []string) [] return env } +func writableStdlibCacheRoot(goenv *env, environ []string) (string, error) { + cacheRoot := strings.TrimSpace(getEnv(environ, "GOCACHE")) + if cacheRoot == "" { + if goenv == nil || strings.TrimSpace(goenv.goroot) == "" { + return "", fmt.Errorf("writable stdlib GOCACHE is empty") + } + cacheRoot = filepath.Join(abs(goenv.goroot), ".gocache") + } + cacheRoot = abs(cacheRoot) + if goenv != nil && goenv.stdlibCache != "" && sameFilePath(cacheRoot, goenv.stdlibCache) { + return "", fmt.Errorf("writable stdlib GOCACHE %s aliases declared cache", cacheRoot) + } + if err := os.MkdirAll(cacheRoot, 0o755); err != nil { + return "", err + } + return cacheRoot, nil +} + func readStdlibCacheManifest(cacheRoot string, packages []string) (map[string]string, error) { if cacheRoot == "" { return nil, nil @@ -1874,18 +1892,17 @@ func resolveStdlibExportsForPackage(goenv *env, pkg string) (map[string]string, cmd.Env = setEnv(cmd.Env, "GOROOT", goenv.goroot) cmd.Env = setEnv(cmd.Env, "GO111MODULE", "on") cmd.Env = setEnv(cmd.Env, "GOWORK", "off") - cachePath := goenv.stdlibCache - if cachePath == "" { - cachePath = getEnv(cmd.Env, "GOCACHE") - } - if cachePath == "" { - cachePath = filepath.Join(goenv.goroot, ".gocache") - } - cachePath = abs(cachePath) - if err := os.MkdirAll(cachePath, 0o755); err != nil { + cachePath, err := writableStdlibCacheRoot(goenv, cmd.Env) + if err != nil { return nil, fmt.Errorf("prepare stdlib closure cache: %w", err) } + if err := seedWovenStdlibCache(goenv, cachePath); err != nil { + return nil, fmt.Errorf("seed stdlib closure cache: %w", err) + } cmd.Env = setEnv(cmd.Env, "GOCACHE", cachePath) + if goenv.stdlibCache != "" { + cmd.Env = setEnv(cmd.Env, orchestrionStdlibCacheEnvVar, goenv.stdlibCache) + } normalizedEnv, err := normalizeGoActionCacheEnv(cmd.Env) if err != nil { return nil, fmt.Errorf("prepare stdlib closure action cache env: %w", err) diff --git a/third_party/rgo/v0_62_0/base/go/tools/builders/importcfg_test.go b/third_party/rgo/v0_62_0/base/go/tools/builders/importcfg_test.go index 3724594d..35a1fa4a 100644 --- a/third_party/rgo/v0_62_0/base/go/tools/builders/importcfg_test.go +++ b/third_party/rgo/v0_62_0/base/go/tools/builders/importcfg_test.go @@ -563,6 +563,47 @@ func TestCacheStdlibGoListBaseEnvUsesSDKGoRoot(t *testing.T) { } } +func TestWritableStdlibCacheRootUsesPrivateCache(t *testing.T) { + privateCache := filepath.Join(t.TempDir(), "private") + declaredCache := filepath.Join(t.TempDir(), "declared") + cacheRoot, err := writableStdlibCacheRoot(&env{ + goroot: t.TempDir(), + stdlibCache: declaredCache, + }, []string{"GOCACHE=" + privateCache}) + if err != nil { + t.Fatalf("writableStdlibCacheRoot error: %v", err) + } + if cacheRoot != abs(privateCache) { + t.Fatalf("writable cache = %q, want %q", cacheRoot, abs(privateCache)) + } + if info, err := os.Stat(cacheRoot); err != nil || !info.IsDir() { + t.Fatalf("writable cache was not created: %v", err) + } +} + +func TestWritableStdlibCacheRootRejectsDeclaredCache(t *testing.T) { + declaredCache := filepath.Join(t.TempDir(), "declared") + _, err := writableStdlibCacheRoot(&env{ + goroot: t.TempDir(), + stdlibCache: declaredCache, + }, []string{"GOCACHE=" + declaredCache}) + if err == nil || !strings.Contains(err.Error(), "aliases declared cache") { + t.Fatalf("writableStdlibCacheRoot error = %v", err) + } +} + +func TestResolveCacheStdlibExportsRejectsDeclaredCacheAsWritable(t *testing.T) { + declaredCache := t.TempDir() + _, err := resolveCacheStdlibExportsAt(&env{ + goroot: "fake-goroot", + sdk: "fake-sdk", + stdlibCache: declaredCache, + }, []string{"fmt"}, declaredCache) + if err == nil || !strings.Contains(err.Error(), "refusing to use declared stdlib cache") { + t.Fatalf("resolveCacheStdlibExportsAt error = %v", err) + } +} + func TestModuleExportCacheManifestRoundTrip(t *testing.T) { root := t.TempDir() paths := orchestrionCachePaths(root, "module-exports", "abc123") diff --git a/third_party/rgo/v0_62_0/base/go/tools/builders/stdlib.go b/third_party/rgo/v0_62_0/base/go/tools/builders/stdlib.go index 08b29926..9b5c36c7 100644 --- a/third_party/rgo/v0_62_0/base/go/tools/builders/stdlib.go +++ b/third_party/rgo/v0_62_0/base/go/tools/builders/stdlib.go @@ -96,20 +96,15 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) os.Setenv("GOROOT", output) goenv.goroot = output - // Create a temporary cache directory. "go build" requires this starting - // in Go 1.12. - cachePath := filepath.Join(output, ".gocache") - if *cacheOut != "" { - cachePath = abs(*cacheOut) - goenv.stdlibCache = cachePath + // Live Go commands use an action-private scratch cache. The separately + // declared cache output is populated only with deterministic woven archives. + cachePath, declaredCache, cleanupCache, err := prepareStdlibCaches(output, *cacheOut) + if err != nil { + return err } + defer cleanupCache() + goenv.stdlibCache = declaredCache os.Setenv("GOCACHE", cachePath) - if err := os.MkdirAll(cachePath, 0o755); err != nil { - return fmt.Errorf("prepare stdlib gocache at %s: %w", cachePath, err) - } - if shouldRemoveStdlibCache(*orchestrion, *cacheOut) { - defer os.RemoveAll(cachePath) - } // Disable modules for the plain stdlib build command. When Orchestrion is // enabled we flip this back on later after preparing a synthetic module that @@ -356,12 +351,33 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) return nil } -// shouldRemoveStdlibCache reports whether the builder owns the cache directory -// as scratch space. A non-empty -cacheout value is a Bazel-declared TreeArtifact -// output, so it must remain present even for plain non-Orchestrion stdlib -// actions. -func shouldRemoveStdlibCache(orchestrionPath, cacheOut string) bool { - return strings.TrimSpace(orchestrionPath) == "" && strings.TrimSpace(cacheOut) == "" +func prepareStdlibCaches(output, cacheOut string) (scratchCache, declaredCache string, cleanup func(), err error) { + scratchCache = filepath.Join(abs(output), ".gocache") + cleanup = func() { + _ = os.RemoveAll(scratchCache) + } + if err := os.MkdirAll(scratchCache, 0o755); err != nil { + return "", "", func() {}, fmt.Errorf("prepare stdlib scratch gocache at %s: %w", scratchCache, err) + } + if strings.TrimSpace(cacheOut) == "" { + return scratchCache, "", cleanup, nil + } + + declaredCache = abs(cacheOut) + if sameFilePath(scratchCache, declaredCache) { + cleanup() + return "", "", func() {}, fmt.Errorf("stdlib scratch gocache aliases declared cache output at %s", declaredCache) + } + if err := os.MkdirAll(declaredCache, 0o755); err != nil { + cleanup() + return "", "", func() {}, fmt.Errorf("prepare declared stdlib cache output at %s: %w", declaredCache, err) + } + return scratchCache, declaredCache, cleanup, nil +} + +func sameFilePath(left, right string) bool { + rel, err := filepath.Rel(abs(left), abs(right)) + return err == nil && rel == "." } func mergeGoDebugSetting(existing, setting string) string { @@ -497,111 +513,101 @@ func syncPersistedOrchestrionExportsToCache(goenv *env, exports map[string]strin defer func() { span.End(err) }() - if goenv == nil || len(exports) == 0 { + if goenv == nil || len(exports) == 0 || len(roots) == 0 || strings.TrimSpace(goenv.stdlibCache) == "" { return nil } - // We have two cache families to keep consistent: - // 1. the Bazel-declared stdlib cache consumed by later compile/link actions - // 2. the shared Datadog/Orchestrion cache used by internal `go list -export` - // dependency resolution when woven deps are injected. - // - // If only one is populated, compile and link can observe different archive - // fingerprints for stdlib packages like log/fmt/flag. Populate both from the - // same woven persisted exports. - candidateCaches := []string{} - seenCaches := map[string]struct{}{} - addCache := func(path string) { - path = strings.TrimSpace(path) - if path == "" { - return - } - path = abs(path) - if _, ok := seenCaches[path]; ok { - return - } - seenCaches[path] = struct{}{} - candidateCaches = append(candidateCaches, path) + scratchCache := strings.TrimSpace(os.Getenv("GOCACHE")) + if scratchCache == "" { + return fmt.Errorf("publish stdlib cache exports: GOCACHE scratch path is empty") } - - addCache(goenv.stdlibCache) - if envCache := strings.TrimSpace(os.Getenv("GOCACHE")); envCache != "" { - addCache(envCache) + scratchCache = abs(scratchCache) + declaredCache := abs(goenv.stdlibCache) + if sameFilePath(scratchCache, declaredCache) { + return fmt.Errorf("publish stdlib cache exports: scratch cache %s aliases declared cache %s", scratchCache, declaredCache) } - if len(candidateCaches) == 0 { - addCache(filepath.Join(abs(goenv.goroot), ".gocache")) + if err := os.MkdirAll(declaredCache, 0o755); err != nil { + return fmt.Errorf("prepare declared stdlib cache exports at %s: %w", declaredCache, err) } + if verbose { + fmt.Fprintf(os.Stderr, "stdlib: resolving cache exports against scratch GOCACHE=%s\n", scratchCache) + } + cacheExports, err := resolveCacheStdlibExportsAt(goenv, roots, scratchCache) + if err != nil { + return err + } + return publishPersistedOrchestrionExportsToCache(exports, cacheExports, scratchCache, declaredCache, verbose) +} - prevCachePath, hadPrevCachePath := os.LookupEnv("GOCACHE") - defer func() { - if hadPrevCachePath { - _ = os.Setenv("GOCACHE", prevCachePath) - } else { - _ = os.Unsetenv("GOCACHE") - } - }() - - for _, cachePath := range candidateCaches { - if err := os.MkdirAll(cachePath, 0o755); err != nil { - return fmt.Errorf("prepare stdlib cache exports at %s: %w", cachePath, err) - } - if err := os.Setenv("GOCACHE", cachePath); err != nil { - return fmt.Errorf("set stdlib cache exports path %s: %w", cachePath, err) - } - if verbose { - fmt.Fprintf(os.Stderr, "stdlib: resolving cache-family exports against GOCACHE=%s\n", cachePath) +func publishPersistedOrchestrionExportsToCache(exports, cacheExports map[string]string, scratchCache, declaredCache string, verbose bool) error { + manifestPath := filepath.Join(abs(declaredCache), orchestrionStdlibCacheManifestName) + if err := os.Remove(manifestPath); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("remove stale stdlib cache manifest at %s: %w", manifestPath, err) + } + packages := make([]string, 0, len(cacheExports)) + for pkg := range cacheExports { + packages = append(packages, pkg) + } + sort.Strings(packages) + var manifest strings.Builder + for _, pkg := range packages { + src := strings.TrimSpace(exports[pkg]) + if src == "" { + return fmt.Errorf("missing persisted stdlib archive for cache package %s", pkg) } - - cacheExports, err := resolveCacheStdlibExportsAt(goenv, roots, cachePath) + dst, relDst, err := projectStdlibCacheArchive(scratchCache, declaredCache, cacheExports[pkg]) if err != nil { + return fmt.Errorf("project stdlib cache archive for %s: %w", pkg, err) + } + if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { return err } - packages := make([]string, 0, len(cacheExports)) - for pkg := range cacheExports { - packages = append(packages, pkg) - } - sort.Strings(packages) - var manifest strings.Builder - for _, pkg := range packages { - src := exports[pkg] - dst, ok := cacheExports[pkg] - if !ok || dst == "" { - continue - } - if strings.TrimSpace(src) == "" { - return fmt.Errorf("missing persisted stdlib archive for cache package %s", pkg) - } - if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { - return err - } - if err := copyArchiveFile(src, dst); err != nil { - return fmt.Errorf("copy persisted stdlib archive %s -> cache %s: %w", src, dst, err) - } - if verbose { - fmt.Fprintf(os.Stderr, "stdlib: synced persisted orchestrion export %s -> cache %s\n", src, dst) - } - relDst := dst - if rel, err := filepath.Rel(cachePath, dst); err == nil { - relDst = rel - } - manifest.WriteString(pkg) - manifest.WriteString("=") - manifest.WriteString(relDst) - manifest.WriteString("\n") - } - if manifest.Len() > 0 { - manifestPath := filepath.Join(cachePath, orchestrionStdlibCacheManifestName) - if err := os.WriteFile(manifestPath, []byte(manifest.String()), 0o644); err != nil { - return fmt.Errorf("write stdlib cache manifest at %s: %w", manifestPath, err) - } - if verbose { - fmt.Fprintf(os.Stderr, "stdlib: wrote stdlib cache manifest %s\n", manifestPath) - } + if err := copyArchiveFile(src, dst); err != nil { + return fmt.Errorf("copy persisted stdlib archive %s -> declared cache %s: %w", src, dst, err) + } + manifest.WriteString(pkg) + manifest.WriteString("=") + manifest.WriteString(relDst) + manifest.WriteString("\n") + if verbose { + fmt.Fprintf(os.Stderr, "stdlib: published persisted orchestrion export %s -> declared cache %s\n", src, dst) } } + if manifest.Len() == 0 { + return nil + } + if err := writeFileAtomically(manifestPath, []byte(manifest.String()), 0o644); err != nil { + return fmt.Errorf("write stdlib cache manifest at %s: %w", manifestPath, err) + } + if verbose { + fmt.Fprintf(os.Stderr, "stdlib: wrote stdlib cache manifest %s\n", manifestPath) + } return nil } +func projectStdlibCacheArchive(scratchCache, declaredCache, scratchArchive string) (destination, relative string, err error) { + scratchRoot := abs(scratchCache) + declaredRoot := abs(declaredCache) + archive := abs(scratchArchive) + rel, err := filepath.Rel(scratchRoot, archive) + if err != nil { + return "", "", err + } + rel = filepath.Clean(rel) + if rel == "." || filepath.IsAbs(rel) || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return "", "", fmt.Errorf("cache archive %s escapes scratch root %s", archive, scratchRoot) + } + if !strings.HasSuffix(filepath.Base(rel), "-d") { + return "", "", fmt.Errorf("cache archive %s is not a Go cache data entry", archive) + } + destination = filepath.Join(declaredRoot, rel) + destinationRel, err := filepath.Rel(declaredRoot, destination) + if err != nil || filepath.Clean(destinationRel) != rel { + return "", "", fmt.Errorf("cache archive destination %s escapes declared root %s", destination, declaredRoot) + } + return destination, filepath.ToSlash(rel), nil +} + func ensureSyntheticOrchestrionToolGo(verbose bool, orchestrionMode string) (func(), error) { const toolFile = "orchestrion.tool.go" if _, err := os.Stat(toolFile); err == nil { diff --git a/third_party/rgo/v0_62_0/base/go/tools/builders/stdlib_test.go b/third_party/rgo/v0_62_0/base/go/tools/builders/stdlib_test.go index b92f214b..7ca400cd 100644 --- a/third_party/rgo/v0_62_0/base/go/tools/builders/stdlib_test.go +++ b/third_party/rgo/v0_62_0/base/go/tools/builders/stdlib_test.go @@ -1,8 +1,12 @@ package main import ( + "bytes" + "crypto/sha256" + "fmt" "os" "path/filepath" + "reflect" "strings" "testing" ) @@ -116,48 +120,223 @@ func TestEnsureImportableStdlibModulePathRewritesAndRestores(t *testing.T) { } } -func TestShouldRemoveStdlibCache(t *testing.T) { - t.Parallel() +func TestPrepareStdlibCachesWithoutDeclaredOutput(t *testing.T) { + output := filepath.Join(t.TempDir(), "stdlib") + scratch, declared, cleanup, err := prepareStdlibCaches(output, "") + if err != nil { + t.Fatalf("prepareStdlibCaches error: %v", err) + } + if want := filepath.Join(abs(output), ".gocache"); scratch != want { + t.Fatalf("scratch cache = %q, want %q", scratch, want) + } + if declared != "" { + t.Fatalf("declared cache = %q, want empty", declared) + } + if info, err := os.Stat(scratch); err != nil || !info.IsDir() { + t.Fatalf("scratch cache was not created: %v", err) + } + cleanup() + if _, err := os.Stat(scratch); !os.IsNotExist(err) { + t.Fatalf("scratch cache still exists after cleanup: %v", err) + } +} + +func TestPrepareStdlibCachesSeparatesDeclaredOutput(t *testing.T) { + root := t.TempDir() + output := filepath.Join(root, "stdlib") + cacheOut := filepath.Join(root, "declared") + scratch, declared, cleanup, err := prepareStdlibCaches(output, cacheOut) + if err != nil { + t.Fatalf("prepareStdlibCaches error: %v", err) + } + if sameFilePath(scratch, declared) { + t.Fatalf("scratch cache %q aliases declared cache %q", scratch, declared) + } + for name, path := range map[string]string{"scratch": scratch, "declared": declared} { + if info, err := os.Stat(path); err != nil || !info.IsDir() { + t.Fatalf("%s cache was not created: %v", name, err) + } + } + cleanup() + if _, err := os.Stat(scratch); !os.IsNotExist(err) { + t.Fatalf("scratch cache still exists after cleanup: %v", err) + } + if info, err := os.Stat(declared); err != nil || !info.IsDir() { + t.Fatalf("declared cache should remain after scratch cleanup: %v", err) + } +} + +func TestPrepareStdlibCachesNormalizesRelativeInputs(t *testing.T) { + root := t.TempDir() + previousWD, err := os.Getwd() + if err != nil { + t.Fatalf("getwd: %v", err) + } + if err := os.Chdir(root); err != nil { + t.Fatalf("chdir: %v", err) + } + defer func() { _ = os.Chdir(previousWD) }() + normalizedRoot, err := os.Getwd() + if err != nil { + t.Fatalf("get normalized workdir: %v", err) + } + + scratch, declared, cleanup, err := prepareStdlibCaches("stdlib", "declared") + if err != nil { + t.Fatalf("prepareStdlibCaches error: %v", err) + } + defer cleanup() + if !filepath.IsAbs(scratch) || !filepath.IsAbs(declared) { + t.Fatalf("cache paths are not absolute: scratch=%q declared=%q", scratch, declared) + } + if scratch != filepath.Join(normalizedRoot, "stdlib", ".gocache") { + t.Fatalf("scratch cache = %q", scratch) + } + if declared != filepath.Join(normalizedRoot, "declared") { + t.Fatalf("declared cache = %q", declared) + } +} + +func TestPrepareStdlibCachesRejectsAliasedOutput(t *testing.T) { + output := filepath.Join(t.TempDir(), "stdlib") + alias := filepath.Join(output, ".gocache") + if _, _, _, err := prepareStdlibCaches(output, alias); err == nil || !strings.Contains(err.Error(), "aliases declared cache") { + t.Fatalf("prepareStdlibCaches alias error = %v", err) + } + if _, err := os.Stat(alias); !os.IsNotExist(err) { + t.Fatalf("aliased scratch cache still exists after failure: %v", err) + } +} + +func TestNewBufferedCommandPreservesWritableGoCache(t *testing.T) { + scratch := filepath.Join(t.TempDir(), "scratch") + declared := filepath.Join(t.TempDir(), "declared") + if err := os.MkdirAll(declared, 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("GOCACHE", scratch) + + cmd := (&env{stdlibCache: declared}).newBufferedCommand([]string{"go", "version"}, &bytes.Buffer{}) + if got := getEnv(cmd.Env, "GOCACHE"); got != scratch { + t.Fatalf("GOCACHE = %q, want writable cache %q", got, scratch) + } + if got := getEnv(cmd.Env, orchestrionStdlibCacheEnvVar); got != declared { + t.Fatalf("%s = %q, want %q", orchestrionStdlibCacheEnvVar, got, declared) + } + + for _, invalid := range []string{"", filepath.Join(t.TempDir(), "missing")} { + cmd = (&env{stdlibCache: invalid}).newBufferedCommand([]string{"go", "version"}, &bytes.Buffer{}) + if got := getEnv(cmd.Env, "GOCACHE"); got != scratch { + t.Fatalf("invalid declared cache %q replaced GOCACHE with %q", invalid, got) + } + if got := getEnv(cmd.Env, orchestrionStdlibCacheEnvVar); got != "" { + t.Fatalf("invalid declared cache %q set %s=%q", invalid, orchestrionStdlibCacheEnvVar, got) + } + } +} +func TestPublishPersistedOrchestrionExportsIsDeterministic(t *testing.T) { + archives := map[string][]byte{ + "fmt": []byte("woven fmt archive"), + "log": []byte("woven log archive"), + } + cachePaths := map[string]string{ + "fmt": filepath.Join("11", "fmt-d"), + "log": filepath.Join("aa", "log-d"), + } + + inventories := make([]map[string]string, 0, 2) + for run := 0; run < 2; run++ { + scratch := filepath.Join(t.TempDir(), "scratch") + declared := filepath.Join(t.TempDir(), "declared") + persisted := filepath.Join(t.TempDir(), "persisted") + if err := os.MkdirAll(scratch, 0o755); err != nil { + t.Fatal(err) + } + exports := make(map[string]string, len(archives)) + resolved := make(map[string]string, len(cachePaths)) + for pkg, data := range archives { + src := filepath.Join(persisted, pkg+".a") + writeTestFile(t, src, data) + exports[pkg] = src + resolved[pkg] = filepath.Join(scratch, cachePaths[pkg]) + writeTestFile(t, resolved[pkg], []byte(fmt.Sprintf("unwoven run %d", run))) + } + writeTestFile(t, filepath.Join(scratch, "11", "fmt-a"), []byte(fmt.Sprintf("timestamp %d", run))) + writeTestFile(t, filepath.Join(scratch, "trim.txt"), []byte(fmt.Sprintf("trim %d", run))) + writeTestFile(t, filepath.Join(scratch, "unrelated", "entry-d"), []byte(fmt.Sprintf("unrelated %d", run))) + + if err := publishPersistedOrchestrionExportsToCache(exports, resolved, scratch, declared, false); err != nil { + t.Fatalf("publish run %d: %v", run, err) + } + inventory := testTreeInventory(t, declared) + inventories = append(inventories, inventory) + if len(inventory) != len(archives)+1 { + t.Fatalf("declared inventory has unexpected entries: %v", inventory) + } + for pkg, data := range archives { + path := filepath.ToSlash(cachePaths[pkg]) + if got := inventory[path]; got != testDigest(data) { + t.Fatalf("published %s digest = %q, want %q", path, got, testDigest(data)) + } + } + manifest, err := os.ReadFile(filepath.Join(declared, orchestrionStdlibCacheManifestName)) + if err != nil { + t.Fatal(err) + } + wantManifest := "fmt=11/fmt-d\nlog=aa/log-d\n" + if string(manifest) != wantManifest { + t.Fatalf("manifest = %q, want %q", string(manifest), wantManifest) + } + } + if !reflect.DeepEqual(inventories[0], inventories[1]) { + t.Fatalf("declared cache inventories differ:\nrun 1: %v\nrun 2: %v", inventories[0], inventories[1]) + } +} + +func TestProjectStdlibCacheArchiveRejectsInvalidPaths(t *testing.T) { + root := t.TempDir() + scratch := filepath.Join(root, "scratch") + declared := filepath.Join(root, "declared") tests := []struct { - name string - orchestrionPath string - cacheOut string - want bool + name string + archive string + want string }{ - { - name: "plain internal cache", - want: true, - }, - { - name: "plain declared cache output", - cacheOut: "bazel-out/bin/external/rules_go/stdlib_/gocache", - want: false, - }, - { - name: "orchestrion internal cache", - orchestrionPath: "external/rules_go_orchestrion_tool/orchestrion", - want: false, - }, - { - name: "orchestrion declared cache output", - orchestrionPath: "external/rules_go_orchestrion_tool/orchestrion", - cacheOut: "bazel-out/bin/external/rules_go/stdlib_/gocache", - want: false, - }, + {name: "outside scratch", archive: filepath.Join(root, "outside", "entry-d"), want: "escapes scratch root"}, + {name: "action index", archive: filepath.Join(scratch, "aa", "entry-a"), want: "not a Go cache data entry"}, } - for _, tt := range tests { - tt := tt t.Run(tt.name, func(t *testing.T) { - t.Parallel() - if got := shouldRemoveStdlibCache(tt.orchestrionPath, tt.cacheOut); got != tt.want { - t.Fatalf("shouldRemoveStdlibCache(%q, %q) = %v, want %v", tt.orchestrionPath, tt.cacheOut, got, tt.want) + if _, _, err := projectStdlibCacheArchive(scratch, declared, tt.archive); err == nil || !strings.Contains(err.Error(), tt.want) { + t.Fatalf("projectStdlibCacheArchive error = %v, want %q", err, tt.want) } }) } } +func TestPublishPersistedOrchestrionExportsFailureLeavesNoManifest(t *testing.T) { + root := t.TempDir() + scratch := filepath.Join(root, "scratch") + declared := filepath.Join(root, "declared") + manifest := filepath.Join(declared, orchestrionStdlibCacheManifestName) + writeTestFile(t, manifest, []byte("stale=stale-d\n")) + + err := publishPersistedOrchestrionExportsToCache( + map[string]string{}, + map[string]string{"fmt": filepath.Join(scratch, "aa", "fmt-d")}, + scratch, + declared, + false, + ) + if err == nil || !strings.Contains(err.Error(), "missing persisted stdlib archive for cache package fmt") { + t.Fatalf("publish error = %v", err) + } + if _, err := os.Stat(manifest); !os.IsNotExist(err) { + t.Fatalf("manifest exists after failed publication: %v", err) + } +} + func TestMergeGoDebugSettingPreservesExistingFlags(t *testing.T) { tests := []struct { name string @@ -238,3 +417,48 @@ func TestPersistOrchestrionStdlibExportsUsesGoToolInstallDir(t *testing.T) { } } } + +func writeTestFile(t *testing.T, path string, data []byte) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("mkdir %s: %v", filepath.Dir(path), err) + } + if err := os.WriteFile(path, data, 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } +} + +func testTreeInventory(t *testing.T, root string) map[string]string { + t.Helper() + inventory := make(map[string]string) + err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if path == root || entry.IsDir() { + return nil + } + if entry.Type()&os.ModeSymlink != 0 { + return fmt.Errorf("unexpected symlink %s", path) + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + rel, err := filepath.Rel(root, path) + if err != nil { + return err + } + inventory[filepath.ToSlash(rel)] = testDigest(data) + return nil + }) + if err != nil { + t.Fatalf("inventory %s: %v", root, err) + } + return inventory +} + +func testDigest(data []byte) string { + sum := sha256.Sum256(data) + return fmt.Sprintf("%x", sum[:]) +} diff --git a/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch index 0025eb8f..63722fd6 100644 --- a/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch @@ -7598,7 +7598,7 @@ index fadb164..00390ca 100644 if err := cmd.Run(); err != nil { diff --git a/go/tools/builders/env_orchestrion.go b/go/tools/builders/env_orchestrion.go new file mode 100644 -index 0000000..fbc392c +index 0000000..73c7992 --- /dev/null +++ b/go/tools/builders/env_orchestrion.go @@ -0,0 +1,81 @@ @@ -7663,7 +7663,8 @@ index 0000000..fbc392c +} + +// newBufferedCommand creates a subprocess command wired to the shared builder -+// buffer and applies the stdlib cache override needed by Orchestrion actions. ++// buffer and advertises the declared stdlib archive source without replacing ++// the subprocess's writable GOCACHE. +func (e *env) newBufferedCommand(args []string, buf *bytes.Buffer) *exec.Cmd { + cmd := exec.Command(args[0], args[1:]...) + cmd.Stdout = buf @@ -7671,7 +7672,6 @@ index 0000000..fbc392c + cmd.Env = os.Environ() + if e.stdlibCache != "" { + if info, err := os.Stat(e.stdlibCache); err == nil && info.IsDir() { -+ cmd.Env = setEnv(cmd.Env, "GOCACHE", e.stdlibCache) + cmd.Env = setEnv(cmd.Env, orchestrionStdlibCacheEnvVar, e.stdlibCache) + } + } @@ -8145,7 +8145,7 @@ index 893a0f6..16955a7 100644 + return "" +} diff --git a/go/tools/builders/importcfg.go b/go/tools/builders/importcfg.go -index c25763a..3f0e203 100644 +index c25763a..4c859ae 100644 --- a/go/tools/builders/importcfg.go +++ b/go/tools/builders/importcfg.go @@ -17,21 +17,94 @@ package main @@ -8243,7 +8243,7 @@ index c25763a..3f0e203 100644 // checkImports verifies that each import in files refers to a // direct dependency in archives or to a standard library package // listed in the file at stdPackageListPath. checkImports returns -@@ -213,6 +286,1678 @@ package with this path is linked.`, +@@ -213,6 +286,1695 @@ package with this path is linked.`, return filename, nil } @@ -9265,29 +9265,17 @@ index c25763a..3f0e203 100644 + baseEnv = setEnv(baseEnv, "GOROOT", goenv.goroot) + baseEnv = setEnv(baseEnv, "GO111MODULE", "off") + baseEnv = setEnv(baseEnv, "GOWORK", "off") -+ orchestrionCachePath := goenv.stdlibCache -+ if orchestrionCachePath == "" { -+ orchestrionCachePath = filepath.Join(goenv.goroot, ".gocache") -+ } -+ cachePath := "" -+ if info, err := os.Stat(orchestrionCachePath); err == nil && info.IsDir() { -+ cachePath = orchestrionCachePath -+ } -+ if cachePath == "" { -+ cachePath = getEnv(baseEnv, "GOCACHE") -+ } -+ if cachePath == "" { -+ if goenv.stdlibCache != "" { -+ cachePath = goenv.stdlibCache -+ } else { -+ cachePath = filepath.Join(goenv.goroot, ".gocache") -+ } -+ } -+ cachePath = abs(cachePath) -+ if err := os.MkdirAll(cachePath, 0o755); err != nil { ++ cachePath, err := writableStdlibCacheRoot(goenv, baseEnv) ++ if err != nil { + return nil, fmt.Errorf("prepare stdlib export cache: %w", err) + } ++ if err := seedWovenStdlibCache(goenv, cachePath); err != nil { ++ return nil, fmt.Errorf("seed stdlib export cache: %w", err) ++ } + baseEnv = setEnv(baseEnv, "GOCACHE", cachePath) ++ if goenv.stdlibCache != "" { ++ baseEnv = setEnv(baseEnv, orchestrionStdlibCacheEnvVar, goenv.stdlibCache) ++ } + if getEnv(baseEnv, "HOME") == "" { + homePath := filepath.Join(goenv.goroot, ".home") + if err := os.MkdirAll(homePath, 0o755); err != nil { @@ -9327,9 +9315,18 @@ index c25763a..3f0e203 100644 +} + +func resolveCacheStdlibExports(goenv *env, packages []string) (map[string]string, error) { -+ cacheRoot := "" -+ if goenv != nil { -+ cacheRoot = goenv.stdlibCache ++ if goenv != nil && goenv.stdlibCache != "" { ++ exports, err := readStdlibCacheManifest(goenv.stdlibCache, packages) ++ if err != nil { ++ return nil, err ++ } ++ if len(exports) > 0 { ++ return exports, nil ++ } ++ } ++ cacheRoot, err := writableStdlibCacheRoot(goenv, os.Environ()) ++ if err != nil { ++ return nil, err + } + return resolveCacheStdlibExportsAt(goenv, packages, cacheRoot) +} @@ -9358,6 +9355,9 @@ index c25763a..3f0e203 100644 + return manifestExports, nil + } + } ++ if goenv.stdlibCache != "" && sameFilePath(cacheRoot, goenv.stdlibCache) { ++ return nil, fmt.Errorf("refusing to use declared stdlib cache %s as writable GOCACHE", abs(goenv.stdlibCache)) ++ } + goenv.goroot = abs(goenv.goroot) + goenv.sdk = abs(goenv.sdk) + if info, err := os.Stat(goenv.goroot); err != nil || !info.IsDir() { @@ -9441,6 +9441,24 @@ index c25763a..3f0e203 100644 + return env +} + ++func writableStdlibCacheRoot(goenv *env, environ []string) (string, error) { ++ cacheRoot := strings.TrimSpace(getEnv(environ, "GOCACHE")) ++ if cacheRoot == "" { ++ if goenv == nil || strings.TrimSpace(goenv.goroot) == "" { ++ return "", fmt.Errorf("writable stdlib GOCACHE is empty") ++ } ++ cacheRoot = filepath.Join(abs(goenv.goroot), ".gocache") ++ } ++ cacheRoot = abs(cacheRoot) ++ if goenv != nil && goenv.stdlibCache != "" && sameFilePath(cacheRoot, goenv.stdlibCache) { ++ return "", fmt.Errorf("writable stdlib GOCACHE %s aliases declared cache", cacheRoot) ++ } ++ if err := os.MkdirAll(cacheRoot, 0o755); err != nil { ++ return "", err ++ } ++ return cacheRoot, nil ++} ++ +func readStdlibCacheManifest(cacheRoot string, packages []string) (map[string]string, error) { + if cacheRoot == "" { + return nil, nil @@ -9835,18 +9853,17 @@ index c25763a..3f0e203 100644 + cmd.Env = setEnv(cmd.Env, "GOROOT", goenv.goroot) + cmd.Env = setEnv(cmd.Env, "GO111MODULE", "on") + cmd.Env = setEnv(cmd.Env, "GOWORK", "off") -+ cachePath := goenv.stdlibCache -+ if cachePath == "" { -+ cachePath = getEnv(cmd.Env, "GOCACHE") -+ } -+ if cachePath == "" { -+ cachePath = filepath.Join(goenv.goroot, ".gocache") -+ } -+ cachePath = abs(cachePath) -+ if err := os.MkdirAll(cachePath, 0o755); err != nil { ++ cachePath, err := writableStdlibCacheRoot(goenv, cmd.Env) ++ if err != nil { + return nil, fmt.Errorf("prepare stdlib closure cache: %w", err) + } ++ if err := seedWovenStdlibCache(goenv, cachePath); err != nil { ++ return nil, fmt.Errorf("seed stdlib closure cache: %w", err) ++ } + cmd.Env = setEnv(cmd.Env, "GOCACHE", cachePath) ++ if goenv.stdlibCache != "" { ++ cmd.Env = setEnv(cmd.Env, orchestrionStdlibCacheEnvVar, goenv.stdlibCache) ++ } + normalizedEnv, err := normalizeGoActionCacheEnv(cmd.Env) + if err != nil { + return nil, fmt.Errorf("prepare stdlib closure action cache env: %w", err) @@ -9924,10 +9941,10 @@ index c25763a..3f0e203 100644 known []string diff --git a/go/tools/builders/importcfg_test.go b/go/tools/builders/importcfg_test.go new file mode 100644 -index 0000000..3724594 +index 0000000..35a1fa4 --- /dev/null +++ b/go/tools/builders/importcfg_test.go -@@ -0,0 +1,816 @@ +@@ -0,0 +1,857 @@ +package main + +import ( @@ -10493,6 +10510,47 @@ index 0000000..3724594 + } +} + ++func TestWritableStdlibCacheRootUsesPrivateCache(t *testing.T) { ++ privateCache := filepath.Join(t.TempDir(), "private") ++ declaredCache := filepath.Join(t.TempDir(), "declared") ++ cacheRoot, err := writableStdlibCacheRoot(&env{ ++ goroot: t.TempDir(), ++ stdlibCache: declaredCache, ++ }, []string{"GOCACHE=" + privateCache}) ++ if err != nil { ++ t.Fatalf("writableStdlibCacheRoot error: %v", err) ++ } ++ if cacheRoot != abs(privateCache) { ++ t.Fatalf("writable cache = %q, want %q", cacheRoot, abs(privateCache)) ++ } ++ if info, err := os.Stat(cacheRoot); err != nil || !info.IsDir() { ++ t.Fatalf("writable cache was not created: %v", err) ++ } ++} ++ ++func TestWritableStdlibCacheRootRejectsDeclaredCache(t *testing.T) { ++ declaredCache := filepath.Join(t.TempDir(), "declared") ++ _, err := writableStdlibCacheRoot(&env{ ++ goroot: t.TempDir(), ++ stdlibCache: declaredCache, ++ }, []string{"GOCACHE=" + declaredCache}) ++ if err == nil || !strings.Contains(err.Error(), "aliases declared cache") { ++ t.Fatalf("writableStdlibCacheRoot error = %v", err) ++ } ++} ++ ++func TestResolveCacheStdlibExportsRejectsDeclaredCacheAsWritable(t *testing.T) { ++ declaredCache := t.TempDir() ++ _, err := resolveCacheStdlibExportsAt(&env{ ++ goroot: "fake-goroot", ++ sdk: "fake-sdk", ++ stdlibCache: declaredCache, ++ }, []string{"fmt"}, declaredCache) ++ if err == nil || !strings.Contains(err.Error(), "refusing to use declared stdlib cache") { ++ t.Fatalf("resolveCacheStdlibExportsAt error = %v", err) ++ } ++} ++ +func TestModuleExportCacheManifestRoundTrip(t *testing.T) { + root := t.TempDir() + paths := orchestrionCachePaths(root, "module-exports", "abc123") @@ -15661,7 +15719,7 @@ index 0000000..9edae16 + } +} diff --git a/go/tools/builders/stdlib.go b/go/tools/builders/stdlib.go -index cfcb991..08b2992 100644 +index cfcb991..9b5c36c 100644 --- a/go/tools/builders/stdlib.go +++ b/go/tools/builders/stdlib.go @@ -21,15 +21,29 @@ import ( @@ -15710,7 +15768,7 @@ index cfcb991..08b2992 100644 goroot := os.Getenv("GOROOT") if goroot == "" { return fmt.Errorf("GOROOT not set") -@@ -58,27 +80,40 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) +@@ -58,27 +80,35 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) } // Link in the bare minimum needed to the new GOROOT @@ -15731,21 +15789,19 @@ index cfcb991..08b2992 100644 os.Setenv("GOROOT", output) + goenv.goroot = output - // Create a temporary cache directory. "go build" requires this starting - // in Go 1.12. - cachePath := filepath.Join(output, ".gocache") -+ if *cacheOut != "" { -+ cachePath = abs(*cacheOut) -+ goenv.stdlibCache = cachePath +- // Create a temporary cache directory. "go build" requires this starting +- // in Go 1.12. +- cachePath := filepath.Join(output, ".gocache") ++ // Live Go commands use an action-private scratch cache. The separately ++ // declared cache output is populated only with deterministic woven archives. ++ cachePath, declaredCache, cleanupCache, err := prepareStdlibCaches(output, *cacheOut) ++ if err != nil { ++ return err + } ++ defer cleanupCache() ++ goenv.stdlibCache = declaredCache os.Setenv("GOCACHE", cachePath) - defer os.RemoveAll(cachePath) -+ if err := os.MkdirAll(cachePath, 0o755); err != nil { -+ return fmt.Errorf("prepare stdlib gocache at %s: %w", cachePath, err) -+ } -+ if shouldRemoveStdlibCache(*orchestrion, *cacheOut) { -+ defer os.RemoveAll(cachePath) -+ } - // Disable modules for the 'go install' command. Depending on the sandboxing - // mode, there may be a go.mod file in a parent directory which will turn @@ -15756,7 +15812,7 @@ index cfcb991..08b2992 100644 os.Setenv("GO111MODULE", "off") // Make sure we have an absolute path to the C compiler. -@@ -89,6 +124,19 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) +@@ -89,6 +119,19 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) for _, path := range filepath.SplitList(os.Getenv("PATH")) { absPaths = append(absPaths, abs(path)) } @@ -15776,7 +15832,7 @@ index cfcb991..08b2992 100644 os.Setenv("PATH", strings.Join(absPaths, string(os.PathListSeparator))) sandboxPath := abs(".") -@@ -113,7 +161,7 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) +@@ -113,7 +156,7 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) } } os.Setenv("CGO_LDFLAGS_ALLOW", b.String()) @@ -15785,7 +15841,7 @@ index cfcb991..08b2992 100644 // Build the commands needed to build the std library in the right mode // NOTE: the go command stamps compiled .a files with build ids, which are -@@ -121,7 +169,112 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) +@@ -121,7 +164,112 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) // creating reproducible builds because the build ids are hashed from // CGO_CFLAGS, which frequently contains absolute paths. As a workaround, // we strip the build ids, since they won't be used after this. @@ -15899,7 +15955,7 @@ index cfcb991..08b2992 100644 if len(build.Default.BuildTags) > 0 { installArgs = append(installArgs, "-tags", strings.Join(build.Default.BuildTags, ",")) } -@@ -135,7 +288,7 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) +@@ -135,7 +283,7 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) installArgs = append(installArgs, "-msan") } if *pgoprofile != "" { @@ -15908,7 +15964,7 @@ index cfcb991..08b2992 100644 } if *shared { gcflags = append(gcflags, "-shared") -@@ -167,8 +320,377 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) +@@ -167,8 +315,388 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) } installArgs = append(installArgs, packages...) @@ -15948,12 +16004,33 @@ index cfcb991..08b2992 100644 return nil } + -+// shouldRemoveStdlibCache reports whether the builder owns the cache directory -+// as scratch space. A non-empty -cacheout value is a Bazel-declared TreeArtifact -+// output, so it must remain present even for plain non-Orchestrion stdlib -+// actions. -+func shouldRemoveStdlibCache(orchestrionPath, cacheOut string) bool { -+ return strings.TrimSpace(orchestrionPath) == "" && strings.TrimSpace(cacheOut) == "" ++func prepareStdlibCaches(output, cacheOut string) (scratchCache, declaredCache string, cleanup func(), err error) { ++ scratchCache = filepath.Join(abs(output), ".gocache") ++ cleanup = func() { ++ _ = os.RemoveAll(scratchCache) ++ } ++ if err := os.MkdirAll(scratchCache, 0o755); err != nil { ++ return "", "", func() {}, fmt.Errorf("prepare stdlib scratch gocache at %s: %w", scratchCache, err) ++ } ++ if strings.TrimSpace(cacheOut) == "" { ++ return scratchCache, "", cleanup, nil ++ } ++ ++ declaredCache = abs(cacheOut) ++ if sameFilePath(scratchCache, declaredCache) { ++ cleanup() ++ return "", "", func() {}, fmt.Errorf("stdlib scratch gocache aliases declared cache output at %s", declaredCache) ++ } ++ if err := os.MkdirAll(declaredCache, 0o755); err != nil { ++ cleanup() ++ return "", "", func() {}, fmt.Errorf("prepare declared stdlib cache output at %s: %w", declaredCache, err) ++ } ++ return scratchCache, declaredCache, cleanup, nil ++} ++ ++func sameFilePath(left, right string) bool { ++ rel, err := filepath.Rel(abs(left), abs(right)) ++ return err == nil && rel == "." +} + +func mergeGoDebugSetting(existing, setting string) string { @@ -16089,111 +16166,101 @@ index cfcb991..08b2992 100644 + defer func() { + span.End(err) + }() -+ if goenv == nil || len(exports) == 0 { ++ if goenv == nil || len(exports) == 0 || len(roots) == 0 || strings.TrimSpace(goenv.stdlibCache) == "" { + return nil + } + -+ // We have two cache families to keep consistent: -+ // 1. the Bazel-declared stdlib cache consumed by later compile/link actions -+ // 2. the shared Datadog/Orchestrion cache used by internal `go list -export` -+ // dependency resolution when woven deps are injected. -+ // -+ // If only one is populated, compile and link can observe different archive -+ // fingerprints for stdlib packages like log/fmt/flag. Populate both from the -+ // same woven persisted exports. -+ candidateCaches := []string{} -+ seenCaches := map[string]struct{}{} -+ addCache := func(path string) { -+ path = strings.TrimSpace(path) -+ if path == "" { -+ return -+ } -+ path = abs(path) -+ if _, ok := seenCaches[path]; ok { -+ return -+ } -+ seenCaches[path] = struct{}{} -+ candidateCaches = append(candidateCaches, path) ++ scratchCache := strings.TrimSpace(os.Getenv("GOCACHE")) ++ if scratchCache == "" { ++ return fmt.Errorf("publish stdlib cache exports: GOCACHE scratch path is empty") + } -+ -+ addCache(goenv.stdlibCache) -+ if envCache := strings.TrimSpace(os.Getenv("GOCACHE")); envCache != "" { -+ addCache(envCache) ++ scratchCache = abs(scratchCache) ++ declaredCache := abs(goenv.stdlibCache) ++ if sameFilePath(scratchCache, declaredCache) { ++ return fmt.Errorf("publish stdlib cache exports: scratch cache %s aliases declared cache %s", scratchCache, declaredCache) + } -+ if len(candidateCaches) == 0 { -+ addCache(filepath.Join(abs(goenv.goroot), ".gocache")) ++ if err := os.MkdirAll(declaredCache, 0o755); err != nil { ++ return fmt.Errorf("prepare declared stdlib cache exports at %s: %w", declaredCache, err) + } ++ if verbose { ++ fmt.Fprintf(os.Stderr, "stdlib: resolving cache exports against scratch GOCACHE=%s\n", scratchCache) ++ } ++ cacheExports, err := resolveCacheStdlibExportsAt(goenv, roots, scratchCache) ++ if err != nil { ++ return err ++ } ++ return publishPersistedOrchestrionExportsToCache(exports, cacheExports, scratchCache, declaredCache, verbose) ++} + -+ prevCachePath, hadPrevCachePath := os.LookupEnv("GOCACHE") -+ defer func() { -+ if hadPrevCachePath { -+ _ = os.Setenv("GOCACHE", prevCachePath) -+ } else { -+ _ = os.Unsetenv("GOCACHE") -+ } -+ }() -+ -+ for _, cachePath := range candidateCaches { -+ if err := os.MkdirAll(cachePath, 0o755); err != nil { -+ return fmt.Errorf("prepare stdlib cache exports at %s: %w", cachePath, err) -+ } -+ if err := os.Setenv("GOCACHE", cachePath); err != nil { -+ return fmt.Errorf("set stdlib cache exports path %s: %w", cachePath, err) -+ } -+ if verbose { -+ fmt.Fprintf(os.Stderr, "stdlib: resolving cache-family exports against GOCACHE=%s\n", cachePath) ++func publishPersistedOrchestrionExportsToCache(exports, cacheExports map[string]string, scratchCache, declaredCache string, verbose bool) error { ++ manifestPath := filepath.Join(abs(declaredCache), orchestrionStdlibCacheManifestName) ++ if err := os.Remove(manifestPath); err != nil && !os.IsNotExist(err) { ++ return fmt.Errorf("remove stale stdlib cache manifest at %s: %w", manifestPath, err) ++ } ++ packages := make([]string, 0, len(cacheExports)) ++ for pkg := range cacheExports { ++ packages = append(packages, pkg) ++ } ++ sort.Strings(packages) ++ var manifest strings.Builder ++ for _, pkg := range packages { ++ src := strings.TrimSpace(exports[pkg]) ++ if src == "" { ++ return fmt.Errorf("missing persisted stdlib archive for cache package %s", pkg) + } -+ -+ cacheExports, err := resolveCacheStdlibExportsAt(goenv, roots, cachePath) ++ dst, relDst, err := projectStdlibCacheArchive(scratchCache, declaredCache, cacheExports[pkg]) + if err != nil { ++ return fmt.Errorf("project stdlib cache archive for %s: %w", pkg, err) ++ } ++ if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { + return err + } -+ packages := make([]string, 0, len(cacheExports)) -+ for pkg := range cacheExports { -+ packages = append(packages, pkg) ++ if err := copyArchiveFile(src, dst); err != nil { ++ return fmt.Errorf("copy persisted stdlib archive %s -> declared cache %s: %w", src, dst, err) + } -+ sort.Strings(packages) -+ var manifest strings.Builder -+ for _, pkg := range packages { -+ src := exports[pkg] -+ dst, ok := cacheExports[pkg] -+ if !ok || dst == "" { -+ continue -+ } -+ if strings.TrimSpace(src) == "" { -+ return fmt.Errorf("missing persisted stdlib archive for cache package %s", pkg) -+ } -+ if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { -+ return err -+ } -+ if err := copyArchiveFile(src, dst); err != nil { -+ return fmt.Errorf("copy persisted stdlib archive %s -> cache %s: %w", src, dst, err) -+ } -+ if verbose { -+ fmt.Fprintf(os.Stderr, "stdlib: synced persisted orchestrion export %s -> cache %s\n", src, dst) -+ } -+ relDst := dst -+ if rel, err := filepath.Rel(cachePath, dst); err == nil { -+ relDst = rel -+ } -+ manifest.WriteString(pkg) -+ manifest.WriteString("=") -+ manifest.WriteString(relDst) -+ manifest.WriteString("\n") -+ } -+ if manifest.Len() > 0 { -+ manifestPath := filepath.Join(cachePath, orchestrionStdlibCacheManifestName) -+ if err := os.WriteFile(manifestPath, []byte(manifest.String()), 0o644); err != nil { -+ return fmt.Errorf("write stdlib cache manifest at %s: %w", manifestPath, err) -+ } -+ if verbose { -+ fmt.Fprintf(os.Stderr, "stdlib: wrote stdlib cache manifest %s\n", manifestPath) -+ } ++ manifest.WriteString(pkg) ++ manifest.WriteString("=") ++ manifest.WriteString(relDst) ++ manifest.WriteString("\n") ++ if verbose { ++ fmt.Fprintf(os.Stderr, "stdlib: published persisted orchestrion export %s -> declared cache %s\n", src, dst) + } + } ++ if manifest.Len() == 0 { ++ return nil ++ } ++ if err := writeFileAtomically(manifestPath, []byte(manifest.String()), 0o644); err != nil { ++ return fmt.Errorf("write stdlib cache manifest at %s: %w", manifestPath, err) ++ } ++ if verbose { ++ fmt.Fprintf(os.Stderr, "stdlib: wrote stdlib cache manifest %s\n", manifestPath) ++ } + return nil +} + ++func projectStdlibCacheArchive(scratchCache, declaredCache, scratchArchive string) (destination, relative string, err error) { ++ scratchRoot := abs(scratchCache) ++ declaredRoot := abs(declaredCache) ++ archive := abs(scratchArchive) ++ rel, err := filepath.Rel(scratchRoot, archive) ++ if err != nil { ++ return "", "", err ++ } ++ rel = filepath.Clean(rel) ++ if rel == "." || filepath.IsAbs(rel) || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { ++ return "", "", fmt.Errorf("cache archive %s escapes scratch root %s", archive, scratchRoot) ++ } ++ if !strings.HasSuffix(filepath.Base(rel), "-d") { ++ return "", "", fmt.Errorf("cache archive %s is not a Go cache data entry", archive) ++ } ++ destination = filepath.Join(declaredRoot, rel) ++ destinationRel, err := filepath.Rel(declaredRoot, destination) ++ if err != nil || filepath.Clean(destinationRel) != rel { ++ return "", "", fmt.Errorf("cache archive destination %s escapes declared root %s", destination, declaredRoot) ++ } ++ return destination, filepath.ToSlash(rel), nil ++} ++ +func ensureSyntheticOrchestrionToolGo(verbose bool, orchestrionMode string) (func(), error) { + const toolFile = "orchestrion.tool.go" + if _, err := os.Stat(toolFile); err == nil { @@ -16288,15 +16355,19 @@ index cfcb991..08b2992 100644 +} diff --git a/go/tools/builders/stdlib_test.go b/go/tools/builders/stdlib_test.go new file mode 100644 -index 0000000..b92f214 +index 0000000..7ca400c --- /dev/null +++ b/go/tools/builders/stdlib_test.go -@@ -0,0 +1,240 @@ +@@ -0,0 +1,464 @@ +package main + +import ( ++ "bytes" ++ "crypto/sha256" ++ "fmt" + "os" + "path/filepath" ++ "reflect" + "strings" + "testing" +) @@ -16410,48 +16481,223 @@ index 0000000..b92f214 + } +} + -+func TestShouldRemoveStdlibCache(t *testing.T) { -+ t.Parallel() ++func TestPrepareStdlibCachesWithoutDeclaredOutput(t *testing.T) { ++ output := filepath.Join(t.TempDir(), "stdlib") ++ scratch, declared, cleanup, err := prepareStdlibCaches(output, "") ++ if err != nil { ++ t.Fatalf("prepareStdlibCaches error: %v", err) ++ } ++ if want := filepath.Join(abs(output), ".gocache"); scratch != want { ++ t.Fatalf("scratch cache = %q, want %q", scratch, want) ++ } ++ if declared != "" { ++ t.Fatalf("declared cache = %q, want empty", declared) ++ } ++ if info, err := os.Stat(scratch); err != nil || !info.IsDir() { ++ t.Fatalf("scratch cache was not created: %v", err) ++ } ++ cleanup() ++ if _, err := os.Stat(scratch); !os.IsNotExist(err) { ++ t.Fatalf("scratch cache still exists after cleanup: %v", err) ++ } ++} ++ ++func TestPrepareStdlibCachesSeparatesDeclaredOutput(t *testing.T) { ++ root := t.TempDir() ++ output := filepath.Join(root, "stdlib") ++ cacheOut := filepath.Join(root, "declared") ++ scratch, declared, cleanup, err := prepareStdlibCaches(output, cacheOut) ++ if err != nil { ++ t.Fatalf("prepareStdlibCaches error: %v", err) ++ } ++ if sameFilePath(scratch, declared) { ++ t.Fatalf("scratch cache %q aliases declared cache %q", scratch, declared) ++ } ++ for name, path := range map[string]string{"scratch": scratch, "declared": declared} { ++ if info, err := os.Stat(path); err != nil || !info.IsDir() { ++ t.Fatalf("%s cache was not created: %v", name, err) ++ } ++ } ++ cleanup() ++ if _, err := os.Stat(scratch); !os.IsNotExist(err) { ++ t.Fatalf("scratch cache still exists after cleanup: %v", err) ++ } ++ if info, err := os.Stat(declared); err != nil || !info.IsDir() { ++ t.Fatalf("declared cache should remain after scratch cleanup: %v", err) ++ } ++} ++ ++func TestPrepareStdlibCachesNormalizesRelativeInputs(t *testing.T) { ++ root := t.TempDir() ++ previousWD, err := os.Getwd() ++ if err != nil { ++ t.Fatalf("getwd: %v", err) ++ } ++ if err := os.Chdir(root); err != nil { ++ t.Fatalf("chdir: %v", err) ++ } ++ defer func() { _ = os.Chdir(previousWD) }() ++ normalizedRoot, err := os.Getwd() ++ if err != nil { ++ t.Fatalf("get normalized workdir: %v", err) ++ } ++ ++ scratch, declared, cleanup, err := prepareStdlibCaches("stdlib", "declared") ++ if err != nil { ++ t.Fatalf("prepareStdlibCaches error: %v", err) ++ } ++ defer cleanup() ++ if !filepath.IsAbs(scratch) || !filepath.IsAbs(declared) { ++ t.Fatalf("cache paths are not absolute: scratch=%q declared=%q", scratch, declared) ++ } ++ if scratch != filepath.Join(normalizedRoot, "stdlib", ".gocache") { ++ t.Fatalf("scratch cache = %q", scratch) ++ } ++ if declared != filepath.Join(normalizedRoot, "declared") { ++ t.Fatalf("declared cache = %q", declared) ++ } ++} ++ ++func TestPrepareStdlibCachesRejectsAliasedOutput(t *testing.T) { ++ output := filepath.Join(t.TempDir(), "stdlib") ++ alias := filepath.Join(output, ".gocache") ++ if _, _, _, err := prepareStdlibCaches(output, alias); err == nil || !strings.Contains(err.Error(), "aliases declared cache") { ++ t.Fatalf("prepareStdlibCaches alias error = %v", err) ++ } ++ if _, err := os.Stat(alias); !os.IsNotExist(err) { ++ t.Fatalf("aliased scratch cache still exists after failure: %v", err) ++ } ++} ++ ++func TestNewBufferedCommandPreservesWritableGoCache(t *testing.T) { ++ scratch := filepath.Join(t.TempDir(), "scratch") ++ declared := filepath.Join(t.TempDir(), "declared") ++ if err := os.MkdirAll(declared, 0o755); err != nil { ++ t.Fatal(err) ++ } ++ t.Setenv("GOCACHE", scratch) ++ ++ cmd := (&env{stdlibCache: declared}).newBufferedCommand([]string{"go", "version"}, &bytes.Buffer{}) ++ if got := getEnv(cmd.Env, "GOCACHE"); got != scratch { ++ t.Fatalf("GOCACHE = %q, want writable cache %q", got, scratch) ++ } ++ if got := getEnv(cmd.Env, orchestrionStdlibCacheEnvVar); got != declared { ++ t.Fatalf("%s = %q, want %q", orchestrionStdlibCacheEnvVar, got, declared) ++ } ++ ++ for _, invalid := range []string{"", filepath.Join(t.TempDir(), "missing")} { ++ cmd = (&env{stdlibCache: invalid}).newBufferedCommand([]string{"go", "version"}, &bytes.Buffer{}) ++ if got := getEnv(cmd.Env, "GOCACHE"); got != scratch { ++ t.Fatalf("invalid declared cache %q replaced GOCACHE with %q", invalid, got) ++ } ++ if got := getEnv(cmd.Env, orchestrionStdlibCacheEnvVar); got != "" { ++ t.Fatalf("invalid declared cache %q set %s=%q", invalid, orchestrionStdlibCacheEnvVar, got) ++ } ++ } ++} ++ ++func TestPublishPersistedOrchestrionExportsIsDeterministic(t *testing.T) { ++ archives := map[string][]byte{ ++ "fmt": []byte("woven fmt archive"), ++ "log": []byte("woven log archive"), ++ } ++ cachePaths := map[string]string{ ++ "fmt": filepath.Join("11", "fmt-d"), ++ "log": filepath.Join("aa", "log-d"), ++ } + ++ inventories := make([]map[string]string, 0, 2) ++ for run := 0; run < 2; run++ { ++ scratch := filepath.Join(t.TempDir(), "scratch") ++ declared := filepath.Join(t.TempDir(), "declared") ++ persisted := filepath.Join(t.TempDir(), "persisted") ++ if err := os.MkdirAll(scratch, 0o755); err != nil { ++ t.Fatal(err) ++ } ++ exports := make(map[string]string, len(archives)) ++ resolved := make(map[string]string, len(cachePaths)) ++ for pkg, data := range archives { ++ src := filepath.Join(persisted, pkg+".a") ++ writeTestFile(t, src, data) ++ exports[pkg] = src ++ resolved[pkg] = filepath.Join(scratch, cachePaths[pkg]) ++ writeTestFile(t, resolved[pkg], []byte(fmt.Sprintf("unwoven run %d", run))) ++ } ++ writeTestFile(t, filepath.Join(scratch, "11", "fmt-a"), []byte(fmt.Sprintf("timestamp %d", run))) ++ writeTestFile(t, filepath.Join(scratch, "trim.txt"), []byte(fmt.Sprintf("trim %d", run))) ++ writeTestFile(t, filepath.Join(scratch, "unrelated", "entry-d"), []byte(fmt.Sprintf("unrelated %d", run))) ++ ++ if err := publishPersistedOrchestrionExportsToCache(exports, resolved, scratch, declared, false); err != nil { ++ t.Fatalf("publish run %d: %v", run, err) ++ } ++ inventory := testTreeInventory(t, declared) ++ inventories = append(inventories, inventory) ++ if len(inventory) != len(archives)+1 { ++ t.Fatalf("declared inventory has unexpected entries: %v", inventory) ++ } ++ for pkg, data := range archives { ++ path := filepath.ToSlash(cachePaths[pkg]) ++ if got := inventory[path]; got != testDigest(data) { ++ t.Fatalf("published %s digest = %q, want %q", path, got, testDigest(data)) ++ } ++ } ++ manifest, err := os.ReadFile(filepath.Join(declared, orchestrionStdlibCacheManifestName)) ++ if err != nil { ++ t.Fatal(err) ++ } ++ wantManifest := "fmt=11/fmt-d\nlog=aa/log-d\n" ++ if string(manifest) != wantManifest { ++ t.Fatalf("manifest = %q, want %q", string(manifest), wantManifest) ++ } ++ } ++ if !reflect.DeepEqual(inventories[0], inventories[1]) { ++ t.Fatalf("declared cache inventories differ:\nrun 1: %v\nrun 2: %v", inventories[0], inventories[1]) ++ } ++} ++ ++func TestProjectStdlibCacheArchiveRejectsInvalidPaths(t *testing.T) { ++ root := t.TempDir() ++ scratch := filepath.Join(root, "scratch") ++ declared := filepath.Join(root, "declared") + tests := []struct { -+ name string -+ orchestrionPath string -+ cacheOut string -+ want bool ++ name string ++ archive string ++ want string + }{ -+ { -+ name: "plain internal cache", -+ want: true, -+ }, -+ { -+ name: "plain declared cache output", -+ cacheOut: "bazel-out/bin/external/rules_go/stdlib_/gocache", -+ want: false, -+ }, -+ { -+ name: "orchestrion internal cache", -+ orchestrionPath: "external/rules_go_orchestrion_tool/orchestrion", -+ want: false, -+ }, -+ { -+ name: "orchestrion declared cache output", -+ orchestrionPath: "external/rules_go_orchestrion_tool/orchestrion", -+ cacheOut: "bazel-out/bin/external/rules_go/stdlib_/gocache", -+ want: false, -+ }, ++ {name: "outside scratch", archive: filepath.Join(root, "outside", "entry-d"), want: "escapes scratch root"}, ++ {name: "action index", archive: filepath.Join(scratch, "aa", "entry-a"), want: "not a Go cache data entry"}, + } -+ + for _, tt := range tests { -+ tt := tt + t.Run(tt.name, func(t *testing.T) { -+ t.Parallel() -+ if got := shouldRemoveStdlibCache(tt.orchestrionPath, tt.cacheOut); got != tt.want { -+ t.Fatalf("shouldRemoveStdlibCache(%q, %q) = %v, want %v", tt.orchestrionPath, tt.cacheOut, got, tt.want) ++ if _, _, err := projectStdlibCacheArchive(scratch, declared, tt.archive); err == nil || !strings.Contains(err.Error(), tt.want) { ++ t.Fatalf("projectStdlibCacheArchive error = %v, want %q", err, tt.want) + } + }) + } +} + ++func TestPublishPersistedOrchestrionExportsFailureLeavesNoManifest(t *testing.T) { ++ root := t.TempDir() ++ scratch := filepath.Join(root, "scratch") ++ declared := filepath.Join(root, "declared") ++ manifest := filepath.Join(declared, orchestrionStdlibCacheManifestName) ++ writeTestFile(t, manifest, []byte("stale=stale-d\n")) ++ ++ err := publishPersistedOrchestrionExportsToCache( ++ map[string]string{}, ++ map[string]string{"fmt": filepath.Join(scratch, "aa", "fmt-d")}, ++ scratch, ++ declared, ++ false, ++ ) ++ if err == nil || !strings.Contains(err.Error(), "missing persisted stdlib archive for cache package fmt") { ++ t.Fatalf("publish error = %v", err) ++ } ++ if _, err := os.Stat(manifest); !os.IsNotExist(err) { ++ t.Fatalf("manifest exists after failed publication: %v", err) ++ } ++} ++ +func TestMergeGoDebugSettingPreservesExistingFlags(t *testing.T) { + tests := []struct { + name string @@ -16532,6 +16778,51 @@ index 0000000..b92f214 + } + } +} ++ ++func writeTestFile(t *testing.T, path string, data []byte) { ++ t.Helper() ++ if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { ++ t.Fatalf("mkdir %s: %v", filepath.Dir(path), err) ++ } ++ if err := os.WriteFile(path, data, 0o644); err != nil { ++ t.Fatalf("write %s: %v", path, err) ++ } ++} ++ ++func testTreeInventory(t *testing.T, root string) map[string]string { ++ t.Helper() ++ inventory := make(map[string]string) ++ err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { ++ if err != nil { ++ return err ++ } ++ if path == root || entry.IsDir() { ++ return nil ++ } ++ if entry.Type()&os.ModeSymlink != 0 { ++ return fmt.Errorf("unexpected symlink %s", path) ++ } ++ data, err := os.ReadFile(path) ++ if err != nil { ++ return err ++ } ++ rel, err := filepath.Rel(root, path) ++ if err != nil { ++ return err ++ } ++ inventory[filepath.ToSlash(rel)] = testDigest(data) ++ return nil ++ }) ++ if err != nil { ++ t.Fatalf("inventory %s: %v", root, err) ++ } ++ return inventory ++} ++ ++func testDigest(data []byte) string { ++ sum := sha256.Sum256(data) ++ return fmt.Sprintf("%x", sum[:]) ++} diff --git a/go/tools/builders/stdliblist.go b/go/tools/builders/stdliblist.go index adbbe4a..4404c30 100644 --- a/go/tools/builders/stdliblist.go diff --git a/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch index 7aa6cecf..1f265f3b 100644 --- a/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch @@ -7594,7 +7594,7 @@ index 2bea17e..154b543 100644 if err := cmd.Run(); err != nil { diff --git a/go/tools/builders/env_orchestrion.go b/go/tools/builders/env_orchestrion.go new file mode 100644 -index 0000000..fbc392c +index 0000000..73c7992 --- /dev/null +++ b/go/tools/builders/env_orchestrion.go @@ -0,0 +1,81 @@ @@ -7659,7 +7659,8 @@ index 0000000..fbc392c +} + +// newBufferedCommand creates a subprocess command wired to the shared builder -+// buffer and applies the stdlib cache override needed by Orchestrion actions. ++// buffer and advertises the declared stdlib archive source without replacing ++// the subprocess's writable GOCACHE. +func (e *env) newBufferedCommand(args []string, buf *bytes.Buffer) *exec.Cmd { + cmd := exec.Command(args[0], args[1:]...) + cmd.Stdout = buf @@ -7667,7 +7668,6 @@ index 0000000..fbc392c + cmd.Env = os.Environ() + if e.stdlibCache != "" { + if info, err := os.Stat(e.stdlibCache); err == nil && info.IsDir() { -+ cmd.Env = setEnv(cmd.Env, "GOCACHE", e.stdlibCache) + cmd.Env = setEnv(cmd.Env, orchestrionStdlibCacheEnvVar, e.stdlibCache) + } + } @@ -8135,7 +8135,7 @@ index 893a0f6..16955a7 100644 + return "" +} diff --git a/go/tools/builders/importcfg.go b/go/tools/builders/importcfg.go -index c25763a..3f0e203 100644 +index c25763a..4c859ae 100644 --- a/go/tools/builders/importcfg.go +++ b/go/tools/builders/importcfg.go @@ -17,21 +17,94 @@ package main @@ -8233,7 +8233,7 @@ index c25763a..3f0e203 100644 // checkImports verifies that each import in files refers to a // direct dependency in archives or to a standard library package // listed in the file at stdPackageListPath. checkImports returns -@@ -213,6 +286,1678 @@ package with this path is linked.`, +@@ -213,6 +286,1695 @@ package with this path is linked.`, return filename, nil } @@ -9255,29 +9255,17 @@ index c25763a..3f0e203 100644 + baseEnv = setEnv(baseEnv, "GOROOT", goenv.goroot) + baseEnv = setEnv(baseEnv, "GO111MODULE", "off") + baseEnv = setEnv(baseEnv, "GOWORK", "off") -+ orchestrionCachePath := goenv.stdlibCache -+ if orchestrionCachePath == "" { -+ orchestrionCachePath = filepath.Join(goenv.goroot, ".gocache") -+ } -+ cachePath := "" -+ if info, err := os.Stat(orchestrionCachePath); err == nil && info.IsDir() { -+ cachePath = orchestrionCachePath -+ } -+ if cachePath == "" { -+ cachePath = getEnv(baseEnv, "GOCACHE") -+ } -+ if cachePath == "" { -+ if goenv.stdlibCache != "" { -+ cachePath = goenv.stdlibCache -+ } else { -+ cachePath = filepath.Join(goenv.goroot, ".gocache") -+ } -+ } -+ cachePath = abs(cachePath) -+ if err := os.MkdirAll(cachePath, 0o755); err != nil { ++ cachePath, err := writableStdlibCacheRoot(goenv, baseEnv) ++ if err != nil { + return nil, fmt.Errorf("prepare stdlib export cache: %w", err) + } ++ if err := seedWovenStdlibCache(goenv, cachePath); err != nil { ++ return nil, fmt.Errorf("seed stdlib export cache: %w", err) ++ } + baseEnv = setEnv(baseEnv, "GOCACHE", cachePath) ++ if goenv.stdlibCache != "" { ++ baseEnv = setEnv(baseEnv, orchestrionStdlibCacheEnvVar, goenv.stdlibCache) ++ } + if getEnv(baseEnv, "HOME") == "" { + homePath := filepath.Join(goenv.goroot, ".home") + if err := os.MkdirAll(homePath, 0o755); err != nil { @@ -9317,9 +9305,18 @@ index c25763a..3f0e203 100644 +} + +func resolveCacheStdlibExports(goenv *env, packages []string) (map[string]string, error) { -+ cacheRoot := "" -+ if goenv != nil { -+ cacheRoot = goenv.stdlibCache ++ if goenv != nil && goenv.stdlibCache != "" { ++ exports, err := readStdlibCacheManifest(goenv.stdlibCache, packages) ++ if err != nil { ++ return nil, err ++ } ++ if len(exports) > 0 { ++ return exports, nil ++ } ++ } ++ cacheRoot, err := writableStdlibCacheRoot(goenv, os.Environ()) ++ if err != nil { ++ return nil, err + } + return resolveCacheStdlibExportsAt(goenv, packages, cacheRoot) +} @@ -9348,6 +9345,9 @@ index c25763a..3f0e203 100644 + return manifestExports, nil + } + } ++ if goenv.stdlibCache != "" && sameFilePath(cacheRoot, goenv.stdlibCache) { ++ return nil, fmt.Errorf("refusing to use declared stdlib cache %s as writable GOCACHE", abs(goenv.stdlibCache)) ++ } + goenv.goroot = abs(goenv.goroot) + goenv.sdk = abs(goenv.sdk) + if info, err := os.Stat(goenv.goroot); err != nil || !info.IsDir() { @@ -9431,6 +9431,24 @@ index c25763a..3f0e203 100644 + return env +} + ++func writableStdlibCacheRoot(goenv *env, environ []string) (string, error) { ++ cacheRoot := strings.TrimSpace(getEnv(environ, "GOCACHE")) ++ if cacheRoot == "" { ++ if goenv == nil || strings.TrimSpace(goenv.goroot) == "" { ++ return "", fmt.Errorf("writable stdlib GOCACHE is empty") ++ } ++ cacheRoot = filepath.Join(abs(goenv.goroot), ".gocache") ++ } ++ cacheRoot = abs(cacheRoot) ++ if goenv != nil && goenv.stdlibCache != "" && sameFilePath(cacheRoot, goenv.stdlibCache) { ++ return "", fmt.Errorf("writable stdlib GOCACHE %s aliases declared cache", cacheRoot) ++ } ++ if err := os.MkdirAll(cacheRoot, 0o755); err != nil { ++ return "", err ++ } ++ return cacheRoot, nil ++} ++ +func readStdlibCacheManifest(cacheRoot string, packages []string) (map[string]string, error) { + if cacheRoot == "" { + return nil, nil @@ -9825,18 +9843,17 @@ index c25763a..3f0e203 100644 + cmd.Env = setEnv(cmd.Env, "GOROOT", goenv.goroot) + cmd.Env = setEnv(cmd.Env, "GO111MODULE", "on") + cmd.Env = setEnv(cmd.Env, "GOWORK", "off") -+ cachePath := goenv.stdlibCache -+ if cachePath == "" { -+ cachePath = getEnv(cmd.Env, "GOCACHE") -+ } -+ if cachePath == "" { -+ cachePath = filepath.Join(goenv.goroot, ".gocache") -+ } -+ cachePath = abs(cachePath) -+ if err := os.MkdirAll(cachePath, 0o755); err != nil { ++ cachePath, err := writableStdlibCacheRoot(goenv, cmd.Env) ++ if err != nil { + return nil, fmt.Errorf("prepare stdlib closure cache: %w", err) + } ++ if err := seedWovenStdlibCache(goenv, cachePath); err != nil { ++ return nil, fmt.Errorf("seed stdlib closure cache: %w", err) ++ } + cmd.Env = setEnv(cmd.Env, "GOCACHE", cachePath) ++ if goenv.stdlibCache != "" { ++ cmd.Env = setEnv(cmd.Env, orchestrionStdlibCacheEnvVar, goenv.stdlibCache) ++ } + normalizedEnv, err := normalizeGoActionCacheEnv(cmd.Env) + if err != nil { + return nil, fmt.Errorf("prepare stdlib closure action cache env: %w", err) @@ -9914,10 +9931,10 @@ index c25763a..3f0e203 100644 known []string diff --git a/go/tools/builders/importcfg_test.go b/go/tools/builders/importcfg_test.go new file mode 100644 -index 0000000..3724594 +index 0000000..35a1fa4 --- /dev/null +++ b/go/tools/builders/importcfg_test.go -@@ -0,0 +1,816 @@ +@@ -0,0 +1,857 @@ +package main + +import ( @@ -10483,6 +10500,47 @@ index 0000000..3724594 + } +} + ++func TestWritableStdlibCacheRootUsesPrivateCache(t *testing.T) { ++ privateCache := filepath.Join(t.TempDir(), "private") ++ declaredCache := filepath.Join(t.TempDir(), "declared") ++ cacheRoot, err := writableStdlibCacheRoot(&env{ ++ goroot: t.TempDir(), ++ stdlibCache: declaredCache, ++ }, []string{"GOCACHE=" + privateCache}) ++ if err != nil { ++ t.Fatalf("writableStdlibCacheRoot error: %v", err) ++ } ++ if cacheRoot != abs(privateCache) { ++ t.Fatalf("writable cache = %q, want %q", cacheRoot, abs(privateCache)) ++ } ++ if info, err := os.Stat(cacheRoot); err != nil || !info.IsDir() { ++ t.Fatalf("writable cache was not created: %v", err) ++ } ++} ++ ++func TestWritableStdlibCacheRootRejectsDeclaredCache(t *testing.T) { ++ declaredCache := filepath.Join(t.TempDir(), "declared") ++ _, err := writableStdlibCacheRoot(&env{ ++ goroot: t.TempDir(), ++ stdlibCache: declaredCache, ++ }, []string{"GOCACHE=" + declaredCache}) ++ if err == nil || !strings.Contains(err.Error(), "aliases declared cache") { ++ t.Fatalf("writableStdlibCacheRoot error = %v", err) ++ } ++} ++ ++func TestResolveCacheStdlibExportsRejectsDeclaredCacheAsWritable(t *testing.T) { ++ declaredCache := t.TempDir() ++ _, err := resolveCacheStdlibExportsAt(&env{ ++ goroot: "fake-goroot", ++ sdk: "fake-sdk", ++ stdlibCache: declaredCache, ++ }, []string{"fmt"}, declaredCache) ++ if err == nil || !strings.Contains(err.Error(), "refusing to use declared stdlib cache") { ++ t.Fatalf("resolveCacheStdlibExportsAt error = %v", err) ++ } ++} ++ +func TestModuleExportCacheManifestRoundTrip(t *testing.T) { + root := t.TempDir() + paths := orchestrionCachePaths(root, "module-exports", "abc123") @@ -15651,7 +15709,7 @@ index 0000000..9edae16 + } +} diff --git a/go/tools/builders/stdlib.go b/go/tools/builders/stdlib.go -index cfcb991..08b2992 100644 +index cfcb991..9b5c36c 100644 --- a/go/tools/builders/stdlib.go +++ b/go/tools/builders/stdlib.go @@ -21,15 +21,29 @@ import ( @@ -15700,7 +15758,7 @@ index cfcb991..08b2992 100644 goroot := os.Getenv("GOROOT") if goroot == "" { return fmt.Errorf("GOROOT not set") -@@ -58,27 +80,40 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) +@@ -58,27 +80,35 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) } // Link in the bare minimum needed to the new GOROOT @@ -15721,21 +15779,19 @@ index cfcb991..08b2992 100644 os.Setenv("GOROOT", output) + goenv.goroot = output - // Create a temporary cache directory. "go build" requires this starting - // in Go 1.12. - cachePath := filepath.Join(output, ".gocache") -+ if *cacheOut != "" { -+ cachePath = abs(*cacheOut) -+ goenv.stdlibCache = cachePath +- // Create a temporary cache directory. "go build" requires this starting +- // in Go 1.12. +- cachePath := filepath.Join(output, ".gocache") ++ // Live Go commands use an action-private scratch cache. The separately ++ // declared cache output is populated only with deterministic woven archives. ++ cachePath, declaredCache, cleanupCache, err := prepareStdlibCaches(output, *cacheOut) ++ if err != nil { ++ return err + } ++ defer cleanupCache() ++ goenv.stdlibCache = declaredCache os.Setenv("GOCACHE", cachePath) - defer os.RemoveAll(cachePath) -+ if err := os.MkdirAll(cachePath, 0o755); err != nil { -+ return fmt.Errorf("prepare stdlib gocache at %s: %w", cachePath, err) -+ } -+ if shouldRemoveStdlibCache(*orchestrion, *cacheOut) { -+ defer os.RemoveAll(cachePath) -+ } - // Disable modules for the 'go install' command. Depending on the sandboxing - // mode, there may be a go.mod file in a parent directory which will turn @@ -15746,7 +15802,7 @@ index cfcb991..08b2992 100644 os.Setenv("GO111MODULE", "off") // Make sure we have an absolute path to the C compiler. -@@ -89,6 +124,19 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) +@@ -89,6 +119,19 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) for _, path := range filepath.SplitList(os.Getenv("PATH")) { absPaths = append(absPaths, abs(path)) } @@ -15766,7 +15822,7 @@ index cfcb991..08b2992 100644 os.Setenv("PATH", strings.Join(absPaths, string(os.PathListSeparator))) sandboxPath := abs(".") -@@ -113,7 +161,7 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) +@@ -113,7 +156,7 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) } } os.Setenv("CGO_LDFLAGS_ALLOW", b.String()) @@ -15775,7 +15831,7 @@ index cfcb991..08b2992 100644 // Build the commands needed to build the std library in the right mode // NOTE: the go command stamps compiled .a files with build ids, which are -@@ -121,7 +169,112 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) +@@ -121,7 +164,112 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) // creating reproducible builds because the build ids are hashed from // CGO_CFLAGS, which frequently contains absolute paths. As a workaround, // we strip the build ids, since they won't be used after this. @@ -15889,7 +15945,7 @@ index cfcb991..08b2992 100644 if len(build.Default.BuildTags) > 0 { installArgs = append(installArgs, "-tags", strings.Join(build.Default.BuildTags, ",")) } -@@ -135,7 +288,7 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) +@@ -135,7 +283,7 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) installArgs = append(installArgs, "-msan") } if *pgoprofile != "" { @@ -15898,7 +15954,7 @@ index cfcb991..08b2992 100644 } if *shared { gcflags = append(gcflags, "-shared") -@@ -167,8 +320,377 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) +@@ -167,8 +315,388 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) } installArgs = append(installArgs, packages...) @@ -15938,12 +15994,33 @@ index cfcb991..08b2992 100644 return nil } + -+// shouldRemoveStdlibCache reports whether the builder owns the cache directory -+// as scratch space. A non-empty -cacheout value is a Bazel-declared TreeArtifact -+// output, so it must remain present even for plain non-Orchestrion stdlib -+// actions. -+func shouldRemoveStdlibCache(orchestrionPath, cacheOut string) bool { -+ return strings.TrimSpace(orchestrionPath) == "" && strings.TrimSpace(cacheOut) == "" ++func prepareStdlibCaches(output, cacheOut string) (scratchCache, declaredCache string, cleanup func(), err error) { ++ scratchCache = filepath.Join(abs(output), ".gocache") ++ cleanup = func() { ++ _ = os.RemoveAll(scratchCache) ++ } ++ if err := os.MkdirAll(scratchCache, 0o755); err != nil { ++ return "", "", func() {}, fmt.Errorf("prepare stdlib scratch gocache at %s: %w", scratchCache, err) ++ } ++ if strings.TrimSpace(cacheOut) == "" { ++ return scratchCache, "", cleanup, nil ++ } ++ ++ declaredCache = abs(cacheOut) ++ if sameFilePath(scratchCache, declaredCache) { ++ cleanup() ++ return "", "", func() {}, fmt.Errorf("stdlib scratch gocache aliases declared cache output at %s", declaredCache) ++ } ++ if err := os.MkdirAll(declaredCache, 0o755); err != nil { ++ cleanup() ++ return "", "", func() {}, fmt.Errorf("prepare declared stdlib cache output at %s: %w", declaredCache, err) ++ } ++ return scratchCache, declaredCache, cleanup, nil ++} ++ ++func sameFilePath(left, right string) bool { ++ rel, err := filepath.Rel(abs(left), abs(right)) ++ return err == nil && rel == "." +} + +func mergeGoDebugSetting(existing, setting string) string { @@ -16079,111 +16156,101 @@ index cfcb991..08b2992 100644 + defer func() { + span.End(err) + }() -+ if goenv == nil || len(exports) == 0 { ++ if goenv == nil || len(exports) == 0 || len(roots) == 0 || strings.TrimSpace(goenv.stdlibCache) == "" { + return nil + } + -+ // We have two cache families to keep consistent: -+ // 1. the Bazel-declared stdlib cache consumed by later compile/link actions -+ // 2. the shared Datadog/Orchestrion cache used by internal `go list -export` -+ // dependency resolution when woven deps are injected. -+ // -+ // If only one is populated, compile and link can observe different archive -+ // fingerprints for stdlib packages like log/fmt/flag. Populate both from the -+ // same woven persisted exports. -+ candidateCaches := []string{} -+ seenCaches := map[string]struct{}{} -+ addCache := func(path string) { -+ path = strings.TrimSpace(path) -+ if path == "" { -+ return -+ } -+ path = abs(path) -+ if _, ok := seenCaches[path]; ok { -+ return -+ } -+ seenCaches[path] = struct{}{} -+ candidateCaches = append(candidateCaches, path) ++ scratchCache := strings.TrimSpace(os.Getenv("GOCACHE")) ++ if scratchCache == "" { ++ return fmt.Errorf("publish stdlib cache exports: GOCACHE scratch path is empty") + } -+ -+ addCache(goenv.stdlibCache) -+ if envCache := strings.TrimSpace(os.Getenv("GOCACHE")); envCache != "" { -+ addCache(envCache) ++ scratchCache = abs(scratchCache) ++ declaredCache := abs(goenv.stdlibCache) ++ if sameFilePath(scratchCache, declaredCache) { ++ return fmt.Errorf("publish stdlib cache exports: scratch cache %s aliases declared cache %s", scratchCache, declaredCache) + } -+ if len(candidateCaches) == 0 { -+ addCache(filepath.Join(abs(goenv.goroot), ".gocache")) ++ if err := os.MkdirAll(declaredCache, 0o755); err != nil { ++ return fmt.Errorf("prepare declared stdlib cache exports at %s: %w", declaredCache, err) + } ++ if verbose { ++ fmt.Fprintf(os.Stderr, "stdlib: resolving cache exports against scratch GOCACHE=%s\n", scratchCache) ++ } ++ cacheExports, err := resolveCacheStdlibExportsAt(goenv, roots, scratchCache) ++ if err != nil { ++ return err ++ } ++ return publishPersistedOrchestrionExportsToCache(exports, cacheExports, scratchCache, declaredCache, verbose) ++} + -+ prevCachePath, hadPrevCachePath := os.LookupEnv("GOCACHE") -+ defer func() { -+ if hadPrevCachePath { -+ _ = os.Setenv("GOCACHE", prevCachePath) -+ } else { -+ _ = os.Unsetenv("GOCACHE") -+ } -+ }() -+ -+ for _, cachePath := range candidateCaches { -+ if err := os.MkdirAll(cachePath, 0o755); err != nil { -+ return fmt.Errorf("prepare stdlib cache exports at %s: %w", cachePath, err) -+ } -+ if err := os.Setenv("GOCACHE", cachePath); err != nil { -+ return fmt.Errorf("set stdlib cache exports path %s: %w", cachePath, err) -+ } -+ if verbose { -+ fmt.Fprintf(os.Stderr, "stdlib: resolving cache-family exports against GOCACHE=%s\n", cachePath) ++func publishPersistedOrchestrionExportsToCache(exports, cacheExports map[string]string, scratchCache, declaredCache string, verbose bool) error { ++ manifestPath := filepath.Join(abs(declaredCache), orchestrionStdlibCacheManifestName) ++ if err := os.Remove(manifestPath); err != nil && !os.IsNotExist(err) { ++ return fmt.Errorf("remove stale stdlib cache manifest at %s: %w", manifestPath, err) ++ } ++ packages := make([]string, 0, len(cacheExports)) ++ for pkg := range cacheExports { ++ packages = append(packages, pkg) ++ } ++ sort.Strings(packages) ++ var manifest strings.Builder ++ for _, pkg := range packages { ++ src := strings.TrimSpace(exports[pkg]) ++ if src == "" { ++ return fmt.Errorf("missing persisted stdlib archive for cache package %s", pkg) + } -+ -+ cacheExports, err := resolveCacheStdlibExportsAt(goenv, roots, cachePath) ++ dst, relDst, err := projectStdlibCacheArchive(scratchCache, declaredCache, cacheExports[pkg]) + if err != nil { ++ return fmt.Errorf("project stdlib cache archive for %s: %w", pkg, err) ++ } ++ if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { + return err + } -+ packages := make([]string, 0, len(cacheExports)) -+ for pkg := range cacheExports { -+ packages = append(packages, pkg) ++ if err := copyArchiveFile(src, dst); err != nil { ++ return fmt.Errorf("copy persisted stdlib archive %s -> declared cache %s: %w", src, dst, err) + } -+ sort.Strings(packages) -+ var manifest strings.Builder -+ for _, pkg := range packages { -+ src := exports[pkg] -+ dst, ok := cacheExports[pkg] -+ if !ok || dst == "" { -+ continue -+ } -+ if strings.TrimSpace(src) == "" { -+ return fmt.Errorf("missing persisted stdlib archive for cache package %s", pkg) -+ } -+ if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { -+ return err -+ } -+ if err := copyArchiveFile(src, dst); err != nil { -+ return fmt.Errorf("copy persisted stdlib archive %s -> cache %s: %w", src, dst, err) -+ } -+ if verbose { -+ fmt.Fprintf(os.Stderr, "stdlib: synced persisted orchestrion export %s -> cache %s\n", src, dst) -+ } -+ relDst := dst -+ if rel, err := filepath.Rel(cachePath, dst); err == nil { -+ relDst = rel -+ } -+ manifest.WriteString(pkg) -+ manifest.WriteString("=") -+ manifest.WriteString(relDst) -+ manifest.WriteString("\n") -+ } -+ if manifest.Len() > 0 { -+ manifestPath := filepath.Join(cachePath, orchestrionStdlibCacheManifestName) -+ if err := os.WriteFile(manifestPath, []byte(manifest.String()), 0o644); err != nil { -+ return fmt.Errorf("write stdlib cache manifest at %s: %w", manifestPath, err) -+ } -+ if verbose { -+ fmt.Fprintf(os.Stderr, "stdlib: wrote stdlib cache manifest %s\n", manifestPath) -+ } ++ manifest.WriteString(pkg) ++ manifest.WriteString("=") ++ manifest.WriteString(relDst) ++ manifest.WriteString("\n") ++ if verbose { ++ fmt.Fprintf(os.Stderr, "stdlib: published persisted orchestrion export %s -> declared cache %s\n", src, dst) + } + } ++ if manifest.Len() == 0 { ++ return nil ++ } ++ if err := writeFileAtomically(manifestPath, []byte(manifest.String()), 0o644); err != nil { ++ return fmt.Errorf("write stdlib cache manifest at %s: %w", manifestPath, err) ++ } ++ if verbose { ++ fmt.Fprintf(os.Stderr, "stdlib: wrote stdlib cache manifest %s\n", manifestPath) ++ } + return nil +} + ++func projectStdlibCacheArchive(scratchCache, declaredCache, scratchArchive string) (destination, relative string, err error) { ++ scratchRoot := abs(scratchCache) ++ declaredRoot := abs(declaredCache) ++ archive := abs(scratchArchive) ++ rel, err := filepath.Rel(scratchRoot, archive) ++ if err != nil { ++ return "", "", err ++ } ++ rel = filepath.Clean(rel) ++ if rel == "." || filepath.IsAbs(rel) || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { ++ return "", "", fmt.Errorf("cache archive %s escapes scratch root %s", archive, scratchRoot) ++ } ++ if !strings.HasSuffix(filepath.Base(rel), "-d") { ++ return "", "", fmt.Errorf("cache archive %s is not a Go cache data entry", archive) ++ } ++ destination = filepath.Join(declaredRoot, rel) ++ destinationRel, err := filepath.Rel(declaredRoot, destination) ++ if err != nil || filepath.Clean(destinationRel) != rel { ++ return "", "", fmt.Errorf("cache archive destination %s escapes declared root %s", destination, declaredRoot) ++ } ++ return destination, filepath.ToSlash(rel), nil ++} ++ +func ensureSyntheticOrchestrionToolGo(verbose bool, orchestrionMode string) (func(), error) { + const toolFile = "orchestrion.tool.go" + if _, err := os.Stat(toolFile); err == nil { @@ -16278,15 +16345,19 @@ index cfcb991..08b2992 100644 +} diff --git a/go/tools/builders/stdlib_test.go b/go/tools/builders/stdlib_test.go new file mode 100644 -index 0000000..b92f214 +index 0000000..7ca400c --- /dev/null +++ b/go/tools/builders/stdlib_test.go -@@ -0,0 +1,240 @@ +@@ -0,0 +1,464 @@ +package main + +import ( ++ "bytes" ++ "crypto/sha256" ++ "fmt" + "os" + "path/filepath" ++ "reflect" + "strings" + "testing" +) @@ -16400,48 +16471,223 @@ index 0000000..b92f214 + } +} + -+func TestShouldRemoveStdlibCache(t *testing.T) { -+ t.Parallel() ++func TestPrepareStdlibCachesWithoutDeclaredOutput(t *testing.T) { ++ output := filepath.Join(t.TempDir(), "stdlib") ++ scratch, declared, cleanup, err := prepareStdlibCaches(output, "") ++ if err != nil { ++ t.Fatalf("prepareStdlibCaches error: %v", err) ++ } ++ if want := filepath.Join(abs(output), ".gocache"); scratch != want { ++ t.Fatalf("scratch cache = %q, want %q", scratch, want) ++ } ++ if declared != "" { ++ t.Fatalf("declared cache = %q, want empty", declared) ++ } ++ if info, err := os.Stat(scratch); err != nil || !info.IsDir() { ++ t.Fatalf("scratch cache was not created: %v", err) ++ } ++ cleanup() ++ if _, err := os.Stat(scratch); !os.IsNotExist(err) { ++ t.Fatalf("scratch cache still exists after cleanup: %v", err) ++ } ++} ++ ++func TestPrepareStdlibCachesSeparatesDeclaredOutput(t *testing.T) { ++ root := t.TempDir() ++ output := filepath.Join(root, "stdlib") ++ cacheOut := filepath.Join(root, "declared") ++ scratch, declared, cleanup, err := prepareStdlibCaches(output, cacheOut) ++ if err != nil { ++ t.Fatalf("prepareStdlibCaches error: %v", err) ++ } ++ if sameFilePath(scratch, declared) { ++ t.Fatalf("scratch cache %q aliases declared cache %q", scratch, declared) ++ } ++ for name, path := range map[string]string{"scratch": scratch, "declared": declared} { ++ if info, err := os.Stat(path); err != nil || !info.IsDir() { ++ t.Fatalf("%s cache was not created: %v", name, err) ++ } ++ } ++ cleanup() ++ if _, err := os.Stat(scratch); !os.IsNotExist(err) { ++ t.Fatalf("scratch cache still exists after cleanup: %v", err) ++ } ++ if info, err := os.Stat(declared); err != nil || !info.IsDir() { ++ t.Fatalf("declared cache should remain after scratch cleanup: %v", err) ++ } ++} ++ ++func TestPrepareStdlibCachesNormalizesRelativeInputs(t *testing.T) { ++ root := t.TempDir() ++ previousWD, err := os.Getwd() ++ if err != nil { ++ t.Fatalf("getwd: %v", err) ++ } ++ if err := os.Chdir(root); err != nil { ++ t.Fatalf("chdir: %v", err) ++ } ++ defer func() { _ = os.Chdir(previousWD) }() ++ normalizedRoot, err := os.Getwd() ++ if err != nil { ++ t.Fatalf("get normalized workdir: %v", err) ++ } ++ ++ scratch, declared, cleanup, err := prepareStdlibCaches("stdlib", "declared") ++ if err != nil { ++ t.Fatalf("prepareStdlibCaches error: %v", err) ++ } ++ defer cleanup() ++ if !filepath.IsAbs(scratch) || !filepath.IsAbs(declared) { ++ t.Fatalf("cache paths are not absolute: scratch=%q declared=%q", scratch, declared) ++ } ++ if scratch != filepath.Join(normalizedRoot, "stdlib", ".gocache") { ++ t.Fatalf("scratch cache = %q", scratch) ++ } ++ if declared != filepath.Join(normalizedRoot, "declared") { ++ t.Fatalf("declared cache = %q", declared) ++ } ++} ++ ++func TestPrepareStdlibCachesRejectsAliasedOutput(t *testing.T) { ++ output := filepath.Join(t.TempDir(), "stdlib") ++ alias := filepath.Join(output, ".gocache") ++ if _, _, _, err := prepareStdlibCaches(output, alias); err == nil || !strings.Contains(err.Error(), "aliases declared cache") { ++ t.Fatalf("prepareStdlibCaches alias error = %v", err) ++ } ++ if _, err := os.Stat(alias); !os.IsNotExist(err) { ++ t.Fatalf("aliased scratch cache still exists after failure: %v", err) ++ } ++} ++ ++func TestNewBufferedCommandPreservesWritableGoCache(t *testing.T) { ++ scratch := filepath.Join(t.TempDir(), "scratch") ++ declared := filepath.Join(t.TempDir(), "declared") ++ if err := os.MkdirAll(declared, 0o755); err != nil { ++ t.Fatal(err) ++ } ++ t.Setenv("GOCACHE", scratch) ++ ++ cmd := (&env{stdlibCache: declared}).newBufferedCommand([]string{"go", "version"}, &bytes.Buffer{}) ++ if got := getEnv(cmd.Env, "GOCACHE"); got != scratch { ++ t.Fatalf("GOCACHE = %q, want writable cache %q", got, scratch) ++ } ++ if got := getEnv(cmd.Env, orchestrionStdlibCacheEnvVar); got != declared { ++ t.Fatalf("%s = %q, want %q", orchestrionStdlibCacheEnvVar, got, declared) ++ } ++ ++ for _, invalid := range []string{"", filepath.Join(t.TempDir(), "missing")} { ++ cmd = (&env{stdlibCache: invalid}).newBufferedCommand([]string{"go", "version"}, &bytes.Buffer{}) ++ if got := getEnv(cmd.Env, "GOCACHE"); got != scratch { ++ t.Fatalf("invalid declared cache %q replaced GOCACHE with %q", invalid, got) ++ } ++ if got := getEnv(cmd.Env, orchestrionStdlibCacheEnvVar); got != "" { ++ t.Fatalf("invalid declared cache %q set %s=%q", invalid, orchestrionStdlibCacheEnvVar, got) ++ } ++ } ++} ++ ++func TestPublishPersistedOrchestrionExportsIsDeterministic(t *testing.T) { ++ archives := map[string][]byte{ ++ "fmt": []byte("woven fmt archive"), ++ "log": []byte("woven log archive"), ++ } ++ cachePaths := map[string]string{ ++ "fmt": filepath.Join("11", "fmt-d"), ++ "log": filepath.Join("aa", "log-d"), ++ } + ++ inventories := make([]map[string]string, 0, 2) ++ for run := 0; run < 2; run++ { ++ scratch := filepath.Join(t.TempDir(), "scratch") ++ declared := filepath.Join(t.TempDir(), "declared") ++ persisted := filepath.Join(t.TempDir(), "persisted") ++ if err := os.MkdirAll(scratch, 0o755); err != nil { ++ t.Fatal(err) ++ } ++ exports := make(map[string]string, len(archives)) ++ resolved := make(map[string]string, len(cachePaths)) ++ for pkg, data := range archives { ++ src := filepath.Join(persisted, pkg+".a") ++ writeTestFile(t, src, data) ++ exports[pkg] = src ++ resolved[pkg] = filepath.Join(scratch, cachePaths[pkg]) ++ writeTestFile(t, resolved[pkg], []byte(fmt.Sprintf("unwoven run %d", run))) ++ } ++ writeTestFile(t, filepath.Join(scratch, "11", "fmt-a"), []byte(fmt.Sprintf("timestamp %d", run))) ++ writeTestFile(t, filepath.Join(scratch, "trim.txt"), []byte(fmt.Sprintf("trim %d", run))) ++ writeTestFile(t, filepath.Join(scratch, "unrelated", "entry-d"), []byte(fmt.Sprintf("unrelated %d", run))) ++ ++ if err := publishPersistedOrchestrionExportsToCache(exports, resolved, scratch, declared, false); err != nil { ++ t.Fatalf("publish run %d: %v", run, err) ++ } ++ inventory := testTreeInventory(t, declared) ++ inventories = append(inventories, inventory) ++ if len(inventory) != len(archives)+1 { ++ t.Fatalf("declared inventory has unexpected entries: %v", inventory) ++ } ++ for pkg, data := range archives { ++ path := filepath.ToSlash(cachePaths[pkg]) ++ if got := inventory[path]; got != testDigest(data) { ++ t.Fatalf("published %s digest = %q, want %q", path, got, testDigest(data)) ++ } ++ } ++ manifest, err := os.ReadFile(filepath.Join(declared, orchestrionStdlibCacheManifestName)) ++ if err != nil { ++ t.Fatal(err) ++ } ++ wantManifest := "fmt=11/fmt-d\nlog=aa/log-d\n" ++ if string(manifest) != wantManifest { ++ t.Fatalf("manifest = %q, want %q", string(manifest), wantManifest) ++ } ++ } ++ if !reflect.DeepEqual(inventories[0], inventories[1]) { ++ t.Fatalf("declared cache inventories differ:\nrun 1: %v\nrun 2: %v", inventories[0], inventories[1]) ++ } ++} ++ ++func TestProjectStdlibCacheArchiveRejectsInvalidPaths(t *testing.T) { ++ root := t.TempDir() ++ scratch := filepath.Join(root, "scratch") ++ declared := filepath.Join(root, "declared") + tests := []struct { -+ name string -+ orchestrionPath string -+ cacheOut string -+ want bool ++ name string ++ archive string ++ want string + }{ -+ { -+ name: "plain internal cache", -+ want: true, -+ }, -+ { -+ name: "plain declared cache output", -+ cacheOut: "bazel-out/bin/external/rules_go/stdlib_/gocache", -+ want: false, -+ }, -+ { -+ name: "orchestrion internal cache", -+ orchestrionPath: "external/rules_go_orchestrion_tool/orchestrion", -+ want: false, -+ }, -+ { -+ name: "orchestrion declared cache output", -+ orchestrionPath: "external/rules_go_orchestrion_tool/orchestrion", -+ cacheOut: "bazel-out/bin/external/rules_go/stdlib_/gocache", -+ want: false, -+ }, ++ {name: "outside scratch", archive: filepath.Join(root, "outside", "entry-d"), want: "escapes scratch root"}, ++ {name: "action index", archive: filepath.Join(scratch, "aa", "entry-a"), want: "not a Go cache data entry"}, + } -+ + for _, tt := range tests { -+ tt := tt + t.Run(tt.name, func(t *testing.T) { -+ t.Parallel() -+ if got := shouldRemoveStdlibCache(tt.orchestrionPath, tt.cacheOut); got != tt.want { -+ t.Fatalf("shouldRemoveStdlibCache(%q, %q) = %v, want %v", tt.orchestrionPath, tt.cacheOut, got, tt.want) ++ if _, _, err := projectStdlibCacheArchive(scratch, declared, tt.archive); err == nil || !strings.Contains(err.Error(), tt.want) { ++ t.Fatalf("projectStdlibCacheArchive error = %v, want %q", err, tt.want) + } + }) + } +} + ++func TestPublishPersistedOrchestrionExportsFailureLeavesNoManifest(t *testing.T) { ++ root := t.TempDir() ++ scratch := filepath.Join(root, "scratch") ++ declared := filepath.Join(root, "declared") ++ manifest := filepath.Join(declared, orchestrionStdlibCacheManifestName) ++ writeTestFile(t, manifest, []byte("stale=stale-d\n")) ++ ++ err := publishPersistedOrchestrionExportsToCache( ++ map[string]string{}, ++ map[string]string{"fmt": filepath.Join(scratch, "aa", "fmt-d")}, ++ scratch, ++ declared, ++ false, ++ ) ++ if err == nil || !strings.Contains(err.Error(), "missing persisted stdlib archive for cache package fmt") { ++ t.Fatalf("publish error = %v", err) ++ } ++ if _, err := os.Stat(manifest); !os.IsNotExist(err) { ++ t.Fatalf("manifest exists after failed publication: %v", err) ++ } ++} ++ +func TestMergeGoDebugSettingPreservesExistingFlags(t *testing.T) { + tests := []struct { + name string @@ -16522,6 +16768,51 @@ index 0000000..b92f214 + } + } +} ++ ++func writeTestFile(t *testing.T, path string, data []byte) { ++ t.Helper() ++ if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { ++ t.Fatalf("mkdir %s: %v", filepath.Dir(path), err) ++ } ++ if err := os.WriteFile(path, data, 0o644); err != nil { ++ t.Fatalf("write %s: %v", path, err) ++ } ++} ++ ++func testTreeInventory(t *testing.T, root string) map[string]string { ++ t.Helper() ++ inventory := make(map[string]string) ++ err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { ++ if err != nil { ++ return err ++ } ++ if path == root || entry.IsDir() { ++ return nil ++ } ++ if entry.Type()&os.ModeSymlink != 0 { ++ return fmt.Errorf("unexpected symlink %s", path) ++ } ++ data, err := os.ReadFile(path) ++ if err != nil { ++ return err ++ } ++ rel, err := filepath.Rel(root, path) ++ if err != nil { ++ return err ++ } ++ inventory[filepath.ToSlash(rel)] = testDigest(data) ++ return nil ++ }) ++ if err != nil { ++ t.Fatalf("inventory %s: %v", root, err) ++ } ++ return inventory ++} ++ ++func testDigest(data []byte) string { ++ sum := sha256.Sum256(data) ++ return fmt.Sprintf("%x", sum[:]) ++} diff --git a/go/tools/builders/stdliblist.go b/go/tools/builders/stdliblist.go index adbbe4a..4404c30 100644 --- a/go/tools/builders/stdliblist.go diff --git a/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch index 7b73deda..f5f4943f 100644 --- a/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch @@ -7598,7 +7598,7 @@ index 2bea17e..154b543 100644 if err := cmd.Run(); err != nil { diff --git a/go/tools/builders/env_orchestrion.go b/go/tools/builders/env_orchestrion.go new file mode 100644 -index 0000000..fbc392c +index 0000000..73c7992 --- /dev/null +++ b/go/tools/builders/env_orchestrion.go @@ -0,0 +1,81 @@ @@ -7663,7 +7663,8 @@ index 0000000..fbc392c +} + +// newBufferedCommand creates a subprocess command wired to the shared builder -+// buffer and applies the stdlib cache override needed by Orchestrion actions. ++// buffer and advertises the declared stdlib archive source without replacing ++// the subprocess's writable GOCACHE. +func (e *env) newBufferedCommand(args []string, buf *bytes.Buffer) *exec.Cmd { + cmd := exec.Command(args[0], args[1:]...) + cmd.Stdout = buf @@ -7671,7 +7672,6 @@ index 0000000..fbc392c + cmd.Env = os.Environ() + if e.stdlibCache != "" { + if info, err := os.Stat(e.stdlibCache); err == nil && info.IsDir() { -+ cmd.Env = setEnv(cmd.Env, "GOCACHE", e.stdlibCache) + cmd.Env = setEnv(cmd.Env, orchestrionStdlibCacheEnvVar, e.stdlibCache) + } + } @@ -8139,7 +8139,7 @@ index 893a0f6..16955a7 100644 + return "" +} diff --git a/go/tools/builders/importcfg.go b/go/tools/builders/importcfg.go -index c25763a..3f0e203 100644 +index c25763a..4c859ae 100644 --- a/go/tools/builders/importcfg.go +++ b/go/tools/builders/importcfg.go @@ -17,21 +17,94 @@ package main @@ -8237,7 +8237,7 @@ index c25763a..3f0e203 100644 // checkImports verifies that each import in files refers to a // direct dependency in archives or to a standard library package // listed in the file at stdPackageListPath. checkImports returns -@@ -213,6 +286,1678 @@ package with this path is linked.`, +@@ -213,6 +286,1695 @@ package with this path is linked.`, return filename, nil } @@ -9259,29 +9259,17 @@ index c25763a..3f0e203 100644 + baseEnv = setEnv(baseEnv, "GOROOT", goenv.goroot) + baseEnv = setEnv(baseEnv, "GO111MODULE", "off") + baseEnv = setEnv(baseEnv, "GOWORK", "off") -+ orchestrionCachePath := goenv.stdlibCache -+ if orchestrionCachePath == "" { -+ orchestrionCachePath = filepath.Join(goenv.goroot, ".gocache") -+ } -+ cachePath := "" -+ if info, err := os.Stat(orchestrionCachePath); err == nil && info.IsDir() { -+ cachePath = orchestrionCachePath -+ } -+ if cachePath == "" { -+ cachePath = getEnv(baseEnv, "GOCACHE") -+ } -+ if cachePath == "" { -+ if goenv.stdlibCache != "" { -+ cachePath = goenv.stdlibCache -+ } else { -+ cachePath = filepath.Join(goenv.goroot, ".gocache") -+ } -+ } -+ cachePath = abs(cachePath) -+ if err := os.MkdirAll(cachePath, 0o755); err != nil { ++ cachePath, err := writableStdlibCacheRoot(goenv, baseEnv) ++ if err != nil { + return nil, fmt.Errorf("prepare stdlib export cache: %w", err) + } ++ if err := seedWovenStdlibCache(goenv, cachePath); err != nil { ++ return nil, fmt.Errorf("seed stdlib export cache: %w", err) ++ } + baseEnv = setEnv(baseEnv, "GOCACHE", cachePath) ++ if goenv.stdlibCache != "" { ++ baseEnv = setEnv(baseEnv, orchestrionStdlibCacheEnvVar, goenv.stdlibCache) ++ } + if getEnv(baseEnv, "HOME") == "" { + homePath := filepath.Join(goenv.goroot, ".home") + if err := os.MkdirAll(homePath, 0o755); err != nil { @@ -9321,9 +9309,18 @@ index c25763a..3f0e203 100644 +} + +func resolveCacheStdlibExports(goenv *env, packages []string) (map[string]string, error) { -+ cacheRoot := "" -+ if goenv != nil { -+ cacheRoot = goenv.stdlibCache ++ if goenv != nil && goenv.stdlibCache != "" { ++ exports, err := readStdlibCacheManifest(goenv.stdlibCache, packages) ++ if err != nil { ++ return nil, err ++ } ++ if len(exports) > 0 { ++ return exports, nil ++ } ++ } ++ cacheRoot, err := writableStdlibCacheRoot(goenv, os.Environ()) ++ if err != nil { ++ return nil, err + } + return resolveCacheStdlibExportsAt(goenv, packages, cacheRoot) +} @@ -9352,6 +9349,9 @@ index c25763a..3f0e203 100644 + return manifestExports, nil + } + } ++ if goenv.stdlibCache != "" && sameFilePath(cacheRoot, goenv.stdlibCache) { ++ return nil, fmt.Errorf("refusing to use declared stdlib cache %s as writable GOCACHE", abs(goenv.stdlibCache)) ++ } + goenv.goroot = abs(goenv.goroot) + goenv.sdk = abs(goenv.sdk) + if info, err := os.Stat(goenv.goroot); err != nil || !info.IsDir() { @@ -9435,6 +9435,24 @@ index c25763a..3f0e203 100644 + return env +} + ++func writableStdlibCacheRoot(goenv *env, environ []string) (string, error) { ++ cacheRoot := strings.TrimSpace(getEnv(environ, "GOCACHE")) ++ if cacheRoot == "" { ++ if goenv == nil || strings.TrimSpace(goenv.goroot) == "" { ++ return "", fmt.Errorf("writable stdlib GOCACHE is empty") ++ } ++ cacheRoot = filepath.Join(abs(goenv.goroot), ".gocache") ++ } ++ cacheRoot = abs(cacheRoot) ++ if goenv != nil && goenv.stdlibCache != "" && sameFilePath(cacheRoot, goenv.stdlibCache) { ++ return "", fmt.Errorf("writable stdlib GOCACHE %s aliases declared cache", cacheRoot) ++ } ++ if err := os.MkdirAll(cacheRoot, 0o755); err != nil { ++ return "", err ++ } ++ return cacheRoot, nil ++} ++ +func readStdlibCacheManifest(cacheRoot string, packages []string) (map[string]string, error) { + if cacheRoot == "" { + return nil, nil @@ -9829,18 +9847,17 @@ index c25763a..3f0e203 100644 + cmd.Env = setEnv(cmd.Env, "GOROOT", goenv.goroot) + cmd.Env = setEnv(cmd.Env, "GO111MODULE", "on") + cmd.Env = setEnv(cmd.Env, "GOWORK", "off") -+ cachePath := goenv.stdlibCache -+ if cachePath == "" { -+ cachePath = getEnv(cmd.Env, "GOCACHE") -+ } -+ if cachePath == "" { -+ cachePath = filepath.Join(goenv.goroot, ".gocache") -+ } -+ cachePath = abs(cachePath) -+ if err := os.MkdirAll(cachePath, 0o755); err != nil { ++ cachePath, err := writableStdlibCacheRoot(goenv, cmd.Env) ++ if err != nil { + return nil, fmt.Errorf("prepare stdlib closure cache: %w", err) + } ++ if err := seedWovenStdlibCache(goenv, cachePath); err != nil { ++ return nil, fmt.Errorf("seed stdlib closure cache: %w", err) ++ } + cmd.Env = setEnv(cmd.Env, "GOCACHE", cachePath) ++ if goenv.stdlibCache != "" { ++ cmd.Env = setEnv(cmd.Env, orchestrionStdlibCacheEnvVar, goenv.stdlibCache) ++ } + normalizedEnv, err := normalizeGoActionCacheEnv(cmd.Env) + if err != nil { + return nil, fmt.Errorf("prepare stdlib closure action cache env: %w", err) @@ -9918,10 +9935,10 @@ index c25763a..3f0e203 100644 known []string diff --git a/go/tools/builders/importcfg_test.go b/go/tools/builders/importcfg_test.go new file mode 100644 -index 0000000..3724594 +index 0000000..35a1fa4 --- /dev/null +++ b/go/tools/builders/importcfg_test.go -@@ -0,0 +1,816 @@ +@@ -0,0 +1,857 @@ +package main + +import ( @@ -10487,6 +10504,47 @@ index 0000000..3724594 + } +} + ++func TestWritableStdlibCacheRootUsesPrivateCache(t *testing.T) { ++ privateCache := filepath.Join(t.TempDir(), "private") ++ declaredCache := filepath.Join(t.TempDir(), "declared") ++ cacheRoot, err := writableStdlibCacheRoot(&env{ ++ goroot: t.TempDir(), ++ stdlibCache: declaredCache, ++ }, []string{"GOCACHE=" + privateCache}) ++ if err != nil { ++ t.Fatalf("writableStdlibCacheRoot error: %v", err) ++ } ++ if cacheRoot != abs(privateCache) { ++ t.Fatalf("writable cache = %q, want %q", cacheRoot, abs(privateCache)) ++ } ++ if info, err := os.Stat(cacheRoot); err != nil || !info.IsDir() { ++ t.Fatalf("writable cache was not created: %v", err) ++ } ++} ++ ++func TestWritableStdlibCacheRootRejectsDeclaredCache(t *testing.T) { ++ declaredCache := filepath.Join(t.TempDir(), "declared") ++ _, err := writableStdlibCacheRoot(&env{ ++ goroot: t.TempDir(), ++ stdlibCache: declaredCache, ++ }, []string{"GOCACHE=" + declaredCache}) ++ if err == nil || !strings.Contains(err.Error(), "aliases declared cache") { ++ t.Fatalf("writableStdlibCacheRoot error = %v", err) ++ } ++} ++ ++func TestResolveCacheStdlibExportsRejectsDeclaredCacheAsWritable(t *testing.T) { ++ declaredCache := t.TempDir() ++ _, err := resolveCacheStdlibExportsAt(&env{ ++ goroot: "fake-goroot", ++ sdk: "fake-sdk", ++ stdlibCache: declaredCache, ++ }, []string{"fmt"}, declaredCache) ++ if err == nil || !strings.Contains(err.Error(), "refusing to use declared stdlib cache") { ++ t.Fatalf("resolveCacheStdlibExportsAt error = %v", err) ++ } ++} ++ +func TestModuleExportCacheManifestRoundTrip(t *testing.T) { + root := t.TempDir() + paths := orchestrionCachePaths(root, "module-exports", "abc123") @@ -15655,7 +15713,7 @@ index 0000000..9edae16 + } +} diff --git a/go/tools/builders/stdlib.go b/go/tools/builders/stdlib.go -index cfcb991..08b2992 100644 +index cfcb991..9b5c36c 100644 --- a/go/tools/builders/stdlib.go +++ b/go/tools/builders/stdlib.go @@ -21,15 +21,29 @@ import ( @@ -15704,7 +15762,7 @@ index cfcb991..08b2992 100644 goroot := os.Getenv("GOROOT") if goroot == "" { return fmt.Errorf("GOROOT not set") -@@ -58,27 +80,40 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) +@@ -58,27 +80,35 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) } // Link in the bare minimum needed to the new GOROOT @@ -15725,21 +15783,19 @@ index cfcb991..08b2992 100644 os.Setenv("GOROOT", output) + goenv.goroot = output - // Create a temporary cache directory. "go build" requires this starting - // in Go 1.12. - cachePath := filepath.Join(output, ".gocache") -+ if *cacheOut != "" { -+ cachePath = abs(*cacheOut) -+ goenv.stdlibCache = cachePath +- // Create a temporary cache directory. "go build" requires this starting +- // in Go 1.12. +- cachePath := filepath.Join(output, ".gocache") ++ // Live Go commands use an action-private scratch cache. The separately ++ // declared cache output is populated only with deterministic woven archives. ++ cachePath, declaredCache, cleanupCache, err := prepareStdlibCaches(output, *cacheOut) ++ if err != nil { ++ return err + } ++ defer cleanupCache() ++ goenv.stdlibCache = declaredCache os.Setenv("GOCACHE", cachePath) - defer os.RemoveAll(cachePath) -+ if err := os.MkdirAll(cachePath, 0o755); err != nil { -+ return fmt.Errorf("prepare stdlib gocache at %s: %w", cachePath, err) -+ } -+ if shouldRemoveStdlibCache(*orchestrion, *cacheOut) { -+ defer os.RemoveAll(cachePath) -+ } - // Disable modules for the 'go install' command. Depending on the sandboxing - // mode, there may be a go.mod file in a parent directory which will turn @@ -15750,7 +15806,7 @@ index cfcb991..08b2992 100644 os.Setenv("GO111MODULE", "off") // Make sure we have an absolute path to the C compiler. -@@ -89,6 +124,19 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) +@@ -89,6 +119,19 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) for _, path := range filepath.SplitList(os.Getenv("PATH")) { absPaths = append(absPaths, abs(path)) } @@ -15770,7 +15826,7 @@ index cfcb991..08b2992 100644 os.Setenv("PATH", strings.Join(absPaths, string(os.PathListSeparator))) sandboxPath := abs(".") -@@ -113,7 +161,7 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) +@@ -113,7 +156,7 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) } } os.Setenv("CGO_LDFLAGS_ALLOW", b.String()) @@ -15779,7 +15835,7 @@ index cfcb991..08b2992 100644 // Build the commands needed to build the std library in the right mode // NOTE: the go command stamps compiled .a files with build ids, which are -@@ -121,7 +169,112 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) +@@ -121,7 +164,112 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) // creating reproducible builds because the build ids are hashed from // CGO_CFLAGS, which frequently contains absolute paths. As a workaround, // we strip the build ids, since they won't be used after this. @@ -15893,7 +15949,7 @@ index cfcb991..08b2992 100644 if len(build.Default.BuildTags) > 0 { installArgs = append(installArgs, "-tags", strings.Join(build.Default.BuildTags, ",")) } -@@ -135,7 +288,7 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) +@@ -135,7 +283,7 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) installArgs = append(installArgs, "-msan") } if *pgoprofile != "" { @@ -15902,7 +15958,7 @@ index cfcb991..08b2992 100644 } if *shared { gcflags = append(gcflags, "-shared") -@@ -167,8 +320,377 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) +@@ -167,8 +315,388 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) } installArgs = append(installArgs, packages...) @@ -15942,12 +15998,33 @@ index cfcb991..08b2992 100644 return nil } + -+// shouldRemoveStdlibCache reports whether the builder owns the cache directory -+// as scratch space. A non-empty -cacheout value is a Bazel-declared TreeArtifact -+// output, so it must remain present even for plain non-Orchestrion stdlib -+// actions. -+func shouldRemoveStdlibCache(orchestrionPath, cacheOut string) bool { -+ return strings.TrimSpace(orchestrionPath) == "" && strings.TrimSpace(cacheOut) == "" ++func prepareStdlibCaches(output, cacheOut string) (scratchCache, declaredCache string, cleanup func(), err error) { ++ scratchCache = filepath.Join(abs(output), ".gocache") ++ cleanup = func() { ++ _ = os.RemoveAll(scratchCache) ++ } ++ if err := os.MkdirAll(scratchCache, 0o755); err != nil { ++ return "", "", func() {}, fmt.Errorf("prepare stdlib scratch gocache at %s: %w", scratchCache, err) ++ } ++ if strings.TrimSpace(cacheOut) == "" { ++ return scratchCache, "", cleanup, nil ++ } ++ ++ declaredCache = abs(cacheOut) ++ if sameFilePath(scratchCache, declaredCache) { ++ cleanup() ++ return "", "", func() {}, fmt.Errorf("stdlib scratch gocache aliases declared cache output at %s", declaredCache) ++ } ++ if err := os.MkdirAll(declaredCache, 0o755); err != nil { ++ cleanup() ++ return "", "", func() {}, fmt.Errorf("prepare declared stdlib cache output at %s: %w", declaredCache, err) ++ } ++ return scratchCache, declaredCache, cleanup, nil ++} ++ ++func sameFilePath(left, right string) bool { ++ rel, err := filepath.Rel(abs(left), abs(right)) ++ return err == nil && rel == "." +} + +func mergeGoDebugSetting(existing, setting string) string { @@ -16083,111 +16160,101 @@ index cfcb991..08b2992 100644 + defer func() { + span.End(err) + }() -+ if goenv == nil || len(exports) == 0 { ++ if goenv == nil || len(exports) == 0 || len(roots) == 0 || strings.TrimSpace(goenv.stdlibCache) == "" { + return nil + } + -+ // We have two cache families to keep consistent: -+ // 1. the Bazel-declared stdlib cache consumed by later compile/link actions -+ // 2. the shared Datadog/Orchestrion cache used by internal `go list -export` -+ // dependency resolution when woven deps are injected. -+ // -+ // If only one is populated, compile and link can observe different archive -+ // fingerprints for stdlib packages like log/fmt/flag. Populate both from the -+ // same woven persisted exports. -+ candidateCaches := []string{} -+ seenCaches := map[string]struct{}{} -+ addCache := func(path string) { -+ path = strings.TrimSpace(path) -+ if path == "" { -+ return -+ } -+ path = abs(path) -+ if _, ok := seenCaches[path]; ok { -+ return -+ } -+ seenCaches[path] = struct{}{} -+ candidateCaches = append(candidateCaches, path) ++ scratchCache := strings.TrimSpace(os.Getenv("GOCACHE")) ++ if scratchCache == "" { ++ return fmt.Errorf("publish stdlib cache exports: GOCACHE scratch path is empty") + } -+ -+ addCache(goenv.stdlibCache) -+ if envCache := strings.TrimSpace(os.Getenv("GOCACHE")); envCache != "" { -+ addCache(envCache) ++ scratchCache = abs(scratchCache) ++ declaredCache := abs(goenv.stdlibCache) ++ if sameFilePath(scratchCache, declaredCache) { ++ return fmt.Errorf("publish stdlib cache exports: scratch cache %s aliases declared cache %s", scratchCache, declaredCache) + } -+ if len(candidateCaches) == 0 { -+ addCache(filepath.Join(abs(goenv.goroot), ".gocache")) ++ if err := os.MkdirAll(declaredCache, 0o755); err != nil { ++ return fmt.Errorf("prepare declared stdlib cache exports at %s: %w", declaredCache, err) + } ++ if verbose { ++ fmt.Fprintf(os.Stderr, "stdlib: resolving cache exports against scratch GOCACHE=%s\n", scratchCache) ++ } ++ cacheExports, err := resolveCacheStdlibExportsAt(goenv, roots, scratchCache) ++ if err != nil { ++ return err ++ } ++ return publishPersistedOrchestrionExportsToCache(exports, cacheExports, scratchCache, declaredCache, verbose) ++} + -+ prevCachePath, hadPrevCachePath := os.LookupEnv("GOCACHE") -+ defer func() { -+ if hadPrevCachePath { -+ _ = os.Setenv("GOCACHE", prevCachePath) -+ } else { -+ _ = os.Unsetenv("GOCACHE") -+ } -+ }() -+ -+ for _, cachePath := range candidateCaches { -+ if err := os.MkdirAll(cachePath, 0o755); err != nil { -+ return fmt.Errorf("prepare stdlib cache exports at %s: %w", cachePath, err) -+ } -+ if err := os.Setenv("GOCACHE", cachePath); err != nil { -+ return fmt.Errorf("set stdlib cache exports path %s: %w", cachePath, err) -+ } -+ if verbose { -+ fmt.Fprintf(os.Stderr, "stdlib: resolving cache-family exports against GOCACHE=%s\n", cachePath) ++func publishPersistedOrchestrionExportsToCache(exports, cacheExports map[string]string, scratchCache, declaredCache string, verbose bool) error { ++ manifestPath := filepath.Join(abs(declaredCache), orchestrionStdlibCacheManifestName) ++ if err := os.Remove(manifestPath); err != nil && !os.IsNotExist(err) { ++ return fmt.Errorf("remove stale stdlib cache manifest at %s: %w", manifestPath, err) ++ } ++ packages := make([]string, 0, len(cacheExports)) ++ for pkg := range cacheExports { ++ packages = append(packages, pkg) ++ } ++ sort.Strings(packages) ++ var manifest strings.Builder ++ for _, pkg := range packages { ++ src := strings.TrimSpace(exports[pkg]) ++ if src == "" { ++ return fmt.Errorf("missing persisted stdlib archive for cache package %s", pkg) + } -+ -+ cacheExports, err := resolveCacheStdlibExportsAt(goenv, roots, cachePath) ++ dst, relDst, err := projectStdlibCacheArchive(scratchCache, declaredCache, cacheExports[pkg]) + if err != nil { ++ return fmt.Errorf("project stdlib cache archive for %s: %w", pkg, err) ++ } ++ if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { + return err + } -+ packages := make([]string, 0, len(cacheExports)) -+ for pkg := range cacheExports { -+ packages = append(packages, pkg) ++ if err := copyArchiveFile(src, dst); err != nil { ++ return fmt.Errorf("copy persisted stdlib archive %s -> declared cache %s: %w", src, dst, err) + } -+ sort.Strings(packages) -+ var manifest strings.Builder -+ for _, pkg := range packages { -+ src := exports[pkg] -+ dst, ok := cacheExports[pkg] -+ if !ok || dst == "" { -+ continue -+ } -+ if strings.TrimSpace(src) == "" { -+ return fmt.Errorf("missing persisted stdlib archive for cache package %s", pkg) -+ } -+ if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { -+ return err -+ } -+ if err := copyArchiveFile(src, dst); err != nil { -+ return fmt.Errorf("copy persisted stdlib archive %s -> cache %s: %w", src, dst, err) -+ } -+ if verbose { -+ fmt.Fprintf(os.Stderr, "stdlib: synced persisted orchestrion export %s -> cache %s\n", src, dst) -+ } -+ relDst := dst -+ if rel, err := filepath.Rel(cachePath, dst); err == nil { -+ relDst = rel -+ } -+ manifest.WriteString(pkg) -+ manifest.WriteString("=") -+ manifest.WriteString(relDst) -+ manifest.WriteString("\n") -+ } -+ if manifest.Len() > 0 { -+ manifestPath := filepath.Join(cachePath, orchestrionStdlibCacheManifestName) -+ if err := os.WriteFile(manifestPath, []byte(manifest.String()), 0o644); err != nil { -+ return fmt.Errorf("write stdlib cache manifest at %s: %w", manifestPath, err) -+ } -+ if verbose { -+ fmt.Fprintf(os.Stderr, "stdlib: wrote stdlib cache manifest %s\n", manifestPath) -+ } ++ manifest.WriteString(pkg) ++ manifest.WriteString("=") ++ manifest.WriteString(relDst) ++ manifest.WriteString("\n") ++ if verbose { ++ fmt.Fprintf(os.Stderr, "stdlib: published persisted orchestrion export %s -> declared cache %s\n", src, dst) + } + } ++ if manifest.Len() == 0 { ++ return nil ++ } ++ if err := writeFileAtomically(manifestPath, []byte(manifest.String()), 0o644); err != nil { ++ return fmt.Errorf("write stdlib cache manifest at %s: %w", manifestPath, err) ++ } ++ if verbose { ++ fmt.Fprintf(os.Stderr, "stdlib: wrote stdlib cache manifest %s\n", manifestPath) ++ } + return nil +} + ++func projectStdlibCacheArchive(scratchCache, declaredCache, scratchArchive string) (destination, relative string, err error) { ++ scratchRoot := abs(scratchCache) ++ declaredRoot := abs(declaredCache) ++ archive := abs(scratchArchive) ++ rel, err := filepath.Rel(scratchRoot, archive) ++ if err != nil { ++ return "", "", err ++ } ++ rel = filepath.Clean(rel) ++ if rel == "." || filepath.IsAbs(rel) || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { ++ return "", "", fmt.Errorf("cache archive %s escapes scratch root %s", archive, scratchRoot) ++ } ++ if !strings.HasSuffix(filepath.Base(rel), "-d") { ++ return "", "", fmt.Errorf("cache archive %s is not a Go cache data entry", archive) ++ } ++ destination = filepath.Join(declaredRoot, rel) ++ destinationRel, err := filepath.Rel(declaredRoot, destination) ++ if err != nil || filepath.Clean(destinationRel) != rel { ++ return "", "", fmt.Errorf("cache archive destination %s escapes declared root %s", destination, declaredRoot) ++ } ++ return destination, filepath.ToSlash(rel), nil ++} ++ +func ensureSyntheticOrchestrionToolGo(verbose bool, orchestrionMode string) (func(), error) { + const toolFile = "orchestrion.tool.go" + if _, err := os.Stat(toolFile); err == nil { @@ -16282,15 +16349,19 @@ index cfcb991..08b2992 100644 +} diff --git a/go/tools/builders/stdlib_test.go b/go/tools/builders/stdlib_test.go new file mode 100644 -index 0000000..b92f214 +index 0000000..7ca400c --- /dev/null +++ b/go/tools/builders/stdlib_test.go -@@ -0,0 +1,240 @@ +@@ -0,0 +1,464 @@ +package main + +import ( ++ "bytes" ++ "crypto/sha256" ++ "fmt" + "os" + "path/filepath" ++ "reflect" + "strings" + "testing" +) @@ -16404,48 +16475,223 @@ index 0000000..b92f214 + } +} + -+func TestShouldRemoveStdlibCache(t *testing.T) { -+ t.Parallel() ++func TestPrepareStdlibCachesWithoutDeclaredOutput(t *testing.T) { ++ output := filepath.Join(t.TempDir(), "stdlib") ++ scratch, declared, cleanup, err := prepareStdlibCaches(output, "") ++ if err != nil { ++ t.Fatalf("prepareStdlibCaches error: %v", err) ++ } ++ if want := filepath.Join(abs(output), ".gocache"); scratch != want { ++ t.Fatalf("scratch cache = %q, want %q", scratch, want) ++ } ++ if declared != "" { ++ t.Fatalf("declared cache = %q, want empty", declared) ++ } ++ if info, err := os.Stat(scratch); err != nil || !info.IsDir() { ++ t.Fatalf("scratch cache was not created: %v", err) ++ } ++ cleanup() ++ if _, err := os.Stat(scratch); !os.IsNotExist(err) { ++ t.Fatalf("scratch cache still exists after cleanup: %v", err) ++ } ++} ++ ++func TestPrepareStdlibCachesSeparatesDeclaredOutput(t *testing.T) { ++ root := t.TempDir() ++ output := filepath.Join(root, "stdlib") ++ cacheOut := filepath.Join(root, "declared") ++ scratch, declared, cleanup, err := prepareStdlibCaches(output, cacheOut) ++ if err != nil { ++ t.Fatalf("prepareStdlibCaches error: %v", err) ++ } ++ if sameFilePath(scratch, declared) { ++ t.Fatalf("scratch cache %q aliases declared cache %q", scratch, declared) ++ } ++ for name, path := range map[string]string{"scratch": scratch, "declared": declared} { ++ if info, err := os.Stat(path); err != nil || !info.IsDir() { ++ t.Fatalf("%s cache was not created: %v", name, err) ++ } ++ } ++ cleanup() ++ if _, err := os.Stat(scratch); !os.IsNotExist(err) { ++ t.Fatalf("scratch cache still exists after cleanup: %v", err) ++ } ++ if info, err := os.Stat(declared); err != nil || !info.IsDir() { ++ t.Fatalf("declared cache should remain after scratch cleanup: %v", err) ++ } ++} ++ ++func TestPrepareStdlibCachesNormalizesRelativeInputs(t *testing.T) { ++ root := t.TempDir() ++ previousWD, err := os.Getwd() ++ if err != nil { ++ t.Fatalf("getwd: %v", err) ++ } ++ if err := os.Chdir(root); err != nil { ++ t.Fatalf("chdir: %v", err) ++ } ++ defer func() { _ = os.Chdir(previousWD) }() ++ normalizedRoot, err := os.Getwd() ++ if err != nil { ++ t.Fatalf("get normalized workdir: %v", err) ++ } ++ ++ scratch, declared, cleanup, err := prepareStdlibCaches("stdlib", "declared") ++ if err != nil { ++ t.Fatalf("prepareStdlibCaches error: %v", err) ++ } ++ defer cleanup() ++ if !filepath.IsAbs(scratch) || !filepath.IsAbs(declared) { ++ t.Fatalf("cache paths are not absolute: scratch=%q declared=%q", scratch, declared) ++ } ++ if scratch != filepath.Join(normalizedRoot, "stdlib", ".gocache") { ++ t.Fatalf("scratch cache = %q", scratch) ++ } ++ if declared != filepath.Join(normalizedRoot, "declared") { ++ t.Fatalf("declared cache = %q", declared) ++ } ++} ++ ++func TestPrepareStdlibCachesRejectsAliasedOutput(t *testing.T) { ++ output := filepath.Join(t.TempDir(), "stdlib") ++ alias := filepath.Join(output, ".gocache") ++ if _, _, _, err := prepareStdlibCaches(output, alias); err == nil || !strings.Contains(err.Error(), "aliases declared cache") { ++ t.Fatalf("prepareStdlibCaches alias error = %v", err) ++ } ++ if _, err := os.Stat(alias); !os.IsNotExist(err) { ++ t.Fatalf("aliased scratch cache still exists after failure: %v", err) ++ } ++} ++ ++func TestNewBufferedCommandPreservesWritableGoCache(t *testing.T) { ++ scratch := filepath.Join(t.TempDir(), "scratch") ++ declared := filepath.Join(t.TempDir(), "declared") ++ if err := os.MkdirAll(declared, 0o755); err != nil { ++ t.Fatal(err) ++ } ++ t.Setenv("GOCACHE", scratch) ++ ++ cmd := (&env{stdlibCache: declared}).newBufferedCommand([]string{"go", "version"}, &bytes.Buffer{}) ++ if got := getEnv(cmd.Env, "GOCACHE"); got != scratch { ++ t.Fatalf("GOCACHE = %q, want writable cache %q", got, scratch) ++ } ++ if got := getEnv(cmd.Env, orchestrionStdlibCacheEnvVar); got != declared { ++ t.Fatalf("%s = %q, want %q", orchestrionStdlibCacheEnvVar, got, declared) ++ } ++ ++ for _, invalid := range []string{"", filepath.Join(t.TempDir(), "missing")} { ++ cmd = (&env{stdlibCache: invalid}).newBufferedCommand([]string{"go", "version"}, &bytes.Buffer{}) ++ if got := getEnv(cmd.Env, "GOCACHE"); got != scratch { ++ t.Fatalf("invalid declared cache %q replaced GOCACHE with %q", invalid, got) ++ } ++ if got := getEnv(cmd.Env, orchestrionStdlibCacheEnvVar); got != "" { ++ t.Fatalf("invalid declared cache %q set %s=%q", invalid, orchestrionStdlibCacheEnvVar, got) ++ } ++ } ++} ++ ++func TestPublishPersistedOrchestrionExportsIsDeterministic(t *testing.T) { ++ archives := map[string][]byte{ ++ "fmt": []byte("woven fmt archive"), ++ "log": []byte("woven log archive"), ++ } ++ cachePaths := map[string]string{ ++ "fmt": filepath.Join("11", "fmt-d"), ++ "log": filepath.Join("aa", "log-d"), ++ } + ++ inventories := make([]map[string]string, 0, 2) ++ for run := 0; run < 2; run++ { ++ scratch := filepath.Join(t.TempDir(), "scratch") ++ declared := filepath.Join(t.TempDir(), "declared") ++ persisted := filepath.Join(t.TempDir(), "persisted") ++ if err := os.MkdirAll(scratch, 0o755); err != nil { ++ t.Fatal(err) ++ } ++ exports := make(map[string]string, len(archives)) ++ resolved := make(map[string]string, len(cachePaths)) ++ for pkg, data := range archives { ++ src := filepath.Join(persisted, pkg+".a") ++ writeTestFile(t, src, data) ++ exports[pkg] = src ++ resolved[pkg] = filepath.Join(scratch, cachePaths[pkg]) ++ writeTestFile(t, resolved[pkg], []byte(fmt.Sprintf("unwoven run %d", run))) ++ } ++ writeTestFile(t, filepath.Join(scratch, "11", "fmt-a"), []byte(fmt.Sprintf("timestamp %d", run))) ++ writeTestFile(t, filepath.Join(scratch, "trim.txt"), []byte(fmt.Sprintf("trim %d", run))) ++ writeTestFile(t, filepath.Join(scratch, "unrelated", "entry-d"), []byte(fmt.Sprintf("unrelated %d", run))) ++ ++ if err := publishPersistedOrchestrionExportsToCache(exports, resolved, scratch, declared, false); err != nil { ++ t.Fatalf("publish run %d: %v", run, err) ++ } ++ inventory := testTreeInventory(t, declared) ++ inventories = append(inventories, inventory) ++ if len(inventory) != len(archives)+1 { ++ t.Fatalf("declared inventory has unexpected entries: %v", inventory) ++ } ++ for pkg, data := range archives { ++ path := filepath.ToSlash(cachePaths[pkg]) ++ if got := inventory[path]; got != testDigest(data) { ++ t.Fatalf("published %s digest = %q, want %q", path, got, testDigest(data)) ++ } ++ } ++ manifest, err := os.ReadFile(filepath.Join(declared, orchestrionStdlibCacheManifestName)) ++ if err != nil { ++ t.Fatal(err) ++ } ++ wantManifest := "fmt=11/fmt-d\nlog=aa/log-d\n" ++ if string(manifest) != wantManifest { ++ t.Fatalf("manifest = %q, want %q", string(manifest), wantManifest) ++ } ++ } ++ if !reflect.DeepEqual(inventories[0], inventories[1]) { ++ t.Fatalf("declared cache inventories differ:\nrun 1: %v\nrun 2: %v", inventories[0], inventories[1]) ++ } ++} ++ ++func TestProjectStdlibCacheArchiveRejectsInvalidPaths(t *testing.T) { ++ root := t.TempDir() ++ scratch := filepath.Join(root, "scratch") ++ declared := filepath.Join(root, "declared") + tests := []struct { -+ name string -+ orchestrionPath string -+ cacheOut string -+ want bool ++ name string ++ archive string ++ want string + }{ -+ { -+ name: "plain internal cache", -+ want: true, -+ }, -+ { -+ name: "plain declared cache output", -+ cacheOut: "bazel-out/bin/external/rules_go/stdlib_/gocache", -+ want: false, -+ }, -+ { -+ name: "orchestrion internal cache", -+ orchestrionPath: "external/rules_go_orchestrion_tool/orchestrion", -+ want: false, -+ }, -+ { -+ name: "orchestrion declared cache output", -+ orchestrionPath: "external/rules_go_orchestrion_tool/orchestrion", -+ cacheOut: "bazel-out/bin/external/rules_go/stdlib_/gocache", -+ want: false, -+ }, ++ {name: "outside scratch", archive: filepath.Join(root, "outside", "entry-d"), want: "escapes scratch root"}, ++ {name: "action index", archive: filepath.Join(scratch, "aa", "entry-a"), want: "not a Go cache data entry"}, + } -+ + for _, tt := range tests { -+ tt := tt + t.Run(tt.name, func(t *testing.T) { -+ t.Parallel() -+ if got := shouldRemoveStdlibCache(tt.orchestrionPath, tt.cacheOut); got != tt.want { -+ t.Fatalf("shouldRemoveStdlibCache(%q, %q) = %v, want %v", tt.orchestrionPath, tt.cacheOut, got, tt.want) ++ if _, _, err := projectStdlibCacheArchive(scratch, declared, tt.archive); err == nil || !strings.Contains(err.Error(), tt.want) { ++ t.Fatalf("projectStdlibCacheArchive error = %v, want %q", err, tt.want) + } + }) + } +} + ++func TestPublishPersistedOrchestrionExportsFailureLeavesNoManifest(t *testing.T) { ++ root := t.TempDir() ++ scratch := filepath.Join(root, "scratch") ++ declared := filepath.Join(root, "declared") ++ manifest := filepath.Join(declared, orchestrionStdlibCacheManifestName) ++ writeTestFile(t, manifest, []byte("stale=stale-d\n")) ++ ++ err := publishPersistedOrchestrionExportsToCache( ++ map[string]string{}, ++ map[string]string{"fmt": filepath.Join(scratch, "aa", "fmt-d")}, ++ scratch, ++ declared, ++ false, ++ ) ++ if err == nil || !strings.Contains(err.Error(), "missing persisted stdlib archive for cache package fmt") { ++ t.Fatalf("publish error = %v", err) ++ } ++ if _, err := os.Stat(manifest); !os.IsNotExist(err) { ++ t.Fatalf("manifest exists after failed publication: %v", err) ++ } ++} ++ +func TestMergeGoDebugSettingPreservesExistingFlags(t *testing.T) { + tests := []struct { + name string @@ -16526,6 +16772,51 @@ index 0000000..b92f214 + } + } +} ++ ++func writeTestFile(t *testing.T, path string, data []byte) { ++ t.Helper() ++ if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { ++ t.Fatalf("mkdir %s: %v", filepath.Dir(path), err) ++ } ++ if err := os.WriteFile(path, data, 0o644); err != nil { ++ t.Fatalf("write %s: %v", path, err) ++ } ++} ++ ++func testTreeInventory(t *testing.T, root string) map[string]string { ++ t.Helper() ++ inventory := make(map[string]string) ++ err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { ++ if err != nil { ++ return err ++ } ++ if path == root || entry.IsDir() { ++ return nil ++ } ++ if entry.Type()&os.ModeSymlink != 0 { ++ return fmt.Errorf("unexpected symlink %s", path) ++ } ++ data, err := os.ReadFile(path) ++ if err != nil { ++ return err ++ } ++ rel, err := filepath.Rel(root, path) ++ if err != nil { ++ return err ++ } ++ inventory[filepath.ToSlash(rel)] = testDigest(data) ++ return nil ++ }) ++ if err != nil { ++ t.Fatalf("inventory %s: %v", root, err) ++ } ++ return inventory ++} ++ ++func testDigest(data []byte) string { ++ sum := sha256.Sum256(data) ++ return fmt.Sprintf("%x", sum[:]) ++} diff --git a/go/tools/builders/stdliblist.go b/go/tools/builders/stdliblist.go index adbbe4a..4404c30 100644 --- a/go/tools/builders/stdliblist.go diff --git a/tools/dev/verify_rules_go_profiles.py b/tools/dev/verify_rules_go_profiles.py index 30ae8ab5..e24011c5 100644 --- a/tools/dev/verify_rules_go_profiles.py +++ b/tools/dev/verify_rules_go_profiles.py @@ -11,9 +11,12 @@ import argparse from contextlib import contextmanager +from dataclasses import dataclass +import hashlib import json import os from pathlib import Path +from pathlib import PurePosixPath import shutil import subprocess import sys @@ -171,6 +174,10 @@ def verify_workspace_runtime_functional_smoke( "--@io_bazel_rules_go//go/private/orchestrion:enabled=true", "--@io_bazel_rules_go//go/private/orchestrion:mode=test_optimization", ] + isolated_cache_flags = [ + "--disk_cache=", + "--remote_cache=", + ] run_bazel( bazel, bazel_output_user_root, @@ -178,11 +185,22 @@ def verify_workspace_runtime_functional_smoke( ["build", *common_flags, "@go_sdk//:builder"], private_safe_patterns=private_safe_patterns, ) - run_bazel( + first_plain = run_stdlib_inventory( + bazel, + bazel_output_user_root, + workspace, + command="build", + mode_flags=[*common_flags, *isolated_cache_flags], + target="//app:hello_test", + private_safe_patterns=private_safe_patterns, + ) + first_orchestrion = run_stdlib_inventory( bazel, bazel_output_user_root, workspace, - ["test", *orchestrion_flags, "--test_output=errors", "//app:hello_test"], + command="test", + mode_flags=[*orchestrion_flags, *isolated_cache_flags, "--test_output=errors"], + target="//app:hello_test", private_safe_patterns=private_safe_patterns, ) aquery = run_bazel( @@ -194,6 +212,34 @@ def verify_workspace_runtime_functional_smoke( ) assert_aquery_contains(aquery.stdout, patch) + replay_output_user_root = work_root / "bazel_output_user_root_replay" + second_plain = run_stdlib_inventory( + bazel, + replay_output_user_root, + workspace, + command="build", + mode_flags=[*common_flags, *isolated_cache_flags], + target="//app:hello_test", + private_safe_patterns=private_safe_patterns, + ) + second_orchestrion = run_stdlib_inventory( + bazel, + replay_output_user_root, + workspace, + command="test", + mode_flags=[*orchestrion_flags, *isolated_cache_flags, "--test_output=errors"], + target="//app:hello_test", + private_safe_patterns=private_safe_patterns, + ) + assert_plain_stdlib_cache(first_plain, patch) + assert_plain_stdlib_cache(second_plain, patch) + assert_orchestrion_stdlib_cache(first_orchestrion, patch) + assert_orchestrion_stdlib_cache(second_orchestrion, patch) + if first_plain != second_plain: + raise ValueError("plain stdlib cache inventories differ for %s" % patch) + if first_orchestrion != second_orchestrion: + raise ValueError("Test Optimization stdlib cache inventories differ for %s" % patch) + def write_smoke_workspace( *, @@ -314,6 +360,161 @@ def run_bazel( return result +@dataclass(frozen=True) +class StdlibCacheSnapshot: + """Canonical declared-cache inventory plus its manifest contents.""" + + inventory: dict[str, str] + manifest: str | None + + +def run_stdlib_inventory( + bazel: Path, + output_user_root: Path, + workspace: Path, + *, + command: str, + mode_flags: list[str], + target: str, + private_safe_patterns: list[str] | None = None, +) -> StdlibCacheSnapshot: + """Execute one stdlib action and inventory its declared cache TreeArtifact.""" + run_bazel( + bazel, + output_user_root, + workspace, + [command, *mode_flags, target], + private_safe_patterns=private_safe_patterns, + ) + output_path_result = run_bazel( + bazel, + output_user_root, + workspace, + ["info", *mode_flags, "output_path"], + private_safe_patterns=private_safe_patterns, + ) + output_path = Path(output_path_result.stdout.strip()) + candidates = sorted( + path + for path in output_path.rglob("gocache") + if path.parent.name == "stdlib_" and path.is_dir() + ) + if not candidates: + raise ValueError( + "expected a declared GoStdlib gocache under %s" % output_path + ) + snapshots = [(path, canonical_tree_inventory(path)) for path in candidates] + manifested = [(path, snapshot) for path, snapshot in snapshots if snapshot.manifest is not None] + if len(manifested) > 1: + raise ValueError( + "multiple declared GoStdlib caches contain manifests under %s: %s" + % (output_path, [path.as_posix() for path, _ in manifested]) + ) + for path, snapshot in snapshots: + if manifested and path == manifested[0][0]: + continue + if snapshot.inventory: + raise ValueError( + "non-selected declared GoStdlib cache is not empty: %s contains %s" + % (path, sorted(snapshot.inventory)) + ) + if manifested: + return manifested[0][1] + return StdlibCacheSnapshot(inventory={}, manifest=None) + + +def canonical_tree_inventory(root: Path) -> StdlibCacheSnapshot: + """Return a deterministic relative-path inventory and reject symlinks.""" + inventory: dict[str, str] = {} + manifest: str | None = None + for path in sorted(root.rglob("*")): + relative = path.relative_to(root).as_posix() + if path.is_symlink(): + raise ValueError("declared stdlib cache contains symlink %s" % relative) + if path.is_dir(): + inventory[relative] = "dir" + continue + if not path.is_file(): + raise ValueError("declared stdlib cache contains non-file %s" % relative) + digest = hashlib.sha256(path.read_bytes()).hexdigest() + inventory[relative] = "file:%s" % digest + if relative == ".orchestrion_stdlib_cache_manifest": + manifest = path.read_text(encoding="utf-8") + return StdlibCacheSnapshot(inventory=inventory, manifest=manifest) + + +def assert_plain_stdlib_cache(snapshot: StdlibCacheSnapshot, patch: Path) -> None: + """Require the plain-mode declared stdlib cache to be empty.""" + if snapshot.inventory or snapshot.manifest is not None: + raise ValueError( + "plain stdlib cache for %s is not empty: %s" + % (patch, sorted(snapshot.inventory)) + ) + + +def assert_orchestrion_stdlib_cache(snapshot: StdlibCacheSnapshot, patch: Path) -> None: + """Require only sorted, manifested Go cache data entries.""" + manifest_name = ".orchestrion_stdlib_cache_manifest" + inventory = snapshot.inventory + manifest_value = inventory.get(manifest_name, "") + if not manifest_value.startswith("file:"): + raise ValueError("Test Optimization stdlib cache for %s has no manifest" % patch) + if snapshot.manifest is None: + raise ValueError("Test Optimization stdlib cache for %s has no manifest contents" % patch) + lines = snapshot.manifest.splitlines() + if not lines: + raise ValueError("Test Optimization stdlib cache manifest for %s is empty" % patch) + + expected = {manifest_name} + archives: set[str] = set() + directories: set[str] = set() + packages: set[str] = set() + package_order: list[str] = [] + for line in lines: + package, separator, relative = line.partition("=") + relative_path = PurePosixPath(relative) + if ( + not separator + or not package + or package in packages + or not relative + or "\\" in relative + or relative_path.is_absolute() + or ".." in relative_path.parts + or relative_path.name.endswith("-a") + or not relative_path.name.endswith("-d") + ): + raise ValueError( + "invalid Test Optimization stdlib cache manifest entry for %s: %r" + % (patch, line) + ) + packages.add(package) + package_order.append(package) + archives.add(relative) + parent = relative_path.parent + while parent != PurePosixPath("."): + directories.add(parent.as_posix()) + parent = parent.parent + + if package_order != sorted(package_order): + raise ValueError("Test Optimization stdlib cache manifest for %s is unsorted" % patch) + + expected.update(archives) + expected.update(directories) + actual = set(inventory) + if actual != expected: + raise ValueError( + "Test Optimization stdlib cache for %s contains unmanifested entries: missing=%s extra=%s" + % (patch, sorted(expected - actual), sorted(actual - expected)) + ) + for relative in archives: + if not inventory[relative].startswith("file:"): + raise ValueError("manifested stdlib archive %s is not a file" % relative) + for relative in directories: + if inventory[relative] != "dir": + raise ValueError("stdlib cache parent %s is not a directory" % relative) + + @contextmanager def temporary_smoke_root(): """Create and remove a smoke tempdir without Python-level Bazel tree cleanup.""" diff --git a/tools/tests/python/test_rules_go_profile_tools.py b/tools/tests/python/test_rules_go_profile_tools.py index 42c75b67..0e6a41b1 100644 --- a/tools/tests/python/test_rules_go_profile_tools.py +++ b/tools/tests/python/test_rules_go_profile_tools.py @@ -424,6 +424,58 @@ def test_run_bazel_scans_captured_output_before_failure_details(self) -> None: private_safe_patterns=["DENYLIST_SENTINEL"], ) + def test_stdlib_cache_snapshot_accepts_manifested_data_entries(self) -> None: + """The determinism verifier accepts only sorted manifested data entries.""" + with tempfile.TemporaryDirectory() as raw_tmp: + root = Path(raw_tmp) + _write(root / "11" / "fmt-d", "woven fmt") + _write(root / "aa" / "log-d", "woven log") + _write(root / "bb" / "runtime-d", "woven runtime") + _write(root / "cc" / "runtime-internal-d", "woven runtime internal") + _write( + root / ".orchestrion_stdlib_cache_manifest", + "fmt=11/fmt-d\n" + "log=aa/log-d\n" + "runtime=bb/runtime-d\n" + "runtime/internal/sys=cc/runtime-internal-d\n", + ) + + snapshot = self.mod.canonical_tree_inventory(root) + self.mod.assert_orchestrion_stdlib_cache(snapshot, Path("profile.patch")) + self.assertEqual( + snapshot.manifest, + "fmt=11/fmt-d\n" + "log=aa/log-d\n" + "runtime=bb/runtime-d\n" + "runtime/internal/sys=cc/runtime-internal-d\n", + ) + + def test_stdlib_cache_snapshot_rejects_unmanifested_entries(self) -> None: + """Action indexes and other unmanifested files fail verification.""" + with tempfile.TemporaryDirectory() as raw_tmp: + root = Path(raw_tmp) + _write(root / "11" / "fmt-d", "woven fmt") + _write(root / "11" / "fmt-a", "timestamped index") + _write( + root / ".orchestrion_stdlib_cache_manifest", + "fmt=11/fmt-d\n", + ) + + snapshot = self.mod.canonical_tree_inventory(root) + with self.assertRaisesRegex(ValueError, "unmanifested entries"): + self.mod.assert_orchestrion_stdlib_cache(snapshot, Path("profile.patch")) + + def test_plain_stdlib_cache_snapshot_must_be_empty(self) -> None: + """Plain mode may declare the cache TreeArtifact but cannot publish files.""" + with tempfile.TemporaryDirectory() as raw_tmp: + root = Path(raw_tmp) + empty = self.mod.canonical_tree_inventory(root) + self.mod.assert_plain_stdlib_cache(empty, Path("profile.patch")) + _write(root / "trim.txt", "nondeterministic metadata") + nonempty = self.mod.canonical_tree_inventory(root) + with self.assertRaisesRegex(ValueError, "is not empty"): + self.mod.assert_plain_stdlib_cache(nonempty, Path("profile.patch")) + if __name__ == "__main__": unittest.main() From 89e28cb34e87e1b5e70c5ccf1f61d0727e185580 Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Wed, 26 Aug 2026 22:21:25 +0200 Subject: [PATCH 28/37] rules_go: stabilize declared stdlib cache paths --- .../go/tools/builders/orchestrion_cache.go | 9 +++ .../v0_60_0/base/go/tools/builders/stdlib.go | 23 +++--- .../base/go/tools/builders/stdlib_test.go | 31 +++++--- .../go/tools/builders/orchestrion_cache.go | 9 +++ .../v0_61_1/base/go/tools/builders/stdlib.go | 23 +++--- .../base/go/tools/builders/stdlib_test.go | 31 +++++--- .../go/tools/builders/orchestrion_cache.go | 9 +++ .../v0_62_0/base/go/tools/builders/stdlib.go | 23 +++--- .../base/go/tools/builders/stdlib_test.go | 31 +++++--- .../v0_60_0/base/0001-full-delta.patch | 75 +++++++++++++------ .../v0_61_1/base/0001-full-delta.patch | 75 +++++++++++++------ .../v0_62_0/base/0001-full-delta.patch | 75 +++++++++++++------ tools/dev/verify_rules_go_profiles.py | 28 ++++++- 13 files changed, 314 insertions(+), 128 deletions(-) diff --git a/third_party/rgo/v0_60_0/base/go/tools/builders/orchestrion_cache.go b/third_party/rgo/v0_60_0/base/go/tools/builders/orchestrion_cache.go index 5cccefd7..876316ee 100644 --- a/third_party/rgo/v0_60_0/base/go/tools/builders/orchestrion_cache.go +++ b/third_party/rgo/v0_60_0/base/go/tools/builders/orchestrion_cache.go @@ -294,6 +294,15 @@ func digestFileOrMissing(path string) (string, error) { return shortDigest(data), nil } +func fullDigestFile(path string) (string, error) { + data, err := os.ReadFile(path) + if err != nil { + return "", err + } + sum := sha256.Sum256(data) + return fmt.Sprintf("%x", sum[:]), nil +} + // goSDKCacheIdentity returns a stable cache identity for the selected Go SDK // without depending on Bazel's output-base-specific execroot path. The identity // is derived from SDK file contents that remain stable across equivalent diff --git a/third_party/rgo/v0_60_0/base/go/tools/builders/stdlib.go b/third_party/rgo/v0_60_0/base/go/tools/builders/stdlib.go index 9b5c36c7..65244aa7 100644 --- a/third_party/rgo/v0_60_0/base/go/tools/builders/stdlib.go +++ b/third_party/rgo/v0_60_0/base/go/tools/builders/stdlib.go @@ -555,7 +555,7 @@ func publishPersistedOrchestrionExportsToCache(exports, cacheExports map[string] if src == "" { return fmt.Errorf("missing persisted stdlib archive for cache package %s", pkg) } - dst, relDst, err := projectStdlibCacheArchive(scratchCache, declaredCache, cacheExports[pkg]) + dst, relDst, err := projectStdlibCacheArchive(scratchCache, declaredCache, cacheExports[pkg], src) if err != nil { return fmt.Errorf("project stdlib cache archive for %s: %w", pkg, err) } @@ -585,27 +585,32 @@ func publishPersistedOrchestrionExportsToCache(exports, cacheExports map[string] return nil } -func projectStdlibCacheArchive(scratchCache, declaredCache, scratchArchive string) (destination, relative string, err error) { +func projectStdlibCacheArchive(scratchCache, declaredCache, scratchArchive, persistedArchive string) (destination, relative string, err error) { scratchRoot := abs(scratchCache) declaredRoot := abs(declaredCache) archive := abs(scratchArchive) - rel, err := filepath.Rel(scratchRoot, archive) + scratchRel, err := filepath.Rel(scratchRoot, archive) if err != nil { return "", "", err } - rel = filepath.Clean(rel) - if rel == "." || filepath.IsAbs(rel) || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + scratchRel = filepath.Clean(scratchRel) + if scratchRel == "." || filepath.IsAbs(scratchRel) || scratchRel == ".." || strings.HasPrefix(scratchRel, ".."+string(filepath.Separator)) { return "", "", fmt.Errorf("cache archive %s escapes scratch root %s", archive, scratchRoot) } - if !strings.HasSuffix(filepath.Base(rel), "-d") { + if !strings.HasSuffix(filepath.Base(scratchRel), "-d") { return "", "", fmt.Errorf("cache archive %s is not a Go cache data entry", archive) } - destination = filepath.Join(declaredRoot, rel) + digest, err := fullDigestFile(persistedArchive) + if err != nil { + return "", "", fmt.Errorf("digest persisted stdlib archive %s: %w", persistedArchive, err) + } + relative = filepath.Join(digest[:2], digest+"-d") + destination = filepath.Join(declaredRoot, relative) destinationRel, err := filepath.Rel(declaredRoot, destination) - if err != nil || filepath.Clean(destinationRel) != rel { + if err != nil || filepath.Clean(destinationRel) != relative { return "", "", fmt.Errorf("cache archive destination %s escapes declared root %s", destination, declaredRoot) } - return destination, filepath.ToSlash(rel), nil + return destination, filepath.ToSlash(relative), nil } func ensureSyntheticOrchestrionToolGo(verbose bool, orchestrionMode string) (func(), error) { diff --git a/third_party/rgo/v0_60_0/base/go/tools/builders/stdlib_test.go b/third_party/rgo/v0_60_0/base/go/tools/builders/stdlib_test.go index 7ca400cd..1d4cf608 100644 --- a/third_party/rgo/v0_60_0/base/go/tools/builders/stdlib_test.go +++ b/third_party/rgo/v0_60_0/base/go/tools/builders/stdlib_test.go @@ -240,9 +240,15 @@ func TestPublishPersistedOrchestrionExportsIsDeterministic(t *testing.T) { "fmt": []byte("woven fmt archive"), "log": []byte("woven log archive"), } - cachePaths := map[string]string{ - "fmt": filepath.Join("11", "fmt-d"), - "log": filepath.Join("aa", "log-d"), + cachePaths := []map[string]string{ + { + "fmt": filepath.Join("11", "fmt-d"), + "log": filepath.Join("aa", "log-d"), + }, + { + "fmt": filepath.Join("22", "different-fmt-d"), + "log": filepath.Join("bb", "different-log-d"), + }, } inventories := make([]map[string]string, 0, 2) @@ -254,12 +260,12 @@ func TestPublishPersistedOrchestrionExportsIsDeterministic(t *testing.T) { t.Fatal(err) } exports := make(map[string]string, len(archives)) - resolved := make(map[string]string, len(cachePaths)) + resolved := make(map[string]string, len(cachePaths[run])) for pkg, data := range archives { src := filepath.Join(persisted, pkg+".a") writeTestFile(t, src, data) exports[pkg] = src - resolved[pkg] = filepath.Join(scratch, cachePaths[pkg]) + resolved[pkg] = filepath.Join(scratch, cachePaths[run][pkg]) writeTestFile(t, resolved[pkg], []byte(fmt.Sprintf("unwoven run %d", run))) } writeTestFile(t, filepath.Join(scratch, "11", "fmt-a"), []byte(fmt.Sprintf("timestamp %d", run))) @@ -274,8 +280,9 @@ func TestPublishPersistedOrchestrionExportsIsDeterministic(t *testing.T) { if len(inventory) != len(archives)+1 { t.Fatalf("declared inventory has unexpected entries: %v", inventory) } - for pkg, data := range archives { - path := filepath.ToSlash(cachePaths[pkg]) + for _, data := range archives { + digest := testDigest(data) + path := filepath.ToSlash(filepath.Join(digest[:2], digest+"-d")) if got := inventory[path]; got != testDigest(data) { t.Fatalf("published %s digest = %q, want %q", path, got, testDigest(data)) } @@ -284,7 +291,13 @@ func TestPublishPersistedOrchestrionExportsIsDeterministic(t *testing.T) { if err != nil { t.Fatal(err) } - wantManifest := "fmt=11/fmt-d\nlog=aa/log-d\n" + fmtDigest := testDigest(archives["fmt"]) + logDigest := testDigest(archives["log"]) + wantManifest := fmt.Sprintf( + "fmt=%s/%s-d\nlog=%s/%s-d\n", + fmtDigest[:2], fmtDigest, + logDigest[:2], logDigest, + ) if string(manifest) != wantManifest { t.Fatalf("manifest = %q, want %q", string(manifest), wantManifest) } @@ -308,7 +321,7 @@ func TestProjectStdlibCacheArchiveRejectsInvalidPaths(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if _, _, err := projectStdlibCacheArchive(scratch, declared, tt.archive); err == nil || !strings.Contains(err.Error(), tt.want) { + if _, _, err := projectStdlibCacheArchive(scratch, declared, tt.archive, filepath.Join(root, "persisted.a")); err == nil || !strings.Contains(err.Error(), tt.want) { t.Fatalf("projectStdlibCacheArchive error = %v, want %q", err, tt.want) } }) diff --git a/third_party/rgo/v0_61_1/base/go/tools/builders/orchestrion_cache.go b/third_party/rgo/v0_61_1/base/go/tools/builders/orchestrion_cache.go index 5cccefd7..876316ee 100644 --- a/third_party/rgo/v0_61_1/base/go/tools/builders/orchestrion_cache.go +++ b/third_party/rgo/v0_61_1/base/go/tools/builders/orchestrion_cache.go @@ -294,6 +294,15 @@ func digestFileOrMissing(path string) (string, error) { return shortDigest(data), nil } +func fullDigestFile(path string) (string, error) { + data, err := os.ReadFile(path) + if err != nil { + return "", err + } + sum := sha256.Sum256(data) + return fmt.Sprintf("%x", sum[:]), nil +} + // goSDKCacheIdentity returns a stable cache identity for the selected Go SDK // without depending on Bazel's output-base-specific execroot path. The identity // is derived from SDK file contents that remain stable across equivalent diff --git a/third_party/rgo/v0_61_1/base/go/tools/builders/stdlib.go b/third_party/rgo/v0_61_1/base/go/tools/builders/stdlib.go index 9b5c36c7..65244aa7 100644 --- a/third_party/rgo/v0_61_1/base/go/tools/builders/stdlib.go +++ b/third_party/rgo/v0_61_1/base/go/tools/builders/stdlib.go @@ -555,7 +555,7 @@ func publishPersistedOrchestrionExportsToCache(exports, cacheExports map[string] if src == "" { return fmt.Errorf("missing persisted stdlib archive for cache package %s", pkg) } - dst, relDst, err := projectStdlibCacheArchive(scratchCache, declaredCache, cacheExports[pkg]) + dst, relDst, err := projectStdlibCacheArchive(scratchCache, declaredCache, cacheExports[pkg], src) if err != nil { return fmt.Errorf("project stdlib cache archive for %s: %w", pkg, err) } @@ -585,27 +585,32 @@ func publishPersistedOrchestrionExportsToCache(exports, cacheExports map[string] return nil } -func projectStdlibCacheArchive(scratchCache, declaredCache, scratchArchive string) (destination, relative string, err error) { +func projectStdlibCacheArchive(scratchCache, declaredCache, scratchArchive, persistedArchive string) (destination, relative string, err error) { scratchRoot := abs(scratchCache) declaredRoot := abs(declaredCache) archive := abs(scratchArchive) - rel, err := filepath.Rel(scratchRoot, archive) + scratchRel, err := filepath.Rel(scratchRoot, archive) if err != nil { return "", "", err } - rel = filepath.Clean(rel) - if rel == "." || filepath.IsAbs(rel) || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + scratchRel = filepath.Clean(scratchRel) + if scratchRel == "." || filepath.IsAbs(scratchRel) || scratchRel == ".." || strings.HasPrefix(scratchRel, ".."+string(filepath.Separator)) { return "", "", fmt.Errorf("cache archive %s escapes scratch root %s", archive, scratchRoot) } - if !strings.HasSuffix(filepath.Base(rel), "-d") { + if !strings.HasSuffix(filepath.Base(scratchRel), "-d") { return "", "", fmt.Errorf("cache archive %s is not a Go cache data entry", archive) } - destination = filepath.Join(declaredRoot, rel) + digest, err := fullDigestFile(persistedArchive) + if err != nil { + return "", "", fmt.Errorf("digest persisted stdlib archive %s: %w", persistedArchive, err) + } + relative = filepath.Join(digest[:2], digest+"-d") + destination = filepath.Join(declaredRoot, relative) destinationRel, err := filepath.Rel(declaredRoot, destination) - if err != nil || filepath.Clean(destinationRel) != rel { + if err != nil || filepath.Clean(destinationRel) != relative { return "", "", fmt.Errorf("cache archive destination %s escapes declared root %s", destination, declaredRoot) } - return destination, filepath.ToSlash(rel), nil + return destination, filepath.ToSlash(relative), nil } func ensureSyntheticOrchestrionToolGo(verbose bool, orchestrionMode string) (func(), error) { diff --git a/third_party/rgo/v0_61_1/base/go/tools/builders/stdlib_test.go b/third_party/rgo/v0_61_1/base/go/tools/builders/stdlib_test.go index 7ca400cd..1d4cf608 100644 --- a/third_party/rgo/v0_61_1/base/go/tools/builders/stdlib_test.go +++ b/third_party/rgo/v0_61_1/base/go/tools/builders/stdlib_test.go @@ -240,9 +240,15 @@ func TestPublishPersistedOrchestrionExportsIsDeterministic(t *testing.T) { "fmt": []byte("woven fmt archive"), "log": []byte("woven log archive"), } - cachePaths := map[string]string{ - "fmt": filepath.Join("11", "fmt-d"), - "log": filepath.Join("aa", "log-d"), + cachePaths := []map[string]string{ + { + "fmt": filepath.Join("11", "fmt-d"), + "log": filepath.Join("aa", "log-d"), + }, + { + "fmt": filepath.Join("22", "different-fmt-d"), + "log": filepath.Join("bb", "different-log-d"), + }, } inventories := make([]map[string]string, 0, 2) @@ -254,12 +260,12 @@ func TestPublishPersistedOrchestrionExportsIsDeterministic(t *testing.T) { t.Fatal(err) } exports := make(map[string]string, len(archives)) - resolved := make(map[string]string, len(cachePaths)) + resolved := make(map[string]string, len(cachePaths[run])) for pkg, data := range archives { src := filepath.Join(persisted, pkg+".a") writeTestFile(t, src, data) exports[pkg] = src - resolved[pkg] = filepath.Join(scratch, cachePaths[pkg]) + resolved[pkg] = filepath.Join(scratch, cachePaths[run][pkg]) writeTestFile(t, resolved[pkg], []byte(fmt.Sprintf("unwoven run %d", run))) } writeTestFile(t, filepath.Join(scratch, "11", "fmt-a"), []byte(fmt.Sprintf("timestamp %d", run))) @@ -274,8 +280,9 @@ func TestPublishPersistedOrchestrionExportsIsDeterministic(t *testing.T) { if len(inventory) != len(archives)+1 { t.Fatalf("declared inventory has unexpected entries: %v", inventory) } - for pkg, data := range archives { - path := filepath.ToSlash(cachePaths[pkg]) + for _, data := range archives { + digest := testDigest(data) + path := filepath.ToSlash(filepath.Join(digest[:2], digest+"-d")) if got := inventory[path]; got != testDigest(data) { t.Fatalf("published %s digest = %q, want %q", path, got, testDigest(data)) } @@ -284,7 +291,13 @@ func TestPublishPersistedOrchestrionExportsIsDeterministic(t *testing.T) { if err != nil { t.Fatal(err) } - wantManifest := "fmt=11/fmt-d\nlog=aa/log-d\n" + fmtDigest := testDigest(archives["fmt"]) + logDigest := testDigest(archives["log"]) + wantManifest := fmt.Sprintf( + "fmt=%s/%s-d\nlog=%s/%s-d\n", + fmtDigest[:2], fmtDigest, + logDigest[:2], logDigest, + ) if string(manifest) != wantManifest { t.Fatalf("manifest = %q, want %q", string(manifest), wantManifest) } @@ -308,7 +321,7 @@ func TestProjectStdlibCacheArchiveRejectsInvalidPaths(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if _, _, err := projectStdlibCacheArchive(scratch, declared, tt.archive); err == nil || !strings.Contains(err.Error(), tt.want) { + if _, _, err := projectStdlibCacheArchive(scratch, declared, tt.archive, filepath.Join(root, "persisted.a")); err == nil || !strings.Contains(err.Error(), tt.want) { t.Fatalf("projectStdlibCacheArchive error = %v, want %q", err, tt.want) } }) diff --git a/third_party/rgo/v0_62_0/base/go/tools/builders/orchestrion_cache.go b/third_party/rgo/v0_62_0/base/go/tools/builders/orchestrion_cache.go index 5cccefd7..876316ee 100644 --- a/third_party/rgo/v0_62_0/base/go/tools/builders/orchestrion_cache.go +++ b/third_party/rgo/v0_62_0/base/go/tools/builders/orchestrion_cache.go @@ -294,6 +294,15 @@ func digestFileOrMissing(path string) (string, error) { return shortDigest(data), nil } +func fullDigestFile(path string) (string, error) { + data, err := os.ReadFile(path) + if err != nil { + return "", err + } + sum := sha256.Sum256(data) + return fmt.Sprintf("%x", sum[:]), nil +} + // goSDKCacheIdentity returns a stable cache identity for the selected Go SDK // without depending on Bazel's output-base-specific execroot path. The identity // is derived from SDK file contents that remain stable across equivalent diff --git a/third_party/rgo/v0_62_0/base/go/tools/builders/stdlib.go b/third_party/rgo/v0_62_0/base/go/tools/builders/stdlib.go index 9b5c36c7..65244aa7 100644 --- a/third_party/rgo/v0_62_0/base/go/tools/builders/stdlib.go +++ b/third_party/rgo/v0_62_0/base/go/tools/builders/stdlib.go @@ -555,7 +555,7 @@ func publishPersistedOrchestrionExportsToCache(exports, cacheExports map[string] if src == "" { return fmt.Errorf("missing persisted stdlib archive for cache package %s", pkg) } - dst, relDst, err := projectStdlibCacheArchive(scratchCache, declaredCache, cacheExports[pkg]) + dst, relDst, err := projectStdlibCacheArchive(scratchCache, declaredCache, cacheExports[pkg], src) if err != nil { return fmt.Errorf("project stdlib cache archive for %s: %w", pkg, err) } @@ -585,27 +585,32 @@ func publishPersistedOrchestrionExportsToCache(exports, cacheExports map[string] return nil } -func projectStdlibCacheArchive(scratchCache, declaredCache, scratchArchive string) (destination, relative string, err error) { +func projectStdlibCacheArchive(scratchCache, declaredCache, scratchArchive, persistedArchive string) (destination, relative string, err error) { scratchRoot := abs(scratchCache) declaredRoot := abs(declaredCache) archive := abs(scratchArchive) - rel, err := filepath.Rel(scratchRoot, archive) + scratchRel, err := filepath.Rel(scratchRoot, archive) if err != nil { return "", "", err } - rel = filepath.Clean(rel) - if rel == "." || filepath.IsAbs(rel) || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + scratchRel = filepath.Clean(scratchRel) + if scratchRel == "." || filepath.IsAbs(scratchRel) || scratchRel == ".." || strings.HasPrefix(scratchRel, ".."+string(filepath.Separator)) { return "", "", fmt.Errorf("cache archive %s escapes scratch root %s", archive, scratchRoot) } - if !strings.HasSuffix(filepath.Base(rel), "-d") { + if !strings.HasSuffix(filepath.Base(scratchRel), "-d") { return "", "", fmt.Errorf("cache archive %s is not a Go cache data entry", archive) } - destination = filepath.Join(declaredRoot, rel) + digest, err := fullDigestFile(persistedArchive) + if err != nil { + return "", "", fmt.Errorf("digest persisted stdlib archive %s: %w", persistedArchive, err) + } + relative = filepath.Join(digest[:2], digest+"-d") + destination = filepath.Join(declaredRoot, relative) destinationRel, err := filepath.Rel(declaredRoot, destination) - if err != nil || filepath.Clean(destinationRel) != rel { + if err != nil || filepath.Clean(destinationRel) != relative { return "", "", fmt.Errorf("cache archive destination %s escapes declared root %s", destination, declaredRoot) } - return destination, filepath.ToSlash(rel), nil + return destination, filepath.ToSlash(relative), nil } func ensureSyntheticOrchestrionToolGo(verbose bool, orchestrionMode string) (func(), error) { diff --git a/third_party/rgo/v0_62_0/base/go/tools/builders/stdlib_test.go b/third_party/rgo/v0_62_0/base/go/tools/builders/stdlib_test.go index 7ca400cd..1d4cf608 100644 --- a/third_party/rgo/v0_62_0/base/go/tools/builders/stdlib_test.go +++ b/third_party/rgo/v0_62_0/base/go/tools/builders/stdlib_test.go @@ -240,9 +240,15 @@ func TestPublishPersistedOrchestrionExportsIsDeterministic(t *testing.T) { "fmt": []byte("woven fmt archive"), "log": []byte("woven log archive"), } - cachePaths := map[string]string{ - "fmt": filepath.Join("11", "fmt-d"), - "log": filepath.Join("aa", "log-d"), + cachePaths := []map[string]string{ + { + "fmt": filepath.Join("11", "fmt-d"), + "log": filepath.Join("aa", "log-d"), + }, + { + "fmt": filepath.Join("22", "different-fmt-d"), + "log": filepath.Join("bb", "different-log-d"), + }, } inventories := make([]map[string]string, 0, 2) @@ -254,12 +260,12 @@ func TestPublishPersistedOrchestrionExportsIsDeterministic(t *testing.T) { t.Fatal(err) } exports := make(map[string]string, len(archives)) - resolved := make(map[string]string, len(cachePaths)) + resolved := make(map[string]string, len(cachePaths[run])) for pkg, data := range archives { src := filepath.Join(persisted, pkg+".a") writeTestFile(t, src, data) exports[pkg] = src - resolved[pkg] = filepath.Join(scratch, cachePaths[pkg]) + resolved[pkg] = filepath.Join(scratch, cachePaths[run][pkg]) writeTestFile(t, resolved[pkg], []byte(fmt.Sprintf("unwoven run %d", run))) } writeTestFile(t, filepath.Join(scratch, "11", "fmt-a"), []byte(fmt.Sprintf("timestamp %d", run))) @@ -274,8 +280,9 @@ func TestPublishPersistedOrchestrionExportsIsDeterministic(t *testing.T) { if len(inventory) != len(archives)+1 { t.Fatalf("declared inventory has unexpected entries: %v", inventory) } - for pkg, data := range archives { - path := filepath.ToSlash(cachePaths[pkg]) + for _, data := range archives { + digest := testDigest(data) + path := filepath.ToSlash(filepath.Join(digest[:2], digest+"-d")) if got := inventory[path]; got != testDigest(data) { t.Fatalf("published %s digest = %q, want %q", path, got, testDigest(data)) } @@ -284,7 +291,13 @@ func TestPublishPersistedOrchestrionExportsIsDeterministic(t *testing.T) { if err != nil { t.Fatal(err) } - wantManifest := "fmt=11/fmt-d\nlog=aa/log-d\n" + fmtDigest := testDigest(archives["fmt"]) + logDigest := testDigest(archives["log"]) + wantManifest := fmt.Sprintf( + "fmt=%s/%s-d\nlog=%s/%s-d\n", + fmtDigest[:2], fmtDigest, + logDigest[:2], logDigest, + ) if string(manifest) != wantManifest { t.Fatalf("manifest = %q, want %q", string(manifest), wantManifest) } @@ -308,7 +321,7 @@ func TestProjectStdlibCacheArchiveRejectsInvalidPaths(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if _, _, err := projectStdlibCacheArchive(scratch, declared, tt.archive); err == nil || !strings.Contains(err.Error(), tt.want) { + if _, _, err := projectStdlibCacheArchive(scratch, declared, tt.archive, filepath.Join(root, "persisted.a")); err == nil || !strings.Contains(err.Error(), tt.want) { t.Fatalf("projectStdlibCacheArchive error = %v, want %q", err, tt.want) } }) diff --git a/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch index 63722fd6..21110798 100644 --- a/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch @@ -12909,10 +12909,10 @@ index 0000000..93f9e48 +} diff --git a/go/tools/builders/orchestrion_cache.go b/go/tools/builders/orchestrion_cache.go new file mode 100644 -index 0000000..5cccefd +index 0000000..876316e --- /dev/null +++ b/go/tools/builders/orchestrion_cache.go -@@ -0,0 +1,348 @@ +@@ -0,0 +1,357 @@ +package main + +import ( @@ -13209,6 +13209,15 @@ index 0000000..5cccefd + return shortDigest(data), nil +} + ++func fullDigestFile(path string) (string, error) { ++ data, err := os.ReadFile(path) ++ if err != nil { ++ return "", err ++ } ++ sum := sha256.Sum256(data) ++ return fmt.Sprintf("%x", sum[:]), nil ++} ++ +// goSDKCacheIdentity returns a stable cache identity for the selected Go SDK +// without depending on Bazel's output-base-specific execroot path. The identity +// is derived from SDK file contents that remain stable across equivalent @@ -15719,7 +15728,7 @@ index 0000000..9edae16 + } +} diff --git a/go/tools/builders/stdlib.go b/go/tools/builders/stdlib.go -index cfcb991..9b5c36c 100644 +index cfcb991..65244aa 100644 --- a/go/tools/builders/stdlib.go +++ b/go/tools/builders/stdlib.go @@ -21,15 +21,29 @@ import ( @@ -15964,7 +15973,7 @@ index cfcb991..9b5c36c 100644 } if *shared { gcflags = append(gcflags, "-shared") -@@ -167,8 +315,388 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) +@@ -167,8 +315,393 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) } installArgs = append(installArgs, packages...) @@ -16208,7 +16217,7 @@ index cfcb991..9b5c36c 100644 + if src == "" { + return fmt.Errorf("missing persisted stdlib archive for cache package %s", pkg) + } -+ dst, relDst, err := projectStdlibCacheArchive(scratchCache, declaredCache, cacheExports[pkg]) ++ dst, relDst, err := projectStdlibCacheArchive(scratchCache, declaredCache, cacheExports[pkg], src) + if err != nil { + return fmt.Errorf("project stdlib cache archive for %s: %w", pkg, err) + } @@ -16238,27 +16247,32 @@ index cfcb991..9b5c36c 100644 + return nil +} + -+func projectStdlibCacheArchive(scratchCache, declaredCache, scratchArchive string) (destination, relative string, err error) { ++func projectStdlibCacheArchive(scratchCache, declaredCache, scratchArchive, persistedArchive string) (destination, relative string, err error) { + scratchRoot := abs(scratchCache) + declaredRoot := abs(declaredCache) + archive := abs(scratchArchive) -+ rel, err := filepath.Rel(scratchRoot, archive) ++ scratchRel, err := filepath.Rel(scratchRoot, archive) + if err != nil { + return "", "", err + } -+ rel = filepath.Clean(rel) -+ if rel == "." || filepath.IsAbs(rel) || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { ++ scratchRel = filepath.Clean(scratchRel) ++ if scratchRel == "." || filepath.IsAbs(scratchRel) || scratchRel == ".." || strings.HasPrefix(scratchRel, ".."+string(filepath.Separator)) { + return "", "", fmt.Errorf("cache archive %s escapes scratch root %s", archive, scratchRoot) + } -+ if !strings.HasSuffix(filepath.Base(rel), "-d") { ++ if !strings.HasSuffix(filepath.Base(scratchRel), "-d") { + return "", "", fmt.Errorf("cache archive %s is not a Go cache data entry", archive) + } -+ destination = filepath.Join(declaredRoot, rel) ++ digest, err := fullDigestFile(persistedArchive) ++ if err != nil { ++ return "", "", fmt.Errorf("digest persisted stdlib archive %s: %w", persistedArchive, err) ++ } ++ relative = filepath.Join(digest[:2], digest+"-d") ++ destination = filepath.Join(declaredRoot, relative) + destinationRel, err := filepath.Rel(declaredRoot, destination) -+ if err != nil || filepath.Clean(destinationRel) != rel { ++ if err != nil || filepath.Clean(destinationRel) != relative { + return "", "", fmt.Errorf("cache archive destination %s escapes declared root %s", destination, declaredRoot) + } -+ return destination, filepath.ToSlash(rel), nil ++ return destination, filepath.ToSlash(relative), nil +} + +func ensureSyntheticOrchestrionToolGo(verbose bool, orchestrionMode string) (func(), error) { @@ -16355,10 +16369,10 @@ index cfcb991..9b5c36c 100644 +} diff --git a/go/tools/builders/stdlib_test.go b/go/tools/builders/stdlib_test.go new file mode 100644 -index 0000000..7ca400c +index 0000000..1d4cf60 --- /dev/null +++ b/go/tools/builders/stdlib_test.go -@@ -0,0 +1,464 @@ +@@ -0,0 +1,477 @@ +package main + +import ( @@ -16601,9 +16615,15 @@ index 0000000..7ca400c + "fmt": []byte("woven fmt archive"), + "log": []byte("woven log archive"), + } -+ cachePaths := map[string]string{ -+ "fmt": filepath.Join("11", "fmt-d"), -+ "log": filepath.Join("aa", "log-d"), ++ cachePaths := []map[string]string{ ++ { ++ "fmt": filepath.Join("11", "fmt-d"), ++ "log": filepath.Join("aa", "log-d"), ++ }, ++ { ++ "fmt": filepath.Join("22", "different-fmt-d"), ++ "log": filepath.Join("bb", "different-log-d"), ++ }, + } + + inventories := make([]map[string]string, 0, 2) @@ -16615,12 +16635,12 @@ index 0000000..7ca400c + t.Fatal(err) + } + exports := make(map[string]string, len(archives)) -+ resolved := make(map[string]string, len(cachePaths)) ++ resolved := make(map[string]string, len(cachePaths[run])) + for pkg, data := range archives { + src := filepath.Join(persisted, pkg+".a") + writeTestFile(t, src, data) + exports[pkg] = src -+ resolved[pkg] = filepath.Join(scratch, cachePaths[pkg]) ++ resolved[pkg] = filepath.Join(scratch, cachePaths[run][pkg]) + writeTestFile(t, resolved[pkg], []byte(fmt.Sprintf("unwoven run %d", run))) + } + writeTestFile(t, filepath.Join(scratch, "11", "fmt-a"), []byte(fmt.Sprintf("timestamp %d", run))) @@ -16635,8 +16655,9 @@ index 0000000..7ca400c + if len(inventory) != len(archives)+1 { + t.Fatalf("declared inventory has unexpected entries: %v", inventory) + } -+ for pkg, data := range archives { -+ path := filepath.ToSlash(cachePaths[pkg]) ++ for _, data := range archives { ++ digest := testDigest(data) ++ path := filepath.ToSlash(filepath.Join(digest[:2], digest+"-d")) + if got := inventory[path]; got != testDigest(data) { + t.Fatalf("published %s digest = %q, want %q", path, got, testDigest(data)) + } @@ -16645,7 +16666,13 @@ index 0000000..7ca400c + if err != nil { + t.Fatal(err) + } -+ wantManifest := "fmt=11/fmt-d\nlog=aa/log-d\n" ++ fmtDigest := testDigest(archives["fmt"]) ++ logDigest := testDigest(archives["log"]) ++ wantManifest := fmt.Sprintf( ++ "fmt=%s/%s-d\nlog=%s/%s-d\n", ++ fmtDigest[:2], fmtDigest, ++ logDigest[:2], logDigest, ++ ) + if string(manifest) != wantManifest { + t.Fatalf("manifest = %q, want %q", string(manifest), wantManifest) + } @@ -16669,7 +16696,7 @@ index 0000000..7ca400c + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { -+ if _, _, err := projectStdlibCacheArchive(scratch, declared, tt.archive); err == nil || !strings.Contains(err.Error(), tt.want) { ++ if _, _, err := projectStdlibCacheArchive(scratch, declared, tt.archive, filepath.Join(root, "persisted.a")); err == nil || !strings.Contains(err.Error(), tt.want) { + t.Fatalf("projectStdlibCacheArchive error = %v, want %q", err, tt.want) + } + }) diff --git a/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch index 1f265f3b..d3ea672b 100644 --- a/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch @@ -12899,10 +12899,10 @@ index 0000000..93f9e48 +} diff --git a/go/tools/builders/orchestrion_cache.go b/go/tools/builders/orchestrion_cache.go new file mode 100644 -index 0000000..5cccefd +index 0000000..876316e --- /dev/null +++ b/go/tools/builders/orchestrion_cache.go -@@ -0,0 +1,348 @@ +@@ -0,0 +1,357 @@ +package main + +import ( @@ -13199,6 +13199,15 @@ index 0000000..5cccefd + return shortDigest(data), nil +} + ++func fullDigestFile(path string) (string, error) { ++ data, err := os.ReadFile(path) ++ if err != nil { ++ return "", err ++ } ++ sum := sha256.Sum256(data) ++ return fmt.Sprintf("%x", sum[:]), nil ++} ++ +// goSDKCacheIdentity returns a stable cache identity for the selected Go SDK +// without depending on Bazel's output-base-specific execroot path. The identity +// is derived from SDK file contents that remain stable across equivalent @@ -15709,7 +15718,7 @@ index 0000000..9edae16 + } +} diff --git a/go/tools/builders/stdlib.go b/go/tools/builders/stdlib.go -index cfcb991..9b5c36c 100644 +index cfcb991..65244aa 100644 --- a/go/tools/builders/stdlib.go +++ b/go/tools/builders/stdlib.go @@ -21,15 +21,29 @@ import ( @@ -15954,7 +15963,7 @@ index cfcb991..9b5c36c 100644 } if *shared { gcflags = append(gcflags, "-shared") -@@ -167,8 +315,388 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) +@@ -167,8 +315,393 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) } installArgs = append(installArgs, packages...) @@ -16198,7 +16207,7 @@ index cfcb991..9b5c36c 100644 + if src == "" { + return fmt.Errorf("missing persisted stdlib archive for cache package %s", pkg) + } -+ dst, relDst, err := projectStdlibCacheArchive(scratchCache, declaredCache, cacheExports[pkg]) ++ dst, relDst, err := projectStdlibCacheArchive(scratchCache, declaredCache, cacheExports[pkg], src) + if err != nil { + return fmt.Errorf("project stdlib cache archive for %s: %w", pkg, err) + } @@ -16228,27 +16237,32 @@ index cfcb991..9b5c36c 100644 + return nil +} + -+func projectStdlibCacheArchive(scratchCache, declaredCache, scratchArchive string) (destination, relative string, err error) { ++func projectStdlibCacheArchive(scratchCache, declaredCache, scratchArchive, persistedArchive string) (destination, relative string, err error) { + scratchRoot := abs(scratchCache) + declaredRoot := abs(declaredCache) + archive := abs(scratchArchive) -+ rel, err := filepath.Rel(scratchRoot, archive) ++ scratchRel, err := filepath.Rel(scratchRoot, archive) + if err != nil { + return "", "", err + } -+ rel = filepath.Clean(rel) -+ if rel == "." || filepath.IsAbs(rel) || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { ++ scratchRel = filepath.Clean(scratchRel) ++ if scratchRel == "." || filepath.IsAbs(scratchRel) || scratchRel == ".." || strings.HasPrefix(scratchRel, ".."+string(filepath.Separator)) { + return "", "", fmt.Errorf("cache archive %s escapes scratch root %s", archive, scratchRoot) + } -+ if !strings.HasSuffix(filepath.Base(rel), "-d") { ++ if !strings.HasSuffix(filepath.Base(scratchRel), "-d") { + return "", "", fmt.Errorf("cache archive %s is not a Go cache data entry", archive) + } -+ destination = filepath.Join(declaredRoot, rel) ++ digest, err := fullDigestFile(persistedArchive) ++ if err != nil { ++ return "", "", fmt.Errorf("digest persisted stdlib archive %s: %w", persistedArchive, err) ++ } ++ relative = filepath.Join(digest[:2], digest+"-d") ++ destination = filepath.Join(declaredRoot, relative) + destinationRel, err := filepath.Rel(declaredRoot, destination) -+ if err != nil || filepath.Clean(destinationRel) != rel { ++ if err != nil || filepath.Clean(destinationRel) != relative { + return "", "", fmt.Errorf("cache archive destination %s escapes declared root %s", destination, declaredRoot) + } -+ return destination, filepath.ToSlash(rel), nil ++ return destination, filepath.ToSlash(relative), nil +} + +func ensureSyntheticOrchestrionToolGo(verbose bool, orchestrionMode string) (func(), error) { @@ -16345,10 +16359,10 @@ index cfcb991..9b5c36c 100644 +} diff --git a/go/tools/builders/stdlib_test.go b/go/tools/builders/stdlib_test.go new file mode 100644 -index 0000000..7ca400c +index 0000000..1d4cf60 --- /dev/null +++ b/go/tools/builders/stdlib_test.go -@@ -0,0 +1,464 @@ +@@ -0,0 +1,477 @@ +package main + +import ( @@ -16591,9 +16605,15 @@ index 0000000..7ca400c + "fmt": []byte("woven fmt archive"), + "log": []byte("woven log archive"), + } -+ cachePaths := map[string]string{ -+ "fmt": filepath.Join("11", "fmt-d"), -+ "log": filepath.Join("aa", "log-d"), ++ cachePaths := []map[string]string{ ++ { ++ "fmt": filepath.Join("11", "fmt-d"), ++ "log": filepath.Join("aa", "log-d"), ++ }, ++ { ++ "fmt": filepath.Join("22", "different-fmt-d"), ++ "log": filepath.Join("bb", "different-log-d"), ++ }, + } + + inventories := make([]map[string]string, 0, 2) @@ -16605,12 +16625,12 @@ index 0000000..7ca400c + t.Fatal(err) + } + exports := make(map[string]string, len(archives)) -+ resolved := make(map[string]string, len(cachePaths)) ++ resolved := make(map[string]string, len(cachePaths[run])) + for pkg, data := range archives { + src := filepath.Join(persisted, pkg+".a") + writeTestFile(t, src, data) + exports[pkg] = src -+ resolved[pkg] = filepath.Join(scratch, cachePaths[pkg]) ++ resolved[pkg] = filepath.Join(scratch, cachePaths[run][pkg]) + writeTestFile(t, resolved[pkg], []byte(fmt.Sprintf("unwoven run %d", run))) + } + writeTestFile(t, filepath.Join(scratch, "11", "fmt-a"), []byte(fmt.Sprintf("timestamp %d", run))) @@ -16625,8 +16645,9 @@ index 0000000..7ca400c + if len(inventory) != len(archives)+1 { + t.Fatalf("declared inventory has unexpected entries: %v", inventory) + } -+ for pkg, data := range archives { -+ path := filepath.ToSlash(cachePaths[pkg]) ++ for _, data := range archives { ++ digest := testDigest(data) ++ path := filepath.ToSlash(filepath.Join(digest[:2], digest+"-d")) + if got := inventory[path]; got != testDigest(data) { + t.Fatalf("published %s digest = %q, want %q", path, got, testDigest(data)) + } @@ -16635,7 +16656,13 @@ index 0000000..7ca400c + if err != nil { + t.Fatal(err) + } -+ wantManifest := "fmt=11/fmt-d\nlog=aa/log-d\n" ++ fmtDigest := testDigest(archives["fmt"]) ++ logDigest := testDigest(archives["log"]) ++ wantManifest := fmt.Sprintf( ++ "fmt=%s/%s-d\nlog=%s/%s-d\n", ++ fmtDigest[:2], fmtDigest, ++ logDigest[:2], logDigest, ++ ) + if string(manifest) != wantManifest { + t.Fatalf("manifest = %q, want %q", string(manifest), wantManifest) + } @@ -16659,7 +16686,7 @@ index 0000000..7ca400c + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { -+ if _, _, err := projectStdlibCacheArchive(scratch, declared, tt.archive); err == nil || !strings.Contains(err.Error(), tt.want) { ++ if _, _, err := projectStdlibCacheArchive(scratch, declared, tt.archive, filepath.Join(root, "persisted.a")); err == nil || !strings.Contains(err.Error(), tt.want) { + t.Fatalf("projectStdlibCacheArchive error = %v, want %q", err, tt.want) + } + }) diff --git a/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch index f5f4943f..c75e9f93 100644 --- a/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch @@ -12903,10 +12903,10 @@ index 0000000..93f9e48 +} diff --git a/go/tools/builders/orchestrion_cache.go b/go/tools/builders/orchestrion_cache.go new file mode 100644 -index 0000000..5cccefd +index 0000000..876316e --- /dev/null +++ b/go/tools/builders/orchestrion_cache.go -@@ -0,0 +1,348 @@ +@@ -0,0 +1,357 @@ +package main + +import ( @@ -13203,6 +13203,15 @@ index 0000000..5cccefd + return shortDigest(data), nil +} + ++func fullDigestFile(path string) (string, error) { ++ data, err := os.ReadFile(path) ++ if err != nil { ++ return "", err ++ } ++ sum := sha256.Sum256(data) ++ return fmt.Sprintf("%x", sum[:]), nil ++} ++ +// goSDKCacheIdentity returns a stable cache identity for the selected Go SDK +// without depending on Bazel's output-base-specific execroot path. The identity +// is derived from SDK file contents that remain stable across equivalent @@ -15713,7 +15722,7 @@ index 0000000..9edae16 + } +} diff --git a/go/tools/builders/stdlib.go b/go/tools/builders/stdlib.go -index cfcb991..9b5c36c 100644 +index cfcb991..65244aa 100644 --- a/go/tools/builders/stdlib.go +++ b/go/tools/builders/stdlib.go @@ -21,15 +21,29 @@ import ( @@ -15958,7 +15967,7 @@ index cfcb991..9b5c36c 100644 } if *shared { gcflags = append(gcflags, "-shared") -@@ -167,8 +315,388 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) +@@ -167,8 +315,393 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) } installArgs = append(installArgs, packages...) @@ -16202,7 +16211,7 @@ index cfcb991..9b5c36c 100644 + if src == "" { + return fmt.Errorf("missing persisted stdlib archive for cache package %s", pkg) + } -+ dst, relDst, err := projectStdlibCacheArchive(scratchCache, declaredCache, cacheExports[pkg]) ++ dst, relDst, err := projectStdlibCacheArchive(scratchCache, declaredCache, cacheExports[pkg], src) + if err != nil { + return fmt.Errorf("project stdlib cache archive for %s: %w", pkg, err) + } @@ -16232,27 +16241,32 @@ index cfcb991..9b5c36c 100644 + return nil +} + -+func projectStdlibCacheArchive(scratchCache, declaredCache, scratchArchive string) (destination, relative string, err error) { ++func projectStdlibCacheArchive(scratchCache, declaredCache, scratchArchive, persistedArchive string) (destination, relative string, err error) { + scratchRoot := abs(scratchCache) + declaredRoot := abs(declaredCache) + archive := abs(scratchArchive) -+ rel, err := filepath.Rel(scratchRoot, archive) ++ scratchRel, err := filepath.Rel(scratchRoot, archive) + if err != nil { + return "", "", err + } -+ rel = filepath.Clean(rel) -+ if rel == "." || filepath.IsAbs(rel) || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { ++ scratchRel = filepath.Clean(scratchRel) ++ if scratchRel == "." || filepath.IsAbs(scratchRel) || scratchRel == ".." || strings.HasPrefix(scratchRel, ".."+string(filepath.Separator)) { + return "", "", fmt.Errorf("cache archive %s escapes scratch root %s", archive, scratchRoot) + } -+ if !strings.HasSuffix(filepath.Base(rel), "-d") { ++ if !strings.HasSuffix(filepath.Base(scratchRel), "-d") { + return "", "", fmt.Errorf("cache archive %s is not a Go cache data entry", archive) + } -+ destination = filepath.Join(declaredRoot, rel) ++ digest, err := fullDigestFile(persistedArchive) ++ if err != nil { ++ return "", "", fmt.Errorf("digest persisted stdlib archive %s: %w", persistedArchive, err) ++ } ++ relative = filepath.Join(digest[:2], digest+"-d") ++ destination = filepath.Join(declaredRoot, relative) + destinationRel, err := filepath.Rel(declaredRoot, destination) -+ if err != nil || filepath.Clean(destinationRel) != rel { ++ if err != nil || filepath.Clean(destinationRel) != relative { + return "", "", fmt.Errorf("cache archive destination %s escapes declared root %s", destination, declaredRoot) + } -+ return destination, filepath.ToSlash(rel), nil ++ return destination, filepath.ToSlash(relative), nil +} + +func ensureSyntheticOrchestrionToolGo(verbose bool, orchestrionMode string) (func(), error) { @@ -16349,10 +16363,10 @@ index cfcb991..9b5c36c 100644 +} diff --git a/go/tools/builders/stdlib_test.go b/go/tools/builders/stdlib_test.go new file mode 100644 -index 0000000..7ca400c +index 0000000..1d4cf60 --- /dev/null +++ b/go/tools/builders/stdlib_test.go -@@ -0,0 +1,464 @@ +@@ -0,0 +1,477 @@ +package main + +import ( @@ -16595,9 +16609,15 @@ index 0000000..7ca400c + "fmt": []byte("woven fmt archive"), + "log": []byte("woven log archive"), + } -+ cachePaths := map[string]string{ -+ "fmt": filepath.Join("11", "fmt-d"), -+ "log": filepath.Join("aa", "log-d"), ++ cachePaths := []map[string]string{ ++ { ++ "fmt": filepath.Join("11", "fmt-d"), ++ "log": filepath.Join("aa", "log-d"), ++ }, ++ { ++ "fmt": filepath.Join("22", "different-fmt-d"), ++ "log": filepath.Join("bb", "different-log-d"), ++ }, + } + + inventories := make([]map[string]string, 0, 2) @@ -16609,12 +16629,12 @@ index 0000000..7ca400c + t.Fatal(err) + } + exports := make(map[string]string, len(archives)) -+ resolved := make(map[string]string, len(cachePaths)) ++ resolved := make(map[string]string, len(cachePaths[run])) + for pkg, data := range archives { + src := filepath.Join(persisted, pkg+".a") + writeTestFile(t, src, data) + exports[pkg] = src -+ resolved[pkg] = filepath.Join(scratch, cachePaths[pkg]) ++ resolved[pkg] = filepath.Join(scratch, cachePaths[run][pkg]) + writeTestFile(t, resolved[pkg], []byte(fmt.Sprintf("unwoven run %d", run))) + } + writeTestFile(t, filepath.Join(scratch, "11", "fmt-a"), []byte(fmt.Sprintf("timestamp %d", run))) @@ -16629,8 +16649,9 @@ index 0000000..7ca400c + if len(inventory) != len(archives)+1 { + t.Fatalf("declared inventory has unexpected entries: %v", inventory) + } -+ for pkg, data := range archives { -+ path := filepath.ToSlash(cachePaths[pkg]) ++ for _, data := range archives { ++ digest := testDigest(data) ++ path := filepath.ToSlash(filepath.Join(digest[:2], digest+"-d")) + if got := inventory[path]; got != testDigest(data) { + t.Fatalf("published %s digest = %q, want %q", path, got, testDigest(data)) + } @@ -16639,7 +16660,13 @@ index 0000000..7ca400c + if err != nil { + t.Fatal(err) + } -+ wantManifest := "fmt=11/fmt-d\nlog=aa/log-d\n" ++ fmtDigest := testDigest(archives["fmt"]) ++ logDigest := testDigest(archives["log"]) ++ wantManifest := fmt.Sprintf( ++ "fmt=%s/%s-d\nlog=%s/%s-d\n", ++ fmtDigest[:2], fmtDigest, ++ logDigest[:2], logDigest, ++ ) + if string(manifest) != wantManifest { + t.Fatalf("manifest = %q, want %q", string(manifest), wantManifest) + } @@ -16663,7 +16690,7 @@ index 0000000..7ca400c + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { -+ if _, _, err := projectStdlibCacheArchive(scratch, declared, tt.archive); err == nil || !strings.Contains(err.Error(), tt.want) { ++ if _, _, err := projectStdlibCacheArchive(scratch, declared, tt.archive, filepath.Join(root, "persisted.a")); err == nil || !strings.Contains(err.Error(), tt.want) { + t.Fatalf("projectStdlibCacheArchive error = %v, want %q", err, tt.want) + } + }) diff --git a/tools/dev/verify_rules_go_profiles.py b/tools/dev/verify_rules_go_profiles.py index e24011c5..ce89db92 100644 --- a/tools/dev/verify_rules_go_profiles.py +++ b/tools/dev/verify_rules_go_profiles.py @@ -236,9 +236,33 @@ def verify_workspace_runtime_functional_smoke( assert_orchestrion_stdlib_cache(first_orchestrion, patch) assert_orchestrion_stdlib_cache(second_orchestrion, patch) if first_plain != second_plain: - raise ValueError("plain stdlib cache inventories differ for %s" % patch) + raise ValueError( + "plain stdlib cache inventories differ for %s: %s" + % (patch, describe_snapshot_difference(first_plain, second_plain)) + ) if first_orchestrion != second_orchestrion: - raise ValueError("Test Optimization stdlib cache inventories differ for %s" % patch) + raise ValueError( + "Test Optimization stdlib cache inventories differ for %s: %s" + % ( + patch, + describe_snapshot_difference(first_orchestrion, second_orchestrion), + ) + ) + + +def describe_snapshot_difference( + first: StdlibCacheSnapshot, second: StdlibCacheSnapshot +) -> str: + """Describe canonical cache entries that differ between two executions.""" + differences = [] + for relative in sorted(set(first.inventory) | set(second.inventory)): + first_value = first.inventory.get(relative, "missing") + second_value = second.inventory.get(relative, "missing") + if first_value != second_value: + differences.append("%s=(%s != %s)" % (relative, first_value, second_value)) + if first.manifest != second.manifest: + differences.append("manifest contents differ") + return ", ".join(differences) or "snapshot metadata differs" def write_smoke_workspace( From 3353f0c589429b63ac4e147aa3387b380be413fc Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Wed, 26 Aug 2026 23:30:53 +0200 Subject: [PATCH 29/37] uploader: preserve partial upload progress --- docs/Uploader_Reference.md | 7 +- tools/core/uploader_bash_runtime.sh.tpl | 51 +++++++- .../core/uploader_powershell_runtime.ps1.tpl | 39 +++++- .../run_test_optimization_ci.ps1 | 6 +- .../run_test_optimization_ci.sh | 6 +- tools/tests/python/test_python_tools.py | 119 ++++++++++++++---- 6 files changed, 193 insertions(+), 35 deletions(-) diff --git a/docs/Uploader_Reference.md b/docs/Uploader_Reference.md index ffb78719..d20f9077 100644 --- a/docs/Uploader_Reference.md +++ b/docs/Uploader_Reference.md @@ -674,8 +674,11 @@ payload discovery/quiescence before proceeding. limit retain the existing unsplit upload path; larger payloads fail locally. PowerShell performs the same split with its built-in JSON support. - Split parts are uploaded in event order and retry independently. A failed - part does not prevent the uploader from attempting the remaining parts. The - source payload is reported as failed and retained unless every part succeeds. + part does not prevent the uploader from attempting the remaining parts. After + partial success, the retry payload is replaced with only the failed parts so + accepted events are not replayed. If every part fails, retry persistence + fails, or payload retention is explicitly enabled, the original source is + retained. - Terminal test-upload failures always log the HTTP status, up to 2,000 characters of the response body, and the uncompressed, compressed, and transmitted byte counts. Response logging does not require debug mode. diff --git a/tools/core/uploader_bash_runtime.sh.tpl b/tools/core/uploader_bash_runtime.sh.tpl index 5372cd2e..a0775f0a 100644 --- a/tools/core/uploader_bash_runtime.sh.tpl +++ b/tools/core/uploader_bash_runtime.sh.tpl @@ -3601,8 +3601,13 @@ test_output_dir_is_freshness_eligible() { log_freshness_skip_once "$outputs_dir" "BEP reported cached result for target $target_label output $output_key" elif [[ "$FRESHNESS_SELECTED_SOURCE" == "bep" && "$FRESHNESS_MODE" == "required" ]]; then if [[ -n "$FRESHNESS_MISSING_OUTPUT_LABELS_FILE" ]] && grep -Fxq "$target_label" "$FRESHNESS_MISSING_OUTPUT_LABELS_FILE" 2>/dev/null; then - log "error: BEP required freshness cannot authorize $outputs_dir because the fresh TestResult for $target_label did not contain a mappable test.outputs reference. Rerun with --remote_download_minimal --remote_download_regex=.*test[.]outputs.* and inspect the BEP testActionOutput entries. If the test run used --zip_undeclared_test_outputs, rerun the uploader with --artifact-source=bep." - exit 2 + log_freshness_skip_once "$outputs_dir" "fresh BEP TestResult for $target_label did not contain a mappable test.outputs reference" + if (( FRESHNESS_SKIP_WAS_EMITTED == 1 )); then + log "warning: BEP required freshness skipped $outputs_dir because the fresh TestResult for $target_label did not contain a mappable test.outputs reference. Rerun with --remote_download_minimal --remote_download_regex=.*test[.]outputs.* and inspect the BEP testActionOutput entries. If the test run used --zip_undeclared_test_outputs, rerun the uploader with --artifact-source=bep." + if (( EXPECTED_TARGETS_CONFIGURED == 0 )); then + ((++UPLOAD_FAILURES)) + fi + fi else log_freshness_skip_once "$outputs_dir" "no fresh BEP TestResult matched target $target_label output $output_key" fi @@ -4919,6 +4924,43 @@ prepare_test_payload_parts() { return 0 } +persist_failed_test_payload_parts() { + local source_file="$1" + shift + local retry_file failed_count=$# + if (( failed_count == 0 )) || [[ "$KEEP_PAYLOADS" == "1" ]]; then + return 0 + fi + retry_file="$(mktemp "${source_file}.retry.XXXXXX" 2>/dev/null || true)" + if [[ -z "$retry_file" ]]; then + log "warning: failed to create retry payload beside source; retaining the original payload: $source_file" + return 1 + fi + if (( failed_count == 1 )); then + if ! cp "$1" "$retry_file"; then + rm -f "$retry_file" 2>/dev/null || true + log "warning: failed to retain the rejected split payload part; retaining the original payload: $source_file" + return 1 + fi + elif ! jq -cs ' + .[0] as $first + | (reduce .[] as $part ([]; . + ($part.events // []))) as $events + | $first + | .events = $events + ' "$@" >"$retry_file"; then + rm -f "$retry_file" 2>/dev/null || true + log "warning: failed to combine rejected split payload parts; retaining the original payload: $source_file" + return 1 + fi + if ! mv -f "$retry_file" "$source_file"; then + rm -f "$retry_file" 2>/dev/null || true + log "warning: failed to replace the source with rejected split payload parts; retaining the original payload: $source_file" + return 1 + fi + log "retained $failed_count failed split payload part(s) for retry: $source_file" + return 0 +} + bounded_upload_response() { local response_file="$1" head -c "$UPLOAD_RESPONSE_LOG_BYTES" "$response_file" 2>/dev/null | tr '\r\n' ' ' @@ -5017,6 +5059,7 @@ send_test_payload_part() { upload_single_test() { local file="$1" local body part part_index=0 part_count failed=0 + local failed_parts=() body="$(mktemp "$TMP_PAYLOAD_DIR/test_payload.XXXXXX" 2>/dev/null || true)" if [[ -z "$body" ]]; then log "error: failed to create temporary test payload: $file" @@ -5044,8 +5087,12 @@ upload_single_test() { ((++part_index)) if ! send_test_payload_part "$file" "$part" "$part_index" "$part_count"; then failed=1 + failed_parts+=("$part") fi done + if (( failed == 1 && ${#failed_parts[@]} < part_count )); then + persist_failed_test_payload_parts "$file" "${failed_parts[@]}" || true + fi cleanup_prepared_test_payloads (( failed == 0 )) } diff --git a/tools/core/uploader_powershell_runtime.ps1.tpl b/tools/core/uploader_powershell_runtime.ps1.tpl index 23abb53a..b6f9844e 100644 --- a/tools/core/uploader_powershell_runtime.ps1.tpl +++ b/tools/core/uploader_powershell_runtime.ps1.tpl @@ -3330,8 +3330,13 @@ function Test-OutputDirFreshnessEligible([string]$OutputsDir) { Write-FreshnessSkipOnce $OutputsDir "BEP reported cached result for target $targetLabel output $outputKey" } elseif ($script:FreshnessSelectedSource -eq "bep" -and $script:FreshnessMode -eq "required") { if ($script:FreshnessMissingOutputLabels.Contains($targetLabel)) { - Log "error: BEP required freshness cannot authorize $OutputsDir because the fresh TestResult for $targetLabel did not contain a mappable test.outputs reference. Rerun with --remote_download_minimal --remote_download_regex=.*test[.]outputs.* and inspect the BEP testActionOutput entries. If the test run used --zip_undeclared_test_outputs, rerun the uploader with --artifact-source=bep." - exit 2 + Write-FreshnessSkipOnce $OutputsDir "fresh BEP TestResult for $targetLabel did not contain a mappable test.outputs reference" + if ($script:FreshnessSkipWasWritten) { + Log "warning: BEP required freshness skipped $OutputsDir because the fresh TestResult for $targetLabel did not contain a mappable test.outputs reference. Rerun with --remote_download_minimal --remote_download_regex=.*test[.]outputs.* and inspect the BEP testActionOutput entries. If the test run used --zip_undeclared_test_outputs, rerun the uploader with --artifact-source=bep." + if (-not $script:ExpectedTargetsConfigured) { + $script:UploadFailures++ + } + } } else { Write-FreshnessSkipOnce $OutputsDir "no fresh BEP TestResult matched target $targetLabel output $outputKey" } @@ -3975,6 +3980,31 @@ function Prepare-TestPayloadParts([string]$BodyPath, [string]$SourcePath) { return [bool]$true } +function Save-FailedTestPayloadParts([string]$SourcePath, [string[]]$FailedParts) { + if ($KeepPayloads -or $null -eq $FailedParts -or $FailedParts.Count -eq 0) { return } + $retryPath = Join-Path (Split-Path -Parent $SourcePath) ((Split-Path -Leaf $SourcePath) + ".retry." + [System.Guid]::NewGuid().ToString("N")) + try { + $retryPayload = Get-Content -LiteralPath $FailedParts[0] -Raw -Encoding UTF8 | ConvertFrom-Json -ErrorAction Stop + if ($FailedParts.Count -gt 1) { + $failedEvents = [System.Collections.Generic.List[object]]::new() + foreach ($partPath in $FailedParts) { + $partPayload = Get-Content -LiteralPath $partPath -Raw -Encoding UTF8 | ConvertFrom-Json -ErrorAction Stop + foreach ($event in @(Get-MapValue $partPayload 'events')) { + $failedEvents.Add($event) | Out-Null + } + } + $retryPayload.events = @($failedEvents.ToArray()) + } + Write-Utf8NoBomFile -Path $retryPath -Content (($retryPayload | ConvertTo-Json -Depth 100 -Compress) + "`n") + [System.IO.File]::Move($retryPath, $SourcePath, $true) + Log "retained $($FailedParts.Count) failed split payload part(s) for retry: $SourcePath" + } catch { + Log "warning: failed to retain rejected split payload parts; retaining the original payload '$SourcePath': $_" + } finally { + Remove-Item -LiteralPath $retryPath -Force -ErrorAction SilentlyContinue + } +} + function Get-TelemetryHeaders([string]$FilePath) { try { $payloadObj = Get-Content -LiteralPath $FilePath -Raw -Encoding UTF8 | ConvertFrom-Json -ErrorAction Stop @@ -4622,6 +4652,7 @@ function Upload-SingleTest([string]$FilePath) { } $partCount = $script:PreparedTestPayloads.Count $failed = $false + $failedParts = [System.Collections.Generic.List[string]]::new() try { for ($index = 0; $index -lt $partCount; $index++) { $part = $script:PreparedTestPayloads[$index] @@ -4634,8 +4665,12 @@ function Upload-SingleTest([string]$FilePath) { $resultStream = @(Send-PostJson $TestUrl $hdrs $part $FilePath ($index + 1) $partCount) if ($resultStream.Count -eq 0 -or -not [bool]$resultStream[-1]) { $failed = $true + $failedParts.Add($part) | Out-Null } } + if ($failed -and $failedParts.Count -lt $partCount) { + Save-FailedTestPayloadParts $FilePath @($failedParts.ToArray()) + } } finally { Clear-PreparedTestPayloads } diff --git a/tools/test_optimization/run_test_optimization_ci.ps1 b/tools/test_optimization/run_test_optimization_ci.ps1 index 2b773a85..8f60d5c1 100644 --- a/tools/test_optimization/run_test_optimization_ci.ps1 +++ b/tools/test_optimization/run_test_optimization_ci.ps1 @@ -92,9 +92,9 @@ if (-not [string]::IsNullOrWhiteSpace($ReportDir)) { $UploaderReportJson = Join-Path $ReportDir "uploader-dry-run-report.json" } } - if ($Upload.IsPresent) { - $UploadReportJson = $UploaderReportJson - } +} +if ($Upload.IsPresent) { + $UploadReportJson = $UploaderReportJson } $keepGeneratedFiles = $KeepTmp.IsPresent -or $env:DD_TEST_OPTIMIZATION_KEEP_TMP -eq "1" diff --git a/tools/test_optimization/run_test_optimization_ci.sh b/tools/test_optimization/run_test_optimization_ci.sh index d9a2ccb7..9c3b295b 100755 --- a/tools/test_optimization/run_test_optimization_ci.sh +++ b/tools/test_optimization/run_test_optimization_ci.sh @@ -192,9 +192,9 @@ if [[ -n "$REPORT_DIR" ]]; then UPLOADER_REPORT_JSON="$REPORT_DIR/uploader-dry-run-report.json" fi fi - if [[ "$DO_UPLOAD" -eq 1 ]]; then - UPLOAD_REPORT_JSON="$UPLOADER_REPORT_JSON" - fi +fi +if [[ "$DO_UPLOAD" -eq 1 ]]; then + UPLOAD_REPORT_JSON="$UPLOADER_REPORT_JSON" fi command_manifest_json="$tmp_root/support-command-manifest.json" diff --git a/tools/tests/python/test_python_tools.py b/tools/tests/python/test_python_tools.py index 00024382..709dd79b 100644 --- a/tools/tests/python/test_python_tools.py +++ b/tools/tests/python/test_python_tools.py @@ -605,9 +605,17 @@ def test_bash_wrapper_uploads_after_partial_validation_failures(self) -> None: env = os.environ.copy() env["BAZEL"] = str(fake_bazel) env["DD_TEST_OPTIMIZATION_TMPDIR"] = str(tmpdir) + uploader_report = root / "custom-uploader-report.json" result = subprocess.run( - [bash, str(wrapper), "--upload", "//pkg:target"], + [ + bash, + str(wrapper), + "--upload", + "--uploader-report-json", + str(uploader_report), + "//pkg:target", + ], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, @@ -624,6 +632,7 @@ def test_bash_wrapper_uploads_after_partial_validation_failures(self) -> None: self.assertEqual(1, len(upload_commands), upload_commands) self.assertIn("--validate-enrichment", upload_commands[0]) self.assertNotIn("--dry-run", upload_commands[0]) + self.assertIn(f"--report-json={uploader_report}", upload_commands[0]) def test_bash_wrapper_support_bundle_preserves_failed_test_status(self) -> None: """Validate Bash support bundle path preserves the original test status.""" @@ -949,6 +958,8 @@ def test_powershell_wrapper_uploads_after_partial_validation_failures(self) -> N tmpdir.mkdir() env = os.environ.copy() env["DD_TEST_OPTIMIZATION_TMPDIR"] = str(tmpdir) + uploader_report = root / "custom-uploader-report.json" + env["DD_TEST_OPTIMIZATION_UPLOADER_REPORT_JSON"] = str(uploader_report) result = subprocess.run( [ @@ -980,6 +991,7 @@ def test_powershell_wrapper_uploads_after_partial_validation_failures(self) -> N self.assertEqual(1, len(upload_commands), upload_commands) self.assertIn("--validate-enrichment", upload_commands[0]) self.assertNotIn("--dry-run", upload_commands[0]) + self.assertIn(f"--report-json={uploader_report}", upload_commands[0]) def test_powershell_wrapper_support_bundle_preserves_failed_test_status(self) -> None: """Validate PowerShell support bundle path preserves the original test status.""" @@ -6277,12 +6289,15 @@ def _write_bep_staging_smoke_fixture(root: Path) -> Path: @staticmethod def _write_mixed_fresh_output_fixture(root: Path) -> tuple[Path, Path]: - """Create one payload-producing and one empty fresh expected output.""" + """Create valid, empty, and locally stale unmappable expected outputs.""" + unmappable_output = root / "bazel-testlogs" / "pkg" / "a_unmappable" / "test.outputs" valid_output = root / "bazel-testlogs" / "pkg" / "valid" / "test.outputs" empty_output = root / "bazel-testlogs" / "pkg" / "empty" / "test.outputs" + unmappable_output.mkdir(parents=True) valid_output.mkdir(parents=True) empty_output.mkdir(parents=True) for output_dir, label in ( + (unmappable_output, "//pkg:a_unmappable"), (valid_output, "//pkg:valid"), (empty_output, "//pkg:empty"), ): @@ -6290,6 +6305,17 @@ def _write_mixed_fresh_output_fixture(root: Path) -> tuple[Path, Path]: json.dumps({"bazel.target": label}), encoding="utf-8", ) + unmappable_payload_dir = unmappable_output / "payloads" / "tests" + unmappable_payload_dir.mkdir(parents=True) + (unmappable_payload_dir / "span_events_stale.json").write_text( + json.dumps({ + "events": [{ + "type": "test", + "content": {"resource": "pkg.stale", "meta": {}, "metrics": {}}, + }], + }), + encoding="utf-8", + ) payload_dir = valid_output / "payloads" / "tests" payload_dir.mkdir(parents=True) (payload_dir / "span_events_valid.json").write_text( @@ -6305,6 +6331,20 @@ def _write_mixed_fresh_output_fixture(root: Path) -> tuple[Path, Path]: TestOptimizationDoctorTests._write_bep( bep, [ + { + "id": { + "testResult": { + "label": "//pkg:a_unmappable", + "run": 1, + "shard": 1, + "attempt": 1, + }, + }, + "testResult": { + "status": "PASSED", + "testActionOutput": [{"name": "test.log", "uri": "file:///tmp/test.log"}], + }, + }, TestOptimizationDoctorTests._bep_test_result( "//pkg:empty", empty_output.as_uri(), @@ -6319,7 +6359,7 @@ def _write_mixed_fresh_output_fixture(root: Path) -> tuple[Path, Path]: expected_targets.write_text( json.dumps({ "schema_version": 1, - "targets": ["//pkg:empty", "//pkg:missing", "//pkg:valid"], + "targets": ["//pkg:a_unmappable", "//pkg:empty", "//pkg:missing", "//pkg:valid"], }), encoding="utf-8", ) @@ -6390,7 +6430,11 @@ def _run_local_test_upload( gzip_enabled: bool, keep_payloads: bool = True, read_only_parent: bool = False, - ) -> tuple[subprocess.CompletedProcess[str], list[dict[str, object]], bool]: + ) -> tuple[ + subprocess.CompletedProcess[str], + list[dict[str, object]], + Optional[dict[str, object]], + ]: """Run one generated uploader against a recording local test intake.""" bash = _require_functional_bash(self) pwsh = _require_command(self, "pwsh", "pwsh is required for generated uploader parity") @@ -6466,7 +6510,10 @@ def _run_local_test_upload( timeout=45, check=False, ) - return result, list(records), source.is_file() + retained_payload = None + if source.is_file(): + retained_payload = json.loads(source.read_text(encoding="utf-8")) + return result, list(records), retained_payload finally: if read_only_parent and payload_dir.exists(): payload_dir.chmod(0o755) @@ -6502,7 +6549,7 @@ def reject(_payload, _records): for runtime in ("Bash", "PowerShell"): for gzip_enabled in (False, True): with self.subTest(runtime=runtime, gzip=gzip_enabled): - result, records, source_exists = self._run_local_test_upload( + result, records, retained_payload = self._run_local_test_upload( runtime, payload, reject, @@ -6510,7 +6557,7 @@ def reject(_payload, _records): ) output = result.stdout + result.stderr self.assertNotEqual(0, result.returncode, output) - self.assertTrue(source_exists) + self.assertIsNotNone(retained_payload) self.assertGreaterEqual(len(records), 1) self.assertIn("http=413", output) self.assertIn("uncompressed_bytes=", output) @@ -6544,7 +6591,7 @@ def retry_second_part(candidate, records): for runtime in ("Bash", "PowerShell"): for gzip_enabled in (False, True): with self.subTest(runtime=runtime, gzip=gzip_enabled): - result, records, source_exists = self._run_local_test_upload( + result, records, retained_payload = self._run_local_test_upload( runtime, payload, retry_second_part, @@ -6553,7 +6600,7 @@ def retry_second_part(candidate, records): ) output = result.stdout + result.stderr self.assertEqual(0, result.returncode, output) - self.assertFalse(source_exists) + self.assertIsNone(retained_payload) self.assertIn("parts=2", output) successful = [record for record in records if record["status"] == 200] self.assertEqual(2, len(successful), records) @@ -6584,7 +6631,7 @@ def accept(_payload, _records): for runtime in ("Bash", "PowerShell"): with self.subTest(runtime=runtime): - result, records, source_exists = self._run_local_test_upload( + result, records, retained_payload = self._run_local_test_upload( runtime, payload, accept, @@ -6593,7 +6640,7 @@ def accept(_payload, _records): ) output = result.stdout + result.stderr self.assertEqual(0, result.returncode, output) - self.assertTrue(source_exists) + self.assertIsNotNone(retained_payload) self.assertEqual([], records) self.assertIn("dry-run would split test payload", output) self.assertIn("into 2 parts", output) @@ -6607,7 +6654,7 @@ def accept(_payload, _records): for runtime in ("Bash", "PowerShell"): with self.subTest(runtime=runtime, result="valid"): - result, records, source_exists = self._run_local_test_upload( + result, records, retained_payload = self._run_local_test_upload( runtime, payload, accept, @@ -6618,13 +6665,13 @@ def accept(_payload, _records): ) output = result.stdout + result.stderr self.assertEqual(0, result.returncode, output) - self.assertFalse(source_exists) + self.assertIsNone(retained_payload) self.assertEqual(1, len(records), records) self.assertIn("validated enriched test payload", output) self.assertNotIn("dry-run validated enriched test payload", output) with self.subTest(runtime=runtime, result="missing-tag"): - result, records, source_exists = self._run_local_test_upload( + result, records, retained_payload = self._run_local_test_upload( runtime, payload, accept, @@ -6634,12 +6681,12 @@ def accept(_payload, _records): ) output = result.stdout + result.stderr self.assertNotEqual(0, result.returncode, output) - self.assertTrue(source_exists) + self.assertIsNotNone(retained_payload) self.assertEqual([], records) self.assertIn("missing expected tag(s): missing.tag", output) - def test_generated_uploaders_continue_after_one_split_part_fails(self) -> None: - """Keep successful parts uploaded while reporting aggregate source failure.""" + def test_generated_uploaders_retry_only_failed_split_parts(self) -> None: + """Persist and retry only rejected parts after reporting aggregate failure.""" payload = self._large_test_payload(event_count=4, blob_bytes=1_300_000) def reject_second_part(candidate, _records): @@ -6650,7 +6697,7 @@ def reject_second_part(candidate, _records): for runtime in ("Bash", "PowerShell"): with self.subTest(runtime=runtime): - result, records, source_exists = self._run_local_test_upload( + result, records, retained_payload = self._run_local_test_upload( runtime, payload, reject_second_part, @@ -6659,7 +6706,7 @@ def reject_second_part(candidate, _records): ) output = result.stdout + result.stderr self.assertNotEqual(0, result.returncode, output) - self.assertTrue(source_exists) + self.assertIsNotNone(retained_payload) first_part = [ record for record in records @@ -6676,6 +6723,30 @@ def reject_second_part(candidate, _records): self.assertTrue(all(record["status"] == 503 for record in failed_part)) self.assertIn("part=2/2", output) self.assertIn("persistent part failure", output) + self.assertIn("retained 1 failed split payload part(s) for retry", output) + assert retained_payload is not None + retained_ids = [ + event["content"]["meta"]["event.id"] + for event in retained_payload["events"] + ] + self.assertEqual(["event-2", "event-3"], retained_ids) + + retry_result, retry_records, retry_payload = self._run_local_test_upload( + runtime, + retained_payload, + lambda _candidate, _records: (200, {}), + gzip_enabled=True, + keep_payloads=False, + ) + retry_output = retry_result.stdout + retry_result.stderr + self.assertEqual(0, retry_result.returncode, retry_output) + self.assertIsNone(retry_payload) + retried_ids = [ + event["content"]["meta"]["event.id"] + for record in retry_records + for event in record["payload"]["events"] + ] + self.assertEqual(["event-2", "event-3"], retried_ids) def test_generated_uploaders_reject_unsplittable_single_event(self) -> None: """Reject one event above the hard intake limit without making a request.""" @@ -6686,7 +6757,7 @@ def accept(_payload, _records): for runtime in ("Bash", "PowerShell"): with self.subTest(runtime=runtime): - result, records, source_exists = self._run_local_test_upload( + result, records, retained_payload = self._run_local_test_upload( runtime, payload, accept, @@ -6694,7 +6765,7 @@ def accept(_payload, _records): ) output = result.stdout + result.stderr self.assertNotEqual(0, result.returncode, output) - self.assertTrue(source_exists) + self.assertIsNotNone(retained_payload) self.assertEqual([], records) self.assertIn("single_event_too_large", output) @@ -6791,7 +6862,7 @@ def test_generated_uploaders_process_valid_payloads_before_reporting_partial_out expected_targets.write_text( json.dumps({ "schema_version": 1, - "targets": ["//pkg:empty", "//pkg:missing", "//pkg:valid"], + "targets": ["//pkg:a_unmappable", "//pkg:empty", "//pkg:missing", "//pkg:valid"], }), encoding="utf-8", ) @@ -6821,6 +6892,7 @@ def test_generated_uploaders_process_valid_payloads_before_reporting_partial_out "no TestResult matched this target); continuing with other fresh outputs", output, ) + self.assertIn("BEP required freshness skipped", output) self.assertIn("fresh expected test output produced no uploadable payloads", output) self.assertIn("//pkg:empty", output) report_doc = json.loads(report.read_text(encoding="utf-8")) @@ -6831,7 +6903,7 @@ def test_generated_uploaders_process_valid_payloads_before_reporting_partial_out expected_targets.write_text( json.dumps({ "schema_version": 1, - "targets": ["//pkg:missing", "//pkg:valid"], + "targets": ["//pkg:a_unmappable", "//pkg:missing", "//pkg:valid"], }), encoding="utf-8", ) @@ -6858,6 +6930,7 @@ def test_generated_uploaders_process_valid_payloads_before_reporting_partial_out partial_output = partial_result.stdout + partial_result.stderr self.assertNotEqual(0, partial_result.returncode, partial_output) self.assertIn("continuing with other fresh outputs", partial_output) + self.assertIn("BEP required freshness skipped", partial_output) partial_report_doc = json.loads(partial_report.read_text(encoding="utf-8")) self.assertEqual("fail", partial_report_doc["status"]) self.assertEqual(1, partial_report_doc["payloads"]["tests"]["processed"]) From 8a859700d61c840b695164326253e6c52dc0d79f Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Wed, 26 Aug 2026 23:52:20 +0200 Subject: [PATCH 30/37] uploader: persist retries from read-only outputs --- tools/core/uploader_bash_runtime.sh.tpl | 11 ++++++++--- tools/core/uploader_powershell_runtime.ps1.tpl | 12 +++++++++++- tools/tests/python/test_python_tools.py | 1 + 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/tools/core/uploader_bash_runtime.sh.tpl b/tools/core/uploader_bash_runtime.sh.tpl index a0775f0a..0573375a 100644 --- a/tools/core/uploader_bash_runtime.sh.tpl +++ b/tools/core/uploader_bash_runtime.sh.tpl @@ -4927,14 +4927,19 @@ prepare_test_payload_parts() { persist_failed_test_payload_parts() { local source_file="$1" shift - local retry_file failed_count=$# + local retry_file source_dir failed_count=$# if (( failed_count == 0 )) || [[ "$KEEP_PAYLOADS" == "1" ]]; then return 0 fi retry_file="$(mktemp "${source_file}.retry.XXXXXX" 2>/dev/null || true)" if [[ -z "$retry_file" ]]; then - log "warning: failed to create retry payload beside source; retaining the original payload: $source_file" - return 1 + source_dir="$(dirname "$source_file")" + chmod u+w "$source_dir" 2>/dev/null || true + retry_file="$(mktemp "${source_file}.retry.XXXXXX" 2>/dev/null || true)" + if [[ -z "$retry_file" ]]; then + log "warning: failed to create retry payload beside source; retaining the original payload: $source_file" + return 1 + fi fi if (( failed_count == 1 )); then if ! cp "$1" "$retry_file"; then diff --git a/tools/core/uploader_powershell_runtime.ps1.tpl b/tools/core/uploader_powershell_runtime.ps1.tpl index b6f9844e..0f5e5f1f 100644 --- a/tools/core/uploader_powershell_runtime.ps1.tpl +++ b/tools/core/uploader_powershell_runtime.ps1.tpl @@ -3982,7 +3982,17 @@ function Prepare-TestPayloadParts([string]$BodyPath, [string]$SourcePath) { function Save-FailedTestPayloadParts([string]$SourcePath, [string[]]$FailedParts) { if ($KeepPayloads -or $null -eq $FailedParts -or $FailedParts.Count -eq 0) { return } - $retryPath = Join-Path (Split-Path -Parent $SourcePath) ((Split-Path -Leaf $SourcePath) + ".retry." + [System.Guid]::NewGuid().ToString("N")) + $sourceDir = Split-Path -Parent $SourcePath + try { + $sourceDirItem = Get-Item -LiteralPath $sourceDir -ErrorAction Stop + if ($sourceDirItem.PSObject.Properties['IsReadOnly'] -and $sourceDirItem.IsReadOnly) { + $sourceDirItem.IsReadOnly = $false + } + } catch {} + if (-not $IsWindows) { + & chmod u+w -- $sourceDir 2>$null + } + $retryPath = Join-Path $sourceDir ((Split-Path -Leaf $SourcePath) + ".retry." + [System.Guid]::NewGuid().ToString("N")) try { $retryPayload = Get-Content -LiteralPath $FailedParts[0] -Raw -Encoding UTF8 | ConvertFrom-Json -ErrorAction Stop if ($FailedParts.Count -gt 1) { diff --git a/tools/tests/python/test_python_tools.py b/tools/tests/python/test_python_tools.py index 709dd79b..5967edd6 100644 --- a/tools/tests/python/test_python_tools.py +++ b/tools/tests/python/test_python_tools.py @@ -6703,6 +6703,7 @@ def reject_second_part(candidate, _records): reject_second_part, gzip_enabled=True, keep_payloads=False, + read_only_parent=True, ) output = result.stdout + result.stderr self.assertNotEqual(0, result.returncode, output) From a9d0009f16fc204303337ca60f4d7d36ec284bca Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Thu, 27 Aug 2026 00:17:57 +0200 Subject: [PATCH 31/37] uploader: support Windows PowerShell partial retries --- tools/core/uploader_powershell_runtime.ps1.tpl | 7 +++++-- tools/tests/python/test_python_tools.py | 17 +++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/tools/core/uploader_powershell_runtime.ps1.tpl b/tools/core/uploader_powershell_runtime.ps1.tpl index 0f5e5f1f..5a917c04 100644 --- a/tools/core/uploader_powershell_runtime.ps1.tpl +++ b/tools/core/uploader_powershell_runtime.ps1.tpl @@ -3989,10 +3989,12 @@ function Save-FailedTestPayloadParts([string]$SourcePath, [string[]]$FailedParts $sourceDirItem.IsReadOnly = $false } } catch {} - if (-not $IsWindows) { + $isWindowsPlatform = [System.Environment]::OSVersion.Platform -eq [System.PlatformID]::Win32NT + if (-not $isWindowsPlatform) { & chmod u+w -- $sourceDir 2>$null } $retryPath = Join-Path $sourceDir ((Split-Path -Leaf $SourcePath) + ".retry." + [System.Guid]::NewGuid().ToString("N")) + $backupPath = Join-Path $sourceDir ((Split-Path -Leaf $SourcePath) + ".backup." + [System.Guid]::NewGuid().ToString("N")) try { $retryPayload = Get-Content -LiteralPath $FailedParts[0] -Raw -Encoding UTF8 | ConvertFrom-Json -ErrorAction Stop if ($FailedParts.Count -gt 1) { @@ -4006,12 +4008,13 @@ function Save-FailedTestPayloadParts([string]$SourcePath, [string[]]$FailedParts $retryPayload.events = @($failedEvents.ToArray()) } Write-Utf8NoBomFile -Path $retryPath -Content (($retryPayload | ConvertTo-Json -Depth 100 -Compress) + "`n") - [System.IO.File]::Move($retryPath, $SourcePath, $true) + [System.IO.File]::Replace($retryPath, $SourcePath, $backupPath) Log "retained $($FailedParts.Count) failed split payload part(s) for retry: $SourcePath" } catch { Log "warning: failed to retain rejected split payload parts; retaining the original payload '$SourcePath': $_" } finally { Remove-Item -LiteralPath $retryPath -Force -ErrorAction SilentlyContinue + Remove-Item -LiteralPath $backupPath -Force -ErrorAction SilentlyContinue } } diff --git a/tools/tests/python/test_python_tools.py b/tools/tests/python/test_python_tools.py index 5967edd6..755efde8 100644 --- a/tools/tests/python/test_python_tools.py +++ b/tools/tests/python/test_python_tools.py @@ -7159,6 +7159,23 @@ def test_bash_runtime_has_no_windows_delegation(self) -> None: self.assertNotIn("cygwin", bash_text) self.assertNotIn("exec powershell.exe", bash_text) + def test_partial_retry_uses_windows_powershell_51_compatible_apis(self) -> None: + """Keep partial retry persistence compatible with Windows PowerShell 5.1.""" + powershell_text = _runfile("tools/core/uploader_powershell_runtime.ps1.tpl").read_text( + encoding="utf-8" + ) + retry_block = self._extract_text_block( + powershell_text, + "function Save-FailedTestPayloadParts", + "function Get-TelemetryHeaders", + ) + + self.assertIn("[System.Environment]::OSVersion.Platform", retry_block) + self.assertIn("[System.PlatformID]::Win32NT", retry_block) + self.assertNotIn("$IsWindows", retry_block) + self.assertIn("[System.IO.File]::Replace($retryPath, $SourcePath, $backupPath)", retry_block) + self.assertNotIn("[System.IO.File]::Move(", retry_block) + def test_bash_runtime_guards_context_enrichment_failures(self) -> None: """Validate bash runtime falls back when jq context enrichment fails.""" bash_text = _runfile("tools/core/uploader_bash_runtime.sh.tpl").read_text(encoding="utf-8") From 30c52bdcb27459e80b9dc20ec343017192325975 Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Thu, 27 Aug 2026 01:08:26 +0200 Subject: [PATCH 32/37] uploader: support PowerShell 5.1 payload cleanup --- tools/core/uploader_powershell_runtime.ps1.tpl | 3 ++- tools/tests/python/test_python_tools.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/tools/core/uploader_powershell_runtime.ps1.tpl b/tools/core/uploader_powershell_runtime.ps1.tpl index 5a917c04..4e70735e 100644 --- a/tools/core/uploader_powershell_runtime.ps1.tpl +++ b/tools/core/uploader_powershell_runtime.ps1.tpl @@ -3781,7 +3781,8 @@ function Remove-PayloadFile([string]$FilePath) { $item.IsReadOnly = $false } } catch {} - if (-not $IsWindows) { + $isWindowsPlatform = [System.Environment]::OSVersion.Platform -eq [System.PlatformID]::Win32NT + if (-not $isWindowsPlatform) { & chmod u+w -- (Split-Path -Parent $FilePath) 2>$null } Remove-Item -LiteralPath $FilePath -Force -ErrorAction SilentlyContinue diff --git a/tools/tests/python/test_python_tools.py b/tools/tests/python/test_python_tools.py index 755efde8..99eb53df 100644 --- a/tools/tests/python/test_python_tools.py +++ b/tools/tests/python/test_python_tools.py @@ -7176,6 +7176,21 @@ def test_partial_retry_uses_windows_powershell_51_compatible_apis(self) -> None: self.assertIn("[System.IO.File]::Replace($retryPath, $SourcePath, $backupPath)", retry_block) self.assertNotIn("[System.IO.File]::Move(", retry_block) + def test_payload_cleanup_uses_windows_powershell_51_compatible_platform_detection(self) -> None: + """Keep successful-upload cleanup compatible with Windows PowerShell 5.1.""" + powershell_text = _runfile("tools/core/uploader_powershell_runtime.ps1.tpl").read_text( + encoding="utf-8" + ) + cleanup_block = self._extract_text_block( + powershell_text, + "function Remove-PayloadFile", + "# Track per-payload upload report counters.", + ) + + self.assertIn("[System.Environment]::OSVersion.Platform", cleanup_block) + self.assertIn("[System.PlatformID]::Win32NT", cleanup_block) + self.assertNotIn("$IsWindows", cleanup_block) + def test_bash_runtime_guards_context_enrichment_failures(self) -> None: """Validate bash runtime falls back when jq context enrichment fails.""" bash_text = _runfile("tools/core/uploader_bash_runtime.sh.tpl").read_text(encoding="utf-8") From 8ab12e13782408ee0b25bd04b9b8fcdb09eb1666 Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Fri, 28 Aug 2026 12:24:42 +0200 Subject: [PATCH 33/37] go: add static Test Optimization targets --- MODULE.bazel | 18 ++ MODULE.bazel.lock | 48 +++- README.md | 40 ++- docs/Configuration_Reference.md | 58 ++++- docs/Installation_Reference.md | 9 +- docs/Language_Onboarding.md | 15 +- docs/Troubleshooting.md | 13 +- docs/Uploader_Reference.md | 36 ++- docs/go_orchestrion_bazel_deep_dive.md | 12 +- docs/internal_monorepo_go_rollout_guide.md | 10 +- docs/rules_go_variant_maintenance_guide.md | 12 +- examples/README.md | 9 +- modules/go/MODULE.bazel | 23 +- modules/go/tests/BUILD.bazel | 95 +++++-- modules/go/tests/test_macro.bzl | 194 +++++++++++--- modules/go/tools/dd_topt_go_bootstrap/main.go | 15 +- .../tools/dd_topt_go_bootstrap/main_test.go | 8 +- modules/go/topt_go_infer.bzl | 44 ++++ modules/go/topt_go_orchestrion.bzl | 27 +- modules/go/topt_go_stdlib.bzl | 10 +- modules/go/topt_go_test.bzl | 137 ++++++++-- .../references/bzlmod-onboarding.md | 6 +- .../references/workspace-onboarding.md | 6 +- .../SKILL.md | 10 +- tools/core/BUILD.bazel | 1 + tools/core/test_optimization_doctor.bzl | 5 + tools/core/test_optimization_doctor.py | 128 ++++++++-- .../test_optimization_repository_state.bzl | 44 ++++ tools/core/test_optimization_sync.bzl | 124 ++++++++- tools/core/test_optimization_targets.bzl | 22 +- tools/core/test_optimization_uploader.bzl | 9 +- tools/core/uploader_bash_runtime.sh.tpl | 236 +++++++++++++++++- .../core/uploader_powershell_runtime.ps1.tpl | 175 ++++++++++++- tools/tests/core/BUILD.bazel | 29 +++ tools/tests/core/test_sync_disabled_utils.bzl | 33 +++ .../test_test_optimization_targets_utils.bzl | 52 ++++ tools/tests/example_stub_repo.bzl | 42 +++- tools/tests/python/test_python_tools.py | 182 ++++++++++++++ 38 files changed, 1697 insertions(+), 240 deletions(-) create mode 100644 tools/core/test_optimization_repository_state.bzl diff --git a/MODULE.bazel b/MODULE.bazel index b70d8ed3..c6cb94a6 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -252,6 +252,21 @@ example_stub_repo.example_stub_repo( labels = _EXAMPLE_STUB_LABELS, service_name = "go-service", ) +example_stub_repo.example_stub_repo( + name = "test_optimization_data_static", + service_name = "static-service", +) +example_stub_repo.example_stub_repo( + name = "test_optimization_data_static_module", + go_module_included = True, + labels = ["example_com_stub"], + service_name = "static-module-service", +) +example_stub_repo.example_stub_repo( + name = "test_optimization_data_static_disabled", + enabled = False, + service_name = "static-disabled-service", +) use_repo( example_stub_repo, "test_optimization_data", @@ -261,4 +276,7 @@ use_repo( "test_optimization_data_nodejs", "test_optimization_data_python", "test_optimization_data_ruby", + "test_optimization_data_static", + "test_optimization_data_static_disabled", + "test_optimization_data_static_module", ) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 5e951a00..2535d417 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -341,8 +341,8 @@ }, "//tools/tests:example_stub_repo.bzl%example_stub_repo_extension": { "general": { - "bzlTransitiveDigest": "TrTrDHJrP0Rre84rDDioXjXxdcPRZwzJZA21GdNKrkE=", - "usagesDigest": "EtFKqYPzQPpUPHFL/jjD4e+uC5rSFI+DlvubVmTxLT0=", + "bzlTransitiveDigest": "l/UY5cKcoI+BOTIVjrd3XT68zZQMJK+5tsxrrQSlMAE=", + "usagesDigest": "iczwt0b8/rtzeNHdvZ+E+5nz7sfkCo47xzKO5qiEi3s=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -502,6 +502,50 @@ "service_name": "go-service", "service_keys": [] } + }, + "test_optimization_data_static": { + "repoRuleId": "@@//tools/tests:example_stub_repo.bzl%example_stub_repo", + "attributes": { + "enabled": true, + "go_module_included": false, + "go_module_path": "example.com/stub", + "go_sanitized_module_path": "example_com_stub", + "labels": [], + "out_dir": ".testoptimization", + "repo_alias": "test_optimization_data_static", + "service_name": "static-service", + "service_keys": [] + } + }, + "test_optimization_data_static_module": { + "repoRuleId": "@@//tools/tests:example_stub_repo.bzl%example_stub_repo", + "attributes": { + "enabled": true, + "go_module_included": true, + "go_module_path": "example.com/stub", + "go_sanitized_module_path": "example_com_stub", + "labels": [ + "example_com_stub" + ], + "out_dir": ".testoptimization", + "repo_alias": "test_optimization_data_static_module", + "service_name": "static-module-service", + "service_keys": [] + } + }, + "test_optimization_data_static_disabled": { + "repoRuleId": "@@//tools/tests:example_stub_repo.bzl%example_stub_repo", + "attributes": { + "enabled": false, + "go_module_included": false, + "go_module_path": "example.com/stub", + "go_sanitized_module_path": "example_com_stub", + "labels": [], + "out_dir": ".testoptimization", + "repo_alias": "test_optimization_data_static_disabled", + "service_name": "static-disabled-service", + "service_keys": [] + } } }, "recordedRepoMappingEntries": [] diff --git a/README.md b/README.md index 702ea61a..34a10c50 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,13 @@ Pick the path that matches your repository: The manifest-driven API is additive. Existing single-service and static multi-service consumers do not need to migrate. +Large consumers may also keep explicit single-service repositories and expose +stable ordinary/optimized target pairs such as `//pkg:test` and +`//pkg:test.topt`. In that model the `.topt` target carries a small literal +descriptor for exactly one service repository; Bazel resolves only the +repositories reached by selected `.topt` targets. The manifest-driven model +remains available for consumers that prefer invocation-scoped discovery. + ### Ordinary and managed execution An ordinary test command omits `--config=test-optimization`. Config-gated Go @@ -185,10 +192,8 @@ config must include public Go bootstrap helpers apply metadata gating by default. Manual Go extension wiring uses the same config-gated default. Direct use of the low-level core sync API in a config-gated Go or Python setup must opt into -`enabled_by_env = True`. Go -additionally needs -`build:test-optimization --@rules_go//go/private/orchestrion:enabled=true` -for Bzlmod, or the same flag with `@io_bazel_rules_go` for WORKSPACE. +`enabled_by_env = True`. Go optimized targets enable Orchestrion through their +own transition; no global `orchestrion:enabled` build setting is required. Removing `--config=test-optimization` provides the complete metadata and runtime opt-out for the Go and Python integrations described below. This release does not change the enablement contract of the other companions. @@ -529,6 +534,28 @@ Go setup. Manual Go callsites should set `orchestrion_mode = "test_optimization"` for standard Go `testing`; the default `general` mode preserves broader generic Orchestrion behavior. +An explicit per-service integration can pass the same macro a local-static +descriptor instead of loading an external repository's `export.bzl`: + +```bzl +dd_topt_go_test( + name = "pkg_go_test.topt", + topt_data = { + "repo_name": "test_optimization_data_go_service", + "service_name": "go-service", + "runtime_module_path": "example.com/repo", + }, + orchestrion_mode = "test_optimization", + srcs = ["pkg_test.go"], +) +``` + +The selected repository must expose the stable state, full-files, and +configured-runtime-module labels generated by `test_optimization_sync`. +Disabled or mismatched repositories fail analysis. This descriptor is an +additive option; generated single-, multi-, and manifest-sync exports keep +their existing behavior. + Large consumers can warm the matching instrumented standard library in a dedicated cache-writing invocation: @@ -544,9 +571,8 @@ bazel build //tools/test_optimization:go_stdlib_warmup \ --config=remote-cache-write ``` -The named config supplies the same Orchestrion enablement used by -`dd_topt_go_test`; the warmup target applies the same Go configuration -transitions, selects `test_optimization` mode, and materializes both the +The warmup target applies the same target-scoped Go configuration transitions, +enables Orchestrion in `test_optimization` mode, and materializes both the instrumented stdlib and its Go build cache. The consumer remains responsible for restricting remote-cache writes to a trusted CI invocation. diff --git a/docs/Configuration_Reference.md b/docs/Configuration_Reference.md index 87cb4521..ec4a9a91 100644 --- a/docs/Configuration_Reference.md +++ b/docs/Configuration_Reference.md @@ -166,6 +166,29 @@ Go target metadata records the selected mode in `bazel.go.orchestrion.mode` and whether the linker optimization was active in `bazel.go.test_binary_linker_optimization`. +### Explicit single-service Go descriptor + +`dd_topt_go_test` also accepts a local-static `topt_data` descriptor for a +consumer-owned stable `.topt` target: + +```bzl +topt_data = { + "repo_name": "test_optimization_data_service", + "service_name": "service-name", + "runtime_module_path": "example.com/repo", + # Optional only when the sync repository uses a non-default out_dir: + # "manifest_label": "@test_optimization_data_service//:custom/manifest.txt", +} +``` + +The descriptor is literal configuration; it does not load the external +repository's `export.bzl`. The macro derives +`:test_optimization_repository_state`, `:test_optimization_files`, and +`:test_optimization_runtime_module` from `repo_name`. The selector fails +analysis when the repository is disabled or its service, Go runtime, or module +identity differs. If the configured module is absent, the stable runtime-module +label is empty and selection uses the canonical full bundle. + ## Sync extension attributes Extension tag: `test_optimization_sync.test_optimization_sync(...)` @@ -201,6 +224,11 @@ Notes: cache-key churn. - For HTTP numeric overrides, `-1` means "do not pin here"; resolution falls back to environment overrides first, then the rule default. +- Single-service repositories expose stable public + `:test_optimization_repository_state` and + `:test_optimization_runtime_module` targets in both enabled and disabled + states. These support explicit static Go targets and do not activate a + repository by themselves. ## Multi-sync extension attributes @@ -307,6 +335,7 @@ Rule: `dd_payload_uploader(...)` | `data` | label_list | `[]` | Data files to include (for example, `context.json` for enrichment) | | `expected_targets` | string_list | `[]` | Optional exact local labels expected in the matching BEP. Fresh and cached results jointly satisfy coverage; missing results are reported while other fresh outputs continue to upload | | `expected_targets_file` | label | unset | Optional schema-v1 exact-target file. Static and file inputs must match when both are non-empty; missing results do not block other fresh uploads | +| `runtime_selection` | bool | `False` | Require repeatable runtime `--expected-target` and `--context-entry` arguments instead of configured expected/context inputs | ## Doctor rule attributes @@ -322,6 +351,7 @@ delete, or rewrite source payloads. | `data` | label_list | `["@test_optimization_data//:test_optimization_context"]` in examples | Context targets bundle `context.json` and `telemetry_facts.json`. Doctor selects `context.json` for Git validation; the same labels can be reused by the uploader for enrichment and rule telemetry | | `expected_targets` | string_list | `[]` | Optional strict list of local Bazel test labels to validate. When empty, the doctor validates discovered Test Optimization output directories and ignores plain non-instrumented test outputs | | `expected_targets_file` | label | unset | Optional schema-v1 JSON file containing the exact invocation-scoped target set. Static and file inputs must match when both are non-empty | +| `runtime_selection` | bool | `False` | Require repeatable runtime `--expected-target` and `--context-entry` arguments instead of configured expected/context inputs | | `require_git_metadata` | bool | `True` | Require `git.repository_url`, `git.commit.sha`, and `git.branch` or `git.tag` in synced context data | | `require_bazel_metadata` | bool | `True` | Require `bazel_target_metadata.json` next to selected payload outputs | | `require_json_payloads` | bool | `True` | Require parseable `.json` payload files | @@ -380,8 +410,9 @@ workspace root package. | `expected_targets` | string_list | `[]` | Exact labels passed to both tools. Doctor validates them strictly; uploader reports missing results while continuing with other fresh payloads | | `expected_targets_file` | label or `None` | `None` | Generated exact-target JSON file forwarded to both tools with the same doctor/uploader semantics for manifest-driven invocations | | `context_data` | label_list or `None` | `["@//:test_optimization_context"]` | Explicit context data labels when the default sync repo label is not enough | -| `doctor_kwargs` | dict or `None` | `{}` | Extra attrs for `dd_test_optimization_doctor`; cannot override `name`, `data`, `expected_targets`, or `expected_targets_file` | -| `uploader_kwargs` | dict or `None` | `{}` | Extra attrs for `dd_payload_uploader`; cannot override `name`, `data`, `expected_targets`, or `expected_targets_file` | +| `runtime_selection` | bool | `False` | Give both generated tools empty static data/expected inputs and require the runner to supply exact runtime targets and keyed contexts | +| `doctor_kwargs` | dict or `None` | `{}` | Extra attrs for `dd_test_optimization_doctor`; cannot override `name`, `data`, `expected_targets`, `expected_targets_file`, or `runtime_selection` | +| `uploader_kwargs` | dict or `None` | `{}` | Extra attrs for `dd_payload_uploader`; cannot override `name`, `data`, `expected_targets`, `expected_targets_file`, or `runtime_selection` | Example: @@ -399,6 +430,20 @@ dd_test_optimization_targets( ) ``` +With `runtime_selection = True`, do not configure `context_data`, +`expected_targets`, or `expected_targets_file`. Invoke both tools with the same +sorted exact selection: + +```text +--expected-target=//pkg:test.topt +--context-entry=test_optimization_data_service=/absolute/path/to/context.json +``` + +Both flags are repeatable. Context entries are keyed by the apparent Bazel +repository name and require a sibling `telemetry_facts.json`; repository, +service, runtime, schema, duplicate, and target-set mismatches fail before +payload discovery or upload. + ## Python snippet generator Tool: @@ -448,11 +493,10 @@ invocation. They do not include `DD_CIVISIBILITY_AGENTLESS_ENABLED`. For Go onboarding, the generated block also contains -`common: --repo_env=DD_TEST_OPTIMIZATION_ENABLED=1` and the existing -`rules_go` analysis setting -`build: --@//go/private/orchestrion:enabled=true`. -`--config=` is the single user-facing switch: removing it disables -both metadata resolution and the real Orchestrion aliases. +`common: --repo_env=DD_TEST_OPTIMIZATION_ENABLED=1`. Optimized Go +targets enable Orchestrion through their own transition, so no global +`orchestrion:enabled` build setting is generated. `--config=` remains +the single user-facing metadata gate. In that disabled state, the patched `rules_go` aliases select package-local empty targets and the gated Orchestrion repository writes its stable empty diff --git a/docs/Installation_Reference.md b/docs/Installation_Reference.md index 5990b6aa..6ceb8eec 100644 --- a/docs/Installation_Reference.md +++ b/docs/Installation_Reference.md @@ -399,7 +399,6 @@ The generated config is named `test-optimization` by default: ```text common:test-optimization --repo_env=DD_TEST_OPTIMIZATION_ENABLED=1 -build:test-optimization --@rules_go//go/private/orchestrion:enabled=true common:test-optimization --repo_env=DD_API_KEY common:test-optimization --repo_env=DD_SITE common:test-optimization --repo_env=DD_GIT_REPOSITORY_URL @@ -973,12 +972,12 @@ and Python onboarding additionally includes: ```text common:test-optimization --repo_env=DD_TEST_OPTIMIZATION_ENABLED=1 -# Go only; use the apparent rules_go repository name: -build:test-optimization --@io_bazel_rules_go//go/private/orchestrion:enabled=true ``` -Python-only consumers omit the Go line. Java, NodeJS, .NET, and Ruby retain -their existing enablement contract and omit both lines in this release. +Optimized Go targets enable Orchestrion through their own transition, so the +config has no global `orchestrion:enabled` setting. Java, NodeJS, .NET, and Ruby +retain their existing enablement contract and omit the metadata gate in this +release. ```text # Repository rule (module/repo phase) — affects refetch diff --git a/docs/Language_Onboarding.md b/docs/Language_Onboarding.md index b828e003..214dbeab 100644 --- a/docs/Language_Onboarding.md +++ b/docs/Language_Onboarding.md @@ -75,17 +75,10 @@ enablement contract of the other companions: common:test-optimization --repo_env=DD_TEST_OPTIMIZATION_ENABLED=1 ``` -Go workspaces also add the existing `rules_go` analysis-time setting. Prefer -the generated block from `dd_topt_go_bootstrap --print-bazelrc-snippet` or -`dd_topt_go_bootstrap --write-bazelrc`, which substitutes the consumer's actual -apparent repository name: - -```text -build:test-optimization --@rules_go//go/private/orchestrion:enabled=true -# Use @io_bazel_rules_go instead of @rules_go in WORKSPACE mode. -``` - -Do not add that Go-only line to Python-only or other non-Go workspaces. +Go optimized targets enable Orchestrion through their own transition. The +generated block therefore needs no global `orchestrion:enabled` setting; this +also allows ordinary and optimized Go tests to run together without changing +the ordinary target's configuration. Pass `DD_GIT_*` only through `--repo_env`. Never forward it as test environment data because that makes Git metadata part of the test action cache diff --git a/docs/Troubleshooting.md b/docs/Troubleshooting.md index 0e27dc12..a7d45cb9 100644 --- a/docs/Troubleshooting.md +++ b/docs/Troubleshooting.md @@ -27,20 +27,15 @@ languages set the metadata repository environment: common:test-optimization --repo_env=DD_TEST_OPTIMIZATION_ENABLED=1 ``` -Go additionally sets the existing `rules_go` Orchestrion flag: - -```bazelrc -build:test-optimization --@rules_go//go/private/orchestrion:enabled=true -``` +Go optimized targets enable Orchestrion through their own transition; do not +add a global `orchestrion:enabled` setting. When the config is omitted, the public Go extension's config-gated default and repositories explicitly configured with `enabled_by_env = True` generate the documented no-fetch stubs. Go aliases select local empty targets; Python keeps the normal consumer runner without Test Optimization metadata or payload -wiring. Python-only consumers omit the Go line. For WORKSPACE Go, replace -`@rules_go` with the apparent repository name used by that workspace. Java, -NodeJS, .NET, and Ruby retain their existing enablement contract in this -release. +wiring. Java, NodeJS, .NET, and Ruby retain their existing enablement contract +in this release. ## Quick triage map diff --git a/docs/Uploader_Reference.md b/docs/Uploader_Reference.md index d20f9077..360cb0c7 100644 --- a/docs/Uploader_Reference.md +++ b/docs/Uploader_Reference.md @@ -35,18 +35,16 @@ The `test-optimization` config should contain the recommended Bazel test flags: `--remote_download_minimal`, `--remote_download_regex=.*test[.]outputs.*`, and `--zip_undeclared_test_outputs`. -For Go consumers using the reusable bootstrap, keep the phase-correct enablement -in the same config: +For Go consumers using the reusable bootstrap, keep metadata enablement in the +same config: ```bazelrc common:test-optimization --repo_env=DD_TEST_OPTIMIZATION_ENABLED=1 -build:test-optimization --@rules_go//go/private/orchestrion:enabled=true ``` -This is one user-facing switch. Removing `--config=test-optimization` disables -both metadata resolution and the Orchestrion analysis aliases; it does not -require a second bool flag. In WORKSPACE repositories, use the apparent -`rules_go` repository name configured by that workspace. +Optimized Go targets enable Orchestrion through their own transition. Removing +`--config=test-optimization` disables metadata resolution; no global +Orchestrion setting or second user-facing bool is required. Config-gated Python consumers use only the `DD_TEST_OPTIMIZATION_ENABLED=1` entry and omit the Go-specific Orchestrion @@ -164,6 +162,30 @@ dd_test_optimization_targets( ) ``` +For a runner that selects explicit per-service repositories at execution time, +create the same pair with `runtime_selection = True` and no static context or +expected-target inputs: + +```bzl +dd_test_optimization_targets( + name = "test_optimization", + runtime_selection = True, +) +``` + +Pass the same repeatable arguments to doctor and uploader: + +```bash +--expected-target=//pkg:test.topt \ +--context-entry=test_optimization_data_service=/absolute/path/to/context.json +``` + +Each context path must name an existing `context.json` with a sibling +`telemetry_facts.json`. The apparent repository key and both documents' service +and runtime identity must agree. Missing, duplicate, mismatched, or incomplete +runtime selections fail before payload discovery, cleanup, enrichment, or +network access. + If your repository is small, the same helper can live in the root package. In large monorepos, prefer `//tools/test_optimization` or another lightweight package to avoid loading unrelated root package wiring when running doctor or diff --git a/docs/go_orchestrion_bazel_deep_dive.md b/docs/go_orchestrion_bazel_deep_dive.md index 762639b7..a9fc0714 100644 --- a/docs/go_orchestrion_bazel_deep_dive.md +++ b/docs/go_orchestrion_bazel_deep_dive.md @@ -307,17 +307,15 @@ in one place. Implementation: - [topt_go_orchestrion.bzl](../modules/go/topt_go_orchestrion.bzl) -In the enabled path, the wrapper rule applies a function transition that sets -only: +In the enabled path, the wrapper rule applies a function transition that sets: ```bzl -"@rules_go//go/private/orchestrion:mode": "general" or "test_optimization" +"@rules_go//go/private/orchestrion:enabled": True, +"@rules_go//go/private/orchestrion:mode": "general" or "test_optimization", ``` -The transition deliberately preserves the existing -`@rules_go//go/private/orchestrion:enabled` setting. The user-facing -`--config=test-optimization` config enables that setting during analysis; -omitting the config leaves it at the `rules_go` default of `False`. +Enablement therefore follows the optimized target edge instead of a global +build setting. Ordinary tests in a mixed invocation remain untransitioned. The wrapper then symlinks the executable produced by the raw target and returns the same runfiles. diff --git a/docs/internal_monorepo_go_rollout_guide.md b/docs/internal_monorepo_go_rollout_guide.md index 5a199e12..b8710aef 100644 --- a/docs/internal_monorepo_go_rollout_guide.md +++ b/docs/internal_monorepo_go_rollout_guide.md @@ -91,17 +91,15 @@ SHA256, and archive prefix generated from the same published commit. `--artifact-staging-dir=`. - Pass `DD_GIT_*` only through `--repo_env`, never through `--test_env`. - Pass uploader credentials at `bazel run` time, not into test actions. -- Keep one user-facing `test-optimization` config with both phase-correct - switches: +- Keep one user-facing `test-optimization` metadata config: ```bazelrc common:test-optimization --repo_env=DD_TEST_OPTIMIZATION_ENABLED=1 - build:test-optimization --@io_bazel_rules_go//go/private/orchestrion:enabled=true ``` - Removing `--config=test-optimization` disables both metadata repositories - and Orchestrion aliases. The public Go helpers enable metadata gating by - default; they do not dynamically control Orchestrion repository declaration. + Optimized targets enable Orchestrion through their own transition. Removing + the config disables metadata repositories; the public Go helpers enable that + metadata gating by default. ## Bootstrap Flow diff --git a/docs/rules_go_variant_maintenance_guide.md b/docs/rules_go_variant_maintenance_guide.md index 9f81aeaa..7e2c3677 100644 --- a/docs/rules_go_variant_maintenance_guide.md +++ b/docs/rules_go_variant_maintenance_guide.md @@ -268,17 +268,15 @@ patch in their private patch order. ## Test Optimization Alias Contract The public base trees keep the existing -`//go/private/orchestrion:enabled` setting as the analysis-time control. Their -stable Orchestrion aliases select package-local empty targets when the setting -is false and the real `rules_go_orchestrion_tool` files when it is true. -Consumers should expose one config that sets both effects: +`//go/private/orchestrion:enabled` setting and stable aliases. The Datadog Go +wrapper transitions that setting only for optimized targets. Consumers expose +one metadata config: ```bazelrc common:test-optimization --repo_env=DD_TEST_OPTIMIZATION_ENABLED=1 -build:test-optimization --@rules_go//go/private/orchestrion:enabled=true ``` The public Go extension reads the metadata environment by default; low-level repositories do so when explicitly configured with `enabled_by_env = True`. -Removing the config is the opt-out and must not require a consumer-owned -duplicate bool flag or stub repository. +Removing the config is the metadata opt-out and must not require a +consumer-owned duplicate bool flag or stub repository. diff --git a/examples/README.md b/examples/README.md index cb0d35ea..b90ebb5f 100644 --- a/examples/README.md +++ b/examples/README.md @@ -48,15 +48,12 @@ repository, use `./bazelw` for local development convenience. ```bazelrc common:test-optimization --repo_env=DD_TEST_OPTIMIZATION_ENABLED=1 - # Go only: - build:test-optimization --@rules_go//go/private/orchestrion:enabled=true ``` Removing `--config=test-optimization` disables metadata resolution and the - matching Go/Python runtime wiring. Go additionally disables Orchestrion - analysis; in WORKSPACE mode, use the apparent `rules_go` repo name configured - by the workspace. Python-only consumers omit the Go line. Other companions - retain their existing enablement contract in this release. + matching Go/Python runtime wiring. Optimized Go targets enable Orchestrion + through their own transition. Other companions retain their existing + enablement contract in this release. ## Single-service (classic) diff --git a/modules/go/MODULE.bazel b/modules/go/MODULE.bazel index 65898b42..ddfbfd5b 100644 --- a/modules/go/MODULE.bazel +++ b/modules/go/MODULE.bazel @@ -33,4 +33,25 @@ example_stub_repo.example_stub_repo( "ruby_service", ], ) -use_repo(example_stub_repo, "test_optimization_data") +example_stub_repo.example_stub_repo( + name = "test_optimization_data_static", + service_name = "static-service", +) +example_stub_repo.example_stub_repo( + name = "test_optimization_data_static_module", + go_module_included = True, + labels = ["example_com_stub"], + service_name = "static-module-service", +) +example_stub_repo.example_stub_repo( + name = "test_optimization_data_static_disabled", + enabled = False, + service_name = "static-disabled-service", +) +use_repo( + example_stub_repo, + "test_optimization_data", + "test_optimization_data_static", + "test_optimization_data_static_disabled", + "test_optimization_data_static_module", +) diff --git a/modules/go/tests/BUILD.bazel b/modules/go/tests/BUILD.bazel index 4cabc375..da9734c9 100644 --- a/modules/go/tests/BUILD.bazel +++ b/modules/go/tests/BUILD.bazel @@ -4,6 +4,10 @@ # This product includes software developed at Datadog # (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. +load( + "@datadog-rules-test-optimization-go//:topt_go_stdlib.bzl", + "dd_topt_go_stdlib_warmup", +) load( ":test_extension.bzl", "go_multi_specs_propagate_non_default_enablement_and_flaky_tests_test", @@ -30,8 +34,6 @@ load( "go_macro_inferred_importpath_metadata_test", "go_macro_multi_service_target", "go_macro_multi_service_wiring_test", - "go_macro_orchestrion_enablement_mismatch_failure_test", - "go_macro_orchestrion_enablement_mismatch_target", "go_macro_orchestrion_pin_files_provider_test", "go_macro_orchestrion_pin_files_target", "go_macro_orchestrion_pin_files_wiring_test", @@ -48,6 +50,15 @@ load( "go_macro_stage_sources_select_wiring_test", "go_macro_stage_sources_target", "go_macro_stage_sources_wiring_test", + "go_macro_static_disabled_failure_test", + "go_macro_static_disabled_target", + "go_macro_static_fallback_metadata_test", + "go_macro_static_identity_mismatch_failure_test", + "go_macro_static_identity_mismatch_target", + "go_macro_static_module_metadata_test", + "go_macro_static_module_target", + "go_macro_static_target", + "go_macro_static_wiring_test", "go_macro_test_optimization_linker_default_target", "go_macro_test_optimization_linker_default_wiring_test", "go_macro_test_optimization_linker_metadata_dbg_test", @@ -63,6 +74,7 @@ load( "go_macro_test_optimization_public_wrapper_mode_test", "has_go_mod_pin_test", "has_package_local_go_mod_test", + "hidden_raw_tags_test", "orch_transition_forwards_mode_test", "orch_wrapper_materialized_actual_non_windows_target", "orch_wrapper_materialized_actual_non_windows_test", @@ -72,8 +84,8 @@ load( "resolve_topt_service_key_missing_target_rule", "resolve_topt_service_key_unknown_failure_test", "resolve_topt_service_key_unknown_target_rule", - "stdlib_warmup_disabled_noop_test", "stdlib_warmup_tags_test", + "stdlib_warmup_target_has_outputs_test", "stdlib_warmup_transition_selects_test_optimization_test", "validate_orchestrion_mode_invalid_failure_test", "validate_orchestrion_mode_invalid_target_rule", @@ -85,10 +97,6 @@ load( "wrapper_output_name_target_rule", "wrapper_output_name_windows_test", ) -load( - "@datadog-rules-test-optimization-go//:topt_go_stdlib.bzl", - "dd_topt_go_stdlib_warmup", -) load( ":test_payloads_selector.bzl", "selector_embed_precedence_target", @@ -699,14 +707,54 @@ validate_test_optimization_pin_files_missing_go_mod_failure_test( target_under_test = ":validate_test_optimization_pin_files_missing_go_mod_target", ) -go_macro_orchestrion_enablement_mismatch_target( - name = "go_macro_orchestrion_enablement_mismatch_target", +go_macro_static_target( + name = "go_macro_static_target", + tags = [ + "dd-requires-docker", + "dd-test-optimization", + "dd-test-optimization-source-manual", + "manual", + ], +) + +go_macro_static_wiring_test( + name = "go_macro_static_wiring_test", + target_under_test = ":go_macro_static_target__raw_go_test", +) + +go_macro_static_fallback_metadata_test( + name = "go_macro_static_fallback_metadata_test", + target_under_test = ":go_macro_static_target_topt_bazel_metadata", +) + +go_macro_static_module_target( + name = "go_macro_static_module_target", tags = ["manual"], ) -go_macro_orchestrion_enablement_mismatch_failure_test( - name = "go_macro_orchestrion_enablement_mismatch_failure_test", - target_under_test = ":go_macro_orchestrion_enablement_mismatch_target", +go_macro_static_module_metadata_test( + name = "go_macro_static_module_metadata_test", + target_under_test = ":go_macro_static_module_target_topt_bazel_metadata", +) + +go_macro_static_disabled_target( + name = "go_macro_static_disabled_target", + tags = ["manual"], +) + +go_macro_static_disabled_failure_test( + name = "go_macro_static_disabled_failure_test", + target_under_test = ":go_macro_static_disabled_target", +) + +go_macro_static_identity_mismatch_target( + name = "go_macro_static_identity_mismatch_target", + tags = ["manual"], +) + +go_macro_static_identity_mismatch_failure_test( + name = "go_macro_static_identity_mismatch_failure_test", + target_under_test = ":go_macro_static_identity_mismatch_target", ) wrapper_output_name_target_rule( @@ -795,13 +843,19 @@ stdlib_warmup_tags_test( timeout = "short", ) +hidden_raw_tags_test( + name = "hidden_raw_tags_test", + size = "small", + timeout = "short", +) + dd_topt_go_stdlib_warmup( - name = "stdlib_warmup_disabled_target", + name = "stdlib_warmup_target", ) -stdlib_warmup_disabled_noop_test( - name = "stdlib_warmup_disabled_noop_test", - target_under_test = ":stdlib_warmup_disabled_target", +stdlib_warmup_target_has_outputs_test( + name = "stdlib_warmup_target_has_outputs_test", + target_under_test = ":stdlib_warmup_target", ) test_suite( @@ -821,7 +875,6 @@ test_suite( ":go_macro_explicit_service_wiring_test", ":go_macro_inferred_importpath_metadata_test", ":go_macro_multi_service_wiring_test", - ":go_macro_orchestrion_enablement_mismatch_failure_test", ":go_macro_orchestrion_pin_files_provider_test", ":go_macro_orchestrion_pin_files_wiring_test", ":go_macro_public_wrapper_test", @@ -831,6 +884,11 @@ test_suite( ":go_macro_stage_sources_rundir_wiring_test", ":go_macro_stage_sources_select_wiring_test", ":go_macro_stage_sources_wiring_test", + ":go_macro_static_disabled_failure_test", + ":go_macro_static_fallback_metadata_test", + ":go_macro_static_identity_mismatch_failure_test", + ":go_macro_static_module_metadata_test", + ":go_macro_static_wiring_test", ":go_macro_test_optimization_linker_default_wiring_test", ":go_macro_test_optimization_linker_metadata_dbg_test", ":go_macro_test_optimization_linker_metadata_default_test", @@ -848,6 +906,7 @@ test_suite( ":go_workspace_single_specs_test", ":has_go_mod_pin_test", ":has_package_local_go_mod_test", + ":hidden_raw_tags_test", ":normalize_user_data_handles_none_test", ":normalize_user_data_invalid_type_failure_test", ":orch_transition_forwards_mode_test", @@ -872,8 +931,8 @@ test_suite( ":selector_override_miss_failure_test", ":selector_override_test", ":service_mapping_entries_filters_non_service_test", - ":stdlib_warmup_disabled_noop_test", ":stdlib_warmup_tags_test", + ":stdlib_warmup_target_has_outputs_test", ":stdlib_warmup_transition_selects_test_optimization_test", ":validate_orchestrion_mode_invalid_failure_test", ":validate_orchestrion_mode_test", diff --git a/modules/go/tests/test_macro.bzl b/modules/go/tests/test_macro.bzl index 3f39fbec..d1fabd68 100644 --- a/modules/go/tests/test_macro.bzl +++ b/modules/go/tests/test_macro.bzl @@ -20,7 +20,11 @@ forwards at analysis time without compiling Go code. load("@bazel_skylib//lib:unittest.bzl", "analysistest", "asserts", "unittest") load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") -load("@datadog-rules-test-optimization-go//:topt_go_infer.bzl", "ToptGoBazelMetadataInfo") +load( + "@datadog-rules-test-optimization-go//:topt_go_infer.bzl", + "ToptGoBazelMetadataInfo", + "topt_go_payloads_selector", +) load( "@datadog-rules-test-optimization-go//:topt_go_orchestrion.bzl", "orch_go_test", @@ -38,14 +42,13 @@ load( "dd_topt_go_test", "has_go_mod_pin_for_tests", "has_package_local_go_mod_for_tests", + "hidden_raw_tags_for_tests", "resolve_topt_service_key_for_tests", "validate_orchestrion_mode_for_tests", "validate_test_optimization_pin_files_for_tests", ) load("@rules_go//go/private/orchestrion:pin_files.bzl", "OrchestrionPinFilesInfo") -_ORCHESTRION_ENABLED_SETTING = str(Label("@rules_go//go/private/orchestrion:enabled")) - ToptGoMacroCaptureInfo = provider( doc = "Captured arguments forwarded by dd_topt_go_test to the underlying go_test rule.", fields = { @@ -54,6 +57,7 @@ ToptGoMacroCaptureInfo = provider( "gc_linkopts": "Forwarded Go linker options.", "importpath": "Forwarded importpath attribute.", "rundir": "Forwarded runtime working directory.", + "tags": "Forwarded raw-test tags.", }, ) @@ -81,6 +85,7 @@ def _go_test_capture_impl(ctx): gc_linkopts = list(ctx.attr.gc_linkopts), importpath = ctx.attr.importpath, rundir = ctx.attr.rundir, + tags = list(ctx.attr.tags), ), ] @@ -236,6 +241,16 @@ def _dynamic_manifest_topt_data(): }) return data +def _local_static_topt_data( + repo_name = "test_optimization_data_static", + service_name = "static-service", + runtime_module_path = "example.com/stub"): + return { + "repo_name": repo_name, + "runtime_module_path": runtime_module_path, + "service_name": service_name, + } + def go_macro_single_service_target(name, tags = None): """Target-under-test: single-service wiring + default rundir path.""" dd_topt_go_test( @@ -261,6 +276,57 @@ def go_macro_dynamic_manifest_target(name, tags = None): tags = tags, ) +def go_macro_static_target(name, tags = None): + """Target under test for the checked-in local-static descriptor path.""" + dd_topt_go_test( + name = name, + topt_data = _local_static_topt_data(), + go_test_rule = _go_test_capture_rule, + tags = tags, + ) + +def go_macro_static_module_target(name, tags = None): + """Target under test for an included stable runtime-module group.""" + dd_topt_go_test( + name = name, + topt_data = _local_static_topt_data( + repo_name = "test_optimization_data_static_module", + service_name = "static-module-service", + ), + go_test_rule = _go_test_capture_rule, + tags = tags, + ) + +def go_macro_static_disabled_target(name, tags = None): + """Target under test for fail-closed disabled static repository state.""" + topt_go_payloads_selector( + name = name, + expected_repo_name = "test_optimization_data_static_disabled", + expected_runtime_module_path = "example.com/static", + expected_service_name = "static-disabled-service", + fallback_importpath = "example.com/static/pkg", + full_files = "@test_optimization_data_static_disabled//:test_optimization_files", + include_per_module = True, + repository_state = "@test_optimization_data_static_disabled//:test_optimization_repository_state", + runtime_module = "@test_optimization_data_static_disabled//:test_optimization_runtime_module", + tags = tags, + ) + +def go_macro_static_identity_mismatch_target(name, tags = None): + """Target under test for descriptor/repository identity disagreement.""" + topt_go_payloads_selector( + name = name, + expected_repo_name = "test_optimization_data_static", + expected_runtime_module_path = "example.com/static", + expected_service_name = "wrong-service", + fallback_importpath = "example.com/static/pkg", + full_files = "@test_optimization_data_static//:test_optimization_files", + include_per_module = True, + repository_state = "@test_optimization_data_static//:test_optimization_repository_state", + runtime_module = "@test_optimization_data_static//:test_optimization_runtime_module", + tags = tags, + ) + def go_macro_disabled_raw_target(name, tags = None): """Target under test for the strict disabled raw go_test branch.""" dd_topt_go_test( @@ -512,15 +578,6 @@ def orch_wrapper_materialized_actual_windows_target(name, tags = None): tags = tags, ) -def go_macro_orchestrion_enablement_mismatch_target(name, tags = None): - """Target under test for an incomplete config-gated Go upgrade.""" - dd_topt_go_test( - name = name, - topt_data = _single_service_topt_data(enabled = True), - go_test_rule = _go_test_capture_rule, - tags = tags, - ) - def _go_macro_single_service_wiring_test_impl(ctx): """Assert env/data/rundir contract for single-service macro usage.""" env = analysistest.begin(ctx) @@ -620,6 +677,46 @@ def _go_macro_dynamic_manifest_payloads_test_impl(ctx): asserts.false(env, _has_file_basename(files, "full_payload.payload")) return analysistest.end(env) +def _go_macro_static_wiring_test_impl(ctx): + """Assert static descriptors use stable repository labels and TO mode.""" + env = analysistest.begin(ctx) + captured = analysistest.target_under_test(env)[ToptGoMacroCaptureInfo] + asserts.true(env, _has_label_suffix(captured.data_labels, ":go_macro_static_target_topt_payloads")) + asserts.true(env, _has_label_suffix(captured.data_labels, ":go_macro_static_target_topt_bazel_metadata")) + asserts.true(env, _has_label_suffix(captured.data_labels, ":.testoptimization/manifest.txt")) + asserts.equals(env, "static-service", captured.env.get("DD_SERVICE")) + asserts.equals(env, ["dd-requires-docker", "manual"], captured.tags) + return analysistest.end(env) + +def _go_macro_static_fallback_metadata_test_impl(ctx): + """Assert an absent stable module is reported as a full-bundle fallback.""" + env = analysistest.begin(ctx) + metadata = analysistest.target_under_test(env)[ToptGoBazelMetadataInfo].metadata + asserts.equals(env, "full_bundle_no_match", metadata["bazel.go.payload_selection"]) + asserts.equals(env, "test_optimization", metadata["bazel.go.orchestrion.mode"]) + return analysistest.end(env) + +def _go_macro_static_module_metadata_test_impl(ctx): + """Assert an included stable module is reported as module selection.""" + env = analysistest.begin(ctx) + metadata = analysistest.target_under_test(env)[ToptGoBazelMetadataInfo].metadata + asserts.equals(env, "module", metadata["bazel.go.payload_selection"]) + return analysistest.end(env) + +def _go_macro_static_disabled_failure_test_impl(ctx): + """Assert explicit static selection fails closed when its repo is disabled.""" + env = analysistest.begin(ctx) + asserts.expect_failure(env, "selected Test Optimization repository") + asserts.expect_failure(env, "is disabled") + return analysistest.end(env) + +def _go_macro_static_identity_mismatch_failure_test_impl(ctx): + """Assert descriptor/repository service mismatches fail analysis.""" + env = analysistest.begin(ctx) + asserts.expect_failure(env, "service identity mismatch") + asserts.expect_failure(env, "wrong-service") + return analysistest.end(env) + def _go_macro_rundir_mismatch_wiring_test_impl(ctx): """Assert custom rundir is honored when explicitly provided.""" env = analysistest.begin(ctx) @@ -957,14 +1054,6 @@ def _validate_test_optimization_pin_files_missing_go_mod_failure_test_impl(ctx): asserts.expect_failure(env, "requires a package-local go.mod or explicit orchestrion_pin_files") return analysistest.end(env) -def _go_macro_orchestrion_enablement_mismatch_failure_test_impl(ctx): - """Assert a partial upgrade fails instead of silently dropping instrumentation.""" - env = analysistest.begin(ctx) - asserts.expect_failure(env, "Test Optimization metadata is enabled but Orchestrion is disabled") - asserts.expect_failure(env, "--config=test-optimization") - asserts.expect_failure(env, "--write-bazelrc") - return analysistest.end(env) - def _wrapper_output_name_non_windows_test_impl(ctx): """Assert non-Windows wrapper names remain extensionless.""" env = analysistest.begin(ctx) @@ -1021,21 +1110,21 @@ def _validate_orchestrion_mode_test_impl(ctx): return unittest.end(env) def _orch_transition_forwards_mode_test_impl(ctx): - """Assert the wrapper transition forwards only the Orchestrion mode.""" + """Assert the wrapper transition enables Orchestrion and forwards its mode.""" env = unittest.begin(ctx) result = orch_transition_impl_for_tests(None, struct(orchestrion_mode = "test_optimization")) - asserts.equals(env, 1, len(result)) + asserts.equals(env, 2, len(result)) + asserts.true(env, result["@rules_go//go/private/orchestrion:enabled"]) asserts.equals(env, "test_optimization", result["@rules_go//go/private/orchestrion:mode"]) - asserts.false(env, "@rules_go//go/private/orchestrion:enabled" in result) return unittest.end(env) def _stdlib_warmup_transition_selects_test_optimization_test_impl(ctx): """Assert the cache-warm target selects the exact instrumented stdlib.""" env = unittest.begin(ctx) result = stdlib_warmup_transition_impl_for_tests(None, None) - asserts.equals(env, 1, len(result)) + asserts.equals(env, 2, len(result)) + asserts.true(env, result["@rules_go//go/private/orchestrion:enabled"]) asserts.equals(env, "test_optimization", result["@rules_go//go/private/orchestrion:mode"]) - asserts.false(env, "@rules_go//go/private/orchestrion:enabled" in result) return unittest.end(env) def _stdlib_warmup_tags_test_impl(ctx): @@ -1046,11 +1135,27 @@ def _stdlib_warmup_tags_test_impl(ctx): asserts.equals(env, ["custom", "manual"], stdlib_warmup_tags_for_tests(["custom", "manual"])) return unittest.end(env) -def _stdlib_warmup_disabled_noop_test_impl(ctx): - """Assert broad builds can analyze the warmup target without enablement.""" +def _hidden_raw_tags_test_impl(ctx): + """Assert public selection markers never leak to the hidden raw test.""" + env = unittest.begin(ctx) + asserts.equals( + env, + ["dd-requires-docker", "manual"], + hidden_raw_tags_for_tests([ + "manual", + "dd-test-optimization", + "dd-requires-docker", + "dd-test-optimization-source-manual", + "manual", + ]), + ) + return unittest.end(env) + +def _stdlib_warmup_target_has_outputs_test_impl(ctx): + """Assert explicit warmup selection no longer depends on a parent flag.""" env = analysistest.begin(ctx) target = analysistest.target_under_test(env) - asserts.equals(env, 0, len(target[DefaultInfo].files.to_list())) + asserts.true(env, len(target[DefaultInfo].files.to_list()) > 0) return analysistest.end(env) def _orch_wrapper_materialized_actual_non_windows_test_impl(ctx): @@ -1178,21 +1283,12 @@ go_macro_explicit_service_wiring_test = analysistest.make( ) go_macro_public_wrapper_test = analysistest.make( _go_macro_public_wrapper_test_impl, - config_settings = { - _ORCHESTRION_ENABLED_SETTING: True, - }, ) go_macro_test_optimization_public_wrapper_mode_test = analysistest.make( _go_macro_test_optimization_public_wrapper_mode_test_impl, - config_settings = { - _ORCHESTRION_ENABLED_SETTING: True, - }, ) go_macro_default_general_public_wrapper_mode_test = analysistest.make( _go_macro_default_general_public_wrapper_mode_test_impl, - config_settings = { - _ORCHESTRION_ENABLED_SETTING: True, - }, ) resolve_topt_service_key_missing_failure_test = analysistest.make( _resolve_topt_service_key_missing_failure_test_impl, @@ -1210,8 +1306,21 @@ validate_test_optimization_pin_files_missing_go_mod_failure_test = analysistest. _validate_test_optimization_pin_files_missing_go_mod_failure_test_impl, expect_failure = True, ) -go_macro_orchestrion_enablement_mismatch_failure_test = analysistest.make( - _go_macro_orchestrion_enablement_mismatch_failure_test_impl, +go_macro_static_wiring_test = analysistest.make( + _go_macro_static_wiring_test_impl, +) +go_macro_static_fallback_metadata_test = analysistest.make( + _go_macro_static_fallback_metadata_test_impl, +) +go_macro_static_module_metadata_test = analysistest.make( + _go_macro_static_module_metadata_test_impl, +) +go_macro_static_disabled_failure_test = analysistest.make( + _go_macro_static_disabled_failure_test_impl, + expect_failure = True, +) +go_macro_static_identity_mismatch_failure_test = analysistest.make( + _go_macro_static_identity_mismatch_failure_test_impl, expect_failure = True, ) wrapper_output_name_non_windows_test = analysistest.make( @@ -1241,8 +1350,11 @@ stdlib_warmup_transition_selects_test_optimization_test = unittest.make( stdlib_warmup_tags_test = unittest.make( _stdlib_warmup_tags_test_impl, ) -stdlib_warmup_disabled_noop_test = analysistest.make( - _stdlib_warmup_disabled_noop_test_impl, +hidden_raw_tags_test = unittest.make( + _hidden_raw_tags_test_impl, +) +stdlib_warmup_target_has_outputs_test = analysistest.make( + _stdlib_warmup_target_has_outputs_test_impl, ) orch_wrapper_materialized_actual_non_windows_test = analysistest.make( _orch_wrapper_materialized_actual_non_windows_test_impl, diff --git a/modules/go/tools/dd_topt_go_bootstrap/main.go b/modules/go/tools/dd_topt_go_bootstrap/main.go index 9527c030..82703d0c 100644 --- a/modules/go/tools/dd_topt_go_bootstrap/main.go +++ b/modules/go/tools/dd_topt_go_bootstrap/main.go @@ -1448,16 +1448,6 @@ func bazelrcSnippet(cfg config) (string, error) { fmt.Fprintf(&buf, "common:%s --repo_env=%s\n", cfg.bazelrcConfig, key) } fmt.Fprintf(&buf, "common:%s --repo_env=DD_TEST_OPTIMIZATION_ENABLED=1\n", cfg.bazelrcConfig) - rulesGoRepoName := cfg.rulesGoRepoName - if !cfg.workspaceMode { - // Bzlmod exposes rules_go under its module name. The WORKSPACE - // repository name is configurable because repository rules may - // remap it (the default there remains io_bazel_rules_go). - rulesGoRepoName = "rules_go" - } else if rulesGoRepoName == "" { - rulesGoRepoName = defaultRulesGoRepoName - } - fmt.Fprintf(&buf, "build:%s --@%s//go/private/orchestrion:enabled=true\n", cfg.bazelrcConfig, rulesGoRepoName) fmt.Fprintf(&buf, "test:%s --remote_download_minimal\n", cfg.bazelrcConfig) fmt.Fprintf(&buf, "test:%s --remote_download_regex=.*test[.]outputs.*\n", cfg.bazelrcConfig) fmt.Fprintf(&buf, "test:%s --zip_undeclared_test_outputs\n", cfg.bazelrcConfig) @@ -1526,7 +1516,7 @@ func validationScript(cfg config) (string, error) { fmt.Fprintf(&buf, "SYNC_REPO=%s\n", shellQuote(cfg.syncRepoName)) fmt.Fprintf(&buf, "DOCTOR_TARGET=%s\n", shellQuote(cfg.validationDoctorTarget)) fmt.Fprintf(&buf, "UPLOAD_TARGET=%s\n", shellQuote(cfg.validationUploadTarget)) - fmt.Fprintf(&buf, "RULES_GO_ENABLED_LABEL=%s\n", shellQuote("@"+apparentRulesGoRepoName(cfg)+"//go/private/orchestrion:enabled")) + fmt.Fprintf(&buf, "RULES_GO_ORCHESTRION_PACKAGE=%s\n", shellQuote("@"+apparentRulesGoRepoName(cfg)+"//go/private/orchestrion")) buf.WriteString("WORKSPACE_DIR=\"$(pwd -P)\"\n") buf.WriteString("BEP_TMP_ROOT=\"\"\n") buf.WriteString("BEP_JSON_DIR=\"\"\n") @@ -1657,7 +1647,7 @@ validate_disabled_bootstrap() { alias_files="$( env -u DD_API_KEY -u DD_SITE -u DD_TEST_OPTIMIZATION_ENABLED \ "${BAZEL}" cquery \ - "${RULES_GO_ENABLED_LABEL%:enabled}:tool_binary" --output=files + "${RULES_GO_ORCHESTRION_PACKAGE}:tool_binary" --output=files )" || return $? if [[ -n "${alias_files}" ]]; then warn "ordinary no-config Orchestrion alias unexpectedly exposed files: ${alias_files}" @@ -1668,7 +1658,6 @@ validate_disabled_bootstrap() { env -u DD_API_KEY -u DD_SITE -u DD_TEST_OPTIMIZATION_ENABLED \ "${BAZEL}" query "--config=${BAZEL_CONFIG}" \ --repo_env=DD_TEST_OPTIMIZATION_ENABLED=0 \ - "--${RULES_GO_ENABLED_LABEL}=false" \ "@${SYNC_REPO}//:test_optimization_files" } diff --git a/modules/go/tools/dd_topt_go_bootstrap/main_test.go b/modules/go/tools/dd_topt_go_bootstrap/main_test.go index 0020d3b0..32b38885 100644 --- a/modules/go/tools/dd_topt_go_bootstrap/main_test.go +++ b/modules/go/tools/dd_topt_go_bootstrap/main_test.go @@ -532,7 +532,6 @@ func TestBazelrcSnippetUsesRepoEnvOnlyForSyncMetadata(t *testing.T) { `common:test-optimization --repo_env=DD_GIT_REPOSITORY_URL`, `common:test-optimization --repo_env=DD_PR_NUMBER`, `common:test-optimization --repo_env=DD_TEST_OPTIMIZATION_ENABLED=1`, - `build:test-optimization --@rules_go//go/private/orchestrion:enabled=true`, `test:test-optimization --remote_download_minimal`, `test:test-optimization --remote_download_regex=.*test[.]outputs.*`, `test:test-optimization --zip_undeclared_test_outputs`, @@ -775,7 +774,7 @@ func TestValidationScriptUsesConfiguredFlowAndUploadOptIn(t *testing.T) { `SYNC_REPO='test_optimization_data_worker'`, `DOCTOR_TARGET='//:dd_test_optimization_doctor'`, `UPLOAD_TARGET='//:dd_upload_payloads'`, - `RULES_GO_ENABLED_LABEL='@rules_go//go/private/orchestrion:enabled'`, + `RULES_GO_ORCHESTRION_PACKAGE='@rules_go//go/private/orchestrion'`, `WORKSPACE_DIR="$(pwd -P)"`, `BEP_TMP_ROOT=""`, `BEP_JSON_DIR=""`, @@ -807,7 +806,7 @@ func TestValidationScriptUsesConfiguredFlowAndUploadOptIn(t *testing.T) { `validate explicit disabled precedence`, `query "@${SYNC_REPO}//:test_optimization_files"`, `"${BAZEL}" cquery \ - "${RULES_GO_ENABLED_LABEL%:enabled}:tool_binary"`, + "${RULES_GO_ORCHESTRION_PACKAGE}:tool_binary"`, `--repo_env=DD_TEST_OPTIMIZATION_ENABLED=0`, `upload skipped; rerun with --upload`, `${BAZEL}" shutdown`, @@ -928,7 +927,7 @@ func TestValidationScriptRunsWithNoControlTargets(t *testing.T) { for _, want := range []string{ "query @test_optimization_data//:test_optimization_files", "cquery @rules_go//go/private/orchestrion:tool_binary --output=files", - "query --config=test-optimization --repo_env=DD_TEST_OPTIMIZATION_ENABLED=0 --@rules_go//go/private/orchestrion:enabled=false @test_optimization_data//:test_optimization_files", + "query --config=test-optimization --repo_env=DD_TEST_OPTIMIZATION_ENABLED=0 @test_optimization_data//:test_optimization_files", "sync --config=test-optimization --repo_env=FETCH_SALT=", "test --config=test-optimization --build_event_json_file=", "//pkg:go_default_test", @@ -1363,7 +1362,6 @@ test:old --test_env=DD_GIT_BRANCH=main } for _, want := range []string{ `common:test-optimization --repo_env=DD_TEST_OPTIMIZATION_ENABLED=1`, - `build:test-optimization --@rules_go//go/private/orchestrion:enabled=true`, } { if !strings.Contains(text, want) { t.Fatalf("migrated managed block missing %q:\n%s", want, text) diff --git a/modules/go/topt_go_infer.bzl b/modules/go/topt_go_infer.bzl index 8a918d5b..e82aeb42 100644 --- a/modules/go/topt_go_infer.bzl +++ b/modules/go/topt_go_infer.bzl @@ -34,6 +34,10 @@ Maintenance notes: the safe full-bundle fallback. """ +load( + "@datadog-rules-test-optimization//tools/core:test_optimization_repository_state.bzl", + "TestOptimizationRepositoryStateInfo", +) load( "@datadog-rules-test-optimization//tools/core:topt_selection_utils.bzl", "select_module_group_name", @@ -74,6 +78,17 @@ def _resolve_payload_selection(ctx): ctx.attr.fallback_importpath or "", ) + if ctx.attr.repository_state: + state = ctx.attr.repository_state[TestOptimizationRepositoryStateInfo] + _validate_static_repository_state(ctx, state) + return struct( + importpath = ip, + importpath_source = importpath_source, + selected_name = "test_optimization_runtime_module" if state.runtime_module_included else "", + chosen = ctx.attr.runtime_module if state.runtime_module_included else None, + selection = "module" if state.runtime_module_included else "full_bundle_no_match", + ) + module_group_names = ctx.attr.module_group_names if module_group_names: if len(module_group_names) != len(ctx.attr.module_groups): @@ -114,6 +129,22 @@ def _resolve_payload_selection(ctx): selection = selection, ) +def _validate_static_repository_state(ctx, state): + """Fail closed when a local descriptor disagrees with its sync repository.""" + if not state.enabled: + reason = state.disabled_reason or "repository synchronization is disabled" + fail("topt_go_payloads_selector: selected Test Optimization repository %r is disabled: %s" % (state.repo_name, reason)) + if state.repo_name != ctx.attr.expected_repo_name: + fail("topt_go_payloads_selector: repository identity mismatch: descriptor=%r repository=%r" % (ctx.attr.expected_repo_name, state.repo_name)) + if state.service_name != ctx.attr.expected_service_name: + fail("topt_go_payloads_selector: service identity mismatch for repository %r: descriptor=%r repository=%r" % (state.repo_name, ctx.attr.expected_service_name, state.service_name)) + if state.runtime_name != "go": + fail("topt_go_payloads_selector: repository %r has runtime %r; expected %r" % (state.repo_name, state.runtime_name, "go")) + if state.runtime_module_path != ctx.attr.expected_runtime_module_path: + fail("topt_go_payloads_selector: runtime module mismatch for repository %r: descriptor=%r repository=%r" % (state.repo_name, ctx.attr.expected_runtime_module_path, state.runtime_module_path)) + if state.runtime_module_included and not ctx.attr.runtime_module: + fail("topt_go_payloads_selector: repository %r reports an included runtime module but exposes no stable runtime-module target" % state.repo_name) + # Provider carrying the inferred importpath string ToptGoImportpathInfo = provider( doc = "Provider carrying the inferred Go package importpath from rules_go.", @@ -276,6 +307,14 @@ topt_go_payloads_selector = rule( # Optional override for the sanitized module label suffix "module_label_override": attr.string(), + + # Static local descriptor state. Dynamic exports leave these empty and + # retain the existing per-module selection behavior above. + "repository_state": attr.label(providers = [TestOptimizationRepositoryStateInfo]), + "runtime_module": attr.label(), + "expected_repo_name": attr.string(), + "expected_service_name": attr.string(), + "expected_runtime_module_path": attr.string(), }, ) @@ -289,6 +328,11 @@ topt_go_bazel_metadata = rule( "module_groups": attr.label_list(), "include_per_module": attr.bool(default = True), "module_label_override": attr.string(), + "repository_state": attr.label(providers = [TestOptimizationRepositoryStateInfo]), + "runtime_module": attr.label(), + "expected_repo_name": attr.string(), + "expected_service_name": attr.string(), + "expected_runtime_module_path": attr.string(), "orchestrion_mode": attr.string(default = "general", values = ["general", "test_optimization"]), "bazel_package": attr.string(mandatory = True), "bazel_target": attr.string(mandatory = True), diff --git a/modules/go/topt_go_orchestrion.bzl b/modules/go/topt_go_orchestrion.bzl index a58363a4..8802b915 100644 --- a/modules/go/topt_go_orchestrion.bzl +++ b/modules/go/topt_go_orchestrion.bzl @@ -6,15 +6,16 @@ """Internal Orchestrion wrapper rule for Go tests.""" -load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") - _BAZEL_TARGET_METADATA_OUTPUT = "bazel_target_metadata.json" +_ORCHESTRION_ENABLED_SETTING = "@rules_go//go/private/orchestrion:enabled" +_ORCHESTRION_MODE_SETTING = "@rules_go//go/private/orchestrion:mode" _ORCHESTRION_MODE_GENERAL = "general" _ORCHESTRION_MODE_TEST_OPTIMIZATION = "test_optimization" def _orch_transition_impl(_settings, _attr): return { - "@rules_go//go/private/orchestrion:mode": _attr.orchestrion_mode, + _ORCHESTRION_ENABLED_SETTING: True, + _ORCHESTRION_MODE_SETTING: _attr.orchestrion_mode, } orch_transition_impl_for_tests = _orch_transition_impl @@ -22,7 +23,10 @@ orch_transition_impl_for_tests = _orch_transition_impl orch_transition = transition( implementation = _orch_transition_impl, inputs = [], - outputs = ["@rules_go//go/private/orchestrion:mode"], + outputs = [ + _ORCHESTRION_ENABLED_SETTING, + _ORCHESTRION_MODE_SETTING, + ], ) def _first_target(dep): @@ -107,13 +111,6 @@ exit /b %%ERRORLEVEL%% ) def _orch_go_test_impl(ctx): - if ctx.attr.test_optimization_enabled and not ctx.attr._orchestrion_enabled[BuildSettingInfo].value: - fail( - "orch_go_test: Test Optimization metadata is enabled but Orchestrion is disabled; " + - "run with --config=test-optimization. Consumers upgrading an existing setup should " + - "rerun dd_topt_go_bootstrap with --write-bazelrc before building tests.", - ) - dep_exe, dep_runfiles = _dep_exec_and_runfiles(ctx.attr.actual) dep_run_environment = _dep_run_environment_info(ctx.attr.actual) metadata_file = ctx.file.metadata @@ -164,17 +161,9 @@ orch_go_test = rule( ], doc = "Internal Orchestrion mode forwarded to the raw go_test target.", ), - "test_optimization_enabled": attr.bool( - default = False, - doc = "Whether the selected generated metadata repository is enabled.", - ), "_allowlist_function_transition": attr.label( default = "@bazel_tools//tools/allowlists/function_transition_allowlist", ), - "_orchestrion_enabled": attr.label( - default = "@rules_go//go/private/orchestrion:enabled", - providers = [BuildSettingInfo], - ), "_windows_constraint": attr.label(default = "@platforms//os:windows"), }, test = True, diff --git a/modules/go/topt_go_stdlib.bzl b/modules/go/topt_go_stdlib.bzl index b1ab5d13..0f00f45e 100644 --- a/modules/go/topt_go_stdlib.bzl +++ b/modules/go/topt_go_stdlib.bzl @@ -6,15 +6,16 @@ """Build target for warming the Test Optimization Go standard library.""" -load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") load("@rules_go//go/private:providers.bzl", "GoStdLib") load("@rules_go//go/private/rules:transition.bzl", "go_transition") +_ORCHESTRION_ENABLED_SETTING = "@rules_go//go/private/orchestrion:enabled" _ORCHESTRION_MODE_SETTING = "@rules_go//go/private/orchestrion:mode" _ORCHESTRION_MODE_TEST_OPTIMIZATION = "test_optimization" def _stdlib_warmup_transition_impl(_settings, _attr): return { + _ORCHESTRION_ENABLED_SETTING: True, _ORCHESTRION_MODE_SETTING: _ORCHESTRION_MODE_TEST_OPTIMIZATION, } @@ -32,6 +33,7 @@ _stdlib_warmup_transition = transition( implementation = _stdlib_warmup_transition_impl, inputs = [], outputs = [ + _ORCHESTRION_ENABLED_SETTING, _ORCHESTRION_MODE_SETTING, ], ) @@ -62,8 +64,6 @@ _go_transition_stdlib_warmup = rule( ) def _dd_topt_go_stdlib_warmup_impl(ctx): - if not ctx.attr._orchestrion_enabled[BuildSettingInfo].value: - return [DefaultInfo()] return [DefaultInfo(files = _first_target(ctx.attr.actual)[DefaultInfo].files)] _dd_topt_go_stdlib_warmup = rule( @@ -73,10 +73,6 @@ _dd_topt_go_stdlib_warmup = rule( mandatory = True, cfg = _stdlib_warmup_transition, ), - "_orchestrion_enabled": attr.label( - default = "@rules_go//go/private/orchestrion:enabled", - providers = [BuildSettingInfo], - ), "_allowlist_function_transition": attr.label( default = "@bazel_tools//tools/allowlists/function_transition_allowlist", ), diff --git a/modules/go/topt_go_test.bzl b/modules/go/topt_go_test.bzl index f5d34428..2c327578 100644 --- a/modules/go/topt_go_test.bzl +++ b/modules/go/topt_go_test.bzl @@ -113,6 +113,17 @@ _ORCHESTRION_MODES = { _ORCHESTRION_MODE_GENERAL: None, _ORCHESTRION_MODE_TEST_OPTIMIZATION: None, } +_STATIC_DESCRIPTOR_KEYS = { + "manifest_label": None, + "repo_name": None, + "runtime_module_path": None, + "service_name": None, +} +_APPARENT_REPO_NAME_CHARS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._+-" +_PUBLIC_WRAPPER_ONLY_TAGS = { + "dd-test-optimization": None, + "dd-test-optimization-source-manual": None, +} _TEST_BINARY_LINKER_OPTIMIZATION_GC_LINKOPTS = select({ # rules_go already strips Go binaries under these Bazel modes, and explicit # debug/no-strip modes should keep their normal linker behavior unless the @@ -161,6 +172,70 @@ def _validate_orchestrion_mode(orchestrion_mode): validate_orchestrion_mode_for_tests = _validate_orchestrion_mode +def _validate_apparent_repo_name(repo_name): + if type(repo_name) != type("") or not repo_name: + fail_with_prefix("dd_topt_go_test", "local-static descriptor repo_name must be a non-empty apparent repository name") + for char in repo_name.elems(): + if char not in _APPARENT_REPO_NAME_CHARS: + fail_with_prefix("dd_topt_go_test", "local-static descriptor repo_name %r contains invalid character %r" % (repo_name, char)) + return repo_name + +def _validate_non_empty_descriptor_string(descriptor, key): + value = descriptor.get(key) + if type(value) != type("") or not value.strip(): + fail_with_prefix("dd_topt_go_test", "local-static descriptor %s must be a non-empty string" % key) + return value.strip() + +def _normalize_static_descriptor(descriptor, topt_service): + """Validate and normalize the checked-in local descriptor shape.""" + if topt_service != None: + fail_with_prefix("dd_topt_go_test", "topt_service cannot be used with a local-static descriptor") + for key in descriptor.keys(): + if key not in _STATIC_DESCRIPTOR_KEYS: + fail_with_prefix("dd_topt_go_test", "local-static descriptor contains unsupported field %r" % key) + + repo_name = _validate_apparent_repo_name(_validate_non_empty_descriptor_string(descriptor, "repo_name")) + service_name = _validate_non_empty_descriptor_string(descriptor, "service_name") + runtime_module_path = _validate_non_empty_descriptor_string(descriptor, "runtime_module_path") + manifest_label = descriptor.get("manifest_label") + if manifest_label == None: + manifest_label = "@%s//:.testoptimization/manifest.txt" % repo_name + elif type(manifest_label) != type("") or not manifest_label: + fail_with_prefix("dd_topt_go_test", "local-static descriptor manifest_label must be a non-empty label string") + + expected_prefix = "@%s//" % repo_name + if not manifest_label.startswith(expected_prefix) or ":" not in manifest_label[len(expected_prefix):]: + fail_with_prefix( + "dd_topt_go_test", + "local-static descriptor manifest_label must be a complete label in repository @%s" % repo_name, + ) + Label(manifest_label) + + return { + "_local_static": True, + "enabled": True, + "manifest_label": manifest_label, + "repo_name": repo_name, + "runtime_module_path": runtime_module_path, + "service_name": service_name, + } + +normalize_static_descriptor_for_tests = _normalize_static_descriptor + +def _hidden_raw_tags(public_tags): + """Keep operational tags on the hidden test and normalize its manual tag.""" + if type(public_tags) == "select": + return public_tags + ["manual"] + tags = [] + for tag in list(public_tags or []): + if tag == "manual" or tag in _PUBLIC_WRAPPER_ONLY_TAGS: + continue + tags.append(tag) + tags.append("manual") + return tags + +hidden_raw_tags_for_tests = _hidden_raw_tags + def _validate_test_optimization_pin_files( orchestrion_mode, package_local_orchestrion_pin_files, @@ -348,10 +423,15 @@ def dd_topt_go_test( if go_test_rule == None: fail_with_prefix("dd_topt_go_test", "go_test_rule override cannot be None") - # Support both shapes: - # 1) Single-service dict with keys: repo_name, labels, set, runtimes - # 2) Aggregator mapping dict: { : single-service dict, ... } - if topt_data.get("repo_name"): + # Support three shapes: + # 1) Local-static descriptor identified by top-level runtime_module_path + # 2) Single-service generated export + # 3) Aggregator mapping of generated service exports + is_local_static = "runtime_module_path" in topt_data + if is_local_static: + _svc = _normalize_static_descriptor(topt_data, topt_service) + orchestrion_mode = _ORCHESTRION_MODE_TEST_OPTIMIZATION + elif topt_data.get("repo_name"): # Single-service shape: caller already selected the service by choosing # this exported dict, so no key resolution is needed. _svc = topt_data @@ -423,7 +503,9 @@ def dd_topt_go_test( if not _is_dict(_go): _go = {} uses_inference = bool(explicit_importpath) or bool(embed_labels) - if uses_inference: + if is_local_static: + include_per_module_files = True + elif uses_inference: # When we can infer importpath, always allow per-module selection. # Missing module matches still safely fall back in selector rule. include_per_module_files = True @@ -440,7 +522,9 @@ def dd_topt_go_test( # Build labels for files/context based on (possibly derived) sync_repo_name. # These labels remain stable public contracts of the generated sync repo. - files_label = resolve_files_label(_svc, sync_repo_name, macro_name = "dd_topt_go_test") + files_label = ( + "@%s//:test_optimization_files" % sync_repo_name if is_local_static else resolve_files_label(_svc, sync_repo_name, macro_name = "dd_topt_go_test") + ) # ------------------------------------------------------------------ # Phase 4: Build environment and selector inputs for analysis-time mapping. @@ -460,12 +544,24 @@ def dd_topt_go_test( # Build the list of per-module groups once (if any were exported) # Use exported sanitized labels directly to avoid re-deriving naming policy # in the macro and drifting from sync-side label generation. - module_labels = resolve_module_labels(_svc, sync_repo_name, macro_name = "dd_topt_go_test") - module_group_names = resolve_module_group_names( - _svc, - module_labels, - macro_name = "dd_topt_go_test", - ) + if is_local_static: + module_labels = [] + module_group_names = [] + repository_state_label = "@%s//:test_optimization_repository_state" % sync_repo_name + runtime_module_label = "@%s//:test_optimization_runtime_module" % sync_repo_name + expected_service_name = _svc["service_name"] + expected_runtime_module_path = _svc["runtime_module_path"] + else: + module_labels = resolve_module_labels(_svc, sync_repo_name, macro_name = "dd_topt_go_test") + module_group_names = resolve_module_group_names( + _svc, + module_labels, + macro_name = "dd_topt_go_test", + ) + repository_state_label = None + runtime_module_label = None + expected_service_name = "" + expected_runtime_module_path = "" # Mirror rules_go's fallback for the actual hidden go_test label. rules_go # only inherits an embed importpath when that path is explicit; otherwise @@ -489,6 +585,11 @@ def dd_topt_go_test( module_groups = module_labels, include_per_module = include_per_module_files, module_label_override = module_label_override, + repository_state = repository_state_label, + runtime_module = runtime_module_label, + expected_repo_name = sync_repo_name if is_local_static else "", + expected_service_name = expected_service_name, + expected_runtime_module_path = expected_runtime_module_path, ) # Emit a small Bazel-owned metadata file next to the built test artifacts. @@ -506,6 +607,11 @@ def dd_topt_go_test( module_groups = module_labels, include_per_module = include_per_module_files, module_label_override = module_label_override or "", + repository_state = repository_state_label, + runtime_module = runtime_module_label, + expected_repo_name = sync_repo_name if is_local_static else "", + expected_service_name = expected_service_name, + expected_runtime_module_path = expected_runtime_module_path, orchestrion_mode = orchestrion_mode, bazel_package = "//%s" % pkg_path if pkg_path else "//", bazel_target = "//%s:%s" % (pkg_path, name) if pkg_path else "//:%s" % name, @@ -574,7 +680,9 @@ def dd_topt_go_test( # manifest_path is emitted by sync metadata and may include slashes. # These paths are rooted at the sync repo package, so target syntax remains # @repo//: (for example @test_optimization_data//:.testoptimization/manifest.txt). - manifest_label = resolve_manifest_label(_svc, sync_repo_name, macro_name = "dd_topt_go_test") + manifest_label = ( + _svc["manifest_label"] if is_local_static else resolve_manifest_label(_svc, sync_repo_name, macro_name = "dd_topt_go_test") + ) data = _append_data_dependencies(data, [manifest_label]) required_env = { "DD_TEST_OPTIMIZATION_MANIFEST_FILE": "$(rlocationpath %s)" % manifest_label, @@ -605,7 +713,7 @@ def dd_topt_go_test( ) user_tags = wrapper_kwargs.get("tags") - kwargs["tags"] = (user_tags or []) + ["manual"] + kwargs["tags"] = _hidden_raw_tags(user_tags) kwargs["visibility"] = ["//visibility:private"] for key, value in raw_passthrough.items(): kwargs[key] = value @@ -624,6 +732,5 @@ def dd_topt_go_test( actual = ":" + raw_name, metadata = ":" + metadata_name, orchestrion_mode = orchestrion_mode, - test_optimization_enabled = bool(_svc.get("enabled", True)), **wrapper_kwargs ) diff --git a/tools/agent-skills/go-test-optimization-onboarding/references/bzlmod-onboarding.md b/tools/agent-skills/go-test-optimization-onboarding/references/bzlmod-onboarding.md index 6fd5bf8b..443e564b 100644 --- a/tools/agent-skills/go-test-optimization-onboarding/references/bzlmod-onboarding.md +++ b/tools/agent-skills/go-test-optimization-onboarding/references/bzlmod-onboarding.md @@ -38,12 +38,10 @@ Add these lines to the named config used by test, doctor, and uploader: ```text common:test-optimization --repo_env=DD_TEST_OPTIMIZATION_ENABLED=1 -build:test-optimization --@rules_go//go/private/orchestrion:enabled=true ``` -The config is the only user-facing switch. Removing -`--config=test-optimization` disables metadata fetching and selects the local -empty Orchestrion aliases; no second Test Optimization flag is required. +The optimized target transition enables Orchestrion. The config remains the +only user-facing metadata switch; no global Orchestrion flag is required. Use a commit that is reachable from `origin/main`. Do not publish branch-only commits in consumer snippets because squash merges can make them disappear. diff --git a/tools/agent-skills/go-test-optimization-onboarding/references/workspace-onboarding.md b/tools/agent-skills/go-test-optimization-onboarding/references/workspace-onboarding.md index bb1bf3bd..62976d94 100644 --- a/tools/agent-skills/go-test-optimization-onboarding/references/workspace-onboarding.md +++ b/tools/agent-skills/go-test-optimization-onboarding/references/workspace-onboarding.md @@ -85,11 +85,10 @@ Add these lines to the named config used by test, doctor, and uploader: ```text common:test-optimization --repo_env=DD_TEST_OPTIMIZATION_ENABLED=1 -build:test-optimization --@io_bazel_rules_go//go/private/orchestrion:enabled=true ``` -`--config=test-optimization` is the only user-facing switch. Omitting it -renders disabled metadata stubs and selects the local empty Orchestrion aliases. +`--config=test-optimization` is the only user-facing metadata switch. The +optimized target transition enables Orchestrion; no global flag is required. For a managed monorepo, declare one aggregate repository after the Rule dependency: @@ -417,7 +416,6 @@ common:test-optimization --repo_env=DD_GIT_PR_BASE_BRANCH common:test-optimization --repo_env=DD_GIT_PR_BASE_BRANCH_SHA common:test-optimization --repo_env=DD_GIT_PR_BASE_BRANCH_HEAD_SHA common:test-optimization --repo_env=DD_PR_NUMBER -build:test-optimization --@io_bazel_rules_go//go/private/orchestrion:enabled=true test:test-optimization --remote_download_minimal test:test-optimization --remote_download_regex=.*test[.]outputs.* test:test-optimization --zip_undeclared_test_outputs diff --git a/tools/agent-skills/rules-go-orchestrion-upstream-migration/SKILL.md b/tools/agent-skills/rules-go-orchestrion-upstream-migration/SKILL.md index 9f859ff2..039d2513 100644 --- a/tools/agent-skills/rules-go-orchestrion-upstream-migration/SKILL.md +++ b/tools/agent-skills/rules-go-orchestrion-upstream-migration/SKILL.md @@ -142,14 +142,12 @@ alias contract from the vendored base tree: ```bazelrc common:test-optimization --repo_env=DD_TEST_OPTIMIZATION_ENABLED=1 -build:test-optimization --@rules_go//go/private/orchestrion:enabled=true ``` -`--config=test-optimization` is the only user-facing switch. Omitting it must -leave metadata bootstrap disabled for public Go extension repositories, and -for low-level repositories explicitly configured with `enabled_by_env = True`, -while selecting local empty Orchestrion aliases. Do not add a consumer-local -duplicate bool flag or collapse the real and empty repository rules. +`--config=test-optimization` is the only user-facing metadata switch. Optimized +targets enable Orchestrion through their transition. Omitting the config must +leave metadata bootstrap disabled for public Go extension repositories and for +low-level repositories explicitly configured with `enabled_by_env = True`. ## Stop Conditions diff --git a/tools/core/BUILD.bazel b/tools/core/BUILD.bazel index c1dbc038..221c395e 100644 --- a/tools/core/BUILD.bazel +++ b/tools/core/BUILD.bazel @@ -15,6 +15,7 @@ exports_files( "test_optimization_sync.bzl", "test_optimization_manifest_sync.bzl", "test_optimization_multi_sync.bzl", + "test_optimization_repository_state.bzl", "test_optimization_context_utils.bzl", "test_optimization_doctor.bzl", "test_optimization_doctor.py", diff --git a/tools/core/test_optimization_doctor.bzl b/tools/core/test_optimization_doctor.bzl index 0ab30c4e..9bb5fc38 100644 --- a/tools/core/test_optimization_doctor.bzl +++ b/tools/core/test_optimization_doctor.bzl @@ -43,6 +43,9 @@ def _doctor_impl(ctx): _validate_expected_targets(ctx.attr.expected_targets) expected_targets_file = ctx.file.expected_targets_file + if ctx.attr.runtime_selection and (ctx.attr.data or ctx.attr.expected_targets or expected_targets_file): + fail_with_prefix(_OWNER, "runtime_selection cannot be combined with configured data, expected_targets, or expected_targets_file") + context_entries = context_manifest_entries_or_fail(ctx.attr.data, ctx.files.data, _OWNER) context_manifest = ctx.actions.declare_file(ctx.label.name + ".context_manifest") ctx.actions.write( @@ -60,6 +63,7 @@ def _doctor_impl(ctx): ' "expected_targets": %s,' % _json_string_list(ctx.attr.expected_targets), ' "expected_targets_file_path": %s,' % json.encode(expected_targets_file.path if expected_targets_file else ""), ' "expected_targets_file_short_path": %s,' % json.encode(expected_targets_file.short_path if expected_targets_file else ""), + ' "runtime_selection": %s,' % _json_bool(ctx.attr.runtime_selection), ' "require_git_metadata": %s,' % _json_bool(ctx.attr.require_git_metadata), ' "require_bazel_metadata": %s,' % _json_bool(ctx.attr.require_bazel_metadata), ' "require_json_payloads": %s,' % _json_bool(ctx.attr.require_json_payloads), @@ -398,6 +402,7 @@ dd_test_optimization_doctor = rule( "data": attr.label_list(allow_files = True, doc = "Context files, normally @test_optimization_data//:test_optimization_context."), "expected_targets": attr.string_list(default = [], doc = "Optional local labels whose bazel-testlogs outputs must be present."), "expected_targets_file": attr.label(allow_single_file = True, doc = "Optional generated JSON file containing invocation-scoped expected targets."), + "runtime_selection": attr.bool(default = False, doc = "Require expected targets and keyed context files to be supplied at runtime."), "require_git_metadata": attr.bool(default = True, doc = "Require repository URL, commit SHA, and branch/tag in context.json."), "require_bazel_metadata": attr.bool(default = True, doc = "Require bazel_target_metadata.json next to payload outputs."), "require_json_payloads": attr.bool(default = True, doc = "Require parseable JSON payload files."), diff --git a/tools/core/test_optimization_doctor.py b/tools/core/test_optimization_doctor.py index bd0d1d22..85231b74 100644 --- a/tools/core/test_optimization_doctor.py +++ b/tools/core/test_optimization_doctor.py @@ -212,6 +212,8 @@ def _parse_args(argv: list[str]) -> argparse.Namespace: """Parse doctor runtime arguments, including optional BEP freshness flags.""" parser = argparse.ArgumentParser() parser.add_argument("--config", required=True) + parser.add_argument("--expected-target", action="append", default=[]) + parser.add_argument("--context-entry", action="append", default=[]) parser.add_argument("--bep-json", action="append", default=[]) parser.add_argument( "--freshness-source", @@ -533,6 +535,8 @@ def _load_expected_targets_file(path: Path) -> list[str]: def _resolve_expected_targets( config: dict[str, Any], config_path: Path, + runtime_targets: list[str] | None = None, + runtime_selection: bool = False, ) -> tuple[list[str], str]: static_targets = config["expected_targets"] for label in static_targets: @@ -544,17 +548,96 @@ def _resolve_expected_targets( "expected_targets_file_path", "expected_targets_file_short_path", ) - if expected_targets_file is None: - return list(static_targets), "static" if static_targets else "discovery" + configured_targets = list(static_targets) + configured_source = "static" if static_targets else "discovery" + if expected_targets_file is not None: + dynamic_targets = _load_expected_targets_file(expected_targets_file) + if static_targets and set(static_targets) != set(dynamic_targets): + _fail( + "static expected_targets and expected_targets_file contain different target sets" + ) + configured_targets = sorted(set(static_targets)) if static_targets else dynamic_targets + configured_source = "static_and_file" if static_targets else "file" + + runtime_targets = runtime_targets or [] + normalized_runtime = [_validate_expected_target_label(label) for label in runtime_targets] + if len(set(normalized_runtime)) != len(normalized_runtime): + _fail("runtime --expected-target arguments contain duplicate target labels") + normalized_runtime = sorted(normalized_runtime) + if normalized_runtime: + if configured_targets and set(configured_targets) != set(normalized_runtime): + _fail("configured and runtime expected targets contain different target sets") + return normalized_runtime, "runtime_and_configured" if configured_targets else "runtime" + if runtime_selection: + _fail("runtime selection requires at least one --expected-target argument") + return configured_targets, configured_source + + +_APPARENT_REPO_NAME_CHARS = frozenset( + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._+-" +) - dynamic_targets = _load_expected_targets_file(expected_targets_file) - if static_targets and set(static_targets) != set(dynamic_targets): - _fail( - "static expected_targets and expected_targets_file contain different target sets" - ) - if static_targets: - return sorted(set(static_targets)), "static_and_file" - return dynamic_targets, "file" + +def _validate_runtime_repo_name(repo_name: str) -> str: + if not repo_name or any(ch not in _APPARENT_REPO_NAME_CHARS for ch in repo_name): + _fail(f"context entry has invalid apparent repository name: {repo_name!r}") + return repo_name + + +def _load_runtime_contexts(entries: list[str]) -> list[tuple[str, Path]]: + """Validate keyed runtime context files and their telemetry siblings.""" + contexts: list[tuple[str, Path]] = [] + seen_repos: set[str] = set() + seen_paths: set[Path] = set() + for entry in entries: + repo_name, separator, raw_path = entry.partition("=") + if not separator or not raw_path: + _fail("--context-entry must use =") + repo_name = _validate_runtime_repo_name(repo_name) + if repo_name in seen_repos: + _fail(f"duplicate --context-entry repository name: {repo_name!r}") + context_path = Path(raw_path).expanduser().resolve() + if context_path.name != "context.json" or not context_path.is_file(): + _fail(f"context entry for {repo_name!r} must reference an existing regular context.json file") + if context_path in seen_paths: + _fail(f"duplicate --context-entry path: {context_path}") + telemetry_path = context_path.with_name("telemetry_facts.json") + if not telemetry_path.is_file(): + _fail(f"context entry for {repo_name!r} is missing sibling telemetry_facts.json") + + context = _load_json(context_path) + if not isinstance(context, dict): + _fail(f"runtime context for {repo_name!r} must be a JSON object") + context_repo = context.get("topt.sync.repository_name") + service_name = context.get("service.name") + runtime_name = context.get("runtime.name") + if context_repo != repo_name: + _fail( + f"runtime context repository mismatch: argument={repo_name!r} context={context_repo!r}" + ) + if not isinstance(service_name, str) or not service_name: + _fail(f"runtime context for {repo_name!r} is missing service.name") + if not isinstance(runtime_name, str) or not runtime_name: + _fail(f"runtime context for {repo_name!r} is missing runtime.name") + + telemetry = _load_json(telemetry_path) + if not isinstance(telemetry, dict): + _fail(f"telemetry facts for {repo_name!r} must be a JSON object") + if telemetry.get("schema_version") != 1: + _fail(f"telemetry facts for {repo_name!r} must use schema_version 1") + if not isinstance(telemetry.get("counts"), list) or not isinstance( + telemetry.get("distributions"), list + ): + _fail(f"telemetry facts for {repo_name!r} has an invalid facts schema") + if telemetry.get("service_name") != service_name: + _fail(f"runtime context and telemetry service mismatch for {repo_name!r}") + if telemetry.get("runtime_name") != runtime_name: + _fail(f"runtime context and telemetry runtime mismatch for {repo_name!r}") + + seen_repos.add(repo_name) + seen_paths.add(context_path) + contexts.append((repo_name, context_path)) + return sorted(contexts, key=lambda item: item[0]) def _expected_target_outputs(testlogs_dir: Path | None, label: str, *, allow_missing: bool = False) -> list[Path]: @@ -1888,7 +1971,10 @@ def _resolve_configured_context_manifest(config: dict[str, Any], config_path: Pa def _validate_git_metadata(context_manifest: Path) -> None: - contexts = _load_contexts(context_manifest) + _validate_git_metadata_contexts(_load_contexts(context_manifest)) + + +def _validate_git_metadata_contexts(contexts: list[tuple[str, Path]]) -> None: if not contexts: _fail("require_git_metadata=True but no context.json was provided in data") for repo_key, context_path in contexts: @@ -2709,16 +2795,28 @@ def _run_doctor(args: argparse.Namespace, report: dict[str, Any]) -> int: if execroot is not None and not os.environ.get(DOCTOR_EXECROOT_ENV): os.environ[DOCTOR_EXECROOT_ENV] = str(execroot) + runtime_selection = bool(config.get("runtime_selection", False)) + expected_targets, expected_targets_source = _resolve_expected_targets( + config, + config_path, + args.expected_target, + runtime_selection, + ) + runtime_contexts = _load_runtime_contexts(args.context_entry) + if runtime_selection and not runtime_contexts: + _fail("runtime selection requires at least one --context-entry argument") + workspace = _workspace_root() testlogs_dir = _resolve_testlogs_dir(workspace, allow_missing=staging_requested) if config["forbid_dd_git_test_env"]: _validate_bazelrc(workspace) if config["require_git_metadata"]: - context_manifest = _resolve_configured_context_manifest(config, config_path) - _validate_git_metadata(context_manifest) - - expected_targets, expected_targets_source = _resolve_expected_targets(config, config_path) + if runtime_contexts: + _validate_git_metadata_contexts(runtime_contexts) + else: + context_manifest = _resolve_configured_context_manifest(config, config_path) + _validate_git_metadata(context_manifest) _update_diagnostic_config( report, config_path=config_path, diff --git a/tools/core/test_optimization_repository_state.bzl b/tools/core/test_optimization_repository_state.bzl new file mode 100644 index 00000000..5bcf380c --- /dev/null +++ b/tools/core/test_optimization_repository_state.bzl @@ -0,0 +1,44 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Analysis-time state exported by a Test Optimization sync repository.""" + +TestOptimizationRepositoryStateInfo = provider( + doc = "Stable identity and enablement state for one synchronized runtime repository.", + fields = { + "disabled_reason": "Safe user-facing reason when synchronization is disabled.", + "enabled": "Whether the repository fetched live Test Optimization metadata.", + "repo_name": "Apparent repository name exported to consumers.", + "runtime_module_included": "Whether the configured runtime module has a dedicated payload group.", + "runtime_module_path": "Configured runtime module path.", + "runtime_name": "Runtime name associated with this repository.", + "service_name": "Service name associated with this repository.", + }, +) + +def _test_optimization_repository_state_impl(ctx): + return [TestOptimizationRepositoryStateInfo( + disabled_reason = ctx.attr.disabled_reason, + enabled = ctx.attr.enabled, + repo_name = ctx.attr.repo_name, + runtime_module_included = ctx.attr.runtime_module_included, + runtime_module_path = ctx.attr.runtime_module_path, + runtime_name = ctx.attr.runtime_name, + service_name = ctx.attr.service_name, + )] + +test_optimization_repository_state = rule( + implementation = _test_optimization_repository_state_impl, + attrs = { + "disabled_reason": attr.string(), + "enabled": attr.bool(mandatory = True), + "repo_name": attr.string(mandatory = True), + "runtime_module_included": attr.bool(mandatory = True), + "runtime_module_path": attr.string(mandatory = True), + "runtime_name": attr.string(mandatory = True), + "service_name": attr.string(mandatory = True), + }, +) diff --git a/tools/core/test_optimization_sync.bzl b/tools/core/test_optimization_sync.bzl index ad2238cf..56fdbb35 100644 --- a/tools/core/test_optimization_sync.bzl +++ b/tools/core/test_optimization_sync.bzl @@ -1472,10 +1472,56 @@ def _render_disabled_export(repo_name, service, runtime_name, runtime_module_pat enabled = False, ) -def _render_disabled_build(settings_file, manifest_file, known_tests_file, test_management_file, flaky_tests_file, context_file, telemetry_facts_file): +def _render_repository_state_target( + enabled, + repo_name, + service_name, + runtime_name, + runtime_module_path, + runtime_module_included, + disabled_reason = ""): + """Render the stable analysis-time repository-state target.""" + return ( + "test_optimization_repository_state(\n" + + ' name = "test_optimization_repository_state",\n' + + " enabled = %s,\n" % ("True" if enabled else "False") + + " repo_name = %s,\n" % _bzl_string_literal(repo_name or "") + + " service_name = %s,\n" % _bzl_string_literal(service_name or "") + + " runtime_name = %s,\n" % _bzl_string_literal(runtime_name or "") + + " runtime_module_path = %s,\n" % _bzl_string_literal(runtime_module_path or "") + + " runtime_module_included = %s,\n" % ("True" if runtime_module_included else "False") + + " disabled_reason = %s,\n" % _bzl_string_literal(disabled_reason or "") + + ' visibility = ["//visibility:public"],\n' + + ")\n" + ) + +def _render_runtime_module_target(module_label = ""): + """Render the stable filegroup for the configured runtime module.""" + srcs = [":module_%s" % module_label] if module_label else [] + return ( + "filegroup(\n" + + ' name = "test_optimization_runtime_module",\n' + + " srcs = %s,\n" % repr(srcs) + + ' visibility = ["//visibility:public"],\n' + + ")\n" + ) + +def _render_disabled_build( + settings_file, + manifest_file, + known_tests_file, + test_management_file, + flaky_tests_file, + context_file, + telemetry_facts_file, + repo_name = "", + service_name = "", + runtime_name = "", + runtime_module_path = ""): """Render public disabled targets with the same file labels as live repos.""" return ( - 'load(":module_runfiles.bzl", "topt_module_files")\n\n' + + 'load(":module_runfiles.bzl", "topt_module_files")\n' + + 'load("@datadog-rules-test-optimization//tools/core:test_optimization_repository_state.bzl", "test_optimization_repository_state")\n\n' + "filegroup(\n" + ' name = "test_optimization_files",\n' + " srcs = %s,\n" % repr([settings_file, manifest_file, known_tests_file, test_management_file, flaky_tests_file]) + @@ -1486,6 +1532,18 @@ def _render_disabled_build(settings_file, manifest_file, known_tests_file, test_ " srcs = %s,\n" % repr([context_file, telemetry_facts_file]) + ' visibility = ["//visibility:public"],\n' + ")\n\n" + + _render_runtime_module_target() + + "\n" + + _render_repository_state_target( + enabled = False, + repo_name = repo_name, + service_name = service_name, + runtime_name = runtime_name, + runtime_module_path = runtime_module_path, + runtime_module_included = False, + disabled_reason = "disabled by repository configuration", + ) + + "\n" + 'exports_files(["export.bzl", %s], visibility = ["//visibility:public"])\n' % repr(manifest_file) ) @@ -1558,6 +1616,10 @@ def _write_disabled_repository(ctx, out_dir, env_data, debug): flaky_tests_file, context_file, telemetry_facts_file, + repo_name = repo_name, + service_name = env_data.get("service") or "", + runtime_name = runtime_name, + runtime_module_path = runtime_module_path, ), ) @@ -1846,6 +1908,8 @@ render_disabled_settings_json_for_tests = _render_disabled_settings_json render_disabled_test_management_json_for_tests = _render_disabled_test_management_json render_disabled_telemetry_facts_json_for_tests = _render_disabled_telemetry_facts_json render_disabled_export_for_tests = _render_disabled_export +render_repository_state_target_for_tests = _render_repository_state_target +render_runtime_module_target_for_tests = _render_runtime_module_target render_disabled_build_for_tests = _render_disabled_build is_test_optimization_enabled_for_tests = _is_test_optimization_enabled repository_environ_for_tests = _TEST_OPTIMIZATION_REPOSITORY_ENVIRON @@ -2164,7 +2228,7 @@ def _build_configurations_json(ctx, debug, osinfo = None, runtime = None): log_debug(debug, "config", "Configurations JSON: %s" % conf_json) return conf_json -def _build_context_tags(ctx, env_data, api_key, debug, osinfo = None, runtime = None): +def _build_context_tags(ctx, env_data, api_key, debug, osinfo = None, runtime = None, repo_name = ""): """Build non-secret context tags stored in generated `context.json`.""" # _build_context_tags: aggregates CI, git, OS, and runtime tags for context.json @@ -2196,6 +2260,7 @@ def _build_context_tags(ctx, env_data, api_key, debug, osinfo = None, runtime = tags["bazel.rule_version"] = TEST_BAZEL_RULE_VERSION tags["bazel.os"] = osinfo.get("platform") or "unknown" tags["bazel.arch"] = osinfo.get("arch") or "unknown" + tags["topt.sync.repository_name"] = repo_name or getattr(ctx.attr, "repo_name", None) or getattr(ctx, "name", "") # Git tags if env_data.get("repository_url"): @@ -2914,7 +2979,15 @@ def _materialize_enabled_context(ctx, spec, emit_surface = True): # Build and write context.json (non-secret metadata) under `out_dir` # so all manifest-relative payload files share a single root. - context_tags = _build_context_tags(ctx, env_data, api_key, debug, osinfo = osinfo, runtime = runtime) + context_tags = _build_context_tags( + ctx, + env_data, + api_key, + debug, + osinfo = osinfo, + runtime = runtime, + repo_name = spec["repo_name"], + ) ctx.file(context_file, json.encode(context_tags) + "\n") ctx.file(telemetry_facts_file, json.encode(telemetry_facts) + "\n") @@ -3014,6 +3087,33 @@ def _materialize_enabled_context(ctx, spec, emit_surface = True): # Unified export file for simpler loading from user repos # Prefer the apparent repo name passed by the extension/WORKSPACE helper; fallback to ctx.name repo_name = spec["repo_name"] + repository_runtime_module_path = "" + repository_runtime_module_label = "" + repository_runtime_module_included = False + if runtime_name == "go": + repository_runtime_module_path = go_module_path + repository_runtime_module_label = sanitized_go_module_path + repository_runtime_module_included = go_module_included + elif runtime_name == "python": + repository_runtime_module_path = python_module_path + repository_runtime_module_label = sanitized_python_module_path + repository_runtime_module_included = python_module_included + elif runtime_name == "java": + repository_runtime_module_path = java_module_path + repository_runtime_module_label = sanitized_java_module_path + repository_runtime_module_included = java_module_included + elif runtime_name == "nodejs": + repository_runtime_module_path = nodejs_module_path + repository_runtime_module_label = sanitized_nodejs_module_path + repository_runtime_module_included = nodejs_module_included + elif runtime_name == "dotnet": + repository_runtime_module_path = dotnet_module_path + repository_runtime_module_label = sanitized_dotnet_module_path + repository_runtime_module_included = dotnet_module_included + elif runtime_name == "ruby": + repository_runtime_module_path = ruby_module_path + repository_runtime_module_label = sanitized_ruby_module_path + repository_runtime_module_included = ruby_module_included export_bzl = _render_export_bzl( repo_name, validated_service, @@ -3059,6 +3159,7 @@ def _materialize_enabled_context(ctx, spec, emit_surface = True): exp = repr(exports) build_content = ( 'load(":module_runfiles.bzl", "topt_module_files")\n' + + 'load("@datadog-rules-test-optimization//tools/core:test_optimization_repository_state.bzl", "test_optimization_repository_state")\n\n' + "filegroup(\n" + ' name = "test_optimization_files",\n' + (" srcs = %s,\n" % exp) + @@ -3146,6 +3247,21 @@ def _materialize_enabled_context(ctx, spec, emit_surface = True): ((" flaky_tests = %s,\n" % _bzl_string_literal(flaky_tests_file)) if flaky_tests_file else "") + ' visibility = ["//visibility:public"],\n' + ")\n") + build_content += ( + "\n" + + _render_runtime_module_target( + repository_runtime_module_label if repository_runtime_module_included else "", + ) + + "\n" + + _render_repository_state_target( + enabled = True, + repo_name = repo_name, + service_name = validated_service, + runtime_name = runtime_name, + runtime_module_path = repository_runtime_module_path, + runtime_module_included = repository_runtime_module_included, + ) + ) if emit_surface: log_debug(debug, "build", "Creating BUILD file with content: %s" % build_content) ctx.report_progress("test_optimization_sync: writing BUILD") diff --git a/tools/core/test_optimization_targets.bzl b/tools/core/test_optimization_targets.bzl index c5c9d168..cf9eb22b 100644 --- a/tools/core/test_optimization_targets.bzl +++ b/tools/core/test_optimization_targets.bzl @@ -16,8 +16,8 @@ them in the workspace root package. load("//tools/core:test_optimization_doctor.bzl", "dd_test_optimization_doctor") load("//tools/core:test_optimization_uploader.bzl", "dd_payload_uploader") -_DOCTOR_CONTROLLED_ATTRS = ["name", "data", "expected_targets", "expected_targets_file"] -_UPLOADER_CONTROLLED_ATTRS = ["name", "data", "expected_targets", "expected_targets_file"] +_DOCTOR_CONTROLLED_ATTRS = ["name", "data", "expected_targets", "expected_targets_file", "runtime_selection"] +_UPLOADER_CONTROLLED_ATTRS = ["name", "data", "expected_targets", "expected_targets_file", "runtime_selection"] def _copy_kwargs(kwargs, label): """Return a defensive copy of optional keyword arguments.""" @@ -57,7 +57,8 @@ def _build_test_optimization_target_specs( context_data, doctor_kwargs, uploader_kwargs, - expected_targets_file = None): + expected_targets_file = None, + runtime_selection = False): """Build normalized doctor/uploader attrs without materializing rules. Args: @@ -86,7 +87,16 @@ def _build_test_optimization_target_specs( ) % doctor_name, ) - normalized_context_data = context_data + if runtime_selection: + if context_data != None: + fail("dd_test_optimization_targets: runtime_selection cannot be combined with context_data") + if expected_targets: + fail("dd_test_optimization_targets: runtime_selection cannot be combined with expected_targets") + if expected_targets_file != None: + fail("dd_test_optimization_targets: runtime_selection cannot be combined with expected_targets_file") + normalized_context_data = [] + else: + normalized_context_data = context_data if normalized_context_data == None: normalized_context_data = ["@%s//:test_optimization_context" % sync_repo_name] @@ -100,6 +110,7 @@ def _build_test_optimization_target_specs( "name": doctor_name, "data": normalized_context_data, "expected_targets": expected_targets, + "runtime_selection": runtime_selection, }) if expected_targets_file != None: doctor_attrs["expected_targets_file"] = expected_targets_file @@ -109,6 +120,7 @@ def _build_test_optimization_target_specs( "name": uploader_name, "data": normalized_context_data, "expected_targets": expected_targets, + "runtime_selection": runtime_selection, }) if expected_targets_file != None: uploader_attrs["expected_targets_file"] = expected_targets_file @@ -128,6 +140,7 @@ def dd_test_optimization_targets( expected_targets = [], expected_targets_file = None, context_data = None, + runtime_selection = False, doctor_kwargs = None, uploader_kwargs = None): """Create the standard doctor and uploader targets for one workspace. @@ -150,6 +163,7 @@ def dd_test_optimization_targets( context_data = context_data, doctor_kwargs = doctor_kwargs, uploader_kwargs = uploader_kwargs, + runtime_selection = runtime_selection, ) dd_test_optimization_doctor(**specs.doctor_attrs) dd_payload_uploader(**specs.uploader_attrs) diff --git a/tools/core/test_optimization_uploader.bzl b/tools/core/test_optimization_uploader.bzl index e5d66125..774db33a 100644 --- a/tools/core/test_optimization_uploader.bzl +++ b/tools/core/test_optimization_uploader.bzl @@ -146,7 +146,8 @@ def _base_template_substitutions( expected_targets_rloc, expected_targets_path, expected_targets_file_rloc, - expected_targets_file_path): + expected_targets_file_path, + runtime_selection): """Build shared template substitutions for Bash/PowerShell scripts.""" return { "quiescent_sec": quiescent_sec, @@ -173,6 +174,7 @@ def _base_template_substitutions( "expected_targets_path": expected_targets_path, "expected_targets_file_rloc": expected_targets_file_rloc, "expected_targets_file_path": expected_targets_file_path, + "runtime_selection": _bool_to_str(runtime_selection), "rules_version": RULES_VERSION, } @@ -679,6 +681,8 @@ def _uploader_impl(ctx): filter_prefix_enabled = ctx.attr.filter_prefix gzip_payloads = ctx.attr.gzip_payloads expected_targets_file = ctx.file.expected_targets_file + if ctx.attr.runtime_selection and (ctx.attr.data or ctx.attr.expected_targets or expected_targets_file): + fail_with_prefix("test_optimization_uploader", "runtime_selection cannot be combined with configured data, expected_targets, or expected_targets_file") expected_targets = ctx.actions.declare_file(ctx.label.name + ".expected_targets") ctx.actions.write( output = expected_targets, @@ -790,6 +794,7 @@ def _uploader_impl(ctx): expected_targets.path, expected_targets_file.short_path if expected_targets_file else "", expected_targets_file.path if expected_targets_file else "", + ctx.attr.runtime_selection, ) bash_substitutions["curl_retry_flags"] = " ".join(_bash_curl_retry_flags_for_tests()) bash_file = ctx.actions.declare_file(ctx.label.name + ".sh") @@ -833,6 +838,7 @@ def _uploader_impl(ctx): expected_targets.path, expected_targets_file.short_path if expected_targets_file else "", expected_targets_file.path if expected_targets_file else "", + ctx.attr.runtime_selection, ), ), is_executable = False, @@ -896,6 +902,7 @@ _dd_payload_uploader_rule = rule( "data": attr.label_list(allow_files = True, doc = "Data files to include in runfiles (e.g., context.json for enrichment)"), "expected_targets": attr.string_list(default = [], doc = "Optional local labels whose current-invocation outputs the uploader must account for."), "expected_targets_file": attr.label(allow_single_file = True, doc = "Optional generated schema-v1 JSON file containing invocation-scoped expected targets."), + "runtime_selection": attr.bool(default = False, doc = "Require expected targets and keyed context files to be supplied at runtime."), # Schema + validator bundled for best-effort payload validation "_schema": attr.label(default = "//tools/core:schemas/agentless-schema.json", allow_single_file = True), "_schema_validator": attr.label(default = "//tools/core:validate_payload_schema.py", allow_single_file = True), diff --git a/tools/core/uploader_bash_runtime.sh.tpl b/tools/core/uploader_bash_runtime.sh.tpl index 0573375a..e5865c12 100644 --- a/tools/core/uploader_bash_runtime.sh.tpl +++ b/tools/core/uploader_bash_runtime.sh.tpl @@ -305,6 +305,197 @@ resolve_runtime_file_path() { resolve_artifact_path "$input_path" } +RUNTIME_SELECTION_REQUIRED="__DDTPL_RUNTIME_SELECTION__" +RUNTIME_EXPECTED_TARGETS=() +RUNTIME_CONTEXT_REPOS=() +RUNTIME_CONTEXT_FILES=() +RUNTIME_TELEMETRY_FACTS_FILES=() + +validate_runtime_expected_target() { + local label="$1" + if [[ ! "$label" =~ ^//[^:[:space:]]*:[^:[:space:]]+$ || "$label" == *"..."* || "$label" == *"*"* || "$label" == *"\\"* ]]; then + log "error: --expected-target must be a fully expanded local //pkg:target label, got '$label'" + exit 2 + fi +} + +validate_runtime_context_pair() { + local repo_key="$1" + local context_file="$2" + local telemetry_file="$3" + if command -v jq >/dev/null 2>&1; then + if ! jq -e --arg repo "$repo_key" ' + type == "object" and + .["topt.sync.repository_name"] == $repo and + (.["service.name"] | type == "string" and length > 0) and + (.["runtime.name"] | type == "string" and length > 0) + ' "$context_file" >/dev/null; then + log "error: runtime context/telemetry identity or schema mismatch for repository '$repo_key'" + exit 2 + fi + local service_name runtime_name + service_name="$(jq -r '.["service.name"]' "$context_file")" + runtime_name="$(jq -r '.["runtime.name"]' "$context_file")" + if ! jq -e --arg service "$service_name" --arg runtime "$runtime_name" ' + type == "object" and + .schema_version == 1 and + .service_name == $service and + .runtime_name == $runtime and + (.counts | type == "array") and + (.distributions | type == "array") + ' "$telemetry_file" >/dev/null; then + log "error: runtime context/telemetry identity or schema mismatch for repository '$repo_key'" + exit 2 + fi + return 0 + fi + if command -v python3 >/dev/null 2>&1; then + if ! python3 - "$repo_key" "$context_file" "$telemetry_file" <<'PY' +import json +import sys + +repo, context_path, telemetry_path = sys.argv[1:] +with open(context_path, encoding="utf-8-sig") as handle: + context = json.load(handle) +with open(telemetry_path, encoding="utf-8-sig") as handle: + telemetry = json.load(handle) +valid = ( + isinstance(context, dict) + and context.get("topt.sync.repository_name") == repo + and isinstance(context.get("service.name"), str) + and bool(context.get("service.name")) + and isinstance(context.get("runtime.name"), str) + and bool(context.get("runtime.name")) + and isinstance(telemetry, dict) + and telemetry.get("schema_version") == 1 + and telemetry.get("service_name") == context.get("service.name") + and telemetry.get("runtime_name") == context.get("runtime.name") + and isinstance(telemetry.get("counts"), list) + and isinstance(telemetry.get("distributions"), list) +) +raise SystemExit(0 if valid else 1) +PY + then + log "error: runtime context/telemetry identity or schema mismatch for repository '$repo_key'" + exit 2 + fi + return 0 + fi + log "error: runtime --context-entry validation requires jq or python3" + exit 2 +} + +scan_runtime_selection_args() { + local args=("$@") + local i=0 arg value repo_key raw_path resolved telemetry existing + while (( i < ${#args[@]} )); do + arg="${args[$i]}" + case "$arg" in + --expected-target) + ((++i)) + if (( i >= ${#args[@]} )); then + log "error: --expected-target requires a local label" + exit 2 + fi + value="${args[$i]}" + ;; + --expected-target=*) + value="${arg#--expected-target=}" + ;; + --context-entry) + ((++i)) + if (( i >= ${#args[@]} )); then + log "error: --context-entry requires =" + exit 2 + fi + value="${args[$i]}" + ;; + --context-entry=*) + value="${arg#--context-entry=}" + ;; + *) + ((++i)) + continue + ;; + esac + + if [[ "$arg" == --expected-target* ]]; then + validate_runtime_expected_target "$value" + for existing in "${RUNTIME_EXPECTED_TARGETS[@]}"; do + if [[ "$existing" == "$value" ]]; then + log "error: duplicate --expected-target '$value'" + exit 2 + fi + done + RUNTIME_EXPECTED_TARGETS+=("$value") + else + repo_key="${value%%=*}" + raw_path="${value#*=}" + if [[ "$repo_key" == "$value" || -z "$repo_key" || -z "$raw_path" || "$repo_key" == *[^A-Za-z0-9._+-]* ]]; then + log "error: --context-entry must use =" + exit 2 + fi + resolved="$(resolve_runtime_file_path "$raw_path")" + if [[ -z "$resolved" || ! -f "$resolved" || "$(basename "$resolved")" != "context.json" ]]; then + log "error: context entry for '$repo_key' must reference an existing regular context.json file" + exit 2 + fi + telemetry="$(dirname "$resolved")/telemetry_facts.json" + if [[ ! -f "$telemetry" ]]; then + log "error: context entry for '$repo_key' is missing sibling telemetry_facts.json" + exit 2 + fi + for existing in "${RUNTIME_CONTEXT_REPOS[@]}"; do + if [[ "$existing" == "$repo_key" ]]; then + log "error: duplicate --context-entry repository '$repo_key'" + exit 2 + fi + done + for existing in "${RUNTIME_CONTEXT_FILES[@]}"; do + if [[ "$existing" == "$resolved" ]]; then + log "error: duplicate --context-entry path '$resolved'" + exit 2 + fi + done + validate_runtime_context_pair "$repo_key" "$resolved" "$telemetry" + RUNTIME_CONTEXT_REPOS+=("$repo_key") + RUNTIME_CONTEXT_FILES+=("$resolved") + RUNTIME_TELEMETRY_FACTS_FILES+=("$telemetry") + fi + ((++i)) + done + + if [[ "$RUNTIME_SELECTION_REQUIRED" == "true" ]]; then + if (( ${#RUNTIME_EXPECTED_TARGETS[@]} == 0 )); then + log "error: runtime selection requires at least one --expected-target" + exit 2 + fi + if (( ${#RUNTIME_CONTEXT_REPOS[@]} == 0 )); then + log "error: runtime selection requires at least one --context-entry" + exit 2 + fi + fi + + local left right tmp + for (( left = 0; left < ${#RUNTIME_CONTEXT_REPOS[@]}; ++left )); do + for (( right = left + 1; right < ${#RUNTIME_CONTEXT_REPOS[@]}; ++right )); do + if [[ "${RUNTIME_CONTEXT_REPOS[$right]}" < "${RUNTIME_CONTEXT_REPOS[$left]}" ]]; then + tmp="${RUNTIME_CONTEXT_REPOS[$left]}" + RUNTIME_CONTEXT_REPOS[$left]="${RUNTIME_CONTEXT_REPOS[$right]}" + RUNTIME_CONTEXT_REPOS[$right]="$tmp" + tmp="${RUNTIME_CONTEXT_FILES[$left]}" + RUNTIME_CONTEXT_FILES[$left]="${RUNTIME_CONTEXT_FILES[$right]}" + RUNTIME_CONTEXT_FILES[$right]="$tmp" + tmp="${RUNTIME_TELEMETRY_FACTS_FILES[$left]}" + RUNTIME_TELEMETRY_FACTS_FILES[$left]="${RUNTIME_TELEMETRY_FACTS_FILES[$right]}" + RUNTIME_TELEMETRY_FACTS_FILES[$right]="$tmp" + fi + done + done +} + +scan_runtime_selection_args "$@" + # Resolve bundled context inputs used for payload enrichment. # Runtime override wins first so callers can reuse an already-fetched context # file without making `bazel run //:dd_upload_payloads` depend on sync labels. @@ -370,7 +561,13 @@ load_context_manifest_entries() { CONTEXT_REPO_COUNT=${#CONTEXT_REPO_KEYS[@]} } -if [[ -n "$CONTEXT_JSON_OVERRIDE" ]]; then +if (( ${#RUNTIME_CONTEXT_REPOS[@]} > 0 )); then + CONTEXT_REPO_KEYS=("${RUNTIME_CONTEXT_REPOS[@]}") + CONTEXT_REPO_FILES=("${RUNTIME_CONTEXT_FILES[@]}") + CONTEXT_REPO_COUNT=${#CONTEXT_REPO_KEYS[@]} + CONTEXT_JSON="${CONTEXT_REPO_FILES[0]}" + dbg "runtime contexts selected for repos: ${CONTEXT_REPO_KEYS[*]}" +elif [[ -n "$CONTEXT_JSON_OVERRIDE" ]]; then CONTEXT_JSON=$(resolve_artifact_path "$CONTEXT_JSON_OVERRIDE") if [[ -n "$CONTEXT_JSON" ]]; then CONTEXT_JSON_FROM_OVERRIDE=1 @@ -379,7 +576,7 @@ if [[ -n "$CONTEXT_JSON_OVERRIDE" ]]; then log "warning: DD_TEST_OPTIMIZATION_CONTEXT_JSON did not resolve to a readable file; falling back to configured data" fi fi -if (( CONTEXT_JSON_FROM_OVERRIDE == 0 )); then +if (( ${#RUNTIME_CONTEXT_REPOS[@]} == 0 && CONTEXT_JSON_FROM_OVERRIDE == 0 )); then CONTEXT_MANIFEST="$(resolve_artifact_path "$CONTEXT_MANIFEST_PATH")" if [[ -n "$CONTEXT_MANIFEST" ]]; then dbg "context manifest resolved via direct path: '$CONTEXT_MANIFEST'" @@ -689,6 +886,8 @@ Options: --dry-run Enrich and validate payloads without uploading or deleting files. --validate-enrichment Require key context and Bazel tags after enrichment, before upload. --expected-enriched-tag TAG Add one required enriched tag; repeatable. Defaults to git and Bazel tags. + --expected-target LABEL Select one exact local target; repeatable. + --context-entry REPO=PATH Select one keyed context.json and telemetry sibling; repeatable. --bep-json PATH BEP JSON file from the matching bazel test invocation; repeatable. --freshness-source SOURCE Cache-safety source: auto, bep, execution_log. Default: auto. --freshness-mode MODE Cache-safety mode: auto, required, optional, or disabled. Default: auto. @@ -729,6 +928,16 @@ while (($# > 0)); do EXPECTED_ENRICHED_TAGS+=("${1#--expected-enriched-tag=}") shift ;; + --expected-target|--context-entry) + if (($# < 2)); then + log "error: $1 requires a value" + exit 2 + fi + shift 2 + ;; + --expected-target=*|--context-entry=*) + shift + ;; --bep-json) if (($# < 2)); then log "error: --bep-json requires a file path" @@ -2986,6 +3195,7 @@ prepare_expected_targets() { fi if [[ -z "$EXPECTED_TARGETS_FILE_PATH" && -z "$EXPECTED_TARGETS_FILE_RLOC" ]]; then + apply_runtime_expected_targets if [[ -s "$EXPECTED_TARGETS_RESOLVED_FILE" ]]; then EXPECTED_TARGETS_CONFIGURED=1 fi @@ -3026,6 +3236,21 @@ prepare_expected_targets() { fi cp "$dynamic_targets" "$EXPECTED_TARGETS_RESOLVED_FILE" EXPECTED_TARGETS_CONFIGURED=1 + apply_runtime_expected_targets +} + +apply_runtime_expected_targets() { + local runtime_targets + (( ${#RUNTIME_EXPECTED_TARGETS[@]} > 0 )) || return 0 + runtime_targets="$TMP_PAYLOAD_DIR/expected_targets_runtime.txt" + printf '%s\n' "${RUNTIME_EXPECTED_TARGETS[@]}" | LC_ALL=C sort >"$runtime_targets" + if [[ -s "$EXPECTED_TARGETS_RESOLVED_FILE" ]] && + ! cmp -s "$EXPECTED_TARGETS_RESOLVED_FILE" "$runtime_targets"; then + log "error: configured and runtime expected targets contain different target sets" + exit 2 + fi + cp "$runtime_targets" "$EXPECTED_TARGETS_RESOLVED_FILE" + EXPECTED_TARGETS_CONFIGURED=1 } filter_bep_rows_to_expected_targets() { @@ -4186,6 +4411,13 @@ resolve_telemetry_facts_sources() { fi fi + for resolved in "${RUNTIME_TELEMETRY_FACTS_FILES[@]}"; do + canonical=$(canonicalize_existing_file "$resolved") + if [[ -n "$canonical" ]]; then + printf '%s\n' "$canonical" >>"$tmp_sources" + fi + done + if [[ ! -s "$tmp_sources" ]]; then rm -f "$tmp_sources" 2>/dev/null || true return 0 diff --git a/tools/core/uploader_powershell_runtime.ps1.tpl b/tools/core/uploader_powershell_runtime.ps1.tpl index 4e70735e..b1198578 100644 --- a/tools/core/uploader_powershell_runtime.ps1.tpl +++ b/tools/core/uploader_powershell_runtime.ps1.tpl @@ -382,6 +382,125 @@ function Log-StartTimeStats([string]$FilePath) { } } +$script:RuntimeSelectionRequired = [System.Convert]::ToBoolean("__DDTPL_RUNTIME_SELECTION__") +$script:RuntimeExpectedTargets = New-Object System.Collections.Generic.List[string] +$script:RuntimeContextEntries = [ordered]@{} +$script:RuntimeTelemetryFacts = New-Object System.Collections.Generic.List[string] + +function Assert-RuntimeExpectedTarget([string]$Label) { + if ( + [string]::IsNullOrWhiteSpace($Label) -or + $Label -notmatch '^//[^:\s]*:[^:\s]+$' -or + $Label.Contains('...') -or + $Label.Contains('*') -or + $Label.Contains('\') + ) { + Log "error: --expected-target must be a fully expanded local //pkg:target label, got '$Label'" + exit 2 + } +} + +function Assert-RuntimeContextPair([string]$RepoKey, [string]$ContextPath, [string]$TelemetryPath) { + try { + $context = Get-Content -LiteralPath $ContextPath -Raw -Encoding UTF8 | ConvertFrom-Json -ErrorAction Stop + $telemetry = Get-Content -LiteralPath $TelemetryPath -Raw -Encoding UTF8 | ConvertFrom-Json -ErrorAction Stop + } catch { + Log "error: runtime context/telemetry must contain valid JSON for repository '$RepoKey'" + exit 2 + } + $contextRepo = [string]$context.PSObject.Properties['topt.sync.repository_name'].Value + $serviceName = [string]$context.PSObject.Properties['service.name'].Value + $runtimeName = [string]$context.PSObject.Properties['runtime.name'].Value + if ( + $contextRepo -cne $RepoKey -or + [string]::IsNullOrWhiteSpace($serviceName) -or + [string]::IsNullOrWhiteSpace($runtimeName) -or + [int]$telemetry.schema_version -ne 1 -or + [string]$telemetry.service_name -cne $serviceName -or + [string]$telemetry.runtime_name -cne $runtimeName -or + $null -eq $telemetry.counts -or + $null -eq $telemetry.distributions + ) { + Log "error: runtime context/telemetry identity or schema mismatch for repository '$RepoKey'" + exit 2 + } +} + +function Initialize-RuntimeSelectionArguments { + $seenTargets = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + $seenPaths = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + for ($index = 0; $index -lt $args.Count; $index++) { + $argument = [string]$args[$index] + $value = $null + $isTarget = $false + if ($argument -eq '--expected-target') { + if ($index + 1 -ge $args.Count) { Log 'error: --expected-target requires a local label'; exit 2 } + $index++ + $value = [string]$args[$index] + $isTarget = $true + } elseif ($argument.StartsWith('--expected-target=')) { + $value = $argument.Substring('--expected-target='.Length) + $isTarget = $true + } elseif ($argument -eq '--context-entry') { + if ($index + 1 -ge $args.Count) { Log 'error: --context-entry requires ='; exit 2 } + $index++ + $value = [string]$args[$index] + } elseif ($argument.StartsWith('--context-entry=')) { + $value = $argument.Substring('--context-entry='.Length) + } else { + continue + } + + if ($isTarget) { + Assert-RuntimeExpectedTarget $value + if (-not $seenTargets.Add($value)) { Log "error: duplicate --expected-target '$value'"; exit 2 } + $script:RuntimeExpectedTargets.Add($value) | Out-Null + continue + } + + $separator = $value.IndexOf('=') + if ($separator -le 0 -or $separator + 1 -ge $value.Length) { + Log 'error: --context-entry must use =' + exit 2 + } + $repoKey = $value.Substring(0, $separator) + $rawPath = $value.Substring($separator + 1) + if ($repoKey -notmatch '^[A-Za-z0-9._+-]+$') { + Log 'error: --context-entry must use =' + exit 2 + } + $resolved = Resolve-RuntimeFilePath $rawPath + if (-not $resolved -or -not (Test-Path -LiteralPath $resolved -PathType Leaf) -or [System.IO.Path]::GetFileName($resolved) -cne 'context.json') { + Log "error: context entry for '$repoKey' must reference an existing regular context.json file" + exit 2 + } + $resolved = [System.IO.Path]::GetFullPath($resolved) + $telemetry = Join-Path (Split-Path -Parent $resolved) 'telemetry_facts.json' + if (-not (Test-Path -LiteralPath $telemetry -PathType Leaf)) { + Log "error: context entry for '$repoKey' is missing sibling telemetry_facts.json" + exit 2 + } + if ($script:RuntimeContextEntries.Contains($repoKey)) { Log "error: duplicate --context-entry repository '$repoKey'"; exit 2 } + if (-not $seenPaths.Add($resolved)) { Log "error: duplicate --context-entry path '$resolved'"; exit 2 } + Assert-RuntimeContextPair $repoKey $resolved $telemetry + $script:RuntimeContextEntries[$repoKey] = $resolved + $script:RuntimeTelemetryFacts.Add([System.IO.Path]::GetFullPath($telemetry)) | Out-Null + } + + if ($script:RuntimeSelectionRequired) { + if ($script:RuntimeExpectedTargets.Count -eq 0) { Log 'error: runtime selection requires at least one --expected-target'; exit 2 } + if ($script:RuntimeContextEntries.Count -eq 0) { Log 'error: runtime selection requires at least one --context-entry'; exit 2 } + } + + $sortedEntries = [ordered]@{} + foreach ($repoKey in @($script:RuntimeContextEntries.Keys | Sort-Object -CaseSensitive)) { + $sortedEntries[$repoKey] = $script:RuntimeContextEntries[$repoKey] + } + $script:RuntimeContextEntries = $sortedEntries +} + +Initialize-RuntimeSelectionArguments @args + # Resolve context.json path (used by upload functions for payload enrichment). # Runtime override wins first so callers can reuse an already-fetched context # file without reintroducing sync repo dependencies at uploader run time. @@ -449,7 +568,11 @@ function Load-ContextManifestEntries { $script:BundledContextEntries[$repoKey] = $resolved } } -if ($ContextJsonOverride) { +if ($script:RuntimeContextEntries.Count -gt 0) { + $script:BundledContextEntries = $script:RuntimeContextEntries + $script:PrimaryContextJson = @($script:BundledContextEntries.Values)[0] + Dbg "runtime contexts selected for repos: $([string]::Join(', ', @($script:BundledContextEntries.Keys)))" +} elseif ($ContextJsonOverride) { $script:PrimaryContextJson = Resolve-ArtifactPath $ContextJsonOverride if ($script:PrimaryContextJson) { $contextJsonFromOverride = $true @@ -736,6 +859,8 @@ function Show-Usage { Write-Host " --dry-run Enrich and validate payloads without uploading or deleting files." Write-Host " --validate-enrichment Require key context and Bazel tags after enrichment, before upload." Write-Host " --expected-enriched-tag TAG Add one required enriched tag; repeatable. Defaults to git and Bazel tags." + Write-Host " --expected-target LABEL Select one exact local target; repeatable." + Write-Host " --context-entry REPO=PATH Select one keyed context.json and telemetry sibling; repeatable." Write-Host " --bep-json PATH BEP JSON file from the matching bazel test invocation; repeatable." Write-Host " --freshness-source SOURCE Cache-safety source: auto, bep, execution_log. Default: auto." Write-Host " --freshness-mode MODE Cache-safety mode: auto, required, optional, or disabled. Default: auto." @@ -776,6 +901,17 @@ for ($i = 0; $i -lt $args.Count; $i++) { $ExpectedEnrichedTags.Add($arg.Substring("--expected-enriched-tag=".Length)) | Out-Null continue } + if ($arg -eq "--expected-target" -or $arg -eq "--context-entry") { + if ($i + 1 -ge $args.Count) { + Log "error: $arg requires a value" + exit 2 + } + $i++ + continue + } + if ($arg.StartsWith("--expected-target=") -or $arg.StartsWith("--context-entry=")) { + continue + } if ($arg -eq "--bep-json") { if ($i + 1 -ge $args.Count) { Log "error: --bep-json requires a file path" @@ -2852,6 +2988,30 @@ function Get-BepFileReferenceCandidates($FileObject) { return $values } +function Set-ExpectedTargetsWithRuntime($ConfiguredTargets, [bool]$HasConfiguredSource) { + $configured = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + foreach ($label in @($ConfiguredTargets)) { + if (-not [string]::IsNullOrWhiteSpace([string]$label)) { $configured.Add([string]$label) | Out-Null } + } + if ($script:RuntimeExpectedTargets.Count -gt 0) { + $runtime = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + foreach ($label in $script:RuntimeExpectedTargets) { $runtime.Add($label) | Out-Null } + if ( + $configured.Count -gt 0 -and + ($configured.Count -ne $runtime.Count -or @($configured | Where-Object { -not $runtime.Contains($_) }).Count -gt 0) + ) { + Log "error: configured and runtime expected targets contain different target sets" + exit 2 + } + $configured = $runtime + } + $script:ExpectedTargets.Clear() + foreach ($label in @($configured | Sort-Object -CaseSensitive)) { + $script:ExpectedTargets.Add($label) | Out-Null + } + $script:ExpectedTargetsConfigured = $HasConfiguredSource -or $script:RuntimeExpectedTargets.Count -gt 0 +} + function Initialize-ExpectedTargets { $script:ExpectedTargets.Clear() $script:ExpectedTargetsConfigured = $false @@ -2873,8 +3033,7 @@ function Initialize-ExpectedTargets { -not [string]::IsNullOrWhiteSpace($script:ExpectedTargetsFileRloc) ) if (-not $dynamicConfigured) { - foreach ($label in $staticTargets) { $script:ExpectedTargets.Add($label) | Out-Null } - $script:ExpectedTargetsConfigured = $staticTargets.Count -gt 0 + Set-ExpectedTargetsWithRuntime $staticTargets ($staticTargets.Count -gt 0) return } @@ -2931,8 +3090,7 @@ function Initialize-ExpectedTargets { exit 2 } } - foreach ($label in $dynamicTargets) { $script:ExpectedTargets.Add($label) | Out-Null } - $script:ExpectedTargetsConfigured = $true + Set-ExpectedTargetsWithRuntime $dynamicTargets $true } function Initialize-BepEligibility { @@ -4116,6 +4274,13 @@ function Resolve-TelemetryFactsSources { } } + foreach ($runtimeFacts in $script:RuntimeTelemetryFacts) { + $canonical = Resolve-CanonicalExistingFile $runtimeFacts + if ($canonical -and $seen.Add($canonical)) { + $sources += ,$canonical + } + } + return (Get-OrdinalSortedStrings $sources) } diff --git a/tools/tests/core/BUILD.bazel b/tools/tests/core/BUILD.bazel index a010ef00..e8b98a49 100644 --- a/tools/tests/core/BUILD.bazel +++ b/tools/tests/core/BUILD.bazel @@ -112,6 +112,7 @@ load( "disabled_context_contract_test", "disabled_export_and_build_shape_test", "disabled_runtime_module_path_precedence_test", + "repository_state_and_runtime_module_rendering_test", "disabled_telemetry_contract_test", "enabled_false_values_test", "enabled_truthy_values_test", @@ -221,6 +222,9 @@ load( "expected_targets_file_test", "expected_targets_test", "explicit_context_data_test", + "runtime_selection_mixed_source_failure_test", + "runtime_selection_mixed_source_target_rule", + "runtime_selection_test", "uploader_controlled_attr_failure_test", "uploader_controlled_attr_target_rule", "uploader_kwargs_test", @@ -486,6 +490,11 @@ uploader_controlled_attr_target_rule( tags = ["manual"], ) +runtime_selection_mixed_source_target_rule( + name = "runtime_selection_mixed_source_target", + tags = ["manual"], +) + # Starlark unit tests for core helpers (sync, uploader, common utils). dd_site_normalization_test( name = "dd_site_normalization_test", @@ -752,6 +761,12 @@ explicit_context_data_test( timeout = "short", ) +runtime_selection_test( + name = "runtime_selection_test", + size = "small", + timeout = "short", +) + explicit_payload_labels_precede_static_fallback_test( name = "explicit_payload_labels_precede_static_fallback_test", size = "small", @@ -802,6 +817,11 @@ uploader_controlled_attr_failure_test( target_under_test = ":uploader_controlled_attr_target", ) +runtime_selection_mixed_source_failure_test( + name = "runtime_selection_mixed_source_failure_test", + target_under_test = ":runtime_selection_mixed_source_target", +) + legacy_context_direct_file_fallback_test( name = "legacy_context_direct_file_fallback_test", target_under_test = ":legacy_context_direct_file_target", @@ -878,6 +898,12 @@ disabled_export_and_build_shape_test( timeout = "short", ) +repository_state_and_runtime_module_rendering_test( + name = "repository_state_and_runtime_module_rendering_test", + size = "small", + timeout = "short", +) + normalize_ref_test( name = "normalize_ref_test", size = "small", @@ -1848,6 +1874,7 @@ test_suite( ":render_template_substitution_test", ":render_template_unescape_only_test", ":repository_environment_contains_bootstrap_inputs_test", + ":repository_state_and_runtime_module_rendering_test", ":required_git_metadata_missing_fields_test", ":resolve_dd_api_base_test", ":runfile_manifest_bash_resolution_test", @@ -1855,6 +1882,8 @@ test_suite( ":runfile_manifest_parser_parity_test", ":runfile_manifest_powershell_resolution_test", ":runtime_module_path_from_environ_test", + ":runtime_selection_mixed_source_failure_test", + ":runtime_selection_test", ":sanitize_label_fragment_test", ":sanitize_repository_url_test", ":service_environment_resolution_test", diff --git a/tools/tests/core/test_sync_disabled_utils.bzl b/tools/tests/core/test_sync_disabled_utils.bzl index ea0d132f..03a69d7e 100644 --- a/tools/tests/core/test_sync_disabled_utils.bzl +++ b/tools/tests/core/test_sync_disabled_utils.bzl @@ -20,6 +20,8 @@ load( "render_disabled_settings_json_for_tests", "render_disabled_telemetry_facts_json_for_tests", "render_disabled_test_management_json_for_tests", + "render_repository_state_target_for_tests", + "render_runtime_module_target_for_tests", "repository_environ_for_tests", "resolve_service_and_environment_for_tests", ) @@ -229,6 +231,8 @@ def _disabled_export_and_build_shape_test(ctx): for fragment in [ 'name = "test_optimization_files"', 'name = "test_optimization_context"', + 'name = "test_optimization_repository_state"', + 'name = "test_optimization_runtime_module"', "custom_topt/cache/http/settings.json", "custom_topt/cache/http/known_tests.json", "custom_topt/cache/http/test_management.json", @@ -238,6 +242,34 @@ def _disabled_export_and_build_shape_test(ctx): asserts.true(env, fragment in build, "missing BUILD fragment %s" % fragment) return unittest.end(env) +def _repository_state_and_runtime_module_rendering_test(ctx): + """Validate stable static-consumer labels represent module hits and misses.""" + env = unittest.begin(ctx) + state = render_repository_state_target_for_tests( + enabled = True, + disabled_reason = "", + repo_name = "test_optimization_data_service", + service_name = "service-name", + runtime_name = "go", + runtime_module_path = "example.com/repo", + runtime_module_included = True, + ) + for fragment in [ + 'name = "test_optimization_repository_state"', + 'repo_name = "test_optimization_data_service"', + 'service_name = "service-name"', + 'runtime_name = "go"', + 'runtime_module_path = "example.com/repo"', + "runtime_module_included = True", + ]: + asserts.true(env, fragment in state, "missing state fragment %s" % fragment) + + module_hit = render_runtime_module_target_for_tests("example_com_repo") + asserts.true(env, 'srcs = [":module_example_com_repo"]' in module_hit) + module_miss = render_runtime_module_target_for_tests() + asserts.true(env, "srcs = []" in module_miss) + return unittest.end(env) + enabled_truthy_values_test = unittest.make(_enabled_truthy_values_test) enabled_false_values_test = unittest.make(_enabled_false_values_test) repository_environment_contains_bootstrap_inputs_test = unittest.make(_repository_environment_contains_bootstrap_inputs_test) @@ -247,3 +279,4 @@ disabled_cache_payloads_contract_test = unittest.make(_disabled_cache_payloads_c disabled_telemetry_contract_test = unittest.make(_disabled_telemetry_contract_test) disabled_export_and_build_shape_test = unittest.make(_disabled_export_and_build_shape_test) disabled_runtime_module_path_precedence_test = unittest.make(_disabled_runtime_module_path_precedence_test) +repository_state_and_runtime_module_rendering_test = unittest.make(_repository_state_and_runtime_module_rendering_test) diff --git a/tools/tests/core/test_test_optimization_targets_utils.bzl b/tools/tests/core/test_test_optimization_targets_utils.bzl index 997c8c3b..b4df4032 100644 --- a/tools/tests/core/test_test_optimization_targets_utils.bzl +++ b/tools/tests/core/test_test_optimization_targets_utils.bzl @@ -51,6 +51,30 @@ def _explicit_context_data_test(ctx): explicit_context_data_test = unittest.make(_explicit_context_data_test) +def _runtime_selection_test(ctx): + """Validate runtime selection owns empty static inputs for both tools.""" + env = unittest.begin(ctx) + specs = build_test_optimization_target_specs_for_tests( + name = "test_optimization", + sync_repo_name = "unused", + doctor_name = "doctor", + uploader_name = "uploader", + expected_targets = [], + context_data = None, + doctor_kwargs = None, + uploader_kwargs = None, + runtime_selection = True, + ) + asserts.equals(env, [], specs.doctor_attrs["data"]) + asserts.equals(env, [], specs.uploader_attrs["data"]) + asserts.equals(env, [], specs.doctor_attrs["expected_targets"]) + asserts.equals(env, [], specs.uploader_attrs["expected_targets"]) + asserts.equals(env, True, specs.doctor_attrs["runtime_selection"]) + asserts.equals(env, True, specs.uploader_attrs["runtime_selection"]) + return unittest.end(env) + +runtime_selection_test = unittest.make(_runtime_selection_test) + def _expected_targets_test(ctx): """Validate strict expected target labels are forwarded to both tools.""" env = unittest.begin(ctx) @@ -200,6 +224,23 @@ def _uploader_controlled_attr_target_impl(_ctx): uploader_controlled_attr_target_rule = rule(implementation = _uploader_controlled_attr_target_impl) +def _runtime_selection_mixed_source_target_impl(_ctx): + """Create a target that combines runtime selection with configured context data.""" + build_test_optimization_target_specs_for_tests( + name = "test_optimization", + sync_repo_name = "test_optimization_data", + doctor_name = "doctor", + uploader_name = "uploader", + expected_targets = [], + context_data = ["@test_optimization_data//:test_optimization_context"], + doctor_kwargs = None, + uploader_kwargs = None, + runtime_selection = True, + ) + return [] + +runtime_selection_mixed_source_target_rule = rule(implementation = _runtime_selection_mixed_source_target_impl) + def _empty_name_failure_test_impl(ctx): """Assert empty helper names fail with an actionable message.""" env = analysistest.begin(ctx) @@ -243,3 +284,14 @@ uploader_controlled_attr_failure_test = analysistest.make( _uploader_controlled_attr_failure_test_impl, expect_failure = True, ) + +def _runtime_selection_mixed_source_failure_test_impl(ctx): + """Assert runtime selection cannot retain a configured context source.""" + env = analysistest.begin(ctx) + asserts.expect_failure(env, "runtime_selection cannot be combined with context_data") + return analysistest.end(env) + +runtime_selection_mixed_source_failure_test = analysistest.make( + _runtime_selection_mixed_source_failure_test_impl, + expect_failure = True, +) diff --git a/tools/tests/example_stub_repo.bzl b/tools/tests/example_stub_repo.bzl index 5f4a9f2e..f5b854b5 100644 --- a/tools/tests/example_stub_repo.bzl +++ b/tools/tests/example_stub_repo.bzl @@ -158,7 +158,13 @@ def _render_stub_build( service_keys = None, module_labels = None, manifest_root = ".testoptimization", - flaky_tests = None): + flaky_tests = None, + repo_name = "test_optimization_data", + service_name = "stub-service", + runtime_module_path = "example.com/stub", + runtime_module_label = "example_com_stub", + runtime_module_included = False, + enabled = True): """Render BUILD content for stub repo targets.""" def _append_filegroups(name_suffix, srcs): @@ -180,7 +186,10 @@ def _render_stub_build( if flaky_tests == None: flaky_tests = "%s/cache/http/flaky_tests.json" % manifest_root files_srcs = [settings, manifest, known_tests, test_management, flaky_tests] - lines = ['load(":module_runfiles.bzl", "topt_module_files")\n\n'] + lines = [ + 'load(":module_runfiles.bzl", "topt_module_files")\n', + 'load("@datadog-rules-test-optimization//tools/core:test_optimization_repository_state.bzl", "test_optimization_repository_state")\n\n', + ] _append_filegroups("", files_srcs) for key in list(service_keys or []): _append_filegroups("_%s" % key, files_srcs) @@ -196,6 +205,29 @@ def _render_stub_build( ' visibility = ["//visibility:public"],\n' + ")\n\n", ) + runtime_label = "" + if runtime_module_included: + runtime_label = runtime_module_label + lines.append( + "filegroup(\n" + + ' name = "test_optimization_runtime_module",\n' + + (" srcs = %s,\n" % repr([":module_%s" % runtime_label] if runtime_label else [])) + + ' visibility = ["//visibility:public"],\n' + + ")\n\n", + ) + lines.append( + "test_optimization_repository_state(\n" + + ' name = "test_optimization_repository_state",\n' + + (" enabled = %s,\n" % ("True" if enabled else "False")) + + (" repo_name = %s,\n" % repr(repo_name)) + + (" service_name = %s,\n" % repr(service_name)) + + ' runtime_name = "go",\n' + + (" runtime_module_path = %s,\n" % repr(runtime_module_path)) + + (" runtime_module_included = %s,\n" % ("True" if runtime_module_included else "False")) + + (" disabled_reason = %s,\n" % repr("" if enabled else "disabled by repository configuration")) + + ' visibility = ["//visibility:public"],\n' + + ")\n\n", + ) lines.append('exports_files(["export.bzl", %s], visibility = ["//visibility:public"])\n' % repr(manifest)) return "".join(lines) @@ -273,6 +305,12 @@ def _example_stub_repo_impl(ctx): module_labels = module_labels, manifest_root = out_dir, flaky_tests = flaky_tests, + repo_name = exported_repo_name, + service_name = ctx.attr.service_name, + runtime_module_path = ctx.attr.go_module_path, + runtime_module_label = ctx.attr.go_sanitized_module_path, + runtime_module_included = ctx.attr.go_module_included, + enabled = ctx.attr.enabled, ) ctx.file("BUILD", build) diff --git a/tools/tests/python/test_python_tools.py b/tools/tests/python/test_python_tools.py index 99eb53df..fa3509b1 100644 --- a/tools/tests/python/test_python_tools.py +++ b/tools/tests/python/test_python_tools.py @@ -218,6 +218,7 @@ def _render_uploader_runtime_template( expected_targets_path: str = "", expected_targets_file_path: str = "", fail_on_error: bool = False, + runtime_selection: bool = False, curl_retry_flags: str = "--retry 3 --retry-delay 2 --retry-connrefused", ) -> str: """Render uploader runtime template placeholders for direct unit tests.""" @@ -243,6 +244,7 @@ def _render_uploader_runtime_template( "ps_name": "generated_uploader.ps1", "quiescent_sec": "10", "rules_version": "test-rules-version", + "runtime_selection": "true" if runtime_selection else "false", "schema_json_path": "", "schema_json_rloc": "", "schema_validator_path": "", @@ -1737,6 +1739,83 @@ def test_resolve_expected_targets_rejects_static_file_mismatch(self) -> None: self.mod._resolve_expected_targets(config, config_path) self.assertIn("different target sets", stderr.getvalue()) + def test_runtime_expected_targets_are_sorted_and_must_match_configured_inputs(self) -> None: + """Validate runtime target selection is exact, deterministic, and fail closed.""" + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + config_path = self._write_doctor_config(root, ["//pkg:a_test", "//pkg:b_test"]) + config = json.loads(config_path.read_text(encoding="utf-8")) + + targets, source = self.mod._resolve_expected_targets( + config, + config_path, + ["//pkg:b_test", "//pkg:a_test"], + True, + ) + self.assertEqual(["//pkg:a_test", "//pkg:b_test"], targets) + self.assertEqual("runtime_and_configured", source) + + stderr = io.StringIO() + with self.assertRaises(SystemExit), mock.patch("sys.stderr", stderr): + self.mod._resolve_expected_targets( + config, + config_path, + ["//pkg:a_test", "//pkg:a_test"], + True, + ) + self.assertIn("duplicate target labels", stderr.getvalue()) + + stderr = io.StringIO() + with self.assertRaises(SystemExit), mock.patch("sys.stderr", stderr): + self.mod._resolve_expected_targets(config, config_path, [], True) + self.assertIn("requires at least one --expected-target", stderr.getvalue()) + + def test_runtime_contexts_validate_and_sort_keyed_context_telemetry_pairs(self) -> None: + """Validate runtime contexts preserve exact repo, service, and runtime identity.""" + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + entries = [] + for repo_name, service_name in (("repo-b", "service-b"), ("repo-a", "service-a")): + repo_dir = root / repo_name + repo_dir.mkdir() + context_path = repo_dir / "context.json" + context_path.write_text( + json.dumps({ + "topt.sync.repository_name": repo_name, + "service.name": service_name, + "runtime.name": "go", + }), + encoding="utf-8", + ) + (repo_dir / "telemetry_facts.json").write_text( + json.dumps({ + "schema_version": 1, + "service_name": service_name, + "runtime_name": "go", + "counts": [], + "distributions": [], + }), + encoding="utf-8", + ) + entries.append(f"{repo_name}={context_path}") + + contexts = self.mod._load_runtime_contexts(entries) + self.assertEqual(["repo-a", "repo-b"], [repo for repo, _ in contexts]) + + stderr = io.StringIO() + with self.assertRaises(SystemExit), mock.patch("sys.stderr", stderr): + self.mod._load_runtime_contexts([entries[0], entries[0]]) + self.assertIn("duplicate --context-entry repository", stderr.getvalue()) + + context_path = contexts[0][1] + context_doc = json.loads(context_path.read_text(encoding="utf-8")) + context_doc["topt.sync.repository_name"] = "other-repo" + context_path.write_text(json.dumps(context_doc), encoding="utf-8") + stderr = io.StringIO() + with self.assertRaises(SystemExit), mock.patch("sys.stderr", stderr): + self.mod._load_runtime_contexts([f"repo-a={context_path}"]) + self.assertIn("repository mismatch", stderr.getvalue()) + def test_expected_targets_file_rejects_duplicates_and_invalid_labels(self) -> None: """Validate generated target input cannot weaken local exactness.""" with tempfile.TemporaryDirectory() as tmp: @@ -7100,6 +7179,109 @@ def test_generated_uploaders_accept_empty_expected_target_set(self) -> None: self.assertEqual(0, report_doc["bep"]["eligible_outputs"]) self.assertEqual(0, report_doc["payloads"]["tests"]["processed"]) + def test_generated_uploaders_validate_runtime_selection_before_discovery(self) -> None: + """Validate Bash and PowerShell accept only complete keyed runtime selections.""" + bash = _require_functional_bash(self) + if os.name == "nt": + self.skipTest("generated PowerShell uploader execution smoke is covered on non-Windows") + pwsh = _require_command(self, "pwsh", "pwsh is required for generated PowerShell execution") + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + context_args = [] + context_paths = [] + for repo_name, service_name in (("repo-b", "service-b"), ("repo-a", "service-a")): + repo_dir = root / repo_name + repo_dir.mkdir() + context_path = repo_dir / "context.json" + context_path.write_text( + json.dumps({ + "topt.sync.repository_name": repo_name, + "service.name": service_name, + "runtime.name": "go", + }), + encoding="utf-8", + ) + (repo_dir / "telemetry_facts.json").write_text( + json.dumps({ + "schema_version": 1, + "service_name": service_name, + "runtime_name": "go", + "counts": [], + "distributions": [], + }), + encoding="utf-8", + ) + context_args.extend(["--context-entry", f"{repo_name}={context_path}"]) + context_paths.append(context_path) + + bash_script = root / "generated_uploader.sh" + bash_script.write_text( + _render_uploader_runtime_template( + "tools/core/uploader_bash_runtime.sh.tpl", + runtime_selection=True, + ), + encoding="utf-8", + ) + bash_script.chmod(0o755) + powershell_script = root / "generated_uploader.ps1" + powershell_script.write_text( + _render_uploader_runtime_template( + "tools/core/uploader_powershell_runtime.ps1.tpl", + runtime_selection=True, + ), + encoding="utf-8", + ) + + for runtime, command in ( + ("Bash", [bash, str(bash_script)]), + ("PowerShell", [pwsh, "-NoLogo", "-NoProfile", "-File", str(powershell_script)]), + ): + with self.subTest(runtime=runtime, case="missing"): + result = subprocess.run( + [*command, "--help"], + cwd=root, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=30, + check=False, + ) + self.assertEqual(2, result.returncode, result.stdout + result.stderr) + self.assertIn("requires at least one --expected-target", result.stdout + result.stderr) + + with self.subTest(runtime=runtime, case="valid-multi-service"): + result = subprocess.run( + [*command, "--expected-target", "//pkg:test.topt", *context_args, "--help"], + cwd=root, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=30, + check=False, + ) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + with self.subTest(runtime=runtime, case="identity-mismatch"): + result = subprocess.run( + [ + *command, + "--expected-target", + "//pkg:test.topt", + "--context-entry", + f"wrong-repo={context_paths[0]}", + "--help", + ], + cwd=root, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=30, + check=False, + ) + self.assertEqual(2, result.returncode, result.stdout + result.stderr) + self.assertIn("identity or schema mismatch", result.stdout + result.stderr) + def _assert_uploader_report_failure(self, report_path: Path, bep_path: Path) -> None: """Validate a failed generated uploader machine-readable report.""" report = json.loads(report_path.read_text(encoding="utf-8")) From 241bbee9fa938b41f754fb4e5e370abad2fc6b3c Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Fri, 28 Aug 2026 12:36:51 +0200 Subject: [PATCH 34/37] uploader: support runtime selection on Bash 3 --- tools/core/uploader_bash_runtime.sh.tpl | 8 ++++---- tools/tests/python/test_python_tools.py | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/tools/core/uploader_bash_runtime.sh.tpl b/tools/core/uploader_bash_runtime.sh.tpl index e5865c12..aa8c0df0 100644 --- a/tools/core/uploader_bash_runtime.sh.tpl +++ b/tools/core/uploader_bash_runtime.sh.tpl @@ -421,7 +421,7 @@ scan_runtime_selection_args() { if [[ "$arg" == --expected-target* ]]; then validate_runtime_expected_target "$value" - for existing in "${RUNTIME_EXPECTED_TARGETS[@]}"; do + for existing in "${RUNTIME_EXPECTED_TARGETS[@]+"${RUNTIME_EXPECTED_TARGETS[@]}"}"; do if [[ "$existing" == "$value" ]]; then log "error: duplicate --expected-target '$value'" exit 2 @@ -445,13 +445,13 @@ scan_runtime_selection_args() { log "error: context entry for '$repo_key' is missing sibling telemetry_facts.json" exit 2 fi - for existing in "${RUNTIME_CONTEXT_REPOS[@]}"; do + for existing in "${RUNTIME_CONTEXT_REPOS[@]+"${RUNTIME_CONTEXT_REPOS[@]}"}"; do if [[ "$existing" == "$repo_key" ]]; then log "error: duplicate --context-entry repository '$repo_key'" exit 2 fi done - for existing in "${RUNTIME_CONTEXT_FILES[@]}"; do + for existing in "${RUNTIME_CONTEXT_FILES[@]+"${RUNTIME_CONTEXT_FILES[@]}"}"; do if [[ "$existing" == "$resolved" ]]; then log "error: duplicate --context-entry path '$resolved'" exit 2 @@ -4411,7 +4411,7 @@ resolve_telemetry_facts_sources() { fi fi - for resolved in "${RUNTIME_TELEMETRY_FACTS_FILES[@]}"; do + for resolved in "${RUNTIME_TELEMETRY_FACTS_FILES[@]+"${RUNTIME_TELEMETRY_FACTS_FILES[@]}"}"; do canonical=$(canonicalize_existing_file "$resolved") if [[ -n "$canonical" ]]; then printf '%s\n' "$canonical" >>"$tmp_sources" diff --git a/tools/tests/python/test_python_tools.py b/tools/tests/python/test_python_tools.py index fa3509b1..ad9797c2 100644 --- a/tools/tests/python/test_python_tools.py +++ b/tools/tests/python/test_python_tools.py @@ -105,7 +105,8 @@ def _load_module(name: str, rel_path: str) -> types.ModuleType: def _require_functional_bash(testcase: unittest.TestCase) -> str: """Return a usable Bash executable or skip the caller's test.""" - bash = shutil.which("bash") + system_bash = Path("/bin/bash") + bash = str(system_bash) if system_bash.is_file() else shutil.which("bash") if bash is None: testcase.skipTest("bash is required for Bash runtime execution") result = subprocess.run( From cd2a14b6f002f062cd128a701e22521163c5090e Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Fri, 28 Aug 2026 13:13:44 +0200 Subject: [PATCH 35/37] tests: render runtime selection in PowerShell harness --- tools/tests/integration/run_mock_server_tests.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/tests/integration/run_mock_server_tests.ps1 b/tools/tests/integration/run_mock_server_tests.ps1 index c682fe1b..efac4c9f 100644 --- a/tools/tests/integration/run_mock_server_tests.ps1 +++ b/tools/tests/integration/run_mock_server_tests.ps1 @@ -234,6 +234,7 @@ function Render-UploaderTemplate { "__DDTPL_SCHEMA_VALIDATOR_PATH__" = "" "__DDTPL_BEP_ARTIFACT_STAGE_HELPER_RLOC__" = "tools/core/bep_artifact_stage_helper.py" "__DDTPL_DOCTOR_RUNTIME_RLOC__" = "tools/core/test_optimization_doctor.py" + "__DDTPL_RUNTIME_SELECTION__" = "false" "__DDTPL_RULES_VERSION__" = "integration-test" } foreach ($entry in $replacements.GetEnumerator()) { From a366f97142607c974d4cb848128614cc6d0f3378 Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Fri, 28 Aug 2026 17:02:36 +0200 Subject: [PATCH 36/37] doctor: ignore ordinary mixed-run artifacts --- tools/core/test_optimization_doctor.py | 11 +++ tools/tests/python/test_python_tools.py | 89 +++++++++++++++++++++++++ 2 files changed, 100 insertions(+) diff --git a/tools/core/test_optimization_doctor.py b/tools/core/test_optimization_doctor.py index 85231b74..4ffad40e 100644 --- a/tools/core/test_optimization_doctor.py +++ b/tools/core/test_optimization_doctor.py @@ -1261,6 +1261,7 @@ def _stage_bep_artifacts( workspace: Path, staging_dir: Path, remote_artifacts: str, + selected_labels: set[str] | None = None, downloader: str = "", downloader_timeout_sec: float = 300.0, ) -> list[StagedBepArtifact]: @@ -1287,6 +1288,8 @@ def warn_once(message: str) -> None: _fail(message) warn_once(message) for ref in refs: + if selected_labels is not None and ref.label not in selected_labels: + continue if ref.cached: continue display_fetch_value = _display_artifact_reference(ref.fetch_value) @@ -2884,12 +2887,19 @@ def _run_doctor(args: argparse.Namespace, report: dict[str, Any]) -> int: staged=staged, ) if staging_enabled and freshness is not None: + selected_labels = set(expected_targets) if expected_targets else None selected_bep_artifact_outputs = _selected_bep_artifact_outputs( freshness, workspace, args.remote_artifacts, ) + if selected_labels is not None: + selected_bep_artifact_outputs = { + output for output in selected_bep_artifact_outputs if output[0] in selected_labels + } blocked_bep_artifact_labels = _blocked_bep_artifact_labels(freshness, args.remote_artifacts) + if selected_labels is not None: + blocked_bep_artifact_labels.intersection_update(selected_labels) _update_diagnostic_bep( report, freshness, @@ -2902,6 +2912,7 @@ def _run_doctor(args: argparse.Namespace, report: dict[str, Any]) -> int: workspace=workspace, staging_dir=staging_base, remote_artifacts=args.remote_artifacts, + selected_labels=selected_labels, downloader=args.bep_artifact_downloader, downloader_timeout_sec=args.bep_artifact_downloader_timeout_sec, ) diff --git a/tools/tests/python/test_python_tools.py b/tools/tests/python/test_python_tools.py index ad9797c2..e593249c 100644 --- a/tools/tests/python/test_python_tools.py +++ b/tools/tests/python/test_python_tools.py @@ -2980,6 +2980,95 @@ def test_doctor_expected_target_validation_uses_staged_target_mapping(self) -> N self.assertEqual(0, rc) + def test_doctor_expected_target_ignores_mixed_ordinary_bep_artifact(self) -> None: + """Validate mixed ordinary outputs are not staged for expected-target validation.""" + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + optimized = self._write_doctor_output( + root / "optimized-artifact", + "module", + "//pkg:target.topt", + ) + ordinary = root / "ordinary-artifact" / "test.outputs" + ordinary.mkdir(parents=True) + (ordinary / "passthrough.txt").write_text("ordinary\n", encoding="utf-8") + config_path = self._write_doctor_config(root, ["//pkg:target.topt"]) + bep = root / "freshness.bep.json" + self._write_bep( + bep, + [ + { + "id": { + "testResult": { + "label": "//pkg:target.topt", + "run": 1, + "shard": 1, + "attempt": 1, + }, + }, + "testResult": { + "status": "PASSED", + "testActionOutput": [ + { + "name": "test.outputs", + "uri": optimized.as_uri(), + "pathPrefix": [ + "bazel-out", + "k8-fastbuild", + "testlogs", + "pkg", + "target.topt", + ], + }, + ], + }, + }, + { + "id": { + "testResult": { + "label": "//tools:ordinary", + "run": 1, + "shard": 1, + "attempt": 1, + }, + }, + "testResult": { + "status": "PASSED", + "testActionOutput": [ + { + "name": "test.outputs", + "uri": ordinary.as_uri(), + "pathPrefix": [ + "bazel-out", + "k8-fastbuild", + "testlogs", + "tools", + "ordinary", + ], + }, + ], + }, + }, + ], + ) + + with mock.patch.dict(os.environ, {"BUILD_WORKSPACE_DIRECTORY": str(root)}, clear=False): + os.environ.pop("TESTLOGS_DIR", None) + rc = self.mod.main([ + "--config", + str(config_path), + "--bep-json", + str(bep), + "--freshness-source=bep", + "--freshness-mode=required", + "--artifact-source=bep", + "--remote-artifacts=download", + "--artifact-staging-dir", + str(root / ".topt" / "bep-artifacts"), + ]) + + self.assertEqual(0, rc) + def test_doctor_required_remote_disabled_fails_remote_only_freshness(self) -> None: """Validate disabling downloads does not authorize stale local fallback in required mode.""" with tempfile.TemporaryDirectory() as tmp: From 7ce7a9ed103824149ff38edfe0ac71003846d55b Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Fri, 28 Aug 2026 22:24:20 +0200 Subject: [PATCH 37/37] uploader: redact runtime context paths --- tools/core/uploader_bash_runtime.sh.tpl | 34 ++++++++++++------- .../core/uploader_powershell_runtime.ps1.tpl | 26 +++++++------- tools/tests/python/test_python_tools.py | 30 +++++++++++++++- 3 files changed, 63 insertions(+), 27 deletions(-) diff --git a/tools/core/uploader_bash_runtime.sh.tpl b/tools/core/uploader_bash_runtime.sh.tpl index aa8c0df0..646aad92 100644 --- a/tools/core/uploader_bash_runtime.sh.tpl +++ b/tools/core/uploader_bash_runtime.sh.tpl @@ -453,7 +453,7 @@ scan_runtime_selection_args() { done for existing in "${RUNTIME_CONTEXT_FILES[@]+"${RUNTIME_CONTEXT_FILES[@]}"}"; do if [[ "$existing" == "$resolved" ]]; then - log "error: duplicate --context-entry path '$resolved'" + log "error: duplicate --context-entry path for repository '$repo_key'" exit 2 fi done @@ -506,7 +506,7 @@ CONTEXT_JSON_PATH="__DDTPL_CONTEXT_JSON_PATH__" TELEMETRY_FACTS_MANIFEST_RLOC="__DDTPL_TELEMETRY_FACTS_MANIFEST_RLOC__" TELEMETRY_FACTS_MANIFEST_PATH="__DDTPL_TELEMETRY_FACTS_MANIFEST_PATH__" CONTEXT_JSON_OVERRIDE="${DD_TEST_OPTIMIZATION_CONTEXT_JSON:-}" -dbg "context.json resolution inputs: override='$CONTEXT_JSON_OVERRIDE' path='$CONTEXT_JSON_PATH' rloc='$CONTEXT_JSON_RLOC' manifest_path='$CONTEXT_MANIFEST_PATH' manifest_rloc='$CONTEXT_MANIFEST_RLOC'" +dbg "resolving configured context.json and manifest inputs" CONTEXT_JSON="" CONTEXT_JSON_FROM_OVERRIDE=0 CONTEXT_MANIFEST="" @@ -571,7 +571,7 @@ elif [[ -n "$CONTEXT_JSON_OVERRIDE" ]]; then CONTEXT_JSON=$(resolve_artifact_path "$CONTEXT_JSON_OVERRIDE") if [[ -n "$CONTEXT_JSON" ]]; then CONTEXT_JSON_FROM_OVERRIDE=1 - dbg "context.json resolved via runtime override: '$CONTEXT_JSON'" + dbg "context.json resolved via runtime override" else log "warning: DD_TEST_OPTIMIZATION_CONTEXT_JSON did not resolve to a readable file; falling back to configured data" fi @@ -579,20 +579,20 @@ fi if (( ${#RUNTIME_CONTEXT_REPOS[@]} == 0 && CONTEXT_JSON_FROM_OVERRIDE == 0 )); then CONTEXT_MANIFEST="$(resolve_artifact_path "$CONTEXT_MANIFEST_PATH")" if [[ -n "$CONTEXT_MANIFEST" ]]; then - dbg "context manifest resolved via direct path: '$CONTEXT_MANIFEST'" + dbg "context manifest resolved via direct path" elif [[ -n "$CONTEXT_MANIFEST_RLOC" ]]; then CONTEXT_MANIFEST="$(resolve_runfile "$CONTEXT_MANIFEST_RLOC")" if [[ -n "$CONTEXT_MANIFEST" ]]; then - dbg "context manifest resolved via runfiles: '$CONTEXT_MANIFEST'" + dbg "context manifest resolved via runfiles" fi fi load_context_manifest_entries "$CONTEXT_MANIFEST" if (( CONTEXT_REPO_COUNT > 0 )); then CONTEXT_JSON="${CONTEXT_REPO_FILES[0]}" if (( CONTEXT_REPO_COUNT == 1 )); then - dbg "context.json resolved from single bundled context: '$CONTEXT_JSON'" + dbg "context.json resolved from single bundled context" else - dbg "primary context.json resolved from bundled manifest: '$CONTEXT_JSON' (repos=${CONTEXT_REPO_KEYS[*]})" + dbg "primary context.json resolved from bundled manifest (repos=${CONTEXT_REPO_KEYS[*]})" fi fi fi @@ -600,21 +600,25 @@ if [[ -z "$CONTEXT_JSON" ]]; then CONTEXT_JSON=$(resolve_artifact_path "$CONTEXT_JSON_PATH") if [[ -n "$CONTEXT_JSON" ]]; then # Direct artifact path is fastest and most deterministic when available. - dbg "context.json resolved via direct path: '$CONTEXT_JSON'" + dbg "context.json resolved via direct path" elif [[ -n "$CONTEXT_JSON_RLOC" ]]; then # Runfiles lookup supports launcher/platform variants and bzlmod naming. CONTEXT_JSON=$(resolve_runfile "$CONTEXT_JSON_RLOC") if [[ -z "$CONTEXT_JSON" ]]; then log "warning: context.json not found in runfiles; payloads will not be enriched" else - dbg "context.json resolved via runfiles: '$CONTEXT_JSON'" + dbg "context.json resolved via runfiles" fi else dbg "context.json not configured in data files; enrichment disabled" fi fi PRIMARY_CONTEXT_JSON="$CONTEXT_JSON" -dbg "primary context.json: ${PRIMARY_CONTEXT_JSON:-} (bundled_contexts=$CONTEXT_REPO_COUNT)" +if [[ -n "$PRIMARY_CONTEXT_JSON" ]]; then + dbg "primary context.json available (bundled_contexts=$CONTEXT_REPO_COUNT)" +else + dbg "primary context.json unavailable (bundled_contexts=$CONTEXT_REPO_COUNT)" +fi dbg "telemetry facts manifest resolution inputs: path='$TELEMETRY_FACTS_MANIFEST_PATH' rloc='$TELEMETRY_FACTS_MANIFEST_RLOC'" TELEMETRY_FACTS_MANIFEST=$(resolve_artifact_path "$TELEMETRY_FACTS_MANIFEST_PATH") @@ -2177,7 +2181,11 @@ dbg_headers() { JQ_AVAILABLE=0 if command -v jq >/dev/null 2>&1; then JQ_AVAILABLE=1; fi dbg "jq available: $JQ_AVAILABLE" -dbg "primary context.json: ${PRIMARY_CONTEXT_JSON:-}" +if [[ -n "$PRIMARY_CONTEXT_JSON" ]]; then + dbg "primary context.json available" +else + dbg "primary context.json unavailable" +fi # CODEOWNERS state (initialized lazily on first enrichment attempt). CODEOWNERS_INITIALIZED=0 @@ -3973,7 +3981,7 @@ select_context_json_for_payload() { return 0 fi - dbg "selected bundled context '$matched_context' for payload '$payload_file' via repo '$repo_key'" + dbg "selected bundled context for payload via repo '$repo_key'" echo "$matched_context" } @@ -4014,7 +4022,7 @@ enrich_with_context() { local infile="$1"; local tmpfile="$2" local selected_ctx_file="" selected_ctx_file="$(select_context_json_for_payload "$infile")" - dbg "enrich_with_context: infile='$infile' outfile='$tmpfile' ctx='${selected_ctx_file:-}' primary='${PRIMARY_CONTEXT_JSON:-}' jq=$JQ_AVAILABLE" + dbg "enrich_with_context: jq=$JQ_AVAILABLE" if (( JQ_AVAILABLE == 0 )); then # No jq means no structural merge; forward original payload unchanged. cp "$infile" "$tmpfile" diff --git a/tools/core/uploader_powershell_runtime.ps1.tpl b/tools/core/uploader_powershell_runtime.ps1.tpl index b1198578..14d47fc7 100644 --- a/tools/core/uploader_powershell_runtime.ps1.tpl +++ b/tools/core/uploader_powershell_runtime.ps1.tpl @@ -481,7 +481,7 @@ function Initialize-RuntimeSelectionArguments { exit 2 } if ($script:RuntimeContextEntries.Contains($repoKey)) { Log "error: duplicate --context-entry repository '$repoKey'"; exit 2 } - if (-not $seenPaths.Add($resolved)) { Log "error: duplicate --context-entry path '$resolved'"; exit 2 } + if (-not $seenPaths.Add($resolved)) { Log "error: duplicate --context-entry path for repository '$repoKey'"; exit 2 } Assert-RuntimeContextPair $repoKey $resolved $telemetry $script:RuntimeContextEntries[$repoKey] = $resolved $script:RuntimeTelemetryFacts.Add([System.IO.Path]::GetFullPath($telemetry)) | Out-Null @@ -511,7 +511,7 @@ $ContextJsonPath = "__DDTPL_CONTEXT_JSON_PATH__" $TelemetryFactsManifestRloc = "__DDTPL_TELEMETRY_FACTS_MANIFEST_RLOC__" $TelemetryFactsManifestPath = "__DDTPL_TELEMETRY_FACTS_MANIFEST_PATH__" $ContextJsonOverride = $env:DD_TEST_OPTIMIZATION_CONTEXT_JSON -Dbg "context.json resolution inputs: override='$ContextJsonOverride' path='$ContextJsonPath' rloc='$ContextJsonRloc' manifest_path='$ContextManifestPath' manifest_rloc='$ContextManifestRloc'" +Dbg "resolving configured context.json and manifest inputs" $script:ContextJson = $null $script:PrimaryContextJson = $null $script:ContextManifest = $null @@ -576,7 +576,7 @@ if ($script:RuntimeContextEntries.Count -gt 0) { $script:PrimaryContextJson = Resolve-ArtifactPath $ContextJsonOverride if ($script:PrimaryContextJson) { $contextJsonFromOverride = $true - Dbg "context.json resolved via runtime override: '$script:PrimaryContextJson'" + Dbg "context.json resolved via runtime override" } else { Log "warning: DD_TEST_OPTIMIZATION_CONTEXT_JSON did not resolve to a readable file; falling back to configured data" } @@ -584,20 +584,20 @@ if ($script:RuntimeContextEntries.Count -gt 0) { if (-not $script:PrimaryContextJson) { $script:ContextManifest = Resolve-ArtifactPath $ContextManifestPath if ($script:ContextManifest) { - Dbg "context manifest resolved via direct path: '$script:ContextManifest'" + Dbg "context manifest resolved via direct path" } elseif ($ContextManifestRloc) { $script:ContextManifest = Resolve-Runfile $ContextManifestRloc if ($script:ContextManifest) { - Dbg "context manifest resolved via runfiles: '$script:ContextManifest'" + Dbg "context manifest resolved via runfiles" } } Load-ContextManifestEntries -ManifestPath $script:ContextManifest if ($script:BundledContextEntries.Count -gt 0) { $script:PrimaryContextJson = @($script:BundledContextEntries.Values)[0] if ($script:BundledContextEntries.Count -eq 1) { - Dbg "context.json resolved from single bundled context: '$script:PrimaryContextJson'" + Dbg "context.json resolved from single bundled context" } else { - Dbg "primary context.json resolved from bundled manifest: '$script:PrimaryContextJson' (repos=$([string]::Join(', ', @($script:BundledContextEntries.Keys))))" + Dbg "primary context.json resolved from bundled manifest (repos=$([string]::Join(', ', @($script:BundledContextEntries.Keys))))" } } } @@ -605,14 +605,14 @@ if (-not $script:PrimaryContextJson) { $script:PrimaryContextJson = Resolve-ArtifactPath $ContextJsonPath if ($script:PrimaryContextJson) { # Direct artifact path is preferred when launcher preserves it. - Dbg "context.json resolved via direct path: '$script:PrimaryContextJson'" + Dbg "context.json resolved via direct path" } elseif ($ContextJsonRloc) { # Runfiles fallback supports manifest-only and bzlmod path variants. $script:PrimaryContextJson = Resolve-Runfile $ContextJsonRloc if (-not $script:PrimaryContextJson) { Log "warning: context.json not found in runfiles; payloads will not be enriched" } else { - Dbg "context.json resolved via runfiles: '$script:PrimaryContextJson'" + Dbg "context.json resolved via runfiles" } } else { Dbg "context.json not configured in data files; enrichment disabled" @@ -1997,7 +1997,7 @@ if ($Agentless -and -not $script:DryRun) { } Dbg "headers prepared (agentless=$Agentless; test headers can be derived from metadata)" -Dbg "primary context.json: $(if ([string]::IsNullOrEmpty($script:PrimaryContextJson)) { '' } else { $script:PrimaryContextJson })" +Dbg "primary context.json $(if ([string]::IsNullOrEmpty($script:PrimaryContextJson)) { 'unavailable' } else { 'available' })" # Optional check: verify fetch-time API key fingerprint matches uploader API key. $ContextFingerprint = $null @@ -3584,7 +3584,7 @@ function Get-ContextInfo([string]$ContextPath) { $info.JsonText = Get-Content -LiteralPath $ContextPath -Raw -Encoding UTF8 $info.Object = $info.JsonText | ConvertFrom-Json -ErrorAction Stop } catch { - Log "warning: failed to parse context.json for payload enrichment: $ContextPath" + Log "warning: failed to parse context.json for payload enrichment" } } @@ -3627,7 +3627,7 @@ function Resolve-ContextJsonForPayload([string]$PayloadFile) { } $matchedContext = [string]$script:BundledContextEntries[$repoKey] - Dbg "selected bundled context '$matchedContext' for payload '$PayloadFile' via repo '$repoKey'" + Dbg "selected bundled context for payload via repo '$repoKey'" return $matchedContext } @@ -3670,7 +3670,7 @@ function Merge-With-Context([string]$infile, [string]$outfile) { $selectedContextPath = Resolve-ContextJsonForPayload $infile $selectedContextInfo = Get-ContextInfo $selectedContextPath $selectedContextObj = if ($selectedContextInfo) { $selectedContextInfo.Object } else { $null } - Dbg "Merge-With-Context: infile='$infile' selected_ctx='$(if ([string]::IsNullOrEmpty($selectedContextPath)) { '' } else { $selectedContextPath })' primary='$(if ([string]::IsNullOrEmpty($script:PrimaryContextJson)) { '' } else { $script:PrimaryContextJson })'" + Dbg "Merge-With-Context: context_selected=$(-not [string]::IsNullOrEmpty($selectedContextPath))" if (-not $payload.metadata) { $payload | Add-Member -NotePropertyName metadata -NotePropertyValue @{} -Force } $meta = Ensure-Hashtable $payload.metadata diff --git a/tools/tests/python/test_python_tools.py b/tools/tests/python/test_python_tools.py index e593249c..e092ebb5 100644 --- a/tools/tests/python/test_python_tools.py +++ b/tools/tests/python/test_python_tools.py @@ -7341,16 +7341,44 @@ def test_generated_uploaders_validate_runtime_selection_before_discovery(self) - self.assertIn("requires at least one --expected-target", result.stdout + result.stderr) with self.subTest(runtime=runtime, case="valid-multi-service"): + debug_env = os.environ.copy() + debug_env["DD_TEST_OPTIMIZATION_DEBUG"] = "1" result = subprocess.run( [*command, "--expected-target", "//pkg:test.topt", *context_args, "--help"], cwd=root, + env=debug_env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=30, check=False, ) - self.assertEqual(0, result.returncode, result.stdout + result.stderr) + output = result.stdout + result.stderr + self.assertEqual(0, result.returncode, output) + for context_path in context_paths: + self.assertNotIn(str(context_path), output) + + with self.subTest(runtime=runtime, case="duplicate-path-redacted"): + duplicate_args = [ + "--context-entry", + f"repo-b={context_paths[0]}", + "--context-entry", + f"repo-c={context_paths[0]}", + ] + result = subprocess.run( + [*command, "--expected-target", "//pkg:test.topt", *duplicate_args, "--help"], + cwd=root, + env=debug_env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=30, + check=False, + ) + output = result.stdout + result.stderr + self.assertEqual(2, result.returncode, output) + self.assertIn("duplicate --context-entry path for repository 'repo-c'", output) + self.assertNotIn(str(context_paths[0]), output) with self.subTest(runtime=runtime, case="identity-mismatch"): result = subprocess.run(