From abcfec8c715092aacceb76b3ad065b133266950f Mon Sep 17 00:00:00 2001 From: tazhate Date: Thu, 28 May 2026 20:11:04 +0300 Subject: [PATCH] fix(registry): drop token call for GAR, parse its nested tag shape MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every op-stack chain in the catalog (~14 of them) was failing in versioncheck with: decode token response: invalid character '<' looking for beginning of value Root cause: us-docker.pkg.dev (Google Artifact Registry) does not implement the Docker Distribution /token endpoint at all — it 302-redirects to console.cloud.google.com and the HTML lands in our JSON decoder. Public GAR repositories don't need a bearer token in the first place; /v2/{repo}/tags/list answers 200 directly. GAR also doesn't return the standard {"tags": [...]} envelope. It returns {"manifest": {digest: {"tag": [...]}}}, ignores ?n= pagination, and dumps every digest in one (~2 MB) response. Split the OCI client into two paths keyed on c.host: - us-docker.pkg.dev -> fetchGARTags: no token, flatten manifest map - everything else -> fetchStandardTags: same token+tags flow as before (ECR Public still works) Added httptest-based coverage for both paths, including the GAR host-prefix normalization, ECR Public's access_token field, and the non-200 error branch. Re-running cmd/versioncheck against master: - base, bob, celo, ink, mode, optimism, soneium now report real UPDATE AVAILABLE entries instead of decode errors - plume (ECR Public path) keeps working Out of scope for this PR (separate items): - ~17 Docker Hub 404s — repos were renamed/moved upstream, fix is per-chain in the adapters - GHCR 403 on morph/sonic/zircuit — same story (probably renamed) - isStableTag missing -synctest/-cdfpl/-overrides suffix variants that op-stack repos use for non-release builds Context: curled the GAR and ECR Public token + tags endpoints by hand to see exactly what each one returns, then split the client. ~1h including the test scaffold. --- internal/registry/oci.go | 163 ++++++++++++++++------- internal/registry/oci_test.go | 239 ++++++++++++++++++++++++++++++++++ 2 files changed, 353 insertions(+), 49 deletions(-) create mode 100644 internal/registry/oci_test.go diff --git a/internal/registry/oci.go b/internal/registry/oci.go index f045e24..2820298 100644 --- a/internal/registry/oci.go +++ b/internal/registry/oci.go @@ -30,13 +30,20 @@ import ( ) // ociClient speaks the Docker Distribution / OCI v2 tag-listing protocol. -// It handles any public registry that issues anonymous Bearer tokens, including -// Google Artifact Registry (us-docker.pkg.dev) and Amazon ECR Public (public.ecr.aws). +// It handles public OCI registries with two flavors: +// +// - Google Artifact Registry (us-docker.pkg.dev): public repos are +// anonymous-readable; tags are returned in a non-standard nested shape +// under {"manifest": {digest: {"tag": [...]}}}. +// - Standard OCI registries (e.g. public.ecr.aws): require a bearer token +// fetched from /token; tags are returned as {"tags": [...]}. type ociClient struct { host string http *http.Client } +const garHost = "us-docker.pkg.dev" + func (c *ociClient) httpClient() *http.Client { if c.http == nil { c.http = &http.Client{Timeout: 15 * time.Second} @@ -44,68 +51,91 @@ func (c *ociClient) httpClient() *http.Client { return c.http } -type ociTokenResponse struct { - Token string `json:"token"` - AccessToken string `json:"access_token"` // ECR Public uses this field -} +// LatestTags fetches tags from the configured OCI registry and filters them by +// policy.TagPattern. The maxResults cap applies after filtering. +func (c *ociClient) LatestTags(ctx context.Context, policy adapters.ChainVersionPolicy, maxResults int) ([]TagEntry, error) { + repo := policy.Repository + if strings.HasPrefix(repo, c.host+"/") { + repo = strings.TrimPrefix(repo, c.host+"/") + } -// getToken obtains an anonymous pull token via the standard OAuth2 scope URL. -// Both GAR and ECR Public follow the same www-authenticate/token pattern. -func (c *ociClient) getToken(ctx context.Context, repo string) (string, error) { - tokenURL := fmt.Sprintf("https://%s/token?scope=repository:%s:pull&service=%s", - c.host, url.QueryEscape(repo), c.host) + var ( + tags []string + err error + ) + if c.host == garHost { + tags, err = c.fetchGARTags(ctx, repo) + } else { + tags, err = c.fetchStandardTags(ctx, repo, maxResults) + } + if err != nil { + return nil, err + } - req, err := http.NewRequestWithContext(ctx, http.MethodGet, tokenURL, nil) + pattern, perr := regexp.Compile(policy.TagPattern) + if perr != nil { + return nil, fmt.Errorf("compile tag pattern %q: %w", policy.TagPattern, perr) + } + + entries := make([]TagEntry, 0, maxResults) + for _, tag := range tags { + if !pattern.MatchString(tag) { + continue + } + entries = append(entries, TagEntry{Tag: tag}) + if len(entries) >= maxResults { + break + } + } + return entries, nil +} + +// fetchGARTags reads all tags from a GAR repository. GAR ignores ?n= pagination +// and dumps every digest's tag list in one response, so we accept the full +// payload (typically a few MB for active op-stack repos) and flatten it. +func (c *ociClient) fetchGARTags(ctx context.Context, repo string) ([]string, error) { + tagsURL := fmt.Sprintf("https://%s/v2/%s/tags/list", c.host, repo) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, tagsURL, nil) if err != nil { - return "", fmt.Errorf("build token request: %w", err) + return nil, fmt.Errorf("build GAR tags request: %w", err) } + req.Header.Set("Accept", "application/json") resp, err := c.httpClient().Do(req) if err != nil { - return "", fmt.Errorf("fetch token from %s: %w", c.host, err) + return nil, fmt.Errorf("GAR tags request: %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("token endpoint %s returned %d", c.host, resp.StatusCode) + return nil, fmt.Errorf("GAR returned %d for %s", resp.StatusCode, repo) } - var tr ociTokenResponse - if err := json.NewDecoder(resp.Body).Decode(&tr); err != nil { - return "", fmt.Errorf("decode token response: %w", err) + var result garTagsResponse + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("decode GAR tags response: %w", err) } - if tr.Token != "" { - return tr.Token, nil + out := make([]string, 0, len(result.Manifest)) + for _, m := range result.Manifest { + out = append(out, m.Tag...) } - return tr.AccessToken, nil + return out, nil } -type ociTagsResponse struct { - Tags []string `json:"tags"` - // next page link is in the Link response header — ignored for now since - // we request enough tags in one shot via ?n= -} - -func (c *ociClient) LatestTags(ctx context.Context, policy adapters.ChainVersionPolicy, maxResults int) ([]TagEntry, error) { - token, err := c.getToken(ctx, policy.Repository) +// fetchStandardTags uses the Docker Distribution token flow: anonymous bearer +// token from /token, then /v2/{repo}/tags/list with the token attached. +func (c *ociClient) fetchStandardTags(ctx context.Context, repo string, maxResults int) ([]string, error) { + token, err := c.getToken(ctx, repo) if err != nil { return nil, err } - // Request more than maxResults so we have room to filter by pattern. fetchN := maxResults * 10 if fetchN < 100 { fetchN = 100 } - // Normalize repo path: strip leading "us-docker.pkg.dev/" or host prefix - // if the caller accidentally included it. - repo := policy.Repository - if strings.HasPrefix(repo, c.host+"/") { - repo = strings.TrimPrefix(repo, c.host+"/") - } - tagsURL := fmt.Sprintf("https://%s/v2/%s/tags/list?n=%d", c.host, repo, fetchN) req, err := http.NewRequestWithContext(ctx, http.MethodGet, tagsURL, nil) if err != nil { @@ -128,21 +158,56 @@ func (c *ociClient) LatestTags(ctx context.Context, policy adapters.ChainVersion if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { return nil, fmt.Errorf("decode tags response: %w", err) } + return result.Tags, nil +} + +type ociTokenResponse struct { + Token string `json:"token"` + AccessToken string `json:"access_token"` // ECR Public uses this field +} + +// getToken obtains an anonymous pull token via the standard OAuth2 scope URL. +// ECR Public follows the standard www-authenticate/token pattern. +func (c *ociClient) getToken(ctx context.Context, repo string) (string, error) { + tokenURL := fmt.Sprintf("https://%s/token?scope=repository:%s:pull&service=%s", + c.host, url.QueryEscape(repo), c.host) - pattern, err := regexp.Compile(policy.TagPattern) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, tokenURL, nil) if err != nil { - return nil, fmt.Errorf("compile tag pattern %q: %w", policy.TagPattern, err) + return "", fmt.Errorf("build token request: %w", err) } - entries := make([]TagEntry, 0, maxResults) - for _, tag := range result.Tags { - if !pattern.MatchString(tag) { - continue - } - entries = append(entries, TagEntry{Tag: tag}) - if len(entries) >= maxResults { - break - } + resp, err := c.httpClient().Do(req) + if err != nil { + return "", fmt.Errorf("fetch token from %s: %w", c.host, err) } - return entries, nil + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("token endpoint %s returned %d", c.host, resp.StatusCode) + } + + var tr ociTokenResponse + if err := json.NewDecoder(resp.Body).Decode(&tr); err != nil { + return "", fmt.Errorf("decode token response: %w", err) + } + + if tr.Token != "" { + return tr.Token, nil + } + return tr.AccessToken, nil +} + +type ociTagsResponse struct { + Tags []string `json:"tags"` +} + +// garTagsResponse is the non-standard envelope returned by Google Artifact +// Registry. Tags are buried inside per-digest manifest entries. +type garTagsResponse struct { + Manifest map[string]garManifestEntry `json:"manifest"` +} + +type garManifestEntry struct { + Tag []string `json:"tag"` } diff --git a/internal/registry/oci_test.go b/internal/registry/oci_test.go new file mode 100644 index 0000000..03f902e --- /dev/null +++ b/internal/registry/oci_test.go @@ -0,0 +1,239 @@ +/* +Copyright (c) 2026 tazhate +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package registry + +import ( + "context" + "net/http" + "net/http/httptest" + "net/url" + "sort" + "strings" + "testing" + "time" + + "github.com/tazhate/chainplane/internal/adapters" +) + +// pointTo rewrites the OCI client's host so it talks to the test server. +// It returns a client configured to dial the test server for HTTPS hostnames. +func pointTo(t *testing.T, srv *httptest.Server, host string) *ociClient { + t.Helper() + u, err := url.Parse(srv.URL) + if err != nil { + t.Fatalf("parse test server URL: %v", err) + } + // Custom transport rewrites scheme+host to the test server. + rt := &rewriteTransport{base: http.DefaultTransport, target: u} + return &ociClient{ + host: host, + http: &http.Client{Transport: rt, Timeout: 5 * time.Second}, + } +} + +type rewriteTransport struct { + base http.RoundTripper + target *url.URL +} + +func (r *rewriteTransport) RoundTrip(req *http.Request) (*http.Response, error) { + req.URL.Scheme = r.target.Scheme + req.URL.Host = r.target.Host + return r.base.RoundTrip(req) +} + +func TestOCIClient_GAR_NoTokenAndFlattensManifest(t *testing.T) { + var tokenHit bool + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasPrefix(r.URL.Path, "/token"): + tokenHit = true + http.Error(w, "should not be called for GAR", http.StatusInternalServerError) + case r.URL.Path == "/v2/oplabs-tools-artifacts/images/op-geth/tags/list": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "child": [], + "manifest": { + "sha256:aaa": {"tag": ["v1.101411.2"]}, + "sha256:bbb": {"tag": ["v1.101408.0", "v1.101408.0-rc1"]}, + "sha256:ccc": {"tag": []}, + "sha256:ddd": {"tag": ["nightly"]} + } + }`)) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + + c := pointTo(t, srv, garHost) + policy := adapters.ChainVersionPolicy{ + Registry: garHost, + Repository: "oplabs-tools-artifacts/images/op-geth", + TagPattern: `^v\d`, + } + + entries, err := c.LatestTags(context.Background(), policy, 10) + if err != nil { + t.Fatalf("LatestTags: %v", err) + } + if tokenHit { + t.Errorf("GAR client must not call /token endpoint") + } + + got := tagSet(entries) + want := []string{"v1.101408.0", "v1.101408.0-rc1", "v1.101411.2"} + sort.Strings(got) + if !equalStrings(got, want) { + t.Errorf("got tags %v, want %v", got, want) + } +} + +func TestOCIClient_GAR_TrimsHostPrefix(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v2/oplabs-tools-artifacts/images/op-geth/tags/list" { + t.Errorf("unexpected request path %q (host prefix not trimmed?)", r.URL.Path) + http.NotFound(w, r) + return + } + _, _ = w.Write([]byte(`{"manifest":{"sha256:x":{"tag":["v1.0.0"]}}}`)) + })) + defer srv.Close() + + c := pointTo(t, srv, garHost) + policy := adapters.ChainVersionPolicy{ + Registry: garHost, + Repository: "us-docker.pkg.dev/oplabs-tools-artifacts/images/op-geth", + TagPattern: `^v\d`, + } + if _, err := c.LatestTags(context.Background(), policy, 5); err != nil { + t.Fatalf("LatestTags: %v", err) + } +} + +func TestOCIClient_Standard_UsesBearerToken(t *testing.T) { + var sawAuth string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == "/token": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"token":"abc123"}`)) + case strings.HasPrefix(r.URL.Path, "/v2/") && strings.HasSuffix(r.URL.Path, "/tags/list"): + sawAuth = r.Header.Get("Authorization") + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"tags":["v3.6.2","v3.6.1","nightly","v3.6.3"]}`)) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + + c := pointTo(t, srv, "public.ecr.aws") + policy := adapters.ChainVersionPolicy{ + Registry: "public.ecr.aws", + Repository: "i6b2w2n6/nitro-node", + TagPattern: `^v\d`, + } + + entries, err := c.LatestTags(context.Background(), policy, 10) + if err != nil { + t.Fatalf("LatestTags: %v", err) + } + if sawAuth != "Bearer abc123" { + t.Errorf("expected Authorization=Bearer abc123, got %q", sawAuth) + } + + want := []string{"v3.6.2", "v3.6.1", "v3.6.3"} + if !equalStrings(tagSet(entries), want) { + t.Errorf("got %v, want %v", tagSet(entries), want) + } +} + +func TestOCIClient_Standard_AccessTokenField(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == "/token": + // ECR Public returns access_token, not token. + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"access_token":"ecr-token"}`)) + case strings.HasSuffix(r.URL.Path, "/tags/list"): + if got := r.Header.Get("Authorization"); got != "Bearer ecr-token" { + t.Errorf("expected Bearer ecr-token, got %q", got) + } + _, _ = w.Write([]byte(`{"tags":["v1.0.0"]}`)) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + + c := pointTo(t, srv, "public.ecr.aws") + policy := adapters.ChainVersionPolicy{ + Registry: "public.ecr.aws", + Repository: "x/y", + TagPattern: `^v\d`, + } + if _, err := c.LatestTags(context.Background(), policy, 5); err != nil { + t.Fatalf("LatestTags: %v", err) + } +} + +func TestOCIClient_GAR_ReturnsErrorOnNon200(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "boom", http.StatusInternalServerError) + })) + defer srv.Close() + + c := pointTo(t, srv, garHost) + policy := adapters.ChainVersionPolicy{ + Registry: garHost, + Repository: "x/y", + TagPattern: `.*`, + } + _, err := c.LatestTags(context.Background(), policy, 5) + if err == nil { + t.Fatal("expected error on 500, got nil") + } + if !strings.Contains(err.Error(), "500") { + t.Errorf("expected error to mention status 500, got %v", err) + } +} + +func tagSet(entries []TagEntry) []string { + out := make([]string, 0, len(entries)) + for _, e := range entries { + out = append(out, e.Tag) + } + return out +} + +func equalStrings(a, b []string) bool { + if len(a) != len(b) { + return false + } + am := map[string]int{} + for _, s := range a { + am[s]++ + } + for _, s := range b { + am[s]-- + if am[s] < 0 { + return false + } + } + return true +}