Cut PR build time: stop deleting the Hugo image cache, drop GOGC=3

.github/workflows/build-and-deploy.yml

@@ -103,6 +103,22 @@ jobs:
           restore-keys: |
             meta-images-
 
+      # Derive the cache key from the Hugo installed above rather than a literal,
+      # so revving hugo-version updates the key on its own. Fails loudly on an
+      # unparseable version rather than silently collapsing every build into a
+      # single unversioned namespace, which is the bug this exists to prevent.
+      - name: Resolve Hugo version for the cache key
+        id: hugo-version
+        run: |
+          hv="$(hugo version)"
+          # -n/p so a non-match yields empty rather than echoing the input back,
+          # and check before appending -extended so the guard can't see a value
+          # that is only the suffix.
+          ver="$(printf '%s' "$hv" | sed -nE 's/^hugo v([0-9]+\.[0-9]+\.[0-9]+).*/\1/p')"
+          [ -n "$ver" ] || { echo "could not parse hugo version from: $hv" >&2; exit 1; }
+          case "$hv" in *+extended*) ver="$ver-extended" ;; esac
+          echo "version=$ver" >> "$GITHUB_OUTPUT"
+
       # Persist Hugo's processed-image cache (resources/_gen). Blog templates
       # process feature images to WebP at several sizes via .Process; without
       # this cache every CI run re-encodes ~2,400 images from scratch (~4-5
@@ -111,9 +127,16 @@ jobs:
         uses: actions/cache@v6
         with:
           path: resources
-          key: hugo-resources-${{ github.sha }}
+          # Keyed on the Hugo resolved above, not a literal, so the four
+          # workflows that share this cache (pull-request.yml,
+          # build-and-deploy.yml, testing-build-and-deploy.yml,
+          # pulumi-cli-docs.yml) stay in step through an upgrade without anyone
+          # remembering to bump them. Hugo does not put its version in its own
+          # cache entries, so without this an upgrade keeps serving images
+          # encoded by the previous version indefinitely.
+          key: hugo-resources-${{ steps.hugo-version.outputs.version }}-${{ github.sha }}
           restore-keys: |
-            hugo-resources-
+            hugo-resources-${{ steps.hugo-version.outputs.version }}-
 
       - name: Record build start time
         run: |

.github/workflows/pull-request.yml

@@ -4,6 +4,17 @@ on:
     branches:
       - master
       - 'release/**'
+
+# Only the newest commit on a PR is worth building. Without this, pushing three
+# commits in quick succession runs three full builds to completion, and the two
+# superseded ones just burn a runner and hold up the queue. Each run publishes to
+# a bucket keyed by PR number *and* head sha, so a cancelled run can only orphan
+# its own preview bucket -- pr-closed.yml deletes every *-pr-<num>-* bucket when
+# the PR closes, and bucket-cleanup-testing.yml sweeps daily.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
 permissions:
       id-token: write # This is required for requesting the JWT
       contents: read # This is required for actions/checkout
@@ -78,6 +89,22 @@ jobs:
           restore-keys: |
             meta-images-
 
+      # Derive the cache key from the Hugo installed above rather than a literal,
+      # so revving hugo-version updates the key on its own. Fails loudly on an
+      # unparseable version rather than silently collapsing every build into a
+      # single unversioned namespace, which is the bug this exists to prevent.
+      - name: Resolve Hugo version for the cache key
+        id: hugo-version
+        run: |
+          hv="$(hugo version)"
+          # -n/p so a non-match yields empty rather than echoing the input back,
+          # and check before appending -extended so the guard can't see a value
+          # that is only the suffix.
+          ver="$(printf '%s' "$hv" | sed -nE 's/^hugo v([0-9]+\.[0-9]+\.[0-9]+).*/\1/p')"
+          [ -n "$ver" ] || { echo "could not parse hugo version from: $hv" >&2; exit 1; }
+          case "$hv" in *+extended*) ver="$ver-extended" ;; esac
+          echo "version=$ver" >> "$GITHUB_OUTPUT"
+
       # Persist Hugo's processed-image cache (resources/_gen). Blog templates
       # process feature images to WebP at several sizes via .Process; without
       # this cache every CI run re-encodes ~2,400 images from scratch (~4-5
@@ -86,9 +113,21 @@ jobs:
         uses: actions/cache@v6
         with:
           path: resources
-          key: hugo-resources-${{ github.sha }}
+          # Keyed on the Hugo resolved above, not a literal, so the four
+          # workflows that share this cache (pull-request.yml,
+          # build-and-deploy.yml, testing-build-and-deploy.yml,
+          # pulumi-cli-docs.yml) stay in step through an upgrade without anyone
+          # remembering to bump them. Hugo does not put its version in its own
+          # cache entries, so without this an upgrade keeps serving images
+          # encoded by the previous version indefinitely.
+          key: hugo-resources-${{ steps.hugo-version.outputs.version }}-${{ github.sha }}
           restore-keys: |
-            hugo-resources-
+            hugo-resources-${{ steps.hugo-version.outputs.version }}-
+
+      # PR runs don't wait on await-in-progress.js the way master deploys do, so
+      # the elapsed time here is build time with no queue wait to subtract.
+      - name: Record build start time
+        run: echo "CI_BUILD_START_EPOCH=$(date +%s)" >> "$GITHUB_ENV"
 
       - name: Build and deploy
         run: make ci_pull_request
@@ -104,6 +143,17 @@ jobs:
           ALGOLIA_APP_ADMIN_KEY: ${{  steps.esc-secrets.outputs.ALGOLIA_APP_ADMIN_KEY }}
           NODE_OPTIONS: "--max_old_space_size=8192"
 
+      # The same guardrail build-and-deploy.yml has. PR builds had none, which is
+      # how they drifted to ~17 minutes without anyone being told. 15 minutes
+      # sits a few minutes above the post-fix baseline, so it should stay quiet
+      # unless the build genuinely regresses; tighten it once that baseline settles.
+      - name: Alert on slow build
+        if: success()
+        env:
+          SLACK_WEBHOOK_URL: ${{ steps.esc-secrets.outputs.SLACK_WEBHOOK_URL }}
+          BUILD_DURATION_THRESHOLD_MINUTES: '15'
+        run: ./scripts/ci-build-duration-alert.sh
+
       - name: Archive test results
         uses: actions/upload-artifact@v7
         with:

.github/workflows/pulumi-cli-docs.yml

@@ -141,16 +141,39 @@ jobs:
           key: meta-images-${{ github.sha }}
           restore-keys: |
             meta-images-
+      # Derive the cache key from the Hugo installed above rather than a literal,
+      # so revving hugo-version updates the key on its own. Fails loudly on an
+      # unparseable version rather than silently collapsing every build into a
+      # single unversioned namespace, which is the bug this exists to prevent.
+      - name: Resolve Hugo version for the cache key
+        id: hugo-version
+        run: |
+          hv="$(hugo version)"
+          # -n/p so a non-match yields empty rather than echoing the input back,
+          # and check before appending -extended so the guard can't see a value
+          # that is only the suffix.
+          ver="$(printf '%s' "$hv" | sed -nE 's/^hugo v([0-9]+\.[0-9]+\.[0-9]+).*/\1/p')"
+          [ -n "$ver" ] || { echo "could not parse hugo version from: $hv" >&2; exit 1; }
+          case "$hv" in *+extended*) ver="$ver-extended" ;; esac
+          echo "version=$ver" >> "$GITHUB_OUTPUT"
+
       # Persist Hugo's processed-image cache (resources/_gen) for the full
       # `make build` below; without it every run re-encodes ~2,400 WebP images
       # from scratch (~4-5 minutes).
       - name: Cache Hugo processed images
         uses: actions/cache@v6
         with:
           path: resources
-          key: hugo-resources-${{ github.sha }}
+          # Keyed on the Hugo resolved above, not a literal, so the four
+          # workflows that share this cache (pull-request.yml,
+          # build-and-deploy.yml, testing-build-and-deploy.yml,
+          # pulumi-cli-docs.yml) stay in step through an upgrade without anyone
+          # remembering to bump them. Hugo does not put its version in its own
+          # cache entries, so without this an upgrade keeps serving images
+          # encoded by the previous version indefinitely.
+          key: hugo-resources-${{ steps.hugo-version.outputs.version }}-${{ github.sha }}
           restore-keys: |
-            hugo-resources-
+            hugo-resources-${{ steps.hugo-version.outputs.version }}-
       - run: make ensure
       - name: clean stale generated command pages
         run: |

.github/workflows/testing-build-and-deploy.yml

@@ -80,6 +80,22 @@ jobs:
           restore-keys: |
             meta-images-
 
+      # Derive the cache key from the Hugo installed above rather than a literal,
+      # so revving hugo-version updates the key on its own. Fails loudly on an
+      # unparseable version rather than silently collapsing every build into a
+      # single unversioned namespace, which is the bug this exists to prevent.
+      - name: Resolve Hugo version for the cache key
+        id: hugo-version
+        run: |
+          hv="$(hugo version)"
+          # -n/p so a non-match yields empty rather than echoing the input back,
+          # and check before appending -extended so the guard can't see a value
+          # that is only the suffix.
+          ver="$(printf '%s' "$hv" | sed -nE 's/^hugo v([0-9]+\.[0-9]+\.[0-9]+).*/\1/p')"
+          [ -n "$ver" ] || { echo "could not parse hugo version from: $hv" >&2; exit 1; }
+          case "$hv" in *+extended*) ver="$ver-extended" ;; esac
+          echo "version=$ver" >> "$GITHUB_OUTPUT"
+
       # Persist Hugo's processed-image cache (resources/_gen). Blog templates
       # process feature images to WebP at several sizes via .Process; without
       # this cache every CI run re-encodes ~2,400 images from scratch (~4-5
@@ -88,9 +104,16 @@ jobs:
         uses: actions/cache@v6
         with:
           path: resources
-          key: hugo-resources-${{ github.sha }}
+          # Keyed on the Hugo resolved above, not a literal, so the four
+          # workflows that share this cache (pull-request.yml,
+          # build-and-deploy.yml, testing-build-and-deploy.yml,
+          # pulumi-cli-docs.yml) stay in step through an upgrade without anyone
+          # remembering to bump them. Hugo does not put its version in its own
+          # cache entries, so without this an upgrade keeps serving images
+          # encoded by the previous version indefinitely.
+          key: hugo-resources-${{ steps.hugo-version.outputs.version }}-${{ github.sha }}
           restore-keys: |
-            hugo-resources-
+            hugo-resources-${{ steps.hugo-version.outputs.version }}-
 
       - name: Build and deploy
         run: make ci_push

BUILD-AND-DEPLOY.md

@@ -4007,7 +4007,7 @@ Complete reference of all build and deployment scripts.
 | **GITHUB_TOKEN** | GitHub API | (auto) | GitHub Actions |
 | **NOBUILD** | Skip rebuilds | `1` | User |
 | **ONLY_TEST** | Test single program | `aws-s3-typescript` | User |
-| **GOGC** | Go GC tuning | `3` | Workflow |
+| **GOMEMLIMIT** | Go soft memory ceiling for Hugo (CI only) | `12GiB` | build-site.sh |
 
 ### AWS Resource Naming Conventions
 

scripts/build-site.sh

@@ -27,15 +27,44 @@ printf "Generating meta images...\n\n"
 node scripts/generate-meta-images.mjs
 
 printf "Running Hugo...\n\n"
-if [ "$1" == "preview" ]; then
+# Hugo previously ran under GOGC=3, which collects once the heap grows 3% over
+# live heap. That capped memory but cost 1.7-3x in wall time, since every
+# allocation-heavy operation (image processing above all) drags a full GC behind
+# it. GOMEMLIMIT expresses the actual intent -- "do not exhaust the runner" --
+# as a soft ceiling, letting Go collect at its normal rate until the build
+# approaches the limit.
+#
+# CI only: this script is the local build path too (`make build`,
+# scripts/laptop-deploy.sh), and 12GiB is sized for the CI runner's 16GB shared
+# with the Node and Pulumi steps -- on a 16GB laptop it would be no ceiling at
+# all. An explicit GOMEMLIMIT always wins, so a memory-constrained machine can
+# set its own.
+if [ -n "${CI:-}" ]; then
+    export GOMEMLIMIT="${GOMEMLIMIT:-12GiB}"
+fi
+
+# --gc prunes cache entries the build no longer references, which is what bounds
+# the growth of the cached resources/ tree (nothing else reclaims superseded
+# entries). The guard below is on the non-preview branches, so this covers every
+# non-preview build under CI: the two deploy workflows, plus any CI job that runs
+# `make build` (pulumi-cli-docs.yml does). That is fine because all of them build
+# the full site and so reference the same set of entries. PR preview builds are
+# the ones deliberately excluded -- they share the same cache namespace, and a
+# build pruning against a narrower view could drop entries the others still need.
+hugo_gc=()
+if [ -n "${CI:-}" ]; then
+    hugo_gc=(--gc)
+fi
+
+if [ "${1:-}" == "preview" ]; then
     export HUGO_BASEURL="http://$(origin_bucket_prefix)-$(build_identifier).s3-website.$(aws_region).amazonaws.com"
-    GOGC=3 hugo --minify --buildFuture --templateMetrics -e "preview"
+    hugo --minify --buildFuture --templateMetrics -e "preview"
 else
     if [ "$DEPLOYMENT_ENVIRONMENT" == "testing" ]; then
         export HUGO_BASEURL="https://www.pulumi-test.io"
-        GOGC=3 hugo --minify --buildFuture --templateMetrics -e "preview"
+        hugo "${hugo_gc[@]}" --minify --buildFuture --templateMetrics -e "preview"
     else
-        GOGC=3 hugo --minify --templateMetrics -e "production"
+        hugo "${hugo_gc[@]}" --minify --templateMetrics -e "production"
     fi
 fi
 

scripts/clean.sh

@@ -1,10 +1,28 @@
 #!/bin/bash
 
-yarn cache clean
-hugo mod clean
+# Locally, this is a full reset: wiping node_modules and the Hugo caches is how
+# you recover from a bad install or stale generated resources.
+#
+# CI is a different situation. The workspace is a fresh clone, so there is
+# nothing stale to remove -- but actions/cache has just restored the yarn cache
+# and Hugo's processed-image cache (resources/) into it. Deleting those here
+# throws the restore away seconds after it happened, which forces Hugo to
+# re-encode every blog feature image on every run. The caches still saved and
+# restored cleanly, so the waste was invisible in the job log.
+#
+# Skip those paths under CI. `hugo mod clean` is in here too: it is a no-op on a
+# fresh runner (no module cache has been downloaded yet at this point), and
+# keeping it out of the CI path removes any question of it reaching into
+# resources/_gen and undoing the guard above. The rest are no-ops on a fresh
+# clone and are left alone so local behavior is unchanged.
+if [ -z "${CI:-}" ]; then
+    yarn cache clean
+    rm -rf resources
+    hugo mod clean
+fi
+
 rm -rf node_modules
 rm -rf infrastructure/node_modules
-rm -rf resources
 rm -rf _vendor
 rm -rf public
 rm -rf cypress/screenshots