RHAIENG-5599: Fix GHA refresh rpm lock file

Add GitHub Action to automate regeneration of rpms.lock.yaml (ODH and RHDS)

Author: ysok

.github/workflows/rpms-lock-renewal.yaml

@@ -22,6 +22,11 @@ on:  # yamllint disable-line rule:truthy
         description: 'Branch to update'
         required: false
         default: 'main'
+      use_cache:
+        description: 'Use GHCR layer cache (Uncheck to rebuild from scratch)'
+        required: false
+        default: true
+        type: boolean
 
 jobs:
   refresh-rpm-lock-files:
@@ -32,15 +37,23 @@ jobs:
       cancel-in-progress: false
     permissions:
       contents: read  # checkout only; push and PR creation use the PAT
+      packages: write  # push/pull rpm-lockfile build cache to ghcr.io
     env:
       BRANCH: ${{ github.event.inputs.branch || 'main' }}
       # Scheduled runs refresh ODH only; RHDS requires manual dispatch with subscription secrets.
       VARIANT: ${{ github.event.inputs.variant || 'odh' }}
       TMPDIR: /home/runner/.local/share/containers/tmpdir
       CONTAINER_HOST: unix:///var/run/podman/podman.sock
       CI: "true"
+      # workflow_dispatch only; scheduled runs always use cache (input unset → true).
+      USE_CACHE: ${{ github.event.inputs.use_cache != 'false' && 'true' || 'false' }}
+      # GitHub Container Registry used for podman build layer cache (see build-notebooks-TEMPLATE.yaml).
+      CACHE: "ghcr.io/${{ github.repository }}/rpm-lockfile-build-cache-${{ github.event.inputs.variant || 'odh' }}"
 
     steps:
+      - name: Downcase CACHE registry path
+        run: echo "CACHE=${CACHE,,}" >> "${GITHUB_ENV}"
+
       - name: Checkout code
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
@@ -55,11 +68,9 @@ jobs:
       - name: Prepare Podman temp directory
         run: mkdir -p "$TMPDIR"
 
-      - name: Install Podman
-        uses: ./.github/actions/install-podman-action
-        with:
-          platform: linux/amd64
-
+      # RHDS secrets and registry auth must be configured before Install Podman.
+      # install-podman-action runs `sudo podman system reset`, which can leave
+      # root-owned files under $HOME/.config/containers and break a later cp.
       - name: Install git-crypt
         if: env.VARIANT == 'rhds'
         uses: ./.github/actions/apt-install
@@ -79,6 +90,9 @@ jobs:
         if: env.VARIANT == 'rhds'
         run: |
           mkdir -p "$HOME/.config/containers"
+          # Don't use sudo here. With CONTAINER_HOST set, podman is a remote client
+          # that forwards auth to the rootful server during builds; credentials
+          # must be readable by the runner user (see build-notebooks-TEMPLATE.yaml).
           cp ci/secrets/pull-secret.json "$HOME/.config/containers/auth.json"
 
       - name: Mask subscription credentials
@@ -96,13 +110,53 @@ jobs:
           echo "::add-mask::${SUBSCRIPTION_ACTIVATION_KEY}"
           echo "::add-mask::${SUBSCRIPTION_ORG}"
 
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121  # v4.1.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Install Podman
+        uses: ./.github/actions/install-podman-action
+        with:
+          platform: linux/amd64
+
       - name: Free up disk space
         uses: ./.github/actions/free-up-disk-space
 
+      - name: Compute CONTAINER_BUILD_CACHE_ARGS
+        id: container-build-cache-args
+        shell: python
+        env:
+          USE_CACHE: ${{ env.USE_CACHE }}
+        run: |
+          import os
+          import sys
+
+          cache = os.environ["CACHE"]
+          event_name = os.environ["GITHUB_EVENT_NAME"]
+          use_cache = os.environ.get("USE_CACHE", "true").lower() == "true"
+
+          if event_name not in ("schedule", "workflow_dispatch"):
+              raise ValueError(f"Unexpected event name: {event_name}")
+
+          if use_cache:
+              cache_args = f"--cache-from {cache} --cache-to {cache}"
+              print("Using and updating GHCR layer cache", file=sys.stderr)
+          else:
+              # Skip --cache-from for a clean rebuild; still publish fresh layers afterward.
+              cache_args = f"--cache-to {cache}"
+              print("Rebuild from scratch (skipping --cache-from); will refresh GHCR cache after build", file=sys.stderr)
+
+          with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as f:
+              f.write(f"CONTAINER_BUILD_CACHE_ARGS={cache_args}\n")
+
       - name: Regenerate rpms.lock.yaml
         env:
           SUBSCRIPTION_ACTIVATION_KEY: ${{ secrets.SUBSCRIPTION_ACTIVATION_KEY }}
           SUBSCRIPTION_ORG: ${{ secrets.SUBSCRIPTION_ORG }}
+          CONTAINER_BUILD_CACHE_ARGS: ${{ steps.container-build-cache-args.outputs.CONTAINER_BUILD_CACHE_ARGS }}
         run: |
           set -euo pipefail
 
@@ -116,7 +170,10 @@ jobs:
             "codeserver/ubi9-python-3.12/prefetch-input/${VARIANT}/rpms.in.yaml"
           )
 
-          echo "Regenerating RPM lockfiles for variant: ${VARIANT}"
+          echo "Regenerating RPM lockfiles for variant: ${VARIANT} (USE_CACHE=${USE_CACHE})"
+          if [[ "$USE_CACHE" == "false" ]]; then
+            podman rmi -f localhost/notebook-rpm-lockfile:latest 2>/dev/null || true
+          fi
           for rpm_input in "${RPM_INPUTS[@]}"; do
             if [[ ! -f "$rpm_input" ]]; then
               echo "Skipping missing input: $rpm_input"
@@ -151,8 +208,14 @@ jobs:
           git commit -m "Update ${VARIANT_UPPER} rpms.lock.yaml lock files"
           git push -u "https://x-access-token:${GH_TOKEN}@github.com/${{ github.repository }}.git" "$BRANCH_NAME"
 
+          # gh pr create fails if the label does not exist yet.
+          gh label create "automated-rpms-lockfile-update" \
+            --description "Auto-created RPM lockfile update PR" \
+            --color "0E8A16" \
+            2>/dev/null || true
+
           gh pr create \
-            --title "Update ${VARIANT_UPPER} rpms.lock.yaml lock files" \
+            --title "GHA: Update ${VARIANT_UPPER} rpms.lock.yaml lock files" \
             --head "$BRANCH_NAME" \
             --body "$(cat <<EOF
           Automated regeneration of \`rpms.lock.yaml\` from \`rpms.in.yaml\` for the **${VARIANT_UPPER}** variant.

scripts/lockfile-generators/Dockerfile.rpm-lockfile

@@ -2,9 +2,9 @@
 #
 # Used by create-rpm-lockfile.sh to resolve RPM packages listed in
 # rpms.in.yaml into a fully pinned rpms.lock.yaml (exact URLs + checksums
-# per architecture).  The image includes rpm-lockfile-prototype, createrepo_c
-# (for repo metadata), and optionally registers with subscription-manager
-# to access RHEL-only repos.
+# per architecture).  The image includes rpm-lockfile-prototype and
+# optionally registers with subscription-manager to access RHEL-only repos.
+# RPM download + repodata for local builds use hermeto-fetch-rpm.sh, not this image.
 #
 # Not used during the actual notebook image build — only for lockfile generation.
 
@@ -31,13 +31,10 @@ RUN if [ -n "$ACTIVATION_KEY" ] && [ -n "$ORG" ]; then \
         # export INJECT_RHEL_VERSION="--releasever=\"${RHEL_VERSION}\" "; \
     fi
 
-# To install modulemd-tools (createrepo_c) for Development only, not for Production
-COPY helpers/rhsm-pulp.repo /etc/yum.repos.d/rhsm-pulp.repo
-
 # Don't run upgrade, it upgrades from 9.6 to 9.7 which is not desired here
 # Add --releasever when RHEL_VERSION is set (e.g. for RHEL/UBI) to pin repo release
 RUN releasever_opt=; [ -n "$RHEL_VERSION" ] && releasever_opt="--releasever=${RHEL_VERSION}"; \
-    dnf $releasever_opt install -y python3 python3-pip python3-dnf skopeo rpm git modulemd-tools patch && \
+    dnf $releasever_opt install -y python3 python3-pip python3-dnf skopeo rpm git patch && \
     dnf clean all
 
 # --index-url: override the base image's pip.conf which may point to a restricted

scripts/lockfile-generators/README.md

@@ -228,14 +228,12 @@ internally. Option 6 (Git submodule) is a manual setup.
 | ----------------------------------------------- | ---------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | `helpers/pylock-to-requirements.py`             | pip              | Convert `pylock.<flavor>.toml` (PEP 751) to pip-compatible `requirements.<flavor>.txt` with `--hash` lines.                                                                                                                              |
 | `helpers/download-pip-packages.py`              | pip              | Standalone pip downloader: downloads wheels/sdists from a `requirements.txt` (with `--hash` lines) into `cachi2/output/deps/pip/`. Not called by `create-requirements-lockfile.sh` (which has its own inline download from pylock.toml). |
-| `helpers/download-rpms.sh`                      | RPM              | Download RPMs from `rpms.lock.yaml` via `wget` into `cachi2/output/deps/rpm/` and create DNF repo metadata. Standalone alternative to `hermeto-fetch-rpm.sh`.                                                                            |
 | `helpers/hermeto-fetch-rpm.sh`                  | RPM              | Download RPMs from `rpms.lock.yaml` using [Hermeto](https://github.com/hermetoproject/hermeto) in a container. Handles RHEL entitlement cert extraction for `cdn.redhat.com` auth. Called by `create-rpm-lockfile.sh --download`.        |
 | `helpers/hermeto-fetch-npm.sh`                  | npm              | Alternative npm fetcher using [Hermeto](https://github.com/hermetoproject/hermeto) in a container.                                                                                                                                       |
 | `helpers/hermeto-fetch-gomod.sh`                | Go modules       | Fetches Go dependencies from a directory with `go.mod`/`go.sum` using [Hermeto](https://github.com/hermetoproject/hermeto) in a container. Output: `cachi2/output/deps/gomod/`. Called by `create-go-lockfile.sh`.                       |
 | `rewrite-npm-urls.sh`                           | npm (Dockerfile) | Rewrites `resolved` URLs in `package-lock.json` / `package.json` to `file:///cachi2/output/deps/npm/`.                                                                                                                                   |
 | `helpers/rpm-lockfile-generate.sh`              | RPM              | Runs `rpm-lockfile-prototype` inside the lockfile container. Not for direct host use.                                                                                                                                                    |
-| `Dockerfile.rpm-lockfile`                       | RPM              | Builds the container image for `create-rpm-lockfile.sh` (includes `rpm-lockfile-prototype` v0.20.0, `createrepo_c`, `modulemd-tools`). Applies patches from `patches/` at build time.                                                    |
-| `helpers/rhsm-pulp.repo`                        | RPM              | DNF repo file for RHEL 9 E4S appstream (used inside the lockfile container to install `modulemd-tools`).                                                                                                                                 |
+| `Dockerfile.rpm-lockfile`                       | RPM              | Builds the container image for `create-rpm-lockfile.sh` (includes `rpm-lockfile-prototype` v0.20.0). Applies patches from `patches/` at build time. Does not install internal-only corp repos. |
 | `patches/apply-patches.sh`                      | RPM (build)      | Applies local patches to pip-installed packages inside the `notebook-rpm-lockfile` container during `docker build`.                                                                                                                      |
 | `patches/rpm-lockfile-prototype-dnf-conf.patch` | RPM (build)      | Adds `RPM_LOCKFILE_MODULE_PLATFORM_ID` and `RPM_LOCKFILE_SKIP_UNAVAILABLE` env var support to `rpm-lockfile-prototype`'s DNF config.                                                                                                     |
 
@@ -590,25 +588,6 @@ and `--org` to register a temporary UBI container and extract fresh certs.
 
 **Requirements:** `podman`, network access.
 
-### Helper: `helpers/download-rpms.sh`
-
-Standalone alternative to `hermeto-fetch-rpm.sh` that downloads RPMs directly
-via `wget`.  Downloads RPMs from a lockfile into `cachi2/output/deps/rpm/`,
-verifies checksums (when `yq` is available), and creates DNF repo metadata using
-the first available method: `createrepo_c` → `createrepo` → container fallback
-(runs `createrepo_c` + `repo2module` + `modifyrepo_c` inside the
-`notebook-rpm-lockfile` image via podman).
-
-Does not handle RHEL entitlement — use `hermeto-fetch-rpm.sh` when downloading
-from `cdn.redhat.com` repos that require subscription certs.
-
-```bash
-./scripts/lockfile-generators/helpers/download-rpms.sh \
-    --lock-file codeserver/ubi9-python-3.12/prefetch-input/odh/rpms.lock.yaml
-```
-
-**Requirements:** `wget`.  `yq` recommended for checksum verification.
-
 ### Helper: `helpers/rpm-lockfile-generate.sh`
 
 Invoked **inside** the `notebook-rpm-lockfile` container by `create-rpm-lockfile.sh`.

scripts/lockfile-generators/create-rpm-lockfile.sh

@@ -109,15 +109,26 @@ else
 fi
 
 # First build the image, so we can run rpm-lockfile-prototype inside it
-echo "--- Building Lockfile Generator Image ---"
-podman build \
-    -f "$SCRIPTS_PATH/Dockerfile.rpm-lockfile" \
-    --platform linux/x86_64 \
-    --build-arg RHEL_VERSION="$RHEL_VERSION" \
-    --build-arg BASE_IMAGE="$BASE_IMAGE" \
-    --build-arg ACTIVATION_KEY="$ACTIVATION_KEY" \
-    --build-arg ORG="$ORG" \
-    -t notebook-rpm-lockfile "$SCRIPTS_PATH"
+CACHE_ARGS=()
+if [[ -n "${CONTAINER_BUILD_CACHE_ARGS:-}" ]]; then
+  # shellcheck disable=SC2206
+  CACHE_ARGS=($CONTAINER_BUILD_CACHE_ARGS)
+fi
+
+if podman image exists localhost/notebook-rpm-lockfile:latest 2>/dev/null; then
+  echo "--- Reusing existing Lockfile Generator Image ---"
+else
+  echo "--- Building Lockfile Generator Image ---"
+  podman build \
+      -f "$SCRIPTS_PATH/Dockerfile.rpm-lockfile" \
+      --platform linux/x86_64 \
+      --build-arg RHEL_VERSION="$RHEL_VERSION" \
+      --build-arg BASE_IMAGE="$BASE_IMAGE" \
+      --build-arg ACTIVATION_KEY="$ACTIVATION_KEY" \
+      --build-arg ORG="$ORG" \
+      "${CACHE_ARGS[@]}" \
+      -t notebook-rpm-lockfile "$SCRIPTS_PATH"
+fi
 
 # Second run rpm-lockfile-prototype to generate the lockfile
 echo "--- Generating Lockfile using rpm-lockfile-prototype --"