fix WPB-27900: add create-build-entry to nix path

Author: mohitrajain

default.nix

@@ -69,7 +69,7 @@ rec {
       list-helm-containers
       mirror-apt-jammy
       generate-gpg1-key
-      #create-build-entry
+      create-build-entry
       # Linting
       shellcheck
 

nix/overlay.nix

@@ -73,13 +73,13 @@ super: {
       wrapProgram $out/bin/list-helm-containers --prefix PATH : '${super.lib.makeBinPath [ self.kubernetes-helm ]}'
     '';
 
-  # create-build-entry = super.runCommandNoCC "create-build-entry"
-  #   {
-  #     nativeBuildInputs = [ super.makeWrapper ];
-  #   }
-  #   ''
-  #     install -Dm755 ${./scripts/create-build-entry.sh} $out/bin/create-build-entry
-  #     wrapProgram $out/bin/create-build-entry --prefix PATH : '${super.lib.makeBinPath (with self; [ bash jq ])}'
-  #   '';
+  create-build-entry = super.runCommandNoCC "create-build-entry"
+    {
+      nativeBuildInputs = [ super.makeWrapper ];
+    }
+    ''
+      install -Dm755 ${./scripts/create-build-entry.sh} $out/bin/create-build-entry
+      wrapProgram $out/bin/create-build-entry --prefix PATH : '${super.lib.makeBinPath (with self; [ bash jq ])}'
+    '';
 
 }

nix/scripts/create-build-entry.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+set -eou pipefail
+
+if [ "$#" -ne 2 ]; then
+  echo "Usage: $0 <docker-image-with-tag> <directory>"
+  exit 1
+fi
+
+IMAGE_WITH_TAG=$1
+DIRECTORY=$2
+
+IMAGE=$(echo "$IMAGE_WITH_TAG" | cut -d':' -f1)
+TAG=$(echo "$IMAGE_WITH_TAG" | cut -d':' -f2)
+
+JSON_FILE="$DIRECTORY/images.json"
+
+if [ ! -d "$DIRECTORY" ]; then
+  mkdir -p "$DIRECTORY"
+fi
+
+append_image_entry() {
+  local image=$1
+  local tag=$2
+  local json_file=$3
+
+  if [ -f "$json_file" ]; then
+    existing_content=$(jq '.' "$json_file")
+
+    new_entry=$(jq -n --arg image "$image" --arg tag "$tag" '{$image: $tag}')
+    updated_content=$(echo "$existing_content" | jq --argjson new_entry "$new_entry" '. += [$new_entry]')
+  else
+    updated_content=$(jq -n --arg image "$image" --arg tag "$tag" '[{$image: $tag}]')
+  fi
+
+  echo "$updated_content" | jq '.' > "$json_file"
+}
+
+append_image_entry "$IMAGE" "$TAG" "$JSON_FILE"

nix/scripts/create-container-dump.sh

@@ -8,33 +8,77 @@ if [[ ! $# -eq 1 ]]; then
   exit 1
 fi
 
+export HTTP_TIMEOUT=600     # Timeout in seconds (default is typically 90)
+export REGISTRY_TIMEOUT=600 # Registry specific timeout
+
+output_dir=$1
 mkdir -p $1
+
 # Download all the docker images into $1, and append its name to an index.txt
 # If this errors out for you, copy default-policy.json from the skopeo repo to
 # /etc/containers/policy.json
 while IFS= read -r image; do
-    # sanitize the image file name, replace slashes with underscores, suffix with .tar
-    image_filename=$(sed -r "s/[:\/]/_/g" <<< $image)
-    image_path=$(realpath $1)/${image_filename}.tar
-    if [[ -e $image_path ]];then
-      echo "Skipping $image_filename…"
+
+# sanitize the image file name, replace slashes with underscores, suffix with .tar
+  image_filename=$(sed -r "s/[:\/]/_/g" <<< "$image")
+  image_path="$(realpath "$1")/${image_filename}.tar"
+
+  if [[ -s "$image_path" ]]; then
+    echo "Skipping $image_filename…"
+    continue
+  fi
+
+  echo "Fetching $image_filename…"
+
+  # All of these images should be publicly fetchable, especially given we
+  # ship public tarballs containing these images.
+  # ci.sh already honors DOCKER_LOGIN, so do the same here, otherwise
+  # fallback to unauthorized fetching.
+
+  # If an image has both a tag and digest, remove the tag. Return the original if there is no match.
+  image_trimmed=$(echo "$image" | sed -E 's/(.+)(:.+(@.+))/\1\3/')
+
+  tmp_path="${image_path}.tmp"
+  rm -f "$tmp_path"
+
+  success=false
+
+  for attempt in {1..5}; do
+    echo "Attempt $attempt/5 for $image_trimmed"
+
+    if [[ -n "${DOCKER_LOGIN:-}" && "$image" =~ quay.io/wire ]]; then
+      skopeo copy --insecure-policy \
+        --src-creds "$DOCKER_LOGIN" \
+        --retry-times 10 \
+        "docker://$image_trimmed" \
+        "docker-archive:${tmp_path}" \
+        --additional-tag "$image" || rc=$?
     else
-      echo "Fetching $image_filename…"
-
-      # All of these images should be publicly fetchable, especially given we
-      # ship public tarballs containing these images.
-      # ci.sh already honors DOCKER_LOGIN, so do the same here, otherwise
-      # fallback to unauthorized fetching.
-
-      # If an image has both a tag and digest, remove the tag. Return the original if there is no match.
-      image_trimmed=$(echo "$image" | sed -E 's/(.+)(:.+(@.+))/\1\3/')
-      if [[ -n "${DOCKER_LOGIN:-}" && "$image" =~ quay.io/wire ]];then
-        skopeo copy --insecure-policy --src-creds "$DOCKER_LOGIN" \
-          docker://$image_trimmed docker-archive:${image_path} --additional-tag $image
-      else
-        skopeo copy --insecure-policy \
-          docker://$image_trimmed docker-archive:${image_path} --additional-tag $image
-      fi
-      echo "${image_filename}.tar" >> $(realpath "$1")/index.txt
+      skopeo copy --insecure-policy \
+        --retry-times 10 \
+        "docker://$image_trimmed" \
+        "docker-archive:${tmp_path}" \
+        --additional-tag "$image" || rc=$?
+    fi
+
+    rc=$?
+
+    if [[ $rc -eq 0 && -s "$tmp_path" ]]; then
+      mv "$tmp_path" "$image_path"
+      success=true
+      break
     fi
+
+    echo "Fetch failed for $image_trimmed with rc=$rc; retrying…"
+    rm -f "$tmp_path"
+    sleep $((attempt * 20))
+  done
+
+  if [[ "$success" != true ]]; then
+    echo "ERROR: failed to fetch $image after retries" >&2
+    exit 1
+  fi
+
+  echo "${image_filename}.tar" >> "$(realpath "$1")/index.txt"
+  create-build-entry "$image" "$output_dir"
 done