[main] micro version bundle generation fixes (#3781)

* fixes bundle generation for .micro >=.2 versions

* fixes

* hack/generate/index-bundles.sh

* make generated-files

* make generated-files

* missed the actual script

* use kafkasource v1beta1 in kitchensink (for upgrade tests)

* kitchensik test, use kafkasource v1beta1 also for readiness checks

* skip EventTransform with SM enabled, SRVKE-1751

Author: maschmid

Makefile

@@ -286,6 +286,8 @@ generate-ci-config:
 
 generate-catalog:
 	./hack/generate/catalog.sh
+	./hack/generate/index-bundles.sh \
+		olm-catalog/serverless-operator-index/index-bundles.yaml
 .PHONY: generate-catalog
 
 generate-override-snapshot: install-tools

hack/generate/annotations.sh

@@ -9,7 +9,7 @@ target="${2:?Provide a target annotations file as arg[2]}"
 source "$(dirname "${BASH_SOURCE[0]}")/../lib/metadata.bash"
 
 declare -A values
-values[operators.operatorframework.io.bundle.channel.default.v1]="$(metadata.get olm.channels.default)"
+values[operators.operatorframework.io.bundle.channel.default.v1]="$(metadata.get olm.channels.operator_default)"
 values[operators.operatorframework.io.bundle.channels.v1]="$(metadata.get olm.channels.default),$(metadata.get 'olm.channels.list[*]' | head -n 2 | tail -n 1)"
 values[operators.operatorframework.io.bundle.package.v1]="$(metadata.get project.name)"
 

hack/generate/catalog.sh

@@ -58,7 +58,7 @@ function generate_catalog {
 
 function add_channel {
   local channel catalog_template catalog current_version current_csv major \
-    minor micro previous_version channel_entry version
+    minor micro previous_version replaces_version channel_entry version
   catalog_template=${1?Pass catalog template path as arg[1]}
   channel=${2:?Pass channel name as arg[2]}
 
@@ -73,8 +73,10 @@ function add_channel {
   # Handle the first entry specifically as it might be a z-stream release.
   if [[ "$micro" == "0" ]]; then
     previous_version="${major}.$(( minor-1 )).${micro}"
+    replaces_version="${major}.$(( minor-1 )).${micro}"
   else
     previous_version="${major}.${minor}.0"
+    replaces_version="${major}.${minor}.$(( micro-1 ))"
   fi
 
   catalog=$(mktemp catalog-XXX.json)
@@ -97,7 +99,7 @@ function add_channel {
   should_add=0
   # Add entry to the channel if doesn't exist yet
   if [[ "${current_csv_entry}" == "" ]]; then
-    replaces="serverless-operator.v${previous_version}"
+    replaces="serverless-operator.v${replaces_version}"
     entry_with_same_replaces=$(yq read "${catalog_template}" "entries[name==${channel}].entries[replaces==${replaces}].name")
     if [[ "${entry_with_same_replaces}" == "" ]]; then
       should_add=1

hack/generate/dockerfile.sh

@@ -14,6 +14,7 @@ declare -A values
 values[NAME]="$(metadata.get project.name)"
 values[LATEST_VERSIONED_CHANNEL]="$(metadata.get 'olm.channels.list[1]')"
 values[DEFAULT_CHANNEL]="$(metadata.get olm.channels.default)"
+values[OPERATOR_DEFAULT_CHANNEL]="$(metadata.get olm.channels.operator_default)"
 values[VERSION]="$(metadata.get project.version)"
 values[SERVING_VERSION]="$(metadata.get dependencies.serving)"
 values[EVENTING_VERSION]="$(metadata.get dependencies.eventing)"
@@ -33,28 +34,6 @@ if [[ "$template" =~ index.Dockerfile ]]; then
     echo "Value: ${before} -> ${values[$before]}"
     sed --in-place "s|__${before}__|${values[${before}]}|" "$target"
   done
-
-  current_version=$(metadata.get 'project.version')
-  major=$(versions.major "$current_version")
-  minor=$(versions.minor "$current_version")
-  micro=$(versions.micro "$current_version")
-
-  # One is already added in template
-  num_csvs=$(( INDEX_IMAGE_NUM_CSVS-1 ))
-
-  # Generate additional entries
-  for i in $(seq $num_csvs); do
-    current_minor=$(( minor-i ))
-    # If the current version is a z-stream then the following entries will
-    # start with the same "minor" version.
-    if [[ "$micro" != "0" ]]; then
-      current_minor=$(( current_minor+1 ))
-    fi
-    current_version="${major}.${current_minor}.0"
-
-    sed --in-place "/opm render/a quay.io/redhat-user-workloads/ocp-serverless-tenant/serverless-operator-${major}${current_minor}/serverless-bundle:${current_version} \\\\" "$target"
-  done
-
 elif [[ "$template" =~ catalog.Dockerfile ]]; then
   while IFS=$'\n' read -r ocp_version; do
     values[OCP_VERSION]="${ocp_version}"

hack/generate/index-bundles.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+
+target="${1:?Provide a target index yaml file as arg[1]}"
+
+# shellcheck disable=SC1091,SC1090
+source "$(dirname "${BASH_SOURCE[0]}")/../lib/__sources__.bash"
+
+function add_previous_bundles {
+  if [[ -n "${REGISTRY_REDHAT_IO_USERNAME:-}" ]] || [[ -n "${REGISTRY_REDHAT_IO_PASSWORD:-}" ]]; then
+    skopeo login registry.redhat.io -u "${REGISTRY_REDHAT_IO_USERNAME}" -p "${REGISTRY_REDHAT_IO_PASSWORD}"
+  fi
+
+  # We're adding just the "previous bundles"
+  num_csvs=$(( INDEX_IMAGE_NUM_CSVS-1 ))
+
+  current_version=$(metadata.get 'project.version')
+  major=$(versions.major "$current_version")
+  minor=$(versions.minor "$current_version")
+  micro=$(versions.micro "$current_version")
+
+  for i in $(seq $num_csvs); do
+    current_minor=$(( minor ))
+    current_micro=$(( micro-i ))
+    if [[ "$current_micro" -le 0 ]]; then
+      current_minor=$((minor + current_micro))
+      current_micro=0
+    fi
+
+    current_version="${major}.${current_minor}.${current_micro}"
+
+    opm render --skip-tls-verify -o yaml "registry.redhat.io/openshift-serverless-1/serverless-operator-bundle:${current_version}" >> "$target"
+  done
+}
+# Clear the file.
+rm -f "${target}"
+
+add_previous_bundles "${target}"

hack/generate/index.sh

@@ -8,7 +8,7 @@ target="${1:?Provide a target index yaml file as arg[1]}"
 source "$(dirname "${BASH_SOURCE[0]}")/../lib/__sources__.bash"
 
 function add_channel_entries {
-  local channel_entry_yaml channel target num_csvs
+  local channel_entry_yaml channel target current_version replaces_version base_version
   channel=${1:?Provide channel name}
   target=${2:?Provide target file name}
   channel_entry_yaml="$(mktemp -t default-entry-XXXXX.yaml)"
@@ -26,51 +26,35 @@ EOF
   minor=$(versions.minor "$current_version")
   micro=$(versions.micro "$current_version")
 
-  # Handle the first entry specifically as it might be a z-stream release.
-  if [[ "$micro" == "0" ]]; then
-    previous_version="${major}.$(( minor-1 )).${micro}"
-  else
-    previous_version="${major}.${minor}.0"
-  fi
+  for i in $(seq "$INDEX_IMAGE_NUM_CSVS"); do
+    current_major=$((major))
+    current_minor=$((minor))
+    current_micro=$((micro - i + 1))
 
-  cat << EOF | yq write --inplace --script - "$channel_entry_yaml"
-  - command: update
-    path: entries[+]
-    value:
-      name: "serverless-operator.v${current_version}"
-      replaces: "serverless-operator.v${previous_version}"
-      skipRange: ">=${previous_version} <${current_version}"
-EOF
-
-  # One is already added above specifically
-  num_csvs=$(( INDEX_IMAGE_NUM_CSVS-1 ))
-
-  # Generate additional entries
-  for i in $(seq $num_csvs); do
-    current_minor=$(( minor-i ))
-    previous_minor=$(( minor-i ))
-    previous_minor=$(( previous_minor-1 ))
-    # If the current version is a z-stream then the following entries will
-    # start with the same "minor" version.
-    if [[ "$micro" != "0" ]]; then
-      current_minor=$(( current_minor+1 ))
-      previous_minor=$(( previous_minor+1 ))
+    if [[ "$current_micro" -le 0 ]]; then
+      current_minor=$((minor + current_micro))
+      current_micro=0
     fi
 
-    current_version="${major}.${current_minor}.0"
-    previous_version="${major}.${previous_minor}.0"
+    current_version="${current_major}.${current_minor}.${current_micro}"
+    if [[ "$current_micro" == "0" ]]; then
+      base_version="${current_major}.$(( current_minor-1 )).0"
+      replaces_version="${current_major}.$(( current_minor-1 )).0"
+    else
+      base_version="${current_major}.${current_minor}.0"
+      replaces_version="${current_major}.${current_minor}.$(( current_micro-1 ))"
+    fi
 
-    # If this is the last item enter only name, without "replaces".
-    if [[ $i -eq $num_csvs ]]; then
+    if [[ $i -eq "$INDEX_IMAGE_NUM_CSVS" ]]; then
       yq write --inplace "$channel_entry_yaml" 'entries[+].name' "serverless-operator.v${current_version}"
     else
       cat << EOF | yq write --inplace --script - "$channel_entry_yaml"
   - command: update
     path: entries[+]
     value:
       name: "serverless-operator.v${current_version}"
-      replaces: "serverless-operator.v${previous_version}"
-      skipRange: ">=${previous_version} <${current_version}"
+      replaces: "serverless-operator.v${replaces_version}"
+      skipRange: ">=${base_version} <${current_version}"
 EOF
     fi
   done

hack/lib/images.bash

@@ -88,19 +88,29 @@ function get_bundle_for_version() {
   app_version=${app_version%.*} # 134.0 -> 134
 
   image=$(image_with_sha "${registry_prefix_quay}${app_version}/serverless-bundle:latest")
+  image_version=$(bundle_image_version "${registry_prefix_quay}${app_version}/serverless-bundle:latest")
+
   # As a backup, try also CI registry.
+  # For .micro releases, it's possible we only have the _previous_ version in Konflux, so also check the version of the bundle
   local ci_bundle="registry.ci.openshift.org/knative/serverless-bundle"
-  if [[ "${image}" == "" ]]; then
+  if [[ "${image}" == "" || "${image_version}" != "${version}" ]]; then
     image=$(image_with_sha "${ci_bundle}:release-${version}" || echo "")
+    image_version=$(bundle_image_version "${ci_bundle}:release-${version}")
   fi
-  if [[ "${image}" == "" ]]; then
+  if [[ "${image}" == "" || "${image_version}" != "${version}" ]]; then
     image=$(image_with_sha "${ci_bundle}:knative-main")
+    image_version=$(bundle_image_version "${ci_bundle}:knative-main")
   fi
 
   if [[ "${image}" == "" ]]; then
+    echo "[ERROR] No image found for $version" > /dev/stderr
     exit 1
   fi
 
+  if [[ "${image_version}" != "${version}" ]]; then
+    echo "[WARNING] Image $image $image_version does not match requested version $version" > /dev/stderr
+  fi
+
   echo "$image"
 }
 
@@ -413,6 +423,12 @@ function image_with_sha {
   echo "${image_without_tag}@${digest}"
 }
 
+function bundle_image_version {
+  image=${1:?"Provide image"}
+  version=$(skopeo inspect --no-tags=true "docker://${image}" | jq -r '.Labels.version')
+  echo "${version}"
+}
+
 function get_app_version_from_tag() {
   local tag app_version
   tag=${1:?"Provide tag for images"}

olm-catalog/serverless-operator-index/Dockerfile

@@ -10,11 +10,13 @@ COPY olm-catalog/serverless-operator-index/configs /configs
 # Copy policy.json for opm
 COPY olm-catalog/serverless-operator-index/policy.json /etc/containers/policy.json
 
+# Copy pre-generated yaml of bundles of the previous releases
+COPY olm-catalog/serverless-operator-index/index-bundles.yaml /index-bundles.yaml
+
 RUN /bin/opm init serverless-operator --default-channel=stable --output yaml >> /configs/index.yaml
+RUN cat /index-bundles.yaml >> /configs/index.yaml
 RUN /bin/opm render --skip-tls-verify -o yaml \
-quay.io/redhat-user-workloads/ocp-serverless-tenant/serverless-operator-135/serverless-bundle:1.35.0 \
-quay.io/redhat-user-workloads/ocp-serverless-tenant/serverless-operator-136/serverless-bundle:1.36.0 \
-      quay.io/redhat-user-workloads/ocp-serverless-tenant/serverless-operator-137/serverless-bundle@sha256:a081d2ca1c4825e3326e2be37e29deb84186c812d2877771a4e86580a9f5024c >> /configs/index.yaml
+      quay.io/redhat-user-workloads/ocp-serverless-tenant/serverless-operator-137/serverless-bundle@sha256:9d6b6510a8ab953c497948af9977e8b3bf45aa3436767853af2e46af11ed7c68 >> /configs/index.yaml
 
 # The base image is expected to contain
 # /bin/opm (with a serve subcommand) and /bin/grpc_health_probe