Skip to content

Commit e3caae8

Browse files
mohitrajainmohitrajain
committed
fix WPB-27900: fix logic for helm_image_tree.json file
1 parent fc7f4b7 commit e3caae8

3 files changed

Lines changed: 105 additions & 22 deletions

File tree

‎nix/scripts/list-helm-containers.sh‎

Lines changed: 93 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,17 @@
55
# those.
66
# In cases where no container image tag has been specified, it'll use `latest`.
77
# The list is sorted and deduplicated, then printed to stdout.
8-
set -x -eou pipefail
8+
set -euo pipefail
99

1010
VALUES_DIR=""
1111
HELM_IMAGE_TREE_FILE=""
12+
VALUES_TYPE=""
13+
14+
# Extract images using yq-go (v4+) syntax
15+
# Note: This requires yq-go to be in PATH (see default.nix)
16+
extract_images() {
17+
yq eval '.. | select(has("image")) | .image' "$1" 2>/dev/null || true
18+
}
1219

1320
# Parse the arguments
1421
for arg in "$@"
@@ -20,19 +27,37 @@ do
2027
HELM_IMAGE_TREE_FILE=*)
2128
HELM_IMAGE_TREE_FILE="${arg#*=}"
2229
;;
30+
VALUES_TYPE=*)
31+
VALUES_TYPE="${arg#*=}"
32+
;;
2333
*)
2434
echo "Unknown argument: $arg" >&2
2535
exit 1
2636
;;
2737
esac
2838
done
2939

30-
if [[ -z "$VALUES_DIR" || -z "$HELM_IMAGE_TREE_FILE" ]]; then
31-
echo "Error: Both VALUES_DIR and HELM_IMAGE_TREE_FILE must be provided." >&2
32-
echo "Usage: $0 VALUES_DIR=<path> HELM_IMAGE_TREE_FILE=<file>" >&2
40+
if [[ -z "$VALUES_DIR" || -z "$HELM_IMAGE_TREE_FILE" || -z "$VALUES_TYPE" ]]; then
41+
echo "Error: VALUES_DIR, HELM_IMAGE_TREE_FILE and VALUES_TYPE must be provided." >&2
42+
echo "Usage: $0 VALUES_DIR=<path> HELM_IMAGE_TREE_FILE=<file> [VALUES_TYPE=<type>]" >&2
3343
exit 1
3444
fi
3545

46+
# create a dependency tree between helm chart and images
47+
append_chart_entry() {
48+
local chart=$1
49+
local images=$2
50+
local json_file=$3
51+
52+
if [ ! -s "$json_file" ]; then
53+
echo '[]' > "$json_file"
54+
fi
55+
56+
existing_content=$(jq '.' "$json_file")
57+
new_entry=$(jq -n --arg chart "$chart" --argjson images "$images" '{"chart": $chart, "images": $images}')
58+
updated_content=$(echo "$existing_content" | jq --argjson new_entry "$new_entry" '. += [$new_entry]')
59+
echo "$updated_content" | jq '.' > "$json_file"
60+
}
3661

3762
# Some of these images don't contain a "latest" tag. We don't to download /ALL/
3863
# of them, but only :latest in that case - it's bad enough there's no proper
@@ -47,27 +72,80 @@ function optionally_complain() {
4772
echo "$image"
4873
else
4974
echo "Container $image without a tag found or pin found. Aborting! Fix this chart. not compatible with offline. Components need explicit tags for that" >&2
75+
exit 1
5076
fi
5177
done
5278
}
5379

5480
images=""
5581
# For each helm chart passed in from stdin, use the example values to
5682
# render the charts, and assemble the list of images this would fetch.
83+
chart_count=0
5784
while IFS= read -r chart; do
58-
echo "Running helm template on chart ${chart}…" >&2
59-
current_images=$(helm template --debug "${chart}" \
60-
--set federate.dtls.tls.key=emptyString \
61-
--set federate.dtls.tls.crt=emptyString \
62-
$( [[ -f "${VALUES_DIR}"/$(basename "${chart}")/prod-values.example.yaml ]] && echo "-f ${VALUES_DIR}/$(basename "${chart}")/prod-values.example.yaml" ) \
63-
$( [[ -f "${VALUES_DIR}"/$(basename "${chart}")/prod-secrets.example.yaml ]] && echo "-f ${VALUES_DIR}/$(basename "${chart}")/prod-secrets.example.yaml" ) \
64-
| yq -r '..|.image? | select(.)' | optionally_complain | sort -u)
85+
chart_count=$((chart_count + 1))
86+
echo "[$chart_count] Running helm template on chart ${chart}…" >&2
87+
set +e # Temporarily disable exit on error
88+
# Determine values file to use (prod first, then demo as fallback)
89+
values_file=""
90+
if [[ -f "${VALUES_DIR}"/$(basename "${chart}")/"${VALUES_TYPE}"-values.example.yaml ]]; then
91+
values_file="${VALUES_DIR}/$(basename "${chart}")/${VALUES_TYPE}-values.example.yaml"
92+
elif [[ -f "${VALUES_DIR}"/$(basename "${chart}")/demo-values.example.yaml ]]; then
93+
values_file="${VALUES_DIR}/$(basename "${chart}")/demo-values.example.yaml"
94+
echo "Using demo values for $(basename $chart) (no ${VALUES_TYPE} values found)" >&2
95+
fi
96+
97+
# Determine secrets file to use
98+
secrets_file=""
99+
if [[ -f "${VALUES_DIR}"/$(basename "${chart}")/"${VALUES_TYPE}"-secrets.example.yaml ]]; then
100+
secrets_file="${VALUES_DIR}/$(basename "${chart}")/${VALUES_TYPE}-secrets.example.yaml"
101+
elif [[ -f "${VALUES_DIR}"/$(basename "${chart}")/demo-secrets.example.yaml ]]; then
102+
secrets_file="${VALUES_DIR}/$(basename "${chart}")/demo-secrets.example.yaml"
103+
fi
104+
105+
# Save helm output to temp file to check exit code before parsing
106+
# This prevents yq from attempting to parse helm error messages
107+
temp_helm_output=$(mktemp)
108+
helm template "${chart}" \
109+
$( [[ -n "$values_file" ]] && echo "-f $values_file" ) \
110+
$( [[ -n "$secrets_file" ]] && echo "-f $secrets_file" ) \
111+
> "$temp_helm_output" 2>&1
112+
113+
helm_exit_code=$?
114+
115+
# Extract images using version-appropriate yq syntax
116+
if [[ $helm_exit_code -eq 0 ]]; then
117+
raw_images=$(extract_images "$temp_helm_output" | grep -v "^null$" | grep -v "^---$" | grep -v "^$" || true)
118+
else
119+
raw_images=""
120+
fi
121+
122+
set -e # Re-enable exit on error
123+
124+
if [[ $helm_exit_code -ne 0 ]]; then
125+
echo "ERROR: Failed to process chart $(basename $chart)" >&2
126+
echo "Chart path: $chart" >&2
127+
echo "Values file: ${values_file:-none}" >&2
128+
echo "Secrets file: ${secrets_file:-none}" >&2
129+
echo "Helm error output:" >&2
130+
cat "$temp_helm_output" >&2
131+
echo "Try running: helm template $chart $([ -n "$values_file" ] && echo "-f $values_file") $([ -n "$secrets_file" ] && echo "-f $secrets_file")" >&2
132+
raw_images=""
133+
fi
134+
135+
rm -f "$temp_helm_output"
136+
137+
# Process extracted images
138+
if [[ -n "$raw_images" ]]; then
139+
current_images=$(echo "$raw_images" | grep -v "^$" | optionally_complain | sort -u)
140+
else
141+
current_images=""
142+
fi
65143

66144
images+="$current_images\n"
67145
if [[ -n "$current_images" ]]; then
68-
basename "${chart}" >> "${HELM_IMAGE_TREE_FILE}"
69-
echo -e "$current_images\n" >> "${HELM_IMAGE_TREE_FILE}"
70-
#echo -e "\n" >> "${HELM_IMAGE_TREE_FILE}"
146+
current_images=$(echo "$current_images" | awk NF)
147+
image_array=$(jq -Rn --arg images "$current_images" '$images | split("\n")')
148+
append_chart_entry "$(basename $chart)" "$image_array" "${HELM_IMAGE_TREE_FILE}"
71149
fi
72150
done
73-
echo -e "$images" | grep . | sort -u
151+
echo -e "$images" | grep . | sort -u || true

‎offline/min-build/build_oci.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,6 @@ tar czf "$OUTPUT_TAR" "${ITEMS_TO_ARCHIVE[@]}"
9696
echo "Dump of versions/helm_image_tree.json"
9797
cat "${OUTPUT_DIR}/versions/helm_image_tree.json"
9898

99-
echo "Dump of mainfest.yaml used"
100-
cat "${OUTPUT_DIR}/mainfest.yaml"
99+
echo "Dump of manifest.yaml used"
100+
cat "${OUTPUT_DIR}/manifest.yaml"
101101

‎offline/tasks/process_charts.sh‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,19 @@ OUTPUT_DIR=""
55
# Default exclude list
66
IMAGE_EXCLUDE_LIST=""
77

8+
# Default values type will expect to use prod values
9+
VALUES_TYPE="prod"
10+
811
# Parse the arguments
912
for arg in "$@"
1013
do
1114
case $arg in
1215
OUTPUT_DIR=*)
1316
OUTPUT_DIR="${arg#*=}"
1417
;;
18+
VALUES_TYPE=*)
19+
VALUES_TYPE="${arg#*=}"
20+
;;
1521
IMAGE_EXCLUDE_LIST=*)
1622
IMAGE_EXCLUDE_LIST="${arg#*=}"
1723
;;
@@ -24,13 +30,13 @@ done
2430

2531
# Check if OUTPUT_DIR is set
2632
if [[ -z "$OUTPUT_DIR" ]]; then
27-
echo "usage: $0 OUTPUT_DIR=\"output-dir\" [IMAGE_EXCLUDE_LIST=\"image1\|image2...\"]" >&2
33+
echo "usage: $0 OUTPUT_DIR=\"output-dir\" [IMAGE_EXCLUDE_LIST=\"image1\|image2...\"] [VALUES_TYPE=\"prod\"]" >&2
2834
exit 1
2935
fi
3036

31-
echo "Processing Helm charts in ${OUTPUT_DIR}"
37+
echo "Processing Helm charts in ${OUTPUT_DIR} with VALUES_TYPE=${VALUES_TYPE}"
3238

33-
HELM_IMAGE_TREE_FILE="${OUTPUT_DIR}/versions/helm_image_tree.txt"
39+
HELM_IMAGE_TREE_FILE="${OUTPUT_DIR}/versions/helm_image_tree.json"
3440
touch "${HELM_IMAGE_TREE_FILE}"
3541

3642
# Check if IMAGE_EXCLUDE_LIST is set, otherwise use a default pattern that matches nothing
@@ -42,13 +48,12 @@ echo "Excluding images matching the pattern: $EXCLUDE_PATTERN"
4248
# containers (e.g. `quay.io_wire_galley-integration_4.22.0`.)
4349
for chartPath in "${OUTPUT_DIR}"/charts/*; do
4450
echo "$chartPath"
45-
done | list-helm-containers VALUES_DIR="${OUTPUT_DIR}"/values HELM_IMAGE_TREE_FILE="$HELM_IMAGE_TREE_FILE" | grep -v "\-integration:" > "${OUTPUT_DIR}"/images
51+
done | list-helm-containers VALUES_DIR="${OUTPUT_DIR}"/values HELM_IMAGE_TREE_FILE="$HELM_IMAGE_TREE_FILE" VALUES_TYPE="$VALUES_TYPE" | grep -v "\-integration:" > "${OUTPUT_DIR}"/images
4652

4753
# Omit integration test
4854
# containers (e.g. `quay.io_wire_galley-integration_4.22.0`.)
4955
sed -i '/-integration/d' "${HELM_IMAGE_TREE_FILE}"
5056

51-
5257
# Replace docker.io/bitnami with docker.io/bitnamilegacy and log updated images
5358
# https://github.com/bitnami/charts/issues/35164
5459
echo "Replacing bitnami with bitnamilegacy..."

0 commit comments

Comments
 (0)