Merge branch 'master' into renovate/azuremonitor-containerinsights-ci…

…prod-3.1.x

.pipelines/e2e.yaml

@@ -20,5 +20,6 @@ pr:
       - pkg/agent/datamodel/sig_config*.go # SIG config changes
       - pkg/agent/datamodel/*.json # SIG version changes
       - pkg/agent/testdata/AKSWindows* # Windows test data
+      - parts/linux/cloud-init/artifacts/components.json # centralized components management file
 jobs:
   - template: ./templates/e2e-template.yaml

.pipelines/templates/e2e-windows-template.yaml

@@ -21,6 +21,10 @@ parameters:
 jobs:
 - job: Setup_Test_Cluster
   steps:
+    - bash: |
+        echo "Cleaning up artifacts..."
+        rm -rf $(Build.ArtifactStagingDirectory)/${{ parameters.windowsImage }}${{ parameters.windowsGpuDriverSuffix }}-clusterConfig
+      displayName: "Clean up artifacts"
     - bash: |
         az login --identity
         az account set -s $(AZURE_E2E_SUBSCRIPTION_ID)
@@ -82,6 +86,10 @@ jobs:
     matrix: $[ dependencies.Generate_Matrix.outputs['matrix.scenarioMatrix'] ]
     maxParallel: 3
   steps:
+    - bash: |
+        echo "Cleaning up artifacts..."
+        rm -rf $(Build.ArtifactStagingDirectory)/${{ parameters.windowsImage }}${{ parameters.windowsGpuDriverSuffix }}-$(SCENARIO_NAME)-logs
+      displayName: "Clean up artifacts"
     - download: current
       artifact: ${{ parameters.windowsImage }}${{ parameters.windowsGpuDriverSuffix }}-clusterConfig
     - bash: |

Makefile

@@ -99,7 +99,6 @@ generate: bootstrap
 	./hack/tools/bin/cue export ./schemas/manifest.cue > ./parts/linux/cloud-init/artifacts/manifest.json
 	@echo "#EOF" >> ./parts/linux/cloud-init/artifacts/manifest.json
 	GENERATE_TEST_DATA="true" go test ./pkg/agent...
-	@$(MAKE) validate-prefetch
 	@echo "running validate-shell to make sure generated cse scripts are correct"
 	@$(MAKE) validate-shell
 	@echo "Running shellspec tests to validate shell/bash scripts"

README.md

@@ -136,3 +136,4 @@ Reference: https://docs.opensource.microsoft.com/tools/cg/cgmanifest.html
 Package:
 
 - Calico Windows: https://docs.projectcalico.org/release-notes/
+

e2e/windows/e2e-helper.sh

@@ -85,6 +85,40 @@ upload_linux_file_to_storage_account() {
     fi
 }
 
+check_linux_file_outdated() {
+    declare -l E2E_RESOURCE_GROUP_NAME="$AZURE_E2E_RESOURCE_GROUP_NAME-$WINDOWS_E2E_IMAGE$WINDOWS_GPU_DRIVER_SUFFIX-$K8S_VERSION"
+    E2E_MC_RESOURCE_GROUP_NAME="MC_${E2E_RESOURCE_GROUP_NAME}_${AZURE_E2E_CLUSTER_NAME}_$AZURE_BUILD_LOCATION"
+    MC_VMSS_NAME=$(az vmss list -g $E2E_MC_RESOURCE_GROUP_NAME --query "[?contains(name, 'nodepool')]" -ojson | jq -r '.[0].name')
+    VMSS_INSTANCE_ID="$(az vmss list-instances --name $MC_VMSS_NAME -g $E2E_MC_RESOURCE_GROUP_NAME | jq -r '.[0].instanceId')"
+
+    linuxFileURL="https://${AZURE_E2E_STORAGE_ACCOUNT_NAME}.blob.core.windows.net/${WINDOWS_E2E_STORAGE_CONTAINER}/${MC_VMSS_NAME}-linux-file.zip"
+
+    # download uploaded linux files to old.zip
+    # compress current linux files to new.zip
+    az vmss run-command invoke --command-id RunShellScript \
+        --resource-group $E2E_MC_RESOURCE_GROUP_NAME \
+        --name $MC_VMSS_NAME \
+        --instance-id $VMSS_INSTANCE_ID \
+        --scripts "cat /etc/kubernetes/azure.json > /home/fields.json; cat /etc/kubernetes/certs/apiserver.crt | base64 -w 0 > /home/apiserver.crt; cat /etc/kubernetes/certs/ca.crt | base64 -w 0 > /home/ca.crt; cat /etc/kubernetes/certs/client.key | base64 -w 0 > /home/client.key; cat /var/lib/kubelet/bootstrap-kubeconfig > /home/bootstrap-kubeconfig; cd /home; zip new.zip fields.json apiserver.crt ca.crt client.key bootstrap-kubeconfig; wget https://aka.ms/downloadazcopy-v10-linux; tar -xvf downloadazcopy-v10-linux; cd ./azcopy_*; export AZCOPY_AUTO_LOGIN_TYPE=\"MSI\"; export AZCOPY_MSI_RESOURCE_STRING=\"${AZURE_MSI_RESOURCE_STRING}\"; ./azcopy copy $linuxFileURL /home/old.zip"
+
+    # Use "unzip -d new new.zip" to unzip the compreesed file to a folder
+    # Use "diff -r -q new old" to compare two folder
+    # If files are different in these two folders, the output message will contain "Files xxx and xxx differ"
+    # Example: "Files /home/new/apiserver.crt and /home/old/apiserver.crt differ"
+    compare_message=$(az vmss run-command invoke --command-id RunShellScript \
+        --resource-group $E2E_MC_RESOURCE_GROUP_NAME \
+        --name $MC_VMSS_NAME \
+        --instance-id $VMSS_INSTANCE_ID \
+        --scripts "apt install unzip --yes; unzip -d /home/new /home/new.zip; unzip -d /home/old /home/old.zip; diff -r -q /home/new /home/old")
+
+    if [[ "$compare_message" == *"differ"* ]]; then
+        err "Linux files are outdated."
+        return
+    fi
+
+    log "Linux files are still the latest."
+}
+
 download_linux_file_from_storage_account() {
     local retval
     retval=0

e2e/windows/e2e-starter.sh

@@ -114,10 +114,11 @@ if [ "$create_cluster" == "true" ]; then
         cleanupOutdatedFiles
     fi
 else
-    if [[ "$(check_linux_file_exists_in_storage_account)" == *"Linux file does not exist in storage account."* ]]; then 
+    if [[ "$(check_linux_file_exists_in_storage_account)" == *"Linux file does not exist in storage account."* ]] || [[ "$(check_linux_file_outdated)" == *"Linux files are outdated."* ]]; then 
         upload_linux_file_to_storage_account
     fi
 fi
+
 download_linux_file_from_storage_account
 log "Download of linux file from storage account completed"
 

parts/linux/cloud-init/artifacts/README-COMPONENTS.md

@@ -70,7 +70,7 @@ Please refer to [components.cue](../../../../schemas/components.cue) for the mos
 	  - `renovateTag` must be exactly one line before `latestVersion` and the optional `previousLatestVersion`. `Renovate.json` requires this tag to parse the versions correctly.
 	  - If you add anything other than the 2 types mentioned above, it won't be monitered by the current configurations of `renovate.json`. For example, you might see `"renovateTag": "<DO_NOT_UPDATE>"` which is actually equivalent to not having any `renovateTag`. Placing `"<DO_NOT_UPDATE>"` here is simply for human readability, but we still recommend including it for consistency and readability.
 	- `latestVersion` and `previousLatestVersion`: to keep the last 2 patch versions in the components.json as well as VHD and keep them auto-updated by Renovate, we will put the latest version in `latestVersion` and the previous latest version `previousLatestVersion`.
-  - `containerImagePrefetch` defines the prefetch optimization for the particular container image, if any. Each `ContainerImagePrefetchOptimizations` object must define a prefetch optimization _at least_ for the `latestVersion`, while optionally defining one of the `previousLatestVersion`. At the end of the day, a prefetch optimization is parameterized by an array of file paths pointing to binaries (relative to the FS of the container image, starting with `/`) to be prefetched during image builder optimization.
+  - `containerImagePrefetch` defines the prefetch optimization for the particular container image, if any. Each `ContainerImagePrefetchOptimizations` object must define a prefetch optimization _at least_ for the `latestVersion`, while optionally defining one of the `previousLatestVersion`. At the end of the day, a prefetch optimization is parameterized by an array of file paths pointing to binaries (relative to the FS of the container image, starting with `/`) to be prefetched during image builder optimization. NOTE: if ever updating prefetch optimizations for container images, please run `make generate` within vhdbuilder/prefetch to update the corresponding test data and re-run prefetch script generation tests.
 
 ### Packages
 `Packages` is a list of `Package` where a `package` has the following scehma:

parts/linux/cloud-init/artifacts/components.json

@@ -65,8 +65,8 @@
         },
         {
           "renovateTag": "registry=https://mcr.microsoft.com, name=containernetworking/azure-cni",
-          "latestVersion": "v1.6.7",
-          "previousLatestVersion": "v1.6.5",
+          "latestVersion": "v1.6.10",
+          "previousLatestVersion": "v1.6.7",
           "containerImagePrefetch": {
             "latestVersion": {
               "binaries": [
@@ -116,8 +116,8 @@
         },
         {
           "renovateTag": "registry=https://mcr.microsoft.com, name=containernetworking/azure-cns",
-          "latestVersion": "v1.6.7",
-          "previousLatestVersion": "v1.6.5",
+          "latestVersion": "v1.6.10",
+          "previousLatestVersion": "v1.6.7",
           "containerImagePrefetch": {
             "latestVersion": {
               "binaries": [
@@ -616,15 +616,15 @@
             "versionsV2": [
               {
                 "renovateTag": "name=moby-containerd, os=ubuntu, release=22.04",
-                "latestVersion": "1.7.20"
+                "latestVersion": "1.7.22-ubuntu22.04u1"
               }
             ]
           },
           "r2004": {
             "versionsV2": [
               {
                 "renovateTag": "name=moby-containerd, os=ubuntu, release=20.04",
-                "latestVersion": "1.7.20"
+                "latestVersion": "1.7.22-ubuntu20.04u1"
               }
             ]
           },
@@ -714,8 +714,8 @@
               },
               {
                 "renovateTag": "<DO_NOT_UPDATE>",
-                "latestVersion": "1.6.7",
-                "previousLatestVersion": "1.6.3"
+                "latestVersion": "1.6.10",
+                "previousLatestVersion": "1.6.7"
               }
             ],
             "downloadURL": "https://acs-mirror.azureedge.net/azure-cni/v${version}/binaries/azure-vnet-cni-linux-${CPU_ARCH}-v${version}.tgz"
@@ -797,6 +797,48 @@
           }
         }
       }
+    },
+    {
+      "name": "containerd-wasm-shims",
+      "downloadLocation": "/usr/local/bin",
+      "downloadURIs": {
+        "default": {
+          "current": {
+            "versionsV2": [
+              {
+                "renovateTag": "<DO_NOT_UPDATE>",
+                "latestVersion": "0.3.0"
+              },
+              {
+                "renovateTag": "<DO_NOT_UPDATE>",
+                "latestVersion": "0.5.1"
+              },
+              {
+                "renovateTag": "<DO_NOT_UPDATE>",
+                "latestVersion": "0.8.0"
+              }
+            ],
+            "downloadURL": "https://acs-mirror.azureedge.net/containerd-wasm-shims/v${version}/linux/${CPU_ARCH}"
+          }
+        }
+      }
+    },
+    {
+      "name": "spinkube",
+      "downloadLocation": "/usr/local/bin",
+      "downloadURIs": {
+        "default": {
+          "current": {
+            "versionsV2": [
+              {
+                "renovateTag": "<DO_NOT_UPDATE>",
+                "latestVersion": "0.15.1"
+              }
+            ],
+            "downloadURL": "https://acs-mirror.azureedge.net/spinkube/v${version}/linux/${CPU_ARCH}"
+          }
+        }
+      }
     }
   ]
-}
+}