From 18f8d0ca93d2f9489545e5fcc3084903a175aa3b Mon Sep 17 00:00:00 2001 From: Jan Lohage Date: Wed, 13 Mar 2024 15:58:45 +0100 Subject: [PATCH 1/6] Allow selection of cilium and calico CNIs in yake-local Signed-off-by: Jan Lohage --- .github/workflows/yake-install.yaml | 4 ++ hack/ci/yake-local/kind-config-no-cni.yaml | 12 +++++ hack/ci/yake-local/work.sh | 55 +++++++++++++++++++++- 3 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 hack/ci/yake-local/kind-config-no-cni.yaml diff --git a/.github/workflows/yake-install.yaml b/.github/workflows/yake-install.yaml index e1890cf40b4..b743fc3f156 100644 --- a/.github/workflows/yake-install.yaml +++ b/.github/workflows/yake-install.yaml @@ -10,6 +10,9 @@ jobs: install: strategy: matrix: + cni: + - calico + - cilium version: - v1.26.14 - v1.27.11 @@ -26,5 +29,6 @@ jobs: - name: install env: K8S_VERSION: ${{ matrix.version }} + CNI: ${{ matrix.cni }} working-directory: hack/ci/yake-local run: ./work.sh diff --git a/hack/ci/yake-local/kind-config-no-cni.yaml b/hack/ci/yake-local/kind-config-no-cni.yaml new file mode 100644 index 00000000000..7329632020f --- /dev/null +++ b/hack/ci/yake-local/kind-config-no-cni.yaml @@ -0,0 +1,12 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: + - role: control-plane + - role: worker + - role: worker + - role: worker +networking: + ipFamily: ipv4 + disableDefaultCNI: true + podSubnet: 10.1.0.0/16 + serviceSubnet: 10.2.0.0/16 diff --git a/hack/ci/yake-local/work.sh b/hack/ci/yake-local/work.sh index ade844be6e3..32604f88a45 100755 --- a/hack/ci/yake-local/work.sh +++ b/hack/ci/yake-local/work.sh @@ -12,6 +12,24 @@ CLUSTERNAME="yake-local" VGARDEN_KUBECONFIG="/tmp/$CLUSTERNAME-apiserver.yaml" K8S_VERSION="${K8S_VERSION:-v1.26.6}" +CNI="${CNI:-default}" + +if [[ $CNI == "default" ]]; then + kindConfig="kind-config.yaml" + useCilium="" + useCalico="" +elif [[ $CNI == "cilium" ]]; then + kindConfig="kind-config-no-cni.yaml" + useCilium="true" + useCalico="" +elif [[ $CNI == "calico" ]]; then + kindConfig="kind-config-no-cni.yaml" + useCilium="" + useCalico="true" +else + echo "unknown CNI '$CNI', use 'default', 'calico' or 'cilium'" + exit 1 +fi # from gardener/gardener hack/kind-up.sh # setup_kind_network is similar to kind's network creation logic, ref https://github.com/kubernetes-sigs/kind/blob/23d2ac0e9c41028fa252dd1340411d70d46e2fd4/pkg/cluster/internal/providers/docker/network.go#L50 @@ -47,12 +65,44 @@ _setup_kind_network() { _create_cluster () { # If export kubeconfig fails, the cluster does not yet exist and we need to create it - $KIND export kubeconfig -n $CLUSTERNAME > /dev/null 2>&1 || $KIND create cluster --config kind-config.yaml --name $CLUSTERNAME --image="kindest/node:$K8S_VERSION" + $KIND export kubeconfig -n $CLUSTERNAME > /dev/null 2>&1 || $KIND create cluster --config "$kindConfig" --name $CLUSTERNAME --image="kindest/node:$K8S_VERSION" $KIND export kubeconfig -n $CLUSTERNAME $KUBECTL config set-context --current --namespace=default } +_create_cni () { + if [[ $useCilium == "true" ]]; then + _create_cilium + elif [[ $useCalico == "true" ]]; then + _create_calico + fi +} + +_create_cilium () { + local VERSION="1.15.1" + $HELM repo add cilium https://helm.cilium.io/ + $HELM repo update cilium + + docker pull "quay.io/cilium/cilium:v$VERSION" + $KIND load docker-image "quay.io/cilium/cilium:v$VERSION" -n $CLUSTERNAME + + $HELM install cilium cilium/cilium --version "$VERSION" \ + --namespace kube-system \ + --set image.pullPolicy=IfNotPresent \ + --set ipam.mode=kubernetes +} + +_create_calico () { + VERSION="v3.27.2" + $KUBECTL apply -f https://raw.githubusercontent.com/projectcalico/calico/$VERSION/manifests/calico.yaml +} + +_wait_for_nodes_ready () { + kubectl wait --for=condition=ready nodes --all --timeout=5m +} + _create_loadbalancer () { + local VERSION= $KUBECTL apply -f https://raw.githubusercontent.com/metallb/metallb/v0.13.12/config/manifests/metallb-native.yaml $KUBECTL wait --namespace metallb-system --for=condition=ready pod --all --timeout=90s @@ -280,9 +330,10 @@ install_kind install_kubectl install_yq install_envsubst - _setup_kind_network _create_cluster +_create_cni +_wait_for_nodes_ready _create_loadbalancer _create_local_git _create_step_ca From 524027f3395b72bdc544dbc91ba6718ec1b6a7ab Mon Sep 17 00:00:00 2001 From: Jan Lohage Date: Wed, 13 Mar 2024 16:22:22 +0100 Subject: [PATCH 2/6] use local kubectl Signed-off-by: Jan Lohage --- hack/ci/yake-local/work.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/ci/yake-local/work.sh b/hack/ci/yake-local/work.sh index 32604f88a45..51d3697d060 100755 --- a/hack/ci/yake-local/work.sh +++ b/hack/ci/yake-local/work.sh @@ -98,7 +98,7 @@ _create_calico () { } _wait_for_nodes_ready () { - kubectl wait --for=condition=ready nodes --all --timeout=5m + $KUBECTL wait --for=condition=ready nodes --all --timeout=5m } _create_loadbalancer () { From 8733a241bb45c331b349957092c8d9d484752637 Mon Sep 17 00:00:00 2001 From: Jan Lohage Date: Thu, 14 Mar 2024 09:05:54 +0100 Subject: [PATCH 3/6] don't fail fast for now Signed-off-by: Jan Lohage --- .github/workflows/yake-install.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/yake-install.yaml b/.github/workflows/yake-install.yaml index b743fc3f156..9e399c1f74f 100644 --- a/.github/workflows/yake-install.yaml +++ b/.github/workflows/yake-install.yaml @@ -9,6 +9,7 @@ on: jobs: install: strategy: + fail-fast: false matrix: cni: - calico From ecde9dc3359ef8dbbb675eb059b227c23f606d81 Mon Sep 17 00:00:00 2001 From: Jan Lohage Date: Thu, 14 Mar 2024 16:57:09 +0100 Subject: [PATCH 4/6] add some log messages for better orientation Signed-off-by: Jan Lohage --- hack/ci/yake-local/work.sh | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/hack/ci/yake-local/work.sh b/hack/ci/yake-local/work.sh index 51d3697d060..388202a4ed4 100755 --- a/hack/ci/yake-local/work.sh +++ b/hack/ci/yake-local/work.sh @@ -31,10 +31,15 @@ else exit 1 fi +_print_heading() { + echo -e "\033[34m$1\033[0m" +} + # from gardener/gardener hack/kind-up.sh # setup_kind_network is similar to kind's network creation logic, ref https://github.com/kubernetes-sigs/kind/blob/23d2ac0e9c41028fa252dd1340411d70d46e2fd4/pkg/cluster/internal/providers/docker/network.go#L50 # In addition to kind's logic, we ensure stable CIDRs that we can rely on in our local setup manifests and code. _setup_kind_network() { + _print_heading "Setup Kind Network" # check if network already exists local existing_network_id existing_network_id="$(docker network list --filter=name=^kind$ --format='{{.ID}}')" @@ -64,6 +69,7 @@ _setup_kind_network() { } _create_cluster () { + _print_heading "Create Cluster" # If export kubeconfig fails, the cluster does not yet exist and we need to create it $KIND export kubeconfig -n $CLUSTERNAME > /dev/null 2>&1 || $KIND create cluster --config "$kindConfig" --name $CLUSTERNAME --image="kindest/node:$K8S_VERSION" $KIND export kubeconfig -n $CLUSTERNAME @@ -71,6 +77,7 @@ _create_cluster () { } _create_cni () { + _print_heading "Create Cni" if [[ $useCilium == "true" ]]; then _create_cilium elif [[ $useCalico == "true" ]]; then @@ -79,6 +86,7 @@ _create_cni () { } _create_cilium () { + _print_heading "Create Cilium" local VERSION="1.15.1" $HELM repo add cilium https://helm.cilium.io/ $HELM repo update cilium @@ -93,15 +101,19 @@ _create_cilium () { } _create_calico () { + _print_heading "Create Calico" VERSION="v3.27.2" $KUBECTL apply -f https://raw.githubusercontent.com/projectcalico/calico/$VERSION/manifests/calico.yaml } _wait_for_nodes_ready () { - $KUBECTL wait --for=condition=ready nodes --all --timeout=5m + _print_heading "Wait For Nodes Ready" + + $KUBECTL wait --for=condition=ready nodes --all --timeout=15m } _create_loadbalancer () { + _print_heading "Create Loadbalancer" local VERSION= $KUBECTL apply -f https://raw.githubusercontent.com/metallb/metallb/v0.13.12/config/manifests/metallb-native.yaml $KUBECTL wait --namespace metallb-system --for=condition=ready pod --all --timeout=90s @@ -125,6 +137,7 @@ EOF } _create_local_git () { + _print_heading "Create Local Git" $KUBECTL apply -f git-server.yaml printf ">>> waiting for git server " @@ -141,6 +154,7 @@ _create_local_git () { } _create_step_ca () { + _print_heading "Create Step Ca" ############# step ca for acme server in kind cluster ################# $HELM repo add smallstep https://smallstep.github.io/helm-charts/ @@ -156,6 +170,7 @@ _create_step_ca () { } _create_local_dns () { + _print_heading "Create Local Dns" ############# knot ################# $KUBECTL apply -f knot.yaml @@ -169,6 +184,7 @@ _create_local_dns () { } _create_flux () { + _print_heading "Create Flux" ############# flux ################# $KUBECTL apply -f ../../../flux-system/gotk-components.yaml @@ -238,6 +254,7 @@ EOF } _patch_ccm() { + _print_heading "Patch Ccm" printf ">>> waiting for deployment cert-controller-manager " until $KUBECTL get deployment cert-controller-manager -n garden >/dev/null 2>&1; do printf . @@ -249,6 +266,7 @@ _patch_ccm() { } _ensure_hosts() { + _print_heading "Ensure Hosts" printf ">>> waiting for hr gardener-runtime " until $KUBECTL get hr gardener-runtime -n flux-system >/dev/null 2>&1; do printf . @@ -283,6 +301,7 @@ _ensure_hosts() { } _create_rbac () { + _print_heading "Create Rbac" $KUBECTL get secrets -n garden garden-kubeconfig-for-admin -o go-template='{{.data.kubeconfig | base64decode }}' > "$VGARDEN_KUBECONFIG" KUBECONFIG="$VGARDEN_KUBECONFIG" $KUBECTL apply -f garden-content/cloudprofile-local.yaml @@ -313,6 +332,7 @@ EOF } _wait_for_initial_seed_ready () { + _print_heading "Wait For Initial Seed Ready" printf ">>> waiting for initial seed to become ready " until providerLocalSAName=$(KUBECONFIG="$VGARDEN_KUBECONFIG" $KUBECTL get seed initial-seed); do From 4227375e9d127a9844a505aefc80f6e183e1b14b Mon Sep 17 00:00:00 2001 From: Jan Lohage Date: Thu, 14 Mar 2024 16:57:35 +0100 Subject: [PATCH 5/6] adjust timeouts Signed-off-by: Jan Lohage --- hack/ci/yake-local/work.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hack/ci/yake-local/work.sh b/hack/ci/yake-local/work.sh index 388202a4ed4..a4923fcfc0e 100755 --- a/hack/ci/yake-local/work.sh +++ b/hack/ci/yake-local/work.sh @@ -116,7 +116,7 @@ _create_loadbalancer () { _print_heading "Create Loadbalancer" local VERSION= $KUBECTL apply -f https://raw.githubusercontent.com/metallb/metallb/v0.13.12/config/manifests/metallb-native.yaml - $KUBECTL wait --namespace metallb-system --for=condition=ready pod --all --timeout=90s + $KUBECTL wait --namespace metallb-system --for=condition=ready pod --all --timeout=3m cat <>> waiting for git server " - $KUBECTL wait --namespace default --for=condition=ready pod --selector=app=git-server --timeout=90s + $KUBECTL wait --namespace default --for=condition=ready pod --selector=app=git-server --timeout=3m gitUrl="http://$($KUBECTL get svc git-server -o jsonpath="{.status.loadBalancer.ingress[0].ip}")/repository.git" git remote add local "$gitUrl" 2>/dev/null || git remote set-url local "$gitUrl" until git fetch local; do From 9e7f660ebf43203c5fca58bdd91c4cb898b0e220 Mon Sep 17 00:00:00 2001 From: Jan Lohage Date: Thu, 14 Mar 2024 16:57:55 +0100 Subject: [PATCH 6/6] set policyCIDRMatchMode=nodes for cilium Signed-off-by: Jan Lohage --- hack/ci/yake-local/work.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hack/ci/yake-local/work.sh b/hack/ci/yake-local/work.sh index a4923fcfc0e..7cf9b905f88 100755 --- a/hack/ci/yake-local/work.sh +++ b/hack/ci/yake-local/work.sh @@ -94,10 +94,11 @@ _create_cilium () { docker pull "quay.io/cilium/cilium:v$VERSION" $KIND load docker-image "quay.io/cilium/cilium:v$VERSION" -n $CLUSTERNAME - $HELM install cilium cilium/cilium --version "$VERSION" \ + $HELM upgrade -i cilium cilium/cilium --version "$VERSION" \ --namespace kube-system \ --set image.pullPolicy=IfNotPresent \ - --set ipam.mode=kubernetes + --set ipam.mode=kubernetes \ + --set policyCIDRMatchMode=nodes } _create_calico () {