Run e2e tests on AL2023 with SELinux (#219)

By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice. --------- Signed-off-by: Burak Varlı <[email protected]> Co-authored-by: Jiayi Nie <[email protected]>
awslabs · Feb 21, 2025 · e519c4e · e519c4e
1 parent 5b2e4b4
commit e519c4e
Show file tree

Hide file tree

Showing 7 changed files with 65 additions and 4 deletions.
diff --git a/.github/matrix.yaml b/.github/matrix.yaml
@@ -26,6 +26,18 @@ matrix:
       arch: "arm"
       family: "Ubuntu2204"
       kubernetes-version: "1.32.1"
+    # Since we only enable enforcing mode for SELinux in AL2023, it's easier to list it in "include"
+    # field rather than trying to exclude all other variants.
+    - cluster-type: "kops"
+      arch: "arm"
+      family: "AmazonLinux2023"
+      kubernetes-version: "1.28.13"
+      selinux-mode: "enforcing"
+    - cluster-type: "eksctl"
+      arch: "x86"
+      family: "AmazonLinux2023"
+      kubernetes-version: "1.32.1"
+      selinux-mode: "enforcing"
   exclude:
     - cluster-type: "kops"
       family: "Bottlerocket"

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
@@ -80,6 +80,7 @@ jobs:
       # and in order to get latest supported version by envtest we convert it to 1.28.
       K8S_VERSION: "${{ matrix.kubernetes-version }}"
       ENVTEST_K8S_VERSION: "${K8S_VERSION%.*}"
+      SELINUX_MODE: "${{ matrix.selinux-mode }}"
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -119,12 +120,15 @@ jobs:
           tests/e2e-kubernetes/scripts/run.sh
 
       # Test SystemdMounter
+      # SystemdMounter tests are skipped if SELinux is enabled as SystemdMounter won't work with SELinux.
       - name: Install the driver (SystemdMounter)
+        if: matrix.selinux-mode != 'enforcing'
         env:
           ACTION: "install_driver"
         run: |
           tests/e2e-kubernetes/scripts/run.sh
       - name: Run E2E Tests (SystemdMounter)
+        if: matrix.selinux-mode != 'enforcing'
         env:
           ACTION: "run_tests"
         run: |
@@ -156,13 +160,13 @@ jobs:
           tests/e2e-kubernetes/scripts/format_benchmark_data.py ${{ env.BENCHMARK_ARTIFACTS_FOLDER }}/benchmark-data.json ${{ env.BENCHMARK_ARTIFACTS_FOLDER }}/quicksight-data.json
           aws s3 cp ${{ env.BENCHMARK_ARTIFACTS_FOLDER }} ${{ env.BENCHMARK_BUCKET }} --recursive
       - name: Post e2e cleanup (SystemdMounter)
-        if: always()
+        if: matrix.selinux-mode != 'enforcing' && always()
         env:
           ACTION: "e2e_cleanup"
         run: |
           tests/e2e-kubernetes/scripts/run.sh
       - name: Uninstall the driver (SystemdMounter)
-        if: always()
+        if: matrix.selinux-mode != 'enforcing' && always()
         env:
           ACTION: "uninstall_driver"
         run: |

diff --git a/tests/e2e-kubernetes/scripts/eksctl-patch-selinux-enforcing.json b/tests/e2e-kubernetes/scripts/eksctl-patch-selinux-enforcing.json
@@ -0,0 +1,12 @@
+[
+    {
+      "op": "add",
+      "path": "/managedNodeGroups/0/preBootstrapCommands",
+      "value": [
+        "dnf install -y container-selinux",
+        "setenforce 1",
+        "sed -i \"s/SELINUX=permissive/SELINUX=enforcing/\" /etc/selinux/config",
+        "reboot"
+      ]
+    }
+  ]
diff --git a/tests/e2e-kubernetes/scripts/eksctl.sh b/tests/e2e-kubernetes/scripts/eksctl.sh
@@ -25,6 +25,7 @@ function eksctl_create_cluster() {
   NODE_TYPE=${10}
   AMI_FAMILY=${11}
   K8S_VERSION=${12}
+  EKSCTL_PATCH_SELINUX_ENFORCING_FILE=${13}
 
   eksctl_delete_cluster "$BIN" "$CLUSTER_NAME" "$REGION"
 
@@ -42,6 +43,12 @@ function eksctl_create_cluster() {
   CLUSTER_FILE_TMP="${CLUSTER_FILE}.tmp"
   ${KUBECTL_BIN} patch -f $CLUSTER_FILE --local --type json --patch "$(cat $EKSCTL_PATCH_FILE)" -o yaml > $CLUSTER_FILE_TMP
   mv $CLUSTER_FILE_TMP $CLUSTER_FILE
+
+  if [ -n "$EKSCTL_PATCH_SELINUX_ENFORCING_FILE" ]; then
+    ${KUBECTL_BIN} patch -f $CLUSTER_FILE --local --type json --patch "$(cat $EKSCTL_PATCH_SELINUX_ENFORCING_FILE)" -o yaml > $CLUSTER_FILE_TMP
+    mv $CLUSTER_FILE_TMP $CLUSTER_FILE
+  fi
+
   ${BIN} create cluster -f "${CLUSTER_FILE}" --kubeconfig "${KUBECONFIG}"
 
   if [ -n "$CI_ROLE_ARN" ]; then

diff --git a/tests/e2e-kubernetes/scripts/kops-patch-node-selinux-enforcing.yaml b/tests/e2e-kubernetes/scripts/kops-patch-node-selinux-enforcing.yaml
@@ -0,0 +1,8 @@
+spec:
+  additionalUserData:
+    - name: selinux-settings.txt
+      type: text/cloud-config
+      content: |
+        #cloud-config
+        selinux:
+            mode: enforcing
diff --git a/tests/e2e-kubernetes/scripts/kops.sh b/tests/e2e-kubernetes/scripts/kops.sh
@@ -33,6 +33,7 @@ function kops_create_cluster() {
   KOPS_PATCH_NODE_FILE=${11}
   KOPS_STATE_FILE=${12}
   SSH_KEY=${13}
+  KOPS_PATCH_NODE_SELINUX_ENFORCING_FILE=${14}
 
   if kops_cluster_exists "${CLUSTER_NAME}" "${BIN}" "${KOPS_STATE_FILE}"; then
     kops_delete_cluster "$BIN" "$CLUSTER_NAME" "$KOPS_STATE_FILE"
@@ -58,6 +59,10 @@ function kops_create_cluster() {
   kops_patch_cluster_file "$CLUSTER_FILE" "$KOPS_PATCH_FILE" "Cluster" ""
   kops_patch_cluster_file "$CLUSTER_FILE" "$KOPS_PATCH_NODE_FILE" "InstanceGroup" "Node"
 
+  if [ -n "$KOPS_PATCH_NODE_SELINUX_ENFORCING_FILE" ]; then
+    kops_patch_cluster_file "$CLUSTER_FILE" "$KOPS_PATCH_NODE_SELINUX_ENFORCING_FILE" "InstanceGroup" "Node"
+  fi
+
   ${BIN} create --state "${KOPS_STATE_FILE}" -f "${CLUSTER_FILE}"
   ${BIN} update cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --yes
   ${BIN} export kubecfg --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --admin --kubeconfig "${KUBECONFIG}"

diff --git a/tests/e2e-kubernetes/scripts/run.sh b/tests/e2e-kubernetes/scripts/run.sh
@@ -27,6 +27,7 @@ KUBECTL_BIN=${KUBECTL_INSTALL_PATH}/kubectl
 CLUSTER_TYPE=${CLUSTER_TYPE:-kops}
 ARCH=${ARCH:-x86}
 AMI_FAMILY=${AMI_FAMILY:-AmazonLinux2}
+SELINUX_MODE=${SELINUX_MODE:-}
 
 # kops: must include patch version (e.g. 1.19.1)
 # eksctl: mustn't include patch version (e.g. 1.19)
@@ -67,11 +68,21 @@ CLUSTER_FILE=${TEST_DIR}/${CLUSTER_NAME}.${CLUSTER_TYPE}.yaml
 KOPS_PATCH_FILE=${KOPS_PATCH_FILE:-${BASE_DIR}/kops-patch.yaml}
 KOPS_PATCH_NODE_FILE=${KOPS_PATCH_NODE_FILE:-${BASE_DIR}/kops-patch-node.yaml}
 KOPS_STATE_FILE=${KOPS_STATE_FILE:-"s3://mountpoint-s3-csi-driver-kops-state-store"}
+KOPS_PATCH_NODE_SELINUX_ENFORCING_FILE=${KOPS_PATCH_NODE_SELINUX_ENFORCING_FILE:-${BASE_DIR}/kops-patch-node-selinux-enforcing.yaml}
+if [[ "${SELINUX_MODE}" != "enforcing" ]]; then
+    KOPS_PATCH_NODE_SELINUX_ENFORCING_FILE=""
+fi
+
 SSH_KEY=${SSH_KEY:-""}
 HELM_RELEASE_NAME=mountpoint-s3-csi-driver
 
 EKSCTL_VERSION=${EKSCTL_VERSION:-0.202.0}
 EKSCTL_PATCH_FILE=${EKSCTL_PATCH_FILE:-${BASE_DIR}/eksctl-patch.json}
+EKSCTL_PATCH_SELINUX_ENFORCING_FILE=${EKSCTL_PATCH_SELINUX_ENFORCING_FILE:-${BASE_DIR}/eksctl-patch-selinux-enforcing.json}
+if [[ "${SELINUX_MODE}" != "enforcing" ]]; then
+    EKSCTL_PATCH_SELINUX_ENFORCING_FILE=""
+fi
+
 CI_ROLE_ARN=${CI_ROLE_ARN:-""}
 
 MOUNTER_KIND=${MOUNTER_KIND:-systemd}
@@ -124,7 +135,8 @@ function create_cluster() {
       "$KOPS_PATCH_FILE" \
       "$KOPS_PATCH_NODE_FILE" \
       "$KOPS_STATE_FILE" \
-      "$SSH_KEY"
+      "$SSH_KEY" \
+      "$KOPS_PATCH_NODE_SELINUX_ENFORCING_FILE"
   elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then
     eksctl_create_cluster \
       "$CLUSTER_NAME" \
@@ -138,7 +150,8 @@ function create_cluster() {
       "$CI_ROLE_ARN" \
       "$INSTANCE_TYPE" \
       "$AMI_FAMILY" \
-      "$K8S_VERSION_EKSCTL"
+      "$K8S_VERSION_EKSCTL" \
+      "$EKSCTL_PATCH_SELINUX_ENFORCING_FILE"
   fi
 }