Handle initial node taints via Terraform

Author: bobbyiliev

misc/nvme-bootstrap/README.md

@@ -10,7 +10,7 @@ Materialize requires fast, locally-attached NVMe storage for optimal performance
 
 1. Automatically detects NVMe instance store devices on your nodes
 2. Creates an LVM volume group from these devices
-3. Configures OpenEBS to provision persistent volumes from this storage
+3. Configures OpenEBS LVM Local-PV to provision persistent volumes from this storage
 4. Makes high-performance storage available to Materialize
 
 ## Prerequisites
@@ -36,10 +36,9 @@ module "materialize" {
   # Use an instance type with NVMe storage
   node_group_instance_types = ["r6gd.2xlarge"]
   node_group_ami_type       = "BOTTLEROCKET_ARM_64"
+  enable_nvme_storage       = true
 
-  # Disable Materialize Operator installation as it will require the storage class
-  # TODO: The module will support this configuration in the future
-  install_materialize_operator = false
+  install_materialize_operator = true
 
   # Other module parameters...
 }
@@ -55,6 +54,8 @@ If you're setting up manually or need to customize the configuration, follow the
 
 ### Step 1: Build and Push the Container Image
 
+> Note: This is temporary and will be replaced with a pre-built image which can be pulled from a public registry.
+
 ```bash
 # Clone the Materialize repository
 git clone https://github.com/MaterializeInc/materialize.git
@@ -70,32 +71,7 @@ docker build -t your-registry/nvme-bootstrap:latest .
 docker push your-registry/nvme-bootstrap:latest
 ```
 
-### Step 2: Install OpenEBS
-
-OpenEBS provides the CSI driver that interfaces with LVM to provide persistent storage:
-
-```bash
-# Add the OpenEBS Helm repository
-helm repo add openebs https://openebs.github.io/charts
-helm repo update
-
-# Create namespace for OpenEBS
-kubectl create namespace openebs
-
-# Install OpenEBS with only the necessary components
-helm install openebs openebs/openebs \
-  --namespace openebs \
-  --set engines.replicated.mayastor.enabled=false
-```
-
-Verify the installation:
-
-```bash
-# Check if the LVM controller is running
-kubectl get pods -n openebs -l role=openebs-lvm
-```
-
-### Step 3: Deploy the NVMe Bootstrap Components
+### Step 2: Deploy the NVMe Bootstrap Components
 
 ```bash
 # Navigate to the Kubernetes manifests directory
@@ -121,9 +97,35 @@ The DaemonSet will:
 3. Create the "instance-store-vg" volume group
 4. Make the storage available for OpenEBS
 
+### Step 3: Install OpenEBS
+
+OpenEBS provides the CSI driver that interfaces with LVM to provide persistent storage:
+
+```bash
+# Add the OpenEBS Helm repository
+helm repo add openebs https://openebs.github.io/charts
+helm repo update
+
+# Create namespace for OpenEBS
+kubectl create namespace openebs
+
+# Install OpenEBS with only the necessary components
+helm install openebs openebs/openebs \
+  --namespace openebs \
+  --set engines.replicated.mayastor.enabled=false
+```
+
+Verify the installation:
+
+```bash
+# Check if the LVM controller is running
+kubectl get pods -n openebs -l role=openebs-lvm
+```
+
 ### Step 4: Create and Test the Storage Class
 
 ```bash
+# TODO: remove this step once the Terraform module handles this
 # Create the StorageClass
 kubectl apply -f storageclass.yaml
 
@@ -139,9 +141,8 @@ A successful test shows your storage class is working correctly.
 To clean up the test resources:
 
 ```bash
-# Delete the test PVC and StorageClass
+# Delete the test PVC
 kubectl delete -f test-pvc.yaml
-kubectl delete -f storageclass.yaml
 ```
 
 ### Step 5: Configure Materialize to Use the Storage Class
@@ -246,19 +247,3 @@ kubectl get pvc -A | grep openebs-lvm-instance-store-ext4
   ```bash
   kubectl get serviceaccount nvme-setup-sa -n kube-system
   ```
-
-## Clean Up
-
-When you're done testing:
-
-```bash
-# Delete test resources
-kubectl delete -f test-pvc.yaml
-kubectl delete -f daemonset.yaml
-kubectl delete -f rbac.yaml
-kubectl delete -f storageclass.yaml
-
-# Delete OpenEBS if no longer needed
-helm uninstall openebs -n openebs
-kubectl delete namespace openebs
-```

misc/nvme-bootstrap/container/Dockerfile

@@ -26,5 +26,3 @@ COPY configure-disks.sh /usr/local/bin/configure-disks.sh
 COPY manage-taints.sh /usr/local/bin/manage-taints.sh
 
 RUN chmod +x /usr/local/bin/configure-disks.sh /usr/local/bin/manage-taints.sh
-
-ENTRYPOINT ["/usr/local/bin/configure-disks.sh"]

misc/nvme-bootstrap/container/configure-disks.sh

@@ -9,47 +9,111 @@
 # the Business Source License, use of this software will be governed
 # by the Apache License, Version 2.0.
 
-set -xeo pipefail
+set -xeuo pipefail
 
-# Volume group name
 VG_NAME="instance-store-vg"
+CLOUD_PROVIDER=""
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --cloud-provider|-c)
+      CLOUD_PROVIDER="$2"
+      shift 2
+      ;;
+    --vg-name|-v)
+      VG_NAME="$2"
+      shift 2
+      ;;
+    --help|-h)
+      echo "Usage: $0 [options]"
+      echo "Options:"
+      echo "  --cloud-provider, -c PROVIDER   Specify cloud provider (aws, gcp, azure, generic)"
+      echo "  --vg-name, -v NAME     Specify volume group name (default: instance-store-vg)"
+      echo "  --help, -h             Show this help message"
+      exit 0
+      ;;
+    *)
+      echo "Unknown option: $1"
+      exit 1
+      ;;
+  esac
+done
 
-# Function to detect cloud provider
 detect_cloud_provider() {
-    # Check for AWS
-    if curl -s -m 5 http://169.254.169.254/latest/meta-data/ >/dev/null; then
+    # Only attempt detection if not explicitly provided
+    if [[ -n "$CLOUD_PROVIDER" ]]; then
+        echo "$CLOUD_PROVIDER"
+        return
+    fi
+
+    # Fall back to AWS detection if no provider is specified
+    if curl -s -m 5 --fail http://169.254.169.254/latest/meta-data/ >/dev/null 2>&1; then
         echo "aws"
         return
     fi
 
-    # Default to generic
+    # Default to generic if detection fails
     echo "generic"
 }
 
-# Cloud provider-specific device detection
+find_aws_bottlerocket_devices() {
+    local nvme_devices=()
+    local BOTTLEROCKET_ROOT="/.bottlerocket/rootfs"
+
+    mapfile -t SSD_NVME_DEVICE_LIST < <(lsblk --json --output-all | \
+        jq -r '.blockdevices[] | select(.model // empty | contains("Amazon EC2 NVMe Instance Storage")) | .path')
+
+    for device in "${SSD_NVME_DEVICE_LIST[@]}"; do
+        nvme_devices+=("$BOTTLEROCKET_ROOT$device")
+    done
+
+    echo "${nvme_devices[@]}"
+}
+
+find_aws_standard_devices() {
+    lsblk --json --output-all | \
+        jq -r '.blockdevices[] | select(.model // empty | contains("Amazon EC2 NVMe Instance Storage")) | .path'
+}
+
+find_aws_devices() {
+    local nvme_devices=()
+
+    # Check if we're running in Bottlerocket
+    if [[ -d "/.bottlerocket" ]]; then
+        # Use mapfile to properly handle the output
+        mapfile -t nvme_devices < <(find_aws_bottlerocket_devices)
+    else
+        # Use mapfile to properly handle the output
+        mapfile -t nvme_devices < <(find_aws_standard_devices)
+    fi
+
+    echo "${nvme_devices[@]}"
+}
+
+find_generic_devices() {
+    lsblk --json --output-all | \
+        jq -r '.blockdevices[] | select(.name | startswith("nvme")) | select(.mountpoint == null and (.children | length == 0)) | .path'
+}
+
 find_nvme_devices() {
     local cloud=$1
     local nvme_devices=()
 
     case $cloud in
         aws)
-            # Handle both standard Linux and Bottlerocket paths
-            if [ -d "/.bottlerocket" ]; then
-                # Bottlerocket specific path
-                BOTTLEROCKET_ROOT="/.bottlerocket/rootfs"
-                mapfile -t SSD_NVME_DEVICE_LIST < <(lsblk --json --output-all | jq -r '.blockdevices[] | select(.model // empty | contains("Amazon EC2 NVMe Instance Storage")) | .path')
-                for device in "${SSD_NVME_DEVICE_LIST[@]}"; do
-                    nvme_devices+=("$BOTTLEROCKET_ROOT$device")
-                done
-            else
-                # Standard EC2 instances
-                mapfile -t nvme_devices < <(lsblk --json --output-all | jq -r '.blockdevices[] | select(.model // empty | contains("Amazon EC2 NVMe Instance Storage")) | .path')
-            fi
+            # Use mapfile to properly handle the output
+            mapfile -t nvme_devices < <(find_aws_devices)
             ;;
-        # Add more cloud providers here
+        # Add more cloud providers here as we support them
+        # gcp)
+        #     mapfile -t nvme_devices < <(find_gcp_devices)
+        #     ;;
+        # azure)
+        #     mapfile -t nvme_devices < <(find_azure_devices)
+        #     ;;
         *)
-            # Generic approach - find all NVMe devices that are not mounted and don't have children (partitions)
-            mapfile -t nvme_devices < <(lsblk --json --output-all | jq -r '.blockdevices[] | select(.name | startswith("nvme")) | select(.mountpoint == null and (.children | length == 0)) | .path')
+            # Generic approach for any other cloud or environment
+            mapfile -t nvme_devices < <(find_generic_devices)
             ;;
     esac
 
@@ -60,7 +124,7 @@ find_nvme_devices() {
 setup_lvm() {
     local -a devices=("$@")
 
-    if [ ${#devices[@]} -eq 0 ]; then
+    if [[ ${#devices[@]} -eq 0 ]]; then
         echo "No suitable NVMe devices found"
         exit 1
     fi
@@ -89,27 +153,19 @@ setup_lvm() {
     return 0
 }
 
-# Main execution
 echo "Starting NVMe disk configuration..."
 
-# Detect cloud provider
+# Detect or use provided cloud provider
 CLOUD_PROVIDER=$(detect_cloud_provider)
-echo "Detected cloud provider: $CLOUD_PROVIDER"
+echo "Using cloud provider: $CLOUD_PROVIDER"
 
 # Find NVMe devices
 mapfile -t NVME_DEVICES < <(find_nvme_devices "$CLOUD_PROVIDER")
 
 # Setup LVM
 if setup_lvm "${NVME_DEVICES[@]}"; then
     echo "NVMe disk configuration completed successfully"
-    # Call taint management script to remove the taint
-    /usr/local/bin/manage-taints.sh remove
-
-    # Keep the container running
-    echo "Setup complete. Container will now stay running for monitoring purposes."
-    while true; do
-        sleep 3600
-    done
+    exit 0
 else
     echo "NVMe disk configuration failed"
     exit 1

misc/nvme-bootstrap/container/manage-taints.sh

@@ -9,40 +9,26 @@
 # the Business Source License, use of this software will be governed
 # by the Apache License, Version 2.0.
 
-set -e
+set -euo pipefail
 
 # Node name is provided via downward API
 NODE_NAME=${NODE_NAME:-$(hostname)}
 TAINT_KEY="disk-unconfigured"
-TAINT_VALUE="true"
-TAINT_EFFECT="NoSchedule"
 
 echo "Starting taint management for node: $NODE_NAME"
 echo "Action: $1"
 
 # Check if necessary environment variables and files exist
 if [ -z "$KUBERNETES_SERVICE_HOST" ] || [ -z "$KUBERNETES_SERVICE_PORT" ]; then
     echo "Error: Kubernetes service environment variables not found"
-    exit 0  # Exit with success to avoid crash loop
+    exit 1
 fi
 
 if [ ! -f "/var/run/secrets/kubernetes.io/serviceaccount/token" ]; then
     echo "Error: Service account token not found"
-    exit 0  # Exit with success to avoid crash loop
+    exit 1
 fi
 
-# Add the taint to the node
-add_taint() {
-    echo "Adding taint $TAINT_KEY=$TAINT_VALUE:$TAINT_EFFECT to node $NODE_NAME"
-
-    kubectl --server="https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}" \
-            --token="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" \
-            --certificate-authority="/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" \
-            taint nodes "$NODE_NAME" "$TAINT_KEY=$TAINT_VALUE:$TAINT_EFFECT" --overwrite || {
-                echo "Warning: Failed to add taint, but continuing anyway"
-            }
-}
-
 # Remove the taint from the node
 remove_taint() {
     echo "Removing taint $TAINT_KEY from node $NODE_NAME"
@@ -56,18 +42,15 @@ remove_taint() {
 }
 
 # Main execution
-ACTION=${1:-"add"}
+ACTION=${1:-"remove"}
 
 case "$ACTION" in
-    add)
-        add_taint
-        ;;
     remove)
         remove_taint
         ;;
     *)
-        echo "Usage: $0 [add|remove]"
-        exit 0  # Exit with success to avoid crash loop
+        echo "Usage: $0 [remove]"
+        exit 1
         ;;
 esac