slurm-single-node.yml

name: slurm-single-node

on:
  push:
  pull_request:

permissions:
  contents: read
  packages: write

jobs:
  host-tools:
    uses: ./.github/workflows/host-tools.yml

  slurm:
    needs: host-tools
    runs-on: ubuntu-24.04
    timeout-minutes: 10

    steps:
      - uses: actions/checkout@v4

      - name: Restore apt package archive cache
        id: restore-apt-archives
        uses: actions/cache/restore@v4
        with:
          path: .ci-cache/apt-archives/*.deb
          key: ${{ runner.os }}-${{ runner.arch }}-apt-archives-slurm-${{ hashFiles('.github/workflows/slurm-single-node.yml') }}
          restore-keys: |
            ${{ runner.os }}-${{ runner.arch }}-apt-archives-slurm-

      - name: Download reusable host tools bundle
        uses: actions/download-artifact@v4
        with:
          name: ${{ needs.host-tools.outputs.artifact-name }}
          path: .ci-artifacts

      - name: Install reusable host tools bundle
        shell: bash
        run: |
          set -euxo pipefail

          tar -C /tmp -xzf ".ci-artifacts/${{ needs.host-tools.outputs.artifact-name }}.tar.gz"

          sudo mkdir -p /usr/local/libexec /usr/local/share
          sudo cp -R /tmp/host-tools/podman-static/usr/. /usr/
          sudo cp -R /tmp/host-tools/podman-static/etc/. /etc/
          sudo cp -R /tmp/host-tools/bats/. /usr/local/
          sudo install -m 0755 /tmp/host-tools/bin/* /usr/local/bin/

          # Keep legacy paths stable for the existing workflow.
          if [ -x /usr/local/libexec/podman/conmon ]; then
            sudo ln -sf /usr/local/libexec/podman/conmon /usr/local/bin/conmon
          fi
          sudo ln -sf /usr/local/bin/squashfuse /usr/bin/squashfuse
          sudo ln -sf /usr/local/bin/squashfuse_ll /usr/bin/squashfuse_ll

          command -v podman
          command -v conmon
          command -v squashfuse_ll
          command -v parallax
          command -v bats

      - name: Allow unprivileged user namespaces (work around Ubuntu 24.04 AppArmor restriction)
        run: |
          set -euxo pipefail

          # Disable Ubuntu's AppArmor restriction that can make `unshare(CLONE_NEWUSER)` fail with EPERM
          if [ -e /proc/sys/kernel/apparmor_restrict_unprivileged_userns ]; then
            echo 0 | sudo tee /proc/sys/kernel/apparmor_restrict_unprivileged_userns
          fi

          # Show status
          sysctl kernel.apparmor_restrict_unprivileged_userns 2>/dev/null || true
          sysctl kernel.unprivileged_userns_clone 2>/dev/null || true
          sysctl user.max_user_namespaces 2>/dev/null || true


      - name: Check user namespaces availability
        run: |
          set -euxo pipefail
          echo "== sysctls =="
          cat /proc/sys/kernel/unprivileged_userns_clone 2>/dev/null || true
          cat /proc/sys/user/max_user_namespaces 2>/dev/null || true
      
          echo "== unshare probe =="
          if unshare -Ur true 2>/dev/null; then
            echo "unshare -Ur: OK (user namespaces usable)"
          else
            echo "unshare -Ur: FAIL (user namespaces not usable here)"
          fi
      
          echo "== kernel config hint =="
          zgrep CONFIG_USER_NS /proc/config.gz 2>/dev/null || true
          grep CONFIG_USER_NS "/boot/config-$(uname -r)" 2>/dev/null || true

      - name: Login to GHCR
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Build libskybox.so in devcontainer (fail early on build)
        if: github.event_name != 'pull_request'
        uses: devcontainers/ci@v0.3
        with:
          configFile: .devcontainer/ubuntu/devcontainer.json
          imageName: ${{ format('ghcr.io/{0}/skybox-devcontainer-cache', github.repository) }}
          cacheFrom: ${{ format('ghcr.io/{0}/skybox-devcontainer-cache:latest', github.repository) }}
          push: always
          runCmd: |
            set -euxo pipefail
            ci_phase_start="${SECONDS}"
            log_phase() {
              local label="$1"
              local now="${SECONDS}"
              echo "::notice::[devcontainer-ci] ${label} (+$((now - ci_phase_start))s)"
              ci_phase_start="${now}"
            }

            # Building skybox
            log_phase "starting cargo build -p skybox --release"
            cargo build --locked -p skybox --release

            # Build and test sarusctl
            log_phase "finished skybox build; starting cargo build -p sarusctl --release"
            cargo build --locked -p sarusctl --release
            log_phase "finished sarusctl build; starting cargo test -p sarusctl --test cli"
            cargo test --locked -p sarusctl --test cli

            # Copy artifact into the workspace
            log_phase "finished sarusctl cli test; copying artifacts"
            mkdir -p dist
            cp -f "${CARGO_TARGET_DIR}/release/libskybox.so" dist/libskybox.so
            cp -f "${CARGO_TARGET_DIR}/release/sarusctl" dist/sarusctl
            ls -l dist/libskybox.so
            ls -l dist/sarusctl
            log_phase "artifacts copied"

      - name: Build libskybox.so in devcontainer (pull request)
        if: github.event_name == 'pull_request'
        uses: devcontainers/ci@v0.3
        with:
          configFile: .devcontainer/ubuntu/devcontainer.json
          imageName: ${{ format('ghcr.io/{0}/skybox-devcontainer-cache', github.repository) }}
          cacheFrom: ${{ format('ghcr.io/{0}/skybox-devcontainer-cache:latest', github.repository) }}
          push: never
          runCmd: |
            set -euxo pipefail
            ci_phase_start="${SECONDS}"
            log_phase() {
              local label="$1"
              local now="${SECONDS}"
              echo "::notice::[devcontainer-ci] ${label} (+$((now - ci_phase_start))s)"
              ci_phase_start="${now}"
            }

            # Building skybox
            log_phase "starting cargo build -p skybox --release"
            cargo build --locked -p skybox --release

            # Build and test sarusctl
            log_phase "finished skybox build; starting cargo build -p sarusctl --release"
            cargo build --locked -p sarusctl --release
            log_phase "finished sarusctl build; starting cargo test -p sarusctl --test cli"
            cargo test --locked -p sarusctl --test cli

            # Copy artifact into the workspace
            log_phase "finished sarusctl cli test; copying artifacts"
            mkdir -p dist
            cp -f "${CARGO_TARGET_DIR}/release/libskybox.so" dist/libskybox.so
            cp -f "${CARGO_TARGET_DIR}/release/sarusctl" dist/sarusctl
            ls -l dist/libskybox.so
            ls -l dist/sarusctl
            log_phase "artifacts copied"
      
      - name: Install libskybox.so into /usr/lib64/slurm on runner
        shell: bash
        run: |
          set -euxo pipefail
      
          sudo mkdir -p /usr/lib64/slurm
          sudo install -m 0755 dist/libskybox.so /usr/lib64/slurm/libskybox.so
          ls -l /usr/lib64/slurm/libskybox.so


      - name: Install Slurm + Munge (+ SlurmDBD + MariaDB)
        run: |
          set -euxo pipefail
          APT_ARCHIVE_CACHE_DIR="${GITHUB_WORKSPACE}/.ci-cache/apt-archives"

          mkdir -p "${APT_ARCHIVE_CACHE_DIR}"

          sudo apt-get update
          sudo apt-get -o Dir::Cache::archives="${APT_ARCHIVE_CACHE_DIR}" install -y --no-install-recommends \
            munge \
            slurmctld slurmd slurm-client slurmdbd \
            mariadb-server


      - name: Configure Munge
        run: |
          set -euxo pipefail

          # Create a fresh key
          sudo dd if=/dev/urandom of=/etc/munge/munge.key bs=1 count=1024 status=none
          sudo chown munge:munge /etc/munge/munge.key
          sudo chmod 0400 /etc/munge/munge.key

          # Start munge w new key
          sudo systemctl enable munge
          sudo systemctl restart munge
          sudo systemctl status --no-pager munge

      - name: Configure single-node Slurm (+ slurmdbd)
        run: |
          set -euxo pipefail

          HOST="$(hostname -s)"

          # Ensure HOST resolves locally!
          if ! getent hosts "$HOST" >/dev/null 2>&1; then
            echo "127.0.1.1 $HOST" | sudo tee -a /etc/hosts >/dev/null
          fi

          # Create slurm service user (if not present)
          if ! id -u slurm >/dev/null 2>&1; then
            sudo useradd -r -m -U -d /var/lib/slurm -s /usr/sbin/nologin slurm
          fi

          CPUS="$(nproc)"
          MEM_MB="$(awk '/MemTotal/ {print int($2/1024)}' /proc/meminfo)"

          sudo install -d -o slurm -g slurm /var/lib/slurm/slurmctld
          sudo install -d -o slurm -g slurm /var/lib/slurm/slurmd
          sudo install -d -o slurm -g slurm /var/log/slurm
          sudo install -d -o root  -g root  /etc/slurm

          # --- MariaDB bootstrap for Slurm accounting (minimal) ---
          sudo systemctl enable --now mariadb
          # Create DB + user SlurmDBD will use
          sudo mysql -e "CREATE DATABASE IF NOT EXISTS slurm_acct_db;"
          sudo mysql -e "CREATE USER IF NOT EXISTS 'slurm'@'localhost' IDENTIFIED BY 'slurm';"
          sudo mysql -e "GRANT ALL PRIVILEGES ON slurm_acct_db.* TO 'slurm'@'localhost';"
          sudo mysql -e "FLUSH PRIVILEGES;"

          # --- slurm.conf ---
          sudo tee /etc/slurm/slurm.conf >/dev/null <<EOF
          ClusterName=ci
          SlurmctldHost=${HOST}
          SlurmUser=slurm

          AuthType=auth/munge
          StateSaveLocation=/var/lib/slurm/slurmctld
          SlurmdSpoolDir=/var/lib/slurm/slurmd

          SlurmctldPort=6817
          SlurmdPort=6818

          # Spank plugin config
          PlugStackConfig=/etc/slurm/plugstack.conf

          # activate debug level for skybox
          SlurmdDebug=debug2

          # no mpi
          MpiDefault=none

          # Scheduling/allocation
          SelectType=select/cons_tres
          SelectTypeParameters=CR_Core

          # No cgroup enforcement
          ProctrackType=proctrack/linuxproc
          TaskPlugin=task/none

          # Avoid fatal "Configured MailProg is invalid" on minimal images
          MailProg=/bin/true

          # Accounting via SlurmDBD
          AccountingStorageType=accounting_storage/slurmdbd

          # Logging
          SlurmctldLogFile=/var/log/slurm/slurmctld.log
          SlurmdLogFile=/var/log/slurm/slurmd.log

          # Single-node definition
          NodeName=${HOST} CPUs=${CPUS} Sockets=1 CoresPerSocket=2 ThreadsPerCore=2 RealMemory=${MEM_MB} State=UNKNOWN
          PartitionName=debug Nodes=${HOST} Default=YES MaxTime=INFINITE State=UP
          EOF

          # --- slurmdbd.conf (permissions MUST be strict) ---
          sudo tee /etc/slurm/slurmdbd.conf >/dev/null <<'EOF'
          AuthType=auth/munge
          DbdHost=localhost
          SlurmUser=slurm

          LogFile=/var/log/slurm/slurmdbd.log
          PidFile=/run/slurmdbd.pid

          StorageType=accounting_storage/mysql
          StorageHost=localhost
          StoragePort=3306
          StorageUser=slurm
          StoragePass=slurm
          StorageLoc=slurm_acct_db
          EOF
          sudo chown slurm:slurm /etc/slurm/slurmdbd.conf
          sudo chmod 0600 /etc/slurm/slurmdbd.conf

          # /etc/slurm-llnl compatibility
          if [ -d /etc/slurm-llnl ] && [ ! -L /etc/slurm-llnl ]; then
            sudo rm -rf /etc/slurm-llnl
          fi
          if [ ! -e /etc/slurm-llnl ]; then
            sudo ln -s /etc/slurm /etc/slurm-llnl
          fi

          sudo chown -R slurm:slurm /var/lib/slurm /var/log/slurm

          echo "=== /etc/slurm/slurm.conf ==="
          sed -n '1,220p' /etc/slurm/slurm.conf
          echo "=== /etc/slurm/slurmdbd.conf (exists) ==="
          sudo ls -l /etc/slurm/slurmdbd.conf


      - name: Install SPANK plugstack + sarus-suite config (Zinal layout)
        run: |
          set -euxo pipefail

          # Create the directory structure
          sudo install -d -m 0755 /etc/slurm
          sudo install -d -m 0755 /etc/slurm/plugstack.conf.d
          sudo install -d -m 0755 /etc/sarus-suite

          # plugstack.conf
          sudo tee /etc/slurm/plugstack.conf >/dev/null <<'EOF'
          #
          # SPANK config file
          #
          include /etc/slurm/plugstack.conf.d/*
          EOF
          sudo chmod 0644 /etc/slurm/plugstack.conf

          # --- plugstack.conf.d entries (create all three) ---

          # skybox.conf
          sudo tee /etc/slurm/plugstack.conf.d/skybox.conf >/dev/null <<'EOF'
          required /usr/lib64/slurm/libskybox.so config_path=/etc/sarus-suite
          EOF
          sudo chmod 0644 /etc/slurm/plugstack.conf.d/skybox.conf

          # /etc/sarus-suite/sarus-suite.conf like in zinal
          sudo tee /etc/sarus-suite/sarus-suite.conf >/dev/null <<'EOF'
          edf_system_search_path = "/tmp/containers/edf"
          parallax_imagestore = "/tmp/.parallax_imagestore"
          parallax_path = "/usr/local/bin/parallax"
          parallax_mount_program = "/usr/local/bin/parallax-mount-program"
          skybox_enabled = true
          tracking_enabled = false
          tracking_tool = "/usr/local/opt/pyxis_tracking_tool/pyxis_tracking_tool"
          EOF
          sudo chmod 0644 /etc/sarus-suite/sarus-suite.conf

          echo "=== /etc/slurm/plugstack.conf ==="
          sudo cat /etc/slurm/plugstack.conf
          echo "=== /etc/slurm/plugstack.conf.d ==="
          sudo ls -la /etc/slurm/plugstack.conf.d
          echo "=== /etc/slurm/plugstack.conf.d/skybox.conf ==="
          sudo cat /etc/slurm/plugstack.conf.d/skybox.conf
          echo "=== /etc/sarus-suite/sarus-suite.conf ==="
          sudo cat /etc/sarus-suite/sarus-suite.conf




      - name: Start Slurm daemons (mariadb -> slurmdbd -> slurmctld -> slurmd)
        run: |
          set -euxo pipefail

          sudo systemctl enable --now mariadb
          sudo systemctl enable --now slurmdbd
          sudo systemctl enable --now slurmctld
          sudo systemctl enable --now slurmd

          sudo systemctl status --no-pager mariadb
          sudo systemctl status --no-pager slurmdbd
          sudo systemctl status --no-pager slurmctld
          sudo systemctl status --no-pager slurmd


      - name: Bootstrap accounting (cluster/account/user)
        run: |
          set -euxo pipefail

          # Create cluster to match ClusterName=ci (idempotent with || true)
          sudo sacctmgr -i add cluster ci || true

          # Create an account + associate the GitHub Actions user
          sudo sacctmgr -i add account default Description="CI default" Organization="ci" || true
          sudo sacctmgr -i add user runner Account=default || true

          echo "=== Associations ==="
          sudo sacctmgr -n show assoc format=Cluster,Account,User || true

      - name: Smoke test slurm (sinfo / scontrol / sbatch)
        run: |
          set -euxo pipefail

          echo "=== Initial cluster state ==="
          sinfo -Nel || true
          scontrol show node || true

          HOST="$(hostname -s)"

          echo "Waiting for node '$HOST' to become ready..."
          for i in $(seq 1 60); do
            state="$(sinfo -h -N -n "$HOST" -o "%T" 2>/dev/null || true)"
            echo "[$i/60] state='${state}'"
            case "$state" in
              idle|mix|alloc|completing)
                echo "Node is ready: $state"
                break
                ;;
            esac
            sleep 1
          done

          echo "=== Post-wait cluster state ==="
          sinfo -Nel
          scontrol show node

          cat > job.sh <<'EOF'
          #!/usr/bin/env bash
          set -euxo pipefail
          echo "Hello from Slurm on $(hostname)"
          EOF
          chmod +x job.sh

          jid="$(sbatch --parsable -p debug -A default -J ci-test --wrap "./job.sh")"
          echo "Submitted: $jid"
          squeue -j "$jid" || true

          for i in $(seq 1 60); do
            state="$(scontrol show job -o "$jid" 2>/dev/null | sed -n 's/.*JobState=\([^ ]*\).*/\1/p' || true)"
            echo "[$i/60] JobState=${state:-UNKNOWN}"
            case "$state" in
              COMPLETED|FAILED|CANCELLED|TIMEOUT|NODE_FAIL|OUT_OF_MEMORY|PREEMPTED)
                break
                ;;
              "")
                # job no longer known to slurmctld (finished and purged)
                break
                ;;
            esac
            sleep 1
          done

          echo "=== Job output (slurm-${jid}.out) ==="
          if [ -f "slurm-${jid}.out" ]; then
            cat "slurm-${jid}.out"
          else
            echo "No slurm-${jid}.out found in $(pwd); listing directory:"
            ls -la

            out_path="$(scontrol show job -dd "$jid" 2>/dev/null | sed -n 's/^ *StdOut=\(.*\)$/\1/p' | tail -n 1 || true)"
            if [ -n "$out_path" ] && [ -f "$out_path" ]; then
              echo "=== Job output (from StdOut=$out_path) ==="
              cat "$out_path"
            else
              echo "Could not locate job stdout file."
            fi
          fi

      - name: Install system dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y \
            lsb-release \
            btrfs-progs \
            libbtrfs-dev \
            lvm2 \
            libdevmapper-dev \
            squashfs-tools \
            fuse-overlayfs \
            inotify-tools


      - name: Tool versions (podman + fuse/squash tools)
        shell: bash
        run: |
          set -euo pipefail
      
          echo "=== OS / Kernel ==="
          uname -a || true
          cat /etc/os-release || true
          echo
      
          # Helper: print "name: path" then try common version flags
          show_ver() {
            local label="$1"; shift
            local bin="$1"; shift || true
      
            echo "--- ${label} ---"
            if [[ "$bin" == /* ]]; then
              if [[ ! -x "$bin" ]]; then
                echo "path: $bin (missing or not executable)"
                echo
                return 0
              fi
              echo "path: $bin"
            else
              if ! command -v "$bin" >/dev/null 2>&1; then
                echo "path: (not found in PATH)"
                echo
                return 0
              fi
              echo "path: $(command -v "$bin")"
            fi
      
            # Try a few common patterns; stop at the first that works.
            if "$bin" --version >/dev/null 2>&1; then
              "$bin" --version
            elif "$bin" version >/dev/null 2>&1; then
              "$bin" version
            elif "$bin" -V >/dev/null 2>&1; then
              "$bin" -V
            elif "$bin" -v >/dev/null 2>&1; then
              "$bin" -v
            else
              echo "version: (no supported version flag detected)"
            fi
            echo
          }
      
          echo "=== Podman static binaries (/usr/local/bin) ==="
          show_ver "podman (static)"       /usr/local/bin/podman
          show_ver "crun (static)"         /usr/local/bin/crun
          show_ver "runc (static)"         /usr/local/bin/runc
          show_ver "fuse-overlayfs (static)" /usr/local/bin/fuse-overlayfs
          show_ver "fusermount3 (static)"  /usr/local/bin/fusermount3
          show_ver "pasta (static)"        /usr/local/bin/pasta
          show_ver "pasta.avx2 (static)"   /usr/local/bin/pasta.avx2
      
          echo "=== System binaries (PATH) you care about ==="
          show_ver "fuse-overlayfs (PATH)" fuse-overlayfs
          show_ver "fusermount3 (PATH)"    fusermount3
          show_ver "crun (PATH)"           crun
          show_ver "runc (PATH)"           runc
          show_ver "podman (PATH)"         podman
          show_ver "slirp4netns (PATH)"    slirp4netns
          show_ver "netavark (PATH)"       netavark
          show_ver "aardvark-dns (PATH)"   aardvark-dns

          echo "=== output for squashfuse ==="
          squashfuse --version |& head -n 1 || true
          echo

          echo "=== Podman view of runtime / helpers (podman info) ==="
          /usr/local/bin/podman info --debug || true


      - name: Rootless Podman host prerequisites (uidmap + subuid/subgid + slirp4netns)
        run: |
          set -euxo pipefail

          # 1) newuidmap/newgidmap (rootless prerequisite)
          # 3) slirp4netns (common rootless networking helper)
          sudo apt-get update
          sudo apt-get install -y --no-install-recommends uidmap slirp4netns

          # 2) Ensure subordinate ID ranges exist for the runner user
          if ! grep -qE '^runner:' /etc/subuid; then
            echo "runner:100000:65536" | sudo tee -a /etc/subuid >/dev/null
          fi
          if ! grep -qE '^runner:' /etc/subgid; then
            echo "runner:100000:65536" | sudo tee -a /etc/subgid >/dev/null
          fi

          # Sanity checks
          command -v newuidmap
          command -v newgidmap
          podman info

      - name: Save apt package archive cache
        if: steps.restore-apt-archives.outputs.cache-hit != 'true' && github.event_name == 'push'
        uses: actions/cache/save@v4
        with:
          path: .ci-cache/apt-archives/*.deb
          key: ${{ steps.restore-apt-archives.outputs.cache-primary-key }}

      - name: Fix AppArmor profile for /usr/local/bin/podman
        run: |
          set -euxo pipefail

          if command -v apparmor_parser >/dev/null 2>&1 && [ -f /etc/apparmor.d/podman ]; then
            sudo sed -Ei 's!^profile podman /usr/bin/podman !profile podman /usr/{bin,local/bin}/podman !' /etc/apparmor.d/podman
            sudo apparmor_parser -r /etc/apparmor.d/podman
          else
            echo "No apparmor_parser or /etc/apparmor.d/podman found; skipping AppArmor adjustment."
          fi

      - name: Write /etc/containers/containers.conf (global)
        shell: bash
        run: |
          set -euo pipefail
          sudo mkdir -p /etc/containers
          sudo tee /etc/containers/containers.conf >/dev/null <<'EOF'
          [engine]
          runtime = "crun"
      
          [engine.runtimes]
          crun = ["/usr/local/bin/crun"]
          EOF

      - name: Create Podman module hpc
        shell: bash
        run: |
          set -euo pipefail
      
          # Resolve actual installed binary paths
          CRUN_PATH="$(command -v crun)"
          CONMON_PATH="$(command -v conmon)"
      
          echo "Detected crun:   ${CRUN_PATH}"
          echo "Detected conmon: ${CONMON_PATH}"
      
          # Create the modules directory Podman searches
          sudo mkdir -p /etc/containers/containers.conf.modules
      
          # Write the 'hpc' module file
          sudo tee /etc/containers/containers.conf.modules/hpc >/dev/null <<EOF
          [containers]
          ipcns = "host"
          netns = "host"
          pidns = "host"
          utsns = "host"
          #userns = "keep-id"
          cgroupns = "host"
          cgroups = "no-conmon"
      
          tz = "local"
      
          # Mounting /tmp as a temporary measure to propagate Slurm PMIx dirs until we have dedicated hook
          mounts = ["type=bind,src=/tmp,dst=/tmp"]
      
          [engine]
          runtime = "crun"
          conmon_path = ["${CONMON_PATH}"]
      
          [engine.runtimes]
          crun = ["${CRUN_PATH}"]
          EOF
      
          # Sanity check: show what we wrote
          sudo ls -l /etc/containers/containers.conf.modules/hpc
          sudo sed -n '1,200p' /etc/containers/containers.conf.modules/hpc



      - name: Run organized Bats smoke tests
        shell: bash
        run: |
          set -euxo pipefail
          bats --print-output-on-failure test/smoke