Skip to content
slurm-single-node

Merge commit '8a37bf7c31dac0202a6a90181bb0378dd4135fe7' #221

Sign in to view logs

Merge commit '8a37bf7c31dac0202a6a90181bb0378dd4135fe7'

Merge commit '8a37bf7c31dac0202a6a90181bb0378dd4135fe7' #221

Workflow file for this run

.github/workflows/slurm-single-node.yml at 97f382d
name: slurm-single-node
on:
push:
pull_request:
permissions:
contents: read
packages: write
jobs:
host-tools:
uses: ./.github/workflows/host-tools.yml
slurm:
needs: host-tools
runs-on: ubuntu-24.04
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
- name: Restore apt package archive cache
id: restore-apt-archives
uses: actions/cache/restore@v4
with:
path: .ci-cache/apt-archives/*.deb
key: ${{ runner.os }}-${{ runner.arch }}-apt-archives-slurm-${{ hashFiles('.github/workflows/slurm-single-node.yml') }}
restore-keys: |
${{ runner.os }}-${{ runner.arch }}-apt-archives-slurm-
- name: Download reusable host tools bundle
uses: actions/download-artifact@v4
with:
name: ${{ needs.host-tools.outputs.artifact-name }}
path: .ci-artifacts
- name: Install reusable host tools bundle
shell: bash
run: |
set -euxo pipefail
tar -C /tmp -xzf ".ci-artifacts/${{ needs.host-tools.outputs.artifact-name }}.tar.gz"
sudo mkdir -p /usr/local/libexec /usr/local/share
sudo cp -R /tmp/host-tools/podman-static/usr/. /usr/
sudo cp -R /tmp/host-tools/podman-static/etc/. /etc/
sudo cp -R /tmp/host-tools/bats/. /usr/local/
sudo install -m 0755 /tmp/host-tools/bin/* /usr/local/bin/
# Keep legacy paths stable for the existing workflow.
if [ -x /usr/local/libexec/podman/conmon ]; then
sudo ln -sf /usr/local/libexec/podman/conmon /usr/local/bin/conmon
fi
sudo ln -sf /usr/local/bin/squashfuse /usr/bin/squashfuse
sudo ln -sf /usr/local/bin/squashfuse_ll /usr/bin/squashfuse_ll
command -v podman
command -v conmon
command -v squashfuse_ll
command -v parallax
command -v bats
- name: Allow unprivileged user namespaces (work around Ubuntu 24.04 AppArmor restriction)
run: |
set -euxo pipefail
# Disable Ubuntu's AppArmor restriction that can make `unshare(CLONE_NEWUSER)` fail with EPERM
if [ -e /proc/sys/kernel/apparmor_restrict_unprivileged_userns ]; then
echo 0 | sudo tee /proc/sys/kernel/apparmor_restrict_unprivileged_userns
fi
# Show status
sysctl kernel.apparmor_restrict_unprivileged_userns 2>/dev/null || true
sysctl kernel.unprivileged_userns_clone 2>/dev/null || true
sysctl user.max_user_namespaces 2>/dev/null || true
- name: Check user namespaces availability
run: |
set -euxo pipefail
echo "== sysctls =="
cat /proc/sys/kernel/unprivileged_userns_clone 2>/dev/null || true
cat /proc/sys/user/max_user_namespaces 2>/dev/null || true
echo "== unshare probe =="
if unshare -Ur true 2>/dev/null; then
echo "unshare -Ur: OK (user namespaces usable)"
else
echo "unshare -Ur: FAIL (user namespaces not usable here)"
fi
echo "== kernel config hint =="
zgrep CONFIG_USER_NS /proc/config.gz 2>/dev/null || true
grep CONFIG_USER_NS "/boot/config-$(uname -r)" 2>/dev/null || true
- name: Login to GHCR
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build libskybox.so in devcontainer (fail early on build)
if: github.event_name != 'pull_request'
uses: devcontainers/ci@v0.3
with:
configFile: .devcontainer/ubuntu/devcontainer.json
imageName: ${{ format('ghcr.io/{0}/skybox-devcontainer-cache', github.repository) }}
cacheFrom: ${{ format('ghcr.io/{0}/skybox-devcontainer-cache:latest', github.repository) }}
push: always
runCmd: |
set -euxo pipefail
ci_phase_start="${SECONDS}"
log_phase() {
local label="$1"
local now="${SECONDS}"
echo "::notice::[devcontainer-ci] ${label} (+$((now - ci_phase_start))s)"
ci_phase_start="${now}"
}
# Building skybox
log_phase "starting cargo build -p skybox --release"
cargo build --locked -p skybox --release
# Build and test sarusctl
log_phase "finished skybox build; starting cargo build -p sarusctl --release"
cargo build --locked -p sarusctl --release
log_phase "finished sarusctl build; starting cargo test -p sarusctl --test cli"
cargo test --locked -p sarusctl --test cli
# Copy artifact into the workspace
log_phase "finished sarusctl cli test; copying artifacts"
mkdir -p dist
cp -f "${CARGO_TARGET_DIR}/release/libskybox.so" dist/libskybox.so
cp -f "${CARGO_TARGET_DIR}/release/sarusctl" dist/sarusctl
ls -l dist/libskybox.so
ls -l dist/sarusctl
log_phase "artifacts copied"
- name: Build libskybox.so in devcontainer (pull request)
if: github.event_name == 'pull_request'
uses: devcontainers/ci@v0.3
with:
configFile: .devcontainer/ubuntu/devcontainer.json
imageName: ${{ format('ghcr.io/{0}/skybox-devcontainer-cache', github.repository) }}
cacheFrom: ${{ format('ghcr.io/{0}/skybox-devcontainer-cache:latest', github.repository) }}
push: never
runCmd: |
set -euxo pipefail
ci_phase_start="${SECONDS}"
log_phase() {
local label="$1"
local now="${SECONDS}"
echo "::notice::[devcontainer-ci] ${label} (+$((now - ci_phase_start))s)"
ci_phase_start="${now}"
}
# Building skybox
log_phase "starting cargo build -p skybox --release"
cargo build --locked -p skybox --release
# Build and test sarusctl
log_phase "finished skybox build; starting cargo build -p sarusctl --release"
cargo build --locked -p sarusctl --release
log_phase "finished sarusctl build; starting cargo test -p sarusctl --test cli"
cargo test --locked -p sarusctl --test cli
# Copy artifact into the workspace
log_phase "finished sarusctl cli test; copying artifacts"
mkdir -p dist
cp -f "${CARGO_TARGET_DIR}/release/libskybox.so" dist/libskybox.so
cp -f "${CARGO_TARGET_DIR}/release/sarusctl" dist/sarusctl
ls -l dist/libskybox.so
ls -l dist/sarusctl
log_phase "artifacts copied"
- name: Install libskybox.so into /usr/lib64/slurm on runner
shell: bash
run: |
set -euxo pipefail
sudo mkdir -p /usr/lib64/slurm
sudo install -m 0755 dist/libskybox.so /usr/lib64/slurm/libskybox.so
ls -l /usr/lib64/slurm/libskybox.so
- name: Install Slurm + Munge (+ SlurmDBD + MariaDB)
run: |
set -euxo pipefail
APT_ARCHIVE_CACHE_DIR="${GITHUB_WORKSPACE}/.ci-cache/apt-archives"
mkdir -p "${APT_ARCHIVE_CACHE_DIR}"
sudo apt-get update
sudo apt-get -o Dir::Cache::archives="${APT_ARCHIVE_CACHE_DIR}" install -y --no-install-recommends \
munge \
slurmctld slurmd slurm-client slurmdbd \
mariadb-server
- name: Configure Munge
run: |
set -euxo pipefail
# Create a fresh key
sudo dd if=/dev/urandom of=/etc/munge/munge.key bs=1 count=1024 status=none
sudo chown munge:munge /etc/munge/munge.key
sudo chmod 0400 /etc/munge/munge.key
# Start munge w new key
sudo systemctl enable munge
sudo systemctl restart munge
sudo systemctl status --no-pager munge
- name: Configure single-node Slurm (+ slurmdbd)
run: |
set -euxo pipefail
HOST="$(hostname -s)"
# Ensure HOST resolves locally!
if ! getent hosts "$HOST" >/dev/null 2>&1; then
echo "127.0.1.1 $HOST" | sudo tee -a /etc/hosts >/dev/null
fi
# Create slurm service user (if not present)
if ! id -u slurm >/dev/null 2>&1; then
sudo useradd -r -m -U -d /var/lib/slurm -s /usr/sbin/nologin slurm
fi
CPUS="$(nproc)"
MEM_MB="$(awk '/MemTotal/ {print int($2/1024)}' /proc/meminfo)"
sudo install -d -o slurm -g slurm /var/lib/slurm/slurmctld
sudo install -d -o slurm -g slurm /var/lib/slurm/slurmd
sudo install -d -o slurm -g slurm /var/log/slurm
sudo install -d -o root -g root /etc/slurm
# --- MariaDB bootstrap for Slurm accounting (minimal) ---
sudo systemctl enable --now mariadb
# Create DB + user SlurmDBD will use
sudo mysql -e "CREATE DATABASE IF NOT EXISTS slurm_acct_db;"
sudo mysql -e "CREATE USER IF NOT EXISTS 'slurm'@'localhost' IDENTIFIED BY 'slurm';"
sudo mysql -e "GRANT ALL PRIVILEGES ON slurm_acct_db.* TO 'slurm'@'localhost';"
sudo mysql -e "FLUSH PRIVILEGES;"
# --- slurm.conf ---
sudo tee /etc/slurm/slurm.conf >/dev/null <<EOF
ClusterName=ci
SlurmctldHost=${HOST}
SlurmUser=slurm
AuthType=auth/munge
StateSaveLocation=/var/lib/slurm/slurmctld
SlurmdSpoolDir=/var/lib/slurm/slurmd
SlurmctldPort=6817
SlurmdPort=6818
# Spank plugin config
PlugStackConfig=/etc/slurm/plugstack.conf
# activate debug level for skybox
SlurmdDebug=debug2
# no mpi
MpiDefault=none
# Scheduling/allocation
SelectType=select/cons_tres
SelectTypeParameters=CR_Core
# No cgroup enforcement
ProctrackType=proctrack/linuxproc
TaskPlugin=task/none
# Avoid fatal "Configured MailProg is invalid" on minimal images
MailProg=/bin/true
# Accounting via SlurmDBD
AccountingStorageType=accounting_storage/slurmdbd
# Logging
SlurmctldLogFile=/var/log/slurm/slurmctld.log
SlurmdLogFile=/var/log/slurm/slurmd.log
# Single-node definition
NodeName=${HOST} CPUs=${CPUS} Sockets=1 CoresPerSocket=2 ThreadsPerCore=2 RealMemory=${MEM_MB} State=UNKNOWN
PartitionName=debug Nodes=${HOST} Default=YES MaxTime=INFINITE State=UP
EOF
# --- slurmdbd.conf (permissions MUST be strict) ---
sudo tee /etc/slurm/slurmdbd.conf >/dev/null <<'EOF'
AuthType=auth/munge
DbdHost=localhost
SlurmUser=slurm
LogFile=/var/log/slurm/slurmdbd.log
PidFile=/run/slurmdbd.pid
StorageType=accounting_storage/mysql
StorageHost=localhost
StoragePort=3306
StorageUser=slurm
StoragePass=slurm
StorageLoc=slurm_acct_db
EOF
sudo chown slurm:slurm /etc/slurm/slurmdbd.conf
sudo chmod 0600 /etc/slurm/slurmdbd.conf
# /etc/slurm-llnl compatibility
if [ -d /etc/slurm-llnl ] && [ ! -L /etc/slurm-llnl ]; then
sudo rm -rf /etc/slurm-llnl
fi
if [ ! -e /etc/slurm-llnl ]; then
sudo ln -s /etc/slurm /etc/slurm-llnl
fi
sudo chown -R slurm:slurm /var/lib/slurm /var/log/slurm
echo "=== /etc/slurm/slurm.conf ==="
sed -n '1,220p' /etc/slurm/slurm.conf
echo "=== /etc/slurm/slurmdbd.conf (exists) ==="
sudo ls -l /etc/slurm/slurmdbd.conf
- name: Install SPANK plugstack + sarus-suite config (Zinal layout)
run: |
set -euxo pipefail
# Create the directory structure
sudo install -d -m 0755 /etc/slurm
sudo install -d -m 0755 /etc/slurm/plugstack.conf.d
sudo install -d -m 0755 /etc/sarus-suite
# plugstack.conf
sudo tee /etc/slurm/plugstack.conf >/dev/null <<'EOF'
#
# SPANK config file
#
include /etc/slurm/plugstack.conf.d/*
EOF
sudo chmod 0644 /etc/slurm/plugstack.conf
# --- plugstack.conf.d entries (create all three) ---
# skybox.conf
sudo tee /etc/slurm/plugstack.conf.d/skybox.conf >/dev/null <<'EOF'
required /usr/lib64/slurm/libskybox.so config_path=/etc/sarus-suite
EOF
sudo chmod 0644 /etc/slurm/plugstack.conf.d/skybox.conf
# /etc/sarus-suite/sarus-suite.conf like in zinal
sudo tee /etc/sarus-suite/sarus-suite.conf >/dev/null <<'EOF'
edf_system_search_path = "/tmp/containers/edf"
parallax_imagestore = "/tmp/.parallax_imagestore"
parallax_path = "/usr/local/bin/parallax"
parallax_mount_program = "/usr/local/bin/parallax-mount-program"
skybox_enabled = true
tracking_enabled = false
tracking_tool = "/usr/local/opt/pyxis_tracking_tool/pyxis_tracking_tool"
EOF
sudo chmod 0644 /etc/sarus-suite/sarus-suite.conf
echo "=== /etc/slurm/plugstack.conf ==="
sudo cat /etc/slurm/plugstack.conf
echo "=== /etc/slurm/plugstack.conf.d ==="
sudo ls -la /etc/slurm/plugstack.conf.d
echo "=== /etc/slurm/plugstack.conf.d/skybox.conf ==="
sudo cat /etc/slurm/plugstack.conf.d/skybox.conf
echo "=== /etc/sarus-suite/sarus-suite.conf ==="
sudo cat /etc/sarus-suite/sarus-suite.conf
- name: Start Slurm daemons (mariadb -> slurmdbd -> slurmctld -> slurmd)
run: |
set -euxo pipefail
sudo systemctl enable --now mariadb
sudo systemctl enable --now slurmdbd
sudo systemctl enable --now slurmctld
sudo systemctl enable --now slurmd
sudo systemctl status --no-pager mariadb
sudo systemctl status --no-pager slurmdbd
sudo systemctl status --no-pager slurmctld
sudo systemctl status --no-pager slurmd
- name: Bootstrap accounting (cluster/account/user)
run: |
set -euxo pipefail
# Create cluster to match ClusterName=ci (idempotent with || true)
sudo sacctmgr -i add cluster ci || true
# Create an account + associate the GitHub Actions user
sudo sacctmgr -i add account default Description="CI default" Organization="ci" || true
sudo sacctmgr -i add user runner Account=default || true
echo "=== Associations ==="
sudo sacctmgr -n show assoc format=Cluster,Account,User || true
- name: Smoke test slurm (sinfo / scontrol / sbatch)
run: |
set -euxo pipefail
echo "=== Initial cluster state ==="
sinfo -Nel || true
scontrol show node || true
HOST="$(hostname -s)"
echo "Waiting for node '$HOST' to become ready..."
for i in $(seq 1 60); do
state="$(sinfo -h -N -n "$HOST" -o "%T" 2>/dev/null || true)"
echo "[$i/60] state='${state}'"
case "$state" in
idle|mix|alloc|completing)
echo "Node is ready: $state"
break
;;
esac
sleep 1
done
echo "=== Post-wait cluster state ==="
sinfo -Nel
scontrol show node
cat > job.sh <<'EOF'
#!/usr/bin/env bash
set -euxo pipefail
echo "Hello from Slurm on $(hostname)"
EOF
chmod +x job.sh
jid="$(sbatch --parsable -p debug -A default -J ci-test --wrap "./job.sh")"
echo "Submitted: $jid"
squeue -j "$jid" || true
for i in $(seq 1 60); do
state="$(scontrol show job -o "$jid" 2>/dev/null | sed -n 's/.*JobState=\([^ ]*\).*/\1/p' || true)"
echo "[$i/60] JobState=${state:-UNKNOWN}"
case "$state" in
COMPLETED|FAILED|CANCELLED|TIMEOUT|NODE_FAIL|OUT_OF_MEMORY|PREEMPTED)
break
;;
"")
# job no longer known to slurmctld (finished and purged)
break
;;
esac
sleep 1
done
echo "=== Job output (slurm-${jid}.out) ==="
if [ -f "slurm-${jid}.out" ]; then
cat "slurm-${jid}.out"
else
echo "No slurm-${jid}.out found in $(pwd); listing directory:"
ls -la
out_path="$(scontrol show job -dd "$jid" 2>/dev/null | sed -n 's/^ *StdOut=\(.*\)$/\1/p' | tail -n 1 || true)"
if [ -n "$out_path" ] && [ -f "$out_path" ]; then
echo "=== Job output (from StdOut=$out_path) ==="
cat "$out_path"
else
echo "Could not locate job stdout file."
fi
fi
- name: Install system dependencies
run: |
sudo apt-get update
sudo apt-get install -y \
lsb-release \
btrfs-progs \
libbtrfs-dev \
lvm2 \
libdevmapper-dev \
squashfs-tools \
fuse-overlayfs \
inotify-tools
- name: Tool versions (podman + fuse/squash tools)
shell: bash
run: |
set -euo pipefail
echo "=== OS / Kernel ==="
uname -a || true
cat /etc/os-release || true
echo
# Helper: print "name: path" then try common version flags
show_ver() {
local label="$1"; shift
local bin="$1"; shift || true
echo "--- ${label} ---"
if [[ "$bin" == /* ]]; then
if [[ ! -x "$bin" ]]; then
echo "path: $bin (missing or not executable)"
echo
return 0
fi
echo "path: $bin"
else
if ! command -v "$bin" >/dev/null 2>&1; then
echo "path: (not found in PATH)"
echo
return 0
fi
echo "path: $(command -v "$bin")"
fi
# Try a few common patterns; stop at the first that works.
if "$bin" --version >/dev/null 2>&1; then
"$bin" --version
elif "$bin" version >/dev/null 2>&1; then
"$bin" version
elif "$bin" -V >/dev/null 2>&1; then
"$bin" -V
elif "$bin" -v >/dev/null 2>&1; then
"$bin" -v
else
echo "version: (no supported version flag detected)"
fi
echo
}
echo "=== Podman static binaries (/usr/local/bin) ==="
show_ver "podman (static)" /usr/local/bin/podman
show_ver "crun (static)" /usr/local/bin/crun
show_ver "runc (static)" /usr/local/bin/runc
show_ver "fuse-overlayfs (static)" /usr/local/bin/fuse-overlayfs
show_ver "fusermount3 (static)" /usr/local/bin/fusermount3
show_ver "pasta (static)" /usr/local/bin/pasta
show_ver "pasta.avx2 (static)" /usr/local/bin/pasta.avx2
echo "=== System binaries (PATH) you care about ==="
show_ver "fuse-overlayfs (PATH)" fuse-overlayfs
show_ver "fusermount3 (PATH)" fusermount3
show_ver "crun (PATH)" crun
show_ver "runc (PATH)" runc
show_ver "podman (PATH)" podman
show_ver "slirp4netns (PATH)" slirp4netns
show_ver "netavark (PATH)" netavark
show_ver "aardvark-dns (PATH)" aardvark-dns
echo "=== output for squashfuse ==="
squashfuse --version |& head -n 1 || true
echo
echo "=== Podman view of runtime / helpers (podman info) ==="
/usr/local/bin/podman info --debug || true
- name: Rootless Podman host prerequisites (uidmap + subuid/subgid + slirp4netns)
run: |
set -euxo pipefail
# 1) newuidmap/newgidmap (rootless prerequisite)
# 3) slirp4netns (common rootless networking helper)
sudo apt-get update
sudo apt-get install -y --no-install-recommends uidmap slirp4netns
# 2) Ensure subordinate ID ranges exist for the runner user
if ! grep -qE '^runner:' /etc/subuid; then
echo "runner:100000:65536" | sudo tee -a /etc/subuid >/dev/null
fi
if ! grep -qE '^runner:' /etc/subgid; then
echo "runner:100000:65536" | sudo tee -a /etc/subgid >/dev/null
fi
# Sanity checks
command -v newuidmap
command -v newgidmap
podman info
- name: Save apt package archive cache
if: steps.restore-apt-archives.outputs.cache-hit != 'true' && github.event_name == 'push'
uses: actions/cache/save@v4
with:
path: .ci-cache/apt-archives/*.deb
key: ${{ steps.restore-apt-archives.outputs.cache-primary-key }}
- name: Fix AppArmor profile for /usr/local/bin/podman
run: |
set -euxo pipefail
if command -v apparmor_parser >/dev/null 2>&1 && [ -f /etc/apparmor.d/podman ]; then
sudo sed -Ei 's!^profile podman /usr/bin/podman !profile podman /usr/{bin,local/bin}/podman !' /etc/apparmor.d/podman
sudo apparmor_parser -r /etc/apparmor.d/podman
else
echo "No apparmor_parser or /etc/apparmor.d/podman found; skipping AppArmor adjustment."
fi
- name: Write /etc/containers/containers.conf (global)
shell: bash
run: |
set -euo pipefail
sudo mkdir -p /etc/containers
sudo tee /etc/containers/containers.conf >/dev/null <<'EOF'
[engine]
runtime = "crun"
[engine.runtimes]
crun = ["/usr/local/bin/crun"]
EOF
- name: Create Podman module hpc
shell: bash
run: |
set -euo pipefail
# Resolve actual installed binary paths
CRUN_PATH="$(command -v crun)"
CONMON_PATH="$(command -v conmon)"
echo "Detected crun: ${CRUN_PATH}"
echo "Detected conmon: ${CONMON_PATH}"
# Create the modules directory Podman searches
sudo mkdir -p /etc/containers/containers.conf.modules
# Write the 'hpc' module file
sudo tee /etc/containers/containers.conf.modules/hpc >/dev/null <<EOF
[containers]
ipcns = "host"
netns = "host"
pidns = "host"
utsns = "host"
#userns = "keep-id"
cgroupns = "host"
cgroups = "no-conmon"
tz = "local"
# Mounting /tmp as a temporary measure to propagate Slurm PMIx dirs until we have dedicated hook
mounts = ["type=bind,src=/tmp,dst=/tmp"]
[engine]
runtime = "crun"
conmon_path = ["${CONMON_PATH}"]
[engine.runtimes]
crun = ["${CRUN_PATH}"]
EOF
# Sanity check: show what we wrote
sudo ls -l /etc/containers/containers.conf.modules/hpc
sudo sed -n '1,200p' /etc/containers/containers.conf.modules/hpc
- name: Run organized Bats smoke tests
shell: bash
run: |
set -euxo pipefail
bats --print-output-on-failure test/smoke