SD-69774 - Fix skybox failures when tracking_tool is not configured #223

Workflow file for this run

.github/workflows/slurm-single-node.yml at 15e09df

	name: slurm-single-node

	on:
	push:
	pull_request:

	permissions:
	contents: read
	packages: write

	jobs:
	host-tools:
	uses: ./.github/workflows/host-tools.yml

	slurm:
	needs: host-tools
	runs-on: ubuntu-24.04
	timeout-minutes: 10

	steps:
	- uses: actions/checkout@v4

	- name: Restore apt package archive cache
	id: restore-apt-archives
	uses: actions/cache/restore@v4
	with:
	path: .ci-cache/apt-archives/*.deb
	key: ${{ runner.os }}-${{ runner.arch }}-apt-archives-slurm-${{ hashFiles('.github/workflows/slurm-single-node.yml') }}
	restore-keys: \|
	${{ runner.os }}-${{ runner.arch }}-apt-archives-slurm-

	- name: Download reusable host tools bundle
	uses: actions/download-artifact@v4
	with:
	name: ${{ needs.host-tools.outputs.artifact-name }}
	path: .ci-artifacts

	- name: Install reusable host tools bundle
	shell: bash
	run: \|
	set -euxo pipefail

	tar -C /tmp -xzf ".ci-artifacts/${{ needs.host-tools.outputs.artifact-name }}.tar.gz"

	sudo mkdir -p /usr/local/libexec /usr/local/share
	sudo cp -R /tmp/host-tools/podman-static/usr/. /usr/
	sudo cp -R /tmp/host-tools/podman-static/etc/. /etc/
	sudo cp -R /tmp/host-tools/bats/. /usr/local/
	sudo install -m 0755 /tmp/host-tools/bin/* /usr/local/bin/

	# Keep legacy paths stable for the existing workflow.
	if [ -x /usr/local/libexec/podman/conmon ]; then
	sudo ln -sf /usr/local/libexec/podman/conmon /usr/local/bin/conmon
	fi
	sudo ln -sf /usr/local/bin/squashfuse /usr/bin/squashfuse
	sudo ln -sf /usr/local/bin/squashfuse_ll /usr/bin/squashfuse_ll

	command -v podman
	command -v conmon
	command -v squashfuse_ll
	command -v parallax
	command -v bats

	- name: Allow unprivileged user namespaces (work around Ubuntu 24.04 AppArmor restriction)
	run: \|
	set -euxo pipefail

	# Disable Ubuntu's AppArmor restriction that can make `unshare(CLONE_NEWUSER)` fail with EPERM
	if [ -e /proc/sys/kernel/apparmor_restrict_unprivileged_userns ]; then
	echo 0 \| sudo tee /proc/sys/kernel/apparmor_restrict_unprivileged_userns
	fi

	# Show status
	sysctl kernel.apparmor_restrict_unprivileged_userns 2>/dev/null \|\| true
	sysctl kernel.unprivileged_userns_clone 2>/dev/null \|\| true
	sysctl user.max_user_namespaces 2>/dev/null \|\| true


	- name: Check user namespaces availability
	run: \|
	set -euxo pipefail
	echo "== sysctls =="
	cat /proc/sys/kernel/unprivileged_userns_clone 2>/dev/null \|\| true
	cat /proc/sys/user/max_user_namespaces 2>/dev/null \|\| true

	echo "== unshare probe =="
	if unshare -Ur true 2>/dev/null; then
	echo "unshare -Ur: OK (user namespaces usable)"
	else
	echo "unshare -Ur: FAIL (user namespaces not usable here)"
	fi

	echo "== kernel config hint =="
	zgrep CONFIG_USER_NS /proc/config.gz 2>/dev/null \|\| true
	grep CONFIG_USER_NS "/boot/config-$(uname -r)" 2>/dev/null \|\| true

	- name: Login to GHCR
	uses: docker/login-action@v3
	with:
	registry: ghcr.io
	username: ${{ github.actor }}
	password: ${{ secrets.GITHUB_TOKEN }}

	- name: Build libskybox.so in devcontainer (fail early on build)
	if: github.event_name != 'pull_request'
	uses: devcontainers/ci@v0.3
	with:
	configFile: .devcontainer/ubuntu/devcontainer.json
	imageName: ${{ format('ghcr.io/{0}/skybox-devcontainer-cache', github.repository) }}
	cacheFrom: ${{ format('ghcr.io/{0}/skybox-devcontainer-cache:latest', github.repository) }}
	push: always
	runCmd: \|
	set -euxo pipefail
	ci_phase_start="${SECONDS}"
	log_phase() {
	local label="$1"
	local now="${SECONDS}"
	echo "::notice::[devcontainer-ci] ${label} (+$((now - ci_phase_start))s)"
	ci_phase_start="${now}"
	}

	# Building skybox
	log_phase "starting cargo build -p skybox --release"
	cargo build --locked -p skybox --release

	# Build and test sarusctl
	log_phase "finished skybox build; starting cargo build -p sarusctl --release"
	cargo build --locked -p sarusctl --release
	log_phase "finished sarusctl build; starting cargo test -p sarusctl --test cli"
	cargo test --locked -p sarusctl --test cli

	# Copy artifact into the workspace
	log_phase "finished sarusctl cli test; copying artifacts"
	mkdir -p dist
	cp -f "${CARGO_TARGET_DIR}/release/libskybox.so" dist/libskybox.so
	cp -f "${CARGO_TARGET_DIR}/release/sarusctl" dist/sarusctl
	ls -l dist/libskybox.so
	ls -l dist/sarusctl
	log_phase "artifacts copied"

	- name: Build libskybox.so in devcontainer (pull request)
	if: github.event_name == 'pull_request'
	uses: devcontainers/ci@v0.3
	with:
	configFile: .devcontainer/ubuntu/devcontainer.json
	imageName: ${{ format('ghcr.io/{0}/skybox-devcontainer-cache', github.repository) }}
	cacheFrom: ${{ format('ghcr.io/{0}/skybox-devcontainer-cache:latest', github.repository) }}
	push: never
	runCmd: \|
	set -euxo pipefail
	ci_phase_start="${SECONDS}"
	log_phase() {
	local label="$1"
	local now="${SECONDS}"
	echo "::notice::[devcontainer-ci] ${label} (+$((now - ci_phase_start))s)"
	ci_phase_start="${now}"
	}

	# Building skybox
	log_phase "starting cargo build -p skybox --release"
	cargo build --locked -p skybox --release

	# Build and test sarusctl
	log_phase "finished skybox build; starting cargo build -p sarusctl --release"
	cargo build --locked -p sarusctl --release
	log_phase "finished sarusctl build; starting cargo test -p sarusctl --test cli"
	cargo test --locked -p sarusctl --test cli

	# Copy artifact into the workspace
	log_phase "finished sarusctl cli test; copying artifacts"
	mkdir -p dist
	cp -f "${CARGO_TARGET_DIR}/release/libskybox.so" dist/libskybox.so
	cp -f "${CARGO_TARGET_DIR}/release/sarusctl" dist/sarusctl
	ls -l dist/libskybox.so
	ls -l dist/sarusctl
	log_phase "artifacts copied"

	- name: Install libskybox.so into /usr/lib64/slurm on runner
	shell: bash
	run: \|
	set -euxo pipefail

	sudo mkdir -p /usr/lib64/slurm
	sudo install -m 0755 dist/libskybox.so /usr/lib64/slurm/libskybox.so
	ls -l /usr/lib64/slurm/libskybox.so


	- name: Install Slurm + Munge (+ SlurmDBD + MariaDB)
	run: \|
	set -euxo pipefail
	APT_ARCHIVE_CACHE_DIR="${GITHUB_WORKSPACE}/.ci-cache/apt-archives"

	mkdir -p "${APT_ARCHIVE_CACHE_DIR}"

	sudo apt-get update
	sudo apt-get -o Dir::Cache::archives="${APT_ARCHIVE_CACHE_DIR}" install -y --no-install-recommends \
	munge \
	slurmctld slurmd slurm-client slurmdbd \
	mariadb-server


	- name: Configure Munge
	run: \|
	set -euxo pipefail

	# Create a fresh key
	sudo dd if=/dev/urandom of=/etc/munge/munge.key bs=1 count=1024 status=none
	sudo chown munge:munge /etc/munge/munge.key
	sudo chmod 0400 /etc/munge/munge.key

	# Start munge w new key
	sudo systemctl enable munge
	sudo systemctl restart munge
	sudo systemctl status --no-pager munge

	- name: Configure single-node Slurm (+ slurmdbd)
	run: \|
	set -euxo pipefail

	HOST="$(hostname -s)"

	# Ensure HOST resolves locally!
	if ! getent hosts "$HOST" >/dev/null 2>&1; then
	echo "127.0.1.1 $HOST" \| sudo tee -a /etc/hosts >/dev/null
	fi

	# Create slurm service user (if not present)
	if ! id -u slurm >/dev/null 2>&1; then
	sudo useradd -r -m -U -d /var/lib/slurm -s /usr/sbin/nologin slurm
	fi

	CPUS="$(nproc)"
	MEM_MB="$(awk '/MemTotal/ {print int($2/1024)}' /proc/meminfo)"

	sudo install -d -o slurm -g slurm /var/lib/slurm/slurmctld
	sudo install -d -o slurm -g slurm /var/lib/slurm/slurmd
	sudo install -d -o slurm -g slurm /var/log/slurm
	sudo install -d -o root -g root /etc/slurm

	# --- MariaDB bootstrap for Slurm accounting (minimal) ---
	sudo systemctl enable --now mariadb
	# Create DB + user SlurmDBD will use
	sudo mysql -e "CREATE DATABASE IF NOT EXISTS slurm_acct_db;"
	sudo mysql -e "CREATE USER IF NOT EXISTS 'slurm'@'localhost' IDENTIFIED BY 'slurm';"
	sudo mysql -e "GRANT ALL PRIVILEGES ON slurm_acct_db.* TO 'slurm'@'localhost';"
	sudo mysql -e "FLUSH PRIVILEGES;"

	# --- slurm.conf ---
	sudo tee /etc/slurm/slurm.conf >/dev/null <<EOF
	ClusterName=ci
	SlurmctldHost=${HOST}
	SlurmUser=slurm

	AuthType=auth/munge
	StateSaveLocation=/var/lib/slurm/slurmctld
	SlurmdSpoolDir=/var/lib/slurm/slurmd

	SlurmctldPort=6817
	SlurmdPort=6818

	# Spank plugin config
	PlugStackConfig=/etc/slurm/plugstack.conf

	# activate debug level for skybox
	SlurmdDebug=debug2

	# no mpi
	MpiDefault=none

	# Scheduling/allocation
	SelectType=select/cons_tres
	SelectTypeParameters=CR_Core

	# No cgroup enforcement
	ProctrackType=proctrack/linuxproc
	TaskPlugin=task/none

	# Avoid fatal "Configured MailProg is invalid" on minimal images
	MailProg=/bin/true

	# Accounting via SlurmDBD
	AccountingStorageType=accounting_storage/slurmdbd

	# Logging
	SlurmctldLogFile=/var/log/slurm/slurmctld.log
	SlurmdLogFile=/var/log/slurm/slurmd.log

	# Single-node definition
	NodeName=${HOST} CPUs=${CPUS} Sockets=1 CoresPerSocket=2 ThreadsPerCore=2 RealMemory=${MEM_MB} State=UNKNOWN
	PartitionName=debug Nodes=${HOST} Default=YES MaxTime=INFINITE State=UP
	EOF

	# --- slurmdbd.conf (permissions MUST be strict) ---
	sudo tee /etc/slurm/slurmdbd.conf >/dev/null <<'EOF'
	AuthType=auth/munge
	DbdHost=localhost
	SlurmUser=slurm

	LogFile=/var/log/slurm/slurmdbd.log
	PidFile=/run/slurmdbd.pid

	StorageType=accounting_storage/mysql
	StorageHost=localhost
	StoragePort=3306
	StorageUser=slurm
	StoragePass=slurm
	StorageLoc=slurm_acct_db
	EOF
	sudo chown slurm:slurm /etc/slurm/slurmdbd.conf
	sudo chmod 0600 /etc/slurm/slurmdbd.conf

	# /etc/slurm-llnl compatibility
	if [ -d /etc/slurm-llnl ] && [ ! -L /etc/slurm-llnl ]; then
	sudo rm -rf /etc/slurm-llnl
	fi
	if [ ! -e /etc/slurm-llnl ]; then
	sudo ln -s /etc/slurm /etc/slurm-llnl
	fi

	sudo chown -R slurm:slurm /var/lib/slurm /var/log/slurm

	echo "=== /etc/slurm/slurm.conf ==="
	sed -n '1,220p' /etc/slurm/slurm.conf
	echo "=== /etc/slurm/slurmdbd.conf (exists) ==="
	sudo ls -l /etc/slurm/slurmdbd.conf


	- name: Install SPANK plugstack + sarus-suite config (Zinal layout)
	run: \|
	set -euxo pipefail

	# Create the directory structure
	sudo install -d -m 0755 /etc/slurm
	sudo install -d -m 0755 /etc/slurm/plugstack.conf.d
	sudo install -d -m 0755 /etc/sarus-suite

	# plugstack.conf
	sudo tee /etc/slurm/plugstack.conf >/dev/null <<'EOF'
	#
	# SPANK config file
	#
	include /etc/slurm/plugstack.conf.d/*
	EOF
	sudo chmod 0644 /etc/slurm/plugstack.conf

	# --- plugstack.conf.d entries (create all three) ---

	# skybox.conf
	sudo tee /etc/slurm/plugstack.conf.d/skybox.conf >/dev/null <<'EOF'
	required /usr/lib64/slurm/libskybox.so config_path=/etc/sarus-suite
	EOF
	sudo chmod 0644 /etc/slurm/plugstack.conf.d/skybox.conf

	# /etc/sarus-suite/sarus-suite.conf like in zinal
	sudo tee /etc/sarus-suite/sarus-suite.conf >/dev/null <<'EOF'
	edf_system_search_path = "/tmp/containers/edf"
	parallax_imagestore = "/tmp/.parallax_imagestore"
	parallax_path = "/usr/local/bin/parallax"
	parallax_mount_program = "/usr/local/bin/parallax-mount-program"
	skybox_enabled = true
	tracking_enabled = false
	tracking_tool = "/usr/local/opt/pyxis_tracking_tool/pyxis_tracking_tool"
	EOF
	sudo chmod 0644 /etc/sarus-suite/sarus-suite.conf

	echo "=== /etc/slurm/plugstack.conf ==="
	sudo cat /etc/slurm/plugstack.conf
	echo "=== /etc/slurm/plugstack.conf.d ==="
	sudo ls -la /etc/slurm/plugstack.conf.d
	echo "=== /etc/slurm/plugstack.conf.d/skybox.conf ==="
	sudo cat /etc/slurm/plugstack.conf.d/skybox.conf
	echo "=== /etc/sarus-suite/sarus-suite.conf ==="
	sudo cat /etc/sarus-suite/sarus-suite.conf




	- name: Start Slurm daemons (mariadb -> slurmdbd -> slurmctld -> slurmd)
	run: \|
	set -euxo pipefail

	sudo systemctl enable --now mariadb
	sudo systemctl enable --now slurmdbd
	sudo systemctl enable --now slurmctld
	sudo systemctl enable --now slurmd

	sudo systemctl status --no-pager mariadb
	sudo systemctl status --no-pager slurmdbd
	sudo systemctl status --no-pager slurmctld
	sudo systemctl status --no-pager slurmd


	- name: Bootstrap accounting (cluster/account/user)
	run: \|
	set -euxo pipefail

	# Create cluster to match ClusterName=ci (idempotent with \|\| true)
	sudo sacctmgr -i add cluster ci \|\| true

	# Create an account + associate the GitHub Actions user
	sudo sacctmgr -i add account default Description="CI default" Organization="ci" \|\| true
	sudo sacctmgr -i add user runner Account=default \|\| true

	echo "=== Associations ==="
	sudo sacctmgr -n show assoc format=Cluster,Account,User \|\| true

	- name: Smoke test slurm (sinfo / scontrol / sbatch)
	run: \|
	set -euxo pipefail

	echo "=== Initial cluster state ==="
	sinfo -Nel \|\| true
	scontrol show node \|\| true

	HOST="$(hostname -s)"

	echo "Waiting for node '$HOST' to become ready..."
	for i in $(seq 1 60); do
	state="$(sinfo -h -N -n "$HOST" -o "%T" 2>/dev/null \|\| true)"
	echo "[$i/60] state='${state}'"
	case "$state" in
	idle\|mix\|alloc\|completing)
	echo "Node is ready: $state"
	break
	;;
	esac
	sleep 1
	done

	echo "=== Post-wait cluster state ==="
	sinfo -Nel
	scontrol show node

	cat > job.sh <<'EOF'
	#!/usr/bin/env bash
	set -euxo pipefail
	echo "Hello from Slurm on $(hostname)"
	EOF
	chmod +x job.sh

	jid="$(sbatch --parsable -p debug -A default -J ci-test --wrap "./job.sh")"
	echo "Submitted: $jid"
	squeue -j "$jid" \|\| true

	for i in $(seq 1 60); do
	state="$(scontrol show job -o "$jid" 2>/dev/null \| sed -n 's/.JobState=$[^ ]$.*/\1/p' \|\| true)"
	echo "[$i/60] JobState=${state:-UNKNOWN}"
	case "$state" in
	COMPLETED\|FAILED\|CANCELLED\|TIMEOUT\|NODE_FAIL\|OUT_OF_MEMORY\|PREEMPTED)
	break
	;;
	"")
	# job no longer known to slurmctld (finished and purged)
	break
	;;
	esac
	sleep 1
	done

	echo "=== Job output (slurm-${jid}.out) ==="
	if [ -f "slurm-${jid}.out" ]; then
	cat "slurm-${jid}.out"
	else
	echo "No slurm-${jid}.out found in $(pwd); listing directory:"
	ls -la

	out_path="$(scontrol show job -dd "$jid" 2>/dev/null \| sed -n 's/^ StdOut=$.$$/\1/p' \| tail -n 1 \|\| true)"
	if [ -n "$out_path" ] && [ -f "$out_path" ]; then
	echo "=== Job output (from StdOut=$out_path) ==="
	cat "$out_path"
	else
	echo "Could not locate job stdout file."
	fi
	fi

	- name: Install system dependencies
	run: \|
	sudo apt-get update
	sudo apt-get install -y \
	lsb-release \
	btrfs-progs \
	libbtrfs-dev \
	lvm2 \
	libdevmapper-dev \
	squashfs-tools \
	fuse-overlayfs \
	inotify-tools


	- name: Tool versions (podman + fuse/squash tools)
	shell: bash
	run: \|
	set -euo pipefail

	echo "=== OS / Kernel ==="
	uname -a \|\| true
	cat /etc/os-release \|\| true
	echo

	# Helper: print "name: path" then try common version flags
	show_ver() {
	local label="$1"; shift
	local bin="$1"; shift \|\| true

	echo "--- ${label} ---"
	if [[ "$bin" == /* ]]; then
	if [[ ! -x "$bin" ]]; then
	echo "path: $bin (missing or not executable)"
	echo
	return 0
	fi
	echo "path: $bin"
	else
	if ! command -v "$bin" >/dev/null 2>&1; then
	echo "path: (not found in PATH)"
	echo
	return 0
	fi
	echo "path: $(command -v "$bin")"
	fi

	# Try a few common patterns; stop at the first that works.
	if "$bin" --version >/dev/null 2>&1; then
	"$bin" --version
	elif "$bin" version >/dev/null 2>&1; then
	"$bin" version
	elif "$bin" -V >/dev/null 2>&1; then
	"$bin" -V
	elif "$bin" -v >/dev/null 2>&1; then
	"$bin" -v
	else
	echo "version: (no supported version flag detected)"
	fi
	echo
	}

	echo "=== Podman static binaries (/usr/local/bin) ==="
	show_ver "podman (static)" /usr/local/bin/podman
	show_ver "crun (static)" /usr/local/bin/crun
	show_ver "runc (static)" /usr/local/bin/runc
	show_ver "fuse-overlayfs (static)" /usr/local/bin/fuse-overlayfs
	show_ver "fusermount3 (static)" /usr/local/bin/fusermount3
	show_ver "pasta (static)" /usr/local/bin/pasta
	show_ver "pasta.avx2 (static)" /usr/local/bin/pasta.avx2

	echo "=== System binaries (PATH) you care about ==="
	show_ver "fuse-overlayfs (PATH)" fuse-overlayfs
	show_ver "fusermount3 (PATH)" fusermount3
	show_ver "crun (PATH)" crun
	show_ver "runc (PATH)" runc
	show_ver "podman (PATH)" podman
	show_ver "slirp4netns (PATH)" slirp4netns
	show_ver "netavark (PATH)" netavark
	show_ver "aardvark-dns (PATH)" aardvark-dns

	echo "=== output for squashfuse ==="
	squashfuse --version \|& head -n 1 \|\| true
	echo

	echo "=== Podman view of runtime / helpers (podman info) ==="
	/usr/local/bin/podman info --debug \|\| true


	- name: Rootless Podman host prerequisites (uidmap + subuid/subgid + slirp4netns)
	run: \|
	set -euxo pipefail

	# 1) newuidmap/newgidmap (rootless prerequisite)
	# 3) slirp4netns (common rootless networking helper)
	sudo apt-get update
	sudo apt-get install -y --no-install-recommends uidmap slirp4netns

	# 2) Ensure subordinate ID ranges exist for the runner user
	if ! grep -qE '^runner:' /etc/subuid; then
	echo "runner:100000:65536" \| sudo tee -a /etc/subuid >/dev/null
	fi
	if ! grep -qE '^runner:' /etc/subgid; then
	echo "runner:100000:65536" \| sudo tee -a /etc/subgid >/dev/null
	fi

	# Sanity checks
	command -v newuidmap
	command -v newgidmap
	podman info

	- name: Save apt package archive cache
	if: steps.restore-apt-archives.outputs.cache-hit != 'true' && github.event_name == 'push'
	uses: actions/cache/save@v4
	with:
	path: .ci-cache/apt-archives/*.deb
	key: ${{ steps.restore-apt-archives.outputs.cache-primary-key }}

	- name: Fix AppArmor profile for /usr/local/bin/podman
	run: \|
	set -euxo pipefail

	if command -v apparmor_parser >/dev/null 2>&1 && [ -f /etc/apparmor.d/podman ]; then
	sudo sed -Ei 's!^profile podman /usr/bin/podman !profile podman /usr/{bin,local/bin}/podman !' /etc/apparmor.d/podman
	sudo apparmor_parser -r /etc/apparmor.d/podman
	else
	echo "No apparmor_parser or /etc/apparmor.d/podman found; skipping AppArmor adjustment."
	fi

	- name: Write /etc/containers/containers.conf (global)
	shell: bash
	run: \|
	set -euo pipefail
	sudo mkdir -p /etc/containers
	sudo tee /etc/containers/containers.conf >/dev/null <<'EOF'
	[engine]
	runtime = "crun"

	[engine.runtimes]
	crun = ["/usr/local/bin/crun"]
	EOF

	- name: Create Podman module hpc
	shell: bash
	run: \|
	set -euo pipefail

	# Resolve actual installed binary paths
	CRUN_PATH="$(command -v crun)"
	CONMON_PATH="$(command -v conmon)"

	echo "Detected crun: ${CRUN_PATH}"
	echo "Detected conmon: ${CONMON_PATH}"

	# Create the modules directory Podman searches
	sudo mkdir -p /etc/containers/containers.conf.modules

	# Write the 'hpc' module file
	sudo tee /etc/containers/containers.conf.modules/hpc >/dev/null <<EOF
	[containers]
	ipcns = "host"
	netns = "host"
	pidns = "host"
	utsns = "host"
	#userns = "keep-id"
	cgroupns = "host"
	cgroups = "no-conmon"

	tz = "local"

	# Mounting /tmp as a temporary measure to propagate Slurm PMIx dirs until we have dedicated hook
	mounts = ["type=bind,src=/tmp,dst=/tmp"]

	[engine]
	runtime = "crun"
	conmon_path = ["${CONMON_PATH}"]

	[engine.runtimes]
	crun = ["${CRUN_PATH}"]
	EOF

	# Sanity check: show what we wrote
	sudo ls -l /etc/containers/containers.conf.modules/hpc
	sudo sed -n '1,200p' /etc/containers/containers.conf.modules/hpc



	- name: Run organized Bats smoke tests
	shell: bash
	run: \|
	set -euxo pipefail
	bats --print-output-on-failure test/smoke

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

SD-69774 - Fix skybox failures when tracking_tool is not configured #223

Workflow file

SD-69774 - Fix skybox failures when tracking_tool is not configured #223

Uh oh!

Workflow file for this run