Merge commit '0d1453a2dd3e2f7f270fcfc36673a6f00e66c9da' #157
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: slurm-single-node | |
| on: | |
| push: | |
| pull_request: | |
| permissions: | |
| contents: read | |
| packages: write | |
| jobs: | |
| slurm: | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 10 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Allow unprivileged user namespaces (work around Ubuntu 24.04 AppArmor restriction) | |
| run: | | |
| set -euxo pipefail | |
| # Disable Ubuntu's AppArmor restriction that can make `unshare(CLONE_NEWUSER)` fail with EPERM | |
| if [ -e /proc/sys/kernel/apparmor_restrict_unprivileged_userns ]; then | |
| echo 0 | sudo tee /proc/sys/kernel/apparmor_restrict_unprivileged_userns | |
| fi | |
| # Show status | |
| sysctl kernel.apparmor_restrict_unprivileged_userns 2>/dev/null || true | |
| sysctl kernel.unprivileged_userns_clone 2>/dev/null || true | |
| sysctl user.max_user_namespaces 2>/dev/null || true | |
| - name: Check user namespaces availability | |
| run: | | |
| set -euxo pipefail | |
| echo "== sysctls ==" | |
| cat /proc/sys/kernel/unprivileged_userns_clone 2>/dev/null || true | |
| cat /proc/sys/user/max_user_namespaces 2>/dev/null || true | |
| echo "== unshare probe ==" | |
| if unshare -Ur true 2>/dev/null; then | |
| echo "unshare -Ur: OK (user namespaces usable)" | |
| else | |
| echo "unshare -Ur: FAIL (user namespaces not usable here)" | |
| fi | |
| echo "== kernel config hint ==" | |
| zgrep CONFIG_USER_NS /proc/config.gz 2>/dev/null || true | |
| grep CONFIG_USER_NS "/boot/config-$(uname -r)" 2>/dev/null || true | |
| - name: Login to GHCR | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build libskybox.so in devcontainer (fail early on build) | |
| uses: devcontainers/ci@v0.3 | |
| with: | |
| configFile: .devcontainer/ubuntu/devcontainer.json | |
| imageName: ghcr.io/${{ github.repository }}/skybox-devcontainer | |
| cacheFrom: ghcr.io/${{ github.repository }}/skybox-devcontainer | |
| push: filter | |
| refFilterForPush: refs/heads/main | |
| runCmd: | | |
| set -euxo pipefail | |
| cargo clean | |
| cargo update | |
| cargo build -p skybox --release | |
| # Copy artifact into the workspace | |
| mkdir -p dist | |
| cp -f target/release/libskybox.so dist/libskybox.so | |
| ls -l dist/libskybox.so | |
| - name: Install libskybox.so into /usr/lib64/slurm on runner | |
| shell: bash | |
| run: | | |
| set -euxo pipefail | |
| sudo mkdir -p /usr/lib64/slurm | |
| sudo install -m 0755 dist/libskybox.so /usr/lib64/slurm/libskybox.so | |
| ls -l /usr/lib64/slurm/libskybox.so | |
| - name: Install Slurm + Munge (+ SlurmDBD + MariaDB) | |
| run: | | |
| set -euxo pipefail | |
| sudo apt-get update | |
| sudo apt-get install -y --no-install-recommends \ | |
| munge \ | |
| slurmctld slurmd slurm-client slurmdbd \ | |
| mariadb-server | |
| - name: Configure Munge | |
| run: | | |
| set -euxo pipefail | |
| # Create a fresh key | |
| sudo dd if=/dev/urandom of=/etc/munge/munge.key bs=1 count=1024 status=none | |
| sudo chown munge:munge /etc/munge/munge.key | |
| sudo chmod 0400 /etc/munge/munge.key | |
| # Start munge w new key | |
| sudo systemctl enable munge | |
| sudo systemctl restart munge | |
| sudo systemctl status --no-pager munge | |
| - name: Configure single-node Slurm (+ slurmdbd) | |
| run: | | |
| set -euxo pipefail | |
| HOST="$(hostname -s)" | |
| # Ensure HOST resolves locally! | |
| if ! getent hosts "$HOST" >/dev/null 2>&1; then | |
| echo "127.0.1.1 $HOST" | sudo tee -a /etc/hosts >/dev/null | |
| fi | |
| # Create slurm service user (if not present) | |
| if ! id -u slurm >/dev/null 2>&1; then | |
| sudo useradd -r -m -U -d /var/lib/slurm -s /usr/sbin/nologin slurm | |
| fi | |
| CPUS="$(nproc)" | |
| MEM_MB="$(awk '/MemTotal/ {print int($2/1024)}' /proc/meminfo)" | |
| sudo install -d -o slurm -g slurm /var/lib/slurm/slurmctld | |
| sudo install -d -o slurm -g slurm /var/lib/slurm/slurmd | |
| sudo install -d -o slurm -g slurm /var/log/slurm | |
| sudo install -d -o root -g root /etc/slurm | |
| # --- MariaDB bootstrap for Slurm accounting (minimal) --- | |
| sudo systemctl enable --now mariadb | |
| # Create DB + user SlurmDBD will use | |
| sudo mysql -e "CREATE DATABASE IF NOT EXISTS slurm_acct_db;" | |
| sudo mysql -e "CREATE USER IF NOT EXISTS 'slurm'@'localhost' IDENTIFIED BY 'slurm';" | |
| sudo mysql -e "GRANT ALL PRIVILEGES ON slurm_acct_db.* TO 'slurm'@'localhost';" | |
| sudo mysql -e "FLUSH PRIVILEGES;" | |
| # --- slurm.conf --- | |
| sudo tee /etc/slurm/slurm.conf >/dev/null <<EOF | |
| ClusterName=ci | |
| SlurmctldHost=${HOST} | |
| SlurmUser=slurm | |
| AuthType=auth/munge | |
| StateSaveLocation=/var/lib/slurm/slurmctld | |
| SlurmdSpoolDir=/var/lib/slurm/slurmd | |
| SlurmctldPort=6817 | |
| SlurmdPort=6818 | |
| # Spank plugin config | |
| PlugStackConfig=/etc/slurm/plugstack.conf | |
| # activate debug level for skybox | |
| SlurmdDebug=debug2 | |
| # no mpi | |
| MpiDefault=none | |
| # Scheduling/allocation | |
| SelectType=select/cons_tres | |
| SelectTypeParameters=CR_Core | |
| # No cgroup enforcement | |
| ProctrackType=proctrack/linuxproc | |
| TaskPlugin=task/none | |
| # Avoid fatal "Configured MailProg is invalid" on minimal images | |
| MailProg=/bin/true | |
| # Accounting via SlurmDBD | |
| AccountingStorageType=accounting_storage/slurmdbd | |
| # Logging | |
| SlurmctldLogFile=/var/log/slurm/slurmctld.log | |
| SlurmdLogFile=/var/log/slurm/slurmd.log | |
| # Single-node definition | |
| NodeName=${HOST} CPUs=${CPUS} Sockets=1 CoresPerSocket=2 ThreadsPerCore=2 RealMemory=${MEM_MB} State=UNKNOWN | |
| PartitionName=debug Nodes=${HOST} Default=YES MaxTime=INFINITE State=UP | |
| EOF | |
| # --- slurmdbd.conf (permissions MUST be strict) --- | |
| sudo tee /etc/slurm/slurmdbd.conf >/dev/null <<'EOF' | |
| AuthType=auth/munge | |
| DbdHost=localhost | |
| SlurmUser=slurm | |
| LogFile=/var/log/slurm/slurmdbd.log | |
| PidFile=/run/slurmdbd.pid | |
| StorageType=accounting_storage/mysql | |
| StorageHost=localhost | |
| StoragePort=3306 | |
| StorageUser=slurm | |
| StoragePass=slurm | |
| StorageLoc=slurm_acct_db | |
| EOF | |
| sudo chown slurm:slurm /etc/slurm/slurmdbd.conf | |
| sudo chmod 0600 /etc/slurm/slurmdbd.conf | |
| # /etc/slurm-llnl compatibility | |
| if [ -d /etc/slurm-llnl ] && [ ! -L /etc/slurm-llnl ]; then | |
| sudo rm -rf /etc/slurm-llnl | |
| fi | |
| if [ ! -e /etc/slurm-llnl ]; then | |
| sudo ln -s /etc/slurm /etc/slurm-llnl | |
| fi | |
| sudo chown -R slurm:slurm /var/lib/slurm /var/log/slurm | |
| echo "=== /etc/slurm/slurm.conf ===" | |
| sed -n '1,220p' /etc/slurm/slurm.conf | |
| echo "=== /etc/slurm/slurmdbd.conf (exists) ===" | |
| sudo ls -l /etc/slurm/slurmdbd.conf | |
| - name: Install SPANK plugstack + sarus-suite config (Zinal layout) | |
| run: | | |
| set -euxo pipefail | |
| # Create the directory structure | |
| sudo install -d -m 0755 /etc/slurm | |
| sudo install -d -m 0755 /etc/slurm/plugstack.conf.d | |
| sudo install -d -m 0755 /etc/sarus-suite | |
| # plugstack.conf | |
| sudo tee /etc/slurm/plugstack.conf >/dev/null <<'EOF' | |
| # | |
| # SPANK config file | |
| # | |
| include /etc/slurm/plugstack.conf.d/* | |
| EOF | |
| sudo chmod 0644 /etc/slurm/plugstack.conf | |
| # --- plugstack.conf.d entries (create all three) --- | |
| # skybox.conf | |
| sudo tee /etc/slurm/plugstack.conf.d/skybox.conf >/dev/null <<'EOF' | |
| required /usr/lib64/slurm/libskybox.so config_path=/etc/sarus-suite | |
| EOF | |
| sudo chmod 0644 /etc/slurm/plugstack.conf.d/skybox.conf | |
| # /etc/sarus-suite/sarus-suite.conf like in zinal | |
| sudo tee /etc/sarus-suite/sarus-suite.conf >/dev/null <<'EOF' | |
| edf_system_search_path = "/tmp/containers/edf" | |
| parallax_imagestore = "/tmp/.parallax_imagestore" | |
| parallax_path = "/usr/local/bin/parallax" | |
| parallax_mount_program = "/usr/local/bin/parallax-mount-program" | |
| skybox_enabled = true | |
| tracking_enabled = false | |
| tracking_tool = "/usr/local/opt/pyxis_tracking_tool/pyxis_tracking_tool" | |
| EOF | |
| sudo chmod 0644 /etc/sarus-suite/sarus-suite.conf | |
| echo "=== /etc/slurm/plugstack.conf ===" | |
| sudo cat /etc/slurm/plugstack.conf | |
| echo "=== /etc/slurm/plugstack.conf.d ===" | |
| sudo ls -la /etc/slurm/plugstack.conf.d | |
| echo "=== /etc/slurm/plugstack.conf.d/skybox.conf ===" | |
| sudo cat /etc/slurm/plugstack.conf.d/skybox.conf | |
| echo "=== /etc/sarus-suite/sarus-suite.conf ===" | |
| sudo cat /etc/sarus-suite/sarus-suite.conf | |
| - name: Start Slurm daemons (mariadb -> slurmdbd -> slurmctld -> slurmd) | |
| run: | | |
| set -euxo pipefail | |
| sudo systemctl enable --now mariadb | |
| sudo systemctl enable --now slurmdbd | |
| sudo systemctl enable --now slurmctld | |
| sudo systemctl enable --now slurmd | |
| sudo systemctl status --no-pager mariadb | |
| sudo systemctl status --no-pager slurmdbd | |
| sudo systemctl status --no-pager slurmctld | |
| sudo systemctl status --no-pager slurmd | |
| - name: Bootstrap accounting (cluster/account/user) | |
| run: | | |
| set -euxo pipefail | |
| # Create cluster to match ClusterName=ci (idempotent with || true) | |
| sudo sacctmgr -i add cluster ci || true | |
| # Create an account + associate the GitHub Actions user | |
| sudo sacctmgr -i add account default Description="CI default" Organization="ci" || true | |
| sudo sacctmgr -i add user runner Account=default || true | |
| echo "=== Associations ===" | |
| sudo sacctmgr -n show assoc format=Cluster,Account,User || true | |
| - name: Smoke test slurm (sinfo / scontrol / sbatch) | |
| run: | | |
| set -euxo pipefail | |
| echo "=== Initial cluster state ===" | |
| sinfo -Nel || true | |
| scontrol show node || true | |
| HOST="$(hostname -s)" | |
| echo "Waiting for node '$HOST' to become ready..." | |
| for i in $(seq 1 60); do | |
| state="$(sinfo -h -N -n "$HOST" -o "%T" 2>/dev/null || true)" | |
| echo "[$i/60] state='${state}'" | |
| case "$state" in | |
| idle|mix|alloc|completing) | |
| echo "Node is ready: $state" | |
| break | |
| ;; | |
| esac | |
| sleep 1 | |
| done | |
| echo "=== Post-wait cluster state ===" | |
| sinfo -Nel | |
| scontrol show node | |
| cat > job.sh <<'EOF' | |
| #!/usr/bin/env bash | |
| set -euxo pipefail | |
| echo "Hello from Slurm on $(hostname)" | |
| EOF | |
| chmod +x job.sh | |
| jid="$(sbatch --parsable -p debug -A default -J ci-test --wrap "./job.sh")" | |
| echo "Submitted: $jid" | |
| squeue -j "$jid" || true | |
| for i in $(seq 1 60); do | |
| state="$(scontrol show job -o "$jid" 2>/dev/null | sed -n 's/.*JobState=\([^ ]*\).*/\1/p' || true)" | |
| echo "[$i/60] JobState=${state:-UNKNOWN}" | |
| case "$state" in | |
| COMPLETED|FAILED|CANCELLED|TIMEOUT|NODE_FAIL|OUT_OF_MEMORY|PREEMPTED) | |
| break | |
| ;; | |
| "") | |
| # job no longer known to slurmctld (finished and purged) | |
| break | |
| ;; | |
| esac | |
| sleep 1 | |
| done | |
| echo "=== Job output (slurm-${jid}.out) ===" | |
| if [ -f "slurm-${jid}.out" ]; then | |
| cat "slurm-${jid}.out" | |
| else | |
| echo "No slurm-${jid}.out found in $(pwd); listing directory:" | |
| ls -la | |
| out_path="$(scontrol show job -dd "$jid" 2>/dev/null | sed -n 's/^ *StdOut=\(.*\)$/\1/p' | tail -n 1 || true)" | |
| if [ -n "$out_path" ] && [ -f "$out_path" ]; then | |
| echo "=== Job output (from StdOut=$out_path) ===" | |
| cat "$out_path" | |
| else | |
| echo "Could not locate job stdout file." | |
| fi | |
| fi | |
| - name: Install system dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y \ | |
| lsb-release \ | |
| wget \ | |
| tar \ | |
| gzip \ | |
| git \ | |
| btrfs-progs \ | |
| libbtrfs-dev \ | |
| lvm2 \ | |
| libdevmapper-dev \ | |
| squashfs-tools \ | |
| fuse-overlayfs \ | |
| inotify-tools \ | |
| build-essential | |
| - name: Install squashfuse + squashfuse_ll into /usr/bin (from tarball) | |
| shell: bash | |
| run: | | |
| set -euxo pipefail | |
| sudo apt-get update | |
| sudo apt-get install -y --no-install-recommends \ | |
| build-essential ca-certificates curl pkg-config \ | |
| libfuse3-dev zlib1g-dev liblzma-dev liblzo2-dev liblz4-dev libzstd-dev libattr1-dev \ | |
| tar | |
| work="$(mktemp -d)" | |
| curl -fsSL -o "$work/squashfuse.tar.gz" \ | |
| https://github.com/vasi/squashfuse/releases/download/0.6.1/squashfuse-0.6.1.tar.gz | |
| tar -xzf "$work/squashfuse.tar.gz" -C "$work" | |
| srcdir="$(find "$work" -maxdepth 1 -type d -name 'squashfuse-*' | head -n 1)" | |
| test -n "$srcdir" | |
| pushd "$srcdir" | |
| # Install into /usr so binaries go to /usr/bin | |
| ./configure --prefix=/usr | |
| # Build everything (including squashfuse_ll if supported) | |
| make -j"$(nproc)" | |
| # Install into /usr/bin | |
| sudo make install | |
| popd | |
| echo "=== Installed paths ===" | |
| ls -l /usr/bin/squashfuse || true | |
| ls -l /usr/bin/squashfuse_ll || true | |
| # Hard requirements | |
| test -x /usr/bin/squashfuse | |
| # If squashfuse_ll still isn't present, fail loudly with context | |
| if [ ! -x /usr/bin/squashfuse_ll ]; then | |
| echo "ERROR: /usr/bin/squashfuse_ll was not installed." | |
| echo "Build outputs containing 'll' (if any):" | |
| find "$srcdir" -maxdepth 2 -type f -name '*squashfuse*ll*' -o -name '*ll*.o' || true | |
| echo "Maybe the build disabled LL binary on this platform/config." | |
| exit 1 | |
| fi | |
| echo "=== Versions ===" | |
| /usr/bin/squashfuse --version || true | |
| /usr/bin/squashfuse_ll --version || true | |
| - name: Install Podman (static binary from mgoltzsche/podman-static) | |
| run: | | |
| set -euxo pipefail | |
| # Ensure we have curl + tar | |
| sudo apt-get update | |
| sudo apt-get install -y --no-install-recommends curl ca-certificates tar gzip | |
| # Ubuntu runners are amd64 | |
| curl -fsSL -o /tmp/podman-linux-amd64.tar.gz \ | |
| https://github.com/mgoltzsche/podman-static/releases/latest/download/podman-linux-amd64.tar.gz | |
| mkdir -p /tmp/podman-static | |
| tar -xzf /tmp/podman-linux-amd64.tar.gz -C /tmp/podman-static | |
| sudo cp -r /tmp/podman-static/podman-linux-amd64/usr /tmp/podman-static/podman-linux-amd64/etc / | |
| # Sanity check | |
| podman --version | |
| podman info || true | |
| - name: Tool versions (podman + fuse/squash tools) | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| echo "=== OS / Kernel ===" | |
| uname -a || true | |
| cat /etc/os-release || true | |
| echo | |
| # Helper: print "name: path" then try common version flags | |
| show_ver() { | |
| local label="$1"; shift | |
| local bin="$1"; shift || true | |
| echo "--- ${label} ---" | |
| if [[ "$bin" == /* ]]; then | |
| if [[ ! -x "$bin" ]]; then | |
| echo "path: $bin (missing or not executable)" | |
| echo | |
| return 0 | |
| fi | |
| echo "path: $bin" | |
| else | |
| if ! command -v "$bin" >/dev/null 2>&1; then | |
| echo "path: (not found in PATH)" | |
| echo | |
| return 0 | |
| fi | |
| echo "path: $(command -v "$bin")" | |
| fi | |
| # Try a few common patterns; stop at the first that works. | |
| if "$bin" --version >/dev/null 2>&1; then | |
| "$bin" --version | |
| elif "$bin" version >/dev/null 2>&1; then | |
| "$bin" version | |
| elif "$bin" -V >/dev/null 2>&1; then | |
| "$bin" -V | |
| elif "$bin" -v >/dev/null 2>&1; then | |
| "$bin" -v | |
| else | |
| echo "version: (no supported version flag detected)" | |
| fi | |
| echo | |
| } | |
| echo "=== Podman static binaries (/usr/local/bin) ===" | |
| show_ver "podman (static)" /usr/local/bin/podman | |
| show_ver "crun (static)" /usr/local/bin/crun | |
| show_ver "runc (static)" /usr/local/bin/runc | |
| show_ver "fuse-overlayfs (static)" /usr/local/bin/fuse-overlayfs | |
| show_ver "fusermount3 (static)" /usr/local/bin/fusermount3 | |
| show_ver "pasta (static)" /usr/local/bin/pasta | |
| show_ver "pasta.avx2 (static)" /usr/local/bin/pasta.avx2 | |
| echo "=== System binaries (PATH) you care about ===" | |
| show_ver "fuse-overlayfs (PATH)" fuse-overlayfs | |
| show_ver "fusermount3 (PATH)" fusermount3 | |
| show_ver "crun (PATH)" crun | |
| show_ver "runc (PATH)" runc | |
| show_ver "podman (PATH)" podman | |
| show_ver "slirp4netns (PATH)" slirp4netns | |
| show_ver "netavark (PATH)" netavark | |
| show_ver "aardvark-dns (PATH)" aardvark-dns | |
| echo "=== output for squashfuse ===" | |
| squashfuse --version |& head -n 1 || true | |
| echo | |
| echo "=== Podman view of runtime / helpers (podman info) ===" | |
| /usr/local/bin/podman info --debug || true | |
| - name: Rootless Podman host prerequisites (uidmap + subuid/subgid + slirp4netns) | |
| run: | | |
| set -euxo pipefail | |
| # 1) newuidmap/newgidmap (rootless prerequisite) | |
| # 3) slirp4netns (common rootless networking helper) | |
| sudo apt-get update | |
| sudo apt-get install -y --no-install-recommends uidmap slirp4netns | |
| # 2) Ensure subordinate ID ranges exist for the runner user | |
| if ! grep -qE '^runner:' /etc/subuid; then | |
| echo "runner:100000:65536" | sudo tee -a /etc/subuid >/dev/null | |
| fi | |
| if ! grep -qE '^runner:' /etc/subgid; then | |
| echo "runner:100000:65536" | sudo tee -a /etc/subgid >/dev/null | |
| fi | |
| # Sanity checks | |
| command -v newuidmap | |
| command -v newgidmap | |
| podman info | |
| - name: Fix AppArmor profile for /usr/local/bin/podman | |
| run: | | |
| set -euxo pipefail | |
| if command -v apparmor_parser >/dev/null 2>&1 && [ -f /etc/apparmor.d/podman ]; then | |
| sudo sed -Ei 's!^profile podman /usr/bin/podman !profile podman /usr/{bin,local/bin}/podman !' /etc/apparmor.d/podman | |
| sudo apparmor_parser -r /etc/apparmor.d/podman | |
| else | |
| echo "No apparmor_parser or /etc/apparmor.d/podman found; skipping AppArmor adjustment." | |
| fi | |
| - name: Write /etc/containers/containers.conf (global) | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| sudo mkdir -p /etc/containers | |
| sudo tee /etc/containers/containers.conf >/dev/null <<'EOF' | |
| [engine] | |
| runtime = "crun" | |
| [engine.runtimes] | |
| crun = ["/usr/local/bin/crun"] | |
| EOF | |
| - name: Create Podman module hpc | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| # Resolve actual installed binary paths | |
| CRUN_PATH="$(command -v crun)" | |
| CONMON_PATH="$(command -v conmon)" | |
| echo "Detected crun: ${CRUN_PATH}" | |
| echo "Detected conmon: ${CONMON_PATH}" | |
| # Create the modules directory Podman searches | |
| sudo mkdir -p /etc/containers/containers.conf.modules | |
| # Write the 'hpc' module file | |
| sudo tee /etc/containers/containers.conf.modules/hpc >/dev/null <<EOF | |
| [containers] | |
| ipcns = "host" | |
| netns = "host" | |
| pidns = "host" | |
| utsns = "host" | |
| #userns = "keep-id" | |
| cgroupns = "host" | |
| cgroups = "no-conmon" | |
| tz = "local" | |
| # Mounting /tmp as a temporary measure to propagate Slurm PMIx dirs until we have dedicated hook | |
| mounts = ["type=bind,src=/tmp,dst=/tmp"] | |
| [engine] | |
| runtime = "crun" | |
| conmon_path = ["${CONMON_PATH}"] | |
| [engine.runtimes] | |
| crun = ["${CRUN_PATH}"] | |
| EOF | |
| # Sanity check: show what we wrote | |
| sudo ls -l /etc/containers/containers.conf.modules/hpc | |
| sudo sed -n '1,200p' /etc/containers/containers.conf.modules/hpc | |
| - name: Rootless Podman smoke test | |
| run: | | |
| set -euxo pipefail | |
| podman --version | |
| podman info | |
| # Rootless sanity checks | |
| podman system migrate || true | |
| # Run a tiny container rootless | |
| podman run --rm docker.io/library/alpine:3.20 echo "ok (rootless podman)" | |
| - name: Install Parallax + mount program (v0.9.10 release binaries) | |
| run: | | |
| set -euxo pipefail | |
| sudo install -d /usr/local/bin | |
| # parallax binary | |
| sudo curl -fsSL \ | |
| -o /usr/local/bin/parallax \ | |
| https://github.com/sarus-suite/parallax/releases/download/v0.9.12/parallax-v0.9.12-ubuntu-24.04-amd64 | |
| sudo chmod 0755 /usr/local/bin/parallax | |
| # mount program (script) | |
| sudo curl -fsSL \ | |
| -o /usr/local/bin/parallax-mount-program \ | |
| https://github.com/sarus-suite/parallax/releases/download/v0.9.12/parallax-mount-program-v0.9.12.sh | |
| sudo chmod 0755 /usr/local/bin/parallax-mount-program | |
| # sanity | |
| parallax --version || true | |
| /usr/local/bin/parallax-mount-program --help >/dev/null 2>&1 || true | |
| - name: Install Bats locally | |
| run: | | |
| set -euxo pipefail | |
| TMP_DIR="$(mktemp -d)" | |
| echo "TMP_DIR=$TMP_DIR" >> "$GITHUB_ENV" | |
| git clone --depth 1 https://github.com/bats-core/bats-core.git "$TMP_DIR/bats-core" | |
| "$TMP_DIR/bats-core/install.sh" "$TMP_DIR/bats" | |
| - name: Parallax basic smoke test (with rootless Podman) | |
| run: | | |
| set -euxo pipefail | |
| export PARALLAX_MP_UID=$(id -u) | |
| export PARALLAX_MP_GID=$(id -g) | |
| export PARALLAX_MP_SQUASHFUSE_CMD=/usr/bin/squashfuse_ll | |
| export PODMAN_BINARY="$(command -v podman)" | |
| export PARALLAX_BINARY="$(command -v parallax)" | |
| export MOUNT_PROGRAM_PATH="$(command -v parallax-mount-program)" | |
| export MKSQUASHFS_PATH="$(command -v mksquashfs)" | |
| export PODMAN_ROOT="$(mktemp -d)" | |
| export PODMAN_RUNROOT="$(mktemp -d)" | |
| export RO_STORAGE="$(mktemp -d)" | |
| export CLEAN_ROOT="$(mktemp -d)" | |
| "$PODMAN_BINARY" --root "$PODMAN_ROOT" --runroot "$PODMAN_RUNROOT" pull busybox:latest | |
| "$PARALLAX_BINARY" \ | |
| --podmanRoot "$PODMAN_ROOT" \ | |
| --roStoragePath "$RO_STORAGE" \ | |
| --mksquashfsPath "$MKSQUASHFS_PATH" \ | |
| --log-level info \ | |
| --migrate \ | |
| --image busybox:latest | |
| "$PODMAN_BINARY" \ | |
| --root "$CLEAN_ROOT" \ | |
| --runroot "$PODMAN_RUNROOT" \ | |
| --storage-opt additionalimagestore="$RO_STORAGE" \ | |
| --storage-opt mount_program="$MOUNT_PROGRAM_PATH" \ | |
| images | |
| "$PODMAN_BINARY" \ | |
| --root "$CLEAN_ROOT" \ | |
| --runroot "$PODMAN_RUNROOT" \ | |
| --storage-opt additionalimagestore="$RO_STORAGE" \ | |
| --storage-opt mount_program="$MOUNT_PROGRAM_PATH" \ | |
| run --rm busybox:latest echo "ok (parallax ro-store)" | |
| # Cleanup (best-effort) | |
| "$PODMAN_BINARY" \ | |
| --root "$PODMAN_ROOT" \ | |
| --runroot "$PODMAN_RUNROOT" \ | |
| --storage-opt mount_program="$MOUNT_PROGRAM_PATH" \ | |
| rmi --all || true | |
| # Try the container exec under Slurm | |
| srun -p debug -A default -J srun-podman-rostore -n 1 bash -lc ' | |
| "$PODMAN_BINARY" \ | |
| --root "$CLEAN_ROOT" \ | |
| --runroot "$PODMAN_RUNROOT" \ | |
| --storage-opt additionalimagestore="$RO_STORAGE" \ | |
| --storage-opt mount_program="$MOUNT_PROGRAM_PATH" \ | |
| run --rm busybox:latest echo "ok (from srun + parallax ro-store)" | |
| ' | |
| # EDF | |
| mkdir -p "$HOME/.edf" | |
| cat > "$HOME/.edf/busybox.toml" <<'EOF' | |
| image = "busybox:latest" | |
| EOF | |
| echo "=== $HOME/.edf/busybox.toml ===" | |
| cat "$HOME/.edf/busybox.toml" | |
| set +e | |
| # dev pts crash debug | |
| #/usr/local/bin/podman --log-level=debug --module hpc \ | |
| #"$PODMAN_BINARY" --module hpc \ | |
| # run --rm \ | |
| # --runtime-flag log=/tmp/crun.log \ | |
| # busybox:latest echo "default ok" || rc=$? | |
| #echo "podman rc=${rc:-0}" | |
| #sudo sed -n '1,200p' /tmp/crun.log || true | |
| #/usr/local/bin/podman --log-level=debug --module hpc \ | |
| # "$PODMAN_BINARY" --module hpc \ | |
| # run --rm --pid=private\ | |
| # --runtime-flag log=/tmp/crun.log \ | |
| # busybox:latest echo "private ok" || rc=$? | |
| #echo "podman rc=${rc:-0}" | |
| #sudo sed -n '1,200p' /tmp/crun.log || true | |
| rm /tmp/parallax-*/mount_program.log | |
| "$PODMAN_BINARY" --module hpc \ | |
| --root "$CLEAN_ROOT" \ | |
| --runroot "$PODMAN_RUNROOT" \ | |
| --storage-opt additionalimagestore="$RO_STORAGE" \ | |
| --storage-opt mount_program="$MOUNT_PROGRAM_PATH" \ | |
| run --rm --userns=keep-id \ | |
| --runtime-flag log=/tmp/crun.log \ | |
| busybox:latest stat -c '%U %u %G %g %n' /bin/sh | |
| ## output mount program logs | |
| echo "=== mount program log tail ===" | |
| sudo tail -n 200 /tmp/parallax-*/mount_program.log || true | |
| sleep 2 | |
| srun -p debug -t 3 -A default -J srun-skybox --chdir=/tmp -n 1 --edf=busybox echo 'ok :D' | |
| ## Test squashfuse parametrization permissions | |
| cat > "$HOME/.edf/ubuntu.toml" <<'EOF' | |
| image = "library/ubuntu:22.04" | |
| EOF | |
| echo "=== $HOME/.edf/ubuntu.toml ===" | |
| cat "$HOME/.edf/ubuntu.toml" | |
| srun -p debug -t 3 -A default -J srun-skybox-2 -n 1 --edf=ubuntu \ | |
| stat -c '%U %u %G %g %n' /etc/os-release /bin/sh | |
| ## output mount program logs | |
| echo "=== mount program log tail ===" | |
| # sudo tail -n 200 /tmp/parallax-*/mount_program.log || true | |
| ## output all slurm related logs that are relevant to debug skybox | |
| echo "=== slurmd log tail ===" | |
| # sudo tail -n 200 /var/log/slurm/slurmd.log || true | |
| echo "=== slurmctld log tail ===" | |
| # sudo tail -n 200 /var/log/slurm/slurmctld.log || true | |
| # test annotation of mount program | |
| rm $HOME/.edf/busybox.toml | |
| cat > "$HOME/.edf/busybox.toml" <<'EOF' | |
| image = "busybox:latest" | |
| [annotations] | |
| com.sarus.parallax_mp_logfile = "/tmp/test_logfile" | |
| com.sarus.parallax_mp_squashfuse_path = "squashfuse_ll" | |
| EOF | |
| echo "=== $HOME/.edf/busybox.toml ===" | |
| cat "$HOME/.edf/busybox.toml" | |
| srun -p debug -t 3 -A default -J logfile-test -n 1 --edf=busybox true | |
| test -s /tmp/test_logfile | |
| tail -n 200 /tmp/test_logfile || true | |
| sudo tail -n 200 /var/log/slurm/slurmd.log || true | |