Add Pixi lockfiles to ensure Python environment reproducibility

.github/workflows/build-push-docker.yaml

@@ -10,16 +10,19 @@ on:
       - "*"
     paths:
       - "Dockerfile.*"
-      - "dask-worker/*"
-      - "jupyterhub/*"
-      - "jupyterlab/*"
-      - "nebari-workflow-controller/*"
+
+      - "*/apt.txt"
+      - "*/postBuild"
 
       - "scripts/*"
 
       - ".github/workflows/build-push-docker.yaml"
     tags:
       - "*"
+  workflow_run:
+    workflows: ["Update Pixi Lockfiles"]
+    types:
+      - completed
 
 env:
   DOCKER_ORG: nebari

.github/workflows/test-images.yaml

@@ -5,14 +5,17 @@ on:
     paths:
       - "Dockerfile.*"
 
-      - "dask-worker/*"
-      - "jupyterhub/*"
-      - "jupyterlab/*"
+      - "*/apt.txt"
+      - "*/postBuild"
 
       - "scripts/*"
 
       - ".github/workflows/build-push-docker.yaml"
       - ".github/workflows/test-images.yaml"
+  workflow_run:
+    workflows: ["Update Pixi Lockfiles"]
+    types:
+      - completed
 
 env:
   DOCKER_ORG: nebari

.github/workflows/update-lockfile.yaml

@@ -0,0 +1,56 @@
+name: Update Pixi Lockfiles
+
+on:
+  workflow_dispatch: null
+  push:
+    branches:
+    - "*"
+    paths:
+    - "*/pixi.toml"
+    - ".github/workflows/update-lockfile.yaml"
+
+jobs:
+  check-manifests:
+    name: "Check Pixi Manifests"
+    runs-on: ubuntu-latest
+
+    outputs:
+      manifests: ${{ steps.changed-manifests.outputs.all_changed_files }}
+
+    steps:
+      - name: "Checkout Repository 🛎️"
+        uses: actions/checkout@v4
+
+      - name: "Check changed Pixi manifests"
+        id: changed-manifests
+        uses: tj-actions/changed-files@v46
+        with:
+          files: "*/pixi.toml"
+          matrix: true
+
+  update-lockfiles:
+    name: "Update Pixi Lockfile"
+    runs-on: ubuntu-latest
+    needs:
+      - check-manifests
+    if: needs.check-manifests.outputs.manifests != '[]'
+    strategy:
+      matrix:
+        manifest: ${{ fromJson(needs.check-manifests.outputs.manifests) }}
+
+    steps:
+      - name: "Checkout Repository 🛎️"
+        uses: actions/checkout@v4
+
+      - name: "Setup Pixi"
+        uses: prefix-dev/[email protected]
+        with:
+          run-install: false
+
+      - name: "Update Pixi Lockfile"
+        run: pixi lock --manifest-path ${{ matrix.manifest }}
+
+      - uses: stefanzweifel/git-auto-commit-action@v5
+        with:
+          commit_message: "Apply automatic changes to Pixi lockfile for ${{ matrix.environment }} environment"
+          file_pattern: "*/pixi.lock"

Dockerfile.dask-worker

@@ -7,7 +7,7 @@
 # docker build -f Dockerfile.dask-worker -t nebari-dask-worker:latest .
 
 ARG BASE_IMAGE=ubuntu:20.04
 FROM $BASE_IMAGE
 LABEL MAINTAINER="Nebari development team"
 
 COPY scripts/install-apt-minimal.sh /opt/scripts/install-apt-minimal.sh
@@ -15,27 +15,25 @@
 
 COPY scripts/fix-permissions /opt/scripts/fix-permissions
 
-ENV MAMBAFORGE_VERSION 4.13.0-1
-ENV MAMBAFORGE_AARCH64_SHA256 69e3c90092f61916da7add745474e15317ed0dc6d48bfe4e4c90f359ba141d23
-ENV MAMBAFORGE_X86_64_SHA256 412b79330e90e49cf7e39a7b6f4752970fcdb8eb54b1a45cc91afe6777e8518c
 SHELL ["/bin/bash", "-c"]
+ENV DEFAULT_ENV=default
 
-ENV PATH=/opt/conda/bin:${PATH}:/opt/scripts
+ENV PATH=/opt/dask-worker/.pixi/envs/${DEFAULT_ENV}/bin:/opt/scripts:${PATH}
 
-# ============== base install ===============
-COPY scripts/install-conda.sh /opt/scripts/install-conda.sh
-
-RUN /opt/scripts/install-conda.sh
+# ========== Install Pixi ============
+RUN curl -fsSL https://pixi.sh/install.sh | bash
+ENV PATH=~/.pixi/bin:${PATH}
 
 # ========== dask-worker install ===========
-COPY dask-worker/environment.yaml /opt/dask-worker/environment.yaml
-COPY scripts/install-conda-environment.sh /opt/scripts/install-conda-environment.sh
-RUN /opt/scripts/install-conda-environment.sh /opt/dask-worker/environment.yaml 'false'
+COPY dask-worker/pixi.toml /opt/dask-worker/pixi.toml
+COPY dask-worker/pixi.lock /opt/dask-worker/pixi.lock
+RUN pixi install --manifest-path /opt/dask-worker/ --locked && \
+    pixi clean --manifest-path /opt/dask-worker/ cache -y
+
+COPY dask-worker/postBuild /opt/dask-worker/postBuild
+RUN /opt/dask-worker/postBuild
 
 # ========== Setup GPU Paths ============
 ENV LD_LIBRARY_PATH=/usr/local/nvidia/lib64
 ENV NVIDIA_PATH=/usr/local/nvidia/bin
 ENV PATH="$NVIDIA_PATH:$PATH"
-
-COPY dask-worker /opt/dask-worker
-RUN /opt/dask-worker/postBuild

Dockerfile.jupyterhub

@@ -14,23 +14,22 @@
 
 COPY scripts/fix-permissions /opt/scripts/fix-permissions
 
-ENV MAMBAFORGE_VERSION 4.13.0-1
-ENV MAMBAFORGE_AARCH64_SHA256 69e3c90092f61916da7add745474e15317ed0dc6d48bfe4e4c90f359ba141d23
-ENV MAMBAFORGE_X86_64_SHA256 412b79330e90e49cf7e39a7b6f4752970fcdb8eb54b1a45cc91afe6777e8518c
 SHELL ["/bin/bash", "-c"]
+ENV DEFAULT_ENV=default
 
-ENV PATH="/opt/conda/bin:$PATH:/opt/scripts"
+ENV PATH=/opt/jupyterhub/.pixi/envs/${DEFAULT_ENV}/bin:/opt/scripts:${PATH}
 
-# ============== base install ===============
-COPY scripts/install-conda.sh /opt/scripts/install-conda.sh
-RUN /opt/scripts/install-conda.sh
+# ========== Install Pixi ============
+RUN curl -fsSL https://pixi.sh/install.sh | bash
+ENV PATH=~/.pixi/bin:${PATH}
 
 # ========== jupyterhub install ===========
-COPY jupyterhub/environment.yaml /opt/jupyterhub/environment.yaml
-COPY scripts/install-conda-environment.sh /opt/scripts/install-conda-environment.sh
-RUN /opt/scripts/install-conda-environment.sh /opt/jupyterhub/environment.yaml 'false'
+COPY jupyterhub/pixi.toml /opt/jupyterhub/pixi.toml
+COPY jupyterhub/pixi.lock /opt/jupyterhub/pixi.lock
+RUN pixi install --manifest-path /opt/jupyterhub/ --locked && \
+    pixi clean --manifest-path /opt/jupyterhub/ cache -y
 
-COPY jupyterhub /opt/jupyterhub
+COPY jupyterhub/postBuild /opt/jupyterhub/postBuild
 RUN /opt/jupyterhub/postBuild
 
 WORKDIR /srv/jupyterhub

Dockerfile.jupyterlab

@@ -7,7 +7,7 @@
 # docker build -f Dockerfile.jupyterlab -t nebari-jupyterlab:latest .
 
 ARG BASE_IMAGE=ubuntu:20.04
 FROM $BASE_IMAGE
 LABEL MAINTAINER="Nebari development team"
 
 ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
@@ -17,25 +17,20 @@
 
 COPY scripts/fix-permissions /opt/scripts/fix-permissions
 
-ENV MAMBAFORGE_VERSION 4.13.0-1
-ENV MAMBAFORGE_AARCH64_SHA256 69e3c90092f61916da7add745474e15317ed0dc6d48bfe4e4c90f359ba141d23
-ENV MAMBAFORGE_X86_64_SHA256 412b79330e90e49cf7e39a7b6f4752970fcdb8eb54b1a45cc91afe6777e8518c
 SHELL ["/bin/bash", "-c"]
-ENV CONDA_DIR=/opt/conda \
-    DEFAULT_ENV=default
+ENV DEFAULT_ENV=default
 # Set timezone
 ENV TZ=America/Chicago
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 
-# Set PATH for Dockerfile so that conda works and some useful scripts are
-# available. Any changes intended to propagate to runtime containers should be
-# set in /etc/profile.d (see setup_shell_behavior.sh)
-ENV PATH=/opt/conda/envs/${DEFAULT_ENV}/bin:/opt/conda/bin:${PATH}:/opt/scripts
+# Set PATH for Dockerfile so that tools in the pixi env and some useful 
+# scripts are available. Any changes intended to propagate to runtime 
+# containers should be set in /etc/profile.d (see setup_shell_behavior.sh)
+ENV PATH=/opt/jupyterlab/.pixi/envs/${DEFAULT_ENV}/bin:/opt/scripts:${PATH}
 
-# ============= base install ===============
-# install conda
-COPY scripts/install-conda.sh /opt/scripts/install-conda.sh
-RUN echo "${SHELL}"; env; cat ~/.bashrc; cat ~/.profile ; /opt/scripts/install-conda.sh
+# ========== Install Pixi ============
+RUN curl -fsSL https://pixi.sh/install.sh | bash
+ENV PATH=~/.pixi/bin:${PATH}
 
 # ========== jupyterlab install ============
 COPY jupyterlab/apt.txt /opt/jupyterlab/apt.txt
@@ -46,22 +41,17 @@
 COPY scripts/install-gitlfs.sh /opt/scripts/install-gitlfs.sh
 RUN /opt/scripts/install-gitlfs.sh
 
-ARG SKIP_CONDA_SOLVE=no
-COPY scripts/install-conda-environment.sh /opt/scripts/install-conda-environment.sh
-COPY jupyterlab/environment.yaml /opt/jupyterlab/environment.yaml
-RUN \
-    if [ "${SKIP_CONDA_SOLVE}" != "no" ];then  \
-    ENV_FILE=/opt/jupyterlab/conda-linux-64.lock ; \
-    else  \
-    ENV_FILE=/opt/jupyterlab/environment.yaml ; \
-    fi ; \
-    /opt/scripts/install-conda-environment.sh "${ENV_FILE}" 'true'
+# Install environment using Pixi
+COPY jupyterlab/pixi.toml /opt/jupyterlab/pixi.toml
+COPY jupyterlab/pixi.lock /opt/jupyterlab/pixi.lock
+RUN pixi install --manifest-path /opt/jupyterlab/ --locked && \
+    pixi clean --manifest-path /opt/jupyterlab/ cache -y
 
 # ========== code-server install ============
-ENV PATH=/opt/conda/envs/${DEFAULT_ENV}/share/code-server/bin:${PATH}
+ENV PATH=/opt/jupyterlab/.pixi/envs/${DEFAULT_ENV}/share/code-server/bin:${PATH}
 COPY scripts/install-code-server.sh /opt/scripts/install-code-server.sh
 
-COPY jupyterlab /opt/jupyterlab
+COPY jupyterlab/postBuild /opt/jupyterlab/postBuild
 RUN /opt/jupyterlab/postBuild
 
 # ========== Setup GPU Paths ============

Dockerfile.workflow-controller

@@ -17,25 +17,20 @@ RUN /opt/scripts/install-apt-minimal.sh
 
 COPY scripts/fix-permissions /opt/scripts/fix-permissions
 
-ENV MAMBAFORGE_VERSION 4.13.0-1
-ENV MAMBAFORGE_AARCH64_SHA256 69e3c90092f61916da7add745474e15317ed0dc6d48bfe4e4c90f359ba141d23
-ENV MAMBAFORGE_X86_64_SHA256 412b79330e90e49cf7e39a7b6f4752970fcdb8eb54b1a45cc91afe6777e8518c
 SHELL ["/bin/bash", "-c"]
-ENV CONDA_DIR=/opt/conda \
-    DEFAULT_ENV=default
+ENV DEFAULT_ENV=default
 # Set timezone
 ENV TZ=America/Chicago
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 
 # Set PATH for Dockerfile so that conda works and some useful scripts are
 # available. Any changes intended to propagate to runtime containers should be
 # set in /etc/profile.d (see setup_shell_behavior.sh)
-ENV PATH=/opt/conda/envs/${DEFAULT_ENV}/bin:/opt/conda/bin:${PATH}:/opt/scripts
+ENV PATH=/opt/nebari-workflow-controller/.pixi/envs/${DEFAULT_ENV}/bin:/opt/scripts:${PATH}
 
-# ============= base install ===============
-# install conda
-COPY scripts/install-conda.sh /opt/scripts/install-conda.sh
-RUN echo "${SHELL}"; env; cat ~/.bashrc; cat ~/.profile ; /opt/scripts/install-conda.sh
+# ========== Install Pixi ============
+RUN curl -fsSL https://pixi.sh/install.sh | bash
+ENV PATH=~/.pixi/bin:${PATH}
 
 # ========== nebari-workflow-controller install ============
 COPY scripts/install-apt.sh /opt/scripts/install-apt.sh
@@ -45,17 +40,10 @@ RUN /opt/scripts/install-apt.sh
 # uncomment to install dev dependencies
 # RUN /opt/scripts/install-apt.sh /opt/nebari-workflow-controller/apt.txt  
 
-ARG SKIP_CONDA_SOLVE=no
-COPY scripts/install-conda-environment.sh /opt/scripts/install-conda-environment.sh
-COPY nebari-workflow-controller/environment.yaml /opt/nebari-workflow-controller/environment.yaml
-RUN \
-    if [ "${SKIP_CONDA_SOLVE}" != "no" ];then  \
-    ENV_FILE=/opt/nebari-workflow-controller/conda-linux-64.lock ; \
-    else  \
-    ENV_FILE=/opt/nebari-workflow-controller/environment.yaml ; \
-    fi ; \
-    /opt/scripts/install-conda-environment.sh "${ENV_FILE}" 'true'
+# ========== nebari-workflow-controller install ===========
+COPY nebari-workflow-controller/pixi.toml /opt/nebari-workflow-controller/pixi.toml
+COPY nebari-workflow-controller/pixi.lock /opt/nebari-workflow-controller/pixi.lock
+RUN pixi install --manifest-path /opt/nebari-workflow-controller/ --locked && \
+    pixi clean --manifest-path /opt/nebari-workflow-controller/ cache -y
 
-COPY nebari-workflow-controller /opt/nebari-workflow-controller
-
-CMD ["python", "-m", "nebari_workflow_controller"]
+CMD ["python", "-m", "nebari_workflow_controller"]