Merge pull request #45 from linea-it/mamba

Updated pipelines (Mamba env)

Author: crisingulani

README.md

@@ -291,11 +291,13 @@ cp orchestration/backend/sh/slurm_env.sh slurm-worker/env.sh
 vi slurm-worker/env.sh    # edit file
 ```
 
-With Conda loaded, run:
+With `micromamba` available in `PATH`, run:
 ```bash
 source slurm-worker/env.sh
 ```
 
+The script defaults to `MAMBA_ROOT_PREFIX=${APP_DIR}/micromamba` and will create the `orchestration` environment there if it does not exist.
+
 And start the application:
 ```bash
 slurm_worker_manager start

backend/Dockerfile

@@ -1,85 +1,51 @@
-# FROM debian:buster-slim as base
 FROM python:3.11-slim as base
 
-# # If this is set to a non-empty string, Python won’t try
-# # to write .pyc files on the import of source modules
 ENV PYTHONDONTWRITEBYTECODE=1
-
-# # Force the stdout and stderr streams to be unbuffered.
-# # This option has no effect on the stdin stream.
 ENV PYTHONUNBUFFERED=1
+ENV MAMBA_ROOT_PREFIX=/opt/micromamba
 
-# Create Group: conda
-RUN groupadd -r conda --gid 900
+RUN groupadd -r mamba --gid 900
 
-# Instalacao Miniconda
 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
     && apt-get -y install --no-install-recommends \
-    procps \
-    gcc \
-    ca-certificates \
-    wget \
-    python3-dev \
-    build-essential \
-    && apt-get clean \
+    procps gcc ca-certificates wget bzip2 python3-dev build-essential && \
+    apt-get clean \
     && rm -rf /var/lib/apt/lists/* \
-    && mkdir -p /opt/conda \
-    && wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O /opt/conda/miniconda.sh \
-    && bash /opt/conda/miniconda.sh -b -u -p /opt/conda \
-    && rm -rf /opt/conda/miniconda.sh \
-    && /opt/conda/bin/conda init bash \
-    && find /opt/conda/ -follow -type f -name '*.a' -delete \
-    && find /opt/conda/ -follow -type f -name '*.js.map' -delete \
-    && /opt/conda/bin/conda clean -afy \
-    && chmod =2775 /opt/conda \
-    && ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh
-
-ENV PATH /opt/conda/bin:$PATH
-
-#-------------- Python 3.11 Environment Stage --------------
-FROM base as env_base
-COPY ./environment.yml .
-
-# Set environment variable to automatically accept conda TOS
-ENV CONDA_ALWAYS_YES=true
-ENV CONDA_AUTO_ACTIVATE_BASE=false
+    && mkdir -p /opt/micromamba \
+    && wget -qO- https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj -C /usr/local/bin --strip-components=1 bin/micromamba && find /opt/micromamba/ -follow -type f -name '*.a' -delete \
+    && find /opt/micromamba/ -follow -type f -name '*.js.map' -delete \
+    && chmod =2775 /opt/micromamba
 
-# Accept conda terms of service using exact commands from error message
-RUN /opt/conda/bin/conda config --set always_yes true && \
-    /opt/conda/bin/conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/main && \
-    /opt/conda/bin/conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/r && \
-    /opt/conda/bin/conda config --set channel_priority flexible && \
-    /opt/conda/bin/conda update -n base -c defaults conda && \
-    /opt/conda/bin/conda env create -f environment.yml
+ENV PATH /usr/local/bin:/opt/micromamba/bin:$PATH
 
-# RUN conda install -c conda-forge uwsgi
+FROM base as env_base
+COPY ./environment.yml .
 
-# Install python packages
+RUN micromamba create --root-prefix "$MAMBA_ROOT_PREFIX" -y -f environment.yml \
+    && micromamba clean --all --yes
 COPY ./requirements.txt /tmp/pip-tmp/
-RUN /bin/bash --login -c "source /opt/conda/bin/activate \
-    && conda activate orchestration \
-    && pip install --upgrade pip wheel setuptools \
-    && pip --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt \
-    && rm -rf /tmp/pip-tmp"
+RUN micromamba run --root-prefix "$MAMBA_ROOT_PREFIX" -n orchestration pip install --upgrade pip wheel setuptools \
+    && micromamba run --root-prefix "$MAMBA_ROOT_PREFIX" -n orchestration pip --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt \
+    && rm -rf /tmp/pip-tmp \
+    && micromamba clean --all --yes
 
 FROM base
 
-# Python 3.11 environment
-COPY --chown=:conda --chmod=775 --from=env_base /opt/conda/envs/orchestration /opt/conda/envs/orchestration
+COPY --chown=:mamba --chmod=775 --from=env_base /opt/micromamba/envs/orchestration /opt/micromamba/envs/orchestration
 
-# add user so we can run things as non-root
-# https://code.visualstudio.com/remote/advancedcontainers/add-nonroot-user
 ARG USERNAME=orch
 ARG USERID=1000
 ARG GROUPID=1000
 ARG BASEDIR=/app
 
 ENV BASE_DIR ${BASEDIR}
 ENV LOG_DIR ${LOG_DIR}
+ENV ORCHESTRATION_ENV_PREFIX /opt/micromamba/envs/orchestration
+ENV MAMBA_ROOT_PREFIX ${BASEDIR}/.micromamba
+ENV PATH /opt/micromamba/envs/orchestration/bin:/usr/local/bin:/opt/micromamba/bin:$PATH
 
-RUN mkdir -p ${BASE_DIR}
+RUN mkdir -p ${BASE_DIR} ${BASE_DIR}/.micromamba
 
-# Copy app files into container
 WORKDIR $BASE_DIR
 COPY . $BASE_DIR
 
@@ -89,20 +55,14 @@ RUN mkdir -p ${BASE_DIR}/django_static \
     ${BASE_DIR}/core/migrations \
     && chmod -R g+w ${BASE_DIR}
 
-# Create group
 RUN groupadd --gid ${GROUPID} $USERNAME
-
-# Create user
 RUN useradd --uid ${USERID} --gid ${GROUPID} --shell /bin/bash --create-home ${USERNAME}
 RUN usermod -a -G ${GROUPID},900 ${USERNAME}
 RUN chown -R ${USERID}:${GROUPID} ${BASE_DIR}
 
 COPY --chmod=0775 ./sh/entrypoint.sh /entrypoint.sh
 
-# Switch to non-priviliged user and run app
 USER $USERNAME
-RUN /bin/bash --login -c "conda init bash \
-    && echo 'source ${BASE_DIR}/sh/env.sh' >> ~/.bashrc \
-    && source ~/.bashrc"
+RUN /bin/bash -lc "echo 'source ${BASE_DIR}/sh/env.sh' >> ~/.bashrc"
 
 ENTRYPOINT [ "/entrypoint.sh" ]

backend/environment.yml

@@ -1,6 +1,5 @@
 name: orchestration
 channels:
-  - defaults
   - conda-forge
 dependencies:
   - python=3.10.8

backend/sh/env.sh

@@ -1,4 +1,7 @@
 #!/bin/bash --login
 
-echo "Activating environment"
-conda activate orchestration
+export MICROMAMBA_BIN="${MICROMAMBA_BIN:-micromamba}"
+export ORCHESTRATION_ENV_PREFIX="${ORCHESTRATION_ENV_PREFIX:-/opt/micromamba/envs/orchestration}"
+export MAMBA_ROOT_PREFIX="${MAMBA_ROOT_PREFIX:-${BASE_DIR}/.micromamba}"
+export ORCHESTRATION_ENV_NAME="${ORCHESTRATION_ENV_NAME:-orchestration}"
+export PATH="${ORCHESTRATION_ENV_PREFIX}/bin:${PATH}"

backend/sh/slurm_env.sh

@@ -2,35 +2,37 @@
 
 export APP_DIR=         # e.g.: /data/apps/app.orch/dev
 export BASE_DIR=        # e.g.: ${APP_DIR}/orchestration
-export CONDA_PATH=      # e.g.: ${APP_DIR}/miniconda3/bin/activate
+export MAMBA_ROOT_PREFIX=   # e.g.: ${APP_DIR}/micromamba
+export MICROMAMBA_BIN=      # e.g.: /usr/local/bin/micromamba
 export RABBITMQ_ENV=    # e.g.: /home/app.orch/rabbitmq/.env
 export WORKER_NAME="slurm"
+export ORCHESTRATION_ENV_NAME="orchestration"
 
 set -a
-. $RABBITMQ_ENV         # environment variables related to rabbitmq running on srvorch-dev server
-. $BASE_DIR/.env
+. "$RABBITMQ_ENV"
+. "$BASE_DIR/.env"
 set +a
 
-echo "Activating environment"
+MICROMAMBA_BIN="${MICROMAMBA_BIN:-micromamba}"
+MAMBA_ROOT_PREFIX="${MAMBA_ROOT_PREFIX:-${APP_DIR}/micromamba}"
+ENV_PREFIX="${MAMBA_ROOT_PREFIX}/envs/${ORCHESTRATION_ENV_NAME}"
 
-# shellcheck source=/data/apps/app.orch/dev/miniconda3/bin/activate
-source $CONDA_PATH || { echo "Failed to activate Conda environment"; exit 1; }
+if ! command -v "$MICROMAMBA_BIN" >/dev/null 2>&1; then
+    echo "Failed to find micromamba in PATH. Set MICROMAMBA_BIN or update PATH."
+    exit 1
+fi
 
 if [ ! -d "$PIPELINES_DIR" ]; then
     echo "Error: PIPELINES_DIR not defined."
     exit 1
 fi
 
-HASENV=$(conda env list | grep 'orchestration ')
-
-if [ -z "$HASENV" ]; then
+if [ ! -d "$ENV_PREFIX" ]; then
     echo "Create virtual environment..."
-    conda env create -f $BASE_DIR/backend/environment.yml
+    "$MICROMAMBA_BIN" create --root-prefix "$MAMBA_ROOT_PREFIX" -y -f "$BASE_DIR/backend/environment.yml"
     echo "Virtual environment created and packages installed."
 fi
 
-conda activate orchestration
-
-export PATH=$PATH:$BASE_DIR/backend/sh/
+export PATH="$ENV_PREFIX/bin:$BASE_DIR/backend/sh/:$PATH"
 
 umask g+w

deploy/env.sh.template

@@ -3,13 +3,13 @@
 export APP_DIR=/scripts/app.orch/[dev|production]
 export BASE_DIR=${APP_DIR}/orchestration
 
-## Conda settings
 export XDG_CACHE_HOME=${APP_DIR}/.cache
-export CONDA_PKGS_DIRS=${APP_DIR}/.conda/pkgs
-export CONDA_PATH=${APP_DIR}/miniconda3/bin/activate
-source $CONDA_PATH
+export MAMBA_ROOT_PREFIX=${APP_DIR}/micromamba
+export MICROMAMBA_BIN=${MICROMAMBA_BIN:-micromamba}
+export ORCHESTRATION_ENV_NAME=${ORCHESTRATION_ENV_NAME:-slurm-worker}
 
-conda activate slurm-worker
+eval "$(${MICROMAMBA_BIN} shell hook --shell bash)"
+micromamba activate "${ORCHESTRATION_ENV_NAME}"
 
 export WORKER_NAME="slurm"
 export SLURM_EXECUTOR="slurm"
@@ -22,11 +22,8 @@ export PATH=$PATH:$BASE_DIR/backend/sh/
 
 umask g+w
 
-## TNO settings
-## Database Connections URI
 export DB_ADMIN_URI=
 export DB_CATALOG_URI=
 export EXECUTION_ENV="linea"
 export PLANETARY_EPHEMERIS_FILEPATH=
 export LEAP_SECONDS_FILEPATH=
-

deploy/environment.yml

@@ -1,6 +1,5 @@
 name: slurm-worker
 channels:
-  - defaults
   - conda-forge
 dependencies:
   - python=3.12

deploy/install_pipelines.sh

@@ -6,7 +6,11 @@ fi
 
 echo "The directory containing the pipelines is: " "$PIPELINES_DIR"
 
-source $(dirname $CONDA_EXE)/activate || { echo "Failed to activate Conda environment"; exit 1; }
+MICROMAMBA_BIN="${MICROMAMBA_BIN:-micromamba}"
+if ! command -v "$MICROMAMBA_BIN" >/dev/null 2>&1; then
+    echo "Failed to find micromamba in PATH"
+    exit 1
+fi
 
 for pipe in $( ls ${PIPELINES_DIR}/*/install.sh)
 do