feat: Add CI/CD pipeline, testing framework, Slurm environment setup,…

.devcontainer/Dockerfile

@@ -8,6 +8,25 @@ ENV PYTHONDONTWRITEBYTECODE=1
 # This option has no effect on the stdin stream.
 ENV PYTHONUNBUFFERED=1
 
+RUN apt-get update \
+    && apt-get -y install --no-install-recommends \
+    apt-transport-https \
+    ca-certificates \
+    curl \
+    gnupg2 \
+    software-properties-common \
+   && curl -fsSL https://download.docker.com/linux/debian/gpg | sudo apt-key add - \
+   && add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable" \
+   && apt-get update \
+   && apt-get -y install --no-install-recommends docker-ce \
+   # Clean up
+   && apt-get autoremove -y \
+   && apt-get clean -y \
+   && rm -rf /var/lib/apt/lists/*
+
+# Symlink docker socket
+RUN ln -s "/var/run/docker-host.sock" "/var/run/docker.sock"
+
 # [Optional] If your requirements rarely change, uncomment this section to add them to the image.
 COPY ./backend/requirements.txt /tmp/pip-tmp/
 RUN pip install --upgrade pip wheel setuptools \

.devcontainer/devcontainer.json

@@ -50,8 +50,10 @@
 			]
 		}
 	},
+	"mounts": [
+		"source=/var/run/docker.sock,target=/var/run/docker-host.sock,type=bind"
+	],
 	"containerEnv": {
-		//"PYTHONPATH": "/workspaces/process_api/backend:$PYTHONPATH"
 		"PYTHONPATH": "/workspaces/${localWorkspaceFolderBasename}:$PYTHONPATH"
 	}
 	// Features to add to the dev container. More info: https://containers.dev/features.

.devcontainer/docker-compose.yml

@@ -11,6 +11,8 @@ services:
       - ./datasets:/datasets # datasets repo
       - ./processes:/processes
       - ./logs:/logs
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
 
     # Overrides default command so things don't shut down after the process ends.
     command: sleep infinity

.github/workflows/push_docker_images.yml

@@ -0,0 +1,55 @@
+name: Build and Push Docker Images
+
+on:
+  push:
+    branches:
+      - 'main'
+    tags:
+      - 'v*'
+  pull_request:
+    branches:
+      - 'main'
+
+jobs:
+
+  build_backend:
+    name: Build and Push Backend
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v3
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        with:
+          images: linea/orchestration
+          tags: |
+            type=raw,enable=${{ startsWith(github.ref, 'refs/tags/v') }},priority=100,prefix=backend_,value=latest
+            type=semver,enable=true,priority=200,prefix=backend_v,suffix=,pattern={{version}}
+            type=sha,enable=true,priority=300,prefix=backend_,suffix=,format=short
+            type=raw,priority=400,prefix=backend_dev_,value=latest
+          flavor: |
+            latest=false
+
+      - name: Log in to Docker Hub
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        with:
+          context: ./backend
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          # build-args: |
+          #   USERNAME=${{ secrets.PZ_USERNAME }}
+          #   USERID=${{ secrets.PZ_USERID }}
+          #   GROUPID=${{ secrets.PZ_GROUPID }}

.github/workflows/test.yml

@@ -0,0 +1,40 @@
+name: Test API workflow
+
+on: [push]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    name: Test python API
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up uWSGI dependencies
+      uses: awalsh128/cache-apt-pkgs-action@latest
+      with:
+        packages: gcc python3-dev build-essential libpcre3 libpcre3-dev
+        version: 1.0
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v4
+      with:
+        # Semantic version range syntax or exact version of a Python version
+        python-version: '3.10'
+        # Optional - x64 or x86 architecture, defaults to x64
+        architecture: 'x64'
+    # You can test your matrix by printing the current Python version
+    - name: Display Python version
+      run: python -c "import sys; print(sys.version)"    
+    - name: Install requirements
+      working-directory: ./backend
+      run: |
+        python -m pip install --upgrade pip wheel
+        pip install -r requirements.txt
+    - name: Create log dir
+      run: mkdir -p /tmp/logs
+    - name: Run tests and collect coverage
+      working-directory: ./backend    
+      run: pytest --cov=./ --cov-report=xml
+    - name: Upload coverage reports to Codecov with GitHub Action
+      uses: codecov/codecov-action@v3
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+

README.md

@@ -2,7 +2,7 @@
 
 Orchestration tool for the Brazilian IDAC applications.  
 
-## Setup Develop Enviroment
+## Setup Enviroment
 
 Clone the repository and access the directory:
 
@@ -71,7 +71,6 @@ $ docker-compose run backend python -c "import secrets; print(secrets.token_urls
 Copy the generated key, replace the `SECRET_KEY` variable value in the `.env` file and uncomment it by removing the "#".
 
 
-
 Create the Django superuser.
 
 Enter in container:
@@ -117,7 +116,7 @@ Point your browser to <http://localhost/admin/oauth2_provider/application/add/>
 - `Algorithm`: keep the default option (No OIDC support)
 
 
-WARNING!!! 
+WARNING!!!
 Fill the form as show in the screenshot below, and before saving take note of **Client id** and **Client secret** we will use it in a minute.
 
 
@@ -259,3 +258,33 @@ docker-compose exec backend bash
 # with the service turned off
 docker-compose run backend bash
 ```
+
+## Setup Slurm Enviroment
+
+### What is it?
+
+Internal application that interacts with the Slurm cluster and must be installed on a submission node. The application is a consumer of rabbitmq, and with each pipeline submission, a Slurm submission file is created and the process is finally submitted.
+
+*note: This installation requires an orchestration running on the same submission node.*
+
+Clone the repository and create a directory that will serve as the base for the application:
+```bash
+git clone https://github.com/linea-it/orchestration.git
+mkdir slurm-worker
+```
+
+Copy the `slurm_env.sh` file and edit the environment variables as needed:
+```bash
+cp orchestration/backend/sh/slurm_env.sh slurm-worker/env.sh
+vi slurm-worker/env.sh    # edit file
+```
+
+With Conda loaded, run:
+```bash
+source slurm-worker/env.sh
+```
+
+And start the application:
+```bash
+slurm_worker_manager start
+```

backend/Dockerfile

@@ -70,6 +70,7 @@ ARG GROUPID=1000
 ARG BASEDIR=/app
 
 ENV BASE_DIR ${BASEDIR}
+ENV LOG_DIR ${LOG_DIR}
 
 RUN mkdir -p ${BASE_DIR}
 
@@ -81,7 +82,7 @@ RUN mkdir -p ${BASE_DIR}/django_static \
     ${BASE_DIR}/django_static/rest_framework \
     ${BASE_DIR}/django_static/admin \
     ${BASE_DIR}/core/migrations \
-    && chmod -R g+w ${BASE_DIR}
+    && chmod -R g+w ${BASE_DIR} 
 
 # Create group
 RUN groupadd --gid ${GROUPID} $USERNAME

backend/core/tests/test_processes.py

@@ -0,0 +1,7 @@
+from rest_framework.test import APITestCase
+
+
+class ProcessesAPIViewTestCase(APITestCase):
+
+    def test_list_product(self):
+        self.assertEqual(1, 1)

backend/orchestration/settings.py

@@ -93,6 +93,12 @@
         "rest_framework.filters.OrderingFilter",
     ],
     "PAGE_SIZE": 10,
+    "TEST_REQUEST_DEFAULT_FORMAT": "json",
+    'TEST_REQUEST_RENDERER_CLASSES': [
+        'rest_framework.renderers.MultiPartRenderer',
+        'rest_framework.renderers.JSONRenderer',
+        'rest_framework.renderers.TemplateHTMLRenderer'
+    ]
 }
 
 MIDDLEWARE = [
@@ -193,7 +199,7 @@
 
 # Logging settings
 
-LOG_DIR = os.getenv("LOG_DIR", "/logs")
+LOG_DIR = os.getenv("LOG_DIR", "/tmp/logs")
 LOGGING_LEVEL = os.getenv("LOGGING_LEVEL", "INFO")
 
 LOGGING = {

backend/requirements.txt

@@ -15,5 +15,8 @@ psycopg2-binary==2.9.9
 pydantic==2.9.2
 pydantic_core==2.23.4
 python-crontab==3.2.0
+pytest==8.3.3
+pytest-cov==5.0.0
+pytest-django==4.9.0
 PyYAML==6.0.2
 requests==2.32.3

backend/setup.cfg

@@ -0,0 +1,2 @@
+[tool:pytest]
+DJANGO_SETTINGS_MODULE = orchestration.settings

backend/sh/slurm_env.sh

@@ -0,0 +1,34 @@
+#!/bin/bash --login
+
+export APP_DIR=         # e.g.: /data/apps/app.orch/dev
+export BASE_DIR=        # e.g.: ${APP_DIR}/orchestration
+export CONDA_PATH=      # e.g.: ${APP_DIR}/miniconda3/bin/activate
+export RABBITMQ_ENV=    # e.g.: /home/app.orch/rabbitmq/.env
+
+set -a
+. $RABBITMQ_ENV         # environment variables related to rabbitmq running on srvorch-dev server 
+. $BASE_DIR/.env
+set +a
+
+echo "Activating environment"
+
+source $CONDA_PATH || { echo "Failed to activate Conda environment"; exit 1; }
+
+if [ ! -d "$PIPELINES_DIR" ]; then
+    echo "Error: PIPELINES_DIR not defined."
+    exit 1
+fi
+
+HASENV=`conda env list | grep 'orchestration '`
+
+if [ -z "$HASENV" ]; then
+    echo "Create virtual environment..."
+    conda env create -f $BASE_DIR/backend/environment.yml
+    echo "Virtual environment created and packages installed."
+fi
+
+conda activate orchestration
+
+export PATH=$PATH:$BASE_DIR/backend/sh/
+
+umask g+w

docker-compose-development.yml

@@ -18,7 +18,7 @@ services:
       - ./pipelines:/pipelines # pipelines repo
       - ./datasets:/datasets # datasets repo
       - ./processes:/processes
-      - ./logs:/logs
+      - ./logs:${LOG_DIR}
       - ./db:/db
     depends_on:
       - rabbitmq