Fix on premise tests

.github/workflows/github-actions-ci.yml

@@ -30,7 +30,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
         run: |
-          sudo apt-get update 
+          sudo apt-get update
           sudo apt-get install -y hdf5-tools curl
           python -m pip install --upgrade pip
           pip install -r requirements.txt
@@ -51,35 +51,12 @@ jobs:
     runs-on: ["self-hosted"]
     strategy:
       matrix:
-        image: ["ipfx_py39:latest", "ipfx_py311:latest"]
+        pyver: ["3.9" , "3.11"]
     steps:
       - uses: actions/checkout@v4
-      - name: run test in docker
+      - name: Build docker image
         run: |
-          docker run \
-          --env-file ~/env.list \
-          --mount type=bind,source=$PWD,target=/local1/github_worker,bind-propagation=rshared \
-          -v /data/informatics/module_test_data/:/data/informatics/module_test_data/ \
-          -v /allen/aibs/informatics/module_test_data/:/allen/aibs/informatics/module_test_data/ \
-          -v /allen/programs/celltypes/production/mousecelltypes/:/allen/programs/celltypes/production/mousecelltypes/ \
-          -v /allen/programs/celltypes/workgroups/279/:/allen/programs/celltypes/workgroups/279/ \
-          -v /allen/programs/celltypes/production/humancelltypes/:/allen/programs/celltypes/production/humancelltypes/ \
-          --workdir /local1/github_worker --rm \
-          --user 1001:1001 \
-          ${{ matrix.image }} \
-              /bin/bash -c "python -m venv .venv; \
-                            source .venv/bin/activate; \
-                            pip install --upgrade pip; \
-                            pip install numpy; \
-                            pip install -r requirements.txt; \
-                            export TEST_API_ENDPOINT=http://api.brain-map.org; \
-                            export SKIP_LIMS=false; \
-                            export TEST_INHOUSE=true; \
-                            export ALLOW_TEST_DOWNLOADS=true; \
-                            export IPFX_TEST_TIMEOUT=60; \
-                            pip install -r requirements-test.txt; \
-                            git config lfs.url 'https://github.com/AllenInstitute/ipfx.git/info/lfs'; \
-                            git lfs env; \
-                            git lfs pull; \
-                            python -m pytest \
-                              --junitxml=test-reports/test.xml --verbose"
+          docker build --build-arg PYTHON_VERSION="${{ matrix.pyver }}" -t "ipfx_${{ matrix.pyver }}:latest" docker
+      - name: Run test in docker
+        run: |
+          DOCKER_IMAGE="ipfx_${{ matrix.pyver }}:latest" ./docker/run_tests_with_docker.sh

.github/workflows/nightly-onprem.yml

@@ -10,33 +10,15 @@ jobs:
     runs-on: ["self-hosted"]
     strategy:
       matrix:
-        image: ["ipfx_py39:latest", "ipfx_py311:latest"]
+        pyver: ["3.9" , "3.11"]
         branch: ["master", "feature/**"]
     steps:
       - uses: actions/checkout@v4
         with:
           ref: ${{ matrix.branch }}
-      - name: run test in docker
+      - name: Build docker image
         run: |
-          docker run \
-          --mount type=bind,source=$PWD,target=/local1/github_worker,bind-propagation=rshared \
-          -v /data/informatics/module_test_data/:/data/informatics/module_test_data/ \
-          -v /allen/aibs/informatics/module_test_data/:/allen/aibs/informatics/module_test_data/ \
-          -v /allen/programs/celltypes/production/mousecelltypes/:/allen/programs/celltypes/production/mousecelltypes/ \
-          -v /allen/programs/celltypes/workgroups/279/:/allen/programs/celltypes/workgroups/279/ \
-          -v /allen/programs/celltypes/production/humancelltypes/:/allen/programs/celltypes/production/humancelltypes/ \
-          --workdir /local1/github_worker --rm \
-          --user 1001:1001 \
-          ${{ matrix.image }} \
-              /bin/bash -c "python -m venv .venv; \
-                            source .venv/bin/activate; \
-                            pip install --upgrade pip; \
-                            pip install numpy; \
-                            pip install -r requirements.txt; \
-                            export TEST_API_ENDPOINT=http://api.brain-map.org; \
-                            pip install -r requirements-test.txt; \
-                            git config lfs.url 'https://github.com/AllenInstitute/ipfx.git/info/lfs'; \
-                            git lfs env; \
-                            git lfs pull; \
-                            python -m pytest \
-                              --junitxml=test-reports/test.xml --verbose"
+          docker build --build-arg PYTHON_VERSION="${{ matrix.pyver }}" -t "ipfx_${{ matrix.pyver }}:latest" docker
+      - name: Run test in docker
+        run: |
+          DOCKER_IMAGE="ipfx_${{ matrix.pyver }}:latest" ./docker/run_tests_with_docker.sh

docker/Dockerfile

@@ -0,0 +1,22 @@
+ARG PYTHON_VERSION=XXX
+FROM python:${PYTHON_VERSION}
+
+ARG APP_UID=1001
+
+ARG APP_GID=1001
+
+RUN apt-get update                    \
+    && apt-get install -y             \
+        hdf5-tools                    \
+        curl                          \
+        git-lfs                       \
+        && rm -rf /var/lib/apt/lists/*
+
+RUN groupadd -g "$APP_GID" ci                               && \
+    useradd -l -u "$APP_UID" -g "$APP_GID" -ms /bin/bash ci
+
+RUN curl https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash
+
+USER ci
+
+WORKDIR /home/ci

docker/README.md

@@ -9,13 +9,5 @@ Our light and on-prem tests are defined in [our workflow file](../.github/workfl
 
 ## Building images
 
-If you are an Allen Institute developer, you will have instructions on how to access the machine running the IPFX self-hosted runner.
-
-On this machine you can create the Docker image corresponding to the Python versions we test on by running:
-
-```
-cd ipfx/docker/py39
-docker build -t ipfx_py39:latest .
-```
-
-And this should be sufficient for the on-prem tests defined in our GitHub workflow to run.
+If you are an Allen Institute developer, you will have instructions on how to access the machine running the
+IPFX self-hosted runner.

docker/run_tests_with_docker.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+set -e
+
+docker run                                                                                                       \
+    --env-file ~/env.list                                                                                        \
+    --mount type=bind,source=$PWD,target=/local1/github_worker,bind-propagation=rshared                          \
+    -v /data/informatics/module_test_data/:/data/informatics/module_test_data/                                   \
+    -v /allen/aibs/informatics/module_test_data/:/allen/aibs/informatics/module_test_data/                       \
+    -v /allen/programs/celltypes/production/mousecelltypes/:/allen/programs/celltypes/production/mousecelltypes/ \
+    -v /allen/programs/celltypes/workgroups/279/:/allen/programs/celltypes/workgroups/279/                       \
+    -v /allen/programs/celltypes/production/humancelltypes/:/allen/programs/celltypes/production/humancelltypes/ \
+    --workdir /local1/github_worker --rm                                                                         \
+    --user 1001:1001                                                                                             \
+    ${DOCKER_IMAGE}                                                                                              \
+      /bin/bash -c "python -m venv .venv;                                                     \
+                    source .venv/bin/activate;                                                \
+                    pip install --upgrade pip;                                                \
+                    pip install numpy;                                                        \
+                    pip install -r requirements.txt;                                          \
+                    export TEST_API_ENDPOINT=http://api.brain-map.org;                        \
+                    export SKIP_LIMS=false;                                                   \
+                    export TEST_INHOUSE=true;                                                 \
+                    export ALLOW_TEST_DOWNLOADS=true;                                         \
+                    export IPFX_TEST_TIMEOUT=60;                                              \
+                    pip install -r requirements-test.txt;                                     \
+                    git config lfs.url 'https://github.com/AllenInstitute/ipfx.git/info/lfs'; \
+                    git lfs env;                                                              \
+                    git lfs pull;                                                             \
+                    python -m pytest --junitxml=test-reports/test.xml --verbose"