diff --git a/.github/workflows/github-actions-ci.yml b/.github/workflows/github-actions-ci.yml index 1ba7e434..372c67b8 100644 --- a/.github/workflows/github-actions-ci.yml +++ b/.github/workflows/github-actions-ci.yml @@ -30,7 +30,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - sudo apt-get update + sudo apt-get update sudo apt-get install -y hdf5-tools curl python -m pip install --upgrade pip pip install -r requirements.txt @@ -51,35 +51,12 @@ jobs: runs-on: ["self-hosted"] strategy: matrix: - image: ["ipfx_py39:latest", "ipfx_py311:latest"] + pyver: ["3.9" , "3.11"] steps: - uses: actions/checkout@v4 - - name: run test in docker + - name: Build docker image run: | - docker run \ - --env-file ~/env.list \ - --mount type=bind,source=$PWD,target=/local1/github_worker,bind-propagation=rshared \ - -v /data/informatics/module_test_data/:/data/informatics/module_test_data/ \ - -v /allen/aibs/informatics/module_test_data/:/allen/aibs/informatics/module_test_data/ \ - -v /allen/programs/celltypes/production/mousecelltypes/:/allen/programs/celltypes/production/mousecelltypes/ \ - -v /allen/programs/celltypes/workgroups/279/:/allen/programs/celltypes/workgroups/279/ \ - -v /allen/programs/celltypes/production/humancelltypes/:/allen/programs/celltypes/production/humancelltypes/ \ - --workdir /local1/github_worker --rm \ - --user 1001:1001 \ - ${{ matrix.image }} \ - /bin/bash -c "python -m venv .venv; \ - source .venv/bin/activate; \ - pip install --upgrade pip; \ - pip install numpy; \ - pip install -r requirements.txt; \ - export TEST_API_ENDPOINT=http://api.brain-map.org; \ - export SKIP_LIMS=false; \ - export TEST_INHOUSE=true; \ - export ALLOW_TEST_DOWNLOADS=true; \ - export IPFX_TEST_TIMEOUT=60; \ - pip install -r requirements-test.txt; \ - git config lfs.url 'https://github.com/AllenInstitute/ipfx.git/info/lfs'; \ - git lfs env; \ - git lfs pull; \ - python -m pytest \ - --junitxml=test-reports/test.xml --verbose" \ No newline at end of file + docker build --build-arg PYTHON_VERSION="${{ matrix.pyver }}" -t "ipfx_${{ matrix.pyver }}:latest" docker + - name: Run test in docker + run: | + DOCKER_IMAGE="ipfx_${{ matrix.pyver }}:latest" ./docker/run_tests_with_docker.sh diff --git a/.github/workflows/nightly-onprem.yml b/.github/workflows/nightly-onprem.yml index 02a67f7a..1f8680d0 100644 --- a/.github/workflows/nightly-onprem.yml +++ b/.github/workflows/nightly-onprem.yml @@ -10,33 +10,15 @@ jobs: runs-on: ["self-hosted"] strategy: matrix: - image: ["ipfx_py39:latest", "ipfx_py311:latest"] + pyver: ["3.9" , "3.11"] branch: ["master", "feature/**"] steps: - uses: actions/checkout@v4 with: ref: ${{ matrix.branch }} - - name: run test in docker + - name: Build docker image run: | - docker run \ - --mount type=bind,source=$PWD,target=/local1/github_worker,bind-propagation=rshared \ - -v /data/informatics/module_test_data/:/data/informatics/module_test_data/ \ - -v /allen/aibs/informatics/module_test_data/:/allen/aibs/informatics/module_test_data/ \ - -v /allen/programs/celltypes/production/mousecelltypes/:/allen/programs/celltypes/production/mousecelltypes/ \ - -v /allen/programs/celltypes/workgroups/279/:/allen/programs/celltypes/workgroups/279/ \ - -v /allen/programs/celltypes/production/humancelltypes/:/allen/programs/celltypes/production/humancelltypes/ \ - --workdir /local1/github_worker --rm \ - --user 1001:1001 \ - ${{ matrix.image }} \ - /bin/bash -c "python -m venv .venv; \ - source .venv/bin/activate; \ - pip install --upgrade pip; \ - pip install numpy; \ - pip install -r requirements.txt; \ - export TEST_API_ENDPOINT=http://api.brain-map.org; \ - pip install -r requirements-test.txt; \ - git config lfs.url 'https://github.com/AllenInstitute/ipfx.git/info/lfs'; \ - git lfs env; \ - git lfs pull; \ - python -m pytest \ - --junitxml=test-reports/test.xml --verbose" + docker build --build-arg PYTHON_VERSION="${{ matrix.pyver }}" -t "ipfx_${{ matrix.pyver }}:latest" docker + - name: Run test in docker + run: | + DOCKER_IMAGE="ipfx_${{ matrix.pyver }}:latest" ./docker/run_tests_with_docker.sh diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..ec19e870 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,22 @@ +ARG PYTHON_VERSION=XXX +FROM python:${PYTHON_VERSION} + +ARG APP_UID=1001 + +ARG APP_GID=1001 + +RUN apt-get update \ + && apt-get install -y \ + hdf5-tools \ + curl \ + git-lfs \ + && rm -rf /var/lib/apt/lists/* + +RUN groupadd -g "$APP_GID" ci && \ + useradd -l -u "$APP_UID" -g "$APP_GID" -ms /bin/bash ci + +RUN curl https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash + +USER ci + +WORKDIR /home/ci diff --git a/docker/README.md b/docker/README.md index 73ca607d..1dd6598a 100644 --- a/docker/README.md +++ b/docker/README.md @@ -9,13 +9,5 @@ Our light and on-prem tests are defined in [our workflow file](../.github/workfl ## Building images -If you are an Allen Institute developer, you will have instructions on how to access the machine running the IPFX self-hosted runner. - -On this machine you can create the Docker image corresponding to the Python versions we test on by running: - -``` -cd ipfx/docker/py39 -docker build -t ipfx_py39:latest . -``` - -And this should be sufficient for the on-prem tests defined in our GitHub workflow to run. +If you are an Allen Institute developer, you will have instructions on how to access the machine running the +IPFX self-hosted runner. diff --git a/docker/py311/Dockerfile b/docker/py311/Dockerfile deleted file mode 100644 index 0eaac054..00000000 --- a/docker/py311/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM python:3.11 - -RUN apt-get update \ - && apt-get install -y \ - hdf5-tools \ - curl \ - git-lfs \ - && rm -rf /var/lib/apt/lists/* - -RUN curl https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash \ No newline at end of file diff --git a/docker/py39/Dockerfile b/docker/py39/Dockerfile deleted file mode 100644 index c52d2229..00000000 --- a/docker/py39/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM python:3.9 - -RUN apt-get update \ - && apt-get install -y \ - hdf5-tools \ - curl \ - git-lfs \ - && rm -rf /var/lib/apt/lists/* - -RUN curl https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash \ No newline at end of file diff --git a/docker/run_tests_with_docker.sh b/docker/run_tests_with_docker.sh new file mode 100755 index 00000000..92cd9d40 --- /dev/null +++ b/docker/run_tests_with_docker.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +set -e + +docker run \ + --env-file ~/env.list \ + --mount type=bind,source=$PWD,target=/local1/github_worker,bind-propagation=rshared \ + -v /data/informatics/module_test_data/:/data/informatics/module_test_data/ \ + -v /allen/aibs/informatics/module_test_data/:/allen/aibs/informatics/module_test_data/ \ + -v /allen/programs/celltypes/production/mousecelltypes/:/allen/programs/celltypes/production/mousecelltypes/ \ + -v /allen/programs/celltypes/workgroups/279/:/allen/programs/celltypes/workgroups/279/ \ + -v /allen/programs/celltypes/production/humancelltypes/:/allen/programs/celltypes/production/humancelltypes/ \ + --workdir /local1/github_worker --rm \ + --user 1001:1001 \ + ${DOCKER_IMAGE} \ + /bin/bash -c "python -m venv .venv; \ + source .venv/bin/activate; \ + pip install --upgrade pip; \ + pip install numpy; \ + pip install -r requirements.txt; \ + export TEST_API_ENDPOINT=http://api.brain-map.org; \ + export SKIP_LIMS=false; \ + export TEST_INHOUSE=true; \ + export ALLOW_TEST_DOWNLOADS=true; \ + export IPFX_TEST_TIMEOUT=60; \ + pip install -r requirements-test.txt; \ + git config lfs.url 'https://github.com/AllenInstitute/ipfx.git/info/lfs'; \ + git lfs env; \ + git lfs pull; \ + python -m pytest --junitxml=test-reports/test.xml --verbose" diff --git a/run_tests_with_docker.sh b/run_tests_with_docker.sh deleted file mode 100644 index 235c3f0a..00000000 --- a/run_tests_with_docker.sh +++ /dev/null @@ -1,11 +0,0 @@ -docker run -v ${PWD}:/root/ipfx \ - -v /data/informatics/module_test_data/:/data/informatics/module_test_data/ \ - -v /allen/aibs/informatics/module_test_data/:/allen/aibs/informatics/module_test_data/ \ - -v /allen/programs/celltypes/production/mousecelltypes/:/allen/programs/celltypes/production/mousecelltypes/ \ - --workdir /root/ipfx --rm -t --shm-size=8g \ - alleninstitute/allensdk_anaconda3 \ - /bin/bash --login -c "source activate py27; \ - export TEST_COMPLETE=true; \ - pip install -r requirements.txt; \ - pip install -r requirements-test.txt; \ - cd tests; py.test || exit 0"