SCANPY-217: Migrate from Cirrus CI to GitHub Action

.cirrus.yml

@@ -16,6 +16,7 @@ env:
   CRON_NIGHTLY_JOB_NAME: "nightly"
 
 only_if: $CIRRUS_USER_COLLABORATOR == 'true' && $CIRRUS_TAG == "" && ($CIRRUS_PR != "" || $CIRRUS_BRANCH == 'master' || $CIRRUS_BRANCH =~ "branch-.*")
+
 linux_container_definition: &LINUX_CONTAINER_DEFINITION
   eks_container:
     dockerfile: .cirrus/poetry.Dockerfile
@@ -31,57 +32,12 @@ linux_container_definition: &LINUX_CONTAINER_DEFINITION
     cpu: 3
     memory: 8G
 
-win_vm_definition: &WINDOWS_VM_DEFINITION
-  env:
-    JF_ALIAS: "jf"
-  ec2_instance:
-    experimental: true # see https://github.com/cirruslabs/cirrus-ci-docs/issues/1051
-    image: base-windows-jdk17-v*
-    platform: windows
-    region: eu-central-1
-    type: c6id.4xlarge
-    preemptible: false
-    use_ssd: true
-
-win_ssd_and_clone:
-  &WIN_SSD_AND_CLONE # copy&paste from https://github.com/SonarSource/sonar-cpp/blob/a8c6f1e45a12393508682a013ac7ee35eb92bece/.cirrus.yml#L45
-  prepare_disk_script:
-    - ps: |
-        Get-Disk -Number 2 | Initialize-Disk -PassThru | New-Partition -UseMaximumSize -DriveLetter Z
-        Format-Volume -DriveLetter Z -FileSystem NTFS -Confirm:$false
-    - echo "CIRRUS_WORKING_DIR=Z:/cirrus-ci-build" >> $CIRRUS_ENV
-  # we don't clone submodules because they are not needed for the tests
-  clone_script: |
-    git config --system core.longpaths true
-    if [ -z "$CIRRUS_PR" ]; then
-      git clone --branch=$CIRRUS_BRANCH https://x-access-token:${CIRRUS_REPO_CLONE_TOKEN}@github.com/${CIRRUS_REPO_FULL_NAME}.git $CIRRUS_WORKING_DIR
-      git reset --hard $CIRRUS_CHANGE_IN_REPO
-    else
-      git clone https://x-access-token:${CIRRUS_REPO_CLONE_TOKEN}@github.com/${CIRRUS_REPO_FULL_NAME}.git $CIRRUS_WORKING_DIR
-      git fetch origin pull/$CIRRUS_PR/head:pull/$CIRRUS_PR
-      git reset --hard $CIRRUS_CHANGE_IN_REPO
-    fi
-
 .jfrog_config_template: &JFROG_CONFIG_TEMPLATE
   jfrog_config_script:
     - $JF_ALIAS config add repox --artifactory-url "$ARTIFACTORY_URL" --access-token "$ARTIFACTORY_PRIVATE_ACCESS_TOKEN"
     - $JF_ALIAS poetry-config --server-id-resolve repox --repo-resolve sonarsource-pypi
     - $JF_ALIAS poetry install --build-name="$CIRRUS_REPO_NAME" --build-number="$CI_BUILD_NUMBER"
 
-poetry_win_install: &POETRY_WIN_INSTALL
-  <<: *WINDOWS_VM_DEFINITION
-  <<: *WIN_SSD_AND_CLONE
-  env:
-    PYTHON_VERSION: 3.12.1
-    POETRY_VERSION: 2.0.1
-  jfrog_win_install_script:
-    - powershell "Start-Process -Wait -Verb RunAs powershell '-NoProfile iwr https://releases.jfrog.io/artifactory/jfrog-cli/v2-jf/[RELEASE]/jfrog-cli-windows-amd64/jf.exe -OutFile $env:SYSTEMROOT\system32\jf.exe'"
-    - jf intro
-  poetry_win_install_script:
-    - source cirrus-env QA
-    - pip install poetry=="$POETRY_VERSION"
-  <<: *JFROG_CONFIG_TEMPLATE
-
 poetry_cache_template: &POETRY_CACHE
   poetry_cache:
     folder: ~/.cache/poetry/
@@ -93,11 +49,6 @@ poetry_cache_template: &POETRY_CACHE
   poetry_install_script:
     - poetry install
 
-.poetry_set_version_template: &POETRY_SET_VERSION
-  poetry_set_version_script:
-    - source set_poetry_build_version "$CI_BUILD_NUMBER"
-    - echo "PROJECT_VERSION=$PROJECT_VERSION" >> $CIRRUS_ENV
-
 .poetry_macos_template: &POETRY_MACOS_TEMPLATE
   <<: *POETRY_CACHE
   jfrog_install_script:
@@ -120,104 +71,6 @@ macos_worker_template: &MACOS_WORKER_DEFINITION
     labels:
       envname: prod
 
-formatting_task:
-  alias: formatting
-  name: "Formatting"
-  <<: *LINUX_CONTAINER_DEFINITION
-  <<: *POETRY_INSTALL
-  formatting_script:
-    - poetry run black src/ tests/ --check
-    - poetry run licenseheaders -t license_header.tmpl -o "SonarSource SA" -y 2011-2024 -n "Sonar Scanner Python" -E .py -d src/
-    - poetry run licenseheaders -t license_header.tmpl -o "SonarSource SA" -y 2011-2024 -n "Sonar Scanner Python" -E .py -d tests/
-    - git diff --name-only --exit-code ./src ./tests
-
-documentation_task:
-  alias: documentation
-  name: "CLI Documentation"
-  <<: *LINUX_CONTAINER_DEFINITION
-  <<: *POETRY_INSTALL
-  cli_docs_script:
-    - poetry run python tools/generate_cli_documentation.py
-    - git diff --exit-code CLI_ARGS.md
-
-analysis_base_linux_template: &ANALYSIS_BASE_LINUX_TEMPLATE
-  <<: *LINUX_CONTAINER_DEFINITION
-  <<: *POETRY_INSTALL
-  # For analysis we don't need to set the build versions, but we still need to access jfrog to recover the dependencies
-  analysis_script:
-    - poetry run pytest --cov-report=xml:coverage.xml --cov-config=pyproject.toml --cov=src --cov-branch tests
-    - poetry run mypy src/ > mypy-report.txt || true # mypy exits with 1 if there are errors
-    - uv venv
-    - source .venv/bin/activate
-    - uv pip install pysonar
-    - . .cirrus/analysis.sh
-  always:
-    pytest_artifacts:
-      path: "coverage.xml"
-      format: junit
-      type: text/xml
-
-analysis_next_task:
-  <<: *ANALYSIS_BASE_LINUX_TEMPLATE
-  alias: sonar_analysis_next
-  name: "NEXT Analysis"
-  env:
-    SONAR_TOKEN: VAULT[development/kv/data/next data.token]
-    SONAR_HOST_URL: https://next.sonarqube.com/sonarqube
-
-analysis_SQC_EU_shadow_task:
-  <<: *ANALYSIS_BASE_LINUX_TEMPLATE
-  # only executed in CRON job AND on master branch
-  only_if: $CIRRUS_CRON == $CRON_NIGHTLY_JOB_NAME && $CIRRUS_BRANCH == "master"
-  alias: sonar_analysis_shadow_sqc_eu
-  name: "SQC-EU Shadow Analysis"
-  env:
-    SONAR_TOKEN: VAULT[development/kv/data/sonarcloud data.token]
-    SONAR_HOST_URL: https://sonarcloud.io
-
-analysis_SQC_US_shadow_task:
-  <<: *ANALYSIS_BASE_LINUX_TEMPLATE
-  # only executed in CRON job AND on master branch
-  only_if: $CIRRUS_CRON == $CRON_NIGHTLY_JOB_NAME && $CIRRUS_BRANCH == "master"
-  alias: sonar_analysis_shadow_sqc_us
-  name: "SQC-US Shadow Analysis"
-  env:
-    SONAR_TOKEN: VAULT[development/kv/data/sonarqube-us data.token]
-    SONAR_HOST_URL: https://sonarqube.us
-
-qa_task:
-  alias: qa
-  matrix:
-    - name: "Test Python 3.9.18"
-      eks_container:
-        docker_arguments:
-          PYTHON_VERSION: 3.9.18
-    - name: "Test Python 3.9.6"
-      eks_container:
-        docker_arguments:
-          PYTHON_VERSION: 3.9.6
-    - name: "Test Python 3.10"
-      eks_container:
-        docker_arguments:
-          PYTHON_VERSION: 3.10.13
-    - name: "Test Python 3.11"
-      eks_container:
-        docker_arguments:
-          PYTHON_VERSION: 3.11.7
-    - name: "Test Python 3.12"
-      eks_container:
-        docker_arguments:
-          PYTHON_VERSION: 3.12.1
-    - name: "Test Python 3.13"
-      eks_container:
-        docker_arguments:
-          PYTHON_VERSION: 3.13.2
-  <<: *LINUX_CONTAINER_DEFINITION
-  <<: *POETRY_INSTALL
-  <<: *POETRY_SET_VERSION
-  qa_script:
-    - poetry run pytest tests/
-
 qa_macos_task:
   alias: qa_macos
   only_if: $CIRRUS_CRON == "macos-its-cron"
@@ -238,35 +91,6 @@ qa_macos_task:
   test_313_script:
     - .cirrus/run_macos_tests.sh "3.13.2"
 
-qa_windows_task:
-  name: "Test Windows"
-  <<: *POETRY_WIN_INSTALL
-  <<: *POETRY_SET_VERSION
-  alias: qa_windows
-  qa_script:
-    - poetry run pytest tests/
-    - exit $LASTEXITCODE
-
-build_task:
-  <<: *LINUX_CONTAINER_DEFINITION
-  <<: *POETRY_INSTALL
-  <<: *POETRY_SET_VERSION
-  alias: build
-  name: "Build"
-  build_script: regular_poetry_build_publish
-
-its_task:
-  name: "Integration Tests"
-  alias: its
-  sonarqube_cache:
-    folder: sonarqube_cache/
-    populate_script: mkdir -p sonarqube_cache && wget -q https://binaries.sonarsource.com/Distribution/sonarqube/sonarqube-$SONARQUBE_VERSION.zip -O sonarqube_cache/sonarqube.zip
-    fingerprint_script: echo "sonarqube-$SONARQUBE_VERSION"
-  <<: *LINUX_CONTAINER_DEFINITION
-  <<: *POETRY_INSTALL
-  its_script:
-    - .cirrus/run_its.sh
-
 its_macos_task:
   name: "[macOS] Integration Tests"
   alias: its_macos
@@ -280,25 +104,7 @@ its_macos_task:
   its_script:
     - .cirrus/run_its.sh
 
-promote_task:
-  depends_on:
-    - formatting
-    - sonar_analysis_next
-    - qa
-    - qa_windows
-    - build
-    - its
-  env:
-    ARTIFACTORY_PROMOTE_ACCESS_TOKEN: VAULT[development/artifactory/token/${CIRRUS_REPO_OWNER}-${CIRRUS_REPO_NAME}-promoter access_token]
-    GITHUB_TOKEN: VAULT[development/github/token/${CIRRUS_REPO_OWNER}-${CIRRUS_REPO_NAME}-promotion token]
-  <<: *LINUX_CONTAINER_DEFINITION
-  <<: *POETRY_INSTALL
-  <<: *POETRY_SET_VERSION
-  promote_script: cirrus_promote
-
 run_iris_task:
-  depends_on:
-    - promote
   <<: *LINUX_CONTAINER_DEFINITION
   # only executed in CRON job AND on master branch
   only_if: $CIRRUS_CRON == $CRON_NIGHTLY_JOB_NAME && $CIRRUS_BRANCH == "master"

.github/actions/config-poetry/action.yml

@@ -0,0 +1,75 @@
+---
+name: Configure Poetry
+description: GitHub Action to configure a poetry project
+
+inputs:
+  python-version:
+    description: The version of python to use
+    default: 3.12.1
+  poetry-version:
+    description: The version of poetry to install
+    default: 2.2.1
+  jfrog-version:
+    description: The version of jFrog to install
+    default: 2.77.0
+  poetry-virtualenvs-path:
+    description: Path to the Poetry virtual environments, relative to GitHub workspace. The folder is cached only if it is a subdirectory of
+      `poetry-cache-dir`.
+    default: .cache/pypoetry/virtualenvs
+  poetry-cache-dir:
+    description: Path to the Poetry cache directory, relative to GitHub workspace.
+    default: .cache/pypoetry
+outputs:
+  BUILD_NUMBER:
+    description: The build number, incremented or reused if already cached
+    value: ${{ steps.get_build_number.outputs.BUILD_NUMBER }}
+
+runs:
+  using: composite
+  steps:
+    - name: Set build parameters
+      shell: bash
+      env:
+        ARTIFACTORY_READER_ROLE: private-reader
+      run: |
+        echo "ARTIFACTORY_READER_ROLE=${ARTIFACTORY_READER_ROLE}" >> "$GITHUB_ENV"
+    - uses: SonarSource/ci-github-actions/get-build-number@v1
+      id: get_build_number
+    - name: Cache local Poetry cache
+      uses: SonarSource/ci-github-actions/cache@v1
+      with:
+        path: ${{ inputs.poetry-cache-dir }}
+        key: poetry-${{ runner.os }}-${{ hashFiles('poetry.lock') }}
+        restore-keys: poetry-${{ runner.os }}-
+    - name: Install mise and Python
+      uses: jdx/mise-action@5ac50f778e26fac95da98d50503682459e86d566 # v3.2.0
+      with:
+        version: 2025.7.12
+        install_args: "python@${{ inputs.python-version }}" 
+    - name: Install jfrog and poetry through mise
+      uses: jdx/mise-action@5ac50f778e26fac95da98d50503682459e86d566 # v3.2.0
+      with:
+        version: 2025.7.12
+        experimental: true # needed to use the http backend for installation of jfrog on windows
+    - name: Vault
+      # yamllint disable rule:line-length
+      id: secrets
+      uses: SonarSource/vault-action-wrapper@320bd31b03e5dacaac6be51bbbb15adf7caccc32 # 3.1.0
+      with:
+        secrets: |
+          development/artifactory/token/{REPO_OWNER_NAME_DASH}-${{ env.ARTIFACTORY_READER_ROLE }} access_token | ARTIFACTORY_ACCESS_TOKEN;
+      # yamllint enable rule:line-length
+    - name: Config Poetry
+      id: config
+      shell: bash
+      env:
+        ARTIFACTORY_URL: https://repox.jfrog.io/artifactory
+        ARTIFACTORY_PYPI_REPO: sonarsource-pypi
+        ARTIFACTORY_ACCESS_TOKEN: ${{ fromJSON(steps.secrets.outputs.vault).ARTIFACTORY_ACCESS_TOKEN }}
+        POETRY_VIRTUALENVS_PATH: ${{ github.workspace }}/${{ inputs.poetry-virtualenvs-path }}
+        POETRY_CACHE_DIR: ${{ github.workspace }}/${{ inputs.poetry-cache-dir }}
+      run: |
+        echo "POETRY_VIRTUALENVS_PATH=${POETRY_VIRTUALENVS_PATH}" >> "$GITHUB_ENV"
+        echo "POETRY_CACHE_DIR=${POETRY_CACHE_DIR}" >> "$GITHUB_ENV"
+        ${GITHUB_ACTION_PATH}/config-poetry.sh
+

.github/actions/config-poetry/config-poetry.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+# Config script for SonarSource Poetry projects.
+
+set -euo pipefail
+
+: "${ARTIFACTORY_URL:?}"
+: "${ARTIFACTORY_PYPI_REPO:?}" "${ARTIFACTORY_ACCESS_TOKEN:?}" 
+: "${BUILD_NUMBER:?}" "${GITHUB_REPOSITORY:?}" 
+
+set_build_env() {
+  export PROJECT=${GITHUB_REPOSITORY#*/}
+  echo "PROJECT: $PROJECT"
+}
+
+config_poetry() {
+  jf config add repox --artifactory-url "$ARTIFACTORY_URL" --access-token "$ARTIFACTORY_ACCESS_TOKEN"
+  jf poetry-config --server-id-resolve repox --repo-resolve "$ARTIFACTORY_PYPI_REPO"
+  jf poetry install --build-name="$PROJECT" --build-number="$BUILD_NUMBER"
+}
+
+main() {
+  set_build_env
+  config_poetry
+}
+
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
+  main "$@"
+fi

.github/scripts/run_its.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+unzip -q sonarqube_cache/sonarqube.zip -d sonarqube
+
+PLATFORM="linux-x86-64"
+if [[ "$(uname)" == "Darwin" ]]; then
+  PLATFORM="macosx-universal-64"
+fi
+
+cd $(ls -d sonarqube/*/)
+./bin/${PLATFORM}/sonar.sh start
+cd -
+
+unset SONAR_TOKEN
+unset SONAR_HOST_URL
+
+poetry install 
+poetry run pytest --its tests/its