diff --git a/.cirrus.yml b/.cirrus.yml index cca6191a..9aaba700 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -16,6 +16,7 @@ env: CRON_NIGHTLY_JOB_NAME: "nightly" only_if: $CIRRUS_USER_COLLABORATOR == 'true' && $CIRRUS_TAG == "" && ($CIRRUS_PR != "" || $CIRRUS_BRANCH == 'master' || $CIRRUS_BRANCH =~ "branch-.*") + linux_container_definition: &LINUX_CONTAINER_DEFINITION eks_container: dockerfile: .cirrus/poetry.Dockerfile @@ -31,57 +32,12 @@ linux_container_definition: &LINUX_CONTAINER_DEFINITION cpu: 3 memory: 8G -win_vm_definition: &WINDOWS_VM_DEFINITION - env: - JF_ALIAS: "jf" - ec2_instance: - experimental: true # see https://github.com/cirruslabs/cirrus-ci-docs/issues/1051 - image: base-windows-jdk17-v* - platform: windows - region: eu-central-1 - type: c6id.4xlarge - preemptible: false - use_ssd: true - -win_ssd_and_clone: - &WIN_SSD_AND_CLONE # copy&paste from https://github.com/SonarSource/sonar-cpp/blob/a8c6f1e45a12393508682a013ac7ee35eb92bece/.cirrus.yml#L45 - prepare_disk_script: - - ps: | - Get-Disk -Number 2 | Initialize-Disk -PassThru | New-Partition -UseMaximumSize -DriveLetter Z - Format-Volume -DriveLetter Z -FileSystem NTFS -Confirm:$false - - echo "CIRRUS_WORKING_DIR=Z:/cirrus-ci-build" >> $CIRRUS_ENV - # we don't clone submodules because they are not needed for the tests - clone_script: | - git config --system core.longpaths true - if [ -z "$CIRRUS_PR" ]; then - git clone --branch=$CIRRUS_BRANCH https://x-access-token:${CIRRUS_REPO_CLONE_TOKEN}@github.com/${CIRRUS_REPO_FULL_NAME}.git $CIRRUS_WORKING_DIR - git reset --hard $CIRRUS_CHANGE_IN_REPO - else - git clone https://x-access-token:${CIRRUS_REPO_CLONE_TOKEN}@github.com/${CIRRUS_REPO_FULL_NAME}.git $CIRRUS_WORKING_DIR - git fetch origin pull/$CIRRUS_PR/head:pull/$CIRRUS_PR - git reset --hard $CIRRUS_CHANGE_IN_REPO - fi - .jfrog_config_template: &JFROG_CONFIG_TEMPLATE jfrog_config_script: - $JF_ALIAS config add repox --artifactory-url "$ARTIFACTORY_URL" --access-token "$ARTIFACTORY_PRIVATE_ACCESS_TOKEN" - $JF_ALIAS poetry-config --server-id-resolve repox --repo-resolve sonarsource-pypi - $JF_ALIAS poetry install --build-name="$CIRRUS_REPO_NAME" --build-number="$CI_BUILD_NUMBER" -poetry_win_install: &POETRY_WIN_INSTALL - <<: *WINDOWS_VM_DEFINITION - <<: *WIN_SSD_AND_CLONE - env: - PYTHON_VERSION: 3.12.1 - POETRY_VERSION: 2.0.1 - jfrog_win_install_script: - - powershell "Start-Process -Wait -Verb RunAs powershell '-NoProfile iwr https://releases.jfrog.io/artifactory/jfrog-cli/v2-jf/[RELEASE]/jfrog-cli-windows-amd64/jf.exe -OutFile $env:SYSTEMROOT\system32\jf.exe'" - - jf intro - poetry_win_install_script: - - source cirrus-env QA - - pip install poetry=="$POETRY_VERSION" - <<: *JFROG_CONFIG_TEMPLATE - poetry_cache_template: &POETRY_CACHE poetry_cache: folder: ~/.cache/poetry/ @@ -93,11 +49,6 @@ poetry_cache_template: &POETRY_CACHE poetry_install_script: - poetry install -.poetry_set_version_template: &POETRY_SET_VERSION - poetry_set_version_script: - - source set_poetry_build_version "$CI_BUILD_NUMBER" - - echo "PROJECT_VERSION=$PROJECT_VERSION" >> $CIRRUS_ENV - .poetry_macos_template: &POETRY_MACOS_TEMPLATE <<: *POETRY_CACHE jfrog_install_script: @@ -120,104 +71,6 @@ macos_worker_template: &MACOS_WORKER_DEFINITION labels: envname: prod -formatting_task: - alias: formatting - name: "Formatting" - <<: *LINUX_CONTAINER_DEFINITION - <<: *POETRY_INSTALL - formatting_script: - - poetry run black src/ tests/ --check - - poetry run licenseheaders -t license_header.tmpl -o "SonarSource SA" -y 2011-2024 -n "Sonar Scanner Python" -E .py -d src/ - - poetry run licenseheaders -t license_header.tmpl -o "SonarSource SA" -y 2011-2024 -n "Sonar Scanner Python" -E .py -d tests/ - - git diff --name-only --exit-code ./src ./tests - -documentation_task: - alias: documentation - name: "CLI Documentation" - <<: *LINUX_CONTAINER_DEFINITION - <<: *POETRY_INSTALL - cli_docs_script: - - poetry run python tools/generate_cli_documentation.py - - git diff --exit-code CLI_ARGS.md - -analysis_base_linux_template: &ANALYSIS_BASE_LINUX_TEMPLATE - <<: *LINUX_CONTAINER_DEFINITION - <<: *POETRY_INSTALL - # For analysis we don't need to set the build versions, but we still need to access jfrog to recover the dependencies - analysis_script: - - poetry run pytest --cov-report=xml:coverage.xml --cov-config=pyproject.toml --cov=src --cov-branch tests - - poetry run mypy src/ > mypy-report.txt || true # mypy exits with 1 if there are errors - - uv venv - - source .venv/bin/activate - - uv pip install pysonar - - . .cirrus/analysis.sh - always: - pytest_artifacts: - path: "coverage.xml" - format: junit - type: text/xml - -analysis_next_task: - <<: *ANALYSIS_BASE_LINUX_TEMPLATE - alias: sonar_analysis_next - name: "NEXT Analysis" - env: - SONAR_TOKEN: VAULT[development/kv/data/next data.token] - SONAR_HOST_URL: https://next.sonarqube.com/sonarqube - -analysis_SQC_EU_shadow_task: - <<: *ANALYSIS_BASE_LINUX_TEMPLATE - # only executed in CRON job AND on master branch - only_if: $CIRRUS_CRON == $CRON_NIGHTLY_JOB_NAME && $CIRRUS_BRANCH == "master" - alias: sonar_analysis_shadow_sqc_eu - name: "SQC-EU Shadow Analysis" - env: - SONAR_TOKEN: VAULT[development/kv/data/sonarcloud data.token] - SONAR_HOST_URL: https://sonarcloud.io - -analysis_SQC_US_shadow_task: - <<: *ANALYSIS_BASE_LINUX_TEMPLATE - # only executed in CRON job AND on master branch - only_if: $CIRRUS_CRON == $CRON_NIGHTLY_JOB_NAME && $CIRRUS_BRANCH == "master" - alias: sonar_analysis_shadow_sqc_us - name: "SQC-US Shadow Analysis" - env: - SONAR_TOKEN: VAULT[development/kv/data/sonarqube-us data.token] - SONAR_HOST_URL: https://sonarqube.us - -qa_task: - alias: qa - matrix: - - name: "Test Python 3.9.18" - eks_container: - docker_arguments: - PYTHON_VERSION: 3.9.18 - - name: "Test Python 3.9.6" - eks_container: - docker_arguments: - PYTHON_VERSION: 3.9.6 - - name: "Test Python 3.10" - eks_container: - docker_arguments: - PYTHON_VERSION: 3.10.13 - - name: "Test Python 3.11" - eks_container: - docker_arguments: - PYTHON_VERSION: 3.11.7 - - name: "Test Python 3.12" - eks_container: - docker_arguments: - PYTHON_VERSION: 3.12.1 - - name: "Test Python 3.13" - eks_container: - docker_arguments: - PYTHON_VERSION: 3.13.2 - <<: *LINUX_CONTAINER_DEFINITION - <<: *POETRY_INSTALL - <<: *POETRY_SET_VERSION - qa_script: - - poetry run pytest tests/ - qa_macos_task: alias: qa_macos only_if: $CIRRUS_CRON == "macos-its-cron" @@ -238,35 +91,6 @@ qa_macos_task: test_313_script: - .cirrus/run_macos_tests.sh "3.13.2" -qa_windows_task: - name: "Test Windows" - <<: *POETRY_WIN_INSTALL - <<: *POETRY_SET_VERSION - alias: qa_windows - qa_script: - - poetry run pytest tests/ - - exit $LASTEXITCODE - -build_task: - <<: *LINUX_CONTAINER_DEFINITION - <<: *POETRY_INSTALL - <<: *POETRY_SET_VERSION - alias: build - name: "Build" - build_script: regular_poetry_build_publish - -its_task: - name: "Integration Tests" - alias: its - sonarqube_cache: - folder: sonarqube_cache/ - populate_script: mkdir -p sonarqube_cache && wget -q https://binaries.sonarsource.com/Distribution/sonarqube/sonarqube-$SONARQUBE_VERSION.zip -O sonarqube_cache/sonarqube.zip - fingerprint_script: echo "sonarqube-$SONARQUBE_VERSION" - <<: *LINUX_CONTAINER_DEFINITION - <<: *POETRY_INSTALL - its_script: - - .cirrus/run_its.sh - its_macos_task: name: "[macOS] Integration Tests" alias: its_macos @@ -280,25 +104,7 @@ its_macos_task: its_script: - .cirrus/run_its.sh -promote_task: - depends_on: - - formatting - - sonar_analysis_next - - qa - - qa_windows - - build - - its - env: - ARTIFACTORY_PROMOTE_ACCESS_TOKEN: VAULT[development/artifactory/token/${CIRRUS_REPO_OWNER}-${CIRRUS_REPO_NAME}-promoter access_token] - GITHUB_TOKEN: VAULT[development/github/token/${CIRRUS_REPO_OWNER}-${CIRRUS_REPO_NAME}-promotion token] - <<: *LINUX_CONTAINER_DEFINITION - <<: *POETRY_INSTALL - <<: *POETRY_SET_VERSION - promote_script: cirrus_promote - run_iris_task: - depends_on: - - promote <<: *LINUX_CONTAINER_DEFINITION # only executed in CRON job AND on master branch only_if: $CIRRUS_CRON == $CRON_NIGHTLY_JOB_NAME && $CIRRUS_BRANCH == "master" diff --git a/.github/actions/config-poetry/action.yml b/.github/actions/config-poetry/action.yml new file mode 100644 index 00000000..c2cc8811 --- /dev/null +++ b/.github/actions/config-poetry/action.yml @@ -0,0 +1,75 @@ +--- +name: Configure Poetry +description: GitHub Action to configure a poetry project + +inputs: + python-version: + description: The version of python to use + default: 3.12.1 + poetry-version: + description: The version of poetry to install + default: 2.2.1 + jfrog-version: + description: The version of jFrog to install + default: 2.77.0 + poetry-virtualenvs-path: + description: Path to the Poetry virtual environments, relative to GitHub workspace. The folder is cached only if it is a subdirectory of + `poetry-cache-dir`. + default: .cache/pypoetry/virtualenvs + poetry-cache-dir: + description: Path to the Poetry cache directory, relative to GitHub workspace. + default: .cache/pypoetry +outputs: + BUILD_NUMBER: + description: The build number, incremented or reused if already cached + value: ${{ steps.get_build_number.outputs.BUILD_NUMBER }} + +runs: + using: composite + steps: + - name: Set build parameters + shell: bash + env: + ARTIFACTORY_READER_ROLE: private-reader + run: | + echo "ARTIFACTORY_READER_ROLE=${ARTIFACTORY_READER_ROLE}" >> "$GITHUB_ENV" + - uses: SonarSource/ci-github-actions/get-build-number@v1 + id: get_build_number + - name: Cache local Poetry cache + uses: SonarSource/ci-github-actions/cache@v1 + with: + path: ${{ inputs.poetry-cache-dir }} + key: poetry-${{ runner.os }}-${{ hashFiles('poetry.lock') }} + restore-keys: poetry-${{ runner.os }}- + - name: Install mise and Python + uses: jdx/mise-action@5ac50f778e26fac95da98d50503682459e86d566 # v3.2.0 + with: + version: 2025.7.12 + install_args: "python@${{ inputs.python-version }}" + - name: Install jfrog and poetry through mise + uses: jdx/mise-action@5ac50f778e26fac95da98d50503682459e86d566 # v3.2.0 + with: + version: 2025.7.12 + experimental: true # needed to use the http backend for installation of jfrog on windows + - name: Vault + # yamllint disable rule:line-length + id: secrets + uses: SonarSource/vault-action-wrapper@320bd31b03e5dacaac6be51bbbb15adf7caccc32 # 3.1.0 + with: + secrets: | + development/artifactory/token/{REPO_OWNER_NAME_DASH}-${{ env.ARTIFACTORY_READER_ROLE }} access_token | ARTIFACTORY_ACCESS_TOKEN; + # yamllint enable rule:line-length + - name: Config Poetry + id: config + shell: bash + env: + ARTIFACTORY_URL: https://repox.jfrog.io/artifactory + ARTIFACTORY_PYPI_REPO: sonarsource-pypi + ARTIFACTORY_ACCESS_TOKEN: ${{ fromJSON(steps.secrets.outputs.vault).ARTIFACTORY_ACCESS_TOKEN }} + POETRY_VIRTUALENVS_PATH: ${{ github.workspace }}/${{ inputs.poetry-virtualenvs-path }} + POETRY_CACHE_DIR: ${{ github.workspace }}/${{ inputs.poetry-cache-dir }} + run: | + echo "POETRY_VIRTUALENVS_PATH=${POETRY_VIRTUALENVS_PATH}" >> "$GITHUB_ENV" + echo "POETRY_CACHE_DIR=${POETRY_CACHE_DIR}" >> "$GITHUB_ENV" + ${GITHUB_ACTION_PATH}/config-poetry.sh + diff --git a/.github/actions/config-poetry/config-poetry.sh b/.github/actions/config-poetry/config-poetry.sh new file mode 100755 index 00000000..98aa7628 --- /dev/null +++ b/.github/actions/config-poetry/config-poetry.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Config script for SonarSource Poetry projects. + +set -euo pipefail + +: "${ARTIFACTORY_URL:?}" +: "${ARTIFACTORY_PYPI_REPO:?}" "${ARTIFACTORY_ACCESS_TOKEN:?}" +: "${BUILD_NUMBER:?}" "${GITHUB_REPOSITORY:?}" + +set_build_env() { + export PROJECT=${GITHUB_REPOSITORY#*/} + echo "PROJECT: $PROJECT" +} + +config_poetry() { + jf config add repox --artifactory-url "$ARTIFACTORY_URL" --access-token "$ARTIFACTORY_ACCESS_TOKEN" + jf poetry-config --server-id-resolve repox --repo-resolve "$ARTIFACTORY_PYPI_REPO" + jf poetry install --build-name="$PROJECT" --build-number="$BUILD_NUMBER" +} + +main() { + set_build_env + config_poetry +} + +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main "$@" +fi diff --git a/.github/scripts/run_its.sh b/.github/scripts/run_its.sh new file mode 100755 index 00000000..857466a3 --- /dev/null +++ b/.github/scripts/run_its.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail + +unzip -q sonarqube_cache/sonarqube.zip -d sonarqube + +PLATFORM="linux-x86-64" +if [[ "$(uname)" == "Darwin" ]]; then + PLATFORM="macosx-universal-64" +fi + +cd $(ls -d sonarqube/*/) +./bin/${PLATFORM}/sonar.sh start +cd - + +unset SONAR_TOKEN +unset SONAR_HOST_URL + +poetry install +poetry run pytest --its tests/its diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..87497753 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,211 @@ +name: Build +on: + push: + branches: [master, branch-*, dogfood-*] + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + build: + name: "Build" + runs-on: github-ubuntu-latest-s + outputs: + build-number: ${{ steps.build-poetry.outputs.BUILD_NUMBER }} + permissions: + id-token: write + contents: write + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + - name: Install mise and tools + uses: jdx/mise-action@e3d7b8d67a7958d1207f6ed871e83b1ea780e7b0 #v3.3.1 + - name: Build the scanner + uses: SonarSource/ci-github-actions/build-poetry@v1 + id: build-poetry + with: + sonar-platform: none + artifactory-reader-role: private-reader + artifactory-deployer-role: qa-deployer + deploy-pull-request: true + + install_deps: + name: "Install and Cache Poetry Dependencies" + runs-on: github-ubuntu-latest-s + permissions: + id-token: write + contents: write + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + - name: Configure poetry + uses: ./.github/actions/config-poetry # We use this job to cache the poetry depend + - run: | + poetry install + + formatting: + name: "Formatting and Licenses headers" + needs: [install_deps] + runs-on: github-ubuntu-latest-s + permissions: + id-token: write + contents: write + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + - name: Configure poetry + uses: ./.github/actions/config-poetry + - run: | + poetry run black src/ tests/ --check + poetry run licenseheaders -t license_header.tmpl -o "SonarSource SA" -y 2011-2024 -n "Sonar Scanner Python" -E .py -d src/ + poetry run licenseheaders -t license_header.tmpl -o "SonarSource SA" -y 2011-2024 -n "Sonar Scanner Python" -E .py -d tests/ + git diff --name-only --exit-code ./src ./tests + + documentation: + name: "CLI Documentation" + runs-on: github-ubuntu-latest-s + needs: [install_deps] + permissions: + id-token: write + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + - name: Install mise and tools + uses: jdx/mise-action@e3d7b8d67a7958d1207f6ed871e83b1ea780e7b0 #v3.3.1 + - name: Check for incorrect documentation + run: | + poetry run python tools/generate_cli_documentation.py + git diff --exit-code CLI_ARGS.md + + coverage: + name: "Coverage report generation" + runs-on: github-ubuntu-latest-s + needs: [install_deps] + permissions: + id-token: write + contents: write + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + - name: Configure poetry + uses: ./.github/actions/config-poetry + - run: | + poetry run pytest --cov-report=xml:coverage.xml --cov-config=pyproject.toml --cov=src --cov-branch tests + poetry run mypy src/ > mypy-report.txt || true + - name: Upload coverage artifacts + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: coverage-reports + path: | + coverage.xml + mypy-report.txt + + analysis: + name: "NEXT Analysis" + runs-on: github-ubuntu-latest-s + needs: [coverage] + permissions: + id-token: write + contents: write + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + - name: Download coverage artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: coverage-reports + - name: Install mise and tools + uses: jdx/mise-action@e3d7b8d67a7958d1207f6ed871e83b1ea780e7b0 #v3.3.1 + - name: Analysis the project on next + uses: SonarSource/ci-github-actions/build-poetry@v1 + with: + sonar-platform: next + artifactory-reader-role: private-reader + artifactory-deployer-role: qa-deployer + + qa: + name: "Test Python ${{ matrix.python-version }}" + runs-on: github-ubuntu-latest-s + needs: [install_deps] + permissions: + id-token: write + contents: write + strategy: + matrix: + python-version: ["3.9.18", "3.9.6", "3.10.13", "3.11.7", "3.12.1", "3.13.2"] + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + - name: Configure poetry + uses: ./.github/actions/config-poetry + with: + python-version: ${{ matrix.python-version }} + - name: Execute the test suite + run: | + poetry run pytest tests/ + + qa-windows: + name: "Test Windows" + runs-on: github-windows-latest-s + needs: [install_deps] + permissions: + id-token: write + contents: write + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + - name: Configure poetry for Windows + uses: ./.github/actions/config-poetry + - name: Execute the test suite + run: | + poetry run pytest tests/ + + its: + name: "Integration Tests" + runs-on: github-ubuntu-latest-s + permissions: + id-token: write + contents: write + env: + SONARQUBE_VERSION: 25.3.0.104237 + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + - name: Cache SonarQube + uses: SonarSource/ci-github-actions/cache@v1 + id: sonarqube-cache + with: + path: sonarqube_cache/ + key: sonarqube-25.3.0.104237 + restore-keys: cache-${{ runner.os }}- + - name: Download SonarQube + if: ${{ !steps.sonarqube-cache.outputs.cache-hit }} + run: | + mkdir -p sonarqube_cache + if [ ! -f sonarqube_cache/sonarqube.zip ]; then + wget -q https://binaries.sonarsource.com/Distribution/sonarqube/sonarqube-$SONARQUBE_VERSION.zip -O sonarqube_cache/sonarqube.zip + fi + - name: Configure poetry + uses: ./.github/actions/config-poetry + - name: Execute the integration tests + run: ./.github/scripts/run_its.sh + + promote: + name: "Promote" + needs: [build, formatting, documentation, coverage, analysis, qa, qa-windows, its] + runs-on: github-ubuntu-latest-s + permissions: + id-token: write + contents: write + steps: + - name: Promote + uses: SonarSource/ci-github-actions/promote@v1 + with: + promote-pull-request: true + build-name: sonar-scanner-python + env: + BUILD_NUMBER: ${{ needs.build.outputs.build-number }} diff --git a/mise.toml b/mise.toml new file mode 100644 index 00000000..80ef1bc0 --- /dev/null +++ b/mise.toml @@ -0,0 +1,13 @@ +[tools] +"pipx:poetry" = "2.2.1" + +[tools."asdf:jfrog-cli"] +version = "2.77.0" +os = ["linux", "macos"] + +[tools."http:jfrog-cli"] +version = "2.77.0" +os = ["windows"] + +[tools."http:jfrog-cli".platforms] +windows-x64 = { url = "https://releases.jfrog.io/artifactory/jfrog-cli/v2-jf/2.77.0/jfrog-cli-windows-amd64/jf.exe", install = "cp ${MISE_DOWNLOAD_PATH} ${MISE_INSTALL_PATH}/bin/jf.exe" }