From 90eea8b54ac4cb795cdea4ed51a88e4829be05a6 Mon Sep 17 00:00:00 2001 From: Andreas Klos Date: Mon, 8 Sep 2025 12:27:26 +0200 Subject: [PATCH] refactor: update GitHub workflows for release management and image building - Added new workflows for creating releases, preparing releases, and publishing libraries on merge. - Implemented logic to derive version from pull request titles and create Git tags/releases accordingly. - Enhanced image building workflows to include digest capturing and improved error handling. - Refactored existing workflows to streamline the process of bumping versions for internal libraries and services. - Introduced scripts for bumping chart versions and updating pyproject dependencies. - Removed obsolete scripts and workflows to clean up the repository. --- .github/workflows/build-images.yml | 110 ++++++++++++++ .github/workflows/create-release.yml | 47 ++++++ .github/workflows/lint-and-test.yml | 8 +- .github/workflows/prepare-release.yml | 74 ++++++++++ .github/workflows/publish-chart.yml | 95 ++++++++++++ .github/workflows/publish-libs-on-merge.yml | 149 +++++++++++++++++++ .github/workflows/semantic-release.yml | 156 -------------------- services/admin-backend/pyproject.toml | 15 +- services/document-extractor/pyproject.toml | 16 +- services/mcp-server/pyproject.toml | 2 +- services/rag-backend/pyproject.toml | 14 +- tools/bump_chart_versions.py | 54 +++++++ tools/bump_pyproject_deps.py | 115 +++++++++++++++ tools/update-helm-values.py | 72 --------- 14 files changed, 687 insertions(+), 240 deletions(-) create mode 100644 .github/workflows/build-images.yml create mode 100644 .github/workflows/create-release.yml create mode 100644 .github/workflows/prepare-release.yml create mode 100644 .github/workflows/publish-chart.yml create mode 100644 .github/workflows/publish-libs-on-merge.yml delete mode 100644 .github/workflows/semantic-release.yml create mode 100644 tools/bump_chart_versions.py create mode 100644 tools/bump_pyproject_deps.py delete mode 100644 tools/update-helm-values.py diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml new file mode 100644 index 00000000..ebd21ffb --- /dev/null +++ b/.github/workflows/build-images.yml @@ -0,0 +1,110 @@ +name: build-images +run-name: build-images ${{ github.event.release.tag_name }} +on: + release: + types: [published] + +permissions: + contents: read + packages: write + +jobs: + prepare: + if: ${{ github.event_name == 'release' }} + runs-on: ubuntu-latest + outputs: + tag: ${{ steps.release_tag.outputs.tag }} + version: ${{ steps.release_tag.outputs.version }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Resolve release tag & version + id: release_tag + run: | + git fetch --tags --force + TAG="${{ github.event.release.tag_name }}" + if [ -z "$TAG" ]; then + echo "No Git tag found to check out" >&2 + exit 1 + fi + VER_NO_V="${TAG#v}" + echo "tag=$TAG" >> $GITHUB_OUTPUT + echo "version=$VER_NO_V" >> $GITHUB_OUTPUT + + build-image: + needs: prepare + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - name: rag-backend + dockerfile: services/rag-backend/Dockerfile + - name: admin-backend + dockerfile: services/admin-backend/Dockerfile + - name: document-extractor + dockerfile: services/document-extractor/Dockerfile + - name: mcp-server + dockerfile: services/mcp-server/Dockerfile + - name: frontend + dockerfile: services/frontend/apps/chat-app/Dockerfile + - name: admin-frontend + dockerfile: services/frontend/apps/admin-app/Dockerfile + env: + REGISTRY: ghcr.io + IMAGE_NS: ${{ github.repository }} + VERSION: ${{ needs.prepare.outputs.version }} + TAG: ${{ needs.prepare.outputs.tag }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Checkout release tag + run: git checkout "$TAG" + - name: Normalize IMAGE_NS to lowercase + run: echo "IMAGE_NS=$(echo '${{ env.IMAGE_NS }}' | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV + - name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GHCR_PAT }} + - name: Set up Buildx + uses: docker/setup-buildx-action@v3 + - name: Build & push ${{ matrix.name }} + run: | + docker buildx build --push \ + -t "$REGISTRY/$IMAGE_NS/${{ matrix.name }}:${VERSION}" \ + -t "$REGISTRY/$IMAGE_NS/${{ matrix.name }}:latest" \ + -f "${{ matrix.dockerfile }}" . + - name: Capture digest + run: | + sudo apt-get update && sudo apt-get install -y jq + ref="$REGISTRY/$IMAGE_NS/${{ matrix.name }}:${VERSION}" + digest=$(docker buildx imagetools inspect "$ref" --format '{{json .Manifest.Digest}}' | jq -r . || true) + jq -n --arg name "${{ matrix.name }}" --arg tag "$VERSION" --arg digest "$digest" '{($name): {tag: $tag, digest: $digest}}' > digest.json + - name: Upload digest artifact + uses: actions/upload-artifact@v4 + with: + name: image-digest-${{ matrix.name }} + path: digest.json + + collect-digests: + needs: [build-image] + runs-on: ubuntu-latest + steps: + - name: Download digest artifacts + uses: actions/download-artifact@v4 + with: + pattern: image-digest-* + merge-multiple: false + - name: Merge digests + run: | + sudo apt-get update && sudo apt-get install -y jq + jq -s 'reduce .[] as $item ({}; . * $item)' image-digest-*/digest.json > image-digests.json + - name: Upload merged digests + uses: actions/upload-artifact@v4 + with: + name: image-digests + path: image-digests.json diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml new file mode 100644 index 00000000..864eedc1 --- /dev/null +++ b/.github/workflows/create-release.yml @@ -0,0 +1,47 @@ +name: create-release +on: + pull_request_target: + types: [closed] + branches: [main] + +permissions: + contents: write + +jobs: + release: + if: >- + ${{ + github.event.pull_request.merged == true && + contains(github.event.pull_request.labels.*.name, 'refresh-locks') + }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Derive version from PR title + id: ver + run: | + TITLE="${{ github.event.pull_request.title }}" + VERSION=$(echo "$TITLE" | sed -nE 's/.*([0-9]+\.[0-9]+\.[0-9]+(\.post[0-9]+)?).*/\1/p' || true) + if [ -z "$VERSION" ]; then + echo "Could not extract version from PR title: $TITLE" >&2 + exit 1 + fi + echo "version=$VERSION" >> $GITHUB_OUTPUT + + - name: Create Git tag + run: | + git config user.name "github-actions" + git config user.email "github-actions@github.com" + git tag -a "v${{ steps.ver.outputs.version }}" -m "Release v${{ steps.ver.outputs.version }}" + git push origin "v${{ steps.ver.outputs.version }}" + + - name: Create GitHub Release + uses: softprops/action-gh-release@v2 + with: + tag_name: v${{ steps.ver.outputs.version }} + name: v${{ steps.ver.outputs.version }} + generate_release_notes: true + token: ${{ secrets.GHCR_PAT }} diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index 23e98420..7a4c9029 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -12,6 +12,12 @@ env: jobs: changes: + if: >- + ${{ + !contains(github.event.pull_request.labels.*.name, 'prepare-release') && + !contains(github.event.pull_request.labels.*.name, 'refresh-locks') && + !contains(github.event.pull_request.labels.*.name, 'chart-bump') + }} name: Detect Changes runs-on: ubuntu-latest outputs: @@ -81,7 +87,7 @@ jobs: - name: Build Docker image run: | - docker build -t $IMAGE_NAME --build-arg dev=1 -f services/${{ matrix.service }}/Dockerfile . + docker build -t $IMAGE_NAME -f services/${{ matrix.service }}/Dockerfile.dev . - name: Run linting run: | diff --git a/.github/workflows/prepare-release.yml b/.github/workflows/prepare-release.yml new file mode 100644 index 00000000..7940e36c --- /dev/null +++ b/.github/workflows/prepare-release.yml @@ -0,0 +1,74 @@ +name: prepare-release +on: + pull_request: + types: [closed] + branches: + - main + +permissions: + contents: write + pull-requests: write + +jobs: + prepare: + if: >- + ${{ + github.event.pull_request.merged && + !contains(github.event.pull_request.labels.*.name, 'prepare-release') && + !contains(github.event.pull_request.labels.*.name, 'refresh-locks') && + !contains(github.event.pull_request.labels.*.name, 'chart-bump') + }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Install semantic-release deps + run: npm ci + + - name: verify-dependencies-integrity + run: npm audit signatures + + - name: Compute next version (dry-run) + id: semrel + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + npx semantic-release --dry-run --no-ci | tee semrel.log + BASE_VERSION=$(grep -Eo "next release version is [0-9]+\.[0-9]+\.[0-9]+" semrel.log | awk '{print $5}') + if [ -z "$BASE_VERSION" ]; then echo "No new release required"; exit 1; fi + VERSION="${BASE_VERSION}.post$(date +%Y%m%d%H%M%S)" + echo "version=$VERSION" >> $GITHUB_OUTPUT + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.13' + + - name: Install bump script deps + run: | + python -m pip install --upgrade pip + python -m pip install "tomlkit==0.13.3" "pyyaml==6.0.2" "packaging==25.0" + + - name: Bump internal libs only (no service pins) + run: | + python tools/bump_pyproject_deps.py --version "${{ steps.semrel.outputs.version }}" --bump-libs + + - name: Commit and open PR + uses: peter-evans/create-pull-request@v6 + with: + branch: chore/release-${{ steps.semrel.outputs.version }} + title: "chore(release): prepare ${{ steps.semrel.outputs.version }}" + body: | + Prepare release ${{ steps.semrel.outputs.version }} + - bump internal libs versions only + commit-message: "chore(release): prepare ${{ steps.semrel.outputs.version }}" + add-paths: | + libs/**/pyproject.toml + labels: prepare-release diff --git a/.github/workflows/publish-chart.yml b/.github/workflows/publish-chart.yml new file mode 100644 index 00000000..89e62fa8 --- /dev/null +++ b/.github/workflows/publish-chart.yml @@ -0,0 +1,95 @@ +name: publish-chart +run-name: publish-chart (post-build-images) +on: + workflow_run: + workflows: [build-images] + types: [completed] + +permissions: + contents: write + pull-requests: write + packages: write + +jobs: + chart: + if: ${{ github.event.workflow_run.conclusion == 'success' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Checkout release tag from triggering run + run: | + git fetch --tags --force + HEAD_SHA="${{ github.event.workflow_run.head_sha }}" + if [ -n "$HEAD_SHA" ]; then + TAG=$(git tag --points-at "$HEAD_SHA" | head -n 1 || true) + if [ -z "$TAG" ]; then + TAG=$(git describe --tags --abbrev=0 "$HEAD_SHA" 2>/dev/null || true) + fi + fi + if [ -z "$TAG" ]; then + echo "No tag found (head_sha=$HEAD_SHA)" >&2 + exit 1 + fi + git checkout "$TAG" + echo "RELEASE_TAG=$TAG" >> $GITHUB_ENV + echo "APP_VERSION=${TAG#v}" >> $GITHUB_ENV + + - name: Expose app version + id: meta + run: echo "app_version=${APP_VERSION}" >> $GITHUB_OUTPUT + + - name: Setup Helm + uses: azure/setup-helm@v4 + + - name: Login to GHCR for Helm OCI + run: echo ${{ secrets.GHCR_PAT }} | helm registry login ghcr.io -u ${{ github.actor }} --password-stdin + + - name: Setup Python (for bump script) + uses: actions/setup-python@v5 + with: + python-version: '3.13' + + - name: Install bump script deps + run: | + python -m pip install --upgrade pip + python -m pip install "pyyaml==6.0.2" "packaging==25.0" + + - name: Bump Chart.yaml (set release version) + env: + APP_VERSION: ${{ env.APP_VERSION }} + run: | + python tools/bump_chart_versions.py --app-version "$APP_VERSION" + + - name: Package and push rag chart + env: + APP_VERSION: ${{ env.APP_VERSION }} + run: | + set -euo pipefail + export HELM_EXPERIMENTAL_OCI=1 + CHART_DIR="infrastructure/rag" + if [ ! -f "$CHART_DIR/Chart.yaml" ]; then + echo "Expected chart at $CHART_DIR/Chart.yaml not found" >&2 + exit 1 + fi + mkdir -p dist + helm dependency update "$CHART_DIR" || true + helm package "$CHART_DIR" --destination dist + PKG=$(ls dist/*.tgz) + helm show chart "$PKG" | grep -E "^version: " + helm push "$PKG" oci://ghcr.io/${{ github.repository_owner }}/charts + + - name: Create PR for chart version bumps + uses: peter-evans/create-pull-request@v6 + with: + base: main + branch: chore/chart-bump-${{ steps.meta.outputs.app_version }} + title: "chore(release): bump chart versions to ${{ steps.meta.outputs.app_version }}" + body: | + Persist Chart.yaml appVersion/version to match release ${{ steps.meta.outputs.app_version }}. + commit-message: "chore(release): bump charts to ${{ steps.meta.outputs.app_version }}" + add-paths: | + infrastructure/**/Chart.yaml + labels: chart-bump diff --git a/.github/workflows/publish-libs-on-merge.yml b/.github/workflows/publish-libs-on-merge.yml new file mode 100644 index 00000000..95dec1b9 --- /dev/null +++ b/.github/workflows/publish-libs-on-merge.yml @@ -0,0 +1,149 @@ +name: publish-libs-on-merge +on: + pull_request: + branches: [main] + types: [closed] + +permissions: + contents: write + pull-requests: write + packages: write + issues: write + +jobs: + publish: + if: ${{ github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'prepare-release') }} + runs-on: ubuntu-latest + outputs: + version: ${{ steps.ver.outputs.version }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Derive version from PR title + id: ver + run: | + TITLE="${{ github.event.pull_request.title }}" + VERSION=$(echo "$TITLE" | sed -nE 's/.*prepare ([0-9]+\.[0-9]+\.[0-9]+(\.post[0-9]+)?).*/\1/p' || true) + if [ -z "$VERSION" ]; then + echo "Could not derive version from PR title" >&2 + exit 1 + fi + echo "version=$VERSION" >> $GITHUB_OUTPUT + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.13' + + - name: Install Poetry + run: | + pip install poetry==2.1.3 + - name: Configure TestPyPI repository #TODO: should be pypi later. + run: | + poetry config repositories.testpypi https://test.pypi.org/legacy/ + - name: Build and publish libs to TestPyPI #TODO: if STACKIT org is created, the gha should be authorized, no token necessary anymore! + env: + POETRY_HTTP_BASIC_TESTPYPI_USERNAME: __token__ + POETRY_HTTP_BASIC_TESTPYPI_PASSWORD: ${{ secrets.TEST_PYPI_TOKEN }} + run: | + for lib in libs/*; do + [ -d "$lib" ] || continue + echo "Publishing $lib" + (cd "$lib" && poetry version "${{ steps.ver.outputs.version }}" && poetry build && poetry publish -r testpypi) + done + lock-services: + runs-on: ubuntu-latest + needs: publish + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.13' + + - name: Install Poetry + run: | + pip install poetry==2.1.3 + - name: Install bump script deps + run: | + python -m pip install --upgrade pip + python -m pip install "tomlkit==0.13.3" + - name: Update service dependency pins to released version + env: + VERSION: ${{ needs.publish.outputs.version }} + run: | + python tools/bump_pyproject_deps.py --version "$VERSION" --bump-service-pins + - name: Install jq + run: sudo apt-get update && sudo apt-get install -y jq + + - name: Wait for TestPyPI indexing + env: + VERSION: ${{ needs.publish.outputs.version }} + run: | + echo "Waiting for TestPyPI to index internal libs for version $VERSION" + for name in admin-api-lib extractor-api-lib rag-core-api rag-core-lib; do + echo "Checking $name==$VERSION" + seen=false + for i in $(seq 1 60); do # up to ~5 minutes + json_ok=false + simple_ok=false + if curl -fsSL "https://test.pypi.org/pypi/$name/json" | jq -e --arg v "$VERSION" '.releases[$v] | length > 0' >/dev/null; then + json_ok=true + fi + # Check simple index also, Poetry resolves via /simple + if curl -fsSL "https://test.pypi.org/simple/$name/" | grep -q "$VERSION"; then + simple_ok=true + fi + if [ "$json_ok" = true ] && [ "$simple_ok" = true ]; then + echo "Found $name==$VERSION on JSON and /simple" + seen=true + break + fi + sleep 5 + done + if [ "$seen" != "true" ]; then + echo "Error: $name==$VERSION not visible on TestPyPI JSON+simple yet" + echo "--- Debug /simple page for $name ---" + curl -fsSL "https://test.pypi.org/simple/$name/" || true + exit 1 + fi + done + - name: Clear poetry caches + run: | + poetry cache clear --all pypi -n || true + poetry cache clear --all testpypi -n || true + + - name: Refresh service lockfiles + run: | + for svc in services/rag-backend services/admin-backend services/document-extractor services/mcp-server; do + if [ -f "$svc/pyproject.toml" ]; then + echo "Locking $svc" + ( + cd "$svc" + poetry lock -v || ( + echo "Lock failed, clearing caches and retrying..."; + poetry cache clear --all pypi -n || true; + poetry cache clear --all testpypi -n || true; + sleep 10; + poetry lock -v + ) + ) + fi + done + - name: Open PR with updated lockfiles and pins + id: cpr + uses: peter-evans/create-pull-request@v6 + with: + branch: chore/refresh-locks-${{ needs.publish.outputs.version }}-${{ github.run_number }} + title: "chore(release): refresh service lockfiles for ${{ needs.publish.outputs.version }}" + body: | + Refresh service poetry.lock files and dependency pins for version ${{ needs.publish.outputs.version }}. + commit-message: "chore(release): refresh service lockfiles and pins" + add-paths: | + services/**/pyproject.toml + services/**/poetry.lock + labels: refresh-locks diff --git a/.github/workflows/semantic-release.yml b/.github/workflows/semantic-release.yml deleted file mode 100644 index c8a3f388..00000000 --- a/.github/workflows/semantic-release.yml +++ /dev/null @@ -1,156 +0,0 @@ -name: semantic-release -on: - workflow_dispatch: - push: - # Only trigger on merged PRs, not on every PR push - pull_request: - types: [closed] - branches: - - main - -permissions: - contents: write - packages: write - -jobs: - semantic-release: - name: semantic-release - runs-on: ubuntu-latest - # Only run on push to main, manual dispatch, or when PR is merged - if: github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && github.event.pull_request.merged == true) - outputs: - new-release-published: ${{ steps.semantic-release.outputs.new-release-published }} - new-release-version: ${{ steps.semantic-release.outputs.new-release-version }} - steps: - - name: checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: false - - - name: setup-node - uses: actions/setup-node@v4 - with: - node-version: "22.13.1" - - - name: create-archives - run: | - TEMP_DIR=$(mktemp -d) - tar --warning=no-file-changed \ - --exclude=".git" \ - --exclude="/.git" \ - --exclude="node_modules" \ - -czf "$TEMP_DIR/action-main-release-trials.tar.gz" . - zip -r "$TEMP_DIR/action-main-release-trials.zip" . \ - -x ".git" "node_modules" - mv "$TEMP_DIR"/*.{tar.gz,zip} . - rm -rf "$TEMP_DIR" - - - name: install-dependencies - run: npm ci - - - name: verify-dependencies-integrity - run: npm audit signatures - - - name: create-semantic-release - id: semantic-release - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - # Run semantic-release and capture the output - npx semantic-release > semantic-release-output.txt 2>&1 || true - - # Check if a new release was published by looking for the success message - if grep -q "Published release" semantic-release-output.txt; then - echo "new-release-published=true" >> $GITHUB_OUTPUT - - # Extract the version from the output - VERSION=$(grep "Published release" semantic-release-output.txt | sed -n 's/.*Published release \([0-9]\+\.[0-9]\+\.[0-9]\+\).*/\1/p') - - if [[ -n "$VERSION" ]]; then - echo "new-release-version=$VERSION" >> $GITHUB_OUTPUT - echo "āœ… New release published: $VERSION" - else - echo "āŒ Could not extract version from semantic-release output" - exit 1 - fi - else - echo "new-release-published=false" >> $GITHUB_OUTPUT - echo "ā„¹ļø No new release published" - fi - - build-and-push-images: - name: build-and-push-images - runs-on: ubuntu-latest - needs: semantic-release - if: needs.semantic-release.outputs.new-release-published == 'true' - strategy: - matrix: - service: - - name: rag-backend - dockerfile: services/rag-backend/Dockerfile - image: rag-backend - build-args: "dev=0" - - name: admin-backend - dockerfile: services/admin-backend/Dockerfile - image: admin-backend - build-args: "dev=0" - - name: document-extractor - dockerfile: services/document-extractor/Dockerfile - image: document-extractor - build-args: "dev=0" - - name: mcp-server - dockerfile: services/mcp-server/Dockerfile - image: mcp-server - build-args: "dev=0" - - name: frontend - dockerfile: services/frontend/apps/chat-app/Dockerfile - image: frontend - build-args: "" - - name: admin-frontend - dockerfile: services/frontend/apps/admin-app/Dockerfile - image: admin-frontend - build-args: "" - steps: - - name: debug-job-inputs - run: | - echo "šŸ” Debug: needs.semantic-release.outputs.new-release-published = ${{ needs.semantic-release.outputs.new-release-published }}" - echo "šŸ” Debug: needs.semantic-release.outputs.new-release-version = ${{ needs.semantic-release.outputs.new-release-version }}" - - - name: checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: false - - - name: login-to-github-container-registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: build-and-push-${{ matrix.service.name }} - run: | - docker build \ - --tag ghcr.io/${{ github.repository_owner }}/rag-template/${{ matrix.service.image }}:v${{ needs.semantic-release.outputs.new-release-version }} \ - --tag ghcr.io/${{ github.repository_owner }}/rag-template/${{ matrix.service.image }}:latest \ - --file ${{ matrix.service.dockerfile }} \ - . - - docker push ghcr.io/${{ github.repository_owner }}/rag-template/${{ matrix.service.image }}:v${{ needs.semantic-release.outputs.new-release-version }} - docker push ghcr.io/${{ github.repository_owner }}/rag-template/${{ matrix.service.image }}:latest - - - name: deployment-summary - if: strategy.job-index == 0 # Only run on first job in matrix - run: | - echo "## šŸš€ Deployment Summary" >> $GITHUB_STEP_SUMMARY - echo "**New version:** v${{ needs.semantic-release.outputs.new-release-version }}" >> $GITHUB_STEP_SUMMARY - echo "**Services built and deployed:**" >> $GITHUB_STEP_SUMMARY - echo "- rag-backend" >> $GITHUB_STEP_SUMMARY - echo "- admin-backend" >> $GITHUB_STEP_SUMMARY - echo "- document-extractor" >> $GITHUB_STEP_SUMMARY - echo "- mcp-server" >> $GITHUB_STEP_SUMMARY - echo "- frontend" >> $GITHUB_STEP_SUMMARY - echo "- admin-frontend" >> $GITHUB_STEP_SUMMARY - echo "**Registry:** ghcr.io/${{ github.repository_owner }}/rag-template" >> $GITHUB_STEP_SUMMARY diff --git a/services/admin-backend/pyproject.toml b/services/admin-backend/pyproject.toml index c6acd17c..eb8b658e 100644 --- a/services/admin-backend/pyproject.toml +++ b/services/admin-backend/pyproject.toml @@ -51,8 +51,8 @@ skip_gitignore = true max-line-length = 120 [tool.poetry] -name = "admin_backend" -version = "0.0.1" +name = "admin-backend" +version = "2.2.1" description = "The admin backend is responsible for the document management. This includes deletion, upload and getting particular documents or document lists." authors = ["STACKIT Data and AI Consulting "] readme = "README.md" @@ -90,4 +90,13 @@ build-backend = "poetry.core.masonry.api" [tool.poetry.dependencies] python = "^3.13" -admin-api-lib = {path = "../../libs/admin-api-lib", develop = true} + +# Prefer PyPI, but allow resolving from TestPyPI for internal libs +[[tool.poetry.source]] +name = "testpypi" +url = "https://test.pypi.org/simple/" +priority = "supplemental" + +[tool.poetry.group.prod.dependencies] +admin-api-lib = "==2.2.1" +rag-core-lib = "==2.2.1" diff --git a/services/document-extractor/pyproject.toml b/services/document-extractor/pyproject.toml index 59769f29..82c531d7 100644 --- a/services/document-extractor/pyproject.toml +++ b/services/document-extractor/pyproject.toml @@ -43,15 +43,14 @@ skip_gitignore = true max-line-length = 120 [tool.poetry] -name = "pdfextractor_server" -version = "0.0.0" -description = "Extracts the content of pdf documents." +name = "document-extractor" +version = "2.2.1" +description = "Extracts content from files and sources like Confluence spaces, sitemaps etc." authors = ["STACKIT Data and AI Consulting "] readme = "README.md" [tool.poetry.dependencies] python = "^3.13" -extractor-api-lib = {path = "../../libs/extractor-api-lib", develop = true} [tool.poetry.group.dev.dependencies] flake8 = "^7.2.0" @@ -86,3 +85,12 @@ httpx = "^0.28.1" [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" + +# Prefer PyPI, but allow resolving from TestPyPI for internal libs +[[tool.poetry.source]] +name = "testpypi" +url = "https://test.pypi.org/simple/" +priority = "supplemental" + +[tool.poetry.group.prod.dependencies] +extractor-api-lib = "==2.2.1" diff --git a/services/mcp-server/pyproject.toml b/services/mcp-server/pyproject.toml index 95fb4949..2b76b417 100644 --- a/services/mcp-server/pyproject.toml +++ b/services/mcp-server/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "rag-mcp-server" -version = "1.0.0" +version = "2.2.1" description = "Offers to use the chat interface of the RAG using MCP" authors = ["STACKIT Data and AI Consulting "] diff --git a/services/rag-backend/pyproject.toml b/services/rag-backend/pyproject.toml index 1e790e11..5a94c2bf 100644 --- a/services/rag-backend/pyproject.toml +++ b/services/rag-backend/pyproject.toml @@ -1,12 +1,11 @@ [tool.poetry] -name = "rag-usecase-example" -version = "0.1.0" +name = "rag-backend" +version = "2.2.1" description = "" authors = ["STACKIT Data and AI Consulting "] [tool.poetry.dependencies] python = "^3.13" -rag-core-api = { path = "../../libs/rag-core-api", develop = true} [tool.poetry.group.dev.dependencies] debugpy = "^1.8.14" @@ -89,3 +88,12 @@ skip_gitignore = true [tool.pylint] max-line-length = 120 +# Prefer PyPI, but allow resolving from TestPyPI for internal libs +[[tool.poetry.source]] +name = "testpypi" +url = "https://test.pypi.org/simple/" +priority = "supplemental" + +[tool.poetry.group.prod.dependencies] +rag-core-api = "==2.2.1" +rag-core-lib = "==2.2.1" diff --git a/tools/bump_chart_versions.py b/tools/bump_chart_versions.py new file mode 100644 index 00000000..9c4da5a0 --- /dev/null +++ b/tools/bump_chart_versions.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +import argparse +import pathlib +import sys +import re + +import yaml + +ROOT = pathlib.Path(__file__).resolve().parents[1] + + +def _to_chart_version(app_version: str) -> str: + """Convert app_version to a SemVer 2.0.0 compliant Helm chart version. + + Examples: + - "2.0.0.post20250904105936" -> "2.0.0-post.20250904105936" + - "2.0.1" -> "2.0.1" + - "2.0.1-rc.1" -> "2.0.1-rc.1" + - Fallback: if an unexpected format is provided, try to keep a valid semver + by extracting the leading MAJOR.MINOR.PATCH. + """ + # Case 1: our prepare-release format "X.Y.Z.post" + m = re.fullmatch(r"(?P\d+\.\d+\.\d+)\.post(?P\d+)", app_version) + if m: + return f"{m.group('base')}-post.{m.group('ts')}" + + # Case 2: already valid semver (optionally with pre-release or build metadata) + if re.fullmatch(r"\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?", app_version): + return app_version + + # Fallback: keep only the base version if present + base = re.match(r"(\d+\.\d+\.\d+)", app_version) + return base.group(1) if base else app_version + + +def bump_chart(chart_path: pathlib.Path, app_version: str): + data = yaml.safe_load(chart_path.read_text()) + data['appVersion'] = str(app_version) + data['version'] = _to_chart_version(str(app_version)) + chart_path.write_text(yaml.safe_dump(data, sort_keys=False)) + + +def main(): + p = argparse.ArgumentParser() + p.add_argument('--app-version', required=True) + args = p.parse_args() + + charts = list((ROOT / 'infrastructure').glob('*/Chart.yaml')) + for ch in charts: + bump_chart(ch, args.app_version) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/bump_pyproject_deps.py b/tools/bump_pyproject_deps.py new file mode 100644 index 00000000..8e3688b8 --- /dev/null +++ b/tools/bump_pyproject_deps.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +import argparse +import pathlib +import re +import sys +from typing import Any, List, Optional + +import tomlkit + +ROOT = pathlib.Path(__file__).resolve().parents[1] + +# Only bump versions for internal libs here +LIBS_VERSION_FILES = [ + ROOT / 'libs' / 'rag-core-lib' / 'pyproject.toml', + ROOT / 'libs' / 'rag-core-api' / 'pyproject.toml', + ROOT / 'libs' / 'admin-api-lib' / 'pyproject.toml', + ROOT / 'libs' / 'extractor-api-lib' / 'pyproject.toml', +] + +# Service pins to update after libs are published +SERVICE_PINS = { + ROOT / 'services' / 'rag-backend' / 'pyproject.toml': { + 'tool.poetry.group.prod.dependencies.rag-core-api': '=={v}', + 'tool.poetry.group.prod.dependencies.rag-core-lib': '=={v}', + }, + ROOT / 'services' / 'admin-backend' / 'pyproject.toml': { + 'tool.poetry.group.prod.dependencies.admin-api-lib': '=={v}', + 'tool.poetry.group.prod.dependencies.rag-core-lib': '=={v}', + }, + ROOT / 'services' / 'document-extractor' / 'pyproject.toml': { + 'tool.poetry.group.prod.dependencies.extractor-api-lib': '=={v}', + }, +} + + +def replace_version_line(text: str, new_version: str) -> str: + lines = text.splitlines(keepends=True) + in_tool_poetry = False + for i, line in enumerate(lines): + stripped = line.strip() + if stripped.startswith('[tool.poetry]'): + in_tool_poetry = True + continue + if in_tool_poetry and stripped.startswith('[') and not stripped.startswith('[tool.poetry]'): + # left the section without finding version; stop scanning section + break + if in_tool_poetry and stripped.startswith('version'): + # Replace only the version value, keep indentation and spacing + lines[i] = re.sub(r'version\s*=\s*"[^"]*"', f'version = "{new_version}"', line) + return ''.join(lines) + # If no version line found, append it to the [tool.poetry] section + out = ''.join(lines) + return out + f"\n[tool.poetry]\nversion = \"{new_version}\"\n" + + +def _get_table(doc: tomlkit.TOMLDocument, path: List[str]) -> Optional[Any]: + ref: Any = doc + for key in path: + try: + if key not in ref: # mapping-like check + return None + ref = ref[key] + except Exception: + return None + return ref + + +def bump(version: str, bump_libs: bool = True, bump_service_pins: bool = True): + # 1) bump libs versions (textual, non-destructive) + if bump_libs: + for file in LIBS_VERSION_FILES: + txt = file.read_text() + new_txt = replace_version_line(txt, version) + file.write_text(new_txt) + print(f"Updated {file} -> tool.poetry.version = {version}") + + # 2) bump service pins only inside [tool.poetry.group.prod.dependencies] + if bump_service_pins: + for file, mapping in SERVICE_PINS.items(): + txt = file.read_text() + doc = tomlkit.parse(txt) + deps = _get_table(doc, [ + 'tool', 'poetry', 'group', 'prod', 'dependencies' + ]) + if deps is None or not hasattr(deps, '__contains__'): + print(f"Skip {file}: prod dependencies table not found") + file.write_text(tomlkit.dumps(doc)) + continue + for dotted, template in mapping.items(): + pkg = dotted.split('.')[-1] + if pkg in deps: + val = template.format(v=version) + deps[pkg] = val + print(f"Pinned {file} -> {pkg} = {val}") + else: + print(f"Skip {file}: {pkg} not present in prod dependencies") + file.write_text(tomlkit.dumps(doc)) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument('--version', required=True) + ap.add_argument('--bump-libs', action='store_true', help='Bump versions in internal libs only') + ap.add_argument('--bump-service-pins', action='store_true', help='Bump service dependency pins only') + args = ap.parse_args() + + # Backward compatibility: if neither flag is provided, do both + bump_libs = args.bump_libs or (not args.bump_libs and not args.bump_service_pins) + bump_service_pins = args.bump_service_pins or (not args.bump_libs and not args.bump_service_pins) + + bump(args.version, bump_libs=bump_libs, bump_service_pins=bump_service_pins) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/update-helm-values.py b/tools/update-helm-values.py deleted file mode 100644 index 283ad051..00000000 --- a/tools/update-helm-values.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env python3 -""" -Update Helm values.yaml with new version tags for specific services only, -preserving all existing formatting and disabling line wrapping via ruamel.yaml. -""" - -import sys -from ruamel.yaml import YAML -from ruamel.yaml.scalarstring import DoubleQuotedScalarString - -def update_helm_values(file_path, new_version): - """Update specific service image tags in values.yaml with ruamel.yaml.""" - yaml = YAML() - yaml.preserve_quotes = True - yaml.width = float('inf') # ā† no wrapping, even for long lines - yaml.indent(mapping=2, sequence=4, offset=2) - - with open(file_path, 'r') as f: - data = yaml.load(f) - - services_to_update = [ - ('backend', 'mcp', 'image', 'tag'), - ('backend', 'image', 'tag'), - ('frontend', 'image', 'tag'), - ('adminBackend', 'image', 'tag'), - ('extractor', 'image', 'tag'), - ('adminFrontend', 'image', 'tag'), - ] - - new_tag = f"v{new_version}" - updated_count = 0 - - for path in services_to_update: - node = data - try: - for key in path[:-1]: - node = node[key] - tag_key = path[-1] - if tag_key in node: - old = node[tag_key] - if str(old) != new_tag: - if hasattr(old, 'style') and old.style in ('"', "'"): - node[tag_key] = DoubleQuotedScalarString(new_tag) - else: - node[tag_key] = new_tag - print(f"āœ… Updated {'.'.join(path)}: {old!r} ā†’ {new_tag!r}") - updated_count += 1 - else: - print(f"āš ļø {'.'.join(path)} already at {new_tag!r}") - else: - print(f"āŒ Could not find {'.'.join(path)}") - except (KeyError, TypeError) as e: - print(f"āŒ Could not access {'.'.join(path)}: {e}") - - if updated_count: - with open(file_path, 'w') as f: - yaml.dump(data, f) - print(f"\nāœ… Updated {updated_count} tag(s) in {file_path}") - return True - else: - print(f"\nāš ļø No changes needed in {file_path}") - return False - -if __name__ == '__main__': - if len(sys.argv) != 2: - print("Usage: update-helm-values.py ") - sys.exit(1) - - version = sys.argv[1] - file_path = 'infrastructure/rag/values.yaml' - success = update_helm_values(file_path, version) - sys.exit(0 if success else 1)