Provider Analysis #213
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Provider Analysis | |
| permissions: | |
| contents: read | |
| on: | |
| schedule: | |
| - cron: '0 */12 * * *' | |
| push: | |
| tags: | |
| - provider-analysis* | |
| env: | |
| STACKQL_CORE_REPOSITORY: ${{ vars.STACKQL_CORE_REPOSITORY != '' && vars.STACKQL_CORE_REPOSITORY || 'stackql/stackql' }} | |
| STACKQL_CORE_REF: ${{ vars.STACKQL_CORE_REF != '' && vars.STACKQL_CORE_REF || 'main' }} | |
| STACKQL_ANY_SDK_REPOSITORY: ${{ vars.STACKQL_ANY_SDK_REPOSITORY != '' && vars.STACKQL_ANY_SDK_REPOSITORY || 'stackql/any-sdk' }} | |
| STACKQL_ANY_SDK_REF: ${{ vars.STACKQL_ANY_SDK_REF != '' && vars.STACKQL_ANY_SDK_REF || 'main' }} | |
| PROVIDER_VERSION_AWS: v26.02.00377 | |
| PROVIDER_VERSION_AZURE: v24.10.00267 | |
| PROVIDER_VERSION_GOOGLE: v25.12.00357 | |
| GOLANG_VERSION: 1.25.3 | |
| PYTHON_VERSION: 3.12 | |
| ANALYSIS_RESULTS_BUCKET: 'stackql-provider-analysis-results' | |
| jobs: | |
| build-anysdk: | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Set up golang | |
| uses: actions/setup-go@v6 | |
| with: | |
| go-version: ^${{ env.GOLANG_VERSION }} | |
| cache: false | |
| id: go | |
| - name: Check out code into the Go module directory | |
| uses: actions/checkout@v7 | |
| - name: Get dependencies | |
| run: | | |
| git config --global url.https://$GH_ACCESS_TOKEN@github.com/.insteadOf https://github.com/ | |
| $GOGETCMD | |
| env: | |
| GOGETCMD: ${{env.GOGETCMD}} | |
| GH_ACCESS_TOKEN: ${{env.GH_ACCESS_TOKEN}} | |
| GOPRIVATE: ${{env.GOPRIVATE}} | |
| - name: Generate Build Flags and Build | |
| env: | |
| BUILDCOMMITSHA: ${{github.sha}} | |
| BUILDBRANCH: ${{github.ref}} | |
| BUILDPLATFORM: ${{runner.os}} | |
| BUILDPATCHVERSION: ${{github.run_number}} | |
| run: | | |
| BUILDSHORTCOMMITSHA=$(echo $BUILDCOMMITSHA | cut -c 1-7) | |
| BUILDDATE=$(date) | |
| echo "BUILDBRANCH: ${BUILDBRANCH}" | |
| echo "BUILDCOMMITSHA: ${BUILDCOMMITSHA}" | |
| echo "BUILDSHORTCOMMITSHA: ${BUILDSHORTCOMMITSHA}" | |
| echo "BUILDDATE: ${BUILDDATE}" | |
| echo "BUILDPLATFORM: ${BUILDPLATFORM}" | |
| go build -x -v \ | |
| -o build/anysdk ./cmd/interrogate | |
| - name: Test | |
| if: success() | |
| run: go test -timeout 240s -v ./... | |
| - name: Upload any-sdk binary | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: anysdk_binary | |
| path: build/anysdk | |
| build-stackql: | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Set up golang | |
| uses: actions/setup-go@v6 | |
| with: | |
| go-version: ^${{ env.GOLANG_VERSION }} | |
| cache: false | |
| id: go | |
| - name: Download core | |
| uses: actions/checkout@v7 | |
| with: | |
| repository: ${{ env.STACKQL_CORE_REPOSITORY }} | |
| ref: ${{ env.STACKQL_CORE_REF }} | |
| token: ${{ secrets.CI_STACKQL_PACKAGE_DOWNLOAD_TOKEN }} | |
| path: stackql-core | |
| - name: Build stackql from core source | |
| working-directory: stackql-core | |
| run: | | |
| go get ./... | |
| python3 cicd/python/build.py --build | |
| - name: Upload stackql binary | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: stackql_binary | |
| path: stackql-core/build/stackql | |
| analysis: | |
| name: Analysis | |
| runs-on: ubuntu-24.04 | |
| needs: | |
| - build-anysdk | |
| - build-stackql | |
| steps: | |
| - name: Check out code into the Go module directory | |
| uses: actions/checkout@v7 | |
| - name: Download stackql binary | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: stackql_binary | |
| path: build | |
| - name: Download any-sdk binary | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: anysdk_binary | |
| path: build | |
| - name: Produce requisite binaries executable and add to path | |
| run: | | |
| echo "${{ github.workspace }}/build" >> $GITHUB_PATH; | |
| chmod +x "${{ github.workspace }}/build/stackql" | |
| chmod +x "${{ github.workspace }}/build/anysdk" | |
| - name: Download core | |
| uses: actions/checkout@v7 | |
| with: | |
| repository: ${{ env.STACKQL_CORE_REPOSITORY }} | |
| ref: ${{ env.STACKQL_CORE_REF }} | |
| path: stackql-core | |
| - name: Pull Providers | |
| run: | | |
| stackql exec "registry pull aws ${PROVIDER_VERSION_AWS};" | |
| stackql exec "registry pull google ${PROVIDER_VERSION_GOOGLE};" | |
| stackql exec "registry pull azure ${PROVIDER_VERSION_AZURE};" | |
| RUN_EPOCH="$(date -u +%s)" | |
| RUN_TS="$(date -u -d "@${RUN_EPOCH}" +%Y-%m-%dT%H-%M-%SZ)" | |
| DATE_PATH="$(date -u -d "@${RUN_EPOCH}" +%Y/%m/%d)" | |
| { | |
| echo "RUN_EPOCH=${RUN_EPOCH}" | |
| echo "DATE_PATH=${DATE_PATH}" | |
| echo "RUN_TS=${RUN_TS}" | |
| } >> "$GITHUB_ENV" | |
| - name: AOT Analysis and Mock Generation | |
| run: | | |
| for provider_cfg in \ | |
| "aws|aws|${PROVIDER_VERSION_AWS}" \ | |
| "azure|azure|${PROVIDER_VERSION_AZURE}" \ | |
| "google|googleapis.com|${PROVIDER_VERSION_GOOGLE}"; do | |
| IFS='|' read -r provider handle version <<< "$provider_cfg" | |
| ./build/anysdk aot \ | |
| ./.stackql \ | |
| ./.stackql/src/${handle}/${version}/provider.yaml \ | |
| -v \ | |
| --mock-output-dir "cicd/out/auto-mocks/${provider}" \ | |
| --mock-expectation-dir "cicd/out/mock-expectations/${provider}" \ | |
| --mock-query-dir "cicd/out/mock-queries/${provider}" \ | |
| --schema-dir cicd/schema-definitions \ | |
| --stdout-file "cicd/out/aot/${RUN_TS}-${provider}-${version}-summary.json" \ | |
| --stderr-file "cicd/out/aot/${RUN_TS}-${provider}-${version}-analysis.jsonl" || true | |
| done | |
| - name: Generate Per-Resource Closures with Provider Docs | |
| run: | | |
| MOCK_PORT=5000 | |
| for provider_cfg in \ | |
| "aws|aws|${PROVIDER_VERSION_AWS}" \ | |
| "azure|azure|${PROVIDER_VERSION_AZURE}" \ | |
| "google|googleapis.com|${PROVIDER_VERSION_GOOGLE}"; do | |
| IFS='|' read -r provider handle version <<< "$provider_cfg" | |
| for svc_file in .stackql/src/${handle}/${version}/services/*.yaml; do | |
| svc_name="$(basename "$svc_file" .yaml)" | |
| ./build/anysdk closure \ | |
| ./.stackql \ | |
| ./.stackql/src/${handle}/${version}/provider.yaml \ | |
| "$svc_name" \ | |
| --provider "$provider" \ | |
| --rewrite-url "http://localhost:${MOCK_PORT}" \ | |
| --output-dir "cicd/out/closures/${provider}" \ | |
| 2>/dev/null || true | |
| done | |
| done | |
| - name: Upload Provider Analysis results | |
| uses: actions/upload-artifact@v7 | |
| if: always() | |
| with: | |
| name: provider-analysis-results | |
| path: cicd/out/aot/ | |
| - name: Upload Auto Mocks | |
| uses: actions/upload-artifact@v7 | |
| if: always() | |
| with: | |
| name: auto-mocks | |
| path: cicd/out/auto-mocks/ | |
| - name: Upload Mock Expectations | |
| uses: actions/upload-artifact@v7 | |
| if: always() | |
| with: | |
| name: mock-expectations | |
| path: cicd/out/mock-expectations/ | |
| - name: Upload Mock Queries | |
| uses: actions/upload-artifact@v7 | |
| if: always() | |
| with: | |
| name: mock-queries | |
| path: cicd/out/mock-queries/ | |
| - name: Upload Closures | |
| uses: actions/upload-artifact@v7 | |
| if: always() | |
| with: | |
| name: closures | |
| path: cicd/out/closures/ | |
| - name: Authenticate to Google Cloud | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| credentials_json: ${{ secrets.ANALYSIS_UPLOAD_GCS }} | |
| - name: Set up Cloud SDK | |
| uses: google-github-actions/setup-gcloud@v2 | |
| - name: Upload all artifacts to GCS | |
| run: | | |
| set -e | |
| DEST="gs://${ANALYSIS_RESULTS_BUCKET}/provider-analysis/${DATE_PATH}/${RUN_TS}" | |
| echo "Uploading to ${DEST}" | |
| gcloud storage cp -r cicd/out/aot "${DEST}/" | |
| gcloud storage cp -r cicd/out/auto-mocks "${DEST}/" | |
| gcloud storage cp -r cicd/out/mock-expectations "${DEST}/" | |
| gcloud storage cp -r cicd/out/mock-queries "${DEST}/" | |
| gcloud storage cp -r cicd/out/closures "${DEST}/" | |
| echo "{\"latest\": \"${DEST}\"}" > latest.json | |
| gcloud storage cp latest.json "gs://${ANALYSIS_RESULTS_BUCKET}/provider-analysis/latest.json" | |