Model Server Tests #295
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Model Server Tests | |
| on: | |
| schedule: | |
| # This cron expression runs the job daily at 16:00 UTC (9am PT) | |
| - cron: "0 16 * * *" | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| env: | |
| # Bedrock | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION_NAME: ${{ vars.AWS_REGION_NAME }} | |
| # API keys for testing | |
| COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} | |
| LITELLM_API_KEY: ${{ secrets.LITELLM_API_KEY }} | |
| LITELLM_API_URL: ${{ secrets.LITELLM_API_URL }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }} | |
| AZURE_API_URL: ${{ vars.AZURE_API_URL }} | |
| jobs: | |
| model-check: | |
| # See https://runs-on.com/runners/linux/ | |
| runs-on: | |
| - runs-on | |
| - runner=4cpu-linux-arm64 | |
| - "run-id=${{ github.run_id }}-model-check" | |
| - "extras=ecr-cache" | |
| timeout-minutes: 45 | |
| env: | |
| PYTHONPATH: ./backend | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # ratchet:actions/checkout@v6 | |
| with: | |
| persist-credentials: false | |
| - name: Setup Python and Install Dependencies | |
| uses: ./.github/actions/setup-python-and-install-dependencies | |
| with: | |
| requirements: | | |
| backend/requirements/default.txt | |
| backend/requirements/dev.txt | |
| - name: Format branch name for cache | |
| id: format-branch | |
| env: | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| REF_NAME: ${{ github.ref_name }} | |
| run: | | |
| if [ -n "${PR_NUMBER}" ]; then | |
| CACHE_SUFFIX="${PR_NUMBER}" | |
| else | |
| # shellcheck disable=SC2001 | |
| CACHE_SUFFIX=$(echo "${REF_NAME}" | sed 's/[^A-Za-z0-9._-]/-/g') | |
| fi | |
| echo "cache-suffix=${CACHE_SUFFIX}" >> $GITHUB_OUTPUT | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 | |
| with: | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_TOKEN }} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f | |
| - name: Build and load | |
| uses: docker/bake-action@5be5f02ff8819ecd3092ea6b2e6261c31774f2b4 # ratchet:docker/bake-action@v6 | |
| env: | |
| TAG: model-server-${{ github.run_id }} | |
| with: | |
| load: true | |
| targets: model-server | |
| set: | | |
| model-server.cache-from=type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-${{ github.event.pull_request.head.sha || github.sha }} | |
| model-server.cache-from=type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-${{ steps.format-branch.outputs.cache-suffix }} | |
| model-server.cache-from=type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache | |
| model-server.cache-from=type=registry,ref=onyxdotapp/onyx-model-server:latest | |
| model-server.cache-to=type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-${{ github.event.pull_request.head.sha || github.sha }},mode=max | |
| model-server.cache-to=type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-${{ steps.format-branch.outputs.cache-suffix }},mode=max | |
| model-server.cache-to=type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache,mode=max | |
| - name: Start Docker containers | |
| id: start_docker | |
| env: | |
| IMAGE_TAG: model-server-${{ github.run_id }} | |
| run: | | |
| cd deployment/docker_compose | |
| docker compose \ | |
| -f docker-compose.yml \ | |
| -f docker-compose.dev.yml \ | |
| up -d --wait \ | |
| inference_model_server | |
| - name: Run Tests | |
| run: | | |
| py.test -o junit_family=xunit2 -xv --ff backend/tests/daily/llm | |
| py.test -o junit_family=xunit2 -xv --ff backend/tests/daily/embedding | |
| - name: Alert on Failure | |
| if: failure() && github.event_name == 'schedule' | |
| uses: ./.github/actions/slack-notify | |
| with: | |
| webhook-url: ${{ secrets.SLACK_WEBHOOK }} | |
| failed-jobs: model-check | |
| title: "🚨 Scheduled Model Tests failed!" | |
| ref-name: ${{ github.ref_name }} | |
| - name: Dump all-container logs (optional) | |
| if: always() | |
| run: | | |
| cd deployment/docker_compose | |
| docker compose logs --no-color > $GITHUB_WORKSPACE/docker-compose.log || true | |
| - name: Upload logs | |
| if: always() | |
| uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f | |
| with: | |
| name: docker-all-logs | |
| path: ${{ github.workspace }}/docker-compose.log |