fix(benchmarks): force reinstall requirement overrides and wait for J… #2285
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: | |
| - "**" | |
| pull_request: | |
| workflow_dispatch: | |
| workflow_call: | |
| secrets: | |
| CODECOV_TOKEN: | |
| required: false | |
| defaults: | |
| run: | |
| shell: bash -l -eo pipefail {0} | |
| jobs: | |
| test: | |
| name: Python ${{ matrix.python-version }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] | |
| env: | |
| STORAGE_EMULATOR_HOST: http://localhost:4443 | |
| steps: | |
| - name: Checkout source | |
| uses: actions/checkout@v5 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup conda | |
| uses: conda-incubator/setup-miniconda@v3 | |
| with: | |
| environment-file: environment_gcsfs.yaml | |
| python-version: ${{ matrix.PY }} | |
| activate-environment: gcsfs_test | |
| - name: Conda info | |
| run: | | |
| conda list | |
| conda --version | |
| - name: install | |
| run: | | |
| pip install -e . | |
| - name: Start GCS Emulator | |
| run: | | |
| docker run -d -p 4443:4443 --name gcsfs_test fsouza/fake-gcs-server:latest -scheme http -public-host 0.0.0.0:4443 -external-url http://localhost:4443 -backend memory | |
| timeout 30s bash -c 'until curl -s http://localhost:4443/storage/v1/b; do sleep 1; done' | |
| - name: Run Standard Bucket Tests with extended feature support turned OFF | |
| run: | | |
| export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/gcsfs/tests/fake-service-account-credentials.json | |
| export GCSFS_EXPERIMENTAL_ZB_HNS_SUPPORT="false" | |
| pytest -vv -s \ | |
| --cov=gcsfs --cov-report=xml \ | |
| --log-format="%(asctime)s %(levelname)s %(message)s" \ | |
| --log-date-format="%H:%M:%S" \ | |
| -n auto \ | |
| gcsfs/ | |
| - name: Run Benchmark Infrastructure Unit Tests | |
| run: | | |
| pip install -r gcsfs/tests/perf/microbenchmarks/requirements.txt | |
| pytest gcsfs/tests/perf/microbenchmarks --run-benchmarks-infra | |
| - name: Run Macrobenchmark Infrastructure Unit Tests | |
| run: | | |
| pip install -r cloudbuild/macrobenchmarks/metrics/requirements.txt | |
| PYTHONPATH=cloudbuild/macrobenchmarks pytest cloudbuild/macrobenchmarks/metrics/tests | |
| - name: Run Release Automation Tests | |
| run: | | |
| pytest -vv .github/release/ | |
| - name: Run all tests (Standard, Zonal & HNS Enabled) with default ON extended feature support | |
| run: | | |
| export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/gcsfs/tests/fake-service-account-credentials.json | |
| pytest -vv -s \ | |
| --cov=gcsfs --cov-append --cov-report=xml \ | |
| --log-format="%(asctime)s %(levelname)s %(message)s" \ | |
| --log-date-format="%H:%M:%S" \ | |
| -n auto \ | |
| gcsfs/ | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v5 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| files: ./coverage.xml | |
| fail_ci_if_error: false | |
| subsystembenchmark-unit-tests: | |
| name: Subsystem benchmark unit tests | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| env: | |
| STORAGE_EMULATOR_HOST: http://localhost:4443 | |
| steps: | |
| - name: Checkout source | |
| uses: actions/checkout@v5 | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.12" | |
| cache: pip | |
| cache-dependency-path: gcsfs/tests/perf/subsystembenchmarks/dataloading/huggingface_datasets/requirements.txt | |
| - name: Install subsystem benchmark dependencies | |
| run: | | |
| python -m pip install -e . | |
| python -m pip install -r gcsfs/tests/perf/subsystembenchmarks/dataloading/huggingface_datasets/requirements.txt | |
| - name: Run subsystem benchmark unit tests | |
| run: pytest gcsfs/tests/perf/subsystembenchmarks --run-benchmarks-infra | |
| lint: | |
| name: lint | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.11" | |
| - uses: pre-commit/action@v3.0.1 |