Update tileserver #55
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Pull Request Backend | |
| on: | |
| pull_request: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number }} | |
| cancel-in-progress: true | |
| env: | |
| python-version: '3.10' | |
| J40_VERSION_LABEL_STRING: ${{ vars.SCORE_VERSION }} | |
| jobs: | |
| # JOB to run change detection | |
| detect-be-changes: | |
| name: Detect backend changes | |
| runs-on: ubuntu-latest | |
| # Required permissions | |
| permissions: | |
| pull-requests: read | |
| # Set job outputs to values from filter step | |
| outputs: | |
| backend: ${{ steps.filter.outputs.backend }} | |
| steps: | |
| # For pull requests it's not necessary to checkout the code | |
| - uses: dorny/paths-filter@v3 | |
| id: filter | |
| with: | |
| filters: | | |
| backend: | |
| - 'data/**' | |
| - '.github/workflows/pr_backend.yml' | |
| code-quality-checks: | |
| name: Code quality checks and tests | |
| needs: detect-be-changes | |
| if: ${{ needs.detect-be-changes.outputs.backend == 'true' }} | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| working-directory: data/data-pipeline | |
| environment: Staging | |
| steps: | |
| - name: Checkout source | |
| uses: actions/checkout@v4 | |
| - name: Print variables to help debug | |
| uses: hmarr/debug-action@v3 | |
| - name: Set up Python ${{ env.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.python-version }} | |
| - name: Load cached Poetry installation | |
| id: cached-poetry-dependencies | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/pypoetry/virtualenvs | |
| key: cejst-poetry-env-${{ runner.os }}-${{ env.python-version }}-${{ hashFiles('**/poetry.lock') }}-${{ hashFiles('.github/workflows/deploy_backend_main.yml') }} | |
| - name: Install poetry | |
| uses: snok/install-poetry@v1 | |
| - name: Install dependencies | |
| run: poetry install | |
| if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' | |
| - name: Check code is formatted | |
| run: poetry run black --check data_pipeline/ | |
| - name: Check code style consistency | |
| run: poetry run flake8 -v data_pipeline/ | |
| - name: Run static code analysis | |
| run: poetry run pylint data_pipeline/ | |
| - name: Check library safety | |
| run: poetry run safety check --ignore 51457 --ignore 44715 --ignore 70612 --ignore 74439 | |
| - name: Run unit tests | |
| run: | | |
| poetry run pytest data_pipeline/ | |
| generate-score-tiles: | |
| name: Score and tile generation | |
| needs: detect-be-changes | |
| if: ${{ needs.detect-be-changes.outputs.backend == 'true' }} | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| working-directory: data/data-pipeline | |
| environment: Staging | |
| steps: | |
| - name: Checkout source | |
| uses: actions/checkout@v4 | |
| - name: Print variables to help debug | |
| uses: hmarr/debug-action@v3 | |
| - name: Set up Python ${{ env.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.python-version }} | |
| - name: Load cached Poetry installation | |
| id: cached-poetry-dependencies | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/pypoetry/virtualenvs | |
| key: cejst-poetry-env-${{ runner.os }}-${{ env.python-version }}-${{ hashFiles('**/poetry.lock') }}-${{ hashFiles('.github/workflows/deploy_backend_main.yml') }} | |
| - name: Install poetry | |
| uses: snok/install-poetry@v1 | |
| - name: Print Poetry settings | |
| run: poetry show -v | |
| - name: Install dependencies | |
| run: poetry add s4cmd && poetry install | |
| if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' | |
| - name: Load cached ETL data | |
| id: cached-etl-data | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| data/data-pipeline/data_pipeline/data/census | |
| data/data-pipeline/data_pipeline/data/dataset | |
| key: cejst-dataset-env-${{ runner.os }}-${{ env.python-version }}-${{ hashFiles('data/data-pipeline/data_pipeline/etl/**/*') }}-${{ hashFiles('data/data-pipeline/data_pipeline/utils.py') }} | |
| - name: Get Census Data | |
| if: steps.cached-etl-data.outputs.cache-hit != 'true' | |
| run: | | |
| poetry run python3 -m data_pipeline.application census-data-download | |
| - name: Run ETL | |
| if: steps.cached-etl-data.outputs.cache-hit != 'true' | |
| run: | | |
| poetry run python3 -m data_pipeline.application etl-run | |
| - name: Generate Score | |
| run: | | |
| poetry run python3 -m data_pipeline.application score-run | |
| - name: Score Compare | |
| run: | | |
| poetry run python3 -m data_pipeline.comparator compare-score | |
| - name: Generate Score Post | |
| run: | | |
| poetry run python3 -m data_pipeline.application generate-score-post | |
| - name: Confirm we generated the version of the score we think we did | |
| if: ${{ env.J40_VERSION_LABEL_STRING == '2.0' || env.J40_VERSION_LABEL_STRING == 'beta' }} | |
| run: | | |
| grep -v "Identified as disadvantaged due to tribal overlap" data_pipeline/data/score/downloadable/* > /dev/null | |
| - name: Generate Score Geo | |
| run: | | |
| poetry run python3 -m data_pipeline.application geo-score | |
| - name: Run smoketest for 1.0 | |
| if: ${{ env.J40_VERSION_LABEL_STRING == '1.0' }} | |
| run: | | |
| poetry run pytest data_pipeline/ -m smoketest | |
| - name: Set timezone for tippecanoe | |
| uses: szenius/[email protected] | |
| with: | |
| timezoneLinux: "America/Los_Angeles" | |
| - name: Get tippecanoe | |
| run: | | |
| sudo apt-get install -y software-properties-common libsqlite3-dev zlib1g-dev | |
| sudo apt-add-repository -y ppa:git-core/ppa | |
| sudo mkdir -p /tmp/tippecanoe-src | |
| sudo git clone https://github.com/mapbox/tippecanoe.git /tmp/tippecanoe-src | |
| - name: Make tippecanoe | |
| working-directory: /tmp/tippecanoe-src | |
| run: | | |
| sudo /usr/bin/bash -c make | |
| mkdir -p /usr/local/bin | |
| cp tippecanoe /usr/local/bin/tippecanoe | |
| tippecanoe -v | |
| - name: Generate Tiles | |
| run: | | |
| poetry run python3 -m data_pipeline.application generate-map-tiles | |
| poetry run python3 -m data_pipeline.application generate-map-tiles --generate-tribal-layer | |