Try pre-building codecs before Pyodide build #80
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and Deploy | |
| on: | |
| push: | |
| branches: | |
| - main | |
| jobs: | |
| build: | |
| permissions: | |
| contents: write | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout the repository | |
| uses: actions/checkout@v3 | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| - name: Install the dependencies | |
| run: | | |
| python -m pip install -r requirements.txt | |
| - name: Prepare Pyodide | |
| run: | | |
| git submodule init pyodide | |
| git submodule update pyodide | |
| - name: Check the Pyodide version | |
| run: | | |
| echo "PYODIDE_SHA_SHORT=$(git submodule status pyodide | cut -c 2-8)" >> $GITHUB_ENV | |
| - name: Cache the Pyodide build | |
| uses: actions/cache@v3 | |
| with: | |
| path: ./pyodide | |
| key: pyodide-${{ env.PYODIDE_SHA_SHORT }}-${{ hashFiles('patches/pyodide.patch') }} | |
| restore-keys: | | |
| pyodide-${{ env.PYODIDE_SHA_SHORT }}-${{ hashFiles('patches/pyodide.patch') }} | |
| pyodide-${{ env.PYODIDE_SHA_SHORT }} | |
| - name: Add a read-only SSH key to pull private dependencies | |
| uses: webfactory/[email protected] | |
| with: | |
| ssh-private-key: ${{ secrets.FCBENCH_SSH_TOKEN }} | |
| - name: Checkout and patch pyodide | |
| run: | | |
| cd pyodide | |
| git apply --whitespace=nowarn ../patches/pyodide.patch | |
| cp -r ../fcpy packages/fcpy/fcpy | |
| git clone [email protected]:juntyr/field-compression-benchmark.git \ | |
| packages/field-compression-benchmark/field-compression-benchmark | |
| cd packages/field-compression-benchmark/field-compression-benchmark | |
| git submodule update --init wasi/wit-deps/wasi-* | |
| git apply ../patches/0001-codecs-build.patch | |
| - name: Pre-build the wasm codecs for fcbench | |
| run: | | |
| cd pyodide/packages/field-compression-benchmark/field-compression-benchmark | |
| # Set up a more modern nightly toolchain with support for emscripten and wasi | |
| rustup toolchain install ${RUSTUP_TOOLCHAIN} | |
| rustup target add wasm32-unknown-emscripten --toolchain ${RUSTUP_TOOLCHAIN} | |
| rustup target add wasm32-unknown-unknown --toolchain ${RUSTUP_TOOLCHAIN} | |
| rustup component add rust-src --toolchain ${RUSTUP_TOOLCHAIN} | |
| # Install nix-portable and add it to the PATH | |
| mkdir -p .nix | |
| wget -q -O - https://github.com/DavHau/nix-portable/releases/download/v012/nix-portable-x86_64 > .nix/nix-portable | |
| cd ./.nix | |
| chmod +x ./nix-portable | |
| ln -s ./nix-portable ./nix | |
| cd .. | |
| # Build the codecs | |
| PATH="$(pwd)/.nix:${PATH}" cargo build -p codecs-build -vv | |
| # Clean up everything after the build | |
| cargo clean | |
| rm -rf ./.nix | |
| chmod -R 777 ~/.nix-portable | |
| rm -rf ~/.nix-portable | |
| rustup toolchain uninstall ${RUSTUP_TOOLCHAIN} | |
| env: | |
| RUSTUP_TOOLCHAIN: "nightly-2024-04-28" | |
| - name: Build Pyodide and all required packages | |
| run: | | |
| cd pyodide | |
| PYODIDE_DOCKER_IMAGE=$(docker build . -q) \ | |
| ./run_docker \ | |
| --non-interactive \ | |
| PYODIDE_PACKAGES="basemap,fcpy,field-compression-benchmark,ipyfilite,ipyloglite,pyodide-http,ipython,earthkit" \ | |
| make | |
| - name: Clean up the pyodide build | |
| run: | | |
| cd pyodide | |
| git apply --whitespace=nowarn --reverse ../patches/pyodide.patch | |
| rm -rf packages/fcpy/fcpy | |
| rm -rf packages/field-compression-benchmark/field-compression-benchmark | |
| rm -rf .docker_home | |
| find . -name jupyter-lite.json -delete | |
| find . -name jupyter_lite_config.json -delete | |
| cd .. | |
| mkdir -p static | |
| cp -r pyodide/dist static/pyodide | |
| rm -rf static/pyodide/*-tests.tar | |
| rm -f static/pyodide/tsconfig.tsbuildinfo | |
| - name: Build the JupyterLite site | |
| run: | | |
| jupyter lite build --output-dir dist | |
| cp ./_headers dist/ | |
| cp -r data dist/ | |
| cp static/favicons/favicon.ico dist/ | |
| find dist/ -name favicon.ico | xargs -L1 cp -f static/favicons/favicon.ico | |
| rm -rf static | |
| - name: Deploy to dist branch | |
| uses: peaceiris/actions-gh-pages@v3 | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| publish_branch: dist | |
| publish_dir: ./dist | |
| force_orphan: true |