From 998308341223d19090c004a455c780d4972e661b Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Tue, 28 Jan 2025 13:19:53 +0000 Subject: [PATCH] build wheels for emscripten (#185) Co-authored-by: Samuel Colvin --- .github/actions/build-pgo-wheel/action.yml | 6 +- .github/workflows/ci.yml | 82 ++++++++++++++++++- crates/jiter-python/package.json | 26 ++++++ .../jiter-python/tests/emscripten_runner.js | 71 ++++++++++++++++ crates/jiter-python/tests/test_jiter.py | 4 + 5 files changed, 183 insertions(+), 6 deletions(-) create mode 100644 crates/jiter-python/package.json create mode 100644 crates/jiter-python/tests/emscripten_runner.js diff --git a/.github/actions/build-pgo-wheel/action.yml b/.github/actions/build-pgo-wheel/action.yml index 824d522..c28e7c7 100644 --- a/.github/actions/build-pgo-wheel/action.yml +++ b/.github/actions/build-pgo-wheel/action.yml @@ -40,9 +40,9 @@ runs: - name: generate pgo data run: | - pip install -U pip - pip install -r tests/requirements.txt - pip install jiter --no-index --no-deps --find-links pgo-wheel --force-reinstall + python -m pip install -U pip + python -m pip install -r tests/requirements.txt + python -m pip install jiter --no-index --no-deps --find-links pgo-wheel --force-reinstall python bench.py jiter jiter-cache RUST_HOST=$(rustc -Vv | grep host | cut -d ' ' -f 2) rustup run ${{ inputs.rust-toolchain }} bash -c 'echo LLVM_PROFDATA=$RUSTUP_HOME/toolchains/$RUSTUP_TOOLCHAIN/lib/rustlib/$RUST_HOST/bin/llvm-profdata >> "$GITHUB_ENV"' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6a8c88f..d27b669 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -395,6 +395,64 @@ jobs: name: pypi_files_${{ matrix.os }}_${{ matrix.interpreter }} path: crates/jiter-python/dist + build-wasm-emscripten: + # only run on push to main and on release + if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'Full Build') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - id: setup-python + name: set up python + uses: quansight-labs/setup-python@v5 + with: + python-version: 3.12 + allow-prereleases: true + + - name: install rust nightly + uses: dtolnay/rust-toolchain@nightly + with: + components: rust-src + targets: wasm32-unknown-emscripten + + - name: cache rust + uses: Swatinem/rust-cache@v2 + + - uses: mymindstorm/setup-emsdk@v14 + with: + # NOTE!: as per https://github.com/pydantic/pydantic-core/pull/149 this version needs to match the version + # in node_modules/pyodide/repodata.json, to get the version, run: + # `cat node_modules/pyodide/repodata.json | python -m json.tool | rg platform` + version: "3.1.58" + actions-cache-folder: emsdk-cache + + - name: install deps + run: pip install -U pip maturin + + - name: build wheels + run: maturin build --release --target wasm32-unknown-emscripten --out dist -i 3.12 + working-directory: crates/jiter-python + + - uses: actions/setup-node@v4 + with: + node-version: "18" + + - run: npm install + working-directory: crates/jiter-python + + - run: npm run test + working-directory: crates/jiter-python + + - run: | + ls -lh dist/ + ls -l dist/ + working-directory: crates/jiter-python + + - uses: actions/upload-artifact@v4 + with: + name: wasm_wheels + path: crates/jiter-python/dist + inspect-pypi-assets: needs: [build, build-sdist, build-pgo] runs-on: ubuntu-latest @@ -524,7 +582,7 @@ jobs: release: needs: [check] - if: "success() && startsWith(github.ref, 'refs/tags/')" + if: success() && startsWith(github.ref, 'refs/tags/') runs-on: ubuntu-latest environment: release @@ -541,8 +599,13 @@ jobs: CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} release-python: - needs: [test-builds-arch, test-builds-os, build-sdist, check] - if: "success() && startsWith(github.ref, 'refs/tags/')" + needs: + - check + - test-builds-arch + - test-builds-os + - build-sdist + - build-wasm-emscripten + if: success() && startsWith(github.ref, 'refs/tags/') runs-on: ubuntu-latest environment: release-python permissions: @@ -573,3 +636,16 @@ jobs: uses: pypa/gh-action-pypi-publish@release/v1 with: packages-dir: dist/ + + - name: get wasm dist artifacts + uses: actions/download-artifact@v4 + with: + name: wasm_wheels + path: wasm + + - name: upload to github release + uses: softprops/action-gh-release@v2 + with: + files: | + wasm/*.whl + prerelease: ${{ contains(github.ref, 'alpha') || contains(github.ref, 'beta') }} diff --git a/crates/jiter-python/package.json b/crates/jiter-python/package.json new file mode 100644 index 0000000..95fab33 --- /dev/null +++ b/crates/jiter-python/package.json @@ -0,0 +1,26 @@ +{ + "name": "jiter", + "version": "1.0.0", + "description": "for running wasm tests.", + "author": "Samuel Colvin", + "license": "MIT", + "homepage": "https://github.com/pydantic/jiter#readme", + "main": "tests/emscripten_runner.js", + "dependencies": { + "prettier": "^2.7.1", + "pyodide": "^0.26.3" + }, + "scripts": { + "test": "node tests/emscripten_runner.js", + "format": "prettier --write 'tests/emscripten_runner.js' 'wasm-preview/*.{html,js}'", + "lint": "prettier --check 'tests/emscripten_runner.js' 'wasm-preview/*.{html,js}'" + }, + "prettier": { + "singleQuote": true, + "trailingComma": "all", + "tabWidth": 2, + "printWidth": 119, + "bracketSpacing": false, + "arrowParens": "avoid" + } +} diff --git a/crates/jiter-python/tests/emscripten_runner.js b/crates/jiter-python/tests/emscripten_runner.js new file mode 100644 index 0000000..86c8cd5 --- /dev/null +++ b/crates/jiter-python/tests/emscripten_runner.js @@ -0,0 +1,71 @@ +const {opendir} = require('node:fs/promises'); +const {loadPyodide} = require('pyodide'); +const path = require('path'); + +async function find_wheel(dist_dir) { + const dir = await opendir(dist_dir); + for await (const dirent of dir) { + if (dirent.name.endsWith('.whl')) { + return path.join(dist_dir, dirent.name); + } + } +} + +async function main() { + const root_dir = path.resolve(__dirname, '..'); + const wheel_path = await find_wheel(path.join(root_dir, 'dist')); + const stdout = [] + const stderr = [] + let errcode = 1; + try { + const pyodide = await loadPyodide({ + stdout: (msg) => { + stdout.push(msg) + }, + stderr: (msg) => { + stderr.push(msg) + } + }); + const FS = pyodide.FS; + FS.mkdir('/test_dir'); + FS.mount(FS.filesystems.NODEFS, {root: path.join(root_dir, 'tests')}, '/test_dir'); + FS.chdir('/test_dir'); + + // mount jiter crate source for benchmark data + FS.mkdir('/jiter'); + FS.mount(FS.filesystems.NODEFS, {root: path.resolve(root_dir, "..", "jiter")}, '/jiter'); + + await pyodide.loadPackage(['micropip', 'pytest']); + // language=python + errcode = await pyodide.runPythonAsync(` +import micropip +import importlib + +# ugly hack to get tests to work on arm64 (my m1 mac) +# see https://github.com/pyodide/pyodide/issues/2840 +# import sys; sys.setrecursionlimit(200) + +await micropip.install([ + 'dirty_equals', + 'file:${wheel_path}' +]) +importlib.invalidate_caches() + +print('installed packages:', micropip.list()) + +import pytest +pytest.main() +`); + } catch (e) { + console.error(e); + process.exit(1); + } + let out = stdout.join('\n') + let err = stderr.join('\n') + console.log('stdout:\n', out) + console.log('stderr:\n', err) + + process.exit(errcode); +} + +main(); diff --git a/crates/jiter-python/tests/test_jiter.py b/crates/jiter-python/tests/test_jiter.py index 31646a0..2b960f6 100644 --- a/crates/jiter-python/tests/test_jiter.py +++ b/crates/jiter-python/tests/test_jiter.py @@ -2,6 +2,7 @@ import json from decimal import Decimal from pathlib import Path +import sys from typing import Any import jiter @@ -352,6 +353,9 @@ def test_against_json(): assert jiter.from_json(data) == json.loads(data) +@pytest.mark.skipif( + sys.platform == 'emscripten', reason='threads not supported on pyodide' +) def test_multithreaded_parsing(): """Basic sanity check that running a parse in multiple threads is fine.""" expected_datas = [json.loads(data) for data in JITER_BENCH_DATAS]