Nightly Compatibility #38
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Nightly Compatibility | |
| on: | |
| schedule: | |
| - cron: "0 3 * * *" # 03:00 UTC daily | |
| workflow_dispatch: # Manual trigger | |
| # No workflow-level permissions — each job declares the minimal set it needs. | |
| jobs: | |
| test-latest-deps: | |
| name: Test with latest dependencies | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.11", "3.12"] | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install with latest deps (no upper bounds) | |
| run: | | |
| pip install --upgrade pip | |
| pip install torch transformers peft datasets accelerate trl pydantic pyyaml tensorboard huggingface_hub requests | |
| pip install pytest pytest-cov ruff | |
| - name: Install ForgeLM (editable, no deps — already installed above) | |
| run: pip install -e . --no-deps | |
| - name: Show dependency versions | |
| run: | | |
| python -c " | |
| import importlib | |
| for pkg in ['torch', 'transformers', 'peft', 'datasets', 'accelerate', 'trl', 'pydantic']: | |
| mod = importlib.import_module(pkg) | |
| print(f'{pkg}: {getattr(mod, \"__version__\", \"unknown\")}') | |
| " | |
| - name: Lint check | |
| run: ruff check . | |
| - name: Run tests | |
| run: pytest tests/ -q --tb=short | |
| - name: CLI smoke test | |
| run: | | |
| forgelm --version | |
| forgelm --config config_template.yaml --dry-run | |
| forgelm --config config_template.yaml --dry-run --output-format json | |
| - name: Quickstart templates smoke test | |
| run: | | |
| # Every bundled template must render to a valid YAML that | |
| # passes pydantic validation. Catches template drift the | |
| # moment a config schema changes. | |
| # | |
| # Note: smoke-tests 4 of 5 templates. `domain-expert` is BYOD | |
| # (no bundled dataset), so `quickstart --dry-run` rejects it | |
| # without `--dataset PATH`. It's covered separately by the | |
| # pytest unit `test_domain_expert_intentionally_has_no_bundled_data`. | |
| forgelm quickstart --list | |
| for tpl in customer-support code-assistant medical-qa-tr grpo-math; do | |
| echo "=== Quickstart dry-run: $tpl ===" | |
| forgelm quickstart "$tpl" --dry-run --output "/tmp/qs-$tpl.yaml" | |
| forgelm --config "/tmp/qs-$tpl.yaml" --dry-run | |
| done | |
| - name: Ingestion + audit smoke test (Phase 11) | |
| run: | | |
| # Minimal end-to-end: TXT in → JSONL out → audit report out. | |
| # Plain TXT path doesn't need the [ingestion] extra; the audit | |
| # module is pure stdlib. Catches CLI wiring drift without paying | |
| # for pypdf / langdetect installs. | |
| mkdir -p /tmp/p11 | |
| echo "Article 10 governs data quality." > /tmp/p11/sample.txt | |
| echo "Section two: representativeness, traceability, bias review." > /tmp/p11/sample2.txt | |
| forgelm ingest /tmp/p11/ --recursive --output /tmp/p11/out.jsonl | |
| test -s /tmp/p11/out.jsonl | |
| forgelm --data-audit /tmp/p11/out.jsonl --output /tmp/p11/audit/ | |
| test -s /tmp/p11/audit/data_audit_report.json | |
| wheel-install-smoke: | |
| # ------------------------------------------------------------------ | |
| # This is the only test that catches package_data globs being broken | |
| # — editable installs always hide this. `pip install -e .` resolves | |
| # forgelm/templates/* via Path(__file__).parent regardless of what | |
| # setuptools would actually copy into the wheel, so a broken | |
| # [tool.setuptools.package-data] entry is invisible until a real user | |
| # runs `pip install forgelm` from PyPI and gets a missing-asset | |
| # FileNotFoundError on `forgelm quickstart`. | |
| # | |
| # We split this into a dedicated job (rather than appending to | |
| # test-latest-deps) because: | |
| # 1. Building a wheel + spawning a fresh venv is unrelated to the | |
| # "latest deps still resolve" axis and adds ~1 minute the matrix | |
| # doesn't need to pay twice. | |
| # 2. A failure here points unambiguously at packaging, not at a | |
| # transitive dep bump that broke training. | |
| # 3. We deliberately install from the wheel WITHOUT --no-deps so | |
| # the install path matches what end-users hit. | |
| # ------------------------------------------------------------------ | |
| name: Wheel install smoke test | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.11" | |
| - name: Build wheel | |
| run: | | |
| set -euo pipefail | |
| pip install --upgrade pip build | |
| python -m build --wheel | |
| - name: Install wheel into a fresh venv and run quickstart from /tmp | |
| run: | | |
| set -euo pipefail | |
| # Build a clean venv so nothing from the checkout's cwd leaks in. | |
| python -m venv /tmp/wheel-test | |
| /tmp/wheel-test/bin/pip install --upgrade pip | |
| # Resolve the wheel glob explicitly: there must be exactly one | |
| # forgelm-*.whl, otherwise the test environment is ambiguous. | |
| shopt -s nullglob | |
| wheels=(dist/forgelm-*.whl) | |
| if [ "${#wheels[@]}" -ne 1 ]; then | |
| echo "Expected exactly one forgelm wheel in dist/; found ${#wheels[@]}: ${wheels[*]:-(none)}" >&2 | |
| exit 1 | |
| fi | |
| wheel="${wheels[0]}" | |
| echo "Installing: $wheel" | |
| /tmp/wheel-test/bin/pip install "$wheel" | |
| # Run from /tmp specifically so the source tree is NOT on | |
| # sys.path — any template asset must come from the wheel's | |
| # site-packages copy, not from the checkout. | |
| cd /tmp | |
| echo "=== forgelm quickstart --list ===" | |
| /tmp/wheel-test/bin/forgelm quickstart --list | tee /tmp/qs-list.txt | |
| # Every registered template must appear in the listing. | |
| for tpl in customer-support code-assistant domain-expert medical-qa-tr grpo-math; do | |
| if ! grep -q "$tpl" /tmp/qs-list.txt; then | |
| echo "MISSING template '$tpl' from quickstart --list output" >&2 | |
| exit 1 | |
| fi | |
| done | |
| echo "=== forgelm quickstart customer-support --dry-run ===" | |
| /tmp/wheel-test/bin/forgelm quickstart customer-support \ | |
| --dry-run --output /tmp/wheel-qs.yaml | |
| if [ ! -f /tmp/wheel-qs.yaml ]; then | |
| echo "Quickstart did not materialize /tmp/wheel-qs.yaml" >&2 | |
| exit 1 | |
| fi | |
| echo "Wheel-install smoke test passed." | |
| test-min-deps: | |
| name: Test with minimum supported versions | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.10" | |
| - name: Install minimum dependency versions | |
| run: | | |
| pip install --upgrade pip | |
| # Install TRL first (most restrictive), let pip resolve transitive deps | |
| pip install \ | |
| "torch==2.1.0" \ | |
| "trl==0.12.0" \ | |
| "peft==0.11.0" \ | |
| "pydantic==2.0.0" \ | |
| "pyyaml==6.0.1" \ | |
| "tensorboard==2.15.0" | |
| pip install pytest pytest-cov | |
| - name: Install ForgeLM | |
| run: pip install -e . --no-deps | |
| - name: Run tests | |
| run: pytest tests/ -q --tb=short | |
| notify-failure: | |
| name: Notify on failure | |
| needs: [test-latest-deps, test-min-deps, wheel-install-smoke] | |
| if: failure() | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read # github-script needs to read the repo | |
| issues: write # required for creating failure notification issues | |
| steps: | |
| - name: Create issue on failure | |
| uses: actions/github-script@v8 | |
| with: | |
| script: | | |
| const title = `Nightly CI failure — ${new Date().toISOString().split('T')[0]}`; | |
| const existing = await github.rest.issues.listForRepo({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| state: 'open', | |
| labels: 'nightly-failure', | |
| per_page: 1, | |
| }); | |
| if (existing.data.length > 0) { | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: existing.data[0].number, | |
| body: `Nightly CI failed again on ${new Date().toISOString().split('T')[0]}.\n\n[View run](${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId})`, | |
| }); | |
| } else { | |
| await github.rest.issues.create({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| title: title, | |
| labels: ['nightly-failure', 'bug'], | |
| body: `## Nightly CI Failure\n\nThe nightly compatibility test failed.\n\n**Run:** [View details](${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId})\n\nThis may indicate a breaking change in a dependency. Check the test logs for details.`, | |
| }); | |
| } |