Nightly Compatibility #51
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Nightly Compatibility | |
| on: | |
| schedule: | |
| - cron: "0 3 * * *" # 03:00 UTC daily | |
| workflow_dispatch: # Manual trigger | |
| # No workflow-level permissions — each job declares the minimal set it needs. | |
| # Set FORGELM_OPERATOR so future operator-identity checks (Phase 3 hardening) | |
| # do not raise on distroless runners where USER is unset. | |
| env: | |
| FORGELM_OPERATOR: ci-smoke | |
| jobs: | |
| test-latest-deps: | |
| name: Test with latest dependencies | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.11", "3.12"] | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install with latest deps (no upper bounds) | |
| # NOTE: intentionally unconstrained — designed to surface upstream-API | |
| # breakage early. The notify-failure job dedups against existing | |
| # 'nightly-failure' issues to bound noise (see also: testing.md | |
| # CI gates discussion). Do NOT add `|| true` here. | |
| run: | | |
| pip install --upgrade pip | |
| pip install torch transformers peft datasets accelerate trl pydantic pyyaml tensorboard huggingface_hub requests | |
| pip install pytest pytest-cov ruff | |
| - name: Install ForgeLM (editable, no deps — already installed above) | |
| run: pip install -e . --no-deps | |
| - name: Show dependency versions | |
| run: | | |
| python -c " | |
| import importlib | |
| for pkg in ['torch', 'transformers', 'peft', 'datasets', 'accelerate', 'trl', 'pydantic']: | |
| mod = importlib.import_module(pkg) | |
| print(f'{pkg}: {getattr(mod, \"__version__\", \"unknown\")}') | |
| " | |
| - name: Lint check | |
| run: ruff check . | |
| - name: Run tests | |
| run: pytest tests/ -q --tb=short | |
| - name: CLI smoke test | |
| run: | | |
| forgelm --version | |
| forgelm --config config_template.yaml --dry-run | |
| forgelm --config config_template.yaml --dry-run --output-format json | |
| - name: Quickstart templates smoke test | |
| run: | | |
| # Every bundled template must render to a valid YAML that | |
| # passes pydantic validation. Catches template drift the | |
| # moment a config schema changes. | |
| # | |
| # Note: smoke-tests 4 of 5 templates. `domain-expert` is BYOD | |
| # (no bundled dataset), so `quickstart --dry-run` rejects it | |
| # without `--dataset PATH`. It's covered separately by the | |
| # pytest unit `test_domain_expert_intentionally_has_no_bundled_data`. | |
| forgelm quickstart --list | |
| for tpl in customer-support code-assistant medical-qa-tr grpo-math; do | |
| echo "=== Quickstart dry-run: $tpl ===" | |
| forgelm quickstart "$tpl" --dry-run --output "/tmp/qs-$tpl.yaml" | |
| forgelm --config "/tmp/qs-$tpl.yaml" --dry-run | |
| done | |
| - name: Forbid git+https in notebooks | |
| run: | | |
| # Notebooks must install ForgeLM from PyPI with a pinned version, | |
| # never from a git+https URL. Closure plan Faz 5 — git+https makes | |
| # Colab installs depend on a specific branch HEAD, breaking | |
| # reproducibility and confusing users when main moves ahead of | |
| # the latest released wheel. | |
| if grep -rn 'git+https://github.com/cemililik/ForgeLM' notebooks/; then | |
| echo "::error::Notebooks must install from PyPI, not git+https" | |
| exit 1 | |
| fi | |
| - name: Ingestion + audit smoke test (Phase 11) | |
| run: | | |
| # Minimal end-to-end: TXT in → JSONL out → audit report out. | |
| # Plain TXT path doesn't need the [ingestion] extra; the audit | |
| # module is pure stdlib. Catches CLI wiring drift without paying | |
| # for pypdf / langdetect installs. | |
| mkdir -p /tmp/p11 | |
| echo "Article 10 governs data quality." > /tmp/p11/sample.txt | |
| echo "Section two: representativeness, traceability, bias review." > /tmp/p11/sample2.txt | |
| forgelm ingest /tmp/p11/ --recursive --output /tmp/p11/out.jsonl | |
| test -s /tmp/p11/out.jsonl | |
| forgelm --data-audit /tmp/p11/out.jsonl --output /tmp/p11/audit/ | |
| test -s /tmp/p11/audit/data_audit_report.json | |
| wheel-install-smoke: | |
| # ------------------------------------------------------------------ | |
| # This is the only test that catches package_data globs being broken | |
| # — editable installs always hide this. `pip install -e .` resolves | |
| # forgelm/templates/* via Path(__file__).parent regardless of what | |
| # setuptools would actually copy into the wheel, so a broken | |
| # [tool.setuptools.package-data] entry is invisible until a real user | |
| # runs `pip install forgelm` from PyPI and gets a missing-asset | |
| # FileNotFoundError on `forgelm quickstart`. | |
| # | |
| # We split this into a dedicated job (rather than appending to | |
| # test-latest-deps) because: | |
| # 1. Building a wheel + spawning a fresh venv is unrelated to the | |
| # "latest deps still resolve" axis and adds ~1 minute the matrix | |
| # doesn't need to pay twice. | |
| # 2. A failure here points unambiguously at packaging, not at a | |
| # transitive dep bump that broke training. | |
| # 3. We deliberately install from the wheel WITHOUT --no-deps so | |
| # the install path matches what end-users hit. | |
| # ------------------------------------------------------------------ | |
| name: Wheel install smoke test | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.11" | |
| - name: Build wheel | |
| run: | | |
| set -euo pipefail | |
| pip install --upgrade pip build | |
| python -m build --wheel | |
| - name: Install wheel into a fresh venv and run quickstart from /tmp | |
| run: | | |
| set -euo pipefail | |
| # Build a clean venv so nothing from the checkout's cwd leaks in. | |
| python -m venv /tmp/wheel-test | |
| /tmp/wheel-test/bin/pip install --upgrade pip | |
| # Resolve the wheel glob explicitly: there must be exactly one | |
| # forgelm-*.whl, otherwise the test environment is ambiguous. | |
| shopt -s nullglob | |
| wheels=(dist/forgelm-*.whl) | |
| if [ "${#wheels[@]}" -ne 1 ]; then | |
| echo "Expected exactly one forgelm wheel in dist/; found ${#wheels[@]}: ${wheels[*]:-(none)}" >&2 | |
| exit 1 | |
| fi | |
| wheel="${wheels[0]}" | |
| echo "Installing: $wheel" | |
| /tmp/wheel-test/bin/pip install "$wheel" | |
| # Run from /tmp specifically so the source tree is NOT on | |
| # sys.path — any template asset must come from the wheel's | |
| # site-packages copy, not from the checkout. | |
| cd /tmp | |
| echo "=== forgelm quickstart --list ===" | |
| /tmp/wheel-test/bin/forgelm quickstart --list | tee /tmp/qs-list.txt | |
| # Every registered template must appear in the listing. | |
| for tpl in customer-support code-assistant domain-expert medical-qa-tr grpo-math; do | |
| if ! grep -q "$tpl" /tmp/qs-list.txt; then | |
| echo "MISSING template '$tpl' from quickstart --list output" >&2 | |
| exit 1 | |
| fi | |
| done | |
| echo "=== forgelm quickstart customer-support --dry-run ===" | |
| /tmp/wheel-test/bin/forgelm quickstart customer-support \ | |
| --dry-run --output /tmp/wheel-qs.yaml | |
| if [ ! -f /tmp/wheel-qs.yaml ]; then | |
| echo "Quickstart did not materialize /tmp/wheel-qs.yaml" >&2 | |
| exit 1 | |
| fi | |
| echo "Wheel-install smoke test passed." | |
| test-min-deps: | |
| name: Test with minimum supported versions | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.10" | |
| - name: Install minimum dependency versions | |
| run: | | |
| pip install --upgrade pip | |
| # Install TRL first (most restrictive), let pip resolve transitive deps | |
| pip install \ | |
| "torch==2.3.0" \ | |
| "trl==0.12.0" \ | |
| "peft==0.11.0" \ | |
| "pydantic==2.0.0" \ | |
| "pyyaml==6.0.1" \ | |
| "tensorboard==2.15.0" | |
| pip install pytest pytest-cov | |
| - name: Install ForgeLM | |
| run: pip install -e . --no-deps | |
| - name: Run tests | |
| run: pytest tests/ -q --tb=short | |
| supply-chain-security: | |
| # ------------------------------------------------------------------ | |
| # Wave 4 / Faz 23: ISO 27001 / SOC 2 Type II alignment supply-chain | |
| # security gate. | |
| # | |
| # Two scans, both run on the latest published deps so we catch CVEs | |
| # the moment a dependency advisory drops, even before the team | |
| # re-bumps versions: | |
| # | |
| # - pip-audit (transitive CVEs against the OSV / GHSA databases) | |
| # - bandit (static-analysis on forgelm/ production code only) | |
| # | |
| # Severity policy (per tools/check_pip_audit.py + tools/check_bandit.py): | |
| # HIGH → fail; MEDIUM → ::warning::; LOW → silent. | |
| # (CRITICAL findings are reported by pip-audit at HIGH tier — same gate.) | |
| # Operators install the same tooling locally via | |
| # `pip install forgelm[security]`. | |
| # ------------------------------------------------------------------ | |
| name: Supply-chain security (pip-audit + bandit) | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.11" | |
| cache: pip | |
| - name: Install ForgeLM with [security] extra | |
| run: | | |
| pip install --upgrade pip | |
| pip install -e ".[security]" | |
| - name: pip-audit (transitive CVE scan) | |
| run: | | |
| # pip-audit returns non-zero on any finding; capture the JSON | |
| # and apply our own severity policy via tools/check_pip_audit.py. | |
| # | |
| # Vulnerability ignores (each ignore must carry a written | |
| # justification + remove-after condition; review at every | |
| # release cycle). Issue #37 tracks the active set. | |
| # | |
| # CVE-2026-1839 — transformers, fix: 5.0.0rc3 (release candidate). | |
| # ForgeLM's pyproject pins ``transformers>=4.38.0,<5.0.0`` and | |
| # the 5.x branch is a major version bump that breaks downstream | |
| # callers (TRL adapter signature changes + tokenizer-config | |
| # API drift). The CVE has no 4.x backport at the time of | |
| # ignore (verified 2026-05-10). Stop-gap until either (a) | |
| # transformers ships a 4.x point release with the fix or | |
| # (b) ForgeLM cuts a tracked major-version-bump cycle that | |
| # raises the upper bound. Re-evaluate at each release. | |
| pip-audit \ | |
| --format json \ | |
| --output /tmp/pip-audit.json \ | |
| --ignore-vuln CVE-2026-1839 \ | |
| || true | |
| python3 tools/check_pip_audit.py /tmp/pip-audit.json | |
| - name: bandit (static security analysis) | |
| run: | | |
| # Same scope/severity policy as ci.yml; nightly cadence catches | |
| # newly-introduced HIGH issues even on long-lived branches. | |
| bandit -c pyproject.toml -r forgelm/ -f json -o /tmp/bandit.json || true | |
| python3 tools/check_bandit.py /tmp/bandit.json | |
| - name: Upload security scan artifacts | |
| if: always() | |
| uses: actions/upload-artifact@v5 | |
| with: | |
| name: supply-chain-scans | |
| path: | | |
| /tmp/pip-audit.json | |
| /tmp/bandit.json | |
| notify-failure: | |
| name: Notify on failure | |
| needs: [test-latest-deps, test-min-deps, wheel-install-smoke, supply-chain-security] | |
| if: failure() | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read # github-script needs to read the repo | |
| issues: write # required for creating failure notification issues | |
| steps: | |
| - name: Create issue on failure | |
| uses: actions/github-script@v8 | |
| with: | |
| script: | | |
| const title = `Nightly CI failure — ${new Date().toISOString().split('T')[0]}`; | |
| const existing = await github.rest.issues.listForRepo({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| state: 'open', | |
| labels: 'nightly-failure', | |
| per_page: 1, | |
| }); | |
| if (existing.data.length > 0) { | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: existing.data[0].number, | |
| body: `Nightly CI failed again on ${new Date().toISOString().split('T')[0]}.\n\n[View run](${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId})`, | |
| }); | |
| } else { | |
| await github.rest.issues.create({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| title: title, | |
| labels: ['nightly-failure', 'bug'], | |
| body: `## Nightly CI Failure\n\nThe nightly compatibility test failed.\n\n**Run:** [View details](${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId})\n\nThis may indicate a breaking change in a dependency. Check the test logs for details.`, | |
| }); | |
| } |