llm-d-benchmark/.pre-commit-config.yaml at main · nilig/llm-d-benchmark · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
repos:
  - repo: local
    hooks:
      # ----------------------------------------------------------------
      # Fast syntax sanity check -- only runs when llmdbenchmark/**.py
      # changes. Catches import errors and syntax regressions before
      # anything heavier runs.
      # ----------------------------------------------------------------
      - id: py-compile
        name: Byte-compile llmdbenchmark
        entry: python -m compileall -q llmdbenchmark
        language: system
        pass_filenames: false
        files: ^llmdbenchmark/.*\.py$
        stages: [pre-commit]

      # ----------------------------------------------------------------
      # Unit tests -- mirrors the `unit-tests` job in
      # .github/workflows/ci-pr-benchmark.yaml.
      # ----------------------------------------------------------------
      - id: pytest
        name: Unit tests (pytest tests/ -x -q)
        entry: python -m pytest tests/ -x -q
        language: system
        pass_filenames: false
        stages: [pre-commit]

      # ----------------------------------------------------------------
      # Render-validation smart check -- detects which scenarios the
      # commit actually touched (via util/precommit_render_changed.py)
      # and renders only those. Scope:
      #
      #   - config/specification/<path>.yaml.j2   -> render <path>
      #   - config/scenarios/<path>.yaml          -> render <path>
      #   - shared render path (templates, parser, plan, executor,
      #     utilities, standup, run, teardown, smoketests, cli.py,
      #     config.py) -> render the cicd/kind-sim canary
      #   - nothing resolved (docs-only commit, test-only commit, ...)
      #     -> render the cicd/kind-sim canary as a baseline sanity
      #     check, so every commit proves the render path is healthy
      #
      # The script prints each spec it is about to render along with
      # the reason it was picked, so it is always obvious what the
      # hook is doing. We rely on CI's full per-spec render job as the
      # final safety net.
      #
      # Important knobs:
      #
      #   pass_filenames: false
      #     We do NOT want pre-commit to hand us the file list. With
      #     pass_filenames: true, pre-commit batches large file lists
      #     across multiple subprocess invocations (to stay under the
      #     OS argv limit) and runs those batches in parallel, which
      #     both duplicates work and scrambles the tty output. With
      #     pass_filenames: false, pre-commit invokes us exactly once
      #     with zero file arguments and the script asks git directly
      #     for the real staged-file list via
      #     ``git diff --cached --name-only --diff-filter=ACMR``. This
      #     also means ``pre-commit run --all-files`` correctly renders
      #     only what's staged (instead of every scenario in the repo)
      #     -- the "--all-files" intent is meaningless for a
      #     change-detecting hook.
      #
      #   require_serial: true
      #     Defensive. With pass_filenames: false there's only one
      #     invocation anyway, but this makes it explicit that the
      #     hook must never be run in parallel with copies of itself.
      # ----------------------------------------------------------------
      - id: render-validation-changed
        name: Render changed scenarios (dry-run)
        entry: python util/precommit_render_changed.py
        language: system
        pass_filenames: false
        require_serial: true
        stages: [pre-commit]

      # ----------------------------------------------------------------
      # Software Bill of Materials -- regenerates SBOM.md from
      # install.sh, pyproject.toml, and config/templates/values/
      # defaults.yaml. Resolves `auto` Helm/image versions against the
      # live registry via the existing VersionResolver.
      #
      # Triggers only when one of those source-of-truth files changes
      # (matches the precommit_render_changed.py performance pattern).
      # When SBOM.md would change, the hook writes the new version and
      # exits non-zero with a message asking the user to `git add` and
      # re-commit -- same convention as `prettier --write` / `black`.
      # ----------------------------------------------------------------
      - id: generate-sbom
        name: Generate SBOM (Software Bill of Materials)
        entry: python util/generate_sbom.py --check
        language: system
        pass_filenames: false
        require_serial: true
        files: ^(install\.sh|pyproject\.toml|config/templates/values/defaults\.yaml|util/generate_sbom\.py)$
        stages: [pre-commit]

  - repo: https://github.com/ibm/detect-secrets
    # If you desire to use a specific version of detect-secrets, you can replace `master` with other git revisions such as branch, tag or commit sha.
    # You are encouraged to use static refs such as tags, instead of branch name
    #
    # Running "pre-commit autoupdate" automatically updates rev to latest tag
    rev: 0.13.1+ibm.64.dss
    hooks:
      - id: detect-secrets # pragma: whitelist secret
        # Add options for detect-secrets-hook binary. You can run `detect-secrets-hook --help` to list out all possible options.
        # You may also run `pre-commit run detect-secrets` to preview the scan result.
        # when "--baseline" without "--use-all-plugins", pre-commit scan with just plugins in baseline file
        # when "--baseline" with "--use-all-plugins", pre-commit scan with all available plugins
        # add "--fail-on-unaudited" to fail pre-commit for unaudited potential secrets
        args: [--baseline, .secrets.baseline, --use-all-plugins]
        # Files the scanner skips entirely. Each entry is a file
        # whose hits are known false positives and whose content
        # changes too often for a baseline audit to be stable.
        # Scope is narrow on purpose — one entry per file — so a
        # new file added alongside these still gets scanned.
        exclude: |
          (?x)^(
            # Docs notebook: cell outputs contain base64-encoded
            # matplotlib PNG figures. Can't use inline pragmas (JSON,
            # no comments) and clearing outputs isn't an option
            # because the rendered figures are the point.
            docs/analysis/analysis\.ipynb
            |
            # BenchmarkReport v0.2 example: the single hit is a
            # synthetic storage path with a unix timestamp and run
            # identifier. Illustrative reference, not a credential.
            llmdbenchmark/analysis/benchmark_report/br_v0_2_example\.yaml
            |
            # BenchmarkReport README: three hits are 32-hex
            # example uid / cfg_id values in a Python code block
            # showing how to construct a BenchmarkReport.
            llmdbenchmark/analysis/benchmark_report/README\.md
          )$

  # ----------------------------------------------------------------
  # Code formating and linting using Ruff. Run ruff-check first to
  # ensure proper code linting and then ruff-format to format your
  # code.
  #
  # These steps can be ran manually at any time using:
  # pre-commit run ruff-check
  #
  # pre-commit run ruff-format
  #
  # * Add the --all-files arg to lint/format all files in the repo.
  # ----------------------------------------------------------------
  - repo: https://github.com/astral-sh/ruff-pre-commit
    rev: v0.15.11
    hooks:
      # Run the linter
      - id: ruff-check
        args: [--fix]
        name: Lint code check
        stages: [pre-commit]

      # Run the formatter
      - id: ruff-format
        args: [] # Add the --diff arg if you only want to see what changes would be made
        name: Format code
        stages: [pre-commit]