Skip to content

Commit ef237fe

Browse files
sabollim-silabssabollim-silabs
committed
pkg-lock
1 parent 6c6d508 commit ef237fe

14 files changed

Lines changed: 889 additions & 35 deletions

‎.github/actions/install-pkg-manifest/action.yaml‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
name: Install package manifest
2+
description: >
3+
Restore Matter packages. In locked mode, install CI Conan lock derived from
4+
SLT pkg.lock, then install local matter_app. Also exports CI_PKG_LOCK for
5+
build_app.sh (slt --check-updates=false).
6+
7+
inputs:
8+
package-version:
9+
description: Version of matter_app to install from the restored Matter cache
10+
required: true
11+
conan-cache-key:
12+
description: Cache key created by the Matter package producer
13+
required: true
14+
use-lockfile:
15+
description: If true, conan install --lockfile + export CI_PKG_LOCK
16+
required: false
17+
default: "true"
18+
fail-on-cache-miss:
19+
description: Fail when the Matter package cache does not exist
20+
required: false
21+
default: "true"
22+
download-parallel:
23+
description: Maximum parallel Conan downloads after restore
24+
required: false
25+
default: "1"
26+
lockfile-dir:
27+
description: Directory with pkg.lock, conan.lock, conanfile.txt
28+
required: false
29+
default: .github/pkg-manifest
30+
conan-remote-name:
31+
description: Conan remote name
32+
required: false
33+
default: silabs-conan-production
34+
conan-remote-url:
35+
description: Conan remote URL
36+
required: false
37+
default: https://artifactory.silabs.net/artifactory/api/conan/silabs-conan-production
38+
39+
outputs:
40+
cache-hit:
41+
description: Whether the Matter package cache was restored
42+
value: ${{ steps.restore.outputs.cache-hit }}
43+
44+
runs:
45+
using: composite
46+
steps:
47+
- name: Prepare Conan home
48+
shell: bash
49+
run: |
50+
set -euo pipefail
51+
: "${CONAN_HOME:?CONAN_HOME must be set (run cache-tools first)}"
52+
rm -rf "${CONAN_HOME}"
53+
mkdir -p "${CONAN_HOME}"
54+
55+
- name: Restore Matter package cache
56+
id: restore
57+
uses: actions/cache/restore@v6
58+
with:
59+
path: ${{ env.CONAN_HOME }}
60+
key: ${{ inputs.conan-cache-key }}
61+
fail-on-cache-miss: ${{ inputs.fail-on-cache-miss }}
62+
63+
- name: Configure Conan downloads
64+
shell: bash
65+
run: |
66+
set -euo pipefail
67+
mkdir -p conan-settings
68+
printf 'core.download:parallel=%s\n' "${{ inputs.download-parallel }}" > conan-settings/global.conf
69+
conan config install conan-settings --type dir
70+
rm -rf conan-settings
71+
72+
REMOTE="${{ inputs.conan-remote-name }}"
73+
conan remote add -f "${REMOTE}" "${{ inputs.conan-remote-url }}"
74+
if [[ -n "${ARTIFACTORY_USER:-}" && -n "${ARTIFACTORY_TOKEN:-}" ]]; then
75+
conan remote login "${REMOTE}" "${ARTIFACTORY_USER}" -p "${ARTIFACTORY_TOKEN}"
76+
else
77+
echo "ARTIFACTORY_USER/TOKEN unset; attempting anonymous Conan access"
78+
fi
79+
80+
- name: Install CI Conan lockfile
81+
if: ${{ inputs.use-lockfile == 'true' }}
82+
shell: bash
83+
run: |
84+
set -euo pipefail
85+
LOCK_DIR="${{ inputs.lockfile-dir }}"
86+
[[ -f "${LOCK_DIR}/conan.lock" ]] || {
87+
echo "Missing ${LOCK_DIR}/conan.lock (generate from pkg.lock)"
88+
exit 1
89+
}
90+
[[ -f "${LOCK_DIR}/conanfile.txt" ]] || {
91+
echo "Missing ${LOCK_DIR}/conanfile.txt"
92+
exit 1
93+
}
94+
[[ -f "${LOCK_DIR}/pkg.lock" ]] || {
95+
echo "Missing ${LOCK_DIR}/pkg.lock"
96+
exit 1
97+
}
98+
99+
REMOTE="${{ inputs.conan-remote-name }}"
100+
echo "Installing locked packages from ${LOCK_DIR}/conan.lock"
101+
conan install "${LOCK_DIR}" \
102+
--lockfile="${LOCK_DIR}/conan.lock" \
103+
-r "${REMOTE}"
104+
105+
ABS_PKG_LOCK="$(cd "${LOCK_DIR}" && pwd)/pkg.lock"
106+
echo "CI_PKG_LOCK=${ABS_PKG_LOCK}" >> "${GITHUB_ENV}"
107+
echo "Using CI pkg.lock for slt: ${ABS_PKG_LOCK}"
108+
109+
- name: Install Matter app package
110+
shell: bash
111+
run: |
112+
set -euo pipefail
113+
slt install "matter_app/${{ inputs.package-version }}@silabs" -e conan

‎.github/pkg-manifest/README.md‎

Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
# CI package dependency locks
2+
3+
This directory holds **CI-only** dependency pins for Matter package validation
4+
workflows (`.github/workflows/matter-packages-validation.yaml` and the package
5+
builders it calls).
6+
7+
These files are **not** part of the `matter` / `matter_app` Conan packages.
8+
Packaging pulls SLC/SDK trees and explicit recipe roots; it does not ship
9+
`.github/`.
10+
11+
## Why this exists
12+
13+
Package CI depends on many Silabs Conan packages (direct and transitive).
14+
Floating ranges (for example in `slc/script/dependency_versions.yaml` /
15+
generated `pkg.slt`) can pick up newly published prereleases and break PR CI
16+
even when Matter code did not change.
17+
18+
This directory stores a **certified pin set**:
19+
20+
1. **PR CI** installs that pin set so builds are reproducible.
21+
2. **Nightly CI** resolves freely with `slt install`, and on success refreshes
22+
the pins so the next day’s PRs pick up a validated set.
23+
24+
## Files
25+
26+
| File | Format | Role |
27+
| --- | --- | --- |
28+
| `pkg.lock` | SLT TOML-like lock | **Source of truth.** Exact pins from `slt install` (versions + Conan recipe revisions, plus archive tools). |
29+
| `conan.lock` | Conan 2 JSON lock (`version: "0.5"`) | **Derived.** Used by `conan install --lockfile=...`. |
30+
| `conanfile.txt` | Conan requires list | **Derived.** Companion graph for `conan install <dir> --lockfile=...`. |
31+
| `README.md` | Markdown | This document. |
32+
33+
### Important: two lock formats
34+
35+
| | `pkg.lock` | `conan.lock` |
36+
| --- | --- | --- |
37+
| Consumer | `slt install` | `conan install --lockfile` |
38+
| Shape | `[dependency]` entries with `installer`, `type`, `ref` | JSON `requires` array of `name/version@user#revision` |
39+
| Compatible with each other? | **No** — do not pass `pkg.lock` to `--lockfile` |
40+
41+
Always treat `pkg.lock` as the source and regenerate Conan files from it.
42+
43+
## Source of truth and conversion
44+
45+
```text
46+
slt install --> pkg.lock --> pkg_lock_to_conan_lock.py --> conan.lock
47+
\--> conanfile.txt
48+
```
49+
50+
Converter script (repo root):
51+
52+
```bash
53+
python3 slc/script/pkg_lock_to_conan_lock.py \
54+
--input .github/pkg-manifest/pkg.lock \
55+
--output .github/pkg-manifest/conan.lock \
56+
--conanfile .github/pkg-manifest/conanfile.txt
57+
```
58+
59+
What the converter does:
60+
61+
- Keeps only entries with `installer = "conan"`.
62+
- Skips archive/tool packages (`commander`, `slc-cli`, `python`, `java21`, `zap`, …).
63+
- By default **omits** `matter` and `matter_app` so CI can install PR-built
64+
Matter packages on top of the locked third-party set.
65+
- Prefers each entry’s `ref = "name/version@silabs#revision"` when present.
66+
- Writes a minimal `conanfile.txt` with exact `name/version@silabs` requires
67+
(revision stripped in the requires list; revisions stay in `conan.lock`).
68+
69+
Optional flag:
70+
71+
```bash
72+
# Include matter/matter_app in the Conan lock (normally not needed in CI)
73+
python3 slc/script/pkg_lock_to_conan_lock.py --include-matter ...
74+
```
75+
76+
## PR CI (locked / stable)
77+
78+
Triggered on pull requests into `main` / `release_*` with `use_lockfile=true`
79+
(default).
80+
81+
Flow (via `.github/actions/install-pkg-manifest` + `packages/build_app.sh`):
82+
83+
1. Restore the Actions cache that contains the PR’s exported `matter` /
84+
`matter_app` packages.
85+
2. Configure the Conan remote (`silabs-conan-production`).
86+
3. Install third-party deps:
87+
88+
```bash
89+
conan install .github/pkg-manifest \
90+
--lockfile=.github/pkg-manifest/conan.lock \
91+
-r silabs-conan-production
92+
```
93+
94+
4. Install the PR Matter app package:
95+
96+
```bash
97+
slt install "matter_app/<PR_VERSION>@silabs" -e conan
98+
```
99+
100+
5. Export `CI_PKG_LOCK` to the absolute path of `pkg.lock`.
101+
6. For each app build, `build_app.sh`:
102+
- Copies `CI_PKG_LOCK` into the app directory as `pkg.lock`.
103+
- Rewrites `matter` / `matter_app` entries to the PR package version
104+
(drops recipe revision so local `export-pkg` packages resolve).
105+
- Runs `slt install --check-updates=false` so SLT does not float.
106+
107+
Result: third-party packages stay at the certified pins; only Matter code under
108+
test comes from the PR.
109+
110+
## Nightly CI (resolve + refresh)
111+
112+
Triggered by `.github/workflows/trigger-matter-packages-validation.yaml` with
113+
`use_lockfile=false`.
114+
115+
Flow:
116+
117+
1. **Build** without installing `.github/pkg-manifest` locks.
118+
2. App builds use normal `slt install` (`CHECK_UPDATES=true`) so newly published
119+
packages matching `pkg.slt` ranges can be pulled.
120+
3. If the package app builds (and OTA check, when enabled) succeed, the
121+
**Update CI pkg.lock and conan.lock** job:
122+
- Runs unlocked `slt install` in `slc/apps/lock_app/wifi`.
123+
- Copies the produced `pkg.lock` to `.github/pkg-manifest/pkg.lock`.
124+
- Runs `pkg_lock_to_conan_lock.py` to refresh `conan.lock` and
125+
`conanfile.txt`.
126+
- Commits and pushes the three files on the branch that was dispatched
127+
(for example `release_2.10-1.6.1`).
128+
129+
If nightly fails, the previous committed locks are left unchanged, so PRs keep
130+
using the last green pin set.
131+
132+
## Manual update (local)
133+
134+
Use this when you want to refresh pins without waiting for nightly:
135+
136+
```bash
137+
# From repo root, with slt/conan configured and remotes available
138+
python3 slc/script/generate_pkg_slt.py -d slc # if pkg.slt missing
139+
rm -f slc/apps/lock_app/wifi/pkg.lock
140+
( cd slc/apps/lock_app/wifi && slt install --check-updates=true )
141+
142+
mkdir -p .github/pkg-manifest
143+
cp -f slc/apps/lock_app/wifi/pkg.lock .github/pkg-manifest/pkg.lock
144+
145+
python3 slc/script/pkg_lock_to_conan_lock.py \
146+
--input .github/pkg-manifest/pkg.lock \
147+
--output .github/pkg-manifest/conan.lock \
148+
--conanfile .github/pkg-manifest/conanfile.txt
149+
150+
# Review diff, then commit all three files together
151+
git add .github/pkg-manifest/pkg.lock \
152+
.github/pkg-manifest/conan.lock \
153+
.github/pkg-manifest/conanfile.txt
154+
```
155+
156+
Always convert after editing or replacing `pkg.lock`. Never hand-edit
157+
`conan.lock` alone unless you also update `pkg.lock` (or accept drift).
158+
159+
## Secrets / remotes
160+
161+
| Name | Purpose |
162+
| --- | --- |
163+
| `ARTIFACTORY_USER` / `ARTIFACTORY_TOKEN` | Optional Conan remote login for `silabs-conan-production` (and lock refresh). If unset, CI tries anonymous access. |
164+
| Job `contents: write` + `GITHUB_TOKEN` | Nightly commit/push of refreshed locks (branch protection may require a bot PAT). |
165+
166+
Remote used by CI:
167+
168+
```text
169+
https://artifactory.silabs.net/artifactory/api/conan/silabs-conan-production
170+
```
171+
172+
## Related code
173+
174+
| Path | Role |
175+
| --- | --- |
176+
| `slc/script/pkg_lock_to_conan_lock.py` | `pkg.lock` → `conan.lock` + `conanfile.txt` |
177+
| `.github/actions/install-pkg-manifest/` | Restore Matter cache; locked Conan install; set `CI_PKG_LOCK` |
178+
| `packages/build_app.sh` | Honors `CI_PKG_LOCK` / `--pkg-lock` and `CHECK_UPDATES` |
179+
| `.github/workflows/matter-packages-validation.yaml` | PR vs nightly mode; nightly lock refresh job |
180+
| `.github/workflows/trigger-matter-packages-validation.yaml` | Daily dispatch with `use_lockfile=false` |
181+
| `slc/script/dependency_versions.yaml` | Floating ranges for local/dev `pkg.slt` generation (not the CI pin set) |
182+
183+
## Seed note
184+
185+
The initial `pkg.lock` was taken from `slc/apps/lock_app/wifi/pkg.lock` (a
186+
representative Wi‑Fi app resolve that also includes shared Thread/platform
187+
packages). Nightly (or a manual refresh) should replace it with a newly
188+
resolved lock after a green package validation run.

‎.github/pkg-manifest/conan.lock‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
{
2+
"version": "0.5",
3+
"requires": [
4+
"zigbee/9.2.0-alpha.46@silabs#8b6c495feeff4aae923ab949e23740788e9f8eb7",
5+
"wiseconnect_common/4.2.0-alpha.66@silabs#52bec072c7581aa2e60fe3012d6e554b2b1cff84",
6+
"wifi/4.2.0-alpha.146@silabs#52bec072c7581aa2e60fe3012d6e554b2b1cff84",
7+
"usb/1.8.0@silabs#2d1c3440a76741001090a046509994a4",
8+
"unity_test_framework/1.1.0@silabs#f418a9460d44f860469de8d9aba3d723",
9+
"thread/3.2.0-alpha.45@silabs#8cce3fd4cfd5bd014a36f01905c4b0b9a111a5aa",
10+
"siwx91x_nwp_interface/2.17.0-alpha.42@silabs#8255e4021015a2cba62472ae5f002ac13d5ed684",
11+
"silabs_lwip/1.2.0-alpha.2@silabs#f5f97a2aff5c62c73de377fec9ba8122bf722831",
12+
"segger/1.2.0@silabs#36f8201f19c01e288b5be67fda8f500d",
13+
"security_tfm/1.2.0@silabs#0943f3f4ec758de891b2468ce589d7f3",
14+
"security_sxsymcrypt/1.2.0@silabs#f242692bb498dc0e96c0544b5d61c41e",
15+
"security_libcryptosoc/1.1.0@silabs#a41bcce963865004cdd183d9f003fd37",
16+
"rail_library/3.2.0-alpha.120@silabs#1912fd314134738688a6dc37f9e0e8d0fb0330f2",
17+
"printf/1.2.0@silabs#7b24522f4c76f3ef6865b3fe5cb974b0",
18+
"platform_siwx91x/4.2.0-alpha.147@silabs#52bec072c7581aa2e60fe3012d6e554b2b1cff84",
19+
"platform_core/1.2.0@silabs#2f3f6a843280008b0111d23b638daa76",
20+
"platform/6.2.0-alpha.60@silabs#3994f55b7c6ffc12bff64bfde70fe671",
21+
"paho_mqtt_embedded/1.0.0-alpha.7@silabs#f50ba31e06d951edb072ff8c0e3fa32e3ab5af4f",
22+
"openthread_forks/3.2.0-alpha.45@silabs#8cce3fd4cfd5bd014a36f01905c4b0b9a111a5aa",
23+
"ninja/1.12.1@silabs#9ce59e07a142d4da1c532b49742fe16d",
24+
"network_stack_sdk/1.0.0-alpha.47@silabs#52bec072c7581aa2e60fe3012d6e554b2b1cff84",
25+
"network_stack/1.0.0-alpha.127@silabs#3afebe8d74635276b58ad9a8c53d71a5b2b2156d",
26+
"multiprotocol/1.2.0-alpha.42@silabs#647fb0ed26ba9c87ec2ed3da4bd9693f4ab4e91f",
27+
"micriumos/6.2.0@silabs#236252f6e565c572a2e9a819050e153d",
28+
"mbedtls/1.2.0@silabs#34f8e2586570896f0be441cc56a4a20d",
29+
"llvm-arm-toolchain-for-embedded/21.1.1@silabs#877f3e0faf9658c618b4c2fe80bbb9fa",
30+
"json_parser/1.1.0-alpha.90@silabs#52bec072c7581aa2e60fe3012d6e554b2b1cff84",
31+
"iot_socket/1.0.0-alpha.84@silabs#52bec072c7581aa2e60fe3012d6e554b2b1cff84",
32+
"gcc-arm-none-eabi/14.2.rel1@silabs#7a5e7220af325e610937338dde99651b",
33+
"freertos/1.2.0@silabs#845766a5077fb17fdf2854ad1786079d",
34+
"delta_dfu/1.1.0@silabs#1ebba4a316f35b36228125a8942d772c72943ec6",
35+
"coremark/1.1.0@silabs#fad338900822dd7b4dd280d6713266f1",
36+
"common_15_4/1.2.0-alpha.8@silabs#647fb0ed26ba9c87ec2ed3da4bd9693f4ab4e91f",
37+
"cmsis/1.2.0@silabs#f8ebee69efcf85687bb21ce7a9d725ce",
38+
"cmake/3.30.2@silabs#8798657aa78b2da38a3ae465f2d10090",
39+
"bootloader/3.4.0@silabs#39519d377fcced2d472ecba7ff6869ca",
40+
"boards/1.2.0@silabs#25a23460427c045026202be2911fb2a3",
41+
"bluetooth_le_siwx91x/4.2.0-alpha.147@silabs#52bec072c7581aa2e60fe3012d6e554b2b1cff84",
42+
"bluetooth_le_host/11.2.0-alpha.69@silabs#91f8b6a6799e6a7e9cecc95fc836f3b6ea30c8e7",
43+
"bluetooth_le_controller/11.2.0-alpha.69@silabs#91f8b6a6799e6a7e9cecc95fc836f3b6ea30c8e7",
44+
"bluetooth_common/1.2.0-alpha.69@silabs#91f8b6a6799e6a7e9cecc95fc836f3b6ea30c8e7",
45+
"bluetooth_bgbuild/11.2.0-alpha.69@silabs#91f8b6a6799e6a7e9cecc95fc836f3b6ea30c8e7",
46+
"bgapi_protocol/11.2.0-alpha.69@silabs#91f8b6a6799e6a7e9cecc95fc836f3b6ea30c8e7",
47+
"base_sdk/2026.12.0@silabs#716052badc1ee855a7abbf99407f2d41",
48+
"azure_freertos_middleware/1.1.0-alpha.71@silabs#52bec072c7581aa2e60fe3012d6e554b2b1cff84",
49+
"aws_sdk/3.1.0-alpha.92@silabs#52bec072c7581aa2e60fe3012d6e554b2b1cff84"
50+
],
51+
"build_requires": [],
52+
"python_requires": [],
53+
"config_requires": []
54+
}

0 commit comments

Comments
 (0)