Skip to content

Commit ec0861f

Browse files
committed
render-pinning: allow conda-build selectors in overrides
switch override parsing to use a similar ruamel.yaml based approach to upstream conda-build so that overrides can contain selectors such as platform specs.
1 parent b0bb681 commit ec0861f

5 files changed

Lines changed: 154 additions & 59 deletions

File tree

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ combines the two, and the files next to each:
8181
- **`patch/*.patch` and `tests/`:** per package, the distribution's file replaces the shared one. In tests, `- if: ros_distro in ["humble", "jazzy"]` blocks select distribution-specific parts.
8282
- **`vinca_pinning.yaml`:** the shared pinning, adjusted by `distro.yaml`:
8383
- `conda_forge_pinning_version` and `conda_forge_migrations` keep a distribution on its own conda-forge pinning, for example until its next full rebuild;
84-
- `pinning_overrides` replace shared pins by name, for example jazzy's Python 3.12.
84+
- `pinning_overrides` replace shared pins by name, for example jazzy's Python 3.12. Items written as a block list keep their conda-build selectors (`- 3.12.* # [not riscv64]`).
8585
- `pixi run rs <distro> render-pinning` turns this into the committed `conda_build_config.yaml`.
8686

8787
A fix that applies to every distribution goes into `shared/`, a version-specific one into `distros/<distro>/`.

‎tools/import_distro.py‎

Lines changed: 1 addition & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -35,21 +35,12 @@
3535
from pathlib import Path
3636

3737
import yaml
38-
from ruamel.yaml import YAML
3938

4039
import robostack as rs
4140

4241
DERIVED_KEYS = ("conda_index", "patch_dir", "rosdistro_snapshot", "rosdistro_additional_recipes", "skip_existing")
4342

4443

45-
def _ry() -> YAML:
46-
ry = YAML()
47-
ry.preserve_quotes = True
48-
ry.width = 4096
49-
ry.indent(mapping=2, sequence=4, offset=2)
50-
return ry
51-
52-
5344
def _norm(x):
5445
if isinstance(x, str):
5546
return x.replace("-", "_")
@@ -125,7 +116,7 @@ def copy_dir(self, path: str, dest: Path) -> None:
125116

126117

127118
def import_distro(src: Source, distro: str | None) -> str:
128-
ry = _ry()
119+
ry = rs.ruamel_yaml()
129120
vinca = ry.load(src.text("vinca.yaml"))
130121
# the old repository's names (ros-<source>-*) follow its distribution, also when
131122
# --distro imports it under another name

‎tools/maintenance.py‎

Lines changed: 4 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -90,16 +90,6 @@ def fold(title: str, body: list[str]) -> None:
9090
return "\n".join(lines)
9191

9292

93-
def _ruamel():
94-
from ruamel.yaml import YAML
95-
96-
ry = YAML()
97-
ry.preserve_quotes = True
98-
ry.width = 4096
99-
ry.indent(mapping=2, sequence=4, offset=2)
100-
return ry
101-
102-
10393
REPODATA_SUBDIRS = ("noarch", "linux-64", "linux-aarch64", "osx-64", "osx-arm64", "win-64")
10494

10595

@@ -167,7 +157,7 @@ def bump_rebuild(distro: str) -> list[str]:
167157
lines = [f"`build_number` {current} → {number} ({source} + 1), mutex {old_mutex} → {new_mutex} (rebuilds every package)"]
168158

169159
if info.is_file():
170-
ry = _ruamel()
160+
ry = rs.ruamel_yaml()
171161
data = ry.load(info.read_text()) or {}
172162
dropped = []
173163
for pkg in list(data):
@@ -246,7 +236,7 @@ def bump_partial(distro: str, packages: dict[str, str]) -> list[str]:
246236
number = (current if released is None else max(current, released)) + 1
247237

248238
info = rs.DISTROS / distro / "pkg_additional_info.yaml"
249-
ry = _ruamel()
239+
ry = rs.ruamel_yaml()
250240
data = (ry.load(info.read_text()) if info.is_file() else None) or {}
251241
for pkg in packages:
252242
entry = data.get(pkg)
@@ -709,7 +699,7 @@ def _available(distro: str) -> set[str]:
709699

710700
def _append_selection(path: Path, packages: list[str]) -> None:
711701
"""Append to packages_select_by_deps keeping the file's comments (round trip)."""
712-
ry = _ruamel()
702+
ry = rs.ruamel_yaml()
713703
data = ry.load(path.read_text()) if path.is_file() else {}
714704
data.setdefault("packages_select_by_deps", [])
715705
items = data["packages_select_by_deps"]
@@ -874,7 +864,7 @@ def merge_build_numbers(path: Path, entries: dict[str, int]) -> list[str]:
874864
packages that changed."""
875865
from vinca.sort_yaml_keys import sort_mapping_keys
876866

877-
ry = _ruamel()
867+
ry = rs.ruamel_yaml()
878868
data = (ry.load(path.read_text()) if path.is_file() else None) or {}
879869
changed = []
880870
for name, number in sorted(entries.items()):

‎tools/robostack.py‎

Lines changed: 93 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
from __future__ import annotations
2121

2222
import argparse
23+
import io
2324
import json
2425
import os
2526
import re
@@ -208,45 +209,103 @@ def run_build(cmd: list[str], cwd: Path, show=None) -> int:
208209
def vinca_pinning(distro: str) -> str:
209210
"""shared/pinning/conda_forge.yaml + overrides.yaml, adjusted by distro.yaml.
210211
211-
Shared override entries that the distribution overrides itself are left out
212-
together with the comment lines above them (comments carry the conda-build
213-
selectors, so this works on text, not on parsed YAML).
212+
The files are read and written with ruamel.yaml's round trip, which keeps
213+
comments, and with them the conda-build selectors (`- 2.39 # [riscv64]`).
214+
A distribution's own pinning_overrides replace the shared entries of the same
215+
name, together with the comment above them. The distribution's own comments
216+
are left out: vinca-pinning-render moves its entries to their place in the
217+
conda-forge pinning, and the comments would end up above other keys.
214218
"""
215-
s = settings(distro)
219+
from ruamel.yaml.comments import CommentedMap, CommentedSeq
220+
221+
yaml = ruamel_yaml()
222+
s = yaml.load((DISTROS / distro / "distro.yaml").read_text()) or {}
216223
own = s.get("pinning_overrides") or {}
217-
out = [f"# Generated by `pixi run rs {distro} render-pinning` from shared/pinning/ and",
218-
f"# distros/{distro}/distro.yaml; edit those instead.", ""]
224+
shared = yaml.load((SHARED / "pinning" / "overrides.yaml").read_text())["pinning_overrides"]
225+
shared_comments = key_comments(shared)
226+
227+
out = CommentedMap()
219228
if s.get("conda_forge_pinning_version"):
220-
out.append(f"conda_forge_pinning_version: {s['conda_forge_pinning_version']}")
221-
migrations = s.get("conda_forge_migrations") or []
222-
out.append("migrations:" if migrations else "migrations: []")
223-
out += [f" - {m}" for m in migrations]
229+
out["conda_forge_pinning_version"] = s["conda_forge_pinning_version"]
230+
out["migrations"] = s.get("conda_forge_migrations") or []
231+
cut_trailing_comments(out["migrations"])
224232
else:
225-
out += [l for l in (SHARED / "pinning" / "conda_forge.yaml").read_text().splitlines()
226-
if not l.startswith("#")]
227-
comments: list[str] = []
228-
skip = False
229-
for line in (SHARED / "pinning" / "overrides.yaml").read_text().splitlines():
230-
if line.startswith("#"):
231-
continue
232-
if line.startswith(" #"):
233-
comments.append(line)
234-
elif line.startswith(" ") and not line.startswith(" "):
235-
skip = line[2:].split(":")[0] in own
236-
if not skip:
237-
out += comments + [line]
238-
comments = []
239-
elif line.startswith(" "):
240-
if not skip:
241-
out.append(line)
242-
else:
243-
out.append(line)
233+
out.update(yaml.load((SHARED / "pinning" / "conda_forge.yaml").read_text()))
234+
overrides = out["pinning_overrides"] = CommentedMap()
235+
for key, value in shared.items():
236+
if key not in own:
237+
overrides[key] = value
238+
if key in shared_comments:
239+
overrides.yaml_set_comment_before_after_key(key, before=shared_comments[key], indent=2)
240+
for key, value in own.items():
241+
if not isinstance(value, list):
242+
value = CommentedSeq([value])
243+
cut_trailing_comments(value)
244+
value.fa.set_block_style()
245+
overrides[key] = value
244246
if own:
245-
out.append(f" # ros-{distro} specific (distros/{distro}/distro.yaml)")
246-
for key, value in own.items():
247-
out.append(f" {key}:")
248-
out += [f" - {v}" for v in (value if isinstance(value, list) else [value])]
249-
return "\n".join(out) + "\n"
247+
overrides.yaml_set_comment_before_after_key(
248+
next(iter(own)), before=f"ros-{distro} specific (distros/{distro}/distro.yaml)", indent=2)
249+
250+
out.yaml_set_start_comment(f"Generated by `pixi run rs {distro} render-pinning` from shared/pinning/ and\n"
251+
f"distros/{distro}/distro.yaml; edit those instead.\n")
252+
stream = io.StringIO()
253+
yaml.dump(out, stream)
254+
return stream.getvalue()
255+
256+
257+
def ruamel_yaml():
258+
"""ruamel.yaml round trip in the layout of the repository's YAML files."""
259+
import ruamel.yaml
260+
261+
yaml = ruamel.yaml.YAML()
262+
yaml.preserve_quotes = True
263+
yaml.width = 4096
264+
yaml.indent(mapping=2, sequence=4, offset=2)
265+
return yaml
266+
267+
268+
def _comment_text(text: str) -> str:
269+
return "\n".join(line.strip().removeprefix("#").removeprefix(" ")
270+
for line in text.splitlines() if line.strip())
271+
272+
273+
def cut_trailing_comments(value) -> str:
274+
"""Cut the comment lines that follow a block list's last item (ruamel keeps
275+
the comment above the next key there) and return them; the item's own
276+
comment, e.g. its selector, stays."""
277+
token = value.ca.items.get(len(value) - 1, [None])[0] if isinstance(value, list) and value else None
278+
if token is None:
279+
return ""
280+
own_line, _, rest = token.value.partition("\n")
281+
if own_line.strip():
282+
token.value = own_line + "\n"
283+
else:
284+
value.ca.items[len(value) - 1][0] = None
285+
return rest
286+
287+
288+
def key_comments(mapping) -> dict[str, str]:
289+
"""The comment above each key of a ruamel mapping, without the "# ".
290+
291+
ruamel attaches it to whatever precedes the key: the mapping (first key), the
292+
end of the previous block list (cut from there, see cut_trailing_comments),
293+
the previous key after a scalar value, or the key itself after a flow value."""
294+
comments = {}
295+
previous = None
296+
for key in mapping:
297+
if previous is None:
298+
text = "".join(t.value for t in (mapping.ca.comment or [None, []])[1] or [])
299+
elif isinstance(mapping[previous], list) and mapping[previous]:
300+
text = cut_trailing_comments(mapping[previous])
301+
else:
302+
token = mapping.ca.items.get(previous, [None] * 3)[2]
303+
text = token.value.partition("\n")[2] if token else ""
304+
text += "".join(t.value for t in mapping.ca.items.get(key, [None, None])[1] or [])
305+
if text := _comment_text(text):
306+
comments[key] = text
307+
previous = key
308+
return comments
250309

251310

252311
def load_yaml(path: Path) -> dict:

‎tools/tests/test_pinning_update.py‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,3 +46,58 @@ def test_pin_report_lists_the_direct_users():
4646
assert "- `zig-compiler`: `(none)` → `zig`" in lines
4747
# nothing used: the folded list comes first
4848
assert maintenance.pin_report({"zig-compiler": (None, "zig")}, {})[0] == "<details>"
49+
50+
51+
def test_vinca_pinning_keeps_selectors_and_replaces_shared_entries(monkeypatch, tmp_path):
52+
(tmp_path / "pinning").mkdir()
53+
(tmp_path / "pinning" / "conda_forge.yaml").write_text("# header\nconda_forge_pinning_version: '1'\nmigrations: []\n")
54+
(tmp_path / "pinning" / "overrides.yaml").write_text(
55+
"pinning_overrides:\n"
56+
" # why channels are dropped\n"
57+
" channel_sources: null\n"
58+
" # why glibc is pinned\n"
59+
" glibc:\n"
60+
" - 2.28 # [not riscv64]\n"
61+
" - 2.39 # [riscv64]\n"
62+
" # why python is pinned\n"
63+
" python:\n"
64+
" - 3.14.*\n"
65+
" # why lua is pinned\n"
66+
" lua:\n"
67+
" - 5.4\n"
68+
)
69+
(tmp_path / "x").mkdir()
70+
(tmp_path / "x" / "distro.yaml").write_text(
71+
"pinning_overrides:\n"
72+
" xtensor: ['0.25.0']\n"
73+
" # not carried over\n"
74+
" python:\n"
75+
" - 3.12.* # [not riscv64]\n"
76+
" - 3.14.*\n"
77+
"# nor is this one\n"
78+
"rosdistro_sync: x/2026-01-01\n"
79+
)
80+
monkeypatch.setattr(rs, "SHARED", tmp_path)
81+
monkeypatch.setattr(rs, "DISTROS", tmp_path)
82+
assert rs.vinca_pinning("x") == (
83+
"# Generated by `pixi run rs x render-pinning` from shared/pinning/ and\n"
84+
"# distros/x/distro.yaml; edit those instead.\n"
85+
"conda_forge_pinning_version: '1'\n"
86+
"migrations: []\n"
87+
"pinning_overrides:\n"
88+
" # why channels are dropped\n"
89+
" channel_sources:\n"
90+
" # why glibc is pinned\n"
91+
" glibc:\n"
92+
" - 2.28 # [not riscv64]\n"
93+
" - 2.39 # [riscv64]\n"
94+
" # why lua is pinned\n"
95+
" lua:\n"
96+
" - 5.4\n"
97+
" # ros-x specific (distros/x/distro.yaml)\n"
98+
" xtensor:\n"
99+
" - '0.25.0'\n"
100+
" python:\n"
101+
" - 3.12.* # [not riscv64]\n"
102+
" - 3.14.*\n"
103+
)

0 commit comments

Comments
 (0)