Skip to content

Commit 2183a9b

Browse files
Tobias-Fischerclaude
Tobias-Fischer
and
claude
committed
Prototype: all RoboStack ROS 2 distributions in one repository
distros/<distro>/ holds only what is specific to a distribution; shared/ holds the rosdep mapping, pinning and tests; tools/robostack.py runs every task for a distribution (pixi run rs <distro> <task>); CI builds only the distributions a PR touches. Data copied from RoboStack/ros-{rolling,humble,lyrical,kilted} main and ros-jazzy codex/cross-distro-sync. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
0 parents  commit 2183a9b

1,148 files changed

Lines changed: 164671 additions & 0 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.gitattributes‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# This file is generated from ros-distro-template (template/.gitattributes).
2+
# If you change it here, upstream the change: comment `@robostack-bot upstream-to-template` on your PR.
3+
4+
# GitHub syntax highlighting
5+
pixi.lock linguist-language=YAML linguist-generated=true

‎.github/FUNDING.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# This file is generated from ros-distro-template (template/.github/FUNDING.yml).
2+
# If you change it here, upstream the change: comment `@robostack-bot upstream-to-template` on your PR.
3+
4+
github: RoboStack
5+
open_collective: robostack

‎.github/workflows/main.yml‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
# After a merge: regenerate the staged build workflows of every distribution the
2+
# merge touched, one build branch per distribution and platform
3+
# (buildbranch_<distro>_<platform>, workflows .github/workflows/build_<distro>_*.yml).
4+
#
5+
# Pushing build branches starts the real builds and uploads, so in this prototype
6+
# it only does that in the RoboStack organisation; elsewhere (or with dry_run) the
7+
# generated workflows are uploaded as an artifact instead.
8+
name: Generate build branches
9+
10+
on:
11+
push:
12+
branches: [main]
13+
workflow_dispatch:
14+
inputs:
15+
distros:
16+
description: Space-separated distributions (default all)
17+
required: false
18+
dry_run:
19+
description: Only upload the generated workflows as an artifact
20+
type: boolean
21+
default: true
22+
23+
permissions:
24+
contents: write
25+
26+
jobs:
27+
plan:
28+
runs-on: ubuntu-latest
29+
outputs:
30+
distros: ${{ steps.plan.outputs.distros }}
31+
steps:
32+
- uses: actions/checkout@v6
33+
with:
34+
fetch-depth: 0
35+
- id: plan
36+
env:
37+
BEFORE: ${{ github.event.before }}
38+
SELECTED: ${{ inputs.distros }}
39+
run: |
40+
if [ -n "$SELECTED" ]; then
41+
distros=$(python3 -c 'import json,os; print(json.dumps(os.environ["SELECTED"].split()))')
42+
elif [ -n "$BEFORE" ] && git cat-file -e "$BEFORE" 2>/dev/null; then
43+
distros=$(python3 tools/robostack.py changed-distros --base "$BEFORE")
44+
else
45+
distros=$(python3 tools/robostack.py changed-distros --base HEAD --all)
46+
fi
47+
echo "distros=$distros" | tee -a "$GITHUB_OUTPUT"
48+
49+
build-branches:
50+
needs: plan
51+
if: needs.plan.outputs.distros != '[]'
52+
runs-on: ubuntu-latest
53+
strategy:
54+
fail-fast: false
55+
matrix:
56+
distro: ${{ fromJSON(needs.plan.outputs.distros) }}
57+
env:
58+
DISTRO: ${{ matrix.distro }}
59+
PUSH: ${{ github.repository_owner == 'RoboStack' && !inputs.dry_run }}
60+
steps:
61+
- uses: actions/checkout@v6
62+
with:
63+
persist-credentials: false
64+
fetch-depth: 0
65+
- uses: prefix-dev/setup-pixi@v0.10.2
66+
with:
67+
frozen: true
68+
pixi-version: v0.75.0
69+
- name: Generate and push the build branches
70+
env:
71+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
72+
PUSH_TOKEN: ${{ secrets.GHA_PAT }}
73+
run: |
74+
set -e
75+
git config --local user.email "action@github.com"
76+
git config --local user.name "GitHub Action"
77+
mkdir -p "$RUNNER_TEMP/workflows"
78+
for platform in linux-64 osx-64 osx-arm64 win-64 linux-aarch64; do
79+
git reset --hard "$GITHUB_SHA" && git clean -fdx -e .pixi
80+
batch=10; [ "$platform" == "win-64" ] && batch=25 # https://github.com/RoboStack/robostack.github.io/issues/105
81+
pixi run -v rs "$DISTRO" generate-recipes --platform "$platform"
82+
pixi run -v rs "$DISTRO" gha --platform "$platform" --batch-size "$batch"
83+
cp .github/workflows/build_"$DISTRO"_*.yml "$RUNNER_TEMP/workflows/"
84+
if [ "$PUSH" == "true" ]; then
85+
branch="buildbranch_${DISTRO}_${platform//-/_}"
86+
git add -f . ':!.pixi'
87+
git commit -qm "Build files for $DISTRO $platform $(date '+%Y-%m-%d-%H%M')"
88+
git push "https://${GITHUB_ACTOR}:${PUSH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" "HEAD:$branch" --force
89+
fi
90+
done
91+
- uses: actions/upload-artifact@v6
92+
with:
93+
name: build-workflows-${{ matrix.distro }}
94+
path: ${{ runner.temp }}/workflows

‎.github/workflows/testpr.yml‎

Lines changed: 190 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,190 @@
1+
# Pull-request build: only the distributions a PR touches are built
2+
# (a change outside distros/ affects all of them), on all five platforms.
3+
on:
4+
pull_request:
5+
workflow_dispatch:
6+
inputs:
7+
distros:
8+
description: Space-separated distributions to build (default all)
9+
required: false
10+
11+
concurrency:
12+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
13+
cancel-in-progress: true
14+
15+
permissions:
16+
actions: read
17+
contents: read
18+
19+
env:
20+
PYTHONIOENCODING: utf-8
21+
SAVE_CACHE_AS_ARTIFACT: 'true'
22+
23+
jobs:
24+
plan:
25+
runs-on: ubuntu-latest
26+
outputs:
27+
distros: ${{ steps.plan.outputs.distros }}
28+
steps:
29+
- uses: actions/checkout@v6
30+
with:
31+
persist-credentials: false
32+
fetch-depth: 0
33+
- name: Which distributions changed
34+
id: plan
35+
env:
36+
BASE: ${{ github.event.pull_request.base.sha }}
37+
SELECTED: ${{ inputs.distros }}
38+
run: |
39+
# plain python + PyYAML of the runner image; no pixi needed for this
40+
if [ -n "$SELECTED" ]; then
41+
distros=$(python3 -c 'import json,os; print(json.dumps(os.environ["SELECTED"].split()))')
42+
elif [ -n "$BASE" ]; then
43+
distros=$(python3 tools/robostack.py changed-distros --base "$BASE")
44+
else
45+
distros=$(python3 tools/robostack.py changed-distros --base HEAD --all)
46+
fi
47+
echo "distros=$distros" | tee -a "$GITHUB_OUTPUT"
48+
49+
sort-check:
50+
runs-on: ubuntu-latest
51+
steps:
52+
- uses: actions/checkout@v6
53+
with:
54+
persist-credentials: false
55+
- uses: prefix-dev/setup-pixi@v0.10.2
56+
with:
57+
frozen: true
58+
pixi-version: v0.75.0
59+
- name: Check sorting
60+
run: pixi run sort --check
61+
- name: Check conda_build_config.yaml is up to date
62+
run: |
63+
pixi run render-pinning
64+
git diff --exit-code -- 'distros/*/conda_build_config.yaml'
65+
66+
build:
67+
needs: plan
68+
if: needs.plan.outputs.distros != '[]'
69+
strategy:
70+
fail-fast: false
71+
matrix:
72+
distro: ${{ fromJSON(needs.plan.outputs.distros) }}
73+
target:
74+
- { os: ubuntu-latest, platform: linux-64 }
75+
- { os: ubuntu-24.04-arm, platform: linux-aarch64 }
76+
- { os: macos-15-intel, platform: osx-64 }
77+
- { os: macos-15, platform: osx-arm64 }
78+
- { os: windows-2022, platform: win-64 }
79+
name: ${{ matrix.distro }} ${{ matrix.target.platform }}
80+
runs-on: ${{ matrix.target.os }}
81+
env:
82+
DISTRO: ${{ matrix.distro }}
83+
PLATFORM: ${{ matrix.target.platform }}
84+
# rattler-build writes to ./output of the distribution, or to CONDA_BLD_PATH (win-64)
85+
CACHE_DIR: ${{ matrix.target.platform == 'win-64' && 'C:/bld/win-64' || format('distros/{0}/output/{1}', matrix.distro, matrix.target.platform) }}
86+
steps:
87+
- uses: actions/checkout@v6
88+
with:
89+
persist-credentials: false
90+
fetch-depth: 0
91+
92+
- uses: prefix-dev/setup-pixi@v0.10.2
93+
with:
94+
frozen: true
95+
pixi-version: v0.75.0
96+
97+
- name: Windows workarounds (short build path, long paths, PATH and SDK clean-up)
98+
if: runner.os == 'Windows'
99+
shell: pwsh
100+
run: |
101+
"CONDA_BLD_PATH=C:\bld\" | Out-File -FilePath $env:GITHUB_ENV -Append
102+
New-Item -ItemType Directory -Force -Path C:\bld | Out-Null
103+
New-ItemProperty -Path "HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem" -Name "LongPathsEnabled" -Value 1 -PropertyType DWORD -Force
104+
Remove-Item -Recurse -Force C:\Strawberry -ErrorAction SilentlyContinue
105+
Remove-Item -Recurse -Force "C:\Program Files (x86)\Windows Kits\10\Include\10.0.17763.0" -ErrorAction SilentlyContinue
106+
107+
- name: Clean up PATH
108+
if: runner.os == 'Windows'
109+
uses: egor-tensin/cleanup-path@v5
110+
with:
111+
dirs: 'C:\Program Files\Git\usr\bin;C:\Program Files\Git\bin;C:\Program Files\Git\cmd;C:\Program Files\Git\mingw64\bin'
112+
113+
- name: Disable git auto-maintenance in source caches
114+
shell: bash -l {0}
115+
run: git config --global maintenance.auto false
116+
117+
- name: Generate recipes
118+
shell: bash -l {0}
119+
env:
120+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
121+
run: |
122+
set -e
123+
for attempt in 1 2 3; do
124+
if pixi run -v rs "$DISTRO" generate-recipes --platform "$PLATFORM" -n; then break; fi
125+
if [[ "${attempt}" == "3" ]]; then echo "Recipe generation failed after 3 attempts" >&2; exit 1; fi
126+
echo "Recipe generation attempt ${attempt} failed; retrying..." >&2
127+
sleep 15
128+
done
129+
130+
- name: Check patches
131+
shell: bash -l {0}
132+
run: |
133+
pixi run rs "$DISTRO" check-orphaned-patches
134+
pixi run rs "$DISTRO" check-patches
135+
136+
- name: Restore build cache
137+
uses: actions/cache/restore@v5
138+
with:
139+
path: ${{ env.CACHE_DIR }}
140+
key: ${{ runner.os }}-${{ matrix.distro }}-${{ matrix.target.platform }}-pr-${{ github.event.pull_request.number }}
141+
restore-keys: |
142+
${{ runner.os }}-${{ matrix.distro }}-${{ matrix.target.platform }}-pr-${{ github.event.pull_request.number }}-
143+
144+
# Temporary rebuild controls live in distros/<distro>/ci.yaml.
145+
- name: Apply ci.yaml cache controls
146+
shell: bash -l {0}
147+
run: |
148+
pixi run python tools/ci_config.py --cache-dir "$CACHE_DIR" --config "distros/$DISTRO/ci.yaml" --vinca "distros/$DISTRO/vinca.yaml"
149+
mkdir -p "$CACHE_DIR"
150+
pixi run rattler-index fs "$CACHE_DIR/.." --force
151+
152+
- name: Build recipes
153+
id: build-recipes
154+
shell: bash -l {0}
155+
run: |
156+
set +e
157+
extra=()
158+
# shorter work dirs for MAX_PATH on Windows
159+
[ "$PLATFORM" == "win-64" ] && extra+=(--no-build-id)
160+
pixi run rs "$DISTRO" build --target-platform "$PLATFORM" "${extra[@]}" 2>&1 | tee rattler-build.log
161+
status=${PIPESTATUS[0]}
162+
echo "exit-code=${status}" >> "$GITHUB_OUTPUT"
163+
exit "$status"
164+
165+
- name: Summarize build result
166+
if: always()
167+
shell: bash -l {0}
168+
run: |
169+
pixi run python tools/summarize_rattler_build.py --log rattler-build.log \
170+
--platform "$DISTRO $PLATFORM" --outcome "${{ steps.build-recipes.outcome }}" >> "$GITHUB_STEP_SUMMARY"
171+
172+
- name: Upload build log
173+
if: always()
174+
uses: actions/upload-artifact@v6
175+
with:
176+
name: build-log-${{ matrix.distro }}-${{ matrix.target.platform }}-${{ github.run_id }}-${{ github.run_attempt }}
177+
path: rattler-build.log
178+
if-no-files-found: warn
179+
retention-days: 7
180+
181+
- name: Save build cache
182+
if: ${{ always() && contains(fromJSON('["success","failure","cancelled"]'), steps.build-recipes.outcome) }}
183+
uses: actions/cache/save@v5
184+
with:
185+
path: ${{ env.CACHE_DIR }}
186+
key: ${{ runner.os }}-${{ matrix.distro }}-${{ matrix.target.platform }}-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }}
187+
188+
- name: Generate the staged build workflows (catches post-merge problems)
189+
shell: bash -l {0}
190+
run: pixi run rs "$DISTRO" gha --platform "$PLATFORM" --batch-size 25

‎.gitignore‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
.pixi
2+
__pycache__
3+
*.egg-info
4+
.DS_Store
5+
# generated per distribution by tools/robostack.py
6+
distros/*/recipes/
7+
distros/*/recipes_only_patch/
8+
distros/*/tests/
9+
distros/*/vinca_pinning.yaml
10+
distros/*/output/
11+
distros/*/*.json
12+
distros/*/buildorder.txt

‎.scripts/build_unix.sh‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
#!/usr/bin/env bash
2+
3+
4+
# Default values
5+
target=""
6+
7+
# Parse arguments
8+
while [[ "$#" -gt 0 ]]; do
9+
case $1 in
10+
--target) target="$2"; shift ;; # Capture value for --target
11+
*) echo "Unknown parameter passed: $1"; exit 1 ;; # Handle invalid options
12+
esac
13+
shift
14+
done
15+
16+
set -xeuo pipefail
17+
export PYTHONUNBUFFERED=1
18+
19+
# ROBOSTACK_DISTRO is set by the generated build workflow (tools/robostack.py gha).
20+
: "${ROBOSTACK_DISTRO:?ROBOSTACK_DISTRO is not set}"
21+
cd "distros/${ROBOSTACK_DISTRO}"
22+
export FEEDSTOCK_ROOT=`pwd`
23+
export "CONDA_BLD_PATH=$HOME/conda-bld/"
24+
25+
# git's background auto-maintenance can hold (and then delete) a lock file in a
26+
# cached source clone while rattler-build copies it, failing the build with
27+
# "FileSystem error: ... .git/objects/maintenance.lock does not exist".
28+
git config --global maintenance.auto false
29+
30+
curl -fsSL https://pixi.sh/install.sh | bash
31+
export PATH="$HOME/.pixi/bin:$PATH"
32+
33+
if [[ "$target" == *"osx"* ]]; then
34+
echo "osx"
35+
export PATH=$(echo $PATH | tr ":" "\n" | grep -v 'homebrew' | xargs | tr ' ' ':')
36+
fi
37+
38+
if [[ "$target" == "emscripten-wasm32" ]]; then
39+
extra_channel="-c https://repo.mamba.pm/emscripten-forge"
40+
cross_compile="--target-platform emscripten-wasm32 --test skip"
41+
42+
else
43+
extra_channel=""
44+
cross_compile=""
45+
fi
46+
47+
48+
for recipe in ${CURRENT_RECIPES[@]}; do
49+
# build-ci (tools/robostack.py) adds the variant config and the distro's channels.
50+
pixi run -v rs "${ROBOSTACK_DISTRO}" build-ci \
51+
--recipe ${FEEDSTOCK_ROOT}/recipes/${recipe} \
52+
${extra_channel} \
53+
--output-dir $CONDA_BLD_PATH \
54+
${cross_compile}
55+
# -m ${FEEDSTOCK_ROOT}/.ci_support/conda_forge_pinnings.yaml \
56+
57+
done
58+
59+
# Check if it build something, this is a hotfix for the skips inside additional_recipes
60+
shopt -s nullglob
61+
conda_packages=( "${CONDA_BLD_PATH}/${target}"*/*.conda )
62+
if (( ${#conda_packages[@]} > 0 )); then
63+
# Upload packages one-by-one; the upload task (tools/robostack.py) skips or overwrites
64+
# packages that already exist, depending on the upload target.
65+
for conda_package in "${conda_packages[@]}"; do
66+
pixi run rs "${ROBOSTACK_DISTRO}" upload "${conda_package}"
67+
done
68+
else
69+
echo "Warning: No .conda files found in ${CONDA_BLD_PATH}/${target}"
70+
echo "This might be due to all the packages being skipped"
71+
fi

0 commit comments

Comments
 (0)