Skip to content

Commit 0c371b1

Browse files
brunissbruniss
committed
add prebuilt deps containers
0 parents  commit 0c371b1

15 files changed

Lines changed: 658 additions & 0 deletions

‎.gitattributes‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
* text=auto
2+
*.sh text eol=lf
3+
*.yml text eol=lf
4+
*.yaml text eol=lf
5+
*.ps1 text eol=crlf
6+

‎.github/workflows/publish-linux.yml‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
name: Publish Linux dependencies
2+
3+
on:
4+
push:
5+
branches: [main]
6+
paths:
7+
- linux/**
8+
- .github/workflows/publish-linux.yml
9+
workflow_dispatch:
10+
11+
permissions:
12+
contents: read
13+
packages: write
14+
15+
concurrency:
16+
group: publish-linux-${{ github.ref }}
17+
cancel-in-progress: true
18+
19+
jobs:
20+
publish:
21+
runs-on: ubuntu-24.04
22+
timeout-minutes: 45
23+
steps:
24+
- uses: actions/checkout@v4
25+
- uses: docker/setup-buildx-action@v3
26+
- uses: docker/login-action@v3
27+
with:
28+
registry: ghcr.io
29+
username: ${{ github.actor }}
30+
password: ${{ secrets.GITHUB_TOKEN }}
31+
- uses: docker/build-push-action@v6
32+
with:
33+
context: linux
34+
file: linux/Dockerfile
35+
platforms: linux/amd64
36+
push: true
37+
tags: |
38+
ghcr.io/scrollprize/vc3d-deps/linux:ubuntu-26.04
39+
ghcr.io/scrollprize/vc3d-deps/linux:sha-${{ github.sha }}
40+
labels: |
41+
org.opencontainers.image.revision=${{ github.sha }}
42+
org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
43+
cache-from: type=gha,scope=vc3d-linux-ubuntu-26.04
44+
cache-to: type=gha,mode=max,scope=vc3d-linux-ubuntu-26.04

‎.github/workflows/publish-macos.yml‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
name: Publish macOS dependencies
2+
3+
on:
4+
push:
5+
branches: [main]
6+
paths:
7+
- macos/**
8+
- .github/workflows/publish-macos.yml
9+
workflow_dispatch:
10+
11+
permissions:
12+
contents: read
13+
packages: write
14+
15+
concurrency:
16+
group: publish-macos-${{ github.ref }}
17+
cancel-in-progress: true
18+
19+
jobs:
20+
publish:
21+
runs-on: macos-15
22+
timeout-minutes: 150
23+
env:
24+
HOMEBREW_NO_AUTO_UPDATE: '1'
25+
HOMEBREW_NO_INSTALL_CLEANUP: '1'
26+
steps:
27+
- uses: actions/checkout@v4
28+
- name: Require Apple Silicon runner
29+
run: test "$(uname -m)" = arm64
30+
- name: Install VC3D dependencies
31+
run: macos/install-deps.sh
32+
- name: Verify toolchain
33+
run: |
34+
/opt/homebrew/opt/llvm/bin/clang++ --version
35+
/opt/homebrew/opt/flang/bin/flang --version
36+
cmake --version
37+
/opt/homebrew/opt/qt/bin/macdeployqt -h >/dev/null
38+
- name: Create Homebrew snapshot
39+
run: |
40+
archive="$RUNNER_TEMP/vc3d-macos-15-arm64.tar.zst"
41+
formulae="$RUNNER_TEMP/vc3d-macos-15-arm64-formulae.txt"
42+
roots=()
43+
while IFS= read -r formula; do
44+
roots+=("$formula")
45+
done < <(sed -e 's/[[:space:]]*#.*//' -e '/^[[:space:]]*$/d' macos/formulae.txt)
46+
{
47+
printf '%s\n' "${roots[@]}"
48+
brew deps --installed --union --include-build --include-optional "${roots[@]}"
49+
} | sort -u > "$formulae"
50+
paths="$RUNNER_TEMP/vc3d-macos-15-arm64-paths.txt"
51+
while IFS= read -r formula; do
52+
cellar=$(brew --cellar "$formula")
53+
printf './%s\n' "${cellar#/opt/homebrew/}"
54+
done < "$formulae" > "$paths"
55+
sudo tar \
56+
-cf - \
57+
-C /opt/homebrew \
58+
-T "$paths" \
59+
| zstd -T0 -10 -o "$archive"
60+
- uses: oras-project/setup-oras@v1
61+
- name: Publish snapshot to GHCR
62+
env:
63+
GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
64+
run: |
65+
printf '%s' "$GHCR_TOKEN" | oras login ghcr.io -u '${{ github.actor }}' --password-stdin
66+
repo="ghcr.io/scrollprize/vc3d-deps/macos"
67+
cd "$RUNNER_TEMP"
68+
artifact="vc3d-macos-15-arm64.tar.zst"
69+
formulae="vc3d-macos-15-arm64-formulae.txt"
70+
oras push "$repo:15-arm64" \
71+
--artifact-type "application/vnd.vc3d.homebrew.snapshot.v1" \
72+
"$artifact:application/vnd.vc3d.homebrew.layer.v1+tar+zstd" \
73+
"$formulae:text/plain" \
74+
--annotation "org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}" \
75+
--annotation "org.opencontainers.image.revision=${{ github.sha }}"
76+
oras cp "$repo:15-arm64" "$repo:sha-${{ github.sha }}"

‎.github/workflows/publish-windows.yml‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
name: Publish Windows dependencies
2+
3+
on:
4+
push:
5+
branches: [main]
6+
paths:
7+
- windows/**
8+
- .github/workflows/publish-windows.yml
9+
workflow_dispatch:
10+
11+
permissions:
12+
contents: read
13+
packages: write
14+
15+
concurrency:
16+
group: publish-windows-${{ github.ref }}
17+
cancel-in-progress: true
18+
19+
jobs:
20+
publish:
21+
runs-on: windows-2025
22+
timeout-minutes: 90
23+
steps:
24+
- uses: actions/checkout@v4
25+
- uses: msys2/setup-msys2@v2
26+
id: msys2
27+
with:
28+
msystem: UCRT64
29+
update: true
30+
- name: Install VC3D dependencies
31+
shell: msys2 {0}
32+
run: windows/install-deps.sh
33+
- name: Verify toolchain
34+
shell: msys2 {0}
35+
run: |
36+
cmake --version
37+
ninja --version
38+
g++ --version
39+
makensis -VERSION
40+
- name: Create UCRT64 snapshot
41+
shell: pwsh
42+
env:
43+
MSYS2_LOCATION: ${{ steps.msys2.outputs.msys2-location }}
44+
run: |
45+
$archive = Join-Path $env:RUNNER_TEMP 'vc3d-windows-ucrt64.7z'
46+
Push-Location $env:MSYS2_LOCATION
47+
& "$env:ProgramFiles\7-Zip\7z.exe" a -t7z -mx=5 -mmt=on $archive '.\*'
48+
if ($LASTEXITCODE -ne 0) { throw "7-Zip failed with exit $LASTEXITCODE" }
49+
Pop-Location
50+
- uses: oras-project/setup-oras@v1
51+
- name: Publish snapshot to GHCR
52+
shell: pwsh
53+
env:
54+
GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
55+
run: |
56+
$env:GHCR_TOKEN | oras login ghcr.io -u '${{ github.actor }}' --password-stdin
57+
Push-Location $env:RUNNER_TEMP
58+
$artifact = 'vc3d-windows-ucrt64.7z'
59+
$repo = 'ghcr.io/scrollprize/vc3d-deps/windows'
60+
$args = @(
61+
'push', "${repo}:ucrt64", "$artifact`:application/vnd.7zip",
62+
'--artifact-type', 'application/vnd.vc3d.msys2.snapshot.v1',
63+
'--annotation', 'org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}',
64+
'--annotation', 'org.opencontainers.image.revision=${{ github.sha }}'
65+
)
66+
& oras @args
67+
if ($LASTEXITCODE -ne 0) { throw "ORAS push failed with exit $LASTEXITCODE" }
68+
& oras cp "${repo}:ucrt64" "${repo}:sha-${{ github.sha }}"
69+
if ($LASTEXITCODE -ne 0) { throw "ORAS tag copy failed with exit $LASTEXITCODE" }
70+
Pop-Location

‎.github/workflows/validate.yml‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
name: Validate
2+
3+
on:
4+
pull_request:
5+
push:
6+
branches: [main]
7+
8+
permissions:
9+
contents: read
10+
11+
jobs:
12+
manifests:
13+
runs-on: ubuntu-24.04
14+
steps:
15+
- uses: actions/checkout@v4
16+
- name: Validate shell scripts
17+
run: |
18+
bash -n linux/install-deps.sh
19+
bash -n windows/install-deps.sh
20+
bash -n macos/install-deps.sh
21+
bash -n macos/restore.sh
22+
- name: Reject duplicate manifest entries
23+
run: |
24+
for manifest in windows/packages.txt macos/formulae.txt; do
25+
duplicates=$(sed -e 's/[[:space:]]*#.*//' -e '/^[[:space:]]*$/d' "$manifest" | sort | uniq -d)
26+
if [[ -n "$duplicates" ]]; then
27+
echo "Duplicate entries in $manifest:"
28+
echo "$duplicates"
29+
exit 1
30+
fi
31+
done
32+
- name: Validate Dockerfile
33+
uses: hadolint/hadolint-action@v3.1.0
34+
with:
35+
dockerfile: linux/Dockerfile
36+
failure-threshold: error
37+

‎.gitignore‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
.DS_Store
2+
*.tar.zst
3+
*.7z
4+

‎README.md‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
# VC3D CI dependencies
2+
3+
Prebuilt build environments for `ScrollPrize/villa`'s
4+
`volume-cartographer` (VC3D) CI jobs.
5+
6+
Installing Qt, OpenCV, Ceres, CGAL, LLVM, and the sparse-solver stack is the
7+
slowest part of a clean VC3D build. This repository installs those dependencies
8+
once and publishes three platform-specific artifacts to GHCR:
9+
10+
| Target | Published artifact | How CI uses it |
11+
| --- | --- | --- |
12+
| Ubuntu 26.04, amd64 | `ghcr.io/scrollprize/vc3d-deps/linux:ubuntu-26.04` | Docker builder image |
13+
| Windows, amd64 | `ghcr.io/scrollprize/vc3d-deps/windows:ucrt64` | MSYS2/UCRT64 snapshot (`.7z`) |
14+
| macOS 15, arm64 | `ghcr.io/scrollprize/vc3d-deps/macos:15-arm64` | Homebrew dependency snapshot (`.tar.zst`) |
15+
16+
The moving platform tags above are convenient for PR CI. Every publication is
17+
also tagged `sha-<commit>` so release jobs can pin an immutable input. For full
18+
reproducibility, resolve and pin an OCI digest rather than a moving tag.
19+
20+
## Publishing
21+
22+
Each platform has a separate workflow. A push to `main` publishes only the
23+
platform whose manifest, build script, or workflow changed. All three workflows
24+
also support manual dispatch.
25+
26+
The repository package must be visible to `ScrollPrize/villa`, and the consuming
27+
job needs `packages: read`. Publishing jobs use the repository `GITHUB_TOKEN`
28+
and need `packages: write`; no long-lived registry credential is required.
29+
30+
## Consuming from `villa`
31+
32+
### Linux
33+
34+
The existing `volume-cartographer/scripts/ci.sh` can pull and run the image, or
35+
the job can use it directly:
36+
37+
```yaml
38+
permissions:
39+
contents: read
40+
packages: read
41+
42+
jobs:
43+
build:
44+
runs-on: ubuntu-24.04
45+
container:
46+
image: ghcr.io/scrollprize/vc3d-deps/linux:ubuntu-26.04
47+
credentials:
48+
username: ${{ github.actor }}
49+
password: ${{ secrets.GITHUB_TOKEN }}
50+
steps:
51+
- uses: actions/checkout@v4
52+
- working-directory: volume-cartographer
53+
run: |
54+
cmake --preset ci-release-tests-gcc -DVC_USE_CCACHE=ON
55+
ninja -C build/ci-release-tests-gcc
56+
ctest --preset ci-release-tests-gcc
57+
```
58+
59+
### Windows
60+
61+
Pull the snapshot with ORAS, restore it, then run the build with the included
62+
MSYS2 Bash. There is no package installation in the consuming job.
63+
64+
```yaml
65+
permissions:
66+
contents: read
67+
packages: read
68+
69+
steps:
70+
- uses: actions/checkout@v4
71+
- uses: actions/checkout@v4
72+
with:
73+
repository: ScrollPrize/vc3d-deps
74+
path: vc3d-deps
75+
- uses: oras-project/setup-oras@v1
76+
- shell: pwsh
77+
env:
78+
GHCR_TOKEN: ${{ github.token }}
79+
run: |
80+
$env:GHCR_TOKEN | oras login ghcr.io -u '${{ github.actor }}' --password-stdin
81+
oras pull ghcr.io/scrollprize/vc3d-deps/windows:ucrt64 -o $env:RUNNER_TEMP
82+
& .\vc3d-deps\windows\restore.ps1 `
83+
-Archive "$env:RUNNER_TEMP\vc3d-windows-ucrt64.7z"
84+
- name: Configure, build, and test
85+
working-directory: volume-cartographer
86+
shell: C:\msys64\usr\bin\bash.exe -leo pipefail {0}
87+
env:
88+
MSYSTEM: UCRT64
89+
CHERE_INVOKING: '1'
90+
PATH: C:\msys64\ucrt64\bin;C:\msys64\usr\bin;${{ env.PATH }}
91+
run: |
92+
cmake --preset ci-windows-mingw -DVC_USE_CCACHE=ON
93+
ninja -C build/ci-windows-mingw
94+
ctest --test-dir build/ci-windows-mingw --output-on-failure
95+
```
96+
97+
The example assumes this repository is checked out at `vc3d-deps`, for example
98+
with `actions/checkout`'s `repository` and `path` inputs.
99+
100+
### macOS
101+
102+
The macOS snapshot is tied to the runner image and architecture in its tag.
103+
Restore it only on `macos-15` arm64 runners.
104+
105+
```yaml
106+
permissions:
107+
contents: read
108+
packages: read
109+
110+
steps:
111+
- uses: actions/checkout@v4
112+
- uses: actions/checkout@v4
113+
with:
114+
repository: ScrollPrize/vc3d-deps
115+
path: vc3d-deps
116+
- uses: oras-project/setup-oras@v1
117+
- shell: bash
118+
env:
119+
GHCR_TOKEN: ${{ github.token }}
120+
run: |
121+
printf '%s' "$GHCR_TOKEN" | oras login ghcr.io -u '${{ github.actor }}' --password-stdin
122+
oras pull ghcr.io/scrollprize/vc3d-deps/macos:15-arm64 -o "$RUNNER_TEMP"
123+
vc3d-deps/macos/restore.sh "$RUNNER_TEMP/vc3d-macos-15-arm64.tar.zst"
124+
- working-directory: volume-cartographer
125+
run: ./scripts/build_macos.sh --ccache
126+
```
127+
128+
The restore helper adds the captured formula closure to Homebrew's standard
129+
Apple Silicon prefix, `/opt/homebrew`, and relinks it. It refuses to run on the
130+
wrong OS, architecture, or prefix. `HOMEBREW_NO_AUTO_UPDATE=1` should remain set
131+
in the build job.
132+
133+
## Updating dependencies
134+
135+
- Linux packages: edit `linux/install-deps.sh`.
136+
- Windows packages: edit `windows/packages.txt`.
137+
- macOS formulae: edit `macos/formulae.txt`.
138+
139+
Keep these lists synchronized with VC3D's CMake configuration and packaging
140+
requirements. Open a PR here first, let the new artifacts publish from `main`,
141+
then update `villa` to consume the new immutable tag or digest.
142+
143+
## Why Windows and macOS are archives
144+
145+
[GitHub Actions container actions run only on Linux][container-actions]. GitHub
146+
offers [custom VM images][custom-images] for larger Linux and Windows runners,
147+
but not macOS, and the current `villa` workflows use standard hosted runners.
148+
Native filesystem snapshots preserve the toolchains needed by NSIS and
149+
`macdeployqt` without changing those runner types.
150+
151+
[container-actions]: https://docs.github.com/actions/concepts/workflows-and-actions/custom-actions#docker-container-actions
152+
[custom-images]: https://docs.github.com/actions/how-tos/manage-runners/larger-runners/use-custom-images

0 commit comments

Comments
 (0)