Skip to content

Commit 7f746b2

Browse files
huydhnhuydhn
committed
Move the tutorials build to OSDC runners
1 parent c80a434 commit 7f746b2

7 files changed

Lines changed: 153 additions & 162 deletions

File tree

‎.github/labeler.yml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
"ciflow/docs":
2+
globs:
3+
- "*.rst"
4+
- "*.py"
5+
- Makefile
6+
- requirements.txt
7+
- _static/**
8+
- "*_source/**"
9+
- .jenkins/**
10+
- .ci/docker/**
11+
- .github/workflows/_build-tutorials-base.yml
12+
- .github/workflows/build-tutorials.yml
13+
draft: false

‎.github/pytorch-probot.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,4 @@
11
tracking_issue: 1896
2+
ciflow_push_tags:
3+
- ciflow/docs
4+
labeler_config: labeler.yml

‎.github/workflows/_build-tutorials-base.yml‎

Lines changed: 123 additions & 134 deletions
Original file line numberDiff line numberDiff line change
@@ -20,138 +20,138 @@ on:
2020
default: "pytorch/tutorials"
2121

2222
jobs:
23-
worker:
24-
name: pytorch_tutorial_build_worker
25-
strategy:
26-
matrix:
27-
include:
28-
- { shard: 1, num_shards: 15, runner: "linux.g5.12xlarge.nvidia.gpu" }
29-
- { shard: 2, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
30-
- { shard: 3, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
31-
- { shard: 4, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
32-
- { shard: 5, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
33-
- { shard: 6, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
34-
- { shard: 7, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
35-
- { shard: 8, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
36-
- { shard: 9, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
37-
- { shard: 10, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
38-
- { shard: 11, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
39-
- { shard: 12, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
40-
- { shard: 13, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
41-
- { shard: 14, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
42-
- { shard: 15, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
43-
fail-fast: false
44-
runs-on: ${{ matrix.runner }}
23+
docker-image:
24+
name: Resolve CI docker image
25+
runs-on: mt-l-x86iavx512-8-64
26+
container:
27+
image: ghcr.io/actions/actions-runner:latest
28+
permissions:
29+
id-token: write
30+
contents: read
31+
outputs:
32+
docker-image: ${{ steps.image.outputs.tag }}
4533
steps:
46-
- name: Setup SSH (Click me for login details)
47-
uses: pytorch/test-infra/.github/actions/setup-ssh@main
48-
with:
49-
github-secret: ${{ secrets.GITHUB_TOKEN }}
50-
instructions: |
51-
All testing is done inside the container, to start an interactive session run:
52-
docker exec -it $(docker container ps --format '{{.ID}}') bash
53-
5434
- name: Checkout Tutorials
5535
uses: actions/checkout@v6
56-
with:
57-
fetch-depth: 0
58-
59-
- name: Setup Linux
60-
uses: pytorch/pytorch/.github/actions/setup-linux@main
6136

62-
- name: Install nvidia driver, nvidia-docker runtime, set GPU_FLAG
63-
uses: pytorch/test-infra/.github/actions/setup-nvidia@main
64-
65-
- name: Calculate/build docker image
66-
id: calculate-docker-image
67-
uses: pytorch/test-infra/.github/actions/calculate-docker-image@main
37+
- name: Configure AWS credentials
38+
uses: aws-actions/configure-aws-credentials@v4
6839
with:
69-
docker-image-name: tutorials
40+
role-to-assume: arn:aws:iam::308535385114:role/arc
41+
aws-region: us-east-1
7042

71-
- name: Pull docker image
72-
uses: pytorch/test-infra/.github/actions/pull-docker-image@main
43+
- name: Login to ECR
44+
uses: aws-actions/amazon-ecr-login@v2
7345
with:
74-
docker-image: ${{ steps.calculate-docker-image.outputs.docker-image }}
46+
registries: "308535385114"
7547

76-
- name: Build
77-
shell: bash
78-
env:
79-
DOCKER_IMAGE: ${{ steps.calculate-docker-image.outputs.docker-image }}
80-
NUM_WORKERS: ${{ matrix.num_shards }}
81-
WORKER_ID: ${{ matrix.shard }}
82-
COMMIT_ID: ${{ github.sha }}
83-
JOB_TYPE: worker
84-
COMMIT_SOURCE: ${{ github.ref }}
85-
USE_NIGHTLY: ${{ inputs.USE_NIGHTLY }}
86-
UPLOAD: ${{ inputs.UPLOAD }}
48+
# Same tag docker-build.yml pushes, built here if it has not landed yet
49+
- name: Resolve the image
50+
id: image
8751
run: |
88-
set -ex
89-
90-
chmod +x ".jenkins/build.sh"
91-
92-
container_name=$(docker run \
93-
${GPU_FLAG:-} \
94-
-e WORKER_ID \
95-
-e NUM_WORKERS \
96-
-e COMMIT_ID \
97-
-e JOB_TYPE \
98-
-e COMMIT_SOURCE \
99-
-e USE_NIGHTLY \
100-
-e UPLOAD \
101-
--env-file="/tmp/github_env_${GITHUB_RUN_ID}" \
102-
--tty \
103-
--detach \
104-
--shm-size=2gb \
105-
--name="${container_name}" \
106-
-v "${GITHUB_WORKSPACE}:/var/lib/workspace" \
107-
-w /var/lib/workspace \
108-
"${DOCKER_IMAGE}"
109-
)
110-
111-
docker exec -u ci-user -t "${container_name}" sh -c ".jenkins/build.sh"
112-
113-
- name: Teardown Linux
114-
uses: pytorch/test-infra/.github/actions/teardown-linux@main
115-
if: always()
52+
set -euo pipefail
53+
git config --global --add safe.directory "${GITHUB_WORKSPACE}"
54+
DOCKER_TAG="$(git rev-parse HEAD:.ci/docker)"
55+
TAG="308535385114.dkr.ecr.us-east-1.amazonaws.com/tutorials/tutorials:${DOCKER_TAG}"
56+
echo "tag=${TAG}" >> "${GITHUB_OUTPUT}"
57+
if docker buildx imagetools inspect "${TAG}" >/dev/null 2>&1; then
58+
echo "exists=true" >> "${GITHUB_OUTPUT}"
59+
else
60+
echo "exists=false" >> "${GITHUB_OUTPUT}"
61+
fi
62+
63+
- name: Build the image
64+
if: steps.image.outputs.exists == 'false'
65+
uses: pytorch/test-infra/.github/actions/docker-build-remote-buildkit@main
66+
with:
67+
command: |
68+
cd .ci/docker
69+
REMOTE_BUILDKIT=1 ./build.sh tutorials -t ${{ steps.image.outputs.tag }}
11670
71+
worker:
72+
name: pytorch_tutorial_build_worker
73+
needs: docker-image
74+
uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main
75+
permissions:
76+
id-token: write
77+
contents: read
78+
strategy:
79+
matrix:
80+
include:
81+
- { shard: 1, num_shards: 15, runner: "mt-l-x86aavx2-45-167-a10g-4" }
82+
- { shard: 2, num_shards: 15, runner: "mt-l-x86aavx2-11-41-a10g" }
83+
- { shard: 3, num_shards: 15, runner: "mt-l-x86aavx2-11-41-a10g" }
84+
- { shard: 4, num_shards: 15, runner: "mt-l-x86aavx2-11-41-a10g" }
85+
- { shard: 5, num_shards: 15, runner: "mt-l-x86aavx2-11-41-a10g" }
86+
- { shard: 6, num_shards: 15, runner: "mt-l-x86aavx2-11-41-a10g" }
87+
- { shard: 7, num_shards: 15, runner: "mt-l-x86aavx2-11-41-a10g" }
88+
- { shard: 8, num_shards: 15, runner: "mt-l-x86aavx2-11-41-a10g" }
89+
- { shard: 9, num_shards: 15, runner: "mt-l-x86aavx2-11-41-a10g" }
90+
- { shard: 10, num_shards: 15, runner: "mt-l-x86aavx2-11-41-a10g" }
91+
- { shard: 11, num_shards: 15, runner: "mt-l-x86aavx2-11-41-a10g" }
92+
- { shard: 12, num_shards: 15, runner: "mt-l-x86aavx2-11-41-a10g" }
93+
- { shard: 13, num_shards: 15, runner: "mt-l-x86aavx2-11-41-a10g" }
94+
- { shard: 14, num_shards: 15, runner: "mt-l-x86aavx2-11-41-a10g" }
95+
- { shard: 15, num_shards: 15, runner: "mt-l-x86aavx2-11-41-a10g" }
96+
fail-fast: false
97+
with:
98+
job-name: pytorch_tutorial_build_worker
99+
runner: ${{ matrix.runner }}
100+
docker-image: ${{ needs.docker-image.outputs.docker-image }}
101+
gpu-arch-type: cuda
102+
fetch-depth: 0
103+
timeout: 120
104+
upload-artifact: ${{ inputs.UPLOAD == 1 && format('tutorials-worker-{0}', matrix.shard) || '' }}
105+
script: |
106+
set -ex
107+
108+
export NUM_WORKERS=${{ matrix.num_shards }}
109+
export WORKER_ID=${{ matrix.shard }}
110+
export COMMIT_ID=${{ github.sha }}
111+
export JOB_TYPE=worker
112+
export COMMIT_SOURCE=${{ github.ref }}
113+
export USE_NIGHTLY=${{ inputs.USE_NIGHTLY }}
114+
export UPLOAD=${{ inputs.UPLOAD }}
115+
116+
.jenkins/build.sh
117+
118+
if [[ "${UPLOAD}" == 1 ]]; then
119+
mv "worker_${WORKER_ID}.7z" "${RUNNER_ARTIFACT_DIR}/"
120+
fi
121+
122+
# Not on linux_job_v3: a job calling a reusable workflow cannot set the environment
123+
# that holds PYTORCHBOT_TOKEN, and the preview goes to a caller-chosen S3 prefix
117124
manager:
118125
name: pytorch_tutorial_build_manager
119-
needs: worker
120-
runs-on: [self-hosted, linux.2xlarge]
126+
needs: [docker-image, worker]
127+
runs-on: mt-l-x86iavx512-8-64
128+
container:
129+
image: ${{ needs.docker-image.outputs.docker-image }}
121130
if: ${{ inputs.UPLOAD == 1 }}
122131
environment: ${{ github.ref == 'refs/heads/main' && 'pytorchbot-env' || '' }}
132+
permissions:
133+
id-token: write
134+
contents: read
135+
timeout-minutes: 120
123136
steps:
124-
- name: Setup SSH (Click me for login details)
125-
uses: pytorch/test-infra/.github/actions/setup-ssh@main
126-
with:
127-
github-secret: ${{ secrets.GITHUB_TOKEN }}
128-
instructions: |
129-
All testing is done inside the container, to start an interactive session run:
130-
docker exec -it $(docker container ps --format '{{.ID}}') bash
137+
# The workspace is created as uid 1001, which the image's ci-user cannot write to
138+
- name: Fix workspace permissions
139+
run: sudo chmod -R 777 "${GITHUB_WORKSPACE}" 2>/dev/null || true
131140

132141
- name: Checkout Tutorials
133142
uses: actions/checkout@v6
134143
with:
135144
fetch-depth: 0
136145

137-
- name: Setup Linux
138-
uses: pytorch/pytorch/.github/actions/setup-linux@main
139-
140-
- name: Calculate/build docker image
141-
id: calculate-docker-image
142-
uses: pytorch/test-infra/.github/actions/calculate-docker-image@main
143-
with:
144-
docker-image-name: tutorials
145-
146-
- name: Pull docker image
147-
uses: pytorch/test-infra/.github/actions/pull-docker-image@main
146+
- name: Download worker archives
147+
uses: actions/download-artifact@v4
148148
with:
149-
docker-image: ${{ steps.calculate-docker-image.outputs.docker-image }}
149+
pattern: tutorials-worker-*
150+
merge-multiple: true
150151

151152
- name: Build
152153
shell: bash
153154
env:
154-
DOCKER_IMAGE: ${{ steps.calculate-docker-image.outputs.docker-image }}
155155
NUM_WORKERS: 15
156156
WORKER_ID: 0
157157
COMMIT_ID: ${{ github.sha }}
@@ -161,39 +161,28 @@ jobs:
161161
USE_NIGHTLY: ${{ inputs.USE_NIGHTLY }}
162162
run: |
163163
set -ex
164-
165-
chmod +x ".jenkins/build.sh"
166-
167-
container_name=$(docker run \
168-
${GPU_FLAG:-} \
169-
-e WORKER_ID \
170-
-e NUM_WORKERS \
171-
-e COMMIT_ID \
172-
-e JOB_TYPE \
173-
-e COMMIT_SOURCE \
174-
-e GITHUB_PYTORCHBOT_TOKEN \
175-
-e USE_NIGHTLY \
176-
--env-file="/tmp/github_env_${GITHUB_RUN_ID}" \
177-
--tty \
178-
--detach \
179-
--name="${container_name}" \
180-
-v "${GITHUB_WORKSPACE}:/var/lib/workspace" \
181-
-w /var/lib/workspace \
182-
"${DOCKER_IMAGE}"
183-
)
184-
185-
docker exec -u ci-user -t "${container_name}" sh -c ".jenkins/build.sh"
164+
.jenkins/build.sh
165+
166+
# The ciflow/docs/<pr> tag still names the pull request, so the preview
167+
# keeps the per-PR prefix. An expression cannot split a ref, hence the step.
168+
- name: Resolve the preview prefix
169+
id: preview
170+
if: ${{ startsWith(github.ref, 'refs/tags/ciflow/docs/') }}
171+
run: echo "prefix=${GITHUB_REF##*/}" >> "${GITHUB_OUTPUT}"
172+
173+
- name: Configure AWS credentials
174+
if: ${{ startsWith(github.ref, 'refs/tags/ciflow/docs/') }}
175+
uses: aws-actions/configure-aws-credentials@v4
176+
with:
177+
role-to-assume: arn:aws:iam::308535385114:role/arc
178+
aws-region: us-east-1
186179

187180
- name: Upload docs preview
188181
uses: seemethere/upload-artifact-s3@v5
189-
if: ${{ github.event_name == 'pull_request' }}
182+
if: ${{ startsWith(github.ref, 'refs/tags/ciflow/docs/') }}
190183
with:
191184
retention-days: 14
192185
s3-bucket: doc-previews
193186
if-no-files-found: error
194187
path: docs
195-
s3-prefix: ${{ inputs.S3_PREFIX }}/${{ github.event.pull_request.number }}
196-
197-
- name: Teardown Linux
198-
uses: pytorch/test-infra/.github/actions/teardown-linux@main
199-
if: always()
188+
s3-prefix: ${{ inputs.S3_PREFIX }}/${{ steps.preview.outputs.prefix }}

‎.github/workflows/build-tutorials-nightly.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,9 @@ concurrency:
2929

3030
jobs:
3131
build:
32+
permissions:
33+
id-token: write
34+
contents: read
3235
uses: ./.github/workflows/_build-tutorials-base.yml
3336
secrets: inherit
3437
with:

‎.github/workflows/build-tutorials.yml‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,23 @@
11
name: Build tutorials
22

33
on:
4-
pull_request:
54
push:
65
branches:
76
- main
7+
# A fork PR gets no OIDC token to build the image or upload the preview, so a
8+
# PR builds through the ciflow/docs label, which pushes this tag from the base repo
9+
tags:
10+
- ciflow/docs/*
811

912
concurrency:
1013
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}-${{ github.event_name == 'schedule' }}
1114
cancel-in-progress: true
1215

1316
jobs:
1417
build:
18+
permissions:
19+
id-token: write
20+
contents: read
1521
uses: ./.github/workflows/_build-tutorials-base.yml
1622
secrets: inherit
1723
with:

‎.github/workflows/docker-build.yml‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,12 @@ name: Docker Build
22

33
on:
44
workflow_dispatch:
5-
pull_request:
6-
paths:
7-
- .ci/docker/**
8-
- .github/workflows/docker-builds.yml
95
push:
106
branches:
117
- main
128
paths:
139
- .ci/docker/**
14-
- .github/workflows/docker-builds.yml
10+
- .github/workflows/docker-build.yml
1511

1612
jobs:
1713
docker-build:

0 commit comments

Comments
 (0)