Skip to content

Commit 2ad3512

Browse files
clee2000clee2000
authored
Build docker image (#2542)
Downloading pip dependencies takes 2-4 min, but downloading the new docker image takes +1min from the pytorch one Not sure how downloading all dependencies to a new image takes in comparison to pulling docker image Docker image scripts are copied from pytorch, I tried to remove stuff but there's definitely a lot left over
1 parent 9ceac50 commit 2ad3512

11 files changed

+344
-118
lines changed

‎.ci/docker/Dockerfile

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
ARG BASE_IMAGE
2+
FROM ${BASE_IMAGE}
3+
4+
ENV DEBIAN_FRONTEND noninteractive
5+
6+
# Install common dependencies (so that this step can be cached separately)
7+
COPY ./common/install_base.sh install_base.sh
8+
RUN bash ./install_base.sh && rm install_base.sh
9+
10+
COPY ./common/install_docs_reqs.sh install_docs_reqs.sh
11+
RUN bash ./install_docs_reqs.sh && rm install_docs_reqs.sh
12+
13+
# Install conda and other packages
14+
ENV ANACONDA_PYTHON_VERSION=3.10
15+
ENV CONDA_CMAKE yes
16+
ENV DOCS yes
17+
ENV PATH /opt/conda/envs/py_$ANACONDA_PYTHON_VERSION/bin:/opt/conda/bin:$PATH
18+
COPY ./requirements.txt /opt/conda/
19+
COPY ./common/install_conda.sh install_conda.sh
20+
COPY ./common/common_utils.sh common_utils.sh
21+
RUN bash ./install_conda.sh && rm install_conda.sh common_utils.sh /opt/conda/requirements.txt
22+
23+
CMD ["bash"]

‎.ci/docker/build.sh

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
set -exu
9+
10+
IMAGE_NAME="$1"
11+
shift
12+
13+
export UBUNTU_VERSION="20.04"
14+
15+
export BASE_IMAGE="ubuntu:${UBUNTU_VERSION}"
16+
echo "Building ${IMAGE_NAME} Docker image"
17+
18+
docker build \
19+
--no-cache \
20+
--progress=plain \
21+
-f Dockerfile \
22+
--build-arg BASE_IMAGE="${BASE_IMAGE}" \
23+
"$@" \
24+
.

‎.ci/docker/common/common_utils.sh

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
#!/bin/bash
2+
3+
# Work around bug where devtoolset replaces sudo and breaks it.
4+
as_ci_user() {
5+
# NB: unsetting the environment variables works around a conda bug
6+
# https://github.com/conda/conda/issues/6576
7+
# NB: Pass on PATH and LD_LIBRARY_PATH to sudo invocation
8+
# NB: This must be run from a directory that the user has access to,
9+
# works around https://github.com/conda/conda-package-handling/pull/34
10+
sudo -E -H env -u SUDO_UID -u SUDO_GID -u SUDO_COMMAND -u SUDO_USER env "PATH=$PATH" "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" $*
11+
}
12+
13+
conda_install() {
14+
# Ensure that the install command don't upgrade/downgrade Python
15+
# This should be called as
16+
# conda_install pkg1 pkg2 ... [-c channel]
17+
as_ci_user conda install -q -n py_$ANACONDA_PYTHON_VERSION -y python="$ANACONDA_PYTHON_VERSION" $*
18+
}
19+
20+
conda_run() {
21+
as_ci_user conda run -n py_$ANACONDA_PYTHON_VERSION --no-capture-output $*
22+
}
23+
24+
pip_install() {
25+
as_ci_user conda run -n py_$ANACONDA_PYTHON_VERSION pip install --progress-bar off $*
26+
}

‎.ci/docker/common/install_base.sh

+47
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
#!/bin/bash
2+
# Based off of https://github.com/pytorch/pytorch/tree/b52e0bf131a4e55cd987176f9c5a8d2ad6783b4f/.ci/docker
3+
4+
set -ex
5+
6+
install_ubuntu() {
7+
# Install common dependencies
8+
apt-get update
9+
# TODO: Some of these may not be necessary
10+
apt-get install -y --no-install-recommends \
11+
build-essential \
12+
ca-certificates \
13+
cmake=3.16* \
14+
curl \
15+
git \
16+
wget \
17+
sudo \
18+
vim \
19+
jq \
20+
vim \
21+
unzip \
22+
gdb \
23+
rsync \
24+
libssl-dev \
25+
p7zip-full \
26+
libglfw3 \
27+
libglfw3-dev \
28+
sox \
29+
libsox-dev \
30+
libsox-fmt-all
31+
32+
# Cleanup package manager
33+
apt-get autoclean && apt-get clean
34+
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
35+
}
36+
37+
# Install base packages depending on the base OS
38+
ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
39+
case "$ID" in
40+
ubuntu)
41+
install_ubuntu
42+
;;
43+
*)
44+
echo "Unable to determine OS..."
45+
exit 1
46+
;;
47+
esac

‎.ci/docker/common/install_conda.sh

+54
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
#!/bin/bash
2+
3+
set -ex
4+
5+
# Optionally install conda
6+
if [ -n "$ANACONDA_PYTHON_VERSION" ]; then
7+
BASE_URL="https://repo.anaconda.com/miniconda"
8+
9+
MAJOR_PYTHON_VERSION=$(echo "$ANACONDA_PYTHON_VERSION" | cut -d . -f 1)
10+
MINOR_PYTHON_VERSION=$(echo "$ANACONDA_PYTHON_VERSION" | cut -d . -f 2)
11+
12+
CONDA_FILE="Miniconda3-latest-Linux-x86_64.sh"
13+
14+
mkdir -p /opt/conda
15+
16+
source "$(dirname "${BASH_SOURCE[0]}")/common_utils.sh"
17+
18+
pushd /tmp
19+
wget -q "${BASE_URL}/${CONDA_FILE}"
20+
# NB: Manually invoke bash per https://github.com/conda/conda/issues/10431
21+
as_ci_user bash "${CONDA_FILE}" -b -f -p "/opt/conda"
22+
popd
23+
24+
# NB: Don't do this, rely on the rpath to get it right
25+
#echo "/opt/conda/lib" > /etc/ld.so.conf.d/conda-python.conf
26+
#ldconfig
27+
sed -e 's|PATH="\(.*\)"|PATH="/opt/conda/bin:\1"|g' -i /etc/environment
28+
export PATH="/opt/conda/bin:$PATH"
29+
30+
# Ensure we run conda in a directory that the user has write access to
31+
pushd /opt/conda
32+
33+
# Prevent conda from updating to 4.14.0, which causes docker build failures
34+
# See https://hud.pytorch.org/pytorch/pytorch/commit/754d7f05b6841e555cea5a4b2c505dd9e0baec1d
35+
# Uncomment the below when resolved to track the latest conda update
36+
# as_ci_user conda update -y -n base conda
37+
38+
# Install correct Python version
39+
as_ci_user conda create -n py_$ANACONDA_PYTHON_VERSION -y python="$ANACONDA_PYTHON_VERSION"
40+
41+
# Use conda cmake in some cases. Conda cmake will be newer than our supported
42+
# min version (3.5 for xenial and 3.10 for bionic), so we only do it in those
43+
# following builds that we know should use conda. Specifically, Ubuntu bionic
44+
# and focal cannot find conda mkl with stock cmake, so we need a cmake from conda
45+
conda_install cmake
46+
47+
# Install pip packages
48+
pip_install -r /opt/conda/requirements.txt
49+
50+
apt-get update
51+
apt-get -y install expect-dev
52+
53+
popd
54+
fi

‎.ci/docker/common/install_docs_reqs.sh

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
#!/bin/bash
2+
# Based off of https://github.com/pytorch/pytorch/tree/b52e0bf131a4e55cd987176f9c5a8d2ad6783b4f/.ci/docker
3+
set -ex
4+
5+
apt-get update
6+
apt-get install -y gpg-agent
7+
8+
curl --retry 3 -sL https://deb.nodesource.com/setup_20.x | sudo -E bash -
9+
sudo apt-get install -y nodejs
10+
11+
curl --retry 3 -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -
12+
echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list
13+
14+
apt-get update
15+
apt-get install -y --no-install-recommends yarn
16+
yarn global add katex --prefix /usr/local
17+
18+
sudo apt-get -y install doxygen
19+
20+
apt-get autoclean && apt-get clean
21+
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*

‎.ci/docker/requirements.txt

+70
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
# --extra-index-url https://download.pytorch.org/whl/cu117/index.html # Use this to run/publish tutorials against the latest binaries during the RC stage. Comment out after the release. Each release verify the correct cuda version.
2+
# Refer to ./jenkins/build.sh for tutorial build instructions
3+
4+
sphinx==5.0.0
5+
sphinx-gallery==0.11.1
6+
sphinx_design
7+
nbsphinx
8+
docutils==0.16
9+
sphinx-copybutton
10+
pypandoc==1.12
11+
pandocfilters
12+
markdown
13+
tqdm==4.66.1
14+
numpy==1.24.4
15+
matplotlib
16+
librosa
17+
torch
18+
torchvision
19+
torchtext
20+
torchdata
21+
networkx
22+
PyHamcrest
23+
bs4
24+
awscliv2==2.1.1
25+
flask
26+
spacy==3.4.1
27+
ray[tune]==2.7.2
28+
tensorboard
29+
jinja2==3.1.3
30+
pytorch-lightning
31+
torchx
32+
torchrl==0.3.0
33+
tensordict==0.3.0
34+
ax-platform
35+
nbformat>==5.9.2
36+
datasets
37+
transformers
38+
torchmultimodal-nightly # needs to be updated to stable as soon as it's avaialable
39+
onnx
40+
onnxscript
41+
onnxruntime
42+
43+
importlib-metadata==6.8.0
44+
45+
# PyTorch Theme
46+
-e git+https://github.com/pytorch/pytorch_sphinx_theme.git#egg=pytorch_sphinx_theme
47+
48+
ipython
49+
50+
sphinxcontrib.katex
51+
# to run examples
52+
boto3
53+
pandas
54+
requests
55+
scikit-image
56+
scipy==1.11.1
57+
numba==0.57.1
58+
pillow==10.2.0
59+
wget
60+
gym==0.26.2
61+
gym-super-mario-bros==7.4.0
62+
pyopengl
63+
gymnasium[mujoco]==0.27.0
64+
timm
65+
iopath
66+
pygame==2.1.2
67+
pycocotools
68+
semilearn==0.3.2
69+
torchao==0.0.3
70+
segment_anything==1.0

‎.github/workflows/build-tutorials.yml

+19-41
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,6 @@ jobs:
3333
- { shard: 15, num_shards: 15, runner: "linux.4xlarge.nvidia.gpu" }
3434
fail-fast: false
3535
runs-on: ${{ matrix.runner }}
36-
env:
37-
DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-focal-cuda12.1-cudnn8-py3-gcc9"
38-
CUDA_VERSION: "9"
3936
steps:
4037
- name: Setup SSH (Click me for login details)
4138
uses: pytorch/test-infra/.github/actions/setup-ssh@main
@@ -54,27 +51,21 @@ jobs:
5451
- name: Install nvidia driver, nvidia-docker runtime, set GPU_FLAG
5552
uses: pytorch/test-infra/.github/actions/setup-nvidia@main
5653

57-
- name: Calculate docker image
58-
shell: bash
59-
id: docker-image
60-
run: |
61-
set -ex
62-
63-
# for some reason, pip installs it in a different place than what is looked at in the py file
64-
pip3 install requests==2.26
65-
pyTorchDockerImageTag=$(python3 .jenkins/get_docker_tag.py)
66-
67-
echo "docker-image=${DOCKER_IMAGE}:${pyTorchDockerImageTag}" >> "${GITHUB_OUTPUT}"
54+
- name: Calculate/build docker image
55+
id: calculate-docker-image
56+
uses: pytorch/test-infra/.github/actions/calculate-docker-image@main
57+
with:
58+
docker-image-name: tutorials
6859

6960
- name: Pull docker image
7061
uses: pytorch/test-infra/.github/actions/pull-docker-image@main
7162
with:
72-
docker-image: ${{ steps.docker-image.outputs.docker-image }}
63+
docker-image: ${{ steps.calculate-docker-image.outputs.docker-image }}
7364

7465
- name: Build
7566
shell: bash
7667
env:
77-
DOCKER_IMAGE: ${{ steps.docker-image.outputs.docker-image }}
68+
DOCKER_IMAGE: ${{ steps.calculate-docker-image.outputs.docker-image }}
7869
NUM_WORKERS: ${{ matrix.num_shards }}
7970
WORKER_ID: ${{ matrix.shard }}
8071
COMMIT_ID: ${{ github.sha }}
@@ -95,16 +86,13 @@ jobs:
9586
--env-file="/tmp/github_env_${GITHUB_RUN_ID}" \
9687
--tty \
9788
--detach \
98-
--user jenkins \
9989
--shm-size=2gb \
10090
--name="${container_name}" \
101-
-v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \
102-
-w /var/lib/jenkins/workspace \
91+
-v "${GITHUB_WORKSPACE}:/var/lib/workspace" \
92+
-w /var/lib/workspace \
10393
"${DOCKER_IMAGE}"
10494
)
10595
106-
echo "rm /opt/cache/bin/*" | docker exec -u root -i "${container_name}" bash
107-
10896
docker exec -t "${container_name}" sh -c ".jenkins/build.sh"
10997
11098
- name: Teardown Linux
@@ -116,9 +104,6 @@ jobs:
116104
needs: worker
117105
runs-on: [self-hosted, linux.2xlarge]
118106
environment: ${{ github.ref == 'refs/heads/main' && 'pytorchbot-env' || '' }}
119-
env:
120-
DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-focal-cuda12.1-cudnn8-py3-gcc9"
121-
CUDA_VERSION: "9"
122107
steps:
123108
- name: Setup SSH (Click me for login details)
124109
uses: pytorch/test-infra/.github/actions/setup-ssh@main
@@ -134,27 +119,21 @@ jobs:
134119
- name: Setup Linux
135120
uses: pytorch/pytorch/.github/actions/setup-linux@main
136121

137-
- name: Calculate docker image
138-
shell: bash
139-
id: docker-image
140-
run: |
141-
set -ex
142-
143-
# for some reason, pip installs it in a different place than what is looked at in the py file
144-
pip3 install requests==2.26
145-
pyTorchDockerImageTag=$(python3 .jenkins/get_docker_tag.py)
146-
147-
echo "docker-image=${DOCKER_IMAGE}:${pyTorchDockerImageTag}" >> "${GITHUB_OUTPUT}"
122+
- name: Calculate/build docker image
123+
id: calculate-docker-image
124+
uses: pytorch/test-infra/.github/actions/calculate-docker-image@main
125+
with:
126+
docker-image-name: tutorials
148127

149128
- name: Pull docker image
150129
uses: pytorch/test-infra/.github/actions/pull-docker-image@main
151130
with:
152-
docker-image: ${{ steps.docker-image.outputs.docker-image }}
131+
docker-image: ${{ steps.calculate-docker-image.outputs.docker-image }}
153132

154133
- name: Build
155134
shell: bash
156135
env:
157-
DOCKER_IMAGE: ${{ steps.docker-image.outputs.docker-image }}
136+
DOCKER_IMAGE: ${{ steps.calculate-docker-image.outputs.docker-image }}
158137
NUM_WORKERS: 15
159138
WORKER_ID: ${{ matrix.shard }}
160139
COMMIT_ID: ${{ github.sha }}
@@ -177,14 +156,13 @@ jobs:
177156
--env-file="/tmp/github_env_${GITHUB_RUN_ID}" \
178157
--tty \
179158
--detach \
180-
--user jenkins \
181159
--name="${container_name}" \
182-
-v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \
183-
-w /var/lib/jenkins/workspace \
160+
-v "${GITHUB_WORKSPACE}:/var/lib/workspace" \
161+
-w /var/lib/workspace \
184162
"${DOCKER_IMAGE}"
185163
)
186164
187-
echo "rm /opt/cache/bin/*" | docker exec -u root -i "${container_name}" bash
165+
docker exec -u root -i "${container_name}" bash
188166
189167
docker exec -t "${container_name}" sh -c ".jenkins/build.sh"
190168

0 commit comments

Comments
 (0)