Skip to content

Commit 95944c4

Browse files
author
Alexander Eldeib
committed
add initial implementation
1 parent 5235e6a commit 95944c4

10 files changed

Lines changed: 311 additions & 8 deletions

File tree

‎.github/workflows/main.yaml‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
name: Building and Pushing to MCR
2+
on: [workflow_dispatch]
3+
# on:
4+
# push:
5+
# branches:
6+
# - main
7+
# workflow_dispatch: {}
8+
permissions:
9+
id-token: write
10+
contents: read
11+
12+
jobs:
13+
publish:
14+
runs-on: ubuntu-latest
15+
steps:
16+
- uses: actions/checkout@v2
17+
- uses: paulhatch/semantic-version@v5.0.0-alpha2
18+
with:
19+
bump_each_commit: true
20+
id: semver
21+
- name: 'Check version'
22+
run: |
23+
echo "version is ${{ steps.semver.output.version }}"
24+
- name: 'Az CLI login'
25+
uses: azure/login@v1
26+
with:
27+
client-id: ${{ secrets.AZURE_CLIENT_ID }}
28+
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
29+
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
30+
- name: 'Run Azure CLI commands'
31+
run: |
32+
docker build -f Dockerfile -t ${{ secrets.AZURE_REGISTRY_SERVER }}/public/aks/aks-gpu:${{ steps.semver.outputs.version }} .
33+
az acr login -n ${{ secrets.AZURE_REGISTRY_SERVER }}
34+
# docker push ${{ secrets.AZURE_REGISTRY_SERVER }}/public/aks/aks-gpu:${{ steps.semver.outputs.version }}
35+
# echo "acr push done"

‎Dockerfile‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
FROM ubuntu:18.04 as gpu
2+
RUN apt update && apt install -y curl xz-utils gnupg2 ca-certificates --no-install-recommends
3+
4+
WORKDIR /opt/gpu
5+
COPY blacklist-nouveau.conf blacklist-nouveau.conf
6+
COPY config.sh config.sh
7+
COPY download.sh download.sh
8+
RUN bash download.sh
9+
10+
FROM ubuntu:18.04
11+
12+
COPY --from=gpu /opt/gpu /opt/gpu
13+
COPY entrypoint.sh /entrypoint.sh
14+
COPY install.sh /opt/actions/install.sh
15+
16+
RUN mkdir -p /mnt
17+
18+
ENTRYPOINT ["/entrypoint.sh"]

‎README.md‎

Lines changed: 24 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,30 @@
1-
# Project
1+
# Driver container image for AKS VHD
22

3-
> This repo has been populated by an initial template to help get you started. Please
4-
> make sure to update the content to build a great experience for community-building.
3+
This repo provides steps to build a container image with all components required for
4+
Kubernetes Nvidia GPU integration. Run it as a privileged container in the host PID namespace.
5+
It will enter the host mount namespace and install the nvidia drivers, container runtime,
6+
and associated libraries on the host, validating their functionality
57

6-
As the maintainer of this project, please make a few updates:
8+
## Build
9+
```
10+
docker build -f Dockerfile -t docker.io/alexeldeib/aks-gpu:latest .
11+
docker push docker.io/alexeldeib/aks-gpu:latest
12+
```
13+
14+
## Run
15+
```bash
16+
mkdir -p /opt/{actions,gpu}
17+
ctr image pull docker.io/alexeldeib/aks-gpu:latest
18+
ctr run --privileged --net-host --with-ns pid:/proc/1/ns/pid --mount type=bind,src=/opt/gpu,dst=/mnt/gpu,options=rbind --mount type=bind,src=/opt/actions,dst=/mnt/actions,options=rbind -t docker.io/alexeldeib/aks-gpu:latest /entrypoint.sh install.sh
19+
```
20+
21+
or Docker (untested...)
22+
```bash
23+
docker run --privileged -v /opt/gpu:/mnt/gpu -v /opt/actions:/mnt/actions docker.io/alexeldeib/aks-gpu:latest
24+
```
25+
26+
Note the `--with-ns pid:/proc/1/ns/pid` and `--privileged`, as well as the bind mounts, these are key.
727

8-
- Improving this README.MD file to provide a great experience
9-
- Updating SUPPORT.MD with content about this project's support experience
10-
- Understanding the security reporting process in SECURITY.MD
11-
- Remove this section from the README
1228

1329
## Contributing
1430

‎blacklist-nouveau.conf‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
blacklist nouveau
2+
options nouveau modeset=0

‎config.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
DRIVER_VERSION="470.57.02"
2+
NVIDIA_CONTAINER_RUNTIME_VERSION="3.6.0"
3+
NVIDIA_CONTAINER_TOOLKIT_VER="1.6.0"
4+
NVIDIA_PACKAGES="libnvidia-container1 libnvidia-container-tools nvidia-container-toolkit"
5+
GPU_DEST="/usr/local/nvidia"

‎download.sh‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
source /etc/os-release
5+
source /opt/gpu/config.sh
6+
7+
NVIDIA_CONTAINER_RUNTIME_VERSION="3.6.0"
8+
NVIDIA_CONTAINER_TOOLKIT_VER="1.6.0"
9+
NVIDIA_PACKAGES="libnvidia-container1 libnvidia-container-tools nvidia-container-toolkit"
10+
GPU_DEST="/usr/local/nvidia"
11+
12+
workdir="$(mktemp -d)"
13+
pushd "$workdir" || exit
14+
15+
# download nvidia drivers, move to permanent cache
16+
curl -fsSLO https://us.download.nvidia.com/tesla/${DRIVER_VERSION}/NVIDIA-Linux-x86_64-${DRIVER_VERSION}.run
17+
mv NVIDIA-Linux-x86_64-${DRIVER_VERSION}.run /opt/gpu/NVIDIA-Linux-x86_64-${DRIVER_VERSION}.run
18+
19+
# download fabricmanager for nvlink based systems, e.g. multi instance gpu vms.
20+
curl -fsSLO https://developer.download.nvidia.com/compute/cuda/redist/fabricmanager/linux-x86_64/fabricmanager-linux-x86_64-${DRIVER_VERSION}-archive.tar.xz
21+
tar -xvf fabricmanager-linux-x86_64-${DRIVER_VERSION}-archive.tar.xz
22+
mv fabricmanager-linux-x86_64-${DRIVER_VERSION}-archive /opt/gpu/fabricmanager-linux-x86_64-${DRIVER_VERSION}
23+
24+
# configure nvidia apt repo to cache packages
25+
curl -fsSLO https://nvidia.github.io/nvidia-docker/gpgkey
26+
gpg --dearmor -o aptnvidia.gpg gpgkey
27+
mv aptnvidia.gpg /etc/apt/trusted.gpg.d/aptnvidia.gpg
28+
curl -fsSL https://nvidia.github.io/nvidia-docker/ubuntu${VERSION_ID}/nvidia-docker.list -o /etc/apt/sources.list.d/nvidia-docker.list
29+
30+
apt update
31+
32+
# download nvidia debian packages for nvidia-container-runtime compat
33+
for apt_package in $NVIDIA_PACKAGES; do
34+
apt-get download ${apt_package}=${NVIDIA_CONTAINER_TOOLKIT_VER}*
35+
mv ${apt_package}_${NVIDIA_CONTAINER_TOOLKIT_VER}* /opt/gpu
36+
done
37+
apt-get download nvidia-container-runtime=${NVIDIA_CONTAINER_RUNTIME_VERSION}*
38+
39+
# move debs to permanent cache
40+
mv nvidia-container-runtime_${NVIDIA_CONTAINER_RUNTIME_VERSION}* /opt/gpu
41+
42+
popd || exit
43+
rm -r "$workdir"

‎entrypoint.sh‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
#!/usr/bin/env bash
2+
set -o errexit
3+
set -o pipefail
4+
set -o nounset
5+
6+
set -x
7+
8+
if [[ -z "${1}" ]]; then
9+
echo "Must provide a non-empty action as first argument"
10+
exit 1
11+
fi
12+
13+
if [[ "${1}" == "copy" ]]; then
14+
echo "copying gpu cache files and exiting"
15+
cp -a /opt/gpu/. /mnt/gpu/
16+
echo "Completed successfully!"
17+
exit 0
18+
fi
19+
20+
ACTION_FILE="/opt/actions/${1}"
21+
22+
if [[ ! -f "$ACTION_FILE" ]]; then
23+
echo "Expected to find action file '$ACTION_FILE', but did not exist"
24+
exit 1
25+
fi
26+
27+
echo "Cleaning up stale actions"
28+
29+
rm -rf /mnt/actions/*
30+
31+
echo "Copying fresh actions"
32+
33+
cp -R /opt/actions/. /mnt/actions
34+
35+
echo "Executing nsenter"
36+
37+
cp -a /opt/gpu/. /mnt/gpu/
38+
nsenter -t 1 -m bash "${ACTION_FILE}"
39+
RESULT="${PIPESTATUS[0]}"
40+
41+
if [ $RESULT -eq 0 ]; then
42+
# Success.
43+
rm -rf /mnt/actions/*
44+
echo "Completed successfully!"
45+
else
46+
echo "Failed during nsenter command execution"
47+
exit 1
48+
fi

‎install.sh‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
#!/usr/bin/env bash
2+
set -euxo pipefail
3+
4+
source /opt/gpu/config.sh
5+
6+
KERNEL_NAME=$(uname -r)
7+
LOG_FILE_NAME="/var/log/nvidia-installer-$(date +%s).log"
8+
9+
# host needs these tools to build and load kernel module, can remove ca-certificates, was only for testing
10+
apt update && apt install -y kmod gcc make dkms initramfs-tools linux-headers-$(uname -r) ca-certificates --no-install-recommends
11+
12+
# install cached nvidia debian packages for container runtime compatibility
13+
for apt_package in $NVIDIA_PACKAGES; do
14+
dpkg -i /opt/gpu/${apt_package}*
15+
done
16+
dpkg -i /opt/gpu/nvidia-container-runtime*
17+
18+
# blacklist nouveau driver, nvidia driver dependency
19+
cp /opt/gpu/blacklist-nouveau.conf /etc/modprobe.d/blacklist-nouveau.conf
20+
update-initramfs -u
21+
22+
# clean up lingering files from previous install
23+
set +e
24+
umount -l /usr/lib/x86_64-linux-gnu || true
25+
umount -l /tmp/overlay || true
26+
rm -r /tmp/overlay
27+
rm -r /opt/gpu/NVIDIA-Linux-x86_64-${DRIVER_VERSION}
28+
set -e
29+
30+
# set up overlayfs to change install location of nvidia libs from /usr/lib/x86_64-linux-gnu to /usr/local/nvidia
31+
# add an extra layer of indirection via tmpfs because it's not possible to have an overlayfs on an overlayfs (i.e., inside a container)
32+
mkdir /tmp/overlay
33+
mount -t tmpfs tmpfs /tmp/overlay
34+
mkdir /tmp/overlay/{workdir,lib64}
35+
mkdir -p ${GPU_DEST}/lib64
36+
mount -t overlay overlay -o lowerdir=/usr/lib/x86_64-linux-gnu,upperdir=/tmp/overlay/lib64,workdir=/tmp/overlay/workdir /usr/lib/x86_64-linux-gnu
37+
38+
# clean up previously uncompressed driver, if it exists
39+
# causes driver installer to fail if it exists
40+
pushd /opt/gpu
41+
# extract runfile, takes some time, so do ahead of time
42+
sh /opt/gpu/NVIDIA-Linux-x86_64-${DRIVER_VERSION}.run -x
43+
popd
44+
45+
# install nvidia drivers
46+
/opt/gpu/NVIDIA-Linux-x86_64-${DRIVER_VERSION}/nvidia-installer -s -k=$KERNEL_NAME --log-file-name=${LOG_FILE_NAME} -a --no-drm --dkms --utility-prefix="${GPU_DEST}" --opengl-prefix="${GPU_DEST}"
47+
48+
# move nvidia libs to correct location from temporary overlayfs
49+
cp -a /tmp/overlay/lib64 ${GPU_DEST}/lib64
50+
51+
# move nvidia binaries to /usr/bin...because we like that?
52+
cp -rvT ${GPU_DEST}/bin /usr/bin
53+
54+
# configure system to know about nvidia lib paths
55+
echo "${GPU_DEST}/lib64" > /etc/ld.so.conf.d/nvidia.conf
56+
ldconfig
57+
58+
# unmount, cleanup
59+
set +e
60+
umount -l /usr/lib/x86_64-linux-gnu
61+
umount /tmp/overlay
62+
rm -r /tmp/overlay
63+
set -e
64+
65+
# validate that nvidia driver is working
66+
dkms status
67+
nvidia-modprobe -u -c0
68+
nvidia-smi

‎justfile‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
default: push
2+
3+
push: (containerize)
4+
docker push docker.io/alexeldeib/aks-gpu:latest
5+
6+
containerize:
7+
docker build -f Dockerfile -t docker.io/alexeldeib/aks-gpu:latest .

‎manifests.yaml‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
apiVersion: apps/v1
2+
kind: DaemonSet
3+
metadata:
4+
name: &name nsenter
5+
labels:
6+
app: *name
7+
spec:
8+
selector:
9+
matchLabels:
10+
app: *name
11+
template:
12+
metadata:
13+
labels:
14+
app: *name
15+
spec:
16+
# affinity:
17+
# nodeAffinity:
18+
# requiredDuringSchedulingIgnoredDuringExecution:
19+
# nodeSelectorTerms:
20+
# - matchExpressions:
21+
# - key: node.kubernetes.io/instance-type
22+
# operator: In
23+
# values:
24+
# - Standard_NP10s
25+
# - Standard_NP20s
26+
# - Standard_NP40s
27+
hostNetwork: true
28+
hostPID: true
29+
containers:
30+
- image: docker.io/alexeldeib/aks-gpu:latest # requires an image with bash, curl, sleep, and nsenter (vanilla ubuntu works)
31+
imagePullPolicy: Always
32+
name: *name
33+
command: ["/entrypoint.sh"]
34+
args: ["install.sh"] # if you don't use my image or build one from Dockerfile, set this to "downloadandinstall"
35+
resources:
36+
requests:
37+
{}
38+
limits:
39+
{}
40+
securityContext:
41+
privileged: true
42+
volumeMounts:
43+
- name: actions
44+
mountPath: "/opt/actions"
45+
- name: hostmount
46+
mountPath: "/mnt/actions"
47+
- name: gpu
48+
mountPath: "/mnt/gpu"
49+
volumes:
50+
- name: gpu
51+
hostPath:
52+
path: /opt/gpu
53+
type: DirectoryOrCreate
54+
- name: hostmount
55+
hostPath:
56+
path: /opt/actions
57+
type: DirectoryOrCreate
58+
- name: actions
59+
configMap:
60+
name: nsenter-actions
61+
---

0 commit comments

Comments
 (0)