Skip to content

Commit f11bcae

Browse files
RuoqingHeRuoqingHe
committed
Introduce riscv64 CI container
Add build scripts for v6.10 riscv64 kernel, qemu-system-riscv64 and opensbi required to boot qemu-system inside docker container. With this approach, we are able to run tests inside qemu-system, while preserving the original output as much as possbile with ssh. This work was inspired by the work done by @endeneer in PR rust-vmm#91, and is the third draft proceeds rust-vmm#101, rust-vmm#104. It is expected to be replaced by the second draft rust-vmm#104 in the future which standardise `riscv64`. Signed-off-by: Ruoqing He <[email protected]>
1 parent 2e2e495 commit f11bcae

9 files changed

+318
-46
lines changed

‎.github/workflows/docker-publish.yml

+69-4
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,20 @@ name: Docker
33
on:
44
push:
55
branches: [ "main" ]
6-
paths: [Dockerfile, .github/workflows/docker-publish.yml, build_container.sh]
6+
paths:
7+
- Dockerfile
8+
- .github/workflows/docker-publish.yml
9+
- build_container.sh
10+
- Dockerfile.riscv64
11+
- riscv64/*
712
pull_request:
813
branches: [ "main" ]
9-
paths: [Dockerfile, .github/workflows/docker-publish.yml, build_container.sh]
14+
paths:
15+
- Dockerfile
16+
- .github/workflows/docker-publish.yml
17+
- build_container.sh
18+
- Dockerfile.riscv64
19+
- riscv64/*
1020

1121
jobs:
1222
build:
@@ -69,7 +79,6 @@ jobs:
6979
cache-from: type=gha
7080
cache-to: type=gha,mode=max
7181

72-
7382
# Sign the resulting Docker image digest except on PRs.
7483
# This will only write to the public Rekor transparency log when the Docker
7584
# repository is public to avoid leaking data. If you would like to publish
@@ -81,4 +90,60 @@ jobs:
8190
COSIGN_EXPERIMENTAL: "true"
8291
# This step uses the identity token to provision an ephemeral certificate
8392
# against the sigstore community Fulcio instance.
84-
run: echo "${{ steps.meta.outputs.tags }}" | xargs -I {} cosign sign {}@${{ steps.build-and-push.outputs.digest }}
93+
run: |
94+
echo "${{ env.VERSION }}" | xargs -I {} cosign sign {}@${{ steps.build-and-push.outputs.digest }}
95+
96+
build-riscv64:
97+
98+
runs-on: ubuntu-latest
99+
permissions:
100+
contents: read
101+
packages: write
102+
id-token: write
103+
104+
steps:
105+
- name: Checkout repository
106+
uses: actions/checkout@v3
107+
108+
- name: Install cosign
109+
if: github.event_name != 'pull_request'
110+
uses: sigstore/[email protected]
111+
- name: Check install!
112+
if: github.event_name != 'pull_request'
113+
run: cosign version
114+
115+
- name: Setup Docker buildx
116+
uses: docker/setup-buildx-action@79abd3f86f79a9d68a23c75a09a9a85889262adf
117+
118+
- name: Log into registry ${{ env.REGISTRY }}
119+
if: github.event_name != 'pull_request'
120+
uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c
121+
with:
122+
username: ${{ secrets.DOCKER_ACCOUNT_ID }}
123+
password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
124+
125+
- name: Generate next docker tag
126+
run: |
127+
NEXT_VERSION=$(./docker.sh print-next-version)
128+
echo "VERSION=${NEXT_VERSION}" >> $GITHUB_ENV
129+
echo "Next version to be published is: ${NEXT_VERSION}"
130+
131+
- name: Build and push Docker image for riscv64
132+
id: build-and-push-riscv64
133+
uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a
134+
with:
135+
context: .
136+
file: Dockerfile.riscv64
137+
push: ${{ github.event_name != 'pull_request' }}
138+
platforms: linux/amd64
139+
tags: ${{ env.VERSION }}-riscv
140+
labels: ${{ steps.meta.outputs.labels }}
141+
cache-from: type=gha
142+
cache-to: type=gha,mode=max
143+
144+
- name: Sign the published Docker image
145+
if: ${{ github.event_name != 'pull_request' }}
146+
env:
147+
COSIGN_EXPERIMENTAL: "true"
148+
run: |
149+
echo "${{ env.VERSION }}-riscv" | xargs -I {} cosign sign {}@${{ steps.build-and-push-riscv.outputs.digest }}

‎Dockerfile.riscv64

+55
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
# Compile QEMU 9.0.2
2+
# ---------------------------------------------------------
3+
FROM ubuntu:22.04 AS qemu_builder
4+
5+
COPY riscv64/build_qemu_system_riscv64.sh /opt/src/scripts/build.sh
6+
RUN /opt/src/scripts/build.sh
7+
8+
# Compile kernel 6.10 since we need AIA drivers
9+
# ---------------------------------------------------------
10+
FROM ubuntu:22.04 AS kernel_builder
11+
12+
COPY riscv64/build_kernel.sh /opt/src/scripts/build.sh
13+
RUN /opt/src/scripts/build.sh
14+
15+
# Compile OpenSBI
16+
# ---------------------------------------------------------
17+
FROM ubuntu:22.04 AS opensbi_builder
18+
19+
COPY riscv64/build_opensbi.sh /opt/src/scripts/build.sh
20+
RUN /opt/src/scripts/build.sh
21+
22+
# Build rootfs with sshd and Rust related packages ready
23+
# ---------------------------------------------------------
24+
FROM --platform=linux/riscv64 riscv64/ubuntu:22.04 AS rootfs_builder
25+
26+
ARG RUST_TOOLCHAIN="1.75.0"
27+
ENV PATH="$PATH:/root/.cargo/bin"
28+
COPY build_container.sh /opt/src/scripts/build.sh
29+
RUN /opt/src/scripts/build.sh
30+
31+
# Finalize
32+
# ---------------------------------------------------------
33+
FROM ubuntu:22.04 AS final
34+
35+
ARG OUTPUT=/output
36+
ARG QEMU_DIR=/opt/qemu
37+
ARG KERNEL_DIR=/opt/kernel
38+
ARG OPENSBI_DIR=/opt/opensbi
39+
ARG ROOTFS_DIR=/opt/rootfs
40+
41+
COPY --from=qemu_builder $OUTPUT $QEMU_DIR
42+
COPY --from=kernel_builder $OUTPUT $KERNEL_DIR
43+
COPY --from=opensbi_builder $OUTPUT $OPENSBI_DIR
44+
COPY --from=rootfs_builder / $ROOTFS_DIR
45+
46+
COPY riscv64/build_finalize.sh /opt/src/scripts/finalize.sh
47+
RUN /opt/src/scripts/finalize.sh
48+
49+
ENV QEMU_DIR=$QEMU_DIR KERNEL_DIR=$KERNEL_DIR \
50+
OPENSBI_DIR=$OPENSBI_DIR ROOTFS_DIR=$ROOTFS_DIR \
51+
WORKDIR=/workdir
52+
53+
# Start qemu-system-riscv64 as a background process
54+
COPY riscv64/start_in_qemu.sh /opt/src/scripts/start.sh
55+
ENTRYPOINT ["/opt/src/scripts/start.sh"]

‎build_container.sh

+71-41
Original file line numberDiff line numberDiff line change
@@ -22,15 +22,24 @@ DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \
2222
libvirglrenderer-dev libvirglrenderer1 \
2323
debhelper-compat libdbus-1-dev libglib2.0-dev meson ninja-build dbus
2424

25+
# `riscv64` specific
26+
if [ "$ARCH" == "riscv64" ]; then
27+
DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \
28+
openssh-server systemd init ifupdown busybox udev isc-dhcp-client
29+
fi
30+
2531
# cleanup
2632
apt-get clean && rm -rf /var/lib/apt/lists/*
2733

2834
# help musl-gcc find linux headers
29-
pushd /usr/include/$ARCH-linux-musl
30-
ln -s ../$ARCH-linux-gnu/asm asm
31-
ln -s ../linux linux
32-
ln -s ../asm-generic asm-generic
33-
popd
35+
# Skip on `riscv64` for now
36+
if [ "$ARCH" != "riscv64" ]; then
37+
pushd /usr/include/$ARCH-linux-musl
38+
ln -s ../$ARCH-linux-gnu/asm asm
39+
ln -s ../linux linux
40+
ln -s ../asm-generic asm-generic
41+
popd
42+
fi
3443

3544
pip3 install --no-cache-dir pytest pexpect boto3 pytest-timeout && apt purge -y python3-pip
3645

@@ -52,46 +61,67 @@ rustup component add miri rust-src --toolchain nightly
5261
rustup component add llvm-tools-preview # needed for coverage
5362

5463
# Install other rust targets.
55-
rustup target add $ARCH-unknown-linux-musl $ARCH-unknown-none
64+
# Skip on `riscv64` for now
65+
if [ "$ARCH" != "riscv64" ]; then
66+
rustup target add $ARCH-unknown-linux-musl $ARCH-unknown-none
67+
fi
5668

5769
cargo install cargo-llvm-cov
5870

5971
# Install aemu, gfxstream, libgpiod and libpipewire (required by vhost-device crate)
60-
pushd /opt
61-
git clone https://android.googlesource.com/platform/hardware/google/aemu
62-
pushd aemu
63-
git checkout v0.1.2-aemu-release
64-
cmake -DAEMU_COMMON_GEN_PKGCONFIG=ON \
65-
-DAEMU_COMMON_BUILD_CONFIG=gfxstream \
66-
-DENABLE_VKCEREAL_TESTS=OFF -B build
67-
cmake --build build -j
68-
cmake --install build
69-
popd
70-
rm -rf aemu
71-
git clone https://android.googlesource.com/platform/hardware/google/gfxstream
72-
pushd gfxstream
73-
git checkout v0.1.2-gfxstream-release
74-
meson setup host-build/
75-
meson install -C host-build/
76-
popd
77-
rm -rf gfxstream
78-
git clone --depth 1 --branch v2.0 https://git.kernel.org/pub/scm/libs/libgpiod/libgpiod.git/
79-
pushd libgpiod
80-
./autogen.sh --prefix=/usr && make && make install
81-
popd
82-
rm -rf libgpiod
83-
wget https://gitlab.freedesktop.org/pipewire/pipewire/-/archive/0.3.71/pipewire-0.3.71.tar.gz
84-
tar xzvf pipewire-0.3.71.tar.gz
85-
pushd pipewire-0.3.71
86-
meson setup builddir --prefix="/usr" -Dbuildtype=release \
87-
-Dauto_features=disabled -Ddocs=disabled -Dtests=disabled \
88-
-Dexamples=disabled -Dinstalled_tests=disabled -Dsession-managers=[] && \
89-
meson compile -C builddir && \
90-
meson install -C builddir
91-
popd
92-
rm -rf pipewire-0.3.71
93-
rm pipewire-0.3.71.tar.gz
94-
popd
72+
# `aemu` has yet supported `riscv64`, skipping `vhost-device` related
73+
# dependencies for `riscv64` at the time being
74+
if [ "$ARCH" != "riscv64" ]; then
75+
pushd /opt
76+
git clone https://android.googlesource.com/platform/hardware/google/aemu
77+
pushd aemu
78+
git checkout v0.1.2-aemu-release
79+
cmake -DAEMU_COMMON_GEN_PKGCONFIG=ON \
80+
-DAEMU_COMMON_BUILD_CONFIG=gfxstream \
81+
-DENABLE_VKCEREAL_TESTS=OFF -B build
82+
cmake --build build -j
83+
cmake --install build
84+
popd
85+
rm -rf aemu
86+
git clone https://android.googlesource.com/platform/hardware/google/gfxstream
87+
pushd gfxstream
88+
git checkout v0.1.2-gfxstream-release
89+
meson setup host-build/
90+
meson install -C host-build/
91+
popd
92+
rm -rf gfxstream
93+
git clone --depth 1 --branch v2.0 https://git.kernel.org/pub/scm/libs/libgpiod/libgpiod.git/
94+
pushd libgpiod
95+
./autogen.sh --prefix=/usr && make && make install
96+
popd
97+
rm -rf libgpiod
98+
wget https://gitlab.freedesktop.org/pipewire/pipewire/-/archive/0.3.71/pipewire-0.3.71.tar.gz
99+
tar xzvf pipewire-0.3.71.tar.gz
100+
pushd pipewire-0.3.71
101+
meson setup builddir --prefix="/usr" -Dbuildtype=release \
102+
-Dauto_features=disabled -Ddocs=disabled -Dtests=disabled \
103+
-Dexamples=disabled -Dinstalled_tests=disabled -Dsession-managers=[] && \
104+
meson compile -C builddir && \
105+
meson install -C builddir
106+
popd
107+
rm -rf pipewire-0.3.71
108+
rm pipewire-0.3.71.tar.gz
109+
popd
110+
fi
95111

96112
# dbus-daemon expects this folder
97113
mkdir /run/dbus
114+
115+
# `riscv64` specific
116+
if [ "$ARCH" == "riscv64" ]; then
117+
# Set passwd for debugging
118+
echo 'root:rustvmm' | chpasswd
119+
# Allow root login
120+
sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/g' /etc/ssh/sshd_config
121+
sed -i 's/#PermitUserEnvironment no/PermitUserEnvironment yes/g' /etc/ssh/sshd_config
122+
# Enable ssh
123+
systemctl enable ssh
124+
mkdir -p /root/.ssh
125+
# Setup network
126+
echo $'auto lo\niface lo inet loopback\n\nauto eth0\niface eth0 inet dhcp\n' > /etc/network/interfaces
127+
fi

‎docker.sh

+2-1
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,12 @@ DOCKER_TAG=rustvmm/dev
77

88
# Get the latest published version. Returns a number.
99
# If latest is v100, returns 100.
10+
# If latest for riscv64 is v100-riscv, returns 100.
1011
# This works as long as we have less than 100 tags because we set the page size to 100,
1112
# once we have more than that this script needs to be updated.
1213
latest(){
1314
curl -L -s 'https://registry.hub.docker.com/v2/repositories/rustvmm/dev/tags?page_size=100'| \
14-
jq '."results"[]["name"]' | sed 's/"//g' | cut -c 2- | grep -E "^[0-9]+$" | sort -n | tail -1
15+
jq '."results"[]["name"]' | sed 's/"//g' | cut -c 2- | grep -E "^[0-9]+" | sort -n | tail -1
1516
}
1617

1718
next_version() {

‎riscv64/build_finalize.sh

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/usr/bin/env bash
2+
set -ex
3+
4+
apt-get update
5+
6+
DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \
7+
openssh-client libslirp-dev libfdt-dev libglib2.0-dev libssl-dev \
8+
libpixman-1-dev netcat
9+
10+
# Setup container ssh config
11+
yes "" | ssh-keygen -P ""
12+
cat /root/.ssh/id_rsa.pub > $ROOTFS_DIR/root/.ssh/authorized_keys
13+
cat > /root/.ssh/config << EOF
14+
Host riscv-qemu
15+
HostName localhost
16+
User root
17+
Port 2222
18+
StrictHostKeyChecking no
19+
EOF
20+
21+
# Set `nameserver` for `resolv.conf`
22+
echo 'nameserver 8.8.8.8' > $ROOTFS_DIR/etc/resolv.conf

‎riscv64/build_kernel.sh

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/usr/bin/env bash
2+
set -ex
3+
4+
apt-get update
5+
6+
KERNEL_TAG=v6.10
7+
OUTPUT=/output
8+
mkdir $OUTPUT
9+
10+
DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \
11+
git python3 python3-pip ninja-build build-essential pkg-config curl bc jq \
12+
libslirp-dev libfdt-dev libglib2.0-dev libssl-dev libpixman-1-dev \
13+
flex bison gcc-riscv64-linux-gnu
14+
15+
git clone --depth 1 --branch $KERNEL_TAG https://github.com/torvalds/linux.git
16+
pushd linux
17+
# Enable kvm module instead of inserting manually
18+
sed -i "s|^CONFIG_KVM=.*|CONFIG_KVM=y|g" arch/riscv/configs/defconfig
19+
make ARCH=riscv CROSS_COMPILE=riscv64-linux-gnu- defconfig && \
20+
make ARCH=riscv CROSS_COMPILE=riscv64-linux-gnu- -j$(nproc)
21+
mv arch/riscv/boot/Image $OUTPUT
22+
popd

‎riscv64/build_opensbi.sh

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/usr/bin/env bash
2+
set -ex
3+
4+
apt-get update
5+
6+
OPENSBI_TAG=v1.3.1
7+
OUTPUT=/output
8+
mkdir $OUTPUT
9+
10+
DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \
11+
git python3 python3-pip ninja-build build-essential pkg-config curl bc jq \
12+
libslirp-dev libfdt-dev libglib2.0-dev libssl-dev libpixman-1-dev \
13+
flex bison gcc-riscv64-linux-gnu
14+
15+
git clone --depth 1 --branch $OPENSBI_TAG https://github.com/riscv-software-src/opensbi.git
16+
pushd opensbi
17+
make -j$(nproc) PLATFORM=generic CROSS_COMPILE=riscv64-linux-gnu-
18+
mv build/platform/generic/firmware/fw_jump.elf $OUTPUT
19+
popd

‎riscv64/build_qemu_system_riscv64.sh

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/usr/bin/env bash
2+
set -ex
3+
4+
apt-get update
5+
6+
QEMU_TAG=v9.0.2
7+
OUTPUT=/output
8+
mkdir $OUTPUT
9+
10+
DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \
11+
git python3 python3-pip ninja-build build-essential pkg-config curl bc jq \
12+
libslirp-dev libfdt-dev libglib2.0-dev libssl-dev libpixman-1-dev \
13+
flex bison
14+
15+
git clone --depth 1 --branch $QEMU_TAG https://gitlab.com/qemu-project/qemu.git
16+
pushd qemu
17+
./configure --target-list=riscv64-softmmu --prefix=$OUTPUT && \
18+
make -j$(nproc) && make install
19+
popd

0 commit comments

Comments
 (0)