Skip to content
Nightly Build & Test

Nightly Build & Test #24

Sign in to view logs

Nightly Build & Test

Nightly Build & Test #24

Workflow file for this run

.github/workflows/nightly.yml at ac865da
name: Nightly Build & Test
on:
schedule:
- cron: '0 16 * * *' # 00:00 Beijing Time (UTC+8)
workflow_dispatch:
inputs:
skip_publish:
description: 'Skip publishing to TestPyPI'
type: boolean
default: false
permissions:
contents: read
concurrency:
group: nightly-${{ github.ref_name }}
cancel-in-progress: true
# ── Workflow Design Notes ──────────────────────────────────────────────────────
# 1. version-stamp computes a forward-looking nightly version (next patch + .devYYYYMMDD)
# that sorts AFTER the current release per PEP 440, e.g. "0.3.12.dev20260720".
# 2. Build jobs pass the nightly version via `version-override` to _build-wheel.yaml
# so each wheel is stamped with the correct dev version for TestPyPI.
# 3. Specialized platform and packaging checks run nightly instead of on every PR.
# 4. publish-testpypi uploads all nightly-* artifacts to TestPyPI.
# 5. nightly-gate aggregates ALL build + test results; notify-failure files an issue.
# ──────────────────────────────────────────────────────────────────────────────
env:
SCCACHE_GHA_ENABLED: "true"
# Gate the dependency root so all downstream jobs are skipped in forks.
jobs:
version-stamp:
if: github.repository == 'kvcache-ai/Mooncake'
runs-on: ubuntu-22.04
outputs:
nightly_version: ${{ steps.version.outputs.nightly_version }}
steps:
- uses: actions/checkout@v4
- name: Compute nightly version
id: version
run: |
BASE_VERSION=$(grep -Po '(?<=^version = ")[^"]+' mooncake-wheel/pyproject.toml)
# Strip .postN suffix if present, then bump patch for forward-looking version
CLEAN_VERSION=$(echo "$BASE_VERSION" | sed 's/\.post[0-9]*$//')
MAJOR=$(echo "$CLEAN_VERSION" | cut -d. -f1)
MINOR=$(echo "$CLEAN_VERSION" | cut -d. -f2)
PATCH=$(echo "$CLEAN_VERSION" | cut -d. -f3)
NEXT_PATCH=$((PATCH + 1))
NIGHTLY_VERSION="${MAJOR}.${MINOR}.${NEXT_PATCH}.dev$(date -u +%Y%m%d)"
echo "nightly_version=$NIGHTLY_VERSION" >> "$GITHUB_OUTPUT"
echo "Nightly version: $NIGHTLY_VERSION"
build-wheels:
needs: version-stamp
strategy:
fail-fast: false
matrix:
include:
- variant: cuda
architecture: x86_64
artifact-prefix: nightly-cuda128-x86
- variant: cuda
architecture: arm64
artifact-prefix: nightly-cuda128-arm64
- variant: cuda13
architecture: x86_64
artifact-prefix: nightly-cuda13-x86
- variant: cuda13
architecture: arm64
artifact-prefix: nightly-cuda13-arm64
- variant: non-cuda
architecture: x86_64
artifact-prefix: nightly-non-cuda-x86
- variant: non-cuda
architecture: arm64
artifact-prefix: nightly-non-cuda-arm64
uses: ./.github/workflows/_build-wheel.yaml
with:
variant: ${{ matrix.variant }}
architecture: ${{ matrix.architecture }}
artifact-prefix: ${{ matrix.artifact-prefix }}
version-override: ${{ needs.version-stamp.outputs.nightly_version }}
tone-sglang-integration:
needs: build-wheels
uses: ./.github/workflows/integration-test.yml
with:
pr_sha: ${{ github.sha }}
source_run_id: ${{ github.run_id }}
artifact_name: nightly-cuda13-x86-py312
secrets: inherit
build-musa:
runs-on: ubuntu-22.04
container: registry.mthreads.com/mcconline/inference/pytorch:2.9.1.post1-py3.10-musa5.2.0-mp31-devel-ubuntu22.04-amd64
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- name: Mark repository as safe
run: git config --global --add safe.directory $GITHUB_WORKSPACE
shell: bash
- name: Configure project
run: |
apt update -y
apt install -y ninja-build
bash -x dependencies.sh -y
mkdir build
cd build
cmake -G Ninja .. -DUSE_MUSA=ON -DUSE_MNNVL=ON -DUSE_ETCD=ON -DSTORE_USE_ETCD=ON -DUSE_CXL=ON -DUSE_TCP=ON -DWITH_EP=ON -DBUILD_UNIT_TESTS=OFF -DBUILD_EXAMPLES=OFF -DENABLE_DEBUG_SYMBOLS=OFF
shell: bash
- name: Build project
run: |
cd build
source ~/.bashrc
# Limit concurrency of Mooncake PG's setup.py build to avoid OOM
export MAX_JOBS=2
# Does not affect the Ninja system
cmake --build .
cmake --install .
shell: bash
build-docker:
name: Build Docker Image
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- name: Free up disk space
uses: ./.github/actions/free-disk-space
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Build Docker image
run: |
docker build -f docker/mooncake.Dockerfile \
--build-arg PYTHON_VERSION=3.10 \
--build-arg EP_TORCH_VERSIONS="2.14.0" \
--build-arg CLEAN_BUILD_ARTIFACTS=1 \
-t mooncake:from-source .
ascend-test:
uses: ./.github/workflows/ci_ascend.yml
secrets: inherit
publish-testpypi:
needs: [version-stamp, build-wheels, nightly-test]
if: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.skip_publish == 'true') }}
runs-on: ubuntu-22.04
environment: nightly
permissions:
contents: read
steps:
- name: Log nightly version
run: |
echo "Publishing nightly version: ${{ needs.version-stamp.outputs.nightly_version }}"
- name: Download all nightly wheel artifacts
uses: actions/download-artifact@v4
with:
path: dist-all
pattern: 'nightly-*'
- name: Collect and rename wheels with nightly version
run: |
mkdir -p dist-publish
find dist-all -name "*.whl" -exec cp {} dist-publish/ \;
echo "Collected wheels:"
ls -la dist-publish/
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Install twine
run: pip install twine
- name: Validate wheels
run: twine check dist-publish/*.whl
- name: Publish to TestPyPI
env:
TWINE_USERNAME: __token__
TWINE_PASSWORD: ${{ secrets.TESTPYPI_API_TOKEN }}
run: |
twine upload --repository testpypi --skip-existing dist-publish/*.whl
nightly-test:
needs: version-stamp
runs-on: ubuntu-22.04
env:
CI: "true"
SCCACHE_GHA_ENABLED: "true"
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- name: Set up Python 3.12
uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
- name: Install and start etcd
run: |
wget -q https://github.com/etcd-io/etcd/releases/download/v3.6.1/etcd-v3.6.1-linux-amd64.tar.gz
tar xzf etcd-v3.6.1-linux-amd64.tar.gz
sudo mv etcd-v3.6.1-linux-amd64/etcd* /usr/local/bin/
etcd --advertise-client-urls http://127.0.0.1:2379 --listen-client-urls http://127.0.0.1:2379 &
sleep 3
ETCDCTL_API=3 etcdctl --endpoints=http://127.0.0.1:2379 endpoint health
- name: Free up disk space
run: |
sudo rm -rf /usr/share/dotnet /opt/ghc /opt/hostedtoolcache/CodeQL /usr/local/lib/android
df -h
- name: Install CUDA Toolkit
uses: Jimver/cuda-toolkit@v0.2.24
with:
cuda: '12.8.1'
linux-local-args: '["--toolkit"]'
method: 'network'
sub-packages: '["nvcc"]'
- name: Install build utilities
run: |
sudo apt-get update
sudo apt-get install -y ninja-build
- name: Run sccache-cache
uses: mozilla-actions/sccache-action@v0.0.9
- name: Configure sccache
uses: actions/github-script@v7
with:
script: |
core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || '');
core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
- name: Configure project (Release, no ASAN)
run: |
sudo apt update -y
sudo bash -x dependencies.sh -y
echo "/usr/local/go/bin" >> "$GITHUB_PATH"
mkdir build && cd build
cmake -G Ninja .. \
-DUSE_HTTP=ON -DUSE_CXL=ON -DUSE_UB=ON -DUSE_ETCD=ON -DUSE_CUDA=ON \
-DWITH_P2P_STORE=ON \
-DSTORE_USE_ETCD=ON -DCMAKE_BUILD_TYPE=Release \
-DBUILD_UNIT_TESTS=ON -DENABLE_SCCACHE=ON
- name: Configure CUDA driver runtime
run: |
cuda_driver_library=$(sed -n \
's/^CUDA_cuda_driver_LIBRARY:FILEPATH=//p' build/CMakeCache.txt)
if [ -z "$cuda_driver_library" ] || [ ! -f "$cuda_driver_library" ]; then
echo "::error::CMake did not resolve the CUDA driver library"
exit 1
fi
cuda_driver_dir=$(dirname "$cuda_driver_library")
if [ ! -e "$cuda_driver_dir/libcuda.so.1" ]; then
sudo ln -s "$(basename "$cuda_driver_library")" \
"$cuda_driver_dir/libcuda.so.1"
fi
echo "LIBRARY_PATH=$cuda_driver_dir:${LIBRARY_PATH:-}" >> "$GITHUB_ENV"
echo "LD_LIBRARY_PATH=$cuda_driver_dir:${LD_LIBRARY_PATH:-}" >> "$GITHUB_ENV"
- name: Build project
run: |
cd build
cmake --build . -j$(nproc)
sudo -E cmake --install .
- name: Build nvlink_allocator.so
run: |
mkdir -p build/mooncake-transfer-engine/nvlink-allocator
cd mooncake-transfer-engine/nvlink-allocator
bash build.sh ../../build/mooncake-transfer-engine/nvlink-allocator/
- name: Start Metadata Server
run: |
cd mooncake-transfer-engine/example/http-metadata-server-python
pip install aiohttp
python ./bootstrap_server.py &
echo "NIGHTLY_METADATA_SERVER_PID=$!" >> "$GITHUB_ENV"
sleep 2
- name: Run CTest unit tests
run: |
# Keep ephemeral TIME_WAIT sockets off mooncake_client's RPC port.
reserved_ports=$(sysctl -n net.ipv4.ip_local_reserved_ports)
sudo sysctl -w "net.ipv4.ip_local_reserved_ports=${reserved_ports:+$reserved_ports,}50052"
cd build
export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}:/usr/local/lib
MC_METADATA_SERVER=http://127.0.0.1:8080/metadata \
DEFAULT_KV_LEASE_TTL=500 \
ctest --parallel $(nproc) --output-on-failure
- name: Run Mooncake Store Rust smoke test
env:
MOONCAKE_STORE_CLUSTER_ID: nightly_rust_cluster
MOONCAKE_STORE_RUST_LINK_ASAN: "0"
run: ./scripts/ci/run_store_rust_smoke.sh
- name: Run Transfer Engine Rust smoke test
env:
MOONCAKE_TE_RUST_LINK_ASAN: "0"
run: ./scripts/ci/run_transfer_engine_rust_smoke.sh
- name: Check Mooncake Store Rust bindings, examples, and tests
env:
MOONCAKE_BUILD_DIR: ${{ github.workspace }}/build
MOONCAKE_STORE_LIB_DIR: ${{ github.workspace }}/build/mooncake-store/src
MOONCAKE_STORE_INCLUDE_DIR: ${{ github.workspace }}/mooncake-store/include
run: |
export LD_LIBRARY_PATH="$GITHUB_WORKSPACE/build/mooncake-asio:$GITHUB_WORKSPACE/build/mooncake-store/src:$GITHUB_WORKSPACE/build/mooncake-store/src/cachelib_memory_allocator:$GITHUB_WORKSPACE/build/mooncake-transfer-engine/src:$GITHUB_WORKSPACE/build/mooncake-transfer-engine/src/common/base:$GITHUB_WORKSPACE/build/mooncake-common/etcd:/usr/local/lib:${LD_LIBRARY_PATH:-}"
cd mooncake-store/rust
cargo test --lib
cargo test --examples --tests --no-run
cargo clean
- name: Verify Mooncake Store Rust dlopen bindings and packaging
env:
MOONCAKE_STORE_INCLUDE_DIR: ${{ github.workspace }}/mooncake-store/include
run: |
cd mooncake-store/rust
cargo run --locked --example generate_dlopen_bindings
git diff --exit-code -- src/generated/ffi_dlopen_bindings.rs
cargo package --no-verify --allow-dirty
crate=$(ls target/package/mooncake_store-*.crate | head -1)
dest=$(mktemp -d)
tar xzf "$crate" -C "$dest"
(cd "$dest"/mooncake_store-* && cargo check --no-default-features --features dlopen)
- name: Run Go store binding integration tests
env:
MOONCAKE_STORE_CLUSTER_ID: nightly_go_cluster
MOONCAKE_STORE_GO_SANITIZED: "0"
run: ./scripts/ci/run_store_go_integration.sh
- name: Build and install Python wheel for integration tests
run: |
python -m venv "${RUNNER_TEMP}/mooncake-nightly-venv"
echo "${RUNNER_TEMP}/mooncake-nightly-venv/bin" >> "$GITHUB_PATH"
source "${RUNNER_TEMP}/mooncake-nightly-venv/bin/activate"
export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}:/usr/local/lib
export CUDA_HOME=/usr/local/cuda
PYTHON_VERSION=3.12 OUTPUT_DIR=dist ./scripts/build_wheel.sh
python -m pip install mooncake-wheel/dist/*.whl
- name: Run Python integration tests (full suite)
env:
MC_METADATA_SERVER: http://127.0.0.1:8080/metadata
RUN_TESTS_METADATA_SERVER_MODE: external
DEFAULT_KV_LEASE_TTL: "500"
TEST_CXL: "1"
run: |
export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}:/usr/local/lib
bash scripts/run_tests.sh
- name: Run Python API end-to-end tests
env:
MOONCAKE_MASTER: "127.0.0.1:50051"
MOONCAKE_TE_META_DATA_SERVER: "http://127.0.0.1:8080/metadata"
MOONCAKE_PROTOCOL: "tcp"
LOCAL_HOSTNAME: "127.0.0.1"
run: |
export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}:/usr/local/lib
kill "$NIGHTLY_METADATA_SERVER_PID" 2>/dev/null || true
for _ in {1..50}; do
if ! kill -0 "$NIGHTLY_METADATA_SERVER_PID" 2>/dev/null; then
break
fi
sleep 0.1
done
if kill -0 "$NIGHTLY_METADATA_SERVER_PID" 2>/dev/null; then
kill -KILL "$NIGHTLY_METADATA_SERVER_PID"
fi
mkdir -p /tmp/mooncake_storage
mooncake_master \
--default_kv_lease_ttl=500 \
--eviction_high_watermark_ratio=0.95 \
--cluster_id=nightly_test_cluster \
--port 50051 \
--enable_http_metadata_server=true &
master_pid=$!
trap 'kill "$master_pid" 2>/dev/null || true; wait "$master_pid" 2>/dev/null || true' EXIT
sleep 3
python -m unittest mooncake-wheel.tests.test_weight_snapshot_api
python scripts/test_async_store.py
python scripts/test_copy_move_api.py
python scripts/test_drain_http_api.py --timeout-sec 90
- name: Run RPC Communicator Bandwidth Test
run: ./scripts/ci/run_rpc_smoke.sh
shell: bash
nightly-coverage:
runs-on: ubuntu-22.04
env:
CI: "true"
SCCACHE_GHA_ENABLED: "true"
CXXFLAGS: --coverage
CFLAGS: --coverage
LDFLAGS: --coverage
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- name: Set up Python 3.12
uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
- name: Install and start etcd
run: |
wget -q https://github.com/etcd-io/etcd/releases/download/v3.6.1/etcd-v3.6.1-linux-amd64.tar.gz
tar xzf etcd-v3.6.1-linux-amd64.tar.gz
sudo mv etcd-v3.6.1-linux-amd64/etcd* /usr/local/bin/
etcd --advertise-client-urls http://127.0.0.1:2379 --listen-client-urls http://127.0.0.1:2379 &
sleep 3
ETCDCTL_API=3 etcdctl --endpoints=http://127.0.0.1:2379 endpoint health
- name: Free up disk space
uses: ./.github/actions/free-disk-space
- name: Install CUDA Toolkit
uses: Jimver/cuda-toolkit@v0.2.24
with:
cuda: '12.8.1'
linux-local-args: '["--toolkit"]'
method: 'network'
sub-packages: '["nvcc"]'
- name: Install coverage tools and build utilities
run: |
sudo apt-get update
sudo apt-get install -y lcov gcovr ninja-build
- name: Test HugeTLB sizing helper
run: python3 scripts/test_hicache_hugepage_requirements.py
- name: Run sccache-cache
uses: mozilla-actions/sccache-action@v0.0.9
- name: Configure sccache
uses: actions/github-script@v7
with:
script: |
core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || '');
core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
- name: Configure project with coverage support
run: |
sudo apt update -y
sudo bash -x dependencies.sh -y
mkdir build
cd build
cmake -G Ninja .. -DUSE_HTTP=ON -DUSE_CXL=ON -DUSE_UB=ON -DUSE_ETCD=ON -DSTORE_USE_ETCD=ON -DENABLE_ASAN=ON -DENABLE_SCCACHE=ON -DCMAKE_BUILD_TYPE=Debug -DENABLE_DEBUG_SYMBOLS=OFF
- name: Build project
run: |
cd build
cmake --build . --parallel "$(nproc)"
sudo cmake --install .
- name: Start Metadata Server
run: |
cd mooncake-transfer-engine/example/http-metadata-server-python
pip install aiohttp
python ./bootstrap_server.py &
sleep 2
- name: Run CTest with coverage
run: |
cd build
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib
MC_METADATA_SERVER=http://127.0.0.1:8080/metadata \
DEFAULT_KV_LEASE_TTL=500 \
ctest --parallel $(nproc) --output-on-failure
- name: Run Mooncake Store Rust sanitizer smoke test
env:
MOONCAKE_STORE_CLUSTER_ID: nightly_asan_rust_cluster
MOONCAKE_STORE_RUST_LINK_ASAN: "1"
run: ./scripts/ci/run_store_rust_smoke.sh
- name: Run Transfer Engine Rust sanitizer smoke test
env:
MOONCAKE_TE_RUST_LINK_ASAN: "1"
run: ./scripts/ci/run_transfer_engine_rust_smoke.sh
- name: Run Go store binding sanitizer integration tests
env:
MOONCAKE_STORE_CLUSTER_ID: nightly_asan_go_cluster
MOONCAKE_STORE_GO_SANITIZED: "1"
run: ./scripts/ci/run_store_go_integration.sh
- name: Generate coverage report
id: coverage
run: |
cd build
echo "=== Starting coverage report generation ==="
echo "Current directory: $(pwd)"
echo "=== Looking for .gcda files ==="
find . -name "*.gcda" 2>/dev/null | head -10 || echo "No .gcda files found"
echo "=== Running lcov ==="
lcov --capture --directory . --output-file coverage.info 2>&1 || {
echo "WARNING: lcov failed to capture coverage data"
echo "Creating minimal lcov-compliant coverage file to allow CI to continue"
echo "TN:dummy" > coverage.filtered.info
echo "SF:/dev/null" >> coverage.filtered.info
echo "DA:0,0" >> coverage.filtered.info
echo "end_of_record" >> coverage.filtered.info
echo "coverage_failed=true" >> "$GITHUB_OUTPUT"
exit 0
}
echo "=== Processing coverage data ==="
lcov --remove coverage.info '/usr/*' '*/test/*' '*/third_party/*' '*/benchmarks/*' --output-file coverage.filtered.info 2>&1 || true
echo "=== Generating HTML report ==="
genhtml coverage.filtered.info --output-directory coverage_report 2>&1 || echo "genhtml failed, continuing..."
echo "=== Coverage summary ==="
lcov --list coverage.filtered.info 2>&1 || echo "lcov list failed"
echo "=== Coverage report generation completed ==="
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v4
with:
files: build/coverage.filtered.info
flags: unittests
name: nightly-code-coverage-report
token: ${{ secrets.CODECOV_TOKEN }}
fail_ci_if_error: false
continue-on-error: true
- name: Check coverage status
if: always()
run: |
if [ "${{ steps.coverage.outputs.coverage_failed }}" = "true" ]; then
echo "Coverage collection failed but the nightly workflow continued"
echo "::warning::Code coverage collection failed. Please check the logs."
else
echo "Coverage collected successfully"
fi
nightly-gate:
# always() bypasses skipped dependencies, so this job needs its own guard.
if: ${{ always() && github.repository == 'kvcache-ai/Mooncake' }}
needs:
- version-stamp
- build-wheels
- tone-sglang-integration
- build-musa
- build-docker
- ascend-test
- publish-testpypi
- nightly-test
- nightly-coverage
runs-on: ubuntu-22.04
steps:
- name: Check results
run: |
echo "=== Nightly Gate Summary ==="
echo "$NEEDS_JSON" | jq -r '
to_entries[] | "\(.key): \(.value.result)"'
failing=$(echo "$NEEDS_JSON" | jq -r \
--arg allow_publish_skip "$ALLOW_PUBLISH_SKIP" '
to_entries[] |
select(
.value.result != "success" and
(.key != "publish-testpypi" or
.value.result != "skipped" or
$allow_publish_skip != "true")
) |
"\(.key): \(.value.result)"')
if [ -n "$failing" ]; then
echo "::error::The following nightly jobs did not succeed:"
echo "$failing"
exit 1
fi
echo "All nightly jobs passed!"
env:
NEEDS_JSON: ${{ toJSON(needs) }}
ALLOW_PUBLISH_SKIP: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.skip_publish == 'true' }}
notify-failure:
if: ${{ always() && needs.nightly-gate.result == 'failure' }}
needs: [nightly-gate]
runs-on: ubuntu-22.04
permissions:
issues: write
steps:
- name: Create failure issue
uses: actions/github-script@v7
with:
script: |
const labelName = 'nightly-failure';
// Ensure the label exists
try {
await github.rest.issues.getLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName,
});
} catch (e) {
if (e.status === 404) {
await github.rest.issues.createLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName,
color: 'd73a4a',
description: 'Nightly CI workflow failure',
});
}
}
const title = `Nightly build/test failure - ${new Date().toISOString().slice(0, 10)}`;
const body = [
'## Nightly Failure Report',
'',
`**Run**: ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`,
`**Branch**: ${context.ref}`,
`**Timestamp**: ${new Date().toISOString()}`,
'',
'Please investigate the failed jobs in the workflow run linked above.',
].join('\n');
await github.rest.issues.create({
owner: context.repo.owner,
repo: context.repo.repo,
title,
body,
labels: [labelName],
});