Merge pull request #808 from jprendes/benchmark-memory

Benchmark memory

Author: Mossaka

.github/workflows/benchmarks.yml

@@ -17,8 +17,6 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-      - name: Fetch submodules
-        run: git submodule update --init --recursive
       - uses: actions-rust-lang/setup-rust-toolchain@v1
         with:
           rustflags: '' #Disable.  By default this action sets environment variable is set to -D warnings.  We manage this in the Makefile
@@ -54,14 +52,56 @@ jobs:
           # Automatically push the benchmark result to gh-pages branch
           # See https://github.com/benchmark-action/github-action-benchmark?tab=readme-ov-file#charts-on-github-pages-1 for more details
           auto-push: ${{ github.event_name == 'schedule' }}
+
+  benchmark-mem:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          rustflags: '' #Disable.  By default this action sets environment variable is set to -D warnings.  We manage this in the Makefile
+      - name: Setup build environment
+        shell: bash
+        run: |
+          os=$(echo "$RUNNER_OS" | tr '[:upper:]' '[:lower:]')
+          ./scripts/setup-$os.sh
+      - name: Build and load shims and wasi-demo-app
+        shell: bash
+        run: |
+          make OPT_PROFILE=release build install test-image load test-image/oci load/oci
+      - name: Run Benchmarks
+        shell: bash
+        run: |
+          set -euxo pipefail
+          for RUNTIME in wasmtime wasmedge wasmer wamr; do
+            sudo ./scripts/benchmark-mem.sh $RUNTIME > bench-mem-$RUNTIME.json
+          done
+          cat bench-mem-* | jq -s 'flatten(1)' > bench-mem.json
+      - name: Store benchmark result
+        uses: benchmark-action/[email protected]
+        with:
+          name: Criterion.rs Benchmark
+          tool: 'customSmallerIsBetter'
+          output-file-path: bench-mem.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          # my experimental local benchmarking seems to have a 20% margin of error. 
+          # So I set the alert threshold to 130% of the previous benchmark result.
+          # If the current benchmark result is more than 130% of the previous benchmark result, it will fail.
+          alert-threshold: '130%'
+          fail-on-alert: ${{ github.event_name == 'schedule' }}
+          alert-comment-cc-users: '@runwasi-committers'
+          # Enable Job Summary
+          summary-always: true
+          # Automatically push the benchmark result to gh-pages branch
+          # See https://github.com/benchmark-action/github-action-benchmark?tab=readme-ov-file#charts-on-github-pages-1 for more details
+          auto-push: ${{ github.event_name == 'schedule' }}
 
   benchmark-http:
     runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v4
-      - name: Fetch submodules
-        run: git submodule update --init --recursive
       - uses: actions-rust-lang/setup-rust-toolchain@v1
         with:
           rustflags: '' #Disable.  By default this action sets environment variable is set to -D warnings.  We manage this in the Makefile

crates/containerd-shim-wasm/src/sandbox/cli.rs

@@ -32,6 +32,42 @@ macro_rules! revision {
     };
 }
 
+#[cfg(target_os = "linux")]
+fn get_mem(pid: u32) -> (usize, usize) {
+    let mut rss = 0;
+    let mut total = 0;
+    for line in std::fs::read_to_string(format!("/proc/{pid}/status"))
+        .unwrap()
+        .lines()
+    {
+        let line = line.trim();
+        // VmPeak is the maximum total virtual memory used so far.
+        // VmHWM (high water mark) is the maximum resident set memory used so far.
+        // See: https://man7.org/linux/man-pages/man5/proc_pid_status.5.html
+        if let Some(rest) = line.strip_prefix("VmPeak:") {
+            if let Some(rest) = rest.strip_suffix("kB") {
+                total = rest.trim().parse().unwrap_or(0);
+            }
+        } else if let Some(rest) = line.strip_prefix("VmHWM:") {
+            if let Some(rest) = rest.strip_suffix("kB") {
+                rss = rest.trim().parse().unwrap_or(0);
+            }
+        }
+    }
+    (rss, total)
+}
+
+#[cfg(target_os = "linux")]
+fn log_mem() {
+    let pid = std::process::id();
+    let (rss, tot) = get_mem(pid);
+    log::info!("Shim peak memory usage was: peak resident set {rss} kB, peak total {tot} kB");
+
+    let pid = zygote::Zygote::global().run(|_| std::process::id(), ());
+    let (rss, tot) = get_mem(pid);
+    log::info!("Zygote peak memory usage was: peak resident set {rss} kB, peak total {tot} kB");
+}
+
 /// Main entry point for the shim.
 ///
 /// If the `opentelemetry` feature is enabled, this function will start the shim with OpenTelemetry tracing.
@@ -70,6 +106,9 @@ pub fn shim_main<'a, I>(
     {
         shim_main_inner::<I>(name, version, revision, shim_version, config);
     }
+
+    #[cfg(target_os = "linux")]
+    log_mem();
 }
 
 #[cfg_attr(feature = "tracing", tracing::instrument(parent = tracing::Span::current(), skip_all, level = "Info"))]

scripts/benchmark-mem.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+# Parse CLI arguments
+RUNTIME=${1:-"wasmtime"}; shift || true
+IMAGE=${1:-"ghcr.io/containerd/runwasi/wasi-demo-app:latest"}; shift || true
+
+if [ $IMAGE == "ghcr.io/containerd/runwasi/wasi-demo-app:latest" ] && [ "$#" == "0" ]; then
+    set -- /wasi-demo-app.wasm echo 'hello'
+fi
+
+# Run the shim and collect logs
+LOG_FILE=$(mktemp)
+journalctl -fn 0 -u containerd | timeout -k 16s 15s grep -m 2 'peak memory usage was' > $LOG_FILE &
+ctr run --null-io --rm --runtime=io.containerd.$RUNTIME.v1 "$IMAGE" testwasm "$@"
+
+# Parse the logs
+wait
+SHIM_MEM=$(cat $LOG_FILE | grep 'Shim peak memory usage was' | sed -E 's/.*peak resident set ([0-9]+) kB.*/\1/')
+ZYGOTE_MEM=$(cat $LOG_FILE | grep 'Zygote peak memory usage was' | sed -E 's/.*peak resident set ([0-9]+) kB.*/\1/')
+rm $LOG_FILE
+
+if [ "$SHIM_MEM" == "" ] || [ "$ZYGOTE_MEM" == "" ]; then
+    exit 1
+fi
+
+TOTAL_MEM=$(( $SHIM_MEM + $ZYGOTE_MEM ))
+
+# Print the JSON for the benchmark report
+cat <<EOF
+[
+    {
+        "name": "$RUNTIME/memory-usage",
+        "unit": "kB",
+        "value": $TOTAL_MEM,
+        "extra": "shim: $SHIM_MEM kB\nzygote: $ZYGOTE_MEM kB"
+    }
+]
+EOF