benchmark.yaml

# Do not run this workflow on pull request since this workflow has permission to modify contents.
on:
  push:
    branches:
      - main

permissions:
  # deployments permission to deploy GitHub pages website
  deployments: write
  # contents permission to update benchmark contents in gh-pages branch
  contents: write

jobs:
  benchmark:
    name: Performance regression check
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: true
      - name: Install libs
        run: |
          sudo apt-get update
          sudo apt-get install build-essential libreadline-dev
      - name: Compile
        run: make TARGET=release
      - name: Run benchmark
        run: |
          gettimems() { date '+%s%3N'; }

          NTIMES=1
          sum=0
          for i in $(seq $NTIMES); do
            before=$(gettimems)
            ./cbcvm bench/bf.rbcvm bench/bench.b
            after=$(gettimems)
            elapsed=$((after - before))
            sum=$((sum + elapsed))
          done
          bfresult=$((sum / NTIMES))

          NTIMES=10
          sum=0
          for i in $(seq $NTIMES); do
            before=$(gettimems)
            ./cbcvm bench/nbody.rbcvm 500000
            after=$(gettimems)
            elapsed=$((after - before))
            sum=$((sum + elapsed))
          done
          nbodyresult=$((sum / NTIMES))

          NTIMES=10
          sum=0
          for i in $(seq $NTIMES); do
            before=$(gettimems)
            ./cbcvm bench/spectral_norm.rbcvm 1000
            after=$(gettimems)
            elapsed=$((after - before))
            sum=$((sum + elapsed))
          done
          spectralnormresult=$((sum / NTIMES))

          NTIMES=1
          sum=0
          for i in $(seq NTIMES); do
            before=$(gettimems)
            ./cbcvm bench/binary_trees.rbcvm 21
            after=$(gettimems)
            elapsed=$((after - before))
            sum=$((sum + elapsed))
          done
          binarytreesresult=$((sum / NTIMES))

          printf '[' >> output.txt
          printf '{"name": "bench.b", "value": %f, "unit": "ms"},' $bfresult >> output.txt
          printf '{"name": "nbody", "value": %f, "unit": "ms"},' $nbodyresult >> output.txt
          printf '{"name": "spectral_norm", "value": %f, "unit": "ms"},' $spectralnormresult >> output.txt
          printf '{"name": "binary_trees", "value": %f, "unit": "ms"}' $binarytreesresult >> output.txt
          printf ']' >> output.txt
      # gh-pages branch is updated and pushed automatically with extracted benchmark data
      - name: Store benchmark result
        uses: benchmark-action/github-action-benchmark@v1
        with:
          name: c-bytecode-vm benchmark
          tool: customSmallerIsBetter
          output-file-path: output.txt
          # Access token to deploy GitHub Pages branch
          github-token: ${{ secrets.GITHUB_TOKEN }}
          # Push and deploy GitHub pages branch automatically
          auto-push: true
          comment-on-alert: true