Bare-metal script for running batch experiments on background eviction and promotion

.github/workflows/build-cachelib-centos-8-5.yml

@@ -13,8 +13,6 @@
 # limitations under the License.
 name: build-cachelib-centos-8.5
 on:
-#   push:
-  pull_request:
   schedule:
      - cron:  '0 9 * * *'
 jobs:

.github/workflows/build-cachelib-centos-long.yml

@@ -0,0 +1,39 @@
+name: build-cachelib-centos-latest
+on:
+  schedule:
+    - cron:  '0 7 * * *'
+
+jobs:
+  build-cachelib-centos8-latest:
+    name: "CentOS/latest - Build CacheLib with all dependencies"
+    runs-on: ubuntu-latest
+    # Docker container image name
+    container: "centos:latest"
+    steps:
+      - name: "update packages"
+        run: dnf upgrade -y
+      - name: "install sudo,git"
+        run: dnf install -y sudo git cmake gcc
+      - name: "System Information"
+        run: |
+          echo === uname ===
+          uname -a
+          echo === /etc/os-release ===
+          cat /etc/os-release
+          echo === df -hl ===
+          df -hl
+          echo === free -h ===
+          free -h
+          echo === top ===
+          top -b -n1 -1 -Eg || timeout 1 top -b -n1
+          echo === env ===
+          env
+          echo === gcc -v ===
+          gcc -v
+      - name: "checkout sources"
+        uses: actions/checkout@v2
+      - name: "build CacheLib using build script"
+        run: ./contrib/build.sh -j -v -T
+      - name: "run tests"
+        timeout-minutes: 60
+        run: cd opt/cachelib/tests && ../../../run_tests.sh long

.github/workflows/build-cachelib-debian.yml

@@ -0,0 +1,43 @@
+name: build-cachelib-debian-10
+on:
+  schedule:
+    - cron:  '30 5 * * 0,3'
+
+jobs:
+  build-cachelib-debian-10:
+    name: "Debian/Buster - Build CacheLib with all dependencies"
+    runs-on: ubuntu-latest
+    # Docker container image name
+    container: "debian:buster-slim"
+    steps:
+      - name: "update packages"
+        run: apt-get update
+      - name: "upgrade packages"
+        run: apt-get -y upgrade
+      - name: "install sudo,git"
+        run: apt-get install -y sudo git procps
+      - name: "System Information"
+        run: |
+          echo === uname ===
+          uname -a
+          echo === /etc/os-release ===
+          cat /etc/os-release
+          echo === df -hl ===
+          df -hl
+          echo === free -h ===
+          free -h
+          echo === top ===
+          top -b -n1 -1 -Eg || timeout 1 top -b -n1 ; true
+          echo === env ===
+          env
+          echo === cc -v ===
+          cc -v || true
+          echo === g++ -v ===
+          g++ - || true
+      - name: "checkout sources"
+        uses: actions/checkout@v2
+      - name: "build CacheLib using build script"
+        run: ./contrib/build.sh -j -v -T
+      - name: "run tests"
+        timeout-minutes: 60
+        run: cd opt/cachelib/tests && ../../../run_tests.sh

.github/workflows/build-cachelib-docker.yml

@@ -0,0 +1,50 @@
+name: build-cachelib-docker
+on:
+  push:
+  pull_request:
+
+jobs:
+  build-cachelib-docker:
+    name: "CentOS/latest - Build CacheLib with all dependencies"
+    runs-on: ubuntu-latest
+    env:
+      REPO:           cachelib
+      GITHUB_REPO:    intel/CacheLib
+      CONTAINER_REG:  ghcr.io/pmem/cachelib
+      CONTAINER_REG_USER:   ${{ secrets.GH_CR_USER }}
+      CONTAINER_REG_PASS:   ${{ secrets.GH_CR_PAT }}
+      FORCE_IMAGE_ACTION:   ${{ secrets.FORCE_IMAGE_ACTION }}
+      HOST_WORKDIR:         ${{ github.workspace }}
+      WORKDIR:              docker
+      IMG_VER:              devel
+    strategy:
+      matrix:
+        CONFIG: ["OS=centos OS_VER=8streams PUSH_IMAGE=1"]
+    steps:
+      - name: "System Information"
+        run: |
+          echo === uname ===
+          uname -a
+          echo === /etc/os-release ===
+          cat /etc/os-release
+          echo === df -hl ===
+          df -hl
+          echo === free -h ===
+          free -h
+          echo === top ===
+          top -b -n1 -1 -Eg || timeout 1 top -b -n1
+          echo === env ===
+          env
+          echo === gcc -v ===
+          gcc -v
+      - name: "checkout sources"
+        uses: actions/checkout@v2
+        with:
+          submodules: recursive
+          fetch-depth: 0
+
+      - name: Pull the image or rebuild and push it
+        run: cd $WORKDIR && ${{ matrix.CONFIG }} ./pull-or-rebuild-image.sh $FORCE_IMAGE_ACTION
+
+      - name: Run the build
+        run: cd $WORKDIR && ${{ matrix.CONFIG }} ./build.sh

.github/workflows/clang-format-check.yml

@@ -1,6 +1,6 @@
 # From: https://github.com/marketplace/actions/clang-format-check#multiple-paths
 name: clang-format Check
-on: [pull_request]
+on: []
 jobs:
   formatting-check:
     name: Formatting Check

MultiTierDataMovement.md

@@ -0,0 +1,90 @@
+# Background Data Movement
+
+In order to reduce the number of online evictions and support asynchronous
+promotion - we have added two periodic workers to handle eviction and promotion.
+
+The diagram below shows a simplified version of how the background evictor
+thread (green) is integrated to the CacheLib architecture. 
+
+<p align="center">
+  <img width="640" height="360" alt="BackgroundEvictor" src="cachelib-background-evictor.png">
+</p>
+
+## Background Evictors
+
+The background evictors scan each class to see if there are objects to move the next (lower)
+tier using a given strategy. Here we document the parameters for the different
+strategies and general parameters. 
+
+- `backgroundEvictorIntervalMilSec`: The interval that this thread runs for - by default
+the background evictor threads will wake up every 10 ms to scan the AllocationClasses. Also,
+the background evictor thread will be woken up everytime there is a failed allocation (from
+a request handling thread) and the current percentage of free memory for the 
+AllocationClass is lower than `lowEvictionAcWatermark`. This may render the interval parameter
+not as important when there are many allocations occuring from request handling threads. 
+
+- `evictorThreads`: The number of background evictors to run - each thread is a assigned
+a set of AllocationClasses to scan and evict objects from. Currently, each thread gets
+an equal number of classes to scan - but as object size distribution may be unequal - future
+versions will attempt to balance the classes among threads. The range is 1 to number of AllocationClasses.
+The default is 1. 
+
+- `maxEvictionBatch`: The number of objects to remove in a given eviction call. The
+default is 40. Lower range is 10 and the upper range is 1000. Too low and we might not
+remove objects at a reasonable rate, too high and it might increase contention with user threads.
+
+- `minEvictionBatch`: Minimum number of items to evict at any time (if there are any
+candidates)
+
+- `maxEvictionPromotionHotness`: Maximum candidates to consider for eviction. This is similar to `maxEvictionBatch`
+but it specifies how many candidates will be taken into consideration, not the actual number of items to evict.
+This option can be used to configure duration of critical section on LRU lock.
+
+
+### FreeThresholdStrategy (default)
+
+- `lowEvictionAcWatermark`: Triggers background eviction thread to run
+when this percentage of the AllocationClass is free. 
+The default is `2.0`, to avoid wasting capacity we don't set this above `10.0`.
+
+- `highEvictionAcWatermark`: Stop the evictions from an AllocationClass when this 
+percentage of the AllocationClass is free. The default is `5.0`, to avoid wasting capacity we
+don't set this above `10`.
+
+
+## Background Promoters
+
+The background promoters scan each class to see if there are objects to move to a lower
+tier using a given strategy. Here we document the parameters for the different
+strategies and general parameters.
+
+- `backgroundPromoterIntervalMilSec`: The interval that this thread runs for - by default
+the background promoter threads will wake up every 10 ms to scan the AllocationClasses for
+objects to promote.
+
+- `promoterThreads`: The number of background promoters to run - each thread is a assigned
+a set of AllocationClasses to scan and promote objects from. Currently, each thread gets
+an equal number of classes to scan - but as object size distribution may be unequal - future
+versions will attempt to balance the classes among threads. The range is `1` to number of AllocationClasses. The default is `1`.
+
+- `maxProtmotionBatch`: The number of objects to promote in a given promotion call. The
+default is 40. Lower range is 10 and the upper range is 1000. Too low and we might not
+remove objects at a reasonable rate, too high and it might increase contention with user threads. 
+
+- `minPromotionBatch`: Minimum number of items to promote at any time (if there are any
+candidates)
+
+- `numDuplicateElements`: This allows us to promote items that have existing handles (read-only) since
+we won't need to modify the data when a user is done with the data. Therefore, for a short time
+the data could reside in both tiers until it is evicted from its current tier. The default is to
+not allow this (0). Setting the value to 100 will enable duplicate elements in tiers.
+
+### Background Promotion Strategy (only one currently)
+
+- `promotionAcWatermark`: Promote items if there is at least this
+percent of free AllocationClasses. Promotion thread will attempt to move `maxPromotionBatch` number of objects
+to that tier. The objects are chosen from the head of the LRU. The default is `4.0`.
+This value should correlate with `lowEvictionAcWatermark`, `highEvictionAcWatermark`, `minAcAllocationWatermark`, `maxAcAllocationWatermark`.
+- `maxPromotionBatch`: The number of objects to promote in batch during BG promotion. Analogous to
+`maxEvictionBatch`. It's value should be lower to decrease contention on hot items.
+

baremetal/README.md

@@ -0,0 +1,51 @@
+# Bare-metal Testing Scripts
+
+
+> ### accelConfig.sh
+>
+>> Set up DSA devices using accel-config.
+>> - OPTIONAL Arg-1: DSA device id. Default: 0
+>> - OPTIONAL Arg-2: Enable/Disable DSA device. Default: yes
+>> - OPTIONAL Arg-3: SHARED WQ id. Default: 1
+>> - OPTIONAL Arg-4: ENGINE count. Default: 4
+>> - OUTPUT Verify DSA devices set up is correct using accel-config
+
+
+> ### runTestAndConvertToJson.py
+>
+>> Run a single test and convert the output TXT to CSV.
+>> This script uses numactl to bind all threads to node 0.
+>> Run with **sudo**
+>> - REQUIRED Arg-1: Cachebench config file path from root directory
+>> - REQUIRED Arg-2: DSA device count
+>> - REQUIRED Arg-3: Number of background evictors
+>> - REQUIRED Arg-4: Eviction batch size
+>> - REQUIRED Arg-5: Number of background promoters
+>> - REQUIRED Arg-6: Promotion batch size
+>> - REQUIRED Arg-7: Output path
+>> - OUTPUT txt and json saved in same path
+
+
+> ### aggregateAndFilterTestResults.py
+>
+>> Gather all output JSON using the file name filter
+>> Run with **sudo**
+>> - REQUIRED Arg-1: Output json directory path
+>> - REQUIRED Arg-2: Filter filenames using this string. Pass null str to gather all files.
+>> - OUTPUT Saved in a csv on the output JSON directory path
+
+
+> ### parseTestResultIntoCsv.py
+>
+>> Parse TXT output and save in JSON format
+>> - REQUIRED Arg-1: output txt file name
+>> - REQUIRED Arg-2: Tag a string for the text
+>> - OUTPUT Save CSV to the same path
+
+
+> ### sample.sh
+>
+>> This shows how to create a test sequence, then parses results and aggregates it.
+>> Run with **sudo**
+
+

baremetal/accelConfig.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+echo "OPTIONAL Arg-1: DSA device id. Default: 0"
+echo "OPTIONAL Arg-2: Enable/Disable DSA device. Default: yes"
+echo "OPTIONAL Arg-3: SHARED WQ id. Default: 1"
+echo "OPTIONAL Arg-4: ENGINE count. Default: 4"
+
+if [ "$#" -ge 5 ]; then
+    echo "ERROR: Incorrect argument count. Expected arg count <= 4"
+    exit 1
+fi
+
+DEVID=${1:-0}
+ENABLE=${2:-yes}
+SWQID=${3:-1}
+NENGS=${4:-4}
+
+DEV=dsa${DEVID}
+SWQ=${DEV}/wq${DEVID}.${SWQID}
+
+echo "=> ${SWQ}:"
+accel-config disable-wq ${SWQ}
+
+echo "=> ${DEV}:"
+accel-config disable-device ${DEV}
+
+if [ "${ENABLE}" != "yes" ]; then
+    echo "Exit after disabling ${DEV}."
+    exit 1
+fi
+
+for ((i=0; i < ${NENGS}; i++))
+do
+    echo "=> ${DEV}/engine${DEVID}.${i}"
+    echo "configured"
+    accel-config config-engine ${DEV}/engine${DEVID}.${i} --group-id=0
+done
+
+accel-config config-wq ${SWQ} --group-id=0
+accel-config config-wq ${SWQ} --priority=1
+accel-config config-wq ${SWQ} --wq-size=128
+accel-config config-wq ${SWQ} --max-batch-size=1024
+accel-config config-wq ${SWQ} --max-transfer-size=2147483648
+accel-config config-wq ${SWQ} --block-on-fault=0
+accel-config config-wq ${SWQ} --type=user
+accel-config config-wq ${SWQ} --name="dsa-test"
+accel-config config-wq ${SWQ} --mode=shared
+accel-config config-wq ${SWQ} --threshold=127
+accel-config config-wq ${SWQ} --driver-name="user"
+
+echo "=> ${DEV}:"
+accel-config enable-device ${DEV}
+
+echo "=> ${SWQ}:"
+accel-config enable-wq ${SWQ}
+