[Draft pull request] Extending benchmarking framework for Kubernetes.

.gitignore

@@ -1,2 +1,3 @@
 **/*local.json
 **/*local.txt
+**/*local.properties

cosmos/scripts/Dockerfile

@@ -1,11 +1,19 @@
 FROM ubuntu:20.04
 RUN apt-get update && apt-get install -y openjdk-8-jdk net-tools vim
-COPY ycsb-azurecosmos-binding-0.18.0-SNAPSHOT/ /usr/app/ycsb-azurecosmos-binding-0.18.0-SNAPSHOT/
-COPY ./run.sh /usr/app/ycsb-azurecosmos-binding-0.18.0-SNAPSHOT/
-COPY ./azure_file_share.sh /usr/app/ycsb-azurecosmos-binding-0.18.0-SNAPSHOT/
-COPY ./azurecosmos.properties /usr/app/ycsb-azurecosmos-binding-0.18.0-SNAPSHOT/
-WORKDIR /usr/app/ycsb-azurecosmos-binding-0.18.0-SNAPSHOT/
-RUN pwd
-RUN chmod +x run.sh
-RUN chmod +x azure_file_share.sh
-CMD /bin/sh ./run.sh
+RUN apt-get -y install maven
+RUN apt-get update && apt-get -y install git-all
+RUN apt-get -y install curl
+RUN curl -sL https://aka.ms/InstallAzureCLIDeb | bash
+RUN apt install -y wget
+RUN wget https://aka.ms/downloadazcopy-v10-linux && tar -xvf downloadazcopy-v10-linux && cp ./azcopy_linux_amd64_*/azcopy /usr/bin/
+RUN apt install -y jq
+RUN apt-get install -y python3
+RUN apt-get install sudo -y
+RUN adduser --disabled-password --gecos '' benchmarking
+RUN adduser benchmarking sudo
+RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> \
+/etc/sudoers
+USER benchmarking
+WORKDIR /home/benchmarking
+RUN curl -H 'Cache-Control: no-cache, no-store' -o execute-kubernetes.sh https://raw.githubusercontent.com/Azure/azure-db-benchmarking/users/ratella/aks/cosmos/scripts/execute-kubernetes.sh
+RUN chmod +x execute-kubernetes.sh

cosmos/scripts/custom-script.sh

@@ -44,7 +44,7 @@ cd YCSB
 echo "########## Pulling Latest YCSB ##########"
 git pull
 echo "########## Building YCSB ##########"
-mvn -pl site.ycsb:$DB_BINDING_NAME-binding -am clean package
+mvn -pl site.ycsb:$DB_BINDING_NAME-binding -am clean package -q
 cp -r ./$DB_BINDING_NAME/target/ycsb-$DB_BINDING_NAME-binding*.tar.gz /tmp/ycsb
 cp -r ./$DB_BINDING_NAME/conf/* /tmp/ycsb
 cd /tmp/ycsb/

cosmos/scripts/execute-kubernetes.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+echo "#####Building custom script url####"
+customScriptUrl="https://raw.githubusercontent.com/${BENCHMARKING_FRAMEWORK_REPO}/${BENCHMARKING_FRAMEWORK_BRANCH}/cosmos/scripts/custom-script.sh"
+export BENCHMARKING_TOOLS_URL="https://github.com/${BENCHMARKING_FRAMEWORK_REPO}.git"
+export BENCHMARKING_TOOLS_BRANCH_NAME=$BENCHMARKING_FRAMEWORK_BRANCH
+export YCSB_GIT_REPO_URL="https://github.com/${YCSB_REPO}.git"
+export YCSB_GIT_BRANCH_NAME=$YCSB_BRANCH
+export VM_NAME=$POD_NAME
+export MACHINE_INDEX=$POD_INDEX
+export VM_COUNT=$POD_COUNT
+
+curl -o custom-script.sh $customScriptUrl
+
+# stdout and stderr will be logged in <$HOME>/agent.out, <$HOME>/agent.err and all output will go to the console
+bash custom-script.sh > >(tee $"/home/${ADMIN_USER_NAME}/agent.out") 2> >(tee "/home/${ADMIN_USER_NAME}/agent.err" >&2)
+

cosmos/sql/tools/java/ycsb/aks/base/templates/benchmarking-deployment-template.yaml

@@ -0,0 +1,166 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: <deploymentname>
+  labels:
+     app: benchmarking
+spec:
+  ttlSecondsAfterFinished: 10
+  template:
+    spec:
+      containers:
+      - name: benchmarking-client
+        image: ravitella/bootstrapper:latest
+        imagePullPolicy: Always
+        command: ["./execute-kubernetes.sh"]
+        resources:
+           limits:
+             memory: "<memorylimit>"
+             cpu: "<cpulimit>"
+        env:
+        - name: DB_BINDING_NAME
+          value: "azurecosmos"
+        - name: ADMIN_USER_NAME
+          value: "benchmarking"
+        - name: POD_NAME
+          value: "<podname>"
+        - name: POD_INDEX
+          value: "<podindex>"  
+        - name: POD_COUNT
+          value: "<podcount>"  
+        - name: WORKLOAD_TYPE
+          value: "workloadc"
+        - name: RESULT_STORAGE_CONNECTION_STRING
+          valueFrom:
+             secretKeyRef:
+               name: benchmarking-secrets
+               key: RESULT_STORAGE_CONNECTION_STRING
+        - name: COSMOS_URI
+          valueFrom:
+             secretKeyRef:
+               name: benchmarking-secrets
+               key: COSMOS_URI
+        - name: COSMOS_KEY
+          valueFrom:
+             secretKeyRef:
+               name: benchmarking-secrets
+               key: COSMOS_KEY  
+        - name: GUID
+          valueFrom:
+             secretKeyRef:
+               name: benchmarking-secrets
+               key: GUID   
+        - name: PREFERRED_REGION_LIST
+          valueFrom:
+             configMapKeyRef:
+               name: benchmarking-recipe-config
+               key: PREFERRED_REGION_LIST
+               optional: true        
+        - name: WRITE_ONLY_OPERATION
+          valueFrom:
+             configMapKeyRef:
+               name: benchmarking-recipe-config
+               key: WRITE_ONLY_OPERATION               
+        - name: TARGET_OPERATIONS_PER_SECOND
+          valueFrom:
+             configMapKeyRef:
+               name: benchmarking-recipe-config
+               key: TARGET_OPERATIONS_PER_SECOND
+        - name: THREAD_COUNT
+          valueFrom:
+             configMapKeyRef:
+               name: benchmarking-recipe-config
+               key: THREAD_COUNT
+        - name: YCSB_OPERATION_COUNT
+          valueFrom:
+             configMapKeyRef:
+               name: benchmarking-recipe-config
+               key: YCSB_OPERATION_COUNT
+        - name: YCSB_RECORD_COUNT
+          valueFrom:
+             configMapKeyRef:
+               name: benchmarking-recipe-config
+               key: YCSB_RECORD_COUNT
+               optional: true 
+        - name: USE_GATEWAY
+          valueFrom:
+             configMapKeyRef:
+               name: benchmarking-recipe-config
+               key: USE_GATEWAY
+        - name: DIAGNOSTICS_LATENCY_THRESHOLD_IN_MS
+          valueFrom:
+             configMapKeyRef:
+               name: benchmarking-recipe-config
+               key: DIAGNOSTICS_LATENCY_THRESHOLD_IN_MS        
+        - name: REQUEST_DISTRIBUTION
+          valueFrom:
+             configMapKeyRef:
+               name: benchmarking-recipe-config
+               key: REQUEST_DISTRIBUTION
+        - name: INSERT_ORDER
+          valueFrom:
+             configMapKeyRef:
+               name: benchmarking-recipe-config
+               key: INSERT_ORDER
+        - name: INCLUDE_EXCEPTION_STACK
+          valueFrom:
+             configMapKeyRef:
+               name: benchmarking-recipe-config
+               key: INCLUDE_EXCEPTION_STACK
+        - name: FIELD_COUNT
+          valueFrom:
+             configMapKeyRef:
+               name: benchmarking-recipe-config
+               key: FIELD_COUNT
+        - name: SKIP_LOAD_PHASE
+          valueFrom:
+             configMapKeyRef:
+               name: benchmarking-recipe-config
+               key: SKIP_LOAD_PHASE
+               optional: true 
+        - name: INSERT_PROPORTION
+          valueFrom:
+             configMapKeyRef:
+               name: benchmarking-recipe-config
+               key: INSERT_PROPORTION
+               optional: true 
+        - name: UPDATE_PROPORTION
+          valueFrom:
+             configMapKeyRef:
+               name: benchmarking-recipe-config
+               key: UPDATE_PROPORTION
+               optional: true 
+        - name: SCAN_PROPORTION
+          valueFrom:
+             configMapKeyRef:
+               name: benchmarking-recipe-config
+               key: SCAN_PROPORTION
+               optional: true 
+        - name: READ_PROPORTION
+          valueFrom:
+             configMapKeyRef:
+               name: benchmarking-recipe-config
+               key: READ_PROPORTION
+               optional: true 
+        - name: BENCHMARKING_FRAMEWORK_REPO
+          valueFrom:
+             configMapKeyRef:
+               name: benchmarking-recipe-config
+               key: BENCHMARKING_FRAMEWORK_REPO
+        - name: BENCHMARKING_FRAMEWORK_BRANCH
+          valueFrom:
+             configMapKeyRef:
+               name: benchmarking-recipe-config
+               key: BENCHMARKING_FRAMEWORK_BRANCH
+        - name: YCSB_REPO
+          valueFrom:
+             configMapKeyRef:
+               name: benchmarking-recipe-config
+               key: YCSB_REPO
+        - name: YCSB_BRANCH
+          valueFrom:
+             configMapKeyRef:
+               name: benchmarking-recipe-config
+               key: YCSB_BRANCH
+      restartPolicy: Never
+  backoffLimit: 0

cosmos/sql/tools/java/ycsb/aks/recipes/read/10-thousand-rps-read/driver.sh

@@ -0,0 +1,83 @@
+#!/bin/bash
+
+while getopts s:u:k:c:r: flag
+do
+    case "${flag}" in
+        s) storageconnstring=${OPTARG};;
+        u) cosmosuri=${OPTARG};;
+        k) cosmosmkey=${OPTARG};;
+        c) clustername=${OPTARG};;
+        r) clusterrg=${OPTARG};;
+    esac
+done
+
+usage(){  
+ echo "Usage: ./driver.sh -s '<storage-connection>' -u '<cosmos-uri>' -k '<cosmos-key>' -c '<aks-cluster-name>' -r '<aks-cluster-resource-group>'"
+ }
+
+if [[ -z "$storageconnstring" ]]
+then
+  echo "Srorage account connection string is missing. Pass it with -s flag"
+  usage
+  exit 1
+fi
+
+if [[ -z "$cosmosuri" ]]
+then
+  echo "Cosmos uri is missing. Pass it with -u flag"
+  usage
+  exit 1
+fi
+
+if [[ -z "$cosmosmkey" ]]
+then
+  echo "Cosmos key is missing. Pass it with -k flag"
+  usage
+  exit 1
+fi
+
+if [[ -z "$clustername" ]]
+then
+  echo "AKS cluster name is missing. Pass it with -c flag"
+  usage
+  exit 1
+fi
+
+if [[ -z "$clusterrg" ]]
+then
+  echo "Resource group name of the AKS cluster is missing. Pass it with -r flag"
+  usage
+  exit 1
+fi
+
+
+# get cluster credentials
+az aks get-credentials -n $clustername -g $clusterrg
+
+# setting the cluster
+kubectl config use-context $clustername
+
+# create recipe configmap from file that containes workload configuration 
+RECIPE_CONFIG_MAP=benchmarking-recipe-config
+STATUS_CHECK=$(kubectl get configmap  $RECIPE_CONFIG_MAP | grep $RECIPE_CONFIG_MAP )
+if [ -n "$STATUS_CHECK" ]
+then
+     kubectl delete configmap $RECIPE_CONFIG_MAP
+fi    
+kubectl create configmap $RECIPE_CONFIG_MAP --from-env-file=./recipe-env-file.properties
+
+# create secrets config map to store secrets and UUID 
+UUID=$(cat /proc/sys/kernel/random/uuid)
+BENCHMARKING_SECRETS=benchmarking-secrets
+STATUS_CHECK=$(kubectl get secrets  $BENCHMARKING_SECRETS | grep $BENCHMARKING_SECRETS )
+if [ -n "$STATUS_CHECK" ]
+then
+     kubectl delete secret $BENCHMARKING_SECRETS
+fi
+kubectl create secret generic $BENCHMARKING_SECRETS --from-literal=GUID=$UUID --from-literal=RESULT_STORAGE_CONNECTION_STRING=$storageconnstring --from-literal=COSMOS_URI=$cosmosuri --from-literal=COSMOS_KEY=$cosmosmkey
+
+# generate base deployment template 
+./generate-deploymnet-file.sh
+
+# create resources
+kubectl apply -f benchmarking-deployment-generated.yaml

cosmos/sql/tools/java/ycsb/aks/recipes/read/10-thousand-rps-read/generate-deploymnet-file.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+generatedfile=benchmarking-deployment-generated.yaml
+propfile="recipe-env-file.properties"
+
+rm -f /p/a/t/h $generatedfile
+
+function prop {
+    grep "${1}" ${propfile} | cut -d'=' -f2
+}
+
+podcount=$(prop 'POD_COUNT')
+memorylimit=$(prop 'MEMORY_LIMIT')
+cpulimit=$(prop 'CPU_LIMIT')
+value=`cat ../../../base/templates/benchmarking-deployment-template.yaml`
+
+for (( i=1; i <= $podcount; i++ ))
+do
+    echo "$value" | sed -e "s/<podcount>/$podcount/;s/<podindex>/$i/;s/<podname>/client$i/;s/<deploymentname>/benchmarking-$i-$podcount/;s/<memorylimit>/$memorylimit/;s/<cpulimit>/$cpulimit/"  >> $generatedfile
+    if [ $podcount -gt 1 ] && [ $i -lt $podcount ]
+    then
+        echo "---" >> $generatedfile
+    fi
+done

cosmos/sql/tools/java/ycsb/aks/recipes/read/10-thousand-rps-read/recipe-env-file.properties

@@ -0,0 +1,20 @@
+BENCHMARKING_FRAMEWORK_REPO=Azure/azure-db-benchmarking
+BENCHMARKING_FRAMEWORK_BRANCH=main
+YCSB_REPO=Azure/YCSB
+YCSB_BRANCH=main
+TARGET_OPERATIONS_PER_SECOND=5000
+THREAD_COUNT=12
+YCSB_OPERATION_COUNT=18000000
+YCSB_RECORD_COUNT=500
+USE_GATEWAY=false
+DIAGNOSTICS_LATENCY_THRESHOLD_IN_MS=-1
+REQUEST_DISTRIBUTION=uniform
+INSERT_ORDER=hashed
+INCLUDE_EXCEPTION_STACK=true
+FIELD_COUNT=10
+WRITE_ONLY_OPERATION=false
+SKIP_LOAD_PHASE=false
+POD_COUNT=2
+MEMORY_LIMIT=4G
+CPU_LIMIT=6000m
+PREFERRED_REGION_LIST=