diff --git a/.dockerignore b/.dockerignore index 2bff7dfe77..85625cd2b4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -22,4 +22,5 @@ antora.yml app.yaml .gae_deploy *-env.txt -.env +**/*.env +packages/serverless-orchestration/local-docker/bot-configs diff --git a/packages/serverless-orchestration/.gitignore b/packages/serverless-orchestration/.gitignore new file mode 100644 index 0000000000..e6926d7b7b --- /dev/null +++ b/packages/serverless-orchestration/.gitignore @@ -0,0 +1 @@ +local-docker/bot-configs/**/*.json diff --git a/packages/serverless-orchestration/README.md b/packages/serverless-orchestration/README.md index 0295cc886d..8653719988 100644 --- a/packages/serverless-orchestration/README.md +++ b/packages/serverless-orchestration/README.md @@ -9,3 +9,5 @@ The two serverless orchestration scripts are: 1. The `ServerlessHub` script which reads in a global configuration file stored and executes parallel serverless instances for each configured bot. This enables one global config file to define all bot instances. This drastically simplifying the devops and management overhead for spinning up new instances as this can be done by simply updating a single config file. 1. The `ServerlessSpoke` script which enables serverless functions to execute any arbitrary command from the UMA Docker container. This can be run on a local machine, within GCP cloud run or GCP cloud function environments. + +Please see the [instructions](./local-docker/README.md) on how to run serverless orchestration scripts when testing locally. diff --git a/packages/serverless-orchestration/local-docker/README.md b/packages/serverless-orchestration/local-docker/README.md new file mode 100644 index 0000000000..44752f4cbc --- /dev/null +++ b/packages/serverless-orchestration/local-docker/README.md @@ -0,0 +1,63 @@ +# Running UMA Serverless Orchestration Locally With Docker + +This document describes how to run the UMA Serverless Orchestration service locally using Docker. This is useful for +testing and debugging bots locally before deploying them to the cloud. + +The instructions below assume you have [Docker](https://www.docker.com/) and its Compose plugin installed and its server +daemon is running on the local machine. Also make sure to add your user to `docker` group in order to avoid running +commands as root. + +You also need to have `jq` installed on your machine as it is used for scripts parsing tested bot configuration files. + +## Build UMA Protocol Docker Image + +In order to build local UMA protocol docker image, run the build script: + +```sh +yarn workspace @uma/serverless-orchestration local-build +``` + +This will build two docker images: + +- `umaprotocol/protocol:local` for the UMA protocol +- `scheduler:local` for the cron scheduler that will trigger bots through local hub service + +## Service configuration + +In the `packages/serverless-orchestration/local-docker/` directory create the required `hub.env` and `spoke.env` files +using the provided templates in [`hub.env.template`](./hub.env.template) and [`spoke.env.template`](./spoke.env.template) +respectively. + +Place all the tested bot configuration files under the `packages/serverless-orchestration/local-docker/bot-configs/serverless-bots` +directory. Configuration files must be formatted as JSON and have a `.json` extension. + +In the [`./bot-configs`](./bot-configs) directory create the required `schedule.json` file using the provided template +in [`schedule.json.example`](./bot-configs/schedule.json.example). This configuration will be used by the cron scheduler +service to trigger bots through the local hub service. + +## Start UMA Serverless Orchestration + +To start the UMA Serverless Orchestration services run: + +```sh +yarn workspace @uma/serverless-orchestration local-up +``` + +This will start the following services: + +- `hub`: local hub service +- `spoke`: local spoke service +- `scheduler`: cron scheduler service + +## Stop UMA Serverless Orchestration + +To stop the UMA Serverless Orchestration services run: + +```sh +yarn workspace @uma/serverless-orchestration local-down +``` + +## Limitations + +Currently, the local UMA Serverless Orchestration does not support running bots that require access to GCP Datastore and +caching service. diff --git a/packages/serverless-orchestration/local-docker/bot-configs/schedule.json.example b/packages/serverless-orchestration/local-docker/bot-configs/schedule.json.example new file mode 100644 index 0000000000..565836fe0f --- /dev/null +++ b/packages/serverless-orchestration/local-docker/bot-configs/schedule.json.example @@ -0,0 +1,20 @@ +# Use this example to create schedule.json for scheduling configured bots. This should be structured as an array of +# objects, each object containing the following properties: +# schedule: A cron expression for when the bot should run. See https://crontab.guru/ for help creating cron expressions. +# bucket: The name of the directory under bot-configs where the bot configuration file is stored. +# configFile: The name of the file containing bot configurations to use for the scheduled run. +# Make sure the corresponding bot configuration file exists before updating the config. +# As an example, the following schedule.json will run the bots in serverless-bots/bot-config.json file every 5 minutes +# and the serverless-bots/bot-config-slow.json file every 30 minutes: +[ + { + "schedule": "*/5 * * * *", + "bucket": "serverless-bots", + "configFile": "bot-config.json" + }, + { + "schedule": "*/30 * * * *", + "bucket": "serverless-bots", + "configFile": "bot-config-slow.json" + } +] diff --git a/packages/serverless-orchestration/local-docker/docker-compose.yml b/packages/serverless-orchestration/local-docker/docker-compose.yml new file mode 100644 index 0000000000..79a93e757b --- /dev/null +++ b/packages/serverless-orchestration/local-docker/docker-compose.yml @@ -0,0 +1,18 @@ +version: "3.9" +services: + hub: + image: umaprotocol/protocol:local + build: ../../.. + env_file: + - hub.env + - bot-config.env + spoke: + image: umaprotocol/protocol:local + build: ../../.. + env_file: spoke.env + scheduler: + image: scheduler:local + build: scheduler + env_file: scheduler.env + environment: + - HUB_URL=http://hub:8080 diff --git a/packages/serverless-orchestration/local-docker/hub.env.template b/packages/serverless-orchestration/local-docker/hub.env.template new file mode 100644 index 0000000000..07d0c9b2c7 --- /dev/null +++ b/packages/serverless-orchestration/local-docker/hub.env.template @@ -0,0 +1,31 @@ +# Note on formatting: Do not enclose variable values in quotes. + +# Entry point for the hub service. +COMMAND=node packages/serverless-orchestration/src/ServerlessHub.js + +# Default spoke URL. Docker compose automatically adds all services to the same network, thus we can use the service +# name as the hostname. +SPOKE_URL=http://spoke:8080 + +# Spoke URLs for different spoke services. This is only required if the tested bot configuration includes spokeUrlName +# property. Since the docker-compose.yml file runs only one spoke service we point all named spoke URLs to the same +# spoke service. +SPOKE_URLS={"large":"http://spoke:8080","small":"http://spoke:8080"} + +# Provide fallback RPC node URL. +CUSTOM_NODE_URL= + +# Hub service name as it appears in logs. +BOT_IDENTIFIER=serverless-hub-mainnet + +# Hub configuration. Provide default timeout for how long to wait for spoke calls to respond. +HUB_CONFIG={"rejectSpokeDelay":900} + +# Stringified JSON configuration in the form of {"defaultWebHookUrl":""} where should be +# replaced with the URL of Slack webhook for the channel to which the hub should send notifications. +SLACK_CONFIG= + +# Stringified JSON configuration in the form of {"integrationKey":""} where +# should be replaced with the integration key of PagerDuty service to which the hub should +# send notifications. +PAGER_DUTY_V2_CONFIG= \ No newline at end of file diff --git a/packages/serverless-orchestration/local-docker/scheduler/Dockerfile b/packages/serverless-orchestration/local-docker/scheduler/Dockerfile new file mode 100644 index 0000000000..af15e203d1 --- /dev/null +++ b/packages/serverless-orchestration/local-docker/scheduler/Dockerfile @@ -0,0 +1,24 @@ +# Pulling Alpine image +FROM alpine:latest + +# Setting up work directory +WORKDIR /scheduler + +# Updating the packages +RUN apk update && \ +apk upgrade --available && sync + +# Installing curl and jq +RUN apk add curl +RUN apk add jq + +# Copying bot runner and entrypoint script into container +COPY run-bots.sh . +COPY entrypoint.sh . + +# Setting up permissions for the scripts +RUN chmod +x run-bots.sh +RUN chmod +x entrypoint.sh + +# Creating entry point to the script that parses environment and starts crond +ENTRYPOINT ["/scheduler/entrypoint.sh"] \ No newline at end of file diff --git a/packages/serverless-orchestration/local-docker/scheduler/entrypoint.sh b/packages/serverless-orchestration/local-docker/scheduler/entrypoint.sh new file mode 100644 index 0000000000..98cb76e5f0 --- /dev/null +++ b/packages/serverless-orchestration/local-docker/scheduler/entrypoint.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +# Parse the BOT_SCHEDULE environment variable and construct the crontab +echo "$BOT_SCHEDULE" | jq -r '.[] | [.schedule, .bucket, .configFile] | @tsv' | + while IFS=$'\t' read -r schedule bucket configFile + do echo "$schedule /scheduler/run-bots.sh $bucket $configFile $HUB_URL" >> /tmp/crontab + done + +crontab /tmp/crontab +crond -f diff --git a/packages/serverless-orchestration/local-docker/scheduler/run-bots.sh b/packages/serverless-orchestration/local-docker/scheduler/run-bots.sh new file mode 100644 index 0000000000..fa5451faf3 --- /dev/null +++ b/packages/serverless-orchestration/local-docker/scheduler/run-bots.sh @@ -0,0 +1,23 @@ +#!/bin/sh + +# Script to trigger bot execution from bucket $1 config file $2 against hub service URL $3 + +generate_post_data() +{ + cat <> "$2" diff --git a/packages/serverless-orchestration/local-docker/scripts/update-config.sh b/packages/serverless-orchestration/local-docker/scripts/update-config.sh new file mode 100644 index 0000000000..881d29f6af --- /dev/null +++ b/packages/serverless-orchestration/local-docker/scripts/update-config.sh @@ -0,0 +1,43 @@ +# Script to convert JSON bot-configs to environment variables passed to the hub service. +# Also creates environment for the scheduler service. + +# Resolve provided root directory. Defaults to current directory if not provided. +if [ -z "$1" ]; then + ROOT_DIR=$(pwd) +else + [ ! -d "$1" ] && { echo "Error: $1 is not a directory"; exit 1; } + ROOT_DIR=$(realpath "$1") +fi + +# Verify required files and directories exist. +[ ! -d "$ROOT_DIR/bot-configs" ] && { echo "Error: bot-configs directory not found"; exit 1; } +[ ! -f "$ROOT_DIR/scripts/json-to-env.sh" ] && { echo "Error: scripts/json-to-env.sh file not found"; exit 1; } + +# Update scheduler.env if bot-configs/schedule.json exists. Otherwise use empty environment. +CONFIG_DIR="$ROOT_DIR/bot-configs" +SCHEDULE_FILE="$CONFIG_DIR/schedule.json" +SCHEDULE_ENV_FILE="$ROOT_DIR/scheduler.env" +if [ -f "$SCHEDULE_FILE" ]; then + # Verify all array objects in schedule.json have required fields. + cat "$SCHEDULE_FILE" | jq -e '.[] | has("schedule") and has("bucket") and has("configFile")' > /dev/null + [ $? -ne 0 ] && { echo "Error: $SCHEDULE_FILE does not contain required fields"; exit 1; } + + # Verify all bucket/configFile values in schedule.json exist as bot-configs files. + cat "$SCHEDULE_FILE" | jq -r '.[] | [.bucket, .configFile] | @tsv' | + while read -r bucket configFile + do [ ! -f "$CONFIG_DIR/$bucket/$configFile" ] && { echo "Error: $CONFIG_DIR/$bucket/$configFile not found"; exit 1; } + done + + echo "Processing $SCHEDULE_FILE ..." + BOT_SCHEDULE=$(cat "$SCHEDULE_FILE" | jq tostring | sed -e 's/\\//g' -e 's/^\"//' -e 's/\"$//') +fi +echo "BOT_SCHEDULE=$BOT_SCHEDULE" > "$SCHEDULE_ENV_FILE" + +# Convert JSON files under bot-configs to bot-config.env file. +CONVERT_SCRIPT="$ROOT_DIR/scripts/json-to-env.sh" +CONFIG_ENV_FILE="$ROOT_DIR/bot-config.env" +echo "Processing JSON files in $CONFIG_DIR and storing environment in $CONFIG_ENV_FILE ..." +rm -f "$CONFIG_ENV_FILE" +find "$CONFIG_DIR" -name "*.json" -exec sh "$CONVERT_SCRIPT" {} "$CONFIG_ENV_FILE" \; + +exit $? diff --git a/packages/serverless-orchestration/local-docker/spoke.env.template b/packages/serverless-orchestration/local-docker/spoke.env.template new file mode 100644 index 0000000000..48be962e04 --- /dev/null +++ b/packages/serverless-orchestration/local-docker/spoke.env.template @@ -0,0 +1,22 @@ +# Note on formatting: Do not enclose variable values in quotes. + +# Entry point for the spoke service. +COMMAND=node packages/serverless-orchestration/src/ServerlessSpoke.js + +# Spoke service name as it appears in logs. +BOT_IDENTIFIER=serverless-spoke + +# Make sure spoke calls are not executed in a loop if this parameter is missing in bot configuration. +POLLING_DELAY=0 + +# Time in seconds to wait for logger to flush. +WAIT_FOR_LOGGER_DELAY=5 + +# Stringified JSON configuration in the form of {"defaultWebHookUrl":""} where should be +# replaced with the URL of Slack webhook for the channel to which the spoke should send notifications. +SLACK_CONFIG= + +# Stringified JSON configuration in the form of {"integrationKey":""} where +# should be replaced with the integration key of PagerDuty service to which the spoke should +# send notifications. +PAGER_DUTY_V2_CONFIG= \ No newline at end of file diff --git a/packages/serverless-orchestration/package.json b/packages/serverless-orchestration/package.json index 52c6b86009..2b956ff67c 100644 --- a/packages/serverless-orchestration/package.json +++ b/packages/serverless-orchestration/package.json @@ -41,7 +41,11 @@ "/src/**/*.js" ], "scripts": { - "test": "HARDHAT_NETWORK=localhost yarn mocha 'test/**/*.js'" + "test": "HARDHAT_NETWORK=localhost yarn mocha 'test/**/*.js'", + "local-build": "sh local-docker/scripts/docker-build.sh local-docker", + "local-config": "sh local-docker/scripts/update-config.sh local-docker", + "local-up": "sh local-docker/scripts/docker-up.sh local-docker", + "local-down": "sh local-docker/scripts/docker-down.sh local-docker" }, "bugs": { "url": "https://github.com/UMAprotocol/protocol/issues"