Merge pull request #6 from toncenter/develop

Improvements and fixes

Author: kdimentionaltree

Dockerfile

@@ -27,20 +27,20 @@ COPY playground/ /app/playground/
 COPY CMakeLists.txt /app/CMakeLists.txt
 COPY external/ /app/external/
 
-ARG TON_REPO
-ARG TON_BRANCH
-RUN if [ -n "$TON_REPO" ]; then \
-        echo "Using ton from ${TON_REPO}:${TON_BRANCH:-master}"; \
+ARG BUILD_WITH_TON_REPO
+ARG BUILD_WITH_TON_REPO
+RUN if [ -n "${BUILD_WITH_TON_REPO}" ]; then \
+        echo "Using ton from ${BUILD_WITH_TON_REPO}:${BUILD_WITH_TON_REPO:-master}"; \
         rm -rf /app/external/ton/ && \
-        git clone --recursive --branch ${TON_BRANCH:-master} ${TON_REPO} /app/external/ton; \
+        git clone --recursive --branch ${BUILD_WITH_TON_REPO:-master} ${BUILD_WITH_TON_REPO} /app/external/ton; \
     else \
         echo "Using ton submodule"; \
     fi
 
 WORKDIR /app/build
 RUN cmake -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 .. && make -j$(nproc) && make install
 RUN mkdir -p /root/.config/gdb && echo "set auto-load safe-path /" > /root/.config/gdb/gdbinit
-COPY ton-http-api/static/ /app/static/
+COPY ton-http-api/static/ /static/
 COPY config/static_config.yaml /app/static_config.yaml
 ENTRYPOINT [ "ton-http-api-cpp" ]
 # end builder
@@ -51,7 +51,7 @@ RUN DEBIAN_FRONTEND=noninteractive apt update -y \
     && apt install -y curl dnsutils libcurl4 libfmt9 libsodium23 libcctz2 libatomic1 libicu74 \
     && rm -rf /var/lib/apt/lists/*
 COPY --from=builder /app/build/ton-http-api/ton-http-api-cpp /usr/bin/
-COPY --from=builder /app/build/tonlib-multiclient/libtonlib_multiclient_lib.so /usr/lib
-COPY ton-http-api/static/ /app/static/
+COPY ton-http-api/static/ /static/
 COPY config/static_config.yaml /app/static_config.yaml
-ENTRYPOINT [ "ton-http-api-cpp" ]
+COPY scripts/entrypoint.sh /app/entrypoint.sh
+ENTRYPOINT [ "/app/entrypoint.sh" ]

README.md

@@ -56,17 +56,17 @@ There are two main ways to run TON HTTP API:
     ```
     - To run testnet version or connect to your private TON node, please specify:
     ```
-    export THACPP_GLOBAL_CONFIG_PATH=private/testnet.json
+    export THACPP_TONLIB_CONFIG_PATH=private/testnet.json
     # or
-    echo "THACPP_GLOBAL_CONFIG_PATH=private/testnet.json" > .env
+    echo "THACPP_TONLIB_CONFIG_PATH=private/testnet.json" > .env
     ```
     - To build an image using custom TON monorepo:
     ```
-    export TON_REPO=https://github.com/ton-blockchain/ton
-    export TON_BRANCH=testnet
+    export BUILD_WITH_TON_REPO=https://github.com/ton-blockchain/ton
+    export BUILD_WITH_TON_BRANCH=testnet
     # or 
-    echo "TON_REPO=https://github.com/ton-blockchain/ton" >> .env
-    export TON_BRANCH=testnet
+    echo "BUILD_WITH_TON_REPO=https://github.com/ton-blockchain/ton" >> .env
+    echo "BUILD_WITH_TON_BRANCH=testnet" >> .env
     ```
 - Pull or build docker image:
     ```bash
@@ -89,6 +89,34 @@ There are two main ways to run TON HTTP API:
     docker compose logs -f -n 100 http-api-cpp
     ```
 
+### Configure via environment variables
+Instead of using `config_vars.yaml` file, you can also configure service via environment variables.
+To do this you need to set environment variables before starting the service and use the following command:
+  ```bash
+  docker compose -f docker-compose.env-vars.yaml up -d
+  ```
+
+List of available environment variables:
+- `THACPP_TONLIB_CONFIG_PATH` - TON global config path (default: `/run/secrets/ton-global-config`)
+- `THACPP_TONLIB_KEYSTORE_PATH` - TONlib keystore path (default: `/tmp/keystore/`)
+- `THACPP_TONLIB_BOC_ENDPOINTS` - Endpoints to duplicate incoming BOCs; should be a list of comma-separated quoted strings like `["http://127.0.0.1:8080", "https://endpoint.com/postBoc"]` (default: `[]`)
+- `THACPP_TONLIB_THREADS` - number of threads for TONlib multiclient (default: `4`)
+- `THACPP_PORT` - API port in container (default: `8081`)
+- `THACPP_MONITOR_PORT` - Monitoring port in container (default: `8082`)
+- `THACPP_MAIN_WORKER_THREADS` - number of threads to serve HTTP requests (default: `4`)
+- `THACPP_FS_WORKER_THREADS` - number of threads for I/O operations (default: `1`)
+- `THACPP_HTTP_WORKER_THREADS` - number of threads for HTTP client (used to duplicate BOCs on external service) (default: `2`)
+- `THACPP_LOG_LEVEL` - api v2 log level (default: `warning`)
+- `THACPP_LOG_PATH` - log output destination (`@stdout`, `@stderr`, `@null`, or `/path/to/file`) (default: `"@stdout"`)
+- `THACPP_SYSTEM_LOG_LEVEL` - userver system logs level (default: `warning`)
+- `THACPP_SYSTEM_LOG_PATH` - userver system log path (default: `"@stdout"`)
+- `THACPP_JSONRPC_LOG_LEVEL` - jsonrpc endpoint log level (default: `warning`)
+- `THACPP_JSONRPC_LOG_PATH` - jsonrpc endpoint log path (default: `"@stdout"`)
+- `THACPP_LOG_FORMAT` - logs format (`tskv`, `ltsv`, `json`) (default: `json`)
+- `THACPP_HTTP_WORKER_USER_AGENT` - HTTP user agent sent to BOC endpoint (default: `empty`)
+- `THACPP_STATIC_CONTENT_DIR` - directory for static content served by API (default: `"/static/"`)
+- `THACPP_MAX_STACK_ENTRY_DEPTH` - max stack entry depth for runGetMethod (higher values increase memory usage) (default: `256`)
+
 ### Local run
 
 - Install dependencies:

config/config_vars.yaml

@@ -1,37 +1,83 @@
-# TONlib parameters
-tonlib_config_path: /run/secrets/ton-global-config  # TON global config path 
-tonlib_keystore_path: /tmp/keystore/  # TONlib keystore path
-tonlib_boc_endpoints: []  # Endpoints to duplicate incoming BOCs
-tonlib_threads: 4  # number of threads for TONlib multiclient
-
-server_port: 8081   # API port in container,
-                    # to change exposed port set THACPP_PORT env variable
-monitor_port: 8082  # Monitoring port in container, 
-                    # to change exposed port set THACPP_MONITOR_PORT env variable
-
-main_worker_threads: 4  # number of threads to serve HTTP requests
-fs_worker_threads: 1    # number of threads for I/O operations, f.e. logging
-http_worker_threads: 2  # number of threads for http client 
-                        # to duplicate BOCs on external service
-
-log_level: warning          # api v2 log level
-log_path: "@stdout"         # log destination, available options:
-                            # @stdout
-                            # @stderr
-                            # @null - don't log
-                            # /path/to/file - log to file
-                            # Note: if you log into file,
-                            # logs will be deleted after deploying new container
-system_log_level: warning   # userver system logs
+##### TONlib parameters #####
+# TON global config path
+# THACPP_TONLIB_CONFIG_PATH
+tonlib_config_path: /run/secrets/ton-global-config
+# TONlib keystore path
+# THACPP_TONLIB_KEYSTORE_PATH
+tonlib_keystore_path: /tmp/keystore/
+# Endpoints to duplicate incoming BOCs
+# should be a list of comma-separated, quoted strings in format `http://host:port`
+# example: ["http://127.0.0.1:8080", "https://endpoint.com/postBoc"]
+# service is expected to accept POST requests with BOC in body and return 200 in case of success
+# request body will have the following format: `{"boc":"base64 encoded boc"}`
+# THACPP_TONLIB_BOC_ENDPOINTS
+tonlib_boc_endpoints: []
+# number of threads for TONlib multiclient
+# THACPP_TONLIB_THREADS
+tonlib_threads: 4
+
+##### Userver parameters #####
+# API port in container
+# THACPP_PORT
+server_port: 8081
+# Monitoring port in container
+# THACPP_MONITOR_PORT
+monitor_port: 8082
+
+# number of threads to serve HTTP requests
+# THACPP_MAIN_WORKER_THREADS
+main_worker_threads: 4
+# number of threads for I/O operations, f.e. logging
+# THACPP_FS_WORKER_THREADS
+fs_worker_threads: 1
+# number of threads for http client
+# to duplicate BOCs on external service
+# THACPP_HTTP_WORKER_THREADS
+http_worker_threads: 2
+
+##### Logging parameters #####
+# for log_path destination use one available options:
+# @stdout
+# @stderr
+# @null - don't log
+# /path/to/file - log to file
+# Note: if you log into file,
+# logs will be deleted after deploying new container
+
+# api v2 log level
+# THACPP_LOG_LEVEL
+log_level: warning
+# THACPP_LOG_PATH
+log_path: "@stdout"
+
+# userver system logs
+# THACPP_SYSTEM_LOG_LEVEL
+system_log_level: warning
+# THACPP_SYSTEM_LOG_PATH
 system_log_path: "@stdout"
 
-jsonrpc_log_level: warning  # jsonrpc endpoint logs
-                            # note that requests to jsonRPC endpoint will appear in both of api-v2 and jsonrpc loggers
+# jsonrpc endpoint logs
+# note that requests to jsonRPC endpoint will appear in both of api-v2 and jsonrpc loggers
+# THACPP_JSONRPC_LOG_LEVEL
+jsonrpc_log_level: warning
+# THACPP_JSONRPC_LOG_PATH
 jsonrpc_log_path: "@stdout"
 
-log_format: json            # logs format, one of `tskv`, `ltsv`, `json`
+# logs format, one of `tskv`, `ltsv`, `json`
+# THACPP_LOG_FORMAT
+log_format: json
+
+# http user agent to set in request to boc endpoint
+# this may be required for external services with anti-ddos and anti-bot protection
+# THACPP_HTTP_WORKER_USER_AGENT
+http_worker_user_agent: empty
 
-http_worker_user_agent: empty  # http user agent to set in request to boc endpoint
+# directory for static content
+# static content will be served from this directory
+# THACPP_STATIC_CONTENT_DIR
+static_content_dir: /static/
 
-static_content_dir: "/static/"  # directory for static content
-max_stack_entry_depth: 100      # max stack entry depth for runGetMethod
+# max stack entry depth for runGetMethod
+# increasing this value may increase memory consumption in runGetMethods
+# THACPP_MAX_STACK_ENTRY_DEPTH
+max_stack_entry_depth: 256

config/static_config.yaml

@@ -3,7 +3,7 @@ components_manager:
     initial_size: 1000
     max_size: 10000
     stack_size: 4194304
-    stack_usage_monitor_enabled: true
+    stack_usage_monitor_enabled: false
   event_thread_pool:
     threads: 2
   task_processors:

docker-compose.env-vars.yaml

@@ -0,0 +1,41 @@
+services:
+  http-api-cpp:
+    image: toncenter/ton-http-api-cpp:latest
+    build:
+      context: .
+      dockerfile: Dockerfile
+      target: http-api-cpp
+      args:
+        - BUILD_WITH_TON_REPO
+        - BUILD_WITH_TON_BRANCH
+    ports:
+      - ${THACPP_PORT:-8081}:8081
+      - ${THACPP_MONITOR_PORT:-8082}:8082
+    environment:
+      - THACPP_TONLIB_CONFIG_PATH
+      - THACPP_TONLIB_KEYSTORE_PATH
+      - THACPP_TONLIB_BOC_ENDPOINTS
+      - THACPP_TONLIB_THREADS
+      - THACPP_MAIN_WORKER_THREADS
+      - THACPP_FS_WORKER_THREADS
+      - THACPP_HTTP_WORKER_THREADS
+      - THACPP_LOG_LEVEL
+      - THACPP_LOG_PATH
+      - THACPP_SYSTEM_LOG_LEVEL
+      - THACPP_SYSTEM_LOG_PATH
+      - THACPP_JSONRPC_LOG_LEVEL
+      - THACPP_JSONRPC_LOG_PATH
+      - THACPP_LOG_FORMAT
+      - THACPP_HTTP_WORKER_USER_AGENT
+      - THACPP_STATIC_CONTENT_DIR
+      - THACPP_MAX_STACK_ENTRY_DEPTH
+    restart: unless-stopped
+#    secrets:
+#      - ton-global-config
+    ulimits:
+      memlock: -1
+      stack: -1
+      core: -1
+#secrets:
+#  ton-global-config:
+#    file: ${THACPP_GLOBAL_CONFIG_PATH:-private/mainnet.json}

docker-compose.yaml

@@ -6,12 +6,14 @@ services:
       dockerfile: Dockerfile
       target: http-api-cpp
       args:
-        - TON_REPO
-        - TON_BRANCH
-    command: --config /app/static_config.yaml --config_vars /run/secrets/config-vars
+        - BUILD_WITH_TON_REPO
+        - BUILD_WITH_TON_BRANCH
     ports:
       - ${THACPP_PORT:-8081}:8081
       - ${THACPP_MONITOR_PORT:-8082}:8082
+    environment:
+      - THACPP_CONFIG_VARS=/run/secrets/config-vars
+      - THACPP_TONLIB_CONFIG_PATH=/run/secrets/ton-global-config
     restart: unless-stopped
     secrets:
       - ton-global-config
@@ -22,6 +24,6 @@ services:
       core: -1
 secrets:
   ton-global-config:
-    file: ${THACPP_GLOBAL_CONFIG_PATH:-private/mainnet.json}
+    file: ${THACPP_TONLIB_CONFIG_PATH:-private/mainnet.json}
   config-vars:
     file: ${THACPP_CONFIG_VARS:-private/config_vars.yaml}

scripts/entrypoint.sh

@@ -0,0 +1,84 @@
+#!/bin/bash
+set -e
+
+# config path
+THACPP_STATIC_CONFIG_PATH=${THACPP_STATIC_CONFIG_PATH:-/app/static_config.yaml}
+if [[ ! -f "$THACPP_STATIC_CONFIG_PATH" ]]; then
+  echo "Static config file '$THACPP_STATIC_CONFIG_PATH' not found}"
+  exit 1
+fi
+PARAMS="--config $THACPP_STATIC_CONFIG_PATH"
+
+# config vars
+USE_ENV_VARS=${USE_ENV_VARS:-0}
+THACPP_CONFIG_VARS=${THACPP_CONFIG_VARS:-/app/config_vars.yaml}
+if [[ ! -f "$THACPP_CONFIG_VARS" ]]; then
+  echo "Config vars file path not found. Config will be rendered from env vars"
+  USE_ENV_VARS=1
+fi
+PARAMS="$PARAMS --config_vars $THACPP_CONFIG_VARS"
+
+if [[ "$USE_ENV_VARS" -eq 1 ]]; then
+  echo "Rendering config from env vars"
+  if [[ "$THACPP_TONLIB_CONFIG_PATH" == "http://"* || "$THACPP_TONLIB_CONFIG_PATH" == "https://"* ]]; then
+    echo "Downloading TON network config from $THACPP_TONLIB_CONFIG_PATH"
+    curl -L --fail -o "/app/ton-config" "$THACPP_TONLIB_CONFIG_PATH"
+    THACPP_TONLIB_CONFIG_PATH=/app/ton-config
+  fi
+  cat <<EOF > /app/config_vars.yaml
+tonlib_config_path: ${THACPP_TONLIB_CONFIG_PATH:-/run/secrets/ton-global-config}
+tonlib_keystore_path: ${THACPP_TONLIB_KEYSTORE_PATH:-/tmp/keystore/}
+tonlib_boc_endpoints: ${THACPP_TONLIB_BOC_ENDPOINTS:-[]}
+tonlib_threads: ${THACPP_TONLIB_THREADS:-4}
+
+server_port: 8081
+monitor_port: 8082
+
+main_worker_threads: ${THACPP_MAIN_WORKER_THREADS:-4}
+fs_worker_threads: ${THACPP_FS_WORKER_THREADS:-1}
+http_worker_threads: ${THACPP_HTTP_WORKER_THREADS:-2}
+
+log_level: ${THACPP_LOG_LEVEL:-warning}
+log_path: "${THACPP_LOG_PATH:-@stdout}"
+
+system_log_level: ${THACPP_SYSTEM_LOG_LEVEL:-warning}
+system_log_path: "${THACPP_SYSTEM_LOG_PATH:-@stdout}"
+
+jsonrpc_log_level: ${THACPP_JSONRPC_LOG_LEVEL:-warning}
+jsonrpc_log_path: "${THACPP_JSONRPC_LOG_PATH:-@stdout}"
+
+log_format: ${THACPP_LOG_FORMAT:-json}
+
+http_worker_user_agent: ${THACPP_HTTP_WORKER_USER_AGENT:-empty}
+
+static_content_dir: ${THACPP_STATIC_CONTENT_DIR:-/static/}
+
+max_stack_entry_depth: ${THACPP_MAX_STACK_ENTRY_DEPTH:-256}
+EOF
+  echo "Config:"
+  cat "$THACPP_CONFIG_VARS"
+fi
+
+# params ready
+echo "params: $PARAMS"
+
+# check global config
+TONLIB_CONFIG_PATH=$(grep '^tonlib_config_path:' "$THACPP_CONFIG_VARS" | awk '{print $2}')
+
+if [[ ! -f "$TONLIB_CONFIG_PATH" ]]; then
+  echo "TON global config '$TONLIB_CONFIG_PATH' not found"
+  exit 1
+fi
+
+# create keystore path
+TONLIB_KEYSTORE_PATH=$(grep '^tonlib_keystore_path:' "$THACPP_CONFIG_VARS" | awk '{print $2}')
+mkdir -p "$TONLIB_KEYSTORE_PATH"
+
+# check static file dir exists
+STATIC_FILES_PATH=$(grep '^static_content_dir:' "$THACPP_CONFIG_VARS" | awk '{print $2}')
+if [[ ! -d "$STATIC_FILES_PATH" ]]; then
+  echo "Config with WWW content not found: '$STATIC_FILES_PATH'"
+  exit 1
+fi
+
+exec ton-http-api-cpp $PARAMS "$@"