Merge pull request #4 from toncenter/develop

Added support for custom TON submodule

Author: kdimentionaltree

.dockerignore

@@ -219,3 +219,4 @@ sandbox/
 .vscode/
 .idea/
 .DS_Store
+.git/

Dockerfile

@@ -10,26 +10,36 @@ RUN DEBIAN_FRONTEND=noninteractive apt update -y \
     libhiredis-dev libidn11-dev libjemalloc2 libjemalloc-dev libkrb5-dev libldap2-dev liblz4-dev \
      libnghttp2-dev libpugixml-dev libsnappy-dev libsasl2-dev libssl-dev libxxhash-dev libyaml-cpp0.8  libyaml-cpp-dev \
     libzstd-dev libssh2-1-dev netbase python3-dev python3-jinja2 python3-venv python3-yaml \
-    ragel yasm zlib1g-dev liblzma-dev libre2-dev clang-format gcc g++ yq \
+    ragel yasm zlib1g-dev liblzma-dev libre2-dev clang-format gcc g++ yq gdb \
     && rm -rf /var/lib/apt/lists/*
 
 ENV CC=/usr/bin/clang
 ENV CXX=/usr/bin/clang++
 ENV CCACHE_DISABLE=1
-ENV USERVER_FEATURE_CRYPTOPP_BLAKE2=0
 ENV BUILD_TON_PLAYGROUND=1
+ENV USERVER_FEATURE_STACK_USAGE_MONITOR=1
 
 COPY examples/ /app/examples/
 COPY py/ /app/py/
-COPY external/ /app/external/
 COPY tonlib-multiclient/ /app/tonlib-multiclient/
 COPY ton-http-api/ /app/ton-http-api/
 COPY playground/ /app/playground/
 COPY CMakeLists.txt /app/CMakeLists.txt
+COPY external/ /app/external/
+
+ARG TON_REPO
+ARG TON_BRANCH
+RUN if [ -n "$TON_REPO" ]; then \
+        echo "Using ton from ${TON_REPO}:${TON_BRANCH:-master}"; \
+        rm -rf /app/external/ton/ && \
+        git clone --recursive --branch ${TON_BRANCH:-master} ${TON_REPO} /app/external/ton; \
+    else \
+        echo "Using ton submodule"; \
+    fi
 
 WORKDIR /app/build
 RUN cmake -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 .. && make -j$(nproc) && make install
-RUN apt update -y && apt install -y gdb && mkdir -p /root/.config/gdb && echo "set auto-load safe-path /" > /root/.config/gdb/gdbinit
+RUN mkdir -p /root/.config/gdb && echo "set auto-load safe-path /" > /root/.config/gdb/gdbinit
 COPY ton-http-api/static/ /app/static/
 COPY config/static_config.yaml /app/static_config.yaml
 ENTRYPOINT [ "ton-http-api-cpp" ]

README.md

@@ -59,6 +59,15 @@ There are two main ways to run TON HTTP API:
     export THACPP_GLOBAL_CONFIG_PATH=private/testnet.json
     # or
     echo "THACPP_GLOBAL_CONFIG_PATH=private/testnet.json" > .env
+    ```
+    - To build an image using custom TON monorepo:
+    ```
+    export TON_REPO=https://github.com/ton-blockchain/ton
+    export TON_BRANCH=testnet
+    # or 
+    echo "TON_REPO=https://github.com/ton-blockchain/ton" >> .env
+    export TON_BRANCH=testnet
+    ```
 - Pull or build docker image:
     ```bash
     docker compose pull

config/static_config.yaml

@@ -1,4 +1,11 @@
 components_manager:
+  coro_pool:
+    initial_size: 1000
+    max_size: 10000
+    stack_size: 4194304
+    stack_usage_monitor_enabled: true
+  event_thread_pool:
+    threads: 2
   task_processors:
     main-task-processor:
       worker_threads: $main_worker_threads
@@ -19,7 +26,7 @@ components_manager:
         port#fallback: 8081
         connection:
           in_buffer_size: 1048576
-          requests_queue_size_threshold: 1024
+          requests_queue_size_threshold: 16384
         task_processor: main-task-processor
         handler-defaults:
           max_url_size: 8192
@@ -66,20 +73,27 @@ components_manager:
       keystore#fallback: /tmp/keystore/
       threads: $tonlib_threads
       threads#fallback: 4
-      # external_message_endpoints: $tonlib_boc_endpoints
-      # external_message_endpoints#fallback: []
       task_processor: main-task-processor
     dns-client:
       fs-task-processor: fs-task-processor
     http-client:
-      pool-statistics-disable: false
+      pool-statistics-disable: true
       thread-name-prefix: http-client
       threads: $http_worker_threads
       threads#fallback: 2
       fs-task-processor: fs-task-processor
       destination-metrics-auto-max-size: 100
       user-agent: $http_worker_user_agent
       user-agent#fallback: empty
+    jsonrpc-http-client:
+      pool-statistics-disable: true
+      thread-name-prefix: jsonrpc-http-client
+      threads: $http_worker_threads
+      threads#fallback: 2
+      fs-task-processor: fs-task-processor
+      destination-metrics-auto-max-size: 0
+      user-agent: $http_worker_user_agent
+      user-agent#fallback: empty
     fs-cache-main:
       dir: $static_content_dir
       dir#fallback: /app/static/
@@ -342,6 +356,7 @@ components_manager:
       logger: api-v2-jsonrpc
       port: $server_port
       port#fallback: 8081
+      log-level: error
 
     #
     # service handlers

docker-compose.yaml

@@ -5,6 +5,9 @@ services:
       context: .
       dockerfile: Dockerfile
       target: http-api-cpp
+      args:
+        - TON_REPO
+        - TON_BRANCH
     command: --config /app/static_config.yaml --config_vars /run/secrets/config-vars
     ports:
       - ${THACPP_PORT:-8081}:8081

ton-http-api/src/converters/transactions.hpp

@@ -111,11 +111,16 @@ inline schemas::v2::Message Convert<schemas::v2::Message>(
         auto r_body_cell = vm::std_boc_deserialize(data.body_);
         if (r_body_cell.is_ok()) {
           auto body_cell = r_body_cell.move_as_ok();
-          vm::CellSlice cs = vm::load_cell_slice(body_cell);
-          if (auto r_body = td::hex_decode(cs.as_bitslice().to_hex()); r_body.is_ok()) {
-            result.message = td::base64_encode(r_body.move_as_ok());
-          } else {
-            result.message_decode_error = r_body.move_as_error_suffix("failed to decode msg body hex: ").to_string();
+          auto r_loaded_cell = body_cell->load_cell();
+          if (r_loaded_cell.is_ok()) {
+            auto loaded_cell = r_loaded_cell.move_as_ok();
+            const size_t n_bytes = (loaded_cell.data_cell->get_bits() + 7) / 8;
+            const size_t n_trailing_bits = 8 - loaded_cell.data_cell->get_bits() % 8;
+            std::string buffer(n_bytes, 0);
+            std::memcpy(buffer.data(), loaded_cell.data_cell->get_data(), n_bytes);
+            // this zeroes last 1 bit which is 1 in C++ and 0 in python implementation
+            buffer[buffer.length() - 1] = static_cast<char>(buffer.back() & ~static_cast<char>((1 << n_trailing_bits) - 1));
+            result.message = td::base64_encode(buffer) + "\n"; // this is back compatibility
           }
         } else {
           result.message_decode_error = "Failed to load cell or get body slice. But why?";

ton-http-api/src/handlers/JsonRpcHandler.cpp

@@ -2,6 +2,7 @@
 
 #include "schemas/v2.hpp"
 #include "userver/clients/http/component.hpp"
+#include <userver/server/component.hpp>
 #include "utils/exceptions.hpp"
 
 namespace ton_http::handlers {
@@ -13,7 +14,7 @@ JsonRpcHandler::JsonRpcHandler(
     logger_(context.FindComponent<userver::components::Logging>().GetLogger(
       config["logger"].As<std::string>("api-v2-jsonrpc")
     )),
-    http_client_(context.FindComponent<userver::components::HttpClient>("http-client").GetHttpClient()),
+    http_client_(context.FindComponent<userver::components::HttpClient>("jsonrpc-http-client").GetHttpClient()),
     base_url_("http://localhost:" + std::to_string(config["port"].As<std::int32_t>()) + "/api/v2/") {
 }
 std::string JsonRpcHandler::HandleRequestThrow(

ton-http-api/src/handlers/runmethod/RunGetMethodHandler.cpp

@@ -49,7 +49,7 @@ td::Result<ton_http::schemas::v2::RunGetMethodResult> ton_http::handlers::RunGet
   );
   auto result_object = converters::Convert(result);
   if (!checkStackDepth(result_object.stack)) {
-    throw utils::TonlibException{"result stack depth >= " + std::to_string(max_stack_entry_depth_), 533};
+    return td::Status::Error(533, "Result stack depth >= " + std::to_string(max_stack_entry_depth_));
   }
   return result_object;
 }

ton-http-api/src/handlers/runmethod/RunGetMethodStdHandler.cpp

@@ -49,7 +49,7 @@ ton_http::handlers::RunGetMethodStdHandler::HandleRequestTonlibThrow(
   );
   auto result_object = converters::Convert<schemas::v2::RunGetMethodStdResult>(result.result);
   if (!checkStackDepth(result_object.stack)) {
-    throw utils::TonlibException{"Result stack depth >= " + std::to_string(max_stack_entry_depth_), 533};
+    return td::Status::Error(533, "Result stack depth >= " + std::to_string(max_stack_entry_depth_));
   }
   return result_object;
 }

ton-http-api/src/main.cpp

@@ -53,6 +53,7 @@ int main(int argc, char* argv[]) {
   // components
   component_list.Append<userver::clients::dns::Component>();
   component_list.Append<userver::components::HttpClient>();
+  component_list.Append<userver::components::HttpClient>("jsonrpc-http-client");
   component_list.Append<userver::components::FsCache>("fs-cache-main");
   component_list.Append<ton_http::core::TonlibComponent>();
   // common handlers