feat: FunC as deamon

crypto/CMakeLists.txt

@@ -412,6 +412,18 @@ if (USE_EMSCRIPTEN)
   target_compile_options(funcfiftlib PRIVATE -fexceptions -fno-stack-protector)
 endif()
 
+# New target: func executable linked with fift-lib
+add_executable(funcplusfift funcfiftlib/funcfiftlib.cpp funcfiftlib/funcfiftlib-main.cpp ${FUNC_LIB_SOURCE})
+target_include_directories(funcplusfift PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>)
+# Link fift-lib, ton_crypto, src_parser, git
+target_link_libraries(funcplusfift PUBLIC fift-lib ton_crypto src_parser git)
+if (WINGETOPT_FOUND)
+  target_link_libraries_system(funcplusfift wingetopt)
+endif()
+
+# Install the new executable
+install(TARGETS funcplusfift RUNTIME DESTINATION bin)
+
 add_executable(tlbc tl/tlbc.cpp)
 target_include_directories(tlbc PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>)
 target_link_libraries(tlbc PUBLIC src_parser)
@@ -448,14 +460,14 @@ file(MAKE_DIRECTORY smartcont/auto)
 if (NOT CMAKE_CROSSCOMPILING OR USE_EMSCRIPTEN)
   set(GENERATE_TLB_CMD tlbc)
   if (NOT USE_EMSCRIPTEN)
-    add_custom_command(
-      WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/block
-      COMMAND ${TURN_OFF_LSAN}
-      COMMAND ${GENERATE_TLB_CMD} -o block-auto -n block::gen -z block.tlb
-      COMMENT "Generate block tlb source files"
-      OUTPUT ${TLB_BLOCK_AUTO}
-      DEPENDS tlbc block/block.tlb
-    )
+#    add_custom_command(
+#      WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/block
+#      COMMAND ${TURN_OFF_LSAN}
+#      COMMAND ${GENERATE_TLB_CMD} -o block-auto -n block::gen -z block.tlb
+#      COMMENT "Generate block tlb source files"
+#      OUTPUT ${TLB_BLOCK_AUTO}
+#      DEPENDS tlbc block/block.tlb
+#    )
   else()
     add_custom_command(
       WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/block

crypto/func/builtins.cpp

@@ -27,10 +27,17 @@ using namespace std::literals::string_literals;
  * 
  */
 
-int glob_func_cnt, undef_func_cnt, glob_var_cnt, const_cnt;
-std::vector<SymDef*> glob_func, glob_vars;
+int stdlib_glob_func_cnt, glob_func_cnt, undef_func_cnt, glob_var_cnt, const_cnt;
+std::vector<SymDef*> stdlib_glob_func, glob_func, glob_vars;
 std::set<std::string> prohibited_var_names;
 
+void reset_builtins() {
+  glob_func_cnt = undef_func_cnt = glob_var_cnt = const_cnt = 0;
+  glob_func.clear();
+  glob_vars.clear();
+  prohibited_var_names.clear();
+}
+
 SymDef* predefine_builtin_func(std::string name, TypeExpr* func_type) {
   if (name.back() == '_') {
     prohibited_var_names.insert(name);

crypto/func/func.cpp

@@ -31,6 +31,7 @@
 #include <getopt.h>
 #include "git.h"
 #include <fstream>
+#include <map>
 #include "td/utils/port/path.h"
 
 namespace funC {
@@ -156,6 +157,10 @@ void generate_output_func(SymDef* func_sym, std::ostream &outs, std::ostream &er
   }
 }
 
+// compile cache
+std::map<sym_idx_t, std::string> compile_cache;
+std::map<size_t, std::string> compile_cache2;
+
 int generate_output(std::ostream &outs, std::ostream &errs) {
   if (asm_preamble) {
     outs << "\"Asm.fif\" include\n";
@@ -175,21 +180,70 @@ int generate_output(std::ostream &outs, std::ostream &errs) {
       outs << func_val->method_id << " DECLMETHOD " << name << "\n";
     }
   }
+  // for (SymDef* func_sym : stdlib_glob_func) {
+  //   SymValCodeFunc* func_val = dynamic_cast<SymValCodeFunc*>(func_sym->value);
+  //   func_assert(func_val);
+  //   std::string name = sym::symbols.get_name(func_sym->sym_idx);
+  //   outs << std::string(indent * 2, ' ');
+  //   if (func_val->method_id.is_null()) {
+  //     outs << "DECLPROC " << name << "\n";
+  //   } else {
+  //     outs << func_val->method_id << " DECLMETHOD " << name << "\n";
+  //   }
+  // }
   for (SymDef* gvar_sym : glob_vars) {
     func_assert(dynamic_cast<SymValGlobVar*>(gvar_sym->value));
     std::string name = sym::symbols.get_name(gvar_sym->sym_idx);
     outs << std::string(indent * 2, ' ') << "DECLGLOBVAR " << name << "\n";
   }
   int errors = 0;
+  int cache_hits = 0;
+  int cache_misses = 0;
   for (SymDef* func_sym : glob_func) {
+    auto key = dynamic_cast<SymValCodeFunc*>(func_sym->value)->source_hash;
+    auto cached = compile_cache2.find(key);
+    if (cached != compile_cache2.end()) {
+      outs << cached->second;
+      cache_hits++;
+      continue;
+    }
+
     try {
-      generate_output_func(func_sym, outs, errs);
+      cache_misses++;
+      std::ostringstream outs2;
+      generate_output_func(func_sym, outs2, errs);
+
+      const auto res = outs2.str();
+      compile_cache2[key] = res;
+
+      outs << res;
     } catch (src::Error& err) {
       errs << "cannot generate code for function `" << sym::symbols.get_name(func_sym->sym_idx) << "`:\n"
                 << err << std::endl;
       ++errors;
     }
   }
+
+  std::cerr << cache_hits << "/" << cache_misses << std::endl;
+
+  // for (SymDef* func_sym : stdlib_glob_func) {
+  //   try {
+  //     auto cached = compile_cache.find(func_sym->sym_idx);
+  //     if (cached != compile_cache.end()) {
+  //       outs << cached->second;
+  //       continue;
+  //     }
+  //
+  //     std::ostringstream outs, errs;
+  //     generate_output_func(func_sym, outs, errs);
+  //
+  //     compile_cache.insert({func_sym->sym_idx, outs.str()});
+  //   } catch (src::Error& err) {
+  //     errs << "cannot generate code for function `" << sym::symbols.get_name(func_sym->sym_idx) << "`:\n"
+  //               << err << std::endl;
+  //     ++errors;
+  //   }
+  // }
   if (program_envelope) {
     outs << "}END>c\n";
   }

crypto/func/func.h

@@ -794,6 +794,8 @@ struct SymValFunc : SymVal {
 
 struct SymValCodeFunc : SymValFunc {
   CodeBlob* code;
+  std::string source;
+  size_t source_hash;
   ~SymValCodeFunc() override = default;
   SymValCodeFunc(int val, TypeExpr* _ft, bool _impure = false) : SymValFunc(val, _ft, _impure), code(nullptr) {
   }
@@ -845,8 +847,8 @@ struct SymValConst : sym::SymValBase {
   }
 };
 
-extern int glob_func_cnt, undef_func_cnt, glob_var_cnt;
-extern std::vector<SymDef*> glob_func, glob_vars;
+extern int stdlib_glob_func_cnt, glob_func_cnt, undef_func_cnt, glob_var_cnt;
+extern std::vector<SymDef*> stdlib_glob_func, glob_func, glob_vars;
 extern std::set<std::string> prohibited_var_names;
 
 /*
@@ -885,12 +887,15 @@ class ReadCallback {
 };
 
 // defined in parse-func.cpp
-bool parse_source(std::istream* is, const src::FileDescr* fdescr);
+bool parse_source(std::istream* is, std::string src, const src::FileDescr* fdescr);
 bool parse_source_file(const char* filename, src::Lexem lex = {}, bool is_main = false);
 bool parse_source_stdin();
 
 extern std::stack<src::SrcLocation> inclusion_locations;
 
+void reset_parser();
+void reset_builtins();
+
 /*
  * 
  *   EXPRESSIONS

crypto/func/parse-func.cpp

@@ -383,8 +383,15 @@ void parse_global_var_decls(Lexer& lex) {
 }
 
 SymValCodeFunc* make_new_glob_func(SymDef* func_sym, TypeExpr* func_type, bool impure = false) {
-  SymValCodeFunc* res = new SymValCodeFunc{glob_func_cnt, func_type, impure};
+  SymValCodeFunc* res = new SymValCodeFunc{stdlib_glob_func_cnt + glob_func_cnt, func_type, impure};
   func_sym->value = res;
+
+  // if (func_sym->loc.fdescr != nullptr && func_sym->loc.fdescr->is_stdlib) {
+  //   stdlib_glob_func.push_back(func_sym);
+  //   stdlib_glob_func_cnt++;
+  //   return res;
+  // }
+
   glob_func.push_back(func_sym);
   glob_func_cnt++;
   return res;
@@ -396,7 +403,7 @@ bool check_global_func(const Lexem& cur, sym_idx_t func_name = 0) {
   }
   SymDef* def = sym::lookup_symbol(func_name);
   if (!def) {
-    cur.loc.show_error(std::string{"undefined function `"} + symbols.get_name(func_name) +
+    cur.loc.show_error(std::string{"undefined function `"} + std::to_string((int)(func_name)) + "  " + symbols.get_name(func_name) +
                        "`, defining a global function of unknown type");
     def = sym::define_global_symbol(func_name, 0, cur.loc);
     func_assert(def && "cannot define global function");
@@ -1239,10 +1246,28 @@ CodeBlob* parse_func_body(Lexer& lex, FormalArgList arg_list, TypeExpr* ret_type
     parse_implicit_ret_stmt(lex, *blob);
   }
   blob->close_blk(lex.cur().loc);
+  blob->loc = SrcLocation(lex.cur().loc);
   lex.expect('}');
   return blob;
 }
 
+SrcLocation skip_func_body(Lexer& lex, FormalArgList arg_list, TypeExpr* ret_type) {
+  int brace_count = 0;
+  if (lex.tp() == '{') {
+    brace_count++;
+    lex.next();
+  }
+  while (brace_count > 0 && lex.tp() != _Eof) {
+    if (lex.tp() == '{') {
+      brace_count++;
+    } else if (lex.tp() == '}') {
+      brace_count--;
+    }
+    lex.next();
+  }
+  return SrcLocation(lex.cur().loc);
+}
+
 SymValAsmFunc* parse_asm_func_body(Lexer& lex, TypeExpr* func_type, const FormalArgList& arg_list, TypeExpr* ret_type,
                                    bool impure = false) {
   auto loc = lex.cur().loc;
@@ -1429,6 +1454,9 @@ TypeExpr* compute_type_closure(TypeExpr* expr, const std::vector<TypeExpr*>& typ
   return expr;
 }
 
+std::map<std::string, std::string> parsed;
+std::map<std::string, std::string> parsed_code;
+
 void parse_func_def(Lexer& lex) {
   SrcLocation loc{lex.cur().loc};
   sym::open_scope(lex);
@@ -1522,8 +1550,16 @@ void parse_func_def(Lexer& lex) {
     if (func_sym_code->code) {
       lex.cur().error("redefinition of function `"s + func_name.str + "`");
     }
-    CodeBlob* code = parse_func_body(lex, arg_list, ret_type);
+
+    CodeBlob* code;
+    std::string range;
+
+    code = parse_func_body(lex, arg_list, ret_type);
     code->name = func_name.str;
+
+    func_sym_code->source = std::move(lex.range(loc, code->loc));
+    func_sym_code->source_hash = std::hash<std::string>{}(func_sym_code->source);;
+
     code->loc = loc;
     // code->print(std::cerr);  // !!!DEBUG!!!
     func_sym_code->code = code;
@@ -1714,9 +1750,16 @@ void parse_pragma(Lexer& lex) {
 
 std::vector<const src::FileDescr*> source_fdescr;
 
+std::map<std::string, src::FileDescr*> stdlib_source_files;
 std::map<std::string, src::FileDescr*> source_files;
 std::stack<src::SrcLocation> inclusion_locations;
 
+void reset_parser() {
+  source_fdescr.clear();
+  source_files.clear();
+  while (!inclusion_locations.empty()) {inclusion_locations.pop();}
+}
+
 void parse_include(Lexer& lex, const src::FileDescr* fdescr) {
   auto include = lex.cur();
   lex.expect(_IncludeHashtag);
@@ -1735,8 +1778,8 @@ void parse_include(Lexer& lex, const src::FileDescr* fdescr) {
   }
 }
 
-bool parse_source(std::istream* is, src::FileDescr* fdescr) {
-  src::SourceReader reader{is, fdescr};
+bool parse_source(std::istream* is, std::string src, src::FileDescr* fdescr) {
+  src::SourceReader reader{is, fdescr, src};
   Lexer lex{reader, true, ";,()[] ~."};
   while (lex.tp() != _Eof) {
     if (lex.tp() == _PragmaHashtag) {
@@ -1764,6 +1807,10 @@ bool parse_source_file(const char* filename, src::Lexem lex, bool is_main) {
     }
   }
 
+  if (stdlib_source_files.find(filename) != stdlib_source_files.end()) {
+      return true;
+  }
+
   auto path_res = read_callback(ReadCallback::Kind::Realpath, filename);
   if (path_res.is_error()) {
     auto error = path_res.move_as_error();
@@ -1786,11 +1833,17 @@ bool parse_source_file(const char* filename, src::Lexem lex, bool is_main) {
   if (lex.tp) { // included
     funC::generated_from += std::string{"incl:"};
   }
+  bool is_stdlib = false; // real_filename.ends_with("stdlib.fc");
   funC::generated_from += std::string{"`"} + filename + "` ";
-  src::FileDescr* cur_source = new src::FileDescr{filename};
-  source_files[real_filename] = cur_source;
+  src::FileDescr* cur_source = new src::FileDescr{filename, false, is_stdlib};
+
+  if (is_stdlib) {
+    stdlib_source_files[real_filename] = cur_source;
+  } else {
+    source_files[real_filename] = cur_source;
+    source_fdescr.push_back(cur_source);
+  }
   cur_source->is_main = is_main;
-  source_fdescr.push_back(cur_source);
   auto file_res = read_callback(ReadCallback::Kind::ReadFile, filename);
   if (file_res.is_error()) {
     auto msg = file_res.move_as_error().message().str();
@@ -1803,7 +1856,7 @@ bool parse_source_file(const char* filename, src::Lexem lex, bool is_main) {
   auto file_str = file_res.move_as_ok();
   std::stringstream ss{file_str};
   inclusion_locations.push(lex.loc);
-  bool res = parse_source(&ss, cur_source);
+  bool res = parse_source(&ss, file_str, cur_source);
   inclusion_locations.pop();
   return res;
 }
@@ -1812,7 +1865,7 @@ bool parse_source_stdin() {
   src::FileDescr* cur_source = new src::FileDescr{"stdin", true};
   cur_source->is_main = true;
   source_fdescr.push_back(cur_source);
-  return parse_source(&std::cin, cur_source);
+  return parse_source(&std::cin, "", cur_source);
 }
 
 }  // namespace funC

crypto/funcfiftlib/funcfiftlib-main.cpp

@@ -0,0 +1,146 @@
+/*
+    This file is part of TON Blockchain source code.
+
+    TON Blockchain is free software; you can redistribute it and/or
+    modify it under the terms of the GNU General Public License
+    as published by the Free Software Foundation; either version 2
+    of the License, or (at your option) any later version.
+
+    TON Blockchain is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with TON Blockchain.  If not, see <http://www.gnu.org/licenses/>.
+
+    In addition, as a special exception, the copyright holders give permission
+    to link the code of portions of this program with the OpenSSL library.
+    You must obey the GNU General Public License in all respects for all
+    of the code used other than OpenSSL. If you modify file(s) with this
+    exception, you may extend this exception to your version of the file(s),
+    but you are not obligated to do so. If you do not wish to do so, delete this
+    exception statement from your version. If you delete this exception statement
+    from all source files in the program, then also delete it here.
+
+    Copyright 2017-2020 Telegram Systems LLP
+*/
+#include "funcfiftlib.h"
+#include "parser/symtable.h"
+
+#include <iostream>
+#include <string>
+#include <cstring>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <unistd.h>
+#include <func/func.h>
+#include <thread>
+#include <vector>
+#include <queue>
+#include <mutex>
+#include <condition_variable>
+
+#define PORT 8333
+
+int counter = 0;
+
+std::queue<int> task_queue;
+std::mutex queue_mutex;
+std::condition_variable condition;
+bool stop = false;
+
+void worker_function() {
+  while (true) {
+    int new_socket;
+    {
+      std::unique_lock<std::mutex> lock(queue_mutex);
+      condition.wait(lock, [] { return !task_queue.empty() || stop; });
+      if (stop && task_queue.empty()) return;
+      new_socket = task_queue.front();
+      task_queue.pop();
+    }
+    char buffer[1024] = {0};
+    read(new_socket, buffer, 1024);
+    const auto res = func_compile(buffer, nullptr);
+    send(new_socket, res, strlen(res), 0);
+    close(new_socket);
+
+    std::cout << res << std::endl;
+
+    counter++;
+    sym::reset_state();
+    funC::reset_parser();
+    funC::reset_builtins();
+  }
+}
+
+int main(const int argc, char **argv) {
+  if (argc < 2) {
+    std::cerr << "Usage: " << argv[0] << " <config.json> or --daemon or --workers <num_threads>" << std::endl;
+    return 1;
+  }
+
+  if (std::string(argv[1]) == "--daemon") {
+    int num_threads = 12;
+    std::vector<std::thread> workers;
+    for (int i = 0; i < num_threads; ++i) {
+      workers.emplace_back(worker_function);
+    }
+
+    int server_fd;
+    struct sockaddr_in address;
+    int opt = 1;
+    int addrlen = sizeof(address);
+
+    if ((server_fd = socket(AF_INET, SOCK_STREAM, 0)) == 0) {
+      perror("socket failed");
+      exit(EXIT_FAILURE);
+    }
+
+    if (setsockopt(server_fd, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt))) {
+      perror("setsockopt");
+      exit(EXIT_FAILURE);
+    }
+    address.sin_family = AF_INET;
+    address.sin_addr.s_addr = INADDR_ANY;
+    address.sin_port = htons(PORT);
+
+    if (bind(server_fd, (struct sockaddr *)&address, sizeof(address))<0) {
+      perror("bind failed");
+      exit(EXIT_FAILURE);
+    }
+    if (listen(server_fd, 3) < 0) {
+      perror("listen");
+      exit(EXIT_FAILURE);
+    }
+    while (true) {
+      int new_socket;
+      if ((new_socket = accept(server_fd, (struct sockaddr *)&address, (socklen_t*)&addrlen))<0) {
+        perror("accept");
+        exit(EXIT_FAILURE);
+      }
+      {
+        std::unique_lock<std::mutex> lock(queue_mutex);
+        task_queue.push(new_socket);
+      }
+      condition.notify_one();
+    }
+
+    {
+      std::unique_lock<std::mutex> lock(queue_mutex);
+      stop = true;
+    }
+    condition.notify_all();
+    for (std::thread &worker : workers) {
+      worker.join();
+    }
+    return 0;
+  }
+
+  // Default behavior: process a single config file
+  auto config_json = std::string(argv[1]);
+  const auto res = func_compile(config_json.data(), nullptr);
+  std::cout << res << std::endl;
+  return 0;
+}

crypto/funcfiftlib/funcfiftlib.cpp

@@ -34,12 +34,14 @@
 #include <sstream>
 #include <iomanip>
 #include "vm/boc.h"
+#include "funcfiftlib.h"
 
 td::Result<std::string> compile_internal(char *config_json) {
   TRY_RESULT(input_json, td::json_decode(td::MutableSlice(config_json)))
   td::JsonObject& config = input_json.get_object();
 
   TRY_RESULT(opt_level, td::get_json_object_int_field(config, "optLevel", false));
+  TRY_RESULT(fift_path, td::get_json_object_string_field(config, "fiftPath", false));
   TRY_RESULT(sources_obj, td::get_json_object_field(config, "sources", td::JsonValue::Type::Array, false));
 
   auto &sources_arr = sources_obj.get_array();
@@ -62,7 +64,7 @@ td::Result<std::string> compile_internal(char *config_json) {
     return td::Status::Error("FunC compilation error: " + errs.str());
   }
 
-  TRY_RESULT(fift_res, fift::compile_asm_program(outs.str(), "/fiftlib/"));
+  TRY_RESULT(fift_res, fift::compile_asm_program(outs.str(), fift_path));
 
   td::JsonBuilder result_json;
   auto obj = result_json.enter_object();
@@ -75,20 +77,6 @@ td::Result<std::string> compile_internal(char *config_json) {
   return result_json.string_builder().as_cslice().str();
 }
 
-/// Callback used to retrieve additional source files or data.
-///
-/// @param _kind The kind of callback (a string).
-/// @param _data The data for the callback (a string).
-/// @param o_contents A pointer to the contents of the file, if found. Allocated via malloc().
-/// @param o_error A pointer to an error message, if there is one. Allocated via malloc().
-///
-/// The callback implementor must use malloc() to allocate storage for
-/// contents or error. The callback implementor must use free() to free
-/// said storage after func_compile returns.
-///
-/// If the callback is not supported, *o_contents and *o_error must be set to NULL.
-typedef void (*CStyleReadFileCallback)(char const* _kind, char const* _data, char** o_contents, char** o_error);
-
 funC::ReadCallback::Callback wrapReadCallback(CStyleReadFileCallback _readCallback)
 {
   funC::ReadCallback::Callback readCallback;

crypto/funcfiftlib/funcfiftlib.h

@@ -0,0 +1,46 @@
+/*
+    This file is part of TON Blockchain source code.
+
+    TON Blockchain is free software; you can redistribute it and/or
+    modify it under the terms of the GNU General Public License
+    as published by the Free Software Foundation; either version 2
+    of the License, or (at your option) any later version.
+
+    TON Blockchain is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with TON Blockchain.  If not, see <http://www.gnu.org/licenses/>.
+
+    In addition, as a special exception, the copyright holders give permission
+    to link the code of portions of this program with the OpenSSL library.
+    You must obey the GNU General Public License in all respects for all
+    of the code used other than OpenSSL. If you modify file(s) with this
+    exception, you may extend this exception to your version of the file(s),
+    but you are not obligated to do so. If you do not wish to do so, delete this
+    exception statement from your version. If you delete this exception statement
+    from all source files in the program, then also delete it here.
+
+    Copyright 2017-2020 Telegram Systems LLP
+*/
+#pragma once
+
+/// Callback used to retrieve additional source files or data.
+///
+/// @param _kind The kind of callback (a string).
+/// @param _data The data for the callback (a string).
+/// @param o_contents A pointer to the contents of the file, if found. Allocated via malloc().
+/// @param o_error A pointer to an error message, if there is one. Allocated via malloc().
+///
+/// The callback implementor must use malloc() to allocate storage for
+/// contents or error. The callback implementor must use free() to free
+/// said storage after func_compile returns.
+///
+/// If the callback is not supported, *o_contents and *o_error must be set to NULL.
+typedef void (*CStyleReadFileCallback)(char const* _kind, char const* _data, char** o_contents, char** o_error);
+
+extern "C" {
+  const char *func_compile(char *config_json, CStyleReadFileCallback callback);
+}

crypto/parser/lexer.h

@@ -66,7 +66,7 @@ struct Lexem {
 };
 
 class Lexer {
-  SourceReader& src;
+public:SourceReader& src;
   bool eof;
   Lexem lexem, peek_lexem;
   unsigned char char_class[128];
@@ -109,6 +109,13 @@ class Lexer {
     return (classify_char(c) & cc::quote_char);
   }
 
+  std::string range(SrcLocation start, SrcLocation end) const {
+    if (this->src.src != "") {
+      return this->src.src.substr(start.char_offs, end.char_offs - start.char_offs);
+    }
+    return "";
+  }
+
  private:
   void set_spec(std::array<int, 3>& arr, std::string setup);
   bool is_multiline_quote(const char* begin, const char* end);

crypto/parser/srcread.cpp

@@ -140,8 +140,8 @@ void ParseError::show(std::ostream& os) const {
   where.show_context(os);
 }
 
-SourceReader::SourceReader(std::istream* _is, FileDescr* _fdescr)
-    : ifs(_is), fdescr(_fdescr), loc(_fdescr), eof(false), cur_line_len(0), start(0), cur(0), end(0) {
+SourceReader::SourceReader(std::istream* _is, FileDescr* _fdescr, std::string src)
+    : ifs(_is), fdescr(_fdescr), loc(_fdescr), eof(false), cur_line_len(0), start(0), cur(0), end(0), src(src) {
   load_line();
 }
 

crypto/parser/srcread.h

@@ -35,8 +35,9 @@ struct FileDescr {
   std::string text;
   std::vector<long> line_offs;
   bool is_stdin;
+  bool is_stdlib;
   bool is_main = false;
-  FileDescr(std::string _fname, bool _stdin = false) : filename(std::move(_fname)), is_stdin(_stdin) {
+  FileDescr(std::string _fname, bool _stdin = false, bool _is_stdlib = false) : filename(std::move(_fname)), is_stdin(_stdin), is_stdlib(_is_stdlib) {
   }
   const char* push_line(std::string new_line);
   const char* convert_offset(long offset, long* line_no, long* line_pos, long* line_size = nullptr) const;
@@ -55,6 +56,7 @@ std::ostream& operator<<(std::ostream& os, const Fatal& fatal);
 
 struct SrcLocation {
   const FileDescr* fdescr;
+  bool is_stdlib;
   long char_offs;
   SrcLocation() : fdescr(nullptr), char_offs(-1) {
   }
@@ -110,17 +112,18 @@ struct ParseError : Error {
 };
 
 class SourceReader {
-  std::istream* ifs;
+public:std::istream* ifs;
   FileDescr* fdescr;
   SrcLocation loc;
   bool eof;
   std::string cur_line;
   int cur_line_len;
   void set_eof();
   const char *start, *cur, *end;
+public: std::string src;
 
  public:
-  SourceReader(std::istream* _is, FileDescr* _fdescr);
+  SourceReader(std::istream* _is, FileDescr* _fdescr, std::string src = "");
   bool load_line();
   bool is_eof() const {
     return eof;

crypto/parser/symtable.cpp

@@ -34,9 +34,22 @@ SymTable<100003> symbols;
 
 SymDef* sym_def[symbols.hprime + 1];
 SymDef* global_sym_def[symbols.hprime + 1];
+SymDef* global_stdlib_sym_def[symbols.hprime + 1];
 std::vector<std::pair<int, SymDef>> symbol_stack;
 std::vector<src::SrcLocation> scope_opened_at;
 
+void reset_state() {
+  // symbols = SymTable<100003>();
+  scope_level = 1;
+  for (int i = 0; i < symbols.hprime + 1; i++) {
+    sym_def[i] = nullptr;
+    global_sym_def[i] = nullptr;
+  }
+
+  symbol_stack.clear();
+  scope_opened_at.clear();
+}
+
 std::string Symbol::unknown_symbol_name(sym_idx_t i) {
   if (!i) {
     return "_";
@@ -82,7 +95,7 @@ SymTableBase& SymTableBase::add_keyword(std::string str, sym_idx_t idx) {
   if (idx <= 0) {
     idx = ++def_kw;
   }
-  sym_idx_t res = gen_lookup(str, -1, idx);
+  sym_idx_t res = gen_lookup(str, 1, idx);
   if (!res) {
     throw SymTableKwRedef{str};
   }
@@ -133,8 +146,12 @@ SymDef* lookup_symbol(sym_idx_t idx, int flags) {
   if ((flags & 1) && sym_def[idx]) {
     return sym_def[idx];
   }
-  if ((flags & 2) && global_sym_def[idx]) {
-    return global_sym_def[idx];
+  if ((flags & 2)) {
+    auto global = global_sym_def[idx];
+    if (global) {
+      return global;
+    }
+    return global_stdlib_sym_def[idx];
   }
   return nullptr;
 }
@@ -147,6 +164,15 @@ SymDef* define_global_symbol(sym_idx_t name_idx, bool force_new, const src::SrcL
   if (!name_idx) {
     return nullptr;
   }
+  // check if the filename includes "stdlib"
+  // if (loc.fdescr != nullptr && loc.fdescr->is_stdlib) {
+  //   auto found = global_stdlib_sym_def[name_idx];
+  //   if (found) {
+  //     return force_new && found->value ? nullptr : found;
+  //   }
+  //   return global_stdlib_sym_def[name_idx] = new SymDef(0, name_idx, loc);
+  // }
+
   auto found = global_sym_def[name_idx];
   if (found) {
     return force_new && found->value ? nullptr : found;

crypto/parser/symtable.h

@@ -119,6 +119,7 @@ class SymTable : public SymTableBase {
 
  public:
   SymTable() : SymTableBase(pp, sym) {
+
   }
   SymTable& add_keyword(std::string str, sym_idx_t idx = 0) {
     SymTableBase::add_keyword(str, idx);
@@ -166,6 +167,8 @@ extern SymDef* global_sym_def[symbols.hprime + 1];
 extern std::vector<std::pair<int, SymDef>> symbol_stack;
 extern std::vector<src::SrcLocation> scope_opened_at;
 
+void reset_state();
+
 void open_scope(src::Lexer& lex);
 void close_scope(src::Lexer& lex);
 SymDef* lookup_symbol(sym_idx_t idx, int flags = 3);