Merge branch 'testnet' into accelerator

crypto/CMakeLists.txt

@@ -358,7 +358,8 @@ target_link_libraries(test-ed25519-crypto PUBLIC ton_crypto)
 
 add_library(fift-lib STATIC ${FIFT_SOURCE})
 target_include_directories(fift-lib PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>)
-target_link_libraries(fift-lib PUBLIC ton_crypto ton_db tdutils ton_block)
+target_link_libraries(fift-lib PUBLIC ton_crypto tdutils ton_block)
+
 if (USE_EMSCRIPTEN)
   target_link_options(fift-lib PRIVATE -fexceptions)
   target_compile_options(fift-lib PRIVATE -fexceptions)

crypto/block/transaction.cpp

@@ -1661,7 +1661,12 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) {
   std::unique_ptr<StringLoggerTail> logger;
   auto vm_log = vm::VmLog();
   if (cfg.with_vm_log) {
-    size_t log_max_size = cfg.vm_log_verbosity > 0 ? 1024 * 1024 : 256;
+    size_t log_max_size = 256;
+    if (cfg.vm_log_verbosity > 4) {
+      log_max_size = 32 << 20;
+    } else if (cfg.vm_log_verbosity > 0) {
+      log_max_size = 1 << 20;
+    }
     logger = std::make_unique<StringLoggerTail>(log_max_size);
     vm_log.log_interface = logger.get();
     vm_log.log_options = td::LogOptions(VERBOSITY_NAME(DEBUG), true, false);
@@ -1673,6 +1678,7 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) {
           vm_log.log_mask |= vm::VmLog::DumpStack;
           if (cfg.vm_log_verbosity > 4) {
             vm_log.log_mask |= vm::VmLog::DumpStackVerbose;
+            vm_log.log_mask |= vm::VmLog::DumpC5;
           }
         }
       }

crypto/common/bitstring.cpp

@@ -640,11 +640,11 @@ long parse_bitstring_hex_literal(unsigned char* buff, std::size_t buff_size, con
   return bits;
 }
 
-long parse_bitstring_binary_literal(BitPtr buff, std::size_t buff_size, const char* str, const char* str_end) {
+long parse_bitstring_binary_literal(BitPtr buff, std::size_t buff_size_bits, const char* str, const char* str_end) {
   const char* ptr = str;
-  while (ptr < str_end && buff_size && (*ptr == '0' || *ptr == '1')) {
+  while (ptr < str_end && buff_size_bits && (*ptr == '0' || *ptr == '1')) {
     *buff++ = (bool)(*ptr++ & 1);
-    --buff_size;
+    --buff_size_bits;
   }
   return td::narrow_cast<long>(ptr == str_end ? ptr - str : str - ptr - 1);
 }

crypto/common/bitstring.h

@@ -58,7 +58,7 @@ unsigned long long bits_load_long_top(ConstBitPtr from, unsigned top_bits);
 long long bits_load_long(ConstBitPtr from, unsigned bits);
 unsigned long long bits_load_ulong(ConstBitPtr from, unsigned bits);
 long parse_bitstring_hex_literal(unsigned char* buff, std::size_t buff_size, const char* str, const char* str_end);
-long parse_bitstring_binary_literal(BitPtr buff, std::size_t buff_size, const char* str, const char* str_end);
+long parse_bitstring_binary_literal(BitPtr buff, std::size_t buff_size_bits, const char* str, const char* str_end);
 
 void bits_sha256(BitPtr to, ConstBitPtr from, std::size_t size);
 

crypto/fift/Fift.cpp

@@ -49,7 +49,6 @@ td::Result<int> Fift::interpret_istream(std::istream& stream, std::string curren
 }
 
 td::Result<int> Fift::do_interpret(IntCtx& ctx, bool is_interactive) {
-  ctx.ton_db = &config_.ton_db;
   ctx.source_lookup = &config_.source_lookup;
   ctx.dictionary = ctx.main_dictionary = ctx.context = config_.dictionary;
   ctx.output_stream = config_.output_stream;

crypto/fift/Fift.h

@@ -19,7 +19,6 @@
 #pragma once
 
 #include "SourceLookup.h"
-#include "vm/db/TonDb.h"
 #include "Dictionary.h"
 
 #include "td/utils/Status.h"
@@ -31,13 +30,11 @@ struct Fift {
  public:
   struct Config {
     fift::SourceLookup source_lookup;
-    vm::TonDb ton_db;
     fift::Dictionary dictionary;
     std::ostream* output_stream{&std::cout};
     std::ostream* error_stream{&std::cerr};
     bool show_backtrace{true};
   };
-  // Fift must own ton_db and dictionary, no concurrent access is allowed
   explicit Fift(Config config);
 
   td::Result<int> interpret_file(std::string fname, std::string current_dir, bool interactive = false);

crypto/fift/fift-main.cpp

@@ -46,8 +46,6 @@
 #include "SourceLookup.h"
 #include "words.h"
 
-#include "vm/db/TonDb.h"
-
 #include "td/utils/logging.h"
 #include "td/utils/misc.h"
 #include "td/utils/Parser.h"
@@ -65,7 +63,6 @@ void usage(const char* progname) {
                "\t-I<source-search-path>\tSets colon-separated (unix) or at-separated (windows) library source include path. If not indicated, "
                "$FIFTPATH is used instead.\n"
                "\t-L<library-fif-file>\tPre-loads a library source file\n"
-               "\t-d<ton-db-path>\tUse a ton database\n"
                "\t-s\tScript mode: use first argument as a fift source file and import remaining arguments as $n)\n"
                "\t-v<verbosity-level>\tSet verbosity level\n"
                "\t-V<version>\tShow fift build information\n";
@@ -94,13 +91,12 @@ int main(int argc, char* const argv[]) {
   bool script_mode = false;
   std::vector<std::string> library_source_files, source_list;
   std::vector<std::string> source_include_path;
-  std::string ton_db_path;
 
   fift::Fift::Config config;
 
   int i;
   int new_verbosity_level = VERBOSITY_NAME(INFO);
-  while (!script_mode && (i = getopt(argc, argv, "hinI:L:d:sv:V")) != -1) {
+  while (!script_mode && (i = getopt(argc, argv, "hinI:L:sv:V")) != -1) {
     switch (i) {
       case 'i':
         interactive = true;
@@ -115,9 +111,6 @@ int main(int argc, char* const argv[]) {
       case 'L':
         library_source_files.emplace_back(optarg);
         break;
-      case 'd':
-        ton_db_path = optarg;
-        break;
       case 's':
         script_mode = true;
         break;
@@ -158,16 +151,6 @@ int main(int argc, char* const argv[]) {
     config.source_lookup.add_include_path(path);
   }
 
-  if (!ton_db_path.empty()) {
-    auto r_ton_db = vm::TonDbImpl::open(ton_db_path);
-    if (r_ton_db.is_error()) {
-      LOG(ERROR) << "Error opening ton database: " << r_ton_db.error().to_string();
-      std::exit(2);
-    }
-    config.ton_db = r_ton_db.move_as_ok();
-    // FIXME //std::atexit([&] { config.ton_db.reset(); });
-  }
-
   fift::init_words_common(config.dictionary);
   fift::init_words_vm(config.dictionary, true);  // enable vm debug
   fift::init_words_ton(config.dictionary);

crypto/fift/utils.cpp

@@ -22,6 +22,8 @@
 #include "td/utils/filesystem.h"
 #include "td/utils/misc.h"
 #include "td/utils/port/path.h"
+#include "vm/boc.h"
+#include <map>
 
 namespace fift {
 namespace {

crypto/fift/words.cpp

@@ -43,8 +43,6 @@
 #include "vm/box.hpp"
 #include "vm/atom.h"
 
-#include "vm/db/TonDb.h"  // only for interpret_db_run_vm{,_parallel}
-
 #include "block/block.h"
 #include "common/global-version.h"
 
@@ -2077,23 +2075,23 @@ void interpret_bitstring_hex_literal(IntCtx& ctx) {
   auto s = ctx.parser->scan_word_to('}');
   unsigned char buff[128];
   int bits = (int)td::bitstring::parse_bitstring_hex_literal(buff, sizeof(buff), s.begin(), s.end());
-  if (bits < 0) {
+  vm::CellBuilder cb;
+  if (bits < 0 || !cb.store_bits_bool(td::ConstBitPtr{buff}, bits)) {
     throw IntError{"Invalid hex bitstring constant"};
   }
-  auto cs = Ref<vm::CellSlice>{true, vm::CellBuilder().store_bits(td::ConstBitPtr{buff}, bits).finalize()};
-  ctx.stack.push(std::move(cs));
+  ctx.stack.push(cb.as_cellslice_ref());
   push_argcount(ctx, 1);
 }
 
 void interpret_bitstring_binary_literal(IntCtx& ctx) {
   auto s = ctx.parser->scan_word_to('}');
   unsigned char buff[128];
-  int bits = (int)td::bitstring::parse_bitstring_binary_literal(buff, sizeof(buff), s.begin(), s.end());
-  if (bits < 0) {
+  int bits = (int)td::bitstring::parse_bitstring_binary_literal(buff, sizeof(buff) * 8, s.begin(), s.end());
+  vm::CellBuilder cb;
+  if (bits < 0 || !cb.store_bits_bool(td::ConstBitPtr{buff}, bits)) {
     throw IntError{"Invalid binary bitstring constant"};
   }
-  auto cs = Ref<vm::CellSlice>{true, vm::CellBuilder().store_bits(td::ConstBitPtr{buff}, bits).finalize()};
-  ctx.stack.push(std::move(cs));
+  ctx.stack.push(cb.as_cellslice_ref());
   push_argcount(ctx, 1);
 }
 
@@ -2721,114 +2719,6 @@ void interpret_vmop_dump(vm::Stack& stack) {
   stack.push_string(std::move(dump));
 }
 
-void do_interpret_db_run_vm_parallel(std::ostream* stream, vm::Stack& stack, vm::TonDb* ton_db_ptr, int threads_n,
-                                     int tasks_n) {
-  if (!ton_db_ptr || !*ton_db_ptr) {
-    throw vm::VmError{vm::Excno::fatal, "Ton database is not available"};
-  }
-  auto& ton_db = *ton_db_ptr;
-  auto txn = ton_db->begin_transaction();
-  auto txn_abort = td::ScopeExit() + [&] { ton_db->abort_transaction(std::move(txn)); };
-
-  struct Task {
-    vm::Ref<vm::CellSlice> code;
-    vm::SmartContractDb smart;
-    td::optional<vm::SmartContractDiff> diff;
-    td::unique_ptr<td::Guard> guard;
-    Ref<vm::Stack> stack;
-    int res{0};
-    Ref<vm::Cell> data;
-    std::string log;
-  };
-  std::vector<Task> tasks(tasks_n);
-  std::vector<td::thread> threads(threads_n);
-
-  for (auto& task : tasks) {
-    task.code = stack.pop_cellslice();
-    auto smart_hash = td::serialize(stack.pop_smallint_range(1000000000));
-    task.smart = txn->begin_smartcontract(smart_hash);
-    task.guard = td::create_lambda_guard([&] { txn->abort_smartcontract(std::move(task.smart)); });
-    auto argsn = stack.pop_smallint_range(100);
-    task.stack = stack.split_top(argsn);
-  }
-
-  std::atomic<int> next_task_i{0};
-  auto run_tasks = [&] {
-    while (true) {
-      auto task_i = next_task_i++;
-      if (task_i >= tasks_n) {
-        break;
-      }
-      auto& task = tasks[task_i];
-      auto data = task.smart->get_root();
-
-      StringLogger logger;
-      vm::VmLog log = create_vm_log(stream ? &logger : nullptr);
-
-      task.res = vm::run_vm_code(task.code, task.stack, 3, &data, std::move(log));
-      task.smart->set_root(data);
-      task.diff = vm::SmartContractDiff(std::move(task.smart));
-      task.data = std::move(data);
-      task.log = std::move(logger.res);
-    }
-  };
-
-  td::Timer timer;
-  for (auto& thread : threads) {
-    thread = td::thread(run_tasks);
-  }
-  run_tasks();
-  for (auto& thread : threads) {
-    thread.join();
-  }
-
-  if (stream) {
-    int id = 0;
-    for (auto& task : tasks) {
-      id++;
-      *stream << "Task #" << id << " vm_log begin" << std::endl;
-      *stream << task.log;
-      *stream << "Task #" << id << " vm_log end" << std::endl;
-    }
-  }
-
-  LOG(ERROR) << timer;
-  timer = {};
-
-  for (auto& task : tasks) {
-    auto retn = task.stack.write().pop_smallint_range(100, -1);
-    if (retn == -1) {
-      retn = task.stack->depth();
-    }
-    stack.push_from_stack(std::move(*task.stack), retn);
-    stack.push_smallint(task.res);
-    stack.push_cell(std::move(task.data));
-    task.guard->dismiss();
-    if (task.diff) {
-      txn->commit_smartcontract(std::move(task.diff.value()));
-    } else {
-      txn->commit_smartcontract(std::move(task.smart));
-    }
-  }
-  LOG(ERROR) << timer;
-  timer = {};
-
-  txn_abort.dismiss();
-  ton_db->commit_transaction(std::move(txn));
-  timer = {};
-  LOG(INFO) << "TonDB stats: \n" << ton_db->stats();
-}
-
-void interpret_db_run_vm(IntCtx& ctx) {
-  do_interpret_db_run_vm_parallel(ctx.error_stream, ctx.stack, ctx.ton_db, 0, 1);
-}
-
-void interpret_db_run_vm_parallel(IntCtx& ctx) {
-  auto threads_n = ctx.stack.pop_smallint_range(32, 0);
-  auto tasks_n = ctx.stack.pop_smallint_range(1000000000);
-  do_interpret_db_run_vm_parallel(ctx.error_stream, ctx.stack, ctx.ton_db, threads_n, tasks_n);
-}
-
 void interpret_store_vm_cont(vm::Stack& stack) {
   auto vmcont = stack.pop_cont();
   auto cb = stack.pop_builder();
@@ -3518,8 +3408,6 @@ void init_words_vm(Dictionary& d, bool enable_debug) {
   // d.def_ctx_word("runvmcode ", std::bind(interpret_run_vm, _1, 0x40));
   // d.def_ctx_word("runvm ", std::bind(interpret_run_vm, _1, 0x45));
   d.def_ctx_word("runvmx ", std::bind(interpret_run_vm, _1, -1));
-  d.def_ctx_word("dbrunvm ", interpret_db_run_vm);
-  d.def_ctx_word("dbrunvm-parallel ", interpret_db_run_vm_parallel);
   d.def_stack_word("vmcont, ", interpret_store_vm_cont);
   d.def_stack_word("vmcont@ ", interpret_fetch_vm_cont);
   d.def_stack_word("(vmoplen) ", interpret_vmop_len);

crypto/func/analyzer.cpp

@@ -17,6 +17,7 @@
     Copyright 2017-2020 Telegram Systems LLP
 */
 #include "func.h"
+#include "vm/boc.h"
 
 namespace funC {
 

crypto/funcfiftlib/funcfiftlib.cpp

@@ -33,6 +33,7 @@
 #include "td/utils/Status.h"
 #include <sstream>
 #include <iomanip>
+#include "vm/boc.h"
 
 td::Result<std::string> compile_internal(char *config_json) {
   TRY_RESULT(input_json, td::json_decode(td::MutableSlice(config_json)))

crypto/test/fift/testdb.fif

@@ -1,102 +0,0 @@
-"Asm.fif" include
-
-PROGRAM{
-
-NEWPROC load_dict
-NEWPROC generate_dict
-NEWPROC save_dict
-
-NEWPROC do_get
-NEWPROC do_set
-NEWPROC do_erase
-
-main PROC:<{
-  DUP 1 INT EQUAL IF:<{
-   DROP
-   do_get CALL
-  }>ELSE<{
-  DUP 2 INT EQUAL IF:<{
-   DROP
-   do_set CALL
-  }>ELSE<{
-  DUP 3 INT EQUAL IF:<{
-   DROP
-   do_erase CALL
-  }> }> }>
-  -1 INT
-}>
-
-do_get PROC:<{
- load_dict CALL
- 32 INT
- DICTIGET
-}>
-
-do_set PROC:<{
- load_dict CALL
- 32 INT
- DICTISET
- save_dict CALL
-}>
-
-do_erase PROC:<{
- load_dict CALL
- 32 INT
- DICTIDEL
- DROP
- save_dict CALL
-}>
-
-generate_dict PROC:<{
-  4 INT 100 INT REPEAT:<{
-    DUP 2DUP MUL ROT 617 INT ADD 1000 INT MOD
-  }>
-  DROP 100 INT
-  NEWDICT
-  SWAP REPEAT:<{
-    s0 s2 XCHG
-    NEWC
-    16 STU
-    s0 s2 XCHG
-    32 INT
-    DICTISETB
-  }>
-}>
-
-load_dict PROC:<{
-  PUSHROOT
-  CTOS DUP SEMPTY IF:<{
-    DROP
-    generate_dict CALL
-  }>
-}>
-
-save_dict PROC:<{
-  NEWC
-  STSLICE
-  ENDC
-  POPROOT
-}>
-
-}END>s constant pmc_prog
-
-{ 1 2 rot pmc_prog } : task_pmc_get
-{ 2 3 rot pmc_prog } : task_pmc_set
-{ 3 2 rot pmc_prog } : task_pmc_erase
-
-{ task_pmc_get dbrunvm 2drop } : pmc_get
-{ task_pmc_set dbrunvm 2drop } : pmc_set
-{ task_pmc_erase dbrunvm 2drop } : pmc_erase
-
-<b x{abacaba} s, <b x{dead} s, x{1dead} sr, b> <s sr, b> <s constant test_value
-
-// 123 "a" pmc_get
-// { csr. } if
-//  123 "a" pmc_set
-
-// test_value 123 x{a} task_pmc_set test_value 123 x{b} task_pmc_set test_value 123 x{c} task_pmc_set 3 3 dbrunvm-parallel
-
-{ test_value 150 rot task_pmc_set } 0 { 1 + 2dup swap execute 7 roll 7 roll } 10000 times 2drop 10000 4 dbrunvm-parallel
-
-// 123 "a" task_pmc_get 123 "b" task_pmc_get 123 "c" task_pmc_get 3 dbrunvm-parallel
-

crypto/vm/log.h

@@ -31,7 +31,7 @@ namespace vm {
 struct VmLog {
   td::LogInterface *log_interface{td::log_interface};
   td::LogOptions log_options{td::log_options};
-  enum { DumpStack = 2, ExecLocation = 4, GasRemaining = 8, DumpStackVerbose = 16 };
+  enum { DumpStack = 2, ExecLocation = 4, GasRemaining = 8, DumpStackVerbose = 16, DumpC5 = 32 };
   int log_mask{1};
   static VmLog Null() {
     VmLog res;

crypto/vm/stack.cpp

@@ -83,6 +83,14 @@ std::string StackEntry::to_lisp_string() const {
   return std::move(os).str();
 }
 
+static std::string cell_to_hex(const td::Ref<vm::Cell> &cell) {
+  auto boc = vm::std_boc_serialize(cell);
+  if (boc.is_ok()) {
+    return td::buffer_to_hex(boc.move_as_ok().as_slice());
+  }
+  return "???";
+}
+
 void StackEntry::dump(std::ostream& os, bool verbose) const {
   switch (tp) {
     case t_null:
@@ -94,12 +102,7 @@ void StackEntry::dump(std::ostream& os, bool verbose) const {
     case t_cell:
       if (ref.not_null()) {
         if (verbose) {
-          std::string serialized = "???";
-          auto boc = vm::std_boc_serialize(as_cell());
-          if (boc.is_ok()) {
-            serialized = td::buffer_to_hex(boc.move_as_ok().as_slice());
-          }
-          os << "C{" << serialized << "}";
+          os << "C{" << cell_to_hex(as_cell()) << "}";
         } else {
           os << "C{" << *as_cell() << "}";
         }
@@ -109,7 +112,12 @@ void StackEntry::dump(std::ostream& os, bool verbose) const {
       break;
     case t_builder:
       if (ref.not_null()) {
-        os << "BC{" << *as_builder() << "}";
+        if (verbose) {
+          Ref<CellBuilder> cb = as_builder();
+          os << "BC{" << cell_to_hex(cb.write().finalize_novm()) << "}";
+        } else {
+          os << "BC{" << *as_builder() << "}";
+        }
       } else {
         os << "BC{null}";
       }
@@ -117,7 +125,13 @@ void StackEntry::dump(std::ostream& os, bool verbose) const {
     case t_slice: {
       if (ref.not_null()) {
         os << "CS{";
-        static_cast<Ref<CellSlice>>(ref)->dump(os, 1, false);
+        if (verbose) {
+          CellBuilder cb;
+          cb.append_cellslice(as_slice());
+          os << cell_to_hex(cb.finalize_novm());
+        } else {
+          static_cast<Ref<CellSlice>>(ref)->dump(os, 1, false);
+        }
         os << '}';
       } else {
         os << "CS{null}";

crypto/vm/utils.cpp

@@ -96,10 +96,10 @@ td::Result<vm::StackEntry> convert_stack_entry(td::Slice str) {
   }
   if (l >= 3 && (str[0] == 'x' || str[0] == 'b') && str[1] == '{' && str.back() == '}') {
     unsigned char buff[128];
-    int bits =
-        (str[0] == 'x')
-            ? (int)td::bitstring::parse_bitstring_hex_literal(buff, sizeof(buff), str.begin() + 2, str.end() - 1)
-            : (int)td::bitstring::parse_bitstring_binary_literal(buff, sizeof(buff), str.begin() + 2, str.end() - 1);
+    int bits = (str[0] == 'x')
+                   ? (int)td::bitstring::parse_bitstring_hex_literal(buff, sizeof(buff), str.begin() + 2, str.end() - 1)
+                   : (int)td::bitstring::parse_bitstring_binary_literal(buff, sizeof(buff) * 8, str.begin() + 2,
+                                                                        str.end() - 1);
     if (bits < 0) {
       return td::Status::Error("failed to parse raw b{...}/x{...} number");
     }

crypto/vm/vm.cpp

@@ -441,10 +441,16 @@ int VmState::step() {
     if (log.log_mask & vm::VmLog::DumpStackVerbose) {
       mode += 4;
     }
+    std::unique_ptr<VmStateInterface> tmp_ctx;
+    // install temporary dummy vm state interface to prevent charging for cell load operations during dump
+    VmStateInterface::Guard guard(tmp_ctx.get());
     stack->dump(ss, mode);
     VM_LOG(this) << "stack:" << ss.str();
   }
   if (stack_trace) {
+    std::unique_ptr<VmStateInterface> tmp_ctx;
+    // install temporary dummy vm state interface to prevent charging for cell load operations during dump
+    VmStateInterface::Guard guard(tmp_ctx.get());
     stack->dump(std::cerr, 3);
   }
   ++steps;
@@ -523,6 +529,13 @@ int VmState::run() {
       res = vmoog.get_errno();  // no ~ for unhandled exceptions (to make their faking impossible)
     }
     if (!parent) {
+      if ((log.log_mask & VmLog::DumpC5) && cstate.committed) {
+        std::stringstream ss;
+        ss << "final c5: ";
+        StackEntry::maybe<Cell>(cstate.c5).dump(ss, true);
+        ss << "\n";
+        VM_LOG(this) << ss.str();
+      }
       return res;
     }
     restore_parent = true;