external/ton
1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-03-09 15:40:10 +00:00
Code Issues Releases Wiki Activity

Merge branch 'testnet' into accelerator

Browse source
This commit is contained in:
SpyCheese 2024-07-29 15:28:00 +03:00
commit d4da0ace74
32 changed files with 296 additions and 367 deletions
Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/build-ton-wasm-emscripten.yml vendored
View file

@ -19,7 +19,7 @@ jobs:
- name: Build TON WASM artifacts
run: |
cd assembly/wasm
cp assembly/wasm/fift-func-wasm-build-ubuntu.sh .
chmod +x fift-func-wasm-build-ubuntu.sh
./fift-func-wasm-build-ubuntu.sh -a

2
assembly/cicd/jenkins/test-builds.groovy
View file

@ -219,7 +219,7 @@ pipeline {
steps {
timeout(time: 180, unit: 'MINUTES') {
sh '''
cd assembly/wasm
cp assembly/wasm/fift-func-wasm-build-ubuntu.sh .
chmod +x fift-func-wasm-build-ubuntu.sh
./fift-func-wasm-build-ubuntu.sh -a
'''

202
assembly/wasm/fift-func-wasm-build-ubuntu.sh
View file

@ -1,5 +1,3 @@
# The script builds funcfift compiler to WASM
# Execute these prerequisites first
# sudo apt update
# sudo apt install -y build-essential git make cmake ninja-build clang libgflags-dev zlib1g-dev libssl-dev \
@ -11,10 +9,12 @@
# sudo ./llvm.sh 16 all
with_artifacts=false
scratch_new=false
while getopts 'a' flag; do
while getopts 'af' flag; do
case "${flag}" in
a) with_artifacts=true ;;
f) scratch_new=true ;;
*) break
;;
esac
@ -24,112 +24,139 @@ export CC=$(which clang-16)
export CXX=$(which clang++-16)
export CCACHE_DISABLE=1
cd ../..
rm -rf openssl zlib emsdk secp256k1 libsodium build
echo `pwd`
if [ "$scratch_new" = true ]; then
echo Compiling openssl zlib lz4 emsdk secp256k1 libsodium emsdk ton
rm -rf openssl zlib lz4 emsdk secp256k1 libsodium build
fi
git clone https://github.com/openssl/openssl.git
cd openssl
git checkout checkout openssl-3.1.4
./config
make -j16
OPENSSL_DIR=`pwd`
cd ..
git clone https://github.com/madler/zlib.git
cd zlib
git checkout v1.3.1
ZLIB_DIR=`pwd`
cd ..
if [ ! -d "openssl" ]; then
git clone https://github.com/openssl/openssl.git
cd openssl
git checkout openssl-3.1.4
./config
make -j16
OPENSSL_DIR=`pwd`
cd ..
else
OPENSSL_DIR=`pwd`/openssl
echo Using compiled openssl at $OPENSSL_DIR
fi
git clone https://github.com/lz4/lz4.git
cd lz4
git checkout v1.9.4
LZ4_DIR=`pwd`
cd ..
if [ ! -d "build" ]; then
mkdir build
cd build
cmake -GNinja -DCMAKE_BUILD_TYPE=Release \
-DCMAKE_CXX_STANDARD=17 \
-DOPENSSL_FOUND=1 \
-DOPENSSL_INCLUDE_DIR=$OPENSSL_DIR/include \
-DOPENSSL_CRYPTO_LIBRARY=$OPENSSL_DIR/libcrypto.so \
-DTON_USE_ABSEIL=OFF ..
git clone https://github.com/bitcoin-core/secp256k1.git
cd secp256k1
git checkout v0.3.2
./autogen.sh
SECP256K1_DIR=`pwd`
cd ..
test $? -eq 0 || { echo "Can't configure TON build"; exit 1; }
ninja fift smc-envelope
test $? -eq 0 || { echo "Can't compile fift "; exit 1; }
rm -rf *
cd ..
else
echo cleaning build...
rm -rf build/*
fi
git clone https://github.com/jedisct1/libsodium
cd libsodium
git checkout 1.0.18-RELEASE
SODIUM_DIR=`pwd`
cd ..
if [ ! -d "emsdk" ]; then
git clone https://github.com/emscripten-core/emsdk.git
echo
echo Using cloned emsdk
fi
mkdir build
cd build
cmake -GNinja -DCMAKE_BUILD_TYPE=Release \
-DCMAKE_CXX_STANDARD=17 \
-DOPENSSL_FOUND=1 \
-DOPENSSL_ROOT_DIR=$OPENSSL_DIR \
-DOPENSSL_INCLUDE_DIR=$OPENSSL_DIR/include \
-DOPENSSL_CRYPTO_LIBRARY=$OPENSSL_DIR/libcrypto.so \
-DOPENSSL_SSL_LIBRARY=$OPENSSL_DIR/libssl.so \
-DTON_USE_ABSEIL=OFF ..
test $? -eq 0 || { echo "Can't configure TON build"; exit 1; }
ninja fift smc-envelope
test $? -eq 0 || { echo "Can't compile fift "; exit 1; }
rm -rf *
cd ..
git clone https://github.com/emscripten-core/emsdk.git
cd emsdk
./emsdk install 3.1.19
./emsdk activate 3.1.19
EMSDK_DIR=`pwd`
ls $EMSDK_DIR
. $EMSDK_DIR/emsdk_env.sh
export CC=$(which emcc)
export CXX=$(which em++)
export CCACHE_DISABLE=1
cd ../openssl
cd ..
make clean
emconfigure ./Configure linux-generic32 no-shared no-dso no-engine no-unit-test
sed -i 's/CROSS_COMPILE=.*/CROSS_COMPILE=/g' Makefile
sed -i 's/-ldl//g' Makefile
sed -i 's/-O3/-Os/g' Makefile
emmake make depend
emmake make -j16
test $? -eq 0 || { echo "Can't compile OpenSSL with emmake "; exit 1; }
if [ ! -f "openssl/openssl_em" ]; then
cd openssl
make clean
emconfigure ./Configure linux-generic32 no-shared no-dso no-engine no-unit-test
sed -i 's/CROSS_COMPILE=.*/CROSS_COMPILE=/g' Makefile
sed -i 's/-ldl//g' Makefile
sed -i 's/-O3/-Os/g' Makefile
emmake make depend
emmake make -j16
test $? -eq 0 || { echo "Can't compile OpenSSL with emmake "; exit 1; }
touch openssl_em
cd ..
else
echo Using compiled openssl with emscripten
fi
cd ../zlib
if [ ! -d "zlib" ]; then
git clone https://github.com/madler/zlib.git
cd zlib
git checkout v1.3.1
ZLIB_DIR=`pwd`
emconfigure ./configure --static
emmake make -j16
test $? -eq 0 || { echo "Can't compile zlib with emmake "; exit 1; }
cd ..
else
ZLIB_DIR=`pwd`/zlib
echo Using compiled zlib with emscripten at $ZLIB_DIR
fi
emconfigure ./configure --static
emmake make -j16
test $? -eq 0 || { echo "Can't compile zlib with emmake "; exit 1; }
if [ ! -d "lz4" ]; then
git clone https://github.com/lz4/lz4.git
cd lz4
git checkout v1.9.4
LZ4_DIR=`pwd`
emmake make -j16
test $? -eq 0 || { echo "Can't compile lz4 with emmake "; exit 1; }
cd ..
else
LZ4_DIR=`pwd`/lz4
echo Using compiled lz4 with emscripten at $LZ4_DIR
fi
cd ../lz4
emmake make -j16
test $? -eq 0 || { echo "Can't compile lz4 with emmake "; exit 1; }
if [ ! -d "secp256k1" ]; then
git clone https://github.com/bitcoin-core/secp256k1.git
cd secp256k1
git checkout v0.3.2
./autogen.sh
SECP256K1_DIR=`pwd`
emconfigure ./configure --enable-module-recovery
emmake make -j16
test $? -eq 0 || { echo "Can't compile secp256k1 with emmake "; exit 1; }
cd ..
else
SECP256K1_DIR=`pwd`/secp256k1
echo Using compiled secp256k1 with emscripten at $SECP256K1_DIR
fi
cd ../secp256k1
if [ ! -d "libsodium" ]; then
git clone https://github.com/jedisct1/libsodium
cd libsodium
git checkout 1.0.18-RELEASE
SODIUM_DIR=`pwd`
emconfigure ./configure --disable-ssp
emmake make -j16
test $? -eq 0 || { echo "Can't compile libsodium with emmake "; exit 1; }
cd ..
else
SODIUM_DIR=`pwd`/libsodium
echo Using compiled libsodium with emscripten at $SODIUM_DIR
fi
emconfigure ./configure --enable-module-recovery
emmake make -j16
test $? -eq 0 || { echo "Can't compile secp256k1 with emmake "; exit 1; }
cd build
cd ../libsodium
emconfigure ./configure --disable-ssp
emmake make -j16
test $? -eq 0 || { echo "Can't compile libsodium with emmake "; exit 1; }
cd ../build
emcmake cmake -DUSE_EMSCRIPTEN=ON -DCMAKE_BUILD_TYPE=Release \
emcmake cmake -DUSE_EMSCRIPTEN=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \
-DZLIB_FOUND=1 \
-DZLIB_LIBRARIES=$ZLIB_DIR/libz.a \
-DZLIB_INCLUDE_DIR=$ZLIB_DIR \
@ -137,18 +164,15 @@ emcmake cmake -DUSE_EMSCRIPTEN=ON -DCMAKE_BUILD_TYPE=Release \
-DLZ4_LIBRARIES=$LZ4_DIR/lib/liblz4.a \
-DLZ4_INCLUDE_DIRS=$LZ4_DIR/lib \
-DOPENSSL_FOUND=1 \
-DOPENSSL_ROOT_DIR=$OPENSSL_DIR \
-DOPENSSL_INCLUDE_DIR=$OPENSSL_DIR/include \
-DOPENSSL_CRYPTO_LIBRARY=$OPENSSL_DIR/libcrypto.a \
-DOPENSSL_SSL_LIBRARY=$OPENSSL_DIR/libssl.a \
-DCMAKE_TOOLCHAIN_FILE=$EMSDK_DIR/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake \
-DCMAKE_CXX_FLAGS="-sUSE_ZLIB=1" \
-DSECP256K1_FOUND=1 \
-DSECP256K1_INCLUDE_DIR=$SECP256K1_DIR/include \
-DSECP256K1_LIBRARY=$SECP256K1_DIR/.libs/libsecp256k1.a \
-DSODIUM_INCLUDE_DIR=$SODIUM_DIR/src/libsodium/include \
-DSODIUM_LIBRARY_RELEASE=$SODIUM_DIR/src/libsodium/.libs/libsodium.a \
-DSODIUM_USE_STATIC_LIBS=ON ..
..
test $? -eq 0 || { echo "Can't configure TON with emmake "; exit 1; }
cp -R ../crypto/smartcont ../crypto/fift/lib crypto

3
crypto/CMakeLists.txt
View file

@ -358,7 +358,8 @@ target_link_libraries(test-ed25519-crypto PUBLIC ton_crypto)
add_library(fift-lib STATIC ${FIFT_SOURCE})
target_include_directories(fift-lib PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>)
target_link_libraries(fift-lib PUBLIC ton_crypto ton_db tdutils ton_block)
target_link_libraries(fift-lib PUBLIC ton_crypto tdutils ton_block)
if (USE_EMSCRIPTEN)
target_link_options(fift-lib PRIVATE -fexceptions)
target_compile_options(fift-lib PRIVATE -fexceptions)

8
crypto/block/transaction.cpp
View file

@ -1661,7 +1661,12 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) {
std::unique_ptr<StringLoggerTail> logger;
auto vm_log = vm::VmLog();
if (cfg.with_vm_log) {
size_t log_max_size = cfg.vm_log_verbosity > 0 ? 1024 * 1024 : 256;
size_t log_max_size = 256;
if (cfg.vm_log_verbosity > 4) {
log_max_size = 32 << 20;
} else if (cfg.vm_log_verbosity > 0) {
log_max_size = 1 << 20;
}
logger = std::make_unique<StringLoggerTail>(log_max_size);
vm_log.log_interface = logger.get();
vm_log.log_options = td::LogOptions(VERBOSITY_NAME(DEBUG), true, false);
@ -1673,6 +1678,7 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) {
vm_log.log_mask |= vm::VmLog::DumpStack;
if (cfg.vm_log_verbosity > 4) {
vm_log.log_mask |= vm::VmLog::DumpStackVerbose;
vm_log.log_mask |= vm::VmLog::DumpC5;
}
}
}

6
crypto/common/bitstring.cpp
View file

@ -640,11 +640,11 @@ long parse_bitstring_hex_literal(unsigned char* buff, std::size_t buff_size, con
return bits;
}
long parse_bitstring_binary_literal(BitPtr buff, std::size_t buff_size, const char* str, const char* str_end) {
long parse_bitstring_binary_literal(BitPtr buff, std::size_t buff_size_bits, const char* str, const char* str_end) {
const char* ptr = str;
while (ptr < str_end && buff_size && (*ptr == '0' || *ptr == '1')) {
while (ptr < str_end && buff_size_bits && (*ptr == '0' || *ptr == '1')) {
*buff++ = (bool)(*ptr++ & 1);
--buff_size;
--buff_size_bits;
}
return td::narrow_cast<long>(ptr == str_end ? ptr - str : str - ptr - 1);
}

2
crypto/common/bitstring.h
View file

@ -58,7 +58,7 @@ unsigned long long bits_load_long_top(ConstBitPtr from, unsigned top_bits);
long long bits_load_long(ConstBitPtr from, unsigned bits);
unsigned long long bits_load_ulong(ConstBitPtr from, unsigned bits);
long parse_bitstring_hex_literal(unsigned char* buff, std::size_t buff_size, const char* str, const char* str_end);
long parse_bitstring_binary_literal(BitPtr buff, std::size_t buff_size, const char* str, const char* str_end);
long parse_bitstring_binary_literal(BitPtr buff, std::size_t buff_size_bits, const char* str, const char* str_end);
void bits_sha256(BitPtr to, ConstBitPtr from, std::size_t size);

1
crypto/fift/Fift.cpp
View file

@ -49,7 +49,6 @@ td::Result<int> Fift::interpret_istream(std::istream& stream, std::string curren
}
td::Result<int> Fift::do_interpret(IntCtx& ctx, bool is_interactive) {
ctx.ton_db = &config_.ton_db;
ctx.source_lookup = &config_.source_lookup;
ctx.dictionary = ctx.main_dictionary = ctx.context = config_.dictionary;
ctx.output_stream = config_.output_stream;

3
crypto/fift/Fift.h
View file

@ -19,7 +19,6 @@
#pragma once
#include "SourceLookup.h"
#include "vm/db/TonDb.h"
#include "Dictionary.h"
#include "td/utils/Status.h"
@ -31,13 +30,11 @@ struct Fift {
public:
struct Config {
fift::SourceLookup source_lookup;
vm::TonDb ton_db;
fift::Dictionary dictionary;
std::ostream* output_stream{&std::cout};
std::ostream* error_stream{&std::cerr};
bool show_backtrace{true};
};
// Fift must own ton_db and dictionary, no concurrent access is allowed
explicit Fift(Config config);
td::Result<int> interpret_file(std::string fname, std::string current_dir, bool interactive = false);

19
crypto/fift/fift-main.cpp
View file

@ -46,8 +46,6 @@
#include "SourceLookup.h"
#include "words.h"
#include "vm/db/TonDb.h"
#include "td/utils/logging.h"
#include "td/utils/misc.h"
#include "td/utils/Parser.h"
@ -65,7 +63,6 @@ void usage(const char* progname) {
"\t-I<source-search-path>\tSets colon-separated (unix) or at-separated (windows) library source include path. If not indicated, "
"$FIFTPATH is used instead.\n"
"\t-L<library-fif-file>\tPre-loads a library source file\n"
"\t-d<ton-db-path>\tUse a ton database\n"
"\t-s\tScript mode: use first argument as a fift source file and import remaining arguments as $n)\n"
"\t-v<verbosity-level>\tSet verbosity level\n"
"\t-V<version>\tShow fift build information\n";
@ -94,13 +91,12 @@ int main(int argc, char* const argv[]) {
bool script_mode = false;
std::vector<std::string> library_source_files, source_list;
std::vector<std::string> source_include_path;
std::string ton_db_path;
fift::Fift::Config config;
int i;
int new_verbosity_level = VERBOSITY_NAME(INFO);
while (!script_mode && (i = getopt(argc, argv, "hinI:L:d:sv:V")) != -1) {
while (!script_mode && (i = getopt(argc, argv, "hinI:L:sv:V")) != -1) {
switch (i) {
case 'i':
interactive = true;
@ -115,9 +111,6 @@ int main(int argc, char* const argv[]) {
case 'L':
library_source_files.emplace_back(optarg);
break;
case 'd':
ton_db_path = optarg;
break;
case 's':
script_mode = true;
break;
@ -158,16 +151,6 @@ int main(int argc, char* const argv[]) {
config.source_lookup.add_include_path(path);
}
if (!ton_db_path.empty()) {
auto r_ton_db = vm::TonDbImpl::open(ton_db_path);
if (r_ton_db.is_error()) {
LOG(ERROR) << "Error opening ton database: " << r_ton_db.error().to_string();
std::exit(2);
}
config.ton_db = r_ton_db.move_as_ok();
// FIXME //std::atexit([&] { config.ton_db.reset(); });
}
fift::init_words_common(config.dictionary);
fift::init_words_vm(config.dictionary, true); // enable vm debug
fift::init_words_ton(config.dictionary);

2
crypto/fift/utils.cpp
View file

@ -22,6 +22,8 @@
#include "td/utils/filesystem.h"
#include "td/utils/misc.h"
#include "td/utils/port/path.h"
#include "vm/boc.h"
#include <map>
namespace fift {
namespace {

126
crypto/fift/words.cpp
View file

@ -43,8 +43,6 @@
#include "vm/box.hpp"
#include "vm/atom.h"
#include "vm/db/TonDb.h" // only for interpret_db_run_vm{,_parallel}
#include "block/block.h"
#include "common/global-version.h"
@ -2077,23 +2075,23 @@ void interpret_bitstring_hex_literal(IntCtx& ctx) {
auto s = ctx.parser->scan_word_to('}');
unsigned char buff[128];
int bits = (int)td::bitstring::parse_bitstring_hex_literal(buff, sizeof(buff), s.begin(), s.end());
if (bits < 0) {
vm::CellBuilder cb;
if (bits < 0 || !cb.store_bits_bool(td::ConstBitPtr{buff}, bits)) {
throw IntError{"Invalid hex bitstring constant"};
}
auto cs = Ref<vm::CellSlice>{true, vm::CellBuilder().store_bits(td::ConstBitPtr{buff}, bits).finalize()};
ctx.stack.push(std::move(cs));
ctx.stack.push(cb.as_cellslice_ref());
push_argcount(ctx, 1);
}
void interpret_bitstring_binary_literal(IntCtx& ctx) {
auto s = ctx.parser->scan_word_to('}');
unsigned char buff[128];
int bits = (int)td::bitstring::parse_bitstring_binary_literal(buff, sizeof(buff), s.begin(), s.end());
if (bits < 0) {
int bits = (int)td::bitstring::parse_bitstring_binary_literal(buff, sizeof(buff) * 8, s.begin(), s.end());
vm::CellBuilder cb;
if (bits < 0 || !cb.store_bits_bool(td::ConstBitPtr{buff}, bits)) {
throw IntError{"Invalid binary bitstring constant"};
}
auto cs = Ref<vm::CellSlice>{true, vm::CellBuilder().store_bits(td::ConstBitPtr{buff}, bits).finalize()};
ctx.stack.push(std::move(cs));
ctx.stack.push(cb.as_cellslice_ref());
push_argcount(ctx, 1);
}
@ -2721,114 +2719,6 @@ void interpret_vmop_dump(vm::Stack& stack) {
stack.push_string(std::move(dump));
}
void do_interpret_db_run_vm_parallel(std::ostream* stream, vm::Stack& stack, vm::TonDb* ton_db_ptr, int threads_n,
int tasks_n) {
if (!ton_db_ptr || !*ton_db_ptr) {
throw vm::VmError{vm::Excno::fatal, "Ton database is not available"};
}
auto& ton_db = *ton_db_ptr;
auto txn = ton_db->begin_transaction();
auto txn_abort = td::ScopeExit() + [&] { ton_db->abort_transaction(std::move(txn)); };
struct Task {
vm::Ref<vm::CellSlice> code;
vm::SmartContractDb smart;
td::optional<vm::SmartContractDiff> diff;
td::unique_ptr<td::Guard> guard;
Ref<vm::Stack> stack;
int res{0};
Ref<vm::Cell> data;
std::string log;
};
std::vector<Task> tasks(tasks_n);
std::vector<td::thread> threads(threads_n);
for (auto& task : tasks) {
task.code = stack.pop_cellslice();
auto smart_hash = td::serialize(stack.pop_smallint_range(1000000000));
task.smart = txn->begin_smartcontract(smart_hash);
task.guard = td::create_lambda_guard([&] { txn->abort_smartcontract(std::move(task.smart)); });
auto argsn = stack.pop_smallint_range(100);
task.stack = stack.split_top(argsn);
}
std::atomic<int> next_task_i{0};
auto run_tasks = [&] {
while (true) {
auto task_i = next_task_i++;
if (task_i >= tasks_n) {
break;
}
auto& task = tasks[task_i];
auto data = task.smart->get_root();
StringLogger logger;
vm::VmLog log = create_vm_log(stream ? &logger : nullptr);
task.res = vm::run_vm_code(task.code, task.stack, 3, &data, std::move(log));
task.smart->set_root(data);
task.diff = vm::SmartContractDiff(std::move(task.smart));
task.data = std::move(data);
task.log = std::move(logger.res);
}
};
td::Timer timer;
for (auto& thread : threads) {
thread = td::thread(run_tasks);
}
run_tasks();
for (auto& thread : threads) {
thread.join();
}
if (stream) {
int id = 0;
for (auto& task : tasks) {
id++;
*stream << "Task #" << id << " vm_log begin" << std::endl;
*stream << task.log;
*stream << "Task #" << id << " vm_log end" << std::endl;
}
}
LOG(ERROR) << timer;
timer = {};
for (auto& task : tasks) {
auto retn = task.stack.write().pop_smallint_range(100, -1);
if (retn == -1) {
retn = task.stack->depth();
}
stack.push_from_stack(std::move(*task.stack), retn);
stack.push_smallint(task.res);
stack.push_cell(std::move(task.data));
task.guard->dismiss();
if (task.diff) {
txn->commit_smartcontract(std::move(task.diff.value()));
} else {
txn->commit_smartcontract(std::move(task.smart));
}
}
LOG(ERROR) << timer;
timer = {};
txn_abort.dismiss();
ton_db->commit_transaction(std::move(txn));
timer = {};
LOG(INFO) << "TonDB stats: \n" << ton_db->stats();
}
void interpret_db_run_vm(IntCtx& ctx) {
do_interpret_db_run_vm_parallel(ctx.error_stream, ctx.stack, ctx.ton_db, 0, 1);
}
void interpret_db_run_vm_parallel(IntCtx& ctx) {
auto threads_n = ctx.stack.pop_smallint_range(32, 0);
auto tasks_n = ctx.stack.pop_smallint_range(1000000000);
do_interpret_db_run_vm_parallel(ctx.error_stream, ctx.stack, ctx.ton_db, threads_n, tasks_n);
}
void interpret_store_vm_cont(vm::Stack& stack) {
auto vmcont = stack.pop_cont();
auto cb = stack.pop_builder();
@ -3518,8 +3408,6 @@ void init_words_vm(Dictionary& d, bool enable_debug) {
// d.def_ctx_word("runvmcode ", std::bind(interpret_run_vm, _1, 0x40));
// d.def_ctx_word("runvm ", std::bind(interpret_run_vm, _1, 0x45));
d.def_ctx_word("runvmx ", std::bind(interpret_run_vm, _1, -1));
d.def_ctx_word("dbrunvm ", interpret_db_run_vm);
d.def_ctx_word("dbrunvm-parallel ", interpret_db_run_vm_parallel);
d.def_stack_word("vmcont, ", interpret_store_vm_cont);
d.def_stack_word("vmcont@ ", interpret_fetch_vm_cont);
d.def_stack_word("(vmoplen) ", interpret_vmop_len);

1
crypto/func/analyzer.cpp
View file

@ -17,6 +17,7 @@
Copyright 2017-2020 Telegram Systems LLP
*/
#include "func.h"
#include "vm/boc.h"
namespace funC {

1
crypto/funcfiftlib/funcfiftlib.cpp
View file

@ -33,6 +33,7 @@
#include "td/utils/Status.h"
#include <sstream>
#include <iomanip>
#include "vm/boc.h"
td::Result<std::string> compile_internal(char *config_json) {
TRY_RESULT(input_json, td::json_decode(td::MutableSlice(config_json)))

102
crypto/test/fift/testdb.fif
View file

@ -1,102 +0,0 @@
"Asm.fif" include
PROGRAM{
NEWPROC load_dict
NEWPROC generate_dict
NEWPROC save_dict
NEWPROC do_get
NEWPROC do_set
NEWPROC do_erase
main PROC:<{
DUP 1 INT EQUAL IF:<{
DROP
do_get CALL
}>ELSE<{
DUP 2 INT EQUAL IF:<{
DROP
do_set CALL
}>ELSE<{
DUP 3 INT EQUAL IF:<{
DROP
do_erase CALL
}> }> }>
-1 INT
}>
do_get PROC:<{
load_dict CALL
32 INT
DICTIGET
}>
do_set PROC:<{
load_dict CALL
32 INT
DICTISET
save_dict CALL
}>
do_erase PROC:<{
load_dict CALL
32 INT
DICTIDEL
DROP
save_dict CALL
}>
generate_dict PROC:<{
4 INT 100 INT REPEAT:<{
DUP 2DUP MUL ROT 617 INT ADD 1000 INT MOD
}>
DROP 100 INT
NEWDICT
SWAP REPEAT:<{
s0 s2 XCHG
NEWC
16 STU
s0 s2 XCHG
32 INT
DICTISETB
}>
}>
load_dict PROC:<{
PUSHROOT
CTOS DUP SEMPTY IF:<{
DROP
generate_dict CALL
}>
}>
save_dict PROC:<{
NEWC
STSLICE
ENDC
POPROOT
}>
}END>s constant pmc_prog
{ 1 2 rot pmc_prog } : task_pmc_get
{ 2 3 rot pmc_prog } : task_pmc_set
{ 3 2 rot pmc_prog } : task_pmc_erase
{ task_pmc_get dbrunvm 2drop } : pmc_get
{ task_pmc_set dbrunvm 2drop } : pmc_set
{ task_pmc_erase dbrunvm 2drop } : pmc_erase
<b x{abacaba} s, <b x{dead} s, x{1dead} sr, b> <s sr, b> <s constant test_value
// 123 "a" pmc_get
// { csr. } if
// 123 "a" pmc_set
// test_value 123 x{a} task_pmc_set test_value 123 x{b} task_pmc_set test_value 123 x{c} task_pmc_set 3 3 dbrunvm-parallel
{ test_value 150 rot task_pmc_set } 0 { 1 + 2dup swap execute 7 roll 7 roll } 10000 times 2drop 10000 4 dbrunvm-parallel
// 123 "a" task_pmc_get 123 "b" task_pmc_get 123 "c" task_pmc_get 3 dbrunvm-parallel

2
crypto/vm/log.h
View file

@ -31,7 +31,7 @@ namespace vm {
struct VmLog {
td::LogInterface *log_interface{td::log_interface};
td::LogOptions log_options{td::log_options};
enum { DumpStack = 2, ExecLocation = 4, GasRemaining = 8, DumpStackVerbose = 16 };
enum { DumpStack = 2, ExecLocation = 4, GasRemaining = 8, DumpStackVerbose = 16, DumpC5 = 32 };
int log_mask{1};
static VmLog Null() {
VmLog res;

30
crypto/vm/stack.cpp
View file

@ -83,6 +83,14 @@ std::string StackEntry::to_lisp_string() const {
return std::move(os).str();
}
static std::string cell_to_hex(const td::Ref<vm::Cell> &cell) {
auto boc = vm::std_boc_serialize(cell);
if (boc.is_ok()) {
return td::buffer_to_hex(boc.move_as_ok().as_slice());
}
return "???";
}
void StackEntry::dump(std::ostream& os, bool verbose) const {
switch (tp) {
case t_null:
@ -94,12 +102,7 @@ void StackEntry::dump(std::ostream& os, bool verbose) const {
case t_cell:
if (ref.not_null()) {
if (verbose) {
std::string serialized = "???";
auto boc = vm::std_boc_serialize(as_cell());
if (boc.is_ok()) {
serialized = td::buffer_to_hex(boc.move_as_ok().as_slice());
}
os << "C{" << serialized << "}";
os << "C{" << cell_to_hex(as_cell()) << "}";
} else {
os << "C{" << *as_cell() << "}";
}
@ -109,7 +112,12 @@ void StackEntry::dump(std::ostream& os, bool verbose) const {
break;
case t_builder:
if (ref.not_null()) {
os << "BC{" << *as_builder() << "}";
if (verbose) {
Ref<CellBuilder> cb = as_builder();
os << "BC{" << cell_to_hex(cb.write().finalize_novm()) << "}";
} else {
os << "BC{" << *as_builder() << "}";
}
} else {
os << "BC{null}";
}
@ -117,7 +125,13 @@ void StackEntry::dump(std::ostream& os, bool verbose) const {
case t_slice: {
if (ref.not_null()) {
os << "CS{";
static_cast<Ref<CellSlice>>(ref)->dump(os, 1, false);
if (verbose) {
CellBuilder cb;
cb.append_cellslice(as_slice());
os << cell_to_hex(cb.finalize_novm());
} else {
static_cast<Ref<CellSlice>>(ref)->dump(os, 1, false);
}
os << '}';
} else {
os << "CS{null}";

8
crypto/vm/utils.cpp
View file

@ -96,10 +96,10 @@ td::Result<vm::StackEntry> convert_stack_entry(td::Slice str) {
}
if (l >= 3 && (str[0] == 'x' || str[0] == 'b') && str[1] == '{' && str.back() == '}') {
unsigned char buff[128];
int bits =
(str[0] == 'x')
? (int)td::bitstring::parse_bitstring_hex_literal(buff, sizeof(buff), str.begin() + 2, str.end() - 1)
: (int)td::bitstring::parse_bitstring_binary_literal(buff, sizeof(buff), str.begin() + 2, str.end() - 1);
int bits = (str[0] == 'x')
? (int)td::bitstring::parse_bitstring_hex_literal(buff, sizeof(buff), str.begin() + 2, str.end() - 1)
: (int)td::bitstring::parse_bitstring_binary_literal(buff, sizeof(buff) * 8, str.begin() + 2,
str.end() - 1);
if (bits < 0) {
return td::Status::Error("failed to parse raw b{...}/x{...} number");
}

13
crypto/vm/vm.cpp
View file

@ -441,10 +441,16 @@ int VmState::step() {
if (log.log_mask & vm::VmLog::DumpStackVerbose) {
mode += 4;
}
std::unique_ptr<VmStateInterface> tmp_ctx;
// install temporary dummy vm state interface to prevent charging for cell load operations during dump
VmStateInterface::Guard guard(tmp_ctx.get());
stack->dump(ss, mode);
VM_LOG(this) << "stack:" << ss.str();
}
if (stack_trace) {
std::unique_ptr<VmStateInterface> tmp_ctx;
// install temporary dummy vm state interface to prevent charging for cell load operations during dump
VmStateInterface::Guard guard(tmp_ctx.get());
stack->dump(std::cerr, 3);
}
++steps;
@ -523,6 +529,13 @@ int VmState::run() {
res = vmoog.get_errno(); // no ~ for unhandled exceptions (to make their faking impossible)
}
if (!parent) {
if ((log.log_mask & VmLog::DumpC5) && cstate.committed) {
std::stringstream ss;
ss << "final c5: ";
StackEntry::maybe<Cell>(cstate.c5).dump(ss, true);
ss << "\n";
VM_LOG(this) << ss.str();
}
return res;
}
restore_parent = true;

4
emulator/CMakeLists.txt
View file

@ -35,7 +35,7 @@ else()
add_library(emulator STATIC ${EMULATOR_SOURCE} ${EMULATOR_HEADERS})
endif()
target_link_libraries(emulator PUBLIC emulator_static)
target_link_libraries(emulator PUBLIC emulator_static git)
generate_export_header(emulator EXPORT_FILE_NAME ${CMAKE_CURRENT_BINARY_DIR}/emulator_export.h)
target_include_directories(emulator PUBLIC
$<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
@ -48,7 +48,7 @@ if (USE_EMSCRIPTEN)
add_executable(emulator-emscripten ${EMULATOR_EMSCRIPTEN_SOURCE})
target_link_libraries(emulator-emscripten PUBLIC emulator)
target_link_options(emulator-emscripten PRIVATE -sEXPORTED_RUNTIME_METHODS=_malloc,free,UTF8ToString,stringToUTF8,allocate,ALLOC_NORMAL,lengthBytesUTF8)
target_link_options(emulator-emscripten PRIVATE -sEXPORTED_FUNCTIONS=_emulate,_free,_run_get_method,_create_emulator,_destroy_emulator,_emulate_with_emulator)
target_link_options(emulator-emscripten PRIVATE -sEXPORTED_FUNCTIONS=_emulate,_free,_run_get_method,_create_emulator,_destroy_emulator,_emulate_with_emulator,_version)
target_link_options(emulator-emscripten PRIVATE -sEXPORT_NAME=EmulatorModule)
target_link_options(emulator-emscripten PRIVATE -sERROR_ON_UNDEFINED_SYMBOLS=0)
target_link_options(emulator-emscripten PRIVATE -Oz)

4
emulator/emulator-emscripten.cpp
View file

@ -254,4 +254,8 @@ const char *run_get_method(const char *params, const char* stack, const char* co
return output;
}
const char *version() {
return emulator_version();
}
}

10
emulator/emulator-extern.cpp
View file

@ -9,6 +9,7 @@
#include "tvm-emulator.hpp"
#include "crypto/vm/stack.hpp"
#include "crypto/vm/memo.h"
#include "git.h"
td::Result<td::Ref<vm::Cell>> boc_b64_to_cell(const char *boc) {
TRY_RESULT_PREFIX(boc_decoded, td::base64_decode(td::Slice(boc)), "Can't decode base64 boc: ");
@ -717,3 +718,12 @@ void tvm_emulator_destroy(void *tvm_emulator) {
void emulator_config_destroy(void *config) {
delete static_cast<block::Config *>(config);
}
const char* emulator_version() {
auto version_json = td::JsonBuilder();
auto obj = version_json.enter_object();
obj("emulatorLibCommitHash", GitMetadata::CommitSHA1());
obj("emulatorLibCommitDate", GitMetadata::CommitDate());
obj.leave();
return strdup(version_json.string_builder().as_cslice().c_str());
}

4
emulator/emulator-extern.h
View file

@ -307,6 +307,10 @@ EMULATOR_EXPORT void tvm_emulator_destroy(void *tvm_emulator);
*/
EMULATOR_EXPORT void emulator_config_destroy(void *config);
/**
* @brief Get git commit hash and date of the library
*/
EMULATOR_EXPORT const char* emulator_version();
#ifdef __cplusplus
} // extern "C"

1
emulator/emulator_export_list
View file

@ -26,3 +26,4 @@ _tvm_emulator_send_external_message
_tvm_emulator_send_internal_message
_tvm_emulator_destroy
_tvm_emulator_emulate_run_method
_emulator_version

12
emulator/test/emulator-tests.cpp
View file

@ -9,7 +9,6 @@
#include "td/utils/base64.h"
#include "td/utils/crypto.h"
#include "td/utils/JsonBuilder.h"
#include "td/utils/Time.h"
#include "smc-envelope/WalletV3.h"
@ -225,7 +224,7 @@ TEST(Emulator, wallet_int_and_ext_msg) {
msg_info.ihr_fee = cb.as_cellslice_ref();
}
msg_info.created_lt = 0;
msg_info.created_at = static_cast<uint32_t>(td::Time::now());
msg_info.created_at = static_cast<uint32_t>(utime);
tlb::csr_pack(message.info, msg_info);
message.init = vm::CellBuilder()
.store_ones(1)
@ -242,6 +241,7 @@ TEST(Emulator, wallet_int_and_ext_msg) {
auto int_msg_boc = td::base64_encode(std_boc_serialize(int_msg).move_as_ok());
std::string int_emu_res = transaction_emulator_emulate_transaction(emulator, none_shard_account_boc.c_str(), int_msg_boc.c_str());
LOG(ERROR) << "int_emu_res = " << int_emu_res;
auto int_result_json = td::json_decode(td::MutableSlice(int_emu_res));
CHECK(int_result_json.is_ok());
@ -293,11 +293,12 @@ TEST(Emulator, wallet_int_and_ext_msg) {
// emulate external message
{
auto ext_body = wallet->make_a_gift_message(priv_key, static_cast<uint32_t>(td::Time::now()) + 60, {ton::WalletV3::Gift{block::StdAddress(0, ton::StdSmcAddress()), 1 * Ton}});
auto ext_body = wallet->make_a_gift_message(priv_key, utime + 60, {ton::WalletV3::Gift{block::StdAddress(0, ton::StdSmcAddress()), 1 * Ton}});
CHECK(ext_body.is_ok());
auto ext_msg = ton::GenericAccount::create_ext_message(address, {}, ext_body.move_as_ok());
auto ext_msg_boc = td::base64_encode(std_boc_serialize(ext_msg).move_as_ok());
std::string ext_emu_res = transaction_emulator_emulate_transaction(emulator, shard_account_after_boc_b64.c_str(), ext_msg_boc.c_str());
LOG(ERROR) << "ext_emu_res = " << ext_emu_res;
auto ext_result_json = td::json_decode(td::MutableSlice(ext_emu_res));
CHECK(ext_result_json.is_ok());
@ -371,9 +372,10 @@ TEST(Emulator, tvm_emulator) {
CHECK(wallet->get_address().rserialize_to(addr_buffer));
auto rand_seed = std::string(64, 'F');
CHECK(tvm_emulator_set_c7(tvm_emulator, addr_buffer, static_cast<uint32_t>(td::Time::now()), 10 * Ton, rand_seed.c_str(), config_boc));
CHECK(tvm_emulator_set_c7(tvm_emulator, addr_buffer, 1337, 10 * Ton, rand_seed.c_str(), config_boc));
std::string tvm_res = tvm_emulator_run_get_method(tvm_emulator, method_id, stack_boc.c_str());
LOG(ERROR) << "tvm_res = " << tvm_res;
auto result_json = td::json_decode(td::MutableSlice(tvm_res));
CHECK(result_json.is_ok());
auto result = result_json.move_as_ok();

1
tddb/td/db/KeyValue.h
View file

@ -18,6 +18,7 @@
*/
#pragma once
#include "td/utils/Status.h"
#include "td/utils/Time.h"
#include "td/utils/logging.h"
#include <functional>
namespace td {

43
tddb/td/db/RocksDb.cpp
View file

@ -258,11 +258,17 @@ Status RocksDb::flush() {
Status RocksDb::begin_snapshot() {
snapshot_.reset(db_->GetSnapshot());
if (options_.snapshot_statistics) {
options_.snapshot_statistics->begin_snapshot(snapshot_.get());
}
return td::Status::OK();
}
Status RocksDb::end_snapshot() {
if (snapshot_) {
if (options_.snapshot_statistics) {
options_.snapshot_statistics->end_snapshot(snapshot_.get());
}
db_->ReleaseSnapshot(snapshot_.release());
}
return td::Status::OK();
@ -271,4 +277,41 @@ Status RocksDb::end_snapshot() {
RocksDb::RocksDb(std::shared_ptr<rocksdb::OptimisticTransactionDB> db, RocksDbOptions options)
: db_(std::move(db)), options_(options) {
}
void RocksDbSnapshotStatistics::begin_snapshot(const rocksdb::Snapshot *snapshot) {
auto lock = std::unique_lock<std::mutex>(mutex_);
auto id = reinterpret_cast<std::uintptr_t>(snapshot);
auto ts = td::Timestamp::now().at();
CHECK(id_to_ts_.emplace(id, ts).second);
CHECK(by_ts_.emplace(ts, id).second);
}
void RocksDbSnapshotStatistics::end_snapshot(const rocksdb::Snapshot *snapshot) {
auto id = reinterpret_cast<std::uintptr_t>(snapshot);
auto it = id_to_ts_.find(id);
CHECK(it != id_to_ts_.end());
auto ts = it->second;
CHECK(by_ts_.erase(std::make_pair(ts, id)) == 1u);
CHECK(id_to_ts_.erase(id) == 1u);
}
td::Timestamp RocksDbSnapshotStatistics::oldest_snapshot_timestamp() const {
auto lock = std::unique_lock<std::mutex>(mutex_);
if (by_ts_.empty()) {
return {};
}
return td::Timestamp::at(by_ts_.begin()->first);
}
std::string RocksDbSnapshotStatistics::to_string() const {
td::Timestamp oldest_snapshot = oldest_snapshot_timestamp();
double value;
if (oldest_snapshot) {
value = td::Timestamp::now().at() - oldest_snapshot.at();
} else {
value = -1;
}
return PSTRING() << "td.rocksdb.snapshot.oldest_snapshot_ago.seconds : " << value << "\n";
}
} // namespace td

18
tddb/td/db/RocksDb.h
View file

@ -26,6 +26,12 @@
#include "td/utils/Status.h"
#include "td/utils/optional.h"
#include "td/utils/Time.h"
#include <map>
#include <mutex>
#include <set>
namespace rocksdb {
class Cache;
class OptimisticTransactionDB;
@ -36,10 +42,22 @@ class Statistics;
} // namespace rocksdb
namespace td {
struct RocksDbSnapshotStatistics {
void begin_snapshot(const rocksdb::Snapshot *snapshot);
void end_snapshot(const rocksdb::Snapshot *snapshot);
td::Timestamp oldest_snapshot_timestamp() const;
std::string to_string() const;
private:
mutable std::mutex mutex_;
std::map<std::uintptr_t, double> id_to_ts_;
std::set<std::pair<double, std::uintptr_t>> by_ts_;
};
struct RocksDbOptions {
std::shared_ptr<rocksdb::Statistics> statistics = nullptr;
std::shared_ptr<rocksdb::Cache> block_cache; // Default - one 1G cache for all RocksDb
std::shared_ptr<RocksDbSnapshotStatistics> snapshot_statistics = nullptr;
bool use_direct_reads = false;
};

13
tddb/test/key_value.cpp
View file

@ -60,9 +60,20 @@ TEST(KeyValue, simple) {
ensure_value(as_slice(x), as_slice(x));
kv.reset();
kv = std::make_unique<td::RocksDb>(td::RocksDb::open(db_name.str()).move_as_ok());
td::RocksDbOptions options;
options.snapshot_statistics = std::make_shared<td::RocksDbSnapshotStatistics>();
kv = std::make_unique<td::RocksDb>(td::RocksDb::open(db_name.str(), options).move_as_ok());
ensure_value("A", "HELLO");
ensure_value(as_slice(x), as_slice(x));
CHECK(!options.snapshot_statistics->oldest_snapshot_timestamp());
auto snapshot = kv->snapshot();
CHECK(options.snapshot_statistics->oldest_snapshot_timestamp());
auto snapshot2 = kv->snapshot();
snapshot.reset();
CHECK(options.snapshot_statistics->oldest_snapshot_timestamp());
snapshot2.reset();
CHECK(!options.snapshot_statistics->oldest_snapshot_timestamp());
};
TEST(KeyValue, async_simple) {

8
tonlib/tonlib/tonlib-cli.cpp
View file

@ -1356,10 +1356,10 @@ class TonlibCli : public td::actor::Actor {
}
if (l >= 3 && (str[0] == 'x' || str[0] == 'b') && str[1] == '{' && str.back() == '}') {
unsigned char buff[128];
int bits =
(str[0] == 'x')
? (int)td::bitstring::parse_bitstring_hex_literal(buff, sizeof(buff), str.begin() + 2, str.end() - 1)
: (int)td::bitstring::parse_bitstring_binary_literal(buff, sizeof(buff), str.begin() + 2, str.end() - 1);
int bits = (str[0] == 'x') ? (int)td::bitstring::parse_bitstring_hex_literal(buff, sizeof(buff), str.begin() + 2,
str.end() - 1)
: (int)td::bitstring::parse_bitstring_binary_literal(buff, sizeof(buff) * 8,
str.begin() + 2, str.end() - 1);
if (bits < 0) {
return td::Status::Error("Failed to parse slice");
}

10
validator/db/celldb.cpp
View file

@ -84,11 +84,13 @@ void CellDbIn::start_up() {
};
CellDbBase::start_up();
td::RocksDbOptions db_options;
if (!opts_->get_disable_rocksdb_stats()) {
statistics_ = td::RocksDb::create_statistics();
statistics_flush_at_ = td::Timestamp::in(60.0);
snapshot_statistics_ = std::make_shared<td::RocksDbSnapshotStatistics>();
db_options.snapshot_statistics = snapshot_statistics_;
}
td::RocksDbOptions db_options;
db_options.statistics = statistics_;
if (opts_->get_celldb_cache_size()) {
db_options.block_cache = td::RocksDb::create_cache(opts_->get_celldb_cache_size().value());
@ -193,7 +195,11 @@ void CellDbIn::get_cell_db_reader(td::Promise<std::shared_ptr<vm::CellDbReader>>
}
void CellDbIn::flush_db_stats() {
auto stats = td::RocksDb::statistics_to_string(statistics_) + cell_db_statistics_.to_string();
if (opts_->get_disable_rocksdb_stats()) {
return;
}
auto stats = td::RocksDb::statistics_to_string(statistics_) + snapshot_statistics_->to_string() +
cell_db_statistics_.to_string();
auto to_file_r =
td::FileFd::open(path_ + "/db_stats.txt", td::FileFd::Truncate | td::FileFd::Create | td::FileFd::Write, 0644);
if (to_file_r.is_error()) {

2
validator/db/celldb.hpp
View file

@ -27,6 +27,7 @@
#include "auto/tl/ton_api.h"
#include "validator.h"
#include "db-utils.h"
#include "td/db/RocksDb.h"
namespace rocksdb {
class Statistics;
@ -139,6 +140,7 @@ class CellDbIn : public CellDbBase {
};
std::shared_ptr<rocksdb::Statistics> statistics_;
std::shared_ptr<td::RocksDbSnapshotStatistics> snapshot_statistics_;
CellDbStatistics cell_db_statistics_;
td::Timestamp statistics_flush_at_ = td::Timestamp::never();