external/ton
1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-03-09 15:40:10 +00:00
Code Issues Releases Wiki Activity

Merge pull request #908 from ton-blockchain/testnet

Browse source 
Merge developer branch
This commit is contained in:
EmelyanenkoK 2024-02-17 17:07:55 +03:00 • committed by GitHub
commit 73621f626e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
67 changed files with 1836 additions and 310 deletions
Download patch file Download diff file Expand all files Collapse all files

16
Changelog.md
View file

@ -1,3 +1,19 @@
## 2024.02 Update
1. Improvement of validator synchronisation:
* Better handling of block broadcasts -> faster sync
* Additional separate overlay among validators as second option for synchronisation
2. Improvements in LS:
* c7 and library context is fully filled up for server-side rungetmethod
* Cache for runmethods and successfull external messages
* Logging of LS requests statistic
3. Precise control of open files:
* almost instantaneous validator start
* `--max-archive-fd` option
* autoremoval of not used temp archive files
* `--archive-preload-period` option
4. Preparatory (not enabled yet) code for addition on new TVM instructions for cheaper fee calculation onchain.
## 2024.01 Update
1. Fixes in how gas in transactions on special accounts is accounted in block limit. Previously, gas was counted as usual, so to conduct elections that costs >30m gas block limit in masterchain was set to 37m gas. To lower the limit for safety reasons it is proposed to caunt gas on special accounts separately. Besides `gas_max` is set to `special_gas_limit` for all types of transactions on special accounts. New behavior is activated through setting `version >= 5` in `ConfigParam 8;`.

4
adnl/adnl-peer-table.hpp
View file

@ -77,6 +77,10 @@ class AdnlPeerTableImpl : public AdnlPeerTable {
td::actor::ActorId<AdnlChannel> channel) override;
void unregister_channel(AdnlChannelIdShort id) override;
void check_id_exists(AdnlNodeIdShort id, td::Promise<bool> promise) override {
promise.set_value(local_ids_.count(id));
}
void write_new_addr_list_to_db(AdnlNodeIdShort local_id, AdnlNodeIdShort peer_id, AdnlDbItem node,
td::Promise<td::Unit> promise) override;
void get_addr_list_from_db(AdnlNodeIdShort local_id, AdnlNodeIdShort peer_id,

2
adnl/adnl-peer.cpp
View file

@ -212,7 +212,9 @@ void AdnlPeerPairImpl::receive_packet_from_channel(AdnlChannelIdShort id, AdnlPa
VLOG(ADNL_NOTICE) << this << ": dropping IN message: outdated channel id" << id;
return;
}
if (channel_inited_) {
channel_ready_ = true;
}
receive_packet_checked(std::move(packet));
}

2
adnl/adnl.h
View file

@ -97,6 +97,8 @@ class Adnl : public AdnlSenderInterface {
virtual void add_id_ex(AdnlNodeIdFull id, AdnlAddressList addr_list, td::uint8 cat, td::uint32 mode) = 0;
virtual void del_id(AdnlNodeIdShort id, td::Promise<td::Unit> promise) = 0;
virtual void check_id_exists(AdnlNodeIdShort id, td::Promise<bool> promise) = 0;
// subscribe to (some) messages(+queries) to this local id
virtual void subscribe(AdnlNodeIdShort dst, std::string prefix, std::unique_ptr<Callback> callback) = 0;
virtual void unsubscribe(AdnlNodeIdShort dst, std::string prefix) = 0;

8
assembly/nix/build-linux-arm64-nix.sh
View file

@ -24,12 +24,14 @@ if [ "$with_tests" = true ]; then
else
nix-build linux-arm64-static.nix
fi
mkdir artifacts
mkdir -p artifacts/lib
cp ./result/bin/* artifacts/
test $? -eq 0 || { echo "No artifacts have been built..."; exit 1; }
chmod +x artifacts/*
rm -rf result
nix-build linux-arm64-tonlib.nix
cp ./result/lib/libtonlibjson.so.0.5 artifacts/libtonlibjson.so
cp ./result/lib/libemulator.so artifacts/
cp -r crypto/fift/lib artifacts/
cp -r crypto/smartcont artifacts/
cp ./result/lib/fift/* artifacts/lib/
cp -r ./result/share/ton/smartcont artifacts/

7
assembly/nix/build-linux-x86-64-nix.sh
View file

@ -25,12 +25,13 @@ else
nix-build linux-x86-64-static.nix
fi
mkdir artifacts
mkdir -p artifacts/lib
cp ./result/bin/* artifacts/
test $? -eq 0 || { echo "No artifacts have been built..."; exit 1; }
chmod +x artifacts/*
rm -rf result
nix-build linux-x86-64-tonlib.nix
cp ./result/lib/libtonlibjson.so.0.5 artifacts/libtonlibjson.so
cp ./result/lib/libemulator.so artifacts/
cp -r crypto/fift/lib artifacts/
cp -r crypto/smartcont artifacts/
cp ./result/lib/fift/* artifacts/lib/
cp -r ./result/share/ton/smartcont artifacts/

8
assembly/nix/build-macos-nix.sh
View file

@ -22,12 +22,14 @@ if [ "$with_tests" = true ]; then
else
nix-build macos-static.nix
fi
mkdir artifacts
mkdir -p artifacts/lib
cp ./result-bin/bin/* artifacts/
test $? -eq 0 || { echo "No artifacts have been built..."; exit 1; }
chmod +x artifacts/*
rm -rf result-bin
nix-build macos-tonlib.nix
cp ./result/lib/libtonlibjson.dylib artifacts/
cp ./result/lib/libemulator.dylib artifacts/
cp -r crypto/fift/lib artifacts/
cp -r crypto/smartcont artifacts/
cp ./result/lib/fift/* artifacts/lib/
cp -r ./result/share/ton/smartcont artifacts/

2
common/global-version.h
View file

@ -19,6 +19,6 @@
namespace ton {
// See doc/GlobalVersions.md
const int SUPPORTED_VERSION = 5;
const int SUPPORTED_VERSION = 6;
}

1
crypto/CMakeLists.txt
View file

@ -401,6 +401,7 @@ if (USE_EMSCRIPTEN)
target_link_options(funcfiftlib PRIVATE -sALLOW_MEMORY_GROWTH=1)
target_link_options(funcfiftlib PRIVATE -sALLOW_TABLE_GROWTH=1)
target_link_options(funcfiftlib PRIVATE --embed-file ${CMAKE_CURRENT_SOURCE_DIR}/fift/lib@/fiftlib)
target_link_options(funcfiftlib PRIVATE --pre-js ${CMAKE_CURRENT_SOURCE_DIR}/funcfiftlib/funcfiftlib-prejs.js)
target_link_options(funcfiftlib PRIVATE -fexceptions)
target_compile_options(funcfiftlib PRIVATE -fexceptions -fno-stack-protector)
endif()

84
crypto/block/mc-config.cpp
View file

@ -621,12 +621,14 @@ td::Result<std::vector<StoragePrices>> Config::get_storage_prices() const {
}
vm::Dictionary dict{std::move(cell), 32};
if (!dict.check_for_each([&res](Ref<vm::CellSlice> cs_ref, td::ConstBitPtr key, int n) -> bool {
block::gen::StoragePrices::Record data;
if (!tlb::csr_unpack(std::move(cs_ref), data) || data.utime_since != key.get_uint(n)) {
auto r_prices = do_get_one_storage_prices(*cs_ref);
if (r_prices.is_error()) {
return false;
}
res.push_back(r_prices.move_as_ok());
if (res.back().valid_since != key.get_uint(n)) {
return false;
}
res.emplace_back(data.utime_since, data.bit_price_ps, data.cell_price_ps, data.mc_bit_price_ps,
data.mc_cell_price_ps);
return true;
})) {
return td::Status::Error("invalid storage prices dictionary in configuration parameter 18");
@ -634,16 +636,25 @@ td::Result<std::vector<StoragePrices>> Config::get_storage_prices() const {
return std::move(res);
}
td::Result<GasLimitsPrices> Config::do_get_gas_limits_prices(td::Ref<vm::Cell> cell, int id) {
td::Result<StoragePrices> Config::do_get_one_storage_prices(vm::CellSlice cs) {
block::gen::StoragePrices::Record data;
if (!tlb::unpack(cs, data)) {
return td::Status::Error("invalid storage prices dictionary in configuration parameter 18");
}
return StoragePrices{data.utime_since, data.bit_price_ps, data.cell_price_ps, data.mc_bit_price_ps,
data.mc_cell_price_ps};
}
td::Result<GasLimitsPrices> Config::do_get_gas_limits_prices(vm::CellSlice cs, int id) {
GasLimitsPrices res;
auto cs = vm::load_cell_slice(cell);
vm::CellSlice cs0 = cs;
block::gen::GasLimitsPrices::Record_gas_flat_pfx flat;
if (tlb::unpack(cs, flat)) {
cs = *flat.other;
res.flat_gas_limit = flat.flat_gas_limit;
res.flat_gas_price = flat.flat_gas_price;
} else {
cs = vm::load_cell_slice(cell);
cs = cs0;
}
auto f = [&](const auto& r, td::uint64 spec_limit) {
res.gas_limit = r.gas_limit;
@ -654,7 +665,6 @@ td::Result<GasLimitsPrices> Config::do_get_gas_limits_prices(td::Ref<vm::Cell> c
res.delete_due_limit = r.delete_due_limit;
};
block::gen::GasLimitsPrices::Record_gas_prices_ext rec;
vm::CellSlice cs0 = cs;
if (tlb::unpack(cs, rec)) {
f(rec, rec.special_gas_limit);
} else {
@ -689,7 +699,7 @@ td::Result<GasLimitsPrices> Config::get_gas_limits_prices(bool is_masterchain) c
if (cell.is_null()) {
return td::Status::Error(PSLICE() << "configuration parameter " << id << " with gas prices is absent");
}
return do_get_gas_limits_prices(std::move(cell), id);
return do_get_gas_limits_prices(vm::load_cell_slice(cell), id);
}
td::Result<MsgPrices> Config::get_msg_prices(bool is_masterchain) const {
@ -698,7 +708,10 @@ td::Result<MsgPrices> Config::get_msg_prices(bool is_masterchain) const {
if (cell.is_null()) {
return td::Status::Error(PSLICE() << "configuration parameter " << id << " with msg prices is absent");
}
auto cs = vm::load_cell_slice(std::move(cell));
return do_get_msg_prices(vm::load_cell_slice(cell), id);
}
td::Result<MsgPrices> Config::do_get_msg_prices(vm::CellSlice cs, int id) {
block::gen::MsgForwardPrices::Record rec;
if (!tlb::unpack(cs, rec)) {
return td::Status::Error(PSLICE() << "configuration parameter " << id
@ -1917,10 +1930,17 @@ std::vector<ton::ValidatorDescr> Config::compute_total_validator_set(int next) c
}
td::Result<SizeLimitsConfig> Config::get_size_limits_config() const {
SizeLimitsConfig limits;
td::Ref<vm::Cell> param = get_config_param(43);
if (param.is_null()) {
return limits;
return do_get_size_limits_config({});
}
return do_get_size_limits_config(vm::load_cell_slice_ref(param));
}
td::Result<SizeLimitsConfig> Config::do_get_size_limits_config(td::Ref<vm::CellSlice> cs) {
SizeLimitsConfig limits;
if (cs.is_null()) {
return limits; // default values
}
auto unpack_v1 = [&](auto& rec) {
limits.max_msg_bits = rec.max_msg_bits;
@ -1939,9 +1959,9 @@ td::Result<SizeLimitsConfig> Config::get_size_limits_config() const {
};
gen::SizeLimitsConfig::Record_size_limits_config rec_v1;
gen::SizeLimitsConfig::Record_size_limits_config_v2 rec_v2;
if (tlb::unpack_cell(param, rec_v1)) {
if (tlb::csr_unpack(cs, rec_v1)) {
unpack_v1(rec_v1);
} else if (tlb::unpack_cell(param, rec_v2)) {
} else if (tlb::csr_unpack(cs, rec_v2)) {
unpack_v2(rec_v2);
} else {
return td::Status::Error("configuration parameter 43 is invalid");
@ -1976,6 +1996,42 @@ BurningConfig Config::get_burning_config() const {
return c;
}
td::Ref<vm::Tuple> Config::get_unpacked_config_tuple(ton::UnixTime now) const {
auto get_param = [&](td::int32 idx) -> vm::StackEntry {
auto cell = get_config_param(idx);
if (cell.is_null()) {
return {};
}
return vm::load_cell_slice_ref(cell);
};
auto get_current_storage_prices = [&]() -> vm::StackEntry {
auto cell = get_config_param(18);
if (cell.is_null()) {
return {};
}
vm::StackEntry res;
vm::Dictionary dict{std::move(cell), 32};
dict.check_for_each([&](Ref<vm::CellSlice> cs_ref, td::ConstBitPtr key, int n) -> bool {
auto utime_since = key.get_uint(n);
if (now >= utime_since) {
res = std::move(cs_ref);
return true;
}
return false;
});
return res;
};
std::vector<vm::StackEntry> tuple;
tuple.push_back(get_current_storage_prices()); // storage_prices
tuple.push_back(get_param(19)); // global_id
tuple.push_back(get_param(20)); // config_mc_gas_prices
tuple.push_back(get_param(21)); // config_gas_prices
tuple.push_back(get_param(24)); // config_mc_fwd_prices
tuple.push_back(get_param(25)); // config_fwd_prices
tuple.push_back(get_param(43)); // size_limits_config
return td::make_cnt_ref<std::vector<vm::StackEntry>>(std::move(tuple));
}
td::Result<std::pair<ton::UnixTime, ton::UnixTime>> Config::unpack_validator_set_start_stop(Ref<vm::Cell> vset_root) {
if (vset_root.is_null()) {
return td::Status::Error("validator set absent");

13
crypto/block/mc-config.h
View file

@ -350,7 +350,11 @@ struct GasLimitsPrices {
td::uint64 freeze_due_limit{0};
td::uint64 delete_due_limit{0};
td::RefInt256 compute_gas_price(td::uint64 gas_used) const;
td::RefInt256 compute_gas_price(td::uint64 gas_used) const {
return gas_used <= flat_gas_limit
? td::make_refint(flat_gas_price)
: td::rshift(td::make_refint(gas_price) * (gas_used - flat_gas_limit), 16, 1) + flat_gas_price;
}
};
// msg_fwd_fees = (lump_price + ceil((bit_price * msg.bits + cell_price * msg.cells)/2^16)) nanograms
@ -365,6 +369,7 @@ struct MsgPrices {
td::uint32 first_frac;
td::uint32 next_frac;
td::uint64 compute_fwd_fees(td::uint64 cells, td::uint64 bits) const;
td::RefInt256 compute_fwd_fees256(td::uint64 cells, td::uint64 bits) const;
std::pair<td::uint64, td::uint64> compute_fwd_ihr_fees(td::uint64 cells, td::uint64 bits,
bool ihr_disabled = false) const;
MsgPrices() = default;
@ -604,9 +609,11 @@ class Config {
bool is_special_smartcontract(const ton::StdSmcAddress& addr) const;
static td::Result<std::unique_ptr<ValidatorSet>> unpack_validator_set(Ref<vm::Cell> valset_root);
td::Result<std::vector<StoragePrices>> get_storage_prices() const;
static td::Result<StoragePrices> do_get_one_storage_prices(vm::CellSlice cs);
td::Result<GasLimitsPrices> get_gas_limits_prices(bool is_masterchain = false) const;
static td::Result<GasLimitsPrices> do_get_gas_limits_prices(td::Ref<vm::Cell> cell, int id);
static td::Result<GasLimitsPrices> do_get_gas_limits_prices(vm::CellSlice cs, int id);
td::Result<MsgPrices> get_msg_prices(bool is_masterchain = false) const;
static td::Result<MsgPrices> do_get_msg_prices(vm::CellSlice cs, int id);
static CatchainValidatorsConfig unpack_catchain_validators_config(Ref<vm::Cell> cell);
CatchainValidatorsConfig get_catchain_validators_config() const;
td::Status visit_validator_params() const;
@ -633,8 +640,10 @@ class Config {
ton::CatchainSeqno cc_seqno) const;
std::vector<ton::ValidatorDescr> compute_total_validator_set(int next) const;
td::Result<SizeLimitsConfig> get_size_limits_config() const;
static td::Result<SizeLimitsConfig> do_get_size_limits_config(td::Ref<vm::CellSlice> cs);
std::unique_ptr<vm::Dictionary> get_suspended_addresses(ton::UnixTime now) const;
BurningConfig get_burning_config() const;
td::Ref<vm::Tuple> get_unpacked_config_tuple(ton::UnixTime now) const;
static std::vector<ton::ValidatorDescr> do_compute_validator_set(const block::CatchainValidatorsConfig& ccv_conf,
ton::ShardIdFull shard,
const block::ValidatorSet& vset, ton::UnixTime time,

27
crypto/block/transaction.cpp
View file

@ -1337,6 +1337,11 @@ Ref<vm::Tuple> Transaction::prepare_vm_c7(const ComputePhaseConfig& cfg) const {
// may only return tuple or raise Error (See crypto/block/mc-config.cpp#2223)
tuple.push_back(cfg.prev_blocks_info.not_null() ? vm::StackEntry(cfg.prev_blocks_info) : vm::StackEntry());
}
if (cfg.global_version >= 6) {
tuple.push_back(cfg.unpacked_config_tuple.not_null() ? vm::StackEntry(cfg.unpacked_config_tuple)
: vm::StackEntry()); // unpacked_config_tuple:[...]
tuple.push_back(due_payment.not_null() ? due_payment : td::zero_refint()); // due_payment:Integer
}
auto tuple_ref = td::make_cnt_ref<std::vector<vm::StackEntry>>(std::move(tuple));
LOG(DEBUG) << "SmartContractInfo initialized with " << vm::StackEntry(tuple_ref).to_string();
return vm::make_tuple_ref(std::move(tuple_ref));
@ -1920,6 +1925,25 @@ td::uint64 MsgPrices::compute_fwd_fees(td::uint64 cells, td::uint64 bits) const
.lo();
}
/**
* Computes the forward fees for a message based on the number of cells and bits.
* Return the result as td::RefInt256
*
* msg_fwd_fees = (lump_price + ceil((bit_price * msg.bits + cell_price * msg.cells)/2^16)) nanograms
* ihr_fwd_fees = ceil((msg_fwd_fees * ihr_price_factor)/2^16) nanograms
* bits in the root cell of a message are not included in msg.bits (lump_price pays for them)
*
* @param cells The number of cells in the message.
* @param bits The number of bits in the message.
*
* @returns The computed forward fees for the message as td::RefInt256j.
*/
td::RefInt256 MsgPrices::compute_fwd_fees256(td::uint64 cells, td::uint64 bits) const {
return td::make_refint(lump_price) +
td::rshift(td::make_refint(bit_price) * bits + td::make_refint(cell_price) * cells, 16,
1); // divide by 2^16 with ceil rounding
}
/**
* Computes the forward fees and IHR fees for a message with the given number of cells and bits.
*
@ -3539,6 +3563,9 @@ td::Status FetchConfigParams::fetch_config_params(
if (compute_phase_cfg->global_version >= 4) {
compute_phase_cfg->prev_blocks_info = std::move(prev_blocks_info);
}
if (compute_phase_cfg->global_version >= 6) {
compute_phase_cfg->unpacked_config_tuple = config.get_unpacked_config_tuple(now);
}
compute_phase_cfg->suspended_addresses = config.get_suspended_addresses(now);
compute_phase_cfg->size_limits = size_limits;
}

1
crypto/block/transaction.h
View file

@ -117,6 +117,7 @@ struct ComputePhaseConfig {
td::uint16 max_vm_data_depth = 512;
int global_version = 0;
Ref<vm::Tuple> prev_blocks_info;
Ref<vm::Tuple> unpacked_config_tuple;
std::unique_ptr<vm::Dictionary> suspended_addresses;
SizeLimitsConfig size_limits;
int vm_log_verbosity = 0;

17
crypto/fift/lib/Asm.fif
View file

@ -2,7 +2,7 @@ library TVM_Asm
// simple TVM Assembler
namespace Asm
Asm definitions
"0.4.4" constant asm-fif-version
"0.4.5" constant asm-fif-version
variable @atend
variable @was-split
@ -813,6 +813,12 @@ x{D761} @Defop LDONES
x{D762} @Defop LDSAME
x{D764} @Defop SDEPTH
x{D765} @Defop CDEPTH
x{D766} @Defop CLEVEL
x{D767} @Defop CLEVELMASK
{ <b x{D76A_} s, swap 2 u, @addopb } : CHASHI
{ <b x{D76E_} s, swap 2 u, @addopb } : CDEPTHI
x{D770} @Defop CHASHIX
x{D771} @Defop CDEPTHIX
//
// continuation / flow control primitives
x{D8} dup @Defop EXECUTE @Defop CALLX
@ -1295,12 +1301,21 @@ x{F82A} @Defop MYCODE
x{F82B} @Defop INCOMINGVALUE
x{F82C} @Defop STORAGEFEES
x{F82D} @Defop PREVBLOCKSINFOTUPLE
x{F82E} @Defop UNPACKEDCONFIGTUPLE
x{F82F} @Defop DUEPAYMENT
x{F830} @Defop CONFIGDICT
x{F832} @Defop CONFIGPARAM
x{F833} @Defop CONFIGOPTPARAM
x{F83400} @Defop PREVMCBLOCKS
x{F83401} @Defop PREVKEYBLOCK
x{F835} @Defop GLOBALID
x{F836} @Defop GETGASFEE
x{F837} @Defop GETSTORAGEFEE
x{F838} @Defop GETFORWARDFEE
x{F839} @Defop GETPRECOMPILEDGAS
x{F83A} @Defop GETORIGINALFWDFEE
x{F83B} @Defop GETGASFEESIMPLE
x{F83C} @Defop GETFORWARDFEESIMPLE
x{F840} @Defop GETGLOBVAR
{ dup 1 31 @rangechk <b x{F85_} s, swap 5 u, @addopb } : GETGLOB

1
crypto/funcfiftlib/funcfiftlib-prejs.js Normal file
View file

@ -0,0 +1 @@
var crypto = { getRandomValues: function(array) { for (var i = 0; i < array.length; i++) array[i] = (Math.random()*256)|0 } };

25
crypto/funcfiftlib/funcfiftlib.cpp
View file

@ -34,29 +34,6 @@
#include <sstream>
#include <iomanip>
std::string escape_json(const std::string &s) {
std::ostringstream o;
for (auto c = s.cbegin(); c != s.cend(); c++) {
switch (*c) {
case '"': o << "\\\""; break;
case '\\': o << "\\\\"; break;
case '\b': o << "\\b"; break;
case '\f': o << "\\f"; break;
case '\n': o << "\\n"; break;
case '\r': o << "\\r"; break;
case '\t': o << "\\t"; break;
default:
if ('\x00' <= *c && *c <= '\x1f') {
o << "\\u"
<< std::hex << std::setw(4) << std::setfill('0') << static_cast<int>(*c);
} else {
o << *c;
}
}
}
return o.str();
}
td::Result<std::string> compile_internal(char *config_json) {
TRY_RESULT(input_json, td::json_decode(td::MutableSlice(config_json)))
auto &obj = input_json.get_object();
@ -91,7 +68,7 @@ td::Result<std::string> compile_internal(char *config_json) {
auto result_obj = result_json.enter_object();
result_obj("status", "ok");
result_obj("codeBoc", td::base64_encode(boc));
result_obj("fiftCode", escape_json(outs.str()));
result_obj("fiftCode", outs.str());
result_obj("codeHashHex", code_cell->get_hash().to_hex());
result_obj.leave();

4
crypto/smc-envelope/SmartContract.cpp
View file

@ -171,6 +171,10 @@ td::Ref<vm::Tuple> prepare_vm_c7(SmartContract::Args args, td::Ref<vm::Cell> cod
// prev_key_block:BlockId ] : PrevBlocksInfo
tuple.push_back(args.prev_blocks_info ? args.prev_blocks_info.value() : vm::StackEntry{}); // prev_block_info
}
if (args.config && args.config.value()->get_global_version() >= 6) {
tuple.push_back(args.config.value()->get_unpacked_config_tuple(now)); // unpacked_config_tuple
tuple.push_back(td::zero_refint()); // due_payment
}
auto tuple_ref = td::make_cnt_ref<std::vector<vm::StackEntry>>(std::move(tuple));
//LOG(DEBUG) << "SmartContractInfo initialized with " << vm::StackEntry(tuple).to_string();
return vm::make_tuple_ref(std::move(tuple_ref));

4
crypto/test/fift.cpp
View file

@ -163,3 +163,7 @@ TEST(Fift, test_bls) {
TEST(Fift, test_bls_ops) {
run_fift("bls_ops.fif");
}
TEST(Fift, test_levels) {
run_fift("levels.fif");
}

75
crypto/test/fift/levels.fif Normal file
View file

@ -0,0 +1,75 @@
"Asm.fif" include
"FiftExt.fif" include
{
dup
."Cell " .dump cr
dup [[ <{ CLEVEL }>s ]] 0 runvmx abort"exitcode != 0" ."Level = " . cr
dup [[ <{ CLEVELMASK }>s ]] 0 runvmx abort"exitcode != 0" ."Level mask = 0b" b. cr
dup dup [[ <{ 0 CHASHI DUP ROT 0 INT CHASHIX EQUAL 55 THROWIFNOT }>s ]] 0 runvmx abort"exitcode != 0" ."Hash_0 = " X. cr
dup dup [[ <{ 1 CHASHI DUP ROT 1 INT CHASHIX EQUAL 55 THROWIFNOT }>s ]] 0 runvmx abort"exitcode != 0" ."Hash_1 = " X. cr
dup dup [[ <{ 2 CHASHI DUP ROT 2 INT CHASHIX EQUAL 55 THROWIFNOT }>s ]] 0 runvmx abort"exitcode != 0" ."Hash_2 = " X. cr
dup dup [[ <{ 3 CHASHI DUP ROT 3 INT CHASHIX EQUAL 55 THROWIFNOT }>s ]] 0 runvmx abort"exitcode != 0" ."Hash_3 = " X. cr
dup dup [[ <{ 0 CDEPTHI DUP ROT 0 INT CDEPTHIX EQUAL 55 THROWIFNOT }>s ]] 0 runvmx abort"exitcode != 0" ."Depth_0 = " . cr
dup dup [[ <{ 1 CDEPTHI DUP ROT 1 INT CDEPTHIX EQUAL 55 THROWIFNOT }>s ]] 0 runvmx abort"exitcode != 0" ."Depth_1 = " . cr
dup dup [[ <{ 2 CDEPTHI DUP ROT 2 INT CDEPTHIX EQUAL 55 THROWIFNOT }>s ]] 0 runvmx abort"exitcode != 0" ."Depth_2 = " . cr
dup dup [[ <{ 3 CDEPTHI DUP ROT 3 INT CDEPTHIX EQUAL 55 THROWIFNOT }>s ]] 0 runvmx abort"exitcode != 0" ."Depth_3 = " . cr
drop
cr
} : print-all
// Ordinary cell of level 0
<b
123 32 u,
<b <b 22 32 u, b> ref, b> ref,
<b b> ref,
b>
print-all
// Prunned branch of level 1
<b
1 8 u,
1 8 u,
0xabcd1111abcd1111abcd1111abcd1111abcd1111abcd1111abcd1111abcd1111 256 u,
14 16 u,
b>spec
print-all
// Prunned branch of level 3
<b
1 8 u,
7 8 u,
0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 256 u,
0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb 256 u,
0xcccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc 256 u,
55 16 u,
44 16 u,
33 16 u,
b>spec
print-all
// Prunned branch of level 3, mask 0b101
<b
1 8 u,
5 8 u,
0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 256 u,
0xcccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc 256 u,
55 16 u,
33 16 u,
b>spec
print-all
// Tree with the previous cell inside
<b
<b
<b
1 8 u,
5 8 u,
0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 256 u,
0xcccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc 256 u,
55 16 u,
33 16 u,
b>spec ref,
b> ref,
b>
print-all

57
crypto/vm/cellops.cpp
View file

@ -1391,6 +1391,55 @@ int exec_slice_depth(VmState* st) {
return 0;
}
int exec_cell_level(VmState* st) {
Stack& stack = st->get_stack();
VM_LOG(st) << "execute CLEVEL";
auto cell = stack.pop_cell();
stack.push_smallint(cell->get_level());
return 0;
}
int exec_cell_level_mask(VmState* st) {
Stack& stack = st->get_stack();
VM_LOG(st) << "execute CLEVELMASK";
auto cell = stack.pop_cell();
stack.push_smallint(cell->get_level_mask().get_mask());
return 0;
}
int exec_cell_hash_i(VmState* st, unsigned args, bool var) {
unsigned i;
Stack& stack = st->get_stack();
if (var) {
VM_LOG(st) << "execute CHASHIX";
i = stack.pop_smallint_range(3);
} else {
i = args & 3;
VM_LOG(st) << "execute CHASHI " << i;
}
auto cell = stack.pop_cell();
std::array<unsigned char, 32> hash = cell->get_hash(i).as_array();
td::RefInt256 res{true};
CHECK(res.write().import_bytes(hash.data(), hash.size(), false));
stack.push_int(std::move(res));
return 0;
}
int exec_cell_depth_i(VmState* st, unsigned args, bool var) {
unsigned i;
Stack& stack = st->get_stack();
if (var) {
VM_LOG(st) << "execute CDEPTHIX";
i = stack.pop_smallint_range(3);
} else {
i = args & 3;
VM_LOG(st) << "execute CDEPTHI " << i;
}
auto cell = stack.pop_cell();
stack.push_smallint(cell->get_depth(i));
return 0;
}
void register_cell_deserialize_ops(OpcodeTable& cp0) {
using namespace std::placeholders;
cp0.insert(OpcodeInstr::mksimple(0xd0, 8, "CTOS", exec_cell_to_slice))
@ -1479,7 +1528,13 @@ void register_cell_deserialize_ops(OpcodeTable& cp0) {
.insert(OpcodeInstr::mksimple(0xd761, 16, "LDONES", std::bind(exec_load_same, _1, "LDONES", 1)))
.insert(OpcodeInstr::mksimple(0xd762, 16, "LDSAME", std::bind(exec_load_same, _1, "LDSAME", -1)))
.insert(OpcodeInstr::mksimple(0xd764, 16, "SDEPTH", exec_slice_depth))
.insert(OpcodeInstr::mksimple(0xd765, 16, "CDEPTH", exec_cell_depth));
.insert(OpcodeInstr::mksimple(0xd765, 16, "CDEPTH", exec_cell_depth))
.insert(OpcodeInstr::mksimple(0xd766, 16, "CLEVEL", exec_cell_level)->require_version(6))
.insert(OpcodeInstr::mksimple(0xd767, 16, "CLEVELMASK", exec_cell_level_mask)->require_version(6))
.insert(OpcodeInstr::mkfixed(0xd768 >> 2, 14, 2, instr::dump_1c_and(3, "CHASHI "), std::bind(exec_cell_hash_i, _1, _2, false))->require_version(6))
.insert(OpcodeInstr::mkfixed(0xd76c >> 2, 14, 2, instr::dump_1c_and(3, "CDEPTHI "), std::bind(exec_cell_depth_i, _1, _2, false))->require_version(6))
.insert(OpcodeInstr::mksimple(0xd770, 16, "CHASHIX ", std::bind(exec_cell_hash_i, _1, 0, true))->require_version(6))
.insert(OpcodeInstr::mksimple(0xd771, 16, "CDEPTHIX ", std::bind(exec_cell_depth_i, _1, 0, true))->require_version(6));
}
void register_cell_ops(OpcodeTable& cp0) {

187
crypto/vm/tonops.cpp
View file

@ -35,6 +35,7 @@
#include "openssl/digest.hpp"
#include <sodium.h>
#include "bls.h"
#include "mc-config.h"
namespace vm {
@ -122,6 +123,20 @@ static const StackEntry& get_param(VmState* st, unsigned idx) {
return tuple_index(t1, idx);
}
// ConfigParams: 18 (only one entry), 19, 20, 21, 24, 25, 43
static td::Ref<CellSlice> get_unpacked_config_param(VmState* st, unsigned idx) {
auto tuple = st->get_c7();
auto t1 = tuple_index(tuple, 0).as_tuple_range(255);
if (t1.is_null()) {
throw VmError{Excno::type_chk, "intermediate value is not a tuple"};
}
auto t2 = tuple_index(t1, 14).as_tuple_range(255);
if (t2.is_null()) {
throw VmError{Excno::type_chk, "intermediate value is not a tuple"};
}
return tuple_index(t2, idx).as_slice();
}
int exec_get_param(VmState* st, unsigned idx, const char* name) {
if (name) {
VM_LOG(st) << "execute " << name;
@ -232,7 +247,18 @@ int exec_get_prev_blocks_info(VmState* st, unsigned idx, const char* name) {
}
int exec_get_global_id(VmState* st) {
Ref<Cell> config = get_param(st, 9).as_cell();
VM_LOG(st) << "execute GLOBALID";
if (st->get_global_version() >= 6) {
Ref<CellSlice> cs = get_unpacked_config_param(st, 1);
if (cs.is_null()) {
throw VmError{Excno::type_chk, "intermediate value is not a slice"};
}
if (cs->size() < 32) {
throw VmError{Excno::cell_und, "invalid global-id config"};
}
st->get_stack().push_smallint(cs->prefetch_long(32));
} else {
Ref<Cell> config = get_param(st, 19).as_cell();
if (config.is_null()) {
throw VmError{Excno::type_chk, "intermediate value is not a cell"};
}
@ -246,6 +272,125 @@ int exec_get_global_id(VmState* st) {
throw VmError{Excno::unknown, "invalid global-id config"};
}
st->get_stack().push_smallint(cs.fetch_long(32));
}
return 0;
}
static block::GasLimitsPrices get_gas_prices(VmState* st, bool is_masterchain) {
Ref<CellSlice> cs = get_unpacked_config_param(st, is_masterchain ? 2 : 3);
if (cs.is_null()) {
throw VmError{Excno::type_chk, "intermediate value is not a slice"};
}
auto r_prices = block::Config::do_get_gas_limits_prices(*cs, is_masterchain ? 20 : 21);
if (r_prices.is_error()) {
throw VmError{Excno::cell_und, PSTRING() << "cannot parse config: " << r_prices.error().message()};
}
return r_prices.move_as_ok();
}
static block::MsgPrices get_msg_prices(VmState* st, bool is_masterchain) {
Ref<CellSlice> cs = get_unpacked_config_param(st, is_masterchain ? 4 : 5);
if (cs.is_null()) {
throw VmError{Excno::type_chk, "intermediate value is not a slice"};
}
auto r_prices = block::Config::do_get_msg_prices(*cs, is_masterchain ? 24 : 25);
if (r_prices.is_error()) {
throw VmError{Excno::cell_und, PSTRING() << "cannot parse config: " << r_prices.error().message()};
}
return r_prices.move_as_ok();
}
int exec_get_gas_fee(VmState* st) {
VM_LOG(st) << "execute GETGASFEE";
Stack& stack = st->get_stack();
bool is_masterchain = stack.pop_bool();
td::uint64 gas = stack.pop_long_range(std::numeric_limits<td::int64>::max(), 0);
block::GasLimitsPrices prices = get_gas_prices(st, is_masterchain);
stack.push_int(prices.compute_gas_price(gas));
return 0;
}
int exec_get_storage_fee(VmState* st) {
VM_LOG(st) << "execute GETSTORAGEFEE";
Stack& stack = st->get_stack();
bool is_masterchain = stack.pop_bool();
td::int64 delta = stack.pop_long_range(std::numeric_limits<td::int64>::max(), 0);
td::uint64 bits = stack.pop_long_range(std::numeric_limits<td::int64>::max(), 0);
td::uint64 cells = stack.pop_long_range(std::numeric_limits<td::int64>::max(), 0);
Ref<CellSlice> cs = get_unpacked_config_param(st, 0);
if (cs.is_null()) {
// null means tat no StoragePrices is active, so the price is 0
stack.push_smallint(0);
return 0;
}
auto r_prices = block::Config::do_get_one_storage_prices(*cs);
if (r_prices.is_error()) {
throw VmError{Excno::cell_und, PSTRING() << "cannot parse config: " << r_prices.error().message()};
}
block::StoragePrices prices = r_prices.move_as_ok();
td::RefInt256 total;
if (is_masterchain) {
total = td::make_refint(cells) * prices.mc_cell_price;
total += td::make_refint(bits) * prices.mc_bit_price;
} else {
total = td::make_refint(cells) * prices.cell_price;
total += td::make_refint(bits) * prices.bit_price;
}
total *= delta;
stack.push_int(td::rshift(total, 16, 1));
return 0;
}
int exec_get_forward_fee(VmState* st) {
VM_LOG(st) << "execute GETFORWARDFEE";
Stack& stack = st->get_stack();
bool is_masterchain = stack.pop_bool();
td::uint64 bits = stack.pop_long_range(std::numeric_limits<td::int64>::max(), 0);
td::uint64 cells = stack.pop_long_range(std::numeric_limits<td::int64>::max(), 0);
block::MsgPrices prices = get_msg_prices(st, is_masterchain);
stack.push_int(prices.compute_fwd_fees256(cells, bits));
return 0;
}
int exec_get_precompiled_gas(VmState* st) {
VM_LOG(st) << "execute GETPRECOMPILEDGAS";
Stack& stack = st->get_stack();
stack.push_null();
return 0;
}
int exec_get_original_fwd_fee(VmState* st) {
VM_LOG(st) << "execute GETORIGINALFWDFEE";
Stack& stack = st->get_stack();
bool is_masterchain = stack.pop_bool();
td::RefInt256 fwd_fee = stack.pop_int_finite();
if (fwd_fee->sgn() < 0) {
throw VmError{Excno::range_chk, "fwd_fee is negative"};
}
block::MsgPrices prices = get_msg_prices(st, is_masterchain);
stack.push_int(td::muldiv(fwd_fee, td::make_refint(1 << 16), td::make_refint((1 << 16) - prices.first_frac)));
return 0;
}
int exec_get_gas_fee_simple(VmState* st) {
VM_LOG(st) << "execute GETGASFEESIMPLE";
Stack& stack = st->get_stack();
bool is_masterchain = stack.pop_bool();
td::uint64 gas = stack.pop_long_range(std::numeric_limits<td::int64>::max(), 0);
block::GasLimitsPrices prices = get_gas_prices(st, is_masterchain);
stack.push_int(td::rshift(td::make_refint(prices.gas_price) * gas, 16, 1));
return 0;
}
int exec_get_forward_fee_simple(VmState* st) {
VM_LOG(st) << "execute GETFORWARDFEESIMPLE";
Stack& stack = st->get_stack();
bool is_masterchain = stack.pop_bool();
td::uint64 bits = stack.pop_long_range(std::numeric_limits<td::int64>::max(), 0);
td::uint64 cells = stack.pop_long_range(std::numeric_limits<td::int64>::max(), 0);
block::MsgPrices prices = get_msg_prices(st, is_masterchain);
stack.push_int(td::rshift(td::make_refint(prices.bit_price) * bits + td::make_refint(prices.cell_price) * cells, 16,
1)); // divide by 2^16 with ceil rounding
return 0;
}
@ -263,13 +408,21 @@ void register_ton_config_ops(OpcodeTable& cp0) {
.insert(OpcodeInstr::mksimple(0xf82b, 16, "INCOMINGVALUE", std::bind(exec_get_param, _1, 11, "INCOMINGVALUE")))
.insert(OpcodeInstr::mksimple(0xf82c, 16, "STORAGEFEES", std::bind(exec_get_param, _1, 12, "STORAGEFEES")))
.insert(OpcodeInstr::mksimple(0xf82d, 16, "PREVBLOCKSINFOTUPLE", std::bind(exec_get_param, _1, 13, "PREVBLOCKSINFOTUPLE")))
.insert(OpcodeInstr::mkfixedrange(0xf82e, 0xf830, 16, 4, instr::dump_1c("GETPARAM "), exec_get_var_param))
.insert(OpcodeInstr::mksimple(0xf82e, 16, "UNPACKEDCONFIGTUPLE", std::bind(exec_get_param, _1, 14, "UNPACKEDCONFIGTUPLE")))
.insert(OpcodeInstr::mksimple(0xf82f, 16, "DUEPAYMENT", std::bind(exec_get_param, _1, 15, "DUEPAYMENT")))
.insert(OpcodeInstr::mksimple(0xf830, 16, "CONFIGDICT", exec_get_config_dict))
.insert(OpcodeInstr::mksimple(0xf832, 16, "CONFIGPARAM", std::bind(exec_get_config_param, _1, false)))
.insert(OpcodeInstr::mksimple(0xf833, 16, "CONFIGOPTPARAM", std::bind(exec_get_config_param, _1, true)))
.insert(OpcodeInstr::mksimple(0xf83400, 24, "PREVMCBLOCKS", std::bind(exec_get_prev_blocks_info, _1, 0, "PREVMCBLOCKS"))->require_version(4))
.insert(OpcodeInstr::mksimple(0xf83401, 24, "PREVKEYBLOCK", std::bind(exec_get_prev_blocks_info, _1, 1, "PREVKEYBLOCK"))->require_version(4))
.insert(OpcodeInstr::mksimple(0xf835, 16, "GLOBALID", exec_get_global_id)->require_version(4))
.insert(OpcodeInstr::mksimple(0xf836, 16, "GETGASFEE", exec_get_gas_fee)->require_version(6))
.insert(OpcodeInstr::mksimple(0xf837, 16, "GETSTORAGEFEE", exec_get_storage_fee)->require_version(6))
.insert(OpcodeInstr::mksimple(0xf838, 16, "GETFORWARDFEE", exec_get_forward_fee)->require_version(6))
.insert(OpcodeInstr::mksimple(0xf839, 16, "GETPRECOMPILEDGAS", exec_get_precompiled_gas)->require_version(6))
.insert(OpcodeInstr::mksimple(0xf83a, 16, "GETORIGINALFWDFEE", exec_get_original_fwd_fee)->require_version(6))
.insert(OpcodeInstr::mksimple(0xf83b, 16, "GETGASFEESIMPLE", exec_get_gas_fee_simple)->require_version(6))
.insert(OpcodeInstr::mksimple(0xf83c, 16, "GETFORWARDFEESIMPLE", exec_get_forward_fee_simple)->require_version(6))
.insert(OpcodeInstr::mksimple(0xf840, 16, "GETGLOBVAR", exec_get_global_var))
.insert(OpcodeInstr::mkfixedrange(0xf841, 0xf860, 16, 5, instr::dump_1c_and(31, "GETGLOB "), exec_get_global))
.insert(OpcodeInstr::mksimple(0xf860, 16, "SETGLOBVAR", exec_set_global_var))
@ -1592,17 +1745,39 @@ int exec_send_message(VmState* st) {
}
bool is_masterchain = parse_addr_workchain(*my_addr) == -1 || (!ext_msg && parse_addr_workchain(*dest) == -1);
td::Ref<CellSlice> prices_cs;
if (st->get_global_version() >= 6) {
prices_cs = get_unpacked_config_param(st, is_masterchain ? 4 : 5);
} else {
Ref<Cell> config_dict = get_param(st, 9).as_cell();
Dictionary config{config_dict, 32};
Ref<Cell> prices_cell = config.lookup_ref(td::BitArray<32>{is_masterchain ? 24 : 25});
block::gen::MsgForwardPrices::Record prices;
if (prices_cell.is_null() || !tlb::unpack_cell(std::move(prices_cell), prices)) {
if (prices_cell.not_null()) {
prices_cs = load_cell_slice_ref(prices_cell);
}
}
if (prices_cs.is_null()) {
throw VmError{Excno::unknown, "invalid prices config"};
}
auto r_prices = block::Config::do_get_msg_prices(*prices_cs, is_masterchain ? 24 : 25);
if (r_prices.is_error()) {
throw VmError{Excno::cell_und, PSTRING() << "cannot parse config: " << r_prices.error().message()};
}
block::MsgPrices prices = r_prices.move_as_ok();
// msg_fwd_fees = (lump_price + ceil((bit_price * msg.bits + cell_price * msg.cells)/2^16)) nanograms
// bits in the root cell of a message are not included in msg.bits (lump_price pays for them)
vm::VmStorageStat stat(1 << 13);
td::uint64 max_cells;
if (st->get_global_version() >= 6) {
auto r_size_limits_config = block::Config::do_get_size_limits_config(get_unpacked_config_param(st, 6));
if (r_size_limits_config.is_error()) {
throw VmError{Excno::cell_und, PSTRING() << "cannot parse config: " << r_size_limits_config.error().message()};
}
max_cells = r_size_limits_config.ok().max_msg_cells;
} else {
max_cells = 1 << 13;
}
vm::VmStorageStat stat(max_cells);
CellSlice cs = load_cell_slice(msg_cell);
cs.skip_first(cs.size());
stat.add_storage(cs);
@ -1650,7 +1825,7 @@ int exec_send_message(VmState* st) {
if (ihr_disabled) {
ihr_fee_short = 0;
} else {
ihr_fee_short = td::uint128(fwd_fee_short).mult(prices.ihr_price_factor).shr(16).lo();
ihr_fee_short = td::uint128(fwd_fee_short).mult(prices.ihr_factor).shr(16).lo();
}
fwd_fee = td::RefInt256{true, fwd_fee_short};
ihr_fee = td::RefInt256{true, ihr_fee_short};

42
doc/GlobalVersions.md
View file

@ -54,3 +54,45 @@ See [this post](https://t.me/tonstatus/88) for details.
* Loading "nested libraries" (i.e. a library cell that points to another library cell) throws an exception.
* Loading a library consumes gas for cell load only once (for the library cell), not twice (both for the library cell and the cell in the library).
* `XLOAD` now works differently. When it takes a library cell, it returns the cell that it points to. This allows loading "nested libraries", if needed.
## Version 6
### c7 tuple
**c7** tuple extended from 14 to 16 elements:
* **14**: tuple that contains some config parameters as cell slices. If the parameter is absent from the config, the value is null. Asm opcode: `UNPACKEDCONFIGTUPLE`.
* **0**: `StoragePrices` from `ConfigParam 18`. Not the whole dict, but only the one StoragePrices entry (one which corresponds to the current time).
* **1**: `ConfigParam 19` (global id).
* **2**: `ConfigParam 20` (mc gas prices).
* **3**: `ConfigParam 21` (gas prices).
* **4**: `ConfigParam 24` (mc fwd fees).
* **5**: `ConfigParam 25` (fwd fees).
* **6**: `ConfigParam 43` (size limits).
* **15**: "[due payment](https://github.com/ton-blockchain/ton/blob/8a9ff339927b22b72819c5125428b70c406da631/crypto/block/block.tlb#L237)" - current debt for storage fee (nanotons). Asm opcode: `DUEPAYMENT`.
### New TVM instructions
#### Fee calculation
* `GETGASFEE` (`gas_used is_mc - price`) - calculates gas fee.
* `GETSTORAGEFEE` (`cells bits seconds is_mc - price`) - calculates storage fees (only current StoragePrices entry is used).
* `GETFORWARDFEE` (`cells bits is_mc - price`) - calculates forward fee.
* `GETPRECOMPILEDGAS` (`- null`) - reserved, currently returns `null`.
* `GETORIGINALFWDFEE` (`fwd_fee is_mc - orig_fwd_fee`) - calculate `fwd_fee * 2^16 / first_frac`. Can be used to get the original `fwd_fee` of the message.
* `GETGASFEESIMPLE` (`gas_used is_mc - price`) - same as `GETGASFEE`, but without flat price (just `(gas_used * price) / 2^16`).
* `GETFORWARDFEESIMPLE` (`cells bits is_mc - price`) - same as `GETFORWARDFEE`, but without lump price (just `(bits*bit_price + cells*cell_price) / 2^16`).
`gas_used`, `cells`, `bits`, `time_delta` are integers in range `0..2^63-1`.
#### Cell operations
Operations for working with Merkle proofs, where cells can have non-zero level and multiple hashes.
* `CLEVEL` (`cell - level`) - returns level of the cell.
* `CLEVELMASK` (`cell - level_mask`) - returns level mask of the cell.
* `i CHASHI` (`cell - hash`) - returns `i`th hash of the cell.
* `i CDEPTHI` (`cell - depth`) - returns `i`th depth of the cell.
* `CHASHIX` (`cell i - hash`) - returns `i`th hash of the cell.
* `CDEPTHIX` (`cell i - depth`) - returns `i`th depth of the cell.
`i` is in range `0..3`.
### Other changes
* `GLOBALID` gets `ConfigParam 19` from the tuple, not from the config dict. This decreases gas usage.
* `SENDMSG` gets `ConfigParam 24/25` (message prices) from the tuple, not from the config dict, and also uses `ConfigParam 43` to get max_msg_cells.

3
emulator/CMakeLists.txt
View file

@ -48,7 +48,7 @@ if (USE_EMSCRIPTEN)
add_executable(emulator-emscripten ${EMULATOR_EMSCRIPTEN_SOURCE})
target_link_libraries(emulator-emscripten PUBLIC emulator)
target_link_options(emulator-emscripten PRIVATE -sEXPORTED_RUNTIME_METHODS=_malloc,free,UTF8ToString,stringToUTF8,allocate,ALLOC_NORMAL,lengthBytesUTF8)
target_link_options(emulator-emscripten PRIVATE -sEXPORTED_FUNCTIONS=_emulate,_free,_run_get_method)
target_link_options(emulator-emscripten PRIVATE -sEXPORTED_FUNCTIONS=_emulate,_free,_run_get_method,_create_emulator,_destroy_emulator,_emulate_with_emulator)
target_link_options(emulator-emscripten PRIVATE -sEXPORT_NAME=EmulatorModule)
target_link_options(emulator-emscripten PRIVATE -sERROR_ON_UNDEFINED_SYMBOLS=0)
target_link_options(emulator-emscripten PRIVATE -Oz)
@ -57,6 +57,7 @@ if (USE_EMSCRIPTEN)
target_link_options(emulator-emscripten PRIVATE -sMODULARIZE=1)
target_link_options(emulator-emscripten PRIVATE -sENVIRONMENT=web)
target_link_options(emulator-emscripten PRIVATE -sFILESYSTEM=0)
target_link_options(emulator-emscripten PRIVATE -sALLOW_MEMORY_GROWTH=1)
target_link_options(emulator-emscripten PRIVATE -fexceptions)
if (USE_EMSCRIPTEN_NO_WASM)
target_link_options(emulator-emscripten PRIVATE -sWASM=0)

43
emulator/emulator-emscripten.cpp
View file

@ -124,9 +124,39 @@ td::Result<GetMethodParams> decode_get_method_params(const char* json) {
return params;
}
class NoopLog : public td::LogInterface {
public:
NoopLog() {
}
void append(td::CSlice new_slice, int log_level) override {
}
void rotate() override {
}
};
extern "C" {
const char *emulate(const char *config, const char* libs, int verbosity, const char* account, const char* message, const char* params) {
void* create_emulator(const char *config, int verbosity) {
NoopLog logger;
td::log_interface = &logger;
SET_VERBOSITY_LEVEL(verbosity_NEVER);
return transaction_emulator_create(config, verbosity);
}
void destroy_emulator(void* em) {
NoopLog logger;
td::log_interface = &logger;
SET_VERBOSITY_LEVEL(verbosity_NEVER);
transaction_emulator_destroy(em);
}
const char *emulate_with_emulator(void* em, const char* libs, const char* account, const char* message, const char* params) {
StringLog logger;
td::log_interface = &logger;
@ -138,8 +168,6 @@ const char *emulate(const char *config, const char* libs, int verbosity, const c
}
auto decoded_params = decoded_params_res.move_as_ok();
auto em = transaction_emulator_create(config, verbosity);
bool rand_seed_set = true;
if (decoded_params.rand_seed_hex) {
rand_seed_set = transaction_emulator_set_rand_seed(em, decoded_params.rand_seed_hex.unwrap().c_str());
@ -162,8 +190,6 @@ const char *emulate(const char *config, const char* libs, int verbosity, const c
result = transaction_emulator_emulate_transaction(em, account, message);
}
transaction_emulator_destroy(em);
const char* output = nullptr;
{
td::JsonBuilder jb;
@ -178,6 +204,13 @@ const char *emulate(const char *config, const char* libs, int verbosity, const c
return output;
}
const char *emulate(const char *config, const char* libs, int verbosity, const char* account, const char* message, const char* params) {
auto em = transaction_emulator_create(config, verbosity);
auto result = emulate_with_emulator(em, libs, account, message, params);
transaction_emulator_destroy(em);
return result;
}
const char *run_get_method(const char *params, const char* stack, const char* config) {
StringLog logger;

40
lite-client/lite-client.cpp
View file

@ -1275,7 +1275,7 @@ bool TestNode::after_parse_run_method(ton::WorkchainId workchain, ton::StdSmcAdd
}
}
});
return start_run_method(workchain, addr, ref_blkid, method_name, std::move(params), ext_mode ? 0x1f : 0,
return start_run_method(workchain, addr, ref_blkid, method_name, std::move(params), ext_mode ? 0x17 : 0,
std::move(P));
}
@ -1449,7 +1449,7 @@ bool TestNode::send_past_vset_query(ton::StdSmcAddress elector_addr) {
}
register_past_vset_info(std::move(S.back()));
});
return start_run_method(ton::masterchainId, elector_addr, mc_last_id_, "past_elections_list", std::move(params), 0x1f,
return start_run_method(ton::masterchainId, elector_addr, mc_last_id_, "past_elections_list", std::move(params), 0x17,
std::move(P));
}
@ -1510,7 +1510,7 @@ void TestNode::send_get_complaints_query(unsigned elect_id, ton::StdSmcAddress e
LOG(ERROR) << "vm virtualization error: " << err.get_msg();
}
});
start_run_method(ton::masterchainId, elector_addr, mc_last_id_, "get_past_complaints", std::move(params), 0x1f,
start_run_method(ton::masterchainId, elector_addr, mc_last_id_, "get_past_complaints", std::move(params), 0x17,
std::move(P));
}
@ -1607,7 +1607,7 @@ void TestNode::send_compute_complaint_price_query(ton::StdSmcAddress elector_add
LOG(ERROR) << "vm virtualization error: " << err.get_msg();
}
});
start_run_method(ton::masterchainId, elector_addr, mc_last_id_, "complaint_storage_price", std::move(params), 0x1f,
start_run_method(ton::masterchainId, elector_addr, mc_last_id_, "complaint_storage_price", std::move(params), 0x17,
std::move(P));
}
@ -1708,7 +1708,7 @@ bool TestNode::dns_resolve_send(ton::WorkchainId workchain, ton::StdSmcAddress a
}
return dns_resolve_finish(workchain, addr, blkid, domain, qdomain, cat, mode, (int)x->to_long(), std::move(cell));
});
return start_run_method(workchain, addr, blkid, "dnsresolve", std::move(params), 0x1f, std::move(P));
return start_run_method(workchain, addr, blkid, "dnsresolve", std::move(params), 0x17, std::move(P));
}
bool TestNode::show_dns_record(std::ostream& os, td::Bits256 cat, Ref<vm::CellSlice> value, bool raw_dump) {
@ -2142,21 +2142,29 @@ void TestNode::run_smc_method(int mode, ton::BlockIdExt ref_blk, ton::BlockIdExt
}
}
if (exit_code != 0) {
LOG(ERROR) << "VM terminated with error code " << exit_code;
out << "result: error " << exit_code << std::endl;
promise.set_error(td::Status::Error(PSLICE() << "VM terminated with non-zero exit code " << exit_code));
return;
}
} else {
stack = vm.get_stack_ref();
{
std::ostringstream os;
os << "result: ";
stack->dump(os, 3);
out << os.str();
}
if (mode & 4) {
if (remote_result.empty()) {
out << "remote result: <none>, exit code " << remote_exit_code;
if (!(mode & 4)) {
if (exit_code != 0) {
LOG(ERROR) << "VM terminated with error code " << exit_code;
promise.set_error(td::Status::Error(PSLICE() << "VM terminated with non-zero exit code " << exit_code));
} else {
promise.set_result(stack->extract_contents());
}
} else {
if (remote_exit_code != 0) {
out << "remote result: error " << remote_exit_code << std::endl;
LOG(ERROR) << "VM terminated with error code " << exit_code;
promise.set_error(td::Status::Error(PSLICE() << "VM terminated with non-zero exit code " << exit_code));
} else if (remote_result.empty()) {
out << "remote result: <none>" << std::endl;
promise.set_value({});
} else {
auto res = vm::std_boc_deserialize(std::move(remote_result));
if (res.is_error()) {
@ -2177,10 +2185,10 @@ void TestNode::run_smc_method(int mode, ton::BlockIdExt ref_blk, ton::BlockIdExt
os << "remote result (not to be trusted): ";
remote_stack->dump(os, 3);
out << os.str();
promise.set_value(remote_stack->extract_contents());
}
}
out.flush();
promise.set_result(stack->extract_contents());
} catch (vm::VmVirtError& err) {
out << "virtualization error while parsing runSmcMethod result: " << err.get_msg();
promise.set_error(
@ -3529,7 +3537,7 @@ void TestNode::continue_check_validator_load2(std::unique_ptr<TestNode::Validato
std::unique_ptr<TestNode::ValidatorLoadInfo> info2, int mode,
std::string file_pfx) {
LOG(INFO) << "continue_check_validator_load2 for blocks " << info1->blk_id.to_str() << " and "
<< info1->blk_id.to_str() << " : requesting block creators data";
<< info2->blk_id.to_str() << " : requesting block creators data";
td::Status st = info1->unpack_vset();
if (st.is_error()) {
LOG(ERROR) << "cannot unpack validator set from block " << info1->blk_id.to_str() << " :" << st.move_as_error();
@ -3627,7 +3635,7 @@ void TestNode::continue_check_validator_load3(std::unique_ptr<TestNode::Validato
auto x1 = info2->created[i].first - info1->created[i].first;
auto y1 = info2->created[i].second - info1->created[i].second;
if (x1 < 0 || y1 < 0 || (x1 | y1) >= (1u << 31)) {
LOG(ERROR) << "impossible situation: validator #i created a negative amount of blocks: " << x1
LOG(ERROR) << "impossible situation: validator #" << i << " created a negative amount of blocks: " << x1
<< " masterchain blocks, " << y1 << " shardchain blocks";
return;
}

30
recent_changelog.md
View file

@ -1,17 +1,17 @@
## 2024.01 Update
## 2024.02 Update
1. Improvement of validator synchronisation:
* Better handling of block broadcasts -> faster sync
* Additional separate overlay among validators as second option for synchronisation
2. Improvements in LS:
* c7 and library context is fully filled up for server-side rungetmethod
* Cache for runmethods and successfull external messages
* Logging of LS requests statistic
3. Precise control of open files:
* almost instantaneous validator start
* `--max-archive-fd` option
* autoremoval of not used temp archive files
* `--archive-preload-period` option
4. Preparatory (not enabled yet) code for addition on new TVM instructions for cheaper fee calculation onchain.
1. Fixes in how gas in transactions on special accounts is accounted in block limit. Previously, gas was counted as usual, so to conduct elections that costs >30m gas block limit in masterchain was set to 37m gas. To lower the limit for safety reasons it is proposed to caunt gas on special accounts separately. Besides `gas_max` is set to `special_gas_limit` for all types of transactions on special accounts. New behavior is activated through setting `version >= 5` in `ConfigParam 8;`.
* Besides update of config temporally increases gas limit on `EQD_v9j1rlsuHHw2FIhcsCFFSD367ldfDdCKcsNmNpIRzUlu` to `special_gas_limit`, see [details](https://t.me/tonstatus/88).
2. Improvements in LS behavior
* Improved detection of the state with all shards applied to decrease rate of `Block is not applied` error
* Better error logs: `block not in db` and `block is not applied` separation
* Fix error in proof generation for blocks after merge
* Fix most of `block is not applied` issues related to sending too recent block in Proofs
* LS now check external messages till `accept_message` (`set_gas`).
3. Improvements in DHT work and storage, CellDb, config.json ammendment, peer misbehavior detection, validator session stats collection, emulator.
4. Change in CTOS and XLOAD behavior activated through setting `version >= 5` in `ConfigParam 8;`:
* Loading "nested libraries" (i.e. a library cell that points to another library cell) throws an exception.
* Loading a library consumes gas for cell load only once (for the library cell), not twice (both for the library cell and the cell in the library).
* `XLOAD` now works differently. When it takes a library cell, it returns the cell that it points to. This allows loading "nested libraries", if needed.
Besides the work of the Core team, this update is based on the efforts of @XaBbl4 (peer misbehavior detection) and @akifoq (CTOS behavior and gas limit scheme for special accounts).

1
test/regression-tests.ans
View file

@ -18,6 +18,7 @@ Test_Fift_test_fiftext_default 2b0db5d4d4bfbc705b959cc787540d7b3a21a71469eac5475
Test_Fift_test_fixed_default 278a19d56b773102caf5c1fe2997ea6c8d0d9e720eff8503feede6398a197eec
Test_Fift_test_hash_ext_default 686fc5680feca5b3bb207768215b27f6872a95128762dee0d7f2c88bc492d62d
Test_Fift_test_hmap_default c269246882039824bb5822e896c3e6e82ef8e1251b6b251f5af8ea9fb8d05067
Test_Fift_test_levels_default 9fba4a7c98aec9000f42846d6e5fd820343ba61d68f9139dd16c88ccda757cf3
Test_Fift_test_namespaces_default e6419619c51332fb5e8bf22043ef415db686c47fe24f03061e5ad831014e7c6c
Test_Fift_test_rist255_default f4d7558f200a656934f986145c19b1dedbe2ad029292a5a975576d6891e25fc4
Test_Fift_test_sort2_default 9b57d47e6a10e7d1bbb565db35400debf2f963031f434742a702ec76555a5d3a

34
tl-utils/lite-utils.cpp
View file

@ -22,6 +22,7 @@
#include "td/utils/tl_storers.h"
#include "td/utils/crypto.h"
#include "crypto/common/bitstring.h"
#include <map>
namespace ton {
@ -129,4 +130,37 @@ td::Bits256 get_tl_object_sha_bits256(const lite_api::Object *T) {
return id256;
}
std::string lite_query_name_by_id(int id) {
static std::map<int, std::string> names = {
{lite_api::liteServer_getMasterchainInfo::ID, "getMasterchainInfo"},
{lite_api::liteServer_getMasterchainInfoExt::ID, "getMasterchainInfoExt"},
{lite_api::liteServer_getTime::ID, "getTime"},
{lite_api::liteServer_getVersion::ID, "getVersion"},
{lite_api::liteServer_getBlock::ID, "getBlock"},
{lite_api::liteServer_getState::ID, "getState"},
{lite_api::liteServer_getBlockHeader::ID, "getBlockHeader"},
{lite_api::liteServer_sendMessage::ID, "sendMessage"},
{lite_api::liteServer_getAccountState::ID, "getAccountState"},
{lite_api::liteServer_getAccountStatePrunned::ID, "getAccountStatePrunned"},
{lite_api::liteServer_runSmcMethod::ID, "runSmcMethod"},
{lite_api::liteServer_getShardInfo::ID, "getShardInfo"},
{lite_api::liteServer_getAllShardsInfo::ID, "getAllShardsInfo"},
{lite_api::liteServer_getOneTransaction::ID, "getOneTransaction"},
{lite_api::liteServer_getTransactions::ID, "getTransactions"},
{lite_api::liteServer_lookupBlock::ID, "lookupBlock"},
{lite_api::liteServer_listBlockTransactions::ID, "listBlockTransactions"},
{lite_api::liteServer_listBlockTransactionsExt::ID, "listBlockTransactionsExt"},
{lite_api::liteServer_getBlockProof::ID, "getBlockProof"},
{lite_api::liteServer_getConfigAll::ID, "getConfigAll"},
{lite_api::liteServer_getConfigParams::ID, "getConfigParams"},
{lite_api::liteServer_getValidatorStats::ID, "getValidatorStats"},
{lite_api::liteServer_getLibraries::ID, "getLibraries"},
{lite_api::liteServer_getShardBlockProof::ID, "getShardBlockProof"}};
auto it = names.find(id);
if (it == names.end()) {
return "unknown";
}
return it->second;
}
} // namespace ton

2
tl-utils/lite-utils.hpp
View file

@ -46,4 +46,6 @@ template <class Tp, std::enable_if_t<std::is_base_of<lite_api::Object, Tp>::valu
td::Bits256 get_tl_object_sha_bits256(const Tp &T) {
return get_tl_object_sha_bits256(static_cast<const lite_api::Object *>(&T));
}
std::string lite_query_name_by_id(int id);
} // namespace ton

2
tl/generate/scheme/ton_api.tl
View file

@ -394,6 +394,8 @@ tonNode.newShardBlockBroadcast block:tonNode.newShardBlock = tonNode.Broadcast;
tonNode.shardPublicOverlayId workchain:int shard:long zero_state_file_hash:int256 = tonNode.ShardPublicOverlayId;
tonNode.privateBlockOverlayId zero_state_file_hash:int256 nodes:(vector int256) = tonNode.PrivateBlockOverlayId;
tonNode.keyBlocks blocks:(vector tonNode.blockIdExt) incomplete:Bool error:Bool = tonNode.KeyBlocks;
ton.blockId root_cell_hash:int256 file_hash:int256 = ton.BlockId;

BIN
tl/generate/scheme/ton_api.tlo
View file

Binary file not shown.

8
ton/ton-types.h
View file

@ -353,6 +353,14 @@ struct BlockBroadcast {
td::uint32 validator_set_hash;
td::BufferSlice data;
td::BufferSlice proof;
BlockBroadcast clone() const {
std::vector<BlockSignature> new_signatures;
for (const BlockSignature& s : signatures) {
new_signatures.emplace_back(s.node, s.signature.clone());
}
return {block_id, std::move(new_signatures), catchain_seqno, validator_set_hash, data.clone(), proof.clone()};
}
};
struct Ed25519_PrivateKey {

4
tonlib/test/offline.cpp
View file

@ -333,8 +333,8 @@ TEST(Tonlib, ConfigParseBug) {
unsigned char buff[128];
int bits = (int)td::bitstring::parse_bitstring_hex_literal(buff, sizeof(buff), literal.begin(), literal.end());
CHECK(bits >= 0);
auto slice = vm::CellBuilder().store_bits(td::ConstBitPtr{buff}, bits).finalize();
block::Config::do_get_gas_limits_prices(std::move(slice), 21).ensure();
auto cell = vm::CellBuilder().store_bits(td::ConstBitPtr{buff}, bits).finalize();
block::Config::do_get_gas_limits_prices(vm::load_cell_slice(cell), 21).ensure();
}
TEST(Tonlib, EncryptionApi) {

2
tonlib/tonlib/TonlibClient.cpp
View file

@ -1216,7 +1216,7 @@ class RemoteRunSmcMethod : public td::actor::Actor {
client_.send_query(
//liteServer.runSmcMethod mode:# id:tonNode.blockIdExt account:liteServer.accountId method_id:long params:bytes = liteServer.RunMethodResult;
ton::lite_api::liteServer_runSmcMethod(
0x1f, ton::create_tl_lite_block_id(query_.block_id.value()),
0x17, ton::create_tl_lite_block_id(query_.block_id.value()),
ton::create_tl_object<ton::lite_api::liteServer_accountId>(query_.address.workchain, query_.address.addr),
method_id, std::move(serialized_stack)),
[self = this](auto r_state) { self->with_run_method_result(std::move(r_state)); },

98
utils/opcode-timing.cpp
View file

@ -11,6 +11,56 @@
#include "td/utils/ScopeGuard.h"
#include "td/utils/StringBuilder.h"
#include "td/utils/Timer.h"
#include "block.h"
#include "td/utils/filesystem.h"
#include "mc-config.h"
td::Ref<vm::Tuple> c7;
void prepare_c7() {
auto now = (td::uint32)td::Clocks::system();
td::Ref<vm::Cell> config_root;
auto config_data = td::read_file("config.boc");
if (config_data.is_ok()) {
LOG(WARNING) << "Reading config from config.boc";
auto r_cell = vm::std_boc_deserialize(config_data.move_as_ok());
r_cell.ensure();
config_root = r_cell.move_as_ok();
}
vm::CellBuilder addr;
addr.store_long(4, 3);
addr.store_long(0, 8);
addr.store_ones(256);
std::vector<vm::StackEntry> tuple = {
td::make_refint(0x076ef1ea), // [ magic:0x076ef1ea
td::zero_refint(), // actions:Integer
td::zero_refint(), // msgs_sent:Integer
td::make_refint(now), // unixtime:Integer
td::make_refint(0), // block_lt:Integer
td::make_refint(0), // trans_lt:Integer
td::make_refint(123), // rand_seed:Integer
block::CurrencyCollection(td::make_refint(10000LL * 1000000000))
.as_vm_tuple(), // balance_remaining:[Integer (Maybe Cell)]
addr.as_cellslice_ref(), // myself:MsgAddressInt
vm::StackEntry::maybe(config_root) // global_config:(Maybe Cell) ] = SmartContractInfo;
};
tuple.push_back({}); // code:Cell
tuple.push_back(block::CurrencyCollection(td::make_refint(2000LL * 1000000000))
.as_vm_tuple()); // in_msg_value:[Integer (Maybe Cell)]
tuple.push_back(td::make_refint(0)); // storage_fees:Integer
tuple.push_back(vm::StackEntry()); // prev_blocks_info
if (config_root.not_null()) {
block::Config config{config_root};
config.unpack().ensure();
tuple.push_back(config.get_unpacked_config_tuple(now)); // unpacked_config_tuple
} else {
tuple.push_back(vm::StackEntry());
}
tuple.push_back(td::zero_refint());
auto tuple_ref = td::make_cnt_ref<std::vector<vm::StackEntry>>(std::move(tuple));
c7 = vm::make_tuple_ref(std::move(tuple_ref));
}
td::Ref<vm::Cell> to_cell(td::Slice s) {
if (s.size() >= 4 && s.substr(0, 4) == "boc:") {
@ -34,9 +84,11 @@ struct runInfo {
long long gasUsage;
int vmReturnCode;
runInfo() : runtime(0.0), gasUsage(0), vmReturnCode(0) {}
runInfo(long double runtime, long long gasUsage, int vmReturnCode) :
runtime(runtime), gasUsage(gasUsage), vmReturnCode(vmReturnCode) {}
runInfo() : runtime(0.0), gasUsage(0), vmReturnCode(0) {
}
runInfo(long double runtime, long long gasUsage, int vmReturnCode)
: runtime(runtime), gasUsage(gasUsage), vmReturnCode(vmReturnCode) {
}
runInfo operator+(const runInfo& addend) const {
return {runtime + addend.runtime, gasUsage + addend.gasUsage, vmReturnCode ? vmReturnCode : addend.vmReturnCode};
@ -45,7 +97,7 @@ struct runInfo {
runInfo& operator+=(const runInfo& addend) {
runtime += addend.runtime;
gasUsage += addend.gasUsage;
if(!vmReturnCode && addend.vmReturnCode) {
if (!vmReturnCode && addend.vmReturnCode) {
vmReturnCode = addend.vmReturnCode;
}
return *this;
@ -68,8 +120,8 @@ vm::Stack prepare_stack(td::Slice command) {
vm::Stack stack;
try {
vm::GasLimits gas_limit;
int ret = vm::run_vm_code(vm::load_cell_slice_ref(cell), stack, 0 /*flags*/, nullptr /*data*/,
vm::VmLog{}, nullptr, &gas_limit, {}, {}, nullptr, 4);
int ret = vm::run_vm_code(vm::load_cell_slice_ref(cell), stack, 0 /*flags*/, nullptr /*data*/, vm::VmLog{}, nullptr,
&gas_limit, {}, c7, nullptr, ton::SUPPORTED_VERSION);
CHECK(ret == 0);
} catch (...) {
LOG(FATAL) << "catch unhandled exception";
@ -83,8 +135,8 @@ runInfo time_run_vm(td::Slice command, td::Ref<vm::Stack> stack) {
CHECK(stack.is_unique());
try {
vm::GasLimits gas_limit;
vm::VmState vm{vm::load_cell_slice_ref(cell), std::move(stack), gas_limit, 0, {}, vm::VmLog{}, {}, {}};
vm.set_global_version(4);
vm::VmState vm{vm::load_cell_slice_ref(cell), std::move(stack), gas_limit, 0, {}, vm::VmLog{}, {}, c7};
vm.set_global_version(ton::SUPPORTED_VERSION);
std::clock_t cStart = std::clock();
int ret = ~vm.run();
std::clock_t cEnd = std::clock();
@ -102,7 +154,7 @@ runtimeStats averageRuntime(td::Slice command, const vm::Stack& stack) {
std::vector<runInfo> values;
values.reserve(samples);
td::Timer t0;
for(size_t i = 0; i < samples; ++i) {
for (size_t i = 0; i < samples; ++i) {
const auto value_empty = time_run_vm(td::Slice(""), td::Ref<vm::Stack>(true, stack));
const auto value_code = time_run_vm(command, td::Ref<vm::Stack>(true, stack));
runInfo value{value_code.runtime - value_empty.runtime, value_code.gasUsage - value_empty.gasUsage,
@ -120,18 +172,16 @@ runtimeStats averageRuntime(td::Slice command, const vm::Stack& stack) {
long double runtimeDiffSum = 0.0;
long double gasDiffSum = 0.0;
bool errored = false;
for(const auto value : values) {
for (const auto value : values) {
const auto runtime = value.runtime - runtimeMean;
const auto gasUsage = static_cast<long double>(value.gasUsage) - gasMean;
runtimeDiffSum += runtime * runtime;
gasDiffSum += gasUsage * gasUsage;
errored = errored || value.errored();
}
return {
{runtimeMean, sqrtl(runtimeDiffSum / static_cast<long double>(samples))},
return {{runtimeMean, sqrtl(runtimeDiffSum / static_cast<long double>(samples))},
{gasMean, sqrtl(gasDiffSum / static_cast<long double>(samples))},
errored
};
errored};
}
runtimeStats timeInstruction(const std::string& setupCode, const std::string& toMeasure) {
@ -141,28 +191,33 @@ runtimeStats timeInstruction(const std::string& setupCode, const std::string& to
int main(int argc, char** argv) {
SET_VERBOSITY_LEVEL(verbosity_ERROR);
if(argc != 2 && argc != 3) {
std::cerr <<
"This utility compares the timing of VM execution against the gas used.\n"
if (argc != 2 && argc != 3) {
std::cerr << "This utility compares the timing of VM execution against the gas used.\n"
"It can be used to discover opcodes or opcode sequences that consume an "
"inordinate amount of computational resources relative to their gas cost.\n"
"\n"
"The utility expects two command line arguments: \n"
"The TVM code used to set up the stack and VM state followed by the TVM code to measure.\n"
"For example, to test the DIVMODC opcode:\n"
"\t$ " << argv[0] << " 80FF801C A90E 2>/dev/null\n"
"\t$ "
<< argv[0]
<< " 80FF801C A90E 2>/dev/null\n"
"\tOPCODE,runtime mean,runtime stddev,gas mean,gas stddev\n"
"\tA90E,0.0066416,0.00233496,26,0\n"
"\n"
"Usage: " << argv[0] << " [TVM_SETUP_BYTECODE] TVM_BYTECODE\n"
"Usage: "
<< argv[0]
<< " [TVM_SETUP_BYTECODE] TVM_BYTECODE\n"
"\tBYTECODE is either:\n"
"\t1. hex-encoded string (e.g. A90E for DIVMODC)\n"
"\t2. boc:<serialized boc in base64> (e.g. boc:te6ccgEBAgEABwABAogBAAJ7)" << std::endl << std::endl;
"\t2. boc:<serialized boc in base64> (e.g. boc:te6ccgEBAgEABwABAogBAAJ7)"
<< std::endl
<< std::endl;
return 1;
}
std::cout << "OPCODE,runtime mean,runtime stddev,gas mean,gas stddev,error" << std::endl;
std::string setup, code;
if(argc == 2) {
if (argc == 2) {
setup = "";
code = argv[1];
} else {
@ -170,6 +225,7 @@ int main(int argc, char** argv) {
code = argv[2];
}
vm::init_vm().ensure();
prepare_c7();
const auto time = timeInstruction(setup, code);
std::cout << std::fixed << std::setprecision(9) << code << "," << time.runtime.mean << "," << time.runtime.stddev
<< "," << time.gasUsage.mean << "," << time.gasUsage.stddev << "," << (int)time.errored << std::endl;

19
validator-engine/validator-engine.cpp
View file

@ -1364,6 +1364,8 @@ td::Status ValidatorEngine::load_global_config() {
validator_options_.write().set_session_logs_file(session_logs_file_);
}
validator_options_.write().set_celldb_compress_depth(celldb_compress_depth_);
validator_options_.write().set_max_open_archive_files(max_open_archive_files_);
validator_options_.write().set_archive_preload_period(archive_preload_period_);
std::vector<ton::BlockIdExt> h;
for (auto &x : conf.validator_->hardforks_) {
@ -3793,6 +3795,23 @@ int main(int argc, char *argv[]) {
});
return td::Status::OK();
});
p.add_checked_option(
'\0', "max-archive-fd", "limit for a number of open file descriptirs in archive manager. 0 is unlimited (default)",
[&](td::Slice s) -> td::Status {
TRY_RESULT(v, td::to_integer_safe<size_t>(s));
acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_max_open_archive_files, v); });
return td::Status::OK();
});
p.add_checked_option(
'\0', "archive-preload-period", "open archive slices for the past X second on startup (default: 0)",
[&](td::Slice s) -> td::Status {
auto v = td::to_double(s);
if (v < 0) {
return td::Status::Error("sync-before should be non-negative");
}
acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_archive_preload_period, v); });
return td::Status::OK();
});
auto S = p.run(argc, argv);
if (S.is_error()) {
LOG(ERROR) << "failed to parse options: " << S.move_as_error();

8
validator-engine/validator-engine.hpp
View file

@ -204,6 +204,8 @@ class ValidatorEngine : public td::actor::Actor {
double archive_ttl_ = 0;
double key_proof_ttl_ = 0;
td::uint32 celldb_compress_depth_ = 0;
size_t max_open_archive_files_ = 0;
double archive_preload_period_ = 0.0;
bool read_config_ = false;
bool started_keyring_ = false;
bool started_ = false;
@ -264,6 +266,12 @@ class ValidatorEngine : public td::actor::Actor {
void set_celldb_compress_depth(td::uint32 value) {
celldb_compress_depth_ = value;
}
void set_max_open_archive_files(size_t value) {
max_open_archive_files_ = value;
}
void set_archive_preload_period(double value) {
archive_preload_period_ = value;
}
void start_up() override;
ValidatorEngine() {
}

3
validator/CMakeLists.txt
View file

@ -43,6 +43,7 @@ set(VALIDATOR_HEADERS
fabric.h
interfaces/db.h
interfaces/external-message.h
interfaces/liteserver.h
interfaces/proof.h
interfaces/shard.h
interfaces/signature-set.h
@ -143,6 +144,8 @@ set(FULL_NODE_SOURCE
full-node-master.h
full-node-master.hpp
full-node-master.cpp
full-node-private-overlay.hpp
full-node-private-overlay.cpp
net/download-block.hpp
net/download-block.cpp

57
validator/db/archive-manager.cpp
View file

@ -55,7 +55,9 @@ std::string PackageId::name() const {
}
}
ArchiveManager::ArchiveManager(td::actor::ActorId<RootDb> root, std::string db_root) : db_root_(db_root) {
ArchiveManager::ArchiveManager(td::actor::ActorId<RootDb> root, std::string db_root,
td::Ref<ValidatorManagerOptions> opts)
: db_root_(db_root), opts_(opts) {
}
void ArchiveManager::add_handle(BlockHandle handle, td::Promise<td::Unit> promise) {
@ -598,9 +600,11 @@ void ArchiveManager::load_package(PackageId id) {
}
}
desc.file = td::actor::create_actor<ArchiveSlice>("slice", id.id, id.key, id.temp, false, db_root_);
desc.file =
td::actor::create_actor<ArchiveSlice>("slice", id.id, id.key, id.temp, false, db_root_, archive_lru_.get());
m.emplace(id, std::move(desc));
update_permanent_slices();
}
const ArchiveManager::FileDescription *ArchiveManager::get_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno,
@ -631,7 +635,8 @@ const ArchiveManager::FileDescription *ArchiveManager::add_file_desc(ShardIdFull
FileDescription new_desc{id, false};
td::mkdir(db_root_ + id.path()).ensure();
std::string prefix = PSTRING() << db_root_ << id.path() << id.name();
new_desc.file = td::actor::create_actor<ArchiveSlice>("slice", id.id, id.key, id.temp, false, db_root_);
new_desc.file =
td::actor::create_actor<ArchiveSlice>("slice", id.id, id.key, id.temp, false, db_root_, archive_lru_.get());
const FileDescription &desc = f.emplace(id, std::move(new_desc));
if (!id.temp) {
update_desc(f, desc, shard, seqno, ts, lt);
@ -673,6 +678,7 @@ const ArchiveManager::FileDescription *ArchiveManager::add_file_desc(ShardIdFull
.ensure();
}
index_->commit_transaction().ensure();
update_permanent_slices();
return &desc;
}
@ -820,6 +826,9 @@ void ArchiveManager::start_up() {
td::mkdir(db_root_ + "/archive/states/").ensure();
td::mkdir(db_root_ + "/files/").ensure();
td::mkdir(db_root_ + "/files/packages/").ensure();
if (opts_->get_max_open_archive_files() > 0) {
archive_lru_ = td::actor::create_actor<ArchiveLru>("archive_lru", opts_->get_max_open_archive_files());
}
index_ = std::make_shared<td::RocksDb>(td::RocksDb::open(db_root_ + "/files/globalindex").move_as_ok());
std::string value;
auto v = index_->get(create_serialize_tl_object<ton_api::db_files_index_key>().as_slice(), value);
@ -876,10 +885,28 @@ void ArchiveManager::start_up() {
}).ensure();
persistent_state_gc(FileHash::zero());
double open_since = td::Clocks::system() - opts_->get_archive_preload_period();
for (auto it = files_.rbegin(); it != files_.rend(); ++it) {
if (it->second.file_actor_id().empty()) {
continue;
}
td::actor::send_closure(it->second.file_actor_id(), &ArchiveSlice::open_files);
bool stop = true;
for (const auto &first_block : it->second.first_blocks) {
if ((double)first_block.second.ts >= open_since) {
stop = false;
break;
}
}
if (stop) {
break;
}
}
}
void ArchiveManager::run_gc(UnixTime ts, UnixTime archive_ttl) {
auto p = get_temp_package_id_by_unixtime(ts);
void ArchiveManager::run_gc(UnixTime mc_ts, UnixTime gc_ts, UnixTime archive_ttl) {
auto p = get_temp_package_id_by_unixtime(std::max(gc_ts, mc_ts - TEMP_PACKAGES_TTL));
std::vector<PackageId> vec;
for (auto &x : temp_files_) {
if (x.first < p) {
@ -907,7 +934,7 @@ void ArchiveManager::run_gc(UnixTime ts, UnixTime archive_ttl) {
if (it == desc.first_blocks.end()) {
continue;
}
if (it->second.ts < ts - archive_ttl) {
if (it->second.ts < gc_ts - archive_ttl) {
vec.push_back(f.first);
}
}
@ -1200,6 +1227,7 @@ void ArchiveManager::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle han
}
}
}
update_permanent_slices();
}
void ArchiveManager::FileMap::shard_index_add(const FileDescription &desc) {
@ -1298,6 +1326,23 @@ const ArchiveManager::FileDescription *ArchiveManager::FileMap::get_next_file_de
return it2->second->deleted ? nullptr : it2->second;
}
void ArchiveManager::update_permanent_slices() {
if (archive_lru_.empty()) {
return;
}
std::vector<PackageId> ids;
if (!files_.empty()) {
ids.push_back(files_.rbegin()->first);
}
if (!key_files_.empty()) {
ids.push_back(key_files_.rbegin()->first);
}
if (!temp_files_.empty()) {
ids.push_back(temp_files_.rbegin()->first);
}
td::actor::send_closure(archive_lru_, &ArchiveLru::set_permanent_slices, std::move(ids));
}
} // namespace validator
} // namespace ton

13
validator/db/archive-manager.hpp
View file

@ -28,7 +28,7 @@ class RootDb;
class ArchiveManager : public td::actor::Actor {
public:
ArchiveManager(td::actor::ActorId<RootDb> root, std::string db_root);
ArchiveManager(td::actor::ActorId<RootDb> root, std::string db_root, td::Ref<ValidatorManagerOptions> opts);
void add_handle(BlockHandle handle, td::Promise<td::Unit> promise);
void update_handle(BlockHandle handle, td::Promise<td::Unit> promise);
@ -58,7 +58,7 @@ class ArchiveManager : public td::actor::Actor {
void truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handle, td::Promise<td::Unit> promise);
//void truncate_continue(BlockSeqno masterchain_seqno, td::Promise<td::Unit> promise);
void run_gc(UnixTime ts, UnixTime archive_ttl);
void run_gc(UnixTime mc_ts, UnixTime gc_ts, UnixTime archive_ttl);
/* from LTDB */
void get_block_by_unix_time(AccountIdPrefixFull account_id, UnixTime ts, td::Promise<ConstBlockHandle> promise);
@ -123,6 +123,9 @@ class ArchiveManager : public td::actor::Actor {
size_t size() const {
return files_.size();
}
bool empty() const {
return files_.empty();
}
std::map<PackageId, FileDescription>::const_iterator lower_bound(const PackageId &x) const {
return files_.lower_bound(x);
}
@ -164,6 +167,7 @@ class ArchiveManager : public td::actor::Actor {
void shard_index_del(const FileDescription &desc);
};
FileMap files_, key_files_, temp_files_;
td::actor::ActorOwn<ArchiveLru> archive_lru_;
BlockSeqno finalized_up_to_{0};
bool async_mode_ = false;
bool huge_transaction_started_ = false;
@ -206,6 +210,7 @@ class ArchiveManager : public td::actor::Actor {
void got_gc_masterchain_handle(ConstBlockHandle handle, FileHash hash);
std::string db_root_;
td::Ref<ValidatorManagerOptions> opts_;
std::shared_ptr<td::KeyValue> index_;
@ -215,6 +220,10 @@ class ArchiveManager : public td::actor::Actor {
PackageId get_temp_package_id() const;
PackageId get_key_package_id(BlockSeqno seqno) const;
PackageId get_temp_package_id_by_unixtime(UnixTime ts) const;
void update_permanent_slices();
static const td::uint32 TEMP_PACKAGES_TTL = 86400 * 7;
};
} // namespace validator

172
validator/db/archive-slice.cpp
View file

@ -21,7 +21,6 @@
#include "td/actor/MultiPromise.h"
#include "validator/fabric.h"
#include "td/db/RocksDb.h"
#include "ton/ton-io.hpp"
#include "td/utils/port/path.h"
#include "common/delay.h"
#include "files-async.hpp"
@ -32,9 +31,16 @@ namespace validator {
void PackageWriter::append(std::string filename, td::BufferSlice data,
td::Promise<std::pair<td::uint64, td::uint64>> promise) {
auto offset = package_->append(std::move(filename), std::move(data), !async_mode_);
auto size = package_->size();
td::uint64 offset, size;
{
auto p = package_.lock();
if (!p) {
promise.set_error(td::Status::Error("Package is closed"));
return;
}
offset = p->append(std::move(filename), std::move(data), !async_mode_);
size = p->size();
}
promise.set_value(std::pair<td::uint64, td::uint64>{offset, size});
}
@ -44,8 +50,10 @@ class PackageReader : public td::actor::Actor {
td::Promise<std::pair<std::string, td::BufferSlice>> promise)
: package_(std::move(package)), offset_(offset), promise_(std::move(promise)) {
}
void start_up() {
promise_.set_result(package_->read(offset_));
void start_up() override {
auto result = package_->read(offset_);
package_ = {};
promise_.set_result(std::move(result));
stop();
}
@ -64,6 +72,7 @@ void ArchiveSlice::add_handle(BlockHandle handle, td::Promise<td::Unit> promise)
update_handle(std::move(handle), std::move(promise));
return;
}
before_query();
CHECK(!key_blocks_only_);
CHECK(!temp_);
CHECK(handle->inited_unix_time());
@ -146,6 +155,7 @@ void ArchiveSlice::update_handle(BlockHandle handle, td::Promise<td::Unit> promi
promise.set_value(td::Unit());
return;
}
before_query();
CHECK(!key_blocks_only_);
begin_transaction();
@ -168,6 +178,7 @@ void ArchiveSlice::add_file(BlockHandle handle, FileReference ref_id, td::Buffer
promise.set_error(td::Status::Error(ErrorCode::notready, "package already gc'd"));
return;
}
before_query();
TRY_RESULT_PROMISE(
promise, p,
choose_package(
@ -179,6 +190,7 @@ void ArchiveSlice::add_file(BlockHandle handle, FileReference ref_id, td::Buffer
promise.set_value(td::Unit());
return;
}
promise = begin_async_query(std::move(promise));
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), idx = p->idx, ref_id, promise = std::move(promise)](
td::Result<std::pair<td::uint64, td::uint64>> R) mutable {
if (R.is_error()) {
@ -217,6 +229,7 @@ void ArchiveSlice::get_handle(BlockIdExt block_id, td::Promise<BlockHandle> prom
promise.set_error(td::Status::Error(ErrorCode::notready, "package already gc'd"));
return;
}
before_query();
CHECK(!key_blocks_only_);
std::string value;
auto R = kv_->get(get_db_key_block_info(block_id), value);
@ -239,6 +252,7 @@ void ArchiveSlice::get_temp_handle(BlockIdExt block_id, td::Promise<ConstBlockHa
promise.set_error(td::Status::Error(ErrorCode::notready, "package already gc'd"));
return;
}
before_query();
CHECK(!key_blocks_only_);
std::string value;
auto R = kv_->get(get_db_key_block_info(block_id), value);
@ -261,6 +275,7 @@ void ArchiveSlice::get_file(ConstBlockHandle handle, FileReference ref_id, td::P
promise.set_error(td::Status::Error(ErrorCode::notready, "package already gc'd"));
return;
}
before_query();
std::string value;
auto R = kv_->get(ref_id.hash().to_hex(), value);
R.ensure();
@ -273,6 +288,7 @@ void ArchiveSlice::get_file(ConstBlockHandle handle, FileReference ref_id, td::P
promise, p,
choose_package(
handle ? handle->id().is_masterchain() ? handle->id().seqno() : handle->masterchain_ref_block() : 0, false));
promise = begin_async_query(std::move(promise));
auto P = td::PromiseCreator::lambda(
[promise = std::move(promise)](td::Result<std::pair<std::string, td::BufferSlice>> R) mutable {
if (R.is_error()) {
@ -292,6 +308,7 @@ void ArchiveSlice::get_block_common(AccountIdPrefixFull account_id,
promise.set_error(td::Status::Error(ErrorCode::notready, "package already gc'd"));
return;
}
before_query();
bool f = false;
BlockIdExt block_id;
td::uint32 ls = 0;
@ -312,7 +329,7 @@ void ArchiveSlice::get_block_common(AccountIdPrefixFull account_id,
auto G = fetch_tl_object<ton_api::db_lt_desc_value>(value, true);
G.ensure();
auto g = G.move_as_ok();
if (compare_desc(*g.get()) > 0) {
if (compare_desc(*g) > 0) {
continue;
}
td::uint32 l = g->first_idx_ - 1;
@ -328,7 +345,7 @@ void ArchiveSlice::get_block_common(AccountIdPrefixFull account_id,
auto E = fetch_tl_object<ton_api::db_lt_el_value>(td::BufferSlice{value}, true);
E.ensure();
auto e = E.move_as_ok();
int cmp_val = compare(*e.get());
int cmp_val = compare(*e);
if (cmp_val < 0) {
rseq = create_block_id(e->id_);
@ -342,9 +359,7 @@ void ArchiveSlice::get_block_common(AccountIdPrefixFull account_id,
}
}
if (rseq.is_valid()) {
if (!block_id.is_valid()) {
block_id = rseq;
} else if (block_id.id.seqno > rseq.id.seqno) {
if (!block_id.is_valid() || block_id.id.seqno > rseq.id.seqno) {
block_id = rseq;
}
}
@ -430,12 +445,15 @@ void ArchiveSlice::get_slice(td::uint64 archive_id, td::uint64 offset, td::uint3
promise.set_error(td::Status::Error(ErrorCode::error, "bad archive id"));
return;
}
before_query();
auto value = static_cast<td::uint32>(archive_id >> 32);
TRY_RESULT_PROMISE(promise, p, choose_package(value, false));
promise = begin_async_query(std::move(promise));
td::actor::create_actor<db::ReadFile>("readfile", p->path, offset, limit, 0, std::move(promise)).release();
}
void ArchiveSlice::get_archive_id(BlockSeqno masterchain_seqno, td::Promise<td::uint64> promise) {
before_query();
if (!sliced_mode_) {
promise.set_result(archive_id_);
} else {
@ -444,14 +462,15 @@ void ArchiveSlice::get_archive_id(BlockSeqno masterchain_seqno, td::Promise<td::
}
}
void ArchiveSlice::start_up() {
PackageId p_id{archive_id_, key_blocks_only_, temp_};
std::string db_path = PSTRING() << db_root_ << p_id.path() << p_id.name() << ".index";
kv_ = std::make_shared<td::RocksDb>(td::RocksDb::open(db_path).move_as_ok());
void ArchiveSlice::before_query() {
if (status_ == st_closed) {
LOG(DEBUG) << "Opening archive slice " << db_path_;
kv_ = std::make_unique<td::RocksDb>(td::RocksDb::open(db_path_).move_as_ok());
std::string value;
auto R2 = kv_->get("status", value);
R2.ensure();
sliced_mode_ = false;
slice_size_ = 100;
if (R2.move_as_ok() == td::KeyValue::GetStatus::Ok) {
if (value == "sliced") {
@ -498,6 +517,54 @@ void ArchiveSlice::start_up() {
add_package(archive_id_, 0, 0);
}
}
}
status_ = st_open;
if (!archive_lru_.empty()) {
td::actor::send_closure(archive_lru_, &ArchiveLru::on_query, actor_id(this), p_id_,
packages_.size() + ESTIMATED_DB_OPEN_FILES);
}
}
void ArchiveSlice::open_files() {
before_query();
}
void ArchiveSlice::close_files() {
if (status_ == st_open) {
if (active_queries_ == 0) {
do_close();
} else {
status_ = st_want_close;
}
}
}
void ArchiveSlice::do_close() {
if (destroyed_) {
return;
}
CHECK(status_ != st_closed && active_queries_ == 0);
LOG(DEBUG) << "Closing archive slice " << db_path_;
status_ = st_closed;
kv_ = {};
packages_.clear();
}
template<typename T>
td::Promise<T> ArchiveSlice::begin_async_query(td::Promise<T> promise) {
++active_queries_;
return [SelfId = actor_id(this), promise = std::move(promise)](td::Result<T> R) mutable {
td::actor::send_closure(SelfId, &ArchiveSlice::end_async_query);
promise.set_result(std::move(R));
};
}
void ArchiveSlice::end_async_query() {
CHECK(active_queries_ > 0);
--active_queries_;
if (active_queries_ == 0 && status_ == st_want_close) {
do_close();
}
}
void ArchiveSlice::begin_transaction() {
@ -521,7 +588,7 @@ void ArchiveSlice::commit_transaction() {
void ArchiveSlice::set_async_mode(bool mode, td::Promise<td::Unit> promise) {
async_mode_ = mode;
if (!async_mode_ && huge_transaction_started_) {
if (!async_mode_ && huge_transaction_started_ && kv_) {
kv_->commit_transaction().ensure();
huge_transaction_size_ = 0;
huge_transaction_started_ = false;
@ -532,16 +599,20 @@ void ArchiveSlice::set_async_mode(bool mode, td::Promise<td::Unit> promise) {
ig.add_promise(std::move(promise));
for (auto &p : packages_) {
td::actor::send_closure(p.writer, &PackageWriter::set_async_mode, mode, std::move(promise));
td::actor::send_closure(p.writer, &PackageWriter::set_async_mode, mode, ig.get_promise());
}
}
ArchiveSlice::ArchiveSlice(td::uint32 archive_id, bool key_blocks_only, bool temp, bool finalized, std::string db_root)
ArchiveSlice::ArchiveSlice(td::uint32 archive_id, bool key_blocks_only, bool temp, bool finalized, std::string db_root,
td::actor::ActorId<ArchiveLru> archive_lru)
: archive_id_(archive_id)
, key_blocks_only_(key_blocks_only)
, temp_(temp)
, finalized_(finalized)
, db_root_(std::move(db_root)) {
, p_id_(archive_id_, key_blocks_only_, temp_)
, db_root_(std::move(db_root))
, archive_lru_(std::move(archive_lru)) {
db_path_ = PSTRING() << db_root_ << p_id_.path() << p_id_.name() << ".index";
}
td::Result<ArchiveSlice::PackageInfo *> ArchiveSlice::choose_package(BlockSeqno masterchain_seqno, bool force) {
@ -587,7 +658,7 @@ void ArchiveSlice::add_package(td::uint32 seqno, td::uint64 size, td::uint32 ver
if (version >= 1) {
pack->truncate(size).ensure();
}
auto writer = td::actor::create_actor<PackageWriter>("writer", pack);
auto writer = td::actor::create_actor<PackageWriter>("writer", pack, async_mode_);
packages_.emplace_back(std::move(pack), std::move(writer), seqno, path, idx, version);
}
@ -611,6 +682,7 @@ void destroy_db(std::string name, td::uint32 attempt, td::Promise<td::Unit> prom
} // namespace
void ArchiveSlice::destroy(td::Promise<td::Unit> promise) {
before_query();
td::MultiPromise mp;
auto ig = mp.init_guard();
ig.add_promise(std::move(promise));
@ -763,6 +835,7 @@ void ArchiveSlice::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handl
destroy(std::move(promise));
return;
}
before_query();
LOG(INFO) << "TRUNCATE: slice " << archive_id_ << " maxseqno= " << max_masterchain_seqno()
<< " truncate_upto=" << masterchain_seqno;
if (max_masterchain_seqno() <= masterchain_seqno) {
@ -819,7 +892,7 @@ void ArchiveSlice::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handl
pack->writer.reset();
td::unlink(pack->path).ensure();
td::rename(pack->path + ".new", pack->path).ensure();
pack->writer = td::actor::create_actor<PackageWriter>("writer", new_package);
pack->writer = td::actor::create_actor<PackageWriter>("writer", new_package, async_mode_);
for (auto idx = pack->idx + 1; idx < packages_.size(); idx++) {
td::unlink(packages_[idx].path).ensure();
@ -831,6 +904,61 @@ void ArchiveSlice::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handl
promise.set_value(td::Unit());
}
static std::tuple<td::uint32, bool, bool> to_tuple(const PackageId &id) {
return {id.id, id.temp, id.key};
}
void ArchiveLru::on_query(td::actor::ActorId<ArchiveSlice> slice, PackageId id, size_t files_count) {
SliceInfo &info = slices_[to_tuple(id)];
if (info.opened_idx != 0) {
total_files_ -= info.files_count;
lru_.erase(info.opened_idx);
}
info.actor = std::move(slice);
total_files_ += (info.files_count = files_count);
info.opened_idx = current_idx_++;
if (!info.is_permanent) {
lru_.emplace(info.opened_idx, id);
}
enforce_limit();
}
void ArchiveLru::set_permanent_slices(std::vector<PackageId> ids) {
for (auto id : permanent_slices_) {
SliceInfo &info = slices_[to_tuple(id)];
if (!info.is_permanent) {
continue;
}
info.is_permanent = false;
if (info.opened_idx) {
lru_.emplace(info.opened_idx, id);
}
}
permanent_slices_ = std::move(ids);
for (auto id : permanent_slices_) {
SliceInfo &info = slices_[to_tuple(id)];
if (info.is_permanent) {
continue;
}
info.is_permanent = true;
if (info.opened_idx) {
lru_.erase(info.opened_idx);
}
}
enforce_limit();
}
void ArchiveLru::enforce_limit() {
while (total_files_ > max_total_files_ && lru_.size() > 1) {
auto it = lru_.begin();
auto it2 = slices_.find(to_tuple(it->second));
lru_.erase(it);
total_files_ -= it2->second.files_count;
td::actor::send_closure(it2->second.actor, &ArchiveSlice::close_files);
it2->second.opened_idx = 0;
}
}
} // namespace validator
} // namespace ton

71
validator/db/archive-slice.hpp
View file

@ -21,6 +21,7 @@
#include "validator/interfaces/db.h"
#include "package.hpp"
#include "fileref.hpp"
#include <map>
namespace ton {
@ -44,7 +45,7 @@ struct PackageId {
std::string path() const;
std::string name() const;
bool is_empty() {
bool is_empty() const {
return id == std::numeric_limits<td::uint32>::max();
}
static PackageId empty(bool key, bool temp) {
@ -54,26 +55,33 @@ struct PackageId {
class PackageWriter : public td::actor::Actor {
public:
PackageWriter(std::shared_ptr<Package> package) : package_(std::move(package)) {
PackageWriter(std::weak_ptr<Package> package, bool async_mode = false)
: package_(std::move(package)), async_mode_(async_mode) {
}
void append(std::string filename, td::BufferSlice data, td::Promise<std::pair<td::uint64, td::uint64>> promise);
void set_async_mode(bool mode, td::Promise<td::Unit> promise) {
async_mode_ = mode;
if (!async_mode_) {
package_->sync();
auto p = package_.lock();
if (p) {
p->sync();
}
}
promise.set_value(td::Unit());
}
private:
std::shared_ptr<Package> package_;
std::weak_ptr<Package> package_;
bool async_mode_ = false;
};
class ArchiveLru;
class ArchiveSlice : public td::actor::Actor {
public:
ArchiveSlice(td::uint32 archive_id, bool key_blocks_only, bool temp, bool finalized, std::string db_root);
ArchiveSlice(td::uint32 archive_id, bool key_blocks_only, bool temp, bool finalized, std::string db_root,
td::actor::ActorId<ArchiveLru> archive_lru);
void get_archive_id(BlockSeqno masterchain_seqno, td::Promise<td::uint64> promise);
@ -95,16 +103,24 @@ class ArchiveSlice : public td::actor::Actor {
void get_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, td::Promise<td::BufferSlice> promise);
void start_up() override;
void destroy(td::Promise<td::Unit> promise);
void truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handle, td::Promise<td::Unit> promise);
void begin_transaction();
void commit_transaction();
void set_async_mode(bool mode, td::Promise<td::Unit> promise);
void open_files();
void close_files();
private:
void written_data(BlockHandle handle, td::Promise<td::Unit> promise);
void before_query();
void do_close();
template<typename T>
td::Promise<T> begin_async_query(td::Promise<T> promise);
void end_async_query();
void begin_transaction();
void commit_transaction();
void add_file_cont(size_t idx, FileReference ref_id, td::uint64 offset, td::uint64 size,
td::Promise<td::Unit> promise);
@ -112,13 +128,14 @@ class ArchiveSlice : public td::actor::Actor {
td::BufferSlice get_db_key_lt_desc(ShardIdFull shard);
td::BufferSlice get_db_key_lt_el(ShardIdFull shard, td::uint32 idx);
td::BufferSlice get_db_key_block_info(BlockIdExt block_id);
td::BufferSlice get_lt_from_db(ShardIdFull shard, td::uint32 idx);
td::uint32 archive_id_;
bool key_blocks_only_;
bool temp_;
bool finalized_;
PackageId p_id_;
std::string db_path_;
bool destroyed_ = false;
bool async_mode_ = false;
@ -127,8 +144,14 @@ class ArchiveSlice : public td::actor::Actor {
td::uint32 huge_transaction_size_ = 0;
td::uint32 slice_size_{100};
enum Status {
st_closed, st_open, st_want_close
} status_ = st_closed;
size_t active_queries_ = 0;
std::string db_root_;
std::shared_ptr<td::KeyValue> kv_;
td::actor::ActorId<ArchiveLru> archive_lru_;
std::unique_ptr<td::KeyValue> kv_;
struct PackageInfo {
PackageInfo(std::shared_ptr<Package> package, td::actor::ActorOwn<PackageWriter> writer, BlockSeqno id,
@ -164,6 +187,32 @@ class ArchiveSlice : public td::actor::Actor {
static constexpr td::uint32 default_package_version() {
return 1;
}
static const size_t ESTIMATED_DB_OPEN_FILES = 5;
};
class ArchiveLru : public td::actor::Actor {
public:
explicit ArchiveLru(size_t max_total_files) : max_total_files_(max_total_files) {
CHECK(max_total_files_ > 0);
}
void on_query(td::actor::ActorId<ArchiveSlice> slice, PackageId id, size_t files_count);
void set_permanent_slices(std::vector<PackageId> ids);
private:
size_t current_idx_ = 1;
struct SliceInfo {
td::actor::ActorId<ArchiveSlice> actor;
size_t files_count = 0;
size_t opened_idx = 0; // 0 - not opened
bool is_permanent = false;
};
std::map<std::tuple<td::uint32, bool, bool>, SliceInfo> slices_;
std::map<size_t, PackageId> lru_;
size_t total_files_ = 0;
size_t max_total_files_ = 0;
std::vector<PackageId> permanent_slices_;
void enforce_limit();
};
} // namespace validator

6
validator/db/rootdb.cpp
View file

@ -400,7 +400,7 @@ void RootDb::start_up() {
cell_db_ = td::actor::create_actor<CellDb>("celldb", actor_id(this), root_path_ + "/celldb/", opts_);
state_db_ = td::actor::create_actor<StateDb>("statedb", actor_id(this), root_path_ + "/state/");
static_files_db_ = td::actor::create_actor<StaticFilesDb>("staticfilesdb", actor_id(this), root_path_ + "/static/");
archive_db_ = td::actor::create_actor<ArchiveManager>("archive", actor_id(this), root_path_);
archive_db_ = td::actor::create_actor<ArchiveManager>("archive", actor_id(this), root_path_, opts_);
}
void RootDb::archive(BlockHandle handle, td::Promise<td::Unit> promise) {
@ -497,8 +497,8 @@ void RootDb::set_async_mode(bool mode, td::Promise<td::Unit> promise) {
td::actor::send_closure(archive_db_, &ArchiveManager::set_async_mode, mode, std::move(promise));
}
void RootDb::run_gc(UnixTime ts, UnixTime archive_ttl) {
td::actor::send_closure(archive_db_, &ArchiveManager::run_gc, ts, archive_ttl);
void RootDb::run_gc(UnixTime mc_ts, UnixTime gc_ts, UnixTime archive_ttl) {
td::actor::send_closure(archive_db_, &ArchiveManager::run_gc, mc_ts, gc_ts, archive_ttl);
}
} // namespace validator

3
validator/db/rootdb.hpp
View file

@ -134,11 +134,10 @@ class RootDb : public Db {
td::Promise<td::BufferSlice> promise) override;
void set_async_mode(bool mode, td::Promise<td::Unit> promise) override;
void run_gc(UnixTime ts, UnixTime archive_ttl) override;
void run_gc(UnixTime mc_ts, UnixTime gc_ts, UnixTime archive_ttl) override;
private:
td::actor::ActorId<ValidatorManager> validator_manager_;
std::string root_path_;
td::Ref<ValidatorManagerOptions> opts_;

22
validator/downloaders/wait-block-state.cpp
View file

@ -109,12 +109,14 @@ void WaitBlockState::start() {
} else if (!handle_->inited_prev() || (!handle_->inited_proof() && !handle_->inited_proof_link())) {
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle = handle_](td::Result<td::BufferSlice> R) {
if (R.is_error()) {
delay_action([SelfId]() { td::actor::send_closure(SelfId, &WaitBlockState::start); }, td::Timestamp::in(0.1));
delay_action([SelfId]() { td::actor::send_closure(SelfId, &WaitBlockState::after_get_proof_link); },
td::Timestamp::in(0.1));
} else {
td::actor::send_closure(SelfId, &WaitBlockState::got_proof_link, R.move_as_ok());
}
});
waiting_proof_link_ = true;
td::actor::send_closure(manager_, &ValidatorManager::send_get_block_proof_link_request, handle_->id(), priority_,
std::move(P));
} else if (prev_state_.is_null()) {
@ -133,12 +135,14 @@ void WaitBlockState::start() {
} else if (handle_->id().is_masterchain() && !handle_->inited_proof()) {
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle = handle_](td::Result<td::BufferSlice> R) {
if (R.is_error()) {
delay_action([SelfId]() { td::actor::send_closure(SelfId, &WaitBlockState::start); }, td::Timestamp::in(0.1));
delay_action([SelfId]() { td::actor::send_closure(SelfId, &WaitBlockState::after_get_proof); },
td::Timestamp::in(0.1));
} else {
td::actor::send_closure(SelfId, &WaitBlockState::got_proof, R.move_as_ok());
}
});
waiting_proof_ = true;
td::actor::send_closure(manager_, &ValidatorManager::send_get_block_proof_request, handle_->id(), priority_,
std::move(P));
} else if (block_.is_null()) {
@ -172,6 +176,9 @@ void WaitBlockState::got_prev_state(td::Ref<ShardState> state) {
}
void WaitBlockState::got_proof_link(td::BufferSlice data) {
if (!waiting_proof_link_) {
return;
}
auto R = create_proof_link(handle_->id(), std::move(data));
if (R.is_error()) {
LOG(INFO) << "received bad proof link: " << R.move_as_error();
@ -182,22 +189,25 @@ void WaitBlockState::got_proof_link(td::BufferSlice data) {
if (R.is_ok()) {
auto h = R.move_as_ok();
CHECK(h->inited_prev());
td::actor::send_closure(SelfId, &WaitBlockState::start);
td::actor::send_closure(SelfId, &WaitBlockState::after_get_proof_link);
} else {
LOG(INFO) << "received bad proof link: " << R.move_as_error();
td::actor::send_closure(SelfId, &WaitBlockState::start);
td::actor::send_closure(SelfId, &WaitBlockState::after_get_proof_link);
}
});
run_check_proof_link_query(handle_->id(), R.move_as_ok(), manager_, timeout_, std::move(P));
}
void WaitBlockState::got_proof(td::BufferSlice data) {
if (!waiting_proof_) {
return;
}
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Unit> R) {
if (R.is_ok()) {
td::actor::send_closure(SelfId, &WaitBlockState::start);
td::actor::send_closure(SelfId, &WaitBlockState::after_get_proof);
} else {
LOG(INFO) << "received bad proof link: " << R.move_as_error();
td::actor::send_closure(SelfId, &WaitBlockState::start);
td::actor::send_closure(SelfId, &WaitBlockState::after_get_proof);
}
});
td::actor::send_closure(manager_, &ValidatorManager::validate_block_proof, handle_->id(), std::move(data),

20
validator/downloaders/wait-block-state.hpp
View file

@ -45,11 +45,9 @@ class WaitBlockState : public td::actor::Actor {
void force_read_from_db();
void start_up() override;
void got_block_handle(BlockHandle handle);
void start();
void got_state_from_db(td::Ref<ShardState> data);
void got_state_from_static_file(td::Ref<ShardState> state, td::BufferSlice data);
void failed_to_get_state_from_db(td::Status reason);
void got_prev_state(td::Ref<ShardState> state);
void failed_to_get_prev_state(td::Status reason);
void got_block_data(td::Ref<BlockData> data);
@ -68,6 +66,22 @@ class WaitBlockState : public td::actor::Actor {
priority_ = priority;
}
// These two methods can be called from ValidatorManagerImpl::written_handle
void after_get_proof_link() {
if (!waiting_proof_link_) {
return;
}
waiting_proof_link_ = false;
start();
}
void after_get_proof() {
if (!waiting_proof_) {
return;
}
waiting_proof_ = false;
start();
}
private:
BlockHandle handle_;
@ -81,6 +95,8 @@ class WaitBlockState : public td::actor::Actor {
td::Ref<BlockData> block_;
bool reading_from_db_ = false;
bool waiting_proof_link_ = false;
bool waiting_proof_ = false;
td::Timestamp next_static_file_attempt_;
td::PerfWarningTimer perf_timer_;

175
validator/full-node-private-overlay.cpp Normal file
View file

@ -0,0 +1,175 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "full-node-private-overlay.hpp"
#include "ton/ton-tl.hpp"
#include "common/delay.h"
namespace ton {
namespace validator {
namespace fullnode {
void FullNodePrivateOverlay::process_broadcast(PublicKeyHash, ton_api::tonNode_blockBroadcast &query) {
std::vector<BlockSignature> signatures;
for (auto &sig : query.signatures_) {
signatures.emplace_back(BlockSignature{sig->who_, std::move(sig->signature_)});
}
BlockIdExt block_id = create_block_id(query.id_);
BlockBroadcast B{block_id,
std::move(signatures),
static_cast<UnixTime>(query.catchain_seqno_),
static_cast<td::uint32>(query.validator_set_hash_),
std::move(query.data_),
std::move(query.proof_)};
auto P = td::PromiseCreator::lambda([](td::Result<td::Unit> R) {
if (R.is_error()) {
if (R.error().code() == ErrorCode::notready) {
LOG(DEBUG) << "dropped broadcast: " << R.move_as_error();
} else {
LOG(INFO) << "dropped broadcast: " << R.move_as_error();
}
}
});
td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::prevalidate_block, std::move(B),
std::move(P));
}
void FullNodePrivateOverlay::process_broadcast(PublicKeyHash, ton_api::tonNode_newShardBlockBroadcast &query) {
td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_shard_block,
create_block_id(query.block_->block_), query.block_->cc_seqno_,
std::move(query.block_->data_));
}
void FullNodePrivateOverlay::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) {
auto B = fetch_tl_object<ton_api::tonNode_Broadcast>(std::move(broadcast), true);
if (B.is_error()) {
return;
}
ton_api::downcast_call(*B.move_as_ok(), [src, Self = this](auto &obj) { Self->process_broadcast(src, obj); });
}
void FullNodePrivateOverlay::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) {
if (!inited_) {
return;
}
auto B = create_serialize_tl_object<ton_api::tonNode_newShardBlockBroadcast>(
create_tl_object<ton_api::tonNode_newShardBlock>(create_tl_block_id(block_id), cc_seqno, std::move(data)));
if (B.size() <= overlay::Overlays::max_simple_broadcast_size()) {
td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_ex, local_id_, overlay_id_,
local_id_.pubkey_hash(), 0, std::move(B));
} else {
td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_,
local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), std::move(B));
}
}
void FullNodePrivateOverlay::send_broadcast(BlockBroadcast broadcast) {
if (!inited_) {
return;
}
std::vector<tl_object_ptr<ton_api::tonNode_blockSignature>> sigs;
for (auto &sig : broadcast.signatures) {
sigs.emplace_back(create_tl_object<ton_api::tonNode_blockSignature>(sig.node, sig.signature.clone()));
}
auto B = create_serialize_tl_object<ton_api::tonNode_blockBroadcast>(
create_tl_block_id(broadcast.block_id), broadcast.catchain_seqno, broadcast.validator_set_hash, std::move(sigs),
broadcast.proof.clone(), broadcast.data.clone());
td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_,
local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), std::move(B));
}
void FullNodePrivateOverlay::start_up() {
std::sort(nodes_.begin(), nodes_.end());
nodes_.erase(std::unique(nodes_.begin(), nodes_.end()), nodes_.end());
std::vector<td::Bits256> nodes;
for (const adnl::AdnlNodeIdShort &id : nodes_) {
nodes.push_back(id.bits256_value());
}
auto X = create_hash_tl_object<ton_api::tonNode_privateBlockOverlayId>(zero_state_file_hash_, std::move(nodes));
td::BufferSlice b{32};
b.as_slice().copy_from(as_slice(X));
overlay_id_full_ = overlay::OverlayIdFull{std::move(b)};
overlay_id_ = overlay_id_full_.compute_short_id();
try_init();
}
void FullNodePrivateOverlay::try_init() {
// Sometimes adnl id is added to validator engine later (or not at all)
td::actor::send_closure(
adnl_, &adnl::Adnl::check_id_exists, local_id_, [SelfId = actor_id(this)](td::Result<bool> R) {
if (R.is_ok() && R.ok()) {
td::actor::send_closure(SelfId, &FullNodePrivateOverlay::init);
} else {
delay_action([SelfId]() { td::actor::send_closure(SelfId, &FullNodePrivateOverlay::try_init); },
td::Timestamp::in(30.0));
}
});
}
void FullNodePrivateOverlay::init() {
LOG(FULL_NODE_INFO) << "Creating private block overlay for adnl id " << local_id_ << " : " << nodes_.size()
<< " nodes";
class Callback : public overlay::Overlays::Callback {
public:
void receive_message(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override {
}
void receive_query(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data,
td::Promise<td::BufferSlice> promise) override {
}
void receive_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override {
td::actor::send_closure(node_, &FullNodePrivateOverlay::receive_broadcast, src, std::move(data));
}
void check_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data,
td::Promise<td::Unit> promise) override {
}
Callback(td::actor::ActorId<FullNodePrivateOverlay> node) : node_(node) {
}
private:
td::actor::ActorId<FullNodePrivateOverlay> node_;
};
overlay::OverlayPrivacyRules rules{overlay::Overlays::max_fec_broadcast_size(),
overlay::CertificateFlags::AllowFec | overlay::CertificateFlags::Trusted,
{}};
td::actor::send_closure(overlays_, &overlay::Overlays::create_private_overlay, local_id_, overlay_id_full_.clone(),
nodes_, std::make_unique<Callback>(actor_id(this)), rules);
td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_id_);
td::actor::send_closure(rldp2_, &rldp2::Rldp::add_id, local_id_);
inited_ = true;
}
void FullNodePrivateOverlay::tear_down() {
if (inited_) {
td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, local_id_, overlay_id_);
}
}
} // namespace fullnode
} // namespace validator
} // namespace ton

86
validator/full-node-private-overlay.hpp Normal file
View file

@ -0,0 +1,86 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "full-node.h"
namespace ton {
namespace validator {
namespace fullnode {
class FullNodePrivateOverlay : public td::actor::Actor {
public:
void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query);
void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query);
template <class T>
void process_broadcast(PublicKeyHash, T &) {
VLOG(FULL_NODE_WARNING) << "dropping unknown broadcast";
}
void receive_broadcast(PublicKeyHash src, td::BufferSlice query);
void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data);
void send_broadcast(BlockBroadcast broadcast);
void start_up() override;
void tear_down() override;
FullNodePrivateOverlay(adnl::AdnlNodeIdShort local_id, std::vector<adnl::AdnlNodeIdShort> nodes,
FileHash zero_state_file_hash, FullNodeConfig config,
td::actor::ActorId<keyring::Keyring> keyring, td::actor::ActorId<adnl::Adnl> adnl,
td::actor::ActorId<rldp::Rldp> rldp, td::actor::ActorId<rldp2::Rldp> rldp2,
td::actor::ActorId<overlay::Overlays> overlays,
td::actor::ActorId<ValidatorManagerInterface> validator_manager)
: local_id_(local_id)
, nodes_(std::move(nodes))
, zero_state_file_hash_(zero_state_file_hash)
, config_(config)
, keyring_(keyring)
, adnl_(adnl)
, rldp_(rldp)
, rldp2_(rldp2)
, overlays_(overlays)
, validator_manager_(validator_manager) {
}
private:
adnl::AdnlNodeIdShort local_id_;
std::vector<adnl::AdnlNodeIdShort> nodes_;
FileHash zero_state_file_hash_;
FullNodeConfig config_;
td::actor::ActorId<keyring::Keyring> keyring_;
td::actor::ActorId<adnl::Adnl> adnl_;
td::actor::ActorId<rldp::Rldp> rldp_;
td::actor::ActorId<rldp2::Rldp> rldp2_;
td::actor::ActorId<overlay::Overlays> overlays_;
td::actor::ActorId<ValidatorManagerInterface> validator_manager_;
bool inited_ = false;
overlay::OverlayIdFull overlay_id_full_;
overlay::OverlayIdShort overlay_id_;
void try_init();
void init();
};
} // namespace fullnode
} // namespace validator
} // namespace ton

15
validator/full-node-shard.cpp
View file

@ -109,6 +109,9 @@ void FullNodeShardImpl::check_broadcast(PublicKeyHash src, td::BufferSlice broad
}
auto q = B.move_as_ok();
if (!processed_ext_msg_broadcasts_.insert(td::sha256_bits256(q->message_->data_)).second) {
return promise.set_error(td::Status::Error("duplicate external message broadcast"));
}
if (config_.ext_messages_broadcast_disabled_) {
promise.set_error(td::Status::Error("rebroadcasting external messages is disabled"));
promise = [manager = validator_manager_, message = q->message_->data_.clone()](td::Result<td::Unit> R) mutable {
@ -703,6 +706,9 @@ void FullNodeShardImpl::send_external_message(td::BufferSlice data) {
});
return;
}
if (!processed_ext_msg_broadcasts_.insert(td::sha256_bits256(data)).second) {
return;
}
auto B = create_serialize_tl_object<ton_api::tonNode_externalMessageBroadcast>(
create_tl_object<ton_api::tonNode_externalMessage>(std::move(data)));
if (B.size() <= overlay::Overlays::max_simple_broadcast_size()) {
@ -852,10 +858,15 @@ void FullNodeShardImpl::alarm() {
update_certificate_at_ = td::Timestamp::never();
}
}
if (cleanup_processed_ext_msg_at_ && cleanup_processed_ext_msg_at_.is_in_past()) {
processed_ext_msg_broadcasts_.clear();
cleanup_processed_ext_msg_at_ = td::Timestamp::in(60.0);
}
alarm_timestamp().relax(sync_completed_at_);
alarm_timestamp().relax(update_certificate_at_);
alarm_timestamp().relax(reload_neighbours_at_);
alarm_timestamp().relax(ping_neighbours_at_);
alarm_timestamp().relax(cleanup_processed_ext_msg_at_);
}
void FullNodeShardImpl::start_up() {
@ -872,8 +883,8 @@ void FullNodeShardImpl::start_up() {
reload_neighbours_at_ = td::Timestamp::now();
ping_neighbours_at_ = td::Timestamp::now();
alarm_timestamp().relax(reload_neighbours_at_);
alarm_timestamp().relax(ping_neighbours_at_);
cleanup_processed_ext_msg_at_ = td::Timestamp::now();
alarm_timestamp().relax(td::Timestamp::now());
}
}

4
validator/full-node-shard.hpp
View file

@ -21,6 +21,7 @@
#include "full-node-shard.h"
#include "td/actor/PromiseFuture.h"
#include "td/utils/port/Poll.h"
#include <set>
namespace ton {
@ -250,6 +251,9 @@ class FullNodeShardImpl : public FullNodeShard {
adnl::AdnlNodeIdShort last_pinged_neighbour_ = adnl::AdnlNodeIdShort::zero();
FullNodeConfig config_;
std::set<td::Bits256> processed_ext_msg_broadcasts_;
td::Timestamp cleanup_processed_ext_msg_at_;
};
} // namespace fullnode

51
validator/full-node.cpp
View file

@ -50,6 +50,7 @@ void FullNodeImpl::add_permanent_key(PublicKeyHash key, td::Promise<td::Unit> pr
for (auto &shard : shards_) {
td::actor::send_closure(shard.second, &FullNodeShard::update_validators, all_validators_, sign_cert_by_);
}
create_private_block_overlay(key);
promise.set_value(td::Unit());
}
@ -74,6 +75,7 @@ void FullNodeImpl::del_permanent_key(PublicKeyHash key, td::Promise<td::Unit> pr
for (auto &shard : shards_) {
td::actor::send_closure(shard.second, &FullNodeShard::update_validators, all_validators_, sign_cert_by_);
}
private_block_overlays_.erase(key);
promise.set_value(td::Unit());
}
@ -179,6 +181,10 @@ void FullNodeImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_s
VLOG(FULL_NODE_WARNING) << "dropping OUT shard block info message to unknown shard";
return;
}
if (!private_block_overlays_.empty()) {
td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateOverlay::send_shard_block_info,
block_id, cc_seqno, data.clone());
}
td::actor::send_closure(shard, &FullNodeShard::send_shard_block_info, block_id, cc_seqno, std::move(data));
}
@ -188,6 +194,10 @@ void FullNodeImpl::send_broadcast(BlockBroadcast broadcast) {
VLOG(FULL_NODE_WARNING) << "dropping OUT broadcast to unknown shard";
return;
}
if (!private_block_overlays_.empty()) {
td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateOverlay::send_broadcast,
broadcast.clone());
}
td::actor::send_closure(shard, &FullNodeShard::send_broadcast, std::move(broadcast));
}
@ -289,6 +299,7 @@ void FullNodeImpl::got_key_block_proof(td::Ref<ProofLink> proof) {
PublicKeyHash l = PublicKeyHash::zero();
std::vector<PublicKeyHash> keys;
std::map<PublicKeyHash, adnl::AdnlNodeIdShort> current_validators;
for (td::int32 i = -1; i <= 1; i++) {
auto r = config->get_total_validator_set(i < 0 ? i : 1 - i);
if (r.not_null()) {
@ -299,9 +310,17 @@ void FullNodeImpl::got_key_block_proof(td::Ref<ProofLink> proof) {
if (local_keys_.count(key)) {
l = key;
}
if (i == 1) {
current_validators[key] = adnl::AdnlNodeIdShort{el.addr.is_zero() ? key.bits256_value() : el.addr};
}
}
}
}
if (current_validators != current_validators_) {
current_validators_ = std::move(current_validators);
update_private_block_overlays();
}
if (keys == all_validators_) {
return;
@ -321,6 +340,7 @@ void FullNodeImpl::got_zero_block_state(td::Ref<ShardState> state) {
PublicKeyHash l = PublicKeyHash::zero();
std::vector<PublicKeyHash> keys;
std::map<PublicKeyHash, adnl::AdnlNodeIdShort> current_validators;
for (td::int32 i = -1; i <= 1; i++) {
auto r = m->get_total_validator_set(i < 0 ? i : 1 - i);
if (r.not_null()) {
@ -331,9 +351,17 @@ void FullNodeImpl::got_zero_block_state(td::Ref<ShardState> state) {
if (local_keys_.count(key)) {
l = key;
}
if (i == 1) {
current_validators[key] = adnl::AdnlNodeIdShort{el.addr.is_zero() ? key.bits256_value() : el.addr};
}
}
}
}
if (current_validators != current_validators_) {
current_validators_ = std::move(current_validators);
update_private_block_overlays();
}
if (keys == all_validators_) {
return;
@ -456,6 +484,29 @@ void FullNodeImpl::start_up() {
std::make_unique<Callback>(actor_id(this)), std::move(P));
}
void FullNodeImpl::update_private_block_overlays() {
private_block_overlays_.clear();
if (local_keys_.empty()) {
return;
}
for (const auto &key : local_keys_) {
create_private_block_overlay(key);
}
}
void FullNodeImpl::create_private_block_overlay(PublicKeyHash key) {
CHECK(local_keys_.count(key));
if (current_validators_.count(key)) {
std::vector<adnl::AdnlNodeIdShort> nodes;
for (const auto &p : current_validators_) {
nodes.push_back(p.second);
}
private_block_overlays_[key] = td::actor::create_actor<FullNodePrivateOverlay>(
"BlocksPrivateOverlay", current_validators_[key], std::move(nodes), zero_state_file_hash_, config_, keyring_,
adnl_, rldp_, rldp2_, overlays_, validator_manager_);
}
}
FullNodeImpl::FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash,
FullNodeConfig config, td::actor::ActorId<keyring::Keyring> keyring,
td::actor::ActorId<adnl::Adnl> adnl, td::actor::ActorId<rldp::Rldp> rldp,

7
validator/full-node.hpp
View file

@ -23,6 +23,7 @@
//#include "ton-node-slave.h"
#include "interfaces/proof.h"
#include "interfaces/shard.h"
#include "full-node-private-overlay.hpp"
#include <map>
#include <set>
@ -111,9 +112,15 @@ class FullNodeImpl : public FullNode {
PublicKeyHash sign_cert_by_;
std::vector<PublicKeyHash> all_validators_;
std::map<PublicKeyHash, adnl::AdnlNodeIdShort> current_validators_;
std::set<PublicKeyHash> local_keys_;
FullNodeConfig config_;
std::map<PublicKeyHash, td::actor::ActorOwn<FullNodePrivateOverlay>> private_block_overlays_;
void update_private_block_overlays();
void create_private_block_overlay(PublicKeyHash key);
};
} // namespace fullnode

1
validator/impl/CMakeLists.txt
View file

@ -32,6 +32,7 @@ set(TON_VALIDATOR_SOURCE
external-message.hpp
ihr-message.hpp
liteserver.hpp
liteserver-cache.hpp
message-queue.hpp
proof.hpp
shard.hpp

5
validator/impl/fabric.cpp
View file

@ -34,6 +34,7 @@
#include "ton/ton-io.hpp"
#include "liteserver.hpp"
#include "validator/fabric.h"
#include "liteserver-cache.hpp"
namespace ton {
@ -46,7 +47,7 @@ td::actor::ActorOwn<Db> create_db_actor(td::actor::ActorId<ValidatorManager> man
td::actor::ActorOwn<LiteServerCache> create_liteserver_cache_actor(td::actor::ActorId<ValidatorManager> manager,
std::string db_root) {
return td::actor::create_actor<LiteServerCache>("cache");
return td::actor::create_actor<LiteServerCacheImpl>("cache");
}
td::Result<td::Ref<BlockData>> create_block(BlockIdExt block_id, td::BufferSlice data) {
@ -244,7 +245,7 @@ void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_b
void run_liteserver_query(td::BufferSlice data, td::actor::ActorId<ValidatorManager> manager,
td::actor::ActorId<LiteServerCache> cache, td::Promise<td::BufferSlice> promise) {
LiteQuery::run_query(std::move(data), std::move(manager), std::move(promise));
LiteQuery::run_query(std::move(data), std::move(manager), std::move(cache), std::move(promise));
}
void run_fetch_account_state(WorkchainId wc, StdSmcAddress addr, td::actor::ActorId<ValidatorManager> manager,

116
validator/impl/liteserver-cache.hpp Normal file
View file

@ -0,0 +1,116 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "interfaces/liteserver.h"
#include <map>
namespace ton::validator {
class LiteServerCacheImpl : public LiteServerCache {
public:
void start_up() override {
alarm();
}
void alarm() override {
alarm_timestamp() = td::Timestamp::in(60.0);
if (queries_cnt_ > 0 || !send_message_cache_.empty()) {
LOG(WARNING) << "LS Cache stats: " << queries_cnt_ << " queries, " << queries_hit_cnt_ << " hits; "
<< cache_.size() << " entries, size=" << total_size_ << "/" << MAX_CACHE_SIZE << "; "
<< send_message_cache_.size() << " different sendMessage queries, " << send_message_error_cnt_
<< " duplicates";
queries_cnt_ = 0;
queries_hit_cnt_ = 0;
send_message_cache_.clear();
send_message_error_cnt_ = 0;
}
}
void lookup(td::Bits256 key, td::Promise<td::BufferSlice> promise) override {
++queries_cnt_;
auto it = cache_.find(key);
if (it == cache_.end()) {
promise.set_error(td::Status::Error("not found"));
return;
}
++queries_hit_cnt_;
auto entry = it->second.get();
entry->remove();
lru_.put(entry);
promise.set_value(entry->value_.clone());
}
void update(td::Bits256 key, td::BufferSlice value) override {
std::unique_ptr<CacheEntry> &entry = cache_[key];
if (entry == nullptr) {
entry = std::make_unique<CacheEntry>(key, std::move(value));
} else {
total_size_ -= entry->size();
entry->value_ = std::move(value);
entry->remove();
}
lru_.put(entry.get());
total_size_ += entry->size();
while (total_size_ > MAX_CACHE_SIZE) {
auto to_remove = (CacheEntry *)lru_.get();
CHECK(to_remove);
total_size_ -= to_remove->size();
to_remove->remove();
cache_.erase(to_remove->key_);
}
}
void process_send_message(td::Bits256 key, td::Promise<td::Unit> promise) override {
if (send_message_cache_.insert(key).second) {
promise.set_result(td::Unit());
} else {
++send_message_error_cnt_;
promise.set_error(td::Status::Error("duplicate message"));
}
}
void drop_send_message_from_cache(td::Bits256 key) override {
send_message_cache_.erase(key);
}
private:
struct CacheEntry : public td::ListNode {
explicit CacheEntry(td::Bits256 key, td::BufferSlice value) : key_(key), value_(std::move(value)) {
}
td::Bits256 key_;
td::BufferSlice value_;
size_t size() const {
return value_.size() + 32 * 2;
}
};
std::map<td::Bits256, std::unique_ptr<CacheEntry>> cache_;
td::ListNode lru_;
size_t total_size_ = 0;
size_t queries_cnt_ = 0, queries_hit_cnt_ = 0;
std::set<td::Bits256> send_message_cache_;
size_t send_message_error_cnt_ = 0;
static const size_t MAX_CACHE_SIZE = 64 << 20;
};
} // namespace ton::validator

131
validator/impl/liteserver.cpp
View file

@ -54,8 +54,11 @@ td::int32 get_tl_tag(td::Slice slice) {
}
void LiteQuery::run_query(td::BufferSlice data, td::actor::ActorId<ValidatorManager> manager,
td::actor::ActorId<LiteServerCache> cache,
td::Promise<td::BufferSlice> promise) {
td::actor::create_actor<LiteQuery>("litequery", std::move(data), std::move(manager), std::move(promise)).release();
td::actor::create_actor<LiteQuery>("litequery", std::move(data), std::move(manager), std::move(cache),
std::move(promise))
.release();
}
void LiteQuery::fetch_account_state(WorkchainId wc, StdSmcAddress acc_addr, td::actor::ActorId<ton::validator::ValidatorManager> manager,
@ -64,8 +67,8 @@ void LiteQuery::fetch_account_state(WorkchainId wc, StdSmcAddress acc_addr, td:
}
LiteQuery::LiteQuery(td::BufferSlice data, td::actor::ActorId<ValidatorManager> manager,
td::Promise<td::BufferSlice> promise)
: query_(std::move(data)), manager_(std::move(manager)), promise_(std::move(promise)) {
td::actor::ActorId<LiteServerCache> cache, td::Promise<td::BufferSlice> promise)
: query_(std::move(data)), manager_(std::move(manager)), cache_(std::move(cache)), promise_(std::move(promise)) {
timeout_ = td::Timestamp::in(default_timeout_msec * 0.001);
}
@ -110,7 +113,10 @@ void LiteQuery::alarm() {
fatal_error(-503, "timeout");
}
bool LiteQuery::finish_query(td::BufferSlice result) {
bool LiteQuery::finish_query(td::BufferSlice result, bool skip_cache_update) {
if (use_cache_ && !skip_cache_update) {
td::actor::send_closure(cache_, &LiteServerCache::update, cache_key_, result.clone());
}
if (promise_) {
promise_.set_result(std::move(result));
stop();
@ -124,19 +130,53 @@ bool LiteQuery::finish_query(td::BufferSlice result) {
void LiteQuery::start_up() {
alarm_timestamp() = timeout_;
if(acc_state_promise_) {
td::actor::send_closure_later(actor_id(this),&LiteQuery::perform_fetchAccountState);
if (acc_state_promise_) {
td::actor::send_closure_later(actor_id(this), &LiteQuery::perform_fetchAccountState);
return;
}
auto F = fetch_tl_object<ton::lite_api::Function>(std::move(query_), true);
auto F = fetch_tl_object<ton::lite_api::Function>(query_, true);
if (F.is_error()) {
td::actor::send_closure(manager_, &ValidatorManager::add_lite_query_stats, 0); // unknown
abort_query(F.move_as_error());
return;
}
query_obj_ = F.move_as_ok();
if (!cache_.empty() && query_obj_->get_id() == lite_api::liteServer_sendMessage::ID) {
// Dropping duplicate "sendMessage"
cache_key_ = td::sha256_bits256(query_);
td::actor::send_closure(cache_, &LiteServerCache::process_send_message, cache_key_,
[SelfId = actor_id(this)](td::Result<td::Unit> R) {
if (R.is_ok()) {
td::actor::send_closure(SelfId, &LiteQuery::perform);
} else {
td::actor::send_closure(SelfId, &LiteQuery::abort_query,
R.move_as_error_prefix("cannot send external message : "));
}
});
return;
}
use_cache_ = !cache_.empty() && query_obj_->get_id() == lite_api::liteServer_runSmcMethod::ID;
if (use_cache_) {
cache_key_ = td::sha256_bits256(query_);
td::actor::send_closure(
cache_, &LiteServerCache::lookup, cache_key_, [SelfId = actor_id(this)](td::Result<td::BufferSlice> R) {
if (R.is_error()) {
td::actor::send_closure(SelfId, &LiteQuery::perform);
} else {
td::actor::send_closure(SelfId, &LiteQuery::finish_query, R.move_as_ok(), true);
}
});
} else {
perform();
}
}
void LiteQuery::perform() {
td::actor::send_closure(manager_, &ValidatorManager::add_lite_query_stats, query_obj_->get_id());
lite_api::downcast_call(
*F.move_as_ok().get(),
*query_obj_,
td::overloaded(
[&](lite_api::liteServer_getTime& q) { this->perform_getTime(); },
[&](lite_api::liteServer_getVersion& q) { this->perform_getVersion(); },
@ -491,15 +531,18 @@ void LiteQuery::perform_sendMessage(td::BufferSlice data) {
auto copy = data.clone();
td::actor::send_closure_later(
manager_, &ValidatorManager::check_external_message, std::move(copy),
[Self = actor_id(this), data = std::move(data), manager = manager_](td::Result<td::Ref<ExtMessage>> res) mutable {
if(res.is_error()) {
[Self = actor_id(this), data = std::move(data), manager = manager_, cache = cache_,
cache_key = cache_key_](td::Result<td::Ref<ExtMessage>> res) mutable {
if (res.is_error()) {
// Don't cache errors
td::actor::send_closure(cache, &LiteServerCache::drop_send_message_from_cache, cache_key);
td::actor::send_closure(Self, &LiteQuery::abort_query,
res.move_as_error_prefix("cannot apply external message to current state : "s));
} else {
LOG(INFO) << "sending an external message to validator manager";
td::actor::send_closure_later(manager, &ValidatorManager::send_external_message, res.move_as_ok());
auto b = ton::create_serialize_tl_object<ton::lite_api::liteServer_sendMsgStatus>(1);
td::actor::send_closure(Self, &LiteQuery::finish_query, std::move(b));
td::actor::send_closure(Self, &LiteQuery::finish_query, std::move(b), false);
}
});
}
@ -805,7 +848,7 @@ void LiteQuery::perform_runSmcMethod(BlockIdExt blkid, WorkchainId workchain, St
fatal_error("more than 64k parameter bytes passed");
return;
}
if (mode & ~0x1f) {
if (mode & ~0x3f) {
fatal_error("unsupported mode in runSmcMethod");
return;
}
@ -1240,14 +1283,17 @@ void LiteQuery::finish_getAccountState(td::BufferSlice shard_proof) {
// same as in lite-client/lite-client-common.cpp
static td::Ref<vm::Tuple> prepare_vm_c7(ton::UnixTime now, ton::LogicalTime lt, td::Ref<vm::CellSlice> my_addr,
const block::CurrencyCollection& balance) {
const block::CurrencyCollection& balance,
const block::ConfigInfo* config = nullptr, td::Ref<vm::Cell> my_code = {},
td::RefInt256 due_payment = td::zero_refint()) {
td::BitArray<256> rand_seed;
td::RefInt256 rand_seed_int{true};
td::Random::secure_bytes(rand_seed.as_slice());
if (!rand_seed_int.unique_write().import_bits(rand_seed.cbits(), 256, false)) {
return {};
}
auto tuple = vm::make_tuple_ref(td::make_refint(0x076ef1ea), // [ magic:0x076ef1ea
std::vector<vm::StackEntry> tuple = {
td::make_refint(0x076ef1ea), // [ magic:0x076ef1ea
td::make_refint(0), // actions:Integer
td::make_refint(0), // msgs_sent:Integer
td::make_refint(now), // unixtime:Integer
@ -1256,9 +1302,26 @@ static td::Ref<vm::Tuple> prepare_vm_c7(ton::UnixTime now, ton::LogicalTime lt,
std::move(rand_seed_int), // rand_seed:Integer
balance.as_vm_tuple(), // balance_remaining:[Integer (Maybe Cell)]
my_addr, // myself:MsgAddressInt
vm::StackEntry()); // global_config:(Maybe Cell) ] = SmartContractInfo;
LOG(DEBUG) << "SmartContractInfo initialized with " << vm::StackEntry(tuple).to_string();
return vm::make_tuple_ref(std::move(tuple));
config ? config->get_root_cell() : vm::StackEntry() // global_config:(Maybe Cell) ] = SmartContractInfo;
};
if (config && config->get_global_version() >= 4) {
tuple.push_back(my_code); // code:Cell
tuple.push_back(block::CurrencyCollection::zero().as_vm_tuple()); // in_msg_value:[Integer (Maybe Cell)]
tuple.push_back(td::zero_refint()); // storage_fees:Integer
// [ wc:Integer shard:Integer seqno:Integer root_hash:Integer file_hash:Integer] = BlockId;
// [ last_mc_blocks:[BlockId...]
// prev_key_block:BlockId ] : PrevBlocksInfo
auto info = config->get_prev_blocks_info();
tuple.push_back(info.is_ok() ? info.move_as_ok() : vm::StackEntry());
}
if (config && config->get_global_version() >= 6) {
tuple.push_back(config->get_unpacked_config_tuple(now)); // unpacked_config_tuple:[...]
tuple.push_back(due_payment); // due_payment:Integer
}
auto tuple_ref = td::make_cnt_ref<std::vector<vm::StackEntry>>(std::move(tuple));
LOG(DEBUG) << "SmartContractInfo initialized with " << vm::StackEntry(tuple_ref).to_string();
return vm::make_tuple_ref(std::move(tuple_ref));
}
void LiteQuery::finish_runSmcMethod(td::BufferSlice shard_proof, td::BufferSlice state_proof, Ref<vm::Cell> acc_root,
@ -1277,12 +1340,14 @@ void LiteQuery::finish_runSmcMethod(td::BufferSlice shard_proof, td::BufferSlice
}
vm::MerkleProofBuilder pb{std::move(acc_root)};
block::gen::Account::Record_account acc;
block::gen::StorageInfo::Record storage_info;
block::gen::AccountStorage::Record store;
block::CurrencyCollection balance;
block::gen::StateInit::Record state_init;
if (!(tlb::unpack_cell(pb.root(), acc) && tlb::csr_unpack(std::move(acc.storage), store) &&
balance.validate_unpack(store.balance) && store.state->prefetch_ulong(1) == 1 &&
store.state.write().advance(1) && tlb::csr_unpack(std::move(store.state), state_init))) {
store.state.write().advance(1) && tlb::csr_unpack(std::move(store.state), state_init) &&
tlb::csr_unpack(std::move(acc.storage_stat), storage_info))) {
LOG(INFO) << "error unpacking account state, or account is frozen or uninitialized";
td::Result<td::BufferSlice> proof_boc;
if (mode & 2) {
@ -1303,12 +1368,35 @@ void LiteQuery::finish_runSmcMethod(td::BufferSlice shard_proof, td::BufferSlice
}
auto code = state_init.code->prefetch_ref();
auto data = state_init.data->prefetch_ref();
auto acc_libs = state_init.library->prefetch_ref();
long long gas_limit = client_method_gas_limit;
td::RefInt256 due_payment;
if (storage_info.due_payment.write().fetch_long(1)) {
due_payment = block::tlb::t_Grams.as_integer(storage_info.due_payment);
} else {
due_payment = td::zero_refint();
}
LOG(DEBUG) << "creating VM with gas limit " << gas_limit;
// **** INIT VM ****
auto r_config = block::ConfigInfo::extract_config(
mc_state_->root_cell(),
block::ConfigInfo::needLibraries | block::ConfigInfo::needCapabilities | block::ConfigInfo::needPrevBlocks);
if (r_config.is_error()) {
fatal_error(r_config.move_as_error());
return;
}
auto config = r_config.move_as_ok();
std::vector<td::Ref<vm::Cell>> libraries;
if (config->get_libraries_root().not_null()) {
libraries.push_back(config->get_libraries_root());
}
if (acc_libs.not_null()) {
libraries.push_back(acc_libs);
}
vm::GasLimits gas{gas_limit, gas_limit};
vm::VmState vm{std::move(code), std::move(stack_), gas, 1, std::move(data), vm::VmLog::Null()};
auto c7 = prepare_vm_c7(gen_utime, gen_lt, td::make_ref<vm::CellSlice>(acc.addr->clone()), balance);
vm::VmState vm{code, std::move(stack_), gas, 1, std::move(data), vm::VmLog::Null(), std::move(libraries)};
auto c7 = prepare_vm_c7(gen_utime, gen_lt, td::make_ref<vm::CellSlice>(acc.addr->clone()), balance, config.get(),
std::move(code), due_payment);
vm.set_c7(c7); // tuple with SmartContractInfo
// vm.incr_stack_trace(1); // enable stack dump after each step
LOG(INFO) << "starting VM to run GET-method of smart contract " << acc_workchain_ << ":" << acc_addr_.to_hex();
@ -1324,6 +1412,9 @@ void LiteQuery::finish_runSmcMethod(td::BufferSlice shard_proof, td::BufferSlice
td::BufferSlice c7_info, result;
if (mode & 8) {
// serialize c7
if (!(mode & 32)) {
c7 = prepare_vm_c7(gen_utime, gen_lt, td::make_ref<vm::CellSlice>(acc.addr->clone()), balance);
}
vm::CellBuilder cb;
if (!(vm::StackEntry{std::move(c7)}.serialize(cb) && cb.finalize_to(cell))) {
fatal_error("cannot serialize c7");

14
validator/impl/liteserver.hpp
View file

@ -27,7 +27,7 @@
#include "shard.hpp"
#include "proof.hpp"
#include "block/block-auto.h"
#include "auto/tl/lite_api.h"
namespace ton {
@ -37,11 +37,16 @@ using td::Ref;
class LiteQuery : public td::actor::Actor {
td::BufferSlice query_;
td::actor::ActorId<ton::validator::ValidatorManager> manager_;
td::actor::ActorId<LiteServerCache> cache_;
td::Timestamp timeout_;
td::Promise<td::BufferSlice> promise_;
td::Promise<std::tuple<td::Ref<vm::CellSlice>,UnixTime,LogicalTime,std::unique_ptr<block::ConfigInfo>>> acc_state_promise_;
tl_object_ptr<ton::lite_api::Function> query_obj_;
bool use_cache_{false};
td::Bits256 cache_key_;
int pending_{0};
int mode_{0};
WorkchainId acc_workchain_;
@ -75,11 +80,11 @@ class LiteQuery : public td::actor::Actor {
ls_capabilities = 7
}; // version 1.1; +1 = build block proof chains, +2 = masterchainInfoExt, +4 = runSmcMethod
LiteQuery(td::BufferSlice data, td::actor::ActorId<ton::validator::ValidatorManager> manager,
td::Promise<td::BufferSlice> promise);
td::actor::ActorId<LiteServerCache> cache, td::Promise<td::BufferSlice> promise);
LiteQuery(WorkchainId wc, StdSmcAddress acc_addr, td::actor::ActorId<ton::validator::ValidatorManager> manager,
td::Promise<std::tuple<td::Ref<vm::CellSlice>,UnixTime,LogicalTime,std::unique_ptr<block::ConfigInfo>>> promise);
static void run_query(td::BufferSlice data, td::actor::ActorId<ton::validator::ValidatorManager> manager,
td::Promise<td::BufferSlice> promise);
td::actor::ActorId<LiteServerCache> cache, td::Promise<td::BufferSlice> promise);
static void fetch_account_state(WorkchainId wc, StdSmcAddress acc_addr, td::actor::ActorId<ton::validator::ValidatorManager> manager,
td::Promise<std::tuple<td::Ref<vm::CellSlice>,UnixTime,LogicalTime,std::unique_ptr<block::ConfigInfo>>> promise);
@ -90,9 +95,10 @@ class LiteQuery : public td::actor::Actor {
bool fatal_error(int err_code, std::string err_msg = "");
void abort_query(td::Status reason);
void abort_query_ext(td::Status reason, std::string err_msg);
bool finish_query(td::BufferSlice result);
bool finish_query(td::BufferSlice result, bool skip_cache_update = false);
void alarm() override;
void start_up() override;
void perform();
void perform_getTime();
void perform_getVersion();
void perform_getMasterchainInfo(int mode);

3
validator/impl/validate-query.cpp
View file

@ -960,6 +960,9 @@ bool ValidateQuery::fetch_config_params() {
}
compute_phase_cfg_.prev_blocks_info = prev_blocks_info.move_as_ok();
}
if (compute_phase_cfg_.global_version >= 6) {
compute_phase_cfg_.unpacked_config_tuple = config_->get_unpacked_config_tuple(now_);
}
compute_phase_cfg_.suspended_addresses = config_->get_suspended_addresses(now_);
compute_phase_cfg_.size_limits = size_limits;
}

2
validator/interfaces/db.h
View file

@ -119,7 +119,7 @@ class Db : public td::actor::Actor {
td::Promise<td::BufferSlice> promise) = 0;
virtual void set_async_mode(bool mode, td::Promise<td::Unit> promise) = 0;
virtual void run_gc(UnixTime ts, UnixTime archive_ttl) = 0;
virtual void run_gc(UnixTime mc_ts, UnixTime gc_ts, UnixTime archive_ttl) = 0;
};
} // namespace validator

18
validator/interfaces/liteserver.h
View file

@ -19,16 +19,20 @@
#pragma once
#include "td/actor/actor.h"
#include "td/utils/buffer.h"
#include "common/bitstring.h"
namespace ton {
namespace validator {
namespace ton::validator {
class LiteServerCache : public td::actor::Actor {
public:
virtual ~LiteServerCache() = default;
~LiteServerCache() override = default;
virtual void lookup(td::Bits256 key, td::Promise<td::BufferSlice> promise) = 0;
virtual void update(td::Bits256 key, td::BufferSlice value) = 0;
virtual void process_send_message(td::Bits256 key, td::Promise<td::Unit> promise) = 0;
virtual void drop_send_message_from_cache(td::Bits256 key) = 0;
};
} // namespace validator
} // namespace ton
} // namespace ton::validator

3
validator/interfaces/validator-manager.h
View file

@ -178,6 +178,9 @@ class ValidatorManager : public ValidatorManagerInterface {
virtual void get_block_by_seqno_from_db_for_litequery(AccountIdPrefixFull account, BlockSeqno seqno,
td::Promise<ConstBlockHandle> promise) = 0;
virtual void add_lite_query_stats(int lite_query_id) {
}
static bool is_persistent_state(UnixTime ts, UnixTime prev_ts) {
return ts / (1 << 17) != prev_ts / (1 << 17);
}

47
validator/manager.cpp
View file

@ -401,6 +401,7 @@ void ValidatorManagerImpl::add_external_message(td::Ref<ExtMessage> msg) {
}
}
void ValidatorManagerImpl::check_external_message(td::BufferSlice data, td::Promise<td::Ref<ExtMessage>> promise) {
++ls_stats_check_ext_messages_;
auto state = do_get_last_liteserver_state();
if (state.is_null()) {
promise.set_error(td::Status::Error(ErrorCode::notready, "not ready"));
@ -1185,7 +1186,7 @@ void ValidatorManagerImpl::write_handle(BlockHandle handle, td::Promise<td::Unit
void ValidatorManagerImpl::written_handle(BlockHandle handle, td::Promise<td::Unit> promise) {
bool received = handle->received();
bool inited_state = handle->received_state();
bool inited_proof = handle->id().is_masterchain() ? handle->inited_proof() : handle->inited_proof();
bool inited_proof = handle->id().is_masterchain() ? handle->inited_proof() : handle->inited_proof_link();
if (handle->need_flush()) {
handle->flush(actor_id(this), handle, std::move(promise));
@ -1198,11 +1199,24 @@ void ValidatorManagerImpl::written_handle(BlockHandle handle, td::Promise<td::Un
td::actor::send_closure(it->second.actor_, &WaitBlockData::force_read_from_db);
}
}
if (inited_state && inited_proof) {
if (inited_state) {
auto it = wait_state_.find(handle->id());
if (it != wait_state_.end()) {
td::actor::send_closure(it->second.actor_, &WaitBlockState::force_read_from_db);
}
} else {
if (handle->inited_proof_link()) {
auto it = wait_state_.find(handle->id());
if (it != wait_state_.end()) {
td::actor::send_closure(it->second.actor_, &WaitBlockState::after_get_proof_link);
}
}
if (handle->id().is_masterchain() && handle->inited_proof()) {
auto it = wait_state_.find(handle->id());
if (it != wait_state_.end()) {
td::actor::send_closure(it->second.actor_, &WaitBlockState::after_get_proof);
}
}
}
promise.set_value(td::Unit());
@ -2371,9 +2385,9 @@ void ValidatorManagerImpl::state_serializer_update(BlockSeqno seqno) {
void ValidatorManagerImpl::alarm() {
try_advance_gc_masterchain_block();
alarm_timestamp() = td::Timestamp::in(1.0);
if (gc_masterchain_handle_) {
td::actor::send_closure(db_, &Db::run_gc, gc_masterchain_handle_->unix_time(),
static_cast<UnixTime>(opts_->archive_ttl()));
if (last_masterchain_block_handle_ && gc_masterchain_handle_) {
td::actor::send_closure(db_, &Db::run_gc, last_masterchain_block_handle_->unix_time(),
gc_masterchain_handle_->unix_time(), static_cast<UnixTime>(opts_->archive_ttl()));
}
if (log_status_at_.is_in_past()) {
if (last_masterchain_block_handle_) {
@ -2453,6 +2467,29 @@ void ValidatorManagerImpl::alarm() {
}
}
alarm_timestamp().relax(check_shard_clients_);
if (log_ls_stats_at_.is_in_past()) {
if (!ls_stats_.empty() || ls_stats_check_ext_messages_ != 0) {
td::StringBuilder sb;
sb << "Liteserver stats (1 minute):";
td::uint32 total = 0;
for (const auto &p : ls_stats_) {
sb << " " << lite_query_name_by_id(p.first) << ":" << p.second;
total += p.second;
}
if (total > 0) {
sb << " TOTAL:" << total;
}
if (ls_stats_check_ext_messages_ > 0) {
sb << " checkExtMessage:" << ls_stats_check_ext_messages_;
}
LOG(WARNING) << sb.as_cslice();
}
ls_stats_.clear();
ls_stats_check_ext_messages_ = 0;
log_ls_stats_at_ = td::Timestamp::in(60.0);
}
alarm_timestamp().relax(log_ls_stats_at_);
}
void ValidatorManagerImpl::update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise<td::Unit> promise) {

8
validator/manager.hpp
View file

@ -575,6 +575,10 @@ class ValidatorManagerImpl : public ValidatorManager {
td::Result<ConstBlockHandle> r_handle,
td::Promise<ConstBlockHandle> promise);
void add_lite_query_stats(int lite_query_id) override {
++ls_stats_[lite_query_id];
}
private:
td::Timestamp resend_shard_blocks_at_;
td::Timestamp check_waiters_at_;
@ -640,6 +644,10 @@ class ValidatorManagerImpl : public ValidatorManager {
private:
std::map<BlockSeqno, WaitList<td::actor::Actor, td::Unit>> shard_client_waiters_;
td::actor::ActorOwn<QueueSizeCounter> queue_size_counter_;
td::Timestamp log_ls_stats_at_;
std::map<int, td::uint32> ls_stats_; // lite_api ID -> count, 0 for unknown
td::uint32 ls_stats_check_ext_messages_{0};
};
} // namespace validator

22
validator/validator-group.cpp
View file

@ -155,9 +155,9 @@ void ValidatorGroup::accept_block_candidate(td::uint32 round_id, PublicKeyHash s
td::actor::send_closure(manager_, &ValidatorManager::log_validator_session_stats, next_block_id, std::move(stats));
auto block =
block_data.size() > 0 ? create_block(next_block_id, std::move(block_data)).move_as_ok() : td::Ref<BlockData>{};
bool send_broadcast = src == local_id_;
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), block_id = next_block_id, block, prev = prev_block_ids_,
sig_set, approve_sig_set,
auto P = td::PromiseCreator::lambda([=, SelfId = actor_id(this), block_id = next_block_id, prev = prev_block_ids_,
promise = std::move(promise)](td::Result<td::Unit> R) mutable {
if (R.is_error()) {
if (R.error().code() == ErrorCode::cancelled) {
@ -166,14 +166,15 @@ void ValidatorGroup::accept_block_candidate(td::uint32 round_id, PublicKeyHash s
}
LOG_CHECK(R.error().code() == ErrorCode::timeout || R.error().code() == ErrorCode::notready) << R.move_as_error();
td::actor::send_closure(SelfId, &ValidatorGroup::retry_accept_block_query, block_id, std::move(block),
std::move(prev), std::move(sig_set), std::move(approve_sig_set), std::move(promise));
std::move(prev), std::move(sig_set), std::move(approve_sig_set), send_broadcast,
std::move(promise));
} else {
promise.set_value(R.move_as_ok());
}
});
run_accept_block_query(next_block_id, std::move(block), prev_block_ids_, validator_set_, std::move(sig_set),
std::move(approve_sig_set), src == local_id_, manager_, std::move(P));
std::move(approve_sig_set), send_broadcast, manager_, std::move(P));
prev_block_ids_ = std::vector<BlockIdExt>{next_block_id};
cached_collated_block_ = nullptr;
approved_candidates_cache_.clear();
@ -181,21 +182,22 @@ void ValidatorGroup::accept_block_candidate(td::uint32 round_id, PublicKeyHash s
void ValidatorGroup::retry_accept_block_query(BlockIdExt block_id, td::Ref<BlockData> block,
std::vector<BlockIdExt> prev, td::Ref<BlockSignatureSet> sig_set,
td::Ref<BlockSignatureSet> approve_sig_set,
td::Ref<BlockSignatureSet> approve_sig_set, bool send_broadcast,
td::Promise<td::Unit> promise) {
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), block_id, block, prev, sig_set, approve_sig_set,
promise = std::move(promise)](td::Result<td::Unit> R) mutable {
auto P = td::PromiseCreator::lambda(
[=, SelfId = actor_id(this), promise = std::move(promise)](td::Result<td::Unit> R) mutable {
if (R.is_error()) {
LOG_CHECK(R.error().code() == ErrorCode::timeout) << R.move_as_error();
td::actor::send_closure(SelfId, &ValidatorGroup::retry_accept_block_query, block_id, std::move(block),
std::move(prev), std::move(sig_set), std::move(approve_sig_set), std::move(promise));
std::move(prev), std::move(sig_set), std::move(approve_sig_set), send_broadcast,
std::move(promise));
} else {
promise.set_value(R.move_as_ok());
}
});
run_accept_block_query(block_id, std::move(block), prev, validator_set_, std::move(sig_set),
std::move(approve_sig_set), false, manager_, std::move(P));
std::move(approve_sig_set), send_broadcast, manager_, std::move(P));
}
void ValidatorGroup::skip_round(td::uint32 round_id) {
@ -347,7 +349,7 @@ void ValidatorGroup::start(std::vector<BlockIdExt> prev, BlockIdExt min_masterch
auto block =
p.block.size() > 0 ? create_block(next_block_id, std::move(p.block)).move_as_ok() : td::Ref<BlockData>{};
retry_accept_block_query(next_block_id, std::move(block), prev_block_ids_, std::move(p.sigs),
std::move(p.approve_sigs), std::move(p.promise));
std::move(p.approve_sigs), false, std::move(p.promise));
prev_block_ids_ = std::vector<BlockIdExt>{next_block_id};
}
postponed_accept_.clear();

2
validator/validator-group.hpp
View file

@ -43,7 +43,7 @@ class ValidatorGroup : public td::actor::Actor {
void skip_round(td::uint32 round);
void retry_accept_block_query(BlockIdExt block_id, td::Ref<BlockData> block, std::vector<BlockIdExt> prev,
td::Ref<BlockSignatureSet> sigs, td::Ref<BlockSignatureSet> approve_sigs,
td::Promise<td::Unit> promise);
bool send_broadcast, td::Promise<td::Unit> promise);
void get_approved_candidate(PublicKey source, RootHash root_hash, FileHash file_hash,
FileHash collated_data_file_hash, td::Promise<BlockCandidate> promise);
BlockIdExt create_next_block_id(RootHash root_hash, FileHash file_hash) const;

14
validator/validator-options.hpp
View file

@ -117,6 +117,12 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
td::uint32 get_celldb_compress_depth() const override {
return celldb_compress_depth_;
}
size_t get_max_open_archive_files() const override {
return max_open_archive_files_;
}
double get_archive_preload_period() const override {
return archive_preload_period_;
}
void set_zero_block_id(BlockIdExt block_id) override {
zero_block_id_ = block_id;
@ -173,6 +179,12 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
void set_celldb_compress_depth(td::uint32 value) override {
celldb_compress_depth_ = value;
}
void set_max_open_archive_files(size_t value) override {
max_open_archive_files_ = value;
}
void set_archive_preload_period(double value) override {
archive_preload_period_ = value;
}
ValidatorManagerOptionsImpl *make_copy() const override {
return new ValidatorManagerOptionsImpl(*this);
@ -216,6 +228,8 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
BlockSeqno sync_upto_{0};
std::string session_logs_file_;
td::uint32 celldb_compress_depth_{0};
size_t max_open_archive_files_ = 0;
double archive_preload_period_ = 0.0;
};
} // namespace validator

4
validator/validator.h
View file

@ -82,6 +82,8 @@ struct ValidatorManagerOptions : public td::CntObject {
virtual BlockSeqno sync_upto() const = 0;
virtual std::string get_session_logs_file() const = 0;
virtual td::uint32 get_celldb_compress_depth() const = 0;
virtual size_t get_max_open_archive_files() const = 0;
virtual double get_archive_preload_period() const = 0;
virtual void set_zero_block_id(BlockIdExt block_id) = 0;
virtual void set_init_block_id(BlockIdExt block_id) = 0;
@ -102,6 +104,8 @@ struct ValidatorManagerOptions : public td::CntObject {
virtual void set_sync_upto(BlockSeqno seqno) = 0;
virtual void set_session_logs_file(std::string f) = 0;
virtual void set_celldb_compress_depth(td::uint32 value) = 0;
virtual void set_max_open_archive_files(size_t value) = 0;
virtual void set_archive_preload_period(double value) = 0;
static td::Ref<ValidatorManagerOptions> create(
BlockIdExt zero_block_id, BlockIdExt init_block_id,