external/ton
1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-03-09 15:40:10 +00:00
Code Issues Releases Wiki Activity

Merge remote-tracking branch 'origin/testnet' into testnet

Browse source
This commit is contained in:
neodiX 2024-12-18 21:15:58 +04:00
commit c5a1234ebc
53 changed files with 1146 additions and 259 deletions
Download patch file Download diff file Expand all files Collapse all files

14
Changelog.md
View file

@ -1,3 +1,17 @@
## 2024.12 Update
1. FunC 0.4.6: Fix in try/catch handling, fixing pure flag for functions stored in variables
2. Merging parts of Accelerator: support of specific shard monitoring, archive/liteserver slice format, support for partial liteservers, proxy liteserver, on-demand neighbour queue loading
3. Fix of asynchronous cell loading
4. Various improvements: caching certificates checks, better block overloading detection, `_malloc` in emulator
5. Introduction of telemetry in overlays
6. Use non-null local-id for tonlib-LS interaction - mitigates MitM attack.
7. Adding `SECP256K1_XONLY_PUBKEY_TWEAK_ADD`, `SETCONTCTRMANY` instructions to TVM (activated by `Config8.version >= 9`)
8. Private keys export via validator-engine-console - required for better backups
9. Fix proof checking in tonlib, `hash` in `raw.Message` in tonlib_api
Besides the work of the core team, this update is based on the efforts of OtterSec and LayerZero (FunC), tg:@throwunless (FunC), Aviv Frenkel and Dima Kogan from Fordefi (LS MitM), @hacker-volodya (Tonlib), OKX team (async cell loading), @krigga (emulator)
## 2024.10 Update
1. Parallel write to celldb: substantial improvement of sync and GC speed, especially with slow disks.

4
create-hardfork/create-hardfork.cpp
View file

@ -272,6 +272,10 @@ class HardforkCreator : public td::actor::Actor {
void download_archive(ton::BlockSeqno masterchain_seqno, ton::ShardIdFull shard_prefix, std::string tmp_dir,
td::Timestamp timeout, td::Promise<std::string> promise) override {
}
void download_out_msg_queue_proof(
ton::ShardIdFull dst_shard, std::vector<ton::BlockIdExt> blocks, block::ImportedMsgQueueLimits limits,
td::Timestamp timeout, td::Promise<std::vector<td::Ref<ton::validator::OutMsgQueueProof>>> promise) override {
}
void new_key_block(ton::validator::BlockHandle handle) override {
}

6
crypto/block/block.cpp
View file

@ -660,6 +660,12 @@ bool EnqueuedMsgDescr::check_key(td::ConstBitPtr key) const {
hash_ == key + 96;
}
bool ImportedMsgQueueLimits::deserialize(vm::CellSlice& cs) {
return cs.fetch_ulong(8) == 0xd3 // imported_msg_queue_limits#d3
&& cs.fetch_uint_to(32, max_bytes) // max_bytes:#
&& cs.fetch_uint_to(32, max_msgs); // max_msgs:#
}
bool ParamLimits::deserialize(vm::CellSlice& cs) {
return cs.fetch_ulong(8) == 0xc3 // param_limits#c3
&& cs.fetch_uint_to(32, limits_[0]) // underload:uint32

10
crypto/block/block.h
View file

@ -216,6 +216,16 @@ static inline std::ostream& operator<<(std::ostream& os, const MsgProcessedUptoC
return proc_coll.print(os);
}
struct ImportedMsgQueueLimits {
// Default values
td::uint32 max_bytes = 1 << 16;
td::uint32 max_msgs = 30;
bool deserialize(vm::CellSlice& cs);
ImportedMsgQueueLimits operator*(td::uint32 x) const {
return {max_bytes * x, max_msgs * x};
}
};
struct ParamLimits {
enum { limits_cnt = 4 };
enum { cl_underload = 0, cl_normal = 1, cl_soft = 2, cl_medium = 3, cl_hard = 4 };

6
crypto/common/bitstring.h
View file

@ -554,11 +554,7 @@ class BitArray {
set_same(0);
}
void set_zero_s() {
volatile uint8* p = data();
auto x = m;
while (x--) {
*p++ = 0;
}
as_slice().fill_zero_secure();
}
void set_ones() {
set_same(1);

8
crypto/vm/db/DynamicBagOfCellsDb.cpp
View file

@ -111,14 +111,16 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat
}
void load_cell_async(td::Slice hash, std::shared_ptr<AsyncExecutor> executor,
td::Promise<Ref<DataCell>> promise) override {
auto promise_ptr = std::make_shared<td::Promise<Ref<DataCell>>>(std::move(promise));
auto info = hash_table_.get_if_exists(hash);
if (info && info->sync_with_db) {
TRY_RESULT_PROMISE(promise, loaded_cell, info->cell->load_cell());
promise.set_result(loaded_cell.data_cell);
executor->execute_async([promise = std::move(promise_ptr), cell = info->cell]() mutable {
TRY_RESULT_PROMISE((*promise), loaded_cell, cell->load_cell());
promise->set_result(loaded_cell.data_cell);
});
return;
}
SimpleExtCellCreator ext_cell_creator(cell_db_reader_);
auto promise_ptr = std::make_shared<td::Promise<Ref<DataCell>>>(std::move(promise));
executor->execute_async(
[executor, loader = *loader_, hash = CellHash::from_slice(hash), db = this,
ext_cell_creator = std::move(ext_cell_creator), promise = std::move(promise_ptr)]() mutable {

4
emulator/CMakeLists.txt
View file

@ -47,8 +47,8 @@ endif()
if (USE_EMSCRIPTEN)
add_executable(emulator-emscripten ${EMULATOR_EMSCRIPTEN_SOURCE})
target_link_libraries(emulator-emscripten PUBLIC emulator)
target_link_options(emulator-emscripten PRIVATE -sEXPORTED_RUNTIME_METHODS=_malloc,free,UTF8ToString,stringToUTF8,allocate,ALLOC_NORMAL,lengthBytesUTF8)
target_link_options(emulator-emscripten PRIVATE -sEXPORTED_FUNCTIONS=_emulate,_free,_run_get_method,_create_emulator,_destroy_emulator,_emulate_with_emulator,_version)
target_link_options(emulator-emscripten PRIVATE -sEXPORTED_RUNTIME_METHODS=UTF8ToString,stringToUTF8,allocate,ALLOC_NORMAL,lengthBytesUTF8)
target_link_options(emulator-emscripten PRIVATE -sEXPORTED_FUNCTIONS=_emulate,_free,_malloc,_run_get_method,_create_emulator,_destroy_emulator,_emulate_with_emulator,_version)
target_link_options(emulator-emscripten PRIVATE -sEXPORT_NAME=EmulatorModule)
target_link_options(emulator-emscripten PRIVATE -sERROR_ON_UNDEFINED_SYMBOLS=0)
target_link_options(emulator-emscripten PRIVATE -Oz)

12
keyring/keyring.cpp
View file

@ -28,7 +28,7 @@ namespace ton {
namespace keyring {
KeyringImpl::PrivateKeyDescr::PrivateKeyDescr(PrivateKey private_key, bool is_temp)
: public_key(private_key.compute_public_key()), is_temp(is_temp) {
: public_key(private_key.compute_public_key()), private_key(private_key), is_temp(is_temp) {
auto D = private_key.create_decryptor_async();
D.ensure();
decryptor_sign = D.move_as_ok();
@ -190,6 +190,16 @@ void KeyringImpl::decrypt_message(PublicKeyHash key_hash, td::BufferSlice data,
}
}
void KeyringImpl::export_all_private_keys(td::Promise<std::vector<PrivateKey>> promise) {
std::vector<PrivateKey> keys;
for (auto& [_, descr] : map_) {
if (!descr->is_temp && descr->private_key.exportable()) {
keys.push_back(descr->private_key);
}
}
promise.set_value(std::move(keys));
}
td::actor::ActorOwn<Keyring> Keyring::create(std::string db_root) {
return td::actor::create_actor<KeyringImpl>("keyring", db_root);
}

2
keyring/keyring.h
View file

@ -44,6 +44,8 @@ class Keyring : public td::actor::Actor {
virtual void decrypt_message(PublicKeyHash key_hash, td::BufferSlice data, td::Promise<td::BufferSlice> promise) = 0;
virtual void export_all_private_keys(td::Promise<std::vector<PrivateKey>> promise) = 0;
static td::actor::ActorOwn<Keyring> create(std::string db_root);
};

3
keyring/keyring.hpp
View file

@ -33,6 +33,7 @@ class KeyringImpl : public Keyring {
td::actor::ActorOwn<DecryptorAsync> decryptor_sign;
td::actor::ActorOwn<DecryptorAsync> decryptor_decrypt;
PublicKey public_key;
PrivateKey private_key;
bool is_temp;
PrivateKeyDescr(PrivateKey private_key, bool is_temp);
};
@ -56,6 +57,8 @@ class KeyringImpl : public Keyring {
void decrypt_message(PublicKeyHash key_hash, td::BufferSlice data, td::Promise<td::BufferSlice> promise) override;
void export_all_private_keys(td::Promise<std::vector<PrivateKey>> promise) override;
KeyringImpl(std::string db_root) : db_root_(db_root) {
}

26
recent_changelog.md
View file

@ -1,17 +1,13 @@
## 2024.10 Update
1. Parallel write to celldb: substantial improvement of sync and GC speed, especially with slow disks.
2. Decreased network traffic: only first block candidate is sent optimistically.
3. Improved channel creation and dht lookups, introduction of semi-private overlays
4. New LS dispatch queue related methods and improvement security
5. Fixing recursion in TVM continuations
6. Improved stats for actors, validator sessions, perf counters, overlays, adnl, rocksdb
7. Migration to C++20
8. Improved block size estimates: account for depth in various structures
9. Fix bug with `<<` optimization in FunC
10. Minor changes of TVM which will be activated by `Config8.version >= 9`
11. Multiple minor improvements
Besides the work of the core team, this update is based on the efforts of @krigga (emulator), Arayz @ TonBit (LS security, TVM recursion), @ret2happy (UB in BLST).
## 2024.12 Update
1. FunC 0.4.6: Fix in try/catch handling, fixing pure flag for functions stored in variables
2. Merging parts of Accelerator: support of specific shard monitoring, archive/liteserver slice format, support for partial liteservers, proxy liteserver, on-demand neighbour queue loading
3. Fix of asynchronous cell loading
4. Various improvements: caching certificates checks, better block overloading detection, `_malloc` in emulator
5. Introduction of telemetry in overlays
6. Use non-null local-id for tonlib-LS interaction - mitigates MitM attack.
7. Adding `SECP256K1_XONLY_PUBKEY_TWEAK_ADD`, `SETCONTCTRMANY` instructions to TVM (activated by `Config8.version >= 9`)
8. Private keys export via validator-engine-console - required for better backups
9. Fix proof checking in tonlib, `hash` in `raw.Message` in tonlib_api
Besides the work of the core team, this update is based on the efforts of OtterSec and LayerZero (FunC), tg:@throwunless (FunC), Aviv Frenkel and Dima Kogan from Fordefi (LS MitM), @hacker-volodya (Tonlib), OKX team (async cell loading), @krigga (emulator)

15
tdutils/td/utils/StringBuilder.h
View file

@ -149,4 +149,19 @@ std::enable_if_t<std::is_arithmetic<T>::value, string> to_string(const T &x) {
return sb.as_cslice().str();
}
template <class SB>
struct LambdaPrintHelper {
SB& sb;
};
template <class SB, class F>
SB& operator<<(const LambdaPrintHelper<SB>& helper, F&& f) {
f(helper.sb);
return helper.sb;
}
struct LambdaPrint {};
inline LambdaPrintHelper<td::StringBuilder> operator<<(td::StringBuilder& sb, const LambdaPrint&) {
return LambdaPrintHelper<td::StringBuilder>{sb};
}
} // namespace td

4
tdutils/td/utils/logging.h
View file

@ -74,6 +74,7 @@
#define LOG(level) LOG_IMPL(level, level, true, ::td::Slice())
#define LOG_IF(level, condition) LOG_IMPL(level, level, condition, #condition)
#define FLOG(level) LOG_IMPL(level, level, true, ::td::Slice()) << td::LambdaPrint{} << [&](auto &sb)
#define VLOG(level) LOG_IMPL(DEBUG, level, true, TD_DEFINE_STR(level))
#define VLOG_IF(level, condition) LOG_IMPL(DEBUG, level, condition, TD_DEFINE_STR(level) " " #condition)
@ -263,6 +264,9 @@ class Logger {
sb_ << other;
return *this;
}
LambdaPrintHelper<td::Logger> operator<<(const LambdaPrint &) {
return LambdaPrintHelper<td::Logger>{*this};
}
MutableCSlice as_cslice() {
return sb_.as_cslice();

4
test/test-ton-collator.cpp
View file

@ -373,6 +373,10 @@ class TestNode : public td::actor::Actor {
void download_archive(ton::BlockSeqno masterchain_seqno, ton::ShardIdFull shard_prefix, std::string tmp_dir,
td::Timestamp timeout, td::Promise<std::string> promise) override {
}
void download_out_msg_queue_proof(
ton::ShardIdFull dst_shard, std::vector<ton::BlockIdExt> blocks, block::ImportedMsgQueueLimits limits,
td::Timestamp timeout, td::Promise<std::vector<td::Ref<ton::validator::OutMsgQueueProof>>> promise) override {
}
void new_key_block(ton::validator::BlockHandle handle) override {
}

9
tl/generate/scheme/ton_api.tl
View file

@ -454,6 +454,10 @@ tonNode.success = tonNode.Success;
tonNode.archiveNotFound = tonNode.ArchiveInfo;
tonNode.archiveInfo id:long = tonNode.ArchiveInfo;
tonNode.importedMsgQueueLimits max_bytes:int max_msgs:int = ImportedMsgQueueLimits;
tonNode.outMsgQueueProof queue_proofs:bytes block_state_proofs:bytes msg_counts:(vector int) = tonNode.OutMsgQueueProof;
tonNode.outMsgQueueProofEmpty = tonNode.OutMsgQueueProof;
tonNode.forgetPeer = tonNode.ForgetPeer;
---functions---
@ -483,6 +487,8 @@ tonNode.downloadKeyBlockProofLink block:tonNode.blockIdExt = tonNode.Data;
tonNode.getArchiveInfo masterchain_seqno:int = tonNode.ArchiveInfo;
tonNode.getShardArchiveInfo masterchain_seqno:int shard_prefix:tonNode.shardId = tonNode.ArchiveInfo;
tonNode.getArchiveSlice archive_id:long offset:long max_size:int = tonNode.Data;
tonNode.getOutMsgQueueProof dst_shard:tonNode.shardId blocks:(vector tonNode.blockIdExt)
limits:tonNode.importedMsgQueueLimits = tonNode.OutMsgQueueProof;
tonNode.getCapabilities = tonNode.Capabilities;
@ -725,6 +731,8 @@ engine.validator.perfTimerStats stats:(vector engine.validator.PerfTimerStatsByN
engine.validator.shardOutQueueSize size:long = engine.validator.ShardOutQueueSize;
engine.validator.exportedPrivateKeys encrypted_data:bytes = engine.validator.ExportedPrivateKeys;
---functions---
@ -755,6 +763,7 @@ engine.validator.delListeningPort ip:int port:int categories:(vector int) priori
engine.validator.delProxy out_ip:int out_port:int categories:(vector int) priority_categories:(vector int) = engine.validator.Success;
engine.validator.sign key_hash:int256 data:bytes = engine.validator.Signature;
engine.validator.exportAllPrivateKeys encryption_key:PublicKey = engine.validator.ExportedPrivateKeys;
engine.validator.getStats = engine.validator.Stats;
engine.validator.getConfig = engine.validator.JsonConfig;

BIN
tl/generate/scheme/ton_api.tlo
View file

Binary file not shown.

20
ton/ton-types.h
View file

@ -120,6 +120,26 @@ struct ShardIdFull {
char buffer[64];
return std::string{buffer, (unsigned)snprintf(buffer, 63, "(%d,%016llx)", workchain, (unsigned long long)shard)};
}
static td::Result<ShardIdFull> parse(td::Slice s) {
// Formats: (0,2000000000000000) (0:2000000000000000) 0,2000000000000000 0:2000000000000000
if (s.empty()) {
return td::Status::Error("empty string");
}
if (s[0] == '(' && s.back() == ')') {
s = s.substr(1, s.size() - 2);
}
auto sep = s.find(':');
if (sep == td::Slice::npos) {
sep = s.find(',');
}
if (sep == td::Slice::npos || s.size() - sep - 1 != 16) {
return td::Status::Error(PSTRING() << "invalid shard " << s);
}
ShardIdFull shard;
TRY_RESULT_ASSIGN(shard.workchain, td::to_integer_safe<td::int32>(s.substr(0, sep)));
TRY_RESULT_ASSIGN(shard.shard, td::hex_to_integer_safe<td::uint64>(s.substr(sep + 1)));
return shard;
}
};
struct AccountIdPrefixFull {

27
tonlib/tonlib/TonlibClient.cpp
View file

@ -5673,19 +5673,9 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_lookupBlock& reques
td::Status check_lookup_block_proof(lite_api_ptr<ton::lite_api::liteServer_lookupBlockResult>& result, int mode, ton::BlockId blkid, ton::BlockIdExt client_mc_blkid, td::uint64 lt, td::uint32 utime) {
try {
ton::BlockIdExt cur_id = ton::create_block_id(result->mc_block_id_);
try {
for (auto& link : result->shard_links_) {
ton::BlockIdExt prev_id = create_block_id(link->id_);
td::BufferSlice proof = std::move(link->proof_);
auto R = vm::std_boc_deserialize(proof);
if (R.is_error()) {
return TonlibError::InvalidBagOfCells("proof");
if (!cur_id.is_masterchain_ext()) {
return td::Status::Error("invalid response: mc block id is not from masterchain");
}
auto block_root = vm::MerkleProof::virtualize(R.move_as_ok(), 1);
if (cur_id.root_hash != block_root->get_hash().bits()) {
return td::Status::Error("invalid block hash in proof");
}
if (cur_id.is_masterchain()) {
if (client_mc_blkid != cur_id) {
auto state = block::check_extract_state_proof(client_mc_blkid, result->client_mc_state_proof_.as_slice(),
result->mc_block_proof_.as_slice());
@ -5703,6 +5693,19 @@ td::Status check_lookup_block_proof(lite_api_ptr<ton::lite_api::liteServer_looku
return td::Status::Error("couldn't check old mc block id");
}
}
try {
for (auto& link : result->shard_links_) {
ton::BlockIdExt prev_id = create_block_id(link->id_);
td::BufferSlice proof = std::move(link->proof_);
auto R = vm::std_boc_deserialize(proof);
if (R.is_error()) {
return TonlibError::InvalidBagOfCells("proof");
}
auto block_root = vm::MerkleProof::virtualize(R.move_as_ok(), 1);
if (cur_id.root_hash != block_root->get_hash().bits()) {
return td::Status::Error("invalid block hash in proof");
}
if (cur_id.is_masterchain()) {
block::gen::Block::Record blk;
block::gen::BlockExtra::Record extra;
block::gen::McBlockExtra::Record mc_extra;

105
validator-engine-console/validator-engine-console-query.cpp
View file

@ -35,6 +35,8 @@
#include "ton/ton-tl.hpp"
#include "td/utils/JsonBuilder.h"
#include "auto/tl/ton_api_json.h"
#include "keys/encryptor.h"
#include "td/utils/port/path.h"
#include "tl/tl_json.h"
#include <cctype>
@ -283,6 +285,66 @@ td::Status SignFileQuery::receive(td::BufferSlice data) {
return td::Status::OK();
}
td::Status ExportAllPrivateKeysQuery::run() {
TRY_RESULT_ASSIGN(directory_, tokenizer_.get_token<std::string>());
TRY_STATUS(tokenizer_.check_endl());
client_pk_ = ton::privkeys::Ed25519::random();
return td::Status::OK();
}
td::Status ExportAllPrivateKeysQuery::send() {
auto b = ton::create_serialize_tl_object<ton::ton_api::engine_validator_exportAllPrivateKeys>(
client_pk_.compute_public_key().tl());
td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise());
return td::Status::OK();
}
td::Status ExportAllPrivateKeysQuery::receive(td::BufferSlice data) {
TRY_RESULT_PREFIX(f, ton::fetch_tl_object<ton::ton_api::engine_validator_exportedPrivateKeys>(data.as_slice(), true),
"received incorrect answer: ");
// Private keys are encrypted using client-provided public key to avoid storing them in
// non-secure buffers (not td::SecureString)
TRY_RESULT_PREFIX(decryptor, client_pk_.create_decryptor(), "cannot create decryptor: ");
TRY_RESULT_PREFIX(keys_data, decryptor->decrypt(f->encrypted_data_.as_slice()), "cannot decrypt data: ");
SCOPE_EXIT {
keys_data.as_slice().fill_zero_secure();
};
td::Slice slice = keys_data.as_slice();
if (slice.size() < 32) {
return td::Status::Error("data is too small");
}
slice.remove_suffix(32);
std::vector<ton::PrivateKey> private_keys;
while (!slice.empty()) {
if (slice.size() < 4) {
return td::Status::Error("unexpected end of data");
}
td::uint32 size;
td::MutableSlice{reinterpret_cast<char *>(&size), 4}.copy_from(slice.substr(0, 4));
if (size > slice.size()) {
return td::Status::Error("unexpected end of data");
}
slice.remove_prefix(4);
TRY_RESULT_PREFIX(private_key, ton::PrivateKey::import(slice.substr(0, size)), "cannot parse private key: ");
if (!private_key.exportable()) {
return td::Status::Error("private key is not exportable");
}
private_keys.push_back(std::move(private_key));
slice.remove_prefix(size);
}
TRY_STATUS_PREFIX(td::mkpath(directory_ + "/"), "cannot create directory " + directory_ + ": ");
td::TerminalIO::out() << "exported " << private_keys.size() << " private keys" << "\n";
for (const ton::PrivateKey &private_key : private_keys) {
std::string hash_hex = private_key.compute_short_id().bits256_value().to_hex();
TRY_STATUS_PREFIX(td::write_file(directory_ + "/" + hash_hex, private_key.export_as_slice()),
"failed to write file: ");
td::TerminalIO::out() << "pubkey_hash " << hash_hex << "\n";
}
td::TerminalIO::out() << "written all files to " << directory_ << "\n";
return td::Status::OK();
}
td::Status AddAdnlAddrQuery::run() {
TRY_RESULT_ASSIGN(key_hash_, tokenizer_.get_token<ton::PublicKeyHash>());
TRY_RESULT_ASSIGN(category_, tokenizer_.get_token<td::uint32>());
@ -979,8 +1041,7 @@ td::Status ImportCertificateQuery::receive(td::BufferSlice data) {
td::Status SignShardOverlayCertificateQuery::run() {
TRY_RESULT_ASSIGN(wc_, tokenizer_.get_token<td::int32>());
TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token<td::int64>() );
TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token<ton::ShardIdFull>() );
TRY_RESULT_ASSIGN(key_, tokenizer_.get_token<ton::PublicKeyHash>());
TRY_RESULT_ASSIGN(expire_at_, tokenizer_.get_token<td::int32>());
TRY_RESULT_ASSIGN(max_size_, tokenizer_.get_token<td::uint32>());
@ -990,8 +1051,9 @@ td::Status SignShardOverlayCertificateQuery::run() {
}
td::Status SignShardOverlayCertificateQuery::send() {
auto b = ton::create_serialize_tl_object<ton::ton_api::engine_validator_signShardOverlayCertificate>
(wc_, shard_, ton::create_tl_object<ton::ton_api::engine_validator_keyHash>(key_.tl()), expire_at_, max_size_);
auto b = ton::create_serialize_tl_object<ton::ton_api::engine_validator_signShardOverlayCertificate>(
shard_.workchain, shard_.shard, ton::create_tl_object<ton::ton_api::engine_validator_keyHash>(key_.tl()),
expire_at_, max_size_);
td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise());
return td::Status::OK();
}
@ -1009,8 +1071,7 @@ td::Status SignShardOverlayCertificateQuery::receive(td::BufferSlice data) {
}
td::Status ImportShardOverlayCertificateQuery::run() {
TRY_RESULT_ASSIGN(wc_, tokenizer_.get_token<td::int32>());
TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token<td::int64>() );
TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token<ton::ShardIdFull>());
TRY_RESULT_ASSIGN(key_, tokenizer_.get_token<ton::PublicKeyHash>());
TRY_RESULT_ASSIGN(in_file_, tokenizer_.get_token<std::string>());
@ -1021,8 +1082,9 @@ td::Status ImportShardOverlayCertificateQuery::send() {
TRY_RESULT(data, td::read_file(in_file_));
TRY_RESULT_PREFIX(cert, ton::fetch_tl_object<ton::ton_api::overlay_Certificate>(data.as_slice(), true),
"incorrect certificate");
auto b = ton::create_serialize_tl_object<ton::ton_api::engine_validator_importShardOverlayCertificate>
(wc_, shard_, ton::create_tl_object<ton::ton_api::engine_validator_keyHash>(key_.tl()), std::move(cert));
auto b = ton::create_serialize_tl_object<ton::ton_api::engine_validator_importShardOverlayCertificate>(
shard_.workchain, shard_.shard, ton::create_tl_object<ton::ton_api::engine_validator_keyHash>(key_.tl()),
std::move(cert));
td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise());
return td::Status::OK();
}
@ -1111,14 +1173,12 @@ td::Status GetPerfTimerStatsJsonQuery::receive(td::BufferSlice data) {
}
td::Status GetShardOutQueueSizeQuery::run() {
TRY_RESULT_ASSIGN(block_id_.workchain, tokenizer_.get_token<int>());
TRY_RESULT_ASSIGN(block_id_.shard, tokenizer_.get_token<long long>());
TRY_RESULT(shard, tokenizer_.get_token<ton::ShardIdFull>());
block_id_.workchain = shard.workchain;
block_id_.shard = shard.shard;
TRY_RESULT_ASSIGN(block_id_.seqno, tokenizer_.get_token<int>());
if (!tokenizer_.endl()) {
ton::ShardIdFull dest;
TRY_RESULT_ASSIGN(dest.workchain, tokenizer_.get_token<int>());
TRY_RESULT_ASSIGN(dest.shard, tokenizer_.get_token<long long>());
dest_ = dest;
TRY_RESULT_ASSIGN(dest_, tokenizer_.get_token<ton::ShardIdFull>());
}
TRY_STATUS(tokenizer_.check_endl());
return td::Status::OK();
@ -1126,8 +1186,7 @@ td::Status GetShardOutQueueSizeQuery::run() {
td::Status GetShardOutQueueSizeQuery::send() {
auto b = ton::create_serialize_tl_object<ton::ton_api::engine_validator_getShardOutQueueSize>(
dest_ ? 1 : 0, ton::create_tl_block_id_simple(block_id_), dest_ ? dest_.value().workchain : 0,
dest_ ? dest_.value().shard : 0);
dest_.is_valid() ? 1 : 0, ton::create_tl_block_id_simple(block_id_), dest_.workchain, dest_.shard);
td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise());
return td::Status::OK();
}
@ -1501,14 +1560,13 @@ td::Status GetAdnlStatsQuery::receive(td::BufferSlice data) {
}
td::Status AddShardQuery::run() {
TRY_RESULT_ASSIGN(wc_, tokenizer_.get_token<td::int32>());
TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token<td::int64>());
TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token<ton::ShardIdFull>());
TRY_STATUS(tokenizer_.check_endl());
return td::Status::OK();
}
td::Status AddShardQuery::send() {
auto b = ton::create_serialize_tl_object<ton::ton_api::engine_validator_addShard>(
ton::create_tl_shard_id(ton::ShardIdFull(wc_, shard_)));
auto b = ton::create_serialize_tl_object<ton::ton_api::engine_validator_addShard>(ton::create_tl_shard_id(shard_));
td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise());
return td::Status::OK();
}
@ -1521,14 +1579,13 @@ td::Status AddShardQuery::receive(td::BufferSlice data) {
}
td::Status DelShardQuery::run() {
TRY_RESULT_ASSIGN(wc_, tokenizer_.get_token<td::int32>());
TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token<td::int64>());
TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token<ton::ShardIdFull>());
TRY_STATUS(tokenizer_.check_endl());
return td::Status::OK();
}
td::Status DelShardQuery::send() {
auto b = ton::create_serialize_tl_object<ton::ton_api::engine_validator_delShard>(
ton::create_tl_shard_id(ton::ShardIdFull(wc_, shard_)));
auto b = ton::create_serialize_tl_object<ton::ton_api::engine_validator_delShard>(ton::create_tl_shard_id(shard_));
td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise());
return td::Status::OK();
}

281
validator-engine-console/validator-engine-console-query.h
View file

@ -36,6 +36,7 @@
#include "ton/ton-types.h"
#include "keys/keys.hpp"
#include "td/utils/base64.h"
class ValidatorEngineConsole;
@ -95,27 +96,25 @@ inline td::Result<td::SharedSlice> Tokenizer::get_token() {
}
template <>
inline td::Result<ton::PublicKeyHash> Tokenizer::get_token() {
TRY_RESULT(S, get_raw_token());
TRY_RESULT(F, td::hex_decode(S));
if (F.size() == 32) {
return ton::PublicKeyHash{td::Slice{F}};
inline td::Result<td::Bits256> Tokenizer::get_token() {
TRY_RESULT(word, get_raw_token());
std::string data;
if (word.size() == 64) {
TRY_RESULT_ASSIGN(data, td::hex_decode(word));
} else if (word.size() == 44) {
TRY_RESULT_ASSIGN(data, td::base64_decode(word));
} else {
return td::Status::Error("cannot parse keyhash: bad length");
}
td::Bits256 v;
v.as_slice().copy_from(data);
return v;
}
template <>
inline td::Result<td::Bits256> Tokenizer::get_token() {
TRY_RESULT(S, get_raw_token());
TRY_RESULT(F, td::hex_decode(S));
if (F.size() == 32) {
td::Bits256 v;
v.as_slice().copy_from(F);
return v;
} else {
return td::Status::Error("cannot parse keyhash: bad length");
}
inline td::Result<ton::PublicKeyHash> Tokenizer::get_token() {
TRY_RESULT(x, get_token<td::Bits256>());
return ton::PublicKeyHash{x};
}
template <>
@ -146,6 +145,18 @@ inline td::Result<std::vector<T>> Tokenizer::get_token_vector() {
}
}
template <>
inline td::Result<ton::ShardIdFull> Tokenizer::get_token() {
TRY_RESULT(word, get_raw_token());
auto r_wc = td::to_integer_safe<ton::WorkchainId>(word);
if (r_wc.is_ok()) {
TRY_RESULT_ASSIGN(word, get_raw_token());
TRY_RESULT(shard, td::to_integer_safe<ton::ShardId>(word));
return ton::ShardIdFull{r_wc.move_as_ok(), shard};
}
return ton::ShardIdFull::parse(word);
}
class QueryRunner {
public:
virtual ~QueryRunner() = default;
@ -222,10 +233,10 @@ class GetTimeQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice R) override;
static std::string get_name() {
return "gettime";
return "get-time";
}
static std::string get_help() {
return "gettime\tshows current server unixtime";
return "get-time\tshows current server unixtime";
}
std::string name() const override {
return get_name();
@ -287,10 +298,10 @@ class NewKeyQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice R) override;
static std::string get_name() {
return "newkey";
return "new-key";
}
static std::string get_help() {
return "newkey\tgenerates new key pair on server";
return "new-key\tgenerates new key pair on server";
}
std::string name() const override {
return get_name();
@ -308,10 +319,10 @@ class ImportPrivateKeyFileQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice R) override;
static std::string get_name() {
return "importf";
return "import-f";
}
static std::string get_help() {
return "importf <filename>\timport private key";
return "import-f <filename>\timport private key";
}
std::string name() const override {
return get_name();
@ -330,10 +341,10 @@ class ExportPublicKeyQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice R) override;
static std::string get_name() {
return "exportpub";
return "export-pub";
}
static std::string get_help() {
return "exportpub <keyhash>\texports public key by key hash";
return "export-pub <keyhash>\texports public key by key hash";
}
std::string name() const override {
return get_name();
@ -352,10 +363,10 @@ class ExportPublicKeyFileQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice R) override;
static std::string get_name() {
return "exportpubf";
return "export-pubf";
}
static std::string get_help() {
return "exportpubf <keyhash> <filename>\texports public key by key hash";
return "export-pub-f <keyhash> <filename>\texports public key by key hash";
}
std::string name() const override {
return get_name();
@ -398,10 +409,10 @@ class SignFileQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "signf";
return "sign-f";
}
static std::string get_help() {
return "signf <keyhash> <infile> <outfile>\tsigns bytestring with privkey";
return "sign-f <keyhash> <infile> <outfile>\tsigns bytestring with privkey";
}
std::string name() const override {
return get_name();
@ -413,6 +424,30 @@ class SignFileQuery : public Query {
std::string out_file_;
};
class ExportAllPrivateKeysQuery : public Query {
public:
ExportAllPrivateKeysQuery(td::actor::ActorId<ValidatorEngineConsole> console, Tokenizer tokenizer)
: Query(console, std::move(tokenizer)) {
}
td::Status run() override;
td::Status send() override;
td::Status receive(td::BufferSlice R) override;
static std::string get_name() {
return "export-all-private-keys";
}
static std::string get_help() {
return "export-all-private-keys <directory>\texports all private keys from validator engine and stores them to "
"<directory>";
}
std::string name() const override {
return get_name();
}
private:
std::string directory_;
ton::PrivateKey client_pk_;
};
class AddAdnlAddrQuery : public Query {
public:
AddAdnlAddrQuery(td::actor::ActorId<ValidatorEngineConsole> console, Tokenizer tokenizer)
@ -422,10 +457,10 @@ class AddAdnlAddrQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "addadnl";
return "add-adnl";
}
static std::string get_help() {
return "addadnl <keyhash> <category>\tuse key as ADNL addr";
return "add-adnl <keyhash> <category>\tuse key as ADNL addr";
}
std::string name() const override {
return get_name();
@ -445,10 +480,10 @@ class AddDhtIdQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "adddht";
return "add-dht";
}
static std::string get_help() {
return "adddht <keyhash>\tcreate DHT node with specified ADNL addr";
return "add-dht <keyhash>\tcreate DHT node with specified ADNL addr";
}
std::string name() const override {
return get_name();
@ -467,10 +502,10 @@ class AddValidatorPermanentKeyQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "addpermkey";
return "add-perm-key";
}
static std::string get_help() {
return "addpermkey <keyhash> <election-date> <expire-at>\tadd validator permanent key";
return "add-perm-key <keyhash> <election-date> <expire-at>\tadd validator permanent key";
}
std::string name() const override {
return get_name();
@ -491,10 +526,10 @@ class AddValidatorTempKeyQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "addtempkey";
return "add-temp-key";
}
static std::string get_help() {
return "addtempkey <permkeyhash> <keyhash> <expireat>\tadd validator temp key";
return "add-temp-key <permkeyhash> <keyhash> <expireat>\tadd validator temp key";
}
std::string name() const override {
return get_name();
@ -515,10 +550,10 @@ class AddValidatorAdnlAddrQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "addvalidatoraddr";
return "add-validator-addr";
}
static std::string get_help() {
return "addvalidatoraddr <permkeyhash> <keyhash> <expireat>\tadd validator ADNL addr";
return "add-validator-addr <permkeyhash> <keyhash> <expireat>\tadd validator ADNL addr";
}
std::string name() const override {
return get_name();
@ -539,10 +574,10 @@ class ChangeFullNodeAdnlAddrQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "changefullnodeaddr";
return "change-full-node-addr";
}
static std::string get_help() {
return "changefullnodeaddr <keyhash>\tchanges fullnode ADNL address";
return "change-full-node-addr <keyhash>\tchanges fullnode ADNL address";
}
std::string name() const override {
return get_name();
@ -561,10 +596,10 @@ class AddLiteServerQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "addliteserver";
return "add-liteserver";
}
static std::string get_help() {
return "addliteserver <port> <keyhash>\tadd liteserver";
return "add-liteserver <port> <keyhash>\tadd liteserver";
}
std::string name() const override {
return get_name();
@ -584,10 +619,10 @@ class DelAdnlAddrQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "deladnl";
return "del-adnl";
}
static std::string get_help() {
return "deladnl <keyhash>\tdel unused ADNL addr";
return "del-adnl <keyhash>\tdel unused ADNL addr";
}
std::string name() const override {
return get_name();
@ -606,10 +641,10 @@ class DelDhtIdQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "deldht";
return "del-dht";
}
static std::string get_help() {
return "deldht <keyhash>\tdel unused DHT node";
return "del-dht <keyhash>\tdel unused DHT node";
}
std::string name() const override {
return get_name();
@ -628,10 +663,10 @@ class DelValidatorPermanentKeyQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "delpermkey";
return "del-perm-key";
}
static std::string get_help() {
return "delpermkey <keyhash>\tforce del unused validator permanent key";
return "del-perm-key <keyhash>\tforce del unused validator permanent key";
}
std::string name() const override {
return get_name();
@ -650,10 +685,10 @@ class DelValidatorTempKeyQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "deltempkey";
return "del-temp-key";
}
static std::string get_help() {
return "deltempkey <permkeyhash> <keyhash>\tforce del unused validator temp key";
return "del-temp-key <permkeyhash> <keyhash>\tforce del unused validator temp key";
}
std::string name() const override {
return get_name();
@ -673,10 +708,10 @@ class DelValidatorAdnlAddrQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "delvalidatoraddr";
return "del-validator-addr";
}
static std::string get_help() {
return "delvalidatoraddr <permkeyhash> <keyhash>\tforce del unused validator ADNL addr";
return "del-validator-addr <permkeyhash> <keyhash>\tforce del unused validator ADNL addr";
}
std::string name() const override {
return get_name();
@ -696,10 +731,10 @@ class GetConfigQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "getconfig";
return "get-config";
}
static std::string get_help() {
return "getconfig\tdownloads current config";
return "get-config\tdownloads current config";
}
std::string name() const override {
return get_name();
@ -717,10 +752,10 @@ class SetVerbosityQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "setverbosity";
return "set-verbosity";
}
static std::string get_help() {
return "setverbosity <value>\tchanges verbosity level";
return "set-verbosity <value>\tchanges verbosity level";
}
std::string name() const override {
return get_name();
@ -739,10 +774,10 @@ class GetStatsQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "getstats";
return "get-stats";
}
static std::string get_help() {
return "getstats\tprints stats";
return "get-stats\tprints stats";
}
std::string name() const override {
return get_name();
@ -783,10 +818,10 @@ class AddNetworkAddressQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "addaddr";
return "add-addr";
}
static std::string get_help() {
return "addaddr <ip> {cats...} {priocats...}\tadds ip address to address list";
return "add-addr <ip> {cats...} {priocats...}\tadds ip address to address list";
}
std::string name() const override {
return get_name();
@ -807,10 +842,10 @@ class AddNetworkProxyAddressQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "addproxyaddr";
return "add-proxy-addr";
}
static std::string get_help() {
return "addproxyaddr <inip> <outip> <id> <secret> {cats...} {priocats...}\tadds ip address to address list";
return "add-proxy-addr <inip> <outip> <id> <secret> {cats...} {priocats...}\tadds ip address to address list";
}
std::string name() const override {
return get_name();
@ -834,10 +869,10 @@ class CreateElectionBidQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "createelectionbid";
return "create-election-bid";
}
static std::string get_help() {
return "createelectionbid <date> <elector> <wallet> <fname>\tcreate election bid";
return "create-election-bid <date> <elector> <wallet> <fname>\tcreate election bid";
}
std::string name() const override {
return get_name();
@ -859,10 +894,10 @@ class CreateProposalVoteQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "createproposalvote";
return "create-proposal-vote";
}
static std::string get_help() {
return "createproposalvote <data> <fname>\tcreate proposal vote";
return "create-proposal-vote <data> <fname>\tcreate proposal vote";
}
std::string name() const override {
return get_name();
@ -882,10 +917,10 @@ class CreateComplaintVoteQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "createcomplaintvote";
return "create-complaint-vote";
}
static std::string get_help() {
return "createcomplaintvote <election-id> <data> <fname>\tcreate proposal vote";
return "create-complaint-vote <election-id> <data> <fname>\tcreate proposal vote";
}
std::string name() const override {
return get_name();
@ -906,10 +941,10 @@ class CheckDhtServersQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "checkdht";
return "check-dht";
}
static std::string get_help() {
return "checkdht <adnlid>\tchecks, which root DHT servers are accessible from this ADNL addr";
return "check-dht <adnlid>\tchecks, which root DHT servers are accessible from this ADNL addr";
}
std::string name() const override {
return get_name();
@ -928,10 +963,10 @@ class GetOverlaysStatsQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "getoverlaysstats";
return "get-overlays-stats";
}
static std::string get_help() {
return "getoverlaysstats\tgets stats for all overlays";
return "get-overlays-stats\tgets stats for all overlays";
}
std::string name() const override {
return get_name();
@ -947,10 +982,10 @@ class GetOverlaysStatsJsonQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "getoverlaysstatsjson";
return "get-overlays-stats-json";
}
static std::string get_help() {
return "getoverlaysstatsjson <outfile>\tgets stats for all overlays and writes to json file";
return "get-overlays-stats-json <outfile>\tgets stats for all overlays and writes to json file";
}
std::string name() const override {
return get_name();
@ -969,10 +1004,11 @@ class SignCertificateQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "signcert";
return "sign-cert";
}
static std::string get_help() {
return "signcert <overlayid> <adnlid> <expireat> <maxsize> <signwith> <outfile>\tsign overlay certificate by <signwith> key";
return "sign-cert <overlayid> <adnlid> <expireat> <maxsize> <signwith> <outfile>\tsign overlay certificate by "
"<signwith> key";
}
std::string name() const override {
return get_name();
@ -1005,10 +1041,10 @@ class ImportCertificateQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "importcert";
return "import-cert";
}
static std::string get_help() {
return "importcert <overlayid> <adnlid> <key> <certfile>\timport overlay certificate for specific key";
return "import-cert <overlayid> <adnlid> <key> <certfile>\timport overlay certificate for specific key";
}
std::string name() const override {
return get_name();
@ -1030,10 +1066,11 @@ class SignShardOverlayCertificateQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "signshardoverlaycert";
return "sign-shard-overlay-cert";
}
static std::string get_help() {
return "signshardoverlaycert <workchain> <shardprefix> <key> <expireat> <maxsize> <outfile>\tsign certificate for <key> in currently active shard overlay";
return "sign-shard-overlay-cert <wc>:<shard> <key> <expireat> <maxsize> <outfile>\tsign certificate "
"for <key> in currently active shard overlay";
}
std::string name() const override {
return get_name();
@ -1041,8 +1078,7 @@ class SignShardOverlayCertificateQuery : public Query {
private:
td::int32 wc_;
td::int64 shard_;
ton::ShardIdFull shard_;
td::int32 expire_at_;
ton::PublicKeyHash key_;
td::uint32 max_size_;
@ -1059,10 +1095,11 @@ class ImportShardOverlayCertificateQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "importshardoverlaycert";
return "import-shard-overlay-cert";
}
static std::string get_help() {
return "importshardoverlaycert <workchain> <shardprefix> <key> <certfile>\timport certificate for <key> in currently active shard overlay";
return "import-shard-overlay-cert <wc>:<shard> <key> <certfile>\timport certificate for <key> in "
"currently active shard overlay";
}
std::string name() const override {
return get_name();
@ -1070,8 +1107,7 @@ class ImportShardOverlayCertificateQuery : public Query {
private:
td::int32 wc_;
td::int64 shard_;
ton::ShardIdFull shard_;
ton::PublicKeyHash key_;
std::string in_file_;
};
@ -1085,10 +1121,10 @@ class GetActorStatsQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "getactorstats";
return "get-actor-stats";
}
static std::string get_help() {
return "getactorstats [<outfile>]\tget actor stats and print it either in stdout or in <outfile>";
return "get-actor-stats [<outfile>]\tget actor stats and print it either in stdout or in <outfile>";
}
std::string name() const override {
return get_name();
@ -1107,10 +1143,11 @@ class GetPerfTimerStatsJsonQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "getperftimerstatsjson";
return "get-perf-timer-stats-json";
}
static std::string get_help() {
return "getperftimerstatsjson <outfile>\tgets min, average and max event processing time for last 60, 300 and 3600 seconds and writes to json file";
return "get-perf-timer-stats-json <outfile>\tgets min, average and max event processing time for last 60, 300 and "
"3600 seconds and writes to json file";
}
std::string name() const override {
return get_name();
@ -1129,10 +1166,10 @@ class GetShardOutQueueSizeQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "getshardoutqueuesize";
return "get-shard-out-queue-size";
}
static std::string get_help() {
return "getshardoutqueuesize <wc> <shard> <seqno> [<dest_wc> <dest_shard>]\treturns number of messages in the "
return "get-shard-out-queue-size <wc>:<shard> <seqno> [<dest_wc>:<dest_shard>]\treturns number of messages in the "
"queue of the given shard. Destination shard is optional.";
}
std::string name() const override {
@ -1141,7 +1178,7 @@ class GetShardOutQueueSizeQuery : public Query {
private:
ton::BlockId block_id_;
td::optional<ton::ShardIdFull> dest_;
ton::ShardIdFull dest_ = ton::ShardIdFull{ton::workchainInvalid};
};
class SetExtMessagesBroadcastDisabledQuery : public Query {
@ -1153,11 +1190,11 @@ class SetExtMessagesBroadcastDisabledQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "setextmessagesbroadcastdisabled";
return "set-ext-messages-broadcast-disabled";
}
static std::string get_help() {
return "setextmessagesbroadcastdisabled <value>\tdisable broadcasting and rebroadcasting ext messages; value is 0 "
"or 1.";
return "set-ext-messages-broadcast-disabled <value>\tdisable broadcasting and rebroadcasting ext messages; value "
"is 0 or 1.";
}
std::string name() const override {
return get_name();
@ -1176,10 +1213,10 @@ class AddCustomOverlayQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "addcustomoverlay";
return "add-custom-overlay";
}
static std::string get_help() {
return "addcustomoverlay <filename>\tadd custom overlay with config from file <filename>";
return "add-custom-overlay <filename>\tadd custom overlay with config from file <filename>";
}
std::string name() const override {
return get_name();
@ -1198,10 +1235,10 @@ class DelCustomOverlayQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "delcustomoverlay";
return "del-custom-overlay";
}
static std::string get_help() {
return "delcustomoverlay <name>\tdelete custom overlay with name <name>";
return "del-custom-overlay <name>\tdelete custom overlay with name <name>";
}
std::string name() const override {
return get_name();
@ -1220,10 +1257,10 @@ class ShowCustomOverlaysQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "showcustomoverlays";
return "show-custom-overlays";
}
static std::string get_help() {
return "showcustomoverlays\tshow all custom overlays";
return "show-custom-overlays\tshow all custom overlays";
}
std::string name() const override {
return get_name();
@ -1239,10 +1276,10 @@ class SetStateSerializerEnabledQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "setstateserializerenabled";
return "set-state-serializer-enabled";
}
static std::string get_help() {
return "setstateserializerenabled <value>\tdisable or enable persistent state serializer; value is 0 or 1";
return "set-state-serializer-enabled <value>\tdisable or enable persistent state serializer; value is 0 or 1";
}
std::string name() const override {
return get_name();
@ -1261,10 +1298,10 @@ class SetCollatorOptionsJsonQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "setcollatoroptionsjson";
return "set-collator-options-json";
}
static std::string get_help() {
return "setcollatoroptionsjson <filename>\tset collator options from file <filename>";
return "set-collator-options-json <filename>\tset collator options from file <filename>";
}
std::string name() const override {
return get_name();
@ -1283,10 +1320,10 @@ class ResetCollatorOptionsQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "resetcollatoroptions";
return "reset-collator-options";
}
static std::string get_help() {
return "resetcollatoroptions\tset collator options to default values";
return "reset-collator-options\tset collator options to default values";
}
std::string name() const override {
return get_name();
@ -1302,10 +1339,10 @@ class GetCollatorOptionsJsonQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "getcollatoroptionsjson";
return "get-collator-options-json";
}
static std::string get_help() {
return "getcollatoroptionsjson <filename>\tsave current collator options to file <filename>";
return "get-collator-options-json <filename>\tsave current collator options to file <filename>";
}
std::string name() const override {
return get_name();
@ -1324,11 +1361,11 @@ class GetAdnlStatsJsonQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "getadnlstatsjson";
return "get-adnl-stats-json";
}
static std::string get_help() {
return "getadnlstatsjson <filename> [all]\tsave adnl stats to <filename>. all - returns all peers (default - only "
"peers with traffic in the last 10 minutes)";
return "get-adnl-stats-json <filename> [all]\tsave adnl stats to <filename>. all - returns all peers (default - "
"only peers with traffic in the last 10 minutes)";
}
std::string name() const override {
return get_name();
@ -1348,11 +1385,11 @@ class GetAdnlStatsQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "getadnlstats";
return "get-adnl-stats";
}
static std::string get_help() {
return "getadnlstats [all]\tdisplay adnl stats. all - returns all peers (default - only peers with traffic in the "
"last 10 minutes)";
return "get-adnl-stats [all]\tdisplay adnl stats. all - returns all peers (default - only peers with traffic in "
"the last 10 minutes)";
}
std::string name() const override {
return get_name();
@ -1372,18 +1409,17 @@ class AddShardQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "addshard";
return "add-shard";
}
static std::string get_help() {
return "addshard <workchain> <shard>\tstart monitoring shard";
return "add-shard <wc>:<shard>\tstart monitoring shard";
}
std::string name() const override {
return get_name();
}
private:
td::int32 wc_;
td::int64 shard_;
ton::ShardIdFull shard_;
};
class DelShardQuery : public Query {
@ -1395,16 +1431,15 @@ class DelShardQuery : public Query {
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "delshard";
return "del-shard";
}
static std::string get_help() {
return "delshard <workchain> <shard>\tstop monitoring shard";
return "del-shard <wc>:<shard>\tstop monitoring shard";
}
std::string name() const override {
return get_name();
}
private:
td::int32 wc_;
td::int64 shard_;
ton::ShardIdFull shard_;
};

11
validator-engine-console/validator-engine-console.cpp
View file

@ -112,6 +112,7 @@ void ValidatorEngineConsole::run() {
add_query_runner(std::make_unique<QueryRunnerImpl<ExportPublicKeyFileQuery>>());
add_query_runner(std::make_unique<QueryRunnerImpl<SignQuery>>());
add_query_runner(std::make_unique<QueryRunnerImpl<SignFileQuery>>());
add_query_runner(std::make_unique<QueryRunnerImpl<ExportAllPrivateKeysQuery>>());
add_query_runner(std::make_unique<QueryRunnerImpl<AddAdnlAddrQuery>>());
add_query_runner(std::make_unique<QueryRunnerImpl<AddDhtIdQuery>>());
add_query_runner(std::make_unique<QueryRunnerImpl<AddValidatorPermanentKeyQuery>>());
@ -205,9 +206,8 @@ void ValidatorEngineConsole::show_help(std::string command, td::Promise<td::Buff
td::TerminalIO::out() << cmd.second->help() << "\n";
}
} else {
auto it = query_runners_.find(command);
if (it != query_runners_.end()) {
td::TerminalIO::out() << it->second->help() << "\n";
if (auto query = get_query(command)) {
td::TerminalIO::out() << query->help() << "\n";
} else {
td::TerminalIO::out() << "unknown command '" << command << "'\n";
}
@ -231,10 +231,9 @@ void ValidatorEngineConsole::parse_line(td::BufferSlice data) {
}
auto name = tokenizer.get_token<std::string>().move_as_ok();
auto it = query_runners_.find(name);
if (it != query_runners_.end()) {
if (auto query = get_query(name)) {
running_queries_++;
it->second->run(actor_id(this), std::move(tokenizer));
query->run(actor_id(this), std::move(tokenizer));
} else {
td::TerminalIO::out() << "unknown command '" << name << "'\n";
}

14
validator-engine-console/validator-engine-console.h
View file

@ -57,9 +57,23 @@ class ValidatorEngineConsole : public td::actor::Actor {
std::unique_ptr<ton::adnl::AdnlExtClient::Callback> make_callback();
std::map<std::string, std::unique_ptr<QueryRunner>> query_runners_;
std::map<std::string, std::string> alternate_names_;
static std::string simplify_name(std::string name) {
std::erase_if(name, [](char c) { return c == '-'; });
return name;
}
void add_query_runner(std::unique_ptr<QueryRunner> runner) {
auto name = runner->name();
query_runners_[name] = std::move(runner);
alternate_names_[simplify_name(name)] = name;
}
QueryRunner* get_query(std::string name) {
auto it = alternate_names_.find(name);
if (it != alternate_names_.end()) {
name = it->second;
}
auto it2 = query_runners_.find(name);
return it2 == query_runners_.end() ? nullptr : it2->second.get();
}
public:

77
validator-engine/validator-engine.cpp
View file

@ -1503,17 +1503,6 @@ td::Status ValidatorEngine::load_global_config() {
h.push_back(b);
}
validator_options_.write().set_hardforks(std::move(h));
auto r_total_mem_stat = td::get_total_mem_stat();
if (r_total_mem_stat.is_error()) {
LOG(ERROR) << "Failed to get total RAM size: " << r_total_mem_stat.move_as_error();
} else {
td::uint64 total_ram = r_total_mem_stat.ok().total_ram;
LOG(WARNING) << "Total RAM = " << td::format::as_size(total_ram);
if (total_ram >= (90ULL << 30)) {
fast_state_serializer_enabled_ = true;
}
}
validator_options_.write().set_fast_state_serializer_enabled(fast_state_serializer_enabled_);
return td::Status::OK();
@ -3333,6 +3322,70 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_sign &que
std::move(query.data_), std::move(P));
}
void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_exportAllPrivateKeys &query,
td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm,
td::Promise<td::BufferSlice> promise) {
if (!(perm & ValidatorEnginePermissions::vep_unsafe)) {
promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized")));
return;
}
if (keyring_.empty()) {
promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started keyring")));
return;
}
ton::PublicKey client_pubkey = ton::PublicKey{query.encryption_key_};
if (!client_pubkey.is_ed25519()) {
promise.set_value(
create_control_query_error(td::Status::Error(ton::ErrorCode::protoviolation, "encryption key is not Ed25519")));
return;
}
td::actor::send_closure(
keyring_, &ton::keyring::Keyring::export_all_private_keys,
[promise = std::move(promise),
client_pubkey = std::move(client_pubkey)](td::Result<std::vector<ton::PrivateKey>> R) mutable {
if (R.is_error()) {
promise.set_value(create_control_query_error(R.move_as_error()));
return;
}
// Private keys are encrypted using client-provided public key to avoid storing them in
// non-secure buffers (not td::SecureString)
std::vector<td::SecureString> serialized_keys;
size_t data_size = 32;
for (const ton::PrivateKey &key : R.ok()) {
serialized_keys.push_back(key.export_as_slice());
data_size += serialized_keys.back().size() + 4;
}
td::SecureString data{data_size};
td::MutableSlice slice = data.as_mutable_slice();
for (const td::SecureString &s : serialized_keys) {
td::uint32 size = td::narrow_cast_safe<td::uint32>(s.size()).move_as_ok();
CHECK(slice.size() >= size + 4);
slice.copy_from(td::Slice{reinterpret_cast<const td::uint8 *>(&size), 4});
slice.remove_prefix(4);
slice.copy_from(s.as_slice());
slice.remove_prefix(s.size());
}
CHECK(slice.size() == 32);
td::Random::secure_bytes(slice);
auto r_encryptor = client_pubkey.create_encryptor();
if (r_encryptor.is_error()) {
promise.set_value(create_control_query_error(r_encryptor.move_as_error_prefix("cannot create encryptor: ")));
return;
}
auto encryptor = r_encryptor.move_as_ok();
auto r_encrypted = encryptor->encrypt(data.as_slice());
if (r_encryptor.is_error()) {
promise.set_value(create_control_query_error(r_encrypted.move_as_error_prefix("cannot encrypt data: ")));
return;
}
promise.set_value(ton::create_serialize_tl_object<ton::ton_api::engine_validator_exportedPrivateKeys>(
r_encrypted.move_as_ok()));
});
}
void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setVerbosity &query, td::BufferSlice data,
ton::PublicKeyHash src, td::uint32 perm, td::Promise<td::BufferSlice> promise) {
if (!(perm & ValidatorEnginePermissions::vep_default)) {
@ -4489,7 +4542,7 @@ int main(int argc, char *argv[]) {
});
p.add_option(
'\0', "fast-state-serializer",
"faster persistent state serializer, but requires more RAM (enabled automatically on machines with >= 90GB RAM)",
"faster persistent state serializer, but requires more RAM",
[&]() {
acts.push_back(
[&x]() { td::actor::send_closure(x, &ValidatorEngine::set_fast_state_serializer_enabled, true); });

2
validator-engine/validator-engine.hpp
View file

@ -475,6 +475,8 @@ class ValidatorEngine : public td::actor::Actor {
td::uint32 perm, td::Promise<td::BufferSlice> promise);
void run_control_query(ton::ton_api::engine_validator_sign &query, td::BufferSlice data, ton::PublicKeyHash src,
td::uint32 perm, td::Promise<td::BufferSlice> promise);
void run_control_query(ton::ton_api::engine_validator_exportAllPrivateKeys &query, td::BufferSlice data,
ton::PublicKeyHash src, td::uint32 perm, td::Promise<td::BufferSlice> promise);
void run_control_query(ton::ton_api::engine_validator_setVerbosity &query, td::BufferSlice data,
ton::PublicKeyHash src, td::uint32 perm, td::Promise<td::BufferSlice> promise);
void run_control_query(ton::ton_api::engine_validator_getStats &query, td::BufferSlice data, ton::PublicKeyHash src,

1
validator/CMakeLists.txt
View file

@ -46,6 +46,7 @@ set(VALIDATOR_HEADERS
interfaces/db.h
interfaces/external-message.h
interfaces/liteserver.h
interfaces/out-msg-queue-proof.h
interfaces/proof.h
interfaces/shard.h
interfaces/signature-set.h

98
validator/full-node-shard.cpp
View file

@ -37,6 +37,7 @@
#include "net/download-proof.hpp"
#include "net/get-next-key-blocks.hpp"
#include "net/download-archive-slice.hpp"
#include "impl/out-msg-queue-proof.hpp"
#include "td/utils/Random.h"
@ -669,6 +670,62 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod
query.offset_, query.max_size_, std::move(promise));
}
void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getOutMsgQueueProof &query,
td::Promise<td::BufferSlice> promise) {
std::vector<BlockIdExt> blocks;
for (const auto &x : query.blocks_) {
BlockIdExt id = create_block_id(x);
if (!id.is_valid_ext()) {
promise.set_error(td::Status::Error("invalid block_id"));
return;
}
if (!shard_is_ancestor(shard_, id.shard_full())) {
promise.set_error(td::Status::Error("query in wrong overlay"));
return;
}
blocks.push_back(create_block_id(x));
}
ShardIdFull dst_shard = create_shard_id(query.dst_shard_);
if (!dst_shard.is_valid_ext()) {
promise.set_error(td::Status::Error("invalid shard"));
return;
}
block::ImportedMsgQueueLimits limits{(td::uint32)query.limits_->max_bytes_, (td::uint32)query.limits_->max_msgs_};
if (limits.max_msgs > 512) {
promise.set_error(td::Status::Error("max_msgs is too big"));
return;
}
if (limits.max_bytes > (1 << 21)) {
promise.set_error(td::Status::Error("max_bytes is too big"));
return;
}
FLOG(DEBUG) {
sb << "Got query getOutMsgQueueProof to shard " << dst_shard.to_str() << " from blocks";
for (const BlockIdExt &id : blocks) {
sb << " " << id.id.to_str();
}
sb << " from " << src;
};
td::actor::send_closure(
full_node_, &FullNode::get_out_msg_queue_query_token,
[=, manager = validator_manager_, blocks = std::move(blocks),
promise = std::move(promise)](td::Result<std::unique_ptr<ActionToken>> R) mutable {
TRY_RESULT_PROMISE(promise, token, std::move(R));
auto P =
td::PromiseCreator::lambda([promise = std::move(promise), token = std::move(token)](
td::Result<tl_object_ptr<ton_api::tonNode_outMsgQueueProof>> R) mutable {
if (R.is_error()) {
promise.set_result(create_serialize_tl_object<ton_api::tonNode_outMsgQueueProofEmpty>());
} else {
promise.set_result(serialize_tl_object(R.move_as_ok(), true));
}
});
td::actor::create_actor<BuildOutMsgQueueProof>("buildqueueproof", dst_shard, std::move(blocks), limits, manager,
std::move(P))
.release();
});
}
void FullNodeShardImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query,
td::Promise<td::BufferSlice> promise) {
if (!active_) {
@ -944,6 +1001,47 @@ void FullNodeShardImpl::download_archive(BlockSeqno masterchain_seqno, ShardIdFu
.release();
}
void FullNodeShardImpl::download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector<BlockIdExt> blocks,
block::ImportedMsgQueueLimits limits, td::Timestamp timeout,
td::Promise<std::vector<td::Ref<OutMsgQueueProof>>> promise) {
// TODO: maybe more complex download (like other requests here)
auto &b = choose_neighbour();
if (b.adnl_id == adnl::AdnlNodeIdShort::zero()) {
promise.set_error(td::Status::Error(ErrorCode::notready, "no nodes"));
return;
}
std::vector<tl_object_ptr<ton_api::tonNode_blockIdExt>> blocks_tl;
for (const BlockIdExt &id : blocks) {
blocks_tl.push_back(create_tl_block_id(id));
}
td::BufferSlice query = create_serialize_tl_object<ton_api::tonNode_getOutMsgQueueProof>(
create_tl_shard_id(dst_shard), std::move(blocks_tl),
create_tl_object<ton_api::tonNode_importedMsgQueueLimits>(limits.max_bytes, limits.max_msgs));
auto P = td::PromiseCreator::lambda(
[=, promise = std::move(promise), blocks = std::move(blocks)](td::Result<td::BufferSlice> R) mutable {
if (R.is_error()) {
promise.set_result(R.move_as_error());
return;
}
TRY_RESULT_PROMISE(promise, f, fetch_tl_object<ton_api::tonNode_OutMsgQueueProof>(R.move_as_ok(), true));
ton_api::downcast_call(
*f, td::overloaded(
[&](ton_api::tonNode_outMsgQueueProofEmpty &x) {
promise.set_error(td::Status::Error("node doesn't have this block"));
},
[&](ton_api::tonNode_outMsgQueueProof &x) {
delay_action(
[=, promise = std::move(promise), blocks = std::move(blocks), x = std::move(x)]() mutable {
promise.set_result(OutMsgQueueProof::fetch(dst_shard, blocks, limits, x));
},
td::Timestamp::now());
}));
});
td::actor::send_closure(overlays_, &overlay::Overlays::send_query_via, b.adnl_id, adnl_id_, overlay_id_,
"get_msg_queue", std::move(P), timeout, std::move(query), 1 << 22, rldp_);
}
void FullNodeShardImpl::set_handle(BlockHandle handle, td::Promise<td::Unit> promise) {
CHECK(!handle_);
handle_ = std::move(handle);

3
validator/full-node-shard.h
View file

@ -66,6 +66,9 @@ class FullNodeShard : public td::actor::Actor {
td::Promise<std::vector<BlockIdExt>> promise) = 0;
virtual void download_archive(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir,
td::Timestamp timeout, td::Promise<std::string> promise) = 0;
virtual void download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector<BlockIdExt> blocks,
block::ImportedMsgQueueLimits limits, td::Timestamp timeout,
td::Promise<std::vector<td::Ref<OutMsgQueueProof>>> promise) = 0;
virtual void set_handle(BlockHandle handle, td::Promise<td::Unit> promise) = 0;

7
validator/full-node-shard.hpp
View file

@ -139,8 +139,8 @@ class FullNodeShardImpl : public FullNodeShard {
td::Promise<td::BufferSlice> promise);
void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveSlice &query,
td::Promise<td::BufferSlice> promise);
// void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_prepareNextKeyBlockProof &query,
// td::Promise<td::BufferSlice> promise);
void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getOutMsgQueueProof &query,
td::Promise<td::BufferSlice> promise);
void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise<td::BufferSlice> promise);
void receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice data);
@ -183,6 +183,9 @@ class FullNodeShardImpl : public FullNodeShard {
td::Promise<std::vector<BlockIdExt>> promise) override;
void download_archive(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir,
td::Timestamp timeout, td::Promise<std::string> promise) override;
void download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector<BlockIdExt> blocks,
block::ImportedMsgQueueLimits limits, td::Timestamp timeout,
td::Promise<std::vector<td::Ref<OutMsgQueueProof>>> promise) override;
void set_handle(BlockHandle handle, td::Promise<td::Unit> promise) override;

30
validator/full-node.cpp
View file

@ -21,6 +21,7 @@
#include "td/actor/MultiPromise.h"
#include "full-node.h"
#include "common/delay.h"
#include "impl/out-msg-queue-proof.hpp"
#include "td/utils/Random.h"
#include "ton/ton-tl.hpp"
@ -430,6 +431,24 @@ void FullNodeImpl::download_archive(BlockSeqno masterchain_seqno, ShardIdFull sh
timeout, std::move(promise));
}
void FullNodeImpl::download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector<BlockIdExt> blocks,
block::ImportedMsgQueueLimits limits, td::Timestamp timeout,
td::Promise<std::vector<td::Ref<OutMsgQueueProof>>> promise) {
if (blocks.empty()) {
promise.set_value({});
return;
}
// All blocks are expected to have the same minsplit shard prefix
auto shard = get_shard(blocks[0].shard_full());
if (shard.empty()) {
VLOG(FULL_NODE_WARNING) << "dropping download msg queue query to unknown shard";
promise.set_error(td::Status::Error(ErrorCode::notready, "shard not ready"));
return;
}
td::actor::send_closure(shard, &FullNodeShard::download_out_msg_queue_proof, dst_shard, std::move(blocks), limits,
timeout, std::move(promise));
}
td::actor::ActorId<FullNodeShard> FullNodeImpl::get_shard(ShardIdFull shard) {
if (shard.is_masterchain()) {
return shards_[ShardIdFull{masterchainId}].actor.get();
@ -557,6 +576,11 @@ void FullNodeImpl::process_block_candidate_broadcast(BlockIdExt block_id, Catcha
std::move(data));
}
void FullNodeImpl::get_out_msg_queue_query_token(td::Promise<std::unique_ptr<ActionToken>> promise) {
td::actor::send_closure(out_msg_queue_query_token_manager_, &TokenManager::get_token, 1, 0, td::Timestamp::in(10.0),
std::move(promise));
}
void FullNodeImpl::set_validator_telemetry_filename(std::string value) {
validator_telemetry_filename_ = std::move(value);
update_validator_telemetry_collector();
@ -645,6 +669,12 @@ void FullNodeImpl::start_up() {
td::actor::send_closure(id_, &FullNodeImpl::download_archive, masterchain_seqno, shard_prefix, std::move(tmp_dir),
timeout, std::move(promise));
}
void download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector<BlockIdExt> blocks,
block::ImportedMsgQueueLimits limits, td::Timestamp timeout,
td::Promise<std::vector<td::Ref<OutMsgQueueProof>>> promise) override {
td::actor::send_closure(id_, &FullNodeImpl::download_out_msg_queue_proof, dst_shard, std::move(blocks), limits,
timeout, std::move(promise));
}
void new_key_block(BlockHandle handle) override {
td::actor::send_closure(id_, &FullNodeImpl::new_key_block, std::move(handle));

1
validator/full-node.h
View file

@ -91,6 +91,7 @@ class FullNode : public td::actor::Actor {
virtual void process_block_broadcast(BlockBroadcast broadcast) = 0;
virtual void process_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno,
td::uint32 validator_set_hash, td::BufferSlice data) = 0;
virtual void get_out_msg_queue_query_token(td::Promise<std::unique_ptr<ActionToken>> promise) = 0;
virtual void set_validator_telemetry_filename(std::string value) = 0;

8
validator/full-node.hpp
View file

@ -28,6 +28,7 @@
#include <map>
#include <set>
#include <queue>
#include <token-manager.h>
namespace ton {
@ -79,6 +80,9 @@ class FullNodeImpl : public FullNode {
void get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeout, td::Promise<std::vector<BlockIdExt>> promise);
void download_archive(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir,
td::Timestamp timeout, td::Promise<std::string> promise);
void download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector<BlockIdExt> blocks,
block::ImportedMsgQueueLimits limits, td::Timestamp timeout,
td::Promise<std::vector<td::Ref<OutMsgQueueProof>>> promise);
void got_key_block_config(td::Ref<ConfigHolder> config);
void new_key_block(BlockHandle handle);
@ -87,6 +91,7 @@ class FullNodeImpl : public FullNode {
void process_block_broadcast(BlockBroadcast broadcast) override;
void process_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash,
td::BufferSlice data) override;
void get_out_msg_queue_query_token(td::Promise<std::unique_ptr<ActionToken>> promise) override;
void set_validator_telemetry_filename(std::string value) override;
@ -160,6 +165,9 @@ class FullNodeImpl : public FullNode {
PublicKeyHash validator_telemetry_collector_key_ = PublicKeyHash::zero();
void update_validator_telemetry_collector();
td::actor::ActorOwn<TokenManager> out_msg_queue_query_token_manager_ =
td::actor::create_actor<TokenManager>("tokens", /* max_tokens = */ 1);
};
} // namespace fullnode

2
validator/impl/CMakeLists.txt
View file

@ -16,6 +16,7 @@ set(TON_VALIDATOR_SOURCE
ihr-message.cpp
liteserver.cpp
message-queue.cpp
out-msg-queue-proof.cpp
proof.cpp
shard.cpp
signature-set.cpp
@ -35,6 +36,7 @@ set(TON_VALIDATOR_SOURCE
liteserver.hpp
liteserver-cache.hpp
message-queue.hpp
out-msg-queue-proof.hpp
proof.hpp
shard.hpp
signature-set.hpp

294
validator/impl/out-msg-queue-proof.cpp Normal file
View file

@ -0,0 +1,294 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
*/
#include "out-msg-queue-proof.hpp"
#include "interfaces/proof.h"
#include "shard.hpp"
#include "vm/cells/MerkleProof.h"
#include "common/delay.h"
#include "interfaces/validator-manager.h"
#include "block/block-parse.h"
#include "block/block-auto.h"
#include "output-queue-merger.h"
namespace ton {
namespace validator {
static td::Status check_no_prunned(const Ref<vm::Cell>& cell) {
if (cell.is_null()) {
return td::Status::OK();
}
TRY_RESULT(loaded_cell, cell->load_cell());
if (loaded_cell.data_cell->get_level() > 0) {
return td::Status::Error("prunned branch");
}
return td::Status::OK();
}
static td::Status check_no_prunned(const vm::CellSlice& cs) {
for (unsigned i = 0; i < cs.size_refs(); ++i) {
TRY_STATUS(check_no_prunned(cs.prefetch_ref(i)));
}
return td::Status::OK();
}
static td::Result<std::vector<td::int32>> process_queue(
ShardIdFull dst_shard, std::vector<std::pair<BlockIdExt, block::gen::OutMsgQueueInfo::Record>> blocks,
block::ImportedMsgQueueLimits limits) {
td::uint64 estimated_proof_size = 0;
td::HashSet<vm::Cell::Hash> visited;
std::function<void(const vm::CellSlice&)> dfs_cs;
auto dfs = [&](const Ref<vm::Cell>& cell) {
if (cell.is_null() || !visited.insert(cell->get_hash()).second) {
return;
}
dfs_cs(vm::CellSlice(vm::NoVm(), cell));
};
dfs_cs = [&](const vm::CellSlice& cs) {
// Based on BlockLimitStatus::estimate_block_size
estimated_proof_size += 12 + (cs.size() + 7) / 8 + cs.size_refs() * 3;
for (unsigned i = 0; i < cs.size_refs(); i++) {
dfs(cs.prefetch_ref(i));
}
};
std::vector<block::OutputQueueMerger::Neighbor> neighbors;
for (auto& b : blocks) {
TRY_STATUS_PREFIX(check_no_prunned(*b.second.proc_info), "invalid proc_info proof: ")
dfs_cs(*b.second.proc_info);
neighbors.emplace_back(b.first, b.second.out_queue->prefetch_ref());
}
block::OutputQueueMerger queue_merger{dst_shard, std::move(neighbors)};
std::vector<td::int32> msg_count(blocks.size());
td::int32 msg_count_total = 0;
bool limit_reached = false;
while (!queue_merger.is_eof()) {
auto kv = queue_merger.extract_cur();
queue_merger.next();
block::EnqueuedMsgDescr enq;
auto msg = kv->msg;
if (!enq.unpack(msg.write())) {
return td::Status::Error("cannot unpack EnqueuedMsgDescr");
}
if (limit_reached) {
break;
}
++msg_count[kv->source];
++msg_count_total;
dfs_cs(*kv->msg);
TRY_STATUS_PREFIX(check_no_prunned(*kv->msg), "invalid message proof: ")
if (estimated_proof_size >= limits.max_bytes || msg_count_total >= (long long)limits.max_msgs) {
limit_reached = true;
}
}
if (!limit_reached) {
std::fill(msg_count.begin(), msg_count.end(), -1);
}
return msg_count;
}
td::Result<tl_object_ptr<ton_api::tonNode_outMsgQueueProof>> OutMsgQueueProof::build(
ShardIdFull dst_shard, std::vector<OneBlock> blocks, block::ImportedMsgQueueLimits limits) {
if (!dst_shard.is_valid_ext()) {
return td::Status::Error("invalid shard");
}
if (blocks.empty()) {
return create_tl_object<ton_api::tonNode_outMsgQueueProof>(td::BufferSlice{}, td::BufferSlice{},
std::vector<td::int32>{});
}
std::vector<td::Ref<vm::Cell>> block_state_proofs;
for (auto& block : blocks) {
if (block.id.seqno() != 0) {
if (block.block_root.is_null()) {
return td::Status::Error("block is null");
}
TRY_RESULT(proof, create_block_state_proof(block.block_root));
block_state_proofs.push_back(std::move(proof));
}
if (!block::ShardConfig::is_neighbor(dst_shard, block.id.shard_full())) {
return td::Status::Error("shards are not neighbors");
}
}
TRY_RESULT(block_state_proof, vm::std_boc_serialize_multi(block_state_proofs));
vm::Dictionary states_dict_pure{32};
for (size_t i = 0; i < blocks.size(); ++i) {
if (blocks[i].state_root.is_null()) {
return td::Status::Error("state is null");
}
states_dict_pure.set_ref(td::BitArray<32>{(long long)i}, blocks[i].state_root);
}
vm::MerkleProofBuilder mpb{states_dict_pure.get_root_cell()};
vm::Dictionary states_dict{mpb.root(), 32};
std::vector<std::pair<BlockIdExt, block::gen::OutMsgQueueInfo::Record>> data(blocks.size());
for (size_t i = 0; i < blocks.size(); ++i) {
data[i].first = blocks[i].id;
TRY_RESULT(state, ShardStateQ::fetch(blocks[i].id, {}, states_dict.lookup_ref(td::BitArray<32>{(long long)i})));
TRY_RESULT(outq_descr, state->message_queue());
block::gen::OutMsgQueueInfo::Record qinfo;
if (!tlb::unpack_cell(outq_descr->root_cell(), data[i].second)) {
return td::Status::Error("invalid message queue");
}
}
TRY_RESULT(msg_count, process_queue(dst_shard, std::move(data), limits));
TRY_RESULT(proof, mpb.extract_proof());
vm::Dictionary states_dict_proof{vm::CellSlice{vm::NoVm(), proof}.prefetch_ref(), 32};
std::vector<td::Ref<vm::Cell>> state_proofs;
for (size_t i = 0; i < blocks.size(); ++i) {
td::Ref<vm::Cell> proof_raw = states_dict_proof.lookup_ref(td::BitArray<32>{(long long)i});
CHECK(proof_raw.not_null());
state_proofs.push_back(vm::CellBuilder::create_merkle_proof(proof_raw));
}
TRY_RESULT(queue_proof, vm::std_boc_serialize_multi(state_proofs));
return create_tl_object<ton_api::tonNode_outMsgQueueProof>(std::move(queue_proof), std::move(block_state_proof),
std::move(msg_count));
}
td::Result<std::vector<td::Ref<OutMsgQueueProof>>> OutMsgQueueProof::fetch(ShardIdFull dst_shard,
std::vector<BlockIdExt> blocks,
block::ImportedMsgQueueLimits limits,
const ton_api::tonNode_outMsgQueueProof& f) {
try {
std::vector<td::Ref<OutMsgQueueProof>> res;
TRY_RESULT(queue_proofs, vm::std_boc_deserialize_multi(f.queue_proofs_, (int)blocks.size()));
TRY_RESULT(block_state_proofs, vm::std_boc_deserialize_multi(f.block_state_proofs_, (int)blocks.size()));
if (queue_proofs.size() != blocks.size()) {
return td::Status::Error("invalid size of queue_proofs");
}
if (f.msg_counts_.size() != blocks.size()) {
return td::Status::Error("invalid size of msg_counts");
}
size_t j = 0;
std::vector<std::pair<BlockIdExt, block::gen::OutMsgQueueInfo::Record>> data(blocks.size());
for (size_t i = 0; i < blocks.size(); ++i) {
td::Bits256 state_root_hash;
Ref<vm::Cell> block_state_proof = {};
if (blocks[i].seqno() == 0) {
state_root_hash = blocks[i].root_hash;
} else {
if (j == block_state_proofs.size()) {
return td::Status::Error("invalid size of block_state_proofs");
}
block_state_proof = block_state_proofs[j++];
TRY_RESULT_ASSIGN(state_root_hash, unpack_block_state_proof(blocks[i], block_state_proof));
}
auto state_root = vm::MerkleProof::virtualize(queue_proofs[i], 1);
if (state_root->get_hash().as_slice() != state_root_hash.as_slice()) {
return td::Status::Error("state root hash mismatch");
}
res.emplace_back(true, blocks[i], state_root, block_state_proof, f.msg_counts_[i]);
data[i].first = blocks[i];
TRY_RESULT(state, ShardStateQ::fetch(blocks[i], {}, state_root));
TRY_RESULT(outq_descr, state->message_queue());
block::gen::OutMsgQueueInfo::Record qinfo;
if (!tlb::unpack_cell(outq_descr->root_cell(), data[i].second)) {
return td::Status::Error("invalid message queue");
}
}
if (j != block_state_proofs.size()) {
return td::Status::Error("invalid size of block_state_proofs");
}
TRY_RESULT(msg_count, process_queue(dst_shard, std::move(data), limits));
if (msg_count != f.msg_counts_) {
return td::Status::Error("incorrect msg_count");
}
return res;
} catch (vm::VmVirtError& err) {
return td::Status::Error(PSTRING() << "invalid proof: " << err.get_msg());
}
}
void BuildOutMsgQueueProof::abort_query(td::Status reason) {
if (promise_) {
FLOG(DEBUG) {
sb << "failed to build msg queue proof to " << dst_shard_.to_str() << " from";
for (const auto& block : blocks_) {
sb << " " << block.id.id.to_str();
}
sb << ": " << reason;
};
promise_.set_error(
reason.move_as_error_prefix(PSTRING() << "failed to build msg queue proof to " << dst_shard_.to_str() << ": "));
}
stop();
}
void BuildOutMsgQueueProof::start_up() {
for (size_t i = 0; i < blocks_.size(); ++i) {
BlockIdExt id = blocks_[i].id;
++pending;
td::actor::send_closure(manager_, &ValidatorManagerInterface::get_shard_state_from_db_short, id,
[SelfId = actor_id(this), i](td::Result<Ref<ShardState>> R) {
if (R.is_error()) {
td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::abort_query,
R.move_as_error_prefix("failed to get shard state: "));
} else {
td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::got_state_root, i,
R.move_as_ok()->root_cell());
}
});
if (id.seqno() != 0) {
++pending;
td::actor::send_closure(manager_, &ValidatorManagerInterface::get_block_data_from_db_short, id,
[SelfId = actor_id(this), i](td::Result<Ref<BlockData>> R) {
if (R.is_error()) {
td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::abort_query,
R.move_as_error_prefix("failed to get block data: "));
} else {
td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::got_block_root, i,
R.move_as_ok()->root_cell());
}
});
}
}
if (pending == 0) {
build_proof();
}
}
void BuildOutMsgQueueProof::got_state_root(size_t i, Ref<vm::Cell> root) {
blocks_[i].state_root = std::move(root);
if (--pending == 0) {
build_proof();
}
}
void BuildOutMsgQueueProof::got_block_root(size_t i, Ref<vm::Cell> root) {
blocks_[i].block_root = std::move(root);
if (--pending == 0) {
build_proof();
}
}
void BuildOutMsgQueueProof::build_proof() {
auto result = OutMsgQueueProof::build(dst_shard_, std::move(blocks_), limits_);
if (result.is_error()) {
LOG(ERROR) << "Failed to build msg queue proof: " << result.error();
}
promise_.set_result(std::move(result));
stop();
}
} // namespace validator
} // namespace ton

64
validator/impl/out-msg-queue-proof.hpp Normal file
View file

@ -0,0 +1,64 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "vm/cells.h"
#include "ton/ton-types.h"
#include "auto/tl/ton_api.h"
#include "interfaces/out-msg-queue-proof.h"
#include "td/actor/actor.h"
#include "interfaces/shard.h"
#include "validator.h"
namespace ton {
namespace validator {
using td::Ref;
class ValidatorManager;
class ValidatorManagerInterface;
class BuildOutMsgQueueProof : public td::actor::Actor {
public:
BuildOutMsgQueueProof(ShardIdFull dst_shard, std::vector<BlockIdExt> blocks, block::ImportedMsgQueueLimits limits,
td::actor::ActorId<ValidatorManagerInterface> manager,
td::Promise<tl_object_ptr<ton_api::tonNode_outMsgQueueProof>> promise)
: dst_shard_(dst_shard), limits_(limits), manager_(manager), promise_(std::move(promise)) {
blocks_.resize(blocks.size());
for (size_t i = 0; i < blocks_.size(); ++i) {
blocks_[i].id = blocks[i];
}
}
void abort_query(td::Status reason);
void start_up() override;
void got_state_root(size_t i, Ref<vm::Cell> root);
void got_block_root(size_t i, Ref<vm::Cell> root);
void build_proof();
private:
ShardIdFull dst_shard_;
std::vector<OutMsgQueueProof::OneBlock> blocks_;
block::ImportedMsgQueueLimits limits_;
td::actor::ActorId<ValidatorManagerInterface> manager_;
td::Promise<tl_object_ptr<ton_api::tonNode_outMsgQueueProof>> promise_;
size_t pending = 0;
};
} // namespace validator
} // namespace ton

35
validator/impl/proof.cpp
View file

@ -162,5 +162,40 @@ td::Result<Ref<vm::Cell>> ProofQ::get_signatures_root() const {
return proof.signatures->prefetch_ref();
}
td::Result<td::Ref<vm::Cell>> create_block_state_proof(td::Ref<vm::Cell> root) {
if (root.is_null()) {
return td::Status::Error("root is null");
}
vm::MerkleProofBuilder mpb{std::move(root)};
block::gen::Block::Record block;
if (!tlb::unpack_cell(mpb.root(), block) || block.state_update->load_cell().is_error()) {
return td::Status::Error("invalid block");
}
TRY_RESULT(proof, mpb.extract_proof());
if (proof.is_null()) {
return td::Status::Error("failed to create proof");
}
return proof;
}
td::Result<RootHash> unpack_block_state_proof(BlockIdExt block_id, td::Ref<vm::Cell> proof) {
auto virt_root = vm::MerkleProof::virtualize(proof, 1);
if (virt_root.is_null()) {
return td::Status::Error("invalid Merkle proof");
}
if (virt_root->get_hash().as_slice() != block_id.root_hash.as_slice()) {
return td::Status::Error("hash mismatch");
}
block::gen::Block::Record block;
if (!tlb::unpack_cell(virt_root, block)) {
return td::Status::Error("invalid block");
}
vm::CellSlice upd_cs{vm::NoVmSpec(), block.state_update};
if (!(upd_cs.is_special() && upd_cs.prefetch_long(8) == 4 && upd_cs.size_ext() == 0x20228)) {
return td::Status::Error("invalid Merkle update");
}
return upd_cs.prefetch_ref(1)->get_hash(0).bits();
}
} // namespace validator
} // namespace ton

57
validator/interfaces/out-msg-queue-proof.h Normal file
View file

@ -0,0 +1,57 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "vm/cells.h"
#include "ton/ton-types.h"
#include "auto/tl/ton_api.h"
#include "block/block.h"
namespace ton {
namespace validator {
using td::Ref;
struct OutMsgQueueProof : public td::CntObject {
OutMsgQueueProof(BlockIdExt block_id, Ref<vm::Cell> state_root, Ref<vm::Cell> block_state_proof,
td::int32 msg_count = -1)
: block_id_(block_id)
, state_root_(std::move(state_root))
, block_state_proof_(std::move(block_state_proof))
, msg_count_(msg_count) {
}
BlockIdExt block_id_;
Ref<vm::Cell> state_root_;
Ref<vm::Cell> block_state_proof_;
td::int32 msg_count_; // -1 - no limit
static td::Result<std::vector<td::Ref<OutMsgQueueProof>>> fetch(ShardIdFull dst_shard, std::vector<BlockIdExt> blocks,
block::ImportedMsgQueueLimits limits,
const ton_api::tonNode_outMsgQueueProof &f);
struct OneBlock {
BlockIdExt id;
Ref<vm::Cell> state_root;
Ref<vm::Cell> block_root;
};
static td::Result<tl_object_ptr<ton_api::tonNode_outMsgQueueProof>> build(ShardIdFull dst_shard,
std::vector<OneBlock> blocks,
block::ImportedMsgQueueLimits limits);
};
} // namespace validator
} // namespace ton

3
validator/interfaces/proof.h
View file

@ -48,6 +48,9 @@ class Proof : virtual public ProofLink {
virtual td::Result<td::Ref<ProofLink>> export_as_proof_link() const = 0;
};
td::Result<td::Ref<vm::Cell>> create_block_state_proof(td::Ref<vm::Cell> root);
td::Result<RootHash> unpack_block_state_proof(BlockIdExt block_id, td::Ref<vm::Cell> proof);
} // namespace validator
} // namespace ton

3
validator/interfaces/validator-manager.h
View file

@ -146,6 +146,9 @@ class ValidatorManager : public ValidatorManagerInterface {
virtual void send_top_shard_block_description(td::Ref<ShardTopBlockDescription> desc) = 0;
virtual void send_block_broadcast(BlockBroadcast broadcast, int mode) = 0;
virtual void send_validator_telemetry(PublicKeyHash key, tl_object_ptr<ton_api::validator_telemetry> telemetry) = 0;
virtual void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector<BlockIdExt> blocks,
block::ImportedMsgQueueLimits limits,
td::Promise<std::vector<td::Ref<OutMsgQueueProof>>> promise) = 0;
virtual void send_download_archive_request(BlockSeqno mc_seqno, ShardIdFull shard_prefix, std::string tmp_dir,
td::Timestamp timeout, td::Promise<std::string> promise) = 0;

7
validator/manager-disk.hpp
View file

@ -265,6 +265,11 @@ class ValidatorManagerImpl : public ValidatorManager {
}
void send_validator_telemetry(PublicKeyHash key, tl_object_ptr<ton_api::validator_telemetry> telemetry) override {
}
void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector<BlockIdExt> blocks,
block::ImportedMsgQueueLimits limits,
td::Promise<std::vector<td::Ref<OutMsgQueueProof>>> promise) override {
UNREACHABLE();
}
void send_download_archive_request(BlockSeqno mc_seqno, ShardIdFull shard_prefix, std::string tmp_dir,
td::Timestamp timeout, td::Promise<std::string> promise) override {
UNREACHABLE();
@ -283,7 +288,7 @@ class ValidatorManagerImpl : public ValidatorManager {
void try_get_static_file(FileHash file_hash, td::Promise<td::BufferSlice> promise) override;
void get_download_token(size_t download_size, td::uint32 priority, td::Timestamp timeout,
td::Promise<std::unique_ptr<DownloadToken>> promise) override {
td::Promise<std::unique_ptr<ActionToken>> promise) override {
promise.set_error(td::Status::Error(ErrorCode::error, "download disabled"));
}

7
validator/manager-hardfork.hpp
View file

@ -335,6 +335,11 @@ class ValidatorManagerImpl : public ValidatorManager {
}
void send_validator_telemetry(PublicKeyHash key, tl_object_ptr<ton_api::validator_telemetry> telemetry) override {
}
void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector<BlockIdExt> blocks,
block::ImportedMsgQueueLimits limits,
td::Promise<std::vector<td::Ref<OutMsgQueueProof>>> promise) override {
UNREACHABLE();
}
void send_download_archive_request(BlockSeqno mc_seqno, ShardIdFull shard_prefix, std::string tmp_dir,
td::Timestamp timeout, td::Promise<std::string> promise) override {
UNREACHABLE();
@ -357,7 +362,7 @@ class ValidatorManagerImpl : public ValidatorManager {
void try_get_static_file(FileHash file_hash, td::Promise<td::BufferSlice> promise) override;
void get_download_token(size_t download_size, td::uint32 priority, td::Timestamp timeout,
td::Promise<std::unique_ptr<DownloadToken>> promise) override {
td::Promise<std::unique_ptr<ActionToken>> promise) override {
promise.set_error(td::Status::Error(ErrorCode::error, "download disabled"));
}

7
validator/manager.cpp
View file

@ -1652,6 +1652,13 @@ void ValidatorManagerImpl::send_validator_telemetry(PublicKeyHash key,
callback_->send_validator_telemetry(key, std::move(telemetry));
}
void ValidatorManagerImpl::send_get_out_msg_queue_proof_request(
ShardIdFull dst_shard, std::vector<BlockIdExt> blocks, block::ImportedMsgQueueLimits limits,
td::Promise<std::vector<td::Ref<OutMsgQueueProof>>> promise) {
callback_->download_out_msg_queue_proof(dst_shard, std::move(blocks), limits, td::Timestamp::in(10.0),
std::move(promise));
}
void ValidatorManagerImpl::send_download_archive_request(BlockSeqno mc_seqno, ShardIdFull shard_prefix,
std::string tmp_dir, td::Timestamp timeout,
td::Promise<std::string> promise) {

7
validator/manager.hpp
View file

@ -512,6 +512,9 @@ class ValidatorManagerImpl : public ValidatorManager {
void send_top_shard_block_description(td::Ref<ShardTopBlockDescription> desc) override;
void send_block_broadcast(BlockBroadcast broadcast, int mode) override;
void send_validator_telemetry(PublicKeyHash key, tl_object_ptr<ton_api::validator_telemetry> telemetry) override;
void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector<BlockIdExt> blocks,
block::ImportedMsgQueueLimits limits,
td::Promise<std::vector<td::Ref<OutMsgQueueProof>>> promise) override;
void send_download_archive_request(BlockSeqno mc_seqno, ShardIdFull shard_prefix, std::string tmp_dir,
td::Timestamp timeout, td::Promise<std::string> promise) override;
@ -524,8 +527,8 @@ class ValidatorManagerImpl : public ValidatorManager {
void try_get_static_file(FileHash file_hash, td::Promise<td::BufferSlice> promise) override;
void get_download_token(size_t download_size, td::uint32 priority, td::Timestamp timeout,
td::Promise<std::unique_ptr<DownloadToken>> promise) override {
td::actor::send_closure(token_manager_, &TokenManager::get_download_token, download_size, priority, timeout,
td::Promise<std::unique_ptr<ActionToken>> promise) override {
td::actor::send_closure(token_manager_, &TokenManager::get_token, download_size, priority, timeout,
std::move(promise));
}

4
validator/net/download-block-new.cpp
View file

@ -144,7 +144,7 @@ void DownloadBlockNew::got_block_handle(BlockHandle handle) {
return;
}
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<std::unique_ptr<DownloadToken>> R) {
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<std::unique_ptr<ActionToken>> R) {
if (R.is_error()) {
td::actor::send_closure(SelfId, &DownloadBlockNew::abort_query,
R.move_as_error_prefix("failed to get download token: "));
@ -156,7 +156,7 @@ void DownloadBlockNew::got_block_handle(BlockHandle handle) {
std::move(P));
}
void DownloadBlockNew::got_download_token(std::unique_ptr<DownloadToken> token) {
void DownloadBlockNew::got_download_token(std::unique_ptr<ActionToken> token) {
token_ = std::move(token);
if (download_from_.is_zero() && client_.empty()) {

4
validator/net/download-block-new.hpp
View file

@ -49,7 +49,7 @@ class DownloadBlockNew : public td::actor::Actor {
void start_up() override;
void got_block_handle(BlockHandle handle);
void got_download_token(std::unique_ptr<DownloadToken> token);
void got_download_token(std::unique_ptr<ActionToken> token);
void got_node_to_download(adnl::AdnlNodeIdShort node);
void got_data(td::BufferSlice data);
void got_data_from_db(td::BufferSlice data);
@ -79,7 +79,7 @@ class DownloadBlockNew : public td::actor::Actor {
bool allow_partial_proof_ = false;
std::unique_ptr<DownloadToken> token_;
std::unique_ptr<ActionToken> token_;
};
} // namespace fullnode

4
validator/net/download-block.cpp
View file

@ -128,7 +128,7 @@ void DownloadBlock::got_block_handle(BlockHandle handle) {
return;
}
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<std::unique_ptr<DownloadToken>> R) {
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<std::unique_ptr<ActionToken>> R) {
if (R.is_error()) {
td::actor::send_closure(SelfId, &DownloadBlock::abort_query,
R.move_as_error_prefix("failed to get download token: "));
@ -140,7 +140,7 @@ void DownloadBlock::got_block_handle(BlockHandle handle) {
std::move(P));
}
void DownloadBlock::got_download_token(std::unique_ptr<DownloadToken> token) {
void DownloadBlock::got_download_token(std::unique_ptr<ActionToken> token) {
token_ = std::move(token);
if (download_from_.is_zero() && !short_ && client_.empty()) {

4
validator/net/download-block.hpp
View file

@ -49,7 +49,7 @@ class DownloadBlock : public td::actor::Actor {
void start_up() override;
void got_block_handle(BlockHandle handle);
void got_download_token(std::unique_ptr<DownloadToken> token);
void got_download_token(std::unique_ptr<ActionToken> token);
void got_node_to_download(adnl::AdnlNodeIdShort node);
void got_block_proof_description(td::BufferSlice proof_description);
void got_block_proof(td::BufferSlice data);
@ -86,7 +86,7 @@ class DownloadBlock : public td::actor::Actor {
bool allow_partial_proof_ = false;
std::unique_ptr<DownloadToken> token_;
std::unique_ptr<ActionToken> token_;
};
} // namespace fullnode

4
validator/net/download-proof.cpp
View file

@ -107,7 +107,7 @@ void DownloadProof::start_up() {
}
void DownloadProof::checked_db() {
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<std::unique_ptr<DownloadToken>> R) {
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<std::unique_ptr<ActionToken>> R) {
if (R.is_error()) {
td::actor::send_closure(SelfId, &DownloadProof::abort_query,
R.move_as_error_prefix("failed to get download token: "));
@ -119,7 +119,7 @@ void DownloadProof::checked_db() {
std::move(P));
}
void DownloadProof::got_download_token(std::unique_ptr<DownloadToken> token) {
void DownloadProof::got_download_token(std::unique_ptr<ActionToken> token) {
token_ = std::move(token);
if (download_from_.is_zero() && client_.empty()) {

4
validator/net/download-proof.hpp
View file

@ -45,7 +45,7 @@ class DownloadProof : public td::actor::Actor {
void start_up() override;
void checked_db();
void got_download_token(std::unique_ptr<DownloadToken> token);
void got_download_token(std::unique_ptr<ActionToken> token);
void got_node_to_download(adnl::AdnlNodeIdShort node);
void got_block_proof_description(td::BufferSlice proof_description);
void got_block_proof(td::BufferSlice data);
@ -72,7 +72,7 @@ class DownloadProof : public td::actor::Actor {
td::BufferSlice data_;
std::unique_ptr<DownloadToken> token_;
std::unique_ptr<ActionToken> token_;
};
} // namespace fullnode

4
validator/net/get-next-key-blocks.cpp
View file

@ -84,7 +84,7 @@ void GetNextKeyBlocks::finish_query() {
void GetNextKeyBlocks::start_up() {
alarm_timestamp() = timeout_;
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<std::unique_ptr<DownloadToken>> R) {
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<std::unique_ptr<ActionToken>> R) {
if (R.is_error()) {
td::actor::send_closure(SelfId, &GetNextKeyBlocks::abort_query,
R.move_as_error_prefix("failed to get download token: "));
@ -96,7 +96,7 @@ void GetNextKeyBlocks::start_up() {
std::move(P));
}
void GetNextKeyBlocks::got_download_token(std::unique_ptr<DownloadToken> token) {
void GetNextKeyBlocks::got_download_token(std::unique_ptr<ActionToken> token) {
token_ = std::move(token);
if (download_from_.is_zero() && client_.empty()) {

4
validator/net/get-next-key-blocks.hpp
View file

@ -44,7 +44,7 @@ class GetNextKeyBlocks : public td::actor::Actor {
void finish_query();
void start_up() override;
void got_download_token(std::unique_ptr<DownloadToken> token);
void got_download_token(std::unique_ptr<ActionToken> token);
void got_node_to_download(adnl::AdnlNodeIdShort node);
void send_request();
void got_result(td::BufferSlice res);
@ -75,7 +75,7 @@ class GetNextKeyBlocks : public td::actor::Actor {
std::vector<BlockIdExt> pending_;
std::vector<BlockIdExt> res_;
std::unique_ptr<DownloadToken> token_;
std::unique_ptr<ActionToken> token_;
};
} // namespace fullnode

34
validator/token-manager.cpp
View file

@ -22,23 +22,23 @@ namespace ton {
namespace validator {
void TokenManager::get_download_token(size_t download_size, td::uint32 priority, td::Timestamp timeout,
td::Promise<std::unique_ptr<DownloadToken>> promise) {
void TokenManager::get_token(size_t size, td::uint32 priority, td::Timestamp timeout,
td::Promise<std::unique_ptr<ActionToken>> promise) {
if (free_priority_tokens_ > 0 && priority > 0) {
--free_priority_tokens_;
promise.set_value(gen_token(download_size, priority));
promise.set_value(gen_token(size, priority));
return;
}
if (free_tokens_ > 0) {
--free_tokens_;
promise.set_value(gen_token(download_size, priority));
promise.set_value(gen_token(size, priority));
return;
}
pending_.emplace(PendingPromiseKey{download_size, priority, seqno_++}, PendingPromise{timeout, std::move(promise)});
pending_.emplace(PendingPromiseKey{size, priority, seqno_++}, PendingPromise{timeout, std::move(promise)});
}
void TokenManager::download_token_cleared(size_t download_size, td::uint32 priority) {
void TokenManager::token_cleared(size_t size, td::uint32 priority) {
(priority ? free_priority_tokens_ : free_tokens_)++;
if (free_priority_tokens_ > max_priority_tokens_) {
free_priority_tokens_--;
@ -47,7 +47,7 @@ void TokenManager::download_token_cleared(size_t download_size, td::uint32 prior
for (auto it = pending_.begin(); it != pending_.end();) {
if (it->first.priority && (free_tokens_ || free_priority_tokens_)) {
it->second.promise.set_value(gen_token(download_size, priority));
it->second.promise.set_value(gen_token(size, priority));
auto it2 = it++;
pending_.erase(it2);
if (free_priority_tokens_ > 0) {
@ -56,7 +56,7 @@ void TokenManager::download_token_cleared(size_t download_size, td::uint32 prior
free_tokens_--;
}
} else if (!it->first.priority && free_tokens_) {
it->second.promise.set_value(gen_token(download_size, priority));
it->second.promise.set_value(gen_token(size, priority));
auto it2 = it++;
pending_.erase(it2);
free_tokens_--;
@ -69,7 +69,7 @@ void TokenManager::download_token_cleared(size_t download_size, td::uint32 prior
void TokenManager::alarm() {
for (auto it = pending_.begin(); it != pending_.end();) {
if (it->second.timeout.is_in_past()) {
it->second.promise.set_error(td::Status::Error(ErrorCode::timeout, "timeout in wait download token"));
it->second.promise.set_error(td::Status::Error(ErrorCode::timeout, "timeout in wait token"));
it = pending_.erase(it);
} else {
it++;
@ -77,23 +77,23 @@ void TokenManager::alarm() {
}
}
std::unique_ptr<DownloadToken> TokenManager::gen_token(size_t download_size, td::uint32 priority) {
class Token : public DownloadToken {
std::unique_ptr<ActionToken> TokenManager::gen_token(size_t size, td::uint32 priority) {
class TokenImpl : public ActionToken {
public:
Token(size_t download_size, td::uint32 priority, td::actor::ActorId<TokenManager> manager)
: download_size_(download_size), priority_(priority), manager_(manager) {
TokenImpl(size_t size, td::uint32 priority, td::actor::ActorId<TokenManager> manager)
: size_(size), priority_(priority), manager_(manager) {
}
~Token() override {
td::actor::send_closure(manager_, &TokenManager::download_token_cleared, download_size_, priority_);
~TokenImpl() override {
td::actor::send_closure(manager_, &TokenManager::token_cleared, size_, priority_);
}
private:
size_t download_size_;
size_t size_;
td::uint32 priority_;
td::actor::ActorId<TokenManager> manager_;
};
return std::make_unique<Token>(download_size, priority, actor_id(this));
return std::make_unique<TokenImpl>(size, priority, actor_id(this));
}
} // namespace validator

15
validator/token-manager.h
View file

@ -31,16 +31,19 @@ class TokenManager : public td::actor::Actor {
public:
TokenManager() {
}
explicit TokenManager(td::uint32 max_tokens)
: free_tokens_(max_tokens), free_priority_tokens_(max_tokens), max_priority_tokens_(max_tokens) {
}
void alarm() override;
void get_download_token(size_t download_size, td::uint32 priority, td::Timestamp timeout,
td::Promise<std::unique_ptr<DownloadToken>> promise);
void download_token_cleared(size_t download_size, td::uint32 priority);
void get_token(size_t size, td::uint32 priority, td::Timestamp timeout,
td::Promise<std::unique_ptr<ActionToken>> promise);
void token_cleared(size_t size, td::uint32 priority);
private:
std::unique_ptr<DownloadToken> gen_token(size_t download_size, td::uint32 priority);
std::unique_ptr<ActionToken> gen_token(size_t size, td::uint32 priority);
struct PendingPromiseKey {
size_t download_size;
size_t size;
td::uint32 priority;
td::uint64 seqno;
@ -50,7 +53,7 @@ class TokenManager : public td::actor::Actor {
};
struct PendingPromise {
td::Timestamp timeout;
td::Promise<std::unique_ptr<DownloadToken>> promise;
td::Promise<std::unique_ptr<ActionToken>> promise;
};
td::uint64 seqno_ = 0;
std::map<PendingPromiseKey, PendingPromise> pending_;

10
validator/validator.h
View file

@ -35,15 +35,16 @@
#include "interfaces/proof.h"
#include "interfaces/shard.h"
#include "catchain/catchain-types.h"
#include "interfaces/out-msg-queue-proof.h"
#include "interfaces/external-message.h"
namespace ton {
namespace validator {
class DownloadToken {
class ActionToken {
public:
virtual ~DownloadToken() = default;
virtual ~ActionToken() = default;
};
struct PerfTimerStats {
@ -186,6 +187,9 @@ class ValidatorManagerInterface : public td::actor::Actor {
td::Promise<std::vector<BlockIdExt>> promise) = 0;
virtual void download_archive(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir,
td::Timestamp timeout, td::Promise<std::string> promise) = 0;
virtual void download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector<BlockIdExt> blocks,
block::ImportedMsgQueueLimits limits, td::Timestamp timeout,
td::Promise<std::vector<td::Ref<OutMsgQueueProof>>> promise) = 0;
virtual void new_key_block(BlockHandle handle) = 0;
virtual void send_validator_telemetry(PublicKeyHash key, tl_object_ptr<ton_api::validator_telemetry> telemetry) = 0;
@ -248,7 +252,7 @@ class ValidatorManagerInterface : public td::actor::Actor {
virtual void add_ext_server_port(td::uint16 port) = 0;
virtual void get_download_token(size_t download_size, td::uint32 priority, td::Timestamp timeout,
td::Promise<std::unique_ptr<DownloadToken>> promise) = 0;
td::Promise<std::unique_ptr<ActionToken>> promise) = 0;
virtual void get_block_data_from_db(ConstBlockHandle handle, td::Promise<td::Ref<BlockData>> promise) = 0;
virtual void get_block_data_from_db_short(BlockIdExt block_id, td::Promise<td::Ref<BlockData>> promise) = 0;