diff --git a/adnl/adnl-node-id.hpp b/adnl/adnl-node-id.hpp index 2d3ade16..43e3629f 100644 --- a/adnl/adnl-node-id.hpp +++ b/adnl/adnl-node-id.hpp @@ -37,7 +37,9 @@ class AdnlNodeIdShort { } explicit AdnlNodeIdShort(td::Bits256 value) : hash_(value) { } - explicit AdnlNodeIdShort(tl_object_ptr obj) : hash_(obj->id_) { + explicit AdnlNodeIdShort(tl_object_ptr &&obj) : hash_(obj->id_) { + } + explicit AdnlNodeIdShort(const tl_object_ptr &obj) : hash_(obj->id_) { } const auto &pubkey_hash() const { diff --git a/dht-server/dht-server.cpp b/dht-server/dht-server.cpp index 006e7408..46a01135 100644 --- a/dht-server/dht-server.cpp +++ b/dht-server/dht-server.cpp @@ -171,8 +171,8 @@ ton::tl_object_ptr Config::tl() const { } return ton::create_tl_object( out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), std::move(col_vec), - ton::PublicKeyHash::zero().tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec), - nullptr, nullptr, std::move(liteserver_vec), std::move(control_vec), std::move(shard_vec), std::move(gc_vec)); + ton::PublicKeyHash::zero().tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec), nullptr, + nullptr, std::move(liteserver_vec), std::move(control_vec), std::move(shard_vec), std::move(gc_vec)); } td::Result Config::config_add_network_addr(td::IPAddress in_ip, td::IPAddress out_ip, @@ -1195,7 +1195,8 @@ int main(int argc, char *argv[]) { SET_VERBOSITY_LEVEL(v); }); p.add_option('V', "version", "shows dht-server build information", [&]() { - std::cout << "dht-server build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; + std::cout << "dht-server build information: [ Commit: " << GitMetadata::CommitSHA1() + << ", Date: " << GitMetadata::CommitDate() << "]\n"; std::exit(0); }); p.add_option('h', "help", "prints_help", [&]() { diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index a4d34428..396c06bc 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -464,6 +464,8 @@ tonNode.outMsgQueueProof queue_proofs:bytes block_state_proofs:bytes msg_counts: tonNode.outMsgQueueProofEmpty = tonNode.OutMsgQueueProof; tonNode.forgetPeer = tonNode.ForgetPeer; +tonNode.newFastSyncMemberCertificate adnl_id:int256 certificate:overlay.MemberCertificate = tonNode.NewFastSyncMemberCertificate; + ---functions--- @@ -501,6 +503,8 @@ tonNode.slave.sendExtMessage message:tonNode.externalMessage = tonNode.Success; tonNode.query = Object; +tonNode.requestFastSyncOverlayMemberCertificate sign_by:int256 adnl_id:int256 slot:int = overlay.MemberCertificate; + ---types--- // bit 0 - started @@ -654,9 +658,11 @@ engine.validator.fullNodeMaster port:int adnl:int256 = engine.validator.FullNode engine.validator.fullNodeSlave ip:int port:int adnl:PublicKey = engine.validator.FullNodeSlave; engine.validator.fullNodeConfig ext_messages_broadcast_disabled:Bool = engine.validator.FullNodeConfig; engine.validator.fastSyncMemberCertificate adnl_id:int256 certificate:overlay.MemberCertificate = engine.validator.FastSyncMemberCertificate; +engine.validator.fastSyncOverlayClient adnl_id:int256 slot:int = engine.validator.FastSyncOverlayClient; engine.validator.collatorNodeWhitelist enabled:Bool adnl_ids:(vector int256) = engine.validator.CollatorNodeWhitelist; engine.validator.extraConfig state_serializer_enabled:Bool fast_sync_member_certificates:(vector engine.validator.fastSyncMemberCertificate) - collator_node_whitelist:engine.validator.collatorNodeWhitelist = engine.validator.ExtraConfig; + collator_node_whitelist:engine.validator.collatorNodeWhitelist + fast_sync_overlay_clients:(vector engine.validator.fastSyncOverlayClient) = engine.validator.ExtraConfig; engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector engine.adnl) dht:(vector engine.dht) validators:(vector engine.validator) collators:(vector engine.collator) @@ -840,6 +846,8 @@ engine.validator.getCollationManagerStats = engine.validator.CollationManagerSta engine.validator.signOverlayMemberCertificate sign_by:int256 adnl_id:int256 slot:int expire_at:int = overlay.MemberCertificate; engine.validator.importFastSyncMemberCertificate adnl_id:int256 certificate:overlay.MemberCertificate = engine.validator.Success; +engine.validator.addFastSyncClient adnl_id:int256 slot:int = engine.validator.Success; +engine.validator.delFastSyncClient adnl_id:int256 = engine.validator.Success; ---types--- diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index cfb78d6f..faef1564 100644 Binary files a/tl/generate/scheme/ton_api.tlo and b/tl/generate/scheme/ton_api.tlo differ diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index 2d78b412..c0170a06 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -807,21 +807,21 @@ td::Status SignCertificateQuery::send() { auto sign = ton::create_serialize_tl_object(signer_.tl(), std::move(cid)); auto pub = ton::create_serialize_tl_object(signer_.tl()); td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(pub), - td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &SignCertificateQuery::handle_error, R.move_as_error()); - } else { - td::actor::send_closure(SelfId, &SignCertificateQuery::receive_pubkey, R.move_as_ok()); - } - })); + td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &SignCertificateQuery::handle_error, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &SignCertificateQuery::receive_pubkey, R.move_as_ok()); + } + })); td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(sign), - td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &SignCertificateQuery::handle_error, R.move_as_error()); - } else { - td::actor::send_closure(SelfId, &SignCertificateQuery::receive_signature, R.move_as_ok()); - } - })); + td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &SignCertificateQuery::handle_error, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &SignCertificateQuery::receive_signature, R.move_as_ok()); + } + })); return td::Status::OK(); } @@ -833,33 +833,32 @@ void SignCertificateQuery::receive_pubkey(td::BufferSlice R) { } pubkey_ = f.move_as_ok(); has_pubkey_ = true; - if(has_signature_) { + if (has_signature_) { save_certificate(); } } - td::Status SignCertificateQuery::receive(td::BufferSlice data) { UNREACHABLE(); } void SignCertificateQuery::receive_signature(td::BufferSlice R) { auto f = ton::fetch_tl_object(R.as_slice(), true); - if(f.is_error()){ + if (f.is_error()) { handle_error(f.move_as_error_prefix("Failed to get signature: ")); return; } signature_ = std::move(f.move_as_ok()->signature_); - if(has_pubkey_) { + if (has_pubkey_) { save_certificate(); } } void SignCertificateQuery::save_certificate() { - auto c = ton::create_serialize_tl_object( - std::move(pubkey_), expire_at_, max_size_, std::move(signature_)); + auto c = ton::create_serialize_tl_object(std::move(pubkey_), expire_at_, max_size_, + std::move(signature_)); auto w = td::write_file(out_file_, c.as_slice()); - if(w.is_error()) { + if (w.is_error()) { handle_error(w.move_as_error_prefix("Failed to write certificate to file: ")); return; } @@ -880,11 +879,8 @@ td::Status ImportCertificateQuery::send() { TRY_RESULT_PREFIX(cert, ton::fetch_tl_object(data.as_slice(), true), "incorrect certificate"); auto b = ton::create_serialize_tl_object( - overlay_, - ton::create_tl_object(id_), - ton::create_tl_object(kh_.tl()), - std::move(cert) - ); + overlay_, ton::create_tl_object(id_), + ton::create_tl_object(kh_.tl()), std::move(cert)); td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); return td::Status::OK(); } @@ -1031,7 +1027,6 @@ td::Status GetOverlaysStatsJsonQuery::receive(td::BufferSlice data) { return td::Status::OK(); } - td::Status ImportCertificateQuery::receive(td::BufferSlice data) { TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), "received incorrect answer: "); @@ -1039,10 +1034,9 @@ td::Status ImportCertificateQuery::receive(td::BufferSlice data) { return td::Status::OK(); } - td::Status SignShardOverlayCertificateQuery::run() { TRY_RESULT_ASSIGN(wc_, tokenizer_.get_token()); - TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token() ); + TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token()); TRY_RESULT_ASSIGN(key_, tokenizer_.get_token()); TRY_RESULT_ASSIGN(expire_at_, tokenizer_.get_token()); TRY_RESULT_ASSIGN(max_size_, tokenizer_.get_token()); @@ -1052,8 +1046,8 @@ td::Status SignShardOverlayCertificateQuery::run() { } td::Status SignShardOverlayCertificateQuery::send() { - auto b = ton::create_serialize_tl_object - (wc_, shard_, ton::create_tl_object(key_.tl()), expire_at_, max_size_); + auto b = ton::create_serialize_tl_object( + wc_, shard_, ton::create_tl_object(key_.tl()), expire_at_, max_size_); td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); return td::Status::OK(); } @@ -1062,7 +1056,7 @@ td::Status SignShardOverlayCertificateQuery::receive(td::BufferSlice data) { TRY_RESULT_PREFIX(c, ton::fetch_tl_object(data.as_slice(), true), "received incorrect cert: "); auto w = td::write_file(out_file_, data.as_slice()); - if(w.is_error()) { + if (w.is_error()) { return w.move_as_error_prefix("Failed to write certificate to file: "); } td::TerminalIO::out() << "saved certificate\n"; @@ -1072,7 +1066,7 @@ td::Status SignShardOverlayCertificateQuery::receive(td::BufferSlice data) { td::Status ImportShardOverlayCertificateQuery::run() { TRY_RESULT_ASSIGN(wc_, tokenizer_.get_token()); - TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token() ); + TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token()); TRY_RESULT_ASSIGN(key_, tokenizer_.get_token()); TRY_RESULT_ASSIGN(in_file_, tokenizer_.get_token()); @@ -1083,8 +1077,8 @@ td::Status ImportShardOverlayCertificateQuery::send() { TRY_RESULT(data, td::read_file(in_file_)); TRY_RESULT_PREFIX(cert, ton::fetch_tl_object(data.as_slice(), true), "incorrect certificate"); - auto b = ton::create_serialize_tl_object - (wc_, shard_, ton::create_tl_object(key_.tl()), std::move(cert)); + auto b = ton::create_serialize_tl_object( + wc_, shard_, ton::create_tl_object(key_.tl()), std::move(cert)); td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); return td::Status::OK(); } @@ -1908,3 +1902,42 @@ td::Status ImportFastSyncMemberCertificateQuery::receive(td::BufferSlice data) { td::TerminalIO::out() << "success\n"; return td::Status::OK(); } + +td::Status AddFastSyncOverlayClientQuery::run() { + TRY_RESULT_ASSIGN(adnl_id_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(slot_, tokenizer_.get_token()); + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status AddFastSyncOverlayClientQuery::send() { + auto b = ton::create_serialize_tl_object(adnl_id_, slot_); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status AddFastSyncOverlayClientQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} + +td::Status DelFastSyncOverlayClientQuery::run() { + TRY_RESULT_ASSIGN(adnl_id_, tokenizer_.get_token()); + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status DelFastSyncOverlayClientQuery::send() { + auto b = ton::create_serialize_tl_object(adnl_id_); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status DelFastSyncOverlayClientQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index c34f4f5b..60e05d59 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -979,9 +979,9 @@ class GetOverlaysStatsJsonQuery : public Query { std::string name() const override { return get_name(); } - -private: - std::string file_name_; + + private: + std::string file_name_; }; class SignCertificateQuery : public Query { @@ -996,7 +996,8 @@ class SignCertificateQuery : public Query { return "signcert"; } static std::string get_help() { - return "signcert \tsign overlay certificate by key"; + return "signcert \tsign overlay certificate by " + " key"; } std::string name() const override { return get_name(); @@ -1004,9 +1005,8 @@ class SignCertificateQuery : public Query { void receive_pubkey(td::BufferSlice R); void receive_signature(td::BufferSlice R); - private: - void save_certificate(); + void save_certificate(); td::Bits256 overlay_; td::Bits256 id_; @@ -1057,14 +1057,14 @@ class SignShardOverlayCertificateQuery : public Query { return "signshardoverlaycert"; } static std::string get_help() { - return "signshardoverlaycert \tsign certificate for in currently active shard overlay"; + return "signshardoverlaycert \tsign certificate for " + " in currently active shard overlay"; } std::string name() const override { return get_name(); } private: - td::int32 wc_; td::int64 shard_; td::int32 expire_at_; @@ -1073,7 +1073,6 @@ class SignShardOverlayCertificateQuery : public Query { std::string out_file_; }; - class ImportShardOverlayCertificateQuery : public Query { public: ImportShardOverlayCertificateQuery(td::actor::ActorId console, Tokenizer tokenizer) @@ -1086,14 +1085,14 @@ class ImportShardOverlayCertificateQuery : public Query { return "importshardoverlaycert"; } static std::string get_help() { - return "importshardoverlaycert \timport certificate for in currently active shard overlay"; + return "importshardoverlaycert \timport certificate for in " + "currently active shard overlay"; } std::string name() const override { return get_name(); } private: - td::int32 wc_; td::int64 shard_; ton::PublicKeyHash key_; @@ -1134,7 +1133,8 @@ class GetPerfTimerStatsJsonQuery : public Query { return "getperftimerstatsjson"; } static std::string get_help() { - return "getperftimerstatsjson \tgets min, average and max event processing time for last 60, 300 and 3600 seconds and writes to json file"; + return "getperftimerstatsjson \tgets min, average and max event processing time for last 60, 300 and 3600 " + "seconds and writes to json file"; } std::string name() const override { return get_name(); @@ -1696,3 +1696,50 @@ class ImportFastSyncMemberCertificateQuery : public Query { td::Bits256 adnl_id_; std::string file_name_; }; + +class AddFastSyncOverlayClientQuery : public Query { + public: + AddFastSyncOverlayClientQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "addfastsyncoverlayclient"; + } + static std::string get_help() { + return "addfastsyncoverlayclient \tstarts issuing member certificates " + "to (hex) on slot (int)"; + } + std::string name() const override { + return get_name(); + } + + private: + td::Bits256 adnl_id_; + td::int32 slot_; +}; + +class DelFastSyncOverlayClientQuery : public Query { + public: + DelFastSyncOverlayClientQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "delfastsyncoverlayclient"; + } + static std::string get_help() { + return "delfastsyncoverlayclient \tstops issuing member certificates " + "to (hex)"; + } + std::string name() const override { + return get_name(); + } + + private: + td::Bits256 adnl_id_; +}; diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index 9e85b567..d8031681 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -168,6 +168,8 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); } bool ValidatorEngineConsole::envelope_send_query(td::BufferSlice query, td::Promise promise) { diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 5f4c7c5c..abba70c5 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -27,9 +27,17 @@ */ #include "validator-engine.hpp" +#include "adnl/adnl-node-id.hpp" #include "auto/tl/ton_api.h" +#include "errorcode.h" +#include "keys/keys.hpp" #include "overlay-manager.h" +#include "overlays.h" +#include "td/actor/PromiseFuture.h" #include "td/actor/actor.h" +#include "td/utils/Status.h" +#include "td/utils/Time.h" +#include "td/utils/buffer.h" #include "tl-utils/tl-utils.hpp" #include "tl/TlObject.h" #include "ton/ton-types.h" @@ -59,6 +67,8 @@ #include "memprof/memprof.h" #include "dht/dht.hpp" +#include +#include #if TD_DARWIN || TD_LINUX #include @@ -182,6 +192,10 @@ Config::Config(const ton::ton_api::engine_validator_config &config) { collator_node_whitelist.emplace(id); } } + for (auto &client : config.extraconfig_->fast_sync_overlay_clients_) { + auto key = ton::adnl::AdnlNodeIdShort{client->adnl_id_}; + fast_sync_overlay_clients.emplace_back(std::move(key), client->slot_); + } } else { state_serializer_enabled = true; } @@ -282,7 +296,8 @@ ton::tl_object_ptr Config::tl() const { } ton::tl_object_ptr extra_config_obj = {}; - if (!state_serializer_enabled || !fast_sync_member_certificates.empty() || collator_node_whitelist_obj) { + if (!state_serializer_enabled || !fast_sync_member_certificates.empty() || collator_node_whitelist_obj || + !fast_sync_overlay_clients.empty()) { // Non-default values extra_config_obj = ton::create_tl_object(); extra_config_obj->state_serializer_enabled_ = state_serializer_enabled; @@ -292,6 +307,11 @@ ton::tl_object_ptr Config::tl() const { certificate.tl())); } extra_config_obj->collator_node_whitelist_ = std::move(collator_node_whitelist_obj); + for (const auto &client : fast_sync_overlay_clients) { + extra_config_obj->fast_sync_overlay_clients_.push_back( + ton::create_tl_object(client.id.bits256_value(), + client.slot)); + } } std::vector> liteserver_vec; @@ -1352,10 +1372,12 @@ void ValidatorEngine::schedule_shutdown(double at) { LOG(DEBUG) << "Scheduled shutdown is in past (" << at << ")"; } else { LOG(INFO) << "Schedule shutdown for " << at << " (in " << ts.in() << "s)"; - ton::delay_action([]() { - LOG(WARNING) << "Shutting down as scheduled"; - std::_Exit(0); - }, ts); + ton::delay_action( + []() { + LOG(WARNING) << "Shutting down as scheduled"; + std::_Exit(0); + }, + ts); } } void ValidatorEngine::start_up() { @@ -1388,12 +1410,14 @@ void ValidatorEngine::alarm() { auto cur_t = config->get_validator_set_start_stop(0); CHECK(cur_t.first > 0); - auto val_set = state_->get_total_validator_set(0); - auto e = val_set->export_vector(); std::set to_del; for (auto &val : config_.validators) { bool is_validator = false; - if (val_set->is_validator(ton::NodeIdShort{val.first.bits256_value()})) { + if (validator_set_next_.not_null() && + validator_set_next_->is_validator(ton::NodeIdShort{val.first.bits256_value()})) { + is_validator = true; + } + if (validator_set_.not_null() && validator_set_->is_validator(ton::NodeIdShort{val.first.bits256_value()})) { is_validator = true; } if (!is_validator && val.second.election_date < cur_t.first && cur_t.first + 600 < state_->get_unix_time()) { @@ -1414,9 +1438,41 @@ void ValidatorEngine::alarm() { need_write = true; } + { + std::set fs_to_del; + for (auto &x : config_.fast_sync_member_certificates) { + if (x.second.is_expired()) { + fs_to_del.insert(x.first); + continue; + } + auto issued_by = x.second.issued_by().compute_short_id().bits256_value(); + if (validator_set_.not_null() && validator_set_->is_validator(issued_by)) { + continue; + } + if (validator_set_prev_.not_null() && validator_set_prev_->is_validator(issued_by)) { + continue; + } + if (validator_set_next_.not_null() && validator_set_next_->is_validator(issued_by)) { + continue; + } + fs_to_del.insert(x.first); + } + if (!fs_to_del.empty()) { + need_write = true; + std::erase_if(config_.fast_sync_member_certificates, + [&](const std::pair &e) { + return !fs_to_del.contains(e.first); + }); + } + } + if (need_write) { write_config([](td::Unit) {}); } + if (issue_fast_sync_overlay_certificates_at_.is_in_past()) { + issue_fast_sync_overlay_certificates_at_ = td::Timestamp::in(60.0); + issue_fast_sync_overlay_certificates(); + } } for (auto &x : config_.gc) { if (running_gc_.count(x) == 0) { @@ -1443,6 +1499,16 @@ void ValidatorEngine::deleted_key(ton::PublicKeyHash x) { } } +void ValidatorEngine::got_state(td::Ref state) { + if (state_.not_null() && state_->get_block_id() == state->get_block_id()) { + return; + } + state_ = std::move(state); + validator_set_ = state_->get_total_validator_set(0); + validator_set_next_ = state_->get_total_validator_set(1); + validator_set_prev_ = state_->get_total_validator_set(-1); +} + td::Status ValidatorEngine::load_global_config() { TRY_RESULT_PREFIX(conf_data, td::read_file(global_config_), "failed to read: "); TRY_RESULT_PREFIX(conf_json, td::json_decode(conf_data.as_slice()), "failed to parse json: "); @@ -1596,25 +1662,24 @@ void ValidatorEngine::set_shard_check_function() { validator_options_.write().set_shard_check_function([](ton::ShardIdFull shard) -> bool { return true; }); } else { std::vector shards = {ton::ShardIdFull(ton::masterchainId)}; - for (const auto& [_, collator_shards] : config_.collators) { - for (const auto& shard : collator_shards) { + for (const auto &[_, collator_shards] : config_.collators) { + for (const auto &shard : collator_shards) { shards.push_back(shard); } } - for (const auto& s : config_.shards_to_monitor) { + for (const auto &s : config_.shards_to_monitor) { shards.push_back(s); } std::sort(shards.begin(), shards.end()); shards.erase(std::unique(shards.begin(), shards.end()), shards.end()); - validator_options_.write().set_shard_check_function( - [shards = std::move(shards)](ton::ShardIdFull shard) -> bool { - for (auto s : shards) { - if (shard_intersects(shard, s)) { - return true; - } - } - return false; - }); + validator_options_.write().set_shard_check_function([shards = std::move(shards)](ton::ShardIdFull shard) -> bool { + for (auto s : shards) { + if (shard_intersects(shard, s)) { + return true; + } + } + return false; + }); } } @@ -2100,7 +2165,8 @@ void ValidatorEngine::started_validator() { } void ValidatorEngine::start_full_node() { - if (!config_.full_node.is_zero() || config_.full_node_slaves.size() > 0) { + if (!config_.full_node.is_zero() || !config_.full_node_slaves.empty()) { + full_node_id_ = ton::adnl::AdnlNodeIdShort{config_.full_node}; auto pk = ton::PrivateKey{ton::privkeys::Ed25519::random()}; auto short_id = pk.compute_short_id(); td::actor::send_closure(keyring_, &ton::keyring::Keyring::add_key, std::move(pk), true, [](td::Unit) {}); @@ -2123,8 +2189,8 @@ void ValidatorEngine::start_full_node() { td::actor::send_closure(SelfId, &ValidatorEngine::started_full_node); }); full_node_ = ton::validator::fullnode::FullNode::create( - short_id, ton::adnl::AdnlNodeIdShort{config_.full_node}, validator_options_->zero_block_id().file_hash, - config_.full_node_config, keyring_.get(), adnl_.get(), rldp_.get(), rldp2_.get(), + short_id, full_node_id_, validator_options_->zero_block_id().file_hash, config_.full_node_config, + keyring_.get(), adnl_.get(), rldp_.get(), rldp2_.get(), default_dht_node_.is_zero() ? td::actor::ActorId{} : dht_nodes_[default_dht_node_].get(), overlay_manager_.get(), validator_manager_.get(), full_node_client_.get(), db_root_, std::move(P)); for (auto &v : config_.validators) { @@ -2143,6 +2209,7 @@ void ValidatorEngine::start_full_node() { validator_telemetry_filename_); } load_custom_overlays_config(); + register_fast_sync_certificate_callback(); } else { started_full_node(); } @@ -2382,9 +2449,17 @@ void ValidatorEngine::try_add_full_node_adnl_addr(ton::PublicKeyHash id, td::Pro return; } - if (!full_node_.empty()) { - td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::update_adnl_id, - ton::adnl::AdnlNodeIdShort{id}, [](td::Unit) {}); + if (!full_node_.empty() && id != full_node_id_.pubkey_hash()) { + td::actor::send_closure( + adnl_.get(), &ton::adnl::Adnl::unsubscribe, full_node_id_, + ton::adnl::Adnl::int_to_bytestring(ton::ton_api::tonNode_newFastSyncMemberCertificate::ID)); + td::actor::send_closure( + adnl_.get(), &ton::adnl::Adnl::unsubscribe, full_node_id_, + ton::adnl::Adnl::int_to_bytestring(ton::ton_api::tonNode_requestFastSyncOverlayMemberCertificate::ID)); + full_node_id_ = ton::adnl::AdnlNodeIdShort{id}; + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::update_adnl_id, full_node_id_, + [](td::Unit) {}); + register_fast_sync_certificate_callback(); } write_config(std::move(promise)); @@ -2644,9 +2719,225 @@ void ValidatorEngine::try_del_proxy(td::uint32 ip, td::int32 port, std::vector validator_engine) + : validator_engine_(std::move(validator_engine)) { + } + void receive_message(ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, + td::BufferSlice data) override { + auto R = ton::fetch_tl_object( + std::move(data), true); + if (R.is_error()) { + return; + } + auto res = R.move_as_ok(); + auto cert = ton::overlay::OverlayMemberCertificate(res->certificate_.get()); + if (cert.empty()) { + return; + } + LOG(DEBUG) << "Received tonNode.newFastSyncMemberCertificate from " << src; + td::actor::send_closure(validator_engine_, &ValidatorEngine::try_import_fast_sync_member_certificate, dst, + std::move(cert), td::PromiseCreator::lambda([](td::Result R) { + if (R.is_error()) { + LOG(WARNING) << "failed to import overlay member certificate: " << R.move_as_error(); + } + })); + } + void receive_query(ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data, + td::Promise promise) override { + auto R = + ton::fetch_tl_object(std::move(data), true); + if (R.is_error()) { + return; + } + auto q = R.move_as_ok(); + td::actor::send_closure( + validator_engine_, &ValidatorEngine::process_fast_sync_overlay_certificate_request, + ton::PublicKeyHash{q->sign_by_}, ton::adnl::AdnlNodeIdShort{q->adnl_id_}, 0, q->slot_, + (td::int32)td::Clocks::system() + 3600, + td::PromiseCreator::lambda( + [promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + return; + } + auto cert = R.move_as_ok(); + promise.set_value(ton::serialize_tl_object(cert.tl(), true)); + })); + } + + private: + td::actor::ActorId validator_engine_; + }; + td::actor::send_closure( + adnl_.get(), &ton::adnl::Adnl::subscribe, full_node_id_, + ton::adnl::Adnl::int_to_bytestring(ton::ton_api::tonNode_newFastSyncMemberCertificate::ID), + std::make_unique(actor_id(this))); + td::actor::send_closure( + adnl_.get(), &ton::adnl::Adnl::subscribe, full_node_id_, + ton::adnl::Adnl::int_to_bytestring(ton::ton_api::tonNode_requestFastSyncOverlayMemberCertificate::ID), + std::make_unique(actor_id(this))); +} + +void ValidatorEngine::try_import_fast_sync_member_certificate(ton::adnl::AdnlNodeIdShort id, + ton::overlay::OverlayMemberCertificate certificate, + td::Promise promise) { + if (!started_ || state_.is_null()) { + return promise.set_error(td::Status::Error("not started")); + } + if (certificate.slot() < 0 || + certificate.slot() >= ton::validator::fullnode::FullNode::MAX_FAST_SYNC_OVERLAY_CLIENTS) { + return promise.set_error(td::Status::Error( + PSTRING() << "invalid slot (max " << ton::validator::fullnode::FullNode::MAX_FAST_SYNC_OVERLAY_CLIENTS + << " clients)")); + } + if (certificate.expire_at() < td::Clocks::system() + 60) { + return promise.set_error(td::Status::Error("certificate expires too soon")); + } + TRY_STATUS_PROMISE_PREFIX(promise, certificate.check_signature(id), "invalid certificate: "); + + auto cert_score = [this](ton::overlay::OverlayMemberCertificate &cert) -> td::int64 { + auto issued_by = cert.issued_by().compute_short_id().bits256_value(); + if (validator_set_next_.not_null() && validator_set_next_->is_validator(issued_by)) { + return cert.expire_at() + (1ll << 32); + } + if (validator_set_.not_null() && validator_set_->is_validator(issued_by)) { + return cert.expire_at() + (1ll << 32); + } + if (validator_set_prev_.not_null() && validator_set_prev_->is_validator(issued_by)) { + return cert.expire_at() + (0ll << 32); + } + return -1; + }; + for (auto &x : config_.fast_sync_member_certificates) { + if (x.first == id) { + // fast check + if (x.second.issued_by() == certificate.issued_by()) { + if (x.second.expire_at() < certificate.expire_at()) { + x.second = std::move(certificate); + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::import_fast_sync_member_certificate, + x.first, x.second); + write_config(std::move(promise)); + return; + } + LOG(DEBUG) << "Not importing certificate: certificate from the same issuer exists with bigger ttl"; + promise.set_value(td::Unit()); + return; + } + auto new_score = cert_score(certificate); + auto old_score = cert_score(x.second); + if (new_score > old_score) { + x.second = std::move(certificate); + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::import_fast_sync_member_certificate, + x.first, x.second); + write_config(std::move(promise)); + return; + } + LOG(DEBUG) << "Not importing certificate: certificate with better score exists"; + promise.set_value(td::Unit()); + return; + } + } + + auto new_score = cert_score(certificate); + if (new_score < 0) { + LOG(DEBUG) << "Not importing certificate: issuer is not a validator"; + promise.set_value(td::Unit()); + return; + } + + auto &x = config_.fast_sync_member_certificates.emplace_back(std::move(id), std::move(certificate)); + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::import_fast_sync_member_certificate, x.first, + x.second); + write_config(std::move(promise)); +} + +void ValidatorEngine::issue_fast_sync_overlay_certificates() { + if (state_.is_null() || config_.fast_sync_overlay_clients.empty() || full_node_id_.is_zero()) { + return; + } + auto issue_by = find_local_validator_for_cert_issuing(); + if (issue_by.is_zero()) { + return; + } + auto src = full_node_id_; + td::int32 expire_at = static_cast(td::Clocks::system()) + 3600; + for (auto &client : config_.fast_sync_overlay_clients) { + issue_fast_sync_overlay_certificate( + issue_by, client.id, 0, client.slot, expire_at, + [src, dst = client.id, adnl = adnl_.get()](td::Result R) { + if (R.is_error()) { + LOG(WARNING) << "cannot issue fast sync overlay certificate for " << dst << ": " << R.move_as_error(); + return; + } + auto cert = R.move_as_ok(); + LOG(INFO) << "Sending fast sync overlay certificate issued by " << cert.issued_by().compute_short_id() + << " to " << dst << " slot " << cert.slot(); + td::actor::send_closure(adnl, &ton::adnl::Adnl::send_message, src, dst, + ton::create_serialize_tl_object( + dst.bits256_value(), cert.tl())); + }); + } +} + +void ValidatorEngine::issue_fast_sync_overlay_certificate(ton::PublicKeyHash issue_by, + ton::adnl::AdnlNodeIdShort issue_to, td::uint32 flags, + td::int32 slot, td::int32 expire_at, + td::Promise promise) { + if (issue_by.is_zero()) { + issue_by = find_local_validator_for_cert_issuing(); + if (issue_by.is_zero()) { + return promise.set_error(td::Status::Error(ton::ErrorCode::notready, "cannot find a local validator")); + } + } + if (expire_at < td::Clocks::system() + 10) { + return promise.set_error(td::Status::Error(ton::ErrorCode::error, "expire at is in too near future")); + } + ton::overlay::OverlayMemberCertificate cert(ton::PublicKey(), flags, slot, expire_at, td::BufferSlice()); + auto to_sign = cert.to_sign_data(issue_to); + td::actor::send_closure(keyring_, &ton::keyring::Keyring::sign_add_get_public_key, issue_by, std::move(to_sign), + [slot, flags, expire_at, promise = std::move(promise)]( + td::Result> R) mutable { + TRY_RESULT_PROMISE_PREFIX(promise, res, std::move(R), "failed to sign certificate: "); + ton::overlay::OverlayMemberCertificate cert(std::move(res.second), flags, slot, expire_at, + std::move(res.first)); + promise.set_value(std::move(cert)); + }); +} + +void ValidatorEngine::process_fast_sync_overlay_certificate_request( + ton::PublicKeyHash issue_by, ton::adnl::AdnlNodeIdShort issue_to, td::uint32 flags, td::int32 slot, + td::int32 expire_at, td::Promise promise) { + for (auto &client : config_.fast_sync_overlay_clients) { + if (client.id == issue_to && (slot < 0 || slot == client.slot)) { + return issue_fast_sync_overlay_certificate(std::move(issue_by), std::move(issue_to), flags, client.slot, + expire_at, std::move(promise)); + } + } + promise.set_error(td::Status::Error(ton::ErrorCode::error, "cannot issue certificate to unknown adnl id")); +} + +ton::PublicKeyHash ValidatorEngine::find_local_validator_for_cert_issuing() { + if (state_.is_null()) { + return ton::PublicKeyHash{}; + } + for (auto& val_set : {validator_set_, validator_set_next_, validator_set_prev_}) { + if (val_set.is_null()) { + continue; + } + for (auto &[val_id, _]: config_.validators) { + if (val_set->is_validator(ton::NodeIdShort{val_id.bits256_value()})) { + return val_id; + } + } + } + return ton::PublicKeyHash::zero(); +} + void ValidatorEngine::load_custom_overlays_config() { - custom_overlays_config_ = - ton::create_tl_object(); + custom_overlays_config_ = ton::create_tl_object(); auto data_R = td::read_file(custom_overlays_config_file()); if (data_R.is_error()) { return; @@ -2699,7 +2990,7 @@ void ValidatorEngine::del_custom_overlay_from_config(std::string name, td::Promi static td::Result> parse_collator_options(td::MutableSlice json_str) { td::Ref ref{true}; - ton::validator::CollatorOptions& opts = ref.write(); + ton::validator::CollatorOptions &opts = ref.write(); // Set default values (from_json leaves missing fields as is) ton::ton_api::engine_validator_collatorOptions f; @@ -3706,7 +3997,7 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importCer return; } auto r = ton::overlay::Certificate::create(std::move(query.cert_)); - if(r.is_error()) { + if (r.is_error()) { promise.set_value(create_control_query_error(r.move_as_error_prefix("Invalid certificate: "))); } //TODO force Overlays::update_certificate to return result @@ -3721,17 +4012,15 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importCer }); */ td::actor::send_closure(overlay_manager_, &ton::overlay::Overlays::update_certificate, - ton::adnl::AdnlNodeIdShort{query.local_id_->id_}, - ton::overlay::OverlayIdShort{query.overlay_id_}, - ton::PublicKeyHash{query.signed_key_->key_hash_}, - r.move_as_ok()); - promise.set_value( - ton::serialize_tl_object(ton::create_tl_object(), true) - ); + ton::adnl::AdnlNodeIdShort{query.local_id_->id_}, + ton::overlay::OverlayIdShort{query.overlay_id_}, + ton::PublicKeyHash{query.signed_key_->key_hash_}, r.move_as_ok()); + promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); } -void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importShardOverlayCertificate &query, td::BufferSlice data, - ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importShardOverlayCertificate &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { if (!(perm & ValidatorEnginePermissions::vep_modify)) { promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); return; @@ -3746,7 +4035,7 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importSha return; } auto r = ton::overlay::Certificate::create(std::move(query.cert_)); - if(r.is_error()) { + if (r.is_error()) { promise.set_value(create_control_query_error(r.move_as_error_prefix("Invalid certificate: "))); } auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result R) mutable { @@ -3758,12 +4047,13 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importSha } }); ton::ShardIdFull shard_id{ton::WorkchainId{query.workchain_}, static_cast(query.shard_)}; - td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::import_shard_overlay_certificate, - shard_id, ton::PublicKeyHash{query.signed_key_->key_hash_}, r.move_as_ok(), std::move(P)); + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::import_shard_overlay_certificate, shard_id, + ton::PublicKeyHash{query.signed_key_->key_hash_}, r.move_as_ok(), std::move(P)); } -void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_signShardOverlayCertificate &query, td::BufferSlice data, - ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_signShardOverlayCertificate &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { if (!(perm & ValidatorEnginePermissions::vep_modify)) { promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); return; @@ -3785,11 +4075,11 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_signShard promise.set_value(R.move_as_ok()); } }); - td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::sign_shard_overlay_certificate, - shard_id, ton::PublicKeyHash{query.signed_key_->key_hash_}, query.expire_at_, query.max_size_, std::move(P)); + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::sign_shard_overlay_certificate, shard_id, + ton::PublicKeyHash{query.signed_key_->key_hash_}, query.expire_at_, query.max_size_, + std::move(P)); } - void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getOverlaysStats &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { if (!(perm & ValidatorEnginePermissions::vep_default)) { @@ -3857,42 +4147,46 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getPerfTi return; } - auto P = td::PromiseCreator::lambda( - [promise = std::move(promise), query = std::move(query)](td::Result> R) mutable { - const std::vector times{60, 300, 3600}; - double now = td::Time::now(); - if (R.is_error()) { - promise.set_value(create_control_query_error(R.move_as_error())); - } else { - auto r = R.move_as_ok(); - std::vector> by_name; - for (const auto &stats : r) { - if (stats.name == query.name_ || query.name_.empty()) { - std::vector> by_time; - for (const auto &t : times) { - double min = std::numeric_limits::lowest(); - double max = std::numeric_limits::max(); - double sum = 0; - int cnt = 0; - for (const auto &stat : stats.stats) { - double time = stat.first; - double duration = stat.second; - if (now - time <= static_cast(t)) { - min = td::min(min, duration); - max = td::max(max, duration); - sum += duration; - ++cnt; - } - } - by_time.push_back(ton::create_tl_object(t, min, sum / static_cast(cnt), max)); + auto P = td::PromiseCreator::lambda([promise = std::move(promise), query = std::move(query)]( + td::Result> R) mutable { + const std::vector times{60, 300, 3600}; + double now = td::Time::now(); + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + auto r = R.move_as_ok(); + std::vector> by_name; + for (const auto &stats : r) { + if (stats.name == query.name_ || query.name_.empty()) { + std::vector> by_time; + for (const auto &t : times) { + double min = std::numeric_limits::lowest(); + double max = std::numeric_limits::max(); + double sum = 0; + int cnt = 0; + for (const auto &stat : stats.stats) { + double time = stat.first; + double duration = stat.second; + if (now - time <= static_cast(t)) { + min = td::min(min, duration); + max = td::max(max, duration); + sum += duration; + ++cnt; } - by_name.push_back(ton::create_tl_object(stats.name, std::move(by_time))); } + by_time.push_back(ton::create_tl_object( + t, min, sum / static_cast(cnt), max)); } - promise.set_value(ton::create_serialize_tl_object(std::move(by_name))); + by_name.push_back(ton::create_tl_object( + stats.name, std::move(by_time))); } - }); - td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::prepare_perf_timer_stats, std::move(P)); + } + promise.set_value( + ton::create_serialize_tl_object(std::move(by_name))); + } + }); + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::prepare_perf_timer_stats, + std::move(P)); } void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getShardOutQueueSize &query, @@ -4015,9 +4309,8 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setExtMes }); } -void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addCustomOverlay &query, - td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, - td::Promise promise) { +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addCustomOverlay &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { if (!(perm & ValidatorEnginePermissions::vep_modify)) { promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); return; @@ -4086,9 +4379,8 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delCustom }); } -void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_showCustomOverlays &query, - td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, - td::Promise promise) { +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_showCustomOverlays &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { if (!(perm & ValidatorEnginePermissions::vep_default)) { promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); return; @@ -4312,7 +4604,8 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addShard if (R.is_error()) { promise.set_value(create_control_query_error(R.move_as_error())); } else { - promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); } }); } @@ -4348,7 +4641,8 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delShard if (R.is_error()) { promise.set_value(create_control_query_error(R.move_as_error())); } else { - promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); } }); } @@ -4484,7 +4778,8 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addCollat if (R.is_error()) { promise.set_value(create_control_query_error(R.move_as_error())); } else { - promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); } }); } @@ -4529,7 +4824,8 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delCollat if (R.is_error()) { promise.set_value(create_control_query_error(R.move_as_error())); } else { - promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); } }); } @@ -4550,30 +4846,16 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_signOverl ton::adnl::AdnlNodeIdShort adnl_id{query.adnl_id_}; int slot = query.slot_; int expire_at = query.expire_at_; - td::actor::send_closure( - keyring_, &ton::keyring::Keyring::get_public_key, public_key_hash, - [=, keyring = keyring_.get(), promise = std::move(promise)](td::Result R) mutable { + + issue_fast_sync_overlay_certificate( + public_key_hash, adnl_id, 0, slot, expire_at, + [promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_value(create_control_query_error(R.move_as_error())); return; } - ton::overlay::OverlayMemberCertificate certificate{R.move_as_ok(), 0, slot, expire_at, td::BufferSlice{}}; - if (certificate.is_expired()) { - promise.set_value( - create_control_query_error(td::Status::Error(ton::ErrorCode::error, "certificate is expired"))); - return; - } - td::BufferSlice to_sign = certificate.to_sign_data(adnl_id); - td::actor::send_closure(keyring, &ton::keyring::Keyring::sign_message, public_key_hash, std::move(to_sign), - [certificate = std::move(certificate), - promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - promise.set_value(create_control_query_error(R.move_as_error())); - return; - } - certificate.set_signature(R.move_as_ok()); - promise.set_value(ton::serialize_tl_object(certificate.tl(), true)); - }); + auto cert = R.move_as_ok(); + promise.set_value(ton::serialize_tl_object(cert.tl(), true)); }); } @@ -4600,23 +4882,92 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importFas promise.set_value(create_control_query_error(std::move(S))); return; } - for (auto &old_cert : config_.fast_sync_member_certificates) { - if (old_cert.first == adnl_id && old_cert.second.issued_by() == certificate.issued_by() && - old_cert.second.expire_at() == certificate.expire_at() && old_cert.second.slot() == certificate.slot()) { - promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "duplicate certificate"))); - return; - } + + try_import_fast_sync_member_certificate( + std::move(adnl_id), std::move(certificate), [promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); + } + }); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addFastSyncClient &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; } - config_.fast_sync_member_certificates.emplace_back(adnl_id, certificate); - write_config([=, promise = std::move(promise), full_node = full_node_.get()](td::Result R) mutable { + ton::adnl::AdnlNodeIdShort adnl_id{query.adnl_id_}; + td::int32 slot = query.slot_; + if (slot < 0 || slot >= ton::validator::fullnode::FullNode::MAX_FAST_SYNC_OVERLAY_CLIENTS) { + promise.set_value(create_control_query_error(td::Status::Error( + PSTRING() << "invalid slot (max " << ton::validator::fullnode::FullNode::MAX_FAST_SYNC_OVERLAY_CLIENTS + << " clients)"))); + return; + } + + bool found = false; + for (auto &c : config_.fast_sync_overlay_clients) { + if (c.slot == slot) { + if (c.id == adnl_id) { + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); + issue_fast_sync_overlay_certificates(); + } else { + promise.set_value(create_control_query_error(td::Status::Error("duplicate slot"))); + } + return; + } + if (c.id == adnl_id) { + found = true; + c.slot = slot; + } + } + if (!found) { + config_.fast_sync_overlay_clients.emplace_back(adnl_id, slot); + } + write_config([promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); + } + }); + issue_fast_sync_overlay_certificates(); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delFastSyncClient &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + + ton::adnl::AdnlNodeIdShort adnl_id{query.adnl_id_}; + for (auto &c : config_.fast_sync_overlay_clients) { + if (c.id == adnl_id) { + std::swap(c, config_.fast_sync_overlay_clients.back()); + config_.fast_sync_overlay_clients.pop_back(); + break; + } + } + write_config([promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_value(create_control_query_error(R.move_as_error())); } else { - if (!full_node.empty()) { - td::actor::send_closure(full_node, &ton::validator::fullnode::FullNode::import_fast_sync_member_certificate, - adnl_id, std::move(certificate)); - } promise.set_value( ton::serialize_tl_object(ton::create_tl_object(), true)); } @@ -4692,9 +5043,8 @@ void ValidatorEngine::get_current_validator_perm_key(td::Promiseget_total_validator_set(0); - CHECK(val_set.not_null()); - auto vec = val_set->export_vector(); + CHECK(validator_set_.not_null()); + auto vec = validator_set_->export_vector(); for (size_t idx = 0; idx < vec.size(); idx++) { auto &el = vec[idx]; ton::PublicKey pub{ton::pubkeys::Ed25519{el.key.as_bits256()}}; diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 89dd22cc..eac31f46 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -27,11 +27,14 @@ */ #pragma once +#include "adnl/adnl-node-id.hpp" #include "adnl/adnl.h" #include "auto/tl/ton_api.h" +#include "overlays.h" #include "rldp/rldp.h" #include "rldp2/rldp.h" #include "dht/dht.h" +#include "td/actor/PromiseFuture.h" #include "validator/manager.h" #include "validator/validator.h" #include "validator/full-node.h" @@ -75,6 +78,13 @@ struct Config { ton::PublicKey key; td::IPAddress addr; }; + struct FastSyncOverlayClient { + FastSyncOverlayClient() = default; + FastSyncOverlayClient(ton::adnl::AdnlNodeIdShort id, td::int32 slot) : id(id), slot(slot) { + } + ton::adnl::AdnlNodeIdShort id; + td::int32 slot; + }; std::map keys_refcnt; td::uint16 out_port; @@ -93,6 +103,7 @@ struct Config { std::map controls; std::set gc; std::vector shards_to_monitor; + std::vector fast_sync_overlay_clients; bool state_serializer_enabled = true; std::vector> @@ -158,6 +169,7 @@ class ValidatorEngine : public td::actor::Actor { td::actor::ActorOwn validator_manager_; td::actor::ActorOwn full_node_client_; td::actor::ActorOwn full_node_; + ton::adnl::AdnlNodeIdShort full_node_id_ = ton::adnl::AdnlNodeIdShort::zero(); std::map> full_node_masters_; td::actor::ActorOwn control_ext_server_; @@ -187,13 +199,13 @@ class ValidatorEngine : public td::actor::Actor { std::map keys_; td::Ref state_; + td::Ref validator_set_, validator_set_prev_, validator_set_next_; + td::Timestamp issue_fast_sync_overlay_certificates_at_ = td::Timestamp::now(); td::Promise get_key_promise(td::MultiPromise::InitGuard &ig); void got_key(ton::PublicKey key); void deleted_key(ton::PublicKeyHash key); - void got_state(td::Ref state) { - state_ = std::move(state); - } + void got_state(td::Ref state); void write_config(td::Promise promise); @@ -420,6 +432,20 @@ class ValidatorEngine : public td::actor::Actor { void try_del_proxy(td::uint32 ip, td::int32 port, std::vector cats, std::vector prio_cats, td::Promise promise); + void register_fast_sync_certificate_callback(); + void try_import_fast_sync_member_certificate(ton::adnl::AdnlNodeIdShort id, + ton::overlay::OverlayMemberCertificate certificate, + td::Promise promise); + + void issue_fast_sync_overlay_certificates(); + void issue_fast_sync_overlay_certificate(ton::PublicKeyHash issue_by, ton::adnl::AdnlNodeIdShort issue_to, + td::uint32 flags, td::int32 slot, td::int32 expire_at, + td::Promise promise); + void process_fast_sync_overlay_certificate_request(ton::PublicKeyHash issue_by, ton::adnl::AdnlNodeIdShort issue_to, + td::uint32 flags, td::int32 slot, td::int32 expire_at, + td::Promise promise); + ton::PublicKeyHash find_local_validator_for_cert_issuing(); + std::string custom_overlays_config_file() const { return db_root_ + "/custom-overlays.json"; } @@ -432,8 +458,8 @@ class ValidatorEngine : public td::actor::Actor { void load_custom_overlays_config(); td::Status write_custom_overlays_config(); - void add_custom_overlay_to_config( - ton::tl_object_ptr overlay, td::Promise promise); + void add_custom_overlay_to_config(ton::tl_object_ptr overlay, + td::Promise promise); void del_custom_overlay_from_config(std::string name, td::Promise promise); void load_collator_options(); @@ -560,6 +586,10 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_getAdnlStats &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_addFastSyncClient &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_delFastSyncClient &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); template void run_control_query(T &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { diff --git a/validator/full-node-fast-sync-overlays.cpp b/validator/full-node-fast-sync-overlays.cpp index 947f9dee..6554c518 100644 --- a/validator/full-node-fast-sync-overlays.cpp +++ b/validator/full-node-fast-sync-overlays.cpp @@ -308,7 +308,7 @@ void FullNodeFastSyncOverlay::init() { options.default_permanent_members_flags_ = overlay::OverlayMemberFlags::DoNotReceiveBroadcasts; } options.local_overlay_member_flags_ = receive_broadcasts_ ? 0 : overlay::OverlayMemberFlags::DoNotReceiveBroadcasts; - options.max_slaves_in_semiprivate_overlay_ = 100000; // TODO: set lower limit (high limit for testing) + options.max_slaves_in_semiprivate_overlay_ = FullNode::MAX_FAST_SYNC_OVERLAY_CLIENTS; td::actor::send_closure(overlays_, &overlay::Overlays::create_semiprivate_overlay, local_id_, overlay_id_full_.clone(), current_validators_adnl_, root_public_keys_, member_certificate_, std::make_unique(actor_id(this)), rules, std::move(scope), options); diff --git a/validator/full-node.h b/validator/full-node.h index f8fc123b..c9a49472 100644 --- a/validator/full-node.h +++ b/validator/full-node.h @@ -111,6 +111,8 @@ class FullNode : public td::actor::Actor { } enum { broadcast_mode_public = 1, broadcast_mode_private_block = 2, broadcast_mode_custom = 4 }; + static constexpr td::int32 MAX_FAST_SYNC_OVERLAY_CLIENTS = 5000; // TODO: set lower limit (high limit for testing) + static td::actor::ActorOwn create( ton::PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, FullNodeConfig config, td::actor::ActorId keyring, td::actor::ActorId adnl, diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 414b3170..b625e4c2 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -100,7 +100,10 @@ class FullNodeImpl : public FullNode { void set_validator_telemetry_filename(std::string value) override; void import_fast_sync_member_certificate(adnl::AdnlNodeIdShort local_id, - overlay::OverlayMemberCertificate cert) override { + overlay::OverlayMemberCertificate cert) override { + VLOG(FULL_NODE_DEBUG) << "Importing fast sync overlay certificate for " << local_id << " issued by " + << cert.issued_by().compute_short_id() << " expires in " + << (double)cert.expire_at() - td::Clocks::system(); fast_sync_overlays_.add_member_certificate(local_id, std::move(cert)); }