From 21c2727dcf6a0d443718bc41ca95722b85486e53 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 3 May 2022 17:19:31 +0300 Subject: [PATCH 01/29] Increase timeout for downloading astate and add logs --- validator/manager-init.cpp | 8 +++++--- validator/manager.cpp | 2 +- validator/net/download-state.cpp | 8 +++++++- validator/shard-client.cpp | 4 ++-- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/validator/manager-init.cpp b/validator/manager-init.cpp index cca320ff..1a322c02 100644 --- a/validator/manager-init.cpp +++ b/validator/manager-init.cpp @@ -227,7 +227,7 @@ void ValidatorManagerMasterchainReiniter::choose_masterchain_state() { } if (!p || ValidatorManager::is_persistent_state(h->unix_time(), p->unix_time())) { auto ttl = ValidatorManager::persistent_state_ttl(h->unix_time()); - double time_to_download = 3600; + double time_to_download = 3600 * 3; if (ttl > td::Clocks::system() + time_to_download) { handle = h; break; @@ -259,7 +259,7 @@ void ValidatorManagerMasterchainReiniter::download_masterchain_state() { } }); td::actor::create_actor("downloadstate", block_id_, block_id_, 2, manager_, - td::Timestamp::in(3600), std::move(P)) + td::Timestamp::in(3600 * 3), std::move(P)) .release(); } @@ -267,7 +267,7 @@ void ValidatorManagerMasterchainReiniter::downloaded_masterchain_state(td::Ref{std::move(state)}; CHECK(handle_->received_state()); CHECK(handle_->is_applied()); - + LOG(INFO) << "downloaded masterchain state"; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { R.ensure(); td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::downloaded_all_shards); @@ -276,6 +276,7 @@ void ValidatorManagerMasterchainReiniter::downloaded_masterchain_state(td::Ref R) { R.ensure(); @@ -286,6 +287,7 @@ void ValidatorManagerMasterchainReiniter::downloaded_all_shards() { void ValidatorManagerMasterchainReiniter::finish() { CHECK(handle_->id().id.seqno == 0 || handle_->is_key_block()); promise_.set_value(ValidatorManagerInitResult{handle_, state_, std::move(client_), handle_, state_, handle_}); + LOG(INFO) << "persistent state download finished"; stop(); } diff --git a/validator/manager.cpp b/validator/manager.cpp index 3abea6e8..9a8fead6 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1328,7 +1328,7 @@ void ValidatorManagerImpl::send_get_zero_state_request(BlockIdExt id, td::uint32 void ValidatorManagerImpl::send_get_persistent_state_request(BlockIdExt id, BlockIdExt masterchain_block_id, td::uint32 priority, td::Promise promise) { - callback_->download_persistent_state(id, masterchain_block_id, priority, td::Timestamp::in(3600.0), + callback_->download_persistent_state(id, masterchain_block_id, priority, td::Timestamp::in(3600 * 3), std::move(promise)); } diff --git a/validator/net/download-state.cpp b/validator/net/download-state.cpp index 0c71133b..49986ce9 100644 --- a/validator/net/download-state.cpp +++ b/validator/net/download-state.cpp @@ -53,7 +53,7 @@ DownloadState::DownloadState(BlockIdExt block_id, BlockIdExt masterchain_block_i void DownloadState::abort_query(td::Status reason) { if (promise_) { if (reason.code() == ErrorCode::notready || reason.code() == ErrorCode::timeout) { - VLOG(FULL_NODE_DEBUG) << "failed to download state " << block_id_ << "from " << download_from_ << ": " << reason; + VLOG(FULL_NODE_DEBUG) << "failed to download state " << block_id_ << " from " << download_from_ << ": " << reason; } else { VLOG(FULL_NODE_NOTICE) << "failed to download state " << block_id_ << " from " << download_from_ << ": " << reason; @@ -115,6 +115,7 @@ void DownloadState::got_block_handle(BlockHandle handle) { void DownloadState::got_node_to_download(adnl::AdnlNodeIdShort node) { download_from_ = node; + LOG(INFO) << "downloading state " << block_id_ << " from " << download_from_; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) mutable { if (R.is_error()) { @@ -187,6 +188,10 @@ void DownloadState::got_block_state_part(td::BufferSlice data, td::uint32 reques sum_ += data.size(); parts_.push_back(std::move(data)); + if (sum_ % (1 << 22) == 0) { + LOG(DEBUG) << "downloading state " << block_id_ << ": total=" << sum_; + } + if (last_part) { td::BufferSlice res{td::narrow_cast(sum_)}; auto S = res.as_slice(); @@ -224,6 +229,7 @@ void DownloadState::got_block_state_part(td::BufferSlice data, td::uint32 reques void DownloadState::got_block_state(td::BufferSlice data) { state_ = std::move(data); + LOG(INFO) << "finished downloading state " << block_id_ << ": total=" << sum_; finish_query(); } diff --git a/validator/shard-client.cpp b/validator/shard-client.cpp index 5cafd3ca..c7d87fb4 100644 --- a/validator/shard-client.cpp +++ b/validator/shard-client.cpp @@ -98,8 +98,8 @@ void ShardClient::start_up_init_mode() { }); td::actor::create_actor("downloadstate", shard->top_block_id(), - masterchain_block_handle_->id(), 2, manager_, td::Timestamp::in(3600), - std::move(P)) + masterchain_block_handle_->id(), 2, manager_, + td::Timestamp::in(3600 * 3), std::move(P)) .release(); } } From 678a8a6a1363131766cd6e8c89738f6d6dc87d29 Mon Sep 17 00:00:00 2001 From: OmicronTau Date: Sat, 13 Nov 2021 16:17:17 +0300 Subject: [PATCH 02/29] Add overlay certificate checks --- catchain/catchain-receiver.cpp | 2 +- overlay/overlay-broadcast.cpp | 45 ++++++++++++++++++-- overlay/overlay-broadcast.hpp | 20 +++++---- overlay/overlay-fec-broadcast.cpp | 11 ++++- overlay/overlay-fec-broadcast.hpp | 4 +- overlay/overlay-manager.cpp | 67 ++++++++++++++++++++++-------- overlay/overlay.cpp | 43 ++++++++++++------- overlay/overlay.hpp | 5 ++- overlay/overlays.h | 48 +++++++++++++++++---- tl/generate/scheme/ton_api.tl | 2 + tl/generate/scheme/ton_api.tlo | Bin 64168 -> 64644 bytes validator/full-node-shard.cpp | 27 +++++++++--- validator/full-node-shard.hpp | 3 ++ validator/manager-disk.hpp | 3 ++ validator/manager-hardfork.hpp | 3 ++ validator/manager.hpp | 5 +++ validator/validator.h | 6 +-- 17 files changed, 232 insertions(+), 62 deletions(-) diff --git a/catchain/catchain-receiver.cpp b/catchain/catchain-receiver.cpp index b9b0373f..e7f5c019 100644 --- a/catchain/catchain-receiver.cpp +++ b/catchain/catchain-receiver.cpp @@ -486,7 +486,7 @@ void CatChainReceiverImpl::start_up() { } td::actor::send_closure(overlay_manager_, &overlay::Overlays::create_private_overlay, get_source(local_idx_)->get_adnl_id(), overlay_full_id_.clone(), std::move(ids), - make_callback(), overlay::OverlayPrivacyRules{0, std::move(root_keys)}); + make_callback(), overlay::OverlayPrivacyRules{0, 0, std::move(root_keys)}); CHECK(root_block_); diff --git a/overlay/overlay-broadcast.cpp b/overlay/overlay-broadcast.cpp index bade50e7..1c05f705 100644 --- a/overlay/overlay-broadcast.cpp +++ b/overlay/overlay-broadcast.cpp @@ -17,8 +17,14 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "overlay-broadcast.hpp" +#include "adnl/adnl-node-id.hpp" +#include "common/util.h" #include "overlay.hpp" #include "keys/encryptor.h" +#include "td/actor/PromiseFuture.h" +#include "td/actor/actor.h" +#include "td/utils/Status.h" +#include "td/utils/port/Stat.h" namespace ton { @@ -33,7 +39,13 @@ td::Status BroadcastSimple::check_duplicate() { } td::Status BroadcastSimple::check_source() { - return overlay_->check_source_eligible(source_, cert_.get(), data_size()); + auto r = overlay_->check_source_eligible(source_, cert_.get(), data_size(), false); + if (r == BroadcastCheckResult::Forbidden) { + return td::Status::Error(ErrorCode::error, "broadcast is forbidden"); + } + + is_valid_ = r == BroadcastCheckResult::Allowed; + return td::Status::OK(); } td::BufferSlice BroadcastSimple::to_sign() { @@ -66,6 +78,14 @@ td::Status BroadcastSimple::distribute() { return td::Status::OK(); } +void BroadcastSimple::broadcast_checked(td::Result R) { + if (R.is_error()) { + return; + } + is_valid_ = true; + run_continue().ignore(); +} + tl_object_ptr BroadcastSimple::tl() const { return create_tl_object(source_.tl(), cert_ ? cert_->tl() : Certificate::empty_tl(), flags_, data_.clone(), date_, signature_.clone()); @@ -75,6 +95,25 @@ td::BufferSlice BroadcastSimple::serialize() { return serialize_tl_object(tl(), true); } +td::Status BroadcastSimple::run_continue() { + TRY_STATUS(distribute()); + deliver(); + return td::Status::OK(); +} + +td::Status BroadcastSimple::run() { + TRY_STATUS(run_checks()); + if (!is_valid_) { + auto P = td::PromiseCreator::lambda( + [id = broadcast_hash_, overlay_id = actor_id(overlay_)](td::Result R) mutable { + td::actor::send_closure(std::move(overlay_id), &OverlayImpl::broadcast_checked, id, std::move(R)); + }); + overlay_->check_broadcast(source_.compute_short_id(), data_.clone(), std::move(P)); + return td::Status::OK(); + } + return run_continue(); +} + td::Status BroadcastSimple::create(OverlayImpl *overlay, tl_object_ptr broadcast) { auto src = PublicKey{broadcast->src_}; auto data_hash = sha256_bits256(broadcast->data_.as_slice()); @@ -86,7 +125,7 @@ td::Status BroadcastSimple::create(OverlayImpl *overlay, tl_object_ptr(broadcast_hash, src, std::move(cert), broadcast->flags_, std::move(broadcast->data_), broadcast->date_, - std::move(broadcast->signature_), overlay); + std::move(broadcast->signature_), false, overlay); TRY_STATUS(B->run()); overlay->register_simple_broadcast(std::move(B)); return td::Status::OK(); @@ -100,7 +139,7 @@ td::Status BroadcastSimple::create_new(td::actor::ActorId overlay, auto date = static_cast(td::Clocks::system()); auto B = std::make_unique(broadcast_hash, PublicKey{}, nullptr, flags, std::move(data), date, - td::BufferSlice{}, nullptr); + td::BufferSlice{}, false, nullptr); auto to_sign = B->to_sign(); auto P = td::PromiseCreator::lambda( diff --git a/overlay/overlay-broadcast.hpp b/overlay/overlay-broadcast.hpp index 665c269d..da29695a 100644 --- a/overlay/overlay-broadcast.hpp +++ b/overlay/overlay-broadcast.hpp @@ -18,9 +18,16 @@ */ #pragma once +#include "adnl/adnl-local-id.h" +#include "adnl/adnl-node-id.hpp" #include "auto/tl/ton_api.h" +#include "common/refcnt.hpp" #include "overlay/overlay.h" +#include "td/actor/PromiseFuture.h" #include "td/utils/List.h" +#include "td/utils/Status.h" +#include "td/utils/buffer.h" +#include "td/utils/common.h" namespace ton { @@ -38,6 +45,7 @@ class BroadcastSimple : public td::ListNode { td::BufferSlice data_; td::uint32 date_; td::BufferSlice signature_; + bool is_valid_{false}; OverlayImpl *overlay_; @@ -52,7 +60,7 @@ class BroadcastSimple : public td::ListNode { public: BroadcastSimple(Overlay::BroadcastHash broadcast_hash, PublicKey source, std::shared_ptr cert, - td::uint32 flags, td::BufferSlice data, td::uint32 date, td::BufferSlice signature, + td::uint32 flags, td::BufferSlice data, td::uint32 date, td::BufferSlice signature, bool is_valid, OverlayImpl *overlay) : broadcast_hash_(broadcast_hash) , source_(std::move(source)) @@ -61,6 +69,7 @@ class BroadcastSimple : public td::ListNode { , data_(std::move(data)) , date_(date) , signature_(std::move(signature)) + , is_valid_(is_valid) , overlay_(overlay) { } @@ -80,17 +89,14 @@ class BroadcastSimple : public td::ListNode { } void deliver(); - td::Status run() { - TRY_STATUS(run_checks()); - TRY_STATUS(distribute()); - deliver(); - return td::Status::OK(); - } + td::Status run(); + td::Status run_continue(); tl_object_ptr tl() const; td::BufferSlice serialize(); void update_overlay(OverlayImpl *overlay); + void broadcast_checked(td::Result R); static td::Status create(OverlayImpl *overlay, tl_object_ptr broadcast); static td::Status create_new(td::actor::ActorId overlay, td::actor::ActorId keyring, diff --git a/overlay/overlay-fec-broadcast.cpp b/overlay/overlay-fec-broadcast.cpp index e368c620..6ab03800 100644 --- a/overlay/overlay-fec-broadcast.cpp +++ b/overlay/overlay-fec-broadcast.cpp @@ -54,7 +54,16 @@ td::Status OverlayFecBroadcastPart::check_duplicate() { } td::Status OverlayFecBroadcastPart::check_source() { - TRY_STATUS(overlay_->check_source_eligible(source_, cert_.get(), broadcast_size_)); + auto r = overlay_->check_source_eligible(source_, cert_.get(), broadcast_size_, true); + if (r == BroadcastCheckResult::Forbidden) { + return td::Status::Error(ErrorCode::error, "broadcast is forbidden"); + } + + // FIXME + if (r == BroadcastCheckResult::NeedCheck) { + return td::Status::Error(ErrorCode::error, "broadcast is forbidden"); + } + if (bcast_) { TRY_STATUS(bcast_->is_eligible_sender(source_)); } diff --git a/overlay/overlay-fec-broadcast.hpp b/overlay/overlay-fec-broadcast.hpp index 6d30d47c..0ce9c2ff 100644 --- a/overlay/overlay-fec-broadcast.hpp +++ b/overlay/overlay-fec-broadcast.hpp @@ -185,6 +185,9 @@ class BroadcastFec : public td::ListNode { } } + void broadcast_checked(td::Result R) { + } + private: bool ready_ = false; @@ -311,4 +314,3 @@ class OverlayFecBroadcastPart : public td::ListNode { } // namespace overlay } // namespace ton - diff --git a/overlay/overlay-manager.cpp b/overlay/overlay-manager.cpp index ae7c3dd5..059ad4e2 100644 --- a/overlay/overlay-manager.cpp +++ b/overlay/overlay-manager.cpp @@ -17,6 +17,7 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "overlay-manager.h" +#include "auto/tl/ton_api.h" #include "overlay.h" #include "adnl/utils.hpp" @@ -268,17 +269,21 @@ void OverlayManager::save_to_db(adnl::AdnlNodeIdShort local_id, OverlayIdShort o db_.set(key, create_serialize_tl_object(std::move(obj))); } -Certificate::Certificate(PublicKey issued_by, td::int32 expire_at, td::uint32 max_size, td::BufferSlice signature) +Certificate::Certificate(PublicKey issued_by, td::int32 expire_at, td::uint32 max_size, td::uint32 flags, + td::BufferSlice signature) : issued_by_(issued_by) , expire_at_(expire_at) , max_size_(max_size) + , flags_(flags) , signature_(td::SharedSlice(signature.as_slice())) { } -Certificate::Certificate(PublicKeyHash issued_by, td::int32 expire_at, td::uint32 max_size, td::BufferSlice signature) +Certificate::Certificate(PublicKeyHash issued_by, td::int32 expire_at, td::uint32 max_size, td::uint32 flags, + td::BufferSlice signature) : issued_by_(issued_by) , expire_at_(expire_at) , max_size_(max_size) + , flags_(flags) , signature_(td::SharedSlice(signature.as_slice())) { } @@ -290,9 +295,19 @@ void Certificate::set_issuer(PublicKey issuer) { issued_by_ = issuer; } +constexpr td::uint32 cert_default_flags(td::uint32 max_size) { + return (max_size > Overlays::max_simple_broadcast_size() ? CertificateFlags::AllowFec : 0) | + CertificateFlags::Trusted; +} + td::BufferSlice Certificate::to_sign(OverlayIdShort overlay_id, PublicKeyHash issued_to) const { - return create_serialize_tl_object(overlay_id.tl(), issued_to.tl(), expire_at_, - max_size_); + if (flags_ == cert_default_flags(max_size_)) { + return create_serialize_tl_object(overlay_id.tl(), issued_to.tl(), expire_at_, + max_size_); + } else { + return create_serialize_tl_object(overlay_id.tl(), issued_to.tl(), expire_at_, + max_size_, flags_); + } } const PublicKeyHash Certificate::issuer_hash() const { @@ -307,32 +322,48 @@ const PublicKey &Certificate::issuer() const { td::Result> Certificate::create(tl_object_ptr cert) { std::shared_ptr res; - ton_api::downcast_call(*cert.get(), td::overloaded([&](ton_api::overlay_emptyCertificate &obj) { res = nullptr; }, - [&](ton_api::overlay_certificate &obj) { - res = std::make_shared( - PublicKey{obj.issued_by_}, obj.expire_at_, - static_cast(obj.max_size_), - std::move(obj.signature_)); - })); + ton_api::downcast_call(*cert.get(), + td::overloaded([&](ton_api::overlay_emptyCertificate &obj) { res = nullptr; }, + [&](ton_api::overlay_certificate &obj) { + res = std::make_shared(PublicKey{obj.issued_by_}, obj.expire_at_, + static_cast(obj.max_size_), + cert_default_flags(obj.max_size_), + std::move(obj.signature_)); + }, + [&](ton_api::overlay_certificateV2 &obj) { + res = std::make_shared(PublicKey{obj.issued_by_}, obj.expire_at_, + static_cast(obj.max_size_), + static_cast(obj.flags_), + std::move(obj.signature_)); + })); return std::move(res); } -td::Status Certificate::check(PublicKeyHash node, OverlayIdShort overlay_id, td::int32 unix_time, - td::uint32 size) const { +BroadcastCheckResult Certificate::check(PublicKeyHash node, OverlayIdShort overlay_id, td::int32 unix_time, + td::uint32 size, bool is_fec) const { if (size > max_size_) { - return td::Status::Error(ErrorCode::protoviolation, "too big broadcast size"); + return BroadcastCheckResult::Forbidden; } if (unix_time > expire_at_) { - return td::Status::Error(ErrorCode::protoviolation, "too old certificate"); + return BroadcastCheckResult::Forbidden; + } + if (is_fec && !(flags_ & CertificateFlags::AllowFec)) { + return BroadcastCheckResult::Forbidden; } - TRY_RESULT(E, issued_by_.get().create_encryptor()); + auto R1 = issued_by_.get().create_encryptor(); + if (R1.is_error()) { + return BroadcastCheckResult::Forbidden; + } + auto E = R1.move_as_ok(); auto B = to_sign(overlay_id, node); - TRY_STATUS(E->check_signature(B.as_slice(), signature_.as_slice())); + if (E->check_signature(B.as_slice(), signature_.as_slice()).is_error()) { + return BroadcastCheckResult::Forbidden; + } - return td::Status::OK(); + return (flags_ & CertificateFlags::Trusted) ? BroadcastCheckResult::Allowed : BroadcastCheckResult::NeedCheck; } tl_object_ptr Certificate::tl() const { diff --git a/overlay/overlay.cpp b/overlay/overlay.cpp index 47019c16..21ec9363 100644 --- a/overlay/overlay.cpp +++ b/overlay/overlay.cpp @@ -391,25 +391,21 @@ td::Status OverlayImpl::check_date(td::uint32 date) { return td::Status::OK(); } -td::Status OverlayImpl::check_source_eligible(PublicKey source, const Certificate *cert, td::uint32 size) { +BroadcastCheckResult OverlayImpl::check_source_eligible(PublicKey source, const Certificate *cert, td::uint32 size, + bool is_fec) { if (size == 0) { - return td::Status::Error(ErrorCode::protoviolation, "empty broadcast"); + return BroadcastCheckResult::Forbidden; } auto short_id = source.compute_short_id(); - auto r = rules_.max_size(source.compute_short_id()); - if (r >= size) { - return td::Status::OK(); + auto r = rules_.check_rules(source.compute_short_id(), size, is_fec); + if (!cert || r == BroadcastCheckResult::Allowed) { + return r; } - if (!cert) { - return td::Status::Error(ErrorCode::protoviolation, "source is not eligible"); - } - TRY_STATUS(cert->check(short_id, overlay_id_, static_cast(td::Clocks::system()), size)); - auto issuer_short = cert->issuer_hash(); - if (rules_.max_size(issuer_short) < size) { - return td::Status::Error(ErrorCode::protoviolation, "bad certificate"); - } - return td::Status::OK(); + + auto r2 = cert->check(short_id, overlay_id_, static_cast(td::Clocks::system()), size, is_fec); + r2 = broadcast_check_result_min(r2, rules_.check_rules(cert->issuer_hash(), size, is_fec)); + return broadcast_check_result_max(r, r2); } td::Status OverlayImpl::check_delivered(BroadcastHash hash) { @@ -539,6 +535,25 @@ void OverlayImpl::set_privacy_rules(OverlayPrivacyRules rules) { rules_ = std::move(rules); } +void OverlayImpl::check_broadcast(PublicKeyHash src, td::BufferSlice data, td::Promise promise) { + callback_->check_broadcast(src, overlay_id_, std::move(data), std::move(promise)); +} + +void OverlayImpl::broadcast_checked(Overlay::BroadcastHash hash, td::Result R) { + { + auto it = broadcasts_.find(hash); + if (it != broadcasts_.end()) { + it->second->broadcast_checked(std::move(R)); + } + } + { + auto it = fec_broadcasts_.find(hash); + if (it != fec_broadcasts_.end()) { + it->second->broadcast_checked(std::move(R)); + } + } +} + } // namespace overlay } // namespace ton diff --git a/overlay/overlay.hpp b/overlay/overlay.hpp index b087fc75..cbddd654 100644 --- a/overlay/overlay.hpp +++ b/overlay/overlay.hpp @@ -144,9 +144,12 @@ class OverlayImpl : public Overlay { void print(td::StringBuilder &sb) override; td::Status check_date(td::uint32 date); - td::Status check_source_eligible(PublicKey source, const Certificate *cert, td::uint32 size); + BroadcastCheckResult check_source_eligible(PublicKey source, const Certificate *cert, td::uint32 size, bool is_fec); td::Status check_delivered(BroadcastHash hash); + void broadcast_checked(Overlay::BroadcastHash hash, td::Result R); + void check_broadcast(PublicKeyHash src, td::BufferSlice data, td::Promise promise); + BroadcastFec *get_fec_broadcast(BroadcastHash hash); void register_fec_broadcast(std::unique_ptr bcast); void register_simple_broadcast(std::unique_ptr bcast); diff --git a/overlay/overlays.h b/overlay/overlays.h index 90f3877e..1ad55453 100644 --- a/overlay/overlays.h +++ b/overlay/overlays.h @@ -21,7 +21,11 @@ #include "adnl/adnl.h" #include "dht/dht.h" +#include "td/actor/PromiseFuture.h" #include "td/actor/actor.h" +#include "td/utils/Status.h" +#include "td/utils/buffer.h" +#include "td/utils/common.h" #include @@ -80,41 +84,64 @@ class OverlayIdFull { td::BufferSlice name_; }; +struct CertificateFlags { + enum Values : td::uint32 { AllowFec = 1, Trusted = 2 }; +}; + +enum BroadcastCheckResult { Forbidden = 1, NeedCheck = 2, Allowed = 3 }; + +inline BroadcastCheckResult broadcast_check_result_max(BroadcastCheckResult l, BroadcastCheckResult r) { + return static_cast(std::max(static_cast(l), static_cast(r))); +} +inline BroadcastCheckResult broadcast_check_result_min(BroadcastCheckResult l, BroadcastCheckResult r) { + return static_cast(std::min(static_cast(l), static_cast(r))); +} + class OverlayPrivacyRules { public: OverlayPrivacyRules() { } OverlayPrivacyRules(td::uint32 size) : max_unath_size_(size) { } - OverlayPrivacyRules(td::uint32 max_size, std::map authorized_keys) - : max_unath_size_(max_size), authorized_keys_(std::move(authorized_keys)) { + OverlayPrivacyRules(td::uint32 max_size, td::uint32 flags, std::map authorized_keys) + : max_unath_size_(max_size), flags_(flags), authorized_keys_(std::move(authorized_keys)) { } - td::uint32 max_size(PublicKeyHash hash) { + BroadcastCheckResult check_rules(PublicKeyHash hash, td::uint32 size, bool is_fec) { auto it = authorized_keys_.find(hash); if (it == authorized_keys_.end()) { - return max_unath_size_; + if (size > max_unath_size_) { + return BroadcastCheckResult::Forbidden; + } + if (!(flags_ & CertificateFlags::AllowFec) && is_fec) { + return BroadcastCheckResult::Forbidden; + } + return (flags_ & CertificateFlags::Trusted) ? BroadcastCheckResult::Allowed : BroadcastCheckResult::NeedCheck; } else { - return it->second; + return it->second >= size ? BroadcastCheckResult::Allowed : BroadcastCheckResult::Forbidden; } } private: td::uint32 max_unath_size_{0}; + td::uint32 flags_{0}; std::map authorized_keys_; }; class Certificate { public: - Certificate(PublicKeyHash issued_by, td::int32 expire_at, td::uint32 max_size, td::BufferSlice signature); - Certificate(PublicKey issued_by, td::int32 expire_at, td::uint32 max_size, td::BufferSlice signature); + Certificate(PublicKeyHash issued_by, td::int32 expire_at, td::uint32 max_size, td::uint32 flags, + td::BufferSlice signature); + Certificate(PublicKey issued_by, td::int32 expire_at, td::uint32 max_size, td::uint32 flags, + td::BufferSlice signature); Certificate() { } void set_signature(td::BufferSlice signature); void set_issuer(PublicKey issuer); td::BufferSlice to_sign(OverlayIdShort overlay_id, PublicKeyHash issued_to) const; - td::Status check(PublicKeyHash node, OverlayIdShort overlay_id, td::int32 unix_time, td::uint32 size) const; + BroadcastCheckResult check(PublicKeyHash node, OverlayIdShort overlay_id, td::int32 unix_time, td::uint32 size, + bool is_fec) const; tl_object_ptr tl() const; const PublicKey &issuer() const; const PublicKeyHash issuer_hash() const; @@ -126,6 +153,7 @@ class Certificate { td::Variant issued_by_; td::int32 expire_at_; td::uint32 max_size_; + td::uint32 flags_; td::SharedSlice signature_; }; @@ -137,6 +165,10 @@ class Overlays : public td::actor::Actor { virtual void receive_query(adnl::AdnlNodeIdShort src, OverlayIdShort overlay_id, td::BufferSlice data, td::Promise promise) = 0; virtual void receive_broadcast(PublicKeyHash src, OverlayIdShort overlay_id, td::BufferSlice data) = 0; + virtual void check_broadcast(PublicKeyHash src, OverlayIdShort overlay_id, td::BufferSlice data, + td::Promise promise) { + promise.set_value(td::Unit()); + } virtual ~Callback() = default; }; diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 57ed9ddc..b226ca7f 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -219,9 +219,11 @@ overlay.broadcastFec.partId broadcast_hash:int256 data_hash:int256 seqno:int = o overlay.broadcast.toSign hash:int256 date:int = overlay.broadcast.ToSign; overlay.certificate issued_by:PublicKey expire_at:int max_size:int signature:bytes = overlay.Certificate; +overlay.certificateV2 issued_by:PublicKey expire_at:int max_size:int flags:int signature:bytes = overlay.Certificate; overlay.emptyCertificate = overlay.Certificate; overlay.certificateId overlay_id:int256 node:int256 expire_at:int max_size:int = overlay.CertificateId; +overlay.certificateIdV2 overlay_id:int256 node:int256 expire_at:int max_size:int flags:int = overlay.CertificateId; overlay.unicast data:bytes = overlay.Broadcast; overlay.broadcast src:PublicKey certificate:overlay.Certificate flags:int data:bytes date:int signature:bytes = overlay.Broadcast; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 823219b5455c649d4cc8f04b6231dcdd439c8c16..32f42b3c4843343c874d1a1a2696af299ab58f1c 100644 GIT binary patch delta 298 zcmZ4SmAU07^9Bt`$=dJLX~OwssYN-7m3q#pMJ1VOnaPPIsmv1{#aZMw9V(t2BPlV_ zL1J@{WCa(a#b#EO_4%wIO$?J6Gu000^VMZXe5KCXq=$|a0 R!8(~YQyFa3=9ZZ%i2&9tb(jDE delta 162 zcmZqq$-Lq#^9Bt`iGSHHmxS}nQj2mDEA^aHi%K%nGLsWaQYSh{u promise) override { + td::actor::send_closure(node_, &FullNodeShardImpl::check_broadcast, src, std::move(data), std::move(promise)); + } Callback(td::actor::ActorId node) : node_(node) { } @@ -95,6 +101,17 @@ void FullNodeShardImpl::create_overlay() { } } +void FullNodeShardImpl::check_broadcast(PublicKeyHash src, td::BufferSlice broadcast, td::Promise promise) { + auto B = fetch_tl_object(std::move(broadcast), true); + if (B.is_error()) { + return promise.set_error(B.move_as_error_prefix("failed to parse external message broadcast: ")); + } + + auto q = B.move_as_ok(); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::check_external_message, + std::move(q->message_->data_), std::move(promise)); +} + void FullNodeShardImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) { td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, adnl_id_, overlay_id_); adnl_id_ = adnl_id; @@ -804,8 +821,9 @@ void FullNodeShardImpl::sign_new_certificate(PublicKeyHash sign_by) { return; } - ton::overlay::Certificate cert{sign_by, static_cast(td::Clocks::system() + 3600), - overlay::Overlays::max_fec_broadcast_size(), td::BufferSlice{}}; + ton::overlay::Certificate cert{ + sign_by, static_cast(td::Clocks::system() + 3600), overlay::Overlays::max_fec_broadcast_size(), + overlay::CertificateFlags::Trusted | overlay::CertificateFlags::AllowFec, td::BufferSlice{}}; auto to_sign = cert.to_sign(overlay_id_, local_id_); auto P = td::PromiseCreator::lambda( @@ -845,7 +863,7 @@ void FullNodeShardImpl::update_validators(std::vector public_key_ authorized_keys.emplace(key, overlay::Overlays::max_fec_broadcast_size()); } - rules_ = overlay::OverlayPrivacyRules{1 << 14, std::move(authorized_keys)}; + rules_ = overlay::OverlayPrivacyRules{1 << 14, 0, std::move(authorized_keys)}; td::actor::send_closure(overlays_, &overlay::Overlays::set_privacy_rules, adnl_id_, overlay_id_, rules_); if (update_cert) { @@ -949,8 +967,7 @@ void FullNodeShardImpl::update_neighbour_stats(adnl::AdnlNodeIdShort adnl_id, do } } -void FullNodeShardImpl::got_neighbour_capabilities(adnl::AdnlNodeIdShort adnl_id, double t, - td::BufferSlice data) { +void FullNodeShardImpl::got_neighbour_capabilities(adnl::AdnlNodeIdShort adnl_id, double t, td::BufferSlice data) { auto it = neighbours_.find(adnl_id); if (it == neighbours_.end()) { return; diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index 4aad637b..2c468374 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -19,6 +19,8 @@ #pragma once #include "full-node-shard.h" +#include "td/actor/PromiseFuture.h" +#include "td/utils/port/Poll.h" namespace ton { @@ -139,6 +141,7 @@ class FullNodeShardImpl : public FullNodeShard { void process_broadcast(PublicKeyHash src, ton_api::tonNode_externalMessageBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query); void receive_broadcast(PublicKeyHash src, td::BufferSlice query); + void check_broadcast(PublicKeyHash src, td::BufferSlice query, td::Promise promise); void send_ihr_message(td::BufferSlice data) override; void send_external_message(td::BufferSlice data) override; diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 71818e6e..fa365ccf 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -124,6 +124,9 @@ class ValidatorManagerImpl : public ValidatorManager { //void get_block_description(BlockIdExt block_id, td::Promise promise) override; void new_external_message(td::BufferSlice data) override; + void check_external_message(td::BufferSlice data, td::Promise promise) override { + UNREACHABLE(); + } void new_ihr_message(td::BufferSlice data) override; void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override; diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 506ee036..9938bcb2 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -144,6 +144,9 @@ class ValidatorManagerImpl : public ValidatorManager { void get_key_block_proof_link(BlockIdExt block_id, td::Promise promise) override; void new_external_message(td::BufferSlice data) override; + void check_external_message(td::BufferSlice data, td::Promise promise) override { + UNREACHABLE(); + } void new_ihr_message(td::BufferSlice data) override; void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override { UNREACHABLE(); diff --git a/validator/manager.hpp b/validator/manager.hpp index 672085d0..08de5060 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -20,6 +20,8 @@ #include "interfaces/validator-manager.h" #include "interfaces/db.h" +#include "td/actor/PromiseFuture.h" +#include "td/utils/port/Poll.h" #include "validator-group.hpp" #include "shard-client.hpp" #include "manager-init.h" @@ -325,6 +327,9 @@ class ValidatorManagerImpl : public ValidatorManager { //void get_block_description(BlockIdExt block_id, td::Promise promise) override; void new_external_message(td::BufferSlice data) override; + void check_external_message(td::BufferSlice data, td::Promise promise) override { + promise.set_value(td::Unit()); + } void new_ihr_message(td::BufferSlice data) override; void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override; diff --git a/validator/validator.h b/validator/validator.h index 54c35ee7..0cbdbe00 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -94,9 +94,8 @@ struct ValidatorManagerOptions : public td::CntObject { BlockIdExt zero_block_id, BlockIdExt init_block_id, std::function check_shard = [](ShardIdFull, CatchainSeqno, ShardCheckMode) { return true; }, - bool allow_blockchain_init = false, double sync_blocks_before = 300, - double block_ttl = 86400 * 7, double state_ttl = 3600, - double archive_ttl = 86400 * 365, double key_proof_ttl = 86400 * 3650, + bool allow_blockchain_init = false, double sync_blocks_before = 300, double block_ttl = 86400 * 7, + double state_ttl = 3600, double archive_ttl = 86400 * 365, double key_proof_ttl = 86400 * 3650, bool initial_sync_disabled = false); }; @@ -176,6 +175,7 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void write_handle(BlockHandle handle, td::Promise promise) = 0; virtual void new_external_message(td::BufferSlice data) = 0; + virtual void check_external_message(td::BufferSlice data, td::Promise promise) = 0; virtual void new_ihr_message(td::BufferSlice data) = 0; virtual void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) = 0; From 69d0472510d44850df14e1cf674f4c55bffc97d5 Mon Sep 17 00:00:00 2001 From: OmicronTau Date: Sat, 13 Nov 2021 17:15:19 +0300 Subject: [PATCH 03/29] Add mempool messages cap --- validator-engine/validator-engine.cpp | 7 +++++++ validator-engine/validator-engine.hpp | 4 ++++ validator/manager.cpp | 3 +++ validator/manager.hpp | 3 +++ validator/validator-options.cpp | 3 ++- validator/validator-options.hpp | 10 +++++++++- validator/validator.h | 3 +++ 7 files changed, 31 insertions(+), 2 deletions(-) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 16f71d5d..9c9d4a98 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1309,6 +1309,9 @@ td::Status ValidatorEngine::load_global_config() { if (state_ttl_ != 0) { validator_options_.write().set_state_ttl(state_ttl_); } + if (max_mempool_num_ != 0) { + validator_options_.write().set_max_mempool_num(max_mempool_num_); + } if (block_ttl_ != 0) { validator_options_.write().set_block_ttl(block_ttl_); } @@ -3336,6 +3339,10 @@ int main(int argc, char *argv[]) { auto v = td::to_double(fname); acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_state_ttl, v); }); }); + p.add_option('m', "mempool-num", "Maximal number of mempool external message", [&](td::Slice fname) { + auto v = td::to_double(fname); + acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_max_mempool_num, v); }); + }); p.add_option('b', "block-ttl", "blocks will be gc'd after this time (in seconds) default=7*86400", [&](td::Slice fname) { auto v = td::to_double(fname); diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 8ce56ab4..07078f95 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -191,6 +191,7 @@ class ValidatorEngine : public td::actor::Actor { std::map control_permissions_; double state_ttl_ = 0; + double max_mempool_num_ = 0; double block_ttl_ = 0; double sync_ttl_ = 0; double archive_ttl_ = 0; @@ -223,6 +224,9 @@ class ValidatorEngine : public td::actor::Actor { void set_state_ttl(double t) { state_ttl_ = t; } + void set_max_mempool_num(double t) { + max_mempool_num_ = t; + } void set_block_ttl(double t) { block_ttl_ = t; } diff --git a/validator/manager.cpp b/validator/manager.cpp index 3abea6e8..1741502d 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -366,6 +366,9 @@ void ValidatorManagerImpl::new_external_message(td::BufferSlice data) { if (!is_validator()) { return; } + if( ext_messages_.size() > max_mempool_num() ) { + return; + } auto R = create_ext_message(std::move(data)); if (R.is_error()) { VLOG(VALIDATOR_NOTICE) << "dropping bad ihr message: " << R.move_as_error(); diff --git a/validator/manager.hpp b/validator/manager.hpp index 08de5060..c2e41a30 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -580,6 +580,9 @@ class ValidatorManagerImpl : public ValidatorManager { double block_ttl() const { return opts_->block_ttl(); } + double max_mempool_num() const { + return opts_->max_mempool_num(); + } private: std::map> shard_client_waiters_; diff --git a/validator/validator-options.cpp b/validator/validator-options.cpp index 30bb050b..93fe05e6 100644 --- a/validator/validator-options.cpp +++ b/validator/validator-options.cpp @@ -27,10 +27,11 @@ namespace validator { td::Ref ValidatorManagerOptions::create( BlockIdExt zero_block_id, BlockIdExt init_block_id, std::function check_shard, bool allow_blockchain_init, - double sync_blocks_before, double block_ttl, double state_ttl, + double sync_blocks_before, double block_ttl, double state_ttl, double max_mempool_num, double archive_ttl, double key_proof_ttl, bool initial_sync_disabled) { return td::make_ref(zero_block_id, init_block_id, std::move(check_shard), allow_blockchain_init, sync_blocks_before, block_ttl, state_ttl, + max_mempool_num, archive_ttl, key_proof_ttl, initial_sync_disabled); } diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index 92597156..e794166f 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -50,6 +50,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { double state_ttl() const override { return state_ttl_; } + double max_mempool_num() const override { + return max_mempool_num_; + } double archive_ttl() const override { return archive_ttl_; } @@ -130,6 +133,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_state_ttl(double value) override { state_ttl_ = value; } + void set_max_mempool_num(double value) override { + max_mempool_num_ = value; + } void set_archive_ttl(double value) override { archive_ttl_ = value; } @@ -163,7 +169,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { ValidatorManagerOptionsImpl(BlockIdExt zero_block_id, BlockIdExt init_block_id, std::function check_shard, bool allow_blockchain_init, double sync_blocks_before, - double block_ttl, double state_ttl, + double block_ttl, double state_ttl, double max_mempool_num, double archive_ttl, double key_proof_ttl, bool initial_sync_disabled) : zero_block_id_(zero_block_id) @@ -173,6 +179,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { , sync_blocks_before_(sync_blocks_before) , block_ttl_(block_ttl) , state_ttl_(state_ttl) + , max_mempool_num_(max_mempool_num) , archive_ttl_(archive_ttl) , key_proof_ttl_(key_proof_ttl) , initial_sync_disabled_(initial_sync_disabled) { @@ -186,6 +193,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { double sync_blocks_before_; double block_ttl_; double state_ttl_; + double max_mempool_num_; double archive_ttl_; double key_proof_ttl_; bool initial_sync_disabled_; diff --git a/validator/validator.h b/validator/validator.h index 0cbdbe00..349824d6 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -56,6 +56,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual double sync_blocks_before() const = 0; virtual double block_ttl() const = 0; virtual double state_ttl() const = 0; + virtual double max_mempool_num() const = 0; virtual double archive_ttl() const = 0; virtual double key_proof_ttl() const = 0; virtual bool initial_sync_disabled() const = 0; @@ -81,6 +82,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual void set_sync_blocks_before(double value) = 0; virtual void set_block_ttl(double value) = 0; virtual void set_state_ttl(double value) = 0; + virtual void set_max_mempool_num(double value) = 0; virtual void set_archive_ttl(double value) = 0; virtual void set_key_proof_ttl(double value) = 0; virtual void set_initial_sync_disabled(bool value) = 0; @@ -96,6 +98,7 @@ struct ValidatorManagerOptions : public td::CntObject { ShardCheckMode) { return true; }, bool allow_blockchain_init = false, double sync_blocks_before = 300, double block_ttl = 86400 * 7, double state_ttl = 3600, double archive_ttl = 86400 * 365, double key_proof_ttl = 86400 * 3650, + double max_mempool_num = 999999, bool initial_sync_disabled = false); }; From 3384d204d2235fec1373722cf6f58c9f4f081b13 Mon Sep 17 00:00:00 2001 From: OmicronTau Date: Sat, 13 Nov 2021 23:28:06 +0300 Subject: [PATCH 04/29] Add checks for external messages --- validator/fabric.h | 4 + validator/impl/collator-impl.h | 21 +++ validator/impl/collator.cpp | 232 ++++++++++++++---------- validator/impl/external-message.cpp | 100 +++++++++- validator/impl/external-message.hpp | 21 ++- validator/impl/fabric.cpp | 9 + validator/impl/liteserver.cpp | 75 ++++++-- validator/impl/liteserver.hpp | 12 ++ validator/interfaces/external-message.h | 2 + validator/manager.cpp | 29 ++- validator/manager.hpp | 10 +- 11 files changed, 396 insertions(+), 119 deletions(-) diff --git a/validator/fabric.h b/validator/fabric.h index a10251dd..67b6ae9e 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -46,6 +46,8 @@ td::Result>> create_new_shard_bloc td::Ref create_signature_set(std::vector sig_set); +void run_check_external_message(td::BufferSlice data, td::actor::ActorId manager, td::Promise promise); + void run_accept_block_query(BlockIdExt id, td::Ref data, std::vector prev, td::Ref validator_set, td::Ref signatures, td::Ref approve_signatures, bool send_broadcast, @@ -81,6 +83,8 @@ void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_b td::Promise promise); void run_liteserver_query(td::BufferSlice data, td::actor::ActorId manager, td::actor::ActorId cache, td::Promise promise); +void run_fetch_account_state(WorkchainId wc, StdSmcAddress addr, td::actor::ActorId manager, + td::Promise,UnixTime,LogicalTime,std::unique_ptr>> promise); void run_validate_shard_block_description(td::BufferSlice data, BlockHandle masterchain_block, td::Ref masterchain_state, td::actor::ActorId manager, td::Timestamp timeout, diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 87aa5494..2dced749 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -103,6 +103,27 @@ class Collator final : public td::actor::Actor { return 2; } + static td::Result> + impl_fetch_config_params(std::unique_ptr config, + Ref* old_mparams, + std::vector* storage_prices, + block::StoragePhaseConfig* storage_phase_cfg, + td::BitArray<256>* rand_seed, + block::ComputePhaseConfig* compute_phase_cfg, + block::ActionPhaseConfig* action_phase_cfg, + td::RefInt256* masterchain_create_fee, + td::RefInt256* basechain_create_fee, + WorkchainId wc); + + static td::Result> + impl_create_ordinary_transaction(Ref msg_root, + block::Account* acc, + UnixTime utime, LogicalTime lt, + block::StoragePhaseConfig* storage_phase_cfg, + block::ComputePhaseConfig* compute_phase_cfg, + block::ActionPhaseConfig* action_phase_cfg, + bool external, LogicalTime after_lt); + private: void start_up() override; void alarm() override; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 9ca4a6cc..6b1c72a6 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -1556,68 +1556,92 @@ bool Collator::init_lt() { } bool Collator::fetch_config_params() { - old_mparams_ = config_->get_config_param(9); - { - auto res = config_->get_storage_prices(); - if (res.is_error()) { + auto res = impl_fetch_config_params(std::move(config_), + &old_mparams_, &storage_prices_, &storage_phase_cfg_, + &rand_seed_, &compute_phase_cfg_, &action_phase_cfg_, + &masterchain_create_fee_, &basechain_create_fee_, + workchain() + ); + if (res.is_error()) { return fatal_error(res.move_as_error()); + } + config_ = res.move_as_ok(); + return true; +} + +td::Result> + Collator::impl_fetch_config_params(std::unique_ptr config, + Ref* old_mparams, + std::vector* storage_prices, + block::StoragePhaseConfig* storage_phase_cfg, + td::BitArray<256>* rand_seed, + block::ComputePhaseConfig* compute_phase_cfg, + block::ActionPhaseConfig* action_phase_cfg, + td::RefInt256* masterchain_create_fee, + td::RefInt256* basechain_create_fee, + WorkchainId wc) { + *old_mparams = config->get_config_param(9); + { + auto res = config->get_storage_prices(); + if (res.is_error()) { + return res.move_as_error(); } - storage_prices_ = res.move_as_ok(); + *storage_prices = res.move_as_ok(); } { // generate rand seed - prng::rand_gen().strong_rand_bytes(rand_seed_.data(), 32); - LOG(DEBUG) << "block random seed set to " << rand_seed_.to_hex(); + prng::rand_gen().strong_rand_bytes(rand_seed->data(), 32); + LOG(DEBUG) << "block random seed set to " << rand_seed->to_hex(); } { // compute compute_phase_cfg / storage_phase_cfg - auto cell = config_->get_config_param(is_masterchain() ? 20 : 21); + auto cell = config->get_config_param(wc == ton::masterchainId ? 20 : 21); if (cell.is_null()) { - return fatal_error("cannot fetch current gas prices and limits from masterchain configuration"); + return td::Status::Error(-668, "cannot fetch current gas prices and limits from masterchain configuration"); } - if (!compute_phase_cfg_.parse_GasLimitsPrices(std::move(cell), storage_phase_cfg_.freeze_due_limit, - storage_phase_cfg_.delete_due_limit)) { - return fatal_error("cannot unpack current gas prices and limits from masterchain configuration"); + if (!compute_phase_cfg->parse_GasLimitsPrices(std::move(cell), storage_phase_cfg->freeze_due_limit, + storage_phase_cfg->delete_due_limit)) { + return td::Status::Error(-668, "cannot unpack current gas prices and limits from masterchain configuration"); } - compute_phase_cfg_.block_rand_seed = rand_seed_; - compute_phase_cfg_.libraries = std::make_unique(config_->get_libraries_root(), 256); - compute_phase_cfg_.global_config = config_->get_root_cell(); + compute_phase_cfg->block_rand_seed = *rand_seed; + compute_phase_cfg->libraries = std::make_unique(config->get_libraries_root(), 256); + compute_phase_cfg->global_config = config->get_root_cell(); } { // compute action_phase_cfg block::gen::MsgForwardPrices::Record rec; - auto cell = config_->get_config_param(24); + auto cell = config->get_config_param(24); if (cell.is_null() || !tlb::unpack_cell(std::move(cell), rec)) { - return fatal_error("cannot fetch masterchain message transfer prices from masterchain configuration"); + return td::Status::Error(-668, "cannot fetch masterchain message transfer prices from masterchain configuration"); } - action_phase_cfg_.fwd_mc = + action_phase_cfg->fwd_mc = block::MsgPrices{rec.lump_price, rec.bit_price, rec.cell_price, rec.ihr_price_factor, (unsigned)rec.first_frac, (unsigned)rec.next_frac}; - cell = config_->get_config_param(25); + cell = config->get_config_param(25); if (cell.is_null() || !tlb::unpack_cell(std::move(cell), rec)) { - return fatal_error("cannot fetch standard message transfer prices from masterchain configuration"); + return td::Status::Error(-668, "cannot fetch standard message transfer prices from masterchain configuration"); } - action_phase_cfg_.fwd_std = + action_phase_cfg->fwd_std = block::MsgPrices{rec.lump_price, rec.bit_price, rec.cell_price, rec.ihr_price_factor, (unsigned)rec.first_frac, (unsigned)rec.next_frac}; - action_phase_cfg_.workchains = &config_->get_workchain_list(); - action_phase_cfg_.bounce_msg_body = (config_->has_capability(ton::capBounceMsgBody) ? 256 : 0); + action_phase_cfg->workchains = &config->get_workchain_list(); + action_phase_cfg->bounce_msg_body = (config->has_capability(ton::capBounceMsgBody) ? 256 : 0); } { // fetch block_grams_created - auto cell = config_->get_config_param(14); + auto cell = config->get_config_param(14); if (cell.is_null()) { - basechain_create_fee_ = masterchain_create_fee_ = td::zero_refint(); + *basechain_create_fee = *masterchain_create_fee = td::zero_refint(); } else { block::gen::BlockCreateFees::Record create_fees; if (!(tlb::unpack_cell(cell, create_fees) && - block::tlb::t_Grams.as_integer_to(create_fees.masterchain_block_fee, masterchain_create_fee_) && - block::tlb::t_Grams.as_integer_to(create_fees.basechain_block_fee, basechain_create_fee_))) { - return fatal_error("cannot unpack BlockCreateFees from configuration parameter #14"); + block::tlb::t_Grams.as_integer_to(create_fees.masterchain_block_fee, *masterchain_create_fee) && + block::tlb::t_Grams.as_integer_to(create_fees.basechain_block_fee, *basechain_create_fee))) { + return td::Status::Error(-668, "cannot unpack BlockCreateFees from configuration parameter #14"); } } } - return true; + return std::move(config); } bool Collator::compute_minted_amount(block::CurrencyCollection& to_mint) { @@ -2218,75 +2242,25 @@ Ref Collator::create_ordinary_transaction(Ref msg_root) { } block::Account* acc = acc_res.move_as_ok(); assert(acc); - if (acc->last_trans_end_lt_ >= start_lt && acc->transactions.empty()) { - fatal_error(PSTRING() << "last transaction time in the state of account " << workchain() << ":" << addr.to_hex() - << " is too large"); - return {}; - } - auto trans_min_lt = start_lt; - if (external) { - // transactions processing external messages must have lt larger than all processed internal messages - trans_min_lt = std::max(trans_min_lt, last_proc_int_msg_.first); - } - std::unique_ptr trans = - std::make_unique(*acc, block::Transaction::tr_ord, trans_min_lt + 1, now_, msg_root); - bool ihr_delivered = false; // FIXME - if (!trans->unpack_input_msg(ihr_delivered, &action_phase_cfg_)) { - if (external) { - // inbound external message was not accepted - LOG(DEBUG) << "inbound external message rejected by account " << addr.to_hex() - << " before smart-contract execution"; + + + auto res = impl_create_ordinary_transaction(msg_root, acc, now_, start_lt, + &storage_phase_cfg_, &compute_phase_cfg_, + &action_phase_cfg_, + external, last_proc_int_msg_.first + ); + if(res.is_error()) { + auto error = res.move_as_error(); + if(error.code() == -701) { + // ignorable errors + LOG(DEBUG) << error.message(); return {}; } - fatal_error("cannot unpack input message for a new transaction"); - return {}; - } - if (trans->bounce_enabled) { - if (!trans->prepare_storage_phase(storage_phase_cfg_, true)) { - fatal_error("cannot create storage phase of a new transaction for smart contract "s + addr.to_hex()); - return {}; - } - if (!external && !trans->prepare_credit_phase()) { - fatal_error("cannot create credit phase of a new transaction for smart contract "s + addr.to_hex()); - return {}; - } - } else { - if (!external && !trans->prepare_credit_phase()) { - fatal_error("cannot create credit phase of a new transaction for smart contract "s + addr.to_hex()); - return {}; - } - if (!trans->prepare_storage_phase(storage_phase_cfg_, true, true)) { - fatal_error("cannot create storage phase of a new transaction for smart contract "s + addr.to_hex()); - return {}; - } - } - if (!trans->prepare_compute_phase(compute_phase_cfg_)) { - fatal_error("cannot create compute phase of a new transaction for smart contract "s + addr.to_hex()); - return {}; - } - if (!trans->compute_phase->accepted) { - if (external) { - // inbound external message was not accepted - LOG(DEBUG) << "inbound external message rejected by transaction " << addr.to_hex(); - return {}; - } else if (trans->compute_phase->skip_reason == block::ComputePhase::sk_none) { - fatal_error("new ordinary transaction for smart contract "s + addr.to_hex() + - " has not been accepted by the smart contract (?)"); - return {}; - } - } - if (trans->compute_phase->success && !trans->prepare_action_phase(action_phase_cfg_)) { - fatal_error("cannot create action phase of a new transaction for smart contract "s + addr.to_hex()); - return {}; - } - if (trans->bounce_enabled && !trans->compute_phase->success && !trans->prepare_bounce_phase(action_phase_cfg_)) { - fatal_error("cannot create bounce phase of a new transaction for smart contract "s + addr.to_hex()); - return {}; - } - if (!trans->serialize()) { - fatal_error("cannot serialize new transaction for smart contract "s + addr.to_hex()); + fatal_error(std::move(error)); return {}; } + std::unique_ptr trans = res.move_as_ok(); + if (!trans->update_limits(*block_limit_status_)) { fatal_error("cannot update block limit status to include the new transaction"); return {}; @@ -2296,6 +2270,7 @@ Ref Collator::create_ordinary_transaction(Ref msg_root) { fatal_error("cannot commit new transaction for smart contract "s + addr.to_hex()); return {}; } + register_new_msgs(*trans); update_max_lt(acc->last_trans_end_lt_); // temporary patch to stop producing dangerous block @@ -2305,6 +2280,75 @@ Ref Collator::create_ordinary_transaction(Ref msg_root) { return trans_root; } +// If td::status::error_code == 669 - Fatal Error block can not be produced +// if td::status::error_code == 701 - Transaction can not be included into block, but it's ok (external or too early internal) +td::Result> Collator::impl_create_ordinary_transaction(Ref msg_root, + block::Account* acc, + UnixTime utime, LogicalTime lt, + block::StoragePhaseConfig* storage_phase_cfg, + block::ComputePhaseConfig* compute_phase_cfg, + block::ActionPhaseConfig* action_phase_cfg, + bool external, LogicalTime after_lt) { + if (acc->last_trans_end_lt_ >= lt && acc->transactions.empty()) { + return td::Status::Error(-669, PSTRING() << "last transaction time in the state of account " << acc->workchain << ":" << acc->addr.to_hex() + << " is too large"); + } + auto trans_min_lt = lt; + if (external) { + // transactions processing external messages must have lt larger than all processed internal messages + trans_min_lt = std::max(trans_min_lt, after_lt); + } + + std::unique_ptr trans = + std::make_unique(*acc, block::Transaction::tr_ord, trans_min_lt + 1, utime, msg_root); + bool ihr_delivered = false; // FIXME + if (!trans->unpack_input_msg(ihr_delivered, action_phase_cfg)) { + if (external) { + // inbound external message was not accepted + return td::Status::Error(-701,"inbound external message rejected by account "s + acc->addr.to_hex() + + " before smart-contract execution"); + } + return td::Status::Error(-669,"cannot unpack input message for a new transaction"); + } + if (trans->bounce_enabled) { + if (!trans->prepare_storage_phase(*storage_phase_cfg, true)) { + return td::Status::Error(-669,"cannot create storage phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } + if (!external && !trans->prepare_credit_phase()) { + return td::Status::Error(-669,"cannot create credit phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } + } else { + if (!external && !trans->prepare_credit_phase()) { + return td::Status::Error(-669,"cannot create credit phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } + if (!trans->prepare_storage_phase(*storage_phase_cfg, true, true)) { + return td::Status::Error(-669,"cannot create storage phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } + } + if (!trans->prepare_compute_phase(*compute_phase_cfg)) { + return td::Status::Error(-669,"cannot create compute phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } + if (!trans->compute_phase->accepted) { + if (external) { + // inbound external message was not accepted + return td::Status::Error(-701,"inbound external message rejected by transaction "s + acc->addr.to_hex()); + } else if (trans->compute_phase->skip_reason == block::ComputePhase::sk_none) { + return td::Status::Error(-669,"new ordinary transaction for smart contract "s + acc->addr.to_hex() + + " has not been accepted by the smart contract (?)"); + } + } + if (trans->compute_phase->success && !trans->prepare_action_phase(*action_phase_cfg)) { + return td::Status::Error(-669,"cannot create action phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } + if (trans->bounce_enabled && !trans->compute_phase->success && !trans->prepare_bounce_phase(*action_phase_cfg)) { + return td::Status::Error(-669,"cannot create bounce phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } + if (!trans->serialize()) { + return td::Status::Error(-669,"cannot serialize new transaction for smart contract "s + acc->addr.to_hex()); + } + return std::move(trans); +} + void Collator::update_max_lt(ton::LogicalTime lt) { CHECK(lt >= start_lt); if (lt > max_lt) { diff --git a/validator/impl/external-message.cpp b/validator/impl/external-message.cpp index 6af721b2..c5f2d097 100644 --- a/validator/impl/external-message.cpp +++ b/validator/impl/external-message.cpp @@ -16,19 +16,25 @@ Copyright 2017-2020 Telegram Systems LLP */ + #include "external-message.hpp" +#include "collator-impl.h" #include "vm/boc.h" #include "block/block-parse.h" #include "block/block-auto.h" #include "block/block-db.h" +#include "fabric.h" +#include "td/actor/actor.h" +#include "td/utils/Random.h" +#include "crypto/openssl/rand.hpp" namespace ton { namespace validator { using td::Ref; -ExtMessageQ::ExtMessageQ(td::BufferSlice data, td::Ref root, AccountIdPrefixFull addr_prefix) - : root_(std::move(root)), addr_prefix_(addr_prefix), data_(std::move(data)) { +ExtMessageQ::ExtMessageQ(td::BufferSlice data, td::Ref root, AccountIdPrefixFull addr_prefix, ton::WorkchainId wc, ton::StdSmcAddress addr) + : root_(std::move(root)), addr_prefix_(addr_prefix), data_(std::move(data)), wc_(wc), addr_(addr) { hash_ = block::compute_file_hash(data_); } @@ -70,7 +76,95 @@ td::Result> ExtMessageQ::create_ext_message(td::BufferSlice dat if (!dest_prefix.is_valid()) { return td::Status::Error("destination of an inbound external message is an invalid blockchain address"); } - return Ref{true, std::move(data), std::move(ext_msg), dest_prefix}; + ton::StdSmcAddress addr; + ton::WorkchainId wc; + if(!block::tlb::t_MsgAddressInt.extract_std_address(info.dest, wc, addr)) { + return td::Status::Error(PSLICE() << "Can't parse destination address"); + } + + return Ref{true, std::move(data), std::move(ext_msg), dest_prefix, wc, addr}; +} + +void ExtMessageQ::run_message(td::BufferSlice data, td::actor::ActorId manager, + td::Promise promise) { + auto R = create_ext_message(std::move(data)); + if (R.is_error()) { + return promise.set_error(R.move_as_error_prefix("failed to parse external message ")); + } + auto M = R.move_as_ok(); + auto root = M->root_cell(); + block::gen::CommonMsgInfo::Record_ext_in_msg_info info; + tlb::unpack_cell_inexact(root, info); // checked in create message + ton::StdSmcAddress addr = M->addr(); + ton::WorkchainId wc = M->wc(); + + run_fetch_account_state(wc, addr, manager, + [promise = std::move(promise), msg_root = root, wc = wc](td::Result,UnixTime,LogicalTime,std::unique_ptr>> res) mutable { + if (res.is_error()) { + promise.set_error(td::Status::Error(PSLICE() << "Failed to get account state")); + } else { + auto tuple = res.move_as_ok(); + block::Account acc; + auto shard_acc = std::move(std::get<0>(tuple)); + auto utime = std::get<1>(tuple); + auto lt = std::get<2>(tuple); + auto config = std::move(std::get<3>(tuple)); + if(!acc.unpack(shard_acc, {}, utime, false)) { + promise.set_error(td::Status::Error(PSLICE() << "Failed to unpack account state")); + } else { + if(run_message_on_account(wc, &acc, utime, lt + 1, msg_root, std::move(config))) { + promise.set_value(td::Unit()); + } else { + promise.set_error(td::Status::Error(PSLICE() << "External message was not accepted")); + } + } + } + } + ); +} + +bool ExtMessageQ::run_message_on_account(ton::WorkchainId wc, + block::Account* acc, + UnixTime utime, LogicalTime lt, + td::Ref msg_root, + std::unique_ptr config) { + + Ref old_mparams; + std::vector storage_prices_; + block::StoragePhaseConfig storage_phase_cfg_{&storage_prices_}; + td::BitArray<256> rand_seed_; + block::ComputePhaseConfig compute_phase_cfg_; + block::ActionPhaseConfig action_phase_cfg_; + td::RefInt256 masterchain_create_fee, basechain_create_fee; + + auto fetch_res = Collator::impl_fetch_config_params(std::move(config), &old_mparams, + &storage_prices_, &storage_phase_cfg_, + &rand_seed_, &compute_phase_cfg_, + &action_phase_cfg_, &masterchain_create_fee, + &basechain_create_fee, wc); + if(fetch_res.is_error()) { + auto error = fetch_res.move_as_error(); + LOG(DEBUG) << "Cannot fetch config params" << error.message(); + return false; + } + + auto res = Collator::impl_create_ordinary_transaction(msg_root, acc, utime, lt, + &storage_phase_cfg_, &compute_phase_cfg_, + &action_phase_cfg_, + true, lt); + if(res.is_error()) { + auto error = res.move_as_error(); + LOG(DEBUG) << "Cannot run message on account" << error.message(); + return false; + } + std::unique_ptr trans = res.move_as_ok(); + + auto trans_root = trans->commit(*acc); + if (trans_root.is_null()) { + LOG(DEBUG) << "cannot commit new transaction for smart contract "; + return false; + } + return true; } } // namespace validator diff --git a/validator/impl/external-message.hpp b/validator/impl/external-message.hpp index 5bb52b11..230258cd 100644 --- a/validator/impl/external-message.hpp +++ b/validator/impl/external-message.hpp @@ -18,9 +18,11 @@ */ #pragma once +#include "interfaces/validator-manager.h" #include "validator/interfaces/external-message.h" #include "auto/tl/ton_api.h" #include "adnl/utils.hpp" +#include "block/transaction.h" namespace ton { @@ -31,6 +33,8 @@ class ExtMessageQ : public ExtMessage { AccountIdPrefixFull addr_prefix_; td::BufferSlice data_; Hash hash_; + ton::WorkchainId wc_; + ton::StdSmcAddress addr_; public: static constexpr unsigned max_ext_msg_size = 65535; @@ -47,8 +51,23 @@ class ExtMessageQ : public ExtMessage { Hash hash() const override { return hash_; } - ExtMessageQ(td::BufferSlice data, td::Ref root, AccountIdPrefixFull shard); + ton::WorkchainId wc() const override { + return wc_; + } + + ton::StdSmcAddress addr() const override { + return addr_; + } + + ExtMessageQ(td::BufferSlice data, td::Ref root, AccountIdPrefixFull shard, ton::WorkchainId wc, ton::StdSmcAddress addr); static td::Result> create_ext_message(td::BufferSlice data); + static void run_message(td::BufferSlice data, td::actor::ActorId manager, + td::Promise promise); + static bool run_message_on_account(ton::WorkchainId wc, + block::Account* acc, + UnixTime utime, LogicalTime lt, + td::Ref msg_root, + std::unique_ptr config); }; } // namespace validator diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index ac296eb4..196595ae 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -116,6 +116,10 @@ td::Result> create_ext_message(td::BufferSlice data) { return std::move(res); } +void run_check_external_message(td::BufferSlice data, td::actor::ActorId manager, td::Promise promise) { + ExtMessageQ::run_message(std::move(data), std::move(manager), std::move(promise)); +} + td::Result> create_ihr_message(td::BufferSlice data) { TRY_RESULT(res, IhrMessageQ::create_ihr_message(std::move(data))); return std::move(res); @@ -237,6 +241,11 @@ void run_liteserver_query(td::BufferSlice data, td::actor::ActorId manager, + td::Promise,UnixTime,LogicalTime,std::unique_ptr>> promise) { + LiteQuery::fetch_account_state(wc, addr, std::move(manager), std::move(promise)); +} + void run_validate_shard_block_description(td::BufferSlice data, BlockHandle masterchain_block, td::Ref masterchain_state, td::actor::ActorId manager, td::Timestamp timeout, diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index b3c0f3eb..510f4906 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -58,15 +58,28 @@ void LiteQuery::run_query(td::BufferSlice data, td::actor::ActorId("litequery", std::move(data), std::move(manager), std::move(promise)).release(); } +void LiteQuery::fetch_account_state(WorkchainId wc, StdSmcAddress acc_addr, td::actor::ActorId manager, + td::Promise,UnixTime,LogicalTime,std::unique_ptr>> promise) { + td::actor::create_actor("litequery", wc, acc_addr, std::move(manager), std::move(promise)).release(); +} + LiteQuery::LiteQuery(td::BufferSlice data, td::actor::ActorId manager, td::Promise promise) : query_(std::move(data)), manager_(std::move(manager)), promise_(std::move(promise)) { timeout_ = td::Timestamp::in(default_timeout_msec * 0.001); } +LiteQuery::LiteQuery(WorkchainId wc, StdSmcAddress acc_addr, td::actor::ActorId manager, + td::Promise,UnixTime,LogicalTime,std::unique_ptr>> promise) + : manager_(std::move(manager)), acc_state_promise_(std::move(promise)), acc_workchain_(wc), acc_addr_(acc_addr) { + timeout_ = td::Timestamp::in(default_timeout_msec * 0.001); +} + void LiteQuery::abort_query(td::Status reason) { LOG(INFO) << "aborted liteserver query: " << reason.to_string(); - if (promise_) { + if (acc_state_promise_) { + acc_state_promise_.set_error(std::move(reason)); + } else if (promise_) { promise_.set_error(std::move(reason)); } stop(); @@ -111,6 +124,11 @@ bool LiteQuery::finish_query(td::BufferSlice result) { void LiteQuery::start_up() { alarm_timestamp() = timeout_; + if(acc_state_promise_) { + td::actor::send_closure_later(actor_id(this),&LiteQuery::perform_fetchAccountState); + return; + } + auto F = fetch_tl_object(std::move(query_), true); if (F.is_error()) { abort_query(F.move_as_error()); @@ -205,17 +223,23 @@ void LiteQuery::perform_getMasterchainInfo(int mode) { } td::actor::send_closure_later( manager_, &ton::validator::ValidatorManager::get_top_masterchain_state_block, - [Self = actor_id(this), mode](td::Result, BlockIdExt>> res) { + [Self = actor_id(this), return_state = bool(acc_state_promise_), mode](td::Result, BlockIdExt>> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { auto pair = res.move_as_ok(); - td::actor::send_closure_later(Self, &LiteQuery::continue_getMasterchainInfo, std::move(pair.first), + auto func = return_state ? &LiteQuery::gotMasterchainInfoForAccountState : &LiteQuery::continue_getMasterchainInfo; + td::actor::send_closure_later(Self, func, std::move(pair.first), pair.second, mode); } }); } +void LiteQuery::gotMasterchainInfoForAccountState(Ref mc_state, BlockIdExt blkid, + int mode) { + perform_getAccountState(blkid, acc_workchain_, acc_addr_, 0x80000000); +} + void LiteQuery::continue_getMasterchainInfo(Ref mc_state, BlockIdExt blkid, int mode) { LOG(INFO) << "obtained data for getMasterchainInfo() : last block = " << blkid.to_str(); @@ -431,15 +455,25 @@ void LiteQuery::continue_getZeroState(BlockIdExt blkid, td::BufferSlice state) { void LiteQuery::perform_sendMessage(td::BufferSlice data) { LOG(INFO) << "started a sendMessage(<" << data.size() << " bytes>) liteserver query"; - auto res = ton::validator::create_ext_message(std::move(data)); - if (res.is_error()) { - abort_query(res.move_as_error()); - return; - } - LOG(INFO) << "sending an external message to validator manager"; - td::actor::send_closure_later(manager_, &ValidatorManager::send_external_message, res.move_as_ok()); - auto b = ton::create_serialize_tl_object(1); - finish_query(std::move(b)); + td::actor::send_closure_later( + manager_, &ValidatorManager::check_external_message, data.clone(), + [Self = actor_id(this), data = std::move(data), manager = manager_](td::Result res) { + if(res.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, + res.move_as_error_prefix("cannot apply external message to current state : "s)); + } else { + auto crm = ton::validator::create_ext_message(data.clone()); + if (crm.is_error()) { + //UNREACHABLE, checks in check_external_message, + td::actor::send_closure(Self, &LiteQuery::abort_query, + crm.move_as_error()); + } + LOG(INFO) << "sending an external message to validator manager"; + td::actor::send_closure_later(manager, &ValidatorManager::send_external_message, crm.move_as_ok()); + auto b = ton::create_serialize_tl_object(1); + td::actor::send_closure(Self, &LiteQuery::finish_query, std::move(b)); + } + }); } bool LiteQuery::request_mc_block_data(BlockIdExt blkid) { @@ -702,6 +736,10 @@ void LiteQuery::continue_getAccountState_0(Ref request_mc_block_data(blkid); } +void LiteQuery::perform_fetchAccountState() { + perform_getMasterchainInfo(-1); +} + void LiteQuery::perform_runSmcMethod(BlockIdExt blkid, WorkchainId workchain, StdSmcAddress addr, int mode, td::int64 method_id, td::BufferSlice params) { LOG(INFO) << "started a runSmcMethod(" << blkid.to_str() << ", " << workchain << ", " << addr.to_hex() << ", " @@ -1010,6 +1048,19 @@ void LiteQuery::finish_getAccountState(td::BufferSlice shard_proof) { } vm::AugmentedDictionary accounts_dict{vm::load_cell_slice_ref(sstate.accounts), 256, block::tlb::aug_ShardAccounts}; auto acc_csr = accounts_dict.lookup(acc_addr_); + if (mode_ & 0x80000000) { + auto config = block::ConfigInfo::extract_config(mc_state_->root_cell(), 0xFFFF); + if (config.is_error()) { + fatal_error(config.move_as_error()); + return; + } + auto rconfig = config.move_as_ok(); + acc_state_promise_.set_value(std::make_tuple( + std::move(acc_csr), sstate.gen_utime, sstate.gen_lt, std::move(rconfig) + )); + return; + } + Ref acc_root; if (acc_csr.not_null()) { acc_root = acc_csr->prefetch_ref(); diff --git a/validator/impl/liteserver.hpp b/validator/impl/liteserver.hpp index f6353c41..24392085 100644 --- a/validator/impl/liteserver.hpp +++ b/validator/impl/liteserver.hpp @@ -26,6 +26,8 @@ #include "block.hpp" #include "shard.hpp" #include "proof.hpp" +#include "block/block-auto.h" + namespace ton { @@ -37,6 +39,9 @@ class LiteQuery : public td::actor::Actor { td::actor::ActorId manager_; td::Timestamp timeout_; td::Promise promise_; + + td::Promise,UnixTime,LogicalTime,std::unique_ptr>> acc_state_promise_; + int pending_{0}; int mode_{0}; WorkchainId acc_workchain_; @@ -71,9 +76,14 @@ class LiteQuery : public td::actor::Actor { }; // version 1.1; +1 = build block proof chains, +2 = masterchainInfoExt, +4 = runSmcMethod LiteQuery(td::BufferSlice data, td::actor::ActorId manager, td::Promise promise); + LiteQuery(WorkchainId wc, StdSmcAddress acc_addr, td::actor::ActorId manager, + td::Promise,UnixTime,LogicalTime,std::unique_ptr>> promise); static void run_query(td::BufferSlice data, td::actor::ActorId manager, td::Promise promise); + static void fetch_account_state(WorkchainId wc, StdSmcAddress acc_addr, td::actor::ActorId manager, + td::Promise,UnixTime,LogicalTime,std::unique_ptr>> promise); + private: bool fatal_error(td::Status error); bool fatal_error(std::string err_msg, int err_code = -400); @@ -87,6 +97,7 @@ class LiteQuery : public td::actor::Actor { void perform_getVersion(); void perform_getMasterchainInfo(int mode); void continue_getMasterchainInfo(Ref mc_state, BlockIdExt blkid, int mode); + void gotMasterchainInfoForAccountState(Ref mc_state, BlockIdExt blkid, int mode); void perform_getBlock(BlockIdExt blkid); void continue_getBlock(BlockIdExt blkid, Ref block); void perform_getBlockHeader(BlockIdExt blkid, int mode); @@ -99,6 +110,7 @@ class LiteQuery : public td::actor::Actor { void continue_getAccountState_0(Ref mc_state, BlockIdExt blkid); void continue_getAccountState(); void finish_getAccountState(td::BufferSlice shard_proof); + void perform_fetchAccountState(); void perform_runSmcMethod(BlockIdExt blkid, WorkchainId workchain, StdSmcAddress addr, int mode, td::int64 method_id, td::BufferSlice params); void finish_runSmcMethod(td::BufferSlice shard_proof, td::BufferSlice state_proof, Ref acc_root, diff --git a/validator/interfaces/external-message.h b/validator/interfaces/external-message.h index f6d9d600..871c624a 100644 --- a/validator/interfaces/external-message.h +++ b/validator/interfaces/external-message.h @@ -35,6 +35,8 @@ class ExtMessage : public td::CntObject { virtual td::BufferSlice serialize() const = 0; virtual td::Ref root_cell() const = 0; virtual Hash hash() const = 0; + virtual ton::WorkchainId wc() const = 0; + virtual ton::StdSmcAddress addr() const = 0; }; } // namespace validator diff --git a/validator/manager.cpp b/validator/manager.cpp index 1741502d..7fcf0274 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -374,13 +374,25 @@ void ValidatorManagerImpl::new_external_message(td::BufferSlice data) { VLOG(VALIDATOR_NOTICE) << "dropping bad ihr message: " << R.move_as_error(); return; } - auto M = std::make_unique>(R.move_as_ok()); - auto id = M->ext_id(); - if (ext_messages_hashes_.count(id.hash) == 0) { - ext_messages_.emplace(id, std::move(M)); - ext_messages_hashes_.emplace(id.hash, id); + add_external_message(R.move_as_ok()); +} + +void ValidatorManagerImpl::add_external_message(td::Ref msg) { + auto message = std::make_unique>(msg); + auto id = message->ext_id(); + auto address = message->address(); + unsigned long per_address_limit = 256; + if(ext_addr_messages_.count(address) < per_address_limit) { + if (ext_messages_hashes_.count(id.hash) == 0) { + ext_messages_.emplace(id, std::move(message)); + ext_messages_hashes_.emplace(id.hash, id); + ext_addr_messages_[address].emplace(id.hash, id); + } } } +void ValidatorManagerImpl::check_external_message(td::BufferSlice data, td::Promise promise) { + run_check_external_message(std::move(data), actor_id(this), std::move(promise)); +} void ValidatorManagerImpl::new_ihr_message(td::BufferSlice data) { if (!is_validator()) { @@ -759,6 +771,7 @@ void ValidatorManagerImpl::get_external_messages(ShardIdFull shard, break; } if (it->second->expired()) { + ext_addr_messages_[it->second->address()].erase(it->first.hash); ext_messages_hashes_.erase(it->first.hash); it = ext_messages_.erase(it); continue; @@ -807,17 +820,20 @@ void ValidatorManagerImpl::complete_external_messages(std::vectorsecond]->address()].erase(it->first); CHECK(ext_messages_.erase(it->second)); ext_messages_hashes_.erase(it); } } + unsigned long soft_mempool_limit = 1024; for (auto &hash : to_delay) { auto it = ext_messages_hashes_.find(hash); if (it != ext_messages_hashes_.end()) { auto it2 = ext_messages_.find(it->second); - if (it2->second->can_postpone()) { + if ((ext_messages_.size() < soft_mempool_limit) && it2->second->can_postpone()) { it2->second->postpone(); } else { + ext_addr_messages_[it2->second->address()].erase(it2->first.hash); ext_messages_.erase(it2); ext_messages_hashes_.erase(it); } @@ -1352,6 +1368,7 @@ void ValidatorManagerImpl::send_get_next_key_blocks_request(BlockIdExt block_id, void ValidatorManagerImpl::send_external_message(td::Ref message) { callback_->send_ext_message(message->shard(), message->serialize()); + add_external_message(std::move(message)); } void ValidatorManagerImpl::send_ihr_message(td::Ref message) { diff --git a/validator/manager.hpp b/validator/manager.hpp index c2e41a30..19ee4cdd 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -74,6 +74,9 @@ class MessageExt { auto hash() const { return message_->hash(); } + auto address() const { + return std::make_pair(message_->wc(), message_->addr()); + } bool is_active() { if (!active_) { if (reactivate_at_.is_in_past()) { @@ -210,6 +213,7 @@ class ValidatorManagerImpl : public ValidatorManager { // DATA FOR COLLATOR std::map> shard_blocks_; std::map, std::unique_ptr>> ext_messages_; + std::map, std::map>> ext_addr_messages_; std::map> ext_messages_hashes_; // IHR ? std::map, std::unique_ptr>> ihr_messages_; @@ -327,9 +331,9 @@ class ValidatorManagerImpl : public ValidatorManager { //void get_block_description(BlockIdExt block_id, td::Promise promise) override; void new_external_message(td::BufferSlice data) override; - void check_external_message(td::BufferSlice data, td::Promise promise) override { - promise.set_value(td::Unit()); - } + void add_external_message(td::Ref message); + void check_external_message(td::BufferSlice data, td::Promise promise) override; + void new_ihr_message(td::BufferSlice data) override; void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override; From cb31a20206a7aea41bb9dadbb5f3ccbd8820d4e5 Mon Sep 17 00:00:00 2001 From: OmicronTau Date: Tue, 30 Nov 2021 15:11:14 +0300 Subject: [PATCH 05/29] Add methods to sign and import certificates --- overlay/overlay-manager.cpp | 51 +++++ overlay/overlay-manager.h | 1 + overlay/overlay.cpp | 15 ++ overlay/overlay.h | 2 +- overlay/overlay.hpp | 2 + overlay/overlays.h | 1 + tdutils/td/utils/DecTree.h | 20 ++ tl/generate/scheme/ton_api.tl | 11 + tl/generate/scheme/ton_api.tlo | Bin 64644 -> 65956 bytes .../validator-engine-console-query.cpp | 193 ++++++++++++++++++ .../validator-engine-console-query.h | 136 ++++++++++++ .../validator-engine-console.cpp | 8 +- validator-engine/validator-engine.cpp | 163 +++++++++++++-- validator-engine/validator-engine.hpp | 9 + validator/full-node-shard.cpp | 33 +++ validator/full-node-shard.h | 4 + validator/full-node-shard.hpp | 4 + validator/full-node.cpp | 32 ++- validator/full-node.h | 7 + validator/full-node.hpp | 8 + 20 files changed, 682 insertions(+), 18 deletions(-) diff --git a/overlay/overlay-manager.cpp b/overlay/overlay-manager.cpp index 059ad4e2..1d133673 100644 --- a/overlay/overlay-manager.cpp +++ b/overlay/overlay-manager.cpp @@ -21,13 +21,18 @@ #include "overlay.h" #include "adnl/utils.hpp" +#include "td/actor/actor.h" +#include "td/actor/common.h" #include "td/utils/Random.h" #include "td/db/RocksDb.h" +#include "td/utils/Status.h" #include "td/utils/overloaded.h" #include "keys/encryptor.h" +#include "td/utils/port/Poll.h" +#include namespace ton { @@ -269,6 +274,52 @@ void OverlayManager::save_to_db(adnl::AdnlNodeIdShort local_id, OverlayIdShort o db_.set(key, create_serialize_tl_object(std::move(obj))); } +void OverlayManager::get_stats(td::Promise> promise) { + class Cb : public td::actor::Actor { + public: + Cb(td::Promise> promise) : promise_(std::move(promise)) { + } + void incr_pending() { + pending_++; + } + void decr_pending() { + if (!--pending_) { + promise_.set_result(create_tl_object(std::move(res_))); + stop(); + } + } + void receive_answer(tl_object_ptr res) { + if (res) { + res_.push_back(std::move(res)); + } + decr_pending(); + } + + private: + std::vector> res_; + size_t pending_{1}; + td::Promise> promise_; + }; + + auto act = td::actor::create_actor("overlaysstatsmerger", std::move(promise)).release(); + + for (auto &a : overlays_) { + for (auto &b : a.second) { + td::actor::send_closure(act, &Cb::incr_pending); + td::actor::send_closure(b.second, &Overlay::get_stats, + [act](td::Result> R) { + if (R.is_ok()) { + td::actor::send_closure(act, &Cb::receive_answer, R.move_as_ok()); + } else { + td::actor::send_closure(act, &Cb::receive_answer, nullptr); + } + }); + } + } + + td::actor::send_closure(act, &Cb::decr_pending); +} + Certificate::Certificate(PublicKey issued_by, td::int32 expire_at, td::uint32 max_size, td::uint32 flags, td::BufferSlice signature) : issued_by_(issued_by) diff --git a/overlay/overlay-manager.h b/overlay/overlay-manager.h index 574d0f91..c6afefaf 100644 --- a/overlay/overlay-manager.h +++ b/overlay/overlay-manager.h @@ -91,6 +91,7 @@ class OverlayManager : public Overlays { void register_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, td::actor::ActorOwn overlay); + void get_stats(td::Promise> promise) override; struct PrintId {}; diff --git a/overlay/overlay.cpp b/overlay/overlay.cpp index 21ec9363..ec13fcd3 100644 --- a/overlay/overlay.cpp +++ b/overlay/overlay.cpp @@ -16,6 +16,7 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include "auto/tl/ton_api.h" #include "td/utils/Random.h" #include "adnl/utils.hpp" @@ -25,6 +26,7 @@ #include "auto/tl/ton_api.hpp" #include "keys/encryptor.h" +#include "td/utils/StringBuilder.h" namespace ton { @@ -554,6 +556,19 @@ void OverlayImpl::broadcast_checked(Overlay::BroadcastHash hash, td::Result> promise) { + auto res = create_tl_object(); + res->adnl_id_ = local_id_.bits256_value(); + res->overlay_id_ = overlay_id_.bits256_value(); + res->overlay_id_full_ = id_full_.pubkey().tl(); + peers_.iterate([&](const adnl::AdnlNodeIdShort &key, const OverlayPeer &peer) { res->nodes_.push_back(key.tl()); }); + + res->stats_.push_back( + create_tl_object("neighbours_cnt", PSTRING() << neighbours_.size())); + + promise.set_value(std::move(res)); +} + } // namespace overlay } // namespace ton diff --git a/overlay/overlay.h b/overlay/overlay.h index cba15145..cce24ed8 100644 --- a/overlay/overlay.h +++ b/overlay/overlay.h @@ -63,6 +63,7 @@ class Overlay : public td::actor::Actor { virtual void add_certificate(PublicKeyHash key, std::shared_ptr) = 0; virtual void set_privacy_rules(OverlayPrivacyRules rules) = 0; virtual void receive_nodes_from_db(tl_object_ptr nodes) = 0; + virtual void get_stats(td::Promise> promise) = 0; //virtual void receive_broadcast(td::BufferSlice data) = 0; //virtual void subscribe(std::unique_ptr callback) = 0; }; @@ -70,4 +71,3 @@ class Overlay : public td::actor::Actor { } // namespace overlay } // namespace ton - diff --git a/overlay/overlay.hpp b/overlay/overlay.hpp index cbddd654..76d17aab 100644 --- a/overlay/overlay.hpp +++ b/overlay/overlay.hpp @@ -190,6 +190,8 @@ class OverlayImpl : public Overlay { std::shared_ptr get_certificate(PublicKeyHash local_id); td::Result get_encryptor(PublicKey source); + void get_stats(td::Promise> promise) override; + private: template void process_query(adnl::AdnlNodeIdShort src, T &query, td::Promise promise) { diff --git a/overlay/overlays.h b/overlay/overlays.h index 1ad55453..aba83ab1 100644 --- a/overlay/overlays.h +++ b/overlay/overlays.h @@ -231,6 +231,7 @@ class Overlays : public td::actor::Actor { virtual void get_overlay_random_peers(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay, td::uint32 max_peers, td::Promise> promise) = 0; + virtual void get_stats(td::Promise> promise) = 0; }; } // namespace overlay diff --git a/tdutils/td/utils/DecTree.h b/tdutils/td/utils/DecTree.h index 80bd0e21..1ebb6cd6 100644 --- a/tdutils/td/utils/DecTree.h +++ b/tdutils/td/utils/DecTree.h @@ -46,6 +46,17 @@ class DecTree { } } + template + void iterate(const FuncT &cb) { + if (left_) { + left_->iterate(cb); + } + cb(key_, value_); + if (right_) { + right_->iterate(cb); + } + } + Node(KeyType key, ValueType value, uint32 y) : size_(1), key_(std::move(key)), value_(std::move(value)), y_(y) { } }; @@ -223,6 +234,15 @@ class DecTree { bool exists(const KeyType &key) const { return get_node(root_, key) != nullptr; } + + template + void iterate(const FuncT &cb) { + if (size() == 0) { + return; + } else { + root_->iterate(cb); + } + } }; } // namespace td diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index b226ca7f..0be70540 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -620,6 +620,11 @@ engine.validator.proposalVote perm_key:int256 to_send:bytes = engine.validator.P engine.validator.dhtServerStatus id:int256 status:int = engine.validator.DhtServerStatus; engine.validator.dhtServersStatus servers:(vector engine.validator.dhtServerStatus) = engine.validator.DhtServersStatus; +engine.validator.overlayStats overlay_id:int256 overlay_id_full:PublicKey adnl_id:int256 nodes:(vector adnl.id.short) + stats:(vector engine.validator.oneStat) = engine.validator.OverlayStats; +engine.validator.overlaysStats overlays:(vector engine.validator.overlayStats) = engine.validator.OverlaysStats; + + ---functions--- engine.validator.getTime = engine.validator.Time; @@ -661,8 +666,14 @@ engine.validator.createComplaintVote election_id:int vote:bytes = engine.validat engine.validator.checkDhtServers id:int256 = engine.validator.DhtServersStatus; +engine.validator.getOverlaysStats = engine.validator.OverlaysStats; + engine.validator.controlQuery data:bytes = Object; +engine.validator.importCertificate overlay_id:int256 local_id:adnl.id.short signed_key:engine.validator.KeyHash cert:overlay.Certificate = engine.validator.Success; +engine.validator.signShardOverlayCertificate workchain:int shard:long signed_key:engine.validator.KeyHash expire_at:int max_size:int = overlay.Certificate; +engine.validator.importShardOverlayCertificate workchain:int shard:long signed_key:engine.validator.KeyHash cert:overlay.Certificate = engine.validator.Success; + ---types--- storage.pong = storage.Pong; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 32f42b3c4843343c874d1a1a2696af299ab58f1c..dbdd73b3db01254391b73828527be4872531d40a 100644 GIT binary patch delta 711 zcmZqq$-JbQh4;~HeJchiII@v(w~t$l+2{3{q7**`Gsd^IFkxK1Qp}tSax5HLx3;k1)8HfdQmu@`70g z0{l?%_{@~}w9=fM$rlPFCa>rhnEYoE5C^Jo`ls64F@hXEnYYv42c!??jjM88G1%P) zH51}K29TPY$Ao-9rbs`#ugL+Go?H;9AOSKEY&=l1$eB%t!?x`pNWwB5=Pi%<`(1^l{lvsm1L%6 zCMTApGITF{vU&2AnFfEkf~Ig+b5Gd(XgB|bZ~5@a|7$X$L*j~xPYfCNYl zOESY>yTyjkEb70 y!)&I<=wMh4@(7Z}lN)D>Pi|S~BZzUXEZu?Rj!i_Ntv?~^xkbSZc+MsC);^jTzb#`BKJ RCthSs7I^hxGsl}bq5#z`8LI#Q diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index 64f57de1..4b5802c6 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -26,9 +26,12 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "validator-engine-console-query.h" +#include "auto/tl/ton_api.h" +#include "td/utils/StringBuilder.h" #include "validator-engine-console.h" #include "terminal/terminal.h" #include "td/utils/filesystem.h" +#include "overlay/overlays.h" #include @@ -720,3 +723,193 @@ td::Status CheckDhtServersQuery::receive(td::BufferSlice data) { } return td::Status::OK(); } + +td::Status SignCertificateQuery::run() { + TRY_RESULT_ASSIGN(overlay_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(id_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(expire_at_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(max_size_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(signer_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(out_file_, tokenizer_.get_token()); + return td::Status::OK(); +} + +td::Status SignCertificateQuery::send() { + auto cid = ton::create_serialize_tl_object(overlay_, id_, expire_at_, max_size_); + auto sign = ton::create_serialize_tl_object(signer_.tl(), std::move(cid)); + auto pub = ton::create_serialize_tl_object(signer_.tl()); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(pub), + td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &SignCertificateQuery::handle_error, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &SignCertificateQuery::receive_pubkey, R.move_as_ok()); + } + })); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(sign), + td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &SignCertificateQuery::handle_error, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &SignCertificateQuery::receive_signature, R.move_as_ok()); + } + })); + return td::Status::OK(); +} + +void SignCertificateQuery::receive_pubkey(td::BufferSlice R) { + auto f = ton::fetch_tl_object(R.as_slice(), true); + if (f.is_error()) { + handle_error(f.move_as_error_prefix("Failed to get pubkey: ")); + return; + } + pubkey_ = f.move_as_ok(); + has_pubkey_ = true; + if(has_signature_) { + save_certificate(); + } +} + + +td::Status SignCertificateQuery::receive(td::BufferSlice data) { + UNREACHABLE(); +} + +void SignCertificateQuery::receive_signature(td::BufferSlice R) { + auto f = ton::fetch_tl_object(R.as_slice(), true); + if(f.is_error()){ + handle_error(f.move_as_error_prefix("Failed to get signature: ")); + return; + } + signature_ = std::move(f.move_as_ok()->signature_); + if(has_pubkey_) { + save_certificate(); + } +} + +void SignCertificateQuery::save_certificate() { + auto c = ton::create_serialize_tl_object( + std::move(pubkey_), expire_at_, max_size_, std::move(signature_)); + auto w = td::write_file(out_file_, c.as_slice()); + if(w.is_error()) { + handle_error(w.move_as_error_prefix("Failed to write certificate to file: ")); + return; + } + td::TerminalIO::out() << "saved certificate\n"; + stop(); +} + +td::Status ImportCertificateQuery::run() { + TRY_RESULT_ASSIGN(overlay_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(id_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(kh_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(in_file_, tokenizer_.get_token()); + return td::Status::OK(); +} + +td::Status ImportCertificateQuery::send() { + TRY_RESULT(data, td::read_file(in_file_)); + TRY_RESULT_PREFIX(cert, ton::fetch_tl_object(data.as_slice(), true), + "incorrect certificate"); + auto b = ton::create_serialize_tl_object( + overlay_, + ton::create_tl_object(id_), + ton::create_tl_object(kh_.tl()), + std::move(cert) + ); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} +td::Status GetOverlaysStatsQuery::run() { + return td::Status::OK(); +} + +td::Status GetOverlaysStatsQuery::send() { + auto b = ton::create_serialize_tl_object(); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status GetOverlaysStatsQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + for (auto &s : f->overlays_) { + td::StringBuilder sb; + sb << "overlay_id=" << s->overlay_id_ << " adnl_id=" << s->adnl_id_ << "\n"; + sb << " nodes:"; + for (auto &n : s->nodes_) { + sb << " " << n->id_ << "\n"; + } + sb << " stats:\n"; + for (auto &t : s->stats_) { + sb << " " << t->key_ << "\t" << t->value_ << "\n"; + } + td::TerminalIO::output(sb.as_cslice()); + } + return td::Status::OK(); +} + + +td::Status ImportCertificateQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "successfully sent certificate to overlay manager\n"; + return td::Status::OK(); +} + + +td::Status SignShardOverlayCertificateQuery::run() { + TRY_RESULT_ASSIGN(wc_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token() ); + TRY_RESULT_ASSIGN(key_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(expire_at_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(max_size_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(out_file_, tokenizer_.get_token()); + + return td::Status::OK(); +} + +td::Status SignShardOverlayCertificateQuery::send() { + auto b = ton::create_serialize_tl_object + (wc_, shard_, ton::create_tl_object(key_.tl()), expire_at_, max_size_); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status SignShardOverlayCertificateQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(c, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect cert: "); + auto w = td::write_file(out_file_, data.as_slice()); + if(w.is_error()) { + return w.move_as_error_prefix("Failed to write certificate to file: "); + } + td::TerminalIO::out() << "saved certificate\n"; + + return td::Status::OK(); +} + +td::Status ImportShardOverlayCertificateQuery::run() { + TRY_RESULT_ASSIGN(wc_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token() ); + TRY_RESULT_ASSIGN(key_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(in_file_, tokenizer_.get_token()); + + return td::Status::OK(); +} + +td::Status ImportShardOverlayCertificateQuery::send() { + TRY_RESULT(data, td::read_file(in_file_)); + TRY_RESULT_PREFIX(cert, ton::fetch_tl_object(data.as_slice(), true), + "incorrect certificate"); + auto b = ton::create_serialize_tl_object + (wc_, shard_, ton::create_tl_object(key_.tl()), std::move(cert)); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status ImportShardOverlayCertificateQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "successfully sent certificate to overlay manager\n"; + return td::Status::OK(); +} diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index e010d76f..769f2052 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -903,3 +903,139 @@ class CheckDhtServersQuery : public Query { private: ton::PublicKeyHash id_; }; + +class GetOverlaysStatsQuery : public Query { + public: + GetOverlaysStatsQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "getoverlaysstats"; + } + static std::string get_help() { + return "getoverlaysstats\tgets stats for all overlays"; + } + std::string name() const override { + return get_name(); + } +}; + +class SignCertificateQuery : public Query { + public: + SignCertificateQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "signcert"; + } + static std::string get_help() { + return "signcert \tsign overlay certificate by key"; + } + std::string name() const override { + return get_name(); + } + void receive_pubkey(td::BufferSlice R); + void receive_signature(td::BufferSlice R); + + + private: + void save_certificate(); + + td::Bits256 overlay_; + td::Bits256 id_; + td::int32 expire_at_; + td::uint32 max_size_; + std::string out_file_; + ton::PublicKeyHash signer_; + td::BufferSlice signature_; + std::unique_ptr pubkey_; + bool has_signature_{0}; + bool has_pubkey_{0}; +}; + +class ImportCertificateQuery : public Query { + public: + ImportCertificateQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "importcert"; + } + static std::string get_help() { + return "importcert \timport overlay certificate for specific key"; + } + std::string name() const override { + return get_name(); + } + + private: + td::Bits256 overlay_; + td::Bits256 id_; + ton::PublicKeyHash kh_; + std::string in_file_; +}; + +class SignShardOverlayCertificateQuery : public Query { + public: + SignShardOverlayCertificateQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "signshardoverlaycert"; + } + static std::string get_help() { + return "signshardoverlaycert \tsign certificate for in currently active shard overlay"; + } + std::string name() const override { + return get_name(); + } + + private: + + td::int32 wc_; + td::int64 shard_; + td::int32 expire_at_; + ton::PublicKeyHash key_; + td::uint32 max_size_; + std::string out_file_; +}; + + +class ImportShardOverlayCertificateQuery : public Query { + public: + ImportShardOverlayCertificateQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "importshardoverlaycert"; + } + static std::string get_help() { + return "importshardoverlaycert \timport certificate for in currently active shard overlay"; + } + std::string name() const override { + return get_name(); + } + + private: + + td::int32 wc_; + td::int64 shard_; + ton::PublicKeyHash key_; + std::string in_file_; +}; + diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index 43a02dc6..899cd53f 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -134,6 +134,11 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); } bool ValidatorEngineConsole::envelope_send_query(td::BufferSlice query, td::Promise promise) { @@ -258,7 +263,8 @@ int main(int argc, char* argv[]) { std::exit(2); }); p.add_option('V', "version", "shows validator-engine-console build information", [&]() { - std::cout << "validator-engine-console build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; + std::cout << "validator-engine-console build information: [ Commit: " << GitMetadata::CommitSHA1() + << ", Date: " << GitMetadata::CommitDate() << "]\n"; std::exit(0); }); p.add_checked_option('a', "address", "server address", [&](td::Slice arg) { diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 9c9d4a98..59279bef 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -27,6 +27,11 @@ */ #include "validator-engine.hpp" +#include "auto/tl/ton_api.h" +#include "overlay-manager.h" +#include "td/actor/actor.h" +#include "tl-utils/tl-utils.hpp" +#include "tl/TlObject.h" #include "ton/ton-types.h" #include "ton/ton-tl.hpp" #include "ton/ton-io.hpp" @@ -3143,6 +3148,135 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_createCom .release(); } +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importCertificate &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (keyring_.empty()) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "keyring not started"))); + return; + } + + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + auto r = ton::overlay::Certificate::create(std::move(query.cert_)); + if(r.is_error()) { + promise.set_value(create_control_query_error(r.move_as_error_prefix("Invalid certificate: "))); + } + //TODO force Overlays::update_certificate to return result + /*auto P = td::PromiseCreator::lambda( + [promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); + } + }); + */ + td::actor::send_closure(overlay_manager_, &ton::overlay::Overlays::update_certificate, + ton::adnl::AdnlNodeIdShort{query.local_id_->id_}, + ton::overlay::OverlayIdShort{query.overlay_id_}, + ton::PublicKeyHash{query.signed_key_->key_hash_}, + r.move_as_ok()); + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true) + ); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importShardOverlayCertificate &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (keyring_.empty()) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "keyring not started"))); + return; + } + + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + auto r = ton::overlay::Certificate::create(std::move(query.cert_)); + if(r.is_error()) { + promise.set_value(create_control_query_error(r.move_as_error_prefix("Invalid certificate: "))); + } + auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error_prefix("failed to import cert: "))); + } else { + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); + } + }); + ton::ShardIdFull shard_id{ton::WorkchainId{query.workchain_}, static_cast(query.shard_)}; + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::import_shard_overlay_certificate, + shard_id, ton::PublicKeyHash{query.signed_key_->key_hash_}, r.move_as_ok(), std::move(P)); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_signShardOverlayCertificate &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (keyring_.empty()) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "keyring not started"))); + return; + } + + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + ton::ShardIdFull shard_id{ton::WorkchainId{query.workchain_}, static_cast(query.shard_)}; + auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error_prefix("failed to import cert: "))); + } else { + promise.set_value(R.move_as_ok()); + } + }); + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::sign_shard_overlay_certificate, + shard_id, ton::PublicKeyHash{query.signed_key_->key_hash_}, query.expire_at_, query.max_size_, std::move(P)); +} + + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getOverlaysStats &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_default)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + + if (keyring_.empty()) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "keyring not started"))); + return; + } + + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + + td::actor::send_closure(overlay_manager_, &ton::overlay::Overlays::get_stats, + [promise = std::move(promise)]( + td::Result> R) mutable { + if (R.is_ok()) { + promise.set_value(ton::serialize_tl_object(R.move_as_ok(), true)); + } else { + promise.set_value(create_control_query_error( + td::Status::Error(ton::ErrorCode::notready, "overlay manager not ready"))); + } + }); +} + void ValidatorEngine::process_control_query(td::uint16 port, ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data, td::Promise promise) { @@ -3294,7 +3428,8 @@ int main(int argc, char *argv[]) { SET_VERBOSITY_LEVEL(v); }); p.add_option('V', "version", "shows validator-engine build information", [&]() { - std::cout << "validator-engine build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; + std::cout << "validator-engine build information: [ Commit: " << GitMetadata::CommitSHA1() + << ", Date: " << GitMetadata::CommitDate() << "]\n"; std::exit(0); }); p.add_option('h', "help", "prints_help", [&]() { @@ -3374,18 +3509,20 @@ int main(int argc, char *argv[]) { acts.push_back([&x, seq]() { td::actor::send_closure(x, &ValidatorEngine::add_unsafe_catchain, seq); }); return td::Status::OK(); }); - p.add_checked_option( - 'F', "unsafe-catchain-rotate", "use forceful and DANGEROUS catchain rotation", [&](td::Slice params) { - auto pos1 = params.find(':'); - TRY_RESULT(b_seq, td::to_integer_safe(params.substr(0, pos1))); - params = params.substr(++pos1, params.size()); - auto pos2 = params.find(':'); - TRY_RESULT(cc_seq, td::to_integer_safe(params.substr(0, pos2))); - params = params.substr(++pos2, params.size()); - auto h = std::stoi(params.substr(0, params.size()).str()); - acts.push_back([&x, b_seq, cc_seq, h]() { td::actor::send_closure(x, &ValidatorEngine::add_unsafe_catchain_rotation, b_seq, cc_seq, h); }); - return td::Status::OK(); - }); + p.add_checked_option('F', "unsafe-catchain-rotate", "use forceful and DANGEROUS catchain rotation", + [&](td::Slice params) { + auto pos1 = params.find(':'); + TRY_RESULT(b_seq, td::to_integer_safe(params.substr(0, pos1))); + params = params.substr(++pos1, params.size()); + auto pos2 = params.find(':'); + TRY_RESULT(cc_seq, td::to_integer_safe(params.substr(0, pos2))); + params = params.substr(++pos2, params.size()); + auto h = std::stoi(params.substr(0, params.size()).str()); + acts.push_back([&x, b_seq, cc_seq, h]() { + td::actor::send_closure(x, &ValidatorEngine::add_unsafe_catchain_rotation, b_seq, cc_seq, h); + }); + return td::Status::OK(); + }); td::uint32 threads = 7; p.add_checked_option( 't', "threads", PSTRING() << "number of threads (default=" << threads << ")", [&](td::Slice fname) { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 07078f95..88ba802d 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -28,6 +28,7 @@ #pragma once #include "adnl/adnl.h" +#include "auto/tl/ton_api.h" #include "rldp/rldp.h" #include "dht/dht.h" #include "validator/manager.h" @@ -395,6 +396,14 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_createComplaintVote &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_importCertificate &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_signShardOverlayCertificate &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_importShardOverlayCertificate &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_getOverlaysStats &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); template void run_control_query(T &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 769ef8f3..72684151 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -115,6 +115,7 @@ void FullNodeShardImpl::check_broadcast(PublicKeyHash src, td::BufferSlice broad void FullNodeShardImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) { td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, adnl_id_, overlay_id_); adnl_id_ = adnl_id; + local_id_ = adnl_id_.pubkey_hash(); create_overlay(); } @@ -848,6 +849,38 @@ void FullNodeShardImpl::signed_new_certificate(ton::overlay::Certificate cert) { td::actor::send_closure(overlays_, &overlay::Overlays::update_certificate, adnl_id_, overlay_id_, local_id_, cert_); } +void FullNodeShardImpl::sign_overlay_certificate(PublicKeyHash signed_key, td::uint32 expire_at, td::uint32 max_size, td::Promise promise) { + auto sign_by = sign_cert_by_; + if (sign_by.is_zero()) { + promise.set_error(td::Status::Error("Node has no key with signing authority")); + return; + } + + ton::overlay::Certificate cert{ + sign_by, static_cast(expire_at), max_size, + overlay::CertificateFlags::Trusted | overlay::CertificateFlags::AllowFec, td::BufferSlice{}}; + auto to_sign = cert.to_sign(overlay_id_, signed_key); + + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), expire_at = expire_at, max_size = max_size, promise = std::move(promise)](td::Result> R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error_prefix("failed to create certificate: failed to sign: ")); + } else { + auto p = R.move_as_ok(); + auto c = ton::create_serialize_tl_object(p.second.tl(), static_cast(expire_at), max_size, std::move(p.first)); + promise.set_value(std::move(c)); + } + }); + td::actor::send_closure(keyring_, &ton::keyring::Keyring::sign_add_get_public_key, sign_by, std::move(to_sign), + std::move(P)); +} + +void FullNodeShardImpl::import_overlay_certificate(PublicKeyHash signed_key, std::shared_ptr cert, td::Promise promise) { + td::actor::send_closure(overlays_, &ton::overlay::Overlays::update_certificate, + adnl_id_, overlay_id_, signed_key, cert); + promise.set_value( td::Unit() ); +} + void FullNodeShardImpl::update_validators(std::vector public_key_hashes, PublicKeyHash local_hash) { if (!client_.empty()) { return; diff --git a/validator/full-node-shard.h b/validator/full-node-shard.h index 40a422e0..c1712baf 100644 --- a/validator/full-node-shard.h +++ b/validator/full-node-shard.h @@ -42,6 +42,10 @@ class FullNodeShard : public td::actor::Actor { virtual void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) = 0; virtual void send_broadcast(BlockBroadcast broadcast) = 0; + virtual void sign_overlay_certificate(PublicKeyHash signed_key, td::uint32 expiry_at, td::uint32 max_size, td::Promise promise) = 0; + virtual void import_overlay_certificate(PublicKeyHash signed_key, std::shared_ptr cert, td::Promise promise) = 0; + + virtual void download_block(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) = 0; virtual void download_zero_state(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index 2c468374..a2dd5cc4 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -170,6 +170,10 @@ class FullNodeShardImpl : public FullNodeShard { void alarm() override; void update_validators(std::vector public_key_hashes, PublicKeyHash local_hash) override; + + void sign_overlay_certificate(PublicKeyHash signed_key, td::uint32 expiry_at, td::uint32 max_size, td::Promise promise) override; + void import_overlay_certificate(PublicKeyHash signed_key, std::shared_ptr cert, td::Promise promise) override; + void sign_new_certificate(PublicKeyHash sign_by); void signed_new_certificate(ton::overlay::Certificate cert); diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 691a8b36..6606f215 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -76,6 +76,27 @@ void FullNodeImpl::del_permanent_key(PublicKeyHash key, td::Promise pr promise.set_value(td::Unit()); } +void FullNodeImpl::sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, + td::uint32 expiry_at, td::uint32 max_size, + td::Promise promise) { + auto it = shards_.find(shard_id); + if(it == shards_.end()) { + promise.set_error(td::Status::Error(ErrorCode::error, "shard not found")); + return; + } + td::actor::send_closure(it->second, &FullNodeShard::sign_overlay_certificate, signed_key, expiry_at, max_size, std::move(promise)); +} + +void FullNodeImpl::import_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, + std::shared_ptr cert, + td::Promise promise) { + auto it = shards_.find(shard_id); + if(it == shards_.end()) { + promise.set_error(td::Status::Error(ErrorCode::error, "shard not found")); + } + td::actor::send_closure(it->second, &FullNodeShard::import_overlay_certificate, signed_key, cert, std::move(promise)); +} + void FullNodeImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) { adnl_id_ = adnl_id; @@ -86,6 +107,7 @@ void FullNodeImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) = 0; virtual void del_permanent_key(PublicKeyHash key, td::Promise promise) = 0; + virtual void sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, + td::uint32 expiry_at, td::uint32 max_size, + td::Promise promise) = 0; + virtual void import_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, + std::shared_ptr cert, + td::Promise promise) = 0; + virtual void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) = 0; static constexpr td::uint32 max_block_size() { diff --git a/validator/full-node.hpp b/validator/full-node.hpp index e9b64f57..6d57f4a8 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -42,6 +42,14 @@ class FullNodeImpl : public FullNode { void add_permanent_key(PublicKeyHash key, td::Promise promise) override; void del_permanent_key(PublicKeyHash key, td::Promise promise) override; + void sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, + td::uint32 expiry_at, td::uint32 max_size, + td::Promise promise) override; + void import_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, + std::shared_ptr cert, + td::Promise promise) override; + + void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) override; void add_shard(ShardIdFull shard); From 703bcd6e32106878b42756cb72b7af5b80f591d3 Mon Sep 17 00:00:00 2001 From: OmicronTau Date: Sun, 5 Dec 2021 17:41:22 +0300 Subject: [PATCH 06/29] Add tests and fixes for modpow2, muldivmod --- CMakeLists.txt | 5 + crypto/CMakeLists.txt | 5 + crypto/common/bigint.hpp | 69 ++- crypto/common/refint.cpp | 8 +- crypto/fift/lib/Asm.fif | 5 + crypto/test/modbigint.cpp | 1070 +++++++++++++++++++++++++++++++++++ crypto/test/test-bigint.cpp | 876 ++++++++++++++++++++++++++++ crypto/vm/arithops.cpp | 26 +- doc/tvm.tex | 5 +- rldp2/CMakeLists.txt | 2 +- 10 files changed, 2017 insertions(+), 54 deletions(-) create mode 100644 crypto/test/modbigint.cpp create mode 100644 crypto/test/test-bigint.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index a97a9c96..da1945ff 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -423,6 +423,9 @@ target_link_libraries(test-vm PRIVATE ton_crypto fift-lib) add_executable(test-smartcont test/test-td-main.cpp ${SMARTCONT_TEST_SOURCE}) target_link_libraries(test-smartcont PRIVATE smc-envelope fift-lib ton_db) +add_executable(test-bigint ${BIGINT_TEST_SOURCE}) +target_link_libraries(test-bigint PRIVATE ton_crypto) + add_executable(test-cells test/test-td-main.cpp ${CELLS_TEST_SOURCE}) target_link_libraries(test-cells PRIVATE ton_crypto) @@ -523,6 +526,7 @@ if (HAS_PARENT) ${FEC_TEST_SOURCE} ${ED25519_TEST_SOURCE} ${TONDB_TEST_SOURCE} + ${BIGNUM_TEST_SOURCE} ${CELLS_TEST_SOURCE} # ${TONVM_TEST_SOURCE} ${FIFT_TEST_SOURCE} ${TONLIB_ONLINE_TEST_SOURCE} PARENT_SCOPE) endif() @@ -536,6 +540,7 @@ set(TEST_OPTIONS "--regression ${CMAKE_CURRENT_SOURCE_DIR}/test/regression-tests separate_arguments(TEST_OPTIONS) add_test(test-ed25519-crypto crypto/test-ed25519-crypto) add_test(test-ed25519 test-ed25519) +add_test(test-bigint test-bigint) add_test(test-vm test-vm ${TEST_OPTIONS}) add_test(test-fift test-fift ${TEST_OPTIONS}) add_test(test-cells test-cells ${TEST_OPTIONS}) diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index 79406ffc..88a0272b 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -259,6 +259,11 @@ set(FIFT_TEST_SOURCE PARENT_SCOPE ) +set(BIGINT_TEST_SOURCE + ${CMAKE_CURRENT_SOURCE_DIR}/test/test-bigint.cpp + PARENT_SCOPE +) + add_library(ton_crypto STATIC ${TON_CRYPTO_SOURCE}) target_include_directories(ton_crypto PUBLIC $ diff --git a/crypto/common/bigint.hpp b/crypto/common/bigint.hpp index e78f4892..94da3a13 100644 --- a/crypto/common/bigint.hpp +++ b/crypto/common/bigint.hpp @@ -264,7 +264,7 @@ class AnyIntView { return digits[size() - 1]; } double top_double() const { - return size() > 1 ? (double)digits[size() - 1] + (double)digits[size() - 2] * (1.0 / Tr::Base) + return size() > 1 ? (double)digits[size() - 1] + (double)digits[size() - 2] * Tr::InvBase : (double)digits[size() - 1]; } bool is_odd_any() const { @@ -314,8 +314,15 @@ class BigIntG { digits[0] = x; } BigIntG(Normalize, word_t x) : n(1) { - digits[0] = x; - normalize_bool(); + if (x >= -Tr::Half && x < Tr::Half) { + digits[0] = x; + } else if (len <= 1) { + digits[0] = x; + normalize_bool(); + } else { + digits[0] = ((x + Tr::Half) & (Tr::Base - 1)) - Tr::Half; + digits[n++] = (x >> Tr::word_shift) + (digits[0] < 0); + } } BigIntG(const BigIntG& x) : n(x.n) { std::memcpy(digits, x.digits, n * sizeof(word_t)); @@ -757,7 +764,7 @@ bool AnyIntView::add_pow2_any(int exponent, int factor) { while (size() <= k) { digits[inc_size()] = 0; } - digits[k] += (factor << dm.rem); + digits[k] += ((word_t)factor << dm.rem); return true; } @@ -1087,12 +1094,16 @@ int AnyIntView::cmp_any(const AnyIntView& yp) const { template int AnyIntView::cmp_any(word_t y) const { - if (size() > 1) { - return top_word() < 0 ? -1 : 1; - } else if (size() == 1) { + if (size() == 1) { return digits[0] < y ? -1 : (digits[0] > y ? 1 : 0); - } else { + } else if (!size()) { return 0x80000000; + } else if (size() == 2 && (y >= Tr::Half || y < -Tr::Half)) { + word_t x0 = digits[0] & (Tr::Base - 1), y0 = y & (Tr::Base - 1); + word_t x1 = digits[1] + (digits[0] >> Tr::word_shift), y1 = (y >> Tr::word_shift); + return x1 < y1 ? -1 : (x1 > y1 ? 1 : (x0 < y0 ? -1 : (x0 > y0 ? 1 : 0))); + } else { + return top_word() < 0 ? -1 : 1; } } @@ -1312,17 +1323,14 @@ bool AnyIntView::mod_div_any(const AnyIntView& yp, AnyIntView& quot, if (k > quot.max_size()) { return invalidate_bool(); } - quot.set_size(max(k,1)); - for(int qi=0; qi< max(k,1); qi++) { - quot.digits[qi]=0; - } + quot.set_size(std::max(k, 1)); + quot.digits[0] = 0; } else { if (k >= quot.max_size()) { return invalidate_bool(); } quot.set_size(k + 1); - double x_top = top_double(); - word_t q = std::llrint(x_top * y_inv * Tr::InvBase); + word_t q = std::llrint(top_double() * y_inv * Tr::InvBase); quot.digits[k] = q; int i = yp.size() - 1; word_t hi = 0; @@ -1337,8 +1345,7 @@ bool AnyIntView::mod_div_any(const AnyIntView& yp, AnyIntView& quot, quot.digits[0] = 0; } while (--k >= 0) { - double x_top = top_double(); - word_t q = std::llrint(x_top * y_inv); + word_t q = std::llrint(top_double() * y_inv); quot.digits[k] = q; for (int i = yp.size() - 1; i >= 0; --i) { Tr::sub_mul(&digits[k + i + 1], &digits[k + i], q, yp.digits[i]); @@ -1346,15 +1353,18 @@ bool AnyIntView::mod_div_any(const AnyIntView& yp, AnyIntView& quot, dec_size(); digits[size() - 1] += (digits[size()] << word_shift); } - if (size() >= yp.size()) { - assert(size() == yp.size()); - double x_top = top_double(); - double t = x_top * y_inv * Tr::InvBase; + if (size() >= yp.size() - 1) { + assert(size() <= yp.size()); + bool grow = (size() < yp.size()); + double t = top_double() * y_inv * (grow ? Tr::InvBase * Tr::InvBase : Tr::InvBase); if (round_mode >= 0) { t += (round_mode ? 1 : 0.5); } word_t q = std::llrint(std::floor(t)); if (q) { + if (grow) { + digits[inc_size()] = 0; + } for (int i = 0; i < size(); i++) { digits[i] -= q * yp.digits[i]; } @@ -1411,6 +1421,7 @@ bool AnyIntView::mod_div_any(const AnyIntView& yp, AnyIntView& quot, return normalize_bool_any(); } +// works for almost-normalized numbers (digits -Base+1 .. Base-1, top non-zero), result also almost-normalized template bool AnyIntView::mod_pow2_any(int exponent) { if (!is_valid()) { @@ -1462,25 +1473,21 @@ bool AnyIntView::mod_pow2_any(int exponent) { if (exponent >= max_size() * word_shift) { return invalidate_bool(); } - if (q - word_shift >= 0) { + if (q - word_shift >= 0) { // original top digit was a non-zero multiple of Base, impossible(?) digits[size() - 1] = 0; digits[inc_size()] = ((word_t)1 << (q - word_shift)); - } - if (q - word_shift == -1 && size() < max_size() - 1) { + } else if (q - word_shift == -1 && size() < max_size()) { digits[size() - 1] = -Tr::Half; digits[inc_size()] = 1; } else { digits[size() - 1] = pow; } return true; - } else if (v >= Tr::Half) { - if (size() == max_size() - 1) { - return invalidate_bool(); - } else { - digits[size() - 1] = v | -Tr::Half; - digits[inc_size()] = ((word_t)1 << (q - word_shift)); - return true; - } + } else if (v >= Tr::Half && size() < max_size()) { + word_t w = (((v >> (word_shift - 1)) + 1) >> 1); + digits[size() - 1] = v - (w << word_shift); + digits[inc_size()] = w; + return true; } else { digits[size() - 1] = v; return true; diff --git a/crypto/common/refint.cpp b/crypto/common/refint.cpp index b79750ce..8e06da5f 100644 --- a/crypto/common/refint.cpp +++ b/crypto/common/refint.cpp @@ -128,12 +128,10 @@ RefInt256 muldiv(RefInt256 x, RefInt256 y, RefInt256 z, int round_mode) { } std::pair muldivmod(RefInt256 x, RefInt256 y, RefInt256 z, int round_mode) { - typename td::BigInt256::DoubleInt tmp{0}; + typename td::BigInt256::DoubleInt tmp{0}, quot; tmp.add_mul(*x, *y); - RefInt256 quot{true}; - tmp.mod_div(*z, quot.unique_write(), round_mode); - quot.write().normalize(); - return std::make_pair(std::move(quot), td::make_refint(tmp)); + tmp.mod_div(*z, quot, round_mode); + return std::make_pair(td::make_refint(quot.normalize()), td::make_refint(tmp)); } RefInt256 operator&(RefInt256 x, RefInt256 y) { diff --git a/crypto/fift/lib/Asm.fif b/crypto/fift/lib/Asm.fif index d61923c1..76381b85 100644 --- a/crypto/fift/lib/Asm.fif +++ b/crypto/fift/lib/Asm.fif @@ -334,9 +334,14 @@ x{A926} @Defop RSHIFTC x{A935} @Defop(8u+1) RSHIFTR# x{A936} @Defop(8u+1) RSHIFTC# x{A938} @Defop(8u+1) MODPOW2# +x{A939} @Defop(8u+1) MODPOW2R# +x{A93A} @Defop(8u+1) MODPOW2C# x{A984} @Defop MULDIV x{A985} @Defop MULDIVR +x{A988} @Defop MULMOD x{A98C} @Defop MULDIVMOD +x{A98D} @Defop MULDIVMODR +x{A98E} @Defop MULDIVMODC x{A9A4} @Defop MULRSHIFT x{A9A5} @Defop MULRSHIFTR x{A9A6} @Defop MULRSHIFTC diff --git a/crypto/test/modbigint.cpp b/crypto/test/modbigint.cpp new file mode 100644 index 00000000..851f0f9b --- /dev/null +++ b/crypto/test/modbigint.cpp @@ -0,0 +1,1070 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include +#include +#include +#include +#include +#include + +namespace modint { + +enum { mod_cnt = 32 }; + +// mod_cnt = 9 => integers -2^268 .. 2^268 +// mod_cnt = 18 => integers -2^537 .. 2^537 +// mod_cnt = 32 => integers -2^955 .. 2^955 +constexpr int mod[mod_cnt] = {999999937, 999999929, 999999893, 999999883, 999999797, 999999761, 999999757, 999999751, + 999999739, 999999733, 999999677, 999999667, 999999613, 999999607, 999999599, 999999587, + 999999541, 999999527, 999999503, 999999491, 999999487, 999999433, 999999391, 999999353, + 99999337, 999999323, 999999229, 999999223, 999999197, 999999193, 999999191, 999999181}; + +// invm[i][j] = mod[i]^(-1) modulo mod[j] +int invm[mod_cnt][mod_cnt]; + +int gcdx(int a, int b, int& u, int& v); + +template +struct ModArray; + +template +struct MixedRadix; + +template +struct ArrayRawDumpRef; + +template +std::ostream& raw_dump_array(std::ostream& os, const std::array& arr) { + os << '['; + for (auto x : arr) { + os << ' ' << x; + } + return os << " ]"; +} + +template +struct MixedRadix { + enum { n = N }; + int a[N]; + MixedRadix(int v) { + set_int(v); + } + MixedRadix() = default; + MixedRadix(const MixedRadix&) = default; + MixedRadix(std::initializer_list l) { + auto sz = std::min(l.size(), (std::size_t)N); + std::copy(l.begin(), l.begin() + sz, a); + std::fill(a + sz, a + N, 0); + } + MixedRadix(const std::array& arr) { + std::copy(arr.begin(), arr.end(), a); + } + template + MixedRadix(const MixedRadix& other) { + static_assert(M >= N); + std::copy(other.a, other.a + N, a); + } + MixedRadix(const ModArray& other); + MixedRadix(const ModArray& other, bool sgnd); + + MixedRadix& set_zero() { + std::fill(a, a + N, 0); + return *this; + } + MixedRadix& set_one() { + a[0] = 1; + std::fill(a + 1, a + N, 0); + return *this; + } + MixedRadix& set_int(int v) { + a[0] = v; + std::fill(a + 1, a + N, 0); + return *this; + } + + MixedRadix copy() const { + return MixedRadix{*this}; + } + + static const int* mod_array() { + return mod; + } + + static int modulus(int i) { + return mod[i]; + } + + int sgn() const { + int i = N - 1; + while (i >= 0 && !a[i]) { + --i; + } + return i < 0 ? 0 : (a[i] > 0 ? 1 : -1); + } + + int cmp(const MixedRadix& other) const { + int i = N - 1; + while (i >= 0 && a[i] == other.a[i]) { + --i; + } + return i < 0 ? 0 : (a[i] > other.a[i] ? 1 : -1); + } + + bool is_small() const { + return !a[N - 1] || a[N - 1] == -1; + } + + bool operator==(const MixedRadix& other) const { + return std::equal(a, a + N, other.a); + } + + bool operator!=(const MixedRadix& other) const { + return !std::equal(a, a + N, other.a); + } + + bool operator<(const MixedRadix& other) const { + return cmp(other) < 0; + } + + bool operator<=(const MixedRadix& other) const { + return cmp(other) <= 0; + } + + bool operator>(const MixedRadix& other) const { + return cmp(other) > 0; + } + + bool operator>=(const MixedRadix& other) const { + return cmp(other) >= 0; + } + + explicit operator bool() const { + return sgn(); + } + + bool operator!() const { + return !sgn(); + } + + MixedRadix& negate() { + int i = 0; + while (i < N - 1 && !a[i]) { + i++; + } + a[i]--; + for (; i < N; i++) { + a[i] = mod[i] - a[i] - 1; + } + a[N - 1] -= mod[N - 1]; + return *this; + } + + static const MixedRadix& pow2(int power); + static MixedRadix negpow2(int power) { + return -pow2(power); + } + + template + const MixedRadix& as_shorter() const { + static_assert(M <= N); + return *reinterpret_cast*>(this); + } + + MixedRadix& import_mod_array(const int* data, bool sgnd = true) { + for (int i = 0; i < N; i++) { + a[i] = data[i] % mod[i]; + } + for (int i = 0; i < N; i++) { + if (a[i] < 0) { + a[i] += mod[i]; + } + for (int j = i + 1; j < N; j++) { + a[j] = (int)((long long)(a[j] - a[i]) * invm[i][j] % mod[j]); + } + } + if (sgnd && a[N - 1] > (mod[N - 1] >> 1)) { + a[N - 1] -= mod[N - 1]; + } + return *this; + } + + MixedRadix& operator=(const MixedRadix&) = default; + + template + MixedRadix& operator=(const MixedRadix& other) { + static_assert(M >= N); + std::copy(other.a, other.a + N, a); + } + + MixedRadix& import_mod_array(const ModArray& other, bool sgnd = true); + + MixedRadix& operator=(const ModArray& other) { + return import_mod_array(other); + } + + MixedRadix& set_sum(const MixedRadix& x, const MixedRadix& y, int factor = 1) { + long long carry = 0; + for (int i = 0; i < N; i++) { + long long acc = x.a[i] + carry + (long long)factor * y.a[i]; + carry = acc / mod[i]; + a[i] = (int)(acc - carry * mod[i]); + if (a[i] < 0) { + a[i] += mod[i]; + --carry; + } + } + if (a[N - 1] >= 0 && carry == -1) { + a[N - 1] -= mod[N - 1]; + } + return *this; + } + + MixedRadix& operator+=(const MixedRadix& other) { + return set_sum(*this, other); + } + + MixedRadix& operator-=(const MixedRadix& other) { + return set_sum(*this, other, -1); + } + + static const MixedRadix& zero(); + static const MixedRadix& one(); + + MixedRadix& operator*=(int factor) { + return set_sum(zero(), *this, factor); + } + + MixedRadix operator-() const { + MixedRadix copy{*this}; + copy.negate(); + return copy; + } + + MixedRadix operator+(const MixedRadix& other) const { + MixedRadix res; + res.set_sum(*this, other); + return res; + } + + MixedRadix operator-(const MixedRadix& other) const { + MixedRadix res; + res.set_sum(*this, other, -1); + return res; + } + + MixedRadix operator*(int factor) const { + MixedRadix res; + res.set_sum(zero(), *this, factor); + return res; + } + + int operator%(int b) const { + int x = a[N - 1] % b; + for (int i = N - 2; i >= 0; --i) { + x = ((long long)x * mod[i] + a[i]) % b; + } + return ((x ^ b) < 0 && x) ? x + b : x; + } + + explicit operator double() const { + double acc = 0.; + for (int i = N - 1; i >= 0; --i) { + acc = acc * mod[i] + a[i]; + } + return acc; + } + + explicit operator long long() const { + long long acc = 0.; + for (int i = N - 1; i >= 0; --i) { + acc = acc * mod[i] + a[i]; + } + return acc; + } + + MixedRadix& to_base(int base) { + int k = N - 1; + while (k > 0 && !a[k]) { + --k; + } + if (k <= 0) { + return *this; + } + for (int i = k - 1; i >= 0; --i) { + // a[i..k] := a[i+1..k] * mod[i] + a[i] + long long carry = a[i]; + for (int j = i; j < k; j++) { + long long t = (long long)a[j + 1] * mod[i] + carry; + carry = t / base; + a[j] = (int)(t - carry * base); + } + a[k] = (int)carry; + } + return *this; + } + + std::ostream& print_dec_destroy(std::ostream& os) { + int s = sgn(); + if (s < 0) { + os << '-'; + negate(); + } else if (!s) { + os << '0'; + return os; + } + to_base(1000000000); + int i = N - 1; + while (!a[i] && i > 0) { + --i; + } + os << a[i]; + while (--i >= 0) { + char buff[12]; + sprintf(buff, "%09d", a[i]); + os << buff; + } + return os; + } + + std::ostream& print_dec(std::ostream& os) const& { + MixedRadix copy{*this}; + return copy.print_dec_destroy(os); + } + + std::ostream& print_dec(std::ostream& os) && { + return print_dec_destroy(os); + } + + std::string to_dec_string_destroy() { + std::ostringstream os; + print_dec_destroy(os); + return std::move(os).str(); + } + + std::string to_dec_string() const& { + MixedRadix copy{*this}; + return copy.to_dec_string_destroy(); + } + + std::string to_dec_string() && { + return to_dec_string_destroy(); + } + + bool to_binary_destroy(unsigned char* arr, int size, bool sgnd = true) { + if (size <= 0) { + return false; + } + int s = (sgnd ? sgn() : 1); + memset(arr, 0, size); + if (s < 0) { + negate(); + } else if (!s) { + return true; + } + to_base(1 << 30); + long long acc = 0; + int bits = 0, j = size; + for (int i = 0; i < N; i++) { + if (!j && a[i]) { + return false; + } + acc += ((long long)a[i] << bits); + bits += 30; + while (bits >= 8 && j > 0) { + arr[--j] = (unsigned char)(acc & 0xff); + bits -= 8; + acc >>= 8; + } + } + while (j > 0) { + arr[--j] = (unsigned char)(acc & 0xff); + acc >>= 8; + } + if (acc) { + return false; + } + if (!sgnd) { + return true; + } + if (s >= 0) { + return arr[0] <= 0x7f; + } + j = size - 1; + while (j >= 0 && !arr[j]) { + --j; + } + assert(j >= 0); + arr[j] = (unsigned char)(-arr[j]); + while (--j >= 0) { + arr[j] = (unsigned char)~arr[j]; + } + return arr[0] >= 0x80; + } + + bool to_binary(unsigned char* arr, int size, bool sgnd = true) const& { + MixedRadix copy{*this}; + return copy.to_binary_destroy(arr, size, sgnd); + } + + bool to_binary(unsigned char* arr, int size, bool sgnd = true) && { + return to_binary_destroy(arr, size, sgnd); + } + + std::ostream& raw_dump(std::ostream& os) const { + return raw_dump_array(os, a); + } + + ArrayRawDumpRef dump() const { + return {a}; + } +}; + +template +struct ModArray { + enum { n = N }; + int a[N]; + ModArray(int v) { + set_int(v); + } + ModArray(long long v) { + set_long(v); + } + ModArray(long v) { + set_long(v); + } + ModArray() = default; + ModArray(const ModArray&) = default; + ModArray(std::initializer_list l) { + auto sz = std::min(l.size(), (std::size_t)N); + std::copy(l.begin(), l.begin() + sz, a); + std::fill(a + sz, a + N, 0); + } + ModArray(const std::array& arr) { + std::copy(arr.begin(), arr.end(), a); + } + template + ModArray(const ModArray& other) { + static_assert(M >= N); + std::copy(other.a, other.a + N, a); + } + ModArray(const int* p) : a(p) { + } + ModArray(std::string str) { + assert(from_dec_string(str) && "not a decimal number"); + } + + ModArray& set_zero() { + std::fill(a, a + N, 0); + return *this; + } + ModArray& set_one() { + std::fill(a, a + N, 1); + return *this; + } + + ModArray& set_int(int v) { + if (v >= 0) { + std::fill(a, a + N, v); + } else { + for (int i = 0; i < N; i++) { + a[i] = mod[i] + v; + } + } + return *this; + } + + ModArray& set_long(long long v) { + for (int i = 0; i < N; i++) { + a[i] = v % mod[i]; + if (a[i] < 0) { + a[i] += mod[i]; + } + } + return *this; + } + + ModArray copy() const { + return ModArray{*this}; + } + + static const int* mod_array() { + return mod; + } + + static int modulus(int i) { + return mod[i]; + } + + static const ModArray& zero(); + static const ModArray& one(); + + ModArray& operator=(const ModArray&) = default; + + template + ModArray& operator=(const ModArray& other) { + static_assert(M >= N); + std::copy(other.a, other.a + N, a); + return *this; + } + + ModArray& negate() { + for (int i = 0; i < N; i++) { + a[i] = (a[i] ? mod[i] - a[i] : 0); + } + return *this; + } + + ModArray& norm_neg() { + for (int i = 0; i < N; i++) { + if (a[i] < 0) { + a[i] += mod[i]; + } + } + return *this; + } + + ModArray& normalize() { + for (int i = 0; i < N; i++) { + a[i] %= mod[i]; + if (a[i] < 0) { + a[i] += mod[i]; + } + } + return *this; + } + + bool is_zero() const { + for (int i = 0; i < N; i++) { + if (a[i]) { + return false; + } + } + return true; + } + + explicit operator bool() const { + return !is_zero(); + } + + bool operator!() const { + return is_zero(); + } + + bool operator==(const ModArray& other) const { + return std::equal(a, a + N, other.a); + } + + bool operator!=(const ModArray& other) const { + return !std::equal(a, a + N, other.a); + } + + bool operator==(long long val) const { + for (int i = 0; i < N; i++) { + int r = (int)(val % mod[i]); + if (a[i] != (r < 0 ? r + mod[i] : r)) { + return false; + } + } + return true; + } + + bool operator!=(long long val) const { + return !operator==(val); + } + + long long try_get_long() const { + return (long long)(MixedRadix<3>(*this)); + } + + bool fits_long() const { + return operator==(try_get_long()); + } + + explicit operator long long() const { + auto v = try_get_long(); + return operator==(v) ? v : -0x8000000000000000; + } + + ModArray& set_sum(const ModArray& x, const ModArray& y) { + for (int i = 0; i < N; i++) { + a[i] = x.a[i] + y.a[i]; + if (a[i] >= mod[i]) { + a[i] -= mod[i]; + } + } + return *this; + } + + ModArray& operator+=(const ModArray& other) { + for (int i = 0; i < N; i++) { + a[i] += other.a[i]; + if (a[i] >= mod[i]) { + a[i] -= mod[i]; + } + } + return *this; + } + + ModArray& operator+=(long long v) { + for (int i = 0; i < N; i++) { + a[i] = (int)((a[i] + v) % mod[i]); + if (a[i] < 0) { + a[i] += mod[i]; + } + } + return *this; + } + + ModArray& operator-=(const ModArray& other) { + for (int i = 0; i < N; i++) { + a[i] -= other.a[i]; + if (a[i] < 0) { + a[i] += mod[i]; + } + } + return *this; + } + + ModArray& operator-=(long long v) { + return (operator+=)(-v); + } + + ModArray& mul_arr(const int other[]) { + for (int i = 0; i < N; i++) { + a[i] = (int)(((long long)a[i] * other[i]) % mod[i]); + } + return *this; + } + + ModArray& operator*=(const ModArray& other) { + return mul_arr(other.a); + } + + template + ModArray& operator*=(const ModArray& other) { + static_assert(M >= N); + return mul_arr(other.a); + } + + ModArray& operator*=(int v) { + for (int i = 0; i < N; i++) { + a[i] = (int)(((long long)a[i] * v) % mod[i]); + } + return (v >= 0 ? *this : norm_neg()); + } + + ModArray& operator*=(long long v) { + for (int i = 0; i < N; i++) { + a[i] = (int)(((long long)a[i] * (v % mod[i])) % mod[i]); + } + return (v >= 0 ? *this : norm_neg()); + } + + ModArray& mul_add(int v, long long w) { + for (int i = 0; i < N; i++) { + a[i] = (int)(((long long)a[i] * v + w) % mod[i]); + if (a[i] < 0) { + a[i] += mod[i]; + } + } + return *this; + } + + // *this = (*this * other) + w + ModArray& mul_add(const ModArray& other, long long w) { + for (int i = 0; i < N; i++) { + a[i] = (int)(((long long)a[i] * other.a[i] + w) % mod[i]); + if (a[i] < 0) { + a[i] += mod[i]; + } + } + return *this; + } + + // *this = (*this << shift) + w + ModArray& lshift_add(int shift, long long w) { + return mul_add(pow2(shift), w); + } + + // *this = *this + other * w + ModArray& add_mul(const ModArray& other, long long w) { + for (int i = 0; i < N; i++) { + a[i] = (int)((a[i] + other.a[i] * w) % mod[i]); + if (a[i] < 0) { + a[i] += mod[i]; + } + } + return *this; + } + + // *this += w << shift + ModArray& add_lshift(int shift, long long w) { + return add_mul(pow2(shift), w); + } + + ModArray operator+(const ModArray& other) const { + ModArray copy{*this}; + copy += other; + return copy; + } + + ModArray operator-(const ModArray& other) const { + ModArray copy{*this}; + copy -= other; + return copy; + } + + ModArray operator+(long long other) const { + ModArray copy{*this}; + copy += other; + return copy; + } + + ModArray operator-(long long other) const { + ModArray copy{*this}; + copy += -other; + return copy; + } + + ModArray operator-() const { + ModArray copy{*this}; + copy.negate(); + return copy; + } + + ModArray operator*(const ModArray& other) const { + ModArray copy{*this}; + copy *= other; + return copy; + } + + ModArray operator*(long long other) const { + ModArray copy{*this}; + copy *= other; + return copy; + } + + bool invert() { + for (int i = 0; i < N; i++) { + int t; + if (gcdx(a[i], mod[i], a[i], t) != 1) { + return false; + } + if (a[i] < 0) { + a[i] += mod[i]; + } + } + return true; + } + + bool try_divide(const ModArray& other) { + for (int i = 0; i < N; i++) { + int q, t; + if (gcdx(other.a[i], mod[i], q, t) != 1) { + return false; + } + a[i] = (int)((long long)a[i] * q % mod[i]); + if (a[i] < 0) { + a[i] += mod[i]; + } + } + return true; + } + + ModArray& operator/=(const ModArray& other) { + assert(try_divide(other) && "division by zero?"); + return *this; + } + + ModArray operator/(const ModArray& other) { + ModArray copy{*this}; + copy /= other; + return copy; + } + + static const ModArray& pow2(int power); + static const ModArray& negpow2(int power); + + ModArray& operator<<=(int lshift) { + return operator*=(pow2(lshift)); + } + + ModArray operator<<(int lshift) const { + return operator*(pow2(lshift)); + } + + ModArray& operator>>=(int rshift) { + return operator/=(pow2(rshift)); + } + + ModArray operator>>(int rshift) const { + return operator/(pow2(rshift)); + } + + template + const ModArray& as_shorter() const { + static_assert(M <= N); + return *reinterpret_cast*>(this); + } + + MixedRadix& to_mixed_radix(MixedRadix& dest, bool sgnd = true) const { + return dest.import_mod_array(a, sgnd); + } + + MixedRadix to_mixed_radix(bool sgnd = true) const { + return MixedRadix(*this, sgnd); + } + + int operator%(int div) const { + return to_mixed_radix() % div; + } + + explicit operator double() const { + return (double)to_mixed_radix(); + } + + std::string to_dec_string() const { + return MixedRadix(*this).to_dec_string(); + } + + std::ostream& print_dec(std::ostream& os, bool sgnd = true) const { + return MixedRadix(*this, sgnd).print_dec(os); + } + + bool to_binary(unsigned char* arr, int size, bool sgnd = true) const { + return MixedRadix(*this, sgnd).to_binary(arr, size, sgnd); + } + + template + bool to_binary(std::array& arr, bool sgnd = true) const { + return to_binary(arr.data(), M, sgnd); + } + + bool from_dec_string(const char* start, const char* end) { + set_zero(); + if (start >= end) { + return false; + } + bool sgn = (*start == '-'); + if (sgn && ++start == end) { + return false; + } + int acc = 0, pow = 1; + while (start < end) { + if (*start < '0' || *start > '9') { + return false; + } + acc = acc * 10 + (*start++ - '0'); + pow *= 10; + if (pow >= 1000000000) { + mul_add(pow, acc); + pow = 1; + acc = 0; + } + } + if (pow > 1) { + mul_add(pow, acc); + } + if (sgn) { + negate(); + } + return true; + } + + bool from_dec_string(std::string str) { + return from_dec_string(str.data(), str.data() + str.size()); + } + + ModArray& from_binary(const unsigned char* arr, int size, bool sgnd = true) { + set_zero(); + if (size <= 0) { + return *this; + } + int i = 0, pow = 0; + long long acc = (sgnd && arr[0] >= 0x80 ? -1 : 0); + while (i < size && arr[i] == (unsigned char)acc) { + i++; + } + for (; i < size; i++) { + pow += 8; + acc = (acc << 8) + arr[i]; + if (pow >= 56) { + lshift_add(pow, acc); + acc = pow = 0; + } + } + if (pow || acc) { + lshift_add(pow, acc); + } + return *this; + } + + template + ModArray& from_binary(const std::array& arr, bool sgnd = true) { + return from_binary(arr.data(), M, sgnd); + } + + std::ostream& raw_dump(std::ostream& os) const { + return raw_dump_array(os, a); + } + + ArrayRawDumpRef dump() const { + return {a}; + } +}; + +template +MixedRadix::MixedRadix(const ModArray& other) { + import_mod_array(other.a); +} + +template +MixedRadix::MixedRadix(const ModArray& other, bool sgnd) { + import_mod_array(other.a, sgnd); +} + +template +MixedRadix& MixedRadix::import_mod_array(const ModArray& other, bool sgnd) { + return import_mod_array(other.a, sgnd); +} + +template +std::ostream& operator<<(std::ostream& os, const ModArray& x) { + return x.print_dec(os); +} + +template +std::ostream& operator<<(std::ostream& os, const MixedRadix& x) { + return x.print_dec(os); +} + +template +std::ostream& operator<<(std::ostream& os, MixedRadix&& x) { + return x.print_dec_destroy(os); +} + +template +struct ArrayRawDumpRef { + const std::array& ref; + ArrayRawDumpRef(const std::array& _ref) : ref(_ref){}; +}; + +template +std::ostream& operator<<(std::ostream& os, ArrayRawDumpRef rd_ref) { + return raw_dump_array(os, rd_ref.ref); +}; + +constexpr int pow2_cnt = 1001; + +ModArray Zero(0), One(1), Pow2[pow2_cnt], NegPow2[pow2_cnt]; +MixedRadix Zero_mr(0), One_mr(1), Pow2_mr[pow2_cnt], NegPow2_mr[pow2_cnt]; + +template +const MixedRadix& MixedRadix::pow2(int power) { + return Pow2_mr[power].as_shorter(); +} + +/* +template +const MixedRadix& MixedRadix::negpow2(int power) { + return NegPow2_mr[power].as_shorter(); +} +*/ + +template +const ModArray& ModArray::pow2(int power) { + return Pow2[power].as_shorter(); +} + +template +const ModArray& ModArray::negpow2(int power) { + return NegPow2[power].as_shorter(); +} + +template +const ModArray& ModArray::zero() { + return Zero.as_shorter(); +} + +template +const ModArray& ModArray::one() { + return One.as_shorter(); +} + +template +const MixedRadix& MixedRadix::zero() { + return Zero_mr.as_shorter(); +} + +template +const MixedRadix& MixedRadix::one() { + return One_mr.as_shorter(); +} + +void init_pow2() { + Pow2[0].set_one(); + Pow2_mr[0].set_one(); + for (int i = 1; i < pow2_cnt; i++) { + Pow2[i].set_sum(Pow2[i - 1], Pow2[i - 1]); + Pow2_mr[i].set_sum(Pow2_mr[i - 1], Pow2_mr[i - 1]); + } + for (int i = 0; i < pow2_cnt; i++) { + NegPow2[i] = -Pow2[i]; + NegPow2_mr[i] = -Pow2_mr[i]; + } +} + +int gcdx(int a, int b, int& u, int& v) { + int a1 = 1, a2 = 0, b1 = 0, b2 = 1; + while (b) { + int q = a / b; + int t = a - q * b; + a = b; + b = t; + t = a1 - q * b1; + a1 = b1; + b1 = t; + t = a2 - q * b2; + a2 = b2; + b2 = t; + } + u = a1; + v = a2; + return a; +} + +void init_invm() { + for (int i = 0; i < mod_cnt; i++) { + assert(mod[i] > 0 && mod[i] <= (1 << 30)); + for (int j = 0; j < i; j++) { + assert(gcdx(mod[i], mod[j], invm[i][j], invm[j][i]) == 1); + if (invm[i][j] < 0) { + invm[i][j] += mod[j]; + } + if (invm[j][i] < 0) { + invm[j][i] += mod[i]; + } + } + } +} + +void init() { + init_invm(); + init_pow2(); +} + +} // namespace modint diff --git a/crypto/test/test-bigint.cpp b/crypto/test/test-bigint.cpp new file mode 100644 index 00000000..bf85e2ad --- /dev/null +++ b/crypto/test/test-bigint.cpp @@ -0,0 +1,876 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include +#include +#include +#include +#include +#include +#include +#include +#include "common/refcnt.hpp" +#include "common/bigint.hpp" +#include "common/refint.h" +#include "modbigint.cpp" + +#include "td/utils/tests.h" + +int mkint_chk_mode = -1, res_chk_mode = 0; +long long iterations = 100000, cur_iteration = -1, debug_iteration = -2; +#define IFDEBUG if (cur_iteration == debug_iteration || debug_iteration == -3) + +using BInt = modint::ModArray<18>; // integers up to 2^537 +using MRInt = modint::MixedRadix<18>; // auxiliary integer representation for printing, comparing etc + +MRInt p2_256, np2_256, p2_63, np2_63; +constexpr long long ll_min = -2 * (1LL << 62), ll_max = ~ll_min; +constexpr double dbl_pow256 = 1.1579208923731619542e77 /* 0x1p256 */; // 2^256 + +std::mt19937_64 Random(666); + +template +bool equal(td::RefInt256 x, T y) { + return !td::cmp(x, y); +} + +bool equal_or_nan(td::RefInt256 x, td::RefInt256 y) { + return equal(x, y) || (!x->is_valid() && !y->fits_bits(257)) || (!y->is_valid() && !x->fits_bits(257)); +} + +#define CHECK_EQ(__x, __y) CHECK(equal(__x, __y)) +#define CHECK_EQ_NAN(__x, __y) CHECK(equal_or_nan(__x, __y)) + +bool mr_in_range(const MRInt& x) { + return x < p2_256 && x >= np2_256; +} + +bool mr_is_small(const MRInt& x) { + return x < p2_63 && x >= np2_63; +} + +bool mr_fits_bits(const MRInt& x, int bits) { + if (bits > 0) { + return x < MRInt::pow2(bits - 1) && x >= MRInt::negpow2(bits - 1); + } else { + return !bits && !x.sgn(); + } +} + +bool mr_ufits_bits(const MRInt& x, int bits) { + return bits >= 0 && x.sgn() >= 0 && x < MRInt::pow2(bits); +} + +struct ShowBin { + unsigned char* data; + ShowBin(unsigned char _data[64]) : data(_data) { + } +}; + +std::ostream& operator<<(std::ostream& os, ShowBin bin) { + int i = 0, s = bin.data[0]; + if (s == 0 || s == 0xff) { + while (i < 64 && bin.data[i] == s) { + i++; + } + } + if (i >= 3) { + os << (s ? "ff..ff" : "00..00"); + } else { + i = 0; + } + constexpr static char hex_digits[] = "0123456789abcdef"; + while (i < 64) { + int t = bin.data[i++]; + os << hex_digits[t >> 4] << hex_digits[t & 15]; + } + return os; +} + +std::ostream& operator<<(std::ostream& os, const td::AnyIntView& x) { + os << '['; + for (int i = 0; i < x.size(); i++) { + os << ' ' << x.digits[i]; + } + os << " ]"; + return os; +} + +template +bool extract_value_any_bool(BInt& val, const td::AnyIntView& x, bool chk_norm = true) { + int n = x.size(); + if (n <= 0 || n > x.max_size() || (!x.digits[n - 1] && n > 1)) { + return false; + } + assert(n == 1 || x.digits[n - 1] != 0); + val.set_zero(); + for (int i = n - 1; i >= 0; --i) { + val.lshift_add(T::word_shift, x.digits[i]); + if (chk_norm && (x.digits[i] < -T::Half || x.digits[i] >= T::Half)) { + return false; // unnormalized + } + } + return true; +} + +template +bool extract_value_bool(BInt& val, const T& x, bool chk_norm = true) { + return extract_value_any_bool(val, x.as_any_int(), chk_norm); +} + +BInt extract_value_any(const td::AnyIntView& x, bool chk_norm = true) { + BInt res; + CHECK(extract_value_any_bool(res, x, chk_norm)); + return res; +} + +template +BInt extract_value(const T& x, bool chk_norm = true) { + return extract_value_any(x.as_any_int(), chk_norm); +} + +template +BInt extract_value_alt(const T& x) { + BInt res; + const int* md = res.mod_array(); + for (int i = 0; i < res.n / 2; i++) { + T copy{x}; + int m1 = md[2 * i], m2 = md[2 * i + 1]; + long long rem = copy.divmod_short((long long)m1 * m2); + res.a[2 * i] = (int)(rem % m1); + res.a[2 * i + 1] = (int)(rem % m2); + } + if (res.n & 1) { + T copy{x}; + res.a[res.n - 1] = (int)copy.divmod_short(md[res.n - 1]); + } + return res; +} + +constexpr int min_spec_int = -0xfd08, max_spec_int = 0xfd07; +// x = sgn*(ord*256+a*16+b) => sgn*((32+a)*2^(ord-2) + b - 8) +// x = -0xfd08 => -2^256 ... x = 0xfd07 => 2^256 - 1 +td::RefInt256 make_special_int(int x, BInt* ptr = nullptr, unsigned char bin[64] = nullptr) { + bool sgn = (x < 0); + if (sgn) { + x = -x; + } + int ord = (x >> 8) - 2, a = 32 + ((x >> 4) & 15), b = (x & 15) - 8; + if (ord < 0) { + a >>= -ord; + ord = 0; + } + if (sgn) { + a = -a; + b = -b; + } + if (ptr) { + ptr->set_int(a); + *ptr <<= ord; + *ptr += b; + } + if (bin) { + int acc = b, r = ord; + for (int i = 63; i >= 0; --i) { + if (r < 8) { + acc += (a << r); + r = 1024; + } + r -= 8; + bin[i] = (unsigned char)(acc & 0xff); + acc >>= 8; + } + } + return (td::make_refint(a) << ord) + b; +} + +int rand_int(int min, int max) { + return min + (int)(Random() % (max - min + 1)); +} + +unsigned randu() { + return (unsigned)(Random() << 16); +} + +bool coin() { + return Random() & (1 << 28); +} + +// returns 0 with probability 1/2, 1 with prob. 1/4, ..., k with prob. 1/2^(k+1) +int randexp(int max = 63, int min = 0) { + return min + __builtin_clzll(Random() | (1ULL << (63 - max + min))); +} + +void bin_add_small(unsigned char bin[64], long long val, int shift = 0) { + val <<= shift & 7; + for (int i = 63 - (shift >> 3); i >= 0 && val; --i) { + val += bin[i]; + bin[i] = (unsigned char)val; + val >>= 8; + } +} + +// adds sgn * (random number less than 2^(ord - ord2)) * 2^ord2 +td::RefInt256 add_random_bits(td::RefInt256 x, BInt& val, unsigned char bin[64], int ord2, int ord, int sgn = 1) { + int t; + do { + t = std::max((ord - 1) & -16, ord2); + int a = sgn * rand_int(0, (1 << (ord - t)) - 1); + // add a << t + val.add_lshift(t, a); + x += td::make_refint(a) << t; + bin_add_small(bin, a, t); + ord = t; + } while (t > ord2); + return x; +} + +// generates a random integer in range -2^256 .. 2^256-1 (and sometimes outside) +// distribution is skewed towards +/- 2^n +/- 2^n +/- smallint, but completely random integers are also generated +td::RefInt256 make_random_int0(BInt& val, unsigned char bin[64]) { + memset(bin, 0, 64); + int ord = rand_int(-257, 257); + if (ord <= 2 && ord >= -2) { + // -2..2 represent themselves + val.set_int(ord); + bin_add_small(bin, ord); + return td::make_refint(ord); + } + int sgn = (ord < 0 ? -1 : 1); + ord = sgn * ord - 1; + int f = std::min(ord, randexp(15)), a = sgn * rand_int(1 << f, (2 << f) - 1); + ord -= f; + // first summand is a << ord + auto res = td::make_refint(a) << ord; + val.set_int(a); + val <<= ord; + bin_add_small(bin, a, ord); + if (!ord) { + // all bits ready + return res; + } + for (int s = 0; s < 2 && ord; s++) { + // decide whether we want an intermediate order (50%), and whether we want randomness above/below that order + int ord2 = (s ? 0 : std::max(0, rand_int(~ord, ord - 1))); + if (!rand_int(0, 4)) { // 20% + // random bits between ord2 and ord + res = add_random_bits(std::move(res), val, bin, ord2, ord, sgn); + } + if (rand_int(0, 4)) { // 80% + // non-zero adjustment + f = randexp(15); + a = rand_int(-(2 << f) + 1, (2 << f) - 1); + ord = std::max(ord2 - f, 0); + // add a << ord + val.add_lshift(ord, a); + res += (td::make_refint(a) << ord); + bin_add_small(bin, a, ord); + } + } + return res; +} + +td::RefInt256 make_random_int(BInt& val, unsigned char bin[64]) { + while (true) { + auto res = make_random_int0(val, bin); + if (res->fits_bits(257)) { + return res; + } + } +} + +void check_one_int_repr(td::RefInt256 x, int mode, int in_range, const BInt* valptr = nullptr, + const unsigned char bin[64] = nullptr) { + CHECK(x.not_null() && (in_range <= -2 || x->is_valid())); + if (!x->is_valid()) { + // not much to check when x is a NaN + unsigned char bytes[64]; + if (valptr) { + // check that the true answer at `valptr` is out of range + CHECK(!mr_in_range(valptr->to_mixed_radix())); + if (mode & 0x200) { + // check BInt binary export + valptr->to_binary(bytes, 64); + if (bin) { + // check that the two true answers match + CHECK(!memcmp(bin, bytes, 64)); + } else { + bin = bytes; + } + } + } + if (bin) { + // check that the true answer in `bin` is out of range + int i = 0, sgn = (bin[0] >= 0x80 ? -1 : 0); + while (i < 32 && bin[i] == (unsigned char)sgn) + ; + CHECK(i < 32); + if (valptr && (mode & 0x100)) { + // check BInt binary export + BInt val2; + val2.from_binary(bin, 64); + CHECK(*valptr == val2); + } + } + return; + } + unsigned char bytes[64]; + CHECK(x->export_bytes(bytes, 64)); + if (bin) { + CHECK(!memcmp(bytes, bin, 64)); + } + BInt val = extract_value(*x); + if (valptr) { + if (val != *valptr) { + std::cerr << "extracted " << val << " from " << x << ' ' << x->as_any_int() << ", expected " << *valptr + << std::endl; + } + CHECK(val == *valptr); + } + if (mode & 1) { + BInt val2 = extract_value_alt(*x); + CHECK(val == val2); + } + if (mode & 2) { + // check binary import + td::BigInt256 y; + y.import_bytes(bytes, 64); + CHECK(y == *x); + } + if (mode & 0x100) { + // check binary import for BInt + BInt val2; + val2.from_binary(bytes, 64); + CHECK(val == val2); + } + // check if small (fits into 64 bits) + long long xval = (long long)val; + bool is_small = (xval != ll_min || val == xval); + CHECK(is_small == x->fits_bits(64)); + if (is_small) { + // special check for small (64-bit) values + CHECK(x->to_long() == xval); + CHECK((long long)__builtin_bswap64(*(long long*)(bytes + 64 - 8)) == xval); + CHECK(in_range); + // check sign + CHECK(x->sgn() == (xval > 0 ? 1 : (xval < 0 ? -1 : 0))); + // check comparison with long long + CHECK(x == xval); + CHECK(!cmp(x, xval)); + if (mode & 4) { + // check constructor from long long + CHECK(!cmp(x, td::make_refint(xval))); + if (xval != ll_min) { + CHECK(x > xval - 1); + CHECK(x > td::make_refint(xval - 1)); + } + if (xval != ll_max) { + CHECK(x < xval + 1); + CHECK(x < td::make_refint(xval + 1)); + } + } + if (!(mode & ~0x107)) { + return; // fast check for small ints in this case + } + } + + MRInt mval(val); // somewhat slow + bool val_in_range = mr_in_range(mval); + CHECK(x->fits_bits(257) == val_in_range); + if (in_range >= 0) { + CHECK((int)val_in_range == in_range); + } + if (mode & 0x200) { + // check binary export for BInt + unsigned char bytes2[64]; + mval.to_binary(bytes2, 64); + CHECK(!memcmp(bytes, bytes2, 64)); + } + // check sign + int sgn = mval.sgn(); + CHECK(x->sgn() == sgn); + CHECK(is_small == mr_is_small(mval)); + if (is_small) { + CHECK((long long)mval == xval); + } + if (mode & 0x10) { + // check decimal export + std::string dec = mval.to_dec_string(); + CHECK(x->to_dec_string() == dec); + // check decimal import + td::BigInt256 y; + int l = y.parse_dec(dec); + CHECK((std::size_t)l == dec.size() && y == *x); + if (mode & 0x1000) { + // check decimal import for BInt + BInt val2; + CHECK(val2.from_dec_string(dec) && val2 == val); + } + } + if (mode & 0x20) { + // check binary bit size + int sz = x->bit_size(); + CHECK(sz >= 0 && sz <= 300); + CHECK(x->fits_bits(sz) && (!sz || !x->fits_bits(sz - 1))); + CHECK(mr_fits_bits(mval, sz) && !mr_fits_bits(mval, sz - 1)); + int usz = x->bit_size(false); + CHECK(sgn >= 0 || usz == 0x7fffffff); + if (sgn >= 0) { + CHECK(x->unsigned_fits_bits(usz) && (!usz || !x->unsigned_fits_bits(usz - 1))); + CHECK(mr_ufits_bits(mval, usz) && !mr_ufits_bits(mval, usz - 1)); + } else { + CHECK(!x->unsigned_fits_bits(256) && !x->unsigned_fits_bits(300)); + } + } +} + +void init_aux() { + np2_256 = p2_256 = MRInt::pow2(256); + np2_256.negate(); + CHECK(np2_256 == MRInt::negpow2(256)); + p2_63 = np2_63 = MRInt::pow2(63); + np2_63.negate(); + CHECK(np2_63 == MRInt::negpow2(63)); +} + +std::vector SpecInt; +BInt SpecIntB[max_spec_int - min_spec_int + 1]; + +void init_check_special_ints() { + std::cerr << "check special ints" << std::endl; + BInt b; + unsigned char binary[64]; + for (int idx = min_spec_int - 512; idx <= max_spec_int + 512; idx++) { + td::RefInt256 x = make_special_int(idx, &b, binary); + check_one_int_repr(x, mkint_chk_mode, idx >= min_spec_int && idx <= max_spec_int, &b, binary); + if (idx >= min_spec_int && idx <= max_spec_int) { + SpecIntB[idx - min_spec_int] = b; + SpecInt.push_back(std::move(x)); + } + } +} + +void check_res(td::RefInt256 y, const BInt& yv) { + check_one_int_repr(std::move(y), res_chk_mode, -2, &yv); +} + +void check_unary_ops_on(td::RefInt256 x, const BInt& xv) { + // NEGATE + BInt yv = -xv; + check_res(-x, yv); + // NOT + check_res(~x, yv -= 1); +} + +void check_unary_ops() { + std::cerr << "check unary ops" << std::endl; + for (int idx = min_spec_int; idx <= max_spec_int; idx++) { + check_unary_ops_on(SpecInt[idx - min_spec_int], SpecIntB[idx - min_spec_int]); + } +} + +void check_pow2_ops(int shift) { + // POW2 + td::RefInt256 r{true}; + r.unique_write().set_pow2(shift); + check_res(r, BInt::pow2(shift)); + // POW2DEC + r.unique_write().set_pow2(shift).add_tiny(-1).normalize(); + check_res(r, BInt::pow2(shift) - 1); + // NEGPOW2 + r.unique_write().set_pow2(shift).negate().normalize(); + check_res(r, -BInt::pow2(shift)); +} + +void check_pow2_ops() { + std::cerr << "check power-2 ops" << std::endl; + for (int i = 0; i <= 256; i++) { + check_pow2_ops(i); + } +} + +void check_shift_ops_on(int shift, td::RefInt256 x, const BInt& xv, const MRInt& mval) { + // LSHIFT + check_res(x << shift, xv << shift); + // FITS + CHECK(x->fits_bits(shift) == mr_fits_bits(mval, shift)); + // UFITS + CHECK(x->unsigned_fits_bits(shift) == mr_ufits_bits(mval, shift)); + // ADDPOW2 / SUBPOW2 + auto y = x; + y.write().add_pow2(shift).normalize(); + check_res(std::move(y), xv + BInt::pow2(shift)); + y = x; + y.write().sub_pow2(shift).normalize(); + check_res(std::move(y), xv - BInt::pow2(shift)); + // RSHIFT, MODPOW2 + for (int round_mode = -1; round_mode <= 1; round_mode++) { + auto r = x, q = td::rshift(x, shift, round_mode); // RSHIFT + CHECK(q.not_null() && q->is_valid()); + r.write().mod_pow2(shift, round_mode).normalize(); // MODPOW2 + CHECK(r.not_null() && r->is_valid()); + if (round_mode < 0) { + CHECK(!cmp(x >> shift, q)); // operator>> should be equivalent to td::rshift + } + BInt qv = extract_value(*q), rv = extract_value(*r); + // check main division equality (q << shift) + r == x + CHECK((qv << shift) + rv == xv); + MRInt rval(rv); + // check remainder range + switch (round_mode) { + case 1: + rval.negate(); // fallthrough + case -1: + CHECK(mr_ufits_bits(rval, shift)); + break; + case 0: + CHECK(mr_fits_bits(rval, shift)); + } + } +} + +void check_shift_ops() { + std::cerr << "check left/right shift ops" << std::endl; + for (int idx = min_spec_int; idx <= max_spec_int; idx++) { + //for (int idx : {-52240, -52239, -52238, -3, -2, -1, 0, 1, 2, 3, 52238, 52239, 52240}) { + const auto& xv = SpecIntB[idx - min_spec_int]; + MRInt mval(xv); + if (!(idx % 1000)) { + std::cerr << "# " << idx << " : " << mval << std::endl; + } + for (int i = 0; i <= 256; i++) { + check_shift_ops_on(i, SpecInt[idx - min_spec_int], xv, mval); + } + } +} + +void check_remainder_range(BInt& rv, const BInt& dv, int rmode = -1) { + if (rmode > 0) { + rv.negate(); + } else if (!rmode) { + rv *= 2; + } + MRInt d(dv), r(rv); + int ds = d.sgn(), rs = r.sgn(); + //std::cerr << "rmode=" << rmode << " ds=" << ds << " rs=" << rs << " d=" << d << " r=" << r << std::endl; + if (!rs) { + return; + } + if (rmode) { + // must have 0 < r < d or 0 > r > d + //if (rs != ds) std::cerr << "iter=" << cur_iteration << " : rmode=" << rmode << " ds=" << ds << " rs=" << rs << " d=" << d << " r=" << r << std::endl; + CHECK(rs == ds); + CHECK(ds * r.cmp(d) < 0); + } else { + // must have -d <= r < d or -d >= r > d + if (rs == -ds) { + r.negate(); + CHECK(ds * r.cmp(d) <= 0); + } else { + CHECK(ds * r.cmp(d) < 0); + } + } +} + +void check_divmod(td::RefInt256 x, const BInt& xv, long long xl, td::RefInt256 y, const BInt& yv, long long yl, + int rmode = -2) { + if (rmode < -1) { + //IFDEBUG std::cerr << " divide " << x << " / " << y << std::endl; + for (rmode = -1; rmode <= 1; rmode++) { + check_divmod(x, xv, xl, y, yv, yl, rmode); + } + return; + } + auto dm = td::divmod(x, y, rmode); + auto q = std::move(dm.first), r = std::move(dm.second); + if (!yl) { + // division by zero + CHECK(q.not_null() && !q->is_valid() && r.not_null() && !r->is_valid()); + return; + } + CHECK(q.not_null() && q->is_valid() && r.not_null() && r->is_valid()); + CHECK_EQ(x, y * q + r); + BInt qv = extract_value(*q), rv = extract_value(*r); + CHECK(xv == yv * qv + rv); + //IFDEBUG std::cerr << " quot=" << q << " rem=" << r << std::endl; + check_remainder_range(rv, yv, rmode); + if (yl != ll_min && rmode == -1) { + // check divmod_short() + auto qq = x; + auto rem = qq.write().divmod_short(yl); + qq.write().normalize(); + CHECK(qq->is_valid()); + CHECK_EQ(qq, q); + CHECK(r == rem); + if (xl != ll_min) { + auto dm = std::lldiv(xl, yl); + if (dm.rem && (dm.rem ^ yl) < 0) { + dm.rem += yl; + dm.quot--; + } + CHECK(q == dm.quot); + CHECK(r == dm.rem); + } + } +} + +void check_binary_ops_on(td::RefInt256 x, const BInt& xv, td::RefInt256 y, const BInt& yv) { + bool x_small = x->fits_bits(62), y_small = y->fits_bits(62); // not 63 + long long xl = x_small ? x->to_long() : ll_min, yl = y_small ? y->to_long() : ll_min; + if (x_small) { + CHECK(x == xl); + } + if (y_small) { + CHECK(y == yl); + } + // ADD, ADDR + auto z = x + y, w = y + x; + CHECK_EQ(z, w); + check_res(z, xv + yv); + // ADDCONST + if (y_small) { + CHECK_EQ(z, x + yl); + } + if (x_small) { + CHECK_EQ(z, y + xl); + } + if (x_small && y_small) { + CHECK_EQ(z, xl + yl); + } + // SUB + z = x - y; + check_res(z, xv - yv); + // SUBCONST + if (y_small) { + CHECK_EQ(z, x - yl); + if (x_small) { + CHECK_EQ(z, xl - yl); + } + } + // SUBR + z = y - x; + check_res(z, yv - xv); + if (x_small) { + CHECK_EQ(z, y - xl); + if (y_small) { + CHECK_EQ(z, yl - xl); + } + } + // CMP + MRInt xmr(xv), ymr(yv); + int cmpv = xmr.cmp(ymr); + CHECK(td::cmp(x, y) == cmpv); + CHECK(td::cmp(y, x) == -cmpv); + if (y_small) { + CHECK(td::cmp(x, yl) == cmpv); + } + if (x_small) { + CHECK(td::cmp(y, xl) == -cmpv); + } + if (x_small && y_small) { + CHECK(cmpv == (xl < yl ? -1 : (xl > yl ? 1 : 0))); + } + // MUL + z = x * y; + BInt zv = xv * yv; + check_res(z, zv); + CHECK_EQ(z, y * x); + // MULCONST + if (y_small) { + CHECK_EQ_NAN(z, x * yl); + } + if (x_small) { + CHECK_EQ_NAN(z, y * xl); + } + if (x_small && y_small && (!yl || std::abs(xl) <= ll_max / std::abs(yl))) { + CHECK_EQ(z, xl * yl); + } + // DIVMOD + if (z->fits_bits(257)) { + int adj = 2 * rand_int(-2, 2) - (int)z->is_odd(); + z += adj; + z >>= 1; + zv += adj; + zv >>= 1; + // z is approximately x * y / 2; divide by y + check_divmod(z, zv, z->fits_bits(62) ? z->to_long() : ll_min, y, yv, yl); + } + check_divmod(x, xv, xl, y, yv, yl); +} + +void finish_check_muldivmod(td::RefInt256 x, const BInt& xv, td::RefInt256 y, const BInt& yv, td::RefInt256 z, + const BInt& zv, td::RefInt256 q, td::RefInt256 r, int rmode) { + static constexpr double eps = 1e-14; + CHECK(q.not_null() && r.not_null()); + //std::cerr << " muldivmod: " << xv << " * " << yv << " / " << zv << " (round " << rmode << ") = " << q << " " << r << std::endl; + if (!zv) { + // division by zero + CHECK(!q->is_valid() && !r->is_valid()); + return; + } + CHECK(r->is_valid()); // remainder always exists if y != 0 + BInt xyv = xv * yv, rv = extract_value(*r); + MRInt xy_mr(xyv), z_mr(zv); + double q0 = (double)xy_mr / (double)z_mr; + if (std::abs(q0) < 1.01 * dbl_pow256) { + // result more or less in range + CHECK(q->is_valid()); + } else if (!q->is_valid()) { + // result out of range, NaN is an acceptable answer + // check that x * y - r is divisible by z + xyv -= rv; + xyv /= zv; + xy_mr = xyv; + double q1 = (double)xy_mr; + CHECK(std::abs(q1 - q0) < eps * std::abs(q0)); + } else { + BInt qv = extract_value(*q); + // must have x * y = z * q + r + CHECK(xv * yv == zv * qv + rv); + } + // check that r is in correct range [0, z) or [0, -z) or [-z/2, z/2) + check_remainder_range(rv, zv, rmode); +} + +void check_muldivmod_on(td::RefInt256 x, const BInt& xv, td::RefInt256 y, const BInt& yv, td::RefInt256 z, + const BInt& zv, int rmode = 2) { + if (rmode < -1) { + for (rmode = -1; rmode <= 1; rmode++) { + check_muldivmod_on(x, xv, y, yv, z, zv, rmode); + } + return; + } else if (rmode > 1) { + rmode = rand_int(-1, 1); + } + // MULDIVMOD + auto qr = td::muldivmod(x, y, z, rmode); + finish_check_muldivmod(std::move(x), xv, std::move(y), yv, std::move(z), zv, std::move(qr.first), + std::move(qr.second), rmode); +} + +void check_mul_rshift_on(td::RefInt256 x, const BInt& xv, td::RefInt256 y, const BInt& yv, int shift, int rmode = 2) { + if (rmode < -1) { + for (rmode = -1; rmode <= 1; rmode++) { + check_mul_rshift_on(x, xv, y, yv, shift, rmode); + } + return; + } else if (rmode > 1) { + rmode = rand_int(-1, 1); + } + // MULRSHIFTMOD + typename td::BigInt256::DoubleInt tmp{0}; + tmp.add_mul(*x, *y); + typename td::BigInt256::DoubleInt tmp2{tmp}; + tmp2.rshift(shift, rmode).normalize(); + tmp.normalize().mod_pow2(shift, rmode).normalize(); + finish_check_muldivmod(std::move(x), xv, std::move(y), yv, {}, BInt::pow2(shift), td::make_refint(tmp2), + td::make_refint(tmp), rmode); +} + +void check_lshift_div_on(td::RefInt256 x, const BInt& xv, td::RefInt256 y, const BInt& yv, int shift, int rmode = 2) { + if (rmode < -1) { + for (rmode = -1; rmode <= 1; rmode++) { + check_lshift_div_on(x, xv, y, yv, shift, rmode); + } + return; + } else if (rmode > 1) { + rmode = rand_int(-1, 1); + } + // LSHIFTDIV + typename td::BigInt256::DoubleInt tmp{*x}, quot; + tmp <<= shift; + tmp.mod_div(*y, quot, rmode); + quot.normalize(); + finish_check_muldivmod(std::move(x), xv, {}, BInt::pow2(shift), std::move(y), yv, td::make_refint(quot), + td::make_refint(tmp), rmode); +} + +void check_random_ops() { + constexpr long long chk_it = 100000; + std::cerr << "check random ops (" << iterations << " iterations)" << std::endl; + BInt xv, yv, zv; + unsigned char xbin[64], ybin[64], zbin[64]; + for (cur_iteration = 0; cur_iteration < iterations; cur_iteration++) { + auto x = make_random_int0(xv, xbin); + if (!(cur_iteration % 10000)) { + std::cerr << "#" << cur_iteration << ": check on " << xv << " = " << ShowBin(xbin) << " = " << x->as_any_int() + << std::endl; + } + check_one_int_repr(x, cur_iteration < chk_it ? -1 : 0, -1, &xv, xbin); + MRInt xmr(xv); + if (!x->fits_bits(257)) { + continue; + } + check_unary_ops_on(x, xv); + for (int j = 0; j < 10; j++) { + int shift = rand_int(0, 256); + //std::cerr << "check shift by " << shift << std::endl; + check_shift_ops_on(shift, x, xv, xmr); + auto y = make_random_int(yv, ybin); + //std::cerr << " y = " << y << " = " << yv << " = " << ShowBin(ybin) << " = " << y->as_any_int() << std::endl; + check_one_int_repr(y, 0, 1, &yv, ybin); + check_binary_ops_on(x, xv, y, yv); + //std::cerr << " *>> " << shift << std::endl; + check_mul_rshift_on(x, xv, y, yv, shift); + //std::cerr << " <as_any_int() << std::endl; + check_muldivmod_on(x, xv, y, yv, z, zv); + } + } +} + +void check_special() { + std::cerr << "run special tests" << std::endl; + check_divmod((td::make_refint(-1) << 207) - 1, BInt::negpow2(207) - 1, ll_min, (td::make_refint(1) << 207) - 1, + BInt::pow2(207) - 1, ll_min); +} + +int main(int argc, char* const argv[]) { + bool do_check_shift_ops = false; + int i; + while ((i = getopt(argc, argv, "hSs:i:")) != -1) { + switch (i) { + case 'S': + do_check_shift_ops = true; + break; + case 's': + Random.seed(atoll(optarg)); + break; + case 'i': + iterations = atoll(optarg); + break; + default: + std::cerr << "unknown option: " << (char)i << std::endl; + // fall through + case 'h': + std::cerr << "usage:\t" << argv[0] << " [-S] [-i] [-s]" << std::endl; + return 2; + } + } + modint::init(); + init_aux(); + init_check_special_ints(); + check_pow2_ops(); + check_unary_ops(); + if (do_check_shift_ops) { + check_shift_ops(); + } + check_special(); + check_random_ops(); + return 0; +} diff --git a/crypto/vm/arithops.cpp b/crypto/vm/arithops.cpp index 823b4408..2831944b 100644 --- a/crypto/vm/arithops.cpp +++ b/crypto/vm/arithops.cpp @@ -387,18 +387,16 @@ int exec_muldivmod(VmState* st, unsigned args, int quiet) { auto z = stack.pop_int(); auto y = stack.pop_int(); auto x = stack.pop_int(); - typename td::BigInt256::DoubleInt tmp{0}; + typename td::BigInt256::DoubleInt tmp{0}, quot; tmp.add_mul(*x, *y); auto q = td::make_refint(); - tmp.mod_div(*z, q.unique_write(), round_mode); + tmp.mod_div(*z, quot, round_mode); switch ((args >> 2) & 3) { case 1: - q.unique_write().normalize(); - stack.push_int_quiet(std::move(q), quiet); + stack.push_int_quiet(td::make_refint(quot.normalize()), quiet); break; case 3: - q.unique_write().normalize(); - stack.push_int_quiet(std::move(q), quiet); + stack.push_int_quiet(td::make_refint(quot.normalize()), quiet); // fallthrough case 2: stack.push_int_quiet(td::make_refint(tmp), quiet); @@ -459,7 +457,7 @@ int exec_mulshrmod(VmState* st, unsigned args, int mode) { } // fallthrough case 2: - tmp.mod_pow2(z, round_mode).normalize(); + tmp.normalize().mod_pow2(z, round_mode).normalize(); stack.push_int_quiet(td::make_refint(tmp), mode & 1); break; } @@ -520,21 +518,17 @@ int exec_shldivmod(VmState* st, unsigned args, int mode) { } auto z = stack.pop_int(); auto x = stack.pop_int(); - typename td::BigInt256::DoubleInt tmp{*x}; + typename td::BigInt256::DoubleInt tmp{*x}, quot; tmp <<= y; switch ((args >> 2) & 3) { case 1: { - auto q = td::make_refint(); - tmp.mod_div(*z, q.unique_write(), round_mode); - q.unique_write().normalize(); - stack.push_int_quiet(std::move(q), mode & 1); + tmp.mod_div(*z, quot, round_mode); + stack.push_int_quiet(td::make_refint(quot.normalize()), mode & 1); break; } case 3: { - auto q = td::make_refint(); - tmp.mod_div(*z, q.unique_write(), round_mode); - q.unique_write().normalize(); - stack.push_int_quiet(std::move(q), mode & 1); + tmp.mod_div(*z, quot, round_mode); + stack.push_int_quiet(td::make_refint(quot.normalize()), mode & 1); stack.push_int_quiet(td::make_refint(tmp), mode & 1); break; } diff --git a/doc/tvm.tex b/doc/tvm.tex index 4f887ac2..71aa7abc 100644 --- a/doc/tvm.tex +++ b/doc/tvm.tex @@ -1531,6 +1531,7 @@ Examples: \item {\tt A934$tt$} --- same as {\tt RSHIFT $tt+1$}: ($x$ -- $\lfloor x\cdot 2^{-tt-1}\rfloor$). \item {\tt A938$tt$} --- {\tt MODPOW2 $tt+1$}: ($x$ -- $x\bmod 2^{tt+1}$). \item {\tt A985} --- {\tt MULDIVR} ($x$ $y$ $z$ -- $q'$), where $q'=\lfloor xy/z+1/2\rfloor$. +\item {\tt A988} --- {\tt MULMOD} ($x$ $y$ $z$ -- $r$), where $r=xy\bmod z=xy-qz$, $q=\lfloor xy/z\rfloor$. This operation always succeeds for $z\neq0$ and returns the correct value of~$r$, even if the intermediate result $xy$ or the quotient $q$ do not fit into 257 bits. \item {\tt A98C} --- {\tt MULDIVMOD} ($x$ $y$ $z$ -- $q$ $r$), where $q:=\lfloor x\cdot y/z\rfloor$, $r:=x\cdot y\bmod z$ (same as {\tt */MOD} in Forth). \item {\tt A9A4} --- {\tt MULRSHIFT} ($x$ $y$ $z$ -- $\lfloor xy\cdot2^{-z}\rfloor$) for $0\leq z\leq 256$. \item {\tt A9A5} --- {\tt MULRSHIFTR} ($x$ $y$ $z$ -- $\lfloor xy\cdot2^{-z}+1/2\rfloor$) for $0\leq z\leq 256$. @@ -1576,10 +1577,12 @@ We opted to make all arithmetic operations ``non-quiet'' (signaling) by default, \begin{itemize} \item {\tt B7xx} --- {\tt QUIET} prefix, transforming any arithmetic operation into its ``quiet'' variant, indicated by prefixing a {\tt Q} to its mnemonic. Such operations return {\tt NaN}s instead of throwing integer overflow exceptions if the results do not fit in {\it Integer\/}s, or if one of their arguments is a {\tt NaN}. Notice that this does not extend to shift amounts and other parameters that must be within a small range (e.g., 0--1023). Also notice that this does not disable type-checking exceptions if a value of a type other than {\it Integer\/} is supplied. \item {\tt B7A0} --- {\tt QADD} ($x$ $y$ -- $x+y$), always works if $x$ and $y$ are {\it Integer\/}s, but returns a {\tt NaN} if the addition cannot be performed. +\item {\tt B7A8} --- {\tt QMUL} ($x$ $y$ -- $xy$), returns the product of $x$ and $y$ if $-2^{256}\leq xy<2^{256}$. Otherwise returns a {\tt NaN}, even if $x=0$ and $y$ is a {\tt NaN}. \item {\tt B7A904} --- {\tt QDIV} ($x$ $y$ -- $\lfloor x/y\rfloor$), returns a {\tt NaN} if $y=0$, or if $y=-1$ and $x=-2^{256}$, or if either of $x$ or $y$ is a {\tt NaN}. +\item {\tt B7A98C} --- {\tt QMULDIVMOD} ($x$ $y$ $z$ -- $q$ $r$), where $q:=\lfloor x\cdot y/z\rfloor$, $r:=x\cdot y\bmod z$. If $z=0$, or if at least one of $x$, $y$, or $z$ is a {\tt NaN}, both $q$ and $r$ are set to {\tt NaN}. Otherwise the correct value of $r$ is always returned, but $q$ is replaced with {\tt NaN} if $q<-2^{256}$ or $q\geq2^{256}$. \item {\tt B7B0} --- {\tt QAND} ($x$ $y$ -- $x\&y$), bitwise ``and'' (similar to {\tt AND}), but returns a {\tt NaN} if either $x$ or $y$ is a {\tt NaN} instead of throwing an integer overflow exception. However, if one of the arguments is zero, and the other is a {\tt NaN}, the result is zero. \item {\tt B7B1} --- {\tt QOR} ($x$ $y$ -- $x\vee y$), bitwise ``or''. If $x=-1$ or $y=-1$, the result is always $-1$, even if the other argument is a {\tt NaN}. -\item {\tt B7B507} --- {\tt QUFITS 8} ($x$ -- $x'$), checks whether $x$ is an unsigned byte (i.e., whether $0\leq x<2^8$), and replaces $x$ with a {\tt NaN} if this is not the case; leaves $x$ intact otherwise (i.e., if $x$ is an unsigned byte). +\item {\tt B7B507} --- {\tt QUFITS 8} ($x$ -- $x'$), checks whether $x$ is an unsigned byte (i.e., whether $0\leq x<2^8$), and replaces $x$ with a {\tt NaN} if this is not the case; leaves $x$ intact otherwise (i.e., if $x$ is an unsigned byte or a {\tt NaN}). \end{itemize} \mysubsection{Comparison primitives} diff --git a/rldp2/CMakeLists.txt b/rldp2/CMakeLists.txt index f93f6e1e..33ae5b67 100644 --- a/rldp2/CMakeLists.txt +++ b/rldp2/CMakeLists.txt @@ -51,7 +51,7 @@ target_include_directories(rldp PUBLIC ${OPENSSL_INCLUDE_DIR} ) if (GSL_FOUND) - target_link_libraries(rldp2 PRIVATE GSL::gsl) + target_link_libraries(rldp2 PRIVATE gsl) target_compile_definitions(rldp2 PRIVATE -DTON_HAVE_GSL=1) endif() target_link_libraries(rldp2 PUBLIC tdutils tdactor fec adnl tl_api) From 1b480e39bae46fa7379f4442d0a75bb0cf3c46a6 Mon Sep 17 00:00:00 2001 From: OmicronTau Date: Thu, 23 Dec 2021 10:39:14 +0300 Subject: [PATCH 07/29] Fix infinity max_gas_limit for get methods --- crypto/smc-envelope/SmartContract.cpp | 2 +- validator/impl/liteserver.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crypto/smc-envelope/SmartContract.cpp b/crypto/smc-envelope/SmartContract.cpp index f3485be9..5f284b57 100644 --- a/crypto/smc-envelope/SmartContract.cpp +++ b/crypto/smc-envelope/SmartContract.cpp @@ -207,7 +207,7 @@ SmartContract::Answer SmartContract::run_get_method(Args args) const { args.c7 = prepare_vm_c7(now, args.balance); } if (!args.limits) { - args.limits = vm::GasLimits{1000000}; + args.limits = vm::GasLimits{1000000, 1000000}; } if (!args.stack) { args.stack = td::Ref(true); diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index 510f4906..6b0b9008 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -1150,7 +1150,7 @@ void LiteQuery::finish_runSmcMethod(td::BufferSlice shard_proof, td::BufferSlice long long gas_limit = client_method_gas_limit; LOG(DEBUG) << "creating VM with gas limit " << gas_limit; // **** INIT VM **** - vm::GasLimits gas{gas_limit}; + vm::GasLimits gas{gas_limit, gas_limit}; vm::VmState vm{std::move(code), std::move(stack_), gas, 1, std::move(data), vm::VmLog::Null()}; auto c7 = prepare_vm_c7(gen_utime, gen_lt, td::make_ref(acc.addr->clone()), balance); vm.set_c7(c7); // tuple with SmartContractInfo From 59fcd7cbd55412d26746a3a838df399cbfb4c4a7 Mon Sep 17 00:00:00 2001 From: OmicronTau Date: Fri, 31 Dec 2021 18:48:51 +0300 Subject: [PATCH 08/29] Deactivate accounts upon destruction in block --- crypto/block/transaction.cpp | 52 ++++++++++++++++++++++--------- crypto/block/transaction.h | 11 ++++--- validator/impl/collator.cpp | 9 ------ validator/impl/validate-query.cpp | 3 +- 4 files changed, 45 insertions(+), 30 deletions(-) diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index 3668a45f..e5cd701d 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -271,7 +271,7 @@ bool Account::recompute_tmp_addr(Ref& tmp_addr, int split_depth, } bool Account::init_rewrite_addr(int split_depth, td::ConstBitPtr orig_addr_rewrite) { - if (split_depth_set_ || !created || !set_split_depth(split_depth)) { + if (split_depth_set_ || !set_split_depth(split_depth)) { return false; } addr_orig = addr; @@ -304,15 +304,8 @@ bool Account::unpack(Ref shard_account, Ref extra, total_state = orig_total_state = account; auto acc_cs = load_cell_slice(std::move(account)); if (block::gen::t_Account.get_tag(acc_cs) == block::gen::Account::account_none) { - status = acc_nonexist; - last_paid = 0; - last_trans_end_lt_ = 0; is_special = special; - if (workchain != ton::workchainInvalid) { - addr_orig = addr; - addr_rewrite = addr.cbits(); - } - return compute_my_addr() && acc_cs.size_ext() == 1; + return acc_cs.size_ext() == 1 && init_new(now); } block::gen::Account::Record_account acc; block::gen::AccountStorage::Record storage; @@ -328,6 +321,7 @@ bool Account::unpack(Ref shard_account, Ref extra, case block::gen::AccountState::account_uninit: status = orig_status = acc_uninit; state_hash = addr; + forget_split_depth(); break; case block::gen::AccountState::account_frozen: status = orig_status = acc_frozen; @@ -396,10 +390,42 @@ bool Account::init_new(ton::UnixTime now) { state_hash = addr_orig; status = orig_status = acc_nonexist; split_depth_set_ = false; - created = true; return true; } +bool Account::forget_split_depth() { + split_depth_set_ = false; + split_depth_ = 0; + addr_orig = addr; + my_addr = my_addr_exact; + addr_rewrite = addr.bits(); + return true; +} + +bool Account::deactivate() { + if (status == acc_active) { + return false; + } + // forget special (tick/tock) info + tick = tock = false; + if (status == acc_nonexist || status == acc_uninit) { + // forget split depth and address rewriting info + forget_split_depth(); + // forget specific state hash for deleted or uninitialized accounts (revert to addr) + state_hash = addr; + } + // forget code and data (only active accounts remember these) + code.clear(); + data.clear(); + library.clear(); + // if deleted, balance must be zero + if (status == acc_nonexist && !balance.is_zero()) { + return false; + } + return true; +} + + bool Account::belongs_to_shard(ton::ShardIdFull shard) const { return workchain == shard.workchain && ton::shard_is_ancestor(shard.shard, addr); } @@ -2214,7 +2240,7 @@ Ref Transaction::commit(Account& acc) { CHECK((const void*)&acc == (const void*)&account); // export all fields modified by the Transaction into original account // NB: this is the only method that modifies account - if (orig_addr_rewrite_set && new_split_depth >= 0 && acc.status == Account::acc_nonexist && + if (orig_addr_rewrite_set && new_split_depth >= 0 && acc.status != Account::acc_active && acc_status == Account::acc_active) { LOG(DEBUG) << "setting address rewriting info for newly-activated account " << acc.addr.to_hex() << " with split_depth=" << new_split_depth @@ -2243,9 +2269,7 @@ Ref Transaction::commit(Account& acc) { acc.tick = new_tick; acc.tock = new_tock; } else { - acc.tick = acc.tock = false; - acc.split_depth_set_ = false; - acc.created = true; + CHECK(acc.deactivate()); } end_lt = 0; acc.push_transaction(root, start_lt); diff --git a/crypto/block/transaction.h b/crypto/block/transaction.h index c1cfbc54..398e2731 100644 --- a/crypto/block/transaction.h +++ b/crypto/block/transaction.h @@ -213,17 +213,16 @@ struct Account { bool is_special{false}; bool tick{false}; bool tock{false}; - bool created{false}; bool split_depth_set_{false}; unsigned char split_depth_{0}; int verbosity{3 * 0}; ton::UnixTime now_{0}; ton::WorkchainId workchain{ton::workchainInvalid}; td::BitArray<32> addr_rewrite; // rewrite (anycast) data, split_depth bits - ton::StdSmcAddress addr; // rewritten address (by replacing a prefix of `addr_orig` with `addr_rewrite`) - ton::StdSmcAddress addr_orig; // address indicated in smart-contract data - Ref my_addr; // address as stored in the smart contract (MsgAddressInt) - Ref my_addr_exact; // exact address without anycast info + ton::StdSmcAddress addr; // rewritten address (by replacing a prefix of `addr_orig` with `addr_rewrite`); it is the key in ShardAccounts + ton::StdSmcAddress addr_orig; // address indicated in smart-contract data (must coincide with hash of StateInit) + Ref my_addr; // address as stored in the smart contract (MsgAddressInt); corresponds to `addr_orig` + anycast info + Ref my_addr_exact; // exact address without anycast info; corresponds to `addr` and has no anycast (rewrite) info ton::LogicalTime last_trans_end_lt_; ton::LogicalTime last_trans_lt_; ton::Bits256 last_trans_hash_; @@ -250,6 +249,7 @@ struct Account { bool set_address(ton::WorkchainId wc, td::ConstBitPtr new_addr); bool unpack(Ref account, Ref extra, ton::UnixTime now, bool special = false); bool init_new(ton::UnixTime now); + bool deactivate(); bool recompute_tmp_addr(Ref& tmp_addr, int split_depth, td::ConstBitPtr orig_addr_rewrite) const; td::RefInt256 compute_storage_fees(ton::UnixTime now, const std::vector& pricing) const; bool is_masterchain() const { @@ -268,6 +268,7 @@ struct Account { friend struct Transaction; bool set_split_depth(int split_depth); bool check_split_depth(int split_depth) const; + bool forget_split_depth(); bool init_rewrite_addr(int split_depth, td::ConstBitPtr orig_addr_rewrite); private: diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 6b1c72a6..5820a2ce 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -1944,7 +1944,6 @@ std::unique_ptr Collator::make_account_from(td::ConstBitPtr addr } auto ptr = std::make_unique(workchain(), addr); if (account.is_null()) { - ptr->created = true; if (!ptr->init_new(now_)) { return nullptr; } @@ -2273,10 +2272,6 @@ Ref Collator::create_ordinary_transaction(Ref msg_root) { register_new_msgs(*trans); update_max_lt(acc->last_trans_end_lt_); - // temporary patch to stop producing dangerous block - if (acc->status == block::Account::acc_nonexist) { - block_full_ = true; - } return trans_root; } @@ -2495,10 +2490,6 @@ int Collator::process_one_new_message(block::NewOutMsg msg, bool enqueue_only, R if (!insert_out_msg(cb.finalize())) { return -1; } - // 6.5. check for temporary patch can be left here - if (block_full_) { - return 3; - } // 7. check whether the block is full now if (!block_limit_status_->fits(block::ParamLimits::cl_normal)) { block_full_ = true; diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index fc90be1d..02a16f83 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -4073,7 +4073,6 @@ std::unique_ptr ValidateQuery::make_account_from(td::ConstBitPtr Ref extra) { auto ptr = std::make_unique(workchain(), addr); if (account.is_null()) { - ptr->created = true; if (!ptr->init_new(now_)) { return nullptr; } @@ -4308,7 +4307,7 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT } } if (is_first && is_masterchain() && account.is_special && account.tick && - (tag != block::gen::TransactionDescr::trans_tick_tock || (td_cs.prefetch_ulong(4) & 1)) && !account.created) { + (tag != block::gen::TransactionDescr::trans_tick_tock || (td_cs.prefetch_ulong(4) & 1)) && account.orig_status == block::Account::acc_active) { return reject_query(PSTRING() << "transaction " << lt << " of account " << addr.to_hex() << " is the first transaction for this special tick account in this block, but the " "transaction is not a tick transaction"); From 112bed04786fba7b3a2d5f3cdb1a62bf0b5d9257 Mon Sep 17 00:00:00 2001 From: sonofmom Date: Mon, 10 Jan 2022 23:07:53 +0100 Subject: [PATCH 09/29] Add base64 key input parameter to LiteClient --- lite-client/lite-client.cpp | 2 ++ lite-client/lite-client.h | 12 ++++++++++++ 2 files changed, 14 insertions(+) diff --git a/lite-client/lite-client.cpp b/lite-client/lite-client.cpp index 1ccfec86..f472af21 100644 --- a/lite-client/lite-client.cpp +++ b/lite-client/lite-client.cpp @@ -4261,6 +4261,8 @@ int main(int argc, char* argv[]) { }); p.add_option('p', "pub", "remote public key", [&](td::Slice arg) { td::actor::send_closure(x, &TestNode::set_public_key, td::BufferSlice{arg}); }); + p.add_option('b', "b64", "remote public key as base64", + [&](td::Slice arg) { td::actor::send_closure(x, &TestNode::decode_public_key, td::BufferSlice{arg}); }); p.add_option('d', "daemonize", "set SIGHUP", [&]() { td::set_signal_handler(td::SignalType::HangUp, [](int sig) { #if TD_DARWIN || TD_LINUX diff --git a/lite-client/lite-client.h b/lite-client/lite-client.h index 7184652a..00d8bd07 100644 --- a/lite-client/lite-client.h +++ b/lite-client/lite-client.h @@ -394,6 +394,18 @@ class TestNode : public td::actor::Actor { } remote_public_key_ = R.move_as_ok(); } + void decode_public_key(td::BufferSlice b64_key) { + auto R = [&]() -> td::Result { + std::string key_bytes = {(char)0xc6, (char)0xb4, (char)0x13, (char)0x48}; + key_bytes = key_bytes + td::base64_decode(b64_key.as_slice().str()).move_as_ok(); + return ton::PublicKey::import(key_bytes); + }(); + + if (R.is_error()) { + LOG(FATAL) << "bad b64 server public key: " << R.move_as_error(); + } + remote_public_key_ = R.move_as_ok(); + } void set_fail_timeout(td::Timestamp ts) { fail_timeout_ = ts; alarm_timestamp().relax(fail_timeout_); From 64615a9766525191b1c825a1098482ad424274df Mon Sep 17 00:00:00 2001 From: Alex Melman Date: Wed, 12 Jan 2022 10:20:16 +0200 Subject: [PATCH 10/29] rocksdb submodule upgrade to 6.27.3 --- third-party/rocksdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third-party/rocksdb b/third-party/rocksdb index 0915c99f..fcf3d75f 160000 --- a/third-party/rocksdb +++ b/third-party/rocksdb @@ -1 +1 @@ -Subproject commit 0915c99f01b46f50af8e02da8b6528156f584b7c +Subproject commit fcf3d75f3f022a6a55ff1222d6b06f8518d38c7c From 7dc980562f1a3f5ff96c8144474a271627da61ff Mon Sep 17 00:00:00 2001 From: Starlight Duck Date: Thu, 20 Jan 2022 23:20:56 +0200 Subject: [PATCH 11/29] tonlib use correct c7: config and address when executing --- crypto/smc-envelope/SmartContract.cpp | 64 ++++++++++++++++++--------- crypto/smc-envelope/SmartContract.h | 18 ++++++++ crypto/vm/vm.cpp | 1 + crypto/vm/vm.h | 4 ++ tonlib/tonlib/LastConfig.cpp | 4 +- tonlib/tonlib/TonlibClient.cpp | 45 +++++++++++++------ tonlib/tonlib/TonlibClient.h | 1 + 7 files changed, 99 insertions(+), 38 deletions(-) diff --git a/crypto/smc-envelope/SmartContract.cpp b/crypto/smc-envelope/SmartContract.cpp index 5f284b57..83e6a4e3 100644 --- a/crypto/smc-envelope/SmartContract.cpp +++ b/crypto/smc-envelope/SmartContract.cpp @@ -52,30 +52,51 @@ td::Ref prepare_vm_stack(td::RefInt256 amount, td::Ref return stack_ref; } -td::Ref prepare_vm_c7(td::uint32 now, td::uint64 balance) { - // TODO: fix initialization of c7 +td::Ref prepare_vm_c7(SmartContract::Args args) { td::BitArray<256> rand_seed; rand_seed.as_slice().fill(0); td::RefInt256 rand_seed_int{true}; rand_seed_int.unique_write().import_bits(rand_seed.cbits(), 256, false); + + td::uint32 now = 0; + if (args.now) { + now = args.now.unwrap(); + } + + vm::CellBuilder cb; + if (args.address) { + td::BigInt256 dest_addr; + dest_addr.import_bits((*args.address).addr.as_bitslice()); + cb.store_ones(1) + .store_zeroes(2) + .store_long((*args.address).workchain, 8) + .store_int256(dest_addr, 256); + } + auto address = cb.finalize(); + auto config = td::Ref(); + + if (args.config) { + config = (*args.config)->get_root_cell(); + } + auto tuple = vm::make_tuple_ref( td::make_refint(0x076ef1ea), // [ magic:0x076ef1ea td::make_refint(0), // actions:Integer td::make_refint(0), // msgs_sent:Integer td::make_refint(now), // unixtime:Integer - td::make_refint(0), // block_lt:Integer - td::make_refint(0), // trans_lt:Integer + td::make_refint(0), //TODO: // block_lt:Integer + td::make_refint(0), //TODO: // trans_lt:Integer std::move(rand_seed_int), // rand_seed:Integer - block::CurrencyCollection(balance).as_vm_tuple(), // balance_remaining:[Integer (Maybe Cell)] - vm::load_cell_slice_ref(vm::CellBuilder().finalize()) // myself:MsgAddressInt - //vm::StackEntry::maybe(td::Ref()) + block::CurrencyCollection(args.balance).as_vm_tuple(), // balance_remaining:[Integer (Maybe Cell)] + vm::load_cell_slice_ref(address), // myself:MsgAddressInt + vm::StackEntry::maybe(config) //vm::StackEntry::maybe(td::Ref()) ); // global_config:(Maybe Cell) ] = SmartContractInfo; //LOG(DEBUG) << "SmartContractInfo initialized with " << vm::StackEntry(tuple).to_string(); return vm::make_tuple_ref(std::move(tuple)); } SmartContract::Answer run_smartcont(SmartContract::State state, td::Ref stack, td::Ref c7, - vm::GasLimits gas, bool ignore_chksig) { + vm::GasLimits gas, bool ignore_chksig, td::Ref libraries) { auto gas_credit = gas.gas_credit; vm::init_op_cp0(); vm::DictionaryBase::get_empty_dictionary(); @@ -108,11 +129,14 @@ SmartContract::Answer run_smartcont(SmartContract::State state, td::Ref SmartContract::get_init_state() const { } SmartContract::Answer SmartContract::run_method(Args args) { - td::uint32 now = 0; - if (args.now) { - now = args.now.unwrap(); - } if (!args.c7) { - args.c7 = prepare_vm_c7(now, args.balance); + args.c7 = prepare_vm_c7(args); } if (!args.limits) { bool is_internal = args.get_method_id().ok() == 0; @@ -193,18 +217,15 @@ SmartContract::Answer SmartContract::run_method(Args args) { CHECK(args.method_id); args.stack.value().write().push_smallint(args.method_id.unwrap()); auto res = - run_smartcont(get_state(), args.stack.unwrap(), args.c7.unwrap(), args.limits.unwrap(), args.ignore_chksig); + run_smartcont(get_state(), args.stack.unwrap(), args.c7.unwrap(), args.limits.unwrap(), args.ignore_chksig, + args.libraries ? args.libraries.unwrap().get_root_cell() : td::Ref{}); state_ = res.new_state; return res; } SmartContract::Answer SmartContract::run_get_method(Args args) const { - td::uint32 now = 0; - if (args.now) { - now = args.now.unwrap(); - } if (!args.c7) { - args.c7 = prepare_vm_c7(now, args.balance); + args.c7 = prepare_vm_c7(args); } if (!args.limits) { args.limits = vm::GasLimits{1000000, 1000000}; @@ -214,7 +235,8 @@ SmartContract::Answer SmartContract::run_get_method(Args args) const { } CHECK(args.method_id); args.stack.value().write().push_smallint(args.method_id.unwrap()); - return run_smartcont(get_state(), args.stack.unwrap(), args.c7.unwrap(), args.limits.unwrap(), args.ignore_chksig); + return run_smartcont(get_state(), args.stack.unwrap(), args.c7.unwrap(), args.limits.unwrap(), args.ignore_chksig, + args.libraries ? args.libraries.unwrap().get_root_cell() : td::Ref{}); } SmartContract::Answer SmartContract::run_get_method(td::Slice method, Args args) const { diff --git a/crypto/smc-envelope/SmartContract.h b/crypto/smc-envelope/SmartContract.h index b26f4885..95dd2e84 100644 --- a/crypto/smc-envelope/SmartContract.h +++ b/crypto/smc-envelope/SmartContract.h @@ -26,6 +26,7 @@ #include "td/utils/crypto.h" #include "block/block.h" +#include "block/mc-config.h" namespace ton { class SmartContract : public td::CntObject { @@ -48,6 +49,7 @@ class SmartContract : public td::CntObject { td::Ref actions; td::int32 code; td::int64 gas_used; + td::ConstBitPtr missing_library{0}; static int output_actions_count(td::Ref list); }; @@ -61,6 +63,10 @@ class SmartContract : public td::CntObject { td::uint64 amount{0}; td::uint64 balance{0}; + td::optional address; + td::optional> config; + td::optional libraries; + Args() { } Args(std::initializer_list stack) @@ -106,6 +112,18 @@ class SmartContract : public td::CntObject { this->balance = balance; return std::move(*this); } + Args&& set_address(block::StdAddress address) { + this->address = address; + return std::move(*this); + } + Args&& set_config(std::shared_ptr& config) { + this->config = config; + return std::move(*this); + } + Args&& set_libraries(vm::Dictionary libraries) { + this->libraries = libraries; + return std::move(*this); + } td::Result get_method_id() const { if (!method_id) { diff --git a/crypto/vm/vm.cpp b/crypto/vm/vm.cpp index 038640a5..fb2f4d1e 100644 --- a/crypto/vm/vm.cpp +++ b/crypto/vm/vm.cpp @@ -604,6 +604,7 @@ Ref VmState::load_library(td::ConstBitPtr hash) { return lib; } } + missing_library = hash; return {}; } diff --git a/crypto/vm/vm.h b/crypto/vm/vm.h index 0fef9642..44aa7f23 100644 --- a/crypto/vm/vm.h +++ b/crypto/vm/vm.h @@ -96,6 +96,7 @@ class VmState final : public VmStateInterface { td::int64 loaded_cells_count{0}; int stack_trace{0}, debug_off{0}; bool chksig_always_succeed{false}; + td::ConstBitPtr missing_library{0}; public: enum { @@ -321,6 +322,9 @@ class VmState final : public VmStateInterface { Ref ref_to_cont(Ref cell) const { return td::make_ref(load_cell_slice_ref(std::move(cell)), get_cp()); } + td::ConstBitPtr get_missing_library() const { + return missing_library; + } private: void init_cregs(bool same_c3 = false, bool push_0 = true); diff --git a/tonlib/tonlib/LastConfig.cpp b/tonlib/tonlib/LastConfig.cpp index d07482ce..0111095b 100644 --- a/tonlib/tonlib/LastConfig.cpp +++ b/tonlib/tonlib/LastConfig.cpp @@ -62,9 +62,7 @@ void LastConfig::with_last_block(td::Result r_last_block) { } auto last_block = r_last_block.move_as_ok(); - auto params = params_; - client_.send_query(ton::lite_api::liteServer_getConfigParams(0, create_tl_lite_block_id(last_block.last_block_id), - std::move(params)), + client_.send_query(ton::lite_api::liteServer_getConfigAll(0, create_tl_lite_block_id(last_block.last_block_id)), [this](auto r_config) { this->on_config(std::move(r_config)); }); } diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index 1ac3a0d7..422dae12 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -861,13 +861,13 @@ class Query { } return res; } - td::Result>> estimate_fees(bool ignore_chksig, const block::Config& cfg) { + td::Result>> estimate_fees(bool ignore_chksig, std::shared_ptr& cfg, vm::Dictionary& libraries) { // gas fees bool is_masterchain = raw_.source->get_address().workchain == ton::masterchainId; - TRY_RESULT(gas_limits_prices, cfg.get_gas_limits_prices(is_masterchain)); - TRY_RESULT(storage_prices, cfg.get_storage_prices()); - TRY_RESULT(masterchain_msg_prices, cfg.get_msg_prices(true)); - TRY_RESULT(basechain_msg_prices, cfg.get_msg_prices(false)); + TRY_RESULT(gas_limits_prices, cfg->get_gas_limits_prices(is_masterchain)); + TRY_RESULT(storage_prices, cfg->get_storage_prices()); + TRY_RESULT(masterchain_msg_prices, cfg->get_msg_prices(true)); + TRY_RESULT(basechain_msg_prices, cfg->get_msg_prices(false)); block::MsgPrices* msg_prices[2] = {&basechain_msg_prices, &masterchain_msg_prices}; auto storage_fee_256 = block::StoragePrices::compute_storage_fees( raw_.source->get_sync_time(), storage_prices, raw_.source->raw().storage_stat, @@ -888,7 +888,9 @@ class Query { .set_limits(gas_limits) .set_balance(raw_.source->get_balance()) .set_now(raw_.source->get_sync_time()) - .set_ignore_chksig(ignore_chksig)); + .set_ignore_chksig(ignore_chksig) + .set_address(raw_.source->get_address()) + .set_config(cfg).set_libraries(libraries)); td::int64 fwd_fee = 0; if (res.success) { LOG(DEBUG) << "output actions:\n" @@ -910,7 +912,7 @@ class Query { for (auto& destination : raw_.destinations) { bool dest_is_masterchain = destination && destination->get_address().workchain == ton::masterchainId; - TRY_RESULT(dest_gas_limits_prices, cfg.get_gas_limits_prices(dest_is_masterchain)); + TRY_RESULT(dest_gas_limits_prices, cfg->get_gas_limits_prices(dest_is_masterchain)); auto dest_storage_fee_256 = destination ? block::StoragePrices::compute_storage_fees( destination->get_sync_time(), storage_prices, destination->raw().storage_stat, @@ -3266,7 +3268,7 @@ void TonlibClient::query_estimate_fees(td::int64 id, bool ignore_chksig, td::Res return; } TRY_RESULT_PROMISE(promise, state, std::move(r_state)); - TRY_RESULT_PROMISE_PREFIX(promise, fees, TRY_VM(it->second->estimate_fees(ignore_chksig, *state.config)), + TRY_RESULT_PROMISE_PREFIX(promise, fees, TRY_VM(it->second->estimate_fees(ignore_chksig, state.config, libraries)), TonlibError::Internal()); promise.set_value(tonlib_api::make_object( fees.first.to_tonlib_api(), td::transform(fees.second, [](auto& x) { return x.to_tonlib_api(); }))); @@ -3493,14 +3495,29 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_runGetMethod& request, args.set_stack(std::move(stack)); args.set_balance(it->second->get_balance()); args.set_now(it->second->get_sync_time()); - auto res = smc->run_get_method(std::move(args)); + args.set_address(it->second->get_address()); - // smc.runResult gas_used:int53 stack:vector exit_code:int32 = smc.RunResult; - std::vector> res_stack; - for (auto& entry : res.stack->as_span()) { - res_stack.push_back(to_tonlib_api(entry)); + auto code = smc->get_state().code; + if (code.not_null()) { + std::vector libraryList; } - promise.set_value(tonlib_api::make_object(res.gas_used, std::move(res_stack), res.code)); + + args.set_libraries(libraries); + + client_.with_last_config([smc=std::move(smc), args=std::move(args), + promise = std::move(promise)](td::Result r_state) mutable { + TRY_RESULT_PROMISE(promise, state, std::move(r_state)); + args.set_config(state.config); + + auto res = smc->run_get_method(std::move(args)); + + // smc.runResult gas_used:int53 stack:vector exit_code:int32 = smc.RunResult; + std::vector> res_stack; + for (auto& entry : res.stack->as_span()) { + res_stack.push_back(to_tonlib_api(entry)); + } + promise.set_value(tonlib_api::make_object(res.gas_used, std::move(res_stack), res.code)); + }); return td::Status::OK(); } diff --git a/tonlib/tonlib/TonlibClient.h b/tonlib/tonlib/TonlibClient.h index bedc1d49..4dfbe8c6 100644 --- a/tonlib/tonlib/TonlibClient.h +++ b/tonlib/tonlib/TonlibClient.h @@ -107,6 +107,7 @@ class TonlibClient : public td::actor::Actor { td::optional block_id; }; QueryContext query_context_; + vm::Dictionary libraries{256}; // network td::actor::ActorOwn raw_client_; From 3fee04e20afd223198b5352520ceb77715cc13e8 Mon Sep 17 00:00:00 2001 From: Starlight Duck Date: Fri, 21 Jan 2022 03:43:35 +0200 Subject: [PATCH 12/29] Support libraries in LS and request+caching in TLC --- crypto/block/mc-config.cpp | 2 +- crypto/smc-envelope/SmartContract.cpp | 3 +- tl/generate/scheme/lite_api.tl | 3 + tl/generate/scheme/lite_api.tlo | Bin 12416 -> 12896 bytes tonlib/tonlib/TonlibClient.cpp | 178 +++++++++++++++++++++++--- tonlib/tonlib/TonlibClient.h | 6 +- validator/impl/liteserver.cpp | 70 ++++++++++ validator/impl/liteserver.hpp | 2 + 8 files changed, 245 insertions(+), 19 deletions(-) diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index 07574ef1..003cd04b 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -2100,7 +2100,7 @@ Ref ConfigInfo::lookup_library(td::ConstBitPtr root_hash) const { return {}; } auto csr = libraries_dict_->lookup(root_hash, 256); - if (csr.is_null() || csr->prefetch_ulong(8) != 0 || !csr->have_refs()) { // shared_lib_descr$00 lib:^Cell + if (csr.is_null() || csr->prefetch_ulong(2) != 0 || !csr->have_refs()) { // shared_lib_descr$00 lib:^Cell return {}; } auto lib = csr->prefetch_ref(); diff --git a/crypto/smc-envelope/SmartContract.cpp b/crypto/smc-envelope/SmartContract.cpp index 83e6a4e3..9a273a08 100644 --- a/crypto/smc-envelope/SmartContract.cpp +++ b/crypto/smc-envelope/SmartContract.cpp @@ -129,8 +129,9 @@ SmartContract::Answer run_smartcont(SmartContract::State state, td::RefE-j$d-O~Uz4o};u2{ISqYF?AaM{Va%R)v zux&dCQolJz)P|2yV)IJn-ON@=+mtRMdpJF{1mfq+)M9kQd7!q(=VTU_Fie(`kei$! NE5ZUcGeTX1697XhbTI$` delta 51 zcmaEm(vZmeXtur;0~F|N(kv.release()); + load_libs_from_disk(); + key_storage_.set_key_value(kv_); last_block_storage_.set_key_value(kv_); auto res = tonlib_api::make_object(); @@ -3474,6 +3476,38 @@ td::Result from_tonlib_api(tonlib_api::tvm_StackEntry& entry) { })); } +void deep_library_search(std::set& set, std::set& visited, + vm::Dictionary libs, td::Ref cell, int depth) { + if (depth <= 0 || set.size() >= 16 || visited.size() >= 256) { + return; + } + auto ins = visited.insert(cell->get_hash()); + if (!ins.second) { + return; // already visited this cell + } + auto r_loaded_cell = cell->load_cell(); + if (r_loaded_cell.is_error()) { + return; + } + auto loaded_cell = r_loaded_cell.move_as_ok(); + if (loaded_cell.data_cell->is_special()) { + if (loaded_cell.data_cell->special_type() == vm::DataCell::SpecialType::Library) { + vm::CellSlice cs(std::move(loaded_cell)); + if (cs.size() != vm::Cell::hash_bits + 8) { + return; + } + auto key = td::Bits256(cs.data_bits() + 8); + if (libs.lookup(key).is_null()) { + set.insert(key); + } + } + return; + } + for (unsigned int i=0; iget_refs_cnt(); i++) { + deep_library_search(set, visited, libs, loaded_cell.data_cell->get_ref(i), depth - 1); + } +} + td::Status TonlibClient::do_request(const tonlib_api::smc_runGetMethod& request, td::Promise>&& promise) { auto it = smcs_.find(request.id_); @@ -3497,30 +3531,120 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_runGetMethod& request, args.set_now(it->second->get_sync_time()); args.set_address(it->second->get_address()); - auto code = smc->get_state().code; - if (code.not_null()) { - std::vector libraryList; - } - - args.set_libraries(libraries); - - client_.with_last_config([smc=std::move(smc), args=std::move(args), - promise = std::move(promise)](td::Result r_state) mutable { + client_.with_last_config([self = this, smc = std::move(smc), args = std::move(args), promise = std::move(promise) + ](td::Result r_state) mutable { TRY_RESULT_PROMISE(promise, state, std::move(r_state)); args.set_config(state.config); - auto res = smc->run_get_method(std::move(args)); - - // smc.runResult gas_used:int53 stack:vector exit_code:int32 = smc.RunResult; - std::vector> res_stack; - for (auto& entry : res.stack->as_span()) { - res_stack.push_back(to_tonlib_api(entry)); + auto code = smc->get_state().code; + if (code.not_null()) { + std::set librarySet; + std::set visited; + deep_library_search(librarySet, visited, self->libraries, code, 24); + std::vector libraryList{librarySet.begin(), librarySet.end()}; + if (libraryList.size() > 0) { + LOG(DEBUG) << "Requesting found libraries in code (" << libraryList.size() << ")"; + self->client_.send_query(ton::lite_api::liteServer_getLibraries(std::move(libraryList)), + [self, smc = std::move(smc), args = std::move(args), promise = std::move(promise)] + (td::Result> r_libraries) mutable + { + if (r_libraries.is_error()) { + LOG(WARNING) << "cannot obtain found libraries: " << r_libraries.move_as_error().to_string(); + } else { + auto libraries = r_libraries.move_as_ok(); + bool updated = false; + for (auto& lr : libraries->result_) { + auto contents = vm::std_boc_deserialize(lr->data_); + if (contents.is_ok() && contents.ok().not_null()) { + if (contents.ok()->get_hash().bits().compare(lr->hash_.cbits(), 256)) { + LOG(WARNING) << "hash mismatch for library " << lr->hash_.to_hex(); + continue; + } + self->libraries.set_ref(lr->hash_, contents.move_as_ok()); + updated = true; + LOG(DEBUG) << "registered library " << lr->hash_.to_hex(); + } else { + LOG(WARNING) << "failed to deserialize library: " << lr->hash_.to_hex(); + } + } + if (updated) { + self->store_libs_to_disk(); + } + } + self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise)); + }); + } + else { + self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise)); + } + } + else { + self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise)); } - promise.set_value(tonlib_api::make_object(res.gas_used, std::move(res_stack), res.code)); }); return td::Status::OK(); } +void TonlibClient::perform_smc_execution(td::Ref smc, ton::SmartContract::Args args, + td::Promise>&& promise) { + + args.set_libraries(libraries); + + auto res = smc->run_get_method(args); + + // smc.runResult gas_used:int53 stack:vector exit_code:int32 = smc.RunResult; + std::vector> res_stack; + for (auto& entry : res.stack->as_span()) { + res_stack.push_back(to_tonlib_api(entry)); + } + + if (res.missing_library.not_null()) { + td::Bits256 hash = res.missing_library; + LOG(DEBUG) << "Requesting missing library: " << hash.to_hex(); + std::vector req = {std::move(hash)}; + client_.send_query(ton::lite_api::liteServer_getLibraries(std::move(req)), + [self = this, res = std::move(res), res_stack = std::move(res_stack), hash = std::move(hash), + smc = std::move(smc), args = std::move(args), promise = std::move(promise)] + (td::Result> r_libraries) mutable + { + if (r_libraries.is_error()) { + LOG(WARNING) << "cannot obtain missing library: " << r_libraries.move_as_error().to_string(); + promise.set_value(tonlib_api::make_object(res.gas_used, std::move(res_stack), res.code)); + return; + } + bool found = false, updated = false; + auto libraries = r_libraries.move_as_ok(); + for (auto& lr : libraries->result_) { + auto contents = vm::std_boc_deserialize(lr->data_); + if (contents.is_ok() && contents.ok().not_null()) { + if (contents.ok()->get_hash().bits().compare(lr->hash_.cbits(), 256)) { + LOG(WARNING) << "hash mismatch for library " << lr->hash_.to_hex(); + continue; + } + found |= (lr->hash_ == hash); + updated = true; + self->libraries.set_ref(lr->hash_, contents.move_as_ok()); + LOG(DEBUG) << "registered library " << lr->hash_.to_hex(); + } else { + LOG(WARNING) << "failed to deserialize library: " << lr->hash_.to_hex(); + } + } + if (updated) { + self->store_libs_to_disk(); + } + if (!found) { + LOG(WARNING) << "cannot obtain library " << hash.to_hex() << ", it may not exist"; + promise.set_value(tonlib_api::make_object(res.gas_used, std::move(res_stack), res.code)); + } else { + self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise)); + } + }); + } + else { + promise.set_value(tonlib_api::make_object(res.gas_used, std::move(res_stack), res.code)); + } +} + td::Result> to_tonlib_api( const ton::ManualDns::EntryData& entry_data) { td::Result> res; @@ -4246,6 +4370,28 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_getBlockHeader& req return td::Status::OK(); } +void TonlibClient::load_libs_from_disk() { + LOG(DEBUG) << "loading libraries from disk cache"; + auto r_data = kv_->get("tonlib.libcache"); + if (r_data.is_error()) { + return; + } + auto r_dict = vm::std_boc_deserialize(r_data.move_as_ok(), true); + if (r_dict.is_error()) { + return; + } + libraries = vm::Dictionary(vm::load_cell_slice(vm::CellBuilder().append_cellslice(vm::load_cell_slice( + r_dict.move_as_ok())).finalize()), 256); + // int n = 0; for (auto&& lr : libraries) n++; + LOG(DEBUG) << "loaded libraries from disk cache"; +} + +void TonlibClient::store_libs_to_disk() { // NB: Dictionary.get_root_cell does not compute_root, and it is protected + kv_->set("tonlib.libcache", vm::std_boc_serialize(vm::CellBuilder().append_cellslice(libraries.get_root()) + .finalize()).move_as_ok().as_slice()); + // int n = 0; for (auto&& lr : libraries) n++; + LOG(DEBUG) << "stored libraries to disk cache"; +} template td::Status TonlibClient::do_request(const tonlib_api::runTests& request, P&&) { diff --git a/tonlib/tonlib/TonlibClient.h b/tonlib/tonlib/TonlibClient.h index 4dfbe8c6..6d80c152 100644 --- a/tonlib/tonlib/TonlibClient.h +++ b/tonlib/tonlib/TonlibClient.h @@ -322,7 +322,8 @@ class TonlibClient : public td::actor::Actor { td::Status do_request(tonlib_api::pchan_unpackPromise& request, td::Promise>&& promise); - + void perform_smc_execution(td::Ref smc, ton::SmartContract::Args args, + td::Promise>&& promise); void do_dns_request(std::string name, td::int32 category, td::int32 ttl, td::optional block_id, block::StdAddress address, td::Promise>&& promise); @@ -358,6 +359,9 @@ class TonlibClient : public td::actor::Actor { void proxy_request(td::int64 query_id, std::string data); + void load_libs_from_disk(); + void store_libs_to_disk(); + friend class TonlibQueryActor; struct Target { bool can_be_empty{true}; diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index 6b0b9008..90d974e0 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -198,6 +198,9 @@ void LiteQuery::start_up() { this->perform_runSmcMethod(ton::create_block_id(q.id_), static_cast(q.account_->workchain_), q.account_->id_, q.mode_, q.method_id_, std::move(q.params_)); }, + [&](lite_api::liteServer_getLibraries& q) { + this->perform_getLibraries(q.library_list_); + }, [&](auto& obj) { this->abort_query(td::Status::Error(ErrorCode::protoviolation, "unknown query")); })); } @@ -781,6 +784,73 @@ void LiteQuery::perform_runSmcMethod(BlockIdExt blkid, WorkchainId workchain, St perform_getAccountState(blkid, workchain, addr, mode | 0x10000); } +void LiteQuery::perform_getLibraries(std::vector library_list) { + LOG(INFO) << "started a getLibraries() liteserver query"; + if (library_list.size() > 16) { + LOG(INFO) << "too many libraries requested, returning only first 16"; + library_list.resize(16); + } + sort( library_list.begin(), library_list.end() ); + library_list.erase( unique( library_list.begin(), library_list.end() ), library_list.end() ); + td::actor::send_closure_later( + manager_, &ton::validator::ValidatorManager::get_top_masterchain_state_block, + [Self = actor_id(this), library_list](td::Result, BlockIdExt>> res) -> void { + if (res.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); + } else { + auto pair = res.move_as_ok(); + td::actor::send_closure_later(Self, &LiteQuery::continue_getLibraries, std::move(pair.first), + pair.second, library_list); + } + }); +} + +void LiteQuery::continue_getLibraries(Ref mc_state, BlockIdExt blkid, std::vector library_list) { + LOG(INFO) << "obtained last masterchain block = " << blkid.to_str(); + base_blk_id_ = blkid; + CHECK(mc_state.not_null()); + mc_state_ = Ref(std::move(mc_state)); + CHECK(mc_state_.not_null()); + + auto rconfig = block::ConfigInfo::extract_config(mc_state_->root_cell(), block::ConfigInfo::needLibraries); + if (rconfig.is_error()) { + fatal_error("cannot extract library list block configuration from masterchain state"); + return; + } + auto config = rconfig.move_as_ok(); + + if (false) { + std::ostringstream os; + vm::load_cell_slice(config->get_libraries_root()).print_rec(os); + LOG(INFO) << "\n" << os.str(); + + auto lib_dict = std::make_unique(config->get_libraries_root(), 256); + for (auto k: *lib_dict) { + std::ostringstream oss; + k.second->print_rec(oss); + LOG(INFO) << "library " << k.first.to_hex(256) << ": \n" << oss.str(); + } + } + + std::vector> a; + for (const auto& hash : library_list) { + LOG(INFO) << "looking for library " << hash.to_hex(); + auto libres = config->lookup_library(hash); + if (libres.is_null()) { + LOG(INFO) << "library lookup result is null"; + continue; + } + auto data = vm::std_boc_serialize(libres); + if (data.is_error()) { + LOG(WARNING) << "library serialization failed: " << data.move_as_error().to_string(); + continue; + } + a.push_back(ton::create_tl_object(hash, data.move_as_ok())); + } + auto b = ton::create_serialize_tl_object(std::move(a)); + finish_query(std::move(b)); +} + void LiteQuery::perform_getOneTransaction(BlockIdExt blkid, WorkchainId workchain, StdSmcAddress addr, LogicalTime lt) { LOG(INFO) << "started a getOneTransaction(" << blkid.to_str() << ", " << workchain << ", " << addr.to_hex() << "," << lt << ") liteserver query"; diff --git a/validator/impl/liteserver.hpp b/validator/impl/liteserver.hpp index 24392085..f1918ceb 100644 --- a/validator/impl/liteserver.hpp +++ b/validator/impl/liteserver.hpp @@ -115,6 +115,8 @@ class LiteQuery : public td::actor::Actor { td::BufferSlice params); void finish_runSmcMethod(td::BufferSlice shard_proof, td::BufferSlice state_proof, Ref acc_root, UnixTime gen_utime, LogicalTime gen_lt); + void perform_getLibraries(std::vector library_list); + void continue_getLibraries(Ref mc_state, BlockIdExt blkid, std::vector library_list); void perform_getOneTransaction(BlockIdExt blkid, WorkchainId workchain, StdSmcAddress addr, LogicalTime lt); void continue_getOneTransaction(); void perform_getTransactions(WorkchainId workchain, StdSmcAddress addr, LogicalTime lt, Bits256 hash, unsigned count); From 76c2764e7ff7fe89c6a37bfd31f9b1d4208b1d9d Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Sat, 19 Feb 2022 16:39:59 +0300 Subject: [PATCH 13/29] Sparse state serialization over time by randomized delay --- validator/state-serializer.cpp | 52 +++++++++++++++++++++++----------- validator/state-serializer.hpp | 3 ++ 2 files changed, 38 insertions(+), 17 deletions(-) diff --git a/validator/state-serializer.cpp b/validator/state-serializer.cpp index 802159c0..63b7b864 100644 --- a/validator/state-serializer.cpp +++ b/validator/state-serializer.cpp @@ -81,6 +81,27 @@ void AsyncStateSerializer::alarm() { td::actor::send_closure(manager_, &ValidatorManager::get_top_masterchain_block, std::move(P)); } +void AsyncStateSerializer::request_masterchain_state() { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &AsyncStateSerializer::fail_handler, + R.move_as_error_prefix("failed to get masterchain state: ")); + } else { + td::actor::send_closure(SelfId, &AsyncStateSerializer::got_masterchain_state, + td::Ref(R.move_as_ok())); + } + }); + td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, masterchain_handle_, std::move(P)); +} + +void AsyncStateSerializer::request_shard_state(BlockIdExt shard) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + R.ensure(); + td::actor::send_closure(SelfId, &AsyncStateSerializer::got_shard_handle, R.move_as_ok()); + }); + return td::actor::send_closure(manager_, &ValidatorManager::get_block_handle, shard, true, std::move(P)); +} + void AsyncStateSerializer::next_iteration() { if (running_) { return; @@ -102,29 +123,26 @@ void AsyncStateSerializer::next_iteration() { if (attempt_ < max_attempt() && last_key_block_id_.id.seqno < last_block_id_.id.seqno && need_serialize(masterchain_handle_)) { if (masterchain_state_.is_null()) { - running_ = true; - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &AsyncStateSerializer::fail_handler, - R.move_as_error_prefix("failed to get masterchain state: ")); - } else { - td::actor::send_closure(SelfId, &AsyncStateSerializer::got_masterchain_state, - td::Ref(R.move_as_ok())); - } - }); - td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, masterchain_handle_, std::move(P)); + // block next attempts immediately, but send actual request later + running_ = true; + delay_action( + [SelfId = actor_id(this)]() { td::actor::send_closure(SelfId, &AsyncStateSerializer::request_masterchain_state); }, + // Masterchain is more important and much lighter than shards + // thus lower delay + td::Timestamp::in(td::Random::fast(0, 600))); return; } while (next_idx_ < shards_.size()) { if (!need_monitor(shards_[next_idx_].shard_full())) { next_idx_++; } else { - running_ = true; - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - R.ensure(); - td::actor::send_closure(SelfId, &AsyncStateSerializer::got_shard_handle, R.move_as_ok()); - }); - td::actor::send_closure(manager_, &ValidatorManager::get_block_handle, shards_[next_idx_], true, std::move(P)); + // block next attempts immediately, but send actual request later + running_ = true; + delay_action( + [SelfId = actor_id(this), shard = shards_[next_idx_]]() { td::actor::send_closure(SelfId, &AsyncStateSerializer::request_shard_state, shard); }, + // Shards are less important and heavier than master + // thus higher delay + td::Timestamp::in(td::Random::fast(0, 4 * 3600))); return; } } diff --git a/validator/state-serializer.hpp b/validator/state-serializer.hpp index 3b8d7052..162e96c1 100644 --- a/validator/state-serializer.hpp +++ b/validator/state-serializer.hpp @@ -65,6 +65,9 @@ class AsyncStateSerializer : public td::actor::Actor { void got_self_state(AsyncSerializerState state); void got_init_handle(BlockHandle handle); + void request_masterchain_state(); + void request_shard_state(BlockIdExt shard); + void next_iteration(); void got_top_masterchain_handle(BlockIdExt block_id); void got_masterchain_handle(BlockHandle handle_); From 45a160d7c19a7c7b6d76485f1ebc5a5b34845cc7 Mon Sep 17 00:00:00 2001 From: ms Date: Thu, 31 Mar 2022 14:29:43 +0200 Subject: [PATCH 14/29] Add getConfigParam to tonlibjson --- tl/generate/scheme/tonlib_api.tl | 5 +++++ tl/generate/scheme/tonlib_api.tlo | Bin 27936 -> 28216 bytes tonlib/tonlib/LastConfig.cpp | 2 +- tonlib/tonlib/LastConfig.h | 9 +------- tonlib/tonlib/TonlibClient.cpp | 36 ++++++++++++++++++++++-------- tonlib/tonlib/TonlibClient.h | 3 +++ 6 files changed, 37 insertions(+), 18 deletions(-) diff --git a/tl/generate/scheme/tonlib_api.tl b/tl/generate/scheme/tonlib_api.tl index b1dd7679..4527cc09 100644 --- a/tl/generate/scheme/tonlib_api.tl +++ b/tl/generate/scheme/tonlib_api.tl @@ -210,6 +210,9 @@ blocks.shortTxId mode:# account:mode.0?bytes lt:mode.1?int64 hash:mode.2?bytes = blocks.transactions id:ton.blockIdExt req_count:int32 incomplete:Bool transactions:vector = blocks.Transactions; blocks.header id:ton.blockIdExt global_id:int32 version:int32 after_merge:Bool after_split:Bool before_split:Bool want_merge:Bool want_split:Bool validator_list_hash_short:int32 catchain_seqno:int32 min_ref_mc_seqno:int32 is_key_block:Bool prev_key_block_seqno:int32 start_lt:int64 end_lt:int64 vert_seqno:# prev_blocks:vector = blocks.Header; //blocks.shortData header:blocks.Header transactions:blocks.Header = blocks.BlockData; + +configInfo config:tvm.cell = ConfigInfo; + ---functions--- init options:options = options.Info; @@ -262,6 +265,8 @@ guessAccount public_key:string rwallet_init_public_key:string = AccountRevisionL getAccountState account_address:accountAddress = FullAccountState; createQuery private_key:InputKey address:accountAddress timeout:int32 action:Action initial_account_state:InitialAccountState = query.Info; +getConfigParam mode:# id:ton.blockIdExt param:# = ConfigInfo; + msg.decrypt input_key:InputKey data:msg.dataEncryptedArray = msg.DataDecryptedArray; msg.decryptWithProof proof:bytes data:msg.dataEncrypted = msg.Data; diff --git a/tl/generate/scheme/tonlib_api.tlo b/tl/generate/scheme/tonlib_api.tlo index 77741950ec220ed7ec01cd6f79333282a16276d4..d5a8344d6d832315af5525ff6b0288e42aa7e453 100644 GIT binary patch delta 174 zcmZ2*i*d&tM&3uW^{p77Abcb5c1D5!p$wW_&iQ$1ndzQ+Y55Ej3nVrlVQk@HY}o9{ zx0+KDET4>!2k~zn6Y^o4yiiDd@&X?b7Lds17vc-d7+p4Z`t9eE?fN3l#Fw600x>lp tu_!T@0j7<4vS)zypH;+YW+FvWa{m3;_E=JL>=d delta 53 zcmdmShjGCzM&3uW^{p77AZ#P=cE-(b7+ZK4>o*tjt>)ZZATh;^(Rnkg|9-B`2~jtM E0oLph*8l(j diff --git a/tonlib/tonlib/LastConfig.cpp b/tonlib/tonlib/LastConfig.cpp index 0111095b..5ee73212 100644 --- a/tonlib/tonlib/LastConfig.cpp +++ b/tonlib/tonlib/LastConfig.cpp @@ -39,7 +39,7 @@ td::StringBuilder& operator<<(td::StringBuilder& sb, const LastConfigState& stat return sb; } -LastConfig::LastConfig(ExtClientRef client, td::unique_ptr callback) : callback_(std::move(callback)) { +LastConfig::LastConfig(ExtClientRef client) { client_.set_client(client); VLOG(last_block) << "State: " << state_; } diff --git a/tonlib/tonlib/LastConfig.h b/tonlib/tonlib/LastConfig.h index 514b4a59..d610165d 100644 --- a/tonlib/tonlib/LastConfig.h +++ b/tonlib/tonlib/LastConfig.h @@ -36,17 +36,10 @@ td::StringBuilder& operator<<(td::StringBuilder& sb, const LastConfigState& stat class LastConfig : public td::actor::Actor { public: - class Callback { - public: - virtual ~Callback() { - } - }; - - explicit LastConfig(ExtClientRef client, td::unique_ptr callback); + explicit LastConfig(ExtClientRef client); void get_last_config(td::Promise promise); private: - td::unique_ptr callback_; ExtClient client_; LastConfigState state_; diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index be2fe6c2..d75fb120 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -1439,17 +1439,9 @@ void TonlibClient::init_last_block(LastBlockState state) { } void TonlibClient::init_last_config() { ref_cnt_++; - class Callback : public LastConfig::Callback { - public: - Callback(td::actor::ActorShared client) : client_(std::move(client)) { - } - - private: - td::actor::ActorShared client_; - }; raw_last_config_ = td::actor::create_actor(td::actor::ActorOptions().with_name("LastConfig").with_poll(false), - get_client_ref(), td::make_unique(td::actor::actor_shared(this))); + get_client_ref()); } void TonlibClient::on_result(td::uint64 id, tonlib_api::object_ptr response) { @@ -4198,6 +4190,32 @@ auto to_lite_api(const tonlib_api::ton_blockIdExt& blk) -> td::Result>&& promise) { + TRY_RESULT(lite_block, to_lite_api(*request.id_)) + auto block = create_block_id(std::move(lite_block)); + auto param = request.param_; + std::vector params = { param }; + + client_.send_query(ton::lite_api::liteServer_getConfigParams(0, std::move(lite_block), std::move(params)), + promise.wrap([block, param](auto r_config) { + auto state = block::check_extract_state_proof(block, r_config->state_proof_.as_slice(), + r_config->config_proof_.as_slice()); + if (state.is_error()) { + LOG(ERROR) << "block::check_extract_state_proof failed: " << state.error(); + } + auto config = block::Config::extract_from_state(std::move(state.move_as_ok()), 0); + if (config.is_error()) { + LOG(ERROR) << "block::Config::extract_from_state failed: " << config.error(); + } + tonlib_api::configInfo config_result; + config_result.config_ = tonlib_api::make_object(to_bytes(config.move_as_ok()->get_config_param(param))); + return tonlib_api::make_object(std::move(config_result)); + })); + + return td::Status::OK(); +} + td::Status TonlibClient::do_request(const tonlib_api::blocks_getMasterchainInfo& masterchain_info, td::Promise>&& promise) { client_.send_query(ton::lite_api::liteServer_getMasterchainInfo(), diff --git a/tonlib/tonlib/TonlibClient.h b/tonlib/tonlib/TonlibClient.h index 6d80c152..716133e4 100644 --- a/tonlib/tonlib/TonlibClient.h +++ b/tonlib/tonlib/TonlibClient.h @@ -357,6 +357,9 @@ class TonlibClient : public td::actor::Actor { td::Status do_request(const tonlib_api::blocks_getBlockHeader& request, td::Promise>&& promise); + td::Status do_request(const tonlib_api::getConfigParam& request, + td::Promise>&& promise); + void proxy_request(td::int64 query_id, std::string data); void load_libs_from_disk(); From 34590176eb786d3c94b136463db2fa65cb159e6b Mon Sep 17 00:00:00 2001 From: akifoq <75585197+akifoq@users.noreply.github.com> Date: Tue, 5 Apr 2022 01:33:09 +0300 Subject: [PATCH 15/29] Fix IF:<{ and IFNOT:<{ empty continuation case The boolean value should be DROPped in the case the continuation is empty. --- crypto/fift/lib/Asm.fif | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crypto/fift/lib/Asm.fif b/crypto/fift/lib/Asm.fif index 76381b85..edd0e24b 100644 --- a/crypto/fift/lib/Asm.fif +++ b/crypto/fift/lib/Asm.fif @@ -694,9 +694,9 @@ x{E30F} @Defop(ref*2) IFREFELSEREF [] execute } : @run-cont-op { triple 1 ' @run-cont-op does create } : @def-cont-op -{ } { PUSHCONT IF } { IFREF } @def-cont-op IF-cont +{ DROP } { PUSHCONT IF } { IFREF } @def-cont-op IF-cont { IFRET } { PUSHCONT IFJMP } { IFJMPREF } @def-cont-op IFJMP-cont -{ } { PUSHCONT IFNOT } { IFNOTREF } @def-cont-op IFNOT-cont +{ DROP } { PUSHCONT IFNOT } { IFNOTREF } @def-cont-op IFNOT-cont { IFNOTRET } { PUSHCONT IFNOTJMP } { IFNOTJMPREF } @def-cont-op IFNOTJMP-cont { dup 2over rot } : 3dup From b4ac32a4520a4ccc88fa61b7915d7a8f328bc2a7 Mon Sep 17 00:00:00 2001 From: ms Date: Wed, 6 Apr 2022 19:34:16 +0200 Subject: [PATCH 16/29] Fix created_lt for in_out messages --- tonlib/tonlib/TonlibClient.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index d75fb120..7f9288b5 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -2302,9 +2302,10 @@ struct ToRawTransactions { return td::Status::Error("Failed to unpack CommonMsgInfo::ext_out_msg_info"); } TRY_RESULT(src, to_std_address(msg_info.src)); + auto created_lt = static_cast(msg_info.created_lt); return tonlib_api::make_object( tonlib_api::make_object(src), - tonlib_api::make_object(), 0, 0, 0, 0, std::move(body_hash), get_data(src)); + tonlib_api::make_object(), 0, 0, 0, created_lt, std::move(body_hash), get_data(src)); } } From ca167e92828b437863d786294e8ede6d66be2f8c Mon Sep 17 00:00:00 2001 From: ms Date: Thu, 14 Apr 2022 22:49:10 +0200 Subject: [PATCH 17/29] Remove limit on count of returned out_msgs --- tonlib/tonlib/TonlibClient.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index 7f9288b5..c6a83ebb 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -2350,7 +2350,7 @@ struct ToRawTransactions { if (trans.outmsg_cnt != 0) { vm::Dictionary dict{trans.r1.out_msgs, 15}; - for (int x = 0; x < trans.outmsg_cnt && x < 100; x++) { + for (int x = 0; x < trans.outmsg_cnt; x++) { TRY_RESULT(out_msg, to_raw_message(dict.lookup_ref(td::BitArray<15>{x}))); fees += out_msg->fwd_fee_; fees += out_msg->ihr_fee_; From 179415f815241be1965de0adea73f829f5b2eae8 Mon Sep 17 00:00:00 2001 From: Andrey Tvorozhkov Date: Sun, 17 Apr 2022 09:27:56 +0300 Subject: [PATCH 18/29] Fix tvm_stackEntrySlice in from_tonlib_api --- tonlib/tonlib/TonlibClient.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index c6a83ebb..0ea3743b 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -3437,7 +3437,8 @@ td::Result from_tonlib_api(tonlib_api::tvm_StackEntry& entry) { [&](tonlib_api::tvm_stackEntryUnsupported& cell) { return td::Status::Error("Unsuppored stack entry"); }, [&](tonlib_api::tvm_stackEntrySlice& cell) -> td::Result { TRY_RESULT(res, vm::std_boc_deserialize(cell.slice_->bytes_)); - return vm::StackEntry{std::move(res)}; + auto slice = vm::load_cell_slice_ref(std::move(res)); + return vm::StackEntry{std::move(slice)}; }, [&](tonlib_api::tvm_stackEntryCell& cell) -> td::Result { TRY_RESULT(res, vm::std_boc_deserialize(cell.cell_->bytes_)); From 8303f2c84f6e978cc97901584ed6dfddd40d899d Mon Sep 17 00:00:00 2001 From: ms Date: Tue, 19 Apr 2022 18:09:06 +0200 Subject: [PATCH 19/29] Fix BlockHeader is_key_block and add gen_utime, flags --- tl/generate/scheme/tonlib_api.tl | 2 +- tl/generate/scheme/tonlib_api.tlo | Bin 28216 -> 28288 bytes tonlib/tonlib/TonlibClient.cpp | 3 +++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tl/generate/scheme/tonlib_api.tl b/tl/generate/scheme/tonlib_api.tl index 4527cc09..d5f97cce 100644 --- a/tl/generate/scheme/tonlib_api.tl +++ b/tl/generate/scheme/tonlib_api.tl @@ -208,7 +208,7 @@ blocks.shards shards:vector = blocks.Shards; blocks.accountTransactionId account:bytes lt:int64 = blocks.AccountTransactionId; blocks.shortTxId mode:# account:mode.0?bytes lt:mode.1?int64 hash:mode.2?bytes = liteServer.TransactionId; blocks.transactions id:ton.blockIdExt req_count:int32 incomplete:Bool transactions:vector = blocks.Transactions; -blocks.header id:ton.blockIdExt global_id:int32 version:int32 after_merge:Bool after_split:Bool before_split:Bool want_merge:Bool want_split:Bool validator_list_hash_short:int32 catchain_seqno:int32 min_ref_mc_seqno:int32 is_key_block:Bool prev_key_block_seqno:int32 start_lt:int64 end_lt:int64 vert_seqno:# prev_blocks:vector = blocks.Header; +blocks.header id:ton.blockIdExt global_id:int32 version:int32 flags:# after_merge:Bool after_split:Bool before_split:Bool want_merge:Bool want_split:Bool validator_list_hash_short:int32 catchain_seqno:int32 min_ref_mc_seqno:int32 is_key_block:Bool prev_key_block_seqno:int32 start_lt:int64 end_lt:int64 gen_utime:int53 vert_seqno:# prev_blocks:vector = blocks.Header; //blocks.shortData header:blocks.Header transactions:blocks.Header = blocks.BlockData; configInfo config:tvm.cell = ConfigInfo; diff --git a/tl/generate/scheme/tonlib_api.tlo b/tl/generate/scheme/tonlib_api.tlo index d5a8344d6d832315af5525ff6b0288e42aa7e453..05c56ac08283bea94ca484f2c04acf3d2ac1d224 100644 GIT binary patch delta 141 zcmdmShq2)<B25imd7di)&z^ut@{Bb%7 delta 102 zcmZp;%ednX Date: Sun, 24 Apr 2022 16:57:38 +0300 Subject: [PATCH 20/29] Asm.fif typo in IFBITJMPs --- crypto/fift/lib/Asm.fif | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crypto/fift/lib/Asm.fif b/crypto/fift/lib/Asm.fif index edd0e24b..71c6a956 100644 --- a/crypto/fift/lib/Asm.fif +++ b/crypto/fift/lib/Asm.fif @@ -654,7 +654,7 @@ x{DB32} dup @Defop BRANCH @Defop RETBOOL x{DB34} @Defop CALLCC x{DB35} @Defop JMPXDATA { dup 1+ 0= { 16 + } if - Date: Mon, 2 May 2022 02:10:21 +0300 Subject: [PATCH 21/29] Add support for `#pragma version` and FunC versioning FunC pragma refined error messages and ^ partials --- crypto/func/func.cpp | 10 ++- crypto/func/func.h | 5 +- crypto/func/keywords.cpp | 2 + crypto/func/parse-func.cpp | 131 ++++++++++++++++++++++++++++++++++++- crypto/func/test/pv.fc | 53 +++++++++++++++ 5 files changed, 197 insertions(+), 4 deletions(-) create mode 100644 crypto/func/test/pv.fc diff --git a/crypto/func/func.cpp b/crypto/func/func.cpp index e8e09961..43bba4ea 100644 --- a/crypto/func/func.cpp +++ b/crypto/func/func.cpp @@ -173,6 +173,7 @@ void usage(const char* progname) { "-R\tInclude operation rewrite comments in the output code\n" "-W\tInclude Fift code to serialize and save generated code into specified BoC file. Enables " "-A and -P.\n" + "\t-s\tOutput semantic version of FunC and exit\n" "\t-V\tShow func build information\n"; std::exit(2); } @@ -182,7 +183,7 @@ std::string output_filename; int main(int argc, char* const argv[]) { int i; bool interactive = false; - while ((i = getopt(argc, argv, "Ahi:Io:O:PRSvW:V")) != -1) { + while ((i = getopt(argc, argv, "Ahi:Io:O:PRsSvW:V")) != -1) { switch (i) { case 'A': funC::asm_preamble = true; @@ -215,8 +216,13 @@ int main(int argc, char* const argv[]) { funC::boc_output_filename = optarg; funC::asm_preamble = funC::program_envelope = true; break; + case 's': + std::cout << funC::func_version << "\n"; + std::exit(0); + break; case 'V': - std::cout << "Func build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; + std::cout << "FunC semantic version: v" << funC::func_version << "\n"; + std::cout << "Build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; std::exit(0); break; case 'h': diff --git a/crypto/func/func.h b/crypto/func/func.h index 1506b974..22935dfe 100644 --- a/crypto/func/func.h +++ b/crypto/func/func.h @@ -38,6 +38,8 @@ extern bool op_rewrite_comments; constexpr int optimize_depth = 20; +const std::string func_version{"0.1.0"}; + enum Keyword { _Eof = -1, _Ident = 0, @@ -106,7 +108,8 @@ enum Keyword { _Operator, _Infix, _Infixl, - _Infixr + _Infixr, + _PragmaHashtag }; void define_keywords(); diff --git a/crypto/func/keywords.cpp b/crypto/func/keywords.cpp index 47975e7f..785c7765 100644 --- a/crypto/func/keywords.cpp +++ b/crypto/func/keywords.cpp @@ -126,6 +126,8 @@ void define_keywords() { .add_keyword("infix", Kw::_Infix) .add_keyword("infixl", Kw::_Infixl) .add_keyword("infixr", Kw::_Infixr); + + sym::symbols.add_keyword("#pragma", Kw::_PragmaHashtag); } } // namespace funC diff --git a/crypto/func/parse-func.cpp b/crypto/func/parse-func.cpp index 3aad90ef..d340a78c 100644 --- a/crypto/func/parse-func.cpp +++ b/crypto/func/parse-func.cpp @@ -1288,13 +1288,142 @@ void parse_func_def(Lexer& lex) { sym::close_scope(lex); } +std::string func_ver_test = func_version; + +void parse_pragma(Lexer& lex) { + auto pragma = lex.cur(); + lex.next(); + if (lex.tp() != _Ident) { + lex.expect(_Ident, "pragma name expected"); + } + auto pragma_name = lex.cur().str; + lex.next(); + if (!pragma_name.compare("version") || !pragma_name.compare("not-version")) { + bool negate = !pragma_name.compare("not-version"); + char op = '='; bool eq = false; + int sem_ver[3] = {0, 0, 0}; + char segs = 1; + if (lex.tp() == _Number) { + sem_ver[0] = std::stoi(lex.cur().str); + } else if (lex.tp() == _Ident) { + auto id1 = lex.cur().str; + char ch1 = id1[0]; + if ((ch1 == '>') || (ch1 == '<') || (ch1 == '=') || (ch1 == '^')) { + op = ch1; + } else { + lex.cur().error("unexpected comparator operation"); + } + if (id1.length() < 2) { + lex.cur().error("expected number after comparator"); + } + if (id1[1] == '=') { + eq = true; + if (id1.length() < 3) { + lex.cur().error("expected number after comparator"); + } + sem_ver[0] = std::stoi(id1.substr(2)); + } else { + sem_ver[0] = std::stoi(id1.substr(1)); + } + } else { + lex.cur().error("expected semver with optional comparator"); + } + lex.next(); + if (lex.tp() != ';') { + if (lex.tp() != _Ident || lex.cur().str[0] != '.') { + lex.cur().error("invalid semver format"); + } + sem_ver[1] = std::stoi(lex.cur().str.substr(1)); + segs = 2; + lex.next(); + } + if (lex.tp() != ';') { + if (lex.tp() != _Ident || lex.cur().str[0] != '.') { + lex.cur().error("invalid semver format"); + } + sem_ver[2] = std::stoi(lex.cur().str.substr(1)); + segs = 3; + lex.next(); + } + // End reading semver from source code + int func_ver[3] = {0, 0, 0}; + std::istringstream iss(func_ver_test); + std::string s; + for (int idx = 0; idx < 3; idx++) { + std::getline(iss, s, '.'); + func_ver[idx] = std::stoi(s); + } + // End parsing embedded semver + std::string semver_expr; + if (negate) { + semver_expr += '!'; + } + semver_expr += op; + if (eq) { + semver_expr += '='; + } + for (int idx = 0; idx < 3; idx++) { + semver_expr += std::to_string(sem_ver[idx]); + if (idx < 2) + semver_expr += '.'; + } + bool match = true; + switch (op) { + case '=': + if ((func_ver[0] != sem_ver[0]) || + (func_ver[1] != sem_ver[1]) || + (func_ver[2] != sem_ver[2])) { + match = false; + } + break; + case '>': + if ( ((func_ver[0] == sem_ver[0]) && (func_ver[1] == sem_ver[1]) && (func_ver[2] == sem_ver[2]) && !eq) || + ((func_ver[0] == sem_ver[0]) && (func_ver[1] == sem_ver[1]) && (func_ver[2] < sem_ver[2])) || + ((func_ver[0] == sem_ver[0]) && (func_ver[1] < sem_ver[1])) || + ((func_ver[0] < sem_ver[0])) ) { + match = false; + } + break; + case '<': + if ( ((func_ver[0] == sem_ver[0]) && (func_ver[1] == sem_ver[1]) && (func_ver[2] == sem_ver[2]) && !eq) || + ((func_ver[0] == sem_ver[0]) && (func_ver[1] == sem_ver[1]) && (func_ver[2] > sem_ver[2])) || + ((func_ver[0] == sem_ver[0]) && (func_ver[1] > sem_ver[1])) || + ((func_ver[0] > sem_ver[0])) ) { + match = false; + } + break; + case '^': + if ( ((segs == 3) && ((func_ver[0] != sem_ver[0]) || (func_ver[1] != sem_ver[1]) || (func_ver[2] < sem_ver[2]))) + || ((segs == 2) && ((func_ver[0] != sem_ver[0]) || (func_ver[1] < sem_ver[1]))) + || ((segs == 1) && ((func_ver[0] < sem_ver[0]))) ) { + match = false; + } + break; + } + if ((match && negate) || (!match && !negate)) { + pragma.error(std::string("FunC version ") + func_ver_test + " does not satisfy condition " + semver_expr); + } + } else if (!pragma_name.compare("test-version-set")) { + if (lex.tp() != _String) { + lex.cur().error("version string expected"); + } + func_ver_test = lex.cur().str; + lex.next(); + } else { + lex.cur().error(std::string{"unknown pragma `"} + pragma_name + "`"); + } + lex.expect(';'); +} + std::vector source_fdescr; bool parse_source(std::istream* is, src::FileDescr* fdescr) { src::SourceReader reader{is, fdescr}; Lexer lex{reader, true, ";,()[] ~."}; while (lex.tp() != _Eof) { - if (lex.tp() == _Global) { + if (lex.tp() == _PragmaHashtag) { + parse_pragma(lex); + } else if (lex.tp() == _Global) { parse_global_var_decls(lex); } else { parse_func_def(lex); diff --git a/crypto/func/test/pv.fc b/crypto/func/test/pv.fc new file mode 100644 index 00000000..d9bcd570 --- /dev/null +++ b/crypto/func/test/pv.fc @@ -0,0 +1,53 @@ +#pragma test-version-set "1.2.3"; + +;; Positive tests +#pragma version ^1.2.0; +#pragma version ^1.2.3; +#pragma version >1.2.0; +#pragma version >0.9.9; +#pragma version <1.3.0; +#pragma version <2.0.0; +#pragma version >=1.2.0; +#pragma version <=1.3.0; +#pragma version >=1.2.3; +#pragma version <=1.2.3; +#pragma version ^1.2.3; +#pragma version 1.2.3; +#pragma version =1.2.3; + +;; Negative tests +#pragma not-version ^1.1.0; +#pragma not-version ^1.0.0; +#pragma not-version ^0.2.3; +#pragma not-version ^2.2.3; +#pragma not-version ^1.3.3; +#pragma not-version >1.2.3; +#pragma not-version <1.2.3; +#pragma not-version ^1.2.4; +#pragma not-version >=1.2.4; +#pragma not-version <=1.2.2; +#pragma not-version 3.2.1; +#pragma not-version =3.2.1; + +;; Test incomplete (partial) version +#pragma version ^1.2; +#pragma version >1.2; +#pragma version <1.3; +#pragma version <2; +#pragma version >=1.2; +#pragma version <=1.3; + +;; Advanced ^ behaviour (partials) +#pragma version ^1.2; +#pragma version ^1.0; +#pragma version ^1; +#pragma version ^0; +#pragma not-version ^1.0.0; +#pragma not-version ^0.0.0; +#pragma not-version ^0.0; +#pragma not-version ^1.3; +#pragma not-version ^2; + +(int) main(int a) { + return a; +} From 0c772185ef86202891cb1ca8ecc8bef78639e980 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Fri, 6 May 2022 10:30:46 +0300 Subject: [PATCH 22/29] Add constants and string literals to funC * Add special string literals "..."? (s,a,u,h,) * Add string literal H (256-bit hash) * Add string literal c (crc32) * Use td::hex_encode instead of homebrew function and add test * Fix error codes and use more generic address * Add support for int and slice constants * Add support for strongly typed constants * Add support for precompiled constant expressions (hard!) Co-authored-by: starlightduck --- crypto/CMakeLists.txt | 12 +-- crypto/func/abscode.cpp | 10 ++ crypto/func/analyzer.cpp | 7 ++ crypto/func/asmops.cpp | 18 ++-- crypto/func/builtins.cpp | 2 +- crypto/func/codegen.cpp | 9 ++ crypto/func/func.h | 49 ++++++++- crypto/func/gen-abscode.cpp | 5 + crypto/func/keywords.cpp | 4 +- crypto/func/parse-func.cpp | 196 +++++++++++++++++++++++++++++++++++- crypto/func/test/co1.fc | 55 ++++++++++ crypto/func/test/s1.fc | 49 +++++++++ crypto/parser/lexer.cpp | 5 + crypto/parser/symtable.h | 2 +- 14 files changed, 397 insertions(+), 26 deletions(-) create mode 100644 crypto/func/test/co1.fc create mode 100644 crypto/func/test/s1.fc diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index 88a0272b..76c8ae9d 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -298,9 +298,14 @@ add_library(src_parser ${PARSER_SOURCE}) target_include_directories(src_parser PUBLIC $) target_link_libraries(src_parser PUBLIC ton_crypto) +add_library(ton_block ${BLOCK_SOURCE}) +target_include_directories(ton_block PUBLIC $ + $ $) +target_link_libraries(ton_block PUBLIC ton_crypto tdutils tdactor tl_api) + add_executable(func func/func.cpp ${FUNC_LIB_SOURCE}) target_include_directories(func PUBLIC $) -target_link_libraries(func PUBLIC ton_crypto src_parser git) +target_link_libraries(func PUBLIC ton_crypto src_parser git ton_block) if (WINGETOPT_FOUND) target_link_libraries_system(func wingetopt) endif() @@ -324,11 +329,6 @@ if (WINGETOPT_FOUND) target_link_libraries_system(pow-miner wingetopt) endif() -add_library(ton_block ${BLOCK_SOURCE}) -target_include_directories(ton_block PUBLIC $ - $ $) -target_link_libraries(ton_block PUBLIC ton_crypto tdutils tdactor tl_api) - set(TURN_OFF_LSAN cd .) if (TON_USE_ASAN AND NOT WIN32) set(TURN_OFF_LSAN export LSAN_OPTIONS=detect_leaks=0) diff --git a/crypto/func/abscode.cpp b/crypto/func/abscode.cpp index 8d73c805..4c893f35 100644 --- a/crypto/func/abscode.cpp +++ b/crypto/func/abscode.cpp @@ -168,6 +168,11 @@ void VarDescr::set_const(td::RefInt256 value) { } } +void VarDescr::set_const(std::string value) { + str_const = value; + val = _Const; +} + void VarDescr::set_const_nan() { set_const(td::make_refint()); } @@ -342,6 +347,11 @@ void Op::show(std::ostream& os, const std::vector& vars, std::string pfx show_var_list(os, left, vars); os << " := " << int_const << std::endl; break; + case _SliceConst: + os << pfx << dis << "SCONST "; + show_var_list(os, left, vars); + os << " := " << str_const << std::endl; + break; case _Import: os << pfx << dis << "IMPORT "; show_var_list(os, left, vars); diff --git a/crypto/func/analyzer.cpp b/crypto/func/analyzer.cpp index 5af435bc..26162521 100644 --- a/crypto/func/analyzer.cpp +++ b/crypto/func/analyzer.cpp @@ -353,6 +353,7 @@ bool Op::compute_used_vars(const CodeBlob& code, bool edit) { } switch (cl) { case _IntConst: + case _SliceConst: case _GlobVar: case _Call: case _CallInd: @@ -540,6 +541,7 @@ bool prune_unreachable(std::unique_ptr& ops) { bool reach; switch (op.cl) { case Op::_IntConst: + case Op::_SliceConst: case Op::_GlobVar: case Op::_SetGlob: case Op::_Call: @@ -707,6 +709,10 @@ VarDescrList Op::fwd_analyze(VarDescrList values) { values.add_newval(left[0]).set_const(int_const); break; } + case _SliceConst: { + values.add_newval(left[0]).set_const(str_const); + break; + } case _Call: { prepare_args(values); auto func = dynamic_cast(fun_ref->value); @@ -848,6 +854,7 @@ bool Op::mark_noreturn() { // fallthrough case _Import: case _IntConst: + case _SliceConst: case _Let: case _Tuple: case _UnTuple: diff --git a/crypto/func/asmops.cpp b/crypto/func/asmops.cpp index f0df9f3d..ccc409d3 100644 --- a/crypto/func/asmops.cpp +++ b/crypto/func/asmops.cpp @@ -55,10 +55,10 @@ std::ostream& operator<<(std::ostream& os, AsmOp::SReg stack_reg) { } } -AsmOp AsmOp::Const(int arg, std::string push_op) { +AsmOp AsmOp::Const(int arg, std::string push_op, td::RefInt256 origin) { std::ostringstream os; os << arg << ' ' << push_op; - return AsmOp::Const(os.str()); + return AsmOp::Const(os.str(), origin); } AsmOp AsmOp::make_stk2(int a, int b, const char* str, int delta) { @@ -166,27 +166,27 @@ AsmOp AsmOp::UnTuple(int a) { AsmOp AsmOp::IntConst(td::RefInt256 x) { if (x->signed_fits_bits(8)) { - return AsmOp::Const(dec_string(std::move(x)) + " PUSHINT"); + return AsmOp::Const(dec_string(std::move(x)) + " PUSHINT", x); } if (!x->is_valid()) { - return AsmOp::Const("PUSHNAN"); + return AsmOp::Const("PUSHNAN", x); } int k = is_pos_pow2(x); if (k >= 0) { - return AsmOp::Const(k, "PUSHPOW2"); + return AsmOp::Const(k, "PUSHPOW2", x); } k = is_pos_pow2(x + 1); if (k >= 0) { - return AsmOp::Const(k, "PUSHPOW2DEC"); + return AsmOp::Const(k, "PUSHPOW2DEC", x); } k = is_pos_pow2(-x); if (k >= 0) { - return AsmOp::Const(k, "PUSHNEGPOW2"); + return AsmOp::Const(k, "PUSHNEGPOW2", x); } if (!x->mod_pow2_short(23)) { - return AsmOp::Const(dec_string(std::move(x)) + " PUSHINTX"); + return AsmOp::Const(dec_string(std::move(x)) + " PUSHINTX", x); } - return AsmOp::Const(dec_string(std::move(x)) + " PUSHINT"); + return AsmOp::Const(dec_string(std::move(x)) + " PUSHINT", x); } AsmOp AsmOp::BoolConst(bool f) { diff --git a/crypto/func/builtins.cpp b/crypto/func/builtins.cpp index 91020be0..075d8f62 100644 --- a/crypto/func/builtins.cpp +++ b/crypto/func/builtins.cpp @@ -27,7 +27,7 @@ using namespace std::literals::string_literals; * */ -int glob_func_cnt, undef_func_cnt, glob_var_cnt; +int glob_func_cnt, undef_func_cnt, glob_var_cnt, const_cnt; std::vector glob_func, glob_vars; SymDef* predefine_builtin_func(std::string name, TypeExpr* func_type) { diff --git a/crypto/func/codegen.cpp b/crypto/func/codegen.cpp index 26d8215e..3e58b995 100644 --- a/crypto/func/codegen.cpp +++ b/crypto/func/codegen.cpp @@ -302,6 +302,15 @@ bool Op::generate_code_step(Stack& stack) { } return true; } + case _SliceConst: { + auto p = next_var_info[left[0]]; + if (!p || p->is_unused()) { + return true; + } + stack.o << AsmOp::Const("x{" + str_const + "} PUSHSLICE"); + stack.push_new_var(left[0]); + return true; + } case _GlobVar: if (dynamic_cast(fun_ref->value)) { bool used = false; diff --git a/crypto/func/func.h b/crypto/func/func.h index 22935dfe..751e50a9 100644 --- a/crypto/func/func.h +++ b/crypto/func/func.h @@ -109,6 +109,7 @@ enum Keyword { _Infix, _Infixl, _Infixr, + _Const, _PragmaHashtag }; @@ -336,6 +337,8 @@ struct VarDescr { static constexpr int FiniteUInt = FiniteInt | _Pos; int val; td::RefInt256 int_const; + std::string str_const; + VarDescr(var_idx_t _idx = -1, int _flags = 0, int _val = 0) : idx(_idx), flags(_flags), val(_val) { } bool operator<(var_idx_t other_idx) const { @@ -406,6 +409,7 @@ struct VarDescr { } void set_const(long long value); void set_const(td::RefInt256 value); + void set_const(std::string value); void set_const_nan(); void operator+=(const VarDescr& y) { flags &= y.flags; @@ -530,7 +534,8 @@ struct Op { _While, _Until, _Repeat, - _Again + _Again, + _SliceConst }; int cl; enum { _Disabled = 1, _Reachable = 2, _NoReturn = 4, _ImpureR = 8, _ImpureW = 16, _Impure = 24 }; @@ -543,6 +548,7 @@ struct Op { std::vector left, right; std::unique_ptr block0, block1; td::RefInt256 int_const; + std::string str_const; Op(const SrcLocation& _where = {}, int _cl = _Undef) : cl(_cl), flags(0), fun_ref(nullptr), where(_where) { } Op(const SrcLocation& _where, int _cl, const std::vector& _left) @@ -554,6 +560,9 @@ struct Op { Op(const SrcLocation& _where, int _cl, const std::vector& _left, td::RefInt256 _const) : cl(_cl), flags(0), fun_ref(nullptr), where(_where), left(_left), int_const(_const) { } + Op(const SrcLocation& _where, int _cl, const std::vector& _left, std::string _const) + : cl(_cl), flags(0), fun_ref(nullptr), where(_where), left(_left), str_const(_const) { + } Op(const SrcLocation& _where, int _cl, const std::vector& _left, const std::vector& _right, SymDef* _fun = nullptr) : cl(_cl), flags(0), fun_ref(_fun), where(_where), left(_left), right(_right) { @@ -784,6 +793,30 @@ struct SymValGlobVar : sym::SymValBase { } }; +struct SymValConst : sym::SymValBase { + td::RefInt256 intval; + std::string strval; + Keyword type; + SymValConst(int idx, td::RefInt256 value) + : sym::SymValBase(_Const, idx), intval(value) { + type = _Int; + } + SymValConst(int idx, std::string value) + : sym::SymValBase(_Const, idx), strval(value) { + type = _Slice; + } + ~SymValConst() override = default; + td::RefInt256 get_int_value() const { + return intval; + } + std::string get_str_value() const { + return strval; + } + Keyword get_type() const { + return type; + } +}; + extern int glob_func_cnt, undef_func_cnt, glob_var_cnt; extern std::vector glob_func, glob_vars; @@ -820,7 +853,8 @@ struct Expr { _LetFirst, _Hole, _Type, - _CondExpr + _CondExpr, + _SliceConst }; int cls; int val{0}; @@ -828,6 +862,7 @@ struct Expr { int flags{0}; SrcLocation here; td::RefInt256 intval; + std::string strval; SymDef* sym{nullptr}; TypeExpr* e_type{nullptr}; std::vector args; @@ -910,6 +945,7 @@ struct AsmOp { int a, b, c; bool gconst{false}; std::string op; + td::RefInt256 origin; struct SReg { int idx; SReg(int _idx) : idx(_idx) { @@ -929,6 +965,9 @@ struct AsmOp { AsmOp(int _t, int _a, int _b, std::string _op) : t(_t), a(_a), b(_b), op(std::move(_op)) { compute_gconst(); } + AsmOp(int _t, int _a, int _b, std::string _op, td::RefInt256 x) : t(_t), a(_a), b(_b), op(std::move(_op)), origin(x) { + compute_gconst(); + } AsmOp(int _t, int _a, int _b, int _c) : t(_t), a(_a), b(_b), c(_c) { } AsmOp(int _t, int _a, int _b, int _c, std::string _op) : t(_t), a(_a), b(_b), c(_c), op(std::move(_op)) { @@ -1047,10 +1086,10 @@ struct AsmOp { static AsmOp make_stk3(int a, int b, int c, const char* str, int delta); static AsmOp IntConst(td::RefInt256 value); static AsmOp BoolConst(bool f); - static AsmOp Const(std::string push_op) { - return AsmOp(a_const, 0, 1, std::move(push_op)); + static AsmOp Const(std::string push_op, td::RefInt256 origin = {}) { + return AsmOp(a_const, 0, 1, std::move(push_op), origin); } - static AsmOp Const(int arg, std::string push_op); + static AsmOp Const(int arg, std::string push_op, td::RefInt256 origin = {}); static AsmOp Comment(std::string comment) { return AsmOp(a_none, std::string{"// "} + comment); } diff --git a/crypto/func/gen-abscode.cpp b/crypto/func/gen-abscode.cpp index c0a8985a..21dce714 100644 --- a/crypto/func/gen-abscode.cpp +++ b/crypto/func/gen-abscode.cpp @@ -362,6 +362,11 @@ std::vector Expr::pre_compile(CodeBlob& code, bool lval) const { code.close_pop_cur(args[2]->here); return rvect; } + case _SliceConst: { + auto rvect = new_tmp_vect(code); + code.emplace_back(here, Op::_SliceConst, rvect, strval); + return rvect; + } default: std::cerr << "expression constructor is " << cls << std::endl; throw src::Fatal{"cannot compile expression with unknown constructor"}; diff --git a/crypto/func/keywords.cpp b/crypto/func/keywords.cpp index 785c7765..b7bba0c2 100644 --- a/crypto/func/keywords.cpp +++ b/crypto/func/keywords.cpp @@ -125,8 +125,8 @@ void define_keywords() { .add_keyword("operator", Kw::_Operator) .add_keyword("infix", Kw::_Infix) .add_keyword("infixl", Kw::_Infixl) - .add_keyword("infixr", Kw::_Infixr); - + .add_keyword("infixr", Kw::_Infixr) + .add_keyword("const", Kw::_Const); sym::symbols.add_keyword("#pragma", Kw::_PragmaHashtag); } diff --git a/crypto/func/parse-func.cpp b/crypto/func/parse-func.cpp index d340a78c..929f5b3c 100644 --- a/crypto/func/parse-func.cpp +++ b/crypto/func/parse-func.cpp @@ -19,6 +19,9 @@ #include "func.h" #include "td/utils/crypto.h" #include "common/refint.h" +#include "openssl/digest.hpp" +#include "block/block.h" +#include "block-parse.h" #include namespace sym { @@ -229,6 +232,83 @@ void parse_global_var_decl(Lexer& lex) { lex.next(); } +extern int const_cnt; +Expr* parse_expr(Lexer& lex, CodeBlob& code, bool nv = false); + +void parse_const_decl(Lexer& lex) { + SrcLocation loc = lex.cur().loc; + int wanted_type = Expr::_None; + if (lex.tp() == _Int) { + wanted_type = Expr::_Const; + lex.next(); + } else if (lex.tp() == _Slice) { + wanted_type = Expr::_SliceConst; + lex.next(); + } + if (lex.tp() != _Ident) { + lex.expect(_Ident, "constant name"); + } + loc = lex.cur().loc; + SymDef* sym_def = sym::define_global_symbol(lex.cur().val, false, loc); + if (!sym_def) { + lex.cur().error_at("cannot define global symbol `", "`"); + } + if (sym_def->value) { + lex.cur().error_at("global symbol `", "` already exists"); + } + lex.next(); + if (lex.tp() != '=') { + lex.cur().error_at("expected = instead of ", ""); + } + lex.next(); + CodeBlob code; + // Handles processing and resolution of literals and consts + auto x = parse_expr(lex, code, false); // also does lex.next() ! + if (x->flags != Expr::_IsRvalue) { + lex.cur().error("expression is not strictly Rvalue"); + } + if ((wanted_type == Expr::_Const) && (x->cls == Expr::_Apply)) + wanted_type = Expr::_None; // Apply is additionally checked to result in an integer + if ((wanted_type != Expr::_None) && (x->cls != wanted_type)) { + lex.cur().error("expression type does not match wanted type"); + } + if (x->cls == Expr::_Const) { // Integer constant + sym_def->value = new SymValConst{const_cnt++, x->intval}; + } else if (x->cls == Expr::_SliceConst) { // Slice constant (string) + sym_def->value = new SymValConst{const_cnt++, x->strval}; + } else if (x->cls == Expr::_Apply) { + code.emplace_back(loc, Op::_Import, std::vector()); + auto tmp_vars = x->pre_compile(code); + code.emplace_back(loc, Op::_Return, std::move(tmp_vars)); + code.emplace_back(loc, Op::_Nop); // This is neccessary to prevent SIGSEGV! + // It is REQUIRED to execute "optimizations" as in func.cpp + code.simplify_var_types(); + code.prune_unreachable_code(); + code.split_vars(true); + for (int i = 0; i < 16; i++) { + code.compute_used_code_vars(); + code.fwd_analyze(); + code.prune_unreachable_code(); + } + code.mark_noreturn(); + AsmOpList out_list(0, &code.vars); + code.generate_code(out_list); + if (out_list.list_.size() != 1) { + lex.cur().error("precompiled expression must result in single operation"); + } + auto op = out_list.list_[0]; + if (!op.is_const()) { + lex.cur().error("precompiled expression must result in compilation time constant"); + } + if (op.origin.is_null() || !op.origin->is_valid()) { + lex.cur().error("precompiled expression did not result in a valid integer constant"); + } + sym_def->value = new SymValConst{const_cnt++, op.origin}; + } else { + lex.cur().error("integer or slice literal or constant expected"); + } +} + FormalArgList parse_formal_args(Lexer& lex) { FormalArgList args; lex.expect('(', "formal argument list"); @@ -246,6 +326,18 @@ FormalArgList parse_formal_args(Lexer& lex) { return args; } +void parse_const_decls(Lexer& lex) { + lex.expect(_Const); + while (true) { + parse_const_decl(lex); + if (lex.tp() != ',') { + break; + } + lex.expect(','); + } + lex.expect(';'); +} + TypeExpr* extract_total_arg_type(const FormalArgList& arg_list) { if (arg_list.empty()) { return TypeExpr::new_unit(); @@ -322,8 +414,6 @@ Expr* make_func_apply(Expr* fun, Expr* x) { return res; } -Expr* parse_expr(Lexer& lex, CodeBlob& code, bool nv = false); - // parse ( E { , E } ) | () | [ E { , E } ] | [] | id | num | _ Expr* parse_expr100(Lexer& lex, CodeBlob& code, bool nv) { if (lex.tp() == '(' || lex.tp() == '[') { @@ -388,6 +478,87 @@ Expr* parse_expr100(Lexer& lex, CodeBlob& code, bool nv) { lex.next(); return res; } + if (t == Lexem::String) { + std::string str = lex.cur().str; + int str_type = lex.cur().val; + Expr* res; + switch (str_type) { + case 0: + case 's': + case 'a': + { + res = new Expr{Expr::_SliceConst, lex.cur().loc}; + res->e_type = TypeExpr::new_atomic(_Slice); + break; + } + case 'u': + case 'h': + case 'H': + case 'c': + { + res = new Expr{Expr::_Const, lex.cur().loc}; + res->e_type = TypeExpr::new_atomic(_Int); + break; + } + default: + { + res = new Expr{Expr::_Const, lex.cur().loc}; + res->e_type = TypeExpr::new_atomic(_Int); + lex.cur().error("invalid string type `" + std::string(1, static_cast(str_type)) + "`"); + return res; + } + } + res->flags = Expr::_IsRvalue; + switch (str_type) { + case 0: { + res->strval = td::hex_encode(str); + break; + } + case 's': { + res->strval = str; + unsigned char buff[128]; + int bits = (int)td::bitstring::parse_bitstring_hex_literal(buff, sizeof(buff), str.data(), str.data() + str.size()); + if (bits < 0) { + lex.cur().error_at("Invalid hex bitstring constant `", "`"); + } + break; + } + case 'a': { // MsgAddressInt + block::StdAddress a; + if (a.parse_addr(str)) { + res->strval = block::tlb::MsgAddressInt().pack_std_address(a)->as_bitslice().to_hex(); + } else { + lex.cur().error_at("invalid standard address `", "`"); + } + break; + } + case 'u': { + res->intval = td::hex_string_to_int256(td::hex_encode(str)); + if (!str.size()) { + lex.cur().error("empty integer ascii-constant"); + } + if (res->intval.is_null()) { + lex.cur().error_at("too long integer ascii-constant `", "`"); + } + break; + } + case 'h': + case 'H': + { + unsigned char hash[32]; + digest::hash_str(hash, str.data(), str.size()); + res->intval = td::bits_to_refint(hash, (str_type == 'h') ? 32 : 256, false); + break; + } + case 'c': + { + res->intval = td::make_refint(td::crc32(td::Slice{str})); + break; + } + } + lex.next(); + return res; + } if (t == '_') { Expr* res = new Expr{Expr::_Hole, lex.cur().loc}; res->val = -1; @@ -429,6 +600,25 @@ Expr* parse_expr100(Lexer& lex, CodeBlob& code, bool nv) { lex.next(); return res; } + if (sym && dynamic_cast(sym->value)) { + auto val = dynamic_cast(sym->value); + Expr* res = new Expr{Expr::_None, lex.cur().loc}; + res->flags = Expr::_IsRvalue; + if (val->type == _Int) { + res->cls = Expr::_Const; + res->intval = val->get_int_value(); + } + else if (val->type == _Slice) { + res->cls = Expr::_SliceConst; + res->strval = val->get_str_value(); + } + else { + lex.cur().error("Invalid symbolic constant type"); + } + res->e_type = TypeExpr::new_atomic(val->type); + lex.next(); + return res; + } bool auto_apply = false; Expr* res = new Expr{Expr::_Var, lex.cur().loc}; if (nv) { @@ -1425,6 +1615,8 @@ bool parse_source(std::istream* is, src::FileDescr* fdescr) { parse_pragma(lex); } else if (lex.tp() == _Global) { parse_global_var_decls(lex); + } else if (lex.tp() == _Const) { + parse_const_decls(lex); } else { parse_func_def(lex); } diff --git a/crypto/func/test/co1.fc b/crypto/func/test/co1.fc new file mode 100644 index 00000000..82071e45 --- /dev/null +++ b/crypto/func/test/co1.fc @@ -0,0 +1,55 @@ +const int1 = 1, int2 = 2; + +const int int101 = 101; +const int int111 = 111; + +const int1r = int1; + +const str1 = "const1", str2 = "aabbcc"s; + +const slice str2r = str2; + +const str1int = 0x636f6e737431; +const str2int = 0xAABBCC; + +const int nibbles = 4; + +int iget1() { return int1; } +int iget2() { return int2; } +int iget3() { return int1 + int2; } + +int iget1r() { return int1r; } + +slice sget1() { return str1; } +slice sget2() { return str2; } +slice sget2r() { return str2r; } + +const int int240 = ((int1 + int2) * 10) << 3; + +int iget240() { return int240; } + +builder newc() asm "NEWC"; +slice endcs(builder b) asm "ENDC" "CTOS"; +int sdeq (slice s1, slice s2) asm "SDEQ"; +builder stslicer(builder b, slice s) asm "STSLICER"; + +_ main() { + int i1 = iget1(); + int i2 = iget2(); + int i3 = iget3(); + + throw_unless(int101, i1 == 1); + throw_unless(102, i2 == 2); + throw_unless(103, i3 == 3); + + slice s1 = sget1(); + slice s2 = sget2(); + slice s3 = newc().stslicer(str1).stslicer(str2r).endcs(); + + throw_unless(int111, sdeq(s1, newc().store_uint(str1int, 12 * nibbles).endcs())); + throw_unless(112, sdeq(s2, newc().store_uint(str2int, 6 * nibbles).endcs())); + throw_unless(113, sdeq(s3, newc().store_uint(0x636f6e737431ABCDEF, 18 * nibbles).endcs())); + + int i4 = iget240(); + throw_unless(104, i4 == 240); +} diff --git a/crypto/func/test/s1.fc b/crypto/func/test/s1.fc new file mode 100644 index 00000000..630d0180 --- /dev/null +++ b/crypto/func/test/s1.fc @@ -0,0 +1,49 @@ +slice ascii_slice() method_id { + return "string"; +} + +slice raw_slice() method_id { + return "abcdef"s; +} + +slice addr_slice() method_id { + return "Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF"a; +} + +int string_hex() method_id { + return "ABCDEFGHIJKLMNOPQRSTUVWXYZ012345"u; +} + +int string_minihash() method_id { + return "transfer(slice, int)"h; +} + +int string_maxihash() method_id { + return "transfer(slice, int)"H; +} + +int string_crc32() method_id { + return "transfer(slice, int)"c; +} + +builder newc() asm "NEWC"; +slice endcs(builder b) asm "ENDC" "CTOS"; +int sdeq (slice s1, slice s2) asm "SDEQ"; + +_ main() { + slice s_ascii = ascii_slice(); + slice s_raw = raw_slice(); + slice s_addr = addr_slice(); + int i_hex = string_hex(); + int i_mini = string_minihash(); + int i_maxi = string_maxihash(); + int i_crc = string_crc32(); + throw_unless(101, sdeq(s_ascii, newc().store_uint(0x737472696E67, 12 * 4).endcs())); + throw_unless(102, sdeq(s_raw, newc().store_uint(0xABCDEF, 6 * 4).endcs())); + throw_unless(103, sdeq(s_addr, newc().store_uint(4, 3).store_int(-1, 8) + .store_uint(0x3333333333333333333333333333333333333333333333333333333333333333, 256).endcs())); + throw_unless(104, i_hex == 0x4142434445464748494A4B4C4D4E4F505152535455565758595A303132333435); + throw_unless(105, i_mini == 0x7a62e8a8); + throw_unless(106, i_maxi == 0x7a62e8a8ebac41bd6de16c65e7be363bc2d2cbc6a0873778dead4795c13db979); + throw_unless(107, i_crc == 2235694568); +} diff --git a/crypto/parser/lexer.cpp b/crypto/parser/lexer.cpp index 87c63c3f..5c5b77d8 100644 --- a/crypto/parser/lexer.cpp +++ b/crypto/parser/lexer.cpp @@ -247,6 +247,11 @@ const Lexem& Lexer::next() { } lexem.set(std::string{src.get_ptr() + 1, end}, src.here(), qc == '`' ? Lexem::Unknown : Lexem::String); src.set_ptr(end + 1); + c = src.cur_char(); + if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) { + lexem.val = c; + src.set_ptr(end + 2); + } // std::cerr << lexem.name_str() << ' ' << lexem.str << std::endl; return lexem; } diff --git a/crypto/parser/symtable.h b/crypto/parser/symtable.h index 81a828a8..9489b2bc 100644 --- a/crypto/parser/symtable.h +++ b/crypto/parser/symtable.h @@ -32,7 +32,7 @@ namespace sym { typedef int var_idx_t; struct SymValBase { - enum { _Param, _Var, _Func, _Typename, _GlobVar }; + enum { _Param, _Var, _Func, _Typename, _GlobVar, _Const }; int type; int idx; SymValBase(int _type, int _idx) : type(_type), idx(_idx) { From 1e0b587023a4177feaaf5fcd834661c489ad9ed8 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Wed, 11 May 2022 21:12:04 +0300 Subject: [PATCH 23/29] Allow untrusted fecs after check (#356) * Allow untrusted fecs after check * FEC: save and resend incoming parts Co-authored-by: OmicronTau <87443739+OmicronTau@users.noreply.github.com> Co-authored-by: OmicronTau --- overlay/overlay-fec-broadcast.cpp | 97 +++++++++++++++++++------------ overlay/overlay-fec-broadcast.hpp | 24 ++++++-- overlay/overlays.h | 1 + validator/full-node-shard.cpp | 2 +- 4 files changed, 83 insertions(+), 41 deletions(-) diff --git a/overlay/overlay-fec-broadcast.cpp b/overlay/overlay-fec-broadcast.cpp index 6ab03800..0ff53346 100644 --- a/overlay/overlay-fec-broadcast.cpp +++ b/overlay/overlay-fec-broadcast.cpp @@ -59,14 +59,15 @@ td::Status OverlayFecBroadcastPart::check_source() { return td::Status::Error(ErrorCode::error, "broadcast is forbidden"); } - // FIXME if (r == BroadcastCheckResult::NeedCheck) { - return td::Status::Error(ErrorCode::error, "broadcast is forbidden"); + untrusted_ = true; + return td::Status::OK(); } if (bcast_) { TRY_STATUS(bcast_->is_eligible_sender(source_)); } + return td::Status::OK(); } @@ -77,6 +78,7 @@ td::Status OverlayFecBroadcastPart::check_signature() { } td::Status OverlayFecBroadcastPart::run_checks() { + TRY_STATUS(check_time()); TRY_STATUS(check_duplicate()); TRY_STATUS(check_source()); @@ -84,7 +86,52 @@ td::Status OverlayFecBroadcastPart::run_checks() { return td::Status::OK(); } +void BroadcastFec::broadcast_checked(td::Result R) { + if (R.is_error()) { + return; + } + overlay_->deliver_broadcast(get_source().compute_short_id(), data_.clone()); + auto manager = overlay_->overlay_manager(); + while (!parts_.empty()) { + distribute_part(parts_.begin()->first); + } +} + +// Do we need status here?? +td::Status BroadcastFec::distribute_part(td::uint32 seqno) { + auto i = parts_.find(seqno); + if (i == parts_.end()) { + // should not get here + return td::Status::OK(); + } + auto tls = std::move(i->second); + parts_.erase(i); + td::BufferSlice data_short = std::move(tls.first); + td::BufferSlice data = std::move(tls.second); + + auto nodes = overlay_->get_neighbours(5); + auto manager = overlay_->overlay_manager(); + + for (auto &n : nodes) { + if (neighbour_completed(n)) { + continue; + } + if (neighbour_received(n)) { + td::actor::send_closure(manager, &OverlayManager::send_message, n, overlay_->local_id(), overlay_->overlay_id(), + data_short.clone()); + } else { + if (hash_.count_leading_zeroes() >= 12) { + VLOG(OVERLAY_INFO) << "broadcast " << hash_ << ": sending part " << seqno << " to " << n; + } + td::actor::send_closure(manager, &OverlayManager::send_message, n, overlay_->local_id(), overlay_->overlay_id(), + data.clone()); + } + } + return td::Status::OK(); +} + td::Status OverlayFecBroadcastPart::apply() { + if (!bcast_) { bcast_ = overlay_->get_fec_broadcast(broadcast_hash_); } @@ -107,7 +154,8 @@ td::Status OverlayFecBroadcastPart::apply() { } if (!bcast_->finalized()) { - TRY_STATUS(bcast_->add_part(seqno_, data_.clone())); + bcast_->set_overlay(overlay_); + TRY_STATUS(bcast_->add_part(seqno_, data_.clone(), export_serialized_short(), export_serialized())); auto R = bcast_->finish(); if (R.is_error()) { auto S = R.move_as_error(); @@ -115,44 +163,22 @@ td::Status OverlayFecBroadcastPart::apply() { return S; } } else { - overlay_->deliver_broadcast(bcast_->get_source().compute_short_id(), R.move_as_ok()); + if(untrusted_) { + auto P = td::PromiseCreator::lambda( + [id = broadcast_hash_, overlay_id = actor_id(overlay_)](td::Result RR) mutable { + td::actor::send_closure(std::move(overlay_id), &OverlayImpl::broadcast_checked, id, std::move(RR)); + }); + overlay_->check_broadcast(bcast_->get_source().compute_short_id(), R.move_as_ok(), std::move(P)); + } else { + overlay_->deliver_broadcast(bcast_->get_source().compute_short_id(), R.move_as_ok()); + } } } - return td::Status::OK(); } td::Status OverlayFecBroadcastPart::distribute() { - auto B = export_serialized(); - auto nodes = overlay_->get_neighbours(5); - - auto manager = overlay_->overlay_manager(); - - td::BufferSlice data; - td::BufferSlice data_short; - - for (auto &n : nodes) { - if (bcast_->neighbour_completed(n)) { - continue; - } - if (bcast_->neighbour_received(n)) { - if (data_short.size() == 0) { - data_short = export_serialized_short(); - } - td::actor::send_closure(manager, &OverlayManager::send_message, n, overlay_->local_id(), overlay_->overlay_id(), - data_short.clone()); - } else { - if (data.size() == 0) { - data = export_serialized(); - } - - if (broadcast_hash_.count_leading_zeroes() >= 12) { - VLOG(OVERLAY_INFO) << "broadcast " << broadcast_hash_ << ": sending part " << part_hash_ << " to " << n; - } - td::actor::send_closure(manager, &OverlayManager::send_message, n, overlay_->local_id(), overlay_->overlay_id(), - data.clone()); - } - } + TRY_STATUS(bcast_->distribute_part(seqno_)); return td::Status::OK(); } @@ -188,7 +214,6 @@ td::BufferSlice OverlayFecBroadcastPart::to_sign() { td::Status OverlayFecBroadcastPart::create(OverlayImpl *overlay, tl_object_ptr broadcast) { TRY_STATUS(overlay->check_date(broadcast->date_)); - auto source = PublicKey{broadcast->src_}; auto part_data_hash = sha256_bits256(broadcast->data_.as_slice()); diff --git a/overlay/overlay-fec-broadcast.hpp b/overlay/overlay-fec-broadcast.hpp index 0ce9c2ff..f27e69fb 100644 --- a/overlay/overlay-fec-broadcast.hpp +++ b/overlay/overlay-fec-broadcast.hpp @@ -82,13 +82,17 @@ class BroadcastFec : public td::ListNode { } } - td::Status add_part(td::uint32 seqno, td::BufferSlice data) { + td::Status add_part(td::uint32 seqno, td::BufferSlice data, + td::BufferSlice serialized_fec_part_short, + td::BufferSlice serialized_fec_part) { CHECK(decoder_); td::fec::Symbol s; s.id = seqno; s.data = std::move(data); decoder_->add_symbol(std::move(s)); + parts_[seqno] = std::pair(std::move(serialized_fec_part_short), + std::move(serialized_fec_part)); return td::Status::OK(); } @@ -106,6 +110,7 @@ class BroadcastFec : public td::ListNode { CHECK(encoder_ != nullptr); ready_ = true; decoder_ = nullptr; + data_ = D.data.clone(); return std::move(D.data); } @@ -185,9 +190,13 @@ class BroadcastFec : public td::ListNode { } } - void broadcast_checked(td::Result R) { + void broadcast_checked(td::Result R); + void set_overlay(OverlayImpl *overlay) { + overlay_ = overlay; } + td::Status distribute_part(td::uint32 seqno); + private: bool ready_ = false; @@ -208,6 +217,10 @@ class BroadcastFec : public td::ListNode { td::uint32 next_seqno_ = 0; td::uint64 received_parts_ = 0; + + std::map> parts_; + OverlayImpl *overlay_; + td::BufferSlice data_; }; class OverlayFecBroadcastPart : public td::ListNode { @@ -228,6 +241,7 @@ class OverlayFecBroadcastPart : public td::ListNode { td::BufferSlice signature_; bool is_short_; + bool untrusted_{false}; BroadcastFec *bcast_; OverlayImpl *overlay_; @@ -280,7 +294,7 @@ class OverlayFecBroadcastPart : public td::ListNode { signature_ = std::move(signature); } void update_overlay(OverlayImpl *overlay); - + tl_object_ptr export_tl(); tl_object_ptr export_tl_short(); td::BufferSlice export_serialized(); @@ -290,7 +304,9 @@ class OverlayFecBroadcastPart : public td::ListNode { td::Status run() { TRY_STATUS(run_checks()); TRY_STATUS(apply()); - TRY_STATUS(distribute()); + if(!untrusted_) { + TRY_STATUS(distribute()); + } return td::Status::OK(); } diff --git a/overlay/overlays.h b/overlay/overlays.h index aba83ab1..799faf5a 100644 --- a/overlay/overlays.h +++ b/overlay/overlays.h @@ -108,6 +108,7 @@ class OverlayPrivacyRules { } BroadcastCheckResult check_rules(PublicKeyHash hash, td::uint32 size, bool is_fec) { + auto it = authorized_keys_.find(hash); if (it == authorized_keys_.end()) { if (size > max_unath_size_) { diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 72684151..279fb6fb 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -896,7 +896,7 @@ void FullNodeShardImpl::update_validators(std::vector public_key_ authorized_keys.emplace(key, overlay::Overlays::max_fec_broadcast_size()); } - rules_ = overlay::OverlayPrivacyRules{1 << 14, 0, std::move(authorized_keys)}; + rules_ = overlay::OverlayPrivacyRules{overlay::Overlays::max_fec_broadcast_size(), overlay::CertificateFlags::AllowFec, std::move(authorized_keys)}; td::actor::send_closure(overlays_, &overlay::Overlays::set_privacy_rules, adnl_id_, overlay_id_, rules_); if (update_cert) { From 9356a16b847a2f2ef7953843f82884cf16df0e26 Mon Sep 17 00:00:00 2001 From: starlightduck Date: Thu, 12 May 2022 12:54:34 +0300 Subject: [PATCH 24/29] Implement `#include` keyword with advanced checks and backtrace --- crypto/func/func.cpp | 17 +++++++++- crypto/func/func.h | 10 ++++-- crypto/func/keywords.cpp | 4 ++- crypto/func/parse-func.cpp | 63 +++++++++++++++++++++++++++++++++++--- crypto/func/test/i1.fc | 14 +++++++++ crypto/func/test/i1sub1.fc | 6 ++++ crypto/func/test/i1sub2.fc | 8 +++++ 7 files changed, 113 insertions(+), 9 deletions(-) create mode 100644 crypto/func/test/i1.fc create mode 100644 crypto/func/test/i1sub1.fc create mode 100644 crypto/func/test/i1sub2.fc diff --git a/crypto/func/func.cpp b/crypto/func/func.cpp index 43bba4ea..b260c885 100644 --- a/crypto/func/func.cpp +++ b/crypto/func/func.cpp @@ -178,6 +178,18 @@ void usage(const char* progname) { std::exit(2); } +void output_inclusion_stack() { + while (!funC::inclusion_locations.empty()) { + src::SrcLocation loc = funC::inclusion_locations.top(); + funC::inclusion_locations.pop(); + if (loc.fdescr) { + std::cerr << "note: included from "; + loc.show(std::cerr); + std::cerr << std::endl; + } + } +} + std::string output_filename; int main(int argc, char* const argv[]) { @@ -241,7 +253,7 @@ int main(int argc, char* const argv[]) { int ok = 0, proc = 0; try { while (optind < argc) { - funC::generated_from += std::string{"`"} + argv[optind] + "` "; + // funC::generated_from += std::string{"`"} + argv[optind] + "` "; ok += funC::parse_source_file(argv[optind++]); proc++; } @@ -268,14 +280,17 @@ int main(int argc, char* const argv[]) { funC::generate_output(); } catch (src::Fatal& fatal) { std::cerr << "fatal: " << fatal << std::endl; + output_inclusion_stack(); std::exit(1); } catch (src::Error& error) { std::cerr << error << std::endl; + output_inclusion_stack(); std::exit(1); } catch (funC::UnifyError& unif_err) { std::cerr << "fatal: "; unif_err.print_message(std::cerr); std::cerr << std::endl; + output_inclusion_stack(); std::exit(1); } } diff --git a/crypto/func/func.h b/crypto/func/func.h index 751e50a9..2f9877a0 100644 --- a/crypto/func/func.h +++ b/crypto/func/func.h @@ -35,10 +35,11 @@ namespace funC { extern int verbosity; extern bool op_rewrite_comments; +extern std::string generated_from; constexpr int optimize_depth = 20; -const std::string func_version{"0.1.0"}; +const std::string func_version{"0.2.0"}; enum Keyword { _Eof = -1, @@ -110,7 +111,8 @@ enum Keyword { _Infixl, _Infixr, _Const, - _PragmaHashtag + _PragmaHashtag, + _IncludeHashtag }; void define_keywords(); @@ -828,9 +830,11 @@ extern std::vector glob_func, glob_vars; // defined in parse-func.cpp bool parse_source(std::istream* is, const src::FileDescr* fdescr); -bool parse_source_file(const char* filename); +bool parse_source_file(const char* filename, src::Lexem lex = {}); bool parse_source_stdin(); +extern std::stack inclusion_locations; + /* * * EXPRESSIONS diff --git a/crypto/func/keywords.cpp b/crypto/func/keywords.cpp index b7bba0c2..3f0fa230 100644 --- a/crypto/func/keywords.cpp +++ b/crypto/func/keywords.cpp @@ -127,7 +127,9 @@ void define_keywords() { .add_keyword("infixl", Kw::_Infixl) .add_keyword("infixr", Kw::_Infixr) .add_keyword("const", Kw::_Const); - sym::symbols.add_keyword("#pragma", Kw::_PragmaHashtag); + + sym::symbols.add_keyword("#pragma", Kw::_PragmaHashtag) + .add_keyword("#include", Kw::_IncludeHashtag); } } // namespace funC diff --git a/crypto/func/parse-func.cpp b/crypto/func/parse-func.cpp index 929f5b3c..c33a3cda 100644 --- a/crypto/func/parse-func.cpp +++ b/crypto/func/parse-func.cpp @@ -23,6 +23,7 @@ #include "block/block.h" #include "block-parse.h" #include +#include namespace sym { @@ -1607,12 +1608,35 @@ void parse_pragma(Lexer& lex) { std::vector source_fdescr; +std::vector source_files; +std::stack inclusion_locations; + +void parse_include(Lexer& lex, const src::FileDescr* fdescr) { + auto include = lex.cur(); + lex.expect(_IncludeHashtag); + if (lex.tp() != _String) { + lex.expect(_String, "source file name"); + } + std::string val = lex.cur().str; + std::string parent_dir = fdescr->filename; + if (parent_dir.rfind('/') != std::string::npos) { + val = parent_dir.substr(0, parent_dir.rfind('/') + 1) + val; + } + lex.next(); + lex.expect(';'); + if (!parse_source_file(val.c_str(), include)) { + include.error(std::string{"failed parsing included file `"} + val + "`"); + } +} + bool parse_source(std::istream* is, src::FileDescr* fdescr) { src::SourceReader reader{is, fdescr}; Lexer lex{reader, true, ";,()[] ~."}; while (lex.tp() != _Eof) { if (lex.tp() == _PragmaHashtag) { parse_pragma(lex); + } else if (lex.tp() == _IncludeHashtag) { + parse_include(lex, fdescr); } else if (lex.tp() == _Global) { parse_global_var_decls(lex); } else if (lex.tp() == _Const) { @@ -1624,17 +1648,48 @@ bool parse_source(std::istream* is, src::FileDescr* fdescr) { return true; } -bool parse_source_file(const char* filename) { +bool parse_source_file(const char* filename, src::Lexem lex) { if (!filename || !*filename) { - throw src::Fatal{"source file name is an empty string"}; + auto msg = "source file name is an empty string"; + if (lex.tp) { + lex.error(msg); + } else { + throw src::Fatal{msg}; + } } + char realpath_buf[PATH_MAX] = {0, }; + realpath(filename, realpath_buf); + std::string real_filename = std::string{realpath_buf}; + if (std::count(source_files.begin(), source_files.end(), real_filename)) { + if (verbosity >= 2) { + if (lex.tp) { + lex.loc.show_warning(std::string{"skipping file "} + real_filename + " because it was already included"); + } else { + std::cerr << "warning: skipping file " << real_filename << " because it was already included" << std::endl; + } + } + return true; + } + if (lex.tp) { // included + funC::generated_from += std::string{"incl:"}; + } + funC::generated_from += std::string{"`"} + filename + "` "; + source_files.push_back(real_filename); src::FileDescr* cur_source = new src::FileDescr{filename}; source_fdescr.push_back(cur_source); std::ifstream ifs{filename}; if (ifs.fail()) { - throw src::Fatal{std::string{"cannot open source file `"} + filename + "`"}; + auto msg = std::string{"cannot open source file `"} + filename + "`"; + if (lex.tp) { + lex.error(msg); + } else { + throw src::Fatal{msg}; + } } - return parse_source(&ifs, cur_source); + inclusion_locations.push(lex.loc); + bool res = parse_source(&ifs, cur_source); + inclusion_locations.pop(); + return res; } bool parse_source_stdin() { diff --git a/crypto/func/test/i1.fc b/crypto/func/test/i1.fc new file mode 100644 index 00000000..2b95e2dd --- /dev/null +++ b/crypto/func/test/i1.fc @@ -0,0 +1,14 @@ +global int i; + +#include "i1sub1.fc"; + +() sub0() impure { i = 0; } + +#include "i1sub2.fc"; + +() main() impure { + sub0(); + sub1(); + sub2(); + i = 9; +} diff --git a/crypto/func/test/i1sub1.fc b/crypto/func/test/i1sub1.fc new file mode 100644 index 00000000..c905bf22 --- /dev/null +++ b/crypto/func/test/i1sub1.fc @@ -0,0 +1,6 @@ +;; DO NOT COMPILE DIRECTLY! +;; Compile i1.fc + +() sub1() impure { + i = 1; +} diff --git a/crypto/func/test/i1sub2.fc b/crypto/func/test/i1sub2.fc new file mode 100644 index 00000000..b7f200db --- /dev/null +++ b/crypto/func/test/i1sub2.fc @@ -0,0 +1,8 @@ +;; DO NOT COMPILE DIRECTLY! +;; Compile i1.fc + +() sub2() impure { + sub1(); + sub0(); + i = 2; +} From eb35cf653ad77b51927e57945a83973427ef66d5 Mon Sep 17 00:00:00 2001 From: starlightduck Date: Thu, 12 May 2022 18:57:03 +0300 Subject: [PATCH 25/29] Use `td::realpath` and add re-inclusion tests --- crypto/func/parse-func.cpp | 6 ++---- crypto/func/test/i1.fc | 2 ++ crypto/func/test/i1sub1.fc | 2 ++ crypto/func/test/i1sub2.fc | 2 ++ 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/crypto/func/parse-func.cpp b/crypto/func/parse-func.cpp index c33a3cda..0d2aa985 100644 --- a/crypto/func/parse-func.cpp +++ b/crypto/func/parse-func.cpp @@ -23,7 +23,7 @@ #include "block/block.h" #include "block-parse.h" #include -#include +#include "td/utils/port/path.h" namespace sym { @@ -1657,9 +1657,7 @@ bool parse_source_file(const char* filename, src::Lexem lex) { throw src::Fatal{msg}; } } - char realpath_buf[PATH_MAX] = {0, }; - realpath(filename, realpath_buf); - std::string real_filename = std::string{realpath_buf}; + std::string real_filename = td::realpath(td::CSlice(filename)).move_as_ok(); if (std::count(source_files.begin(), source_files.end(), real_filename)) { if (verbosity >= 2) { if (lex.tp) { diff --git a/crypto/func/test/i1.fc b/crypto/func/test/i1.fc index 2b95e2dd..04f7889e 100644 --- a/crypto/func/test/i1.fc +++ b/crypto/func/test/i1.fc @@ -12,3 +12,5 @@ global int i; sub2(); i = 9; } + +#include "../test/i1sub2.fc"; \ No newline at end of file diff --git a/crypto/func/test/i1sub1.fc b/crypto/func/test/i1sub1.fc index c905bf22..62bdd177 100644 --- a/crypto/func/test/i1sub1.fc +++ b/crypto/func/test/i1sub1.fc @@ -1,6 +1,8 @@ ;; DO NOT COMPILE DIRECTLY! ;; Compile i1.fc +#include "i1sub1.fc"; + () sub1() impure { i = 1; } diff --git a/crypto/func/test/i1sub2.fc b/crypto/func/test/i1sub2.fc index b7f200db..0c48e192 100644 --- a/crypto/func/test/i1sub2.fc +++ b/crypto/func/test/i1sub2.fc @@ -1,6 +1,8 @@ ;; DO NOT COMPILE DIRECTLY! ;; Compile i1.fc +#include "./i1sub1.fc"; + () sub2() impure { sub1(); sub0(); From a973a51a87508a4068d2503db8b82545ca21fe51 Mon Sep 17 00:00:00 2001 From: dungeon-master-666 Date: Sat, 14 May 2022 22:27:03 +0200 Subject: [PATCH 26/29] Put back LastConfig callback. --- tonlib/tonlib/LastConfig.cpp | 2 +- tonlib/tonlib/LastConfig.h | 9 ++++++++- tonlib/tonlib/TonlibClient.cpp | 10 +++++++++- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/tonlib/tonlib/LastConfig.cpp b/tonlib/tonlib/LastConfig.cpp index 5ee73212..0111095b 100644 --- a/tonlib/tonlib/LastConfig.cpp +++ b/tonlib/tonlib/LastConfig.cpp @@ -39,7 +39,7 @@ td::StringBuilder& operator<<(td::StringBuilder& sb, const LastConfigState& stat return sb; } -LastConfig::LastConfig(ExtClientRef client) { +LastConfig::LastConfig(ExtClientRef client, td::unique_ptr callback) : callback_(std::move(callback)) { client_.set_client(client); VLOG(last_block) << "State: " << state_; } diff --git a/tonlib/tonlib/LastConfig.h b/tonlib/tonlib/LastConfig.h index d610165d..514b4a59 100644 --- a/tonlib/tonlib/LastConfig.h +++ b/tonlib/tonlib/LastConfig.h @@ -36,10 +36,17 @@ td::StringBuilder& operator<<(td::StringBuilder& sb, const LastConfigState& stat class LastConfig : public td::actor::Actor { public: - explicit LastConfig(ExtClientRef client); + class Callback { + public: + virtual ~Callback() { + } + }; + + explicit LastConfig(ExtClientRef client, td::unique_ptr callback); void get_last_config(td::Promise promise); private: + td::unique_ptr callback_; ExtClient client_; LastConfigState state_; diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index 8bcf53c9..2696bb1a 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -1439,9 +1439,17 @@ void TonlibClient::init_last_block(LastBlockState state) { } void TonlibClient::init_last_config() { ref_cnt_++; + class Callback : public LastConfig::Callback { + public: + Callback(td::actor::ActorShared client) : client_(std::move(client)) { + } + + private: + td::actor::ActorShared client_; + }; raw_last_config_ = td::actor::create_actor(td::actor::ActorOptions().with_name("LastConfig").with_poll(false), - get_client_ref()); + get_client_ref(), td::make_unique(td::actor::actor_shared(this))); } void TonlibClient::on_result(td::uint64 id, tonlib_api::object_ptr response) { From c07394aab50c96a3652d4ef0657b358c41685bc9 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Sun, 15 May 2022 17:51:24 +0300 Subject: [PATCH 27/29] Optimization of persistent state serialization (#364) * Fix double serialization of masterchain; increase sync_blocks_before * Improve logging in DownloadState * Write persistent state directly to file instead of a buffer * Don't keep ref to masterchain state in AsyncStateSerializer * Sparse state serialization over longer period Co-authored-by: SpyCheese --- crypto/vm/boc.cpp | 200 +++++++++++++++++++---- crypto/vm/boc.h | 23 +-- validator/db/archive-manager.cpp | 25 ++- validator/db/archive-manager.hpp | 5 + validator/db/files-async.hpp | 42 +++-- validator/db/rootdb.cpp | 7 + validator/db/rootdb.hpp | 3 + validator/impl/shard.cpp | 25 +++ validator/impl/shard.hpp | 1 + validator/interfaces/db.h | 3 + validator/interfaces/shard.h | 1 + validator/interfaces/validator-manager.h | 3 + validator/manager-disk.cpp | 7 + validator/manager-disk.hpp | 3 + validator/manager-hardfork.hpp | 5 + validator/manager.cpp | 7 + validator/manager.hpp | 3 + validator/net/download-state.cpp | 9 +- validator/net/download-state.hpp | 3 + validator/state-serializer.cpp | 32 ++-- validator/state-serializer.hpp | 2 +- validator/validator.h | 2 +- 22 files changed, 327 insertions(+), 84 deletions(-) diff --git a/crypto/vm/boc.cpp b/crypto/vm/boc.cpp index c53595ff..8ff244d0 100644 --- a/crypto/vm/boc.cpp +++ b/crypto/vm/boc.cpp @@ -475,15 +475,128 @@ std::string BagOfCells::extract_string() const { return std::string{serialized.data(), serialized.data() + serialized.size()}; } -void BagOfCells::store_uint(unsigned long long value, unsigned bytes) { - unsigned char* ptr = store_ptr += bytes; - store_chk(); - while (bytes) { - *--ptr = value & 0xff; - value >>= 8; - --bytes; +namespace { +struct BufferWriter { + BufferWriter(unsigned char* store_start, unsigned char* store_end) + : store_start(store_start), store_ptr(store_start), store_end(store_end) {} + + size_t position() const { + return store_ptr - store_start; } - DCHECK(!bytes); + size_t remaining() const { + return store_end - store_ptr; + } + void chk() const { + DCHECK(store_ptr <= store_end); + } + bool empty() const { + return store_ptr == store_end; + } + void store_uint(unsigned long long value, unsigned bytes) { + unsigned char* ptr = store_ptr += bytes; + chk(); + while (bytes) { + *--ptr = value & 0xff; + value >>= 8; + --bytes; + } + DCHECK(!bytes); + } + void store_bytes(unsigned char const* data, size_t s) { + store_ptr += s; + chk(); + memcpy(store_ptr - s, data, s); + } + unsigned get_crc32() const { + return td::crc32c(td::Slice{store_start, store_ptr}); + } + + private: + unsigned char* store_start; + unsigned char* store_ptr; + unsigned char* store_end; +}; + +struct FileWriter { + FileWriter(td::FileFd& fd, size_t expected_size) + : fd(fd), expected_size(expected_size) {} + + ~FileWriter() { + flush(); + } + + size_t position() const { + return flushed_size + writer.position(); + } + size_t remaining() const { + return expected_size - position(); + } + void chk() const { + DCHECK(position() <= expected_size); + } + bool empty() const { + return remaining() == 0; + } + void store_uint(unsigned long long value, unsigned bytes) { + flush_if_needed(bytes); + writer.store_uint(value, bytes); + } + void store_bytes(unsigned char const* data, size_t s) { + flush_if_needed(s); + writer.store_bytes(data, s); + } + unsigned get_crc32() const { + unsigned char const* start = buf.data(); + unsigned char const* end = start + writer.position(); + return td::crc32c_extend(current_crc32, td::Slice(start, end)); + } + + td::Status finalize() { + flush(); + return std::move(res); + } + + private: + void flush_if_needed(size_t s) { + DCHECK(s <= BUF_SIZE); + if (s > BUF_SIZE - writer.position()) { + flush(); + } + } + + void flush() { + chk(); + unsigned char* start = buf.data(); + unsigned char* end = start + writer.position(); + if (start == end) { + return; + } + flushed_size += end - start; + current_crc32 = td::crc32c_extend(current_crc32, td::Slice(start, end)); + if (res.is_ok()) { + while (end > start) { + auto R = fd.write(td::Slice(start, end)); + if (R.is_error()) { + res = R.move_as_error(); + break; + } + size_t s = R.move_as_ok(); + start += s; + } + } + writer = BufferWriter(buf.data(), buf.data() + buf.size()); + } + + td::FileFd& fd; + size_t expected_size; + size_t flushed_size = 0; + unsigned current_crc32 = td::crc32c(td::Slice()); + + static const size_t BUF_SIZE = 1 << 22; + std::vector buf = std::vector(BUF_SIZE, '\0'); + BufferWriter writer = BufferWriter(buf.data(), buf.data() + buf.size()); + td::Status res = td::Status::OK(); +}; } //serialized_boc#672fb0ac has_idx:(## 1) has_crc32c:(## 1) @@ -497,13 +610,16 @@ void BagOfCells::store_uint(unsigned long long value, unsigned bytes) { // index:(cells * ##(off_bytes * 8)) // cell_data:(tot_cells_size * [ uint8 ]) // = BagOfCells; -std::size_t BagOfCells::serialize_to(unsigned char* buffer, std::size_t buff_size, int mode) { - std::size_t size_est = estimate_serialized_size(mode); - if (!size_est || size_est > buff_size) { - return 0; - } - init_store(buffer, buffer + size_est); - store_uint(info.magic, 4); +template +std::size_t BagOfCells::serialize_to_impl(WriterT& writer, int mode) { + auto store_ref = [&](unsigned long long value) { + writer.store_uint(value, info.ref_byte_size); + }; + auto store_offset = [&](unsigned long long value) { + writer.store_uint(value, info.offset_byte_size); + }; + + writer.store_uint(info.magic, 4); td::uint8 byte{0}; if (info.has_index) { @@ -520,9 +636,9 @@ std::size_t BagOfCells::serialize_to(unsigned char* buffer, std::size_t buff_siz return 0; } byte |= static_cast(info.ref_byte_size); - store_uint(byte, 1); + writer.store_uint(byte, 1); - store_uint(info.offset_byte_size, 1); + writer.store_uint(info.offset_byte_size, 1); store_ref(cell_count); store_ref(root_count); store_ref(0); @@ -532,7 +648,7 @@ std::size_t BagOfCells::serialize_to(unsigned char* buffer, std::size_t buff_siz DCHECK(k >= 0 && k < cell_count); store_ref(k); } - DCHECK(store_ptr - buffer == (long long)info.index_offset); + DCHECK(writer.position() == info.index_offset); DCHECK((unsigned)cell_count == cell_list_.size()); if (info.has_index) { std::size_t offs = 0; @@ -551,8 +667,8 @@ std::size_t BagOfCells::serialize_to(unsigned char* buffer, std::size_t buff_siz } DCHECK(offs == info.data_size); } - DCHECK(store_ptr - buffer == (long long)info.data_offset); - unsigned char* keep_ptr = store_ptr; + DCHECK(writer.position() == info.data_offset); + size_t keep_position = writer.position(); for (int i = 0; i < cell_count; ++i) { const auto& dc_info = cell_list_[cell_count - 1 - i]; const Ref& dc = dc_info.dc_ref; @@ -560,9 +676,9 @@ std::size_t BagOfCells::serialize_to(unsigned char* buffer, std::size_t buff_siz if (dc_info.is_root_cell && (mode & Mode::WithTopHash)) { with_hash = true; } - int s = dc->serialize(store_ptr, 256, with_hash); - store_ptr += s; - store_chk(); + unsigned char buf[256]; + int s = dc->serialize(buf, 256, with_hash); + writer.store_bytes(buf, s); DCHECK(dc->size_refs() == dc_info.ref_num); // std::cerr << (dc_info.is_special() ? '*' : ' ') << i << '<' << (int)dc_info.wt << ">:"; for (unsigned j = 0; j < dc_info.ref_num; ++j) { @@ -573,16 +689,38 @@ std::size_t BagOfCells::serialize_to(unsigned char* buffer, std::size_t buff_siz } // std::cerr << std::endl; } - store_chk(); - DCHECK(store_ptr - keep_ptr == (long long)info.data_size); - DCHECK(store_end - store_ptr == (info.has_crc32c ? 4 : 0)); + writer.chk(); + DCHECK(writer.position() - keep_position == info.data_size); + DCHECK(writer.remaining() == (info.has_crc32c ? 4 : 0)); if (info.has_crc32c) { - // compute crc32c of buffer .. store_ptr - unsigned crc = td::crc32c(td::Slice{buffer, store_ptr}); - store_uint(td::bswap32(crc), 4); + unsigned crc = writer.get_crc32(); + writer.store_uint(td::bswap32(crc), 4); } - DCHECK(store_empty()); - return store_ptr - buffer; + DCHECK(writer.empty()); + return writer.position(); +} + +std::size_t BagOfCells::serialize_to(unsigned char* buffer, std::size_t buff_size, int mode) { + std::size_t size_est = estimate_serialized_size(mode); + if (!size_est || size_est > buff_size) { + return 0; + } + BufferWriter writer{buffer, buffer + size_est}; + return serialize_to_impl(writer, mode); +} + +td::Status BagOfCells::serialize_to_file(td::FileFd& fd, int mode) { + std::size_t size_est = estimate_serialized_size(mode); + if (!size_est) { + return td::Status::Error("no cells to serialize to this bag of cells"); + } + FileWriter writer{fd, size_est}; + size_t s = serialize_to_impl(writer, mode); + TRY_STATUS(writer.finalize()); + if (s != size_est) { + return td::Status::Error("error while serializing a bag of cells: actual serialized size differs from estimated"); + } + return td::Status::OK(); } unsigned long long BagOfCells::Info::read_int(const unsigned char* ptr, unsigned bytes) { diff --git a/crypto/vm/boc.h b/crypto/vm/boc.h index cc277a8c..02078e27 100644 --- a/crypto/vm/boc.h +++ b/crypto/vm/boc.h @@ -23,6 +23,7 @@ #include "td/utils/buffer.h" #include "td/utils/HashMap.h" #include "td/utils/HashSet.h" +#include "td/utils/port/FileFd.h" namespace vm { using td::Ref; @@ -216,8 +217,6 @@ class BagOfCells { int max_depth{1024}; Info info; unsigned long long data_bytes{0}; - unsigned char* store_ptr{nullptr}; - unsigned char* store_end{nullptr}; td::HashMap cells; struct CellInfo { Ref dc_ref; @@ -267,6 +266,9 @@ class BagOfCells { std::string serialize_to_string(int mode = 0); td::Result serialize_to_slice(int mode = 0); std::size_t serialize_to(unsigned char* buffer, std::size_t buff_size, int mode = 0); + td::Status serialize_to_file(td::FileFd& fd, int mode = 0); + template + std::size_t serialize_to_impl(WriterT& writer, int mode = 0); std::string extract_string() const; td::Result deserialize(const td::Slice& data, int max_roots = default_max_roots); @@ -295,23 +297,6 @@ class BagOfCells { cell_list_.clear(); } td::uint64 compute_sizes(int mode, int& r_size, int& o_size); - void init_store(unsigned char* from, unsigned char* to) { - store_ptr = from; - store_end = to; - } - void store_chk() const { - DCHECK(store_ptr <= store_end); - } - bool store_empty() const { - return store_ptr == store_end; - } - void store_uint(unsigned long long value, unsigned bytes); - void store_ref(unsigned long long value) { - store_uint(value, info.ref_byte_size); - } - void store_offset(unsigned long long value) { - store_uint(value, info.offset_byte_size); - } void reorder_cells(); int revisit(int cell_idx, int force = 0); unsigned long long get_idx_entry_raw(int index); diff --git a/validator/db/archive-manager.cpp b/validator/db/archive-manager.cpp index fb8f2c00..e7614154 100644 --- a/validator/db/archive-manager.cpp +++ b/validator/db/archive-manager.cpp @@ -336,6 +336,28 @@ void ArchiveManager::add_zero_state(BlockIdExt block_id, td::BufferSlice data, t void ArchiveManager::add_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice data, td::Promise promise) { + auto create_writer = [&](std::string path, td::Promise P) { + td::actor::create_actor("writefile", db_root_ + "/archive/tmp/", + std::move(path), std::move(data), std::move(P)) + .release(); + }; + add_persistent_state_impl(block_id, masterchain_block_id, std::move(promise), std::move(create_writer)); +} + +void ArchiveManager::add_persistent_state_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, + std::function write_state, + td::Promise promise) { + auto create_writer = [&](std::string path, td::Promise P) { + td::actor::create_actor("writefile", db_root_ + "/archive/tmp/", + std::move(path), std::move(write_state), std::move(P)) + .release(); + }; + add_persistent_state_impl(block_id, masterchain_block_id, std::move(promise), std::move(create_writer)); +} + +void ArchiveManager::add_persistent_state_impl(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise, + std::function)> create_writer) { auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}}; auto hash = id.hash(); if (perm_states_.find(hash) != perm_states_.end()) { @@ -353,8 +375,7 @@ void ArchiveManager::add_persistent_state(BlockIdExt block_id, BlockIdExt master promise.set_value(td::Unit()); } }); - td::actor::create_actor("writefile", db_root_ + "/archive/tmp/", path, std::move(data), std::move(P)) - .release(); + create_writer(std::move(path), std::move(P)); } void ArchiveManager::get_zero_state(BlockIdExt block_id, td::Promise promise) { diff --git a/validator/db/archive-manager.hpp b/validator/db/archive-manager.hpp index dbe6e4d4..31e0e6b6 100644 --- a/validator/db/archive-manager.hpp +++ b/validator/db/archive-manager.hpp @@ -45,6 +45,9 @@ class ArchiveManager : public td::actor::Actor { void add_zero_state(BlockIdExt block_id, td::BufferSlice data, td::Promise promise); void add_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice data, td::Promise promise); + void add_persistent_state_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, + std::function write_state, + td::Promise promise); void get_zero_state(BlockIdExt block_id, td::Promise promise); void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise); void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, @@ -137,6 +140,8 @@ class ArchiveManager : public td::actor::Actor { PackageId get_max_temp_file_desc_idx(); PackageId get_prev_temp_file_desc_idx(PackageId id); + void add_persistent_state_impl(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise, + std::function)> create_writer); void written_perm_state(FileReferenceShort id); void persistent_state_gc(FileHash last); diff --git a/validator/db/files-async.hpp b/validator/db/files-async.hpp index 1f2ab5ce..2da534bf 100644 --- a/validator/db/files-async.hpp +++ b/validator/db/files-async.hpp @@ -52,30 +52,50 @@ class WriteFile : public td::actor::Actor { auto res = R.move_as_ok(); auto file = std::move(res.first); auto old_name = res.second; - td::uint64 offset = 0; - while (data_.size() > 0) { - auto R = file.pwrite(data_.as_slice(), offset); - auto s = R.move_as_ok(); - offset += s; - data_.confirm_read(s); + auto status = write_data_(file); + if (!status.is_error()) { + status = file.sync(); + } + if (status.is_error()) { + td::unlink(old_name); + promise_.set_error(std::move(status)); + stop(); + return; } - file.sync().ensure(); if (new_name_.length() > 0) { - td::rename(old_name, new_name_).ensure(); - promise_.set_value(std::move(new_name_)); + status = td::rename(old_name, new_name_); + if (status.is_error()) { + promise_.set_error(std::move(status)); + } else { + promise_.set_value(std::move(new_name_)); + } } else { promise_.set_value(std::move(old_name)); } stop(); } + WriteFile(std::string tmp_dir, std::string new_name, std::function write_data, + td::Promise promise) + : tmp_dir_(tmp_dir), new_name_(new_name), write_data_(std::move(write_data)), promise_(std::move(promise)) { + } WriteFile(std::string tmp_dir, std::string new_name, td::BufferSlice data, td::Promise promise) - : tmp_dir_(tmp_dir), new_name_(new_name), data_(std::move(data)), promise_(std::move(promise)) { + : tmp_dir_(tmp_dir), new_name_(new_name), promise_(std::move(promise)) { + write_data_ = [data_ptr = std::make_shared(std::move(data))] (td::FileFd& fd) { + auto data = std::move(*data_ptr); + td::uint64 offset = 0; + while (data.size() > 0) { + TRY_RESULT(s, fd.pwrite(data.as_slice(), offset)); + offset += s; + data.confirm_read(s); + } + return td::Status::OK(); + }; } private: const std::string tmp_dir_; std::string new_name_; - td::BufferSlice data_; + std::function write_data_; td::Promise promise_; }; diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index ef88ad5f..58e3dd2f 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -276,6 +276,13 @@ void RootDb::store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterc std::move(state), std::move(promise)); } +void RootDb::store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, + std::function write_data, + td::Promise promise) { + td::actor::send_closure(archive_db_, &ArchiveManager::add_persistent_state_gen, block_id, masterchain_block_id, + std::move(write_data), std::move(promise)); +} + void RootDb::get_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise) { td::actor::send_closure(archive_db_, &ArchiveManager::get_persistent_state, block_id, masterchain_block_id, diff --git a/validator/db/rootdb.hpp b/validator/db/rootdb.hpp index c4d642e7..2654d482 100644 --- a/validator/db/rootdb.hpp +++ b/validator/db/rootdb.hpp @@ -69,6 +69,9 @@ class RootDb : public Db { void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, td::Promise promise) override; + void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, + std::function write_data, + td::Promise promise) override; void get_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise) override; void get_persistent_state_file_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, diff --git a/validator/impl/shard.cpp b/validator/impl/shard.cpp index 18f71d32..a96f1a81 100644 --- a/validator/impl/shard.cpp +++ b/validator/impl/shard.cpp @@ -26,6 +26,7 @@ #include "vm/cells/MerkleUpdate.h" #include "block/block-parse.h" #include "block/block-auto.h" +#include "td/utils/filesystem.h" #define LAZY_STATE_DESERIALIZE 1 @@ -301,6 +302,30 @@ td::Result ShardStateQ::serialize() const { return st_res.move_as_ok(); } +td::Status ShardStateQ::serialize_to_file(td::FileFd& fd) const { + td::PerfWarningTimer perf_timer_{"serializestate", 0.1}; + if (!data.is_null()) { + auto cur_data = data.clone(); + while (cur_data.size() > 0) { + TRY_RESULT(s, fd.write(cur_data.as_slice())); + cur_data.confirm_read(s); + } + return td::Status::OK(); + } + if (root.is_null()) { + return td::Status::Error(-666, "cannot serialize an uninitialized state"); + } + vm::BagOfCells new_boc; + new_boc.set_root(root); + TRY_STATUS(new_boc.import_cells()); + auto st_res = new_boc.serialize_to_file(fd, 31); + if (st_res.is_error()) { + LOG(ERROR) << "cannot serialize a shardchain state"; + return st_res.move_as_error(); + } + return td::Status::OK(); +} + MasterchainStateQ::MasterchainStateQ(const BlockIdExt& _id, td::BufferSlice _data) : MasterchainState(), ShardStateQ(_id, std::move(_data)) { } diff --git a/validator/impl/shard.hpp b/validator/impl/shard.hpp index 862b0fc5..0056df11 100644 --- a/validator/impl/shard.hpp +++ b/validator/impl/shard.hpp @@ -87,6 +87,7 @@ class ShardStateQ : virtual public ShardState { td::Result> merge_with(const ShardState& with) const override; td::Result, Ref>> split() const override; td::Result serialize() const override; + td::Status serialize_to_file(td::FileFd& fd) const override; }; #if TD_MSVC diff --git a/validator/interfaces/db.h b/validator/interfaces/db.h index f49a7819..9983572b 100644 --- a/validator/interfaces/db.h +++ b/validator/interfaces/db.h @@ -53,6 +53,9 @@ class Db : public td::actor::Actor { virtual void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, td::Promise promise) = 0; + virtual void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, + std::function write_data, + td::Promise promise) = 0; virtual void get_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise) = 0; virtual void get_persistent_state_file_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, diff --git a/validator/interfaces/shard.h b/validator/interfaces/shard.h index 0cfbda43..96ca8da2 100644 --- a/validator/interfaces/shard.h +++ b/validator/interfaces/shard.h @@ -55,6 +55,7 @@ class ShardState : public td::CntObject { virtual td::Result, td::Ref>> split() const = 0; virtual td::Result serialize() const = 0; + virtual td::Status serialize_to_file(td::FileFd& fd) const = 0; }; class MasterchainState : virtual public ShardState { diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 1d4a7c9f..42e99ecc 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -57,6 +57,9 @@ class ValidatorManager : public ValidatorManagerInterface { td::Promise> promise) = 0; virtual void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, td::Promise promise) = 0; + virtual void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, + std::function write_data, + td::Promise promise) = 0; virtual void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) = 0; virtual void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) = 0; diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index e43e37b2..404f7fe8 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -680,6 +680,13 @@ void ValidatorManagerImpl::store_persistent_state_file(BlockIdExt block_id, Bloc std::move(promise)); } +void ValidatorManagerImpl::store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, + std::function write_data, + td::Promise promise) { + td::actor::send_closure(db_, &Db::store_persistent_state_file_gen, block_id, masterchain_block_id, + std::move(write_data), std::move(promise)); +} + void ValidatorManagerImpl::store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) { td::actor::send_closure(db_, &Db::store_zero_state_file, block_id, std::move(state), std::move(promise)); diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index fa365ccf..bb740b4c 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -143,6 +143,9 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override; void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, td::Promise promise) override; + void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, + std::function write_data, + td::Promise promise) override; void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) override; void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) override; diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 9938bcb2..71157dcc 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -169,6 +169,11 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise promise) override { UNREACHABLE(); } + void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, + std::function write_data, + td::Promise promise) override { + UNREACHABLE(); + } void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) override { UNREACHABLE(); } diff --git a/validator/manager.cpp b/validator/manager.cpp index e6fe5954..28e5cd3a 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1051,6 +1051,13 @@ void ValidatorManagerImpl::store_persistent_state_file(BlockIdExt block_id, Bloc std::move(promise)); } +void ValidatorManagerImpl::store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, + std::function write_data, + td::Promise promise) { + td::actor::send_closure(db_, &Db::store_persistent_state_file_gen, block_id, masterchain_block_id, + std::move(write_data), std::move(promise)); +} + void ValidatorManagerImpl::store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) { td::actor::send_closure(db_, &Db::store_zero_state_file, block_id, std::move(state), std::move(promise)); diff --git a/validator/manager.hpp b/validator/manager.hpp index 19ee4cdd..123625b2 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -347,6 +347,9 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override; void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, td::Promise promise) override; + void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, + std::function write_data, + td::Promise promise) override; void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) override; void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) override; diff --git a/validator/net/download-state.cpp b/validator/net/download-state.cpp index 49986ce9..fedfaae8 100644 --- a/validator/net/download-state.cpp +++ b/validator/net/download-state.cpp @@ -149,6 +149,7 @@ void DownloadState::got_block_state_description(td::BufferSlice data) { abort_query(F.move_as_error()); return; } + prev_logged_timer_ = td::Timer(); ton_api::downcast_call( *F.move_as_ok().get(), @@ -188,8 +189,12 @@ void DownloadState::got_block_state_part(td::BufferSlice data, td::uint32 reques sum_ += data.size(); parts_.push_back(std::move(data)); - if (sum_ % (1 << 22) == 0) { - LOG(DEBUG) << "downloading state " << block_id_ << ": total=" << sum_; + double elapsed = prev_logged_timer_.elapsed(); + if (elapsed > 10.0) { + prev_logged_timer_ = td::Timer(); + LOG(INFO) << "downloading state " << block_id_ << ": total=" << sum_ << + " (" << double(sum_ - prev_logged_sum_) / elapsed << " B/s)"; + prev_logged_sum_ = sum_; } if (last_part) { diff --git a/validator/net/download-state.hpp b/validator/net/download-state.hpp index d965e3c7..a586f61f 100644 --- a/validator/net/download-state.hpp +++ b/validator/net/download-state.hpp @@ -72,6 +72,9 @@ class DownloadState : public td::actor::Actor { td::BufferSlice state_; std::vector parts_; td::uint64 sum_ = 0; + + td::uint64 prev_logged_sum_ = 0; + td::Timer prev_logged_timer_; }; } // namespace fullnode diff --git a/validator/state-serializer.cpp b/validator/state-serializer.cpp index 63b7b864..daf2de11 100644 --- a/validator/state-serializer.cpp +++ b/validator/state-serializer.cpp @@ -122,14 +122,12 @@ void AsyncStateSerializer::next_iteration() { CHECK(masterchain_handle_->id() == last_block_id_); if (attempt_ < max_attempt() && last_key_block_id_.id.seqno < last_block_id_.id.seqno && need_serialize(masterchain_handle_)) { - if (masterchain_state_.is_null()) { + if (!have_masterchain_state_) { // block next attempts immediately, but send actual request later running_ = true; delay_action( [SelfId = actor_id(this)]() { td::actor::send_closure(SelfId, &AsyncStateSerializer::request_masterchain_state); }, - // Masterchain is more important and much lighter than shards - // thus lower delay - td::Timestamp::in(td::Random::fast(0, 600))); + td::Timestamp::in(td::Random::fast(0, 3600))); return; } while (next_idx_ < shards_.size()) { @@ -140,8 +138,6 @@ void AsyncStateSerializer::next_iteration() { running_ = true; delay_action( [SelfId = actor_id(this), shard = shards_[next_idx_]]() { td::actor::send_closure(SelfId, &AsyncStateSerializer::request_shard_state, shard); }, - // Shards are less important and heavier than master - // thus higher delay td::Timestamp::in(td::Random::fast(0, 4 * 3600))); return; } @@ -162,7 +158,7 @@ void AsyncStateSerializer::next_iteration() { } if (masterchain_handle_->inited_next_left()) { last_block_id_ = masterchain_handle_->one_next(true); - masterchain_state_ = td::Ref{}; + have_masterchain_state_ = false; masterchain_handle_ = nullptr; saved_to_db_ = false; shards_.clear(); @@ -186,25 +182,25 @@ void AsyncStateSerializer::got_masterchain_handle(BlockHandle handle) { } void AsyncStateSerializer::got_masterchain_state(td::Ref state) { - masterchain_state_ = state; + have_masterchain_state_ = true; CHECK(next_idx_ == 0); CHECK(shards_.size() == 0); - auto vec = masterchain_state_->get_shards(); - shards_.push_back(masterchain_handle_->id()); + auto vec = state->get_shards(); for (auto &v : vec) { shards_.push_back(v->top_block_id()); } - auto B = masterchain_state_->serialize(); - B.ensure(); + auto write_data = [state] (td::FileFd& fd) { + return state->serialize_to_file(fd); + }; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { R.ensure(); td::actor::send_closure(SelfId, &AsyncStateSerializer::stored_masterchain_state); }); - td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file, masterchain_handle_->id(), - masterchain_handle_->id(), B.move_as_ok(), std::move(P)); + td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file_gen, masterchain_handle_->id(), + masterchain_handle_->id(), write_data, std::move(P)); } void AsyncStateSerializer::stored_masterchain_state() { @@ -225,13 +221,15 @@ void AsyncStateSerializer::got_shard_handle(BlockHandle handle) { } void AsyncStateSerializer::got_shard_state(BlockHandle handle, td::Ref state) { - auto B = state->serialize().move_as_ok(); + auto write_data = [state] (td::FileFd& fd) { + return state->serialize_to_file(fd); + }; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { R.ensure(); td::actor::send_closure(SelfId, &AsyncStateSerializer::success_handler); }); - td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file, handle->id(), - masterchain_handle_->id(), std::move(B), std::move(P)); + td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file_gen, handle->id(), + masterchain_handle_->id(), write_data, std::move(P)); LOG(INFO) << "storing persistent state for " << masterchain_handle_->id().seqno() << ":" << handle->id().id.shard; next_idx_++; } diff --git a/validator/state-serializer.hpp b/validator/state-serializer.hpp index 162e96c1..14261df7 100644 --- a/validator/state-serializer.hpp +++ b/validator/state-serializer.hpp @@ -43,7 +43,7 @@ class AsyncStateSerializer : public td::actor::Actor { td::uint32 next_idx_ = 0; BlockHandle masterchain_handle_; - td::Ref masterchain_state_; + bool have_masterchain_state_ = false; std::vector shards_; diff --git a/validator/validator.h b/validator/validator.h index 349824d6..46110c92 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -96,7 +96,7 @@ struct ValidatorManagerOptions : public td::CntObject { BlockIdExt zero_block_id, BlockIdExt init_block_id, std::function check_shard = [](ShardIdFull, CatchainSeqno, ShardCheckMode) { return true; }, - bool allow_blockchain_init = false, double sync_blocks_before = 300, double block_ttl = 86400 * 7, + bool allow_blockchain_init = false, double sync_blocks_before = 86400, double block_ttl = 86400 * 7, double state_ttl = 3600, double archive_ttl = 86400 * 365, double key_proof_ttl = 86400 * 3650, double max_mempool_num = 999999, bool initial_sync_disabled = false); From 3b4166be69a0e34027e2ac1cd0ef69c9efb2371a Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 16 May 2022 12:29:39 +0300 Subject: [PATCH 28/29] Add logs in AsyncStateSerializer --- validator/state-serializer.cpp | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/validator/state-serializer.cpp b/validator/state-serializer.cpp index daf2de11..82cf31cc 100644 --- a/validator/state-serializer.cpp +++ b/validator/state-serializer.cpp @@ -123,11 +123,12 @@ void AsyncStateSerializer::next_iteration() { if (attempt_ < max_attempt() && last_key_block_id_.id.seqno < last_block_id_.id.seqno && need_serialize(masterchain_handle_)) { if (!have_masterchain_state_) { - // block next attempts immediately, but send actual request later - running_ = true; - delay_action( - [SelfId = actor_id(this)]() { td::actor::send_closure(SelfId, &AsyncStateSerializer::request_masterchain_state); }, - td::Timestamp::in(td::Random::fast(0, 3600))); + LOG(INFO) << "started serializing persistent state for " << masterchain_handle_->id().seqno(); + // block next attempts immediately, but send actual request later + running_ = true; + delay_action( + [SelfId = actor_id(this)]() { td::actor::send_closure(SelfId, &AsyncStateSerializer::request_masterchain_state); }, + td::Timestamp::in(td::Random::fast(0, 3600))); return; } while (next_idx_ < shards_.size()) { @@ -142,6 +143,7 @@ void AsyncStateSerializer::next_iteration() { return; } } + LOG(INFO) << "finished serializing persistent state for " << masterchain_handle_->id().seqno(); last_key_block_ts_ = masterchain_handle_->unix_time(); last_key_block_id_ = masterchain_handle_->id(); } @@ -182,6 +184,7 @@ void AsyncStateSerializer::got_masterchain_handle(BlockHandle handle) { } void AsyncStateSerializer::got_masterchain_state(td::Ref state) { + LOG(INFO) << "serializing masterchain state " << masterchain_handle_->id().seqno(); have_masterchain_state_ = true; CHECK(next_idx_ == 0); CHECK(shards_.size() == 0); @@ -204,6 +207,7 @@ void AsyncStateSerializer::got_masterchain_state(td::Ref state } void AsyncStateSerializer::stored_masterchain_state() { + LOG(INFO) << "finished serializing masterchain state " << masterchain_handle_->id().seqno(); running_ = false; next_iteration(); } @@ -221,16 +225,17 @@ void AsyncStateSerializer::got_shard_handle(BlockHandle handle) { } void AsyncStateSerializer::got_shard_state(BlockHandle handle, td::Ref state) { + LOG(INFO) << "serializing shard state " << handle->id().seqno(); auto write_data = [state] (td::FileFd& fd) { return state->serialize_to_file(fd); }; - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result R) { R.ensure(); + LOG(INFO) << "finished serializing shard state " << handle->id().seqno(); td::actor::send_closure(SelfId, &AsyncStateSerializer::success_handler); }); td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file_gen, handle->id(), masterchain_handle_->id(), write_data, std::move(P)); - LOG(INFO) << "storing persistent state for " << masterchain_handle_->id().seqno() << ":" << handle->id().id.shard; next_idx_++; } From b9481fb08e5c34010c768bb03c46d9573eb6143a Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Mon, 16 May 2022 22:47:01 +0300 Subject: [PATCH 29/29] Add Changelog to May 22 release --- Changelog.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 Changelog.md diff --git a/Changelog.md b/Changelog.md new file mode 100644 index 00000000..e9052c0b --- /dev/null +++ b/Changelog.md @@ -0,0 +1,14 @@ +## 05.2022 Update +* Initial synchronization improved: adjusted timeouts for state download and the way of choosing which state to download. Nodes with low network speed and/or bad connectivity will synchronize faster and consistently. +* Improved peer-to-peer network stability and DDoS resistance: now peers will only relay valid messages to the network. Large messages, which require splitting for relaying, will be retranslated as well, but only after the node gets all parts, and reassembles and checks them. Validators may sign certificates for network peers, which allow relaying large messages by parts without checks. It is used now by validators to faster relay new blocks. Sign and import certificate commands are exposed via `validator-engine-console`. +* Fixed some rare edge cases in TVM arithmetic operations related to big numbers (`2**63+`) +* Improved fixes used to combat wrong activate-destruct-activate contract behavior last November. +* Improved tonlib: support libraries (with client-side caching), getmethods completely fill c7 register, getmethods support slice arguments, improved messages listing for transactions, added extended block header params, added getConfig method. +* RocksDB updated to a newer version. +* Improved persistent state serialization: memory usage during serialization was optimized; the start of serialization on different nodes was sparsed. +* FunC update: support for string literals and constants (including precompiled constant expressions), semver, `include` expressions. +* Fixed rarely manifested bugs in `Asm.fif`. +* LiteClient supports key as cli parameter. +* Improved Liteserver DoS resistance for running getmethods. + +Besides the work of the core team, this update is based on the efforts of @tvorogme (added support for slice arguments and noted bugs in Asm.fif), @akifoq (fixed bug in Asm.fif), @cryshado (noted strange behavior of LS, which, upon inspection, turned out to be a vector of DoS attack).