From 678a8a6a1363131766cd6e8c89738f6d6dc87d29 Mon Sep 17 00:00:00 2001 From: OmicronTau Date: Sat, 13 Nov 2021 16:17:17 +0300 Subject: [PATCH 01/21] Add overlay certificate checks --- catchain/catchain-receiver.cpp | 2 +- overlay/overlay-broadcast.cpp | 45 ++++++++++++++++++-- overlay/overlay-broadcast.hpp | 20 +++++---- overlay/overlay-fec-broadcast.cpp | 11 ++++- overlay/overlay-fec-broadcast.hpp | 4 +- overlay/overlay-manager.cpp | 67 ++++++++++++++++++++++-------- overlay/overlay.cpp | 43 ++++++++++++------- overlay/overlay.hpp | 5 ++- overlay/overlays.h | 48 +++++++++++++++++---- tl/generate/scheme/ton_api.tl | 2 + tl/generate/scheme/ton_api.tlo | Bin 64168 -> 64644 bytes validator/full-node-shard.cpp | 27 +++++++++--- validator/full-node-shard.hpp | 3 ++ validator/manager-disk.hpp | 3 ++ validator/manager-hardfork.hpp | 3 ++ validator/manager.hpp | 5 +++ validator/validator.h | 6 +-- 17 files changed, 232 insertions(+), 62 deletions(-) diff --git a/catchain/catchain-receiver.cpp b/catchain/catchain-receiver.cpp index b9b0373f..e7f5c019 100644 --- a/catchain/catchain-receiver.cpp +++ b/catchain/catchain-receiver.cpp @@ -486,7 +486,7 @@ void CatChainReceiverImpl::start_up() { } td::actor::send_closure(overlay_manager_, &overlay::Overlays::create_private_overlay, get_source(local_idx_)->get_adnl_id(), overlay_full_id_.clone(), std::move(ids), - make_callback(), overlay::OverlayPrivacyRules{0, std::move(root_keys)}); + make_callback(), overlay::OverlayPrivacyRules{0, 0, std::move(root_keys)}); CHECK(root_block_); diff --git a/overlay/overlay-broadcast.cpp b/overlay/overlay-broadcast.cpp index bade50e7..1c05f705 100644 --- a/overlay/overlay-broadcast.cpp +++ b/overlay/overlay-broadcast.cpp @@ -17,8 +17,14 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "overlay-broadcast.hpp" +#include "adnl/adnl-node-id.hpp" +#include "common/util.h" #include "overlay.hpp" #include "keys/encryptor.h" +#include "td/actor/PromiseFuture.h" +#include "td/actor/actor.h" +#include "td/utils/Status.h" +#include "td/utils/port/Stat.h" namespace ton { @@ -33,7 +39,13 @@ td::Status BroadcastSimple::check_duplicate() { } td::Status BroadcastSimple::check_source() { - return overlay_->check_source_eligible(source_, cert_.get(), data_size()); + auto r = overlay_->check_source_eligible(source_, cert_.get(), data_size(), false); + if (r == BroadcastCheckResult::Forbidden) { + return td::Status::Error(ErrorCode::error, "broadcast is forbidden"); + } + + is_valid_ = r == BroadcastCheckResult::Allowed; + return td::Status::OK(); } td::BufferSlice BroadcastSimple::to_sign() { @@ -66,6 +78,14 @@ td::Status BroadcastSimple::distribute() { return td::Status::OK(); } +void BroadcastSimple::broadcast_checked(td::Result R) { + if (R.is_error()) { + return; + } + is_valid_ = true; + run_continue().ignore(); +} + tl_object_ptr BroadcastSimple::tl() const { return create_tl_object(source_.tl(), cert_ ? cert_->tl() : Certificate::empty_tl(), flags_, data_.clone(), date_, signature_.clone()); @@ -75,6 +95,25 @@ td::BufferSlice BroadcastSimple::serialize() { return serialize_tl_object(tl(), true); } +td::Status BroadcastSimple::run_continue() { + TRY_STATUS(distribute()); + deliver(); + return td::Status::OK(); +} + +td::Status BroadcastSimple::run() { + TRY_STATUS(run_checks()); + if (!is_valid_) { + auto P = td::PromiseCreator::lambda( + [id = broadcast_hash_, overlay_id = actor_id(overlay_)](td::Result R) mutable { + td::actor::send_closure(std::move(overlay_id), &OverlayImpl::broadcast_checked, id, std::move(R)); + }); + overlay_->check_broadcast(source_.compute_short_id(), data_.clone(), std::move(P)); + return td::Status::OK(); + } + return run_continue(); +} + td::Status BroadcastSimple::create(OverlayImpl *overlay, tl_object_ptr broadcast) { auto src = PublicKey{broadcast->src_}; auto data_hash = sha256_bits256(broadcast->data_.as_slice()); @@ -86,7 +125,7 @@ td::Status BroadcastSimple::create(OverlayImpl *overlay, tl_object_ptr(broadcast_hash, src, std::move(cert), broadcast->flags_, std::move(broadcast->data_), broadcast->date_, - std::move(broadcast->signature_), overlay); + std::move(broadcast->signature_), false, overlay); TRY_STATUS(B->run()); overlay->register_simple_broadcast(std::move(B)); return td::Status::OK(); @@ -100,7 +139,7 @@ td::Status BroadcastSimple::create_new(td::actor::ActorId overlay, auto date = static_cast(td::Clocks::system()); auto B = std::make_unique(broadcast_hash, PublicKey{}, nullptr, flags, std::move(data), date, - td::BufferSlice{}, nullptr); + td::BufferSlice{}, false, nullptr); auto to_sign = B->to_sign(); auto P = td::PromiseCreator::lambda( diff --git a/overlay/overlay-broadcast.hpp b/overlay/overlay-broadcast.hpp index 665c269d..da29695a 100644 --- a/overlay/overlay-broadcast.hpp +++ b/overlay/overlay-broadcast.hpp @@ -18,9 +18,16 @@ */ #pragma once +#include "adnl/adnl-local-id.h" +#include "adnl/adnl-node-id.hpp" #include "auto/tl/ton_api.h" +#include "common/refcnt.hpp" #include "overlay/overlay.h" +#include "td/actor/PromiseFuture.h" #include "td/utils/List.h" +#include "td/utils/Status.h" +#include "td/utils/buffer.h" +#include "td/utils/common.h" namespace ton { @@ -38,6 +45,7 @@ class BroadcastSimple : public td::ListNode { td::BufferSlice data_; td::uint32 date_; td::BufferSlice signature_; + bool is_valid_{false}; OverlayImpl *overlay_; @@ -52,7 +60,7 @@ class BroadcastSimple : public td::ListNode { public: BroadcastSimple(Overlay::BroadcastHash broadcast_hash, PublicKey source, std::shared_ptr cert, - td::uint32 flags, td::BufferSlice data, td::uint32 date, td::BufferSlice signature, + td::uint32 flags, td::BufferSlice data, td::uint32 date, td::BufferSlice signature, bool is_valid, OverlayImpl *overlay) : broadcast_hash_(broadcast_hash) , source_(std::move(source)) @@ -61,6 +69,7 @@ class BroadcastSimple : public td::ListNode { , data_(std::move(data)) , date_(date) , signature_(std::move(signature)) + , is_valid_(is_valid) , overlay_(overlay) { } @@ -80,17 +89,14 @@ class BroadcastSimple : public td::ListNode { } void deliver(); - td::Status run() { - TRY_STATUS(run_checks()); - TRY_STATUS(distribute()); - deliver(); - return td::Status::OK(); - } + td::Status run(); + td::Status run_continue(); tl_object_ptr tl() const; td::BufferSlice serialize(); void update_overlay(OverlayImpl *overlay); + void broadcast_checked(td::Result R); static td::Status create(OverlayImpl *overlay, tl_object_ptr broadcast); static td::Status create_new(td::actor::ActorId overlay, td::actor::ActorId keyring, diff --git a/overlay/overlay-fec-broadcast.cpp b/overlay/overlay-fec-broadcast.cpp index e368c620..6ab03800 100644 --- a/overlay/overlay-fec-broadcast.cpp +++ b/overlay/overlay-fec-broadcast.cpp @@ -54,7 +54,16 @@ td::Status OverlayFecBroadcastPart::check_duplicate() { } td::Status OverlayFecBroadcastPart::check_source() { - TRY_STATUS(overlay_->check_source_eligible(source_, cert_.get(), broadcast_size_)); + auto r = overlay_->check_source_eligible(source_, cert_.get(), broadcast_size_, true); + if (r == BroadcastCheckResult::Forbidden) { + return td::Status::Error(ErrorCode::error, "broadcast is forbidden"); + } + + // FIXME + if (r == BroadcastCheckResult::NeedCheck) { + return td::Status::Error(ErrorCode::error, "broadcast is forbidden"); + } + if (bcast_) { TRY_STATUS(bcast_->is_eligible_sender(source_)); } diff --git a/overlay/overlay-fec-broadcast.hpp b/overlay/overlay-fec-broadcast.hpp index 6d30d47c..0ce9c2ff 100644 --- a/overlay/overlay-fec-broadcast.hpp +++ b/overlay/overlay-fec-broadcast.hpp @@ -185,6 +185,9 @@ class BroadcastFec : public td::ListNode { } } + void broadcast_checked(td::Result R) { + } + private: bool ready_ = false; @@ -311,4 +314,3 @@ class OverlayFecBroadcastPart : public td::ListNode { } // namespace overlay } // namespace ton - diff --git a/overlay/overlay-manager.cpp b/overlay/overlay-manager.cpp index ae7c3dd5..059ad4e2 100644 --- a/overlay/overlay-manager.cpp +++ b/overlay/overlay-manager.cpp @@ -17,6 +17,7 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "overlay-manager.h" +#include "auto/tl/ton_api.h" #include "overlay.h" #include "adnl/utils.hpp" @@ -268,17 +269,21 @@ void OverlayManager::save_to_db(adnl::AdnlNodeIdShort local_id, OverlayIdShort o db_.set(key, create_serialize_tl_object(std::move(obj))); } -Certificate::Certificate(PublicKey issued_by, td::int32 expire_at, td::uint32 max_size, td::BufferSlice signature) +Certificate::Certificate(PublicKey issued_by, td::int32 expire_at, td::uint32 max_size, td::uint32 flags, + td::BufferSlice signature) : issued_by_(issued_by) , expire_at_(expire_at) , max_size_(max_size) + , flags_(flags) , signature_(td::SharedSlice(signature.as_slice())) { } -Certificate::Certificate(PublicKeyHash issued_by, td::int32 expire_at, td::uint32 max_size, td::BufferSlice signature) +Certificate::Certificate(PublicKeyHash issued_by, td::int32 expire_at, td::uint32 max_size, td::uint32 flags, + td::BufferSlice signature) : issued_by_(issued_by) , expire_at_(expire_at) , max_size_(max_size) + , flags_(flags) , signature_(td::SharedSlice(signature.as_slice())) { } @@ -290,9 +295,19 @@ void Certificate::set_issuer(PublicKey issuer) { issued_by_ = issuer; } +constexpr td::uint32 cert_default_flags(td::uint32 max_size) { + return (max_size > Overlays::max_simple_broadcast_size() ? CertificateFlags::AllowFec : 0) | + CertificateFlags::Trusted; +} + td::BufferSlice Certificate::to_sign(OverlayIdShort overlay_id, PublicKeyHash issued_to) const { - return create_serialize_tl_object(overlay_id.tl(), issued_to.tl(), expire_at_, - max_size_); + if (flags_ == cert_default_flags(max_size_)) { + return create_serialize_tl_object(overlay_id.tl(), issued_to.tl(), expire_at_, + max_size_); + } else { + return create_serialize_tl_object(overlay_id.tl(), issued_to.tl(), expire_at_, + max_size_, flags_); + } } const PublicKeyHash Certificate::issuer_hash() const { @@ -307,32 +322,48 @@ const PublicKey &Certificate::issuer() const { td::Result> Certificate::create(tl_object_ptr cert) { std::shared_ptr res; - ton_api::downcast_call(*cert.get(), td::overloaded([&](ton_api::overlay_emptyCertificate &obj) { res = nullptr; }, - [&](ton_api::overlay_certificate &obj) { - res = std::make_shared( - PublicKey{obj.issued_by_}, obj.expire_at_, - static_cast(obj.max_size_), - std::move(obj.signature_)); - })); + ton_api::downcast_call(*cert.get(), + td::overloaded([&](ton_api::overlay_emptyCertificate &obj) { res = nullptr; }, + [&](ton_api::overlay_certificate &obj) { + res = std::make_shared(PublicKey{obj.issued_by_}, obj.expire_at_, + static_cast(obj.max_size_), + cert_default_flags(obj.max_size_), + std::move(obj.signature_)); + }, + [&](ton_api::overlay_certificateV2 &obj) { + res = std::make_shared(PublicKey{obj.issued_by_}, obj.expire_at_, + static_cast(obj.max_size_), + static_cast(obj.flags_), + std::move(obj.signature_)); + })); return std::move(res); } -td::Status Certificate::check(PublicKeyHash node, OverlayIdShort overlay_id, td::int32 unix_time, - td::uint32 size) const { +BroadcastCheckResult Certificate::check(PublicKeyHash node, OverlayIdShort overlay_id, td::int32 unix_time, + td::uint32 size, bool is_fec) const { if (size > max_size_) { - return td::Status::Error(ErrorCode::protoviolation, "too big broadcast size"); + return BroadcastCheckResult::Forbidden; } if (unix_time > expire_at_) { - return td::Status::Error(ErrorCode::protoviolation, "too old certificate"); + return BroadcastCheckResult::Forbidden; + } + if (is_fec && !(flags_ & CertificateFlags::AllowFec)) { + return BroadcastCheckResult::Forbidden; } - TRY_RESULT(E, issued_by_.get().create_encryptor()); + auto R1 = issued_by_.get().create_encryptor(); + if (R1.is_error()) { + return BroadcastCheckResult::Forbidden; + } + auto E = R1.move_as_ok(); auto B = to_sign(overlay_id, node); - TRY_STATUS(E->check_signature(B.as_slice(), signature_.as_slice())); + if (E->check_signature(B.as_slice(), signature_.as_slice()).is_error()) { + return BroadcastCheckResult::Forbidden; + } - return td::Status::OK(); + return (flags_ & CertificateFlags::Trusted) ? BroadcastCheckResult::Allowed : BroadcastCheckResult::NeedCheck; } tl_object_ptr Certificate::tl() const { diff --git a/overlay/overlay.cpp b/overlay/overlay.cpp index 47019c16..21ec9363 100644 --- a/overlay/overlay.cpp +++ b/overlay/overlay.cpp @@ -391,25 +391,21 @@ td::Status OverlayImpl::check_date(td::uint32 date) { return td::Status::OK(); } -td::Status OverlayImpl::check_source_eligible(PublicKey source, const Certificate *cert, td::uint32 size) { +BroadcastCheckResult OverlayImpl::check_source_eligible(PublicKey source, const Certificate *cert, td::uint32 size, + bool is_fec) { if (size == 0) { - return td::Status::Error(ErrorCode::protoviolation, "empty broadcast"); + return BroadcastCheckResult::Forbidden; } auto short_id = source.compute_short_id(); - auto r = rules_.max_size(source.compute_short_id()); - if (r >= size) { - return td::Status::OK(); + auto r = rules_.check_rules(source.compute_short_id(), size, is_fec); + if (!cert || r == BroadcastCheckResult::Allowed) { + return r; } - if (!cert) { - return td::Status::Error(ErrorCode::protoviolation, "source is not eligible"); - } - TRY_STATUS(cert->check(short_id, overlay_id_, static_cast(td::Clocks::system()), size)); - auto issuer_short = cert->issuer_hash(); - if (rules_.max_size(issuer_short) < size) { - return td::Status::Error(ErrorCode::protoviolation, "bad certificate"); - } - return td::Status::OK(); + + auto r2 = cert->check(short_id, overlay_id_, static_cast(td::Clocks::system()), size, is_fec); + r2 = broadcast_check_result_min(r2, rules_.check_rules(cert->issuer_hash(), size, is_fec)); + return broadcast_check_result_max(r, r2); } td::Status OverlayImpl::check_delivered(BroadcastHash hash) { @@ -539,6 +535,25 @@ void OverlayImpl::set_privacy_rules(OverlayPrivacyRules rules) { rules_ = std::move(rules); } +void OverlayImpl::check_broadcast(PublicKeyHash src, td::BufferSlice data, td::Promise promise) { + callback_->check_broadcast(src, overlay_id_, std::move(data), std::move(promise)); +} + +void OverlayImpl::broadcast_checked(Overlay::BroadcastHash hash, td::Result R) { + { + auto it = broadcasts_.find(hash); + if (it != broadcasts_.end()) { + it->second->broadcast_checked(std::move(R)); + } + } + { + auto it = fec_broadcasts_.find(hash); + if (it != fec_broadcasts_.end()) { + it->second->broadcast_checked(std::move(R)); + } + } +} + } // namespace overlay } // namespace ton diff --git a/overlay/overlay.hpp b/overlay/overlay.hpp index b087fc75..cbddd654 100644 --- a/overlay/overlay.hpp +++ b/overlay/overlay.hpp @@ -144,9 +144,12 @@ class OverlayImpl : public Overlay { void print(td::StringBuilder &sb) override; td::Status check_date(td::uint32 date); - td::Status check_source_eligible(PublicKey source, const Certificate *cert, td::uint32 size); + BroadcastCheckResult check_source_eligible(PublicKey source, const Certificate *cert, td::uint32 size, bool is_fec); td::Status check_delivered(BroadcastHash hash); + void broadcast_checked(Overlay::BroadcastHash hash, td::Result R); + void check_broadcast(PublicKeyHash src, td::BufferSlice data, td::Promise promise); + BroadcastFec *get_fec_broadcast(BroadcastHash hash); void register_fec_broadcast(std::unique_ptr bcast); void register_simple_broadcast(std::unique_ptr bcast); diff --git a/overlay/overlays.h b/overlay/overlays.h index 90f3877e..1ad55453 100644 --- a/overlay/overlays.h +++ b/overlay/overlays.h @@ -21,7 +21,11 @@ #include "adnl/adnl.h" #include "dht/dht.h" +#include "td/actor/PromiseFuture.h" #include "td/actor/actor.h" +#include "td/utils/Status.h" +#include "td/utils/buffer.h" +#include "td/utils/common.h" #include @@ -80,41 +84,64 @@ class OverlayIdFull { td::BufferSlice name_; }; +struct CertificateFlags { + enum Values : td::uint32 { AllowFec = 1, Trusted = 2 }; +}; + +enum BroadcastCheckResult { Forbidden = 1, NeedCheck = 2, Allowed = 3 }; + +inline BroadcastCheckResult broadcast_check_result_max(BroadcastCheckResult l, BroadcastCheckResult r) { + return static_cast(std::max(static_cast(l), static_cast(r))); +} +inline BroadcastCheckResult broadcast_check_result_min(BroadcastCheckResult l, BroadcastCheckResult r) { + return static_cast(std::min(static_cast(l), static_cast(r))); +} + class OverlayPrivacyRules { public: OverlayPrivacyRules() { } OverlayPrivacyRules(td::uint32 size) : max_unath_size_(size) { } - OverlayPrivacyRules(td::uint32 max_size, std::map authorized_keys) - : max_unath_size_(max_size), authorized_keys_(std::move(authorized_keys)) { + OverlayPrivacyRules(td::uint32 max_size, td::uint32 flags, std::map authorized_keys) + : max_unath_size_(max_size), flags_(flags), authorized_keys_(std::move(authorized_keys)) { } - td::uint32 max_size(PublicKeyHash hash) { + BroadcastCheckResult check_rules(PublicKeyHash hash, td::uint32 size, bool is_fec) { auto it = authorized_keys_.find(hash); if (it == authorized_keys_.end()) { - return max_unath_size_; + if (size > max_unath_size_) { + return BroadcastCheckResult::Forbidden; + } + if (!(flags_ & CertificateFlags::AllowFec) && is_fec) { + return BroadcastCheckResult::Forbidden; + } + return (flags_ & CertificateFlags::Trusted) ? BroadcastCheckResult::Allowed : BroadcastCheckResult::NeedCheck; } else { - return it->second; + return it->second >= size ? BroadcastCheckResult::Allowed : BroadcastCheckResult::Forbidden; } } private: td::uint32 max_unath_size_{0}; + td::uint32 flags_{0}; std::map authorized_keys_; }; class Certificate { public: - Certificate(PublicKeyHash issued_by, td::int32 expire_at, td::uint32 max_size, td::BufferSlice signature); - Certificate(PublicKey issued_by, td::int32 expire_at, td::uint32 max_size, td::BufferSlice signature); + Certificate(PublicKeyHash issued_by, td::int32 expire_at, td::uint32 max_size, td::uint32 flags, + td::BufferSlice signature); + Certificate(PublicKey issued_by, td::int32 expire_at, td::uint32 max_size, td::uint32 flags, + td::BufferSlice signature); Certificate() { } void set_signature(td::BufferSlice signature); void set_issuer(PublicKey issuer); td::BufferSlice to_sign(OverlayIdShort overlay_id, PublicKeyHash issued_to) const; - td::Status check(PublicKeyHash node, OverlayIdShort overlay_id, td::int32 unix_time, td::uint32 size) const; + BroadcastCheckResult check(PublicKeyHash node, OverlayIdShort overlay_id, td::int32 unix_time, td::uint32 size, + bool is_fec) const; tl_object_ptr tl() const; const PublicKey &issuer() const; const PublicKeyHash issuer_hash() const; @@ -126,6 +153,7 @@ class Certificate { td::Variant issued_by_; td::int32 expire_at_; td::uint32 max_size_; + td::uint32 flags_; td::SharedSlice signature_; }; @@ -137,6 +165,10 @@ class Overlays : public td::actor::Actor { virtual void receive_query(adnl::AdnlNodeIdShort src, OverlayIdShort overlay_id, td::BufferSlice data, td::Promise promise) = 0; virtual void receive_broadcast(PublicKeyHash src, OverlayIdShort overlay_id, td::BufferSlice data) = 0; + virtual void check_broadcast(PublicKeyHash src, OverlayIdShort overlay_id, td::BufferSlice data, + td::Promise promise) { + promise.set_value(td::Unit()); + } virtual ~Callback() = default; }; diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 57ed9ddc..b226ca7f 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -219,9 +219,11 @@ overlay.broadcastFec.partId broadcast_hash:int256 data_hash:int256 seqno:int = o overlay.broadcast.toSign hash:int256 date:int = overlay.broadcast.ToSign; overlay.certificate issued_by:PublicKey expire_at:int max_size:int signature:bytes = overlay.Certificate; +overlay.certificateV2 issued_by:PublicKey expire_at:int max_size:int flags:int signature:bytes = overlay.Certificate; overlay.emptyCertificate = overlay.Certificate; overlay.certificateId overlay_id:int256 node:int256 expire_at:int max_size:int = overlay.CertificateId; +overlay.certificateIdV2 overlay_id:int256 node:int256 expire_at:int max_size:int flags:int = overlay.CertificateId; overlay.unicast data:bytes = overlay.Broadcast; overlay.broadcast src:PublicKey certificate:overlay.Certificate flags:int data:bytes date:int signature:bytes = overlay.Broadcast; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 823219b5455c649d4cc8f04b6231dcdd439c8c16..32f42b3c4843343c874d1a1a2696af299ab58f1c 100644 GIT binary patch delta 298 zcmZ4SmAU07^9Bt`$=dJLX~OwssYN-7m3q#pMJ1VOnaPPIsmv1{#aZMw9V(t2BPlV_ zL1J@{WCa(a#b#EO_4%wIO$?J6Gu000^VMZXe5KCXq=$|a0 R!8(~YQyFa3=9ZZ%i2&9tb(jDE delta 162 zcmZqq$-Lq#^9Bt`iGSHHmxS}nQj2mDEA^aHi%K%nGLsWaQYSh{u promise) override { + td::actor::send_closure(node_, &FullNodeShardImpl::check_broadcast, src, std::move(data), std::move(promise)); + } Callback(td::actor::ActorId node) : node_(node) { } @@ -95,6 +101,17 @@ void FullNodeShardImpl::create_overlay() { } } +void FullNodeShardImpl::check_broadcast(PublicKeyHash src, td::BufferSlice broadcast, td::Promise promise) { + auto B = fetch_tl_object(std::move(broadcast), true); + if (B.is_error()) { + return promise.set_error(B.move_as_error_prefix("failed to parse external message broadcast: ")); + } + + auto q = B.move_as_ok(); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::check_external_message, + std::move(q->message_->data_), std::move(promise)); +} + void FullNodeShardImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) { td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, adnl_id_, overlay_id_); adnl_id_ = adnl_id; @@ -804,8 +821,9 @@ void FullNodeShardImpl::sign_new_certificate(PublicKeyHash sign_by) { return; } - ton::overlay::Certificate cert{sign_by, static_cast(td::Clocks::system() + 3600), - overlay::Overlays::max_fec_broadcast_size(), td::BufferSlice{}}; + ton::overlay::Certificate cert{ + sign_by, static_cast(td::Clocks::system() + 3600), overlay::Overlays::max_fec_broadcast_size(), + overlay::CertificateFlags::Trusted | overlay::CertificateFlags::AllowFec, td::BufferSlice{}}; auto to_sign = cert.to_sign(overlay_id_, local_id_); auto P = td::PromiseCreator::lambda( @@ -845,7 +863,7 @@ void FullNodeShardImpl::update_validators(std::vector public_key_ authorized_keys.emplace(key, overlay::Overlays::max_fec_broadcast_size()); } - rules_ = overlay::OverlayPrivacyRules{1 << 14, std::move(authorized_keys)}; + rules_ = overlay::OverlayPrivacyRules{1 << 14, 0, std::move(authorized_keys)}; td::actor::send_closure(overlays_, &overlay::Overlays::set_privacy_rules, adnl_id_, overlay_id_, rules_); if (update_cert) { @@ -949,8 +967,7 @@ void FullNodeShardImpl::update_neighbour_stats(adnl::AdnlNodeIdShort adnl_id, do } } -void FullNodeShardImpl::got_neighbour_capabilities(adnl::AdnlNodeIdShort adnl_id, double t, - td::BufferSlice data) { +void FullNodeShardImpl::got_neighbour_capabilities(adnl::AdnlNodeIdShort adnl_id, double t, td::BufferSlice data) { auto it = neighbours_.find(adnl_id); if (it == neighbours_.end()) { return; diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index 4aad637b..2c468374 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -19,6 +19,8 @@ #pragma once #include "full-node-shard.h" +#include "td/actor/PromiseFuture.h" +#include "td/utils/port/Poll.h" namespace ton { @@ -139,6 +141,7 @@ class FullNodeShardImpl : public FullNodeShard { void process_broadcast(PublicKeyHash src, ton_api::tonNode_externalMessageBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query); void receive_broadcast(PublicKeyHash src, td::BufferSlice query); + void check_broadcast(PublicKeyHash src, td::BufferSlice query, td::Promise promise); void send_ihr_message(td::BufferSlice data) override; void send_external_message(td::BufferSlice data) override; diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 71818e6e..fa365ccf 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -124,6 +124,9 @@ class ValidatorManagerImpl : public ValidatorManager { //void get_block_description(BlockIdExt block_id, td::Promise promise) override; void new_external_message(td::BufferSlice data) override; + void check_external_message(td::BufferSlice data, td::Promise promise) override { + UNREACHABLE(); + } void new_ihr_message(td::BufferSlice data) override; void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override; diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 506ee036..9938bcb2 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -144,6 +144,9 @@ class ValidatorManagerImpl : public ValidatorManager { void get_key_block_proof_link(BlockIdExt block_id, td::Promise promise) override; void new_external_message(td::BufferSlice data) override; + void check_external_message(td::BufferSlice data, td::Promise promise) override { + UNREACHABLE(); + } void new_ihr_message(td::BufferSlice data) override; void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override { UNREACHABLE(); diff --git a/validator/manager.hpp b/validator/manager.hpp index 672085d0..08de5060 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -20,6 +20,8 @@ #include "interfaces/validator-manager.h" #include "interfaces/db.h" +#include "td/actor/PromiseFuture.h" +#include "td/utils/port/Poll.h" #include "validator-group.hpp" #include "shard-client.hpp" #include "manager-init.h" @@ -325,6 +327,9 @@ class ValidatorManagerImpl : public ValidatorManager { //void get_block_description(BlockIdExt block_id, td::Promise promise) override; void new_external_message(td::BufferSlice data) override; + void check_external_message(td::BufferSlice data, td::Promise promise) override { + promise.set_value(td::Unit()); + } void new_ihr_message(td::BufferSlice data) override; void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override; diff --git a/validator/validator.h b/validator/validator.h index 54c35ee7..0cbdbe00 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -94,9 +94,8 @@ struct ValidatorManagerOptions : public td::CntObject { BlockIdExt zero_block_id, BlockIdExt init_block_id, std::function check_shard = [](ShardIdFull, CatchainSeqno, ShardCheckMode) { return true; }, - bool allow_blockchain_init = false, double sync_blocks_before = 300, - double block_ttl = 86400 * 7, double state_ttl = 3600, - double archive_ttl = 86400 * 365, double key_proof_ttl = 86400 * 3650, + bool allow_blockchain_init = false, double sync_blocks_before = 300, double block_ttl = 86400 * 7, + double state_ttl = 3600, double archive_ttl = 86400 * 365, double key_proof_ttl = 86400 * 3650, bool initial_sync_disabled = false); }; @@ -176,6 +175,7 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void write_handle(BlockHandle handle, td::Promise promise) = 0; virtual void new_external_message(td::BufferSlice data) = 0; + virtual void check_external_message(td::BufferSlice data, td::Promise promise) = 0; virtual void new_ihr_message(td::BufferSlice data) = 0; virtual void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) = 0; From 69d0472510d44850df14e1cf674f4c55bffc97d5 Mon Sep 17 00:00:00 2001 From: OmicronTau Date: Sat, 13 Nov 2021 17:15:19 +0300 Subject: [PATCH 02/21] Add mempool messages cap --- validator-engine/validator-engine.cpp | 7 +++++++ validator-engine/validator-engine.hpp | 4 ++++ validator/manager.cpp | 3 +++ validator/manager.hpp | 3 +++ validator/validator-options.cpp | 3 ++- validator/validator-options.hpp | 10 +++++++++- validator/validator.h | 3 +++ 7 files changed, 31 insertions(+), 2 deletions(-) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 16f71d5d..9c9d4a98 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1309,6 +1309,9 @@ td::Status ValidatorEngine::load_global_config() { if (state_ttl_ != 0) { validator_options_.write().set_state_ttl(state_ttl_); } + if (max_mempool_num_ != 0) { + validator_options_.write().set_max_mempool_num(max_mempool_num_); + } if (block_ttl_ != 0) { validator_options_.write().set_block_ttl(block_ttl_); } @@ -3336,6 +3339,10 @@ int main(int argc, char *argv[]) { auto v = td::to_double(fname); acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_state_ttl, v); }); }); + p.add_option('m', "mempool-num", "Maximal number of mempool external message", [&](td::Slice fname) { + auto v = td::to_double(fname); + acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_max_mempool_num, v); }); + }); p.add_option('b', "block-ttl", "blocks will be gc'd after this time (in seconds) default=7*86400", [&](td::Slice fname) { auto v = td::to_double(fname); diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 8ce56ab4..07078f95 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -191,6 +191,7 @@ class ValidatorEngine : public td::actor::Actor { std::map control_permissions_; double state_ttl_ = 0; + double max_mempool_num_ = 0; double block_ttl_ = 0; double sync_ttl_ = 0; double archive_ttl_ = 0; @@ -223,6 +224,9 @@ class ValidatorEngine : public td::actor::Actor { void set_state_ttl(double t) { state_ttl_ = t; } + void set_max_mempool_num(double t) { + max_mempool_num_ = t; + } void set_block_ttl(double t) { block_ttl_ = t; } diff --git a/validator/manager.cpp b/validator/manager.cpp index 3abea6e8..1741502d 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -366,6 +366,9 @@ void ValidatorManagerImpl::new_external_message(td::BufferSlice data) { if (!is_validator()) { return; } + if( ext_messages_.size() > max_mempool_num() ) { + return; + } auto R = create_ext_message(std::move(data)); if (R.is_error()) { VLOG(VALIDATOR_NOTICE) << "dropping bad ihr message: " << R.move_as_error(); diff --git a/validator/manager.hpp b/validator/manager.hpp index 08de5060..c2e41a30 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -580,6 +580,9 @@ class ValidatorManagerImpl : public ValidatorManager { double block_ttl() const { return opts_->block_ttl(); } + double max_mempool_num() const { + return opts_->max_mempool_num(); + } private: std::map> shard_client_waiters_; diff --git a/validator/validator-options.cpp b/validator/validator-options.cpp index 30bb050b..93fe05e6 100644 --- a/validator/validator-options.cpp +++ b/validator/validator-options.cpp @@ -27,10 +27,11 @@ namespace validator { td::Ref ValidatorManagerOptions::create( BlockIdExt zero_block_id, BlockIdExt init_block_id, std::function check_shard, bool allow_blockchain_init, - double sync_blocks_before, double block_ttl, double state_ttl, + double sync_blocks_before, double block_ttl, double state_ttl, double max_mempool_num, double archive_ttl, double key_proof_ttl, bool initial_sync_disabled) { return td::make_ref(zero_block_id, init_block_id, std::move(check_shard), allow_blockchain_init, sync_blocks_before, block_ttl, state_ttl, + max_mempool_num, archive_ttl, key_proof_ttl, initial_sync_disabled); } diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index 92597156..e794166f 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -50,6 +50,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { double state_ttl() const override { return state_ttl_; } + double max_mempool_num() const override { + return max_mempool_num_; + } double archive_ttl() const override { return archive_ttl_; } @@ -130,6 +133,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_state_ttl(double value) override { state_ttl_ = value; } + void set_max_mempool_num(double value) override { + max_mempool_num_ = value; + } void set_archive_ttl(double value) override { archive_ttl_ = value; } @@ -163,7 +169,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { ValidatorManagerOptionsImpl(BlockIdExt zero_block_id, BlockIdExt init_block_id, std::function check_shard, bool allow_blockchain_init, double sync_blocks_before, - double block_ttl, double state_ttl, + double block_ttl, double state_ttl, double max_mempool_num, double archive_ttl, double key_proof_ttl, bool initial_sync_disabled) : zero_block_id_(zero_block_id) @@ -173,6 +179,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { , sync_blocks_before_(sync_blocks_before) , block_ttl_(block_ttl) , state_ttl_(state_ttl) + , max_mempool_num_(max_mempool_num) , archive_ttl_(archive_ttl) , key_proof_ttl_(key_proof_ttl) , initial_sync_disabled_(initial_sync_disabled) { @@ -186,6 +193,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { double sync_blocks_before_; double block_ttl_; double state_ttl_; + double max_mempool_num_; double archive_ttl_; double key_proof_ttl_; bool initial_sync_disabled_; diff --git a/validator/validator.h b/validator/validator.h index 0cbdbe00..349824d6 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -56,6 +56,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual double sync_blocks_before() const = 0; virtual double block_ttl() const = 0; virtual double state_ttl() const = 0; + virtual double max_mempool_num() const = 0; virtual double archive_ttl() const = 0; virtual double key_proof_ttl() const = 0; virtual bool initial_sync_disabled() const = 0; @@ -81,6 +82,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual void set_sync_blocks_before(double value) = 0; virtual void set_block_ttl(double value) = 0; virtual void set_state_ttl(double value) = 0; + virtual void set_max_mempool_num(double value) = 0; virtual void set_archive_ttl(double value) = 0; virtual void set_key_proof_ttl(double value) = 0; virtual void set_initial_sync_disabled(bool value) = 0; @@ -96,6 +98,7 @@ struct ValidatorManagerOptions : public td::CntObject { ShardCheckMode) { return true; }, bool allow_blockchain_init = false, double sync_blocks_before = 300, double block_ttl = 86400 * 7, double state_ttl = 3600, double archive_ttl = 86400 * 365, double key_proof_ttl = 86400 * 3650, + double max_mempool_num = 999999, bool initial_sync_disabled = false); }; From 3384d204d2235fec1373722cf6f58c9f4f081b13 Mon Sep 17 00:00:00 2001 From: OmicronTau Date: Sat, 13 Nov 2021 23:28:06 +0300 Subject: [PATCH 03/21] Add checks for external messages --- validator/fabric.h | 4 + validator/impl/collator-impl.h | 21 +++ validator/impl/collator.cpp | 232 ++++++++++++++---------- validator/impl/external-message.cpp | 100 +++++++++- validator/impl/external-message.hpp | 21 ++- validator/impl/fabric.cpp | 9 + validator/impl/liteserver.cpp | 75 ++++++-- validator/impl/liteserver.hpp | 12 ++ validator/interfaces/external-message.h | 2 + validator/manager.cpp | 29 ++- validator/manager.hpp | 10 +- 11 files changed, 396 insertions(+), 119 deletions(-) diff --git a/validator/fabric.h b/validator/fabric.h index a10251dd..67b6ae9e 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -46,6 +46,8 @@ td::Result>> create_new_shard_bloc td::Ref create_signature_set(std::vector sig_set); +void run_check_external_message(td::BufferSlice data, td::actor::ActorId manager, td::Promise promise); + void run_accept_block_query(BlockIdExt id, td::Ref data, std::vector prev, td::Ref validator_set, td::Ref signatures, td::Ref approve_signatures, bool send_broadcast, @@ -81,6 +83,8 @@ void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_b td::Promise promise); void run_liteserver_query(td::BufferSlice data, td::actor::ActorId manager, td::actor::ActorId cache, td::Promise promise); +void run_fetch_account_state(WorkchainId wc, StdSmcAddress addr, td::actor::ActorId manager, + td::Promise,UnixTime,LogicalTime,std::unique_ptr>> promise); void run_validate_shard_block_description(td::BufferSlice data, BlockHandle masterchain_block, td::Ref masterchain_state, td::actor::ActorId manager, td::Timestamp timeout, diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 87aa5494..2dced749 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -103,6 +103,27 @@ class Collator final : public td::actor::Actor { return 2; } + static td::Result> + impl_fetch_config_params(std::unique_ptr config, + Ref* old_mparams, + std::vector* storage_prices, + block::StoragePhaseConfig* storage_phase_cfg, + td::BitArray<256>* rand_seed, + block::ComputePhaseConfig* compute_phase_cfg, + block::ActionPhaseConfig* action_phase_cfg, + td::RefInt256* masterchain_create_fee, + td::RefInt256* basechain_create_fee, + WorkchainId wc); + + static td::Result> + impl_create_ordinary_transaction(Ref msg_root, + block::Account* acc, + UnixTime utime, LogicalTime lt, + block::StoragePhaseConfig* storage_phase_cfg, + block::ComputePhaseConfig* compute_phase_cfg, + block::ActionPhaseConfig* action_phase_cfg, + bool external, LogicalTime after_lt); + private: void start_up() override; void alarm() override; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 9ca4a6cc..6b1c72a6 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -1556,68 +1556,92 @@ bool Collator::init_lt() { } bool Collator::fetch_config_params() { - old_mparams_ = config_->get_config_param(9); - { - auto res = config_->get_storage_prices(); - if (res.is_error()) { + auto res = impl_fetch_config_params(std::move(config_), + &old_mparams_, &storage_prices_, &storage_phase_cfg_, + &rand_seed_, &compute_phase_cfg_, &action_phase_cfg_, + &masterchain_create_fee_, &basechain_create_fee_, + workchain() + ); + if (res.is_error()) { return fatal_error(res.move_as_error()); + } + config_ = res.move_as_ok(); + return true; +} + +td::Result> + Collator::impl_fetch_config_params(std::unique_ptr config, + Ref* old_mparams, + std::vector* storage_prices, + block::StoragePhaseConfig* storage_phase_cfg, + td::BitArray<256>* rand_seed, + block::ComputePhaseConfig* compute_phase_cfg, + block::ActionPhaseConfig* action_phase_cfg, + td::RefInt256* masterchain_create_fee, + td::RefInt256* basechain_create_fee, + WorkchainId wc) { + *old_mparams = config->get_config_param(9); + { + auto res = config->get_storage_prices(); + if (res.is_error()) { + return res.move_as_error(); } - storage_prices_ = res.move_as_ok(); + *storage_prices = res.move_as_ok(); } { // generate rand seed - prng::rand_gen().strong_rand_bytes(rand_seed_.data(), 32); - LOG(DEBUG) << "block random seed set to " << rand_seed_.to_hex(); + prng::rand_gen().strong_rand_bytes(rand_seed->data(), 32); + LOG(DEBUG) << "block random seed set to " << rand_seed->to_hex(); } { // compute compute_phase_cfg / storage_phase_cfg - auto cell = config_->get_config_param(is_masterchain() ? 20 : 21); + auto cell = config->get_config_param(wc == ton::masterchainId ? 20 : 21); if (cell.is_null()) { - return fatal_error("cannot fetch current gas prices and limits from masterchain configuration"); + return td::Status::Error(-668, "cannot fetch current gas prices and limits from masterchain configuration"); } - if (!compute_phase_cfg_.parse_GasLimitsPrices(std::move(cell), storage_phase_cfg_.freeze_due_limit, - storage_phase_cfg_.delete_due_limit)) { - return fatal_error("cannot unpack current gas prices and limits from masterchain configuration"); + if (!compute_phase_cfg->parse_GasLimitsPrices(std::move(cell), storage_phase_cfg->freeze_due_limit, + storage_phase_cfg->delete_due_limit)) { + return td::Status::Error(-668, "cannot unpack current gas prices and limits from masterchain configuration"); } - compute_phase_cfg_.block_rand_seed = rand_seed_; - compute_phase_cfg_.libraries = std::make_unique(config_->get_libraries_root(), 256); - compute_phase_cfg_.global_config = config_->get_root_cell(); + compute_phase_cfg->block_rand_seed = *rand_seed; + compute_phase_cfg->libraries = std::make_unique(config->get_libraries_root(), 256); + compute_phase_cfg->global_config = config->get_root_cell(); } { // compute action_phase_cfg block::gen::MsgForwardPrices::Record rec; - auto cell = config_->get_config_param(24); + auto cell = config->get_config_param(24); if (cell.is_null() || !tlb::unpack_cell(std::move(cell), rec)) { - return fatal_error("cannot fetch masterchain message transfer prices from masterchain configuration"); + return td::Status::Error(-668, "cannot fetch masterchain message transfer prices from masterchain configuration"); } - action_phase_cfg_.fwd_mc = + action_phase_cfg->fwd_mc = block::MsgPrices{rec.lump_price, rec.bit_price, rec.cell_price, rec.ihr_price_factor, (unsigned)rec.first_frac, (unsigned)rec.next_frac}; - cell = config_->get_config_param(25); + cell = config->get_config_param(25); if (cell.is_null() || !tlb::unpack_cell(std::move(cell), rec)) { - return fatal_error("cannot fetch standard message transfer prices from masterchain configuration"); + return td::Status::Error(-668, "cannot fetch standard message transfer prices from masterchain configuration"); } - action_phase_cfg_.fwd_std = + action_phase_cfg->fwd_std = block::MsgPrices{rec.lump_price, rec.bit_price, rec.cell_price, rec.ihr_price_factor, (unsigned)rec.first_frac, (unsigned)rec.next_frac}; - action_phase_cfg_.workchains = &config_->get_workchain_list(); - action_phase_cfg_.bounce_msg_body = (config_->has_capability(ton::capBounceMsgBody) ? 256 : 0); + action_phase_cfg->workchains = &config->get_workchain_list(); + action_phase_cfg->bounce_msg_body = (config->has_capability(ton::capBounceMsgBody) ? 256 : 0); } { // fetch block_grams_created - auto cell = config_->get_config_param(14); + auto cell = config->get_config_param(14); if (cell.is_null()) { - basechain_create_fee_ = masterchain_create_fee_ = td::zero_refint(); + *basechain_create_fee = *masterchain_create_fee = td::zero_refint(); } else { block::gen::BlockCreateFees::Record create_fees; if (!(tlb::unpack_cell(cell, create_fees) && - block::tlb::t_Grams.as_integer_to(create_fees.masterchain_block_fee, masterchain_create_fee_) && - block::tlb::t_Grams.as_integer_to(create_fees.basechain_block_fee, basechain_create_fee_))) { - return fatal_error("cannot unpack BlockCreateFees from configuration parameter #14"); + block::tlb::t_Grams.as_integer_to(create_fees.masterchain_block_fee, *masterchain_create_fee) && + block::tlb::t_Grams.as_integer_to(create_fees.basechain_block_fee, *basechain_create_fee))) { + return td::Status::Error(-668, "cannot unpack BlockCreateFees from configuration parameter #14"); } } } - return true; + return std::move(config); } bool Collator::compute_minted_amount(block::CurrencyCollection& to_mint) { @@ -2218,75 +2242,25 @@ Ref Collator::create_ordinary_transaction(Ref msg_root) { } block::Account* acc = acc_res.move_as_ok(); assert(acc); - if (acc->last_trans_end_lt_ >= start_lt && acc->transactions.empty()) { - fatal_error(PSTRING() << "last transaction time in the state of account " << workchain() << ":" << addr.to_hex() - << " is too large"); - return {}; - } - auto trans_min_lt = start_lt; - if (external) { - // transactions processing external messages must have lt larger than all processed internal messages - trans_min_lt = std::max(trans_min_lt, last_proc_int_msg_.first); - } - std::unique_ptr trans = - std::make_unique(*acc, block::Transaction::tr_ord, trans_min_lt + 1, now_, msg_root); - bool ihr_delivered = false; // FIXME - if (!trans->unpack_input_msg(ihr_delivered, &action_phase_cfg_)) { - if (external) { - // inbound external message was not accepted - LOG(DEBUG) << "inbound external message rejected by account " << addr.to_hex() - << " before smart-contract execution"; + + + auto res = impl_create_ordinary_transaction(msg_root, acc, now_, start_lt, + &storage_phase_cfg_, &compute_phase_cfg_, + &action_phase_cfg_, + external, last_proc_int_msg_.first + ); + if(res.is_error()) { + auto error = res.move_as_error(); + if(error.code() == -701) { + // ignorable errors + LOG(DEBUG) << error.message(); return {}; } - fatal_error("cannot unpack input message for a new transaction"); - return {}; - } - if (trans->bounce_enabled) { - if (!trans->prepare_storage_phase(storage_phase_cfg_, true)) { - fatal_error("cannot create storage phase of a new transaction for smart contract "s + addr.to_hex()); - return {}; - } - if (!external && !trans->prepare_credit_phase()) { - fatal_error("cannot create credit phase of a new transaction for smart contract "s + addr.to_hex()); - return {}; - } - } else { - if (!external && !trans->prepare_credit_phase()) { - fatal_error("cannot create credit phase of a new transaction for smart contract "s + addr.to_hex()); - return {}; - } - if (!trans->prepare_storage_phase(storage_phase_cfg_, true, true)) { - fatal_error("cannot create storage phase of a new transaction for smart contract "s + addr.to_hex()); - return {}; - } - } - if (!trans->prepare_compute_phase(compute_phase_cfg_)) { - fatal_error("cannot create compute phase of a new transaction for smart contract "s + addr.to_hex()); - return {}; - } - if (!trans->compute_phase->accepted) { - if (external) { - // inbound external message was not accepted - LOG(DEBUG) << "inbound external message rejected by transaction " << addr.to_hex(); - return {}; - } else if (trans->compute_phase->skip_reason == block::ComputePhase::sk_none) { - fatal_error("new ordinary transaction for smart contract "s + addr.to_hex() + - " has not been accepted by the smart contract (?)"); - return {}; - } - } - if (trans->compute_phase->success && !trans->prepare_action_phase(action_phase_cfg_)) { - fatal_error("cannot create action phase of a new transaction for smart contract "s + addr.to_hex()); - return {}; - } - if (trans->bounce_enabled && !trans->compute_phase->success && !trans->prepare_bounce_phase(action_phase_cfg_)) { - fatal_error("cannot create bounce phase of a new transaction for smart contract "s + addr.to_hex()); - return {}; - } - if (!trans->serialize()) { - fatal_error("cannot serialize new transaction for smart contract "s + addr.to_hex()); + fatal_error(std::move(error)); return {}; } + std::unique_ptr trans = res.move_as_ok(); + if (!trans->update_limits(*block_limit_status_)) { fatal_error("cannot update block limit status to include the new transaction"); return {}; @@ -2296,6 +2270,7 @@ Ref Collator::create_ordinary_transaction(Ref msg_root) { fatal_error("cannot commit new transaction for smart contract "s + addr.to_hex()); return {}; } + register_new_msgs(*trans); update_max_lt(acc->last_trans_end_lt_); // temporary patch to stop producing dangerous block @@ -2305,6 +2280,75 @@ Ref Collator::create_ordinary_transaction(Ref msg_root) { return trans_root; } +// If td::status::error_code == 669 - Fatal Error block can not be produced +// if td::status::error_code == 701 - Transaction can not be included into block, but it's ok (external or too early internal) +td::Result> Collator::impl_create_ordinary_transaction(Ref msg_root, + block::Account* acc, + UnixTime utime, LogicalTime lt, + block::StoragePhaseConfig* storage_phase_cfg, + block::ComputePhaseConfig* compute_phase_cfg, + block::ActionPhaseConfig* action_phase_cfg, + bool external, LogicalTime after_lt) { + if (acc->last_trans_end_lt_ >= lt && acc->transactions.empty()) { + return td::Status::Error(-669, PSTRING() << "last transaction time in the state of account " << acc->workchain << ":" << acc->addr.to_hex() + << " is too large"); + } + auto trans_min_lt = lt; + if (external) { + // transactions processing external messages must have lt larger than all processed internal messages + trans_min_lt = std::max(trans_min_lt, after_lt); + } + + std::unique_ptr trans = + std::make_unique(*acc, block::Transaction::tr_ord, trans_min_lt + 1, utime, msg_root); + bool ihr_delivered = false; // FIXME + if (!trans->unpack_input_msg(ihr_delivered, action_phase_cfg)) { + if (external) { + // inbound external message was not accepted + return td::Status::Error(-701,"inbound external message rejected by account "s + acc->addr.to_hex() + + " before smart-contract execution"); + } + return td::Status::Error(-669,"cannot unpack input message for a new transaction"); + } + if (trans->bounce_enabled) { + if (!trans->prepare_storage_phase(*storage_phase_cfg, true)) { + return td::Status::Error(-669,"cannot create storage phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } + if (!external && !trans->prepare_credit_phase()) { + return td::Status::Error(-669,"cannot create credit phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } + } else { + if (!external && !trans->prepare_credit_phase()) { + return td::Status::Error(-669,"cannot create credit phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } + if (!trans->prepare_storage_phase(*storage_phase_cfg, true, true)) { + return td::Status::Error(-669,"cannot create storage phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } + } + if (!trans->prepare_compute_phase(*compute_phase_cfg)) { + return td::Status::Error(-669,"cannot create compute phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } + if (!trans->compute_phase->accepted) { + if (external) { + // inbound external message was not accepted + return td::Status::Error(-701,"inbound external message rejected by transaction "s + acc->addr.to_hex()); + } else if (trans->compute_phase->skip_reason == block::ComputePhase::sk_none) { + return td::Status::Error(-669,"new ordinary transaction for smart contract "s + acc->addr.to_hex() + + " has not been accepted by the smart contract (?)"); + } + } + if (trans->compute_phase->success && !trans->prepare_action_phase(*action_phase_cfg)) { + return td::Status::Error(-669,"cannot create action phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } + if (trans->bounce_enabled && !trans->compute_phase->success && !trans->prepare_bounce_phase(*action_phase_cfg)) { + return td::Status::Error(-669,"cannot create bounce phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } + if (!trans->serialize()) { + return td::Status::Error(-669,"cannot serialize new transaction for smart contract "s + acc->addr.to_hex()); + } + return std::move(trans); +} + void Collator::update_max_lt(ton::LogicalTime lt) { CHECK(lt >= start_lt); if (lt > max_lt) { diff --git a/validator/impl/external-message.cpp b/validator/impl/external-message.cpp index 6af721b2..c5f2d097 100644 --- a/validator/impl/external-message.cpp +++ b/validator/impl/external-message.cpp @@ -16,19 +16,25 @@ Copyright 2017-2020 Telegram Systems LLP */ + #include "external-message.hpp" +#include "collator-impl.h" #include "vm/boc.h" #include "block/block-parse.h" #include "block/block-auto.h" #include "block/block-db.h" +#include "fabric.h" +#include "td/actor/actor.h" +#include "td/utils/Random.h" +#include "crypto/openssl/rand.hpp" namespace ton { namespace validator { using td::Ref; -ExtMessageQ::ExtMessageQ(td::BufferSlice data, td::Ref root, AccountIdPrefixFull addr_prefix) - : root_(std::move(root)), addr_prefix_(addr_prefix), data_(std::move(data)) { +ExtMessageQ::ExtMessageQ(td::BufferSlice data, td::Ref root, AccountIdPrefixFull addr_prefix, ton::WorkchainId wc, ton::StdSmcAddress addr) + : root_(std::move(root)), addr_prefix_(addr_prefix), data_(std::move(data)), wc_(wc), addr_(addr) { hash_ = block::compute_file_hash(data_); } @@ -70,7 +76,95 @@ td::Result> ExtMessageQ::create_ext_message(td::BufferSlice dat if (!dest_prefix.is_valid()) { return td::Status::Error("destination of an inbound external message is an invalid blockchain address"); } - return Ref{true, std::move(data), std::move(ext_msg), dest_prefix}; + ton::StdSmcAddress addr; + ton::WorkchainId wc; + if(!block::tlb::t_MsgAddressInt.extract_std_address(info.dest, wc, addr)) { + return td::Status::Error(PSLICE() << "Can't parse destination address"); + } + + return Ref{true, std::move(data), std::move(ext_msg), dest_prefix, wc, addr}; +} + +void ExtMessageQ::run_message(td::BufferSlice data, td::actor::ActorId manager, + td::Promise promise) { + auto R = create_ext_message(std::move(data)); + if (R.is_error()) { + return promise.set_error(R.move_as_error_prefix("failed to parse external message ")); + } + auto M = R.move_as_ok(); + auto root = M->root_cell(); + block::gen::CommonMsgInfo::Record_ext_in_msg_info info; + tlb::unpack_cell_inexact(root, info); // checked in create message + ton::StdSmcAddress addr = M->addr(); + ton::WorkchainId wc = M->wc(); + + run_fetch_account_state(wc, addr, manager, + [promise = std::move(promise), msg_root = root, wc = wc](td::Result,UnixTime,LogicalTime,std::unique_ptr>> res) mutable { + if (res.is_error()) { + promise.set_error(td::Status::Error(PSLICE() << "Failed to get account state")); + } else { + auto tuple = res.move_as_ok(); + block::Account acc; + auto shard_acc = std::move(std::get<0>(tuple)); + auto utime = std::get<1>(tuple); + auto lt = std::get<2>(tuple); + auto config = std::move(std::get<3>(tuple)); + if(!acc.unpack(shard_acc, {}, utime, false)) { + promise.set_error(td::Status::Error(PSLICE() << "Failed to unpack account state")); + } else { + if(run_message_on_account(wc, &acc, utime, lt + 1, msg_root, std::move(config))) { + promise.set_value(td::Unit()); + } else { + promise.set_error(td::Status::Error(PSLICE() << "External message was not accepted")); + } + } + } + } + ); +} + +bool ExtMessageQ::run_message_on_account(ton::WorkchainId wc, + block::Account* acc, + UnixTime utime, LogicalTime lt, + td::Ref msg_root, + std::unique_ptr config) { + + Ref old_mparams; + std::vector storage_prices_; + block::StoragePhaseConfig storage_phase_cfg_{&storage_prices_}; + td::BitArray<256> rand_seed_; + block::ComputePhaseConfig compute_phase_cfg_; + block::ActionPhaseConfig action_phase_cfg_; + td::RefInt256 masterchain_create_fee, basechain_create_fee; + + auto fetch_res = Collator::impl_fetch_config_params(std::move(config), &old_mparams, + &storage_prices_, &storage_phase_cfg_, + &rand_seed_, &compute_phase_cfg_, + &action_phase_cfg_, &masterchain_create_fee, + &basechain_create_fee, wc); + if(fetch_res.is_error()) { + auto error = fetch_res.move_as_error(); + LOG(DEBUG) << "Cannot fetch config params" << error.message(); + return false; + } + + auto res = Collator::impl_create_ordinary_transaction(msg_root, acc, utime, lt, + &storage_phase_cfg_, &compute_phase_cfg_, + &action_phase_cfg_, + true, lt); + if(res.is_error()) { + auto error = res.move_as_error(); + LOG(DEBUG) << "Cannot run message on account" << error.message(); + return false; + } + std::unique_ptr trans = res.move_as_ok(); + + auto trans_root = trans->commit(*acc); + if (trans_root.is_null()) { + LOG(DEBUG) << "cannot commit new transaction for smart contract "; + return false; + } + return true; } } // namespace validator diff --git a/validator/impl/external-message.hpp b/validator/impl/external-message.hpp index 5bb52b11..230258cd 100644 --- a/validator/impl/external-message.hpp +++ b/validator/impl/external-message.hpp @@ -18,9 +18,11 @@ */ #pragma once +#include "interfaces/validator-manager.h" #include "validator/interfaces/external-message.h" #include "auto/tl/ton_api.h" #include "adnl/utils.hpp" +#include "block/transaction.h" namespace ton { @@ -31,6 +33,8 @@ class ExtMessageQ : public ExtMessage { AccountIdPrefixFull addr_prefix_; td::BufferSlice data_; Hash hash_; + ton::WorkchainId wc_; + ton::StdSmcAddress addr_; public: static constexpr unsigned max_ext_msg_size = 65535; @@ -47,8 +51,23 @@ class ExtMessageQ : public ExtMessage { Hash hash() const override { return hash_; } - ExtMessageQ(td::BufferSlice data, td::Ref root, AccountIdPrefixFull shard); + ton::WorkchainId wc() const override { + return wc_; + } + + ton::StdSmcAddress addr() const override { + return addr_; + } + + ExtMessageQ(td::BufferSlice data, td::Ref root, AccountIdPrefixFull shard, ton::WorkchainId wc, ton::StdSmcAddress addr); static td::Result> create_ext_message(td::BufferSlice data); + static void run_message(td::BufferSlice data, td::actor::ActorId manager, + td::Promise promise); + static bool run_message_on_account(ton::WorkchainId wc, + block::Account* acc, + UnixTime utime, LogicalTime lt, + td::Ref msg_root, + std::unique_ptr config); }; } // namespace validator diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index ac296eb4..196595ae 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -116,6 +116,10 @@ td::Result> create_ext_message(td::BufferSlice data) { return std::move(res); } +void run_check_external_message(td::BufferSlice data, td::actor::ActorId manager, td::Promise promise) { + ExtMessageQ::run_message(std::move(data), std::move(manager), std::move(promise)); +} + td::Result> create_ihr_message(td::BufferSlice data) { TRY_RESULT(res, IhrMessageQ::create_ihr_message(std::move(data))); return std::move(res); @@ -237,6 +241,11 @@ void run_liteserver_query(td::BufferSlice data, td::actor::ActorId manager, + td::Promise,UnixTime,LogicalTime,std::unique_ptr>> promise) { + LiteQuery::fetch_account_state(wc, addr, std::move(manager), std::move(promise)); +} + void run_validate_shard_block_description(td::BufferSlice data, BlockHandle masterchain_block, td::Ref masterchain_state, td::actor::ActorId manager, td::Timestamp timeout, diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index b3c0f3eb..510f4906 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -58,15 +58,28 @@ void LiteQuery::run_query(td::BufferSlice data, td::actor::ActorId("litequery", std::move(data), std::move(manager), std::move(promise)).release(); } +void LiteQuery::fetch_account_state(WorkchainId wc, StdSmcAddress acc_addr, td::actor::ActorId manager, + td::Promise,UnixTime,LogicalTime,std::unique_ptr>> promise) { + td::actor::create_actor("litequery", wc, acc_addr, std::move(manager), std::move(promise)).release(); +} + LiteQuery::LiteQuery(td::BufferSlice data, td::actor::ActorId manager, td::Promise promise) : query_(std::move(data)), manager_(std::move(manager)), promise_(std::move(promise)) { timeout_ = td::Timestamp::in(default_timeout_msec * 0.001); } +LiteQuery::LiteQuery(WorkchainId wc, StdSmcAddress acc_addr, td::actor::ActorId manager, + td::Promise,UnixTime,LogicalTime,std::unique_ptr>> promise) + : manager_(std::move(manager)), acc_state_promise_(std::move(promise)), acc_workchain_(wc), acc_addr_(acc_addr) { + timeout_ = td::Timestamp::in(default_timeout_msec * 0.001); +} + void LiteQuery::abort_query(td::Status reason) { LOG(INFO) << "aborted liteserver query: " << reason.to_string(); - if (promise_) { + if (acc_state_promise_) { + acc_state_promise_.set_error(std::move(reason)); + } else if (promise_) { promise_.set_error(std::move(reason)); } stop(); @@ -111,6 +124,11 @@ bool LiteQuery::finish_query(td::BufferSlice result) { void LiteQuery::start_up() { alarm_timestamp() = timeout_; + if(acc_state_promise_) { + td::actor::send_closure_later(actor_id(this),&LiteQuery::perform_fetchAccountState); + return; + } + auto F = fetch_tl_object(std::move(query_), true); if (F.is_error()) { abort_query(F.move_as_error()); @@ -205,17 +223,23 @@ void LiteQuery::perform_getMasterchainInfo(int mode) { } td::actor::send_closure_later( manager_, &ton::validator::ValidatorManager::get_top_masterchain_state_block, - [Self = actor_id(this), mode](td::Result, BlockIdExt>> res) { + [Self = actor_id(this), return_state = bool(acc_state_promise_), mode](td::Result, BlockIdExt>> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { auto pair = res.move_as_ok(); - td::actor::send_closure_later(Self, &LiteQuery::continue_getMasterchainInfo, std::move(pair.first), + auto func = return_state ? &LiteQuery::gotMasterchainInfoForAccountState : &LiteQuery::continue_getMasterchainInfo; + td::actor::send_closure_later(Self, func, std::move(pair.first), pair.second, mode); } }); } +void LiteQuery::gotMasterchainInfoForAccountState(Ref mc_state, BlockIdExt blkid, + int mode) { + perform_getAccountState(blkid, acc_workchain_, acc_addr_, 0x80000000); +} + void LiteQuery::continue_getMasterchainInfo(Ref mc_state, BlockIdExt blkid, int mode) { LOG(INFO) << "obtained data for getMasterchainInfo() : last block = " << blkid.to_str(); @@ -431,15 +455,25 @@ void LiteQuery::continue_getZeroState(BlockIdExt blkid, td::BufferSlice state) { void LiteQuery::perform_sendMessage(td::BufferSlice data) { LOG(INFO) << "started a sendMessage(<" << data.size() << " bytes>) liteserver query"; - auto res = ton::validator::create_ext_message(std::move(data)); - if (res.is_error()) { - abort_query(res.move_as_error()); - return; - } - LOG(INFO) << "sending an external message to validator manager"; - td::actor::send_closure_later(manager_, &ValidatorManager::send_external_message, res.move_as_ok()); - auto b = ton::create_serialize_tl_object(1); - finish_query(std::move(b)); + td::actor::send_closure_later( + manager_, &ValidatorManager::check_external_message, data.clone(), + [Self = actor_id(this), data = std::move(data), manager = manager_](td::Result res) { + if(res.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, + res.move_as_error_prefix("cannot apply external message to current state : "s)); + } else { + auto crm = ton::validator::create_ext_message(data.clone()); + if (crm.is_error()) { + //UNREACHABLE, checks in check_external_message, + td::actor::send_closure(Self, &LiteQuery::abort_query, + crm.move_as_error()); + } + LOG(INFO) << "sending an external message to validator manager"; + td::actor::send_closure_later(manager, &ValidatorManager::send_external_message, crm.move_as_ok()); + auto b = ton::create_serialize_tl_object(1); + td::actor::send_closure(Self, &LiteQuery::finish_query, std::move(b)); + } + }); } bool LiteQuery::request_mc_block_data(BlockIdExt blkid) { @@ -702,6 +736,10 @@ void LiteQuery::continue_getAccountState_0(Ref request_mc_block_data(blkid); } +void LiteQuery::perform_fetchAccountState() { + perform_getMasterchainInfo(-1); +} + void LiteQuery::perform_runSmcMethod(BlockIdExt blkid, WorkchainId workchain, StdSmcAddress addr, int mode, td::int64 method_id, td::BufferSlice params) { LOG(INFO) << "started a runSmcMethod(" << blkid.to_str() << ", " << workchain << ", " << addr.to_hex() << ", " @@ -1010,6 +1048,19 @@ void LiteQuery::finish_getAccountState(td::BufferSlice shard_proof) { } vm::AugmentedDictionary accounts_dict{vm::load_cell_slice_ref(sstate.accounts), 256, block::tlb::aug_ShardAccounts}; auto acc_csr = accounts_dict.lookup(acc_addr_); + if (mode_ & 0x80000000) { + auto config = block::ConfigInfo::extract_config(mc_state_->root_cell(), 0xFFFF); + if (config.is_error()) { + fatal_error(config.move_as_error()); + return; + } + auto rconfig = config.move_as_ok(); + acc_state_promise_.set_value(std::make_tuple( + std::move(acc_csr), sstate.gen_utime, sstate.gen_lt, std::move(rconfig) + )); + return; + } + Ref acc_root; if (acc_csr.not_null()) { acc_root = acc_csr->prefetch_ref(); diff --git a/validator/impl/liteserver.hpp b/validator/impl/liteserver.hpp index f6353c41..24392085 100644 --- a/validator/impl/liteserver.hpp +++ b/validator/impl/liteserver.hpp @@ -26,6 +26,8 @@ #include "block.hpp" #include "shard.hpp" #include "proof.hpp" +#include "block/block-auto.h" + namespace ton { @@ -37,6 +39,9 @@ class LiteQuery : public td::actor::Actor { td::actor::ActorId manager_; td::Timestamp timeout_; td::Promise promise_; + + td::Promise,UnixTime,LogicalTime,std::unique_ptr>> acc_state_promise_; + int pending_{0}; int mode_{0}; WorkchainId acc_workchain_; @@ -71,9 +76,14 @@ class LiteQuery : public td::actor::Actor { }; // version 1.1; +1 = build block proof chains, +2 = masterchainInfoExt, +4 = runSmcMethod LiteQuery(td::BufferSlice data, td::actor::ActorId manager, td::Promise promise); + LiteQuery(WorkchainId wc, StdSmcAddress acc_addr, td::actor::ActorId manager, + td::Promise,UnixTime,LogicalTime,std::unique_ptr>> promise); static void run_query(td::BufferSlice data, td::actor::ActorId manager, td::Promise promise); + static void fetch_account_state(WorkchainId wc, StdSmcAddress acc_addr, td::actor::ActorId manager, + td::Promise,UnixTime,LogicalTime,std::unique_ptr>> promise); + private: bool fatal_error(td::Status error); bool fatal_error(std::string err_msg, int err_code = -400); @@ -87,6 +97,7 @@ class LiteQuery : public td::actor::Actor { void perform_getVersion(); void perform_getMasterchainInfo(int mode); void continue_getMasterchainInfo(Ref mc_state, BlockIdExt blkid, int mode); + void gotMasterchainInfoForAccountState(Ref mc_state, BlockIdExt blkid, int mode); void perform_getBlock(BlockIdExt blkid); void continue_getBlock(BlockIdExt blkid, Ref block); void perform_getBlockHeader(BlockIdExt blkid, int mode); @@ -99,6 +110,7 @@ class LiteQuery : public td::actor::Actor { void continue_getAccountState_0(Ref mc_state, BlockIdExt blkid); void continue_getAccountState(); void finish_getAccountState(td::BufferSlice shard_proof); + void perform_fetchAccountState(); void perform_runSmcMethod(BlockIdExt blkid, WorkchainId workchain, StdSmcAddress addr, int mode, td::int64 method_id, td::BufferSlice params); void finish_runSmcMethod(td::BufferSlice shard_proof, td::BufferSlice state_proof, Ref acc_root, diff --git a/validator/interfaces/external-message.h b/validator/interfaces/external-message.h index f6d9d600..871c624a 100644 --- a/validator/interfaces/external-message.h +++ b/validator/interfaces/external-message.h @@ -35,6 +35,8 @@ class ExtMessage : public td::CntObject { virtual td::BufferSlice serialize() const = 0; virtual td::Ref root_cell() const = 0; virtual Hash hash() const = 0; + virtual ton::WorkchainId wc() const = 0; + virtual ton::StdSmcAddress addr() const = 0; }; } // namespace validator diff --git a/validator/manager.cpp b/validator/manager.cpp index 1741502d..7fcf0274 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -374,13 +374,25 @@ void ValidatorManagerImpl::new_external_message(td::BufferSlice data) { VLOG(VALIDATOR_NOTICE) << "dropping bad ihr message: " << R.move_as_error(); return; } - auto M = std::make_unique>(R.move_as_ok()); - auto id = M->ext_id(); - if (ext_messages_hashes_.count(id.hash) == 0) { - ext_messages_.emplace(id, std::move(M)); - ext_messages_hashes_.emplace(id.hash, id); + add_external_message(R.move_as_ok()); +} + +void ValidatorManagerImpl::add_external_message(td::Ref msg) { + auto message = std::make_unique>(msg); + auto id = message->ext_id(); + auto address = message->address(); + unsigned long per_address_limit = 256; + if(ext_addr_messages_.count(address) < per_address_limit) { + if (ext_messages_hashes_.count(id.hash) == 0) { + ext_messages_.emplace(id, std::move(message)); + ext_messages_hashes_.emplace(id.hash, id); + ext_addr_messages_[address].emplace(id.hash, id); + } } } +void ValidatorManagerImpl::check_external_message(td::BufferSlice data, td::Promise promise) { + run_check_external_message(std::move(data), actor_id(this), std::move(promise)); +} void ValidatorManagerImpl::new_ihr_message(td::BufferSlice data) { if (!is_validator()) { @@ -759,6 +771,7 @@ void ValidatorManagerImpl::get_external_messages(ShardIdFull shard, break; } if (it->second->expired()) { + ext_addr_messages_[it->second->address()].erase(it->first.hash); ext_messages_hashes_.erase(it->first.hash); it = ext_messages_.erase(it); continue; @@ -807,17 +820,20 @@ void ValidatorManagerImpl::complete_external_messages(std::vectorsecond]->address()].erase(it->first); CHECK(ext_messages_.erase(it->second)); ext_messages_hashes_.erase(it); } } + unsigned long soft_mempool_limit = 1024; for (auto &hash : to_delay) { auto it = ext_messages_hashes_.find(hash); if (it != ext_messages_hashes_.end()) { auto it2 = ext_messages_.find(it->second); - if (it2->second->can_postpone()) { + if ((ext_messages_.size() < soft_mempool_limit) && it2->second->can_postpone()) { it2->second->postpone(); } else { + ext_addr_messages_[it2->second->address()].erase(it2->first.hash); ext_messages_.erase(it2); ext_messages_hashes_.erase(it); } @@ -1352,6 +1368,7 @@ void ValidatorManagerImpl::send_get_next_key_blocks_request(BlockIdExt block_id, void ValidatorManagerImpl::send_external_message(td::Ref message) { callback_->send_ext_message(message->shard(), message->serialize()); + add_external_message(std::move(message)); } void ValidatorManagerImpl::send_ihr_message(td::Ref message) { diff --git a/validator/manager.hpp b/validator/manager.hpp index c2e41a30..19ee4cdd 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -74,6 +74,9 @@ class MessageExt { auto hash() const { return message_->hash(); } + auto address() const { + return std::make_pair(message_->wc(), message_->addr()); + } bool is_active() { if (!active_) { if (reactivate_at_.is_in_past()) { @@ -210,6 +213,7 @@ class ValidatorManagerImpl : public ValidatorManager { // DATA FOR COLLATOR std::map> shard_blocks_; std::map, std::unique_ptr>> ext_messages_; + std::map, std::map>> ext_addr_messages_; std::map> ext_messages_hashes_; // IHR ? std::map, std::unique_ptr>> ihr_messages_; @@ -327,9 +331,9 @@ class ValidatorManagerImpl : public ValidatorManager { //void get_block_description(BlockIdExt block_id, td::Promise promise) override; void new_external_message(td::BufferSlice data) override; - void check_external_message(td::BufferSlice data, td::Promise promise) override { - promise.set_value(td::Unit()); - } + void add_external_message(td::Ref message); + void check_external_message(td::BufferSlice data, td::Promise promise) override; + void new_ihr_message(td::BufferSlice data) override; void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override; From cb31a20206a7aea41bb9dadbb5f3ccbd8820d4e5 Mon Sep 17 00:00:00 2001 From: OmicronTau Date: Tue, 30 Nov 2021 15:11:14 +0300 Subject: [PATCH 04/21] Add methods to sign and import certificates --- overlay/overlay-manager.cpp | 51 +++++ overlay/overlay-manager.h | 1 + overlay/overlay.cpp | 15 ++ overlay/overlay.h | 2 +- overlay/overlay.hpp | 2 + overlay/overlays.h | 1 + tdutils/td/utils/DecTree.h | 20 ++ tl/generate/scheme/ton_api.tl | 11 + tl/generate/scheme/ton_api.tlo | Bin 64644 -> 65956 bytes .../validator-engine-console-query.cpp | 193 ++++++++++++++++++ .../validator-engine-console-query.h | 136 ++++++++++++ .../validator-engine-console.cpp | 8 +- validator-engine/validator-engine.cpp | 163 +++++++++++++-- validator-engine/validator-engine.hpp | 9 + validator/full-node-shard.cpp | 33 +++ validator/full-node-shard.h | 4 + validator/full-node-shard.hpp | 4 + validator/full-node.cpp | 32 ++- validator/full-node.h | 7 + validator/full-node.hpp | 8 + 20 files changed, 682 insertions(+), 18 deletions(-) diff --git a/overlay/overlay-manager.cpp b/overlay/overlay-manager.cpp index 059ad4e2..1d133673 100644 --- a/overlay/overlay-manager.cpp +++ b/overlay/overlay-manager.cpp @@ -21,13 +21,18 @@ #include "overlay.h" #include "adnl/utils.hpp" +#include "td/actor/actor.h" +#include "td/actor/common.h" #include "td/utils/Random.h" #include "td/db/RocksDb.h" +#include "td/utils/Status.h" #include "td/utils/overloaded.h" #include "keys/encryptor.h" +#include "td/utils/port/Poll.h" +#include namespace ton { @@ -269,6 +274,52 @@ void OverlayManager::save_to_db(adnl::AdnlNodeIdShort local_id, OverlayIdShort o db_.set(key, create_serialize_tl_object(std::move(obj))); } +void OverlayManager::get_stats(td::Promise> promise) { + class Cb : public td::actor::Actor { + public: + Cb(td::Promise> promise) : promise_(std::move(promise)) { + } + void incr_pending() { + pending_++; + } + void decr_pending() { + if (!--pending_) { + promise_.set_result(create_tl_object(std::move(res_))); + stop(); + } + } + void receive_answer(tl_object_ptr res) { + if (res) { + res_.push_back(std::move(res)); + } + decr_pending(); + } + + private: + std::vector> res_; + size_t pending_{1}; + td::Promise> promise_; + }; + + auto act = td::actor::create_actor("overlaysstatsmerger", std::move(promise)).release(); + + for (auto &a : overlays_) { + for (auto &b : a.second) { + td::actor::send_closure(act, &Cb::incr_pending); + td::actor::send_closure(b.second, &Overlay::get_stats, + [act](td::Result> R) { + if (R.is_ok()) { + td::actor::send_closure(act, &Cb::receive_answer, R.move_as_ok()); + } else { + td::actor::send_closure(act, &Cb::receive_answer, nullptr); + } + }); + } + } + + td::actor::send_closure(act, &Cb::decr_pending); +} + Certificate::Certificate(PublicKey issued_by, td::int32 expire_at, td::uint32 max_size, td::uint32 flags, td::BufferSlice signature) : issued_by_(issued_by) diff --git a/overlay/overlay-manager.h b/overlay/overlay-manager.h index 574d0f91..c6afefaf 100644 --- a/overlay/overlay-manager.h +++ b/overlay/overlay-manager.h @@ -91,6 +91,7 @@ class OverlayManager : public Overlays { void register_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, td::actor::ActorOwn overlay); + void get_stats(td::Promise> promise) override; struct PrintId {}; diff --git a/overlay/overlay.cpp b/overlay/overlay.cpp index 21ec9363..ec13fcd3 100644 --- a/overlay/overlay.cpp +++ b/overlay/overlay.cpp @@ -16,6 +16,7 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include "auto/tl/ton_api.h" #include "td/utils/Random.h" #include "adnl/utils.hpp" @@ -25,6 +26,7 @@ #include "auto/tl/ton_api.hpp" #include "keys/encryptor.h" +#include "td/utils/StringBuilder.h" namespace ton { @@ -554,6 +556,19 @@ void OverlayImpl::broadcast_checked(Overlay::BroadcastHash hash, td::Result> promise) { + auto res = create_tl_object(); + res->adnl_id_ = local_id_.bits256_value(); + res->overlay_id_ = overlay_id_.bits256_value(); + res->overlay_id_full_ = id_full_.pubkey().tl(); + peers_.iterate([&](const adnl::AdnlNodeIdShort &key, const OverlayPeer &peer) { res->nodes_.push_back(key.tl()); }); + + res->stats_.push_back( + create_tl_object("neighbours_cnt", PSTRING() << neighbours_.size())); + + promise.set_value(std::move(res)); +} + } // namespace overlay } // namespace ton diff --git a/overlay/overlay.h b/overlay/overlay.h index cba15145..cce24ed8 100644 --- a/overlay/overlay.h +++ b/overlay/overlay.h @@ -63,6 +63,7 @@ class Overlay : public td::actor::Actor { virtual void add_certificate(PublicKeyHash key, std::shared_ptr) = 0; virtual void set_privacy_rules(OverlayPrivacyRules rules) = 0; virtual void receive_nodes_from_db(tl_object_ptr nodes) = 0; + virtual void get_stats(td::Promise> promise) = 0; //virtual void receive_broadcast(td::BufferSlice data) = 0; //virtual void subscribe(std::unique_ptr callback) = 0; }; @@ -70,4 +71,3 @@ class Overlay : public td::actor::Actor { } // namespace overlay } // namespace ton - diff --git a/overlay/overlay.hpp b/overlay/overlay.hpp index cbddd654..76d17aab 100644 --- a/overlay/overlay.hpp +++ b/overlay/overlay.hpp @@ -190,6 +190,8 @@ class OverlayImpl : public Overlay { std::shared_ptr get_certificate(PublicKeyHash local_id); td::Result get_encryptor(PublicKey source); + void get_stats(td::Promise> promise) override; + private: template void process_query(adnl::AdnlNodeIdShort src, T &query, td::Promise promise) { diff --git a/overlay/overlays.h b/overlay/overlays.h index 1ad55453..aba83ab1 100644 --- a/overlay/overlays.h +++ b/overlay/overlays.h @@ -231,6 +231,7 @@ class Overlays : public td::actor::Actor { virtual void get_overlay_random_peers(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay, td::uint32 max_peers, td::Promise> promise) = 0; + virtual void get_stats(td::Promise> promise) = 0; }; } // namespace overlay diff --git a/tdutils/td/utils/DecTree.h b/tdutils/td/utils/DecTree.h index 80bd0e21..1ebb6cd6 100644 --- a/tdutils/td/utils/DecTree.h +++ b/tdutils/td/utils/DecTree.h @@ -46,6 +46,17 @@ class DecTree { } } + template + void iterate(const FuncT &cb) { + if (left_) { + left_->iterate(cb); + } + cb(key_, value_); + if (right_) { + right_->iterate(cb); + } + } + Node(KeyType key, ValueType value, uint32 y) : size_(1), key_(std::move(key)), value_(std::move(value)), y_(y) { } }; @@ -223,6 +234,15 @@ class DecTree { bool exists(const KeyType &key) const { return get_node(root_, key) != nullptr; } + + template + void iterate(const FuncT &cb) { + if (size() == 0) { + return; + } else { + root_->iterate(cb); + } + } }; } // namespace td diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index b226ca7f..0be70540 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -620,6 +620,11 @@ engine.validator.proposalVote perm_key:int256 to_send:bytes = engine.validator.P engine.validator.dhtServerStatus id:int256 status:int = engine.validator.DhtServerStatus; engine.validator.dhtServersStatus servers:(vector engine.validator.dhtServerStatus) = engine.validator.DhtServersStatus; +engine.validator.overlayStats overlay_id:int256 overlay_id_full:PublicKey adnl_id:int256 nodes:(vector adnl.id.short) + stats:(vector engine.validator.oneStat) = engine.validator.OverlayStats; +engine.validator.overlaysStats overlays:(vector engine.validator.overlayStats) = engine.validator.OverlaysStats; + + ---functions--- engine.validator.getTime = engine.validator.Time; @@ -661,8 +666,14 @@ engine.validator.createComplaintVote election_id:int vote:bytes = engine.validat engine.validator.checkDhtServers id:int256 = engine.validator.DhtServersStatus; +engine.validator.getOverlaysStats = engine.validator.OverlaysStats; + engine.validator.controlQuery data:bytes = Object; +engine.validator.importCertificate overlay_id:int256 local_id:adnl.id.short signed_key:engine.validator.KeyHash cert:overlay.Certificate = engine.validator.Success; +engine.validator.signShardOverlayCertificate workchain:int shard:long signed_key:engine.validator.KeyHash expire_at:int max_size:int = overlay.Certificate; +engine.validator.importShardOverlayCertificate workchain:int shard:long signed_key:engine.validator.KeyHash cert:overlay.Certificate = engine.validator.Success; + ---types--- storage.pong = storage.Pong; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 32f42b3c4843343c874d1a1a2696af299ab58f1c..dbdd73b3db01254391b73828527be4872531d40a 100644 GIT binary patch delta 711 zcmZqq$-JbQh4;~HeJchiII@v(w~t$l+2{3{q7**`Gsd^IFkxK1Qp}tSax5HLx3;k1)8HfdQmu@`70g z0{l?%_{@~}w9=fM$rlPFCa>rhnEYoE5C^Jo`ls64F@hXEnYYv42c!??jjM88G1%P) zH51}K29TPY$Ao-9rbs`#ugL+Go?H;9AOSKEY&=l1$eB%t!?x`pNWwB5=Pi%<`(1^l{lvsm1L%6 zCMTApGITF{vU&2AnFfEkf~Ig+b5Gd(XgB|bZ~5@a|7$X$L*j~xPYfCNYl zOESY>yTyjkEb70 y!)&I<=wMh4@(7Z}lN)D>Pi|S~BZzUXEZu?Rj!i_Ntv?~^xkbSZc+MsC);^jTzb#`BKJ RCthSs7I^hxGsl}bq5#z`8LI#Q diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index 64f57de1..4b5802c6 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -26,9 +26,12 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "validator-engine-console-query.h" +#include "auto/tl/ton_api.h" +#include "td/utils/StringBuilder.h" #include "validator-engine-console.h" #include "terminal/terminal.h" #include "td/utils/filesystem.h" +#include "overlay/overlays.h" #include @@ -720,3 +723,193 @@ td::Status CheckDhtServersQuery::receive(td::BufferSlice data) { } return td::Status::OK(); } + +td::Status SignCertificateQuery::run() { + TRY_RESULT_ASSIGN(overlay_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(id_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(expire_at_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(max_size_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(signer_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(out_file_, tokenizer_.get_token()); + return td::Status::OK(); +} + +td::Status SignCertificateQuery::send() { + auto cid = ton::create_serialize_tl_object(overlay_, id_, expire_at_, max_size_); + auto sign = ton::create_serialize_tl_object(signer_.tl(), std::move(cid)); + auto pub = ton::create_serialize_tl_object(signer_.tl()); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(pub), + td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &SignCertificateQuery::handle_error, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &SignCertificateQuery::receive_pubkey, R.move_as_ok()); + } + })); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(sign), + td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &SignCertificateQuery::handle_error, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &SignCertificateQuery::receive_signature, R.move_as_ok()); + } + })); + return td::Status::OK(); +} + +void SignCertificateQuery::receive_pubkey(td::BufferSlice R) { + auto f = ton::fetch_tl_object(R.as_slice(), true); + if (f.is_error()) { + handle_error(f.move_as_error_prefix("Failed to get pubkey: ")); + return; + } + pubkey_ = f.move_as_ok(); + has_pubkey_ = true; + if(has_signature_) { + save_certificate(); + } +} + + +td::Status SignCertificateQuery::receive(td::BufferSlice data) { + UNREACHABLE(); +} + +void SignCertificateQuery::receive_signature(td::BufferSlice R) { + auto f = ton::fetch_tl_object(R.as_slice(), true); + if(f.is_error()){ + handle_error(f.move_as_error_prefix("Failed to get signature: ")); + return; + } + signature_ = std::move(f.move_as_ok()->signature_); + if(has_pubkey_) { + save_certificate(); + } +} + +void SignCertificateQuery::save_certificate() { + auto c = ton::create_serialize_tl_object( + std::move(pubkey_), expire_at_, max_size_, std::move(signature_)); + auto w = td::write_file(out_file_, c.as_slice()); + if(w.is_error()) { + handle_error(w.move_as_error_prefix("Failed to write certificate to file: ")); + return; + } + td::TerminalIO::out() << "saved certificate\n"; + stop(); +} + +td::Status ImportCertificateQuery::run() { + TRY_RESULT_ASSIGN(overlay_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(id_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(kh_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(in_file_, tokenizer_.get_token()); + return td::Status::OK(); +} + +td::Status ImportCertificateQuery::send() { + TRY_RESULT(data, td::read_file(in_file_)); + TRY_RESULT_PREFIX(cert, ton::fetch_tl_object(data.as_slice(), true), + "incorrect certificate"); + auto b = ton::create_serialize_tl_object( + overlay_, + ton::create_tl_object(id_), + ton::create_tl_object(kh_.tl()), + std::move(cert) + ); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} +td::Status GetOverlaysStatsQuery::run() { + return td::Status::OK(); +} + +td::Status GetOverlaysStatsQuery::send() { + auto b = ton::create_serialize_tl_object(); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status GetOverlaysStatsQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + for (auto &s : f->overlays_) { + td::StringBuilder sb; + sb << "overlay_id=" << s->overlay_id_ << " adnl_id=" << s->adnl_id_ << "\n"; + sb << " nodes:"; + for (auto &n : s->nodes_) { + sb << " " << n->id_ << "\n"; + } + sb << " stats:\n"; + for (auto &t : s->stats_) { + sb << " " << t->key_ << "\t" << t->value_ << "\n"; + } + td::TerminalIO::output(sb.as_cslice()); + } + return td::Status::OK(); +} + + +td::Status ImportCertificateQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "successfully sent certificate to overlay manager\n"; + return td::Status::OK(); +} + + +td::Status SignShardOverlayCertificateQuery::run() { + TRY_RESULT_ASSIGN(wc_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token() ); + TRY_RESULT_ASSIGN(key_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(expire_at_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(max_size_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(out_file_, tokenizer_.get_token()); + + return td::Status::OK(); +} + +td::Status SignShardOverlayCertificateQuery::send() { + auto b = ton::create_serialize_tl_object + (wc_, shard_, ton::create_tl_object(key_.tl()), expire_at_, max_size_); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status SignShardOverlayCertificateQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(c, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect cert: "); + auto w = td::write_file(out_file_, data.as_slice()); + if(w.is_error()) { + return w.move_as_error_prefix("Failed to write certificate to file: "); + } + td::TerminalIO::out() << "saved certificate\n"; + + return td::Status::OK(); +} + +td::Status ImportShardOverlayCertificateQuery::run() { + TRY_RESULT_ASSIGN(wc_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token() ); + TRY_RESULT_ASSIGN(key_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(in_file_, tokenizer_.get_token()); + + return td::Status::OK(); +} + +td::Status ImportShardOverlayCertificateQuery::send() { + TRY_RESULT(data, td::read_file(in_file_)); + TRY_RESULT_PREFIX(cert, ton::fetch_tl_object(data.as_slice(), true), + "incorrect certificate"); + auto b = ton::create_serialize_tl_object + (wc_, shard_, ton::create_tl_object(key_.tl()), std::move(cert)); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status ImportShardOverlayCertificateQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "successfully sent certificate to overlay manager\n"; + return td::Status::OK(); +} diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index e010d76f..769f2052 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -903,3 +903,139 @@ class CheckDhtServersQuery : public Query { private: ton::PublicKeyHash id_; }; + +class GetOverlaysStatsQuery : public Query { + public: + GetOverlaysStatsQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "getoverlaysstats"; + } + static std::string get_help() { + return "getoverlaysstats\tgets stats for all overlays"; + } + std::string name() const override { + return get_name(); + } +}; + +class SignCertificateQuery : public Query { + public: + SignCertificateQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "signcert"; + } + static std::string get_help() { + return "signcert \tsign overlay certificate by key"; + } + std::string name() const override { + return get_name(); + } + void receive_pubkey(td::BufferSlice R); + void receive_signature(td::BufferSlice R); + + + private: + void save_certificate(); + + td::Bits256 overlay_; + td::Bits256 id_; + td::int32 expire_at_; + td::uint32 max_size_; + std::string out_file_; + ton::PublicKeyHash signer_; + td::BufferSlice signature_; + std::unique_ptr pubkey_; + bool has_signature_{0}; + bool has_pubkey_{0}; +}; + +class ImportCertificateQuery : public Query { + public: + ImportCertificateQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "importcert"; + } + static std::string get_help() { + return "importcert \timport overlay certificate for specific key"; + } + std::string name() const override { + return get_name(); + } + + private: + td::Bits256 overlay_; + td::Bits256 id_; + ton::PublicKeyHash kh_; + std::string in_file_; +}; + +class SignShardOverlayCertificateQuery : public Query { + public: + SignShardOverlayCertificateQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "signshardoverlaycert"; + } + static std::string get_help() { + return "signshardoverlaycert \tsign certificate for in currently active shard overlay"; + } + std::string name() const override { + return get_name(); + } + + private: + + td::int32 wc_; + td::int64 shard_; + td::int32 expire_at_; + ton::PublicKeyHash key_; + td::uint32 max_size_; + std::string out_file_; +}; + + +class ImportShardOverlayCertificateQuery : public Query { + public: + ImportShardOverlayCertificateQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "importshardoverlaycert"; + } + static std::string get_help() { + return "importshardoverlaycert \timport certificate for in currently active shard overlay"; + } + std::string name() const override { + return get_name(); + } + + private: + + td::int32 wc_; + td::int64 shard_; + ton::PublicKeyHash key_; + std::string in_file_; +}; + diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index 43a02dc6..899cd53f 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -134,6 +134,11 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); } bool ValidatorEngineConsole::envelope_send_query(td::BufferSlice query, td::Promise promise) { @@ -258,7 +263,8 @@ int main(int argc, char* argv[]) { std::exit(2); }); p.add_option('V', "version", "shows validator-engine-console build information", [&]() { - std::cout << "validator-engine-console build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; + std::cout << "validator-engine-console build information: [ Commit: " << GitMetadata::CommitSHA1() + << ", Date: " << GitMetadata::CommitDate() << "]\n"; std::exit(0); }); p.add_checked_option('a', "address", "server address", [&](td::Slice arg) { diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 9c9d4a98..59279bef 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -27,6 +27,11 @@ */ #include "validator-engine.hpp" +#include "auto/tl/ton_api.h" +#include "overlay-manager.h" +#include "td/actor/actor.h" +#include "tl-utils/tl-utils.hpp" +#include "tl/TlObject.h" #include "ton/ton-types.h" #include "ton/ton-tl.hpp" #include "ton/ton-io.hpp" @@ -3143,6 +3148,135 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_createCom .release(); } +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importCertificate &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (keyring_.empty()) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "keyring not started"))); + return; + } + + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + auto r = ton::overlay::Certificate::create(std::move(query.cert_)); + if(r.is_error()) { + promise.set_value(create_control_query_error(r.move_as_error_prefix("Invalid certificate: "))); + } + //TODO force Overlays::update_certificate to return result + /*auto P = td::PromiseCreator::lambda( + [promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); + } + }); + */ + td::actor::send_closure(overlay_manager_, &ton::overlay::Overlays::update_certificate, + ton::adnl::AdnlNodeIdShort{query.local_id_->id_}, + ton::overlay::OverlayIdShort{query.overlay_id_}, + ton::PublicKeyHash{query.signed_key_->key_hash_}, + r.move_as_ok()); + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true) + ); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importShardOverlayCertificate &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (keyring_.empty()) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "keyring not started"))); + return; + } + + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + auto r = ton::overlay::Certificate::create(std::move(query.cert_)); + if(r.is_error()) { + promise.set_value(create_control_query_error(r.move_as_error_prefix("Invalid certificate: "))); + } + auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error_prefix("failed to import cert: "))); + } else { + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); + } + }); + ton::ShardIdFull shard_id{ton::WorkchainId{query.workchain_}, static_cast(query.shard_)}; + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::import_shard_overlay_certificate, + shard_id, ton::PublicKeyHash{query.signed_key_->key_hash_}, r.move_as_ok(), std::move(P)); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_signShardOverlayCertificate &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (keyring_.empty()) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "keyring not started"))); + return; + } + + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + ton::ShardIdFull shard_id{ton::WorkchainId{query.workchain_}, static_cast(query.shard_)}; + auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error_prefix("failed to import cert: "))); + } else { + promise.set_value(R.move_as_ok()); + } + }); + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::sign_shard_overlay_certificate, + shard_id, ton::PublicKeyHash{query.signed_key_->key_hash_}, query.expire_at_, query.max_size_, std::move(P)); +} + + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getOverlaysStats &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_default)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + + if (keyring_.empty()) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "keyring not started"))); + return; + } + + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + + td::actor::send_closure(overlay_manager_, &ton::overlay::Overlays::get_stats, + [promise = std::move(promise)]( + td::Result> R) mutable { + if (R.is_ok()) { + promise.set_value(ton::serialize_tl_object(R.move_as_ok(), true)); + } else { + promise.set_value(create_control_query_error( + td::Status::Error(ton::ErrorCode::notready, "overlay manager not ready"))); + } + }); +} + void ValidatorEngine::process_control_query(td::uint16 port, ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data, td::Promise promise) { @@ -3294,7 +3428,8 @@ int main(int argc, char *argv[]) { SET_VERBOSITY_LEVEL(v); }); p.add_option('V', "version", "shows validator-engine build information", [&]() { - std::cout << "validator-engine build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; + std::cout << "validator-engine build information: [ Commit: " << GitMetadata::CommitSHA1() + << ", Date: " << GitMetadata::CommitDate() << "]\n"; std::exit(0); }); p.add_option('h', "help", "prints_help", [&]() { @@ -3374,18 +3509,20 @@ int main(int argc, char *argv[]) { acts.push_back([&x, seq]() { td::actor::send_closure(x, &ValidatorEngine::add_unsafe_catchain, seq); }); return td::Status::OK(); }); - p.add_checked_option( - 'F', "unsafe-catchain-rotate", "use forceful and DANGEROUS catchain rotation", [&](td::Slice params) { - auto pos1 = params.find(':'); - TRY_RESULT(b_seq, td::to_integer_safe(params.substr(0, pos1))); - params = params.substr(++pos1, params.size()); - auto pos2 = params.find(':'); - TRY_RESULT(cc_seq, td::to_integer_safe(params.substr(0, pos2))); - params = params.substr(++pos2, params.size()); - auto h = std::stoi(params.substr(0, params.size()).str()); - acts.push_back([&x, b_seq, cc_seq, h]() { td::actor::send_closure(x, &ValidatorEngine::add_unsafe_catchain_rotation, b_seq, cc_seq, h); }); - return td::Status::OK(); - }); + p.add_checked_option('F', "unsafe-catchain-rotate", "use forceful and DANGEROUS catchain rotation", + [&](td::Slice params) { + auto pos1 = params.find(':'); + TRY_RESULT(b_seq, td::to_integer_safe(params.substr(0, pos1))); + params = params.substr(++pos1, params.size()); + auto pos2 = params.find(':'); + TRY_RESULT(cc_seq, td::to_integer_safe(params.substr(0, pos2))); + params = params.substr(++pos2, params.size()); + auto h = std::stoi(params.substr(0, params.size()).str()); + acts.push_back([&x, b_seq, cc_seq, h]() { + td::actor::send_closure(x, &ValidatorEngine::add_unsafe_catchain_rotation, b_seq, cc_seq, h); + }); + return td::Status::OK(); + }); td::uint32 threads = 7; p.add_checked_option( 't', "threads", PSTRING() << "number of threads (default=" << threads << ")", [&](td::Slice fname) { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 07078f95..88ba802d 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -28,6 +28,7 @@ #pragma once #include "adnl/adnl.h" +#include "auto/tl/ton_api.h" #include "rldp/rldp.h" #include "dht/dht.h" #include "validator/manager.h" @@ -395,6 +396,14 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_createComplaintVote &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_importCertificate &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_signShardOverlayCertificate &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_importShardOverlayCertificate &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_getOverlaysStats &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); template void run_control_query(T &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 769ef8f3..72684151 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -115,6 +115,7 @@ void FullNodeShardImpl::check_broadcast(PublicKeyHash src, td::BufferSlice broad void FullNodeShardImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) { td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, adnl_id_, overlay_id_); adnl_id_ = adnl_id; + local_id_ = adnl_id_.pubkey_hash(); create_overlay(); } @@ -848,6 +849,38 @@ void FullNodeShardImpl::signed_new_certificate(ton::overlay::Certificate cert) { td::actor::send_closure(overlays_, &overlay::Overlays::update_certificate, adnl_id_, overlay_id_, local_id_, cert_); } +void FullNodeShardImpl::sign_overlay_certificate(PublicKeyHash signed_key, td::uint32 expire_at, td::uint32 max_size, td::Promise promise) { + auto sign_by = sign_cert_by_; + if (sign_by.is_zero()) { + promise.set_error(td::Status::Error("Node has no key with signing authority")); + return; + } + + ton::overlay::Certificate cert{ + sign_by, static_cast(expire_at), max_size, + overlay::CertificateFlags::Trusted | overlay::CertificateFlags::AllowFec, td::BufferSlice{}}; + auto to_sign = cert.to_sign(overlay_id_, signed_key); + + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), expire_at = expire_at, max_size = max_size, promise = std::move(promise)](td::Result> R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error_prefix("failed to create certificate: failed to sign: ")); + } else { + auto p = R.move_as_ok(); + auto c = ton::create_serialize_tl_object(p.second.tl(), static_cast(expire_at), max_size, std::move(p.first)); + promise.set_value(std::move(c)); + } + }); + td::actor::send_closure(keyring_, &ton::keyring::Keyring::sign_add_get_public_key, sign_by, std::move(to_sign), + std::move(P)); +} + +void FullNodeShardImpl::import_overlay_certificate(PublicKeyHash signed_key, std::shared_ptr cert, td::Promise promise) { + td::actor::send_closure(overlays_, &ton::overlay::Overlays::update_certificate, + adnl_id_, overlay_id_, signed_key, cert); + promise.set_value( td::Unit() ); +} + void FullNodeShardImpl::update_validators(std::vector public_key_hashes, PublicKeyHash local_hash) { if (!client_.empty()) { return; diff --git a/validator/full-node-shard.h b/validator/full-node-shard.h index 40a422e0..c1712baf 100644 --- a/validator/full-node-shard.h +++ b/validator/full-node-shard.h @@ -42,6 +42,10 @@ class FullNodeShard : public td::actor::Actor { virtual void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) = 0; virtual void send_broadcast(BlockBroadcast broadcast) = 0; + virtual void sign_overlay_certificate(PublicKeyHash signed_key, td::uint32 expiry_at, td::uint32 max_size, td::Promise promise) = 0; + virtual void import_overlay_certificate(PublicKeyHash signed_key, std::shared_ptr cert, td::Promise promise) = 0; + + virtual void download_block(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) = 0; virtual void download_zero_state(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index 2c468374..a2dd5cc4 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -170,6 +170,10 @@ class FullNodeShardImpl : public FullNodeShard { void alarm() override; void update_validators(std::vector public_key_hashes, PublicKeyHash local_hash) override; + + void sign_overlay_certificate(PublicKeyHash signed_key, td::uint32 expiry_at, td::uint32 max_size, td::Promise promise) override; + void import_overlay_certificate(PublicKeyHash signed_key, std::shared_ptr cert, td::Promise promise) override; + void sign_new_certificate(PublicKeyHash sign_by); void signed_new_certificate(ton::overlay::Certificate cert); diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 691a8b36..6606f215 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -76,6 +76,27 @@ void FullNodeImpl::del_permanent_key(PublicKeyHash key, td::Promise pr promise.set_value(td::Unit()); } +void FullNodeImpl::sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, + td::uint32 expiry_at, td::uint32 max_size, + td::Promise promise) { + auto it = shards_.find(shard_id); + if(it == shards_.end()) { + promise.set_error(td::Status::Error(ErrorCode::error, "shard not found")); + return; + } + td::actor::send_closure(it->second, &FullNodeShard::sign_overlay_certificate, signed_key, expiry_at, max_size, std::move(promise)); +} + +void FullNodeImpl::import_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, + std::shared_ptr cert, + td::Promise promise) { + auto it = shards_.find(shard_id); + if(it == shards_.end()) { + promise.set_error(td::Status::Error(ErrorCode::error, "shard not found")); + } + td::actor::send_closure(it->second, &FullNodeShard::import_overlay_certificate, signed_key, cert, std::move(promise)); +} + void FullNodeImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) { adnl_id_ = adnl_id; @@ -86,6 +107,7 @@ void FullNodeImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) = 0; virtual void del_permanent_key(PublicKeyHash key, td::Promise promise) = 0; + virtual void sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, + td::uint32 expiry_at, td::uint32 max_size, + td::Promise promise) = 0; + virtual void import_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, + std::shared_ptr cert, + td::Promise promise) = 0; + virtual void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) = 0; static constexpr td::uint32 max_block_size() { diff --git a/validator/full-node.hpp b/validator/full-node.hpp index e9b64f57..6d57f4a8 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -42,6 +42,14 @@ class FullNodeImpl : public FullNode { void add_permanent_key(PublicKeyHash key, td::Promise promise) override; void del_permanent_key(PublicKeyHash key, td::Promise promise) override; + void sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, + td::uint32 expiry_at, td::uint32 max_size, + td::Promise promise) override; + void import_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, + std::shared_ptr cert, + td::Promise promise) override; + + void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) override; void add_shard(ShardIdFull shard); From 703bcd6e32106878b42756cb72b7af5b80f591d3 Mon Sep 17 00:00:00 2001 From: OmicronTau Date: Sun, 5 Dec 2021 17:41:22 +0300 Subject: [PATCH 05/21] Add tests and fixes for modpow2, muldivmod --- CMakeLists.txt | 5 + crypto/CMakeLists.txt | 5 + crypto/common/bigint.hpp | 69 ++- crypto/common/refint.cpp | 8 +- crypto/fift/lib/Asm.fif | 5 + crypto/test/modbigint.cpp | 1070 +++++++++++++++++++++++++++++++++++ crypto/test/test-bigint.cpp | 876 ++++++++++++++++++++++++++++ crypto/vm/arithops.cpp | 26 +- doc/tvm.tex | 5 +- rldp2/CMakeLists.txt | 2 +- 10 files changed, 2017 insertions(+), 54 deletions(-) create mode 100644 crypto/test/modbigint.cpp create mode 100644 crypto/test/test-bigint.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index a97a9c96..da1945ff 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -423,6 +423,9 @@ target_link_libraries(test-vm PRIVATE ton_crypto fift-lib) add_executable(test-smartcont test/test-td-main.cpp ${SMARTCONT_TEST_SOURCE}) target_link_libraries(test-smartcont PRIVATE smc-envelope fift-lib ton_db) +add_executable(test-bigint ${BIGINT_TEST_SOURCE}) +target_link_libraries(test-bigint PRIVATE ton_crypto) + add_executable(test-cells test/test-td-main.cpp ${CELLS_TEST_SOURCE}) target_link_libraries(test-cells PRIVATE ton_crypto) @@ -523,6 +526,7 @@ if (HAS_PARENT) ${FEC_TEST_SOURCE} ${ED25519_TEST_SOURCE} ${TONDB_TEST_SOURCE} + ${BIGNUM_TEST_SOURCE} ${CELLS_TEST_SOURCE} # ${TONVM_TEST_SOURCE} ${FIFT_TEST_SOURCE} ${TONLIB_ONLINE_TEST_SOURCE} PARENT_SCOPE) endif() @@ -536,6 +540,7 @@ set(TEST_OPTIONS "--regression ${CMAKE_CURRENT_SOURCE_DIR}/test/regression-tests separate_arguments(TEST_OPTIONS) add_test(test-ed25519-crypto crypto/test-ed25519-crypto) add_test(test-ed25519 test-ed25519) +add_test(test-bigint test-bigint) add_test(test-vm test-vm ${TEST_OPTIONS}) add_test(test-fift test-fift ${TEST_OPTIONS}) add_test(test-cells test-cells ${TEST_OPTIONS}) diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index 79406ffc..88a0272b 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -259,6 +259,11 @@ set(FIFT_TEST_SOURCE PARENT_SCOPE ) +set(BIGINT_TEST_SOURCE + ${CMAKE_CURRENT_SOURCE_DIR}/test/test-bigint.cpp + PARENT_SCOPE +) + add_library(ton_crypto STATIC ${TON_CRYPTO_SOURCE}) target_include_directories(ton_crypto PUBLIC $ diff --git a/crypto/common/bigint.hpp b/crypto/common/bigint.hpp index e78f4892..94da3a13 100644 --- a/crypto/common/bigint.hpp +++ b/crypto/common/bigint.hpp @@ -264,7 +264,7 @@ class AnyIntView { return digits[size() - 1]; } double top_double() const { - return size() > 1 ? (double)digits[size() - 1] + (double)digits[size() - 2] * (1.0 / Tr::Base) + return size() > 1 ? (double)digits[size() - 1] + (double)digits[size() - 2] * Tr::InvBase : (double)digits[size() - 1]; } bool is_odd_any() const { @@ -314,8 +314,15 @@ class BigIntG { digits[0] = x; } BigIntG(Normalize, word_t x) : n(1) { - digits[0] = x; - normalize_bool(); + if (x >= -Tr::Half && x < Tr::Half) { + digits[0] = x; + } else if (len <= 1) { + digits[0] = x; + normalize_bool(); + } else { + digits[0] = ((x + Tr::Half) & (Tr::Base - 1)) - Tr::Half; + digits[n++] = (x >> Tr::word_shift) + (digits[0] < 0); + } } BigIntG(const BigIntG& x) : n(x.n) { std::memcpy(digits, x.digits, n * sizeof(word_t)); @@ -757,7 +764,7 @@ bool AnyIntView::add_pow2_any(int exponent, int factor) { while (size() <= k) { digits[inc_size()] = 0; } - digits[k] += (factor << dm.rem); + digits[k] += ((word_t)factor << dm.rem); return true; } @@ -1087,12 +1094,16 @@ int AnyIntView::cmp_any(const AnyIntView& yp) const { template int AnyIntView::cmp_any(word_t y) const { - if (size() > 1) { - return top_word() < 0 ? -1 : 1; - } else if (size() == 1) { + if (size() == 1) { return digits[0] < y ? -1 : (digits[0] > y ? 1 : 0); - } else { + } else if (!size()) { return 0x80000000; + } else if (size() == 2 && (y >= Tr::Half || y < -Tr::Half)) { + word_t x0 = digits[0] & (Tr::Base - 1), y0 = y & (Tr::Base - 1); + word_t x1 = digits[1] + (digits[0] >> Tr::word_shift), y1 = (y >> Tr::word_shift); + return x1 < y1 ? -1 : (x1 > y1 ? 1 : (x0 < y0 ? -1 : (x0 > y0 ? 1 : 0))); + } else { + return top_word() < 0 ? -1 : 1; } } @@ -1312,17 +1323,14 @@ bool AnyIntView::mod_div_any(const AnyIntView& yp, AnyIntView& quot, if (k > quot.max_size()) { return invalidate_bool(); } - quot.set_size(max(k,1)); - for(int qi=0; qi< max(k,1); qi++) { - quot.digits[qi]=0; - } + quot.set_size(std::max(k, 1)); + quot.digits[0] = 0; } else { if (k >= quot.max_size()) { return invalidate_bool(); } quot.set_size(k + 1); - double x_top = top_double(); - word_t q = std::llrint(x_top * y_inv * Tr::InvBase); + word_t q = std::llrint(top_double() * y_inv * Tr::InvBase); quot.digits[k] = q; int i = yp.size() - 1; word_t hi = 0; @@ -1337,8 +1345,7 @@ bool AnyIntView::mod_div_any(const AnyIntView& yp, AnyIntView& quot, quot.digits[0] = 0; } while (--k >= 0) { - double x_top = top_double(); - word_t q = std::llrint(x_top * y_inv); + word_t q = std::llrint(top_double() * y_inv); quot.digits[k] = q; for (int i = yp.size() - 1; i >= 0; --i) { Tr::sub_mul(&digits[k + i + 1], &digits[k + i], q, yp.digits[i]); @@ -1346,15 +1353,18 @@ bool AnyIntView::mod_div_any(const AnyIntView& yp, AnyIntView& quot, dec_size(); digits[size() - 1] += (digits[size()] << word_shift); } - if (size() >= yp.size()) { - assert(size() == yp.size()); - double x_top = top_double(); - double t = x_top * y_inv * Tr::InvBase; + if (size() >= yp.size() - 1) { + assert(size() <= yp.size()); + bool grow = (size() < yp.size()); + double t = top_double() * y_inv * (grow ? Tr::InvBase * Tr::InvBase : Tr::InvBase); if (round_mode >= 0) { t += (round_mode ? 1 : 0.5); } word_t q = std::llrint(std::floor(t)); if (q) { + if (grow) { + digits[inc_size()] = 0; + } for (int i = 0; i < size(); i++) { digits[i] -= q * yp.digits[i]; } @@ -1411,6 +1421,7 @@ bool AnyIntView::mod_div_any(const AnyIntView& yp, AnyIntView& quot, return normalize_bool_any(); } +// works for almost-normalized numbers (digits -Base+1 .. Base-1, top non-zero), result also almost-normalized template bool AnyIntView::mod_pow2_any(int exponent) { if (!is_valid()) { @@ -1462,25 +1473,21 @@ bool AnyIntView::mod_pow2_any(int exponent) { if (exponent >= max_size() * word_shift) { return invalidate_bool(); } - if (q - word_shift >= 0) { + if (q - word_shift >= 0) { // original top digit was a non-zero multiple of Base, impossible(?) digits[size() - 1] = 0; digits[inc_size()] = ((word_t)1 << (q - word_shift)); - } - if (q - word_shift == -1 && size() < max_size() - 1) { + } else if (q - word_shift == -1 && size() < max_size()) { digits[size() - 1] = -Tr::Half; digits[inc_size()] = 1; } else { digits[size() - 1] = pow; } return true; - } else if (v >= Tr::Half) { - if (size() == max_size() - 1) { - return invalidate_bool(); - } else { - digits[size() - 1] = v | -Tr::Half; - digits[inc_size()] = ((word_t)1 << (q - word_shift)); - return true; - } + } else if (v >= Tr::Half && size() < max_size()) { + word_t w = (((v >> (word_shift - 1)) + 1) >> 1); + digits[size() - 1] = v - (w << word_shift); + digits[inc_size()] = w; + return true; } else { digits[size() - 1] = v; return true; diff --git a/crypto/common/refint.cpp b/crypto/common/refint.cpp index b79750ce..8e06da5f 100644 --- a/crypto/common/refint.cpp +++ b/crypto/common/refint.cpp @@ -128,12 +128,10 @@ RefInt256 muldiv(RefInt256 x, RefInt256 y, RefInt256 z, int round_mode) { } std::pair muldivmod(RefInt256 x, RefInt256 y, RefInt256 z, int round_mode) { - typename td::BigInt256::DoubleInt tmp{0}; + typename td::BigInt256::DoubleInt tmp{0}, quot; tmp.add_mul(*x, *y); - RefInt256 quot{true}; - tmp.mod_div(*z, quot.unique_write(), round_mode); - quot.write().normalize(); - return std::make_pair(std::move(quot), td::make_refint(tmp)); + tmp.mod_div(*z, quot, round_mode); + return std::make_pair(td::make_refint(quot.normalize()), td::make_refint(tmp)); } RefInt256 operator&(RefInt256 x, RefInt256 y) { diff --git a/crypto/fift/lib/Asm.fif b/crypto/fift/lib/Asm.fif index d61923c1..76381b85 100644 --- a/crypto/fift/lib/Asm.fif +++ b/crypto/fift/lib/Asm.fif @@ -334,9 +334,14 @@ x{A926} @Defop RSHIFTC x{A935} @Defop(8u+1) RSHIFTR# x{A936} @Defop(8u+1) RSHIFTC# x{A938} @Defop(8u+1) MODPOW2# +x{A939} @Defop(8u+1) MODPOW2R# +x{A93A} @Defop(8u+1) MODPOW2C# x{A984} @Defop MULDIV x{A985} @Defop MULDIVR +x{A988} @Defop MULMOD x{A98C} @Defop MULDIVMOD +x{A98D} @Defop MULDIVMODR +x{A98E} @Defop MULDIVMODC x{A9A4} @Defop MULRSHIFT x{A9A5} @Defop MULRSHIFTR x{A9A6} @Defop MULRSHIFTC diff --git a/crypto/test/modbigint.cpp b/crypto/test/modbigint.cpp new file mode 100644 index 00000000..851f0f9b --- /dev/null +++ b/crypto/test/modbigint.cpp @@ -0,0 +1,1070 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include +#include +#include +#include +#include +#include + +namespace modint { + +enum { mod_cnt = 32 }; + +// mod_cnt = 9 => integers -2^268 .. 2^268 +// mod_cnt = 18 => integers -2^537 .. 2^537 +// mod_cnt = 32 => integers -2^955 .. 2^955 +constexpr int mod[mod_cnt] = {999999937, 999999929, 999999893, 999999883, 999999797, 999999761, 999999757, 999999751, + 999999739, 999999733, 999999677, 999999667, 999999613, 999999607, 999999599, 999999587, + 999999541, 999999527, 999999503, 999999491, 999999487, 999999433, 999999391, 999999353, + 99999337, 999999323, 999999229, 999999223, 999999197, 999999193, 999999191, 999999181}; + +// invm[i][j] = mod[i]^(-1) modulo mod[j] +int invm[mod_cnt][mod_cnt]; + +int gcdx(int a, int b, int& u, int& v); + +template +struct ModArray; + +template +struct MixedRadix; + +template +struct ArrayRawDumpRef; + +template +std::ostream& raw_dump_array(std::ostream& os, const std::array& arr) { + os << '['; + for (auto x : arr) { + os << ' ' << x; + } + return os << " ]"; +} + +template +struct MixedRadix { + enum { n = N }; + int a[N]; + MixedRadix(int v) { + set_int(v); + } + MixedRadix() = default; + MixedRadix(const MixedRadix&) = default; + MixedRadix(std::initializer_list l) { + auto sz = std::min(l.size(), (std::size_t)N); + std::copy(l.begin(), l.begin() + sz, a); + std::fill(a + sz, a + N, 0); + } + MixedRadix(const std::array& arr) { + std::copy(arr.begin(), arr.end(), a); + } + template + MixedRadix(const MixedRadix& other) { + static_assert(M >= N); + std::copy(other.a, other.a + N, a); + } + MixedRadix(const ModArray& other); + MixedRadix(const ModArray& other, bool sgnd); + + MixedRadix& set_zero() { + std::fill(a, a + N, 0); + return *this; + } + MixedRadix& set_one() { + a[0] = 1; + std::fill(a + 1, a + N, 0); + return *this; + } + MixedRadix& set_int(int v) { + a[0] = v; + std::fill(a + 1, a + N, 0); + return *this; + } + + MixedRadix copy() const { + return MixedRadix{*this}; + } + + static const int* mod_array() { + return mod; + } + + static int modulus(int i) { + return mod[i]; + } + + int sgn() const { + int i = N - 1; + while (i >= 0 && !a[i]) { + --i; + } + return i < 0 ? 0 : (a[i] > 0 ? 1 : -1); + } + + int cmp(const MixedRadix& other) const { + int i = N - 1; + while (i >= 0 && a[i] == other.a[i]) { + --i; + } + return i < 0 ? 0 : (a[i] > other.a[i] ? 1 : -1); + } + + bool is_small() const { + return !a[N - 1] || a[N - 1] == -1; + } + + bool operator==(const MixedRadix& other) const { + return std::equal(a, a + N, other.a); + } + + bool operator!=(const MixedRadix& other) const { + return !std::equal(a, a + N, other.a); + } + + bool operator<(const MixedRadix& other) const { + return cmp(other) < 0; + } + + bool operator<=(const MixedRadix& other) const { + return cmp(other) <= 0; + } + + bool operator>(const MixedRadix& other) const { + return cmp(other) > 0; + } + + bool operator>=(const MixedRadix& other) const { + return cmp(other) >= 0; + } + + explicit operator bool() const { + return sgn(); + } + + bool operator!() const { + return !sgn(); + } + + MixedRadix& negate() { + int i = 0; + while (i < N - 1 && !a[i]) { + i++; + } + a[i]--; + for (; i < N; i++) { + a[i] = mod[i] - a[i] - 1; + } + a[N - 1] -= mod[N - 1]; + return *this; + } + + static const MixedRadix& pow2(int power); + static MixedRadix negpow2(int power) { + return -pow2(power); + } + + template + const MixedRadix& as_shorter() const { + static_assert(M <= N); + return *reinterpret_cast*>(this); + } + + MixedRadix& import_mod_array(const int* data, bool sgnd = true) { + for (int i = 0; i < N; i++) { + a[i] = data[i] % mod[i]; + } + for (int i = 0; i < N; i++) { + if (a[i] < 0) { + a[i] += mod[i]; + } + for (int j = i + 1; j < N; j++) { + a[j] = (int)((long long)(a[j] - a[i]) * invm[i][j] % mod[j]); + } + } + if (sgnd && a[N - 1] > (mod[N - 1] >> 1)) { + a[N - 1] -= mod[N - 1]; + } + return *this; + } + + MixedRadix& operator=(const MixedRadix&) = default; + + template + MixedRadix& operator=(const MixedRadix& other) { + static_assert(M >= N); + std::copy(other.a, other.a + N, a); + } + + MixedRadix& import_mod_array(const ModArray& other, bool sgnd = true); + + MixedRadix& operator=(const ModArray& other) { + return import_mod_array(other); + } + + MixedRadix& set_sum(const MixedRadix& x, const MixedRadix& y, int factor = 1) { + long long carry = 0; + for (int i = 0; i < N; i++) { + long long acc = x.a[i] + carry + (long long)factor * y.a[i]; + carry = acc / mod[i]; + a[i] = (int)(acc - carry * mod[i]); + if (a[i] < 0) { + a[i] += mod[i]; + --carry; + } + } + if (a[N - 1] >= 0 && carry == -1) { + a[N - 1] -= mod[N - 1]; + } + return *this; + } + + MixedRadix& operator+=(const MixedRadix& other) { + return set_sum(*this, other); + } + + MixedRadix& operator-=(const MixedRadix& other) { + return set_sum(*this, other, -1); + } + + static const MixedRadix& zero(); + static const MixedRadix& one(); + + MixedRadix& operator*=(int factor) { + return set_sum(zero(), *this, factor); + } + + MixedRadix operator-() const { + MixedRadix copy{*this}; + copy.negate(); + return copy; + } + + MixedRadix operator+(const MixedRadix& other) const { + MixedRadix res; + res.set_sum(*this, other); + return res; + } + + MixedRadix operator-(const MixedRadix& other) const { + MixedRadix res; + res.set_sum(*this, other, -1); + return res; + } + + MixedRadix operator*(int factor) const { + MixedRadix res; + res.set_sum(zero(), *this, factor); + return res; + } + + int operator%(int b) const { + int x = a[N - 1] % b; + for (int i = N - 2; i >= 0; --i) { + x = ((long long)x * mod[i] + a[i]) % b; + } + return ((x ^ b) < 0 && x) ? x + b : x; + } + + explicit operator double() const { + double acc = 0.; + for (int i = N - 1; i >= 0; --i) { + acc = acc * mod[i] + a[i]; + } + return acc; + } + + explicit operator long long() const { + long long acc = 0.; + for (int i = N - 1; i >= 0; --i) { + acc = acc * mod[i] + a[i]; + } + return acc; + } + + MixedRadix& to_base(int base) { + int k = N - 1; + while (k > 0 && !a[k]) { + --k; + } + if (k <= 0) { + return *this; + } + for (int i = k - 1; i >= 0; --i) { + // a[i..k] := a[i+1..k] * mod[i] + a[i] + long long carry = a[i]; + for (int j = i; j < k; j++) { + long long t = (long long)a[j + 1] * mod[i] + carry; + carry = t / base; + a[j] = (int)(t - carry * base); + } + a[k] = (int)carry; + } + return *this; + } + + std::ostream& print_dec_destroy(std::ostream& os) { + int s = sgn(); + if (s < 0) { + os << '-'; + negate(); + } else if (!s) { + os << '0'; + return os; + } + to_base(1000000000); + int i = N - 1; + while (!a[i] && i > 0) { + --i; + } + os << a[i]; + while (--i >= 0) { + char buff[12]; + sprintf(buff, "%09d", a[i]); + os << buff; + } + return os; + } + + std::ostream& print_dec(std::ostream& os) const& { + MixedRadix copy{*this}; + return copy.print_dec_destroy(os); + } + + std::ostream& print_dec(std::ostream& os) && { + return print_dec_destroy(os); + } + + std::string to_dec_string_destroy() { + std::ostringstream os; + print_dec_destroy(os); + return std::move(os).str(); + } + + std::string to_dec_string() const& { + MixedRadix copy{*this}; + return copy.to_dec_string_destroy(); + } + + std::string to_dec_string() && { + return to_dec_string_destroy(); + } + + bool to_binary_destroy(unsigned char* arr, int size, bool sgnd = true) { + if (size <= 0) { + return false; + } + int s = (sgnd ? sgn() : 1); + memset(arr, 0, size); + if (s < 0) { + negate(); + } else if (!s) { + return true; + } + to_base(1 << 30); + long long acc = 0; + int bits = 0, j = size; + for (int i = 0; i < N; i++) { + if (!j && a[i]) { + return false; + } + acc += ((long long)a[i] << bits); + bits += 30; + while (bits >= 8 && j > 0) { + arr[--j] = (unsigned char)(acc & 0xff); + bits -= 8; + acc >>= 8; + } + } + while (j > 0) { + arr[--j] = (unsigned char)(acc & 0xff); + acc >>= 8; + } + if (acc) { + return false; + } + if (!sgnd) { + return true; + } + if (s >= 0) { + return arr[0] <= 0x7f; + } + j = size - 1; + while (j >= 0 && !arr[j]) { + --j; + } + assert(j >= 0); + arr[j] = (unsigned char)(-arr[j]); + while (--j >= 0) { + arr[j] = (unsigned char)~arr[j]; + } + return arr[0] >= 0x80; + } + + bool to_binary(unsigned char* arr, int size, bool sgnd = true) const& { + MixedRadix copy{*this}; + return copy.to_binary_destroy(arr, size, sgnd); + } + + bool to_binary(unsigned char* arr, int size, bool sgnd = true) && { + return to_binary_destroy(arr, size, sgnd); + } + + std::ostream& raw_dump(std::ostream& os) const { + return raw_dump_array(os, a); + } + + ArrayRawDumpRef dump() const { + return {a}; + } +}; + +template +struct ModArray { + enum { n = N }; + int a[N]; + ModArray(int v) { + set_int(v); + } + ModArray(long long v) { + set_long(v); + } + ModArray(long v) { + set_long(v); + } + ModArray() = default; + ModArray(const ModArray&) = default; + ModArray(std::initializer_list l) { + auto sz = std::min(l.size(), (std::size_t)N); + std::copy(l.begin(), l.begin() + sz, a); + std::fill(a + sz, a + N, 0); + } + ModArray(const std::array& arr) { + std::copy(arr.begin(), arr.end(), a); + } + template + ModArray(const ModArray& other) { + static_assert(M >= N); + std::copy(other.a, other.a + N, a); + } + ModArray(const int* p) : a(p) { + } + ModArray(std::string str) { + assert(from_dec_string(str) && "not a decimal number"); + } + + ModArray& set_zero() { + std::fill(a, a + N, 0); + return *this; + } + ModArray& set_one() { + std::fill(a, a + N, 1); + return *this; + } + + ModArray& set_int(int v) { + if (v >= 0) { + std::fill(a, a + N, v); + } else { + for (int i = 0; i < N; i++) { + a[i] = mod[i] + v; + } + } + return *this; + } + + ModArray& set_long(long long v) { + for (int i = 0; i < N; i++) { + a[i] = v % mod[i]; + if (a[i] < 0) { + a[i] += mod[i]; + } + } + return *this; + } + + ModArray copy() const { + return ModArray{*this}; + } + + static const int* mod_array() { + return mod; + } + + static int modulus(int i) { + return mod[i]; + } + + static const ModArray& zero(); + static const ModArray& one(); + + ModArray& operator=(const ModArray&) = default; + + template + ModArray& operator=(const ModArray& other) { + static_assert(M >= N); + std::copy(other.a, other.a + N, a); + return *this; + } + + ModArray& negate() { + for (int i = 0; i < N; i++) { + a[i] = (a[i] ? mod[i] - a[i] : 0); + } + return *this; + } + + ModArray& norm_neg() { + for (int i = 0; i < N; i++) { + if (a[i] < 0) { + a[i] += mod[i]; + } + } + return *this; + } + + ModArray& normalize() { + for (int i = 0; i < N; i++) { + a[i] %= mod[i]; + if (a[i] < 0) { + a[i] += mod[i]; + } + } + return *this; + } + + bool is_zero() const { + for (int i = 0; i < N; i++) { + if (a[i]) { + return false; + } + } + return true; + } + + explicit operator bool() const { + return !is_zero(); + } + + bool operator!() const { + return is_zero(); + } + + bool operator==(const ModArray& other) const { + return std::equal(a, a + N, other.a); + } + + bool operator!=(const ModArray& other) const { + return !std::equal(a, a + N, other.a); + } + + bool operator==(long long val) const { + for (int i = 0; i < N; i++) { + int r = (int)(val % mod[i]); + if (a[i] != (r < 0 ? r + mod[i] : r)) { + return false; + } + } + return true; + } + + bool operator!=(long long val) const { + return !operator==(val); + } + + long long try_get_long() const { + return (long long)(MixedRadix<3>(*this)); + } + + bool fits_long() const { + return operator==(try_get_long()); + } + + explicit operator long long() const { + auto v = try_get_long(); + return operator==(v) ? v : -0x8000000000000000; + } + + ModArray& set_sum(const ModArray& x, const ModArray& y) { + for (int i = 0; i < N; i++) { + a[i] = x.a[i] + y.a[i]; + if (a[i] >= mod[i]) { + a[i] -= mod[i]; + } + } + return *this; + } + + ModArray& operator+=(const ModArray& other) { + for (int i = 0; i < N; i++) { + a[i] += other.a[i]; + if (a[i] >= mod[i]) { + a[i] -= mod[i]; + } + } + return *this; + } + + ModArray& operator+=(long long v) { + for (int i = 0; i < N; i++) { + a[i] = (int)((a[i] + v) % mod[i]); + if (a[i] < 0) { + a[i] += mod[i]; + } + } + return *this; + } + + ModArray& operator-=(const ModArray& other) { + for (int i = 0; i < N; i++) { + a[i] -= other.a[i]; + if (a[i] < 0) { + a[i] += mod[i]; + } + } + return *this; + } + + ModArray& operator-=(long long v) { + return (operator+=)(-v); + } + + ModArray& mul_arr(const int other[]) { + for (int i = 0; i < N; i++) { + a[i] = (int)(((long long)a[i] * other[i]) % mod[i]); + } + return *this; + } + + ModArray& operator*=(const ModArray& other) { + return mul_arr(other.a); + } + + template + ModArray& operator*=(const ModArray& other) { + static_assert(M >= N); + return mul_arr(other.a); + } + + ModArray& operator*=(int v) { + for (int i = 0; i < N; i++) { + a[i] = (int)(((long long)a[i] * v) % mod[i]); + } + return (v >= 0 ? *this : norm_neg()); + } + + ModArray& operator*=(long long v) { + for (int i = 0; i < N; i++) { + a[i] = (int)(((long long)a[i] * (v % mod[i])) % mod[i]); + } + return (v >= 0 ? *this : norm_neg()); + } + + ModArray& mul_add(int v, long long w) { + for (int i = 0; i < N; i++) { + a[i] = (int)(((long long)a[i] * v + w) % mod[i]); + if (a[i] < 0) { + a[i] += mod[i]; + } + } + return *this; + } + + // *this = (*this * other) + w + ModArray& mul_add(const ModArray& other, long long w) { + for (int i = 0; i < N; i++) { + a[i] = (int)(((long long)a[i] * other.a[i] + w) % mod[i]); + if (a[i] < 0) { + a[i] += mod[i]; + } + } + return *this; + } + + // *this = (*this << shift) + w + ModArray& lshift_add(int shift, long long w) { + return mul_add(pow2(shift), w); + } + + // *this = *this + other * w + ModArray& add_mul(const ModArray& other, long long w) { + for (int i = 0; i < N; i++) { + a[i] = (int)((a[i] + other.a[i] * w) % mod[i]); + if (a[i] < 0) { + a[i] += mod[i]; + } + } + return *this; + } + + // *this += w << shift + ModArray& add_lshift(int shift, long long w) { + return add_mul(pow2(shift), w); + } + + ModArray operator+(const ModArray& other) const { + ModArray copy{*this}; + copy += other; + return copy; + } + + ModArray operator-(const ModArray& other) const { + ModArray copy{*this}; + copy -= other; + return copy; + } + + ModArray operator+(long long other) const { + ModArray copy{*this}; + copy += other; + return copy; + } + + ModArray operator-(long long other) const { + ModArray copy{*this}; + copy += -other; + return copy; + } + + ModArray operator-() const { + ModArray copy{*this}; + copy.negate(); + return copy; + } + + ModArray operator*(const ModArray& other) const { + ModArray copy{*this}; + copy *= other; + return copy; + } + + ModArray operator*(long long other) const { + ModArray copy{*this}; + copy *= other; + return copy; + } + + bool invert() { + for (int i = 0; i < N; i++) { + int t; + if (gcdx(a[i], mod[i], a[i], t) != 1) { + return false; + } + if (a[i] < 0) { + a[i] += mod[i]; + } + } + return true; + } + + bool try_divide(const ModArray& other) { + for (int i = 0; i < N; i++) { + int q, t; + if (gcdx(other.a[i], mod[i], q, t) != 1) { + return false; + } + a[i] = (int)((long long)a[i] * q % mod[i]); + if (a[i] < 0) { + a[i] += mod[i]; + } + } + return true; + } + + ModArray& operator/=(const ModArray& other) { + assert(try_divide(other) && "division by zero?"); + return *this; + } + + ModArray operator/(const ModArray& other) { + ModArray copy{*this}; + copy /= other; + return copy; + } + + static const ModArray& pow2(int power); + static const ModArray& negpow2(int power); + + ModArray& operator<<=(int lshift) { + return operator*=(pow2(lshift)); + } + + ModArray operator<<(int lshift) const { + return operator*(pow2(lshift)); + } + + ModArray& operator>>=(int rshift) { + return operator/=(pow2(rshift)); + } + + ModArray operator>>(int rshift) const { + return operator/(pow2(rshift)); + } + + template + const ModArray& as_shorter() const { + static_assert(M <= N); + return *reinterpret_cast*>(this); + } + + MixedRadix& to_mixed_radix(MixedRadix& dest, bool sgnd = true) const { + return dest.import_mod_array(a, sgnd); + } + + MixedRadix to_mixed_radix(bool sgnd = true) const { + return MixedRadix(*this, sgnd); + } + + int operator%(int div) const { + return to_mixed_radix() % div; + } + + explicit operator double() const { + return (double)to_mixed_radix(); + } + + std::string to_dec_string() const { + return MixedRadix(*this).to_dec_string(); + } + + std::ostream& print_dec(std::ostream& os, bool sgnd = true) const { + return MixedRadix(*this, sgnd).print_dec(os); + } + + bool to_binary(unsigned char* arr, int size, bool sgnd = true) const { + return MixedRadix(*this, sgnd).to_binary(arr, size, sgnd); + } + + template + bool to_binary(std::array& arr, bool sgnd = true) const { + return to_binary(arr.data(), M, sgnd); + } + + bool from_dec_string(const char* start, const char* end) { + set_zero(); + if (start >= end) { + return false; + } + bool sgn = (*start == '-'); + if (sgn && ++start == end) { + return false; + } + int acc = 0, pow = 1; + while (start < end) { + if (*start < '0' || *start > '9') { + return false; + } + acc = acc * 10 + (*start++ - '0'); + pow *= 10; + if (pow >= 1000000000) { + mul_add(pow, acc); + pow = 1; + acc = 0; + } + } + if (pow > 1) { + mul_add(pow, acc); + } + if (sgn) { + negate(); + } + return true; + } + + bool from_dec_string(std::string str) { + return from_dec_string(str.data(), str.data() + str.size()); + } + + ModArray& from_binary(const unsigned char* arr, int size, bool sgnd = true) { + set_zero(); + if (size <= 0) { + return *this; + } + int i = 0, pow = 0; + long long acc = (sgnd && arr[0] >= 0x80 ? -1 : 0); + while (i < size && arr[i] == (unsigned char)acc) { + i++; + } + for (; i < size; i++) { + pow += 8; + acc = (acc << 8) + arr[i]; + if (pow >= 56) { + lshift_add(pow, acc); + acc = pow = 0; + } + } + if (pow || acc) { + lshift_add(pow, acc); + } + return *this; + } + + template + ModArray& from_binary(const std::array& arr, bool sgnd = true) { + return from_binary(arr.data(), M, sgnd); + } + + std::ostream& raw_dump(std::ostream& os) const { + return raw_dump_array(os, a); + } + + ArrayRawDumpRef dump() const { + return {a}; + } +}; + +template +MixedRadix::MixedRadix(const ModArray& other) { + import_mod_array(other.a); +} + +template +MixedRadix::MixedRadix(const ModArray& other, bool sgnd) { + import_mod_array(other.a, sgnd); +} + +template +MixedRadix& MixedRadix::import_mod_array(const ModArray& other, bool sgnd) { + return import_mod_array(other.a, sgnd); +} + +template +std::ostream& operator<<(std::ostream& os, const ModArray& x) { + return x.print_dec(os); +} + +template +std::ostream& operator<<(std::ostream& os, const MixedRadix& x) { + return x.print_dec(os); +} + +template +std::ostream& operator<<(std::ostream& os, MixedRadix&& x) { + return x.print_dec_destroy(os); +} + +template +struct ArrayRawDumpRef { + const std::array& ref; + ArrayRawDumpRef(const std::array& _ref) : ref(_ref){}; +}; + +template +std::ostream& operator<<(std::ostream& os, ArrayRawDumpRef rd_ref) { + return raw_dump_array(os, rd_ref.ref); +}; + +constexpr int pow2_cnt = 1001; + +ModArray Zero(0), One(1), Pow2[pow2_cnt], NegPow2[pow2_cnt]; +MixedRadix Zero_mr(0), One_mr(1), Pow2_mr[pow2_cnt], NegPow2_mr[pow2_cnt]; + +template +const MixedRadix& MixedRadix::pow2(int power) { + return Pow2_mr[power].as_shorter(); +} + +/* +template +const MixedRadix& MixedRadix::negpow2(int power) { + return NegPow2_mr[power].as_shorter(); +} +*/ + +template +const ModArray& ModArray::pow2(int power) { + return Pow2[power].as_shorter(); +} + +template +const ModArray& ModArray::negpow2(int power) { + return NegPow2[power].as_shorter(); +} + +template +const ModArray& ModArray::zero() { + return Zero.as_shorter(); +} + +template +const ModArray& ModArray::one() { + return One.as_shorter(); +} + +template +const MixedRadix& MixedRadix::zero() { + return Zero_mr.as_shorter(); +} + +template +const MixedRadix& MixedRadix::one() { + return One_mr.as_shorter(); +} + +void init_pow2() { + Pow2[0].set_one(); + Pow2_mr[0].set_one(); + for (int i = 1; i < pow2_cnt; i++) { + Pow2[i].set_sum(Pow2[i - 1], Pow2[i - 1]); + Pow2_mr[i].set_sum(Pow2_mr[i - 1], Pow2_mr[i - 1]); + } + for (int i = 0; i < pow2_cnt; i++) { + NegPow2[i] = -Pow2[i]; + NegPow2_mr[i] = -Pow2_mr[i]; + } +} + +int gcdx(int a, int b, int& u, int& v) { + int a1 = 1, a2 = 0, b1 = 0, b2 = 1; + while (b) { + int q = a / b; + int t = a - q * b; + a = b; + b = t; + t = a1 - q * b1; + a1 = b1; + b1 = t; + t = a2 - q * b2; + a2 = b2; + b2 = t; + } + u = a1; + v = a2; + return a; +} + +void init_invm() { + for (int i = 0; i < mod_cnt; i++) { + assert(mod[i] > 0 && mod[i] <= (1 << 30)); + for (int j = 0; j < i; j++) { + assert(gcdx(mod[i], mod[j], invm[i][j], invm[j][i]) == 1); + if (invm[i][j] < 0) { + invm[i][j] += mod[j]; + } + if (invm[j][i] < 0) { + invm[j][i] += mod[i]; + } + } + } +} + +void init() { + init_invm(); + init_pow2(); +} + +} // namespace modint diff --git a/crypto/test/test-bigint.cpp b/crypto/test/test-bigint.cpp new file mode 100644 index 00000000..bf85e2ad --- /dev/null +++ b/crypto/test/test-bigint.cpp @@ -0,0 +1,876 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include +#include +#include +#include +#include +#include +#include +#include +#include "common/refcnt.hpp" +#include "common/bigint.hpp" +#include "common/refint.h" +#include "modbigint.cpp" + +#include "td/utils/tests.h" + +int mkint_chk_mode = -1, res_chk_mode = 0; +long long iterations = 100000, cur_iteration = -1, debug_iteration = -2; +#define IFDEBUG if (cur_iteration == debug_iteration || debug_iteration == -3) + +using BInt = modint::ModArray<18>; // integers up to 2^537 +using MRInt = modint::MixedRadix<18>; // auxiliary integer representation for printing, comparing etc + +MRInt p2_256, np2_256, p2_63, np2_63; +constexpr long long ll_min = -2 * (1LL << 62), ll_max = ~ll_min; +constexpr double dbl_pow256 = 1.1579208923731619542e77 /* 0x1p256 */; // 2^256 + +std::mt19937_64 Random(666); + +template +bool equal(td::RefInt256 x, T y) { + return !td::cmp(x, y); +} + +bool equal_or_nan(td::RefInt256 x, td::RefInt256 y) { + return equal(x, y) || (!x->is_valid() && !y->fits_bits(257)) || (!y->is_valid() && !x->fits_bits(257)); +} + +#define CHECK_EQ(__x, __y) CHECK(equal(__x, __y)) +#define CHECK_EQ_NAN(__x, __y) CHECK(equal_or_nan(__x, __y)) + +bool mr_in_range(const MRInt& x) { + return x < p2_256 && x >= np2_256; +} + +bool mr_is_small(const MRInt& x) { + return x < p2_63 && x >= np2_63; +} + +bool mr_fits_bits(const MRInt& x, int bits) { + if (bits > 0) { + return x < MRInt::pow2(bits - 1) && x >= MRInt::negpow2(bits - 1); + } else { + return !bits && !x.sgn(); + } +} + +bool mr_ufits_bits(const MRInt& x, int bits) { + return bits >= 0 && x.sgn() >= 0 && x < MRInt::pow2(bits); +} + +struct ShowBin { + unsigned char* data; + ShowBin(unsigned char _data[64]) : data(_data) { + } +}; + +std::ostream& operator<<(std::ostream& os, ShowBin bin) { + int i = 0, s = bin.data[0]; + if (s == 0 || s == 0xff) { + while (i < 64 && bin.data[i] == s) { + i++; + } + } + if (i >= 3) { + os << (s ? "ff..ff" : "00..00"); + } else { + i = 0; + } + constexpr static char hex_digits[] = "0123456789abcdef"; + while (i < 64) { + int t = bin.data[i++]; + os << hex_digits[t >> 4] << hex_digits[t & 15]; + } + return os; +} + +std::ostream& operator<<(std::ostream& os, const td::AnyIntView& x) { + os << '['; + for (int i = 0; i < x.size(); i++) { + os << ' ' << x.digits[i]; + } + os << " ]"; + return os; +} + +template +bool extract_value_any_bool(BInt& val, const td::AnyIntView& x, bool chk_norm = true) { + int n = x.size(); + if (n <= 0 || n > x.max_size() || (!x.digits[n - 1] && n > 1)) { + return false; + } + assert(n == 1 || x.digits[n - 1] != 0); + val.set_zero(); + for (int i = n - 1; i >= 0; --i) { + val.lshift_add(T::word_shift, x.digits[i]); + if (chk_norm && (x.digits[i] < -T::Half || x.digits[i] >= T::Half)) { + return false; // unnormalized + } + } + return true; +} + +template +bool extract_value_bool(BInt& val, const T& x, bool chk_norm = true) { + return extract_value_any_bool(val, x.as_any_int(), chk_norm); +} + +BInt extract_value_any(const td::AnyIntView& x, bool chk_norm = true) { + BInt res; + CHECK(extract_value_any_bool(res, x, chk_norm)); + return res; +} + +template +BInt extract_value(const T& x, bool chk_norm = true) { + return extract_value_any(x.as_any_int(), chk_norm); +} + +template +BInt extract_value_alt(const T& x) { + BInt res; + const int* md = res.mod_array(); + for (int i = 0; i < res.n / 2; i++) { + T copy{x}; + int m1 = md[2 * i], m2 = md[2 * i + 1]; + long long rem = copy.divmod_short((long long)m1 * m2); + res.a[2 * i] = (int)(rem % m1); + res.a[2 * i + 1] = (int)(rem % m2); + } + if (res.n & 1) { + T copy{x}; + res.a[res.n - 1] = (int)copy.divmod_short(md[res.n - 1]); + } + return res; +} + +constexpr int min_spec_int = -0xfd08, max_spec_int = 0xfd07; +// x = sgn*(ord*256+a*16+b) => sgn*((32+a)*2^(ord-2) + b - 8) +// x = -0xfd08 => -2^256 ... x = 0xfd07 => 2^256 - 1 +td::RefInt256 make_special_int(int x, BInt* ptr = nullptr, unsigned char bin[64] = nullptr) { + bool sgn = (x < 0); + if (sgn) { + x = -x; + } + int ord = (x >> 8) - 2, a = 32 + ((x >> 4) & 15), b = (x & 15) - 8; + if (ord < 0) { + a >>= -ord; + ord = 0; + } + if (sgn) { + a = -a; + b = -b; + } + if (ptr) { + ptr->set_int(a); + *ptr <<= ord; + *ptr += b; + } + if (bin) { + int acc = b, r = ord; + for (int i = 63; i >= 0; --i) { + if (r < 8) { + acc += (a << r); + r = 1024; + } + r -= 8; + bin[i] = (unsigned char)(acc & 0xff); + acc >>= 8; + } + } + return (td::make_refint(a) << ord) + b; +} + +int rand_int(int min, int max) { + return min + (int)(Random() % (max - min + 1)); +} + +unsigned randu() { + return (unsigned)(Random() << 16); +} + +bool coin() { + return Random() & (1 << 28); +} + +// returns 0 with probability 1/2, 1 with prob. 1/4, ..., k with prob. 1/2^(k+1) +int randexp(int max = 63, int min = 0) { + return min + __builtin_clzll(Random() | (1ULL << (63 - max + min))); +} + +void bin_add_small(unsigned char bin[64], long long val, int shift = 0) { + val <<= shift & 7; + for (int i = 63 - (shift >> 3); i >= 0 && val; --i) { + val += bin[i]; + bin[i] = (unsigned char)val; + val >>= 8; + } +} + +// adds sgn * (random number less than 2^(ord - ord2)) * 2^ord2 +td::RefInt256 add_random_bits(td::RefInt256 x, BInt& val, unsigned char bin[64], int ord2, int ord, int sgn = 1) { + int t; + do { + t = std::max((ord - 1) & -16, ord2); + int a = sgn * rand_int(0, (1 << (ord - t)) - 1); + // add a << t + val.add_lshift(t, a); + x += td::make_refint(a) << t; + bin_add_small(bin, a, t); + ord = t; + } while (t > ord2); + return x; +} + +// generates a random integer in range -2^256 .. 2^256-1 (and sometimes outside) +// distribution is skewed towards +/- 2^n +/- 2^n +/- smallint, but completely random integers are also generated +td::RefInt256 make_random_int0(BInt& val, unsigned char bin[64]) { + memset(bin, 0, 64); + int ord = rand_int(-257, 257); + if (ord <= 2 && ord >= -2) { + // -2..2 represent themselves + val.set_int(ord); + bin_add_small(bin, ord); + return td::make_refint(ord); + } + int sgn = (ord < 0 ? -1 : 1); + ord = sgn * ord - 1; + int f = std::min(ord, randexp(15)), a = sgn * rand_int(1 << f, (2 << f) - 1); + ord -= f; + // first summand is a << ord + auto res = td::make_refint(a) << ord; + val.set_int(a); + val <<= ord; + bin_add_small(bin, a, ord); + if (!ord) { + // all bits ready + return res; + } + for (int s = 0; s < 2 && ord; s++) { + // decide whether we want an intermediate order (50%), and whether we want randomness above/below that order + int ord2 = (s ? 0 : std::max(0, rand_int(~ord, ord - 1))); + if (!rand_int(0, 4)) { // 20% + // random bits between ord2 and ord + res = add_random_bits(std::move(res), val, bin, ord2, ord, sgn); + } + if (rand_int(0, 4)) { // 80% + // non-zero adjustment + f = randexp(15); + a = rand_int(-(2 << f) + 1, (2 << f) - 1); + ord = std::max(ord2 - f, 0); + // add a << ord + val.add_lshift(ord, a); + res += (td::make_refint(a) << ord); + bin_add_small(bin, a, ord); + } + } + return res; +} + +td::RefInt256 make_random_int(BInt& val, unsigned char bin[64]) { + while (true) { + auto res = make_random_int0(val, bin); + if (res->fits_bits(257)) { + return res; + } + } +} + +void check_one_int_repr(td::RefInt256 x, int mode, int in_range, const BInt* valptr = nullptr, + const unsigned char bin[64] = nullptr) { + CHECK(x.not_null() && (in_range <= -2 || x->is_valid())); + if (!x->is_valid()) { + // not much to check when x is a NaN + unsigned char bytes[64]; + if (valptr) { + // check that the true answer at `valptr` is out of range + CHECK(!mr_in_range(valptr->to_mixed_radix())); + if (mode & 0x200) { + // check BInt binary export + valptr->to_binary(bytes, 64); + if (bin) { + // check that the two true answers match + CHECK(!memcmp(bin, bytes, 64)); + } else { + bin = bytes; + } + } + } + if (bin) { + // check that the true answer in `bin` is out of range + int i = 0, sgn = (bin[0] >= 0x80 ? -1 : 0); + while (i < 32 && bin[i] == (unsigned char)sgn) + ; + CHECK(i < 32); + if (valptr && (mode & 0x100)) { + // check BInt binary export + BInt val2; + val2.from_binary(bin, 64); + CHECK(*valptr == val2); + } + } + return; + } + unsigned char bytes[64]; + CHECK(x->export_bytes(bytes, 64)); + if (bin) { + CHECK(!memcmp(bytes, bin, 64)); + } + BInt val = extract_value(*x); + if (valptr) { + if (val != *valptr) { + std::cerr << "extracted " << val << " from " << x << ' ' << x->as_any_int() << ", expected " << *valptr + << std::endl; + } + CHECK(val == *valptr); + } + if (mode & 1) { + BInt val2 = extract_value_alt(*x); + CHECK(val == val2); + } + if (mode & 2) { + // check binary import + td::BigInt256 y; + y.import_bytes(bytes, 64); + CHECK(y == *x); + } + if (mode & 0x100) { + // check binary import for BInt + BInt val2; + val2.from_binary(bytes, 64); + CHECK(val == val2); + } + // check if small (fits into 64 bits) + long long xval = (long long)val; + bool is_small = (xval != ll_min || val == xval); + CHECK(is_small == x->fits_bits(64)); + if (is_small) { + // special check for small (64-bit) values + CHECK(x->to_long() == xval); + CHECK((long long)__builtin_bswap64(*(long long*)(bytes + 64 - 8)) == xval); + CHECK(in_range); + // check sign + CHECK(x->sgn() == (xval > 0 ? 1 : (xval < 0 ? -1 : 0))); + // check comparison with long long + CHECK(x == xval); + CHECK(!cmp(x, xval)); + if (mode & 4) { + // check constructor from long long + CHECK(!cmp(x, td::make_refint(xval))); + if (xval != ll_min) { + CHECK(x > xval - 1); + CHECK(x > td::make_refint(xval - 1)); + } + if (xval != ll_max) { + CHECK(x < xval + 1); + CHECK(x < td::make_refint(xval + 1)); + } + } + if (!(mode & ~0x107)) { + return; // fast check for small ints in this case + } + } + + MRInt mval(val); // somewhat slow + bool val_in_range = mr_in_range(mval); + CHECK(x->fits_bits(257) == val_in_range); + if (in_range >= 0) { + CHECK((int)val_in_range == in_range); + } + if (mode & 0x200) { + // check binary export for BInt + unsigned char bytes2[64]; + mval.to_binary(bytes2, 64); + CHECK(!memcmp(bytes, bytes2, 64)); + } + // check sign + int sgn = mval.sgn(); + CHECK(x->sgn() == sgn); + CHECK(is_small == mr_is_small(mval)); + if (is_small) { + CHECK((long long)mval == xval); + } + if (mode & 0x10) { + // check decimal export + std::string dec = mval.to_dec_string(); + CHECK(x->to_dec_string() == dec); + // check decimal import + td::BigInt256 y; + int l = y.parse_dec(dec); + CHECK((std::size_t)l == dec.size() && y == *x); + if (mode & 0x1000) { + // check decimal import for BInt + BInt val2; + CHECK(val2.from_dec_string(dec) && val2 == val); + } + } + if (mode & 0x20) { + // check binary bit size + int sz = x->bit_size(); + CHECK(sz >= 0 && sz <= 300); + CHECK(x->fits_bits(sz) && (!sz || !x->fits_bits(sz - 1))); + CHECK(mr_fits_bits(mval, sz) && !mr_fits_bits(mval, sz - 1)); + int usz = x->bit_size(false); + CHECK(sgn >= 0 || usz == 0x7fffffff); + if (sgn >= 0) { + CHECK(x->unsigned_fits_bits(usz) && (!usz || !x->unsigned_fits_bits(usz - 1))); + CHECK(mr_ufits_bits(mval, usz) && !mr_ufits_bits(mval, usz - 1)); + } else { + CHECK(!x->unsigned_fits_bits(256) && !x->unsigned_fits_bits(300)); + } + } +} + +void init_aux() { + np2_256 = p2_256 = MRInt::pow2(256); + np2_256.negate(); + CHECK(np2_256 == MRInt::negpow2(256)); + p2_63 = np2_63 = MRInt::pow2(63); + np2_63.negate(); + CHECK(np2_63 == MRInt::negpow2(63)); +} + +std::vector SpecInt; +BInt SpecIntB[max_spec_int - min_spec_int + 1]; + +void init_check_special_ints() { + std::cerr << "check special ints" << std::endl; + BInt b; + unsigned char binary[64]; + for (int idx = min_spec_int - 512; idx <= max_spec_int + 512; idx++) { + td::RefInt256 x = make_special_int(idx, &b, binary); + check_one_int_repr(x, mkint_chk_mode, idx >= min_spec_int && idx <= max_spec_int, &b, binary); + if (idx >= min_spec_int && idx <= max_spec_int) { + SpecIntB[idx - min_spec_int] = b; + SpecInt.push_back(std::move(x)); + } + } +} + +void check_res(td::RefInt256 y, const BInt& yv) { + check_one_int_repr(std::move(y), res_chk_mode, -2, &yv); +} + +void check_unary_ops_on(td::RefInt256 x, const BInt& xv) { + // NEGATE + BInt yv = -xv; + check_res(-x, yv); + // NOT + check_res(~x, yv -= 1); +} + +void check_unary_ops() { + std::cerr << "check unary ops" << std::endl; + for (int idx = min_spec_int; idx <= max_spec_int; idx++) { + check_unary_ops_on(SpecInt[idx - min_spec_int], SpecIntB[idx - min_spec_int]); + } +} + +void check_pow2_ops(int shift) { + // POW2 + td::RefInt256 r{true}; + r.unique_write().set_pow2(shift); + check_res(r, BInt::pow2(shift)); + // POW2DEC + r.unique_write().set_pow2(shift).add_tiny(-1).normalize(); + check_res(r, BInt::pow2(shift) - 1); + // NEGPOW2 + r.unique_write().set_pow2(shift).negate().normalize(); + check_res(r, -BInt::pow2(shift)); +} + +void check_pow2_ops() { + std::cerr << "check power-2 ops" << std::endl; + for (int i = 0; i <= 256; i++) { + check_pow2_ops(i); + } +} + +void check_shift_ops_on(int shift, td::RefInt256 x, const BInt& xv, const MRInt& mval) { + // LSHIFT + check_res(x << shift, xv << shift); + // FITS + CHECK(x->fits_bits(shift) == mr_fits_bits(mval, shift)); + // UFITS + CHECK(x->unsigned_fits_bits(shift) == mr_ufits_bits(mval, shift)); + // ADDPOW2 / SUBPOW2 + auto y = x; + y.write().add_pow2(shift).normalize(); + check_res(std::move(y), xv + BInt::pow2(shift)); + y = x; + y.write().sub_pow2(shift).normalize(); + check_res(std::move(y), xv - BInt::pow2(shift)); + // RSHIFT, MODPOW2 + for (int round_mode = -1; round_mode <= 1; round_mode++) { + auto r = x, q = td::rshift(x, shift, round_mode); // RSHIFT + CHECK(q.not_null() && q->is_valid()); + r.write().mod_pow2(shift, round_mode).normalize(); // MODPOW2 + CHECK(r.not_null() && r->is_valid()); + if (round_mode < 0) { + CHECK(!cmp(x >> shift, q)); // operator>> should be equivalent to td::rshift + } + BInt qv = extract_value(*q), rv = extract_value(*r); + // check main division equality (q << shift) + r == x + CHECK((qv << shift) + rv == xv); + MRInt rval(rv); + // check remainder range + switch (round_mode) { + case 1: + rval.negate(); // fallthrough + case -1: + CHECK(mr_ufits_bits(rval, shift)); + break; + case 0: + CHECK(mr_fits_bits(rval, shift)); + } + } +} + +void check_shift_ops() { + std::cerr << "check left/right shift ops" << std::endl; + for (int idx = min_spec_int; idx <= max_spec_int; idx++) { + //for (int idx : {-52240, -52239, -52238, -3, -2, -1, 0, 1, 2, 3, 52238, 52239, 52240}) { + const auto& xv = SpecIntB[idx - min_spec_int]; + MRInt mval(xv); + if (!(idx % 1000)) { + std::cerr << "# " << idx << " : " << mval << std::endl; + } + for (int i = 0; i <= 256; i++) { + check_shift_ops_on(i, SpecInt[idx - min_spec_int], xv, mval); + } + } +} + +void check_remainder_range(BInt& rv, const BInt& dv, int rmode = -1) { + if (rmode > 0) { + rv.negate(); + } else if (!rmode) { + rv *= 2; + } + MRInt d(dv), r(rv); + int ds = d.sgn(), rs = r.sgn(); + //std::cerr << "rmode=" << rmode << " ds=" << ds << " rs=" << rs << " d=" << d << " r=" << r << std::endl; + if (!rs) { + return; + } + if (rmode) { + // must have 0 < r < d or 0 > r > d + //if (rs != ds) std::cerr << "iter=" << cur_iteration << " : rmode=" << rmode << " ds=" << ds << " rs=" << rs << " d=" << d << " r=" << r << std::endl; + CHECK(rs == ds); + CHECK(ds * r.cmp(d) < 0); + } else { + // must have -d <= r < d or -d >= r > d + if (rs == -ds) { + r.negate(); + CHECK(ds * r.cmp(d) <= 0); + } else { + CHECK(ds * r.cmp(d) < 0); + } + } +} + +void check_divmod(td::RefInt256 x, const BInt& xv, long long xl, td::RefInt256 y, const BInt& yv, long long yl, + int rmode = -2) { + if (rmode < -1) { + //IFDEBUG std::cerr << " divide " << x << " / " << y << std::endl; + for (rmode = -1; rmode <= 1; rmode++) { + check_divmod(x, xv, xl, y, yv, yl, rmode); + } + return; + } + auto dm = td::divmod(x, y, rmode); + auto q = std::move(dm.first), r = std::move(dm.second); + if (!yl) { + // division by zero + CHECK(q.not_null() && !q->is_valid() && r.not_null() && !r->is_valid()); + return; + } + CHECK(q.not_null() && q->is_valid() && r.not_null() && r->is_valid()); + CHECK_EQ(x, y * q + r); + BInt qv = extract_value(*q), rv = extract_value(*r); + CHECK(xv == yv * qv + rv); + //IFDEBUG std::cerr << " quot=" << q << " rem=" << r << std::endl; + check_remainder_range(rv, yv, rmode); + if (yl != ll_min && rmode == -1) { + // check divmod_short() + auto qq = x; + auto rem = qq.write().divmod_short(yl); + qq.write().normalize(); + CHECK(qq->is_valid()); + CHECK_EQ(qq, q); + CHECK(r == rem); + if (xl != ll_min) { + auto dm = std::lldiv(xl, yl); + if (dm.rem && (dm.rem ^ yl) < 0) { + dm.rem += yl; + dm.quot--; + } + CHECK(q == dm.quot); + CHECK(r == dm.rem); + } + } +} + +void check_binary_ops_on(td::RefInt256 x, const BInt& xv, td::RefInt256 y, const BInt& yv) { + bool x_small = x->fits_bits(62), y_small = y->fits_bits(62); // not 63 + long long xl = x_small ? x->to_long() : ll_min, yl = y_small ? y->to_long() : ll_min; + if (x_small) { + CHECK(x == xl); + } + if (y_small) { + CHECK(y == yl); + } + // ADD, ADDR + auto z = x + y, w = y + x; + CHECK_EQ(z, w); + check_res(z, xv + yv); + // ADDCONST + if (y_small) { + CHECK_EQ(z, x + yl); + } + if (x_small) { + CHECK_EQ(z, y + xl); + } + if (x_small && y_small) { + CHECK_EQ(z, xl + yl); + } + // SUB + z = x - y; + check_res(z, xv - yv); + // SUBCONST + if (y_small) { + CHECK_EQ(z, x - yl); + if (x_small) { + CHECK_EQ(z, xl - yl); + } + } + // SUBR + z = y - x; + check_res(z, yv - xv); + if (x_small) { + CHECK_EQ(z, y - xl); + if (y_small) { + CHECK_EQ(z, yl - xl); + } + } + // CMP + MRInt xmr(xv), ymr(yv); + int cmpv = xmr.cmp(ymr); + CHECK(td::cmp(x, y) == cmpv); + CHECK(td::cmp(y, x) == -cmpv); + if (y_small) { + CHECK(td::cmp(x, yl) == cmpv); + } + if (x_small) { + CHECK(td::cmp(y, xl) == -cmpv); + } + if (x_small && y_small) { + CHECK(cmpv == (xl < yl ? -1 : (xl > yl ? 1 : 0))); + } + // MUL + z = x * y; + BInt zv = xv * yv; + check_res(z, zv); + CHECK_EQ(z, y * x); + // MULCONST + if (y_small) { + CHECK_EQ_NAN(z, x * yl); + } + if (x_small) { + CHECK_EQ_NAN(z, y * xl); + } + if (x_small && y_small && (!yl || std::abs(xl) <= ll_max / std::abs(yl))) { + CHECK_EQ(z, xl * yl); + } + // DIVMOD + if (z->fits_bits(257)) { + int adj = 2 * rand_int(-2, 2) - (int)z->is_odd(); + z += adj; + z >>= 1; + zv += adj; + zv >>= 1; + // z is approximately x * y / 2; divide by y + check_divmod(z, zv, z->fits_bits(62) ? z->to_long() : ll_min, y, yv, yl); + } + check_divmod(x, xv, xl, y, yv, yl); +} + +void finish_check_muldivmod(td::RefInt256 x, const BInt& xv, td::RefInt256 y, const BInt& yv, td::RefInt256 z, + const BInt& zv, td::RefInt256 q, td::RefInt256 r, int rmode) { + static constexpr double eps = 1e-14; + CHECK(q.not_null() && r.not_null()); + //std::cerr << " muldivmod: " << xv << " * " << yv << " / " << zv << " (round " << rmode << ") = " << q << " " << r << std::endl; + if (!zv) { + // division by zero + CHECK(!q->is_valid() && !r->is_valid()); + return; + } + CHECK(r->is_valid()); // remainder always exists if y != 0 + BInt xyv = xv * yv, rv = extract_value(*r); + MRInt xy_mr(xyv), z_mr(zv); + double q0 = (double)xy_mr / (double)z_mr; + if (std::abs(q0) < 1.01 * dbl_pow256) { + // result more or less in range + CHECK(q->is_valid()); + } else if (!q->is_valid()) { + // result out of range, NaN is an acceptable answer + // check that x * y - r is divisible by z + xyv -= rv; + xyv /= zv; + xy_mr = xyv; + double q1 = (double)xy_mr; + CHECK(std::abs(q1 - q0) < eps * std::abs(q0)); + } else { + BInt qv = extract_value(*q); + // must have x * y = z * q + r + CHECK(xv * yv == zv * qv + rv); + } + // check that r is in correct range [0, z) or [0, -z) or [-z/2, z/2) + check_remainder_range(rv, zv, rmode); +} + +void check_muldivmod_on(td::RefInt256 x, const BInt& xv, td::RefInt256 y, const BInt& yv, td::RefInt256 z, + const BInt& zv, int rmode = 2) { + if (rmode < -1) { + for (rmode = -1; rmode <= 1; rmode++) { + check_muldivmod_on(x, xv, y, yv, z, zv, rmode); + } + return; + } else if (rmode > 1) { + rmode = rand_int(-1, 1); + } + // MULDIVMOD + auto qr = td::muldivmod(x, y, z, rmode); + finish_check_muldivmod(std::move(x), xv, std::move(y), yv, std::move(z), zv, std::move(qr.first), + std::move(qr.second), rmode); +} + +void check_mul_rshift_on(td::RefInt256 x, const BInt& xv, td::RefInt256 y, const BInt& yv, int shift, int rmode = 2) { + if (rmode < -1) { + for (rmode = -1; rmode <= 1; rmode++) { + check_mul_rshift_on(x, xv, y, yv, shift, rmode); + } + return; + } else if (rmode > 1) { + rmode = rand_int(-1, 1); + } + // MULRSHIFTMOD + typename td::BigInt256::DoubleInt tmp{0}; + tmp.add_mul(*x, *y); + typename td::BigInt256::DoubleInt tmp2{tmp}; + tmp2.rshift(shift, rmode).normalize(); + tmp.normalize().mod_pow2(shift, rmode).normalize(); + finish_check_muldivmod(std::move(x), xv, std::move(y), yv, {}, BInt::pow2(shift), td::make_refint(tmp2), + td::make_refint(tmp), rmode); +} + +void check_lshift_div_on(td::RefInt256 x, const BInt& xv, td::RefInt256 y, const BInt& yv, int shift, int rmode = 2) { + if (rmode < -1) { + for (rmode = -1; rmode <= 1; rmode++) { + check_lshift_div_on(x, xv, y, yv, shift, rmode); + } + return; + } else if (rmode > 1) { + rmode = rand_int(-1, 1); + } + // LSHIFTDIV + typename td::BigInt256::DoubleInt tmp{*x}, quot; + tmp <<= shift; + tmp.mod_div(*y, quot, rmode); + quot.normalize(); + finish_check_muldivmod(std::move(x), xv, {}, BInt::pow2(shift), std::move(y), yv, td::make_refint(quot), + td::make_refint(tmp), rmode); +} + +void check_random_ops() { + constexpr long long chk_it = 100000; + std::cerr << "check random ops (" << iterations << " iterations)" << std::endl; + BInt xv, yv, zv; + unsigned char xbin[64], ybin[64], zbin[64]; + for (cur_iteration = 0; cur_iteration < iterations; cur_iteration++) { + auto x = make_random_int0(xv, xbin); + if (!(cur_iteration % 10000)) { + std::cerr << "#" << cur_iteration << ": check on " << xv << " = " << ShowBin(xbin) << " = " << x->as_any_int() + << std::endl; + } + check_one_int_repr(x, cur_iteration < chk_it ? -1 : 0, -1, &xv, xbin); + MRInt xmr(xv); + if (!x->fits_bits(257)) { + continue; + } + check_unary_ops_on(x, xv); + for (int j = 0; j < 10; j++) { + int shift = rand_int(0, 256); + //std::cerr << "check shift by " << shift << std::endl; + check_shift_ops_on(shift, x, xv, xmr); + auto y = make_random_int(yv, ybin); + //std::cerr << " y = " << y << " = " << yv << " = " << ShowBin(ybin) << " = " << y->as_any_int() << std::endl; + check_one_int_repr(y, 0, 1, &yv, ybin); + check_binary_ops_on(x, xv, y, yv); + //std::cerr << " *>> " << shift << std::endl; + check_mul_rshift_on(x, xv, y, yv, shift); + //std::cerr << " <as_any_int() << std::endl; + check_muldivmod_on(x, xv, y, yv, z, zv); + } + } +} + +void check_special() { + std::cerr << "run special tests" << std::endl; + check_divmod((td::make_refint(-1) << 207) - 1, BInt::negpow2(207) - 1, ll_min, (td::make_refint(1) << 207) - 1, + BInt::pow2(207) - 1, ll_min); +} + +int main(int argc, char* const argv[]) { + bool do_check_shift_ops = false; + int i; + while ((i = getopt(argc, argv, "hSs:i:")) != -1) { + switch (i) { + case 'S': + do_check_shift_ops = true; + break; + case 's': + Random.seed(atoll(optarg)); + break; + case 'i': + iterations = atoll(optarg); + break; + default: + std::cerr << "unknown option: " << (char)i << std::endl; + // fall through + case 'h': + std::cerr << "usage:\t" << argv[0] << " [-S] [-i] [-s]" << std::endl; + return 2; + } + } + modint::init(); + init_aux(); + init_check_special_ints(); + check_pow2_ops(); + check_unary_ops(); + if (do_check_shift_ops) { + check_shift_ops(); + } + check_special(); + check_random_ops(); + return 0; +} diff --git a/crypto/vm/arithops.cpp b/crypto/vm/arithops.cpp index 823b4408..2831944b 100644 --- a/crypto/vm/arithops.cpp +++ b/crypto/vm/arithops.cpp @@ -387,18 +387,16 @@ int exec_muldivmod(VmState* st, unsigned args, int quiet) { auto z = stack.pop_int(); auto y = stack.pop_int(); auto x = stack.pop_int(); - typename td::BigInt256::DoubleInt tmp{0}; + typename td::BigInt256::DoubleInt tmp{0}, quot; tmp.add_mul(*x, *y); auto q = td::make_refint(); - tmp.mod_div(*z, q.unique_write(), round_mode); + tmp.mod_div(*z, quot, round_mode); switch ((args >> 2) & 3) { case 1: - q.unique_write().normalize(); - stack.push_int_quiet(std::move(q), quiet); + stack.push_int_quiet(td::make_refint(quot.normalize()), quiet); break; case 3: - q.unique_write().normalize(); - stack.push_int_quiet(std::move(q), quiet); + stack.push_int_quiet(td::make_refint(quot.normalize()), quiet); // fallthrough case 2: stack.push_int_quiet(td::make_refint(tmp), quiet); @@ -459,7 +457,7 @@ int exec_mulshrmod(VmState* st, unsigned args, int mode) { } // fallthrough case 2: - tmp.mod_pow2(z, round_mode).normalize(); + tmp.normalize().mod_pow2(z, round_mode).normalize(); stack.push_int_quiet(td::make_refint(tmp), mode & 1); break; } @@ -520,21 +518,17 @@ int exec_shldivmod(VmState* st, unsigned args, int mode) { } auto z = stack.pop_int(); auto x = stack.pop_int(); - typename td::BigInt256::DoubleInt tmp{*x}; + typename td::BigInt256::DoubleInt tmp{*x}, quot; tmp <<= y; switch ((args >> 2) & 3) { case 1: { - auto q = td::make_refint(); - tmp.mod_div(*z, q.unique_write(), round_mode); - q.unique_write().normalize(); - stack.push_int_quiet(std::move(q), mode & 1); + tmp.mod_div(*z, quot, round_mode); + stack.push_int_quiet(td::make_refint(quot.normalize()), mode & 1); break; } case 3: { - auto q = td::make_refint(); - tmp.mod_div(*z, q.unique_write(), round_mode); - q.unique_write().normalize(); - stack.push_int_quiet(std::move(q), mode & 1); + tmp.mod_div(*z, quot, round_mode); + stack.push_int_quiet(td::make_refint(quot.normalize()), mode & 1); stack.push_int_quiet(td::make_refint(tmp), mode & 1); break; } diff --git a/doc/tvm.tex b/doc/tvm.tex index 4f887ac2..71aa7abc 100644 --- a/doc/tvm.tex +++ b/doc/tvm.tex @@ -1531,6 +1531,7 @@ Examples: \item {\tt A934$tt$} --- same as {\tt RSHIFT $tt+1$}: ($x$ -- $\lfloor x\cdot 2^{-tt-1}\rfloor$). \item {\tt A938$tt$} --- {\tt MODPOW2 $tt+1$}: ($x$ -- $x\bmod 2^{tt+1}$). \item {\tt A985} --- {\tt MULDIVR} ($x$ $y$ $z$ -- $q'$), where $q'=\lfloor xy/z+1/2\rfloor$. +\item {\tt A988} --- {\tt MULMOD} ($x$ $y$ $z$ -- $r$), where $r=xy\bmod z=xy-qz$, $q=\lfloor xy/z\rfloor$. This operation always succeeds for $z\neq0$ and returns the correct value of~$r$, even if the intermediate result $xy$ or the quotient $q$ do not fit into 257 bits. \item {\tt A98C} --- {\tt MULDIVMOD} ($x$ $y$ $z$ -- $q$ $r$), where $q:=\lfloor x\cdot y/z\rfloor$, $r:=x\cdot y\bmod z$ (same as {\tt */MOD} in Forth). \item {\tt A9A4} --- {\tt MULRSHIFT} ($x$ $y$ $z$ -- $\lfloor xy\cdot2^{-z}\rfloor$) for $0\leq z\leq 256$. \item {\tt A9A5} --- {\tt MULRSHIFTR} ($x$ $y$ $z$ -- $\lfloor xy\cdot2^{-z}+1/2\rfloor$) for $0\leq z\leq 256$. @@ -1576,10 +1577,12 @@ We opted to make all arithmetic operations ``non-quiet'' (signaling) by default, \begin{itemize} \item {\tt B7xx} --- {\tt QUIET} prefix, transforming any arithmetic operation into its ``quiet'' variant, indicated by prefixing a {\tt Q} to its mnemonic. Such operations return {\tt NaN}s instead of throwing integer overflow exceptions if the results do not fit in {\it Integer\/}s, or if one of their arguments is a {\tt NaN}. Notice that this does not extend to shift amounts and other parameters that must be within a small range (e.g., 0--1023). Also notice that this does not disable type-checking exceptions if a value of a type other than {\it Integer\/} is supplied. \item {\tt B7A0} --- {\tt QADD} ($x$ $y$ -- $x+y$), always works if $x$ and $y$ are {\it Integer\/}s, but returns a {\tt NaN} if the addition cannot be performed. +\item {\tt B7A8} --- {\tt QMUL} ($x$ $y$ -- $xy$), returns the product of $x$ and $y$ if $-2^{256}\leq xy<2^{256}$. Otherwise returns a {\tt NaN}, even if $x=0$ and $y$ is a {\tt NaN}. \item {\tt B7A904} --- {\tt QDIV} ($x$ $y$ -- $\lfloor x/y\rfloor$), returns a {\tt NaN} if $y=0$, or if $y=-1$ and $x=-2^{256}$, or if either of $x$ or $y$ is a {\tt NaN}. +\item {\tt B7A98C} --- {\tt QMULDIVMOD} ($x$ $y$ $z$ -- $q$ $r$), where $q:=\lfloor x\cdot y/z\rfloor$, $r:=x\cdot y\bmod z$. If $z=0$, or if at least one of $x$, $y$, or $z$ is a {\tt NaN}, both $q$ and $r$ are set to {\tt NaN}. Otherwise the correct value of $r$ is always returned, but $q$ is replaced with {\tt NaN} if $q<-2^{256}$ or $q\geq2^{256}$. \item {\tt B7B0} --- {\tt QAND} ($x$ $y$ -- $x\&y$), bitwise ``and'' (similar to {\tt AND}), but returns a {\tt NaN} if either $x$ or $y$ is a {\tt NaN} instead of throwing an integer overflow exception. However, if one of the arguments is zero, and the other is a {\tt NaN}, the result is zero. \item {\tt B7B1} --- {\tt QOR} ($x$ $y$ -- $x\vee y$), bitwise ``or''. If $x=-1$ or $y=-1$, the result is always $-1$, even if the other argument is a {\tt NaN}. -\item {\tt B7B507} --- {\tt QUFITS 8} ($x$ -- $x'$), checks whether $x$ is an unsigned byte (i.e., whether $0\leq x<2^8$), and replaces $x$ with a {\tt NaN} if this is not the case; leaves $x$ intact otherwise (i.e., if $x$ is an unsigned byte). +\item {\tt B7B507} --- {\tt QUFITS 8} ($x$ -- $x'$), checks whether $x$ is an unsigned byte (i.e., whether $0\leq x<2^8$), and replaces $x$ with a {\tt NaN} if this is not the case; leaves $x$ intact otherwise (i.e., if $x$ is an unsigned byte or a {\tt NaN}). \end{itemize} \mysubsection{Comparison primitives} diff --git a/rldp2/CMakeLists.txt b/rldp2/CMakeLists.txt index f93f6e1e..33ae5b67 100644 --- a/rldp2/CMakeLists.txt +++ b/rldp2/CMakeLists.txt @@ -51,7 +51,7 @@ target_include_directories(rldp PUBLIC ${OPENSSL_INCLUDE_DIR} ) if (GSL_FOUND) - target_link_libraries(rldp2 PRIVATE GSL::gsl) + target_link_libraries(rldp2 PRIVATE gsl) target_compile_definitions(rldp2 PRIVATE -DTON_HAVE_GSL=1) endif() target_link_libraries(rldp2 PUBLIC tdutils tdactor fec adnl tl_api) From 1b480e39bae46fa7379f4442d0a75bb0cf3c46a6 Mon Sep 17 00:00:00 2001 From: OmicronTau Date: Thu, 23 Dec 2021 10:39:14 +0300 Subject: [PATCH 06/21] Fix infinity max_gas_limit for get methods --- crypto/smc-envelope/SmartContract.cpp | 2 +- validator/impl/liteserver.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crypto/smc-envelope/SmartContract.cpp b/crypto/smc-envelope/SmartContract.cpp index f3485be9..5f284b57 100644 --- a/crypto/smc-envelope/SmartContract.cpp +++ b/crypto/smc-envelope/SmartContract.cpp @@ -207,7 +207,7 @@ SmartContract::Answer SmartContract::run_get_method(Args args) const { args.c7 = prepare_vm_c7(now, args.balance); } if (!args.limits) { - args.limits = vm::GasLimits{1000000}; + args.limits = vm::GasLimits{1000000, 1000000}; } if (!args.stack) { args.stack = td::Ref(true); diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index 510f4906..6b0b9008 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -1150,7 +1150,7 @@ void LiteQuery::finish_runSmcMethod(td::BufferSlice shard_proof, td::BufferSlice long long gas_limit = client_method_gas_limit; LOG(DEBUG) << "creating VM with gas limit " << gas_limit; // **** INIT VM **** - vm::GasLimits gas{gas_limit}; + vm::GasLimits gas{gas_limit, gas_limit}; vm::VmState vm{std::move(code), std::move(stack_), gas, 1, std::move(data), vm::VmLog::Null()}; auto c7 = prepare_vm_c7(gen_utime, gen_lt, td::make_ref(acc.addr->clone()), balance); vm.set_c7(c7); // tuple with SmartContractInfo From 59fcd7cbd55412d26746a3a838df399cbfb4c4a7 Mon Sep 17 00:00:00 2001 From: OmicronTau Date: Fri, 31 Dec 2021 18:48:51 +0300 Subject: [PATCH 07/21] Deactivate accounts upon destruction in block --- crypto/block/transaction.cpp | 52 ++++++++++++++++++++++--------- crypto/block/transaction.h | 11 ++++--- validator/impl/collator.cpp | 9 ------ validator/impl/validate-query.cpp | 3 +- 4 files changed, 45 insertions(+), 30 deletions(-) diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index 3668a45f..e5cd701d 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -271,7 +271,7 @@ bool Account::recompute_tmp_addr(Ref& tmp_addr, int split_depth, } bool Account::init_rewrite_addr(int split_depth, td::ConstBitPtr orig_addr_rewrite) { - if (split_depth_set_ || !created || !set_split_depth(split_depth)) { + if (split_depth_set_ || !set_split_depth(split_depth)) { return false; } addr_orig = addr; @@ -304,15 +304,8 @@ bool Account::unpack(Ref shard_account, Ref extra, total_state = orig_total_state = account; auto acc_cs = load_cell_slice(std::move(account)); if (block::gen::t_Account.get_tag(acc_cs) == block::gen::Account::account_none) { - status = acc_nonexist; - last_paid = 0; - last_trans_end_lt_ = 0; is_special = special; - if (workchain != ton::workchainInvalid) { - addr_orig = addr; - addr_rewrite = addr.cbits(); - } - return compute_my_addr() && acc_cs.size_ext() == 1; + return acc_cs.size_ext() == 1 && init_new(now); } block::gen::Account::Record_account acc; block::gen::AccountStorage::Record storage; @@ -328,6 +321,7 @@ bool Account::unpack(Ref shard_account, Ref extra, case block::gen::AccountState::account_uninit: status = orig_status = acc_uninit; state_hash = addr; + forget_split_depth(); break; case block::gen::AccountState::account_frozen: status = orig_status = acc_frozen; @@ -396,10 +390,42 @@ bool Account::init_new(ton::UnixTime now) { state_hash = addr_orig; status = orig_status = acc_nonexist; split_depth_set_ = false; - created = true; return true; } +bool Account::forget_split_depth() { + split_depth_set_ = false; + split_depth_ = 0; + addr_orig = addr; + my_addr = my_addr_exact; + addr_rewrite = addr.bits(); + return true; +} + +bool Account::deactivate() { + if (status == acc_active) { + return false; + } + // forget special (tick/tock) info + tick = tock = false; + if (status == acc_nonexist || status == acc_uninit) { + // forget split depth and address rewriting info + forget_split_depth(); + // forget specific state hash for deleted or uninitialized accounts (revert to addr) + state_hash = addr; + } + // forget code and data (only active accounts remember these) + code.clear(); + data.clear(); + library.clear(); + // if deleted, balance must be zero + if (status == acc_nonexist && !balance.is_zero()) { + return false; + } + return true; +} + + bool Account::belongs_to_shard(ton::ShardIdFull shard) const { return workchain == shard.workchain && ton::shard_is_ancestor(shard.shard, addr); } @@ -2214,7 +2240,7 @@ Ref Transaction::commit(Account& acc) { CHECK((const void*)&acc == (const void*)&account); // export all fields modified by the Transaction into original account // NB: this is the only method that modifies account - if (orig_addr_rewrite_set && new_split_depth >= 0 && acc.status == Account::acc_nonexist && + if (orig_addr_rewrite_set && new_split_depth >= 0 && acc.status != Account::acc_active && acc_status == Account::acc_active) { LOG(DEBUG) << "setting address rewriting info for newly-activated account " << acc.addr.to_hex() << " with split_depth=" << new_split_depth @@ -2243,9 +2269,7 @@ Ref Transaction::commit(Account& acc) { acc.tick = new_tick; acc.tock = new_tock; } else { - acc.tick = acc.tock = false; - acc.split_depth_set_ = false; - acc.created = true; + CHECK(acc.deactivate()); } end_lt = 0; acc.push_transaction(root, start_lt); diff --git a/crypto/block/transaction.h b/crypto/block/transaction.h index c1cfbc54..398e2731 100644 --- a/crypto/block/transaction.h +++ b/crypto/block/transaction.h @@ -213,17 +213,16 @@ struct Account { bool is_special{false}; bool tick{false}; bool tock{false}; - bool created{false}; bool split_depth_set_{false}; unsigned char split_depth_{0}; int verbosity{3 * 0}; ton::UnixTime now_{0}; ton::WorkchainId workchain{ton::workchainInvalid}; td::BitArray<32> addr_rewrite; // rewrite (anycast) data, split_depth bits - ton::StdSmcAddress addr; // rewritten address (by replacing a prefix of `addr_orig` with `addr_rewrite`) - ton::StdSmcAddress addr_orig; // address indicated in smart-contract data - Ref my_addr; // address as stored in the smart contract (MsgAddressInt) - Ref my_addr_exact; // exact address without anycast info + ton::StdSmcAddress addr; // rewritten address (by replacing a prefix of `addr_orig` with `addr_rewrite`); it is the key in ShardAccounts + ton::StdSmcAddress addr_orig; // address indicated in smart-contract data (must coincide with hash of StateInit) + Ref my_addr; // address as stored in the smart contract (MsgAddressInt); corresponds to `addr_orig` + anycast info + Ref my_addr_exact; // exact address without anycast info; corresponds to `addr` and has no anycast (rewrite) info ton::LogicalTime last_trans_end_lt_; ton::LogicalTime last_trans_lt_; ton::Bits256 last_trans_hash_; @@ -250,6 +249,7 @@ struct Account { bool set_address(ton::WorkchainId wc, td::ConstBitPtr new_addr); bool unpack(Ref account, Ref extra, ton::UnixTime now, bool special = false); bool init_new(ton::UnixTime now); + bool deactivate(); bool recompute_tmp_addr(Ref& tmp_addr, int split_depth, td::ConstBitPtr orig_addr_rewrite) const; td::RefInt256 compute_storage_fees(ton::UnixTime now, const std::vector& pricing) const; bool is_masterchain() const { @@ -268,6 +268,7 @@ struct Account { friend struct Transaction; bool set_split_depth(int split_depth); bool check_split_depth(int split_depth) const; + bool forget_split_depth(); bool init_rewrite_addr(int split_depth, td::ConstBitPtr orig_addr_rewrite); private: diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 6b1c72a6..5820a2ce 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -1944,7 +1944,6 @@ std::unique_ptr Collator::make_account_from(td::ConstBitPtr addr } auto ptr = std::make_unique(workchain(), addr); if (account.is_null()) { - ptr->created = true; if (!ptr->init_new(now_)) { return nullptr; } @@ -2273,10 +2272,6 @@ Ref Collator::create_ordinary_transaction(Ref msg_root) { register_new_msgs(*trans); update_max_lt(acc->last_trans_end_lt_); - // temporary patch to stop producing dangerous block - if (acc->status == block::Account::acc_nonexist) { - block_full_ = true; - } return trans_root; } @@ -2495,10 +2490,6 @@ int Collator::process_one_new_message(block::NewOutMsg msg, bool enqueue_only, R if (!insert_out_msg(cb.finalize())) { return -1; } - // 6.5. check for temporary patch can be left here - if (block_full_) { - return 3; - } // 7. check whether the block is full now if (!block_limit_status_->fits(block::ParamLimits::cl_normal)) { block_full_ = true; diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index fc90be1d..02a16f83 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -4073,7 +4073,6 @@ std::unique_ptr ValidateQuery::make_account_from(td::ConstBitPtr Ref extra) { auto ptr = std::make_unique(workchain(), addr); if (account.is_null()) { - ptr->created = true; if (!ptr->init_new(now_)) { return nullptr; } @@ -4308,7 +4307,7 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT } } if (is_first && is_masterchain() && account.is_special && account.tick && - (tag != block::gen::TransactionDescr::trans_tick_tock || (td_cs.prefetch_ulong(4) & 1)) && !account.created) { + (tag != block::gen::TransactionDescr::trans_tick_tock || (td_cs.prefetch_ulong(4) & 1)) && account.orig_status == block::Account::acc_active) { return reject_query(PSTRING() << "transaction " << lt << " of account " << addr.to_hex() << " is the first transaction for this special tick account in this block, but the " "transaction is not a tick transaction"); From 112bed04786fba7b3a2d5f3cdb1a62bf0b5d9257 Mon Sep 17 00:00:00 2001 From: sonofmom Date: Mon, 10 Jan 2022 23:07:53 +0100 Subject: [PATCH 08/21] Add base64 key input parameter to LiteClient --- lite-client/lite-client.cpp | 2 ++ lite-client/lite-client.h | 12 ++++++++++++ 2 files changed, 14 insertions(+) diff --git a/lite-client/lite-client.cpp b/lite-client/lite-client.cpp index 1ccfec86..f472af21 100644 --- a/lite-client/lite-client.cpp +++ b/lite-client/lite-client.cpp @@ -4261,6 +4261,8 @@ int main(int argc, char* argv[]) { }); p.add_option('p', "pub", "remote public key", [&](td::Slice arg) { td::actor::send_closure(x, &TestNode::set_public_key, td::BufferSlice{arg}); }); + p.add_option('b', "b64", "remote public key as base64", + [&](td::Slice arg) { td::actor::send_closure(x, &TestNode::decode_public_key, td::BufferSlice{arg}); }); p.add_option('d', "daemonize", "set SIGHUP", [&]() { td::set_signal_handler(td::SignalType::HangUp, [](int sig) { #if TD_DARWIN || TD_LINUX diff --git a/lite-client/lite-client.h b/lite-client/lite-client.h index 7184652a..00d8bd07 100644 --- a/lite-client/lite-client.h +++ b/lite-client/lite-client.h @@ -394,6 +394,18 @@ class TestNode : public td::actor::Actor { } remote_public_key_ = R.move_as_ok(); } + void decode_public_key(td::BufferSlice b64_key) { + auto R = [&]() -> td::Result { + std::string key_bytes = {(char)0xc6, (char)0xb4, (char)0x13, (char)0x48}; + key_bytes = key_bytes + td::base64_decode(b64_key.as_slice().str()).move_as_ok(); + return ton::PublicKey::import(key_bytes); + }(); + + if (R.is_error()) { + LOG(FATAL) << "bad b64 server public key: " << R.move_as_error(); + } + remote_public_key_ = R.move_as_ok(); + } void set_fail_timeout(td::Timestamp ts) { fail_timeout_ = ts; alarm_timestamp().relax(fail_timeout_); From 64615a9766525191b1c825a1098482ad424274df Mon Sep 17 00:00:00 2001 From: Alex Melman Date: Wed, 12 Jan 2022 10:20:16 +0200 Subject: [PATCH 09/21] rocksdb submodule upgrade to 6.27.3 --- third-party/rocksdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third-party/rocksdb b/third-party/rocksdb index 0915c99f..fcf3d75f 160000 --- a/third-party/rocksdb +++ b/third-party/rocksdb @@ -1 +1 @@ -Subproject commit 0915c99f01b46f50af8e02da8b6528156f584b7c +Subproject commit fcf3d75f3f022a6a55ff1222d6b06f8518d38c7c From 7dc980562f1a3f5ff96c8144474a271627da61ff Mon Sep 17 00:00:00 2001 From: Starlight Duck Date: Thu, 20 Jan 2022 23:20:56 +0200 Subject: [PATCH 10/21] tonlib use correct c7: config and address when executing --- crypto/smc-envelope/SmartContract.cpp | 64 ++++++++++++++++++--------- crypto/smc-envelope/SmartContract.h | 18 ++++++++ crypto/vm/vm.cpp | 1 + crypto/vm/vm.h | 4 ++ tonlib/tonlib/LastConfig.cpp | 4 +- tonlib/tonlib/TonlibClient.cpp | 45 +++++++++++++------ tonlib/tonlib/TonlibClient.h | 1 + 7 files changed, 99 insertions(+), 38 deletions(-) diff --git a/crypto/smc-envelope/SmartContract.cpp b/crypto/smc-envelope/SmartContract.cpp index 5f284b57..83e6a4e3 100644 --- a/crypto/smc-envelope/SmartContract.cpp +++ b/crypto/smc-envelope/SmartContract.cpp @@ -52,30 +52,51 @@ td::Ref prepare_vm_stack(td::RefInt256 amount, td::Ref return stack_ref; } -td::Ref prepare_vm_c7(td::uint32 now, td::uint64 balance) { - // TODO: fix initialization of c7 +td::Ref prepare_vm_c7(SmartContract::Args args) { td::BitArray<256> rand_seed; rand_seed.as_slice().fill(0); td::RefInt256 rand_seed_int{true}; rand_seed_int.unique_write().import_bits(rand_seed.cbits(), 256, false); + + td::uint32 now = 0; + if (args.now) { + now = args.now.unwrap(); + } + + vm::CellBuilder cb; + if (args.address) { + td::BigInt256 dest_addr; + dest_addr.import_bits((*args.address).addr.as_bitslice()); + cb.store_ones(1) + .store_zeroes(2) + .store_long((*args.address).workchain, 8) + .store_int256(dest_addr, 256); + } + auto address = cb.finalize(); + auto config = td::Ref(); + + if (args.config) { + config = (*args.config)->get_root_cell(); + } + auto tuple = vm::make_tuple_ref( td::make_refint(0x076ef1ea), // [ magic:0x076ef1ea td::make_refint(0), // actions:Integer td::make_refint(0), // msgs_sent:Integer td::make_refint(now), // unixtime:Integer - td::make_refint(0), // block_lt:Integer - td::make_refint(0), // trans_lt:Integer + td::make_refint(0), //TODO: // block_lt:Integer + td::make_refint(0), //TODO: // trans_lt:Integer std::move(rand_seed_int), // rand_seed:Integer - block::CurrencyCollection(balance).as_vm_tuple(), // balance_remaining:[Integer (Maybe Cell)] - vm::load_cell_slice_ref(vm::CellBuilder().finalize()) // myself:MsgAddressInt - //vm::StackEntry::maybe(td::Ref()) + block::CurrencyCollection(args.balance).as_vm_tuple(), // balance_remaining:[Integer (Maybe Cell)] + vm::load_cell_slice_ref(address), // myself:MsgAddressInt + vm::StackEntry::maybe(config) //vm::StackEntry::maybe(td::Ref()) ); // global_config:(Maybe Cell) ] = SmartContractInfo; //LOG(DEBUG) << "SmartContractInfo initialized with " << vm::StackEntry(tuple).to_string(); return vm::make_tuple_ref(std::move(tuple)); } SmartContract::Answer run_smartcont(SmartContract::State state, td::Ref stack, td::Ref c7, - vm::GasLimits gas, bool ignore_chksig) { + vm::GasLimits gas, bool ignore_chksig, td::Ref libraries) { auto gas_credit = gas.gas_credit; vm::init_op_cp0(); vm::DictionaryBase::get_empty_dictionary(); @@ -108,11 +129,14 @@ SmartContract::Answer run_smartcont(SmartContract::State state, td::Ref SmartContract::get_init_state() const { } SmartContract::Answer SmartContract::run_method(Args args) { - td::uint32 now = 0; - if (args.now) { - now = args.now.unwrap(); - } if (!args.c7) { - args.c7 = prepare_vm_c7(now, args.balance); + args.c7 = prepare_vm_c7(args); } if (!args.limits) { bool is_internal = args.get_method_id().ok() == 0; @@ -193,18 +217,15 @@ SmartContract::Answer SmartContract::run_method(Args args) { CHECK(args.method_id); args.stack.value().write().push_smallint(args.method_id.unwrap()); auto res = - run_smartcont(get_state(), args.stack.unwrap(), args.c7.unwrap(), args.limits.unwrap(), args.ignore_chksig); + run_smartcont(get_state(), args.stack.unwrap(), args.c7.unwrap(), args.limits.unwrap(), args.ignore_chksig, + args.libraries ? args.libraries.unwrap().get_root_cell() : td::Ref{}); state_ = res.new_state; return res; } SmartContract::Answer SmartContract::run_get_method(Args args) const { - td::uint32 now = 0; - if (args.now) { - now = args.now.unwrap(); - } if (!args.c7) { - args.c7 = prepare_vm_c7(now, args.balance); + args.c7 = prepare_vm_c7(args); } if (!args.limits) { args.limits = vm::GasLimits{1000000, 1000000}; @@ -214,7 +235,8 @@ SmartContract::Answer SmartContract::run_get_method(Args args) const { } CHECK(args.method_id); args.stack.value().write().push_smallint(args.method_id.unwrap()); - return run_smartcont(get_state(), args.stack.unwrap(), args.c7.unwrap(), args.limits.unwrap(), args.ignore_chksig); + return run_smartcont(get_state(), args.stack.unwrap(), args.c7.unwrap(), args.limits.unwrap(), args.ignore_chksig, + args.libraries ? args.libraries.unwrap().get_root_cell() : td::Ref{}); } SmartContract::Answer SmartContract::run_get_method(td::Slice method, Args args) const { diff --git a/crypto/smc-envelope/SmartContract.h b/crypto/smc-envelope/SmartContract.h index b26f4885..95dd2e84 100644 --- a/crypto/smc-envelope/SmartContract.h +++ b/crypto/smc-envelope/SmartContract.h @@ -26,6 +26,7 @@ #include "td/utils/crypto.h" #include "block/block.h" +#include "block/mc-config.h" namespace ton { class SmartContract : public td::CntObject { @@ -48,6 +49,7 @@ class SmartContract : public td::CntObject { td::Ref actions; td::int32 code; td::int64 gas_used; + td::ConstBitPtr missing_library{0}; static int output_actions_count(td::Ref list); }; @@ -61,6 +63,10 @@ class SmartContract : public td::CntObject { td::uint64 amount{0}; td::uint64 balance{0}; + td::optional address; + td::optional> config; + td::optional libraries; + Args() { } Args(std::initializer_list stack) @@ -106,6 +112,18 @@ class SmartContract : public td::CntObject { this->balance = balance; return std::move(*this); } + Args&& set_address(block::StdAddress address) { + this->address = address; + return std::move(*this); + } + Args&& set_config(std::shared_ptr& config) { + this->config = config; + return std::move(*this); + } + Args&& set_libraries(vm::Dictionary libraries) { + this->libraries = libraries; + return std::move(*this); + } td::Result get_method_id() const { if (!method_id) { diff --git a/crypto/vm/vm.cpp b/crypto/vm/vm.cpp index 038640a5..fb2f4d1e 100644 --- a/crypto/vm/vm.cpp +++ b/crypto/vm/vm.cpp @@ -604,6 +604,7 @@ Ref VmState::load_library(td::ConstBitPtr hash) { return lib; } } + missing_library = hash; return {}; } diff --git a/crypto/vm/vm.h b/crypto/vm/vm.h index 0fef9642..44aa7f23 100644 --- a/crypto/vm/vm.h +++ b/crypto/vm/vm.h @@ -96,6 +96,7 @@ class VmState final : public VmStateInterface { td::int64 loaded_cells_count{0}; int stack_trace{0}, debug_off{0}; bool chksig_always_succeed{false}; + td::ConstBitPtr missing_library{0}; public: enum { @@ -321,6 +322,9 @@ class VmState final : public VmStateInterface { Ref ref_to_cont(Ref cell) const { return td::make_ref(load_cell_slice_ref(std::move(cell)), get_cp()); } + td::ConstBitPtr get_missing_library() const { + return missing_library; + } private: void init_cregs(bool same_c3 = false, bool push_0 = true); diff --git a/tonlib/tonlib/LastConfig.cpp b/tonlib/tonlib/LastConfig.cpp index d07482ce..0111095b 100644 --- a/tonlib/tonlib/LastConfig.cpp +++ b/tonlib/tonlib/LastConfig.cpp @@ -62,9 +62,7 @@ void LastConfig::with_last_block(td::Result r_last_block) { } auto last_block = r_last_block.move_as_ok(); - auto params = params_; - client_.send_query(ton::lite_api::liteServer_getConfigParams(0, create_tl_lite_block_id(last_block.last_block_id), - std::move(params)), + client_.send_query(ton::lite_api::liteServer_getConfigAll(0, create_tl_lite_block_id(last_block.last_block_id)), [this](auto r_config) { this->on_config(std::move(r_config)); }); } diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index 1ac3a0d7..422dae12 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -861,13 +861,13 @@ class Query { } return res; } - td::Result>> estimate_fees(bool ignore_chksig, const block::Config& cfg) { + td::Result>> estimate_fees(bool ignore_chksig, std::shared_ptr& cfg, vm::Dictionary& libraries) { // gas fees bool is_masterchain = raw_.source->get_address().workchain == ton::masterchainId; - TRY_RESULT(gas_limits_prices, cfg.get_gas_limits_prices(is_masterchain)); - TRY_RESULT(storage_prices, cfg.get_storage_prices()); - TRY_RESULT(masterchain_msg_prices, cfg.get_msg_prices(true)); - TRY_RESULT(basechain_msg_prices, cfg.get_msg_prices(false)); + TRY_RESULT(gas_limits_prices, cfg->get_gas_limits_prices(is_masterchain)); + TRY_RESULT(storage_prices, cfg->get_storage_prices()); + TRY_RESULT(masterchain_msg_prices, cfg->get_msg_prices(true)); + TRY_RESULT(basechain_msg_prices, cfg->get_msg_prices(false)); block::MsgPrices* msg_prices[2] = {&basechain_msg_prices, &masterchain_msg_prices}; auto storage_fee_256 = block::StoragePrices::compute_storage_fees( raw_.source->get_sync_time(), storage_prices, raw_.source->raw().storage_stat, @@ -888,7 +888,9 @@ class Query { .set_limits(gas_limits) .set_balance(raw_.source->get_balance()) .set_now(raw_.source->get_sync_time()) - .set_ignore_chksig(ignore_chksig)); + .set_ignore_chksig(ignore_chksig) + .set_address(raw_.source->get_address()) + .set_config(cfg).set_libraries(libraries)); td::int64 fwd_fee = 0; if (res.success) { LOG(DEBUG) << "output actions:\n" @@ -910,7 +912,7 @@ class Query { for (auto& destination : raw_.destinations) { bool dest_is_masterchain = destination && destination->get_address().workchain == ton::masterchainId; - TRY_RESULT(dest_gas_limits_prices, cfg.get_gas_limits_prices(dest_is_masterchain)); + TRY_RESULT(dest_gas_limits_prices, cfg->get_gas_limits_prices(dest_is_masterchain)); auto dest_storage_fee_256 = destination ? block::StoragePrices::compute_storage_fees( destination->get_sync_time(), storage_prices, destination->raw().storage_stat, @@ -3266,7 +3268,7 @@ void TonlibClient::query_estimate_fees(td::int64 id, bool ignore_chksig, td::Res return; } TRY_RESULT_PROMISE(promise, state, std::move(r_state)); - TRY_RESULT_PROMISE_PREFIX(promise, fees, TRY_VM(it->second->estimate_fees(ignore_chksig, *state.config)), + TRY_RESULT_PROMISE_PREFIX(promise, fees, TRY_VM(it->second->estimate_fees(ignore_chksig, state.config, libraries)), TonlibError::Internal()); promise.set_value(tonlib_api::make_object( fees.first.to_tonlib_api(), td::transform(fees.second, [](auto& x) { return x.to_tonlib_api(); }))); @@ -3493,14 +3495,29 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_runGetMethod& request, args.set_stack(std::move(stack)); args.set_balance(it->second->get_balance()); args.set_now(it->second->get_sync_time()); - auto res = smc->run_get_method(std::move(args)); + args.set_address(it->second->get_address()); - // smc.runResult gas_used:int53 stack:vector exit_code:int32 = smc.RunResult; - std::vector> res_stack; - for (auto& entry : res.stack->as_span()) { - res_stack.push_back(to_tonlib_api(entry)); + auto code = smc->get_state().code; + if (code.not_null()) { + std::vector libraryList; } - promise.set_value(tonlib_api::make_object(res.gas_used, std::move(res_stack), res.code)); + + args.set_libraries(libraries); + + client_.with_last_config([smc=std::move(smc), args=std::move(args), + promise = std::move(promise)](td::Result r_state) mutable { + TRY_RESULT_PROMISE(promise, state, std::move(r_state)); + args.set_config(state.config); + + auto res = smc->run_get_method(std::move(args)); + + // smc.runResult gas_used:int53 stack:vector exit_code:int32 = smc.RunResult; + std::vector> res_stack; + for (auto& entry : res.stack->as_span()) { + res_stack.push_back(to_tonlib_api(entry)); + } + promise.set_value(tonlib_api::make_object(res.gas_used, std::move(res_stack), res.code)); + }); return td::Status::OK(); } diff --git a/tonlib/tonlib/TonlibClient.h b/tonlib/tonlib/TonlibClient.h index bedc1d49..4dfbe8c6 100644 --- a/tonlib/tonlib/TonlibClient.h +++ b/tonlib/tonlib/TonlibClient.h @@ -107,6 +107,7 @@ class TonlibClient : public td::actor::Actor { td::optional block_id; }; QueryContext query_context_; + vm::Dictionary libraries{256}; // network td::actor::ActorOwn raw_client_; From 3fee04e20afd223198b5352520ceb77715cc13e8 Mon Sep 17 00:00:00 2001 From: Starlight Duck Date: Fri, 21 Jan 2022 03:43:35 +0200 Subject: [PATCH 11/21] Support libraries in LS and request+caching in TLC --- crypto/block/mc-config.cpp | 2 +- crypto/smc-envelope/SmartContract.cpp | 3 +- tl/generate/scheme/lite_api.tl | 3 + tl/generate/scheme/lite_api.tlo | Bin 12416 -> 12896 bytes tonlib/tonlib/TonlibClient.cpp | 178 +++++++++++++++++++++++--- tonlib/tonlib/TonlibClient.h | 6 +- validator/impl/liteserver.cpp | 70 ++++++++++ validator/impl/liteserver.hpp | 2 + 8 files changed, 245 insertions(+), 19 deletions(-) diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index 07574ef1..003cd04b 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -2100,7 +2100,7 @@ Ref ConfigInfo::lookup_library(td::ConstBitPtr root_hash) const { return {}; } auto csr = libraries_dict_->lookup(root_hash, 256); - if (csr.is_null() || csr->prefetch_ulong(8) != 0 || !csr->have_refs()) { // shared_lib_descr$00 lib:^Cell + if (csr.is_null() || csr->prefetch_ulong(2) != 0 || !csr->have_refs()) { // shared_lib_descr$00 lib:^Cell return {}; } auto lib = csr->prefetch_ref(); diff --git a/crypto/smc-envelope/SmartContract.cpp b/crypto/smc-envelope/SmartContract.cpp index 83e6a4e3..9a273a08 100644 --- a/crypto/smc-envelope/SmartContract.cpp +++ b/crypto/smc-envelope/SmartContract.cpp @@ -129,8 +129,9 @@ SmartContract::Answer run_smartcont(SmartContract::State state, td::RefE-j$d-O~Uz4o};u2{ISqYF?AaM{Va%R)v zux&dCQolJz)P|2yV)IJn-ON@=+mtRMdpJF{1mfq+)M9kQd7!q(=VTU_Fie(`kei$! NE5ZUcGeTX1697XhbTI$` delta 51 zcmaEm(vZmeXtur;0~F|N(kv.release()); + load_libs_from_disk(); + key_storage_.set_key_value(kv_); last_block_storage_.set_key_value(kv_); auto res = tonlib_api::make_object(); @@ -3474,6 +3476,38 @@ td::Result from_tonlib_api(tonlib_api::tvm_StackEntry& entry) { })); } +void deep_library_search(std::set& set, std::set& visited, + vm::Dictionary libs, td::Ref cell, int depth) { + if (depth <= 0 || set.size() >= 16 || visited.size() >= 256) { + return; + } + auto ins = visited.insert(cell->get_hash()); + if (!ins.second) { + return; // already visited this cell + } + auto r_loaded_cell = cell->load_cell(); + if (r_loaded_cell.is_error()) { + return; + } + auto loaded_cell = r_loaded_cell.move_as_ok(); + if (loaded_cell.data_cell->is_special()) { + if (loaded_cell.data_cell->special_type() == vm::DataCell::SpecialType::Library) { + vm::CellSlice cs(std::move(loaded_cell)); + if (cs.size() != vm::Cell::hash_bits + 8) { + return; + } + auto key = td::Bits256(cs.data_bits() + 8); + if (libs.lookup(key).is_null()) { + set.insert(key); + } + } + return; + } + for (unsigned int i=0; iget_refs_cnt(); i++) { + deep_library_search(set, visited, libs, loaded_cell.data_cell->get_ref(i), depth - 1); + } +} + td::Status TonlibClient::do_request(const tonlib_api::smc_runGetMethod& request, td::Promise>&& promise) { auto it = smcs_.find(request.id_); @@ -3497,30 +3531,120 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_runGetMethod& request, args.set_now(it->second->get_sync_time()); args.set_address(it->second->get_address()); - auto code = smc->get_state().code; - if (code.not_null()) { - std::vector libraryList; - } - - args.set_libraries(libraries); - - client_.with_last_config([smc=std::move(smc), args=std::move(args), - promise = std::move(promise)](td::Result r_state) mutable { + client_.with_last_config([self = this, smc = std::move(smc), args = std::move(args), promise = std::move(promise) + ](td::Result r_state) mutable { TRY_RESULT_PROMISE(promise, state, std::move(r_state)); args.set_config(state.config); - auto res = smc->run_get_method(std::move(args)); - - // smc.runResult gas_used:int53 stack:vector exit_code:int32 = smc.RunResult; - std::vector> res_stack; - for (auto& entry : res.stack->as_span()) { - res_stack.push_back(to_tonlib_api(entry)); + auto code = smc->get_state().code; + if (code.not_null()) { + std::set librarySet; + std::set visited; + deep_library_search(librarySet, visited, self->libraries, code, 24); + std::vector libraryList{librarySet.begin(), librarySet.end()}; + if (libraryList.size() > 0) { + LOG(DEBUG) << "Requesting found libraries in code (" << libraryList.size() << ")"; + self->client_.send_query(ton::lite_api::liteServer_getLibraries(std::move(libraryList)), + [self, smc = std::move(smc), args = std::move(args), promise = std::move(promise)] + (td::Result> r_libraries) mutable + { + if (r_libraries.is_error()) { + LOG(WARNING) << "cannot obtain found libraries: " << r_libraries.move_as_error().to_string(); + } else { + auto libraries = r_libraries.move_as_ok(); + bool updated = false; + for (auto& lr : libraries->result_) { + auto contents = vm::std_boc_deserialize(lr->data_); + if (contents.is_ok() && contents.ok().not_null()) { + if (contents.ok()->get_hash().bits().compare(lr->hash_.cbits(), 256)) { + LOG(WARNING) << "hash mismatch for library " << lr->hash_.to_hex(); + continue; + } + self->libraries.set_ref(lr->hash_, contents.move_as_ok()); + updated = true; + LOG(DEBUG) << "registered library " << lr->hash_.to_hex(); + } else { + LOG(WARNING) << "failed to deserialize library: " << lr->hash_.to_hex(); + } + } + if (updated) { + self->store_libs_to_disk(); + } + } + self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise)); + }); + } + else { + self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise)); + } + } + else { + self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise)); } - promise.set_value(tonlib_api::make_object(res.gas_used, std::move(res_stack), res.code)); }); return td::Status::OK(); } +void TonlibClient::perform_smc_execution(td::Ref smc, ton::SmartContract::Args args, + td::Promise>&& promise) { + + args.set_libraries(libraries); + + auto res = smc->run_get_method(args); + + // smc.runResult gas_used:int53 stack:vector exit_code:int32 = smc.RunResult; + std::vector> res_stack; + for (auto& entry : res.stack->as_span()) { + res_stack.push_back(to_tonlib_api(entry)); + } + + if (res.missing_library.not_null()) { + td::Bits256 hash = res.missing_library; + LOG(DEBUG) << "Requesting missing library: " << hash.to_hex(); + std::vector req = {std::move(hash)}; + client_.send_query(ton::lite_api::liteServer_getLibraries(std::move(req)), + [self = this, res = std::move(res), res_stack = std::move(res_stack), hash = std::move(hash), + smc = std::move(smc), args = std::move(args), promise = std::move(promise)] + (td::Result> r_libraries) mutable + { + if (r_libraries.is_error()) { + LOG(WARNING) << "cannot obtain missing library: " << r_libraries.move_as_error().to_string(); + promise.set_value(tonlib_api::make_object(res.gas_used, std::move(res_stack), res.code)); + return; + } + bool found = false, updated = false; + auto libraries = r_libraries.move_as_ok(); + for (auto& lr : libraries->result_) { + auto contents = vm::std_boc_deserialize(lr->data_); + if (contents.is_ok() && contents.ok().not_null()) { + if (contents.ok()->get_hash().bits().compare(lr->hash_.cbits(), 256)) { + LOG(WARNING) << "hash mismatch for library " << lr->hash_.to_hex(); + continue; + } + found |= (lr->hash_ == hash); + updated = true; + self->libraries.set_ref(lr->hash_, contents.move_as_ok()); + LOG(DEBUG) << "registered library " << lr->hash_.to_hex(); + } else { + LOG(WARNING) << "failed to deserialize library: " << lr->hash_.to_hex(); + } + } + if (updated) { + self->store_libs_to_disk(); + } + if (!found) { + LOG(WARNING) << "cannot obtain library " << hash.to_hex() << ", it may not exist"; + promise.set_value(tonlib_api::make_object(res.gas_used, std::move(res_stack), res.code)); + } else { + self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise)); + } + }); + } + else { + promise.set_value(tonlib_api::make_object(res.gas_used, std::move(res_stack), res.code)); + } +} + td::Result> to_tonlib_api( const ton::ManualDns::EntryData& entry_data) { td::Result> res; @@ -4246,6 +4370,28 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_getBlockHeader& req return td::Status::OK(); } +void TonlibClient::load_libs_from_disk() { + LOG(DEBUG) << "loading libraries from disk cache"; + auto r_data = kv_->get("tonlib.libcache"); + if (r_data.is_error()) { + return; + } + auto r_dict = vm::std_boc_deserialize(r_data.move_as_ok(), true); + if (r_dict.is_error()) { + return; + } + libraries = vm::Dictionary(vm::load_cell_slice(vm::CellBuilder().append_cellslice(vm::load_cell_slice( + r_dict.move_as_ok())).finalize()), 256); + // int n = 0; for (auto&& lr : libraries) n++; + LOG(DEBUG) << "loaded libraries from disk cache"; +} + +void TonlibClient::store_libs_to_disk() { // NB: Dictionary.get_root_cell does not compute_root, and it is protected + kv_->set("tonlib.libcache", vm::std_boc_serialize(vm::CellBuilder().append_cellslice(libraries.get_root()) + .finalize()).move_as_ok().as_slice()); + // int n = 0; for (auto&& lr : libraries) n++; + LOG(DEBUG) << "stored libraries to disk cache"; +} template td::Status TonlibClient::do_request(const tonlib_api::runTests& request, P&&) { diff --git a/tonlib/tonlib/TonlibClient.h b/tonlib/tonlib/TonlibClient.h index 4dfbe8c6..6d80c152 100644 --- a/tonlib/tonlib/TonlibClient.h +++ b/tonlib/tonlib/TonlibClient.h @@ -322,7 +322,8 @@ class TonlibClient : public td::actor::Actor { td::Status do_request(tonlib_api::pchan_unpackPromise& request, td::Promise>&& promise); - + void perform_smc_execution(td::Ref smc, ton::SmartContract::Args args, + td::Promise>&& promise); void do_dns_request(std::string name, td::int32 category, td::int32 ttl, td::optional block_id, block::StdAddress address, td::Promise>&& promise); @@ -358,6 +359,9 @@ class TonlibClient : public td::actor::Actor { void proxy_request(td::int64 query_id, std::string data); + void load_libs_from_disk(); + void store_libs_to_disk(); + friend class TonlibQueryActor; struct Target { bool can_be_empty{true}; diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index 6b0b9008..90d974e0 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -198,6 +198,9 @@ void LiteQuery::start_up() { this->perform_runSmcMethod(ton::create_block_id(q.id_), static_cast(q.account_->workchain_), q.account_->id_, q.mode_, q.method_id_, std::move(q.params_)); }, + [&](lite_api::liteServer_getLibraries& q) { + this->perform_getLibraries(q.library_list_); + }, [&](auto& obj) { this->abort_query(td::Status::Error(ErrorCode::protoviolation, "unknown query")); })); } @@ -781,6 +784,73 @@ void LiteQuery::perform_runSmcMethod(BlockIdExt blkid, WorkchainId workchain, St perform_getAccountState(blkid, workchain, addr, mode | 0x10000); } +void LiteQuery::perform_getLibraries(std::vector library_list) { + LOG(INFO) << "started a getLibraries() liteserver query"; + if (library_list.size() > 16) { + LOG(INFO) << "too many libraries requested, returning only first 16"; + library_list.resize(16); + } + sort( library_list.begin(), library_list.end() ); + library_list.erase( unique( library_list.begin(), library_list.end() ), library_list.end() ); + td::actor::send_closure_later( + manager_, &ton::validator::ValidatorManager::get_top_masterchain_state_block, + [Self = actor_id(this), library_list](td::Result, BlockIdExt>> res) -> void { + if (res.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); + } else { + auto pair = res.move_as_ok(); + td::actor::send_closure_later(Self, &LiteQuery::continue_getLibraries, std::move(pair.first), + pair.second, library_list); + } + }); +} + +void LiteQuery::continue_getLibraries(Ref mc_state, BlockIdExt blkid, std::vector library_list) { + LOG(INFO) << "obtained last masterchain block = " << blkid.to_str(); + base_blk_id_ = blkid; + CHECK(mc_state.not_null()); + mc_state_ = Ref(std::move(mc_state)); + CHECK(mc_state_.not_null()); + + auto rconfig = block::ConfigInfo::extract_config(mc_state_->root_cell(), block::ConfigInfo::needLibraries); + if (rconfig.is_error()) { + fatal_error("cannot extract library list block configuration from masterchain state"); + return; + } + auto config = rconfig.move_as_ok(); + + if (false) { + std::ostringstream os; + vm::load_cell_slice(config->get_libraries_root()).print_rec(os); + LOG(INFO) << "\n" << os.str(); + + auto lib_dict = std::make_unique(config->get_libraries_root(), 256); + for (auto k: *lib_dict) { + std::ostringstream oss; + k.second->print_rec(oss); + LOG(INFO) << "library " << k.first.to_hex(256) << ": \n" << oss.str(); + } + } + + std::vector> a; + for (const auto& hash : library_list) { + LOG(INFO) << "looking for library " << hash.to_hex(); + auto libres = config->lookup_library(hash); + if (libres.is_null()) { + LOG(INFO) << "library lookup result is null"; + continue; + } + auto data = vm::std_boc_serialize(libres); + if (data.is_error()) { + LOG(WARNING) << "library serialization failed: " << data.move_as_error().to_string(); + continue; + } + a.push_back(ton::create_tl_object(hash, data.move_as_ok())); + } + auto b = ton::create_serialize_tl_object(std::move(a)); + finish_query(std::move(b)); +} + void LiteQuery::perform_getOneTransaction(BlockIdExt blkid, WorkchainId workchain, StdSmcAddress addr, LogicalTime lt) { LOG(INFO) << "started a getOneTransaction(" << blkid.to_str() << ", " << workchain << ", " << addr.to_hex() << "," << lt << ") liteserver query"; diff --git a/validator/impl/liteserver.hpp b/validator/impl/liteserver.hpp index 24392085..f1918ceb 100644 --- a/validator/impl/liteserver.hpp +++ b/validator/impl/liteserver.hpp @@ -115,6 +115,8 @@ class LiteQuery : public td::actor::Actor { td::BufferSlice params); void finish_runSmcMethod(td::BufferSlice shard_proof, td::BufferSlice state_proof, Ref acc_root, UnixTime gen_utime, LogicalTime gen_lt); + void perform_getLibraries(std::vector library_list); + void continue_getLibraries(Ref mc_state, BlockIdExt blkid, std::vector library_list); void perform_getOneTransaction(BlockIdExt blkid, WorkchainId workchain, StdSmcAddress addr, LogicalTime lt); void continue_getOneTransaction(); void perform_getTransactions(WorkchainId workchain, StdSmcAddress addr, LogicalTime lt, Bits256 hash, unsigned count); From 76c2764e7ff7fe89c6a37bfd31f9b1d4208b1d9d Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Sat, 19 Feb 2022 16:39:59 +0300 Subject: [PATCH 12/21] Sparse state serialization over time by randomized delay --- validator/state-serializer.cpp | 52 +++++++++++++++++++++++----------- validator/state-serializer.hpp | 3 ++ 2 files changed, 38 insertions(+), 17 deletions(-) diff --git a/validator/state-serializer.cpp b/validator/state-serializer.cpp index 802159c0..63b7b864 100644 --- a/validator/state-serializer.cpp +++ b/validator/state-serializer.cpp @@ -81,6 +81,27 @@ void AsyncStateSerializer::alarm() { td::actor::send_closure(manager_, &ValidatorManager::get_top_masterchain_block, std::move(P)); } +void AsyncStateSerializer::request_masterchain_state() { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &AsyncStateSerializer::fail_handler, + R.move_as_error_prefix("failed to get masterchain state: ")); + } else { + td::actor::send_closure(SelfId, &AsyncStateSerializer::got_masterchain_state, + td::Ref(R.move_as_ok())); + } + }); + td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, masterchain_handle_, std::move(P)); +} + +void AsyncStateSerializer::request_shard_state(BlockIdExt shard) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + R.ensure(); + td::actor::send_closure(SelfId, &AsyncStateSerializer::got_shard_handle, R.move_as_ok()); + }); + return td::actor::send_closure(manager_, &ValidatorManager::get_block_handle, shard, true, std::move(P)); +} + void AsyncStateSerializer::next_iteration() { if (running_) { return; @@ -102,29 +123,26 @@ void AsyncStateSerializer::next_iteration() { if (attempt_ < max_attempt() && last_key_block_id_.id.seqno < last_block_id_.id.seqno && need_serialize(masterchain_handle_)) { if (masterchain_state_.is_null()) { - running_ = true; - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &AsyncStateSerializer::fail_handler, - R.move_as_error_prefix("failed to get masterchain state: ")); - } else { - td::actor::send_closure(SelfId, &AsyncStateSerializer::got_masterchain_state, - td::Ref(R.move_as_ok())); - } - }); - td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, masterchain_handle_, std::move(P)); + // block next attempts immediately, but send actual request later + running_ = true; + delay_action( + [SelfId = actor_id(this)]() { td::actor::send_closure(SelfId, &AsyncStateSerializer::request_masterchain_state); }, + // Masterchain is more important and much lighter than shards + // thus lower delay + td::Timestamp::in(td::Random::fast(0, 600))); return; } while (next_idx_ < shards_.size()) { if (!need_monitor(shards_[next_idx_].shard_full())) { next_idx_++; } else { - running_ = true; - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - R.ensure(); - td::actor::send_closure(SelfId, &AsyncStateSerializer::got_shard_handle, R.move_as_ok()); - }); - td::actor::send_closure(manager_, &ValidatorManager::get_block_handle, shards_[next_idx_], true, std::move(P)); + // block next attempts immediately, but send actual request later + running_ = true; + delay_action( + [SelfId = actor_id(this), shard = shards_[next_idx_]]() { td::actor::send_closure(SelfId, &AsyncStateSerializer::request_shard_state, shard); }, + // Shards are less important and heavier than master + // thus higher delay + td::Timestamp::in(td::Random::fast(0, 4 * 3600))); return; } } diff --git a/validator/state-serializer.hpp b/validator/state-serializer.hpp index 3b8d7052..162e96c1 100644 --- a/validator/state-serializer.hpp +++ b/validator/state-serializer.hpp @@ -65,6 +65,9 @@ class AsyncStateSerializer : public td::actor::Actor { void got_self_state(AsyncSerializerState state); void got_init_handle(BlockHandle handle); + void request_masterchain_state(); + void request_shard_state(BlockIdExt shard); + void next_iteration(); void got_top_masterchain_handle(BlockIdExt block_id); void got_masterchain_handle(BlockHandle handle_); From 45a160d7c19a7c7b6d76485f1ebc5a5b34845cc7 Mon Sep 17 00:00:00 2001 From: ms Date: Thu, 31 Mar 2022 14:29:43 +0200 Subject: [PATCH 13/21] Add getConfigParam to tonlibjson --- tl/generate/scheme/tonlib_api.tl | 5 +++++ tl/generate/scheme/tonlib_api.tlo | Bin 27936 -> 28216 bytes tonlib/tonlib/LastConfig.cpp | 2 +- tonlib/tonlib/LastConfig.h | 9 +------- tonlib/tonlib/TonlibClient.cpp | 36 ++++++++++++++++++++++-------- tonlib/tonlib/TonlibClient.h | 3 +++ 6 files changed, 37 insertions(+), 18 deletions(-) diff --git a/tl/generate/scheme/tonlib_api.tl b/tl/generate/scheme/tonlib_api.tl index b1dd7679..4527cc09 100644 --- a/tl/generate/scheme/tonlib_api.tl +++ b/tl/generate/scheme/tonlib_api.tl @@ -210,6 +210,9 @@ blocks.shortTxId mode:# account:mode.0?bytes lt:mode.1?int64 hash:mode.2?bytes = blocks.transactions id:ton.blockIdExt req_count:int32 incomplete:Bool transactions:vector = blocks.Transactions; blocks.header id:ton.blockIdExt global_id:int32 version:int32 after_merge:Bool after_split:Bool before_split:Bool want_merge:Bool want_split:Bool validator_list_hash_short:int32 catchain_seqno:int32 min_ref_mc_seqno:int32 is_key_block:Bool prev_key_block_seqno:int32 start_lt:int64 end_lt:int64 vert_seqno:# prev_blocks:vector = blocks.Header; //blocks.shortData header:blocks.Header transactions:blocks.Header = blocks.BlockData; + +configInfo config:tvm.cell = ConfigInfo; + ---functions--- init options:options = options.Info; @@ -262,6 +265,8 @@ guessAccount public_key:string rwallet_init_public_key:string = AccountRevisionL getAccountState account_address:accountAddress = FullAccountState; createQuery private_key:InputKey address:accountAddress timeout:int32 action:Action initial_account_state:InitialAccountState = query.Info; +getConfigParam mode:# id:ton.blockIdExt param:# = ConfigInfo; + msg.decrypt input_key:InputKey data:msg.dataEncryptedArray = msg.DataDecryptedArray; msg.decryptWithProof proof:bytes data:msg.dataEncrypted = msg.Data; diff --git a/tl/generate/scheme/tonlib_api.tlo b/tl/generate/scheme/tonlib_api.tlo index 77741950ec220ed7ec01cd6f79333282a16276d4..d5a8344d6d832315af5525ff6b0288e42aa7e453 100644 GIT binary patch delta 174 zcmZ2*i*d&tM&3uW^{p77Abcb5c1D5!p$wW_&iQ$1ndzQ+Y55Ej3nVrlVQk@HY}o9{ zx0+KDET4>!2k~zn6Y^o4yiiDd@&X?b7Lds17vc-d7+p4Z`t9eE?fN3l#Fw600x>lp tu_!T@0j7<4vS)zypH;+YW+FvWa{m3;_E=JL>=d delta 53 zcmdmShjGCzM&3uW^{p77AZ#P=cE-(b7+ZK4>o*tjt>)ZZATh;^(Rnkg|9-B`2~jtM E0oLph*8l(j diff --git a/tonlib/tonlib/LastConfig.cpp b/tonlib/tonlib/LastConfig.cpp index 0111095b..5ee73212 100644 --- a/tonlib/tonlib/LastConfig.cpp +++ b/tonlib/tonlib/LastConfig.cpp @@ -39,7 +39,7 @@ td::StringBuilder& operator<<(td::StringBuilder& sb, const LastConfigState& stat return sb; } -LastConfig::LastConfig(ExtClientRef client, td::unique_ptr callback) : callback_(std::move(callback)) { +LastConfig::LastConfig(ExtClientRef client) { client_.set_client(client); VLOG(last_block) << "State: " << state_; } diff --git a/tonlib/tonlib/LastConfig.h b/tonlib/tonlib/LastConfig.h index 514b4a59..d610165d 100644 --- a/tonlib/tonlib/LastConfig.h +++ b/tonlib/tonlib/LastConfig.h @@ -36,17 +36,10 @@ td::StringBuilder& operator<<(td::StringBuilder& sb, const LastConfigState& stat class LastConfig : public td::actor::Actor { public: - class Callback { - public: - virtual ~Callback() { - } - }; - - explicit LastConfig(ExtClientRef client, td::unique_ptr callback); + explicit LastConfig(ExtClientRef client); void get_last_config(td::Promise promise); private: - td::unique_ptr callback_; ExtClient client_; LastConfigState state_; diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index be2fe6c2..d75fb120 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -1439,17 +1439,9 @@ void TonlibClient::init_last_block(LastBlockState state) { } void TonlibClient::init_last_config() { ref_cnt_++; - class Callback : public LastConfig::Callback { - public: - Callback(td::actor::ActorShared client) : client_(std::move(client)) { - } - - private: - td::actor::ActorShared client_; - }; raw_last_config_ = td::actor::create_actor(td::actor::ActorOptions().with_name("LastConfig").with_poll(false), - get_client_ref(), td::make_unique(td::actor::actor_shared(this))); + get_client_ref()); } void TonlibClient::on_result(td::uint64 id, tonlib_api::object_ptr response) { @@ -4198,6 +4190,32 @@ auto to_lite_api(const tonlib_api::ton_blockIdExt& blk) -> td::Result>&& promise) { + TRY_RESULT(lite_block, to_lite_api(*request.id_)) + auto block = create_block_id(std::move(lite_block)); + auto param = request.param_; + std::vector params = { param }; + + client_.send_query(ton::lite_api::liteServer_getConfigParams(0, std::move(lite_block), std::move(params)), + promise.wrap([block, param](auto r_config) { + auto state = block::check_extract_state_proof(block, r_config->state_proof_.as_slice(), + r_config->config_proof_.as_slice()); + if (state.is_error()) { + LOG(ERROR) << "block::check_extract_state_proof failed: " << state.error(); + } + auto config = block::Config::extract_from_state(std::move(state.move_as_ok()), 0); + if (config.is_error()) { + LOG(ERROR) << "block::Config::extract_from_state failed: " << config.error(); + } + tonlib_api::configInfo config_result; + config_result.config_ = tonlib_api::make_object(to_bytes(config.move_as_ok()->get_config_param(param))); + return tonlib_api::make_object(std::move(config_result)); + })); + + return td::Status::OK(); +} + td::Status TonlibClient::do_request(const tonlib_api::blocks_getMasterchainInfo& masterchain_info, td::Promise>&& promise) { client_.send_query(ton::lite_api::liteServer_getMasterchainInfo(), diff --git a/tonlib/tonlib/TonlibClient.h b/tonlib/tonlib/TonlibClient.h index 6d80c152..716133e4 100644 --- a/tonlib/tonlib/TonlibClient.h +++ b/tonlib/tonlib/TonlibClient.h @@ -357,6 +357,9 @@ class TonlibClient : public td::actor::Actor { td::Status do_request(const tonlib_api::blocks_getBlockHeader& request, td::Promise>&& promise); + td::Status do_request(const tonlib_api::getConfigParam& request, + td::Promise>&& promise); + void proxy_request(td::int64 query_id, std::string data); void load_libs_from_disk(); From 34590176eb786d3c94b136463db2fa65cb159e6b Mon Sep 17 00:00:00 2001 From: akifoq <75585197+akifoq@users.noreply.github.com> Date: Tue, 5 Apr 2022 01:33:09 +0300 Subject: [PATCH 14/21] Fix IF:<{ and IFNOT:<{ empty continuation case The boolean value should be DROPped in the case the continuation is empty. --- crypto/fift/lib/Asm.fif | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crypto/fift/lib/Asm.fif b/crypto/fift/lib/Asm.fif index 76381b85..edd0e24b 100644 --- a/crypto/fift/lib/Asm.fif +++ b/crypto/fift/lib/Asm.fif @@ -694,9 +694,9 @@ x{E30F} @Defop(ref*2) IFREFELSEREF [] execute } : @run-cont-op { triple 1 ' @run-cont-op does create } : @def-cont-op -{ } { PUSHCONT IF } { IFREF } @def-cont-op IF-cont +{ DROP } { PUSHCONT IF } { IFREF } @def-cont-op IF-cont { IFRET } { PUSHCONT IFJMP } { IFJMPREF } @def-cont-op IFJMP-cont -{ } { PUSHCONT IFNOT } { IFNOTREF } @def-cont-op IFNOT-cont +{ DROP } { PUSHCONT IFNOT } { IFNOTREF } @def-cont-op IFNOT-cont { IFNOTRET } { PUSHCONT IFNOTJMP } { IFNOTJMPREF } @def-cont-op IFNOTJMP-cont { dup 2over rot } : 3dup From b4ac32a4520a4ccc88fa61b7915d7a8f328bc2a7 Mon Sep 17 00:00:00 2001 From: ms Date: Wed, 6 Apr 2022 19:34:16 +0200 Subject: [PATCH 15/21] Fix created_lt for in_out messages --- tonlib/tonlib/TonlibClient.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index d75fb120..7f9288b5 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -2302,9 +2302,10 @@ struct ToRawTransactions { return td::Status::Error("Failed to unpack CommonMsgInfo::ext_out_msg_info"); } TRY_RESULT(src, to_std_address(msg_info.src)); + auto created_lt = static_cast(msg_info.created_lt); return tonlib_api::make_object( tonlib_api::make_object(src), - tonlib_api::make_object(), 0, 0, 0, 0, std::move(body_hash), get_data(src)); + tonlib_api::make_object(), 0, 0, 0, created_lt, std::move(body_hash), get_data(src)); } } From ca167e92828b437863d786294e8ede6d66be2f8c Mon Sep 17 00:00:00 2001 From: ms Date: Thu, 14 Apr 2022 22:49:10 +0200 Subject: [PATCH 16/21] Remove limit on count of returned out_msgs --- tonlib/tonlib/TonlibClient.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index 7f9288b5..c6a83ebb 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -2350,7 +2350,7 @@ struct ToRawTransactions { if (trans.outmsg_cnt != 0) { vm::Dictionary dict{trans.r1.out_msgs, 15}; - for (int x = 0; x < trans.outmsg_cnt && x < 100; x++) { + for (int x = 0; x < trans.outmsg_cnt; x++) { TRY_RESULT(out_msg, to_raw_message(dict.lookup_ref(td::BitArray<15>{x}))); fees += out_msg->fwd_fee_; fees += out_msg->ihr_fee_; From 179415f815241be1965de0adea73f829f5b2eae8 Mon Sep 17 00:00:00 2001 From: Andrey Tvorozhkov Date: Sun, 17 Apr 2022 09:27:56 +0300 Subject: [PATCH 17/21] Fix tvm_stackEntrySlice in from_tonlib_api --- tonlib/tonlib/TonlibClient.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index c6a83ebb..0ea3743b 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -3437,7 +3437,8 @@ td::Result from_tonlib_api(tonlib_api::tvm_StackEntry& entry) { [&](tonlib_api::tvm_stackEntryUnsupported& cell) { return td::Status::Error("Unsuppored stack entry"); }, [&](tonlib_api::tvm_stackEntrySlice& cell) -> td::Result { TRY_RESULT(res, vm::std_boc_deserialize(cell.slice_->bytes_)); - return vm::StackEntry{std::move(res)}; + auto slice = vm::load_cell_slice_ref(std::move(res)); + return vm::StackEntry{std::move(slice)}; }, [&](tonlib_api::tvm_stackEntryCell& cell) -> td::Result { TRY_RESULT(res, vm::std_boc_deserialize(cell.cell_->bytes_)); From 8303f2c84f6e978cc97901584ed6dfddd40d899d Mon Sep 17 00:00:00 2001 From: ms Date: Tue, 19 Apr 2022 18:09:06 +0200 Subject: [PATCH 18/21] Fix BlockHeader is_key_block and add gen_utime, flags --- tl/generate/scheme/tonlib_api.tl | 2 +- tl/generate/scheme/tonlib_api.tlo | Bin 28216 -> 28288 bytes tonlib/tonlib/TonlibClient.cpp | 3 +++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tl/generate/scheme/tonlib_api.tl b/tl/generate/scheme/tonlib_api.tl index 4527cc09..d5f97cce 100644 --- a/tl/generate/scheme/tonlib_api.tl +++ b/tl/generate/scheme/tonlib_api.tl @@ -208,7 +208,7 @@ blocks.shards shards:vector = blocks.Shards; blocks.accountTransactionId account:bytes lt:int64 = blocks.AccountTransactionId; blocks.shortTxId mode:# account:mode.0?bytes lt:mode.1?int64 hash:mode.2?bytes = liteServer.TransactionId; blocks.transactions id:ton.blockIdExt req_count:int32 incomplete:Bool transactions:vector = blocks.Transactions; -blocks.header id:ton.blockIdExt global_id:int32 version:int32 after_merge:Bool after_split:Bool before_split:Bool want_merge:Bool want_split:Bool validator_list_hash_short:int32 catchain_seqno:int32 min_ref_mc_seqno:int32 is_key_block:Bool prev_key_block_seqno:int32 start_lt:int64 end_lt:int64 vert_seqno:# prev_blocks:vector = blocks.Header; +blocks.header id:ton.blockIdExt global_id:int32 version:int32 flags:# after_merge:Bool after_split:Bool before_split:Bool want_merge:Bool want_split:Bool validator_list_hash_short:int32 catchain_seqno:int32 min_ref_mc_seqno:int32 is_key_block:Bool prev_key_block_seqno:int32 start_lt:int64 end_lt:int64 gen_utime:int53 vert_seqno:# prev_blocks:vector = blocks.Header; //blocks.shortData header:blocks.Header transactions:blocks.Header = blocks.BlockData; configInfo config:tvm.cell = ConfigInfo; diff --git a/tl/generate/scheme/tonlib_api.tlo b/tl/generate/scheme/tonlib_api.tlo index d5a8344d6d832315af5525ff6b0288e42aa7e453..05c56ac08283bea94ca484f2c04acf3d2ac1d224 100644 GIT binary patch delta 141 zcmdmShq2)<B25imd7di)&z^ut@{Bb%7 delta 102 zcmZp;%ednX Date: Sun, 24 Apr 2022 16:57:38 +0300 Subject: [PATCH 19/21] Asm.fif typo in IFBITJMPs --- crypto/fift/lib/Asm.fif | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crypto/fift/lib/Asm.fif b/crypto/fift/lib/Asm.fif index edd0e24b..71c6a956 100644 --- a/crypto/fift/lib/Asm.fif +++ b/crypto/fift/lib/Asm.fif @@ -654,7 +654,7 @@ x{DB32} dup @Defop BRANCH @Defop RETBOOL x{DB34} @Defop CALLCC x{DB35} @Defop JMPXDATA { dup 1+ 0= { 16 + } if - Date: Mon, 2 May 2022 02:10:21 +0300 Subject: [PATCH 20/21] Add support for `#pragma version` and FunC versioning FunC pragma refined error messages and ^ partials --- crypto/func/func.cpp | 10 ++- crypto/func/func.h | 5 +- crypto/func/keywords.cpp | 2 + crypto/func/parse-func.cpp | 131 ++++++++++++++++++++++++++++++++++++- crypto/func/test/pv.fc | 53 +++++++++++++++ 5 files changed, 197 insertions(+), 4 deletions(-) create mode 100644 crypto/func/test/pv.fc diff --git a/crypto/func/func.cpp b/crypto/func/func.cpp index e8e09961..43bba4ea 100644 --- a/crypto/func/func.cpp +++ b/crypto/func/func.cpp @@ -173,6 +173,7 @@ void usage(const char* progname) { "-R\tInclude operation rewrite comments in the output code\n" "-W\tInclude Fift code to serialize and save generated code into specified BoC file. Enables " "-A and -P.\n" + "\t-s\tOutput semantic version of FunC and exit\n" "\t-V\tShow func build information\n"; std::exit(2); } @@ -182,7 +183,7 @@ std::string output_filename; int main(int argc, char* const argv[]) { int i; bool interactive = false; - while ((i = getopt(argc, argv, "Ahi:Io:O:PRSvW:V")) != -1) { + while ((i = getopt(argc, argv, "Ahi:Io:O:PRsSvW:V")) != -1) { switch (i) { case 'A': funC::asm_preamble = true; @@ -215,8 +216,13 @@ int main(int argc, char* const argv[]) { funC::boc_output_filename = optarg; funC::asm_preamble = funC::program_envelope = true; break; + case 's': + std::cout << funC::func_version << "\n"; + std::exit(0); + break; case 'V': - std::cout << "Func build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; + std::cout << "FunC semantic version: v" << funC::func_version << "\n"; + std::cout << "Build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; std::exit(0); break; case 'h': diff --git a/crypto/func/func.h b/crypto/func/func.h index 1506b974..22935dfe 100644 --- a/crypto/func/func.h +++ b/crypto/func/func.h @@ -38,6 +38,8 @@ extern bool op_rewrite_comments; constexpr int optimize_depth = 20; +const std::string func_version{"0.1.0"}; + enum Keyword { _Eof = -1, _Ident = 0, @@ -106,7 +108,8 @@ enum Keyword { _Operator, _Infix, _Infixl, - _Infixr + _Infixr, + _PragmaHashtag }; void define_keywords(); diff --git a/crypto/func/keywords.cpp b/crypto/func/keywords.cpp index 47975e7f..785c7765 100644 --- a/crypto/func/keywords.cpp +++ b/crypto/func/keywords.cpp @@ -126,6 +126,8 @@ void define_keywords() { .add_keyword("infix", Kw::_Infix) .add_keyword("infixl", Kw::_Infixl) .add_keyword("infixr", Kw::_Infixr); + + sym::symbols.add_keyword("#pragma", Kw::_PragmaHashtag); } } // namespace funC diff --git a/crypto/func/parse-func.cpp b/crypto/func/parse-func.cpp index 3aad90ef..d340a78c 100644 --- a/crypto/func/parse-func.cpp +++ b/crypto/func/parse-func.cpp @@ -1288,13 +1288,142 @@ void parse_func_def(Lexer& lex) { sym::close_scope(lex); } +std::string func_ver_test = func_version; + +void parse_pragma(Lexer& lex) { + auto pragma = lex.cur(); + lex.next(); + if (lex.tp() != _Ident) { + lex.expect(_Ident, "pragma name expected"); + } + auto pragma_name = lex.cur().str; + lex.next(); + if (!pragma_name.compare("version") || !pragma_name.compare("not-version")) { + bool negate = !pragma_name.compare("not-version"); + char op = '='; bool eq = false; + int sem_ver[3] = {0, 0, 0}; + char segs = 1; + if (lex.tp() == _Number) { + sem_ver[0] = std::stoi(lex.cur().str); + } else if (lex.tp() == _Ident) { + auto id1 = lex.cur().str; + char ch1 = id1[0]; + if ((ch1 == '>') || (ch1 == '<') || (ch1 == '=') || (ch1 == '^')) { + op = ch1; + } else { + lex.cur().error("unexpected comparator operation"); + } + if (id1.length() < 2) { + lex.cur().error("expected number after comparator"); + } + if (id1[1] == '=') { + eq = true; + if (id1.length() < 3) { + lex.cur().error("expected number after comparator"); + } + sem_ver[0] = std::stoi(id1.substr(2)); + } else { + sem_ver[0] = std::stoi(id1.substr(1)); + } + } else { + lex.cur().error("expected semver with optional comparator"); + } + lex.next(); + if (lex.tp() != ';') { + if (lex.tp() != _Ident || lex.cur().str[0] != '.') { + lex.cur().error("invalid semver format"); + } + sem_ver[1] = std::stoi(lex.cur().str.substr(1)); + segs = 2; + lex.next(); + } + if (lex.tp() != ';') { + if (lex.tp() != _Ident || lex.cur().str[0] != '.') { + lex.cur().error("invalid semver format"); + } + sem_ver[2] = std::stoi(lex.cur().str.substr(1)); + segs = 3; + lex.next(); + } + // End reading semver from source code + int func_ver[3] = {0, 0, 0}; + std::istringstream iss(func_ver_test); + std::string s; + for (int idx = 0; idx < 3; idx++) { + std::getline(iss, s, '.'); + func_ver[idx] = std::stoi(s); + } + // End parsing embedded semver + std::string semver_expr; + if (negate) { + semver_expr += '!'; + } + semver_expr += op; + if (eq) { + semver_expr += '='; + } + for (int idx = 0; idx < 3; idx++) { + semver_expr += std::to_string(sem_ver[idx]); + if (idx < 2) + semver_expr += '.'; + } + bool match = true; + switch (op) { + case '=': + if ((func_ver[0] != sem_ver[0]) || + (func_ver[1] != sem_ver[1]) || + (func_ver[2] != sem_ver[2])) { + match = false; + } + break; + case '>': + if ( ((func_ver[0] == sem_ver[0]) && (func_ver[1] == sem_ver[1]) && (func_ver[2] == sem_ver[2]) && !eq) || + ((func_ver[0] == sem_ver[0]) && (func_ver[1] == sem_ver[1]) && (func_ver[2] < sem_ver[2])) || + ((func_ver[0] == sem_ver[0]) && (func_ver[1] < sem_ver[1])) || + ((func_ver[0] < sem_ver[0])) ) { + match = false; + } + break; + case '<': + if ( ((func_ver[0] == sem_ver[0]) && (func_ver[1] == sem_ver[1]) && (func_ver[2] == sem_ver[2]) && !eq) || + ((func_ver[0] == sem_ver[0]) && (func_ver[1] == sem_ver[1]) && (func_ver[2] > sem_ver[2])) || + ((func_ver[0] == sem_ver[0]) && (func_ver[1] > sem_ver[1])) || + ((func_ver[0] > sem_ver[0])) ) { + match = false; + } + break; + case '^': + if ( ((segs == 3) && ((func_ver[0] != sem_ver[0]) || (func_ver[1] != sem_ver[1]) || (func_ver[2] < sem_ver[2]))) + || ((segs == 2) && ((func_ver[0] != sem_ver[0]) || (func_ver[1] < sem_ver[1]))) + || ((segs == 1) && ((func_ver[0] < sem_ver[0]))) ) { + match = false; + } + break; + } + if ((match && negate) || (!match && !negate)) { + pragma.error(std::string("FunC version ") + func_ver_test + " does not satisfy condition " + semver_expr); + } + } else if (!pragma_name.compare("test-version-set")) { + if (lex.tp() != _String) { + lex.cur().error("version string expected"); + } + func_ver_test = lex.cur().str; + lex.next(); + } else { + lex.cur().error(std::string{"unknown pragma `"} + pragma_name + "`"); + } + lex.expect(';'); +} + std::vector source_fdescr; bool parse_source(std::istream* is, src::FileDescr* fdescr) { src::SourceReader reader{is, fdescr}; Lexer lex{reader, true, ";,()[] ~."}; while (lex.tp() != _Eof) { - if (lex.tp() == _Global) { + if (lex.tp() == _PragmaHashtag) { + parse_pragma(lex); + } else if (lex.tp() == _Global) { parse_global_var_decls(lex); } else { parse_func_def(lex); diff --git a/crypto/func/test/pv.fc b/crypto/func/test/pv.fc new file mode 100644 index 00000000..d9bcd570 --- /dev/null +++ b/crypto/func/test/pv.fc @@ -0,0 +1,53 @@ +#pragma test-version-set "1.2.3"; + +;; Positive tests +#pragma version ^1.2.0; +#pragma version ^1.2.3; +#pragma version >1.2.0; +#pragma version >0.9.9; +#pragma version <1.3.0; +#pragma version <2.0.0; +#pragma version >=1.2.0; +#pragma version <=1.3.0; +#pragma version >=1.2.3; +#pragma version <=1.2.3; +#pragma version ^1.2.3; +#pragma version 1.2.3; +#pragma version =1.2.3; + +;; Negative tests +#pragma not-version ^1.1.0; +#pragma not-version ^1.0.0; +#pragma not-version ^0.2.3; +#pragma not-version ^2.2.3; +#pragma not-version ^1.3.3; +#pragma not-version >1.2.3; +#pragma not-version <1.2.3; +#pragma not-version ^1.2.4; +#pragma not-version >=1.2.4; +#pragma not-version <=1.2.2; +#pragma not-version 3.2.1; +#pragma not-version =3.2.1; + +;; Test incomplete (partial) version +#pragma version ^1.2; +#pragma version >1.2; +#pragma version <1.3; +#pragma version <2; +#pragma version >=1.2; +#pragma version <=1.3; + +;; Advanced ^ behaviour (partials) +#pragma version ^1.2; +#pragma version ^1.0; +#pragma version ^1; +#pragma version ^0; +#pragma not-version ^1.0.0; +#pragma not-version ^0.0.0; +#pragma not-version ^0.0; +#pragma not-version ^1.3; +#pragma not-version ^2; + +(int) main(int a) { + return a; +} From 0c772185ef86202891cb1ca8ecc8bef78639e980 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Fri, 6 May 2022 10:30:46 +0300 Subject: [PATCH 21/21] Add constants and string literals to funC * Add special string literals "..."? (s,a,u,h,) * Add string literal H (256-bit hash) * Add string literal c (crc32) * Use td::hex_encode instead of homebrew function and add test * Fix error codes and use more generic address * Add support for int and slice constants * Add support for strongly typed constants * Add support for precompiled constant expressions (hard!) Co-authored-by: starlightduck --- crypto/CMakeLists.txt | 12 +-- crypto/func/abscode.cpp | 10 ++ crypto/func/analyzer.cpp | 7 ++ crypto/func/asmops.cpp | 18 ++-- crypto/func/builtins.cpp | 2 +- crypto/func/codegen.cpp | 9 ++ crypto/func/func.h | 49 ++++++++- crypto/func/gen-abscode.cpp | 5 + crypto/func/keywords.cpp | 4 +- crypto/func/parse-func.cpp | 196 +++++++++++++++++++++++++++++++++++- crypto/func/test/co1.fc | 55 ++++++++++ crypto/func/test/s1.fc | 49 +++++++++ crypto/parser/lexer.cpp | 5 + crypto/parser/symtable.h | 2 +- 14 files changed, 397 insertions(+), 26 deletions(-) create mode 100644 crypto/func/test/co1.fc create mode 100644 crypto/func/test/s1.fc diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index 88a0272b..76c8ae9d 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -298,9 +298,14 @@ add_library(src_parser ${PARSER_SOURCE}) target_include_directories(src_parser PUBLIC $) target_link_libraries(src_parser PUBLIC ton_crypto) +add_library(ton_block ${BLOCK_SOURCE}) +target_include_directories(ton_block PUBLIC $ + $ $) +target_link_libraries(ton_block PUBLIC ton_crypto tdutils tdactor tl_api) + add_executable(func func/func.cpp ${FUNC_LIB_SOURCE}) target_include_directories(func PUBLIC $) -target_link_libraries(func PUBLIC ton_crypto src_parser git) +target_link_libraries(func PUBLIC ton_crypto src_parser git ton_block) if (WINGETOPT_FOUND) target_link_libraries_system(func wingetopt) endif() @@ -324,11 +329,6 @@ if (WINGETOPT_FOUND) target_link_libraries_system(pow-miner wingetopt) endif() -add_library(ton_block ${BLOCK_SOURCE}) -target_include_directories(ton_block PUBLIC $ - $ $) -target_link_libraries(ton_block PUBLIC ton_crypto tdutils tdactor tl_api) - set(TURN_OFF_LSAN cd .) if (TON_USE_ASAN AND NOT WIN32) set(TURN_OFF_LSAN export LSAN_OPTIONS=detect_leaks=0) diff --git a/crypto/func/abscode.cpp b/crypto/func/abscode.cpp index 8d73c805..4c893f35 100644 --- a/crypto/func/abscode.cpp +++ b/crypto/func/abscode.cpp @@ -168,6 +168,11 @@ void VarDescr::set_const(td::RefInt256 value) { } } +void VarDescr::set_const(std::string value) { + str_const = value; + val = _Const; +} + void VarDescr::set_const_nan() { set_const(td::make_refint()); } @@ -342,6 +347,11 @@ void Op::show(std::ostream& os, const std::vector& vars, std::string pfx show_var_list(os, left, vars); os << " := " << int_const << std::endl; break; + case _SliceConst: + os << pfx << dis << "SCONST "; + show_var_list(os, left, vars); + os << " := " << str_const << std::endl; + break; case _Import: os << pfx << dis << "IMPORT "; show_var_list(os, left, vars); diff --git a/crypto/func/analyzer.cpp b/crypto/func/analyzer.cpp index 5af435bc..26162521 100644 --- a/crypto/func/analyzer.cpp +++ b/crypto/func/analyzer.cpp @@ -353,6 +353,7 @@ bool Op::compute_used_vars(const CodeBlob& code, bool edit) { } switch (cl) { case _IntConst: + case _SliceConst: case _GlobVar: case _Call: case _CallInd: @@ -540,6 +541,7 @@ bool prune_unreachable(std::unique_ptr& ops) { bool reach; switch (op.cl) { case Op::_IntConst: + case Op::_SliceConst: case Op::_GlobVar: case Op::_SetGlob: case Op::_Call: @@ -707,6 +709,10 @@ VarDescrList Op::fwd_analyze(VarDescrList values) { values.add_newval(left[0]).set_const(int_const); break; } + case _SliceConst: { + values.add_newval(left[0]).set_const(str_const); + break; + } case _Call: { prepare_args(values); auto func = dynamic_cast(fun_ref->value); @@ -848,6 +854,7 @@ bool Op::mark_noreturn() { // fallthrough case _Import: case _IntConst: + case _SliceConst: case _Let: case _Tuple: case _UnTuple: diff --git a/crypto/func/asmops.cpp b/crypto/func/asmops.cpp index f0df9f3d..ccc409d3 100644 --- a/crypto/func/asmops.cpp +++ b/crypto/func/asmops.cpp @@ -55,10 +55,10 @@ std::ostream& operator<<(std::ostream& os, AsmOp::SReg stack_reg) { } } -AsmOp AsmOp::Const(int arg, std::string push_op) { +AsmOp AsmOp::Const(int arg, std::string push_op, td::RefInt256 origin) { std::ostringstream os; os << arg << ' ' << push_op; - return AsmOp::Const(os.str()); + return AsmOp::Const(os.str(), origin); } AsmOp AsmOp::make_stk2(int a, int b, const char* str, int delta) { @@ -166,27 +166,27 @@ AsmOp AsmOp::UnTuple(int a) { AsmOp AsmOp::IntConst(td::RefInt256 x) { if (x->signed_fits_bits(8)) { - return AsmOp::Const(dec_string(std::move(x)) + " PUSHINT"); + return AsmOp::Const(dec_string(std::move(x)) + " PUSHINT", x); } if (!x->is_valid()) { - return AsmOp::Const("PUSHNAN"); + return AsmOp::Const("PUSHNAN", x); } int k = is_pos_pow2(x); if (k >= 0) { - return AsmOp::Const(k, "PUSHPOW2"); + return AsmOp::Const(k, "PUSHPOW2", x); } k = is_pos_pow2(x + 1); if (k >= 0) { - return AsmOp::Const(k, "PUSHPOW2DEC"); + return AsmOp::Const(k, "PUSHPOW2DEC", x); } k = is_pos_pow2(-x); if (k >= 0) { - return AsmOp::Const(k, "PUSHNEGPOW2"); + return AsmOp::Const(k, "PUSHNEGPOW2", x); } if (!x->mod_pow2_short(23)) { - return AsmOp::Const(dec_string(std::move(x)) + " PUSHINTX"); + return AsmOp::Const(dec_string(std::move(x)) + " PUSHINTX", x); } - return AsmOp::Const(dec_string(std::move(x)) + " PUSHINT"); + return AsmOp::Const(dec_string(std::move(x)) + " PUSHINT", x); } AsmOp AsmOp::BoolConst(bool f) { diff --git a/crypto/func/builtins.cpp b/crypto/func/builtins.cpp index 91020be0..075d8f62 100644 --- a/crypto/func/builtins.cpp +++ b/crypto/func/builtins.cpp @@ -27,7 +27,7 @@ using namespace std::literals::string_literals; * */ -int glob_func_cnt, undef_func_cnt, glob_var_cnt; +int glob_func_cnt, undef_func_cnt, glob_var_cnt, const_cnt; std::vector glob_func, glob_vars; SymDef* predefine_builtin_func(std::string name, TypeExpr* func_type) { diff --git a/crypto/func/codegen.cpp b/crypto/func/codegen.cpp index 26d8215e..3e58b995 100644 --- a/crypto/func/codegen.cpp +++ b/crypto/func/codegen.cpp @@ -302,6 +302,15 @@ bool Op::generate_code_step(Stack& stack) { } return true; } + case _SliceConst: { + auto p = next_var_info[left[0]]; + if (!p || p->is_unused()) { + return true; + } + stack.o << AsmOp::Const("x{" + str_const + "} PUSHSLICE"); + stack.push_new_var(left[0]); + return true; + } case _GlobVar: if (dynamic_cast(fun_ref->value)) { bool used = false; diff --git a/crypto/func/func.h b/crypto/func/func.h index 22935dfe..751e50a9 100644 --- a/crypto/func/func.h +++ b/crypto/func/func.h @@ -109,6 +109,7 @@ enum Keyword { _Infix, _Infixl, _Infixr, + _Const, _PragmaHashtag }; @@ -336,6 +337,8 @@ struct VarDescr { static constexpr int FiniteUInt = FiniteInt | _Pos; int val; td::RefInt256 int_const; + std::string str_const; + VarDescr(var_idx_t _idx = -1, int _flags = 0, int _val = 0) : idx(_idx), flags(_flags), val(_val) { } bool operator<(var_idx_t other_idx) const { @@ -406,6 +409,7 @@ struct VarDescr { } void set_const(long long value); void set_const(td::RefInt256 value); + void set_const(std::string value); void set_const_nan(); void operator+=(const VarDescr& y) { flags &= y.flags; @@ -530,7 +534,8 @@ struct Op { _While, _Until, _Repeat, - _Again + _Again, + _SliceConst }; int cl; enum { _Disabled = 1, _Reachable = 2, _NoReturn = 4, _ImpureR = 8, _ImpureW = 16, _Impure = 24 }; @@ -543,6 +548,7 @@ struct Op { std::vector left, right; std::unique_ptr block0, block1; td::RefInt256 int_const; + std::string str_const; Op(const SrcLocation& _where = {}, int _cl = _Undef) : cl(_cl), flags(0), fun_ref(nullptr), where(_where) { } Op(const SrcLocation& _where, int _cl, const std::vector& _left) @@ -554,6 +560,9 @@ struct Op { Op(const SrcLocation& _where, int _cl, const std::vector& _left, td::RefInt256 _const) : cl(_cl), flags(0), fun_ref(nullptr), where(_where), left(_left), int_const(_const) { } + Op(const SrcLocation& _where, int _cl, const std::vector& _left, std::string _const) + : cl(_cl), flags(0), fun_ref(nullptr), where(_where), left(_left), str_const(_const) { + } Op(const SrcLocation& _where, int _cl, const std::vector& _left, const std::vector& _right, SymDef* _fun = nullptr) : cl(_cl), flags(0), fun_ref(_fun), where(_where), left(_left), right(_right) { @@ -784,6 +793,30 @@ struct SymValGlobVar : sym::SymValBase { } }; +struct SymValConst : sym::SymValBase { + td::RefInt256 intval; + std::string strval; + Keyword type; + SymValConst(int idx, td::RefInt256 value) + : sym::SymValBase(_Const, idx), intval(value) { + type = _Int; + } + SymValConst(int idx, std::string value) + : sym::SymValBase(_Const, idx), strval(value) { + type = _Slice; + } + ~SymValConst() override = default; + td::RefInt256 get_int_value() const { + return intval; + } + std::string get_str_value() const { + return strval; + } + Keyword get_type() const { + return type; + } +}; + extern int glob_func_cnt, undef_func_cnt, glob_var_cnt; extern std::vector glob_func, glob_vars; @@ -820,7 +853,8 @@ struct Expr { _LetFirst, _Hole, _Type, - _CondExpr + _CondExpr, + _SliceConst }; int cls; int val{0}; @@ -828,6 +862,7 @@ struct Expr { int flags{0}; SrcLocation here; td::RefInt256 intval; + std::string strval; SymDef* sym{nullptr}; TypeExpr* e_type{nullptr}; std::vector args; @@ -910,6 +945,7 @@ struct AsmOp { int a, b, c; bool gconst{false}; std::string op; + td::RefInt256 origin; struct SReg { int idx; SReg(int _idx) : idx(_idx) { @@ -929,6 +965,9 @@ struct AsmOp { AsmOp(int _t, int _a, int _b, std::string _op) : t(_t), a(_a), b(_b), op(std::move(_op)) { compute_gconst(); } + AsmOp(int _t, int _a, int _b, std::string _op, td::RefInt256 x) : t(_t), a(_a), b(_b), op(std::move(_op)), origin(x) { + compute_gconst(); + } AsmOp(int _t, int _a, int _b, int _c) : t(_t), a(_a), b(_b), c(_c) { } AsmOp(int _t, int _a, int _b, int _c, std::string _op) : t(_t), a(_a), b(_b), c(_c), op(std::move(_op)) { @@ -1047,10 +1086,10 @@ struct AsmOp { static AsmOp make_stk3(int a, int b, int c, const char* str, int delta); static AsmOp IntConst(td::RefInt256 value); static AsmOp BoolConst(bool f); - static AsmOp Const(std::string push_op) { - return AsmOp(a_const, 0, 1, std::move(push_op)); + static AsmOp Const(std::string push_op, td::RefInt256 origin = {}) { + return AsmOp(a_const, 0, 1, std::move(push_op), origin); } - static AsmOp Const(int arg, std::string push_op); + static AsmOp Const(int arg, std::string push_op, td::RefInt256 origin = {}); static AsmOp Comment(std::string comment) { return AsmOp(a_none, std::string{"// "} + comment); } diff --git a/crypto/func/gen-abscode.cpp b/crypto/func/gen-abscode.cpp index c0a8985a..21dce714 100644 --- a/crypto/func/gen-abscode.cpp +++ b/crypto/func/gen-abscode.cpp @@ -362,6 +362,11 @@ std::vector Expr::pre_compile(CodeBlob& code, bool lval) const { code.close_pop_cur(args[2]->here); return rvect; } + case _SliceConst: { + auto rvect = new_tmp_vect(code); + code.emplace_back(here, Op::_SliceConst, rvect, strval); + return rvect; + } default: std::cerr << "expression constructor is " << cls << std::endl; throw src::Fatal{"cannot compile expression with unknown constructor"}; diff --git a/crypto/func/keywords.cpp b/crypto/func/keywords.cpp index 785c7765..b7bba0c2 100644 --- a/crypto/func/keywords.cpp +++ b/crypto/func/keywords.cpp @@ -125,8 +125,8 @@ void define_keywords() { .add_keyword("operator", Kw::_Operator) .add_keyword("infix", Kw::_Infix) .add_keyword("infixl", Kw::_Infixl) - .add_keyword("infixr", Kw::_Infixr); - + .add_keyword("infixr", Kw::_Infixr) + .add_keyword("const", Kw::_Const); sym::symbols.add_keyword("#pragma", Kw::_PragmaHashtag); } diff --git a/crypto/func/parse-func.cpp b/crypto/func/parse-func.cpp index d340a78c..929f5b3c 100644 --- a/crypto/func/parse-func.cpp +++ b/crypto/func/parse-func.cpp @@ -19,6 +19,9 @@ #include "func.h" #include "td/utils/crypto.h" #include "common/refint.h" +#include "openssl/digest.hpp" +#include "block/block.h" +#include "block-parse.h" #include namespace sym { @@ -229,6 +232,83 @@ void parse_global_var_decl(Lexer& lex) { lex.next(); } +extern int const_cnt; +Expr* parse_expr(Lexer& lex, CodeBlob& code, bool nv = false); + +void parse_const_decl(Lexer& lex) { + SrcLocation loc = lex.cur().loc; + int wanted_type = Expr::_None; + if (lex.tp() == _Int) { + wanted_type = Expr::_Const; + lex.next(); + } else if (lex.tp() == _Slice) { + wanted_type = Expr::_SliceConst; + lex.next(); + } + if (lex.tp() != _Ident) { + lex.expect(_Ident, "constant name"); + } + loc = lex.cur().loc; + SymDef* sym_def = sym::define_global_symbol(lex.cur().val, false, loc); + if (!sym_def) { + lex.cur().error_at("cannot define global symbol `", "`"); + } + if (sym_def->value) { + lex.cur().error_at("global symbol `", "` already exists"); + } + lex.next(); + if (lex.tp() != '=') { + lex.cur().error_at("expected = instead of ", ""); + } + lex.next(); + CodeBlob code; + // Handles processing and resolution of literals and consts + auto x = parse_expr(lex, code, false); // also does lex.next() ! + if (x->flags != Expr::_IsRvalue) { + lex.cur().error("expression is not strictly Rvalue"); + } + if ((wanted_type == Expr::_Const) && (x->cls == Expr::_Apply)) + wanted_type = Expr::_None; // Apply is additionally checked to result in an integer + if ((wanted_type != Expr::_None) && (x->cls != wanted_type)) { + lex.cur().error("expression type does not match wanted type"); + } + if (x->cls == Expr::_Const) { // Integer constant + sym_def->value = new SymValConst{const_cnt++, x->intval}; + } else if (x->cls == Expr::_SliceConst) { // Slice constant (string) + sym_def->value = new SymValConst{const_cnt++, x->strval}; + } else if (x->cls == Expr::_Apply) { + code.emplace_back(loc, Op::_Import, std::vector()); + auto tmp_vars = x->pre_compile(code); + code.emplace_back(loc, Op::_Return, std::move(tmp_vars)); + code.emplace_back(loc, Op::_Nop); // This is neccessary to prevent SIGSEGV! + // It is REQUIRED to execute "optimizations" as in func.cpp + code.simplify_var_types(); + code.prune_unreachable_code(); + code.split_vars(true); + for (int i = 0; i < 16; i++) { + code.compute_used_code_vars(); + code.fwd_analyze(); + code.prune_unreachable_code(); + } + code.mark_noreturn(); + AsmOpList out_list(0, &code.vars); + code.generate_code(out_list); + if (out_list.list_.size() != 1) { + lex.cur().error("precompiled expression must result in single operation"); + } + auto op = out_list.list_[0]; + if (!op.is_const()) { + lex.cur().error("precompiled expression must result in compilation time constant"); + } + if (op.origin.is_null() || !op.origin->is_valid()) { + lex.cur().error("precompiled expression did not result in a valid integer constant"); + } + sym_def->value = new SymValConst{const_cnt++, op.origin}; + } else { + lex.cur().error("integer or slice literal or constant expected"); + } +} + FormalArgList parse_formal_args(Lexer& lex) { FormalArgList args; lex.expect('(', "formal argument list"); @@ -246,6 +326,18 @@ FormalArgList parse_formal_args(Lexer& lex) { return args; } +void parse_const_decls(Lexer& lex) { + lex.expect(_Const); + while (true) { + parse_const_decl(lex); + if (lex.tp() != ',') { + break; + } + lex.expect(','); + } + lex.expect(';'); +} + TypeExpr* extract_total_arg_type(const FormalArgList& arg_list) { if (arg_list.empty()) { return TypeExpr::new_unit(); @@ -322,8 +414,6 @@ Expr* make_func_apply(Expr* fun, Expr* x) { return res; } -Expr* parse_expr(Lexer& lex, CodeBlob& code, bool nv = false); - // parse ( E { , E } ) | () | [ E { , E } ] | [] | id | num | _ Expr* parse_expr100(Lexer& lex, CodeBlob& code, bool nv) { if (lex.tp() == '(' || lex.tp() == '[') { @@ -388,6 +478,87 @@ Expr* parse_expr100(Lexer& lex, CodeBlob& code, bool nv) { lex.next(); return res; } + if (t == Lexem::String) { + std::string str = lex.cur().str; + int str_type = lex.cur().val; + Expr* res; + switch (str_type) { + case 0: + case 's': + case 'a': + { + res = new Expr{Expr::_SliceConst, lex.cur().loc}; + res->e_type = TypeExpr::new_atomic(_Slice); + break; + } + case 'u': + case 'h': + case 'H': + case 'c': + { + res = new Expr{Expr::_Const, lex.cur().loc}; + res->e_type = TypeExpr::new_atomic(_Int); + break; + } + default: + { + res = new Expr{Expr::_Const, lex.cur().loc}; + res->e_type = TypeExpr::new_atomic(_Int); + lex.cur().error("invalid string type `" + std::string(1, static_cast(str_type)) + "`"); + return res; + } + } + res->flags = Expr::_IsRvalue; + switch (str_type) { + case 0: { + res->strval = td::hex_encode(str); + break; + } + case 's': { + res->strval = str; + unsigned char buff[128]; + int bits = (int)td::bitstring::parse_bitstring_hex_literal(buff, sizeof(buff), str.data(), str.data() + str.size()); + if (bits < 0) { + lex.cur().error_at("Invalid hex bitstring constant `", "`"); + } + break; + } + case 'a': { // MsgAddressInt + block::StdAddress a; + if (a.parse_addr(str)) { + res->strval = block::tlb::MsgAddressInt().pack_std_address(a)->as_bitslice().to_hex(); + } else { + lex.cur().error_at("invalid standard address `", "`"); + } + break; + } + case 'u': { + res->intval = td::hex_string_to_int256(td::hex_encode(str)); + if (!str.size()) { + lex.cur().error("empty integer ascii-constant"); + } + if (res->intval.is_null()) { + lex.cur().error_at("too long integer ascii-constant `", "`"); + } + break; + } + case 'h': + case 'H': + { + unsigned char hash[32]; + digest::hash_str(hash, str.data(), str.size()); + res->intval = td::bits_to_refint(hash, (str_type == 'h') ? 32 : 256, false); + break; + } + case 'c': + { + res->intval = td::make_refint(td::crc32(td::Slice{str})); + break; + } + } + lex.next(); + return res; + } if (t == '_') { Expr* res = new Expr{Expr::_Hole, lex.cur().loc}; res->val = -1; @@ -429,6 +600,25 @@ Expr* parse_expr100(Lexer& lex, CodeBlob& code, bool nv) { lex.next(); return res; } + if (sym && dynamic_cast(sym->value)) { + auto val = dynamic_cast(sym->value); + Expr* res = new Expr{Expr::_None, lex.cur().loc}; + res->flags = Expr::_IsRvalue; + if (val->type == _Int) { + res->cls = Expr::_Const; + res->intval = val->get_int_value(); + } + else if (val->type == _Slice) { + res->cls = Expr::_SliceConst; + res->strval = val->get_str_value(); + } + else { + lex.cur().error("Invalid symbolic constant type"); + } + res->e_type = TypeExpr::new_atomic(val->type); + lex.next(); + return res; + } bool auto_apply = false; Expr* res = new Expr{Expr::_Var, lex.cur().loc}; if (nv) { @@ -1425,6 +1615,8 @@ bool parse_source(std::istream* is, src::FileDescr* fdescr) { parse_pragma(lex); } else if (lex.tp() == _Global) { parse_global_var_decls(lex); + } else if (lex.tp() == _Const) { + parse_const_decls(lex); } else { parse_func_def(lex); } diff --git a/crypto/func/test/co1.fc b/crypto/func/test/co1.fc new file mode 100644 index 00000000..82071e45 --- /dev/null +++ b/crypto/func/test/co1.fc @@ -0,0 +1,55 @@ +const int1 = 1, int2 = 2; + +const int int101 = 101; +const int int111 = 111; + +const int1r = int1; + +const str1 = "const1", str2 = "aabbcc"s; + +const slice str2r = str2; + +const str1int = 0x636f6e737431; +const str2int = 0xAABBCC; + +const int nibbles = 4; + +int iget1() { return int1; } +int iget2() { return int2; } +int iget3() { return int1 + int2; } + +int iget1r() { return int1r; } + +slice sget1() { return str1; } +slice sget2() { return str2; } +slice sget2r() { return str2r; } + +const int int240 = ((int1 + int2) * 10) << 3; + +int iget240() { return int240; } + +builder newc() asm "NEWC"; +slice endcs(builder b) asm "ENDC" "CTOS"; +int sdeq (slice s1, slice s2) asm "SDEQ"; +builder stslicer(builder b, slice s) asm "STSLICER"; + +_ main() { + int i1 = iget1(); + int i2 = iget2(); + int i3 = iget3(); + + throw_unless(int101, i1 == 1); + throw_unless(102, i2 == 2); + throw_unless(103, i3 == 3); + + slice s1 = sget1(); + slice s2 = sget2(); + slice s3 = newc().stslicer(str1).stslicer(str2r).endcs(); + + throw_unless(int111, sdeq(s1, newc().store_uint(str1int, 12 * nibbles).endcs())); + throw_unless(112, sdeq(s2, newc().store_uint(str2int, 6 * nibbles).endcs())); + throw_unless(113, sdeq(s3, newc().store_uint(0x636f6e737431ABCDEF, 18 * nibbles).endcs())); + + int i4 = iget240(); + throw_unless(104, i4 == 240); +} diff --git a/crypto/func/test/s1.fc b/crypto/func/test/s1.fc new file mode 100644 index 00000000..630d0180 --- /dev/null +++ b/crypto/func/test/s1.fc @@ -0,0 +1,49 @@ +slice ascii_slice() method_id { + return "string"; +} + +slice raw_slice() method_id { + return "abcdef"s; +} + +slice addr_slice() method_id { + return "Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF"a; +} + +int string_hex() method_id { + return "ABCDEFGHIJKLMNOPQRSTUVWXYZ012345"u; +} + +int string_minihash() method_id { + return "transfer(slice, int)"h; +} + +int string_maxihash() method_id { + return "transfer(slice, int)"H; +} + +int string_crc32() method_id { + return "transfer(slice, int)"c; +} + +builder newc() asm "NEWC"; +slice endcs(builder b) asm "ENDC" "CTOS"; +int sdeq (slice s1, slice s2) asm "SDEQ"; + +_ main() { + slice s_ascii = ascii_slice(); + slice s_raw = raw_slice(); + slice s_addr = addr_slice(); + int i_hex = string_hex(); + int i_mini = string_minihash(); + int i_maxi = string_maxihash(); + int i_crc = string_crc32(); + throw_unless(101, sdeq(s_ascii, newc().store_uint(0x737472696E67, 12 * 4).endcs())); + throw_unless(102, sdeq(s_raw, newc().store_uint(0xABCDEF, 6 * 4).endcs())); + throw_unless(103, sdeq(s_addr, newc().store_uint(4, 3).store_int(-1, 8) + .store_uint(0x3333333333333333333333333333333333333333333333333333333333333333, 256).endcs())); + throw_unless(104, i_hex == 0x4142434445464748494A4B4C4D4E4F505152535455565758595A303132333435); + throw_unless(105, i_mini == 0x7a62e8a8); + throw_unless(106, i_maxi == 0x7a62e8a8ebac41bd6de16c65e7be363bc2d2cbc6a0873778dead4795c13db979); + throw_unless(107, i_crc == 2235694568); +} diff --git a/crypto/parser/lexer.cpp b/crypto/parser/lexer.cpp index 87c63c3f..5c5b77d8 100644 --- a/crypto/parser/lexer.cpp +++ b/crypto/parser/lexer.cpp @@ -247,6 +247,11 @@ const Lexem& Lexer::next() { } lexem.set(std::string{src.get_ptr() + 1, end}, src.here(), qc == '`' ? Lexem::Unknown : Lexem::String); src.set_ptr(end + 1); + c = src.cur_char(); + if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) { + lexem.val = c; + src.set_ptr(end + 2); + } // std::cerr << lexem.name_str() << ' ' << lexem.str << std::endl; return lexem; } diff --git a/crypto/parser/symtable.h b/crypto/parser/symtable.h index 81a828a8..9489b2bc 100644 --- a/crypto/parser/symtable.h +++ b/crypto/parser/symtable.h @@ -32,7 +32,7 @@ namespace sym { typedef int var_idx_t; struct SymValBase { - enum { _Param, _Var, _Func, _Typename, _GlobVar }; + enum { _Param, _Var, _Func, _Typename, _GlobVar, _Const }; int type; int idx; SymValBase(int _type, int _idx) : type(_type), idx(_idx) {