From be824e96493414938e0791bdc0c2cd410a131698 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 22 Aug 2022 10:25:28 +0300 Subject: [PATCH 01/34] Check is_applied in liteserver --- validator/impl/liteserver.cpp | 227 ++++++++++++++++++++++------------ validator/impl/liteserver.hpp | 1 + 2 files changed, 149 insertions(+), 79 deletions(-) diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index 023a84ae..255742f9 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -269,15 +269,21 @@ void LiteQuery::perform_getBlock(BlockIdExt blkid) { fatal_error("invalid BlockIdExt"); return; } - td::actor::send_closure_later(manager_, &ValidatorManager::get_block_data_from_db_short, blkid, - [Self = actor_id(this), blkid](td::Result> res) { - if (res.is_error()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); - } else { - td::actor::send_closure_later(Self, &LiteQuery::continue_getBlock, blkid, - res.move_as_ok()); - } - }); + get_block_handle_checked(blkid, [manager = manager_, Self = actor_id(this), blkid](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); + return; + } + td::actor::send_closure_later(manager, &ValidatorManager::get_block_data_from_db, R.move_as_ok(), + [=](td::Result> res) { + if (res.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); + } else { + td::actor::send_closure_later(Self, &LiteQuery::continue_getBlock, blkid, + res.move_as_ok()); + } + }); + }); } void LiteQuery::continue_getBlock(BlockIdExt blkid, Ref block) { @@ -295,15 +301,21 @@ void LiteQuery::perform_getBlockHeader(BlockIdExt blkid, int mode) { fatal_error("invalid BlockIdExt"); return; } - td::actor::send_closure_later(manager_, &ValidatorManager::get_block_data_from_db_short, blkid, - [Self = actor_id(this), blkid, mode](td::Result> res) { - if (res.is_error()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); - } else { - td::actor::send_closure_later(Self, &LiteQuery::continue_getBlockHeader, blkid, - mode, res.move_as_ok()); - } - }); + get_block_handle_checked(blkid, [=, manager = manager_, Self = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); + return; + } + td::actor::send_closure_later(manager, &ValidatorManager::get_block_data_from_db, R.move_as_ok(), + [=](td::Result> res) { + if (res.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); + } else { + td::actor::send_closure_later(Self, &LiteQuery::continue_getBlockHeader, blkid, + mode, res.move_as_ok()); + } + }); + }); } static bool visit(Ref cell); @@ -409,27 +421,33 @@ void LiteQuery::perform_getState(BlockIdExt blkid) { fatal_error("cannot request total state: possibly too large"); return; } - if (blkid.id.seqno) { - td::actor::send_closure_later(manager_, &ValidatorManager::get_shard_state_from_db_short, blkid, - [Self = actor_id(this), blkid](td::Result> res) { - if (res.is_error()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); - } else { - td::actor::send_closure_later(Self, &LiteQuery::continue_getState, blkid, - res.move_as_ok()); - } - }); - } else { - td::actor::send_closure_later(manager_, &ValidatorManager::get_zero_state, blkid, - [Self = actor_id(this), blkid](td::Result res) { - if (res.is_error()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); - } else { - td::actor::send_closure_later(Self, &LiteQuery::continue_getZeroState, blkid, - res.move_as_ok()); - } - }); - } + get_block_handle_checked(blkid, [=, manager = manager_, Self = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); + return; + } + if (blkid.id.seqno) { + td::actor::send_closure_later(manager, &ValidatorManager::get_shard_state_from_db, R.move_as_ok(), + [=](td::Result> res) { + if (res.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); + } else { + td::actor::send_closure_later(Self, &LiteQuery::continue_getState, blkid, + res.move_as_ok()); + } + }); + } else { + td::actor::send_closure_later(manager, &ValidatorManager::get_zero_state, blkid, + [=](td::Result res) { + if (res.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); + } else { + td::actor::send_closure_later(Self, &LiteQuery::continue_getZeroState, blkid, + res.move_as_ok()); + } + }); + } + }); } void LiteQuery::continue_getState(BlockIdExt blkid, Ref state) { @@ -481,6 +499,23 @@ void LiteQuery::perform_sendMessage(td::BufferSlice data) { }); } +void LiteQuery::get_block_handle_checked(BlockIdExt blkid, td::Promise promise) { + auto P = td::PromiseCreator::lambda( + [promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + } else { + auto handle = R.move_as_ok(); + if (handle->is_applied()) { + promise.set_result(std::move(handle)); + } else { + promise.set_error(td::Status::Error("block is not applied")); + } + } + }); + td::actor::send_closure(manager_, &ValidatorManager::get_block_handle, blkid, false, std::move(P)); +} + bool LiteQuery::request_mc_block_data(BlockIdExt blkid) { if (!blkid.is_masterchain() || !blkid.is_valid_full()) { return fatal_error("reference block must belong to the masterchain"); @@ -579,16 +614,22 @@ bool LiteQuery::request_block_state(BlockIdExt blkid) { } blk_id_ = blkid; ++pending_; - td::actor::send_closure_later( - manager_, &ValidatorManager::get_shard_state_from_db_short, blkid, - [Self = actor_id(this), blkid](td::Result> res) { - if (res.is_error()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, - res.move_as_error_prefix("cannot load state for "s + blkid.to_str() + " : ")); - } else { - td::actor::send_closure_later(Self, &LiteQuery::got_block_state, blkid, res.move_as_ok()); - } - }); + get_block_handle_checked(blkid, [=, manager = manager_, Self = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); + return; + } + td::actor::send_closure_later( + manager, &ValidatorManager::get_shard_state_from_db, R.move_as_ok(), + [=](td::Result> res) { + if (res.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, + res.move_as_error_prefix("cannot load state for "s + blkid.to_str() + " : ")); + } else { + td::actor::send_closure_later(Self, &LiteQuery::got_block_state, blkid, res.move_as_ok()); + } + }); + }); return true; } @@ -601,16 +642,22 @@ bool LiteQuery::request_block_data(BlockIdExt blkid) { } blk_id_ = blkid; ++pending_; - td::actor::send_closure_later( - manager_, &ValidatorManager::get_block_data_from_db_short, blkid, - [Self = actor_id(this), blkid](td::Result> res) { - if (res.is_error()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, - res.move_as_error_prefix("cannot load block "s + blkid.to_str() + " : ")); - } else { - td::actor::send_closure_later(Self, &LiteQuery::got_block_data, blkid, res.move_as_ok()); - } - }); + get_block_handle_checked(blkid, [=, manager = manager_, Self = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); + return; + } + td::actor::send_closure_later( + manager, &ValidatorManager::get_block_data_from_db, R.move_as_ok(), + [=](td::Result> res) { + if (res.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, + res.move_as_error_prefix("cannot load block "s + blkid.to_str() + " : ")); + } else { + td::actor::send_closure_later(Self, &LiteQuery::got_block_data, blkid, res.move_as_ok()); + } + }); + }); return true; } @@ -646,16 +693,23 @@ bool LiteQuery::request_proof_link(BlockIdExt blkid) { }); }); } else { - td::actor::send_closure_later( - manager_, &ValidatorManager::get_block_proof_link_from_db_short, blkid, - [Self = actor_id(this), blkid](td::Result> res) { - if (res.is_error()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, - res.move_as_error_prefix("cannot load proof link for "s + blkid.to_str() + " : ")); - } else { - td::actor::send_closure_later(Self, &LiteQuery::got_block_proof_link, blkid, res.move_as_ok()); - } - }); + get_block_handle_checked(blkid, [=, manager = manager_, Self = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); + return; + } + td::actor::send_closure_later( + manager, &ValidatorManager::get_block_proof_link_from_db, R.move_as_ok(), + [=](td::Result> res) { + if (res.is_error()) { + td::actor::send_closure( + Self, &LiteQuery::abort_query, + res.move_as_error_prefix("cannot load proof link for "s + blkid.to_str() + " : ")); + } else { + td::actor::send_closure_later(Self, &LiteQuery::got_block_proof_link, blkid, res.move_as_ok()); + } + }); + }); } return true; } @@ -672,16 +726,22 @@ bool LiteQuery::request_zero_state(BlockIdExt blkid) { } blk_id_ = blkid; ++pending_; - td::actor::send_closure_later( - manager_, &ValidatorManager::get_zero_state, blkid, - [Self = actor_id(this), blkid](td::Result res) { - if (res.is_error()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, - res.move_as_error_prefix("cannot load zerostate of "s + blkid.to_str() + " : ")); - } else { - td::actor::send_closure_later(Self, &LiteQuery::got_zero_state, blkid, res.move_as_ok()); - } - }); + get_block_handle_checked(blkid, [=, manager = manager_, Self = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); + return; + } + td::actor::send_closure_later( + manager, &ValidatorManager::get_zero_state, blkid, + [=](td::Result res) { + if (res.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, + res.move_as_error_prefix("cannot load zerostate of "s + blkid.to_str() + " : ")); + } else { + td::actor::send_closure_later(Self, &LiteQuery::got_zero_state, blkid, res.move_as_ok()); + } + }); + }); return true; } @@ -1397,6 +1457,11 @@ void LiteQuery::continue_getTransactions(unsigned remaining, bool exact) { td::actor::send_closure(Self, &LiteQuery::abort_getTransactions, res.move_as_error(), ton::BlockIdExt{}); } else { auto handle = res.move_as_ok(); + if (!handle->is_applied()) { + td::actor::send_closure(Self, &LiteQuery::abort_getTransactions, td::Status::Error("block is not applied"), + ton::BlockIdExt{}); + return; + } LOG(DEBUG) << "requesting data for block " << handle->id().to_str(); td::actor::send_closure_later(manager, &ValidatorManager::get_block_data_from_db, handle, [Self, blkid = handle->id(), remaining](td::Result> res) { @@ -1722,6 +1787,10 @@ void LiteQuery::perform_lookupBlock(BlockId blkid, int mode, LogicalTime lt, Uni td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { auto handle = res.move_as_ok(); + if (!handle->is_applied()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, td::Status::Error("block is not applied")); + return; + } LOG(DEBUG) << "requesting data for block " << handle->id().to_str(); td::actor::send_closure_later(manager, &ValidatorManager::get_block_data_from_db, handle, [Self, blkid = handle->id(), mode](td::Result> res) { diff --git a/validator/impl/liteserver.hpp b/validator/impl/liteserver.hpp index f1918ceb..2f2e8cba 100644 --- a/validator/impl/liteserver.hpp +++ b/validator/impl/liteserver.hpp @@ -152,6 +152,7 @@ class LiteQuery : public td::actor::Actor { void finish_loadPrevKeyBlock(ton::BlockIdExt blkid, td::Result> res, td::Promise>> promise); + void get_block_handle_checked(BlockIdExt blkid, td::Promise promise); bool request_block_data(BlockIdExt blkid); bool request_block_state(BlockIdExt blkid); bool request_block_data_state(BlockIdExt blkid); From 34c1c548c45dd86ab1e180d8d154cbd6d7db42ea Mon Sep 17 00:00:00 2001 From: andreypfau Date: Mon, 22 Aug 2022 11:41:46 +0300 Subject: [PATCH 02/34] Fix AdnlProxy Pong: 'data' used after it was moved --- adnl/adnl-proxy.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/adnl/adnl-proxy.cpp b/adnl/adnl-proxy.cpp index 10171e0c..81ec8dcd 100644 --- a/adnl/adnl-proxy.cpp +++ b/adnl/adnl-proxy.cpp @@ -188,7 +188,6 @@ void Receiver::receive_from_client(td::IPAddress addr, td::BufferSlice data) { p.data = std::move(data); p.adnl_start_time = start_time(); p.seqno = out_seqno_; - p.data = std::move(data); auto enc = proxy_->encrypt(std::move(p)); From c931e6dc31fb05cf4fcac9e6fd020c308ba29ab4 Mon Sep 17 00:00:00 2001 From: Anthony Tsivarev Date: Thu, 25 Aug 2022 11:04:08 +0300 Subject: [PATCH 03/34] Add social badges Add telegram, Twitter and Stack Overflow badges in Readme --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index df8fefa8..71509731 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,16 @@ +[![Telegram Group][telegram-badge]][telegram-url] +[![Twitter Group][twitter-badge]][twitter-url] +[![Stack Overflow Group][stack-overflow-badge]][stack-overflow-url] + +[telegram-badge]: https://img.shields.io/badge/-TON%20Community-2CA5E0?style=flat&logo=telegram&logoColor=white +[telegram-url]: https://t.me/toncoin +[twitter-badge]: https://img.shields.io/twitter/follow/ton_blockchain +[twitter-url]: https://twitter.com/ton_blockchain +[stack-overflow-badge]: https://img.shields.io/badge/-Stack%20Overflow-FE7A16?style=flat&logo=stack-overflow&logoColor=white +[stack-overflow-url]: https://stackoverflow.com/questions/tagged/ton + + + # TON Main TON monorepo, which includes the code of the node/validator, lite-client, tonlib, FunC compiler, etc. From 1c356a190dd5baee9d9547a460887efb56cfa159 Mon Sep 17 00:00:00 2001 From: tolya-yanot <1449561+tolya-yanot@users.noreply.github.com> Date: Thu, 25 Aug 2022 23:36:19 +0300 Subject: [PATCH 04/34] update README.md badges --- README.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 71509731..177bc3ee 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,12 @@ -[![Telegram Group][telegram-badge]][telegram-url] -[![Twitter Group][twitter-badge]][twitter-url] [![Stack Overflow Group][stack-overflow-badge]][stack-overflow-url] +[![Telegram Foundation Group][telegram-foundation-badge]][telegram-foundation-url] +[![Telegram Community Group][telegram-community-badge]][telegram-community-url] +[![Twitter Group][twitter-badge]][twitter-url] -[telegram-badge]: https://img.shields.io/badge/-TON%20Community-2CA5E0?style=flat&logo=telegram&logoColor=white -[telegram-url]: https://t.me/toncoin +[telegram-foundation-badge]: https://img.shields.io/badge/-TON%20Foundation-2CA5E0?style=flat&logo=telegram&logoColor=white +[telegram-community-badge]: https://img.shields.io/badge/-TON%20Community-2CA5E0?style=flat&logo=telegram&logoColor=white +[telegram-foundation-url]: https://t.me/tonblockchain +[telegram-community-url]: https://t.me/toncoin [twitter-badge]: https://img.shields.io/twitter/follow/ton_blockchain [twitter-url]: https://twitter.com/ton_blockchain [stack-overflow-badge]: https://img.shields.io/badge/-Stack%20Overflow-FE7A16?style=flat&logo=stack-overflow&logoColor=white From b44e5403c8be61f219f1d3941ad2827ac7b06a03 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 29 Aug 2022 11:04:03 +0300 Subject: [PATCH 05/34] Fix DNS resolve in liteclient and tonlib --- crypto/block/block.tlb | 2 +- crypto/smc-envelope/ManualDns.cpp | 6 +++++- lite-client/lite-client.cpp | 5 +++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 20bacaa4..474d1aaa 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -814,7 +814,7 @@ vmc_pushint$1111 value:int32 next:^VmCont = VmCont; // // DNS RECORDS // -_ (HashmapE 256 DNSRecord) = DNS_RecordSet; +_ (HashmapE 256 ^DNSRecord) = DNS_RecordSet; chunk_ref$_ {n:#} ref:^(TextChunks (n + 1)) = TextChunkRef (n + 1); chunk_ref_empty$_ = TextChunkRef 0; diff --git a/crypto/smc-envelope/ManualDns.cpp b/crypto/smc-envelope/ManualDns.cpp index 8f5097c0..42d379c2 100644 --- a/crypto/smc-envelope/ManualDns.cpp +++ b/crypto/smc-envelope/ManualDns.cpp @@ -433,8 +433,12 @@ td::Result> ManualDns::resolve_raw_or_throw(td: } else { if (category.is_zero()) { vm::Dictionary dict(std::move(data), 256); - dict.check_for_each([&](auto cs, td::ConstBitPtr key, int n) { + dict.check_for_each([&](td::Ref cs, td::ConstBitPtr key, int n) { CHECK(n == 256); + if (cs.is_null() || cs->size_ext() != 0x10000) { + return true; + } + cs = vm::load_cell_slice_ref(cs->prefetch_ref()); vec.push_back({name.str(), td::Bits256(key), cs}); return true; }); diff --git a/lite-client/lite-client.cpp b/lite-client/lite-client.cpp index aad3ae6e..ecaa97bc 100644 --- a/lite-client/lite-client.cpp +++ b/lite-client/lite-client.cpp @@ -1812,10 +1812,11 @@ void TestNode::dns_resolve_finish(ton::WorkchainId workchain, ton::StdSmcAddress if (!dict.check_for_each([this, &out](Ref cs, td::ConstBitPtr key, int n) { CHECK(n == 256); td::Bits256 x{key}; - /*if (cs.is_null() || cs->size_ext() != 0x10000) { + if (cs.is_null() || cs->size_ext() != 0x10000) { out << "category " << x << " : value is not a reference" << std::endl; return true; - }*/ + } + cs = vm::load_cell_slice_ref(cs->prefetch_ref()); std::ostringstream os; (void)show_dns_record(os, x, cs, true); out << "category " << x << " : " << os.str() << std::endl; From da65be6ee2985590975d39790d1c7bd2e8c2ad04 Mon Sep 17 00:00:00 2001 From: tolya-yanot <1449561+tolya-yanot@users.noreply.github.com> Date: Tue, 30 Aug 2022 03:30:00 +0300 Subject: [PATCH 06/34] Fix DNS resolve in liteclient and tonlib --- crypto/block/block.tlb | 2 +- crypto/smc-envelope/ManualDns.cpp | 6 +++++- lite-client/lite-client.cpp | 5 +++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index dae517d4..788d2f96 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -814,7 +814,7 @@ vmc_pushint$1111 value:int32 next:^VmCont = VmCont; // // DNS RECORDS // -_ (HashmapE 256 DNSRecord) = DNS_RecordSet; +_ (HashmapE 256 ^DNSRecord) = DNS_RecordSet; chunk_ref$_ {n:#} ref:^(TextChunks (n + 1)) = TextChunkRef (n + 1); chunk_ref_empty$_ = TextChunkRef 0; diff --git a/crypto/smc-envelope/ManualDns.cpp b/crypto/smc-envelope/ManualDns.cpp index 8f5097c0..42d379c2 100644 --- a/crypto/smc-envelope/ManualDns.cpp +++ b/crypto/smc-envelope/ManualDns.cpp @@ -433,8 +433,12 @@ td::Result> ManualDns::resolve_raw_or_throw(td: } else { if (category.is_zero()) { vm::Dictionary dict(std::move(data), 256); - dict.check_for_each([&](auto cs, td::ConstBitPtr key, int n) { + dict.check_for_each([&](td::Ref cs, td::ConstBitPtr key, int n) { CHECK(n == 256); + if (cs.is_null() || cs->size_ext() != 0x10000) { + return true; + } + cs = vm::load_cell_slice_ref(cs->prefetch_ref()); vec.push_back({name.str(), td::Bits256(key), cs}); return true; }); diff --git a/lite-client/lite-client.cpp b/lite-client/lite-client.cpp index aad3ae6e..ecaa97bc 100644 --- a/lite-client/lite-client.cpp +++ b/lite-client/lite-client.cpp @@ -1812,10 +1812,11 @@ void TestNode::dns_resolve_finish(ton::WorkchainId workchain, ton::StdSmcAddress if (!dict.check_for_each([this, &out](Ref cs, td::ConstBitPtr key, int n) { CHECK(n == 256); td::Bits256 x{key}; - /*if (cs.is_null() || cs->size_ext() != 0x10000) { + if (cs.is_null() || cs->size_ext() != 0x10000) { out << "category " << x << " : value is not a reference" << std::endl; return true; - }*/ + } + cs = vm::load_cell_slice_ref(cs->prefetch_ref()); std::ostringstream os; (void)show_dns_record(os, x, cs, true); out << "category " << x << " : " << os.str() << std::endl; From 3e24af5d1588b5e229e22f062b2d7e61429cf62b Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 9 Sep 2022 11:03:53 +0300 Subject: [PATCH 07/34] Tonlib function getMasterchainBlockSignatures --- tl/generate/scheme/tonlib_api.tl | 4 + tl/generate/scheme/tonlib_api.tlo | Bin 28572 -> 29068 bytes tonlib/tonlib/TonlibClient.cpp | 152 ++++++++++++++++++++++++++++++ tonlib/tonlib/TonlibClient.h | 2 + tonlib/tonlib/tonlib-cli.cpp | 16 ++++ 5 files changed, 174 insertions(+) diff --git a/tl/generate/scheme/tonlib_api.tl b/tl/generate/scheme/tonlib_api.tl index c1c3966f..cfbbf9b9 100644 --- a/tl/generate/scheme/tonlib_api.tl +++ b/tl/generate/scheme/tonlib_api.tl @@ -214,6 +214,9 @@ blocks.transactions id:ton.blockIdExt req_count:int32 incomplete:Bool transactio blocks.header id:ton.blockIdExt global_id:int32 version:int32 flags:# after_merge:Bool after_split:Bool before_split:Bool want_merge:Bool want_split:Bool validator_list_hash_short:int32 catchain_seqno:int32 min_ref_mc_seqno:int32 is_key_block:Bool prev_key_block_seqno:int32 start_lt:int64 end_lt:int64 gen_utime:int53 vert_seqno:# prev_blocks:vector = blocks.Header; //blocks.shortData header:blocks.Header transactions:blocks.Header = blocks.BlockData; +blocks.signature node_id_short:int256 signature:bytes = blocks.Signature; +blocks.blockSignatures id:ton.blockIdExt signatures:(vector blocks.signature) = blocks.BlockSignatures; + configInfo config:tvm.cell = ConfigInfo; ---functions--- @@ -300,6 +303,7 @@ blocks.getShards id:ton.blockIdExt = blocks.Shards; blocks.lookupBlock mode:int32 id:ton.blockId lt:int64 utime:int32 = ton.BlockIdExt; blocks.getTransactions id:ton.blockIdExt mode:# count:# after:blocks.accountTransactionId = blocks.Transactions; blocks.getBlockHeader id:ton.blockIdExt = blocks.Header; +blocks.getMasterchainBlockSignatures seqno:int32 = blocks.BlockSignatures; onLiteServerQueryResult id:int64 bytes:bytes = Ok; onLiteServerQueryError id:int64 error:error = Ok; diff --git a/tl/generate/scheme/tonlib_api.tlo b/tl/generate/scheme/tonlib_api.tlo index 0bf3f39e013e73ca71abc5f29e0d8b63c799f58c..4903ee67a0109bc73a6bd53bc4f61efc09016baf 100644 GIT binary patch delta 315 zcmbPppRwmLBk!Zx`c@235WSICm|bS}ZN?X3Njdq+*~NNJASyUBJuk7Ov?#ThVPb*A zWCivf(SsdA+XbL%;L?)|nWQ(bVV7WMY}tI2Ph3U;*(9)HxYA+ z*IUX;fQ)8@I;hB*O^3s_?I1|;<_lUjEM~|S!!-g`gT!wh6Y^mKvZbHh*W}I1Pf3l> zOo=bf$S*3HtQRgd`GTFo?lFi_WOSMQ MGVt+co8SOB03bLLjQ{`u diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index 6e55108d..0f73bf61 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -41,6 +41,7 @@ #include "block/check-proof.h" #include "ton/lite-tl.hpp" #include "ton/ton-shard.h" +#include "lite-client/lite-client-common.h" #include "vm/boc.h" #include "vm/cellops.h" @@ -1316,6 +1317,148 @@ class GetRawAccountState : public td::actor::Actor { } }; +class GetMasterchainBlockSignatures : public td::actor::Actor { + public: + GetMasterchainBlockSignatures(ExtClientRef ext_client_ref, ton::BlockSeqno seqno, td::actor::ActorShared<> parent, + td::Promise>&& promise) + : block_id_short_(ton::masterchainId, ton::shardIdAll, seqno) + , parent_(std::move(parent)) + , promise_(std::move(promise)) { + client_.set_client(ext_client_ref); + } + + void start_up() override { + if (block_id_short_.seqno == 0) { + abort(td::Status::Error("can't get signatures of block #0")); + return; + } + client_.with_last_block([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &GetMasterchainBlockSignatures::abort, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &GetMasterchainBlockSignatures::got_last_block, R.ok().last_block_id); + } + }); + } + + void got_last_block(ton::BlockIdExt id) { + last_block_ = id; + prev_block_id_short_ = block_id_short_; + prev_block_id_short_.seqno--; + client_.send_query( + ton::lite_api::liteServer_lookupBlock(1, ton::create_tl_lite_block_id_simple(prev_block_id_short_), 0, 0), + [SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &GetMasterchainBlockSignatures::abort, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &GetMasterchainBlockSignatures::got_prev_block_id, + ton::create_block_id(R.ok()->id_)); + } + }); + } + + void got_prev_block_id(ton::BlockIdExt id) { + prev_block_id_ = id; + if (prev_block_id_.id != prev_block_id_short_) { + abort(td::Status::Error("got incorrect block header from liteserver")); + return; + } + client_.send_query( + ton::lite_api::liteServer_getBlockProof(0x1001, ton::create_tl_lite_block_id(last_block_), + ton::create_tl_lite_block_id(prev_block_id_)), + [SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &GetMasterchainBlockSignatures::abort, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &GetMasterchainBlockSignatures::got_prev_proof, R.move_as_ok()); + } + }); + } + + void got_prev_proof(lite_api_ptr proof) { + auto R = liteclient::deserialize_proof_chain(std::move(proof)); + if (R.is_error()) { + abort(R.move_as_error()); + return; + } + auto chain = R.move_as_ok(); + if (chain->from != last_block_ || chain->to != prev_block_id_ || !chain->complete) { + abort(td::Status::Error("got invalid proof chain")); + return; + } + auto S = chain->validate(); + if (S.is_error()) { + abort(std::move(S)); + return; + } + client_.send_query( + ton::lite_api::liteServer_lookupBlock(1, ton::create_tl_lite_block_id_simple(block_id_short_), 0, 0), + [SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &GetMasterchainBlockSignatures::abort, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &GetMasterchainBlockSignatures::got_block_id, + ton::create_block_id(R.ok()->id_)); + } + }); + } + + void got_block_id(ton::BlockIdExt id) { + block_id_ = id; + client_.send_query( + ton::lite_api::liteServer_getBlockProof(0x1001, ton::create_tl_lite_block_id(prev_block_id_), + ton::create_tl_lite_block_id(block_id_)), + [SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &GetMasterchainBlockSignatures::abort, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &GetMasterchainBlockSignatures::got_proof, R.move_as_ok()); + } + }); + } + + void got_proof(lite_api_ptr proof) { + auto R = liteclient::deserialize_proof_chain(std::move(proof)); + if (R.is_error()) { + abort(R.move_as_error()); + return; + } + auto chain = R.move_as_ok(); + if (chain->from != prev_block_id_ || chain->to != block_id_ || !chain->complete || chain->links.empty() || + chain->last_link().signatures.empty()) { + abort(td::Status::Error("got invalid proof chain")); + return; + } + auto S = chain->validate(); + if (S.is_error()) { + abort(std::move(S)); + return; + } + std::vector> signatures; + for (const auto& s : chain->last_link().signatures) { + signatures.push_back(ton::create_tl_object(s.node, s.signature.as_slice().str())); + } + promise_.set_result( + ton::create_tl_object(to_tonlib_api(block_id_), std::move(signatures))); + stop(); + } + + void abort(td::Status error) { + promise_.set_error(std::move(error)); + stop(); + } + + private: + ton::BlockId block_id_short_; + td::actor::ActorShared<> parent_; + td::Promise> promise_; + ExtClient client_; + ton::BlockIdExt block_id_; + ton::BlockId prev_block_id_short_; + ton::BlockIdExt prev_block_id_; + ton::BlockIdExt last_block_; +}; + TonlibClient::TonlibClient(td::unique_ptr callback) : callback_(std::move(callback)) { } TonlibClient::~TonlibClient() = default; @@ -4365,6 +4508,15 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_getBlockHeader& req return td::Status::OK(); } +td::Status TonlibClient::do_request(const tonlib_api::blocks_getMasterchainBlockSignatures& request, + td::Promise>&& promise) { + auto actor_id = actor_id_++; + actors_[actor_id] = td::actor::create_actor( + "GetMasterchainBlockSignatures", client_.get_client(), request.seqno_, actor_shared(this, actor_id), + std::move(promise)); + return td::Status::OK(); +} + void TonlibClient::load_libs_from_disk() { LOG(DEBUG) << "loading libraries from disk cache"; auto r_data = kv_->get("tonlib.libcache"); diff --git a/tonlib/tonlib/TonlibClient.h b/tonlib/tonlib/TonlibClient.h index 09c102cd..0e7e2f58 100644 --- a/tonlib/tonlib/TonlibClient.h +++ b/tonlib/tonlib/TonlibClient.h @@ -357,6 +357,8 @@ class TonlibClient : public td::actor::Actor { td::Promise>&& promise); td::Status do_request(const tonlib_api::blocks_getBlockHeader& request, td::Promise>&& promise); + td::Status do_request(const tonlib_api::blocks_getMasterchainBlockSignatures& request, + td::Promise>&& promise); td::Status do_request(const tonlib_api::getConfigParam& request, td::Promise>&& promise); diff --git a/tonlib/tonlib/tonlib-cli.cpp b/tonlib/tonlib/tonlib-cli.cpp index be550454..e889234a 100644 --- a/tonlib/tonlib/tonlib-cli.cpp +++ b/tonlib/tonlib/tonlib-cli.cpp @@ -430,6 +430,7 @@ class TonlibCli : public td::actor::Actor { << "\t 'e' modifier - encrypt all messages\n" << "\t 'k' modifier - use fake key\n" << "\t 'c' modifier - just esmitate fees\n"; + td::TerminalIO::out() << "getmasterchainsignatures - get sigratures of masterchain block \n"; } else if (cmd == "genkey") { generate_key(); } else if (cmd == "exit" || cmd == "quit") { @@ -510,6 +511,9 @@ class TonlibCli : public td::actor::Actor { auto key = parser.read_word(); auto init_key = parser.read_word(); guess_account(key, init_key, std::move(cmd_promise)); + } else if (cmd == "getmasterchainsignatures") { + auto seqno = parser.read_word(); + run_get_masterchain_block_signatures(seqno, std::move(cmd_promise)); } else { cmd_promise.set_error(td::Status::Error(PSLICE() << "Unkwnown query `" << cmd << "`")); } @@ -2096,6 +2100,18 @@ class TonlibCli : public td::actor::Actor { })); } + void run_get_masterchain_block_signatures(td::Slice seqno_s, td::Promise promise) { + TRY_RESULT_PROMISE(promise, seqno, td::to_integer_safe(seqno_s)); + send_query(make_object(seqno), promise.wrap([](auto signatures) { + td::TerminalIO::out() << "Signatures: " << signatures->signatures_.size() << "\n"; + for (const auto& s : signatures->signatures_) { + td::TerminalIO::out() << " " << s->node_id_short_ << " : " << td::base64_encode(td::Slice(s->signature_)) + << "\n"; + } + return td::Unit(); + })); + } + void get_history2(td::Slice key, td::Result> r_state, td::Promise promise) { TRY_RESULT_PROMISE(promise, state, std::move(r_state)); From 8329a589942b69547a9a132b5d091976391b1a8b Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Sat, 10 Sep 2022 12:57:13 +0300 Subject: [PATCH 08/34] Add GetPerfTimerStats (#451) * Add getperfwarningtimeraverage query for validator-engine-console * Fix for getperfwarningtimeraverage query * getperftimerstats * Add history max size: 1 hour * PerfWarningTimer: move callback instead of copy * PerfWarningTimer: fix move constructor bug * PerfWarningTimer: fix bug: lifetime of the callback was greater than lifetime of the local variable 'manager' captured by reference * Fix validate-query.cpp: 'manager' used after it was moved * PerfWarningTimer: remove logs * getperftimerstats: write to json file * getperftimerstatsjson Co-authored-by: legaii Co-authored-by: Ivan Siomash <106972486+legaii@users.noreply.github.com> --- tdutils/td/utils/Timer.cpp | 11 ++-- tdutils/td/utils/Timer.h | 5 +- tl/generate/scheme/ton_api.tl | 6 +++ tl/generate/scheme/ton_api.tlo | Bin 68244 -> 69024 bytes .../validator-engine-console-query.cpp | 50 +++++++++++++++++ .../validator-engine-console-query.h | 21 ++++++++ .../validator-engine-console.cpp | 1 + validator-engine/validator-engine.cpp | 51 ++++++++++++++++++ validator-engine/validator-engine.hpp | 2 + validator/apply-block.hpp | 7 ++- validator/downloaders/wait-block-data.hpp | 7 ++- validator/downloaders/wait-block-state.hpp | 7 ++- validator/impl/accept-block.cpp | 15 ++++-- validator/impl/accept-block.hpp | 2 +- validator/impl/check-proof.hpp | 17 ++++-- validator/impl/collator-impl.h | 2 +- validator/impl/collator.cpp | 5 +- validator/impl/validate-query.cpp | 7 ++- validator/impl/validate-query.hpp | 2 +- validator/manager-disk.hpp | 7 +++ validator/manager-hardfork.hpp | 7 +++ validator/manager.cpp | 18 +++++++ validator/manager.hpp | 5 ++ validator/validate-broadcast.hpp | 7 ++- validator/validator.h | 9 ++++ 25 files changed, 244 insertions(+), 27 deletions(-) diff --git a/tdutils/td/utils/Timer.cpp b/tdutils/td/utils/Timer.cpp index acd77f63..f33a30dc 100644 --- a/tdutils/td/utils/Timer.cpp +++ b/tdutils/td/utils/Timer.cpp @@ -60,12 +60,12 @@ StringBuilder &operator<<(StringBuilder &string_builder, const Timer &timer) { return string_builder << format::as_time(timer.elapsed()); } -PerfWarningTimer::PerfWarningTimer(string name, double max_duration) - : name_(std::move(name)), start_at_(Time::now()), max_duration_(max_duration) { +PerfWarningTimer::PerfWarningTimer(string name, double max_duration, std::function&& callback) + : name_(std::move(name)), start_at_(Time::now()), max_duration_(max_duration), callback_(std::move(callback)) { } PerfWarningTimer::PerfWarningTimer(PerfWarningTimer &&other) - : name_(std::move(other.name_)), start_at_(other.start_at_), max_duration_(other.max_duration_) { + : name_(std::move(other.name_)), start_at_(other.start_at_), max_duration_(other.max_duration_), callback_(std::move(other.callback_)) { other.start_at_ = 0; } @@ -78,8 +78,9 @@ void PerfWarningTimer::reset() { return; } double duration = Time::now() - start_at_; - LOG_IF(WARNING, duration > max_duration_) - << "SLOW: " << tag("name", name_) << tag("duration", format::as_time(duration)); + //LOG_IF(WARNING, duration > max_duration_) + //<< "SLOW: " << tag("name", name_) << tag("duration", format::as_time(duration)); + callback_(duration); start_at_ = 0; } diff --git a/tdutils/td/utils/Timer.h b/tdutils/td/utils/Timer.h index 7dec45cf..64f3e934 100644 --- a/tdutils/td/utils/Timer.h +++ b/tdutils/td/utils/Timer.h @@ -20,6 +20,8 @@ #include "td/utils/StringBuilder.h" +#include + namespace td { class Timer { @@ -44,7 +46,7 @@ class Timer { class PerfWarningTimer { public: - explicit PerfWarningTimer(string name, double max_duration = 0.1); + explicit PerfWarningTimer(string name, double max_duration = 0.1, std::function&& callback = [] (double) {}); PerfWarningTimer(const PerfWarningTimer &) = delete; PerfWarningTimer &operator=(const PerfWarningTimer &) = delete; PerfWarningTimer(PerfWarningTimer &&other); @@ -56,6 +58,7 @@ class PerfWarningTimer { string name_; double start_at_{0}; double max_duration_{0}; + std::function callback_; }; } // namespace td diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 7dc6d8ca..78a682da 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -630,6 +630,10 @@ engine.validator.overlayStatsNode adnl_id:int256 ip_addr:string bdcst_errors:int engine.validator.overlayStats overlay_id:int256 overlay_id_full:PublicKey adnl_id:int256 scope:string nodes:(vector engine.validator.overlayStatsNode) stats:(vector engine.validator.oneStat) = engine.validator.OverlayStats; engine.validator.overlaysStats overlays:(vector engine.validator.overlayStats) = engine.validator.OverlaysStats; +engine.validator.onePerfTimerStat time:int min:double avg:double max:double = engine.validator.OnePerfTimerStat; +engine.validator.perfTimerStatsByName name:string stats:(vector engine.validator.OnePerfTimerStat) = engine.validator.PerfTimerStatsByName; +engine.validator.perfTimerStats stats:(vector engine.validator.PerfTimerStatsByName) = engine.validator.PerfTimerStats; + ---functions--- @@ -680,6 +684,8 @@ engine.validator.importCertificate overlay_id:int256 local_id:adnl.id.short sign engine.validator.signShardOverlayCertificate workchain:int shard:long signed_key:engine.validator.KeyHash expire_at:int max_size:int = overlay.Certificate; engine.validator.importShardOverlayCertificate workchain:int shard:long signed_key:engine.validator.KeyHash cert:overlay.Certificate = engine.validator.Success; +engine.validator.getPerfTimerStats name:string = engine.validator.PerfTimerStats; + ---types--- storage.pong = storage.Pong; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index c49612788983ca6776a9977c3b0bbb6bc2701e05..8abffd746bad60da62e384b2460103ef2cac77a4 100644 GIT binary patch delta 481 zcmbO-m1V(f7T!m*^{p77;LJwebWyd8#mW;EQ}fa@^HTN75_2+B5=-)n^!)Qu15%69 zLNaqxi-JoMOD6kqNKY;meXk-NdLdpOLph4t;>jC0BsI?NYEmVj&Z*KbF?aGoZt2Y_ zVh;R_PMepieotZrJ81I44iO0yQyCaQ5;u{FgBFpHHrH#2Y2NeLF_#IkfS zots!O>7)oxku#ePhi%(IkcQ0@IyhJ~u)Ded*@wj-HIvu$8iYInIvo~9Frz?{FtMw0 zTrnUunBGG*5bV8F1_qGY$tx!4PTtw8ECF&8%pjQIAO)LydV2gA<2Fm)wiji6|HJ8( j(qzF8XhBfS0CEN>2pA`?ouEAV!vqc%kjQ4A4<1qgS`4wL delta 62 zcmZ25n`O#W7T!m*^{p77;N(W$bkWIML_cg+5qIEcwBMYn_C0B{OxGXg%@+L)evGl3 Ociy!Z-JJ5-MhXCk_!hnZ diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index 35d2e65d..261c0616 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -1005,3 +1005,53 @@ td::Status ImportShardOverlayCertificateQuery::receive(td::BufferSlice data) { td::TerminalIO::out() << "successfully sent certificate to overlay manager\n"; return td::Status::OK(); } + +td::Status GetPerfTimerStatsJsonQuery::run() { + TRY_RESULT_ASSIGN(file_name_, tokenizer_.get_token()); + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status GetPerfTimerStatsJsonQuery::send() { + auto b = ton::create_serialize_tl_object(""); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status GetPerfTimerStatsJsonQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + std::ofstream sb(file_name_); + + sb << "{"; + bool gtail = false; + for (const auto &v : f->stats_) { + if (gtail) { + sb << ","; + } else { + gtail = true; + } + + sb << "\n '" << v->name_ << "': {"; + bool tail = false; + for (const auto &stat : v->stats_) { + if (tail) { + sb << ","; + } else { + tail = true; + } + + sb << "\n " << stat->time_ << ": ["; + sb << "\n " << stat->min_ << ","; + sb << "\n " << stat->avg_ << ","; + sb << "\n " << stat->max_; + sb << "\n ]"; + } + sb << "\n }"; + } + sb << "\n}\n"; + sb << std::flush; + + td::TerminalIO::output(std::string("wrote stats to " + file_name_ + "\n")); + return td::Status::OK(); +} diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index 15b77314..ab2141dd 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -1075,3 +1075,24 @@ class ImportShardOverlayCertificateQuery : public Query { std::string in_file_; }; +class GetPerfTimerStatsJsonQuery : public Query { + public: + GetPerfTimerStatsJsonQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "getperftimerstatsjson"; + } + static std::string get_help() { + return "getperftimerstatsjson \tgets min, average and max event processing time for last 60, 300 and 3600 seconds and writes to json file"; + } + std::string name() const override { + return get_name(); + } + + private: + std::string file_name_; +}; diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index 1b3f3376..5ce8526b 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -140,6 +140,7 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); } bool ValidatorEngineConsole::envelope_send_query(td::BufferSlice query, td::Promise promise) { diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index f380b003..9679ee01 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -66,9 +66,11 @@ #include #include #include +#include #include #include "git.h" + Config::Config() { out_port = 3278; full_node = ton::PublicKeyHash::zero(); @@ -3280,6 +3282,55 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getOverla }); } +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getPerfTimerStats &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_default)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + + if (validator_manager_.empty()) { + promise.set_value( + create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "validator manager not started"))); + return; + } + + auto P = td::PromiseCreator::lambda( + [promise = std::move(promise), query = std::move(query)](td::Result> R) mutable { + const std::vector times{60, 300, 3600}; + double now = td::Time::now(); + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + auto r = R.move_as_ok(); + std::vector> by_name; + for (const auto &stats : r) { + if (stats.name == query.name_ || query.name_.empty()) { + std::vector> by_time; + for (const auto &t : times) { + double min = std::numeric_limits::lowest(); + double max = std::numeric_limits::max(); + double sum = 0; + int cnt = 0; + for (const auto &[time, duration] : stats.stats) { + if (now - time <= static_cast(t)) { + min = td::min(min, duration); + max = td::max(max, duration); + sum += duration; + ++cnt; + } + } + by_time.push_back(ton::create_tl_object(t, min, sum / static_cast(cnt), max)); + } + by_name.push_back(ton::create_tl_object(stats.name, std::move(by_time))); + } + } + promise.set_value(ton::create_serialize_tl_object(std::move(by_name))); + } + }); + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::prepare_perf_timer_stats, std::move(P)); +} + void ValidatorEngine::process_control_query(td::uint16 port, ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data, td::Promise promise) { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 3f4fea2c..7284a5be 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -407,6 +407,8 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_getOverlaysStats &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_getPerfTimerStats &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); template void run_control_query(T &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { diff --git a/validator/apply-block.hpp b/validator/apply-block.hpp index 67ee3911..95deb025 100644 --- a/validator/apply-block.hpp +++ b/validator/apply-block.hpp @@ -46,7 +46,10 @@ class ApplyBlock : public td::actor::Actor { , masterchain_block_id_(masterchain_block_id) , manager_(manager) , timeout_(timeout) - , promise_(std::move(promise)) { + , promise_(std::move(promise)) + , perf_timer_("applyblock", 0.1, [manager](double duration) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "applyblock", duration); + }) { } static constexpr td::uint32 apply_block_priority() { @@ -78,7 +81,7 @@ class ApplyBlock : public td::actor::Actor { BlockHandle handle_; td::Ref state_; - td::PerfWarningTimer perf_timer_{"applyblock", 0.1}; + td::PerfWarningTimer perf_timer_; }; } // namespace validator diff --git a/validator/downloaders/wait-block-data.hpp b/validator/downloaders/wait-block-data.hpp index 73097d30..9a03b1cb 100644 --- a/validator/downloaders/wait-block-data.hpp +++ b/validator/downloaders/wait-block-data.hpp @@ -35,7 +35,10 @@ class WaitBlockData : public td::actor::Actor { , priority_(priority) , manager_(manager) , timeout_(timeout) - , promise_(std::move(promise)) { + , promise_(std::move(promise)) + , perf_timer_("waitdata", 1.0, [manager](double duration) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "waitdata", duration); + }) { } void update_timeout(td::Timestamp timeout, td::uint32 priority) { @@ -74,7 +77,7 @@ class WaitBlockData : public td::actor::Actor { bool is_hardfork_ = false; td::Timestamp try_read_static_file_ = td::Timestamp::now(); - //td::PerfWarningTimer perf_timer_{"waitdata", 1.0}; + td::PerfWarningTimer perf_timer_; }; } // namespace validator diff --git a/validator/downloaders/wait-block-state.hpp b/validator/downloaders/wait-block-state.hpp index a93f2971..7cdc0699 100644 --- a/validator/downloaders/wait-block-state.hpp +++ b/validator/downloaders/wait-block-state.hpp @@ -32,7 +32,10 @@ class WaitBlockState : public td::actor::Actor { , priority_(priority) , manager_(manager) , timeout_(timeout) - , promise_(std::move(promise)) { + , promise_(std::move(promise)) + , perf_timer_("waitstate", 1.0, [manager](double duration) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "waitstate", duration); + }) { } void abort_query(td::Status reason); @@ -80,7 +83,7 @@ class WaitBlockState : public td::actor::Actor { bool reading_from_db_ = false; td::Timestamp next_static_file_attempt_; - //td::PerfWarningTimer perf_timer_{"waitstate", 1.0}; + td::PerfWarningTimer perf_timer_; }; } // namespace validator diff --git a/validator/impl/accept-block.cpp b/validator/impl/accept-block.cpp index a5c4d9a1..ff953b66 100644 --- a/validator/impl/accept-block.cpp +++ b/validator/impl/accept-block.cpp @@ -53,7 +53,10 @@ AcceptBlockQuery::AcceptBlockQuery(BlockIdExt id, td::Ref data, std:: , is_fork_(false) , send_broadcast_(send_broadcast) , manager_(manager) - , promise_(std::move(promise)) { + , promise_(std::move(promise)) + , perf_timer_("acceptblock", 0.1, [manager](double duration) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "acceptblock", duration); + }) { state_keep_old_hash_.clear(); state_old_hash_.clear(); state_hash_.clear(); @@ -71,7 +74,10 @@ AcceptBlockQuery::AcceptBlockQuery(AcceptBlockQuery::IsFake fake, BlockIdExt id, , is_fork_(false) , send_broadcast_(false) , manager_(manager) - , promise_(std::move(promise)) { + , promise_(std::move(promise)) + , perf_timer_("acceptblock", 0.1, [manager](double duration) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "acceptblock", duration); + }) { state_keep_old_hash_.clear(); state_old_hash_.clear(); state_hash_.clear(); @@ -86,7 +92,10 @@ AcceptBlockQuery::AcceptBlockQuery(ForceFork ffork, BlockIdExt id, td::Ref top_block_descr_; - td::PerfWarningTimer perf_timer_{"acceptblock", 0.1}; + td::PerfWarningTimer perf_timer_; bool fatal_error(std::string msg, int code = -666); static bool check_send_error(td::actor::ActorId SelfId, td::Status error); diff --git a/validator/impl/check-proof.hpp b/validator/impl/check-proof.hpp index 4461b5e1..5c9a29b5 100644 --- a/validator/impl/check-proof.hpp +++ b/validator/impl/check-proof.hpp @@ -47,7 +47,10 @@ class CheckProof : public td::actor::Actor { , manager_(manager) , timeout_(timeout) , promise_(std::move(promise)) - , skip_check_signatures_(skip_check_signatures) { + , skip_check_signatures_(skip_check_signatures) + , perf_timer_("checkproof", 0.1, [manager](double duration) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "checkproof", duration); + }) { } CheckProof(BlockIdExt id, td::Ref proof, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, bool skip_check_signatures, td::Ref known_state) @@ -58,7 +61,10 @@ class CheckProof : public td::actor::Actor { , timeout_(timeout) , promise_(std::move(promise)) , state_(std::move(known_state)) - , skip_check_signatures_(skip_check_signatures) { + , skip_check_signatures_(skip_check_signatures) + , perf_timer_("checkproof", 0.1, [manager](double duration) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "checkproof", duration); + }) { } CheckProof(BlockIdExt id, td::Ref proof_link, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise) @@ -67,7 +73,10 @@ class CheckProof : public td::actor::Actor { , proof_(std::move(proof_link)) , manager_(manager) , timeout_(timeout) - , promise_(std::move(promise)) { + , promise_(std::move(promise)) + , perf_timer_("checkproof", 0.1, [manager](double duration) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "checkproof", duration); + }) { } private: @@ -114,7 +123,7 @@ class CheckProof : public td::actor::Actor { bool skip_check_signatures_{false}; bool sig_ok_{false}; - td::PerfWarningTimer perf_timer_{"checkproof", 0.1}; + td::PerfWarningTimer perf_timer_; static bool check_send_error(td::actor::ActorId SelfId, td::Status error); template diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 2dced749..1c9faa58 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -208,7 +208,7 @@ class Collator final : public td::actor::Actor { std::vector> collated_roots_; std::unique_ptr block_candidate; - td::PerfWarningTimer perf_timer_{"collate", 0.1}; + td::PerfWarningTimer perf_timer_; // block::Account* lookup_account(td::ConstBitPtr addr) const; std::unique_ptr make_account_from(td::ConstBitPtr addr, Ref account, diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 6aedcc21..84d75e88 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -67,7 +67,10 @@ Collator::Collator(ShardIdFull shard, bool is_hardfork, UnixTime min_ts, BlockId , validator_set_(std::move(validator_set)) , manager(manager) , timeout(timeout) - , main_promise(std::move(promise)) { + , main_promise(std::move(promise)) + , perf_timer_("collate", 0.1, [manager](double duration) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "collate", duration); + }) { } void Collator::start_up() { diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 97a4bab9..61a98bb9 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -59,12 +59,15 @@ ValidateQuery::ValidateQuery(ShardIdFull shard, UnixTime min_ts, BlockIdExt min_ , prev_blocks(std::move(prev)) , block_candidate(std::move(candidate)) , validator_set_(std::move(validator_set)) - , manager(std::move(manager)) + , manager(manager) , timeout(timeout) , main_promise(std::move(promise)) , is_fake_(is_fake) , shard_pfx_(shard_.shard) - , shard_pfx_len_(ton::shard_prefix_length(shard_)) { + , shard_pfx_len_(ton::shard_prefix_length(shard_)) + , perf_timer_("validateblock", 0.1, [manager](double duration) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "validateblock", duration); + }) { proc_hash_.zero(); } diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index 2aef04af..4ddbc445 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -228,7 +228,7 @@ class ValidateQuery : public td::actor::Actor { std::vector> lib_publishers_, lib_publishers2_; - td::PerfWarningTimer perf_timer_{"validateblock", 0.1}; + td::PerfWarningTimer perf_timer_; static constexpr td::uint32 priority() { return 2; diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index b56f8b3e..d5b35d03 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -358,6 +358,13 @@ class ValidatorManagerImpl : public ValidatorManager { UNREACHABLE(); } + void prepare_perf_timer_stats(td::Promise> promise) override { + UNREACHABLE(); + } + + void add_perf_timer_stat(std::string name, double duration) override { + } + void truncate(BlockSeqno seqno, ConstBlockHandle handle, td::Promise promise) override { UNREACHABLE(); } diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 11470cfd..4fb81de6 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -417,6 +417,13 @@ class ValidatorManagerImpl : public ValidatorManager { UNREACHABLE(); } + void prepare_perf_timer_stats(td::Promise> promise) override { + UNREACHABLE(); + } + + void add_perf_timer_stat(std::string name, double duration) override { + } + void truncate(BlockSeqno seqno, ConstBlockHandle handle, td::Promise promise) override { UNREACHABLE(); } diff --git a/validator/manager.cpp b/validator/manager.cpp index d6a9b4dd..b830bc84 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2495,6 +2495,24 @@ void ValidatorManagerImpl::prepare_stats(td::Promise> promise) { + promise.set_value(std::vector(perf_timer_stats)); +} + +void ValidatorManagerImpl::add_perf_timer_stat(std::string name, double duration) { + for (auto &s : perf_timer_stats) { + if (s.name == name) { + double now = td::Time::now(); + while (!s.stats.empty() && s.stats.front().first < now - 3600.0) { + s.stats.pop_front(); + } + s.stats.push_back({td::Time::now(), duration}); + return; + } + } + perf_timer_stats.push_back({name, {{td::Time::now(), duration}}}); +} + void ValidatorManagerImpl::truncate(BlockSeqno seqno, ConstBlockHandle handle, td::Promise promise) { td::actor::send_closure(db_, &Db::truncate, seqno, std::move(handle), std::move(promise)); } diff --git a/validator/manager.hpp b/validator/manager.hpp index d7f2c8a3..761fbad7 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -255,6 +255,8 @@ class ValidatorManagerImpl : public ValidatorManager { std::map, std::vector>>> pending_masterchain_states_; + std::vector perf_timer_stats; + void new_masterchain_block(); void update_shards(); void update_shard_blocks(); @@ -526,6 +528,9 @@ class ValidatorManagerImpl : public ValidatorManager { void prepare_stats(td::Promise>> promise) override; + void prepare_perf_timer_stats(td::Promise> promise) override; + void add_perf_timer_stat(std::string name, double duration) override; + void truncate(BlockSeqno seqno, ConstBlockHandle handle, td::Promise promise) override; void wait_shard_client_state(BlockSeqno seqno, td::Timestamp timeout, td::Promise promise) override; diff --git a/validator/validate-broadcast.hpp b/validator/validate-broadcast.hpp index 8a6fb149..74496fa3 100644 --- a/validator/validate-broadcast.hpp +++ b/validator/validate-broadcast.hpp @@ -44,7 +44,7 @@ class ValidateBroadcast : public td::actor::Actor { td::Ref proof_link_; BlockHandle handle_; - td::PerfWarningTimer perf_timer_{"validatebroadcast", 0.1}; + td::PerfWarningTimer perf_timer_; bool exact_key_block_handle_; td::Ref key_proof_link_; @@ -60,7 +60,10 @@ class ValidateBroadcast : public td::actor::Actor { , last_known_masterchain_block_handle_(std::move(last_known_masterchain_block_handle)) , manager_(manager) , timeout_(timeout) - , promise_(std::move(promise)) { + , promise_(std::move(promise)) + , perf_timer_("validatebroadcast", 0.1, [manager](double duration) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "validatebroadcast", duration); + }) { } void start_up() override; diff --git a/validator/validator.h b/validator/validator.h index fce119a4..49d80286 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -19,6 +19,7 @@ #pragma once #include +#include #include "td/actor/actor.h" @@ -44,6 +45,11 @@ class DownloadToken { virtual ~DownloadToken() = default; }; +struct PerfTimerStats { + std::string name; + std::deque> stats; // +}; + struct ValidatorManagerOptions : public td::CntObject { public: enum class ShardCheckMode { m_monitor, m_validate }; @@ -214,6 +220,9 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void run_ext_query(td::BufferSlice data, td::Promise promise) = 0; virtual void prepare_stats(td::Promise>> promise) = 0; + + virtual void prepare_perf_timer_stats(td::Promise> promise) = 0; + virtual void add_perf_timer_stat(std::string name, double duration) = 0; }; } // namespace validator From e2cca03a78c1632cfa6fb2e9dc37d05509ee6ccb Mon Sep 17 00:00:00 2001 From: AlexeyFSL <112364963+AlexeyFSL@users.noreply.github.com> Date: Wed, 14 Sep 2022 16:36:01 +0700 Subject: [PATCH 09/34] Func and Fift lib for WASM (#455) * Add social badges Add telegram, Twitter and Stack Overflow badges in Readme * update README.md badges * patch for wasm build * fix narrowing conversion error for clang compiler * refactor func code * funcfift lib implementation * fix funcfift lib fix CMakeFile * fix rvalue missing * remove unused field from result json * name fix remove unused target * rename * added script for building funcfiftlib to wasm * fix json fild names * fix commit hash for script * added version function to funcfiftlib * update commit hash for script * add realpath fail processing fix DISABLE_EXCEPTION_CATCHING option * update hash in script Co-authored-by: Anthony Tsivarev Co-authored-by: EmelyanenkoK Co-authored-by: tolya-yanot <1449561+tolya-yanot@users.noreply.github.com> --- .github/script/fift-func-wasm-build-ubuntu.sh | 79 +++++++ CMakeLists.txt | 12 +- crypto/CMakeLists.txt | 50 ++++- crypto/func/func-main.cpp | 127 +++++++++++ crypto/func/func.cpp | 201 ++++++------------ crypto/func/func.h | 15 ++ crypto/func/parse-func.cpp | 9 +- crypto/funcfiftlib/funcfiftlib.cpp | 131 ++++++++++++ crypto/vm/large-boc-serializer.cpp | 2 +- tdutils/td/utils/port/config.h | 8 +- tonlib/CMakeLists.txt | 20 +- 11 files changed, 491 insertions(+), 163 deletions(-) create mode 100755 .github/script/fift-func-wasm-build-ubuntu.sh create mode 100644 crypto/func/func-main.cpp create mode 100644 crypto/funcfiftlib/funcfiftlib.cpp diff --git a/.github/script/fift-func-wasm-build-ubuntu.sh b/.github/script/fift-func-wasm-build-ubuntu.sh new file mode 100755 index 00000000..505ce137 --- /dev/null +++ b/.github/script/fift-func-wasm-build-ubuntu.sh @@ -0,0 +1,79 @@ +# The script build funcfift compiler to WASM + +# dependencies: +#sudo apt-get install -y build-essential git make cmake clang libgflags-dev zlib1g-dev libssl-dev libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev python3-pip nodejs + +export CC=$(which clang) +export CXX=$(which clang++) +export CCACHE_DISABLE=1 + +git clone https://github.com/openssl/openssl.git +cd openssl +git checkout OpenSSL_1_1_1j + +./config +make -j4 + +OPENSSL_DIR=`pwd` + +cd .. + +git clone https://github.com/madler/zlib.git +cd zlib +ZLIB_DIR=`pwd` + +cd .. + +# clone ton repo +git clone --recursive https://github.com/the-ton-tech/ton-blockchain.git + +# only to generate auto-block.cpp + +cd ton-blockchain +git pull +git checkout 1566a23b2bece49fd1de9ab2f35e88297d22829f +mkdir build +cd build +cmake -DCMAKE_BUILD_TYPE=Release -DZLIB_LIBRARY=/usr/lib/x86_64-linux-gnu/libz.so -DZLIB_INCLUDE_DIR=$ZLIB_DIR -DOPENSSL_ROOT_DIR=$OPENSSL_DIR -DOPENSSL_INCLUDE_DIR=$OPENSSL_DIR/include -DOPENSSL_CRYPTO_LIBRARY=$OPENSSL_DIR/libcrypto.so -DOPENSSL_SSL_LIBRARY=$OPENSSL_DIR/libssl.so .. +make -j4 fift + +rm -rf * + +cd ../.. + +git clone https://github.com/emscripten-core/emsdk.git +cd emsdk +./emsdk install latest +./emsdk activate latest +EMSDK_DIR=`pwd` + +source $EMSDK_DIR/emsdk_env.sh +export CC=$(which emcc) +export CXX=$(which em++) +export CCACHE_DISABLE=1 + +cd ../zlib + +emconfigure ./configure --static +emmake make -j4 +ZLIB_DIR=`pwd` + +cd ../openssl + +make clean +emconfigure ./Configure linux-generic32 no-shared no-dso no-engine no-unit-test no-ui +sed -i 's/CROSS_COMPILE=.*/CROSS_COMPILE=/g' Makefile +sed -i 's/-ldl//g' Makefile +sed -i 's/-O3/-Os/g' Makefile +emmake make depend +emmake make -j4 + +cd ../ton-blockchain + +cd build + +emcmake cmake -DUSE_EMSCRIPTEN=ON -DCMAKE_BUILD_TYPE=Release -DZLIB_LIBRARY=$ZLIB_DIR/libz.a -DZLIB_INCLUDE_DIR=$ZLIB_DIR -DOPENSSL_ROOT_DIR=$OPENSSL_DIR -DOPENSSL_INCLUDE_DIR=$OPENSSL_DIR/include -DOPENSSL_CRYPTO_LIBRARY=$OPENSSL_DIR/libcrypto.a -DOPENSSL_SSL_LIBRARY=$OPENSSL_DIR/libssl.a -DCMAKE_TOOLCHAIN_FILE=$EMSDK_DIR/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake -DCMAKE_CXX_FLAGS="-pthread -sUSE_ZLIB=1" .. + +cp -R ../crypto/smartcont ../crypto/fift/lib crypto + +emmake make -j4 funcfiftlib diff --git a/CMakeLists.txt b/CMakeLists.txt index 44dbae8b..936b74e1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -84,7 +84,11 @@ set(CMAKE_CXX_STANDARD_REQUIRED TRUE) set(CMAKE_CXX_EXTENSIONS FALSE) #BEGIN internal +option(USE_EMSCRIPTEN "Use \"ON\" for config building wasm." OFF) option(TON_ONLY_TONLIB "Use \"ON\" to build only tonlib." OFF) +if (USE_EMSCRIPTEN) + set(TON_ONLY_TONLIB true) +endif() if (TON_ONLY_TONLIB) set(NOT_TON_ONLY_TONLIB false) else() @@ -242,10 +246,14 @@ elseif (CLANG OR GCC) set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} -fvisibility=hidden -Wl,-dead_strip,-x,-S") else() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ffunction-sections -fdata-sections") - set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--gc-sections -Wl,--exclude-libs,ALL") + if (NOT USE_EMSCRIPTEN) + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--gc-sections -Wl,--exclude-libs,ALL") + endif() set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections") if (NOT TON_USE_ASAN AND NOT TON_USE_TSAN AND NOT MEMPROF) - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--exclude-libs,ALL") + if (NOT USE_EMSCRIPTEN) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--exclude-libs,ALL") + endif() endif() endif() endif() diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index 58b98088..c8c85370 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -179,6 +179,7 @@ set(FUNC_LIB_SOURCE func/stack-transform.cpp func/optimize.cpp func/codegen.cpp + func/func.cpp ) set(TLB_BLOCK_AUTO @@ -266,6 +267,8 @@ set(BIGINT_TEST_SOURCE PARENT_SCOPE ) +set(USE_EMSCRIPTEN ${USE_EMSCRIPTEN} PARENT_SCOPE) + add_library(ton_crypto STATIC ${TON_CRYPTO_SOURCE}) target_include_directories(ton_crypto PUBLIC $ @@ -305,13 +308,30 @@ target_include_directories(ton_block PUBLIC $ $) target_link_libraries(ton_block PUBLIC ton_crypto tdutils tdactor tl_api) -add_executable(func func/func.cpp ${FUNC_LIB_SOURCE}) +add_executable(func func/func-main.cpp ${FUNC_LIB_SOURCE}) target_include_directories(func PUBLIC $) target_link_libraries(func PUBLIC ton_crypto src_parser git ton_block) if (WINGETOPT_FOUND) target_link_libraries_system(func wingetopt) endif() +if (USE_EMSCRIPTEN) + add_executable(funcfiftlib funcfiftlib/funcfiftlib.cpp ${FUNC_LIB_SOURCE}) + target_include_directories(funcfiftlib PUBLIC $) + target_link_libraries(funcfiftlib PUBLIC fift-lib src_parser git) + target_link_options(funcfiftlib PRIVATE -sEXPORTED_RUNTIME_METHODS=FS,ccall,cwrap,_malloc,free,UTF8ToString,stringToUTF8) + target_link_options(funcfiftlib PRIVATE -sEXPORTED_FUNCTIONS=_func_compile,_version) + target_link_options(funcfiftlib PRIVATE -sEXPORT_NAME=CompilerModule) + target_link_options(funcfiftlib PRIVATE -sERROR_ON_UNDEFINED_SYMBOLS=0) + target_link_options(funcfiftlib PRIVATE -sFILESYSTEM=1) + target_link_options(funcfiftlib PRIVATE -Oz) + target_link_options(funcfiftlib PRIVATE -sIGNORE_MISSING_MAIN=1) + target_link_options(funcfiftlib PRIVATE -sAUTO_NATIVE_LIBRARIES=0) + target_link_options(funcfiftlib PRIVATE -sMODULARIZE=1) + target_link_options(funcfiftlib PRIVATE --embed-file ${CMAKE_CURRENT_SOURCE_DIR}/fift/lib@/fiftlib) + target_compile_options(funcfiftlib PRIVATE -sDISABLE_EXCEPTION_CATCHING=0) +endif() + add_executable(tlbc tl/tlbc.cpp) target_include_directories(tlbc PUBLIC $) target_link_libraries(tlbc PUBLIC ton_crypto src_parser) @@ -337,16 +357,26 @@ if (TON_USE_ASAN AND NOT WIN32) endif() file(MAKE_DIRECTORY smartcont/auto) -if (NOT CMAKE_CROSSCOMPILING) +if (NOT CMAKE_CROSSCOMPILING OR USE_EMSCRIPTEN) set(GENERATE_TLB_CMD tlbc) - add_custom_command( - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/block - COMMAND ${TURN_OFF_LSAN} - COMMAND ${GENERATE_TLB_CMD} -o block-auto -n block::gen -z block.tlb - COMMENT "Generate block tlb source files" - OUTPUT ${TLB_BLOCK_AUTO} - DEPENDS tlbc block/block.tlb - ) + if (NOT USE_EMSCRIPTEN) + add_custom_command( + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/block + COMMAND ${TURN_OFF_LSAN} + COMMAND ${GENERATE_TLB_CMD} -o block-auto -n block::gen -z block.tlb + COMMENT "Generate block tlb source files" + OUTPUT ${TLB_BLOCK_AUTO} + DEPENDS tlbc block/block.tlb + ) + else() + add_custom_command( + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/block + COMMAND ${TURN_OFF_LSAN} + COMMENT "Generate block tlb source files" + OUTPUT ${TLB_BLOCK_AUTO} + DEPENDS tlbc block/block.tlb + ) + endif() add_custom_target(tlb_generate_block DEPENDS ${TLB_BLOCK_AUTO}) add_dependencies(ton_block tlb_generate_block) diff --git a/crypto/func/func-main.cpp b/crypto/func/func-main.cpp new file mode 100644 index 00000000..45194ea3 --- /dev/null +++ b/crypto/func/func-main.cpp @@ -0,0 +1,127 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. + + Copyright 2017-2020 Telegram Systems LLP +*/ +#include "func.h" +#include "parser/srcread.h" +#include "parser/lexer.h" +#include "parser/symtable.h" +#include +#include +#include "git.h" + +void usage(const char* progname) { + std::cerr + << "usage: " << progname + << " [-vIAPSR][-O][-i][-o][-W] { ...}\n" + "\tGenerates Fift TVM assembler code from a funC source\n" + "-I\tEnables interactive mode (parse stdin)\n" + "-o\tWrites generated code into specified file instead of stdout\n" + "-v\tIncreases verbosity level (extra information output into stderr)\n" + "-i\tSets indentation for the output code (in two-space units)\n" + "-A\tPrefix code with `\"Asm.fif\" include` preamble\n" + "-O\tSets optimization level (2 by default)\n" + "-P\tEnvelope code into PROGRAM{ ... }END>c\n" + "-S\tInclude stack layout comments in the output code\n" + "-R\tInclude operation rewrite comments in the output code\n" + "-W\tInclude Fift code to serialize and save generated code into specified BoC file. Enables " + "-A and -P.\n" + "\t-s\tOutput semantic version of FunC and exit\n" + "\t-V\tShow func build information\n"; + std::exit(2); +} + +int main(int argc, char* const argv[]) { + int i; + std::string output_filename; + while ((i = getopt(argc, argv, "Ahi:Io:O:PRsSvW:V")) != -1) { + switch (i) { + case 'A': + funC::asm_preamble = true; + break; + case 'I': + funC::interactive = true; + break; + case 'i': + funC::indent = std::max(0, atoi(optarg)); + break; + case 'o': + output_filename = optarg; + break; + case 'O': + funC::opt_level = std::max(0, atoi(optarg)); + break; + case 'P': + funC::program_envelope = true; + break; + case 'R': + funC::op_rewrite_comments = true; + break; + case 'S': + funC::stack_layout_comments = true; + break; + case 'v': + ++funC::verbosity; + break; + case 'W': + funC::boc_output_filename = optarg; + funC::asm_preamble = funC::program_envelope = true; + break; + case 's': + std::cout << funC::func_version << "\n"; + std::exit(0); + break; + case 'V': + std::cout << "FunC semantic version: v" << funC::func_version << "\n"; + std::cout << "Build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; + std::exit(0); + break; + case 'h': + default: + usage(argv[0]); + } + } + + std::ostream *outs = &std::cout; + + std::unique_ptr fs; + if (!output_filename.empty()) { + fs = std::make_unique(output_filename, fs->trunc | fs->out); + if (!fs->is_open()) { + std::cerr << "failed to create output file " << output_filename << '\n'; + return 2; + } + outs = fs.get(); + } + + std::vector sources; + + while (optind < argc) { + sources.push_back(std::string(argv[optind++])); + } + + return funC::func_proceed(sources, *outs, std::cerr); +} diff --git a/crypto/func/func.cpp b/crypto/func/func.cpp index b5d769cb..3daac5d7 100644 --- a/crypto/func/func.cpp +++ b/crypto/func/func.cpp @@ -28,16 +28,14 @@ #include "func.h" #include "parser/srcread.h" #include "parser/lexer.h" -#include "parser/symtable.h" #include -#include #include "git.h" namespace funC { int verbosity, indent, opt_level = 2; bool stack_layout_comments, op_rewrite_comments, program_envelope, asm_preamble; -std::ostream* outs = &std::cout; +bool interactive = false; std::string generated_from, boc_output_filename; /* @@ -46,58 +44,59 @@ std::string generated_from, boc_output_filename; * */ -void generate_output_func(SymDef* func_sym) { +void generate_output_func(SymDef* func_sym, std::ostream &outs, std::ostream &errs) { SymValCodeFunc* func_val = dynamic_cast(func_sym->value); assert(func_val); std::string name = sym::symbols.get_name(func_sym->sym_idx); if (verbosity >= 2) { - std::cerr << "\n\n=========================\nfunction " << name << " : " << func_val->get_type() << std::endl; + errs << "\n\n=========================\nfunction " << name << " : " << func_val->get_type() << std::endl; } if (!func_val->code) { - std::cerr << "( function `" << name << "` undefined )\n"; + errs << "( function `" << name << "` undefined )\n"; + throw src::ParseError(func_sym->loc, name); } else { CodeBlob& code = *(func_val->code); if (verbosity >= 3) { - code.print(std::cerr, 9); + code.print(errs, 9); } code.simplify_var_types(); if (verbosity >= 5) { - std::cerr << "after simplify_var_types: \n"; - code.print(std::cerr, 0); + errs << "after simplify_var_types: \n"; + code.print(errs, 0); } code.prune_unreachable_code(); if (verbosity >= 5) { - std::cerr << "after prune_unreachable: \n"; - code.print(std::cerr, 0); + errs << "after prune_unreachable: \n"; + code.print(errs, 0); } code.split_vars(true); if (verbosity >= 5) { - std::cerr << "after split_vars: \n"; - code.print(std::cerr, 0); + errs << "after split_vars: \n"; + code.print(errs, 0); } for (int i = 0; i < 8; i++) { code.compute_used_code_vars(); if (verbosity >= 4) { - std::cerr << "after compute_used_vars: \n"; - code.print(std::cerr, 6); + errs << "after compute_used_vars: \n"; + code.print(errs, 6); } code.fwd_analyze(); if (verbosity >= 5) { - std::cerr << "after fwd_analyze: \n"; - code.print(std::cerr, 6); + errs << "after fwd_analyze: \n"; + code.print(errs, 6); } code.prune_unreachable_code(); if (verbosity >= 5) { - std::cerr << "after prune_unreachable: \n"; - code.print(std::cerr, 6); + errs << "after prune_unreachable: \n"; + code.print(errs, 6); } } code.mark_noreturn(); if (verbosity >= 3) { - code.print(std::cerr, 15); + code.print(errs, 15); } if (verbosity >= 2) { - std::cerr << "\n---------- resulting code for " << name << " -------------\n"; + errs << "\n---------- resulting code for " << name << " -------------\n"; } bool inline_func = (func_val->flags & 1); bool inline_ref = (func_val->flags & 2); @@ -107,7 +106,7 @@ void generate_output_func(SymDef* func_sym) { } else if (inline_ref) { modifier = "REF"; } - *outs << std::string(indent * 2, ' ') << name << " PROC" << modifier << ":<{\n"; + outs << std::string(indent * 2, ' ') << name << " PROC" << modifier << ":<{\n"; int mode = 0; if (stack_layout_comments) { mode |= Stack::_StkCmt | Stack::_CptStkCmt; @@ -120,145 +119,71 @@ void generate_output_func(SymDef* func_sym) { if (fv && (fv->flags & 1) && code.ops->noreturn()) { mode |= Stack::_InlineFunc; } - code.generate_code(*outs, mode, indent + 1); - *outs << std::string(indent * 2, ' ') << "}>\n"; + code.generate_code(outs, mode, indent + 1); + outs << std::string(indent * 2, ' ') << "}>\n"; if (verbosity >= 2) { - std::cerr << "--------------\n"; + errs << "--------------\n"; } } } -int generate_output() { +int generate_output(std::ostream &outs, std::ostream &errs) { if (asm_preamble) { - *outs << "\"Asm.fif\" include\n"; + outs << "\"Asm.fif\" include\n"; } - *outs << "// automatically generated from " << generated_from << std::endl; + outs << "// automatically generated from " << generated_from << std::endl; if (program_envelope) { - *outs << "PROGRAM{\n"; + outs << "PROGRAM{\n"; } for (SymDef* func_sym : glob_func) { SymValCodeFunc* func_val = dynamic_cast(func_sym->value); assert(func_val); std::string name = sym::symbols.get_name(func_sym->sym_idx); - *outs << std::string(indent * 2, ' '); + outs << std::string(indent * 2, ' '); if (func_val->method_id.is_null()) { - *outs << "DECLPROC " << name << "\n"; + outs << "DECLPROC " << name << "\n"; } else { - *outs << func_val->method_id << " DECLMETHOD " << name << "\n"; + outs << func_val->method_id << " DECLMETHOD " << name << "\n"; } } for (SymDef* gvar_sym : glob_vars) { assert(dynamic_cast(gvar_sym->value)); std::string name = sym::symbols.get_name(gvar_sym->sym_idx); - *outs << std::string(indent * 2, ' ') << "DECLGLOBVAR " << name << "\n"; + outs << std::string(indent * 2, ' ') << "DECLGLOBVAR " << name << "\n"; } int errors = 0; for (SymDef* func_sym : glob_func) { try { - generate_output_func(func_sym); + generate_output_func(func_sym, outs, errs); } catch (src::Error& err) { - std::cerr << "cannot generate code for function `" << sym::symbols.get_name(func_sym->sym_idx) << "`:\n" + errs << "cannot generate code for function `" << sym::symbols.get_name(func_sym->sym_idx) << "`:\n" << err << std::endl; ++errors; } } if (program_envelope) { - *outs << "}END>c\n"; + outs << "}END>c\n"; } if (!boc_output_filename.empty()) { - *outs << "2 boc+>B \"" << boc_output_filename << "\" B>file\n"; + outs << "2 boc+>B \"" << boc_output_filename << "\" B>file\n"; } return errors; } -} // namespace funC - -void usage(const char* progname) { - std::cerr - << "usage: " << progname - << " [-vIAPSR][-O][-i][-o][-W] { ...}\n" - "\tGenerates Fift TVM assembler code from a funC source\n" - "-I\tEnables interactive mode (parse stdin)\n" - "-o\tWrites generated code into specified file instead of stdout\n" - "-v\tIncreases verbosity level (extra information output into stderr)\n" - "-i\tSets indentation for the output code (in two-space units)\n" - "-A\tPrefix code with `\"Asm.fif\" include` preamble\n" - "-O\tSets optimization level (2 by default)\n" - "-P\tEnvelope code into PROGRAM{ ... }END>c\n" - "-S\tInclude stack layout comments in the output code\n" - "-R\tInclude operation rewrite comments in the output code\n" - "-W\tInclude Fift code to serialize and save generated code into specified BoC file. Enables " - "-A and -P.\n" - "\t-s\tOutput semantic version of FunC and exit\n" - "\t-V\tShow func build information\n"; - std::exit(2); -} - -void output_inclusion_stack() { +void output_inclusion_stack(std::ostream &errs) { while (!funC::inclusion_locations.empty()) { src::SrcLocation loc = funC::inclusion_locations.top(); funC::inclusion_locations.pop(); if (loc.fdescr) { - std::cerr << "note: included from "; - loc.show(std::cerr); - std::cerr << std::endl; + errs << "note: included from "; + loc.show(errs); + errs << std::endl; } } } -std::string output_filename; - -int main(int argc, char* const argv[]) { - int i; - bool interactive = false; - while ((i = getopt(argc, argv, "Ahi:Io:O:PRsSvW:V")) != -1) { - switch (i) { - case 'A': - funC::asm_preamble = true; - break; - case 'I': - interactive = true; - break; - case 'i': - funC::indent = std::max(0, atoi(optarg)); - break; - case 'o': - output_filename = optarg; - break; - case 'O': - funC::opt_level = std::max(0, atoi(optarg)); - break; - case 'P': - funC::program_envelope = true; - break; - case 'R': - funC::op_rewrite_comments = true; - break; - case 'S': - funC::stack_layout_comments = true; - break; - case 'v': - ++funC::verbosity; - break; - case 'W': - funC::boc_output_filename = optarg; - funC::asm_preamble = funC::program_envelope = true; - break; - case 's': - std::cout << funC::func_version << "\n"; - std::exit(0); - break; - case 'V': - std::cout << "FunC semantic version: v" << funC::func_version << "\n"; - std::cout << "Build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; - std::exit(0); - break; - case 'h': - default: - usage(argv[0]); - } - } +int func_proceed(const std::vector &sources, std::ostream &outs, std::ostream &errs) { if (funC::program_envelope && !funC::indent) { funC::indent = 1; } @@ -268,12 +193,11 @@ int main(int argc, char* const argv[]) { int ok = 0, proc = 0; try { - while (optind < argc) { - // funC::generated_from += std::string{"`"} + argv[optind] + "` "; - ok += funC::parse_source_file(argv[optind++]); + for (auto src : sources) { + ok += funC::parse_source_file(src.c_str()); proc++; } - if (interactive) { + if (funC::interactive) { funC::generated_from += "stdin "; ok += funC::parse_source_stdin(); proc++; @@ -284,29 +208,24 @@ int main(int argc, char* const argv[]) { if (!proc) { throw src::Fatal{"no source files, no output"}; } - std::unique_ptr fs; - if (!output_filename.empty()) { - fs = std::make_unique(output_filename, fs->trunc | fs->out); - if (!fs->is_open()) { - std::cerr << "failed to create output file " << output_filename << '\n'; - return 2; - } - funC::outs = fs.get(); - } - funC::generate_output(); + return funC::generate_output(outs, errs); } catch (src::Fatal& fatal) { - std::cerr << "fatal: " << fatal << std::endl; - output_inclusion_stack(); - std::exit(1); + errs << "fatal: " << fatal << std::endl; + output_inclusion_stack(errs); + return 2; } catch (src::Error& error) { - std::cerr << error << std::endl; - output_inclusion_stack(); - std::exit(1); + errs << error << std::endl; + output_inclusion_stack(errs); + return 2; } catch (funC::UnifyError& unif_err) { - std::cerr << "fatal: "; - unif_err.print_message(std::cerr); - std::cerr << std::endl; - output_inclusion_stack(); - std::exit(1); + errs << "fatal: "; + unif_err.print_message(errs); + errs << std::endl; + output_inclusion_stack(errs); + return 2; } + + return 0; } + +} // namespace funC \ No newline at end of file diff --git a/crypto/func/func.h b/crypto/func/func.h index 886a21ed..2ea1015e 100644 --- a/crypto/func/func.h +++ b/crypto/func/func.h @@ -1665,4 +1665,19 @@ AsmOp push_const(td::RefInt256 x); void define_builtins(); + +extern int verbosity, indent, opt_level; +extern bool stack_layout_comments, op_rewrite_comments, program_envelope, asm_preamble, interactive; +extern std::string generated_from, boc_output_filename; + +/* + * + * OUTPUT CODE GENERATOR + * + */ + +int func_proceed(const std::vector &sources, std::ostream &outs, std::ostream &errs); + } // namespace funC + + diff --git a/crypto/func/parse-func.cpp b/crypto/func/parse-func.cpp index 0d2aa985..439dfb88 100644 --- a/crypto/func/parse-func.cpp +++ b/crypto/func/parse-func.cpp @@ -1657,7 +1657,14 @@ bool parse_source_file(const char* filename, src::Lexem lex) { throw src::Fatal{msg}; } } - std::string real_filename = td::realpath(td::CSlice(filename)).move_as_ok(); + + auto path_res = td::realpath(td::CSlice(filename)); + if (path_res.is_error()) { + auto error = path_res.move_as_error(); + lex.error(error.message().c_str()); + return false; + } + std::string real_filename = path_res.move_as_ok(); if (std::count(source_files.begin(), source_files.end(), real_filename)) { if (verbosity >= 2) { if (lex.tp) { diff --git a/crypto/funcfiftlib/funcfiftlib.cpp b/crypto/funcfiftlib/funcfiftlib.cpp new file mode 100644 index 00000000..6c8912bc --- /dev/null +++ b/crypto/funcfiftlib/funcfiftlib.cpp @@ -0,0 +1,131 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. + + Copyright 2017-2020 Telegram Systems LLP +*/ +#include "func/func.h" +#include "git.h" +#include "td/utils/JsonBuilder.h" +#include "fift/utils.h" +#include "td/utils/base64.h" +#include +#include + +std::string escape_json(const std::string &s) { + std::ostringstream o; + for (auto c = s.cbegin(); c != s.cend(); c++) { + switch (*c) { + case '"': o << "\\\""; break; + case '\\': o << "\\\\"; break; + case '\b': o << "\\b"; break; + case '\f': o << "\\f"; break; + case '\n': o << "\\n"; break; + case '\r': o << "\\r"; break; + case '\t': o << "\\t"; break; + default: + if ('\x00' <= *c && *c <= '\x1f') { + o << "\\u" + << std::hex << std::setw(4) << std::setfill('0') << static_cast(*c); + } else { + o << *c; + } + } + } + return o.str(); +} + +td::Result compile_internal(char *config_json) { + TRY_RESULT(input_json, td::json_decode(td::MutableSlice(config_json))) + auto &obj = input_json.get_object(); + + TRY_RESULT(opt_level, td::get_json_object_int_field(obj, "optLevel", false)); + TRY_RESULT(sources_obj, td::get_json_object_field(obj, "sources", td::JsonValue::Type::Array, false)); + + auto &sources_arr = sources_obj.get_array(); + + std::vector sources; + + for (auto &item : sources_arr) { + sources.push_back(item.get_string().str()); + } + + funC::opt_level = std::max(0, opt_level); + funC::program_envelope = true; + funC::verbosity = 0; + funC::indent = 1; + + std::ostringstream outs, errs; + auto compile_res = funC::func_proceed(sources, outs, errs); + + if (compile_res != 0) { + return td::Status::Error(std::string("Func compilation error: ") + errs.str()); + } + + TRY_RESULT(code_cell, fift::compile_asm(outs.str(), "/fiftlib/", false)); + TRY_RESULT(boc, vm::std_boc_serialize(code_cell)); + + td::JsonBuilder result_json; + auto result_obj = result_json.enter_object(); + result_obj("status", "ok"); + result_obj("codeBoc", td::base64_encode(boc)); + result_obj("fiftCode", escape_json(outs.str())); + result_obj.leave(); + + outs.clear(); + errs.clear(); + + return result_json.string_builder().as_cslice().str(); +} + +extern "C" { + +const char* version() { + auto version_json = td::JsonBuilder(); + auto obj = version_json.enter_object(); + obj("funcVersion", funC::func_version); + obj("funcFiftLibCommitHash", GitMetadata::CommitSHA1()); + obj("funcFiftLibCommitDate", GitMetadata::CommitDate()); + obj.leave(); + return strdup(version_json.string_builder().as_cslice().c_str()); +} + +const char *func_compile(char *config_json) { + auto res = compile_internal(config_json); + + if (res.is_error()) { + auto result = res.move_as_error(); + auto error_res = td::JsonBuilder(); + auto error_o = error_res.enter_object(); + error_o("status", "error"); + error_o("message", result.message().str()); + error_o.leave(); + return strdup(error_res.string_builder().as_cslice().c_str()); + } + + auto res_string = res.move_as_ok(); + + return strdup(res_string.c_str()); +} +} diff --git a/crypto/vm/large-boc-serializer.cpp b/crypto/vm/large-boc-serializer.cpp index 765bd9a6..fe16b767 100644 --- a/crypto/vm/large-boc-serializer.cpp +++ b/crypto/vm/large-boc-serializer.cpp @@ -313,7 +313,7 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { return td::Status::Error("bag of cells is too large"); } - boc_writers::FileWriter writer{fd, info.total_size}; + boc_writers::FileWriter writer{fd, (size_t) info.total_size}; auto store_ref = [&](unsigned long long value) { writer.store_uint(value, info.ref_byte_size); }; diff --git a/tdutils/td/utils/port/config.h b/tdutils/td/utils/port/config.h index 2bd671b0..77143668 100644 --- a/tdutils/td/utils/port/config.h +++ b/tdutils/td/utils/port/config.h @@ -39,7 +39,7 @@ #define TD_EVENTFD_BSD 1 #elif TD_EMSCRIPTEN #define TD_POLL_POLL 1 - #define TD_EVENTFD_UNSUPPORTED 1 + // #define TD_EVENTFD_UNSUPPORTED 1 #elif TD_DARWIN #define TD_POLL_KQUEUE 1 #define TD_EVENTFD_BSD 1 @@ -51,7 +51,11 @@ #endif #if TD_EMSCRIPTEN - #define TD_THREAD_UNSUPPORTED 1 + // #define TD_THREAD_UNSUPPORTED 1 + #define TD_POLL_EPOLL 1 + #define TD_EVENTFD_UNSUPPORTED 0 + #define TD_THREAD_PTHREAD 1 + #define TD_EVENTFD_LINUX 1 #elif TD_TIZEN || TD_LINUX || TD_DARWIN #define TD_THREAD_PTHREAD 1 #else diff --git a/tonlib/CMakeLists.txt b/tonlib/CMakeLists.txt index 5b6530a6..061778af 100644 --- a/tonlib/CMakeLists.txt +++ b/tonlib/CMakeLists.txt @@ -49,6 +49,8 @@ set(TONLIB_SOURCE set(TONLIB_OFFLINE_TEST_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/test/offline.cpp PARENT_SCOPE) set(TONLIB_ONLINE_TEST_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/test/online.cpp PARENT_SCOPE) +set(USE_EMSCRIPTEN ${USE_EMSCRIPTEN} PARENT_SCOPE) + add_library(tonlib STATIC ${TONLIB_SOURCE}) target_include_directories(tonlib PUBLIC @@ -87,8 +89,12 @@ set(TONLIB_JSON_HEADERS tonlib/tonlib_client_json.h) set(TONLIB_JSON_SOURCE tonlib/tonlib_client_json.cpp) include(GenerateExportHeader) +if (NOT USE_EMSCRIPTEN) + add_library(tonlibjson SHARED ${TONLIB_JSON_SOURCE} ${TONLIB_JSON_HEADERS}) +else() + add_library(tonlibjson STATIC ${TONLIB_JSON_SOURCE} ${TONLIB_JSON_HEADERS}) +endif() -add_library(tonlibjson SHARED ${TONLIB_JSON_SOURCE} ${TONLIB_JSON_HEADERS}) target_link_libraries(tonlibjson PRIVATE tonlibjson_private) generate_export_header(tonlibjson EXPORT_FILE_NAME ${CMAKE_CURRENT_BINARY_DIR}/tonlib/tonlibjson_export.h) target_include_directories(tonlibjson PUBLIC @@ -149,11 +155,13 @@ endif() install(FILES ${TONLIB_JSON_HEADERS} ${CMAKE_CURRENT_BINARY_DIR}/tonlib/tonlibjson_export.h DESTINATION include/tonlib/) -install(EXPORT Tonlib - FILE TonlibTargets.cmake - NAMESPACE Tonlib:: - DESTINATION lib/cmake/Tonlib -) +if (NOT USE_EMSCRIPTEN) + install(EXPORT Tonlib + FILE TonlibTargets.cmake + NAMESPACE Tonlib:: + DESTINATION lib/cmake/Tonlib + ) +endif() include(CMakePackageConfigHelpers) write_basic_package_version_file("TonlibConfigVersion.cmake" VERSION ${TON_VERSION} From e40d323fcef788123390038bce10a9845014e8ed Mon Sep 17 00:00:00 2001 From: Andrey Tvorozhkov Date: Wed, 14 Sep 2022 12:36:54 +0300 Subject: [PATCH 10/34] Add ~strdump and STRDUMP TVM OP (#452) * Add ~strdump and STRDUMP TVM OP Add ~strdump and STRDUMP TVM OP Add ~strdump and STRDUMP TVM OP * STRDUMP fixes Co-authored-by: Andrey Tvorozhkov --- crypto/func/builtins.cpp | 2 ++ crypto/vm/debugops.cpp | 41 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/crypto/func/builtins.cpp b/crypto/func/builtins.cpp index 075d8f62..9a3c27e5 100644 --- a/crypto/func/builtins.cpp +++ b/crypto/func/builtins.cpp @@ -1063,6 +1063,8 @@ void define_builtins() { AsmOp::Nop()); define_builtin_func("~dump", TypeExpr::new_forall({X}, TypeExpr::new_map(X, TypeExpr::new_tensor({X, Unit}))), AsmOp::Custom("s0 DUMP", 1, 1), true); + define_builtin_func("~strdump", TypeExpr::new_forall({X}, TypeExpr::new_map(X, TypeExpr::new_tensor({X, Unit}))), + AsmOp::Custom("STRDUMP", 1, 1), true); define_builtin_func("run_method0", TypeExpr::new_map(Int, Unit), [](auto a, auto b, auto c) { return compile_run_method(a, b, c, 0, false); }, true); define_builtin_func("run_method1", TypeExpr::new_forall({X}, TypeExpr::new_map(TypeExpr::new_tensor({Int, X}), Unit)), diff --git a/crypto/vm/debugops.cpp b/crypto/vm/debugops.cpp index 0ec205a3..3f27de23 100644 --- a/crypto/vm/debugops.cpp +++ b/crypto/vm/debugops.cpp @@ -105,6 +105,43 @@ int exec_dump_value(VmState* st, unsigned arg) { return 0; } +int exec_dump_string(VmState* st) { + VM_LOG(st) << "execute STRDUMP"; + if (!vm_debug_enabled) { + return 0; + } + + Stack& stack = st->get_stack(); + + if (stack.depth() > 0){ + auto cs = stack[0].as_slice(); + + if (cs.not_null()) { // wanted t_slice + auto size = cs->size(); + + if (size % 8 == 0) { + auto cnt = size / 8; + + unsigned char tmp[128]; + cs.write().fetch_bytes(tmp, cnt); + std::string s{tmp, tmp + cnt}; + + std::cerr << "#DEBUG#: " << s << std::endl; + } + else { + std::cerr << "#DEBUG#: slice contains not valid bits count" << std::endl; + } + + } else { + std::cerr << "#DEBUG#: is not a slice" << std::endl; + } + } else { + std::cerr << "#DEBUG#: s0 is absent" << std::endl; + } + + return 0; +} + void register_debug_ops(OpcodeTable& cp0) { using namespace std::placeholders; if (!vm_debug_enabled) { @@ -113,7 +150,9 @@ void register_debug_ops(OpcodeTable& cp0) { } else { // NB: all non-redefined opcodes in fe00..feff should be redirected to dummy debug definitions cp0.insert(OpcodeInstr::mksimple(0xfe00, 16, "DUMPSTK", exec_dump_stack)) - .insert(OpcodeInstr::mkfixedrange(0xfe01, 0xfe20, 16, 8, instr::dump_1c_and(0xff, "DEBUG "), exec_dummy_debug)) + .insert(OpcodeInstr::mkfixedrange(0xfe01, 0xfe14, 16, 8, instr::dump_1c_and(0xff, "DEBUG "), exec_dummy_debug)) + .insert(OpcodeInstr::mksimple(0xfe14, 16,"STRDUMP", exec_dump_string)) + .insert(OpcodeInstr::mkfixedrange(0xfe15, 0xfe20, 16, 8, instr::dump_1c_and(0xff, "DEBUG "), exec_dummy_debug)) .insert(OpcodeInstr::mkfixed(0xfe2, 12, 4, instr::dump_1sr("DUMP"), exec_dump_value)) .insert(OpcodeInstr::mkfixedrange(0xfe30, 0xfef0, 16, 8, instr::dump_1c_and(0xff, "DEBUG "), exec_dummy_debug)) .insert(OpcodeInstr::mkext(0xfef, 12, 4, dump_dummy_debug_str, exec_dummy_debug_str, compute_len_debug_str)); From bd5f4f61acc9556a8d0bba3048360c912b9a90cf Mon Sep 17 00:00:00 2001 From: Andrey Tvorozhkov Date: Thu, 15 Sep 2022 09:37:27 +0300 Subject: [PATCH 11/34] Fix dump of null items (#459) Co-authored-by: Andrey Tvorozhkov --- crypto/vm/stack.cpp | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/crypto/vm/stack.cpp b/crypto/vm/stack.cpp index c580a3d0..9a17c8e2 100644 --- a/crypto/vm/stack.cpp +++ b/crypto/vm/stack.cpp @@ -90,15 +90,27 @@ void StackEntry::dump(std::ostream& os) const { os << dec_string(as_int()); break; case t_cell: - os << "C{" << static_cast>(ref)->get_hash().to_hex() << "}"; + if (ref.not_null()) { + os << "C{" << static_cast>(ref)->get_hash().to_hex() << "}"; + } else { + os << "C{null}"; + } break; case t_builder: - os << "BC{" << static_cast>(ref)->to_hex() << "}"; + if (ref.not_null()) { + os << "BC{" << static_cast>(ref)->to_hex() << "}"; + } else { + os << "BC{null}"; + } break; case t_slice: { - os << "CS{"; - static_cast>(ref)->dump(os, 1, false); - os << '}'; + if (ref.not_null()) { + os << "CS{"; + static_cast>(ref)->dump(os, 1, false); + os << '}'; + } else { + os << "CS{null}"; + } break; } case t_string: From 5b2e96c2fa4b9d9128bd63168fa59cdee67508d3 Mon Sep 17 00:00:00 2001 From: Ivan Siomash <106972486+legaii@users.noreply.github.com> Date: Thu, 15 Sep 2022 10:26:17 +0300 Subject: [PATCH 12/34] Fixes for GetPerfTimerStats (#457) * getperftimerstatsjson: fix json format * getperftimerstatsjson: fix td::min ambiguous template parameter --- validator-engine-console/validator-engine-console-query.cpp | 4 ++-- validator-engine/validator-engine.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index 261c0616..1aee7cc4 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -1032,7 +1032,7 @@ td::Status GetPerfTimerStatsJsonQuery::receive(td::BufferSlice data) { gtail = true; } - sb << "\n '" << v->name_ << "': {"; + sb << "\n \"" << v->name_ << "\": {"; bool tail = false; for (const auto &stat : v->stats_) { if (tail) { @@ -1041,7 +1041,7 @@ td::Status GetPerfTimerStatsJsonQuery::receive(td::BufferSlice data) { tail = true; } - sb << "\n " << stat->time_ << ": ["; + sb << "\n \"" << stat->time_ << "\": ["; sb << "\n " << stat->min_ << ","; sb << "\n " << stat->avg_ << ","; sb << "\n " << stat->max_; diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 9679ee01..a8e46eed 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -3314,8 +3314,8 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getPerfTi int cnt = 0; for (const auto &[time, duration] : stats.stats) { if (now - time <= static_cast(t)) { - min = td::min(min, duration); - max = td::max(max, duration); + min = td::min(min, duration); + max = td::max(max, duration); sum += duration; ++cnt; } From d9a5b2ccdc1290c8860f63363fff61cb71086416 Mon Sep 17 00:00:00 2001 From: ms <98183742+dungeon-master-666@users.noreply.github.com> Date: Sun, 18 Sep 2022 15:01:07 +0200 Subject: [PATCH 13/34] Add tonlib function raw.getTransactionsV2 (#461) * Add parameter count to raw.getTransactions * fix tlo * Add tonlib function raw.getTransactionsV2 --- tl/generate/scheme/tonlib_api.tl | 1 + tl/generate/scheme/tonlib_api.tlo | Bin 29120 -> 29384 bytes tonlib/tonlib/TonlibClient.cpp | 59 +++++++++++++++++++++++++++--- tonlib/tonlib/TonlibClient.h | 2 + 4 files changed, 56 insertions(+), 6 deletions(-) diff --git a/tl/generate/scheme/tonlib_api.tl b/tl/generate/scheme/tonlib_api.tl index c4d8eb98..408b1c5d 100644 --- a/tl/generate/scheme/tonlib_api.tl +++ b/tl/generate/scheme/tonlib_api.tl @@ -251,6 +251,7 @@ getBip39Hints prefix:string = Bip39Hints; //raw.init initial_account_state:raw.initialAccountState = Ok; raw.getAccountState account_address:accountAddress = raw.FullAccountState; raw.getTransactions private_key:InputKey account_address:accountAddress from_transaction_id:internal.transactionId = raw.Transactions; +raw.getTransactionsV2 private_key:InputKey account_address:accountAddress from_transaction_id:internal.transactionId count:# try_decode_messages:Bool = raw.Transactions; raw.sendMessage body:bytes = Ok; raw.sendMessageReturnHash body:bytes = raw.ExtMessageInfo; raw.createAndSendMessage destination:accountAddress initial_account_state:bytes data:bytes = Ok; diff --git a/tl/generate/scheme/tonlib_api.tlo b/tl/generate/scheme/tonlib_api.tlo index 69cf1595913df0f72ca80fe016a3e3761d9f3359..e2d32ceac2e8c09c646fc8f61e3b93c00d6e41ae 100644 GIT binary patch delta 95 zcmV-l0HFWC;{nL!0kE)90Y|g3Q8yYOrhGr%6>?#BE@x$QRB~Z%b75n2X>V?GRx$tp z?dbZ1lQ32=lYnbblbBNwlY~ to_balance(td::Ref balance_ref) { class GetTransactionHistory : public td::actor::Actor { public: - GetTransactionHistory(ExtClientRef ext_client_ref, block::StdAddress address, ton::LogicalTime lt, ton::Bits256 hash, + GetTransactionHistory(ExtClientRef ext_client_ref, block::StdAddress address, ton::LogicalTime lt, ton::Bits256 hash, td::int32 count, td::actor::ActorShared<> parent, td::Promise promise) : address_(std::move(address)) , lt_(std::move(lt)) , hash_(std::move(hash)) + , count_(count) , parent_(std::move(parent)) , promise_(std::move(promise)) { client_.set_client(ext_client_ref); @@ -975,7 +976,7 @@ class GetTransactionHistory : public td::actor::Actor { ton::LogicalTime lt_; ton::Bits256 hash_; ExtClient client_; - td::int32 count_{10}; + td::int32 count_; td::actor::ActorShared<> parent_; td::Promise promise_; @@ -2170,10 +2171,14 @@ td::Result to_std_address(td::Ref cs) { return TRY_VM(to_std_address_or_throw(std::move(cs))); } struct ToRawTransactions { - explicit ToRawTransactions(td::optional private_key) : private_key_(std::move(private_key)) { + explicit ToRawTransactions(td::optional private_key, bool try_decode_messages = true) + : private_key_(std::move(private_key)) + , try_decode_messages_(try_decode_messages) { } td::optional private_key_; + bool try_decode_messages_; + td::Result> to_raw_message_or_throw(td::Ref cell) { block::gen::Message::Record message; if (!tlb::type_unpack_cell(cell, block::gen::t_Message_Any, message)) { @@ -2192,7 +2197,7 @@ struct ToRawTransactions { auto get_data = [body = std::move(body), body_cell, this](td::Slice salt) mutable { tonlib_api::object_ptr data; - if (body->size() >= 32 && static_cast(body->prefetch_long(32)) <= 1) { + if (try_decode_messages_ && body->size() >= 32 && static_cast(body->prefetch_long(32)) <= 1) { auto type = body.write().fetch_long(32); td::Status status; @@ -2203,7 +2208,6 @@ struct ToRawTransactions { if (type == 0) { data = tonlib_api::make_object(r_body_message.move_as_ok()); } else { - LOG(ERROR) << "TRY DECRYPT"; auto encrypted_message = r_body_message.move_as_ok(); auto r_decrypted_message = [&]() -> td::Result { if (!private_key_) { @@ -2464,13 +2468,56 @@ td::Status TonlibClient::do_request(tonlib_api::raw_getTransactions& request, auto actor_id = actor_id_++; actors_[actor_id] = td::actor::create_actor( - "GetTransactionHistory", client_.get_client(), account_address, lt, hash, actor_shared(this, actor_id), + "GetTransactionHistory", client_.get_client(), account_address, lt, hash, 10, actor_shared(this, actor_id), promise.wrap([private_key = std::move(private_key)](auto&& x) mutable { return ToRawTransactions(std::move(private_key)).to_raw_transactions(std::move(x)); })); return td::Status::OK(); } +td::Status TonlibClient::do_request(tonlib_api::raw_getTransactionsV2& request, + td::Promise>&& promise) { + if (!request.account_address_) { + return TonlibError::EmptyField("account_address"); + } + if (!request.from_transaction_id_) { + return TonlibError::EmptyField("from_transaction_id"); + } + TRY_RESULT(account_address, get_account_address(request.account_address_->account_address_)); + td::optional private_key; + if (request.private_key_) { + TRY_RESULT(input_key, from_tonlib(*request.private_key_)); + //NB: optional has lot of problems. We use emplace to migitate them + td::optional o_status; + //NB: rely on (and assert) that GetPrivateKey is a synchonous request + make_request(int_api::GetPrivateKey{std::move(input_key)}, [&](auto&& r_key) { + if (r_key.is_error()) { + o_status.emplace(r_key.move_as_error()); + return; + } + o_status.emplace(td::Status::OK()); + private_key = td::Ed25519::PrivateKey(std::move(r_key.move_as_ok().private_key)); + }); + TRY_STATUS(o_status.unwrap()); + } + auto lt = request.from_transaction_id_->lt_; + auto hash_str = request.from_transaction_id_->hash_; + if (hash_str.size() != 32) { + return td::Status::Error(400, "Invalid transaction id hash size"); + } + td::Bits256 hash; + hash.as_slice().copy_from(hash_str); + td::int32 count = request.count_ ? request.count_ : 10; + + auto actor_id = actor_id_++; + actors_[actor_id] = td::actor::create_actor( + "GetTransactionHistory", client_.get_client(), account_address, lt, hash, count, actor_shared(this, actor_id), + promise.wrap([private_key = std::move(private_key), try_decode_messages = request.try_decode_messages_](auto&& x) mutable { + return ToRawTransactions(std::move(private_key), try_decode_messages).to_raw_transactions(std::move(x)); + })); + return td::Status::OK(); +} + td::Status TonlibClient::do_request(const tonlib_api::getAccountState& request, td::Promise>&& promise) { if (!request.account_address_) { diff --git a/tonlib/tonlib/TonlibClient.h b/tonlib/tonlib/TonlibClient.h index 10c83f21..02c06e17 100644 --- a/tonlib/tonlib/TonlibClient.h +++ b/tonlib/tonlib/TonlibClient.h @@ -236,6 +236,8 @@ class TonlibClient : public td::actor::Actor { td::Promise>&& promise); td::Status do_request(tonlib_api::raw_getTransactions& request, td::Promise>&& promise); + td::Status do_request(tonlib_api::raw_getTransactionsV2& request, + td::Promise>&& promise); td::Status do_request(const tonlib_api::getAccountState& request, td::Promise>&& promise); From 1464ae0b0aab8479c300985325d90a6dc1cd8f84 Mon Sep 17 00:00:00 2001 From: Ivan Siomash <106972486+legaii@users.noreply.github.com> Date: Sun, 18 Sep 2022 16:02:08 +0300 Subject: [PATCH 14/34] Remove c++17 structure bindings from GetPerfTimerStats (#460) * Remove c++17 structure bindings --- validator-engine/validator-engine.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index a8e46eed..b6eb8b26 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -3312,7 +3312,9 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getPerfTi double max = std::numeric_limits::max(); double sum = 0; int cnt = 0; - for (const auto &[time, duration] : stats.stats) { + for (const auto &stat : stats.stats) { + double time = stat.first; + double duration = stat.second; if (now - time <= static_cast(t)) { min = td::min(min, duration); max = td::max(max, duration); From 9c6787d2ff27dcb29fad562a9ca64ac650d66d34 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Mon, 19 Sep 2022 17:31:54 +0300 Subject: [PATCH 15/34] Fix undefined behavior code (#464) * Fix UB in arithmetics * Fix misaligned allocations in validator sessions * Fix integer overflow in bigint.hpp * Fix potential UB Co-authored-by: SpyCheese --- crypto/block/block-parse.h | 2 +- crypto/common/bigint.hpp | 61 ++++++++++--------- crypto/common/bitstring.cpp | 10 +-- crypto/tl/tlblib.cpp | 7 ++- crypto/vm/cells/CellBuilder.cpp | 2 +- crypto/vm/tonops.cpp | 2 +- tonlib/tonlib/TonlibClient.cpp | 53 ++++++++-------- .../validator-session-description.cpp | 20 ++++-- 8 files changed, 86 insertions(+), 71 deletions(-) diff --git a/crypto/block/block-parse.h b/crypto/block/block-parse.h index 1c490383..3e2ff81a 100644 --- a/crypto/block/block-parse.h +++ b/crypto/block/block-parse.h @@ -982,7 +982,7 @@ struct ShardIdent::Record { int shard_pfx_bits; int workchain_id; unsigned long long shard_prefix; - Record() : shard_pfx_bits(-1) { + Record() : shard_pfx_bits(-1), workchain_id(ton::workchainInvalid), shard_prefix(0) { } Record(int _pfxlen, int _wcid, unsigned long long _pfx) : shard_pfx_bits(_pfxlen), workchain_id(_wcid), shard_prefix(_pfx) { diff --git a/crypto/common/bigint.hpp b/crypto/common/bigint.hpp index 94da3a13..5fa8e2da 100644 --- a/crypto/common/bigint.hpp +++ b/crypto/common/bigint.hpp @@ -176,6 +176,7 @@ class AnyIntView { public: enum { word_bits = Tr::word_bits, word_shift = Tr::word_shift }; typedef typename Tr::word_t word_t; + typedef typename Tr::uword_t uword_t; int& n_; PropagateConstSpan digits; @@ -320,7 +321,7 @@ class BigIntG { digits[0] = x; normalize_bool(); } else { - digits[0] = ((x + Tr::Half) & (Tr::Base - 1)) - Tr::Half; + digits[0] = ((x ^ Tr::Half) & (Tr::Base - 1)) - Tr::Half; digits[n++] = (x >> Tr::word_shift) + (digits[0] < 0); } } @@ -675,7 +676,7 @@ class BigIntG { return n > 0 && !(digits[0] & 1); } word_t mod_pow2_short(int pow) const { - return n > 0 ? digits[0] & ((1 << pow) - 1) : 0; + return n > 0 ? digits[0] & ((1ULL << pow) - 1) : 0; } private: @@ -764,7 +765,7 @@ bool AnyIntView::add_pow2_any(int exponent, int factor) { while (size() <= k) { digits[inc_size()] = 0; } - digits[k] += ((word_t)factor << dm.rem); + digits[k] += factor * ((word_t)1 << dm.rem); return true; } @@ -969,7 +970,7 @@ bool AnyIntView::add_mul_any(const AnyIntView& yp, const AnyIntView& if (hi && hi != -1) { return invalidate_bool(); } - digits[size() - 1] += (hi << word_shift); + digits[size() - 1] += ((uword_t)hi << word_shift); } return true; } @@ -1014,7 +1015,7 @@ int AnyIntView::sgn_un_any() const { } int i = size() - 2; do { - v <<= word_shift; + v *= Tr::Base; word_t w = digits[i]; if (w >= -v + Tr::MaxDenorm) { return 1; @@ -1059,7 +1060,7 @@ typename Tr::word_t AnyIntView::to_long_any() const { } else if (size() == 1) { return digits[0]; } else { - word_t v = digits[0] + (digits[1] << word_shift); // approximation mod 2^64 + word_t v = (uword_t)digits[0] + ((uword_t)digits[1] << word_shift); // approximation mod 2^64 word_t w = (v & (Tr::Base - 1)) - digits[0]; w >>= word_shift; w += (v >> word_shift); // excess of approximation divided by Tr::Base @@ -1120,7 +1121,7 @@ int AnyIntView::cmp_un_any(const AnyIntView& yp) const { return -1; } while (xn > yn) { - v <<= word_shift; + v *= Tr::Base; word_t w = T::eval(digits[--xn]); if (w >= -v + Tr::MaxDenorm) { return 1; @@ -1137,7 +1138,7 @@ int AnyIntView::cmp_un_any(const AnyIntView& yp) const { return -1; } while (yn > xn) { - v <<= word_shift; + v *= Tr::Base; word_t w = yp.digits[--yn]; if (w <= v - Tr::MaxDenorm) { return 1; @@ -1150,7 +1151,7 @@ int AnyIntView::cmp_un_any(const AnyIntView& yp) const { v = 0; } while (--xn >= 0) { - v <<= word_shift; + v *= Tr::Base; word_t w = T::eval(digits[xn]) - yp.digits[xn]; if (w >= -v + Tr::MaxDenorm) { return 1; @@ -1197,7 +1198,7 @@ int AnyIntView::divmod_tiny_any(int y) { } int rem = 0; for (int i = size() - 1; i >= 0; i--) { - auto divmod = std::div(digits[i] + ((word_t)rem << word_shift), (word_t)y); + auto divmod = std::div(digits[i] + ((uword_t)rem << word_shift), (word_t)y); digits[i] = divmod.quot; rem = (int)divmod.rem; if ((rem ^ y) < 0 && rem) { @@ -1267,7 +1268,7 @@ bool AnyIntView::mul_add_short_any(word_t y, word_t z) { z += (digits[size() - 1] >> word_shift); digits[size() - 1] &= Tr::Base - 1; if (!z || z == -1) { - digits[size() - 1] += (z << word_shift); + digits[size() - 1] += ((uword_t)z << word_shift); return true; } else { return false; @@ -1338,7 +1339,7 @@ bool AnyIntView::mod_div_any(const AnyIntView& yp, AnyIntView& quot, while (--i >= 0) { Tr::sub_mul(&digits[k + i + 1], &digits[k + i], q, yp.digits[i]); } - digits[size() - 1] += (hi << word_shift); + digits[size() - 1] += ((uword_t)hi << word_shift); } } else { quot.set_size(1); @@ -1351,7 +1352,7 @@ bool AnyIntView::mod_div_any(const AnyIntView& yp, AnyIntView& quot, Tr::sub_mul(&digits[k + i + 1], &digits[k + i], q, yp.digits[i]); } dec_size(); - digits[size() - 1] += (digits[size()] << word_shift); + digits[size() - 1] += ((uword_t)digits[size()] << word_shift); } if (size() >= yp.size() - 1) { assert(size() <= yp.size()); @@ -1455,7 +1456,7 @@ bool AnyIntView::mod_pow2_any(int exponent) { dec_size(); q += word_shift; } - word_t pow = ((word_t)1 << q); + uword_t pow = ((uword_t)1 << q); word_t v = digits[size() - 1] & (pow - 1); if (!v) { int k = size() - 1; @@ -1485,7 +1486,7 @@ bool AnyIntView::mod_pow2_any(int exponent) { return true; } else if (v >= Tr::Half && size() < max_size()) { word_t w = (((v >> (word_shift - 1)) + 1) >> 1); - digits[size() - 1] = v - (w << word_shift); + digits[size() - 1] = (uword_t)v - ((uword_t)w << word_shift); digits[inc_size()] = w; return true; } else { @@ -1623,7 +1624,7 @@ bool AnyIntView::lshift_any(int exponent) { } else if (v != -1) { return invalidate_bool(); } else { - digits[size() - 1] += (v << word_shift); + digits[size() - 1] += ((uword_t)v << word_shift); } } if (q) { @@ -1750,7 +1751,7 @@ int AnyIntView::bit_size_any(bool sgnd) const { int k = size() - 1; word_t q = digits[k]; if (k > 0 && q < Tr::MaxDenorm / 2) { - q <<= word_shift; + q *= Tr::Base; q += digits[--k]; } if (!k) { @@ -1766,7 +1767,7 @@ int AnyIntView::bit_size_any(bool sgnd) const { } else if (q <= -Tr::MaxDenorm / 2) { return s; } - q <<= word_shift; + q *= Tr::Base; q += digits[--k]; } return q >= 0 ? s + 1 : s; @@ -1774,7 +1775,7 @@ int AnyIntView::bit_size_any(bool sgnd) const { int k = size() - 1; word_t q = digits[k]; if (k > 0 && q > -Tr::MaxDenorm / 2) { - q <<= word_shift; + q *= Tr::Base; q += digits[--k]; } if (!k) { @@ -1790,7 +1791,7 @@ int AnyIntView::bit_size_any(bool sgnd) const { } else if (q <= -Tr::MaxDenorm / 2) { return s + 1; } - q <<= word_shift; + q *= Tr::Base; q += digits[--k]; } return q >= 0 ? s : s + 1; @@ -1817,7 +1818,7 @@ bool AnyIntView::export_bytes_any(unsigned char* buff, std::size_t buff_size for (int i = 0; i < size(); i++) { if ((word_shift & 7) && word_shift + 8 >= word_bits && k >= word_bits - word_shift - 1) { int k1 = 8 - k; - v += (digits[i] << k) & 0xff; + v += ((uword_t)digits[i] << k) & 0xff; if (ptr > buff) { *--ptr = (unsigned char)(v & 0xff); } else if ((unsigned char)(v & 0xff) != s) { @@ -1827,7 +1828,7 @@ bool AnyIntView::export_bytes_any(unsigned char* buff, std::size_t buff_size v += (digits[i] >> k1); k += word_shift - 8; } else { - v += (digits[i] << k); + v += ((uword_t)digits[i] << k); k += word_shift; } while (k >= 8) { @@ -1868,7 +1869,7 @@ bool AnyIntView::export_bytes_lsb_any(unsigned char* buff, std::size_t buff_ for (int i = 0; i < size(); i++) { if ((word_shift & 7) && word_shift + 8 >= word_bits && k >= word_bits - word_shift - 1) { int k1 = 8 - k; - v += (digits[i] << k) & 0xff; + v += ((uword_t)digits[i] << k) & 0xff; if (buff < end) { *buff++ = (unsigned char)(v & 0xff); } else if ((unsigned char)(v & 0xff) != s) { @@ -1878,7 +1879,7 @@ bool AnyIntView::export_bytes_lsb_any(unsigned char* buff, std::size_t buff_ v += (digits[i] >> k1); k += word_shift - 8; } else { - v += (digits[i] << k); + v += ((uword_t)digits[i] << k); k += word_shift; } while (k >= 8) { @@ -1922,7 +1923,7 @@ bool AnyIntView::export_bits_any(unsigned char* buff, int offs, unsigned bit return false; } } - td::bitstring::bits_store_long_top(buff, offs, v << (64 - bits), bits); + td::bitstring::bits_store_long_top(buff, offs, (unsigned long long)v << (64 - bits), bits); } else { if (!sgnd && v < 0) { return false; @@ -1945,7 +1946,7 @@ bool AnyIntView::export_bits_any(unsigned char* buff, int offs, unsigned bit for (int i = 0; i < size(); i++) { if (word_shift + 8 >= word_bits && k >= word_bits - word_shift - 1) { int k1 = 8 - k; - v += (digits[i] << k) & 0xff; + v += ((uword_t)digits[i] << k) & 0xff; if (ptr > buff) { if (--ptr > buff) { *ptr = (unsigned char)(v & 0xff); @@ -1963,7 +1964,7 @@ bool AnyIntView::export_bits_any(unsigned char* buff, int offs, unsigned bit v += (digits[i] >> k1); k += word_shift - 8; } else { - v += (digits[i] << k); + v += ((uword_t)digits[i] << k); k += word_shift; } while (k >= 8) { @@ -2028,7 +2029,7 @@ bool AnyIntView::import_bytes_any(const unsigned char* buff, std::size_t buf return invalidate_bool(); } } - v |= (((word_t) * --ptr) << k); + v |= (((uword_t) * --ptr) << k); k += 8; } if (s) { @@ -2043,7 +2044,9 @@ bool AnyIntView::import_bits_any(const unsigned char* buff, int offs, unsign if (bits < word_shift) { set_size(1); unsigned long long val = td::bitstring::bits_load_long_top(buff, offs, bits); - if (sgnd) { + if (bits == 0) { + digits[0] = 0; + } else if (sgnd) { digits[0] = ((long long)val >> (64 - bits)); } else { digits[0] = (val >> (64 - bits)); diff --git a/crypto/common/bitstring.cpp b/crypto/common/bitstring.cpp index 0a273949..aabc6984 100644 --- a/crypto/common/bitstring.cpp +++ b/crypto/common/bitstring.cpp @@ -191,7 +191,7 @@ void bits_memcpy(unsigned char* to, int to_offs, const unsigned char* from, int *to++ = (unsigned char)(acc >> b); } if (b > 0) { - *to = (unsigned char)((*to & (0xff >> b)) | ((int)acc << (8 - b))); + *to = (unsigned char)((*to & (0xff >> b)) | ((unsigned)acc << (8 - b))); } } } @@ -301,7 +301,7 @@ std::size_t bits_memscan(const unsigned char* ptr, int offs, std::size_t bit_cou ptr++; } while (rem >= 8 && !td::is_aligned_pointer<8>(ptr)) { - v = ((*ptr++ ^ xor_val) << 24); + v = ((unsigned)(*ptr++ ^ xor_val) << 24); // std::cerr << "[B] rem=" << rem << " ptr=" << (const void*)(ptr - 1) << " v=" << std::hex << v << std::dec << std::endl; if (v) { return bit_count - rem + td::count_leading_zeroes_non_zero32(v); @@ -319,7 +319,7 @@ std::size_t bits_memscan(const unsigned char* ptr, int offs, std::size_t bit_cou rem -= 64; } while (rem >= 8) { - v = ((*ptr++ ^ xor_val) << 24); + v = ((unsigned)(*ptr++ ^ xor_val) << 24); // std::cerr << "[D] rem=" << rem << " ptr=" << (const void*)(ptr - 1) << " v=" << std::hex << v << std::dec << std::endl; if (v) { return bit_count - rem + td::count_leading_zeroes_non_zero32(v); @@ -327,7 +327,7 @@ std::size_t bits_memscan(const unsigned char* ptr, int offs, std::size_t bit_cou rem -= 8; } if (rem > 0) { - v = ((*ptr ^ xor_val) << 24); + v = ((unsigned)(*ptr ^ xor_val) << 24); // std::cerr << "[E] rem=" << rem << " ptr=" << (const void*)ptr << " v=" << std::hex << v << std::dec << std::endl; c = td::count_leading_zeroes32(v); return c < rem ? bit_count - rem + c : bit_count; @@ -505,7 +505,7 @@ unsigned long long bits_load_long_top(ConstBitPtr from, unsigned top_bits) { } unsigned long long bits_load_ulong(ConstBitPtr from, unsigned bits) { - return bits_load_long_top(from, bits) >> (64 - bits); + return bits == 0 ? 0 : bits_load_long_top(from, bits) >> (64 - bits); } long long bits_load_long(ConstBitPtr from, unsigned bits) { diff --git a/crypto/tl/tlblib.cpp b/crypto/tl/tlblib.cpp index d0c1b538..0e0e5626 100644 --- a/crypto/tl/tlblib.cpp +++ b/crypto/tl/tlblib.cpp @@ -125,8 +125,11 @@ bool TupleT::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { } bool TLB::validate_ref_internal(int* ops, Ref cell_ref, bool weak) const { - if (ops && --*ops < 0) { - return false; + if (ops) { + if (*ops <= 0) { + return false; + } + --*ops; } bool is_special; auto cs = load_cell_slice_special(std::move(cell_ref), is_special); diff --git a/crypto/vm/cells/CellBuilder.cpp b/crypto/vm/cells/CellBuilder.cpp index 481c2581..43058531 100644 --- a/crypto/vm/cells/CellBuilder.cpp +++ b/crypto/vm/cells/CellBuilder.cpp @@ -335,7 +335,7 @@ bool CellBuilder::store_ulong_rchk_bool(unsigned long long val, unsigned val_bit } CellBuilder& CellBuilder::store_long(long long val, unsigned val_bits) { - return store_long_top(val << (64 - val_bits), val_bits); + return store_long_top(val_bits == 0 ? 0 : (unsigned long long)val << (64 - val_bits), val_bits); } CellBuilder& CellBuilder::store_long_top(unsigned long long val, unsigned top_bits) { diff --git a/crypto/vm/tonops.cpp b/crypto/vm/tonops.cpp index 6a4cceaa..a164df83 100644 --- a/crypto/vm/tonops.cpp +++ b/crypto/vm/tonops.cpp @@ -474,7 +474,7 @@ int exec_store_var_integer(VmState* st, int len_bits, bool sgnd, bool quiet) { stack.check_underflow(2); auto x = stack.pop_int(); auto cbr = stack.pop_builder(); - unsigned len = ((x->bit_size(sgnd) + 7) >> 3); + unsigned len = (((unsigned)x->bit_size(sgnd) + 7) >> 3); if (len >= (1u << len_bits)) { throw VmError{Excno::range_chk}; } diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index 14fd4100..0bb49763 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -4451,35 +4451,36 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_getMasterchainInfo& } td::Status TonlibClient::do_request(const tonlib_api::blocks_getShards& request, - td::Promise>&& promise) { + td::Promise>&& promise) { TRY_RESULT(block, to_lite_api(*request.id_)) client_.send_query(ton::lite_api::liteServer_getAllShardsInfo(std::move(block)), - promise.wrap([](lite_api_ptr&& all_shards_info) { - td::BufferSlice proof = std::move((*all_shards_info).proof_); - td::BufferSlice data = std::move((*all_shards_info).data_); - if (data.empty()) { - //return td::Status::Error("shard configuration is empty"); - } else { - auto R = vm::std_boc_deserialize(data.clone()); - if (R.is_error()) { - //return td::Status::Error("cannot deserialize shard configuration"); - } - auto root = R.move_as_ok(); - block::ShardConfig sh_conf; - if (!sh_conf.unpack(vm::load_cell_slice_ref(root))) { - //return td::Status::Error("cannot extract shard block list from shard configuration"); - } else { - auto ids = sh_conf.get_shard_hash_ids(true); - tonlib_api::blocks_shards shards; - for (auto id : ids) { - auto ref = sh_conf.get_shard_hash(ton::ShardIdFull(id)); - if (ref.not_null()) { - shards.shards_.push_back(to_tonlib_api(ref->top_block_id())); - } - } + promise.wrap([](lite_api_ptr&& all_shards_info) + -> td::Result> { + td::BufferSlice proof = std::move((*all_shards_info).proof_); + td::BufferSlice data = std::move((*all_shards_info).data_); + if (data.empty()) { + return td::Status::Error("shard configuration is empty"); + } else { + auto R = vm::std_boc_deserialize(data.clone()); + if (R.is_error()) { + return R.move_as_error_prefix("cannot deserialize shard configuration: "); + } + auto root = R.move_as_ok(); + block::ShardConfig sh_conf; + if (!sh_conf.unpack(vm::load_cell_slice_ref(root))) { + return td::Status::Error("cannot extract shard block list from shard configuration"); + } else { + auto ids = sh_conf.get_shard_hash_ids(true); + tonlib_api::blocks_shards shards; + for (auto id : ids) { + auto ref = sh_conf.get_shard_hash(ton::ShardIdFull(id)); + if (ref.not_null()) { + shards.shards_.push_back(to_tonlib_api(ref->top_block_id())); + } + } return tonlib_api::make_object(std::move(shards)); - } - } + } + } })); return td::Status::OK(); } diff --git a/validator-session/validator-session-description.cpp b/validator-session/validator-session-description.cpp index 42059053..eb4c3a8e 100644 --- a/validator-session/validator-session-description.cpp +++ b/validator-session/validator-session-description.cpp @@ -162,20 +162,28 @@ void ValidatorSessionDescriptionImpl::update_hash(const RootObject *obj, HashTyp } void *ValidatorSessionDescriptionImpl::alloc(size_t size, size_t align, bool temp) { + CHECK(align && !(align & (align - 1))); // align should be a power of 2 + auto get_padding = [&](const uint8_t* ptr) { + return (-(size_t)ptr) & (align - 1); + }; if (temp) { + pdata_temp_ptr_ += get_padding(pdata_temp_ + pdata_temp_ptr_); auto s = pdata_temp_ptr_; pdata_temp_ptr_ += size; CHECK(s + size <= pdata_temp_size_); return static_cast(pdata_temp_ + s); } else { while (true) { - auto s = pdata_perm_ptr_; - pdata_perm_ptr_ += size; - - if (pdata_perm_ptr_ <= pdata_perm_.size() * pdata_perm_size_) { - return static_cast(pdata_perm_[s / pdata_perm_size_] + (s % pdata_perm_size_)); + size_t idx = pdata_perm_ptr_ / pdata_perm_size_; + if (idx < pdata_perm_.size()) { + auto ptr = pdata_perm_[idx] + (pdata_perm_ptr_ % pdata_perm_size_); + pdata_perm_ptr_ += get_padding(ptr); + ptr += get_padding(ptr); + pdata_perm_ptr_ += size; + if (pdata_perm_ptr_ <= pdata_perm_.size() * pdata_perm_size_) { + return static_cast(ptr); + } } - pdata_perm_.push_back(new td::uint8[pdata_perm_size_]); } } From 2512f0287b32fe616ff82a2c02e42008b0c23e18 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 20 Sep 2022 10:49:28 +0300 Subject: [PATCH 16/34] Tonlib method getShardBlockProof (#465) * Tonlib method getShardBlockProof * Clarify limit of result.size() in getShardBlockProof --- tl/generate/scheme/lite_api.tl | 3 + tl/generate/scheme/lite_api.tlo | Bin 12896 -> 13396 bytes tl/generate/scheme/tonlib_api.tl | 4 + tl/generate/scheme/tonlib_api.tlo | Bin 29880 -> 30836 bytes tonlib/tonlib/TonlibClient.cpp | 192 ++++++++++++++++++++++++++++++ tonlib/tonlib/TonlibClient.h | 2 + validator/impl/liteserver.cpp | 133 +++++++++++++++++++++ validator/impl/liteserver.hpp | 3 + 8 files changed, 337 insertions(+) diff --git a/tl/generate/scheme/lite_api.tl b/tl/generate/scheme/lite_api.tl index d31ef569..b2a72dc9 100644 --- a/tl/generate/scheme/lite_api.tl +++ b/tl/generate/scheme/lite_api.tl @@ -52,6 +52,8 @@ liteServer.partialBlockProof complete:Bool from:tonNode.blockIdExt to:tonNode.bl liteServer.configInfo mode:# id:tonNode.blockIdExt state_proof:bytes config_proof:bytes = liteServer.ConfigInfo; liteServer.validatorStats mode:# id:tonNode.blockIdExt count:int complete:Bool state_proof:bytes data_proof:bytes = liteServer.ValidatorStats; liteServer.libraryResult result:(vector liteServer.libraryEntry) = liteServer.LibraryResult; +liteServer.shardBlockLink id:tonNode.blockIdExt proof:bytes = liteServer.ShardBlockLink; +liteServer.shardBlockProof masterchain_id:tonNode.blockIdExt links:(vector liteServer.shardBlockLink) = liteServer.ShardBlockProof; liteServer.debug.verbosity value:int = liteServer.debug.Verbosity; @@ -78,6 +80,7 @@ liteServer.getConfigAll mode:# id:tonNode.blockIdExt = liteServer.ConfigInfo; liteServer.getConfigParams mode:# id:tonNode.blockIdExt param_list:(vector int) = liteServer.ConfigInfo; liteServer.getValidatorStats#091a58bc mode:# id:tonNode.blockIdExt limit:int start_after:mode.0?int256 modified_after:mode.2?int = liteServer.ValidatorStats; liteServer.getLibraries library_list:(vector int256) = liteServer.LibraryResult; +liteServer.getShardBlockProof id:tonNode.blockIdExt = liteServer.ShardBlockProof; liteServer.queryPrefix = Object; liteServer.query data:bytes = Object; diff --git a/tl/generate/scheme/lite_api.tlo b/tl/generate/scheme/lite_api.tlo index eb8e321137b650cc7badcf44cee0f92b7db944b6..a015d93265e1565c6ade590701b540a76a604c53 100644 GIT binary patch delta 322 zcmaEmawUWJ(QJJy1}HGt$orjDq3AsCWyze(lGNbTqO#N?z2J<*q7FZQDVR37a2C>u@qkZst|(XJrjwUgje^d7+lD0NCFUd%zY= P7Ese+0jr;&b%7lKxq@q| delta 51 zcmcbT@gRlw(QJJy1}MkhI2k22&sFPZ-8@0(2s;3i CTMu9W diff --git a/tl/generate/scheme/tonlib_api.tl b/tl/generate/scheme/tonlib_api.tl index 99bbc927..79185d5b 100644 --- a/tl/generate/scheme/tonlib_api.tl +++ b/tl/generate/scheme/tonlib_api.tl @@ -219,6 +219,9 @@ blocks.header id:ton.blockIdExt global_id:int32 version:int32 flags:# after_merg blocks.signature node_id_short:int256 signature:bytes = blocks.Signature; blocks.blockSignatures id:ton.blockIdExt signatures:(vector blocks.signature) = blocks.BlockSignatures; +blocks.shardBlockLink id:ton.blockIdExt proof:bytes = blocks.ShardBlockLink; +blocks.blockLinkBack to_key_block:Bool from:ton.blockIdExt to:ton.blockIdExt dest_proof:bytes proof:bytes state_proof:bytes = blocks.BlockLinkBack; +blocks.shardBlockProof from:ton.blockIdExt mc_id:ton.blockIdExt links:(vector blocks.shardBlockLink) mc_proof:(vector blocks.blockLinkBack) = blocks.ShardBlockProof; configInfo config:tvm.cell = ConfigInfo; @@ -311,6 +314,7 @@ blocks.lookupBlock mode:int32 id:ton.blockId lt:int64 utime:int32 = ton.BlockIdE blocks.getTransactions id:ton.blockIdExt mode:# count:# after:blocks.accountTransactionId = blocks.Transactions; blocks.getBlockHeader id:ton.blockIdExt = blocks.Header; blocks.getMasterchainBlockSignatures seqno:int32 = blocks.BlockSignatures; +blocks.getShardBlockProof id:ton.blockIdExt mode:# from:mode.0?ton.blockIdExt = blocks.ShardBlockProof; onLiteServerQueryResult id:int64 bytes:bytes = Ok; onLiteServerQueryError id:int64 error:error = Ok; diff --git a/tl/generate/scheme/tonlib_api.tlo b/tl/generate/scheme/tonlib_api.tlo index 0cc2a6d9bf9172aec03adf92c5df338b2df218db..ee457609da33cca7655480b0531417fd19ccf958 100644 GIT binary patch delta 539 zcmdn-lJUz2M&3uW^{p77AYmh~FuSB_Lyx0KQciwycCnrli1NwI%XUgk&YoPzBt2Py zeTr13=!0dVP!+)$iA5=JWs~igq-28Qul*LokPaxy&rf5RSRk=^3;Pvz#?H;1{NghD z$d-Y%z$^ng2_$#(n2-+}kS+b}z9vse{$zPQRd$xNqWoM2hRNr(C7GFkV&}D`xKdJ! zOX3SaR!sgcAUSzSssaypaYiib3`rLNCO3AUTkWS)eXPaw$7&ZgPBP3fR@GtU16So!l!RFA1`Z z5o$chj~qZ%5N|y6l$QXx7$#ff%%;O(+jbCS$L2TsF3OCan+=1fGpanF_f}F88reKIIgfWwry4w5SQ8Pp&P-nnZwvZnbBjj NW7u@Y%`Fih parent, + td::Promise>&& promise) + : id_(id), from_(from), parent_(std::move(parent)), promise_(std::move(promise)) { + client_.set_client(ext_client_ref); + } + + void start_up() override { + if (from_.is_masterchain_ext()) { + got_from_block(from_); + } else { + client_.with_last_block([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &GetShardBlockProof::abort, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &GetShardBlockProof::got_from_block, R.move_as_ok().last_block_id); + } + }); + } + } + + void got_from_block(ton::BlockIdExt from) { + from_ = from; + CHECK(from_.is_masterchain_ext()); + client_.send_query( + ton::lite_api::liteServer_getShardBlockProof(ton::create_tl_lite_block_id(id_)), + [SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &GetShardBlockProof::abort, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &GetShardBlockProof::got_shard_block_proof, R.move_as_ok()); + } + }); + } + + void got_shard_block_proof(lite_api_ptr result) { + mc_id_ = create_block_id(std::move(result->masterchain_id_)); + if (!mc_id_.is_masterchain_ext()) { + abort(td::Status::Error("got invalid masterchain block id")); + return; + } + if (result->links_.size() > 8) { + abort(td::Status::Error("chain is too long")); + return; + } + ton::BlockIdExt cur_id = mc_id_; + try { + for (auto& link : result->links_) { + ton::BlockIdExt prev_id = create_block_id(link->id_); + td::BufferSlice proof = std::move(link->proof_); + auto R = vm::std_boc_deserialize(proof); + if (R.is_error()) { + abort(TonlibError::InvalidBagOfCells("proof")); + return; + } + auto block_root = vm::MerkleProof::virtualize(R.move_as_ok(), 1); + if (cur_id.root_hash != block_root->get_hash().bits()) { + abort(td::Status::Error("invalid block hash in proof")); + return; + } + if (cur_id.is_masterchain()) { + block::gen::Block::Record blk; + block::gen::BlockExtra::Record extra; + block::gen::McBlockExtra::Record mc_extra; + if (!tlb::unpack_cell(block_root, blk) || !tlb::unpack_cell(blk.extra, extra) || !extra.custom->have_refs() || + !tlb::unpack_cell(extra.custom->prefetch_ref(), mc_extra)) { + abort(td::Status::Error("cannot unpack block header")); + return; + } + block::ShardConfig shards(mc_extra.shard_hashes->prefetch_ref()); + td::Ref shard_hash = shards.get_shard_hash(prev_id.shard_full(), true); + if (shard_hash.is_null() || shard_hash->top_block_id() != prev_id) { + abort(td::Status::Error("invalid proof chain: prev block is not in mc shard list")); + return; + } + } else { + std::vector prev; + ton::BlockIdExt mc_blkid; + bool after_split; + td::Status S = block::unpack_block_prev_blk_try(block_root, cur_id, prev, mc_blkid, after_split); + if (S.is_error()) { + abort(std::move(S)); + return; + } + CHECK(prev.size() == 1 || prev.size() == 2); + bool found = prev_id == prev[0] || (prev.size() == 2 && prev_id == prev[1]); + if (!found) { + abort(td::Status::Error("invalid proof chain: prev block is not in prev blocks list")); + return; + } + } + links_.emplace_back(prev_id, std::move(proof)); + cur_id = prev_id; + } + } catch (vm::VmVirtError& err) { + abort(err.as_status()); + return; + } + if (cur_id != id_) { + abort(td::Status::Error("got invalid proof chain")); + return; + } + + if (mc_id_.seqno() > from_.seqno()) { + abort(td::Status::Error("from mc block is too old")); + return; + } + + client_.send_query( + ton::lite_api::liteServer_getBlockProof(0x1001, ton::create_tl_lite_block_id(from_), + ton::create_tl_lite_block_id(mc_id_)), + [SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &GetShardBlockProof::abort, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &GetShardBlockProof::got_mc_proof, R.move_as_ok()); + } + }); + } + + void got_mc_proof(lite_api_ptr result) { + auto R = liteclient::deserialize_proof_chain(std::move(result)); + if (R.is_error()) { + abort(R.move_as_error()); + return; + } + auto chain = R.move_as_ok(); + if (chain->from != from_ || chain->to != mc_id_ || !chain->complete || chain->link_count() > 1) { + abort(td::Status::Error("got invalid proof chain")); + return; + } + auto S = chain->validate(); + if (S.is_error()) { + abort(std::move(S)); + return; + } + + std::vector> links; + std::vector> mc_proof; + for (const auto& p : links_) { + links.push_back( + ton::create_tl_object(to_tonlib_api(p.first), p.second.as_slice().str())); + } + if (chain->link_count() == 1) { + auto& link = chain->last_link(); + td::BufferSlice dest_proof = vm::std_boc_serialize(link.dest_proof).move_as_ok(); + td::BufferSlice proof = vm::std_boc_serialize(link.proof).move_as_ok(); + td::BufferSlice state_proof = vm::std_boc_serialize(link.state_proof).move_as_ok(); + mc_proof.push_back(ton::create_tl_object( + link.is_key, to_tonlib_api(link.from), to_tonlib_api(link.to), dest_proof.as_slice().str(), + proof.as_slice().str(), state_proof.as_slice().str())); + } + + promise_.set_result(ton::create_tl_object( + to_tonlib_api(from_), to_tonlib_api(mc_id_), std::move(links), std::move(mc_proof))); + stop(); + } + + void abort(td::Status error) { + promise_.set_error(std::move(error)); + stop(); + } + + private: + ton::BlockIdExt id_, from_, mc_id_; + td::actor::ActorShared<> parent_; + td::Promise> promise_; + ExtClient client_; + std::vector> links_; +}; + TonlibClient::TonlibClient(td::unique_ptr callback) : callback_(std::move(callback)) { } TonlibClient::~TonlibClient() = default; @@ -4413,6 +4586,12 @@ auto to_lite_api(const tonlib_api::ton_blockIdExt& blk) -> td::Result to_block_id(const tonlib_api::ton_blockIdExt& blk) { + TRY_RESULT(root_hash, to_bits256(blk.root_hash_, "blk.root_hash")) + TRY_RESULT(file_hash, to_bits256(blk.file_hash_, "blk.file_hash")) + return ton::BlockIdExt(blk.workchain_, blk.shard_, blk.seqno_, root_hash, file_hash); +} + td::Status TonlibClient::do_request(const tonlib_api::getConfigParam& request, td::Promise>&& promise) { TRY_RESULT(lite_block, to_lite_api(*request.id_)) @@ -4624,6 +4803,19 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_getMasterchainBlock return td::Status::OK(); } +td::Status TonlibClient::do_request(const tonlib_api::blocks_getShardBlockProof& request, + td::Promise>&& promise) { + TRY_RESULT(id, to_block_id(*request.id_)); + ton::BlockIdExt from; + if (request.mode_ & 1) { + TRY_RESULT_ASSIGN(from, to_block_id(*request.id_)); + } + auto actor_id = actor_id_++; + actors_[actor_id] = td::actor::create_actor("GetShardBlockProof", client_.get_client(), id, from, + actor_shared(this, actor_id), std::move(promise)); + return td::Status::OK(); +} + void TonlibClient::load_libs_from_disk() { LOG(DEBUG) << "loading libraries from disk cache"; auto r_data = kv_->get("tonlib.libcache"); diff --git a/tonlib/tonlib/TonlibClient.h b/tonlib/tonlib/TonlibClient.h index bbb9b900..800d8c80 100644 --- a/tonlib/tonlib/TonlibClient.h +++ b/tonlib/tonlib/TonlibClient.h @@ -366,6 +366,8 @@ class TonlibClient : public td::actor::Actor { td::Promise>&& promise); td::Status do_request(const tonlib_api::blocks_getMasterchainBlockSignatures& request, td::Promise>&& promise); + td::Status do_request(const tonlib_api::blocks_getShardBlockProof& request, + td::Promise>&& promise); td::Status do_request(const tonlib_api::getConfigParam& request, td::Promise>&& promise); diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index 255742f9..fadc596b 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -201,6 +201,9 @@ void LiteQuery::start_up() { [&](lite_api::liteServer_getLibraries& q) { this->perform_getLibraries(q.library_list_); }, + [&](lite_api::liteServer_getShardBlockProof& q) { + this->perform_getShardBlockProof(create_block_id(q.id_)); + }, [&](auto& obj) { this->abort_query(td::Status::Error(ErrorCode::protoviolation, "unknown query")); })); } @@ -2432,5 +2435,135 @@ void LiteQuery::continue_getValidatorStats(int mode, int limit, Bits256 start_af finish_query(std::move(b)); } +void LiteQuery::perform_getShardBlockProof(BlockIdExt blkid) { + LOG(INFO) << "started a getMasterchainInfo(" << blkid.to_str() << ") liteserver query"; + if (!blkid.is_valid_ext()) { + fatal_error("invalid block id"); + return; + } + if (blkid.is_masterchain()) { + LOG(INFO) << "getShardBlockProof() query completed"; + auto b = create_serialize_tl_object( + create_tl_lite_block_id(blkid), std::vector>()); + finish_query(std::move(b)); + return; + } + blk_id_ = blkid; + get_block_handle_checked(blkid, [manager = manager_, Self = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); + return; + } + ConstBlockHandle handle = R.move_as_ok(); + if (!handle->inited_masterchain_ref_block()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, td::Status::Error("block doesn't have masterchain ref")); + return; + } + AccountIdPrefixFull pfx{masterchainId, shardIdAll}; + td::actor::send_closure_later( + manager, &ValidatorManager::get_block_by_seqno_from_db, pfx, handle->masterchain_ref_block(), + [Self, manager](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); + } else { + ConstBlockHandle handle = R.move_as_ok(); + td::actor::send_closure_later( + manager, &ValidatorManager::get_block_data_from_db, handle, [Self](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); + } else { + td::actor::send_closure_later(Self, &LiteQuery::continue_getShardBlockProof, R.move_as_ok(), + std::vector>()); + } + }); + } + }); + }); +} + +void LiteQuery::continue_getShardBlockProof(Ref cur_block, + std::vector> result) { + BlockIdExt cur_id = cur_block->block_id(); + BlockIdExt prev_id; + vm::MerkleProofBuilder mpb{cur_block->root_cell()}; + if (cur_id.is_masterchain()) { + base_blk_id_ = cur_id; + block::gen::Block::Record blk; + block::gen::BlockExtra::Record extra; + block::gen::McBlockExtra::Record mc_extra; + if (!tlb::unpack_cell(mpb.root(), blk) || !tlb::unpack_cell(blk.extra, extra) || !extra.custom->have_refs() || + !tlb::unpack_cell(extra.custom->prefetch_ref(), mc_extra)) { + fatal_error("cannot unpack header of block "s + cur_id.to_str()); + return; + } + block::ShardConfig shards(mc_extra.shard_hashes->prefetch_ref()); + ShardIdFull shard_id = blk_id_.shard_full(); + shard_id.shard = (shard_id.shard & ~(1 << (63 - shard_id.pfx_len()))) | 1; + Ref shard_hash = shards.get_shard_hash(shard_id, false); + if (shard_hash.is_null()) { + fatal_error("shard not found"); + return; + } + prev_id = shard_hash->top_block_id(); + } else { + std::vector prev; + BlockIdExt mc_blkid; + bool after_split; + td::Status S = block::unpack_block_prev_blk_try(mpb.root(), cur_id, prev, mc_blkid, after_split); + if (S.is_error()) { + fatal_error(std::move(S)); + return; + } + bool found = false; + for (const BlockIdExt& id : prev) { + if (shard_intersects(id.shard_full(), blk_id_.shard_full())) { + found = true; + prev_id = id; + break; + } + } + if (!found) { + fatal_error("failed to find block chain"); + return; + } + } + auto proof = mpb.extract_proof_boc(); + if (proof.is_error()) { + fatal_error(proof.move_as_error_prefix("cannot serialize Merkle proof : ")); + return; + } + result.emplace_back(prev_id, proof.move_as_ok()); + + if (prev_id == blk_id_) { + CHECK(base_blk_id_.is_masterchain()); + std::vector> links; + for (auto& p : result) { + links.push_back( + create_tl_object(create_tl_lite_block_id(p.first), std::move(p.second))); + } + LOG(INFO) << "getShardBlockProof() query completed"; + auto b = create_serialize_tl_object(create_tl_lite_block_id(base_blk_id_), + std::move(links)); + finish_query(std::move(b)); + return; + } + if (result.size() == 8) { + // Chains of shardblocks between masterchain blocks can't be longer than 8 (see collator.cpp:991) + fatal_error("proof chain is too long"); + return; + } + + td::actor::send_closure_later( + manager_, &ValidatorManager::get_block_data_from_db_short, prev_id, + [Self = actor_id(this), result = std::move(result)](td::Result> R) mutable { + if (R.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); + } else { + td::actor::send_closure_later(Self, &LiteQuery::continue_getShardBlockProof, R.move_as_ok(), + std::move(result)); + } + }); +} + } // namespace validator } // namespace ton diff --git a/validator/impl/liteserver.hpp b/validator/impl/liteserver.hpp index 2f2e8cba..47970aae 100644 --- a/validator/impl/liteserver.hpp +++ b/validator/impl/liteserver.hpp @@ -145,6 +145,9 @@ class LiteQuery : public td::actor::Actor { bool construct_proof_link_back_cont(ton::BlockIdExt cur, ton::BlockIdExt next); bool adjust_last_proof_link(ton::BlockIdExt cur, Ref block_root); bool finish_proof_chain(ton::BlockIdExt id); + void perform_getShardBlockProof(BlockIdExt blkid); + void continue_getShardBlockProof(Ref cur_block, + std::vector> result); void load_prevKeyBlock(ton::BlockIdExt blkid, td::Promise>>); void continue_loadPrevKeyBlock(ton::BlockIdExt blkid, td::Result, BlockIdExt>> res, From 3b1d33f54305c97301b3daa9ff6c39849490c86d Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Tue, 20 Sep 2022 14:16:53 +0300 Subject: [PATCH 17/34] Allow constants with the same name and value (#462) Co-authored-by: legaii --- crypto/func/parse-func.cpp | 21 +++++++++++++++------ crypto/func/test/co2.fc | 15 +++++++++++++++ 2 files changed, 30 insertions(+), 6 deletions(-) create mode 100644 crypto/func/test/co2.fc diff --git a/crypto/func/parse-func.cpp b/crypto/func/parse-func.cpp index 439dfb88..6118a433 100644 --- a/crypto/func/parse-func.cpp +++ b/crypto/func/parse-func.cpp @@ -254,9 +254,7 @@ void parse_const_decl(Lexer& lex) { if (!sym_def) { lex.cur().error_at("cannot define global symbol `", "`"); } - if (sym_def->value) { - lex.cur().error_at("global symbol `", "` already exists"); - } + Lexem ident = lex.cur(); lex.next(); if (lex.tp() != '=') { lex.cur().error_at("expected = instead of ", ""); @@ -273,10 +271,11 @@ void parse_const_decl(Lexer& lex) { if ((wanted_type != Expr::_None) && (x->cls != wanted_type)) { lex.cur().error("expression type does not match wanted type"); } + SymValConst* new_value = nullptr; if (x->cls == Expr::_Const) { // Integer constant - sym_def->value = new SymValConst{const_cnt++, x->intval}; + new_value = new SymValConst{const_cnt++, x->intval}; } else if (x->cls == Expr::_SliceConst) { // Slice constant (string) - sym_def->value = new SymValConst{const_cnt++, x->strval}; + new_value = new SymValConst{const_cnt++, x->strval}; } else if (x->cls == Expr::_Apply) { code.emplace_back(loc, Op::_Import, std::vector()); auto tmp_vars = x->pre_compile(code); @@ -304,10 +303,20 @@ void parse_const_decl(Lexer& lex) { if (op.origin.is_null() || !op.origin->is_valid()) { lex.cur().error("precompiled expression did not result in a valid integer constant"); } - sym_def->value = new SymValConst{const_cnt++, op.origin}; + new_value = new SymValConst{const_cnt++, op.origin}; } else { lex.cur().error("integer or slice literal or constant expected"); } + if (sym_def->value) { + SymValConst* old_value = dynamic_cast(sym_def->value); + Keyword new_type = new_value->get_type(); + if (!old_value || old_value->get_type() != new_type || + (new_type == _Int && *old_value->get_int_value() != *new_value->get_int_value()) || + (new_type == _Slice && old_value->get_str_value() != new_value->get_str_value())) { + ident.error_at("global symbol `", "` already exists"); + } + } + sym_def->value = new_value; } FormalArgList parse_formal_args(Lexer& lex) { diff --git a/crypto/func/test/co2.fc b/crypto/func/test/co2.fc new file mode 100644 index 00000000..f5fcb748 --- /dev/null +++ b/crypto/func/test/co2.fc @@ -0,0 +1,15 @@ +const int x = 5; +const slice s = "abacaba"; +const int y = 3; +const slice s = "abacaba"; +const int x = 5; +const int z = 4, z = 4; + +int sdeq (slice s1, slice s2) asm "SDEQ"; + +() main() { + throw_unless(101, x == 5); + throw_unless(102, y == 3); + throw_unless(103, z == 4); + throw_unless(104, sdeq(s, "abacaba")); +} From 3c380e0a7b94a1404f1e7902a919c68dc144d8a0 Mon Sep 17 00:00:00 2001 From: Starlight Duck Date: Tue, 20 Sep 2022 14:19:42 +0300 Subject: [PATCH 18/34] Implement compilation and pre-computation of logical operations (#437) * Implement compilation and pre-computation of logical operations (and, or, xor, not) * Fix emulate_and optimization * Fix variable flags in emulate_not * Rename co2.fc to co3.fc Co-authored-by: EmelyanenkoK --- crypto/func/builtins.cpp | 133 ++++++++++++++++++++++++++++++++++++--- crypto/func/test/co3.fc | 24 +++++++ 2 files changed, 150 insertions(+), 7 deletions(-) create mode 100644 crypto/func/test/co3.fc diff --git a/crypto/func/builtins.cpp b/crypto/func/builtins.cpp index 9a3c27e5..756de2f8 100644 --- a/crypto/func/builtins.cpp +++ b/crypto/func/builtins.cpp @@ -173,6 +173,74 @@ int emulate_mul(int a, int b) { return r; } +int emulate_and(int a, int b) { + int both = a & b, any = a | b; + int r = VarDescr::_Int; + if (any & VarDescr::_Nan) { + return r | VarDescr::_Nan; + } + r |= VarDescr::_Finite; + if (any & VarDescr::_Zero) { + return VarDescr::ConstZero; + } + r |= both & (VarDescr::_Even | VarDescr::_Odd); + r |= both & (VarDescr::_Bit | VarDescr::_Bool); + if (both & VarDescr::_Odd) { + r |= VarDescr::_NonZero; + } + return r; +} + +int emulate_or(int a, int b) { + if (b & VarDescr::_Zero) { + return a; + } else if (a & VarDescr::_Zero) { + return b; + } + int both = a & b, any = a | b; + int r = VarDescr::_Int; + if (any & VarDescr::_Nan) { + return r | VarDescr::_Nan; + } + r |= VarDescr::_Finite; + r |= any & VarDescr::_NonZero; + r |= any & VarDescr::_Odd; + r |= both & VarDescr::_Even; + return r; +} + +int emulate_xor(int a, int b) { + if (b & VarDescr::_Zero) { + return a; + } else if (a & VarDescr::_Zero) { + return b; + } + int both = a & b, any = a | b; + int r = VarDescr::_Int; + if (any & VarDescr::_Nan) { + return r | VarDescr::_Nan; + } + r |= VarDescr::_Finite; + r |= both & VarDescr::_Even; + if (both & VarDescr::_Odd) { + r |= VarDescr::_Even; + } + return r; +} + +int emulate_not(int a) { + int f = VarDescr::_Even | VarDescr::_Odd; + if ((a & f) && (~a & f)) { + a ^= f; + } + f = VarDescr::_Pos | VarDescr::_Neg; + if ((a & f) && (~a & f)) { + a ^= f; + } + a &= ~(VarDescr::_Zero | VarDescr::_NonZero | VarDescr::_Bit); + return a; +} + int emulate_lshift(int a, int b) { if (((a | b) & VarDescr::_Nan) || !(~b & (VarDescr::_Neg | VarDescr::_NonZero))) { return VarDescr::_Int | VarDescr::_Nan; @@ -427,6 +495,57 @@ AsmOp compile_negate(std::vector& res, std::vector& args) { return exec_op("NEGATE", 1); } +AsmOp compile_and(std::vector& res, std::vector& args) { + assert(res.size() == 1 && args.size() == 2); + VarDescr &r = res[0], &x = args[0], &y = args[1]; + if (x.is_int_const() && y.is_int_const()) { + r.set_const(x.int_const & y.int_const); + x.unused(); + y.unused(); + return push_const(r.int_const); + } + r.val = emulate_and(x.val, y.val); + return exec_op("AND", 2); +} + +AsmOp compile_or(std::vector& res, std::vector& args) { + assert(res.size() == 1 && args.size() == 2); + VarDescr &r = res[0], &x = args[0], &y = args[1]; + if (x.is_int_const() && y.is_int_const()) { + r.set_const(x.int_const | y.int_const); + x.unused(); + y.unused(); + return push_const(r.int_const); + } + r.val = emulate_or(x.val, y.val); + return exec_op("OR", 2); +} + +AsmOp compile_xor(std::vector& res, std::vector& args) { + assert(res.size() == 1 && args.size() == 2); + VarDescr &r = res[0], &x = args[0], &y = args[1]; + if (x.is_int_const() && y.is_int_const()) { + r.set_const(x.int_const ^ y.int_const); + x.unused(); + y.unused(); + return push_const(r.int_const); + } + r.val = emulate_xor(x.val, y.val); + return exec_op("XOR", 2); +} + +AsmOp compile_not(std::vector& res, std::vector& args) { + assert(res.size() == 1 && args.size() == 1); + VarDescr &r = res[0], &x = args[0]; + if (x.is_int_const()) { + r.set_const(~x.int_const); + x.unused(); + return push_const(r.int_const); + } + r.val = emulate_not(x.val); + return exec_op("NOT", 1); +} + AsmOp compile_mul_internal(VarDescr& r, VarDescr& x, VarDescr& y) { if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const * y.int_const); @@ -1000,10 +1119,10 @@ void define_builtins() { define_builtin_func("_>>_", arith_bin_op, std::bind(compile_rshift, _1, _2, -1)); define_builtin_func("_~>>_", arith_bin_op, std::bind(compile_rshift, _1, _2, 0)); define_builtin_func("_^>>_", arith_bin_op, std::bind(compile_rshift, _1, _2, 1)); - define_builtin_func("_&_", arith_bin_op, AsmOp::Custom("AND", 2)); - define_builtin_func("_|_", arith_bin_op, AsmOp::Custom("OR", 2)); - define_builtin_func("_^_", arith_bin_op, AsmOp::Custom("XOR", 2)); - define_builtin_func("~_", arith_un_op, AsmOp::Custom("NOT", 1)); + define_builtin_func("_&_", arith_bin_op, compile_and); + define_builtin_func("_|_", arith_bin_op, compile_or); + define_builtin_func("_^_", arith_bin_op, compile_xor); + define_builtin_func("~_", arith_un_op, compile_not); define_builtin_func("^_+=_", arith_bin_op, compile_add); define_builtin_func("^_-=_", arith_bin_op, compile_sub); define_builtin_func("^_*=_", arith_bin_op, compile_mul); @@ -1017,9 +1136,9 @@ void define_builtins() { define_builtin_func("^_>>=_", arith_bin_op, std::bind(compile_rshift, _1, _2, -1)); define_builtin_func("^_~>>=_", arith_bin_op, std::bind(compile_rshift, _1, _2, 0)); define_builtin_func("^_^>>=_", arith_bin_op, std::bind(compile_rshift, _1, _2, 1)); - define_builtin_func("^_&=_", arith_bin_op, AsmOp::Custom("AND", 2)); - define_builtin_func("^_|=_", arith_bin_op, AsmOp::Custom("OR", 2)); - define_builtin_func("^_^=_", arith_bin_op, AsmOp::Custom("XOR", 2)); + define_builtin_func("^_&=_", arith_bin_op, compile_and); + define_builtin_func("^_|=_", arith_bin_op, compile_or); + define_builtin_func("^_^=_", arith_bin_op, compile_xor); define_builtin_func("muldiv", TypeExpr::new_map(Int3, Int), std::bind(compile_muldiv, _1, _2, -1)); define_builtin_func("muldivr", TypeExpr::new_map(Int3, Int), std::bind(compile_muldiv, _1, _2, 0)); define_builtin_func("muldivc", TypeExpr::new_map(Int3, Int), std::bind(compile_muldiv, _1, _2, 1)); diff --git a/crypto/func/test/co3.fc b/crypto/func/test/co3.fc new file mode 100644 index 00000000..398592d3 --- /dev/null +++ b/crypto/func/test/co3.fc @@ -0,0 +1,24 @@ +const val1 = 123456789; +const val2 = 987654321; +const val3 = 135792468; +const val4 = 246813579; + +const prec_and = val1 & val2; +const prec_or = val1 | val2; +const prec_xor = val1 ^ val2; +const prec_logic = ((val1 & val2) | val3) ^ val4; +const prec_nand = val1 & (~ val2); + +int get_and() { return prec_and; } +int get_or() { return prec_or; } +int get_xor() { return prec_xor; } +int get_logic() { return prec_logic; } +int get_nand() { return prec_nand; } + +_ main() { + throw_unless(101, get_and() == 39471121); + throw_unless(102, get_or() == 1071639989); + throw_unless(103, get_xor() == 1032168868); + throw_unless(104, get_logic() == 82599134); + throw_unless(105, get_nand() == 83985668); +} From d11580dfb3b81ea5d00775502737d59c155adfb2 Mon Sep 17 00:00:00 2001 From: Trinketer22 <109865562+Trinketer22@users.noreply.github.com> Date: Tue, 20 Sep 2022 22:26:59 +0300 Subject: [PATCH 20/34] Compiler -march fix for not supported arch (#444) I think there is an issue with the way -march flag is handled in the current build system. It is set to native (heavy cpu specific optimizations) via TON_ARCH and it is never checked if it is supported by the compiler. That's what is causing all those issues with apple m1 builds or any other arm system. Without checking early, build will only fail at link stage. Strict arch support checking is due. 1. Check if "-march=${TON_ARCH}" flag is supported by the compiler Adds it to CXX_FLAGS if al good 2. Otherwise if such flag is not supported but TON_ARCH is set to default we continue execution without setting march and leaving arch decisions to cmake build system 3. Otherwise user specified TON_ARCH intentionally and current arch is not supported by the compiler. Therefore we terminate build process immediately and let user know what's the matter. Co-authored-by: Trinketer22 --- CMakeLists.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 44dbae8b..b3daab9c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -197,10 +197,13 @@ find_package(Threads REQUIRED) find_package(ZLIB REQUIRED) if (TON_ARCH AND NOT MSVC) + CHECK_CXX_COMPILER_FLAG( "-march=${TON_ARCH}" COMPILER_OPT_ARCH_SUPPORTED ) if (TON_ARCH STREQUAL "apple-m1") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcpu=${TON_ARCH}") - else() + elseif(COMPILER_OPT_ARCH_SUPPORTED) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=${TON_ARCH}") + elseif(NOT TON_ARCH STREQUAL "native") + message(FATAL_ERROR "Compiler doesn't support arch ${TON_ARCH}") endif() endif() if (THREADS_HAVE_PTHREAD_ARG) From bb5bc6178cfd57bf291c8ea055fce8054ef3c9eb Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 21 Sep 2022 11:04:40 +0300 Subject: [PATCH 21/34] Change recursion to loop in CellSlice --- crypto/block/transaction.cpp | 4 +- crypto/vm/cells/CellSlice.cpp | 71 ++++++++++++++++--------------- validator/impl/validate-query.cpp | 4 -- 3 files changed, 38 insertions(+), 41 deletions(-) diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index f5fb18f1..3b2334d6 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -327,8 +327,7 @@ bool Account::unpack(Ref shard_account, Ref extra, block::gen::t_ShardAccount.print(std::cerr, *shard_account); } block::gen::ShardAccount::Record acc_info; - if (!(block::gen::t_ShardAccount.validate_csr(shard_account) && - block::tlb::t_ShardAccount.validate_csr(shard_account) && tlb::unpack_exact(shard_account.write(), acc_info))) { + if (!(block::tlb::t_ShardAccount.validate_csr(shard_account) && tlb::unpack_exact(shard_account.write(), acc_info))) { LOG(ERROR) << "account " << addr.to_hex() << " state is invalid"; return false; } @@ -2013,7 +2012,6 @@ bool Transaction::compute_state() { std::cerr << "new account state: "; block::gen::t_Account.print_ref(std::cerr, new_total_state); } - CHECK(block::gen::t_Account.validate_ref(new_total_state)); CHECK(block::tlb::t_Account.validate_ref(new_total_state)); return true; } diff --git a/crypto/vm/cells/CellSlice.cpp b/crypto/vm/cells/CellSlice.cpp index 90b78558..e1df5759 100644 --- a/crypto/vm/cells/CellSlice.cpp +++ b/crypto/vm/cells/CellSlice.cpp @@ -1055,44 +1055,47 @@ std::ostream& operator<<(std::ostream& os, Ref cs_ref) { // If can_be_special is not null, then it is allowed to load special cell // Flag whether loaded cell is actually special will be stored into can_be_special -VirtualCell::LoadedCell load_cell_slice_impl(const Ref& cell, bool* can_be_special) { - auto* vm_state_interface = VmStateInterface::get(); - if (vm_state_interface) { - vm_state_interface->register_cell_load(cell->get_hash()); - } - auto r_loaded_cell = cell->load_cell(); - if (r_loaded_cell.is_error()) { - throw VmError{Excno::cell_und, "failed to load cell"}; - } - auto loaded_cell = r_loaded_cell.move_as_ok(); - if (loaded_cell.data_cell->special_type() == DataCell::SpecialType::PrunnedBranch) { - auto virtualization = loaded_cell.virt.get_virtualization(); - if (virtualization != 0) { - throw VmVirtError{virtualization}; +VirtualCell::LoadedCell load_cell_slice_impl(Ref cell, bool* can_be_special) { + while (true) { + auto* vm_state_interface = VmStateInterface::get(); + if (vm_state_interface) { + vm_state_interface->register_cell_load(cell->get_hash()); } - } - if (can_be_special) { - *can_be_special = loaded_cell.data_cell->is_special(); - } else if (loaded_cell.data_cell->is_special()) { - if (loaded_cell.data_cell->special_type() == DataCell::SpecialType::Library) { - if (vm_state_interface) { - CellSlice cs(std::move(loaded_cell)); - DCHECK(cs.size() == Cell::hash_bits + 8); - auto library_cell = vm_state_interface->load_library(cs.data_bits() + 8); - if (library_cell.not_null()) { - //TODO: fix infinity loop - return load_cell_slice_impl(library_cell, nullptr); - } - throw VmError{Excno::cell_und, "failed to load library cell"}; + auto r_loaded_cell = cell->load_cell(); + if (r_loaded_cell.is_error()) { + throw VmError{Excno::cell_und, "failed to load cell"}; + } + auto loaded_cell = r_loaded_cell.move_as_ok(); + if (loaded_cell.data_cell->special_type() == DataCell::SpecialType::PrunnedBranch) { + auto virtualization = loaded_cell.virt.get_virtualization(); + if (virtualization != 0) { + throw VmVirtError{virtualization}; } - throw VmError{Excno::cell_und, "failed to load library cell (no vm_state_interface available)"}; - } else if (loaded_cell.data_cell->special_type() == DataCell::SpecialType::PrunnedBranch) { - CHECK(loaded_cell.virt.get_virtualization() == 0); - throw VmError{Excno::cell_und, "trying to load prunned cell"}; } - throw VmError{Excno::cell_und, "unexpected special cell"}; + if (can_be_special) { + *can_be_special = loaded_cell.data_cell->is_special(); + } else if (loaded_cell.data_cell->is_special()) { + if (loaded_cell.data_cell->special_type() == DataCell::SpecialType::Library) { + if (vm_state_interface) { + CellSlice cs(std::move(loaded_cell)); + DCHECK(cs.size() == Cell::hash_bits + 8); + auto library_cell = vm_state_interface->load_library(cs.data_bits() + 8); + if (library_cell.not_null()) { + cell = library_cell; + can_be_special = nullptr; + continue; + } + throw VmError{Excno::cell_und, "failed to load library cell"}; + } + throw VmError{Excno::cell_und, "failed to load library cell (no vm_state_interface available)"}; + } else if (loaded_cell.data_cell->special_type() == DataCell::SpecialType::PrunnedBranch) { + CHECK(loaded_cell.virt.get_virtualization() == 0); + throw VmError{Excno::cell_und, "trying to load prunned cell"}; + } + throw VmError{Excno::cell_und, "unexpected special cell"}; + } + return loaded_cell; } - return loaded_cell; } CellSlice load_cell_slice(const Ref& cell) { diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 61a98bb9..0649473d 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -2313,10 +2313,6 @@ bool ValidateQuery::precheck_one_account_update(td::ConstBitPtr acc_id, Ref Date: Thu, 22 Sep 2022 16:54:26 +0300 Subject: [PATCH 22/34] Add triple quotes asms (#463) * Add python-like triple quotes for multiline strings * Add test for multiline asm * Allow asm definition duplicate * Asm duplicate: add test & fixes * Fix multiline asm * Fix asm duplicate Co-authored-by: legaii --- crypto/func/func.h | 1 + crypto/func/parse-func.cpp | 53 +++++++++++++++++++++++++++++++------- crypto/func/test/s2.fc | 26 +++++++++++++++++++ crypto/parser/lexer.cpp | 49 +++++++++++++++++++++++++++++++++-- crypto/parser/lexer.h | 5 +++- crypto/tl/tlbc.cpp | 2 +- 6 files changed, 123 insertions(+), 13 deletions(-) create mode 100644 crypto/func/test/s2.fc diff --git a/crypto/func/func.h b/crypto/func/func.h index 2ea1015e..c86f7e6b 100644 --- a/crypto/func/func.h +++ b/crypto/func/func.h @@ -1631,6 +1631,7 @@ inline compile_func_t make_ext_compile(AsmOp op) { struct SymValAsmFunc : SymValFunc { simple_compile_func_t simple_compile; compile_func_t ext_compile; + td::uint64 crc; ~SymValAsmFunc() override = default; SymValAsmFunc(TypeExpr* ft, const AsmOp& _macro, bool impure = false) : SymValFunc(-1, ft, impure), simple_compile(make_simple_compile(_macro)) { diff --git a/crypto/func/parse-func.cpp b/crypto/func/parse-func.cpp index 6118a433..92dfe47b 100644 --- a/crypto/func/parse-func.cpp +++ b/crypto/func/parse-func.cpp @@ -1270,19 +1270,48 @@ SymValAsmFunc* parse_asm_func_body(Lexer& lex, TypeExpr* func_type, const Formal lex.expect(')'); } while (lex.tp() == _String) { - asm_ops.push_back(AsmOp::Parse(lex.cur().str, cnt, width)); - lex.next(); - if (asm_ops.back().is_custom()) { - cnt = width; + std::string ops = lex.cur().str; // \n\n... + std::string op; + for (const char& c : ops) { + if (c == '\n') { + if (!op.empty()) { + asm_ops.push_back(AsmOp::Parse(op, cnt, width)); + if (asm_ops.back().is_custom()) { + cnt = width; + } + op.clear(); + } + } else { + op.push_back(c); + } } + if (!op.empty()) { + asm_ops.push_back(AsmOp::Parse(op, cnt, width)); + if (asm_ops.back().is_custom()) { + cnt = width; + } + } + lex.next(); } if (asm_ops.empty()) { throw src::ParseError{lex.cur().loc, "string with assembler instruction expected"}; } lex.expect(';'); + std::string crc_s; + for (const AsmOp& asm_op : asm_ops) { + crc_s += asm_op.op; + } + crc_s.push_back(impure); + for (const int& x : arg_order) { + crc_s += std::string((const char*) (&x), (const char*) (&x + 1)); + } + for (const int& x : ret_order) { + crc_s += std::string((const char*) (&x), (const char*) (&x + 1)); + } auto res = new SymValAsmFunc{func_type, asm_ops, impure}; res->arg_order = std::move(arg_order); res->ret_order = std::move(ret_order); + res->crc = td::crc64(crc_s); return res; } @@ -1448,16 +1477,22 @@ void parse_func_def(Lexer& lex) { // code->print(std::cerr); // !!!DEBUG!!! func_sym_code->code = code; } else { + Lexem asm_lexem = lex.cur(); + SymValAsmFunc* asm_func = parse_asm_func_body(lex, func_type, arg_list, ret_type, impure); if (func_sym_val) { if (dynamic_cast(func_sym_val)) { - lex.cur().error("function `"s + func_name.str + "` was already declared as an ordinary function"); + asm_lexem.error("function `"s + func_name.str + "` was already declared as an ordinary function"); } - if (dynamic_cast(func_sym_val)) { - lex.cur().error("redefinition of built-in assembler function `"s + func_name.str + "`"); + SymValAsmFunc* asm_func_old = dynamic_cast(func_sym_val); + if (asm_func_old) { + if (asm_func->crc != asm_func_old->crc) { + asm_lexem.error("redefinition of built-in assembler function `"s + func_name.str + "`"); + } + } else { + asm_lexem.error("redefinition of previously (somehow) defined function `"s + func_name.str + "`"); } - lex.cur().error("redefinition of previously (somehow) defined function `"s + func_name.str + "`"); } - func_sym->value = parse_asm_func_body(lex, func_type, arg_list, ret_type, impure); + func_sym->value = asm_func; } if (method_id.not_null()) { auto val = dynamic_cast(func_sym->value); diff --git a/crypto/func/test/s2.fc b/crypto/func/test/s2.fc new file mode 100644 index 00000000..c6df49d5 --- /dev/null +++ b/crypto/func/test/s2.fc @@ -0,0 +1,26 @@ +slice test1() asm """ + "Test" $>s + PUSHSLICE +"""; + +slice test2() asm """ + "Hello" + " " + "World" + $+ $+ $>s + PUSHSLICE +"""; + +int sdeq (slice s1, slice s2) asm """SDEQ"""; +int sdeq (slice s1, slice s2) asm "SDEQ" ""; +int sdeq (slice s1, slice s2) asm "" """ +SDEQ +"""; + +() main() { + slice s = test1(); + throw_unless(101, sdeq(s, "Test")); + + slice s = test2(); + throw_unless(102, sdeq(s, "Hello World")); +} diff --git a/crypto/parser/lexer.cpp b/crypto/parser/lexer.cpp index 5c5b77d8..624d8dd2 100644 --- a/crypto/parser/lexer.cpp +++ b/crypto/parser/lexer.cpp @@ -125,8 +125,9 @@ int Lexem::set(std::string _str, const SrcLocation& _loc, int _tp, int _val) { } Lexer::Lexer(SourceReader& _src, bool init, std::string active_chars, std::string eol_cmts, std::string open_cmts, - std::string close_cmts, std::string quote_chars) - : src(_src), eof(false), lexem("", src.here(), Lexem::Undefined), peek_lexem("", {}, Lexem::Undefined) { + std::string close_cmts, std::string quote_chars, std::string multiline_quote) + : src(_src), eof(false), lexem("", src.here(), Lexem::Undefined), peek_lexem("", {}, Lexem::Undefined), + multiline_quote(std::move(multiline_quote)) { std::memset(char_class, 0, sizeof(char_class)); unsigned char activity = cc::active; for (char c : active_chars) { @@ -171,6 +172,19 @@ void Lexer::set_spec(std::array& arr, std::string setup) { } } +bool Lexer::is_multiline_quote(const char* begin, const char* end) { + if (multiline_quote.empty()) { + return false; + } + for (const char& c : multiline_quote) { + if (begin == end || *begin != c) { + return false; + } + ++begin; + } + return true; +} + void Lexer::expect(int exp_tp, const char* msg) { if (tp() != exp_tp) { throw ParseError{lexem.loc, (msg ? std::string{msg} : Lexem::lexem_name_str(exp_tp)) + " expected instead of " + @@ -234,6 +248,37 @@ const Lexem& Lexer::next() { } return lexem.clear(src.here(), Lexem::Eof); } + if (is_multiline_quote(src.get_ptr(), src.get_end_ptr())) { + src.advance(multiline_quote.size()); + const char* begin = src.get_ptr(); + const char* end = nullptr; + SrcLocation here = src.here(); + std::string body; + while (!src.is_eof()) { + if (src.is_eoln()) { + body.push_back('\n'); + src.load_line(); + continue; + } + if (is_multiline_quote(src.get_ptr(), src.get_end_ptr())) { + end = src.get_ptr(); + src.advance(multiline_quote.size()); + break; + } + body.push_back(src.cur_char()); + src.advance(1); + } + if (!end) { + src.error("string extends past end of file"); + } + lexem.set(body, here, Lexem::String); + int c = src.cur_char(); + if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) { + lexem.val = c; + src.advance(1); + } + return lexem; + } int c = src.cur_char(); const char* end = src.get_ptr(); if (is_quote_char(c) || c == '`') { diff --git a/crypto/parser/lexer.h b/crypto/parser/lexer.h index 12556e46..686d8eac 100644 --- a/crypto/parser/lexer.h +++ b/crypto/parser/lexer.h @@ -71,6 +71,7 @@ class Lexer { Lexem lexem, peek_lexem; unsigned char char_class[128]; std::array eol_cmt, cmt_op, cmt_cl; + std::string multiline_quote; enum cc { left_active = 2, right_active = 1, active = 3, allow_repeat = 4, quote_char = 8 }; public: @@ -78,7 +79,8 @@ class Lexer { return eof; } Lexer(SourceReader& _src, bool init = false, std::string active_chars = ";,() ~.", std::string eol_cmts = ";;", - std::string open_cmts = "{-", std::string close_cmts = "-}", std::string quote_chars = "\""); + std::string open_cmts = "{-", std::string close_cmts = "-}", std::string quote_chars = "\"", + std::string multiline_quote = "\"\"\""); const Lexem& next(); const Lexem& cur() const { return lexem; @@ -109,6 +111,7 @@ class Lexer { private: void set_spec(std::array& arr, std::string setup); + bool is_multiline_quote(const char* begin, const char* end); }; } // namespace src diff --git a/crypto/tl/tlbc.cpp b/crypto/tl/tlbc.cpp index 127eacb0..01b8a31a 100644 --- a/crypto/tl/tlbc.cpp +++ b/crypto/tl/tlbc.cpp @@ -2423,7 +2423,7 @@ std::vector source_fdescr; bool parse_source(std::istream* is, src::FileDescr* fdescr) { src::SourceReader reader{is, fdescr}; - src::Lexer lex{reader, true, "(){}:;? #$. ^~ #", "//", "/*", "*/"}; + src::Lexer lex{reader, true, "(){}:;? #$. ^~ #", "//", "/*", "*/", ""}; while (lex.tp() != src::_Eof) { parse_constructor_def(lex); // std::cerr << lex.cur().str << '\t' << lex.cur().name_str() << std::endl; From 6a72aba9af79481dfc2626f615455bab9cba232a Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Thu, 22 Sep 2022 16:55:14 +0300 Subject: [PATCH 23/34] Bump func version --- crypto/func/func.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/func/func.h b/crypto/func/func.h index c86f7e6b..5db99725 100644 --- a/crypto/func/func.h +++ b/crypto/func/func.h @@ -39,7 +39,7 @@ extern std::string generated_from; constexpr int optimize_depth = 20; -const std::string func_version{"0.2.0"}; +const std::string func_version{"0.3.0"}; enum Keyword { _Eof = -1, From 5c2ad4a6c7318500877450816c6f43d89eb84fa4 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 19 May 2022 18:32:07 +0300 Subject: [PATCH 24/34] Tests for func with scripts --- crypto/func/auto-tests/run_tests.py | 91 +++++++ crypto/func/auto-tests/stress_tester.py | 239 ++++++++++++++++++ crypto/func/auto-tests/tests/a10.fc | 17 ++ crypto/func/auto-tests/tests/a6.fc | 89 +++++++ crypto/func/auto-tests/tests/a6_1.fc | 16 ++ crypto/func/auto-tests/tests/a6_5.fc | 24 ++ crypto/func/auto-tests/tests/a7.fc | 24 ++ crypto/func/auto-tests/tests/c2.fc | 17 ++ crypto/func/auto-tests/tests/c2_1.fc | 14 + crypto/func/auto-tests/tests/co1.fc | 60 +++++ .../auto-tests/tests/code_after_ifelse.fc | 19 ++ crypto/func/auto-tests/tests/inline_big.fc | 61 +++++ crypto/func/auto-tests/tests/inline_if.fc | 26 ++ crypto/func/auto-tests/tests/inline_loops.fc | 43 ++++ crypto/func/auto-tests/tests/method_id.fc | 12 + crypto/func/auto-tests/tests/s1.fc | 54 ++++ .../func/auto-tests/tests/unbalanced_ret.fc | 17 ++ .../auto-tests/tests/unbalanced_ret_inline.fc | 18 ++ .../auto-tests/tests/unbalanced_ret_loops.fc | 48 ++++ .../auto-tests/tests/unbalanced_ret_nested.fc | 35 +++ crypto/func/auto-tests/tests/w1.fc | 14 + crypto/func/auto-tests/tests/w2.fc | 18 ++ crypto/func/auto-tests/tests/w6.fc | 17 ++ crypto/func/auto-tests/tests/w7.fc | 24 ++ crypto/func/auto-tests/tests/w9.fc | 14 + 25 files changed, 1011 insertions(+) create mode 100644 crypto/func/auto-tests/run_tests.py create mode 100644 crypto/func/auto-tests/stress_tester.py create mode 100644 crypto/func/auto-tests/tests/a10.fc create mode 100644 crypto/func/auto-tests/tests/a6.fc create mode 100644 crypto/func/auto-tests/tests/a6_1.fc create mode 100644 crypto/func/auto-tests/tests/a6_5.fc create mode 100644 crypto/func/auto-tests/tests/a7.fc create mode 100644 crypto/func/auto-tests/tests/c2.fc create mode 100644 crypto/func/auto-tests/tests/c2_1.fc create mode 100644 crypto/func/auto-tests/tests/co1.fc create mode 100644 crypto/func/auto-tests/tests/code_after_ifelse.fc create mode 100644 crypto/func/auto-tests/tests/inline_big.fc create mode 100644 crypto/func/auto-tests/tests/inline_if.fc create mode 100644 crypto/func/auto-tests/tests/inline_loops.fc create mode 100644 crypto/func/auto-tests/tests/method_id.fc create mode 100644 crypto/func/auto-tests/tests/s1.fc create mode 100644 crypto/func/auto-tests/tests/unbalanced_ret.fc create mode 100644 crypto/func/auto-tests/tests/unbalanced_ret_inline.fc create mode 100644 crypto/func/auto-tests/tests/unbalanced_ret_loops.fc create mode 100644 crypto/func/auto-tests/tests/unbalanced_ret_nested.fc create mode 100644 crypto/func/auto-tests/tests/w1.fc create mode 100644 crypto/func/auto-tests/tests/w2.fc create mode 100644 crypto/func/auto-tests/tests/w6.fc create mode 100644 crypto/func/auto-tests/tests/w7.fc create mode 100644 crypto/func/auto-tests/tests/w9.fc diff --git a/crypto/func/auto-tests/run_tests.py b/crypto/func/auto-tests/run_tests.py new file mode 100644 index 00000000..ae9c990c --- /dev/null +++ b/crypto/func/auto-tests/run_tests.py @@ -0,0 +1,91 @@ +import os +import os.path +import subprocess +import sys +import tempfile + +def getenv(name, default=None): + if name in os.environ: + return os.environ[name] + if default is None: + print("Environment variable", name, "is not set", file=sys.stderr) + exit(1) + return default + +FUNC_EXECUTABLE = getenv("FUNC_EXECUTABLE", "func") +FIFT_EXECUTABLE = getenv("FIFT_EXECUTABLE", "fift") +#FUNC_STDLIB = getenv("FUNC_STDLIB") +FIFT_LIBS = getenv("FIFT_LIBS") +TMP_DIR = tempfile.mkdtemp() +COMPILED_FIF = os.path.join(TMP_DIR, "compiled.fif") +RUNNER_FIF = os.path.join(TMP_DIR, "runner.fif") + +if len(sys.argv) != 2: + print("Usage : run_tests.py tests_dir", file=sys.stderr) + exit(1) +TESTS_DIR = sys.argv[1] + +class ExecutionError(Exception): + pass + +def compile_func(f): + res = subprocess.run([FUNC_EXECUTABLE, "-o", COMPILED_FIF, "-SPA", f], capture_output=True, timeout=10) + if res.returncode != 0: + raise ExecutionError(str(res.stderr, "utf-8")) + +def run_runner(): + res = subprocess.run([FIFT_EXECUTABLE, "-I", FIFT_LIBS, RUNNER_FIF], capture_output=True, timeout=10) + if res.returncode != 0: + raise ExecutionError(str(res.stderr, "utf-8")) + s = str(res.stdout, "utf-8") + s = [x.strip() for x in s.split("\n")] + return [x for x in s if x != ""] + +tests = [s for s in os.listdir(TESTS_DIR) if s.endswith(".fc")] +tests.sort() +print("Found", len(tests), "tests", file=sys.stderr) +for ti, tf in enumerate(tests): + print("Running test %d/%d: %s" % (ti + 1, len(tests), tf), file=sys.stderr) + tf = os.path.join(TESTS_DIR, tf) + try: + compile_func(tf) + except ExecutionError as e: + print(file=sys.stderr) + print("Compilation error", file=sys.stderr) + print(e, file=sys.stderr) + exit(2) + with open(tf, "r") as fd: + lines = fd.readlines() + cases = [] + for s in lines: + s = [x.strip() for x in s.split("|")] + if len(s) == 4 and s[0].strip() == "TESTCASE": + cases.append(s[1:]) + if len(cases) == 0: + print(file=sys.stderr) + print("Error: no test cases", file=sys.stderr) + exit(2) + + with open(RUNNER_FIF, "w") as f: + print("\"%s\" include = %d);" % (var, self.n), file=f) + +def write_function(f, name, body, inline=False, inline_ref=False, method_id=None): + print("_ %s(int x)" % name, file=f, end="") + if inline: + print(" inline", file=f, end="") + if inline_ref: + print(" inline_ref", file=f, end="") + if method_id is not None: + print(" method_id(%d)" % method_id, file=f, end="") + print(" {", file=f) + for i in range(VAR_CNT): + print(" int v%d = 0;" % i, file=f) + body.write(f, 1); + print("}", file=f) + +def gen_code(xl, xr, with_return, loop_depth=0): + code = [] + for _ in range(random.randint(0, 2)): + if random.randint(0, 3) == 0 and loop_depth < 3: + c = gen_code(xl, xr, False, loop_depth + 1) + code.append(CodeRepeat(random.randint(0, 3), c, random.randint(0, 2))) + elif xr - xl > 1: + xmid = random.randrange(xl + 1, xr) + ret = random.choice((0, 0, 0, 0, 0, 1, 2)) + c1 = gen_code(xl, xmid, ret == 1, loop_depth) + if random.randrange(5) == 0: + c2 = CodeEmpty() + else: + c2 = gen_code(xmid, xr, ret == 2, loop_depth) + code.append(CodeIfRange(xl, xmid, c1, c2)) + if with_return: + if xr - xl == 1: + code.append(CodeReturn(random.randrange(10**9))) + else: + xmid = random.randrange(xl + 1, xr) + c1 = gen_code(xl, xmid, True, loop_depth) + c2 = gen_code(xmid, xr, True, loop_depth) + code.append(CodeIfRange(xl, xmid, c1, c2)) + for _ in range(random.randint(0, 3)): + pos = random.randint(0, len(code)) + code.insert(pos, CodeAdd(random.randrange(VAR_CNT), random.randint(0, 10**6))) + if len(code) == 0: + return CodeEmpty() + return CodeBlock(code) + +class ExecutionError(Exception): + pass + +def compile_func(fc, fif): + res = subprocess.run([FUNC_EXECUTABLE, "-o", fif, "-SPA", fc], capture_output=True) + if res.returncode != 0: + raise ExecutionError(str(res.stderr, "utf-8")) + +def runvm(compiled_fif, xl, xr): + runner = os.path.join(TMP_DIR, "runner.fif") + with open(runner, "w") as f: + print("\"%s\" include a) { + x -= 1; + z = 1; + } + return (y, z); +} + +{- + method_id | in | out +TESTCASE | 0 | 101 15 | 100 1 +TESTCASE | 0 | 101 14 | 100 1 +TESTCASE | 0 | 101 10 | 100 0 +TESTCASE | 0 | 100 10 | 100 0 +-} diff --git a/crypto/func/auto-tests/tests/a6.fc b/crypto/func/auto-tests/tests/a6.fc new file mode 100644 index 00000000..05a49fab --- /dev/null +++ b/crypto/func/auto-tests/tests/a6.fc @@ -0,0 +1,89 @@ +(int, int) f(int a, int b, int c, int d, int e, int f) { + ;; solve a 2x2 linear equation + int D = a * d - b * c; + int Dx = e * d - b * f; + int Dy = a * f - e * c; + return (Dx / D, Dy / D); +} + +int calc_phi() { + var n = 1; + repeat (70) { n *= 10; } + var p = var q = 1; + do { + (p, q) = (q, p + q); + } until (q > n); + return muldivr(p, n, q); +} + +int calc_sqrt2() { + var n = 1; + repeat (70) { n *= 10; } + var p = var q = 1; + do { + var t = p + q; + (p, q) = (q, t + q); + } until (q > n); + return muldivr(p, n, q); +} + +var calc_root(m) { + int base = 1; + repeat(70) { base *= 10; } + var (a, b, c) = (1, 0, - m); + var (p1, q1, p2, q2) = (1, 0, 0, 1); + do { + int k = -1; + var (a1, b1, c1) = (0, 0, 0); + do { + k += 1; + (a1, b1, c1) = (a, b, c); + c += b; + c += b += a; + } until (c > 0); + (a, b, c) = (- c1, - b1, - a1); + (p1, q1) = (k * p1 + q1, p1); + (p2, q2) = (k * p2 + q2, p2); + } until (p1 > base); + return (p1, q1, p2, q2); +} + +{- +operator _/%_ infix 20; + +(int, int) ((int x) /% (int y)) { + return (x / y, x % y); +} + +(int, int) _/%_ (int x, int y) { + return (x / y, x % y); +} +-} + +int ataninv(int base, int q) { ;; computes base*atan(1/q) + base ~/= q; + q *= - q; + int sum = 0; + int n = 1; + do { + sum += base ~/ n; + base ~/= q; + n += 2; + } until base == 0; + return sum; +} + +int calc_pi() { + int base = 64; + repeat (70) { base *= 10; } + return (ataninv(base << 2, 5) - ataninv(base, 239)) ~>> 4; +} + +int main() { + return calc_pi(); +} + +{- + method_id | in | out +TESTCASE | 0 | | 31415926535897932384626433832795028841971693993751058209749445923078164 +-} diff --git a/crypto/func/auto-tests/tests/a6_1.fc b/crypto/func/auto-tests/tests/a6_1.fc new file mode 100644 index 00000000..b6341df0 --- /dev/null +++ b/crypto/func/auto-tests/tests/a6_1.fc @@ -0,0 +1,16 @@ +(int, int) main(int a, int b, int c, int d, int e, int f) { + int D = a * d - b * c; + int Dx = e * d - b * f; + int Dy = a * f - e * c; + return (Dx / D, Dy / D); +} + +{- + method_id | in | out +TESTCASE | 0 | 1 1 1 -1 10 6 | 8 2 +TESTCASE | 0 | 817 -31 624 -241 132272 272276 | 132 -788 +TESTCASE | 0 | -886 562 498 -212 -36452 -68958 | -505 -861 +TESTCASE | 0 | 448 -433 -444 792 150012 -356232 | -218 -572 +TESTCASE | 0 | -40 -821 433 -734 -721629 -741724 | -206 889 +TESTCASE | 0 | -261 -98 -494 868 -166153 733738 | 263 995 +-} diff --git a/crypto/func/auto-tests/tests/a6_5.fc b/crypto/func/auto-tests/tests/a6_5.fc new file mode 100644 index 00000000..06b5cc9d --- /dev/null +++ b/crypto/func/auto-tests/tests/a6_5.fc @@ -0,0 +1,24 @@ +var twice(f, x) { + return f (f x); +} + +_ sqr(x) { + return x * x; +} + +var main(x) { + var f = sqr; + return twice(f, x) * f(x); +} + +var pow6(x) method_id(4) { + return twice(sqr, x) * sqr(x); +} + +{- + method_id | in | out +TESTCASE | 0 | 3 | 729 +TESTCASE | 0 | 10 | 1000000 +TESTCASE | 4 | 3 | 729 +TESTCASE | 4 | 10 | 1000000 +-} diff --git a/crypto/func/auto-tests/tests/a7.fc b/crypto/func/auto-tests/tests/a7.fc new file mode 100644 index 00000000..356759d4 --- /dev/null +++ b/crypto/func/auto-tests/tests/a7.fc @@ -0,0 +1,24 @@ +() main() { } +int steps(int x) method_id(1) { + var n = 0; + while (x > 1) { + n += 1; + if (x & 1) { + x = 3 * x + 1; + } else { + x >>= 1; + } + } + return n; +} + +{- + method_id | in | out +TESTCASE | 1 | 1 | 0 +TESTCASE | 1 | 2 | 1 +TESTCASE | 1 | 5 | 5 +TESTCASE | 1 | 19 | 20 +TESTCASE | 1 | 27 | 111 +TESTCASE | 1 | 100 | 25 +-} + diff --git a/crypto/func/auto-tests/tests/c2.fc b/crypto/func/auto-tests/tests/c2.fc new file mode 100644 index 00000000..27f8b88f --- /dev/null +++ b/crypto/func/auto-tests/tests/c2.fc @@ -0,0 +1,17 @@ +global ((int, int) -> int) op; + +int check_assoc(int a, int b, int c) { + return op(op(a, b), c) == op(a, op(b, c)); +} + +int main(int x, int y, int z) { + op = _+_; + return check_assoc(x, y, z); +} + +{- + method_id | in | out +TESTCASE | 0 | 2 3 9 | -1 +TESTCASE | 0 | 11 22 44 | -1 +TESTCASE | 0 | -1 -10 -20 | -1 +-} diff --git a/crypto/func/auto-tests/tests/c2_1.fc b/crypto/func/auto-tests/tests/c2_1.fc new file mode 100644 index 00000000..ef7183b6 --- /dev/null +++ b/crypto/func/auto-tests/tests/c2_1.fc @@ -0,0 +1,14 @@ +_ check_assoc(op, a, b, c) { + return op(op(a, b), c) == op(a, op(b, c)); +} + +int main(int x, int y, int z) { + return check_assoc(_+_, x, y, z); +} + +{- + method_id | in | out +TESTCASE | 0 | 2 3 9 | -1 +TESTCASE | 0 | 11 22 44 | -1 +TESTCASE | 0 | -1 -10 -20 | -1 +-} diff --git a/crypto/func/auto-tests/tests/co1.fc b/crypto/func/auto-tests/tests/co1.fc new file mode 100644 index 00000000..44067243 --- /dev/null +++ b/crypto/func/auto-tests/tests/co1.fc @@ -0,0 +1,60 @@ +const int1 = 1, int2 = 2; + +const int int101 = 101; +const int int111 = 111; + +const int1r = int1; + +const str1 = "const1", str2 = "aabbcc"s; + +const slice str2r = str2; + +const str1int = 0x636f6e737431; +const str2int = 0xAABBCC; + +const int nibbles = 4; + +int iget1() { return int1; } +int iget2() { return int2; } +int iget3() { return int1 + int2; } + +int iget1r() { return int1r; } + +slice sget1() { return str1; } +slice sget2() { return str2; } +slice sget2r() { return str2r; } + +const int int240 = ((int1 + int2) * 10) << 3; + +int iget240() { return int240; } + +builder newc() asm "NEWC"; +slice endcs(builder b) asm "ENDC" "CTOS"; +int sdeq (slice s1, slice s2) asm "SDEQ"; +builder stslicer(builder b, slice s) asm "STSLICER"; + +_ main() { + int i1 = iget1(); + int i2 = iget2(); + int i3 = iget3(); + + throw_unless(int101, i1 == 1); + throw_unless(102, i2 == 2); + throw_unless(103, i3 == 3); + + slice s1 = sget1(); + slice s2 = sget2(); + slice s3 = newc().stslicer(str1).stslicer(str2r).endcs(); + + throw_unless(int111, sdeq(s1, newc().store_uint(str1int, 12 * nibbles).endcs())); + throw_unless(112, sdeq(s2, newc().store_uint(str2int, 6 * nibbles).endcs())); + throw_unless(113, sdeq(s3, newc().store_uint(0x636f6e737431AABBCC, 18 * nibbles).endcs())); + + int i4 = iget240(); + throw_unless(104, i4 == 240); + return 0; +} + +{- +TESTCASE | 0 | | 0 +-} diff --git a/crypto/func/auto-tests/tests/code_after_ifelse.fc b/crypto/func/auto-tests/tests/code_after_ifelse.fc new file mode 100644 index 00000000..49e082c7 --- /dev/null +++ b/crypto/func/auto-tests/tests/code_after_ifelse.fc @@ -0,0 +1,19 @@ +int foo(int x) inline method_id(1) { + if (x == 1) { + return 111; + } else { + x *= 2; + } + return x + 1; +} +(int, int) main(int x) { + return (foo(x), 222); +} + +{- + method_id | in | out +TESTCASE | 1 | 1 | 111 +TESTCASE | 1 | 3 | 7 +TESTCASE | 0 | 1 | 111 222 +TESTCASE | 0 | 3 | 7 222 +-} diff --git a/crypto/func/auto-tests/tests/inline_big.fc b/crypto/func/auto-tests/tests/inline_big.fc new file mode 100644 index 00000000..61ad436e --- /dev/null +++ b/crypto/func/auto-tests/tests/inline_big.fc @@ -0,0 +1,61 @@ +int foo(int x) inline { + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + return x; +} + +(int) main(int x) { + return foo(x) * 10 + 5; +} +{- + method_id | in | out +TESTCASE | 0 | 9 | 9111111111111111111111111111111111111111111111111115 +-} diff --git a/crypto/func/auto-tests/tests/inline_if.fc b/crypto/func/auto-tests/tests/inline_if.fc new file mode 100644 index 00000000..390f0fd3 --- /dev/null +++ b/crypto/func/auto-tests/tests/inline_if.fc @@ -0,0 +1,26 @@ +int foo1(int x) { + if (x == 1) { + return 1; + } + return 2; +} +int foo2(int x) inline { + if (x == 1) { + return 11; + } + return 22; +} +int foo3(int x) inline_ref { + if (x == 1) { + return 111; + } + return 222; +} +(int, int, int) main(int x) { + return (foo1(x) + 1, foo2(x) + 1, foo3(x) + 1); +} +{- + method_id | in | out +TESTCASE | 0 | 1 | 2 12 112 +TESTCASE | 0 | 2 | 3 23 223 +-} diff --git a/crypto/func/auto-tests/tests/inline_loops.fc b/crypto/func/auto-tests/tests/inline_loops.fc new file mode 100644 index 00000000..9f1f45fc --- /dev/null +++ b/crypto/func/auto-tests/tests/inline_loops.fc @@ -0,0 +1,43 @@ +global int g; + +_ foo_repeat() impure inline { + g = 1; + repeat(5) { + g *= 2; + } +} + +int foo_until() impure inline { + g = 1; + int i = 0; + do { + g *= 2; + i += 1; + } until (i >= 8); + return i; +} + +int foo_while() impure inline { + g = 1; + int i = 0; + while (i < 10) { + g *= 2; + i += 1; + } + return i; +} + +_ main() { + foo_repeat(); + int x = g; + foo_until(); + int y = g; + foo_while(); + int z = g; + return (x, y, z); +} + +{- + method_id | in | out +TESTCASE | 0 | | 32 256 1024 +-} diff --git a/crypto/func/auto-tests/tests/method_id.fc b/crypto/func/auto-tests/tests/method_id.fc new file mode 100644 index 00000000..b4e0cd3b --- /dev/null +++ b/crypto/func/auto-tests/tests/method_id.fc @@ -0,0 +1,12 @@ +int foo1() method_id(1) { return 111; } +int foo2() method_id(3) { return 222; } +int foo3() method_id(10) { return 333; } +int main() { return 999; } + +{- + method_id | in | out +TESTCASE | 1 | | 111 +TESTCASE | 3 | | 222 +TESTCASE | 10 | | 333 +TESTCASE | 0 | | 999 +-} diff --git a/crypto/func/auto-tests/tests/s1.fc b/crypto/func/auto-tests/tests/s1.fc new file mode 100644 index 00000000..1541943d --- /dev/null +++ b/crypto/func/auto-tests/tests/s1.fc @@ -0,0 +1,54 @@ +slice ascii_slice() method_id { + return "string"; +} + +slice raw_slice() method_id { + return "abcdef"s; +} + +slice addr_slice() method_id { + return "Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF"a; +} + +int string_hex() method_id { + return "ABCDEFGHIJKLMNOPQRSTUVWXYZ012345"u; +} + +int string_minihash() method_id { + return "transfer(slice, int)"h; +} + +int string_maxihash() method_id { + return "transfer(slice, int)"H; +} + +int string_crc32() method_id { + return "transfer(slice, int)"c; +} + +builder newc() asm "NEWC"; +slice endcs(builder b) asm "ENDC" "CTOS"; +int sdeq (slice s1, slice s2) asm "SDEQ"; + +_ main() { + slice s_ascii = ascii_slice(); + slice s_raw = raw_slice(); + slice s_addr = addr_slice(); + int i_hex = string_hex(); + int i_mini = string_minihash(); + int i_maxi = string_maxihash(); + int i_crc = string_crc32(); + throw_unless(101, sdeq(s_ascii, newc().store_uint(0x737472696E67, 12 * 4).endcs())); + throw_unless(102, sdeq(s_raw, newc().store_uint(0xABCDEF, 6 * 4).endcs())); + throw_unless(103, sdeq(s_addr, newc().store_uint(4, 3).store_int(-1, 8) + .store_uint(0x3333333333333333333333333333333333333333333333333333333333333333, 256).endcs())); + throw_unless(104, i_hex == 0x4142434445464748494A4B4C4D4E4F505152535455565758595A303132333435); + throw_unless(105, i_mini == 0x7a62e8a8); + throw_unless(106, i_maxi == 0x7a62e8a8ebac41bd6de16c65e7be363bc2d2cbc6a0873778dead4795c13db979); + throw_unless(107, i_crc == 2235694568); + return 0; +} + +{- +TESTCASE | 0 | | 0 +-} diff --git a/crypto/func/auto-tests/tests/unbalanced_ret.fc b/crypto/func/auto-tests/tests/unbalanced_ret.fc new file mode 100644 index 00000000..0e4ef6ae --- /dev/null +++ b/crypto/func/auto-tests/tests/unbalanced_ret.fc @@ -0,0 +1,17 @@ +(int, int) main(int x) { + int y = 5; + if (x < 0) { + x *= 2; + y += 1; + if (x == -10) { + return (111, 0); + } + } + return (x + 1, y); +} +{- + method_id | in | out +TESTCASE | 0 | 10 | 11 5 +TESTCASE | 0 | -5 | 111 0 +TESTCASE | 0 | -4 | -7 6 +-} diff --git a/crypto/func/auto-tests/tests/unbalanced_ret_inline.fc b/crypto/func/auto-tests/tests/unbalanced_ret_inline.fc new file mode 100644 index 00000000..6d169345 --- /dev/null +++ b/crypto/func/auto-tests/tests/unbalanced_ret_inline.fc @@ -0,0 +1,18 @@ +int foo(int x) inline { + if (x < 0) { + x *= 2; + if (x == -10) { + return 111; + } + } + return x + 1; +} +int main(int x) { + return foo(x) * 10; +} +{- + method_id | in | out +TESTCASE | 0 | 10 | 110 +TESTCASE | 0 | -5 | 1110 +TESTCASE | 0 | -4 | -70 +-} diff --git a/crypto/func/auto-tests/tests/unbalanced_ret_loops.fc b/crypto/func/auto-tests/tests/unbalanced_ret_loops.fc new file mode 100644 index 00000000..104ec00d --- /dev/null +++ b/crypto/func/auto-tests/tests/unbalanced_ret_loops.fc @@ -0,0 +1,48 @@ +_ main() { } + +int foo_repeat(int x) method_id(1) { + repeat(10) { + x += 10; + if (x >= 100) { + return x; + } + } + return -1; +} + +int foo_while(int x) method_id(2) { + int i = 0; + while (i < 10) { + x += 10; + if (x >= 100) { + return x; + } + i += 1; + } + return -1; +} + +int foo_until(int x) method_id(3) { + int i = 0; + do { + x += 10; + if (x >= 100) { + return x; + } + i += 1; + } until (i >= 10); + return -1; +} + +{- + method_id | in | out +TESTCASE | 1 | 40 | 100 +TESTCASE | 1 | 33 | 103 +TESTCASE | 1 | -5 | -1 +TESTCASE | 2 | 40 | 100 +TESTCASE | 2 | 33 | 103 +TESTCASE | 2 | -5 | -1 +TESTCASE | 3 | 40 | 100 +TESTCASE | 3 | 33 | 103 +TESTCASE | 3 | -5 | -1 +-} diff --git a/crypto/func/auto-tests/tests/unbalanced_ret_nested.fc b/crypto/func/auto-tests/tests/unbalanced_ret_nested.fc new file mode 100644 index 00000000..7ab4bdf0 --- /dev/null +++ b/crypto/func/auto-tests/tests/unbalanced_ret_nested.fc @@ -0,0 +1,35 @@ +int foo(int y) { + if (y < 0) { + y *= 2; + if (y == -10) { + return 111; + } + } + return y + 1; +} +(int, int) bar(int x, int y) { + if (x < 0) { + y = foo(y); + x *= 2; + if (x == -10) { + return (111, y); + } + } + return (x + 1, y); +} +(int, int) main(int x, int y) { + (x, y) = bar(x, y); + return (x, y * 10); +} +{- + method_id | in | out +TESTCASE | 0 | 3 3 | 4 30 +TESTCASE | 0 | 3 -5 | 4 -50 +TESTCASE | 0 | 3 -4 | 4 -40 +TESTCASE | 0 | -5 3 | 111 40 +TESTCASE | 0 | -5 -5 | 111 1110 +TESTCASE | 0 | -5 -4 | 111 -70 +TESTCASE | 0 | -4 3 | -7 40 +TESTCASE | 0 | -4 -5 | -7 1110 +TESTCASE | 0 | -4 -4 | -7 -70 +-} diff --git a/crypto/func/auto-tests/tests/w1.fc b/crypto/func/auto-tests/tests/w1.fc new file mode 100644 index 00000000..3d6a8f69 --- /dev/null +++ b/crypto/func/auto-tests/tests/w1.fc @@ -0,0 +1,14 @@ +(int, int) main(int id) { + if (id > 0) { + if (id > 10) { + return (2 * id, 3 * id); + } + } + return (5, 6); +} +{- + method_id | in | out +TESTCASE | 0 | 0 | 5 6 +TESTCASE | 0 | 4 | 5 6 +TESTCASE | 0 | 11 | 22 33 +-} diff --git a/crypto/func/auto-tests/tests/w2.fc b/crypto/func/auto-tests/tests/w2.fc new file mode 100644 index 00000000..b9e5fb0e --- /dev/null +++ b/crypto/func/auto-tests/tests/w2.fc @@ -0,0 +1,18 @@ +_ f(cs) { + return (cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), + cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), + cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), + cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), + cs~load_uint(8), cs~load_uint(8), cs~load_uint(8), cs~load_uint(8)); +} + +_ main(cs) { + var (x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, + x11, x12, x13, x14, x15, x16, x17, x18, x19) = f(cs); + return x0 + x1 + x2 + x3 + x4 + x5 + x6 + x7 + x8 + x9 + + x10 + x11 + x12 + x13 + x14 + x15 + x16 + x17 + x18 + x19; +} +{- + method_id | in | out +TESTCASE | 0 | x{000102030405060708090a0b0c0d0e0f10111213} | 190 +-} diff --git a/crypto/func/auto-tests/tests/w6.fc b/crypto/func/auto-tests/tests/w6.fc new file mode 100644 index 00000000..9b91cdf7 --- /dev/null +++ b/crypto/func/auto-tests/tests/w6.fc @@ -0,0 +1,17 @@ +int main(int x) { + int i = 0; + ;; int f = false; + do { + i = i + 1; + if (i > 5) { + return 1; + } + int f = (i * i == 64); + } until (f); + return -1; +} + +{- + method_id | in | out +TESTCASE | 0 | 0 | 1 +-} diff --git a/crypto/func/auto-tests/tests/w7.fc b/crypto/func/auto-tests/tests/w7.fc new file mode 100644 index 00000000..991b3e64 --- /dev/null +++ b/crypto/func/auto-tests/tests/w7.fc @@ -0,0 +1,24 @@ +int test(int y) method_id(1) { + int x = 1; + if (y > 0) { + return 1; + } + return x > 0; +} + +int f(int y) method_id(2) { + if (y > 0) { + return 1; + } + return 2; +} + +_ main() { } + +{- + method_id | in | out +TESTCASE | 1 | 10 | 1 +TESTCASE | 1 | -5 | -1 +TESTCASE | 2 | 10 | 1 +TESTCASE | 2 | -5 | 2 +-} diff --git a/crypto/func/auto-tests/tests/w9.fc b/crypto/func/auto-tests/tests/w9.fc new file mode 100644 index 00000000..f299dec1 --- /dev/null +++ b/crypto/func/auto-tests/tests/w9.fc @@ -0,0 +1,14 @@ +_ main(s) { + var (z, t) = (17, s); + while (z > 0) { + t = s; + z -= 1; + } + return ~ t; +} + +{- + method_id | in | out +TESTCASE | 0 | 1 | -2 +TESTCASE | 0 | 5 | -6 +-} From 7d810da5be85cde725cfd94611c1cb1d15c170be Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 27 Sep 2022 11:51:30 +0300 Subject: [PATCH 25/34] Fix array indexing in get_neighbours (#474) --- overlay/overlay.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/overlay/overlay.hpp b/overlay/overlay.hpp index 1d5f9861..f2f26a9a 100644 --- a/overlay/overlay.hpp +++ b/overlay/overlay.hpp @@ -192,7 +192,7 @@ class OverlayImpl : public Overlay { } else { std::vector vec; for (td::uint32 i = 0; i < max_size; i++) { - vec.push_back(neighbours_[td::Random::fast(0, static_cast(neighbours_.size()))]); + vec.push_back(neighbours_[td::Random::fast(0, static_cast(neighbours_.size()) - 1)]); } return vec; } From fc8da44940bcaa60fbcef28058c6cf9c76120841 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 29 Sep 2022 09:24:15 +0300 Subject: [PATCH 26/34] Fix compilation of muliplication (#475) --- crypto/func/builtins.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crypto/func/builtins.cpp b/crypto/func/builtins.cpp index 756de2f8..22f4282c 100644 --- a/crypto/func/builtins.cpp +++ b/crypto/func/builtins.cpp @@ -561,6 +561,7 @@ AsmOp compile_mul_internal(VarDescr& r, VarDescr& x, VarDescr& y) { if (y.always_zero() && x.always_finite()) { // dubious optimization: NaN * 0 = ? r.set_const(y.int_const); + x.unused(); return push_const(r.int_const); } if (*y.int_const == 1 && x.always_finite()) { @@ -587,6 +588,7 @@ AsmOp compile_mul_internal(VarDescr& r, VarDescr& x, VarDescr& y) { if (x.always_zero() && y.always_finite()) { // dubious optimization: NaN * 0 = ? r.set_const(x.int_const); + y.unused(); return push_const(r.int_const); } if (*x.int_const == 1 && y.always_finite()) { From c55b6f84a57050ec2f5f998592d777fae0d3db66 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 27 Jun 2022 19:38:11 +0300 Subject: [PATCH 27/34] HTTP tunnel in rldp-http-proxy --- http/http-connection.cpp | 35 +-- http/http-connection.h | 4 +- http/http-inbound-connection.h | 3 +- http/http-outbound-connection.h | 3 +- http/http.cpp | 55 ++-- http/http.h | 19 +- rldp-http-proxy/rldp-http-proxy.cpp | 413 ++++++++++++++++++++++++---- 7 files changed, 432 insertions(+), 100 deletions(-) diff --git a/http/http-connection.cpp b/http/http-connection.cpp index 6ba1a813..f1c3c58c 100644 --- a/http/http-connection.cpp +++ b/http/http-connection.cpp @@ -58,9 +58,7 @@ void HttpConnection::loop() { } TRY_STATUS(buffered_fd_.flush_write()); if (writing_payload_ && buffered_fd_.left_unwritten() < fd_high_watermark()) { - auto w = buffered_fd_.left_unwritten(); - continue_payload_write(); - written = buffered_fd_.left_unwritten() > w; + written = continue_payload_write(); } if (close_after_write_ && !writing_payload_ && !buffered_fd_.left_unwritten()) { LOG(INFO) << "close after write"; @@ -120,7 +118,7 @@ void HttpConnection::write_payload(std::shared_ptr payload) { class Cb : public HttpPayload::Callback { public: - Cb(td::actor::ActorId conn) : conn_(conn) { + Cb(td::actor::ActorId conn, size_t watermark) : conn_(conn), watermark_(watermark) { } void run(size_t ready_bytes) override { if (!reached_ && ready_bytes >= watermark_) { @@ -135,23 +133,23 @@ void HttpConnection::write_payload(std::shared_ptr payload) { } private: - size_t watermark_ = chunk_size(); - bool reached_ = false; - td::actor::ActorId conn_; + size_t watermark_; + bool reached_ = false; }; - writing_payload_->add_callback(std::make_unique(actor_id(this))); + writing_payload_->add_callback(std::make_unique( + actor_id(this), writing_payload_->payload_type() == HttpPayload::PayloadType::pt_tunnel ? 1 : chunk_size())); continue_payload_write(); } -void HttpConnection::continue_payload_write() { +bool HttpConnection::continue_payload_write() { if (!writing_payload_) { - return; + return false; } if (writing_payload_->is_error()) { stop(); - return; + return false; } auto t = writing_payload_->payload_type(); @@ -159,19 +157,24 @@ void HttpConnection::continue_payload_write() { t = HttpPayload::PayloadType::pt_chunked; } + bool wrote = false; while (!writing_payload_->written()) { if (buffered_fd_.left_unwritten() > fd_high_watermark()) { - return; + return wrote; } - if (!writing_payload_->parse_completed() && writing_payload_->ready_bytes() < chunk_size()) { - return; + bool is_tunnel = writing_payload_->payload_type() == HttpPayload::PayloadType::pt_tunnel; + if (!is_tunnel && !writing_payload_->parse_completed() && writing_payload_->ready_bytes() < chunk_size()) { + return wrote; } - writing_payload_->store_http(buffered_fd_.output_buffer(), chunk_size(), t); + if (is_tunnel && writing_payload_->ready_bytes() == 0) { + return wrote; + } + wrote |= writing_payload_->store_http(buffered_fd_.output_buffer(), chunk_size(), t); } if (writing_payload_->parse_completed() && writing_payload_->written()) { payload_written(); - return; } + return wrote; } td::Status HttpConnection::read_payload(HttpResponse *response) { diff --git a/http/http-connection.h b/http/http-connection.h index f31af484..840ac8a1 100644 --- a/http/http-connection.h +++ b/http/http-connection.h @@ -65,9 +65,7 @@ class HttpConnection : public td::actor::Actor, public td::ObserverBase { void send_request(std::unique_ptr request, std::shared_ptr payload); void send_response(std::unique_ptr response, std::shared_ptr payload); void write_payload(std::shared_ptr payload); - void continue_payload_write(); - td::Status receive_request(); - td::Status receive_response(); + bool continue_payload_write(); td::Status read_payload(HttpRequest *request); td::Status read_payload(HttpResponse *response); td::Status read_payload(std::shared_ptr payload); diff --git a/http/http-inbound-connection.h b/http/http-inbound-connection.h index 876c3902..c63e0a30 100644 --- a/http/http-inbound-connection.h +++ b/http/http-inbound-connection.h @@ -35,7 +35,8 @@ class HttpInboundConnection : public HttpConnection { td::Status receive_eof() override { found_eof_ = true; if (reading_payload_) { - if (reading_payload_->payload_type() != HttpPayload::PayloadType::pt_eof) { + if (reading_payload_->payload_type() != HttpPayload::PayloadType::pt_eof && + reading_payload_->payload_type() != HttpPayload::PayloadType::pt_tunnel) { return td::Status::Error("unexpected EOF"); } else { reading_payload_->complete_parse(); diff --git a/http/http-outbound-connection.h b/http/http-outbound-connection.h index a3f5d513..a3d75258 100644 --- a/http/http-outbound-connection.h +++ b/http/http-outbound-connection.h @@ -44,7 +44,8 @@ class HttpOutboundConnection : public HttpConnection { td::Status receive_eof() override { found_eof_ = true; if (reading_payload_) { - if (reading_payload_->payload_type() != HttpPayload::PayloadType::pt_eof) { + if (reading_payload_->payload_type() != HttpPayload::PayloadType::pt_eof && + reading_payload_->payload_type() != HttpPayload::PayloadType::pt_tunnel) { return td::Status::Error("unexpected EOF"); } else { LOG(INFO) << "stopping (EOF payload)"; diff --git a/http/http.cpp b/http/http.cpp index b3c56c9e..427032f4 100644 --- a/http/http.cpp +++ b/http/http.cpp @@ -164,6 +164,8 @@ td::Result> HttpRequest::create_empty_payload() { if (!need_payload()) { return std::make_shared(HttpPayload::PayloadType::pt_empty); + } else if (method_ == "CONNECT") { + return std::make_shared(HttpPayload::PayloadType::pt_tunnel, low_watermark(), high_watermark()); } else if (found_content_length_) { return std::make_shared(HttpPayload::PayloadType::pt_content_length, low_watermark(), high_watermark(), content_length_); @@ -175,7 +177,7 @@ td::Result> HttpRequest::create_empty_payload() { } bool HttpRequest::need_payload() const { - return found_content_length_ || found_transfer_encoding_; + return found_content_length_ || found_transfer_encoding_ || method_ == "CONNECT"; } td::Status HttpRequest::add_header(HttpHeader header) { @@ -284,7 +286,8 @@ td::Status HttpPayload::parse(td::ChainBufferReader &input) { case PayloadType::pt_empty: UNREACHABLE(); case PayloadType::pt_eof: - cur_chunk_size_ = 1 << 30; + case PayloadType::pt_tunnel: + cur_chunk_size_ = 1LL << 60; break; case PayloadType::pt_chunked: state_ = ParseState::reading_crlf; @@ -480,17 +483,18 @@ void HttpPayload::run_callbacks() { } } -void HttpPayload::store_http(td::ChainBufferWriter &output, size_t max_size, HttpPayload::PayloadType store_type) { +bool HttpPayload::store_http(td::ChainBufferWriter &output, size_t max_size, HttpPayload::PayloadType store_type) { if (store_type == PayloadType::pt_empty) { - return; + return false; } slice_gc(); + bool wrote = false; while (chunks_.size() > 0 && max_size > 0) { auto cur_state = state_.load(std::memory_order_consume); auto s = get_slice(max_size); if (s.size() == 0) { if (cur_state != ParseState::reading_trailer && cur_state != ParseState::completed) { - return; + return wrote; } else { break; } @@ -500,28 +504,33 @@ void HttpPayload::store_http(td::ChainBufferWriter &output, size_t max_size, Htt if (store_type == PayloadType::pt_chunked) { char buf[64]; ::sprintf(buf, "%lx\r\n", s.size()); - output.append(td::Slice(buf, strlen(buf))); + auto slice = td::Slice(buf, strlen(buf)); + wrote |= !slice.empty(); + output.append(slice); } + wrote |= !s.empty(); output.append(std::move(s)); if (store_type == PayloadType::pt_chunked) { output.append(td::Slice("\r\n", 2)); + wrote = true; } } if (chunks_.size() != 0) { - return; + return wrote; } if (!written_zero_chunk_) { if (store_type == PayloadType::pt_chunked) { output.append(td::Slice("0\r\n", 3)); + wrote = true; } written_zero_chunk_ = true; } if (store_type != PayloadType::pt_chunked) { written_trailer_ = true; - return; + return wrote; } while (max_size > 0) { @@ -529,15 +538,16 @@ void HttpPayload::store_http(td::ChainBufferWriter &output, size_t max_size, Htt HttpHeader h = get_header(); if (h.empty()) { if (cur_state != ParseState::completed) { - return; + return wrote; } else { break; } } auto s = h.size(); h.store_http(output); + wrote = true; if (max_size <= s) { - return; + return wrote; } max_size -= s; } @@ -545,7 +555,9 @@ void HttpPayload::store_http(td::ChainBufferWriter &output, size_t max_size, Htt if (!written_trailer_) { output.append(td::Slice("\r\n", 2)); written_trailer_ = true; + wrote = true; } + return wrote; } tl_object_ptr HttpPayload::store_tl(size_t max_size) { @@ -729,17 +741,18 @@ td::Result> HttpResponse::parse(std::unique_ptr> HttpResponse::create(std::string proto_version, td::uint32 code, std::string reason, bool force_no_payload, - bool keep_alive) { + bool keep_alive, bool is_tunnel) { if (proto_version != "HTTP/1.0" && proto_version != "HTTP/1.1") { return td::Status::Error(PSTRING() << "unsupported http version '" << proto_version << "'"); } @@ -749,7 +762,7 @@ td::Result> HttpResponse::create(std::string proto } return std::make_unique(std::move(proto_version), code, std::move(reason), force_no_payload, - keep_alive); + keep_alive, is_tunnel); } td::Status HttpResponse::complete_parse_header() { @@ -767,6 +780,8 @@ td::Result> HttpResponse::create_empty_payload() { if (!need_payload()) { return std::make_shared(HttpPayload::PayloadType::pt_empty); + } else if (is_tunnel_) { + return std::make_shared(HttpPayload::PayloadType::pt_tunnel, low_watermark(), high_watermark()); } else if (found_content_length_) { return std::make_shared(HttpPayload::PayloadType::pt_content_length, low_watermark(), high_watermark(), content_length_); @@ -828,10 +843,12 @@ void HttpResponse::store_http(td::ChainBufferWriter &output) { for (auto &x : options_) { x.store_http(output); } - if (keep_alive_) { - HttpHeader{"Connection", "Keep-Alive"}.store_http(output); - } else { - HttpHeader{"Connection", "Close"}.store_http(output); + if (!is_tunnel_) { + if (keep_alive_) { + HttpHeader{"Connection", "Keep-Alive"}.store_http(output); + } else { + HttpHeader{"Connection", "Close"}.store_http(output); + } } output.append(td::Slice("\r\n", 2)); } @@ -893,7 +910,9 @@ void answer_error(HttpStatusCode code, std::string reason, } auto response = HttpResponse::create("HTTP/1.0", code, reason, false, false).move_as_ok(); response->add_header(HttpHeader{"Content-Length", "0"}); + response->complete_parse_header(); auto payload = response->create_empty_payload().move_as_ok(); + payload->complete_parse(); CHECK(payload->parse_completed()); promise.set_value(std::make_pair(std::move(response), std::move(payload))); } diff --git a/http/http.h b/http/http.h index f5e2087a..8e109455 100644 --- a/http/http.h +++ b/http/http.h @@ -64,7 +64,7 @@ td::Result get_header(std::string line); class HttpPayload { public: - enum class PayloadType { pt_empty, pt_eof, pt_chunked, pt_content_length }; + enum class PayloadType { pt_empty, pt_eof, pt_chunked, pt_content_length, pt_tunnel }; HttpPayload(PayloadType t, size_t low_watermark, size_t high_watermark, td::uint64 size) : type_(t), low_watermark_(low_watermark), high_watermark_(high_watermark), cur_chunk_size_(size) { CHECK(t == PayloadType::pt_content_length); @@ -75,17 +75,15 @@ class HttpPayload { CHECK(t != PayloadType::pt_content_length); CHECK(t != PayloadType::pt_empty); switch (t) { - case PayloadType::pt_empty: - UNREACHABLE(); case PayloadType::pt_eof: + case PayloadType::pt_tunnel: state_ = ParseState::reading_chunk_data; break; case PayloadType::pt_chunked: state_ = ParseState::reading_chunk_header; break; - case PayloadType::pt_content_length: - state_ = ParseState::reading_chunk_data; - break; + default: + UNREACHABLE(); } } HttpPayload(PayloadType t) : type_(t) { @@ -136,7 +134,7 @@ class HttpPayload { void slice_gc(); HttpHeader get_header(); - void store_http(td::ChainBufferWriter &output, size_t max_size, HttpPayload::PayloadType store_type); + bool store_http(td::ChainBufferWriter &output, size_t max_size, HttpPayload::PayloadType store_type); tl_object_ptr store_tl(size_t max_size); bool written() const { @@ -267,9 +265,11 @@ class HttpResponse { } static td::Result> create(std::string proto_version, td::uint32 code, - std::string reason, bool force_no_payload, bool keep_alive); + std::string reason, bool force_no_payload, bool keep_alive, + bool is_tunnel = false); - HttpResponse(std::string proto_version, td::uint32 code, std::string reason, bool force_no_payload, bool keep_alive); + HttpResponse(std::string proto_version, td::uint32 code, std::string reason, bool force_no_payload, bool keep_alive, + bool is_tunnel = false); bool check_parse_header_completed() const; bool keep_alive() const { @@ -323,6 +323,7 @@ class HttpResponse { bool keep_alive_ = false; std::vector options_; + bool is_tunnel_ = false; }; void answer_error(HttpStatusCode code, std::string reason, diff --git a/rldp-http-proxy/rldp-http-proxy.cpp b/rldp-http-proxy/rldp-http-proxy.cpp index fb5ae4c1..6ecb8e3a 100644 --- a/rldp-http-proxy/rldp-http-proxy.cpp +++ b/rldp-http-proxy/rldp-http-proxy.cpp @@ -52,6 +52,8 @@ #include #include #include "git.h" +#include "td/utils/BufferedFd.h" +#include "common/delay.h" #if TD_DARWIN || TD_LINUX #include @@ -128,8 +130,15 @@ class HttpRldpPayloadReceiver : public td::actor::Actor { public: HttpRldpPayloadReceiver(std::shared_ptr payload, td::Bits256 transfer_id, ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort local_id, - td::actor::ActorId adnl, td::actor::ActorId rldp) - : payload_(std::move(payload)), id_(transfer_id), src_(src), local_id_(local_id), adnl_(adnl), rldp_(rldp) { + td::actor::ActorId adnl, td::actor::ActorId rldp, + bool is_tunnel = false) + : payload_(std::move(payload)) + , id_(transfer_id) + , src_(src) + , local_id_(local_id) + , adnl_(adnl) + , rldp_(rldp) + , is_tunnel_(is_tunnel) { } void start_up() override { @@ -178,13 +187,14 @@ class HttpRldpPayloadReceiver : public td::actor::Actor { auto f = ton::create_serialize_tl_object( id_, seqno_++, static_cast(chunk_size())); + auto timeout = td::Timestamp::in(is_tunnel_ ? 60.0 : 15.0); td::actor::send_closure(rldp_, &ton::rldp::Rldp::send_query_ex, local_id_, src_, "payload part", std::move(P), - td::Timestamp::in(15.0), std::move(f), 2 * chunk_size() + 1024); + timeout, std::move(f), 2 * chunk_size() + 1024); } void add_data(td::BufferSlice data) { LOG(INFO) << "HttpPayloadReceiver: received answer (size " << data.size() << ")"; - auto F = ton::fetch_tl_object(std::move(data), true); + auto F = ton::fetch_tl_object(data, true); if (F.is_error()) { abort_query(F.move_as_error()); return; @@ -243,14 +253,20 @@ class HttpRldpPayloadReceiver : public td::actor::Actor { bool sent_ = false; td::int32 seqno_ = 0; + bool is_tunnel_; }; class HttpRldpPayloadSender : public td::actor::Actor { public: HttpRldpPayloadSender(std::shared_ptr payload, td::Bits256 transfer_id, ton::adnl::AdnlNodeIdShort local_id, td::actor::ActorId adnl, - td::actor::ActorId rldp) - : payload_(std::move(payload)), id_(transfer_id), local_id_(local_id), adnl_(adnl), rldp_(rldp) { + td::actor::ActorId rldp, bool is_tunnel = false) + : payload_(std::move(payload)) + , id_(transfer_id) + , local_id_(local_id) + , adnl_(adnl) + , rldp_(rldp) + , is_tunnel_(is_tunnel) { } std::string generate_prefix() const { @@ -287,32 +303,36 @@ class HttpRldpPayloadSender : public td::actor::Actor { class Cb : public ton::http::HttpPayload::Callback { public: - Cb(td::actor::ActorId id) : self_id_(id) { + Cb(td::actor::ActorId id, size_t watermark) : self_id_(id), watermark_(watermark) { } void run(size_t ready_bytes) override { if (!reached_ && ready_bytes >= watermark_) { reached_ = true; - td::actor::send_closure(self_id_, &HttpRldpPayloadSender::try_answer_query); + td::actor::send_closure(self_id_, &HttpRldpPayloadSender::try_answer_query, false); } else if (reached_ && ready_bytes < watermark_) { reached_ = false; } } void completed() override { - td::actor::send_closure(self_id_, &HttpRldpPayloadSender::try_answer_query); + td::actor::send_closure(self_id_, &HttpRldpPayloadSender::try_answer_query, false); } private: - size_t watermark_ = ton::http::HttpRequest::low_watermark(); bool reached_ = false; td::actor::ActorId self_id_; + size_t watermark_; }; - payload_->add_callback(std::make_unique(actor_id(this))); + payload_->add_callback(std::make_unique(actor_id(this), + is_tunnel_ ? 1 : ton::http::HttpRequest::low_watermark())); - alarm_timestamp() = td::Timestamp::in(10.0); + alarm_timestamp() = td::Timestamp::in(is_tunnel_ ? 60.0 : 10.0); } - void try_answer_query() { + void try_answer_query(bool from_timer = false) { + if (from_timer) { + active_timer_ = false; + } if (!cur_query_promise_) { return; } @@ -321,6 +341,15 @@ class HttpRldpPayloadSender : public td::actor::Actor { } if (payload_->parse_completed() || payload_->ready_bytes() >= ton::http::HttpRequest::low_watermark()) { answer_query(); + } else if (!is_tunnel_ || payload_->ready_bytes() == 0) { + return; + } else if (from_timer) { + answer_query(); + } else if (!active_timer_) { + active_timer_ = true; + ton::delay_action([SelfId = actor_id(this)]() { + td::actor::send_closure(SelfId, &HttpRldpPayloadSender::try_answer_query, true); + }, td::Timestamp::in(0.001)); } } @@ -348,16 +377,12 @@ class HttpRldpPayloadSender : public td::actor::Actor { LOG(INFO) << "received request. size=" << cur_query_size_ << " parse_completed=" << payload_->parse_completed() << " ready_bytes=" << payload_->ready_bytes(); - if (payload_->parse_completed() || payload_->ready_bytes() >= ton::http::HttpRequest::low_watermark()) { - answer_query(); - return; - } - - alarm_timestamp() = td::Timestamp::in(10.0); + alarm_timestamp() = td::Timestamp::in(is_tunnel_ ? 50.0 : 10.0); + try_answer_query(false); } void receive_query(td::BufferSlice data, td::Promise promise) { - auto F = ton::fetch_tl_object(std::move(data), true); + auto F = ton::fetch_tl_object(data, true); if (F.is_error()) { LOG(INFO) << "failed to parse query: " << F.move_as_error(); return; @@ -367,6 +392,10 @@ class HttpRldpPayloadSender : public td::actor::Actor { void alarm() override { if (cur_query_promise_) { + if (is_tunnel_) { + answer_query(); + return; + } LOG(INFO) << "timeout on inbound connection. closing http transfer"; } else { LOG(INFO) << "timeout on RLDP connection. closing http transfer"; @@ -382,7 +411,7 @@ class HttpRldpPayloadSender : public td::actor::Actor { } seqno_++; - alarm_timestamp() = td::Timestamp::in(30.0); + alarm_timestamp() = td::Timestamp::in(is_tunnel_ ? 60.0 : 30.0); } void abort_query(td::Status error) { @@ -412,6 +441,7 @@ class HttpRldpPayloadSender : public td::actor::Actor { size_t cur_query_size_; td::Promise cur_query_promise_; + bool is_tunnel_, active_timer_ = false; }; class RldpHttpProxy; @@ -452,7 +482,8 @@ class TcpToRldpRequestSender : public td::actor::Actor { } }); - td::actor::create_actor("HttpPayloadSender", request_payload_, id_, local_id_, adnl_, rldp_) + td::actor::create_actor("HttpPayloadSender", request_payload_, id_, local_id_, adnl_, rldp_, + is_tunnel()) .release(); auto f = ton::serialize_tl_object(request_->store_tl(id_), true); @@ -461,13 +492,14 @@ class TcpToRldpRequestSender : public td::actor::Actor { } void got_result(td::BufferSlice data) { - auto F = ton::fetch_tl_object(std::move(data), true); + auto F = ton::fetch_tl_object(data, true); if (F.is_error()) { abort_query(F.move_as_error()); return; } auto f = F.move_as_ok(); - auto R = ton::http::HttpResponse::create(f->http_version_, f->status_code_, f->reason_, false, true); + auto R = ton::http::HttpResponse::create( + f->http_version_, f->status_code_, f->reason_, false, true, is_tunnel() && f->status_code_ == 200); if (R.is_error()) { abort_query(R.move_as_error()); return; @@ -498,7 +530,7 @@ class TcpToRldpRequestSender : public td::actor::Actor { } }); td::actor::create_actor("HttpPayloadReceiver", response_payload_, id_, dst_, local_id_, - adnl_, rldp_) + adnl_, rldp_, is_tunnel()) .release(); promise_.set_value(std::make_pair(std::move(response_), std::move(response_payload_))); @@ -515,6 +547,10 @@ class TcpToRldpRequestSender : public td::actor::Actor { } protected: + bool is_tunnel() const { + return request_->method() == "CONNECT"; + } + td::Bits256 id_; ton::adnl::AdnlNodeIdShort local_id_; @@ -535,6 +571,208 @@ class TcpToRldpRequestSender : public td::actor::Actor { std::shared_ptr response_payload_; }; +class RldpTcpTunnel : public td::actor::Actor, private td::ObserverBase { + public: + RldpTcpTunnel(td::Bits256 transfer_id, ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort local_id, + td::actor::ActorId adnl, td::actor::ActorId rldp, td::SocketFd fd) + : id_(transfer_id) + , src_(src) + , local_id_(local_id) + , adnl_(std::move(adnl)) + , rldp_(std::move(rldp)) + , fd_(std::move(fd)) { + } + + void start_up() override { + self_ = actor_id(this); + td::actor::SchedulerContext::get()->get_poll().subscribe(fd_.get_poll_info().extract_pollable_fd(this), + td::PollFlags::ReadWrite()); + + class Cb : public ton::adnl::Adnl::Callback { + public: + explicit Cb(td::actor::ActorId id) : self_id_(std::move(id)) { + } + void receive_message(ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, + td::BufferSlice data) override { + LOG(INFO) << "rldp tcp tunnel: dropping message"; + } + void receive_query(ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data, + td::Promise promise) override { + td::actor::send_closure(self_id_, &RldpTcpTunnel::receive_query, std::move(data), std::move(promise)); + } + + private: + td::actor::ActorId self_id_; + }; + td::actor::send_closure(adnl_, &ton::adnl::Adnl::subscribe, local_id_, generate_prefix(), + std::make_unique(actor_id(this))); + process(); + } + + void tear_down() override { + LOG(INFO) << "RldpTcpTunnel: tear_down"; + td::actor::send_closure(adnl_, &ton::adnl::Adnl::unsubscribe, local_id_, generate_prefix()); + td::actor::SchedulerContext::get()->get_poll().unsubscribe(fd_.get_poll_info().get_pollable_fd_ref()); + } + + void notify() override { + td::actor::send_closure(self_, &RldpTcpTunnel::process); + } + + void request_data() { + if (close_ || sent_request_) { + return; + } + sent_request_ = true; + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + td::actor::send_closure(SelfId, &RldpTcpTunnel::got_data_from_rldp, std::move(R)); + }); + + auto f = ton::create_serialize_tl_object(id_, out_seqno_++, 1 << 17); + td::actor::send_closure(rldp_, &ton::rldp::Rldp::send_query_ex, local_id_, src_, "payload part", std::move(P), + td::Timestamp::in(60.0), std::move(f), (1 << 18) + 1024); + } + + void receive_query(td::BufferSlice data, td::Promise promise) { + auto F = ton::fetch_tl_object(data, true); + if (F.is_error()) { + LOG(INFO) << "failed to parse query: " << F.error(); + promise.set_error(F.move_as_error()); + return; + } + auto f = F.move_as_ok(); + if (cur_promise_) { + LOG(INFO) << "failed to process query: previous query is active"; + promise.set_error(td::Status::Error("previous query is active")); + return; + } + if (f->seqno_ != cur_seqno_) { + LOG(INFO) << "failed to process query: seqno mismatch"; + promise.set_error(td::Status::Error("seqno mismatch")); + return; + } + LOG(INFO) << "RldpTcpTunnel: received query, seqno=" << cur_seqno_; + cur_promise_ = std::move(promise); + cur_max_chunk_size_ = f->max_chunk_size_; + alarm_timestamp() = td::Timestamp::in(50.0); + process(); + } + + void got_data_from_rldp(td::Result R) { + if (R.is_error()) { + abort(R.move_as_error()); + return; + } + td::BufferSlice data = R.move_as_ok(); + LOG(INFO) << "RldpTcpTunnel: received data from rldp: size=" << data.size(); + sent_request_ = false; + auto F = ton::fetch_tl_object(data, true); + if (F.is_error()) { + abort(F.move_as_error()); + return; + } + auto f = F.move_as_ok(); + fd_.output_buffer().append(std::move(f->data_)); + if (f->last_) { + got_last_part_ = true; + } + process(); + } + + void process() { + if (!close_) { + auto status = [&] { + TRY_STATUS(fd_.flush_read()); + TRY_STATUS(fd_.flush_write()); + close_ = td::can_close(fd_); + return td::Status::OK(); + }(); + if (status.is_error()) { + abort(std::move(status)); + return; + } + } + if (got_last_part_) { + close_ = true; + } + answer_query(); + request_data(); + } + + void answer_query(bool allow_empty = false, bool from_timer = false) { + if (from_timer) { + active_timer_ = false; + } + auto &input = fd_.input_buffer(); + if (cur_promise_ && (!input.empty() || close_ || allow_empty)) { + if (!from_timer && !close_ && !allow_empty && input.size() < ton::http::HttpRequest::low_watermark()) { + if (!active_timer_) { + active_timer_ = true; + ton::delay_action([SelfId = actor_id(this)]() { + td::actor::send_closure(SelfId, &RldpTcpTunnel::answer_query, false, true); + }, td::Timestamp::in(0.001)); + } + return; + } + size_t s = std::min(input.size(), cur_max_chunk_size_); + td::BufferSlice data(s); + LOG(INFO) << "RldpTcpTunnel: sending data to rldp: size=" << data.size(); + input.advance(s, td::as_mutable_slice(data)); + cur_promise_.set_result(ton::create_serialize_tl_object( + std::move(data), std::vector>(), close_)); + ++cur_seqno_; + cur_promise_.reset(); + alarm_timestamp() = td::Timestamp::never(); + if (close_) { + stop(); + return; + } + } + } + + void alarm() override { + answer_query(true, false); + } + + void abort(td::Status status) { + LOG(INFO) << "RldpTcpTunnel error: " << status; + if (cur_promise_) { + cur_promise_.set_error(status.move_as_error()); + } + stop(); + } + + private: + std::string generate_prefix() const { + std::string x(static_cast(36), '\0'); + auto S = td::MutableSlice{x}; + CHECK(S.size() == 36); + + auto id = ton::ton_api::http_getNextPayloadPart::ID; + S.copy_from(td::Slice(reinterpret_cast(&id), 4)); + S.remove_prefix(4); + S.copy_from(id_.as_slice()); + return x; + } + + td::Bits256 id_; + + ton::adnl::AdnlNodeIdShort src_; + ton::adnl::AdnlNodeIdShort local_id_; + td::actor::ActorId adnl_; + td::actor::ActorId rldp_; + + td::BufferedFd fd_; + + td::actor::ActorId self_; + + td::int32 cur_seqno_ = 0, cur_max_chunk_size_ = 0; + td::Promise cur_promise_; + td::int32 out_seqno_ = 0; + bool close_ = false, sent_request_ = false, got_last_part_ = false; + bool active_timer_ = false; +}; + class RldpToTcpRequestSender : public td::actor::Actor { public: RldpToTcpRequestSender(td::Bits256 id, ton::adnl::AdnlNodeIdShort local_id, ton::adnl::AdnlNodeIdShort dst, @@ -627,8 +865,8 @@ class RldpHttpProxy : public td::actor::Actor { client_port_ = port; } - void set_local_host(std::string name, td::IPAddress remote) { - local_hosts_.emplace_back(std::move(name), std::move(remote)); + void set_local_host(std::string host, td::uint16 port, td::IPAddress remote) { + hosts_[host].ports_[port].remote_addr_ = remote; } void receive_request_result(td::uint64 id, td::Result> R) { @@ -688,7 +926,7 @@ class RldpHttpProxy : public td::actor::Actor { } void store_dht() { - for (auto &serv : local_hosts_) { + for (auto &serv : hosts_) { if (serv.first != "*") { for (auto &serv_id : server_ids_) { ton::PublicKey key = ton::pubkeys::Unenc{"http." + serv.first}; @@ -765,7 +1003,7 @@ class RldpHttpProxy : public td::actor::Actor { } void run_cont() { - if (is_client_ && local_hosts_.size() > 0) { + if (is_client_ && hosts_.size() > 0) { LOG(ERROR) << "client-only node cannot be server"; std::_Exit(2); } @@ -876,9 +1114,6 @@ class RldpHttpProxy : public td::actor::Actor { ton::adnl::Adnl::int_to_bytestring(ton::ton_api::http_request::ID), std::make_unique(actor_id(this))); } - for (auto &serv : local_hosts_) { - servers_.emplace(serv.first, td::actor::create_actor("remote", serv.second)); - } rldp_ = ton::rldp::Rldp::create(adnl_.get()); td::actor::send_closure(rldp_, &ton::rldp::Rldp::add_id, local_id_); @@ -938,7 +1173,7 @@ class RldpHttpProxy : public td::actor::Actor { void receive_rldp_request(ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data, td::Promise promise) { LOG(INFO) << "got HTTP request over rldp from " << src; - TRY_RESULT_PROMISE(promise, f, ton::fetch_tl_object(std::move(data), true)); + TRY_RESULT_PROMISE(promise, f, ton::fetch_tl_object(data, true)); TRY_RESULT_PROMISE(promise, request, ton::http::HttpRequest::create(f->method_, f->url_, f->http_version_)); for (auto &x : f->headers_) { ton::http::HttpHeader h{x->name_, x->value_}; @@ -947,7 +1182,8 @@ class RldpHttpProxy : public td::actor::Actor { } TRY_STATUS_PROMISE(promise, request->complete_parse_header()); auto host = request->host(); - if (host.size() == 0) { + td::uint16 port = 80; + if (host.empty()) { host = request->url(); if (host.size() >= 7 && host.substr(0, 7) == "http://") { host = host.substr(7); @@ -972,29 +1208,60 @@ class RldpHttpProxy : public td::actor::Actor { { auto p = host.find(':'); if (p != std::string::npos) { + try { + port = (td::uint16)std::stoul(host.substr(p + 1)); + } catch (const std::logic_error &) { + port = 80; + promise.set_error(td::Status::Error(PSLICE() << "Invalid port '" << host.substr(p + 1) << "'")); + return; + } host = host.substr(0, p); } } std::transform(host.begin(), host.end(), host.begin(), [](unsigned char c) { return std::tolower(c); }); - auto it = servers_.find(host); - if (it == servers_.end()) { - it = servers_.find("*"); - if (it == servers_.end()) { + auto it = hosts_.find(host); + if (it == hosts_.end()) { + it = hosts_.find("*"); + if (it == hosts_.end()) { promise.set_error(td::Status::Error(ton::ErrorCode::error, "unknown server name")); return; } } + auto it2 = it->second.ports_.find(port); + if (it2 == it->second.ports_.end()) { + promise.set_error(td::Status::Error(ton::ErrorCode::error, "unknown host:port")); + return; + } + auto &server = it2->second; + if (request->method() == "CONNECT") { + LOG(INFO) << "starting HTTP tunnel over RLDP to " << server.remote_addr_; + start_tcp_tunnel(f->id_, src, dst, server.remote_addr_, std::move(promise)); + return; + } + + if (server.http_remote_.empty()) { + server.http_remote_ = td::actor::create_actor("remote", server.remote_addr_); + } TRY_RESULT_PROMISE(promise, payload, request->create_empty_payload()); LOG(INFO) << "starting HTTP over RLDP request"; td::actor::create_actor("inboundreq", f->id_, dst, src, std::move(request), std::move(payload), std::move(promise), adnl_.get(), rldp_.get(), - it->second.get()) + server.http_remote_.get()) .release(); } + void start_tcp_tunnel(td::Bits256 id, ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort local_id, + td::IPAddress ip, td::Promise promise) { + TRY_RESULT_PROMISE(promise, fd, td::SocketFd::open(ip)); + td::actor::create_actor(td::actor::ActorOptions().with_name("tunnel").with_poll(), id, src, local_id, + adnl_.get(), rldp_.get(), std::move(fd)).release(); + promise.set_result(ton::create_serialize_tl_object( + "HTTP/1.1", 200, "Connection Established", std::vector>())); + } + void add_adnl_addr(ton::adnl::AdnlNodeIdShort id) { server_ids_.insert(id); } @@ -1008,10 +1275,17 @@ class RldpHttpProxy : public td::actor::Actor { } private: + struct Host { + struct Server { + td::IPAddress remote_addr_; + td::actor::ActorOwn http_remote_; + }; + std::map ports_; + }; + td::uint16 port_{0}; td::IPAddress addr_; std::string global_config_; - std::vector> local_hosts_; bool is_client_{false}; td::uint16 client_port_{0}; @@ -1023,7 +1297,7 @@ class RldpHttpProxy : public td::actor::Actor { td::actor::ActorOwn server_; std::map dns_; - std::map> servers_; + std::map hosts_; td::actor::ActorOwn keyring_; td::actor::ActorOwn adnl_network_manager_; @@ -1123,6 +1397,42 @@ int main(int argc, char *argv[]) { td::log_interface = td::default_log_interface; }; + auto add_local_host = [&](const std::string& local, const std::string& remote) -> td::Status { + std::string host; + std::vector ports; + auto p = local.find(':'); + if (p == std::string::npos) { + host = local; + ports = {80, 443}; + } else { + host = local.substr(0, p); + ++p; + while (p < local.size()) { + auto p2 = local.find(',', p); + if (p2 == std::string::npos) { + p2 = local.size(); + } + try { + ports.push_back((td::uint16)std::stoul(local.substr(p, p2 - p))); + } catch (const std::logic_error& e) { + return td::Status::Error(PSLICE() << "Invalid port: " << local.substr(p, p2 - p)); + } + p = p2 + 1; + } + } + for (td::uint16 port : ports) { + std::string cur_remote = remote; + if (cur_remote.find(':') == std::string::npos) { + cur_remote += ':'; + cur_remote += std::to_string(port); + } + td::IPAddress addr; + TRY_STATUS(addr.init_host_port(cur_remote)); + td::actor::send_closure(x, &RldpHttpProxy::set_local_host, host, port, addr); + } + return td::Status::OK(); + }; + td::OptionParser p; p.set_description( "A simple rldp-to-http and http-to-rldp proxy for running and accessing ton sites\n" @@ -1136,7 +1446,8 @@ int main(int argc, char *argv[]) { SET_VERBOSITY_LEVEL(v); }); p.add_option('V', "version", "shows rldp-http-proxy build information", [&]() { - std::cout << "rldp-http-proxy build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; + std::cout << "rldp-http-proxy build information: [ Commit: " << GitMetadata::CommitSHA1() + << ", Date: " << GitMetadata::CommitDate() << "]\n"; std::exit(0); }); p.add_option('h', "help", "prints a help message", [&]() { @@ -1170,27 +1481,25 @@ int main(int argc, char *argv[]) { }); p.add_option('C', "global-config", "global TON configuration file", [&](td::Slice arg) { td::actor::send_closure(x, &RldpHttpProxy::set_global_config, arg.str()); }); - p.add_checked_option('L', "local", "http hostname that will be proxied to http server at localhost:80", + p.add_checked_option('L', "local", + ":, hostname that will be proxied to localhost\n" + " is a comma-separated list of ports (may be omitted, default: 80, 443)\n", [&](td::Slice arg) -> td::Status { - td::IPAddress addr; - TRY_STATUS(addr.init_ipv4_port("127.0.0.1", 80)); - td::actor::send_closure(x, &RldpHttpProxy::set_local_host, arg.str(), addr); - return td::Status::OK(); + return add_local_host(arg.str(), "127.0.0.1"); }); p.add_option('D', "db", "db root", [&](td::Slice arg) { td::actor::send_closure(x, &RldpHttpProxy::set_db_root, arg.str()); }); p.add_checked_option( 'R', "remote", - "@:, indicates a http hostname that will be proxied to remote http server at :", + ":@:, indicates a hostname that will be proxied to remote server at :\n" + " is a comma-separated list of ports (may be omitted, default: 80,433)\n" + " is a remote port (may be omitted, default: same as host's port)", [&](td::Slice arg) -> td::Status { auto ch = arg.find('@'); if (ch == td::Slice::npos) { return td::Status::Error("bad format for --remote"); } - td::IPAddress addr; - TRY_STATUS(addr.init_host_port(arg.substr(ch + 1).str())); - td::actor::send_closure(x, &RldpHttpProxy::set_local_host, arg.substr(0, ch).str(), addr); - return td::Status::OK(); + return add_local_host(arg.substr(0, ch).str(), arg.substr(ch + 1).str()); }); p.add_option('d', "daemonize", "set SIGHUP", [&]() { td::set_signal_handler(td::SignalType::HangUp, [](int sig) { From 9107bcaf24917c1f845cf28bff09181f0a2596ab Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 29 Jun 2022 15:53:48 +0300 Subject: [PATCH 28/34] Correctly return errors from proxy --- http/http-inbound-connection.cpp | 25 ++++++--- http/http-inbound-connection.h | 2 +- http/http.cpp | 2 +- rldp-http-proxy/rldp-http-proxy.cpp | 80 +++++++++++++++++++--------- tl/generate/scheme/ton_api.tl | 2 +- tl/generate/scheme/ton_api.tlo | Bin 67196 -> 67232 bytes 6 files changed, 75 insertions(+), 36 deletions(-) diff --git a/http/http-inbound-connection.cpp b/http/http-inbound-connection.cpp index ef2f3a58..89c9c123 100644 --- a/http/http-inbound-connection.cpp +++ b/http/http-inbound-connection.cpp @@ -44,13 +44,22 @@ void HttpInboundConnection::send_server_error() { loop(); } -void HttpInboundConnection::send_proxy_error() { - static const auto s = - "HTTP/1.1 502 Bad Gateway\r\n" - "Connection: keep-alive\r\n" - "Content-length: 0\r\n" - "\r\n"; - buffered_fd_.output_buffer().append(td::Slice(s, strlen(s))); +void HttpInboundConnection::send_proxy_error(td::Status error) { + if (error.code() == ErrorCode::timeout) { + static const auto s = + "HTTP/1.1 504 Gateway Timeout\r\n" + "Connection: keep-alive\r\n" + "Content-length: 0\r\n" + "\r\n"; + buffered_fd_.output_buffer().append(td::Slice(s, strlen(s))); + } else { + static const auto s = + "HTTP/1.1 502 Bad Gateway\r\n" + "Connection: keep-alive\r\n" + "Content-length: 0\r\n" + "\r\n"; + buffered_fd_.output_buffer().append(td::Slice(s, strlen(s))); + } loop(); } @@ -83,7 +92,7 @@ td::Status HttpInboundConnection::receive(td::ChainBufferReader &input) { auto a = R.move_as_ok(); td::actor::send_closure(SelfId, &HttpInboundConnection::send_answer, std::move(a.first), std::move(a.second)); } else { - td::actor::send_closure(SelfId, &HttpInboundConnection::send_proxy_error); + td::actor::send_closure(SelfId, &HttpInboundConnection::send_proxy_error, R.move_as_error()); } }); http_callback_->receive_request(std::move(cur_request_), payload, std::move(P)); diff --git a/http/http-inbound-connection.h b/http/http-inbound-connection.h index c63e0a30..e3602267 100644 --- a/http/http-inbound-connection.h +++ b/http/http-inbound-connection.h @@ -54,7 +54,7 @@ class HttpInboundConnection : public HttpConnection { void send_client_error(); void send_server_error(); - void send_proxy_error(); + void send_proxy_error(td::Status error); void payload_written() override { writing_payload_ = nullptr; diff --git a/http/http.cpp b/http/http.cpp index 427032f4..c99a5ec4 100644 --- a/http/http.cpp +++ b/http/http.cpp @@ -864,7 +864,7 @@ tl_object_ptr HttpResponse::store_tl() { } else { headers.push_back(HttpHeader{"Connection", "Close"}.store_tl()); } - return create_tl_object(proto_version_, code_, reason_, std::move(headers)); + return create_tl_object(proto_version_, code_, reason_, std::move(headers), false); } td::Status HttpHeader::basic_check() { diff --git a/rldp-http-proxy/rldp-http-proxy.cpp b/rldp-http-proxy/rldp-http-proxy.cpp index 6ecb8e3a..98088713 100644 --- a/rldp-http-proxy/rldp-http-proxy.cpp +++ b/rldp-http-proxy/rldp-http-proxy.cpp @@ -122,10 +122,15 @@ class HttpRemote : public td::actor::Actor { private: td::IPAddress addr_; - bool ready_ = false; + bool ready_ = true; td::actor::ActorOwn client_; }; +td::BufferSlice create_error_response(const std::string& proto_version, int code, const std::string& reason) { + return ton::create_serialize_tl_object( + proto_version, code, reason, std::vector>(), true); +} + class HttpRldpPayloadReceiver : public td::actor::Actor { public: HttpRldpPayloadReceiver(std::shared_ptr payload, td::Bits256 transfer_id, @@ -499,7 +504,7 @@ class TcpToRldpRequestSender : public td::actor::Actor { } auto f = F.move_as_ok(); auto R = ton::http::HttpResponse::create( - f->http_version_, f->status_code_, f->reason_, false, true, is_tunnel() && f->status_code_ == 200); + f->http_version_, f->status_code_, f->reason_, f->no_payload_, true, is_tunnel() && f->status_code_ == 200); if (R.is_error()) { abort_query(R.move_as_error()); return; @@ -529,9 +534,13 @@ class TcpToRldpRequestSender : public td::actor::Actor { td::actor::send_closure(SelfId, &TcpToRldpRequestSender::finished_payload_transfer); } }); - td::actor::create_actor("HttpPayloadReceiver", response_payload_, id_, dst_, local_id_, - adnl_, rldp_, is_tunnel()) - .release(); + if (f->no_payload_) { + response_payload_->complete_parse(); + } else { + td::actor::create_actor("HttpPayloadReceiver", response_payload_, id_, dst_, local_id_, + adnl_, rldp_, is_tunnel()) + .release(); + } promise_.set_value(std::make_pair(std::move(response_), std::move(response_payload_))); stop(); @@ -543,6 +552,7 @@ class TcpToRldpRequestSender : public td::actor::Actor { void abort_query(td::Status error) { LOG(INFO) << "aborting http over rldp query: " << error; + promise_.set_error(std::move(error)); stop(); } @@ -808,9 +818,11 @@ class RldpToTcpRequestSender : public td::actor::Actor { } void got_result(std::pair, std::shared_ptr> R) { - td::actor::create_actor("HttpPayloadSender(R)", std::move(R.second), id_, local_id_, adnl_, - rldp_) - .release(); + if (R.first->need_payload()) { + td::actor::create_actor("HttpPayloadSender(R)", std::move(R.second), id_, local_id_, adnl_, + rldp_) + .release(); + } auto f = ton::serialize_tl_object(R.first->store_tl(), true); promise_.set_value(std::move(f)); stop(); @@ -818,7 +830,7 @@ class RldpToTcpRequestSender : public td::actor::Actor { void abort_query(td::Status error) { LOG(INFO) << "aborting http over rldp query: " << error; - promise_.set_error(std::move(error)); + promise_.set_result(create_error_response(request_->proto_version(), 502, "Bad Gateway")); stop(); } @@ -1174,13 +1186,22 @@ class RldpHttpProxy : public td::actor::Actor { td::Promise promise) { LOG(INFO) << "got HTTP request over rldp from " << src; TRY_RESULT_PROMISE(promise, f, ton::fetch_tl_object(data, true)); - TRY_RESULT_PROMISE(promise, request, ton::http::HttpRequest::create(f->method_, f->url_, f->http_version_)); - for (auto &x : f->headers_) { - ton::http::HttpHeader h{x->name_, x->value_}; - TRY_STATUS_PROMISE(promise, h.basic_check()); - request->add_header(std::move(h)); + std::unique_ptr request; + auto S = [&]() { + TRY_RESULT_ASSIGN(request, ton::http::HttpRequest::create(f->method_, f->url_, f->http_version_)); + for (auto &x : f->headers_) { + ton::http::HttpHeader h{x->name_, x->value_}; + TRY_STATUS(h.basic_check()); + request->add_header(std::move(h)); + } + TRY_STATUS(request->complete_parse_header()); + return td::Status::OK(); + }(); + if (S.is_error()) { + LOG(INFO) << "Failed to parse http request: " << S; + promise.set_result(create_error_response(f->http_version_, 400, "Bad Request")); + return; } - TRY_STATUS_PROMISE(promise, request->complete_parse_header()); auto host = request->host(); td::uint16 port = 80; if (host.empty()) { @@ -1212,7 +1233,7 @@ class RldpHttpProxy : public td::actor::Actor { port = (td::uint16)std::stoul(host.substr(p + 1)); } catch (const std::logic_error &) { port = 80; - promise.set_error(td::Status::Error(PSLICE() << "Invalid port '" << host.substr(p + 1) << "'")); + promise.set_result(create_error_response(f->http_version_, 400, "Bad Request")); return; } host = host.substr(0, p); @@ -1224,19 +1245,19 @@ class RldpHttpProxy : public td::actor::Actor { if (it == hosts_.end()) { it = hosts_.find("*"); if (it == hosts_.end()) { - promise.set_error(td::Status::Error(ton::ErrorCode::error, "unknown server name")); + promise.set_result(create_error_response(f->http_version_, 502, "Bad Gateway")); return; } } auto it2 = it->second.ports_.find(port); if (it2 == it->second.ports_.end()) { - promise.set_error(td::Status::Error(ton::ErrorCode::error, "unknown host:port")); + promise.set_result(create_error_response(f->http_version_, 502, "Bad Gateway")); return; } auto &server = it2->second; if (request->method() == "CONNECT") { LOG(INFO) << "starting HTTP tunnel over RLDP to " << server.remote_addr_; - start_tcp_tunnel(f->id_, src, dst, server.remote_addr_, std::move(promise)); + start_tcp_tunnel(f->id_, src, dst, f->http_version_, server.remote_addr_, std::move(promise)); return; } @@ -1244,22 +1265,31 @@ class RldpHttpProxy : public td::actor::Actor { server.http_remote_ = td::actor::create_actor("remote", server.remote_addr_); } - TRY_RESULT_PROMISE(promise, payload, request->create_empty_payload()); + auto payload = request->create_empty_payload(); + if (payload.is_error()) { + promise.set_result(create_error_response(f->http_version_, 502, "Bad Gateway")); + return; + } LOG(INFO) << "starting HTTP over RLDP request"; td::actor::create_actor("inboundreq", f->id_, dst, src, std::move(request), - std::move(payload), std::move(promise), adnl_.get(), rldp_.get(), + payload.move_as_ok(), std::move(promise), adnl_.get(), rldp_.get(), server.http_remote_.get()) .release(); } void start_tcp_tunnel(td::Bits256 id, ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort local_id, - td::IPAddress ip, td::Promise promise) { - TRY_RESULT_PROMISE(promise, fd, td::SocketFd::open(ip)); + std::string http_version, td::IPAddress ip, td::Promise promise) { + auto fd = td::SocketFd::open(ip); + if (fd.is_error()) { + promise.set_result(create_error_response(http_version, 502, "Bad Gateway")); + return; + } td::actor::create_actor(td::actor::ActorOptions().with_name("tunnel").with_poll(), id, src, local_id, - adnl_.get(), rldp_.get(), std::move(fd)).release(); + adnl_.get(), rldp_.get(), fd.move_as_ok()).release(); promise.set_result(ton::create_serialize_tl_object( - "HTTP/1.1", 200, "Connection Established", std::vector>())); + http_version, 200, "Connection Established", std::vector>(), + false)); } void add_adnl_addr(ton::adnl::AdnlNodeIdShort id) { diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 87dadd8a..bed1fc10 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -705,7 +705,7 @@ storage.queryPrefix id:int256 = Object; http.header name:string value:string = http.Header; http.payloadPart data:bytes trailer:(vector http.header) last:Bool = http.PayloadPart; -http.response http_version:string status_code:int reason:string headers:(vector http.header) = http.Response; +http.response http_version:string status_code:int reason:string headers:(vector http.header) no_payload:Bool = http.Response; ---functions--- diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 9fb0cc4371598866241af28d9d103d99f314bafd..f7279060e24450dfc7bc722bc2365da1243d3f4b 100644 GIT binary patch delta 115 zcmeybBAV)15_loaR{r4|?D=M|?iFo48v9ux9mnXEf; nF;9^*n+}I<+d+`%rb%DoIbqVQlP^{(PgeLe2V&fTPY;9v+5j(t From 440c42acdeef50b0aafe6f4de34f3c995be8eb17 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 30 Jun 2022 16:04:54 +0300 Subject: [PATCH 29/34] rldp-http-proxy: add DNS cache --- rldp-http-proxy/CMakeLists.txt | 2 +- rldp-http-proxy/DNSResolver.cpp | 104 +++++++++++++++++ rldp-http-proxy/DNSResolver.h | 49 ++++++++ rldp-http-proxy/TonlibClient.cpp | 72 ++++++++++++ rldp-http-proxy/TonlibClient.h | 47 ++++++++ rldp-http-proxy/rldp-http-proxy.cpp | 166 +++++++--------------------- 6 files changed, 312 insertions(+), 128 deletions(-) create mode 100644 rldp-http-proxy/DNSResolver.cpp create mode 100644 rldp-http-proxy/DNSResolver.h create mode 100644 rldp-http-proxy/TonlibClient.cpp create mode 100644 rldp-http-proxy/TonlibClient.h diff --git a/rldp-http-proxy/CMakeLists.txt b/rldp-http-proxy/CMakeLists.txt index 8fa4ac55..7ba66ccb 100644 --- a/rldp-http-proxy/CMakeLists.txt +++ b/rldp-http-proxy/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) -add_executable(rldp-http-proxy rldp-http-proxy.cpp) +add_executable(rldp-http-proxy rldp-http-proxy.cpp DNSResolver.h TonlibClient.h TonlibClient.cpp DNSResolver.cpp) target_include_directories(rldp-http-proxy PUBLIC $) target_link_libraries(rldp-http-proxy PRIVATE tonhttp rldp dht tonlib git) diff --git a/rldp-http-proxy/DNSResolver.cpp b/rldp-http-proxy/DNSResolver.cpp new file mode 100644 index 00000000..b6dc1bba --- /dev/null +++ b/rldp-http-proxy/DNSResolver.cpp @@ -0,0 +1,104 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. +*/ +#include "DNSResolver.h" +#include "td/utils/overloaded.h" + +static const double CACHE_TIMEOUT_HARD = 300.0; +static const double CACHE_TIMEOUT_SOFT = 270.0; + +DNSResolver::DNSResolver(td::actor::ActorId tonlib_client) : tonlib_client_(std::move(tonlib_client)) { +} + +void DNSResolver::start_up() { + auto obj = tonlib_api::make_object(); + auto P = td::PromiseCreator::lambda([](td::Result>) {}); + td::actor::send_closure(tonlib_client_, &TonlibClient::send_request, std::move(obj), std::move(P)); +} + +void DNSResolver::resolve(std::string host, td::Promise promise) { + auto it = cache_.find(host); + if (it != cache_.end()) { + const CacheEntry &entry = it->second; + double now = td::Time::now(); + if (now < entry.created_at_ + CACHE_TIMEOUT_HARD) { + promise.set_result(entry.id_); + promise.reset(); + if (now < entry.created_at_ + CACHE_TIMEOUT_SOFT) { + return; + } + } + } + + auto obj = tonlib_api::make_object(nullptr, host, 0, 16); + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), promise = std::move(promise), host = std::move(host)]( + td::Result> R) mutable { + if (R.is_error()) { + if (promise) { + promise.set_result(R.move_as_error()); + } + } else { + auto v = R.move_as_ok(); + auto obj = dynamic_cast(v.get()); + if (obj == nullptr) { + promise.set_result(td::Status::Error("invalid response from tonlib")); + return; + } + ton::adnl::AdnlNodeIdShort id; + td::uint32 cnt = 0; + for (auto &e : obj->entries_) { + tonlib_api::downcast_call(*e->entry_.get(), + td::overloaded( + [&](tonlib_api::dns_entryDataAdnlAddress &x) { + if (td::Random::fast(0, cnt) == 0) { + auto R = ton::adnl::AdnlNodeIdShort::parse(x.adnl_address_->adnl_address_); + if (R.is_ok()) { + id = R.move_as_ok(); + cnt++; + } + } + }, + [&](auto &x) {})); + } + if (cnt == 0) { + if (promise) { + promise.set_error(td::Status::Error("no DNS entries")); + } + } else { + td::actor::send_closure(SelfId, &DNSResolver::save_to_cache, std::move(host), id); + if (promise) { + promise.set_result(id); + } + } + } + }); + td::actor::send_closure(tonlib_client_, &TonlibClient::send_request, std::move(obj), std::move(P)); +} + +void DNSResolver::save_to_cache(std::string host, ton::adnl::AdnlNodeIdShort id) { + CacheEntry &entry = cache_[host]; + entry.id_ = id; + entry.created_at_ = td::Time::now(); +} diff --git a/rldp-http-proxy/DNSResolver.h b/rldp-http-proxy/DNSResolver.h new file mode 100644 index 00000000..f87b5e5a --- /dev/null +++ b/rldp-http-proxy/DNSResolver.h @@ -0,0 +1,49 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. +*/ +#pragma once +#include "td/actor/actor.h" +#include "TonlibClient.h" +#include "adnl/adnl.h" +#include "td/actor/PromiseFuture.h" + +class DNSResolver : public td::actor::Actor { + public: + explicit DNSResolver(td::actor::ActorId tonlib_client); + + void start_up() override; + void resolve(std::string host, td::Promise promise); + + private: + void save_to_cache(std::string host, ton::adnl::AdnlNodeIdShort id); + + td::actor::ActorId tonlib_client_; + + struct CacheEntry { + ton::adnl::AdnlNodeIdShort id_; + double created_at_; + }; + std::map cache_; +}; \ No newline at end of file diff --git a/rldp-http-proxy/TonlibClient.cpp b/rldp-http-proxy/TonlibClient.cpp new file mode 100644 index 00000000..a8e18b81 --- /dev/null +++ b/rldp-http-proxy/TonlibClient.cpp @@ -0,0 +1,72 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. +*/ +#include "TonlibClient.h" + +TonlibClient::TonlibClient(ton::tl_object_ptr options) : options_(std::move(options)) { +} + +void TonlibClient::start_up() { + class Cb : public tonlib::TonlibCallback { + public: + explicit Cb(td::actor::ActorId self_id) : self_id_(self_id) { + } + void on_result(std::uint64_t id, tonlib_api::object_ptr result) override { + td::actor::send_closure(self_id_, &TonlibClient::receive_request_result, id, std::move(result)); + } + void on_error(std::uint64_t id, tonlib_api::object_ptr error) override { + td::actor::send_closure(self_id_, &TonlibClient::receive_request_result, id, + td::Status::Error(error->code_, std::move(error->message_))); + } + + private: + td::actor::ActorId self_id_; + }; + + tonlib_client_ = td::actor::create_actor("tonlibclient", td::make_unique(actor_id(this))); + auto init = tonlib_api::make_object(std::move(options_)); + auto P = td::PromiseCreator::lambda([](td::Result> R) mutable { + R.ensure(); + }); + send_request(std::move(init), std::move(P)); +} + +void TonlibClient::send_request(tonlib_api::object_ptr obj, + td::Promise> promise) { + auto id = next_request_id_++; + CHECK(requests_.emplace(id, std::move(promise)).second); + td::actor::send_closure(tonlib_client_, &tonlib::TonlibClient::request, id, std::move(obj)); +} + +void TonlibClient::receive_request_result(td::uint64 id, td::Result> R) { + if (id == 0) { + return; + } + auto it = requests_.find(id); + CHECK(it != requests_.end()); + auto promise = std::move(it->second); + requests_.erase(it); + promise.set_result(std::move(R)); +} \ No newline at end of file diff --git a/rldp-http-proxy/TonlibClient.h b/rldp-http-proxy/TonlibClient.h new file mode 100644 index 00000000..0b75c6c0 --- /dev/null +++ b/rldp-http-proxy/TonlibClient.h @@ -0,0 +1,47 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. +*/ +#pragma once +#include "td/actor/actor.h" +#include "auto/tl/tonlib_api.hpp" +#include "tonlib/tonlib/TonlibClient.h" + +class TonlibClient : public td::actor::Actor { + public: + explicit TonlibClient(ton::tl_object_ptr options); + + void start_up() override; + + void send_request(tonlib_api::object_ptr obj, + td::Promise> promise); + + private: + void receive_request_result(td::uint64 id, td::Result> R); + + ton::tl_object_ptr options_; + td::actor::ActorOwn tonlib_client_; + std::map>> requests_; + td::uint64 next_request_id_{1}; +}; diff --git a/rldp-http-proxy/rldp-http-proxy.cpp b/rldp-http-proxy/rldp-http-proxy.cpp index 98088713..a5faa002 100644 --- a/rldp-http-proxy/rldp-http-proxy.cpp +++ b/rldp-http-proxy/rldp-http-proxy.cpp @@ -55,6 +55,9 @@ #include "td/utils/BufferedFd.h" #include "common/delay.h" +#include "TonlibClient.h" +#include "DNSResolver.h" + #if TD_DARWIN || TD_LINUX #include #endif @@ -126,7 +129,7 @@ class HttpRemote : public td::actor::Actor { td::actor::ActorOwn client_; }; -td::BufferSlice create_error_response(const std::string& proto_version, int code, const std::string& reason) { +td::BufferSlice create_error_response(const std::string &proto_version, int code, const std::string &reason) { return ton::create_serialize_tl_object( proto_version, code, reason, std::vector>(), true); } @@ -328,8 +331,8 @@ class HttpRldpPayloadSender : public td::actor::Actor { size_t watermark_; }; - payload_->add_callback(std::make_unique(actor_id(this), - is_tunnel_ ? 1 : ton::http::HttpRequest::low_watermark())); + payload_->add_callback( + std::make_unique(actor_id(this), is_tunnel_ ? 1 : ton::http::HttpRequest::low_watermark())); alarm_timestamp() = td::Timestamp::in(is_tunnel_ ? 60.0 : 10.0); } @@ -352,9 +355,11 @@ class HttpRldpPayloadSender : public td::actor::Actor { answer_query(); } else if (!active_timer_) { active_timer_ = true; - ton::delay_action([SelfId = actor_id(this)]() { - td::actor::send_closure(SelfId, &HttpRldpPayloadSender::try_answer_query, true); - }, td::Timestamp::in(0.001)); + ton::delay_action( + [SelfId = actor_id(this)]() { + td::actor::send_closure(SelfId, &HttpRldpPayloadSender::try_answer_query, true); + }, + td::Timestamp::in(0.001)); } } @@ -437,7 +442,6 @@ class HttpRldpPayloadSender : public td::actor::Actor { td::Bits256 id_; - bool sent_ = false; td::int32 seqno_ = 0; ton::adnl::AdnlNodeIdShort local_id_; @@ -458,7 +462,7 @@ class TcpToRldpRequestSender : public td::actor::Actor { std::shared_ptr request_payload, td::Promise, std::shared_ptr>> promise, td::actor::ActorId adnl, td::actor::ActorId dht, - td::actor::ActorId rldp, td::actor::ActorId proxy) + td::actor::ActorId rldp, td::actor::ActorId dns_resolver) : local_id_(local_id) , host_(std::move(host)) , request_(std::move(request)) @@ -467,7 +471,7 @@ class TcpToRldpRequestSender : public td::actor::Actor { , adnl_(adnl) , dht_(dht) , rldp_(rldp) - , proxy_(proxy) { + , dns_resolver_(dns_resolver) { } void start_up() override { resolve(); @@ -503,8 +507,8 @@ class TcpToRldpRequestSender : public td::actor::Actor { return; } auto f = F.move_as_ok(); - auto R = ton::http::HttpResponse::create( - f->http_version_, f->status_code_, f->reason_, f->no_payload_, true, is_tunnel() && f->status_code_ == 200); + auto R = ton::http::HttpResponse::create(f->http_version_, f->status_code_, f->reason_, f->no_payload_, true, + is_tunnel() && f->status_code_ == 200); if (R.is_error()) { abort_query(R.move_as_error()); return; @@ -575,7 +579,7 @@ class TcpToRldpRequestSender : public td::actor::Actor { td::actor::ActorId adnl_; td::actor::ActorId dht_; td::actor::ActorId rldp_; - td::actor::ActorId proxy_; + td::actor::ActorId dns_resolver_; std::unique_ptr response_; std::shared_ptr response_payload_; @@ -718,9 +722,11 @@ class RldpTcpTunnel : public td::actor::Actor, private td::ObserverBase { if (!from_timer && !close_ && !allow_empty && input.size() < ton::http::HttpRequest::low_watermark()) { if (!active_timer_) { active_timer_ = true; - ton::delay_action([SelfId = actor_id(this)]() { - td::actor::send_closure(SelfId, &RldpTcpTunnel::answer_query, false, true); - }, td::Timestamp::in(0.001)); + ton::delay_action( + [SelfId = actor_id(this)]() { + td::actor::send_closure(SelfId, &RldpTcpTunnel::answer_query, false, true); + }, + td::Timestamp::in(0.001)); } return; } @@ -853,8 +859,7 @@ class RldpToTcpRequestSender : public td::actor::Actor { class RldpHttpProxy : public td::actor::Actor { public: - RldpHttpProxy() { - } + RldpHttpProxy() = default; void set_port(td::uint16 port) { if (port_) { @@ -881,27 +886,6 @@ class RldpHttpProxy : public td::actor::Actor { hosts_[host].ports_[port].remote_addr_ = remote; } - void receive_request_result(td::uint64 id, td::Result> R) { - if (id == 0) { - return; - } - auto it = tonlib_requests_.find(id); - CHECK(it != tonlib_requests_.end()); - auto promise = std::move(it->second); - tonlib_requests_.erase(it); - - promise.set_result(std::move(R)); - } - - void send_tonlib_request(tonlib_api::object_ptr obj, - td::Promise> promise) { - auto id = next_tonlib_requests_id_++; - - CHECK(tonlib_requests_.emplace(id, std::move(promise)).second); - - td::actor::send_closure(tonlib_client_, &tonlib::TonlibClient::request, id, std::move(obj)); - } - td::Status load_global_config() { TRY_RESULT_PREFIX(conf_data, td::read_file(global_config_), "failed to read: "); TRY_RESULT_PREFIX(conf_json, td::json_decode(conf_data.as_slice()), "failed to parse json: "); @@ -916,24 +900,6 @@ class RldpHttpProxy : public td::actor::Actor { TRY_RESULT_PREFIX(dht, ton::dht::Dht::create_global_config(std::move(conf.dht_)), "bad [dht] section: "); dht_config_ = std::move(dht); - class Cb : public tonlib::TonlibCallback { - public: - Cb(td::actor::ActorId self_id) : self_id_(self_id) { - } - void on_result(std::uint64_t id, tonlib_api::object_ptr result) override { - td::actor::send_closure(self_id_, &RldpHttpProxy::receive_request_result, id, std::move(result)); - } - void on_error(std::uint64_t id, tonlib_api::object_ptr error) override { - td::actor::send_closure(self_id_, &RldpHttpProxy::receive_request_result, id, - td::Status::Error(error->code_, std::move(error->message_))); - } - - private: - td::actor::ActorId self_id_; - }; - - tonlib_client_ = td::actor::create_actor("tonlibclient", td::make_unique(actor_id(this))); - return td::Status::OK(); } @@ -972,7 +938,7 @@ class RldpHttpProxy : public td::actor::Actor { { auto S = load_global_config(); if (S.is_error()) { - LOG(INFO) << S; + LOG(ERROR) << S; std::_Exit(2); } } @@ -999,19 +965,15 @@ class RldpHttpProxy : public td::actor::Actor { }); td::actor::send_closure(keyring_, &ton::keyring::Keyring::get_public_key, x.pubkey_hash(), std::move(Q)); } - auto Q = td::PromiseCreator::lambda( - [promise = ig.get_promise()](td::Result> R) mutable { - R.ensure(); - promise.set_value(td::Unit()); - }); auto conf_dataR = td::read_file(global_config_); conf_dataR.ensure(); - auto req = tonlib_api::make_object(tonlib_api::make_object( + auto tonlib_options = tonlib_api::make_object( tonlib_api::make_object(conf_dataR.move_as_ok().as_slice().str(), "", false, false), - tonlib_api::make_object())); - send_tonlib_request(std::move(req), std::move(Q)); + tonlib_api::make_object()); + tonlib_client_ = td::actor::create_actor("tonlibclient", std::move(tonlib_options)); + dns_resolver_ = td::actor::create_actor("dnsresolver", tonlib_client_.get()); } void run_cont() { @@ -1178,7 +1140,7 @@ class RldpHttpProxy : public td::actor::Actor { td::actor::create_actor("outboundreq", local_id_, host, std::move(request), std::move(payload), std::move(promise), adnl_.get(), dht_.get(), - rldp_.get(), actor_id(this)) + rldp_.get(), dns_resolver_.get()) .release(); } @@ -1326,7 +1288,6 @@ class RldpHttpProxy : public td::actor::Actor { ton::adnl::AdnlNodeIdShort dht_id_; td::actor::ActorOwn server_; - std::map dns_; std::map hosts_; td::actor::ActorOwn keyring_; @@ -1340,9 +1301,8 @@ class RldpHttpProxy : public td::actor::Actor { std::string db_root_ = "."; bool proxy_all_ = false; - td::actor::ActorOwn tonlib_client_; - std::map>> tonlib_requests_; - td::uint64 next_tonlib_requests_id_{1}; + td::actor::ActorOwn tonlib_client_; + td::actor::ActorOwn dns_resolver_; }; void TcpToRldpRequestSender::resolve() { @@ -1357,63 +1317,15 @@ void TcpToRldpRequestSender::resolve() { resolved(R.move_as_ok()); return; } - if (false) { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &TcpToRldpRequestSender::abort_query, R.move_as_error()); - return; - } - auto value = R.move_as_ok(); - if (value.value().size() != 32) { - td::actor::send_closure(SelfId, &TcpToRldpRequestSender::abort_query, td::Status::Error("bad value in dht")); - return; - } - - ton::PublicKeyHash h{value.value().as_slice()}; - td::actor::send_closure(SelfId, &TcpToRldpRequestSender::resolved, ton::adnl::AdnlNodeIdShort{h}); - }); - - ton::PublicKey key = ton::pubkeys::Unenc{"http." + host_}; - ton::dht::DhtKey dht_key{key.compute_short_id(), "http." + host_, 0}; - td::actor::send_closure(dht_, &ton::dht::Dht::get_value, std::move(dht_key), std::move(P)); - } else { - td::Bits256 category = td::sha256_bits256(td::Slice("site", 4)); - auto obj = tonlib_api::make_object(nullptr, host_, category, 16); - - auto P = - td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &TcpToRldpRequestSender::abort_query, - R.move_as_error_prefix("failed to resolve: ")); - } else { - auto v = R.move_as_ok(); - auto obj = static_cast(v.get()); - ton::adnl::AdnlNodeIdShort id; - td::uint32 cnt = 0; - for (auto &e : obj->entries_) { - tonlib_api::downcast_call( - *e->entry_.get(), td::overloaded( - [&](tonlib_api::dns_entryDataAdnlAddress &x) { - if (td::Random::fast(0, cnt) == 0) { - auto R = ton::adnl::AdnlNodeIdShort::parse(x.adnl_address_->adnl_address_); - if (R.is_ok()) { - id = R.move_as_ok(); - cnt++; - } - } - }, - [&](auto &x) {})); - } - if (cnt == 0) { - td::actor::send_closure(SelfId, &TcpToRldpRequestSender::abort_query, - td::Status::Error(ton::ErrorCode::notready, "failed to resolve")); - } else { - td::actor::send_closure(SelfId, &TcpToRldpRequestSender::resolved, id); - } - } - }); - td::actor::send_closure(proxy_, &RldpHttpProxy::send_tonlib_request, std::move(obj), std::move(P)); - } + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &TcpToRldpRequestSender::abort_query, + R.move_as_error_prefix("failed to resolve: ")); + } else { + td::actor::send_closure(SelfId, &TcpToRldpRequestSender::resolved, R.move_as_ok()); + } + }); + td::actor::send_closure(dns_resolver_, &DNSResolver::resolve, host_, std::move(P)); } int main(int argc, char *argv[]) { From e39e2d884fcaecbe7ef285ccbc99327f2fdda8db Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 23 Aug 2022 22:39:41 +0300 Subject: [PATCH 30/34] Use new dns interface --- rldp-http-proxy/DNSResolver.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rldp-http-proxy/DNSResolver.cpp b/rldp-http-proxy/DNSResolver.cpp index b6dc1bba..a7519742 100644 --- a/rldp-http-proxy/DNSResolver.cpp +++ b/rldp-http-proxy/DNSResolver.cpp @@ -52,7 +52,8 @@ void DNSResolver::resolve(std::string host, td::Promise(nullptr, host, 0, 16); + td::Bits256 category = td::sha256_bits256(td::Slice("site", 4)); + auto obj = tonlib_api::make_object(nullptr, host, category, 16); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), promise = std::move(promise), host = std::move(host)]( td::Result> R) mutable { if (R.is_error()) { From 05bbd9e0ee980ebd9acb712020072c2bcbd60257 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 29 Sep 2022 18:41:47 +0300 Subject: [PATCH 31/34] Remove Content-Length limit --- http/http.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/http/http.cpp b/http/http.cpp index c99a5ec4..63819957 100644 --- a/http/http.cpp +++ b/http/http.cpp @@ -193,9 +193,6 @@ td::Status HttpRequest::add_header(HttpHeader header) { if (found_transfer_encoding_ || found_content_length_) { return td::Status::Error("duplicate Content-Length/Transfer-Encoding"); } - if (len > HttpRequest::max_payload_size()) { - return td::Status::Error("too big Content-Length"); - } content_length_ = len; found_content_length_ = true; } else if (lc_name == "transfer-encoding") { @@ -809,9 +806,6 @@ td::Status HttpResponse::add_header(HttpHeader header) { if (found_transfer_encoding_ || found_content_length_) { return td::Status::Error("duplicate Content-Length/Transfer-Encoding"); } - if (len > HttpRequest::max_payload_size()) { - return td::Status::Error("too big Content-Length"); - } content_length_ = len; found_content_length_ = true; } else if (lc_name == "transfer-encoding") { From a6e38b7cdeda78a87e09fd0ccaf9c860d34eafd8 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 29 Sep 2022 23:29:42 +0300 Subject: [PATCH 32/34] Fix writing payload to http --- http/http.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http/http.cpp b/http/http.cpp index 63819957..cefe1a47 100644 --- a/http/http.cpp +++ b/http/http.cpp @@ -514,7 +514,7 @@ bool HttpPayload::store_http(td::ChainBufferWriter &output, size_t max_size, Htt wrote = true; } } - if (chunks_.size() != 0) { + if (chunks_.size() != 0 || !parse_completed()) { return wrote; } if (!written_zero_chunk_) { From e05af0563515270731b70004a0e23614bd5c9cb5 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 3 Oct 2022 09:14:50 +0300 Subject: [PATCH 33/34] Check fec type in incoming messages in RLDP --- fec/fec.cpp | 48 ++++++++++++++++++++----------- overlay/overlay-fec-broadcast.cpp | 2 +- rldp/rldp-peer.cpp | 4 +++ 3 files changed, 36 insertions(+), 18 deletions(-) diff --git a/fec/fec.cpp b/fec/fec.cpp index b194a173..102df038 100644 --- a/fec/fec.cpp +++ b/fec/fec.cpp @@ -19,6 +19,7 @@ #include "fec.h" #include "td/utils/overloaded.h" #include "auto/tl/ton_api.hpp" +#include "td/utils/misc.h" namespace ton { @@ -98,24 +99,37 @@ td::uint32 FecType::symbol_size() const { } td::Result FecType::create(tl_object_ptr obj) { + td::int32 data_size_int, symbol_size_int, symbols_count_int; + ton_api::downcast_call(*obj, td::overloaded([&](const auto &obj) { + data_size_int = obj.data_size_; + symbol_size_int = obj.symbol_size_; + symbols_count_int = obj.symbols_count_; + })); + TRY_RESULT(data_size, td::narrow_cast_safe(data_size_int)); + TRY_RESULT(symbol_size, td::narrow_cast_safe(symbol_size_int)); + TRY_RESULT(symbols_count, td::narrow_cast_safe(symbols_count_int)); + + if (symbol_size == 0) { + return td::Status::Error("invalid fec type: symbol_size is 0"); + } + if (symbol_size > 1 << 11) { + return td::Status::Error("invalid fec type: symbol_size is too big"); + } + if (symbols_count != (data_size + symbol_size - 1) / symbol_size) { + return td::Status::Error("invalid fec type: wrong symbols_count"); + } FecType T; - ton_api::downcast_call( - *obj.get(), td::overloaded( - [&](const ton_api::fec_raptorQ &obj) { - T.type_ = td::fec::RaptorQEncoder::Parameters{static_cast(obj.data_size_), - static_cast(obj.symbol_size_), - static_cast(obj.symbols_count_)}; - }, - [&](const ton_api::fec_roundRobin &obj) { - T.type_ = td::fec::RoundRobinEncoder::Parameters{static_cast(obj.data_size_), - static_cast(obj.symbol_size_), - static_cast(obj.symbols_count_)}; - }, - [&](const ton_api::fec_online &obj) { - T.type_ = td::fec::OnlineEncoder::Parameters{static_cast(obj.data_size_), - static_cast(obj.symbol_size_), - static_cast(obj.symbols_count_)}; - })); + ton_api::downcast_call(*obj, + td::overloaded( + [&](const ton_api::fec_raptorQ &obj) { + T.type_ = td::fec::RaptorQEncoder::Parameters{data_size, symbol_size, symbols_count}; + }, + [&](const ton_api::fec_roundRobin &obj) { + T.type_ = td::fec::RoundRobinEncoder::Parameters{data_size, symbol_size, symbols_count}; + }, + [&](const ton_api::fec_online &obj) { + T.type_ = td::fec::OnlineEncoder::Parameters{data_size, symbol_size, symbols_count}; + })); return T; } diff --git a/overlay/overlay-fec-broadcast.cpp b/overlay/overlay-fec-broadcast.cpp index 7ff08309..aed5248b 100644 --- a/overlay/overlay-fec-broadcast.cpp +++ b/overlay/overlay-fec-broadcast.cpp @@ -28,8 +28,8 @@ td::Result> BroadcastFec::create(Overlay::Broadcas Overlay::BroadcastDataHash data_hash, td::uint32 flags, td::uint32 date, fec::FecType fec_type) { auto F = std::make_unique(hash, std::move(src), data_hash, flags, date, std::move(fec_type)); - TRY_STATUS(F->init_fec_type()); TRY_STATUS(F->run_checks()); + TRY_STATUS(F->init_fec_type()); return std::move(F); } diff --git a/rldp/rldp-peer.cpp b/rldp/rldp-peer.cpp index 6192d070..c599e949 100644 --- a/rldp/rldp-peer.cpp +++ b/rldp/rldp-peer.cpp @@ -125,6 +125,10 @@ void RldpTransferReceiverImpl::receive_part(fec::FecType fec_type, td::uint32 pa } if (!decoder_) { + if (offset_ + fec_type.size() > total_size_) { + VLOG(RLDP_NOTICE) << "failed to create decoder: data size in fec type is too big"; + return; + } auto D = fec_type.create_decoder(); if (D.is_error()) { VLOG(RLDP_WARNING) << "failed to create decoder: " << D.move_as_error(); From 495327099cfee92fefe3f53198a8cb9411355a62 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 3 Oct 2022 09:15:41 +0300 Subject: [PATCH 34/34] Check level of c4 and c5 --- crypto/vm/vm.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/vm/vm.cpp b/crypto/vm/vm.cpp index fb2f4d1e..6668c6d5 100644 --- a/crypto/vm/vm.cpp +++ b/crypto/vm/vm.cpp @@ -503,7 +503,7 @@ int VmState::run() { bool VmState::try_commit() { if (cr.d[0].not_null() && cr.d[1].not_null() && cr.d[0]->get_depth() <= max_data_depth && - cr.d[1]->get_depth() <= max_data_depth) { + cr.d[1]->get_depth() <= max_data_depth && cr.d[0]->get_level() == 0 && cr.d[1]->get_level() == 0) { cstate.c4 = cr.d[0]; cstate.c5 = cr.d[1]; cstate.committed = true;