diff --git a/CMake/FindLZ4.cmake b/CMake/FindLZ4.cmake new file mode 100644 index 00000000..e13eca55 --- /dev/null +++ b/CMake/FindLZ4.cmake @@ -0,0 +1,39 @@ +############################################################################### +# Find LZ4 +# +# This sets the following variables: +# LZ4_FOUND - True if LZ4 was found. +# LZ4_INCLUDE_DIRS - Directories containing the LZ4 include files. +# LZ4_LIBRARIES - Libraries needed to use LZ4. +# LZ4_LIBRARY - Library needed to use LZ4. +# LZ4_LIBRARY_DIRS - Library needed to use LZ4. + +find_package(PkgConfig REQUIRED) + +# If found, LZ$_* variables will be defined +pkg_check_modules(LZ4 REQUIRED liblz4) + +if(NOT LZ4_FOUND) + find_path(LZ4_INCLUDE_DIR lz4.h + HINTS "${LZ4_ROOT}" "$ENV{LZ4_ROOT}" + PATHS "$ENV{PROGRAMFILES}/lz4" "$ENV{PROGRAMW6432}/lz4" + PATH_SUFFIXES include) + + find_library(LZ4_LIBRARY + NAMES lz4 lz4_static + HINTS "${LZ4_ROOT}" "$ENV{LZ4_ROOT}" + PATHS "$ENV{PROGRAMFILES}/lz4" "$ENV{PROGRAMW6432}/lz4" + PATH_SUFFIXES lib) + + if(LZ4_LIBRARY) + set(LZ4_LIBRARIES ${LZ4_LIBRARY}) + get_filename_component(LZ4_LIBRARY_DIRS ${LZ4_LIBRARY} DIRECTORY) + endif() +else() + find_library(LZ4_LIBRARY + NAMES lz4 lz4_static + PATHS ${LZ4_LIBRARY_DIRS} + NO_DEFAULT_PATH) +endif() + +mark_as_advanced(LZ4_LIBRARY LZ4_INCLUDE_DIRS LZ4_LIBRARY_DIRS LZ4_LIBRARIES) \ No newline at end of file diff --git a/tdutils/CMakeLists.txt b/tdutils/CMakeLists.txt index f1e4b1ea..7eebc910 100644 --- a/tdutils/CMakeLists.txt +++ b/tdutils/CMakeLists.txt @@ -15,6 +15,7 @@ if (NOT DEFINED CMAKE_INSTALL_LIBDIR) endif() find_package(PkgConfig REQUIRED) +find_package(LZ4) if (NOT ZLIB_FOUND) pkg_check_modules(ZLIB zlib) endif() @@ -280,6 +281,15 @@ if (TDUTILS_MIME_TYPE) ) endif() +if (LZ4_FOUND) + set(TD_HAVE_LZ4 1) + set(TDUTILS_SOURCE + ${TDUTILS_SOURCE} + td/utils/lz4.cpp + td/utils/lz4.h + ) +endif() + set(TDUTILS_TEST_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/test/buffer.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test/ConcurrentHashMap.cpp @@ -338,6 +348,11 @@ endif() if (CRC32C_FOUND) target_link_libraries(tdutils PRIVATE crc32c) endif() + +if (LZ4_FOUND) + target_link_libraries(tdutils PRIVATE ${LZ4_LIBRARIES}) +endif() + if (ABSL_FOUND) target_link_libraries_system(tdutils absl::flat_hash_map absl::flat_hash_set absl::hash) endif() diff --git a/tdutils/td/utils/config.h.in b/tdutils/td/utils/config.h.in index f8b89aeb..3f4e1bf2 100644 --- a/tdutils/td/utils/config.h.in +++ b/tdutils/td/utils/config.h.in @@ -3,6 +3,7 @@ #cmakedefine01 TD_HAVE_OPENSSL #cmakedefine01 TD_HAVE_ZLIB #cmakedefine01 TD_HAVE_CRC32C +#cmakedefine01 TD_HAVE_LZ4 #cmakedefine01 TD_HAVE_COROUTINES #cmakedefine01 TD_HAVE_ABSL #cmakedefine01 TD_FD_DEBUG diff --git a/tdutils/td/utils/lz4.cpp b/tdutils/td/utils/lz4.cpp new file mode 100644 index 00000000..ebf456aa --- /dev/null +++ b/tdutils/td/utils/lz4.cpp @@ -0,0 +1,48 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "td/utils/buffer.h" +#include "td/utils/misc.h" +#include + +namespace td { + +td::BufferSlice lz4_compress(td::Slice data) { + int size = narrow_cast(data.size()); + int buf_size = LZ4_compressBound(size); + td::BufferSlice compressed(buf_size); + int compressed_size = LZ4_compress_default(data.data(), compressed.data(), size, buf_size); + CHECK(compressed_size > 0); + return td::BufferSlice{compressed.as_slice().substr(0, compressed_size)}; +} + +td::Result lz4_decompress(td::Slice data, int max_decompressed_size) { + TRY_RESULT(size, narrow_cast_safe(data.size())); + if (max_decompressed_size < 0) { + return td::Status::Error("invalid max_decompressed_size"); + } + td::BufferSlice decompressed(max_decompressed_size); + int result = LZ4_decompress_safe(data.data(), decompressed.data(), size, max_decompressed_size); + if (result < 0) { + return td::Status::Error(PSTRING() << "lz4 decompression failed, error code: " << result); + } + if (result == max_decompressed_size) { + return decompressed; + } + return td::BufferSlice{decompressed.as_slice().substr(0, result)}; +} + +} // namespace td diff --git a/tdutils/td/utils/lz4.h b/tdutils/td/utils/lz4.h new file mode 100644 index 00000000..fbbc470f --- /dev/null +++ b/tdutils/td/utils/lz4.h @@ -0,0 +1,27 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "td/utils/buffer.h" +#include "td/utils/Status.h" + +namespace td { + +td::BufferSlice lz4_compress(td::Slice data); +td::Result lz4_decompress(td::Slice data, int max_decompressed_size); + +} // namespace td diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 0f94e7f1..159a0c88 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -309,6 +309,7 @@ validatorSession.candidateId src:int256 root_hash:int256 file_hash:int256 collat validatorSession.blockUpdate ts:long actions:(vector validatorSession.round.Message) state:int = validatorSession.BlockUpdate; validatorSession.candidate src:int256 round:int root_hash:int256 data:bytes collated_data:bytes = validatorSession.Candidate; +validatorSession.compressedCandidate flags:# src:int256 round:int root_hash:int256 decompressed_size:int data:bytes = validatorSession.Candidate; validatorSession.config catchain_idle_timeout:double catchain_max_deps:int round_candidates:int next_candidate_delay:double round_attempt_duration:int max_round_attempts:int max_block_size:int max_collated_data_size:int = validatorSession.Config; @@ -385,9 +386,13 @@ tonNode.externalMessage data:bytes = tonNode.ExternalMessage; tonNode.newShardBlock block:tonNode.blockIdExt cc_seqno:int data:bytes = tonNode.NewShardBlock; +tonNode.blockBroadcastCompressed.data signatures:(vector tonNode.blockSignature) proof_data:bytes = tonNode.blockBroadcaseCompressed.Data; + tonNode.blockBroadcast id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int signatures:(vector tonNode.blockSignature) proof:bytes data:bytes = tonNode.Broadcast; +tonNode.blockBroadcastCompressed id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int + flags:# compressed:bytes = tonNode.Broadcast; tonNode.ihrMessageBroadcast message:tonNode.ihrMessage = tonNode.Broadcast; tonNode.externalMessageBroadcast message:tonNode.externalMessage = tonNode.Broadcast; tonNode.newShardBlockBroadcast block:tonNode.newShardBlock = tonNode.Broadcast; @@ -401,9 +406,8 @@ tonNode.keyBlocks blocks:(vector tonNode.blockIdExt) incomplete:Bool error:Bool ton.blockId root_cell_hash:int256 file_hash:int256 = ton.BlockId; ton.blockIdApprove root_cell_hash:int256 file_hash:int256 = ton.BlockId; -tonNode.dataList data:(vector bytes) = tonNode.DataList; - tonNode.dataFull id:tonNode.blockIdExt proof:bytes block:bytes is_link:Bool = tonNode.DataFull; +tonNode.dataFullCompressed id:tonNode.blockIdExt flags:# compressed:bytes is_link:Bool = tonNode.DataFull; tonNode.dataFullEmpty = tonNode.DataFull; tonNode.capabilities version:int capabilities:long = tonNode.Capabilities; @@ -430,18 +434,13 @@ tonNode.getNextKeyBlockIds block:tonNode.blockIdExt max_size:int = tonNode.KeyBl tonNode.downloadNextBlockFull prev_block:tonNode.blockIdExt = tonNode.DataFull; tonNode.downloadBlockFull block:tonNode.blockIdExt = tonNode.DataFull; tonNode.downloadBlock block:tonNode.blockIdExt = tonNode.Data; -tonNode.downloadBlocks blocks:(vector tonNode.blockIdExt) = tonNode.DataList; tonNode.downloadPersistentState block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt = tonNode.Data; tonNode.downloadPersistentStateSlice block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt offset:long max_size:long = tonNode.Data; tonNode.downloadZeroState block:tonNode.blockIdExt = tonNode.Data; tonNode.downloadBlockProof block:tonNode.blockIdExt = tonNode.Data; tonNode.downloadKeyBlockProof block:tonNode.blockIdExt = tonNode.Data; -tonNode.downloadBlockProofs blocks:(vector tonNode.blockIdExt) = tonNode.DataList; -tonNode.downloadKeyBlockProofs blocks:(vector tonNode.blockIdExt) = tonNode.DataList; tonNode.downloadBlockProofLink block:tonNode.blockIdExt = tonNode.Data; tonNode.downloadKeyBlockProofLink block:tonNode.blockIdExt = tonNode.Data; -tonNode.downloadBlockProofLinks blocks:(vector tonNode.blockIdExt) = tonNode.DataList; -tonNode.downloadKeyBlockProofLinks blocks:(vector tonNode.blockIdExt) = tonNode.DataList; tonNode.getArchiveInfo masterchain_seqno:int = tonNode.ArchiveInfo; tonNode.getArchiveSlice archive_id:long offset:long max_size:int = tonNode.Data; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index e71fb0f7..18df7794 100644 Binary files a/tl/generate/scheme/ton_api.tlo and b/tl/generate/scheme/ton_api.tlo differ diff --git a/validator-session/CMakeLists.txt b/validator-session/CMakeLists.txt index c769f4d8..41911a8b 100644 --- a/validator-session/CMakeLists.txt +++ b/validator-session/CMakeLists.txt @@ -5,12 +5,14 @@ if (NOT OPENSSL_FOUND) endif() set(VALIDATOR_SESSION_SOURCE + candidate-serializer.cpp persistent-vector.cpp validator-session-description.cpp validator-session-state.cpp validator-session.cpp validator-session-round-attempt-state.cpp + candidate-serializer.h persistent-vector.h validator-session-description.h validator-session-description.hpp diff --git a/validator-session/candidate-serializer.cpp b/validator-session/candidate-serializer.cpp new file mode 100644 index 00000000..c220b4a9 --- /dev/null +++ b/validator-session/candidate-serializer.cpp @@ -0,0 +1,76 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "candidate-serializer.h" +#include "tl-utils/tl-utils.hpp" +#include "vm/boc.h" +#include "td/utils/lz4.h" +#include "validator-session-types.h" + +namespace ton::validatorsession { + +td::Result serialize_candidate(const tl_object_ptr &block, + bool compression_enabled) { + if (!compression_enabled) { + return serialize_tl_object(block, true); + } + vm::BagOfCells boc1, boc2; + TRY_STATUS(boc1.deserialize(block->data_)); + if (boc1.get_root_count() != 1) { + return td::Status::Error("block candidate should have exactly one root"); + } + std::vector> roots = {boc1.get_root_cell()}; + TRY_STATUS(boc2.deserialize(block->collated_data_)); + for (int i = 0; i < boc2.get_root_count(); ++i) { + roots.push_back(boc2.get_root_cell(i)); + } + TRY_RESULT(data, vm::std_boc_serialize_multi(std::move(roots), 2)); + td::BufferSlice compressed = td::lz4_compress(data); + LOG(VALIDATOR_SESSION_DEBUG) << "Compressing block candidate: " << block->data_.size() + block->collated_data_.size() + << " -> " << compressed.size(); + return create_serialize_tl_object( + 0, block->src_, block->round_, block->root_hash_, (int)data.size(), std::move(compressed)); +} + +td::Result> deserialize_candidate(td::Slice data, + bool compression_enabled, + int max_decompressed_data_size) { + if (!compression_enabled) { + return fetch_tl_object(data, true); + } + TRY_RESULT(f, fetch_tl_object(data, true)); + if (f->decompressed_size_ > max_decompressed_data_size) { + return td::Status::Error("decompressed size is too big"); + } + TRY_RESULT(decompressed, td::lz4_decompress(f->data_, f->decompressed_size_)); + if (decompressed.size() != (size_t)f->decompressed_size_) { + return td::Status::Error("decompressed size mismatch"); + } + TRY_RESULT(roots, vm::std_boc_deserialize_multi(decompressed)); + if (roots.empty()) { + return td::Status::Error("boc is empty"); + } + TRY_RESULT(block_data, vm::std_boc_serialize(roots[0], 31)); + roots.erase(roots.begin()); + TRY_RESULT(collated_data, vm::std_boc_serialize_multi(std::move(roots), 31)); + LOG(VALIDATOR_SESSION_DEBUG) << "Decompressing block candidate: " << f->data_.size() << " -> " + << block_data.size() + collated_data.size(); + return create_tl_object(f->src_, f->round_, f->root_hash_, std::move(block_data), + std::move(collated_data)); +} + +} // namespace ton::validatorsession diff --git a/validator-session/candidate-serializer.h b/validator-session/candidate-serializer.h new file mode 100644 index 00000000..030a412c --- /dev/null +++ b/validator-session/candidate-serializer.h @@ -0,0 +1,29 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "ton/ton-types.h" +#include "auto/tl/ton_api.h" + +namespace ton::validatorsession { + +td::Result serialize_candidate(const tl_object_ptr &block, + bool compression_enabled); +td::Result> deserialize_candidate(td::Slice data, + bool compression_enabled, + int max_decompressed_data_size); + +} // namespace ton::validatorsession diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index 38e2ea23..51650524 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -19,6 +19,7 @@ #include "validator-session.hpp" #include "td/utils/Random.h" #include "td/utils/crypto.h" +#include "candidate-serializer.h" namespace ton { @@ -221,7 +222,9 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice // Note: src is not necessarily equal to the sender of this message: // If requested using get_broadcast_p2p, src is the creator of the block, sender possibly is some other node. auto src_idx = description().get_source_idx(src); - auto R = fetch_tl_object(data.clone(), true); + auto R = + deserialize_candidate(data, compress_block_candidates_, + description().opts().max_block_size + description().opts().max_collated_data_size + 1024); if (R.is_error()) { VLOG(VALIDATOR_SESSION_WARNING) << this << "[node " << src << "][broadcast " << sha256_bits256(data.as_slice()) << "]: failed to parse: " << R.move_as_error(); @@ -343,17 +346,17 @@ void ValidatorSessionImpl::process_query(PublicKeyHash src, td::BufferSlice data } CHECK(block); - auto P = td::PromiseCreator::lambda( - [promise = std::move(promise), src = f->id_->src_, round_id](td::Result R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error_prefix("failed to get candidate: ")); - } else { - auto c = R.move_as_ok(); - auto obj = create_tl_object( - src, round_id, c.id.root_hash, std::move(c.data), std::move(c.collated_data)); - promise.set_value(serialize_tl_object(obj, true)); - } - }); + auto P = td::PromiseCreator::lambda([promise = std::move(promise), src = f->id_->src_, round_id, + compress = compress_block_candidates_](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error_prefix("failed to get candidate: ")); + } else { + auto c = R.move_as_ok(); + auto obj = create_tl_object(src, round_id, c.id.root_hash, std::move(c.data), + std::move(c.collated_data)); + promise.set_result(serialize_candidate(obj, compress)); + } + }); callback_->get_approved_candidate(description().get_source_public_key(block->get_src_idx()), f->id_->root_hash_, f->id_->file_hash_, f->id_->collated_data_file_hash_, std::move(P)); @@ -431,7 +434,7 @@ void ValidatorSessionImpl::generated_block(td::uint32 round, ValidatorSessionCan auto b = create_tl_object(local_id().tl(), round, root_hash, std::move(data), std::move(collated_data)); - auto B = serialize_tl_object(b, true); + auto B = serialize_candidate(b, compress_block_candidates_).move_as_ok(); auto block_id = description().candidate_id(local_idx(), root_hash, file_hash, collated_data_file_hash); @@ -862,7 +865,8 @@ void ValidatorSessionImpl::on_catchain_started() { if (x) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), round = virtual_state_->cur_round_seqno(), src = description().get_source_id(x->get_src_idx()), - root_hash = x->get_root_hash()](td::Result R) { + root_hash = x->get_root_hash(), + compress = compress_block_candidates_](td::Result R) { if (R.is_error()) { LOG(ERROR) << "failed to get candidate: " << R.move_as_error(); } else { @@ -870,7 +874,7 @@ void ValidatorSessionImpl::on_catchain_started() { auto broadcast = create_tl_object( src.tl(), round, root_hash, std::move(B.data), std::move(B.collated_data)); td::actor::send_closure(SelfId, &ValidatorSessionImpl::process_broadcast, src, - serialize_tl_object(broadcast, true), td::optional(), + serialize_candidate(broadcast, compress).move_as_ok(), td::optional(), false); } }); @@ -898,6 +902,7 @@ ValidatorSessionImpl::ValidatorSessionImpl(catchain::CatChainSessionId session_i , rldp_(rldp) , overlay_manager_(overlays) , allow_unsafe_self_blocks_resync_(allow_unsafe_self_blocks_resync) { + compress_block_candidates_ = opts.proto_version >= 3; description_ = ValidatorSessionDescription::create(std::move(opts), nodes, local_id); src_round_candidate_.resize(description_->get_total_nodes()); } diff --git a/validator-session/validator-session.hpp b/validator-session/validator-session.hpp index e61e9010..619f7178 100644 --- a/validator-session/validator-session.hpp +++ b/validator-session/validator-session.hpp @@ -156,6 +156,7 @@ class ValidatorSessionImpl : public ValidatorSession { bool started_ = false; bool catchain_started_ = false; bool allow_unsafe_self_blocks_resync_; + bool compress_block_candidates_ = false; ValidatorSessionStats cur_stats_; void stats_init(); diff --git a/validator/CMakeLists.txt b/validator/CMakeLists.txt index 832374c6..a25ac7e1 100644 --- a/validator/CMakeLists.txt +++ b/validator/CMakeLists.txt @@ -146,6 +146,8 @@ set(FULL_NODE_SOURCE full-node-master.cpp full-node-private-overlay.hpp full-node-private-overlay.cpp + full-node-serializer.hpp + full-node-serializer.cpp net/download-block.hpp net/download-block.cpp diff --git a/validator/full-node-private-overlay.cpp b/validator/full-node-private-overlay.cpp index ea72230b..559dba17 100644 --- a/validator/full-node-private-overlay.cpp +++ b/validator/full-node-private-overlay.cpp @@ -14,11 +14,10 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ -#pragma once - #include "full-node-private-overlay.hpp" #include "ton/ton-tl.hpp" #include "common/delay.h" +#include "full-node-serializer.hpp" namespace ton { @@ -26,20 +25,20 @@ namespace validator { namespace fullnode { -void FullNodePrivateOverlay::process_broadcast(PublicKeyHash, ton_api::tonNode_blockBroadcast &query) { - std::vector signatures; - for (auto &sig : query.signatures_) { - signatures.emplace_back(BlockSignature{sig->who_, std::move(sig->signature_)}); +void FullNodePrivateOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query) { + process_block_broadcast(src, query); +} + +void FullNodePrivateOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query) { + process_block_broadcast(src, query); +} + +void FullNodePrivateOverlay::process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { + auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size()); + if (B.is_error()) { + LOG(DEBUG) << "dropped broadcast: " << B.move_as_error(); + return; } - - BlockIdExt block_id = create_block_id(query.id_); - BlockBroadcast B{block_id, - std::move(signatures), - static_cast(query.catchain_seqno_), - static_cast(query.validator_set_hash_), - std::move(query.data_), - std::move(query.proof_)}; - auto P = td::PromiseCreator::lambda([](td::Result R) { if (R.is_error()) { if (R.error().code() == ErrorCode::notready) { @@ -49,7 +48,7 @@ void FullNodePrivateOverlay::process_broadcast(PublicKeyHash, ton_api::tonNode_b } } }); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::prevalidate_block, std::move(B), + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::prevalidate_block, B.move_as_ok(), std::move(P)); } @@ -87,15 +86,13 @@ void FullNodePrivateOverlay::send_broadcast(BlockBroadcast broadcast) { if (!inited_) { return; } - std::vector> sigs; - for (auto &sig : broadcast.signatures) { - sigs.emplace_back(create_tl_object(sig.node, sig.signature.clone())); + auto B = serialize_block_broadcast(broadcast, false); // compression_enabled = false + if (B.is_error()) { + VLOG(FULL_NODE_WARNING) << "failed to serialize block broadcast: " << B.move_as_error(); + return; } - auto B = create_serialize_tl_object( - create_tl_block_id(broadcast.block_id), broadcast.catchain_seqno, broadcast.validator_set_hash, std::move(sigs), - broadcast.proof.clone(), broadcast.data.clone()); td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, - local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), std::move(B)); + local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); } void FullNodePrivateOverlay::start_up() { diff --git a/validator/full-node-private-overlay.hpp b/validator/full-node-private-overlay.hpp index 6463fda2..e6497a87 100644 --- a/validator/full-node-private-overlay.hpp +++ b/validator/full-node-private-overlay.hpp @@ -27,6 +27,9 @@ namespace fullnode { class FullNodePrivateOverlay : public td::actor::Actor { public: void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query); + void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query); template void process_broadcast(PublicKeyHash, T &) { diff --git a/validator/full-node-serializer.cpp b/validator/full-node-serializer.cpp new file mode 100644 index 00000000..42e68286 --- /dev/null +++ b/validator/full-node-serializer.cpp @@ -0,0 +1,155 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "full-node-serializer.hpp" +#include "ton/ton-tl.hpp" +#include "tl-utils/common-utils.hpp" +#include "auto/tl/ton_api.hpp" +#include "tl-utils/tl-utils.hpp" +#include "vm/boc.h" +#include "td/utils/lz4.h" +#include "full-node.h" +#include "td/utils/overloaded.h" + +namespace ton::validator::fullnode { + +td::Result serialize_block_broadcast(const BlockBroadcast& broadcast, bool compression_enabled) { + std::vector> sigs; + for (auto& sig : broadcast.signatures) { + sigs.emplace_back(create_tl_object(sig.node, sig.signature.clone())); + } + if (!compression_enabled) { + return create_serialize_tl_object( + create_tl_block_id(broadcast.block_id), broadcast.catchain_seqno, broadcast.validator_set_hash, std::move(sigs), + broadcast.proof.clone(), broadcast.data.clone()); + } + + TRY_RESULT(proof_root, vm::std_boc_deserialize(broadcast.proof)); + TRY_RESULT(data_root, vm::std_boc_deserialize(broadcast.data)); + TRY_RESULT(boc, vm::std_boc_serialize_multi({proof_root, data_root}, 2)); + td::BufferSlice data = + create_serialize_tl_object(std::move(sigs), std::move(boc)); + td::BufferSlice compressed = td::lz4_compress(data); + VLOG(FULL_NODE_DEBUG) << "Compressing block broadcast: " + << broadcast.data.size() + broadcast.proof.size() + broadcast.signatures.size() * 96 << " -> " + << compressed.size(); + return create_serialize_tl_object( + create_tl_block_id(broadcast.block_id), broadcast.catchain_seqno, broadcast.validator_set_hash, 0, + std::move(compressed)); +} + +static td::Result deserialize_block_broadcast(ton_api::tonNode_blockBroadcast& f) { + std::vector signatures; + for (auto& sig : f.signatures_) { + signatures.emplace_back(BlockSignature{sig->who_, std::move(sig->signature_)}); + } + return BlockBroadcast{create_block_id(f.id_), + std::move(signatures), + static_cast(f.catchain_seqno_), + static_cast(f.validator_set_hash_), + std::move(f.data_), + std::move(f.proof_)}; +} + +static td::Result deserialize_block_broadcast(ton_api::tonNode_blockBroadcastCompressed& f, + int max_decompressed_size) { + TRY_RESULT(decompressed, td::lz4_decompress(f.compressed_, max_decompressed_size)); + TRY_RESULT(f2, fetch_tl_object(decompressed, true)); + std::vector signatures; + for (auto& sig : f2->signatures_) { + signatures.emplace_back(BlockSignature{sig->who_, std::move(sig->signature_)}); + } + TRY_RESULT(roots, vm::std_boc_deserialize_multi(f2->proof_data_, 2)); + if (roots.size() != 2) { + return td::Status::Error("expected 2 roots in boc"); + } + TRY_RESULT(proof, vm::std_boc_serialize(roots[0], 0)); + TRY_RESULT(data, vm::std_boc_serialize(roots[1], 31)); + VLOG(FULL_NODE_DEBUG) << "Decompressing block broadcast: " << f.compressed_.size() << " -> " + << data.size() + proof.size() + signatures.size() * 96; + return BlockBroadcast{create_block_id(f.id_), + std::move(signatures), + static_cast(f.catchain_seqno_), + static_cast(f.validator_set_hash_), + std::move(data), + std::move(proof)}; +} + +td::Result deserialize_block_broadcast(ton_api::tonNode_Broadcast& obj, + int max_decompressed_data_size) { + td::Result B; + ton_api::downcast_call(obj, + td::overloaded([&](ton_api::tonNode_blockBroadcast& f) { B = deserialize_block_broadcast(f); }, + [&](ton_api::tonNode_blockBroadcastCompressed& f) { + B = deserialize_block_broadcast(f, max_decompressed_data_size); + }, + [&](auto&) { B = td::Status::Error("unknown broadcast type"); })); + return B; +} + +td::Result serialize_block_full(const BlockIdExt& id, td::Slice proof, td::Slice data, + bool is_proof_link, bool compression_enabled) { + if (!compression_enabled) { + return create_serialize_tl_object(create_tl_block_id(id), td::BufferSlice(proof), + td::BufferSlice(data), is_proof_link); + } + TRY_RESULT(proof_root, vm::std_boc_deserialize(proof)); + TRY_RESULT(data_root, vm::std_boc_deserialize(data)); + TRY_RESULT(boc, vm::std_boc_serialize_multi({proof_root, data_root}, 2)); + td::BufferSlice compressed = td::lz4_compress(boc); + VLOG(FULL_NODE_DEBUG) << "Compressing block full: " << data.size() + proof.size() << " -> " << compressed.size(); + return create_serialize_tl_object(create_tl_block_id(id), 0, + std::move(compressed), is_proof_link); +} + +static td::Status deserialize_block_full(ton_api::tonNode_dataFull& f, BlockIdExt& id, td::BufferSlice& proof, + td::BufferSlice& data, bool& is_proof_link) { + id = create_block_id(f.id_); + proof = std::move(f.proof_); + data = std::move(f.block_); + is_proof_link = f.is_link_; + return td::Status::OK(); +} + +static td::Status deserialize_block_full(ton_api::tonNode_dataFullCompressed& f, BlockIdExt& id, td::BufferSlice& proof, + td::BufferSlice& data, bool& is_proof_link, int max_decompressed_size) { + TRY_RESULT(decompressed, td::lz4_decompress(f.compressed_, max_decompressed_size)); + TRY_RESULT(roots, vm::std_boc_deserialize_multi(decompressed, 2)); + if (roots.size() != 2) { + return td::Status::Error("expected 2 roots in boc"); + } + TRY_RESULT_ASSIGN(proof, vm::std_boc_serialize(roots[0], 0)); + TRY_RESULT_ASSIGN(data, vm::std_boc_serialize(roots[1], 31)); + VLOG(FULL_NODE_DEBUG) << "Decompressing block full: " << f.compressed_.size() << " -> " << data.size() + proof.size(); + id = create_block_id(f.id_); + is_proof_link = f.is_link_; + return td::Status::OK(); +} + +td::Status deserialize_block_full(ton_api::tonNode_DataFull& obj, BlockIdExt& id, td::BufferSlice& proof, + td::BufferSlice& data, bool& is_proof_link, int max_decompressed_data_size) { + td::Status S; + ton_api::downcast_call( + obj, td::overloaded( + [&](ton_api::tonNode_dataFull& f) { S = deserialize_block_full(f, id, proof, data, is_proof_link); }, + [&](ton_api::tonNode_dataFullCompressed& f) { + S = deserialize_block_full(f, id, proof, data, is_proof_link, max_decompressed_data_size); + }, + [&](auto&) { S = td::Status::Error("unknown data type"); })); + return S; +} + +} // namespace ton::validator::fullnode diff --git a/validator/full-node-serializer.hpp b/validator/full-node-serializer.hpp new file mode 100644 index 00000000..a5c73cbc --- /dev/null +++ b/validator/full-node-serializer.hpp @@ -0,0 +1,31 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "ton/ton-types.h" +#include "auto/tl/ton_api.h" + +namespace ton::validator::fullnode { + +td::Result serialize_block_broadcast(const BlockBroadcast& broadcast, bool compression_enabled); +td::Result deserialize_block_broadcast(ton_api::tonNode_Broadcast& obj, int max_decompressed_data_size); + +td::Result serialize_block_full(const BlockIdExt& id, td::Slice proof, td::Slice data, + bool is_proof_link, bool compression_enabled); +td::Status deserialize_block_full(ton_api::tonNode_DataFull& obj, BlockIdExt& id, td::BufferSlice& proof, + td::BufferSlice& data, bool& is_proof_link, int max_decompressed_data_size); + +} // namespace ton::validator::fullnode diff --git a/validator/full-node-shard-queries.hpp b/validator/full-node-shard-queries.hpp index 1a2cd716..17068229 100644 --- a/validator/full-node-shard-queries.hpp +++ b/validator/full-node-shard-queries.hpp @@ -20,6 +20,7 @@ #include "validator/validator.h" #include "ton/ton-tl.hpp" +#include "full-node-serializer.hpp" namespace ton { @@ -38,8 +39,8 @@ class BlockFullSender : public td::actor::Actor { stop(); } void finish_query() { - promise_.set_value(create_serialize_tl_object( - create_tl_block_id(block_id_), std::move(proof_), std::move(data_), is_proof_link_)); + promise_.set_result( + serialize_block_full(block_id_, proof_, data_, is_proof_link_, false)); // compression_enabled = false stop(); } void start_up() override { diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 8e42716b..9f798fa8 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -21,6 +21,7 @@ #include "td/utils/SharedSlice.h" #include "full-node-shard.hpp" #include "full-node-shard-queries.hpp" +#include "full-node-serializer.hpp" #include "ton/ton-shard.h" #include "ton/ton-tl.hpp" @@ -645,19 +646,19 @@ void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_ne } void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query) { - std::vector signatures; - for (auto &sig : query.signatures_) { - signatures.emplace_back(BlockSignature{sig->who_, std::move(sig->signature_)}); + process_block_broadcast(src, query); +} + +void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query) { + process_block_broadcast(src, query); +} + +void FullNodeShardImpl::process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { + auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size()); + if (B.is_error()) { + LOG(DEBUG) << "dropped broadcast: " << B.move_as_error(); + return; } - - BlockIdExt block_id = create_block_id(query.id_); - BlockBroadcast B{block_id, - std::move(signatures), - static_cast(query.catchain_seqno_), - static_cast(query.validator_set_hash_), - std::move(query.data_), - std::move(query.proof_)}; - auto P = td::PromiseCreator::lambda([](td::Result R) { if (R.is_error()) { if (R.error().code() == ErrorCode::notready) { @@ -667,7 +668,7 @@ void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_bl } } }); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::prevalidate_block, std::move(B), + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::prevalidate_block, B.move_as_ok(), std::move(P)); } @@ -749,15 +750,13 @@ void FullNodeShardImpl::send_broadcast(BlockBroadcast broadcast) { UNREACHABLE(); return; } - std::vector> sigs; - for (auto &sig : broadcast.signatures) { - sigs.emplace_back(create_tl_object(sig.node, sig.signature.clone())); + auto B = serialize_block_broadcast(broadcast, false); // compression_enabled = false + if (B.is_error()) { + VLOG(FULL_NODE_WARNING) << "failed to serialize block broadcast: " << B.move_as_error(); + return; } - auto B = create_serialize_tl_object( - create_tl_block_id(broadcast.block_id), broadcast.catchain_seqno, broadcast.validator_set_hash, std::move(sigs), - broadcast.proof.clone(), broadcast.data.clone()); td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, adnl_id_, overlay_id_, local_id_, - overlay::Overlays::BroadcastFlagAnySender(), std::move(B)); + overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); } void FullNodeShardImpl::download_block(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index 0525474e..c8b5301a 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -71,7 +71,7 @@ class FullNodeShardImpl : public FullNodeShard { return 2; } static constexpr td::uint64 proto_capabilities() { - return 2; + return 3; } static constexpr td::uint32 max_neighbours() { return 16; @@ -146,6 +146,9 @@ class FullNodeShardImpl : public FullNodeShard { void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise promise); void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query); + void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_ihrMessageBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_externalMessageBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query); diff --git a/validator/net/download-block-new.cpp b/validator/net/download-block-new.cpp index 14754f64..9ec36e33 100644 --- a/validator/net/download-block-new.cpp +++ b/validator/net/download-block-new.cpp @@ -23,6 +23,7 @@ #include "td/utils/overloaded.h" #include "ton/ton-io.hpp" #include "validator/full-node.h" +#include "full-node-serializer.hpp" namespace ton { @@ -219,52 +220,54 @@ void DownloadBlockNew::got_data(td::BufferSlice data) { } auto f = F.move_as_ok(); + if (f->get_id() == ton_api::tonNode_dataFullEmpty::ID) { + abort_query(td::Status::Error(ErrorCode::notready, "node doesn't have this block")); + return; + } + BlockIdExt id; + td::BufferSlice proof, block_data; + bool is_link; + td::Status S = deserialize_block_full(*f, id, proof, block_data, is_link, overlay::Overlays::max_fec_broadcast_size()); + if (S.is_error()) { + abort_query(S.move_as_error_prefix("cannot deserialize block: ")); + return; + } - ton_api::downcast_call( - *f.get(), - td::overloaded( - [&](ton_api::tonNode_dataFullEmpty &x) { - abort_query(td::Status::Error(ErrorCode::notready, "node doesn't have this block")); - }, - [&, self = this](ton_api::tonNode_dataFull &x) { - if (!allow_partial_proof_ && x.is_link_) { - abort_query(td::Status::Error(ErrorCode::notready, "node doesn't have proof for this block")); - return; - } - auto id = create_block_id(x.id_); - if (block_id_.is_valid() && id != block_id_) { - abort_query(td::Status::Error(ErrorCode::notready, "received data for wrong block")); - return; - } - block_.id = id; - block_.data = std::move(x.block_); - if (td::sha256_bits256(block_.data.as_slice()) != id.file_hash) { - abort_query(td::Status::Error(ErrorCode::notready, "received data with bad hash")); - return; - } + if (!allow_partial_proof_ && is_link) { + abort_query(td::Status::Error(ErrorCode::notready, "node doesn't have proof for this block")); + return; + } + if (block_id_.is_valid() && id != block_id_) { + abort_query(td::Status::Error(ErrorCode::notready, "received data for wrong block")); + return; + } + block_.id = id; + block_.data = std::move(block_data); + if (td::sha256_bits256(block_.data.as_slice()) != id.file_hash) { + abort_query(td::Status::Error(ErrorCode::notready, "received data with bad hash")); + return; + } - auto P = td::PromiseCreator::lambda([SelfId = actor_id(self)](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &DownloadBlockNew::abort_query, - R.move_as_error_prefix("received bad proof: ")); - } else { - td::actor::send_closure(SelfId, &DownloadBlockNew::checked_block_proof); - } - }); - if (block_id_.is_valid()) { - if (x.is_link_) { - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::validate_block_proof_link, - block_id_, std::move(x.proof_), std::move(P)); - } else { - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::validate_block_proof, block_id_, - std::move(x.proof_), std::move(P)); - } - } else { - CHECK(!x.is_link_); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::validate_block_is_next_proof, - prev_id_, id, std::move(x.proof_), std::move(P)); - } - })); + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &DownloadBlockNew::abort_query, R.move_as_error_prefix("received bad proof: ")); + } else { + td::actor::send_closure(SelfId, &DownloadBlockNew::checked_block_proof); + } + }); + if (block_id_.is_valid()) { + if (is_link) { + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::validate_block_proof_link, block_id_, + std::move(proof), std::move(P)); + } else { + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::validate_block_proof, block_id_, + std::move(proof), std::move(P)); + } + } else { + CHECK(!is_link); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::validate_block_is_next_proof, prev_id_, id, + std::move(proof), std::move(P)); + } } void DownloadBlockNew::got_data_from_db(td::BufferSlice data) {