diff --git a/CMake/FindLZ4.cmake b/CMake/FindLZ4.cmake
new file mode 100644
index 00000000..e13eca55
--- /dev/null
+++ b/CMake/FindLZ4.cmake
@@ -0,0 +1,39 @@
+###############################################################################
+# Find LZ4
+#
+# This sets the following variables:
+# LZ4_FOUND - True if LZ4 was found.
+# LZ4_INCLUDE_DIRS - Directories containing the LZ4 include files.
+# LZ4_LIBRARIES - Libraries needed to use LZ4.
+# LZ4_LIBRARY - Library needed to use LZ4.
+# LZ4_LIBRARY_DIRS - Library needed to use LZ4.
+
+find_package(PkgConfig REQUIRED)
+
+# If found, LZ$_* variables will be defined
+pkg_check_modules(LZ4 REQUIRED liblz4)
+
+if(NOT LZ4_FOUND)
+ find_path(LZ4_INCLUDE_DIR lz4.h
+ HINTS "${LZ4_ROOT}" "$ENV{LZ4_ROOT}"
+ PATHS "$ENV{PROGRAMFILES}/lz4" "$ENV{PROGRAMW6432}/lz4"
+ PATH_SUFFIXES include)
+
+ find_library(LZ4_LIBRARY
+ NAMES lz4 lz4_static
+ HINTS "${LZ4_ROOT}" "$ENV{LZ4_ROOT}"
+ PATHS "$ENV{PROGRAMFILES}/lz4" "$ENV{PROGRAMW6432}/lz4"
+ PATH_SUFFIXES lib)
+
+ if(LZ4_LIBRARY)
+ set(LZ4_LIBRARIES ${LZ4_LIBRARY})
+ get_filename_component(LZ4_LIBRARY_DIRS ${LZ4_LIBRARY} DIRECTORY)
+ endif()
+else()
+ find_library(LZ4_LIBRARY
+ NAMES lz4 lz4_static
+ PATHS ${LZ4_LIBRARY_DIRS}
+ NO_DEFAULT_PATH)
+endif()
+
+mark_as_advanced(LZ4_LIBRARY LZ4_INCLUDE_DIRS LZ4_LIBRARY_DIRS LZ4_LIBRARIES)
\ No newline at end of file
diff --git a/tdutils/CMakeLists.txt b/tdutils/CMakeLists.txt
index f1e4b1ea..7eebc910 100644
--- a/tdutils/CMakeLists.txt
+++ b/tdutils/CMakeLists.txt
@@ -15,6 +15,7 @@ if (NOT DEFINED CMAKE_INSTALL_LIBDIR)
endif()
find_package(PkgConfig REQUIRED)
+find_package(LZ4)
if (NOT ZLIB_FOUND)
pkg_check_modules(ZLIB zlib)
endif()
@@ -280,6 +281,15 @@ if (TDUTILS_MIME_TYPE)
)
endif()
+if (LZ4_FOUND)
+ set(TD_HAVE_LZ4 1)
+ set(TDUTILS_SOURCE
+ ${TDUTILS_SOURCE}
+ td/utils/lz4.cpp
+ td/utils/lz4.h
+ )
+endif()
+
set(TDUTILS_TEST_SOURCE
${CMAKE_CURRENT_SOURCE_DIR}/test/buffer.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/ConcurrentHashMap.cpp
@@ -338,6 +348,11 @@ endif()
if (CRC32C_FOUND)
target_link_libraries(tdutils PRIVATE crc32c)
endif()
+
+if (LZ4_FOUND)
+ target_link_libraries(tdutils PRIVATE ${LZ4_LIBRARIES})
+endif()
+
if (ABSL_FOUND)
target_link_libraries_system(tdutils absl::flat_hash_map absl::flat_hash_set absl::hash)
endif()
diff --git a/tdutils/td/utils/config.h.in b/tdutils/td/utils/config.h.in
index f8b89aeb..3f4e1bf2 100644
--- a/tdutils/td/utils/config.h.in
+++ b/tdutils/td/utils/config.h.in
@@ -3,6 +3,7 @@
#cmakedefine01 TD_HAVE_OPENSSL
#cmakedefine01 TD_HAVE_ZLIB
#cmakedefine01 TD_HAVE_CRC32C
+#cmakedefine01 TD_HAVE_LZ4
#cmakedefine01 TD_HAVE_COROUTINES
#cmakedefine01 TD_HAVE_ABSL
#cmakedefine01 TD_FD_DEBUG
diff --git a/tdutils/td/utils/lz4.cpp b/tdutils/td/utils/lz4.cpp
new file mode 100644
index 00000000..ebf456aa
--- /dev/null
+++ b/tdutils/td/utils/lz4.cpp
@@ -0,0 +1,48 @@
+/*
+ This file is part of TON Blockchain Library.
+
+ TON Blockchain Library is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as published by
+ the Free Software Foundation, either version 2 of the License, or
+ (at your option) any later version.
+
+ TON Blockchain Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public License
+ along with TON Blockchain Library. If not, see .
+*/
+#include "td/utils/buffer.h"
+#include "td/utils/misc.h"
+#include
+
+namespace td {
+
+td::BufferSlice lz4_compress(td::Slice data) {
+ int size = narrow_cast(data.size());
+ int buf_size = LZ4_compressBound(size);
+ td::BufferSlice compressed(buf_size);
+ int compressed_size = LZ4_compress_default(data.data(), compressed.data(), size, buf_size);
+ CHECK(compressed_size > 0);
+ return td::BufferSlice{compressed.as_slice().substr(0, compressed_size)};
+}
+
+td::Result lz4_decompress(td::Slice data, int max_decompressed_size) {
+ TRY_RESULT(size, narrow_cast_safe(data.size()));
+ if (max_decompressed_size < 0) {
+ return td::Status::Error("invalid max_decompressed_size");
+ }
+ td::BufferSlice decompressed(max_decompressed_size);
+ int result = LZ4_decompress_safe(data.data(), decompressed.data(), size, max_decompressed_size);
+ if (result < 0) {
+ return td::Status::Error(PSTRING() << "lz4 decompression failed, error code: " << result);
+ }
+ if (result == max_decompressed_size) {
+ return decompressed;
+ }
+ return td::BufferSlice{decompressed.as_slice().substr(0, result)};
+}
+
+} // namespace td
diff --git a/tdutils/td/utils/lz4.h b/tdutils/td/utils/lz4.h
new file mode 100644
index 00000000..fbbc470f
--- /dev/null
+++ b/tdutils/td/utils/lz4.h
@@ -0,0 +1,27 @@
+/*
+ This file is part of TON Blockchain Library.
+
+ TON Blockchain Library is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as published by
+ the Free Software Foundation, either version 2 of the License, or
+ (at your option) any later version.
+
+ TON Blockchain Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public License
+ along with TON Blockchain Library. If not, see .
+*/
+#pragma once
+
+#include "td/utils/buffer.h"
+#include "td/utils/Status.h"
+
+namespace td {
+
+td::BufferSlice lz4_compress(td::Slice data);
+td::Result lz4_decompress(td::Slice data, int max_decompressed_size);
+
+} // namespace td
diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl
index 0f94e7f1..159a0c88 100644
--- a/tl/generate/scheme/ton_api.tl
+++ b/tl/generate/scheme/ton_api.tl
@@ -309,6 +309,7 @@ validatorSession.candidateId src:int256 root_hash:int256 file_hash:int256 collat
validatorSession.blockUpdate ts:long actions:(vector validatorSession.round.Message) state:int = validatorSession.BlockUpdate;
validatorSession.candidate src:int256 round:int root_hash:int256 data:bytes collated_data:bytes = validatorSession.Candidate;
+validatorSession.compressedCandidate flags:# src:int256 round:int root_hash:int256 decompressed_size:int data:bytes = validatorSession.Candidate;
validatorSession.config catchain_idle_timeout:double catchain_max_deps:int round_candidates:int next_candidate_delay:double round_attempt_duration:int
max_round_attempts:int max_block_size:int max_collated_data_size:int = validatorSession.Config;
@@ -385,9 +386,13 @@ tonNode.externalMessage data:bytes = tonNode.ExternalMessage;
tonNode.newShardBlock block:tonNode.blockIdExt cc_seqno:int data:bytes = tonNode.NewShardBlock;
+tonNode.blockBroadcastCompressed.data signatures:(vector tonNode.blockSignature) proof_data:bytes = tonNode.blockBroadcaseCompressed.Data;
+
tonNode.blockBroadcast id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int
signatures:(vector tonNode.blockSignature)
proof:bytes data:bytes = tonNode.Broadcast;
+tonNode.blockBroadcastCompressed id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int
+ flags:# compressed:bytes = tonNode.Broadcast;
tonNode.ihrMessageBroadcast message:tonNode.ihrMessage = tonNode.Broadcast;
tonNode.externalMessageBroadcast message:tonNode.externalMessage = tonNode.Broadcast;
tonNode.newShardBlockBroadcast block:tonNode.newShardBlock = tonNode.Broadcast;
@@ -401,9 +406,8 @@ tonNode.keyBlocks blocks:(vector tonNode.blockIdExt) incomplete:Bool error:Bool
ton.blockId root_cell_hash:int256 file_hash:int256 = ton.BlockId;
ton.blockIdApprove root_cell_hash:int256 file_hash:int256 = ton.BlockId;
-tonNode.dataList data:(vector bytes) = tonNode.DataList;
-
tonNode.dataFull id:tonNode.blockIdExt proof:bytes block:bytes is_link:Bool = tonNode.DataFull;
+tonNode.dataFullCompressed id:tonNode.blockIdExt flags:# compressed:bytes is_link:Bool = tonNode.DataFull;
tonNode.dataFullEmpty = tonNode.DataFull;
tonNode.capabilities version:int capabilities:long = tonNode.Capabilities;
@@ -430,18 +434,13 @@ tonNode.getNextKeyBlockIds block:tonNode.blockIdExt max_size:int = tonNode.KeyBl
tonNode.downloadNextBlockFull prev_block:tonNode.blockIdExt = tonNode.DataFull;
tonNode.downloadBlockFull block:tonNode.blockIdExt = tonNode.DataFull;
tonNode.downloadBlock block:tonNode.blockIdExt = tonNode.Data;
-tonNode.downloadBlocks blocks:(vector tonNode.blockIdExt) = tonNode.DataList;
tonNode.downloadPersistentState block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt = tonNode.Data;
tonNode.downloadPersistentStateSlice block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt offset:long max_size:long = tonNode.Data;
tonNode.downloadZeroState block:tonNode.blockIdExt = tonNode.Data;
tonNode.downloadBlockProof block:tonNode.blockIdExt = tonNode.Data;
tonNode.downloadKeyBlockProof block:tonNode.blockIdExt = tonNode.Data;
-tonNode.downloadBlockProofs blocks:(vector tonNode.blockIdExt) = tonNode.DataList;
-tonNode.downloadKeyBlockProofs blocks:(vector tonNode.blockIdExt) = tonNode.DataList;
tonNode.downloadBlockProofLink block:tonNode.blockIdExt = tonNode.Data;
tonNode.downloadKeyBlockProofLink block:tonNode.blockIdExt = tonNode.Data;
-tonNode.downloadBlockProofLinks blocks:(vector tonNode.blockIdExt) = tonNode.DataList;
-tonNode.downloadKeyBlockProofLinks blocks:(vector tonNode.blockIdExt) = tonNode.DataList;
tonNode.getArchiveInfo masterchain_seqno:int = tonNode.ArchiveInfo;
tonNode.getArchiveSlice archive_id:long offset:long max_size:int = tonNode.Data;
diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo
index e71fb0f7..18df7794 100644
Binary files a/tl/generate/scheme/ton_api.tlo and b/tl/generate/scheme/ton_api.tlo differ
diff --git a/validator-session/CMakeLists.txt b/validator-session/CMakeLists.txt
index c769f4d8..41911a8b 100644
--- a/validator-session/CMakeLists.txt
+++ b/validator-session/CMakeLists.txt
@@ -5,12 +5,14 @@ if (NOT OPENSSL_FOUND)
endif()
set(VALIDATOR_SESSION_SOURCE
+ candidate-serializer.cpp
persistent-vector.cpp
validator-session-description.cpp
validator-session-state.cpp
validator-session.cpp
validator-session-round-attempt-state.cpp
+ candidate-serializer.h
persistent-vector.h
validator-session-description.h
validator-session-description.hpp
diff --git a/validator-session/candidate-serializer.cpp b/validator-session/candidate-serializer.cpp
new file mode 100644
index 00000000..c220b4a9
--- /dev/null
+++ b/validator-session/candidate-serializer.cpp
@@ -0,0 +1,76 @@
+/*
+ This file is part of TON Blockchain Library.
+
+ TON Blockchain Library is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as published by
+ the Free Software Foundation, either version 2 of the License, or
+ (at your option) any later version.
+
+ TON Blockchain Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public License
+ along with TON Blockchain Library. If not, see .
+*/
+#pragma once
+#include "candidate-serializer.h"
+#include "tl-utils/tl-utils.hpp"
+#include "vm/boc.h"
+#include "td/utils/lz4.h"
+#include "validator-session-types.h"
+
+namespace ton::validatorsession {
+
+td::Result serialize_candidate(const tl_object_ptr &block,
+ bool compression_enabled) {
+ if (!compression_enabled) {
+ return serialize_tl_object(block, true);
+ }
+ vm::BagOfCells boc1, boc2;
+ TRY_STATUS(boc1.deserialize(block->data_));
+ if (boc1.get_root_count() != 1) {
+ return td::Status::Error("block candidate should have exactly one root");
+ }
+ std::vector> roots = {boc1.get_root_cell()};
+ TRY_STATUS(boc2.deserialize(block->collated_data_));
+ for (int i = 0; i < boc2.get_root_count(); ++i) {
+ roots.push_back(boc2.get_root_cell(i));
+ }
+ TRY_RESULT(data, vm::std_boc_serialize_multi(std::move(roots), 2));
+ td::BufferSlice compressed = td::lz4_compress(data);
+ LOG(VALIDATOR_SESSION_DEBUG) << "Compressing block candidate: " << block->data_.size() + block->collated_data_.size()
+ << " -> " << compressed.size();
+ return create_serialize_tl_object(
+ 0, block->src_, block->round_, block->root_hash_, (int)data.size(), std::move(compressed));
+}
+
+td::Result> deserialize_candidate(td::Slice data,
+ bool compression_enabled,
+ int max_decompressed_data_size) {
+ if (!compression_enabled) {
+ return fetch_tl_object(data, true);
+ }
+ TRY_RESULT(f, fetch_tl_object(data, true));
+ if (f->decompressed_size_ > max_decompressed_data_size) {
+ return td::Status::Error("decompressed size is too big");
+ }
+ TRY_RESULT(decompressed, td::lz4_decompress(f->data_, f->decompressed_size_));
+ if (decompressed.size() != (size_t)f->decompressed_size_) {
+ return td::Status::Error("decompressed size mismatch");
+ }
+ TRY_RESULT(roots, vm::std_boc_deserialize_multi(decompressed));
+ if (roots.empty()) {
+ return td::Status::Error("boc is empty");
+ }
+ TRY_RESULT(block_data, vm::std_boc_serialize(roots[0], 31));
+ roots.erase(roots.begin());
+ TRY_RESULT(collated_data, vm::std_boc_serialize_multi(std::move(roots), 31));
+ LOG(VALIDATOR_SESSION_DEBUG) << "Decompressing block candidate: " << f->data_.size() << " -> "
+ << block_data.size() + collated_data.size();
+ return create_tl_object(f->src_, f->round_, f->root_hash_, std::move(block_data),
+ std::move(collated_data));
+}
+
+} // namespace ton::validatorsession
diff --git a/validator-session/candidate-serializer.h b/validator-session/candidate-serializer.h
new file mode 100644
index 00000000..030a412c
--- /dev/null
+++ b/validator-session/candidate-serializer.h
@@ -0,0 +1,29 @@
+/*
+ This file is part of TON Blockchain Library.
+
+ TON Blockchain Library is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as published by
+ the Free Software Foundation, either version 2 of the License, or
+ (at your option) any later version.
+
+ TON Blockchain Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public License
+ along with TON Blockchain Library. If not, see .
+*/
+#pragma once
+#include "ton/ton-types.h"
+#include "auto/tl/ton_api.h"
+
+namespace ton::validatorsession {
+
+td::Result serialize_candidate(const tl_object_ptr &block,
+ bool compression_enabled);
+td::Result> deserialize_candidate(td::Slice data,
+ bool compression_enabled,
+ int max_decompressed_data_size);
+
+} // namespace ton::validatorsession
diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp
index 38e2ea23..51650524 100644
--- a/validator-session/validator-session.cpp
+++ b/validator-session/validator-session.cpp
@@ -19,6 +19,7 @@
#include "validator-session.hpp"
#include "td/utils/Random.h"
#include "td/utils/crypto.h"
+#include "candidate-serializer.h"
namespace ton {
@@ -221,7 +222,9 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice
// Note: src is not necessarily equal to the sender of this message:
// If requested using get_broadcast_p2p, src is the creator of the block, sender possibly is some other node.
auto src_idx = description().get_source_idx(src);
- auto R = fetch_tl_object(data.clone(), true);
+ auto R =
+ deserialize_candidate(data, compress_block_candidates_,
+ description().opts().max_block_size + description().opts().max_collated_data_size + 1024);
if (R.is_error()) {
VLOG(VALIDATOR_SESSION_WARNING) << this << "[node " << src << "][broadcast " << sha256_bits256(data.as_slice())
<< "]: failed to parse: " << R.move_as_error();
@@ -343,17 +346,17 @@ void ValidatorSessionImpl::process_query(PublicKeyHash src, td::BufferSlice data
}
CHECK(block);
- auto P = td::PromiseCreator::lambda(
- [promise = std::move(promise), src = f->id_->src_, round_id](td::Result R) mutable {
- if (R.is_error()) {
- promise.set_error(R.move_as_error_prefix("failed to get candidate: "));
- } else {
- auto c = R.move_as_ok();
- auto obj = create_tl_object(
- src, round_id, c.id.root_hash, std::move(c.data), std::move(c.collated_data));
- promise.set_value(serialize_tl_object(obj, true));
- }
- });
+ auto P = td::PromiseCreator::lambda([promise = std::move(promise), src = f->id_->src_, round_id,
+ compress = compress_block_candidates_](td::Result R) mutable {
+ if (R.is_error()) {
+ promise.set_error(R.move_as_error_prefix("failed to get candidate: "));
+ } else {
+ auto c = R.move_as_ok();
+ auto obj = create_tl_object(src, round_id, c.id.root_hash, std::move(c.data),
+ std::move(c.collated_data));
+ promise.set_result(serialize_candidate(obj, compress));
+ }
+ });
callback_->get_approved_candidate(description().get_source_public_key(block->get_src_idx()), f->id_->root_hash_,
f->id_->file_hash_, f->id_->collated_data_file_hash_, std::move(P));
@@ -431,7 +434,7 @@ void ValidatorSessionImpl::generated_block(td::uint32 round, ValidatorSessionCan
auto b = create_tl_object(local_id().tl(), round, root_hash, std::move(data),
std::move(collated_data));
- auto B = serialize_tl_object(b, true);
+ auto B = serialize_candidate(b, compress_block_candidates_).move_as_ok();
auto block_id = description().candidate_id(local_idx(), root_hash, file_hash, collated_data_file_hash);
@@ -862,7 +865,8 @@ void ValidatorSessionImpl::on_catchain_started() {
if (x) {
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), round = virtual_state_->cur_round_seqno(),
src = description().get_source_id(x->get_src_idx()),
- root_hash = x->get_root_hash()](td::Result R) {
+ root_hash = x->get_root_hash(),
+ compress = compress_block_candidates_](td::Result R) {
if (R.is_error()) {
LOG(ERROR) << "failed to get candidate: " << R.move_as_error();
} else {
@@ -870,7 +874,7 @@ void ValidatorSessionImpl::on_catchain_started() {
auto broadcast = create_tl_object(
src.tl(), round, root_hash, std::move(B.data), std::move(B.collated_data));
td::actor::send_closure(SelfId, &ValidatorSessionImpl::process_broadcast, src,
- serialize_tl_object(broadcast, true), td::optional(),
+ serialize_candidate(broadcast, compress).move_as_ok(), td::optional(),
false);
}
});
@@ -898,6 +902,7 @@ ValidatorSessionImpl::ValidatorSessionImpl(catchain::CatChainSessionId session_i
, rldp_(rldp)
, overlay_manager_(overlays)
, allow_unsafe_self_blocks_resync_(allow_unsafe_self_blocks_resync) {
+ compress_block_candidates_ = opts.proto_version >= 3;
description_ = ValidatorSessionDescription::create(std::move(opts), nodes, local_id);
src_round_candidate_.resize(description_->get_total_nodes());
}
diff --git a/validator-session/validator-session.hpp b/validator-session/validator-session.hpp
index e61e9010..619f7178 100644
--- a/validator-session/validator-session.hpp
+++ b/validator-session/validator-session.hpp
@@ -156,6 +156,7 @@ class ValidatorSessionImpl : public ValidatorSession {
bool started_ = false;
bool catchain_started_ = false;
bool allow_unsafe_self_blocks_resync_;
+ bool compress_block_candidates_ = false;
ValidatorSessionStats cur_stats_;
void stats_init();
diff --git a/validator/CMakeLists.txt b/validator/CMakeLists.txt
index 832374c6..a25ac7e1 100644
--- a/validator/CMakeLists.txt
+++ b/validator/CMakeLists.txt
@@ -146,6 +146,8 @@ set(FULL_NODE_SOURCE
full-node-master.cpp
full-node-private-overlay.hpp
full-node-private-overlay.cpp
+ full-node-serializer.hpp
+ full-node-serializer.cpp
net/download-block.hpp
net/download-block.cpp
diff --git a/validator/full-node-private-overlay.cpp b/validator/full-node-private-overlay.cpp
index ea72230b..559dba17 100644
--- a/validator/full-node-private-overlay.cpp
+++ b/validator/full-node-private-overlay.cpp
@@ -14,11 +14,10 @@
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see .
*/
-#pragma once
-
#include "full-node-private-overlay.hpp"
#include "ton/ton-tl.hpp"
#include "common/delay.h"
+#include "full-node-serializer.hpp"
namespace ton {
@@ -26,20 +25,20 @@ namespace validator {
namespace fullnode {
-void FullNodePrivateOverlay::process_broadcast(PublicKeyHash, ton_api::tonNode_blockBroadcast &query) {
- std::vector signatures;
- for (auto &sig : query.signatures_) {
- signatures.emplace_back(BlockSignature{sig->who_, std::move(sig->signature_)});
+void FullNodePrivateOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query) {
+ process_block_broadcast(src, query);
+}
+
+void FullNodePrivateOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query) {
+ process_block_broadcast(src, query);
+}
+
+void FullNodePrivateOverlay::process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) {
+ auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size());
+ if (B.is_error()) {
+ LOG(DEBUG) << "dropped broadcast: " << B.move_as_error();
+ return;
}
-
- BlockIdExt block_id = create_block_id(query.id_);
- BlockBroadcast B{block_id,
- std::move(signatures),
- static_cast(query.catchain_seqno_),
- static_cast(query.validator_set_hash_),
- std::move(query.data_),
- std::move(query.proof_)};
-
auto P = td::PromiseCreator::lambda([](td::Result R) {
if (R.is_error()) {
if (R.error().code() == ErrorCode::notready) {
@@ -49,7 +48,7 @@ void FullNodePrivateOverlay::process_broadcast(PublicKeyHash, ton_api::tonNode_b
}
}
});
- td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::prevalidate_block, std::move(B),
+ td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::prevalidate_block, B.move_as_ok(),
std::move(P));
}
@@ -87,15 +86,13 @@ void FullNodePrivateOverlay::send_broadcast(BlockBroadcast broadcast) {
if (!inited_) {
return;
}
- std::vector> sigs;
- for (auto &sig : broadcast.signatures) {
- sigs.emplace_back(create_tl_object(sig.node, sig.signature.clone()));
+ auto B = serialize_block_broadcast(broadcast, false); // compression_enabled = false
+ if (B.is_error()) {
+ VLOG(FULL_NODE_WARNING) << "failed to serialize block broadcast: " << B.move_as_error();
+ return;
}
- auto B = create_serialize_tl_object(
- create_tl_block_id(broadcast.block_id), broadcast.catchain_seqno, broadcast.validator_set_hash, std::move(sigs),
- broadcast.proof.clone(), broadcast.data.clone());
td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_,
- local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), std::move(B));
+ local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok());
}
void FullNodePrivateOverlay::start_up() {
diff --git a/validator/full-node-private-overlay.hpp b/validator/full-node-private-overlay.hpp
index 6463fda2..e6497a87 100644
--- a/validator/full-node-private-overlay.hpp
+++ b/validator/full-node-private-overlay.hpp
@@ -27,6 +27,9 @@ namespace fullnode {
class FullNodePrivateOverlay : public td::actor::Actor {
public:
void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query);
+ void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query);
+ void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query);
+
void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query);
template
void process_broadcast(PublicKeyHash, T &) {
diff --git a/validator/full-node-serializer.cpp b/validator/full-node-serializer.cpp
new file mode 100644
index 00000000..42e68286
--- /dev/null
+++ b/validator/full-node-serializer.cpp
@@ -0,0 +1,155 @@
+/*
+ This file is part of TON Blockchain Library.
+
+ TON Blockchain Library is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as published by
+ the Free Software Foundation, either version 2 of the License, or
+ (at your option) any later version.
+
+ TON Blockchain Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public License
+ along with TON Blockchain Library. If not, see .
+*/
+#include "full-node-serializer.hpp"
+#include "ton/ton-tl.hpp"
+#include "tl-utils/common-utils.hpp"
+#include "auto/tl/ton_api.hpp"
+#include "tl-utils/tl-utils.hpp"
+#include "vm/boc.h"
+#include "td/utils/lz4.h"
+#include "full-node.h"
+#include "td/utils/overloaded.h"
+
+namespace ton::validator::fullnode {
+
+td::Result serialize_block_broadcast(const BlockBroadcast& broadcast, bool compression_enabled) {
+ std::vector> sigs;
+ for (auto& sig : broadcast.signatures) {
+ sigs.emplace_back(create_tl_object(sig.node, sig.signature.clone()));
+ }
+ if (!compression_enabled) {
+ return create_serialize_tl_object(
+ create_tl_block_id(broadcast.block_id), broadcast.catchain_seqno, broadcast.validator_set_hash, std::move(sigs),
+ broadcast.proof.clone(), broadcast.data.clone());
+ }
+
+ TRY_RESULT(proof_root, vm::std_boc_deserialize(broadcast.proof));
+ TRY_RESULT(data_root, vm::std_boc_deserialize(broadcast.data));
+ TRY_RESULT(boc, vm::std_boc_serialize_multi({proof_root, data_root}, 2));
+ td::BufferSlice data =
+ create_serialize_tl_object(std::move(sigs), std::move(boc));
+ td::BufferSlice compressed = td::lz4_compress(data);
+ VLOG(FULL_NODE_DEBUG) << "Compressing block broadcast: "
+ << broadcast.data.size() + broadcast.proof.size() + broadcast.signatures.size() * 96 << " -> "
+ << compressed.size();
+ return create_serialize_tl_object(
+ create_tl_block_id(broadcast.block_id), broadcast.catchain_seqno, broadcast.validator_set_hash, 0,
+ std::move(compressed));
+}
+
+static td::Result deserialize_block_broadcast(ton_api::tonNode_blockBroadcast& f) {
+ std::vector signatures;
+ for (auto& sig : f.signatures_) {
+ signatures.emplace_back(BlockSignature{sig->who_, std::move(sig->signature_)});
+ }
+ return BlockBroadcast{create_block_id(f.id_),
+ std::move(signatures),
+ static_cast(f.catchain_seqno_),
+ static_cast(f.validator_set_hash_),
+ std::move(f.data_),
+ std::move(f.proof_)};
+}
+
+static td::Result deserialize_block_broadcast(ton_api::tonNode_blockBroadcastCompressed& f,
+ int max_decompressed_size) {
+ TRY_RESULT(decompressed, td::lz4_decompress(f.compressed_, max_decompressed_size));
+ TRY_RESULT(f2, fetch_tl_object(decompressed, true));
+ std::vector signatures;
+ for (auto& sig : f2->signatures_) {
+ signatures.emplace_back(BlockSignature{sig->who_, std::move(sig->signature_)});
+ }
+ TRY_RESULT(roots, vm::std_boc_deserialize_multi(f2->proof_data_, 2));
+ if (roots.size() != 2) {
+ return td::Status::Error("expected 2 roots in boc");
+ }
+ TRY_RESULT(proof, vm::std_boc_serialize(roots[0], 0));
+ TRY_RESULT(data, vm::std_boc_serialize(roots[1], 31));
+ VLOG(FULL_NODE_DEBUG) << "Decompressing block broadcast: " << f.compressed_.size() << " -> "
+ << data.size() + proof.size() + signatures.size() * 96;
+ return BlockBroadcast{create_block_id(f.id_),
+ std::move(signatures),
+ static_cast(f.catchain_seqno_),
+ static_cast(f.validator_set_hash_),
+ std::move(data),
+ std::move(proof)};
+}
+
+td::Result deserialize_block_broadcast(ton_api::tonNode_Broadcast& obj,
+ int max_decompressed_data_size) {
+ td::Result B;
+ ton_api::downcast_call(obj,
+ td::overloaded([&](ton_api::tonNode_blockBroadcast& f) { B = deserialize_block_broadcast(f); },
+ [&](ton_api::tonNode_blockBroadcastCompressed& f) {
+ B = deserialize_block_broadcast(f, max_decompressed_data_size);
+ },
+ [&](auto&) { B = td::Status::Error("unknown broadcast type"); }));
+ return B;
+}
+
+td::Result serialize_block_full(const BlockIdExt& id, td::Slice proof, td::Slice data,
+ bool is_proof_link, bool compression_enabled) {
+ if (!compression_enabled) {
+ return create_serialize_tl_object(create_tl_block_id(id), td::BufferSlice(proof),
+ td::BufferSlice(data), is_proof_link);
+ }
+ TRY_RESULT(proof_root, vm::std_boc_deserialize(proof));
+ TRY_RESULT(data_root, vm::std_boc_deserialize(data));
+ TRY_RESULT(boc, vm::std_boc_serialize_multi({proof_root, data_root}, 2));
+ td::BufferSlice compressed = td::lz4_compress(boc);
+ VLOG(FULL_NODE_DEBUG) << "Compressing block full: " << data.size() + proof.size() << " -> " << compressed.size();
+ return create_serialize_tl_object(create_tl_block_id(id), 0,
+ std::move(compressed), is_proof_link);
+}
+
+static td::Status deserialize_block_full(ton_api::tonNode_dataFull& f, BlockIdExt& id, td::BufferSlice& proof,
+ td::BufferSlice& data, bool& is_proof_link) {
+ id = create_block_id(f.id_);
+ proof = std::move(f.proof_);
+ data = std::move(f.block_);
+ is_proof_link = f.is_link_;
+ return td::Status::OK();
+}
+
+static td::Status deserialize_block_full(ton_api::tonNode_dataFullCompressed& f, BlockIdExt& id, td::BufferSlice& proof,
+ td::BufferSlice& data, bool& is_proof_link, int max_decompressed_size) {
+ TRY_RESULT(decompressed, td::lz4_decompress(f.compressed_, max_decompressed_size));
+ TRY_RESULT(roots, vm::std_boc_deserialize_multi(decompressed, 2));
+ if (roots.size() != 2) {
+ return td::Status::Error("expected 2 roots in boc");
+ }
+ TRY_RESULT_ASSIGN(proof, vm::std_boc_serialize(roots[0], 0));
+ TRY_RESULT_ASSIGN(data, vm::std_boc_serialize(roots[1], 31));
+ VLOG(FULL_NODE_DEBUG) << "Decompressing block full: " << f.compressed_.size() << " -> " << data.size() + proof.size();
+ id = create_block_id(f.id_);
+ is_proof_link = f.is_link_;
+ return td::Status::OK();
+}
+
+td::Status deserialize_block_full(ton_api::tonNode_DataFull& obj, BlockIdExt& id, td::BufferSlice& proof,
+ td::BufferSlice& data, bool& is_proof_link, int max_decompressed_data_size) {
+ td::Status S;
+ ton_api::downcast_call(
+ obj, td::overloaded(
+ [&](ton_api::tonNode_dataFull& f) { S = deserialize_block_full(f, id, proof, data, is_proof_link); },
+ [&](ton_api::tonNode_dataFullCompressed& f) {
+ S = deserialize_block_full(f, id, proof, data, is_proof_link, max_decompressed_data_size);
+ },
+ [&](auto&) { S = td::Status::Error("unknown data type"); }));
+ return S;
+}
+
+} // namespace ton::validator::fullnode
diff --git a/validator/full-node-serializer.hpp b/validator/full-node-serializer.hpp
new file mode 100644
index 00000000..a5c73cbc
--- /dev/null
+++ b/validator/full-node-serializer.hpp
@@ -0,0 +1,31 @@
+/*
+ This file is part of TON Blockchain Library.
+
+ TON Blockchain Library is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as published by
+ the Free Software Foundation, either version 2 of the License, or
+ (at your option) any later version.
+
+ TON Blockchain Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public License
+ along with TON Blockchain Library. If not, see .
+*/
+#pragma once
+#include "ton/ton-types.h"
+#include "auto/tl/ton_api.h"
+
+namespace ton::validator::fullnode {
+
+td::Result serialize_block_broadcast(const BlockBroadcast& broadcast, bool compression_enabled);
+td::Result deserialize_block_broadcast(ton_api::tonNode_Broadcast& obj, int max_decompressed_data_size);
+
+td::Result serialize_block_full(const BlockIdExt& id, td::Slice proof, td::Slice data,
+ bool is_proof_link, bool compression_enabled);
+td::Status deserialize_block_full(ton_api::tonNode_DataFull& obj, BlockIdExt& id, td::BufferSlice& proof,
+ td::BufferSlice& data, bool& is_proof_link, int max_decompressed_data_size);
+
+} // namespace ton::validator::fullnode
diff --git a/validator/full-node-shard-queries.hpp b/validator/full-node-shard-queries.hpp
index 1a2cd716..17068229 100644
--- a/validator/full-node-shard-queries.hpp
+++ b/validator/full-node-shard-queries.hpp
@@ -20,6 +20,7 @@
#include "validator/validator.h"
#include "ton/ton-tl.hpp"
+#include "full-node-serializer.hpp"
namespace ton {
@@ -38,8 +39,8 @@ class BlockFullSender : public td::actor::Actor {
stop();
}
void finish_query() {
- promise_.set_value(create_serialize_tl_object(
- create_tl_block_id(block_id_), std::move(proof_), std::move(data_), is_proof_link_));
+ promise_.set_result(
+ serialize_block_full(block_id_, proof_, data_, is_proof_link_, false)); // compression_enabled = false
stop();
}
void start_up() override {
diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp
index 8e42716b..9f798fa8 100644
--- a/validator/full-node-shard.cpp
+++ b/validator/full-node-shard.cpp
@@ -21,6 +21,7 @@
#include "td/utils/SharedSlice.h"
#include "full-node-shard.hpp"
#include "full-node-shard-queries.hpp"
+#include "full-node-serializer.hpp"
#include "ton/ton-shard.h"
#include "ton/ton-tl.hpp"
@@ -645,19 +646,19 @@ void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_ne
}
void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query) {
- std::vector signatures;
- for (auto &sig : query.signatures_) {
- signatures.emplace_back(BlockSignature{sig->who_, std::move(sig->signature_)});
+ process_block_broadcast(src, query);
+}
+
+void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query) {
+ process_block_broadcast(src, query);
+}
+
+void FullNodeShardImpl::process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) {
+ auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size());
+ if (B.is_error()) {
+ LOG(DEBUG) << "dropped broadcast: " << B.move_as_error();
+ return;
}
-
- BlockIdExt block_id = create_block_id(query.id_);
- BlockBroadcast B{block_id,
- std::move(signatures),
- static_cast(query.catchain_seqno_),
- static_cast(query.validator_set_hash_),
- std::move(query.data_),
- std::move(query.proof_)};
-
auto P = td::PromiseCreator::lambda([](td::Result R) {
if (R.is_error()) {
if (R.error().code() == ErrorCode::notready) {
@@ -667,7 +668,7 @@ void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_bl
}
}
});
- td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::prevalidate_block, std::move(B),
+ td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::prevalidate_block, B.move_as_ok(),
std::move(P));
}
@@ -749,15 +750,13 @@ void FullNodeShardImpl::send_broadcast(BlockBroadcast broadcast) {
UNREACHABLE();
return;
}
- std::vector> sigs;
- for (auto &sig : broadcast.signatures) {
- sigs.emplace_back(create_tl_object(sig.node, sig.signature.clone()));
+ auto B = serialize_block_broadcast(broadcast, false); // compression_enabled = false
+ if (B.is_error()) {
+ VLOG(FULL_NODE_WARNING) << "failed to serialize block broadcast: " << B.move_as_error();
+ return;
}
- auto B = create_serialize_tl_object(
- create_tl_block_id(broadcast.block_id), broadcast.catchain_seqno, broadcast.validator_set_hash, std::move(sigs),
- broadcast.proof.clone(), broadcast.data.clone());
td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, adnl_id_, overlay_id_, local_id_,
- overlay::Overlays::BroadcastFlagAnySender(), std::move(B));
+ overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok());
}
void FullNodeShardImpl::download_block(BlockIdExt id, td::uint32 priority, td::Timestamp timeout,
diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp
index 0525474e..c8b5301a 100644
--- a/validator/full-node-shard.hpp
+++ b/validator/full-node-shard.hpp
@@ -71,7 +71,7 @@ class FullNodeShardImpl : public FullNodeShard {
return 2;
}
static constexpr td::uint64 proto_capabilities() {
- return 2;
+ return 3;
}
static constexpr td::uint32 max_neighbours() {
return 16;
@@ -146,6 +146,9 @@ class FullNodeShardImpl : public FullNodeShard {
void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise promise);
void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query);
+ void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query);
+ void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query);
+
void process_broadcast(PublicKeyHash src, ton_api::tonNode_ihrMessageBroadcast &query);
void process_broadcast(PublicKeyHash src, ton_api::tonNode_externalMessageBroadcast &query);
void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query);
diff --git a/validator/net/download-block-new.cpp b/validator/net/download-block-new.cpp
index 14754f64..9ec36e33 100644
--- a/validator/net/download-block-new.cpp
+++ b/validator/net/download-block-new.cpp
@@ -23,6 +23,7 @@
#include "td/utils/overloaded.h"
#include "ton/ton-io.hpp"
#include "validator/full-node.h"
+#include "full-node-serializer.hpp"
namespace ton {
@@ -219,52 +220,54 @@ void DownloadBlockNew::got_data(td::BufferSlice data) {
}
auto f = F.move_as_ok();
+ if (f->get_id() == ton_api::tonNode_dataFullEmpty::ID) {
+ abort_query(td::Status::Error(ErrorCode::notready, "node doesn't have this block"));
+ return;
+ }
+ BlockIdExt id;
+ td::BufferSlice proof, block_data;
+ bool is_link;
+ td::Status S = deserialize_block_full(*f, id, proof, block_data, is_link, overlay::Overlays::max_fec_broadcast_size());
+ if (S.is_error()) {
+ abort_query(S.move_as_error_prefix("cannot deserialize block: "));
+ return;
+ }
- ton_api::downcast_call(
- *f.get(),
- td::overloaded(
- [&](ton_api::tonNode_dataFullEmpty &x) {
- abort_query(td::Status::Error(ErrorCode::notready, "node doesn't have this block"));
- },
- [&, self = this](ton_api::tonNode_dataFull &x) {
- if (!allow_partial_proof_ && x.is_link_) {
- abort_query(td::Status::Error(ErrorCode::notready, "node doesn't have proof for this block"));
- return;
- }
- auto id = create_block_id(x.id_);
- if (block_id_.is_valid() && id != block_id_) {
- abort_query(td::Status::Error(ErrorCode::notready, "received data for wrong block"));
- return;
- }
- block_.id = id;
- block_.data = std::move(x.block_);
- if (td::sha256_bits256(block_.data.as_slice()) != id.file_hash) {
- abort_query(td::Status::Error(ErrorCode::notready, "received data with bad hash"));
- return;
- }
+ if (!allow_partial_proof_ && is_link) {
+ abort_query(td::Status::Error(ErrorCode::notready, "node doesn't have proof for this block"));
+ return;
+ }
+ if (block_id_.is_valid() && id != block_id_) {
+ abort_query(td::Status::Error(ErrorCode::notready, "received data for wrong block"));
+ return;
+ }
+ block_.id = id;
+ block_.data = std::move(block_data);
+ if (td::sha256_bits256(block_.data.as_slice()) != id.file_hash) {
+ abort_query(td::Status::Error(ErrorCode::notready, "received data with bad hash"));
+ return;
+ }
- auto P = td::PromiseCreator::lambda([SelfId = actor_id(self)](td::Result R) {
- if (R.is_error()) {
- td::actor::send_closure(SelfId, &DownloadBlockNew::abort_query,
- R.move_as_error_prefix("received bad proof: "));
- } else {
- td::actor::send_closure(SelfId, &DownloadBlockNew::checked_block_proof);
- }
- });
- if (block_id_.is_valid()) {
- if (x.is_link_) {
- td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::validate_block_proof_link,
- block_id_, std::move(x.proof_), std::move(P));
- } else {
- td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::validate_block_proof, block_id_,
- std::move(x.proof_), std::move(P));
- }
- } else {
- CHECK(!x.is_link_);
- td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::validate_block_is_next_proof,
- prev_id_, id, std::move(x.proof_), std::move(P));
- }
- }));
+ auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) {
+ if (R.is_error()) {
+ td::actor::send_closure(SelfId, &DownloadBlockNew::abort_query, R.move_as_error_prefix("received bad proof: "));
+ } else {
+ td::actor::send_closure(SelfId, &DownloadBlockNew::checked_block_proof);
+ }
+ });
+ if (block_id_.is_valid()) {
+ if (is_link) {
+ td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::validate_block_proof_link, block_id_,
+ std::move(proof), std::move(P));
+ } else {
+ td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::validate_block_proof, block_id_,
+ std::move(proof), std::move(P));
+ }
+ } else {
+ CHECK(!is_link);
+ td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::validate_block_is_next_proof, prev_id_, id,
+ std::move(proof), std::move(P));
+ }
}
void DownloadBlockNew::got_data_from_db(td::BufferSlice data) {