Exp/compress candidates (#942)

* Compress block candidates in validator-session

* Compress blocks in full-node (disabled for now)

---------

Co-authored-by: SpyCheese <mikle98@yandex.ru>

CMake/FindLZ4.cmake

@@ -0,0 +1,39 @@
+###############################################################################
+# Find LZ4
+#
+# This sets the following variables:
+# LZ4_FOUND        - True if LZ4 was found.
+# LZ4_INCLUDE_DIRS - Directories containing the LZ4 include files.
+# LZ4_LIBRARIES    - Libraries needed to use LZ4.
+# LZ4_LIBRARY      - Library needed to use LZ4.
+# LZ4_LIBRARY_DIRS - Library needed to use LZ4.
+
+find_package(PkgConfig REQUIRED)
+
+# If found, LZ$_* variables will be defined
+pkg_check_modules(LZ4 REQUIRED liblz4)
+
+if(NOT LZ4_FOUND)
+  find_path(LZ4_INCLUDE_DIR lz4.h
+    HINTS "${LZ4_ROOT}" "$ENV{LZ4_ROOT}"
+    PATHS "$ENV{PROGRAMFILES}/lz4" "$ENV{PROGRAMW6432}/lz4"
+    PATH_SUFFIXES include)
+
+  find_library(LZ4_LIBRARY
+    NAMES lz4 lz4_static
+    HINTS "${LZ4_ROOT}" "$ENV{LZ4_ROOT}"
+    PATHS "$ENV{PROGRAMFILES}/lz4" "$ENV{PROGRAMW6432}/lz4"
+    PATH_SUFFIXES lib)
+
+  if(LZ4_LIBRARY)
+    set(LZ4_LIBRARIES ${LZ4_LIBRARY})
+    get_filename_component(LZ4_LIBRARY_DIRS ${LZ4_LIBRARY} DIRECTORY)
+  endif()
+else()
+  find_library(LZ4_LIBRARY
+    NAMES lz4 lz4_static
+    PATHS ${LZ4_LIBRARY_DIRS}
+    NO_DEFAULT_PATH)
+endif()
+
+mark_as_advanced(LZ4_LIBRARY LZ4_INCLUDE_DIRS LZ4_LIBRARY_DIRS LZ4_LIBRARIES)

tdutils/CMakeLists.txt

@@ -15,6 +15,7 @@ if (NOT DEFINED CMAKE_INSTALL_LIBDIR)
 endif()
 
 find_package(PkgConfig REQUIRED)
+find_package(LZ4)
 if (NOT ZLIB_FOUND)
   pkg_check_modules(ZLIB zlib)
 endif()
@@ -280,6 +281,15 @@ if (TDUTILS_MIME_TYPE)
   )
 endif()
 
+if (LZ4_FOUND)
+  set(TD_HAVE_LZ4 1)
+  set(TDUTILS_SOURCE
+    ${TDUTILS_SOURCE}
+    td/utils/lz4.cpp
+    td/utils/lz4.h
+    )
+endif()
+
 set(TDUTILS_TEST_SOURCE
   ${CMAKE_CURRENT_SOURCE_DIR}/test/buffer.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/test/ConcurrentHashMap.cpp
@@ -338,6 +348,11 @@ endif()
 if (CRC32C_FOUND)
   target_link_libraries(tdutils PRIVATE crc32c)
 endif()
+
+if (LZ4_FOUND)
+  target_link_libraries(tdutils PRIVATE ${LZ4_LIBRARIES})
+endif()
+
 if (ABSL_FOUND)
   target_link_libraries_system(tdutils absl::flat_hash_map absl::flat_hash_set absl::hash)
 endif()

tdutils/td/utils/config.h.in

@@ -3,6 +3,7 @@
 #cmakedefine01 TD_HAVE_OPENSSL
 #cmakedefine01 TD_HAVE_ZLIB
 #cmakedefine01 TD_HAVE_CRC32C
+#cmakedefine01 TD_HAVE_LZ4
 #cmakedefine01 TD_HAVE_COROUTINES
 #cmakedefine01 TD_HAVE_ABSL
 #cmakedefine01 TD_FD_DEBUG

tdutils/td/utils/lz4.cpp

@@ -0,0 +1,48 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+*/
+#include "td/utils/buffer.h"
+#include "td/utils/misc.h"
+#include <lz4.h>
+
+namespace td {
+
+td::BufferSlice lz4_compress(td::Slice data) {
+  int size = narrow_cast<int>(data.size());
+  int buf_size = LZ4_compressBound(size);
+  td::BufferSlice compressed(buf_size);
+  int compressed_size = LZ4_compress_default(data.data(), compressed.data(), size, buf_size);
+  CHECK(compressed_size > 0);
+  return td::BufferSlice{compressed.as_slice().substr(0, compressed_size)};
+}
+
+td::Result<td::BufferSlice> lz4_decompress(td::Slice data, int max_decompressed_size) {
+  TRY_RESULT(size, narrow_cast_safe<int>(data.size()));
+  if (max_decompressed_size < 0) {
+    return td::Status::Error("invalid max_decompressed_size");
+  }
+  td::BufferSlice decompressed(max_decompressed_size);
+  int result = LZ4_decompress_safe(data.data(), decompressed.data(), size, max_decompressed_size);
+  if (result < 0) {
+    return td::Status::Error(PSTRING() << "lz4 decompression failed, error code: " << result);
+  }
+  if (result == max_decompressed_size) {
+    return decompressed;
+  }
+  return td::BufferSlice{decompressed.as_slice().substr(0, result)};
+}
+
+}  // namespace td

tdutils/td/utils/lz4.h

@@ -0,0 +1,27 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+*/
+#pragma once
+
+#include "td/utils/buffer.h"
+#include "td/utils/Status.h"
+
+namespace td {
+
+td::BufferSlice lz4_compress(td::Slice data);
+td::Result<td::BufferSlice> lz4_decompress(td::Slice data, int max_decompressed_size);
+
+}  // namespace td

tl/generate/scheme/ton_api.tl

@@ -309,6 +309,7 @@ validatorSession.candidateId src:int256 root_hash:int256 file_hash:int256 collat
 
 validatorSession.blockUpdate ts:long actions:(vector validatorSession.round.Message) state:int = validatorSession.BlockUpdate;
 validatorSession.candidate src:int256 round:int root_hash:int256 data:bytes collated_data:bytes = validatorSession.Candidate;
+validatorSession.compressedCandidate flags:# src:int256 round:int root_hash:int256 decompressed_size:int data:bytes = validatorSession.Candidate;
 
 validatorSession.config catchain_idle_timeout:double catchain_max_deps:int round_candidates:int next_candidate_delay:double round_attempt_duration:int
         max_round_attempts:int max_block_size:int max_collated_data_size:int = validatorSession.Config;
@@ -385,9 +386,13 @@ tonNode.externalMessage data:bytes = tonNode.ExternalMessage;
 
 tonNode.newShardBlock block:tonNode.blockIdExt cc_seqno:int data:bytes = tonNode.NewShardBlock;
 
+tonNode.blockBroadcastCompressed.data signatures:(vector tonNode.blockSignature) proof_data:bytes = tonNode.blockBroadcaseCompressed.Data;
+
 tonNode.blockBroadcast id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int 
               signatures:(vector tonNode.blockSignature) 
               proof:bytes data:bytes = tonNode.Broadcast;
+tonNode.blockBroadcastCompressed id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int
+              flags:# compressed:bytes = tonNode.Broadcast;
 tonNode.ihrMessageBroadcast message:tonNode.ihrMessage = tonNode.Broadcast;
 tonNode.externalMessageBroadcast message:tonNode.externalMessage = tonNode.Broadcast;
 tonNode.newShardBlockBroadcast block:tonNode.newShardBlock = tonNode.Broadcast;
@@ -401,9 +406,8 @@ tonNode.keyBlocks blocks:(vector tonNode.blockIdExt) incomplete:Bool error:Bool
 ton.blockId root_cell_hash:int256 file_hash:int256 = ton.BlockId;
 ton.blockIdApprove root_cell_hash:int256 file_hash:int256 = ton.BlockId;
 
-tonNode.dataList data:(vector bytes) = tonNode.DataList;
-
 tonNode.dataFull id:tonNode.blockIdExt proof:bytes block:bytes is_link:Bool = tonNode.DataFull;
+tonNode.dataFullCompressed id:tonNode.blockIdExt flags:# compressed:bytes is_link:Bool = tonNode.DataFull;
 tonNode.dataFullEmpty = tonNode.DataFull;
 
 tonNode.capabilities version:int capabilities:long = tonNode.Capabilities;
@@ -430,18 +434,13 @@ tonNode.getNextKeyBlockIds block:tonNode.blockIdExt max_size:int = tonNode.KeyBl
 tonNode.downloadNextBlockFull prev_block:tonNode.blockIdExt = tonNode.DataFull;
 tonNode.downloadBlockFull block:tonNode.blockIdExt = tonNode.DataFull;
 tonNode.downloadBlock block:tonNode.blockIdExt = tonNode.Data;
-tonNode.downloadBlocks blocks:(vector tonNode.blockIdExt) = tonNode.DataList;
 tonNode.downloadPersistentState block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt = tonNode.Data;
 tonNode.downloadPersistentStateSlice block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt offset:long max_size:long = tonNode.Data;
 tonNode.downloadZeroState block:tonNode.blockIdExt = tonNode.Data;
 tonNode.downloadBlockProof block:tonNode.blockIdExt = tonNode.Data;
 tonNode.downloadKeyBlockProof block:tonNode.blockIdExt = tonNode.Data;
-tonNode.downloadBlockProofs blocks:(vector tonNode.blockIdExt) = tonNode.DataList;
-tonNode.downloadKeyBlockProofs blocks:(vector tonNode.blockIdExt) = tonNode.DataList;
 tonNode.downloadBlockProofLink block:tonNode.blockIdExt = tonNode.Data;
 tonNode.downloadKeyBlockProofLink block:tonNode.blockIdExt = tonNode.Data;
-tonNode.downloadBlockProofLinks blocks:(vector tonNode.blockIdExt) = tonNode.DataList;
-tonNode.downloadKeyBlockProofLinks blocks:(vector tonNode.blockIdExt) = tonNode.DataList;
 tonNode.getArchiveInfo masterchain_seqno:int = tonNode.ArchiveInfo;
 tonNode.getArchiveSlice archive_id:long offset:long max_size:int = tonNode.Data;
 

validator-session/CMakeLists.txt

@@ -5,12 +5,14 @@ if (NOT OPENSSL_FOUND)
 endif()
 
 set(VALIDATOR_SESSION_SOURCE
+  candidate-serializer.cpp
   persistent-vector.cpp
   validator-session-description.cpp
   validator-session-state.cpp
   validator-session.cpp
   validator-session-round-attempt-state.cpp
 
+  candidate-serializer.h
   persistent-vector.h
   validator-session-description.h
   validator-session-description.hpp

validator-session/candidate-serializer.cpp

@@ -0,0 +1,76 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+*/
+#pragma once
+#include "candidate-serializer.h"
+#include "tl-utils/tl-utils.hpp"
+#include "vm/boc.h"
+#include "td/utils/lz4.h"
+#include "validator-session-types.h"
+
+namespace ton::validatorsession {
+
+td::Result<td::BufferSlice> serialize_candidate(const tl_object_ptr<ton_api::validatorSession_candidate> &block,
+                                                bool compression_enabled) {
+  if (!compression_enabled) {
+    return serialize_tl_object(block, true);
+  }
+  vm::BagOfCells boc1, boc2;
+  TRY_STATUS(boc1.deserialize(block->data_));
+  if (boc1.get_root_count() != 1) {
+    return td::Status::Error("block candidate should have exactly one root");
+  }
+  std::vector<td::Ref<vm::Cell>> roots = {boc1.get_root_cell()};
+  TRY_STATUS(boc2.deserialize(block->collated_data_));
+  for (int i = 0; i < boc2.get_root_count(); ++i) {
+    roots.push_back(boc2.get_root_cell(i));
+  }
+  TRY_RESULT(data, vm::std_boc_serialize_multi(std::move(roots), 2));
+  td::BufferSlice compressed = td::lz4_compress(data);
+  LOG(VALIDATOR_SESSION_DEBUG) << "Compressing block candidate: " << block->data_.size() + block->collated_data_.size()
+                               << " -> " << compressed.size();
+  return create_serialize_tl_object<ton_api::validatorSession_compressedCandidate>(
+      0, block->src_, block->round_, block->root_hash_, (int)data.size(), std::move(compressed));
+}
+
+td::Result<tl_object_ptr<ton_api::validatorSession_candidate>> deserialize_candidate(td::Slice data,
+                                                                                     bool compression_enabled,
+                                                                                     int max_decompressed_data_size) {
+  if (!compression_enabled) {
+    return fetch_tl_object<ton_api::validatorSession_candidate>(data, true);
+  }
+  TRY_RESULT(f, fetch_tl_object<ton_api::validatorSession_compressedCandidate>(data, true));
+  if (f->decompressed_size_ > max_decompressed_data_size) {
+    return td::Status::Error("decompressed size is too big");
+  }
+  TRY_RESULT(decompressed, td::lz4_decompress(f->data_, f->decompressed_size_));
+  if (decompressed.size() != (size_t)f->decompressed_size_) {
+    return td::Status::Error("decompressed size mismatch");
+  }
+  TRY_RESULT(roots, vm::std_boc_deserialize_multi(decompressed));
+  if (roots.empty()) {
+    return td::Status::Error("boc is empty");
+  }
+  TRY_RESULT(block_data, vm::std_boc_serialize(roots[0], 31));
+  roots.erase(roots.begin());
+  TRY_RESULT(collated_data, vm::std_boc_serialize_multi(std::move(roots), 31));
+  LOG(VALIDATOR_SESSION_DEBUG) << "Decompressing block candidate: " << f->data_.size() << " -> "
+                               << block_data.size() + collated_data.size();
+  return create_tl_object<ton_api::validatorSession_candidate>(f->src_, f->round_, f->root_hash_, std::move(block_data),
+                                                               std::move(collated_data));
+}
+
+}  // namespace ton::validatorsession

validator-session/candidate-serializer.h

@@ -0,0 +1,29 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+*/
+#pragma once
+#include "ton/ton-types.h"
+#include "auto/tl/ton_api.h"
+
+namespace ton::validatorsession {
+
+td::Result<td::BufferSlice> serialize_candidate(const tl_object_ptr<ton_api::validatorSession_candidate> &block,
+                                                bool compression_enabled);
+td::Result<tl_object_ptr<ton_api::validatorSession_candidate>> deserialize_candidate(td::Slice data,
+                                                                                     bool compression_enabled,
+                                                                                     int max_decompressed_data_size);
+
+}  // namespace ton::validatorsession

validator-session/validator-session.cpp

@@ -19,6 +19,7 @@
 #include "validator-session.hpp"
 #include "td/utils/Random.h"
 #include "td/utils/crypto.h"
+#include "candidate-serializer.h"
 
 namespace ton {
 
@@ -221,7 +222,9 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice
   // Note: src is not necessarily equal to the sender of this message:
   // If requested using get_broadcast_p2p, src is the creator of the block, sender possibly is some other node.
   auto src_idx = description().get_source_idx(src);
-  auto R = fetch_tl_object<ton_api::validatorSession_candidate>(data.clone(), true);
+  auto R =
+      deserialize_candidate(data, compress_block_candidates_,
+                            description().opts().max_block_size + description().opts().max_collated_data_size + 1024);
   if (R.is_error()) {
     VLOG(VALIDATOR_SESSION_WARNING) << this << "[node " << src << "][broadcast " << sha256_bits256(data.as_slice())
                                     << "]: failed to parse: " << R.move_as_error();
@@ -343,17 +346,17 @@ void ValidatorSessionImpl::process_query(PublicKeyHash src, td::BufferSlice data
   }
   CHECK(block);
 
-  auto P = td::PromiseCreator::lambda(
+  auto P = td::PromiseCreator::lambda([promise = std::move(promise), src = f->id_->src_, round_id,
-      [promise = std::move(promise), src = f->id_->src_, round_id](td::Result<BlockCandidate> R) mutable {
+                                       compress = compress_block_candidates_](td::Result<BlockCandidate> R) mutable {
-        if (R.is_error()) {
+    if (R.is_error()) {
-          promise.set_error(R.move_as_error_prefix("failed to get candidate: "));
+      promise.set_error(R.move_as_error_prefix("failed to get candidate: "));
-        } else {
+    } else {
-          auto c = R.move_as_ok();
+      auto c = R.move_as_ok();
-          auto obj = create_tl_object<ton_api::validatorSession_candidate>(
+      auto obj = create_tl_object<ton_api::validatorSession_candidate>(src, round_id, c.id.root_hash, std::move(c.data),
-              src, round_id, c.id.root_hash, std::move(c.data), std::move(c.collated_data));
+                                                                       std::move(c.collated_data));
-          promise.set_value(serialize_tl_object(obj, true));
+      promise.set_result(serialize_candidate(obj, compress));
-        }
+    }
-      });
+  });
 
   callback_->get_approved_candidate(description().get_source_public_key(block->get_src_idx()), f->id_->root_hash_,
                                     f->id_->file_hash_, f->id_->collated_data_file_hash_, std::move(P));
@@ -431,7 +434,7 @@ void ValidatorSessionImpl::generated_block(td::uint32 round, ValidatorSessionCan
   auto b = create_tl_object<ton_api::validatorSession_candidate>(local_id().tl(), round, root_hash, std::move(data),
                                                                  std::move(collated_data));
 
-  auto B = serialize_tl_object(b, true);
+  auto B = serialize_candidate(b, compress_block_candidates_).move_as_ok();
 
   auto block_id = description().candidate_id(local_idx(), root_hash, file_hash, collated_data_file_hash);
 
@@ -862,7 +865,8 @@ void ValidatorSessionImpl::on_catchain_started() {
     if (x) {
       auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), round = virtual_state_->cur_round_seqno(),
                                            src = description().get_source_id(x->get_src_idx()),
-                                           root_hash = x->get_root_hash()](td::Result<BlockCandidate> R) {
+                                           root_hash = x->get_root_hash(),
+                                           compress = compress_block_candidates_](td::Result<BlockCandidate> R) {
         if (R.is_error()) {
           LOG(ERROR) << "failed to get candidate: " << R.move_as_error();
         } else {
@@ -870,7 +874,7 @@ void ValidatorSessionImpl::on_catchain_started() {
           auto broadcast = create_tl_object<ton_api::validatorSession_candidate>(
               src.tl(), round, root_hash, std::move(B.data), std::move(B.collated_data));
           td::actor::send_closure(SelfId, &ValidatorSessionImpl::process_broadcast, src,
-                                  serialize_tl_object(broadcast, true), td::optional<ValidatorSessionCandidateId>(),
+                                  serialize_candidate(broadcast, compress).move_as_ok(), td::optional<ValidatorSessionCandidateId>(),
                                   false);
         }
       });
@@ -898,6 +902,7 @@ ValidatorSessionImpl::ValidatorSessionImpl(catchain::CatChainSessionId session_i
     , rldp_(rldp)
     , overlay_manager_(overlays)
     , allow_unsafe_self_blocks_resync_(allow_unsafe_self_blocks_resync) {
+  compress_block_candidates_ = opts.proto_version >= 3;
   description_ = ValidatorSessionDescription::create(std::move(opts), nodes, local_id);
   src_round_candidate_.resize(description_->get_total_nodes());
 }

validator-session/validator-session.hpp

@@ -156,6 +156,7 @@ class ValidatorSessionImpl : public ValidatorSession {
   bool started_ = false;
   bool catchain_started_ = false;
   bool allow_unsafe_self_blocks_resync_;
+  bool compress_block_candidates_ = false;
 
   ValidatorSessionStats cur_stats_;
   void stats_init();

validator/CMakeLists.txt

@@ -146,6 +146,8 @@ set(FULL_NODE_SOURCE
   full-node-master.cpp
   full-node-private-overlay.hpp
   full-node-private-overlay.cpp
+  full-node-serializer.hpp
+  full-node-serializer.cpp
 
   net/download-block.hpp
   net/download-block.cpp

validator/full-node-private-overlay.cpp

@@ -14,11 +14,10 @@
     You should have received a copy of the GNU Lesser General Public License
     along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
 */
-#pragma once
-
 #include "full-node-private-overlay.hpp"
 #include "ton/ton-tl.hpp"
 #include "common/delay.h"
+#include "full-node-serializer.hpp"
 
 namespace ton {
 
@@ -26,20 +25,20 @@ namespace validator {
 
 namespace fullnode {
 
-void FullNodePrivateOverlay::process_broadcast(PublicKeyHash, ton_api::tonNode_blockBroadcast &query) {
+void FullNodePrivateOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query) {
-  std::vector<BlockSignature> signatures;
+  process_block_broadcast(src, query);
-  for (auto &sig : query.signatures_) {
+}
-    signatures.emplace_back(BlockSignature{sig->who_, std::move(sig->signature_)});
+
+void FullNodePrivateOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query) {
+  process_block_broadcast(src, query);
+}
+
+void FullNodePrivateOverlay::process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) {
+  auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size());
+  if (B.is_error()) {
+    LOG(DEBUG) << "dropped broadcast: " << B.move_as_error();
+    return;
   }
-
-  BlockIdExt block_id = create_block_id(query.id_);
-  BlockBroadcast B{block_id,
-                   std::move(signatures),
-                   static_cast<UnixTime>(query.catchain_seqno_),
-                   static_cast<td::uint32>(query.validator_set_hash_),
-                   std::move(query.data_),
-                   std::move(query.proof_)};
-
   auto P = td::PromiseCreator::lambda([](td::Result<td::Unit> R) {
     if (R.is_error()) {
       if (R.error().code() == ErrorCode::notready) {
@@ -49,7 +48,7 @@ void FullNodePrivateOverlay::process_broadcast(PublicKeyHash, ton_api::tonNode_b
       }
     }
   });
-  td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::prevalidate_block, std::move(B),
+  td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::prevalidate_block, B.move_as_ok(),
                           std::move(P));
 }
 
@@ -87,15 +86,13 @@ void FullNodePrivateOverlay::send_broadcast(BlockBroadcast broadcast) {
   if (!inited_) {
     return;
   }
-  std::vector<tl_object_ptr<ton_api::tonNode_blockSignature>> sigs;
+  auto B = serialize_block_broadcast(broadcast, false);  // compression_enabled = false
-  for (auto &sig : broadcast.signatures) {
+  if (B.is_error()) {
-    sigs.emplace_back(create_tl_object<ton_api::tonNode_blockSignature>(sig.node, sig.signature.clone()));
+    VLOG(FULL_NODE_WARNING) << "failed to serialize block broadcast: " << B.move_as_error();
+    return;
   }
-  auto B = create_serialize_tl_object<ton_api::tonNode_blockBroadcast>(
-      create_tl_block_id(broadcast.block_id), broadcast.catchain_seqno, broadcast.validator_set_hash, std::move(sigs),
-      broadcast.proof.clone(), broadcast.data.clone());
   td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_,
-                          local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), std::move(B));
+                          local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok());
 }
 
 void FullNodePrivateOverlay::start_up() {

validator/full-node-private-overlay.hpp

@@ -27,6 +27,9 @@ namespace fullnode {
 class FullNodePrivateOverlay : public td::actor::Actor {
  public:
   void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query);
+  void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query);
+  void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query);
+
   void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query);
   template <class T>
   void process_broadcast(PublicKeyHash, T &) {

validator/full-node-serializer.cpp

@@ -0,0 +1,155 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+*/
+#include "full-node-serializer.hpp"
+#include "ton/ton-tl.hpp"
+#include "tl-utils/common-utils.hpp"
+#include "auto/tl/ton_api.hpp"
+#include "tl-utils/tl-utils.hpp"
+#include "vm/boc.h"
+#include "td/utils/lz4.h"
+#include "full-node.h"
+#include "td/utils/overloaded.h"
+
+namespace ton::validator::fullnode {
+
+td::Result<td::BufferSlice> serialize_block_broadcast(const BlockBroadcast& broadcast, bool compression_enabled) {
+  std::vector<tl_object_ptr<ton_api::tonNode_blockSignature>> sigs;
+  for (auto& sig : broadcast.signatures) {
+    sigs.emplace_back(create_tl_object<ton_api::tonNode_blockSignature>(sig.node, sig.signature.clone()));
+  }
+  if (!compression_enabled) {
+    return create_serialize_tl_object<ton_api::tonNode_blockBroadcast>(
+        create_tl_block_id(broadcast.block_id), broadcast.catchain_seqno, broadcast.validator_set_hash, std::move(sigs),
+        broadcast.proof.clone(), broadcast.data.clone());
+  }
+
+  TRY_RESULT(proof_root, vm::std_boc_deserialize(broadcast.proof));
+  TRY_RESULT(data_root, vm::std_boc_deserialize(broadcast.data));
+  TRY_RESULT(boc, vm::std_boc_serialize_multi({proof_root, data_root}, 2));
+  td::BufferSlice data =
+      create_serialize_tl_object<ton_api::tonNode_blockBroadcastCompressed_data>(std::move(sigs), std::move(boc));
+  td::BufferSlice compressed = td::lz4_compress(data);
+  VLOG(FULL_NODE_DEBUG) << "Compressing block broadcast: "
+                        << broadcast.data.size() + broadcast.proof.size() + broadcast.signatures.size() * 96 << " -> "
+                        << compressed.size();
+  return create_serialize_tl_object<ton_api::tonNode_blockBroadcastCompressed>(
+      create_tl_block_id(broadcast.block_id), broadcast.catchain_seqno, broadcast.validator_set_hash, 0,
+      std::move(compressed));
+}
+
+static td::Result<BlockBroadcast> deserialize_block_broadcast(ton_api::tonNode_blockBroadcast& f) {
+  std::vector<BlockSignature> signatures;
+  for (auto& sig : f.signatures_) {
+    signatures.emplace_back(BlockSignature{sig->who_, std::move(sig->signature_)});
+  }
+  return BlockBroadcast{create_block_id(f.id_),
+                        std::move(signatures),
+                        static_cast<UnixTime>(f.catchain_seqno_),
+                        static_cast<td::uint32>(f.validator_set_hash_),
+                        std::move(f.data_),
+                        std::move(f.proof_)};
+}
+
+static td::Result<BlockBroadcast> deserialize_block_broadcast(ton_api::tonNode_blockBroadcastCompressed& f,
+                                                              int max_decompressed_size) {
+  TRY_RESULT(decompressed, td::lz4_decompress(f.compressed_, max_decompressed_size));
+  TRY_RESULT(f2, fetch_tl_object<ton_api::tonNode_blockBroadcastCompressed_data>(decompressed, true));
+  std::vector<BlockSignature> signatures;
+  for (auto& sig : f2->signatures_) {
+    signatures.emplace_back(BlockSignature{sig->who_, std::move(sig->signature_)});
+  }
+  TRY_RESULT(roots, vm::std_boc_deserialize_multi(f2->proof_data_, 2));
+  if (roots.size() != 2) {
+    return td::Status::Error("expected 2 roots in boc");
+  }
+  TRY_RESULT(proof, vm::std_boc_serialize(roots[0], 0));
+  TRY_RESULT(data, vm::std_boc_serialize(roots[1], 31));
+  VLOG(FULL_NODE_DEBUG) << "Decompressing block broadcast: " << f.compressed_.size() << " -> "
+                        << data.size() + proof.size() + signatures.size() * 96;
+  return BlockBroadcast{create_block_id(f.id_),
+                        std::move(signatures),
+                        static_cast<UnixTime>(f.catchain_seqno_),
+                        static_cast<td::uint32>(f.validator_set_hash_),
+                        std::move(data),
+                        std::move(proof)};
+}
+
+td::Result<BlockBroadcast> deserialize_block_broadcast(ton_api::tonNode_Broadcast& obj,
+                                                       int max_decompressed_data_size) {
+  td::Result<BlockBroadcast> B;
+  ton_api::downcast_call(obj,
+                         td::overloaded([&](ton_api::tonNode_blockBroadcast& f) { B = deserialize_block_broadcast(f); },
+                                        [&](ton_api::tonNode_blockBroadcastCompressed& f) {
+                                          B = deserialize_block_broadcast(f, max_decompressed_data_size);
+                                        },
+                                        [&](auto&) { B = td::Status::Error("unknown broadcast type"); }));
+  return B;
+}
+
+td::Result<td::BufferSlice> serialize_block_full(const BlockIdExt& id, td::Slice proof, td::Slice data,
+                                                 bool is_proof_link, bool compression_enabled) {
+  if (!compression_enabled) {
+    return create_serialize_tl_object<ton_api::tonNode_dataFull>(create_tl_block_id(id), td::BufferSlice(proof),
+                                                                 td::BufferSlice(data), is_proof_link);
+  }
+  TRY_RESULT(proof_root, vm::std_boc_deserialize(proof));
+  TRY_RESULT(data_root, vm::std_boc_deserialize(data));
+  TRY_RESULT(boc, vm::std_boc_serialize_multi({proof_root, data_root}, 2));
+  td::BufferSlice compressed = td::lz4_compress(boc);
+  VLOG(FULL_NODE_DEBUG) << "Compressing block full: " << data.size() + proof.size() << " -> " << compressed.size();
+  return create_serialize_tl_object<ton_api::tonNode_dataFullCompressed>(create_tl_block_id(id), 0,
+                                                                         std::move(compressed), is_proof_link);
+}
+
+static td::Status deserialize_block_full(ton_api::tonNode_dataFull& f, BlockIdExt& id, td::BufferSlice& proof,
+                                         td::BufferSlice& data, bool& is_proof_link) {
+  id = create_block_id(f.id_);
+  proof = std::move(f.proof_);
+  data = std::move(f.block_);
+  is_proof_link = f.is_link_;
+  return td::Status::OK();
+}
+
+static td::Status deserialize_block_full(ton_api::tonNode_dataFullCompressed& f, BlockIdExt& id, td::BufferSlice& proof,
+                                         td::BufferSlice& data, bool& is_proof_link, int max_decompressed_size) {
+  TRY_RESULT(decompressed, td::lz4_decompress(f.compressed_, max_decompressed_size));
+  TRY_RESULT(roots, vm::std_boc_deserialize_multi(decompressed, 2));
+  if (roots.size() != 2) {
+    return td::Status::Error("expected 2 roots in boc");
+  }
+  TRY_RESULT_ASSIGN(proof, vm::std_boc_serialize(roots[0], 0));
+  TRY_RESULT_ASSIGN(data, vm::std_boc_serialize(roots[1], 31));
+  VLOG(FULL_NODE_DEBUG) << "Decompressing block full: " << f.compressed_.size() << " -> " << data.size() + proof.size();
+  id = create_block_id(f.id_);
+  is_proof_link = f.is_link_;
+  return td::Status::OK();
+}
+
+td::Status deserialize_block_full(ton_api::tonNode_DataFull& obj, BlockIdExt& id, td::BufferSlice& proof,
+                                  td::BufferSlice& data, bool& is_proof_link, int max_decompressed_data_size) {
+  td::Status S;
+  ton_api::downcast_call(
+      obj, td::overloaded(
+               [&](ton_api::tonNode_dataFull& f) { S = deserialize_block_full(f, id, proof, data, is_proof_link); },
+               [&](ton_api::tonNode_dataFullCompressed& f) {
+                 S = deserialize_block_full(f, id, proof, data, is_proof_link, max_decompressed_data_size);
+               },
+               [&](auto&) { S = td::Status::Error("unknown data type"); }));
+  return S;
+}
+
+}  // namespace ton::validator::fullnode

validator/full-node-serializer.hpp

@@ -0,0 +1,31 @@
+/*
+    This file is part of TON Blockchain Library.
+
+    TON Blockchain Library is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TON Blockchain Library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
+*/
+#pragma once
+#include "ton/ton-types.h"
+#include "auto/tl/ton_api.h"
+
+namespace ton::validator::fullnode {
+
+td::Result<td::BufferSlice> serialize_block_broadcast(const BlockBroadcast& broadcast, bool compression_enabled);
+td::Result<BlockBroadcast> deserialize_block_broadcast(ton_api::tonNode_Broadcast& obj, int max_decompressed_data_size);
+
+td::Result<td::BufferSlice> serialize_block_full(const BlockIdExt& id, td::Slice proof, td::Slice data,
+                                                 bool is_proof_link, bool compression_enabled);
+td::Status deserialize_block_full(ton_api::tonNode_DataFull& obj, BlockIdExt& id, td::BufferSlice& proof,
+                                  td::BufferSlice& data, bool& is_proof_link, int max_decompressed_data_size);
+
+}  // namespace ton::validator::fullnode

validator/full-node-shard-queries.hpp

@@ -20,6 +20,7 @@
 
 #include "validator/validator.h"
 #include "ton/ton-tl.hpp"
+#include "full-node-serializer.hpp"
 
 namespace ton {
 
@@ -38,8 +39,8 @@ class BlockFullSender : public td::actor::Actor {
     stop();
   }
   void finish_query() {
-    promise_.set_value(create_serialize_tl_object<ton_api::tonNode_dataFull>(
+    promise_.set_result(
-        create_tl_block_id(block_id_), std::move(proof_), std::move(data_), is_proof_link_));
+        serialize_block_full(block_id_, proof_, data_, is_proof_link_, false));  // compression_enabled = false
     stop();
   }
   void start_up() override {

validator/full-node-shard.cpp

@@ -21,6 +21,7 @@
 #include "td/utils/SharedSlice.h"
 #include "full-node-shard.hpp"
 #include "full-node-shard-queries.hpp"
+#include "full-node-serializer.hpp"
 
 #include "ton/ton-shard.h"
 #include "ton/ton-tl.hpp"
@@ -645,19 +646,19 @@ void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_ne
 }
 
 void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query) {
-  std::vector<BlockSignature> signatures;
+  process_block_broadcast(src, query);
-  for (auto &sig : query.signatures_) {
+}
-    signatures.emplace_back(BlockSignature{sig->who_, std::move(sig->signature_)});
+
+void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query) {
+  process_block_broadcast(src, query);
+}
+
+void FullNodeShardImpl::process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) {
+  auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size());
+  if (B.is_error()) {
+    LOG(DEBUG) << "dropped broadcast: " << B.move_as_error();
+    return;
   }
-
-  BlockIdExt block_id = create_block_id(query.id_);
-  BlockBroadcast B{block_id,
-                   std::move(signatures),
-                   static_cast<UnixTime>(query.catchain_seqno_),
-                   static_cast<td::uint32>(query.validator_set_hash_),
-                   std::move(query.data_),
-                   std::move(query.proof_)};
-
   auto P = td::PromiseCreator::lambda([](td::Result<td::Unit> R) {
     if (R.is_error()) {
       if (R.error().code() == ErrorCode::notready) {
@@ -667,7 +668,7 @@ void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_bl
       }
     }
   });
-  td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::prevalidate_block, std::move(B),
+  td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::prevalidate_block, B.move_as_ok(),
                           std::move(P));
 }
 
@@ -749,15 +750,13 @@ void FullNodeShardImpl::send_broadcast(BlockBroadcast broadcast) {
     UNREACHABLE();
     return;
   }
-  std::vector<tl_object_ptr<ton_api::tonNode_blockSignature>> sigs;
+  auto B = serialize_block_broadcast(broadcast, false);  // compression_enabled = false
-  for (auto &sig : broadcast.signatures) {
+  if (B.is_error()) {
-    sigs.emplace_back(create_tl_object<ton_api::tonNode_blockSignature>(sig.node, sig.signature.clone()));
+    VLOG(FULL_NODE_WARNING) << "failed to serialize block broadcast: " << B.move_as_error();
+    return;
   }
-  auto B = create_serialize_tl_object<ton_api::tonNode_blockBroadcast>(
-      create_tl_block_id(broadcast.block_id), broadcast.catchain_seqno, broadcast.validator_set_hash, std::move(sigs),
-      broadcast.proof.clone(), broadcast.data.clone());
   td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, adnl_id_, overlay_id_, local_id_,
-                          overlay::Overlays::BroadcastFlagAnySender(), std::move(B));
+                          overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok());
 }
 
 void FullNodeShardImpl::download_block(BlockIdExt id, td::uint32 priority, td::Timestamp timeout,

validator/full-node-shard.hpp

@@ -71,7 +71,7 @@ class FullNodeShardImpl : public FullNodeShard {
     return 2;
   }
   static constexpr td::uint64 proto_capabilities() {
-    return 2;
+    return 3;
   }
   static constexpr td::uint32 max_neighbours() {
     return 16;
@@ -146,6 +146,9 @@ class FullNodeShardImpl : public FullNodeShard {
   void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise<td::BufferSlice> promise);
 
   void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query);
+  void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query);
+  void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query);
+
   void process_broadcast(PublicKeyHash src, ton_api::tonNode_ihrMessageBroadcast &query);
   void process_broadcast(PublicKeyHash src, ton_api::tonNode_externalMessageBroadcast &query);
   void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query);

validator/net/download-block-new.cpp

@@ -23,6 +23,7 @@
 #include "td/utils/overloaded.h"
 #include "ton/ton-io.hpp"
 #include "validator/full-node.h"
+#include "full-node-serializer.hpp"
 
 namespace ton {
 
@@ -219,52 +220,54 @@ void DownloadBlockNew::got_data(td::BufferSlice data) {
   }
 
   auto f = F.move_as_ok();
+  if (f->get_id() == ton_api::tonNode_dataFullEmpty::ID) {
+    abort_query(td::Status::Error(ErrorCode::notready, "node doesn't have this block"));
+    return;
+  }
+  BlockIdExt id;
+  td::BufferSlice proof, block_data;
+  bool is_link;
+  td::Status S = deserialize_block_full(*f, id, proof, block_data, is_link, overlay::Overlays::max_fec_broadcast_size());
+  if (S.is_error()) {
+    abort_query(S.move_as_error_prefix("cannot deserialize block: "));
+    return;
+  }
 
-  ton_api::downcast_call(
+  if (!allow_partial_proof_ && is_link) {
-      *f.get(),
+    abort_query(td::Status::Error(ErrorCode::notready, "node doesn't have proof for this block"));
-      td::overloaded(
+    return;
-          [&](ton_api::tonNode_dataFullEmpty &x) {
+  }
-            abort_query(td::Status::Error(ErrorCode::notready, "node doesn't have this block"));
+  if (block_id_.is_valid() && id != block_id_) {
-          },
+    abort_query(td::Status::Error(ErrorCode::notready, "received data for wrong block"));
-          [&, self = this](ton_api::tonNode_dataFull &x) {
+    return;
-            if (!allow_partial_proof_ && x.is_link_) {
+  }
-              abort_query(td::Status::Error(ErrorCode::notready, "node doesn't have proof for this block"));
+  block_.id = id;
-              return;
+  block_.data = std::move(block_data);
-            }
+  if (td::sha256_bits256(block_.data.as_slice()) != id.file_hash) {
-            auto id = create_block_id(x.id_);
+    abort_query(td::Status::Error(ErrorCode::notready, "received data with bad hash"));
-            if (block_id_.is_valid() && id != block_id_) {
+    return;
-              abort_query(td::Status::Error(ErrorCode::notready, "received data for wrong block"));
+  }
-              return;
-            }
-            block_.id = id;
-            block_.data = std::move(x.block_);
-            if (td::sha256_bits256(block_.data.as_slice()) != id.file_hash) {
-              abort_query(td::Status::Error(ErrorCode::notready, "received data with bad hash"));
-              return;
-            }
 
-            auto P = td::PromiseCreator::lambda([SelfId = actor_id(self)](td::Result<td::Unit> R) {
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Unit> R) {
-              if (R.is_error()) {
+    if (R.is_error()) {
-                td::actor::send_closure(SelfId, &DownloadBlockNew::abort_query,
+      td::actor::send_closure(SelfId, &DownloadBlockNew::abort_query, R.move_as_error_prefix("received bad proof: "));
-                                        R.move_as_error_prefix("received bad proof: "));
+    } else {
-              } else {
+      td::actor::send_closure(SelfId, &DownloadBlockNew::checked_block_proof);
-                td::actor::send_closure(SelfId, &DownloadBlockNew::checked_block_proof);
+    }
-              }
+  });
-            });
+  if (block_id_.is_valid()) {
-            if (block_id_.is_valid()) {
+    if (is_link) {
-              if (x.is_link_) {
+      td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::validate_block_proof_link, block_id_,
-                td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::validate_block_proof_link,
+                              std::move(proof), std::move(P));
-                                        block_id_, std::move(x.proof_), std::move(P));
+    } else {
-              } else {
+      td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::validate_block_proof, block_id_,
-                td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::validate_block_proof, block_id_,
+                              std::move(proof), std::move(P));
-                                        std::move(x.proof_), std::move(P));
+    }
-              }
+  } else {
-            } else {
+    CHECK(!is_link);
-              CHECK(!x.is_link_);
+    td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::validate_block_is_next_proof, prev_id_, id,
-              td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::validate_block_is_next_proof,
+                            std::move(proof), std::move(P));
-                                      prev_id_, id, std::move(x.proof_), std::move(P));
+  }
-            }
-          }));
 }
 
 void DownloadBlockNew::got_data_from_db(td::BufferSlice data) {