1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-03-09 15:40:10 +00:00

Exp/compress candidates (#942)

* Compress block candidates in validator-session

* Compress blocks in full-node (disabled for now)

---------

Co-authored-by: SpyCheese <mikle98@yandex.ru>
This commit is contained in:
EmelyanenkoK 2024-03-26 14:52:46 +03:00 committed by GitHub
parent 9452c367e4
commit 0bcebe8a0e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
21 changed files with 548 additions and 112 deletions

View file

@ -146,6 +146,8 @@ set(FULL_NODE_SOURCE
full-node-master.cpp
full-node-private-overlay.hpp
full-node-private-overlay.cpp
full-node-serializer.hpp
full-node-serializer.cpp
net/download-block.hpp
net/download-block.cpp

View file

@ -14,11 +14,10 @@
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "full-node-private-overlay.hpp"
#include "ton/ton-tl.hpp"
#include "common/delay.h"
#include "full-node-serializer.hpp"
namespace ton {
@ -26,20 +25,20 @@ namespace validator {
namespace fullnode {
void FullNodePrivateOverlay::process_broadcast(PublicKeyHash, ton_api::tonNode_blockBroadcast &query) {
std::vector<BlockSignature> signatures;
for (auto &sig : query.signatures_) {
signatures.emplace_back(BlockSignature{sig->who_, std::move(sig->signature_)});
void FullNodePrivateOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query) {
process_block_broadcast(src, query);
}
void FullNodePrivateOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query) {
process_block_broadcast(src, query);
}
void FullNodePrivateOverlay::process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) {
auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size());
if (B.is_error()) {
LOG(DEBUG) << "dropped broadcast: " << B.move_as_error();
return;
}
BlockIdExt block_id = create_block_id(query.id_);
BlockBroadcast B{block_id,
std::move(signatures),
static_cast<UnixTime>(query.catchain_seqno_),
static_cast<td::uint32>(query.validator_set_hash_),
std::move(query.data_),
std::move(query.proof_)};
auto P = td::PromiseCreator::lambda([](td::Result<td::Unit> R) {
if (R.is_error()) {
if (R.error().code() == ErrorCode::notready) {
@ -49,7 +48,7 @@ void FullNodePrivateOverlay::process_broadcast(PublicKeyHash, ton_api::tonNode_b
}
}
});
td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::prevalidate_block, std::move(B),
td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::prevalidate_block, B.move_as_ok(),
std::move(P));
}
@ -87,15 +86,13 @@ void FullNodePrivateOverlay::send_broadcast(BlockBroadcast broadcast) {
if (!inited_) {
return;
}
std::vector<tl_object_ptr<ton_api::tonNode_blockSignature>> sigs;
for (auto &sig : broadcast.signatures) {
sigs.emplace_back(create_tl_object<ton_api::tonNode_blockSignature>(sig.node, sig.signature.clone()));
auto B = serialize_block_broadcast(broadcast, false); // compression_enabled = false
if (B.is_error()) {
VLOG(FULL_NODE_WARNING) << "failed to serialize block broadcast: " << B.move_as_error();
return;
}
auto B = create_serialize_tl_object<ton_api::tonNode_blockBroadcast>(
create_tl_block_id(broadcast.block_id), broadcast.catchain_seqno, broadcast.validator_set_hash, std::move(sigs),
broadcast.proof.clone(), broadcast.data.clone());
td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_,
local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), std::move(B));
local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok());
}
void FullNodePrivateOverlay::start_up() {

View file

@ -27,6 +27,9 @@ namespace fullnode {
class FullNodePrivateOverlay : public td::actor::Actor {
public:
void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query);
void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query);
void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query);
void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query);
template <class T>
void process_broadcast(PublicKeyHash, T &) {

View file

@ -0,0 +1,155 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
*/
#include "full-node-serializer.hpp"
#include "ton/ton-tl.hpp"
#include "tl-utils/common-utils.hpp"
#include "auto/tl/ton_api.hpp"
#include "tl-utils/tl-utils.hpp"
#include "vm/boc.h"
#include "td/utils/lz4.h"
#include "full-node.h"
#include "td/utils/overloaded.h"
namespace ton::validator::fullnode {
td::Result<td::BufferSlice> serialize_block_broadcast(const BlockBroadcast& broadcast, bool compression_enabled) {
std::vector<tl_object_ptr<ton_api::tonNode_blockSignature>> sigs;
for (auto& sig : broadcast.signatures) {
sigs.emplace_back(create_tl_object<ton_api::tonNode_blockSignature>(sig.node, sig.signature.clone()));
}
if (!compression_enabled) {
return create_serialize_tl_object<ton_api::tonNode_blockBroadcast>(
create_tl_block_id(broadcast.block_id), broadcast.catchain_seqno, broadcast.validator_set_hash, std::move(sigs),
broadcast.proof.clone(), broadcast.data.clone());
}
TRY_RESULT(proof_root, vm::std_boc_deserialize(broadcast.proof));
TRY_RESULT(data_root, vm::std_boc_deserialize(broadcast.data));
TRY_RESULT(boc, vm::std_boc_serialize_multi({proof_root, data_root}, 2));
td::BufferSlice data =
create_serialize_tl_object<ton_api::tonNode_blockBroadcastCompressed_data>(std::move(sigs), std::move(boc));
td::BufferSlice compressed = td::lz4_compress(data);
VLOG(FULL_NODE_DEBUG) << "Compressing block broadcast: "
<< broadcast.data.size() + broadcast.proof.size() + broadcast.signatures.size() * 96 << " -> "
<< compressed.size();
return create_serialize_tl_object<ton_api::tonNode_blockBroadcastCompressed>(
create_tl_block_id(broadcast.block_id), broadcast.catchain_seqno, broadcast.validator_set_hash, 0,
std::move(compressed));
}
static td::Result<BlockBroadcast> deserialize_block_broadcast(ton_api::tonNode_blockBroadcast& f) {
std::vector<BlockSignature> signatures;
for (auto& sig : f.signatures_) {
signatures.emplace_back(BlockSignature{sig->who_, std::move(sig->signature_)});
}
return BlockBroadcast{create_block_id(f.id_),
std::move(signatures),
static_cast<UnixTime>(f.catchain_seqno_),
static_cast<td::uint32>(f.validator_set_hash_),
std::move(f.data_),
std::move(f.proof_)};
}
static td::Result<BlockBroadcast> deserialize_block_broadcast(ton_api::tonNode_blockBroadcastCompressed& f,
int max_decompressed_size) {
TRY_RESULT(decompressed, td::lz4_decompress(f.compressed_, max_decompressed_size));
TRY_RESULT(f2, fetch_tl_object<ton_api::tonNode_blockBroadcastCompressed_data>(decompressed, true));
std::vector<BlockSignature> signatures;
for (auto& sig : f2->signatures_) {
signatures.emplace_back(BlockSignature{sig->who_, std::move(sig->signature_)});
}
TRY_RESULT(roots, vm::std_boc_deserialize_multi(f2->proof_data_, 2));
if (roots.size() != 2) {
return td::Status::Error("expected 2 roots in boc");
}
TRY_RESULT(proof, vm::std_boc_serialize(roots[0], 0));
TRY_RESULT(data, vm::std_boc_serialize(roots[1], 31));
VLOG(FULL_NODE_DEBUG) << "Decompressing block broadcast: " << f.compressed_.size() << " -> "
<< data.size() + proof.size() + signatures.size() * 96;
return BlockBroadcast{create_block_id(f.id_),
std::move(signatures),
static_cast<UnixTime>(f.catchain_seqno_),
static_cast<td::uint32>(f.validator_set_hash_),
std::move(data),
std::move(proof)};
}
td::Result<BlockBroadcast> deserialize_block_broadcast(ton_api::tonNode_Broadcast& obj,
int max_decompressed_data_size) {
td::Result<BlockBroadcast> B;
ton_api::downcast_call(obj,
td::overloaded([&](ton_api::tonNode_blockBroadcast& f) { B = deserialize_block_broadcast(f); },
[&](ton_api::tonNode_blockBroadcastCompressed& f) {
B = deserialize_block_broadcast(f, max_decompressed_data_size);
},
[&](auto&) { B = td::Status::Error("unknown broadcast type"); }));
return B;
}
td::Result<td::BufferSlice> serialize_block_full(const BlockIdExt& id, td::Slice proof, td::Slice data,
bool is_proof_link, bool compression_enabled) {
if (!compression_enabled) {
return create_serialize_tl_object<ton_api::tonNode_dataFull>(create_tl_block_id(id), td::BufferSlice(proof),
td::BufferSlice(data), is_proof_link);
}
TRY_RESULT(proof_root, vm::std_boc_deserialize(proof));
TRY_RESULT(data_root, vm::std_boc_deserialize(data));
TRY_RESULT(boc, vm::std_boc_serialize_multi({proof_root, data_root}, 2));
td::BufferSlice compressed = td::lz4_compress(boc);
VLOG(FULL_NODE_DEBUG) << "Compressing block full: " << data.size() + proof.size() << " -> " << compressed.size();
return create_serialize_tl_object<ton_api::tonNode_dataFullCompressed>(create_tl_block_id(id), 0,
std::move(compressed), is_proof_link);
}
static td::Status deserialize_block_full(ton_api::tonNode_dataFull& f, BlockIdExt& id, td::BufferSlice& proof,
td::BufferSlice& data, bool& is_proof_link) {
id = create_block_id(f.id_);
proof = std::move(f.proof_);
data = std::move(f.block_);
is_proof_link = f.is_link_;
return td::Status::OK();
}
static td::Status deserialize_block_full(ton_api::tonNode_dataFullCompressed& f, BlockIdExt& id, td::BufferSlice& proof,
td::BufferSlice& data, bool& is_proof_link, int max_decompressed_size) {
TRY_RESULT(decompressed, td::lz4_decompress(f.compressed_, max_decompressed_size));
TRY_RESULT(roots, vm::std_boc_deserialize_multi(decompressed, 2));
if (roots.size() != 2) {
return td::Status::Error("expected 2 roots in boc");
}
TRY_RESULT_ASSIGN(proof, vm::std_boc_serialize(roots[0], 0));
TRY_RESULT_ASSIGN(data, vm::std_boc_serialize(roots[1], 31));
VLOG(FULL_NODE_DEBUG) << "Decompressing block full: " << f.compressed_.size() << " -> " << data.size() + proof.size();
id = create_block_id(f.id_);
is_proof_link = f.is_link_;
return td::Status::OK();
}
td::Status deserialize_block_full(ton_api::tonNode_DataFull& obj, BlockIdExt& id, td::BufferSlice& proof,
td::BufferSlice& data, bool& is_proof_link, int max_decompressed_data_size) {
td::Status S;
ton_api::downcast_call(
obj, td::overloaded(
[&](ton_api::tonNode_dataFull& f) { S = deserialize_block_full(f, id, proof, data, is_proof_link); },
[&](ton_api::tonNode_dataFullCompressed& f) {
S = deserialize_block_full(f, id, proof, data, is_proof_link, max_decompressed_data_size);
},
[&](auto&) { S = td::Status::Error("unknown data type"); }));
return S;
}
} // namespace ton::validator::fullnode

View file

@ -0,0 +1,31 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "ton/ton-types.h"
#include "auto/tl/ton_api.h"
namespace ton::validator::fullnode {
td::Result<td::BufferSlice> serialize_block_broadcast(const BlockBroadcast& broadcast, bool compression_enabled);
td::Result<BlockBroadcast> deserialize_block_broadcast(ton_api::tonNode_Broadcast& obj, int max_decompressed_data_size);
td::Result<td::BufferSlice> serialize_block_full(const BlockIdExt& id, td::Slice proof, td::Slice data,
bool is_proof_link, bool compression_enabled);
td::Status deserialize_block_full(ton_api::tonNode_DataFull& obj, BlockIdExt& id, td::BufferSlice& proof,
td::BufferSlice& data, bool& is_proof_link, int max_decompressed_data_size);
} // namespace ton::validator::fullnode

View file

@ -20,6 +20,7 @@
#include "validator/validator.h"
#include "ton/ton-tl.hpp"
#include "full-node-serializer.hpp"
namespace ton {
@ -38,8 +39,8 @@ class BlockFullSender : public td::actor::Actor {
stop();
}
void finish_query() {
promise_.set_value(create_serialize_tl_object<ton_api::tonNode_dataFull>(
create_tl_block_id(block_id_), std::move(proof_), std::move(data_), is_proof_link_));
promise_.set_result(
serialize_block_full(block_id_, proof_, data_, is_proof_link_, false)); // compression_enabled = false
stop();
}
void start_up() override {

View file

@ -21,6 +21,7 @@
#include "td/utils/SharedSlice.h"
#include "full-node-shard.hpp"
#include "full-node-shard-queries.hpp"
#include "full-node-serializer.hpp"
#include "ton/ton-shard.h"
#include "ton/ton-tl.hpp"
@ -645,19 +646,19 @@ void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_ne
}
void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query) {
std::vector<BlockSignature> signatures;
for (auto &sig : query.signatures_) {
signatures.emplace_back(BlockSignature{sig->who_, std::move(sig->signature_)});
process_block_broadcast(src, query);
}
void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query) {
process_block_broadcast(src, query);
}
void FullNodeShardImpl::process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) {
auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size());
if (B.is_error()) {
LOG(DEBUG) << "dropped broadcast: " << B.move_as_error();
return;
}
BlockIdExt block_id = create_block_id(query.id_);
BlockBroadcast B{block_id,
std::move(signatures),
static_cast<UnixTime>(query.catchain_seqno_),
static_cast<td::uint32>(query.validator_set_hash_),
std::move(query.data_),
std::move(query.proof_)};
auto P = td::PromiseCreator::lambda([](td::Result<td::Unit> R) {
if (R.is_error()) {
if (R.error().code() == ErrorCode::notready) {
@ -667,7 +668,7 @@ void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_bl
}
}
});
td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::prevalidate_block, std::move(B),
td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::prevalidate_block, B.move_as_ok(),
std::move(P));
}
@ -749,15 +750,13 @@ void FullNodeShardImpl::send_broadcast(BlockBroadcast broadcast) {
UNREACHABLE();
return;
}
std::vector<tl_object_ptr<ton_api::tonNode_blockSignature>> sigs;
for (auto &sig : broadcast.signatures) {
sigs.emplace_back(create_tl_object<ton_api::tonNode_blockSignature>(sig.node, sig.signature.clone()));
auto B = serialize_block_broadcast(broadcast, false); // compression_enabled = false
if (B.is_error()) {
VLOG(FULL_NODE_WARNING) << "failed to serialize block broadcast: " << B.move_as_error();
return;
}
auto B = create_serialize_tl_object<ton_api::tonNode_blockBroadcast>(
create_tl_block_id(broadcast.block_id), broadcast.catchain_seqno, broadcast.validator_set_hash, std::move(sigs),
broadcast.proof.clone(), broadcast.data.clone());
td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, adnl_id_, overlay_id_, local_id_,
overlay::Overlays::BroadcastFlagAnySender(), std::move(B));
overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok());
}
void FullNodeShardImpl::download_block(BlockIdExt id, td::uint32 priority, td::Timestamp timeout,

View file

@ -71,7 +71,7 @@ class FullNodeShardImpl : public FullNodeShard {
return 2;
}
static constexpr td::uint64 proto_capabilities() {
return 2;
return 3;
}
static constexpr td::uint32 max_neighbours() {
return 16;
@ -146,6 +146,9 @@ class FullNodeShardImpl : public FullNodeShard {
void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise<td::BufferSlice> promise);
void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query);
void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query);
void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query);
void process_broadcast(PublicKeyHash src, ton_api::tonNode_ihrMessageBroadcast &query);
void process_broadcast(PublicKeyHash src, ton_api::tonNode_externalMessageBroadcast &query);
void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query);

View file

@ -23,6 +23,7 @@
#include "td/utils/overloaded.h"
#include "ton/ton-io.hpp"
#include "validator/full-node.h"
#include "full-node-serializer.hpp"
namespace ton {
@ -219,52 +220,54 @@ void DownloadBlockNew::got_data(td::BufferSlice data) {
}
auto f = F.move_as_ok();
if (f->get_id() == ton_api::tonNode_dataFullEmpty::ID) {
abort_query(td::Status::Error(ErrorCode::notready, "node doesn't have this block"));
return;
}
BlockIdExt id;
td::BufferSlice proof, block_data;
bool is_link;
td::Status S = deserialize_block_full(*f, id, proof, block_data, is_link, overlay::Overlays::max_fec_broadcast_size());
if (S.is_error()) {
abort_query(S.move_as_error_prefix("cannot deserialize block: "));
return;
}
ton_api::downcast_call(
*f.get(),
td::overloaded(
[&](ton_api::tonNode_dataFullEmpty &x) {
abort_query(td::Status::Error(ErrorCode::notready, "node doesn't have this block"));
},
[&, self = this](ton_api::tonNode_dataFull &x) {
if (!allow_partial_proof_ && x.is_link_) {
abort_query(td::Status::Error(ErrorCode::notready, "node doesn't have proof for this block"));
return;
}
auto id = create_block_id(x.id_);
if (block_id_.is_valid() && id != block_id_) {
abort_query(td::Status::Error(ErrorCode::notready, "received data for wrong block"));
return;
}
block_.id = id;
block_.data = std::move(x.block_);
if (td::sha256_bits256(block_.data.as_slice()) != id.file_hash) {
abort_query(td::Status::Error(ErrorCode::notready, "received data with bad hash"));
return;
}
if (!allow_partial_proof_ && is_link) {
abort_query(td::Status::Error(ErrorCode::notready, "node doesn't have proof for this block"));
return;
}
if (block_id_.is_valid() && id != block_id_) {
abort_query(td::Status::Error(ErrorCode::notready, "received data for wrong block"));
return;
}
block_.id = id;
block_.data = std::move(block_data);
if (td::sha256_bits256(block_.data.as_slice()) != id.file_hash) {
abort_query(td::Status::Error(ErrorCode::notready, "received data with bad hash"));
return;
}
auto P = td::PromiseCreator::lambda([SelfId = actor_id(self)](td::Result<td::Unit> R) {
if (R.is_error()) {
td::actor::send_closure(SelfId, &DownloadBlockNew::abort_query,
R.move_as_error_prefix("received bad proof: "));
} else {
td::actor::send_closure(SelfId, &DownloadBlockNew::checked_block_proof);
}
});
if (block_id_.is_valid()) {
if (x.is_link_) {
td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::validate_block_proof_link,
block_id_, std::move(x.proof_), std::move(P));
} else {
td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::validate_block_proof, block_id_,
std::move(x.proof_), std::move(P));
}
} else {
CHECK(!x.is_link_);
td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::validate_block_is_next_proof,
prev_id_, id, std::move(x.proof_), std::move(P));
}
}));
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Unit> R) {
if (R.is_error()) {
td::actor::send_closure(SelfId, &DownloadBlockNew::abort_query, R.move_as_error_prefix("received bad proof: "));
} else {
td::actor::send_closure(SelfId, &DownloadBlockNew::checked_block_proof);
}
});
if (block_id_.is_valid()) {
if (is_link) {
td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::validate_block_proof_link, block_id_,
std::move(proof), std::move(P));
} else {
td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::validate_block_proof, block_id_,
std::move(proof), std::move(P));
}
} else {
CHECK(!is_link);
td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::validate_block_is_next_proof, prev_id_, id,
std::move(proof), std::move(P));
}
}
void DownloadBlockNew::got_data_from_db(td::BufferSlice data) {