external/ton
1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-03-09 15:40:10 +00:00
Code Issues Releases Wiki Activity

Exp/compress candidates (#942)

Browse source 
* Compress block candidates in validator-session

* Compress blocks in full-node (disabled for now)

---------

Co-authored-by: SpyCheese <mikle98@yandex.ru>
This commit is contained in:
EmelyanenkoK 2024-03-26 14:52:46 +03:00 • committed by GitHub
parent 9452c367e4
commit 0bcebe8a0e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
21 changed files with 548 additions and 112 deletions
Download patch file Download diff file Expand all files Collapse all files

2
validator-session/CMakeLists.txt
View file

@ -5,12 +5,14 @@ if (NOT OPENSSL_FOUND)
endif()
set(VALIDATOR_SESSION_SOURCE
candidate-serializer.cpp
persistent-vector.cpp
validator-session-description.cpp
validator-session-state.cpp
validator-session.cpp
validator-session-round-attempt-state.cpp
candidate-serializer.h
persistent-vector.h
validator-session-description.h
validator-session-description.hpp

76
validator-session/candidate-serializer.cpp Normal file
View file

@ -0,0 +1,76 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "candidate-serializer.h"
#include "tl-utils/tl-utils.hpp"
#include "vm/boc.h"
#include "td/utils/lz4.h"
#include "validator-session-types.h"
namespace ton::validatorsession {
td::Result<td::BufferSlice> serialize_candidate(const tl_object_ptr<ton_api::validatorSession_candidate> &block,
bool compression_enabled) {
if (!compression_enabled) {
return serialize_tl_object(block, true);
}
vm::BagOfCells boc1, boc2;
TRY_STATUS(boc1.deserialize(block->data_));
if (boc1.get_root_count() != 1) {
return td::Status::Error("block candidate should have exactly one root");
}
std::vector<td::Ref<vm::Cell>> roots = {boc1.get_root_cell()};
TRY_STATUS(boc2.deserialize(block->collated_data_));
for (int i = 0; i < boc2.get_root_count(); ++i) {
roots.push_back(boc2.get_root_cell(i));
}
TRY_RESULT(data, vm::std_boc_serialize_multi(std::move(roots), 2));
td::BufferSlice compressed = td::lz4_compress(data);
LOG(VALIDATOR_SESSION_DEBUG) << "Compressing block candidate: " << block->data_.size() + block->collated_data_.size()
<< " -> " << compressed.size();
return create_serialize_tl_object<ton_api::validatorSession_compressedCandidate>(
0, block->src_, block->round_, block->root_hash_, (int)data.size(), std::move(compressed));
}
td::Result<tl_object_ptr<ton_api::validatorSession_candidate>> deserialize_candidate(td::Slice data,
bool compression_enabled,
int max_decompressed_data_size) {
if (!compression_enabled) {
return fetch_tl_object<ton_api::validatorSession_candidate>(data, true);
}
TRY_RESULT(f, fetch_tl_object<ton_api::validatorSession_compressedCandidate>(data, true));
if (f->decompressed_size_ > max_decompressed_data_size) {
return td::Status::Error("decompressed size is too big");
}
TRY_RESULT(decompressed, td::lz4_decompress(f->data_, f->decompressed_size_));
if (decompressed.size() != (size_t)f->decompressed_size_) {
return td::Status::Error("decompressed size mismatch");
}
TRY_RESULT(roots, vm::std_boc_deserialize_multi(decompressed));
if (roots.empty()) {
return td::Status::Error("boc is empty");
}
TRY_RESULT(block_data, vm::std_boc_serialize(roots[0], 31));
roots.erase(roots.begin());
TRY_RESULT(collated_data, vm::std_boc_serialize_multi(std::move(roots), 31));
LOG(VALIDATOR_SESSION_DEBUG) << "Decompressing block candidate: " << f->data_.size() << " -> "
<< block_data.size() + collated_data.size();
return create_tl_object<ton_api::validatorSession_candidate>(f->src_, f->round_, f->root_hash_, std::move(block_data),
std::move(collated_data));
}
} // namespace ton::validatorsession

29
validator-session/candidate-serializer.h Normal file
View file

@ -0,0 +1,29 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "ton/ton-types.h"
#include "auto/tl/ton_api.h"
namespace ton::validatorsession {
td::Result<td::BufferSlice> serialize_candidate(const tl_object_ptr<ton_api::validatorSession_candidate> &block,
bool compression_enabled);
td::Result<tl_object_ptr<ton_api::validatorSession_candidate>> deserialize_candidate(td::Slice data,
bool compression_enabled,
int max_decompressed_data_size);
} // namespace ton::validatorsession

35
validator-session/validator-session.cpp
View file

@ -19,6 +19,7 @@
#include "validator-session.hpp"
#include "td/utils/Random.h"
#include "td/utils/crypto.h"
#include "candidate-serializer.h"
namespace ton {
@ -221,7 +222,9 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice
// Note: src is not necessarily equal to the sender of this message:
// If requested using get_broadcast_p2p, src is the creator of the block, sender possibly is some other node.
auto src_idx = description().get_source_idx(src);
auto R = fetch_tl_object<ton_api::validatorSession_candidate>(data.clone(), true);
auto R =
deserialize_candidate(data, compress_block_candidates_,
description().opts().max_block_size + description().opts().max_collated_data_size + 1024);
if (R.is_error()) {
VLOG(VALIDATOR_SESSION_WARNING) << this << "[node " << src << "][broadcast " << sha256_bits256(data.as_slice())
<< "]: failed to parse: " << R.move_as_error();
@ -343,17 +346,17 @@ void ValidatorSessionImpl::process_query(PublicKeyHash src, td::BufferSlice data
}
CHECK(block);
auto P = td::PromiseCreator::lambda(
[promise = std::move(promise), src = f->id_->src_, round_id](td::Result<BlockCandidate> R) mutable {
if (R.is_error()) {
promise.set_error(R.move_as_error_prefix("failed to get candidate: "));
} else {
auto c = R.move_as_ok();
auto obj = create_tl_object<ton_api::validatorSession_candidate>(
src, round_id, c.id.root_hash, std::move(c.data), std::move(c.collated_data));
promise.set_value(serialize_tl_object(obj, true));
}
});
auto P = td::PromiseCreator::lambda([promise = std::move(promise), src = f->id_->src_, round_id,
compress = compress_block_candidates_](td::Result<BlockCandidate> R) mutable {
if (R.is_error()) {
promise.set_error(R.move_as_error_prefix("failed to get candidate: "));
} else {
auto c = R.move_as_ok();
auto obj = create_tl_object<ton_api::validatorSession_candidate>(src, round_id, c.id.root_hash, std::move(c.data),
std::move(c.collated_data));
promise.set_result(serialize_candidate(obj, compress));
}
});
callback_->get_approved_candidate(description().get_source_public_key(block->get_src_idx()), f->id_->root_hash_,
f->id_->file_hash_, f->id_->collated_data_file_hash_, std::move(P));
@ -431,7 +434,7 @@ void ValidatorSessionImpl::generated_block(td::uint32 round, ValidatorSessionCan
auto b = create_tl_object<ton_api::validatorSession_candidate>(local_id().tl(), round, root_hash, std::move(data),
std::move(collated_data));
auto B = serialize_tl_object(b, true);
auto B = serialize_candidate(b, compress_block_candidates_).move_as_ok();
auto block_id = description().candidate_id(local_idx(), root_hash, file_hash, collated_data_file_hash);
@ -862,7 +865,8 @@ void ValidatorSessionImpl::on_catchain_started() {
if (x) {
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), round = virtual_state_->cur_round_seqno(),
src = description().get_source_id(x->get_src_idx()),
root_hash = x->get_root_hash()](td::Result<BlockCandidate> R) {
root_hash = x->get_root_hash(),
compress = compress_block_candidates_](td::Result<BlockCandidate> R) {
if (R.is_error()) {
LOG(ERROR) << "failed to get candidate: " << R.move_as_error();
} else {
@ -870,7 +874,7 @@ void ValidatorSessionImpl::on_catchain_started() {
auto broadcast = create_tl_object<ton_api::validatorSession_candidate>(
src.tl(), round, root_hash, std::move(B.data), std::move(B.collated_data));
td::actor::send_closure(SelfId, &ValidatorSessionImpl::process_broadcast, src,
serialize_tl_object(broadcast, true), td::optional<ValidatorSessionCandidateId>(),
serialize_candidate(broadcast, compress).move_as_ok(), td::optional<ValidatorSessionCandidateId>(),
false);
}
});
@ -898,6 +902,7 @@ ValidatorSessionImpl::ValidatorSessionImpl(catchain::CatChainSessionId session_i
, rldp_(rldp)
, overlay_manager_(overlays)
, allow_unsafe_self_blocks_resync_(allow_unsafe_self_blocks_resync) {
compress_block_candidates_ = opts.proto_version >= 3;
description_ = ValidatorSessionDescription::create(std::move(opts), nodes, local_id);
src_round_candidate_.resize(description_->get_total_nodes());
}

1
validator-session/validator-session.hpp
View file

@ -156,6 +156,7 @@ class ValidatorSessionImpl : public ValidatorSession {
bool started_ = false;
bool catchain_started_ = false;
bool allow_unsafe_self_blocks_resync_;
bool compress_block_candidates_ = false;
ValidatorSessionStats cur_stats_;
void stats_init();