external/ton
1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-02-12 19:22:37 +00:00
Code Issues Releases Wiki Activity

Merge pull request #433 from ton-blockchain/testnet

Browse source 
Merge node updates
This commit is contained in:
EmelyanenkoK 2022-08-08 12:48:00 +03:00 committed by GitHub
commit ce65245a69
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
92 changed files with 1959 additions and 751 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.github/workflows/docker-ubuntu-image.yml vendored
View file

@ -3,8 +3,6 @@ name: Docker Ubuntu 18.04 image
on:
workflow_dispatch:
push:
branches:
- 'master'
env:
REGISTRY: ghcr.io
@ -36,4 +34,4 @@ jobs:
with:
push: true
context: ./docker
tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest

4
.github/workflows/macos-10.15-compile.yml vendored
View file

@ -3,8 +3,6 @@ name: C/C++ CI macOS-10.15 Compile
on:
workflow_dispatch:
push:
branches:
- 'master'
jobs:
build:
@ -40,4 +38,4 @@ jobs:
uses: actions/upload-artifact@master
with:
name: ton-macos-binaries
path: artifacts
path: artifacts

4
.github/workflows/ubuntu-18.04-ton-ccpcheck.yml vendored
View file

@ -3,8 +3,6 @@ name: Ubuntu 18.04 TON ccpcheck
on:
workflow_dispatch:
push:
branches:
- 'master'
jobs:
build:
@ -28,4 +26,4 @@ jobs:
uses: actions/upload-artifact@v1
with:
name: ton-ccpcheck-report
path: output
path: output

40
.github/workflows/ubuntu-compile.yml vendored Normal file
View file

@ -0,0 +1,40 @@
name: Ubuntu Compile
on: [push,workflow_dispatch]
jobs:
build:
strategy:
fail-fast: false
matrix:
os: [ubuntu-18.04, ubuntu-20.04, ubuntu-22.04]
runs-on: ${{ matrix.os }}
steps:
- name: Check out repository
uses: actions/checkout@v2
with:
submodules: 'recursive'
- name: Install libraries
run: |
sudo apt update
sudo apt install -y build-essential git make cmake clang libgflags-dev zlib1g-dev libssl-dev libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev ninja-build
- name: Configure & Build
run: |
export CC=$(which clang)
export CXX=$(which clang++)
export CCACHE_DISABLE=1
mkdir build-${{ matrix.os }}
cd build-${{ matrix.os }}
cmake -GNinja -DCMAKE_BUILD_TYPE=Release ..
ninja fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork
- name: find & copy binaries
run: |
mkdir artifacts-${{ matrix.os }}
cp build-${{ matrix.os }}/crypto/fift build-${{ matrix.os }}/crypto/tlbc build-${{ matrix.os }}/crypto/func build-${{ matrix.os }}/crypto/create-state build-${{ matrix.os }}/validator-engine-console/validator-engine-console build-${{ matrix.os }}/tonlib/tonlib-cli build-${{ matrix.os }}/tonlib/libtonlibjson.so.0.5 build-${{ matrix.os }}/http/http-proxy build-${{ matrix.os }}/rldp-http-proxy/rldp-http-proxy build-${{ matrix.os }}/dht-server/dht-server build-${{ matrix.os }}/lite-client/lite-client build-${{ matrix.os }}/validator-engine/validator-engine build-${{ matrix.os }}/utils/generate-random-id build-${{ matrix.os }}/utils/json2tlo build-${{ matrix.os }}/adnl/adnl-proxy artifacts-${{ matrix.os }}
- name: Upload artifacts
uses: actions/upload-artifact@master
with:
name: ton-binaries-${{ matrix.os }}
path: artifacts-${{ matrix.os }}

4
.github/workflows/windows2019x64-compile.yml vendored
View file

@ -1,10 +1,8 @@
name: C/C++ CI Windows Server 2019 x64 Compile
name: Windows Server 2019 x64 Compile
on:
workflow_dispatch:
push:
branches:
- 'master'
defaults:
run:

3
.gitignore vendored
View file

@ -11,4 +11,5 @@ crypto/smartcont/auto/
test/regression-tests.cache/
*.swp
**/*build*/
.idea
.idea
.vscode

31
CMakeLists.txt
View file

@ -1,4 +1,4 @@
cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR)
cmake_minimum_required(VERSION 3.1 FATAL_ERROR)
project(TON VERSION 0.5 LANGUAGES C CXX)
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
@ -79,6 +79,10 @@ else()
set(HAVE_SSE42 FALSE)
endif()
set(CMAKE_CXX_STANDARD 14)
set(CMAKE_CXX_STANDARD_REQUIRED TRUE)
set(CMAKE_CXX_EXTENSIONS FALSE)
#BEGIN internal
option(TON_ONLY_TONLIB "Use \"ON\" to build only tonlib." OFF)
if (TON_ONLY_TONLIB)
@ -107,6 +111,7 @@ endif()
if (TON_USE_ABSEIL)
message("Add abseil-cpp")
set(ABSL_PROPAGATE_CXX_STD TRUE)
add_subdirectory(third-party/abseil-cpp EXCLUDE_FROM_ALL)
set(ABSL_FOUND 1)
endif()
@ -186,26 +191,6 @@ endif()
include(CheckCXXCompilerFlag)
if (GCC OR CLANG OR INTEL)
if (WIN32 AND INTEL)
set(STD14_FLAG /Qstd=c++14)
else()
set(STD14_FLAG -std=c++14)
endif()
check_cxx_compiler_flag(${STD14_FLAG} HAVE_STD14)
if (NOT HAVE_STD14)
string(REPLACE "c++14" "c++1y" STD14_FLAG "${STD14_FLAG}")
check_cxx_compiler_flag(${STD14_FLAG} HAVE_STD1Y)
set(HAVE_STD14 ${HAVE_STD1Y})
endif()
elseif (MSVC)
set(HAVE_STD14 MSVC_VERSION>=1900)
endif()
if (NOT HAVE_STD14)
message(FATAL_ERROR "No C++14 support in the compiler. Please upgrade the compiler.")
endif()
set(CMAKE_THREAD_PREFER_PTHREAD ON)
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)
@ -250,7 +235,7 @@ if (MSVC)
add_definitions(-D_SCL_SECURE_NO_WARNINGS -D_CRT_SECURE_NO_WARNINGS)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /MP /W4 /wd4100 /wd4127 /wd4324 /wd4456 /wd4457 /wd4458 /wd4505 /wd4702")
elseif (CLANG OR GCC)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${STD14_FLAG} -fno-omit-frame-pointer")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer")
if (APPLE)
#use "-Wl,-exported_symbols_list,${CMAKE_CURRENT_SOURCE_DIR}/export_list" for exported symbols
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fvisibility=hidden -Wl,-dead_strip,-x,-S")
@ -263,8 +248,6 @@ elseif (CLANG OR GCC)
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--exclude-libs,ALL")
endif()
endif()
elseif (INTEL)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${STD14_FLAG}")
endif()
if (WIN32)

6
adnl/adnl-address-list.cpp
View file

@ -39,6 +39,9 @@ class AdnlNetworkConnectionUdp : public AdnlNetworkConnection {
void start_up() override {
callback_->on_change_state(true);
}
void get_ip_str(td::Promise<td::string> promise) override {
promise.set_value(PSTRING() << addr_.get_ip_str().str() << ":" << addr_.get_port());
}
AdnlNetworkConnectionUdp(td::actor::ActorId<AdnlNetworkManager> network_manager, td::uint32 ip, td::uint16 port,
std::unique_ptr<AdnlNetworkConnection::Callback> callback);
@ -88,6 +91,9 @@ class AdnlNetworkConnectionTunnel : public AdnlNetworkConnection {
pub_key_hash_ = pub_key_.compute_short_id();
//ready_.store(true, std::memory_order_release);
}
void get_ip_str(td::Promise<td::string> promise) override {
promise.set_value("tunnel");
}
AdnlNetworkConnectionTunnel(td::actor::ActorId<AdnlNetworkManager> network_manager, td::actor::ActorId<Adnl> adnl,
adnl::AdnlNodeIdShort adnl_id, PublicKey pubkey,

2
adnl/adnl-network-manager.h
View file

@ -50,6 +50,8 @@ class AdnlNetworkConnection : public td::actor::Actor {
virtual void send(AdnlNodeIdShort src, AdnlNodeIdShort dst, td::uint32 priority, td::BufferSlice message) = 0;
virtual bool is_alive() const = 0;
virtual bool is_active() const = 0;
virtual void get_ip_str(td::Promise<td::string> promise) = 0;
virtual ~AdnlNetworkConnection() = default;
};

9
adnl/adnl-peer-table.cpp
View file

@ -376,6 +376,15 @@ void AdnlPeerTableImpl::create_tunnel(AdnlNodeIdShort dst, td::uint32 size,
td::Promise<std::pair<td::actor::ActorOwn<AdnlTunnel>, AdnlAddress>> promise) {
}
void AdnlPeerTableImpl::get_conn_ip_str(AdnlNodeIdShort l_id, AdnlNodeIdShort p_id, td::Promise<td::string> promise) {
auto it = peers_.find(p_id);
if (it == peers_.end()) {
promise.set_value("undefined");
return;
}
td::actor::send_closure(it->second, &AdnlPeer::get_conn_ip_str, l_id, std::move(promise));
}
} // namespace adnl
} // namespace ton

1
adnl/adnl-peer-table.h
View file

@ -110,6 +110,7 @@ class AdnlPeerTable : public Adnl {
virtual void deliver_query(AdnlNodeIdShort src, AdnlNodeIdShort dst, td::BufferSlice data,
td::Promise<td::BufferSlice> promise) = 0;
virtual void decrypt_message(AdnlNodeIdShort dst, td::BufferSlice data, td::Promise<td::BufferSlice> promise) = 0;
virtual void get_conn_ip_str(AdnlNodeIdShort l_id, AdnlNodeIdShort p_id, td::Promise<td::string> promise) = 0;
};
} // namespace adnl

1
adnl/adnl-peer-table.hpp
View file

@ -102,6 +102,7 @@ class AdnlPeerTableImpl : public AdnlPeerTable {
void create_tunnel(AdnlNodeIdShort dst, td::uint32 size,
td::Promise<std::pair<td::actor::ActorOwn<AdnlTunnel>, AdnlAddress>> promise) override;
void get_conn_ip_str(AdnlNodeIdShort l_id, AdnlNodeIdShort p_id, td::Promise<td::string> promise) override;
struct PrintId {};
PrintId print_id() const {

32
adnl/adnl-peer.cpp
View file

@ -725,6 +725,28 @@ void AdnlPeerPairImpl::update_addr_list(AdnlAddressList addr_list) {
(priority ? priority_addr_list_ : addr_list_) = addr_list;
}
void AdnlPeerPairImpl::get_conn_ip_str(td::Promise<td::string> promise) {
if (conns_.size() == 0 && priority_conns_.size() == 0) {
promise.set_value("undefined");
return;
}
for (auto &conn : priority_conns_) {
if (conn.ready()) {
td::actor::send_closure(conn.conn, &AdnlNetworkConnection::get_ip_str, std::move(promise));
return;
}
}
for (auto &conn : conns_) {
if (conn.ready()) {
td::actor::send_closure(conn.conn, &AdnlNetworkConnection::get_ip_str, std::move(promise));
return;
}
}
promise.set_value("undefined");
}
void AdnlPeerImpl::update_id(AdnlNodeIdFull id) {
CHECK(id.compute_short_id() == peer_id_short_);
if (!peer_id_.empty()) {
@ -841,6 +863,16 @@ void AdnlPeerImpl::update_dht_node(td::actor::ActorId<dht::Dht> dht_node) {
}
}
void AdnlPeerImpl::get_conn_ip_str(AdnlNodeIdShort l_id, td::Promise<td::string> promise) {
auto it = peer_pairs_.find(l_id);
if (it == peer_pairs_.end()) {
promise.set_value("undefined");
return;
}
td::actor::send_closure(it->second, &AdnlPeerPair::get_conn_ip_str, std::move(promise));
}
void AdnlPeerImpl::update_addr_list(AdnlNodeIdShort local_id, td::uint32 local_mode,
td::actor::ActorId<AdnlLocalId> local_actor, AdnlAddressList addr_list) {
auto it = peer_pairs_.find(local_id);

2
adnl/adnl-peer.h
View file

@ -58,6 +58,7 @@ class AdnlPeerPair : public td::actor::Actor {
virtual void update_dht_node(td::actor::ActorId<dht::Dht> dht_node) = 0;
virtual void update_peer_id(AdnlNodeIdFull id) = 0;
virtual void update_addr_list(AdnlAddressList addr_list) = 0;
virtual void get_conn_ip_str(td::Promise<td::string> promise) = 0;
static td::actor::ActorOwn<AdnlPeerPair> create(td::actor::ActorId<AdnlNetworkManager> network_manager,
td::actor::ActorId<AdnlPeerTable> peer_table, td::uint32 local_mode,
@ -98,6 +99,7 @@ class AdnlPeer : public td::actor::Actor {
virtual void update_addr_list(AdnlNodeIdShort local_id, td::uint32 local_mode,
td::actor::ActorId<AdnlLocalId> local_actor, AdnlAddressList addr_list) = 0;
virtual void update_dht_node(td::actor::ActorId<dht::Dht> dht_node) = 0;
virtual void get_conn_ip_str(AdnlNodeIdShort l_id, td::Promise<td::string> promise) = 0;
};
} // namespace adnl

3
adnl/adnl-peer.hpp
View file

@ -88,6 +88,8 @@ class AdnlPeerPairImpl : public AdnlPeerPair {
void update_addr_list(AdnlAddressList addr_list) override;
void update_peer_id(AdnlNodeIdFull id) override;
void get_conn_ip_str(td::Promise<td::string> promise) override;
void got_data_from_db(td::Result<AdnlDbItem> R);
void got_data_from_static_nodes(td::Result<AdnlNode> R);
void got_data_from_dht(td::Result<AdnlNode> R);
@ -265,6 +267,7 @@ class AdnlPeerImpl : public AdnlPeer {
void update_addr_list(AdnlNodeIdShort local_id, td::uint32 local_mode, td::actor::ActorId<AdnlLocalId> local_actor,
AdnlAddressList addr_list) override;
void update_dht_node(td::actor::ActorId<dht::Dht> dht_node) override;
void get_conn_ip_str(AdnlNodeIdShort l_id, td::Promise<td::string> promise) override;
//void check_signature(td::BufferSlice data, td::BufferSlice signature, td::Promise<td::Unit> promise) override;
AdnlPeerImpl(td::actor::ActorId<AdnlNetworkManager> network_manager, td::actor::ActorId<AdnlPeerTable> peer_table,

1
adnl/adnl.h
View file

@ -56,6 +56,7 @@ class AdnlSenderInterface : public td::actor::Actor {
virtual void send_query_ex(AdnlNodeIdShort src, AdnlNodeIdShort dst, std::string name,
td::Promise<td::BufferSlice> promise, td::Timestamp timeout, td::BufferSlice data,
td::uint64 max_answer_size) = 0;
virtual void get_conn_ip_str(AdnlNodeIdShort l_id, AdnlNodeIdShort p_id, td::Promise<td::string> promise) = 0;
};
class AdnlTunnel : public td::actor::Actor {};

14
catchain/catchain-block.cpp
View file

@ -22,8 +22,8 @@ namespace ton {
namespace catchain {
std::unique_ptr<CatChainBlock> CatChainBlock::create(td::uint32 src, td::uint32 fork, PublicKeyHash src_hash,
CatChainBlockHeight height, CatChainBlockHash hash,
std::unique_ptr<CatChainBlock> CatChainBlock::create(td::uint32 src, td::uint32 fork, const PublicKeyHash &src_hash,
CatChainBlockHeight height, const CatChainBlockHash &hash,
td::SharedSlice payload, CatChainBlock *prev,
std::vector<CatChainBlock *> deps,
std::vector<CatChainBlockHeight> vt) {
@ -31,10 +31,10 @@ std::unique_ptr<CatChainBlock> CatChainBlock::create(td::uint32 src, td::uint32
std::move(deps), std::move(vt));
}
CatChainBlockImpl::CatChainBlockImpl(td::uint32 src, td::uint32 fork, PublicKeyHash src_hash,
CatChainBlockHeight height, CatChainBlockHash hash, td::SharedSlice payload,
CatChainBlock *prev, std::vector<CatChainBlock *> deps,
std::vector<CatChainBlockHeight> vt)
CatChainBlockImpl::CatChainBlockImpl(td::uint32 src, td::uint32 fork, const PublicKeyHash &src_hash,
CatChainBlockHeight height, const CatChainBlockHash &hash,
td::SharedSlice payload, CatChainBlock *prev,
std::vector<CatChainBlock *> deps, std::vector<CatChainBlockHeight> vt)
: src_(src)
, fork_(fork)
, src_hash_(src_hash)
@ -47,7 +47,7 @@ CatChainBlockImpl::CatChainBlockImpl(td::uint32 src, td::uint32 fork, PublicKeyH
}
bool CatChainBlockImpl::is_descendant_of(CatChainBlock *block) {
auto fork = block->fork();
td::uint32 fork = block->fork();
if (fork >= vt_.size()) {
return false;
}

4
catchain/catchain-block.hpp
View file

@ -103,8 +103,8 @@ class CatChainBlockImpl : public CatChainBlock {
bool is_descendant_of(CatChainBlock *block) override;
CatChainBlockImpl(td::uint32 src, td::uint32 fork, PublicKeyHash src_hash, CatChainBlockHeight height,
CatChainBlockHash hash, td::SharedSlice payload, CatChainBlock *prev,
CatChainBlockImpl(td::uint32 src, td::uint32 fork, const PublicKeyHash &src_hash, CatChainBlockHeight height,
const CatChainBlockHash &hash, td::SharedSlice payload, CatChainBlock *prev,
std::vector<CatChainBlock *> deps, std::vector<CatChainBlockHeight> vt);
};

149
catchain/catchain-received-block.cpp
View file

@ -33,7 +33,7 @@ void CatChainReceivedBlockImpl::initialize(tl_object_ptr<ton_api::catchain_block
}
payload_ = std::move(payload);
CHECK(payload_.size() > 0);
CHECK(!payload_.empty());
prev_ = dynamic_cast<CatChainReceivedBlockImpl *>(chain_->create_block(std::move(block->data_->prev_)));
CHECK(prev_ != nullptr);
@ -51,7 +51,7 @@ void CatChainReceivedBlockImpl::initialize(tl_object_ptr<ton_api::catchain_block
set_ill();
return;
}
for (auto &X : block_deps_) {
for (CatChainReceivedBlockImpl *X : block_deps_) {
if (X->is_ill()) {
set_ill();
return;
@ -63,17 +63,17 @@ void CatChainReceivedBlockImpl::initialize(tl_object_ptr<ton_api::catchain_block
if (!prev_->delivered()) {
pending_deps++;
} else {
update_deps(prev_);
update_vt(prev_);
}
if (!prev_->delivered()) {
prev_->add_rev_dep(this);
}
}
for (auto &X : block_deps_) {
for (CatChainReceivedBlockImpl *X : block_deps_) {
if (!X->delivered()) {
pending_deps++;
} else {
update_deps(X);
update_vt(X);
}
if (!X->delivered()) {
X->add_rev_dep(this);
@ -121,11 +121,11 @@ void CatChainReceivedBlockImpl::initialize_fork() {
}
}
if (deps_.size() < fork_id_ + 1) {
deps_.resize(fork_id_ + 1, 0);
if (vt_.size() < fork_id_ + 1) {
vt_.resize(fork_id_ + 1, 0);
}
CHECK(deps_[fork_id_] < height_);
deps_[fork_id_] = height_;
CHECK(vt_[fork_id_] < height_);
vt_[fork_id_] = height_;
}
void CatChainReceivedBlockImpl::pre_deliver(ton_api::catchain_block_data_fork &b) {
@ -153,7 +153,7 @@ void CatChainReceivedBlockImpl::pre_deliver(ton_api::catchain_block_data_fork &b
return;
}
auto S = chain_->get_source(b.left_->src_);
CatChainReceiverSource *S = chain_->get_source(b.left_->src_);
S->on_found_fork_proof(
create_serialize_tl_object<ton_api::catchain_block_data_fork>(std::move(b.left_), std::move(b.right_)));
S->blame(fork_id_, height_);
@ -173,15 +173,15 @@ void CatChainReceivedBlockImpl::pre_deliver() {
CHECK(pending_deps_ == 0);
CHECK(in_db_);
auto M = chain_->get_source(source_id_);
CatChainReceiverSource *M = chain_->get_source(source_id_);
auto d = prev_ ? &prev_->deps_ : nullptr;
std::vector<CatChainBlockHeight> *d = prev_ ? &prev_->vt_ : nullptr;
for (auto &X : block_deps_) {
auto S = chain_->get_source(X->get_source_id());
auto &f = S->get_forks();
for (CatChainReceivedBlockImpl *X : block_deps_) {
CatChainReceiverSource *S = chain_->get_source(X->get_source_id());
const std::vector<td::uint32> &f = S->get_forks();
if (d) {
auto &dd = *d;
const std::vector<CatChainBlockHeight> &dd = *d;
if (X->get_fork_id() < dd.size() && X->get_height() <= dd[X->get_fork_id()]) {
VLOG(CATCHAIN_WARNING) << this << ": has direct dep from source " << X->get_source_id() << " and prev block "
<< " has newer indirect dep";
@ -190,8 +190,8 @@ void CatChainReceivedBlockImpl::pre_deliver() {
}
}
if (S->blamed() && d) {
auto &dd = *d;
for (auto x : f) {
const std::vector<CatChainBlockHeight> &dd = *d;
for (td::uint32 x : f) {
if (x != X->get_fork_id() && dd.size() > x && dd[x] > 0) {
VLOG(CATCHAIN_WARNING) << this << ": has direct dep from source " << X->get_source_id() << " and prev block "
<< " has indirect dep to another fork of this source " << x << " " << X->get_fork_id()
@ -201,7 +201,7 @@ void CatChainReceivedBlockImpl::pre_deliver() {
return;
}
}
auto v = S->get_blamed_heights();
std::vector<CatChainBlockHeight> v = S->get_blamed_heights();
for (size_t i = 0; i < v.size() && i < dd.size(); i++) {
if (v[i] > 0 && dd[i] >= v[i]) {
@ -220,7 +220,7 @@ void CatChainReceivedBlockImpl::pre_deliver() {
if (X.is_error()) {
is_custom_ = true;
} else {
ton_api::downcast_call(*X.move_as_ok().get(), [Self = this](auto &obj) { Self->pre_deliver(obj); });
ton_api::downcast_call(*X.move_as_ok(), [Self = this](auto &obj) { Self->pre_deliver(obj); });
}
}
@ -237,7 +237,7 @@ void CatChainReceivedBlockImpl::deliver() {
state_ = bs_delivered;
VLOG(CATCHAIN_DEBUG) << this << ": delivered";
for (auto &B : rev_deps_) {
for (CatChainReceivedBlockImpl *B : rev_deps_) {
B->dep_delivered(this);
}
rev_deps_.clear();
@ -250,10 +250,10 @@ void CatChainReceivedBlockImpl::set_ill() {
return;
}
VLOG(CATCHAIN_WARNING) << this << ": got ill";
auto M = chain_->get_source(source_id_);
CatChainReceiverSource *M = chain_->get_source(source_id_);
M->blame();
state_ = bs_ill;
for (auto &B : rev_deps_) {
for (CatChainReceivedBlockImpl *B : rev_deps_) {
B->dep_ill(this);
}
}
@ -262,14 +262,14 @@ void CatChainReceivedBlockImpl::dep_ill(CatChainReceivedBlockImpl *block) {
set_ill();
}
void CatChainReceivedBlockImpl::update_deps(CatChainReceivedBlockImpl *block) {
auto &d = block->deps_;
if (d.size() > deps_.size()) {
deps_.resize(d.size(), 0);
void CatChainReceivedBlockImpl::update_vt(CatChainReceivedBlockImpl *block) {
const std::vector<CatChainBlockHeight> &d = block->vt_;
if (d.size() > vt_.size()) {
vt_.resize(d.size(), 0);
}
for (size_t i = 0; i < d.size(); i++) {
if (deps_[i] < d[i]) {
deps_[i] = d[i];
if (vt_[i] < d[i]) {
vt_[i] = d[i];
}
}
}
@ -279,7 +279,7 @@ void CatChainReceivedBlockImpl::dep_delivered(CatChainReceivedBlockImpl *block)
return;
}
CHECK(!block->is_ill());
update_deps(block);
update_vt(block);
pending_deps_--;
if (pending_deps_ == 0 && in_db_) {
schedule();
@ -332,35 +332,37 @@ void CatChainReceivedBlockImpl::find_pending_deps(std::vector<CatChainBlockHash>
if (prev_) {
prev_->find_pending_deps(vec, max_size);
}
for (auto &X : block_deps_) {
for (CatChainReceivedBlockImpl *X : block_deps_) {
X->find_pending_deps(vec, max_size);
}
}
tl_object_ptr<ton_api::catchain_block_id> CatChainReceivedBlock::block_id(CatChainReceiver *chain,
tl_object_ptr<ton_api::catchain_block> &block,
td::Slice payload) {
tl_object_ptr<ton_api::catchain_block_id> CatChainReceivedBlock::block_id(
const CatChainReceiver *chain, const tl_object_ptr<ton_api::catchain_block> &block, const td::Slice &payload) {
td::Bits256 hash = data_payload_hash(chain, block->data_, payload);
return create_tl_object<ton_api::catchain_block_id>(block->incarnation_, chain->get_source_hash(block->src_).tl(),
block->height_, sha256_bits256(payload));
block->height_, hash);
}
tl_object_ptr<ton_api::catchain_block_id> CatChainReceivedBlock::block_id(
CatChainReceiver *chain, tl_object_ptr<ton_api::catchain_block_dep> &block) {
const CatChainReceiver *chain, const tl_object_ptr<ton_api::catchain_block_dep> &block) {
return create_tl_object<ton_api::catchain_block_id>(
chain->get_incarnation(), chain->get_source_hash(block->src_).tl(), block->height_, block->data_hash_);
}
CatChainBlockHash CatChainReceivedBlock::block_hash(CatChainReceiver *chain,
tl_object_ptr<ton_api::catchain_block> &block, td::Slice payload) {
CatChainBlockHash CatChainReceivedBlock::block_hash(const CatChainReceiver *chain,
const tl_object_ptr<ton_api::catchain_block> &block,
const td::Slice &payload) {
return get_tl_object_sha_bits256(block_id(chain, block, payload));
}
CatChainBlockHash CatChainReceivedBlock::block_hash(CatChainReceiver *chain,
tl_object_ptr<ton_api::catchain_block_dep> &block) {
CatChainBlockHash CatChainReceivedBlock::block_hash(const CatChainReceiver *chain,
const tl_object_ptr<ton_api::catchain_block_dep> &block) {
return get_tl_object_sha_bits256(block_id(chain, block));
}
td::Status CatChainReceivedBlock::pre_validate_block(CatChainReceiver *chain,
tl_object_ptr<ton_api::catchain_block> &block, td::Slice payload) {
td::Status CatChainReceivedBlock::pre_validate_block(const CatChainReceiver *chain,
const tl_object_ptr<ton_api::catchain_block> &block,
const td::Slice &payload) {
CHECK(block->incarnation_ == chain->get_incarnation());
if (block->height_ <= 0) {
return td::Status::Error(ErrorCode::protoviolation, std::string("bad height ") + std::to_string(block->height_));
@ -397,7 +399,7 @@ td::Status CatChainReceivedBlock::pre_validate_block(CatChainReceiver *chain,
std::set<td::uint32> used;
used.insert(block->src_);
for (auto &X : block->data_->deps_) {
for (const auto &X : block->data_->deps_) {
if (used.find(X->src_) != used.end()) {
return td::Status::Error(ErrorCode::protoviolation, "two deps from same source");
}
@ -405,19 +407,19 @@ td::Status CatChainReceivedBlock::pre_validate_block(CatChainReceiver *chain,
}
TRY_STATUS(chain->validate_block_sync(block->data_->prev_));
for (auto &X : block->data_->deps_) {
for (const auto &X : block->data_->deps_) {
TRY_STATUS(chain->validate_block_sync(X));
}
if (payload.size() == 0) {
if (payload.empty()) {
return td::Status::Error(ErrorCode::protoviolation, "empty payload");
}
return td::Status::OK();
}
td::Status CatChainReceivedBlock::pre_validate_block(CatChainReceiver *chain,
tl_object_ptr<ton_api::catchain_block_dep> &block) {
td::Status CatChainReceivedBlock::pre_validate_block(const CatChainReceiver *chain,
const tl_object_ptr<ton_api::catchain_block_dep> &block) {
if (block->height_ < 0) {
return td::Status::Error(ErrorCode::protoviolation, std::string("bad height ") + std::to_string(block->height_));
}
@ -430,7 +432,7 @@ td::Status CatChainReceivedBlock::pre_validate_block(CatChainReceiver *chain,
return td::Status::Error(ErrorCode::protoviolation,
std::string("bad src (first block) ") + std::to_string(block->src_));
}
if (block->data_hash_ != chain->get_incarnation() || block->signature_.size() != 0) {
if (block->data_hash_ != chain->get_incarnation() || !block->signature_.empty()) {
return td::Status::Error(ErrorCode::protoviolation, std::string("bad first block"));
}
}
@ -443,9 +445,10 @@ tl_object_ptr<ton_api::catchain_block> CatChainReceivedBlockImpl::export_tl() co
CHECK(height_ > 0);
std::vector<tl_object_ptr<ton_api::catchain_block_dep>> deps;
for (auto &B : block_deps_) {
for (CatChainReceivedBlockImpl *B : block_deps_) {
deps.push_back(B->export_tl_dep());
}
CHECK(deps.size() <= chain_->opts().max_deps)
return create_tl_object<ton_api::catchain_block>(
chain_->get_incarnation(), source_id_, height_,
@ -454,35 +457,34 @@ tl_object_ptr<ton_api::catchain_block> CatChainReceivedBlockImpl::export_tl() co
}
tl_object_ptr<ton_api::catchain_block_dep> CatChainReceivedBlockImpl::export_tl_dep() const {
return create_tl_object<ton_api::catchain_block_dep>(source_id_, height_, data_hash_,
return create_tl_object<ton_api::catchain_block_dep>(source_id_, height_, data_payload_hash_,
signature_.clone_as_buffer_slice());
}
CatChainReceivedBlockImpl::CatChainReceivedBlockImpl(td::uint32 source_id, CatChainSessionId hash,
CatChainReceivedBlockImpl::CatChainReceivedBlockImpl(td::uint32 source_id, const CatChainBlockPayloadHash &hash,
CatChainReceiver *chain) {
chain_ = chain;
state_ = bs_delivered;
fork_id_ = 0;
source_id_ = source_id;
data_ = nullptr;
prev_ = nullptr;
height_ = 0;
data_hash_ = hash;
hash_ = get_tl_object_sha_bits256(create_tl_object<ton_api::catchain_block_id>(
chain->get_incarnation(), chain->get_incarnation(), height_, data_hash_));
data_payload_hash_ = hash;
block_id_hash_ = get_tl_object_sha_bits256(create_tl_object<ton_api::catchain_block_id>(
chain->get_incarnation(), chain->get_incarnation(), height_, data_payload_hash_));
}
CatChainReceivedBlockImpl::CatChainReceivedBlockImpl(tl_object_ptr<ton_api::catchain_block> block,
td::SharedSlice payload, CatChainReceiver *chain) {
chain_ = chain;
data_hash_ = sha256_bits256(payload.as_slice());
hash_ = get_tl_object_sha_bits256(create_tl_object<ton_api::catchain_block_id>(
block->incarnation_, chain->get_source_hash(block->src_).tl(), block->height_, data_hash_));
data_payload_hash_ = data_payload_hash(chain, block->data_, payload);
block_id_hash_ = get_tl_object_sha_bits256(create_tl_object<ton_api::catchain_block_id>(
block->incarnation_, chain->get_source_hash(block->src_).tl(), block->height_, data_payload_hash_));
height_ = block->height_;
source_id_ = block->src_;
CHECK(height_ <= get_max_block_height(chain->opts(), chain->get_sources_cnt()));
auto S = chain_->get_source(source_id_);
CatChainReceiverSource *S = chain_->get_source(source_id_);
S->on_new_block(this);
initialize(std::move(block), std::move(payload));
@ -491,14 +493,14 @@ CatChainReceivedBlockImpl::CatChainReceivedBlockImpl(tl_object_ptr<ton_api::catc
CatChainReceivedBlockImpl::CatChainReceivedBlockImpl(tl_object_ptr<ton_api::catchain_block_dep> block,
CatChainReceiver *chain) {
chain_ = chain;
data_hash_ = block->data_hash_;
data_payload_hash_ = block->data_hash_;
source_id_ = block->src_;
signature_ = td::SharedSlice{block->signature_.as_slice()};
hash_ = get_tl_object_sha_bits256(create_tl_object<ton_api::catchain_block_id>(
chain_->get_incarnation(), chain_->get_source_hash(source_id_).tl(), block->height_, data_hash_));
block_id_hash_ = get_tl_object_sha_bits256(create_tl_object<ton_api::catchain_block_id>(
chain_->get_incarnation(), chain_->get_source_hash(source_id_).tl(), block->height_, data_payload_hash_));
height_ = block->height_;
auto S = chain_->get_source(source_id_);
CatChainReceiverSource *S = chain_->get_source(source_id_);
S->on_new_block(this);
}
@ -513,9 +515,24 @@ std::unique_ptr<CatChainReceivedBlock> CatChainReceivedBlock::create(tl_object_p
}
std::unique_ptr<CatChainReceivedBlock> CatChainReceivedBlock::create_root(td::uint32 source_id,
CatChainSessionId data_hash,
CatChainSessionId session_id,
CatChainReceiver *chain) {
return std::make_unique<CatChainReceivedBlockImpl>(source_id, data_hash, chain);
return std::make_unique<CatChainReceivedBlockImpl>(source_id, session_id, chain);
}
CatChainBlockPayloadHash CatChainReceivedBlock::data_payload_hash(
const CatChainReceiver *chain, const tl_object_ptr<ton_api::catchain_block_data> &data, const td::Slice &payload) {
td::Bits256 hash = sha256_bits256(payload);
if (chain->opts().block_hash_covers_data) {
td::Bits256 data_hash = get_tl_object_sha_bits256(data);
char buf[32 * 2];
CHECK(hash.as_array().size() == 32 && data_hash.as_array().size() == 32);
std::copy(hash.as_array().begin(), hash.as_array().end(), buf);
std::copy(data_hash.as_array().begin(), data_hash.as_array().end(), buf + 32);
return sha256_bits256(td::Slice(buf, buf + 64));
} else {
return hash;
}
}
} // namespace catchain

37
catchain/catchain-received-block.h
View file

@ -30,7 +30,6 @@ namespace catchain {
class CatChainReceiver;
class CatChainReceiverSource;
class CatChainReceiverFork;
class CatChainReceivedBlock {
public:
@ -43,7 +42,7 @@ class CatChainReceivedBlock {
virtual CatChainReceivedBlock *get_prev() const = 0;
virtual CatChainBlockHash get_prev_hash() const = 0;
virtual const std::vector<CatChainBlockHeight> &get_deps() const = 0;
virtual const std::vector<CatChainBlockHeight> &get_vt() const = 0;
virtual std::vector<CatChainBlockHash> get_dep_hashes() const = 0;
virtual CatChainReceiver *get_chain() const = 0;
@ -56,6 +55,8 @@ class CatChainReceivedBlock {
virtual void find_pending_deps(std::vector<CatChainBlockHash> &vec, td::uint32 max_size) const = 0;
virtual bool has_rev_deps() const = 0;
public:
// state
virtual bool initialized() const = 0;
@ -76,20 +77,28 @@ class CatChainReceivedBlock {
td::SharedSlice payload, CatChainReceiver *chain);
static std::unique_ptr<CatChainReceivedBlock> create(tl_object_ptr<ton_api::catchain_block_dep> block,
CatChainReceiver *chain);
static std::unique_ptr<CatChainReceivedBlock> create_root(td::uint32 source_id, CatChainBlockPayloadHash data_hash,
static std::unique_ptr<CatChainReceivedBlock> create_root(td::uint32 source_id, CatChainSessionId session_id,
CatChainReceiver *chain);
static tl_object_ptr<ton_api::catchain_block_id> block_id(CatChainReceiver *chain,
tl_object_ptr<ton_api::catchain_block> &block,
td::Slice payload);
static tl_object_ptr<ton_api::catchain_block_id> block_id(CatChainReceiver *chain,
tl_object_ptr<ton_api::catchain_block_dep> &block);
static CatChainBlockHash block_hash(CatChainReceiver *chain, tl_object_ptr<ton_api::catchain_block> &block,
td::Slice payload);
static CatChainBlockHash block_hash(CatChainReceiver *chain, tl_object_ptr<ton_api::catchain_block_dep> &block);
static td::Status pre_validate_block(CatChainReceiver *chain, tl_object_ptr<ton_api::catchain_block> &block,
td::Slice payload);
static td::Status pre_validate_block(CatChainReceiver *chain, tl_object_ptr<ton_api::catchain_block_dep> &block);
static tl_object_ptr<ton_api::catchain_block_id> block_id(const CatChainReceiver *chain,
const tl_object_ptr<ton_api::catchain_block> &block,
const td::Slice &payload);
static tl_object_ptr<ton_api::catchain_block_id> block_id(const CatChainReceiver *chain,
const tl_object_ptr<ton_api::catchain_block_dep> &block);
static CatChainBlockHash block_hash(const CatChainReceiver *chain,
const tl_object_ptr<ton_api::catchain_block> &block,
const td::Slice &payload);
static CatChainBlockHash block_hash(const CatChainReceiver *chain,
const tl_object_ptr<ton_api::catchain_block_dep> &block);
static td::Status pre_validate_block(const CatChainReceiver *chain,
const tl_object_ptr<ton_api::catchain_block> &block,
const td::Slice &payload);
static td::Status pre_validate_block(const CatChainReceiver *chain,
const tl_object_ptr<ton_api::catchain_block_dep> &block);
static CatChainBlockPayloadHash data_payload_hash(const CatChainReceiver *chain,
const tl_object_ptr<ton_api::catchain_block_data> &data,
const td::Slice &payload);
virtual ~CatChainReceivedBlock() = default;
};

28
catchain/catchain-received-block.hpp
View file

@ -26,15 +26,14 @@ namespace catchain {
class CatChainReceiver;
class CatChainReceiverSource;
class CatChainReceiverFork;
class CatChainReceivedBlockImpl : public CatChainReceivedBlock {
class CatChainReceivedBlockImpl final : public CatChainReceivedBlock {
public:
const td::SharedSlice &get_payload() const override {
return payload_;
}
CatChainBlockHash get_hash() const override {
return hash_;
return block_id_hash_;
}
const td::SharedSlice &get_signature() const override {
return signature_;
@ -46,8 +45,8 @@ class CatChainReceivedBlockImpl : public CatChainReceivedBlock {
CatChainReceivedBlock *get_prev() const override;
CatChainBlockHash get_prev_hash() const override;
const std::vector<CatChainBlockHeight> &get_deps() const override {
return deps_;
const std::vector<CatChainBlockHeight> &get_vt() const override {
return vt_;
}
std::vector<CatChainBlockHash> get_dep_hashes() const override;
@ -69,6 +68,10 @@ class CatChainReceivedBlockImpl : public CatChainReceivedBlock {
void find_pending_deps(std::vector<CatChainBlockHash> &vec, td::uint32 max_size) const override;
bool has_rev_deps() const override {
return !rev_deps_.empty();
}
public:
bool initialized() const override {
return state_ >= bs_initialized;
@ -114,7 +117,7 @@ class CatChainReceivedBlockImpl : public CatChainReceivedBlock {
CatChainReceiver *chain);
CatChainReceivedBlockImpl(tl_object_ptr<ton_api::catchain_block_dep> block, CatChainReceiver *chain);
CatChainReceivedBlockImpl(td::uint32 source_id, CatChainSessionId hash, CatChainReceiver *chain);
CatChainReceivedBlockImpl(td::uint32 source_id, const CatChainSessionId &hash, CatChainReceiver *chain);
private:
enum State {
@ -124,31 +127,28 @@ class CatChainReceivedBlockImpl : public CatChainReceivedBlock {
bs_delivered,
} state_ = bs_none;
void update_deps(CatChainReceivedBlockImpl *block);
void update_vt(CatChainReceivedBlockImpl *block);
void add_rev_dep(CatChainReceivedBlockImpl *block);
void add_child_dep(CatChainReceivedBlockImpl *block);
void initialize_fork();
void on_ready_to_deliver();
td::uint32 fork_id_{0};
td::uint32 source_id_;
CatChainReceiver *chain_;
tl_object_ptr<ton_api::catchain_block_inner_Data> data_;
td::SharedSlice payload_;
CatChainBlockHash hash_;
CatChainBlockPayloadHash data_hash_;
CatChainBlockHash block_id_hash_{};
CatChainBlockPayloadHash data_payload_hash_{};
CatChainReceivedBlockImpl *prev_;
CatChainReceivedBlockImpl *prev_ = nullptr;
CatChainBlockHeight height_;
CatChainReceivedBlockImpl *next_ = nullptr;
std::vector<CatChainReceivedBlockImpl *> block_deps_;
std::vector<CatChainBlockHeight> deps_;
std::vector<CatChainBlockHeight> vt_;
td::SharedSlice signature_;

24
catchain/catchain-receiver-interface.h
View file

@ -35,33 +35,33 @@ class CatChainReceiverInterface : public td::actor::Actor {
CatChainBlockHash prev, std::vector<CatChainBlockHash> deps,
std::vector<CatChainBlockHeight> vt, td::SharedSlice data) = 0;
virtual void blame(td::uint32 src_id) = 0;
virtual void on_custom_message(PublicKeyHash src, td::BufferSlice data) = 0;
virtual void on_custom_query(PublicKeyHash src, td::BufferSlice data, td::Promise<td::BufferSlice> promise) = 0;
virtual void on_broadcast(PublicKeyHash src, td::BufferSlice data) = 0;
virtual void on_custom_query(const PublicKeyHash &src, td::BufferSlice data,
td::Promise<td::BufferSlice> promise) = 0;
virtual void on_broadcast(const PublicKeyHash &src, td::BufferSlice data) = 0;
virtual void start() = 0;
virtual ~Callback() = default;
};
virtual void add_block(td::BufferSlice payload, std::vector<CatChainBlockHash> deps) = 0;
virtual void debug_add_fork(td::BufferSlice payload, CatChainBlockHeight height,
std::vector<CatChainBlockHash> deps) = 0;
virtual void blame_node(td::uint32 idx) = 0;
virtual void send_fec_broadcast(td::BufferSlice data) = 0;
virtual void send_custom_query_data(PublicKeyHash dst, std::string name, td::Promise<td::BufferSlice> promise,
virtual void send_custom_query_data(const PublicKeyHash &dst, std::string name, td::Promise<td::BufferSlice> promise,
td::Timestamp timeout, td::BufferSlice query) = 0;
virtual void send_custom_query_data_via(PublicKeyHash dst, std::string name, td::Promise<td::BufferSlice> promise,
td::Timestamp timeout, td::BufferSlice query, td::uint64 max_answer_size,
virtual void send_custom_query_data_via(const PublicKeyHash &dst, std::string name,
td::Promise<td::BufferSlice> promise, td::Timestamp timeout,
td::BufferSlice query, td::uint64 max_answer_size,
td::actor::ActorId<adnl::AdnlSenderInterface> via) = 0;
virtual void send_custom_message_data(PublicKeyHash dst, td::BufferSlice query) = 0;
virtual void send_custom_message_data(const PublicKeyHash &dst, td::BufferSlice query) = 0;
virtual void destroy() = 0;
static td::actor::ActorOwn<CatChainReceiverInterface> create(
std::unique_ptr<Callback> callback, CatChainOptions opts, td::actor::ActorId<keyring::Keyring> keyring,
std::unique_ptr<Callback> callback, const CatChainOptions &opts, td::actor::ActorId<keyring::Keyring> keyring,
td::actor::ActorId<adnl::Adnl> adnl, td::actor::ActorId<overlay::Overlays> overlay_manager,
std::vector<CatChainNode> ids, PublicKeyHash local_id, CatChainSessionId unique_hash, std::string db_root,
std::string db_suffix, bool allow_unsafe_self_blocks_resync);
const std::vector<CatChainNode> &ids, const PublicKeyHash &local_id, const CatChainSessionId &unique_hash,
std::string db_root, std::string db_suffix, bool allow_unsafe_self_blocks_resync);
virtual ~CatChainReceiverInterface() = default;
~CatChainReceiverInterface() override = default;
};
} // namespace catchain

21
catchain/catchain-receiver-source.cpp
View file

@ -24,10 +24,10 @@ namespace ton {
namespace catchain {
td::uint32 CatChainReceiverSourceImpl::add_fork() {
if (fork_ids_.size() > 0) {
if (!fork_ids_.empty()) {
blame();
}
auto F = chain_->add_fork();
td::uint32 F = chain_->add_fork();
CHECK(F > 0);
fork_ids_.push_back(F);
@ -60,7 +60,7 @@ td::Result<std::unique_ptr<CatChainReceiverSource>> CatChainReceiverSource::crea
void CatChainReceiverSourceImpl::blame(td::uint32 fork, CatChainBlockHeight height) {
blame();
if (blamed_heights_.size() > 0) {
if (!blamed_heights_.empty()) {
if (blamed_heights_.size() <= fork) {
blamed_heights_.resize(fork + 1, 0);
}
@ -130,19 +130,6 @@ void CatChainReceiverSourceImpl::block_delivered(CatChainBlockHeight height) {
}
}
td::Status CatChainReceiverSourceImpl::validate_dep_sync(tl_object_ptr<ton_api::catchain_block_dep> &dep) {
auto S = std::move(dep->signature_);
auto str = serialize_tl_object(dep, true);
dep->signature_ = std::move(S);
auto R = encryptor_sync_->check_signature(str.as_slice(), dep->signature_.as_slice());
if (R.is_error()) {
return R.move_as_error();
}
return td::Status::OK();
}
void CatChainReceiverSourceImpl::on_new_block(CatChainReceivedBlock *block) {
if (fork_is_found()) {
return;
@ -165,7 +152,7 @@ void CatChainReceiverSourceImpl::on_new_block(CatChainReceivedBlock *block) {
blocks_[block->get_height()] = block;
}
void CatChainReceiverSourceImpl::on_found_fork_proof(td::Slice proof) {
void CatChainReceiverSourceImpl::on_found_fork_proof(const td::Slice &proof) {
if (!fork_is_found()) {
fetch_tl_object<ton_api::catchain_block_data_fork>(proof, true).ensure();
fork_proof_ = td::SharedSlice{proof};

3
catchain/catchain-receiver-source.h
View file

@ -56,9 +56,8 @@ class CatChainReceiverSource {
virtual bool has_unreceived() const = 0;
virtual bool has_undelivered() const = 0;
virtual td::Status validate_dep_sync(tl_object_ptr<ton_api::catchain_block_dep> &dep) = 0;
virtual void on_new_block(CatChainReceivedBlock *block) = 0;
virtual void on_found_fork_proof(td::Slice fork) = 0;
virtual void on_found_fork_proof(const td::Slice &fork) = 0;
virtual td::BufferSlice fork_proof() const = 0;
virtual bool fork_is_found() const = 0;

8
catchain/catchain-receiver-source.hpp
View file

@ -23,6 +23,7 @@
#include "catchain-receiver-source.h"
#include "catchain-receiver.h"
#include "catchain-received-block.h"
#include <queue>
namespace ton {
@ -82,7 +83,7 @@ class CatChainReceiverSourceImpl : public CatChainReceiverSource {
if (blamed()) {
return true;
}
if (!blocks_.size()) {
if (blocks_.empty()) {
return false;
}
CHECK(blocks_.rbegin()->second->get_height() >= received_height_);
@ -93,9 +94,8 @@ class CatChainReceiverSourceImpl : public CatChainReceiverSource {
}
CatChainReceivedBlock *get_block(CatChainBlockHeight height) const override;
td::Status validate_dep_sync(tl_object_ptr<ton_api::catchain_block_dep> &dep) override;
void on_new_block(CatChainReceivedBlock *block) override;
void on_found_fork_proof(td::Slice proof) override;
void on_found_fork_proof(const td::Slice &proof) override;
bool fork_is_found() const override {
return !fork_proof_.empty();
}
@ -103,7 +103,7 @@ class CatChainReceiverSourceImpl : public CatChainReceiverSource {
if (!fork_proof_.empty()) {
return fork_proof_.clone_as_buffer_slice();
} else {
return td::BufferSlice();
return {};
}
}

383
catchain/catchain-receiver.cpp
View file

@ -17,6 +17,7 @@
Copyright 2017-2020 Telegram Systems LLP
*/
#include <set>
#include <utility>
#include "td/actor/PromiseFuture.h"
#include "td/utils/Random.h"
#include "td/db/RocksDb.h"
@ -30,6 +31,25 @@ namespace ton {
namespace catchain {
static const td::uint32 MAX_NEIGHBOURS = 5;
static const double EXPECTED_UNSAFE_INITIAL_SYNC_DURATION = 300.0;
static const double EXPECTED_INITIAL_SYNC_DURATION = 5.0;
static const td::uint32 OVERLAY_MAX_ALLOWED_PACKET_SIZE = 16 * 1024 * 1024;
static const double NEIGHBOURS_ROTATE_INTERVAL_MIN = 60;
static const double NEIGHBOURS_ROTATE_INTERVAL_MAX = 120;
static const td::uint32 MAX_QUERY_BLOCKS = 100;
static const td::uint32 MAX_QUERY_HEIGHT = 100;
static const td::uint32 GET_DIFFERENCE_MAX_SEND = 100;
static const double GET_DIFFERENCE_TIMEOUT = 5.0;
static const double GET_BLOCK_TIMEOUT = 2.0;
static const td::uint32 MAX_PENDING_DEPS = 16;
static const double EXPECTED_INITIAL_SYNC_DURATION_WITH_UNPROCESSED = 60.0;
static const double SYNC_INTERVAL_MIN = 0.1;
static const double SYNC_INTERVAL_MAX = 0.2;
static const td::uint32 SYNC_ITERATIONS = 3;
static const double DESTROY_DB_DELAY = 1.0;
static const td::uint32 DESTROY_DB_MAX_ATTEMPTS = 10;
PublicKeyHash CatChainReceiverImpl::get_source_hash(td::uint32 source_id) const {
CHECK(source_id < sources_.size());
return sources_[source_id]->get_hash();
@ -45,18 +65,19 @@ void CatChainReceiverImpl::deliver_block(CatChainReceivedBlock *block) {
<< " custom=" << block->is_custom();
callback_->new_block(block->get_source_id(), block->get_fork_id(), block->get_hash(), block->get_height(),
block->get_height() == 1 ? CatChainBlockHash::zero() : block->get_prev_hash(),
block->get_dep_hashes(), block->get_deps(),
block->get_dep_hashes(), block->get_vt(),
block->is_custom() ? block->get_payload().clone() : td::SharedSlice());
std::vector<adnl::AdnlNodeIdShort> v;
for (auto it : neighbours_) {
auto S = get_source(it);
for (td::uint32 it : neighbours_) {
CatChainReceiverSource *S = get_source(it);
v.push_back(S->get_adnl_id());
}
auto update = create_tl_object<ton_api::catchain_blockUpdate>(block->export_tl());
auto D = serialize_tl_object(update, true, block->get_payload().as_slice());
td::BufferSlice D = serialize_tl_object(update, true, block->get_payload().as_slice());
CHECK(D.size() <= opts_.max_serialized_block_size);
td::actor::send_closure(overlay_manager_, &overlay::Overlays::send_multiple_messages, std::move(v),
get_source(local_idx_)->get_adnl_id(), overlay_id_, std::move(D));
@ -64,8 +85,8 @@ void CatChainReceiverImpl::deliver_block(CatChainReceivedBlock *block) {
void CatChainReceiverImpl::receive_block(adnl::AdnlNodeIdShort src, tl_object_ptr<ton_api::catchain_block> block,
td::BufferSlice payload) {
auto id = CatChainReceivedBlock::block_hash(this, block, payload);
auto B = get_block(id);
CatChainBlockHash id = CatChainReceivedBlock::block_hash(this, block, payload);
CatChainReceivedBlock *B = get_block(id);
if (B && B->initialized()) {
return;
}
@ -76,7 +97,27 @@ void CatChainReceiverImpl::receive_block(adnl::AdnlNodeIdShort src, tl_object_pt
return;
}
auto S = validate_block_sync(block, payload.as_slice());
td::uint64 max_block_height = get_max_block_height(opts_, sources_.size());
if ((td::uint32)block->height_ > max_block_height) {
VLOG(CATCHAIN_WARNING) << this << ": received too many blocks from " << src
<< " (limit=" << max_block_height << ")";
return;
}
td::uint32 src_id = block->src_;
if (src_id >= get_sources_cnt()) {
VLOG(CATCHAIN_WARNING) << this << ": received broken block from " << src << ": bad src " << block->src_;
return;
}
CatChainReceiverSource *source = get_source(src_id);
if (source->fork_is_found()) {
if (B == nullptr || !B->has_rev_deps()) {
VLOG(CATCHAIN_WARNING) << this << ": dropping block from source " << src_id << ": source has a fork";
return;
}
}
td::Status S = validate_block_sync(block, payload.as_slice());
if (S.is_error()) {
VLOG(CATCHAIN_WARNING) << this << ": received broken block from " << src << ": " << S.move_as_error();
@ -89,11 +130,11 @@ void CatChainReceiverImpl::receive_block(adnl::AdnlNodeIdShort src, tl_object_pt
<< " (unsafe=" << allow_unsafe_self_blocks_resync_ << ")";
} else {
LOG(ERROR) << this << ": received unknown SELF block from " << src << ". UPDATING LOCAL DATABASE. UNSAFE";
initial_sync_complete_at_ = td::Timestamp::in(300.0);
initial_sync_complete_at_ = td::Timestamp::in(EXPECTED_UNSAFE_INITIAL_SYNC_DURATION);
}
}
auto raw_data = serialize_tl_object(block, true, payload.as_slice());
td::BufferSlice raw_data = serialize_tl_object(block, true, payload.as_slice());
create_block(std::move(block), td::SharedSlice{payload.as_slice()});
if (!opts_.debug_disable_db) {
@ -104,6 +145,11 @@ void CatChainReceiverImpl::receive_block(adnl::AdnlNodeIdShort src, tl_object_pt
}
void CatChainReceiverImpl::receive_block_answer(adnl::AdnlNodeIdShort src, td::BufferSlice data) {
if (data.size() > opts_.max_serialized_block_size) {
VLOG(CATCHAIN_INFO) << this << ": received bad block result " << src << ": too big (size="
<< data.size() << ", limit=" << opts_.max_serialized_block_size << ")";
return;
}
auto F = fetch_tl_prefix<ton_api::catchain_BlockResult>(data, true);
if (F.is_error()) {
VLOG(CATCHAIN_INFO) << this << ": received bad block result: " << F.move_as_error();
@ -111,7 +157,7 @@ void CatChainReceiverImpl::receive_block_answer(adnl::AdnlNodeIdShort src, td::B
}
auto f = F.move_as_ok();
ton_api::downcast_call(
*f.get(),
*f,
td::overloaded(
[&](ton_api::catchain_blockNotFound &r) { VLOG(CATCHAIN_INFO) << this << ": catchain block not found"; },
[&](ton_api::catchain_blockResult &r) { receive_block(src, std::move(r.block_), std::move(data)); }));
@ -129,6 +175,11 @@ void CatChainReceiverImpl::receive_message_from_overlay(adnl::AdnlNodeIdShort sr
VLOG(CATCHAIN_INFO) << this << ": dropping block update from blamed source " << src;
return;
}*/
if (data.size() > opts_.max_serialized_block_size) {
VLOG(CATCHAIN_WARNING) << this << ": dropping broken block from " << src << ": too big (size="
<< data.size() << ", limit=" << opts_.max_serialized_block_size << ")";
return;
}
auto R = fetch_tl_prefix<ton_api::catchain_blockUpdate>(data, true);
if (R.is_error()) {
VLOG(CATCHAIN_WARNING) << this << ": dropping broken block from " << src << ": " << R.move_as_error();
@ -139,14 +190,14 @@ void CatChainReceiverImpl::receive_message_from_overlay(adnl::AdnlNodeIdShort sr
receive_block(src, std::move(U->block_), std::move(data));
}
void CatChainReceiverImpl::receive_broadcast_from_overlay(PublicKeyHash src, td::BufferSlice data) {
void CatChainReceiverImpl::receive_broadcast_from_overlay(const PublicKeyHash &src, td::BufferSlice data) {
if (!read_db_) {
return;
}
callback_->on_broadcast(src, std::move(data));
}
/*void CatChainReceiverImpl::send_block(PublicKeyHash src, tl_object_ptr<ton_api::catchain_block> block,
/*void CatChainReceiverImpl::send_block(const PublicKeyHash &src, tl_object_ptr<ton_api::catchain_block> block,
td::BufferSlice payload) {
CHECK(read_db_);
CHECK(src == local_id_);
@ -164,7 +215,7 @@ CatChainReceivedBlock *CatChainReceiverImpl::create_block(tl_object_ptr<ton_api:
if (block->height_ == 0) {
return root_block_;
}
auto hash = CatChainReceivedBlock::block_hash(this, block, payload.as_slice());
CatChainBlockHash hash = CatChainReceivedBlock::block_hash(this, block, payload.as_slice());
auto it = blocks_.find(hash);
if (it != blocks_.end()) {
@ -173,9 +224,8 @@ CatChainReceivedBlock *CatChainReceiverImpl::create_block(tl_object_ptr<ton_api:
}
return it->second.get();
} else {
blocks_.emplace(hash, CatChainReceivedBlock::create(std::move(block), std::move(payload), this));
it = blocks_.find(hash);
return it->second.get();
auto r = blocks_.emplace(hash, CatChainReceivedBlock::create(std::move(block), std::move(payload), this));
return r.first->second.get();
}
}
@ -183,7 +233,7 @@ CatChainReceivedBlock *CatChainReceiverImpl::create_block(tl_object_ptr<ton_api:
if (block->height_ == 0) {
return root_block_;
}
auto hash = CatChainReceivedBlock::block_hash(this, block);
CatChainBlockHash hash = CatChainReceivedBlock::block_hash(this, block);
auto it = blocks_.find(hash);
if (it != blocks_.end()) {
return it->second.get();
@ -194,20 +244,20 @@ CatChainReceivedBlock *CatChainReceiverImpl::create_block(tl_object_ptr<ton_api:
}
}
td::Status CatChainReceiverImpl::validate_block_sync(tl_object_ptr<ton_api::catchain_block_dep> &dep) {
td::Status CatChainReceiverImpl::validate_block_sync(const tl_object_ptr<ton_api::catchain_block_dep> &dep) const {
TRY_STATUS_PREFIX(CatChainReceivedBlock::pre_validate_block(this, dep), "failed to validate block: ");
if (dep->height_ > 0) {
auto id = CatChainReceivedBlock::block_id(this, dep);
auto B = serialize_tl_object(id, true);
auto block = get_block(get_tl_object_sha_bits256(id));
td::BufferSlice B = serialize_tl_object(id, true);
CatChainReceivedBlock *block = get_block(get_tl_object_sha_bits256(id));
if (block) {
return td::Status::OK();
}
auto S = get_source_by_hash(PublicKeyHash{id->src_});
CatChainReceiverSource *S = get_source_by_hash(PublicKeyHash{id->src_});
CHECK(S != nullptr);
auto E = S->get_encryptor_sync();
Encryptor *E = S->get_encryptor_sync();
CHECK(E != nullptr);
return E->check_signature(B.as_slice(), dep->signature_.as_slice());
} else {
@ -215,17 +265,18 @@ td::Status CatChainReceiverImpl::validate_block_sync(tl_object_ptr<ton_api::catc
}
}
td::Status CatChainReceiverImpl::validate_block_sync(tl_object_ptr<ton_api::catchain_block> &block, td::Slice payload) {
td::Status CatChainReceiverImpl::validate_block_sync(const tl_object_ptr<ton_api::catchain_block> &block,
const td::Slice &payload) const {
//LOG(INFO) << ton_api::to_string(block);
TRY_STATUS_PREFIX(CatChainReceivedBlock::pre_validate_block(this, block, payload), "failed to validate block: ");
if (block->height_ > 0) {
auto id = CatChainReceivedBlock::block_id(this, block, payload);
auto B = serialize_tl_object(id, true);
td::BufferSlice B = serialize_tl_object(id, true);
auto S = get_source_by_hash(PublicKeyHash{id->src_});
CatChainReceiverSource *S = get_source_by_hash(PublicKeyHash{id->src_});
CHECK(S != nullptr);
auto E = S->get_encryptor_sync();
Encryptor *E = S->get_encryptor_sync();
CHECK(E != nullptr);
return E->check_signature(B.as_slice(), block->signature_.as_slice());
} else {
@ -235,7 +286,7 @@ td::Status CatChainReceiverImpl::validate_block_sync(tl_object_ptr<ton_api::catc
void CatChainReceiverImpl::run_scheduler() {
while (!to_run_.empty()) {
auto B = to_run_.front();
CatChainReceivedBlock *B = to_run_.front();
to_run_.pop_front();
B->run();
@ -265,7 +316,7 @@ void CatChainReceiverImpl::add_block_cont_3(tl_object_ptr<ton_api::catchain_bloc
}
active_send_ = false;
if (pending_blocks_.size() > 0) {
if (!pending_blocks_.empty()) {
auto B = std::move(pending_blocks_.front());
pending_blocks_.pop_front();
add_block(std::move(B->payload_), std::move(B->deps_));
@ -278,9 +329,9 @@ void CatChainReceiverImpl::add_block_cont_2(tl_object_ptr<ton_api::catchain_bloc
return;
}
auto id = CatChainReceivedBlock::block_hash(this, block, payload);
CatChainBlockHash id = CatChainReceivedBlock::block_hash(this, block, payload);
td::BufferSlice raw_data{32};
td::BufferSlice raw_data{id.as_array().size()};
raw_data.as_slice().copy_from(as_slice(id));
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), block = std::move(block),
@ -298,9 +349,9 @@ void CatChainReceiverImpl::add_block_cont(tl_object_ptr<ton_api::catchain_block>
add_block_cont_2(std::move(block), std::move(payload));
return;
}
auto id = CatChainReceivedBlock::block_hash(this, block, payload.as_slice());
CatChainBlockHash id = CatChainReceivedBlock::block_hash(this, block, payload.as_slice());
auto raw_data = serialize_tl_object(block, true, payload.as_slice());
td::BufferSlice raw_data = serialize_tl_object(block, true, payload.as_slice());
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), block = std::move(block),
payload = std::move(payload)](td::Result<td::Unit> R) mutable {
@ -319,7 +370,7 @@ void CatChainReceiverImpl::add_block(td::BufferSlice payload, std::vector<CatCha
}
active_send_ = true;
auto S = get_source_by_hash(local_id_);
CatChainReceiverSource *S = get_source_by_hash(local_id_);
CHECK(S != nullptr);
CHECK(S->get_id() == local_idx_);
if (!intentional_fork_) {
@ -331,7 +382,7 @@ void CatChainReceiverImpl::add_block(td::BufferSlice payload, std::vector<CatCha
std::vector<tl_object_ptr<ton_api::catchain_block_dep>> deps_arr;
deps_arr.resize(deps.size());
for (size_t i = 0; i < deps.size(); i++) {
auto B = get_block(deps[i]);
CatChainReceivedBlock *B = get_block(deps[i]);
LOG_CHECK(B != nullptr) << this << ": cannot find block with hash " << deps[i];
if (!intentional_fork_) {
CHECK(B->get_source_id() != local_idx_);
@ -339,13 +390,13 @@ void CatChainReceiverImpl::add_block(td::BufferSlice payload, std::vector<CatCha
deps_arr[i] = B->export_tl_dep();
}
auto height = prev->height_ + 1;
int height = prev->height_ + 1;
auto block_data = create_tl_object<ton_api::catchain_block_data>(std::move(prev), std::move(deps_arr));
auto block = create_tl_object<ton_api::catchain_block>(incarnation_, local_idx_, height, std::move(block_data),
td::BufferSlice());
auto id = CatChainReceivedBlock::block_id(this, block, payload);
auto id_s = serialize_tl_object(id, true);
td::BufferSlice id_s = serialize_tl_object(id, true);
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), print_id = print_id(), block = std::move(block),
payload = std::move(payload)](td::Result<td::BufferSlice> R) mutable {
@ -362,24 +413,24 @@ void CatChainReceiverImpl::add_block(td::BufferSlice payload, std::vector<CatCha
void CatChainReceiverImpl::debug_add_fork_cont(tl_object_ptr<ton_api::catchain_block> block, td::BufferSlice payload) {
validate_block_sync(block, payload.as_slice()).ensure();
auto B = create_block(std::move(block), td::SharedSlice{payload.as_slice()});
CatChainReceivedBlock *B = create_block(std::move(block), td::SharedSlice{payload.as_slice()});
B->written();
run_scheduler();
CHECK(B->delivered());
active_send_ = false;
if (pending_blocks_.size() > 0) {
auto B = std::move(pending_blocks_.front());
if (!pending_blocks_.empty()) {
auto pending_block = std::move(pending_blocks_.front());
pending_blocks_.pop_front();
add_block(std::move(B->payload_), std::move(B->deps_));
add_block(std::move(pending_block->payload_), std::move(pending_block->deps_));
}
}
void CatChainReceiverImpl::debug_add_fork(td::BufferSlice payload, CatChainBlockHeight height,
std::vector<CatChainBlockHash> deps) {
intentional_fork_ = true;
auto S = get_source_by_hash(local_id_);
CatChainReceiverSource *S = get_source_by_hash(local_id_);
CHECK(S != nullptr);
CHECK(S->get_id() == local_idx_);
@ -399,7 +450,7 @@ void CatChainReceiverImpl::debug_add_fork(td::BufferSlice payload, CatChainBlock
std::vector<tl_object_ptr<ton_api::catchain_block_dep>> deps_arr;
deps_arr.resize(deps.size());
for (size_t i = 0; i < deps.size(); i++) {
auto B = get_block(deps[i]);
CatChainReceivedBlock *B = get_block(deps[i]);
LOG_CHECK(B != nullptr) << this << ": cannot find block with hash " << deps[i];
CHECK(B->get_source_id() != local_idx_);
deps_arr[i] = B->export_tl_dep();
@ -410,7 +461,7 @@ void CatChainReceiverImpl::debug_add_fork(td::BufferSlice payload, CatChainBlock
td::BufferSlice());
auto id = CatChainReceivedBlock::block_id(this, block, payload);
auto id_s = serialize_tl_object(id, true);
td::BufferSlice id_s = serialize_tl_object(id, true);
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), print_id = print_id(), block = std::move(block),
payload = std::move(payload)](td::Result<td::BufferSlice> R) mutable {
@ -425,29 +476,33 @@ void CatChainReceiverImpl::debug_add_fork(td::BufferSlice payload, CatChainBlock
td::actor::send_closure_later(keyring_, &keyring::Keyring::sign_message, local_id_, std::move(id_s), std::move(P));
}
CatChainReceiverImpl::CatChainReceiverImpl(std::unique_ptr<Callback> callback, CatChainOptions opts,
CatChainReceiverImpl::CatChainReceiverImpl(std::unique_ptr<Callback> callback,
const CatChainOptions &opts,
td::actor::ActorId<keyring::Keyring> keyring,
td::actor::ActorId<adnl::Adnl> adnl,
td::actor::ActorId<overlay::Overlays> overlay_manager,
std::vector<CatChainNode> ids, PublicKeyHash local_id,
CatChainSessionId unique_hash, std::string db_root, std::string db_suffix,
const std::vector<CatChainNode> &ids,
const PublicKeyHash &local_id,
const CatChainSessionId &unique_hash,
std::string db_root,
std::string db_suffix,
bool allow_unsafe_self_blocks_resync)
: callback_(std::move(callback))
, opts_(std::move(opts))
, keyring_(keyring)
, adnl_(adnl)
, overlay_manager_(overlay_manager)
, opts_(opts)
, keyring_(std::move(keyring))
, adnl_(std::move(adnl))
, overlay_manager_(std::move(overlay_manager))
, local_id_(local_id)
, db_root_(db_root)
, db_suffix_(db_suffix)
, db_root_(std::move(db_root))
, db_suffix_(std::move(db_suffix))
, allow_unsafe_self_blocks_resync_(allow_unsafe_self_blocks_resync) {
std::vector<td::Bits256> short_ids;
local_idx_ = static_cast<td::uint32>(ids.size());
for (auto &id : ids) {
td::uint32 seq = static_cast<td::uint32>(sources_.size());
for (const CatChainNode &id : ids) {
auto seq = static_cast<td::uint32>(sources_.size());
auto R = CatChainReceiverSource::create(this, id.pub_key, id.adnl_id, seq);
auto S = R.move_as_ok();
auto h = id.pub_key.compute_short_id();
PublicKeyHash h = id.pub_key.compute_short_id();
short_ids.push_back(h.bits256_value());
sources_hashes_[h] = seq;
sources_adnl_addrs_[id.adnl_id] = seq;
@ -477,12 +532,13 @@ CatChainReceiverImpl::CatChainReceiverImpl(std::unique_ptr<Callback> callback, C
void CatChainReceiverImpl::start_up() {
std::vector<adnl::AdnlNodeIdShort> ids;
ids.reserve(get_sources_cnt());
for (td::uint32 i = 0; i < get_sources_cnt(); i++) {
ids.push_back(get_source(i)->get_adnl_id());
}
std::map<PublicKeyHash, td::uint32> root_keys;
for (td::uint32 i = 0; i < get_sources_cnt(); i++) {
root_keys.emplace(get_source(i)->get_hash(), 16 << 20);
root_keys.emplace(get_source(i)->get_hash(), OVERLAY_MAX_ALLOWED_PACKET_SIZE);
}
td::actor::send_closure(overlay_manager_, &overlay::Overlays::create_private_overlay,
get_source(local_idx_)->get_adnl_id(), overlay_full_id_.clone(), std::move(ids),
@ -498,13 +554,13 @@ void CatChainReceiverImpl::start_up() {
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<DbType::GetResult> R) {
R.ensure();
auto g = R.move_as_ok();
DbType::GetResult g = R.move_as_ok();
if (g.status == td::KeyValue::GetStatus::NotFound) {
td::actor::send_closure(SelfId, &CatChainReceiverImpl::read_db);
} else {
auto B = std::move(g.value);
CHECK(B.size() == 32);
td::BufferSlice B = std::move(g.value);
CatChainBlockHash x;
CHECK(B.size() == x.as_array().size());
as_slice(x).copy_from(B.as_slice());
td::actor::send_closure(SelfId, &CatChainReceiverImpl::read_db_from, x);
}
@ -527,7 +583,7 @@ void CatChainReceiverImpl::read_db_from(CatChainBlockHash id) {
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), id](td::Result<DbType::GetResult> R) {
R.ensure();
auto g = R.move_as_ok();
DbType::GetResult g = R.move_as_ok();
CHECK(g.status == td::KeyValue::GetStatus::Ok);
td::actor::send_closure(SelfId, &CatChainReceiverImpl::read_block_from_db, id, std::move(g.value));
@ -543,12 +599,12 @@ void CatChainReceiverImpl::read_block_from_db(CatChainBlockHash id, td::BufferSl
F.ensure();
auto block = F.move_as_ok();
auto payload = std::move(data);
td::BufferSlice payload = std::move(data);
auto block_id = CatChainReceivedBlock::block_hash(this, block, payload);
CatChainBlockHash block_id = CatChainReceivedBlock::block_hash(this, block, payload);
CHECK(block_id == id);
auto B = get_block(id);
CatChainReceivedBlock *B = get_block(id);
if (B && B->initialized()) {
CHECK(B->in_db());
if (!pending_in_db_) {
@ -557,7 +613,7 @@ void CatChainReceiverImpl::read_block_from_db(CatChainBlockHash id, td::BufferSl
return;
}
auto source = get_source(block->src_);
CatChainReceiverSource *source = get_source(block->src_);
CHECK(source != nullptr);
CHECK(block->incarnation_ == incarnation_);
@ -565,18 +621,18 @@ void CatChainReceiverImpl::read_block_from_db(CatChainBlockHash id, td::BufferSl
validate_block_sync(block, payload).ensure();
B = create_block(std::move(block), td::SharedSlice{payload.as_slice()});
B->written();
CHECK(B);
B->written();
auto deps = B->get_dep_hashes();
std::vector<CatChainBlockHash> deps = B->get_dep_hashes();
deps.push_back(B->get_prev_hash());
for (auto &dep : deps) {
auto dep_block = get_block(dep);
for (const CatChainBlockHash &dep : deps) {
CatChainReceivedBlock *dep_block = get_block(dep);
if (!dep_block || !dep_block->initialized()) {
pending_in_db_++;
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), dep](td::Result<DbType::GetResult> R) {
R.ensure();
auto g = R.move_as_ok();
DbType::GetResult g = R.move_as_ok();
CHECK(g.status == td::KeyValue::GetStatus::Ok);
td::actor::send_closure(SelfId, &CatChainReceiverImpl::read_block_from_db, dep, std::move(g.value));
@ -601,26 +657,28 @@ void CatChainReceiverImpl::read_db() {
read_db_ = true;
next_rotate_ = td::Timestamp::in(60 + td::Random::fast(0, 60));
next_sync_ = td::Timestamp::in(0.001 * td::Random::fast(0, 60));
initial_sync_complete_at_ = td::Timestamp::in(allow_unsafe_self_blocks_resync_ ? 300.0 : 5.0);
next_rotate_ = td::Timestamp::in(td::Random::fast(NEIGHBOURS_ROTATE_INTERVAL_MIN, NEIGHBOURS_ROTATE_INTERVAL_MAX));
next_sync_ = td::Timestamp::in(
0.001 * td::Random::fast(NEIGHBOURS_ROTATE_INTERVAL_MIN, NEIGHBOURS_ROTATE_INTERVAL_MAX));
initial_sync_complete_at_ = td::Timestamp::in(
allow_unsafe_self_blocks_resync_ ? EXPECTED_UNSAFE_INITIAL_SYNC_DURATION : EXPECTED_INITIAL_SYNC_DURATION);
alarm_timestamp().relax(next_rotate_);
alarm_timestamp().relax(next_sync_);
alarm_timestamp().relax(initial_sync_complete_at_);
}
td::actor::ActorOwn<CatChainReceiverInterface> CatChainReceiverInterface::create(
std::unique_ptr<Callback> callback, CatChainOptions opts, td::actor::ActorId<keyring::Keyring> keyring,
std::unique_ptr<Callback> callback, const CatChainOptions &opts, td::actor::ActorId<keyring::Keyring> keyring,
td::actor::ActorId<adnl::Adnl> adnl, td::actor::ActorId<overlay::Overlays> overlay_manager,
std::vector<CatChainNode> ids, PublicKeyHash local_id, CatChainSessionId unique_hash, std::string db_root,
std::string db_suffix, bool allow_unsafe_self_blocks_resync) {
const std::vector<CatChainNode> &ids, const PublicKeyHash &local_id, const CatChainSessionId &unique_hash,
std::string db_root, std::string db_suffix, bool allow_unsafe_self_blocks_resync) {
auto A = td::actor::create_actor<CatChainReceiverImpl>(
"catchainreceiver", std::move(callback), std::move(opts), keyring, adnl, overlay_manager, std::move(ids),
local_id, unique_hash, db_root, db_suffix, allow_unsafe_self_blocks_resync);
"catchainreceiver", std::move(callback), opts, std::move(keyring), std::move(adnl), std::move(overlay_manager),
ids, local_id, unique_hash, std::move(db_root), std::move(db_suffix), allow_unsafe_self_blocks_resync);
return std::move(A);
}
CatChainReceiverSource *CatChainReceiverImpl::get_source_by_hash(PublicKeyHash source_hash) const {
CatChainReceiverSource *CatChainReceiverImpl::get_source_by_hash(const PublicKeyHash &source_hash) const {
auto it = sources_hashes_.find(source_hash);
if (it == sources_hashes_.end()) {
return nullptr;
@ -650,10 +708,10 @@ void CatChainReceiverImpl::receive_query_from_overlay(adnl::AdnlNodeIdShort src,
return;
}
auto f = F.move_as_ok();
ton_api::downcast_call(*f.get(), [&](auto &obj) { this->process_query(src, obj, std::move(promise)); });
ton_api::downcast_call(*f, [&](auto &obj) { this->process_query(src, std::move(obj), std::move(promise)); });
}
void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getBlock &query,
void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getBlock query,
td::Promise<td::BufferSlice> promise) {
auto it = blocks_.find(query.block_);
if (it == blocks_.end() || it->second->get_height() == 0 || !it->second->initialized()) {
@ -664,19 +722,20 @@ void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::cat
}
}
void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getBlocks &query,
void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getBlocks query,
td::Promise<td::BufferSlice> promise) {
if (query.blocks_.size() > 100) {
if (query.blocks_.size() > MAX_QUERY_BLOCKS) {
promise.set_error(td::Status::Error(ErrorCode::protoviolation, "too many blocks"));
return;
}
td::int32 cnt = 0;
for (auto &b : query.blocks_) {
for (const CatChainBlockHash &b : query.blocks_) {
auto it = blocks_.find(b);
if (it != blocks_.end() && it->second->get_height() > 0) {
auto block = create_tl_object<ton_api::catchain_blockUpdate>(it->second->export_tl());
CHECK(it->second->get_payload().size() > 0);
auto B = serialize_tl_object(block, true, it->second->get_payload().clone());
CHECK(!it->second->get_payload().empty());
td::BufferSlice B = serialize_tl_object(block, true, it->second->get_payload().clone());
CHECK(B.size() <= opts_.max_serialized_block_size);
td::actor::send_closure(overlay_manager_, &overlay::Overlays::send_message, src,
get_source(local_idx_)->get_adnl_id(), overlay_id_, std::move(B));
cnt++;
@ -685,19 +744,19 @@ void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::cat
promise.set_value(serialize_tl_object(create_tl_object<ton_api::catchain_sent>(cnt), true));
}
void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getBlockHistory &query,
void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getBlockHistory query,
td::Promise<td::BufferSlice> promise) {
auto h = query.height_;
int64_t h = query.height_;
if (h <= 0) {
promise.set_error(td::Status::Error(ErrorCode::protoviolation, "not-positive height"));
return;
}
if (h > 100) {
h = 100;
if (h > MAX_QUERY_HEIGHT) {
h = MAX_QUERY_HEIGHT;
}
std::set<CatChainBlockHash> s{query.stop_if_.begin(), query.stop_if_.end()};
auto B = get_block(query.block_);
CatChainReceivedBlock *B = get_block(query.block_);
if (B == nullptr) {
promise.set_value(serialize_tl_object(create_tl_object<ton_api::catchain_sent>(0), true));
return;
@ -711,8 +770,9 @@ void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::cat
break;
}
auto block = create_tl_object<ton_api::catchain_blockUpdate>(B->export_tl());
CHECK(B->get_payload().size() > 0);
auto BB = serialize_tl_object(block, true, B->get_payload().as_slice());
CHECK(!B->get_payload().empty());
td::BufferSlice BB = serialize_tl_object(block, true, B->get_payload().as_slice());
CHECK(BB.size() <= opts_.max_serialized_block_size);
td::actor::send_closure(overlay_manager_, &overlay::Overlays::send_message, src,
get_source(local_idx_)->get_adnl_id(), overlay_id_, std::move(BB));
B = B->get_prev();
@ -721,7 +781,7 @@ void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::cat
promise.set_value(serialize_tl_object(create_tl_object<ton_api::catchain_sent>(cnt), true));
}
void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getDifference &query,
void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getDifference query,
td::Promise<td::BufferSlice> promise) {
auto &vt = query.rt_;
if (vt.size() != get_sources_cnt()) {
@ -731,7 +791,7 @@ void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::cat
}
for (td::uint32 i = 0; i < get_sources_cnt(); i++) {
if (vt[i] >= 0) {
auto S = get_source(i);
CatChainReceiverSource *S = get_source(i);
if (S->fork_is_found()) {
auto obj = fetch_tl_object<ton_api::catchain_block_data_fork>(S->fork_proof(), true);
obj.ensure();
@ -744,26 +804,21 @@ void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::cat
}
std::vector<td::int32> my_vt(get_sources_cnt());
td::uint64 total = 0;
for (td::uint32 i = 0; i < get_sources_cnt(); i++) {
if (vt[i] >= 0) {
auto x = static_cast<CatChainBlockHeight>(vt[i]);
auto S = get_source(i);
if (S->delivered_height() > x) {
total += S->delivered_height() - x;
}
my_vt[i] = S->delivered_height();
CatChainReceiverSource *S = get_source(i);
my_vt[i] = static_cast<td::int32>(S->delivered_height());
} else {
my_vt[i] = -1;
}
}
const td::uint32 max_send = 100;
const td::uint32 max_send = GET_DIFFERENCE_MAX_SEND;
td::int32 l = 0;
td::int32 r = max_send + 1;
while (r - l > 1) {
td::int32 x = (r + l) / 2;
td::int32 left = 0;
td::int32 right = max_send + 1;
while (right - left > 1) {
td::int32 x = (right + left) / 2;
td::uint64 sum = 0;
for (td::uint32 i = 0; i < get_sources_cnt(); i++) {
if (vt[i] >= 0 && my_vt[i] > vt[i]) {
@ -771,23 +826,23 @@ void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::cat
}
}
if (sum > max_send) {
r = x;
right = x;
} else {
l = x;
left = x;
}
}
CHECK(r > 0);
CHECK(right > 0);
for (td::uint32 i = 0; i < get_sources_cnt(); i++) {
if (vt[i] >= 0 && my_vt[i] > vt[i]) {
auto S = get_source(i);
auto t = (my_vt[i] - vt[i] > r) ? r : (my_vt[i] - vt[i]);
CHECK(t > 0);
CatChainReceiverSource *S = get_source(i);
td::int32 t = (my_vt[i] - vt[i] > right) ? right : (my_vt[i] - vt[i]);
while (t-- > 0) {
auto M = S->get_block(++vt[i]);
CatChainReceivedBlock *M = S->get_block(++vt[i]);
CHECK(M != nullptr);
auto block = create_tl_object<ton_api::catchain_blockUpdate>(M->export_tl());
CHECK(M->get_payload().size() > 0);
auto BB = serialize_tl_object(block, true, M->get_payload().as_slice());
CHECK(!M->get_payload().empty());
td::BufferSlice BB = serialize_tl_object(block, true, M->get_payload().as_slice());
CHECK(BB.size() <= opts_.max_serialized_block_size);
td::actor::send_closure(overlay_manager_, &overlay::Overlays::send_message, src,
get_source(local_idx_)->get_adnl_id(), overlay_id_, std::move(BB));
}
@ -798,7 +853,7 @@ void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::cat
}
void CatChainReceiverImpl::got_fork_proof(td::BufferSlice data) {
auto F = fetch_tl_object<ton_api::catchain_differenceFork>(std::move(data), true);
auto F = fetch_tl_object<ton_api::catchain_differenceFork>(data, true);
if (F.is_error()) {
VLOG(CATCHAIN_WARNING) << this << ": received bad fork proof: " << F.move_as_error();
return;
@ -825,7 +880,7 @@ void CatChainReceiverImpl::got_fork_proof(td::BufferSlice data) {
return;
}
auto S = get_source(f->left_->src_);
CatChainReceiverSource *S = get_source(f->left_->src_);
S->on_found_fork_proof(
create_serialize_tl_object<ton_api::catchain_block_data_fork>(std::move(f->left_), std::move(f->right_)));
S->blame();
@ -835,24 +890,24 @@ void CatChainReceiverImpl::synchronize_with(CatChainReceiverSource *S) {
CHECK(!S->blamed());
std::vector<td::int32> rt(get_sources_cnt());
for (td::uint32 i = 0; i < get_sources_cnt(); i++) {
auto SS = get_source(i);
CatChainReceiverSource *SS = get_source(i);
if (SS->blamed()) {
rt[i] = -1;
} else {
rt[i] = S->delivered_height();
rt[i] = static_cast<td::int32>(S->delivered_height());
}
}
auto P = td::PromiseCreator::lambda(
[SelfId = actor_id(this), src = S->get_hash(), print_id = print_id()](td::Result<td::BufferSlice> R) {
if (R.is_error()) {
VLOG(CATCHAIN_INFO) << print_id << ": timedout syncronize query to " << src;
VLOG(CATCHAIN_INFO) << print_id << ": timedout synchronize query to " << src;
return;
}
auto data = R.move_as_ok();
td::BufferSlice data = R.move_as_ok();
auto X = fetch_tl_object<ton_api::catchain_Difference>(data.clone(), true);
if (X.is_error()) {
VLOG(CATCHAIN_WARNING) << print_id << ": received incorrect answer to syncronize query from " << src << ": "
VLOG(CATCHAIN_WARNING) << print_id << ": received incorrect answer to synchronize query from " << src << ": "
<< X.move_as_error();
return;
}
@ -866,49 +921,49 @@ void CatChainReceiverImpl::synchronize_with(CatChainReceiverSource *S) {
});
td::actor::send_closure(overlay_manager_, &overlay::Overlays::send_query, S->get_adnl_id(),
get_source(local_idx_)->get_adnl_id(), overlay_id_, "sync", std::move(P),
td::Timestamp::in(5.0),
td::Timestamp::in(GET_DIFFERENCE_TIMEOUT),
serialize_tl_object(create_tl_object<ton_api::catchain_getDifference>(std::move(rt)), true));
if (S->delivered_height() < S->received_height()) {
auto B = S->get_block(S->delivered_height() + 1);
CatChainReceivedBlock *B = S->get_block(S->delivered_height() + 1);
CHECK(B->initialized());
std::vector<CatChainBlockHash> vec;
B->find_pending_deps(vec, 16);
B->find_pending_deps(vec, MAX_PENDING_DEPS);
for (auto &hash : vec) {
auto P = td::PromiseCreator::lambda(
for (const CatChainBlockHash &hash : vec) {
auto PP = td::PromiseCreator::lambda(
[SelfId = actor_id(this), print_id = print_id(), src = S->get_adnl_id()](td::Result<td::BufferSlice> R) {
if (R.is_error()) {
VLOG(CATCHAIN_INFO) << print_id << ": timedout syncronize query to " << src;
VLOG(CATCHAIN_INFO) << print_id << ": timedout synchronize query to " << src;
} else {
td::actor::send_closure(SelfId, &CatChainReceiverImpl::receive_block_answer, src, R.move_as_ok());
}
});
auto query = serialize_tl_object(create_tl_object<ton_api::catchain_getBlock>(hash), true);
td::BufferSlice query = serialize_tl_object(create_tl_object<ton_api::catchain_getBlock>(hash), true);
td::actor::send_closure(overlay_manager_, &overlay::Overlays::send_query, S->get_adnl_id(),
get_source(local_idx_)->get_adnl_id(), overlay_id_, "sync blocks", std::move(P),
td::Timestamp::in(2.0), std::move(query));
get_source(local_idx_)->get_adnl_id(), overlay_id_, "sync blocks", std::move(PP),
td::Timestamp::in(GET_BLOCK_TIMEOUT), std::move(query));
}
}
}
void CatChainReceiverImpl::choose_neighbours() {
std::vector<td::uint32> n;
n.resize(get_max_neighbours());
n.resize(MAX_NEIGHBOURS);
td::uint32 size = 0;
for (td::uint32 i = 0; i < get_sources_cnt(); i++) {
if (i == local_idx_) {
continue;
}
auto S = get_source(i);
CatChainReceiverSource *S = get_source(i);
if (!S->blamed()) {
size++;
if (size <= n.size()) {
n[size - 1] = i;
} else {
td::uint32 id = td::Random::fast(0, size - 1);
td::uint32 id = td::Random::fast(0, static_cast<td::int32>(size) - 1);
if (id < n.size()) {
n[id] = i;
}
@ -922,15 +977,15 @@ void CatChainReceiverImpl::choose_neighbours() {
}
bool CatChainReceiverImpl::unsafe_start_up_check_completed() {
auto S = get_source(local_idx_);
CatChainReceiverSource *S = get_source(local_idx_);
CHECK(!S->blamed());
if (S->has_unreceived() || S->has_undelivered()) {
LOG(INFO) << "catchain: has_unreceived=" << S->has_unreceived() << " has_undelivered=" << S->has_undelivered();
run_scheduler();
initial_sync_complete_at_ = td::Timestamp::in(60.0);
initial_sync_complete_at_ = td::Timestamp::in(EXPECTED_INITIAL_SYNC_DURATION_WITH_UNPROCESSED);
return false;
}
auto h = S->delivered_height();
CatChainBlockHeight h = S->delivered_height();
if (h == 0) {
CHECK(last_sent_block_->get_height() == 0);
CHECK(!unsafe_root_block_writing_);
@ -941,20 +996,20 @@ bool CatChainReceiverImpl::unsafe_start_up_check_completed() {
return true;
}
if (unsafe_root_block_writing_) {
initial_sync_complete_at_ = td::Timestamp::in(5.0);
initial_sync_complete_at_ = td::Timestamp::in(EXPECTED_INITIAL_SYNC_DURATION);
LOG(INFO) << "catchain: writing=true";
return false;
}
unsafe_root_block_writing_ = true;
auto B = S->get_block(h);
CatChainReceivedBlock *B = S->get_block(h);
CHECK(B != nullptr);
CHECK(B->delivered());
CHECK(B->in_db());
auto id = B->get_hash();
CatChainBlockHash id = B->get_hash();
td::BufferSlice raw_data{32};
td::BufferSlice raw_data{id.as_array().size()};
raw_data.as_slice().copy_from(as_slice(id));
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), block = B](td::Result<td::Unit> R) mutable {
@ -963,7 +1018,7 @@ bool CatChainReceiverImpl::unsafe_start_up_check_completed() {
});
db_.set(CatChainBlockHash::zero(), std::move(raw_data), std::move(P), 0);
initial_sync_complete_at_ = td::Timestamp::in(5.0);
initial_sync_complete_at_ = td::Timestamp::in(EXPECTED_INITIAL_SYNC_DURATION);
LOG(INFO) << "catchain: need update root";
return false;
}
@ -977,9 +1032,9 @@ void CatChainReceiverImpl::written_unsafe_root_block(CatChainReceivedBlock *bloc
void CatChainReceiverImpl::alarm() {
alarm_timestamp() = td::Timestamp::never();
if (next_sync_ && next_sync_.is_in_past()) {
next_sync_ = td::Timestamp::in(td::Random::fast(0.1, 0.2));
for (auto i = 0; i < 3; i++) {
auto S = get_source(td::Random::fast(0, get_sources_cnt() - 1));
next_sync_ = td::Timestamp::in(td::Random::fast(SYNC_INTERVAL_MIN, SYNC_INTERVAL_MAX));
for (unsigned i = 0; i < SYNC_ITERATIONS; i++) {
CatChainReceiverSource *S = get_source(td::Random::fast(0, static_cast<td::int32>(get_sources_cnt()) - 1));
CHECK(S != nullptr);
if (!S->blamed()) {
synchronize_with(S);
@ -988,7 +1043,7 @@ void CatChainReceiverImpl::alarm() {
}
}
if (next_rotate_ && next_rotate_.is_in_past()) {
next_rotate_ = td::Timestamp::in(td::Random::fast(60.0, 120.0));
next_rotate_ = td::Timestamp::in(td::Random::fast(NEIGHBOURS_ROTATE_INTERVAL_MIN, NEIGHBOURS_ROTATE_INTERVAL_MAX));
choose_neighbours();
}
if (!started_ && read_db_ && initial_sync_complete_at_ && initial_sync_complete_at_.is_in_past()) {
@ -1013,58 +1068,68 @@ void CatChainReceiverImpl::send_fec_broadcast(td::BufferSlice data) {
td::actor::send_closure(overlay_manager_, &overlay::Overlays::send_broadcast_fec_ex,
get_source(local_idx_)->get_adnl_id(), overlay_id_, local_id_, 0, std::move(data));
}
void CatChainReceiverImpl::send_custom_query_data(PublicKeyHash dst, std::string name,
void CatChainReceiverImpl::send_custom_query_data(const PublicKeyHash &dst, std::string name,
td::Promise<td::BufferSlice> promise, td::Timestamp timeout,
td::BufferSlice query) {
auto S = get_source_by_hash(dst);
CatChainReceiverSource *S = get_source_by_hash(dst);
CHECK(S != nullptr);
td::actor::send_closure(overlay_manager_, &overlay::Overlays::send_query, S->get_adnl_id(),
get_source(local_idx_)->get_adnl_id(), overlay_id_, std::move(name), std::move(promise),
timeout, std::move(query));
}
void CatChainReceiverImpl::send_custom_query_data_via(PublicKeyHash dst, std::string name,
void CatChainReceiverImpl::send_custom_query_data_via(const PublicKeyHash &dst, std::string name,
td::Promise<td::BufferSlice> promise, td::Timestamp timeout,
td::BufferSlice query, td::uint64 max_answer_size,
td::actor::ActorId<adnl::AdnlSenderInterface> via) {
auto S = get_source_by_hash(dst);
CatChainReceiverSource *S = get_source_by_hash(dst);
CHECK(S != nullptr);
td::actor::send_closure(overlay_manager_, &overlay::Overlays::send_query_via, S->get_adnl_id(),
get_source(local_idx_)->get_adnl_id(), overlay_id_, std::move(name), std::move(promise),
timeout, std::move(query), max_answer_size, via);
}
void CatChainReceiverImpl::send_custom_message_data(PublicKeyHash dst, td::BufferSlice data) {
auto S = get_source_by_hash(dst);
void CatChainReceiverImpl::send_custom_message_data(const PublicKeyHash &dst, td::BufferSlice data) {
CatChainReceiverSource *S = get_source_by_hash(dst);
CHECK(S != nullptr);
td::actor::send_closure(overlay_manager_, &overlay::Overlays::send_message, S->get_adnl_id(),
get_source(local_idx_)->get_adnl_id(), overlay_id_, std::move(data));
}
void CatChainReceiverImpl::block_written_to_db(CatChainBlockHash hash) {
auto block = get_block(hash);
CatChainReceivedBlock *block = get_block(hash);
CHECK(block);
block->written();
run_scheduler();
}
static void destroy_db(std::string name, td::uint32 attempt) {
static void destroy_db(const std::string& name, td::uint32 attempt) {
auto S = td::RocksDb::destroy(name);
if (S.is_ok()) {
return;
}
if (S.is_error() && attempt >= 10) {
LOG(ERROR) << "failed to destroy catchain " << name << ": " << S;
} else {
if (S.is_error()) {
LOG(DEBUG) << "failed to destroy catchain " << name << ": " << S;
delay_action([name, attempt]() { destroy_db(name, attempt); }, td::Timestamp::in(1.0));
if (attempt < DESTROY_DB_MAX_ATTEMPTS) {
delay_action([name, attempt]() { destroy_db(name, attempt + 1); }, td::Timestamp::in(DESTROY_DB_DELAY));
}
}
}
void CatChainReceiverImpl::destroy() {
auto name = db_root_ + "/catchainreceiver" + db_suffix_ + td::base64url_encode(as_slice(incarnation_));
delay_action([name]() { destroy_db(name, 0); }, td::Timestamp::in(1.0));
delay_action([name]() { destroy_db(name, 0); }, td::Timestamp::in(DESTROY_DB_DELAY));
stop();
}
td::uint64 get_max_block_height(const CatChainOptions& opts, size_t sources_cnt) {
if (opts.max_block_height_coeff == 0) {
return std::numeric_limits<td::uint64>::max();
}
return opts.max_block_height_coeff * (1 + (sources_cnt + opts.max_deps - 1) / opts.max_deps) / 1000;
}
} // namespace catchain
} // namespace ton

10
catchain/catchain-receiver.h
View file

@ -44,9 +44,6 @@ class CatChainReceiver : public CatChainReceiverInterface {
CatChainSessionId instance_;
PublicKeyHash local_id_;
};
td::uint32 get_max_neighbours() const {
return 5;
}
virtual PrintId print_id() const = 0;
virtual CatChainReceivedBlock *create_block(tl_object_ptr<ton_api::catchain_block> block,
td::SharedSlice payload) = 0;
@ -64,12 +61,15 @@ class CatChainReceiver : public CatChainReceiverInterface {
virtual const CatChainOptions &opts() const = 0;
virtual td::Status validate_block_sync(tl_object_ptr<ton_api::catchain_block_dep> &dep) = 0;
virtual td::Status validate_block_sync(tl_object_ptr<ton_api::catchain_block> &block, td::Slice payload) = 0;
virtual td::Status validate_block_sync(const tl_object_ptr<ton_api::catchain_block_dep> &dep) const = 0;
virtual td::Status validate_block_sync(const tl_object_ptr<ton_api::catchain_block> &block,
const td::Slice &payload) const = 0;
virtual ~CatChainReceiver() = default;
};
td::uint64 get_max_block_height(const CatChainOptions& opts, size_t sources_cnt);
} // namespace catchain
} // namespace ton

44
catchain/catchain-receiver.hpp
View file

@ -33,7 +33,7 @@ namespace ton {
namespace catchain {
class CatChainReceiverImpl : public CatChainReceiver {
class CatChainReceiverImpl final : public CatChainReceiver {
public:
PrintId print_id() const override {
return PrintId{incarnation_, local_id_};
@ -62,7 +62,7 @@ class CatChainReceiverImpl : public CatChainReceiver {
return sources_[source_id].get();
}
PublicKeyHash get_source_hash(td::uint32 source_id) const override;
CatChainReceiverSource *get_source_by_hash(PublicKeyHash source_hash) const;
CatChainReceiverSource *get_source_by_hash(const PublicKeyHash &source_hash) const;
CatChainReceiverSource *get_source_by_adnl_id(adnl::AdnlNodeIdShort source_hash) const;
td::uint32 add_fork() override;
@ -72,39 +72,39 @@ class CatChainReceiverImpl : public CatChainReceiver {
void receive_message_from_overlay(adnl::AdnlNodeIdShort src, td::BufferSlice data);
void receive_query_from_overlay(adnl::AdnlNodeIdShort src, td::BufferSlice data,
td::Promise<td::BufferSlice> promise);
void process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getBlock &query,
void process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getBlock query, td::Promise<td::BufferSlice> promise);
void process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getBlocks query,
td::Promise<td::BufferSlice> promise);
void process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getBlocks &query,
void process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getBlockHistory query,
td::Promise<td::BufferSlice> promise);
void process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getBlockHistory &query,
td::Promise<td::BufferSlice> promise);
void process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getDifference &query,
void process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getDifference query,
td::Promise<td::BufferSlice> promise);
template <class T>
void process_query(adnl::AdnlNodeIdShort src, T &query, td::Promise<td::BufferSlice> promise) {
void process_query(adnl::AdnlNodeIdShort src, const T &query, td::Promise<td::BufferSlice> promise) {
//LOG(WARNING) << this << ": unknown query from " << src;
callback_->on_custom_query(get_source_by_adnl_id(src)->get_hash(), serialize_tl_object(&query, true),
std::move(promise));
}
void receive_broadcast_from_overlay(PublicKeyHash src, td::BufferSlice data);
void receive_broadcast_from_overlay(const PublicKeyHash &src, td::BufferSlice data);
void receive_block(adnl::AdnlNodeIdShort src, tl_object_ptr<ton_api::catchain_block> block, td::BufferSlice payload);
void receive_block_answer(adnl::AdnlNodeIdShort src, td::BufferSlice);
//void send_block(PublicKeyHash src, tl_object_ptr<ton_api::catchain_block> block, td::BufferSlice payload);
//void send_block(const PublicKeyHash &src, tl_object_ptr<ton_api::catchain_block> block, td::BufferSlice payload);
CatChainReceivedBlock *create_block(tl_object_ptr<ton_api::catchain_block> block, td::SharedSlice payload) override;
CatChainReceivedBlock *create_block(tl_object_ptr<ton_api::catchain_block_dep> block) override;
td::Status validate_block_sync(tl_object_ptr<ton_api::catchain_block_dep> &dep) override;
td::Status validate_block_sync(tl_object_ptr<ton_api::catchain_block> &block, td::Slice payload) override;
td::Status validate_block_sync(const tl_object_ptr<ton_api::catchain_block_dep> &dep) const override;
td::Status validate_block_sync(const tl_object_ptr<ton_api::catchain_block> &block,
const td::Slice &payload) const override;
void send_fec_broadcast(td::BufferSlice data) override;
void send_custom_query_data(PublicKeyHash dst, std::string name, td::Promise<td::BufferSlice> promise,
void send_custom_query_data(const PublicKeyHash &dst, std::string name, td::Promise<td::BufferSlice> promise,
td::Timestamp timeout, td::BufferSlice query) override;
void send_custom_query_data_via(PublicKeyHash dst, std::string name, td::Promise<td::BufferSlice> promise,
void send_custom_query_data_via(const PublicKeyHash &dst, std::string name, td::Promise<td::BufferSlice> promise,
td::Timestamp timeout, td::BufferSlice query, td::uint64 max_answer_size,
td::actor::ActorId<adnl::AdnlSenderInterface> via) override;
void send_custom_message_data(PublicKeyHash dst, td::BufferSlice query) override;
void send_custom_message_data(const PublicKeyHash &dst, td::BufferSlice query) override;
void run_scheduler();
void add_block(td::BufferSlice data, std::vector<CatChainBlockHash> deps) override;
@ -117,8 +117,6 @@ class CatChainReceiverImpl : public CatChainReceiver {
void on_blame(td::uint32 src) override {
callback_->blame(src);
}
void blame_node(td::uint32 idx) override {
}
const CatChainOptions &opts() const override {
return opts_;
}
@ -141,11 +139,11 @@ class CatChainReceiverImpl : public CatChainReceiver {
CatChainReceivedBlock *get_block(CatChainBlockHash hash) const;
CatChainReceiverImpl(std::unique_ptr<Callback> callback, CatChainOptions opts,
CatChainReceiverImpl(std::unique_ptr<Callback> callback, const CatChainOptions &opts,
td::actor::ActorId<keyring::Keyring> keyring, td::actor::ActorId<adnl::Adnl> adnl,
td::actor::ActorId<overlay::Overlays>, std::vector<CatChainNode> ids, PublicKeyHash local_id,
CatChainBlockHash unique_hash, std::string db_root, std::string db_suffix,
bool allow_unsafe_self_blocks_resync);
td::actor::ActorId<overlay::Overlays> overlays, const std::vector<CatChainNode> &ids,
const PublicKeyHash &local_id, const CatChainBlockHash &unique_hash, std::string db_root,
std::string db_suffix, bool allow_unsafe_self_blocks_resync);
private:
std::unique_ptr<overlay::Overlays::Callback> make_callback() {
@ -164,7 +162,7 @@ class CatChainReceiverImpl : public CatChainReceiver {
void receive_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override {
td::actor::send_closure(id_, &CatChainReceiverImpl::receive_broadcast_from_overlay, src, std::move(data));
}
Callback(td::actor::ActorId<CatChainReceiverImpl> id) : id_(std::move(id)) {
explicit Callback(td::actor::ActorId<CatChainReceiverImpl> id) : id_(std::move(id)) {
}
private:
@ -199,7 +197,7 @@ class CatChainReceiverImpl : public CatChainReceiver {
CatChainReceivedBlock *root_block_;
CatChainReceivedBlock *last_sent_block_;
CatChainSessionId incarnation_;
CatChainSessionId incarnation_{};
std::unique_ptr<Callback> callback_;
CatChainOptions opts_;

8
catchain/catchain-types.h
View file

@ -20,6 +20,7 @@
#include "td/utils/int_types.h"
#include "adnl/adnl-node-id.hpp"
#include "ton/ton-types.h"
namespace ton {
@ -35,13 +36,6 @@ struct CatChainNode {
PublicKey pub_key;
};
struct CatChainOptions {
double idle_timeout = 16.0;
td::uint32 max_deps = 4;
bool debug_disable_db = false;
};
} // namespace catchain
} // namespace ton

79
catchain/catchain.cpp
View file

@ -18,9 +18,9 @@
*/
#include "catchain-types.h"
#include "catchain.hpp"
#include "catchain-receiver.h"
#include "adnl/utils.hpp"
#include <utility>
#include "catchain-receiver.h"
namespace ton {
@ -32,10 +32,11 @@ void CatChainImpl::send_process() {
std::vector<CatChainBlock *> v;
std::vector<CatChainBlockHash> w;
while (top_blocks_.size() > 0 && v.size() < opts_.max_deps) {
auto B = *top_blocks_.get_random();
CatChainBlock *B = *top_blocks_.get_random();
CHECK(B != nullptr);
top_blocks_.remove(B->hash());
if (B->source() == sources_.size() || !blamed_sources_[B->source()]) {
CHECK(B->source() < sources_.size());
if (!blamed_sources_[B->source()]) {
w.push_back(B->hash());
v.push_back(B);
set_processed(B);
@ -52,13 +53,13 @@ void CatChainImpl::send_preprocess(CatChainBlock *block) {
if (block->preprocess_is_sent()) {
return;
}
auto prev = block->prev();
CatChainBlock *prev = block->prev();
if (prev) {
send_preprocess(prev);
}
auto deps = block->deps();
for (auto X : deps) {
const std::vector<CatChainBlock *> &deps = block->deps();
for (CatChainBlock *X : deps) {
send_preprocess(X);
}
@ -72,13 +73,13 @@ void CatChainImpl::set_processed(CatChainBlock *block) {
if (block->is_processed()) {
return;
}
auto prev = block->prev();
CatChainBlock *prev = block->prev();
if (prev) {
set_processed(prev);
}
auto deps = block->deps();
for (auto X : deps) {
const std::vector<CatChainBlock *> &deps = block->deps();
for (CatChainBlock *X : deps) {
set_processed(X);
}
@ -133,6 +134,7 @@ void CatChainImpl::on_new_block(td::uint32 src_id, td::uint32 fork, CatChainBloc
}
}
CHECK(src_id < sources_.size());
std::vector<CatChainBlock *> v;
v.resize(deps.size());
for (size_t i = 0; i < deps.size(); i++) {
@ -143,12 +145,12 @@ void CatChainImpl::on_new_block(td::uint32 src_id, td::uint32 fork, CatChainBloc
CHECK(v[i] != nullptr);
}
CHECK(src_id < sources_.size());
auto src_hash = sources_[src_id];
CHECK(height <= get_max_block_height(opts_, sources_.size()));
PublicKeyHash src_hash = sources_[src_id];
blocks_[hash] =
CatChainBlock::create(src_id, fork, src_hash, height, hash, std::move(data), p, std::move(v), std::move(vt));
auto B = get_block(hash);
CatChainBlock *B = get_block(hash);
CHECK(B != nullptr);
if (!blamed_sources_[src_id]) {
@ -177,7 +179,7 @@ void CatChainImpl::on_blame(td::uint32 src_id) {
auto size = static_cast<td::uint32>(sources_.size());
for (td::uint32 i = 0; i < size; i++) {
if (!blamed_sources_[i] && top_source_blocks_[i] && i != local_idx_) {
auto B = top_source_blocks_[i];
CatChainBlock *B = top_source_blocks_[i];
bool f = true;
if (B->is_processed()) {
continue;
@ -197,15 +199,11 @@ void CatChainImpl::on_blame(td::uint32 src_id) {
}
}
void CatChainImpl::on_custom_message(PublicKeyHash src, td::BufferSlice data) {
callback_->process_message(src, std::move(data));
}
void CatChainImpl::on_custom_query(PublicKeyHash src, td::BufferSlice data, td::Promise<td::BufferSlice> promise) {
void CatChainImpl::on_custom_query(const PublicKeyHash &src, td::BufferSlice data, td::Promise<td::BufferSlice> promise) {
callback_->process_query(src, std::move(data), std::move(promise));
}
void CatChainImpl::on_broadcast(PublicKeyHash src, td::BufferSlice data) {
void CatChainImpl::on_broadcast(const PublicKeyHash &src, td::BufferSlice data) {
VLOG(CATCHAIN_INFO) << this << ": processing broadcast";
callback_->process_broadcast(src, std::move(data));
VLOG(CATCHAIN_INFO) << this << ": sent processing broadcast";
@ -219,18 +217,18 @@ void CatChainImpl::on_receiver_started() {
send_process();
}
CatChainImpl::CatChainImpl(std::unique_ptr<Callback> callback, CatChainOptions opts,
CatChainImpl::CatChainImpl(std::unique_ptr<Callback> callback, const CatChainOptions &opts,
td::actor::ActorId<keyring::Keyring> keyring, td::actor::ActorId<adnl::Adnl> adnl,
td::actor::ActorId<overlay::Overlays> overlay_manager, std::vector<CatChainNode> ids,
PublicKeyHash local_id, CatChainSessionId unique_hash, std::string db_root,
std::string db_suffix, bool allow_unsafe_self_blocks_resync)
: opts_(std::move(opts))
, db_root_(db_root)
, db_suffix_(db_suffix)
const PublicKeyHash &local_id, const CatChainSessionId &unique_hash,
std::string db_root, std::string db_suffix, bool allow_unsafe_self_blocks_resync)
: opts_(opts)
, unique_hash_(unique_hash)
, db_root_(std::move(db_root))
, db_suffix_(std::move(db_suffix))
, allow_unsafe_self_blocks_resync_(allow_unsafe_self_blocks_resync) {
callback_ = std::move(callback);
sources_.resize(ids.size());
unique_hash_ = unique_hash;
for (size_t i = 0; i < ids.size(); i++) {
sources_[i] = ids[i].pub_key.compute_short_id();
if (sources_[i] == local_id) {
@ -240,7 +238,8 @@ CatChainImpl::CatChainImpl(std::unique_ptr<Callback> callback, CatChainOptions o
blamed_sources_.resize(ids.size(), false);
top_source_blocks_.resize(ids.size(), nullptr);
args_ = std::make_unique<Args>(keyring, adnl, overlay_manager, std::move(ids), local_id, unique_hash);
args_ = std::make_unique<Args>(std::move(keyring), std::move(adnl), std::move(overlay_manager), std::move(ids),
local_id, unique_hash);
}
void CatChainImpl::alarm() {
@ -263,19 +262,16 @@ void CatChainImpl::start_up() {
void blame(td::uint32 src_id) override {
td::actor::send_closure(id_, &CatChainImpl::on_blame, src_id);
}
void on_custom_message(PublicKeyHash src, td::BufferSlice data) override {
td::actor::send_closure(id_, &CatChainImpl::on_custom_message, src, std::move(data));
}
void on_custom_query(PublicKeyHash src, td::BufferSlice data, td::Promise<td::BufferSlice> promise) override {
void on_custom_query(const PublicKeyHash &src, td::BufferSlice data, td::Promise<td::BufferSlice> promise) override {
td::actor::send_closure(id_, &CatChainImpl::on_custom_query, src, std::move(data), std::move(promise));
}
void on_broadcast(PublicKeyHash src, td::BufferSlice data) override {
void on_broadcast(const PublicKeyHash &src, td::BufferSlice data) override {
td::actor::send_closure(id_, &CatChainImpl::on_broadcast, src, std::move(data));
}
void start() override {
td::actor::send_closure(id_, &CatChainImpl::on_receiver_started);
}
ChainCb(td::actor::ActorId<CatChainImpl> id) : id_(id) {
explicit ChainCb(td::actor::ActorId<CatChainImpl> id) : id_(std::move(id)) {
}
private:
@ -285,22 +281,23 @@ void CatChainImpl::start_up() {
auto cb = std::make_unique<ChainCb>(actor_id(this));
receiver_ = CatChainReceiverInterface::create(
std::move(cb), opts_, args_->keyring, args_->adnl, args_->overlay_manager, std::move(args_->ids), args_->local_id,
std::move(cb), opts_, args_->keyring, args_->adnl, args_->overlay_manager, args_->ids, args_->local_id,
args_->unique_hash, db_root_, db_suffix_, allow_unsafe_self_blocks_resync_);
args_ = nullptr;
//alarm_timestamp() = td::Timestamp::in(opts_.idle_timeout);
}
td::actor::ActorOwn<CatChain> CatChain::create(std::unique_ptr<Callback> callback, CatChainOptions opts,
td::actor::ActorOwn<CatChain> CatChain::create(std::unique_ptr<Callback> callback, const CatChainOptions &opts,
td::actor::ActorId<keyring::Keyring> keyring,
td::actor::ActorId<adnl::Adnl> adnl,
td::actor::ActorId<overlay::Overlays> overlay_manager,
std::vector<CatChainNode> ids, PublicKeyHash local_id,
CatChainSessionId unique_hash, std::string db_root,
std::vector<CatChainNode> ids, const PublicKeyHash &local_id,
const CatChainSessionId &unique_hash, std::string db_root,
std::string db_suffix, bool allow_unsafe_self_blocks_resync) {
return td::actor::create_actor<CatChainImpl>("catchain", std::move(callback), std::move(opts), keyring, adnl,
overlay_manager, std::move(ids), local_id, unique_hash, db_root,
db_suffix, allow_unsafe_self_blocks_resync);
return td::actor::create_actor<CatChainImpl>("catchain", std::move(callback), opts, std::move(keyring),
std::move(adnl), std::move(overlay_manager), std::move(ids),
local_id, unique_hash, std::move(db_root), std::move(db_suffix),
allow_unsafe_self_blocks_resync);
}
CatChainBlock *CatChainImpl::get_block(CatChainBlockHash hash) const {

27
catchain/catchain.h
View file

@ -58,8 +58,8 @@ class CatChainBlock {
virtual bool is_descendant_of(CatChainBlock *block) = 0;
static std::unique_ptr<CatChainBlock> create(td::uint32 src, td::uint32 fork_id, PublicKeyHash src_hash,
CatChainBlockHeight height, CatChainBlockHash hash,
static std::unique_ptr<CatChainBlock> create(td::uint32 src, td::uint32 fork_id, const PublicKeyHash &src_hash,
CatChainBlockHeight height, const CatChainBlockHash &hash,
td::SharedSlice payload, CatChainBlock *prev,
std::vector<CatChainBlock *> deps, std::vector<CatChainBlockHeight> vt);
@ -73,9 +73,10 @@ class CatChain : public td::actor::Actor {
virtual void process_blocks(std::vector<CatChainBlock *> blocks) = 0;
virtual void finished_processing() = 0;
virtual void preprocess_block(CatChainBlock *block) = 0;
virtual void process_broadcast(PublicKeyHash src, td::BufferSlice data) = 0;
virtual void process_message(PublicKeyHash src, td::BufferSlice data) = 0;
virtual void process_query(PublicKeyHash src, td::BufferSlice data, td::Promise<td::BufferSlice> promise) = 0;
virtual void process_broadcast(const PublicKeyHash &src, td::BufferSlice data) = 0;
virtual void process_message(const PublicKeyHash &src, td::BufferSlice data) = 0;
virtual void process_query(const PublicKeyHash &src, td::BufferSlice data,
td::Promise<td::BufferSlice> promise) = 0;
virtual void started() = 0;
virtual ~Callback() = default;
};
@ -89,22 +90,22 @@ class CatChain : public td::actor::Actor {
virtual void debug_add_fork(td::BufferSlice payload, CatChainBlockHeight height) = 0;
virtual void send_broadcast(td::BufferSlice data) = 0;
virtual void send_message(PublicKeyHash dst, td::BufferSlice data) = 0;
virtual void send_query(PublicKeyHash dst, std::string name, td::Promise<td::BufferSlice> promise,
virtual void send_message(const PublicKeyHash &dst, td::BufferSlice data) = 0;
virtual void send_query(const PublicKeyHash &dst, std::string name, td::Promise<td::BufferSlice> promise,
td::Timestamp timeout, td::BufferSlice query) = 0;
virtual void send_query_via(PublicKeyHash dst, std::string name, td::Promise<td::BufferSlice> promise,
virtual void send_query_via(const PublicKeyHash &dst, std::string name, td::Promise<td::BufferSlice> promise,
td::Timestamp timeout, td::BufferSlice query, td::uint64 max_answer_size,
td::actor::ActorId<adnl::AdnlSenderInterface> via) = 0;
virtual void destroy() = 0;
static td::actor::ActorOwn<CatChain> create(std::unique_ptr<Callback> callback, CatChainOptions opts,
static td::actor::ActorOwn<CatChain> create(std::unique_ptr<Callback> callback, const CatChainOptions &opts,
td::actor::ActorId<keyring::Keyring> keyring,
td::actor::ActorId<adnl::Adnl> adnl,
td::actor::ActorId<overlay::Overlays> overlay_manager,
std::vector<CatChainNode> ids, PublicKeyHash local_id,
CatChainSessionId unique_hash, std::string db_root, std::string db_suffix,
bool allow_unsafe_self_blocks_resync);
virtual ~CatChain() = default;
std::vector<CatChainNode> ids, const PublicKeyHash &local_id,
const CatChainSessionId &unique_hash, std::string db_root,
std::string db_suffix, bool allow_unsafe_self_blocks_resync);
~CatChain() override = default;
};
} // namespace catchain

34
catchain/catchain.hpp
View file

@ -67,11 +67,11 @@ class CatChainImpl : public CatChain {
CatChainSessionId unique_hash;
Args(td::actor::ActorId<keyring::Keyring> keyring, td::actor::ActorId<adnl::Adnl> adnl,
td::actor::ActorId<overlay::Overlays> overlay_manager, std::vector<CatChainNode> ids, PublicKeyHash local_id,
CatChainSessionId unique_hash)
: keyring(keyring)
, adnl(adnl)
, overlay_manager(overlay_manager)
td::actor::ActorId<overlay::Overlays> overlay_manager, std::vector<CatChainNode> ids,
const PublicKeyHash &local_id, const CatChainSessionId &unique_hash)
: keyring(std::move(keyring))
, adnl(std::move(adnl))
, overlay_manager(std::move(overlay_manager))
, ids(std::move(ids))
, local_id(local_id)
, unique_hash(unique_hash) {
@ -88,9 +88,8 @@ class CatChainImpl : public CatChain {
CatChainBlockHash prev, std::vector<CatChainBlockHash> deps, std::vector<CatChainBlockHeight> vt,
td::SharedSlice data);
void on_blame(td::uint32 src_id);
void on_custom_message(PublicKeyHash src, td::BufferSlice data);
void on_custom_query(PublicKeyHash src, td::BufferSlice data, td::Promise<td::BufferSlice> promise);
void on_broadcast(PublicKeyHash src, td::BufferSlice data);
void on_custom_query(const PublicKeyHash &src, td::BufferSlice data, td::Promise<td::BufferSlice> promise);
void on_broadcast(const PublicKeyHash &src, td::BufferSlice data);
void on_receiver_started();
void processed_block(td::BufferSlice payload) override;
void need_new_block(td::Timestamp t) override;
@ -102,25 +101,26 @@ class CatChainImpl : public CatChain {
void send_broadcast(td::BufferSlice data) override {
td::actor::send_closure(receiver_, &CatChainReceiverInterface::send_fec_broadcast, std::move(data));
}
void send_message(PublicKeyHash dst, td::BufferSlice data) override {
void send_message(const PublicKeyHash &dst, td::BufferSlice data) override {
td::actor::send_closure(receiver_, &CatChainReceiverInterface::send_custom_message_data, dst, std::move(data));
}
void send_query(PublicKeyHash dst, std::string name, td::Promise<td::BufferSlice> promise, td::Timestamp timeout,
td::BufferSlice query) override {
void send_query(const PublicKeyHash &dst, std::string name, td::Promise<td::BufferSlice> promise,
td::Timestamp timeout, td::BufferSlice query) override {
td::actor::send_closure(receiver_, &CatChainReceiverInterface::send_custom_query_data, dst, name,
std::move(promise), timeout, std::move(query));
}
void send_query_via(PublicKeyHash dst, std::string name, td::Promise<td::BufferSlice> promise, td::Timestamp timeout,
td::BufferSlice query, td::uint64 max_answer_size,
void send_query_via(const PublicKeyHash &dst, std::string name, td::Promise<td::BufferSlice> promise,
td::Timestamp timeout, td::BufferSlice query, td::uint64 max_answer_size,
td::actor::ActorId<adnl::AdnlSenderInterface> via) override {
td::actor::send_closure(receiver_, &CatChainReceiverInterface::send_custom_query_data_via, dst, name,
std::move(promise), timeout, std::move(query), max_answer_size, via);
}
void destroy() override;
CatChainImpl(std::unique_ptr<Callback> callback, CatChainOptions opts, td::actor::ActorId<keyring::Keyring> keyring,
td::actor::ActorId<adnl::Adnl> adnl, td::actor::ActorId<overlay::Overlays> overlay_manager,
std::vector<CatChainNode> ids, PublicKeyHash local_id, CatChainSessionId unique_hash,
std::string db_root, std::string db_suffix, bool allow_unsafe_self_blocks_resync);
CatChainImpl(std::unique_ptr<Callback> callback, const CatChainOptions &opts,
td::actor::ActorId<keyring::Keyring> keyring, td::actor::ActorId<adnl::Adnl> adnl,
td::actor::ActorId<overlay::Overlays> overlay_manager, std::vector<CatChainNode> ids,
const PublicKeyHash &local_id, const CatChainSessionId &unique_hash, std::string db_root,
std::string db_suffix, bool allow_unsafe_self_blocks_resync);
void alarm() override;
void start_up() override;

2
crypto/CMakeLists.txt
View file

@ -36,6 +36,7 @@ set(TON_CRYPTO_SOURCE
vm/debugops.cpp
vm/tonops.cpp
vm/boc.cpp
vm/large-boc-serializer.cpp
vm/utils.cpp
vm/vm.cpp
tl/tlblib.cpp
@ -68,6 +69,7 @@ set(TON_CRYPTO_SOURCE
vm/arithops.h
vm/atom.h
vm/boc.h
vm/boc-writers.h
vm/box.hpp
vm/cellops.h
vm/continuation.h

7
crypto/block/block.tlb
View file

@ -703,6 +703,13 @@ consensus_config_v3#d8 flags:(## 7) { flags = 0 } new_catchain_ids:Bool
max_block_bytes:uint32 max_collated_bytes:uint32
proto_version:uint16 = ConsensusConfig;
consensus_config_v4#d9 flags:(## 7) { flags = 0 } new_catchain_ids:Bool
round_candidates:(## 8) { round_candidates >= 1 }
next_candidate_delay_ms:uint32 consensus_timeout_ms:uint32
fast_attempts:uint32 attempt_duration:uint32 catchain_max_deps:uint32
max_block_bytes:uint32 max_collated_bytes:uint32
proto_version:uint16 catchain_max_blocks_coeff:uint32 = ConsensusConfig;
_ CatchainConfig = ConfigParam 28;
_ ConsensusConfig = ConfigParam 29;

48
crypto/block/mc-config.cpp
View file

@ -310,32 +310,52 @@ td::Status Config::visit_validator_params() const {
ton::ValidatorSessionConfig Config::get_consensus_config() const {
auto cc = get_config_param(29);
ton::ValidatorSessionConfig c;
auto set = [&c](auto& r) {
c.catchain_idle_timeout = r.consensus_timeout_ms * 0.001;
c.catchain_max_deps = r.catchain_max_deps;
auto set_v1 = [&](auto& r) {
c.catchain_opts.idle_timeout = r.consensus_timeout_ms * 0.001;
c.catchain_opts.max_deps = r.catchain_max_deps;
c.round_candidates = r.round_candidates;
c.next_candidate_delay = r.next_candidate_delay_ms * 0.001;
c.round_attempt_duration = r.attempt_duration;
c.max_round_attempts = r.fast_attempts;
c.max_block_size = r.max_block_bytes;
c.max_collated_data_size = r.max_collated_bytes;
return true;
};
auto set_new_cc_ids = [&c] (auto& r) {
auto set_v2 = [&] (auto& r) {
set_v1(r);
c.new_catchain_ids = r.new_catchain_ids;
return true;
};
auto set_proto = [&c](auto& r) {
auto set_v3 = [&](auto& r) {
set_v2(r);
c.proto_version = r.proto_version;
return true;
};
auto set_v4 = [&](auto& r) {
set_v3(r);
td::uint64 max_blocks_coeff = r.catchain_max_blocks_coeff;
if (max_blocks_coeff == 0) {
c.catchain_opts.max_block_height_coeff = 0;
} else {
auto catchain_config = get_catchain_validators_config();
td::uint64 catchain_lifetime = std::max(catchain_config.mc_cc_lifetime, catchain_config.shard_cc_lifetime);
c.catchain_opts.max_block_height_coeff = catchain_lifetime * max_blocks_coeff;
}
};
if (cc.not_null()) {
block::gen::ConsensusConfig::Record_consensus_config_v3 r2;
block::gen::ConsensusConfig::Record_consensus_config_new r1;
block::gen::ConsensusConfig::Record_consensus_config r0;
(tlb::unpack_cell(cc, r2) && set(r2) && set_new_cc_ids(r2) && set_proto(r2)) ||
(tlb::unpack_cell(cc, r1) && set(r1) && set_new_cc_ids(r1)) ||
(tlb::unpack_cell(cc, r0) && set(r0));
block::gen::ConsensusConfig::Record_consensus_config_v4 r4;
block::gen::ConsensusConfig::Record_consensus_config_v3 r3;
block::gen::ConsensusConfig::Record_consensus_config_new r2;
block::gen::ConsensusConfig::Record_consensus_config r1;
if (tlb::unpack_cell(cc, r4)) {
set_v4(r4);
} else if (tlb::unpack_cell(cc, r3)) {
set_v3(r3);
} else if (tlb::unpack_cell(cc, r2)) {
set_v2(r2);
} else if (tlb::unpack_cell(cc, r1)) {
set_v1(r1);
}
}
if (c.proto_version >= ton::ValidatorSessionConfig::BLOCK_HASH_COVERS_DATA_FROM_VERSION) {
c.catchain_opts.block_hash_covers_data = true;
}
return c;
}

47
crypto/block/transaction.cpp
View file

@ -25,6 +25,41 @@
#include "ton/ton-shard.h"
#include "vm/vm.h"
namespace {
class StringLoggerTail : public td::LogInterface {
public:
explicit StringLoggerTail(size_t max_size = 256) : buf(max_size, '\0') {}
void append(td::CSlice slice) override {
if (slice.size() > buf.size()) {
slice.remove_prefix(slice.size() - buf.size());
}
while (!slice.empty()) {
size_t s = std::min(buf.size() - pos, slice.size());
std::copy(slice.begin(), slice.begin() + s, buf.begin() + pos);
pos += s;
if (pos == buf.size()) {
pos = 0;
truncated = true;
}
slice.remove_prefix(s);
}
}
std::string get_log() const {
if (truncated) {
std::string res = buf;
std::rotate(res.begin(), res.begin() + pos, res.end());
return res;
} else {
return buf.substr(0, pos);
}
}
private:
std::string buf;
size_t pos = 0;
bool truncated = false;
};
}
namespace block {
using td::Ref;
@ -1001,7 +1036,14 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) {
vm::GasLimits gas{(long long)cp.gas_limit, (long long)cp.gas_max, (long long)cp.gas_credit};
LOG(DEBUG) << "creating VM";
vm::VmState vm{new_code, std::move(stack), gas, 1, new_data, vm::VmLog(), compute_vm_libraries(cfg)};
std::unique_ptr<StringLoggerTail> logger;
auto vm_log = vm::VmLog();
if (cfg.with_vm_log) {
logger = std::make_unique<StringLoggerTail>();
vm_log.log_interface = logger.get();
vm_log.log_options = td::LogOptions(VERBOSITY_NAME(DEBUG), true, false);
}
vm::VmState vm{new_code, std::move(stack), gas, 1, new_data, vm_log, compute_vm_libraries(cfg)};
vm.set_c7(prepare_vm_c7(cfg)); // tuple with SmartContractInfo
// vm.incr_stack_trace(1); // enable stack dump after each step
@ -1024,6 +1066,9 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) {
LOG(INFO) << "steps: " << vm.get_steps_count() << " gas: used=" << gas.gas_consumed() << ", max=" << gas.gas_max
<< ", limit=" << gas.gas_limit << ", credit=" << gas.gas_credit;
LOG(INFO) << "out_of_gas=" << cp.out_of_gas << ", accepted=" << cp.accepted << ", success=" << cp.success;
if (logger != nullptr) {
cp.vm_log = logger->get_log();
}
if (cp.success) {
cp.new_data = vm.get_committed_state().c4; // c4 -> persistent data
cp.actions = vm.get_committed_state().c5; // c5 -> action list

2
crypto/block/transaction.h
View file

@ -106,6 +106,7 @@ struct ComputePhaseConfig {
std::unique_ptr<vm::Dictionary> libraries;
Ref<vm::Cell> global_config;
td::BitArray<256> block_rand_seed;
bool with_vm_log{false};
ComputePhaseConfig(td::uint64 _gas_price = 0, td::uint64 _gas_limit = 0, td::uint64 _gas_credit = 0)
: gas_price(_gas_price), gas_limit(_gas_limit), special_gas_limit(_gas_limit), gas_credit(_gas_credit) {
compute_threshold();
@ -171,6 +172,7 @@ struct ComputePhase {
Ref<vm::Cell> in_msg;
Ref<vm::Cell> new_data;
Ref<vm::Cell> actions;
std::string vm_log;
};
struct ActionPhase {

146
crypto/vm/boc-writers.h Normal file
View file

@ -0,0 +1,146 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "td/utils/port/FileFd.h"
#include "td/utils/crypto.h"
#include <vector>
namespace vm {
namespace boc_writers {
struct BufferWriter {
BufferWriter(unsigned char* store_start, unsigned char* store_end)
: store_start(store_start), store_ptr(store_start), store_end(store_end) {}
size_t position() const {
return store_ptr - store_start;
}
size_t remaining() const {
return store_end - store_ptr;
}
void chk() const {
DCHECK(store_ptr <= store_end);
}
bool empty() const {
return store_ptr == store_end;
}
void store_uint(unsigned long long value, unsigned bytes) {
unsigned char* ptr = store_ptr += bytes;
chk();
while (bytes) {
*--ptr = value & 0xff;
value >>= 8;
--bytes;
}
DCHECK(!bytes);
}
void store_bytes(unsigned char const* data, size_t s) {
store_ptr += s;
chk();
memcpy(store_ptr - s, data, s);
}
unsigned get_crc32() const {
return td::crc32c(td::Slice{store_start, store_ptr});
}
private:
unsigned char* store_start;
unsigned char* store_ptr;
unsigned char* store_end;
};
struct FileWriter {
FileWriter(td::FileFd& fd, size_t expected_size)
: fd(fd), expected_size(expected_size) {}
~FileWriter() {
flush();
}
size_t position() const {
return flushed_size + writer.position();
}
size_t remaining() const {
return expected_size - position();
}
void chk() const {
DCHECK(position() <= expected_size);
}
bool empty() const {
return remaining() == 0;
}
void store_uint(unsigned long long value, unsigned bytes) {
flush_if_needed(bytes);
writer.store_uint(value, bytes);
}
void store_bytes(unsigned char const* data, size_t s) {
flush_if_needed(s);
writer.store_bytes(data, s);
}
unsigned get_crc32() const {
unsigned char const* start = buf.data();
unsigned char const* end = start + writer.position();
return td::crc32c_extend(current_crc32, td::Slice(start, end));
}
td::Status finalize() {
flush();
return std::move(res);
}
private:
void flush_if_needed(size_t s) {
DCHECK(s <= BUF_SIZE);
if (s > BUF_SIZE - writer.position()) {
flush();
}
}
void flush() {
chk();
unsigned char* start = buf.data();
unsigned char* end = start + writer.position();
if (start == end) {
return;
}
flushed_size += end - start;
current_crc32 = td::crc32c_extend(current_crc32, td::Slice(start, end));
if (res.is_ok()) {
while (end > start) {
auto R = fd.write(td::Slice(start, end));
if (R.is_error()) {
res = R.move_as_error();
break;
}
size_t s = R.move_as_ok();
start += s;
}
}
writer = BufferWriter(buf.data(), buf.data() + buf.size());
}
td::FileFd& fd;
size_t expected_size;
size_t flushed_size = 0;
unsigned current_crc32 = td::crc32c(td::Slice());
static const size_t BUF_SIZE = 1 << 22;
std::vector<unsigned char> buf = std::vector<unsigned char>(BUF_SIZE, '\0');
BufferWriter writer = BufferWriter(buf.data(), buf.data() + buf.size());
td::Status res = td::Status::OK();
};
}
}

136
crypto/vm/boc.cpp
View file

@ -20,6 +20,7 @@
#include <iomanip>
#include <algorithm>
#include "vm/boc.h"
#include "vm/boc-writers.h"
#include "vm/cells.h"
#include "vm/cellslice.h"
#include "td/utils/bits.h"
@ -180,6 +181,7 @@ int BagOfCells::add_root(td::Ref<vm::Cell> add_root) {
return 1;
}
// Changes in this function may require corresponding changes in crypto/vm/large-boc-serializer.cpp
td::Status BagOfCells::import_cells() {
cells_clear();
for (auto& root : roots) {
@ -197,6 +199,7 @@ td::Status BagOfCells::import_cells() {
return td::Status::OK();
}
// Changes in this function may require corresponding changes in crypto/vm/large-boc-serializer.cpp
td::Result<int> BagOfCells::import_cell(td::Ref<vm::Cell> cell, int depth) {
if (depth > max_depth) {
return td::Status::Error("error while importing a cell into a bag of cells: cell depth too large");
@ -246,6 +249,7 @@ td::Result<int> BagOfCells::import_cell(td::Ref<vm::Cell> cell, int depth) {
return cell_count++;
}
// Changes in this function may require corresponding changes in crypto/vm/large-boc-serializer.cpp
void BagOfCells::reorder_cells() {
int_hashes = 0;
for (int i = cell_count - 1; i >= 0; --i) {
@ -323,6 +327,7 @@ void BagOfCells::reorder_cells() {
// force=0 : previsit (recursively until special cells are found; then visit them)
// force=1 : visit (allocate and process all children)
// force=2 : allocate (assign a new index; can be run only after visiting)
// Changes in this function may require corresponding changes in crypto/vm/large-boc-serializer.cpp
int BagOfCells::revisit(int cell_idx, int force) {
DCHECK(cell_idx >= 0 && cell_idx < cell_count);
CellInfo& dci = cell_list_[cell_idx];
@ -369,6 +374,7 @@ int BagOfCells::revisit(int cell_idx, int force) {
return dci.new_idx = -3; // mark as visited (and all children processed)
}
// Changes in this function may require corresponding changes in crypto/vm/large-boc-serializer.cpp
td::uint64 BagOfCells::compute_sizes(int mode, int& r_size, int& o_size) {
int rs = 0, os = 0;
if (!root_count || !data_bytes) {
@ -395,6 +401,7 @@ td::uint64 BagOfCells::compute_sizes(int mode, int& r_size, int& o_size) {
return data_bytes_adj;
}
// Changes in this function may require corresponding changes in crypto/vm/large-boc-serializer.cpp
std::size_t BagOfCells::estimate_serialized_size(int mode) {
if ((mode & Mode::WithCacheBits) && !(mode & Mode::WithIndex)) {
info.invalidate();
@ -475,130 +482,6 @@ std::string BagOfCells::extract_string() const {
return std::string{serialized.data(), serialized.data() + serialized.size()};
}
namespace {
struct BufferWriter {
BufferWriter(unsigned char* store_start, unsigned char* store_end)
: store_start(store_start), store_ptr(store_start), store_end(store_end) {}
size_t position() const {
return store_ptr - store_start;
}
size_t remaining() const {
return store_end - store_ptr;
}
void chk() const {
DCHECK(store_ptr <= store_end);
}
bool empty() const {
return store_ptr == store_end;
}
void store_uint(unsigned long long value, unsigned bytes) {
unsigned char* ptr = store_ptr += bytes;
chk();
while (bytes) {
*--ptr = value & 0xff;
value >>= 8;
--bytes;
}
DCHECK(!bytes);
}
void store_bytes(unsigned char const* data, size_t s) {
store_ptr += s;
chk();
memcpy(store_ptr - s, data, s);
}
unsigned get_crc32() const {
return td::crc32c(td::Slice{store_start, store_ptr});
}
private:
unsigned char* store_start;
unsigned char* store_ptr;
unsigned char* store_end;
};
struct FileWriter {
FileWriter(td::FileFd& fd, size_t expected_size)
: fd(fd), expected_size(expected_size) {}
~FileWriter() {
flush();
}
size_t position() const {
return flushed_size + writer.position();
}
size_t remaining() const {
return expected_size - position();
}
void chk() const {
DCHECK(position() <= expected_size);
}
bool empty() const {
return remaining() == 0;
}
void store_uint(unsigned long long value, unsigned bytes) {
flush_if_needed(bytes);
writer.store_uint(value, bytes);
}
void store_bytes(unsigned char const* data, size_t s) {
flush_if_needed(s);
writer.store_bytes(data, s);
}
unsigned get_crc32() const {
unsigned char const* start = buf.data();
unsigned char const* end = start + writer.position();
return td::crc32c_extend(current_crc32, td::Slice(start, end));
}
td::Status finalize() {
flush();
return std::move(res);
}
private:
void flush_if_needed(size_t s) {
DCHECK(s <= BUF_SIZE);
if (s > BUF_SIZE - writer.position()) {
flush();
}
}
void flush() {
chk();
unsigned char* start = buf.data();
unsigned char* end = start + writer.position();
if (start == end) {
return;
}
flushed_size += end - start;
current_crc32 = td::crc32c_extend(current_crc32, td::Slice(start, end));
if (res.is_ok()) {
while (end > start) {
auto R = fd.write(td::Slice(start, end));
if (R.is_error()) {
res = R.move_as_error();
break;
}
size_t s = R.move_as_ok();
start += s;
}
}
writer = BufferWriter(buf.data(), buf.data() + buf.size());
}
td::FileFd& fd;
size_t expected_size;
size_t flushed_size = 0;
unsigned current_crc32 = td::crc32c(td::Slice());
static const size_t BUF_SIZE = 1 << 22;
std::vector<unsigned char> buf = std::vector<unsigned char>(BUF_SIZE, '\0');
BufferWriter writer = BufferWriter(buf.data(), buf.data() + buf.size());
td::Status res = td::Status::OK();
};
}
//serialized_boc#672fb0ac has_idx:(## 1) has_crc32c:(## 1)
// has_cache_bits:(## 1) flags:(## 2) { flags = 0 }
// size:(## 3) { size <= 4 }
@ -610,6 +493,7 @@ struct FileWriter {
// index:(cells * ##(off_bytes * 8))
// cell_data:(tot_cells_size * [ uint8 ])
// = BagOfCells;
// Changes in this function may require corresponding changes in crypto/vm/large-boc-serializer.cpp
template<typename WriterT>
std::size_t BagOfCells::serialize_to_impl(WriterT& writer, int mode) {
auto store_ref = [&](unsigned long long value) {
@ -705,7 +589,7 @@ std::size_t BagOfCells::serialize_to(unsigned char* buffer, std::size_t buff_siz
if (!size_est || size_est > buff_size) {
return 0;
}
BufferWriter writer{buffer, buffer + size_est};
boc_writers::BufferWriter writer{buffer, buffer + size_est};
return serialize_to_impl(writer, mode);
}
@ -714,7 +598,7 @@ td::Status BagOfCells::serialize_to_file(td::FileFd& fd, int mode) {
if (!size_est) {
return td::Status::Error("no cells to serialize to this bag of cells");
}
FileWriter writer{fd, size_est};
boc_writers::FileWriter writer{fd, size_est};
size_t s = serialize_to_impl(writer, mode);
TRY_STATUS(writer.finalize());
if (s != size_est) {

4
crypto/vm/boc.h
View file

@ -18,6 +18,7 @@
*/
#pragma once
#include <set>
#include "vm/db/DynamicBagOfCellsDb.h"
#include "vm/cells.h"
#include "td/utils/Status.h"
#include "td/utils/buffer.h"
@ -314,4 +315,7 @@ td::Result<std::vector<Ref<Cell>>> std_boc_deserialize_multi(td::Slice data,
int max_roots = BagOfCells::default_max_roots);
td::Result<td::BufferSlice> std_boc_serialize_multi(std::vector<Ref<Cell>> root, int mode = 0);
td::Status std_boc_serialize_to_file_large(std::shared_ptr<CellDbReader> reader, Cell::Hash root_hash,
td::FileFd& fd, int mode = 0);
} // namespace vm

7
crypto/vm/db/CellHashTable.h
View file

@ -64,6 +64,13 @@ class CellHashTable {
size_t size() const {
return set_.size();
}
InfoT* get_if_exists(td::Slice hash) {
auto it = set_.find(hash);
if (it != set_.end()) {
return &const_cast<InfoT &>(*it);
}
return nullptr;
}
private:
std::set<InfoT, std::less<>> set_;

92
crypto/vm/db/DynamicBagOfCellsDb.cpp
View file

@ -31,12 +31,6 @@
namespace vm {
namespace {
class CellDbReader {
public:
virtual ~CellDbReader() = default;
virtual td::Result<Ref<DataCell>> load_cell(td::Slice hash) = 0;
};
struct DynamicBocExtCellExtra {
std::shared_ptr<CellDbReader> reader;
};
@ -92,6 +86,40 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat
TRY_RESULT(loaded_cell, get_cell_info_force(hash).cell->load_cell());
return std::move(loaded_cell.data_cell);
}
void load_cell_async(td::Slice hash, std::shared_ptr<AsyncExecutor> executor,
td::Promise<Ref<DataCell>> promise) override {
auto info = hash_table_.get_if_exists(hash);
if (info && info->sync_with_db) {
TRY_RESULT_PROMISE(promise, loaded_cell, info->cell->load_cell());
promise.set_result(loaded_cell.data_cell);
return;
}
SimpleExtCellCreator ext_cell_creator(cell_db_reader_);
auto promise_ptr = std::make_shared<td::Promise<Ref<DataCell>>>(std::move(promise));
executor->execute_async(
[executor, loader = *loader_, hash = CellHash::from_slice(hash), db = this,
ext_cell_creator = std::move(ext_cell_creator), promise = std::move(promise_ptr)]() mutable {
TRY_RESULT_PROMISE((*promise), res, loader.load(hash.as_slice(), true, ext_cell_creator));
if (res.status != CellLoader::LoadResult::Ok) {
promise->set_error(td::Status::Error("cell not found"));
return;
}
Ref<Cell> cell = res.cell();
executor->execute_sync([hash, db, res = std::move(res),
ext_cell_creator = std::move(ext_cell_creator)]() mutable {
db->hash_table_.apply(hash.as_slice(), [&](CellInfo &info) {
db->update_cell_info_loaded(info, hash.as_slice(), std::move(res));
});
for (auto &ext_cell : ext_cell_creator.get_created_cells()) {
auto ext_cell_hash = ext_cell->get_hash();
db->hash_table_.apply(ext_cell_hash.as_slice(), [&](CellInfo &info) {
db->update_cell_info_created_ext(info, std::move(ext_cell));
});
}
});
promise->set_result(std::move(cell));
});
}
CellInfo &get_cell_info_force(td::Slice hash) {
return hash_table_.apply(hash, [&](CellInfo &info) { update_cell_info_force(info, hash); });
}
@ -176,6 +204,10 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat
return td::Status::OK();
}
std::shared_ptr<CellDbReader> get_cell_db_reader() override {
return cell_db_reader_;
}
td::Status set_loader(std::unique_ptr<CellLoader> loader) override {
reset_cell_db_reader();
loader_ = std::move(loader);
@ -200,6 +232,27 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat
return res;
}
class SimpleExtCellCreator : public ExtCellCreator {
public:
explicit SimpleExtCellCreator(std::shared_ptr<CellDbReader> cell_db_reader) :
cell_db_reader_(std::move(cell_db_reader)) {}
td::Result<Ref<Cell>> ext_cell(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) override {
TRY_RESULT(ext_cell, DynamicBocExtCell::create(PrunnedCellInfo{level_mask, hash, depth},
DynamicBocExtCellExtra{cell_db_reader_}));
created_cells_.push_back(ext_cell);
return std::move(ext_cell);
}
std::vector<Ref<Cell>>& get_created_cells() {
return created_cells_;
}
private:
std::vector<Ref<Cell>> created_cells_;
std::shared_ptr<CellDbReader> cell_db_reader_;
};
class CellDbReaderImpl : public CellDbReader,
private ExtCellCreator,
public std::enable_shared_from_this<CellDbReaderImpl> {
@ -486,6 +539,33 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat
info.sync_with_db = true;
}
// same as update_cell_info_force, but with cell provided by a caller
void update_cell_info_loaded(CellInfo &info, td::Slice hash, CellLoader::LoadResult res) {
if (info.sync_with_db) {
return;
}
DCHECK(res.status == CellLoader::LoadResult::Ok);
info.cell = std::move(res.cell());
CHECK(info.cell->get_hash().as_slice() == hash);
info.in_db = true;
info.db_refcnt = res.refcnt();
info.sync_with_db = true;
}
// same as update_cell_info_lazy, but with cell provided by a caller
void update_cell_info_created_ext(CellInfo &info, Ref<Cell> cell) {
if (info.sync_with_db) {
CHECK(info.cell.not_null());
CHECK(info.cell->get_level_mask() == cell->get_level_mask());
CHECK(info.cell->get_hash() == cell->get_hash());
return;
}
if (info.cell.is_null()) {
info.cell = std::move(cell);
info.in_db = true;
}
}
td::Result<Ref<Cell>> create_empty_ext_cell(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) {
TRY_RESULT(res, DynamicBocExtCell::create(PrunnedCellInfo{level_mask, hash, depth},
DynamicBocExtCellExtra{cell_db_reader_}));

18
crypto/vm/db/DynamicBagOfCellsDb.h
View file

@ -21,6 +21,7 @@
#include "td/utils/Slice.h"
#include "td/utils/Status.h"
#include "td/actor/PromiseFuture.h"
namespace vm {
class CellLoader;
@ -34,6 +35,12 @@ class ExtCellCreator {
virtual td::Result<Ref<Cell>> ext_cell(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) = 0;
};
class CellDbReader {
public:
virtual ~CellDbReader() = default;
virtual td::Result<Ref<DataCell>> load_cell(td::Slice hash) = 0;
};
class DynamicBagOfCellsDb {
public:
virtual ~DynamicBagOfCellsDb() = default;
@ -52,11 +59,22 @@ class DynamicBagOfCellsDb {
virtual td::Status prepare_commit() = 0;
virtual Stats get_stats_diff() = 0;
virtual td::Status commit(CellStorer &) = 0;
virtual std::shared_ptr<CellDbReader> get_cell_db_reader() = 0;
// restart with new loader will also reset stats_diff
virtual td::Status set_loader(std::unique_ptr<CellLoader> loader) = 0;
static std::unique_ptr<DynamicBagOfCellsDb> create();
class AsyncExecutor {
public:
virtual ~AsyncExecutor() {}
virtual void execute_async(std::function<void()> f) = 0;
virtual void execute_sync(std::function<void()> f) = 0;
};
virtual void load_cell_async(td::Slice hash, std::shared_ptr<AsyncExecutor> executor,
td::Promise<Ref<DataCell>> promise) = 0;
};
} // namespace vm

411
crypto/vm/large-boc-serializer.cpp Normal file
View file

@ -0,0 +1,411 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
*/
#include "vm/boc.h"
#include "vm/boc-writers.h"
#include "vm/cellslice.h"
#include "td/utils/misc.h"
namespace vm {
namespace {
// LargeBocSerializer implements serialization of the bag of cells in the standard way
// (equivalent to the implementation in crypto/vm/boc.cpp)
// Changes in this file may require corresponding changes in boc.cpp
class LargeBocSerializer {
public:
using Hash = Cell::Hash;
explicit LargeBocSerializer(std::shared_ptr<CellDbReader> reader) : reader(std::move(reader)) {}
void add_root(Hash root);
td::Status import_cells();
td::Status serialize(td::FileFd& fd, int mode);
private:
std::shared_ptr<CellDbReader> reader;
struct CellInfo {
std::array<int, 4> ref_idx;
int idx;
unsigned short serialized_size;
unsigned char wt;
unsigned char hcnt : 6;
bool should_cache : 1;
bool is_root_cell : 1;
CellInfo(int idx, const std::array<int, 4>& ref_list) : ref_idx(ref_list), idx(idx) {
hcnt = 0;
should_cache = is_root_cell = 0;
}
bool is_special() const {
return !wt;
}
unsigned get_ref_num() const {
for (unsigned i = 0; i < 4; ++i) {
if (ref_idx[i] == -1) {
return i;
}
}
return 4;
}
};
std::map<Hash, CellInfo> cells;
std::vector<std::pair<const Hash, CellInfo>*> cell_list;
struct RootInfo {
RootInfo(Hash hash, int idx) : hash(hash), idx(idx) {}
Hash hash;
int idx;
};
std::vector<RootInfo> roots;
int cell_count = 0, int_refs = 0, int_hashes = 0, top_hashes = 0;
int rv_idx = 0;
unsigned long long data_bytes = 0;
td::Result<int> import_cell(Hash hash, int depth = 0);
void reorder_cells();
int revisit(int cell_idx, int force = 0);
td::uint64 compute_sizes(int mode, int& r_size, int& o_size);
};
void LargeBocSerializer::add_root(Hash root) {
roots.emplace_back(root, -1);
}
td::Status LargeBocSerializer::import_cells() {
for (auto& root : roots) {
TRY_RESULT(idx, import_cell(root.hash));
root.idx = idx;
}
reorder_cells();
CHECK(!cell_list.empty());
return td::Status::OK();
}
td::Result<int> LargeBocSerializer::import_cell(Hash hash, int depth) {
if (depth > Cell::max_depth) {
return td::Status::Error("error while importing a cell into a bag of cells: cell depth too large");
}
auto it = cells.find(hash);
if (it != cells.end()) {
it->second.should_cache = true;
return it->second.idx;
}
TRY_RESULT(cell, reader->load_cell(hash.as_slice()));
if (cell->get_virtualization() != 0) {
return td::Status::Error(
"error while importing a cell into a bag of cells: cell has non-zero virtualization level");
}
CellSlice cs(std::move(cell));
std::array<int, 4> refs;
std::fill(refs.begin(), refs.end(), -1);
DCHECK(cs.size_refs() <= 4);
unsigned sum_child_wt = 1;
for (unsigned i = 0; i < cs.size_refs(); i++) {
TRY_RESULT(ref, import_cell(cs.prefetch_ref(i)->get_hash(), depth + 1));
refs[i] = ref;
sum_child_wt += cell_list[ref]->second.wt;
++int_refs;
}
auto dc = cs.move_as_loaded_cell().data_cell;
auto res = cells.emplace(hash, CellInfo(cell_count, refs));
DCHECK(res.second);
cell_list.push_back(&*res.first);
CellInfo& dc_info = res.first->second;
dc_info.wt = (unsigned char)std::min(0xffU, sum_child_wt);
unsigned hcnt = dc->get_level_mask().get_hashes_count();
DCHECK(hcnt <= 4);
dc_info.hcnt = (unsigned char)hcnt;
TRY_RESULT(serialized_size, td::narrow_cast_safe<unsigned short>(dc->get_serialized_size()));
data_bytes += dc_info.serialized_size = serialized_size;
return cell_count++;
}
void LargeBocSerializer::reorder_cells() {
for (auto ptr : cell_list) {
ptr->second.idx = -1;
}
int_hashes = 0;
for (int i = cell_count - 1; i >= 0; --i) {
CellInfo& dci = cell_list[i]->second;
int s = dci.get_ref_num(), c = s, sum = BagOfCells::max_cell_whs - 1, mask = 0;
for (int j = 0; j < s; ++j) {
CellInfo& dcj = cell_list[dci.ref_idx[j]]->second;
int limit = (BagOfCells::max_cell_whs - 1 + j) / s;
if (dcj.wt <= limit) {
sum -= dcj.wt;
--c;
mask |= (1 << j);
}
}
if (c) {
for (int j = 0; j < s; ++j) {
if (!(mask & (1 << j))) {
CellInfo& dcj = cell_list[dci.ref_idx[j]]->second;
int limit = sum++ / c;
if (dcj.wt > limit) {
dcj.wt = (unsigned char)limit;
}
}
}
}
}
for (int i = 0; i < cell_count; i++) {
CellInfo& dci = cell_list[i]->second;
int s = dci.get_ref_num(), sum = 1;
for (int j = 0; j < s; ++j) {
sum += cell_list[dci.ref_idx[j]]->second.wt;
}
DCHECK(sum <= BagOfCells::max_cell_whs);
if (sum <= dci.wt) {
dci.wt = (unsigned char)sum;
} else {
dci.wt = 0;
int_hashes += dci.hcnt;
}
}
top_hashes = 0;
for (auto& root_info : roots) {
auto& cell_info = cell_list[root_info.idx]->second;
if (cell_info.is_root_cell) {
cell_info.is_root_cell = true;
if (cell_info.wt) {
top_hashes += cell_info.hcnt;
}
}
}
if (cell_count > 0) {
rv_idx = 0;
for (const auto& root_info : roots) {
revisit(root_info.idx, 0);
revisit(root_info.idx, 1);
}
for (const auto& root_info : roots) {
revisit(root_info.idx, 2);
}
for (auto& root_info : roots) {
root_info.idx = cell_list[root_info.idx]->second.idx;
}
DCHECK(rv_idx == cell_count);
for (int i = 0; i < cell_count; ++i) {
while (cell_list[i]->second.idx != i) {
std::swap(cell_list[i], cell_list[cell_list[i]->second.idx]);
}
}
}
}
int LargeBocSerializer::revisit(int cell_idx, int force) {
DCHECK(cell_idx >= 0 && cell_idx < cell_count);
CellInfo& dci = cell_list[cell_idx]->second;
if (dci.idx >= 0) {
return dci.idx;
}
if (!force) {
// previsit
if (dci.idx != -1) {
// already previsited or visited
return dci.idx;
}
int n = dci.get_ref_num();
for (int j = n - 1; j >= 0; --j) {
int child_idx = dci.ref_idx[j];
// either previsit or visit child, depending on whether it is special
revisit(dci.ref_idx[j], cell_list[child_idx]->second.is_special());
}
return dci.idx = -2; // mark as previsited
}
if (force > 1) {
// time to allocate
auto i = dci.idx = rv_idx++;
return i;
}
if (dci.idx == -3) {
// already visited
return dci.idx;
}
if (dci.is_special()) {
// if current cell is special, previsit it first
revisit(cell_idx, 0);
}
// visit children
int n = dci.get_ref_num();
for (int j = n - 1; j >= 0; --j) {
revisit(dci.ref_idx[j], 1);
}
// allocate children
for (int j = n - 1; j >= 0; --j) {
dci.ref_idx[j] = revisit(dci.ref_idx[j], 2);
}
return dci.idx = -3; // mark as visited (and all children processed)
}
td::uint64 LargeBocSerializer::compute_sizes(int mode, int& r_size, int& o_size) {
using Mode = BagOfCells::Mode;
int rs = 0, os = 0;
if (roots.empty() || !data_bytes) {
r_size = o_size = 0;
return 0;
}
while (cell_count >= (1LL << (rs << 3))) {
rs++;
}
td::uint64 hashes =
(((mode & Mode::WithTopHash) ? top_hashes : 0) + ((mode & Mode::WithIntHashes) ? int_hashes : 0)) *
(Cell::hash_bytes + Cell::depth_bytes);
td::uint64 data_bytes_adj = data_bytes + (unsigned long long)int_refs * rs + hashes;
td::uint64 max_offset = (mode & Mode::WithCacheBits) ? data_bytes_adj * 2 : data_bytes_adj;
while (max_offset >= (1ULL << (os << 3))) {
os++;
}
if (rs > 4 || os > 8) {
r_size = o_size = 0;
return 0;
}
r_size = rs;
o_size = os;
return data_bytes_adj;
}
td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) {
using Mode = BagOfCells::Mode;
BagOfCells::Info info;
if ((mode & Mode::WithCacheBits) && !(mode & Mode::WithIndex)) {
return td::Status::Error("invalid flags");
}
auto data_bytes_adj = compute_sizes(mode, info.ref_byte_size, info.offset_byte_size);
if (data_bytes_adj == 0) {
return td::Status::Error("no cells to serialize");
}
info.valid = true;
info.has_crc32c = mode & Mode::WithCRC32C;
info.has_index = mode & Mode::WithIndex;
info.has_cache_bits = mode & Mode::WithCacheBits;
info.root_count = (int)roots.size();
info.cell_count = cell_count;
info.absent_count = 0;
int crc_size = info.has_crc32c ? 4 : 0;
info.roots_offset = 4 + 1 + 1 + 3 * info.ref_byte_size + info.offset_byte_size;
info.index_offset = info.roots_offset + info.root_count * info.ref_byte_size;
info.data_offset = info.index_offset;
if (info.has_index) {
info.data_offset += (long long)cell_count * info.offset_byte_size;
}
info.magic = BagOfCells::Info::boc_generic;
info.data_size = data_bytes_adj;
info.total_size = info.data_offset + data_bytes_adj + crc_size;
auto res = td::narrow_cast_safe<size_t>(info.total_size);
if (res.is_error()) {
return td::Status::Error("bag of cells is too large");
}
boc_writers::FileWriter writer{fd, info.total_size};
auto store_ref = [&](unsigned long long value) {
writer.store_uint(value, info.ref_byte_size);
};
auto store_offset = [&](unsigned long long value) {
writer.store_uint(value, info.offset_byte_size);
};
writer.store_uint(info.magic, 4);
td::uint8 byte{0};
if (info.has_index) {
byte |= 1 << 7;
}
if (info.has_crc32c) {
byte |= 1 << 6;
}
if (info.has_cache_bits) {
byte |= 1 << 5;
}
byte |= (td::uint8)info.ref_byte_size;
writer.store_uint(byte, 1);
writer.store_uint(info.offset_byte_size, 1);
store_ref(cell_count);
store_ref(roots.size());
store_ref(0);
store_offset(info.data_size);
for (const auto& root_info : roots) {
int k = cell_count - 1 - root_info.idx;
DCHECK(k >= 0 && k < cell_count);
store_ref(k);
}
DCHECK(writer.position() == info.index_offset);
DCHECK((unsigned)cell_count == cell_list.size());
if (info.has_index) {
std::size_t offs = 0;
for (int i = cell_count - 1; i >= 0; --i) {
const auto& dc_info = cell_list[i]->second;
bool with_hash = (mode & Mode::WithIntHashes) && !dc_info.wt;
if (dc_info.is_root_cell && (mode & Mode::WithTopHash)) {
with_hash = true;
}
int hash_size = 0;
if (with_hash) {
hash_size = (Cell::hash_bytes + Cell::depth_bytes) * dc_info.hcnt;
}
offs += dc_info.serialized_size + hash_size + dc_info.get_ref_num() * info.ref_byte_size;
auto fixed_offset = offs;
if (info.has_cache_bits) {
fixed_offset = offs * 2 + dc_info.should_cache;
}
store_offset(fixed_offset);
}
DCHECK(offs == info.data_size);
}
DCHECK(writer.position() == info.data_offset);
size_t keep_position = writer.position();
for (int i = 0; i < cell_count; ++i) {
auto hash = cell_list[cell_count - 1 - i]->first;
const auto& dc_info = cell_list[cell_count - 1 - i]->second;
TRY_RESULT(dc, reader->load_cell(hash.as_slice()));
bool with_hash = (mode & Mode::WithIntHashes) && !dc_info.wt;
if (dc_info.is_root_cell && (mode & Mode::WithTopHash)) {
with_hash = true;
}
unsigned char buf[256];
int s = dc->serialize(buf, 256, with_hash);
writer.store_bytes(buf, s);
DCHECK(dc->size_refs() == dc_info.get_ref_num());
unsigned ref_num = dc_info.get_ref_num();
for (unsigned j = 0; j < ref_num; ++j) {
int k = cell_count - 1 - dc_info.ref_idx[j];
DCHECK(k > i && k < cell_count);
store_ref(k);
}
}
DCHECK(writer.position() - keep_position == info.data_size);
if (info.has_crc32c) {
unsigned crc = writer.get_crc32();
writer.store_uint(td::bswap32(crc), 4);
}
DCHECK(writer.empty());
return writer.finalize();
}
}
td::Status std_boc_serialize_to_file_large(std::shared_ptr<CellDbReader> reader, Cell::Hash root_hash,
td::FileFd& fd, int mode) {
CHECK(reader != nullptr)
LargeBocSerializer serializer(reader);
serializer.add_root(root_hash);
TRY_STATUS(serializer.import_cells());
return serializer.serialize(fd, mode);
}
}

22
doc/catchain-dos.md Normal file
View file

@ -0,0 +1,22 @@
# Catchain DoS protection
## Consensus-safe upgrade
Since catchain protocol describes the way validators come to consensus, it should be updated in simultaneous manner to ensure that at any moment of time at least 2/3 of validators (measured by weight) are using the same version of the protocol. Innate way to do it in TON is to use Configuration parameters, when absence or old version of some configuration parameter corresponds to the old behavior, while new version controls updated catchain protocol behavior.
That way, update of catchain protocol should be done in the following way: each validator upgrades its node software and votes for a new config parameter. This method ensures that switching to updated catchain protocol will happen only after 3/4 (more than 2/3) of validators upgrade.
## Covering block dependencies by hash
Catchain protocol version is controlled by `ConfigParam 29`: `proto_version` field in constructors `consensus_config_v3` and older (for other constructors `proto_version` is equal to `0`).
For catchain protocol version higher or equal to 2 hash covers catchain block dependencies, that prevents relaying of blocks with altered dependencies.
## Limiting maximal catchain block height and size
### Catchain block size
All catchain messages, except `REJECT` message, have (and had) a limited size. After update `REJECT` message will be limited to 1024 bytes. At the same time one block contains at most number of block candidates per round messages. That way, 16kb limit per catchain block should be enough to prevent DoS.
### Limiting block height
Another potential DoS is related to a situation when a malbehaviouring node sends too many catchain blocks. Note that limiting of maximal number of blocks per second is not a good solution since it will hinder synchronization after node disconnect.
At the same time, catchain groups exist for quite short period of time (around a few hunderd seconds), while number of blocks production is determined by "natural block production speed" on the one hand and number of blocks generated to decrease dependencies size on the other. In any case, total number of blocks is limited by `catchain_lifetime * natural_block_production_speed * (1 + number_of_catchain_participants / max_dependencies_size)`.
To prevent DoS attack we limit maximal height of the block which will be processed by node by `catchain_lifetime * natural_block_production_speed * (1 + number_of_catchain_participants / max_dependencies_size)`, where `catchain_lifetime` is set by `ConfigParam 28` (`CatchainConfig`), `natural_block_production_speed` and `max_dependencies_size` are set by `ConfigParam 29` (`ConsensusConfig`) (`natural_block_production_speed` is calculated as `catchain_max_blocks_coeff / 1000`) and `number_of_catchain_participants` is set from catchain group configuration.
By default, `catchain_max_blocks_coeff` is set to zero: special value which means that there is no limitation on catchain block height. It is recommended to set `catchain_max_blocks_coeff` to `10000`: we expect that natural production rate is about one block per 3 seconds, so we set the coefficient to allow 30 times higher block production than expected. At the same time, this number is low enough to prevent resource-intensive attacks.
To prevent DoS with many blocks of the same height, nodes will ignore any blocks from a "bad" node (i.e. one which created a fork) unless the new block is required by some other block from a "good" node.

7
overlay/overlay-broadcast.cpp
View file

@ -80,6 +80,7 @@ td::Status BroadcastSimple::distribute() {
void BroadcastSimple::broadcast_checked(td::Result<td::Unit> R) {
if (R.is_error()) {
td::actor::send_closure(actor_id(overlay_), &OverlayImpl::update_peer_err_ctr, src_peer_id_, false);
return;
}
is_valid_ = true;
@ -114,7 +115,7 @@ td::Status BroadcastSimple::run() {
return run_continue();
}
td::Status BroadcastSimple::create(OverlayImpl *overlay, tl_object_ptr<ton_api::overlay_broadcast> broadcast) {
td::Status BroadcastSimple::create(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, tl_object_ptr<ton_api::overlay_broadcast> broadcast) {
auto src = PublicKey{broadcast->src_};
auto data_hash = sha256_bits256(broadcast->data_.as_slice());
auto broadcast_hash = compute_broadcast_id(src, data_hash, broadcast->flags_);
@ -125,7 +126,7 @@ td::Status BroadcastSimple::create(OverlayImpl *overlay, tl_object_ptr<ton_api::
auto B = std::make_unique<BroadcastSimple>(broadcast_hash, src, std::move(cert), broadcast->flags_,
std::move(broadcast->data_), broadcast->date_,
std::move(broadcast->signature_), false, overlay);
std::move(broadcast->signature_), false, overlay, src_peer_id);
TRY_STATUS(B->run());
overlay->register_simple_broadcast(std::move(B));
return td::Status::OK();
@ -139,7 +140,7 @@ td::Status BroadcastSimple::create_new(td::actor::ActorId<OverlayImpl> overlay,
auto date = static_cast<td::uint32>(td::Clocks::system());
auto B = std::make_unique<BroadcastSimple>(broadcast_hash, PublicKey{}, nullptr, flags, std::move(data), date,
td::BufferSlice{}, false, nullptr);
td::BufferSlice{}, false, nullptr, adnl::AdnlNodeIdShort::zero());
auto to_sign = B->to_sign();
auto P = td::PromiseCreator::lambda(

9
overlay/overlay-broadcast.hpp
View file

@ -49,6 +49,8 @@ class BroadcastSimple : public td::ListNode {
OverlayImpl *overlay_;
adnl::AdnlNodeIdShort src_peer_id_ = adnl::AdnlNodeIdShort::zero();
td::Status check_time();
td::Status check_duplicate();
td::Status check_source();
@ -61,7 +63,7 @@ class BroadcastSimple : public td::ListNode {
public:
BroadcastSimple(Overlay::BroadcastHash broadcast_hash, PublicKey source, std::shared_ptr<Certificate> cert,
td::uint32 flags, td::BufferSlice data, td::uint32 date, td::BufferSlice signature, bool is_valid,
OverlayImpl *overlay)
OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id)
: broadcast_hash_(broadcast_hash)
, source_(std::move(source))
, cert_(std::move(cert))
@ -70,7 +72,8 @@ class BroadcastSimple : public td::ListNode {
, date_(date)
, signature_(std::move(signature))
, is_valid_(is_valid)
, overlay_(overlay) {
, overlay_(overlay)
, src_peer_id_(src_peer_id) {
}
Overlay::BroadcastHash get_hash() const {
@ -98,7 +101,7 @@ class BroadcastSimple : public td::ListNode {
void update_overlay(OverlayImpl *overlay);
void broadcast_checked(td::Result<td::Unit> R);
static td::Status create(OverlayImpl *overlay, tl_object_ptr<ton_api::overlay_broadcast> broadcast);
static td::Status create(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, tl_object_ptr<ton_api::overlay_broadcast> broadcast);
static td::Status create_new(td::actor::ActorId<OverlayImpl> overlay, td::actor::ActorId<keyring::Keyring> keyring,
PublicKeyHash local_id, td::BufferSlice data, td::uint32 flags);

14
overlay/overlay-fec-broadcast.cpp
View file

@ -88,6 +88,7 @@ td::Status OverlayFecBroadcastPart::run_checks() {
void BroadcastFec::broadcast_checked(td::Result<td::Unit> R) {
if (R.is_error()) {
td::actor::send_closure(actor_id(overlay_), &OverlayImpl::update_peer_err_ctr, src_peer_id_, true);
return;
}
overlay_->deliver_broadcast(get_source().compute_short_id(), data_.clone());
@ -155,6 +156,7 @@ td::Status OverlayFecBroadcastPart::apply() {
if (!bcast_->finalized()) {
bcast_->set_overlay(overlay_);
bcast_->set_src_peer_id(src_peer_id_);
TRY_STATUS(bcast_->add_part(seqno_, data_.clone(), export_serialized_short(), export_serialized()));
auto R = bcast_->finish();
if (R.is_error()) {
@ -211,7 +213,7 @@ td::BufferSlice OverlayFecBroadcastPart::to_sign() {
return serialize_tl_object(obj, true);
}
td::Status OverlayFecBroadcastPart::create(OverlayImpl *overlay,
td::Status OverlayFecBroadcastPart::create(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id,
tl_object_ptr<ton_api::overlay_broadcastFec> broadcast) {
TRY_STATUS(overlay->check_date(broadcast->date_));
auto source = PublicKey{broadcast->src_};
@ -244,12 +246,13 @@ td::Status OverlayFecBroadcastPart::create(OverlayImpl *overlay,
std::move(broadcast->signature_),
false,
overlay->get_fec_broadcast(broadcast_hash),
overlay};
overlay,
src_peer_id};
TRY_STATUS(B.run());
return td::Status::OK();
}
td::Status OverlayFecBroadcastPart::create(OverlayImpl *overlay,
td::Status OverlayFecBroadcastPart::create(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id,
tl_object_ptr<ton_api::overlay_broadcastFecShort> broadcast) {
auto bcast = overlay->get_fec_broadcast(broadcast->broadcast_hash_);
if (!bcast) {
@ -285,7 +288,8 @@ td::Status OverlayFecBroadcastPart::create(OverlayImpl *overlay,
std::move(broadcast->signature_),
true,
bcast,
overlay};
overlay,
src_peer_id};
TRY_STATUS(B.run());
return td::Status::OK();
}
@ -300,7 +304,7 @@ td::Status OverlayFecBroadcastPart::create_new(OverlayImpl *overlay, td::actor::
auto B = std::make_unique<OverlayFecBroadcastPart>(
broadcast_hash, part_hash, PublicKey{}, overlay->get_certificate(local_id), data_hash, size, flags,
part_data_hash, std::move(part), seqno, std::move(fec_type), date, td::BufferSlice{}, false, nullptr, overlay);
part_data_hash, std::move(part), seqno, std::move(fec_type), date, td::BufferSlice{}, false, nullptr, overlay, adnl::AdnlNodeIdShort::zero());
auto to_sign = B->to_sign();
auto P = td::PromiseCreator::lambda(

14
overlay/overlay-fec-broadcast.hpp
View file

@ -194,6 +194,9 @@ class BroadcastFec : public td::ListNode {
void set_overlay(OverlayImpl *overlay) {
overlay_ = overlay;
}
void set_src_peer_id(adnl::AdnlNodeIdShort src_peer_id) {
src_peer_id_ = src_peer_id;
}
td::Status distribute_part(td::uint32 seqno);
@ -220,6 +223,7 @@ class BroadcastFec : public td::ListNode {
std::map<td::uint32, std::pair<td::BufferSlice, td::BufferSlice>> parts_;
OverlayImpl *overlay_;
adnl::AdnlNodeIdShort src_peer_id_ = adnl::AdnlNodeIdShort::zero();
td::BufferSlice data_;
};
@ -245,6 +249,7 @@ class OverlayFecBroadcastPart : public td::ListNode {
BroadcastFec *bcast_;
OverlayImpl *overlay_;
adnl::AdnlNodeIdShort src_peer_id_ = adnl::AdnlNodeIdShort::zero();
td::Status check_time();
td::Status check_duplicate();
@ -260,7 +265,7 @@ class OverlayFecBroadcastPart : public td::ListNode {
std::shared_ptr<Certificate> cert, Overlay::BroadcastDataHash data_hash, td::uint32 data_size,
td::uint32 flags, Overlay::BroadcastDataHash part_data_hash, td::BufferSlice data,
td::uint32 seqno, fec::FecType fec_type, td::uint32 date, td::BufferSlice signature,
bool is_short, BroadcastFec *bcast, OverlayImpl *overlay)
bool is_short, BroadcastFec *bcast, OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id)
: broadcast_hash_(broadcast_hash)
, part_hash_(part_hash)
, source_(std::move(source))
@ -276,7 +281,8 @@ class OverlayFecBroadcastPart : public td::ListNode {
, signature_(std::move(signature))
, is_short_(is_short)
, bcast_(bcast)
, overlay_(overlay) {
, overlay_(overlay)
, src_peer_id_(src_peer_id) {
}
td::uint32 data_size() const {
@ -310,8 +316,8 @@ class OverlayFecBroadcastPart : public td::ListNode {
return td::Status::OK();
}
static td::Status create(OverlayImpl *overlay, tl_object_ptr<ton_api::overlay_broadcastFec> broadcast);
static td::Status create(OverlayImpl *overlay, tl_object_ptr<ton_api::overlay_broadcastFecShort> broadcast);
static td::Status create(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, tl_object_ptr<ton_api::overlay_broadcastFec> broadcast);
static td::Status create(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, tl_object_ptr<ton_api::overlay_broadcastFecShort> broadcast);
static td::Status create_new(OverlayImpl *overlay, td::actor::ActorId<OverlayImpl> overlay_actor_id,
PublicKeyHash local_id, Overlay::BroadcastDataHash data_hash, td::uint32 size,
td::uint32 flags, td::BufferSlice part, td::uint32 seqno, fec::FecType fec_type,

27
overlay/overlay-manager.cpp
View file

@ -90,12 +90,12 @@ void OverlayManager::delete_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdSho
}
void OverlayManager::create_public_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
std::unique_ptr<Callback> callback, OverlayPrivacyRules rules) {
std::unique_ptr<Callback> callback, OverlayPrivacyRules rules, td::string scope) {
CHECK(!dht_node_.empty());
auto id = overlay_id.compute_short_id();
register_overlay(local_id, id,
Overlay::create(keyring_, adnl_, actor_id(this), dht_node_, local_id, std::move(overlay_id),
std::move(callback), std::move(rules)));
std::move(callback), std::move(rules), scope));
}
void OverlayManager::create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
@ -128,6 +128,7 @@ void OverlayManager::receive_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeId
return;
}
td::actor::send_closure(it2->second, &Overlay::update_throughput_in_ctr, src, (td::uint32)data.size(), false);
td::actor::send_closure(it2->second, &Overlay::receive_message, src, std::move(data));
}
@ -152,12 +153,12 @@ void OverlayManager::receive_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdSh
}
auto it2 = it->second.find(OverlayIdShort{M->overlay_});
if (it2 == it->second.end()) {
VLOG(OVERLAY_NOTICE) << this << ": query to localid not in overlay " << M->overlay_ << "@" << dst << " from "
<< src;
VLOG(OVERLAY_NOTICE) << this << ": query to localid not in overlay " << M->overlay_ << "@" << dst << " from " << src;
promise.set_error(td::Status::Error(ErrorCode::protoviolation, PSTRING() << "bad overlay_id " << M->overlay_));
return;
}
td::actor::send_closure(it2->second, &Overlay::update_throughput_in_ctr, src, (td::uint32)data.size(), true);
td::actor::send_closure(it2->second, &Overlay::receive_query, src, std::move(data), std::move(promise));
}
@ -166,6 +167,15 @@ void OverlayManager::send_query_via(adnl::AdnlNodeIdShort dst, adnl::AdnlNodeIdS
td::BufferSlice query, td::uint64 max_answer_size,
td::actor::ActorId<adnl::AdnlSenderInterface> via) {
CHECK(query.size() <= adnl::Adnl::huge_packet_max_size());
auto it = overlays_.find(src);
if (it != overlays_.end()) {
auto it2 = it->second.find(overlay_id);
if (it2 != it->second.end()) {
td::actor::send_closure(it2->second, &Overlay::update_throughput_out_ctr, dst, (td::uint32)query.size(), true);
}
}
td::actor::send_closure(
via, &adnl::AdnlSenderInterface::send_query_ex, src, dst, std::move(name), std::move(promise), timeout,
create_serialize_tl_object_suffix<ton_api::overlay_query>(query.as_slice(), overlay_id.tl()), max_answer_size);
@ -174,6 +184,15 @@ void OverlayManager::send_query_via(adnl::AdnlNodeIdShort dst, adnl::AdnlNodeIdS
void OverlayManager::send_message_via(adnl::AdnlNodeIdShort dst, adnl::AdnlNodeIdShort src, OverlayIdShort overlay_id,
td::BufferSlice object, td::actor::ActorId<adnl::AdnlSenderInterface> via) {
CHECK(object.size() <= adnl::Adnl::huge_packet_max_size());
auto it = overlays_.find(src);
if (it != overlays_.end()) {
auto it2 = it->second.find(overlay_id);
if (it2 != it->second.end()) {
td::actor::send_closure(it2->second, &Overlay::update_throughput_out_ctr, dst, (td::uint32)object.size(), false);
}
}
td::actor::send_closure(
via, &adnl::AdnlSenderInterface::send_message, src, dst,
create_serialize_tl_object_suffix<ton_api::overlay_message>(object.as_slice(), overlay_id.tl()));

2
overlay/overlay-manager.h
View file

@ -51,7 +51,7 @@ class OverlayManager : public Overlays {
void update_dht_node(td::actor::ActorId<dht::Dht> dht) override;
void create_public_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
std::unique_ptr<Callback> callback, OverlayPrivacyRules rules) override;
std::unique_ptr<Callback> callback, OverlayPrivacyRules rules, td::string scope) override;
void create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
std::vector<adnl::AdnlNodeIdShort> nodes, std::unique_ptr<Callback> callback,
OverlayPrivacyRules rules) override;

77
overlay/overlay.cpp
View file

@ -37,10 +37,10 @@ td::actor::ActorOwn<Overlay> Overlay::create(td::actor::ActorId<keyring::Keyring
td::actor::ActorId<OverlayManager> manager,
td::actor::ActorId<dht::Dht> dht_node, adnl::AdnlNodeIdShort local_id,
OverlayIdFull overlay_id, std::unique_ptr<Overlays::Callback> callback,
OverlayPrivacyRules rules) {
OverlayPrivacyRules rules, td::string scope) {
auto R = td::actor::create_actor<OverlayImpl>("overlay", keyring, adnl, manager, dht_node, local_id,
std::move(overlay_id), true, std::vector<adnl::AdnlNodeIdShort>(),
std::move(callback), std::move(rules));
std::move(callback), std::move(rules), scope);
return td::actor::ActorOwn<Overlay>(std::move(R));
}
@ -60,7 +60,7 @@ OverlayImpl::OverlayImpl(td::actor::ActorId<keyring::Keyring> keyring, td::actor
td::actor::ActorId<OverlayManager> manager, td::actor::ActorId<dht::Dht> dht_node,
adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, bool pub,
std::vector<adnl::AdnlNodeIdShort> nodes, std::unique_ptr<Overlays::Callback> callback,
OverlayPrivacyRules rules)
OverlayPrivacyRules rules, td::string scope)
: keyring_(keyring)
, adnl_(adnl)
, manager_(manager)
@ -69,7 +69,8 @@ OverlayImpl::OverlayImpl(td::actor::ActorId<keyring::Keyring> keyring, td::actor
, id_full_(std::move(overlay_id))
, callback_(std::move(callback))
, public_(pub)
, rules_(std::move(rules)) {
, rules_(std::move(rules))
, scope_(scope) {
overlay_id_ = id_full_.compute_short_id();
VLOG(OVERLAY_INFO) << this << ": creating " << (public_ ? "public" : "private");
@ -161,17 +162,17 @@ void OverlayImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data,
td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from,
tl_object_ptr<ton_api::overlay_broadcast> bcast) {
return BroadcastSimple::create(this, std::move(bcast));
return BroadcastSimple::create(this, message_from, std::move(bcast));
}
td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from,
tl_object_ptr<ton_api::overlay_broadcastFec> b) {
return OverlayFecBroadcastPart::create(this, std::move(b));
return OverlayFecBroadcastPart::create(this, message_from, std::move(b));
}
td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from,
tl_object_ptr<ton_api::overlay_broadcastFecShort> b) {
return OverlayFecBroadcastPart::create(this, std::move(b));
return OverlayFecBroadcastPart::create(this, message_from, std::move(b));
}
td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from,
@ -236,6 +237,36 @@ void OverlayImpl::receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice dat
void OverlayImpl::alarm() {
bcast_gc();
if(update_throughput_at_.is_in_past()) {
double t_elapsed = td::Time::now() - last_throughput_update_.at();
auto SelfId = actor_id(this);
peers_.iterate([&](const adnl::AdnlNodeIdShort &key, OverlayPeer &peer) {
peer.throughput_out_bytes = static_cast<td::uint32>(peer.throughput_out_bytes_ctr / t_elapsed);
peer.throughput_in_bytes = static_cast<td::uint32>(peer.throughput_in_bytes_ctr / t_elapsed);
peer.throughput_out_packets = static_cast<td::uint32>(peer.throughput_out_packets_ctr / t_elapsed);
peer.throughput_in_packets = static_cast<td::uint32>(peer.throughput_in_packets_ctr / t_elapsed);
peer.throughput_out_bytes_ctr = 0;
peer.throughput_in_bytes_ctr = 0;
peer.throughput_out_packets_ctr = 0;
peer.throughput_in_packets_ctr = 0;
auto P = td::PromiseCreator::lambda([SelfId, peer_id = key](td::Result<td::string> result) {
result.ensure();
td::actor::send_closure(SelfId, &Overlay::update_peer_ip_str, peer_id, result.move_as_ok());
});
td::actor::send_closure(adnl_, &adnl::AdnlSenderInterface::get_conn_ip_str, local_id_, key, std::move(P));
});
update_throughput_at_ = td::Timestamp::in(50.0);
last_throughput_update_ = td::Timestamp::now();
}
if (public_) {
if (peers_.size() > 0) {
auto P = get_random_peer();
@ -541,6 +572,17 @@ void OverlayImpl::check_broadcast(PublicKeyHash src, td::BufferSlice data, td::P
callback_->check_broadcast(src, overlay_id_, std::move(data), std::move(promise));
}
void OverlayImpl::update_peer_err_ctr(adnl::AdnlNodeIdShort peer_id, bool is_fec) {
auto src_peer = peers_.get(peer_id);
if(src_peer) {
if(is_fec) {
src_peer->fec_broadcast_errors++;
} else {
src_peer->broadcast_errors++;
}
}
}
void OverlayImpl::broadcast_checked(Overlay::BroadcastHash hash, td::Result<td::Unit> R) {
{
auto it = broadcasts_.find(hash);
@ -561,7 +603,26 @@ void OverlayImpl::get_stats(td::Promise<tl_object_ptr<ton_api::engine_validator_
res->adnl_id_ = local_id_.bits256_value();
res->overlay_id_ = overlay_id_.bits256_value();
res->overlay_id_full_ = id_full_.pubkey().tl();
peers_.iterate([&](const adnl::AdnlNodeIdShort &key, const OverlayPeer &peer) { res->nodes_.push_back(key.tl()); });
res->scope_ = scope_;
peers_.iterate([&](const adnl::AdnlNodeIdShort &key, const OverlayPeer &peer) {
auto node_obj = create_tl_object<ton_api::engine_validator_overlayStatsNode>();
node_obj->adnl_id_ = key.bits256_value();
node_obj->t_out_bytes_ = peer.throughput_out_bytes;
node_obj->t_in_bytes_ = peer.throughput_in_bytes;
node_obj->t_out_pckts_ = peer.throughput_out_packets;
node_obj->t_in_pckts_ = peer.throughput_in_packets;
node_obj->ip_addr_ = peer.ip_addr_str;
node_obj->last_in_query_ = static_cast<td::uint32>(peer.last_in_query_at.at_unix());
node_obj->last_out_query_ = static_cast<td::uint32>(peer.last_out_query_at.at_unix());
node_obj->bdcst_errors_ = peer.broadcast_errors;
node_obj->fec_bdcst_errors_ = peer.fec_broadcast_errors;
res->nodes_.push_back(std::move(node_obj));
});
res->stats_.push_back(
create_tl_object<ton_api::engine_validator_oneStat>("neighbours_cnt", PSTRING() << neighbours_.size()));

5
overlay/overlay.h
View file

@ -42,7 +42,7 @@ class Overlay : public td::actor::Actor {
td::actor::ActorId<OverlayManager> manager,
td::actor::ActorId<dht::Dht> dht_node, adnl::AdnlNodeIdShort local_id,
OverlayIdFull overlay_id, std::unique_ptr<Overlays::Callback> callback,
OverlayPrivacyRules rules);
OverlayPrivacyRules rules, td::string scope);
static td::actor::ActorOwn<Overlay> create(td::actor::ActorId<keyring::Keyring> keyring,
td::actor::ActorId<adnl::Adnl> adnl,
td::actor::ActorId<OverlayManager> manager,
@ -64,6 +64,9 @@ class Overlay : public td::actor::Actor {
virtual void set_privacy_rules(OverlayPrivacyRules rules) = 0;
virtual void receive_nodes_from_db(tl_object_ptr<ton_api::overlay_nodes> nodes) = 0;
virtual void get_stats(td::Promise<tl_object_ptr<ton_api::engine_validator_overlayStats>> promise) = 0;
virtual void update_throughput_out_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) = 0;
virtual void update_throughput_in_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) = 0;
virtual void update_peer_ip_str(adnl::AdnlNodeIdShort peer_id, td::string ip_str) = 0;
//virtual void receive_broadcast(td::BufferSlice data) = 0;
//virtual void subscribe(std::unique_ptr<Overlays::Callback> callback) = 0;
};

63
overlay/overlay.hpp
View file

@ -79,6 +79,26 @@ class OverlayPeer {
td::int32 get_version() const {
return node_.version();
}
td::uint32 throughput_out_bytes = 0;
td::uint32 throughput_in_bytes = 0;
td::uint32 throughput_out_packets = 0;
td::uint32 throughput_in_packets = 0;
td::uint32 throughput_out_bytes_ctr = 0;
td::uint32 throughput_in_bytes_ctr = 0;
td::uint32 throughput_out_packets_ctr = 0;
td::uint32 throughput_in_packets_ctr = 0;
td::uint32 broadcast_errors = 0;
td::uint32 fec_broadcast_errors = 0;
td::Timestamp last_in_query_at = td::Timestamp::now();
td::Timestamp last_out_query_at = td::Timestamp::now();
td::string ip_addr_str = "undefined";
private:
OverlayNode node_;
@ -93,7 +113,7 @@ class OverlayImpl : public Overlay {
td::actor::ActorId<OverlayManager> manager, td::actor::ActorId<dht::Dht> dht_node,
adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, bool pub,
std::vector<adnl::AdnlNodeIdShort> nodes, std::unique_ptr<Overlays::Callback> callback,
OverlayPrivacyRules rules);
OverlayPrivacyRules rules, td::string scope = "{ \"type\": \"undefined\" }");
void update_dht_node(td::actor::ActorId<dht::Dht> dht) override {
dht_node_ = dht;
}
@ -109,6 +129,9 @@ class OverlayImpl : public Overlay {
void alarm() override;
void start_up() override {
update_throughput_at_ = td::Timestamp::in(50.0);
last_throughput_update_ = td::Timestamp::now();
if (public_) {
update_db_at_ = td::Timestamp::in(60.0);
}
@ -150,6 +173,8 @@ class OverlayImpl : public Overlay {
void broadcast_checked(Overlay::BroadcastHash hash, td::Result<td::Unit> R);
void check_broadcast(PublicKeyHash src, td::BufferSlice data, td::Promise<td::Unit> promise);
void update_peer_err_ctr(adnl::AdnlNodeIdShort peer_id, bool is_fec);
BroadcastFec *get_fec_broadcast(BroadcastHash hash);
void register_fec_broadcast(std::unique_ptr<BroadcastFec> bcast);
void register_simple_broadcast(std::unique_ptr<BroadcastSimple> bcast);
@ -191,6 +216,39 @@ class OverlayImpl : public Overlay {
td::Result<Encryptor *> get_encryptor(PublicKey source);
void get_stats(td::Promise<tl_object_ptr<ton_api::engine_validator_overlayStats>> promise) override;
void update_throughput_out_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) override {
auto out_peer = peers_.get(peer_id);
if(out_peer) {
out_peer->throughput_out_bytes_ctr += msg_size;
out_peer->throughput_out_packets_ctr++;
if(is_query)
{
out_peer->last_out_query_at = td::Timestamp::now();
}
}
}
void update_throughput_in_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) override {
auto in_peer = peers_.get(peer_id);
if(in_peer) {
in_peer->throughput_in_bytes_ctr += msg_size;
in_peer->throughput_in_packets_ctr++;
if(is_query)
{
in_peer->last_in_query_at = td::Timestamp::now();
}
}
}
void update_peer_ip_str(adnl::AdnlNodeIdShort peer_id, td::string ip_str) override {
auto fpeer = peers_.get(peer_id);
if(fpeer) {
fpeer->ip_addr_str = ip_str;
}
}
private:
template <class T>
@ -236,6 +294,8 @@ class OverlayImpl : public Overlay {
td::DecTree<adnl::AdnlNodeIdShort, OverlayPeer> peers_;
td::Timestamp next_dht_query_ = td::Timestamp::in(1.0);
td::Timestamp update_db_at_;
td::Timestamp update_throughput_at_;
td::Timestamp last_throughput_update_;
std::unique_ptr<Overlays::Callback> callback_;
@ -291,6 +351,7 @@ class OverlayImpl : public Overlay {
bool public_;
bool semi_public_ = false;
OverlayPrivacyRules rules_;
td::string scope_;
std::map<PublicKeyHash, std::shared_ptr<Certificate>> certs_;
class CachedEncryptor : public td::ListNode {

2
overlay/overlays.h
View file

@ -193,7 +193,7 @@ class Overlays : public td::actor::Actor {
virtual void update_dht_node(td::actor::ActorId<dht::Dht> dht) = 0;
virtual void create_public_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
std::unique_ptr<Callback> callback, OverlayPrivacyRules rules) = 0;
std::unique_ptr<Callback> callback, OverlayPrivacyRules rules, td::string scope) = 0;
virtual void create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
std::vector<adnl::AdnlNodeIdShort> nodes, std::unique_ptr<Callback> callback,
OverlayPrivacyRules rules) = 0;

1
rldp/rldp-in.hpp
View file

@ -99,6 +99,7 @@ class RldpIn : public RldpImpl {
void in_transfer_completed(TransferId transfer_id);
void add_id(adnl::AdnlNodeIdShort local_id) override;
void get_conn_ip_str(adnl::AdnlNodeIdShort l_id, adnl::AdnlNodeIdShort p_id, td::Promise<td::string> promise) override;
RldpIn(td::actor::ActorId<adnl::AdnlPeerTable> adnl) : adnl_(adnl) {
}

4
rldp/rldp.cpp
View file

@ -259,6 +259,10 @@ void RldpIn::add_id(adnl::AdnlNodeIdShort local_id) {
local_ids_.insert(local_id);
}
void RldpIn::get_conn_ip_str(adnl::AdnlNodeIdShort l_id, adnl::AdnlNodeIdShort p_id, td::Promise<td::string> promise) {
td::actor::send_closure(adnl_, &adnl::AdnlPeerTable::get_conn_ip_str, l_id, p_id, std::move(promise));
}
std::unique_ptr<adnl::Adnl::Callback> RldpIn::make_adnl_callback() {
class Callback : public adnl::Adnl::Callback {
private:

2
rldp2/rldp-in.hpp
View file

@ -90,6 +90,8 @@ class RldpIn : public RldpImpl {
void add_id(adnl::AdnlNodeIdShort local_id) override;
void get_conn_ip_str(adnl::AdnlNodeIdShort l_id, adnl::AdnlNodeIdShort p_id, td::Promise<td::string> promise) override;
RldpIn(td::actor::ActorId<adnl::AdnlPeerTable> adnl) : adnl_(adnl) {
}

4
rldp2/rldp.cpp
View file

@ -217,6 +217,10 @@ void RldpIn::add_id(adnl::AdnlNodeIdShort local_id) {
local_ids_.insert(local_id);
}
void RldpIn::get_conn_ip_str(adnl::AdnlNodeIdShort l_id, adnl::AdnlNodeIdShort p_id, td::Promise<td::string> promise) {
td::actor::send_closure(adnl_, &adnl::AdnlPeerTable::get_conn_ip_str, l_id, p_id, std::move(promise));
}
std::unique_ptr<adnl::Adnl::Callback> RldpIn::make_adnl_callback() {
class Callback : public adnl::Adnl::Callback {
private:

2
storage/storage-cli.cpp
View file

@ -95,7 +95,7 @@ class PeerManager : public td::actor::Actor {
}
};
send_closure(overlays_, &ton::overlay::Overlays::create_public_overlay, adnl_id_, overlay_id_.clone(),
std::make_unique<Callback>(), rules);
std::make_unique<Callback>(), rules, "{ \"type\": \"storage\" }");
}
void tear_down() override {
send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, adnl_id_, overlay_id_.compute_short_id());

22
tdutils/td/utils/port/ServerSocketFd.cpp
View file

@ -253,25 +253,9 @@ class ServerSocketFdImpl {
}
auto error = Status::PosixError(accept_errno, PSLICE() << "Accept from " << get_native_fd() << " has failed");
switch (accept_errno) {
case EBADF:
case EFAULT:
case EINVAL:
case ENOTSOCK:
case EOPNOTSUPP:
LOG(FATAL) << error;
UNREACHABLE();
break;
default:
LOG(ERROR) << error;
// fallthrough
case EMFILE:
case ENFILE:
case ECONNABORTED: //???
get_poll_info().clear_flags(PollFlags::Read());
get_poll_info().add_flags(PollFlags::Close());
return std::move(error);
}
get_poll_info().clear_flags(PollFlags::Read());
get_poll_info().add_flags(PollFlags::Close());
return std::move(error);
}
Status get_pending_error() {

10
test/test-catchain.cpp
View file

@ -122,8 +122,9 @@ class CatChainInst : public td::actor::Actor {
void start_up() override {
alarm_timestamp() = td::Timestamp::in(0.1);
ton::catchain::CatChainOptions opts;
ton::CatChainOptions opts;
opts.debug_disable_db = true;
//opts.block_hash_covers_data = true;
std::vector<ton::catchain::CatChainNode> nodes;
for (auto &n : nodes_) {
@ -157,13 +158,14 @@ class CatChainInst : public td::actor::Actor {
void preprocess_block(ton::catchain::CatChainBlock *block) override {
td::actor::send_closure(id_, &CatChainInst::preprocess_block, std::move(block));
}
void process_broadcast(ton::PublicKeyHash src, td::BufferSlice data) override {
void process_broadcast(const ton::PublicKeyHash &src, td::BufferSlice data) override {
UNREACHABLE();
}
void process_message(ton::PublicKeyHash src, td::BufferSlice data) override {
void process_message(const ton::PublicKeyHash &src, td::BufferSlice data) override {
UNREACHABLE();
}
void process_query(ton::PublicKeyHash src, td::BufferSlice data, td::Promise<td::BufferSlice> promise) override {
void process_query(const ton::PublicKeyHash &src, td::BufferSlice data,
td::Promise<td::BufferSlice> promise) override {
UNREACHABLE();
}
void started() override {

2
third-party/abseil-cpp vendored

@ -1 +1 @@
Subproject commit df3ea785d8c30a9503321a3d35ee7d35808f190d
Subproject commit 215105818dfde3174fe799600bb0f3cae233d0bf

2
third-party/crc32c vendored

@ -1 +1 @@
Subproject commit fd33bccdf4824716df2552f7addd735264f77872
Subproject commit 02e65f4fd3065d27b2e29324800ca6d04df16126

14
tl/generate/scheme/ton_api.tl
View file

@ -259,7 +259,7 @@ catchain.blockUpdate block:catchain.block = catchain.Update;
catchain.block.data.badBlock block:catchain.block = catchain.block.inner.Data;
catchain.block.data.fork left:catchain.block.Dep right:catchain.block.Dep = catchain.block.inner.Data;
catchain.block.data.nop = catchain.block.inner.Data;
catchain.block.data.vector msgs:(vector bytes) = catchain.block.inner.Data;
//catchain.block.data.vector msgs:(vector bytes) = catchain.block.inner.Data;
//catchain.block.data.custom = catchain.block.inner.Data;
catchain.firstblock unique_hash:int256 nodes:(vector int256) = catchain.FirstBlock;
@ -309,11 +309,14 @@ validatorSession.candidate src:int256 round:int root_hash:int256 data:bytes coll
validatorSession.config catchain_idle_timeout:double catchain_max_deps:int round_candidates:int next_candidate_delay:double round_attempt_duration:int
max_round_attempts:int max_block_size:int max_collated_data_size:int = validatorSession.Config;
validatorSession.configVersioned catchain_idle_timeout:double catchain_max_deps:int round_candidates:int next_candidate_delay:double round_attempt_duration:int
max_round_attempts:int max_block_size:int max_collated_data_size:int version:int = validatorSession.Config;
validatorSession.configNew catchain_idle_timeout:double catchain_max_deps:int round_candidates:int next_candidate_delay:double round_attempt_duration:int
max_round_attempts:int max_block_size:int max_collated_data_size:int new_catchain_ids:Bool = validatorSession.Config;
validatorSession.catchainOptions idle_timeout:double max_deps:int max_block_size:int block_hash_covers_data:Bool
max_block_height_ceoff:int debug_disable_db:Bool = validatorSession.CatChainOptions;
validatorSession.configVersioned catchain_opts:validatorSession.CatChainOptions round_candidates:int next_candidate_delay:double
round_attempt_duration:int max_round_attempts:int max_block_size:int max_collated_data_size:int version:int = validatorSession.Config;
---functions---
validatorSession.ping hash:long = validatorSession.Pong;
@ -620,8 +623,9 @@ engine.validator.proposalVote perm_key:int256 to_send:bytes = engine.validator.P
engine.validator.dhtServerStatus id:int256 status:int = engine.validator.DhtServerStatus;
engine.validator.dhtServersStatus servers:(vector engine.validator.dhtServerStatus) = engine.validator.DhtServersStatus;
engine.validator.overlayStats overlay_id:int256 overlay_id_full:PublicKey adnl_id:int256 nodes:(vector adnl.id.short)
stats:(vector engine.validator.oneStat) = engine.validator.OverlayStats;
engine.validator.overlayStatsNode adnl_id:int256 ip_addr:string bdcst_errors:int fec_bdcst_errors:int last_in_query:int last_out_query:int t_out_bytes:int t_in_bytes:int t_out_pckts:int t_in_pckts:int = engine.validator.OverlayStatsNode;
engine.validator.overlayStats overlay_id:int256 overlay_id_full:PublicKey adnl_id:int256 scope:string nodes:(vector engine.validator.overlayStatsNode) stats:(vector engine.validator.oneStat) = engine.validator.OverlayStats;
engine.validator.overlaysStats overlays:(vector engine.validator.overlayStats) = engine.validator.OverlaysStats;

BIN
tl/generate/scheme/ton_api.tlo
View file

Binary file not shown.

19
ton/ton-types.h
View file

@ -24,6 +24,7 @@
#include "td/utils/Slice.h"
#include "td/utils/UInt.h"
#include "td/utils/misc.h"
#include "td/utils/optional.h"
#include <cinttypes>
@ -447,11 +448,23 @@ struct ValidatorDescr {
}
};
struct CatChainOptions {
double idle_timeout = 16.0;
td::uint32 max_deps = 4;
td::uint32 max_serialized_block_size = 16 * 1024;
bool block_hash_covers_data = false;
// Max block height = max_block_height_coeff * (1 + N / max_deps) / 1000
// N - number of participants
// 0 - unlimited
td::uint64 max_block_height_coeff = 0;
bool debug_disable_db = false;
};
struct ValidatorSessionConfig {
td::uint32 proto_version = 0;
/* double */ double catchain_idle_timeout = 16.0;
td::uint32 catchain_max_deps = 4;
CatChainOptions catchain_opts;
td::uint32 round_candidates = 3;
/* double */ double next_candidate_delay = 2.0;
@ -462,6 +475,8 @@ struct ValidatorSessionConfig {
td::uint32 max_collated_data_size = (4 << 20);
bool new_catchain_ids = false;
static const td::uint32 BLOCK_HASH_COVERS_DATA_FROM_VERSION = 2;
};
} // namespace ton

97
validator-engine-console/validator-engine-console-query.cpp
View file

@ -34,6 +34,7 @@
#include "overlay/overlays.h"
#include <cctype>
#include <fstream>
Tokenizer::Tokenizer(td::BufferSlice data) : data_(std::move(data)) {
remaining_ = data_.as_slice();
@ -835,11 +836,25 @@ td::Status GetOverlaysStatsQuery::receive(td::BufferSlice data) {
"received incorrect answer: ");
for (auto &s : f->overlays_) {
td::StringBuilder sb;
sb << "overlay_id=" << s->overlay_id_ << " adnl_id=" << s->adnl_id_ << "\n";
sb << " nodes:";
sb << "overlay_id: " << s->overlay_id_ << " adnl_id: " << s->adnl_id_ << " scope: " << s->scope_ << "\n";
sb << " nodes:\n";
td::uint32 overlay_t_out_bytes = 0;
td::uint32 overlay_t_out_pckts = 0;
td::uint32 overlay_t_in_bytes = 0;
td::uint32 overlay_t_in_pckts = 0;
for (auto &n : s->nodes_) {
sb << " " << n->id_ << "\n";
sb << " adnl_id: " << n->adnl_id_ << " ip_addr: " << n->ip_addr_ << " broadcast_errors: " << n->bdcst_errors_ << " fec_broadcast_errors: " << n->fec_bdcst_errors_ << " last_in_query: " << n->last_in_query_ << " (" << time_to_human(n->last_in_query_) << ")" << " last_out_query: " << n->last_out_query_ << " (" << time_to_human(n->last_out_query_) << ")" << "\n throughput:\n out: " << n->t_out_bytes_ << " bytes/sec, " << n->t_out_pckts_ << " pckts/sec\n in: " << n->t_in_bytes_ << " bytes/sec, " << n->t_in_pckts_ << " pckts/sec\n";
overlay_t_out_bytes += n->t_out_bytes_;
overlay_t_out_pckts += n->t_out_pckts_;
overlay_t_in_bytes += n->t_in_bytes_;
overlay_t_in_pckts += n->t_in_pckts_;
}
sb << " total_throughput:\n out: " << overlay_t_out_bytes << " bytes/sec, " << overlay_t_out_pckts << " pckts/sec\n in: " << overlay_t_in_bytes << " bytes/sec, " << overlay_t_in_pckts << " pckts/sec\n";
sb << " stats:\n";
for (auto &t : s->stats_) {
sb << " " << t->key_ << "\t" << t->value_ << "\n";
@ -849,6 +864,82 @@ td::Status GetOverlaysStatsQuery::receive(td::BufferSlice data) {
return td::Status::OK();
}
td::Status GetOverlaysStatsJsonQuery::run() {
TRY_RESULT_ASSIGN(file_name_, tokenizer_.get_token<std::string>());
TRY_STATUS(tokenizer_.check_endl());
return td::Status::OK();
}
td::Status GetOverlaysStatsJsonQuery::send() {
auto b = ton::create_serialize_tl_object<ton::ton_api::engine_validator_getOverlaysStats>();
td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise());
return td::Status::OK();
}
td::Status GetOverlaysStatsJsonQuery::receive(td::BufferSlice data) {
TRY_RESULT_PREFIX(f, ton::fetch_tl_object<ton::ton_api::engine_validator_overlaysStats>(data.as_slice(), true),
"received incorrect answer: ");
std::ofstream sb(file_name_);
sb << "[\n";
bool rtail = false;
for (auto &s : f->overlays_) {
if(rtail) {
sb << ",\n";
} else {
rtail = true;
}
sb << "{\n \"overlay_id\": \"" << s->overlay_id_ << "\",\n \"adnl_id\": \"" << s->adnl_id_ << "\",\n \"scope\": " << s->scope_ << ",\n";
sb << " \"nodes\": [\n";
td::uint32 overlay_t_out_bytes = 0;
td::uint32 overlay_t_out_pckts = 0;
td::uint32 overlay_t_in_bytes = 0;
td::uint32 overlay_t_in_pckts = 0;
bool tail = false;
for (auto &n : s->nodes_) {
if(tail) {
sb << ",\n";
} else {
tail = true;
}
sb << " {\n \"adnl_id\": \"" << n->adnl_id_ << "\",\n \"ip_addr\": \"" << n->ip_addr_ << "\",\n \"broadcast_errors\": " << n->bdcst_errors_ << ",\n \"fec_broadcast_errors\": " << n->fec_bdcst_errors_ << ",\n \"last_in_query_unix\": " << n->last_in_query_ << ",\n \"last_in_query_human\": \"" << time_to_human(n->last_in_query_) << "\",\n" << " \"last_out_query_unix\": " << n->last_out_query_ << ",\n \"last_out_query_human\": \"" << time_to_human(n->last_out_query_) << "\",\n" << "\n \"throughput\": { \"out_bytes_sec\": " << n->t_out_bytes_ << ", \"out_pckts_sec\": " << n->t_out_pckts_ << ", \"in_bytes_sec\": " << n->t_in_bytes_ << ", \"in_pckts_sec\": " << n->t_in_pckts_ << " }\n }";
overlay_t_out_bytes += n->t_out_bytes_;
overlay_t_out_pckts += n->t_out_pckts_;
overlay_t_in_bytes += n->t_in_bytes_;
overlay_t_in_pckts += n->t_in_pckts_;
}
sb << " ],\n";
sb << " \"total_throughput\": { \"out_bytes_sec\": " << overlay_t_out_bytes << ", \"out_pckts_sec\": " << overlay_t_out_pckts << ", \"in_bytes_sec\": " << overlay_t_in_bytes << ", \"in_pckts_sec\": " << overlay_t_in_pckts << " },\n";
sb << " \"stats\": {\n";
tail = false;
for (auto &t : s->stats_) {
if(tail) {
sb << ",\n";
} else {
tail = true;
}
sb << " \"" << t->key_ << "\": \"" << t->value_ << "\"";
}
sb << "\n }\n";
sb << "}\n";
}
sb << "]\n";
sb << std::flush;
td::TerminalIO::output(std::string("wrote stats to " + file_name_ + "\n"));
return td::Status::OK();
}
td::Status ImportCertificateQuery::receive(td::BufferSlice data) {
TRY_RESULT_PREFIX(f, ton::fetch_tl_object<ton::ton_api::engine_validator_success>(data.as_slice(), true),

36
validator-engine-console/validator-engine-console-query.h
View file

@ -193,6 +193,20 @@ class Query : public td::actor::Actor {
virtual std::string name() const = 0;
void handle_error(td::Status error);
static std::string time_to_human(int unixtime) {
char time_buffer[80];
time_t rawtime = unixtime;
struct tm tInfo;
#if defined(_WIN32) || defined(_WIN64)
struct tm* timeinfo = localtime_s(&tInfo, &rawtime) ? nullptr : &tInfo;
#else
struct tm* timeinfo = localtime_r(&rawtime, &tInfo);
#endif
assert(timeinfo == &tInfo);
strftime(time_buffer, 80, "%c", timeinfo);
return std::string(time_buffer);
}
protected:
td::actor::ActorId<ValidatorEngineConsole> console_;
Tokenizer tokenizer_;
@ -923,6 +937,28 @@ class GetOverlaysStatsQuery : public Query {
}
};
class GetOverlaysStatsJsonQuery : public Query {
public:
GetOverlaysStatsJsonQuery(td::actor::ActorId<ValidatorEngineConsole> console, Tokenizer tokenizer)
: Query(console, std::move(tokenizer)) {
}
td::Status run() override;
td::Status send() override;
td::Status receive(td::BufferSlice data) override;
static std::string get_name() {
return "getoverlaysstatsjson";
}
static std::string get_help() {
return "getoverlaysstatsjson <outfile>\tgets stats for all overlays and writes to json file";
}
std::string name() const override {
return get_name();
}
private:
std::string file_name_;
};
class SignCertificateQuery : public Query {
public:
SignCertificateQuery(td::actor::ActorId<ValidatorEngineConsole> console, Tokenizer tokenizer)

1
validator-engine-console/validator-engine-console.cpp
View file

@ -137,6 +137,7 @@ void ValidatorEngineConsole::run() {
add_query_runner(std::make_unique<QueryRunnerImpl<SignCertificateQuery>>());
add_query_runner(std::make_unique<QueryRunnerImpl<ImportCertificateQuery>>());
add_query_runner(std::make_unique<QueryRunnerImpl<GetOverlaysStatsQuery>>());
add_query_runner(std::make_unique<QueryRunnerImpl<GetOverlaysStatsJsonQuery>>());
add_query_runner(std::make_unique<QueryRunnerImpl<ImportShardOverlayCertificateQuery>>());
add_query_runner(std::make_unique<QueryRunnerImpl<SignShardOverlayCertificateQuery>>());
}

9
validator-session/validator-session-types.h
View file

@ -37,11 +37,10 @@ inline ValidatorSessionCandidateId skip_round_candidate_id() {
}
struct ValidatorSessionOptions {
ValidatorSessionOptions() {
}
ValidatorSessionOptions(const ValidatorSessionConfig &conf);
double catchain_idle_timeout = 16.0;
td::uint32 catchain_max_deps = 4;
ValidatorSessionOptions() = default;
explicit ValidatorSessionOptions(const ValidatorSessionConfig &conf);
CatChainOptions catchain_opts;
td::uint32 round_candidates = 3;
double next_candidate_delay = 2.0;

36
validator-session/validator-session.cpp
View file

@ -337,6 +337,9 @@ void ValidatorSessionImpl::candidate_decision_fail(td::uint32 round, ValidatorSe
}
LOG(ERROR) << this << ": failed candidate " << hash << ": " << result;
pending_approve_.erase(hash);
if (result.size() > MAX_REJECT_REASON_SIZE) {
result.resize(MAX_REJECT_REASON_SIZE);
}
pending_reject_.emplace(hash, td::BufferSlice{result});
rejected_.insert(hash);
}
@ -840,10 +843,8 @@ void ValidatorSessionImpl::start() {
auto w = description().export_catchain_nodes();
catchain_ = catchain::CatChain::create(
make_catchain_callback(),
catchain::CatChainOptions{description().opts().catchain_idle_timeout, description().opts().catchain_max_deps},
keyring_, adnl_, overlay_manager_, std::move(w), local_id(), unique_hash_, db_root_, db_suffix_,
allow_unsafe_self_blocks_resync_);
make_catchain_callback(), description().opts().catchain_opts, keyring_, adnl_, overlay_manager_, std::move(w),
local_id(), unique_hash_, db_root_, db_suffix_, allow_unsafe_self_blocks_resync_);
check_all();
}
@ -883,26 +884,27 @@ td::actor::ActorOwn<ValidatorSession> ValidatorSession::create(
td::Bits256 ValidatorSessionOptions::get_hash() const {
if(!proto_version) {
if (!new_catchain_ids) {
return create_hash_tl_object<ton_api::validatorSession_config>(
catchain_idle_timeout, catchain_max_deps, round_candidates, next_candidate_delay, round_attempt_duration,
max_round_attempts, max_block_size, max_collated_data_size);
return create_hash_tl_object<ton_api::validatorSession_config>(
catchain_opts.idle_timeout, catchain_opts.max_deps, round_candidates, next_candidate_delay,
round_attempt_duration, max_round_attempts, max_block_size, max_collated_data_size);
} else {
return create_hash_tl_object<ton_api::validatorSession_configNew>(
catchain_idle_timeout, catchain_max_deps, round_candidates, next_candidate_delay, round_attempt_duration,
max_round_attempts, max_block_size, max_collated_data_size, new_catchain_ids);
return create_hash_tl_object<ton_api::validatorSession_configNew>(
catchain_opts.idle_timeout, catchain_opts.max_deps, round_candidates, next_candidate_delay,
round_attempt_duration, max_round_attempts, max_block_size, max_collated_data_size, new_catchain_ids);
}
} else {
return create_hash_tl_object<ton_api::validatorSession_configVersioned>(
catchain_idle_timeout, catchain_max_deps, round_candidates, next_candidate_delay, round_attempt_duration,
max_round_attempts, max_block_size, max_collated_data_size, proto_version);
}
return create_hash_tl_object<ton_api::validatorSession_configVersioned>(
create_tl_object<ton_api::validatorSession_catchainOptions>(
catchain_opts.idle_timeout, catchain_opts.max_deps, catchain_opts.max_serialized_block_size,
catchain_opts.block_hash_covers_data, catchain_opts.max_block_height_coeff, catchain_opts.debug_disable_db),
round_candidates, next_candidate_delay, round_attempt_duration,
max_round_attempts, max_block_size, max_collated_data_size, proto_version);
}
}
ValidatorSessionOptions::ValidatorSessionOptions(const ValidatorSessionConfig &conf) {
proto_version = conf.proto_version;
catchain_idle_timeout = conf.catchain_idle_timeout;
catchain_max_deps = conf.catchain_max_deps;
catchain_opts = conf.catchain_opts;
max_block_size = conf.max_block_size;
max_collated_data_size = conf.max_collated_data_size;
max_round_attempts = conf.max_round_attempts;

10
validator-session/validator-session.hpp
View file

@ -109,13 +109,14 @@ class ValidatorSessionImpl : public ValidatorSession {
void preprocess_block(catchain::CatChainBlock *block) override {
td::actor::send_closure(id_, &ValidatorSessionImpl::preprocess_block, block);
}
void process_broadcast(PublicKeyHash src, td::BufferSlice data) override {
void process_broadcast(const PublicKeyHash &src, td::BufferSlice data) override {
td::actor::send_closure(id_, &ValidatorSessionImpl::process_broadcast, src, std::move(data));
}
void process_message(PublicKeyHash src, td::BufferSlice data) override {
void process_message(const PublicKeyHash &src, td::BufferSlice data) override {
td::actor::send_closure(id_, &ValidatorSessionImpl::process_message, src, std::move(data));
}
void process_query(PublicKeyHash src, td::BufferSlice data, td::Promise<td::BufferSlice> promise) override {
void process_query(const PublicKeyHash &src, td::BufferSlice data,
td::Promise<td::BufferSlice> promise) override {
td::actor::send_closure(id_, &ValidatorSessionImpl::process_query, src, std::move(data), std::move(promise));
}
void started() override {
@ -195,6 +196,9 @@ class ValidatorSessionImpl : public ValidatorSession {
PrintId print_id() const override {
return PrintId{unique_hash_, description_->get_source_id(description_->get_self_idx())};
}
private:
static const size_t MAX_REJECT_REASON_SIZE = 1024;
};
} // namespace validatorsession

62
validator/db/celldb.cpp
View file

@ -28,11 +28,46 @@ namespace ton {
namespace validator {
class CellDbAsyncExecutor : public vm::DynamicBagOfCellsDb::AsyncExecutor {
public:
explicit CellDbAsyncExecutor(td::actor::ActorId<CellDbBase> cell_db) : cell_db_(std::move(cell_db)) {
}
void execute_async(std::function<void()> f) {
class Runner : public td::actor::Actor {
public:
explicit Runner(std::function<void()> f) : f_(std::move(f)) {}
void start_up() {
f_();
stop();
}
private:
std::function<void()> f_;
};
td::actor::create_actor<Runner>("executeasync", std::move(f)).release();
}
void execute_sync(std::function<void()> f) {
td::actor::send_closure(cell_db_, &CellDbBase::execute_sync, std::move(f));
}
private:
td::actor::ActorId<CellDbBase> cell_db_;
};
void CellDbBase::start_up() {
async_executor = std::make_shared<CellDbAsyncExecutor>(actor_id(this));
}
void CellDbBase::execute_sync(std::function<void()> f) {
f();
}
CellDbIn::CellDbIn(td::actor::ActorId<RootDb> root_db, td::actor::ActorId<CellDb> parent, std::string path)
: root_db_(root_db), parent_(parent), path_(std::move(path)) {
}
void CellDbIn::start_up() {
CellDbBase::start_up();
cell_db_ = std::make_shared<td::RocksDb>(td::RocksDb::open(path_).move_as_ok());
boc_ = vm::DynamicBagOfCellsDb::create();
@ -52,8 +87,7 @@ void CellDbIn::start_up() {
}
void CellDbIn::load_cell(RootHash hash, td::Promise<td::Ref<vm::DataCell>> promise) {
td::PerfWarningTimer{"loadcell", 0.1};
promise.set_result(boc_->load_cell(hash.as_slice()));
boc_->load_cell_async(hash.as_slice(), async_executor, std::move(promise));
}
void CellDbIn::store_cell(BlockIdExt block_id, td::Ref<vm::Cell> cell, td::Promise<td::Ref<vm::DataCell>> promise) {
@ -102,6 +136,10 @@ void CellDbIn::store_cell(BlockIdExt block_id, td::Ref<vm::Cell> cell, td::Promi
promise.set_result(boc_->load_cell(cell->get_hash().as_slice()));
}
void CellDbIn::get_cell_db_reader(td::Promise<std::shared_ptr<vm::CellDbReader>> promise) {
promise.set_result(boc_->get_cell_db_reader());
}
void CellDbIn::alarm() {
auto R = get_block(last_gc_);
R.ensure();
@ -251,12 +289,15 @@ void CellDb::load_cell(RootHash hash, td::Promise<td::Ref<vm::DataCell>> promise
if (!started_) {
td::actor::send_closure(cell_db_, &CellDbIn::load_cell, hash, std::move(promise));
} else {
auto R = boc_->load_cell(hash.as_slice());
if (R.is_error()) {
td::actor::send_closure(cell_db_, &CellDbIn::load_cell, hash, std::move(promise));
} else {
promise.set_result(R.move_as_ok());
}
auto P = td::PromiseCreator::lambda(
[cell_db_in = cell_db_.get(), hash, promise = std::move(promise)](td::Result<td::Ref<vm::DataCell>> R) mutable {
if (R.is_error()) {
td::actor::send_closure(cell_db_in, &CellDbIn::load_cell, hash, std::move(promise));
} else {
promise.set_result(R.move_as_ok());
}
});
boc_->load_cell_async(hash.as_slice(), async_executor, std::move(P));
}
}
@ -264,7 +305,12 @@ void CellDb::store_cell(BlockIdExt block_id, td::Ref<vm::Cell> cell, td::Promise
td::actor::send_closure(cell_db_, &CellDbIn::store_cell, block_id, std::move(cell), std::move(promise));
}
void CellDb::get_cell_db_reader(td::Promise<std::shared_ptr<vm::CellDbReader>> promise) {
td::actor::send_closure(cell_db_, &CellDbIn::get_cell_db_reader, std::move(promise));
}
void CellDb::start_up() {
CellDbBase::start_up();
boc_ = vm::DynamicBagOfCellsDb::create();
cell_db_ = td::actor::create_actor<CellDbIn>("celldbin", root_db_, actor_id(this), path_);
}

17
validator/db/celldb.hpp
View file

@ -33,13 +33,25 @@ namespace validator {
class RootDb;
class CellDb;
class CellDbAsyncExecutor;
class CellDbIn : public td::actor::Actor {
class CellDbBase : public td::actor::Actor {
public:
virtual void start_up();
protected:
std::shared_ptr<vm::DynamicBagOfCellsDb::AsyncExecutor> async_executor;
private:
void execute_sync(std::function<void()> f);
friend CellDbAsyncExecutor;
};
class CellDbIn : public CellDbBase {
public:
using KeyHash = td::Bits256;
void load_cell(RootHash hash, td::Promise<td::Ref<vm::DataCell>> promise);
void store_cell(BlockIdExt block_id, td::Ref<vm::Cell> cell, td::Promise<td::Ref<vm::DataCell>> promise);
void get_cell_db_reader(td::Promise<std::shared_ptr<vm::CellDbReader>> promise);
CellDbIn(td::actor::ActorId<RootDb> root_db, td::actor::ActorId<CellDb> parent, std::string path);
@ -88,7 +100,7 @@ class CellDbIn : public td::actor::Actor {
KeyHash last_gc_;
};
class CellDb : public td::actor::Actor {
class CellDb : public CellDbBase {
public:
void load_cell(RootHash hash, td::Promise<td::Ref<vm::DataCell>> promise);
void store_cell(BlockIdExt block_id, td::Ref<vm::Cell> cell, td::Promise<td::Ref<vm::DataCell>> promise);
@ -96,6 +108,7 @@ class CellDb : public td::actor::Actor {
started_ = true;
boc_->set_loader(std::make_unique<vm::CellLoader>(std::move(snapshot))).ensure();
}
void get_cell_db_reader(td::Promise<std::shared_ptr<vm::CellDbReader>> promise);
CellDb(td::actor::ActorId<RootDb> root_db, std::string path) : root_db_(root_db), path_(path) {
}

4
validator/db/rootdb.cpp
View file

@ -270,6 +270,10 @@ void RootDb::get_block_state(ConstBlockHandle handle, td::Promise<td::Ref<ShardS
}
}
void RootDb::get_cell_db_reader(td::Promise<std::shared_ptr<vm::CellDbReader>> promise) {
td::actor::send_closure(cell_db_, &CellDb::get_cell_db_reader, std::move(promise));
}
void RootDb::store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state,
td::Promise<td::Unit> promise) {
td::actor::send_closure(archive_db_, &ArchiveManager::add_persistent_state, block_id, masterchain_block_id,

1
validator/db/rootdb.hpp
View file

@ -60,6 +60,7 @@ class RootDb : public Db {
void store_block_state(BlockHandle handle, td::Ref<ShardState> state,
td::Promise<td::Ref<ShardState>> promise) override;
void get_block_state(ConstBlockHandle handle, td::Promise<td::Ref<ShardState>> promise) override;
void get_cell_db_reader(td::Promise<std::shared_ptr<vm::CellDbReader>> promise) override;
void store_block_handle(BlockHandle handle, td::Promise<td::Unit> promise) override;
void get_block_handle(BlockIdExt id, td::Promise<BlockHandle> promise) override;

2
validator/full-node-shard.cpp
View file

@ -93,7 +93,7 @@ void FullNodeShardImpl::create_overlay() {
};
td::actor::send_closure(overlays_, &overlay::Overlays::create_public_overlay, adnl_id_, overlay_id_full_.clone(),
std::make_unique<Callback>(actor_id(this)), rules_);
std::make_unique<Callback>(actor_id(this)), rules_, PSTRING() << "{ \"type\": \"shard\", \"shard_id\": " << get_shard() << ", \"workchain_id\": " << get_workchain() << " }");
td::actor::send_closure(rldp_, &rldp::Rldp::add_id, adnl_id_);
if (cert_) {

17
validator/impl/collator.cpp
View file

@ -2326,7 +2326,12 @@ td::Result<std::unique_ptr<block::Transaction>> Collator::impl_create_ordinary_t
if (!trans->compute_phase->accepted) {
if (external) {
// inbound external message was not accepted
return td::Status::Error(-701,"inbound external message rejected by transaction "s + acc->addr.to_hex());
auto const& cp = *trans->compute_phase;
return td::Status::Error(
-701,
PSLICE() << "inbound external message rejected by transaction " << acc->addr.to_hex() << ":\n" <<
"exitcode=" << cp.exit_code << ", steps=" << cp.vm_steps << ", gas_used=" << cp.gas_used <<
(cp.vm_log.empty() ? "" : "\nVM Log (truncated):\n..." + cp.vm_log));
} else if (trans->compute_phase->skip_reason == block::ComputePhase::sk_none) {
return td::Status::Error(-669,"new ordinary transaction for smart contract "s + acc->addr.to_hex() +
" has not been accepted by the smart contract (?)");
@ -3026,9 +3031,9 @@ void Collator::register_new_msgs(block::Transaction& trans) {
}
/*
*
*
* Generate (parts of) new state and block
*
*
*/
bool store_ext_blk_ref_to(vm::CellBuilder& cb, const ton::BlockIdExt& id_ext, ton::LogicalTime end_lt) {
@ -3161,7 +3166,7 @@ bool Collator::create_mc_state_extra() {
" contains an invalid configuration in its data, IGNORING CHANGES";
ignore_cfg_changes = true;
} else {
cfg0 = cfg_dict.lookup_ref(td::BitArray<32>(1 - 1));
cfg0 = cfg_dict.lookup_ref(td::BitArray<32>{(long long) 0});
}
bool changed_cfg = false;
if (cfg0.not_null()) {
@ -4036,9 +4041,9 @@ void Collator::return_block_candidate(td::Result<td::Unit> saved) {
}
/*
*
*
* Collator register methods
*
*
*/
td::Result<bool> Collator::register_external_message_cell(Ref<vm::Cell> ext_msg, const ExtMessage::Hash& ext_hash) {

35
validator/impl/external-message.cpp
View file

@ -112,10 +112,12 @@ void ExtMessageQ::run_message(td::BufferSlice data, td::actor::ActorId<ton::vali
if(!acc.unpack(shard_acc, {}, utime, false)) {
promise.set_error(td::Status::Error(PSLICE() << "Failed to unpack account state"));
} else {
if(run_message_on_account(wc, &acc, utime, lt + 1, msg_root, std::move(config))) {
auto status = run_message_on_account(wc, &acc, utime, lt + 1, msg_root, std::move(config));
if (status.is_ok()) {
promise.set_value(td::Unit());
} else {
promise.set_error(td::Status::Error(PSLICE() << "External message was not accepted"));
promise.set_error(td::Status::Error(
PSLICE() << "External message was not accepted\n" << status.message()));
}
}
}
@ -123,11 +125,11 @@ void ExtMessageQ::run_message(td::BufferSlice data, td::actor::ActorId<ton::vali
);
}
bool ExtMessageQ::run_message_on_account(ton::WorkchainId wc,
block::Account* acc,
UnixTime utime, LogicalTime lt,
td::Ref<vm::Cell> msg_root,
std::unique_ptr<block::ConfigInfo> config) {
td::Status ExtMessageQ::run_message_on_account(ton::WorkchainId wc,
block::Account* acc,
UnixTime utime, LogicalTime lt,
td::Ref<vm::Cell> msg_root,
std::unique_ptr<block::ConfigInfo> config) {
Ref<vm::Cell> old_mparams;
std::vector<block::StoragePrices> storage_prices_;
@ -143,28 +145,29 @@ bool ExtMessageQ::run_message_on_account(ton::WorkchainId wc,
&action_phase_cfg_, &masterchain_create_fee,
&basechain_create_fee, wc);
if(fetch_res.is_error()) {
auto error = fetch_res.move_as_error();
LOG(DEBUG) << "Cannot fetch config params" << error.message();
return false;
auto error = fetch_res.move_as_error();
LOG(DEBUG) << "Cannot fetch config params: " << error.message();
return error.move_as_error_prefix("Cannot fetch config params: ");
}
compute_phase_cfg_.with_vm_log = true;
auto res = Collator::impl_create_ordinary_transaction(msg_root, acc, utime, lt,
&storage_phase_cfg_, &compute_phase_cfg_,
&action_phase_cfg_,
true, lt);
if(res.is_error()) {
auto error = res.move_as_error();
LOG(DEBUG) << "Cannot run message on account" << error.message();
return false;
auto error = res.move_as_error();
LOG(DEBUG) << "Cannot run message on account: " << error.message();
return error.move_as_error_prefix("Cannot run message on account: ");
}
std::unique_ptr<block::Transaction> trans = res.move_as_ok();
auto trans_root = trans->commit(*acc);
if (trans_root.is_null()) {
LOG(DEBUG) << "cannot commit new transaction for smart contract ";
return false;
LOG(DEBUG) << "Cannot commit new transaction for smart contract";
return td::Status::Error("Cannot commit new transaction for smart contract");
}
return true;
return td::Status::OK();
}
} // namespace validator

10
validator/impl/external-message.hpp
View file

@ -63,11 +63,11 @@ class ExtMessageQ : public ExtMessage {
static td::Result<td::Ref<ExtMessageQ>> create_ext_message(td::BufferSlice data);
static void run_message(td::BufferSlice data, td::actor::ActorId<ton::validator::ValidatorManager> manager,
td::Promise<td::Unit> promise);
static bool run_message_on_account(ton::WorkchainId wc,
block::Account* acc,
UnixTime utime, LogicalTime lt,
td::Ref<vm::Cell> msg_root,
std::unique_ptr<block::ConfigInfo> config);
static td::Status run_message_on_account(ton::WorkchainId wc,
block::Account* acc,
UnixTime utime, LogicalTime lt,
td::Ref<vm::Cell> msg_root,
std::unique_ptr<block::ConfigInfo> config);
};
} // namespace validator

18
validator/impl/validate-query.cpp
View file

@ -150,9 +150,9 @@ void ValidateQuery::finish_query() {
}
/*
*
*
* INITIAL PARSE & LOAD REQUIRED DATA
*
*
*/
void ValidateQuery::start_up() {
@ -982,9 +982,9 @@ bool ValidateQuery::check_this_shard_mc_info() {
}
/*
*
*
* METHODS CALLED FROM try_validate() stage 0
*
*
*/
bool ValidateQuery::compute_prev_state() {
@ -1844,9 +1844,9 @@ bool ValidateQuery::check_utime_lt() {
}
/*
*
*
* METHODS CALLED FROM try_validate() stage 1
*
*
*/
// almost the same as in Collator
@ -4956,7 +4956,7 @@ bool ValidateQuery::check_config_update(Ref<vm::CellSlice> old_conf_params, Ref<
return reject_query("no important parameters have been changed, but the block is marked as a key block");
}
vm::Dictionary dict1{ocfg_root, 32};
auto param0 = dict1.lookup_ref(td::BitArray<32>{1 - 1});
auto param0 = dict1.lookup_ref(td::BitArray<32>{(long long) 0});
if (param0.is_null()) {
if (cfg_acc_changed) {
return reject_query("new state of old configuration smart contract "s + old_cfg_addr.to_hex() +
@ -5431,10 +5431,10 @@ bool ValidateQuery::check_mc_block_extra() {
}
/*
*
*
* MAIN VALIDATOR FUNCTION
* (invokes other methods in a suitable order)
*
*
*/
bool ValidateQuery::try_validate() {

1
validator/interfaces/db.h
View file

@ -50,6 +50,7 @@ class Db : public td::actor::Actor {
virtual void store_block_state(BlockHandle handle, td::Ref<ShardState> state,
td::Promise<td::Ref<ShardState>> promise) = 0;
virtual void get_block_state(ConstBlockHandle handle, td::Promise<td::Ref<ShardState>> promise) = 0;
virtual void get_cell_db_reader(td::Promise<std::shared_ptr<vm::CellDbReader>> promise) = 0;
virtual void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state,
td::Promise<td::Unit> promise) = 0;

2
validator/interfaces/validator-manager.h
View file

@ -27,6 +27,7 @@
#include "message-queue.h"
#include "validator/validator.h"
#include "liteserver.h"
#include "crypto/vm/db/DynamicBagOfCellsDb.h"
namespace ton {
@ -55,6 +56,7 @@ class ValidatorManager : public ValidatorManagerInterface {
public:
virtual void set_block_state(BlockHandle handle, td::Ref<ShardState> state,
td::Promise<td::Ref<ShardState>> promise) = 0;
virtual void get_cell_db_reader(td::Promise<std::shared_ptr<vm::CellDbReader>> promise) = 0;
virtual void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state,
td::Promise<td::Unit> promise) = 0;
virtual void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id,

4
validator/manager-disk.cpp
View file

@ -674,6 +674,10 @@ void ValidatorManagerImpl::set_block_state(BlockHandle handle, td::Ref<ShardStat
td::actor::send_closure(db_, &Db::store_block_state, handle, state, std::move(promise));
}
void ValidatorManagerImpl::get_cell_db_reader(td::Promise<std::shared_ptr<vm::CellDbReader>> promise) {
td::actor::send_closure(db_, &Db::get_cell_db_reader, std::move(promise));
}
void ValidatorManagerImpl::store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id,
td::BufferSlice state, td::Promise<td::Unit> promise) {
td::actor::send_closure(db_, &Db::store_persistent_state_file, block_id, masterchain_block_id, std::move(state),

1
validator/manager-disk.hpp
View file

@ -141,6 +141,7 @@ class ValidatorManagerImpl : public ValidatorManager {
void set_block_state(BlockHandle handle, td::Ref<ShardState> state,
td::Promise<td::Ref<ShardState>> promise) override;
void get_cell_db_reader(td::Promise<std::shared_ptr<vm::CellDbReader>> promise) override;
void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state,
td::Promise<td::Unit> promise) override;
void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id,

4
validator/manager-hardfork.cpp
View file

@ -530,6 +530,10 @@ void ValidatorManagerImpl::get_block_handle(BlockIdExt id, bool force, td::Promi
td::actor::send_closure(db_, &Db::get_block_handle, id, std::move(P));
}
void ValidatorManagerImpl::get_cell_db_reader(td::Promise<std::shared_ptr<vm::CellDbReader>> promise) {
td::actor::send_closure(db_, &Db::get_cell_db_reader, std::move(promise));
}
void ValidatorManagerImpl::register_block_handle(BlockHandle handle, td::Promise<BlockHandle> promise) {
auto it = handles_.find(handle->id());
if (it != handles_.end()) {

1
validator/manager-hardfork.hpp
View file

@ -165,6 +165,7 @@ class ValidatorManagerImpl : public ValidatorManager {
td::Promise<td::Ref<ShardState>> promise) override {
UNREACHABLE();
}
void get_cell_db_reader(td::Promise<std::shared_ptr<vm::CellDbReader>> promise) override;
void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state,
td::Promise<td::Unit> promise) override {
UNREACHABLE();

4
validator/manager.cpp
View file

@ -1045,6 +1045,10 @@ void ValidatorManagerImpl::set_block_state(BlockHandle handle, td::Ref<ShardStat
td::actor::send_closure(db_, &Db::store_block_state, handle, state, std::move(P));
}
void ValidatorManagerImpl::get_cell_db_reader(td::Promise<std::shared_ptr<vm::CellDbReader>> promise) {
td::actor::send_closure(db_, &Db::get_cell_db_reader, std::move(promise));
}
void ValidatorManagerImpl::store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id,
td::BufferSlice state, td::Promise<td::Unit> promise) {
td::actor::send_closure(db_, &Db::store_persistent_state_file, block_id, masterchain_block_id, std::move(state),

1
validator/manager.hpp
View file

@ -345,6 +345,7 @@ class ValidatorManagerImpl : public ValidatorManager {
void set_block_state(BlockHandle handle, td::Ref<ShardState> state,
td::Promise<td::Ref<ShardState>> promise) override;
void get_cell_db_reader(td::Promise<std::shared_ptr<vm::CellDbReader>> promise) override;
void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state,
td::Promise<td::Unit> promise) override;
void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id,

71
validator/state-serializer.cpp
View file

@ -82,24 +82,24 @@ void AsyncStateSerializer::alarm() {
}
void AsyncStateSerializer::request_masterchain_state() {
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Ref<ShardState>> R) {
if (R.is_error()) {
td::actor::send_closure(SelfId, &AsyncStateSerializer::fail_handler,
R.move_as_error_prefix("failed to get masterchain state: "));
} else {
td::actor::send_closure(SelfId, &AsyncStateSerializer::got_masterchain_state,
td::Ref<MasterchainState>(R.move_as_ok()));
}
});
td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, masterchain_handle_, std::move(P));
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Ref<ShardState>> R) {
if (R.is_error()) {
td::actor::send_closure(SelfId, &AsyncStateSerializer::fail_handler,
R.move_as_error_prefix("failed to get masterchain state: "));
} else {
td::actor::send_closure(SelfId, &AsyncStateSerializer::got_masterchain_state,
td::Ref<MasterchainState>(R.move_as_ok()));
}
});
td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, masterchain_handle_, std::move(P));
}
void AsyncStateSerializer::request_shard_state(BlockIdExt shard) {
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<BlockHandle> R) {
R.ensure();
td::actor::send_closure(SelfId, &AsyncStateSerializer::got_shard_handle, R.move_as_ok());
});
return td::actor::send_closure(manager_, &ValidatorManager::get_block_handle, shard, true, std::move(P));
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<BlockHandle> R) {
R.ensure();
td::actor::send_closure(SelfId, &AsyncStateSerializer::got_shard_handle, R.move_as_ok());
});
return td::actor::send_closure(manager_, &ValidatorManager::get_block_handle, shard, true, std::move(P));
}
void AsyncStateSerializer::next_iteration() {
@ -122,8 +122,21 @@ void AsyncStateSerializer::next_iteration() {
CHECK(masterchain_handle_->id() == last_block_id_);
if (attempt_ < max_attempt() && last_key_block_id_.id.seqno < last_block_id_.id.seqno &&
need_serialize(masterchain_handle_)) {
if (!cell_db_reader_) {
running_ = true;
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<std::shared_ptr<vm::CellDbReader>> R) {
if (R.is_error()) {
td::actor::send_closure(SelfId, &AsyncStateSerializer::fail_handler,
R.move_as_error_prefix("failed to get cell db reader: "));
} else {
td::actor::send_closure(SelfId, &AsyncStateSerializer::got_cell_db_reader, R.move_as_ok());
}
});
td::actor::send_closure(manager_, &ValidatorManager::get_cell_db_reader, std::move(P));
return;
}
if (!have_masterchain_state_) {
LOG(INFO) << "started serializing persistent state for " << masterchain_handle_->id().seqno();
LOG(INFO) << "started serializing persistent state for " << masterchain_handle_->id().id;
// block next attempts immediately, but send actual request later
running_ = true;
delay_action(
@ -143,9 +156,10 @@ void AsyncStateSerializer::next_iteration() {
return;
}
}
LOG(INFO) << "finished serializing persistent state for " << masterchain_handle_->id().seqno();
LOG(INFO) << "finished serializing persistent state for " << masterchain_handle_->id().id;
last_key_block_ts_ = masterchain_handle_->unix_time();
last_key_block_id_ = masterchain_handle_->id();
cell_db_reader_ = nullptr;
}
if (!saved_to_db_) {
running_ = true;
@ -175,6 +189,13 @@ void AsyncStateSerializer::got_top_masterchain_handle(BlockIdExt block_id) {
}
}
void AsyncStateSerializer::got_cell_db_reader(std::shared_ptr<vm::CellDbReader> cell_db_reader) {
cell_db_reader_ = std::move(cell_db_reader);
running_ = false;
attempt_ = 0;
next_iteration();
}
void AsyncStateSerializer::got_masterchain_handle(BlockHandle handle) {
CHECK(!masterchain_handle_);
masterchain_handle_ = std::move(handle);
@ -184,7 +205,7 @@ void AsyncStateSerializer::got_masterchain_handle(BlockHandle handle) {
}
void AsyncStateSerializer::got_masterchain_state(td::Ref<MasterchainState> state) {
LOG(INFO) << "serializing masterchain state " << masterchain_handle_->id().seqno();
LOG(INFO) << "serializing masterchain state " << masterchain_handle_->id().id;
have_masterchain_state_ = true;
CHECK(next_idx_ == 0);
CHECK(shards_.size() == 0);
@ -194,8 +215,8 @@ void AsyncStateSerializer::got_masterchain_state(td::Ref<MasterchainState> state
shards_.push_back(v->top_block_id());
}
auto write_data = [state] (td::FileFd& fd) {
return state->serialize_to_file(fd);
auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader = cell_db_reader_] (td::FileFd& fd) {
return vm::std_boc_serialize_to_file_large(cell_db_reader, hash, fd, 31);
};
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Unit> R) {
R.ensure();
@ -207,7 +228,7 @@ void AsyncStateSerializer::got_masterchain_state(td::Ref<MasterchainState> state
}
void AsyncStateSerializer::stored_masterchain_state() {
LOG(INFO) << "finished serializing masterchain state " << masterchain_handle_->id().seqno();
LOG(INFO) << "finished serializing masterchain state " << masterchain_handle_->id().id;
running_ = false;
next_iteration();
}
@ -225,13 +246,13 @@ void AsyncStateSerializer::got_shard_handle(BlockHandle handle) {
}
void AsyncStateSerializer::got_shard_state(BlockHandle handle, td::Ref<ShardState> state) {
LOG(INFO) << "serializing shard state " << handle->id().seqno();
auto write_data = [state] (td::FileFd& fd) {
return state->serialize_to_file(fd);
LOG(INFO) << "serializing shard state " << handle->id().id;
auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader = cell_db_reader_] (td::FileFd& fd) {
return vm::std_boc_serialize_to_file_large(cell_db_reader, hash, fd, 31);
};
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result<td::Unit> R) {
R.ensure();
LOG(INFO) << "finished serializing shard state " << handle->id().seqno();
LOG(INFO) << "finished serializing shard state " << handle->id().id;
td::actor::send_closure(SelfId, &AsyncStateSerializer::success_handler);
});
td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file_gen, handle->id(),

2
validator/state-serializer.hpp
View file

@ -42,6 +42,7 @@ class AsyncStateSerializer : public td::actor::Actor {
td::uint32 next_idx_ = 0;
std::shared_ptr<vm::CellDbReader> cell_db_reader_ = nullptr;
BlockHandle masterchain_handle_;
bool have_masterchain_state_ = false;
@ -70,6 +71,7 @@ class AsyncStateSerializer : public td::actor::Actor {
void next_iteration();
void got_top_masterchain_handle(BlockIdExt block_id);
void got_cell_db_reader(std::shared_ptr<vm::CellDbReader> cell_db_reader);
void got_masterchain_handle(BlockHandle handle_);
void got_masterchain_state(td::Ref<MasterchainState> state);
void stored_masterchain_state();