From 89b8717cf8bb43833ff5325ae17bbaa91119aaa8 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Tue, 24 May 2022 21:17:11 +0300 Subject: [PATCH 01/16] Persistent optimization (#382) * State serialization consuming less RAM * Move large-boc-serializer to crypto/vm, refactor code Co-authored-by: SpyCheese --- crypto/CMakeLists.txt | 2 + crypto/vm/boc-writers.h | 146 ++++++++ crypto/vm/boc.cpp | 136 +------- crypto/vm/boc.h | 4 + crypto/vm/db/DynamicBagOfCellsDb.cpp | 10 +- crypto/vm/db/DynamicBagOfCellsDb.h | 7 + crypto/vm/large-boc-serializer.cpp | 411 +++++++++++++++++++++++ validator/db/celldb.cpp | 8 + validator/db/celldb.hpp | 2 + validator/db/rootdb.cpp | 4 + validator/db/rootdb.hpp | 1 + validator/interfaces/db.h | 1 + validator/interfaces/validator-manager.h | 2 + validator/manager-disk.cpp | 4 + validator/manager-disk.hpp | 1 + validator/manager-hardfork.cpp | 4 + validator/manager-hardfork.hpp | 1 + validator/manager.cpp | 4 + validator/manager.hpp | 1 + validator/state-serializer.cpp | 71 ++-- validator/state-serializer.hpp | 2 + 21 files changed, 665 insertions(+), 157 deletions(-) create mode 100644 crypto/vm/boc-writers.h create mode 100644 crypto/vm/large-boc-serializer.cpp diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index 76c8ae9d..58b98088 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -36,6 +36,7 @@ set(TON_CRYPTO_SOURCE vm/debugops.cpp vm/tonops.cpp vm/boc.cpp + vm/large-boc-serializer.cpp vm/utils.cpp vm/vm.cpp tl/tlblib.cpp @@ -68,6 +69,7 @@ set(TON_CRYPTO_SOURCE vm/arithops.h vm/atom.h vm/boc.h + vm/boc-writers.h vm/box.hpp vm/cellops.h vm/continuation.h diff --git a/crypto/vm/boc-writers.h b/crypto/vm/boc-writers.h new file mode 100644 index 00000000..e33886df --- /dev/null +++ b/crypto/vm/boc-writers.h @@ -0,0 +1,146 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "td/utils/port/FileFd.h" +#include "td/utils/crypto.h" +#include + +namespace vm { +namespace boc_writers { +struct BufferWriter { + BufferWriter(unsigned char* store_start, unsigned char* store_end) + : store_start(store_start), store_ptr(store_start), store_end(store_end) {} + + size_t position() const { + return store_ptr - store_start; + } + size_t remaining() const { + return store_end - store_ptr; + } + void chk() const { + DCHECK(store_ptr <= store_end); + } + bool empty() const { + return store_ptr == store_end; + } + void store_uint(unsigned long long value, unsigned bytes) { + unsigned char* ptr = store_ptr += bytes; + chk(); + while (bytes) { + *--ptr = value & 0xff; + value >>= 8; + --bytes; + } + DCHECK(!bytes); + } + void store_bytes(unsigned char const* data, size_t s) { + store_ptr += s; + chk(); + memcpy(store_ptr - s, data, s); + } + unsigned get_crc32() const { + return td::crc32c(td::Slice{store_start, store_ptr}); + } + + private: + unsigned char* store_start; + unsigned char* store_ptr; + unsigned char* store_end; +}; + +struct FileWriter { + FileWriter(td::FileFd& fd, size_t expected_size) + : fd(fd), expected_size(expected_size) {} + + ~FileWriter() { + flush(); + } + + size_t position() const { + return flushed_size + writer.position(); + } + size_t remaining() const { + return expected_size - position(); + } + void chk() const { + DCHECK(position() <= expected_size); + } + bool empty() const { + return remaining() == 0; + } + void store_uint(unsigned long long value, unsigned bytes) { + flush_if_needed(bytes); + writer.store_uint(value, bytes); + } + void store_bytes(unsigned char const* data, size_t s) { + flush_if_needed(s); + writer.store_bytes(data, s); + } + unsigned get_crc32() const { + unsigned char const* start = buf.data(); + unsigned char const* end = start + writer.position(); + return td::crc32c_extend(current_crc32, td::Slice(start, end)); + } + + td::Status finalize() { + flush(); + return std::move(res); + } + + private: + void flush_if_needed(size_t s) { + DCHECK(s <= BUF_SIZE); + if (s > BUF_SIZE - writer.position()) { + flush(); + } + } + + void flush() { + chk(); + unsigned char* start = buf.data(); + unsigned char* end = start + writer.position(); + if (start == end) { + return; + } + flushed_size += end - start; + current_crc32 = td::crc32c_extend(current_crc32, td::Slice(start, end)); + if (res.is_ok()) { + while (end > start) { + auto R = fd.write(td::Slice(start, end)); + if (R.is_error()) { + res = R.move_as_error(); + break; + } + size_t s = R.move_as_ok(); + start += s; + } + } + writer = BufferWriter(buf.data(), buf.data() + buf.size()); + } + + td::FileFd& fd; + size_t expected_size; + size_t flushed_size = 0; + unsigned current_crc32 = td::crc32c(td::Slice()); + + static const size_t BUF_SIZE = 1 << 22; + std::vector buf = std::vector(BUF_SIZE, '\0'); + BufferWriter writer = BufferWriter(buf.data(), buf.data() + buf.size()); + td::Status res = td::Status::OK(); +}; +} +} \ No newline at end of file diff --git a/crypto/vm/boc.cpp b/crypto/vm/boc.cpp index 8ff244d0..d35291d3 100644 --- a/crypto/vm/boc.cpp +++ b/crypto/vm/boc.cpp @@ -20,6 +20,7 @@ #include #include #include "vm/boc.h" +#include "vm/boc-writers.h" #include "vm/cells.h" #include "vm/cellslice.h" #include "td/utils/bits.h" @@ -180,6 +181,7 @@ int BagOfCells::add_root(td::Ref add_root) { return 1; } +// Changes in this function may require corresponding changes in crypto/vm/large-boc-serializer.cpp td::Status BagOfCells::import_cells() { cells_clear(); for (auto& root : roots) { @@ -197,6 +199,7 @@ td::Status BagOfCells::import_cells() { return td::Status::OK(); } +// Changes in this function may require corresponding changes in crypto/vm/large-boc-serializer.cpp td::Result BagOfCells::import_cell(td::Ref cell, int depth) { if (depth > max_depth) { return td::Status::Error("error while importing a cell into a bag of cells: cell depth too large"); @@ -246,6 +249,7 @@ td::Result BagOfCells::import_cell(td::Ref cell, int depth) { return cell_count++; } +// Changes in this function may require corresponding changes in crypto/vm/large-boc-serializer.cpp void BagOfCells::reorder_cells() { int_hashes = 0; for (int i = cell_count - 1; i >= 0; --i) { @@ -323,6 +327,7 @@ void BagOfCells::reorder_cells() { // force=0 : previsit (recursively until special cells are found; then visit them) // force=1 : visit (allocate and process all children) // force=2 : allocate (assign a new index; can be run only after visiting) +// Changes in this function may require corresponding changes in crypto/vm/large-boc-serializer.cpp int BagOfCells::revisit(int cell_idx, int force) { DCHECK(cell_idx >= 0 && cell_idx < cell_count); CellInfo& dci = cell_list_[cell_idx]; @@ -369,6 +374,7 @@ int BagOfCells::revisit(int cell_idx, int force) { return dci.new_idx = -3; // mark as visited (and all children processed) } +// Changes in this function may require corresponding changes in crypto/vm/large-boc-serializer.cpp td::uint64 BagOfCells::compute_sizes(int mode, int& r_size, int& o_size) { int rs = 0, os = 0; if (!root_count || !data_bytes) { @@ -395,6 +401,7 @@ td::uint64 BagOfCells::compute_sizes(int mode, int& r_size, int& o_size) { return data_bytes_adj; } +// Changes in this function may require corresponding changes in crypto/vm/large-boc-serializer.cpp std::size_t BagOfCells::estimate_serialized_size(int mode) { if ((mode & Mode::WithCacheBits) && !(mode & Mode::WithIndex)) { info.invalidate(); @@ -475,130 +482,6 @@ std::string BagOfCells::extract_string() const { return std::string{serialized.data(), serialized.data() + serialized.size()}; } -namespace { -struct BufferWriter { - BufferWriter(unsigned char* store_start, unsigned char* store_end) - : store_start(store_start), store_ptr(store_start), store_end(store_end) {} - - size_t position() const { - return store_ptr - store_start; - } - size_t remaining() const { - return store_end - store_ptr; - } - void chk() const { - DCHECK(store_ptr <= store_end); - } - bool empty() const { - return store_ptr == store_end; - } - void store_uint(unsigned long long value, unsigned bytes) { - unsigned char* ptr = store_ptr += bytes; - chk(); - while (bytes) { - *--ptr = value & 0xff; - value >>= 8; - --bytes; - } - DCHECK(!bytes); - } - void store_bytes(unsigned char const* data, size_t s) { - store_ptr += s; - chk(); - memcpy(store_ptr - s, data, s); - } - unsigned get_crc32() const { - return td::crc32c(td::Slice{store_start, store_ptr}); - } - - private: - unsigned char* store_start; - unsigned char* store_ptr; - unsigned char* store_end; -}; - -struct FileWriter { - FileWriter(td::FileFd& fd, size_t expected_size) - : fd(fd), expected_size(expected_size) {} - - ~FileWriter() { - flush(); - } - - size_t position() const { - return flushed_size + writer.position(); - } - size_t remaining() const { - return expected_size - position(); - } - void chk() const { - DCHECK(position() <= expected_size); - } - bool empty() const { - return remaining() == 0; - } - void store_uint(unsigned long long value, unsigned bytes) { - flush_if_needed(bytes); - writer.store_uint(value, bytes); - } - void store_bytes(unsigned char const* data, size_t s) { - flush_if_needed(s); - writer.store_bytes(data, s); - } - unsigned get_crc32() const { - unsigned char const* start = buf.data(); - unsigned char const* end = start + writer.position(); - return td::crc32c_extend(current_crc32, td::Slice(start, end)); - } - - td::Status finalize() { - flush(); - return std::move(res); - } - - private: - void flush_if_needed(size_t s) { - DCHECK(s <= BUF_SIZE); - if (s > BUF_SIZE - writer.position()) { - flush(); - } - } - - void flush() { - chk(); - unsigned char* start = buf.data(); - unsigned char* end = start + writer.position(); - if (start == end) { - return; - } - flushed_size += end - start; - current_crc32 = td::crc32c_extend(current_crc32, td::Slice(start, end)); - if (res.is_ok()) { - while (end > start) { - auto R = fd.write(td::Slice(start, end)); - if (R.is_error()) { - res = R.move_as_error(); - break; - } - size_t s = R.move_as_ok(); - start += s; - } - } - writer = BufferWriter(buf.data(), buf.data() + buf.size()); - } - - td::FileFd& fd; - size_t expected_size; - size_t flushed_size = 0; - unsigned current_crc32 = td::crc32c(td::Slice()); - - static const size_t BUF_SIZE = 1 << 22; - std::vector buf = std::vector(BUF_SIZE, '\0'); - BufferWriter writer = BufferWriter(buf.data(), buf.data() + buf.size()); - td::Status res = td::Status::OK(); -}; -} - //serialized_boc#672fb0ac has_idx:(## 1) has_crc32c:(## 1) // has_cache_bits:(## 1) flags:(## 2) { flags = 0 } // size:(## 3) { size <= 4 } @@ -610,6 +493,7 @@ struct FileWriter { // index:(cells * ##(off_bytes * 8)) // cell_data:(tot_cells_size * [ uint8 ]) // = BagOfCells; +// Changes in this function may require corresponding changes in crypto/vm/large-boc-serializer.cpp template std::size_t BagOfCells::serialize_to_impl(WriterT& writer, int mode) { auto store_ref = [&](unsigned long long value) { @@ -705,7 +589,7 @@ std::size_t BagOfCells::serialize_to(unsigned char* buffer, std::size_t buff_siz if (!size_est || size_est > buff_size) { return 0; } - BufferWriter writer{buffer, buffer + size_est}; + boc_writers::BufferWriter writer{buffer, buffer + size_est}; return serialize_to_impl(writer, mode); } @@ -714,7 +598,7 @@ td::Status BagOfCells::serialize_to_file(td::FileFd& fd, int mode) { if (!size_est) { return td::Status::Error("no cells to serialize to this bag of cells"); } - FileWriter writer{fd, size_est}; + boc_writers::FileWriter writer{fd, size_est}; size_t s = serialize_to_impl(writer, mode); TRY_STATUS(writer.finalize()); if (s != size_est) { diff --git a/crypto/vm/boc.h b/crypto/vm/boc.h index 02078e27..2fae1846 100644 --- a/crypto/vm/boc.h +++ b/crypto/vm/boc.h @@ -18,6 +18,7 @@ */ #pragma once #include +#include "vm/db/DynamicBagOfCellsDb.h" #include "vm/cells.h" #include "td/utils/Status.h" #include "td/utils/buffer.h" @@ -314,4 +315,7 @@ td::Result>> std_boc_deserialize_multi(td::Slice data, int max_roots = BagOfCells::default_max_roots); td::Result std_boc_serialize_multi(std::vector> root, int mode = 0); +td::Status std_boc_serialize_to_file_large(std::shared_ptr reader, Cell::Hash root_hash, + td::FileFd& fd, int mode = 0); + } // namespace vm diff --git a/crypto/vm/db/DynamicBagOfCellsDb.cpp b/crypto/vm/db/DynamicBagOfCellsDb.cpp index 5fcc3923..636be9c9 100644 --- a/crypto/vm/db/DynamicBagOfCellsDb.cpp +++ b/crypto/vm/db/DynamicBagOfCellsDb.cpp @@ -31,12 +31,6 @@ namespace vm { namespace { -class CellDbReader { - public: - virtual ~CellDbReader() = default; - virtual td::Result> load_cell(td::Slice hash) = 0; -}; - struct DynamicBocExtCellExtra { std::shared_ptr reader; }; @@ -176,6 +170,10 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat return td::Status::OK(); } + std::shared_ptr get_cell_db_reader() override { + return cell_db_reader_; + } + td::Status set_loader(std::unique_ptr loader) override { reset_cell_db_reader(); loader_ = std::move(loader); diff --git a/crypto/vm/db/DynamicBagOfCellsDb.h b/crypto/vm/db/DynamicBagOfCellsDb.h index 67d92ecf..9a87c619 100644 --- a/crypto/vm/db/DynamicBagOfCellsDb.h +++ b/crypto/vm/db/DynamicBagOfCellsDb.h @@ -34,6 +34,12 @@ class ExtCellCreator { virtual td::Result> ext_cell(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) = 0; }; +class CellDbReader { + public: + virtual ~CellDbReader() = default; + virtual td::Result> load_cell(td::Slice hash) = 0; +}; + class DynamicBagOfCellsDb { public: virtual ~DynamicBagOfCellsDb() = default; @@ -52,6 +58,7 @@ class DynamicBagOfCellsDb { virtual td::Status prepare_commit() = 0; virtual Stats get_stats_diff() = 0; virtual td::Status commit(CellStorer &) = 0; + virtual std::shared_ptr get_cell_db_reader() = 0; // restart with new loader will also reset stats_diff virtual td::Status set_loader(std::unique_ptr loader) = 0; diff --git a/crypto/vm/large-boc-serializer.cpp b/crypto/vm/large-boc-serializer.cpp new file mode 100644 index 00000000..31a393ec --- /dev/null +++ b/crypto/vm/large-boc-serializer.cpp @@ -0,0 +1,411 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "vm/boc.h" +#include "vm/boc-writers.h" +#include "vm/cellslice.h" +#include "td/utils/misc.h" + +namespace vm { + +namespace { +// LargeBocSerializer implements serialization of the bag of cells in the standard way +// (equivalent to the implementation in crypto/vm/boc.cpp) +// Changes in this file may require corresponding changes in boc.cpp +class LargeBocSerializer { + public: + using Hash = Cell::Hash; + + explicit LargeBocSerializer(std::shared_ptr reader) : reader(std::move(reader)) {} + + void add_root(Hash root); + td::Status import_cells(); + td::Status serialize(td::FileFd& fd, int mode); + + private: + std::shared_ptr reader; + struct CellInfo { + std::array ref_idx; + int idx; + unsigned short serialized_size; + unsigned char wt; + unsigned char hcnt : 6; + bool should_cache : 1; + bool is_root_cell : 1; + CellInfo(int idx, const std::array& ref_list) : ref_idx(ref_list), idx(idx) { + hcnt = 0; + should_cache = is_root_cell = 0; + } + bool is_special() const { + return !wt; + } + unsigned get_ref_num() const { + for (unsigned i = 0; i < 4; ++i) { + if (ref_idx[i] == -1) { + return i; + } + } + return 4; + } + }; + std::map cells; + std::vector*> cell_list; + struct RootInfo { + RootInfo(Hash hash, int idx) : hash(hash), idx(idx) {} + Hash hash; + int idx; + }; + std::vector roots; + int cell_count = 0, int_refs = 0, int_hashes = 0, top_hashes = 0; + int rv_idx = 0; + unsigned long long data_bytes = 0; + + td::Result import_cell(Hash hash, int depth = 0); + void reorder_cells(); + int revisit(int cell_idx, int force = 0); + td::uint64 compute_sizes(int mode, int& r_size, int& o_size); +}; + +void LargeBocSerializer::add_root(Hash root) { + roots.emplace_back(root, -1); +} + +td::Status LargeBocSerializer::import_cells() { + for (auto& root : roots) { + TRY_RESULT(idx, import_cell(root.hash)); + root.idx = idx; + } + reorder_cells(); + CHECK(!cell_list.empty()); + return td::Status::OK(); +} + +td::Result LargeBocSerializer::import_cell(Hash hash, int depth) { + if (depth > Cell::max_depth) { + return td::Status::Error("error while importing a cell into a bag of cells: cell depth too large"); + } + auto it = cells.find(hash); + if (it != cells.end()) { + it->second.should_cache = true; + return it->second.idx; + } + TRY_RESULT(cell, reader->load_cell(hash.as_slice())); + if (cell->get_virtualization() != 0) { + return td::Status::Error( + "error while importing a cell into a bag of cells: cell has non-zero virtualization level"); + } + CellSlice cs(std::move(cell)); + std::array refs; + std::fill(refs.begin(), refs.end(), -1); + DCHECK(cs.size_refs() <= 4); + unsigned sum_child_wt = 1; + for (unsigned i = 0; i < cs.size_refs(); i++) { + TRY_RESULT(ref, import_cell(cs.prefetch_ref(i)->get_hash(), depth + 1)); + refs[i] = ref; + sum_child_wt += cell_list[ref]->second.wt; + ++int_refs; + } + auto dc = cs.move_as_loaded_cell().data_cell; + auto res = cells.emplace(hash, CellInfo(cell_count, refs)); + DCHECK(res.second); + cell_list.push_back(&*res.first); + CellInfo& dc_info = res.first->second; + dc_info.wt = (unsigned char)std::min(0xffU, sum_child_wt); + unsigned hcnt = dc->get_level_mask().get_hashes_count(); + DCHECK(hcnt <= 4); + dc_info.hcnt = (unsigned char)hcnt; + TRY_RESULT(serialized_size, td::narrow_cast_safe(dc->get_serialized_size())); + data_bytes += dc_info.serialized_size = serialized_size; + return cell_count++; +} + +void LargeBocSerializer::reorder_cells() { + for (auto ptr : cell_list) { + ptr->second.idx = -1; + } + int_hashes = 0; + for (int i = cell_count - 1; i >= 0; --i) { + CellInfo& dci = cell_list[i]->second; + int s = dci.get_ref_num(), c = s, sum = BagOfCells::max_cell_whs - 1, mask = 0; + for (int j = 0; j < s; ++j) { + CellInfo& dcj = cell_list[dci.ref_idx[j]]->second; + int limit = (BagOfCells::max_cell_whs - 1 + j) / s; + if (dcj.wt <= limit) { + sum -= dcj.wt; + --c; + mask |= (1 << j); + } + } + if (c) { + for (int j = 0; j < s; ++j) { + if (!(mask & (1 << j))) { + CellInfo& dcj = cell_list[dci.ref_idx[j]]->second; + int limit = sum++ / c; + if (dcj.wt > limit) { + dcj.wt = (unsigned char)limit; + } + } + } + } + } + for (int i = 0; i < cell_count; i++) { + CellInfo& dci = cell_list[i]->second; + int s = dci.get_ref_num(), sum = 1; + for (int j = 0; j < s; ++j) { + sum += cell_list[dci.ref_idx[j]]->second.wt; + } + DCHECK(sum <= BagOfCells::max_cell_whs); + if (sum <= dci.wt) { + dci.wt = (unsigned char)sum; + } else { + dci.wt = 0; + int_hashes += dci.hcnt; + } + } + top_hashes = 0; + for (auto& root_info : roots) { + auto& cell_info = cell_list[root_info.idx]->second; + if (cell_info.is_root_cell) { + cell_info.is_root_cell = true; + if (cell_info.wt) { + top_hashes += cell_info.hcnt; + } + } + } + if (cell_count > 0) { + rv_idx = 0; + + for (const auto& root_info : roots) { + revisit(root_info.idx, 0); + revisit(root_info.idx, 1); + } + for (const auto& root_info : roots) { + revisit(root_info.idx, 2); + } + for (auto& root_info : roots) { + root_info.idx = cell_list[root_info.idx]->second.idx; + } + + DCHECK(rv_idx == cell_count); + for (int i = 0; i < cell_count; ++i) { + while (cell_list[i]->second.idx != i) { + std::swap(cell_list[i], cell_list[cell_list[i]->second.idx]); + } + } + } +} + +int LargeBocSerializer::revisit(int cell_idx, int force) { + DCHECK(cell_idx >= 0 && cell_idx < cell_count); + CellInfo& dci = cell_list[cell_idx]->second; + if (dci.idx >= 0) { + return dci.idx; + } + if (!force) { + // previsit + if (dci.idx != -1) { + // already previsited or visited + return dci.idx; + } + int n = dci.get_ref_num(); + for (int j = n - 1; j >= 0; --j) { + int child_idx = dci.ref_idx[j]; + // either previsit or visit child, depending on whether it is special + revisit(dci.ref_idx[j], cell_list[child_idx]->second.is_special()); + } + return dci.idx = -2; // mark as previsited + } + if (force > 1) { + // time to allocate + auto i = dci.idx = rv_idx++; + return i; + } + if (dci.idx == -3) { + // already visited + return dci.idx; + } + if (dci.is_special()) { + // if current cell is special, previsit it first + revisit(cell_idx, 0); + } + // visit children + int n = dci.get_ref_num(); + for (int j = n - 1; j >= 0; --j) { + revisit(dci.ref_idx[j], 1); + } + // allocate children + for (int j = n - 1; j >= 0; --j) { + dci.ref_idx[j] = revisit(dci.ref_idx[j], 2); + } + return dci.idx = -3; // mark as visited (and all children processed) +} + +td::uint64 LargeBocSerializer::compute_sizes(int mode, int& r_size, int& o_size) { + using Mode = BagOfCells::Mode; + int rs = 0, os = 0; + if (roots.empty() || !data_bytes) { + r_size = o_size = 0; + return 0; + } + while (cell_count >= (1LL << (rs << 3))) { + rs++; + } + td::uint64 hashes = + (((mode & Mode::WithTopHash) ? top_hashes : 0) + ((mode & Mode::WithIntHashes) ? int_hashes : 0)) * + (Cell::hash_bytes + Cell::depth_bytes); + td::uint64 data_bytes_adj = data_bytes + (unsigned long long)int_refs * rs + hashes; + td::uint64 max_offset = (mode & Mode::WithCacheBits) ? data_bytes_adj * 2 : data_bytes_adj; + while (max_offset >= (1ULL << (os << 3))) { + os++; + } + if (rs > 4 || os > 8) { + r_size = o_size = 0; + return 0; + } + r_size = rs; + o_size = os; + return data_bytes_adj; +} + +td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { + using Mode = BagOfCells::Mode; + BagOfCells::Info info; + if ((mode & Mode::WithCacheBits) && !(mode & Mode::WithIndex)) { + return td::Status::Error("invalid flags"); + } + auto data_bytes_adj = compute_sizes(mode, info.ref_byte_size, info.offset_byte_size); + if (data_bytes_adj == 0) { + return td::Status::Error("no cells to serialize"); + } + info.valid = true; + info.has_crc32c = mode & Mode::WithCRC32C; + info.has_index = mode & Mode::WithIndex; + info.has_cache_bits = mode & Mode::WithCacheBits; + info.root_count = (int)roots.size(); + info.cell_count = cell_count; + info.absent_count = 0; + int crc_size = info.has_crc32c ? 4 : 0; + info.roots_offset = 4 + 1 + 1 + 3 * info.ref_byte_size + info.offset_byte_size; + info.index_offset = info.roots_offset + info.root_count * info.ref_byte_size; + info.data_offset = info.index_offset; + if (info.has_index) { + info.data_offset += (long long)cell_count * info.offset_byte_size; + } + info.magic = BagOfCells::Info::boc_generic; + info.data_size = data_bytes_adj; + info.total_size = info.data_offset + data_bytes_adj + crc_size; + auto res = td::narrow_cast_safe(info.total_size); + if (res.is_error()) { + return td::Status::Error("bag of cells is too large"); + } + + boc_writers::FileWriter writer{fd, info.total_size}; + auto store_ref = [&](unsigned long long value) { + writer.store_uint(value, info.ref_byte_size); + }; + auto store_offset = [&](unsigned long long value) { + writer.store_uint(value, info.offset_byte_size); + }; + + writer.store_uint(info.magic, 4); + + td::uint8 byte{0}; + if (info.has_index) { + byte |= 1 << 7; + } + if (info.has_crc32c) { + byte |= 1 << 6; + } + if (info.has_cache_bits) { + byte |= 1 << 5; + } + byte |= (td::uint8)info.ref_byte_size; + writer.store_uint(byte, 1); + + writer.store_uint(info.offset_byte_size, 1); + store_ref(cell_count); + store_ref(roots.size()); + store_ref(0); + store_offset(info.data_size); + for (const auto& root_info : roots) { + int k = cell_count - 1 - root_info.idx; + DCHECK(k >= 0 && k < cell_count); + store_ref(k); + } + DCHECK(writer.position() == info.index_offset); + DCHECK((unsigned)cell_count == cell_list.size()); + if (info.has_index) { + std::size_t offs = 0; + for (int i = cell_count - 1; i >= 0; --i) { + const auto& dc_info = cell_list[i]->second; + bool with_hash = (mode & Mode::WithIntHashes) && !dc_info.wt; + if (dc_info.is_root_cell && (mode & Mode::WithTopHash)) { + with_hash = true; + } + int hash_size = 0; + if (with_hash) { + hash_size = (Cell::hash_bytes + Cell::depth_bytes) * dc_info.hcnt; + } + offs += dc_info.serialized_size + hash_size + dc_info.get_ref_num() * info.ref_byte_size; + auto fixed_offset = offs; + if (info.has_cache_bits) { + fixed_offset = offs * 2 + dc_info.should_cache; + } + store_offset(fixed_offset); + } + DCHECK(offs == info.data_size); + } + DCHECK(writer.position() == info.data_offset); + size_t keep_position = writer.position(); + for (int i = 0; i < cell_count; ++i) { + auto hash = cell_list[cell_count - 1 - i]->first; + const auto& dc_info = cell_list[cell_count - 1 - i]->second; + TRY_RESULT(dc, reader->load_cell(hash.as_slice())); + bool with_hash = (mode & Mode::WithIntHashes) && !dc_info.wt; + if (dc_info.is_root_cell && (mode & Mode::WithTopHash)) { + with_hash = true; + } + unsigned char buf[256]; + int s = dc->serialize(buf, 256, with_hash); + writer.store_bytes(buf, s); + DCHECK(dc->size_refs() == dc_info.get_ref_num()); + unsigned ref_num = dc_info.get_ref_num(); + for (unsigned j = 0; j < ref_num; ++j) { + int k = cell_count - 1 - dc_info.ref_idx[j]; + DCHECK(k > i && k < cell_count); + store_ref(k); + } + } + DCHECK(writer.position() - keep_position == info.data_size); + if (info.has_crc32c) { + unsigned crc = writer.get_crc32(); + writer.store_uint(td::bswap32(crc), 4); + } + DCHECK(writer.empty()); + return writer.finalize(); +} +} + +td::Status std_boc_serialize_to_file_large(std::shared_ptr reader, Cell::Hash root_hash, + td::FileFd& fd, int mode) { + CHECK(reader != nullptr) + LargeBocSerializer serializer(reader); + serializer.add_root(root_hash); + TRY_STATUS(serializer.import_cells()); + return serializer.serialize(fd, mode); +} + +} diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index 6b1d99d8..e7458407 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -102,6 +102,10 @@ void CellDbIn::store_cell(BlockIdExt block_id, td::Ref cell, td::Promi promise.set_result(boc_->load_cell(cell->get_hash().as_slice())); } +void CellDbIn::get_cell_db_reader(td::Promise> promise) { + promise.set_result(boc_->get_cell_db_reader()); +} + void CellDbIn::alarm() { auto R = get_block(last_gc_); R.ensure(); @@ -264,6 +268,10 @@ void CellDb::store_cell(BlockIdExt block_id, td::Ref cell, td::Promise td::actor::send_closure(cell_db_, &CellDbIn::store_cell, block_id, std::move(cell), std::move(promise)); } +void CellDb::get_cell_db_reader(td::Promise> promise) { + td::actor::send_closure(cell_db_, &CellDbIn::get_cell_db_reader, std::move(promise)); +} + void CellDb::start_up() { boc_ = vm::DynamicBagOfCellsDb::create(); cell_db_ = td::actor::create_actor("celldbin", root_db_, actor_id(this), path_); diff --git a/validator/db/celldb.hpp b/validator/db/celldb.hpp index 691d8848..4eb1d6a8 100644 --- a/validator/db/celldb.hpp +++ b/validator/db/celldb.hpp @@ -40,6 +40,7 @@ class CellDbIn : public td::actor::Actor { void load_cell(RootHash hash, td::Promise> promise); void store_cell(BlockIdExt block_id, td::Ref cell, td::Promise> promise); + void get_cell_db_reader(td::Promise> promise); CellDbIn(td::actor::ActorId root_db, td::actor::ActorId parent, std::string path); @@ -96,6 +97,7 @@ class CellDb : public td::actor::Actor { started_ = true; boc_->set_loader(std::make_unique(std::move(snapshot))).ensure(); } + void get_cell_db_reader(td::Promise> promise); CellDb(td::actor::ActorId root_db, std::string path) : root_db_(root_db), path_(path) { } diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index 58e3dd2f..a7a1becf 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -270,6 +270,10 @@ void RootDb::get_block_state(ConstBlockHandle handle, td::Promise> promise) { + td::actor::send_closure(cell_db_, &CellDb::get_cell_db_reader, std::move(promise)); +} + void RootDb::store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, td::Promise promise) { td::actor::send_closure(archive_db_, &ArchiveManager::add_persistent_state, block_id, masterchain_block_id, diff --git a/validator/db/rootdb.hpp b/validator/db/rootdb.hpp index 2654d482..9b0d52a6 100644 --- a/validator/db/rootdb.hpp +++ b/validator/db/rootdb.hpp @@ -60,6 +60,7 @@ class RootDb : public Db { void store_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) override; void get_block_state(ConstBlockHandle handle, td::Promise> promise) override; + void get_cell_db_reader(td::Promise> promise) override; void store_block_handle(BlockHandle handle, td::Promise promise) override; void get_block_handle(BlockIdExt id, td::Promise promise) override; diff --git a/validator/interfaces/db.h b/validator/interfaces/db.h index 9983572b..ba4d9dda 100644 --- a/validator/interfaces/db.h +++ b/validator/interfaces/db.h @@ -50,6 +50,7 @@ class Db : public td::actor::Actor { virtual void store_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) = 0; virtual void get_block_state(ConstBlockHandle handle, td::Promise> promise) = 0; + virtual void get_cell_db_reader(td::Promise> promise) = 0; virtual void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, td::Promise promise) = 0; diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 42e99ecc..77f728eb 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -27,6 +27,7 @@ #include "message-queue.h" #include "validator/validator.h" #include "liteserver.h" +#include "crypto/vm/db/DynamicBagOfCellsDb.h" namespace ton { @@ -55,6 +56,7 @@ class ValidatorManager : public ValidatorManagerInterface { public: virtual void set_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) = 0; + virtual void get_cell_db_reader(td::Promise> promise) = 0; virtual void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, td::Promise promise) = 0; virtual void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index 404f7fe8..46ad50fa 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -674,6 +674,10 @@ void ValidatorManagerImpl::set_block_state(BlockHandle handle, td::Ref> promise) { + td::actor::send_closure(db_, &Db::get_cell_db_reader, std::move(promise)); +} + void ValidatorManagerImpl::store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, td::Promise promise) { td::actor::send_closure(db_, &Db::store_persistent_state_file, block_id, masterchain_block_id, std::move(state), diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index bb740b4c..59475723 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -141,6 +141,7 @@ class ValidatorManagerImpl : public ValidatorManager { void set_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) override; + void get_cell_db_reader(td::Promise> promise) override; void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, td::Promise promise) override; void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, diff --git a/validator/manager-hardfork.cpp b/validator/manager-hardfork.cpp index 7b6ab8e9..d9718fa7 100644 --- a/validator/manager-hardfork.cpp +++ b/validator/manager-hardfork.cpp @@ -530,6 +530,10 @@ void ValidatorManagerImpl::get_block_handle(BlockIdExt id, bool force, td::Promi td::actor::send_closure(db_, &Db::get_block_handle, id, std::move(P)); } +void ValidatorManagerImpl::get_cell_db_reader(td::Promise> promise) { + td::actor::send_closure(db_, &Db::get_cell_db_reader, std::move(promise)); +} + void ValidatorManagerImpl::register_block_handle(BlockHandle handle, td::Promise promise) { auto it = handles_.find(handle->id()); if (it != handles_.end()) { diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 71157dcc..98539f7e 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -165,6 +165,7 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override { UNREACHABLE(); } + void get_cell_db_reader(td::Promise> promise) override; void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, td::Promise promise) override { UNREACHABLE(); diff --git a/validator/manager.cpp b/validator/manager.cpp index 28e5cd3a..c33d23c2 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1045,6 +1045,10 @@ void ValidatorManagerImpl::set_block_state(BlockHandle handle, td::Ref> promise) { + td::actor::send_closure(db_, &Db::get_cell_db_reader, std::move(promise)); +} + void ValidatorManagerImpl::store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, td::Promise promise) { td::actor::send_closure(db_, &Db::store_persistent_state_file, block_id, masterchain_block_id, std::move(state), diff --git a/validator/manager.hpp b/validator/manager.hpp index 123625b2..e053fa94 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -345,6 +345,7 @@ class ValidatorManagerImpl : public ValidatorManager { void set_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) override; + void get_cell_db_reader(td::Promise> promise) override; void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, td::Promise promise) override; void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, diff --git a/validator/state-serializer.cpp b/validator/state-serializer.cpp index 82cf31cc..d5d2f000 100644 --- a/validator/state-serializer.cpp +++ b/validator/state-serializer.cpp @@ -82,24 +82,24 @@ void AsyncStateSerializer::alarm() { } void AsyncStateSerializer::request_masterchain_state() { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &AsyncStateSerializer::fail_handler, - R.move_as_error_prefix("failed to get masterchain state: ")); - } else { - td::actor::send_closure(SelfId, &AsyncStateSerializer::got_masterchain_state, - td::Ref(R.move_as_ok())); - } - }); - td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, masterchain_handle_, std::move(P)); + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &AsyncStateSerializer::fail_handler, + R.move_as_error_prefix("failed to get masterchain state: ")); + } else { + td::actor::send_closure(SelfId, &AsyncStateSerializer::got_masterchain_state, + td::Ref(R.move_as_ok())); + } + }); + td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, masterchain_handle_, std::move(P)); } void AsyncStateSerializer::request_shard_state(BlockIdExt shard) { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - R.ensure(); - td::actor::send_closure(SelfId, &AsyncStateSerializer::got_shard_handle, R.move_as_ok()); - }); - return td::actor::send_closure(manager_, &ValidatorManager::get_block_handle, shard, true, std::move(P)); + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + R.ensure(); + td::actor::send_closure(SelfId, &AsyncStateSerializer::got_shard_handle, R.move_as_ok()); + }); + return td::actor::send_closure(manager_, &ValidatorManager::get_block_handle, shard, true, std::move(P)); } void AsyncStateSerializer::next_iteration() { @@ -122,8 +122,21 @@ void AsyncStateSerializer::next_iteration() { CHECK(masterchain_handle_->id() == last_block_id_); if (attempt_ < max_attempt() && last_key_block_id_.id.seqno < last_block_id_.id.seqno && need_serialize(masterchain_handle_)) { + if (!cell_db_reader_) { + running_ = true; + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &AsyncStateSerializer::fail_handler, + R.move_as_error_prefix("failed to get cell db reader: ")); + } else { + td::actor::send_closure(SelfId, &AsyncStateSerializer::got_cell_db_reader, R.move_as_ok()); + } + }); + td::actor::send_closure(manager_, &ValidatorManager::get_cell_db_reader, std::move(P)); + return; + } if (!have_masterchain_state_) { - LOG(INFO) << "started serializing persistent state for " << masterchain_handle_->id().seqno(); + LOG(INFO) << "started serializing persistent state for " << masterchain_handle_->id().id; // block next attempts immediately, but send actual request later running_ = true; delay_action( @@ -143,9 +156,10 @@ void AsyncStateSerializer::next_iteration() { return; } } - LOG(INFO) << "finished serializing persistent state for " << masterchain_handle_->id().seqno(); + LOG(INFO) << "finished serializing persistent state for " << masterchain_handle_->id().id; last_key_block_ts_ = masterchain_handle_->unix_time(); last_key_block_id_ = masterchain_handle_->id(); + cell_db_reader_ = nullptr; } if (!saved_to_db_) { running_ = true; @@ -175,6 +189,13 @@ void AsyncStateSerializer::got_top_masterchain_handle(BlockIdExt block_id) { } } +void AsyncStateSerializer::got_cell_db_reader(std::shared_ptr cell_db_reader) { + cell_db_reader_ = std::move(cell_db_reader); + running_ = false; + attempt_ = 0; + next_iteration(); +} + void AsyncStateSerializer::got_masterchain_handle(BlockHandle handle) { CHECK(!masterchain_handle_); masterchain_handle_ = std::move(handle); @@ -184,7 +205,7 @@ void AsyncStateSerializer::got_masterchain_handle(BlockHandle handle) { } void AsyncStateSerializer::got_masterchain_state(td::Ref state) { - LOG(INFO) << "serializing masterchain state " << masterchain_handle_->id().seqno(); + LOG(INFO) << "serializing masterchain state " << masterchain_handle_->id().id; have_masterchain_state_ = true; CHECK(next_idx_ == 0); CHECK(shards_.size() == 0); @@ -194,8 +215,8 @@ void AsyncStateSerializer::got_masterchain_state(td::Ref state shards_.push_back(v->top_block_id()); } - auto write_data = [state] (td::FileFd& fd) { - return state->serialize_to_file(fd); + auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader = cell_db_reader_] (td::FileFd& fd) { + return vm::std_boc_serialize_to_file_large(cell_db_reader, hash, fd, 31); }; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { R.ensure(); @@ -207,7 +228,7 @@ void AsyncStateSerializer::got_masterchain_state(td::Ref state } void AsyncStateSerializer::stored_masterchain_state() { - LOG(INFO) << "finished serializing masterchain state " << masterchain_handle_->id().seqno(); + LOG(INFO) << "finished serializing masterchain state " << masterchain_handle_->id().id; running_ = false; next_iteration(); } @@ -225,13 +246,13 @@ void AsyncStateSerializer::got_shard_handle(BlockHandle handle) { } void AsyncStateSerializer::got_shard_state(BlockHandle handle, td::Ref state) { - LOG(INFO) << "serializing shard state " << handle->id().seqno(); - auto write_data = [state] (td::FileFd& fd) { - return state->serialize_to_file(fd); + LOG(INFO) << "serializing shard state " << handle->id().id; + auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader = cell_db_reader_] (td::FileFd& fd) { + return vm::std_boc_serialize_to_file_large(cell_db_reader, hash, fd, 31); }; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result R) { R.ensure(); - LOG(INFO) << "finished serializing shard state " << handle->id().seqno(); + LOG(INFO) << "finished serializing shard state " << handle->id().id; td::actor::send_closure(SelfId, &AsyncStateSerializer::success_handler); }); td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file_gen, handle->id(), diff --git a/validator/state-serializer.hpp b/validator/state-serializer.hpp index 14261df7..e971a3f8 100644 --- a/validator/state-serializer.hpp +++ b/validator/state-serializer.hpp @@ -42,6 +42,7 @@ class AsyncStateSerializer : public td::actor::Actor { td::uint32 next_idx_ = 0; + std::shared_ptr cell_db_reader_ = nullptr; BlockHandle masterchain_handle_; bool have_masterchain_state_ = false; @@ -70,6 +71,7 @@ class AsyncStateSerializer : public td::actor::Actor { void next_iteration(); void got_top_masterchain_handle(BlockIdExt block_id); + void got_cell_db_reader(std::shared_ptr cell_db_reader); void got_masterchain_handle(BlockHandle handle_); void got_masterchain_state(td::Ref state); void stored_masterchain_state(); From ca00f0ed917f30cf6db42ac7dfd5022244584ebb Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Tue, 24 May 2022 21:21:58 +0300 Subject: [PATCH 02/16] Extended error notes for unacceptable external messages (#383) * patch sendMessage * Verbose error message for "sendMessage" LS query * More verbose errors in external-message.cpp Co-authored-by: dungeon-master-666 Co-authored-by: SpyCheese --- crypto/block/transaction.cpp | 47 ++++++++++++++++++++++++++++- crypto/block/transaction.h | 2 ++ validator/impl/collator.cpp | 7 ++++- validator/impl/external-message.cpp | 35 +++++++++++---------- validator/impl/external-message.hpp | 10 +++--- validator/impl/liteserver.cpp | 14 +++++---- 6 files changed, 86 insertions(+), 29 deletions(-) diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index e5cd701d..f5fb18f1 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -25,6 +25,41 @@ #include "ton/ton-shard.h" #include "vm/vm.h" +namespace { +class StringLoggerTail : public td::LogInterface { + public: + explicit StringLoggerTail(size_t max_size = 256) : buf(max_size, '\0') {} + void append(td::CSlice slice) override { + if (slice.size() > buf.size()) { + slice.remove_prefix(slice.size() - buf.size()); + } + while (!slice.empty()) { + size_t s = std::min(buf.size() - pos, slice.size()); + std::copy(slice.begin(), slice.begin() + s, buf.begin() + pos); + pos += s; + if (pos == buf.size()) { + pos = 0; + truncated = true; + } + slice.remove_prefix(s); + } + } + std::string get_log() const { + if (truncated) { + std::string res = buf; + std::rotate(res.begin(), res.begin() + pos, res.end()); + return res; + } else { + return buf.substr(0, pos); + } + } + private: + std::string buf; + size_t pos = 0; + bool truncated = false; +}; +} + namespace block { using td::Ref; @@ -1001,7 +1036,14 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) { vm::GasLimits gas{(long long)cp.gas_limit, (long long)cp.gas_max, (long long)cp.gas_credit}; LOG(DEBUG) << "creating VM"; - vm::VmState vm{new_code, std::move(stack), gas, 1, new_data, vm::VmLog(), compute_vm_libraries(cfg)}; + std::unique_ptr logger; + auto vm_log = vm::VmLog(); + if (cfg.with_vm_log) { + logger = std::make_unique(); + vm_log.log_interface = logger.get(); + vm_log.log_options = td::LogOptions(VERBOSITY_NAME(DEBUG), true, false); + } + vm::VmState vm{new_code, std::move(stack), gas, 1, new_data, vm_log, compute_vm_libraries(cfg)}; vm.set_c7(prepare_vm_c7(cfg)); // tuple with SmartContractInfo // vm.incr_stack_trace(1); // enable stack dump after each step @@ -1024,6 +1066,9 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) { LOG(INFO) << "steps: " << vm.get_steps_count() << " gas: used=" << gas.gas_consumed() << ", max=" << gas.gas_max << ", limit=" << gas.gas_limit << ", credit=" << gas.gas_credit; LOG(INFO) << "out_of_gas=" << cp.out_of_gas << ", accepted=" << cp.accepted << ", success=" << cp.success; + if (logger != nullptr) { + cp.vm_log = logger->get_log(); + } if (cp.success) { cp.new_data = vm.get_committed_state().c4; // c4 -> persistent data cp.actions = vm.get_committed_state().c5; // c5 -> action list diff --git a/crypto/block/transaction.h b/crypto/block/transaction.h index 398e2731..75762260 100644 --- a/crypto/block/transaction.h +++ b/crypto/block/transaction.h @@ -106,6 +106,7 @@ struct ComputePhaseConfig { std::unique_ptr libraries; Ref global_config; td::BitArray<256> block_rand_seed; + bool with_vm_log{false}; ComputePhaseConfig(td::uint64 _gas_price = 0, td::uint64 _gas_limit = 0, td::uint64 _gas_credit = 0) : gas_price(_gas_price), gas_limit(_gas_limit), special_gas_limit(_gas_limit), gas_credit(_gas_credit) { compute_threshold(); @@ -171,6 +172,7 @@ struct ComputePhase { Ref in_msg; Ref new_data; Ref actions; + std::string vm_log; }; struct ActionPhase { diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 5820a2ce..798e6564 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -2326,7 +2326,12 @@ td::Result> Collator::impl_create_ordinary_t if (!trans->compute_phase->accepted) { if (external) { // inbound external message was not accepted - return td::Status::Error(-701,"inbound external message rejected by transaction "s + acc->addr.to_hex()); + auto const& cp = *trans->compute_phase; + return td::Status::Error( + -701, + PSLICE() << "inbound external message rejected by transaction " << acc->addr.to_hex() << ":\n" << + "exitcode=" << cp.exit_code << ", steps=" << cp.vm_steps << ", gas_used=" << cp.gas_used << + (cp.vm_log.empty() ? "" : "\nVM Log (truncated):\n..." + cp.vm_log)); } else if (trans->compute_phase->skip_reason == block::ComputePhase::sk_none) { return td::Status::Error(-669,"new ordinary transaction for smart contract "s + acc->addr.to_hex() + " has not been accepted by the smart contract (?)"); diff --git a/validator/impl/external-message.cpp b/validator/impl/external-message.cpp index c5f2d097..755370ca 100644 --- a/validator/impl/external-message.cpp +++ b/validator/impl/external-message.cpp @@ -112,10 +112,12 @@ void ExtMessageQ::run_message(td::BufferSlice data, td::actor::ActorId msg_root, - std::unique_ptr config) { +td::Status ExtMessageQ::run_message_on_account(ton::WorkchainId wc, + block::Account* acc, + UnixTime utime, LogicalTime lt, + td::Ref msg_root, + std::unique_ptr config) { Ref old_mparams; std::vector storage_prices_; @@ -143,28 +145,29 @@ bool ExtMessageQ::run_message_on_account(ton::WorkchainId wc, &action_phase_cfg_, &masterchain_create_fee, &basechain_create_fee, wc); if(fetch_res.is_error()) { - auto error = fetch_res.move_as_error(); - LOG(DEBUG) << "Cannot fetch config params" << error.message(); - return false; + auto error = fetch_res.move_as_error(); + LOG(DEBUG) << "Cannot fetch config params: " << error.message(); + return error.move_as_error_prefix("Cannot fetch config params: "); } + compute_phase_cfg_.with_vm_log = true; auto res = Collator::impl_create_ordinary_transaction(msg_root, acc, utime, lt, &storage_phase_cfg_, &compute_phase_cfg_, &action_phase_cfg_, true, lt); if(res.is_error()) { - auto error = res.move_as_error(); - LOG(DEBUG) << "Cannot run message on account" << error.message(); - return false; + auto error = res.move_as_error(); + LOG(DEBUG) << "Cannot run message on account: " << error.message(); + return error.move_as_error_prefix("Cannot run message on account: "); } std::unique_ptr trans = res.move_as_ok(); auto trans_root = trans->commit(*acc); if (trans_root.is_null()) { - LOG(DEBUG) << "cannot commit new transaction for smart contract "; - return false; + LOG(DEBUG) << "Cannot commit new transaction for smart contract"; + return td::Status::Error("Cannot commit new transaction for smart contract"); } - return true; + return td::Status::OK(); } } // namespace validator diff --git a/validator/impl/external-message.hpp b/validator/impl/external-message.hpp index 230258cd..1d1ec640 100644 --- a/validator/impl/external-message.hpp +++ b/validator/impl/external-message.hpp @@ -63,11 +63,11 @@ class ExtMessageQ : public ExtMessage { static td::Result> create_ext_message(td::BufferSlice data); static void run_message(td::BufferSlice data, td::actor::ActorId manager, td::Promise promise); - static bool run_message_on_account(ton::WorkchainId wc, - block::Account* acc, - UnixTime utime, LogicalTime lt, - td::Ref msg_root, - std::unique_ptr config); + static td::Status run_message_on_account(ton::WorkchainId wc, + block::Account* acc, + UnixTime utime, LogicalTime lt, + td::Ref msg_root, + std::unique_ptr config); }; } // namespace validator diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index 90d974e0..023a84ae 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -458,18 +458,20 @@ void LiteQuery::continue_getZeroState(BlockIdExt blkid, td::BufferSlice state) { void LiteQuery::perform_sendMessage(td::BufferSlice data) { LOG(INFO) << "started a sendMessage(<" << data.size() << " bytes>) liteserver query"; - td::actor::send_closure_later( - manager_, &ValidatorManager::check_external_message, data.clone(), - [Self = actor_id(this), data = std::move(data), manager = manager_](td::Result res) { + auto copy = data.clone(); + td::actor::send_closure_later( + manager_, &ValidatorManager::check_external_message, std::move(copy), + [Self = actor_id(this), data = std::move(data), manager = manager_](td::Result res) mutable { if(res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, - res.move_as_error_prefix("cannot apply external message to current state : "s)); + res.move_as_error_prefix("cannot apply external message to current state : "s)); } else { - auto crm = ton::validator::create_ext_message(data.clone()); + auto crm = ton::validator::create_ext_message(std::move(data)); if (crm.is_error()) { //UNREACHABLE, checks in check_external_message, td::actor::send_closure(Self, &LiteQuery::abort_query, - crm.move_as_error()); + crm.move_as_error()); + return; } LOG(INFO) << "sending an external message to validator manager"; td::actor::send_closure_later(manager, &ValidatorManager::send_external_message, crm.move_as_ok()); From 845cbca1e5328e2656e12940f2a8c2323e112d2c Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Wed, 1 Jun 2022 10:03:50 +0300 Subject: [PATCH 03/16] Make asynchronous celldb interface (#388) * Asynchronous load_cell in celldb Co-authored-by: SpyCheese --- crypto/vm/db/CellHashTable.h | 7 +++ crypto/vm/db/DynamicBagOfCellsDb.cpp | 82 ++++++++++++++++++++++++++++ crypto/vm/db/DynamicBagOfCellsDb.h | 11 ++++ validator/db/celldb.cpp | 54 +++++++++++++++--- validator/db/celldb.hpp | 15 ++++- 5 files changed, 159 insertions(+), 10 deletions(-) diff --git a/crypto/vm/db/CellHashTable.h b/crypto/vm/db/CellHashTable.h index acb3b3fc..7d0308b7 100644 --- a/crypto/vm/db/CellHashTable.h +++ b/crypto/vm/db/CellHashTable.h @@ -64,6 +64,13 @@ class CellHashTable { size_t size() const { return set_.size(); } + InfoT* get_if_exists(td::Slice hash) { + auto it = set_.find(hash); + if (it != set_.end()) { + return &const_cast(*it); + } + return nullptr; + } private: std::set> set_; diff --git a/crypto/vm/db/DynamicBagOfCellsDb.cpp b/crypto/vm/db/DynamicBagOfCellsDb.cpp index 636be9c9..4ff4ec30 100644 --- a/crypto/vm/db/DynamicBagOfCellsDb.cpp +++ b/crypto/vm/db/DynamicBagOfCellsDb.cpp @@ -86,6 +86,40 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat TRY_RESULT(loaded_cell, get_cell_info_force(hash).cell->load_cell()); return std::move(loaded_cell.data_cell); } + void load_cell_async(td::Slice hash, std::shared_ptr executor, + td::Promise> promise) override { + auto info = hash_table_.get_if_exists(hash); + if (info && info->sync_with_db) { + TRY_RESULT_PROMISE(promise, loaded_cell, info->cell->load_cell()); + promise.set_result(loaded_cell.data_cell); + return; + } + SimpleExtCellCreator ext_cell_creator(cell_db_reader_); + auto promise_ptr = std::make_shared>>(std::move(promise)); + executor->execute_async( + [executor, loader = *loader_, hash = CellHash::from_slice(hash), db = this, + ext_cell_creator = std::move(ext_cell_creator), promise = std::move(promise_ptr)]() mutable { + TRY_RESULT_PROMISE((*promise), res, loader.load(hash.as_slice(), true, ext_cell_creator)); + if (res.status != CellLoader::LoadResult::Ok) { + promise->set_error(td::Status::Error("cell not found")); + return; + } + Ref cell = res.cell(); + executor->execute_sync([hash, db, res = std::move(res), + ext_cell_creator = std::move(ext_cell_creator)]() mutable { + db->hash_table_.apply(hash.as_slice(), [&](CellInfo &info) { + db->update_cell_info_loaded(info, hash.as_slice(), std::move(res)); + }); + for (auto &ext_cell : ext_cell_creator.get_created_cells()) { + auto ext_cell_hash = ext_cell->get_hash(); + db->hash_table_.apply(ext_cell_hash.as_slice(), [&](CellInfo &info) { + db->update_cell_info_created_ext(info, std::move(ext_cell)); + }); + } + }); + promise->set_result(std::move(cell)); + }); + } CellInfo &get_cell_info_force(td::Slice hash) { return hash_table_.apply(hash, [&](CellInfo &info) { update_cell_info_force(info, hash); }); } @@ -198,6 +232,27 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat return res; } + class SimpleExtCellCreator : public ExtCellCreator { + public: + explicit SimpleExtCellCreator(std::shared_ptr cell_db_reader) : + cell_db_reader_(std::move(cell_db_reader)) {} + + td::Result> ext_cell(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) override { + TRY_RESULT(ext_cell, DynamicBocExtCell::create(PrunnedCellInfo{level_mask, hash, depth}, + DynamicBocExtCellExtra{cell_db_reader_})); + created_cells_.push_back(ext_cell); + return std::move(ext_cell); + } + + std::vector>& get_created_cells() { + return created_cells_; + } + + private: + std::vector> created_cells_; + std::shared_ptr cell_db_reader_; + }; + class CellDbReaderImpl : public CellDbReader, private ExtCellCreator, public std::enable_shared_from_this { @@ -484,6 +539,33 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat info.sync_with_db = true; } + // same as update_cell_info_force, but with cell provided by a caller + void update_cell_info_loaded(CellInfo &info, td::Slice hash, CellLoader::LoadResult res) { + if (info.sync_with_db) { + return; + } + DCHECK(res.status == CellLoader::LoadResult::Ok); + info.cell = std::move(res.cell()); + CHECK(info.cell->get_hash().as_slice() == hash); + info.in_db = true; + info.db_refcnt = res.refcnt(); + info.sync_with_db = true; + } + + // same as update_cell_info_lazy, but with cell provided by a caller + void update_cell_info_created_ext(CellInfo &info, Ref cell) { + if (info.sync_with_db) { + CHECK(info.cell.not_null()); + CHECK(info.cell->get_level_mask() == cell->get_level_mask()); + CHECK(info.cell->get_hash() == cell->get_hash()); + return; + } + if (info.cell.is_null()) { + info.cell = std::move(cell); + info.in_db = true; + } + } + td::Result> create_empty_ext_cell(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) { TRY_RESULT(res, DynamicBocExtCell::create(PrunnedCellInfo{level_mask, hash, depth}, DynamicBocExtCellExtra{cell_db_reader_})); diff --git a/crypto/vm/db/DynamicBagOfCellsDb.h b/crypto/vm/db/DynamicBagOfCellsDb.h index 9a87c619..3569208c 100644 --- a/crypto/vm/db/DynamicBagOfCellsDb.h +++ b/crypto/vm/db/DynamicBagOfCellsDb.h @@ -21,6 +21,7 @@ #include "td/utils/Slice.h" #include "td/utils/Status.h" +#include "td/actor/PromiseFuture.h" namespace vm { class CellLoader; @@ -64,6 +65,16 @@ class DynamicBagOfCellsDb { virtual td::Status set_loader(std::unique_ptr loader) = 0; static std::unique_ptr create(); + + class AsyncExecutor { + public: + virtual ~AsyncExecutor() {} + virtual void execute_async(std::function f) = 0; + virtual void execute_sync(std::function f) = 0; + }; + + virtual void load_cell_async(td::Slice hash, std::shared_ptr executor, + td::Promise> promise) = 0; }; } // namespace vm diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index e7458407..caefe8e4 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -28,11 +28,46 @@ namespace ton { namespace validator { +class CellDbAsyncExecutor : public vm::DynamicBagOfCellsDb::AsyncExecutor { + public: + explicit CellDbAsyncExecutor(td::actor::ActorId cell_db) : cell_db_(std::move(cell_db)) { + } + + void execute_async(std::function f) { + class Runner : public td::actor::Actor { + public: + explicit Runner(std::function f) : f_(std::move(f)) {} + void start_up() { + f_(); + stop(); + } + private: + std::function f_; + }; + td::actor::create_actor("executeasync", std::move(f)).release(); + } + + void execute_sync(std::function f) { + td::actor::send_closure(cell_db_, &CellDbBase::execute_sync, std::move(f)); + } + private: + td::actor::ActorId cell_db_; +}; + +void CellDbBase::start_up() { + async_executor = std::make_shared(actor_id(this)); +} + +void CellDbBase::execute_sync(std::function f) { + f(); +} + CellDbIn::CellDbIn(td::actor::ActorId root_db, td::actor::ActorId parent, std::string path) : root_db_(root_db), parent_(parent), path_(std::move(path)) { } void CellDbIn::start_up() { + CellDbBase::start_up(); cell_db_ = std::make_shared(td::RocksDb::open(path_).move_as_ok()); boc_ = vm::DynamicBagOfCellsDb::create(); @@ -52,8 +87,7 @@ void CellDbIn::start_up() { } void CellDbIn::load_cell(RootHash hash, td::Promise> promise) { - td::PerfWarningTimer{"loadcell", 0.1}; - promise.set_result(boc_->load_cell(hash.as_slice())); + boc_->load_cell_async(hash.as_slice(), async_executor, std::move(promise)); } void CellDbIn::store_cell(BlockIdExt block_id, td::Ref cell, td::Promise> promise) { @@ -255,12 +289,15 @@ void CellDb::load_cell(RootHash hash, td::Promise> promise if (!started_) { td::actor::send_closure(cell_db_, &CellDbIn::load_cell, hash, std::move(promise)); } else { - auto R = boc_->load_cell(hash.as_slice()); - if (R.is_error()) { - td::actor::send_closure(cell_db_, &CellDbIn::load_cell, hash, std::move(promise)); - } else { - promise.set_result(R.move_as_ok()); - } + auto P = td::PromiseCreator::lambda( + [cell_db_in = cell_db_.get(), hash, promise = std::move(promise)](td::Result> R) mutable { + if (R.is_error()) { + td::actor::send_closure(cell_db_in, &CellDbIn::load_cell, hash, std::move(promise)); + } else { + promise.set_result(R.move_as_ok()); + } + }); + boc_->load_cell_async(hash.as_slice(), async_executor, std::move(P)); } } @@ -273,6 +310,7 @@ void CellDb::get_cell_db_reader(td::Promise> p } void CellDb::start_up() { + CellDbBase::start_up(); boc_ = vm::DynamicBagOfCellsDb::create(); cell_db_ = td::actor::create_actor("celldbin", root_db_, actor_id(this), path_); } diff --git a/validator/db/celldb.hpp b/validator/db/celldb.hpp index 4eb1d6a8..e54526b9 100644 --- a/validator/db/celldb.hpp +++ b/validator/db/celldb.hpp @@ -33,8 +33,19 @@ namespace validator { class RootDb; class CellDb; +class CellDbAsyncExecutor; -class CellDbIn : public td::actor::Actor { +class CellDbBase : public td::actor::Actor { + public: + virtual void start_up(); + protected: + std::shared_ptr async_executor; + private: + void execute_sync(std::function f); + friend CellDbAsyncExecutor; +}; + +class CellDbIn : public CellDbBase { public: using KeyHash = td::Bits256; @@ -89,7 +100,7 @@ class CellDbIn : public td::actor::Actor { KeyHash last_gc_; }; -class CellDb : public td::actor::Actor { +class CellDb : public CellDbBase { public: void load_cell(RootHash hash, td::Promise> promise); void store_cell(BlockIdExt block_id, td::Ref cell, td::Promise> promise); From 65e3177def2fa2b637d791cd3441f9ad5ad98e8b Mon Sep 17 00:00:00 2001 From: neodiX42 Date: Fri, 3 Jun 2022 15:01:55 +0200 Subject: [PATCH 04/16] Fix windows validator-engine.exe runtime access violation error (#387) * Fix runtime access violation error on Windows --- .github/workflows/docker-ubuntu-image.yml | 22 +++--- .github/workflows/macos-10.15-compile.yml | 8 ++- .github/workflows/macos-10.15-tonlib-java.yml | 60 ++++++++++++++++ .github/workflows/ubuntu-18.04-compile.yml | 31 ++++---- .../workflows/ubuntu-18.04-ton-ccpcheck.yml | 31 ++++++++ .../workflows/ubuntu-18.04-tonlib-java.yml | 50 +++++++++++++ .github/workflows/windows2019x64-compile.yml | 11 ++- .../workflows/windows2019x64-tonlib-java.yml | 71 +++++++++++++++++++ doc/tblkch.tex | 2 +- doc/tvm.tex | 2 +- validator/impl/collator.cpp | 10 +-- validator/impl/validate-query.cpp | 18 ++--- 12 files changed, 262 insertions(+), 54 deletions(-) create mode 100644 .github/workflows/macos-10.15-tonlib-java.yml create mode 100644 .github/workflows/ubuntu-18.04-ton-ccpcheck.yml create mode 100644 .github/workflows/ubuntu-18.04-tonlib-java.yml create mode 100644 .github/workflows/windows2019x64-tonlib-java.yml diff --git a/.github/workflows/docker-ubuntu-image.yml b/.github/workflows/docker-ubuntu-image.yml index 4fd9d1da..2f0d49ee 100644 --- a/.github/workflows/docker-ubuntu-image.yml +++ b/.github/workflows/docker-ubuntu-image.yml @@ -1,4 +1,4 @@ -name: Build Docker Ubuntu Image +name: Docker Ubuntu 18.04 image on: workflow_dispatch: @@ -12,34 +12,28 @@ env: jobs: build-and-push: - runs-on: ubuntu-latest + runs-on: ubuntu-18.04 steps: - - - name: Checkout + - name: Checkout uses: actions/checkout@v2 - - - name: Set up QEMU + - name: Set up QEMU uses: docker/setup-qemu-action@v1 - - - name: Set up Docker Buildx + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 - - - name: Login to GitHub Container Registry + - name: Login to GitHub Container Registry uses: docker/login-action@v1 with: registry: ${{ env.REGISTRY }} username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push + - name: Build and push id: docker_build uses: docker/build-push-action@v2 with: push: true context: ./docker - tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest - + tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest \ No newline at end of file diff --git a/.github/workflows/macos-10.15-compile.yml b/.github/workflows/macos-10.15-compile.yml index 1aa36a9f..351b3ea0 100644 --- a/.github/workflows/macos-10.15-compile.yml +++ b/.github/workflows/macos-10.15-compile.yml @@ -1,6 +1,10 @@ name: C/C++ CI macOS-10.15 Compile -on: [push,workflow_dispatch] +on: + workflow_dispatch: + push: + branches: + - 'master' jobs: build: @@ -36,4 +40,4 @@ jobs: uses: actions/upload-artifact@master with: name: ton-macos-binaries - path: artifacts + path: artifacts \ No newline at end of file diff --git a/.github/workflows/macos-10.15-tonlib-java.yml b/.github/workflows/macos-10.15-tonlib-java.yml new file mode 100644 index 00000000..e5199f58 --- /dev/null +++ b/.github/workflows/macos-10.15-tonlib-java.yml @@ -0,0 +1,60 @@ +name: macOS-10.15 tonlib-java + + +on: + workflow_dispatch: + push: + branches: + - 'wallets' +jobs: + build: + + runs-on: macos-10.15 + + steps: + - name: Check out repository + uses: actions/checkout@v2 + with: + submodules: 'recursive' + + - name: Compile OpenSSL + run: | + git clone https://github.com/openssl/openssl openssl_1_1_1 + cd openssl_1_1_1 + git checkout OpenSSL_1_1_1-stable + ./Configure --prefix=/usr/local/macos darwin64-x86_64-cc -static -mmacosx-version-min=10.15 + make build_libs -j4 + + - name: Configure & Build + run: | + rootPath=`pwd` + + export CC=$(which clang) + export CXX=$(which clang++) + export CCACHE_DISABLE=1 + + export JAVA_AWT_LIBRARY=NotNeeded + export JAVA_JVM_LIBRARY=NotNeeded + export JAVA_INCLUDE_PATH=${JAVA_HOME}/include + export JAVA_AWT_INCLUDE_PATH=${JAVA_HOME}/include + export JAVA_INCLUDE_PATH2=${JAVA_HOME}/include/darwin + + cd example/android/ + mkdir build + cd build + + cmake -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$rootPath/openssl_1_1_1/include -DOPENSSL_CRYPTO_LIBRARY=$rootPath/openssl_1_1_1/libcrypto.a -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.15 -DCMAKE_CXX_FLAGS="-stdlib=libc++" -DTON_ONLY_TONLIB=ON .. + cmake --build . --target prepare_cross_compiling + cmake --build . --target native-lib --config Release + + - name: find & copy binaries + run: | + mkdir -p artifacts/tonlib-java + cp example/android/src/drinkless/org/ton/TonApi.java artifacts/tonlib-java/ + cp example/android/build/libnative-lib.dylib artifacts/tonlib-java/ + + - name: Upload artifacts + uses: actions/upload-artifact@master + with: + name: tonlib-macos-java + path: artifacts diff --git a/.github/workflows/ubuntu-18.04-compile.yml b/.github/workflows/ubuntu-18.04-compile.yml index f27185bb..e215c7bd 100644 --- a/.github/workflows/ubuntu-18.04-compile.yml +++ b/.github/workflows/ubuntu-18.04-compile.yml @@ -12,28 +12,21 @@ jobs: uses: actions/checkout@v2 with: submodules: 'recursive' - - name: Run Cppcheck - uses: Bedzior/run-cppcheck@master - with: - enabled checks: all - enable inconclusive: true - generate report: true - - name: Upload report - uses: actions/upload-artifact@v1 - with: - name: report - path: output - - name: mkdir + + - name: Install libraries run: | + sudo apt update + sudo apt install -y build-essential git make cmake clang libgflags-dev zlib1g-dev libssl-dev libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev ninja-build + + - name: Configure & Build + run: | + export CC=$(which clang) + export CXX=$(which clang++) + export CCACHE_DISABLE=1 mkdir build - - name: cmake all - run: | cd build - cmake -DCMAKE_BUILD_TYPE=Release .. - - name: make all - run: | - cd build - make -j4 fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork + cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. + ninja fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork - name: find & copy binaries run: | diff --git a/.github/workflows/ubuntu-18.04-ton-ccpcheck.yml b/.github/workflows/ubuntu-18.04-ton-ccpcheck.yml new file mode 100644 index 00000000..97c46efb --- /dev/null +++ b/.github/workflows/ubuntu-18.04-ton-ccpcheck.yml @@ -0,0 +1,31 @@ +name: Ubuntu 18.04 TON ccpcheck + +on: + workflow_dispatch: + push: + branches: + - 'master' + +jobs: + build: + + runs-on: ubuntu-18.04 + + steps: + - name: Check out repository + uses: actions/checkout@v2 + with: + submodules: 'recursive' + + - name: Run Cppcheck + uses: Bedzior/run-cppcheck@master + with: + enabled checks: all + enable inconclusive: true + generate report: true + + - name: Upload report + uses: actions/upload-artifact@v1 + with: + name: ton-ccpcheck-report + path: output \ No newline at end of file diff --git a/.github/workflows/ubuntu-18.04-tonlib-java.yml b/.github/workflows/ubuntu-18.04-tonlib-java.yml new file mode 100644 index 00000000..8c746b26 --- /dev/null +++ b/.github/workflows/ubuntu-18.04-tonlib-java.yml @@ -0,0 +1,50 @@ +name: Ubuntu 18.04 tonlib-java + +on: + workflow_dispatch: + push: + branches: + - 'wallets' + +jobs: + build: + + runs-on: ubuntu-18.04 + + steps: + - name: Check out repository + uses: actions/checkout@v2 + with: + submodules: 'recursive' + + - name: Install libraries + run: | + sudo apt update + sudo apt install -y build-essential git make cmake clang libgflags-dev zlib1g-dev libssl-dev libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev ninja-build + + - name: Configure & Build + run: | + export JAVA_AWT_LIBRARY=NotNeeded + export JAVA_JVM_LIBRARY=NotNeeded + export JAVA_INCLUDE_PATH=${JAVA_HOME}/include + export JAVA_AWT_INCLUDE_PATH=${JAVA_HOME}/include + export JAVA_INCLUDE_PATH2=${JAVA_HOME}/include/linux + + cd example/android/ + mkdir build + cd build + cmake -DTON_ONLY_TONLIB=ON .. + cmake --build . --target prepare_cross_compiling + cmake --build . --target native-lib + + - name: find & copy binaries + run: | + mkdir -p artifacts/tonlib-java + cp example/android/src/drinkless/org/ton/TonApi.java artifacts/tonlib-java/ + cp example/android/build/libnative-lib.so artifacts/tonlib-java/ + + - name: Upload artifacts + uses: actions/upload-artifact@master + with: + name: tonlib-ubuntu-java + path: artifacts \ No newline at end of file diff --git a/.github/workflows/windows2019x64-compile.yml b/.github/workflows/windows2019x64-compile.yml index 65c9a929..43303e65 100644 --- a/.github/workflows/windows2019x64-compile.yml +++ b/.github/workflows/windows2019x64-compile.yml @@ -1,6 +1,10 @@ name: C/C++ CI Windows Server 2019 x64 Compile -on: [push,workflow_dispatch] +on: + workflow_dispatch: + push: + branches: + - 'master' defaults: run: @@ -49,8 +53,8 @@ jobs: cmake -DZLIB_FOUND=1 -DZLIB_INCLUDE_DIR=%root%\zlib -DZLIB_LIBRARY=%root%\zlib\contrib\vstudio\vc14\x64\ZlibStatReleaseWithoutAsm\zlibstat.lib -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=%root%/openssl-1.1/x64/include -DOPENSSL_CRYPTO_LIBRARY=%root%/openssl-1.1/x64/lib/libcrypto.lib -DCMAKE_CXX_FLAGS="/DTD_WINDOWS=1 /EHsc /bigobj /W0" .. - name: Build - run: | - cd build + run: | + cd build cmake --build . --target fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork --config Release - name: Show executables @@ -69,6 +73,7 @@ jobs: run: | mkdir artifacts for /f %%a in ('dir *.exe /b /a /s') do copy /Y %%a artifacts + copy build\tonlib\Release\tonlibjson.dll artifacts - name: Upload artifacts uses: actions/upload-artifact@master diff --git a/.github/workflows/windows2019x64-tonlib-java.yml b/.github/workflows/windows2019x64-tonlib-java.yml new file mode 100644 index 00000000..f03212b8 --- /dev/null +++ b/.github/workflows/windows2019x64-tonlib-java.yml @@ -0,0 +1,71 @@ +name: Windows 2019 tonlib-java + +on: + workflow_dispatch: + push: + branches: + - 'wallets' + +defaults: + run: + shell: cmd + +jobs: + build: + + runs-on: windows-2019 + + steps: + - name: Get Current OS version + run: | + systeminfo | findstr /B /C:"OS Name" /C:"OS Version" + - name: Check out current repository + uses: actions/checkout@v2 + with: + submodules: 'recursive' + + - name: Check out zlib repository + uses: actions/checkout@v2 + with: + repository: desktop-app/zlib + path: zlib + + - name: Setup msbuild.exe + uses: microsoft/setup-msbuild@v1.0.2 + + - name: Compile zlib Win64 + run: | + cd zlib\contrib\vstudio\vc14 + msbuild zlibstat.vcxproj /p:Configuration=ReleaseWithoutAsm /p:platform=x64 -p:PlatformToolset=v142 + + - name: Install precompiled OpenSSL Win64 + run: | + curl -Lo openssl-1.1.1o.zip https://github.com/neodiX42/precompiled-openssl-win64/raw/main/openssl-1.1.1o.zip + jar xf openssl-1.1.1o.zip + + - name: Configure & Build + run: | + set JAVA_AWT_LIBRARY=NotNeeded + set JAVA_JVM_LIBRARY=NotNeeded + set JAVA_INCLUDE_PATH=${JAVA_HOME}/include + set JAVA_AWT_INCLUDE_PATH=${JAVA_HOME}/include + set JAVA_INCLUDE_PATH2=${JAVA_HOME}/include/win32 + + set root=%cd% + echo %root% + cd example/android + mkdir build + cd build + cmake -DZLIB_FOUND=1 -DZLIB_INCLUDE_DIR=%root%\zlib -DZLIB_LIBRARY=%root%\zlib\contrib\vstudio\vc14\x64\ZlibStatReleaseWithoutAsm\zlibstat.lib -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=%root%/openssl-1.1/x64/include -DOPENSSL_CRYPTO_LIBRARY=%root%/openssl-1.1/x64/lib/libcrypto.lib -DTON_ONLY_TONLIB=ON -DCMAKE_CXX_FLAGS="/DTD_WINDOWS=1 /EHsc /bigobj /W0" .. + cmake --build . --target native-lib --config Release + + - name: Find & copy binaries + run: | + mkdir tonlib-java + cp example/android/build/Release/native-lib.dll tonlib-java/ + + - name: Upload artifacts + uses: actions/upload-artifact@master + with: + name: tonlib-win64-java + path: tonlib-java diff --git a/doc/tblkch.tex b/doc/tblkch.tex index e8d7f44f..e7b3b195 100644 --- a/doc/tblkch.tex +++ b/doc/tblkch.tex @@ -865,7 +865,7 @@ bit$_ _:(## 1) = Bit; The serialization of source and destination addresses is defined by the following TL-B scheme: \begin{verbatim} addr_none$00 = MsgAddressExt; -addr_extern$01 len:(## 8) external_address:(len * Bit) +addr_extern$01 len:(## 9) external_address:(len * Bit) = MsgAddressExt; anycast_info$_ depth:(## 5) rewrite_pfx:(depth * Bit) = Anycast; addr_std$10 anycast:(Maybe Anycast) diff --git a/doc/tvm.tex b/doc/tvm.tex index 71aa7abc..f5067cc8 100644 --- a/doc/tvm.tex +++ b/doc/tvm.tex @@ -2277,7 +2277,7 @@ The ``global variables'' may be helpful in implementing some high-level smart-co The message and address manipulation primitives listed below serialize and deserialize values according to the following TL-B scheme (cf.~\ptref{sp:tlb.brief}): \begin{verbatim} addr_none$00 = MsgAddressExt; -addr_extern$01 len:(## 8) external_address:(bits len) +addr_extern$01 len:(## 9) external_address:(bits len) = MsgAddressExt; anycast_info$_ depth:(#<= 30) { depth >= 1 } rewrite_pfx:(bits depth) = Anycast; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 798e6564..6aedcc21 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -3031,9 +3031,9 @@ void Collator::register_new_msgs(block::Transaction& trans) { } /* - * + * * Generate (parts of) new state and block - * + * */ bool store_ext_blk_ref_to(vm::CellBuilder& cb, const ton::BlockIdExt& id_ext, ton::LogicalTime end_lt) { @@ -3166,7 +3166,7 @@ bool Collator::create_mc_state_extra() { " contains an invalid configuration in its data, IGNORING CHANGES"; ignore_cfg_changes = true; } else { - cfg0 = cfg_dict.lookup_ref(td::BitArray<32>(1 - 1)); + cfg0 = cfg_dict.lookup_ref(td::BitArray<32>{(long long) 0}); } bool changed_cfg = false; if (cfg0.not_null()) { @@ -4041,9 +4041,9 @@ void Collator::return_block_candidate(td::Result saved) { } /* - * + * * Collator register methods - * + * */ td::Result Collator::register_external_message_cell(Ref ext_msg, const ExtMessage::Hash& ext_hash) { diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 02a16f83..97a4bab9 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -150,9 +150,9 @@ void ValidateQuery::finish_query() { } /* - * + * * INITIAL PARSE & LOAD REQUIRED DATA - * + * */ void ValidateQuery::start_up() { @@ -982,9 +982,9 @@ bool ValidateQuery::check_this_shard_mc_info() { } /* - * + * * METHODS CALLED FROM try_validate() stage 0 - * + * */ bool ValidateQuery::compute_prev_state() { @@ -1844,9 +1844,9 @@ bool ValidateQuery::check_utime_lt() { } /* - * + * * METHODS CALLED FROM try_validate() stage 1 - * + * */ // almost the same as in Collator @@ -4956,7 +4956,7 @@ bool ValidateQuery::check_config_update(Ref old_conf_params, Ref< return reject_query("no important parameters have been changed, but the block is marked as a key block"); } vm::Dictionary dict1{ocfg_root, 32}; - auto param0 = dict1.lookup_ref(td::BitArray<32>{1 - 1}); + auto param0 = dict1.lookup_ref(td::BitArray<32>{(long long) 0}); if (param0.is_null()) { if (cfg_acc_changed) { return reject_query("new state of old configuration smart contract "s + old_cfg_addr.to_hex() + @@ -5431,10 +5431,10 @@ bool ValidateQuery::check_mc_block_extra() { } /* - * + * * MAIN VALIDATOR FUNCTION * (invokes other methods in a suitable order) - * + * */ bool ValidateQuery::try_validate() { From 31f96f91edf41e4d28de2b0e534672f340f2a6c0 Mon Sep 17 00:00:00 2001 From: CAnon Date: Mon, 6 Jun 2022 23:45:08 +0800 Subject: [PATCH 05/16] Add vscode to gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 4e9758de..54d9ffc7 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,5 @@ crypto/smartcont/auto/ test/regression-tests.cache/ *.swp **/*build*/ -.idea \ No newline at end of file +.idea +.vscode \ No newline at end of file From e30049930a7372a3c1d28a1e59956af8eb489439 Mon Sep 17 00:00:00 2001 From: Tsenilov Oleg Date: Tue, 7 Jun 2022 16:24:26 +0300 Subject: [PATCH 06/16] Enhance overlay stats output (#386) * Expand overlay stats * Add scope and peer broadcast errors to stats * Add json output format Co-authored-by: EmelyanenkoK --- adnl/adnl-address-list.cpp | 6 ++ adnl/adnl-network-manager.h | 2 + adnl/adnl-peer-table.cpp | 9 ++ adnl/adnl-peer-table.h | 1 + adnl/adnl-peer-table.hpp | 1 + adnl/adnl-peer.cpp | 32 ++++++ adnl/adnl-peer.h | 2 + adnl/adnl-peer.hpp | 3 + adnl/adnl.h | 1 + overlay/overlay-broadcast.cpp | 7 +- overlay/overlay-broadcast.hpp | 9 +- overlay/overlay-fec-broadcast.cpp | 14 ++- overlay/overlay-fec-broadcast.hpp | 14 ++- overlay/overlay-manager.cpp | 27 ++++- overlay/overlay-manager.h | 2 +- overlay/overlay.cpp | 76 ++++++++++++-- overlay/overlay.h | 5 +- overlay/overlay.hpp | 63 +++++++++++- overlay/overlays.h | 2 +- rldp/rldp-in.hpp | 1 + rldp/rldp.cpp | 4 + rldp2/rldp-in.hpp | 2 + rldp2/rldp.cpp | 4 + storage/storage-cli.cpp | 2 +- tl/generate/scheme/ton_api.tl | 5 +- tl/generate/scheme/ton_api.tlo | Bin 65956 -> 66500 bytes .../validator-engine-console-query.cpp | 97 +++++++++++++++++- .../validator-engine-console-query.h | 40 ++++++++ .../validator-engine-console.cpp | 1 + validator/full-node-shard.cpp | 2 +- 30 files changed, 396 insertions(+), 38 deletions(-) diff --git a/adnl/adnl-address-list.cpp b/adnl/adnl-address-list.cpp index 064d4e56..7bd36374 100644 --- a/adnl/adnl-address-list.cpp +++ b/adnl/adnl-address-list.cpp @@ -39,6 +39,9 @@ class AdnlNetworkConnectionUdp : public AdnlNetworkConnection { void start_up() override { callback_->on_change_state(true); } + void get_ip_str(td::Promise promise) override { + promise.set_value(PSTRING() << addr_.get_ip_str().str() << ":" << addr_.get_port()); + } AdnlNetworkConnectionUdp(td::actor::ActorId network_manager, td::uint32 ip, td::uint16 port, std::unique_ptr callback); @@ -88,6 +91,9 @@ class AdnlNetworkConnectionTunnel : public AdnlNetworkConnection { pub_key_hash_ = pub_key_.compute_short_id(); //ready_.store(true, std::memory_order_release); } + void get_ip_str(td::Promise promise) override { + promise.set_value("tunnel"); + } AdnlNetworkConnectionTunnel(td::actor::ActorId network_manager, td::actor::ActorId adnl, adnl::AdnlNodeIdShort adnl_id, PublicKey pubkey, diff --git a/adnl/adnl-network-manager.h b/adnl/adnl-network-manager.h index 61c37d85..67cf602a 100644 --- a/adnl/adnl-network-manager.h +++ b/adnl/adnl-network-manager.h @@ -50,6 +50,8 @@ class AdnlNetworkConnection : public td::actor::Actor { virtual void send(AdnlNodeIdShort src, AdnlNodeIdShort dst, td::uint32 priority, td::BufferSlice message) = 0; virtual bool is_alive() const = 0; virtual bool is_active() const = 0; + + virtual void get_ip_str(td::Promise promise) = 0; virtual ~AdnlNetworkConnection() = default; }; diff --git a/adnl/adnl-peer-table.cpp b/adnl/adnl-peer-table.cpp index a334ef26..54891515 100644 --- a/adnl/adnl-peer-table.cpp +++ b/adnl/adnl-peer-table.cpp @@ -376,6 +376,15 @@ void AdnlPeerTableImpl::create_tunnel(AdnlNodeIdShort dst, td::uint32 size, td::Promise, AdnlAddress>> promise) { } +void AdnlPeerTableImpl::get_conn_ip_str(AdnlNodeIdShort l_id, AdnlNodeIdShort p_id, td::Promise promise) { + auto it = peers_.find(p_id); + if (it == peers_.end()) { + promise.set_value("undefined"); + return; + } + td::actor::send_closure(it->second, &AdnlPeer::get_conn_ip_str, l_id, std::move(promise)); +} + } // namespace adnl } // namespace ton diff --git a/adnl/adnl-peer-table.h b/adnl/adnl-peer-table.h index 59cced97..cb7da613 100644 --- a/adnl/adnl-peer-table.h +++ b/adnl/adnl-peer-table.h @@ -110,6 +110,7 @@ class AdnlPeerTable : public Adnl { virtual void deliver_query(AdnlNodeIdShort src, AdnlNodeIdShort dst, td::BufferSlice data, td::Promise promise) = 0; virtual void decrypt_message(AdnlNodeIdShort dst, td::BufferSlice data, td::Promise promise) = 0; + virtual void get_conn_ip_str(AdnlNodeIdShort l_id, AdnlNodeIdShort p_id, td::Promise promise) = 0; }; } // namespace adnl diff --git a/adnl/adnl-peer-table.hpp b/adnl/adnl-peer-table.hpp index 04f493bd..2a27a802 100644 --- a/adnl/adnl-peer-table.hpp +++ b/adnl/adnl-peer-table.hpp @@ -102,6 +102,7 @@ class AdnlPeerTableImpl : public AdnlPeerTable { void create_tunnel(AdnlNodeIdShort dst, td::uint32 size, td::Promise, AdnlAddress>> promise) override; + void get_conn_ip_str(AdnlNodeIdShort l_id, AdnlNodeIdShort p_id, td::Promise promise) override; struct PrintId {}; PrintId print_id() const { diff --git a/adnl/adnl-peer.cpp b/adnl/adnl-peer.cpp index 132f795a..35ba2a11 100644 --- a/adnl/adnl-peer.cpp +++ b/adnl/adnl-peer.cpp @@ -725,6 +725,28 @@ void AdnlPeerPairImpl::update_addr_list(AdnlAddressList addr_list) { (priority ? priority_addr_list_ : addr_list_) = addr_list; } +void AdnlPeerPairImpl::get_conn_ip_str(td::Promise promise) { + if (conns_.size() == 0 && priority_conns_.size() == 0) { + promise.set_value("undefined"); + return; + } + + for (auto &conn : priority_conns_) { + if (conn.ready()) { + td::actor::send_closure(conn.conn, &AdnlNetworkConnection::get_ip_str, std::move(promise)); + return; + } + } + for (auto &conn : conns_) { + if (conn.ready()) { + td::actor::send_closure(conn.conn, &AdnlNetworkConnection::get_ip_str, std::move(promise)); + return; + } + } + + promise.set_value("undefined"); +} + void AdnlPeerImpl::update_id(AdnlNodeIdFull id) { CHECK(id.compute_short_id() == peer_id_short_); if (!peer_id_.empty()) { @@ -841,6 +863,16 @@ void AdnlPeerImpl::update_dht_node(td::actor::ActorId dht_node) { } } +void AdnlPeerImpl::get_conn_ip_str(AdnlNodeIdShort l_id, td::Promise promise) { + auto it = peer_pairs_.find(l_id); + if (it == peer_pairs_.end()) { + promise.set_value("undefined"); + return; + } + + td::actor::send_closure(it->second, &AdnlPeerPair::get_conn_ip_str, std::move(promise)); +} + void AdnlPeerImpl::update_addr_list(AdnlNodeIdShort local_id, td::uint32 local_mode, td::actor::ActorId local_actor, AdnlAddressList addr_list) { auto it = peer_pairs_.find(local_id); diff --git a/adnl/adnl-peer.h b/adnl/adnl-peer.h index 2f671f9e..8488e82e 100644 --- a/adnl/adnl-peer.h +++ b/adnl/adnl-peer.h @@ -58,6 +58,7 @@ class AdnlPeerPair : public td::actor::Actor { virtual void update_dht_node(td::actor::ActorId dht_node) = 0; virtual void update_peer_id(AdnlNodeIdFull id) = 0; virtual void update_addr_list(AdnlAddressList addr_list) = 0; + virtual void get_conn_ip_str(td::Promise promise) = 0; static td::actor::ActorOwn create(td::actor::ActorId network_manager, td::actor::ActorId peer_table, td::uint32 local_mode, @@ -98,6 +99,7 @@ class AdnlPeer : public td::actor::Actor { virtual void update_addr_list(AdnlNodeIdShort local_id, td::uint32 local_mode, td::actor::ActorId local_actor, AdnlAddressList addr_list) = 0; virtual void update_dht_node(td::actor::ActorId dht_node) = 0; + virtual void get_conn_ip_str(AdnlNodeIdShort l_id, td::Promise promise) = 0; }; } // namespace adnl diff --git a/adnl/adnl-peer.hpp b/adnl/adnl-peer.hpp index 66b6b60c..410c0f75 100644 --- a/adnl/adnl-peer.hpp +++ b/adnl/adnl-peer.hpp @@ -88,6 +88,8 @@ class AdnlPeerPairImpl : public AdnlPeerPair { void update_addr_list(AdnlAddressList addr_list) override; void update_peer_id(AdnlNodeIdFull id) override; + void get_conn_ip_str(td::Promise promise) override; + void got_data_from_db(td::Result R); void got_data_from_static_nodes(td::Result R); void got_data_from_dht(td::Result R); @@ -265,6 +267,7 @@ class AdnlPeerImpl : public AdnlPeer { void update_addr_list(AdnlNodeIdShort local_id, td::uint32 local_mode, td::actor::ActorId local_actor, AdnlAddressList addr_list) override; void update_dht_node(td::actor::ActorId dht_node) override; + void get_conn_ip_str(AdnlNodeIdShort l_id, td::Promise promise) override; //void check_signature(td::BufferSlice data, td::BufferSlice signature, td::Promise promise) override; AdnlPeerImpl(td::actor::ActorId network_manager, td::actor::ActorId peer_table, diff --git a/adnl/adnl.h b/adnl/adnl.h index 917f5afb..b7dad216 100644 --- a/adnl/adnl.h +++ b/adnl/adnl.h @@ -56,6 +56,7 @@ class AdnlSenderInterface : public td::actor::Actor { virtual void send_query_ex(AdnlNodeIdShort src, AdnlNodeIdShort dst, std::string name, td::Promise promise, td::Timestamp timeout, td::BufferSlice data, td::uint64 max_answer_size) = 0; + virtual void get_conn_ip_str(AdnlNodeIdShort l_id, AdnlNodeIdShort p_id, td::Promise promise) = 0; }; class AdnlTunnel : public td::actor::Actor {}; diff --git a/overlay/overlay-broadcast.cpp b/overlay/overlay-broadcast.cpp index 1c05f705..03991b76 100644 --- a/overlay/overlay-broadcast.cpp +++ b/overlay/overlay-broadcast.cpp @@ -80,6 +80,7 @@ td::Status BroadcastSimple::distribute() { void BroadcastSimple::broadcast_checked(td::Result R) { if (R.is_error()) { + td::actor::send_closure(actor_id(overlay_), &OverlayImpl::update_peer_err_ctr, src_peer_id_, false); return; } is_valid_ = true; @@ -114,7 +115,7 @@ td::Status BroadcastSimple::run() { return run_continue(); } -td::Status BroadcastSimple::create(OverlayImpl *overlay, tl_object_ptr broadcast) { +td::Status BroadcastSimple::create(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, tl_object_ptr broadcast) { auto src = PublicKey{broadcast->src_}; auto data_hash = sha256_bits256(broadcast->data_.as_slice()); auto broadcast_hash = compute_broadcast_id(src, data_hash, broadcast->flags_); @@ -125,7 +126,7 @@ td::Status BroadcastSimple::create(OverlayImpl *overlay, tl_object_ptr(broadcast_hash, src, std::move(cert), broadcast->flags_, std::move(broadcast->data_), broadcast->date_, - std::move(broadcast->signature_), false, overlay); + std::move(broadcast->signature_), false, overlay, src_peer_id); TRY_STATUS(B->run()); overlay->register_simple_broadcast(std::move(B)); return td::Status::OK(); @@ -139,7 +140,7 @@ td::Status BroadcastSimple::create_new(td::actor::ActorId overlay, auto date = static_cast(td::Clocks::system()); auto B = std::make_unique(broadcast_hash, PublicKey{}, nullptr, flags, std::move(data), date, - td::BufferSlice{}, false, nullptr); + td::BufferSlice{}, false, nullptr, adnl::AdnlNodeIdShort::zero()); auto to_sign = B->to_sign(); auto P = td::PromiseCreator::lambda( diff --git a/overlay/overlay-broadcast.hpp b/overlay/overlay-broadcast.hpp index da29695a..e7b39eec 100644 --- a/overlay/overlay-broadcast.hpp +++ b/overlay/overlay-broadcast.hpp @@ -49,6 +49,8 @@ class BroadcastSimple : public td::ListNode { OverlayImpl *overlay_; + adnl::AdnlNodeIdShort src_peer_id_ = adnl::AdnlNodeIdShort::zero(); + td::Status check_time(); td::Status check_duplicate(); td::Status check_source(); @@ -61,7 +63,7 @@ class BroadcastSimple : public td::ListNode { public: BroadcastSimple(Overlay::BroadcastHash broadcast_hash, PublicKey source, std::shared_ptr cert, td::uint32 flags, td::BufferSlice data, td::uint32 date, td::BufferSlice signature, bool is_valid, - OverlayImpl *overlay) + OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id) : broadcast_hash_(broadcast_hash) , source_(std::move(source)) , cert_(std::move(cert)) @@ -70,7 +72,8 @@ class BroadcastSimple : public td::ListNode { , date_(date) , signature_(std::move(signature)) , is_valid_(is_valid) - , overlay_(overlay) { + , overlay_(overlay) + , src_peer_id_(src_peer_id) { } Overlay::BroadcastHash get_hash() const { @@ -98,7 +101,7 @@ class BroadcastSimple : public td::ListNode { void update_overlay(OverlayImpl *overlay); void broadcast_checked(td::Result R); - static td::Status create(OverlayImpl *overlay, tl_object_ptr broadcast); + static td::Status create(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, tl_object_ptr broadcast); static td::Status create_new(td::actor::ActorId overlay, td::actor::ActorId keyring, PublicKeyHash local_id, td::BufferSlice data, td::uint32 flags); diff --git a/overlay/overlay-fec-broadcast.cpp b/overlay/overlay-fec-broadcast.cpp index 0ff53346..7ff08309 100644 --- a/overlay/overlay-fec-broadcast.cpp +++ b/overlay/overlay-fec-broadcast.cpp @@ -88,6 +88,7 @@ td::Status OverlayFecBroadcastPart::run_checks() { void BroadcastFec::broadcast_checked(td::Result R) { if (R.is_error()) { + td::actor::send_closure(actor_id(overlay_), &OverlayImpl::update_peer_err_ctr, src_peer_id_, true); return; } overlay_->deliver_broadcast(get_source().compute_short_id(), data_.clone()); @@ -155,6 +156,7 @@ td::Status OverlayFecBroadcastPart::apply() { if (!bcast_->finalized()) { bcast_->set_overlay(overlay_); + bcast_->set_src_peer_id(src_peer_id_); TRY_STATUS(bcast_->add_part(seqno_, data_.clone(), export_serialized_short(), export_serialized())); auto R = bcast_->finish(); if (R.is_error()) { @@ -211,7 +213,7 @@ td::BufferSlice OverlayFecBroadcastPart::to_sign() { return serialize_tl_object(obj, true); } -td::Status OverlayFecBroadcastPart::create(OverlayImpl *overlay, +td::Status OverlayFecBroadcastPart::create(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, tl_object_ptr broadcast) { TRY_STATUS(overlay->check_date(broadcast->date_)); auto source = PublicKey{broadcast->src_}; @@ -244,12 +246,13 @@ td::Status OverlayFecBroadcastPart::create(OverlayImpl *overlay, std::move(broadcast->signature_), false, overlay->get_fec_broadcast(broadcast_hash), - overlay}; + overlay, + src_peer_id}; TRY_STATUS(B.run()); return td::Status::OK(); } -td::Status OverlayFecBroadcastPart::create(OverlayImpl *overlay, +td::Status OverlayFecBroadcastPart::create(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, tl_object_ptr broadcast) { auto bcast = overlay->get_fec_broadcast(broadcast->broadcast_hash_); if (!bcast) { @@ -285,7 +288,8 @@ td::Status OverlayFecBroadcastPart::create(OverlayImpl *overlay, std::move(broadcast->signature_), true, bcast, - overlay}; + overlay, + src_peer_id}; TRY_STATUS(B.run()); return td::Status::OK(); } @@ -300,7 +304,7 @@ td::Status OverlayFecBroadcastPart::create_new(OverlayImpl *overlay, td::actor:: auto B = std::make_unique( broadcast_hash, part_hash, PublicKey{}, overlay->get_certificate(local_id), data_hash, size, flags, - part_data_hash, std::move(part), seqno, std::move(fec_type), date, td::BufferSlice{}, false, nullptr, overlay); + part_data_hash, std::move(part), seqno, std::move(fec_type), date, td::BufferSlice{}, false, nullptr, overlay, adnl::AdnlNodeIdShort::zero()); auto to_sign = B->to_sign(); auto P = td::PromiseCreator::lambda( diff --git a/overlay/overlay-fec-broadcast.hpp b/overlay/overlay-fec-broadcast.hpp index f27e69fb..612af22f 100644 --- a/overlay/overlay-fec-broadcast.hpp +++ b/overlay/overlay-fec-broadcast.hpp @@ -194,6 +194,9 @@ class BroadcastFec : public td::ListNode { void set_overlay(OverlayImpl *overlay) { overlay_ = overlay; } + void set_src_peer_id(adnl::AdnlNodeIdShort src_peer_id) { + src_peer_id_ = src_peer_id; + } td::Status distribute_part(td::uint32 seqno); @@ -220,6 +223,7 @@ class BroadcastFec : public td::ListNode { std::map> parts_; OverlayImpl *overlay_; + adnl::AdnlNodeIdShort src_peer_id_ = adnl::AdnlNodeIdShort::zero(); td::BufferSlice data_; }; @@ -245,6 +249,7 @@ class OverlayFecBroadcastPart : public td::ListNode { BroadcastFec *bcast_; OverlayImpl *overlay_; + adnl::AdnlNodeIdShort src_peer_id_ = adnl::AdnlNodeIdShort::zero(); td::Status check_time(); td::Status check_duplicate(); @@ -260,7 +265,7 @@ class OverlayFecBroadcastPart : public td::ListNode { std::shared_ptr cert, Overlay::BroadcastDataHash data_hash, td::uint32 data_size, td::uint32 flags, Overlay::BroadcastDataHash part_data_hash, td::BufferSlice data, td::uint32 seqno, fec::FecType fec_type, td::uint32 date, td::BufferSlice signature, - bool is_short, BroadcastFec *bcast, OverlayImpl *overlay) + bool is_short, BroadcastFec *bcast, OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id) : broadcast_hash_(broadcast_hash) , part_hash_(part_hash) , source_(std::move(source)) @@ -276,7 +281,8 @@ class OverlayFecBroadcastPart : public td::ListNode { , signature_(std::move(signature)) , is_short_(is_short) , bcast_(bcast) - , overlay_(overlay) { + , overlay_(overlay) + , src_peer_id_(src_peer_id) { } td::uint32 data_size() const { @@ -310,8 +316,8 @@ class OverlayFecBroadcastPart : public td::ListNode { return td::Status::OK(); } - static td::Status create(OverlayImpl *overlay, tl_object_ptr broadcast); - static td::Status create(OverlayImpl *overlay, tl_object_ptr broadcast); + static td::Status create(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, tl_object_ptr broadcast); + static td::Status create(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, tl_object_ptr broadcast); static td::Status create_new(OverlayImpl *overlay, td::actor::ActorId overlay_actor_id, PublicKeyHash local_id, Overlay::BroadcastDataHash data_hash, td::uint32 size, td::uint32 flags, td::BufferSlice part, td::uint32 seqno, fec::FecType fec_type, diff --git a/overlay/overlay-manager.cpp b/overlay/overlay-manager.cpp index 1d133673..41750b4f 100644 --- a/overlay/overlay-manager.cpp +++ b/overlay/overlay-manager.cpp @@ -90,12 +90,12 @@ void OverlayManager::delete_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdSho } void OverlayManager::create_public_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, - std::unique_ptr callback, OverlayPrivacyRules rules) { + std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope) { CHECK(!dht_node_.empty()); auto id = overlay_id.compute_short_id(); register_overlay(local_id, id, Overlay::create(keyring_, adnl_, actor_id(this), dht_node_, local_id, std::move(overlay_id), - std::move(callback), std::move(rules))); + std::move(callback), std::move(rules), scope)); } void OverlayManager::create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, @@ -128,6 +128,7 @@ void OverlayManager::receive_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeId return; } + td::actor::send_closure(it2->second, &Overlay::update_throughput_in_ctr, src, (td::uint32)data.size(), false); td::actor::send_closure(it2->second, &Overlay::receive_message, src, std::move(data)); } @@ -152,12 +153,12 @@ void OverlayManager::receive_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdSh } auto it2 = it->second.find(OverlayIdShort{M->overlay_}); if (it2 == it->second.end()) { - VLOG(OVERLAY_NOTICE) << this << ": query to localid not in overlay " << M->overlay_ << "@" << dst << " from " - << src; + VLOG(OVERLAY_NOTICE) << this << ": query to localid not in overlay " << M->overlay_ << "@" << dst << " from " << src; promise.set_error(td::Status::Error(ErrorCode::protoviolation, PSTRING() << "bad overlay_id " << M->overlay_)); return; } + td::actor::send_closure(it2->second, &Overlay::update_throughput_in_ctr, src, (td::uint32)data.size(), true); td::actor::send_closure(it2->second, &Overlay::receive_query, src, std::move(data), std::move(promise)); } @@ -166,6 +167,15 @@ void OverlayManager::send_query_via(adnl::AdnlNodeIdShort dst, adnl::AdnlNodeIdS td::BufferSlice query, td::uint64 max_answer_size, td::actor::ActorId via) { CHECK(query.size() <= adnl::Adnl::huge_packet_max_size()); + + auto it = overlays_.find(src); + if (it != overlays_.end()) { + auto it2 = it->second.find(overlay_id); + if (it2 != it->second.end()) { + td::actor::send_closure(it2->second, &Overlay::update_throughput_out_ctr, dst, (td::uint32)query.size(), true); + } + } + td::actor::send_closure( via, &adnl::AdnlSenderInterface::send_query_ex, src, dst, std::move(name), std::move(promise), timeout, create_serialize_tl_object_suffix(query.as_slice(), overlay_id.tl()), max_answer_size); @@ -174,6 +184,15 @@ void OverlayManager::send_query_via(adnl::AdnlNodeIdShort dst, adnl::AdnlNodeIdS void OverlayManager::send_message_via(adnl::AdnlNodeIdShort dst, adnl::AdnlNodeIdShort src, OverlayIdShort overlay_id, td::BufferSlice object, td::actor::ActorId via) { CHECK(object.size() <= adnl::Adnl::huge_packet_max_size()); + + auto it = overlays_.find(src); + if (it != overlays_.end()) { + auto it2 = it->second.find(overlay_id); + if (it2 != it->second.end()) { + td::actor::send_closure(it2->second, &Overlay::update_throughput_out_ctr, dst, (td::uint32)object.size(), false); + } + } + td::actor::send_closure( via, &adnl::AdnlSenderInterface::send_message, src, dst, create_serialize_tl_object_suffix(object.as_slice(), overlay_id.tl())); diff --git a/overlay/overlay-manager.h b/overlay/overlay-manager.h index c6afefaf..d9739bf3 100644 --- a/overlay/overlay-manager.h +++ b/overlay/overlay-manager.h @@ -51,7 +51,7 @@ class OverlayManager : public Overlays { void update_dht_node(td::actor::ActorId dht) override; void create_public_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, - std::unique_ptr callback, OverlayPrivacyRules rules) override; + std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope) override; void create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::vector nodes, std::unique_ptr callback, OverlayPrivacyRules rules) override; diff --git a/overlay/overlay.cpp b/overlay/overlay.cpp index ec13fcd3..2574932b 100644 --- a/overlay/overlay.cpp +++ b/overlay/overlay.cpp @@ -37,10 +37,10 @@ td::actor::ActorOwn Overlay::create(td::actor::ActorId manager, td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::unique_ptr callback, - OverlayPrivacyRules rules) { + OverlayPrivacyRules rules, td::string scope) { auto R = td::actor::create_actor("overlay", keyring, adnl, manager, dht_node, local_id, std::move(overlay_id), true, std::vector(), - std::move(callback), std::move(rules)); + std::move(callback), std::move(rules), scope); return td::actor::ActorOwn(std::move(R)); } @@ -60,7 +60,7 @@ OverlayImpl::OverlayImpl(td::actor::ActorId keyring, td::actor td::actor::ActorId manager, td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, bool pub, std::vector nodes, std::unique_ptr callback, - OverlayPrivacyRules rules) + OverlayPrivacyRules rules, td::string scope) : keyring_(keyring) , adnl_(adnl) , manager_(manager) @@ -69,7 +69,8 @@ OverlayImpl::OverlayImpl(td::actor::ActorId keyring, td::actor , id_full_(std::move(overlay_id)) , callback_(std::move(callback)) , public_(pub) - , rules_(std::move(rules)) { + , rules_(std::move(rules)) + , scope_(scope) { overlay_id_ = id_full_.compute_short_id(); VLOG(OVERLAY_INFO) << this << ": creating " << (public_ ? "public" : "private"); @@ -161,17 +162,17 @@ void OverlayImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr bcast) { - return BroadcastSimple::create(this, std::move(bcast)); + return BroadcastSimple::create(this, message_from, std::move(bcast)); } td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr b) { - return OverlayFecBroadcastPart::create(this, std::move(b)); + return OverlayFecBroadcastPart::create(this, message_from, std::move(b)); } td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr b) { - return OverlayFecBroadcastPart::create(this, std::move(b)); + return OverlayFecBroadcastPart::create(this, message_from, std::move(b)); } td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, @@ -236,6 +237,35 @@ void OverlayImpl::receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice dat void OverlayImpl::alarm() { bcast_gc(); + + if(update_throughput_at_.is_in_past()) { + double t_elapsed = td::Time::now() - last_throughput_update_.at(); + + peers_.iterate([&](const adnl::AdnlNodeIdShort &key, OverlayPeer &peer) { + peer.throughput_out_bytes = static_cast(peer.throughput_out_bytes_ctr / t_elapsed); + peer.throughput_in_bytes = static_cast(peer.throughput_in_bytes_ctr / t_elapsed); + + peer.throughput_out_packets = static_cast(peer.throughput_out_packets_ctr / t_elapsed); + peer.throughput_in_packets = static_cast(peer.throughput_in_packets_ctr / t_elapsed); + + peer.throughput_out_bytes_ctr = 0; + peer.throughput_in_bytes_ctr = 0; + + peer.throughput_out_packets_ctr = 0; + peer.throughput_in_packets_ctr = 0; + + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), peer_id = key](td::Result result) { + result.ensure(); + td::actor::send_closure(SelfId, &Overlay::update_peer_ip_str, peer_id, result.move_as_ok()); + }); + + td::actor::send_closure(adnl_, &adnl::AdnlSenderInterface::get_conn_ip_str, local_id_, key, std::move(P)); + }); + + update_throughput_at_ = td::Timestamp::in(50.0); + last_throughput_update_ = td::Timestamp::now(); + } + if (public_) { if (peers_.size() > 0) { auto P = get_random_peer(); @@ -541,6 +571,17 @@ void OverlayImpl::check_broadcast(PublicKeyHash src, td::BufferSlice data, td::P callback_->check_broadcast(src, overlay_id_, std::move(data), std::move(promise)); } +void OverlayImpl::update_peer_err_ctr(adnl::AdnlNodeIdShort peer_id, bool is_fec) { + auto src_peer = peers_.get(peer_id); + if(src_peer) { + if(is_fec) { + src_peer->fec_broadcast_errors++; + } else { + src_peer->broadcast_errors++; + } + } +} + void OverlayImpl::broadcast_checked(Overlay::BroadcastHash hash, td::Result R) { { auto it = broadcasts_.find(hash); @@ -561,7 +602,26 @@ void OverlayImpl::get_stats(td::Promiseadnl_id_ = local_id_.bits256_value(); res->overlay_id_ = overlay_id_.bits256_value(); res->overlay_id_full_ = id_full_.pubkey().tl(); - peers_.iterate([&](const adnl::AdnlNodeIdShort &key, const OverlayPeer &peer) { res->nodes_.push_back(key.tl()); }); + res->scope_ = scope_; + peers_.iterate([&](const adnl::AdnlNodeIdShort &key, const OverlayPeer &peer) { + auto node_obj = create_tl_object(); + node_obj->adnl_id_ = key.bits256_value(); + node_obj->t_out_bytes_ = peer.throughput_out_bytes; + node_obj->t_in_bytes_ = peer.throughput_in_bytes; + + node_obj->t_out_pckts_ = peer.throughput_out_packets; + node_obj->t_in_pckts_ = peer.throughput_in_packets; + + node_obj->ip_addr_ = peer.ip_addr_str; + + node_obj->last_in_query_ = static_cast(peer.last_in_query_at.at_unix()); + node_obj->last_out_query_ = static_cast(peer.last_out_query_at.at_unix()); + + node_obj->bdcst_errors_ = peer.broadcast_errors; + node_obj->fec_bdcst_errors_ = peer.fec_broadcast_errors; + + res->nodes_.push_back(std::move(node_obj)); + }); res->stats_.push_back( create_tl_object("neighbours_cnt", PSTRING() << neighbours_.size())); diff --git a/overlay/overlay.h b/overlay/overlay.h index cce24ed8..51eab1b4 100644 --- a/overlay/overlay.h +++ b/overlay/overlay.h @@ -42,7 +42,7 @@ class Overlay : public td::actor::Actor { td::actor::ActorId manager, td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::unique_ptr callback, - OverlayPrivacyRules rules); + OverlayPrivacyRules rules, td::string scope); static td::actor::ActorOwn create(td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId manager, @@ -64,6 +64,9 @@ class Overlay : public td::actor::Actor { virtual void set_privacy_rules(OverlayPrivacyRules rules) = 0; virtual void receive_nodes_from_db(tl_object_ptr nodes) = 0; virtual void get_stats(td::Promise> promise) = 0; + virtual void update_throughput_out_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) = 0; + virtual void update_throughput_in_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) = 0; + virtual void update_peer_ip_str(adnl::AdnlNodeIdShort peer_id, td::string ip_str) = 0; //virtual void receive_broadcast(td::BufferSlice data) = 0; //virtual void subscribe(std::unique_ptr callback) = 0; }; diff --git a/overlay/overlay.hpp b/overlay/overlay.hpp index 76d17aab..1d5f9861 100644 --- a/overlay/overlay.hpp +++ b/overlay/overlay.hpp @@ -79,6 +79,26 @@ class OverlayPeer { td::int32 get_version() const { return node_.version(); } + + td::uint32 throughput_out_bytes = 0; + td::uint32 throughput_in_bytes = 0; + + td::uint32 throughput_out_packets = 0; + td::uint32 throughput_in_packets = 0; + + td::uint32 throughput_out_bytes_ctr = 0; + td::uint32 throughput_in_bytes_ctr = 0; + + td::uint32 throughput_out_packets_ctr = 0; + td::uint32 throughput_in_packets_ctr = 0; + + td::uint32 broadcast_errors = 0; + td::uint32 fec_broadcast_errors = 0; + + td::Timestamp last_in_query_at = td::Timestamp::now(); + td::Timestamp last_out_query_at = td::Timestamp::now(); + + td::string ip_addr_str = "undefined"; private: OverlayNode node_; @@ -93,7 +113,7 @@ class OverlayImpl : public Overlay { td::actor::ActorId manager, td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, bool pub, std::vector nodes, std::unique_ptr callback, - OverlayPrivacyRules rules); + OverlayPrivacyRules rules, td::string scope = "{ \"type\": \"undefined\" }"); void update_dht_node(td::actor::ActorId dht) override { dht_node_ = dht; } @@ -109,6 +129,9 @@ class OverlayImpl : public Overlay { void alarm() override; void start_up() override { + update_throughput_at_ = td::Timestamp::in(50.0); + last_throughput_update_ = td::Timestamp::now(); + if (public_) { update_db_at_ = td::Timestamp::in(60.0); } @@ -150,6 +173,8 @@ class OverlayImpl : public Overlay { void broadcast_checked(Overlay::BroadcastHash hash, td::Result R); void check_broadcast(PublicKeyHash src, td::BufferSlice data, td::Promise promise); + void update_peer_err_ctr(adnl::AdnlNodeIdShort peer_id, bool is_fec); + BroadcastFec *get_fec_broadcast(BroadcastHash hash); void register_fec_broadcast(std::unique_ptr bcast); void register_simple_broadcast(std::unique_ptr bcast); @@ -191,6 +216,39 @@ class OverlayImpl : public Overlay { td::Result get_encryptor(PublicKey source); void get_stats(td::Promise> promise) override; + + void update_throughput_out_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) override { + auto out_peer = peers_.get(peer_id); + if(out_peer) { + out_peer->throughput_out_bytes_ctr += msg_size; + out_peer->throughput_out_packets_ctr++; + + if(is_query) + { + out_peer->last_out_query_at = td::Timestamp::now(); + } + } + } + + void update_throughput_in_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) override { + auto in_peer = peers_.get(peer_id); + if(in_peer) { + in_peer->throughput_in_bytes_ctr += msg_size; + in_peer->throughput_in_packets_ctr++; + + if(is_query) + { + in_peer->last_in_query_at = td::Timestamp::now(); + } + } + } + + void update_peer_ip_str(adnl::AdnlNodeIdShort peer_id, td::string ip_str) override { + auto fpeer = peers_.get(peer_id); + if(fpeer) { + fpeer->ip_addr_str = ip_str; + } + } private: template @@ -236,6 +294,8 @@ class OverlayImpl : public Overlay { td::DecTree peers_; td::Timestamp next_dht_query_ = td::Timestamp::in(1.0); td::Timestamp update_db_at_; + td::Timestamp update_throughput_at_; + td::Timestamp last_throughput_update_; std::unique_ptr callback_; @@ -291,6 +351,7 @@ class OverlayImpl : public Overlay { bool public_; bool semi_public_ = false; OverlayPrivacyRules rules_; + td::string scope_; std::map> certs_; class CachedEncryptor : public td::ListNode { diff --git a/overlay/overlays.h b/overlay/overlays.h index 799faf5a..45316254 100644 --- a/overlay/overlays.h +++ b/overlay/overlays.h @@ -193,7 +193,7 @@ class Overlays : public td::actor::Actor { virtual void update_dht_node(td::actor::ActorId dht) = 0; virtual void create_public_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, - std::unique_ptr callback, OverlayPrivacyRules rules) = 0; + std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope) = 0; virtual void create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::vector nodes, std::unique_ptr callback, OverlayPrivacyRules rules) = 0; diff --git a/rldp/rldp-in.hpp b/rldp/rldp-in.hpp index 196f5edc..b4981999 100644 --- a/rldp/rldp-in.hpp +++ b/rldp/rldp-in.hpp @@ -99,6 +99,7 @@ class RldpIn : public RldpImpl { void in_transfer_completed(TransferId transfer_id); void add_id(adnl::AdnlNodeIdShort local_id) override; + void get_conn_ip_str(adnl::AdnlNodeIdShort l_id, adnl::AdnlNodeIdShort p_id, td::Promise promise) override; RldpIn(td::actor::ActorId adnl) : adnl_(adnl) { } diff --git a/rldp/rldp.cpp b/rldp/rldp.cpp index ac787ca5..9b38dcb8 100644 --- a/rldp/rldp.cpp +++ b/rldp/rldp.cpp @@ -259,6 +259,10 @@ void RldpIn::add_id(adnl::AdnlNodeIdShort local_id) { local_ids_.insert(local_id); } +void RldpIn::get_conn_ip_str(adnl::AdnlNodeIdShort l_id, adnl::AdnlNodeIdShort p_id, td::Promise promise) { + td::actor::send_closure(adnl_, &adnl::AdnlPeerTable::get_conn_ip_str, l_id, p_id, std::move(promise)); +} + std::unique_ptr RldpIn::make_adnl_callback() { class Callback : public adnl::Adnl::Callback { private: diff --git a/rldp2/rldp-in.hpp b/rldp2/rldp-in.hpp index e1697c4d..c2e46d2a 100644 --- a/rldp2/rldp-in.hpp +++ b/rldp2/rldp-in.hpp @@ -90,6 +90,8 @@ class RldpIn : public RldpImpl { void add_id(adnl::AdnlNodeIdShort local_id) override; + void get_conn_ip_str(adnl::AdnlNodeIdShort l_id, adnl::AdnlNodeIdShort p_id, td::Promise promise) override; + RldpIn(td::actor::ActorId adnl) : adnl_(adnl) { } diff --git a/rldp2/rldp.cpp b/rldp2/rldp.cpp index 7a608c14..765e38a5 100644 --- a/rldp2/rldp.cpp +++ b/rldp2/rldp.cpp @@ -217,6 +217,10 @@ void RldpIn::add_id(adnl::AdnlNodeIdShort local_id) { local_ids_.insert(local_id); } +void RldpIn::get_conn_ip_str(adnl::AdnlNodeIdShort l_id, adnl::AdnlNodeIdShort p_id, td::Promise promise) { + td::actor::send_closure(adnl_, &adnl::AdnlPeerTable::get_conn_ip_str, l_id, p_id, std::move(promise)); +} + std::unique_ptr RldpIn::make_adnl_callback() { class Callback : public adnl::Adnl::Callback { private: diff --git a/storage/storage-cli.cpp b/storage/storage-cli.cpp index fcbbc55f..06cb0055 100644 --- a/storage/storage-cli.cpp +++ b/storage/storage-cli.cpp @@ -95,7 +95,7 @@ class PeerManager : public td::actor::Actor { } }; send_closure(overlays_, &ton::overlay::Overlays::create_public_overlay, adnl_id_, overlay_id_.clone(), - std::make_unique(), rules); + std::make_unique(), rules, "{ \"type\": \"storage\" }"); } void tear_down() override { send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, adnl_id_, overlay_id_.compute_short_id()); diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 0be70540..7f52198b 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -620,8 +620,9 @@ engine.validator.proposalVote perm_key:int256 to_send:bytes = engine.validator.P engine.validator.dhtServerStatus id:int256 status:int = engine.validator.DhtServerStatus; engine.validator.dhtServersStatus servers:(vector engine.validator.dhtServerStatus) = engine.validator.DhtServersStatus; -engine.validator.overlayStats overlay_id:int256 overlay_id_full:PublicKey adnl_id:int256 nodes:(vector adnl.id.short) - stats:(vector engine.validator.oneStat) = engine.validator.OverlayStats; +engine.validator.overlayStatsNode adnl_id:int256 ip_addr:string bdcst_errors:int fec_bdcst_errors:int last_in_query:int last_out_query:int t_out_bytes:int t_in_bytes:int t_out_pckts:int t_in_pckts:int = engine.validator.OverlayStatsNode; + +engine.validator.overlayStats overlay_id:int256 overlay_id_full:PublicKey adnl_id:int256 scope:string nodes:(vector engine.validator.overlayStatsNode) stats:(vector engine.validator.oneStat) = engine.validator.OverlayStats; engine.validator.overlaysStats overlays:(vector engine.validator.overlayStats) = engine.validator.OverlaysStats; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index dbdd73b3db01254391b73828527be4872531d40a..fad15df58f4f7f8b0e0059b41a944aa25e7faba6 100644 GIT binary patch delta 339 zcmZ3|%yOiih4;~HeJchiIJ%K{z9`GjUkmO}-XLnhdQ+k3r{d&|qOvT0`6;QJ&xx+% zW3=8}tMWb>qNbojffJ#oXi#X#&R?2zRL*}=d7;)BfNnjCpgd6Fk9duBm=VoFL8&^QTRo}`rI z;*$8(qN4nw;>nJd(t-kMsmbx^qWru$i9ltUdGUp%sYR6xllLE&;sXoimzF@pCf938 z@^F{Lg9MT)OHzv`8Ax(~wE%gOA8LqC7O-T6$`>SOmw@EC!17?;# delta 103 zcmX@o&a$MLh4;~HeJchiII@vujFZnf-H;7tn))1S*$7r>Augd#ms9=WzCrI|@ pF(Dt;$&RnIC$n^VPTuRQJo(8Ifyo}d77%4~x--Cx%^W>@$^ofHDMJ7N diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index 4b5802c6..89fe9f1a 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -34,6 +34,7 @@ #include "overlay/overlays.h" #include +#include Tokenizer::Tokenizer(td::BufferSlice data) : data_(std::move(data)) { remaining_ = data_.as_slice(); @@ -835,11 +836,25 @@ td::Status GetOverlaysStatsQuery::receive(td::BufferSlice data) { "received incorrect answer: "); for (auto &s : f->overlays_) { td::StringBuilder sb; - sb << "overlay_id=" << s->overlay_id_ << " adnl_id=" << s->adnl_id_ << "\n"; - sb << " nodes:"; + sb << "overlay_id: " << s->overlay_id_ << " adnl_id: " << s->adnl_id_ << " scope: " << s->scope_ << "\n"; + sb << " nodes:\n"; + + td::uint32 overlay_t_out_bytes = 0; + td::uint32 overlay_t_out_pckts = 0; + td::uint32 overlay_t_in_bytes = 0; + td::uint32 overlay_t_in_pckts = 0; + for (auto &n : s->nodes_) { - sb << " " << n->id_ << "\n"; + sb << " adnl_id: " << n->adnl_id_ << " ip_addr: " << n->ip_addr_ << " broadcast_errors: " << n->bdcst_errors_ << " fec_broadcast_errors: " << n->fec_bdcst_errors_ << " last_in_query: " << n->last_in_query_ << " (" << time_to_human(n->last_in_query_) << ")" << " last_out_query: " << n->last_out_query_ << " (" << time_to_human(n->last_out_query_) << ")" << "\n throughput:\n out: " << n->t_out_bytes_ << " bytes/sec, " << n->t_out_pckts_ << " pckts/sec\n in: " << n->t_in_bytes_ << " bytes/sec, " << n->t_in_pckts_ << " pckts/sec\n"; + + overlay_t_out_bytes += n->t_out_bytes_; + overlay_t_out_pckts += n->t_out_pckts_; + + overlay_t_in_bytes += n->t_in_bytes_; + overlay_t_in_pckts += n->t_in_pckts_; } + sb << " total_throughput:\n out: " << overlay_t_out_bytes << " bytes/sec, " << overlay_t_out_pckts << " pckts/sec\n in: " << overlay_t_in_bytes << " bytes/sec, " << overlay_t_in_pckts << " pckts/sec\n"; + sb << " stats:\n"; for (auto &t : s->stats_) { sb << " " << t->key_ << "\t" << t->value_ << "\n"; @@ -849,6 +864,82 @@ td::Status GetOverlaysStatsQuery::receive(td::BufferSlice data) { return td::Status::OK(); } +td::Status GetOverlaysStatsJsonQuery::run() { + TRY_RESULT_ASSIGN(file_name_, tokenizer_.get_token()); + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status GetOverlaysStatsJsonQuery::send() { + auto b = ton::create_serialize_tl_object(); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status GetOverlaysStatsJsonQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + std::ofstream sb(file_name_); + + sb << "[\n"; + bool rtail = false; + for (auto &s : f->overlays_) { + if(rtail) { + sb << ",\n"; + } else { + rtail = true; + } + + sb << "{\n \"overlay_id\": \"" << s->overlay_id_ << "\",\n \"adnl_id\": \"" << s->adnl_id_ << "\",\n \"scope\": " << s->scope_ << ",\n"; + sb << " \"nodes\": [\n"; + + td::uint32 overlay_t_out_bytes = 0; + td::uint32 overlay_t_out_pckts = 0; + td::uint32 overlay_t_in_bytes = 0; + td::uint32 overlay_t_in_pckts = 0; + + bool tail = false; + for (auto &n : s->nodes_) { + if(tail) { + sb << ",\n"; + } else { + tail = true; + } + + sb << " {\n \"adnl_id\": \"" << n->adnl_id_ << "\",\n \"ip_addr\": \"" << n->ip_addr_ << "\",\n \"broadcast_errors\": " << n->bdcst_errors_ << ",\n \"fec_broadcast_errors\": " << n->fec_bdcst_errors_ << ",\n \"last_in_query_unix\": " << n->last_in_query_ << ",\n \"last_in_query_human\": \"" << time_to_human(n->last_in_query_) << "\",\n" << " \"last_out_query_unix\": " << n->last_out_query_ << ",\n \"last_out_query_human\": \"" << time_to_human(n->last_out_query_) << "\",\n" << "\n \"throughput\": { \"out_bytes_sec\": " << n->t_out_bytes_ << ", \"out_pckts_sec\": " << n->t_out_pckts_ << ", \"in_bytes_sec\": " << n->t_in_bytes_ << ", \"in_pckts_sec\": " << n->t_in_pckts_ << " }\n }"; + + overlay_t_out_bytes += n->t_out_bytes_; + overlay_t_out_pckts += n->t_out_pckts_; + + overlay_t_in_bytes += n->t_in_bytes_; + overlay_t_in_pckts += n->t_in_pckts_; + } + sb << " ],\n"; + + sb << " \"total_throughput\": { \"out_bytes_sec\": " << overlay_t_out_bytes << ", \"out_pckts_sec\": " << overlay_t_out_pckts << ", \"in_bytes_sec\": " << overlay_t_in_bytes << ", \"in_pckts_sec\": " << overlay_t_in_pckts << " },\n"; + + sb << " \"stats\": {\n"; + + tail = false; + for (auto &t : s->stats_) { + if(tail) { + sb << ",\n"; + } else { + tail = true; + } + + sb << " \"" << t->key_ << "\": \"" << t->value_ << "\""; + } + sb << "\n }\n"; + sb << "}\n"; + } + sb << "]\n"; + sb << std::flush; + + td::TerminalIO::output(std::string("wrote stats to " + file_name_ + "\n")); + return td::Status::OK(); +} + td::Status ImportCertificateQuery::receive(td::BufferSlice data) { TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index 769f2052..558a0893 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -918,11 +918,51 @@ class GetOverlaysStatsQuery : public Query { static std::string get_help() { return "getoverlaysstats\tgets stats for all overlays"; } + static std::string time_to_human(int unixtime) { + char time_buffer[80]; + time_t rawtime = unixtime; + struct tm tInfo; + struct tm* timeinfo = localtime_r(&rawtime, &tInfo); + assert(timeinfo == &tInfo); + strftime(time_buffer, 80, "%c", timeinfo); + return std::string(time_buffer); + } std::string name() const override { return get_name(); } }; +class GetOverlaysStatsJsonQuery : public Query { + public: + GetOverlaysStatsJsonQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "getoverlaysstatsjson"; + } + static std::string get_help() { + return "getoverlaysstatsjson \tgets stats for all overlays and writes to json file"; + } + static std::string time_to_human(int unixtime) { + char time_buffer[80]; + time_t rawtime = unixtime; + struct tm tInfo; + struct tm* timeinfo = localtime_r(&rawtime, &tInfo); + assert(timeinfo == &tInfo); + strftime(time_buffer, 80, "%c", timeinfo); + return std::string(time_buffer); + } + std::string name() const override { + return get_name(); + } + +private: + std::string file_name_; +}; + class SignCertificateQuery : public Query { public: SignCertificateQuery(td::actor::ActorId console, Tokenizer tokenizer) diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index 899cd53f..8e5f37b3 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -137,6 +137,7 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); } diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 279fb6fb..1094c896 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -93,7 +93,7 @@ void FullNodeShardImpl::create_overlay() { }; td::actor::send_closure(overlays_, &overlay::Overlays::create_public_overlay, adnl_id_, overlay_id_full_.clone(), - std::make_unique(actor_id(this)), rules_); + std::make_unique(actor_id(this)), rules_, PSTRING() << "{ \"type\": \"shard\", \"shard_id\": " << get_shard() << ", \"workchain_id\": " << get_workchain() << " }"); td::actor::send_closure(rldp_, &rldp::Rldp::add_id, adnl_id_); if (cert_) { From 127f5778dc399b939c8582641cac0d3f6f38e379 Mon Sep 17 00:00:00 2001 From: Doge Date: Tue, 14 Jun 2022 01:00:33 +0800 Subject: [PATCH 07/16] Upgrade abseil cpp (#392) * Upgrade abseil cpp --- third-party/abseil-cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third-party/abseil-cpp b/third-party/abseil-cpp index df3ea785..21510581 160000 --- a/third-party/abseil-cpp +++ b/third-party/abseil-cpp @@ -1 +1 @@ -Subproject commit df3ea785d8c30a9503321a3d35ee7d35808f190d +Subproject commit 215105818dfde3174fe799600bb0f3cae233d0bf From b398862cf8635ac18d1f9676ba6b1ea52e207fe8 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 27 Jun 2022 10:22:39 +0300 Subject: [PATCH 08/16] Fix error handling in ServerSocketFd.cpp --- tdutils/td/utils/port/ServerSocketFd.cpp | 22 +++------------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/tdutils/td/utils/port/ServerSocketFd.cpp b/tdutils/td/utils/port/ServerSocketFd.cpp index 3fc69f17..d92dd8e2 100644 --- a/tdutils/td/utils/port/ServerSocketFd.cpp +++ b/tdutils/td/utils/port/ServerSocketFd.cpp @@ -253,25 +253,9 @@ class ServerSocketFdImpl { } auto error = Status::PosixError(accept_errno, PSLICE() << "Accept from " << get_native_fd() << " has failed"); - switch (accept_errno) { - case EBADF: - case EFAULT: - case EINVAL: - case ENOTSOCK: - case EOPNOTSUPP: - LOG(FATAL) << error; - UNREACHABLE(); - break; - default: - LOG(ERROR) << error; - // fallthrough - case EMFILE: - case ENFILE: - case ECONNABORTED: //??? - get_poll_info().clear_flags(PollFlags::Read()); - get_poll_info().add_flags(PollFlags::Close()); - return std::move(error); - } + get_poll_info().clear_flags(PollFlags::Read()); + get_poll_info().add_flags(PollFlags::Close()); + return std::move(error); } Status get_pending_error() { From 3d8d7b5c28a78c835c310931ccabd82d53432d50 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 27 Jun 2022 10:24:21 +0300 Subject: [PATCH 09/16] Update crc32c to 1.1.2 --- third-party/crc32c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third-party/crc32c b/third-party/crc32c index fd33bccd..02e65f4f 160000 --- a/third-party/crc32c +++ b/third-party/crc32c @@ -1 +1 @@ -Subproject commit fd33bccdf4824716df2552f7addd735264f77872 +Subproject commit 02e65f4fd3065d27b2e29324800ca6d04df16126 From 6fb01df1d267c57fbb4d3e2f3c3019c181b3ef6e Mon Sep 17 00:00:00 2001 From: neodiX42 Date: Wed, 29 Jun 2022 22:08:02 +0300 Subject: [PATCH 10/16] Compile TON against Ubuntu 18.04, 20.04 and 22.04 in a single github action (#415) * Update and rename ubuntu-18.04-compile.yml to ubuntu-compile.yml * Update ubuntu-compile.yml segregate output binaries --- .github/workflows/ubuntu-18.04-compile.yml | 40 ---------------------- .github/workflows/ubuntu-compile.yml | 40 ++++++++++++++++++++++ 2 files changed, 40 insertions(+), 40 deletions(-) delete mode 100644 .github/workflows/ubuntu-18.04-compile.yml create mode 100644 .github/workflows/ubuntu-compile.yml diff --git a/.github/workflows/ubuntu-18.04-compile.yml b/.github/workflows/ubuntu-18.04-compile.yml deleted file mode 100644 index e215c7bd..00000000 --- a/.github/workflows/ubuntu-18.04-compile.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: C/C++ CI Ubuntu 18.04 Compile - -on: [push,workflow_dispatch] - -jobs: - build: - - runs-on: ubuntu-18.04 - - steps: - - name: Check out repository - uses: actions/checkout@v2 - with: - submodules: 'recursive' - - - name: Install libraries - run: | - sudo apt update - sudo apt install -y build-essential git make cmake clang libgflags-dev zlib1g-dev libssl-dev libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev ninja-build - - - name: Configure & Build - run: | - export CC=$(which clang) - export CXX=$(which clang++) - export CCACHE_DISABLE=1 - mkdir build - cd build - cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. - ninja fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork - - - name: find & copy binaries - run: | - mkdir artifacts - cp build/crypto/fift build/crypto/tlbc build/crypto/func build/crypto/create-state build/validator-engine-console/validator-engine-console build/tonlib/tonlib-cli build/tonlib/libtonlibjson.so.0.5 build/http/http-proxy build/rldp-http-proxy/rldp-http-proxy build/dht-server/dht-server build/lite-client/lite-client build/validator-engine/validator-engine build/utils/generate-random-id build/utils/json2tlo build/adnl/adnl-proxy artifacts - - - name: Upload artifacts - uses: actions/upload-artifact@master - with: - name: ton-binaries - path: artifacts diff --git a/.github/workflows/ubuntu-compile.yml b/.github/workflows/ubuntu-compile.yml new file mode 100644 index 00000000..2862a471 --- /dev/null +++ b/.github/workflows/ubuntu-compile.yml @@ -0,0 +1,40 @@ +name: Ubuntu Compile + +on: [push,workflow_dispatch] + +jobs: + build: + strategy: + fail-fast: false + matrix: + os: [ubuntu-18.04, ubuntu-20.04, ubuntu-22.04] + runs-on: ${{ matrix.os }} + + steps: + - name: Check out repository + uses: actions/checkout@v2 + with: + submodules: 'recursive' + + - name: Install libraries + run: | + sudo apt update + sudo apt install -y build-essential git make cmake clang libgflags-dev zlib1g-dev libssl-dev libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev ninja-build + - name: Configure & Build + run: | + export CC=$(which clang) + export CXX=$(which clang++) + export CCACHE_DISABLE=1 + mkdir build-${{ matrix.os }} + cd build-${{ matrix.os }} + cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. + ninja fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork + - name: find & copy binaries + run: | + mkdir artifacts-${{ matrix.os }} + cp build-${{ matrix.os }}/crypto/fift build-${{ matrix.os }}/crypto/tlbc build-${{ matrix.os }}/crypto/func build-${{ matrix.os }}/crypto/create-state build-${{ matrix.os }}/validator-engine-console/validator-engine-console build-${{ matrix.os }}/tonlib/tonlib-cli build-${{ matrix.os }}/tonlib/libtonlibjson.so.0.5 build-${{ matrix.os }}/http/http-proxy build-${{ matrix.os }}/rldp-http-proxy/rldp-http-proxy build-${{ matrix.os }}/dht-server/dht-server build-${{ matrix.os }}/lite-client/lite-client build-${{ matrix.os }}/validator-engine/validator-engine build-${{ matrix.os }}/utils/generate-random-id build-${{ matrix.os }}/utils/json2tlo build-${{ matrix.os }}/adnl/adnl-proxy artifacts-${{ matrix.os }} + - name: Upload artifacts + uses: actions/upload-artifact@master + with: + name: ton-binaries-${{ matrix.os }} + path: artifacts-${{ matrix.os }} From f6bf3d696d94c19251e7ac4c52d9d3830832ce3e Mon Sep 17 00:00:00 2001 From: neodiX42 Date: Thu, 30 Jun 2022 15:18:11 +0300 Subject: [PATCH 11/16] Update windows2019x64-compile.yml (#418) Allow Windows github action to run against any branch --- .github/workflows/windows2019x64-compile.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/windows2019x64-compile.yml b/.github/workflows/windows2019x64-compile.yml index 43303e65..f1dce51c 100644 --- a/.github/workflows/windows2019x64-compile.yml +++ b/.github/workflows/windows2019x64-compile.yml @@ -1,10 +1,8 @@ -name: C/C++ CI Windows Server 2019 x64 Compile +name: Windows Server 2019 x64 Compile on: workflow_dispatch: push: - branches: - - 'master' defaults: run: From 90e06e33949cbbee32ff1b21d8f6ec22edf89d44 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 30 Jun 2022 15:18:40 +0300 Subject: [PATCH 12/16] Fix compilation errors in windows (#417) --- overlay/overlay.cpp | 5 +-- .../validator-engine-console-query.h | 32 ++++++++----------- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/overlay/overlay.cpp b/overlay/overlay.cpp index 2574932b..3c707e60 100644 --- a/overlay/overlay.cpp +++ b/overlay/overlay.cpp @@ -240,7 +240,8 @@ void OverlayImpl::alarm() { if(update_throughput_at_.is_in_past()) { double t_elapsed = td::Time::now() - last_throughput_update_.at(); - + + auto SelfId = actor_id(this); peers_.iterate([&](const adnl::AdnlNodeIdShort &key, OverlayPeer &peer) { peer.throughput_out_bytes = static_cast(peer.throughput_out_bytes_ctr / t_elapsed); peer.throughput_in_bytes = static_cast(peer.throughput_in_bytes_ctr / t_elapsed); @@ -254,7 +255,7 @@ void OverlayImpl::alarm() { peer.throughput_out_packets_ctr = 0; peer.throughput_in_packets_ctr = 0; - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), peer_id = key](td::Result result) { + auto P = td::PromiseCreator::lambda([SelfId, peer_id = key](td::Result result) { result.ensure(); td::actor::send_closure(SelfId, &Overlay::update_peer_ip_str, peer_id, result.move_as_ok()); }); diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index 558a0893..15b77314 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -193,6 +193,20 @@ class Query : public td::actor::Actor { virtual std::string name() const = 0; void handle_error(td::Status error); + static std::string time_to_human(int unixtime) { + char time_buffer[80]; + time_t rawtime = unixtime; + struct tm tInfo; +#if defined(_WIN32) || defined(_WIN64) + struct tm* timeinfo = localtime_s(&tInfo, &rawtime) ? nullptr : &tInfo; +#else + struct tm* timeinfo = localtime_r(&rawtime, &tInfo); +#endif + assert(timeinfo == &tInfo); + strftime(time_buffer, 80, "%c", timeinfo); + return std::string(time_buffer); + } + protected: td::actor::ActorId console_; Tokenizer tokenizer_; @@ -918,15 +932,6 @@ class GetOverlaysStatsQuery : public Query { static std::string get_help() { return "getoverlaysstats\tgets stats for all overlays"; } - static std::string time_to_human(int unixtime) { - char time_buffer[80]; - time_t rawtime = unixtime; - struct tm tInfo; - struct tm* timeinfo = localtime_r(&rawtime, &tInfo); - assert(timeinfo == &tInfo); - strftime(time_buffer, 80, "%c", timeinfo); - return std::string(time_buffer); - } std::string name() const override { return get_name(); } @@ -946,15 +951,6 @@ class GetOverlaysStatsJsonQuery : public Query { static std::string get_help() { return "getoverlaysstatsjson \tgets stats for all overlays and writes to json file"; } - static std::string time_to_human(int unixtime) { - char time_buffer[80]; - time_t rawtime = unixtime; - struct tm tInfo; - struct tm* timeinfo = localtime_r(&rawtime, &tInfo); - assert(timeinfo == &tInfo); - strftime(time_buffer, 80, "%c", timeinfo); - return std::string(time_buffer); - } std::string name() const override { return get_name(); } From fecf760aaec39f02ea95fa5c5eafd0222335274b Mon Sep 17 00:00:00 2001 From: neodiX42 Date: Tue, 5 Jul 2022 19:52:12 +0300 Subject: [PATCH 13/16] Adjust Mac OS non-arm build for upgraded abseil (#416) * Update and rename ubuntu-18.04-compile.yml to ubuntu-compile.yml * Update ubuntu-compile.yml segregate output binaries * Update macos-10.15-compile.yml Add testnet branch to github action and compilation flag -std=c++17 * Update macos-10.15-compile.yml remove branch filter * Update windows2019x64-compile.yml * Update windows2019x64-compile.yml add flag /std:c++17 * Update windows2019x64-compile.yml * Update windows2019x64-compile.yml * Update ubuntu-compile.yml Add -DCMAKE_CXX_FLAGS="-std=c++17" * use CMAKE_CXX_STANDARD * Update macos-10.15-compile.yml remove -std=c++17 flag * Update ubuntu-compile.yml remove -DCMAKE_CXX_FLAGS="-std=c++17" flag * Update ubuntu-compile.yml remove space * Update ubuntu-18.04-ton-ccpcheck.yml remove branch filter * Update docker-ubuntu-image.yml remove branch filter Co-authored-by: dungeon-master-666 --- .github/workflows/docker-ubuntu-image.yml | 4 +-- .github/workflows/macos-10.15-compile.yml | 4 +-- .../workflows/ubuntu-18.04-ton-ccpcheck.yml | 4 +-- CMakeLists.txt | 31 +++++-------------- 4 files changed, 10 insertions(+), 33 deletions(-) diff --git a/.github/workflows/docker-ubuntu-image.yml b/.github/workflows/docker-ubuntu-image.yml index 2f0d49ee..ad83c099 100644 --- a/.github/workflows/docker-ubuntu-image.yml +++ b/.github/workflows/docker-ubuntu-image.yml @@ -3,8 +3,6 @@ name: Docker Ubuntu 18.04 image on: workflow_dispatch: push: - branches: - - 'master' env: REGISTRY: ghcr.io @@ -36,4 +34,4 @@ jobs: with: push: true context: ./docker - tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest \ No newline at end of file + tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest diff --git a/.github/workflows/macos-10.15-compile.yml b/.github/workflows/macos-10.15-compile.yml index 351b3ea0..5f2d94b0 100644 --- a/.github/workflows/macos-10.15-compile.yml +++ b/.github/workflows/macos-10.15-compile.yml @@ -3,8 +3,6 @@ name: C/C++ CI macOS-10.15 Compile on: workflow_dispatch: push: - branches: - - 'master' jobs: build: @@ -40,4 +38,4 @@ jobs: uses: actions/upload-artifact@master with: name: ton-macos-binaries - path: artifacts \ No newline at end of file + path: artifacts diff --git a/.github/workflows/ubuntu-18.04-ton-ccpcheck.yml b/.github/workflows/ubuntu-18.04-ton-ccpcheck.yml index 97c46efb..e8d4548e 100644 --- a/.github/workflows/ubuntu-18.04-ton-ccpcheck.yml +++ b/.github/workflows/ubuntu-18.04-ton-ccpcheck.yml @@ -3,8 +3,6 @@ name: Ubuntu 18.04 TON ccpcheck on: workflow_dispatch: push: - branches: - - 'master' jobs: build: @@ -28,4 +26,4 @@ jobs: uses: actions/upload-artifact@v1 with: name: ton-ccpcheck-report - path: output \ No newline at end of file + path: output diff --git a/CMakeLists.txt b/CMakeLists.txt index da1945ff..d8f9deb3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +cmake_minimum_required(VERSION 3.1 FATAL_ERROR) project(TON VERSION 0.5 LANGUAGES C CXX) set(CMAKE_POSITION_INDEPENDENT_CODE ON) @@ -79,6 +79,10 @@ else() set(HAVE_SSE42 FALSE) endif() +set(CMAKE_CXX_STANDARD 14) +set(CMAKE_CXX_STANDARD_REQUIRED TRUE) +set(CMAKE_CXX_EXTENSIONS FALSE) + #BEGIN internal option(TON_ONLY_TONLIB "Use \"ON\" to build only tonlib." OFF) if (TON_ONLY_TONLIB) @@ -99,6 +103,7 @@ set(TON_ARCH "native" CACHE STRING "Architecture, will be passed to -march=") if (TON_USE_ABSEIL) message("Add abseil-cpp") + set(ABSL_PROPAGATE_CXX_STD TRUE) add_subdirectory(third-party/abseil-cpp EXCLUDE_FROM_ALL) set(ABSL_FOUND 1) endif() @@ -178,26 +183,6 @@ endif() include(CheckCXXCompilerFlag) -if (GCC OR CLANG OR INTEL) - if (WIN32 AND INTEL) - set(STD14_FLAG /Qstd=c++14) - else() - set(STD14_FLAG -std=c++14) - endif() - check_cxx_compiler_flag(${STD14_FLAG} HAVE_STD14) - if (NOT HAVE_STD14) - string(REPLACE "c++14" "c++1y" STD14_FLAG "${STD14_FLAG}") - check_cxx_compiler_flag(${STD14_FLAG} HAVE_STD1Y) - set(HAVE_STD14 ${HAVE_STD1Y}) - endif() -elseif (MSVC) - set(HAVE_STD14 MSVC_VERSION>=1900) -endif() - -if (NOT HAVE_STD14) - message(FATAL_ERROR "No C++14 support in the compiler. Please upgrade the compiler.") -endif() - set(CMAKE_THREAD_PREFER_PTHREAD ON) set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) @@ -238,7 +223,7 @@ if (MSVC) add_definitions(-D_SCL_SECURE_NO_WARNINGS -D_CRT_SECURE_NO_WARNINGS) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /MP /W4 /wd4100 /wd4127 /wd4324 /wd4456 /wd4457 /wd4458 /wd4505 /wd4702") elseif (CLANG OR GCC) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${STD14_FLAG} -fno-omit-frame-pointer") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer") if (APPLE) #use "-Wl,-exported_symbols_list,${CMAKE_CURRENT_SOURCE_DIR}/export_list" for exported symbols set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fvisibility=hidden -Wl,-dead_strip,-x,-S") @@ -251,8 +236,6 @@ elseif (CLANG OR GCC) set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--exclude-libs,ALL") endif() endif() -elseif (INTEL) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${STD14_FLAG}") endif() if (WIN32) From 40cec56e28689a6e87f634ef852cf0bb36ea0bab Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Thu, 4 Aug 2022 14:48:19 +0300 Subject: [PATCH 14/16] A series of FunC improvements (#378) * Don't use IFJMP/IFNOTJMP in inline functions * Fix incorrect ifelse branch code generation https://github.com/ton-blockchain/ton/issues/374 * Make generate_code_all clearer * Don't replace IFJMP with IF in inner blocks in inline functions * Allow unbalance if/else by using RETALT * Fix wrong PUSHCONT * Bugfix in IF code generation for inline functions * Fix unbalanced if/else * Bugfix and improvements in code generation * Fix analyzing while(0) in func https://github.com/ton-blockchain/ton/issues/377 * FunC and Asm.fif: Fix inlining large functions https://github.com/ton-blockchain/ton/issues/375 Co-authored-by: SpyCheese --- crypto/fift/lib/Asm.fif | 12 +++- crypto/func/analyzer.cpp | 2 +- crypto/func/codegen.cpp | 151 +++++++++++++++++++++------------------ crypto/func/func.cpp | 26 +++++-- crypto/func/func.h | 34 ++++++++- 5 files changed, 144 insertions(+), 81 deletions(-) diff --git a/crypto/fift/lib/Asm.fif b/crypto/fift/lib/Asm.fif index 71c6a956..5ae44c11 100644 --- a/crypto/fift/lib/Asm.fif +++ b/crypto/fift/lib/Asm.fif @@ -1,15 +1,17 @@ library TVM_Asm // simple TVM Assembler variable @atend +variable @was-split +false @was-split ! { "not in asm context" abort } @atend ! { `normal eq? not abort"must be terminated by }>" } : @normal? { @atend @ 1 { @atend ! @normal? } does @atend ! } : @pushatend { @pushatend +{ false @was-split ! `normal @endblk } : }> { }> b> } : }>c { }>c s -{ @atend @ 2 { @atend ! rot b> ref, swap @endblk } does @atend ! ref, swap @endblk } does @atend ! c & ops) { // block1 never executed op.block0->last().next = std::move(op.next); ops = std::move(op.block0); - return false; + return prune_unreachable(ops); } else if (c_var && c_var->always_true()) { if (!prune_unreachable(op.block1)) { // block1 never returns diff --git a/crypto/func/codegen.cpp b/crypto/func/codegen.cpp index 3e58b995..1f504bcd 100644 --- a/crypto/func/codegen.cpp +++ b/crypto/func/codegen.cpp @@ -277,12 +277,16 @@ bool Op::generate_code_step(Stack& stack) { stack.drop_vars_except(var_info); stack.opt_show(); const auto& next_var_info = next->var_info; + bool inline_func = stack.mode & Stack::_InlineFunc; switch (cl) { case _Nop: case _Import: return true; case _Return: { stack.enforce_state(left); + if (stack.o.retalt_ && (stack.mode & Stack::_NeedRetAlt)) { + stack.o << "RETALT"; + } stack.opt_show(); return false; } @@ -532,9 +536,7 @@ bool Op::generate_code_step(Stack& stack) { return true; } if (!next->noreturn() && (block0->noreturn() != block1->noreturn())) { - // simple fix of unbalanced returns in if/else branches - // (to be replaced with a finer condition working in loop bodies) - throw src::ParseError{where, "`if` and `else` branches should both return or both not return"}; + stack.o.retalt_ = true; } var_idx_t x = left[0]; stack.rearrange_top(x, var_info[x] && var_info[x]->is_last()); @@ -542,71 +544,57 @@ bool Op::generate_code_step(Stack& stack) { stack.opt_show(); stack.s.pop_back(); stack.modified(); - if (block1->is_empty()) { + if (inline_func && (block0->noreturn() || block1->noreturn())) { + bool is0 = block0->noreturn(); + Op* block_noreturn = is0 ? block0.get() : block1.get(); + Op* block_other = is0 ? block1.get() : block0.get(); + stack.mode &= ~Stack::_InlineFunc; + stack.o << (is0 ? "IF:<{" : "IFNOT:<{"); + stack.o.indent(); + Stack stack_copy{stack}; + block_noreturn->generate_code_all(stack_copy); + stack.o.undent(); + stack.o << "}>ELSE<{"; + stack.o.indent(); + block_other->generate_code_all(stack); + if (!block_other->noreturn()) { + next->generate_code_all(stack); + } + stack.o.undent(); + stack.o << "}>"; + return false; + } + if (block1->is_empty() || block0->is_empty()) { + bool is0 = block1->is_empty(); + Op* block = is0 ? block0.get() : block1.get(); // if (left) block0; ... - if (block0->noreturn()) { - stack.o << "IFJMP:<{"; - stack.o.indent(); - Stack stack_copy{stack}; - block0->generate_code_all(stack_copy); - stack.o.undent(); - stack.o << "}>"; - return true; - } - stack.o << "IF:<{"; - stack.o.indent(); - Stack stack_copy{stack}, stack_target{stack}; - stack_target.disable_output(); - stack_target.drop_vars_except(next->var_info); - block0->generate_code_all(stack_copy); - stack_copy.drop_vars_except(var_info); - stack_copy.opt_show(); - if (stack_copy == stack) { - stack.o.undent(); - stack.o << "}>"; - return true; - } - // stack_copy.drop_vars_except(next->var_info); - stack_copy.enforce_state(stack_target.vars()); - stack_copy.opt_show(); - if (stack_copy.vars() == stack.vars()) { - stack.o.undent(); - stack.o << "}>"; - stack.merge_const(stack_copy); - return true; - } - stack.o.undent(); - stack.o << "}>ELSE<{"; - stack.o.indent(); - stack.merge_state(stack_copy); - stack.opt_show(); - stack.o.undent(); - stack.o << "}>"; - return true; - } - if (block0->is_empty()) { // if (!left) block1; ... - if (block1->noreturn()) { - stack.o << "IFNOTJMP:<{"; + if (block->noreturn()) { + stack.o << (is0 ? "IFJMP:<{" : "IFNOTJMP:<{"); stack.o.indent(); Stack stack_copy{stack}; - block1->generate_code_all(stack_copy); + stack_copy.mode &= ~Stack::_InlineFunc; + stack_copy.mode |= next->noreturn() ? 0 : Stack::_NeedRetAlt; + block->generate_code_all(stack_copy); stack.o.undent(); stack.o << "}>"; return true; } - stack.o << "IFNOT:<{"; + stack.o << (is0 ? "IF:<{" : "IFNOT:<{"); stack.o.indent(); Stack stack_copy{stack}, stack_target{stack}; stack_target.disable_output(); stack_target.drop_vars_except(next->var_info); - block1->generate_code_all(stack_copy); + stack_copy.mode &= ~Stack::_InlineFunc; + block->generate_code_all(stack_copy); stack_copy.drop_vars_except(var_info); stack_copy.opt_show(); - if (stack_copy.vars() == stack.vars()) { + if ((is0 && stack_copy == stack) || (!is0 && stack_copy.vars() == stack.vars())) { stack.o.undent(); stack.o << "}>"; - stack.merge_const(stack_copy); + if (!is0) { + stack.merge_const(stack_copy); + } return true; } // stack_copy.drop_vars_except(next->var_info); @@ -627,33 +615,32 @@ bool Op::generate_code_step(Stack& stack) { stack.o << "}>"; return true; } - if (block0->noreturn()) { - stack.o << "IFJMP:<{"; + if (block0->noreturn() || block1->noreturn()) { + bool is0 = block0->noreturn(); + Op* block_noreturn = is0 ? block0.get() : block1.get(); + Op* block_other = is0 ? block1.get() : block0.get(); + stack.o << (is0 ? "IFJMP:<{" : "IFNOTJMP:<{"); stack.o.indent(); Stack stack_copy{stack}; - block0->generate_code_all(stack_copy); + stack_copy.mode &= ~Stack::_InlineFunc; + stack_copy.mode |= (block_other->noreturn() || next->noreturn()) ? 0 : Stack::_NeedRetAlt; + block_noreturn->generate_code_all(stack_copy); stack.o.undent(); stack.o << "}>"; - return block1->generate_code_all(stack); - } - if (block1->noreturn()) { - stack.o << "IFNOTJMP:<{"; - stack.o.indent(); - Stack stack_copy{stack}; - block1->generate_code_all(stack_copy); - stack.o.undent(); - stack.o << "}>"; - return block0->generate_code_all(stack); + block_other->generate_code_all(stack); + return !block_other->noreturn(); } stack.o << "IF:<{"; stack.o.indent(); Stack stack_copy{stack}; + stack_copy.mode &= ~Stack::_InlineFunc; block0->generate_code_all(stack_copy); stack_copy.drop_vars_except(next->var_info); stack_copy.opt_show(); stack.o.undent(); stack.o << "}>ELSE<{"; stack.o.indent(); + stack.mode &= ~Stack::_InlineFunc; block1->generate_code_all(stack); stack.merge_state(stack_copy); stack.opt_show(); @@ -669,11 +656,16 @@ bool Op::generate_code_step(Stack& stack) { stack.opt_show(); stack.s.pop_back(); stack.modified(); + if (block0->noreturn()) { + stack.o.retalt_ = true; + } if (true || !next->is_empty()) { stack.o << "REPEAT:<{"; stack.o.indent(); stack.forget_const(); StackLayout layout1 = stack.vars(); + stack.mode &= ~Stack::_InlineFunc; + stack.mode |= Stack::_NeedRetAlt; block0->generate_code_all(stack); stack.enforce_state(std::move(layout1)); stack.opt_show(); @@ -693,11 +685,16 @@ bool Op::generate_code_step(Stack& stack) { case _Again: { stack.drop_vars_except(block0->var_info); stack.opt_show(); - if (!next->is_empty()) { + if (block0->noreturn()) { + stack.o.retalt_ = true; + } + if (!next->is_empty() || inline_func) { stack.o << "AGAIN:<{"; stack.o.indent(); stack.forget_const(); StackLayout layout1 = stack.vars(); + stack.mode &= ~Stack::_InlineFunc; + stack.mode |= Stack::_NeedRetAlt; block0->generate_code_all(stack); stack.enforce_state(std::move(layout1)); stack.opt_show(); @@ -717,11 +714,16 @@ bool Op::generate_code_step(Stack& stack) { case _Until: { // stack.drop_vars_except(block0->var_info); // stack.opt_show(); + if (block0->noreturn()) { + stack.o.retalt_ = true; + } if (true || !next->is_empty()) { stack.o << "UNTIL:<{"; stack.o.indent(); stack.forget_const(); auto layout1 = stack.vars(); + stack.mode &= ~Stack::_InlineFunc; + stack.mode |= Stack::_NeedRetAlt; block0->generate_code_all(stack); layout1.push_back(left[0]); stack.enforce_state(std::move(layout1)); @@ -749,9 +751,14 @@ bool Op::generate_code_step(Stack& stack) { stack.opt_show(); StackLayout layout1 = stack.vars(); bool next_empty = false && next->is_empty(); + if (block0->noreturn()) { + stack.o.retalt_ = true; + } stack.o << "WHILE:<{"; stack.o.indent(); stack.forget_const(); + stack.mode &= ~Stack::_InlineFunc; + stack.mode |= Stack::_NeedRetAlt; block0->generate_code_all(stack); stack.rearrange_top(x, !next->var_info[x] && !block1->var_info[x]); stack.opt_show(); @@ -781,11 +788,12 @@ bool Op::generate_code_step(Stack& stack) { } } -bool Op::generate_code_all(Stack& stack) { - if (generate_code_step(stack) && next) { - return next->generate_code_all(stack); - } else { - return false; +void Op::generate_code_all(Stack& stack) { + int saved_mode = stack.mode; + auto cont = generate_code_step(stack); + stack.mode = (stack.mode & ~Stack::_ModeSave) | (saved_mode & Stack::_ModeSave); + if (cont && next) { + next->generate_code_all(stack); } } @@ -796,6 +804,7 @@ void CodeBlob::generate_code(AsmOpList& out, int mode) { stack.push_new_var(x); } ops->generate_code_all(stack); + stack.apply_wrappers(); if (!(mode & Stack::_DisableOpt)) { optimize_code(out); } diff --git a/crypto/func/func.cpp b/crypto/func/func.cpp index b260c885..b5d769cb 100644 --- a/crypto/func/func.cpp +++ b/crypto/func/func.cpp @@ -99,12 +99,28 @@ void generate_output_func(SymDef* func_sym) { if (verbosity >= 2) { std::cerr << "\n---------- resulting code for " << name << " -------------\n"; } + bool inline_func = (func_val->flags & 1); bool inline_ref = (func_val->flags & 2); - *outs << std::string(indent * 2, ' ') << name << " PROC" << (inline_ref ? "REF" : "") << ":<{\n"; - code.generate_code( - *outs, - (stack_layout_comments ? Stack::_StkCmt | Stack::_CptStkCmt : 0) | (opt_level < 2 ? Stack::_DisableOpt : 0), - indent + 1); + const char* modifier = ""; + if (inline_func) { + modifier = "INLINE"; + } else if (inline_ref) { + modifier = "REF"; + } + *outs << std::string(indent * 2, ' ') << name << " PROC" << modifier << ":<{\n"; + int mode = 0; + if (stack_layout_comments) { + mode |= Stack::_StkCmt | Stack::_CptStkCmt; + } + if (opt_level < 2) { + mode |= Stack::_DisableOpt; + } + auto fv = dynamic_cast(func_sym->value); + // Flags: 1 - inline, 2 - inline_ref + if (fv && (fv->flags & 1) && code.ops->noreturn()) { + mode |= Stack::_InlineFunc; + } + code.generate_code(*outs, mode, indent + 1); *outs << std::string(indent * 2, ' ') << "}>\n"; if (verbosity >= 2) { std::cerr << "--------------\n"; diff --git a/crypto/func/func.h b/crypto/func/func.h index 2f9877a0..886a21ed 100644 --- a/crypto/func/func.h +++ b/crypto/func/func.h @@ -613,7 +613,7 @@ struct Op { return !(flags & _Impure); } bool generate_code_step(Stack& stack); - bool generate_code_all(Stack& stack); + void generate_code_all(Stack& stack); Op& last() { return next ? next->last() : *this; } @@ -1121,6 +1121,7 @@ struct AsmOpList { int indent_{0}; const std::vector* var_names_{nullptr}; std::vector constants_; + bool retalt_{false}; void out(std::ostream& os, int mode = 0) const; AsmOpList(int indent = 0, const std::vector* var_names = nullptr) : indent_(indent), var_names_(var_names) { } @@ -1168,6 +1169,19 @@ struct AsmOpList { void set_indent(int new_indent) { indent_ = new_indent; } + void insert(size_t pos, std::string str) { + insert(pos, AsmOp(AsmOp::a_custom, 255, 255, str)); + } + void insert(size_t pos, const AsmOp& op) { + auto ip = list_.begin() + pos; + ip = list_.insert(ip, op); + ip->indent = (ip == list_.begin()) ? indent_ : (ip - 1)->indent; + } + void indent_all() { + for (auto &op : list_) { + ++op.indent; + } + } }; inline std::ostream& operator<<(std::ostream& os, const AsmOpList& op_list) { @@ -1498,7 +1512,12 @@ void optimize_code(AsmOpList& ops); struct Stack { StackLayoutExt s; AsmOpList& o; - enum { _StkCmt = 1, _CptStkCmt = 2, _DisableOpt = 4, _DisableOut = 128, _Shown = 256, _Garbage = -0x10000 }; + enum { + _StkCmt = 1, _CptStkCmt = 2, _DisableOpt = 4, _DisableOut = 128, _Shown = 256, + _InlineFunc = 512, _NeedRetAlt = 1024, + _ModeSave = _InlineFunc | _NeedRetAlt, + _Garbage = -0x10000 + }; int mode; Stack(AsmOpList& _o, int _mode = 0) : o(_o), mode(_mode) { } @@ -1571,6 +1590,17 @@ struct Stack { bool operator==(const Stack& y) const & { return s == y.s; } + void apply_wrappers() { + if (o.retalt_) { + o.insert(0, "SAMEALTSAVE"); + if (mode & _InlineFunc) { + o.indent_all(); + o.insert(0, "CONT:<{"); + o << "}>"; + o << "EXECUTE"; + } + } + } }; /* From 8d7f1bba7387a006e7e5a6f6eec3177d1f25b80c Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 8 Aug 2022 09:31:36 +0300 Subject: [PATCH 15/16] Update catchain (#432) * Update catchain * Update ton_api.tlo --- README.md | 51 ++- catchain/catchain-block.cpp | 14 +- catchain/catchain-block.hpp | 4 +- catchain/catchain-received-block.cpp | 149 ++++---- catchain/catchain-received-block.h | 37 +- catchain/catchain-received-block.hpp | 28 +- catchain/catchain-receiver-interface.h | 24 +- catchain/catchain-receiver-source.cpp | 21 +- catchain/catchain-receiver-source.h | 3 +- catchain/catchain-receiver-source.hpp | 8 +- catchain/catchain-receiver.cpp | 383 ++++++++++++-------- catchain/catchain-receiver.h | 10 +- catchain/catchain-receiver.hpp | 44 ++- catchain/catchain-types.h | 8 +- catchain/catchain.cpp | 79 ++-- catchain/catchain.h | 27 +- catchain/catchain.hpp | 34 +- crypto/block/block.tlb | 7 + crypto/block/mc-config.cpp | 48 ++- doc/catchain-dos.md | 22 ++ test/test-catchain.cpp | 10 +- tl/generate/scheme/ton_api.tl | 9 +- tl/generate/scheme/ton_api.tlo | Bin 66500 -> 66724 bytes ton/ton-types.h | 19 +- validator-session/validator-session-types.h | 9 +- validator-session/validator-session.cpp | 36 +- validator-session/validator-session.hpp | 10 +- 27 files changed, 617 insertions(+), 477 deletions(-) create mode 100644 doc/catchain-dos.md diff --git a/README.md b/README.md index df8fefa8..efa5e971 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,26 @@ -# TON +# TON CatChain -Main TON monorepo, which includes the code of the node/validator, lite-client, tonlib, FunC compiler, etc. - -## Updates flow: - -* **master branch** - mainnet is running on this stable branch. - - Only emergency updates, urgent updates, or updates that do not affect the main codebase (GitHub workflows / docker images / documentation) are committed directly to this branch. - -* **testnet branch** - testnet is running on this branch. The branch contains a set of new updates. After testing, the testnet branch is merged into the master branch and then a new set of updates is added to testnet branch. - -* **backlog** - other branches that are candidates to getting into the testnet branch in the next iteration. - -Usually, the response to your pull request will indicate which section it falls into. - - -## "Soft" Pull Request rules - -* Thou shall not merge your own PRs, at least one person should review the PR and merge it (4-eyes rule) -* Thou shall make sure that workflows are cleanly completed for your PR before considering merge - -## Workflows responsibility -If a CI workflow fails not because of your changes but workflow issues, try to fix it yourself or contact one of the persons listed below via Telegram messenger: - -* **C/C++ CI (ccpp-linux.yml)**: TBD -* **C/C++ CI Win64 Compile (ccpp-win64.yml)**: TBD +| Name | Fixed? | Comment | +|------|---|---------------------------------------| +|CA2-01| ✔️ | | +|CA2-02| ✔️ | | +|CA5-01| ✔️ | `hash(payload)` now also covers block data.
This behavior is enabled by a flag in config. | +|CA5-02| ✔️ | Size of a block is now limited (16kb).
Block height is now limited, and blocks from misbehaviouring nodes are now ignored.
See `doc/catchain-dos.md` for details. | +|CA5-03| ✔️ | | +|CA5-04| ✔️ | | +|CA5-05| ✔️ | | +|CA5-06| ✔️ | | +|CA5-07| ✔️ | | +|CA5-08| ✔️ | | +|CA5-09| ✔️ | | +|CA5-10| ✔️ | | +|CA5-11| ✔️ | | +|CAA-01| ✔️ | Fixed in the same way as CA5-01 (hash now covers block data). | +|CAA-02| ✔️ | | +|CAE-01| ✔️ | | +|CAN-01| ✔️ | | +|CAN-02| ✔️ | | +|CAT-01| ✔️ | | +|CAT-02| ✔️ | | +|CAT-03| ✔️ | | +|CAT-04| ✔️ | | diff --git a/catchain/catchain-block.cpp b/catchain/catchain-block.cpp index f4e76674..e12c89f9 100644 --- a/catchain/catchain-block.cpp +++ b/catchain/catchain-block.cpp @@ -22,8 +22,8 @@ namespace ton { namespace catchain { -std::unique_ptr CatChainBlock::create(td::uint32 src, td::uint32 fork, PublicKeyHash src_hash, - CatChainBlockHeight height, CatChainBlockHash hash, +std::unique_ptr CatChainBlock::create(td::uint32 src, td::uint32 fork, const PublicKeyHash &src_hash, + CatChainBlockHeight height, const CatChainBlockHash &hash, td::SharedSlice payload, CatChainBlock *prev, std::vector deps, std::vector vt) { @@ -31,10 +31,10 @@ std::unique_ptr CatChainBlock::create(td::uint32 src, td::uint32 std::move(deps), std::move(vt)); } -CatChainBlockImpl::CatChainBlockImpl(td::uint32 src, td::uint32 fork, PublicKeyHash src_hash, - CatChainBlockHeight height, CatChainBlockHash hash, td::SharedSlice payload, - CatChainBlock *prev, std::vector deps, - std::vector vt) +CatChainBlockImpl::CatChainBlockImpl(td::uint32 src, td::uint32 fork, const PublicKeyHash &src_hash, + CatChainBlockHeight height, const CatChainBlockHash &hash, + td::SharedSlice payload, CatChainBlock *prev, + std::vector deps, std::vector vt) : src_(src) , fork_(fork) , src_hash_(src_hash) @@ -47,7 +47,7 @@ CatChainBlockImpl::CatChainBlockImpl(td::uint32 src, td::uint32 fork, PublicKeyH } bool CatChainBlockImpl::is_descendant_of(CatChainBlock *block) { - auto fork = block->fork(); + td::uint32 fork = block->fork(); if (fork >= vt_.size()) { return false; } diff --git a/catchain/catchain-block.hpp b/catchain/catchain-block.hpp index 8d66d3c1..d37b4dc5 100644 --- a/catchain/catchain-block.hpp +++ b/catchain/catchain-block.hpp @@ -103,8 +103,8 @@ class CatChainBlockImpl : public CatChainBlock { bool is_descendant_of(CatChainBlock *block) override; - CatChainBlockImpl(td::uint32 src, td::uint32 fork, PublicKeyHash src_hash, CatChainBlockHeight height, - CatChainBlockHash hash, td::SharedSlice payload, CatChainBlock *prev, + CatChainBlockImpl(td::uint32 src, td::uint32 fork, const PublicKeyHash &src_hash, CatChainBlockHeight height, + const CatChainBlockHash &hash, td::SharedSlice payload, CatChainBlock *prev, std::vector deps, std::vector vt); }; diff --git a/catchain/catchain-received-block.cpp b/catchain/catchain-received-block.cpp index 7a17d3ee..5996e967 100644 --- a/catchain/catchain-received-block.cpp +++ b/catchain/catchain-received-block.cpp @@ -33,7 +33,7 @@ void CatChainReceivedBlockImpl::initialize(tl_object_ptr 0); + CHECK(!payload_.empty()); prev_ = dynamic_cast(chain_->create_block(std::move(block->data_->prev_))); CHECK(prev_ != nullptr); @@ -51,7 +51,7 @@ void CatChainReceivedBlockImpl::initialize(tl_object_ptris_ill()) { set_ill(); return; @@ -63,17 +63,17 @@ void CatChainReceivedBlockImpl::initialize(tl_object_ptrdelivered()) { pending_deps++; } else { - update_deps(prev_); + update_vt(prev_); } if (!prev_->delivered()) { prev_->add_rev_dep(this); } } - for (auto &X : block_deps_) { + for (CatChainReceivedBlockImpl *X : block_deps_) { if (!X->delivered()) { pending_deps++; } else { - update_deps(X); + update_vt(X); } if (!X->delivered()) { X->add_rev_dep(this); @@ -121,11 +121,11 @@ void CatChainReceivedBlockImpl::initialize_fork() { } } - if (deps_.size() < fork_id_ + 1) { - deps_.resize(fork_id_ + 1, 0); + if (vt_.size() < fork_id_ + 1) { + vt_.resize(fork_id_ + 1, 0); } - CHECK(deps_[fork_id_] < height_); - deps_[fork_id_] = height_; + CHECK(vt_[fork_id_] < height_); + vt_[fork_id_] = height_; } void CatChainReceivedBlockImpl::pre_deliver(ton_api::catchain_block_data_fork &b) { @@ -153,7 +153,7 @@ void CatChainReceivedBlockImpl::pre_deliver(ton_api::catchain_block_data_fork &b return; } - auto S = chain_->get_source(b.left_->src_); + CatChainReceiverSource *S = chain_->get_source(b.left_->src_); S->on_found_fork_proof( create_serialize_tl_object(std::move(b.left_), std::move(b.right_))); S->blame(fork_id_, height_); @@ -173,15 +173,15 @@ void CatChainReceivedBlockImpl::pre_deliver() { CHECK(pending_deps_ == 0); CHECK(in_db_); - auto M = chain_->get_source(source_id_); + CatChainReceiverSource *M = chain_->get_source(source_id_); - auto d = prev_ ? &prev_->deps_ : nullptr; + std::vector *d = prev_ ? &prev_->vt_ : nullptr; - for (auto &X : block_deps_) { - auto S = chain_->get_source(X->get_source_id()); - auto &f = S->get_forks(); + for (CatChainReceivedBlockImpl *X : block_deps_) { + CatChainReceiverSource *S = chain_->get_source(X->get_source_id()); + const std::vector &f = S->get_forks(); if (d) { - auto &dd = *d; + const std::vector &dd = *d; if (X->get_fork_id() < dd.size() && X->get_height() <= dd[X->get_fork_id()]) { VLOG(CATCHAIN_WARNING) << this << ": has direct dep from source " << X->get_source_id() << " and prev block " << " has newer indirect dep"; @@ -190,8 +190,8 @@ void CatChainReceivedBlockImpl::pre_deliver() { } } if (S->blamed() && d) { - auto &dd = *d; - for (auto x : f) { + const std::vector &dd = *d; + for (td::uint32 x : f) { if (x != X->get_fork_id() && dd.size() > x && dd[x] > 0) { VLOG(CATCHAIN_WARNING) << this << ": has direct dep from source " << X->get_source_id() << " and prev block " << " has indirect dep to another fork of this source " << x << " " << X->get_fork_id() @@ -201,7 +201,7 @@ void CatChainReceivedBlockImpl::pre_deliver() { return; } } - auto v = S->get_blamed_heights(); + std::vector v = S->get_blamed_heights(); for (size_t i = 0; i < v.size() && i < dd.size(); i++) { if (v[i] > 0 && dd[i] >= v[i]) { @@ -220,7 +220,7 @@ void CatChainReceivedBlockImpl::pre_deliver() { if (X.is_error()) { is_custom_ = true; } else { - ton_api::downcast_call(*X.move_as_ok().get(), [Self = this](auto &obj) { Self->pre_deliver(obj); }); + ton_api::downcast_call(*X.move_as_ok(), [Self = this](auto &obj) { Self->pre_deliver(obj); }); } } @@ -237,7 +237,7 @@ void CatChainReceivedBlockImpl::deliver() { state_ = bs_delivered; VLOG(CATCHAIN_DEBUG) << this << ": delivered"; - for (auto &B : rev_deps_) { + for (CatChainReceivedBlockImpl *B : rev_deps_) { B->dep_delivered(this); } rev_deps_.clear(); @@ -250,10 +250,10 @@ void CatChainReceivedBlockImpl::set_ill() { return; } VLOG(CATCHAIN_WARNING) << this << ": got ill"; - auto M = chain_->get_source(source_id_); + CatChainReceiverSource *M = chain_->get_source(source_id_); M->blame(); state_ = bs_ill; - for (auto &B : rev_deps_) { + for (CatChainReceivedBlockImpl *B : rev_deps_) { B->dep_ill(this); } } @@ -262,14 +262,14 @@ void CatChainReceivedBlockImpl::dep_ill(CatChainReceivedBlockImpl *block) { set_ill(); } -void CatChainReceivedBlockImpl::update_deps(CatChainReceivedBlockImpl *block) { - auto &d = block->deps_; - if (d.size() > deps_.size()) { - deps_.resize(d.size(), 0); +void CatChainReceivedBlockImpl::update_vt(CatChainReceivedBlockImpl *block) { + const std::vector &d = block->vt_; + if (d.size() > vt_.size()) { + vt_.resize(d.size(), 0); } for (size_t i = 0; i < d.size(); i++) { - if (deps_[i] < d[i]) { - deps_[i] = d[i]; + if (vt_[i] < d[i]) { + vt_[i] = d[i]; } } } @@ -279,7 +279,7 @@ void CatChainReceivedBlockImpl::dep_delivered(CatChainReceivedBlockImpl *block) return; } CHECK(!block->is_ill()); - update_deps(block); + update_vt(block); pending_deps_--; if (pending_deps_ == 0 && in_db_) { schedule(); @@ -332,35 +332,37 @@ void CatChainReceivedBlockImpl::find_pending_deps(std::vector if (prev_) { prev_->find_pending_deps(vec, max_size); } - for (auto &X : block_deps_) { + for (CatChainReceivedBlockImpl *X : block_deps_) { X->find_pending_deps(vec, max_size); } } -tl_object_ptr CatChainReceivedBlock::block_id(CatChainReceiver *chain, - tl_object_ptr &block, - td::Slice payload) { +tl_object_ptr CatChainReceivedBlock::block_id( + const CatChainReceiver *chain, const tl_object_ptr &block, const td::Slice &payload) { + td::Bits256 hash = data_payload_hash(chain, block->data_, payload); return create_tl_object(block->incarnation_, chain->get_source_hash(block->src_).tl(), - block->height_, sha256_bits256(payload)); + block->height_, hash); } tl_object_ptr CatChainReceivedBlock::block_id( - CatChainReceiver *chain, tl_object_ptr &block) { + const CatChainReceiver *chain, const tl_object_ptr &block) { return create_tl_object( chain->get_incarnation(), chain->get_source_hash(block->src_).tl(), block->height_, block->data_hash_); } -CatChainBlockHash CatChainReceivedBlock::block_hash(CatChainReceiver *chain, - tl_object_ptr &block, td::Slice payload) { +CatChainBlockHash CatChainReceivedBlock::block_hash(const CatChainReceiver *chain, + const tl_object_ptr &block, + const td::Slice &payload) { return get_tl_object_sha_bits256(block_id(chain, block, payload)); } -CatChainBlockHash CatChainReceivedBlock::block_hash(CatChainReceiver *chain, - tl_object_ptr &block) { +CatChainBlockHash CatChainReceivedBlock::block_hash(const CatChainReceiver *chain, + const tl_object_ptr &block) { return get_tl_object_sha_bits256(block_id(chain, block)); } -td::Status CatChainReceivedBlock::pre_validate_block(CatChainReceiver *chain, - tl_object_ptr &block, td::Slice payload) { +td::Status CatChainReceivedBlock::pre_validate_block(const CatChainReceiver *chain, + const tl_object_ptr &block, + const td::Slice &payload) { CHECK(block->incarnation_ == chain->get_incarnation()); if (block->height_ <= 0) { return td::Status::Error(ErrorCode::protoviolation, std::string("bad height ") + std::to_string(block->height_)); @@ -397,7 +399,7 @@ td::Status CatChainReceivedBlock::pre_validate_block(CatChainReceiver *chain, std::set used; used.insert(block->src_); - for (auto &X : block->data_->deps_) { + for (const auto &X : block->data_->deps_) { if (used.find(X->src_) != used.end()) { return td::Status::Error(ErrorCode::protoviolation, "two deps from same source"); } @@ -405,19 +407,19 @@ td::Status CatChainReceivedBlock::pre_validate_block(CatChainReceiver *chain, } TRY_STATUS(chain->validate_block_sync(block->data_->prev_)); - for (auto &X : block->data_->deps_) { + for (const auto &X : block->data_->deps_) { TRY_STATUS(chain->validate_block_sync(X)); } - if (payload.size() == 0) { + if (payload.empty()) { return td::Status::Error(ErrorCode::protoviolation, "empty payload"); } return td::Status::OK(); } -td::Status CatChainReceivedBlock::pre_validate_block(CatChainReceiver *chain, - tl_object_ptr &block) { +td::Status CatChainReceivedBlock::pre_validate_block(const CatChainReceiver *chain, + const tl_object_ptr &block) { if (block->height_ < 0) { return td::Status::Error(ErrorCode::protoviolation, std::string("bad height ") + std::to_string(block->height_)); } @@ -430,7 +432,7 @@ td::Status CatChainReceivedBlock::pre_validate_block(CatChainReceiver *chain, return td::Status::Error(ErrorCode::protoviolation, std::string("bad src (first block) ") + std::to_string(block->src_)); } - if (block->data_hash_ != chain->get_incarnation() || block->signature_.size() != 0) { + if (block->data_hash_ != chain->get_incarnation() || !block->signature_.empty()) { return td::Status::Error(ErrorCode::protoviolation, std::string("bad first block")); } } @@ -443,9 +445,10 @@ tl_object_ptr CatChainReceivedBlockImpl::export_tl() co CHECK(height_ > 0); std::vector> deps; - for (auto &B : block_deps_) { + for (CatChainReceivedBlockImpl *B : block_deps_) { deps.push_back(B->export_tl_dep()); } + CHECK(deps.size() <= chain_->opts().max_deps) return create_tl_object( chain_->get_incarnation(), source_id_, height_, @@ -454,35 +457,34 @@ tl_object_ptr CatChainReceivedBlockImpl::export_tl() co } tl_object_ptr CatChainReceivedBlockImpl::export_tl_dep() const { - return create_tl_object(source_id_, height_, data_hash_, + return create_tl_object(source_id_, height_, data_payload_hash_, signature_.clone_as_buffer_slice()); } -CatChainReceivedBlockImpl::CatChainReceivedBlockImpl(td::uint32 source_id, CatChainSessionId hash, +CatChainReceivedBlockImpl::CatChainReceivedBlockImpl(td::uint32 source_id, const CatChainBlockPayloadHash &hash, CatChainReceiver *chain) { chain_ = chain; state_ = bs_delivered; fork_id_ = 0; source_id_ = source_id; - data_ = nullptr; - prev_ = nullptr; height_ = 0; - data_hash_ = hash; - hash_ = get_tl_object_sha_bits256(create_tl_object( - chain->get_incarnation(), chain->get_incarnation(), height_, data_hash_)); + data_payload_hash_ = hash; + block_id_hash_ = get_tl_object_sha_bits256(create_tl_object( + chain->get_incarnation(), chain->get_incarnation(), height_, data_payload_hash_)); } CatChainReceivedBlockImpl::CatChainReceivedBlockImpl(tl_object_ptr block, td::SharedSlice payload, CatChainReceiver *chain) { chain_ = chain; - data_hash_ = sha256_bits256(payload.as_slice()); - hash_ = get_tl_object_sha_bits256(create_tl_object( - block->incarnation_, chain->get_source_hash(block->src_).tl(), block->height_, data_hash_)); + data_payload_hash_ = data_payload_hash(chain, block->data_, payload); + block_id_hash_ = get_tl_object_sha_bits256(create_tl_object( + block->incarnation_, chain->get_source_hash(block->src_).tl(), block->height_, data_payload_hash_)); height_ = block->height_; source_id_ = block->src_; + CHECK(height_ <= get_max_block_height(chain->opts(), chain->get_sources_cnt())); - auto S = chain_->get_source(source_id_); + CatChainReceiverSource *S = chain_->get_source(source_id_); S->on_new_block(this); initialize(std::move(block), std::move(payload)); @@ -491,14 +493,14 @@ CatChainReceivedBlockImpl::CatChainReceivedBlockImpl(tl_object_ptr block, CatChainReceiver *chain) { chain_ = chain; - data_hash_ = block->data_hash_; + data_payload_hash_ = block->data_hash_; source_id_ = block->src_; signature_ = td::SharedSlice{block->signature_.as_slice()}; - hash_ = get_tl_object_sha_bits256(create_tl_object( - chain_->get_incarnation(), chain_->get_source_hash(source_id_).tl(), block->height_, data_hash_)); + block_id_hash_ = get_tl_object_sha_bits256(create_tl_object( + chain_->get_incarnation(), chain_->get_source_hash(source_id_).tl(), block->height_, data_payload_hash_)); height_ = block->height_; - auto S = chain_->get_source(source_id_); + CatChainReceiverSource *S = chain_->get_source(source_id_); S->on_new_block(this); } @@ -513,9 +515,24 @@ std::unique_ptr CatChainReceivedBlock::create(tl_object_p } std::unique_ptr CatChainReceivedBlock::create_root(td::uint32 source_id, - CatChainSessionId data_hash, + CatChainSessionId session_id, CatChainReceiver *chain) { - return std::make_unique(source_id, data_hash, chain); + return std::make_unique(source_id, session_id, chain); +} + +CatChainBlockPayloadHash CatChainReceivedBlock::data_payload_hash( + const CatChainReceiver *chain, const tl_object_ptr &data, const td::Slice &payload) { + td::Bits256 hash = sha256_bits256(payload); + if (chain->opts().block_hash_covers_data) { + td::Bits256 data_hash = get_tl_object_sha_bits256(data); + char buf[32 * 2]; + CHECK(hash.as_array().size() == 32 && data_hash.as_array().size() == 32); + std::copy(hash.as_array().begin(), hash.as_array().end(), buf); + std::copy(data_hash.as_array().begin(), data_hash.as_array().end(), buf + 32); + return sha256_bits256(td::Slice(buf, buf + 64)); + } else { + return hash; + } } } // namespace catchain diff --git a/catchain/catchain-received-block.h b/catchain/catchain-received-block.h index 8a5dfa90..3c4f5d76 100644 --- a/catchain/catchain-received-block.h +++ b/catchain/catchain-received-block.h @@ -30,7 +30,6 @@ namespace catchain { class CatChainReceiver; class CatChainReceiverSource; -class CatChainReceiverFork; class CatChainReceivedBlock { public: @@ -43,7 +42,7 @@ class CatChainReceivedBlock { virtual CatChainReceivedBlock *get_prev() const = 0; virtual CatChainBlockHash get_prev_hash() const = 0; - virtual const std::vector &get_deps() const = 0; + virtual const std::vector &get_vt() const = 0; virtual std::vector get_dep_hashes() const = 0; virtual CatChainReceiver *get_chain() const = 0; @@ -56,6 +55,8 @@ class CatChainReceivedBlock { virtual void find_pending_deps(std::vector &vec, td::uint32 max_size) const = 0; + virtual bool has_rev_deps() const = 0; + public: // state virtual bool initialized() const = 0; @@ -76,20 +77,28 @@ class CatChainReceivedBlock { td::SharedSlice payload, CatChainReceiver *chain); static std::unique_ptr create(tl_object_ptr block, CatChainReceiver *chain); - static std::unique_ptr create_root(td::uint32 source_id, CatChainBlockPayloadHash data_hash, + static std::unique_ptr create_root(td::uint32 source_id, CatChainSessionId session_id, CatChainReceiver *chain); - static tl_object_ptr block_id(CatChainReceiver *chain, - tl_object_ptr &block, - td::Slice payload); - static tl_object_ptr block_id(CatChainReceiver *chain, - tl_object_ptr &block); - static CatChainBlockHash block_hash(CatChainReceiver *chain, tl_object_ptr &block, - td::Slice payload); - static CatChainBlockHash block_hash(CatChainReceiver *chain, tl_object_ptr &block); - static td::Status pre_validate_block(CatChainReceiver *chain, tl_object_ptr &block, - td::Slice payload); - static td::Status pre_validate_block(CatChainReceiver *chain, tl_object_ptr &block); + static tl_object_ptr block_id(const CatChainReceiver *chain, + const tl_object_ptr &block, + const td::Slice &payload); + static tl_object_ptr block_id(const CatChainReceiver *chain, + const tl_object_ptr &block); + static CatChainBlockHash block_hash(const CatChainReceiver *chain, + const tl_object_ptr &block, + const td::Slice &payload); + static CatChainBlockHash block_hash(const CatChainReceiver *chain, + const tl_object_ptr &block); + static td::Status pre_validate_block(const CatChainReceiver *chain, + const tl_object_ptr &block, + const td::Slice &payload); + static td::Status pre_validate_block(const CatChainReceiver *chain, + const tl_object_ptr &block); + static CatChainBlockPayloadHash data_payload_hash(const CatChainReceiver *chain, + const tl_object_ptr &data, + const td::Slice &payload); + virtual ~CatChainReceivedBlock() = default; }; diff --git a/catchain/catchain-received-block.hpp b/catchain/catchain-received-block.hpp index 27f9ec61..f1270ab0 100644 --- a/catchain/catchain-received-block.hpp +++ b/catchain/catchain-received-block.hpp @@ -26,15 +26,14 @@ namespace catchain { class CatChainReceiver; class CatChainReceiverSource; -class CatChainReceiverFork; -class CatChainReceivedBlockImpl : public CatChainReceivedBlock { +class CatChainReceivedBlockImpl final : public CatChainReceivedBlock { public: const td::SharedSlice &get_payload() const override { return payload_; } CatChainBlockHash get_hash() const override { - return hash_; + return block_id_hash_; } const td::SharedSlice &get_signature() const override { return signature_; @@ -46,8 +45,8 @@ class CatChainReceivedBlockImpl : public CatChainReceivedBlock { CatChainReceivedBlock *get_prev() const override; CatChainBlockHash get_prev_hash() const override; - const std::vector &get_deps() const override { - return deps_; + const std::vector &get_vt() const override { + return vt_; } std::vector get_dep_hashes() const override; @@ -69,6 +68,10 @@ class CatChainReceivedBlockImpl : public CatChainReceivedBlock { void find_pending_deps(std::vector &vec, td::uint32 max_size) const override; + bool has_rev_deps() const override { + return !rev_deps_.empty(); + } + public: bool initialized() const override { return state_ >= bs_initialized; @@ -114,7 +117,7 @@ class CatChainReceivedBlockImpl : public CatChainReceivedBlock { CatChainReceiver *chain); CatChainReceivedBlockImpl(tl_object_ptr block, CatChainReceiver *chain); - CatChainReceivedBlockImpl(td::uint32 source_id, CatChainSessionId hash, CatChainReceiver *chain); + CatChainReceivedBlockImpl(td::uint32 source_id, const CatChainSessionId &hash, CatChainReceiver *chain); private: enum State { @@ -124,31 +127,28 @@ class CatChainReceivedBlockImpl : public CatChainReceivedBlock { bs_delivered, } state_ = bs_none; - void update_deps(CatChainReceivedBlockImpl *block); + void update_vt(CatChainReceivedBlockImpl *block); void add_rev_dep(CatChainReceivedBlockImpl *block); - void add_child_dep(CatChainReceivedBlockImpl *block); void initialize_fork(); - void on_ready_to_deliver(); td::uint32 fork_id_{0}; td::uint32 source_id_; CatChainReceiver *chain_; - tl_object_ptr data_; td::SharedSlice payload_; - CatChainBlockHash hash_; - CatChainBlockPayloadHash data_hash_; + CatChainBlockHash block_id_hash_{}; + CatChainBlockPayloadHash data_payload_hash_{}; - CatChainReceivedBlockImpl *prev_; + CatChainReceivedBlockImpl *prev_ = nullptr; CatChainBlockHeight height_; CatChainReceivedBlockImpl *next_ = nullptr; std::vector block_deps_; - std::vector deps_; + std::vector vt_; td::SharedSlice signature_; diff --git a/catchain/catchain-receiver-interface.h b/catchain/catchain-receiver-interface.h index af983402..c8f1ef66 100644 --- a/catchain/catchain-receiver-interface.h +++ b/catchain/catchain-receiver-interface.h @@ -35,33 +35,33 @@ class CatChainReceiverInterface : public td::actor::Actor { CatChainBlockHash prev, std::vector deps, std::vector vt, td::SharedSlice data) = 0; virtual void blame(td::uint32 src_id) = 0; - virtual void on_custom_message(PublicKeyHash src, td::BufferSlice data) = 0; - virtual void on_custom_query(PublicKeyHash src, td::BufferSlice data, td::Promise promise) = 0; - virtual void on_broadcast(PublicKeyHash src, td::BufferSlice data) = 0; + virtual void on_custom_query(const PublicKeyHash &src, td::BufferSlice data, + td::Promise promise) = 0; + virtual void on_broadcast(const PublicKeyHash &src, td::BufferSlice data) = 0; virtual void start() = 0; virtual ~Callback() = default; }; virtual void add_block(td::BufferSlice payload, std::vector deps) = 0; virtual void debug_add_fork(td::BufferSlice payload, CatChainBlockHeight height, std::vector deps) = 0; - virtual void blame_node(td::uint32 idx) = 0; virtual void send_fec_broadcast(td::BufferSlice data) = 0; - virtual void send_custom_query_data(PublicKeyHash dst, std::string name, td::Promise promise, + virtual void send_custom_query_data(const PublicKeyHash &dst, std::string name, td::Promise promise, td::Timestamp timeout, td::BufferSlice query) = 0; - virtual void send_custom_query_data_via(PublicKeyHash dst, std::string name, td::Promise promise, - td::Timestamp timeout, td::BufferSlice query, td::uint64 max_answer_size, + virtual void send_custom_query_data_via(const PublicKeyHash &dst, std::string name, + td::Promise promise, td::Timestamp timeout, + td::BufferSlice query, td::uint64 max_answer_size, td::actor::ActorId via) = 0; - virtual void send_custom_message_data(PublicKeyHash dst, td::BufferSlice query) = 0; + virtual void send_custom_message_data(const PublicKeyHash &dst, td::BufferSlice query) = 0; virtual void destroy() = 0; static td::actor::ActorOwn create( - std::unique_ptr callback, CatChainOptions opts, td::actor::ActorId keyring, + std::unique_ptr callback, const CatChainOptions &opts, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId overlay_manager, - std::vector ids, PublicKeyHash local_id, CatChainSessionId unique_hash, std::string db_root, - std::string db_suffix, bool allow_unsafe_self_blocks_resync); + const std::vector &ids, const PublicKeyHash &local_id, const CatChainSessionId &unique_hash, + std::string db_root, std::string db_suffix, bool allow_unsafe_self_blocks_resync); - virtual ~CatChainReceiverInterface() = default; + ~CatChainReceiverInterface() override = default; }; } // namespace catchain diff --git a/catchain/catchain-receiver-source.cpp b/catchain/catchain-receiver-source.cpp index aca522dc..6a9e0777 100644 --- a/catchain/catchain-receiver-source.cpp +++ b/catchain/catchain-receiver-source.cpp @@ -24,10 +24,10 @@ namespace ton { namespace catchain { td::uint32 CatChainReceiverSourceImpl::add_fork() { - if (fork_ids_.size() > 0) { + if (!fork_ids_.empty()) { blame(); } - auto F = chain_->add_fork(); + td::uint32 F = chain_->add_fork(); CHECK(F > 0); fork_ids_.push_back(F); @@ -60,7 +60,7 @@ td::Result> CatChainReceiverSource::crea void CatChainReceiverSourceImpl::blame(td::uint32 fork, CatChainBlockHeight height) { blame(); - if (blamed_heights_.size() > 0) { + if (!blamed_heights_.empty()) { if (blamed_heights_.size() <= fork) { blamed_heights_.resize(fork + 1, 0); } @@ -130,19 +130,6 @@ void CatChainReceiverSourceImpl::block_delivered(CatChainBlockHeight height) { } } -td::Status CatChainReceiverSourceImpl::validate_dep_sync(tl_object_ptr &dep) { - auto S = std::move(dep->signature_); - auto str = serialize_tl_object(dep, true); - dep->signature_ = std::move(S); - - auto R = encryptor_sync_->check_signature(str.as_slice(), dep->signature_.as_slice()); - if (R.is_error()) { - return R.move_as_error(); - } - - return td::Status::OK(); -} - void CatChainReceiverSourceImpl::on_new_block(CatChainReceivedBlock *block) { if (fork_is_found()) { return; @@ -165,7 +152,7 @@ void CatChainReceiverSourceImpl::on_new_block(CatChainReceivedBlock *block) { blocks_[block->get_height()] = block; } -void CatChainReceiverSourceImpl::on_found_fork_proof(td::Slice proof) { +void CatChainReceiverSourceImpl::on_found_fork_proof(const td::Slice &proof) { if (!fork_is_found()) { fetch_tl_object(proof, true).ensure(); fork_proof_ = td::SharedSlice{proof}; diff --git a/catchain/catchain-receiver-source.h b/catchain/catchain-receiver-source.h index 11faf3db..e805c2a5 100644 --- a/catchain/catchain-receiver-source.h +++ b/catchain/catchain-receiver-source.h @@ -56,9 +56,8 @@ class CatChainReceiverSource { virtual bool has_unreceived() const = 0; virtual bool has_undelivered() const = 0; - virtual td::Status validate_dep_sync(tl_object_ptr &dep) = 0; virtual void on_new_block(CatChainReceivedBlock *block) = 0; - virtual void on_found_fork_proof(td::Slice fork) = 0; + virtual void on_found_fork_proof(const td::Slice &fork) = 0; virtual td::BufferSlice fork_proof() const = 0; virtual bool fork_is_found() const = 0; diff --git a/catchain/catchain-receiver-source.hpp b/catchain/catchain-receiver-source.hpp index 74082b45..0785d8c2 100644 --- a/catchain/catchain-receiver-source.hpp +++ b/catchain/catchain-receiver-source.hpp @@ -23,6 +23,7 @@ #include "catchain-receiver-source.h" #include "catchain-receiver.h" #include "catchain-received-block.h" +#include namespace ton { @@ -82,7 +83,7 @@ class CatChainReceiverSourceImpl : public CatChainReceiverSource { if (blamed()) { return true; } - if (!blocks_.size()) { + if (blocks_.empty()) { return false; } CHECK(blocks_.rbegin()->second->get_height() >= received_height_); @@ -93,9 +94,8 @@ class CatChainReceiverSourceImpl : public CatChainReceiverSource { } CatChainReceivedBlock *get_block(CatChainBlockHeight height) const override; - td::Status validate_dep_sync(tl_object_ptr &dep) override; void on_new_block(CatChainReceivedBlock *block) override; - void on_found_fork_proof(td::Slice proof) override; + void on_found_fork_proof(const td::Slice &proof) override; bool fork_is_found() const override { return !fork_proof_.empty(); } @@ -103,7 +103,7 @@ class CatChainReceiverSourceImpl : public CatChainReceiverSource { if (!fork_proof_.empty()) { return fork_proof_.clone_as_buffer_slice(); } else { - return td::BufferSlice(); + return {}; } } diff --git a/catchain/catchain-receiver.cpp b/catchain/catchain-receiver.cpp index e7f5c019..488fbb9d 100644 --- a/catchain/catchain-receiver.cpp +++ b/catchain/catchain-receiver.cpp @@ -17,6 +17,7 @@ Copyright 2017-2020 Telegram Systems LLP */ #include +#include #include "td/actor/PromiseFuture.h" #include "td/utils/Random.h" #include "td/db/RocksDb.h" @@ -30,6 +31,25 @@ namespace ton { namespace catchain { +static const td::uint32 MAX_NEIGHBOURS = 5; +static const double EXPECTED_UNSAFE_INITIAL_SYNC_DURATION = 300.0; +static const double EXPECTED_INITIAL_SYNC_DURATION = 5.0; +static const td::uint32 OVERLAY_MAX_ALLOWED_PACKET_SIZE = 16 * 1024 * 1024; +static const double NEIGHBOURS_ROTATE_INTERVAL_MIN = 60; +static const double NEIGHBOURS_ROTATE_INTERVAL_MAX = 120; +static const td::uint32 MAX_QUERY_BLOCKS = 100; +static const td::uint32 MAX_QUERY_HEIGHT = 100; +static const td::uint32 GET_DIFFERENCE_MAX_SEND = 100; +static const double GET_DIFFERENCE_TIMEOUT = 5.0; +static const double GET_BLOCK_TIMEOUT = 2.0; +static const td::uint32 MAX_PENDING_DEPS = 16; +static const double EXPECTED_INITIAL_SYNC_DURATION_WITH_UNPROCESSED = 60.0; +static const double SYNC_INTERVAL_MIN = 0.1; +static const double SYNC_INTERVAL_MAX = 0.2; +static const td::uint32 SYNC_ITERATIONS = 3; +static const double DESTROY_DB_DELAY = 1.0; +static const td::uint32 DESTROY_DB_MAX_ATTEMPTS = 10; + PublicKeyHash CatChainReceiverImpl::get_source_hash(td::uint32 source_id) const { CHECK(source_id < sources_.size()); return sources_[source_id]->get_hash(); @@ -45,18 +65,19 @@ void CatChainReceiverImpl::deliver_block(CatChainReceivedBlock *block) { << " custom=" << block->is_custom(); callback_->new_block(block->get_source_id(), block->get_fork_id(), block->get_hash(), block->get_height(), block->get_height() == 1 ? CatChainBlockHash::zero() : block->get_prev_hash(), - block->get_dep_hashes(), block->get_deps(), + block->get_dep_hashes(), block->get_vt(), block->is_custom() ? block->get_payload().clone() : td::SharedSlice()); std::vector v; - for (auto it : neighbours_) { - auto S = get_source(it); + for (td::uint32 it : neighbours_) { + CatChainReceiverSource *S = get_source(it); v.push_back(S->get_adnl_id()); } auto update = create_tl_object(block->export_tl()); - auto D = serialize_tl_object(update, true, block->get_payload().as_slice()); + td::BufferSlice D = serialize_tl_object(update, true, block->get_payload().as_slice()); + CHECK(D.size() <= opts_.max_serialized_block_size); td::actor::send_closure(overlay_manager_, &overlay::Overlays::send_multiple_messages, std::move(v), get_source(local_idx_)->get_adnl_id(), overlay_id_, std::move(D)); @@ -64,8 +85,8 @@ void CatChainReceiverImpl::deliver_block(CatChainReceivedBlock *block) { void CatChainReceiverImpl::receive_block(adnl::AdnlNodeIdShort src, tl_object_ptr block, td::BufferSlice payload) { - auto id = CatChainReceivedBlock::block_hash(this, block, payload); - auto B = get_block(id); + CatChainBlockHash id = CatChainReceivedBlock::block_hash(this, block, payload); + CatChainReceivedBlock *B = get_block(id); if (B && B->initialized()) { return; } @@ -76,7 +97,27 @@ void CatChainReceiverImpl::receive_block(adnl::AdnlNodeIdShort src, tl_object_pt return; } - auto S = validate_block_sync(block, payload.as_slice()); + td::uint64 max_block_height = get_max_block_height(opts_, sources_.size()); + if ((td::uint32)block->height_ > max_block_height) { + VLOG(CATCHAIN_WARNING) << this << ": received too many blocks from " << src + << " (limit=" << max_block_height << ")"; + return; + } + + td::uint32 src_id = block->src_; + if (src_id >= get_sources_cnt()) { + VLOG(CATCHAIN_WARNING) << this << ": received broken block from " << src << ": bad src " << block->src_; + return; + } + CatChainReceiverSource *source = get_source(src_id); + if (source->fork_is_found()) { + if (B == nullptr || !B->has_rev_deps()) { + VLOG(CATCHAIN_WARNING) << this << ": dropping block from source " << src_id << ": source has a fork"; + return; + } + } + + td::Status S = validate_block_sync(block, payload.as_slice()); if (S.is_error()) { VLOG(CATCHAIN_WARNING) << this << ": received broken block from " << src << ": " << S.move_as_error(); @@ -89,11 +130,11 @@ void CatChainReceiverImpl::receive_block(adnl::AdnlNodeIdShort src, tl_object_pt << " (unsafe=" << allow_unsafe_self_blocks_resync_ << ")"; } else { LOG(ERROR) << this << ": received unknown SELF block from " << src << ". UPDATING LOCAL DATABASE. UNSAFE"; - initial_sync_complete_at_ = td::Timestamp::in(300.0); + initial_sync_complete_at_ = td::Timestamp::in(EXPECTED_UNSAFE_INITIAL_SYNC_DURATION); } } - auto raw_data = serialize_tl_object(block, true, payload.as_slice()); + td::BufferSlice raw_data = serialize_tl_object(block, true, payload.as_slice()); create_block(std::move(block), td::SharedSlice{payload.as_slice()}); if (!opts_.debug_disable_db) { @@ -104,6 +145,11 @@ void CatChainReceiverImpl::receive_block(adnl::AdnlNodeIdShort src, tl_object_pt } void CatChainReceiverImpl::receive_block_answer(adnl::AdnlNodeIdShort src, td::BufferSlice data) { + if (data.size() > opts_.max_serialized_block_size) { + VLOG(CATCHAIN_INFO) << this << ": received bad block result " << src << ": too big (size=" + << data.size() << ", limit=" << opts_.max_serialized_block_size << ")"; + return; + } auto F = fetch_tl_prefix(data, true); if (F.is_error()) { VLOG(CATCHAIN_INFO) << this << ": received bad block result: " << F.move_as_error(); @@ -111,7 +157,7 @@ void CatChainReceiverImpl::receive_block_answer(adnl::AdnlNodeIdShort src, td::B } auto f = F.move_as_ok(); ton_api::downcast_call( - *f.get(), + *f, td::overloaded( [&](ton_api::catchain_blockNotFound &r) { VLOG(CATCHAIN_INFO) << this << ": catchain block not found"; }, [&](ton_api::catchain_blockResult &r) { receive_block(src, std::move(r.block_), std::move(data)); })); @@ -129,6 +175,11 @@ void CatChainReceiverImpl::receive_message_from_overlay(adnl::AdnlNodeIdShort sr VLOG(CATCHAIN_INFO) << this << ": dropping block update from blamed source " << src; return; }*/ + if (data.size() > opts_.max_serialized_block_size) { + VLOG(CATCHAIN_WARNING) << this << ": dropping broken block from " << src << ": too big (size=" + << data.size() << ", limit=" << opts_.max_serialized_block_size << ")"; + return; + } auto R = fetch_tl_prefix(data, true); if (R.is_error()) { VLOG(CATCHAIN_WARNING) << this << ": dropping broken block from " << src << ": " << R.move_as_error(); @@ -139,14 +190,14 @@ void CatChainReceiverImpl::receive_message_from_overlay(adnl::AdnlNodeIdShort sr receive_block(src, std::move(U->block_), std::move(data)); } -void CatChainReceiverImpl::receive_broadcast_from_overlay(PublicKeyHash src, td::BufferSlice data) { +void CatChainReceiverImpl::receive_broadcast_from_overlay(const PublicKeyHash &src, td::BufferSlice data) { if (!read_db_) { return; } callback_->on_broadcast(src, std::move(data)); } -/*void CatChainReceiverImpl::send_block(PublicKeyHash src, tl_object_ptr block, +/*void CatChainReceiverImpl::send_block(const PublicKeyHash &src, tl_object_ptr block, td::BufferSlice payload) { CHECK(read_db_); CHECK(src == local_id_); @@ -164,7 +215,7 @@ CatChainReceivedBlock *CatChainReceiverImpl::create_block(tl_object_ptrheight_ == 0) { return root_block_; } - auto hash = CatChainReceivedBlock::block_hash(this, block, payload.as_slice()); + CatChainBlockHash hash = CatChainReceivedBlock::block_hash(this, block, payload.as_slice()); auto it = blocks_.find(hash); if (it != blocks_.end()) { @@ -173,9 +224,8 @@ CatChainReceivedBlock *CatChainReceiverImpl::create_block(tl_object_ptrsecond.get(); } else { - blocks_.emplace(hash, CatChainReceivedBlock::create(std::move(block), std::move(payload), this)); - it = blocks_.find(hash); - return it->second.get(); + auto r = blocks_.emplace(hash, CatChainReceivedBlock::create(std::move(block), std::move(payload), this)); + return r.first->second.get(); } } @@ -183,7 +233,7 @@ CatChainReceivedBlock *CatChainReceiverImpl::create_block(tl_object_ptrheight_ == 0) { return root_block_; } - auto hash = CatChainReceivedBlock::block_hash(this, block); + CatChainBlockHash hash = CatChainReceivedBlock::block_hash(this, block); auto it = blocks_.find(hash); if (it != blocks_.end()) { return it->second.get(); @@ -194,20 +244,20 @@ CatChainReceivedBlock *CatChainReceiverImpl::create_block(tl_object_ptr &dep) { +td::Status CatChainReceiverImpl::validate_block_sync(const tl_object_ptr &dep) const { TRY_STATUS_PREFIX(CatChainReceivedBlock::pre_validate_block(this, dep), "failed to validate block: "); if (dep->height_ > 0) { auto id = CatChainReceivedBlock::block_id(this, dep); - auto B = serialize_tl_object(id, true); - auto block = get_block(get_tl_object_sha_bits256(id)); + td::BufferSlice B = serialize_tl_object(id, true); + CatChainReceivedBlock *block = get_block(get_tl_object_sha_bits256(id)); if (block) { return td::Status::OK(); } - auto S = get_source_by_hash(PublicKeyHash{id->src_}); + CatChainReceiverSource *S = get_source_by_hash(PublicKeyHash{id->src_}); CHECK(S != nullptr); - auto E = S->get_encryptor_sync(); + Encryptor *E = S->get_encryptor_sync(); CHECK(E != nullptr); return E->check_signature(B.as_slice(), dep->signature_.as_slice()); } else { @@ -215,17 +265,18 @@ td::Status CatChainReceiverImpl::validate_block_sync(tl_object_ptr &block, td::Slice payload) { +td::Status CatChainReceiverImpl::validate_block_sync(const tl_object_ptr &block, + const td::Slice &payload) const { //LOG(INFO) << ton_api::to_string(block); TRY_STATUS_PREFIX(CatChainReceivedBlock::pre_validate_block(this, block, payload), "failed to validate block: "); if (block->height_ > 0) { auto id = CatChainReceivedBlock::block_id(this, block, payload); - auto B = serialize_tl_object(id, true); + td::BufferSlice B = serialize_tl_object(id, true); - auto S = get_source_by_hash(PublicKeyHash{id->src_}); + CatChainReceiverSource *S = get_source_by_hash(PublicKeyHash{id->src_}); CHECK(S != nullptr); - auto E = S->get_encryptor_sync(); + Encryptor *E = S->get_encryptor_sync(); CHECK(E != nullptr); return E->check_signature(B.as_slice(), block->signature_.as_slice()); } else { @@ -235,7 +286,7 @@ td::Status CatChainReceiverImpl::validate_block_sync(tl_object_ptrrun(); @@ -265,7 +316,7 @@ void CatChainReceiverImpl::add_block_cont_3(tl_object_ptr 0) { + if (!pending_blocks_.empty()) { auto B = std::move(pending_blocks_.front()); pending_blocks_.pop_front(); add_block(std::move(B->payload_), std::move(B->deps_)); @@ -278,9 +329,9 @@ void CatChainReceiverImpl::add_block_cont_2(tl_object_ptr add_block_cont_2(std::move(block), std::move(payload)); return; } - auto id = CatChainReceivedBlock::block_hash(this, block, payload.as_slice()); + CatChainBlockHash id = CatChainReceivedBlock::block_hash(this, block, payload.as_slice()); - auto raw_data = serialize_tl_object(block, true, payload.as_slice()); + td::BufferSlice raw_data = serialize_tl_object(block, true, payload.as_slice()); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), block = std::move(block), payload = std::move(payload)](td::Result R) mutable { @@ -319,7 +370,7 @@ void CatChainReceiverImpl::add_block(td::BufferSlice payload, std::vectorget_id() == local_idx_); if (!intentional_fork_) { @@ -331,7 +382,7 @@ void CatChainReceiverImpl::add_block(td::BufferSlice payload, std::vector> deps_arr; deps_arr.resize(deps.size()); for (size_t i = 0; i < deps.size(); i++) { - auto B = get_block(deps[i]); + CatChainReceivedBlock *B = get_block(deps[i]); LOG_CHECK(B != nullptr) << this << ": cannot find block with hash " << deps[i]; if (!intentional_fork_) { CHECK(B->get_source_id() != local_idx_); @@ -339,13 +390,13 @@ void CatChainReceiverImpl::add_block(td::BufferSlice payload, std::vectorexport_tl_dep(); } - auto height = prev->height_ + 1; + int height = prev->height_ + 1; auto block_data = create_tl_object(std::move(prev), std::move(deps_arr)); auto block = create_tl_object(incarnation_, local_idx_, height, std::move(block_data), td::BufferSlice()); auto id = CatChainReceivedBlock::block_id(this, block, payload); - auto id_s = serialize_tl_object(id, true); + td::BufferSlice id_s = serialize_tl_object(id, true); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), print_id = print_id(), block = std::move(block), payload = std::move(payload)](td::Result R) mutable { @@ -362,24 +413,24 @@ void CatChainReceiverImpl::add_block(td::BufferSlice payload, std::vector block, td::BufferSlice payload) { validate_block_sync(block, payload.as_slice()).ensure(); - auto B = create_block(std::move(block), td::SharedSlice{payload.as_slice()}); + CatChainReceivedBlock *B = create_block(std::move(block), td::SharedSlice{payload.as_slice()}); B->written(); run_scheduler(); CHECK(B->delivered()); active_send_ = false; - if (pending_blocks_.size() > 0) { - auto B = std::move(pending_blocks_.front()); + if (!pending_blocks_.empty()) { + auto pending_block = std::move(pending_blocks_.front()); pending_blocks_.pop_front(); - add_block(std::move(B->payload_), std::move(B->deps_)); + add_block(std::move(pending_block->payload_), std::move(pending_block->deps_)); } } void CatChainReceiverImpl::debug_add_fork(td::BufferSlice payload, CatChainBlockHeight height, std::vector deps) { intentional_fork_ = true; - auto S = get_source_by_hash(local_id_); + CatChainReceiverSource *S = get_source_by_hash(local_id_); CHECK(S != nullptr); CHECK(S->get_id() == local_idx_); @@ -399,7 +450,7 @@ void CatChainReceiverImpl::debug_add_fork(td::BufferSlice payload, CatChainBlock std::vector> deps_arr; deps_arr.resize(deps.size()); for (size_t i = 0; i < deps.size(); i++) { - auto B = get_block(deps[i]); + CatChainReceivedBlock *B = get_block(deps[i]); LOG_CHECK(B != nullptr) << this << ": cannot find block with hash " << deps[i]; CHECK(B->get_source_id() != local_idx_); deps_arr[i] = B->export_tl_dep(); @@ -410,7 +461,7 @@ void CatChainReceiverImpl::debug_add_fork(td::BufferSlice payload, CatChainBlock td::BufferSlice()); auto id = CatChainReceivedBlock::block_id(this, block, payload); - auto id_s = serialize_tl_object(id, true); + td::BufferSlice id_s = serialize_tl_object(id, true); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), print_id = print_id(), block = std::move(block), payload = std::move(payload)](td::Result R) mutable { @@ -425,29 +476,33 @@ void CatChainReceiverImpl::debug_add_fork(td::BufferSlice payload, CatChainBlock td::actor::send_closure_later(keyring_, &keyring::Keyring::sign_message, local_id_, std::move(id_s), std::move(P)); } -CatChainReceiverImpl::CatChainReceiverImpl(std::unique_ptr callback, CatChainOptions opts, +CatChainReceiverImpl::CatChainReceiverImpl(std::unique_ptr callback, + const CatChainOptions &opts, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId overlay_manager, - std::vector ids, PublicKeyHash local_id, - CatChainSessionId unique_hash, std::string db_root, std::string db_suffix, + const std::vector &ids, + const PublicKeyHash &local_id, + const CatChainSessionId &unique_hash, + std::string db_root, + std::string db_suffix, bool allow_unsafe_self_blocks_resync) : callback_(std::move(callback)) - , opts_(std::move(opts)) - , keyring_(keyring) - , adnl_(adnl) - , overlay_manager_(overlay_manager) + , opts_(opts) + , keyring_(std::move(keyring)) + , adnl_(std::move(adnl)) + , overlay_manager_(std::move(overlay_manager)) , local_id_(local_id) - , db_root_(db_root) - , db_suffix_(db_suffix) + , db_root_(std::move(db_root)) + , db_suffix_(std::move(db_suffix)) , allow_unsafe_self_blocks_resync_(allow_unsafe_self_blocks_resync) { std::vector short_ids; local_idx_ = static_cast(ids.size()); - for (auto &id : ids) { - td::uint32 seq = static_cast(sources_.size()); + for (const CatChainNode &id : ids) { + auto seq = static_cast(sources_.size()); auto R = CatChainReceiverSource::create(this, id.pub_key, id.adnl_id, seq); auto S = R.move_as_ok(); - auto h = id.pub_key.compute_short_id(); + PublicKeyHash h = id.pub_key.compute_short_id(); short_ids.push_back(h.bits256_value()); sources_hashes_[h] = seq; sources_adnl_addrs_[id.adnl_id] = seq; @@ -477,12 +532,13 @@ CatChainReceiverImpl::CatChainReceiverImpl(std::unique_ptr callback, C void CatChainReceiverImpl::start_up() { std::vector ids; + ids.reserve(get_sources_cnt()); for (td::uint32 i = 0; i < get_sources_cnt(); i++) { ids.push_back(get_source(i)->get_adnl_id()); } std::map root_keys; for (td::uint32 i = 0; i < get_sources_cnt(); i++) { - root_keys.emplace(get_source(i)->get_hash(), 16 << 20); + root_keys.emplace(get_source(i)->get_hash(), OVERLAY_MAX_ALLOWED_PACKET_SIZE); } td::actor::send_closure(overlay_manager_, &overlay::Overlays::create_private_overlay, get_source(local_idx_)->get_adnl_id(), overlay_full_id_.clone(), std::move(ids), @@ -498,13 +554,13 @@ void CatChainReceiverImpl::start_up() { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { R.ensure(); - auto g = R.move_as_ok(); + DbType::GetResult g = R.move_as_ok(); if (g.status == td::KeyValue::GetStatus::NotFound) { td::actor::send_closure(SelfId, &CatChainReceiverImpl::read_db); } else { - auto B = std::move(g.value); - CHECK(B.size() == 32); + td::BufferSlice B = std::move(g.value); CatChainBlockHash x; + CHECK(B.size() == x.as_array().size()); as_slice(x).copy_from(B.as_slice()); td::actor::send_closure(SelfId, &CatChainReceiverImpl::read_db_from, x); } @@ -527,7 +583,7 @@ void CatChainReceiverImpl::read_db_from(CatChainBlockHash id) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), id](td::Result R) { R.ensure(); - auto g = R.move_as_ok(); + DbType::GetResult g = R.move_as_ok(); CHECK(g.status == td::KeyValue::GetStatus::Ok); td::actor::send_closure(SelfId, &CatChainReceiverImpl::read_block_from_db, id, std::move(g.value)); @@ -543,12 +599,12 @@ void CatChainReceiverImpl::read_block_from_db(CatChainBlockHash id, td::BufferSl F.ensure(); auto block = F.move_as_ok(); - auto payload = std::move(data); + td::BufferSlice payload = std::move(data); - auto block_id = CatChainReceivedBlock::block_hash(this, block, payload); + CatChainBlockHash block_id = CatChainReceivedBlock::block_hash(this, block, payload); CHECK(block_id == id); - auto B = get_block(id); + CatChainReceivedBlock *B = get_block(id); if (B && B->initialized()) { CHECK(B->in_db()); if (!pending_in_db_) { @@ -557,7 +613,7 @@ void CatChainReceiverImpl::read_block_from_db(CatChainBlockHash id, td::BufferSl return; } - auto source = get_source(block->src_); + CatChainReceiverSource *source = get_source(block->src_); CHECK(source != nullptr); CHECK(block->incarnation_ == incarnation_); @@ -565,18 +621,18 @@ void CatChainReceiverImpl::read_block_from_db(CatChainBlockHash id, td::BufferSl validate_block_sync(block, payload).ensure(); B = create_block(std::move(block), td::SharedSlice{payload.as_slice()}); - B->written(); CHECK(B); + B->written(); - auto deps = B->get_dep_hashes(); + std::vector deps = B->get_dep_hashes(); deps.push_back(B->get_prev_hash()); - for (auto &dep : deps) { - auto dep_block = get_block(dep); + for (const CatChainBlockHash &dep : deps) { + CatChainReceivedBlock *dep_block = get_block(dep); if (!dep_block || !dep_block->initialized()) { pending_in_db_++; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), dep](td::Result R) { R.ensure(); - auto g = R.move_as_ok(); + DbType::GetResult g = R.move_as_ok(); CHECK(g.status == td::KeyValue::GetStatus::Ok); td::actor::send_closure(SelfId, &CatChainReceiverImpl::read_block_from_db, dep, std::move(g.value)); @@ -601,26 +657,28 @@ void CatChainReceiverImpl::read_db() { read_db_ = true; - next_rotate_ = td::Timestamp::in(60 + td::Random::fast(0, 60)); - next_sync_ = td::Timestamp::in(0.001 * td::Random::fast(0, 60)); - initial_sync_complete_at_ = td::Timestamp::in(allow_unsafe_self_blocks_resync_ ? 300.0 : 5.0); + next_rotate_ = td::Timestamp::in(td::Random::fast(NEIGHBOURS_ROTATE_INTERVAL_MIN, NEIGHBOURS_ROTATE_INTERVAL_MAX)); + next_sync_ = td::Timestamp::in( + 0.001 * td::Random::fast(NEIGHBOURS_ROTATE_INTERVAL_MIN, NEIGHBOURS_ROTATE_INTERVAL_MAX)); + initial_sync_complete_at_ = td::Timestamp::in( + allow_unsafe_self_blocks_resync_ ? EXPECTED_UNSAFE_INITIAL_SYNC_DURATION : EXPECTED_INITIAL_SYNC_DURATION); alarm_timestamp().relax(next_rotate_); alarm_timestamp().relax(next_sync_); alarm_timestamp().relax(initial_sync_complete_at_); } td::actor::ActorOwn CatChainReceiverInterface::create( - std::unique_ptr callback, CatChainOptions opts, td::actor::ActorId keyring, + std::unique_ptr callback, const CatChainOptions &opts, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId overlay_manager, - std::vector ids, PublicKeyHash local_id, CatChainSessionId unique_hash, std::string db_root, - std::string db_suffix, bool allow_unsafe_self_blocks_resync) { + const std::vector &ids, const PublicKeyHash &local_id, const CatChainSessionId &unique_hash, + std::string db_root, std::string db_suffix, bool allow_unsafe_self_blocks_resync) { auto A = td::actor::create_actor( - "catchainreceiver", std::move(callback), std::move(opts), keyring, adnl, overlay_manager, std::move(ids), - local_id, unique_hash, db_root, db_suffix, allow_unsafe_self_blocks_resync); + "catchainreceiver", std::move(callback), opts, std::move(keyring), std::move(adnl), std::move(overlay_manager), + ids, local_id, unique_hash, std::move(db_root), std::move(db_suffix), allow_unsafe_self_blocks_resync); return std::move(A); } -CatChainReceiverSource *CatChainReceiverImpl::get_source_by_hash(PublicKeyHash source_hash) const { +CatChainReceiverSource *CatChainReceiverImpl::get_source_by_hash(const PublicKeyHash &source_hash) const { auto it = sources_hashes_.find(source_hash); if (it == sources_hashes_.end()) { return nullptr; @@ -650,10 +708,10 @@ void CatChainReceiverImpl::receive_query_from_overlay(adnl::AdnlNodeIdShort src, return; } auto f = F.move_as_ok(); - ton_api::downcast_call(*f.get(), [&](auto &obj) { this->process_query(src, obj, std::move(promise)); }); + ton_api::downcast_call(*f, [&](auto &obj) { this->process_query(src, std::move(obj), std::move(promise)); }); } -void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getBlock &query, +void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getBlock query, td::Promise promise) { auto it = blocks_.find(query.block_); if (it == blocks_.end() || it->second->get_height() == 0 || !it->second->initialized()) { @@ -664,19 +722,20 @@ void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::cat } } -void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getBlocks &query, +void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getBlocks query, td::Promise promise) { - if (query.blocks_.size() > 100) { + if (query.blocks_.size() > MAX_QUERY_BLOCKS) { promise.set_error(td::Status::Error(ErrorCode::protoviolation, "too many blocks")); return; } td::int32 cnt = 0; - for (auto &b : query.blocks_) { + for (const CatChainBlockHash &b : query.blocks_) { auto it = blocks_.find(b); if (it != blocks_.end() && it->second->get_height() > 0) { auto block = create_tl_object(it->second->export_tl()); - CHECK(it->second->get_payload().size() > 0); - auto B = serialize_tl_object(block, true, it->second->get_payload().clone()); + CHECK(!it->second->get_payload().empty()); + td::BufferSlice B = serialize_tl_object(block, true, it->second->get_payload().clone()); + CHECK(B.size() <= opts_.max_serialized_block_size); td::actor::send_closure(overlay_manager_, &overlay::Overlays::send_message, src, get_source(local_idx_)->get_adnl_id(), overlay_id_, std::move(B)); cnt++; @@ -685,19 +744,19 @@ void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::cat promise.set_value(serialize_tl_object(create_tl_object(cnt), true)); } -void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getBlockHistory &query, +void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getBlockHistory query, td::Promise promise) { - auto h = query.height_; + int64_t h = query.height_; if (h <= 0) { promise.set_error(td::Status::Error(ErrorCode::protoviolation, "not-positive height")); return; } - if (h > 100) { - h = 100; + if (h > MAX_QUERY_HEIGHT) { + h = MAX_QUERY_HEIGHT; } std::set s{query.stop_if_.begin(), query.stop_if_.end()}; - auto B = get_block(query.block_); + CatChainReceivedBlock *B = get_block(query.block_); if (B == nullptr) { promise.set_value(serialize_tl_object(create_tl_object(0), true)); return; @@ -711,8 +770,9 @@ void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::cat break; } auto block = create_tl_object(B->export_tl()); - CHECK(B->get_payload().size() > 0); - auto BB = serialize_tl_object(block, true, B->get_payload().as_slice()); + CHECK(!B->get_payload().empty()); + td::BufferSlice BB = serialize_tl_object(block, true, B->get_payload().as_slice()); + CHECK(BB.size() <= opts_.max_serialized_block_size); td::actor::send_closure(overlay_manager_, &overlay::Overlays::send_message, src, get_source(local_idx_)->get_adnl_id(), overlay_id_, std::move(BB)); B = B->get_prev(); @@ -721,7 +781,7 @@ void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::cat promise.set_value(serialize_tl_object(create_tl_object(cnt), true)); } -void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getDifference &query, +void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getDifference query, td::Promise promise) { auto &vt = query.rt_; if (vt.size() != get_sources_cnt()) { @@ -731,7 +791,7 @@ void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::cat } for (td::uint32 i = 0; i < get_sources_cnt(); i++) { if (vt[i] >= 0) { - auto S = get_source(i); + CatChainReceiverSource *S = get_source(i); if (S->fork_is_found()) { auto obj = fetch_tl_object(S->fork_proof(), true); obj.ensure(); @@ -744,26 +804,21 @@ void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::cat } std::vector my_vt(get_sources_cnt()); - td::uint64 total = 0; for (td::uint32 i = 0; i < get_sources_cnt(); i++) { if (vt[i] >= 0) { - auto x = static_cast(vt[i]); - auto S = get_source(i); - if (S->delivered_height() > x) { - total += S->delivered_height() - x; - } - my_vt[i] = S->delivered_height(); + CatChainReceiverSource *S = get_source(i); + my_vt[i] = static_cast(S->delivered_height()); } else { my_vt[i] = -1; } } - const td::uint32 max_send = 100; + const td::uint32 max_send = GET_DIFFERENCE_MAX_SEND; - td::int32 l = 0; - td::int32 r = max_send + 1; - while (r - l > 1) { - td::int32 x = (r + l) / 2; + td::int32 left = 0; + td::int32 right = max_send + 1; + while (right - left > 1) { + td::int32 x = (right + left) / 2; td::uint64 sum = 0; for (td::uint32 i = 0; i < get_sources_cnt(); i++) { if (vt[i] >= 0 && my_vt[i] > vt[i]) { @@ -771,23 +826,23 @@ void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::cat } } if (sum > max_send) { - r = x; + right = x; } else { - l = x; + left = x; } } - CHECK(r > 0); + CHECK(right > 0); for (td::uint32 i = 0; i < get_sources_cnt(); i++) { if (vt[i] >= 0 && my_vt[i] > vt[i]) { - auto S = get_source(i); - auto t = (my_vt[i] - vt[i] > r) ? r : (my_vt[i] - vt[i]); - CHECK(t > 0); + CatChainReceiverSource *S = get_source(i); + td::int32 t = (my_vt[i] - vt[i] > right) ? right : (my_vt[i] - vt[i]); while (t-- > 0) { - auto M = S->get_block(++vt[i]); + CatChainReceivedBlock *M = S->get_block(++vt[i]); CHECK(M != nullptr); auto block = create_tl_object(M->export_tl()); - CHECK(M->get_payload().size() > 0); - auto BB = serialize_tl_object(block, true, M->get_payload().as_slice()); + CHECK(!M->get_payload().empty()); + td::BufferSlice BB = serialize_tl_object(block, true, M->get_payload().as_slice()); + CHECK(BB.size() <= opts_.max_serialized_block_size); td::actor::send_closure(overlay_manager_, &overlay::Overlays::send_message, src, get_source(local_idx_)->get_adnl_id(), overlay_id_, std::move(BB)); } @@ -798,7 +853,7 @@ void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::cat } void CatChainReceiverImpl::got_fork_proof(td::BufferSlice data) { - auto F = fetch_tl_object(std::move(data), true); + auto F = fetch_tl_object(data, true); if (F.is_error()) { VLOG(CATCHAIN_WARNING) << this << ": received bad fork proof: " << F.move_as_error(); return; @@ -825,7 +880,7 @@ void CatChainReceiverImpl::got_fork_proof(td::BufferSlice data) { return; } - auto S = get_source(f->left_->src_); + CatChainReceiverSource *S = get_source(f->left_->src_); S->on_found_fork_proof( create_serialize_tl_object(std::move(f->left_), std::move(f->right_))); S->blame(); @@ -835,24 +890,24 @@ void CatChainReceiverImpl::synchronize_with(CatChainReceiverSource *S) { CHECK(!S->blamed()); std::vector rt(get_sources_cnt()); for (td::uint32 i = 0; i < get_sources_cnt(); i++) { - auto SS = get_source(i); + CatChainReceiverSource *SS = get_source(i); if (SS->blamed()) { rt[i] = -1; } else { - rt[i] = S->delivered_height(); + rt[i] = static_cast(S->delivered_height()); } } auto P = td::PromiseCreator::lambda( [SelfId = actor_id(this), src = S->get_hash(), print_id = print_id()](td::Result R) { if (R.is_error()) { - VLOG(CATCHAIN_INFO) << print_id << ": timedout syncronize query to " << src; + VLOG(CATCHAIN_INFO) << print_id << ": timedout synchronize query to " << src; return; } - auto data = R.move_as_ok(); + td::BufferSlice data = R.move_as_ok(); auto X = fetch_tl_object(data.clone(), true); if (X.is_error()) { - VLOG(CATCHAIN_WARNING) << print_id << ": received incorrect answer to syncronize query from " << src << ": " + VLOG(CATCHAIN_WARNING) << print_id << ": received incorrect answer to synchronize query from " << src << ": " << X.move_as_error(); return; } @@ -866,49 +921,49 @@ void CatChainReceiverImpl::synchronize_with(CatChainReceiverSource *S) { }); td::actor::send_closure(overlay_manager_, &overlay::Overlays::send_query, S->get_adnl_id(), get_source(local_idx_)->get_adnl_id(), overlay_id_, "sync", std::move(P), - td::Timestamp::in(5.0), + td::Timestamp::in(GET_DIFFERENCE_TIMEOUT), serialize_tl_object(create_tl_object(std::move(rt)), true)); if (S->delivered_height() < S->received_height()) { - auto B = S->get_block(S->delivered_height() + 1); + CatChainReceivedBlock *B = S->get_block(S->delivered_height() + 1); CHECK(B->initialized()); std::vector vec; - B->find_pending_deps(vec, 16); + B->find_pending_deps(vec, MAX_PENDING_DEPS); - for (auto &hash : vec) { - auto P = td::PromiseCreator::lambda( + for (const CatChainBlockHash &hash : vec) { + auto PP = td::PromiseCreator::lambda( [SelfId = actor_id(this), print_id = print_id(), src = S->get_adnl_id()](td::Result R) { if (R.is_error()) { - VLOG(CATCHAIN_INFO) << print_id << ": timedout syncronize query to " << src; + VLOG(CATCHAIN_INFO) << print_id << ": timedout synchronize query to " << src; } else { td::actor::send_closure(SelfId, &CatChainReceiverImpl::receive_block_answer, src, R.move_as_ok()); } }); - auto query = serialize_tl_object(create_tl_object(hash), true); + td::BufferSlice query = serialize_tl_object(create_tl_object(hash), true); td::actor::send_closure(overlay_manager_, &overlay::Overlays::send_query, S->get_adnl_id(), - get_source(local_idx_)->get_adnl_id(), overlay_id_, "sync blocks", std::move(P), - td::Timestamp::in(2.0), std::move(query)); + get_source(local_idx_)->get_adnl_id(), overlay_id_, "sync blocks", std::move(PP), + td::Timestamp::in(GET_BLOCK_TIMEOUT), std::move(query)); } } } void CatChainReceiverImpl::choose_neighbours() { std::vector n; - n.resize(get_max_neighbours()); + n.resize(MAX_NEIGHBOURS); td::uint32 size = 0; for (td::uint32 i = 0; i < get_sources_cnt(); i++) { if (i == local_idx_) { continue; } - auto S = get_source(i); + CatChainReceiverSource *S = get_source(i); if (!S->blamed()) { size++; if (size <= n.size()) { n[size - 1] = i; } else { - td::uint32 id = td::Random::fast(0, size - 1); + td::uint32 id = td::Random::fast(0, static_cast(size) - 1); if (id < n.size()) { n[id] = i; } @@ -922,15 +977,15 @@ void CatChainReceiverImpl::choose_neighbours() { } bool CatChainReceiverImpl::unsafe_start_up_check_completed() { - auto S = get_source(local_idx_); + CatChainReceiverSource *S = get_source(local_idx_); CHECK(!S->blamed()); if (S->has_unreceived() || S->has_undelivered()) { LOG(INFO) << "catchain: has_unreceived=" << S->has_unreceived() << " has_undelivered=" << S->has_undelivered(); run_scheduler(); - initial_sync_complete_at_ = td::Timestamp::in(60.0); + initial_sync_complete_at_ = td::Timestamp::in(EXPECTED_INITIAL_SYNC_DURATION_WITH_UNPROCESSED); return false; } - auto h = S->delivered_height(); + CatChainBlockHeight h = S->delivered_height(); if (h == 0) { CHECK(last_sent_block_->get_height() == 0); CHECK(!unsafe_root_block_writing_); @@ -941,20 +996,20 @@ bool CatChainReceiverImpl::unsafe_start_up_check_completed() { return true; } if (unsafe_root_block_writing_) { - initial_sync_complete_at_ = td::Timestamp::in(5.0); + initial_sync_complete_at_ = td::Timestamp::in(EXPECTED_INITIAL_SYNC_DURATION); LOG(INFO) << "catchain: writing=true"; return false; } unsafe_root_block_writing_ = true; - auto B = S->get_block(h); + CatChainReceivedBlock *B = S->get_block(h); CHECK(B != nullptr); CHECK(B->delivered()); CHECK(B->in_db()); - auto id = B->get_hash(); + CatChainBlockHash id = B->get_hash(); - td::BufferSlice raw_data{32}; + td::BufferSlice raw_data{id.as_array().size()}; raw_data.as_slice().copy_from(as_slice(id)); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), block = B](td::Result R) mutable { @@ -963,7 +1018,7 @@ bool CatChainReceiverImpl::unsafe_start_up_check_completed() { }); db_.set(CatChainBlockHash::zero(), std::move(raw_data), std::move(P), 0); - initial_sync_complete_at_ = td::Timestamp::in(5.0); + initial_sync_complete_at_ = td::Timestamp::in(EXPECTED_INITIAL_SYNC_DURATION); LOG(INFO) << "catchain: need update root"; return false; } @@ -977,9 +1032,9 @@ void CatChainReceiverImpl::written_unsafe_root_block(CatChainReceivedBlock *bloc void CatChainReceiverImpl::alarm() { alarm_timestamp() = td::Timestamp::never(); if (next_sync_ && next_sync_.is_in_past()) { - next_sync_ = td::Timestamp::in(td::Random::fast(0.1, 0.2)); - for (auto i = 0; i < 3; i++) { - auto S = get_source(td::Random::fast(0, get_sources_cnt() - 1)); + next_sync_ = td::Timestamp::in(td::Random::fast(SYNC_INTERVAL_MIN, SYNC_INTERVAL_MAX)); + for (unsigned i = 0; i < SYNC_ITERATIONS; i++) { + CatChainReceiverSource *S = get_source(td::Random::fast(0, static_cast(get_sources_cnt()) - 1)); CHECK(S != nullptr); if (!S->blamed()) { synchronize_with(S); @@ -988,7 +1043,7 @@ void CatChainReceiverImpl::alarm() { } } if (next_rotate_ && next_rotate_.is_in_past()) { - next_rotate_ = td::Timestamp::in(td::Random::fast(60.0, 120.0)); + next_rotate_ = td::Timestamp::in(td::Random::fast(NEIGHBOURS_ROTATE_INTERVAL_MIN, NEIGHBOURS_ROTATE_INTERVAL_MAX)); choose_neighbours(); } if (!started_ && read_db_ && initial_sync_complete_at_ && initial_sync_complete_at_.is_in_past()) { @@ -1013,58 +1068,68 @@ void CatChainReceiverImpl::send_fec_broadcast(td::BufferSlice data) { td::actor::send_closure(overlay_manager_, &overlay::Overlays::send_broadcast_fec_ex, get_source(local_idx_)->get_adnl_id(), overlay_id_, local_id_, 0, std::move(data)); } -void CatChainReceiverImpl::send_custom_query_data(PublicKeyHash dst, std::string name, +void CatChainReceiverImpl::send_custom_query_data(const PublicKeyHash &dst, std::string name, td::Promise promise, td::Timestamp timeout, td::BufferSlice query) { - auto S = get_source_by_hash(dst); + CatChainReceiverSource *S = get_source_by_hash(dst); + CHECK(S != nullptr); td::actor::send_closure(overlay_manager_, &overlay::Overlays::send_query, S->get_adnl_id(), get_source(local_idx_)->get_adnl_id(), overlay_id_, std::move(name), std::move(promise), timeout, std::move(query)); } -void CatChainReceiverImpl::send_custom_query_data_via(PublicKeyHash dst, std::string name, +void CatChainReceiverImpl::send_custom_query_data_via(const PublicKeyHash &dst, std::string name, td::Promise promise, td::Timestamp timeout, td::BufferSlice query, td::uint64 max_answer_size, td::actor::ActorId via) { - auto S = get_source_by_hash(dst); + CatChainReceiverSource *S = get_source_by_hash(dst); + CHECK(S != nullptr); td::actor::send_closure(overlay_manager_, &overlay::Overlays::send_query_via, S->get_adnl_id(), get_source(local_idx_)->get_adnl_id(), overlay_id_, std::move(name), std::move(promise), timeout, std::move(query), max_answer_size, via); } -void CatChainReceiverImpl::send_custom_message_data(PublicKeyHash dst, td::BufferSlice data) { - auto S = get_source_by_hash(dst); +void CatChainReceiverImpl::send_custom_message_data(const PublicKeyHash &dst, td::BufferSlice data) { + CatChainReceiverSource *S = get_source_by_hash(dst); + CHECK(S != nullptr); td::actor::send_closure(overlay_manager_, &overlay::Overlays::send_message, S->get_adnl_id(), get_source(local_idx_)->get_adnl_id(), overlay_id_, std::move(data)); } void CatChainReceiverImpl::block_written_to_db(CatChainBlockHash hash) { - auto block = get_block(hash); + CatChainReceivedBlock *block = get_block(hash); CHECK(block); block->written(); run_scheduler(); } -static void destroy_db(std::string name, td::uint32 attempt) { +static void destroy_db(const std::string& name, td::uint32 attempt) { auto S = td::RocksDb::destroy(name); if (S.is_ok()) { return; } - if (S.is_error() && attempt >= 10) { - LOG(ERROR) << "failed to destroy catchain " << name << ": " << S; - } else { + if (S.is_error()) { LOG(DEBUG) << "failed to destroy catchain " << name << ": " << S; - delay_action([name, attempt]() { destroy_db(name, attempt); }, td::Timestamp::in(1.0)); + if (attempt < DESTROY_DB_MAX_ATTEMPTS) { + delay_action([name, attempt]() { destroy_db(name, attempt + 1); }, td::Timestamp::in(DESTROY_DB_DELAY)); + } } } void CatChainReceiverImpl::destroy() { auto name = db_root_ + "/catchainreceiver" + db_suffix_ + td::base64url_encode(as_slice(incarnation_)); - delay_action([name]() { destroy_db(name, 0); }, td::Timestamp::in(1.0)); + delay_action([name]() { destroy_db(name, 0); }, td::Timestamp::in(DESTROY_DB_DELAY)); stop(); } +td::uint64 get_max_block_height(const CatChainOptions& opts, size_t sources_cnt) { + if (opts.max_block_height_coeff == 0) { + return std::numeric_limits::max(); + } + return opts.max_block_height_coeff * (1 + (sources_cnt + opts.max_deps - 1) / opts.max_deps) / 1000; +} + } // namespace catchain } // namespace ton diff --git a/catchain/catchain-receiver.h b/catchain/catchain-receiver.h index c3bae8e0..2c959fbc 100644 --- a/catchain/catchain-receiver.h +++ b/catchain/catchain-receiver.h @@ -44,9 +44,6 @@ class CatChainReceiver : public CatChainReceiverInterface { CatChainSessionId instance_; PublicKeyHash local_id_; }; - td::uint32 get_max_neighbours() const { - return 5; - } virtual PrintId print_id() const = 0; virtual CatChainReceivedBlock *create_block(tl_object_ptr block, td::SharedSlice payload) = 0; @@ -64,12 +61,15 @@ class CatChainReceiver : public CatChainReceiverInterface { virtual const CatChainOptions &opts() const = 0; - virtual td::Status validate_block_sync(tl_object_ptr &dep) = 0; - virtual td::Status validate_block_sync(tl_object_ptr &block, td::Slice payload) = 0; + virtual td::Status validate_block_sync(const tl_object_ptr &dep) const = 0; + virtual td::Status validate_block_sync(const tl_object_ptr &block, + const td::Slice &payload) const = 0; virtual ~CatChainReceiver() = default; }; +td::uint64 get_max_block_height(const CatChainOptions& opts, size_t sources_cnt); + } // namespace catchain } // namespace ton diff --git a/catchain/catchain-receiver.hpp b/catchain/catchain-receiver.hpp index b41c8af0..5da4001c 100644 --- a/catchain/catchain-receiver.hpp +++ b/catchain/catchain-receiver.hpp @@ -33,7 +33,7 @@ namespace ton { namespace catchain { -class CatChainReceiverImpl : public CatChainReceiver { +class CatChainReceiverImpl final : public CatChainReceiver { public: PrintId print_id() const override { return PrintId{incarnation_, local_id_}; @@ -62,7 +62,7 @@ class CatChainReceiverImpl : public CatChainReceiver { return sources_[source_id].get(); } PublicKeyHash get_source_hash(td::uint32 source_id) const override; - CatChainReceiverSource *get_source_by_hash(PublicKeyHash source_hash) const; + CatChainReceiverSource *get_source_by_hash(const PublicKeyHash &source_hash) const; CatChainReceiverSource *get_source_by_adnl_id(adnl::AdnlNodeIdShort source_hash) const; td::uint32 add_fork() override; @@ -72,39 +72,39 @@ class CatChainReceiverImpl : public CatChainReceiver { void receive_message_from_overlay(adnl::AdnlNodeIdShort src, td::BufferSlice data); void receive_query_from_overlay(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise); - void process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getBlock &query, + void process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getBlock query, td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getBlocks query, td::Promise promise); - void process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getBlocks &query, + void process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getBlockHistory query, td::Promise promise); - void process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getBlockHistory &query, - td::Promise promise); - void process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getDifference &query, + void process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getDifference query, td::Promise promise); template - void process_query(adnl::AdnlNodeIdShort src, T &query, td::Promise promise) { + void process_query(adnl::AdnlNodeIdShort src, const T &query, td::Promise promise) { //LOG(WARNING) << this << ": unknown query from " << src; callback_->on_custom_query(get_source_by_adnl_id(src)->get_hash(), serialize_tl_object(&query, true), std::move(promise)); } - void receive_broadcast_from_overlay(PublicKeyHash src, td::BufferSlice data); + void receive_broadcast_from_overlay(const PublicKeyHash &src, td::BufferSlice data); void receive_block(adnl::AdnlNodeIdShort src, tl_object_ptr block, td::BufferSlice payload); void receive_block_answer(adnl::AdnlNodeIdShort src, td::BufferSlice); - //void send_block(PublicKeyHash src, tl_object_ptr block, td::BufferSlice payload); + //void send_block(const PublicKeyHash &src, tl_object_ptr block, td::BufferSlice payload); CatChainReceivedBlock *create_block(tl_object_ptr block, td::SharedSlice payload) override; CatChainReceivedBlock *create_block(tl_object_ptr block) override; - td::Status validate_block_sync(tl_object_ptr &dep) override; - td::Status validate_block_sync(tl_object_ptr &block, td::Slice payload) override; + td::Status validate_block_sync(const tl_object_ptr &dep) const override; + td::Status validate_block_sync(const tl_object_ptr &block, + const td::Slice &payload) const override; void send_fec_broadcast(td::BufferSlice data) override; - void send_custom_query_data(PublicKeyHash dst, std::string name, td::Promise promise, + void send_custom_query_data(const PublicKeyHash &dst, std::string name, td::Promise promise, td::Timestamp timeout, td::BufferSlice query) override; - void send_custom_query_data_via(PublicKeyHash dst, std::string name, td::Promise promise, + void send_custom_query_data_via(const PublicKeyHash &dst, std::string name, td::Promise promise, td::Timestamp timeout, td::BufferSlice query, td::uint64 max_answer_size, td::actor::ActorId via) override; - void send_custom_message_data(PublicKeyHash dst, td::BufferSlice query) override; + void send_custom_message_data(const PublicKeyHash &dst, td::BufferSlice query) override; void run_scheduler(); void add_block(td::BufferSlice data, std::vector deps) override; @@ -117,8 +117,6 @@ class CatChainReceiverImpl : public CatChainReceiver { void on_blame(td::uint32 src) override { callback_->blame(src); } - void blame_node(td::uint32 idx) override { - } const CatChainOptions &opts() const override { return opts_; } @@ -141,11 +139,11 @@ class CatChainReceiverImpl : public CatChainReceiver { CatChainReceivedBlock *get_block(CatChainBlockHash hash) const; - CatChainReceiverImpl(std::unique_ptr callback, CatChainOptions opts, + CatChainReceiverImpl(std::unique_ptr callback, const CatChainOptions &opts, td::actor::ActorId keyring, td::actor::ActorId adnl, - td::actor::ActorId, std::vector ids, PublicKeyHash local_id, - CatChainBlockHash unique_hash, std::string db_root, std::string db_suffix, - bool allow_unsafe_self_blocks_resync); + td::actor::ActorId overlays, const std::vector &ids, + const PublicKeyHash &local_id, const CatChainBlockHash &unique_hash, std::string db_root, + std::string db_suffix, bool allow_unsafe_self_blocks_resync); private: std::unique_ptr make_callback() { @@ -164,7 +162,7 @@ class CatChainReceiverImpl : public CatChainReceiver { void receive_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { td::actor::send_closure(id_, &CatChainReceiverImpl::receive_broadcast_from_overlay, src, std::move(data)); } - Callback(td::actor::ActorId id) : id_(std::move(id)) { + explicit Callback(td::actor::ActorId id) : id_(std::move(id)) { } private: @@ -199,7 +197,7 @@ class CatChainReceiverImpl : public CatChainReceiver { CatChainReceivedBlock *root_block_; CatChainReceivedBlock *last_sent_block_; - CatChainSessionId incarnation_; + CatChainSessionId incarnation_{}; std::unique_ptr callback_; CatChainOptions opts_; diff --git a/catchain/catchain-types.h b/catchain/catchain-types.h index 414e5946..4031acc5 100644 --- a/catchain/catchain-types.h +++ b/catchain/catchain-types.h @@ -20,6 +20,7 @@ #include "td/utils/int_types.h" #include "adnl/adnl-node-id.hpp" +#include "ton/ton-types.h" namespace ton { @@ -35,13 +36,6 @@ struct CatChainNode { PublicKey pub_key; }; -struct CatChainOptions { - double idle_timeout = 16.0; - td::uint32 max_deps = 4; - - bool debug_disable_db = false; -}; - } // namespace catchain } // namespace ton diff --git a/catchain/catchain.cpp b/catchain/catchain.cpp index abbc8b5c..001840fd 100644 --- a/catchain/catchain.cpp +++ b/catchain/catchain.cpp @@ -18,9 +18,9 @@ */ #include "catchain-types.h" #include "catchain.hpp" -#include "catchain-receiver.h" -#include "adnl/utils.hpp" +#include +#include "catchain-receiver.h" namespace ton { @@ -32,10 +32,11 @@ void CatChainImpl::send_process() { std::vector v; std::vector w; while (top_blocks_.size() > 0 && v.size() < opts_.max_deps) { - auto B = *top_blocks_.get_random(); + CatChainBlock *B = *top_blocks_.get_random(); CHECK(B != nullptr); top_blocks_.remove(B->hash()); - if (B->source() == sources_.size() || !blamed_sources_[B->source()]) { + CHECK(B->source() < sources_.size()); + if (!blamed_sources_[B->source()]) { w.push_back(B->hash()); v.push_back(B); set_processed(B); @@ -52,13 +53,13 @@ void CatChainImpl::send_preprocess(CatChainBlock *block) { if (block->preprocess_is_sent()) { return; } - auto prev = block->prev(); + CatChainBlock *prev = block->prev(); if (prev) { send_preprocess(prev); } - auto deps = block->deps(); - for (auto X : deps) { + const std::vector &deps = block->deps(); + for (CatChainBlock *X : deps) { send_preprocess(X); } @@ -72,13 +73,13 @@ void CatChainImpl::set_processed(CatChainBlock *block) { if (block->is_processed()) { return; } - auto prev = block->prev(); + CatChainBlock *prev = block->prev(); if (prev) { set_processed(prev); } - auto deps = block->deps(); - for (auto X : deps) { + const std::vector &deps = block->deps(); + for (CatChainBlock *X : deps) { set_processed(X); } @@ -133,6 +134,7 @@ void CatChainImpl::on_new_block(td::uint32 src_id, td::uint32 fork, CatChainBloc } } + CHECK(src_id < sources_.size()); std::vector v; v.resize(deps.size()); for (size_t i = 0; i < deps.size(); i++) { @@ -143,12 +145,12 @@ void CatChainImpl::on_new_block(td::uint32 src_id, td::uint32 fork, CatChainBloc CHECK(v[i] != nullptr); } - CHECK(src_id < sources_.size()); - auto src_hash = sources_[src_id]; + CHECK(height <= get_max_block_height(opts_, sources_.size())); + PublicKeyHash src_hash = sources_[src_id]; blocks_[hash] = CatChainBlock::create(src_id, fork, src_hash, height, hash, std::move(data), p, std::move(v), std::move(vt)); - auto B = get_block(hash); + CatChainBlock *B = get_block(hash); CHECK(B != nullptr); if (!blamed_sources_[src_id]) { @@ -177,7 +179,7 @@ void CatChainImpl::on_blame(td::uint32 src_id) { auto size = static_cast(sources_.size()); for (td::uint32 i = 0; i < size; i++) { if (!blamed_sources_[i] && top_source_blocks_[i] && i != local_idx_) { - auto B = top_source_blocks_[i]; + CatChainBlock *B = top_source_blocks_[i]; bool f = true; if (B->is_processed()) { continue; @@ -197,15 +199,11 @@ void CatChainImpl::on_blame(td::uint32 src_id) { } } -void CatChainImpl::on_custom_message(PublicKeyHash src, td::BufferSlice data) { - callback_->process_message(src, std::move(data)); -} - -void CatChainImpl::on_custom_query(PublicKeyHash src, td::BufferSlice data, td::Promise promise) { +void CatChainImpl::on_custom_query(const PublicKeyHash &src, td::BufferSlice data, td::Promise promise) { callback_->process_query(src, std::move(data), std::move(promise)); } -void CatChainImpl::on_broadcast(PublicKeyHash src, td::BufferSlice data) { +void CatChainImpl::on_broadcast(const PublicKeyHash &src, td::BufferSlice data) { VLOG(CATCHAIN_INFO) << this << ": processing broadcast"; callback_->process_broadcast(src, std::move(data)); VLOG(CATCHAIN_INFO) << this << ": sent processing broadcast"; @@ -219,18 +217,18 @@ void CatChainImpl::on_receiver_started() { send_process(); } -CatChainImpl::CatChainImpl(std::unique_ptr callback, CatChainOptions opts, +CatChainImpl::CatChainImpl(std::unique_ptr callback, const CatChainOptions &opts, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId overlay_manager, std::vector ids, - PublicKeyHash local_id, CatChainSessionId unique_hash, std::string db_root, - std::string db_suffix, bool allow_unsafe_self_blocks_resync) - : opts_(std::move(opts)) - , db_root_(db_root) - , db_suffix_(db_suffix) + const PublicKeyHash &local_id, const CatChainSessionId &unique_hash, + std::string db_root, std::string db_suffix, bool allow_unsafe_self_blocks_resync) + : opts_(opts) + , unique_hash_(unique_hash) + , db_root_(std::move(db_root)) + , db_suffix_(std::move(db_suffix)) , allow_unsafe_self_blocks_resync_(allow_unsafe_self_blocks_resync) { callback_ = std::move(callback); sources_.resize(ids.size()); - unique_hash_ = unique_hash; for (size_t i = 0; i < ids.size(); i++) { sources_[i] = ids[i].pub_key.compute_short_id(); if (sources_[i] == local_id) { @@ -240,7 +238,8 @@ CatChainImpl::CatChainImpl(std::unique_ptr callback, CatChainOptions o blamed_sources_.resize(ids.size(), false); top_source_blocks_.resize(ids.size(), nullptr); - args_ = std::make_unique(keyring, adnl, overlay_manager, std::move(ids), local_id, unique_hash); + args_ = std::make_unique(std::move(keyring), std::move(adnl), std::move(overlay_manager), std::move(ids), + local_id, unique_hash); } void CatChainImpl::alarm() { @@ -263,19 +262,16 @@ void CatChainImpl::start_up() { void blame(td::uint32 src_id) override { td::actor::send_closure(id_, &CatChainImpl::on_blame, src_id); } - void on_custom_message(PublicKeyHash src, td::BufferSlice data) override { - td::actor::send_closure(id_, &CatChainImpl::on_custom_message, src, std::move(data)); - } - void on_custom_query(PublicKeyHash src, td::BufferSlice data, td::Promise promise) override { + void on_custom_query(const PublicKeyHash &src, td::BufferSlice data, td::Promise promise) override { td::actor::send_closure(id_, &CatChainImpl::on_custom_query, src, std::move(data), std::move(promise)); } - void on_broadcast(PublicKeyHash src, td::BufferSlice data) override { + void on_broadcast(const PublicKeyHash &src, td::BufferSlice data) override { td::actor::send_closure(id_, &CatChainImpl::on_broadcast, src, std::move(data)); } void start() override { td::actor::send_closure(id_, &CatChainImpl::on_receiver_started); } - ChainCb(td::actor::ActorId id) : id_(id) { + explicit ChainCb(td::actor::ActorId id) : id_(std::move(id)) { } private: @@ -285,22 +281,23 @@ void CatChainImpl::start_up() { auto cb = std::make_unique(actor_id(this)); receiver_ = CatChainReceiverInterface::create( - std::move(cb), opts_, args_->keyring, args_->adnl, args_->overlay_manager, std::move(args_->ids), args_->local_id, + std::move(cb), opts_, args_->keyring, args_->adnl, args_->overlay_manager, args_->ids, args_->local_id, args_->unique_hash, db_root_, db_suffix_, allow_unsafe_self_blocks_resync_); args_ = nullptr; //alarm_timestamp() = td::Timestamp::in(opts_.idle_timeout); } -td::actor::ActorOwn CatChain::create(std::unique_ptr callback, CatChainOptions opts, +td::actor::ActorOwn CatChain::create(std::unique_ptr callback, const CatChainOptions &opts, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId overlay_manager, - std::vector ids, PublicKeyHash local_id, - CatChainSessionId unique_hash, std::string db_root, + std::vector ids, const PublicKeyHash &local_id, + const CatChainSessionId &unique_hash, std::string db_root, std::string db_suffix, bool allow_unsafe_self_blocks_resync) { - return td::actor::create_actor("catchain", std::move(callback), std::move(opts), keyring, adnl, - overlay_manager, std::move(ids), local_id, unique_hash, db_root, - db_suffix, allow_unsafe_self_blocks_resync); + return td::actor::create_actor("catchain", std::move(callback), opts, std::move(keyring), + std::move(adnl), std::move(overlay_manager), std::move(ids), + local_id, unique_hash, std::move(db_root), std::move(db_suffix), + allow_unsafe_self_blocks_resync); } CatChainBlock *CatChainImpl::get_block(CatChainBlockHash hash) const { diff --git a/catchain/catchain.h b/catchain/catchain.h index 6e832e49..912957e5 100644 --- a/catchain/catchain.h +++ b/catchain/catchain.h @@ -58,8 +58,8 @@ class CatChainBlock { virtual bool is_descendant_of(CatChainBlock *block) = 0; - static std::unique_ptr create(td::uint32 src, td::uint32 fork_id, PublicKeyHash src_hash, - CatChainBlockHeight height, CatChainBlockHash hash, + static std::unique_ptr create(td::uint32 src, td::uint32 fork_id, const PublicKeyHash &src_hash, + CatChainBlockHeight height, const CatChainBlockHash &hash, td::SharedSlice payload, CatChainBlock *prev, std::vector deps, std::vector vt); @@ -73,9 +73,10 @@ class CatChain : public td::actor::Actor { virtual void process_blocks(std::vector blocks) = 0; virtual void finished_processing() = 0; virtual void preprocess_block(CatChainBlock *block) = 0; - virtual void process_broadcast(PublicKeyHash src, td::BufferSlice data) = 0; - virtual void process_message(PublicKeyHash src, td::BufferSlice data) = 0; - virtual void process_query(PublicKeyHash src, td::BufferSlice data, td::Promise promise) = 0; + virtual void process_broadcast(const PublicKeyHash &src, td::BufferSlice data) = 0; + virtual void process_message(const PublicKeyHash &src, td::BufferSlice data) = 0; + virtual void process_query(const PublicKeyHash &src, td::BufferSlice data, + td::Promise promise) = 0; virtual void started() = 0; virtual ~Callback() = default; }; @@ -89,22 +90,22 @@ class CatChain : public td::actor::Actor { virtual void debug_add_fork(td::BufferSlice payload, CatChainBlockHeight height) = 0; virtual void send_broadcast(td::BufferSlice data) = 0; - virtual void send_message(PublicKeyHash dst, td::BufferSlice data) = 0; - virtual void send_query(PublicKeyHash dst, std::string name, td::Promise promise, + virtual void send_message(const PublicKeyHash &dst, td::BufferSlice data) = 0; + virtual void send_query(const PublicKeyHash &dst, std::string name, td::Promise promise, td::Timestamp timeout, td::BufferSlice query) = 0; - virtual void send_query_via(PublicKeyHash dst, std::string name, td::Promise promise, + virtual void send_query_via(const PublicKeyHash &dst, std::string name, td::Promise promise, td::Timestamp timeout, td::BufferSlice query, td::uint64 max_answer_size, td::actor::ActorId via) = 0; virtual void destroy() = 0; - static td::actor::ActorOwn create(std::unique_ptr callback, CatChainOptions opts, + static td::actor::ActorOwn create(std::unique_ptr callback, const CatChainOptions &opts, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId overlay_manager, - std::vector ids, PublicKeyHash local_id, - CatChainSessionId unique_hash, std::string db_root, std::string db_suffix, - bool allow_unsafe_self_blocks_resync); - virtual ~CatChain() = default; + std::vector ids, const PublicKeyHash &local_id, + const CatChainSessionId &unique_hash, std::string db_root, + std::string db_suffix, bool allow_unsafe_self_blocks_resync); + ~CatChain() override = default; }; } // namespace catchain diff --git a/catchain/catchain.hpp b/catchain/catchain.hpp index 2a1bc608..8c8bb99a 100644 --- a/catchain/catchain.hpp +++ b/catchain/catchain.hpp @@ -67,11 +67,11 @@ class CatChainImpl : public CatChain { CatChainSessionId unique_hash; Args(td::actor::ActorId keyring, td::actor::ActorId adnl, - td::actor::ActorId overlay_manager, std::vector ids, PublicKeyHash local_id, - CatChainSessionId unique_hash) - : keyring(keyring) - , adnl(adnl) - , overlay_manager(overlay_manager) + td::actor::ActorId overlay_manager, std::vector ids, + const PublicKeyHash &local_id, const CatChainSessionId &unique_hash) + : keyring(std::move(keyring)) + , adnl(std::move(adnl)) + , overlay_manager(std::move(overlay_manager)) , ids(std::move(ids)) , local_id(local_id) , unique_hash(unique_hash) { @@ -88,9 +88,8 @@ class CatChainImpl : public CatChain { CatChainBlockHash prev, std::vector deps, std::vector vt, td::SharedSlice data); void on_blame(td::uint32 src_id); - void on_custom_message(PublicKeyHash src, td::BufferSlice data); - void on_custom_query(PublicKeyHash src, td::BufferSlice data, td::Promise promise); - void on_broadcast(PublicKeyHash src, td::BufferSlice data); + void on_custom_query(const PublicKeyHash &src, td::BufferSlice data, td::Promise promise); + void on_broadcast(const PublicKeyHash &src, td::BufferSlice data); void on_receiver_started(); void processed_block(td::BufferSlice payload) override; void need_new_block(td::Timestamp t) override; @@ -102,25 +101,26 @@ class CatChainImpl : public CatChain { void send_broadcast(td::BufferSlice data) override { td::actor::send_closure(receiver_, &CatChainReceiverInterface::send_fec_broadcast, std::move(data)); } - void send_message(PublicKeyHash dst, td::BufferSlice data) override { + void send_message(const PublicKeyHash &dst, td::BufferSlice data) override { td::actor::send_closure(receiver_, &CatChainReceiverInterface::send_custom_message_data, dst, std::move(data)); } - void send_query(PublicKeyHash dst, std::string name, td::Promise promise, td::Timestamp timeout, - td::BufferSlice query) override { + void send_query(const PublicKeyHash &dst, std::string name, td::Promise promise, + td::Timestamp timeout, td::BufferSlice query) override { td::actor::send_closure(receiver_, &CatChainReceiverInterface::send_custom_query_data, dst, name, std::move(promise), timeout, std::move(query)); } - void send_query_via(PublicKeyHash dst, std::string name, td::Promise promise, td::Timestamp timeout, - td::BufferSlice query, td::uint64 max_answer_size, + void send_query_via(const PublicKeyHash &dst, std::string name, td::Promise promise, + td::Timestamp timeout, td::BufferSlice query, td::uint64 max_answer_size, td::actor::ActorId via) override { td::actor::send_closure(receiver_, &CatChainReceiverInterface::send_custom_query_data_via, dst, name, std::move(promise), timeout, std::move(query), max_answer_size, via); } void destroy() override; - CatChainImpl(std::unique_ptr callback, CatChainOptions opts, td::actor::ActorId keyring, - td::actor::ActorId adnl, td::actor::ActorId overlay_manager, - std::vector ids, PublicKeyHash local_id, CatChainSessionId unique_hash, - std::string db_root, std::string db_suffix, bool allow_unsafe_self_blocks_resync); + CatChainImpl(std::unique_ptr callback, const CatChainOptions &opts, + td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId overlay_manager, std::vector ids, + const PublicKeyHash &local_id, const CatChainSessionId &unique_hash, std::string db_root, + std::string db_suffix, bool allow_unsafe_self_blocks_resync); void alarm() override; void start_up() override; diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 46b29cf2..dae517d4 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -703,6 +703,13 @@ consensus_config_v3#d8 flags:(## 7) { flags = 0 } new_catchain_ids:Bool max_block_bytes:uint32 max_collated_bytes:uint32 proto_version:uint16 = ConsensusConfig; +consensus_config_v4#d9 flags:(## 7) { flags = 0 } new_catchain_ids:Bool + round_candidates:(## 8) { round_candidates >= 1 } + next_candidate_delay_ms:uint32 consensus_timeout_ms:uint32 + fast_attempts:uint32 attempt_duration:uint32 catchain_max_deps:uint32 + max_block_bytes:uint32 max_collated_bytes:uint32 + proto_version:uint16 catchain_max_blocks_coeff:uint32 = ConsensusConfig; + _ CatchainConfig = ConfigParam 28; _ ConsensusConfig = ConfigParam 29; diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index 003cd04b..b5409a62 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -310,32 +310,52 @@ td::Status Config::visit_validator_params() const { ton::ValidatorSessionConfig Config::get_consensus_config() const { auto cc = get_config_param(29); ton::ValidatorSessionConfig c; - auto set = [&c](auto& r) { - c.catchain_idle_timeout = r.consensus_timeout_ms * 0.001; - c.catchain_max_deps = r.catchain_max_deps; + auto set_v1 = [&](auto& r) { + c.catchain_opts.idle_timeout = r.consensus_timeout_ms * 0.001; + c.catchain_opts.max_deps = r.catchain_max_deps; c.round_candidates = r.round_candidates; c.next_candidate_delay = r.next_candidate_delay_ms * 0.001; c.round_attempt_duration = r.attempt_duration; c.max_round_attempts = r.fast_attempts; c.max_block_size = r.max_block_bytes; c.max_collated_data_size = r.max_collated_bytes; - return true; }; - auto set_new_cc_ids = [&c] (auto& r) { + auto set_v2 = [&] (auto& r) { + set_v1(r); c.new_catchain_ids = r.new_catchain_ids; - return true; }; - auto set_proto = [&c](auto& r) { + auto set_v3 = [&](auto& r) { + set_v2(r); c.proto_version = r.proto_version; - return true; + }; + auto set_v4 = [&](auto& r) { + set_v3(r); + td::uint64 max_blocks_coeff = r.catchain_max_blocks_coeff; + if (max_blocks_coeff == 0) { + c.catchain_opts.max_block_height_coeff = 0; + } else { + auto catchain_config = get_catchain_validators_config(); + td::uint64 catchain_lifetime = std::max(catchain_config.mc_cc_lifetime, catchain_config.shard_cc_lifetime); + c.catchain_opts.max_block_height_coeff = catchain_lifetime * max_blocks_coeff; + } }; if (cc.not_null()) { - block::gen::ConsensusConfig::Record_consensus_config_v3 r2; - block::gen::ConsensusConfig::Record_consensus_config_new r1; - block::gen::ConsensusConfig::Record_consensus_config r0; - (tlb::unpack_cell(cc, r2) && set(r2) && set_new_cc_ids(r2) && set_proto(r2)) || - (tlb::unpack_cell(cc, r1) && set(r1) && set_new_cc_ids(r1)) || - (tlb::unpack_cell(cc, r0) && set(r0)); + block::gen::ConsensusConfig::Record_consensus_config_v4 r4; + block::gen::ConsensusConfig::Record_consensus_config_v3 r3; + block::gen::ConsensusConfig::Record_consensus_config_new r2; + block::gen::ConsensusConfig::Record_consensus_config r1; + if (tlb::unpack_cell(cc, r4)) { + set_v4(r4); + } else if (tlb::unpack_cell(cc, r3)) { + set_v3(r3); + } else if (tlb::unpack_cell(cc, r2)) { + set_v2(r2); + } else if (tlb::unpack_cell(cc, r1)) { + set_v1(r1); + } + } + if (c.proto_version >= ton::ValidatorSessionConfig::BLOCK_HASH_COVERS_DATA_FROM_VERSION) { + c.catchain_opts.block_hash_covers_data = true; } return c; } diff --git a/doc/catchain-dos.md b/doc/catchain-dos.md new file mode 100644 index 00000000..028cb95a --- /dev/null +++ b/doc/catchain-dos.md @@ -0,0 +1,22 @@ +# Catchain DoS protection +## Consensus-safe upgrade +Since catchain protocol describes the way validators come to consensus, it should be updated in simultaneous manner to ensure that at any moment of time at least 2/3 of validators (measured by weight) are using the same version of the protocol. Innate way to do it in TON is to use Configuration parameters, when absence or old version of some configuration parameter corresponds to the old behavior, while new version controls updated catchain protocol behavior. + +That way, update of catchain protocol should be done in the following way: each validator upgrades its node software and votes for a new config parameter. This method ensures that switching to updated catchain protocol will happen only after 3/4 (more than 2/3) of validators upgrade. + +## Covering block dependencies by hash +Catchain protocol version is controlled by `ConfigParam 29`: `proto_version` field in constructors `consensus_config_v3` and older (for other constructors `proto_version` is equal to `0`). + +For catchain protocol version higher or equal to 2 hash covers catchain block dependencies, that prevents relaying of blocks with altered dependencies. + +## Limiting maximal catchain block height and size +### Catchain block size +All catchain messages, except `REJECT` message, have (and had) a limited size. After update `REJECT` message will be limited to 1024 bytes. At the same time one block contains at most number of block candidates per round messages. That way, 16kb limit per catchain block should be enough to prevent DoS. +### Limiting block height +Another potential DoS is related to a situation when a malbehaviouring node sends too many catchain blocks. Note that limiting of maximal number of blocks per second is not a good solution since it will hinder synchronization after node disconnect. +At the same time, catchain groups exist for quite short period of time (around a few hunderd seconds), while number of blocks production is determined by "natural block production speed" on the one hand and number of blocks generated to decrease dependencies size on the other. In any case, total number of blocks is limited by `catchain_lifetime * natural_block_production_speed * (1 + number_of_catchain_participants / max_dependencies_size)`. +To prevent DoS attack we limit maximal height of the block which will be processed by node by `catchain_lifetime * natural_block_production_speed * (1 + number_of_catchain_participants / max_dependencies_size)`, where `catchain_lifetime` is set by `ConfigParam 28` (`CatchainConfig`), `natural_block_production_speed` and `max_dependencies_size` are set by `ConfigParam 29` (`ConsensusConfig`) (`natural_block_production_speed` is calculated as `catchain_max_blocks_coeff / 1000`) and `number_of_catchain_participants` is set from catchain group configuration. + +By default, `catchain_max_blocks_coeff` is set to zero: special value which means that there is no limitation on catchain block height. It is recommended to set `catchain_max_blocks_coeff` to `10000`: we expect that natural production rate is about one block per 3 seconds, so we set the coefficient to allow 30 times higher block production than expected. At the same time, this number is low enough to prevent resource-intensive attacks. + +To prevent DoS with many blocks of the same height, nodes will ignore any blocks from a "bad" node (i.e. one which created a fork) unless the new block is required by some other block from a "good" node. diff --git a/test/test-catchain.cpp b/test/test-catchain.cpp index 88f5c493..53b32924 100644 --- a/test/test-catchain.cpp +++ b/test/test-catchain.cpp @@ -122,8 +122,9 @@ class CatChainInst : public td::actor::Actor { void start_up() override { alarm_timestamp() = td::Timestamp::in(0.1); - ton::catchain::CatChainOptions opts; + ton::CatChainOptions opts; opts.debug_disable_db = true; + //opts.block_hash_covers_data = true; std::vector nodes; for (auto &n : nodes_) { @@ -157,13 +158,14 @@ class CatChainInst : public td::actor::Actor { void preprocess_block(ton::catchain::CatChainBlock *block) override { td::actor::send_closure(id_, &CatChainInst::preprocess_block, std::move(block)); } - void process_broadcast(ton::PublicKeyHash src, td::BufferSlice data) override { + void process_broadcast(const ton::PublicKeyHash &src, td::BufferSlice data) override { UNREACHABLE(); } - void process_message(ton::PublicKeyHash src, td::BufferSlice data) override { + void process_message(const ton::PublicKeyHash &src, td::BufferSlice data) override { UNREACHABLE(); } - void process_query(ton::PublicKeyHash src, td::BufferSlice data, td::Promise promise) override { + void process_query(const ton::PublicKeyHash &src, td::BufferSlice data, + td::Promise promise) override { UNREACHABLE(); } void started() override { diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 7f52198b..a1fddf2b 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -259,7 +259,7 @@ catchain.blockUpdate block:catchain.block = catchain.Update; catchain.block.data.badBlock block:catchain.block = catchain.block.inner.Data; catchain.block.data.fork left:catchain.block.Dep right:catchain.block.Dep = catchain.block.inner.Data; catchain.block.data.nop = catchain.block.inner.Data; -catchain.block.data.vector msgs:(vector bytes) = catchain.block.inner.Data; +//catchain.block.data.vector msgs:(vector bytes) = catchain.block.inner.Data; //catchain.block.data.custom = catchain.block.inner.Data; catchain.firstblock unique_hash:int256 nodes:(vector int256) = catchain.FirstBlock; @@ -309,11 +309,14 @@ validatorSession.candidate src:int256 round:int root_hash:int256 data:bytes coll validatorSession.config catchain_idle_timeout:double catchain_max_deps:int round_candidates:int next_candidate_delay:double round_attempt_duration:int max_round_attempts:int max_block_size:int max_collated_data_size:int = validatorSession.Config; -validatorSession.configVersioned catchain_idle_timeout:double catchain_max_deps:int round_candidates:int next_candidate_delay:double round_attempt_duration:int - max_round_attempts:int max_block_size:int max_collated_data_size:int version:int = validatorSession.Config; validatorSession.configNew catchain_idle_timeout:double catchain_max_deps:int round_candidates:int next_candidate_delay:double round_attempt_duration:int max_round_attempts:int max_block_size:int max_collated_data_size:int new_catchain_ids:Bool = validatorSession.Config; +validatorSession.catchainOptions idle_timeout:double max_deps:int max_block_size:int block_hash_covers_data:Bool + max_block_height_ceoff:int debug_disable_db:Bool = validatorSession.CatChainOptions; +validatorSession.configVersioned catchain_opts:validatorSession.CatChainOptions round_candidates:int next_candidate_delay:double + round_attempt_duration:int max_round_attempts:int max_block_size:int max_collated_data_size:int version:int = validatorSession.Config; + ---functions--- validatorSession.ping hash:long = validatorSession.Pong; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index fad15df58f4f7f8b0e0059b41a944aa25e7faba6..dd308bc0040c8e17b01d8f9698e6db5fd3b48c1d 100644 GIT binary patch delta 455 zcmX@o&a$MFh4;~HeJchiIJS{@Iy=jiE5F(%uV9y9WS;0KzWEOO6lvCHo{tI?CNnAv z^OiVgBxdIM7nEe?=M_&*#=r)cn$t$&DwaIXQ9@E8L{=$21^;pm4l~5zLxgaY_dwwBpnXMiEd5f&2;rSLL{3 zJkEp)P2O-qMHeIi@&X4?0^|i=m@nh=3rdQ?>KQo%dStswzR)&}! Ia;Cr>0E$_#$p8QV delta 283 zcmZ3|$#SHfh4;~HeJchiIJ%K{Iy=k1-@%V2uV9y9WSQtFzWEOO6ls>2!%rADZ&9w$ zg-8nc7(f^)fjSUIfUm;jErC$c7l9g)AVY2*6Y>F>T;$BA!(rQY5F`e(>Z%-9jF$e& z6shFIlH`oU%sjoMoc!c$y_Cd~M7^@q @@ -447,11 +448,23 @@ struct ValidatorDescr { } }; +struct CatChainOptions { + double idle_timeout = 16.0; + td::uint32 max_deps = 4; + td::uint32 max_serialized_block_size = 16 * 1024; + bool block_hash_covers_data = false; + // Max block height = max_block_height_coeff * (1 + N / max_deps) / 1000 + // N - number of participants + // 0 - unlimited + td::uint64 max_block_height_coeff = 0; + + bool debug_disable_db = false; +}; + struct ValidatorSessionConfig { td::uint32 proto_version = 0; - /* double */ double catchain_idle_timeout = 16.0; - td::uint32 catchain_max_deps = 4; + CatChainOptions catchain_opts; td::uint32 round_candidates = 3; /* double */ double next_candidate_delay = 2.0; @@ -462,6 +475,8 @@ struct ValidatorSessionConfig { td::uint32 max_collated_data_size = (4 << 20); bool new_catchain_ids = false; + + static const td::uint32 BLOCK_HASH_COVERS_DATA_FROM_VERSION = 2; }; } // namespace ton diff --git a/validator-session/validator-session-types.h b/validator-session/validator-session-types.h index 812c20b8..f1eef563 100644 --- a/validator-session/validator-session-types.h +++ b/validator-session/validator-session-types.h @@ -37,11 +37,10 @@ inline ValidatorSessionCandidateId skip_round_candidate_id() { } struct ValidatorSessionOptions { - ValidatorSessionOptions() { - } - ValidatorSessionOptions(const ValidatorSessionConfig &conf); - double catchain_idle_timeout = 16.0; - td::uint32 catchain_max_deps = 4; + ValidatorSessionOptions() = default; + explicit ValidatorSessionOptions(const ValidatorSessionConfig &conf); + + CatChainOptions catchain_opts; td::uint32 round_candidates = 3; double next_candidate_delay = 2.0; diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index 088d2957..4edd3d58 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -337,6 +337,9 @@ void ValidatorSessionImpl::candidate_decision_fail(td::uint32 round, ValidatorSe } LOG(ERROR) << this << ": failed candidate " << hash << ": " << result; pending_approve_.erase(hash); + if (result.size() > MAX_REJECT_REASON_SIZE) { + result.resize(MAX_REJECT_REASON_SIZE); + } pending_reject_.emplace(hash, td::BufferSlice{result}); rejected_.insert(hash); } @@ -840,10 +843,8 @@ void ValidatorSessionImpl::start() { auto w = description().export_catchain_nodes(); catchain_ = catchain::CatChain::create( - make_catchain_callback(), - catchain::CatChainOptions{description().opts().catchain_idle_timeout, description().opts().catchain_max_deps}, - keyring_, adnl_, overlay_manager_, std::move(w), local_id(), unique_hash_, db_root_, db_suffix_, - allow_unsafe_self_blocks_resync_); + make_catchain_callback(), description().opts().catchain_opts, keyring_, adnl_, overlay_manager_, std::move(w), + local_id(), unique_hash_, db_root_, db_suffix_, allow_unsafe_self_blocks_resync_); check_all(); } @@ -883,26 +884,27 @@ td::actor::ActorOwn ValidatorSession::create( td::Bits256 ValidatorSessionOptions::get_hash() const { if(!proto_version) { if (!new_catchain_ids) { - return create_hash_tl_object( - catchain_idle_timeout, catchain_max_deps, round_candidates, next_candidate_delay, round_attempt_duration, - max_round_attempts, max_block_size, max_collated_data_size); + return create_hash_tl_object( + catchain_opts.idle_timeout, catchain_opts.max_deps, round_candidates, next_candidate_delay, + round_attempt_duration, max_round_attempts, max_block_size, max_collated_data_size); } else { - return create_hash_tl_object( - catchain_idle_timeout, catchain_max_deps, round_candidates, next_candidate_delay, round_attempt_duration, - max_round_attempts, max_block_size, max_collated_data_size, new_catchain_ids); + return create_hash_tl_object( + catchain_opts.idle_timeout, catchain_opts.max_deps, round_candidates, next_candidate_delay, + round_attempt_duration, max_round_attempts, max_block_size, max_collated_data_size, new_catchain_ids); } } else { - return create_hash_tl_object( - catchain_idle_timeout, catchain_max_deps, round_candidates, next_candidate_delay, round_attempt_duration, - max_round_attempts, max_block_size, max_collated_data_size, proto_version); - } - + return create_hash_tl_object( + create_tl_object( + catchain_opts.idle_timeout, catchain_opts.max_deps, catchain_opts.max_serialized_block_size, + catchain_opts.block_hash_covers_data, catchain_opts.max_block_height_coeff, catchain_opts.debug_disable_db), + round_candidates, next_candidate_delay, round_attempt_duration, + max_round_attempts, max_block_size, max_collated_data_size, proto_version); + } } ValidatorSessionOptions::ValidatorSessionOptions(const ValidatorSessionConfig &conf) { proto_version = conf.proto_version; - catchain_idle_timeout = conf.catchain_idle_timeout; - catchain_max_deps = conf.catchain_max_deps; + catchain_opts = conf.catchain_opts; max_block_size = conf.max_block_size; max_collated_data_size = conf.max_collated_data_size; max_round_attempts = conf.max_round_attempts; diff --git a/validator-session/validator-session.hpp b/validator-session/validator-session.hpp index f9c94a2f..9b7f9508 100644 --- a/validator-session/validator-session.hpp +++ b/validator-session/validator-session.hpp @@ -109,13 +109,14 @@ class ValidatorSessionImpl : public ValidatorSession { void preprocess_block(catchain::CatChainBlock *block) override { td::actor::send_closure(id_, &ValidatorSessionImpl::preprocess_block, block); } - void process_broadcast(PublicKeyHash src, td::BufferSlice data) override { + void process_broadcast(const PublicKeyHash &src, td::BufferSlice data) override { td::actor::send_closure(id_, &ValidatorSessionImpl::process_broadcast, src, std::move(data)); } - void process_message(PublicKeyHash src, td::BufferSlice data) override { + void process_message(const PublicKeyHash &src, td::BufferSlice data) override { td::actor::send_closure(id_, &ValidatorSessionImpl::process_message, src, std::move(data)); } - void process_query(PublicKeyHash src, td::BufferSlice data, td::Promise promise) override { + void process_query(const PublicKeyHash &src, td::BufferSlice data, + td::Promise promise) override { td::actor::send_closure(id_, &ValidatorSessionImpl::process_query, src, std::move(data), std::move(promise)); } void started() override { @@ -195,6 +196,9 @@ class ValidatorSessionImpl : public ValidatorSession { PrintId print_id() const override { return PrintId{unique_hash_, description_->get_source_id(description_->get_self_idx())}; } + + private: + static const size_t MAX_REJECT_REASON_SIZE = 1024; }; } // namespace validatorsession From acba17a025857f2d679ae93c61243ff6285618f8 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Mon, 8 Aug 2022 09:42:23 +0300 Subject: [PATCH 16/16] Fix README --- README.md | 51 ++++++++++++++++++++++++++------------------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index efa5e971..df8fefa8 100644 --- a/README.md +++ b/README.md @@ -1,26 +1,27 @@ -# TON CatChain +# TON -| Name | Fixed? | Comment | -|------|---|---------------------------------------| -|CA2-01| ✔️ | | -|CA2-02| ✔️ | | -|CA5-01| ✔️ | `hash(payload)` now also covers block data.
This behavior is enabled by a flag in config. | -|CA5-02| ✔️ | Size of a block is now limited (16kb).
Block height is now limited, and blocks from misbehaviouring nodes are now ignored.
See `doc/catchain-dos.md` for details. | -|CA5-03| ✔️ | | -|CA5-04| ✔️ | | -|CA5-05| ✔️ | | -|CA5-06| ✔️ | | -|CA5-07| ✔️ | | -|CA5-08| ✔️ | | -|CA5-09| ✔️ | | -|CA5-10| ✔️ | | -|CA5-11| ✔️ | | -|CAA-01| ✔️ | Fixed in the same way as CA5-01 (hash now covers block data). | -|CAA-02| ✔️ | | -|CAE-01| ✔️ | | -|CAN-01| ✔️ | | -|CAN-02| ✔️ | | -|CAT-01| ✔️ | | -|CAT-02| ✔️ | | -|CAT-03| ✔️ | | -|CAT-04| ✔️ | | +Main TON monorepo, which includes the code of the node/validator, lite-client, tonlib, FunC compiler, etc. + +## Updates flow: + +* **master branch** - mainnet is running on this stable branch. + + Only emergency updates, urgent updates, or updates that do not affect the main codebase (GitHub workflows / docker images / documentation) are committed directly to this branch. + +* **testnet branch** - testnet is running on this branch. The branch contains a set of new updates. After testing, the testnet branch is merged into the master branch and then a new set of updates is added to testnet branch. + +* **backlog** - other branches that are candidates to getting into the testnet branch in the next iteration. + +Usually, the response to your pull request will indicate which section it falls into. + + +## "Soft" Pull Request rules + +* Thou shall not merge your own PRs, at least one person should review the PR and merge it (4-eyes rule) +* Thou shall make sure that workflows are cleanly completed for your PR before considering merge + +## Workflows responsibility +If a CI workflow fails not because of your changes but workflow issues, try to fix it yourself or contact one of the persons listed below via Telegram messenger: + +* **C/C++ CI (ccpp-linux.yml)**: TBD +* **C/C++ CI Win64 Compile (ccpp-win64.yml)**: TBD