From 3e806bf46e30b0803f5ade93b2bf8e6052220260 Mon Sep 17 00:00:00 2001 From: Marat <98183742+dungeon-master-666@users.noreply.github.com> Date: Tue, 15 Oct 2024 16:43:09 +0200 Subject: [PATCH 1/6] Limit RocksDB logs size (#1278) * fix logging of failed destroy attempts * limit rocksdb log size --- tddb/td/db/RocksDb.cpp | 2 ++ validator/db/archive-slice.cpp | 13 ++++--------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/tddb/td/db/RocksDb.cpp b/tddb/td/db/RocksDb.cpp index 0be79cb3..f1aa64a5 100644 --- a/tddb/td/db/RocksDb.cpp +++ b/tddb/td/db/RocksDb.cpp @@ -85,6 +85,8 @@ Result RocksDb::open(std::string path, RocksDbOptions options) { db_options.bytes_per_sync = 1 << 20; db_options.writable_file_max_buffer_size = 2 << 14; db_options.statistics = options.statistics; + db_options.max_log_file_size = 100 << 20; + db_options.keep_log_file_num = 1; rocksdb::OptimisticTransactionDBOptions occ_options; occ_options.validate_policy = rocksdb::OccValidationPolicy::kValidateSerial; rocksdb::ColumnFamilyOptions cf_options(db_options); diff --git a/validator/db/archive-slice.cpp b/validator/db/archive-slice.cpp index d392431a..83fe144b 100644 --- a/validator/db/archive-slice.cpp +++ b/validator/db/archive-slice.cpp @@ -768,22 +768,19 @@ void destroy_db(std::string name, td::uint32 attempt, td::Promise prom promise.set_value(td::Unit()); return; } - if (S.is_error() && attempt > 0 && attempt % 64 == 0) { + if (attempt > 0 && attempt % 64 == 0) { LOG(ERROR) << "failed to destroy index " << name << ": " << S; } else { LOG(DEBUG) << "failed to destroy index " << name << ": " << S; } delay_action( - [name, attempt, promise = std::move(promise)]() mutable { destroy_db(name, attempt, std::move(promise)); }, + [name, attempt, promise = std::move(promise)]() mutable { destroy_db(name, attempt + 1, std::move(promise)); }, td::Timestamp::in(1.0)); } } // namespace void ArchiveSlice::destroy(td::Promise promise) { before_query(); - td::MultiPromise mp; - auto ig = mp.init_guard(); - ig.add_promise(std::move(promise)); destroyed_ = true; for (auto &p : packages_) { @@ -795,10 +792,8 @@ void ArchiveSlice::destroy(td::Promise promise) { packages_.clear(); kv_ = nullptr; - PackageId p_id{archive_id_, key_blocks_only_, temp_}; - std::string db_path = PSTRING() << db_root_ << p_id.path() << p_id.name() << ".index"; - delay_action([name = db_path, attempt = 0, - promise = ig.get_promise()]() mutable { destroy_db(name, attempt, std::move(promise)); }, + delay_action([name = db_path_, attempt = 0, + promise = std::move(promise)]() mutable { destroy_db(name, attempt, std::move(promise)); }, td::Timestamp::in(0.0)); } From 5920f800ded13189bedaef79c5b46c09f9e84d37 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 15 Oct 2024 17:43:46 +0300 Subject: [PATCH 2/6] Improve gas consumption in VmState::jump_to (#1276) --- crypto/vm/vm.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crypto/vm/vm.h b/crypto/vm/vm.h index f202e55f..7afa355c 100644 --- a/crypto/vm/vm.h +++ b/crypto/vm/vm.h @@ -373,11 +373,11 @@ class VmState final : public VmStateInterface { int jump_to(Ref cont) { int res = 0, cnt = 0; while (cont.not_null()) { - cnt++; cont = cont->is_unique() ? cont.unique_write().jump_w(this, res) : cont->jump(this, res); - } - if (global_version >= 9 && cnt > free_nested_cont_jump) { - consume_gas(cnt - free_nested_cont_jump); + cnt++; + if (cnt > free_nested_cont_jump && global_version >= 9) { + consume_gas(1); + } } return res; } From 603d1d14b9c7c42c6a8308573ef053bfc0a9ef5b Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 15 Oct 2024 17:59:12 +0300 Subject: [PATCH 3/6] More precise calculation of expected shard blocks in checkload (#1279) * More precise calculation of expected shard blocks in checkload * Fix monte-carlo checkload --- crypto/block/create-state.cpp | 2 +- crypto/block/mc-config.cpp | 9 +- crypto/block/mc-config.h | 8 +- lite-client/lite-client.cpp | 302 ++++++++++++++++++++++++++++------ lite-client/lite-client.h | 20 +++ validator/impl/collator.cpp | 2 +- 6 files changed, 279 insertions(+), 64 deletions(-) diff --git a/crypto/block/create-state.cpp b/crypto/block/create-state.cpp index 348377e9..c8c8b970 100644 --- a/crypto/block/create-state.cpp +++ b/crypto/block/create-state.cpp @@ -426,7 +426,7 @@ bool store_validator_list_hash(vm::CellBuilder& cb) { LOG_CHECK(vset) << "unpacked validator set is empty"; auto ccvc = block::Config::unpack_catchain_validators_config(config_dict.lookup_ref(td::BitArray<32>{28})); ton::ShardIdFull shard{ton::masterchainId}; - auto nodes = block::Config::do_compute_validator_set(ccvc, shard, *vset, now, 0); + auto nodes = block::Config::do_compute_validator_set(ccvc, shard, *vset, 0); LOG_CHECK(!nodes.empty()) << "validator node list in unpacked validator set is empty"; auto vset_hash = block::compute_validator_set_hash(0, shard, std::move(nodes)); LOG(DEBUG) << "initial validator set hash is " << vset_hash; diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index 16b4897a..56ee85ae 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -1746,7 +1746,7 @@ ton::CatchainSeqno ConfigInfo::get_shard_cc_seqno(ton::ShardIdFull shard) const std::vector Config::compute_validator_set(ton::ShardIdFull shard, const block::ValidatorSet& vset, ton::UnixTime time, ton::CatchainSeqno cc_seqno) const { - return do_compute_validator_set(get_catchain_validators_config(), shard, vset, time, cc_seqno); + return do_compute_validator_set(get_catchain_validators_config(), shard, vset, cc_seqno); } std::vector Config::compute_validator_set(ton::ShardIdFull shard, ton::UnixTime time, @@ -1773,7 +1773,7 @@ std::vector ConfigInfo::compute_validator_set_cc(ton::Shard if (cc_seqno_delta) { cc_seqno = *cc_seqno_delta += cc_seqno; } - return do_compute_validator_set(get_catchain_validators_config(), shard, vset, time, cc_seqno); + return do_compute_validator_set(get_catchain_validators_config(), shard, vset, cc_seqno); } std::vector ConfigInfo::compute_validator_set_cc(ton::ShardIdFull shard, ton::UnixTime time, @@ -1856,9 +1856,8 @@ int ValidatorSet::lookup_public_key(td::ConstBitPtr pubkey) const { return -1; } -std::vector Config::do_compute_validator_set(const block::CatchainValidatorsConfig& ccv_conf, - ton::ShardIdFull shard, - const block::ValidatorSet& vset, ton::UnixTime time, +std::vector Config::do_compute_validator_set(const CatchainValidatorsConfig& ccv_conf, + ton::ShardIdFull shard, const ValidatorSet& vset, ton::CatchainSeqno cc_seqno) { // LOG(DEBUG) << "in Config::do_compute_validator_set() for " << shard.to_str() << " ; cc_seqno=" << cc_seqno; std::vector nodes; diff --git a/crypto/block/mc-config.h b/crypto/block/mc-config.h index 2c4de0c6..7cb6613d 100644 --- a/crypto/block/mc-config.h +++ b/crypto/block/mc-config.h @@ -456,10 +456,11 @@ class ShardConfig { ShardConfig() = default; ShardConfig(const ShardConfig& other); ShardConfig(ShardConfig&& other) = default; - ShardConfig(Ref shard_hashes, Ref mc_shard_hash = {}) + explicit ShardConfig(Ref shard_hashes, Ref mc_shard_hash = {}) : shard_hashes_(std::move(shard_hashes)), mc_shard_hash_(std::move(mc_shard_hash)) { init(); } + ShardConfig& operator=(ShardConfig&& other) = default; bool is_valid() const { return valid_; } @@ -659,9 +660,8 @@ class Config { BurningConfig get_burning_config() const; td::Ref get_unpacked_config_tuple(ton::UnixTime now) const; PrecompiledContractsConfig get_precompiled_contracts_config() const; - static std::vector do_compute_validator_set(const block::CatchainValidatorsConfig& ccv_conf, - ton::ShardIdFull shard, - const block::ValidatorSet& vset, ton::UnixTime time, + static std::vector do_compute_validator_set(const CatchainValidatorsConfig& ccv_conf, + ton::ShardIdFull shard, const ValidatorSet& vset, ton::CatchainSeqno cc_seqno); static td::Result> unpack_config(Ref config_root, diff --git a/lite-client/lite-client.cpp b/lite-client/lite-client.cpp index 96891b53..1a4201a7 100644 --- a/lite-client/lite-client.cpp +++ b/lite-client/lite-client.cpp @@ -965,8 +965,8 @@ bool TestNode::show_help(std::string command) { "recentcreatorstats [ []]\tLists block creator statistics " "updated after by validator public " "key\n" - "checkload[all|severe] []\tChecks whether all validators worked " - "properly during specified time " + "checkload[all|severe][-v2] []\tChecks whether all validators " + "worked properly during specified time " "interval, and optionally saves proofs into -.boc\n" "loadproofcheck \tChecks a validator misbehavior proof previously created by checkload\n" "pastvalsets\tLists known past validator set ids and their hashes\n" @@ -1102,8 +1102,15 @@ bool TestNode::do_parse_line() { return parse_block_id_ext(blkid) && (!mode || parse_uint32(utime)) && (seekeoln() ? (mode |= 0x100) : parse_uint32(count)) && (seekeoln() || (parse_hash(hash) && (mode |= 1))) && seekeoln() && get_creator_stats(blkid, mode, count, hash, utime); - } else if (word == "checkload" || word == "checkloadall" || word == "checkloadsevere") { - int time1, time2, mode = (word == "checkloadsevere"); + } else if (word == "checkload" || word == "checkloadall" || word == "checkloadsevere" || word == "checkload-v2" || + word == "checkloadall-v2" || word == "checkloadsevere-v2") { + int time1, time2, mode = 0; + if (word == "checkloadsevere" || word == "checkloadsevere-v2") { + mode |= 1; + } + if (td::ends_with(word, "-v2")) { + mode |= 4; + } std::string file_pfx; return parse_int32(time1) && parse_int32(time2) && (seekeoln() || ((mode |= 2) && get_word_to(file_pfx))) && seekeoln() && check_validator_load(time1, time2, mode, file_pfx); @@ -3706,7 +3713,7 @@ void TestNode::continue_check_validator_load2(std::unique_ptr info2, int mode, std::string file_pfx) { LOG(INFO) << "continue_check_validator_load3 for blocks " << info1->blk_id.to_str() << " and " - << info2->blk_id.to_str() << " with mode=" << mode << " and file prefix `" << file_pfx - << "`: comparing block creators data"; + << info2->blk_id.to_str() << " with mode=" << mode << " and file prefix `" << file_pfx; + + if (mode & 4) { + ton::BlockSeqno start_seqno = info1->blk_id.seqno(); + ton::BlockSeqno end_seqno = info2->blk_id.seqno(); + block::ValidatorSet validator_set = *info1->vset; + if (info1->config->get_config_param(28)->get_hash() != info2->config->get_config_param(28)->get_hash()) { + LOG(ERROR) << "Catchain validator config (28) changed between the first and the last block"; + return; + } + auto catchain_config = std::make_unique( + block::Config::unpack_catchain_validators_config(info1->config->get_config_param(28))); + load_validator_shard_shares( + start_seqno, end_seqno, std::move(validator_set), std::move(catchain_config), + [=, this, info1 = std::move(info1), + info2 = std::move(info2)](td::Result> R) mutable { + if (R.is_error()) { + LOG(ERROR) << "failed to load validator shard shares: " << R.move_as_error(); + } else { + continue_check_validator_load4(std::move(info1), std::move(info2), mode, file_pfx, R.move_as_ok()); + } + }); + } else { + continue_check_validator_load4(std::move(info1), std::move(info2), mode, std::move(file_pfx), {}); + } +} + +void TestNode::continue_check_validator_load4(std::unique_ptr info1, + std::unique_ptr info2, int mode, + std::string file_pfx, + std::map exact_shard_shares) { + LOG(INFO) << "continue_check_validator_load4 for blocks " << info1->blk_id.to_str() << " and " + << info2->blk_id.to_str() << " with mode=" << mode << " and file prefix `" << file_pfx; if (info1->created_total.first <= 0 || info2->created_total.first <= 0) { LOG(ERROR) << "no total created blocks statistics"; return; } td::TerminalIO::out() << "total: (" << info1->created_total.first << "," << info1->created_total.second << ") -> (" << info2->created_total.first << "," << info2->created_total.second << ")\n"; - auto x = info2->created_total.first - info1->created_total.first; - auto y = info2->created_total.second - info1->created_total.second; - td::int64 xs = 0, ys = 0; - if (x <= 0 || y < 0 || (x | y) >= (1u << 31)) { - LOG(ERROR) << "impossible situation: zero or no blocks created: " << x << " masterchain blocks, " << y - << " shardchain blocks"; + auto created_total_mc = info2->created_total.first - info1->created_total.first; + auto created_total_bc = info2->created_total.second - info1->created_total.second; + td::int64 created_mc_sum = 0, created_bc_sum = 0; + if (created_total_mc <= 0 || created_total_bc < 0 || (created_total_mc | created_total_bc) >= (1U << 31)) { + LOG(ERROR) << "impossible situation: zero or no blocks created: " << created_total_mc << " masterchain blocks, " + << created_total_bc << " shardchain blocks"; return; } - std::pair created_total{(int)x, (int)y}; int count = info1->vset->total; CHECK(info2->vset->total == count); CHECK((int)info1->created.size() == count); CHECK((int)info2->created.size() == count); - std::vector> d; - d.reserve(count); + std::vector> vals_created; + vals_created.reserve(count); for (int i = 0; i < count; i++) { - auto x1 = info2->created[i].first - info1->created[i].first; - auto y1 = info2->created[i].second - info1->created[i].second; - if (x1 < 0 || y1 < 0 || (x1 | y1) >= (1u << 31)) { - LOG(ERROR) << "impossible situation: validator #" << i << " created a negative amount of blocks: " << x1 - << " masterchain blocks, " << y1 << " shardchain blocks"; + auto created_mc = info2->created[i].first - info1->created[i].first; + auto created_bc = info2->created[i].second - info1->created[i].second; + if (created_mc < 0 || created_bc < 0 || (created_mc | created_bc) >= (1u << 31)) { + LOG(ERROR) << "impossible situation: validator #" << i << " created a negative amount of blocks: " << created_mc + << " masterchain blocks, " << created_bc << " shardchain blocks"; return; } - xs += x1; - ys += y1; - d.emplace_back((int)x1, (int)y1); - td::TerminalIO::out() << "val #" << i << ": created (" << x1 << "," << y1 << ") ; was (" << info1->created[i].first - << "," << info1->created[i].second << ")\n"; + created_mc_sum += created_mc; + created_bc_sum += created_bc; + vals_created.emplace_back((int)created_mc, (int)created_bc); + td::TerminalIO::out() << "val #" << i << ": created (" << created_mc << "," << created_bc << ") ; was (" + << info1->created[i].first << "," << info1->created[i].second << ")\n"; } - if (xs != x || ys != y) { - LOG(ERROR) << "cannot account for all blocks created: total is (" << x << "," << y - << "), but the sum for all validators is (" << xs << "," << ys << ")"; + if (created_mc_sum != created_total_mc || created_bc_sum != created_total_bc) { + LOG(ERROR) << "cannot account for all blocks created: total is (" << created_total_mc << "," << created_total_bc + << "), but the sum for all validators is (" << created_mc_sum << "," << created_bc_sum << ")"; return; } - td::TerminalIO::out() << "total: (" << x << "," << y << ")\n"; + td::TerminalIO::out() << "total: (" << created_total_mc << "," << created_total_bc << ")\n"; auto ccfg = block::Config::unpack_catchain_validators_config(info2->config->get_config_param(28)); auto ccfg_old = block::Config::unpack_catchain_validators_config(info1->config->get_config_param(28)); if (ccfg.shard_val_num != ccfg_old.shard_val_num || ccfg.shard_val_num <= 0) { @@ -3797,57 +3834,216 @@ void TestNode::continue_check_validator_load3(std::unique_ptrvset->main; - if (info1->vset->main != main_count || main_count <= 0) { - LOG(ERROR) << "masterchain validator group size changed from " << info1->vset->main << " to " << main_count + int shard_vals = ccfg.shard_val_num, master_vals = info2->vset->main; + if (info1->vset->main != master_vals || master_vals <= 0) { + LOG(ERROR) << "masterchain validator group size changed from " << info1->vset->main << " to " << master_vals << ", or is not positive"; return; } - int cnt = 0, cnt_ok = 0; - double chunk_size = ccfg.shard_val_lifetime / 3. / shard_count; - block::MtCarloComputeShare shard_share(shard_count, info2->vset->export_scaled_validator_weights()); + + bool use_exact_shard_share = mode & 4; + int proofs_cnt = 0, proofs_cnt_ok = 0; + double chunk_size = ccfg.shard_val_lifetime / 3. / shard_vals; + + std::vector mtc_shard_share; + if (use_exact_shard_share) { + LOG(INFO) << "using exact shard shares"; + td::uint64 exact_shard_shares_sum = 0; + for (auto& [_, count] : exact_shard_shares) { + exact_shard_shares_sum += count; + } + if ((td::int64)exact_shard_shares_sum != shard_vals * created_bc_sum) { + LOG(ERROR) << "unexpected total shard shares: blocks=" << created_bc_sum << ", shard_vals=" << shard_vals + << ", expected_sum=" << shard_vals * created_bc_sum << ", found=" << exact_shard_shares_sum; + return; + } + } else { + LOG(INFO) << "using MtCarloComputeShare"; + block::MtCarloComputeShare mtc(shard_vals, info2->vset->export_scaled_validator_weights()); + if (!mtc.is_ok()) { + LOG(ERROR) << "failed to compute shard shares"; + return; + } + mtc_shard_share.resize(count); + for (size_t i = 0; i < count; ++i) { + mtc_shard_share[i] = mtc[i]; + } + } + + auto validators = info1->vset->export_validator_set(); for (int i = 0; i < count; i++) { - int x1 = d[i].first, y1 = d[i].second; - bool is_masterchain_validator = i < main_count; - double xe = (is_masterchain_validator ? (double)xs / main_count : 0); - double ye = shard_share[i] * (double)ys / shard_count; + int created_mc = vals_created[i].first, created_bc = vals_created[i].second; + bool is_masterchain_validator = i < master_vals; + + double expected_created_mc = (is_masterchain_validator ? (double)created_mc_sum / master_vals : 0); + double prob_mc = create_prob(created_mc, .9 * expected_created_mc); + + double expected_created_bc, prob_bc; + if (use_exact_shard_share) { + expected_created_bc = (double)exact_shard_shares[validators[i].key.as_bits256()] / shard_vals; + prob_bc = create_prob(created_bc, .9 * expected_created_bc); + } else { + expected_created_bc = mtc_shard_share[i] * (double)created_bc_sum / shard_vals; + prob_bc = shard_create_prob(created_bc, .9 * expected_created_bc, chunk_size); + } + td::Bits256 pk = info2->vset->list[i].pubkey.as_bits256(); - double p1 = create_prob(x1, .9 * xe), p2 = shard_create_prob(y1, .9 * ye, chunk_size); - td::TerminalIO::out() << "val #" << i << ": pubkey " << pk.to_hex() << ", blocks created (" << x1 << "," << y1 - << "), expected (" << xe << "," << ye << "), probabilities " << p1 << " and " << p2 << "\n"; - if ((is_masterchain_validator ? p1 : p2) < .00001) { + td::TerminalIO::out() << "val #" << i << ": pubkey " << pk.to_hex() << ", blocks created (" << created_mc << "," + << created_bc << "), expected (" << expected_created_mc << "," << expected_created_bc + << "), probabilities " << prob_mc << " and " << prob_bc << "\n"; + if ((is_masterchain_validator ? prob_mc : prob_bc) < .00001) { LOG(ERROR) << "validator #" << i << " with pubkey " << pk.to_hex() << " : serious misbehavior detected: created less than 90% of the expected amount of blocks with " "probability 99.999% : created (" - << x1 << "," << y1 << "), expected (" << xe << "," << ye << ") masterchain/shardchain blocks\n"; + << created_mc << "," << created_bc << "), expected (" << expected_created_mc << "," + << expected_created_bc << ") masterchain/shardchain blocks\n"; if (mode & 2) { - auto st = write_val_create_proof(*info1, *info2, i, true, file_pfx, ++cnt); + auto st = write_val_create_proof(*info1, *info2, i, true, file_pfx, ++proofs_cnt); if (st.is_error()) { LOG(ERROR) << "cannot create proof: " << st.move_as_error(); } else { - cnt_ok++; + proofs_cnt_ok++; } } - } else if ((is_masterchain_validator ? p1 : p2) < .005) { + } else if ((is_masterchain_validator ? prob_mc : prob_bc) < .005) { LOG(ERROR) << "validator #" << i << " with pubkey " << pk.to_hex() << " : moderate misbehavior detected: created less than 90% of the expected amount of blocks with " "probability 99.5% : created (" - << x1 << "," << y1 << "), expected (" << xe << "," << ye << ") masterchain/shardchain blocks\n"; + << created_mc << "," << created_bc << "), expected (" << expected_created_mc << "," + << expected_created_bc << ") masterchain/shardchain blocks\n"; if ((mode & 3) == 2) { - auto st = write_val_create_proof(*info1, *info2, i, false, file_pfx, ++cnt); + auto st = write_val_create_proof(*info1, *info2, i, false, file_pfx, ++proofs_cnt); if (st.is_error()) { LOG(ERROR) << "cannot create proof: " << st.move_as_error(); } else { - cnt_ok++; + proofs_cnt_ok++; } } } } - if (cnt > 0) { - LOG(INFO) << cnt_ok << " out of " << cnt << " proofs written to " << file_pfx << "-*.boc"; + if (proofs_cnt > 0) { + LOG(INFO) << proofs_cnt_ok << " out of " << proofs_cnt << " proofs written to " << file_pfx << "-*.boc"; } } +void TestNode::load_validator_shard_shares(ton::BlockSeqno start_seqno, ton::BlockSeqno end_seqno, + block::ValidatorSet validator_set, + std::unique_ptr catchain_config, + td::Promise> promise) { + CHECK(start_seqno <= end_seqno); + LOG(INFO) << "loading shard shares from mc blocks " << start_seqno << ".." << end_seqno << " (" + << end_seqno - start_seqno + 1 << " blocks)"; + auto state = std::make_shared(); + state->start_seqno = start_seqno; + state->end_seqno = end_seqno; + state->validator_set = std::move(validator_set); + state->catchain_config = std::move(catchain_config); + state->shard_configs.resize(end_seqno - start_seqno + 1); + state->promise = std::move(promise); + load_validator_shard_shares_cont(std::move(state)); +} + +void TestNode::load_validator_shard_shares_cont(std::shared_ptr state) { + if (!state->promise) { + return; + } + if (state->loaded % 100 == 0) { + LOG(INFO) << "loaded " << state->loaded << "/" << state->shard_configs.size() << " mc blocks"; + } + while (state->cur_idx < state->shard_configs.size() && state->pending < 8) { + load_block_shard_configuration(state->start_seqno + state->cur_idx, + [this, state, idx = state->cur_idx](td::Result R) mutable { + if (R.is_error()) { + state->promise.set_error(R.move_as_error()); + state->promise = {}; + } else { + state->shard_configs[idx] = R.move_as_ok(); + --state->pending; + ++state->loaded; + load_validator_shard_shares_cont(std::move(state)); + } + }); + ++state->pending; + ++state->cur_idx; + } + + if (state->loaded != state->shard_configs.size()) { + return; + } + LOG(INFO) << "loaded all " << state->shard_configs.size() << " mc blocks, computing shard shares"; + std::map result; + try { + for (size_t idx = 0; idx + 1 < state->shard_configs.size(); ++idx) { + block::ShardConfig& shards1 = state->shard_configs[idx]; + block::ShardConfig& shards2 = state->shard_configs[idx + 1]; + + // Compute validator groups, see ValidatorManagerImpl::update_shards + auto process_shard = [&](ton::ShardIdFull shard, ton::BlockSeqno first_seqno) { + auto desc2 = shards2.get_shard_hash(shard); + if (desc2.is_null() || desc2->seqno() < first_seqno) { + return; + } + td::uint32 blocks_count = desc2->seqno() - first_seqno + 1; + ton::CatchainSeqno cc_seqno = shards1.get_shard_cc_seqno(shard); + auto val_set = + block::ConfigInfo::do_compute_validator_set(*state->catchain_config, shard, state->validator_set, cc_seqno); + for (const auto &val : val_set) { + result[val.key.as_bits256()] += blocks_count; + } + }; + + for (const ton::BlockId& id : shards1.get_shard_hash_ids()) { + ton::ShardIdFull shard = id.shard_full(); + auto desc = shards1.get_shard_hash(shard); + CHECK(desc.not_null()); + if (desc->before_split()) { + ton::ShardIdFull l_shard = shard_child(shard, true); + ton::ShardIdFull r_shard = shard_child(shard, false); + process_shard(l_shard, desc->seqno() + 1); + process_shard(r_shard, desc->seqno() + 1); + } else if (desc->before_merge()) { + if (is_right_child(shard)) { + continue; + } + ton::ShardIdFull sibling_shard = shard_sibling(shard); + auto sibling_desc = shards1.get_shard_hash(sibling_shard); + CHECK(sibling_desc.not_null()); + ton::ShardIdFull p_shard = shard_parent(shard); + process_shard(p_shard, std::max(desc->seqno(), sibling_desc->seqno()) + 1); + } else { + process_shard(shard, desc->seqno() + 1); + } + } + } + } catch (vm::VmError &e) { + state->promise.set_error(e.as_status("cannot parse shard hashes: ")); + return; + } + state->promise.set_value(std::move(result)); +} + +void TestNode::load_block_shard_configuration(ton::BlockSeqno seqno, td::Promise promise) { + lookup_block( + ton::ShardIdFull{ton::masterchainId}, 1, seqno, + [this, promise = std::move(promise)](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, res, std::move(R)); + auto b = ton::serialize_tl_object( + ton::create_tl_object(ton::create_tl_lite_block_id(res.blk_id)), + true); + envelope_send_query(std::move(b), [this, promise = std::move(promise)](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, data, std::move(R)); + TRY_RESULT_PROMISE(promise, f, ton::fetch_tl_object(data, true)); + TRY_RESULT_PROMISE(promise, root, vm::std_boc_deserialize(f->data_)); + block::ShardConfig sh_conf; + if (!sh_conf.unpack(load_cell_slice_ref(root))) { + promise.set_error(td::Status::Error("cannot extract shard block list from shard configuration")); + } else { + promise.set_value(std::move(sh_conf)); + } + }); + }); +} + bool compute_punishment_default(int interval, bool severe, td::RefInt256& fine, unsigned& fine_part) { if (interval <= 1000) { return false; // no punishments for less than 1000 seconds diff --git a/lite-client/lite-client.h b/lite-client/lite-client.h index 6fb7f9ab..90a2fb8a 100644 --- a/lite-client/lite-client.h +++ b/lite-client/lite-client.h @@ -282,6 +282,26 @@ class TestNode : public td::actor::Actor { void continue_check_validator_load3(std::unique_ptr info1, std::unique_ptr info2, int mode = 0, std::string file_pfx = ""); + void continue_check_validator_load4(std::unique_ptr info1, + std::unique_ptr info2, int mode, std::string file_pfx, + std::map exact_shard_shares); + + struct LoadValidatorShardSharesState { + ton::BlockSeqno start_seqno; + ton::BlockSeqno end_seqno; + block::ValidatorSet validator_set; + std::unique_ptr catchain_config; + std::vector shard_configs; + td::uint32 cur_idx = 0, pending = 0, loaded = 0; + td::Promise> promise; + }; + void load_validator_shard_shares(ton::BlockSeqno start_seqno, ton::BlockSeqno end_seqno, + block::ValidatorSet validator_set, + std::unique_ptr catchain_config, + td::Promise> promise); + void load_validator_shard_shares_cont(std::shared_ptr state); + void load_block_shard_configuration(ton::BlockSeqno seqno, td::Promise promise); + td::Status write_val_create_proof(ValidatorLoadInfo& info1, ValidatorLoadInfo& info2, int idx, bool severe, std::string file_pfx, int cnt); bool load_creator_stats(std::unique_ptr load_to, diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 0708fb52..6b5d7614 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -4516,7 +4516,7 @@ bool Collator::create_mc_state_extra() { cc_updated = true; LOG(INFO) << "increased masterchain catchain seqno to " << val_info.catchain_seqno; } - auto nodes = block::Config::do_compute_validator_set(ccvc, shard_, *cur_validators, now_, val_info.catchain_seqno); + auto nodes = block::Config::do_compute_validator_set(ccvc, shard_, *cur_validators, val_info.catchain_seqno); LOG_CHECK(!nodes.empty()) << "validator node list in unpacked validator set is empty"; auto vlist_hash = block::compute_validator_set_hash(/* val_info.catchain_seqno */ 0, shard_, std::move(nodes)); From 96ddbcd674655917faa0d93baf88a7f6504d98c9 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Sat, 19 Oct 2024 12:33:11 +0300 Subject: [PATCH 4/6] Update changelog --- Changelog.md | 16 ++++++++++++++++ recent_changelog.md | 25 +++++++++++++------------ 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/Changelog.md b/Changelog.md index e5df2f97..86d86f59 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,19 @@ +## 2024.10 Update + +1. Parallel write to celldb: substantial improvement of sync and GC speed, especially with slow disks. +2. Decreased network traffic: only first block candidate is sent optimistically. +3. Improved channel creation and dht lookups, introduction of semi-private overlays +4. New LS dispatch queue related methods and improvement security +5. Fixing recursion in TVM continuations +6. Improved stats for actors, validator sessions, perf counters, overlays, adnl, rocksdb +7. Migration to C++20 +8. Improved block size estimates: account for depth in various structures +9. Fix bug with `<<` optimization in FunC +10. Minor changes of TVM which will be activated by `Config8.version >= 9` +11. Multiple minor improvements + +Besides the work of the core team, this update is based on the efforts of @krigga (emulator), Arayz @ TonBit (LS security, TVM recursion). + ## 2024.08 Update 1. Introduction of dispatch queues, message envelopes with transaction chain metadata, and explicitly stored msg_queue size, which will be activated by `Config8.version >= 8` and new `Config8.capabilities` bits: `capStoreOutMsgQueueSize`, `capMsgMetadata`, `capDeferMessages`. diff --git a/recent_changelog.md b/recent_changelog.md index 930fa9c9..464e2b94 100644 --- a/recent_changelog.md +++ b/recent_changelog.md @@ -1,16 +1,17 @@ -## 2024.08 Update +## 2024.10 Update -1. Introduction of dispatch queues, message envelopes with transaction chain metadata, and explicitly stored msg_queue size, which will be activated by `Config8.version >= 8` and new `Config8.capabilities` bits: `capStoreOutMsgQueueSize`, `capMsgMetadata`, `capDeferMessages`. -2. A number of changes to transcation executor which will activated for `Config8.version >= 8`: - - Check mode on invalid `action_send_msg`. Ignore action if `IGNORE_ERROR` (+2) bit is set, bounce if `BOUNCE_ON_FAIL` (+16) bit is set. - - Slightly change random seed generation to fix mix of `addr_rewrite` and `addr`. - - Fill in `skipped_actions` for both invalid and valid messages with `IGNORE_ERROR` mode that can't be sent. - - Allow unfreeze through external messages. - - Don't use user-provided `fwd_fee` and `ihr_fee` for internal messages. -3. A few issues with broadcasts were fixed: stop on receiving last piece, response to AdnlMessageCreateChannel -4. A number of fixes and improvements for emulator and tonlib: correct work with config_addr, not accepted externals, bounces, debug ops gas consumption, added version and c5 dump, fixed tonlib crashes -5. Added new flags and commands to the node, in particular `--fast-state-serializer`, `getcollatoroptionsjson`, `setcollatoroptionsjson` +1. Parallel write to celldb: substantial improvement of sync and GC speed, especially with slow disks. +2. Decreased network traffic: only first block candidate is sent optimistically. +3. Improved channel creation and dht lookups, introduction of semi-private overlays +4. New LS dispatch queue related methods and improvement security +5. Fixing recursion in TVM continuations +6. Improved stats for actors, validator sessions, perf counters, overlays, adnl, rocksdb +7. Migration to C++20 +8. Improved block size estimates: account for depth in various structures +9. Fix bug with `<<` optimization in FunC +10. Minor changes of TVM which will be activated by `Config8.version >= 9` +11. Multiple minor improvements -Besides the work of the core team, this update is based on the efforts of @krigga (emulator), stonfi team, in particular @dbaranovstonfi and @hey-researcher (emulator), and @loeul, @xiaoxianBoy, @simlecode (typos in comments and docs). +Besides the work of the core team, this update is based on the efforts of @krigga (emulator), Arayz @ TonBit (LS security, TVM recursion). From c1ee18c90293384366ed3dbee376cb814b463a2f Mon Sep 17 00:00:00 2001 From: "Victor S." <53380262+1IxI1@users.noreply.github.com> Date: Sat, 19 Oct 2024 12:34:08 +0300 Subject: [PATCH 5/6] Add state-init option to wallet-v3.fif script (#1282) --- crypto/smartcont/wallet-v3.fif | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crypto/smartcont/wallet-v3.fif b/crypto/smartcont/wallet-v3.fif index c090dc09..f44e0c92 100644 --- a/crypto/smartcont/wallet-v3.fif +++ b/crypto/smartcont/wallet-v3.fif @@ -13,7 +13,7 @@ variable extra-currencies { extra-currencies @ cc+ extra-currencies ! } : extra-cc+! begin-options - " [-x *] [-n|-b] [-t] [-B ] [-C ] []" +cr +tab + " [-x *] [-n|-b] [-t] [-B ] [-C ] [-I ] []" +cr +tab +"Creates a request to advanced wallet created by new-wallet-v3.fif, with private key loaded from file .pk " +"and address from .addr, and saves it into .boc ('wallet-query.boc' by default)" disable-digit-options generic-help-setopt @@ -29,6 +29,8 @@ begin-options "Sets the payload of the transfer message" option-help "C" "--comment" { =: comment } short-long-option-arg "Sets the comment to be sent in the transfer message" option-help + "I" "--with-init" { =: init-file } short-long-option-arg + "Indicates filename with BoC containing StateInit for internal message" option-help "m" "--mode" { parse-int =: send-mode } short-long-option-arg "Sets transfer mode (0..255) for SENDRAWMSG (" send-mode (.) $+ +" by default)" option-help @@ -57,14 +59,18 @@ file-base +".pk" load-keypair nip constant wallet_pk def? body-boc-file { @' body-boc-file file>B B>boc } { comment simple-transfer-body } cond constant body-cell +def? init-file { @' init-file file>B B>boc From 8daf5c335a12d5a0d045b98b94be14050e02301a Mon Sep 17 00:00:00 2001 From: neodix42 Date: Sat, 19 Oct 2024 14:03:10 +0400 Subject: [PATCH 6/6] Improve artifacts' stripping (#1286) * add ton build on mac-15 gh action * rename action titles * fix https://github.com/ton-blockchain/ton/issues/1246 * improve artifacts' stripping * improve artifacts' stripping * use strip -xSX on mac on github runner * use strip -xSX on mac on github runner + sudo --- .../build-ton-macos-15-arm64-shared.yml | 25 ++++++++++++++ .../build-ton-macos-arm64-shared.yml | 2 +- CMake/FindSodium.cmake | 8 ++++- assembly/native/build-macos-portable.sh | 34 +++++++++---------- assembly/native/build-macos-shared.sh | 34 +++++++++---------- assembly/native/build-ubuntu-portable.sh | 2 +- assembly/native/build-ubuntu-shared.sh | 2 +- assembly/native/build-windows-2019.bat | 2 +- assembly/native/build-windows.bat | 2 +- assembly/nix/build-linux-arm64-nix.sh | 19 +++++++++++ assembly/nix/build-linux-x86-64-nix.sh | 19 +++++++++++ assembly/nix/build-macos-nix.sh | 19 +++++++++++ 12 files changed, 128 insertions(+), 40 deletions(-) create mode 100644 .github/workflows/build-ton-macos-15-arm64-shared.yml diff --git a/.github/workflows/build-ton-macos-15-arm64-shared.yml b/.github/workflows/build-ton-macos-15-arm64-shared.yml new file mode 100644 index 00000000..8379a0b0 --- /dev/null +++ b/.github/workflows/build-ton-macos-15-arm64-shared.yml @@ -0,0 +1,25 @@ +name: MacOS-15 TON build (shared, arm64) + +on: [push,workflow_dispatch,workflow_call] + +jobs: + build: + runs-on: macos-15 + + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - name: Build TON + run: | + cp assembly/native/build-macos-shared.sh . + chmod +x build-macos-shared.sh + ./build-macos-shared.sh -t -a + + - name: Upload artifacts + uses: actions/upload-artifact@master + with: + name: ton-binaries-macos-15 + path: artifacts diff --git a/.github/workflows/build-ton-macos-arm64-shared.yml b/.github/workflows/build-ton-macos-arm64-shared.yml index 2a68272c..aadd23dd 100644 --- a/.github/workflows/build-ton-macos-arm64-shared.yml +++ b/.github/workflows/build-ton-macos-arm64-shared.yml @@ -1,4 +1,4 @@ -name: MacOS TON build (shared, arm64) +name: MacOS-14 TON build (shared, arm64) on: [push,workflow_dispatch,workflow_call] diff --git a/CMake/FindSodium.cmake b/CMake/FindSodium.cmake index c11e46f1..3818ef89 100644 --- a/CMake/FindSodium.cmake +++ b/CMake/FindSodium.cmake @@ -127,10 +127,16 @@ elseif (WIN32) endif() string(APPEND _PLATFORM_PATH "/$$CONFIG$$") + message(STATUS "MSVC_VERSION ${MSVC_VERSION}") if (MSVC_VERSION LESS 1900) math(EXPR _VS_VERSION "${MSVC_VERSION} / 10 - 60") else() - math(EXPR _VS_VERSION "${MSVC_VERSION} / 10 - 50") + if (MSVC_VERSION EQUAL 1941) + math(EXPR _VS_VERSION "${MSVC_VERSION} / 10 - 51") + else() + math(EXPR _VS_VERSION "${MSVC_VERSION} / 10 - 50") + endif() + endif() string(APPEND _PLATFORM_PATH "/v${_VS_VERSION}") diff --git a/assembly/native/build-macos-portable.sh b/assembly/native/build-macos-portable.sh index 471f29af..fa4702ab 100644 --- a/assembly/native/build-macos-portable.sh +++ b/assembly/native/build-macos-portable.sh @@ -168,23 +168,23 @@ else test $? -eq 0 || { echo "Can't compile ton"; exit 1; } fi -strip storage/storage-daemon/storage-daemon -strip storage/storage-daemon/storage-daemon-cli -strip blockchain-explorer/blockchain-explorer -strip crypto/fift -strip crypto/func -strip crypto/create-state -strip crypto/tlbc -strip validator-engine-console/validator-engine-console -strip tonlib/tonlib-cli -strip http/http-proxy -strip rldp-http-proxy/rldp-http-proxy -strip dht-server/dht-server -strip lite-client/lite-client -strip validator-engine/validator-engine -strip utils/generate-random-id -strip utils/json2tlo -strip adnl/adnl-proxy +strip -s storage/storage-daemon/storage-daemon +strip -s storage/storage-daemon/storage-daemon-cli +strip -s blockchain-explorer/blockchain-explorer +strip -s crypto/fift +strip -s crypto/func +strip -s crypto/create-state +strip -s crypto/tlbc +strip -s validator-engine-console/validator-engine-console +strip -s tonlib/tonlib-cli +strip -s http/http-proxy +strip -s rldp-http-proxy/rldp-http-proxy +strip -s dht-server/dht-server +strip -s lite-client/lite-client +strip -s validator-engine/validator-engine +strip -s utils/generate-random-id +strip -s utils/json2tlo +strip -s adnl/adnl-proxy cd .. diff --git a/assembly/native/build-macos-shared.sh b/assembly/native/build-macos-shared.sh index 20cb70cf..7fdcfb94 100644 --- a/assembly/native/build-macos-shared.sh +++ b/assembly/native/build-macos-shared.sh @@ -97,23 +97,23 @@ else fi -strip storage/storage-daemon/storage-daemon -strip storage/storage-daemon/storage-daemon-cli -strip blockchain-explorer/blockchain-explorer -strip crypto/fift -strip crypto/func -strip crypto/create-state -strip crypto/tlbc -strip validator-engine-console/validator-engine-console -strip tonlib/tonlib-cli -strip http/http-proxy -strip rldp-http-proxy/rldp-http-proxy -strip dht-server/dht-server -strip lite-client/lite-client -strip validator-engine/validator-engine -strip utils/generate-random-id -strip utils/json2tlo -strip adnl/adnl-proxy +strip -s storage/storage-daemon/storage-daemon +strip -s storage/storage-daemon/storage-daemon-cli +strip -s blockchain-explorer/blockchain-explorer +strip -s crypto/fift +strip -s crypto/func +strip -s crypto/create-state +strip -s crypto/tlbc +strip -s validator-engine-console/validator-engine-console +strip -s tonlib/tonlib-cli +strip -s http/http-proxy +strip -s rldp-http-proxy/rldp-http-proxy +strip -s dht-server/dht-server +strip -s lite-client/lite-client +strip -s validator-engine/validator-engine +strip -s utils/generate-random-id +strip -s utils/json2tlo +strip -s adnl/adnl-proxy cd .. diff --git a/assembly/native/build-ubuntu-portable.sh b/assembly/native/build-ubuntu-portable.sh index 39b49a38..73ae5926 100644 --- a/assembly/native/build-ubuntu-portable.sh +++ b/assembly/native/build-ubuntu-portable.sh @@ -160,7 +160,7 @@ ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli \ test $? -eq 0 || { echo "Can't compile ton"; exit 1; } fi -strip -g storage/storage-daemon/storage-daemon \ +strip -s storage/storage-daemon/storage-daemon \ storage/storage-daemon/storage-daemon-cli \ blockchain-explorer/blockchain-explorer \ crypto/fift \ diff --git a/assembly/native/build-ubuntu-shared.sh b/assembly/native/build-ubuntu-shared.sh index a62698d2..00b9aa9b 100644 --- a/assembly/native/build-ubuntu-shared.sh +++ b/assembly/native/build-ubuntu-shared.sh @@ -68,7 +68,7 @@ ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli \ test $? -eq 0 || { echo "Can't compile ton"; exit 1; } fi -strip -g storage/storage-daemon/storage-daemon \ +strip -s storage/storage-daemon/storage-daemon \ storage/storage-daemon/storage-daemon-cli \ blockchain-explorer/blockchain-explorer \ crypto/fift \ diff --git a/assembly/native/build-windows-2019.bat b/assembly/native/build-windows-2019.bat index b528b05a..f728b88f 100644 --- a/assembly/native/build-windows-2019.bat +++ b/assembly/native/build-windows-2019.bat @@ -216,6 +216,6 @@ build\validator-engine\validator-engine.exe ^ build\utils\generate-random-id.exe ^ build\utils\json2tlo.exe ^ build\adnl\adnl-proxy.exe ^ -build\emulator\emulator.dll) do (strip -g %%I & copy %%I artifacts\) +build\emulator\emulator.dll) do (strip -s %%I & copy %%I artifacts\) xcopy /e /k /h /i crypto\smartcont artifacts\smartcont xcopy /e /k /h /i crypto\fift\lib artifacts\lib diff --git a/assembly/native/build-windows.bat b/assembly/native/build-windows.bat index a9871aa9..aa0fd69a 100644 --- a/assembly/native/build-windows.bat +++ b/assembly/native/build-windows.bat @@ -217,6 +217,6 @@ build\validator-engine\validator-engine.exe ^ build\utils\generate-random-id.exe ^ build\utils\json2tlo.exe ^ build\adnl\adnl-proxy.exe ^ -build\emulator\emulator.dll) do (strip -g %%I & copy %%I artifacts\) +build\emulator\emulator.dll) do (strip -s %%I & copy %%I artifacts\) xcopy /e /k /h /i crypto\smartcont artifacts\smartcont xcopy /e /k /h /i crypto\fift\lib artifacts\lib diff --git a/assembly/nix/build-linux-arm64-nix.sh b/assembly/nix/build-linux-arm64-nix.sh index 6fc2fab2..2c7df521 100644 --- a/assembly/nix/build-linux-arm64-nix.sh +++ b/assembly/nix/build-linux-arm64-nix.sh @@ -36,3 +36,22 @@ cp ./result/lib/libemulator.so artifacts/ cp ./result/lib/fift/* artifacts/lib/ cp -r ./result/share/ton/smartcont artifacts/ chmod -R +x artifacts +cd artifacts +sudo strip -s storage-daemon \ + storage-daemon-cli \ + blockchain-explorer \ + fift \ + tlbc \ + func \ + create-state \ + validator-engine-console \ + tonlib-cli \ + http-proxy \ + rldp-http-proxy \ + dht-server \ + lite-client \ + validator-engine \ + generate-random-id \ + adnl-proxy \ + libemulator.so \ + libtonlibjson.so diff --git a/assembly/nix/build-linux-x86-64-nix.sh b/assembly/nix/build-linux-x86-64-nix.sh index 30ab79f7..ae478ec2 100644 --- a/assembly/nix/build-linux-x86-64-nix.sh +++ b/assembly/nix/build-linux-x86-64-nix.sh @@ -36,3 +36,22 @@ cp ./result/lib/libemulator.so artifacts/ cp ./result/lib/fift/* artifacts/lib/ cp -r ./result/share/ton/smartcont artifacts/ chmod -R +x artifacts +cd artifacts +sudo strip -s storage-daemon \ + storage-daemon-cli \ + blockchain-explorer \ + fift \ + tlbc \ + func \ + create-state \ + validator-engine-console \ + tonlib-cli \ + http-proxy \ + rldp-http-proxy \ + dht-server \ + lite-client \ + validator-engine \ + generate-random-id \ + adnl-proxy \ + libemulator.so \ + libtonlibjson.so diff --git a/assembly/nix/build-macos-nix.sh b/assembly/nix/build-macos-nix.sh index 8a07bea2..c92eddb2 100644 --- a/assembly/nix/build-macos-nix.sh +++ b/assembly/nix/build-macos-nix.sh @@ -36,3 +36,22 @@ cp ./result/lib/libemulator.dylib artifacts/ cp ./result/lib/fift/* artifacts/lib/ cp -r ./result/share/ton/smartcont artifacts/ chmod -R +x artifacts +cd artifacts +sudo strip -xSX storage-daemon \ + storage-daemon-cli \ + blockchain-explorer \ + fift \ + tlbc \ + func \ + create-state \ + validator-engine-console \ + tonlib-cli \ + http-proxy \ + rldp-http-proxy \ + dht-server \ + lite-client \ + validator-engine \ + generate-random-id \ + adnl-proxy \ + libemulator.dylib \ + libtonlibjson.dylib