1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-03-09 15:40:10 +00:00

updated submodules, bugfixes

- added new fift/func code for validator complaint creation
- bugfixes in validator
- updates in tonlib
- new versions of rocksdb/abseil
- hardfork support
This commit is contained in:
ton 2020-04-27 16:01:46 +04:00
parent 16a4566091
commit 9f008b129f
129 changed files with 8438 additions and 879 deletions

View file

@ -241,6 +241,14 @@ bool MsgAddressInt::extract_std_address(vm::CellSlice& cs, ton::WorkchainId& wor
return false;
}
bool MsgAddressInt::extract_std_address(Ref<vm::CellSlice> cs_ref, block::StdAddress& addr, bool rewrite) const {
return extract_std_address(std::move(cs_ref), addr.workchain, addr.addr, rewrite);
}
bool MsgAddressInt::extract_std_address(vm::CellSlice& cs, block::StdAddress& addr, bool rewrite) const {
return extract_std_address(cs, addr.workchain, addr.addr, rewrite);
}
bool MsgAddressInt::store_std_address(vm::CellBuilder& cb, ton::WorkchainId workchain,
const ton::StdSmcAddress& addr) const {
if (workchain >= -128 && workchain < 128) {
@ -263,6 +271,14 @@ Ref<vm::CellSlice> MsgAddressInt::pack_std_address(ton::WorkchainId workchain, c
}
}
bool MsgAddressInt::store_std_address(vm::CellBuilder& cb, const block::StdAddress& addr) const {
return store_std_address(cb, addr.workchain, addr.addr);
}
Ref<vm::CellSlice> MsgAddressInt::pack_std_address(const block::StdAddress& addr) const {
return pack_std_address(addr.workchain, addr.addr);
}
const MsgAddressInt t_MsgAddressInt;
bool MsgAddress::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const {

View file

@ -298,8 +298,13 @@ struct MsgAddressInt final : TLB_Complex {
bool rewrite = true) const;
bool extract_std_address(vm::CellSlice& cs, ton::WorkchainId& workchain, ton::StdSmcAddress& addr,
bool rewrite = true) const;
bool extract_std_address(Ref<vm::CellSlice> cs_ref, block::StdAddress& addr, bool rewrite = true) const;
bool extract_std_address(vm::CellSlice& cs, block::StdAddress& addr, bool rewrite = true) const;
bool store_std_address(vm::CellBuilder& cb, ton::WorkchainId workchain, const ton::StdSmcAddress& addr) const;
Ref<vm::CellSlice> pack_std_address(ton::WorkchainId workchain, const ton::StdSmcAddress& addr) const;
bool store_std_address(vm::CellBuilder& cb, const block::StdAddress& addr) const;
Ref<vm::CellSlice> pack_std_address(const block::StdAddress& addr) const;
};
extern const MsgAddressInt t_MsgAddressInt;

View file

@ -27,6 +27,7 @@
#include "td/utils/crypto.h"
#include "td/utils/tl_storers.h"
#include "td/utils/misc.h"
#include "td/utils/Random.h"
namespace block {
using namespace std::literals::string_literals;
@ -369,14 +370,14 @@ std::unique_ptr<MsgProcessedUptoCollection> MsgProcessedUptoCollection::unpack(t
return v && v->valid ? std::move(v) : std::unique_ptr<MsgProcessedUptoCollection>{};
}
bool MsgProcessedUpto::contains(const MsgProcessedUpto& other) const & {
bool MsgProcessedUpto::contains(const MsgProcessedUpto& other) const& {
return ton::shard_is_ancestor(shard, other.shard) && mc_seqno >= other.mc_seqno &&
(last_inmsg_lt > other.last_inmsg_lt ||
(last_inmsg_lt == other.last_inmsg_lt && !(last_inmsg_hash < other.last_inmsg_hash)));
}
bool MsgProcessedUpto::contains(ton::ShardId other_shard, ton::LogicalTime other_lt, td::ConstBitPtr other_hash,
ton::BlockSeqno other_mc_seqno) const & {
ton::BlockSeqno other_mc_seqno) const& {
return ton::shard_is_ancestor(shard, other_shard) && mc_seqno >= other_mc_seqno &&
(last_inmsg_lt > other_lt || (last_inmsg_lt == other_lt && !(last_inmsg_hash < other_hash)));
}
@ -1546,6 +1547,89 @@ bool unpack_CreatorStats(Ref<vm::CellSlice> cs, DiscountedCounter& mc_cnt, Disco
}
}
/*
*
* Monte Carlo simulator for computing the share of shardchain blocks generated by each validator
*
*/
bool MtCarloComputeShare::compute() {
ok = false;
if (W.size() >= (1U << 31) || W.empty()) {
return false;
}
K = std::min(K, N);
if (K <= 0 || iterations <= 0) {
return false;
}
double tot_weight = 0., acc = 0.;
for (int i = 0; i < N; i++) {
if (W[i] <= 0.) {
return false;
}
tot_weight += W[i];
}
CW.resize(N);
RW.resize(N);
for (int i = 0; i < N; i++) {
CW[i] = acc;
acc += W[i] /= tot_weight;
RW[i] = 0.;
}
R0 = 0.;
H.resize(N);
A.resize(K);
for (long long it = 0; it < iterations; ++it) {
gen_vset();
}
for (int i = 0; i < N; i++) {
RW[i] = W[i] * (RW[i] + R0) / (double)iterations;
}
return ok = true;
}
void MtCarloComputeShare::gen_vset() {
double total_wt = 1.;
int hc = 0;
for (int i = 0; i < K; i++) {
CHECK(total_wt > 0);
double inv_wt = 1. / total_wt;
R0 += inv_wt;
for (int j = 0; j < i; j++) {
RW[A[j]] -= inv_wt;
}
// double p = drand48() * total_wt;
double p = (double)td::Random::fast_uint64() * total_wt / (1. * (1LL << 32) * (1LL << 32));
for (int h = 0; h < hc; h++) {
if (p < H[h].first) {
break;
}
p += H[h].second;
}
int a = -1, b = N, c;
while (b - a > 1) {
c = ((a + b) >> 1);
if (CW[c] <= p) {
a = c;
} else {
b = c;
}
}
CHECK(a >= 0 && a < N);
CHECK(total_wt >= W[a]);
total_wt -= W[a];
double x = CW[a];
c = hc++;
while (c > 0 && H[c - 1].first > x) {
H[c] = H[c - 1];
--c;
}
H[c].first = x;
H[c].second = W[a];
A[i] = a;
}
}
/*
*
* Other block-related functions
@ -1723,7 +1807,7 @@ ton::AccountIdPrefixFull interpolate_addr(const ton::AccountIdPrefixFull& src, c
unsigned long long mask = (std::numeric_limits<td::uint64>::max() >> (d - 32));
return ton::AccountIdPrefixFull{dest.workchain, (dest.account_id_prefix & ~mask) | (src.account_id_prefix & mask)};
} else {
int mask = (-1 >> d);
int mask = (int)(~0U >> d);
return ton::AccountIdPrefixFull{(dest.workchain & ~mask) | (src.workchain & mask), src.account_id_prefix};
}
}

View file

@ -163,12 +163,12 @@ struct MsgProcessedUpto {
MsgProcessedUpto(ton::ShardId _shard, ton::BlockSeqno _mcseqno, ton::LogicalTime _lt, td::ConstBitPtr _hash)
: shard(_shard), mc_seqno(_mcseqno), last_inmsg_lt(_lt), last_inmsg_hash(_hash) {
}
bool operator<(const MsgProcessedUpto& other) const & {
bool operator<(const MsgProcessedUpto& other) const& {
return shard < other.shard || (shard == other.shard && mc_seqno < other.mc_seqno);
}
bool contains(const MsgProcessedUpto& other) const &;
bool contains(const MsgProcessedUpto& other) const&;
bool contains(ton::ShardId other_shard, ton::LogicalTime other_lt, td::ConstBitPtr other_hash,
ton::BlockSeqno other_mc_seqno) const &;
ton::BlockSeqno other_mc_seqno) const&;
// NB: this is for checking whether we have already imported an internal message
bool already_processed(const EnqueuedMsgDescr& msg) const;
bool can_check_processed() const {
@ -596,6 +596,62 @@ struct BlockProofChain {
td::Status validate(td::CancellationToken cancellation_token = {});
};
// compute the share of shardchain blocks generated by each validator using Monte Carlo method
class MtCarloComputeShare {
int K, N;
long long iterations;
std::vector<double> W;
std::vector<double> CW, RW;
std::vector<std::pair<double, double>> H;
std::vector<int> A;
double R0;
bool ok;
public:
MtCarloComputeShare(int subset_size, const std::vector<double>& weights, long long iteration_count = 1000000)
: K(subset_size), N((int)weights.size()), iterations(iteration_count), W(weights), ok(false) {
compute();
}
MtCarloComputeShare(int subset_size, int set_size, const double* weights, long long iteration_count = 1000000)
: K(subset_size), N(set_size), iterations(iteration_count), W(weights, weights + set_size), ok(false) {
compute();
}
bool is_ok() const {
return ok;
}
const double* share_array() const {
return ok ? RW.data() : nullptr;
}
const double* weights_array() const {
return ok ? W.data() : nullptr;
}
double operator[](int i) const {
return ok ? RW.at(i) : -1.;
}
double share(int i) const {
return ok ? RW.at(i) : -1.;
}
double weight(int i) const {
return ok ? W.at(i) : -1.;
}
int size() const {
return N;
}
int subset_size() const {
return K;
}
long long performed_iterations() const {
return iterations;
}
private:
bool set_error() {
return ok = false;
}
bool compute();
void gen_vset();
};
int filter_out_msg_queue(vm::AugmentedDictionary& out_queue, ton::ShardIdFull old_shard, ton::ShardIdFull subshard);
std::ostream& operator<<(std::ostream& os, const ShardId& shard_id);

View file

@ -272,6 +272,7 @@ transaction$0111 account_addr:bits256 lt:uint64
old:^X new:^X = MERKLE_UPDATE X;
update_hashes#72 {X:Type} old_hash:bits256 new_hash:bits256
= HASH_UPDATE X;
!merkle_proof#03 {X:Type} virtual_hash:bits256 depth:uint16 virtual_root:^X = MERKLE_PROOF X;
acc_trans#5 account_addr:bits256
transactions:(HashmapAug 64 ^Transaction CurrencyCollection)
@ -731,8 +732,11 @@ top_block_descr_set#4ac789f3 collection:(HashmapE 96 ^TopBlockDescr) = TopBlockD
//
// VALIDATOR MISBEHAVIOR COMPLAINTS
//
no_blk_gen mc_blk_ref:ExtBlkRef from_utime:uint32 to_utime:uint32 state_proof:^Cell prod_proof:^Cell = ComplaintDescr;
validator_complaint#bc validator_pubkey:uint256 description:^ComplaintDescr created_at:uint32 severity:uint8 reward_addr:uint256 paid:Grams suggested_fine:Grams suggested_fine_part:uint32 = ValidatorComplaint;
prod_info#34 utime:uint32 mc_blk_ref:ExtBlkRef state_proof:^(MERKLE_PROOF Block)
prod_proof:^(MERKLE_PROOF ShardState) = ProducerInfo;
no_blk_gen from_utime:uint32 prod_info:^ProducerInfo = ComplaintDescr;
no_blk_gen_diff prod_info_old:^ProducerInfo prod_info_new:^ProducerInfo = ComplaintDescr;
validator_complaint#bc validator_pubkey:bits256 description:^ComplaintDescr created_at:uint32 severity:uint8 reward_addr:uint256 paid:Grams suggested_fine:Grams suggested_fine_part:uint32 = ValidatorComplaint;
complaint_status#2d complaint:^ValidatorComplaint voters:(HashmapE 16 True) vset_id:uint256 weight_remaining:int64 = ValidatorComplaintStatus;
//
@ -808,3 +812,25 @@ cap_method_pubkey#71f4 = SmcCapability;
cap_is_wallet#2177 = SmcCapability;
cap_name#ff name:Text = SmcCapability;
//
// PAYMENT CHANNELS
//
chan_config$_ init_timeout:uint32 close_timeout:uint32 a_key:bits256 b_key:bits256
a_addr:^MsgAddressInt b_addr:^MsgAddressInt channel_id:uint64 = ChanConfig;
chan_state_init$000 signed_A:Bool signed_B:Bool min_A:Grams min_B:Grams expire_at:uint32 A:Grams B:Grams = ChanState;
chan_state_close$001 signed_A:Bool signed_B:Bool promise_A:Grams promise_B:Grams expire_at:uint32 A:Grams B:Grams = ChanState;
chan_state_payout$010 A:Grams B:Grams = ChanState;
chan_promise$_ channel_id:uint64 promise_A:Grams promise_B:Grams = ChanPromise;
chan_signed_promise#_ sig:(Maybe ^bits512) promise:ChanPromise = ChanSignedPromise;
chan_msg_init#27317822 inc_A:Grams inc_B:Grams min_A:Grams min_B:Grams channel_id:uint64 = ChanMsg;
chan_msg_close#f28ae183 extra_A:Grams extra_B:Grams promise:ChanSignedPromise = ChanMsg;
chan_msg_timeout#43278a28 = ChanMsg;
chan_signed_msg$_ sig_A:(Maybe ^bits512) sig_B:(Maybe ^bits512) msg:ChanMsg = ChanSignedMsg;
chan_data$_ config:^ChanConfig state:^ChanState = ChanData;

View file

@ -31,7 +31,7 @@
namespace block {
using namespace std::literals::string_literals;
td::Status check_block_header_proof(td::Ref<vm::Cell> root, ton::BlockIdExt blkid, ton::Bits256* store_shard_hash_to,
td::Status check_block_header_proof(td::Ref<vm::Cell> root, ton::BlockIdExt blkid, ton::Bits256* store_state_hash_to,
bool check_state_hash, td::uint32* save_utime, ton::LogicalTime* save_lt) {
ton::RootHash vhash{root->get_hash().bits()};
if (vhash != blkid.root_hash) {
@ -53,7 +53,7 @@ td::Status check_block_header_proof(td::Ref<vm::Cell> root, ton::BlockIdExt blki
if (save_lt) {
*save_lt = info.end_lt;
}
if (store_shard_hash_to) {
if (store_state_hash_to) {
vm::CellSlice upd_cs{vm::NoVmSpec(), blk.state_update};
if (!(upd_cs.is_special() && upd_cs.prefetch_long(8) == 4 // merkle update
&& upd_cs.size_ext() == 0x20228)) {
@ -61,11 +61,11 @@ td::Status check_block_header_proof(td::Ref<vm::Cell> root, ton::BlockIdExt blki
}
auto upd_hash = upd_cs.prefetch_ref(1)->get_hash(0);
if (!check_state_hash) {
*store_shard_hash_to = upd_hash.bits();
} else if (store_shard_hash_to->compare(upd_hash.bits())) {
*store_state_hash_to = upd_hash.bits();
} else if (store_state_hash_to->compare(upd_hash.bits())) {
return td::Status::Error(PSTRING() << "state hash mismatch in block header of " << blkid.to_str()
<< " : header declares " << upd_hash.bits().to_hex(256) << " expected "
<< store_shard_hash_to->to_hex());
<< store_state_hash_to->to_hex());
}
}
return td::Status::OK();

View file

@ -25,7 +25,7 @@ namespace block {
using td::Ref;
td::Status check_block_header_proof(td::Ref<vm::Cell> root, ton::BlockIdExt blkid,
ton::Bits256* store_shard_hash_to = nullptr, bool check_state_hash = false,
ton::Bits256* store_state_hash_to = nullptr, bool check_state_hash = false,
td::uint32* save_utime = nullptr, ton::LogicalTime* save_lt = nullptr);
td::Status check_shard_proof(ton::BlockIdExt blk, ton::BlockIdExt shard_blk, td::Slice shard_proof);
td::Status check_account_proof(td::Slice proof, ton::BlockIdExt shard_blk, const block::StdAddress& addr,

View file

@ -1785,6 +1785,31 @@ std::vector<ton::ValidatorDescr> ValidatorSet::export_validator_set() const {
return l;
}
std::map<ton::Bits256, int> ValidatorSet::compute_validator_map() const {
std::map<ton::Bits256, int> res;
for (int i = 0; i < (int)list.size(); i++) {
res.emplace(list[i].pubkey.as_bits256(), i);
}
return res;
}
std::vector<double> ValidatorSet::export_scaled_validator_weights() const {
std::vector<double> res;
for (const auto& node : list) {
res.push_back((double)node.weight / (double)total_weight);
}
return res;
}
int ValidatorSet::lookup_public_key(td::ConstBitPtr pubkey) const {
for (int i = 0; i < (int)list.size(); i++) {
if (list[i].pubkey.as_bits256() == pubkey) {
return i;
}
}
return -1;
}
std::vector<ton::ValidatorDescr> Config::do_compute_validator_set(const block::CatchainValidatorsConfig& ccv_conf,
ton::ShardIdFull shard,
const block::ValidatorSet& vset, ton::UnixTime time,

View file

@ -71,6 +71,12 @@ struct ValidatorSet {
}
const ValidatorDescr& at_weight(td::uint64 weight_pos) const;
std::vector<ton::ValidatorDescr> export_validator_set() const;
std::map<ton::Bits256, int> compute_validator_map() const;
std::vector<double> export_scaled_validator_weights() const;
int lookup_public_key(td::ConstBitPtr pubkey) const;
int lookup_public_key(const td::Bits256& pubkey) const {
return lookup_public_key(pubkey.bits());
}
};
#pragma pack(push, 1)

View file

@ -0,0 +1,199 @@
/*
This file is part of TON Blockchain source code.
TON Blockchain is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.
TON Blockchain is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with TON Blockchain. If not, see <http://www.gnu.org/licenses/>.
In addition, as a special exception, the copyright holders give permission
to link the code of portions of this program with the OpenSSL library.
You must obey the GNU General Public License in all respects for all
of the code used other than OpenSSL. If you modify file(s) with this
exception, you may extend this exception to your version of the file(s),
but you are not obligated to do so. If you do not wish to do so, delete this
exception statement from your version. If you delete this exception statement
from all source files in the program, then also delete it here.
Copyright 2020 Telegram Systems LLP
*/
#include <iostream>
#include "td/utils/Random.h"
#include "td/utils/misc.h"
#include "block/block.h"
#include <getopt.h>
const int MAX_N = 1000, MAX_K = 100, DEFAULT_K = 7;
int verbosity;
int N, K = DEFAULT_K;
long long iterations = 1000000;
td::uint64 TWL, WL[MAX_N];
double W[MAX_N], CW[MAX_N + 1], RW[MAX_N], R0;
int A[MAX_N], C[MAX_N];
long long TC;
void gen_vset() {
static std::pair<double, double> H[MAX_N];
double total_wt = 1.;
int hc = 0;
for (int i = 0; i < K; i++) {
CHECK(total_wt > 0);
double inv_wt = 1. / total_wt;
R0 += inv_wt; // advanced mtcarlo stats
for (int j = 0; j < i; j++) {
RW[A[j]] -= inv_wt; // advanced mtcarlo stats
}
// double p = drand48() * total_wt;
double p = (double)td::Random::fast_uint64() * total_wt / (1. * (1LL << 32) * (1LL << 32));
for (int h = 0; h < hc; h++) {
if (p < H[h].first) {
break;
}
p += H[h].second;
}
int a = -1, b = N, c;
while (b - a > 1) {
c = ((a + b) >> 1);
if (CW[c] <= p) {
a = c;
} else {
b = c;
}
}
CHECK(a >= 0 && a < N);
CHECK(total_wt >= W[a]);
total_wt -= W[a];
double x = CW[a];
c = hc++;
while (c > 0 && H[c - 1].first > x) {
H[c] = H[c - 1];
--c;
}
H[c].first = x;
H[c].second = W[a];
A[i] = a;
C[a]++; // simple mtcarlo stats
// std::cout << a << ' ';
}
// std::cout << std::endl;
++TC; // simple mtcarlo stats
}
void mt_carlo() {
for (int i = 0; i < N; i++) {
C[i] = 0;
RW[i] = 0.;
}
TC = 0;
R0 = 0.;
std::cout << "running " << iterations << " steps of Monte Carlo simulation\n";
for (long long it = 0; it < iterations; ++it) {
gen_vset();
}
for (int i = 0; i < N; i++) {
RW[i] = W[i] * (RW[i] + R0) / (double)iterations;
}
}
double B[MAX_N];
void compute_bad_approx() {
static double S[MAX_K + 1];
S[0] = 1.;
for (int i = 1; i <= K; i++) {
S[i] = 0.;
}
for (int i = 0; i < N; i++) {
double p = W[i];
for (int j = K; j > 0; j--) {
S[j] += p * S[j - 1];
}
}
double Sk = S[K];
for (int i = 0; i < N; i++) {
double t = 1., p = W[i];
for (int j = 1; j <= K; j++) {
t = S[j] - p * t;
}
B[i] = 1. - t / Sk;
}
}
void usage() {
std::cout
<< "usage: test-weight-distr [-k<shard-val-num>][-m<iterations>][-s<rand-seed>]\nReads the set of validator "
"weights from stdin and emulates validator shard distribution load\n\t-k <shard-val-num>\tSets the number of "
"validators generating each shard\n\t-m <iterations>\tMonte Carlo simulation steps\n";
std::exit(2);
}
int main(int argc, char* const argv[]) {
int i;
int new_verbosity_level = VERBOSITY_NAME(INFO);
// long seed = 0;
while ((i = getopt(argc, argv, "hs:k:m:v:")) != -1) {
switch (i) {
case 'k':
K = td::to_integer<int>(td::Slice(optarg));
CHECK(K > 0 && K <= 100);
break;
case 'm':
iterations = td::to_integer<long long>(td::Slice(optarg));
CHECK(iterations > 0);
break;
case 's':
// seed = td::to_integer<long>(td::Slice(optarg));
// srand48(seed);
break;
case 'v':
new_verbosity_level = VERBOSITY_NAME(FATAL) + (verbosity = td::to_integer<int>(td::Slice(optarg)));
break;
case 'h':
usage();
std::exit(2);
default:
usage();
std::exit(2);
}
}
SET_VERBOSITY_LEVEL(new_verbosity_level);
for (N = 0; N < MAX_N && (std::cin >> WL[N]); N++) {
CHECK(WL[N] > 0);
TWL += WL[N];
}
CHECK(std::cin.eof());
CHECK(N > 0 && TWL > 0 && N <= MAX_N);
K = std::min(K, N);
CHECK(K > 0 && K <= MAX_K);
double acc = 0.;
for (i = 0; i < N; i++) {
CW[i] = acc;
acc += W[i] = (double)WL[i] / (double)TWL;
std::cout << "#" << i << ":\t" << W[i] << std::endl;
}
compute_bad_approx();
mt_carlo();
std::cout << "result of Monte Carlo simulation (" << iterations << " iterations):" << std::endl;
std::cout << "idx\tweight\tmtcarlo1\tmtcarlo2\tapprox\n";
for (i = 0; i < N; i++) {
std::cout << "#" << i << ":\t" << W[i] << '\t' << (double)C[i] / (double)iterations << '\t' << RW[i] << '\t' << B[i]
<< std::endl;
}
// same computation, but using a MtCarloComputeShare object
block::MtCarloComputeShare MT(K, N, W, iterations);
std::cout << "-----------------------\n";
for (i = 0; i < N; i++) {
std::cout << '#' << i << ":\t" << MT.weight(i) << '\t' << MT.share(i) << std::endl;
}
return 0;
}

View file

@ -1429,6 +1429,9 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap,
if (!tlb::csr_unpack(msg.info, erec)) {
return -1;
}
if (act_rec.mode & ~3) {
return -1; // invalid mode for an external message
}
info.src = std::move(erec.src);
info.dest = std::move(erec.dest);
// created_lt and created_at are ignored