1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-03-09 15:40:10 +00:00

New account storage stat for accelerator

This commit is contained in:
SpyCheese 2025-03-05 17:02:16 +03:00
parent 15da5e846b
commit 928f02e6a1
14 changed files with 514 additions and 127 deletions

View file

@ -53,51 +53,83 @@ AccountStorageStat& AccountStorageStat::operator=(AccountStorageStat&& other) {
return *this;
}
td::Result<AccountStorageStat::CellInfo> AccountStorageStat::add_root(const Ref<vm::Cell>& cell) {
roots_.push_back(cell);
return add_cell(cell);
}
td::Status AccountStorageStat::remove_root(const Ref<vm::Cell>& cell) {
auto it = std::find_if(roots_.begin(), roots_.end(),
[&](const Ref<vm::Cell>& c) { return c->get_hash() == cell->get_hash(); });
if (it == roots_.end()) {
return td::Status::Error(PSTRING() << "no such root " << cell->get_hash().to_hex());
}
roots_.erase(it);
return remove_cell(cell);
}
td::Result<AccountStorageStat::CellInfo> AccountStorageStat::replace_roots(std::vector<Ref<vm::Cell>> new_roots) {
std::vector<Ref<vm::Cell>> old_roots = roots_;
std::erase_if(new_roots, [](const Ref<vm::Cell>& c) { return c.is_null(); });
auto cmp = [](const Ref<vm::Cell>& c1, const Ref<vm::Cell>& c2) { return c1->get_hash() < c2->get_hash(); };
std::sort(new_roots.begin(), new_roots.end(), cmp);
std::sort(roots_.begin(), roots_.end(), cmp);
std::vector<Ref<vm::Cell>> to_add, to_del;
std::set_difference(new_roots.begin(), new_roots.end(), roots_.begin(), roots_.end(), std::back_inserter(to_add),
cmp);
std::set_difference(roots_.begin(), roots_.end(), new_roots.begin(), new_roots.end(), std::back_inserter(to_del),
cmp);
td::uint32 max_merkle_depth = 0;
for (const Ref<vm::Cell>& root : new_roots) {
if (root.is_null()) {
continue;
}
TRY_RESULT(info, add_root(root));
for (const Ref<vm::Cell>& root : to_add) {
TRY_RESULT(info, add_cell(root));
max_merkle_depth = std::max(max_merkle_depth, info.max_merkle_depth);
}
for (const Ref<vm::Cell>& root : old_roots) {
TRY_STATUS(remove_root(root));
for (const Ref<vm::Cell>& root : to_del) {
TRY_STATUS(remove_cell(root));
}
roots_ = std::move(new_roots);
td::Status S = td::Status::OK();
cache_.for_each([&](Entry& e) {
if (S.is_ok()) {
S = commit_entry(e);
}
});
TRY_STATUS(std::move(S));
return CellInfo{max_merkle_depth};
}
void AccountStorageStat::add_hint(const td::HashSet<vm::CellHash>& hint) {
td::HashSet<vm::CellHash> visited;
std::function<void(const Ref<vm::Cell>&, bool)> dfs = [&](const Ref<vm::Cell>& cell, bool is_root) {
if (!visited.insert(cell->get_hash()).second) {
return;
}
Entry& e = get_entry(cell);
e.exists = e.exists_known = true;
if (is_root) {
fetch_entry(e).ignore();
if (e.max_merkle_depth && e.max_merkle_depth.value() != 0) {
return;
}
}
if (hint.contains(cell->get_hash())) {
bool spec;
vm::CellSlice cs = vm::load_cell_slice_special(cell, spec);
for (unsigned i = 0; i < cs.size_refs(); ++i) {
dfs(cs.prefetch_ref(i), false);
}
}
};
for (const Ref<vm::Cell>& root : roots_) {
dfs(root, true);
}
}
td::Result<AccountStorageStat::CellInfo> AccountStorageStat::add_cell(const Ref<vm::Cell>& cell) {
Entry& e = get_entry(cell);
++e.refcnt;
if (e.refcnt == 0) {
return td::Status::Error(PSTRING() << "cell " << cell->get_hash().to_hex() << ": refcnt overflow");
if (!e.exists_known || e.refcnt_diff < 0) {
TRY_STATUS(fetch_entry(e));
}
if (e.refcnt != 1) {
update_dict(e);
return CellInfo{e.max_merkle_depth};
++e.refcnt_diff;
if (e.exists || e.refcnt_diff > 1 || (e.refcnt && e.refcnt.value() + e.refcnt_diff != 1)) {
if (!e.max_merkle_depth) {
TRY_STATUS(fetch_entry(e));
if (!e.max_merkle_depth) {
return td::Status::Error(PSTRING() << "unexpected unknown Merkle depth of cell " << cell->get_hash());
}
}
return CellInfo{e.max_merkle_depth.value()};
}
td::uint32 max_merkle_depth = 0;
vm::CellSlice cs{vm::NoVm{}, cell};
++total_cells_;
total_bits_ += cs.size();
bool spec;
vm::CellSlice cs = vm::load_cell_slice_special(cell, spec);
for (unsigned i = 0; i < cs.size_refs(); ++i) {
TRY_RESULT(info, add_cell(cs.prefetch_ref(i)));
max_merkle_depth = std::max(max_merkle_depth, info.max_merkle_depth);
@ -109,64 +141,99 @@ td::Result<AccountStorageStat::CellInfo> AccountStorageStat::add_cell(const Ref<
max_merkle_depth = std::min(max_merkle_depth, MERKLE_DEPTH_LIMIT);
Entry& e2 = get_entry(cell);
e2.max_merkle_depth = max_merkle_depth;
update_dict(e2);
return CellInfo{max_merkle_depth};
}
td::Status AccountStorageStat::remove_cell(const Ref<vm::Cell>& cell) {
Entry& e = get_entry(cell);
if (e.refcnt == 0) {
return td::Status::Error(PSTRING() << "cell " << cell->get_hash().to_hex() << " is not in the dict");
if (!e.exists_known) {
TRY_STATUS(fetch_entry(e));
}
--e.refcnt;
update_dict(e);
if (e.refcnt != 0) {
if (!e.exists) {
return td::Status::Error(PSTRING() << "Failed to remove cell " << cell->get_hash().to_hex()
<< " : does not exist in the dict");
}
--e.refcnt_diff;
if (e.refcnt_diff < 0 && !e.refcnt) {
TRY_STATUS(fetch_entry(e));
}
if (e.refcnt.value() + e.refcnt_diff != 0) {
return td::Status::OK();
}
vm::CellSlice cs{vm::NoVm{}, std::move(cell)};
if (total_cells_ == 0 || total_bits_ < cs.size()) {
return td::Status::Error("total_cell/total_bits becomes negative");
}
--total_cells_;
total_bits_ -= cs.size();
bool spec;
vm::CellSlice cs = vm::load_cell_slice_special(cell, spec);
for (unsigned i = 0; i < cs.size_refs(); ++i) {
TRY_STATUS(remove_cell(cs.prefetch_ref(i)));
}
return td::Status::OK();
}
bool AccountStorageStat::Entry::serialize(vm::CellBuilder& cb) const {
return cb.store_long_bool(refcnt, 32) && cb.store_long_bool(max_merkle_depth, 2);
}
void AccountStorageStat::Entry::fetch(Ref<vm::CellSlice> cs) {
if (cs.is_null()) {
refcnt = max_merkle_depth = 0;
} else {
refcnt = (td::uint32)cs.write().fetch_ulong(32);
max_merkle_depth = (td::uint32)cs.write().fetch_ulong(2);
}
}
AccountStorageStat::Entry& AccountStorageStat::get_entry(const Ref<vm::Cell>& cell) {
return cache_.apply(cell->get_hash().as_slice(), [&](Entry& e) {
if (e.inited) {
return;
}
e.inited = true;
e.hash = cell->get_hash();
e.fetch(dict_.lookup(e.hash.as_bitslice()));
e.cell = cell;
});
}
void AccountStorageStat::update_dict(const Entry& e) {
if (e.refcnt == 0) {
dict_.lookup_delete(e.hash.as_bitslice());
} else {
vm::CellBuilder cb;
CHECK(e.serialize(cb));
dict_.set_builder(e.hash.as_bitslice(), cb);
td::Status AccountStorageStat::fetch_entry(Entry& e) {
if (e.exists_known && e.refcnt && (!e.exists || e.max_merkle_depth)) {
return td::Status::OK();
}
auto cs = dict_.lookup(e.cell->get_hash().as_bitslice());
if (cs.is_null()) {
e.exists = false;
e.refcnt = 0;
} else {
if (cs->size_ext() != 32 + 2) {
return td::Status::Error(PSTRING() << "invalid record for cell " << e.cell->get_hash().to_hex());
}
e.exists = true;
e.refcnt = (td::uint32)cs.write().fetch_ulong(32);
e.max_merkle_depth = (td::uint32)cs.write().fetch_ulong(2);
if (e.refcnt.value() == 0) {
return td::Status::Error(PSTRING() << "invalid refcnt=0 for cell " << e.cell->get_hash().to_hex());
}
}
e.exists_known = true;
return td::Status::OK();
}
td::Status AccountStorageStat::commit_entry(Entry& e) {
if (e.refcnt_diff == 0) {
return td::Status::OK();
}
TRY_STATUS(fetch_entry(e));
e.refcnt.value() += e.refcnt_diff;
e.refcnt_diff = 0;
bool spec;
if (e.refcnt.value() == 0) {
--total_cells_;
total_bits_ -= vm::load_cell_slice_special(e.cell, spec).size();
e.exists = false;
if (dict_.lookup_delete(e.cell->get_hash().as_bitslice()).is_null()) {
return td::Status::Error(PSTRING() << "Failed to delete entry " << e.cell->get_hash().to_hex());
}
} else {
if (!e.exists) {
++total_cells_;
total_bits_ += vm::load_cell_slice_special(e.cell, spec).size();
}
e.exists = true;
if (!e.max_merkle_depth) {
return td::Status::Error(PSTRING() << "Failed to store entry " << e.cell->get_hash().to_hex()
<< " : unknown merkle depth");
}
vm::CellBuilder cb;
dict_.set_builder(e.cell->get_hash().as_bitslice(), cb);
CHECK(cb.store_long_bool(e.refcnt.value(), 32) && cb.store_long_bool(e.max_merkle_depth.value(), 2));
if (!dict_.set_builder(e.cell->get_hash().as_bitslice(), cb)) {
return td::Status::Error(PSTRING() << "Failed to store entry " << e.cell->get_hash().to_hex());
}
}
return td::Status::OK();
}
} // namespace block

View file

@ -58,9 +58,8 @@ class AccountStorageStat {
return dict_.is_empty() ? td::Bits256::zero() : td::Bits256{dict_.get_root_cell()->get_hash().bits()};
}
td::Result<CellInfo> add_root(const Ref<vm::Cell> &cell);
td::Status remove_root(const Ref<vm::Cell> &cell);
td::Result<CellInfo> replace_roots(std::vector<Ref<vm::Cell>> new_roots);
td::Result<CellInfo> replace_roots(std::vector<Ref<vm::Cell>> hint);
void add_hint(const td::HashSet<vm::CellHash> &visited);
private:
vm::Dictionary dict_;
@ -72,15 +71,14 @@ class AccountStorageStat {
struct Entry {
bool inited = false;
vm::Cell::Hash hash;
td::uint32 refcnt = 0;
td::uint32 max_merkle_depth = 0;
void fetch(Ref<vm::CellSlice> cs);
bool serialize(vm::CellBuilder &cb) const;
Ref<vm::Cell> cell;
bool exists_known = false;
bool exists = false;
td::optional<td::uint32> refcnt, max_merkle_depth;
td::int32 refcnt_diff = 0;
vm::Cell::Hash key() const {
return hash;
return cell->get_hash();
}
bool operator<(const Entry &other) const {
return key() < other.key();
@ -111,7 +109,8 @@ class AccountStorageStat {
vm::CellHashTable<Entry> cache_;
Entry &get_entry(const Ref<vm::Cell> &cell);
void update_dict(const Entry &e);
td::Status fetch_entry(Entry &e);
td::Status commit_entry(Entry &e);
static constexpr td::uint32 MERKLE_DEPTH_LIMIT = 3;
};

View file

@ -852,6 +852,7 @@ top_block_descr#d5 proof_for:BlockIdExt signatures:(Maybe ^BlockSignatures)
// COLLATED DATA
//
top_block_descr_set#4ac789f3 collection:(HashmapE 96 ^TopBlockDescr) = TopBlockDescrSet;
account_storage_dict_proof#37c1e3fc proof:^Cell = AccountStorageDictProof;
//
// VALIDATOR MISBEHAVIOR COMPLAINTS

View file

@ -262,6 +262,7 @@ bool Account::unpack_storage_info(vm::CellSlice& cs) {
} else {
storage_dict_hash = {};
}
orig_storage_dict_hash = storage_dict_hash;
if (info.due_payment->prefetch_ulong(1) == 1) {
vm::CellSlice& cs2 = info.due_payment.write();
cs2.advance(1);
@ -310,8 +311,8 @@ bool Account::unpack_state(vm::CellSlice& cs) {
tock = z & 1;
LOG(DEBUG) << "tick=" << tick << ", tock=" << tock;
}
code = state.code->prefetch_ref();
data = state.data->prefetch_ref();
code = orig_code = state.code->prefetch_ref();
data = orig_data = state.data->prefetch_ref();
library = orig_library = state.library->prefetch_ref();
return true;
}
@ -531,7 +532,7 @@ bool Account::init_new(ton::UnixTime now) {
now_ = now;
last_paid = 0;
storage_used = {};
storage_dict_hash = {};
orig_storage_dict_hash = storage_dict_hash = {};
due_payment = td::zero_refint();
balance.set_zero();
if (my_addr_exact.is_null()) {
@ -562,6 +563,113 @@ bool Account::init_new(ton::UnixTime now) {
return true;
}
/**
* Removes extra currencies dict from AccountStorage.
*
* This is used for computing account storage stats.
*
* @param storage_cs AccountStorage as CellSlice.
*
* @returns AccountStorage without extra currencies as CellSlice.
*/
static td::Ref<vm::CellSlice> storage_without_extra_currencies(td::Ref<vm::CellSlice> storage_cs) {
block::gen::AccountStorage::Record rec;
if (!block::gen::csr_unpack(storage_cs, rec)) {
LOG(ERROR) << "failed to unpack AccountStorage";
return {};
}
if (rec.balance->size_refs() > 0) {
block::gen::CurrencyCollection::Record balance;
if (!block::gen::csr_unpack(rec.balance, balance)) {
LOG(ERROR) << "failed to unpack AccountStorage";
return {};
}
balance.other = vm::CellBuilder{}.store_zeroes(1).as_cellslice_ref();
if (!block::gen::csr_pack(rec.balance, balance)) {
LOG(ERROR) << "failed to pack AccountStorage";
return {};
}
}
td::Ref<vm::CellSlice> result;
if (!block::gen::csr_pack(result, rec)) {
LOG(ERROR) << "failed to pack AccountStorage";
return {};
}
return result;
}
/**
* Computes storage dict of the account from scratch.
* This requires storage_dict_hash to be set, as it guarantees that the stored storage_used was computed recently
* (in older versions it included extra currency balance, in newer versions it does not).
*
* @returns Root of the dictionary, or Error
*/
td::Result<Ref<vm::Cell>> Account::compute_account_storage_dict() const {
if (storage.is_null()) {
return td::Status::Error("cannot compute storage dict: empty storage");
}
if (!storage_dict_hash) {
return td::Status::Error("cannot compute storage dict: storage_dict_hash is not set");
}
AccountStorageStat stat;
auto storage_for_stat = storage_without_extra_currencies(storage);
if (storage_for_stat.is_null()) {
return td::Status::Error("cannot compute storage dict: invalid storage");
}
TRY_STATUS(stat.replace_roots(storage_for_stat->prefetch_all_refs()).move_as_status());
// Root of AccountStorage is not counted in AccountStorageStat
td::uint64 expected_cells = stat.get_total_cells() + 1;
td::uint64 expected_bits = stat.get_total_bits() + storage->size();
if (expected_cells != storage_used.cells || expected_bits != storage_used.bits) {
return td::Status::Error(PSTRING() << "invalid storage_used: computed cells=" << expected_cells
<< " bits=" << expected_bits << ", found cells" << storage_used.cells
<< " bits=" << storage_used.bits);
}
if (storage_dict_hash.value() != stat.get_dict_hash()) {
return td::Status::Error(PSTRING() << "invalid storage dict hash: computed " << stat.get_dict_hash().to_hex()
<< ", found " << storage_dict_hash.value().to_hex());
}
return stat.get_dict_root();
}
/**
* Initializes account_storage_stat of the account using the existing dict_root.
* This is not strictly necessary, as the storage stat is recomputed in Transaction.
* However, it can be used to optimize cell usage.
* This requires storage_dict_hash to be set, as it guarantees that the stored storage_used was computed recently
* (in older versions it included extra currency balance, in newer versions it does not).
*
* @param dict_root Root of the storage dictionary.
*
* @returns Status of the operation.
*/
td::Status Account::init_account_storage_stat(Ref<vm::Cell> dict_root) {
if (storage.is_null()) {
if (dict_root.not_null()) {
return td::Status::Error("storage is null, but dict_root is not null");
}
account_storage_stat = {};
return td::Status::OK();
}
if (!storage_dict_hash) {
return td::Status::Error("cannot init storage dict: storage_dict_hash is not set");
}
// Root of AccountStorage is not counted in AccountStorageStat
if (storage_used.cells < 1 || storage_used.bits < storage->size()) {
return td::Status::Error(PSTRING() << "storage_used is too small: cells=" << storage_used.cells
<< " bits=" << storage_used.bits << " storage_root_bits=" << storage->size());
}
AccountStorageStat new_stat(std::move(dict_root), storage->prefetch_all_refs(), storage_used.cells - 1,
storage_used.bits - storage->size());
if (storage_dict_hash.value() != new_stat.get_dict_hash()) {
return td::Status::Error(PSTRING() << "invalid storage dict hash: computed " << new_stat.get_dict_hash().to_hex()
<< ", found " << storage_dict_hash.value().to_hex());
}
account_storage_stat = std::move(new_stat);
return td::Status::OK();
}
/**
* Resets the split depth of the account.
*
@ -1814,6 +1922,7 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) {
<< cfg.flat_gas_limit << "]; remaining balance=" << balance.to_str();
CHECK(td::sgn(balance.grams) >= 0);
}
cp.vm_loaded_cells = vm.extract_loaded_cells();
return true;
}
@ -2956,6 +3065,9 @@ td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits,
{
TD_PERF_COUNTER(transaction_storage_stat_a);
td::Timer timer;
if (update_storage_stat && compute_phase) {
storage_stat.add_hint(compute_phase->vm_loaded_cells);
}
TRY_RESULT(info, storage_stat.replace_roots({new_code, new_data, new_library}));
if (info.max_merkle_depth > max_allowed_merkle_depth) {
return td::Status::Error("too big Merkle depth");
@ -3131,41 +3243,6 @@ bool Account::store_acc_status(vm::CellBuilder& cb, int acc_status) const {
return cb.store_long_bool(v, 2);
}
/**
* Removes extra currencies dict from AccountStorage.
*
* This is used for computing account storage stats.
*
* @param storage_cs AccountStorage as CellSlice.
*
* @returns AccountStorage without extra currencies as CellSlice.
*/
static td::Ref<vm::CellSlice> storage_without_extra_currencies(td::Ref<vm::CellSlice> storage_cs) {
block::gen::AccountStorage::Record rec;
if (!block::gen::csr_unpack(storage_cs, rec)) {
LOG(ERROR) << "failed to unpack AccountStorage";
return {};
}
if (rec.balance->size_refs() > 0) {
block::gen::CurrencyCollection::Record balance;
if (!block::gen::csr_unpack(rec.balance, balance)) {
LOG(ERROR) << "failed to unpack AccountStorage";
return {};
}
balance.other = vm::CellBuilder{}.store_zeroes(1).as_cellslice_ref();
if (!block::gen::csr_pack(rec.balance, balance)) {
LOG(ERROR) << "failed to pack AccountStorage";
return {};
}
}
td::Ref<vm::CellSlice> result;
if (!block::gen::csr_pack(result, rec)) {
LOG(ERROR) << "failed to pack AccountStorage";
return {};
}
return result;
}
namespace transaction {
/**
* Computes the new state of the account.
@ -3284,6 +3361,9 @@ bool Transaction::compute_state(const SerializeConfig& cfg) {
}
AccountStorageStat& stats = new_account_storage_stat.value_force();
// Don't check Merkle depth and size here - they were checked in check_state_limits
if (compute_phase) {
stats.add_hint(compute_phase->vm_loaded_cells);
}
auto S = stats.replace_roots(new_storage->prefetch_all_refs()).move_as_status();
if (S.is_error()) {
LOG(ERROR) << "Cannot recompute storage stats for account " << account.addr.to_hex() << ": " << S.move_as_error();
@ -3306,6 +3386,26 @@ bool Transaction::compute_state(const SerializeConfig& cfg) {
if (!cfg.store_storage_dict_hash) {
new_storage_dict_hash = {};
}
if (false) {
vm::CellStorageStat control_stats;
control_stats.add_used_storage(new_storage);
if (control_stats.bits != new_storage_used.bits || control_stats.cells != new_storage_used.cells) {
LOG(ERROR) << " [ QQQQQQ 1 Wrong storage stat " << account.workchain << ":" << account.addr.to_hex() << " "
<< start_lt << " : " << new_storage_used.cells << "," << new_storage_used.bits
<< " != " << control_stats.cells << "," << control_stats.bits << " ] ";
return false;
}
AccountStorageStat control_stats_2;
control_stats_2.replace_roots(new_storage->prefetch_all_refs());
if (control_stats_2.get_total_bits() + new_storage->size() != new_storage_used.bits ||
control_stats_2.get_total_cells() + 1 != new_storage_used.cells) {
LOG(ERROR) << " [ QQQQQQ 2 Wrong storage stat " << account.workchain << ":" << account.addr.to_hex() << " "
<< start_lt << " : " << new_storage_used.cells << "," << new_storage_used.bits
<< " != " << control_stats_2.get_total_cells() + 1 << ","
<< control_stats_2.get_total_bits() + new_storage->size() << " ] ";
return false;
}
}
CHECK(cb.store_long_bool(1, 1) // account$1
&& cb.append_cellslice_bool(account.my_addr) // addr:MsgAddressInt

View file

@ -208,6 +208,7 @@ struct ComputePhase {
Ref<vm::Cell> actions;
std::string vm_log;
td::optional<td::uint64> precompiled_gas_usage;
td::HashSet<vm::CellHash> vm_loaded_cells;
};
struct ActionPhase {
@ -272,6 +273,7 @@ struct Account {
ton::UnixTime last_paid;
StorageUsed storage_used;
td::optional<td::Bits256> storage_dict_hash;
td::optional<td::Bits256> orig_storage_dict_hash;
td::optional<AccountStorageStat> account_storage_stat;
block::CurrencyCollection balance;
@ -281,7 +283,8 @@ struct Account {
Ref<vm::CellSlice> storage; // AccountStorage
Ref<vm::CellSlice> inner_state; // StateInit
ton::Bits256 state_hash; // hash of StateInit for frozen accounts
Ref<vm::Cell> code, data, library, orig_library;
Ref<vm::Cell> code, data, library;
Ref<vm::Cell> orig_code, orig_data, orig_library;
std::vector<LtCellRef> transactions;
Account() = default;
Account(ton::WorkchainId wc, td::ConstBitPtr _addr) : workchain(wc), addr(_addr) {
@ -295,6 +298,8 @@ struct Account {
bool set_address(ton::WorkchainId wc, td::ConstBitPtr new_addr);
bool unpack(Ref<vm::CellSlice> account, ton::UnixTime now, bool special);
bool init_new(ton::UnixTime now);
td::Result<Ref<vm::Cell>> compute_account_storage_dict() const;
td::Status init_account_storage_stat(Ref<vm::Cell> dict_root);
bool deactivate();
bool recompute_tmp_addr(Ref<vm::CellSlice>& tmp_addr, int split_depth, td::ConstBitPtr orig_addr_rewrite) const;
td::RefInt256 compute_storage_fees(ton::UnixTime now, const std::vector<block::StoragePrices>& pricing) const;