1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-03-09 15:40:10 +00:00

New account storage stat

This commit is contained in:
SpyCheese 2025-02-28 18:00:36 +03:00
parent 44e7e091b2
commit 57d7c2a895
17 changed files with 479 additions and 177 deletions

View file

@ -19,6 +19,6 @@
namespace ton {
// See doc/GlobalVersions.md
constexpr int SUPPORTED_VERSION = 10;
constexpr int SUPPORTED_VERSION = 11;
}

View file

@ -214,6 +214,8 @@ set(BLOCK_SOURCE
block/mc-config.cpp
block/output-queue-merger.cpp
block/transaction.cpp
block/account-storage-stat.h
block/account-storage-stat.cpp
block/precompiled-smc/PrecompiledSmartContract.cpp
${TLB_BLOCK_AUTO}

View file

@ -0,0 +1,172 @@
/*
This file is part of TON Blockchain source code.
TON Blockchain is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.
TON Blockchain is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with TON Blockchain. If not, see <http://www.gnu.org/licenses/>.
*/
#include "account-storage-stat.h"
namespace block {
AccountStorageStat::AccountStorageStat() : AccountStorageStat({}, {}, 0, 0) {
}
AccountStorageStat::AccountStorageStat(Ref<vm::Cell> dict_root, std::vector<Ref<vm::Cell>> roots,
td::uint64 total_cells, td::uint64 total_bits)
: dict_(std::move(dict_root), 256), roots_(std::move(roots)), total_cells_(total_cells), total_bits_(total_bits) {
}
AccountStorageStat::AccountStorageStat(const AccountStorageStat& other)
: AccountStorageStat(other.dict_.get_root_cell(), other.roots_, other.total_cells_, other.total_bits_) {
}
AccountStorageStat::AccountStorageStat(AccountStorageStat&& other)
: AccountStorageStat(other.dict_.get_root_cell(), std::move(other.roots_), other.total_cells_, other.total_bits_) {
cache_ = std::move(other.cache_);
}
AccountStorageStat& AccountStorageStat::operator=(const AccountStorageStat& other) {
dict_ = other.dict_;
total_cells_ = other.total_cells_;
total_bits_ = other.total_bits_;
roots_ = other.roots_;
cache_ = {};
return *this;
}
AccountStorageStat& AccountStorageStat::operator=(AccountStorageStat&& other) {
dict_ = std::move(other.dict_);
total_cells_ = other.total_cells_;
total_bits_ = other.total_bits_;
roots_ = std::move(other.roots_);
cache_ = std::move(other.cache_);
return *this;
}
td::Result<AccountStorageStat::CellInfo> AccountStorageStat::add_root(const Ref<vm::Cell>& cell) {
roots_.push_back(cell);
return add_cell(cell);
}
td::Status AccountStorageStat::remove_root(const Ref<vm::Cell>& cell) {
auto it = std::find_if(roots_.begin(), roots_.end(),
[&](const Ref<vm::Cell>& c) { return c->get_hash() == cell->get_hash(); });
if (it == roots_.end()) {
return td::Status::Error(PSTRING() << "no such root " << cell->get_hash().to_hex());
}
roots_.erase(it);
return remove_cell(cell);
}
td::Result<AccountStorageStat::CellInfo> AccountStorageStat::replace_roots(std::vector<Ref<vm::Cell>> new_roots) {
std::vector<Ref<vm::Cell>> old_roots = roots_;
td::uint32 max_merkle_depth = 0;
for (const Ref<vm::Cell>& root : new_roots) {
if (root.is_null()) {
continue;
}
TRY_RESULT(info, add_root(root));
max_merkle_depth = std::max(max_merkle_depth, info.max_merkle_depth);
}
for (const Ref<vm::Cell>& root : old_roots) {
TRY_STATUS(remove_root(root));
}
return CellInfo{max_merkle_depth};
}
td::Result<AccountStorageStat::CellInfo> AccountStorageStat::add_cell(const Ref<vm::Cell>& cell) {
Entry& e = get_entry(cell);
++e.refcnt;
if (e.refcnt == 0) {
return td::Status::Error(PSTRING() << "cell " << cell->get_hash().to_hex() << ": refcnt overflow");
}
if (e.refcnt != 1) {
update_dict(e);
return CellInfo{e.max_merkle_depth};
}
td::uint32 max_merkle_depth = 0;
vm::CellSlice cs{vm::NoVm{}, cell};
++total_cells_;
total_bits_ += cs.size();
for (unsigned i = 0; i < cs.size_refs(); ++i) {
TRY_RESULT(info, add_cell(cs.prefetch_ref(i)));
max_merkle_depth = std::max(max_merkle_depth, info.max_merkle_depth);
}
if (cs.special_type() == vm::CellTraits::SpecialType::MerkleProof ||
cs.special_type() == vm::CellTraits::SpecialType::MerkleUpdate) {
++max_merkle_depth;
}
max_merkle_depth = std::min(max_merkle_depth, MERKLE_DEPTH_LIMIT);
Entry& e2 = get_entry(cell);
e2.max_merkle_depth = max_merkle_depth;
update_dict(e2);
return CellInfo{max_merkle_depth};
}
td::Status AccountStorageStat::remove_cell(const Ref<vm::Cell>& cell) {
Entry& e = get_entry(cell);
if (e.refcnt == 0) {
return td::Status::Error(PSTRING() << "cell " << cell->get_hash().to_hex() << " is not in the dict");
}
--e.refcnt;
update_dict(e);
if (e.refcnt != 0) {
return td::Status::OK();
}
vm::CellSlice cs{vm::NoVm{}, std::move(cell)};
if (total_cells_ == 0 || total_bits_ < cs.size()) {
return td::Status::Error("total_cell/total_bits becomes negative");
}
--total_cells_;
total_bits_ -= cs.size();
for (unsigned i = 0; i < cs.size_refs(); ++i) {
TRY_STATUS(remove_cell(cs.prefetch_ref(i)));
}
return td::Status::OK();
}
bool AccountStorageStat::Entry::serialize(vm::CellBuilder& cb) const {
return cb.store_long_bool(refcnt, 32) && cb.store_long_bool(max_merkle_depth, 2);
}
void AccountStorageStat::Entry::fetch(Ref<vm::CellSlice> cs) {
if (cs.is_null()) {
refcnt = max_merkle_depth = 0;
} else {
refcnt = (td::uint32)cs.write().fetch_ulong(32);
max_merkle_depth = (td::uint32)cs.write().fetch_ulong(2);
}
}
AccountStorageStat::Entry& AccountStorageStat::get_entry(const Ref<vm::Cell>& cell) {
return cache_.apply(cell->get_hash().as_slice(), [&](Entry& e) {
if (e.inited) {
return;
}
e.inited = true;
e.hash = cell->get_hash();
e.fetch(dict_.lookup(e.hash.as_bitslice()));
});
}
void AccountStorageStat::update_dict(const Entry& e) {
if (e.refcnt == 0) {
dict_.lookup_delete(e.hash.as_bitslice());
} else {
vm::CellBuilder cb;
CHECK(e.serialize(cb));
dict_.set_builder(e.hash.as_bitslice(), cb);
}
}
} // namespace block

View file

@ -0,0 +1,119 @@
/*
This file is part of TON Blockchain source code.
TON Blockchain is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.
TON Blockchain is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with TON Blockchain. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "common/refcnt.hpp"
#include "vm/dict.h"
#include "ton/ton-types.h"
#include "ton/ton-shard.h"
#include "common/bitstring.h"
#include "block.h"
#include "vm/db/CellHashTable.h"
namespace block {
using td::Ref;
class AccountStorageStat {
public:
struct CellInfo {
td::uint32 max_merkle_depth = 0;
};
AccountStorageStat();
AccountStorageStat(Ref<vm::Cell> dict_root, std::vector<Ref<vm::Cell>> roots, td::uint64 total_cells,
td::uint64 total_bits);
AccountStorageStat(const AccountStorageStat &other);
AccountStorageStat(AccountStorageStat &&other);
~AccountStorageStat() = default;
AccountStorageStat &operator=(const AccountStorageStat &other);
AccountStorageStat &operator=(AccountStorageStat &&other);
td::uint64 get_total_cells() const {
return total_cells_;
}
td::uint64 get_total_bits() const {
return total_bits_;
}
Ref<vm::Cell> get_dict_root() const {
return dict_.get_root_cell();
}
td::Bits256 get_dict_hash() const {
return dict_.is_empty() ? td::Bits256::zero() : td::Bits256{dict_.get_root_cell()->get_hash().bits()};
}
td::Result<CellInfo> add_root(const Ref<vm::Cell> &cell);
td::Status remove_root(const Ref<vm::Cell> &cell);
td::Result<CellInfo> replace_roots(std::vector<Ref<vm::Cell>> new_roots);
private:
vm::Dictionary dict_;
td::uint64 total_cells_, total_bits_;
std::vector<Ref<vm::Cell>> roots_;
td::Result<CellInfo> add_cell(const Ref<vm::Cell> &cell);
td::Status remove_cell(const Ref<vm::Cell> &cell);
struct Entry {
bool inited = false;
vm::Cell::Hash hash;
td::uint32 refcnt = 0;
td::uint32 max_merkle_depth = 0;
void fetch(Ref<vm::CellSlice> cs);
bool serialize(vm::CellBuilder &cb) const;
vm::Cell::Hash key() const {
return hash;
}
bool operator<(const Entry &other) const {
return key() < other.key();
}
struct Eq {
using is_transparent = void; // Pred to use
bool operator()(const Entry &info, const Entry &other_info) const {
return info.key() == other_info.key();
}
bool operator()(const Entry &info, td::Slice hash) const {
return info.key().as_slice() == hash;
}
bool operator()(td::Slice hash, const Entry &info) const {
return info.key().as_slice() == hash;
}
};
struct Hash {
using is_transparent = void; // Pred to use
using transparent_key_equal = Eq;
size_t operator()(td::Slice hash) const {
return cell_hash_slice_hash(hash);
}
size_t operator()(const Entry &info) const {
return cell_hash_slice_hash(info.key().as_slice());
}
};
};
vm::CellHashTable<Entry> cache_;
Entry &get_entry(const Ref<vm::Cell> &cell);
void update_dict(const Entry &e);
static constexpr td::uint32 MERKLE_DEPTH_LIMIT = 3;
};
} // namespace block

View file

@ -961,40 +961,33 @@ const RefTo<MsgEnvelope> t_Ref_MsgEnvelope;
bool StorageUsed::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const {
return t_VarUInteger_7.validate_skip(ops, cs, weak) // cells:(VarUInteger 7)
&& t_VarUInteger_7.validate_skip(ops, cs, weak) // bits:(VarUInteger 7)
&& t_VarUInteger_7.validate_skip(ops, cs, weak); // public_cells:(VarUInteger 7)
&& t_VarUInteger_7.validate_skip(ops, cs, weak); // bits:(VarUInteger 7)
}
bool StorageUsed::skip(vm::CellSlice& cs) const {
return t_VarUInteger_7.skip(cs) // cells:(VarUInteger 7)
&& t_VarUInteger_7.skip(cs) // bits:(VarUInteger 7)
&& t_VarUInteger_7.skip(cs); // public_cells:(VarUInteger 7)
&& t_VarUInteger_7.skip(cs); // bits:(VarUInteger 7)
}
const StorageUsed t_StorageUsed;
bool StorageUsedShort::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const {
return t_VarUInteger_7.validate_skip(ops, cs, weak) // cells:(VarUInteger 7)
&& t_VarUInteger_7.validate_skip(ops, cs, weak); // bits:(VarUInteger 7)
}
bool StorageUsedShort::skip(vm::CellSlice& cs) const {
return t_VarUInteger_7.skip(cs) // cells:(VarUInteger 7)
&& t_VarUInteger_7.skip(cs); // bits:(VarUInteger 7)
}
const StorageUsedShort t_StorageUsedShort;
const Maybe<Grams> t_Maybe_Grams;
bool StorageInfo::skip(vm::CellSlice& cs) const {
int extra_tag = 0;
return t_StorageUsed.skip(cs) // used:StorageUsed
&& cs.fetch_uint_to(3, extra_tag) // storage_extra:StorageExtraInfo
&& (extra_tag == 0 || cs.advance(256)) // storage_extra_info$001 dict_hash:uint256 = StorageExtraInfo;
&& cs.advance(32) // last_paid:uint32
&& t_Maybe_Grams.skip(cs); // due_payment:(Maybe Grams)
}
bool StorageInfo::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const {
int extra_tag = 0;
return t_StorageUsed.validate_skip(ops, cs, weak) // used:StorageUsed
&& cs.fetch_uint_to(3, extra_tag) // storage_extra:StorageExtraInfo
&& (extra_tag == 0 ||
(extra_tag == 1 && cs.advance(256))) // storage_extra_info$001 dict_hash:uint256 = StorageExtraInfo;
&& cs.advance(32) // last_paid:uint32
&& t_Maybe_Grams.validate_skip(ops, cs, weak); // due_payment:(Maybe Grams)
}
@ -1368,7 +1361,7 @@ bool TrActionPhase::skip(vm::CellSlice& cs) const {
&& cs.advance(16 * 4 + 256) // tot_actions:uint16 spec_actions:uint16
// skipped_actions:uint16 msgs_created:uint16
// action_list_hash:uint256
&& t_StorageUsedShort.skip(cs); // tot_msg_size:StorageUsedShort
&& t_StorageUsed.skip(cs); // tot_msg_size:StorageUsed
}
bool TrActionPhase::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const {
@ -1381,7 +1374,7 @@ bool TrActionPhase::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const
&& cs.advance(16 * 4 + 256) // tot_actions:uint16 spec_actions:uint16
// skipped_actions:uint16 msgs_created:uint16
// action_list_hash:uint256
&& t_StorageUsedShort.validate_skip(ops, cs, weak); // tot_msg_size:StorageUsed
&& t_StorageUsed.validate_skip(ops, cs, weak); // tot_msg_size:StorageUsed
}
const TrActionPhase t_TrActionPhase;
@ -1392,11 +1385,11 @@ bool TrBouncePhase::skip(vm::CellSlice& cs) const {
return cs.advance(2); // tr_phase_bounce_negfunds$00
case tr_phase_bounce_nofunds:
return cs.advance(2) // tr_phase_bounce_nofunds$01
&& t_StorageUsedShort.skip(cs) // msg_size:StorageUsedShort
&& t_StorageUsed.skip(cs) // msg_size:StorageUsed
&& t_Grams.skip(cs); // req_fwd_fees:Grams
case tr_phase_bounce_ok:
return cs.advance(1) // tr_phase_bounce_ok$1
&& t_StorageUsedShort.skip(cs) // msg_size:StorageUsedShort
&& t_StorageUsed.skip(cs) // msg_size:StorageUsed
&& t_Grams.skip(cs) // msg_fees:Grams
&& t_Grams.skip(cs); // fwd_fees:Grams
}
@ -1409,11 +1402,11 @@ bool TrBouncePhase::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const
return cs.advance(2); // tr_phase_bounce_negfunds$00
case tr_phase_bounce_nofunds:
return cs.advance(2) // tr_phase_bounce_nofunds$01
&& t_StorageUsedShort.validate_skip(ops, cs, weak) // msg_size:StorageUsedShort
&& t_StorageUsed.validate_skip(ops, cs, weak) // msg_size:StorageUsed
&& t_Grams.validate_skip(ops, cs, weak); // req_fwd_fees:Grams
case tr_phase_bounce_ok:
return cs.advance(1) // tr_phase_bounce_ok$1
&& t_StorageUsedShort.validate_skip(ops, cs, weak) // msg_size:StorageUsedShort
&& t_StorageUsed.validate_skip(ops, cs, weak) // msg_size:StorageUsed
&& t_Grams.validate_skip(ops, cs, weak) // msg_fees:Grams
&& t_Grams.validate_skip(ops, cs, weak); // fwd_fees:Grams
}

View file

@ -493,13 +493,6 @@ struct StorageUsed final : TLB_Complex {
extern const StorageUsed t_StorageUsed;
struct StorageUsedShort final : TLB_Complex {
bool skip(vm::CellSlice& cs) const override;
bool validate_skip(int* ops, vm::CellSlice& cs, bool weak = false) const override;
};
extern const StorageUsedShort t_StorageUsedShort;
struct StorageInfo final : TLB_Complex {
bool skip(vm::CellSlice& cs) const override;
bool validate_skip(int* ops, vm::CellSlice& cs, bool weak = false) const override;

View file

@ -246,14 +246,13 @@ out_msg_queue_extra#0 dispatch_queue:DispatchQueue out_queue_size:(Maybe uint48)
_ out_queue:OutMsgQueue proc_info:ProcessedInfo
extra:(Maybe OutMsgQueueExtra) = OutMsgQueueInfo;
//
storage_used$_ cells:(VarUInteger 7) bits:(VarUInteger 7)
public_cells:(VarUInteger 7) = StorageUsed;
storage_used_short$_ cells:(VarUInteger 7)
bits:(VarUInteger 7) = StorageUsedShort;
storage_extra_none$000 = StorageExtraInfo;
storage_extra_info$001 dict_hash:uint256 = StorageExtraInfo;
storage_info$_ used:StorageUsed last_paid:uint32
storage_used$_ cells:(VarUInteger 7) bits:(VarUInteger 7) = StorageUsed;
storage_info$_ used:StorageUsed storage_extra:StorageExtraInfo last_paid:uint32
due_payment:(Maybe Grams) = StorageInfo;
account_none$0 = Account;
@ -341,13 +340,13 @@ tr_phase_action$_ success:Bool valid:Bool no_funds:Bool
total_fwd_fees:(Maybe Grams) total_action_fees:(Maybe Grams)
result_code:int32 result_arg:(Maybe int32) tot_actions:uint16
spec_actions:uint16 skipped_actions:uint16 msgs_created:uint16
action_list_hash:bits256 tot_msg_size:StorageUsedShort
action_list_hash:bits256 tot_msg_size:StorageUsed
= TrActionPhase;
tr_phase_bounce_negfunds$00 = TrBouncePhase;
tr_phase_bounce_nofunds$01 msg_size:StorageUsedShort
tr_phase_bounce_nofunds$01 msg_size:StorageUsed
req_fwd_fees:Grams = TrBouncePhase;
tr_phase_bounce_ok$1 msg_size:StorageUsedShort
tr_phase_bounce_ok$1 msg_size:StorageUsed
msg_fees:Grams fwd_fees:Grams = TrBouncePhase;
//
trans_ord$0000 credit_first:Bool

View file

@ -339,9 +339,10 @@ td::RefInt256 create_smartcontract(td::RefInt256 smc_addr, Ref<vm::Cell> code, R
THRERR("Cannot serialize addr:MsgAddressInt of the new smart contract");
// storage_stat:StorageInfo -> storage_stat.used:StorageUsed
PDO(block::store_UInt7(cb, stats.cells) // cells:(VarUInteger 7)
&& block::store_UInt7(cb, stats.bits) // bits:(VarUInteger 7)
&& block::store_UInt7(cb, stats.public_cells)); // public_cells:(VarUInteger 7)
&& block::store_UInt7(cb, stats.bits)) // bits:(VarUInteger 7)
THRERR("Cannot serialize used:StorageUsed of the new smart contract");
PDO(cb.store_zeroes_bool(3)); // extra:StorageExtraInfo
THRERR("Cannot serialize storage_extra:StorageExtraInfo of the new smart contract");
PDO(cb.store_long_bool(0, 33)); // last_paid:uint32 due_payment:(Maybe Grams)
PDO(cb.append_data_cell_bool(storage)); // storage:AccountStorage
THRERR("Cannot create Account of the new smart contract");

View file

@ -156,6 +156,10 @@ class McShardHashI : public td::CntObject {
virtual bool before_merge() const = 0;
};
struct StorageUsed {
td::uint64 cells = 0, bits = 0;
};
struct McShardHash : public McShardHashI {
ton::BlockIdExt blk_;
ton::LogicalTime start_lt_, end_lt_;
@ -336,7 +340,7 @@ struct StoragePrices {
, mc_cell_price(_mc_cprice) {
}
static td::RefInt256 compute_storage_fees(ton::UnixTime now, const std::vector<block::StoragePrices>& pricing,
const vm::CellStorageStat& storage_stat, ton::UnixTime last_paid,
const StorageUsed& storage_used, ton::UnixTime last_paid,
bool is_special, bool is_masterchain);
};

View file

@ -257,6 +257,11 @@ bool Account::unpack_storage_info(vm::CellSlice& cs) {
return false;
}
last_paid = info.last_paid;
if (info.storage_extra.write().fetch_long(3) == 1) {
info.storage_extra->prefetch_bits_to(storage_dict_hash.value_force());
} else {
storage_dict_hash = {};
}
if (info.due_payment->prefetch_ulong(1) == 1) {
vm::CellSlice& cs2 = info.due_payment.write();
cs2.advance(1);
@ -268,11 +273,9 @@ bool Account::unpack_storage_info(vm::CellSlice& cs) {
due_payment = td::zero_refint();
}
unsigned long long u = 0;
u |= storage_stat.cells = block::tlb::t_VarUInteger_7.as_uint(*used.cells);
u |= storage_stat.bits = block::tlb::t_VarUInteger_7.as_uint(*used.bits);
u |= storage_stat.public_cells = block::tlb::t_VarUInteger_7.as_uint(*used.public_cells);
LOG(DEBUG) << "last_paid=" << last_paid << "; cells=" << storage_stat.cells << " bits=" << storage_stat.bits
<< " public_cells=" << storage_stat.public_cells;
u |= storage_used.cells = block::tlb::t_VarUInteger_7.as_uint(*used.cells);
u |= storage_used.bits = block::tlb::t_VarUInteger_7.as_uint(*used.bits);
LOG(DEBUG) << "last_paid=" << last_paid << "; cells=" << storage_used.cells << " bits=" << storage_used.bits;
return (u != std::numeric_limits<td::uint64>::max());
}
@ -527,7 +530,8 @@ bool Account::init_new(ton::UnixTime now) {
last_trans_hash_.set_zero();
now_ = now;
last_paid = 0;
storage_stat.clear();
storage_used = {};
storage_dict_hash = {};
due_payment = td::zero_refint();
balance.set_zero();
if (my_addr_exact.is_null()) {
@ -617,12 +621,12 @@ bool Account::belongs_to_shard(ton::ShardIdFull shard) const {
* @param payment The total sum to be updated.
* @param delta The time delta for which the payment is calculated.
* @param prices The storage prices.
* @param storage Account storage statistics.
* @param storage_used Account storage statistics.
* @param is_mc A flag indicating whether the account is in the masterchain.
*/
void add_partial_storage_payment(td::BigInt256& payment, ton::UnixTime delta, const block::StoragePrices& prices,
const vm::CellStorageStat& storage, bool is_mc) {
td::BigInt256 c{(long long)storage.cells}, b{(long long)storage.bits};
const StorageUsed& storage_used, bool is_mc) {
td::BigInt256 c{(long long)storage_used.cells}, b{(long long)storage_used.bits};
if (is_mc) {
// storage.cells * prices.mc_cell_price + storage.bits * prices.mc_bit_price;
c.mul_short(prices.mc_cell_price);
@ -643,7 +647,7 @@ void add_partial_storage_payment(td::BigInt256& payment, ton::UnixTime delta, co
*
* @param now The current Unix time.
* @param pricing The vector of storage prices.
* @param storage_stat Account storage statistics.
* @param storage_used Account storage statistics.
* @param last_paid The Unix time when the last payment was made.
* @param is_special A flag indicating if the account is special.
* @param is_masterchain A flag indicating if the account is in the masterchain.
@ -651,7 +655,7 @@ void add_partial_storage_payment(td::BigInt256& payment, ton::UnixTime delta, co
* @returns The computed storage fees as RefInt256.
*/
td::RefInt256 StoragePrices::compute_storage_fees(ton::UnixTime now, const std::vector<block::StoragePrices>& pricing,
const vm::CellStorageStat& storage_stat, ton::UnixTime last_paid,
const StorageUsed& storage_used, ton::UnixTime last_paid,
bool is_special, bool is_masterchain) {
if (now <= last_paid || !last_paid || is_special || pricing.empty() || now <= pricing[0].valid_since) {
return td::zero_refint();
@ -669,7 +673,7 @@ td::RefInt256 StoragePrices::compute_storage_fees(ton::UnixTime now, const std::
ton::UnixTime valid_until = (i < n - 1 ? std::min(now, pricing[i + 1].valid_since) : now);
if (upto < valid_until) {
assert(upto >= pricing[i].valid_since);
add_partial_storage_payment(total.unique_write(), valid_until - upto, pricing[i], storage_stat, is_masterchain);
add_partial_storage_payment(total.unique_write(), valid_until - upto, pricing[i], storage_used, is_masterchain);
}
upto = valid_until;
}
@ -685,7 +689,7 @@ td::RefInt256 StoragePrices::compute_storage_fees(ton::UnixTime now, const std::
* @returns The computed storage fees as RefInt256.
*/
td::RefInt256 Account::compute_storage_fees(ton::UnixTime now, const std::vector<block::StoragePrices>& pricing) const {
return StoragePrices::compute_storage_fees(now, pricing, storage_stat, last_paid, is_special, is_masterchain());
return StoragePrices::compute_storage_fees(now, pricing, storage_used, last_paid, is_special, is_masterchain());
}
namespace transaction {
@ -1848,7 +1852,7 @@ bool Transaction::prepare_action_phase(const ActionPhaseConfig& cfg) {
if (S.is_error()) {
// Rollback changes to state, fail action phase
LOG(INFO) << "Account state size exceeded limits: " << S.move_as_error();
new_storage_stat.clear();
new_account_storage_stat = {};
new_code = old_code;
new_data = old_data;
new_library = old_library;
@ -2104,7 +2108,7 @@ int Transaction::try_action_change_library(vm::CellSlice& cs, ActionPhase& ap, c
LOG(DEBUG) << "added " << ((rec.mode >> 1) ? "public" : "private") << " library with hash " << hash.to_hex();
}
new_library = std::move(dict).extract_root_cell();
} catch (vm::VmError& vme) {
} catch (vm::VmError&) {
return 42;
}
ap.spec_actions++;
@ -2931,7 +2935,7 @@ static td::uint32 get_public_libraries_diff_count(const td::Ref<vm::Cell>& old_l
* This function is not called for special accounts.
*
* @param size_limits The size limits configuration.
* @param update_storage_stat Store storage stat in the Transaction's CellStorageStat.
* @param update_storage_stat Store storage stat in the Transaction's AccountStorageStat.
*
* @returns A `td::Status` indicating the result of the check.
* - If the state limits are within the allowed range, returns OK.
@ -2939,60 +2943,47 @@ static td::uint32 get_public_libraries_diff_count(const td::Ref<vm::Cell>& old_l
*/
td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, bool update_storage_stat) {
auto cell_equal = [](const td::Ref<vm::Cell>& a, const td::Ref<vm::Cell>& b) -> bool {
if (a.is_null()) {
return b.is_null();
}
if (b.is_null()) {
return false;
}
return a->get_hash() == b->get_hash();
return a.is_null() || b.is_null() ? a.is_null() == b.is_null() : a->get_hash() == b->get_hash();
};
if (cell_equal(account.code, new_code) && cell_equal(account.data, new_data) &&
cell_equal(account.library, new_library)) {
return td::Status::OK();
}
vm::CellStorageStat storage_stat;
storage_stat.limit_cells = size_limits.max_acc_state_cells;
storage_stat.limit_bits = size_limits.max_acc_state_bits;
AccountStorageStat storage_stat;
if (update_storage_stat && account.account_storage_stat) {
storage_stat = account.account_storage_stat.value();
}
{
TD_PERF_COUNTER(transaction_storage_stat_a);
td::Timer timer;
auto add_used_storage = [&](const td::Ref<vm::Cell>& cell) -> td::Status {
if (cell.not_null()) {
TRY_RESULT(res, storage_stat.add_used_storage(cell));
if (res.max_merkle_depth > max_allowed_merkle_depth) {
return td::Status::Error("too big merkle depth");
TRY_RESULT(info, storage_stat.replace_roots({new_code, new_data, new_library}));
if (info.max_merkle_depth > max_allowed_merkle_depth) {
return td::Status::Error("too big Merkle depth");
}
}
return td::Status::OK();
};
TRY_STATUS(add_used_storage(new_code));
TRY_STATUS(add_used_storage(new_data));
TRY_STATUS(add_used_storage(new_library));
if (timer.elapsed() > 0.1) {
LOG(INFO) << "Compute used storage took " << timer.elapsed() << "s";
LOG(INFO) << "Compute used storage (1) took " << timer.elapsed() << "s";
}
}
if (acc_status == Account::acc_active) {
storage_stat.clear_limit();
} else {
storage_stat.clear();
if (storage_stat.get_total_cells() > size_limits.max_acc_state_cells ||
storage_stat.get_total_bits() > size_limits.max_acc_state_bits) {
return td::Status::Error(PSTRING() << "account state is too big: cells=" << storage_stat.get_total_cells()
<< ", bits=" << storage_stat.get_total_bits()
<< " (max cells=" << size_limits.max_acc_state_cells
<< ", max bits=" << size_limits.max_acc_state_bits << ")");
}
if (account.is_masterchain() && !cell_equal(account.library, new_library)) {
auto libraries_count = get_public_libraries_count(new_library);
if (libraries_count > size_limits.max_acc_public_libraries) {
return td::Status::Error(PSTRING() << "too many public libraries: " << libraries_count << " (max "
<< size_limits.max_acc_public_libraries << ")");
}
td::Status res;
if (storage_stat.cells > size_limits.max_acc_state_cells || storage_stat.bits > size_limits.max_acc_state_bits) {
res = td::Status::Error(PSTRING() << "account state is too big");
} else if (account.is_masterchain() && !cell_equal(account.library, new_library) &&
get_public_libraries_count(new_library) > size_limits.max_acc_public_libraries) {
res = td::Status::Error("too many public libraries");
} else {
res = td::Status::OK();
}
if (update_storage_stat) {
// storage_stat will be reused in compute_state()
new_storage_stat = std::move(storage_stat);
new_account_storage_stat.value_force() = std::move(storage_stat);
}
return res;
return td::Status::OK();
}
/**
@ -3140,44 +3131,6 @@ bool Account::store_acc_status(vm::CellBuilder& cb, int acc_status) const {
return cb.store_long_bool(v, 2);
}
/**
* Tries to update the storage statistics based on the old storage statistics and old account state without fully recomputing it.
*
* It succeeds if only root cell of AccountStorage is changed.
* old_cs and new_cell are AccountStorage without extra currencies (if global_version >= 10).
*
* @param old_stat The old storage statistics.
* @param old_cs The old AccountStorage.
* @param new_cell The new AccountStorage.
*
* @returns An optional value of type vm::CellStorageStat. If the update is successful, it returns the new storage statistics. Otherwise, it returns an empty optional.
*/
static td::optional<vm::CellStorageStat> try_update_storage_stat(const vm::CellStorageStat& old_stat,
td::Ref<vm::CellSlice> old_cs,
td::Ref<vm::Cell> new_cell) {
if (old_stat.cells == 0 || old_cs.is_null()) {
return {};
}
vm::CellSlice new_cs = vm::CellSlice(vm::NoVm(), new_cell);
if (old_cs->size_refs() != new_cs.size_refs()) {
return {};
}
for (unsigned i = 0; i < old_cs->size_refs(); ++i) {
if (old_cs->prefetch_ref(i)->get_hash() != new_cs.prefetch_ref(i)->get_hash()) {
return {};
}
}
if (old_stat.bits < old_cs->size()) {
return {};
}
vm::CellStorageStat new_stat;
new_stat.cells = old_stat.cells;
new_stat.bits = old_stat.bits - old_cs->size() + new_cs.size();
new_stat.public_cells = old_stat.public_cells;
return new_stat;
}
/**
* Removes extra currencies dict from AccountStorage.
*
@ -3185,9 +3138,9 @@ static td::optional<vm::CellStorageStat> try_update_storage_stat(const vm::CellS
*
* @param storage_cs AccountStorage as CellSlice.
*
* @returns AccountStorage without extra currencies as Cell.
* @returns AccountStorage without extra currencies as CellSlice.
*/
static td::Ref<vm::Cell> storage_without_extra_currencies(td::Ref<vm::CellSlice> storage_cs) {
static td::Ref<vm::CellSlice> storage_without_extra_currencies(td::Ref<vm::CellSlice> storage_cs) {
block::gen::AccountStorage::Record rec;
if (!block::gen::csr_unpack(storage_cs, rec)) {
LOG(ERROR) << "failed to unpack AccountStorage";
@ -3205,18 +3158,20 @@ static td::Ref<vm::Cell> storage_without_extra_currencies(td::Ref<vm::CellSlice>
return {};
}
}
td::Ref<vm::Cell> cell;
if (!block::gen::pack_cell(cell, rec)) {
td::Ref<vm::CellSlice> result;
if (!block::gen::csr_pack(result, rec)) {
LOG(ERROR) << "failed to pack AccountStorage";
return {};
}
return cell;
return result;
}
namespace transaction {
/**
* Computes the new state of the account.
*
* @param cfg The configuration for the serialization phase.
*
* @returns True if the state computation is successful, false otherwise.
*/
bool Transaction::compute_state(const SerializeConfig& cfg) {
@ -3290,37 +3245,74 @@ bool Transaction::compute_state(const SerializeConfig& cfg) {
} else {
new_inner_state.clear();
}
vm::CellStorageStat& stats = new_storage_stat;
td::Ref<vm::CellSlice> old_storage_for_stat = account.storage;
td::Ref<vm::Cell> new_storage_for_stat = storage;
td::Ref<vm::CellSlice> new_storage_for_stat = new_storage;
if (cfg.extra_currency_v2) {
new_storage_for_stat = storage_without_extra_currencies(new_storage);
if (new_storage_for_stat.is_null()) {
return false;
}
if (old_storage_for_stat.not_null()) {
old_storage_for_stat = vm::load_cell_slice_ref(storage_without_extra_currencies(old_storage_for_stat));
old_storage_for_stat = storage_without_extra_currencies(old_storage_for_stat);
if (old_storage_for_stat.is_null()) {
return false;
}
}
} else if (cfg.store_storage_dict_hash) {
LOG(ERROR) << "unsupported store_storage_dict_hash=true, extra_currency_v2=false";
return false;
}
auto new_stats = try_update_storage_stat(account.storage_stat, old_storage_for_stat, storage);
if (new_stats) {
stats = new_stats.unwrap();
bool storage_refs_changed = false;
if (old_storage_for_stat.is_null() || new_storage_for_stat->size_refs() != old_storage_for_stat->size_refs()) {
storage_refs_changed = true;
} else {
for (unsigned i = 0; i < new_storage_for_stat->size_refs(); i++) {
if (new_storage_for_stat->prefetch_ref(i)->get_hash() != old_storage_for_stat->prefetch_ref(i)->get_hash()) {
storage_refs_changed = true;
break;
}
}
}
if (storage_refs_changed || (cfg.store_storage_dict_hash && !account.storage_dict_hash)) {
TD_PERF_COUNTER(transaction_storage_stat_b);
td::Timer timer;
stats.add_used_storage(new_storage_for_stat).ensure();
if (!new_account_storage_stat && account.account_storage_stat) {
new_account_storage_stat = account.account_storage_stat;
}
AccountStorageStat& stats = new_account_storage_stat.value_force();
// Don't check Merkle depth and size here - they were checked in check_state_limits
auto S = stats.replace_roots(new_storage->prefetch_all_refs()).move_as_status();
if (S.is_error()) {
LOG(ERROR) << "Cannot recompute storage stats for account " << account.addr.to_hex() << ": " << S.move_as_error();
return false;
}
new_storage_dict_hash = stats.get_dict_hash();
// Root of AccountStorage is not counted in AccountStorageStat
new_storage_used.cells = stats.get_total_cells() + 1;
new_storage_used.bits = stats.get_total_bits() + new_storage->size();
if (timer.elapsed() > 0.1) {
LOG(INFO) << "Compute used storage took " << timer.elapsed() << "s";
LOG(INFO) << "Compute used storage (2) took " << timer.elapsed() << "s";
}
} else {
new_storage_used = account.storage_used;
new_storage_used.bits -= old_storage_for_stat->size();
new_storage_used.bits += new_storage_for_stat->size();
new_storage_dict_hash = account.storage_dict_hash;
new_account_storage_stat = account.account_storage_stat;
}
if (!cfg.store_storage_dict_hash) {
new_storage_dict_hash = {};
}
CHECK(cb.store_long_bool(1, 1) // account$1
&& cb.append_cellslice_bool(account.my_addr) // addr:MsgAddressInt
&& block::store_UInt7(cb, stats.cells) // storage_used$_ cells:(VarUInteger 7)
&& block::store_UInt7(cb, stats.bits) // bits:(VarUInteger 7)
&& block::store_UInt7(cb, stats.public_cells) // public_cells:(VarUInteger 7)
&& block::store_UInt7(cb, new_storage_used.cells) // storage_used$_ cells:(VarUInteger 7)
&& block::store_UInt7(cb, new_storage_used.bits) // bits:(VarUInteger 7)
&& cb.store_long_bool(new_storage_dict_hash ? 1 : 0, 3) // extra:StorageExtraInfo
&& (!new_storage_dict_hash || cb.store_bits_bool(new_storage_dict_hash.value())) // dict_hash:uint256
&& cb.store_long_bool(last_paid, 32)); // last_paid:uint32
if (due_payment.not_null() && td::sgn(due_payment) != 0) {
CHECK(cb.store_long_bool(1, 1) && block::tlb::t_Grams.store_integer_ref(cb, due_payment));
@ -3328,7 +3320,7 @@ bool Transaction::compute_state(const SerializeConfig& cfg) {
} else {
CHECK(cb.store_long_bool(0, 1));
}
CHECK(cb.append_data_cell_bool(std::move(storage)));
CHECK(cb.append_cellslice_bool(new_storage));
new_total_state = cb.finalize();
if (verbosity > 2) {
FLOG(INFO) {
@ -3345,6 +3337,8 @@ bool Transaction::compute_state(const SerializeConfig& cfg) {
*
* Updates root.
*
* @param cfg The configuration for the serialization.
*
* @returns True if the serialization is successful, False otherwise.
*/
bool Transaction::serialize(const SerializeConfig& cfg) {
@ -3688,7 +3682,9 @@ Ref<vm::Cell> Transaction::commit(Account& acc) {
acc.last_trans_end_lt_ = end_lt;
acc.last_trans_hash_ = root->get_hash().bits();
acc.last_paid = last_paid;
acc.storage_stat = new_storage_stat;
acc.storage_used = new_storage_used;
acc.account_storage_stat = std::move(new_account_storage_stat);
acc.storage_dict_hash = new_storage_dict_hash;
acc.storage = new_storage;
acc.balance = std::move(balance);
acc.due_payment = std::move(due_payment);
@ -3936,6 +3932,7 @@ td::Status FetchConfigParams::fetch_config_params(
}
{
serialize_cfg->extra_currency_v2 = config.get_global_version() >= 10;
serialize_cfg->store_storage_dict_hash = config.get_global_version() >= 11;
}
{
// fetch block_grams_created

View file

@ -17,6 +17,7 @@
Copyright 2017-2020 Telegram Systems LLP
*/
#pragma once
#include "account-storage-stat.h"
#include "common/refcnt.hpp"
#include "common/refint.h"
#include "vm/cells.h"
@ -179,6 +180,7 @@ struct ActionPhaseConfig {
struct SerializeConfig {
bool extra_currency_v2{false};
bool store_storage_dict_hash{false};
};
struct CreditPhase {
@ -266,8 +268,12 @@ struct Account {
ton::LogicalTime last_trans_lt_;
ton::Bits256 last_trans_hash_;
ton::LogicalTime block_lt;
ton::UnixTime last_paid;
vm::CellStorageStat storage_stat;
StorageUsed storage_used;
td::optional<td::Bits256> storage_dict_hash;
td::optional<AccountStorageStat> account_storage_stat;
block::CurrencyCollection balance;
td::RefInt256 due_payment;
Ref<vm::Cell> orig_total_state; // ^Account
@ -377,7 +383,9 @@ struct Transaction {
std::unique_ptr<ComputePhase> compute_phase;
std::unique_ptr<ActionPhase> action_phase;
std::unique_ptr<BouncePhase> bounce_phase;
vm::CellStorageStat new_storage_stat;
StorageUsed new_storage_used;
td::optional<AccountStorageStat> new_account_storage_stat;
td::optional<td::Bits256> new_storage_dict_hash;
bool gas_limit_overridden{false};
Transaction(const Account& _account, int ttype, ton::LogicalTime req_start_lt, ton::UnixTime _now,
Ref<vm::Cell> _inmsg = {});

View file

@ -113,21 +113,19 @@ class NewCellStorageStat {
struct CellStorageStat {
unsigned long long cells;
unsigned long long bits;
unsigned long long public_cells;
struct CellInfo {
td::uint32 max_merkle_depth = 0;
};
td::HashMap<vm::Cell::Hash, CellInfo> seen;
CellStorageStat() : cells(0), bits(0), public_cells(0) {
CellStorageStat() : cells(0), bits(0) {
}
explicit CellStorageStat(unsigned long long limit_cells)
: cells(0), bits(0), public_cells(0), limit_cells(limit_cells) {
explicit CellStorageStat(unsigned long long limit_cells) : cells(0), bits(0), limit_cells(limit_cells) {
}
void clear_seen() {
seen.clear();
}
void clear() {
cells = bits = public_cells = 0;
cells = bits = 0;
clear_limit();
clear_seen();
}

View file

@ -773,6 +773,14 @@ bool CellSlice::prefetch_maybe_ref(Ref<vm::Cell>& res) const {
}
}
std::vector<Ref<Cell>> CellSlice::prefetch_all_refs() const {
std::vector<Ref<Cell>> res(size_refs());
for (unsigned i = 0; i < size_refs(); ++i) {
res[i] = prefetch_ref(i);
}
return res;
}
bool CellSlice::fetch_maybe_ref(Ref<vm::Cell>& res) {
auto z = prefetch_ulong(1);
if (!z) {

View file

@ -190,6 +190,7 @@ class CellSlice : public td::CntObject {
}
bool fetch_maybe_ref(Ref<Cell>& ref);
bool prefetch_maybe_ref(Ref<Cell>& ref) const;
std::vector<Ref<Cell>> prefetch_all_refs() const;
td::BitSlice fetch_bits(unsigned bits);
td::BitSlice prefetch_bits(unsigned bits) const;
td::Ref<CellSlice> fetch_subslice(unsigned bits, unsigned refs = 0);

View file

@ -156,3 +156,10 @@ Example: if the last masterchain block seqno is `19071` then the list contains b
### TVM changes
- `SENDMSG` calculates messages size and fees without extra currencies, uses new +64 and +128 mode behavior.
- `SENDMSG` does not check the number of extra currencies.
## Version 11
### New account storage stat
Along with the storage stat (cells and bits count), each account now stores the hash of the **storage dict**.
**Storage dict** is the dictionary that stores refcnt for each cell in the account state.
This is required to help computing storage stats in the future, after collator-validator separation.

View file

@ -180,7 +180,7 @@ struct RawAccountState {
td::Ref<vm::Cell> extra_currencies;
ton::UnixTime storage_last_paid{0};
vm::CellStorageStat storage_stat;
block::StorageUsed storage_used;
td::Ref<vm::Cell> code;
td::Ref<vm::Cell> data;
@ -1036,7 +1036,7 @@ class Query {
TRY_RESULT(basechain_msg_prices, cfg->get_msg_prices(false));
block::MsgPrices* msg_prices[2] = {&basechain_msg_prices, &masterchain_msg_prices};
auto storage_fee_256 = block::StoragePrices::compute_storage_fees(
raw_.source->get_sync_time(), storage_prices, raw_.source->raw().storage_stat,
raw_.source->get_sync_time(), storage_prices, raw_.source->raw().storage_used,
raw_.source->raw().storage_last_paid, false, is_masterchain);
auto storage_fee = storage_fee_256.is_null() ? 0 : storage_fee_256->to_long();
@ -1085,7 +1085,7 @@ class Query {
TRY_RESULT(dest_gas_limits_prices, cfg->get_gas_limits_prices(dest_is_masterchain));
auto dest_storage_fee_256 =
destination ? block::StoragePrices::compute_storage_fees(
destination->get_sync_time(), storage_prices, destination->raw().storage_stat,
destination->get_sync_time(), storage_prices, destination->raw().storage_used,
destination->raw().storage_last_paid, false, is_masterchain)
: td::make_refint(0);
Fee dst_fee;
@ -1399,17 +1399,16 @@ class GetRawAccountState : public td::actor::Actor {
return td::Status::Error("Failed to unpack StorageInfo");
}
unsigned long long u = 0;
vm::CellStorageStat storage_stat;
block::StorageUsed storage_stat;
u |= storage_stat.cells = block::tlb::t_VarUInteger_7.as_uint(*storage_used.cells);
u |= storage_stat.bits = block::tlb::t_VarUInteger_7.as_uint(*storage_used.bits);
u |= storage_stat.public_cells = block::tlb::t_VarUInteger_7.as_uint(*storage_used.public_cells);
//LOG(DEBUG) << "last_paid=" << res.storage_last_paid << "; cells=" << storage_stat.cells
//<< " bits=" << storage_stat.bits << " public_cells=" << storage_stat.public_cells;
//<< " bits=" << storage_stat.bits;
if (u == std::numeric_limits<td::uint64>::max()) {
return td::Status::Error("Failed to unpack StorageStat");
}
res.storage_stat = storage_stat;
res.storage_used = storage_stat;
}
block::gen::AccountStorage::Record storage;
@ -2089,7 +2088,7 @@ class RunEmulator : public TonlibQueryActor {
raw.balance = balance.grams->to_long();
raw.extra_currencies = balance.extra;
raw.storage_last_paid = std::move(account.last_paid);
raw.storage_stat = std::move(account.storage_stat);
raw.storage_used = account.storage_used;
raw.code = std::move(account.code);
raw.data = std::move(account.data);
raw.state = std::move(account.total_state);

View file

@ -1008,6 +1008,7 @@ bool ValidateQuery::fetch_config_params() {
}
{
serialize_cfg_.extra_currency_v2 = config_->get_global_version() >= 10;
serialize_cfg_.store_storage_dict_hash = config_->get_global_version() >= 11;
}
{
// fetch block_grams_created