mirror of
https://github.com/ton-blockchain/ton
synced 2025-03-09 15:40:10 +00:00
updated func/fift
- updated func/fift - updated liteclient/liteserver - bugfixes
This commit is contained in:
ton
parent
d41ce55305
commit
acf16718e6
45 changed files with 1360 additions and 185 deletions
|
|
@ -39,6 +39,7 @@
|
|||
#include "td/utils/port/path.h"
|
||||
#include "td/utils/format.h"
|
||||
#include "td/utils/misc.h"
|
||||
#include "td/utils/optional.h"
|
||||
#include "td/utils/tests.h"
|
||||
#include "td/utils/tl_parsers.h"
|
||||
#include "td/utils/tl_helpers.h"
|
||||
|
|
@ -529,16 +530,20 @@ TEST(Cell, MerkleProofCombine) {
|
|||
ASSERT_EQ(exploration_b.log, exploration2.log);
|
||||
}
|
||||
|
||||
Ref<Cell> proof_union;
|
||||
{
|
||||
proof_union = MerkleProof::combine(proof1, proof2);
|
||||
auto check = [&](auto proof_union) {
|
||||
auto virtualized_proof = MerkleProof::virtualize(proof_union, 1);
|
||||
auto exploration_a = CellExplorer::explore(virtualized_proof, exploration1.ops);
|
||||
auto exploration_b = CellExplorer::explore(virtualized_proof, exploration2.ops);
|
||||
ASSERT_EQ(exploration_a.log, exploration1.log);
|
||||
ASSERT_EQ(exploration_b.log, exploration2.log);
|
||||
};
|
||||
auto proof_union = MerkleProof::combine(proof1, proof2);
|
||||
ASSERT_EQ(proof_union->get_hash(), proof12->get_hash());
|
||||
check(proof_union);
|
||||
|
||||
auto virtualized_proof = MerkleProof::virtualize(proof_union, 1);
|
||||
auto exploration_a = CellExplorer::explore(virtualized_proof, exploration1.ops);
|
||||
auto exploration_b = CellExplorer::explore(virtualized_proof, exploration2.ops);
|
||||
ASSERT_EQ(exploration_a.log, exploration1.log);
|
||||
ASSERT_EQ(exploration_b.log, exploration2.log);
|
||||
auto proof_union_fast = MerkleProof::combine_fast(proof1, proof2);
|
||||
check(proof_union_fast);
|
||||
}
|
||||
{
|
||||
auto cell = MerkleProof::virtualize(proof12, 1);
|
||||
|
|
@ -1194,6 +1199,88 @@ TEST(Cell, MerkleProofArrayHands) {
|
|||
test_boc_deserializer_full(proof).ensure();
|
||||
test_boc_deserializer_full(CellBuilder::create_merkle_proof(proof)).ensure();
|
||||
}
|
||||
|
||||
TEST(Cell, MerkleProofCombineArray) {
|
||||
size_t n = 1 << 15;
|
||||
std::vector<td::uint64> data;
|
||||
for (size_t i = 0; i < n; i++) {
|
||||
data.push_back(i / 3);
|
||||
}
|
||||
CompactArray arr(data);
|
||||
|
||||
td::Ref<vm::Cell> root = vm::CellBuilder::create_merkle_proof(arr.merkle_proof({}));
|
||||
td::Timer timer;
|
||||
for (size_t i = 0; i < n; i++) {
|
||||
auto new_root = vm::CellBuilder::create_merkle_proof(arr.merkle_proof({i}));
|
||||
root = vm::MerkleProof::combine_fast(root, new_root);
|
||||
if ((i - 1) % 100 == 0) {
|
||||
LOG(ERROR) << timer;
|
||||
timer = {};
|
||||
}
|
||||
}
|
||||
|
||||
CompactArray arr2(n, vm::MerkleProof::virtualize(root, 1));
|
||||
for (size_t i = 0; i < n; i++) {
|
||||
CHECK(arr.get(i) == arr2.get(i));
|
||||
}
|
||||
}
|
||||
|
||||
TEST(Cell, MerkleProofCombineArray2) {
|
||||
auto a = vm::CellBuilder().store_long(1, 8).finalize();
|
||||
auto b = vm::CellBuilder().store_long(2, 8).finalize();
|
||||
auto c = vm::CellBuilder().store_long(3, 8).finalize();
|
||||
auto d = vm::CellBuilder().store_long(4, 8).finalize();
|
||||
auto left = vm::CellBuilder().store_ref(a).store_ref(b).finalize();
|
||||
auto right = vm::CellBuilder().store_ref(c).store_ref(d).finalize();
|
||||
auto x = vm::CellBuilder().store_ref(left).store_ref(right).finalize();
|
||||
size_t n = 18;
|
||||
//TODO: n = 100, currently TL
|
||||
for (size_t i = 0; i < n; i++) {
|
||||
x = vm::CellBuilder().store_ref(x).store_ref(x).finalize();
|
||||
}
|
||||
|
||||
td::Ref<vm::Cell> root;
|
||||
auto apply_op = [&](auto op) {
|
||||
auto usage_tree = std::make_shared<CellUsageTree>();
|
||||
auto usage_cell = UsageCell::create(x, usage_tree->root_ptr());
|
||||
root = usage_cell;
|
||||
op();
|
||||
return MerkleProof::generate(root, usage_tree.get());
|
||||
};
|
||||
|
||||
auto first = apply_op([&] {
|
||||
auto x = root;
|
||||
while (true) {
|
||||
auto cs = vm::load_cell_slice(x);
|
||||
if (cs.size_refs() == 0) {
|
||||
break;
|
||||
}
|
||||
x = cs.prefetch_ref(0);
|
||||
}
|
||||
});
|
||||
auto second = apply_op([&] {
|
||||
auto x = root;
|
||||
while (true) {
|
||||
auto cs = vm::load_cell_slice(x);
|
||||
if (cs.size_refs() == 0) {
|
||||
break;
|
||||
}
|
||||
x = cs.prefetch_ref(1);
|
||||
}
|
||||
});
|
||||
|
||||
{
|
||||
td::Timer t;
|
||||
auto x = vm::MerkleProof::combine(first, second);
|
||||
LOG(ERROR) << "slow " << t;
|
||||
}
|
||||
{
|
||||
td::Timer t;
|
||||
auto x = vm::MerkleProof::combine_fast(first, second);
|
||||
LOG(ERROR) << "fast " << t;
|
||||
}
|
||||
}
|
||||
|
||||
TEST(Cell, MerkleUpdateHands) {
|
||||
auto data = CellBuilder{}.store_bytes("pruned data").store_ref(CellBuilder{}.finalize()).finalize();
|
||||
auto node = CellBuilder{}.store_bytes("protected data").store_ref(data).finalize();
|
||||
|
|
@ -1965,6 +2052,222 @@ TEST(Ref, AtomicRef) {
|
|||
LOG(ERROR) << String::total_strings.sum();
|
||||
}
|
||||
|
||||
class FileMerkleTree {
|
||||
public:
|
||||
FileMerkleTree(size_t chunks_count, td::Ref<vm::Cell> root = {}) {
|
||||
log_n_ = 0;
|
||||
while ((size_t(1) << log_n_) < chunks_count) {
|
||||
log_n_++;
|
||||
}
|
||||
n_ = size_t(1) << log_n_;
|
||||
mark_.resize(n_ * 2);
|
||||
proof_.resize(n_ * 2);
|
||||
|
||||
CHECK(n_ == chunks_count); // TODO: support other chunks_count
|
||||
//auto x = vm::CellBuilder().finalize();
|
||||
root_ = std::move(root);
|
||||
}
|
||||
|
||||
struct Chunk {
|
||||
td::size_t index{0};
|
||||
td::Slice hash;
|
||||
};
|
||||
|
||||
void remove_chunk(td::size_t index) {
|
||||
CHECK(index < n_);
|
||||
index += n_;
|
||||
while (proof_[index].not_null()) {
|
||||
proof_[index] = {};
|
||||
index /= 2;
|
||||
}
|
||||
}
|
||||
|
||||
bool has_chunk(td::size_t index) const {
|
||||
CHECK(index < n_);
|
||||
index += n_;
|
||||
return proof_[index].not_null();
|
||||
}
|
||||
|
||||
void add_chunk(td::size_t index, td::Slice hash) {
|
||||
CHECK(hash.size() == 32);
|
||||
CHECK(index < n_);
|
||||
index += n_;
|
||||
auto cell = vm::CellBuilder().store_bytes(hash).finalize();
|
||||
CHECK(proof_[index].is_null());
|
||||
proof_[index] = std::move(cell);
|
||||
for (index /= 2; index != 0; index /= 2) {
|
||||
CHECK(proof_[index].is_null());
|
||||
auto &left = proof_[index * 2];
|
||||
auto &right = proof_[index * 2 + 1];
|
||||
if (left.not_null() && right.not_null()) {
|
||||
proof_[index] = vm::CellBuilder().store_ref(left).store_ref(right).finalize();
|
||||
} else {
|
||||
mark_[index] = mark_id_;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
td::Status validate_proof(td::Ref<vm::Cell> new_root) {
|
||||
// TODO: check structure
|
||||
return td::Status::OK();
|
||||
}
|
||||
|
||||
td::Status add_proof(td::Ref<vm::Cell> new_root) {
|
||||
TRY_STATUS(validate_proof(new_root));
|
||||
auto combined = vm::MerkleProof::combine_fast_raw(root_, new_root);
|
||||
if (combined.is_null()) {
|
||||
return td::Status::Error("Can't combine proofs");
|
||||
}
|
||||
root_ = std::move(combined);
|
||||
return td::Status::OK();
|
||||
}
|
||||
|
||||
td::Status try_add_chunks(td::Span<Chunk> chunks) {
|
||||
for (auto chunk : chunks) {
|
||||
if (has_chunk(chunk.index)) {
|
||||
return td::Status::Error("Already has chunk");
|
||||
}
|
||||
}
|
||||
mark_id_++;
|
||||
for (auto chunk : chunks) {
|
||||
add_chunk(chunk.index, chunk.hash);
|
||||
}
|
||||
auto r_new_root = merge(root_, 1);
|
||||
if (r_new_root.is_error()) {
|
||||
for (auto chunk : chunks) {
|
||||
remove_chunk(chunk.index);
|
||||
}
|
||||
return r_new_root.move_as_error();
|
||||
}
|
||||
root_ = r_new_root.move_as_ok();
|
||||
return td::Status::OK();
|
||||
}
|
||||
|
||||
td::Result<td::Ref<vm::Cell>> merge(td::Ref<vm::Cell> root, size_t index) {
|
||||
const auto &down = proof_[index];
|
||||
if (down.not_null()) {
|
||||
if (down->get_hash() != root->get_hash(0)) {
|
||||
return td::Status::Error("Hash mismatch");
|
||||
}
|
||||
return down;
|
||||
}
|
||||
|
||||
if (mark_[index] != mark_id_) {
|
||||
return root;
|
||||
}
|
||||
|
||||
vm::CellSlice cs(vm::NoVm(), root);
|
||||
if (cs.is_special()) {
|
||||
return td::Status::Error("Proof is not enough to validate chunks");
|
||||
}
|
||||
|
||||
CHECK(cs.size_refs() == 2);
|
||||
vm::CellBuilder cb;
|
||||
cb.store_bits(cs.fetch_bits(cs.size()));
|
||||
TRY_RESULT(left, merge(cs.fetch_ref(), index * 2));
|
||||
TRY_RESULT(right, merge(cs.fetch_ref(), index * 2 + 1));
|
||||
cb.store_ref(std::move(left)).store_ref(std::move(right));
|
||||
return cb.finalize();
|
||||
}
|
||||
|
||||
void init_proof() {
|
||||
CHECK(proof_[1].not_null());
|
||||
root_ = proof_[1];
|
||||
}
|
||||
|
||||
td::Result<td::Ref<vm::Cell>> gen_proof(size_t l, size_t r) {
|
||||
auto usage_tree = std::make_shared<vm::CellUsageTree>();
|
||||
auto usage_cell = vm::UsageCell::create(root_, usage_tree->root_ptr());
|
||||
TRY_STATUS(do_gen_proof(std::move(usage_cell), 0, n_ - 1, l, r));
|
||||
auto res = vm::MerkleProof::generate_raw(root_, usage_tree.get());
|
||||
CHECK(res.not_null());
|
||||
return res;
|
||||
}
|
||||
|
||||
private:
|
||||
td::size_t n_; // n = 2^log_n
|
||||
td::size_t log_n_;
|
||||
td::size_t mark_id_{0};
|
||||
std::vector<td::size_t> mark_; // n_ * 2
|
||||
std::vector<td::Ref<vm::Cell>> proof_; // n_ * 2
|
||||
td::Ref<vm::Cell> root_;
|
||||
|
||||
td::Status do_gen_proof(td::Ref<vm::Cell> node, size_t il, size_t ir, size_t l, size_t r) {
|
||||
if (ir < l || il > r) {
|
||||
return td::Status::OK();
|
||||
}
|
||||
if (l <= il && ir <= r) {
|
||||
return td::Status::OK();
|
||||
}
|
||||
vm::CellSlice cs(vm::NoVm(), std::move(node));
|
||||
if (cs.is_special()) {
|
||||
return td::Status::Error("Can't generate a proof");
|
||||
}
|
||||
CHECK(cs.size_refs() == 2);
|
||||
auto ic = (il + ir) / 2;
|
||||
TRY_STATUS(do_gen_proof(cs.fetch_ref(), il, ic, l, r));
|
||||
TRY_STATUS(do_gen_proof(cs.fetch_ref(), ic + 1, ir, l, r));
|
||||
return td::Status::OK();
|
||||
}
|
||||
};
|
||||
|
||||
TEST(FileMerkleTree, Manual) {
|
||||
// create big random file
|
||||
size_t chunk_size = 768;
|
||||
// for simplicity numer of chunks in a file is a power of two
|
||||
size_t chunks_count = 1 << 16;
|
||||
size_t file_size = chunk_size * chunks_count;
|
||||
td::Timer timer;
|
||||
LOG(INFO) << "Generate random string";
|
||||
const auto file = td::rand_string('a', 'z', td::narrow_cast<int>(file_size));
|
||||
LOG(INFO) << timer;
|
||||
|
||||
timer = {};
|
||||
LOG(INFO) << "Calculate all hashes";
|
||||
std::vector<td::UInt256> hashes(chunks_count);
|
||||
for (size_t i = 0; i < chunks_count; i++) {
|
||||
td::sha256(td::Slice(file).substr(i * chunk_size, chunk_size), hashes[i].as_slice());
|
||||
}
|
||||
LOG(INFO) << timer;
|
||||
|
||||
timer = {};
|
||||
LOG(INFO) << "Init merkle tree";
|
||||
FileMerkleTree tree(chunks_count);
|
||||
for (size_t i = 0; i < chunks_count; i++) {
|
||||
tree.add_chunk(i, hashes[i].as_slice());
|
||||
}
|
||||
tree.init_proof();
|
||||
LOG(INFO) << timer;
|
||||
|
||||
auto root_proof = tree.gen_proof(0, chunks_count - 1).move_as_ok();
|
||||
|
||||
// first download each chunk one by one
|
||||
|
||||
for (size_t stride : {1 << 6, 1}) {
|
||||
timer = {};
|
||||
LOG(INFO) << "Gen all proofs, stride = " << stride;
|
||||
for (size_t i = 0; i < chunks_count; i += stride) {
|
||||
tree.gen_proof(i, i + stride - 1).move_as_ok();
|
||||
}
|
||||
LOG(INFO) << timer;
|
||||
timer = {};
|
||||
LOG(INFO) << "Proof size: " << vm::std_boc_serialize(tree.gen_proof(0, stride - 1).move_as_ok()).ok().size();
|
||||
LOG(INFO) << "Download file, stride = " << stride;
|
||||
{
|
||||
FileMerkleTree new_tree(chunks_count, root_proof);
|
||||
for (size_t i = 0; i < chunks_count; i += stride) {
|
||||
new_tree.add_proof(tree.gen_proof(i, i + stride - 1).move_as_ok()).ensure();
|
||||
std::vector<FileMerkleTree::Chunk> chunks;
|
||||
for (size_t j = 0; j < stride; j++) {
|
||||
chunks.push_back({i + j, hashes[i + j].as_slice()});
|
||||
}
|
||||
new_tree.try_add_chunks(chunks).ensure();
|
||||
}
|
||||
}
|
||||
LOG(INFO) << timer;
|
||||
}
|
||||
}
|
||||
|
||||
//TEST(Tmp, Boc) {
|
||||
//LOG(ERROR) << "A";
|
||||
//auto data = td::read_file("boc");
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue