external/ton
1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-03-09 15:40:10 +00:00
Code Issues Releases Wiki Activity

celldb in-memory mode (--celldb-in-memory option)

Browse source
This commit is contained in:
birydrad 2024-09-09 18:08:15 +02:00
parent 420029b056
commit 1723562748
48 changed files with 1966 additions and 201 deletions
Download patch file Download diff file Expand all files Collapse all files

77
crypto/vm/boc.cpp
View file

@ -183,6 +183,9 @@ int BagOfCells::add_root(td::Ref<vm::Cell> add_root) {
// Changes in this function may require corresponding changes in crypto/vm/large-boc-serializer.cpp
td::Status BagOfCells::import_cells() {
if (logger_ptr_) {
logger_ptr_->start_stage("import_cells");
}
cells_clear();
for (auto& root : roots) {
auto res = import_cell(root.cell, 0);
@ -196,6 +199,9 @@ td::Status BagOfCells::import_cells() {
//LOG(INFO) << "[cells: " << cell_count << ", refs: " << int_refs << ", bytes: " << data_bytes
//<< ", internal hashes: " << int_hashes << ", top hashes: " << top_hashes << "]";
CHECK(cell_count != 0);
if (logger_ptr_) {
logger_ptr_->finish_stage(PSLICE() << cell_count << " cells");
}
return td::Status::OK();
}
@ -207,6 +213,9 @@ td::Result<int> BagOfCells::import_cell(td::Ref<vm::Cell> cell, int depth) {
if (cell.is_null()) {
return td::Status::Error("error while importing a cell into a bag of cells: cell is null");
}
if (logger_ptr_) {
TRY_STATUS(logger_ptr_->on_cell_processed());
}
auto it = cells.find(cell->get_hash());
if (it != cells.end()) {
auto pos = it->second;
@ -436,17 +445,19 @@ std::size_t BagOfCells::estimate_serialized_size(int mode) {
return res.ok();
}
BagOfCells& BagOfCells::serialize(int mode) {
td::Status BagOfCells::serialize(int mode) {
std::size_t size_est = estimate_serialized_size(mode);
if (!size_est) {
serialized.clear();
return *this;
return td::Status::OK();
}
serialized.resize(size_est);
if (serialize_to(const_cast<unsigned char*>(serialized.data()), serialized.size(), mode) != size_est) {
TRY_RESULT(size, serialize_to(const_cast<unsigned char*>(serialized.data()), serialized.size(), mode));
if (size != size_est) {
serialized.clear();
return td::Status::Error("serialization failed");
}
return *this;
return td::Status::OK();
}
std::string BagOfCells::serialize_to_string(int mode) {
@ -456,8 +467,8 @@ std::string BagOfCells::serialize_to_string(int mode) {
}
std::string res;
res.resize(size_est, 0);
if (serialize_to(const_cast<unsigned char*>(reinterpret_cast<const unsigned char*>(res.data())), res.size(), mode) ==
res.size()) {
if (serialize_to(const_cast<unsigned char*>(reinterpret_cast<const unsigned char*>(res.data())), res.size(), mode)
.move_as_ok() == res.size()) {
return res;
} else {
return {};
@ -470,8 +481,9 @@ td::Result<td::BufferSlice> BagOfCells::serialize_to_slice(int mode) {
return td::Status::Error("no cells to serialize to this bag of cells");
}
td::BufferSlice res(size_est);
if (serialize_to(const_cast<unsigned char*>(reinterpret_cast<const unsigned char*>(res.data())), res.size(), mode) ==
res.size()) {
TRY_RESULT(size, serialize_to(const_cast<unsigned char*>(reinterpret_cast<const unsigned char*>(res.data())),
res.size(), mode));
if (size == res.size()) {
return std::move(res);
} else {
return td::Status::Error("error while serializing a bag of cells: actual serialized size differs from estimated");
@ -494,14 +506,10 @@ std::string BagOfCells::extract_string() const {
// cell_data:(tot_cells_size * [ uint8 ])
// = BagOfCells;
// Changes in this function may require corresponding changes in crypto/vm/large-boc-serializer.cpp
template<typename WriterT>
std::size_t BagOfCells::serialize_to_impl(WriterT& writer, int mode) {
auto store_ref = [&](unsigned long long value) {
writer.store_uint(value, info.ref_byte_size);
};
auto store_offset = [&](unsigned long long value) {
writer.store_uint(value, info.offset_byte_size);
};
template <typename WriterT>
td::Result<std::size_t> BagOfCells::serialize_to_impl(WriterT& writer, int mode) {
auto store_ref = [&](unsigned long long value) { writer.store_uint(value, info.ref_byte_size); };
auto store_offset = [&](unsigned long long value) { writer.store_uint(value, info.offset_byte_size); };
writer.store_uint(info.magic, 4);
@ -536,6 +544,9 @@ std::size_t BagOfCells::serialize_to_impl(WriterT& writer, int mode) {
DCHECK((unsigned)cell_count == cell_list_.size());
if (info.has_index) {
std::size_t offs = 0;
if (logger_ptr_) {
logger_ptr_->start_stage("generate_index");
}
for (int i = cell_count - 1; i >= 0; --i) {
const Ref<DataCell>& dc = cell_list_[i].dc_ref;
bool with_hash = (mode & Mode::WithIntHashes) && !cell_list_[i].wt;
@ -548,11 +559,20 @@ std::size_t BagOfCells::serialize_to_impl(WriterT& writer, int mode) {
fixed_offset = offs * 2 + cell_list_[i].should_cache;
}
store_offset(fixed_offset);
if (logger_ptr_) {
TRY_STATUS(logger_ptr_->on_cell_processed());
}
}
if (logger_ptr_) {
logger_ptr_->finish_stage("");
}
DCHECK(offs == info.data_size);
}
DCHECK(writer.position() == info.data_offset);
size_t keep_position = writer.position();
if (logger_ptr_) {
logger_ptr_->start_stage("serialize");
}
for (int i = 0; i < cell_count; ++i) {
const auto& dc_info = cell_list_[cell_count - 1 - i];
const Ref<DataCell>& dc = dc_info.dc_ref;
@ -572,6 +592,9 @@ std::size_t BagOfCells::serialize_to_impl(WriterT& writer, int mode) {
// std::cerr << ' ' << k;
}
// std::cerr << std::endl;
if (logger_ptr_) {
TRY_STATUS(logger_ptr_->on_cell_processed());
}
}
writer.chk();
DCHECK(writer.position() - keep_position == info.data_size);
@ -580,11 +603,14 @@ std::size_t BagOfCells::serialize_to_impl(WriterT& writer, int mode) {
unsigned crc = writer.get_crc32();
writer.store_uint(td::bswap32(crc), 4);
}
if (logger_ptr_) {
logger_ptr_->finish_stage(PSLICE() << cell_count << " cells, " << writer.position() << " bytes");
}
DCHECK(writer.empty());
return writer.position();
}
std::size_t BagOfCells::serialize_to(unsigned char* buffer, std::size_t buff_size, int mode) {
td::Result<std::size_t> BagOfCells::serialize_to(unsigned char* buffer, std::size_t buff_size, int mode) {
std::size_t size_est = estimate_serialized_size(mode);
if (!size_est || size_est > buff_size) {
return 0;
@ -599,7 +625,7 @@ td::Status BagOfCells::serialize_to_file(td::FileFd& fd, int mode) {
return td::Status::Error("no cells to serialize to this bag of cells");
}
boc_writers::FileWriter writer{fd, size_est};
size_t s = serialize_to_impl(writer, mode);
TRY_RESULT(s, serialize_to_impl(writer, mode));
TRY_STATUS(writer.finalize());
if (s != size_est) {
return td::Status::Error("error while serializing a bag of cells: actual serialized size differs from estimated");
@ -1001,6 +1027,21 @@ td::Result<td::BufferSlice> std_boc_serialize_multi(std::vector<Ref<Cell>> roots
}
return boc.serialize_to_slice(mode);
}
td::Status std_boc_serialize_to_file(Ref<Cell> root, td::FileFd& fd, int mode,
td::CancellationToken cancellation_token) {
if (root.is_null()) {
return td::Status::Error("cannot serialize a null cell reference into a bag of cells");
}
td::Timer timer;
BagOfCellsLogger logger(std::move(cancellation_token));
BagOfCells boc;
boc.set_logger(&logger);
boc.add_root(std::move(root));
TRY_STATUS(boc.import_cells());
TRY_STATUS(boc.serialize_to_file(fd, mode));
LOG(ERROR) << "serialization took " << timer.elapsed() << "s";
return td::Status::OK();
}
/*
*