Merge branch 'refs/heads/testnet' into block-generation

Changelog.md

@@ -1,3 +1,17 @@
+## 2024.04 Update
+
+1. Emulator: Single call optimized runGetMethod added
+2. Tonlib: a series of proof improvements, also breaking Change in `liteServer.getAllShardsInfo` method (see below)
+3. DB: usage statistics now collected, outdated persistent states are not serialized
+4. LS: fast `getOutMsgQueueSizes` added, preliminary support of non-final block requests
+5. Network: lz4 compression of block candidates (disabled by default).
+6. Overlays: add custom overlays
+7. Transaction Executor: fixed issue with due_payment collection
+
+* `liteServer.getAllShardsInfo` method was updated for better efficiency. Previously, field proof contained BoC with two roots: one for BlockState from block's root and another for ShardHashes from BlockState. Now, it returns a single-root proof BoC, specifically the merkle proof of ShardHashes directly from the block's root, streamlining data access and integrity. Checking of the proof requires to check that ShardHashes in the `data` correspond to ShardHashes from the block.
+
+Besides the work of the core team, this update is based on the efforts of @akifoq (due_payment issue).
+
 ## 2024.03 Update
 
 1. Preparatory (not enabled yet) code for pre-compiled smart-contract.

Dockerfile

@@ -1,6 +1,6 @@
 FROM ubuntu:22.04 as builder
 RUN apt-get update && \
-	DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gperf wget git ninja-build libsecp256k1-dev libsodium-dev libmicrohttpd-dev pkg-config autoconf automake libtool && \
+	DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gperf wget git ninja-build libsecp256k1-dev libsodium-dev libmicrohttpd-dev liblz4-dev pkg-config autoconf automake libtool && \
 	rm -rf /var/lib/apt/lists/*
 ENV CC clang
 ENV CXX clang++
@@ -19,7 +19,7 @@ RUN mkdir build && \
 
 FROM ubuntu:22.04
 RUN apt-get update && \
-    apt-get install -y wget libatomic1 openssl libsecp256k1-dev libsodium-dev libmicrohttpd-dev && \
+    apt-get install -y wget libatomic1 openssl libsecp256k1-dev libsodium-dev libmicrohttpd-dev liblz4-dev && \
     rm -rf /var/lib/apt/lists/*
 
 RUN mkdir -p /var/ton-work/db && \
@@ -36,4 +36,4 @@ WORKDIR /var/ton-work/db
 COPY ./docker/init.sh ./docker/control.template ./
 RUN chmod +x init.sh
 
-ENTRYPOINT ["/var/ton-work/db/init.sh"]
+ENTRYPOINT ["/var/ton-work/db/init.sh"]

recent_changelog.md

@@ -5,11 +5,9 @@
 3. DB: usage statistics now collected, outdated persistent states are not serialized
 4. LS: fast `getOutMsgQueueSizes` added, preliminary support of non-final block requests
 5. Network: lz4 compression of block candidates (disabled by default).
-
+6. Overlays: add custom overlays
-
+7. Transaction Executor: fixed issue with due_payment collection
-
----
 
 * `liteServer.getAllShardsInfo` method was updated for better efficiency. Previously, field proof contained BoC with two roots: one for BlockState from block's root and another for ShardHashes from BlockState. Now, it returns a single-root proof BoC, specifically the merkle proof of ShardHashes directly from the block's root, streamlining data access and integrity. Checking of the proof requires to check that ShardHashes in the `data` correspond to ShardHashes from the block.
 
-
+Besides the work of the core team, this update is based on the efforts of @akifoq (due_payment issue).

tonlib/tonlib/TonlibClient.cpp

@@ -1890,7 +1890,9 @@ class RunEmulator : public TonlibQueryActor {
     if (stopped_) {
       return;
     }
-    get_block_id([self = this](td::Result<FullBlockId>&& block_id) { self->set_block_id(std::move(block_id)); });
+    get_block_id([SelfId = actor_id(this)](td::Result<FullBlockId>&& block_id) {
+      td::actor::send_closure(SelfId, &RunEmulator::set_block_id, std::move(block_id));
+    });
   }
 
   void set_block_id(td::Result<FullBlockId>&& block_id) {
@@ -1899,8 +1901,12 @@ class RunEmulator : public TonlibQueryActor {
     } else {
       block_id_ = block_id.move_as_ok();
 
-      get_mc_state_root([self = this](td::Result<td::Ref<vm::Cell>>&& mc_state_root) { self->set_mc_state_root(std::move(mc_state_root)); });
+      get_mc_state_root([SelfId = actor_id(this)](td::Result<td::Ref<vm::Cell>>&& mc_state_root) {
-      get_account_state([self = this](td::Result<td::unique_ptr<AccountState>>&& state) { self->set_account_state(std::move(state)); });
+        td::actor::send_closure(SelfId, &RunEmulator::set_mc_state_root, std::move(mc_state_root));
+      });
+      get_account_state([SelfId = actor_id(this)](td::Result<td::unique_ptr<AccountState>>&& state) {
+        td::actor::send_closure(SelfId, &RunEmulator::set_account_state, std::move(state));
+      });
       check(get_transactions(0));
 
       inc();
@@ -1922,7 +1928,9 @@ class RunEmulator : public TonlibQueryActor {
     } else {
       account_state_ = account_state.move_as_ok();
       send_query(int_api::ScanAndLoadGlobalLibs{account_state_->get_raw_state()},
-                 [self = this](td::Result<vm::Dictionary> R) { self->set_global_libraries(std::move(R)); });
+                 [SelfId = actor_id(this)](td::Result<vm::Dictionary> R) {
+                   td::actor::send_closure(SelfId, &RunEmulator::set_global_libraries, std::move(R));
+                 });
     }
   }
 
@@ -5517,7 +5525,7 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_getShards& request,
                           }
 
                           block::ShardConfig sh_conf;
-                          if (!sh_conf.unpack(mc_extra.shard_hashes)) {
+                          if (!sh_conf.unpack(data_csr)) {
                             return td::Status::Error("cannot extract shard block list from shard configuration");
                           }
                           auto ids = sh_conf.get_shard_hash_ids(true);
@@ -5540,7 +5548,9 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_getShards& request,
   return td::Status::OK();
 }
 
-td::Status check_lookup_block_proof(lite_api_ptr<ton::lite_api::liteServer_lookupBlockResult>& result, int mode, ton::BlockId blkid, ton::BlockIdExt client_mc_blkid, td::uint64 lt, td::uint32 utime);
+td::Status check_lookup_block_proof(lite_api_ptr<ton::lite_api::liteServer_lookupBlockResult>& result, int mode,
+                                    ton::BlockId blkid, ton::BlockIdExt client_mc_blkid, td::uint64 lt,
+                                    td::uint32 utime);
 
 td::Status TonlibClient::do_request(const tonlib_api::blocks_lookupBlock& request,
                         td::Promise<object_ptr<tonlib_api::ton_blockIdExt>>&& promise) {
@@ -5726,7 +5736,7 @@ auto to_tonlib_api(const ton::lite_api::liteServer_transactionId& txid)
 
 td::Status check_block_transactions_proof(lite_api_ptr<ton::lite_api::liteServer_blockTransactions>& bTxes, int32_t mode,
     ton::LogicalTime start_lt, td::Bits256 start_addr, td::Bits256 root_hash, int req_count) {
-  if (mode & ton::lite_api::liteServer_listBlockTransactions::WANT_PROOF_MASK == 0) {
+  if ((mode & ton::lite_api::liteServer_listBlockTransactions::WANT_PROOF_MASK) == 0) {
     return td::Status::OK();
   }
   constexpr int max_answer_transactions = 256;

tonlib/tonlib/tonlib-cli.cpp

@@ -1418,7 +1418,8 @@ class TonlibCli : public td::actor::Actor {
                                if (r_cell.is_error()) {
                                  sb << "<INVALID_CELL>";
                                }
-                               auto cs = vm::load_cell_slice(r_cell.move_as_ok());
+                               bool spec = true;
+                               auto cs = vm::load_cell_slice_special(r_cell.move_as_ok(), spec);
                                std::stringstream ss;
                                cs.print_rec(ss);
                                sb << ss.str();

validator/manager.cpp

@@ -2598,9 +2598,9 @@ void ValidatorManagerImpl::state_serializer_update(BlockSeqno seqno) {
 void ValidatorManagerImpl::alarm() {
   try_advance_gc_masterchain_block();
   alarm_timestamp() = td::Timestamp::in(1.0);
-  if (last_masterchain_block_handle_ && gc_masterchain_handle_) {
+  if (shard_client_handle_ && gc_masterchain_handle_) {
-    td::actor::send_closure(db_, &Db::run_gc, last_masterchain_block_handle_->unix_time(),
+    td::actor::send_closure(db_, &Db::run_gc, shard_client_handle_->unix_time(), gc_masterchain_handle_->unix_time(),
-                            gc_masterchain_handle_->unix_time(), static_cast<UnixTime>(opts_->archive_ttl()));
+                            static_cast<UnixTime>(opts_->archive_ttl()));
   }
   if (log_status_at_.is_in_past()) {
     if (last_masterchain_block_handle_) {