Merge pull request #877 from ton-blockchain/testnet

Merge developer branch

.github/workflows/ton-x86-64-linux.yml

@@ -24,7 +24,7 @@ jobs:
       run: |
         cp assembly/nix/build-linux-x86-64-nix.sh .
         chmod +x build-linux-x86-64-nix.sh
-        ./build-linux-x86-64-nix.sh
+        ./build-linux-x86-64-nix.sh -t
 
     - name: Simple binaries test
       run: |
@@ -38,4 +38,4 @@ jobs:
       uses: actions/upload-artifact@master
       with:
         name: ton-x86_64-linux-binaries
-        path: artifacts
+        path: artifacts

.github/workflows/ton-x86-64-macos.yml

@@ -20,7 +20,7 @@ jobs:
       run: |
         cp assembly/nix/build-macos-nix.sh .
         chmod +x build-macos-nix.sh
-        ./build-macos-nix.sh
+        ./build-macos-nix.sh -t
 
     - name: Simple binaries test
       run: |
@@ -34,4 +34,4 @@ jobs:
       uses: actions/upload-artifact@master
       with:
         name: ton-x86_64-macos-binaries
-        path: artifacts
+        path: artifacts

Changelog.md

@@ -1,14 +1,20 @@
 ## 2024.01 Update
 
-1. Fixes in how gas in transactions on special accounts is accounted in block limit. Previously, gas was counted as usual, so to conduct elections that costs >30m gas block limit in masterchain was set to 37m gas. To lower the limit for safety reasons it is proposed to not count gas on special accounts. Besides `gas_max` is set to `special_gas_limit` for all types of transactions on special accounts. New behavior is activated through setting `gas_prices_v3` in `ConfigParam 20;`.
+1. Fixes in how gas in transactions on special accounts is accounted in block limit. Previously, gas was counted as usual, so to conduct elections that costs >30m gas block limit in masterchain was set to 37m gas. To lower the limit for safety reasons it is proposed to caunt gas on special accounts separately. Besides `gas_max` is set to `special_gas_limit` for all types of transactions on special accounts. New behavior is activated through setting `version >= 5` in `ConfigParam 8;`.
    * Besides update of config temporally increases gas limit on `EQD_v9j1rlsuHHw2FIhcsCFFSD367ldfDdCKcsNmNpIRzUlu` to `special_gas_limit`, see [details](https://t.me/tonstatus/88).
 2. Improvements in LS behavior
    * Improved detection of the state with all shards applied to decrease rate of `Block is not applied` error
    * Better error logs: `block not in db` and `block is not applied` separation
    * Fix error in proof generation for blocks after merge
+   * Fix most of `block is not applied` issues related to sending too recent block in Proofs
+   * LS now check external messages till `accept_message` (`set_gas`).
 3. Improvements in DHT work and storage, CellDb, config.json ammendment, peer misbehavior detection, validator session stats collection, emulator.
+4. Change in CTOS and XLOAD behavior activated through setting `version >= 5` in `ConfigParam 8;`:
+   * Loading "nested libraries" (i.e. a library cell that points to another library cell) throws an exception.
+   * Loading a library consumes gas for cell load only once (for the library cell), not twice (both for the library cell and the cell in the library).
+   * `XLOAD` now works differently. When it takes a library cell, it returns the cell that it points to. This allows loading "nested libraries", if needed.
 
-Besides the work of the core team, this update is based on the efforts of @XaBbl4 (peer misbehavior detection).
+Besides the work of the Core team, this update is based on the efforts of @XaBbl4 (peer misbehavior detection) and @akifoq (CTOS behavior and gas limit scheme for special accounts).
 
 ## 2023.12 Update
 

assembly/cicd/jenkins/test-builds.groovy

@@ -31,7 +31,7 @@ pipeline {
                             sh '''
                                 cp assembly/nix/build-linux-x86-64-nix.sh .
                                 chmod +x build-linux-x86-64-nix.sh
-                                ./build-linux-x86-64-nix.sh
+                                ./build-linux-x86-64-nix.sh -t
                             '''
                             sh '''
                                 cd artifacts
@@ -69,7 +69,7 @@ pipeline {
                             sh '''
                                 cp assembly/nix/build-linux-arm64-nix.sh .
                                 chmod +x build-linux-arm64-nix.sh
-                                ./build-linux-arm64-nix.sh
+                                ./build-linux-arm64-nix.sh -t
                             '''
                             sh '''
                                 cd artifacts
@@ -107,7 +107,7 @@ pipeline {
                             sh '''
                                 cp assembly/nix/build-macos-nix.sh .
                                 chmod +x build-macos-nix.sh
-                                ./build-macos-nix.sh
+                                ./build-macos-nix.sh -t
                             '''
                             sh '''
                                 cd artifacts
@@ -145,7 +145,7 @@ pipeline {
                             sh '''
                                 cp assembly/nix/build-macos-nix.sh .
                                 chmod +x build-macos-nix.sh
-                                ./build-macos-nix.sh
+                                ./build-macos-nix.sh -t
                             '''
                             sh '''
                                 cd artifacts
@@ -233,4 +233,4 @@ pipeline {
             }
         }
     }
-}
+}

assembly/nix/build-linux-arm64-nix.sh

@@ -3,12 +3,27 @@
 nix-build --version
 test $? -eq 0 || { echo "Nix is not installed!"; exit 1; }
 
+with_tests=false
+
+
+while getopts 't' flag; do
+  case "${flag}" in
+    t) with_tests=true ;;
+    *) break
+       ;;
+  esac
+done
+
 cp assembly/nix/linux-arm64* .
 cp assembly/nix/microhttpd.nix .
 cp assembly/nix/openssl.nix .
 export NIX_PATH=nixpkgs=https://github.com/nixOS/nixpkgs/archive/23.05.tar.gz
 
-nix-build linux-arm64-static.nix
+if [ "$with_tests" = true ]; then
+  nix-build linux-arm64-static.nix --arg testing true
+else
+  nix-build linux-arm64-static.nix
+fi
 mkdir artifacts
 cp ./result/bin/* artifacts/
 chmod +x artifacts/*
@@ -17,4 +32,4 @@ nix-build linux-arm64-tonlib.nix
 cp ./result/lib/libtonlibjson.so.0.5 artifacts/libtonlibjson.so
 cp ./result/lib/libemulator.so artifacts/
 cp -r crypto/fift/lib artifacts/
-cp -r crypto/smartcont artifacts/
+cp -r crypto/smartcont artifacts/

assembly/nix/build-linux-x86-64-nix.sh

@@ -3,12 +3,28 @@
 nix-build --version
 test $? -eq 0 || { echo "Nix is not installed!"; exit 1; }
 
+with_tests=false
+
+
+while getopts 't' flag; do
+  case "${flag}" in
+    t) with_tests=true ;;
+    *) break
+       ;;
+  esac
+done
+
 cp assembly/nix/linux-x86-64* .
 cp assembly/nix/microhttpd.nix .
 cp assembly/nix/openssl.nix .
 export NIX_PATH=nixpkgs=https://github.com/nixOS/nixpkgs/archive/23.05.tar.gz
 
-nix-build linux-x86-64-static.nix
+if [ "$with_tests" = true ]; then
+  nix-build linux-x86-64-static.nix --arg testing true
+else
+  nix-build linux-x86-64-static.nix
+fi
+
 mkdir artifacts
 cp ./result/bin/* artifacts/
 chmod +x artifacts/*
@@ -17,4 +33,4 @@ nix-build linux-x86-64-tonlib.nix
 cp ./result/lib/libtonlibjson.so.0.5 artifacts/libtonlibjson.so
 cp ./result/lib/libemulator.so artifacts/
 cp -r crypto/fift/lib artifacts/
-cp -r crypto/smartcont artifacts/
+cp -r crypto/smartcont artifacts/

assembly/nix/build-macos-nix.sh

@@ -3,9 +3,25 @@
 nix-build --version
 test $? -eq 0 || { echo "Nix is not installed!"; exit 1; }
 
+with_tests=false
+
+
+while getopts 't' flag; do
+  case "${flag}" in
+    t) with_tests=true ;;
+    *) break
+       ;;
+  esac
+done
+
 cp assembly/nix/macos-* .
 export NIX_PATH=nixpkgs=https://github.com/nixOS/nixpkgs/archive/23.05.tar.gz
-nix-build macos-static.nix
+
+if [ "$with_tests" = true ]; then
+  nix-build macos-static.nix --arg testing true
+else
+  nix-build macos-static.nix
+fi
 mkdir artifacts
 cp ./result-bin/bin/* artifacts/
 chmod +x artifacts/*
@@ -14,4 +30,4 @@ nix-build macos-tonlib.nix
 cp ./result/lib/libtonlibjson.dylib artifacts/
 cp ./result/lib/libemulator.dylib artifacts/
 cp -r crypto/fift/lib artifacts/
-cp -r crypto/smartcont artifacts/
+cp -r crypto/smartcont artifacts/

assembly/nix/linux-arm64-static.nix

@@ -3,6 +3,7 @@
 { pkgs ? import <nixpkgs> { system = builtins.currentSystem; }
 , lib ? pkgs.lib
 , stdenv ? pkgs.stdenv
+, testing ? false
 }:
 let
   microhttpdmy = (import ./microhttpd.nix) {};
@@ -25,7 +26,7 @@ stdenv.mkDerivation {
     ];
 
   makeStatic = true;
-  doCheck = true;
+  doCheck = testing;
 
   cmakeFlags = [
     "-DTON_USE_ABSEIL=OFF"

assembly/nix/linux-x86-64-static.nix

@@ -3,6 +3,7 @@
 { pkgs ? import <nixpkgs> { system = builtins.currentSystem; }
 , lib ? pkgs.lib
 , stdenv ? pkgs.stdenv
+, testing ? false
 }:
 let
   microhttpdmy = (import ./microhttpd.nix) {};
@@ -25,7 +26,7 @@ stdenv.mkDerivation {
     ];
 
   makeStatic = true;
-  doCheck = true;
+  doCheck = testing;
 
   cmakeFlags = [
     "-DTON_USE_ABSEIL=OFF"

assembly/nix/macos-static.nix

@@ -3,6 +3,7 @@
 { pkgs ? import <nixpkgs> { system = builtins.currentSystem; }
 , lib ? pkgs.lib
 , stdenv ? pkgs.stdenv
+, testing ? false
 }:
 
 pkgs.llvmPackages_14.stdenv.mkDerivation {
@@ -29,7 +30,7 @@ pkgs.llvmPackages_14.stdenv.mkDerivation {
 
   dontAddStaticConfigureFlags = true;
   makeStatic = true;
-  doCheck = true;
+  doCheck = testing;
 
   configureFlags = [];
 
@@ -62,4 +63,4 @@ pkgs.llvmPackages_14.stdenv.mkDerivation {
       done
   '';
   outputs = [ "bin" "out" ];
-}
+}

common/global-version.h

@@ -19,6 +19,6 @@
 namespace ton {
 
 // See doc/GlobalVersions.md
-const int SUPPORTED_VERSION = 4;
+const int SUPPORTED_VERSION = 5;
 
 }

crypto/block/block.tlb

@@ -696,11 +696,6 @@ gas_prices_ext#de gas_price:uint64 gas_limit:uint64 special_gas_limit:uint64 gas
   block_gas_limit:uint64 freeze_due_limit:uint64 delete_due_limit:uint64 
   = GasLimitsPrices;
 
-// same fields as gas_prices_ext; behavior differs
-gas_prices_v3#df gas_price:uint64 gas_limit:uint64 special_gas_limit:uint64 gas_credit:uint64
-  block_gas_limit:uint64 freeze_due_limit:uint64 delete_due_limit:uint64
-  = GasLimitsPrices;
-
 gas_flat_pfx#d1 flat_gas_limit:uint64 flat_gas_price:uint64 other:GasLimitsPrices
   = GasLimitsPrices;
 

crypto/block/mc-config.cpp

@@ -654,13 +654,9 @@ td::Result<GasLimitsPrices> Config::do_get_gas_limits_prices(td::Ref<vm::Cell> c
     res.delete_due_limit = r.delete_due_limit;
   };
   block::gen::GasLimitsPrices::Record_gas_prices_ext rec;
-  block::gen::GasLimitsPrices::Record_gas_prices_v3 rec_v3;
   vm::CellSlice cs0 = cs;
   if (tlb::unpack(cs, rec)) {
     f(rec, rec.special_gas_limit);
-  } else if (tlb::unpack(cs = cs0, rec_v3)) {
-    f(rec_v3, rec_v3.special_gas_limit);
-    res.special_full_limit = true;
   } else {
     block::gen::GasLimitsPrices::Record_gas_prices rec0;
     if (tlb::unpack(cs = cs0, rec0)) {

crypto/block/mc-config.h

@@ -349,7 +349,6 @@ struct GasLimitsPrices {
   td::uint64 block_gas_limit{0};
   td::uint64 freeze_due_limit{0};
   td::uint64 delete_due_limit{0};
-  bool special_full_limit{false};
 
   td::RefInt256 compute_gas_price(td::uint64 gas_used) const;
 };

crypto/block/transaction.cpp

@@ -1039,12 +1039,8 @@ bool ComputePhaseConfig::parse_GasLimitsPrices_internal(Ref<vm::CellSlice> cs, t
     delete_due_limit = td::make_refint(r.delete_due_limit);
   };
   block::gen::GasLimitsPrices::Record_gas_prices_ext rec;
-  block::gen::GasLimitsPrices::Record_gas_prices_v3 rec_v3;
   if (tlb::csr_unpack(cs, rec)) {
     f(rec, rec.special_gas_limit);
-  } else if (tlb::csr_unpack(cs, rec_v3)) {
-    f(rec_v3, rec_v3.special_gas_limit);
-    special_gas_full = true;
   } else {
     block::gen::GasLimitsPrices::Record_gas_prices rec0;
     if (tlb::csr_unpack(std::move(cs), rec0)) {
@@ -1153,8 +1149,8 @@ namespace transaction {
  * not enough to clean up old queires, thus locking funds inside.
  * See comment in crypto/smartcont/highload-wallet-v2-code.fc for details on why this happened.
  * Account address: EQD_v9j1rlsuHHw2FIhcsCFFSD367ldfDdCKcsNmNpIRzUlu
- * It was proposed to validators to increase gas limit for this account for a limited amount of time (until 2024-02-16).
+ * It was proposed to validators to increase gas limit for this account for a limited amount of time (until 2024-02-29).
- * It is activated by setting gas_prices_v3 in ConfigParam 20 (config_mc_gas_prices).
+ * It is activated by setting global version to 5 in ConfigParam 8.
  * This config change also activates new behavior for special accounts in masterchain.
  *
  * @param cfg The compute phase configuration.
@@ -1164,10 +1160,10 @@ namespace transaction {
  * @returns True if gas_limit override is required, false otherwise
  */
 static bool override_gas_limit(const ComputePhaseConfig& cfg, ton::UnixTime now, const Account& account) {
-  if (!cfg.mc_gas_prices.special_full_limit) {
+  if (!cfg.special_gas_full) {
     return false;
   }
-  ton::UnixTime until = 1708041600;  // 2024-02-16 00:00:00 UTC
+  ton::UnixTime until = 1709164800;  // 2024-02-29 00:00:00 UTC
   ton::WorkchainId wc = 0;
   const char* addr_hex = "FFBFD8F5AE5B2E1C7C3614885CB02145483DFAEE575F0DD08A72C366369211CD";
   return now < until && account.workchain == wc && account.addr.to_hex() == addr_hex;
@@ -1563,6 +1559,7 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) {
   vm.set_global_version(cfg.global_version);
   vm.set_c7(prepare_vm_c7(cfg));  // tuple with SmartContractInfo
   vm.set_chksig_always_succeed(cfg.ignore_chksig);
+  vm.set_stop_on_accept_message(cfg.stop_on_accept_message);
   // vm.incr_stack_trace(1);    // enable stack dump after each step
 
   LOG(DEBUG) << "starting VM";
@@ -3533,6 +3530,7 @@ td::Status FetchConfigParams::fetch_config_params(
     TRY_RESULT_PREFIX(mc_gas_prices, config.get_gas_limits_prices(true),
                       "cannot unpack masterchain gas prices and limits: ");
     compute_phase_cfg->mc_gas_prices = std::move(mc_gas_prices);
+    compute_phase_cfg->special_gas_full = config.get_global_version() >= 5;
     storage_phase_cfg->enable_due_payment = config.get_global_version() >= 4;
     compute_phase_cfg->block_rand_seed = *rand_seed;
     compute_phase_cfg->max_vm_data_depth = size_limits.max_vm_data_depth;

crypto/block/transaction.h

@@ -120,6 +120,7 @@ struct ComputePhaseConfig {
   std::unique_ptr<vm::Dictionary> suspended_addresses;
   SizeLimitsConfig size_limits;
   int vm_log_verbosity = 0;
+  bool stop_on_accept_message = false;
 
   ComputePhaseConfig() : gas_price(0), gas_limit(0), special_gas_limit(0), gas_credit(0) {
     compute_threshold();

crypto/vm/cellops.cpp

@@ -892,6 +892,40 @@ int exec_load_special_cell(VmState* st, bool quiet) {
   Stack& stack = st->get_stack();
   VM_LOG(st) << "execute XLOAD" << (quiet ? "Q" : "");
   auto cell = stack.pop_cell();
+  if (st->get_global_version() >= 5) {
+    st->register_cell_load(cell->get_hash());
+    auto r_loaded_cell = cell->load_cell();
+    if (r_loaded_cell.is_error()) {
+      if (quiet) {
+        stack.push_bool(false);
+        return 0;
+      } else {
+        throw VmError{Excno::cell_und, "failed to load cell"};
+      }
+    }
+    auto loaded_cell = r_loaded_cell.move_as_ok();
+    if (loaded_cell.data_cell->is_special()) {
+      if (loaded_cell.data_cell->special_type() != CellTraits::SpecialType::Library) {
+        if (quiet) {
+          stack.push_bool(false);
+          return 0;
+        } else {
+          throw VmError{Excno::cell_und, "unexpected special cell"};
+        }
+      }
+      CellSlice cs(std::move(loaded_cell));
+      DCHECK(cs.size() == Cell::hash_bits + 8);
+      cell = st->load_library(cs.data_bits() + 8);
+      if (cell.is_null()) {
+        if (quiet) {
+          stack.push_bool(false);
+          return 0;
+        } else {
+          throw VmError{Excno::cell_und, "failed to load library cell"};
+        }
+      }
+    }
+  }
   stack.push_cell(cell);
   if (quiet) {
     stack.push_bool(true);

crypto/vm/cells/CellSlice.cpp

@@ -1056,9 +1056,10 @@ std::ostream& operator<<(std::ostream& os, Ref<CellSlice> cs_ref) {
 // If can_be_special is not null, then it is allowed to load special cell
 // Flag whether loaded cell is actually special will be stored into can_be_special
 VirtualCell::LoadedCell load_cell_slice_impl(Ref<Cell> cell, bool* can_be_special) {
+  auto* vm_state_interface = VmStateInterface::get();
+  bool library_loaded = false;
   while (true) {
-    auto* vm_state_interface = VmStateInterface::get();
+    if (vm_state_interface && !library_loaded) {
-    if (vm_state_interface) {
       vm_state_interface->register_cell_load(cell->get_hash());
     }
     auto r_loaded_cell = cell->load_cell();
@@ -1077,6 +1078,12 @@ VirtualCell::LoadedCell load_cell_slice_impl(Ref<Cell> cell, bool* can_be_specia
     } else if (loaded_cell.data_cell->is_special()) {
       if (loaded_cell.data_cell->special_type() == DataCell::SpecialType::Library) {
         if (vm_state_interface) {
+          if (vm_state_interface->get_global_version() >= 5) {
+            if (library_loaded) {
+              throw VmError{Excno::cell_und, "failed to load library cell: recursive library cells are not allowed"};
+            }
+            library_loaded = true;
+          }
           CellSlice cs(std::move(loaded_cell));
           DCHECK(cs.size() == Cell::hash_bits + 8);
           auto library_cell = vm_state_interface->load_library(cs.data_bits() + 8);

crypto/vm/tonops.cpp

@@ -67,6 +67,10 @@ int exec_set_gas_generic(VmState* st, long long new_gas_limit) {
     throw VmNoGas{};
   }
   st->change_gas_limit(new_gas_limit);
+  if (st->get_stop_on_accept_message()) {
+    VM_LOG(st) << "External message is accepted, stopping TVM";
+    return st->jump(td::Ref<QuitCont>{true, 0});
+  }
   return 0;
 }
 

crypto/vm/vm.h

@@ -98,6 +98,7 @@ class VmState final : public VmStateInterface {
   td::HashSet<CellHash> loaded_cells;
   int stack_trace{0}, debug_off{0};
   bool chksig_always_succeed{false};
+  bool stop_on_accept_message{false};
   td::optional<td::Bits256> missing_library;
   td::uint16 max_data_depth = 512; // Default value
   int global_version{0};
@@ -339,7 +340,7 @@ class VmState final : public VmStateInterface {
   void preclear_cr(const ControlRegs& save) {
     cr &= save;
   }
-  int get_global_version() const {
+  int get_global_version() const override {
     return global_version;
   }
   void set_global_version(int version) {
@@ -381,6 +382,12 @@ class VmState final : public VmStateInterface {
   bool get_chksig_always_succeed() const {
     return chksig_always_succeed;
   }
+  void set_stop_on_accept_message(bool flag) {
+    stop_on_accept_message = flag;
+  }
+  bool get_stop_on_accept_message() const {
+    return stop_on_accept_message;
+  }
   Ref<OrdCont> ref_to_cont(Ref<Cell> cell) const {
     return td::make_ref<OrdCont>(load_cell_slice_ref(std::move(cell)), get_cp());
   }

crypto/vm/vmstate.h

@@ -19,6 +19,7 @@
 #pragma once
 #include "common/refcnt.hpp"
 #include "vm/cells.h"
+#include "common/global-version.h"
 
 #include "td/utils/Context.h"
 
@@ -38,6 +39,9 @@ class VmStateInterface : public td::Context<VmStateInterface> {
   virtual bool register_op(int op_units = 1) {
     return true;
   };
+  virtual int get_global_version() const {
+    return ton::SUPPORTED_VERSION;
+  }
 };
 
 }  // namespace vm

doc/GlobalVersions.md

@@ -36,4 +36,21 @@ intermediate value before division (e.g. `(xy+w)/z`).
 * Flag +16 in actions "Send message", "Reserve", "Change library" causes bounce if action fails.
 
 ### Storage phase
-* Unpaid storage fee is now saved to `due_payment`
+* Unpaid storage fee is now saved to `due_payment`
+
+## Version 5
+
+### Gas limits
+Version 5 enables higher gas limits for special contracts.
+
+* Gas limit for all transactions on special contracts is set to `special_gas_limit` from `ConfigParam 20` (which is 35M at the moment of writing). 
+Previously only ticktock transactions had this limit, while ordinary transactions had a default limit of `gas_limit` gas (1M).
+* Gas usage of special contracts is not taken into account when checking block limits. This allows keeping masterchain block limits low
+while having high gas limits for elector.
+* Gas limit on `EQD_v9j1rlsuHHw2FIhcsCFFSD367ldfDdCKcsNmNpIRzUlu` is increased to `special_gas_limit * 2` until 2024-02-29.
+See [this post](https://t.me/tonstatus/88) for details.
+
+### Loading libraries
+* Loading "nested libraries" (i.e. a library cell that points to another library cell) throws an exception.
+* Loading a library consumes gas for cell load only once (for the library cell), not twice (both for the library cell and the cell in the library).
+* `XLOAD` now works differently. When it takes a library cell, it returns the cell that it points to. This allows loading "nested libraries", if needed.

recent_changelog.md

@@ -1,11 +1,17 @@
 ## 2024.01 Update
 
-1. Fixes in how gas in transactions on special accounts is accounted in block limit. Previously, gas was counted as usual, so to conduct elections that costs >30m gas block limit in masterchain was set to 37m gas. To lower the limit for safety reasons it is proposed to not count gas on special accounts. Besides `gas_max` is set to `special_gas_limit` for all types of transactions on special accounts. New behavior is activated through setting `gas_prices_v3` in `ConfigParam 20;`.
+1. Fixes in how gas in transactions on special accounts is accounted in block limit. Previously, gas was counted as usual, so to conduct elections that costs >30m gas block limit in masterchain was set to 37m gas. To lower the limit for safety reasons it is proposed to caunt gas on special accounts separately. Besides `gas_max` is set to `special_gas_limit` for all types of transactions on special accounts. New behavior is activated through setting `version >= 5` in `ConfigParam 8;`.
    * Besides update of config temporally increases gas limit on `EQD_v9j1rlsuHHw2FIhcsCFFSD367ldfDdCKcsNmNpIRzUlu` to `special_gas_limit`, see [details](https://t.me/tonstatus/88).
 2. Improvements in LS behavior
    * Improved detection of the state with all shards applied to decrease rate of `Block is not applied` error
    * Better error logs: `block not in db` and `block is not applied` separation
    * Fix error in proof generation for blocks after merge
+   * Fix most of `block is not applied` issues related to sending too recent block in Proofs
+   * LS now check external messages till `accept_message` (`set_gas`).
 3. Improvements in DHT work and storage, CellDb, config.json ammendment, peer misbehavior detection, validator session stats collection, emulator.
+4. Change in CTOS and XLOAD behavior activated through setting `version >= 5` in `ConfigParam 8;`:
+   * Loading "nested libraries" (i.e. a library cell that points to another library cell) throws an exception.
+   * Loading a library consumes gas for cell load only once (for the library cell), not twice (both for the library cell and the cell in the library).
+   * `XLOAD` now works differently. When it takes a library cell, it returns the cell that it points to. This allows loading "nested libraries", if needed.
 
-Besides the work of the core team, this update is based on the efforts of @XaBbl4 (peer misbehavior detection).
+Besides the work of the Core team, this update is based on the efforts of @XaBbl4 (peer misbehavior detection) and @akifoq (CTOS behavior and gas limit scheme for special accounts).

test/test-validator-session-state.cpp

@@ -34,6 +34,7 @@
 
 #include "validator-session/validator-session-description.h"
 #include "validator-session/validator-session-state.h"
+#include "validator-session/validator-session-description.hpp"
 
 #include <limits>
 #include <memory>
@@ -48,17 +49,13 @@ class Description : public ton::validatorsession::ValidatorSessionDescription {
     return 0;
   }
   void *alloc(size_t size, size_t align, bool temp) override {
-    size = (size + 15) / 16 * 16;
+    return (temp ? mem_temp_ : mem_perm_).alloc(size, align);
-    td::uint32 idx = temp ? 1 : 0;
-    auto s = pdata_cur_[idx].fetch_add(size);
-    CHECK(s + size <= pdata_size_[idx]);
-    return static_cast<void *>(pdata_[idx] + s);
   }
   bool is_persistent(const void *ptr) const override {
-    return ptr == nullptr || (ptr >= pdata_[0] && ptr < pdata_[0] + pdata_size_[0]);
+    return mem_perm_.contains(ptr);
   }
   void clear_temp_memory() override {
-    pdata_cur_[1] = 0;
+    mem_temp_.clear();
   }
 
   ton::PublicKeyHash get_source_id(td::uint32 idx) const override {
@@ -189,21 +186,8 @@ class Description : public ton::validatorsession::ValidatorSessionDescription {
     return opts_;
   }
 
-  ~Description() {
-    delete[] pdata_[0];
-    delete[] pdata_[1];
-  }
-
   Description(ton::validatorsession::ValidatorSessionOptions opts, td::uint32 total_nodes)
-      : opts_(opts), total_nodes_(total_nodes) {
+      : opts_(opts), total_nodes_(total_nodes), mem_perm_(1 << 30), mem_temp_(1 << 22) {
-    pdata_size_[0] =
-        static_cast<std::size_t>(std::numeric_limits<std::size_t>::max() < (1ull << 32) ? 1ull << 30 : 1ull << 33);
-    pdata_size_[1] = 1 << 22;
-    pdata_[0] = new td::uint8[pdata_size_[0]];
-    pdata_[1] = new td::uint8[pdata_size_[1]];
-    pdata_cur_[0] = 0;
-    pdata_cur_[1] = 0;
-
     for (auto &el : cache_) {
       Cached v{nullptr};
       el.store(v, std::memory_order_relaxed);
@@ -224,9 +208,7 @@ class Description : public ton::validatorsession::ValidatorSessionDescription {
   };
   std::array<std::atomic<Cached>, cache_size> cache_;
 
-  td::uint8 *pdata_[2];
+  ton::validatorsession::ValidatorSessionDescriptionImpl::MemPool mem_perm_, mem_temp_;
-  std::atomic<size_t> pdata_cur_[2];
-  size_t pdata_size_[2];
 };
 
 double myrand() {

validator-session/validator-session-description.hpp

@@ -58,6 +58,7 @@ class ValidatorSessionDescriptionImpl : public ValidatorSessionDescription {
   };
   std::array<std::atomic<Cached>, cache_size> cache_;
 
+ public:
   class MemPool {
    public:
     explicit MemPool(size_t chunk_size);
@@ -71,6 +72,8 @@ class ValidatorSessionDescriptionImpl : public ValidatorSessionDescription {
     std::vector<td::uint8 *> data_;
     size_t ptr_ = 0;
   };
+
+ private:
   MemPool mem_perm_ = MemPool(mem_chunk_size_perm);
   MemPool mem_temp_ = MemPool(mem_chunk_size_temp);
 

validator/impl/collator-impl.h

@@ -244,7 +244,7 @@ class Collator final : public td::actor::Actor {
                                   Ref<vm::Cell>& in_msg);
   bool create_ticktock_transactions(int mask);
   bool create_ticktock_transaction(const ton::StdSmcAddress& smc_addr, ton::LogicalTime req_start_lt, int mask);
-  Ref<vm::Cell> create_ordinary_transaction(Ref<vm::Cell> msg_root);
+  Ref<vm::Cell> create_ordinary_transaction(Ref<vm::Cell> msg_root, bool is_special_tx = false);
   bool check_cur_validator_set();
   bool unpack_last_mc_state();
   bool unpack_last_state();

validator/impl/collator.cpp

@@ -2667,8 +2667,7 @@ bool Collator::create_ticktock_transaction(const ton::StdSmcAddress& smc_addr, t
     return fatal_error(td::Status::Error(
         -666, std::string{"cannot serialize new transaction for smart contract "} + smc_addr.to_hex()));
   }
-  if (!trans->update_limits(*block_limit_status_,
+  if (!trans->update_limits(*block_limit_status_, /* with_gas = */ false)) {
-                            /* with_gas = */ !(acc->is_special && compute_phase_cfg_.special_gas_full))) {
     return fatal_error(-666, "cannot update block limit status to include the new transaction");
   }
   if (trans->commit(*acc).is_null()) {
@@ -2684,10 +2683,11 @@ bool Collator::create_ticktock_transaction(const ton::StdSmcAddress& smc_addr, t
  * Creates an ordinary transaction using a given message.
  *
  * @param msg_root The root of the message to be processed serialized using Message TLB-scheme.
+ * @param is_special_tx True if creating a special transaction (mint/recover), false otherwise.
  *
  * @returns The root of the serialized transaction, or an empty reference if the transaction creation fails.
  */
-Ref<vm::Cell> Collator::create_ordinary_transaction(Ref<vm::Cell> msg_root) {
+Ref<vm::Cell> Collator::create_ordinary_transaction(Ref<vm::Cell> msg_root, bool is_special_tx) {
   ton::StdSmcAddress addr;
   auto cs = vm::load_cell_slice(msg_root);
   bool external;
@@ -2746,7 +2746,7 @@ Ref<vm::Cell> Collator::create_ordinary_transaction(Ref<vm::Cell> msg_root) {
   std::unique_ptr<block::transaction::Transaction> trans = res.move_as_ok();
 
   if (!trans->update_limits(*block_limit_status_,
-                            /* with_gas = */ !(acc->is_special && compute_phase_cfg_.special_gas_full))) {
+                            /* with_gas = */ !(is_special_tx && compute_phase_cfg_.special_gas_full))) {
     fatal_error("cannot update block limit status to include the new transaction");
     return {};
   }
@@ -3019,7 +3019,7 @@ int Collator::process_one_new_message(block::NewOutMsg msg, bool enqueue_only, R
     return -1;
   }
   // 1. create a Transaction processing this Message
-  auto trans_root = create_ordinary_transaction(msg.msg);
+  auto trans_root = create_ordinary_transaction(msg.msg, is_special != nullptr);
   if (trans_root.is_null()) {
     fatal_error("cannot create transaction for re-processing output message");
     return -1;

validator/impl/external-message.cpp

@@ -156,6 +156,7 @@ td::Status ExtMessageQ::run_message_on_account(ton::WorkchainId wc,
    }
    compute_phase_cfg_.libraries = std::make_unique<vm::Dictionary>(config->get_libraries_root(), 256);
    compute_phase_cfg_.with_vm_log = true;
+   compute_phase_cfg_.stop_on_accept_message = true;
 
    auto res = Collator::impl_create_ordinary_transaction(msg_root, acc, utime, lt,
                                                     &storage_phase_cfg_, &compute_phase_cfg_,

validator/impl/validate-query.cpp

@@ -945,6 +945,7 @@ bool ValidateQuery::fetch_config_params() {
       return fatal_error(mc_gas_prices.move_as_error_prefix("cannot unpack masterchain gas prices and limits: "));
     }
     compute_phase_cfg_.mc_gas_prices = mc_gas_prices.move_as_ok();
+    compute_phase_cfg_.special_gas_full = config_->get_global_version() >= 5;
     storage_phase_cfg_.enable_due_payment = config_->get_global_version() >= 4;
     compute_phase_cfg_.block_rand_seed = rand_seed_;
     compute_phase_cfg_.libraries = std::make_unique<vm::Dictionary>(config_->get_libraries_root(), 256);
@@ -4684,6 +4685,7 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT
   bool external{false}, ihr_delivered{false}, need_credit_phase{false};
   // check input message
   block::CurrencyCollection money_imported(0), money_exported(0);
+  bool is_special_tx = false;  // recover/mint transaction
   if (in_msg_root.not_null()) {
     auto in_descr_cs = in_msg_dict_->lookup(in_msg_root->get_hash().as_bitslice());
     if (in_descr_cs.is_null()) {
@@ -4699,6 +4701,7 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT
                                     << " has an invalid InMsg record (not one of msg_import_ext, msg_import_fin, "
                                        "msg_import_imm or msg_import_ihr)");
     }
+    is_special_tx = is_special_in_msg(*in_descr_cs);
     // once we know there is a InMsg with correct hash, we already know that it contains a message with this hash (by the verification of InMsg), so it is our message
     // have still to check its destination address and imported value
     // and that it refers to this transaction
@@ -4716,7 +4719,7 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT
                                       << " processed inbound message created later at logical time "
                                       << info.created_lt);
       }
-      if (info.created_lt != start_lt_ || !is_special_in_msg(*in_descr_cs)) {
+      if (info.created_lt != start_lt_ || !is_special_tx) {
         msg_proc_lt_.emplace_back(addr, lt, info.created_lt);
       }
       dest = std::move(info.dest);
@@ -5056,19 +5059,31 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT
     return reject_query(PSTRING() << "cannot re-create the serialization of  transaction " << lt
                                   << " for smart contract " << addr.to_hex());
   }
-  if (!trs->update_limits(*block_limit_status_,
+  if (!trs->update_limits(*block_limit_status_, /* with_gas = */ false, /* with_size = */ false)) {
-                          /* with_gas = */ !account.is_special && !trs->gas_limit_overridden,
-                          /* with_size = */ false)) {
     return fatal_error(PSTRING() << "cannot update block limit status to include transaction " << lt << " of account "
                                  << addr.to_hex());
   }
-  if (block_limit_status_->gas_used > block_limits_->gas.hard() + compute_phase_cfg_.gas_limit) {
+
-    // Note that block_limit_status_->gas_used does not include transactions in special accounts
+  // Collator should stop if total gas usage exceeds limits, including transactions on special accounts, but without
+  // ticktocks and mint/recover.
+  // Here Validator checks a weaker condition
+  if (!is_special_tx && !trs->gas_limit_overridden && trans_type == block::transaction::Transaction::tr_ord) {
+    (account.is_special ? total_special_gas_used_ : total_gas_used_) += trs->gas_used();
+  }
+  if (total_gas_used_ > block_limits_->gas.hard() + compute_phase_cfg_.gas_limit) {
     return reject_query(PSTRING() << "gas block limits are exceeded: total_gas_used > gas_limit_hard + trx_gas_limit ("
-                                  << "total_gas_used=" << block_limit_status_->gas_used
+                                  << "total_gas_used=" << total_gas_used_
                                   << ", gas_limit_hard=" << block_limits_->gas.hard()
                                   << ", trx_gas_limit=" << compute_phase_cfg_.gas_limit << ")");
   }
+  if (total_special_gas_used_ > block_limits_->gas.hard() + compute_phase_cfg_.special_gas_limit) {
+    return reject_query(
+        PSTRING() << "gas block limits are exceeded: total_special_gas_used > gas_limit_hard + special_gas_limit ("
+                  << "total_special_gas_used=" << total_special_gas_used_
+                  << ", gas_limit_hard=" << block_limits_->gas.hard()
+                  << ", special_gas_limit=" << compute_phase_cfg_.special_gas_limit << ")");
+  }
+
   auto trans_root2 = trs->commit(account);
   if (trans_root2.is_null()) {
     return reject_query(PSTRING() << "the re-created transaction " << lt << " for smart contract " << addr.to_hex()

validator/impl/validate-query.hpp

@@ -195,6 +195,7 @@ class ValidateQuery : public td::actor::Actor {
   ton::LogicalTime prev_key_block_lt_;
   std::unique_ptr<block::BlockLimits> block_limits_;
   std::unique_ptr<block::BlockLimitStatus> block_limit_status_;
+  td::uint64 total_gas_used_{0}, total_special_gas_used_{0};
 
   LogicalTime start_lt_, end_lt_;
   UnixTime prev_now_{~0u}, now_{~0u};

validator/manager.cpp

@@ -2460,8 +2460,8 @@ void ValidatorManagerImpl::update_shard_client_state(BlockIdExt masterchain_bloc
 }
 
 void ValidatorManagerImpl::get_shard_client_state(bool from_db, td::Promise<BlockIdExt> promise) {
-  if (!shard_client_.empty() && !from_db) {
+  if (shard_client_handle_ && !from_db) {
-    td::actor::send_closure(shard_client_, &ShardClient::get_processed_masterchain_block_id, std::move(promise));
+    promise.set_result(shard_client_handle_->id());
   } else {
     td::actor::send_closure(db_, &Db::get_shard_client_state, std::move(promise));
   }