Simplify loading virtual states from collated data, fix message queue cleanup

validator/fabric.h

@@ -40,7 +40,7 @@ td::Result<td::Ref<Proof>> create_proof(BlockIdExt masterchain_block_id, td::Buf
 td::Result<td::Ref<ProofLink>> create_proof_link(BlockIdExt block_id, td::BufferSlice proof);
 td::Result<td::Ref<BlockSignatureSet>> create_signature_set(td::BufferSlice sig_set);
 td::Result<td::Ref<ShardState>> create_shard_state(BlockIdExt block_id, td::BufferSlice data);
-td::Result<td::Ref<ShardState>> create_shard_state(BlockIdExt block_id, td::Ref<vm::DataCell> root_cell);
+td::Result<td::Ref<ShardState>> create_shard_state(BlockIdExt block_id, td::Ref<vm::Cell> root_cell);
 td::Result<BlockHandle> create_block_handle(td::BufferSlice data);
 td::Result<BlockHandle> create_block_handle(td::Slice data);
 td::Result<ConstBlockHandle> create_temp_block_handle(td::BufferSlice data);

validator/impl/collator.cpp

@@ -2450,8 +2450,7 @@ bool Collator::out_msg_queue_cleanup() {
     auto pure_out_msg_queue =
         std::make_unique<vm::AugmentedDictionary>(r_cell.move_as_ok().data_cell, 352, block::tlb::aug_OutMsgQueue);
     td::uint32 deleted = 0;
-    bool fail = false;
+    bool ok = pure_out_msg_queue->check_for_each([&](Ref<vm::CellSlice> value, td::ConstBitPtr key, int n) -> bool {
-    pure_out_msg_queue->check_for_each([&](Ref<vm::CellSlice> value, td::ConstBitPtr key, int n) -> bool {
       vm::CellSlice& cs = value.write();
       assert(n == 352);
       block::EnqueuedMsgDescr enq_msg_descr;
@@ -2461,7 +2460,6 @@ bool Collator::out_msg_queue_cleanup() {
             && enq_msg_descr.check_key(key)      // check key
             && enq_msg_descr.lt_ == created_lt)) {
         LOG(ERROR) << "cannot unpack EnqueuedMsg with key " << key.to_hex(n);
-        fail = true;
         return false;
       }
       LOG(DEBUG) << "scanning outbound message with (lt,hash)=(" << enq_msg_descr.lt_ << ","
@@ -2495,7 +2493,6 @@ bool Collator::out_msg_queue_cleanup() {
         if (!dequeue_message(std::move(enq_msg_descr.msg_env_), deliver_lt)) {
           fatal_error(PSTRING() << "cannot dequeue outbound message with (lt,hash)=(" << enq_msg_descr.lt_ << ","
                                 << enq_msg_descr.hash_.to_hex() << ") by inserting a msg_export_deq record");
-          fail = true;
           return false;
         }
         register_out_msg_queue_op();
@@ -2504,11 +2501,11 @@ bool Collator::out_msg_queue_cleanup() {
           block_limit_class_ = std::max(block_limit_class_, block_limit_status_->classify());
         }
       }
-      return !delivered;
+      return true;
     });
     LOG(WARNING) << "deleted " << deleted << " messages from out_msg_queue after merge, remaining queue size is "
                  << out_msg_queue_size_;
-    if (fail) {
+    if (!ok) {
       return fatal_error("error scanning/updating OutMsgQueue");
     }
   } else {
@@ -5801,7 +5798,19 @@ bool Collator::create_collated_data() {
     if (state_proof.is_null()) {
       return fatal_error("cannot generate Merkle proof for previous state");
     }
-    proofs[prev_state_root_->get_hash().bits()] = std::move(state_proof);
+    if (after_merge_) {
+      bool special;
+      auto cs = vm::load_cell_slice_special(state_proof, special);
+      CHECK(cs.special_type() == vm::CellTraits::SpecialType::MerkleProof);
+      cs = vm::load_cell_slice(cs.prefetch_ref(0));
+      CHECK(cs.size_refs() == 2);
+      CHECK(cs.size() == 32);
+      CHECK(cs.prefetch_ulong(32) == 0x5f327da5U);
+      proofs[cs.prefetch_ref(0)->get_hash(0).bits()] = vm::CellBuilder::create_merkle_proof(cs.prefetch_ref(0));
+      proofs[cs.prefetch_ref(1)->get_hash(0).bits()] = vm::CellBuilder::create_merkle_proof(cs.prefetch_ref(1));
+    } else {
+      proofs[prev_state_root_->get_hash().bits()] = std::move(state_proof);
+    }
   }
   // 4. Proofs for message queues
   for (vm::MerkleProofBuilder &mpb : neighbor_proof_builders_) {

validator/impl/fabric.cpp

@@ -84,7 +84,7 @@ td::Result<td::Ref<ShardState>> create_shard_state(BlockIdExt block_id, td::Buff
   }
 }
 
-td::Result<td::Ref<ShardState>> create_shard_state(BlockIdExt block_id, td::Ref<vm::DataCell> root_cell) {
+td::Result<td::Ref<ShardState>> create_shard_state(BlockIdExt block_id, td::Ref<vm::Cell> root_cell) {
   auto res = ShardStateQ::fetch(block_id, {}, std::move(root_cell));
   if (res.is_error()) {
     return res.move_as_error();

validator/impl/validate-query.cpp

@@ -1243,10 +1243,21 @@ bool ValidateQuery::check_this_shard_mc_info() {
  * @returns True if the previous state is computed successfully, false otherwise.
  */
 bool ValidateQuery::compute_prev_state() {
-  if (!is_masterchain() && full_collated_data_) {
-    return compute_prev_state_from_collated_data();
-  }
   CHECK(prev_states.size() == 1u + after_merge_);
+  CHECK(prev_states.size() == prev_blocks.size());
+  if (!is_masterchain() && full_collated_data_) {
+    for (size_t i = 0; i < prev_states.size(); i++) {
+      Ref<vm::Cell> root = get_virt_state_root(prev_blocks[i]);
+      if (root.is_null()) {
+        return reject_query(PSTRING() << "cannot get previous state from collated data: " << prev_blocks[i].to_str());
+      }
+      auto r_state = create_shard_state(prev_blocks[i], std::move(root));
+      if (r_state.is_error()) {
+        return reject_query("failed to parse previous state from collated data", r_state.move_as_error());
+      }
+      prev_states[i] = r_state.move_as_ok();
+    }
+  }
   // Extend validator timeout if previous block is too old
   UnixTime prev_ts = prev_states[0]->get_unix_time();
   if (after_merge_) {
@@ -1274,46 +1285,6 @@ bool ValidateQuery::compute_prev_state() {
   return true;
 }
 
-bool ValidateQuery::compute_prev_state_from_collated_data() {
-  td::Bits256 state_hash;
-  if (id_.seqno() == 1) {
-    if (prev_blocks.size() != 1) {
-      return reject_query("seqno is 1, but number of previous blocks is not 1");
-    }
-    state_hash = prev_blocks[0].root_hash;
-  } else {
-    std::vector<Ref<vm::Cell>> prev_state_roots(prev_blocks.size());
-    for (size_t i = 0; i < prev_blocks.size(); ++i) {
-      prev_state_roots[i] = get_virt_state_root(prev_blocks[i].root_hash);
-      if (prev_state_roots[i].is_null()) {
-        return reject_query(PSTRING() << "cannot get hash of previous state root: " << prev_blocks[i]);
-      }
-    }
-
-    if (prev_state_roots.size() == 1) {
-      state_hash = prev_state_roots[0]->get_hash().bits();
-    } else {
-      CHECK(prev_state_roots.size() == 2);
-      Ref<vm::Cell> merged;
-      if (!block::gen::t_ShardState.cell_pack_split_state(merged, prev_state_roots[0], prev_state_roots[1])) {
-        return fatal_error(-667, "cannot construct mechanically merged previously state");
-      }
-      state_hash = merged->get_hash().bits();
-    }
-  }
-  if (state_hash != prev_state_hash_) {
-    return reject_query("previous state hash mismatch for block "s + id_.to_str() + " : block header declares " +
-                        prev_state_hash_.to_hex() + " , actual " + state_hash.to_hex());
-  }
-  auto it = virt_roots_.find(state_hash);
-  if (it == virt_roots_.end()) {
-    return reject_query(PSTRING() << "no state root for previous block in collated data (hash = "
-                                  << state_hash.to_hex() << ")");
-  }
-  prev_state_root_ = it->second;
-  return true;
-}
-
 /**
  * Computes the next shard state using the previous state and the block's Merkle update.
  */
@@ -1574,21 +1545,11 @@ bool ValidateQuery::request_neighbor_queues() {
         ++i;
         continue;
       }
-      td::Bits256 state_root_hash;
+      auto state_root = get_virt_state_root(descr.blk_);
-      if (descr.blk_.seqno() == 0) {
+      if (state_root.is_null()) {
-        state_root_hash = descr.blk_.root_hash;
+        return reject_query(PSTRING() << "cannot get state root form collated data: " << descr.blk_.to_str());
-      } else {
-        Ref<vm::Cell> state_root = get_virt_state_root(descr.blk_.root_hash);
-        if (state_root.is_null()) {
-          return reject_query(PSTRING() << "cannot get hash of state root: " << descr.blk_);
-        }
-        state_root_hash = state_root->get_hash().bits();
       }
-      auto it = virt_roots_.find(state_root_hash);
+      auto state = ShardStateQ::fetch(descr.blk_, {}, std::move(state_root));
-      if (it == virt_roots_.end()) {
-        return reject_query(PSTRING() << "cannot get state root form collated data: " << descr.blk_);
-      }
-      auto state = ShardStateQ::fetch(descr.blk_, {}, it->second);
       if (state.is_error()) {
         return reject_query("cannot fetch shard state from collated data", state.move_as_error());
       }
@@ -6898,22 +6859,23 @@ bool ValidateQuery::postcheck_value_flow() {
   return true;
 }
 
-Ref<vm::Cell> ValidateQuery::get_virt_state_root(td::Bits256 block_root_hash) {
+Ref<vm::Cell> ValidateQuery::get_virt_state_root(const BlockIdExt& block_id) {
-  auto it = virt_roots_.find(block_root_hash);
+  auto it = virt_roots_.find(block_id.root_hash);
   if (it == virt_roots_.end()) {
     return {};
   }
   Ref<vm::Cell> root = it->second;
+  if (block_id.seqno() == 0) {
+    return root;
+  }
   block::gen::Block::Record block;
   if (!tlb::unpack_cell(root, block)) {
     return {};
   }
   vm::CellSlice upd_cs{vm::NoVmSpec(), block.state_update};
-  if (!(upd_cs.is_special() && upd_cs.prefetch_long(8) == 4  // merkle update
+  td::Bits256 state_root_hash = upd_cs.prefetch_ref(1)->get_hash(0).bits();
-        && upd_cs.size_ext() == 0x20228)) {
+  it = virt_roots_.find(state_root_hash);
-    return {};
+  return it == virt_roots_.end() ? Ref<vm::Cell>{} : it->second;
-  }
-  return vm::MerkleProof::virtualize_raw(upd_cs.prefetch_ref(1), {0, 1});
 }
 
 /**

validator/impl/validate-query.hpp

@@ -306,7 +306,6 @@ class ValidateQuery : public td::actor::Actor {
   bool extract_collated_data();
   bool try_validate();
   bool compute_prev_state();
-  bool compute_prev_state_from_collated_data();
   bool compute_next_state();
   bool unpack_merge_prev_state();
   bool unpack_prev_state();
@@ -397,7 +396,7 @@ class ValidateQuery : public td::actor::Actor {
                            const block::CurrencyCollection& create);
   bool check_mc_block_extra();
 
-  Ref<vm::Cell> get_virt_state_root(td::Bits256 block_root_hash);
+  Ref<vm::Cell> get_virt_state_root(const BlockIdExt& block_id);
 
   bool check_timeout() {
     if (timeout && timeout.is_in_past()) {