Exporting and importing candidates using validator-engine-console; lite mode for validator group

validator/impl/validate-query.cpp

@@ -449,6 +449,7 @@ bool ValidateQuery::init_parse() {
     return reject_query("after_merge value mismatch in block header");
   }
   rand_seed_ = extra.rand_seed;
+  created_by_ = extra.created_by;
   if (created_by_ != extra.created_by) {
     return reject_query("block candidate "s + id_.to_str() + " has creator " + created_by_.to_hex() +
                         " but the block header contains different value " + extra.created_by.to_hex());

validator/interfaces/validator-manager.h

@@ -169,6 +169,8 @@ class ValidatorManager : public ValidatorManagerInterface {
 
   virtual void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) = 0;
 
+  virtual void wait_block_candidate(BlockId block_id, td::Timestamp timeout, td::Promise<BlockCandidate> promise) = 0;
+
   static bool is_persistent_state(UnixTime ts, UnixTime prev_ts) {
     return ts / (1 << 17) != prev_ts / (1 << 17);
   }

validator/manager-disk.hpp

@@ -371,6 +371,18 @@ class ValidatorManagerImpl : public ValidatorManager {
       td::Promise<tl_object_ptr<ton_api::engine_validator_validatorSessionsInfo>> promise) override {
     UNREACHABLE();
   }
+  void generate_block_candidate(BlockId block_id, td::Promise<BlockCandidate> promise) override {
+    UNREACHABLE();
+  }
+  void get_required_block_candidates(td::Promise<std::vector<BlockId>> promise) override {
+    UNREACHABLE();
+  }
+  void import_block_candidate(BlockCandidate candidate, td::Promise<td::Unit> promise) override {
+    UNREACHABLE();
+  }
+  void wait_block_candidate(BlockId block_id, td::Timestamp timeout, td::Promise<BlockCandidate> promise) override {
+    UNREACHABLE();
+  }
 
  private:
   PublicKeyHash local_id_;

validator/manager-hardfork.hpp

@@ -430,6 +430,19 @@ class ValidatorManagerImpl : public ValidatorManager {
       td::Promise<tl_object_ptr<ton_api::engine_validator_validatorSessionsInfo>> promise) override {
     UNREACHABLE();
   }
+  void generate_block_candidate(BlockId block_id, td::Promise<BlockCandidate> promise) override {
+    UNREACHABLE();
+  }
+  void get_required_block_candidates(td::Promise<std::vector<BlockId>> promise) override {
+    UNREACHABLE();
+  }
+  void import_block_candidate(BlockCandidate candidate, td::Promise<td::Unit> promise) override {
+    UNREACHABLE();
+  }
+  void wait_block_candidate(BlockId block_id, td::Timestamp timeout, td::Promise<BlockCandidate> promise) override {
+    UNREACHABLE();
+  }
+
 
  private:
   td::Ref<ValidatorManagerOptions> opts_;

validator/manager.cpp

@@ -2031,7 +2031,7 @@ td::actor::ActorOwn<ValidatorGroup> ValidatorManagerImpl::create_validator_group
     auto G = td::actor::create_actor<ValidatorGroup>(
         "validatorgroup", shard, validator_id, session_id, validator_set, opts, keyring_, adnl_, rldp_, overlays_,
         db_root_, actor_id(this), init_session,
-        opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno()));
+        opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno()), true);
     return G;
   }
 }
@@ -2583,6 +2583,119 @@ void ValidatorManagerImpl::get_validator_sessions_info(
   IntermediateData::step({std::move(groups), {}, std::move(promise)});
 }
 
+void ValidatorManagerImpl::generate_block_candidate(BlockId block_id, td::Promise<BlockCandidate> promise) {
+  if (!block_id.is_valid_full()) {
+    promise.set_error(td::Status::Error("invalid block id"));
+    return;
+  }
+  if (last_masterchain_state_.is_null()) {
+    promise.set_error(td::Status::Error("not started"));
+    return;
+  }
+  ShardIdFull shard_id = block_id.shard_full();
+  std::vector<BlockIdExt> prev;
+  auto shard = last_masterchain_state_->get_shard_from_config(shard_id);
+  if (shard.not_null()) {
+    if (shard->before_split()) {
+      promise.set_error(td::Status::Error("shard is before_split"));
+      return;
+    }
+    if (shard->before_merge()) {
+      promise.set_error(td::Status::Error("shard is before_merge"));
+      return;
+    }
+    prev.push_back(shard->top_block_id());
+  } else {
+    auto parent = shard_id.pfx_len() == 0 ? td::Ref<McShardHash>()
+                                          : last_masterchain_state_->get_shard_from_config(shard_parent(shard_id));
+    if (parent.not_null() && parent->before_split()) {
+      prev.push_back(parent->top_block_id());
+    } else {
+      auto child_l = last_masterchain_state_->get_shard_from_config(shard_child(shard_id, true));
+      auto child_r = last_masterchain_state_->get_shard_from_config(shard_child(shard_id, false));
+      if (child_l.not_null() && child_r.not_null() && child_l->before_merge() && child_r->before_merge()) {
+        prev.push_back(child_l->top_block_id());
+        prev.push_back(child_r->top_block_id());
+      }
+    }
+    if (prev.empty()) {
+      promise.set_error(td::Status::Error("no such shard"));
+      return;
+    }
+  }
+
+  BlockSeqno next_seqno = 0;
+  for (const BlockIdExt& prev_id : prev) {
+    next_seqno = std::max(next_seqno, prev_id.seqno() + 1);
+  }
+  if (next_seqno != block_id.seqno) {
+    promise.set_error(td::Status::Error(PSTRING() << "seqno mismatch: asked for seqno " << block_id.seqno
+                                                  << ", but actual next seqno is " << next_seqno));
+    return;
+  }
+
+  Ed25519_PublicKey local_id{Bits256::zero()};
+  td::Ref<ValidatorSet> validator_set = last_masterchain_state_->get_validator_set(shard_id);
+  if (validator_set.is_null()) {
+    promise.set_error(td::Status::Error("cannot get validator set"));
+    return;
+  }
+  run_collate_query(shard_id, last_masterchain_state_->get_unix_time(), last_masterchain_block_id_, std::move(prev),
+                    local_id, std::move(validator_set), actor_id(this), td::Timestamp::in(10.0), std::move(promise));
+}
+
+void ValidatorManagerImpl::get_required_block_candidates(td::Promise<std::vector<BlockId>> promise) {
+  std::vector<BlockId> block_ids;
+  for (const auto& p : pending_block_candidates_) {
+    block_ids.push_back(p.first);
+  }
+  promise.set_result(std::move(block_ids));
+}
+
+void ValidatorManagerImpl::import_block_candidate(BlockCandidate candidate, td::Promise<td::Unit> promise) {
+  auto it = pending_block_candidates_.find(candidate.id.id);
+  if (it != pending_block_candidates_.end()) {
+    while (!it->second.empty()) {
+      auto promise = std::move(it->second.back().first);
+      it->second.pop_back();
+      if (it->second.empty()) {
+        promise.set_result(std::move(candidate));
+      } else {
+        promise.set_result(candidate.clone());
+      }
+    }
+    pending_block_candidates_.erase(it);
+  }
+  promise.set_result(td::Unit());
+}
+
+void ValidatorManagerImpl::wait_block_candidate(BlockId block_id, td::Timestamp timeout,
+                                                td::Promise<BlockCandidate> promise) {
+  pending_block_candidates_[block_id].emplace_back(std::move(promise), timeout);
+  delay_action([SelfId = actor_id(this), block_id, timeout]() {
+    td::actor::send_closure(SelfId, &ValidatorManagerImpl::cleanup_old_pending_candidates, block_id, timeout);
+  }, timeout);
+}
+
+void ValidatorManagerImpl::cleanup_old_pending_candidates(BlockId block_id, td::Timestamp now) {
+  auto it = pending_block_candidates_.find(block_id);
+  if (it == pending_block_candidates_.end()) {
+    return;
+  }
+  it->second.erase(std::remove_if(it->second.begin(), it->second.end(),
+                                  [&](std::pair<td::Promise<BlockCandidate>, td::Timestamp> &p) {
+                                    if (p.second.is_in_past(now)) {
+                                      p.first.set_error(td::Status::Error(ErrorCode::timeout, "timeout"));
+                                      return true;
+                                    }
+                                    return false;
+                                  }),
+                   it->second.end());
+  if (it->second.empty()) {
+    pending_block_candidates_.erase(it);
+  }
+}
+
 td::actor::ActorOwn<ValidatorManagerInterface> ValidatorManagerFactory::create(
     td::Ref<ValidatorManagerOptions> opts, std::string db_root, td::actor::ActorId<keyring::Keyring> keyring,
     td::actor::ActorId<adnl::Adnl> adnl, td::actor::ActorId<rldp::Rldp> rldp,

validator/manager.hpp

@@ -534,7 +534,12 @@ class ValidatorManagerImpl : public ValidatorManager {
   void get_validator_sessions_info(
       td::Promise<tl_object_ptr<ton_api::engine_validator_validatorSessionsInfo>> promise) override;
 
-    private:
+  void generate_block_candidate(BlockId block_id, td::Promise<BlockCandidate> promise) override;
+  void get_required_block_candidates(td::Promise<std::vector<BlockId>> promise) override;
+  void import_block_candidate(BlockCandidate candidate, td::Promise<td::Unit> promise) override;
+  void wait_block_candidate(BlockId block_id, td::Timestamp timeout, td::Promise<BlockCandidate> promise) override;
+
+ private:
   td::Timestamp resend_shard_blocks_at_;
   td::Timestamp check_waiters_at_;
   td::Timestamp check_shard_clients_;
@@ -598,6 +603,9 @@ class ValidatorManagerImpl : public ValidatorManager {
 
  private:
   std::map<BlockSeqno, WaitList<td::actor::Actor, td::Unit>> shard_client_waiters_;
+
+  std::map<BlockId, std::vector<std::pair<td::Promise<BlockCandidate>, td::Timestamp>>> pending_block_candidates_;
+  void cleanup_old_pending_candidates(BlockId block_id, td::Timestamp now);
 };
 
 }  // namespace validator

validator/validator-group.cpp

@@ -35,6 +35,22 @@ void ValidatorGroup::generate_block_candidate(td::uint32 round_id, td::Promise<B
     promise.set_error(td::Status::Error(ErrorCode::notready, "cannot collate block: group not started"));
     return;
   }
+  if (lite_mode_) {
+    auto P = td::PromiseCreator::lambda(
+        [promise = std::move(promise),
+         pubkey = Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}](td::Result<BlockCandidate> R) mutable {
+          if (R.is_error()) {
+            promise.set_error(R.move_as_error());
+          } else {
+            BlockCandidate candidate = R.move_as_ok();
+            candidate.pubkey = pubkey;
+            promise.set_result(std::move(candidate));
+          }
+        });
+    td::actor::send_closure(manager_, &ValidatorManager::wait_block_candidate, create_next_block_id_simple(),
+                            td::Timestamp::in(15.0), std::move(P));
+    return;
+  }
   run_collate_query(shard_, min_ts_, min_masterchain_block_id_, prev_block_ids_,
                     Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}, validator_set_, manager_,
                     td::Timestamp::in(10.0), std::move(promise));
@@ -79,7 +95,7 @@ void ValidatorGroup::validate_block_candidate(td::uint32 round_id, BlockCandidat
   VLOG(VALIDATOR_DEBUG) << "validating block candidate " << next_block_id;
   block.id = next_block_id;
   run_validate_query(shard_, min_ts_, min_masterchain_block_id_, prev_block_ids_, std::move(block), validator_set_,
-                     manager_, td::Timestamp::in(10.0), std::move(P));
+                     manager_, td::Timestamp::in(10.0), std::move(P), lite_mode_ ? ValidateMode::lite : 0);
 }
 
 void ValidatorGroup::accept_block_candidate(td::uint32 round_id, PublicKeyHash src, td::BufferSlice block_data,

validator/validator-group.hpp

@@ -64,7 +64,7 @@ class ValidatorGroup : public td::actor::Actor {
                  td::actor::ActorId<keyring::Keyring> keyring, td::actor::ActorId<adnl::Adnl> adnl,
                  td::actor::ActorId<rldp::Rldp> rldp, td::actor::ActorId<overlay::Overlays> overlays,
                  std::string db_root, td::actor::ActorId<ValidatorManager> validator_manager, bool create_session,
-                 bool allow_unsafe_self_blocks_resync)
+                 bool allow_unsafe_self_blocks_resync, bool lite_mode = false)
       : shard_(shard)
       , local_id_(std::move(local_id))
       , session_id_(session_id)
@@ -77,7 +77,8 @@ class ValidatorGroup : public td::actor::Actor {
       , db_root_(std::move(db_root))
       , manager_(validator_manager)
       , init_(create_session)
-      , allow_unsafe_self_blocks_resync_(allow_unsafe_self_blocks_resync) {
+      , allow_unsafe_self_blocks_resync_(allow_unsafe_self_blocks_resync)
+      , lite_mode_(lite_mode) {
   }
 
  private:
@@ -118,6 +119,7 @@ class ValidatorGroup : public td::actor::Actor {
   bool init_ = false;
   bool started_ = false;
   bool allow_unsafe_self_blocks_resync_;
+  bool lite_mode_ = false;
   td::uint32 last_known_round_id_ = 0;
 };
 

validator/validator.h

@@ -216,6 +216,10 @@ class ValidatorManagerInterface : public td::actor::Actor {
   virtual void prepare_stats(td::Promise<std::vector<std::pair<std::string, std::string>>> promise) = 0;
   virtual void get_validator_sessions_info(
       td::Promise<tl_object_ptr<ton_api::engine_validator_validatorSessionsInfo>> promise) = 0;
+
+  virtual void generate_block_candidate(BlockId block_id, td::Promise<BlockCandidate> promise) = 0;
+  virtual void get_required_block_candidates(td::Promise<std::vector<BlockId>> promise) = 0;
+  virtual void import_block_candidate(BlockCandidate candidate, td::Promise<td::Unit> promise) = 0;
 };
 
 }  // namespace validator