Accelerator: partial fullnodes (#1393)

* Accelerator: partial fullnodes

1) Node can monitor a subset of shards
2) New archive slice format (sharded)
3) Validators are still required to have all shards
4) Support partial liteservers in lite-client, blockchain explorer, tonlib
5) Proxy liteserver

* Fix compilation error

validator/downloaders/wait-block-data.cpp

@@ -106,13 +106,24 @@ void WaitBlockData::start() {
     });
 
     td::actor::send_closure(manager_, &ValidatorManager::try_get_static_file, handle_->id().file_hash, std::move(P));
+  } else if (try_get_candidate_) {
+    try_get_candidate_ = false;
+    td::actor::send_closure(
+        manager_, &ValidatorManager::get_candidate_data_by_block_id_from_db, handle_->id(),
+        [SelfId = actor_id(this), id = handle_->id()](td::Result<td::BufferSlice> R) {
+          if (R.is_error()) {
+            td::actor::send_closure(SelfId, &WaitBlockData::start);
+          } else {
+            td::actor::send_closure(SelfId, &WaitBlockData::loaded_data, ReceivedBlock{id, R.move_as_ok()});
+          }
+        });
   } else {
     auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<ReceivedBlock> R) {
       if (R.is_error()) {
         td::actor::send_closure(SelfId, &WaitBlockData::failed_to_get_block_data_from_net,
                                 R.move_as_error_prefix("net error: "));
       } else {
-        td::actor::send_closure(SelfId, &WaitBlockData::got_data_from_net, R.move_as_ok());
+        td::actor::send_closure(SelfId, &WaitBlockData::loaded_data, R.move_as_ok());
       }
     });
 
@@ -137,16 +148,16 @@ void WaitBlockData::failed_to_get_block_data_from_net(td::Status reason) {
                td::Timestamp::in(0.1));
 }
 
-void WaitBlockData::got_data_from_net(ReceivedBlock block) {
+void WaitBlockData::loaded_data(ReceivedBlock block) {
   auto X = create_block(std::move(block));
   if (X.is_error()) {
     failed_to_get_block_data_from_net(X.move_as_error_prefix("bad block from net: "));
     return;
   }
-  got_block_data_from_net(X.move_as_ok());
+  loaded_block_data(X.move_as_ok());
 }
 
-void WaitBlockData::got_block_data_from_net(td::Ref<BlockData> block) {
+void WaitBlockData::loaded_block_data(td::Ref<BlockData> block) {
   if (data_.not_null()) {
     return;
   }

validator/downloaders/wait-block-data.hpp

@@ -30,15 +30,16 @@ class ValidatorManager;
 class WaitBlockData : public td::actor::Actor {
  public:
   WaitBlockData(BlockHandle handle, td::uint32 priority, td::actor::ActorId<ValidatorManager> manager,
-                td::Timestamp timeout, td::Promise<td::Ref<BlockData>> promise)
+                td::Timestamp timeout, bool try_get_candidate, td::Promise<td::Ref<BlockData>> promise)
       : handle_(std::move(handle))
       , priority_(priority)
       , manager_(manager)
       , timeout_(timeout)
+      , try_get_candidate_(try_get_candidate)
       , promise_(std::move(promise))
       , perf_timer_("waitdata", 1.0, [manager](double duration) {
-          send_closure(manager, &ValidatorManager::add_perf_timer_stat, "waitdata", duration);
-        }) {
+        send_closure(manager, &ValidatorManager::add_perf_timer_stat, "waitdata", duration);
+      }) {
   }
 
   void update_timeout(td::Timestamp timeout, td::uint32 priority) {
@@ -57,8 +58,8 @@ class WaitBlockData : public td::actor::Actor {
   void set_is_hardfork(bool value);
   void start();
   void got_block_data_from_db(td::Ref<BlockData> data);
-  void got_data_from_net(ReceivedBlock data);
-  void got_block_data_from_net(td::Ref<BlockData> block);
+  void loaded_data(ReceivedBlock data);
+  void loaded_block_data(td::Ref<BlockData> block);
   void checked_proof_link();
   void failed_to_get_block_data_from_net(td::Status reason);
 
@@ -73,6 +74,7 @@ class WaitBlockData : public td::actor::Actor {
 
   td::actor::ActorId<ValidatorManager> manager_;
   td::Timestamp timeout_;
+  bool try_get_candidate_;
   td::Promise<td::Ref<BlockData>> promise_;
 
   td::Ref<BlockData> data_;

validator/downloaders/wait-block-state.cpp

@@ -21,6 +21,7 @@
 #include "ton/ton-io.hpp"
 #include "common/checksum.h"
 #include "common/delay.h"
+#include "validator/downloaders/download-state.hpp"
 
 namespace ton {
 
@@ -106,6 +107,19 @@ void WaitBlockState::start() {
     });
     td::actor::send_closure(manager_, &ValidatorManager::send_get_zero_state_request, handle_->id(), priority_,
                             std::move(P));
+  } else if (check_persistent_state_desc()) {
+    auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Ref<ShardState>> R) {
+      if (R.is_error()) {
+        LOG(WARNING) << "failed to get persistent state: " << R.move_as_error();
+        td::actor::send_closure(SelfId, &WaitBlockState::start);
+      } else {
+        td::actor::send_closure(SelfId, &WaitBlockState::written_state, R.move_as_ok());
+      }
+    });
+    BlockIdExt masterchain_id = persistent_state_desc_->masterchain_id;
+    td::actor::create_actor<DownloadShardState>("downloadstate", handle_->id(), masterchain_id, priority_, manager_,
+                                                timeout_, std::move(P))
+        .release();
   } else if (!handle_->inited_prev() || (!handle_->inited_proof() && !handle_->inited_proof_link())) {
     auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle = handle_](td::Result<td::BufferSlice> R) {
       if (R.is_error()) {

validator/downloaders/wait-block-state.hpp

@@ -27,12 +27,14 @@ namespace validator {
 class WaitBlockState : public td::actor::Actor {
  public:
   WaitBlockState(BlockHandle handle, td::uint32 priority, td::actor::ActorId<ValidatorManager> manager,
-                 td::Timestamp timeout, td::Promise<td::Ref<ShardState>> promise)
+                 td::Timestamp timeout, td::Promise<td::Ref<ShardState>> promise,
+                 td::Ref<PersistentStateDescription> persistent_state_desc = {})
       : handle_(std::move(handle))
       , priority_(priority)
       , manager_(manager)
       , timeout_(timeout)
       , promise_(std::move(promise))
+      , persistent_state_desc_(std::move(persistent_state_desc))
       , perf_timer_("waitstate", 1.0, [manager](double duration) {
           send_closure(manager, &ValidatorManager::add_perf_timer_stat, "waitstate", duration);
         }) {
@@ -90,6 +92,7 @@ class WaitBlockState : public td::actor::Actor {
   td::actor::ActorId<ValidatorManager> manager_;
   td::Timestamp timeout_;
   td::Promise<td::Ref<ShardState>> promise_;
+  td::Ref<PersistentStateDescription> persistent_state_desc_;
 
   td::Ref<ShardState> prev_state_;
   td::Ref<BlockData> block_;
@@ -99,7 +102,15 @@ class WaitBlockState : public td::actor::Actor {
   bool waiting_proof_ = false;
   td::Timestamp next_static_file_attempt_;
 
-  td::PerfWarningTimer perf_timer_;
+  td::PerfWarningTimer perf_timer_{"waitstate", 1.0};
+
+  bool check_persistent_state_desc() const {
+    if (persistent_state_desc_.is_null()) {
+      return false;
+    }
+    auto now = (UnixTime)td::Clocks::system();
+    return persistent_state_desc_->end_time > now + 3600 && persistent_state_desc_->start_time < now - 6 * 3600;
+  }
 };
 
 }  // namespace validator