Accelerator: partial fullnodes (#1393)

* Accelerator: partial fullnodes

1) Node can monitor a subset of shards
2) New archive slice format (sharded)
3) Validators are still required to have all shards
4) Support partial liteservers in lite-client, blockchain explorer, tonlib
5) Proxy liteserver

* Fix compilation error

validator-engine-console/validator-engine-console-query.cpp

@@ -1,4 +1,4 @@
-/* 
+/*
     This file is part of TON Blockchain source code.
 
     TON Blockchain is free software; you can redistribute it and/or
@@ -35,6 +35,7 @@
 #include "ton/ton-tl.hpp"
 #include "td/utils/JsonBuilder.h"
 #include "auto/tl/ton_api_json.h"
+#include "tl/tl_json.h"
 
 #include <cctype>
 #include <fstream>
@@ -948,8 +949,18 @@ td::Status GetOverlaysStatsJsonQuery::receive(td::BufferSlice data) {
 
       sb << "   \"" << t->key_ << "\": \"" << t->value_ << "\"";
     }
-    sb << "\n  }\n";
-    sb << "}\n";
+    sb << "\n  }";
+    if (!s->extra_.empty()) {
+      sb << ",\n  \"extra\": ";
+      for (char c : s->extra_) {
+        if (c == '\n') {
+          sb << "\n  ";
+        } else {
+          sb << c;
+        }
+      }
+    }
+    sb << "\n}\n";
   }
   sb << "]\n";
   sb << std::flush;
@@ -1216,6 +1227,12 @@ td::Status ShowCustomOverlaysQuery::receive(td::BufferSlice data) {
                                     : "")
                             << (node->block_sender_ ? " (block sender)" : "") << "\n";
     }
+    if (!overlay->sender_shards_.empty()) {
+      td::TerminalIO::out() << "Sender shards:\n";
+      for (const auto &shard : overlay->sender_shards_) {
+        td::TerminalIO::out() << "  " << ton::create_shard_id(shard).to_str() << "\n";
+      }
+    }
     td::TerminalIO::out() << "\n";
   }
   return td::Status::OK();
@@ -1482,3 +1499,43 @@ td::Status GetAdnlStatsQuery::receive(td::BufferSlice data) {
   td::TerminalIO::out() << sb.as_cslice();
   return td::Status::OK();
 }
+
+td::Status AddShardQuery::run() {
+  TRY_RESULT_ASSIGN(wc_, tokenizer_.get_token<td::int32>());
+  TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token<td::int64>());
+  return td::Status::OK();
+}
+
+td::Status AddShardQuery::send() {
+  auto b = ton::create_serialize_tl_object<ton::ton_api::engine_validator_addShard>(
+      ton::create_tl_shard_id(ton::ShardIdFull(wc_, shard_)));
+  td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise());
+  return td::Status::OK();
+}
+
+td::Status AddShardQuery::receive(td::BufferSlice data) {
+  TRY_RESULT_PREFIX(f, ton::fetch_tl_object<ton::ton_api::engine_validator_success>(data.as_slice(), true),
+                    "received incorrect answer: ");
+  td::TerminalIO::out() << "successfully added shard\n";
+  return td::Status::OK();
+}
+
+td::Status DelShardQuery::run() {
+  TRY_RESULT_ASSIGN(wc_, tokenizer_.get_token<td::int32>());
+  TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token<td::int64>());
+  return td::Status::OK();
+}
+
+td::Status DelShardQuery::send() {
+  auto b = ton::create_serialize_tl_object<ton::ton_api::engine_validator_delShard>(
+      ton::create_tl_shard_id(ton::ShardIdFull(wc_, shard_)));
+  td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise());
+  return td::Status::OK();
+}
+
+td::Status DelShardQuery::receive(td::BufferSlice data) {
+  TRY_RESULT_PREFIX(f, ton::fetch_tl_object<ton::ton_api::engine_validator_success>(data.as_slice(), true),
+                    "received incorrect answer: ");
+  td::TerminalIO::out() << "successfully removed shard\n";
+  return td::Status::OK();
+}

validator-engine-console/validator-engine-console-query.h

@@ -1362,3 +1362,49 @@ class GetAdnlStatsQuery : public Query {
   std::string file_name_;
   bool all_ = false;
 };
+
+class AddShardQuery : public Query {
+ public:
+  AddShardQuery(td::actor::ActorId<ValidatorEngineConsole> console, Tokenizer tokenizer)
+      : Query(console, std::move(tokenizer)) {
+  }
+  td::Status run() override;
+  td::Status send() override;
+  td::Status receive(td::BufferSlice data) override;
+  static std::string get_name() {
+    return "addshard";
+  }
+  static std::string get_help() {
+    return "addshard <workchain> <shard>\tstart monitoring shard";
+  }
+  std::string name() const override {
+    return get_name();
+  }
+
+ private:
+  td::int32 wc_;
+  td::int64 shard_;
+};
+
+class DelShardQuery : public Query {
+ public:
+  DelShardQuery(td::actor::ActorId<ValidatorEngineConsole> console, Tokenizer tokenizer)
+      : Query(console, std::move(tokenizer)) {
+  }
+  td::Status run() override;
+  td::Status send() override;
+  td::Status receive(td::BufferSlice data) override;
+  static std::string get_name() {
+    return "delshard";
+  }
+  static std::string get_help() {
+    return "delshard <workchain> <shard>\tstop monitoring shard";
+  }
+  std::string name() const override {
+    return get_name();
+  }
+
+ private:
+  td::int32 wc_;
+  td::int64 shard_;
+};

validator-engine-console/validator-engine-console.cpp

@@ -153,6 +153,8 @@ void ValidatorEngineConsole::run() {
   add_query_runner(std::make_unique<QueryRunnerImpl<GetCollatorOptionsJsonQuery>>());
   add_query_runner(std::make_unique<QueryRunnerImpl<GetAdnlStatsJsonQuery>>());
   add_query_runner(std::make_unique<QueryRunnerImpl<GetAdnlStatsQuery>>());
+  add_query_runner(std::make_unique<QueryRunnerImpl<AddShardQuery>>());
+  add_query_runner(std::make_unique<QueryRunnerImpl<DelShardQuery>>());
 }
 
 bool ValidatorEngineConsole::envelope_send_query(td::BufferSlice query, td::Promise<td::BufferSlice> promise) {