Add custom overlays for external messages (#949)

* Private overlay for external messages

* Improve ext msg overlays

* Manage from validator console
* Bypass out queue size limit for high-priority messages
* Shuffle messages in get_external_messages

* Cleanup mempool when creating validator group

* Improve private overlays for externals

1. Allow using validator adnl ids in addition to fullnode ids
2. Set priority per sender, not per overlay
3. Require the same overlay name for all nodes
4. Enable lz4 in private block overlay

* Fix typo, add debug logs

* Enable lz4 in private block overlay by config

Change proto_version for lz4 in catchain overlays to 4

* Add logs for broadcasts in fullnode

---------

Co-authored-by: SpyCheese <mikle98@yandex.ru>

validator-engine-console/validator-engine-console-query.cpp

@@ -33,6 +33,8 @@
 #include "td/utils/filesystem.h"
 #include "overlay/overlays.h"
 #include "ton/ton-tl.hpp"
+#include "td/utils/JsonBuilder.h"
+#include "auto/tl/ton_api_json.h"
 
 #include <cctype>
 #include <fstream>
@@ -1107,3 +1109,73 @@ td::Status SetExtMessagesBroadcastDisabledQuery::receive(td::BufferSlice data) {
   td::TerminalIO::out() << "success\n";
   return td::Status::OK();
 }
+
+td::Status AddPrivateExtMsgOverlayQuery::run() {
+  TRY_RESULT_ASSIGN(file_name_, tokenizer_.get_token<std::string>());
+  TRY_STATUS(tokenizer_.check_endl());
+  return td::Status::OK();
+}
+
+td::Status AddPrivateExtMsgOverlayQuery::send() {
+  TRY_RESULT(data, td::read_file(file_name_));
+  TRY_RESULT(json, td::json_decode(data.as_slice()));
+  auto overlay = ton::create_tl_object<ton::ton_api::engine_validator_privateExtMsgOverlay>();
+  TRY_STATUS(ton::ton_api::from_json(*overlay, json.get_object()));
+  auto b = ton::create_serialize_tl_object<ton::ton_api::engine_validator_addPrivateExtMsgOverlay>(std::move(overlay));
+  td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise());
+  return td::Status::OK();
+}
+
+td::Status AddPrivateExtMsgOverlayQuery::receive(td::BufferSlice data) {
+  TRY_RESULT_PREFIX(f, ton::fetch_tl_object<ton::ton_api::engine_validator_success>(data.as_slice(), true),
+                    "received incorrect answer: ");
+  td::TerminalIO::out() << "success\n";
+  return td::Status::OK();
+}
+
+td::Status DelPrivateExtMsgOverlayQuery::run() {
+  TRY_RESULT_ASSIGN(name_, tokenizer_.get_token<std::string>());
+  TRY_STATUS(tokenizer_.check_endl());
+  return td::Status::OK();
+}
+
+td::Status DelPrivateExtMsgOverlayQuery::send() {
+  auto b = ton::create_serialize_tl_object<ton::ton_api::engine_validator_delPrivateExtMsgOverlay>(name_);
+  td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise());
+  return td::Status::OK();
+}
+
+td::Status DelPrivateExtMsgOverlayQuery::receive(td::BufferSlice data) {
+  TRY_RESULT_PREFIX(f, ton::fetch_tl_object<ton::ton_api::engine_validator_success>(data.as_slice(), true),
+                    "received incorrect answer: ");
+  td::TerminalIO::out() << "success\n";
+  return td::Status::OK();
+}
+
+td::Status ShowPrivateExtMsgOverlaysQuery::run() {
+  TRY_STATUS(tokenizer_.check_endl());
+  return td::Status::OK();
+}
+
+td::Status ShowPrivateExtMsgOverlaysQuery::send() {
+  auto b = ton::create_serialize_tl_object<ton::ton_api::engine_validator_showPrivateExtMsgOverlays>();
+  td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise());
+  return td::Status::OK();
+}
+
+td::Status ShowPrivateExtMsgOverlaysQuery::receive(td::BufferSlice data) {
+  TRY_RESULT_PREFIX(
+      f, ton::fetch_tl_object<ton::ton_api::engine_validator_privateExtMsgOverlaysConfig>(data.as_slice(), true),
+      "received incorrect answer: ");
+  td::TerminalIO::out() << f->overlays_.size() << " private overlays:\n\n";
+  for (const auto &overlay : f->overlays_) {
+    td::TerminalIO::out() << "Overlay \"" << overlay->name_ << "\": " << overlay->nodes_.size() << " nodes\n";
+    for (const auto &node : overlay->nodes_) {
+      td::TerminalIO::out() << "  " << node->adnl_id_
+                            << (node->sender_ ? (PSTRING() << " (sender, p=" << node->sender_priority_ << ")") : "")
+                            << "\n";
+    }
+    td::TerminalIO::out() << "\n";
+  }
+  return td::Status::OK();
+}

validator-engine-console/validator-engine-console-query.h

@@ -1144,3 +1144,67 @@ class SetExtMessagesBroadcastDisabledQuery : public Query {
  private:
   bool value;
 };
+
+class AddPrivateExtMsgOverlayQuery : public Query {
+ public:
+  AddPrivateExtMsgOverlayQuery(td::actor::ActorId<ValidatorEngineConsole> console, Tokenizer tokenizer)
+      : Query(console, std::move(tokenizer)) {
+  }
+  td::Status run() override;
+  td::Status send() override;
+  td::Status receive(td::BufferSlice data) override;
+  static std::string get_name() {
+    return "addprivateextmsgoverlay";
+  }
+  static std::string get_help() {
+    return "addprivateextmsgoverlay <filename>\tadd private overlay for external messages with config from file "
+           "<filename>";
+  }
+  std::string name() const override {
+    return get_name();
+  }
+
+ private:
+  std::string file_name_;
+};
+
+class DelPrivateExtMsgOverlayQuery : public Query {
+ public:
+  DelPrivateExtMsgOverlayQuery(td::actor::ActorId<ValidatorEngineConsole> console, Tokenizer tokenizer)
+      : Query(console, std::move(tokenizer)) {
+  }
+  td::Status run() override;
+  td::Status send() override;
+  td::Status receive(td::BufferSlice data) override;
+  static std::string get_name() {
+    return "delprivateextmsgoverlay";
+  }
+  static std::string get_help() {
+    return "delprivateextmsgoverlay <name>\tdelete private overlay for external messages with name <name>";
+  }
+  std::string name() const override {
+    return get_name();
+  }
+
+ private:
+  std::string name_;
+};
+
+class ShowPrivateExtMsgOverlaysQuery : public Query {
+ public:
+  ShowPrivateExtMsgOverlaysQuery(td::actor::ActorId<ValidatorEngineConsole> console, Tokenizer tokenizer)
+      : Query(console, std::move(tokenizer)) {
+  }
+  td::Status run() override;
+  td::Status send() override;
+  td::Status receive(td::BufferSlice data) override;
+  static std::string get_name() {
+    return "showprivateextmsgoverlays";
+  }
+  static std::string get_help() {
+    return "showprivateextmsgoverlays\tshow all private overlay for external messages";
+  }
+  std::string name() const override {
+    return get_name();
+  }
+};

validator-engine-console/validator-engine-console.cpp

@@ -143,6 +143,9 @@ void ValidatorEngineConsole::run() {
   add_query_runner(std::make_unique<QueryRunnerImpl<GetPerfTimerStatsJsonQuery>>());
   add_query_runner(std::make_unique<QueryRunnerImpl<GetShardOutQueueSizeQuery>>());
   add_query_runner(std::make_unique<QueryRunnerImpl<SetExtMessagesBroadcastDisabledQuery>>());
+  add_query_runner(std::make_unique<QueryRunnerImpl<AddPrivateExtMsgOverlayQuery>>());
+  add_query_runner(std::make_unique<QueryRunnerImpl<DelPrivateExtMsgOverlayQuery>>());
+  add_query_runner(std::make_unique<QueryRunnerImpl<ShowPrivateExtMsgOverlaysQuery>>());
 }
 
 bool ValidatorEngineConsole::envelope_send_query(td::BufferSlice query, td::Promise<td::BufferSlice> promise) {