external/ton
1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-03-09 15:40:10 +00:00
Code Issues Releases Wiki Activity

Merge branch 'testnet' into accelerator

Browse source
This commit is contained in:
SpyCheese 2024-10-09 21:10:36 +03:00
commit df21424047
44 changed files with 373 additions and 180 deletions
Download patch file Download diff file Expand all files Collapse all files

14
validator/downloaders/download-state.cpp
View file

@ -50,7 +50,16 @@ void DownloadShardState::start_up() {
void DownloadShardState::got_block_handle(BlockHandle handle) {
handle_ = std::move(handle);
download_state();
if (handle_->received_state()) {
LOG(WARNING) << "shard state " << block_id_.to_str() << " already stored in db";
td::actor::send_closure(manager_, &ValidatorManagerInterface::get_shard_state_from_db, handle_,
[SelfId = actor_id(this)](td::Result<td::Ref<ShardState>> R) {
R.ensure();
td::actor::send_closure(SelfId, &DownloadShardState::written_shard_state, R.move_as_ok());
});
} else {
download_state();
}
}
void DownloadShardState::retry() {
@ -165,6 +174,7 @@ void DownloadShardState::downloaded_shard_state(td::BufferSlice data) {
}
void DownloadShardState::checked_shard_state() {
LOG(WARNING) << "checked shard state " << block_id_.to_str();
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Unit> R) {
R.ensure();
td::actor::send_closure(SelfId, &DownloadShardState::written_shard_state_file);
@ -179,6 +189,7 @@ void DownloadShardState::checked_shard_state() {
}
void DownloadShardState::written_shard_state_file() {
LOG(WARNING) << "written shard state file " << block_id_.to_str();
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Ref<ShardState>> R) {
R.ensure();
td::actor::send_closure(SelfId, &DownloadShardState::written_shard_state, R.move_as_ok());
@ -207,6 +218,7 @@ void DownloadShardState::written_shard_state(td::Ref<ShardState> state) {
}
void DownloadShardState::written_block_handle() {
LOG(WARNING) << "finished downloading and storing shard state " << block_id_.to_str();
finish_query();
}

2
validator/fabric.h
View file

@ -57,7 +57,7 @@ void run_check_external_message(td::Ref<ExtMessage> message, td::actor::ActorId<
void run_accept_block_query(BlockIdExt id, td::Ref<BlockData> data, std::vector<BlockIdExt> prev,
td::Ref<ValidatorSet> validator_set, td::Ref<BlockSignatureSet> signatures,
td::Ref<BlockSignatureSet> approve_signatures, bool send_broadcast, bool apply,
td::Ref<BlockSignatureSet> approve_signatures, int send_broadcast_mode, bool apply,
td::actor::ActorId<ValidatorManager> manager, td::Promise<td::Unit> promise);
void run_fake_accept_block_query(BlockIdExt id, td::Ref<BlockData> data, std::vector<BlockIdExt> prev,
td::Ref<ValidatorSet> validator_set, td::actor::ActorId<ValidatorManager> manager,

17
validator/full-node.cpp
View file

@ -354,17 +354,16 @@ void FullNodeImpl::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_se
}
}
void FullNodeImpl::send_broadcast(BlockBroadcast broadcast, bool custom_overlays_only) {
send_block_broadcast_to_custom_overlays(broadcast);
if (custom_overlays_only) {
return;
void FullNodeImpl::send_broadcast(BlockBroadcast broadcast, int mode) {
if (mode & broadcast_mode_custom) {
send_block_broadcast_to_custom_overlays(broadcast);
}
auto shard = get_shard(broadcast.block_id.shard_full());
if (shard.empty()) {
VLOG(FULL_NODE_WARNING) << "dropping OUT broadcast to unknown shard";
return;
}
if (broadcast.block_id.is_masterchain()) {
if (mode & broadcast_mode_private_block) {
if (!private_block_overlays_.empty()) {
td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_broadcast,
broadcast.clone());
@ -374,7 +373,9 @@ void FullNodeImpl::send_broadcast(BlockBroadcast broadcast, bool custom_overlays
td::actor::send_closure(fast_sync_overlay, &FullNodeFastSyncOverlay::send_broadcast, broadcast.clone());
}
}
td::actor::send_closure(shard, &FullNodeShard::send_broadcast, std::move(broadcast));
if (mode & broadcast_mode_public) {
td::actor::send_closure(shard, &FullNodeShard::send_broadcast, std::move(broadcast));
}
}
void FullNodeImpl::download_block(BlockIdExt id, td::uint32 priority, td::Timestamp timeout,
@ -625,8 +626,8 @@ void FullNodeImpl::start_up() {
td::actor::send_closure(id_, &FullNodeImpl::send_block_candidate, block_id, cc_seqno, validator_set_hash,
std::move(data));
}
void send_broadcast(BlockBroadcast broadcast, bool custom_overlays_only) override {
td::actor::send_closure(id_, &FullNodeImpl::send_broadcast, std::move(broadcast), custom_overlays_only);
void send_broadcast(BlockBroadcast broadcast, int mode) override {
td::actor::send_closure(id_, &FullNodeImpl::send_broadcast, std::move(broadcast), mode);
}
void download_block(BlockIdExt id, td::uint32 priority, td::Timestamp timeout,
td::Promise<ReceivedBlock> promise) override {

1
validator/full-node.h
View file

@ -106,6 +106,7 @@ class FullNode : public td::actor::Actor {
static constexpr td::uint64 max_state_size() {
return 4ull << 30;
}
enum { broadcast_mode_public = 1, broadcast_mode_private_block = 2, broadcast_mode_custom = 4 };
static td::actor::ActorOwn<FullNode> create(
ton::PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, FullNodeConfig config,

2
validator/full-node.hpp
View file

@ -69,7 +69,7 @@ class FullNodeImpl : public FullNode {
void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqnp, td::BufferSlice data);
void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash,
td::BufferSlice data);
void send_broadcast(BlockBroadcast broadcast, bool custom_overlays_only);
void send_broadcast(BlockBroadcast broadcast, int mode);
void download_block(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise<ReceivedBlock> promise);
void download_zero_state(BlockIdExt id, td::uint32 priority, td::Timestamp timeout,
td::Promise<td::BufferSlice> promise);

13
validator/impl/accept-block.cpp
View file

@ -41,7 +41,7 @@ using namespace std::literals::string_literals;
AcceptBlockQuery::AcceptBlockQuery(BlockIdExt id, td::Ref<BlockData> data, std::vector<BlockIdExt> prev,
td::Ref<ValidatorSet> validator_set, td::Ref<BlockSignatureSet> signatures,
td::Ref<BlockSignatureSet> approve_signatures, bool send_broadcast, bool apply,
td::Ref<BlockSignatureSet> approve_signatures, int send_broadcast_mode, bool apply,
td::actor::ActorId<ValidatorManager> manager, td::Promise<td::Unit> promise)
: id_(id)
, data_(std::move(data))
@ -51,7 +51,7 @@ AcceptBlockQuery::AcceptBlockQuery(BlockIdExt id, td::Ref<BlockData> data, std::
, approve_signatures_(std::move(approve_signatures))
, is_fake_(false)
, is_fork_(false)
, send_broadcast_(send_broadcast)
, send_broadcast_mode_(send_broadcast_mode)
, apply_(apply)
, manager_(manager)
, promise_(std::move(promise))
@ -73,7 +73,6 @@ AcceptBlockQuery::AcceptBlockQuery(AcceptBlockQuery::IsFake fake, BlockIdExt id,
, validator_set_(std::move(validator_set))
, is_fake_(true)
, is_fork_(false)
, send_broadcast_(false)
, manager_(manager)
, promise_(std::move(promise))
, perf_timer_("acceptblock", 0.1, [manager](double duration) {
@ -91,7 +90,6 @@ AcceptBlockQuery::AcceptBlockQuery(ForceFork ffork, BlockIdExt id, td::Ref<Block
, data_(std::move(data))
, is_fake_(true)
, is_fork_(true)
, send_broadcast_(false)
, manager_(manager)
, promise_(std::move(promise))
, perf_timer_("acceptblock", 0.1, [manager](double duration) {
@ -937,6 +935,10 @@ void AcceptBlockQuery::written_block_info_2() {
}
void AcceptBlockQuery::applied() {
if (send_broadcast_mode_ == 0) {
finish_query();
return;
}
BlockBroadcast b;
b.data = data_->data();
b.block_id = id_;
@ -956,8 +958,7 @@ void AcceptBlockQuery::applied() {
}
// do not wait for answer
td::actor::send_closure_later(manager_, &ValidatorManager::send_block_broadcast, std::move(b),
/* custom_overlays_only = */ !send_broadcast_);
td::actor::send_closure_later(manager_, &ValidatorManager::send_block_broadcast, std::move(b), send_broadcast_mode_);
finish_query();
}

4
validator/impl/accept-block.hpp
View file

@ -50,7 +50,7 @@ class AcceptBlockQuery : public td::actor::Actor {
struct ForceFork {};
AcceptBlockQuery(BlockIdExt id, td::Ref<BlockData> data, std::vector<BlockIdExt> prev,
td::Ref<ValidatorSet> validator_set, td::Ref<BlockSignatureSet> signatures,
td::Ref<BlockSignatureSet> approve_signatures, bool send_broadcast, bool apply,
td::Ref<BlockSignatureSet> approve_signatures, int send_broadcast_mode, bool apply,
td::actor::ActorId<ValidatorManager> manager, td::Promise<td::Unit> promise);
AcceptBlockQuery(IsFake fake, BlockIdExt id, td::Ref<BlockData> data, std::vector<BlockIdExt> prev,
td::Ref<ValidatorSet> validator_set, td::actor::ActorId<ValidatorManager> manager,
@ -99,7 +99,7 @@ class AcceptBlockQuery : public td::actor::Actor {
Ref<BlockSignatureSetQ> approve_signatures_;
bool is_fake_;
bool is_fork_;
bool send_broadcast_;
int send_broadcast_mode_{0};
bool apply_ = true;
bool ancestors_split_{false}, is_key_block_{false};
td::Timestamp timeout_ = td::Timestamp::in(600.0);

4
validator/impl/fabric.cpp
View file

@ -131,11 +131,11 @@ td::Result<td::Ref<IhrMessage>> create_ihr_message(td::BufferSlice data) {
void run_accept_block_query(BlockIdExt id, td::Ref<BlockData> data, std::vector<BlockIdExt> prev,
td::Ref<ValidatorSet> validator_set, td::Ref<BlockSignatureSet> signatures,
td::Ref<BlockSignatureSet> approve_signatures, bool send_broadcast, bool apply,
td::Ref<BlockSignatureSet> approve_signatures, int send_broadcast_mode, bool apply,
td::actor::ActorId<ValidatorManager> manager, td::Promise<td::Unit> promise) {
td::actor::create_actor<AcceptBlockQuery>(
PSTRING() << "accept" << id.id.to_str(), id, std::move(data), prev, std::move(validator_set),
std::move(signatures), std::move(approve_signatures), send_broadcast, apply, manager, std::move(promise))
std::move(signatures), std::move(approve_signatures), send_broadcast_mode, apply, manager, std::move(promise))
.release();
}

4
validator/interfaces/validator-manager.h
View file

@ -104,6 +104,8 @@ class ValidatorManager : public ValidatorManagerInterface {
virtual void set_block_candidate(BlockIdExt id, BlockCandidate candidate, CatchainSeqno cc_seqno,
td::uint32 validator_set_hash, td::Promise<td::Unit> promise) = 0;
virtual void send_block_candidate_broadcast(BlockIdExt id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash,
td::BufferSlice data) = 0;
virtual void wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority,
td::Timestamp timeout, td::Promise<td::Ref<ShardState>> promise) = 0;
@ -143,7 +145,7 @@ class ValidatorManager : public ValidatorManagerInterface {
virtual void send_external_message(td::Ref<ExtMessage> message) = 0;
virtual void send_ihr_message(td::Ref<IhrMessage> message) = 0;
virtual void send_top_shard_block_description(td::Ref<ShardTopBlockDescription> desc) = 0;
virtual void send_block_broadcast(BlockBroadcast broadcast, bool custom_overlays_only) = 0;
virtual void send_block_broadcast(BlockBroadcast broadcast, int mode) = 0;
virtual void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector<BlockIdExt> blocks,
block::ImportedMsgQueueLimits limits,
td::Promise<std::vector<td::Ref<OutMsgQueueProof>>> promise) = 0;

5
validator/manager-disk.cpp
View file

@ -785,6 +785,11 @@ void ValidatorManagerImpl::set_block_candidate(BlockIdExt id, BlockCandidate can
td::actor::send_closure(db_, &Db::store_block_candidate, std::move(candidate), std::move(promise));
}
void ValidatorManagerImpl::send_block_candidate_broadcast(BlockIdExt id, CatchainSeqno cc_seqno,
td::uint32 validator_set_hash, td::BufferSlice data) {
callback_->send_block_candidate(id, cc_seqno, validator_set_hash, std::move(data));
}
void ValidatorManagerImpl::write_handle(BlockHandle handle, td::Promise<td::Unit> promise) {
td::actor::send_closure(db_, &Db::store_block_handle, std::move(handle), std::move(promise));
}

4
validator/manager-disk.hpp
View file

@ -189,6 +189,8 @@ class ValidatorManagerImpl : public ValidatorManager {
void set_block_candidate(BlockIdExt id, BlockCandidate candidate, CatchainSeqno cc_seqno,
td::uint32 validator_set_hash, td::Promise<td::Unit> promise) override;
void send_block_candidate_broadcast(BlockIdExt id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash,
td::BufferSlice data) override;
void wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority, td::Timestamp timeout,
td::Promise<td::Ref<ShardState>> promise) override;
@ -263,7 +265,7 @@ class ValidatorManagerImpl : public ValidatorManager {
new_ihr_message(message->serialize());
}
void send_top_shard_block_description(td::Ref<ShardTopBlockDescription> desc) override;
void send_block_broadcast(BlockBroadcast broadcast, bool custom_overlays_only) override {
void send_block_broadcast(BlockBroadcast broadcast, int mode) override {
}
void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector<BlockIdExt> blocks,
block::ImportedMsgQueueLimits limits,

6
validator/manager-hardfork.hpp
View file

@ -231,6 +231,10 @@ class ValidatorManagerImpl : public ValidatorManager {
td::uint32 validator_set_hash, td::Promise<td::Unit> promise) override {
promise.set_value(td::Unit());
}
void send_block_candidate_broadcast(BlockIdExt id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash,
td::BufferSlice data) {
callback_->send_block_candidate(id, cc_seqno, validator_set_hash, std::move(data));
}
void wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority, td::Timestamp timeout,
td::Promise<td::Ref<ShardState>> promise) override;
@ -331,7 +335,7 @@ class ValidatorManagerImpl : public ValidatorManager {
void send_top_shard_block_description(td::Ref<ShardTopBlockDescription> desc) override {
UNREACHABLE();
}
void send_block_broadcast(BlockBroadcast broadcast, bool custom_overlays_only) override {
void send_block_broadcast(BlockBroadcast broadcast, int mode) override {
}
void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector<BlockIdExt> blocks,
block::ImportedMsgQueueLimits limits,

2
validator/manager-init.cpp
View file

@ -227,7 +227,7 @@ void ValidatorManagerMasterchainReiniter::choose_masterchain_state() {
}
if (!p || ValidatorManager::is_persistent_state(h->unix_time(), p->unix_time())) {
auto ttl = ValidatorManager::persistent_state_ttl(h->unix_time());
double time_to_download = 3600 * 3;
double time_to_download = 3600 * 8;
if (ttl > td::Clocks::system() + time_to_download) {
handle = h;
break;

10
validator/manager.cpp
View file

@ -1428,11 +1428,15 @@ void ValidatorManagerImpl::set_block_candidate(BlockIdExt id, BlockCandidate can
}
if (!id.is_masterchain()) {
add_cached_block_candidate(ReceivedBlock{id, candidate.data.clone()});
callback_->send_block_candidate(id, cc_seqno, validator_set_hash, candidate.data.clone());
}
td::actor::send_closure(db_, &Db::store_block_candidate, std::move(candidate), std::move(promise));
}
void ValidatorManagerImpl::send_block_candidate_broadcast(BlockIdExt id, CatchainSeqno cc_seqno,
td::uint32 validator_set_hash, td::BufferSlice data) {
callback_->send_block_candidate(id, cc_seqno, validator_set_hash, std::move(data));
}
void ValidatorManagerImpl::write_handle(BlockHandle handle, td::Promise<td::Unit> promise) {
auto P = td::PromiseCreator::lambda(
[SelfId = actor_id(this), handle, promise = std::move(promise)](td::Result<td::Unit> R) mutable {
@ -1761,8 +1765,8 @@ void ValidatorManagerImpl::send_top_shard_block_description(td::Ref<ShardTopBloc
}
}
void ValidatorManagerImpl::send_block_broadcast(BlockBroadcast broadcast, bool custom_overlays_only) {
callback_->send_broadcast(std::move(broadcast), custom_overlays_only);
void ValidatorManagerImpl::send_block_broadcast(BlockBroadcast broadcast, int mode) {
callback_->send_broadcast(std::move(broadcast), mode);
}
void ValidatorManagerImpl::send_get_out_msg_queue_proof_request(

4
validator/manager.hpp
View file

@ -454,6 +454,8 @@ class ValidatorManagerImpl : public ValidatorManager {
void set_block_candidate(BlockIdExt id, BlockCandidate candidate, CatchainSeqno cc_seqno,
td::uint32 validator_set_hash, td::Promise<td::Unit> promise) override;
void send_block_candidate_broadcast(BlockIdExt id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash,
td::BufferSlice data) override;
void wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority, td::Timestamp timeout,
td::Promise<td::Ref<ShardState>> promise) override;
@ -518,7 +520,7 @@ class ValidatorManagerImpl : public ValidatorManager {
void send_external_message(td::Ref<ExtMessage> message) override;
void send_ihr_message(td::Ref<IhrMessage> message) override;
void send_top_shard_block_description(td::Ref<ShardTopBlockDescription> desc) override;
void send_block_broadcast(BlockBroadcast broadcast, bool custom_overlays_only) override;
void send_block_broadcast(BlockBroadcast broadcast, int mode) override;
void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector<BlockIdExt> blocks,
block::ImportedMsgQueueLimits limits,
td::Promise<std::vector<td::Ref<OutMsgQueueProof>>> promise) override;

28
validator/net/download-state.cpp
View file

@ -52,12 +52,7 @@ DownloadState::DownloadState(BlockIdExt block_id, BlockIdExt masterchain_block_i
void DownloadState::abort_query(td::Status reason) {
if (promise_) {
if (reason.code() == ErrorCode::notready || reason.code() == ErrorCode::timeout) {
VLOG(FULL_NODE_DEBUG) << "failed to download state " << block_id_ << " from " << download_from_ << ": " << reason;
} else {
VLOG(FULL_NODE_NOTICE) << "failed to download state " << block_id_ << " from " << download_from_ << ": "
<< reason;
}
LOG(WARNING) << "failed to download state " << block_id_.to_str() << " from " << download_from_ << ": " << reason;
promise_.set_error(std::move(reason));
}
stop();
@ -77,6 +72,19 @@ void DownloadState::finish_query() {
void DownloadState::start_up() {
alarm_timestamp() = timeout_;
td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state, block_id_,
masterchain_block_id_,
[SelfId = actor_id(this), block_id = block_id_](td::Result<td::BufferSlice> R) {
if (R.is_error()) {
td::actor::send_closure(SelfId, &DownloadState::get_block_handle);
} else {
LOG(WARNING) << "got block state from disk: " << block_id.to_str();
td::actor::send_closure(SelfId, &DownloadState::got_block_state, R.move_as_ok());
}
});
}
void DownloadState::get_block_handle() {
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<BlockHandle> R) {
if (R.is_error()) {
td::actor::send_closure(SelfId, &DownloadState::abort_query, R.move_as_error());
@ -115,7 +123,7 @@ void DownloadState::got_block_handle(BlockHandle handle) {
void DownloadState::got_node_to_download(adnl::AdnlNodeIdShort node) {
download_from_ = node;
LOG(INFO) << "downloading state " << block_id_.to_str() << " from " << download_from_;
LOG(WARNING) << "downloading state " << block_id_.to_str() << " from " << download_from_;
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::BufferSlice> R) mutable {
if (R.is_error()) {
@ -192,8 +200,8 @@ void DownloadState::got_block_state_part(td::BufferSlice data, td::uint32 reques
double elapsed = prev_logged_timer_.elapsed();
if (elapsed > 10.0) {
prev_logged_timer_ = td::Timer();
LOG(INFO) << "downloading state " << block_id_.to_str() << ": total=" << sum_ << " ("
<< td::format::as_size((td::uint64)(double(sum_ - prev_logged_sum_) / elapsed)) << "/s)";
LOG(WARNING) << "downloading state " << block_id_.to_str() << ": " << td::format::as_size(sum_) << " ("
<< td::format::as_size((td::uint64)(double(sum_ - prev_logged_sum_) / elapsed)) << "/s)";
prev_logged_sum_ = sum_;
}
@ -234,7 +242,7 @@ void DownloadState::got_block_state_part(td::BufferSlice data, td::uint32 reques
void DownloadState::got_block_state(td::BufferSlice data) {
state_ = std::move(data);
LOG(INFO) << "finished downloading state " << block_id_.to_str() << ": total=" << sum_;
LOG(WARNING) << "finished downloading state " << block_id_.to_str() << ": " << td::format::as_size(state_.size());
finish_query();
}

1
validator/net/download-state.hpp
View file

@ -43,6 +43,7 @@ class DownloadState : public td::actor::Actor {
void finish_query();
void start_up() override;
void get_block_handle();
void got_block_handle(BlockHandle handle);
void got_node_to_download(adnl::AdnlNodeIdShort node);
void got_block_state_description(td::BufferSlice data_description);

43
validator/shard-client.cpp
View file

@ -74,29 +74,30 @@ void ShardClient::got_init_state_from_db(td::Ref<MasterchainState> state) {
}
void ShardClient::start_up_init_mode() {
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Unit> R) {
R.ensure();
td::actor::send_closure(SelfId, &ShardClient::apply_all_shards);
});
td::MultiPromise mp;
auto ig = mp.init_guard();
ig.add_promise(std::move(P));
auto vec = masterchain_state_->get_shards();
for (auto &shard : vec) {
if (opts_->need_monitor(shard->shard(), masterchain_state_)) {
auto P = td::PromiseCreator::lambda([promise = ig.get_promise()](td::Result<td::Ref<ShardState>> R) mutable {
R.ensure();
promise.set_value(td::Unit());
});
td::actor::create_actor<DownloadShardState>("downloadstate", shard->top_block_id(),
masterchain_block_handle_->id(), 2, manager_,
td::Timestamp::in(3600 * 3), std::move(P))
.release();
std::vector<BlockIdExt> shards;
for (const auto& s : masterchain_state_->get_shards()) {
if (opts_->need_monitor(s->shard(), masterchain_state_)) {
shards.push_back(s->top_block_id());
}
}
download_shard_states(masterchain_block_handle_->id(), std::move(shards), 0);
}
void ShardClient::download_shard_states(BlockIdExt masterchain_block_id, std::vector<BlockIdExt> shards, size_t idx) {
if (idx >= shards.size()) {
LOG(WARNING) << "downloaded all shard states";
applied_all_shards();
return;
}
BlockIdExt block_id = shards[idx];
td::actor::create_actor<DownloadShardState>(
"downloadstate", block_id, masterchain_block_handle_->id(), 2, manager_, td::Timestamp::in(3600 * 5),
[=, SelfId = actor_id(this), shards = std::move(shards)](td::Result<td::Ref<ShardState>> R) {
R.ensure();
td::actor::send_closure(SelfId, &ShardClient::download_shard_states, masterchain_block_id, std::move(shards),
idx + 1);
})
.release();
}
void ShardClient::applied_all_shards() {

9
validator/shard-client.hpp
View file

@ -64,19 +64,12 @@ class ShardClient : public td::actor::Actor {
void start_up() override;
void start_up_init_mode();
void start_up_init_mode_finished();
void download_shard_states(BlockIdExt masterchain_block_id, std::vector<BlockIdExt> shards, size_t idx);
void start();
void got_state_from_db(BlockIdExt masterchain_block_id);
void got_init_handle_from_db(BlockHandle handle);
void got_init_state_from_db(td::Ref<MasterchainState> state);
void im_download_shard_state(BlockIdExt block_id, td::Promise<td::Unit> promise);
void im_downloaded_zero_state(BlockIdExt block_id, td::BufferSlice data, td::Promise<td::Unit> promise);
void im_downloaded_proof_link(BlockIdExt block_id, td::BufferSlice data, td::Promise<td::Unit> promise);
void im_checked_proof_link(BlockIdExt block_id, td::Promise<td::Unit> promise);
void im_downloaded_shard_state(BlockIdExt block_id, td::Promise<td::Unit> promise);
void im_got_shard_handle(BlockHandle handle, td::Promise<td::Unit> promise);
void new_masterchain_block_id(BlockIdExt masterchain_block_id);
void got_masterchain_block_handle(BlockHandle handle);
void download_masterchain_state();

130
validator/validator-group.cpp
View file

@ -18,6 +18,7 @@
*/
#include "validator-group.hpp"
#include "fabric.h"
#include "full-node-master.hpp"
#include "ton/ton-io.hpp"
#include "td/utils/overloaded.h"
#include "common/delay.h"
@ -30,8 +31,14 @@ namespace ton {
namespace validator {
static bool need_send_candidate_broadcast(const validatorsession::BlockSourceInfo &source_info, bool is_masterchain) {
return source_info.first_block_round == source_info.round && source_info.source_priority == 0 && !is_masterchain;
}
void ValidatorGroup::generate_block_candidate(
td::uint32 round_id, td::Promise<validatorsession::ValidatorSession::GeneratedCandidate> promise) {
validatorsession::BlockSourceInfo source_info,
td::Promise<validatorsession::ValidatorSession::GeneratedCandidate> promise) {
td::uint32 round_id = source_info.round;
if (round_id > last_known_round_id_) {
last_known_round_id_ = round_id;
}
@ -53,14 +60,16 @@ void ValidatorGroup::generate_block_candidate(
cached_collated_block_->promises.push_back(promise.wrap([](BlockCandidate &&res) {
return validatorsession::ValidatorSession::GeneratedCandidate{std::move(res), false};
}));
td::Promise<BlockCandidate> P = [SelfId = actor_id(this),
cache = cached_collated_block_](td::Result<BlockCandidate> R) {
td::actor::send_closure(SelfId, &ValidatorGroup::generated_block_candidate, std::move(cache), std::move(R));
td::Promise<BlockCandidate> P = [SelfId = actor_id(this), cache = cached_collated_block_,
source_info](td::Result<BlockCandidate> R) {
td::actor::send_closure(SelfId, &ValidatorGroup::generated_block_candidate, source_info, std::move(cache),
std::move(R));
};
collate_block(round_id, td::Timestamp::in(10.0), std::move(P));
collate_block(source_info, td::Timestamp::in(10.0), std::move(P));
}
void ValidatorGroup::generated_block_candidate(std::shared_ptr<CachedCollatedBlock> cache,
void ValidatorGroup::generated_block_candidate(validatorsession::BlockSourceInfo source_info,
std::shared_ptr<CachedCollatedBlock> cache,
td::Result<BlockCandidate> R) {
if (R.is_error()) {
for (auto &p : cache->promises) {
@ -72,6 +81,9 @@ void ValidatorGroup::generated_block_candidate(std::shared_ptr<CachedCollatedBlo
} else {
auto candidate = R.move_as_ok();
add_available_block_candidate(candidate.pubkey.as_bits256(), candidate.id, candidate.collated_file_hash);
if (need_send_candidate_broadcast(source_info, shard_.is_masterchain())) {
send_block_candidate_broadcast(candidate.id, candidate.data.clone());
}
cache->result = std::move(candidate);
for (auto &p : cache->promises) {
p.set_value(cache->result.value().clone());
@ -80,8 +92,9 @@ void ValidatorGroup::generated_block_candidate(std::shared_ptr<CachedCollatedBlo
cache->promises.clear();
}
void ValidatorGroup::validate_block_candidate(td::uint32 round_id, BlockCandidate block,
void ValidatorGroup::validate_block_candidate(validatorsession::BlockSourceInfo source_info, BlockCandidate block,
td::Promise<std::pair<UnixTime, bool>> promise) {
td::uint32 round_id = source_info.round;
if (round_id > last_known_round_id_) {
last_known_round_id_ = round_id;
}
@ -100,7 +113,7 @@ void ValidatorGroup::validate_block_candidate(td::uint32 round_id, BlockCandidat
return;
}
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), round_id, block = block.clone(),
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), source_info, block = block.clone(),
promise = std::move(promise)](td::Result<ValidateCandidateResult> R) mutable {
if (R.is_error()) {
auto S = R.move_as_error();
@ -108,19 +121,22 @@ void ValidatorGroup::validate_block_candidate(td::uint32 round_id, BlockCandidat
LOG(ERROR) << "failed to validate candidate: " << S;
}
delay_action(
[SelfId, round_id, block = std::move(block), promise = std::move(promise)]() mutable {
td::actor::send_closure(SelfId, &ValidatorGroup::validate_block_candidate, round_id, std::move(block),
std::move(promise));
[SelfId, source_info, block = std::move(block), promise = std::move(promise)]() mutable {
td::actor::send_closure(SelfId, &ValidatorGroup::validate_block_candidate, std::move(source_info),
std::move(block), std::move(promise));
},
td::Timestamp::in(0.1));
} else {
auto v = R.move_as_ok();
v.visit(td::overloaded(
[&](UnixTime ts) {
td::actor::send_closure(SelfId, &ValidatorGroup::update_approve_cache, block_to_cache_key(block),
ts);
td::actor::send_closure(SelfId, &ValidatorGroup::update_approve_cache, block_to_cache_key(block), ts);
td::actor::send_closure(SelfId, &ValidatorGroup::add_available_block_candidate, block.pubkey.as_bits256(),
block.id, block.collated_file_hash);
if (need_send_candidate_broadcast(source_info, block.id.is_masterchain())) {
td::actor::send_closure(SelfId, &ValidatorGroup::send_block_candidate_broadcast, block.id,
block.data.clone());
}
promise.set_value({ts, false});
},
[&](CandidateReject reject) {
@ -142,13 +158,14 @@ void ValidatorGroup::update_approve_cache(CacheKey key, UnixTime value) {
approved_candidates_cache_[key] = value;
}
void ValidatorGroup::accept_block_candidate(td::uint32 round_id, PublicKeyHash src, td::BufferSlice block_data,
void ValidatorGroup::accept_block_candidate(validatorsession::BlockSourceInfo source_info, td::BufferSlice block_data,
RootHash root_hash, FileHash file_hash,
std::vector<BlockSignature> signatures,
std::vector<BlockSignature> approve_signatures,
validatorsession::ValidatorSessionStats stats,
td::Promise<td::Unit> promise) {
stats.cc_seqno = validator_set_->get_catchain_seqno();
td::uint32 round_id = source_info.round;
if (round_id >= last_known_round_id_) {
last_known_round_id_ = round_id + 1;
}
@ -167,8 +184,23 @@ void ValidatorGroup::accept_block_candidate(td::uint32 round_id, PublicKeyHash s
td::actor::send_closure(manager_, &ValidatorManager::log_validator_session_stats, next_block_id, std::move(stats));
auto block =
block_data.size() > 0 ? create_block(next_block_id, std::move(block_data)).move_as_ok() : td::Ref<BlockData>{};
// Creator of the block sends broadcast to public overlays
// Creator of the block sends broadcast to private block overlay unless candidate broadcast was sent
// Any node sends broadcast to custom overlays unless candidate broadcast was sent
int send_broadcast_mode = 0;
bool sent_candidate = sent_candidate_broadcasts_.contains(block->block_id());
if (source_info.source.compute_short_id() == local_id_) {
send_broadcast_mode |= fullnode::FullNode::broadcast_mode_public;
if (!sent_candidate) {
send_broadcast_mode |= fullnode::FullNode::broadcast_mode_private_block;
}
}
if (!sent_candidate) {
send_broadcast_mode |= fullnode::FullNode::broadcast_mode_custom;
}
accept_block_query(next_block_id, std::move(block), std::move(prev_block_ids_), std::move(sig_set),
std::move(approve_sig_set), src == local_id_, std::move(promise));
std::move(approve_sig_set), send_broadcast_mode, std::move(promise));
prev_block_ids_ = std::vector<BlockIdExt>{next_block_id};
cached_collated_block_ = nullptr;
approved_candidates_cache_.clear();
@ -177,7 +209,7 @@ void ValidatorGroup::accept_block_candidate(td::uint32 round_id, PublicKeyHash s
void ValidatorGroup::accept_block_query(BlockIdExt block_id, td::Ref<BlockData> block, std::vector<BlockIdExt> prev,
td::Ref<BlockSignatureSet> sig_set, td::Ref<BlockSignatureSet> approve_sig_set,
bool send_broadcast, td::Promise<td::Unit> promise, bool is_retry) {
int send_broadcast_mode, td::Promise<td::Unit> promise, bool is_retry) {
auto P = td::PromiseCreator::lambda([=, SelfId = actor_id(this),
promise = std::move(promise)](td::Result<td::Unit> R) mutable {
if (R.is_error()) {
@ -187,7 +219,7 @@ void ValidatorGroup::accept_block_query(BlockIdExt block_id, td::Ref<BlockData>
}
LOG_CHECK(R.error().code() == ErrorCode::timeout || R.error().code() == ErrorCode::notready) << R.move_as_error();
td::actor::send_closure(SelfId, &ValidatorGroup::accept_block_query, block_id, std::move(block),
std::move(prev), std::move(sig_set), std::move(approve_sig_set), send_broadcast,
std::move(prev), std::move(sig_set), std::move(approve_sig_set), send_broadcast_mode,
std::move(promise), true);
} else {
promise.set_value(R.move_as_ok());
@ -195,7 +227,7 @@ void ValidatorGroup::accept_block_query(BlockIdExt block_id, td::Ref<BlockData>
});
run_accept_block_query(block_id, std::move(block), std::move(prev), validator_set_, std::move(sig_set),
std::move(approve_sig_set), send_broadcast, monitoring_shard_, manager_, std::move(P));
std::move(approve_sig_set), send_broadcast_mode, monitoring_shard_, manager_, std::move(P));
}
void ValidatorGroup::skip_round(td::uint32 round_id) {
@ -231,8 +263,9 @@ std::unique_ptr<validatorsession::ValidatorSession::Callback> ValidatorGroup::ma
public:
Callback(td::actor::ActorId<ValidatorGroup> id) : id_(id) {
}
void on_candidate(td::uint32 round, PublicKey source, validatorsession::ValidatorSessionRootHash root_hash,
td::BufferSlice data, td::BufferSlice collated_data,
void on_candidate(validatorsession::BlockSourceInfo source_info,
validatorsession::ValidatorSessionRootHash root_hash, td::BufferSlice data,
td::BufferSlice collated_data,
td::Promise<validatorsession::ValidatorSession::CandidateDecision> promise) override {
auto P =
td::PromiseCreator::lambda([promise = std::move(promise)](td::Result<std::pair<td::uint32, bool>> R) mutable {
@ -247,18 +280,19 @@ std::unique_ptr<validatorsession::ValidatorSession::Callback> ValidatorGroup::ma
}
});
BlockCandidate candidate{Ed25519_PublicKey{source.ed25519_value().raw()},
BlockCandidate candidate{Ed25519_PublicKey{source_info.source.ed25519_value().raw()},
BlockIdExt{0, 0, 0, root_hash, sha256_bits256(data.as_slice())},
sha256_bits256(collated_data.as_slice()), data.clone(), collated_data.clone()};
td::actor::send_closure(id_, &ValidatorGroup::validate_block_candidate, round, std::move(candidate),
std::move(P));
td::actor::send_closure(id_, &ValidatorGroup::validate_block_candidate, std::move(source_info),
std::move(candidate), std::move(P));
}
void on_generate_slot(td::uint32 round,
void on_generate_slot(validatorsession::BlockSourceInfo source_info,
td::Promise<validatorsession::ValidatorSession::GeneratedCandidate> promise) override {
td::actor::send_closure(id_, &ValidatorGroup::generate_block_candidate, round, std::move(promise));
td::actor::send_closure(id_, &ValidatorGroup::generate_block_candidate, std::move(source_info),
std::move(promise));
}
void on_block_committed(td::uint32 round, PublicKey source, validatorsession::ValidatorSessionRootHash root_hash,
void on_block_committed(validatorsession::BlockSourceInfo source_info, validatorsession::ValidatorSessionRootHash root_hash,
validatorsession::ValidatorSessionFileHash file_hash, td::BufferSlice data,
std::vector<std::pair<PublicKeyHash, td::BufferSlice>> signatures,
std::vector<std::pair<PublicKeyHash, td::BufferSlice>> approve_signatures,
@ -272,9 +306,9 @@ std::unique_ptr<validatorsession::ValidatorSession::Callback> ValidatorGroup::ma
approve_sigs.emplace_back(BlockSignature{sig.first.bits256_value(), std::move(sig.second)});
}
auto P = td::PromiseCreator::lambda([](td::Result<td::Unit>) {});
td::actor::send_closure(id_, &ValidatorGroup::accept_block_candidate, round, source.compute_short_id(),
std::move(data), root_hash, file_hash, std::move(sigs), std::move(approve_sigs),
std::move(stats), std::move(P));
td::actor::send_closure(id_, &ValidatorGroup::accept_block_candidate, std::move(source_info), std::move(data),
root_hash, file_hash, std::move(sigs), std::move(approve_sigs), std::move(stats),
std::move(P));
}
void on_block_skipped(td::uint32 round) override {
td::actor::send_closure(id_, &ValidatorGroup::skip_round, round);
@ -363,7 +397,7 @@ void ValidatorGroup::start(std::vector<BlockIdExt> prev, BlockIdExt min_masterch
auto block =
p.block.size() > 0 ? create_block(next_block_id, std::move(p.block)).move_as_ok() : td::Ref<BlockData>{};
accept_block_query(next_block_id, std::move(block), std::move(prev_block_ids_), std::move(p.sigs),
std::move(p.approve_sigs), false, std::move(p.promise));
std::move(p.approve_sigs), 0, std::move(p.promise));
prev_block_ids_ = std::vector<BlockIdExt>{next_block_id};
}
postponed_accept_.clear();
@ -463,9 +497,9 @@ void ValidatorGroup::get_validator_group_info_for_litequery_cont(
promise.set_result(std::move(result));
}
void ValidatorGroup::collate_block(td::uint32 round_id, td::Timestamp timeout, td::Promise<BlockCandidate> promise,
unsigned max_retries) {
if (round_id < last_known_round_id_) {
void ValidatorGroup::collate_block(validatorsession::BlockSourceInfo source_info, td::Timestamp timeout,
td::Promise<BlockCandidate> promise, unsigned max_retries) {
if (source_info.round < last_known_round_id_) {
promise.set_error(td::Status::Error("too old"));
return;
}
@ -528,7 +562,7 @@ void ValidatorGroup::collate_block(td::uint32 round_id, td::Timestamp timeout, t
LOG(WARNING) << "collate query for " << next_block_id.to_str() << ": " << R.error() << ", time=" << timer.elapsed()
<< "s, " << (retry ? "retrying" : "giving up");
if (retry) {
td::actor::send_closure(SelfId, &ValidatorGroup::collate_block, round_id, timeout, std::move(promise),
td::actor::send_closure(SelfId, &ValidatorGroup::collate_block, source_info, timeout, std::move(promise),
max_retries - 1);
} else {
promise.set_result(td::Status::Error(ErrorCode::timeout, "timeout"));
@ -549,7 +583,7 @@ void ValidatorGroup::collate_block(td::uint32 round_id, td::Timestamp timeout, t
promise.set_error(R.move_as_error_prefix("rldp query failed: "));
return;
}
td::actor::send_closure(SelfId, &ValidatorGroup::receive_collate_query_response, round_id, R.move_as_ok(),
td::actor::send_closure(SelfId, &ValidatorGroup::receive_collate_query_response, source_info, R.move_as_ok(),
trusted_collator, std::move(promise));
});
LOG(INFO) << "sending collate query for " << next_block_id.to_str() << ": send to " << collator_adnl_id;
@ -560,24 +594,24 @@ void ValidatorGroup::collate_block(td::uint32 round_id, td::Timestamp timeout, t
std::move(P), timeout, std::move(query), max_answer_size);
}
void ValidatorGroup::receive_collate_query_response(td::uint32 round_id, td::BufferSlice data, bool trusted_collator,
td::Promise<BlockCandidate> promise) {
if (round_id < last_known_round_id_) {
void ValidatorGroup::receive_collate_query_response(validatorsession::BlockSourceInfo source_info, td::BufferSlice data,
bool trusted_collator, td::Promise<BlockCandidate> promise) {
if (source_info.round < last_known_round_id_) {
promise.set_error(td::Status::Error("too old"));
return;
}
TRY_RESULT_PROMISE(promise, f, fetch_tl_object<ton_api::collatorNode_GenerateBlockResult>(data, true));
td::Result<BlockCandidate> res;
ton_api::downcast_call(*f, td::overloaded(
[&](ton_api::collatorNode_generateBlockError &r) {
td::Status error = td::Status::Error(r.code_, r.message_);
res = error.move_as_error_prefix("collate query: ");
},
[&](ton_api::collatorNode_generateBlockSuccess &r) {
res = CollatorNode::deserialize_candidate(
std::move(r.candidate_),
config_.max_block_size + config_.max_collated_data_size + 1024);
}));
[&](ton_api::collatorNode_generateBlockError &r) {
td::Status error = td::Status::Error(r.code_, r.message_);
res = error.move_as_error_prefix("collate query: ");
},
[&](ton_api::collatorNode_generateBlockSuccess &r) {
res = CollatorNode::deserialize_candidate(
std::move(r.candidate_),
config_.max_block_size + config_.max_collated_data_size + 1024);
}));
TRY_RESULT_PROMISE(promise, candidate, std::move(res));
if (candidate.pubkey.as_bits256() != local_id_full_.ed25519_value().raw()) {
promise.set_error(td::Status::Error("collate query: block candidate source mismatch"));
@ -600,7 +634,7 @@ void ValidatorGroup::receive_collate_query_response(td::uint32 round_id, td::Buf
}
promise.set_result(std::move(candidate));
});
validate_block_candidate(round_id, std::move(candidate), std::move(P));
validate_block_candidate(source_info, std::move(candidate), std::move(P));
}
} // namespace validator

29
validator/validator-group.hpp
View file

@ -34,18 +34,18 @@ class ValidatorManager;
class ValidatorGroup : public td::actor::Actor {
public:
void generate_block_candidate(td::uint32 round_id,
void generate_block_candidate(validatorsession::BlockSourceInfo source_info,
td::Promise<validatorsession::ValidatorSession::GeneratedCandidate> promise);
void validate_block_candidate(td::uint32 round_id, BlockCandidate block,
void validate_block_candidate(validatorsession::BlockSourceInfo source_info, BlockCandidate block,
td::Promise<std::pair<UnixTime, bool>> promise);
void accept_block_candidate(td::uint32 round_id, PublicKeyHash src, td::BufferSlice block, RootHash root_hash,
void accept_block_candidate(validatorsession::BlockSourceInfo source_info, td::BufferSlice block, RootHash root_hash,
FileHash file_hash, std::vector<BlockSignature> signatures,
std::vector<BlockSignature> approve_signatures,
validatorsession::ValidatorSessionStats stats, td::Promise<td::Unit> promise);
void skip_round(td::uint32 round);
void accept_block_query(BlockIdExt block_id, td::Ref<BlockData> block, std::vector<BlockIdExt> prev,
td::Ref<BlockSignatureSet> sigs, td::Ref<BlockSignatureSet> approve_sigs,
bool send_broadcast, td::Promise<td::Unit> promise, bool is_retry = false);
int send_broadcast_mode, td::Promise<td::Unit> promise, bool is_retry = false);
void get_approved_candidate(PublicKey source, RootHash root_hash, FileHash file_hash,
FileHash collated_data_file_hash, td::Promise<BlockCandidate> promise);
BlockIdExt create_next_block_id(RootHash root_hash, FileHash file_hash) const;
@ -97,10 +97,10 @@ class ValidatorGroup : public td::actor::Actor {
private:
std::unique_ptr<validatorsession::ValidatorSession::Callback> make_validator_session_callback();
void collate_block(td::uint32 round_id, td::Timestamp timeout, td::Promise<BlockCandidate> promise,
unsigned max_retries = 4);
void receive_collate_query_response(td::uint32 round_id, td::BufferSlice data, bool trusted_collator,
td::Promise<BlockCandidate> promise);
void collate_block(validatorsession::BlockSourceInfo source_info, td::Timestamp timeout,
td::Promise<BlockCandidate> promise, unsigned max_retries = 4);
void receive_collate_query_response(validatorsession::BlockSourceInfo source_info, td::BufferSlice data,
bool trusted_collator, td::Promise<BlockCandidate> promise);
struct PostponedAccept {
RootHash root_hash;
@ -150,7 +150,8 @@ class ValidatorGroup : public td::actor::Actor {
std::shared_ptr<CachedCollatedBlock> cached_collated_block_;
td::CancellationTokenSource cancellation_token_source_;
void generated_block_candidate(std::shared_ptr<CachedCollatedBlock> cache, td::Result<BlockCandidate> R);
void generated_block_candidate(validatorsession::BlockSourceInfo source_info,
std::shared_ptr<CachedCollatedBlock> cache, td::Result<BlockCandidate> R);
using CacheKey = std::tuple<td::Bits256, BlockIdExt, FileHash, FileHash>;
std::map<CacheKey, UnixTime> approved_candidates_cache_;
@ -171,6 +172,16 @@ class ValidatorGroup : public td::actor::Actor {
void add_available_block_candidate(td::Bits256 source, BlockIdExt id, FileHash collated_data_hash) {
available_block_candidates_.emplace(source, id, collated_data_hash);
}
std::set<BlockIdExt> sent_candidate_broadcasts_;
void send_block_candidate_broadcast(BlockIdExt id, td::BufferSlice data) {
if (sent_candidate_broadcasts_.insert(id).second) {
td::actor::send_closure(manager_, &ValidatorManager::send_block_candidate_broadcast, id,
validator_set_->get_catchain_seqno(), validator_set_->get_validator_set_hash(),
std::move(data));
}
}
};
} // namespace validator

2
validator/validator.h
View file

@ -188,7 +188,7 @@ class ValidatorManagerInterface : public td::actor::Actor {
virtual void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) = 0;
virtual void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash,
td::BufferSlice data) = 0;
virtual void send_broadcast(BlockBroadcast broadcast, bool custom_overlays_only = false) = 0;
virtual void send_broadcast(BlockBroadcast broadcast, int mode) = 0;
virtual void download_block(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout,
td::Promise<ReceivedBlock> promise) = 0;
virtual void download_zero_state(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout,