external/ton
1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-03-09 15:40:10 +00:00
Code Issues Releases Wiki Activity

TON Storage utilities (#564)

Browse source 
* Rename chunk to piece in MerkleTree for consistency

* Refactor PeerManager

* Make PeerState thread-safe

* Download torrent by hash

* First version of storage daemon

* Download torrents partially

* Improve storing and loading torrent state in DB

* Rewrite MerkleTree

* "Remove torrent" in storage daemon

* Process errors, fix bugs in storage

* Move TonlibClientWrapper from rldp-http-proxy to tonlib

* Initial version of storage provider

* Move interaction with contracts to smc-util

* Improve TonlibClientWrapper interface

* Various improvements in storage provider

* Fix TorrentCreator.cpp

* Improve interface for partial download

* Client mode in storage-daemon

* Improve interface of storage-daemon-cli

* Fix calculating speed, show peers in storage-daemon

* Use permanent adnl id in storage daemon

* Fix sending large "storage.addUpdate" messages

* Improve printing torrents in cli

* Update tlo

* Fix RldpSender::on_ack

* Update storage provider

* Add "address" parameter to get-provider-params

* Allow client to close storage contract

* Limit torrent description

* Add more logs to storage provider

* smc.forget tonlib method

* Use smc.forget in storage daemon

* Optimize sending messages in smc-util.cpp

* Fix verbosity, remove excessive logs

* Json output in storage-daemon-cli

* Update storage provider contracts

* Fix rldp2 acks

* Change verbosity of logs in rldp2

* Update help and output of commands and in storage-daemon-cli

Co-authored-by: SpyCheese <mikle98@yandex.ru>
This commit is contained in:
EmelyanenkoK 2022-12-22 12:24:13 +03:00 • committed by GitHub
parent 434dc487a4
commit 360ef54e6b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
75 changed files with 8872 additions and 1148 deletions
Download patch file Download diff file Expand all files Collapse all files

3
rldp2/BdwStats.cpp
View file

@ -18,6 +18,7 @@
*/
#include "BdwStats.h"
#include "rldp.hpp"
namespace ton {
namespace rldp2 {
@ -39,7 +40,7 @@ void BdwStats::on_packet_ack(const PacketInfo &info, td::Timestamp sent_at, td::
auto ack_passed = now.at() - info.delivered_now.at();
auto passed = td::max(sent_passed, ack_passed);
if (passed < 0.01) {
LOG(ERROR) << "Invalid passed " << passed;
VLOG(RLDP_WARNING) << "Invalid passed " << passed;
}
auto delivered = delivered_count - info.delivered_count;
on_rate_sample((double)delivered / passed, now, info.is_paused);

20
rldp2/RldpConnection.cpp
View file

@ -18,6 +18,7 @@
*/
#include "RldpConnection.h"
#include "rldp.hpp"
#include "td/utils/overloaded.h"
#include "td/utils/Random.h"
@ -83,7 +84,7 @@ td::Timestamp RldpConnection::loop_limits(td::Timestamp now) {
outbound_transfers_.erase(it);
to_on_sent_.emplace_back(limit->transfer_id, std::move(error));
} else {
LOG(ERROR) << "Timeout on unknown transfer " << limit->transfer_id.to_hex();
VLOG(RLDP_WARNING) << "Timeout on unknown transfer " << limit->transfer_id.to_hex();
}
}
limits_set_.erase(*limit);
@ -113,7 +114,7 @@ void RldpConnection::send(TransferId transfer_id, td::BufferSlice data, td::Time
td::Random::secure_bytes(transfer_id.as_slice());
} else {
if (outbound_transfers_.find(transfer_id) != outbound_transfers_.end()) {
LOG(WARNING) << "Skip resend of " << transfer_id.to_hex();
VLOG(RLDP_WARNING) << "Skip resend of " << transfer_id.to_hex();
return;
}
}
@ -143,17 +144,6 @@ void RldpConnection::loop_bbr(td::Timestamp now) {
double speed = bbr_.get_rate();
td::uint32 congestion_window = bbr_.get_window_size();
static td::Timestamp next;
//FIXME: remove this UNSAFE debug output
if (next.is_in_past(now)) {
next = td::Timestamp::in(1, now);
if (td::actor::core::ActorExecuteContext::get()->actor().get_actor_info_ptr()->get_name() == "Alice") {
LOG(ERROR) << "speed=" << td::format::as_size((td::int64)speed * 768) << " "
<< "cgw=" << td::format::as_size((td::int64)congestion_window * 768) << " "
<< "loss=" << loss_stats_.loss * 100 << "%";
}
}
pacer_.set_speed(speed);
congestion_window_ = congestion_window;
}
@ -301,7 +291,7 @@ void RldpConnection::receive_raw_obj(ton::ton_api::rldp2_messagePart &part) {
max_size = limit_it->max_size;
}
if (total_size > max_size) {
LOG(INFO) << "Drop too big rldp query " << part.total_size_ << " > " << max_size;
VLOG(RLDP_INFO) << "Drop too big rldp query " << part.total_size_ << " > " << max_size;
return;
}
@ -324,7 +314,7 @@ void RldpConnection::receive_raw_obj(ton::ton_api::rldp2_messagePart &part) {
}
return {};
}
if (in_part->receiver.on_received(part.seqno_, td::Timestamp::now())) {
if (in_part->receiver.on_received(part.seqno_ + 1, td::Timestamp::now())) {
TRY_STATUS_PREFIX(in_part->decoder->add_symbol({static_cast<td::uint32>(part.seqno_), std::move(part.data_)}),
td::Status::Error(ErrorCode::protoviolation, "invalid symbol"));
if (in_part->decoder->may_try_decode()) {

5
rldp2/RldpReceiver.cpp
View file

@ -33,9 +33,8 @@ void RldpReceiver::on_ack_sent(td::Timestamp now) {
//LOG(ERROR) << "RESEND ACK " << cnt_;
}
cnt_++;
if (cnt_ > 7) {
send_ack_at_ = {};
} else {
send_ack_at_ = {};
if (cnt_ <= 7) {
send_ack_at_.relax(td::Timestamp::at(now.at() + config_.ack_delay * (1 << cnt_)));
}
}

1
rldp2/RldpSender.cpp
View file

@ -49,7 +49,6 @@ SenderPackets::Update RldpSender::on_ack(const Ack &ack, double ack_delay, td::T
//LOG(ERROR) << "ON ACK " << ack.max_seqno << " " << ack.received_mask << " " << ack.received_count;
auto update = packets_.on_ack(ack);
if (!update.was_max_updated) {
CHECK(!update.new_received);
return update;
}

5
rldp2/RttStats.cpp
View file

@ -18,17 +18,18 @@
*/
#include "RttStats.h"
#include "rldp.hpp"
#include <cmath>
namespace ton {
namespace rldp2 {
void RttStats::on_rtt_sample(double rtt_sample, double ack_delay, td::Timestamp now) {
if (rtt_sample < 0.001 || rtt_sample > 10) {
LOG(WARNING) << "Suspicious rtt sample " << rtt_sample;
VLOG(RLDP_WARNING) << "Suspicious rtt sample " << rtt_sample;
return;
}
if (ack_delay < -1e-9 || ack_delay > 10) {
LOG(WARNING) << "Suspicious ack_delay " << ack_delay;
VLOG(RLDP_WARNING) << "Suspicious ack_delay " << ack_delay;
return;
}
rtt_sample = td::max(0.01, rtt_sample);