mirror of
https://github.com/ton-blockchain/ton
synced 2025-03-09 15:40:10 +00:00
Recent updates in storage (#667)
* Fix error handling in Torrent.cpp, improve choosing peers for upload * Various improvements in storage daemon "get-pieces-info" Store "added at" Improve calculating up/down speed Improve TL protocol for future compatibility Remove empty directories on "--remove-files" Better windows support Debug logs in PeerActor More restrictions on TorrentInfo Bugfixes * Global speed limits for download and upload +bugfix * Reset download/upload speed on changing settings or completion * Exclude some system files in TorrentCreator
This commit is contained in:
SpyCheese
GitHub
parent
e3af63e6c0
commit
bb21f732fd
21 changed files with 974 additions and 213 deletions
|
|
@ -50,6 +50,29 @@ void StorageManager::start_up() {
|
|||
|
||||
db_ = std::make_shared<db::DbType>(
|
||||
std::make_shared<td::RocksDb>(td::RocksDb::open(db_root_ + "/torrent-db").move_as_ok()));
|
||||
|
||||
db::db_get<ton_api::storage_db_config>(
|
||||
*db_, create_hash_tl_object<ton_api::storage_db_key_config>(), true,
|
||||
[SelfId = actor_id(this)](td::Result<tl_object_ptr<ton_api::storage_db_config>> R) {
|
||||
if (R.is_error()) {
|
||||
LOG(ERROR) << "Failed to load config from db: " << R.move_as_error();
|
||||
td::actor::send_closure(SelfId, &StorageManager::loaded_config_from_db, nullptr);
|
||||
} else {
|
||||
td::actor::send_closure(SelfId, &StorageManager::loaded_config_from_db, R.move_as_ok());
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
void StorageManager::loaded_config_from_db(tl_object_ptr<ton_api::storage_db_config> config) {
|
||||
if (config) {
|
||||
LOG(INFO) << "Loaded config from DB. Speed limits: download=" << config->download_speed_limit_
|
||||
<< ", upload=" << config->upload_speed_limit_;
|
||||
download_speed_limit_ = config->download_speed_limit_;
|
||||
upload_speed_limit_ = config->upload_speed_limit_;
|
||||
td::actor::send_closure(download_speed_limiter_, &SpeedLimiter::set_max_speed, download_speed_limit_);
|
||||
td::actor::send_closure(upload_speed_limiter_, &SpeedLimiter::set_max_speed, upload_speed_limit_);
|
||||
}
|
||||
|
||||
db::db_get<ton_api::storage_db_torrentList>(
|
||||
*db_, create_hash_tl_object<ton_api::storage_db_key_torrentList>(), true,
|
||||
[SelfId = actor_id(this)](td::Result<tl_object_ptr<ton_api::storage_db_torrentList>> R) {
|
||||
|
|
@ -79,6 +102,7 @@ void StorageManager::load_torrents_from_db(std::vector<td::Bits256> torrents) {
|
|||
client_mode_, overlays_, adnl_, rldp_);
|
||||
NodeActor::load_from_db(
|
||||
db_, hash, create_callback(hash, entry.closing_state), PeerManager::create_callback(entry.peer_manager.get()),
|
||||
SpeedLimiters{download_speed_limiter_.get(), upload_speed_limiter_.get()},
|
||||
[SelfId = actor_id(this), hash,
|
||||
promise = ig.get_promise()](td::Result<td::actor::ActorOwn<NodeActor>> R) mutable {
|
||||
td::actor::send_closure(SelfId, &StorageManager::loaded_torrent_from_db, hash, std::move(R));
|
||||
|
|
@ -162,9 +186,9 @@ td::Status StorageManager::add_torrent_impl(Torrent torrent, bool start_download
|
|||
client_mode_, overlays_, adnl_, rldp_);
|
||||
auto context = PeerManager::create_callback(entry.peer_manager.get());
|
||||
LOG(INFO) << "Added torrent " << hash.to_hex() << " , root_dir = " << torrent.get_root_dir();
|
||||
entry.actor =
|
||||
td::actor::create_actor<NodeActor>("Node", 1, std::move(torrent), create_callback(hash, entry.closing_state),
|
||||
std::move(context), db_, start_download, allow_upload);
|
||||
entry.actor = td::actor::create_actor<NodeActor>(
|
||||
"Node", 1, std::move(torrent), create_callback(hash, entry.closing_state), std::move(context), db_,
|
||||
SpeedLimiters{download_speed_limiter_.get(), upload_speed_limiter_.get()}, start_download, allow_upload);
|
||||
return td::Status::OK();
|
||||
}
|
||||
|
||||
|
|
@ -210,6 +234,18 @@ void StorageManager::get_all_torrents(td::Promise<std::vector<td::Bits256>> prom
|
|||
promise.set_result(std::move(result));
|
||||
}
|
||||
|
||||
void StorageManager::db_store_config() {
|
||||
auto config = create_tl_object<ton_api::storage_db_config>();
|
||||
config->download_speed_limit_ = download_speed_limit_;
|
||||
config->upload_speed_limit_ = upload_speed_limit_;
|
||||
db_->set(create_hash_tl_object<ton_api::storage_db_key_config>(), serialize_tl_object(config, true),
|
||||
[](td::Result<td::Unit> R) {
|
||||
if (R.is_error()) {
|
||||
LOG(ERROR) << "Failed to save config to db: " << R.move_as_error();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
void StorageManager::db_store_torrent_list() {
|
||||
std::vector<td::Bits256> torrents;
|
||||
for (const auto& p : torrents_) {
|
||||
|
|
@ -259,19 +295,55 @@ void StorageManager::load_from(td::Bits256 hash, td::optional<TorrentMeta> meta,
|
|||
std::move(promise));
|
||||
}
|
||||
|
||||
static bool try_rm_empty_dir(const std::string& path) {
|
||||
auto stat = td::stat(path);
|
||||
if (stat.is_error() || !stat.ok().is_dir_) {
|
||||
return true;
|
||||
}
|
||||
size_t cnt = 0;
|
||||
td::WalkPath::run(path, [&](td::CSlice name, td::WalkPath::Type type) {
|
||||
if (type != td::WalkPath::Type::ExitDir) {
|
||||
++cnt;
|
||||
}
|
||||
if (cnt < 2) {
|
||||
return td::WalkPath::Action::Continue;
|
||||
} else {
|
||||
return td::WalkPath::Action::Abort;
|
||||
}
|
||||
}).ignore();
|
||||
if (cnt == 1) {
|
||||
td::rmdir(path).ignore();
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
void StorageManager::on_torrent_closed(Torrent torrent, std::shared_ptr<TorrentEntry::ClosingState> closing_state) {
|
||||
if (!closing_state->removing) {
|
||||
return;
|
||||
}
|
||||
if (closing_state->remove_files && torrent.inited_header()) {
|
||||
// Ignore all errors: files may just not exist
|
||||
size_t files_count = torrent.get_files_count().unwrap();
|
||||
for (size_t i = 0; i < files_count; ++i) {
|
||||
std::string path = torrent.get_file_path(i);
|
||||
td::unlink(path).ignore();
|
||||
// TODO: Check errors, remove empty directories
|
||||
std::string name = torrent.get_file_name(i).str();
|
||||
for (int j = (int)name.size() - 1; j >= 0; --j) {
|
||||
if (name[j] == '/') {
|
||||
name.resize(j + 1);
|
||||
if (!try_rm_empty_dir(torrent.get_root_dir() + '/' + torrent.get_header().dir_name + '/' + name)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!torrent.get_header().dir_name.empty()) {
|
||||
try_rm_empty_dir(torrent.get_root_dir() + '/' + torrent.get_header().dir_name);
|
||||
}
|
||||
}
|
||||
}
|
||||
td::rmrf(db_root_ + "/torrent-files/" + torrent.get_hash().to_hex()).ignore();
|
||||
std::string path = db_root_ + "/torrent-files/" + torrent.get_hash().to_hex();
|
||||
td::rmrf(path).ignore();
|
||||
NodeActor::cleanup_db(db_, torrent.get_hash(),
|
||||
[promise = std::move(closing_state->promise)](td::Result<td::Unit> R) mutable {
|
||||
if (R.is_error()) {
|
||||
|
|
@ -292,4 +364,28 @@ void StorageManager::get_peers_info(td::Bits256 hash,
|
|||
td::actor::send_closure(entry->actor, &NodeActor::get_peers_info, std::move(promise));
|
||||
}
|
||||
|
||||
void StorageManager::get_speed_limits(td::Promise<std::pair<double, double>> promise) {
|
||||
promise.set_result(std::make_pair(download_speed_limit_, upload_speed_limit_));
|
||||
}
|
||||
|
||||
void StorageManager::set_download_speed_limit(double max_speed) {
|
||||
if (max_speed < 0.0) {
|
||||
max_speed = -1.0;
|
||||
}
|
||||
LOG(INFO) << "Set download speed limit to " << max_speed;
|
||||
download_speed_limit_ = max_speed;
|
||||
td::actor::send_closure(download_speed_limiter_, &SpeedLimiter::set_max_speed, max_speed);
|
||||
db_store_config();
|
||||
}
|
||||
|
||||
void StorageManager::set_upload_speed_limit(double max_speed) {
|
||||
if (max_speed < 0.0) {
|
||||
max_speed = -1.0;
|
||||
}
|
||||
LOG(INFO) << "Set upload speed limit to " << max_speed;
|
||||
upload_speed_limit_ = max_speed;
|
||||
td::actor::send_closure(upload_speed_limiter_, &SpeedLimiter::set_max_speed, max_speed);
|
||||
db_store_config();
|
||||
}
|
||||
|
||||
} // namespace ton
|
||||
Loading…
Add table
Add a link
Reference in a new issue