integrating the existing state of TON Storage / TON Payments / CPS Fift development branches

tdutils/td/utils/Heap.h

@@ -49,6 +49,10 @@ class KHeap {
     return array_[0].key_;
   }
 
+  HeapNode *top() const {
+    return array_[0].node_;
+  }
+
   HeapNode *pop() {
     CHECK(!empty());
     HeapNode *result = array_[0].node_;

tdutils/td/utils/PathView.h

@@ -54,6 +54,9 @@ class PathView {
   Slice parent_dir() const {
     return path_.substr(0, last_slash_ + 1);
   }
+  Slice parent_dir_noslash() const {
+    return last_slash_ <= 0 ? td::Slice(".") : path_.substr(0, last_slash_);
+  }
 
   Slice extension() const {
     if (last_dot_ == static_cast<int32>(path_.size())) {

tdutils/td/utils/Random.cpp

@@ -192,5 +192,8 @@ uint64 Random::Xorshift128plus::operator()() {
 int Random::Xorshift128plus::fast(int min, int max) {
   return static_cast<int>((*this)() % (max - min + 1) + min);
 }
+int64 Random::Xorshift128plus::fast64(int64 min, int64 max) {
+  return static_cast<int64>((*this)() % (max - min + 1) + min);
+}
 
 }  // namespace td

tdutils/td/utils/Random.h

@@ -20,6 +20,7 @@
 
 #include "td/utils/common.h"
 #include "td/utils/Slice.h"
+#include "td/utils/Span.h"
 
 namespace td {
 
@@ -45,16 +46,31 @@ class Random {
   static int fast(int min, int max);
   static double fast(double min, double max);
 
+  class Fast {
+   public:
+    uint64 operator()() {
+      return fast_uint64();
+    }
+  };
   class Xorshift128plus {
    public:
     explicit Xorshift128plus(uint64 seed);
     Xorshift128plus(uint64 seed_a, uint64 seed_b);
     uint64 operator()();
     int fast(int min, int max);
+    int64 fast64(int64 min, int64 max);
 
    private:
     uint64 seed_[2];
   };
 };
 
+template <class T, class R>
+void random_shuffle(td::MutableSpan<T> v, R &rnd) {
+  for (std::size_t i = 1; i < v.size(); i++) {
+    auto pos = static_cast<std::size_t>(rnd() % (i + 1));
+    std::swap(v[i], v[pos]);
+  }
+}
+
 }  // namespace td

tdutils/td/utils/Span.h

@@ -86,6 +86,26 @@ class SpanImpl {
     return data_[i];
   }
 
+  InnerT &back() {
+    DCHECK(!empty());
+    return data_[size() - 1];
+  }
+
+  const InnerT &back() const {
+    DCHECK(!empty());
+    return data_[size() - 1];
+  }
+
+  InnerT &front() {
+    DCHECK(!empty());
+    return data_[0];
+  }
+
+  const InnerT &front() const {
+    DCHECK(!empty());
+    return data_[0];
+  }
+
   InnerT *data() const {
     return data_;
   }
@@ -109,8 +129,9 @@ class SpanImpl {
   }
 
   SpanImpl &truncate(size_t size) {
-    CHECK(size <= size_);
-    size_ = size;
+    if (size < size_) {
+      size_ = size;
+    }
     return *this;
   }
 
@@ -136,9 +157,45 @@ template <class T>
 Span<T> span(const T *ptr, size_t size) {
   return Span<T>(ptr, size);
 }
+template <class T>
+Span<T> span(const std::vector<T> &vec) {
+  return Span<T>(vec);
+}
+
 template <class T>
 MutableSpan<T> mutable_span(T *ptr, size_t size) {
   return MutableSpan<T>(ptr, size);
 }
+template <class T>
+MutableSpan<T> mutable_span(std::vector<T> &vec) {
+  return MutableSpan<T>(vec);
+}
+
+template <class T>
+Span<T> span_one(const T &value) {
+  return td::Span<T>(&value, 1);
+}
+template <class T>
+MutableSpan<T> mutable_span_one(T &value) {
+  return td::MutableSpan<T>(&value, 1);
+}
+
+template <class T>
+Span<T> as_span(Span<T> span) {
+  return span;
+}
+template <class T>
+Span<T> as_span(const std::vector<T> &vec) {
+  return Span<T>(vec);
+}
+
+template <class T>
+MutableSpan<T> as_mutable_span(MutableSpan<T> span) {
+  return span;
+}
+template <class T>
+MutableSpan<T> as_mutable_span(std::vector<T> &vec) {
+  return MutableSpan<T>(vec);
+}
 
 }  // namespace td

tdutils/td/utils/Status.h

@@ -582,6 +582,22 @@ class Result {
     *this = Result<T>();
   }
 
+  template <class F>
+  td::Result<decltype(std::declval<F>()(std::declval<T>()))> move_map(F &&f) {
+    if (is_error()) {
+      return move_as_error();
+    }
+    return f(move_as_ok());
+  }
+
+  template <class F>
+  decltype(std::declval<F>()(std::declval<T>())) move_fmap(F &&f) {
+    if (is_error()) {
+      return move_as_error();
+    }
+    return f(move_as_ok());
+  }
+
  private:
   Status status_;
   union {

tdutils/td/utils/ThreadSafeCounter.h

@@ -38,6 +38,13 @@ class ThreadSafeMultiCounter {
     tls_.for_each([&](auto &value) { res += value[index].load(); });
     return res;
   }
+  void clear() {
+    tls_.for_each([&](auto &value) {
+      for (auto &x : value) {
+        x = 0;
+      }
+    });
+  }
 
  private:
   ThreadLocalStorage<std::array<std::atomic<int64>, N>> tls_;
@@ -108,6 +115,11 @@ class NamedThreadSafeCounter {
     }
   }
 
+  void clear() {
+    std::unique_lock<std::mutex> guard(mutex_);
+    counter_.clear();
+  }
+
   friend StringBuilder &operator<<(StringBuilder &sb, const NamedThreadSafeCounter &counter) {
     counter.for_each([&sb](Slice name, int64 cnt) { sb << name << ": " << cnt << "\n"; });
     return sb;

tdutils/td/utils/Time.cpp

@@ -19,14 +19,36 @@
 #include "td/utils/Time.h"
 
 #include <cmath>
+#include <atomic>
 
 namespace td {
 
 bool operator==(Timestamp a, Timestamp b) {
   return std::abs(a.at() - b.at()) < 1e-6;
 }
+namespace {
+std::atomic<double> time_diff;
+}
 double Time::now() {
+  return now_unadjusted() + time_diff.load(std::memory_order_relaxed);
+}
+
+double Time::now_unadjusted() {
   return Clocks::monotonic();
 }
 
+void Time::jump_in_future(double at) {
+  auto old_time_diff = time_diff.load();
+
+  while (true) {
+    auto diff = at - now();
+    if (diff < 0) {
+      return;
+    }
+    if (time_diff.compare_exchange_strong(old_time_diff, old_time_diff + diff)) {
+      return;
+    }
+  }
+}
+
 }  // namespace td

tdutils/td/utils/Time.h

@@ -40,6 +40,10 @@ class Time {
     // As an alternative we may say that now_cached is a thread local copy of now
     return now();
   }
+  static double now_unadjusted();
+
+  // Used for testing. After jump_in_future(at) is called, now() >= at.
+  static void jump_in_future(double at);
 };
 
 inline void relax_timeout_at(double *timeout, double new_timeout) {
@@ -70,12 +74,15 @@ class Timestamp {
     return Timestamp{timeout - td::Clocks::system() + Time::now()};
   }
 
-  static Timestamp in(double timeout) {
-    return Timestamp{Time::now_cached() + timeout};
+  static Timestamp in(double timeout, td::Timestamp now = td::Timestamp::now_cached()) {
+    return Timestamp{now.at() + timeout};
   }
 
+  bool is_in_past(td::Timestamp now) const {
+    return at_ <= now.at();
+  }
   bool is_in_past() const {
-    return at_ <= Time::now_cached();
+    return is_in_past(now_cached());
   }
 
   explicit operator bool() const {
@@ -111,6 +118,10 @@ class Timestamp {
   }
 };
 
+inline bool operator<(const Timestamp &a, const Timestamp &b) {
+  return a.at() < b.at();
+}
+
 template <class StorerT>
 void store(const Timestamp &timestamp, StorerT &storer) {
   storer.store_binary(timestamp.at() - Time::now() + Clocks::system());

tdutils/td/utils/TimedStat.h

@@ -19,6 +19,7 @@
 #pragma once
 
 #include "td/utils/common.h"
+#include "td/utils/optional.h"
 
 #include <utility>
 
@@ -80,4 +81,27 @@ class TimedStat {
   }
 };
 
+template <class T, class Cmp>
+struct MinMaxStat {
+ public:
+  using Event = T;
+  void on_event(Event event) {
+    if (!best_ || Cmp()(event, best_.value())) {
+      best_ = event;
+    }
+  }
+  td::optional<T> get_stat() const {
+    return best_.copy();
+  }
+
+ private:
+  td::optional<T> best_;
+};
+
+template <class T>
+using MinStat = MinMaxStat<T, std::less<>>;
+
+template <class T>
+using MaxStat = MinMaxStat<T, std::greater<>>;
+
 }  // namespace td

tdutils/td/utils/VectorQueue.h

@@ -50,6 +50,12 @@ class VectorQueue {
   T &back() {
     return vector_.back();
   }
+  const T &front() const {
+    return vector_[read_pos_];
+  }
+  const T &back() const {
+    return vector_.back();
+  }
   bool empty() const {
     return size() == 0;
   }

tdutils/td/utils/bits.h

@@ -270,4 +270,40 @@ inline int32 count_bits64(uint64 x) {
 
 #endif
 
+struct BitsRange {
+  td::uint64 bits{0};
+  mutable td::int32 pos{-1};
+
+  explicit BitsRange(td::uint64 bits = 0) : bits{bits}, pos{-1} {
+  }
+
+  BitsRange begin() const {
+    return *this;
+  }
+
+  BitsRange end() const {
+    return BitsRange{};
+  }
+
+  td::int32 operator*() const {
+    if (pos == -1) {
+      pos = td::count_trailing_zeroes64(bits);
+    }
+    return pos;
+  }
+
+  bool operator!=(const BitsRange &other) const {
+    return bits != other.bits;
+  }
+
+  BitsRange &operator++() {
+    auto i = **this;
+    if (i != 64) {
+      bits ^= 1ull << i;
+    }
+    pos = -1;
+    return *this;
+  }
+};
+
 }  // namespace td

tdutils/td/utils/format.h

@@ -25,6 +25,7 @@
 
 #include <tuple>
 #include <utility>
+#include <set>
 
 namespace td {
 namespace format {
@@ -333,5 +334,9 @@ template <class T>
 StringBuilder &operator<<(StringBuilder &stream, const vector<T> &vec) {
   return stream << format::as_array(vec);
 }
+template <class T>
+StringBuilder &operator<<(StringBuilder &stream, const std::set<T> &vec) {
+  return stream << format::as_array(vec);
+}
 
 }  // namespace td

tdutils/td/utils/optional.h

@@ -59,9 +59,11 @@ class optional {
     return impl_.is_ok();
   }
   T &value() {
+    DCHECK(*this);
     return impl_.ok_ref();
   }
   const T &value() const {
+    DCHECK(*this);
     return impl_.ok_ref();
   }
   T &operator*() {

tdutils/td/utils/tests.cpp

@@ -204,6 +204,7 @@ bool TestsRunner::run_all_step() {
       }
       LOG(ERROR) << "Run test " << tag("name", name);
       state_.start = Time::now();
+      state_.start_unadjusted = Time::now_unadjusted();
       state_.is_running = true;
     }
 
@@ -211,7 +212,13 @@ bool TestsRunner::run_all_step() {
       break;
     }
 
-    LOG(ERROR) << format::as_time(Time::now() - state_.start);
+    auto passed = Time::now() - state_.start;
+    auto real_passed = Time::now_unadjusted() - state_.start_unadjusted;
+    if (real_passed + 1e-9 > passed) {
+      LOG(ERROR) << format::as_time(passed);
+    } else {
+      LOG(ERROR) << format::as_time(passed) << " real[" << format::as_time(real_passed) << "]";
+    }
     if (regression_tester_) {
       regression_tester_->save_db();
     }

tdutils/td/utils/tests.h

@@ -25,6 +25,7 @@
 #include "td/utils/port/thread.h"
 #include "td/utils/Random.h"
 #include "td/utils/Slice.h"
+#include "td/utils/Span.h"
 #include "td/utils/Status.h"
 
 #include <atomic>
@@ -117,6 +118,7 @@ class TestsRunner : public TestContext {
     size_t it{0};
     bool is_running = false;
     double start{0};
+    double start_unadjusted{0};
     size_t end{0};
   };
   bool stress_flag_{false};

tdutils/test/heap.cpp

@@ -38,8 +38,8 @@ TEST(Heap, sort_random_perm) {
   for (int i = 0; i < n; i++) {
     v[i] = i;
   }
-  std::srand(123);
-  std::random_shuffle(v.begin(), v.end());
+  td::Random::Xorshift128plus rnd(123);
+  td::random_shuffle(as_mutable_span(v), rnd);
   std::vector<HeapNode> nodes(n);
   KHeap<int> kheap;
   for (int i = 0; i < n; i++) {

tdutils/test/misc.cpp

@@ -710,6 +710,39 @@ TEST(Misc, Bits) {
   ASSERT_EQ(4, count_bits64((1ull << 63) | 7));
 }
 
+TEST(Misc, BitsRange) {
+  auto to_vec_a = [](td::uint64 x) {
+    std::vector<td::int32> bits;
+    for (auto i : td::BitsRange(x)) {
+      bits.push_back(i);
+    }
+    return bits;
+  };
+
+  auto to_vec_b = [](td::uint64 x) {
+    std::vector<td::int32> bits;
+    td::int32 pos = 0;
+    while (x != 0) {
+      if ((x & 1) != 0) {
+        bits.push_back(pos);
+      }
+      x >>= 1;
+      pos++;
+    }
+    return bits;
+  };
+
+  auto do_check = [](std::vector<td::int32> a, std::vector<td::int32> b) { ASSERT_EQ(b, a); };
+  auto check = [&](td::uint64 x) { do_check(to_vec_a(x), to_vec_b(x)); };
+
+  do_check(to_vec_a(21), {0, 2, 4});
+  for (int x = 0; x < 100; x++) {
+    check(x);
+    check(std::numeric_limits<td::uint32>::max() - x);
+    check(std::numeric_limits<td::uint64>::max() - x);
+  }
+}
+
 #if !TD_THREAD_UNSUPPORTED
 TEST(Misc, Time) {
   Stage run;