external/ton
1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-03-09 15:40:10 +00:00
Code Issues Releases Wiki Activity

initial commit

Browse source
This commit is contained in:
initial commit 2019-09-07 14:03:22 +04:00 • committed by vvaltman
commit c2da007f40
1610 changed files with 398047 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

76
tdfec/CMakeLists.txt Normal file
View file

@ -0,0 +1,76 @@
cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR)
set(TDFEC_SOURCE
td/fec/raptorq/Rfc.cpp
td/fec/raptorq/Rfc.h
td/fec/raptorq/Decoder.h
td/fec/raptorq/Decoder.cpp
td/fec/raptorq/Encoder.h
td/fec/raptorq/Encoder.cpp
td/fec/raptorq/RawEncoder.cpp
td/fec/raptorq/RawEncoder.h
td/fec/raptorq/Solver.h
td/fec/raptorq/Solver.cpp
td/fec/common/SymbolRef.h
td/fec/common/SymbolsView.h
td/fec/common/SymbolsView.cpp
td/fec/online/Encoder.h
td/fec/online/Encoder.cpp
td/fec/online/Decoder.h
td/fec/online/Decoder.cpp
td/fec/online/Rfc.h
td/fec/online/Rfc.cpp
td/fec/algebra/BeliefPropagationDecoding.h
td/fec/algebra/BeliefPropagationDecoding.cpp
td/fec/algebra/GaussianElimination.h
td/fec/algebra/GaussianElimination.cpp
td/fec/algebra/InactivationDecoding.h
td/fec/algebra/InactivationDecoding.cpp
td/fec/algebra/SparseMatrixGF2.h
td/fec/algebra/SparseMatrixGF2.cpp
td/fec/algebra/MatrixGF2.h
td/fec/algebra/MatrixGF2.cpp
td/fec/algebra/MatrixGF256.h
td/fec/algebra/MatrixGF256.cpp
td/fec/algebra/Octet.h
td/fec/algebra/Octet.cpp
td/fec/algebra/Simd.h
td/fec/fec.cpp
td/fec/fec.h
)
add_library(tdfec STATIC ${TDFEC_SOURCE})
target_include_directories(tdfec PUBLIC
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>/..
)
target_link_libraries(tdfec PUBLIC tdutils)
if (USE_LIBRAPTORQ)
set(THIRD_PARTY_FEC_SOURCE
test/LibRaptorQ.h
test/LibRaptorQ.cpp
)
add_library(third_party_fec STATIC ${THIRD_PARTY_FEC_SOURCE})
target_include_directories(third_party_fec PUBLIC
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>/..
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>/../third-party/libraptorq/src
)
target_link_libraries(third_party_fec PRIVATE tdutils RaptorQ_Static)
endif()
set(FEC_TEST_SOURCE
${CMAKE_CURRENT_SOURCE_DIR}/test/fec-test.cpp
PARENT_SCOPE
)
add_subdirectory(benchmark)

5
tdfec/benchmark/CMakeLists.txt Normal file
View file

@ -0,0 +1,5 @@
cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR)
add_executable(benchmark-fec benchmark.cpp )
target_include_directories(benchmark-fec PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>)
target_link_libraries(benchmark-fec PRIVATE tdfec)

304
tdfec/benchmark/benchmark.cpp Normal file
View file

@ -0,0 +1,304 @@
/*
This file is part of TON Blockchain source code.
TON Blockchain is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.
TON Blockchain is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with TON Blockchain. If not, see <http://www.gnu.org/licenses/>.
In addition, as a special exception, the copyright holders give permission
to link the code of portions of this program with the OpenSSL library.
You must obey the GNU General Public License in all respects for all
of the code used other than OpenSSL. If you modify file(s) with this
exception, you may extend this exception to your version of the file(s),
but you are not obligated to do so. If you do not wish to do so, delete this
exception statement from your version. If you delete this exception statement
from all source files in the program, then also delete it here.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/utils/benchmark.h"
#include "td/utils/format.h"
#include "td/utils/Random.h"
#include "td/utils/Time.h"
#include "td/utils/tests.h"
#include "td/fec/fec.h"
#include "td/fec/algebra/Octet.h"
#include "td/fec/algebra/GaussianElimination.h"
#include "td/fec/algebra/Simd.h"
#include <cstdio>
template <class Simd, size_t size = 256>
class Simd_gf256_from_gf2 : public td::Benchmark {
public:
explicit Simd_gf256_from_gf2(std::string description) : description_(std::move(description)) {
for (size_t i = 0; i < size; i++) {
src[i] = td::uint8(td::Random::fast(0, 255));
}
}
std::string get_description() const override {
return PSTRING() << "gf256_from_gf2 " << description_ << " " << size;
}
void run(int n) override {
for (int i = 0; i < n; i++) {
Simd::gf256_from_gf2(dest, src, size);
}
td::do_not_optimize_away(dest[0]);
}
private:
alignas(32) td::uint8 dest[8 * size];
alignas(32) td::uint8 src[size];
std::string description_;
};
template <class Simd, size_t size = 256 * 8>
class Simd_gf256_add : public td::Benchmark {
public:
explicit Simd_gf256_add(std::string description) : description_(std::move(description)) {
for (size_t i = 0; i < size; i++) {
src[i] = td::uint8(td::Random::fast(0, 255));
}
}
std::string get_description() const override {
return PSTRING() << "gf256_add " << description_ << " " << size;
}
void run(int n) override {
for (int i = 0; i < n; i++) {
Simd::gf256_add(dest, src, size);
}
td::do_not_optimize_away(dest[0]);
}
private:
alignas(32) td::uint8 dest[size];
alignas(32) td::uint8 src[size];
std::string description_;
};
template <class Simd, size_t size = 256 * 8>
class Simd_gf256_add_mul : public td::Benchmark {
public:
explicit Simd_gf256_add_mul(std::string description) : description_(std::move(description)) {
for (size_t i = 0; i < size; i++) {
src[i] = td::uint8(td::Random::fast(0, 255));
}
}
std::string get_description() const override {
return PSTRING() << "gf256_add_mul " << description_ << " " << size;
}
void run(int n) override {
for (int i = 0; i < n; i++) {
Simd::gf256_add_mul(dest, src, 211, size);
}
td::do_not_optimize_away(dest[0]);
}
private:
alignas(32) td::uint8 dest[size];
alignas(32) td::uint8 src[size];
std::string description_;
};
template <class Simd, size_t size = 256 * 8>
class Simd_gf256_mul : public td::Benchmark {
public:
explicit Simd_gf256_mul(std::string description) : description_(std::move(description)) {
for (size_t i = 0; i < size; i++) {
src[i] = td::uint8(td::Random::fast(0, 255));
}
}
std::string get_description() const override {
return PSTRING() << "gf256_mul " << description_ << " " << size;
}
void run(int n) override {
for (int i = 0; i < n; i++) {
Simd::gf256_mul(src, 211, size);
}
td::do_not_optimize_away(dest[0]);
}
private:
alignas(32) td::uint8 dest[size];
alignas(32) td::uint8 src[size];
std::string description_;
};
class GaussBenchmark : public td::Benchmark {
public:
GaussBenchmark(size_t n) : n_(n) {
for (size_t i = 0; i < A_.rows(); i++) {
for (size_t j = 0; j < A_.cols(); j++) {
A_.set(i, j, td::Octet(td::uint8(td::Random::fast_uint32())));
}
}
for (size_t i = 0; i < D_.rows(); i++) {
for (size_t j = 0; j < D_.cols(); j++) {
D_.set(i, j, td::Octet(td::uint8(td::Random::fast_uint32())));
}
}
}
std::string get_description() const override {
return PSTRING() << "GaussBenchmark " << n_;
}
void run(int n) override {
for (int j = 0; j < n; j++) {
auto A = A_.copy();
auto D = D_.copy();
td::GaussianElimination::run(std::move(A), std::move(D));
}
}
private:
size_t n_;
td::MatrixGF256 A_{n_, n_};
td::MatrixGF256 D_{n_, n_ / 3};
};
class SolverBenchmark : public td::Benchmark {
public:
SolverBenchmark(size_t data_size, size_t symbol_size) {
data_ = td::BufferSlice(td::rand_string('a', 'z', td::narrow_cast<int>(data_size)));
symbol_size_ = symbol_size;
}
std::string get_description() const override {
return PSTRING() << "SolverBenchmark " << data_.size() << " " << symbol_size_;
}
void run(int n) override {
for (int j = 0; j < n; j++) {
td::fec::RaptorQEncoder::create(data_.clone(), symbol_size_);
}
}
private:
td::BufferSlice data_;
size_t symbol_size_;
};
template <class Encoder, class Decoder>
class FecBenchmark : public td::Benchmark {
public:
FecBenchmark(td::uint32 symbol_size, td::uint32 symbols_count, std::string name)
: symbol_size_(symbol_size), symbols_count_(symbols_count), name_(std::move(name)) {
data_ = td::BufferSlice(symbols_count_ * symbol_size_);
}
std::string get_description() const override {
return PSTRING() << "FecBenchmark " << name_ << " " << td::tag("symbols_count", symbols_count_)
<< td::tag("symbol_size", symbol_size_);
}
void run(int n) override {
for (int i = 0; i < n; i++) {
auto encoder = Encoder::create(data_.clone(), symbol_size_);
std::vector<td::fec::Symbol> symbols;
auto parameters = encoder->get_parameters();
auto decoder = Decoder::create(parameters);
size_t sent_symbols = 0;
for (td::uint32 j = 0; j < data_.size() / symbol_size_ * 20; j++) {
if (td::Random::fast(0, 5) != 0) {
if (encoder->get_info().ready_symbol_count <= j) {
encoder->prepare_more_symbols();
}
decoder->add_symbol(encoder->gen_symbol(j));
sent_symbols++;
if (decoder->may_try_decode()) {
auto res = decoder->try_decode(false);
if (res.is_ok()) {
if (sent_symbols > static_cast<size_t>(static_cast<double>(parameters.symbols_count) * 1.05)) {
LOG(ERROR) << sent_symbols << " / " << parameters.symbols_count;
}
//ASSERT_EQ(res.data.as_slice(), data);
break;
}
}
}
}
}
}
size_t symbol_size_;
size_t symbols_count_;
std::string name_;
td::BufferSlice data_;
};
template <template <class T, size_t size> class O, size_t size = 256 * 8>
void bench_simd() {
bench(O<td::Simd_null, size>("baseline"));
#if TD_SSSE3
bench(O<td::Simd_sse, size>("SSE"));
#endif
#if TD_AVX2
bench(O<td::Simd_avx, size>("AVX"));
#endif
}
void run_encode_benchmark() {
constexpr size_t TARGET_TOTAL_BYTES = 100 * 1024 * 1024;
constexpr size_t SYMBOLS_COUNT[11] = {10, 100, 250, 500, 1000, 2000, 4000, 10000, 20000, 40000, 56403};
td::uint64 junk = 0;
for (int it = 0; it < 11; it++) {
//for (int it = 0; true;) {
auto symbol_count = SYMBOLS_COUNT[it];
auto symbol_size = 512;
auto elements = symbol_count * symbol_size;
td::BufferSlice data(elements);
td::Random::Xorshift128plus rnd(123);
for (auto &c : data.as_slice()) {
c = static_cast<td::uint8>(rnd());
}
double now = td::Time::now();
auto iterations = TARGET_TOTAL_BYTES / elements;
for (size_t i = 0; i < iterations; i++) {
auto encoder = td::fec::RaptorQEncoder::create(data.clone(), symbol_size);
encoder->prepare_more_symbols();
junk += encoder->gen_symbol(10000000).data.as_slice()[0];
}
double elapsed = td::Time::now() - now;
double throughput = ((double)elements * (double)iterations * 8.0) / 1024 / 1024 / elapsed;
fprintf(stderr, "symbol count = %d, encoded %d MB in %.3lfsecs, throughtput: %.1lfMbit/s\n", (int)symbol_count,
(int)(elements * iterations / 1024 / 1024), elapsed, throughput);
}
td::do_not_optimize_away(junk);
}
int main(void) {
SET_VERBOSITY_LEVEL(VERBOSITY_NAME(ERROR));
run_encode_benchmark();
bench_simd<Simd_gf256_mul, 32>();
bench_simd<Simd_gf256_add_mul, 32>();
bench_simd<Simd_gf256_add, 32>();
bench_simd<Simd_gf256_from_gf2, 32>();
bench_simd<Simd_gf256_mul>();
bench_simd<Simd_gf256_add_mul>();
bench_simd<Simd_gf256_add>();
bench_simd<Simd_gf256_from_gf2, 256>();
bench(GaussBenchmark(15));
bench(GaussBenchmark(1000));
bench(FecBenchmark<td::fec::RaptorQEncoder, td::fec::RaptorQDecoder>(512, 20, "RaptorQ"));
bench(FecBenchmark<td::fec::RaptorQEncoder, td::fec::RaptorQDecoder>(200, 1000, "RaptorQ"));
bench(FecBenchmark<td::fec::OnlineEncoder, td::fec::OnlineDecoder>(200, 1000, "Online"));
for (int symbol_size = 32; symbol_size <= 8192; symbol_size *= 2) {
bench(FecBenchmark<td::fec::OnlineEncoder, td::fec::OnlineDecoder>(symbol_size, 50000, "Online"));
bench(FecBenchmark<td::fec::RaptorQEncoder, td::fec::RaptorQDecoder>(symbol_size, 50000, "RaptorQ"));
}
SET_VERBOSITY_LEVEL(VERBOSITY_NAME(WARNING));
bench(SolverBenchmark(50000 * 200, 200));
return 0;
}

114
tdfec/td/fec/algebra/BeliefPropagationDecoding.cpp Normal file
View file

@ -0,0 +1,114 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/fec/algebra/BeliefPropagationDecoding.h"
namespace td {
BeliefPropagationDecoding::BeliefPropagationDecoding(size_t symbols_count, size_t symbol_size)
: max_equation_count_{static_cast<size_t>(static_cast<double>(symbols_count) * 1.1 + 5)}
, C_{symbols_count, symbol_size}
, D_{max_equation_count_, symbol_size} {
equations_.reserve(max_equation_count_);
symbols_.resize(symbols_count);
edges_.resize(1);
}
Slice BeliefPropagationDecoding::get_symbol(uint32 symbol_id) const {
CHECK(symbols_[symbol_id].is_ready);
return C_.row(symbol_id);
}
void BeliefPropagationDecoding::add_equation(Span<uint32> symbol_ids, Slice data) {
if (equations_.size() >= D_.rows()) {
MatrixGF256 new_D(D_.rows() * 2, D_.cols());
new_D.set_from(D_, 0, 0);
D_ = std::move(new_D);
}
CHECK(symbol_ids.size() != 0);
uint32 equation_id = static_cast<uint32>(equations_.size());
D_.row_set(equation_id, data);
EquationInfo equation;
for (auto symbol_id : symbol_ids) {
CHECK(symbol_id < symbols_.size());
auto &symbol = symbols_[symbol_id];
if (symbol.is_ready) {
D_.row_add(equation_id, C_.row(symbol_id));
} else {
equation.symbols_xor ^= symbol_id;
equation.symbols_count++;
edges_.push_back({equation_id, symbol.head_});
symbol.head_ = uint32(edges_.size() - 1);
}
}
if (equation.symbols_count == 0) {
return;
}
equations_.push_back(equation);
if (equation.symbols_count == 1) {
ready_equations_.push_back(equation_id);
loop();
}
}
bool BeliefPropagationDecoding::is_ready() const {
return ready_symbols().size() == C_.rows();
}
Span<uint32> BeliefPropagationDecoding::ready_symbols() const {
return ready_symbols_;
}
void BeliefPropagationDecoding::loop() {
while (!is_ready() && !ready_equations_.empty()) {
auto equation_id = ready_equations_.back();
ready_equations_.pop_back();
auto &equation = equations_[equation_id];
LOG_CHECK(equation.symbols_count <= 1) << equation.symbols_count;
if (equation.symbols_count == 0) {
continue;
}
auto symbol_id = equation.symbols_xor;
auto &symbol = symbols_[symbol_id];
LOG_CHECK(symbol_id < symbols_.size())
<< equation.symbols_xor << " " << equation.symbols_count << " " << equation_id;
if (symbol.is_ready) {
continue;
}
C_.row_set(symbol_id, D_.row(equation_id));
symbol.is_ready = true;
ready_symbols_.push_back(symbol_id);
for (auto i = symbol.head_; i != 0;) {
auto &edge = edges_[i];
auto next_equation_id = edge.value;
i = edge.next;
D_.row_add(next_equation_id, C_.row(symbol_id));
auto &next_equation = equations_[next_equation_id];
next_equation.symbols_xor ^= symbol_id;
next_equation.symbols_count--;
if (next_equation.symbols_count == 1) {
ready_equations_.push_back(next_equation_id);
}
}
}
}
} // namespace td

62
tdfec/td/fec/algebra/BeliefPropagationDecoding.h Normal file
View file

@ -0,0 +1,62 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/buffer.h"
#include "td/utils/Span.h"
#include "td/fec/algebra/MatrixGF256.h"
namespace td {
class BeliefPropagationDecoding {
public:
explicit BeliefPropagationDecoding(size_t symbols_count, size_t symbol_size);
void add_equation(Span<uint32> symbol_ids, Slice data);
bool is_ready() const;
Span<uint32> ready_symbols() const;
Slice get_symbol(uint32 symbol_id) const;
private:
struct SymbolInfo {
bool is_ready{false};
uint32 head_ = 0;
};
struct EquationInfo {
uint32 symbols_xor{0};
uint32 symbols_count{0};
};
size_t max_equation_count_;
MatrixGF256 C_;
MatrixGF256 D_;
std::vector<SymbolInfo> symbols_;
std::vector<EquationInfo> equations_;
std::vector<uint32> ready_equations_;
std::vector<uint32> ready_symbols_;
struct Edge {
uint32 value;
uint32 next;
};
std::vector<Edge> edges_;
void loop();
};
} // namespace td

59
tdfec/td/fec/algebra/GaussianElimination.cpp Normal file
View file

@ -0,0 +1,59 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/fec/algebra/GaussianElimination.h"
namespace td {
Result<MatrixGF256> GaussianElimination::run(MatrixGF256 A, MatrixGF256 D) {
const size_t cols = A.cols();
const size_t rows = A.rows();
CHECK(cols <= rows);
std::vector<uint32> row_perm(rows);
for (uint32 i = 0; i < rows; i++) {
row_perm[i] = i;
}
for (size_t row = 0; row < cols; row++) {
size_t non_zero_row = row;
for (; non_zero_row < rows && A.get(row_perm[non_zero_row], row).is_zero(); non_zero_row++) {
}
if (non_zero_row == rows) {
return Status::Error("Non solvable");
}
if (non_zero_row != row) {
std::swap(row_perm[non_zero_row], row_perm[row]);
}
auto mul = A.get(row_perm[row], row).inverse();
A.row_multiply(row_perm[row], mul);
D.row_multiply(row_perm[row], mul);
CHECK(A.get(row_perm[row], row).value() == 1);
for (size_t zero_row = 0; zero_row < rows; zero_row++) {
if (zero_row == row) {
continue;
}
auto x = A.get(row_perm[zero_row], row);
if (!x.is_zero()) {
A.row_add_mul(row_perm[zero_row], row_perm[row], x);
D.row_add_mul(row_perm[zero_row], row_perm[row], x);
}
}
}
return D.apply_row_permutation(row_perm);
}
} // namespace td

28
tdfec/td/fec/algebra/GaussianElimination.h Normal file
View file

@ -0,0 +1,28 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/fec/algebra/MatrixGF256.h"
namespace td {
class GaussianElimination {
public:
static Result<MatrixGF256> run(MatrixGF256 A, MatrixGF256 D);
};
} // namespace td

186
tdfec/td/fec/algebra/InactivationDecoding.cpp Normal file
View file

@ -0,0 +1,186 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/fec/algebra/InactivationDecoding.h"
#include "td/utils/logging.h"
#include <algorithm>
namespace td {
InactivationDecodingResult InactivationDecoding::run() {
init();
loop();
LOG(DEBUG) << tag("A_small.cols", L_.cols() - p_rows_.size()) << tag("Total columns", L_.cols()) << tag("PI", PI_)
<< tag("A_small.cols - PI", L_.cols() - PI_ - p_rows_.size());
for (uint32 row = 0; row < rows_; row++) {
if (!was_row_[row]) {
p_rows_.push_back(row);
}
}
uint32 side = narrow_cast<uint32>(p_cols_.size());
std::reverse(inactive_cols_.begin(), inactive_cols_.end());
for (auto col : inactive_cols_) {
p_cols_.push_back(col);
}
for (uint32 i = 0; i < PI_; i++) {
p_cols_.push_back(cols_ + i);
}
check(side);
return {side, std::move(p_rows_), std::move(p_cols_)};
}
void InactivationDecoding::init() {
was_row_ = vector<bool>(rows_, false);
was_col_ = vector<bool>(cols_, false);
col_cnt_ = vector<uint32>(cols_, 0);
row_cnt_ = vector<uint32>(rows_, 0);
row_xor_ = vector<uint32>(rows_, 0);
L_.generate([&](auto row, auto col) {
if (col >= cols_) {
return;
}
col_cnt_[col]++;
row_cnt_[row]++;
row_xor_[row] ^= col;
});
sort_rows();
}
void InactivationDecoding::loop() {
while (row_cnt_offset_[1] != rows_) {
auto row = sorted_rows_[row_cnt_offset_[1]];
uint32 col = choose_col(row);
LOG_CHECK(col_cnt_[col] >= 1) << col;
auto cnt = row_cnt_[row];
CHECK(row_cnt_offset_[cnt] == row_cnt_offset_[1]);
CHECK(row_pos_[row] == row_cnt_offset_[1]);
p_cols_.push_back(col);
p_rows_.push_back(row);
if (cnt == 1) {
inactivate_col(col);
} else {
for (auto x : L_rows_.col(row)) {
if (x >= cols_ || was_col_[x]) {
continue;
}
if (x != col) {
inactive_cols_.push_back(x);
}
inactivate_col(x);
}
}
was_row_[row] = true;
}
}
void InactivationDecoding::check_sorted() {
for (size_t i = 0; i < rows_; i++) {
CHECK(sorted_rows_[row_pos_[i]] == i);
}
for (size_t i = 1; i < rows_; i++) {
CHECK(row_cnt_[sorted_rows_[i - 1]] <= row_cnt_[sorted_rows_[i]]);
}
for (size_t i = 1; i <= cols_ + 1; i++) {
CHECK(row_cnt_offset_[i - 1] <= row_cnt_offset_[i]);
}
for (size_t i = 0; i < rows_; i++) {
auto pos = row_pos_[i];
auto cnt = row_cnt_[i];
CHECK(pos >= row_cnt_offset_[cnt]);
CHECK(pos < row_cnt_offset_[cnt + 1]);
}
}
uint32 InactivationDecoding::choose_col(uint32 row) {
auto cnt = row_cnt_[row];
if (cnt == 1) {
return row_xor_[row];
}
uint32 best_col = uint32(-1);
for (auto col : L_rows_.col(row)) {
if (col >= cols_ || was_col_[col]) {
continue;
}
DCHECK(col_cnt_[col] >= 1);
if (best_col == uint32(-1) || col_cnt_[col] < col_cnt_[best_col]) {
best_col = col;
}
}
DCHECK(best_col != uint32(-1));
return best_col;
}
void InactivationDecoding::inactivate_col(uint32 col) {
was_col_[col] = true;
for (auto row : L_.col(col)) {
if (was_row_[row]) {
continue;
}
auto pos = row_pos_[row];
DCHECK(sorted_rows_[pos] == row);
auto cnt = row_cnt_[row];
LOG_DCHECK(cnt >= 1) << row << " " << col;
auto offset = row_cnt_offset_[cnt];
std::swap(sorted_rows_[pos], sorted_rows_[offset]);
row_pos_[sorted_rows_[pos]] = pos;
row_pos_[sorted_rows_[offset]] = offset;
row_cnt_offset_[cnt]++;
row_cnt_[row]--;
row_xor_[row] ^= col;
}
}
void InactivationDecoding::sort_rows() {
vector<uint32> offset(cols_ + 2, 0);
for (size_t i = 0; i < rows_; i++) {
offset[row_cnt_[i] + 1]++;
}
for (size_t i = 1; i <= cols_ + 1; i++) {
offset[i] += offset[i - 1];
}
row_cnt_offset_ = offset;
sorted_rows_.resize(rows_);
row_pos_.resize(rows_);
for (uint32 i = 0; i < rows_; i++) {
auto pos = offset[row_cnt_[i]]++;
sorted_rows_[pos] = i;
row_pos_[i] = pos;
}
}
void InactivationDecoding::check(uint32 side) {
auto inv_p_cols = inverse_permutation(p_cols_);
auto inv_p_rows = inverse_permutation(p_rows_);
for (uint32 i = 0; i < side; i++) {
CHECK(inv_p_cols[p_cols_[i]] == i);
auto col = L_.col(p_cols_[i]);
CHECK(col.size() >= 1);
for (auto x : col) {
CHECK(inv_p_rows[x] >= i);
}
}
}
} // namespace td

73
tdfec/td/fec/algebra/InactivationDecoding.h Normal file
View file

@ -0,0 +1,73 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#include "td/fec/algebra/SparseMatrixGF2.h"
namespace td {
struct InactivationDecodingResult {
uint32 size;
vector<uint32> p_rows;
vector<uint32> p_cols;
};
class InactivationDecoding {
public:
InactivationDecoding(const SparseMatrixGF2 &L, uint32 PI) : L_(L), PI_(PI) {
}
InactivationDecodingResult run();
private:
const SparseMatrixGF2 &L_;
uint32 PI_;
const SparseMatrixGF2 L_rows_{L_.transpose()};
const uint32 cols_ = L_.cols() - PI_;
const uint32 rows_ = L_.rows();
vector<bool> was_row_;
vector<bool> was_col_;
vector<uint32> col_cnt_;
vector<uint32> row_cnt_;
vector<uint32> row_xor_;
vector<uint32> sorted_rows_;
vector<uint32> row_cnt_offset_;
vector<uint32> row_pos_;
vector<uint32> p_rows_;
vector<uint32> p_cols_;
vector<uint32> inactive_cols_;
void init();
void loop();
void check_sorted();
uint32 choose_col(uint32 row);
void inactivate_col(uint32 col);
void sort_rows();
void check(uint32 side);
};
} // namespace td

20
tdfec/td/fec/algebra/MatrixGF2.cpp Normal file
View file

@ -0,0 +1,20 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/fec/algebra/MatrixGF2.h"
char disable_linker_warning_about_empty_file_matrixgf2_cpp TD_UNUSED;

103
tdfec/td/fec/algebra/MatrixGF2.h Normal file
View file

@ -0,0 +1,103 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/fec/algebra/MatrixGF256.h"
namespace td {
class MatrixGF2 {
public:
MatrixGF2(size_t rows, size_t cols) : rows_(rows), cols_(cols) {
CHECK(Simd::alignment() % 8 == 0);
stride_ = ((cols_ + 7) / 8 + Simd::alignment() - 1) / Simd::alignment() * Simd::alignment();
CHECK(stride_ * 8 >= cols_);
storage_ = std::make_unique<uint8[]>(stride_ * rows + Simd::alignment() - 1);
matrix_ = storage_.get();
while (!Simd::is_aligned_pointer(matrix_)) {
matrix_++;
}
CHECK(Simd::is_aligned_pointer(matrix_));
CHECK(Simd::is_aligned_pointer(matrix_ + stride_));
CHECK(static_cast<size_t>(matrix_ - storage_.get()) < Simd::alignment());
}
void set_zero() {
std::fill(matrix_, matrix_ + stride_ * rows_, 0);
}
size_t rows() const {
return rows_;
}
size_t cols() const {
return cols_;
}
void set_one(size_t row, size_t col) {
DCHECK(row < rows_ && col < cols_);
matrix_[row * stride_ + col / 8] |= uint8(1 << (col % 8));
}
bool get(size_t row, size_t col) const {
DCHECK(row < rows_ && col < cols_);
return (matrix_[row * stride_ + col / 8] & (uint8(1) << (col % 8))) != 0;
}
// row(a) += row(b)
void row_add(size_t a, size_t b) {
row_add(row_ptr(a), row_ptr(b));
}
void row_add(size_t a, Slice b) {
DCHECK(b.size() == stride_);
row_add(row_ptr(a), b.ubegin());
}
Slice row(size_t a) const {
return Slice(row_ptr(a), stride_);
}
MutableSlice row(size_t a) {
return MutableSlice(row_ptr(a), stride_);
}
void row_set(size_t a, Slice b) {
row(a).copy_from(b);
}
MatrixGF256 to_gf256() const {
MatrixGF256 res(rows(), cols());
for (size_t i = 0; i < rows(); i++) {
Simd::gf256_from_gf2(res.row(i).data(), row(i).data(), ((cols_ + 7) / 8 + 3) / 4 * 4);
}
return res;
}
private:
uint8* matrix_;
size_t rows_;
size_t cols_;
size_t stride_;
std::unique_ptr<uint8[]> storage_;
uint8* row_ptr(size_t row) {
return matrix_ + stride_ * row;
}
const uint8* row_ptr(size_t row) const {
return matrix_ + stride_ * row;
}
void row_add(uint8* pa, const uint8* pb) {
Simd::gf256_add(pa, pb, stride_);
}
};
} // namespace td

20
tdfec/td/fec/algebra/MatrixGF256.cpp Normal file
View file

@ -0,0 +1,20 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/fec/algebra/MatrixGF256.h"
char disable_linker_warning_about_empty_file_matrixgf256_cpp TD_UNUSED;

199
tdfec/td/fec/algebra/MatrixGF256.h Normal file
View file

@ -0,0 +1,199 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/fec/algebra/Octet.h"
#include "td/fec/algebra/Simd.h"
#include "td/utils/Span.h"
#include "td/utils/format.h"
namespace td {
class MatrixGF256 {
public:
MatrixGF256(size_t rows, size_t cols) : rows_(rows), cols_(cols) {
stride_ = (cols_ + Simd::alignment() - 1) / Simd::alignment() * Simd::alignment();
storage_ = std::make_unique<uint8[]>(stride_ * rows + Simd::alignment() - 1);
matrix_ = storage_.get();
while (!Simd::is_aligned_pointer(matrix_)) {
matrix_++;
}
CHECK(Simd::is_aligned_pointer(matrix_));
CHECK(Simd::is_aligned_pointer(matrix_ + stride_));
CHECK(static_cast<size_t>(matrix_ - storage_.get()) < Simd::alignment());
}
void set_zero() {
std::fill(matrix_, matrix_ + stride_ * rows_, 0);
}
size_t rows() const {
return rows_;
}
size_t cols() const {
return cols_;
}
MatrixGF256 apply_row_permutation(Span<uint32> permutation) {
MatrixGF256 res(rows_, cols_);
for (size_t row = 0; row < rows_; row++) {
res.row(row).copy_from(this->row(permutation[row]));
}
return res;
}
Octet get(size_t row, size_t col) const {
DCHECK(row < rows_ && col < cols_);
return Octet(matrix_[row * stride_ + col]);
}
void set(size_t row, size_t col, Octet o) {
DCHECK(row < rows_ && col < cols_);
matrix_[row * stride_ + col] = o.value();
}
void row_multiply(size_t row, Octet o) {
uint8* p = row_ptr(row);
Simd::gf256_mul(p, o.value(), stride_);
}
Slice row(size_t row) const {
return Slice(row_ptr(row), cols());
}
MutableSlice row(size_t row) {
return MutableSlice(row_ptr(row), cols());
}
template <class M>
void set_from(const M& m, size_t row_offset, size_t col_offset) {
auto to = block_view(row_offset, col_offset, rows() - row_offset, cols() - col_offset);
for (size_t i = 0; i < m.rows(); i++) {
to.row(i).copy_from(m.row(i));
}
}
MatrixGF256 copy() {
MatrixGF256 res(rows(), cols());
res.set_from(*this, 0, 0);
return res;
}
void add(const MatrixGF256& m) {
CHECK(m.rows() == rows());
CHECK(m.cols() == cols());
for (size_t i = 0; i < m.rows(); i++) {
auto* to = row_ptr(i);
auto* from = m.row_ptr(i);
row_add(to, from);
}
}
// row(a) += row(b) * m
void row_add_mul(size_t a, size_t b, Octet m) {
row_add_mul(row_ptr(a), row_ptr(b), m);
}
void row_add_mul(size_t a, Slice b, Octet m) {
row_add_mul(row_ptr(a), b.ubegin(), m);
}
// row(a) += row(b)
void row_add(size_t a, size_t b) {
row_add(row_ptr(a), row_ptr(b));
}
void row_add(size_t a, Slice b) {
row_add(row_ptr(a), b.ubegin());
}
void row_set(size_t a, Slice b) {
row(a).copy_from(b);
}
class BlockView {
public:
BlockView(size_t row_offset, size_t col_offset, size_t row_size, size_t col_size, MatrixGF256& m)
: row_offset_(row_offset), col_offset_(col_offset), row_size_(row_size), col_size_(col_size), m_(m) {
}
size_t cols() const {
return col_size_;
}
size_t rows() const {
return row_size_;
}
Slice row(size_t row) const {
return m_.row(row_offset_ + row).remove_prefix(col_offset_);
}
MutableSlice row(size_t row) {
return m_.row(row_offset_ + row).remove_prefix(col_offset_);
}
private:
size_t row_offset_;
size_t col_offset_;
size_t row_size_;
size_t col_size_;
MatrixGF256& m_;
};
BlockView block_view(size_t row_offset, size_t col_offset, size_t row_size, size_t col_size) {
return BlockView(row_offset, col_offset, row_size, col_size, *this);
}
private:
uint8* matrix_;
size_t rows_;
size_t cols_;
size_t stride_;
std::unique_ptr<uint8[]> storage_;
uint8* row_ptr(size_t row) {
return matrix_ + stride_ * row;
}
const uint8* row_ptr(size_t row) const {
return matrix_ + stride_ * row;
}
void row_add_mul(uint8* ap, const uint8* bp, Octet m) {
uint8 u = m.value();
if (u == 0) {
return;
}
if (u == 1) {
return row_add(ap, bp);
}
Simd::gf256_add_mul(ap, bp, u, stride_);
}
void row_add(uint8* ap, const uint8* bp) {
Simd::gf256_add(ap, bp, stride_);
}
};
inline StringBuilder& operator<<(StringBuilder& sb, const MatrixGF256& m) {
sb << "\n";
for (uint32 i = 0; i < m.rows(); i++) {
auto row = m.row(i);
for (uint32 j = 0; j < m.cols(); j++) {
uint8 x = row[j];
sb << " " << format::hex_digit(x / 16) << format::hex_digit(x % 16);
}
sb << "\n";
}
return sb;
}
} // namespace td

9242
tdfec/td/fec/algebra/Octet.cpp Normal file
View file

File diff suppressed because it is too large Load diff

152
tdfec/td/fec/algebra/Octet.h Normal file
View file

@ -0,0 +1,152 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/logging.h"
#include <array>
#include <iomanip>
namespace td {
class Octet {
public:
Octet() = default;
explicit Octet(const uint8 val) : data_(val) {
}
uint8 value() const {
return data_;
}
Octet& operator+=(const Octet a) {
data_ ^= a.data_;
return *this;
}
friend Octet operator+(Octet lhs, const Octet rhs) {
return lhs += rhs;
}
Octet& operator-=(const Octet a) {
return *this += a;
}
friend Octet operator-(Octet lhs, const Octet rhs) {
return lhs + rhs;
}
Octet& operator*=(const Octet a) {
if (data_ != 0 && a.data_ != 0) {
data_ = oct_exp(oct_log(uint8(data_ - 1)) + oct_log(uint8(a.data_ - 1)));
} else {
data_ = 0;
}
return *this;
}
friend Octet operator*(Octet lhs, const Octet rhs) {
return lhs *= rhs;
}
Octet& operator/=(const Octet a) {
if (a.data_ != 0 && data_ != 0) {
data_ = oct_exp(oct_log(uint8(data_ - 1)) - oct_log(uint8(a.data_ - 1)) + 255);
}
return *this;
}
friend Octet operator/(Octet lhs, const Octet rhs) {
return lhs /= rhs;
}
Octet inverse() const {
return Octet(oct_exp(255 - oct_log(uint8(data_ - 1))));
}
bool operator==(const Octet a) const {
return data_ == a.data_;
}
bool operator!=(const Octet a) const {
return data_ != a.data_;
}
bool operator<(const Octet a) const {
return data_ < a.data_;
}
bool operator>(const Octet a) const {
return data_ > a.data_;
}
bool is_zero() const {
return data_ == 0;
}
friend std::ostream& operator<<(std::ostream& os, const Octet m) {
os << std::hex << static_cast<uint32>(m.data_);
return os;
}
static constexpr uint8 oct_exp(uint32 data) {
constexpr std::array<uint8, 510> precalc = {
{1, 2, 4, 8, 16, 32, 64, 128, 29, 58, 116, 232, 205, 135, 19, 38, 76, 152, 45, 90, 180, 117,
234, 201, 143, 3, 6, 12, 24, 48, 96, 192, 157, 39, 78, 156, 37, 74, 148, 53, 106, 212, 181, 119,
238, 193, 159, 35, 70, 140, 5, 10, 20, 40, 80, 160, 93, 186, 105, 210, 185, 111, 222, 161, 95, 190,
97, 194, 153, 47, 94, 188, 101, 202, 137, 15, 30, 60, 120, 240, 253, 231, 211, 187, 107, 214, 177, 127,
254, 225, 223, 163, 91, 182, 113, 226, 217, 175, 67, 134, 17, 34, 68, 136, 13, 26, 52, 104, 208, 189,
103, 206, 129, 31, 62, 124, 248, 237, 199, 147, 59, 118, 236, 197, 151, 51, 102, 204, 133, 23, 46, 92,
184, 109, 218, 169, 79, 158, 33, 66, 132, 21, 42, 84, 168, 77, 154, 41, 82, 164, 85, 170, 73, 146,
57, 114, 228, 213, 183, 115, 230, 209, 191, 99, 198, 145, 63, 126, 252, 229, 215, 179, 123, 246, 241, 255,
227, 219, 171, 75, 150, 49, 98, 196, 149, 55, 110, 220, 165, 87, 174, 65, 130, 25, 50, 100, 200, 141,
7, 14, 28, 56, 112, 224, 221, 167, 83, 166, 81, 162, 89, 178, 121, 242, 249, 239, 195, 155, 43, 86,
172, 69, 138, 9, 18, 36, 72, 144, 61, 122, 244, 245, 247, 243, 251, 235, 203, 139, 11, 22, 44, 88,
176, 125, 250, 233, 207, 131, 27, 54, 108, 216, 173, 71, 142, 1, 2, 4, 8, 16, 32, 64, 128, 29,
58, 116, 232, 205, 135, 19, 38, 76, 152, 45, 90, 180, 117, 234, 201, 143, 3, 6, 12, 24, 48, 96,
192, 157, 39, 78, 156, 37, 74, 148, 53, 106, 212, 181, 119, 238, 193, 159, 35, 70, 140, 5, 10, 20,
40, 80, 160, 93, 186, 105, 210, 185, 111, 222, 161, 95, 190, 97, 194, 153, 47, 94, 188, 101, 202, 137,
15, 30, 60, 120, 240, 253, 231, 211, 187, 107, 214, 177, 127, 254, 225, 223, 163, 91, 182, 113, 226, 217,
175, 67, 134, 17, 34, 68, 136, 13, 26, 52, 104, 208, 189, 103, 206, 129, 31, 62, 124, 248, 237, 199,
147, 59, 118, 236, 197, 151, 51, 102, 204, 133, 23, 46, 92, 184, 109, 218, 169, 79, 158, 33, 66, 132,
21, 42, 84, 168, 77, 154, 41, 82, 164, 85, 170, 73, 146, 57, 114, 228, 213, 183, 115, 230, 209, 191,
99, 198, 145, 63, 126, 252, 229, 215, 179, 123, 246, 241, 255, 227, 219, 171, 75, 150, 49, 98, 196, 149,
55, 110, 220, 165, 87, 174, 65, 130, 25, 50, 100, 200, 141, 7, 14, 28, 56, 112, 224, 221, 167, 83,
166, 81, 162, 89, 178, 121, 242, 249, 239, 195, 155, 43, 86, 172, 69, 138, 9, 18, 36, 72, 144, 61,
122, 244, 245, 247, 243, 251, 235, 203, 139, 11, 22, 44, 88, 176, 125, 250, 233, 207, 131, 27, 54, 108,
216, 173, 71, 142}};
return precalc[data];
}
static constexpr uint8 oct_log(uint8 data) {
constexpr std::array<uint8, 255> precalc = {
{0, 1, 25, 2, 50, 26, 198, 3, 223, 51, 238, 27, 104, 199, 75, 4, 100, 224, 14, 52, 141, 239,
129, 28, 193, 105, 248, 200, 8, 76, 113, 5, 138, 101, 47, 225, 36, 15, 33, 53, 147, 142, 218, 240,
18, 130, 69, 29, 181, 194, 125, 106, 39, 249, 185, 201, 154, 9, 120, 77, 228, 114, 166, 6, 191, 139,
98, 102, 221, 48, 253, 226, 152, 37, 179, 16, 145, 34, 136, 54, 208, 148, 206, 143, 150, 219, 189, 241,
210, 19, 92, 131, 56, 70, 64, 30, 66, 182, 163, 195, 72, 126, 110, 107, 58, 40, 84, 250, 133, 186,
61, 202, 94, 155, 159, 10, 21, 121, 43, 78, 212, 229, 172, 115, 243, 167, 87, 7, 112, 192, 247, 140,
128, 99, 13, 103, 74, 222, 237, 49, 197, 254, 24, 227, 165, 153, 119, 38, 184, 180, 124, 17, 68, 146,
217, 35, 32, 137, 46, 55, 63, 209, 91, 149, 188, 207, 205, 144, 135, 151, 178, 220, 252, 190, 97, 242,
86, 211, 171, 20, 42, 93, 158, 132, 60, 57, 83, 71, 109, 65, 162, 31, 45, 67, 216, 183, 123, 164,
118, 196, 23, 73, 236, 127, 12, 111, 246, 108, 161, 59, 82, 41, 157, 85, 170, 251, 96, 134, 177, 187,
204, 62, 90, 203, 89, 95, 176, 156, 169, 160, 81, 11, 245, 22, 235, 122, 117, 44, 215, 79, 174, 213,
233, 230, 231, 173, 232, 116, 214, 244, 234, 168, 80, 88, 175}};
return precalc[data];
}
static const uint8 OctMulLo[256][16];
static const uint8 OctMulHi[256][16];
private:
uint8 data_;
};
} // namespace td

266
tdfec/td/fec/algebra/Simd.h Normal file
View file

@ -0,0 +1,266 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/misc.h"
#include "td/fec/algebra/Octet.h"
#if __SSSE3__
#define TD_SSE3 1
#endif
#if __AVX2__
#define TD_AVX2 1
#define TD_SSE3 1
#endif
#if TD_AVX2
#include <immintrin.h> /* avx2 */
#elif TD_SSE3
#include <tmmintrin.h> /* ssse3 */
#endif
namespace td {
class Simd_null {
public:
static constexpr size_t alignment() {
return 32; // gf256_from_gf2 relies on 32 alignment
}
static std::string get_name() {
return "Without simd";
}
static bool is_aligned_pointer(const void *ptr) {
return ::td::is_aligned_pointer<alignment()>(ptr);
}
static void gf256_add(void *a, const void *b, size_t size) {
DCHECK(is_aligned_pointer(a));
DCHECK(is_aligned_pointer(b));
uint8 *ap = reinterpret_cast<uint8 *>(a);
const uint8 *bp = reinterpret_cast<const uint8 *>(b);
for (size_t i = 0; i < size; i++) {
ap[i] ^= bp[i];
}
}
static void gf256_mul(void *a, uint8 u, size_t size) {
DCHECK(is_aligned_pointer(a));
uint8 *ap = reinterpret_cast<uint8 *>(a);
for (size_t i = 0; i < size; i++) {
ap[i] = (Octet(ap[i]) * Octet(u)).value();
}
}
static void gf256_add_mul(void *a, const void *b, uint8 u, size_t size) {
DCHECK(is_aligned_pointer(a));
DCHECK(is_aligned_pointer(b));
uint8 *ap = reinterpret_cast<uint8 *>(a);
const uint8 *bp = reinterpret_cast<const uint8 *>(b);
for (size_t i = 0; i < size; i++) {
ap[i] = (Octet(ap[i]) + Octet(bp[i]) * Octet(u)).value();
}
}
static void gf256_from_gf2(void *a, const void *b, size_t size) {
uint8 *ap = reinterpret_cast<uint8 *>(a);
const uint8 *bp = reinterpret_cast<const uint8 *>(b);
size *= 8;
for (size_t i = 0; i < size; i++, ap++) {
*ap = (bp[i / 8] >> (i % 8)) & 1;
}
}
};
#if TD_SSE3
class Simd_sse : public Simd_null {
public:
static constexpr size_t alignment() {
return 32;
}
static std::string get_name() {
return "With SSE";
}
static bool is_aligned_pointer(const void *ptr) {
return ::td::is_aligned_pointer<alignment()>(ptr);
}
static void gf256_add(void *a, const void *b, size_t size) {
DCHECK(is_aligned_pointer(a));
DCHECK(is_aligned_pointer(b));
uint8 *ap = reinterpret_cast<uint8 *>(a);
const uint8 *bp = reinterpret_cast<const uint8 *>(b);
__m128i *ap128 = reinterpret_cast<__m128i *>(ap);
const __m128i *bp128 = reinterpret_cast<const __m128i *>(bp);
for (size_t idx = 0; idx < size; idx += 16) {
_mm_storeu_si128(ap128, _mm_xor_si128(_mm_loadu_si128(ap128), _mm_loadu_si128(bp128)));
ap128++;
bp128++;
}
}
static void gf256_mul(void *a, uint8 u, size_t size) {
DCHECK(is_aligned_pointer(a));
uint8 *ap = reinterpret_cast<uint8 *>(a);
const __m128i mask = _mm_set1_epi8(0x0f);
const __m128i urow_hi = _mm_loadu_si128(reinterpret_cast<const __m128i *>(Octet::OctMulHi[u]));
const __m128i urow_lo = _mm_loadu_si128(reinterpret_cast<const __m128i *>(Octet::OctMulLo[u]));
__m128i *ap128 = reinterpret_cast<__m128i *>(ap);
for (size_t idx = 0; idx < size; idx += 16) {
__m128i ax = _mm_loadu_si128(ap128);
__m128i lo = _mm_and_si128(ax, mask);
ax = _mm_srli_epi64(ax, 4);
__m128i hi = _mm_and_si128(ax, mask);
lo = _mm_shuffle_epi8(urow_lo, lo);
hi = _mm_shuffle_epi8(urow_hi, hi);
_mm_storeu_si128(ap128, _mm_xor_si128(lo, hi));
ap128++;
}
}
static void gf256_add_mul(void *a, const void *b, uint8 u, size_t size) {
DCHECK(is_aligned_pointer(a));
DCHECK(is_aligned_pointer(b));
uint8 *ap = reinterpret_cast<uint8 *>(a);
const uint8 *bp = reinterpret_cast<const uint8 *>(b);
const __m128i mask = _mm_set1_epi8(0x0f);
const __m128i urow_hi = _mm_loadu_si128(reinterpret_cast<const __m128i *>(Octet::OctMulHi[u]));
const __m128i urow_lo = _mm_loadu_si128(reinterpret_cast<const __m128i *>(Octet::OctMulLo[u]));
__m128i *ap128 = reinterpret_cast<__m128i *>(ap);
const __m128i *bp128 = reinterpret_cast<const __m128i *>(bp);
for (size_t idx = 0; idx < size; idx += 16) {
__m128i bx = _mm_loadu_si128(bp128++);
__m128i lo = _mm_and_si128(bx, mask);
bx = _mm_srli_epi64(bx, 4);
__m128i hi = _mm_and_si128(bx, mask);
lo = _mm_shuffle_epi8(urow_lo, lo);
hi = _mm_shuffle_epi8(urow_hi, hi);
_mm_storeu_si128(ap128, _mm_xor_si128(_mm_loadu_si128(ap128), _mm_xor_si128(lo, hi)));
ap128++;
}
}
};
#endif // SSSE3
#ifdef TD_AVX2
class Simd_avx : public Simd_sse {
public:
static std::string get_name() {
return "With AVX";
}
static void gf256_add(void *a, const void *b, size_t size) {
DCHECK(is_aligned_pointer(a));
DCHECK(is_aligned_pointer(b));
uint8 *ap = reinterpret_cast<uint8 *>(a);
const uint8 *bp = reinterpret_cast<const uint8 *>(b);
__m256i *ap256 = reinterpret_cast<__m256i *>(ap);
const __m256i *bp256 = reinterpret_cast<const __m256i *>(bp);
for (size_t idx = 0; idx < size; idx += 32) {
_mm256_storeu_si256(ap256, _mm256_xor_si256(_mm256_loadu_si256(ap256), _mm256_loadu_si256(bp256)));
ap256++;
bp256++;
}
}
static __m256i get_mask(const uint32 mask) {
// abcd -> abcd * 8
__m256i vmask(_mm256_set1_epi32(mask));
// abcd * 8 -> aaaaaaaabbbbbbbbccccccccdddddddd
const __m256i shuffle(
_mm256_setr_epi64x(0x0000000000000000, 0x0101010101010101, 0x0202020202020202, 0x0303030303030303));
vmask = _mm256_shuffle_epi8(vmask, shuffle);
const __m256i bit_mask(_mm256_set1_epi64x(0x7fbfdfeff7fbfdfe));
vmask = _mm256_or_si256(vmask, bit_mask);
return _mm256_and_si256(_mm256_cmpeq_epi8(vmask, _mm256_set1_epi64x(-1)), _mm256_set1_epi8(1));
}
static void gf256_from_gf2(void *a, const void *b, size_t size) {
DCHECK(is_aligned_pointer(a));
DCHECK(size % 4 == 0);
__m256i *ap256 = reinterpret_cast<__m256i *>(a);
const uint32 *bp = reinterpret_cast<const uint32 *>(b);
size /= 4;
for (size_t i = 0; i < size; i++, bp++, ap256++) {
*ap256 = get_mask(*bp);
}
}
static __attribute__((noinline)) void gf256_mul(void *a, uint8 u, size_t size) {
const __m128i urow_hi_small = _mm_load_si128(reinterpret_cast<const __m128i *>(Octet::OctMulHi[u]));
const __m256i urow_hi = _mm256_broadcastsi128_si256(urow_hi_small);
const __m128i urow_lo_small = _mm_load_si128(reinterpret_cast<const __m128i *>(Octet::OctMulLo[u]));
const __m256i urow_lo = _mm256_broadcastsi128_si256(urow_lo_small);
const __m256i mask = _mm256_set1_epi8(0x0f);
__m256i *ap256 = (__m256i *)a;
for (size_t idx = 0; idx < size; idx += 32) {
__m256i ax = _mm256_load_si256(ap256);
__m256i lo = _mm256_and_si256(ax, mask);
ax = _mm256_srli_epi64(ax, 4);
__m256i hi = _mm256_and_si256(ax, mask);
lo = _mm256_shuffle_epi8(urow_lo, lo);
hi = _mm256_shuffle_epi8(urow_hi, hi);
_mm256_store_si256(ap256, _mm256_xor_si256(lo, hi));
ap256++;
}
}
static __attribute__((noinline)) void gf256_add_mul(void *a, const void *b, uint8 u, size_t size) {
const __m128i urow_hi_small = _mm_load_si128(reinterpret_cast<const __m128i *>(Octet::OctMulHi[u]));
const __m256i urow_hi = _mm256_broadcastsi128_si256(urow_hi_small);
const __m128i urow_lo_small = _mm_load_si128(reinterpret_cast<const __m128i *>(Octet::OctMulLo[u]));
const __m256i urow_lo = _mm256_broadcastsi128_si256(urow_lo_small);
const __m256i mask = _mm256_set1_epi8(0x0f);
__m256i *ap256 = (__m256i *)a;
const __m256i *bp256 = (const __m256i *)b;
for (size_t idx = 0; idx < size; idx += 32) {
__m256i bx = _mm256_load_si256(bp256++);
__m256i lo = _mm256_and_si256(bx, mask);
bx = _mm256_srli_epi64(bx, 4);
__m256i hi = _mm256_and_si256(bx, mask);
lo = _mm256_shuffle_epi8(urow_lo, lo);
hi = _mm256_shuffle_epi8(urow_hi, hi);
_mm256_store_si256(ap256, _mm256_xor_si256(_mm256_load_si256(ap256), _mm256_xor_si256(lo, hi)));
ap256++;
}
}
};
#endif // AVX2
#if TD_AVX2
using Simd = Simd_avx;
#elif TD_SSE3
using Simd = Simd_sse;
#else
using Simd = Simd_null;
#endif
} // namespace td

20
tdfec/td/fec/algebra/SparseMatrixGF2.cpp Normal file
View file

@ -0,0 +1,20 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/fec/algebra/SparseMatrixGF2.h"
char disable_linker_warning_about_empty_file_sparsematrixgf2_cpp TD_UNUSED;

277
tdfec/td/fec/algebra/SparseMatrixGF2.h Normal file
View file

@ -0,0 +1,277 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/fec/algebra/MatrixGF2.h"
namespace td {
inline std::vector<uint32> inverse_permutation(Span<uint32> p) {
std::vector<uint32> res(p.size());
for (size_t i = 0; i < p.size(); i++) {
res[p[i]] = narrow_cast<uint32>(i);
}
return res;
}
class IdentityGenerator {
public:
IdentityGenerator(uint32 N) : N(N) {
}
template <class F>
void generate(F&& f) const {
for (uint32 col = 0; col < N; col++) {
f(col, col);
}
}
uint32 non_zeroes() const {
return N;
}
uint32 cols() const {
return N;
}
uint32 rows() const {
return N;
}
private:
uint32 N;
};
template <class GeneratorT>
class PermutationGenerator {
public:
PermutationGenerator(const GeneratorT& m, Span<uint32> p) : m_(m), p_(inverse_permutation(p)) {
}
uint32 non_zeroes() const {
return m_.non_zeroes();
}
uint32 rows() const {
return m_.rows();
}
uint32 cols() const {
return m_.cols();
}
template <class F>
void generate(F&& f) const {
m_.generate([&f, this](auto row, auto col) { f(row, p_[col]); });
}
private:
const GeneratorT& m_;
std::vector<uint32> p_;
};
template <class GeneratorT>
class TransposeGenerator {
public:
explicit TransposeGenerator(const GeneratorT& m) : m_(m) {
}
uint32 non_zeroes() const {
return m_.non_zeroes();
}
uint32 rows() const {
return m_.cols();
}
uint32 cols() const {
return m_.rows();
}
template <class F>
void generate(F&& f) const {
m_.generate([&f](auto row, auto col) { f(col, row); });
}
private:
const GeneratorT& m_;
std::vector<uint32> p_;
};
class SparseMatrixGF2 {
public:
uint32 non_zeroes() const {
return narrow_cast<uint32>(data_.size());
}
Span<uint32> col(uint32 i) const {
return Span<uint32>(data_.data() + col_offset_[i], col_size(i));
}
uint32 col_size(uint32 i) const {
return col_offset_[i + 1] - col_offset_[i];
}
uint32 cols() const {
return cols_;
}
uint32 rows() const {
return rows_;
}
template <class F>
void generate(F&& f) const {
return block_for_each(0, 0, rows_, cols_, f);
}
template <class F>
void block_for_each(uint32 row_from, uint32 col_from, uint32 row_size, uint32 col_size, F&& f) const {
auto col_till = col_from + col_size;
auto row_till = row_from + row_size;
for (uint32 col_i = col_from; col_i < col_till; col_i++) {
auto col_span = col(col_i);
auto* it = row_from == 0 ? col_span.begin() : std::lower_bound(col_span.begin(), col_span.end(), row_from);
while (it != col_span.end() && *it < row_till) {
f(*it - row_from, col_i - col_from);
it++;
}
}
}
MatrixGF2 block_dense(uint32 row_from, uint32 col_from, uint32 row_size, uint32 col_size) const {
MatrixGF2 res(row_size, col_size);
res.set_zero();
block_for_each(row_from, col_from, row_size, col_size, [&](auto row, auto col) { res.set_one(row, col); });
return res;
}
template <class GeneratorT>
explicit SparseMatrixGF2(GeneratorT&& generator) : rows_(generator.rows()), cols_(generator.cols()) {
data_.resize(generator.non_zeroes());
col_offset_.resize(cols_ + 1, 0);
generator.generate([&](uint32 row, uint32 col) {
LOG_DCHECK(row < rows_ && col < cols_) << "(" << row << "," << col << ") (" << rows_ << "," << cols_ << ")";
col_offset_[col + 1]++;
});
for (uint32 i = 1; i < col_offset_.size(); i++) {
col_offset_[i] += col_offset_[i - 1];
}
auto col_pos = col_offset_;
generator.generate([&](uint32 row, uint32 col) { data_[col_pos[col]++] = row; });
for (uint32 col_i = 0; col_i < cols_; col_i++) {
auto c = col(col_i);
for (size_t j = 1; j < c.size(); j++) {
LOG_DCHECK(c[j] > c[j - 1]) << c[j] << " > " << c[j - 1] << tag("row", col_i);
}
}
}
class BlockView {
public:
BlockView(uint32 row_offset, uint32 col_offset, uint32 row_size, uint32 col_size, const SparseMatrixGF2& m)
: row_offset_(row_offset), col_offset_(col_offset), row_size_(row_size), col_size_(col_size), m_(m) {
}
uint32 cols() const {
return col_size_;
}
uint32 rows() const {
return row_size_;
}
uint32 non_zeroes() const {
uint32 res = 0;
m_.block_for_each(row_offset_, col_offset_, row_size_, col_size_, [&res](auto row, auto col) { res++; });
return res;
}
template <class F>
void generate(F&& f) const {
m_.block_for_each(row_offset_, col_offset_, row_size_, col_size_, [&f](auto row, auto col) { f(row, col); });
}
private:
uint32 row_offset_;
uint32 col_offset_;
uint32 row_size_;
uint32 col_size_;
const SparseMatrixGF2& m_;
};
SparseMatrixGF2 block_sparse(uint32 row_from, uint32 col_from, uint32 row_size, uint32 col_size) const {
return SparseMatrixGF2(BlockView(row_from, col_from, row_size, col_size, *this));
}
SparseMatrixGF2 transpose() const {
return SparseMatrixGF2(TransposeGenerator<SparseMatrixGF2>(*this));
}
SparseMatrixGF2 apply_col_permutation(Span<uint32> p) const {
return SparseMatrixGF2(PermutationGenerator<SparseMatrixGF2>(*this, p));
}
SparseMatrixGF2 apply_row_permutation(Span<uint32> p) const {
return transpose().apply_col_permutation(p).transpose();
}
private:
uint32 rows_{0};
uint32 cols_{0};
std::vector<uint32> data_;
std::vector<uint32> col_offset_;
};
template <class M>
M operator*(const SparseMatrixGF2& a, const M& b) {
M res(a.rows(), b.cols());
res.set_zero();
a.generate([&](auto row, auto col) { res.row_add(row, b.row(col)); });
return res;
}
template <class... ArgsT>
class BlockGenerator {
public:
BlockGenerator(uint32 rows, uint32 cols, ArgsT&&... args)
: rows_(rows), cols_(cols), tuple_(std::forward_as_tuple(std::forward<ArgsT>(args)...)) {
}
template <class F>
void generate(F&& f) const {
uint32 row_offset = 0;
uint32 next_row_offset = 0;
uint32 col_offset = 0;
tuple_for_each(tuple_, [&](auto& g) {
if (col_offset == 0) {
next_row_offset = row_offset + g.rows();
} else {
CHECK(next_row_offset == row_offset + g.rows());
}
g.generate([&](auto row, auto col) { f(row_offset + row, col_offset + col); });
col_offset += g.cols();
if (col_offset >= cols_) {
CHECK(col_offset == cols_);
col_offset = 0;
row_offset = next_row_offset;
}
});
}
uint32 non_zeroes() const {
uint32 res = 0;
tuple_for_each(tuple_, [&](auto& g) { res += g.non_zeroes(); });
return res;
}
uint32 cols() const {
return cols_;
}
uint32 rows() const {
return rows_;
}
private:
uint32 rows_;
uint32 cols_;
std::tuple<ArgsT...> tuple_;
};
template <class... ArgsT>
auto block_generator(uint32 rows, uint32 cols, ArgsT&&... args) {
return BlockGenerator<ArgsT...>(rows, cols, std::forward<ArgsT>(args)...);
}
} // namespace td

28
tdfec/td/fec/common/SymbolRef.h Normal file
View file

@ -0,0 +1,28 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/Slice.h"
namespace td {
namespace raptorq {
struct SymbolRef {
uint32 id;
Slice data;
};
} // namespace raptorq
} // namespace td

46
tdfec/td/fec/common/SymbolsView.cpp Normal file
View file

@ -0,0 +1,46 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/fec/common/SymbolsView.h"
namespace td {
namespace raptorq {
SymbolsView::SymbolsView(size_t symbols_count, size_t symbol_size, Slice data) {
symbols_.reserve(symbols_count);
zero_symbol_ = std::string(symbol_size, '\0');
last_symbol_ = std::string(symbol_size, '\0');
for (uint32 symbol_i = 0; symbol_i < symbols_count; symbol_i++) {
auto offset = symbol_i * symbol_size;
Slice symbol;
if (offset < data.size()) {
symbol = data.substr(offset).truncate(symbol_size);
}
Slice slice;
if (symbol.empty()) {
slice = zero_symbol_;
} else if (symbol.size() < symbol_size) {
MutableSlice(last_symbol_).copy_from(symbol);
slice = last_symbol_;
} else {
slice = symbol;
}
symbols_.push_back(SymbolRef{symbol_i, slice});
}
}
} // namespace raptorq
} // namespace td

41
tdfec/td/fec/common/SymbolsView.h Normal file
View file

@ -0,0 +1,41 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/fec/common/SymbolRef.h"
#include "td/utils/Span.h"
namespace td {
namespace raptorq {
class SymbolsView {
public:
SymbolsView(size_t symbols_count, size_t symbol_size, Slice data);
SymbolsView(const SymbolsView &) = delete;
SymbolsView(SymbolsView &&) = delete;
Span<SymbolRef> symbols() const {
return symbols_;
}
private:
std::vector<SymbolRef> symbols_;
std::string zero_symbol_;
std::string last_symbol_;
};
} // namespace raptorq
} // namespace td

212
tdfec/td/fec/fec.cpp Normal file
View file

@ -0,0 +1,212 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/fec/fec.h"
#include "td/fec/raptorq/Encoder.h"
#include "td/fec/raptorq/Decoder.h"
#include "td/fec/online/Encoder.h"
#include "td/fec/online/Decoder.h"
namespace td {
namespace fec {
std::unique_ptr<RoundRobinEncoder> RoundRobinEncoder::create(BufferSlice data, size_t max_symbol_size) {
CHECK(max_symbol_size > 0);
return std::make_unique<RoundRobinEncoder>(std::move(data), max_symbol_size);
}
Symbol RoundRobinEncoder::gen_symbol(uint32 id) {
id %= narrow_cast<uint32>(symbols_count_);
auto offset = symbol_size_ * id;
return Symbol{id, data_.from_slice(data_.as_slice().substr(offset).truncate(symbol_size_))};
};
RoundRobinEncoder::Parameters RoundRobinEncoder::get_parameters() const {
Parameters parameters;
parameters.symbol_size = symbol_size_;
parameters.data_size = data_.size();
parameters.symbols_count = symbols_count_;
return parameters;
}
RoundRobinEncoder::RoundRobinEncoder(BufferSlice data, size_t max_symbol_size)
: data_(std::move(data)), symbol_size_(max_symbol_size) {
symbols_count_ = (data_.size() + symbol_size_ - 1) / symbol_size_;
rounded_data_size_ = symbols_count_ * symbol_size_;
}
std::unique_ptr<RoundRobinDecoder> RoundRobinDecoder::create(RoundRobinEncoder::Parameters parameters) {
return std::make_unique<RoundRobinDecoder>(parameters);
}
bool RoundRobinDecoder::may_try_decode() const {
return left_symbols_ == 0;
}
Result<DataWithEncoder> RoundRobinDecoder::try_decode(bool need_encoder) {
if (!may_try_decode()) {
return Status::Error("Not ready");
}
std::unique_ptr<Encoder> encoder;
if (need_encoder) {
encoder = RoundRobinEncoder::create(data_.copy(), symbol_size_);
}
return DataWithEncoder{data_.clone(), std::move(encoder)};
}
Status RoundRobinDecoder::add_symbol(Symbol symbol) {
if (symbol.data.size() != symbol_size_) {
return Status::Error("Symbol has invalid length");
}
auto pos = symbol.id % symbols_count_;
if (data_mask_[pos]) {
return td::Status::OK();
}
data_mask_[pos] = true;
auto offset = pos * symbol_size_;
data_.as_slice().substr(offset).truncate(symbol_size_).copy_from(symbol.data.as_slice());
left_symbols_--;
return td::Status::OK();
}
RoundRobinDecoder::RoundRobinDecoder(RoundRobinEncoder::Parameters parameters)
: data_(BufferSlice(parameters.data_size))
, data_mask_(parameters.symbols_count, false)
, left_symbols_(parameters.symbols_count)
, symbol_size_(parameters.symbol_size)
, symbols_count_(parameters.symbols_count) {
}
std::unique_ptr<RaptorQEncoder> RaptorQEncoder::create(BufferSlice data, size_t max_symbol_size) {
auto encoder = raptorq::Encoder::create(max_symbol_size, std::move(data)).move_as_ok();
return std::make_unique<RaptorQEncoder>(std::move(encoder));
}
Symbol RaptorQEncoder::gen_symbol(uint32 id) {
BufferSlice data(encoder_->get_parameters().symbol_size);
encoder_->gen_symbol(id, data.as_slice()).ensure();
return Symbol{id, std::move(data)};
}
RaptorQEncoder::Info RaptorQEncoder::get_info() const {
auto info = encoder_->get_info();
return {info.symbol_count, info.ready_symbol_count};
}
void RaptorQEncoder::prepare_more_symbols() {
encoder_->precalc();
}
RaptorQEncoder::Parameters RaptorQEncoder::get_parameters() const {
Parameters res;
auto p = encoder_->get_parameters();
res.symbol_size = p.symbol_size;
res.data_size = p.data_size;
res.symbols_count = p.symbols_count;
return res;
}
RaptorQEncoder::RaptorQEncoder(std::unique_ptr<raptorq::Encoder> encoder) : encoder_(std::move(encoder)) {
}
RaptorQEncoder::~RaptorQEncoder() = default;
std::unique_ptr<RaptorQDecoder> RaptorQDecoder::create(RaptorQEncoder::Parameters parameters) {
raptorq::Encoder::Parameters p;
p.symbol_size = parameters.symbol_size;
p.data_size = parameters.data_size;
p.symbols_count = parameters.symbols_count;
return std::make_unique<RaptorQDecoder>(raptorq::Decoder::create(p).move_as_ok());
}
bool RaptorQDecoder::may_try_decode() const {
return decoder_->may_try_decode();
}
Result<DataWithEncoder> RaptorQDecoder::try_decode(bool need_encoder) {
TRY_RESULT(data_with_encoder, decoder_->try_decode(need_encoder));
DataWithEncoder res;
res.data = std::move(data_with_encoder.data);
res.encoder = std::make_unique<RaptorQEncoder>(std::move(data_with_encoder.encoder));
return std::move(res);
}
Status RaptorQDecoder::add_symbol(Symbol symbol) {
return decoder_->add_symbol({symbol.id, symbol.data.as_slice()});
}
RaptorQDecoder::RaptorQDecoder(std::unique_ptr<raptorq::Decoder> decoder) : decoder_(std::move(decoder)) {
}
RaptorQDecoder::~RaptorQDecoder() = default;
std::unique_ptr<OnlineEncoder> OnlineEncoder::create(BufferSlice data, size_t max_symbol_size) {
auto encoder = online_code::Encoder::create(max_symbol_size, std::move(data)).move_as_ok();
return std::make_unique<OnlineEncoder>(std::move(encoder));
}
Symbol OnlineEncoder::gen_symbol(uint32 id) {
BufferSlice data(encoder_->get_parameters().symbol_size);
encoder_->gen_symbol(id, data.as_slice());
return Symbol{id, std::move(data)};
}
OnlineEncoder::Parameters OnlineEncoder::get_parameters() const {
Parameters res;
auto p = encoder_->get_parameters();
res.symbol_size = p.symbol_size;
res.data_size = p.data_size;
res.symbols_count = p.symbols_count;
return res;
}
OnlineEncoder::OnlineEncoder(std::unique_ptr<online_code::Encoder> encoder) : encoder_(std::move(encoder)) {
}
OnlineEncoder::~OnlineEncoder() = default;
std::unique_ptr<OnlineDecoder> OnlineDecoder::create(OnlineEncoder::Parameters parameters) {
online_code::Encoder::Parameters p;
p.symbol_size = parameters.symbol_size;
p.data_size = parameters.data_size;
p.symbols_count = parameters.symbols_count;
return std::make_unique<OnlineDecoder>(online_code::Decoder::create(p).move_as_ok(), parameters.symbol_size);
}
bool OnlineDecoder::may_try_decode() const {
return decoder_->is_ready();
}
Result<DataWithEncoder> OnlineDecoder::try_decode(bool need_encoder) {
if (!may_try_decode()) {
return Status::Error("Not ready yet");
}
std::unique_ptr<Encoder> encoder;
auto data = decoder_->get_data();
if (need_encoder) {
encoder = RoundRobinEncoder::create(data.copy(), symbol_size_);
}
return DataWithEncoder{std::move(data), nullptr};
}
Status OnlineDecoder::add_symbol(Symbol symbol) {
return decoder_->add_symbol({symbol.id, symbol.data.as_slice()});
}
OnlineDecoder::OnlineDecoder(std::unique_ptr<online_code::Decoder> decoder, size_t symbol_size)
: decoder_(std::move(decoder)), symbol_size_(symbol_size) {
}
OnlineDecoder::~OnlineDecoder() = default;
} // namespace fec
} // namespace td

196
tdfec/td/fec/fec.h Normal file
View file

@ -0,0 +1,196 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/buffer.h"
namespace td {
namespace raptorq {
class Encoder;
class Decoder;
} // namespace raptorq
namespace online_code {
class Encoder;
class Decoder;
} // namespace online_code
} // namespace td
namespace td {
namespace fec {
struct Symbol {
uint32 id;
BufferSlice data;
};
class Encoder;
struct DataWithEncoder {
BufferSlice data;
std::unique_ptr<Encoder> encoder;
};
class Encoder {
public:
virtual Symbol gen_symbol(uint32 id) = 0;
struct Info {
uint32 symbol_count;
uint32 ready_symbol_count;
};
virtual Info get_info() const {
return Info{static_cast<uint32>(-1), static_cast<uint32>(-1)};
}
// Concurrent calls are forbidden
virtual void prepare_more_symbols() {
}
// struct Parameters;
// Parameters get_parameters();
virtual ~Encoder() {
}
};
class Decoder {
public:
virtual ~Decoder() {
}
virtual bool may_try_decode() const = 0;
virtual Result<DataWithEncoder> try_decode(bool need_encoder) = 0;
virtual Status add_symbol(Symbol symbol) = 0;
};
class RoundRobinEncoder : public Encoder {
public:
static std::unique_ptr<RoundRobinEncoder> create(BufferSlice data, size_t max_symbol_size);
Symbol gen_symbol(uint32 id) override;
struct Parameters {
size_t data_size;
size_t symbol_size;
size_t symbols_count;
};
Parameters get_parameters() const;
RoundRobinEncoder(BufferSlice data, size_t max_symbol_size);
private:
BufferSlice data_;
size_t symbol_size_;
size_t symbols_count_;
size_t rounded_data_size_;
};
class RoundRobinDecoder : public Decoder {
public:
static std::unique_ptr<RoundRobinDecoder> create(RoundRobinEncoder::Parameters parameters);
bool may_try_decode() const override;
Result<DataWithEncoder> try_decode(bool need_encoder) override;
Status add_symbol(Symbol symbol) override;
RoundRobinDecoder(RoundRobinEncoder::Parameters parameters);
private:
BufferSlice data_;
vector<bool> data_mask_;
size_t left_symbols_;
size_t symbol_size_;
size_t symbols_count_;
};
class RaptorQEncoder : public Encoder {
public:
static std::unique_ptr<RaptorQEncoder> create(BufferSlice data, size_t max_symbol_size);
Symbol gen_symbol(uint32 id) override;
Info get_info() const override;
void prepare_more_symbols() override;
struct Parameters {
size_t data_size;
size_t symbol_size;
size_t symbols_count;
};
Parameters get_parameters() const;
RaptorQEncoder(std::unique_ptr<raptorq::Encoder> encoder);
~RaptorQEncoder();
private:
std::unique_ptr<raptorq::Encoder> encoder_;
};
class RaptorQDecoder : public Decoder {
public:
static std::unique_ptr<RaptorQDecoder> create(RaptorQEncoder::Parameters parameters);
bool may_try_decode() const override;
Result<DataWithEncoder> try_decode(bool need_encoder) override;
Status add_symbol(Symbol symbol) override;
RaptorQDecoder(std::unique_ptr<raptorq::Decoder> decoder);
~RaptorQDecoder();
private:
std::unique_ptr<raptorq::Decoder> decoder_;
BufferSlice res_;
};
class OnlineEncoder : public Encoder {
public:
static std::unique_ptr<OnlineEncoder> create(BufferSlice data, size_t max_symbol_size);
Symbol gen_symbol(uint32 id) override;
struct Parameters {
size_t data_size;
size_t symbol_size;
size_t symbols_count;
};
Parameters get_parameters() const;
OnlineEncoder(std::unique_ptr<online_code::Encoder> encoder);
~OnlineEncoder();
private:
std::unique_ptr<online_code::Encoder> encoder_;
};
class OnlineDecoder : public Decoder {
public:
static std::unique_ptr<OnlineDecoder> create(OnlineEncoder::Parameters parameters);
bool may_try_decode() const override;
Result<DataWithEncoder> try_decode(bool need_encoder) override;
Status add_symbol(Symbol symbol) override;
OnlineDecoder(std::unique_ptr<online_code::Decoder> decoder, size_t symbol_size);
~OnlineDecoder();
private:
std::unique_ptr<online_code::Decoder> decoder_;
size_t symbol_size_;
BufferSlice res_;
};
} // namespace fec
} // namespace td

77
tdfec/td/fec/online/Decoder.cpp Normal file
View file

@ -0,0 +1,77 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/fec/online/Decoder.h"
namespace td {
namespace online_code {
Result<std::unique_ptr<Decoder>> Decoder::create(Encoder::Parameters p) {
return std::make_unique<Decoder>(p.symbol_size, p.data_size);
}
Decoder::Decoder(size_t symbol_size, size_t data_size)
: parameters_(Rfc::get_parameters((data_size + symbol_size - 1) / symbol_size).move_as_ok())
, symbol_size_(symbol_size)
, data_size_(data_size) {
MatrixGF256 d_{1, symbol_size_};
d_.set_zero();
std::vector<uint32> e_cnt(parameters_.outer_encoding_blocks_count(), 0);
std::vector<std::vector<uint32>> e(parameters_.outer_encoding_blocks_count());
parameters_.outer_encoding_for_each([&](uint32 i, uint32 j) { e_cnt[i]++; });
for (uint32 i = 0; i < parameters_.outer_encoding_blocks_count(); i++) {
e[i].reserve(e_cnt[i]);
}
parameters_.outer_encoding_for_each([&](uint32 i, uint32 j) { e[i].push_back(j); });
for (uint32 i = 0; i < parameters_.outer_encoding_blocks_count(); i++) {
decoding_.add_equation(e[i], d_.row(0));
}
}
Status Decoder::add_symbol(raptorq::SymbolRef symbol_ref) {
if (symbol_ref.data.size() != symbol_size_) {
return Status::Error("Symbol has invalid length");
}
auto offset = decoding_.ready_symbols().size();
decoding_.add_equation(parameters_.get_inner_encoding_row(symbol_ref.id), symbol_ref.data);
//LOG(ERROR) << inner_decoding_.ready_symbols().size();
for (auto symbol_id : decoding_.ready_symbols().substr(offset)) {
if (symbol_id < parameters_.source_blocks_count()) {
ready_cnt_++;
}
}
return Status::OK();
}
bool Decoder::is_ready() const {
return ready_cnt_ == parameters_.source_blocks_count();
}
BufferSlice Decoder::get_data() const {
CHECK(is_ready());
BufferSlice res(data_size_);
for (uint32 i = 0; i < parameters_.source_blocks_count(); i++) {
auto dest = res.as_slice().substr(i * symbol_size_);
dest.copy_from(decoding_.get_symbol(i).truncate(dest.size()));
}
return res;
}
} // namespace online_code
} // namespace td

49
tdfec/td/fec/online/Decoder.h Normal file
View file

@ -0,0 +1,49 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/fec/online/Rfc.h"
#include "td/fec/common/SymbolRef.h"
#include "td/fec/algebra/BeliefPropagationDecoding.h"
#include "td/fec/online/Encoder.h"
namespace td {
namespace online_code {
class Decoder {
public:
static Result<std::unique_ptr<Decoder>> create(Encoder::Parameters p);
Decoder(size_t symbol_size, size_t data_size);
Status add_symbol(raptorq::SymbolRef symbol_ref);
bool is_ready() const;
BufferSlice get_data() const;
private:
Rfc::Parameters parameters_;
size_t symbol_size_;
size_t data_size_;
size_t ready_cnt_{0};
BeliefPropagationDecoding decoding_{parameters_.source_blocks_count() + parameters_.outer_encoding_blocks_count(),
symbol_size_};
};
} // namespace online_code
} // namespace td

63
tdfec/td/fec/online/Encoder.cpp Normal file
View file

@ -0,0 +1,63 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/fec/online/Encoder.h"
#include "td/fec/common/SymbolsView.h"
namespace td {
namespace online_code {
Result<std::unique_ptr<Encoder>> Encoder::create(size_t symbol_size, BufferSlice data) {
return std::make_unique<Encoder>(symbol_size, data.as_slice());
}
Encoder::Encoder(size_t symbol_size, Slice data)
: parameters_(Rfc::get_parameters((data.size() + symbol_size - 1) / symbol_size).move_as_ok())
, C_(parameters_.source_blocks_count() + parameters_.outer_encoding_blocks_count(), symbol_size)
, d_(1, symbol_size)
, data_size_(data.size()) {
raptorq::SymbolsView view(parameters_.source_blocks_count() + parameters_.outer_encoding_blocks_count(), symbol_size,
data);
for (auto &symbol : view.symbols()) {
C_.row_set(symbol.id, symbol.data);
}
parameters_.outer_encoding_for_each([&](uint32 i, uint32 j) {
if (j < parameters_.source_blocks_count()) {
C_.row_add(i + parameters_.source_blocks_count(), C_.row(j));
}
});
}
Encoder::Parameters Encoder::get_parameters() const {
Parameters p;
p.symbols_count = parameters_.source_blocks_count();
p.symbol_size = C_.cols();
p.data_size = data_size_;
return p;
}
Status Encoder::gen_symbol(uint32 symbol_id, MutableSlice slice) {
d_.set_zero();
for (auto id : parameters_.get_inner_encoding_row(symbol_id)) {
d_.row_add(0, C_.row(id));
}
slice.copy_from(d_.row(0));
return Status::OK();
}
} // namespace online_code
} // namespace td

48
tdfec/td/fec/online/Encoder.h Normal file
View file

@ -0,0 +1,48 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/fec/online/Rfc.h"
#include "td/fec/algebra/MatrixGF256.h"
#include "td/utils/buffer.h"
namespace td {
namespace online_code {
class Encoder {
public:
struct Parameters {
size_t symbols_count;
size_t symbol_size;
size_t data_size;
};
static Result<std::unique_ptr<Encoder>> create(size_t symbol_size, BufferSlice data);
Encoder(size_t symbol_size, Slice data);
Parameters get_parameters() const;
Status gen_symbol(uint32 symbol_id, MutableSlice slice);
private:
Rfc::Parameters parameters_;
MatrixGF256 C_;
MatrixGF256 d_;
size_t data_size_;
};
} // namespace online_code
} // namespace td

79
tdfec/td/fec/online/Rfc.cpp Normal file
View file

@ -0,0 +1,79 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/fec/online/Rfc.h"
namespace td {
namespace online_code {
Result<Rfc::Parameters> Rfc::get_parameters(size_t K) {
return Rfc::Parameters(K, 0.001, 3);
}
Rfc::Parameters::Parameters(size_t source_blocks, double epsilon, uint32 quality)
: source_blocks_(static_cast<uint32>(source_blocks))
, epsilon_(epsilon)
, quality_(quality)
, degree_distribution_{epsilon}
, outer_encoding_blocks_count_{
static_cast<uint32>(1 + 0.55 * quality * epsilon * static_cast<double>(source_blocks))} {
}
size_t Rfc::Parameters::source_blocks_count() const {
return source_blocks_;
}
size_t Rfc::Parameters::outer_encoding_blocks_count() const {
return outer_encoding_blocks_count_;
}
size_t Rfc::Parameters::estimated_packets() const {
return size_t(static_cast<double>(source_blocks_count() + outer_encoding_blocks_count()) * (1 + epsilon_));
}
Span<uint32> Rfc::Parameters::get_inner_encoding_row(size_t row_id) const {
Random r(static_cast<uint32>(row_id));
uint32 degree = degree_distribution_.get_degree(static_cast<uint32>(r() % (1 << 20)) * 1.0 / (1 << 20));
inner_distribution_.set_random(&r);
MutableSpan<uint32> res(row_buffer_.data(), degree);
for (auto &x : res) {
x = inner_distribution_();
}
//std::sort(res.begin(), res.end());
//res.truncate(std::unique(res.begin(), res.end()) - res.begin());
return res;
}
Rfc::Parameters::DegreeDistribution::DegreeDistribution(double epsilon) {
uint32 F = static_cast<uint32>(log(epsilon * epsilon / 4) / log(1 - epsilon / 2) + 1 - 1e-9);
double x = 1 - (1 + 1.0 / F) / (1 + epsilon);
p_.reserve(F);
p_.push_back(x);
for (uint32 i = 2; i <= F; i++) {
auto y = (1 - x) * F / ((F - 1.0) * i * (i - 1));
p_.push_back(p_.back() + y);
}
}
uint32 Rfc::Parameters::DegreeDistribution::get_degree(double x) const {
for (uint32 i = 0; i < p_.size(); i++) {
if (x < p_[i]) {
return i + 1;
}
}
UNREACHABLE();
return 1;
}
size_t Rfc::Parameters::DegreeDistribution::get_max_degree() const {
return p_.size();
}
} // namespace online_code
} // namespace td

103
tdfec/td/fec/online/Rfc.h Normal file
View file

@ -0,0 +1,103 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include <random>
#include <algorithm>
#include "td/utils/Status.h"
#include "td/utils/Span.h"
#include "td/utils/Random.h"
namespace td {
namespace online_code {
template <class Random>
class UniformDistributionSimple {
public:
UniformDistributionSimple(uint32 L, uint32 R) : L_(L), R_(R) {
}
void set_random(Random *random) {
random_ = random;
}
uint32 operator()() {
return static_cast<uint32>((*random_)() % (R_ - L_ + 1)) + L_;
}
private:
Random *random_{nullptr};
uint32 L_;
uint32 R_;
};
using Random = Random::Xorshift128plus;
using UniformDistribution = UniformDistributionSimple<Random>;
class Rfc {
public:
class Parameters {
public:
Parameters(size_t source_blocks, double epsilon, uint32 quality);
size_t source_blocks_count() const;
size_t outer_encoding_blocks_count() const;
size_t estimated_packets() const;
template <class F>
void outer_encoding_for_each(F &&f) const {
Random r(static_cast<uint32>(1));
outer_distribution_.set_random(&r);
for (uint32 j = 0; j < outer_encoding_blocks_count(); j++) {
f(j, uint32(source_blocks_count() + j));
}
for (uint32 i = 0; i < source_blocks_count(); i++) {
for (uint32 j = 0; j < quality_; j++) {
f(outer_distribution_(), i);
}
}
}
Span<uint32> get_inner_encoding_row(size_t row_id) const;
private:
struct DegreeDistribution {
public:
DegreeDistribution(double epsilon);
uint32 get_degree(double x) const;
size_t get_max_degree() const;
private:
std::vector<double> p_;
};
uint32 source_blocks_;
double epsilon_;
uint32 quality_;
DegreeDistribution degree_distribution_;
size_t outer_encoding_blocks_count_;
mutable std::vector<uint32> row_buffer_ = std::vector<uint32>(degree_distribution_.get_max_degree(), 0);
mutable UniformDistribution outer_distribution_{0, static_cast<uint32>(outer_encoding_blocks_count()) - 1};
mutable UniformDistribution inner_distribution_{
0, static_cast<uint32>(outer_encoding_blocks_count() + source_blocks_count()) - 1};
};
static Result<Parameters> get_parameters(size_t K);
};
} // namespace online_code
} // namespace td

151
tdfec/td/fec/raptorq/Decoder.cpp Normal file
View file

@ -0,0 +1,151 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/fec/raptorq/Decoder.h"
namespace td {
namespace raptorq {
Result<std::unique_ptr<Decoder>> Decoder::create(Encoder::Parameters p) {
TRY_RESULT(rfc_p, Rfc::get_parameters(p.symbols_count));
return std::make_unique<Decoder>(rfc_p, p.symbol_size, p.data_size);
}
Decoder::Decoder(const Rfc::Parameters &p, size_t symbol_size, size_t data_size) : p_(p) {
mask_ = vector<bool>(p.K, false);
mask_size_ = 0;
data_ = BufferSlice(p.K * symbol_size);
data_size_ = data_size;
symbol_size_ = symbol_size;
}
bool Decoder::may_try_decode() const {
return may_decode_;
}
Status Decoder::add_symbol(SymbolRef symbol) {
if (symbol.data.size() != symbol_size_) {
return Status::Error("Symbol has invalid length");
}
if (symbol.id < p_.K) {
add_small_symbol(symbol);
return Status::OK();
}
if (mask_size_ + slow_symbols_set_.size() >= p_.K + 10) {
return Status::OK();
}
add_big_symbol(symbol);
return Status::OK();
}
Result<Decoder::DataWithEncoder> Decoder::try_decode(bool need_encoder) {
if (!may_decode_) {
return Status::Error("Need more symbols");
}
optional<RawEncoder> raw_encoder;
if (mask_size_ < p_.K) {
flush_symbols();
may_decode_ = false;
TRY_RESULT(C, Solver::run(p_, symbols_));
raw_encoder = RawEncoder(p_, std::move(C));
for (uint32 i = 0; i < p_.K; i++) {
if (!mask_[i]) {
(*raw_encoder).gen_symbol(i, data_.as_slice().substr(i * symbol_size_, symbol_size_));
mask_[i] = true;
mask_size_++;
}
}
}
auto data = data_.from_slice(data_.as_slice().truncate(data_size_));
std::unique_ptr<Encoder> encoder;
if (need_encoder) {
encoder = std::make_unique<Encoder>(p_, symbol_size_, data.copy(), std::move(raw_encoder));
}
return DataWithEncoder{std::move(data), std::move(encoder)};
}
void Decoder::add_small_symbol(SymbolRef symbol) {
if (mask_[symbol.id]) {
return;
}
mask_size_++;
mask_[symbol.id] = true;
auto slice = data_.as_slice().substr(symbol.id * symbol_size_, symbol_size_);
slice.copy_from(symbol.data);
if (flush_symbols_) {
symbols_.push_back({symbol.id, slice});
}
update_may_decode();
}
void Decoder::add_big_symbol(SymbolRef symbol) {
if (!slow_path_) {
on_first_slow_path();
}
symbol.id += p_.K_padded - p_.K;
if (slow_symbols_set_.size() == slow_symbols_) {
// Got at least p.K + 10 different symbols
return;
}
size_t offset = slow_symbols_set_.size() * symbol_size_;
if (!slow_symbols_set_.insert(symbol.id).second) {
return;
}
auto slice = buffer_.as_slice().substr(offset, symbol_size_);
slice.copy_from(symbol.data);
symbols_.push_back({symbol.id, slice});
update_may_decode();
}
void Decoder::update_may_decode() {
size_t total_symbols = mask_size_ + slow_symbols_set_.size();
if (total_symbols < p_.K) {
return;
}
may_decode_ = true;
}
void Decoder::on_first_slow_path() {
slow_path_ = true;
slow_symbols_ = p_.K + 10 - mask_size_;
buffer_ = BufferSlice(slow_symbols_ * symbol_size_);
symbols_.reserve(p_.K + 10);
}
void Decoder::flush_symbols() {
if (flush_symbols_) {
return;
}
flush_symbols_ = true;
zero_symbol_ = std::string(symbol_size_, '\0');
for (uint32 i = p_.K; i < p_.K_padded; i++) {
symbols_.push_back({i, Slice(zero_symbol_)});
}
for (uint32 i = 0; i < p_.K; i++) {
if (mask_[i]) {
symbols_.push_back({i, data_.as_slice().substr(i * symbol_size_, symbol_size_)});
}
}
}
} // namespace raptorq
} // namespace td

71
tdfec/td/fec/raptorq/Decoder.h Normal file
View file

@ -0,0 +1,71 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/fec/raptorq/Solver.h"
#include "td/fec/raptorq/Encoder.h"
#include <set>
namespace td {
namespace raptorq {
class Decoder {
public:
static Result<std::unique_ptr<Decoder>> create(Encoder::Parameters p);
Decoder(const Rfc::Parameters &p, size_t symbol_size, size_t data_size);
Status add_symbol(SymbolRef symbol);
struct DataWithEncoder {
BufferSlice data;
std::unique_ptr<Encoder> encoder;
};
Result<DataWithEncoder> try_decode(bool need_encoder);
bool may_try_decode() const;
private:
Rfc::Parameters p_;
size_t symbol_size_;
bool may_decode_{false};
vector<bool> mask_;
size_t mask_size_{0};
BufferSlice data_;
size_t data_size_;
bool flush_symbols_{false};
bool slow_path_{false};
size_t slow_symbols_;
BufferSlice buffer_;
std::vector<SymbolRef> symbols_;
std::set<uint32> slow_symbols_set_;
std::string zero_symbol_;
void add_small_symbol(SymbolRef symbol);
void add_big_symbol(SymbolRef symbol);
void update_may_decode();
void on_first_slow_path();
void flush_symbols();
};
} // namespace raptorq
} // namespace td

81
tdfec/td/fec/raptorq/Encoder.cpp Normal file
View file

@ -0,0 +1,81 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/fec/raptorq/Encoder.h"
#include "td/fec/raptorq/Solver.h"
namespace td {
namespace raptorq {
Result<std::unique_ptr<Encoder>> Encoder::create(size_t symbol_size, BufferSlice data) {
TRY_RESULT(p, Rfc::get_parameters((data.size() + symbol_size - 1) / symbol_size));
return std::make_unique<Encoder>(p, symbol_size, std::move(data));
}
Encoder::Encoder(Rfc::Parameters p, size_t symbol_size, BufferSlice data, optional<RawEncoder> raw_encoder)
: p_(p)
, symbol_size_(symbol_size)
, data_(std::move(data))
, raw_encoder_{std::move(raw_encoder)}
, has_encoder_{bool(raw_encoder_)} {
}
Encoder::Parameters Encoder::get_parameters() const {
Parameters res;
res.symbol_size = symbol_size_;
res.data_size = data_.size();
res.symbols_count = p_.K;
return res;
}
Encoder::Info Encoder::get_info() const {
Info res;
res.symbol_count = 1 << 24;
res.ready_symbol_count = has_precalc() ? res.symbol_count : p_.K;
return res;
}
Status Encoder::gen_symbol(uint32 id, MutableSlice slice) {
if (id < p_.K) {
slice.copy_from(first_symbols_.symbols()[id].data);
return Status::OK();
}
if (!has_precalc()) {
return Status::Error("Precalc is not finished");
}
(*raw_encoder_).gen_symbol(id + p_.K_padded - p_.K, slice);
return Status::OK();
}
bool Encoder::has_precalc() const {
return has_encoder_.load(std::memory_order_acquire);
}
void Encoder::precalc() {
if (has_precalc()) {
return;
}
auto r_C = Solver::run(p_, first_symbols_.symbols());
LOG_IF(FATAL, r_C.is_error()) << r_C.error();
raw_encoder_ = RawEncoder(p_, r_C.move_as_ok());
has_encoder_ = true;
}
} // namespace raptorq
} // namespace td

68
tdfec/td/fec/raptorq/Encoder.h Normal file
View file

@ -0,0 +1,68 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/fec/raptorq/RawEncoder.h"
#include "td/fec/common/SymbolsView.h"
#include "td/utils/optional.h"
#include "td/utils/buffer.h"
#include <atomic>
namespace td {
namespace raptorq {
class Encoder {
public:
struct Parameters {
size_t symbols_count;
size_t symbol_size;
size_t data_size;
};
struct Info {
uint32 symbol_count;
uint32 ready_symbol_count;
};
static Result<std::unique_ptr<Encoder>> create(size_t symbol_size, BufferSlice data);
static std::unique_ptr<Encoder> create(size_t symbol_size, RawEncoder raw_encoder);
Encoder(Rfc::Parameters p, size_t symbol_size, BufferSlice data, optional<RawEncoder> raw_encoder = {});
Status gen_symbol(uint32 id, MutableSlice slice);
Parameters get_parameters() const;
Info get_info() const;
bool has_precalc() const;
// Must be called only once. No concurrent calls are allowed.
// Also it may be and should be called from another thread.
void precalc();
private:
Rfc::Parameters p_;
size_t symbol_size_;
BufferSlice data_;
SymbolsView first_symbols_{p_.K_padded, symbol_size_, data_.as_slice()};
optional<RawEncoder> raw_encoder_;
std::atomic<bool> has_encoder_{false};
};
} // namespace raptorq
} // namespace td

30
tdfec/td/fec/raptorq/RawEncoder.cpp Normal file
View file

@ -0,0 +1,30 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/fec/raptorq/RawEncoder.h"
namespace td {
namespace raptorq {
void RawEncoder::gen_symbol(uint32 id, MutableSlice to) const {
CHECK(to.size() == symbol_size());
d_.set_zero();
p_.encoding_row_for_each(p_.get_encoding_row(id), [&](auto row) { d_.row_add(0, C_.row(row)); });
to.copy_from(d_.row(0).truncate(symbol_size()));
}
} // namespace raptorq
} // namespace td

42
tdfec/td/fec/raptorq/RawEncoder.h Normal file
View file

@ -0,0 +1,42 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/fec/raptorq/Rfc.h"
#include "td/fec/algebra/MatrixGF256.h"
namespace td {
namespace raptorq {
class RawEncoder {
public:
RawEncoder(Rfc::Parameters p, MatrixGF256 C) : p_(p), C_(std::move(C)), d_{1, symbol_size()} {
}
size_t symbol_size() const {
return C_.cols();
}
void gen_symbol(uint32 id, MutableSlice to) const;
private:
Rfc::Parameters p_;
MatrixGF256 C_;
mutable MatrixGF256 d_;
};
} // namespace raptorq
} // namespace td

446
tdfec/td/fec/raptorq/Rfc.cpp Normal file
View file

@ -0,0 +1,446 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/fec/raptorq/Rfc.h"
#include <array>
namespace td {
namespace raptorq {
// 5.5. Random Numbers
//
// The four arrays V0, V1, V2, and V3 used in Section 5.3.5.1 are
// provided below. There are 256 entries in each of the four arrays.
// The indexing into each array starts at 0, and the entries are 32-bit
// unsigned integers.
namespace {
constexpr size_t V_size{256};
constexpr std::array<uint32, V_size> V0 = {
{251291136, 3952231631, 3370958628, 4070167936, 123631495, 3351110283, 3218676425, 2011642291, 774603218,
2402805061, 1004366930, 1843948209, 428891132, 3746331984, 1591258008, 3067016507, 1433388735, 504005498,
2032657933, 3419319784, 2805686246, 3102436986, 3808671154, 2501582075, 3978944421, 246043949, 4016898363,
649743608, 1974987508, 2651273766, 2357956801, 689605112, 715807172, 2722736134, 191939188, 3535520147,
3277019569, 1470435941, 3763101702, 3232409631, 122701163, 3920852693, 782246947, 372121310, 2995604341,
2045698575, 2332962102, 4005368743, 218596347, 3415381967, 4207612806, 861117671, 3676575285, 2581671944,
3312220480, 681232419, 307306866, 4112503940, 1158111502, 709227802, 2724140433, 4201101115, 4215970289,
4048876515, 3031661061, 1909085522, 510985033, 1361682810, 129243379, 3142379587, 2569842483, 3033268270,
1658118006, 932109358, 1982290045, 2983082771, 3007670818, 3448104768, 683749698, 778296777, 1399125101,
1939403708, 1692176003, 3868299200, 1422476658, 593093658, 1878973865, 2526292949, 1591602827, 3986158854,
3964389521, 2695031039, 1942050155, 424618399, 1347204291, 2669179716, 2434425874, 2540801947, 1384069776,
4123580443, 1523670218, 2708475297, 1046771089, 2229796016, 1255426612, 4213663089, 1521339547, 3041843489,
420130494, 10677091, 515623176, 3457502702, 2115821274, 2720124766, 3242576090, 854310108, 425973987,
325832382, 1796851292, 2462744411, 1976681690, 1408671665, 1228817808, 3917210003, 263976645, 2593736473,
2471651269, 4291353919, 650792940, 1191583883, 3046561335, 2466530435, 2545983082, 969168436, 2019348792,
2268075521, 1169345068, 3250240009, 3963499681, 2560755113, 911182396, 760842409, 3569308693, 2687243553,
381854665, 2613828404, 2761078866, 1456668111, 883760091, 3294951678, 1604598575, 1985308198, 1014570543,
2724959607, 3062518035, 3115293053, 138853680, 4160398285, 3322241130, 2068983570, 2247491078, 3669524410,
1575146607, 828029864, 3732001371, 3422026452, 3370954177, 4006626915, 543812220, 1243116171, 3928372514,
2791443445, 4081325272, 2280435605, 885616073, 616452097, 3188863436, 2780382310, 2340014831, 1208439576,
258356309, 3837963200, 2075009450, 3214181212, 3303882142, 880813252, 1355575717, 207231484, 2420803184,
358923368, 1617557768, 3272161958, 1771154147, 2842106362, 1751209208, 1421030790, 658316681, 194065839,
3241510581, 38625260, 301875395, 4176141739, 297312930, 2137802113, 1502984205, 3669376622, 3728477036,
234652930, 2213589897, 2734638932, 1129721478, 3187422815, 2859178611, 3284308411, 3819792700, 3557526733,
451874476, 1740576081, 3592838701, 1709429513, 3702918379, 3533351328, 1641660745, 179350258, 2380520112,
3936163904, 3685256204, 3156252216, 1854258901, 2861641019, 3176611298, 834787554, 331353807, 517858103,
3010168884, 4012642001, 2217188075, 3756943137, 3077882590, 2054995199, 3081443129, 3895398812, 1141097543,
2376261053, 2626898255, 2554703076, 401233789, 1460049922, 678083952, 1064990737, 940909784, 1673396780,
528881783, 1712547446, 3629685652, 1358307511}};
constexpr std::array<uint32, V_size> V1 = {
{807385413, 2043073223, 3336749796, 1302105833, 2278607931, 541015020, 1684564270, 372709334, 3508252125,
1768346005, 1270451292, 2603029534, 2049387273, 3891424859, 2152948345, 4114760273, 915180310, 3754787998,
700503826, 2131559305, 1308908630, 224437350, 4065424007, 3638665944, 1679385496, 3431345226, 1779595665,
3068494238, 1424062773, 1033448464, 4050396853, 3302235057, 420600373, 2868446243, 311689386, 259047959,
4057180909, 1575367248, 4151214153, 110249784, 3006865921, 4293710613, 3501256572, 998007483, 499288295,
1205710710, 2997199489, 640417429, 3044194711, 486690751, 2686640734, 2394526209, 2521660077, 49993987,
3843885867, 4201106668, 415906198, 19296841, 2402488407, 2137119134, 1744097284, 579965637, 2037662632,
852173610, 2681403713, 1047144830, 2982173936, 910285038, 4187576520, 2589870048, 989448887, 3292758024,
506322719, 176010738, 1865471968, 2619324712, 564829442, 1996870325, 339697593, 4071072948, 3618966336,
2111320126, 1093955153, 957978696, 892010560, 1854601078, 1873407527, 2498544695, 2694156259, 1927339682,
1650555729, 183933047, 3061444337, 2067387204, 228962564, 3904109414, 1595995433, 1780701372, 2463145963,
307281463, 3237929991, 3852995239, 2398693510, 3754138664, 522074127, 146352474, 4104915256, 3029415884,
3545667983, 332038910, 976628269, 3123492423, 3041418372, 2258059298, 2139377204, 3243642973, 3226247917,
3674004636, 2698992189, 3453843574, 1963216666, 3509855005, 2358481858, 747331248, 1957348676, 1097574450,
2435697214, 3870972145, 1888833893, 2914085525, 4161315584, 1273113343, 3269644828, 3681293816, 412536684,
1156034077, 3823026442, 1066971017, 3598330293, 1979273937, 2079029895, 1195045909, 1071986421, 2712821515,
3377754595, 2184151095, 750918864, 2585729879, 4249895712, 1832579367, 1192240192, 946734366, 31230688,
3174399083, 3549375728, 1642430184, 1904857554, 861877404, 3277825584, 4267074718, 3122860549, 666423581,
644189126, 226475395, 307789415, 1196105631, 3191691839, 782852669, 1608507813, 1847685900, 4069766876,
3931548641, 2526471011, 766865139, 2115084288, 4259411376, 3323683436, 568512177, 3736601419, 1800276898,
4012458395, 1823982, 27980198, 2023839966, 869505096, 431161506, 1024804023, 1853869307, 3393537983,
1500703614, 3019471560, 1351086955, 3096933631, 3034634988, 2544598006, 1230942551, 3362230798, 159984793,
491590373, 3993872886, 3681855622, 903593547, 3535062472, 1799803217, 772984149, 895863112, 1899036275,
4187322100, 101856048, 234650315, 3183125617, 3190039692, 525584357, 1286834489, 455810374, 1869181575,
922673938, 3877430102, 3422391938, 1414347295, 1971054608, 3061798054, 830555096, 2822905141, 167033190,
1079139428, 4210126723, 3593797804, 429192890, 372093950, 1779187770, 3312189287, 204349348, 452421568,
2800540462, 3733109044, 1235082423, 1765319556, 3174729780, 3762994475, 3171962488, 442160826, 198349622,
45942637, 1324086311, 2901868599, 678860040, 3812229107, 19936821, 1119590141, 3640121682, 3545931032,
2102949142, 2828208598, 3603378023, 4135048896}};
constexpr std::array<uint32, V_size> V2 = {
{1629829892, 282540176, 2794583710, 496504798, 2990494426, 3070701851, 2575963183, 4094823972, 2775723650,
4079480416, 176028725, 2246241423, 3732217647, 2196843075, 1306949278, 4170992780, 4039345809, 3209664269,
3387499533, 293063229, 3660290503, 2648440860, 2531406539, 3537879412, 773374739, 4184691853, 1804207821,
3347126643, 3479377103, 3970515774,
1891731298, 2368003842, 3537588307, 2969158410, 4230745262, 831906319, 2935838131, 264029468, 120852739,
3200326460, 355445271, 2296305141, 1566296040, 1760127056, 20073893, 3427103620, 2866979760, 2359075957,
2025314291, 1725696734, 3346087406, 2690756527, 99815156, 4248519977, 2253762642, 3274144518, 598024568,
3299672435, 556579346, 4121041856, 2896948975, 3620123492, 918453629, 3249461198, 2231414958, 3803272287,
3657597946, 2588911389, 242262274, 1725007475, 2026427718, 46776484, 2873281403, 2919275846, 3177933051,
1918859160, 2517854537, 1857818511, 3234262050, 479353687, 200201308, 2801945841, 1621715769, 483977159,
423502325, 3689396064, 1850168397, 3359959416, 3459831930, 841488699, 3570506095, 930267420, 1564520841,
2505122797, 593824107, 1116572080, 819179184, 3139123629, 1414339336, 1076360795, 512403845, 177759256,
1701060666, 2239736419, 515179302, 2935012727, 3821357612, 1376520851, 2700745271, 966853647, 1041862223,
715860553, 171592961, 1607044257, 1227236688, 3647136358, 1417559141, 4087067551, 2241705880, 4194136288,
1439041934, 20464430, 119668151, 2021257232, 2551262694, 1381539058, 4082839035, 498179069, 311508499,
3580908637, 2889149671, 142719814, 1232184754, 3356662582, 2973775623, 1469897084, 1728205304, 1415793613,
50111003, 3133413359, 4074115275, 2710540611, 2700083070, 2457757663, 2612845330, 3775943755, 2469309260,
2560142753, 3020996369, 1691667711, 4219602776, 1687672168, 1017921622, 2307642321, 368711460, 3282925988,
213208029, 4150757489, 3443211944, 2846101972, 4106826684, 4272438675, 2199416468, 3710621281, 497564971,
285138276, 765042313, 916220877, 3402623607, 2768784621, 1722849097, 3386397442, 487920061, 3569027007,
3424544196, 217781973, 2356938519, 3252429414, 145109750, 2692588106, 2454747135, 1299493354, 4120241887,
2088917094, 932304329, 1442609203, 952586974, 3509186750, 753369054, 854421006, 1954046388, 2708927882,
4047539230, 3048925996, 1667505809, 805166441, 1182069088, 4265546268, 4215029527, 3374748959, 373532666,
2454243090, 2371530493, 3651087521, 2619878153, 1651809518, 1553646893, 1227452842, 703887512, 3696674163,
2552507603, 2635912901, 895130484, 3287782244, 3098973502, 990078774, 3780326506, 2290845203, 41729428,
1949580860, 2283959805, 1036946170, 1694887523, 4880696, 466000198, 2765355283, 3318686998, 1266458025,
3919578154, 3545413527, 2627009988, 3744680394, 1696890173, 3250684705, 4142417708, 915739411, 3308488877,
1289361460, 2942552331, 1169105979, 3342228712, 698560958, 1356041230, 2401944293, 107705232, 3701895363,
903928723, 3646581385, 844950914, 1944371367, 3863894844, 2946773319, 1972431613, 1706989237, 29917467,
3497665928}};
constexpr std::array<uint32, V_size> V3 = {
{1191369816, 744902811, 2539772235, 3213192037, 3286061266, 1200571165, 2463281260, 754888894, 714651270,
1968220972, 3628497775, 1277626456, 1493398934, 364289757, 2055487592, 3913468088, 2930259465, 902504567,
3967050355, 2056499403, 692132390, 186386657, 832834706, 859795816, 1283120926, 2253183716, 3003475205,
1755803552, 2239315142, 4271056352, 2184848469, 769228092, 1249230754, 1193269205, 2660094102, 642979613,
1687087994, 2726106182, 446402913, 4122186606, 3771347282, 37667136, 192775425, 3578702187, 1952659096,
3989584400, 3069013882, 2900516158, 4045316336, 3057163251, 1702104819, 4116613420, 3575472384, 2674023117,
1409126723, 3215095429, 1430726429, 2544497368, 1029565676, 1855801827, 4262184627, 1854326881, 2906728593,
3277836557, 2787697002, 2787333385, 3105430738, 2477073192, 748038573, 1088396515, 1611204853, 201964005,
3745818380, 3654683549, 3816120877, 3915783622, 2563198722, 1181149055, 33158084, 3723047845, 3790270906,
3832415204, 2959617497, 372900708, 1286738499, 1932439099, 3677748309, 2454711182, 2757856469, 2134027055,
2780052465, 3190347618, 3758510138, 3626329451, 1120743107, 1623585693, 1389834102, 2719230375, 3038609003,
462617590, 260254189, 3706349764, 2556762744, 2874272296, 2502399286, 4216263978, 2683431180, 2168560535,
3561507175, 668095726, 680412330, 3726693946, 4180630637, 3335170953, 942140968, 2711851085, 2059233412,
4265696278, 3204373534, 232855056, 881788313, 2258252172, 2043595984, 3758795150, 3615341325, 2138837681,
1351208537, 2923692473, 3402482785, 2105383425, 2346772751, 499245323, 3417846006, 2366116814, 2543090583,
1828551634, 3148696244, 3853884867, 1364737681, 2200687771, 2689775688, 232720625, 4071657318, 2671968983,
3531415031, 1212852141, 867923311, 3740109711, 1923146533, 3237071777, 3100729255, 3247856816, 906742566,
4047640575, 4007211572, 3495700105, 1171285262, 2835682655, 1634301229, 3115169925, 2289874706, 2252450179,
944880097, 371933491, 1649074501, 2208617414, 2524305981, 2496569844, 2667037160, 1257550794, 3399219045,
3194894295, 1643249887, 342911473, 891025733, 3146861835, 3789181526, 938847812, 1854580183, 2112653794,
2960702988, 1238603378, 2205280635, 1666784014, 2520274614, 3355493726, 2310872278, 3153920489, 2745882591,
1200203158, 3033612415, 2311650167, 1048129133, 4206710184, 4209176741, 2640950279, 2096382177, 4116899089,
3631017851, 4104488173, 1857650503, 3801102932, 445806934, 3055654640, 897898279, 3234007399, 1325494930,
2982247189, 1619020475, 2720040856, 885096170, 3485255499, 2983202469, 3891011124, 546522756, 1524439205,
2644317889, 2170076800, 2969618716, 961183518, 1081831074, 1037015347, 3289016286, 2331748669, 620887395,
303042654, 3990027945, 1562756376, 3413341792, 2059647769, 2823844432, 674595301, 2457639984, 4076754716,
2447737904, 1583323324, 625627134, 3076006391, 345777990, 1684954145, 879227329, 3436182180, 1522273219,
3802543817, 1456017040, 1897819847, 2970081129, 1382576028, 3820044861, 1044428167, 612252599, 3340478395,
2150613904, 3397625662, 3573635640, 3432275192}};
} // namespace
// 5.3.5.1. Random Number Generator
// The random number generator Rand[y, i, m] is defined as follows,
// where y is a non-negative integer, i is a non-negative integer less
// than 256, and m is a positive integer, and the value produced is an
// integer between 0 and m-1. Let V0, V1, V2, and V3 be the arrays
// provided in Section 5.5.
//
// Let
//
// o x0 = (y + i) mod 2^^8
//
// o x1 = (floor(y / 2^^8) + i) mod 2^^8
//
// o x2 = (floor(y / 2^^16) + i) mod 2^^8
//
// o x3 = (floor(y / 2^^24) + i) mod 2^^8
//
// Then
//
// Rand[y, i, m] = (V0[x0] ^ V1[x1] ^ V2[x2] ^ V3[x3]) % m
uint32 Rfc::random(uint32 y, uint32 i, uint32 m) {
auto x0 = (y + i) % 256;
auto x1 = (y / 256 + i) % 256;
auto x2 = (y / 256 / 256 + i) % 256;
auto x3 = (y / 256 / 256 / 256 + i) % 256;
auto res = V0[x0] ^ V1[x1] ^ V2[x2] ^ V3[x3];
if (m != 0) {
res %= m;
}
return res;
}
namespace {
constexpr std::array<Rfc::RawParameters, 1232> rfc_parameters = {
{{10, 254, 7, 10, 17}, {12, 630, 7, 10, 19}, {18, 682, 11, 10, 29},
{20, 293, 11, 10, 31}, {26, 80, 11, 10, 37}, {30, 566, 11, 10, 41},
{32, 860, 11, 10, 43}, {36, 267, 11, 10, 47}, {42, 822, 11, 10, 53},
{46, 506, 13, 10, 59}, {48, 589, 13, 10, 61}, {49, 87, 13, 10, 61},
{55, 520, 13, 10, 67}, {60, 159, 13, 10, 71}, {62, 235, 13, 10, 73},
{69, 157, 13, 10, 79}, {75, 502, 17, 10, 89}, {84, 334, 17, 10, 97},
{88, 583, 17, 10, 101}, {91, 66, 17, 10, 103}, {95, 352, 17, 10, 107},
{97, 365, 17, 10, 109}, {101, 562, 17, 10, 113}, {114, 5, 19, 10, 127},
{119, 603, 19, 10, 131}, {125, 721, 19, 10, 137}, {127, 28, 19, 10, 139},
{138, 660, 19, 10, 149}, {140, 829, 19, 10, 151}, {149, 900, 23, 10, 163},
{153, 930, 23, 10, 167}, {160, 814, 23, 10, 173}, {166, 661, 23, 10, 179},
{168, 693, 23, 10, 181}, {179, 780, 23, 10, 191}, {181, 605, 23, 10, 193},
{185, 551, 23, 10, 197}, {187, 777, 23, 10, 199}, {200, 491, 23, 10, 211},
{213, 396, 23, 10, 223}, {217, 764, 29, 10, 233}, {225, 843, 29, 10, 241},
{236, 646, 29, 10, 251}, {242, 557, 29, 10, 257}, {248, 608, 29, 10, 263},
{257, 265, 29, 10, 271}, {263, 505, 29, 10, 277}, {269, 722, 29, 10, 283},
{280, 263, 29, 10, 293}, {295, 999, 29, 10, 307}, {301, 874, 29, 10, 313},
{305, 160, 29, 10, 317}, {324, 575, 31, 10, 337}, {337, 210, 31, 10, 349},
{341, 513, 31, 10, 353}, {347, 503, 31, 10, 359}, {355, 558, 31, 10, 367},
{362, 932, 31, 10, 373}, {368, 404, 31, 10, 379}, {372, 520, 37, 10, 389},
{380, 846, 37, 10, 397}, {385, 485, 37, 10, 401}, {393, 728, 37, 10, 409},
{405, 554, 37, 10, 421}, {418, 471, 37, 10, 433}, {428, 641, 37, 10, 443},
{434, 732, 37, 10, 449}, {447, 193, 37, 10, 461}, {453, 934, 37, 10, 467},
{466, 864, 37, 10, 479}, {478, 790, 37, 10, 491}, {486, 912, 37, 10, 499},
{491, 617, 37, 10, 503}, {497, 587, 37, 10, 509}, {511, 800, 37, 10, 523},
{526, 923, 41, 10, 541}, {532, 998, 41, 10, 547}, {542, 92, 41, 10, 557},
{549, 497, 41, 10, 563}, {557, 559, 41, 10, 571}, {563, 667, 41, 10, 577},
{573, 912, 41, 10, 587}, {580, 262, 41, 10, 593}, {588, 152, 41, 10, 601},
{594, 526, 41, 10, 607}, {600, 268, 41, 10, 613}, {606, 212, 41, 10, 619},
{619, 45, 41, 10, 631}, {633, 898, 43, 10, 647}, {640, 527, 43, 10, 653},
{648, 558, 43, 10, 661}, {666, 460, 47, 10, 683}, {675, 5, 47, 10, 691},
{685, 895, 47, 10, 701}, {693, 996, 47, 10, 709}, {703, 282, 47, 10, 719},
{718, 513, 47, 10, 733}, {728, 865, 47, 10, 743}, {736, 870, 47, 10, 751},
{747, 239, 47, 10, 761}, {759, 452, 47, 10, 773}, {778, 862, 53, 10, 797},
{792, 852, 53, 10, 811}, {802, 643, 53, 10, 821}, {811, 543, 53, 10, 829},
{821, 447, 53, 10, 839}, {835, 321, 53, 10, 853}, {845, 287, 53, 10, 863},
{860, 12, 53, 10, 877}, {870, 251, 53, 10, 887}, {891, 30, 53, 10, 907},
{903, 621, 53, 10, 919}, {913, 555, 53, 10, 929}, {926, 127, 53, 10, 941},
{938, 400, 53, 10, 953}, {950, 91, 59, 10, 971}, {963, 916, 59, 10, 983},
{977, 935, 59, 10, 997}, {989, 691, 59, 10, 1009}, {1002, 299, 59, 10, 1021},
{1020, 282, 59, 10, 1039}, {1032, 824, 59, 10, 1051}, {1050, 536, 59, 11, 1069},
{1074, 596, 59, 11, 1093}, {1085, 28, 59, 11, 1103}, {1099, 947, 59, 11, 1117},
{1111, 162, 59, 11, 1129}, {1136, 536, 59, 11, 1153}, {1152, 1000, 61, 11, 1171},
{1169, 251, 61, 11, 1187}, {1183, 673, 61, 11, 1201}, {1205, 559, 61, 11, 1223},
{1220, 923, 61, 11, 1237}, {1236, 81, 67, 11, 1259}, {1255, 478, 67, 11, 1277},
{1269, 198, 67, 11, 1291}, {1285, 137, 67, 11, 1307}, {1306, 75, 67, 11, 1327},
{1347, 29, 67, 11, 1367}, {1361, 231, 67, 11, 1381}, {1389, 532, 67, 11, 1409},
{1404, 58, 67, 11, 1423}, {1420, 60, 67, 11, 1439}, {1436, 964, 71, 11, 1459},
{1461, 624, 71, 11, 1483}, {1477, 502, 71, 11, 1499}, {1502, 636, 71, 11, 1523},
{1522, 986, 71, 11, 1543}, {1539, 950, 71, 11, 1559}, {1561, 735, 73, 11, 1583},
{1579, 866, 73, 11, 1601}, {1600, 203, 73, 11, 1621}, {1616, 83, 73, 11, 1637},
{1649, 14, 73, 11, 1669}, {1673, 522, 79, 11, 1699}, {1698, 226, 79, 11, 1723},
{1716, 282, 79, 11, 1741}, {1734, 88, 79, 11, 1759}, {1759, 636, 79, 11, 1783},
{1777, 860, 79, 11, 1801}, {1800, 324, 79, 11, 1823}, {1824, 424, 79, 11, 1847},
{1844, 999, 79, 11, 1867}, {1863, 682, 83, 11, 1889}, {1887, 814, 83, 11, 1913},
{1906, 979, 83, 11, 1931}, {1926, 538, 83, 11, 1951}, {1954, 278, 83, 11, 1979},
{1979, 580, 83, 11, 2003}, {2005, 773, 83, 11, 2029}, {2040, 911, 89, 11, 2069},
{2070, 506, 89, 11, 2099}, {2103, 628, 89, 11, 2131}, {2125, 282, 89, 11, 2153},
{2152, 309, 89, 11, 2179}, {2195, 858, 89, 11, 2221}, {2217, 442, 89, 11, 2243},
{2247, 654, 89, 11, 2273}, {2278, 82, 97, 11, 2311}, {2315, 428, 97, 11, 2347},
{2339, 442, 97, 11, 2371}, {2367, 283, 97, 11, 2399}, {2392, 538, 97, 11, 2423},
{2416, 189, 97, 11, 2447}, {2447, 438, 97, 11, 2477}, {2473, 912, 97, 11, 2503},
{2502, 1, 97, 11, 2531}, {2528, 167, 97, 11, 2557}, {2565, 272, 97, 11, 2593},
{2601, 209, 101, 11, 2633}, {2640, 927, 101, 11, 2671}, {2668, 386, 101, 11, 2699},
{2701, 653, 101, 11, 2731}, {2737, 669, 101, 11, 2767}, {2772, 431, 101, 11, 2801},
{2802, 793, 103, 11, 2833}, {2831, 588, 103, 11, 2861}, {2875, 777, 107, 11, 2909},
{2906, 939, 107, 11, 2939}, {2938, 864, 107, 11, 2971}, {2979, 627, 107, 11, 3011},
{3015, 265, 109, 11, 3049}, {3056, 976, 109, 11, 3089}, {3101, 988, 113, 11, 3137},
{3151, 507, 113, 11, 3187}, {3186, 640, 113, 11, 3221}, {3224, 15, 113, 11, 3259},
{3265, 667, 113, 11, 3299}, {3299, 24, 127, 11, 3347}, {3344, 877, 127, 11, 3391},
{3387, 240, 127, 11, 3433}, {3423, 720, 127, 11, 3469}, {3466, 93, 127, 11, 3511},
{3502, 919, 127, 11, 3547}, {3539, 635, 127, 11, 3583}, {3579, 174, 127, 11, 3623},
{3616, 647, 127, 11, 3659}, {3658, 820, 127, 11, 3701}, {3697, 56, 127, 11, 3739},
{3751, 485, 127, 11, 3793}, {3792, 210, 127, 11, 3833}, {3840, 124, 127, 11, 3881},
{3883, 546, 127, 11, 3923}, {3924, 954, 131, 11, 3967}, {3970, 262, 131, 11, 4013},
{4015, 927, 131, 11, 4057}, {4069, 957, 131, 11, 4111}, {4112, 726, 137, 11, 4159},
{4165, 583, 137, 11, 4211}, {4207, 782, 137, 11, 4253}, {4252, 37, 137, 11, 4297},
{4318, 758, 137, 11, 4363}, {4365, 777, 137, 11, 4409}, {4418, 104, 139, 11, 4463},
{4468, 476, 139, 11, 4513}, {4513, 113, 149, 11, 4567}, {4567, 313, 149, 11, 4621},
{4626, 102, 149, 11, 4679}, {4681, 501, 149, 11, 4733}, {4731, 332, 149, 11, 4783},
{4780, 786, 149, 11, 4831}, {4838, 99, 149, 11, 4889}, {4901, 658, 149, 11, 4951},
{4954, 794, 149, 11, 5003}, {5008, 37, 151, 11, 5059}, {5063, 471, 151, 11, 5113},
{5116, 94, 157, 11, 5171}, {5172, 873, 157, 11, 5227}, {5225, 918, 157, 11, 5279},
{5279, 945, 157, 11, 5333}, {5334, 211, 157, 11, 5387}, {5391, 341, 157, 11, 5443},
{5449, 11, 163, 11, 5507}, {5506, 578, 163, 11, 5563}, {5566, 494, 163, 11, 5623},
{5637, 694, 163, 11, 5693}, {5694, 252, 163, 11, 5749}, {5763, 451, 167, 11, 5821},
{5823, 83, 167, 11, 5881}, {5896, 689, 167, 11, 5953}, {5975, 488, 173, 11, 6037},
{6039, 214, 173, 11, 6101}, {6102, 17, 173, 11, 6163}, {6169, 469, 173, 11, 6229},
{6233, 263, 179, 11, 6299}, {6296, 309, 179, 11, 6361}, {6363, 984, 179, 11, 6427},
{6427, 123, 179, 11, 6491}, {6518, 360, 179, 11, 6581}, {6589, 863, 181, 11, 6653},
{6655, 122, 181, 11, 6719}, {6730, 522, 191, 11, 6803}, {6799, 539, 191, 11, 6871},
{6878, 181, 191, 11, 6949}, {6956, 64, 191, 11, 7027}, {7033, 387, 191, 11, 7103},
{7108, 967, 191, 11, 7177}, {7185, 843, 191, 11, 7253}, {7281, 999, 193, 11, 7351},
{7360, 76, 197, 11, 7433}, {7445, 142, 197, 11, 7517}, {7520, 599, 197, 11, 7591},
{7596, 576, 199, 11, 7669}, {7675, 176, 211, 11, 7759}, {7770, 392, 211, 11, 7853},
{7855, 332, 211, 11, 7937}, {7935, 291, 211, 11, 8017}, {8030, 913, 211, 11, 8111},
{8111, 608, 211, 11, 8191}, {8194, 212, 211, 11, 8273}, {8290, 696, 211, 11, 8369},
{8377, 931, 223, 11, 8467}, {8474, 326, 223, 11, 8563}, {8559, 228, 223, 11, 8647},
{8654, 706, 223, 11, 8741}, {8744, 144, 223, 11, 8831}, {8837, 83, 223, 11, 8923},
{8928, 743, 223, 11, 9013}, {9019, 187, 223, 11, 9103}, {9111, 654, 227, 11, 9199},
{9206, 359, 227, 11, 9293}, {9303, 493, 229, 11, 9391}, {9400, 369, 233, 11, 9491},
{9497, 981, 233, 11, 9587}, {9601, 276, 239, 11, 9697}, {9708, 647, 239, 11, 9803},
{9813, 389, 239, 11, 9907}, {9916, 80, 239, 11, 10009}, {10017, 396, 241, 11, 10111},
{10120, 580, 251, 11, 10223}, {10241, 873, 251, 11, 10343}, {10351, 15, 251, 11, 10453},
{10458, 976, 251, 11, 10559}, {10567, 584, 251, 11, 10667}, {10676, 267, 257, 11, 10781},
{10787, 876, 257, 11, 10891}, {10899, 642, 257, 12, 11003}, {11015, 794, 257, 12, 11119},
{11130, 78, 263, 12, 11239}, {11245, 736, 263, 12, 11353}, {11358, 882, 269, 12, 11471},
{11475, 251, 269, 12, 11587}, {11590, 434, 269, 12, 11701}, {11711, 204, 269, 12, 11821},
{11829, 256, 271, 12, 11941}, {11956, 106, 277, 12, 12073}, {12087, 375, 277, 12, 12203},
{12208, 148, 277, 12, 12323}, {12333, 496, 281, 12, 12451}, {12460, 88, 281, 12, 12577},
{12593, 826, 293, 12, 12721}, {12726, 71, 293, 12, 12853}, {12857, 925, 293, 12, 12983},
{13002, 760, 293, 12, 13127}, {13143, 130, 293, 12, 13267}, {13284, 641, 307, 12, 13421},
{13417, 400, 307, 12, 13553}, {13558, 480, 307, 12, 13693}, {13695, 76, 307, 12, 13829},
{13833, 665, 307, 12, 13967}, {13974, 910, 307, 12, 14107}, {14115, 467, 311, 12, 14251},
{14272, 964, 311, 12, 14407}, {14415, 625, 313, 12, 14551}, {14560, 362, 317, 12, 14699},
{14713, 759, 317, 12, 14851}, {14862, 728, 331, 12, 15013}, {15011, 343, 331, 12, 15161},
{15170, 113, 331, 12, 15319}, {15325, 137, 331, 12, 15473}, {15496, 308, 331, 12, 15643},
{15651, 800, 337, 12, 15803}, {15808, 177, 337, 12, 15959}, {15977, 961, 337, 12, 16127},
{16161, 958, 347, 12, 16319}, {16336, 72, 347, 12, 16493}, {16505, 732, 347, 12, 16661},
{16674, 145, 349, 12, 16831}, {16851, 577, 353, 12, 17011}, {17024, 305, 353, 12, 17183},
{17195, 50, 359, 12, 17359}, {17376, 351, 359, 12, 17539}, {17559, 175, 367, 12, 17729},
{17742, 727, 367, 12, 17911}, {17929, 902, 367, 12, 18097}, {18116, 409, 373, 12, 18289},
{18309, 776, 373, 12, 18481}, {18503, 586, 379, 12, 18679}, {18694, 451, 379, 12, 18869},
{18909, 287, 383, 12, 19087}, {19126, 246, 389, 12, 19309}, {19325, 222, 389, 12, 19507},
{19539, 563, 397, 12, 19727}, {19740, 839, 397, 12, 19927}, {19939, 897, 401, 12, 20129},
{20152, 409, 401, 12, 20341}, {20355, 618, 409, 12, 20551}, {20564, 439, 409, 12, 20759},
{20778, 95, 419, 13, 20983}, {20988, 448, 419, 13, 21191}, {21199, 133, 419, 13, 21401},
{21412, 938, 419, 13, 21613}, {21629, 423, 431, 13, 21841}, {21852, 90, 431, 13, 22063},
{22073, 640, 431, 13, 22283}, {22301, 922, 433, 13, 22511}, {22536, 250, 439, 13, 22751},
{22779, 367, 439, 13, 22993}, {23010, 447, 443, 13, 23227}, {23252, 559, 449, 13, 23473},
{23491, 121, 457, 13, 23719}, {23730, 623, 457, 13, 23957}, {23971, 450, 457, 13, 24197},
{24215, 253, 461, 13, 24443}, {24476, 106, 467, 13, 24709}, {24721, 863, 467, 13, 24953},
{24976, 148, 479, 13, 25219}, {25230, 427, 479, 13, 25471}, {25493, 138, 479, 13, 25733},
{25756, 794, 487, 13, 26003}, {26022, 247, 487, 13, 26267}, {26291, 562, 491, 13, 26539},
{26566, 53, 499, 13, 26821}, {26838, 135, 499, 13, 27091}, {27111, 21, 503, 13, 27367},
{27392, 201, 509, 13, 27653}, {27682, 169, 521, 13, 27953}, {27959, 70, 521, 13, 28229},
{28248, 386, 521, 13, 28517}, {28548, 226, 523, 13, 28817}, {28845, 3, 541, 13, 29131},
{29138, 769, 541, 13, 29423}, {29434, 590, 541, 13, 29717}, {29731, 672, 541, 13, 30013},
{30037, 713, 547, 13, 30323}, {30346, 967, 547, 13, 30631}, {30654, 368, 557, 14, 30949},
{30974, 348, 557, 14, 31267}, {31285, 119, 563, 14, 31583}, {31605, 503, 569, 14, 31907},
{31948, 181, 571, 14, 32251}, {32272, 394, 577, 14, 32579}, {32601, 189, 587, 14, 32917},
{32932, 210, 587, 14, 33247}, {33282, 62, 593, 14, 33601}, {33623, 273, 593, 14, 33941},
{33961, 554, 599, 14, 34283}, {34302, 936, 607, 14, 34631}, {34654, 483, 607, 14, 34981},
{35031, 397, 613, 14, 35363}, {35395, 241, 619, 14, 35731}, {35750, 500, 631, 14, 36097},
{36112, 12, 631, 14, 36457}, {36479, 958, 641, 14, 36833}, {36849, 524, 641, 14, 37201},
{37227, 8, 643, 14, 37579}, {37606, 100, 653, 14, 37967}, {37992, 339, 653, 14, 38351},
{38385, 804, 659, 14, 38749}, {38787, 510, 673, 14, 39163}, {39176, 18, 673, 14, 39551},
{39576, 412, 677, 14, 39953}, {39980, 394, 683, 14, 40361}, {40398, 830, 691, 15, 40787},
{40816, 535, 701, 15, 41213}, {41226, 199, 701, 15, 41621}, {41641, 27, 709, 15, 42043},
{42067, 298, 709, 15, 42467}, {42490, 368, 719, 15, 42899}, {42916, 755, 727, 15, 43331},
{43388, 379, 727, 15, 43801}, {43840, 73, 733, 15, 44257}, {44279, 387, 739, 15, 44701},
{44729, 457, 751, 15, 45161}, {45183, 761, 751, 15, 45613}, {45638, 855, 757, 15, 46073},
{46104, 370, 769, 15, 46549}, {46574, 261, 769, 15, 47017}, {47047, 299, 787, 15, 47507},
{47523, 920, 787, 15, 47981}, {48007, 269, 787, 15, 48463}, {48489, 862, 797, 15, 48953},
{48976, 349, 809, 15, 49451}, {49470, 103, 809, 15, 49943}, {49978, 115, 821, 15, 50461},
{50511, 93, 821, 16, 50993}, {51017, 982, 827, 16, 51503}, {51530, 432, 839, 16, 52027},
{52062, 340, 853, 16, 52571}, {52586, 173, 853, 16, 53093}, {53114, 421, 857, 16, 53623},
{53650, 330, 863, 16, 54163}, {54188, 624, 877, 16, 54713}, {54735, 233, 877, 16, 55259},
{55289, 362, 883, 16, 55817}, {55843, 963, 907, 16, 56393}, {56403, 471, 907, 16, 56951}}};
} // namespace
Result<Rfc::Parameters> Rfc::get_parameters(size_t K) {
for (auto &p : rfc_parameters) {
if (p.K_padded >= K) {
return Rfc::Parameters(narrow_cast<uint32>(K), p);
}
}
return Status::Error("K is too big");
}
Rfc::Parameters::Parameters(uint32 K, RawParameters raw_parameters) {
this->K = K;
K_padded = raw_parameters.K_padded;
J = raw_parameters.J;
S = raw_parameters.S;
H = raw_parameters.H;
W = raw_parameters.W;
L = K_padded + S + H;
P = L - W;
U = P - H;
B = W - S;
P1 = P + 1;
auto is_prime = [](uint32 n) {
if (n <= 3)
return true;
if (n % 2 == 0 || n % 3 == 0)
return false;
uint32 i = 5;
uint32 w = 2;
while (i * i <= n) {
if (n % i == 0)
return false;
i += w;
w = 6 - w;
}
return true;
};
while (!is_prime(P1)) {
P1++;
}
}
uint32 Rfc::Parameters::get_degree(uint32 v) const {
static const std::array<uint32, 31> degree_distribution = {
{0, 5243, 529531, 704294, 791675, 844104, 879057, 904023, 922747, 937311, 948962,
958494, 966438, 973160, 978921, 983914, 988283, 992138, 995565, 998631, 1001391, 1003887,
1006157, 1008229, 1010129, 1011876, 1013490, 1014983, 1016370, 1017662, 1048576}};
for (uint32 i = 0; i < degree_distribution.size(); i++) {
if (v < degree_distribution[i]) {
return std::min(W - 2, i);
}
}
UNREACHABLE();
return 0;
}
Rfc::EncodingRow Rfc::Parameters::get_encoding_row(uint32 X /*ISI*/) const {
uint32 A = 53591 + J * 997;
if (A % 2 == 0) {
A++;
}
uint32 B_local = 10267 * (J + 1);
uint32 y = B_local + X * A;
uint32 v = random(y, 0, 1 << 20);
uint32 d = get_degree(v);
uint32 a = 1 + random(y, 1, W - 1);
uint32 b = random(y, 2, W);
uint32 d1;
if (d < 4) {
d1 = 2 + random(X, 3, 2);
} else {
d1 = 2;
}
uint32 a1 = 1 + random(X, 4, P1 - 1);
uint32 b1 = random(X, 5, P1);
EncodingRow r;
std::tie(r.d, r.a, r.b, r.d1, r.a1, r.b1) = std::tie(d, a, b, d1, a1, b1);
return r;
}
uint32 Rfc::Parameters::encoding_row_size(const EncodingRow &t) const {
return t.d + t.d1;
}
} // namespace raptorq
} // namespace td

243
tdfec/td/fec/raptorq/Rfc.h Normal file
View file

@ -0,0 +1,243 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/Status.h"
#include "td/fec/algebra/MatrixGF256.h"
#include "td/fec/algebra/SparseMatrixGF2.h"
namespace td {
namespace raptorq {
class Rfc {
public:
struct RawParameters {
uint32 K_padded;
uint32 J;
uint32 S;
uint32 H;
uint32 W;
};
struct EncodingRow {
uint32 d; // [1,30] LT degree
uint32 a; // [0,W)
uint32 b; // [0,W)
uint32 d1; // [2,3] PI degree
uint32 a1; // [0,P1)
uint32 b1; // [0,P1)
};
struct Parameters {
uint32 K;
uint32 K_padded;
uint32 J;
uint32 S;
uint32 H;
uint32 W;
uint32 L;
uint32 P;
uint32 P1;
uint32 U;
uint32 B;
EncodingRow get_encoding_row(uint32 X /*ISI*/) const;
uint32 encoding_row_size(const EncodingRow &t) const;
template <class F>
void encoding_row_for_each(EncodingRow t, F &&f) const {
f(t.b);
for (uint16 j = 1; j < t.d; ++j) {
t.b = (t.b + t.a) % W;
f(t.b);
}
while (t.b1 >= P)
t.b1 = (t.b1 + t.a1) % P1;
f(W + t.b1);
for (uint16 j = 1; j < t.d1; ++j) {
t.b1 = (t.b1 + t.a1) % P1;
while (t.b1 >= P)
t.b1 = (t.b1 + t.a1) % P1;
f(W + t.b1);
}
}
Parameters(uint32 K, RawParameters raw_parameters);
class LDPC1 {
public:
LDPC1(uint32 S, uint32 B) : S(S), B(B) {
}
static void sort_inplace(uint32 &a, uint32 &b, uint32 &c) {
if (a > c) {
std::swap(a, c);
}
if (b > c) {
std::swap(b, c);
}
if (a > b) {
std::swap(a, b);
}
DCHECK(a < b && b < c);
}
template <class F>
void generate(F &&f) const {
for (uint32 col = 0; col < B; col++) {
uint32 i = col / S;
uint32 shift = col % S;
uint32 a = shift;
uint32 b = (i + 1 + shift) % S;
uint32 c = (2 * (i + 1) + shift) % S;
DCHECK(a != b);
DCHECK(a != c);
DCHECK(b != c);
sort_inplace(a, b, c);
f(a, col);
f(b, col);
f(c, col);
}
}
uint32 non_zeroes() const {
return B * 3;
}
uint32 cols() const {
return B;
}
uint32 rows() const {
return S;
}
private:
uint32 S;
uint32 B;
};
// 1100000
// 0110000
// 0011000
// .......
// 1100000
class LDPC2 {
public:
LDPC2(uint32 rows, uint32 cols) : rows_(rows), cols_(cols) {
}
template <class F>
void generate(F &&f) const {
for (uint32 row = 0; row < rows_; row++) {
f(row, row % cols_);
f(row, (row + 1) % cols_);
}
}
uint32 non_zeroes() const {
return rows_ * 2;
}
uint32 cols() const {
return cols_;
}
uint32 rows() const {
return rows_;
}
private:
uint32 rows_;
uint32 cols_;
};
static MatrixGF256 HDPC_multiply(const uint32 rows, MatrixGF256 v) {
Octet alpha = Octet(Octet::oct_exp(1));
for (uint32 i = 1; i < v.rows(); i++) {
v.row_add_mul(i, i - 1, alpha);
}
MatrixGF256 u(rows, v.cols());
u.set_zero();
for (uint32 i = 0; i < rows; i++) {
u.row_add_mul(i, v.row(narrow_cast<uint32>(v.rows() - 1)), Octet(Octet::oct_exp(i % 255)));
}
for (uint32 col = 0; col + 1 < v.rows(); col++) {
auto a = Rfc::random(col + 1, 6, rows);
auto b = (a + Rfc::random(col + 1, 7, rows - 1) + 1) % rows;
u.row_add(a, v.row(col));
u.row_add(b, v.row(col));
}
return u;
}
class ENC {
public:
ENC(const Rfc::Parameters &p, Span<Rfc::EncodingRow> encoding_rows) : p(p), encoding_rows(encoding_rows) {
}
template <class F>
void generate(F &&f) const {
uint32 row = 0;
for (auto encoding_row : encoding_rows) {
p.encoding_row_for_each(encoding_row, [&f, row](auto col) { f(row, col); });
row++;
}
}
uint32 non_zeroes() const {
uint32 res = 0;
for (auto encoding_row : encoding_rows) {
res += p.encoding_row_size(encoding_row);
}
return res;
}
uint32 cols() const {
return p.L;
}
uint32 rows() const {
return narrow_cast<uint32>(encoding_rows.size());
}
private:
const Rfc::Parameters &p;
Span<Rfc::EncodingRow> encoding_rows;
};
LDPC1 get_LDPC1() const {
return LDPC1(S, B);
}
LDPC2 get_LDPC2() const {
return LDPC2(S, P);
}
ENC get_ENC(Span<EncodingRow> encoding_rows) const {
return ENC(*this, encoding_rows);
}
MatrixGF256 HDPC_multiply(MatrixGF256 v) const {
return HDPC_multiply(H, std::move(v));
}
SparseMatrixGF2 get_A_upper(Span<EncodingRow> encoding_rows) const {
return SparseMatrixGF2(block_generator(narrow_cast<uint32>(S + encoding_rows.size()), L, get_LDPC1(),
IdentityGenerator(S), get_LDPC2(), get_ENC(encoding_rows)));
}
private:
uint32 get_degree(uint32 v) const;
};
static uint32 random(uint32 y, uint32 i, uint32 m);
static Result<Parameters> get_parameters(size_t K);
};
} // namespace raptorq
} // namespace td

244
tdfec/td/fec/raptorq/Solver.cpp Normal file
View file

@ -0,0 +1,244 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/fec/raptorq/Solver.h"
#include "td/fec/algebra/GaussianElimination.h"
#include "td/fec/algebra/InactivationDecoding.h"
#include "td/utils/Timer.h"
#include <map>
namespace td {
namespace raptorq {
MatrixGF256 create_D(const Rfc::Parameters &p, Span<SymbolRef> symbols) {
auto symbol_size = symbols[0].data.size();
MatrixGF256 D(p.S + p.H + symbols.size(), symbol_size);
D.set_zero();
auto offset = p.S;
for (auto &symbol : symbols) {
for (size_t i = 0; i < symbol_size; i++) {
D.set(offset, i, Octet(symbol.data[i]));
}
offset++;
}
return D;
}
Result<MatrixGF256> Solver::run(const Rfc::Parameters &p, Span<SymbolRef> symbols) {
if (0) { // turns out gauss is slower even for small symbols count
auto encoding_rows = transform(symbols, [&p](auto &symbol) { return p.get_encoding_row(symbol.id); });
MatrixGF256 A(p.S + p.H + symbols.size(), p.L);
A.set_zero();
auto A_upper = p.get_A_upper(encoding_rows);
A_upper.block_for_each(0, 0, A_upper.rows(), A_upper.cols(), [&](auto x, auto y) { A.set(x, y, Octet(1)); });
MatrixGF256 tmp(A.cols() - p.H, A.cols() - p.H);
tmp.set_zero();
for (size_t i = 0; i < tmp.cols(); i++) {
tmp.set(i, i, Octet(1));
}
auto HDCP = p.HDPC_multiply(std::move(tmp));
MatrixGF256 HDCP2(p.H, p.L - p.H);
MatrixGF256 IH(p.H, p.H);
IH.set_zero();
for (size_t i = 0; i < p.H; i++) {
IH.set(i, i, Octet(1));
}
A.set_from(HDCP, A_upper.rows(), 0);
A.set_from(IH, A_upper.rows(), HDCP.cols());
auto D = create_D(p, symbols);
auto C = GaussianElimination::run(std::move(A), std::move(D));
return C;
}
PerfWarningTimer x("solve");
Timer timer;
auto perf_log = [&](Slice message) {
if (GET_VERBOSITY_LEVEL() > VERBOSITY_NAME(DEBUG)) {
static std::map<std::string, double> total;
static double total_all = 0;
auto elapsed = timer.elapsed();
auto current_total = total[message.str()] += elapsed;
total_all += elapsed;
LOG(DEBUG) << "PERF: " << message << " " << timer << " " << current_total / total_all * 100;
timer = {};
}
};
// Solve linear system
// A * C = D
// C - intermeidate symbols
// D - encoded symbols and restriction symbols.
//
// A:
// +--------+-----+-------+
// | LDPC1 | I_S | LDPC2|
// +--------+-----+-------+
// | ENC |
// +---------------+------+
// | HDCP | I_H |
// +---------------+------+
CHECK(p.K_padded <= symbols.size());
auto encoding_rows = transform(symbols, [&p](auto &symbol) { return p.get_encoding_row(symbol.id); });
// Generate matrix A_upper: sparse part of A, first S + K_padded rows.
SparseMatrixGF2 A_upper = p.get_A_upper(encoding_rows);
auto D = create_D(p, symbols);
perf_log("Generate sparse matrix");
// Run indactivation decoding.
// Lets call resulting lower triangualr matrix U
auto decoding_result = InactivationDecoding(A_upper, p.P).run();
perf_log("Inactivation decoding");
uint32 U_size = decoding_result.size;
auto row_permutation = std::move(decoding_result.p_rows);
while (row_permutation.size() < D.rows()) {
row_permutation.push_back(narrow_cast<uint32>(row_permutation.size()));
}
auto col_permutation = std::move(decoding_result.p_cols);
// +--------+---------+ +---------+
// | U | E | | D_upper |
// +--------+---------+ +---------+
// | G_left | G_right | * C = | |
// +--------+--+------+ | D_lower |
// |HDCP | I_H | | |
// +-----------+------+ +---------+
D = D.apply_row_permutation(row_permutation);
perf_log("D: apply permutation");
A_upper = A_upper.apply_row_permutation(row_permutation).apply_col_permutation(col_permutation);
perf_log("A_upper: apply permutation");
auto E = A_upper.block_dense(0, U_size, U_size, p.L - U_size);
perf_log("Calc E");
MatrixGF256 C(A_upper.cols(), D.cols());
C.set_from(D.block_view(0, 0, U_size, D.cols()), 0, 0);
// Make U Identity matrix and calculate E and D_upper.
for (uint32 i = 0; i < U_size; i++) {
for (auto row : A_upper.col(i)) {
if (row == i) {
continue;
}
if (row >= U_size) {
break;
}
E.row_add(row, i);
D.row_add(row, i); // this is SLOW
}
}
perf_log("Triangular -> Identity");
auto HDPC_left_multiply = [&](const MatrixGF256 &m) {
MatrixGF256 T(p.K_padded + p.S, m.cols());
T.set_zero();
for (uint32 i = 0; i < m.rows(); i++) {
T.row_set(col_permutation[i], m.row(i));
}
return p.HDPC_multiply(std::move(T));
};
SparseMatrixGF2 G_left = A_upper.block_sparse(U_size, 0, A_upper.rows() - U_size, U_size);
perf_log("G_left");
// Calculate small_A_upper
// small_A_upper = G_right
MatrixGF256 small_A_upper(A_upper.rows() - U_size, A_upper.cols() - U_size);
A_upper.block_for_each(U_size, U_size, A_upper.rows() - U_size, A_upper.cols() - U_size,
[&](auto row, auto col) { small_A_upper.set(row, col, Octet(1)); });
// small_A_upper += G_left * E
small_A_upper.add((G_left * E).to_gf256());
perf_log("small_A_upper");
// Calculate small_A_lower
MatrixGF256 small_A_lower(p.H, A_upper.cols() - U_size);
small_A_lower.set_zero();
for (uint32 i = 1; i <= p.H; i++) {
small_A_lower.set(small_A_lower.rows() - i, small_A_lower.cols() - i, Octet(1));
}
// Calculate HDPC_right and set it into small_A_lower
MatrixGF256 T(p.K_padded + p.S, p.K_padded + p.S - U_size);
T.set_zero();
for (uint32 i = 0; i < T.cols(); i++) {
T.set(col_permutation[i + T.rows() - T.cols()], i, Octet(1));
}
MatrixGF256 HDCP_right = p.HDPC_multiply(std::move(T));
small_A_lower.set_from(HDCP_right, 0, 0);
perf_log("small_A_lower");
// small_A_lower += HDPC_left * E
auto t = E.to_gf256();
perf_log("t");
small_A_lower.add(HDPC_left_multiply(std::move(t)));
perf_log("small_A_lower += HDPC_left * E");
MatrixGF256 D_upper(U_size, D.cols());
D_upper.set_from(D.block_view(0, 0, D_upper.rows(), D_upper.cols()), 0, 0);
// small_D_upper
MatrixGF256 small_D_upper(A_upper.rows() - U_size, D.cols());
small_D_upper.set_from(D.block_view(U_size, 0, small_D_upper.rows(), small_D_upper.cols()), 0, 0);
small_D_upper.add(G_left * D_upper);
perf_log("small_D_upper");
// small_D_lower
MatrixGF256 small_D_lower(p.H, D.cols());
small_D_lower.set_from(D.block_view(A_upper.rows(), 0, small_D_lower.rows(), small_D_lower.cols()), 0, 0);
perf_log("small_D_lower");
small_D_lower.add(HDPC_left_multiply(D_upper));
perf_log("small_D_lower += HDPC_left * D_upper");
// Combine small_A from small_A_lower and small_A_upper
MatrixGF256 small_A(small_A_upper.rows() + small_A_lower.rows(), small_A_upper.cols());
small_A.set_from(small_A_upper, 0, 0);
small_A.set_from(small_A_lower, small_A_upper.rows(), 0);
// Combine small_D from small_D_lower and small_D_upper
MatrixGF256 small_D(small_D_upper.rows() + small_D_lower.rows(), small_D_upper.cols());
small_D.set_from(small_D_upper, 0, 0);
small_D.set_from(small_D_lower, small_D_upper.rows(), 0);
TRY_RESULT(small_C, GaussianElimination::run(std::move(small_A), std::move(small_D)));
perf_log("gauss");
C.set_from(small_C.block_view(0, 0, C.rows() - U_size, C.cols()), U_size, 0);
SparseMatrixGF2 A_upper_t = A_upper.transpose();
for (uint32 row = 0; row < U_size; row++) {
for (auto col : A_upper_t.col(row)) {
if (col == row) {
continue;
}
C.row_add(row, col);
}
}
perf_log("Calc result");
auto res = C.apply_row_permutation(inverse_permutation(col_permutation));
perf_log("Apply permutation");
return std::move(res);
}
} // namespace raptorq
} // namespace td

35
tdfec/td/fec/raptorq/Solver.h Normal file
View file

@ -0,0 +1,35 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/Slice.h"
#include "td/utils/Span.h"
#include "td/fec/raptorq/Rfc.h"
#include "td/fec/common/SymbolRef.h"
namespace td {
namespace raptorq {
class Solver {
public:
static Result<MatrixGF256> run(const Rfc::Parameters &p, Span<SymbolRef> symbols);
};
} // namespace raptorq
} // namespace td

157
tdfec/test/LibRaptorQ.cpp Normal file
View file

@ -0,0 +1,157 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "LibRaptorQ.h"
#include "RaptorQ/RaptorQ_v1_hdr.hpp"
namespace RaptorQ = RaptorQ__v1;
namespace td {
namespace fec {
class SlowRaptorQEncoder::Impl {
public:
Impl(BufferSlice data, size_t max_symbol_size)
: encoder_(create_encoder(data.size(), max_symbol_size)), data_(std::move(data)), symbol_size_(max_symbol_size) {
encoder_->set_data(data_.as_slice().ubegin(), data_.as_slice().uend());
}
Symbol gen_symbol(uint32 id) {
BufferSlice symbol_data(symbol_size_);
auto *begin = symbol_data.as_slice().ubegin();
auto *end = symbol_data.as_slice().uend();
encoder_->encode(begin, end, id);
return {id, symbol_data.from_slice(Slice(symbol_data.as_slice().ubegin(), begin))};
}
Parameters get_parameters() const {
return Parameters{encoder_->symbols(), encoder_->symbol_size(), data_.size()};
}
Info get_info() const {
return {1 << 24, computed_ ? encoder_->max_repair() : encoder_->symbols()};
}
void prepare_more_symbols() {
encoder_->compute_sync();
computed_ = true;
}
private:
using Encoder = RaptorQ::Encoder<uint8 *, uint8 *>;
std::unique_ptr<Encoder> encoder_;
BufferSlice data_;
size_t symbol_size_;
bool computed_{false};
static std::unique_ptr<Encoder> create_encoder(size_t data_size, size_t symbol_size) {
auto min_symbols = (data_size + symbol_size - 1) / symbol_size;
RaptorQ::Block_Size block = RaptorQ::Block_Size::Block_10;
for (auto blk : *RaptorQ::blocks) {
// RaptorQ::blocks is a pointer to an array, just scan it to find your
// block.
if (static_cast<uint16>(blk) >= min_symbols) {
block = blk;
break;
}
}
CHECK(static_cast<uint16>(block) >= min_symbols);
return std::make_unique<Encoder>(block, symbol_size);
}
};
class SlowRaptorQDecoder::Impl {
public:
using Decoder = RaptorQ::Decoder<uint8 *, uint8 *>;
Impl(SlowRaptorQEncoder::Parameters parameters)
: decoder_(RaptorQ::Block_Size(parameters.symbols_count), parameters.symbol_size, Decoder::Report::COMPLETE)
, data_size_(parameters.data_size) {
}
bool may_try_decode() const {
return decoder_.can_decode();
}
Result<DataWithEncoder> try_decode(bool need_encoder) {
decoder_.decode_once();
if (!decoder_.ready()) {
return Status::Error("Not ready");
}
BufferSlice data(data_size_);
auto begin = data.as_slice().ubegin();
decoder_.decode_bytes(begin, data.as_slice().uend(), 0, 0);
std::unique_ptr<Encoder> encoder;
if (need_encoder) {
encoder = SlowRaptorQEncoder::create(data.copy(), decoder_.symbol_size());
}
return DataWithEncoder{std::move(data), std::move(encoder)};
}
void add_symbol(Symbol symbol) {
auto begin = symbol.data.as_slice().ubegin();
decoder_.add_symbol(begin, symbol.data.as_slice().uend(), symbol.id);
}
private:
Decoder decoder_;
size_t data_size_;
};
SlowRaptorQEncoder::~SlowRaptorQEncoder() = default;
std::unique_ptr<SlowRaptorQEncoder> SlowRaptorQEncoder::create(BufferSlice data, size_t max_symbol_size) {
return std::make_unique<SlowRaptorQEncoder>(std::move(data), max_symbol_size);
}
SlowRaptorQEncoder::SlowRaptorQEncoder(BufferSlice data, size_t max_symbol_size) {
impl_ = std::make_unique<Impl>(std::move(data), max_symbol_size);
}
Symbol SlowRaptorQEncoder::gen_symbol(uint32 id) {
return impl_->gen_symbol(id);
}
SlowRaptorQEncoder::Parameters SlowRaptorQEncoder::get_parameters() const {
return impl_->get_parameters();
}
Encoder::Info SlowRaptorQEncoder::get_info() const {
return impl_->get_info();
}
void SlowRaptorQEncoder::prepare_more_symbols() {
impl_->prepare_more_symbols();
}
SlowRaptorQDecoder::~SlowRaptorQDecoder() = default;
std::unique_ptr<SlowRaptorQDecoder> SlowRaptorQDecoder::create(SlowRaptorQEncoder::Parameters parameters) {
return std::make_unique<SlowRaptorQDecoder>(std::move(parameters));
}
bool SlowRaptorQDecoder::may_try_decode() const {
return impl_->may_try_decode();
}
Result<DataWithEncoder> SlowRaptorQDecoder::try_decode(bool need_encoder) {
return impl_->try_decode(need_encoder);
}
void SlowRaptorQDecoder::add_symbol(Symbol symbol) {
impl_->add_symbol(std::move(symbol));
}
SlowRaptorQDecoder::SlowRaptorQDecoder(SlowRaptorQEncoder::Parameters parameters) {
impl_ = std::make_unique<Impl>(std::move(parameters));
}
} // namespace fec
} // namespace td

68
tdfec/test/LibRaptorQ.h Normal file
View file

@ -0,0 +1,68 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/buffer.h"
#include "td/fec/fec.h"
namespace td {
namespace fec {
class SlowRaptorQEncoder : public Encoder {
public:
~SlowRaptorQEncoder();
static std::unique_ptr<SlowRaptorQEncoder> create(BufferSlice data, size_t max_symbol_size);
Symbol gen_symbol(uint32 id) override;
Info get_info() const override;
void prepare_more_symbols() override;
struct Parameters {
size_t symbols_count;
size_t symbol_size;
size_t data_size;
};
SlowRaptorQEncoder(BufferSlice data, size_t max_symbol_size);
Parameters get_parameters() const;
private:
class Impl;
std::unique_ptr<Impl> impl_;
};
class SlowRaptorQDecoder : public Decoder {
public:
~SlowRaptorQDecoder();
static std::unique_ptr<SlowRaptorQDecoder> create(SlowRaptorQEncoder::Parameters parameters);
bool may_try_decode() const override;
Result<DataWithEncoder> try_decode(bool need_encoder) override;
void add_symbol(Symbol symbol) override;
SlowRaptorQDecoder(SlowRaptorQEncoder::Parameters parameters);
private:
class Impl;
std::unique_ptr<Impl> impl_;
};
} // namespace fec
} // namespace td

242
tdfec/test/fec-test.cpp Normal file
View file

@ -0,0 +1,242 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
//#include "fec.h"
//#include "raptorq/Solver.h"
#include "td/fec/fec.h"
#include "td/fec/raptorq/Encoder.h"
#include "td/fec/raptorq/Decoder.h"
#if USE_LIBRAPTORQ
#include "LibRaptorQ.h"
#endif
#include "td/utils/tests.h"
#include <string>
td::Slice get_long_string() {
const size_t max_symbol_size = 200;
const size_t symbols_count = 100;
static std::string data = td::rand_string('a', 'z', max_symbol_size * symbols_count);
return data;
}
TEST(Fec, Simd) {
constexpr size_t size = td::Simd::alignment() * 1024;
alignas(td::Simd::alignment()) td::uint8 a[size];
alignas(td::Simd::alignment()) td::uint8 a_copy[size];
alignas(td::Simd::alignment()) td::uint8 b[size];
alignas(td::Simd::alignment()) td::uint8 d[8 * size];
td::Random::Xorshift128plus rnd(123);
for (auto k_size : {1, 2, 10, 1024}) {
auto a_size = k_size * td::Simd::alignment();
LOG(ERROR) << a_size;
for (size_t i = 0; i < a_size; i++) {
a_copy[i] = rnd() & 255;
b[i] = rnd() & 255;
}
std::vector<std::string> res;
std::vector<std::string> other_res;
bool is_first = true;
auto save_str = [&](auto str) {
if (is_first) {
res.push_back(str);
} else {
other_res.push_back(str);
}
};
auto save_a = [&] { save_str(td::Slice(a, a_size).str()); };
auto save_d = [&] { save_str(td::Slice(d, a_size * 8).str()); };
auto run = [&](auto simd) {
LOG(ERROR) << simd.get_name();
std::memcpy(a, a_copy, a_size);
simd.gf256_add(a, b, a_size);
save_a();
for (td::uint32 o = 0; o < 256; o++) {
std::memcpy(a, a_copy, a_size);
simd.gf256_add_mul(a, b, td::uint8(o), a_size);
save_a();
std::memcpy(a, a_copy, a_size);
simd.gf256_mul(a, td::uint8(o), a_size);
save_a();
}
std::memcpy(a, a_copy, a_size);
simd.gf256_from_gf2(d, a, a_size);
save_d();
if (is_first) {
is_first = false;
} else {
CHECK(res == other_res);
other_res.clear();
}
};
run(td::Simd_null());
#if TD_SSE3
run(td::Simd_sse());
#endif
#if TD_AVX2
run(td::Simd_avx());
#endif
run(td::Simd());
}
}
static const td::Slice tmp = get_long_string();
TEST(Fec, RaptorQFirstSymbols) {
auto data = get_long_string();
auto encoder = td::raptorq::Encoder::create(200, td::BufferSlice(data)).move_as_ok();
auto parameters = encoder->get_parameters();
auto decoder = td::raptorq::Decoder::create(parameters).move_as_ok();
std::string symbol(parameters.symbol_size, '\0');
std::string new_symbol(parameters.symbol_size, '\0');
encoder->precalc();
for (td::uint32 i = 0; i < 2; i++) {
encoder->gen_symbol(i + (1 << 21), symbol);
decoder->add_symbol({i + (1 << 21), td::Slice(symbol)});
}
for (td::uint32 i = 0; i < parameters.symbols_count; i++) {
td::uint32 id = i;
encoder->gen_symbol(id, symbol);
decoder->add_symbol({id, td::Slice(symbol)});
if (decoder->may_try_decode()) {
auto r = decoder->try_decode(true);
if (r.is_ok()) {
ASSERT_EQ(r.ok().data, data);
auto new_encoder = std::move(r.move_as_ok().encoder);
new_encoder->precalc();
auto check_id = [&](td::uint32 id) {
encoder->gen_symbol(id, symbol);
new_encoder->gen_symbol(id, new_symbol);
ASSERT_EQ(symbol, new_symbol);
};
check_id(0);
check_id(1);
check_id(1000000);
LOG(ERROR) << "ok";
return;
} else {
LOG(WARNING) << "SKIP";
}
}
}
UNREACHABLE();
}
TEST(Fec, RaptorQRandomSymbols) {
auto data = get_long_string();
auto encoder = td::raptorq::Encoder::create(200, td::BufferSlice(data)).move_as_ok();
encoder->precalc();
auto parameters = encoder->get_parameters();
auto decoder = td::raptorq::Decoder::create(parameters).move_as_ok();
std::string symbol(parameters.symbol_size, '\0');
for (size_t i = 0; i < parameters.symbols_count + 10; i++) {
auto id = td::Random::fast_uint32();
encoder->gen_symbol(id, symbol);
decoder->add_symbol({id, td::Slice(symbol)});
if (decoder->may_try_decode()) {
auto r = decoder->try_decode(false);
if (r.is_ok()) {
ASSERT_EQ(r.ok().data, data);
return;
}
}
}
UNREACHABLE();
}
template <class Encoder, class Decoder>
void fec_test(td::Slice data, size_t max_symbol_size) {
LOG(ERROR) << "!";
auto encoder = Encoder::create(td::BufferSlice(data), max_symbol_size);
LOG(ERROR) << "?";
std::vector<td::fec::Symbol> symbols;
auto parameters = encoder->get_parameters();
auto decoder = Decoder::create(parameters);
LOG(ERROR) << "?";
size_t sent_symbols = 0;
for (td::uint32 i = 0; i < data.size() / max_symbol_size * 20; i++) {
if (td::Random::fast(0, 5) != 0) {
if (encoder->get_info().ready_symbol_count <= i) {
encoder->prepare_more_symbols();
}
decoder->add_symbol(encoder->gen_symbol(i));
sent_symbols++;
if (decoder->may_try_decode()) {
auto res = decoder->try_decode(false);
if (res.is_ok()) {
ASSERT_EQ(res.ok().data.as_slice(), data);
LOG(ERROR) << sent_symbols << " / " << parameters.symbols_count;
return;
}
}
}
}
UNREACHABLE();
}
TEST(Fec, RoundRobin) {
const size_t max_symbol_size = 200;
std::string data = td::rand_string('a', 'z', max_symbol_size * 400);
fec_test<td::fec::RoundRobinEncoder, td::fec::RoundRobinDecoder>(data, max_symbol_size);
}
TEST(Fec, Online) {
const size_t max_symbol_size = 200;
std::string data = td::rand_string('a', 'z', max_symbol_size * 50000);
fec_test<td::fec::OnlineEncoder, td::fec::OnlineDecoder>(data, max_symbol_size);
}
#if USE_LIBRAPTORQ
TEST(Fec, SlowRaptorQ) {
const size_t max_symbol_size = 200;
std::string data = td::rand_string('a', 'z', max_symbol_size * 200);
fec_test<td::fec::SlowRaptorQEncoder, td::fec::SlowRaptorQDecoder>(data, max_symbol_size);
}
#endif
TEST(Fec, RaptorQFull) {
const size_t max_symbol_size = 200;
std::string data = td::rand_string('a', 'z', max_symbol_size * 50000);
fec_test<td::fec::RaptorQEncoder, td::fec::RaptorQDecoder>(data, max_symbol_size);
}
#if USE_LIBRAPTORQ
TEST(Fec, RaptorQEncoder) {
const size_t max_symbol_size = 200;
std::string data = td::rand_string('a', 'z', max_symbol_size * 200);
auto reference_encoder = td::fec::SlowRaptorQEncoder::create(td::BufferSlice(data), max_symbol_size);
auto checked_encoder = td::fec::RaptorQEncoder::create(td::BufferSlice(data), max_symbol_size);
reference_encoder->prepare_more_symbols();
checked_encoder->prepare_more_symbols();
for (td::uint32 i = 0; i < 1000000; i++) {
auto reference_symbol = reference_encoder->gen_symbol(i);
auto checked_symbol = checked_encoder->gen_symbol(i);
ASSERT_EQ(reference_symbol.data.as_slice(), checked_symbol.data.as_slice());
}
}
#endif