external/ton
1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-03-09 15:40:10 +00:00
Code Issues Releases Wiki Activity

initial commit

Browse source
This commit is contained in:
initial commit 2019-09-07 14:03:22 +04:00 • committed by vvaltman
commit c2da007f40
1610 changed files with 398047 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

114
tdfec/td/fec/algebra/BeliefPropagationDecoding.cpp Normal file
View file

@ -0,0 +1,114 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/fec/algebra/BeliefPropagationDecoding.h"
namespace td {
BeliefPropagationDecoding::BeliefPropagationDecoding(size_t symbols_count, size_t symbol_size)
: max_equation_count_{static_cast<size_t>(static_cast<double>(symbols_count) * 1.1 + 5)}
, C_{symbols_count, symbol_size}
, D_{max_equation_count_, symbol_size} {
equations_.reserve(max_equation_count_);
symbols_.resize(symbols_count);
edges_.resize(1);
}
Slice BeliefPropagationDecoding::get_symbol(uint32 symbol_id) const {
CHECK(symbols_[symbol_id].is_ready);
return C_.row(symbol_id);
}
void BeliefPropagationDecoding::add_equation(Span<uint32> symbol_ids, Slice data) {
if (equations_.size() >= D_.rows()) {
MatrixGF256 new_D(D_.rows() * 2, D_.cols());
new_D.set_from(D_, 0, 0);
D_ = std::move(new_D);
}
CHECK(symbol_ids.size() != 0);
uint32 equation_id = static_cast<uint32>(equations_.size());
D_.row_set(equation_id, data);
EquationInfo equation;
for (auto symbol_id : symbol_ids) {
CHECK(symbol_id < symbols_.size());
auto &symbol = symbols_[symbol_id];
if (symbol.is_ready) {
D_.row_add(equation_id, C_.row(symbol_id));
} else {
equation.symbols_xor ^= symbol_id;
equation.symbols_count++;
edges_.push_back({equation_id, symbol.head_});
symbol.head_ = uint32(edges_.size() - 1);
}
}
if (equation.symbols_count == 0) {
return;
}
equations_.push_back(equation);
if (equation.symbols_count == 1) {
ready_equations_.push_back(equation_id);
loop();
}
}
bool BeliefPropagationDecoding::is_ready() const {
return ready_symbols().size() == C_.rows();
}
Span<uint32> BeliefPropagationDecoding::ready_symbols() const {
return ready_symbols_;
}
void BeliefPropagationDecoding::loop() {
while (!is_ready() && !ready_equations_.empty()) {
auto equation_id = ready_equations_.back();
ready_equations_.pop_back();
auto &equation = equations_[equation_id];
LOG_CHECK(equation.symbols_count <= 1) << equation.symbols_count;
if (equation.symbols_count == 0) {
continue;
}
auto symbol_id = equation.symbols_xor;
auto &symbol = symbols_[symbol_id];
LOG_CHECK(symbol_id < symbols_.size())
<< equation.symbols_xor << " " << equation.symbols_count << " " << equation_id;
if (symbol.is_ready) {
continue;
}
C_.row_set(symbol_id, D_.row(equation_id));
symbol.is_ready = true;
ready_symbols_.push_back(symbol_id);
for (auto i = symbol.head_; i != 0;) {
auto &edge = edges_[i];
auto next_equation_id = edge.value;
i = edge.next;
D_.row_add(next_equation_id, C_.row(symbol_id));
auto &next_equation = equations_[next_equation_id];
next_equation.symbols_xor ^= symbol_id;
next_equation.symbols_count--;
if (next_equation.symbols_count == 1) {
ready_equations_.push_back(next_equation_id);
}
}
}
}
} // namespace td

62
tdfec/td/fec/algebra/BeliefPropagationDecoding.h Normal file
View file

@ -0,0 +1,62 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/buffer.h"
#include "td/utils/Span.h"
#include "td/fec/algebra/MatrixGF256.h"
namespace td {
class BeliefPropagationDecoding {
public:
explicit BeliefPropagationDecoding(size_t symbols_count, size_t symbol_size);
void add_equation(Span<uint32> symbol_ids, Slice data);
bool is_ready() const;
Span<uint32> ready_symbols() const;
Slice get_symbol(uint32 symbol_id) const;
private:
struct SymbolInfo {
bool is_ready{false};
uint32 head_ = 0;
};
struct EquationInfo {
uint32 symbols_xor{0};
uint32 symbols_count{0};
};
size_t max_equation_count_;
MatrixGF256 C_;
MatrixGF256 D_;
std::vector<SymbolInfo> symbols_;
std::vector<EquationInfo> equations_;
std::vector<uint32> ready_equations_;
std::vector<uint32> ready_symbols_;
struct Edge {
uint32 value;
uint32 next;
};
std::vector<Edge> edges_;
void loop();
};
} // namespace td

59
tdfec/td/fec/algebra/GaussianElimination.cpp Normal file
View file

@ -0,0 +1,59 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/fec/algebra/GaussianElimination.h"
namespace td {
Result<MatrixGF256> GaussianElimination::run(MatrixGF256 A, MatrixGF256 D) {
const size_t cols = A.cols();
const size_t rows = A.rows();
CHECK(cols <= rows);
std::vector<uint32> row_perm(rows);
for (uint32 i = 0; i < rows; i++) {
row_perm[i] = i;
}
for (size_t row = 0; row < cols; row++) {
size_t non_zero_row = row;
for (; non_zero_row < rows && A.get(row_perm[non_zero_row], row).is_zero(); non_zero_row++) {
}
if (non_zero_row == rows) {
return Status::Error("Non solvable");
}
if (non_zero_row != row) {
std::swap(row_perm[non_zero_row], row_perm[row]);
}
auto mul = A.get(row_perm[row], row).inverse();
A.row_multiply(row_perm[row], mul);
D.row_multiply(row_perm[row], mul);
CHECK(A.get(row_perm[row], row).value() == 1);
for (size_t zero_row = 0; zero_row < rows; zero_row++) {
if (zero_row == row) {
continue;
}
auto x = A.get(row_perm[zero_row], row);
if (!x.is_zero()) {
A.row_add_mul(row_perm[zero_row], row_perm[row], x);
D.row_add_mul(row_perm[zero_row], row_perm[row], x);
}
}
}
return D.apply_row_permutation(row_perm);
}
} // namespace td

28
tdfec/td/fec/algebra/GaussianElimination.h Normal file
View file

@ -0,0 +1,28 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/fec/algebra/MatrixGF256.h"
namespace td {
class GaussianElimination {
public:
static Result<MatrixGF256> run(MatrixGF256 A, MatrixGF256 D);
};
} // namespace td

186
tdfec/td/fec/algebra/InactivationDecoding.cpp Normal file
View file

@ -0,0 +1,186 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/fec/algebra/InactivationDecoding.h"
#include "td/utils/logging.h"
#include <algorithm>
namespace td {
InactivationDecodingResult InactivationDecoding::run() {
init();
loop();
LOG(DEBUG) << tag("A_small.cols", L_.cols() - p_rows_.size()) << tag("Total columns", L_.cols()) << tag("PI", PI_)
<< tag("A_small.cols - PI", L_.cols() - PI_ - p_rows_.size());
for (uint32 row = 0; row < rows_; row++) {
if (!was_row_[row]) {
p_rows_.push_back(row);
}
}
uint32 side = narrow_cast<uint32>(p_cols_.size());
std::reverse(inactive_cols_.begin(), inactive_cols_.end());
for (auto col : inactive_cols_) {
p_cols_.push_back(col);
}
for (uint32 i = 0; i < PI_; i++) {
p_cols_.push_back(cols_ + i);
}
check(side);
return {side, std::move(p_rows_), std::move(p_cols_)};
}
void InactivationDecoding::init() {
was_row_ = vector<bool>(rows_, false);
was_col_ = vector<bool>(cols_, false);
col_cnt_ = vector<uint32>(cols_, 0);
row_cnt_ = vector<uint32>(rows_, 0);
row_xor_ = vector<uint32>(rows_, 0);
L_.generate([&](auto row, auto col) {
if (col >= cols_) {
return;
}
col_cnt_[col]++;
row_cnt_[row]++;
row_xor_[row] ^= col;
});
sort_rows();
}
void InactivationDecoding::loop() {
while (row_cnt_offset_[1] != rows_) {
auto row = sorted_rows_[row_cnt_offset_[1]];
uint32 col = choose_col(row);
LOG_CHECK(col_cnt_[col] >= 1) << col;
auto cnt = row_cnt_[row];
CHECK(row_cnt_offset_[cnt] == row_cnt_offset_[1]);
CHECK(row_pos_[row] == row_cnt_offset_[1]);
p_cols_.push_back(col);
p_rows_.push_back(row);
if (cnt == 1) {
inactivate_col(col);
} else {
for (auto x : L_rows_.col(row)) {
if (x >= cols_ || was_col_[x]) {
continue;
}
if (x != col) {
inactive_cols_.push_back(x);
}
inactivate_col(x);
}
}
was_row_[row] = true;
}
}
void InactivationDecoding::check_sorted() {
for (size_t i = 0; i < rows_; i++) {
CHECK(sorted_rows_[row_pos_[i]] == i);
}
for (size_t i = 1; i < rows_; i++) {
CHECK(row_cnt_[sorted_rows_[i - 1]] <= row_cnt_[sorted_rows_[i]]);
}
for (size_t i = 1; i <= cols_ + 1; i++) {
CHECK(row_cnt_offset_[i - 1] <= row_cnt_offset_[i]);
}
for (size_t i = 0; i < rows_; i++) {
auto pos = row_pos_[i];
auto cnt = row_cnt_[i];
CHECK(pos >= row_cnt_offset_[cnt]);
CHECK(pos < row_cnt_offset_[cnt + 1]);
}
}
uint32 InactivationDecoding::choose_col(uint32 row) {
auto cnt = row_cnt_[row];
if (cnt == 1) {
return row_xor_[row];
}
uint32 best_col = uint32(-1);
for (auto col : L_rows_.col(row)) {
if (col >= cols_ || was_col_[col]) {
continue;
}
DCHECK(col_cnt_[col] >= 1);
if (best_col == uint32(-1) || col_cnt_[col] < col_cnt_[best_col]) {
best_col = col;
}
}
DCHECK(best_col != uint32(-1));
return best_col;
}
void InactivationDecoding::inactivate_col(uint32 col) {
was_col_[col] = true;
for (auto row : L_.col(col)) {
if (was_row_[row]) {
continue;
}
auto pos = row_pos_[row];
DCHECK(sorted_rows_[pos] == row);
auto cnt = row_cnt_[row];
LOG_DCHECK(cnt >= 1) << row << " " << col;
auto offset = row_cnt_offset_[cnt];
std::swap(sorted_rows_[pos], sorted_rows_[offset]);
row_pos_[sorted_rows_[pos]] = pos;
row_pos_[sorted_rows_[offset]] = offset;
row_cnt_offset_[cnt]++;
row_cnt_[row]--;
row_xor_[row] ^= col;
}
}
void InactivationDecoding::sort_rows() {
vector<uint32> offset(cols_ + 2, 0);
for (size_t i = 0; i < rows_; i++) {
offset[row_cnt_[i] + 1]++;
}
for (size_t i = 1; i <= cols_ + 1; i++) {
offset[i] += offset[i - 1];
}
row_cnt_offset_ = offset;
sorted_rows_.resize(rows_);
row_pos_.resize(rows_);
for (uint32 i = 0; i < rows_; i++) {
auto pos = offset[row_cnt_[i]]++;
sorted_rows_[pos] = i;
row_pos_[i] = pos;
}
}
void InactivationDecoding::check(uint32 side) {
auto inv_p_cols = inverse_permutation(p_cols_);
auto inv_p_rows = inverse_permutation(p_rows_);
for (uint32 i = 0; i < side; i++) {
CHECK(inv_p_cols[p_cols_[i]] == i);
auto col = L_.col(p_cols_[i]);
CHECK(col.size() >= 1);
for (auto x : col) {
CHECK(inv_p_rows[x] >= i);
}
}
}
} // namespace td

73
tdfec/td/fec/algebra/InactivationDecoding.h Normal file
View file

@ -0,0 +1,73 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#include "td/fec/algebra/SparseMatrixGF2.h"
namespace td {
struct InactivationDecodingResult {
uint32 size;
vector<uint32> p_rows;
vector<uint32> p_cols;
};
class InactivationDecoding {
public:
InactivationDecoding(const SparseMatrixGF2 &L, uint32 PI) : L_(L), PI_(PI) {
}
InactivationDecodingResult run();
private:
const SparseMatrixGF2 &L_;
uint32 PI_;
const SparseMatrixGF2 L_rows_{L_.transpose()};
const uint32 cols_ = L_.cols() - PI_;
const uint32 rows_ = L_.rows();
vector<bool> was_row_;
vector<bool> was_col_;
vector<uint32> col_cnt_;
vector<uint32> row_cnt_;
vector<uint32> row_xor_;
vector<uint32> sorted_rows_;
vector<uint32> row_cnt_offset_;
vector<uint32> row_pos_;
vector<uint32> p_rows_;
vector<uint32> p_cols_;
vector<uint32> inactive_cols_;
void init();
void loop();
void check_sorted();
uint32 choose_col(uint32 row);
void inactivate_col(uint32 col);
void sort_rows();
void check(uint32 side);
};
} // namespace td

20
tdfec/td/fec/algebra/MatrixGF2.cpp Normal file
View file

@ -0,0 +1,20 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/fec/algebra/MatrixGF2.h"
char disable_linker_warning_about_empty_file_matrixgf2_cpp TD_UNUSED;

103
tdfec/td/fec/algebra/MatrixGF2.h Normal file
View file

@ -0,0 +1,103 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/fec/algebra/MatrixGF256.h"
namespace td {
class MatrixGF2 {
public:
MatrixGF2(size_t rows, size_t cols) : rows_(rows), cols_(cols) {
CHECK(Simd::alignment() % 8 == 0);
stride_ = ((cols_ + 7) / 8 + Simd::alignment() - 1) / Simd::alignment() * Simd::alignment();
CHECK(stride_ * 8 >= cols_);
storage_ = std::make_unique<uint8[]>(stride_ * rows + Simd::alignment() - 1);
matrix_ = storage_.get();
while (!Simd::is_aligned_pointer(matrix_)) {
matrix_++;
}
CHECK(Simd::is_aligned_pointer(matrix_));
CHECK(Simd::is_aligned_pointer(matrix_ + stride_));
CHECK(static_cast<size_t>(matrix_ - storage_.get()) < Simd::alignment());
}
void set_zero() {
std::fill(matrix_, matrix_ + stride_ * rows_, 0);
}
size_t rows() const {
return rows_;
}
size_t cols() const {
return cols_;
}
void set_one(size_t row, size_t col) {
DCHECK(row < rows_ && col < cols_);
matrix_[row * stride_ + col / 8] |= uint8(1 << (col % 8));
}
bool get(size_t row, size_t col) const {
DCHECK(row < rows_ && col < cols_);
return (matrix_[row * stride_ + col / 8] & (uint8(1) << (col % 8))) != 0;
}
// row(a) += row(b)
void row_add(size_t a, size_t b) {
row_add(row_ptr(a), row_ptr(b));
}
void row_add(size_t a, Slice b) {
DCHECK(b.size() == stride_);
row_add(row_ptr(a), b.ubegin());
}
Slice row(size_t a) const {
return Slice(row_ptr(a), stride_);
}
MutableSlice row(size_t a) {
return MutableSlice(row_ptr(a), stride_);
}
void row_set(size_t a, Slice b) {
row(a).copy_from(b);
}
MatrixGF256 to_gf256() const {
MatrixGF256 res(rows(), cols());
for (size_t i = 0; i < rows(); i++) {
Simd::gf256_from_gf2(res.row(i).data(), row(i).data(), ((cols_ + 7) / 8 + 3) / 4 * 4);
}
return res;
}
private:
uint8* matrix_;
size_t rows_;
size_t cols_;
size_t stride_;
std::unique_ptr<uint8[]> storage_;
uint8* row_ptr(size_t row) {
return matrix_ + stride_ * row;
}
const uint8* row_ptr(size_t row) const {
return matrix_ + stride_ * row;
}
void row_add(uint8* pa, const uint8* pb) {
Simd::gf256_add(pa, pb, stride_);
}
};
} // namespace td

20
tdfec/td/fec/algebra/MatrixGF256.cpp Normal file
View file

@ -0,0 +1,20 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/fec/algebra/MatrixGF256.h"
char disable_linker_warning_about_empty_file_matrixgf256_cpp TD_UNUSED;

199
tdfec/td/fec/algebra/MatrixGF256.h Normal file
View file

@ -0,0 +1,199 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/fec/algebra/Octet.h"
#include "td/fec/algebra/Simd.h"
#include "td/utils/Span.h"
#include "td/utils/format.h"
namespace td {
class MatrixGF256 {
public:
MatrixGF256(size_t rows, size_t cols) : rows_(rows), cols_(cols) {
stride_ = (cols_ + Simd::alignment() - 1) / Simd::alignment() * Simd::alignment();
storage_ = std::make_unique<uint8[]>(stride_ * rows + Simd::alignment() - 1);
matrix_ = storage_.get();
while (!Simd::is_aligned_pointer(matrix_)) {
matrix_++;
}
CHECK(Simd::is_aligned_pointer(matrix_));
CHECK(Simd::is_aligned_pointer(matrix_ + stride_));
CHECK(static_cast<size_t>(matrix_ - storage_.get()) < Simd::alignment());
}
void set_zero() {
std::fill(matrix_, matrix_ + stride_ * rows_, 0);
}
size_t rows() const {
return rows_;
}
size_t cols() const {
return cols_;
}
MatrixGF256 apply_row_permutation(Span<uint32> permutation) {
MatrixGF256 res(rows_, cols_);
for (size_t row = 0; row < rows_; row++) {
res.row(row).copy_from(this->row(permutation[row]));
}
return res;
}
Octet get(size_t row, size_t col) const {
DCHECK(row < rows_ && col < cols_);
return Octet(matrix_[row * stride_ + col]);
}
void set(size_t row, size_t col, Octet o) {
DCHECK(row < rows_ && col < cols_);
matrix_[row * stride_ + col] = o.value();
}
void row_multiply(size_t row, Octet o) {
uint8* p = row_ptr(row);
Simd::gf256_mul(p, o.value(), stride_);
}
Slice row(size_t row) const {
return Slice(row_ptr(row), cols());
}
MutableSlice row(size_t row) {
return MutableSlice(row_ptr(row), cols());
}
template <class M>
void set_from(const M& m, size_t row_offset, size_t col_offset) {
auto to = block_view(row_offset, col_offset, rows() - row_offset, cols() - col_offset);
for (size_t i = 0; i < m.rows(); i++) {
to.row(i).copy_from(m.row(i));
}
}
MatrixGF256 copy() {
MatrixGF256 res(rows(), cols());
res.set_from(*this, 0, 0);
return res;
}
void add(const MatrixGF256& m) {
CHECK(m.rows() == rows());
CHECK(m.cols() == cols());
for (size_t i = 0; i < m.rows(); i++) {
auto* to = row_ptr(i);
auto* from = m.row_ptr(i);
row_add(to, from);
}
}
// row(a) += row(b) * m
void row_add_mul(size_t a, size_t b, Octet m) {
row_add_mul(row_ptr(a), row_ptr(b), m);
}
void row_add_mul(size_t a, Slice b, Octet m) {
row_add_mul(row_ptr(a), b.ubegin(), m);
}
// row(a) += row(b)
void row_add(size_t a, size_t b) {
row_add(row_ptr(a), row_ptr(b));
}
void row_add(size_t a, Slice b) {
row_add(row_ptr(a), b.ubegin());
}
void row_set(size_t a, Slice b) {
row(a).copy_from(b);
}
class BlockView {
public:
BlockView(size_t row_offset, size_t col_offset, size_t row_size, size_t col_size, MatrixGF256& m)
: row_offset_(row_offset), col_offset_(col_offset), row_size_(row_size), col_size_(col_size), m_(m) {
}
size_t cols() const {
return col_size_;
}
size_t rows() const {
return row_size_;
}
Slice row(size_t row) const {
return m_.row(row_offset_ + row).remove_prefix(col_offset_);
}
MutableSlice row(size_t row) {
return m_.row(row_offset_ + row).remove_prefix(col_offset_);
}
private:
size_t row_offset_;
size_t col_offset_;
size_t row_size_;
size_t col_size_;
MatrixGF256& m_;
};
BlockView block_view(size_t row_offset, size_t col_offset, size_t row_size, size_t col_size) {
return BlockView(row_offset, col_offset, row_size, col_size, *this);
}
private:
uint8* matrix_;
size_t rows_;
size_t cols_;
size_t stride_;
std::unique_ptr<uint8[]> storage_;
uint8* row_ptr(size_t row) {
return matrix_ + stride_ * row;
}
const uint8* row_ptr(size_t row) const {
return matrix_ + stride_ * row;
}
void row_add_mul(uint8* ap, const uint8* bp, Octet m) {
uint8 u = m.value();
if (u == 0) {
return;
}
if (u == 1) {
return row_add(ap, bp);
}
Simd::gf256_add_mul(ap, bp, u, stride_);
}
void row_add(uint8* ap, const uint8* bp) {
Simd::gf256_add(ap, bp, stride_);
}
};
inline StringBuilder& operator<<(StringBuilder& sb, const MatrixGF256& m) {
sb << "\n";
for (uint32 i = 0; i < m.rows(); i++) {
auto row = m.row(i);
for (uint32 j = 0; j < m.cols(); j++) {
uint8 x = row[j];
sb << " " << format::hex_digit(x / 16) << format::hex_digit(x % 16);
}
sb << "\n";
}
return sb;
}
} // namespace td

9242
tdfec/td/fec/algebra/Octet.cpp Normal file
View file

File diff suppressed because it is too large Load diff

152
tdfec/td/fec/algebra/Octet.h Normal file
View file

@ -0,0 +1,152 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/logging.h"
#include <array>
#include <iomanip>
namespace td {
class Octet {
public:
Octet() = default;
explicit Octet(const uint8 val) : data_(val) {
}
uint8 value() const {
return data_;
}
Octet& operator+=(const Octet a) {
data_ ^= a.data_;
return *this;
}
friend Octet operator+(Octet lhs, const Octet rhs) {
return lhs += rhs;
}
Octet& operator-=(const Octet a) {
return *this += a;
}
friend Octet operator-(Octet lhs, const Octet rhs) {
return lhs + rhs;
}
Octet& operator*=(const Octet a) {
if (data_ != 0 && a.data_ != 0) {
data_ = oct_exp(oct_log(uint8(data_ - 1)) + oct_log(uint8(a.data_ - 1)));
} else {
data_ = 0;
}
return *this;
}
friend Octet operator*(Octet lhs, const Octet rhs) {
return lhs *= rhs;
}
Octet& operator/=(const Octet a) {
if (a.data_ != 0 && data_ != 0) {
data_ = oct_exp(oct_log(uint8(data_ - 1)) - oct_log(uint8(a.data_ - 1)) + 255);
}
return *this;
}
friend Octet operator/(Octet lhs, const Octet rhs) {
return lhs /= rhs;
}
Octet inverse() const {
return Octet(oct_exp(255 - oct_log(uint8(data_ - 1))));
}
bool operator==(const Octet a) const {
return data_ == a.data_;
}
bool operator!=(const Octet a) const {
return data_ != a.data_;
}
bool operator<(const Octet a) const {
return data_ < a.data_;
}
bool operator>(const Octet a) const {
return data_ > a.data_;
}
bool is_zero() const {
return data_ == 0;
}
friend std::ostream& operator<<(std::ostream& os, const Octet m) {
os << std::hex << static_cast<uint32>(m.data_);
return os;
}
static constexpr uint8 oct_exp(uint32 data) {
constexpr std::array<uint8, 510> precalc = {
{1, 2, 4, 8, 16, 32, 64, 128, 29, 58, 116, 232, 205, 135, 19, 38, 76, 152, 45, 90, 180, 117,
234, 201, 143, 3, 6, 12, 24, 48, 96, 192, 157, 39, 78, 156, 37, 74, 148, 53, 106, 212, 181, 119,
238, 193, 159, 35, 70, 140, 5, 10, 20, 40, 80, 160, 93, 186, 105, 210, 185, 111, 222, 161, 95, 190,
97, 194, 153, 47, 94, 188, 101, 202, 137, 15, 30, 60, 120, 240, 253, 231, 211, 187, 107, 214, 177, 127,
254, 225, 223, 163, 91, 182, 113, 226, 217, 175, 67, 134, 17, 34, 68, 136, 13, 26, 52, 104, 208, 189,
103, 206, 129, 31, 62, 124, 248, 237, 199, 147, 59, 118, 236, 197, 151, 51, 102, 204, 133, 23, 46, 92,
184, 109, 218, 169, 79, 158, 33, 66, 132, 21, 42, 84, 168, 77, 154, 41, 82, 164, 85, 170, 73, 146,
57, 114, 228, 213, 183, 115, 230, 209, 191, 99, 198, 145, 63, 126, 252, 229, 215, 179, 123, 246, 241, 255,
227, 219, 171, 75, 150, 49, 98, 196, 149, 55, 110, 220, 165, 87, 174, 65, 130, 25, 50, 100, 200, 141,
7, 14, 28, 56, 112, 224, 221, 167, 83, 166, 81, 162, 89, 178, 121, 242, 249, 239, 195, 155, 43, 86,
172, 69, 138, 9, 18, 36, 72, 144, 61, 122, 244, 245, 247, 243, 251, 235, 203, 139, 11, 22, 44, 88,
176, 125, 250, 233, 207, 131, 27, 54, 108, 216, 173, 71, 142, 1, 2, 4, 8, 16, 32, 64, 128, 29,
58, 116, 232, 205, 135, 19, 38, 76, 152, 45, 90, 180, 117, 234, 201, 143, 3, 6, 12, 24, 48, 96,
192, 157, 39, 78, 156, 37, 74, 148, 53, 106, 212, 181, 119, 238, 193, 159, 35, 70, 140, 5, 10, 20,
40, 80, 160, 93, 186, 105, 210, 185, 111, 222, 161, 95, 190, 97, 194, 153, 47, 94, 188, 101, 202, 137,
15, 30, 60, 120, 240, 253, 231, 211, 187, 107, 214, 177, 127, 254, 225, 223, 163, 91, 182, 113, 226, 217,
175, 67, 134, 17, 34, 68, 136, 13, 26, 52, 104, 208, 189, 103, 206, 129, 31, 62, 124, 248, 237, 199,
147, 59, 118, 236, 197, 151, 51, 102, 204, 133, 23, 46, 92, 184, 109, 218, 169, 79, 158, 33, 66, 132,
21, 42, 84, 168, 77, 154, 41, 82, 164, 85, 170, 73, 146, 57, 114, 228, 213, 183, 115, 230, 209, 191,
99, 198, 145, 63, 126, 252, 229, 215, 179, 123, 246, 241, 255, 227, 219, 171, 75, 150, 49, 98, 196, 149,
55, 110, 220, 165, 87, 174, 65, 130, 25, 50, 100, 200, 141, 7, 14, 28, 56, 112, 224, 221, 167, 83,
166, 81, 162, 89, 178, 121, 242, 249, 239, 195, 155, 43, 86, 172, 69, 138, 9, 18, 36, 72, 144, 61,
122, 244, 245, 247, 243, 251, 235, 203, 139, 11, 22, 44, 88, 176, 125, 250, 233, 207, 131, 27, 54, 108,
216, 173, 71, 142}};
return precalc[data];
}
static constexpr uint8 oct_log(uint8 data) {
constexpr std::array<uint8, 255> precalc = {
{0, 1, 25, 2, 50, 26, 198, 3, 223, 51, 238, 27, 104, 199, 75, 4, 100, 224, 14, 52, 141, 239,
129, 28, 193, 105, 248, 200, 8, 76, 113, 5, 138, 101, 47, 225, 36, 15, 33, 53, 147, 142, 218, 240,
18, 130, 69, 29, 181, 194, 125, 106, 39, 249, 185, 201, 154, 9, 120, 77, 228, 114, 166, 6, 191, 139,
98, 102, 221, 48, 253, 226, 152, 37, 179, 16, 145, 34, 136, 54, 208, 148, 206, 143, 150, 219, 189, 241,
210, 19, 92, 131, 56, 70, 64, 30, 66, 182, 163, 195, 72, 126, 110, 107, 58, 40, 84, 250, 133, 186,
61, 202, 94, 155, 159, 10, 21, 121, 43, 78, 212, 229, 172, 115, 243, 167, 87, 7, 112, 192, 247, 140,
128, 99, 13, 103, 74, 222, 237, 49, 197, 254, 24, 227, 165, 153, 119, 38, 184, 180, 124, 17, 68, 146,
217, 35, 32, 137, 46, 55, 63, 209, 91, 149, 188, 207, 205, 144, 135, 151, 178, 220, 252, 190, 97, 242,
86, 211, 171, 20, 42, 93, 158, 132, 60, 57, 83, 71, 109, 65, 162, 31, 45, 67, 216, 183, 123, 164,
118, 196, 23, 73, 236, 127, 12, 111, 246, 108, 161, 59, 82, 41, 157, 85, 170, 251, 96, 134, 177, 187,
204, 62, 90, 203, 89, 95, 176, 156, 169, 160, 81, 11, 245, 22, 235, 122, 117, 44, 215, 79, 174, 213,
233, 230, 231, 173, 232, 116, 214, 244, 234, 168, 80, 88, 175}};
return precalc[data];
}
static const uint8 OctMulLo[256][16];
static const uint8 OctMulHi[256][16];
private:
uint8 data_;
};
} // namespace td

266
tdfec/td/fec/algebra/Simd.h Normal file
View file

@ -0,0 +1,266 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/misc.h"
#include "td/fec/algebra/Octet.h"
#if __SSSE3__
#define TD_SSE3 1
#endif
#if __AVX2__
#define TD_AVX2 1
#define TD_SSE3 1
#endif
#if TD_AVX2
#include <immintrin.h> /* avx2 */
#elif TD_SSE3
#include <tmmintrin.h> /* ssse3 */
#endif
namespace td {
class Simd_null {
public:
static constexpr size_t alignment() {
return 32; // gf256_from_gf2 relies on 32 alignment
}
static std::string get_name() {
return "Without simd";
}
static bool is_aligned_pointer(const void *ptr) {
return ::td::is_aligned_pointer<alignment()>(ptr);
}
static void gf256_add(void *a, const void *b, size_t size) {
DCHECK(is_aligned_pointer(a));
DCHECK(is_aligned_pointer(b));
uint8 *ap = reinterpret_cast<uint8 *>(a);
const uint8 *bp = reinterpret_cast<const uint8 *>(b);
for (size_t i = 0; i < size; i++) {
ap[i] ^= bp[i];
}
}
static void gf256_mul(void *a, uint8 u, size_t size) {
DCHECK(is_aligned_pointer(a));
uint8 *ap = reinterpret_cast<uint8 *>(a);
for (size_t i = 0; i < size; i++) {
ap[i] = (Octet(ap[i]) * Octet(u)).value();
}
}
static void gf256_add_mul(void *a, const void *b, uint8 u, size_t size) {
DCHECK(is_aligned_pointer(a));
DCHECK(is_aligned_pointer(b));
uint8 *ap = reinterpret_cast<uint8 *>(a);
const uint8 *bp = reinterpret_cast<const uint8 *>(b);
for (size_t i = 0; i < size; i++) {
ap[i] = (Octet(ap[i]) + Octet(bp[i]) * Octet(u)).value();
}
}
static void gf256_from_gf2(void *a, const void *b, size_t size) {
uint8 *ap = reinterpret_cast<uint8 *>(a);
const uint8 *bp = reinterpret_cast<const uint8 *>(b);
size *= 8;
for (size_t i = 0; i < size; i++, ap++) {
*ap = (bp[i / 8] >> (i % 8)) & 1;
}
}
};
#if TD_SSE3
class Simd_sse : public Simd_null {
public:
static constexpr size_t alignment() {
return 32;
}
static std::string get_name() {
return "With SSE";
}
static bool is_aligned_pointer(const void *ptr) {
return ::td::is_aligned_pointer<alignment()>(ptr);
}
static void gf256_add(void *a, const void *b, size_t size) {
DCHECK(is_aligned_pointer(a));
DCHECK(is_aligned_pointer(b));
uint8 *ap = reinterpret_cast<uint8 *>(a);
const uint8 *bp = reinterpret_cast<const uint8 *>(b);
__m128i *ap128 = reinterpret_cast<__m128i *>(ap);
const __m128i *bp128 = reinterpret_cast<const __m128i *>(bp);
for (size_t idx = 0; idx < size; idx += 16) {
_mm_storeu_si128(ap128, _mm_xor_si128(_mm_loadu_si128(ap128), _mm_loadu_si128(bp128)));
ap128++;
bp128++;
}
}
static void gf256_mul(void *a, uint8 u, size_t size) {
DCHECK(is_aligned_pointer(a));
uint8 *ap = reinterpret_cast<uint8 *>(a);
const __m128i mask = _mm_set1_epi8(0x0f);
const __m128i urow_hi = _mm_loadu_si128(reinterpret_cast<const __m128i *>(Octet::OctMulHi[u]));
const __m128i urow_lo = _mm_loadu_si128(reinterpret_cast<const __m128i *>(Octet::OctMulLo[u]));
__m128i *ap128 = reinterpret_cast<__m128i *>(ap);
for (size_t idx = 0; idx < size; idx += 16) {
__m128i ax = _mm_loadu_si128(ap128);
__m128i lo = _mm_and_si128(ax, mask);
ax = _mm_srli_epi64(ax, 4);
__m128i hi = _mm_and_si128(ax, mask);
lo = _mm_shuffle_epi8(urow_lo, lo);
hi = _mm_shuffle_epi8(urow_hi, hi);
_mm_storeu_si128(ap128, _mm_xor_si128(lo, hi));
ap128++;
}
}
static void gf256_add_mul(void *a, const void *b, uint8 u, size_t size) {
DCHECK(is_aligned_pointer(a));
DCHECK(is_aligned_pointer(b));
uint8 *ap = reinterpret_cast<uint8 *>(a);
const uint8 *bp = reinterpret_cast<const uint8 *>(b);
const __m128i mask = _mm_set1_epi8(0x0f);
const __m128i urow_hi = _mm_loadu_si128(reinterpret_cast<const __m128i *>(Octet::OctMulHi[u]));
const __m128i urow_lo = _mm_loadu_si128(reinterpret_cast<const __m128i *>(Octet::OctMulLo[u]));
__m128i *ap128 = reinterpret_cast<__m128i *>(ap);
const __m128i *bp128 = reinterpret_cast<const __m128i *>(bp);
for (size_t idx = 0; idx < size; idx += 16) {
__m128i bx = _mm_loadu_si128(bp128++);
__m128i lo = _mm_and_si128(bx, mask);
bx = _mm_srli_epi64(bx, 4);
__m128i hi = _mm_and_si128(bx, mask);
lo = _mm_shuffle_epi8(urow_lo, lo);
hi = _mm_shuffle_epi8(urow_hi, hi);
_mm_storeu_si128(ap128, _mm_xor_si128(_mm_loadu_si128(ap128), _mm_xor_si128(lo, hi)));
ap128++;
}
}
};
#endif // SSSE3
#ifdef TD_AVX2
class Simd_avx : public Simd_sse {
public:
static std::string get_name() {
return "With AVX";
}
static void gf256_add(void *a, const void *b, size_t size) {
DCHECK(is_aligned_pointer(a));
DCHECK(is_aligned_pointer(b));
uint8 *ap = reinterpret_cast<uint8 *>(a);
const uint8 *bp = reinterpret_cast<const uint8 *>(b);
__m256i *ap256 = reinterpret_cast<__m256i *>(ap);
const __m256i *bp256 = reinterpret_cast<const __m256i *>(bp);
for (size_t idx = 0; idx < size; idx += 32) {
_mm256_storeu_si256(ap256, _mm256_xor_si256(_mm256_loadu_si256(ap256), _mm256_loadu_si256(bp256)));
ap256++;
bp256++;
}
}
static __m256i get_mask(const uint32 mask) {
// abcd -> abcd * 8
__m256i vmask(_mm256_set1_epi32(mask));
// abcd * 8 -> aaaaaaaabbbbbbbbccccccccdddddddd
const __m256i shuffle(
_mm256_setr_epi64x(0x0000000000000000, 0x0101010101010101, 0x0202020202020202, 0x0303030303030303));
vmask = _mm256_shuffle_epi8(vmask, shuffle);
const __m256i bit_mask(_mm256_set1_epi64x(0x7fbfdfeff7fbfdfe));
vmask = _mm256_or_si256(vmask, bit_mask);
return _mm256_and_si256(_mm256_cmpeq_epi8(vmask, _mm256_set1_epi64x(-1)), _mm256_set1_epi8(1));
}
static void gf256_from_gf2(void *a, const void *b, size_t size) {
DCHECK(is_aligned_pointer(a));
DCHECK(size % 4 == 0);
__m256i *ap256 = reinterpret_cast<__m256i *>(a);
const uint32 *bp = reinterpret_cast<const uint32 *>(b);
size /= 4;
for (size_t i = 0; i < size; i++, bp++, ap256++) {
*ap256 = get_mask(*bp);
}
}
static __attribute__((noinline)) void gf256_mul(void *a, uint8 u, size_t size) {
const __m128i urow_hi_small = _mm_load_si128(reinterpret_cast<const __m128i *>(Octet::OctMulHi[u]));
const __m256i urow_hi = _mm256_broadcastsi128_si256(urow_hi_small);
const __m128i urow_lo_small = _mm_load_si128(reinterpret_cast<const __m128i *>(Octet::OctMulLo[u]));
const __m256i urow_lo = _mm256_broadcastsi128_si256(urow_lo_small);
const __m256i mask = _mm256_set1_epi8(0x0f);
__m256i *ap256 = (__m256i *)a;
for (size_t idx = 0; idx < size; idx += 32) {
__m256i ax = _mm256_load_si256(ap256);
__m256i lo = _mm256_and_si256(ax, mask);
ax = _mm256_srli_epi64(ax, 4);
__m256i hi = _mm256_and_si256(ax, mask);
lo = _mm256_shuffle_epi8(urow_lo, lo);
hi = _mm256_shuffle_epi8(urow_hi, hi);
_mm256_store_si256(ap256, _mm256_xor_si256(lo, hi));
ap256++;
}
}
static __attribute__((noinline)) void gf256_add_mul(void *a, const void *b, uint8 u, size_t size) {
const __m128i urow_hi_small = _mm_load_si128(reinterpret_cast<const __m128i *>(Octet::OctMulHi[u]));
const __m256i urow_hi = _mm256_broadcastsi128_si256(urow_hi_small);
const __m128i urow_lo_small = _mm_load_si128(reinterpret_cast<const __m128i *>(Octet::OctMulLo[u]));
const __m256i urow_lo = _mm256_broadcastsi128_si256(urow_lo_small);
const __m256i mask = _mm256_set1_epi8(0x0f);
__m256i *ap256 = (__m256i *)a;
const __m256i *bp256 = (const __m256i *)b;
for (size_t idx = 0; idx < size; idx += 32) {
__m256i bx = _mm256_load_si256(bp256++);
__m256i lo = _mm256_and_si256(bx, mask);
bx = _mm256_srli_epi64(bx, 4);
__m256i hi = _mm256_and_si256(bx, mask);
lo = _mm256_shuffle_epi8(urow_lo, lo);
hi = _mm256_shuffle_epi8(urow_hi, hi);
_mm256_store_si256(ap256, _mm256_xor_si256(_mm256_load_si256(ap256), _mm256_xor_si256(lo, hi)));
ap256++;
}
}
};
#endif // AVX2
#if TD_AVX2
using Simd = Simd_avx;
#elif TD_SSE3
using Simd = Simd_sse;
#else
using Simd = Simd_null;
#endif
} // namespace td

20
tdfec/td/fec/algebra/SparseMatrixGF2.cpp Normal file
View file

@ -0,0 +1,20 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/fec/algebra/SparseMatrixGF2.h"
char disable_linker_warning_about_empty_file_sparsematrixgf2_cpp TD_UNUSED;

277
tdfec/td/fec/algebra/SparseMatrixGF2.h Normal file
View file

@ -0,0 +1,277 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/fec/algebra/MatrixGF2.h"
namespace td {
inline std::vector<uint32> inverse_permutation(Span<uint32> p) {
std::vector<uint32> res(p.size());
for (size_t i = 0; i < p.size(); i++) {
res[p[i]] = narrow_cast<uint32>(i);
}
return res;
}
class IdentityGenerator {
public:
IdentityGenerator(uint32 N) : N(N) {
}
template <class F>
void generate(F&& f) const {
for (uint32 col = 0; col < N; col++) {
f(col, col);
}
}
uint32 non_zeroes() const {
return N;
}
uint32 cols() const {
return N;
}
uint32 rows() const {
return N;
}
private:
uint32 N;
};
template <class GeneratorT>
class PermutationGenerator {
public:
PermutationGenerator(const GeneratorT& m, Span<uint32> p) : m_(m), p_(inverse_permutation(p)) {
}
uint32 non_zeroes() const {
return m_.non_zeroes();
}
uint32 rows() const {
return m_.rows();
}
uint32 cols() const {
return m_.cols();
}
template <class F>
void generate(F&& f) const {
m_.generate([&f, this](auto row, auto col) { f(row, p_[col]); });
}
private:
const GeneratorT& m_;
std::vector<uint32> p_;
};
template <class GeneratorT>
class TransposeGenerator {
public:
explicit TransposeGenerator(const GeneratorT& m) : m_(m) {
}
uint32 non_zeroes() const {
return m_.non_zeroes();
}
uint32 rows() const {
return m_.cols();
}
uint32 cols() const {
return m_.rows();
}
template <class F>
void generate(F&& f) const {
m_.generate([&f](auto row, auto col) { f(col, row); });
}
private:
const GeneratorT& m_;
std::vector<uint32> p_;
};
class SparseMatrixGF2 {
public:
uint32 non_zeroes() const {
return narrow_cast<uint32>(data_.size());
}
Span<uint32> col(uint32 i) const {
return Span<uint32>(data_.data() + col_offset_[i], col_size(i));
}
uint32 col_size(uint32 i) const {
return col_offset_[i + 1] - col_offset_[i];
}
uint32 cols() const {
return cols_;
}
uint32 rows() const {
return rows_;
}
template <class F>
void generate(F&& f) const {
return block_for_each(0, 0, rows_, cols_, f);
}
template <class F>
void block_for_each(uint32 row_from, uint32 col_from, uint32 row_size, uint32 col_size, F&& f) const {
auto col_till = col_from + col_size;
auto row_till = row_from + row_size;
for (uint32 col_i = col_from; col_i < col_till; col_i++) {
auto col_span = col(col_i);
auto* it = row_from == 0 ? col_span.begin() : std::lower_bound(col_span.begin(), col_span.end(), row_from);
while (it != col_span.end() && *it < row_till) {
f(*it - row_from, col_i - col_from);
it++;
}
}
}
MatrixGF2 block_dense(uint32 row_from, uint32 col_from, uint32 row_size, uint32 col_size) const {
MatrixGF2 res(row_size, col_size);
res.set_zero();
block_for_each(row_from, col_from, row_size, col_size, [&](auto row, auto col) { res.set_one(row, col); });
return res;
}
template <class GeneratorT>
explicit SparseMatrixGF2(GeneratorT&& generator) : rows_(generator.rows()), cols_(generator.cols()) {
data_.resize(generator.non_zeroes());
col_offset_.resize(cols_ + 1, 0);
generator.generate([&](uint32 row, uint32 col) {
LOG_DCHECK(row < rows_ && col < cols_) << "(" << row << "," << col << ") (" << rows_ << "," << cols_ << ")";
col_offset_[col + 1]++;
});
for (uint32 i = 1; i < col_offset_.size(); i++) {
col_offset_[i] += col_offset_[i - 1];
}
auto col_pos = col_offset_;
generator.generate([&](uint32 row, uint32 col) { data_[col_pos[col]++] = row; });
for (uint32 col_i = 0; col_i < cols_; col_i++) {
auto c = col(col_i);
for (size_t j = 1; j < c.size(); j++) {
LOG_DCHECK(c[j] > c[j - 1]) << c[j] << " > " << c[j - 1] << tag("row", col_i);
}
}
}
class BlockView {
public:
BlockView(uint32 row_offset, uint32 col_offset, uint32 row_size, uint32 col_size, const SparseMatrixGF2& m)
: row_offset_(row_offset), col_offset_(col_offset), row_size_(row_size), col_size_(col_size), m_(m) {
}
uint32 cols() const {
return col_size_;
}
uint32 rows() const {
return row_size_;
}
uint32 non_zeroes() const {
uint32 res = 0;
m_.block_for_each(row_offset_, col_offset_, row_size_, col_size_, [&res](auto row, auto col) { res++; });
return res;
}
template <class F>
void generate(F&& f) const {
m_.block_for_each(row_offset_, col_offset_, row_size_, col_size_, [&f](auto row, auto col) { f(row, col); });
}
private:
uint32 row_offset_;
uint32 col_offset_;
uint32 row_size_;
uint32 col_size_;
const SparseMatrixGF2& m_;
};
SparseMatrixGF2 block_sparse(uint32 row_from, uint32 col_from, uint32 row_size, uint32 col_size) const {
return SparseMatrixGF2(BlockView(row_from, col_from, row_size, col_size, *this));
}
SparseMatrixGF2 transpose() const {
return SparseMatrixGF2(TransposeGenerator<SparseMatrixGF2>(*this));
}
SparseMatrixGF2 apply_col_permutation(Span<uint32> p) const {
return SparseMatrixGF2(PermutationGenerator<SparseMatrixGF2>(*this, p));
}
SparseMatrixGF2 apply_row_permutation(Span<uint32> p) const {
return transpose().apply_col_permutation(p).transpose();
}
private:
uint32 rows_{0};
uint32 cols_{0};
std::vector<uint32> data_;
std::vector<uint32> col_offset_;
};
template <class M>
M operator*(const SparseMatrixGF2& a, const M& b) {
M res(a.rows(), b.cols());
res.set_zero();
a.generate([&](auto row, auto col) { res.row_add(row, b.row(col)); });
return res;
}
template <class... ArgsT>
class BlockGenerator {
public:
BlockGenerator(uint32 rows, uint32 cols, ArgsT&&... args)
: rows_(rows), cols_(cols), tuple_(std::forward_as_tuple(std::forward<ArgsT>(args)...)) {
}
template <class F>
void generate(F&& f) const {
uint32 row_offset = 0;
uint32 next_row_offset = 0;
uint32 col_offset = 0;
tuple_for_each(tuple_, [&](auto& g) {
if (col_offset == 0) {
next_row_offset = row_offset + g.rows();
} else {
CHECK(next_row_offset == row_offset + g.rows());
}
g.generate([&](auto row, auto col) { f(row_offset + row, col_offset + col); });
col_offset += g.cols();
if (col_offset >= cols_) {
CHECK(col_offset == cols_);
col_offset = 0;
row_offset = next_row_offset;
}
});
}
uint32 non_zeroes() const {
uint32 res = 0;
tuple_for_each(tuple_, [&](auto& g) { res += g.non_zeroes(); });
return res;
}
uint32 cols() const {
return cols_;
}
uint32 rows() const {
return rows_;
}
private:
uint32 rows_;
uint32 cols_;
std::tuple<ArgsT...> tuple_;
};
template <class... ArgsT>
auto block_generator(uint32 rows, uint32 cols, ArgsT&&... args) {
return BlockGenerator<ArgsT...>(rows, cols, std::forward<ArgsT>(args)...);
}
} // namespace td

28
tdfec/td/fec/common/SymbolRef.h Normal file
View file

@ -0,0 +1,28 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/Slice.h"
namespace td {
namespace raptorq {
struct SymbolRef {
uint32 id;
Slice data;
};
} // namespace raptorq
} // namespace td

46
tdfec/td/fec/common/SymbolsView.cpp Normal file
View file

@ -0,0 +1,46 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/fec/common/SymbolsView.h"
namespace td {
namespace raptorq {
SymbolsView::SymbolsView(size_t symbols_count, size_t symbol_size, Slice data) {
symbols_.reserve(symbols_count);
zero_symbol_ = std::string(symbol_size, '\0');
last_symbol_ = std::string(symbol_size, '\0');
for (uint32 symbol_i = 0; symbol_i < symbols_count; symbol_i++) {
auto offset = symbol_i * symbol_size;
Slice symbol;
if (offset < data.size()) {
symbol = data.substr(offset).truncate(symbol_size);
}
Slice slice;
if (symbol.empty()) {
slice = zero_symbol_;
} else if (symbol.size() < symbol_size) {
MutableSlice(last_symbol_).copy_from(symbol);
slice = last_symbol_;
} else {
slice = symbol;
}
symbols_.push_back(SymbolRef{symbol_i, slice});
}
}
} // namespace raptorq
} // namespace td

41
tdfec/td/fec/common/SymbolsView.h Normal file
View file

@ -0,0 +1,41 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/fec/common/SymbolRef.h"
#include "td/utils/Span.h"
namespace td {
namespace raptorq {
class SymbolsView {
public:
SymbolsView(size_t symbols_count, size_t symbol_size, Slice data);
SymbolsView(const SymbolsView &) = delete;
SymbolsView(SymbolsView &&) = delete;
Span<SymbolRef> symbols() const {
return symbols_;
}
private:
std::vector<SymbolRef> symbols_;
std::string zero_symbol_;
std::string last_symbol_;
};
} // namespace raptorq
} // namespace td

212
tdfec/td/fec/fec.cpp Normal file
View file

@ -0,0 +1,212 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/fec/fec.h"
#include "td/fec/raptorq/Encoder.h"
#include "td/fec/raptorq/Decoder.h"
#include "td/fec/online/Encoder.h"
#include "td/fec/online/Decoder.h"
namespace td {
namespace fec {
std::unique_ptr<RoundRobinEncoder> RoundRobinEncoder::create(BufferSlice data, size_t max_symbol_size) {
CHECK(max_symbol_size > 0);
return std::make_unique<RoundRobinEncoder>(std::move(data), max_symbol_size);
}
Symbol RoundRobinEncoder::gen_symbol(uint32 id) {
id %= narrow_cast<uint32>(symbols_count_);
auto offset = symbol_size_ * id;
return Symbol{id, data_.from_slice(data_.as_slice().substr(offset).truncate(symbol_size_))};
};
RoundRobinEncoder::Parameters RoundRobinEncoder::get_parameters() const {
Parameters parameters;
parameters.symbol_size = symbol_size_;
parameters.data_size = data_.size();
parameters.symbols_count = symbols_count_;
return parameters;
}
RoundRobinEncoder::RoundRobinEncoder(BufferSlice data, size_t max_symbol_size)
: data_(std::move(data)), symbol_size_(max_symbol_size) {
symbols_count_ = (data_.size() + symbol_size_ - 1) / symbol_size_;
rounded_data_size_ = symbols_count_ * symbol_size_;
}
std::unique_ptr<RoundRobinDecoder> RoundRobinDecoder::create(RoundRobinEncoder::Parameters parameters) {
return std::make_unique<RoundRobinDecoder>(parameters);
}
bool RoundRobinDecoder::may_try_decode() const {
return left_symbols_ == 0;
}
Result<DataWithEncoder> RoundRobinDecoder::try_decode(bool need_encoder) {
if (!may_try_decode()) {
return Status::Error("Not ready");
}
std::unique_ptr<Encoder> encoder;
if (need_encoder) {
encoder = RoundRobinEncoder::create(data_.copy(), symbol_size_);
}
return DataWithEncoder{data_.clone(), std::move(encoder)};
}
Status RoundRobinDecoder::add_symbol(Symbol symbol) {
if (symbol.data.size() != symbol_size_) {
return Status::Error("Symbol has invalid length");
}
auto pos = symbol.id % symbols_count_;
if (data_mask_[pos]) {
return td::Status::OK();
}
data_mask_[pos] = true;
auto offset = pos * symbol_size_;
data_.as_slice().substr(offset).truncate(symbol_size_).copy_from(symbol.data.as_slice());
left_symbols_--;
return td::Status::OK();
}
RoundRobinDecoder::RoundRobinDecoder(RoundRobinEncoder::Parameters parameters)
: data_(BufferSlice(parameters.data_size))
, data_mask_(parameters.symbols_count, false)
, left_symbols_(parameters.symbols_count)
, symbol_size_(parameters.symbol_size)
, symbols_count_(parameters.symbols_count) {
}
std::unique_ptr<RaptorQEncoder> RaptorQEncoder::create(BufferSlice data, size_t max_symbol_size) {
auto encoder = raptorq::Encoder::create(max_symbol_size, std::move(data)).move_as_ok();
return std::make_unique<RaptorQEncoder>(std::move(encoder));
}
Symbol RaptorQEncoder::gen_symbol(uint32 id) {
BufferSlice data(encoder_->get_parameters().symbol_size);
encoder_->gen_symbol(id, data.as_slice()).ensure();
return Symbol{id, std::move(data)};
}
RaptorQEncoder::Info RaptorQEncoder::get_info() const {
auto info = encoder_->get_info();
return {info.symbol_count, info.ready_symbol_count};
}
void RaptorQEncoder::prepare_more_symbols() {
encoder_->precalc();
}
RaptorQEncoder::Parameters RaptorQEncoder::get_parameters() const {
Parameters res;
auto p = encoder_->get_parameters();
res.symbol_size = p.symbol_size;
res.data_size = p.data_size;
res.symbols_count = p.symbols_count;
return res;
}
RaptorQEncoder::RaptorQEncoder(std::unique_ptr<raptorq::Encoder> encoder) : encoder_(std::move(encoder)) {
}
RaptorQEncoder::~RaptorQEncoder() = default;
std::unique_ptr<RaptorQDecoder> RaptorQDecoder::create(RaptorQEncoder::Parameters parameters) {
raptorq::Encoder::Parameters p;
p.symbol_size = parameters.symbol_size;
p.data_size = parameters.data_size;
p.symbols_count = parameters.symbols_count;
return std::make_unique<RaptorQDecoder>(raptorq::Decoder::create(p).move_as_ok());
}
bool RaptorQDecoder::may_try_decode() const {
return decoder_->may_try_decode();
}
Result<DataWithEncoder> RaptorQDecoder::try_decode(bool need_encoder) {
TRY_RESULT(data_with_encoder, decoder_->try_decode(need_encoder));
DataWithEncoder res;
res.data = std::move(data_with_encoder.data);
res.encoder = std::make_unique<RaptorQEncoder>(std::move(data_with_encoder.encoder));
return std::move(res);
}
Status RaptorQDecoder::add_symbol(Symbol symbol) {
return decoder_->add_symbol({symbol.id, symbol.data.as_slice()});
}
RaptorQDecoder::RaptorQDecoder(std::unique_ptr<raptorq::Decoder> decoder) : decoder_(std::move(decoder)) {
}
RaptorQDecoder::~RaptorQDecoder() = default;
std::unique_ptr<OnlineEncoder> OnlineEncoder::create(BufferSlice data, size_t max_symbol_size) {
auto encoder = online_code::Encoder::create(max_symbol_size, std::move(data)).move_as_ok();
return std::make_unique<OnlineEncoder>(std::move(encoder));
}
Symbol OnlineEncoder::gen_symbol(uint32 id) {
BufferSlice data(encoder_->get_parameters().symbol_size);
encoder_->gen_symbol(id, data.as_slice());
return Symbol{id, std::move(data)};
}
OnlineEncoder::Parameters OnlineEncoder::get_parameters() const {
Parameters res;
auto p = encoder_->get_parameters();
res.symbol_size = p.symbol_size;
res.data_size = p.data_size;
res.symbols_count = p.symbols_count;
return res;
}
OnlineEncoder::OnlineEncoder(std::unique_ptr<online_code::Encoder> encoder) : encoder_(std::move(encoder)) {
}
OnlineEncoder::~OnlineEncoder() = default;
std::unique_ptr<OnlineDecoder> OnlineDecoder::create(OnlineEncoder::Parameters parameters) {
online_code::Encoder::Parameters p;
p.symbol_size = parameters.symbol_size;
p.data_size = parameters.data_size;
p.symbols_count = parameters.symbols_count;
return std::make_unique<OnlineDecoder>(online_code::Decoder::create(p).move_as_ok(), parameters.symbol_size);
}
bool OnlineDecoder::may_try_decode() const {
return decoder_->is_ready();
}
Result<DataWithEncoder> OnlineDecoder::try_decode(bool need_encoder) {
if (!may_try_decode()) {
return Status::Error("Not ready yet");
}
std::unique_ptr<Encoder> encoder;
auto data = decoder_->get_data();
if (need_encoder) {
encoder = RoundRobinEncoder::create(data.copy(), symbol_size_);
}
return DataWithEncoder{std::move(data), nullptr};
}
Status OnlineDecoder::add_symbol(Symbol symbol) {
return decoder_->add_symbol({symbol.id, symbol.data.as_slice()});
}
OnlineDecoder::OnlineDecoder(std::unique_ptr<online_code::Decoder> decoder, size_t symbol_size)
: decoder_(std::move(decoder)), symbol_size_(symbol_size) {
}
OnlineDecoder::~OnlineDecoder() = default;
} // namespace fec
} // namespace td

196
tdfec/td/fec/fec.h Normal file
View file

@ -0,0 +1,196 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/buffer.h"
namespace td {
namespace raptorq {
class Encoder;
class Decoder;
} // namespace raptorq
namespace online_code {
class Encoder;
class Decoder;
} // namespace online_code
} // namespace td
namespace td {
namespace fec {
struct Symbol {
uint32 id;
BufferSlice data;
};
class Encoder;
struct DataWithEncoder {
BufferSlice data;
std::unique_ptr<Encoder> encoder;
};
class Encoder {
public:
virtual Symbol gen_symbol(uint32 id) = 0;
struct Info {
uint32 symbol_count;
uint32 ready_symbol_count;
};
virtual Info get_info() const {
return Info{static_cast<uint32>(-1), static_cast<uint32>(-1)};
}
// Concurrent calls are forbidden
virtual void prepare_more_symbols() {
}
// struct Parameters;
// Parameters get_parameters();
virtual ~Encoder() {
}
};
class Decoder {
public:
virtual ~Decoder() {
}
virtual bool may_try_decode() const = 0;
virtual Result<DataWithEncoder> try_decode(bool need_encoder) = 0;
virtual Status add_symbol(Symbol symbol) = 0;
};
class RoundRobinEncoder : public Encoder {
public:
static std::unique_ptr<RoundRobinEncoder> create(BufferSlice data, size_t max_symbol_size);
Symbol gen_symbol(uint32 id) override;
struct Parameters {
size_t data_size;
size_t symbol_size;
size_t symbols_count;
};
Parameters get_parameters() const;
RoundRobinEncoder(BufferSlice data, size_t max_symbol_size);
private:
BufferSlice data_;
size_t symbol_size_;
size_t symbols_count_;
size_t rounded_data_size_;
};
class RoundRobinDecoder : public Decoder {
public:
static std::unique_ptr<RoundRobinDecoder> create(RoundRobinEncoder::Parameters parameters);
bool may_try_decode() const override;
Result<DataWithEncoder> try_decode(bool need_encoder) override;
Status add_symbol(Symbol symbol) override;
RoundRobinDecoder(RoundRobinEncoder::Parameters parameters);
private:
BufferSlice data_;
vector<bool> data_mask_;
size_t left_symbols_;
size_t symbol_size_;
size_t symbols_count_;
};
class RaptorQEncoder : public Encoder {
public:
static std::unique_ptr<RaptorQEncoder> create(BufferSlice data, size_t max_symbol_size);
Symbol gen_symbol(uint32 id) override;
Info get_info() const override;
void prepare_more_symbols() override;
struct Parameters {
size_t data_size;
size_t symbol_size;
size_t symbols_count;
};
Parameters get_parameters() const;
RaptorQEncoder(std::unique_ptr<raptorq::Encoder> encoder);
~RaptorQEncoder();
private:
std::unique_ptr<raptorq::Encoder> encoder_;
};
class RaptorQDecoder : public Decoder {
public:
static std::unique_ptr<RaptorQDecoder> create(RaptorQEncoder::Parameters parameters);
bool may_try_decode() const override;
Result<DataWithEncoder> try_decode(bool need_encoder) override;
Status add_symbol(Symbol symbol) override;
RaptorQDecoder(std::unique_ptr<raptorq::Decoder> decoder);
~RaptorQDecoder();
private:
std::unique_ptr<raptorq::Decoder> decoder_;
BufferSlice res_;
};
class OnlineEncoder : public Encoder {
public:
static std::unique_ptr<OnlineEncoder> create(BufferSlice data, size_t max_symbol_size);
Symbol gen_symbol(uint32 id) override;
struct Parameters {
size_t data_size;
size_t symbol_size;
size_t symbols_count;
};
Parameters get_parameters() const;
OnlineEncoder(std::unique_ptr<online_code::Encoder> encoder);
~OnlineEncoder();
private:
std::unique_ptr<online_code::Encoder> encoder_;
};
class OnlineDecoder : public Decoder {
public:
static std::unique_ptr<OnlineDecoder> create(OnlineEncoder::Parameters parameters);
bool may_try_decode() const override;
Result<DataWithEncoder> try_decode(bool need_encoder) override;
Status add_symbol(Symbol symbol) override;
OnlineDecoder(std::unique_ptr<online_code::Decoder> decoder, size_t symbol_size);
~OnlineDecoder();
private:
std::unique_ptr<online_code::Decoder> decoder_;
size_t symbol_size_;
BufferSlice res_;
};
} // namespace fec
} // namespace td

77
tdfec/td/fec/online/Decoder.cpp Normal file
View file

@ -0,0 +1,77 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/fec/online/Decoder.h"
namespace td {
namespace online_code {
Result<std::unique_ptr<Decoder>> Decoder::create(Encoder::Parameters p) {
return std::make_unique<Decoder>(p.symbol_size, p.data_size);
}
Decoder::Decoder(size_t symbol_size, size_t data_size)
: parameters_(Rfc::get_parameters((data_size + symbol_size - 1) / symbol_size).move_as_ok())
, symbol_size_(symbol_size)
, data_size_(data_size) {
MatrixGF256 d_{1, symbol_size_};
d_.set_zero();
std::vector<uint32> e_cnt(parameters_.outer_encoding_blocks_count(), 0);
std::vector<std::vector<uint32>> e(parameters_.outer_encoding_blocks_count());
parameters_.outer_encoding_for_each([&](uint32 i, uint32 j) { e_cnt[i]++; });
for (uint32 i = 0; i < parameters_.outer_encoding_blocks_count(); i++) {
e[i].reserve(e_cnt[i]);
}
parameters_.outer_encoding_for_each([&](uint32 i, uint32 j) { e[i].push_back(j); });
for (uint32 i = 0; i < parameters_.outer_encoding_blocks_count(); i++) {
decoding_.add_equation(e[i], d_.row(0));
}
}
Status Decoder::add_symbol(raptorq::SymbolRef symbol_ref) {
if (symbol_ref.data.size() != symbol_size_) {
return Status::Error("Symbol has invalid length");
}
auto offset = decoding_.ready_symbols().size();
decoding_.add_equation(parameters_.get_inner_encoding_row(symbol_ref.id), symbol_ref.data);
//LOG(ERROR) << inner_decoding_.ready_symbols().size();
for (auto symbol_id : decoding_.ready_symbols().substr(offset)) {
if (symbol_id < parameters_.source_blocks_count()) {
ready_cnt_++;
}
}
return Status::OK();
}
bool Decoder::is_ready() const {
return ready_cnt_ == parameters_.source_blocks_count();
}
BufferSlice Decoder::get_data() const {
CHECK(is_ready());
BufferSlice res(data_size_);
for (uint32 i = 0; i < parameters_.source_blocks_count(); i++) {
auto dest = res.as_slice().substr(i * symbol_size_);
dest.copy_from(decoding_.get_symbol(i).truncate(dest.size()));
}
return res;
}
} // namespace online_code
} // namespace td

49
tdfec/td/fec/online/Decoder.h Normal file
View file

@ -0,0 +1,49 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/fec/online/Rfc.h"
#include "td/fec/common/SymbolRef.h"
#include "td/fec/algebra/BeliefPropagationDecoding.h"
#include "td/fec/online/Encoder.h"
namespace td {
namespace online_code {
class Decoder {
public:
static Result<std::unique_ptr<Decoder>> create(Encoder::Parameters p);
Decoder(size_t symbol_size, size_t data_size);
Status add_symbol(raptorq::SymbolRef symbol_ref);
bool is_ready() const;
BufferSlice get_data() const;
private:
Rfc::Parameters parameters_;
size_t symbol_size_;
size_t data_size_;
size_t ready_cnt_{0};
BeliefPropagationDecoding decoding_{parameters_.source_blocks_count() + parameters_.outer_encoding_blocks_count(),
symbol_size_};
};
} // namespace online_code
} // namespace td

63
tdfec/td/fec/online/Encoder.cpp Normal file
View file

@ -0,0 +1,63 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/fec/online/Encoder.h"
#include "td/fec/common/SymbolsView.h"
namespace td {
namespace online_code {
Result<std::unique_ptr<Encoder>> Encoder::create(size_t symbol_size, BufferSlice data) {
return std::make_unique<Encoder>(symbol_size, data.as_slice());
}
Encoder::Encoder(size_t symbol_size, Slice data)
: parameters_(Rfc::get_parameters((data.size() + symbol_size - 1) / symbol_size).move_as_ok())
, C_(parameters_.source_blocks_count() + parameters_.outer_encoding_blocks_count(), symbol_size)
, d_(1, symbol_size)
, data_size_(data.size()) {
raptorq::SymbolsView view(parameters_.source_blocks_count() + parameters_.outer_encoding_blocks_count(), symbol_size,
data);
for (auto &symbol : view.symbols()) {
C_.row_set(symbol.id, symbol.data);
}
parameters_.outer_encoding_for_each([&](uint32 i, uint32 j) {
if (j < parameters_.source_blocks_count()) {
C_.row_add(i + parameters_.source_blocks_count(), C_.row(j));
}
});
}
Encoder::Parameters Encoder::get_parameters() const {
Parameters p;
p.symbols_count = parameters_.source_blocks_count();
p.symbol_size = C_.cols();
p.data_size = data_size_;
return p;
}
Status Encoder::gen_symbol(uint32 symbol_id, MutableSlice slice) {
d_.set_zero();
for (auto id : parameters_.get_inner_encoding_row(symbol_id)) {
d_.row_add(0, C_.row(id));
}
slice.copy_from(d_.row(0));
return Status::OK();
}
} // namespace online_code
} // namespace td

48
tdfec/td/fec/online/Encoder.h Normal file
View file

@ -0,0 +1,48 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/fec/online/Rfc.h"
#include "td/fec/algebra/MatrixGF256.h"
#include "td/utils/buffer.h"
namespace td {
namespace online_code {
class Encoder {
public:
struct Parameters {
size_t symbols_count;
size_t symbol_size;
size_t data_size;
};
static Result<std::unique_ptr<Encoder>> create(size_t symbol_size, BufferSlice data);
Encoder(size_t symbol_size, Slice data);
Parameters get_parameters() const;
Status gen_symbol(uint32 symbol_id, MutableSlice slice);
private:
Rfc::Parameters parameters_;
MatrixGF256 C_;
MatrixGF256 d_;
size_t data_size_;
};
} // namespace online_code
} // namespace td

79
tdfec/td/fec/online/Rfc.cpp Normal file
View file

@ -0,0 +1,79 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/fec/online/Rfc.h"
namespace td {
namespace online_code {
Result<Rfc::Parameters> Rfc::get_parameters(size_t K) {
return Rfc::Parameters(K, 0.001, 3);
}
Rfc::Parameters::Parameters(size_t source_blocks, double epsilon, uint32 quality)
: source_blocks_(static_cast<uint32>(source_blocks))
, epsilon_(epsilon)
, quality_(quality)
, degree_distribution_{epsilon}
, outer_encoding_blocks_count_{
static_cast<uint32>(1 + 0.55 * quality * epsilon * static_cast<double>(source_blocks))} {
}
size_t Rfc::Parameters::source_blocks_count() const {
return source_blocks_;
}
size_t Rfc::Parameters::outer_encoding_blocks_count() const {
return outer_encoding_blocks_count_;
}
size_t Rfc::Parameters::estimated_packets() const {
return size_t(static_cast<double>(source_blocks_count() + outer_encoding_blocks_count()) * (1 + epsilon_));
}
Span<uint32> Rfc::Parameters::get_inner_encoding_row(size_t row_id) const {
Random r(static_cast<uint32>(row_id));
uint32 degree = degree_distribution_.get_degree(static_cast<uint32>(r() % (1 << 20)) * 1.0 / (1 << 20));
inner_distribution_.set_random(&r);
MutableSpan<uint32> res(row_buffer_.data(), degree);
for (auto &x : res) {
x = inner_distribution_();
}
//std::sort(res.begin(), res.end());
//res.truncate(std::unique(res.begin(), res.end()) - res.begin());
return res;
}
Rfc::Parameters::DegreeDistribution::DegreeDistribution(double epsilon) {
uint32 F = static_cast<uint32>(log(epsilon * epsilon / 4) / log(1 - epsilon / 2) + 1 - 1e-9);
double x = 1 - (1 + 1.0 / F) / (1 + epsilon);
p_.reserve(F);
p_.push_back(x);
for (uint32 i = 2; i <= F; i++) {
auto y = (1 - x) * F / ((F - 1.0) * i * (i - 1));
p_.push_back(p_.back() + y);
}
}
uint32 Rfc::Parameters::DegreeDistribution::get_degree(double x) const {
for (uint32 i = 0; i < p_.size(); i++) {
if (x < p_[i]) {
return i + 1;
}
}
UNREACHABLE();
return 1;
}
size_t Rfc::Parameters::DegreeDistribution::get_max_degree() const {
return p_.size();
}
} // namespace online_code
} // namespace td

103
tdfec/td/fec/online/Rfc.h Normal file
View file

@ -0,0 +1,103 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include <random>
#include <algorithm>
#include "td/utils/Status.h"
#include "td/utils/Span.h"
#include "td/utils/Random.h"
namespace td {
namespace online_code {
template <class Random>
class UniformDistributionSimple {
public:
UniformDistributionSimple(uint32 L, uint32 R) : L_(L), R_(R) {
}
void set_random(Random *random) {
random_ = random;
}
uint32 operator()() {
return static_cast<uint32>((*random_)() % (R_ - L_ + 1)) + L_;
}
private:
Random *random_{nullptr};
uint32 L_;
uint32 R_;
};
using Random = Random::Xorshift128plus;
using UniformDistribution = UniformDistributionSimple<Random>;
class Rfc {
public:
class Parameters {
public:
Parameters(size_t source_blocks, double epsilon, uint32 quality);
size_t source_blocks_count() const;
size_t outer_encoding_blocks_count() const;
size_t estimated_packets() const;
template <class F>
void outer_encoding_for_each(F &&f) const {
Random r(static_cast<uint32>(1));
outer_distribution_.set_random(&r);
for (uint32 j = 0; j < outer_encoding_blocks_count(); j++) {
f(j, uint32(source_blocks_count() + j));
}
for (uint32 i = 0; i < source_blocks_count(); i++) {
for (uint32 j = 0; j < quality_; j++) {
f(outer_distribution_(), i);
}
}
}
Span<uint32> get_inner_encoding_row(size_t row_id) const;
private:
struct DegreeDistribution {
public:
DegreeDistribution(double epsilon);
uint32 get_degree(double x) const;
size_t get_max_degree() const;
private:
std::vector<double> p_;
};
uint32 source_blocks_;
double epsilon_;
uint32 quality_;
DegreeDistribution degree_distribution_;
size_t outer_encoding_blocks_count_;
mutable std::vector<uint32> row_buffer_ = std::vector<uint32>(degree_distribution_.get_max_degree(), 0);
mutable UniformDistribution outer_distribution_{0, static_cast<uint32>(outer_encoding_blocks_count()) - 1};
mutable UniformDistribution inner_distribution_{
0, static_cast<uint32>(outer_encoding_blocks_count() + source_blocks_count()) - 1};
};
static Result<Parameters> get_parameters(size_t K);
};
} // namespace online_code
} // namespace td

151
tdfec/td/fec/raptorq/Decoder.cpp Normal file
View file

@ -0,0 +1,151 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/fec/raptorq/Decoder.h"
namespace td {
namespace raptorq {
Result<std::unique_ptr<Decoder>> Decoder::create(Encoder::Parameters p) {
TRY_RESULT(rfc_p, Rfc::get_parameters(p.symbols_count));
return std::make_unique<Decoder>(rfc_p, p.symbol_size, p.data_size);
}
Decoder::Decoder(const Rfc::Parameters &p, size_t symbol_size, size_t data_size) : p_(p) {
mask_ = vector<bool>(p.K, false);
mask_size_ = 0;
data_ = BufferSlice(p.K * symbol_size);
data_size_ = data_size;
symbol_size_ = symbol_size;
}
bool Decoder::may_try_decode() const {
return may_decode_;
}
Status Decoder::add_symbol(SymbolRef symbol) {
if (symbol.data.size() != symbol_size_) {
return Status::Error("Symbol has invalid length");
}
if (symbol.id < p_.K) {
add_small_symbol(symbol);
return Status::OK();
}
if (mask_size_ + slow_symbols_set_.size() >= p_.K + 10) {
return Status::OK();
}
add_big_symbol(symbol);
return Status::OK();
}
Result<Decoder::DataWithEncoder> Decoder::try_decode(bool need_encoder) {
if (!may_decode_) {
return Status::Error("Need more symbols");
}
optional<RawEncoder> raw_encoder;
if (mask_size_ < p_.K) {
flush_symbols();
may_decode_ = false;
TRY_RESULT(C, Solver::run(p_, symbols_));
raw_encoder = RawEncoder(p_, std::move(C));
for (uint32 i = 0; i < p_.K; i++) {
if (!mask_[i]) {
(*raw_encoder).gen_symbol(i, data_.as_slice().substr(i * symbol_size_, symbol_size_));
mask_[i] = true;
mask_size_++;
}
}
}
auto data = data_.from_slice(data_.as_slice().truncate(data_size_));
std::unique_ptr<Encoder> encoder;
if (need_encoder) {
encoder = std::make_unique<Encoder>(p_, symbol_size_, data.copy(), std::move(raw_encoder));
}
return DataWithEncoder{std::move(data), std::move(encoder)};
}
void Decoder::add_small_symbol(SymbolRef symbol) {
if (mask_[symbol.id]) {
return;
}
mask_size_++;
mask_[symbol.id] = true;
auto slice = data_.as_slice().substr(symbol.id * symbol_size_, symbol_size_);
slice.copy_from(symbol.data);
if (flush_symbols_) {
symbols_.push_back({symbol.id, slice});
}
update_may_decode();
}
void Decoder::add_big_symbol(SymbolRef symbol) {
if (!slow_path_) {
on_first_slow_path();
}
symbol.id += p_.K_padded - p_.K;
if (slow_symbols_set_.size() == slow_symbols_) {
// Got at least p.K + 10 different symbols
return;
}
size_t offset = slow_symbols_set_.size() * symbol_size_;
if (!slow_symbols_set_.insert(symbol.id).second) {
return;
}
auto slice = buffer_.as_slice().substr(offset, symbol_size_);
slice.copy_from(symbol.data);
symbols_.push_back({symbol.id, slice});
update_may_decode();
}
void Decoder::update_may_decode() {
size_t total_symbols = mask_size_ + slow_symbols_set_.size();
if (total_symbols < p_.K) {
return;
}
may_decode_ = true;
}
void Decoder::on_first_slow_path() {
slow_path_ = true;
slow_symbols_ = p_.K + 10 - mask_size_;
buffer_ = BufferSlice(slow_symbols_ * symbol_size_);
symbols_.reserve(p_.K + 10);
}
void Decoder::flush_symbols() {
if (flush_symbols_) {
return;
}
flush_symbols_ = true;
zero_symbol_ = std::string(symbol_size_, '\0');
for (uint32 i = p_.K; i < p_.K_padded; i++) {
symbols_.push_back({i, Slice(zero_symbol_)});
}
for (uint32 i = 0; i < p_.K; i++) {
if (mask_[i]) {
symbols_.push_back({i, data_.as_slice().substr(i * symbol_size_, symbol_size_)});
}
}
}
} // namespace raptorq
} // namespace td

71
tdfec/td/fec/raptorq/Decoder.h Normal file
View file

@ -0,0 +1,71 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/fec/raptorq/Solver.h"
#include "td/fec/raptorq/Encoder.h"
#include <set>
namespace td {
namespace raptorq {
class Decoder {
public:
static Result<std::unique_ptr<Decoder>> create(Encoder::Parameters p);
Decoder(const Rfc::Parameters &p, size_t symbol_size, size_t data_size);
Status add_symbol(SymbolRef symbol);
struct DataWithEncoder {
BufferSlice data;
std::unique_ptr<Encoder> encoder;
};
Result<DataWithEncoder> try_decode(bool need_encoder);
bool may_try_decode() const;
private:
Rfc::Parameters p_;
size_t symbol_size_;
bool may_decode_{false};
vector<bool> mask_;
size_t mask_size_{0};
BufferSlice data_;
size_t data_size_;
bool flush_symbols_{false};
bool slow_path_{false};
size_t slow_symbols_;
BufferSlice buffer_;
std::vector<SymbolRef> symbols_;
std::set<uint32> slow_symbols_set_;
std::string zero_symbol_;
void add_small_symbol(SymbolRef symbol);
void add_big_symbol(SymbolRef symbol);
void update_may_decode();
void on_first_slow_path();
void flush_symbols();
};
} // namespace raptorq
} // namespace td

81
tdfec/td/fec/raptorq/Encoder.cpp Normal file
View file

@ -0,0 +1,81 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/fec/raptorq/Encoder.h"
#include "td/fec/raptorq/Solver.h"
namespace td {
namespace raptorq {
Result<std::unique_ptr<Encoder>> Encoder::create(size_t symbol_size, BufferSlice data) {
TRY_RESULT(p, Rfc::get_parameters((data.size() + symbol_size - 1) / symbol_size));
return std::make_unique<Encoder>(p, symbol_size, std::move(data));
}
Encoder::Encoder(Rfc::Parameters p, size_t symbol_size, BufferSlice data, optional<RawEncoder> raw_encoder)
: p_(p)
, symbol_size_(symbol_size)
, data_(std::move(data))
, raw_encoder_{std::move(raw_encoder)}
, has_encoder_{bool(raw_encoder_)} {
}
Encoder::Parameters Encoder::get_parameters() const {
Parameters res;
res.symbol_size = symbol_size_;
res.data_size = data_.size();
res.symbols_count = p_.K;
return res;
}
Encoder::Info Encoder::get_info() const {
Info res;
res.symbol_count = 1 << 24;
res.ready_symbol_count = has_precalc() ? res.symbol_count : p_.K;
return res;
}
Status Encoder::gen_symbol(uint32 id, MutableSlice slice) {
if (id < p_.K) {
slice.copy_from(first_symbols_.symbols()[id].data);
return Status::OK();
}
if (!has_precalc()) {
return Status::Error("Precalc is not finished");
}
(*raw_encoder_).gen_symbol(id + p_.K_padded - p_.K, slice);
return Status::OK();
}
bool Encoder::has_precalc() const {
return has_encoder_.load(std::memory_order_acquire);
}
void Encoder::precalc() {
if (has_precalc()) {
return;
}
auto r_C = Solver::run(p_, first_symbols_.symbols());
LOG_IF(FATAL, r_C.is_error()) << r_C.error();
raw_encoder_ = RawEncoder(p_, r_C.move_as_ok());
has_encoder_ = true;
}
} // namespace raptorq
} // namespace td

68
tdfec/td/fec/raptorq/Encoder.h Normal file
View file

@ -0,0 +1,68 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/fec/raptorq/RawEncoder.h"
#include "td/fec/common/SymbolsView.h"
#include "td/utils/optional.h"
#include "td/utils/buffer.h"
#include <atomic>
namespace td {
namespace raptorq {
class Encoder {
public:
struct Parameters {
size_t symbols_count;
size_t symbol_size;
size_t data_size;
};
struct Info {
uint32 symbol_count;
uint32 ready_symbol_count;
};
static Result<std::unique_ptr<Encoder>> create(size_t symbol_size, BufferSlice data);
static std::unique_ptr<Encoder> create(size_t symbol_size, RawEncoder raw_encoder);
Encoder(Rfc::Parameters p, size_t symbol_size, BufferSlice data, optional<RawEncoder> raw_encoder = {});
Status gen_symbol(uint32 id, MutableSlice slice);
Parameters get_parameters() const;
Info get_info() const;
bool has_precalc() const;
// Must be called only once. No concurrent calls are allowed.
// Also it may be and should be called from another thread.
void precalc();
private:
Rfc::Parameters p_;
size_t symbol_size_;
BufferSlice data_;
SymbolsView first_symbols_{p_.K_padded, symbol_size_, data_.as_slice()};
optional<RawEncoder> raw_encoder_;
std::atomic<bool> has_encoder_{false};
};
} // namespace raptorq
} // namespace td

30
tdfec/td/fec/raptorq/RawEncoder.cpp Normal file
View file

@ -0,0 +1,30 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/fec/raptorq/RawEncoder.h"
namespace td {
namespace raptorq {
void RawEncoder::gen_symbol(uint32 id, MutableSlice to) const {
CHECK(to.size() == symbol_size());
d_.set_zero();
p_.encoding_row_for_each(p_.get_encoding_row(id), [&](auto row) { d_.row_add(0, C_.row(row)); });
to.copy_from(d_.row(0).truncate(symbol_size()));
}
} // namespace raptorq
} // namespace td

42
tdfec/td/fec/raptorq/RawEncoder.h Normal file
View file

@ -0,0 +1,42 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/fec/raptorq/Rfc.h"
#include "td/fec/algebra/MatrixGF256.h"
namespace td {
namespace raptorq {
class RawEncoder {
public:
RawEncoder(Rfc::Parameters p, MatrixGF256 C) : p_(p), C_(std::move(C)), d_{1, symbol_size()} {
}
size_t symbol_size() const {
return C_.cols();
}
void gen_symbol(uint32 id, MutableSlice to) const;
private:
Rfc::Parameters p_;
MatrixGF256 C_;
mutable MatrixGF256 d_;
};
} // namespace raptorq
} // namespace td

446
tdfec/td/fec/raptorq/Rfc.cpp Normal file
View file

@ -0,0 +1,446 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/fec/raptorq/Rfc.h"
#include <array>
namespace td {
namespace raptorq {
// 5.5. Random Numbers
//
// The four arrays V0, V1, V2, and V3 used in Section 5.3.5.1 are
// provided below. There are 256 entries in each of the four arrays.
// The indexing into each array starts at 0, and the entries are 32-bit
// unsigned integers.
namespace {
constexpr size_t V_size{256};
constexpr std::array<uint32, V_size> V0 = {
{251291136, 3952231631, 3370958628, 4070167936, 123631495, 3351110283, 3218676425, 2011642291, 774603218,
2402805061, 1004366930, 1843948209, 428891132, 3746331984, 1591258008, 3067016507, 1433388735, 504005498,
2032657933, 3419319784, 2805686246, 3102436986, 3808671154, 2501582075, 3978944421, 246043949, 4016898363,
649743608, 1974987508, 2651273766, 2357956801, 689605112, 715807172, 2722736134, 191939188, 3535520147,
3277019569, 1470435941, 3763101702, 3232409631, 122701163, 3920852693, 782246947, 372121310, 2995604341,
2045698575, 2332962102, 4005368743, 218596347, 3415381967, 4207612806, 861117671, 3676575285, 2581671944,
3312220480, 681232419, 307306866, 4112503940, 1158111502, 709227802, 2724140433, 4201101115, 4215970289,
4048876515, 3031661061, 1909085522, 510985033, 1361682810, 129243379, 3142379587, 2569842483, 3033268270,
1658118006, 932109358, 1982290045, 2983082771, 3007670818, 3448104768, 683749698, 778296777, 1399125101,
1939403708, 1692176003, 3868299200, 1422476658, 593093658, 1878973865, 2526292949, 1591602827, 3986158854,
3964389521, 2695031039, 1942050155, 424618399, 1347204291, 2669179716, 2434425874, 2540801947, 1384069776,
4123580443, 1523670218, 2708475297, 1046771089, 2229796016, 1255426612, 4213663089, 1521339547, 3041843489,
420130494, 10677091, 515623176, 3457502702, 2115821274, 2720124766, 3242576090, 854310108, 425973987,
325832382, 1796851292, 2462744411, 1976681690, 1408671665, 1228817808, 3917210003, 263976645, 2593736473,
2471651269, 4291353919, 650792940, 1191583883, 3046561335, 2466530435, 2545983082, 969168436, 2019348792,
2268075521, 1169345068, 3250240009, 3963499681, 2560755113, 911182396, 760842409, 3569308693, 2687243553,
381854665, 2613828404, 2761078866, 1456668111, 883760091, 3294951678, 1604598575, 1985308198, 1014570543,
2724959607, 3062518035, 3115293053, 138853680, 4160398285, 3322241130, 2068983570, 2247491078, 3669524410,
1575146607, 828029864, 3732001371, 3422026452, 3370954177, 4006626915, 543812220, 1243116171, 3928372514,
2791443445, 4081325272, 2280435605, 885616073, 616452097, 3188863436, 2780382310, 2340014831, 1208439576,
258356309, 3837963200, 2075009450, 3214181212, 3303882142, 880813252, 1355575717, 207231484, 2420803184,
358923368, 1617557768, 3272161958, 1771154147, 2842106362, 1751209208, 1421030790, 658316681, 194065839,
3241510581, 38625260, 301875395, 4176141739, 297312930, 2137802113, 1502984205, 3669376622, 3728477036,
234652930, 2213589897, 2734638932, 1129721478, 3187422815, 2859178611, 3284308411, 3819792700, 3557526733,
451874476, 1740576081, 3592838701, 1709429513, 3702918379, 3533351328, 1641660745, 179350258, 2380520112,
3936163904, 3685256204, 3156252216, 1854258901, 2861641019, 3176611298, 834787554, 331353807, 517858103,
3010168884, 4012642001, 2217188075, 3756943137, 3077882590, 2054995199, 3081443129, 3895398812, 1141097543,
2376261053, 2626898255, 2554703076, 401233789, 1460049922, 678083952, 1064990737, 940909784, 1673396780,
528881783, 1712547446, 3629685652, 1358307511}};
constexpr std::array<uint32, V_size> V1 = {
{807385413, 2043073223, 3336749796, 1302105833, 2278607931, 541015020, 1684564270, 372709334, 3508252125,
1768346005, 1270451292, 2603029534, 2049387273, 3891424859, 2152948345, 4114760273, 915180310, 3754787998,
700503826, 2131559305, 1308908630, 224437350, 4065424007, 3638665944, 1679385496, 3431345226, 1779595665,
3068494238, 1424062773, 1033448464, 4050396853, 3302235057, 420600373, 2868446243, 311689386, 259047959,
4057180909, 1575367248, 4151214153, 110249784, 3006865921, 4293710613, 3501256572, 998007483, 499288295,
1205710710, 2997199489, 640417429, 3044194711, 486690751, 2686640734, 2394526209, 2521660077, 49993987,
3843885867, 4201106668, 415906198, 19296841, 2402488407, 2137119134, 1744097284, 579965637, 2037662632,
852173610, 2681403713, 1047144830, 2982173936, 910285038, 4187576520, 2589870048, 989448887, 3292758024,
506322719, 176010738, 1865471968, 2619324712, 564829442, 1996870325, 339697593, 4071072948, 3618966336,
2111320126, 1093955153, 957978696, 892010560, 1854601078, 1873407527, 2498544695, 2694156259, 1927339682,
1650555729, 183933047, 3061444337, 2067387204, 228962564, 3904109414, 1595995433, 1780701372, 2463145963,
307281463, 3237929991, 3852995239, 2398693510, 3754138664, 522074127, 146352474, 4104915256, 3029415884,
3545667983, 332038910, 976628269, 3123492423, 3041418372, 2258059298, 2139377204, 3243642973, 3226247917,
3674004636, 2698992189, 3453843574, 1963216666, 3509855005, 2358481858, 747331248, 1957348676, 1097574450,
2435697214, 3870972145, 1888833893, 2914085525, 4161315584, 1273113343, 3269644828, 3681293816, 412536684,
1156034077, 3823026442, 1066971017, 3598330293, 1979273937, 2079029895, 1195045909, 1071986421, 2712821515,
3377754595, 2184151095, 750918864, 2585729879, 4249895712, 1832579367, 1192240192, 946734366, 31230688,
3174399083, 3549375728, 1642430184, 1904857554, 861877404, 3277825584, 4267074718, 3122860549, 666423581,
644189126, 226475395, 307789415, 1196105631, 3191691839, 782852669, 1608507813, 1847685900, 4069766876,
3931548641, 2526471011, 766865139, 2115084288, 4259411376, 3323683436, 568512177, 3736601419, 1800276898,
4012458395, 1823982, 27980198, 2023839966, 869505096, 431161506, 1024804023, 1853869307, 3393537983,
1500703614, 3019471560, 1351086955, 3096933631, 3034634988, 2544598006, 1230942551, 3362230798, 159984793,
491590373, 3993872886, 3681855622, 903593547, 3535062472, 1799803217, 772984149, 895863112, 1899036275,
4187322100, 101856048, 234650315, 3183125617, 3190039692, 525584357, 1286834489, 455810374, 1869181575,
922673938, 3877430102, 3422391938, 1414347295, 1971054608, 3061798054, 830555096, 2822905141, 167033190,
1079139428, 4210126723, 3593797804, 429192890, 372093950, 1779187770, 3312189287, 204349348, 452421568,
2800540462, 3733109044, 1235082423, 1765319556, 3174729780, 3762994475, 3171962488, 442160826, 198349622,
45942637, 1324086311, 2901868599, 678860040, 3812229107, 19936821, 1119590141, 3640121682, 3545931032,
2102949142, 2828208598, 3603378023, 4135048896}};
constexpr std::array<uint32, V_size> V2 = {
{1629829892, 282540176, 2794583710, 496504798, 2990494426, 3070701851, 2575963183, 4094823972, 2775723650,
4079480416, 176028725, 2246241423, 3732217647, 2196843075, 1306949278, 4170992780, 4039345809, 3209664269,
3387499533, 293063229, 3660290503, 2648440860, 2531406539, 3537879412, 773374739, 4184691853, 1804207821,
3347126643, 3479377103, 3970515774,
1891731298, 2368003842, 3537588307, 2969158410, 4230745262, 831906319, 2935838131, 264029468, 120852739,
3200326460, 355445271, 2296305141, 1566296040, 1760127056, 20073893, 3427103620, 2866979760, 2359075957,
2025314291, 1725696734, 3346087406, 2690756527, 99815156, 4248519977, 2253762642, 3274144518, 598024568,
3299672435, 556579346, 4121041856, 2896948975, 3620123492, 918453629, 3249461198, 2231414958, 3803272287,
3657597946, 2588911389, 242262274, 1725007475, 2026427718, 46776484, 2873281403, 2919275846, 3177933051,
1918859160, 2517854537, 1857818511, 3234262050, 479353687, 200201308, 2801945841, 1621715769, 483977159,
423502325, 3689396064, 1850168397, 3359959416, 3459831930, 841488699, 3570506095, 930267420, 1564520841,
2505122797, 593824107, 1116572080, 819179184, 3139123629, 1414339336, 1076360795, 512403845, 177759256,
1701060666, 2239736419, 515179302, 2935012727, 3821357612, 1376520851, 2700745271, 966853647, 1041862223,
715860553, 171592961, 1607044257, 1227236688, 3647136358, 1417559141, 4087067551, 2241705880, 4194136288,
1439041934, 20464430, 119668151, 2021257232, 2551262694, 1381539058, 4082839035, 498179069, 311508499,
3580908637, 2889149671, 142719814, 1232184754, 3356662582, 2973775623, 1469897084, 1728205304, 1415793613,
50111003, 3133413359, 4074115275, 2710540611, 2700083070, 2457757663, 2612845330, 3775943755, 2469309260,
2560142753, 3020996369, 1691667711, 4219602776, 1687672168, 1017921622, 2307642321, 368711460, 3282925988,
213208029, 4150757489, 3443211944, 2846101972, 4106826684, 4272438675, 2199416468, 3710621281, 497564971,
285138276, 765042313, 916220877, 3402623607, 2768784621, 1722849097, 3386397442, 487920061, 3569027007,
3424544196, 217781973, 2356938519, 3252429414, 145109750, 2692588106, 2454747135, 1299493354, 4120241887,
2088917094, 932304329, 1442609203, 952586974, 3509186750, 753369054, 854421006, 1954046388, 2708927882,
4047539230, 3048925996, 1667505809, 805166441, 1182069088, 4265546268, 4215029527, 3374748959, 373532666,
2454243090, 2371530493, 3651087521, 2619878153, 1651809518, 1553646893, 1227452842, 703887512, 3696674163,
2552507603, 2635912901, 895130484, 3287782244, 3098973502, 990078774, 3780326506, 2290845203, 41729428,
1949580860, 2283959805, 1036946170, 1694887523, 4880696, 466000198, 2765355283, 3318686998, 1266458025,
3919578154, 3545413527, 2627009988, 3744680394, 1696890173, 3250684705, 4142417708, 915739411, 3308488877,
1289361460, 2942552331, 1169105979, 3342228712, 698560958, 1356041230, 2401944293, 107705232, 3701895363,
903928723, 3646581385, 844950914, 1944371367, 3863894844, 2946773319, 1972431613, 1706989237, 29917467,
3497665928}};
constexpr std::array<uint32, V_size> V3 = {
{1191369816, 744902811, 2539772235, 3213192037, 3286061266, 1200571165, 2463281260, 754888894, 714651270,
1968220972, 3628497775, 1277626456, 1493398934, 364289757, 2055487592, 3913468088, 2930259465, 902504567,
3967050355, 2056499403, 692132390, 186386657, 832834706, 859795816, 1283120926, 2253183716, 3003475205,
1755803552, 2239315142, 4271056352, 2184848469, 769228092, 1249230754, 1193269205, 2660094102, 642979613,
1687087994, 2726106182, 446402913, 4122186606, 3771347282, 37667136, 192775425, 3578702187, 1952659096,
3989584400, 3069013882, 2900516158, 4045316336, 3057163251, 1702104819, 4116613420, 3575472384, 2674023117,
1409126723, 3215095429, 1430726429, 2544497368, 1029565676, 1855801827, 4262184627, 1854326881, 2906728593,
3277836557, 2787697002, 2787333385, 3105430738, 2477073192, 748038573, 1088396515, 1611204853, 201964005,
3745818380, 3654683549, 3816120877, 3915783622, 2563198722, 1181149055, 33158084, 3723047845, 3790270906,
3832415204, 2959617497, 372900708, 1286738499, 1932439099, 3677748309, 2454711182, 2757856469, 2134027055,
2780052465, 3190347618, 3758510138, 3626329451, 1120743107, 1623585693, 1389834102, 2719230375, 3038609003,
462617590, 260254189, 3706349764, 2556762744, 2874272296, 2502399286, 4216263978, 2683431180, 2168560535,
3561507175, 668095726, 680412330, 3726693946, 4180630637, 3335170953, 942140968, 2711851085, 2059233412,
4265696278, 3204373534, 232855056, 881788313, 2258252172, 2043595984, 3758795150, 3615341325, 2138837681,
1351208537, 2923692473, 3402482785, 2105383425, 2346772751, 499245323, 3417846006, 2366116814, 2543090583,
1828551634, 3148696244, 3853884867, 1364737681, 2200687771, 2689775688, 232720625, 4071657318, 2671968983,
3531415031, 1212852141, 867923311, 3740109711, 1923146533, 3237071777, 3100729255, 3247856816, 906742566,
4047640575, 4007211572, 3495700105, 1171285262, 2835682655, 1634301229, 3115169925, 2289874706, 2252450179,
944880097, 371933491, 1649074501, 2208617414, 2524305981, 2496569844, 2667037160, 1257550794, 3399219045,
3194894295, 1643249887, 342911473, 891025733, 3146861835, 3789181526, 938847812, 1854580183, 2112653794,
2960702988, 1238603378, 2205280635, 1666784014, 2520274614, 3355493726, 2310872278, 3153920489, 2745882591,
1200203158, 3033612415, 2311650167, 1048129133, 4206710184, 4209176741, 2640950279, 2096382177, 4116899089,
3631017851, 4104488173, 1857650503, 3801102932, 445806934, 3055654640, 897898279, 3234007399, 1325494930,
2982247189, 1619020475, 2720040856, 885096170, 3485255499, 2983202469, 3891011124, 546522756, 1524439205,
2644317889, 2170076800, 2969618716, 961183518, 1081831074, 1037015347, 3289016286, 2331748669, 620887395,
303042654, 3990027945, 1562756376, 3413341792, 2059647769, 2823844432, 674595301, 2457639984, 4076754716,
2447737904, 1583323324, 625627134, 3076006391, 345777990, 1684954145, 879227329, 3436182180, 1522273219,
3802543817, 1456017040, 1897819847, 2970081129, 1382576028, 3820044861, 1044428167, 612252599, 3340478395,
2150613904, 3397625662, 3573635640, 3432275192}};
} // namespace
// 5.3.5.1. Random Number Generator
// The random number generator Rand[y, i, m] is defined as follows,
// where y is a non-negative integer, i is a non-negative integer less
// than 256, and m is a positive integer, and the value produced is an
// integer between 0 and m-1. Let V0, V1, V2, and V3 be the arrays
// provided in Section 5.5.
//
// Let
//
// o x0 = (y + i) mod 2^^8
//
// o x1 = (floor(y / 2^^8) + i) mod 2^^8
//
// o x2 = (floor(y / 2^^16) + i) mod 2^^8
//
// o x3 = (floor(y / 2^^24) + i) mod 2^^8
//
// Then
//
// Rand[y, i, m] = (V0[x0] ^ V1[x1] ^ V2[x2] ^ V3[x3]) % m
uint32 Rfc::random(uint32 y, uint32 i, uint32 m) {
auto x0 = (y + i) % 256;
auto x1 = (y / 256 + i) % 256;
auto x2 = (y / 256 / 256 + i) % 256;
auto x3 = (y / 256 / 256 / 256 + i) % 256;
auto res = V0[x0] ^ V1[x1] ^ V2[x2] ^ V3[x3];
if (m != 0) {
res %= m;
}
return res;
}
namespace {
constexpr std::array<Rfc::RawParameters, 1232> rfc_parameters = {
{{10, 254, 7, 10, 17}, {12, 630, 7, 10, 19}, {18, 682, 11, 10, 29},
{20, 293, 11, 10, 31}, {26, 80, 11, 10, 37}, {30, 566, 11, 10, 41},
{32, 860, 11, 10, 43}, {36, 267, 11, 10, 47}, {42, 822, 11, 10, 53},
{46, 506, 13, 10, 59}, {48, 589, 13, 10, 61}, {49, 87, 13, 10, 61},
{55, 520, 13, 10, 67}, {60, 159, 13, 10, 71}, {62, 235, 13, 10, 73},
{69, 157, 13, 10, 79}, {75, 502, 17, 10, 89}, {84, 334, 17, 10, 97},
{88, 583, 17, 10, 101}, {91, 66, 17, 10, 103}, {95, 352, 17, 10, 107},
{97, 365, 17, 10, 109}, {101, 562, 17, 10, 113}, {114, 5, 19, 10, 127},
{119, 603, 19, 10, 131}, {125, 721, 19, 10, 137}, {127, 28, 19, 10, 139},
{138, 660, 19, 10, 149}, {140, 829, 19, 10, 151}, {149, 900, 23, 10, 163},
{153, 930, 23, 10, 167}, {160, 814, 23, 10, 173}, {166, 661, 23, 10, 179},
{168, 693, 23, 10, 181}, {179, 780, 23, 10, 191}, {181, 605, 23, 10, 193},
{185, 551, 23, 10, 197}, {187, 777, 23, 10, 199}, {200, 491, 23, 10, 211},
{213, 396, 23, 10, 223}, {217, 764, 29, 10, 233}, {225, 843, 29, 10, 241},
{236, 646, 29, 10, 251}, {242, 557, 29, 10, 257}, {248, 608, 29, 10, 263},
{257, 265, 29, 10, 271}, {263, 505, 29, 10, 277}, {269, 722, 29, 10, 283},
{280, 263, 29, 10, 293}, {295, 999, 29, 10, 307}, {301, 874, 29, 10, 313},
{305, 160, 29, 10, 317}, {324, 575, 31, 10, 337}, {337, 210, 31, 10, 349},
{341, 513, 31, 10, 353}, {347, 503, 31, 10, 359}, {355, 558, 31, 10, 367},
{362, 932, 31, 10, 373}, {368, 404, 31, 10, 379}, {372, 520, 37, 10, 389},
{380, 846, 37, 10, 397}, {385, 485, 37, 10, 401}, {393, 728, 37, 10, 409},
{405, 554, 37, 10, 421}, {418, 471, 37, 10, 433}, {428, 641, 37, 10, 443},
{434, 732, 37, 10, 449}, {447, 193, 37, 10, 461}, {453, 934, 37, 10, 467},
{466, 864, 37, 10, 479}, {478, 790, 37, 10, 491}, {486, 912, 37, 10, 499},
{491, 617, 37, 10, 503}, {497, 587, 37, 10, 509}, {511, 800, 37, 10, 523},
{526, 923, 41, 10, 541}, {532, 998, 41, 10, 547}, {542, 92, 41, 10, 557},
{549, 497, 41, 10, 563}, {557, 559, 41, 10, 571}, {563, 667, 41, 10, 577},
{573, 912, 41, 10, 587}, {580, 262, 41, 10, 593}, {588, 152, 41, 10, 601},
{594, 526, 41, 10, 607}, {600, 268, 41, 10, 613}, {606, 212, 41, 10, 619},
{619, 45, 41, 10, 631}, {633, 898, 43, 10, 647}, {640, 527, 43, 10, 653},
{648, 558, 43, 10, 661}, {666, 460, 47, 10, 683}, {675, 5, 47, 10, 691},
{685, 895, 47, 10, 701}, {693, 996, 47, 10, 709}, {703, 282, 47, 10, 719},
{718, 513, 47, 10, 733}, {728, 865, 47, 10, 743}, {736, 870, 47, 10, 751},
{747, 239, 47, 10, 761}, {759, 452, 47, 10, 773}, {778, 862, 53, 10, 797},
{792, 852, 53, 10, 811}, {802, 643, 53, 10, 821}, {811, 543, 53, 10, 829},
{821, 447, 53, 10, 839}, {835, 321, 53, 10, 853}, {845, 287, 53, 10, 863},
{860, 12, 53, 10, 877}, {870, 251, 53, 10, 887}, {891, 30, 53, 10, 907},
{903, 621, 53, 10, 919}, {913, 555, 53, 10, 929}, {926, 127, 53, 10, 941},
{938, 400, 53, 10, 953}, {950, 91, 59, 10, 971}, {963, 916, 59, 10, 983},
{977, 935, 59, 10, 997}, {989, 691, 59, 10, 1009}, {1002, 299, 59, 10, 1021},
{1020, 282, 59, 10, 1039}, {1032, 824, 59, 10, 1051}, {1050, 536, 59, 11, 1069},
{1074, 596, 59, 11, 1093}, {1085, 28, 59, 11, 1103}, {1099, 947, 59, 11, 1117},
{1111, 162, 59, 11, 1129}, {1136, 536, 59, 11, 1153}, {1152, 1000, 61, 11, 1171},
{1169, 251, 61, 11, 1187}, {1183, 673, 61, 11, 1201}, {1205, 559, 61, 11, 1223},
{1220, 923, 61, 11, 1237}, {1236, 81, 67, 11, 1259}, {1255, 478, 67, 11, 1277},
{1269, 198, 67, 11, 1291}, {1285, 137, 67, 11, 1307}, {1306, 75, 67, 11, 1327},
{1347, 29, 67, 11, 1367}, {1361, 231, 67, 11, 1381}, {1389, 532, 67, 11, 1409},
{1404, 58, 67, 11, 1423}, {1420, 60, 67, 11, 1439}, {1436, 964, 71, 11, 1459},
{1461, 624, 71, 11, 1483}, {1477, 502, 71, 11, 1499}, {1502, 636, 71, 11, 1523},
{1522, 986, 71, 11, 1543}, {1539, 950, 71, 11, 1559}, {1561, 735, 73, 11, 1583},
{1579, 866, 73, 11, 1601}, {1600, 203, 73, 11, 1621}, {1616, 83, 73, 11, 1637},
{1649, 14, 73, 11, 1669}, {1673, 522, 79, 11, 1699}, {1698, 226, 79, 11, 1723},
{1716, 282, 79, 11, 1741}, {1734, 88, 79, 11, 1759}, {1759, 636, 79, 11, 1783},
{1777, 860, 79, 11, 1801}, {1800, 324, 79, 11, 1823}, {1824, 424, 79, 11, 1847},
{1844, 999, 79, 11, 1867}, {1863, 682, 83, 11, 1889}, {1887, 814, 83, 11, 1913},
{1906, 979, 83, 11, 1931}, {1926, 538, 83, 11, 1951}, {1954, 278, 83, 11, 1979},
{1979, 580, 83, 11, 2003}, {2005, 773, 83, 11, 2029}, {2040, 911, 89, 11, 2069},
{2070, 506, 89, 11, 2099}, {2103, 628, 89, 11, 2131}, {2125, 282, 89, 11, 2153},
{2152, 309, 89, 11, 2179}, {2195, 858, 89, 11, 2221}, {2217, 442, 89, 11, 2243},
{2247, 654, 89, 11, 2273}, {2278, 82, 97, 11, 2311}, {2315, 428, 97, 11, 2347},
{2339, 442, 97, 11, 2371}, {2367, 283, 97, 11, 2399}, {2392, 538, 97, 11, 2423},
{2416, 189, 97, 11, 2447}, {2447, 438, 97, 11, 2477}, {2473, 912, 97, 11, 2503},
{2502, 1, 97, 11, 2531}, {2528, 167, 97, 11, 2557}, {2565, 272, 97, 11, 2593},
{2601, 209, 101, 11, 2633}, {2640, 927, 101, 11, 2671}, {2668, 386, 101, 11, 2699},
{2701, 653, 101, 11, 2731}, {2737, 669, 101, 11, 2767}, {2772, 431, 101, 11, 2801},
{2802, 793, 103, 11, 2833}, {2831, 588, 103, 11, 2861}, {2875, 777, 107, 11, 2909},
{2906, 939, 107, 11, 2939}, {2938, 864, 107, 11, 2971}, {2979, 627, 107, 11, 3011},
{3015, 265, 109, 11, 3049}, {3056, 976, 109, 11, 3089}, {3101, 988, 113, 11, 3137},
{3151, 507, 113, 11, 3187}, {3186, 640, 113, 11, 3221}, {3224, 15, 113, 11, 3259},
{3265, 667, 113, 11, 3299}, {3299, 24, 127, 11, 3347}, {3344, 877, 127, 11, 3391},
{3387, 240, 127, 11, 3433}, {3423, 720, 127, 11, 3469}, {3466, 93, 127, 11, 3511},
{3502, 919, 127, 11, 3547}, {3539, 635, 127, 11, 3583}, {3579, 174, 127, 11, 3623},
{3616, 647, 127, 11, 3659}, {3658, 820, 127, 11, 3701}, {3697, 56, 127, 11, 3739},
{3751, 485, 127, 11, 3793}, {3792, 210, 127, 11, 3833}, {3840, 124, 127, 11, 3881},
{3883, 546, 127, 11, 3923}, {3924, 954, 131, 11, 3967}, {3970, 262, 131, 11, 4013},
{4015, 927, 131, 11, 4057}, {4069, 957, 131, 11, 4111}, {4112, 726, 137, 11, 4159},
{4165, 583, 137, 11, 4211}, {4207, 782, 137, 11, 4253}, {4252, 37, 137, 11, 4297},
{4318, 758, 137, 11, 4363}, {4365, 777, 137, 11, 4409}, {4418, 104, 139, 11, 4463},
{4468, 476, 139, 11, 4513}, {4513, 113, 149, 11, 4567}, {4567, 313, 149, 11, 4621},
{4626, 102, 149, 11, 4679}, {4681, 501, 149, 11, 4733}, {4731, 332, 149, 11, 4783},
{4780, 786, 149, 11, 4831}, {4838, 99, 149, 11, 4889}, {4901, 658, 149, 11, 4951},
{4954, 794, 149, 11, 5003}, {5008, 37, 151, 11, 5059}, {5063, 471, 151, 11, 5113},
{5116, 94, 157, 11, 5171}, {5172, 873, 157, 11, 5227}, {5225, 918, 157, 11, 5279},
{5279, 945, 157, 11, 5333}, {5334, 211, 157, 11, 5387}, {5391, 341, 157, 11, 5443},
{5449, 11, 163, 11, 5507}, {5506, 578, 163, 11, 5563}, {5566, 494, 163, 11, 5623},
{5637, 694, 163, 11, 5693}, {5694, 252, 163, 11, 5749}, {5763, 451, 167, 11, 5821},
{5823, 83, 167, 11, 5881}, {5896, 689, 167, 11, 5953}, {5975, 488, 173, 11, 6037},
{6039, 214, 173, 11, 6101}, {6102, 17, 173, 11, 6163}, {6169, 469, 173, 11, 6229},
{6233, 263, 179, 11, 6299}, {6296, 309, 179, 11, 6361}, {6363, 984, 179, 11, 6427},
{6427, 123, 179, 11, 6491}, {6518, 360, 179, 11, 6581}, {6589, 863, 181, 11, 6653},
{6655, 122, 181, 11, 6719}, {6730, 522, 191, 11, 6803}, {6799, 539, 191, 11, 6871},
{6878, 181, 191, 11, 6949}, {6956, 64, 191, 11, 7027}, {7033, 387, 191, 11, 7103},
{7108, 967, 191, 11, 7177}, {7185, 843, 191, 11, 7253}, {7281, 999, 193, 11, 7351},
{7360, 76, 197, 11, 7433}, {7445, 142, 197, 11, 7517}, {7520, 599, 197, 11, 7591},
{7596, 576, 199, 11, 7669}, {7675, 176, 211, 11, 7759}, {7770, 392, 211, 11, 7853},
{7855, 332, 211, 11, 7937}, {7935, 291, 211, 11, 8017}, {8030, 913, 211, 11, 8111},
{8111, 608, 211, 11, 8191}, {8194, 212, 211, 11, 8273}, {8290, 696, 211, 11, 8369},
{8377, 931, 223, 11, 8467}, {8474, 326, 223, 11, 8563}, {8559, 228, 223, 11, 8647},
{8654, 706, 223, 11, 8741}, {8744, 144, 223, 11, 8831}, {8837, 83, 223, 11, 8923},
{8928, 743, 223, 11, 9013}, {9019, 187, 223, 11, 9103}, {9111, 654, 227, 11, 9199},
{9206, 359, 227, 11, 9293}, {9303, 493, 229, 11, 9391}, {9400, 369, 233, 11, 9491},
{9497, 981, 233, 11, 9587}, {9601, 276, 239, 11, 9697}, {9708, 647, 239, 11, 9803},
{9813, 389, 239, 11, 9907}, {9916, 80, 239, 11, 10009}, {10017, 396, 241, 11, 10111},
{10120, 580, 251, 11, 10223}, {10241, 873, 251, 11, 10343}, {10351, 15, 251, 11, 10453},
{10458, 976, 251, 11, 10559}, {10567, 584, 251, 11, 10667}, {10676, 267, 257, 11, 10781},
{10787, 876, 257, 11, 10891}, {10899, 642, 257, 12, 11003}, {11015, 794, 257, 12, 11119},
{11130, 78, 263, 12, 11239}, {11245, 736, 263, 12, 11353}, {11358, 882, 269, 12, 11471},
{11475, 251, 269, 12, 11587}, {11590, 434, 269, 12, 11701}, {11711, 204, 269, 12, 11821},
{11829, 256, 271, 12, 11941}, {11956, 106, 277, 12, 12073}, {12087, 375, 277, 12, 12203},
{12208, 148, 277, 12, 12323}, {12333, 496, 281, 12, 12451}, {12460, 88, 281, 12, 12577},
{12593, 826, 293, 12, 12721}, {12726, 71, 293, 12, 12853}, {12857, 925, 293, 12, 12983},
{13002, 760, 293, 12, 13127}, {13143, 130, 293, 12, 13267}, {13284, 641, 307, 12, 13421},
{13417, 400, 307, 12, 13553}, {13558, 480, 307, 12, 13693}, {13695, 76, 307, 12, 13829},
{13833, 665, 307, 12, 13967}, {13974, 910, 307, 12, 14107}, {14115, 467, 311, 12, 14251},
{14272, 964, 311, 12, 14407}, {14415, 625, 313, 12, 14551}, {14560, 362, 317, 12, 14699},
{14713, 759, 317, 12, 14851}, {14862, 728, 331, 12, 15013}, {15011, 343, 331, 12, 15161},
{15170, 113, 331, 12, 15319}, {15325, 137, 331, 12, 15473}, {15496, 308, 331, 12, 15643},
{15651, 800, 337, 12, 15803}, {15808, 177, 337, 12, 15959}, {15977, 961, 337, 12, 16127},
{16161, 958, 347, 12, 16319}, {16336, 72, 347, 12, 16493}, {16505, 732, 347, 12, 16661},
{16674, 145, 349, 12, 16831}, {16851, 577, 353, 12, 17011}, {17024, 305, 353, 12, 17183},
{17195, 50, 359, 12, 17359}, {17376, 351, 359, 12, 17539}, {17559, 175, 367, 12, 17729},
{17742, 727, 367, 12, 17911}, {17929, 902, 367, 12, 18097}, {18116, 409, 373, 12, 18289},
{18309, 776, 373, 12, 18481}, {18503, 586, 379, 12, 18679}, {18694, 451, 379, 12, 18869},
{18909, 287, 383, 12, 19087}, {19126, 246, 389, 12, 19309}, {19325, 222, 389, 12, 19507},
{19539, 563, 397, 12, 19727}, {19740, 839, 397, 12, 19927}, {19939, 897, 401, 12, 20129},
{20152, 409, 401, 12, 20341}, {20355, 618, 409, 12, 20551}, {20564, 439, 409, 12, 20759},
{20778, 95, 419, 13, 20983}, {20988, 448, 419, 13, 21191}, {21199, 133, 419, 13, 21401},
{21412, 938, 419, 13, 21613}, {21629, 423, 431, 13, 21841}, {21852, 90, 431, 13, 22063},
{22073, 640, 431, 13, 22283}, {22301, 922, 433, 13, 22511}, {22536, 250, 439, 13, 22751},
{22779, 367, 439, 13, 22993}, {23010, 447, 443, 13, 23227}, {23252, 559, 449, 13, 23473},
{23491, 121, 457, 13, 23719}, {23730, 623, 457, 13, 23957}, {23971, 450, 457, 13, 24197},
{24215, 253, 461, 13, 24443}, {24476, 106, 467, 13, 24709}, {24721, 863, 467, 13, 24953},
{24976, 148, 479, 13, 25219}, {25230, 427, 479, 13, 25471}, {25493, 138, 479, 13, 25733},
{25756, 794, 487, 13, 26003}, {26022, 247, 487, 13, 26267}, {26291, 562, 491, 13, 26539},
{26566, 53, 499, 13, 26821}, {26838, 135, 499, 13, 27091}, {27111, 21, 503, 13, 27367},
{27392, 201, 509, 13, 27653}, {27682, 169, 521, 13, 27953}, {27959, 70, 521, 13, 28229},
{28248, 386, 521, 13, 28517}, {28548, 226, 523, 13, 28817}, {28845, 3, 541, 13, 29131},
{29138, 769, 541, 13, 29423}, {29434, 590, 541, 13, 29717}, {29731, 672, 541, 13, 30013},
{30037, 713, 547, 13, 30323}, {30346, 967, 547, 13, 30631}, {30654, 368, 557, 14, 30949},
{30974, 348, 557, 14, 31267}, {31285, 119, 563, 14, 31583}, {31605, 503, 569, 14, 31907},
{31948, 181, 571, 14, 32251}, {32272, 394, 577, 14, 32579}, {32601, 189, 587, 14, 32917},
{32932, 210, 587, 14, 33247}, {33282, 62, 593, 14, 33601}, {33623, 273, 593, 14, 33941},
{33961, 554, 599, 14, 34283}, {34302, 936, 607, 14, 34631}, {34654, 483, 607, 14, 34981},
{35031, 397, 613, 14, 35363}, {35395, 241, 619, 14, 35731}, {35750, 500, 631, 14, 36097},
{36112, 12, 631, 14, 36457}, {36479, 958, 641, 14, 36833}, {36849, 524, 641, 14, 37201},
{37227, 8, 643, 14, 37579}, {37606, 100, 653, 14, 37967}, {37992, 339, 653, 14, 38351},
{38385, 804, 659, 14, 38749}, {38787, 510, 673, 14, 39163}, {39176, 18, 673, 14, 39551},
{39576, 412, 677, 14, 39953}, {39980, 394, 683, 14, 40361}, {40398, 830, 691, 15, 40787},
{40816, 535, 701, 15, 41213}, {41226, 199, 701, 15, 41621}, {41641, 27, 709, 15, 42043},
{42067, 298, 709, 15, 42467}, {42490, 368, 719, 15, 42899}, {42916, 755, 727, 15, 43331},
{43388, 379, 727, 15, 43801}, {43840, 73, 733, 15, 44257}, {44279, 387, 739, 15, 44701},
{44729, 457, 751, 15, 45161}, {45183, 761, 751, 15, 45613}, {45638, 855, 757, 15, 46073},
{46104, 370, 769, 15, 46549}, {46574, 261, 769, 15, 47017}, {47047, 299, 787, 15, 47507},
{47523, 920, 787, 15, 47981}, {48007, 269, 787, 15, 48463}, {48489, 862, 797, 15, 48953},
{48976, 349, 809, 15, 49451}, {49470, 103, 809, 15, 49943}, {49978, 115, 821, 15, 50461},
{50511, 93, 821, 16, 50993}, {51017, 982, 827, 16, 51503}, {51530, 432, 839, 16, 52027},
{52062, 340, 853, 16, 52571}, {52586, 173, 853, 16, 53093}, {53114, 421, 857, 16, 53623},
{53650, 330, 863, 16, 54163}, {54188, 624, 877, 16, 54713}, {54735, 233, 877, 16, 55259},
{55289, 362, 883, 16, 55817}, {55843, 963, 907, 16, 56393}, {56403, 471, 907, 16, 56951}}};
} // namespace
Result<Rfc::Parameters> Rfc::get_parameters(size_t K) {
for (auto &p : rfc_parameters) {
if (p.K_padded >= K) {
return Rfc::Parameters(narrow_cast<uint32>(K), p);
}
}
return Status::Error("K is too big");
}
Rfc::Parameters::Parameters(uint32 K, RawParameters raw_parameters) {
this->K = K;
K_padded = raw_parameters.K_padded;
J = raw_parameters.J;
S = raw_parameters.S;
H = raw_parameters.H;
W = raw_parameters.W;
L = K_padded + S + H;
P = L - W;
U = P - H;
B = W - S;
P1 = P + 1;
auto is_prime = [](uint32 n) {
if (n <= 3)
return true;
if (n % 2 == 0 || n % 3 == 0)
return false;
uint32 i = 5;
uint32 w = 2;
while (i * i <= n) {
if (n % i == 0)
return false;
i += w;
w = 6 - w;
}
return true;
};
while (!is_prime(P1)) {
P1++;
}
}
uint32 Rfc::Parameters::get_degree(uint32 v) const {
static const std::array<uint32, 31> degree_distribution = {
{0, 5243, 529531, 704294, 791675, 844104, 879057, 904023, 922747, 937311, 948962,
958494, 966438, 973160, 978921, 983914, 988283, 992138, 995565, 998631, 1001391, 1003887,
1006157, 1008229, 1010129, 1011876, 1013490, 1014983, 1016370, 1017662, 1048576}};
for (uint32 i = 0; i < degree_distribution.size(); i++) {
if (v < degree_distribution[i]) {
return std::min(W - 2, i);
}
}
UNREACHABLE();
return 0;
}
Rfc::EncodingRow Rfc::Parameters::get_encoding_row(uint32 X /*ISI*/) const {
uint32 A = 53591 + J * 997;
if (A % 2 == 0) {
A++;
}
uint32 B_local = 10267 * (J + 1);
uint32 y = B_local + X * A;
uint32 v = random(y, 0, 1 << 20);
uint32 d = get_degree(v);
uint32 a = 1 + random(y, 1, W - 1);
uint32 b = random(y, 2, W);
uint32 d1;
if (d < 4) {
d1 = 2 + random(X, 3, 2);
} else {
d1 = 2;
}
uint32 a1 = 1 + random(X, 4, P1 - 1);
uint32 b1 = random(X, 5, P1);
EncodingRow r;
std::tie(r.d, r.a, r.b, r.d1, r.a1, r.b1) = std::tie(d, a, b, d1, a1, b1);
return r;
}
uint32 Rfc::Parameters::encoding_row_size(const EncodingRow &t) const {
return t.d + t.d1;
}
} // namespace raptorq
} // namespace td

243
tdfec/td/fec/raptorq/Rfc.h Normal file
View file

@ -0,0 +1,243 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/Status.h"
#include "td/fec/algebra/MatrixGF256.h"
#include "td/fec/algebra/SparseMatrixGF2.h"
namespace td {
namespace raptorq {
class Rfc {
public:
struct RawParameters {
uint32 K_padded;
uint32 J;
uint32 S;
uint32 H;
uint32 W;
};
struct EncodingRow {
uint32 d; // [1,30] LT degree
uint32 a; // [0,W)
uint32 b; // [0,W)
uint32 d1; // [2,3] PI degree
uint32 a1; // [0,P1)
uint32 b1; // [0,P1)
};
struct Parameters {
uint32 K;
uint32 K_padded;
uint32 J;
uint32 S;
uint32 H;
uint32 W;
uint32 L;
uint32 P;
uint32 P1;
uint32 U;
uint32 B;
EncodingRow get_encoding_row(uint32 X /*ISI*/) const;
uint32 encoding_row_size(const EncodingRow &t) const;
template <class F>
void encoding_row_for_each(EncodingRow t, F &&f) const {
f(t.b);
for (uint16 j = 1; j < t.d; ++j) {
t.b = (t.b + t.a) % W;
f(t.b);
}
while (t.b1 >= P)
t.b1 = (t.b1 + t.a1) % P1;
f(W + t.b1);
for (uint16 j = 1; j < t.d1; ++j) {
t.b1 = (t.b1 + t.a1) % P1;
while (t.b1 >= P)
t.b1 = (t.b1 + t.a1) % P1;
f(W + t.b1);
}
}
Parameters(uint32 K, RawParameters raw_parameters);
class LDPC1 {
public:
LDPC1(uint32 S, uint32 B) : S(S), B(B) {
}
static void sort_inplace(uint32 &a, uint32 &b, uint32 &c) {
if (a > c) {
std::swap(a, c);
}
if (b > c) {
std::swap(b, c);
}
if (a > b) {
std::swap(a, b);
}
DCHECK(a < b && b < c);
}
template <class F>
void generate(F &&f) const {
for (uint32 col = 0; col < B; col++) {
uint32 i = col / S;
uint32 shift = col % S;
uint32 a = shift;
uint32 b = (i + 1 + shift) % S;
uint32 c = (2 * (i + 1) + shift) % S;
DCHECK(a != b);
DCHECK(a != c);
DCHECK(b != c);
sort_inplace(a, b, c);
f(a, col);
f(b, col);
f(c, col);
}
}
uint32 non_zeroes() const {
return B * 3;
}
uint32 cols() const {
return B;
}
uint32 rows() const {
return S;
}
private:
uint32 S;
uint32 B;
};
// 1100000
// 0110000
// 0011000
// .......
// 1100000
class LDPC2 {
public:
LDPC2(uint32 rows, uint32 cols) : rows_(rows), cols_(cols) {
}
template <class F>
void generate(F &&f) const {
for (uint32 row = 0; row < rows_; row++) {
f(row, row % cols_);
f(row, (row + 1) % cols_);
}
}
uint32 non_zeroes() const {
return rows_ * 2;
}
uint32 cols() const {
return cols_;
}
uint32 rows() const {
return rows_;
}
private:
uint32 rows_;
uint32 cols_;
};
static MatrixGF256 HDPC_multiply(const uint32 rows, MatrixGF256 v) {
Octet alpha = Octet(Octet::oct_exp(1));
for (uint32 i = 1; i < v.rows(); i++) {
v.row_add_mul(i, i - 1, alpha);
}
MatrixGF256 u(rows, v.cols());
u.set_zero();
for (uint32 i = 0; i < rows; i++) {
u.row_add_mul(i, v.row(narrow_cast<uint32>(v.rows() - 1)), Octet(Octet::oct_exp(i % 255)));
}
for (uint32 col = 0; col + 1 < v.rows(); col++) {
auto a = Rfc::random(col + 1, 6, rows);
auto b = (a + Rfc::random(col + 1, 7, rows - 1) + 1) % rows;
u.row_add(a, v.row(col));
u.row_add(b, v.row(col));
}
return u;
}
class ENC {
public:
ENC(const Rfc::Parameters &p, Span<Rfc::EncodingRow> encoding_rows) : p(p), encoding_rows(encoding_rows) {
}
template <class F>
void generate(F &&f) const {
uint32 row = 0;
for (auto encoding_row : encoding_rows) {
p.encoding_row_for_each(encoding_row, [&f, row](auto col) { f(row, col); });
row++;
}
}
uint32 non_zeroes() const {
uint32 res = 0;
for (auto encoding_row : encoding_rows) {
res += p.encoding_row_size(encoding_row);
}
return res;
}
uint32 cols() const {
return p.L;
}
uint32 rows() const {
return narrow_cast<uint32>(encoding_rows.size());
}
private:
const Rfc::Parameters &p;
Span<Rfc::EncodingRow> encoding_rows;
};
LDPC1 get_LDPC1() const {
return LDPC1(S, B);
}
LDPC2 get_LDPC2() const {
return LDPC2(S, P);
}
ENC get_ENC(Span<EncodingRow> encoding_rows) const {
return ENC(*this, encoding_rows);
}
MatrixGF256 HDPC_multiply(MatrixGF256 v) const {
return HDPC_multiply(H, std::move(v));
}
SparseMatrixGF2 get_A_upper(Span<EncodingRow> encoding_rows) const {
return SparseMatrixGF2(block_generator(narrow_cast<uint32>(S + encoding_rows.size()), L, get_LDPC1(),
IdentityGenerator(S), get_LDPC2(), get_ENC(encoding_rows)));
}
private:
uint32 get_degree(uint32 v) const;
};
static uint32 random(uint32 y, uint32 i, uint32 m);
static Result<Parameters> get_parameters(size_t K);
};
} // namespace raptorq
} // namespace td

244
tdfec/td/fec/raptorq/Solver.cpp Normal file
View file

@ -0,0 +1,244 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/fec/raptorq/Solver.h"
#include "td/fec/algebra/GaussianElimination.h"
#include "td/fec/algebra/InactivationDecoding.h"
#include "td/utils/Timer.h"
#include <map>
namespace td {
namespace raptorq {
MatrixGF256 create_D(const Rfc::Parameters &p, Span<SymbolRef> symbols) {
auto symbol_size = symbols[0].data.size();
MatrixGF256 D(p.S + p.H + symbols.size(), symbol_size);
D.set_zero();
auto offset = p.S;
for (auto &symbol : symbols) {
for (size_t i = 0; i < symbol_size; i++) {
D.set(offset, i, Octet(symbol.data[i]));
}
offset++;
}
return D;
}
Result<MatrixGF256> Solver::run(const Rfc::Parameters &p, Span<SymbolRef> symbols) {
if (0) { // turns out gauss is slower even for small symbols count
auto encoding_rows = transform(symbols, [&p](auto &symbol) { return p.get_encoding_row(symbol.id); });
MatrixGF256 A(p.S + p.H + symbols.size(), p.L);
A.set_zero();
auto A_upper = p.get_A_upper(encoding_rows);
A_upper.block_for_each(0, 0, A_upper.rows(), A_upper.cols(), [&](auto x, auto y) { A.set(x, y, Octet(1)); });
MatrixGF256 tmp(A.cols() - p.H, A.cols() - p.H);
tmp.set_zero();
for (size_t i = 0; i < tmp.cols(); i++) {
tmp.set(i, i, Octet(1));
}
auto HDCP = p.HDPC_multiply(std::move(tmp));
MatrixGF256 HDCP2(p.H, p.L - p.H);
MatrixGF256 IH(p.H, p.H);
IH.set_zero();
for (size_t i = 0; i < p.H; i++) {
IH.set(i, i, Octet(1));
}
A.set_from(HDCP, A_upper.rows(), 0);
A.set_from(IH, A_upper.rows(), HDCP.cols());
auto D = create_D(p, symbols);
auto C = GaussianElimination::run(std::move(A), std::move(D));
return C;
}
PerfWarningTimer x("solve");
Timer timer;
auto perf_log = [&](Slice message) {
if (GET_VERBOSITY_LEVEL() > VERBOSITY_NAME(DEBUG)) {
static std::map<std::string, double> total;
static double total_all = 0;
auto elapsed = timer.elapsed();
auto current_total = total[message.str()] += elapsed;
total_all += elapsed;
LOG(DEBUG) << "PERF: " << message << " " << timer << " " << current_total / total_all * 100;
timer = {};
}
};
// Solve linear system
// A * C = D
// C - intermeidate symbols
// D - encoded symbols and restriction symbols.
//
// A:
// +--------+-----+-------+
// | LDPC1 | I_S | LDPC2|
// +--------+-----+-------+
// | ENC |
// +---------------+------+
// | HDCP | I_H |
// +---------------+------+
CHECK(p.K_padded <= symbols.size());
auto encoding_rows = transform(symbols, [&p](auto &symbol) { return p.get_encoding_row(symbol.id); });
// Generate matrix A_upper: sparse part of A, first S + K_padded rows.
SparseMatrixGF2 A_upper = p.get_A_upper(encoding_rows);
auto D = create_D(p, symbols);
perf_log("Generate sparse matrix");
// Run indactivation decoding.
// Lets call resulting lower triangualr matrix U
auto decoding_result = InactivationDecoding(A_upper, p.P).run();
perf_log("Inactivation decoding");
uint32 U_size = decoding_result.size;
auto row_permutation = std::move(decoding_result.p_rows);
while (row_permutation.size() < D.rows()) {
row_permutation.push_back(narrow_cast<uint32>(row_permutation.size()));
}
auto col_permutation = std::move(decoding_result.p_cols);
// +--------+---------+ +---------+
// | U | E | | D_upper |
// +--------+---------+ +---------+
// | G_left | G_right | * C = | |
// +--------+--+------+ | D_lower |
// |HDCP | I_H | | |
// +-----------+------+ +---------+
D = D.apply_row_permutation(row_permutation);
perf_log("D: apply permutation");
A_upper = A_upper.apply_row_permutation(row_permutation).apply_col_permutation(col_permutation);
perf_log("A_upper: apply permutation");
auto E = A_upper.block_dense(0, U_size, U_size, p.L - U_size);
perf_log("Calc E");
MatrixGF256 C(A_upper.cols(), D.cols());
C.set_from(D.block_view(0, 0, U_size, D.cols()), 0, 0);
// Make U Identity matrix and calculate E and D_upper.
for (uint32 i = 0; i < U_size; i++) {
for (auto row : A_upper.col(i)) {
if (row == i) {
continue;
}
if (row >= U_size) {
break;
}
E.row_add(row, i);
D.row_add(row, i); // this is SLOW
}
}
perf_log("Triangular -> Identity");
auto HDPC_left_multiply = [&](const MatrixGF256 &m) {
MatrixGF256 T(p.K_padded + p.S, m.cols());
T.set_zero();
for (uint32 i = 0; i < m.rows(); i++) {
T.row_set(col_permutation[i], m.row(i));
}
return p.HDPC_multiply(std::move(T));
};
SparseMatrixGF2 G_left = A_upper.block_sparse(U_size, 0, A_upper.rows() - U_size, U_size);
perf_log("G_left");
// Calculate small_A_upper
// small_A_upper = G_right
MatrixGF256 small_A_upper(A_upper.rows() - U_size, A_upper.cols() - U_size);
A_upper.block_for_each(U_size, U_size, A_upper.rows() - U_size, A_upper.cols() - U_size,
[&](auto row, auto col) { small_A_upper.set(row, col, Octet(1)); });
// small_A_upper += G_left * E
small_A_upper.add((G_left * E).to_gf256());
perf_log("small_A_upper");
// Calculate small_A_lower
MatrixGF256 small_A_lower(p.H, A_upper.cols() - U_size);
small_A_lower.set_zero();
for (uint32 i = 1; i <= p.H; i++) {
small_A_lower.set(small_A_lower.rows() - i, small_A_lower.cols() - i, Octet(1));
}
// Calculate HDPC_right and set it into small_A_lower
MatrixGF256 T(p.K_padded + p.S, p.K_padded + p.S - U_size);
T.set_zero();
for (uint32 i = 0; i < T.cols(); i++) {
T.set(col_permutation[i + T.rows() - T.cols()], i, Octet(1));
}
MatrixGF256 HDCP_right = p.HDPC_multiply(std::move(T));
small_A_lower.set_from(HDCP_right, 0, 0);
perf_log("small_A_lower");
// small_A_lower += HDPC_left * E
auto t = E.to_gf256();
perf_log("t");
small_A_lower.add(HDPC_left_multiply(std::move(t)));
perf_log("small_A_lower += HDPC_left * E");
MatrixGF256 D_upper(U_size, D.cols());
D_upper.set_from(D.block_view(0, 0, D_upper.rows(), D_upper.cols()), 0, 0);
// small_D_upper
MatrixGF256 small_D_upper(A_upper.rows() - U_size, D.cols());
small_D_upper.set_from(D.block_view(U_size, 0, small_D_upper.rows(), small_D_upper.cols()), 0, 0);
small_D_upper.add(G_left * D_upper);
perf_log("small_D_upper");
// small_D_lower
MatrixGF256 small_D_lower(p.H, D.cols());
small_D_lower.set_from(D.block_view(A_upper.rows(), 0, small_D_lower.rows(), small_D_lower.cols()), 0, 0);
perf_log("small_D_lower");
small_D_lower.add(HDPC_left_multiply(D_upper));
perf_log("small_D_lower += HDPC_left * D_upper");
// Combine small_A from small_A_lower and small_A_upper
MatrixGF256 small_A(small_A_upper.rows() + small_A_lower.rows(), small_A_upper.cols());
small_A.set_from(small_A_upper, 0, 0);
small_A.set_from(small_A_lower, small_A_upper.rows(), 0);
// Combine small_D from small_D_lower and small_D_upper
MatrixGF256 small_D(small_D_upper.rows() + small_D_lower.rows(), small_D_upper.cols());
small_D.set_from(small_D_upper, 0, 0);
small_D.set_from(small_D_lower, small_D_upper.rows(), 0);
TRY_RESULT(small_C, GaussianElimination::run(std::move(small_A), std::move(small_D)));
perf_log("gauss");
C.set_from(small_C.block_view(0, 0, C.rows() - U_size, C.cols()), U_size, 0);
SparseMatrixGF2 A_upper_t = A_upper.transpose();
for (uint32 row = 0; row < U_size; row++) {
for (auto col : A_upper_t.col(row)) {
if (col == row) {
continue;
}
C.row_add(row, col);
}
}
perf_log("Calc result");
auto res = C.apply_row_permutation(inverse_permutation(col_permutation));
perf_log("Apply permutation");
return std::move(res);
}
} // namespace raptorq
} // namespace td

35
tdfec/td/fec/raptorq/Solver.h Normal file
View file

@ -0,0 +1,35 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/Slice.h"
#include "td/utils/Span.h"
#include "td/fec/raptorq/Rfc.h"
#include "td/fec/common/SymbolRef.h"
namespace td {
namespace raptorq {
class Solver {
public:
static Result<MatrixGF256> run(const Rfc::Parameters &p, Span<SymbolRef> symbols);
};
} // namespace raptorq
} // namespace td