external/ton
1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-03-09 15:40:10 +00:00
Code Issues Releases Wiki Activity

initial commit

Browse source
This commit is contained in:
initial commit 2019-09-07 14:03:22 +04:00 • committed by vvaltman
commit c2da007f40
1610 changed files with 398047 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

12
tdutils/.gitrepo Normal file
View file

@ -0,0 +1,12 @@
; DO NOT EDIT (unless you know what you are doing)
;
; This subdirectory is a git "subrepo", and this file is maintained by the
; git-subrepo command. See https://github.com/git-commands/git-subrepo#readme
;
[subrepo]
remote = git@github.com:arseny30/tdutils.git
branch = master
commit = cebe6b0ee08089006ff9ed7d3c4ec38be23b22fa
parent = ce4bbf3c37dcc2fdce25ba07d3d8432ce2d77d85
cmdver = 0.4.0
method = merge

330
tdutils/CMakeLists.txt Normal file
View file

@ -0,0 +1,330 @@
cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR)
if (WIN32)
if (WINGETOPT_FOUND)
set(TD_HAVE_GETOPT 1)
endif()
else()
set(TD_HAVE_GETOPT 1)
endif()
if (NOT ZLIB_FOUND)
find_package(ZLIB)
endif()
if (ZLIB_FOUND)
set(TD_HAVE_ZLIB 1)
message(STATUS "Found ZLIB: ${ZLIB_INCLUDE_DIR} ${ZLIB_LIBRARIES}")
# OpenSSL internally depends on zlib
if (NOT OPENSSL_FOUND)
find_package(OpenSSL)
endif()
if (OPENSSL_FOUND)
set(TD_HAVE_OPENSSL 1)
endif()
endif()
if (CRC32C_FOUND)
set(TD_HAVE_CRC32C 1)
endif()
if (ABSL_FOUND)
set(TD_HAVE_ABSL 1)
endif()
configure_file(td/utils/config.h.in td/utils/config.h @ONLY)
add_subdirectory(generate)
# TDUTILS
set_source_files_properties(${TDMIME_AUTO} PROPERTIES GENERATED TRUE)
if (CLANG OR GCC)
set_property(SOURCE ${TDMIME_AUTO} APPEND_STRING PROPERTY COMPILE_FLAGS " -Wno-conversion")
elseif (MSVC)
set_property(SOURCE ${TDMIME_AUTO} APPEND_STRING PROPERTY COMPILE_FLAGS " /wd4267")
endif()
if (CLANG)
set_property(SOURCE ${TDMIME_AUTO} APPEND_STRING PROPERTY COMPILE_FLAGS " -Wno-deprecated-register")
endif()
set(TDUTILS_SOURCE
td/utils/port/Clocks.cpp
td/utils/port/FileFd.cpp
td/utils/port/IPAddress.cpp
td/utils/port/MemoryMapping.cpp
td/utils/port/path.cpp
td/utils/port/PollFlags.cpp
td/utils/port/ServerSocketFd.cpp
td/utils/port/signals.cpp
td/utils/port/sleep.cpp
td/utils/port/SocketFd.cpp
td/utils/port/stacktrace.cpp
td/utils/port/Stat.cpp
td/utils/port/StdStreams.cpp
td/utils/port/thread_local.cpp
td/utils/port/user.cpp
td/utils/port/UdpSocketFd.cpp
td/utils/port/wstring_convert.cpp
td/utils/port/detail/Epoll.cpp
td/utils/port/detail/EventFdBsd.cpp
td/utils/port/detail/EventFdLinux.cpp
td/utils/port/detail/EventFdWindows.cpp
td/utils/port/detail/Iocp.cpp
td/utils/port/detail/KQueue.cpp
td/utils/port/detail/NativeFd.cpp
td/utils/port/detail/Poll.cpp
td/utils/port/detail/Select.cpp
td/utils/port/detail/ThreadIdGuard.cpp
td/utils/port/detail/WineventPoll.cpp
${TDMIME_AUTO}
td/utils/base64.cpp
td/utils/BigNum.cpp
td/utils/buffer.cpp
td/utils/BufferedUdp.cpp
td/utils/check.cpp
td/utils/crypto.cpp
td/utils/FileLog.cpp
td/utils/filesystem.cpp
td/utils/find_boundary.cpp
td/utils/Gzip.cpp
td/utils/GzipByteFlow.cpp
td/utils/Hints.cpp
td/utils/HttpUrl.cpp
td/utils/JsonBuilder.cpp
td/utils/logging.cpp
td/utils/misc.cpp
td/utils/MimeType.cpp
td/utils/MpmcQueue.cpp
td/utils/OptionsParser.cpp
td/utils/Random.cpp
td/utils/Slice.cpp
td/utils/SharedSlice.cpp
td/utils/StackAllocator.cpp
td/utils/Status.cpp
td/utils/StringBuilder.cpp
td/utils/Time.cpp
td/utils/Timer.cpp
td/utils/TsFileLog.cpp
td/utils/tests.cpp
td/utils/tl_parsers.cpp
td/utils/translit.cpp
td/utils/unicode.cpp
td/utils/utf8.cpp
td/utils/port/Clocks.h
td/utils/port/config.h
td/utils/port/CxCli.h
td/utils/port/EventFd.h
td/utils/port/EventFdBase.h
td/utils/port/FileFd.h
td/utils/port/IPAddress.h
td/utils/port/IoSlice.h
td/utils/port/MemoryMapping.h
td/utils/port/path.h
td/utils/port/platform.h
td/utils/port/Poll.h
td/utils/port/PollBase.h
td/utils/port/PollFlags.h
td/utils/port/RwMutex.h
td/utils/port/ServerSocketFd.h
td/utils/port/signals.h
td/utils/port/sleep.h
td/utils/port/SocketFd.h
td/utils/port/stacktrace.h
td/utils/port/Stat.h
td/utils/port/StdStreams.h
td/utils/port/thread.h
td/utils/port/thread_local.h
td/utils/port/user.h
td/utils/port/UdpSocketFd.h
td/utils/port/wstring_convert.h
td/utils/port/detail/Epoll.h
td/utils/port/detail/EventFdBsd.h
td/utils/port/detail/EventFdLinux.h
td/utils/port/detail/EventFdWindows.h
td/utils/port/detail/Iocp.h
td/utils/port/detail/KQueue.h
td/utils/port/detail/NativeFd.h
td/utils/port/detail/Poll.h
td/utils/port/detail/PollableFd.h
td/utils/port/detail/Select.h
td/utils/port/detail/ThreadIdGuard.h
td/utils/port/detail/ThreadPthread.h
td/utils/port/detail/ThreadStl.h
td/utils/port/detail/WineventPoll.h
td/utils/AesCtrByteFlow.h
td/utils/as.h
td/utils/base64.h
td/utils/benchmark.h
td/utils/BigNum.h
td/utils/bits.h
td/utils/buffer.h
td/utils/BufferedFd.h
td/utils/BufferedReader.h
td/utils/BufferedUdp.h
td/utils/ByteFlow.h
td/utils/CancellationToken.h
td/utils/ChangesProcessor.h
td/utils/check.h
td/utils/Closure.h
td/utils/common.h
td/utils/ConcurrentHashTable.h
td/utils/Container.h
td/utils/Context.h
td/utils/crypto.h
td/utils/DecTree.h
td/utils/Destructor.h
td/utils/Enumerator.h
td/utils/EpochBasedMemoryReclamation.h
td/utils/FileLog.h
td/utils/filesystem.h
td/utils/find_boundary.h
td/utils/FloodControlFast.h
td/utils/FloodControlStrict.h
td/utils/format.h
td/utils/Gzip.h
td/utils/GzipByteFlow.h
td/utils/Hash.h
td/utils/HashMap.h
td/utils/HashSet.h
td/utils/HazardPointers.h
td/utils/Heap.h
td/utils/Hints.h
td/utils/HttpUrl.h
td/utils/int_types.h
td/utils/invoke.h
td/utils/JsonBuilder.h
td/utils/List.h
td/utils/logging.h
td/utils/MemoryLog.h
td/utils/MimeType.h
td/utils/misc.h
td/utils/MovableValue.h
td/utils/MpmcQueue.h
td/utils/MpmcWaiter.h
td/utils/MpscPollableQueue.h
td/utils/MpscLinkQueue.h
td/utils/Named.h
td/utils/ObjectPool.h
td/utils/Observer.h
td/utils/optional.h
td/utils/OptionsParser.h
td/utils/OrderedEventsProcessor.h
td/utils/overloaded.h
td/utils/Parser.h
td/utils/PathView.h
td/utils/queue.h
td/utils/Random.h
td/utils/ScopeGuard.h
td/utils/SharedObjectPool.h
td/utils/Slice-decl.h
td/utils/Slice.h
td/utils/Span.h
td/utils/SpinLock.h
td/utils/StackAllocator.h
td/utils/Status.h
td/utils/Storer.h
td/utils/StorerBase.h
td/utils/StringBuilder.h
td/utils/tests.h
td/utils/ThreadSafeCounter.h
td/utils/Time.h
td/utils/TimedStat.h
td/utils/Timer.h
td/utils/TsFileLog.h
td/utils/tl_helpers.h
td/utils/tl_parsers.h
td/utils/tl_storers.h
td/utils/translit.h
td/utils/type_traits.h
td/utils/UInt.h
td/utils/uint128.h
td/utils/unicode.h
td/utils/unique_ptr.h
td/utils/utf8.h
td/utils/Variant.h
td/utils/VectorQueue.h
)
set(TDUTILS_TEST_SOURCE
${CMAKE_CURRENT_SOURCE_DIR}/test/buffer.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/ConcurrentHashMap.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/crypto.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/Enumerator.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/filesystem.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/gzip.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/HazardPointers.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/heap.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/json.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/log.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/misc.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/MpmcQueue.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/MpmcWaiter.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/MpscLinkQueue.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/OrderedEventsProcessor.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/port.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/pq.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/SharedObjectPool.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/SharedSlice.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/variant.cpp
PARENT_SCOPE
)
#RULES
#LIBRARIES
add_library(tdutils STATIC ${TDUTILS_SOURCE})
if (WIN32)
# find_library(WS2_32_LIBRARY ws2_32)
# find_library(MSWSOCK_LIBRARY Mswsock)
# target_link_libraries(tdutils PRIVATE ${WS2_32_LIBRARY} ${MSWSOCK_LIBRARY})
target_link_libraries(tdutils PRIVATE ws2_32 Mswsock Normaliz)
endif()
if (NOT CMAKE_CROSSCOMPILING)
add_dependencies(tdutils tdmime_auto)
endif()
if (DEFINED CMAKE_THREAD_LIBS_INIT)
target_link_libraries(tdutils PUBLIC ${CMAKE_THREAD_LIBS_INIT})
endif()
target_include_directories(tdutils PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>)
if (OPENSSL_FOUND)
target_link_libraries(tdutils PRIVATE ${OPENSSL_CRYPTO_LIBRARY} ${CMAKE_DL_LIBS} ${ZLIB_LIBRARIES})
target_include_directories(tdutils SYSTEM PRIVATE ${OPENSSL_INCLUDE_DIR})
endif()
if (ZLIB_FOUND)
target_link_libraries(tdutils PRIVATE ${ZLIB_LIBRARIES})
target_include_directories(tdutils SYSTEM PRIVATE ${ZLIB_INCLUDE_DIR})
endif()
if (CRC32C_FOUND)
target_link_libraries(tdutils PRIVATE crc32c)
endif()
if (ABSL_FOUND)
target_link_libraries_system(tdutils absl::flat_hash_map absl::flat_hash_set absl::hash)
endif()
if (WIN32 AND WINGETOPT_FOUND)
target_link_libraries(tdutils PRIVATE wingetopt)
endif()
if (ANDROID)
target_link_libraries(tdutils PRIVATE log)
endif()
if (CMAKE_HOST_SYSTEM_NAME MATCHES "NetBSD")
target_link_libraries(tdutils PUBLIC /usr/pkg/gcc5/i486--netbsdelf/lib/libatomic.so)
endif()
install(TARGETS tdutils EXPORT TdTargets
LIBRARY DESTINATION lib
ARCHIVE DESTINATION lib
RUNTIME DESTINATION bin
INCLUDES DESTINATION include
)

64
tdutils/generate/CMakeLists.txt Normal file
View file

@ -0,0 +1,64 @@
cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR)
# Generates files for MIME type <-> extension conversions
# DEPENDS ON: gperf grep bash/powershell
file(MAKE_DIRECTORY auto)
set(TDMIME_SOURCE
${CMAKE_CURRENT_SOURCE_DIR}/auto/mime_type_to_extension.cpp
${CMAKE_CURRENT_SOURCE_DIR}/auto/extension_to_mime_type.cpp
)
set(TDMIME_AUTO
${TDMIME_SOURCE}
PARENT_SCOPE
)
add_custom_target(tdmime_auto DEPENDS ${TDMIME_SOURCE})
if (NOT CMAKE_CROSSCOMPILING)
find_program(GPERF_EXECUTABLE gperf)
if (NOT GPERF_EXECUTABLE)
message(FATAL_ERROR "Could NOT find gperf. Add path to gperf executable to PATH environment variable or specify it manually using GPERF_EXECUTABLE option, i. e. 'cmake -DGPERF_EXECUTABLE:FILEPATH=\"<path to gperf executable>\"'.")
endif()
set(GPERF_FILES
${CMAKE_CURRENT_SOURCE_DIR}/auto/mime_type_to_extension.gperf
${CMAKE_CURRENT_SOURCE_DIR}/auto/extension_to_mime_type.gperf
)
set(GPERF_GEN_SOURCE generate_mime_types_gperf.cpp)
add_executable(generate_mime_types_gperf ${GPERF_GEN_SOURCE})
add_custom_command(
OUTPUT ${GPERF_FILES}
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
COMMAND generate_mime_types_gperf mime_types.txt ${GPERF_FILES}
DEPENDS generate_mime_types_gperf mime_types.txt
)
if (WIN32)
set(MIME_TYPE_TO_EXTENSION_CMD ${GPERF_EXECUTABLE} -m100 --output-file=auto/mime_type_to_extension.cpp auto/mime_type_to_extension.gperf)
else()
set(MIME_TYPE_TO_EXTENSION_CMD ${GPERF_EXECUTABLE} -m100 auto/mime_type_to_extension.gperf | grep -v __gnu_inline__ > auto/mime_type_to_extension.cpp)
endif()
add_custom_command(
OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/auto/mime_type_to_extension.cpp
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
COMMAND ${MIME_TYPE_TO_EXTENSION_CMD}
DEPENDS auto/mime_type_to_extension.gperf
)
if (WIN32)
set(EXTENSION_TO_MIME_TYPE_CMD ${GPERF_EXECUTABLE} -m100 --output-file=auto/extension_to_mime_type.cpp auto/extension_to_mime_type.gperf)
else()
set(EXTENSION_TO_MIME_TYPE_CMD ${GPERF_EXECUTABLE} -m100 auto/extension_to_mime_type.gperf | grep -v __gnu_inline__ > auto/extension_to_mime_type.cpp)
endif()
add_custom_command(
OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/auto/extension_to_mime_type.cpp
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
COMMAND ${EXTENSION_TO_MIME_TYPE_CMD}
DEPENDS auto/extension_to_mime_type.gperf
)
endif()

177
tdutils/generate/generate_mime_types_gperf.cpp Normal file
View file

@ -0,0 +1,177 @@
/*
This file is part of TON Blockchain source code.
TON Blockchain is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.
TON Blockchain is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with TON Blockchain. If not, see <http://www.gnu.org/licenses/>.
In addition, as a special exception, the copyright holders give permission
to link the code of portions of this program with the OpenSSL library.
You must obey the GNU General Public License in all respects for all
of the code used other than OpenSSL. If you modify file(s) with this
exception, you may extend this exception to your version of the file(s),
but you are not obligated to do so. If you do not wish to do so, delete this
exception statement from your version. If you delete this exception statement
from all source files in the program, then also delete it here.
Copyright 2017-2019 Telegram Systems LLP
*/
#include <algorithm>
#include <cassert>
#include <cstddef>
#include <fstream>
#include <iostream>
#include <map>
#include <string>
#include <tuple>
#include <utility>
#include <vector>
static std::pair<std::string, std::string> split(std::string s, char delimiter = ' ') {
auto delimiter_pos = s.find(delimiter);
if (delimiter_pos == std::string::npos) {
return {std::move(s), ""};
} else {
auto head = s.substr(0, delimiter_pos);
auto tail = s.substr(delimiter_pos + 1);
return {head, tail};
}
}
static bool generate(const char *file_name, const char *from_name, const char *to_name,
const std::map<std::string, std::string> &map) {
// binary mode is needed for MSYS2 gperf
std::ofstream out(file_name, std::ios_base::trunc | std::ios_base::binary);
if (!out) {
std::cerr << "Can't open output file \"" << file_name << std::endl;
return false;
}
out << "%struct-type\n";
out << "%ignore-case\n";
out << "%language=ANSI-C\n";
out << "%readonly-tables\n";
out << "%includes\n";
out << "%enum\n";
out << "%define slot-name " << from_name << "\n";
out << "%define initializer-suffix ,nullptr\n";
out << "%define slot-name " << from_name << "\n";
out << "%define hash-function-name " << from_name << "_hash\n";
out << "%define lookup-function-name search_" << from_name << "\n";
// out << "%define class-name " << from_name << "_to_" << to_name << "\n";
out << "struct " << from_name << "_and_" << to_name << " {\n";
out << " const char *" << from_name << ";\n";
out << " const char *" << to_name << ";\n";
out << "}\n";
out << "%%\n";
for (auto &value : map) {
out << '"' << value.first << "\", \"" << value.second << '"' << "\n";
}
out << "%%\n";
out << "const char *" << from_name << "_to_" << to_name << "(const char *" << from_name << ", size_t " << from_name
<< "_len) {\n";
out << " const auto &result = search_" << from_name << "(" << from_name << ", " << from_name << "_len);\n";
out << " if (result == nullptr) {\n";
out << " return nullptr;\n";
out << " }\n";
out << "\n";
out << " return result->" << to_name << ";\n";
out << "}\n";
return true;
}
static bool is_private_mime_type(const std::string &mime_type) {
return mime_type.find("/x-") != std::string::npos;
}
int main(int argc, char *argv[]) {
if (argc != 4) {
std::cerr << "Wrong number of arguments supplied. Expected 'generate_mime_types_gperf <mime_types.txt> "
"<mime_type_to_extension.cpp> <extension_to_mime_type.cpp>'"
<< std::endl;
return EXIT_FAILURE;
}
std::ifstream mime_types_file(argv[1]);
if (!mime_types_file) {
std::cerr << "Can't open input file \"" << argv[1] << std::endl;
return EXIT_FAILURE;
}
std::map<std::string, std::string> mime_type_to_extension;
std::map<std::string, std::string> extension_to_mime_type;
std::string line;
while (std::getline(mime_types_file, line)) {
while (!line.empty() && (line.back() == '\r' || line.back() == '\n')) {
line.pop_back();
}
std::string mime_type;
std::string extensions_string;
std::tie(mime_type, extensions_string) = split(line, '\t');
if (mime_type.empty()) {
std::cerr << "Wrong MIME type description \"" << line << "\"" << std::endl;
continue;
}
auto extensions_start_position = extensions_string.find_first_not_of(" \t");
if (extensions_start_position == std::string::npos) {
std::cerr << "Wrong MIME type description \"" << line << "\"" << std::endl;
continue;
}
extensions_string = extensions_string.substr(extensions_start_position);
std::vector<std::string> extensions;
while (!extensions_string.empty()) {
extensions.push_back("");
std::tie(extensions.back(), extensions_string) = split(extensions_string);
}
assert(!extensions.empty());
std::map<std::string, std::string> preffered_extensions{{"image/jpeg", "jpg"}, {"audio/mpeg", "mp3"},
{"audio/midi", "midi"}, {"text/x-pascal", "pas"},
{"text/x-asm", "asm"}, {"video/quicktime", "mov"}};
std::size_t index = 0;
if (preffered_extensions.count(mime_type) != 0) {
index = std::find(extensions.begin(), extensions.end(), preffered_extensions[mime_type]) - extensions.begin();
assert(index < extensions.size());
}
if (mime_type_to_extension.emplace_hint(mime_type_to_extension.end(), mime_type, extensions[index])->second !=
extensions[index]) {
std::cerr << "MIME type \"" << mime_type << "\" has more than one extensions list" << std::endl;
}
for (auto &extension : extensions) {
if (!extension_to_mime_type.emplace(extension, mime_type).second) {
if (is_private_mime_type(extension_to_mime_type[extension]) == is_private_mime_type(mime_type)) {
std::cerr << "Extension \"" << extension << "\" matches more than one type" << std::endl;
} else {
if (!is_private_mime_type(mime_type)) {
extension_to_mime_type[extension] = mime_type;
}
}
}
}
}
if (!generate(argv[2], "mime_type", "extension", mime_type_to_extension)) {
return EXIT_FAILURE;
}
if (!generate(argv[3], "extension", "mime_type", extension_to_mime_type)) {
return EXIT_FAILURE;
}
}

777
tdutils/generate/mime_types.txt Normal file
View file

@ -0,0 +1,777 @@
application/andrew-inset ez
application/applixware aw
application/atom+xml atom
application/atomcat+xml atomcat
application/atomsvc+xml atomsvc
application/ccxml+xml ccxml
application/cdmi-capability cdmia
application/cdmi-container cdmic
application/cdmi-domain cdmid
application/cdmi-object cdmio
application/cdmi-queue cdmiq
application/cu-seeme cu
application/davmount+xml davmount
application/docbook+xml dbk
application/dssc+der dssc
application/dssc+xml xdssc
application/ecmascript es
application/emma+xml emma
application/epub+zip epub
application/exi exi
application/font-tdpfr pfr
application/gml+xml gml
application/gpx+xml gpx
application/gxf gxf
application/hyperstudio stk
application/inkml+xml ink inkml
application/ipfix ipfix
application/java-archive jar
application/java-serialized-object ser
application/java-vm class
application/javascript js
application/json json
application/jsonml+json jsonml
application/lost+xml lostxml
application/mac-binhex40 hqx
application/mac-compactpro cpt
application/mads+xml mads
application/marc mrc
application/marcxml+xml mrcx
application/mathematica ma nb mb
application/mathml+xml mathml
application/mbox mbox
application/mediaservercontrol+xml mscml
application/metalink+xml metalink
application/metalink4+xml meta4
application/mets+xml mets
application/mods+xml mods
application/mp21 m21 mp21
application/mp4 mp4s
application/msword doc dot
application/mxf mxf
application/octet-stream bin dms lrf mar so dist distz pkg bpk dump elc deploy
application/oda oda
application/oebps-package+xml opf
application/ogg ogx
application/omdoc+xml omdoc
application/onenote onetoc onetoc2 onetmp onepkg
application/oxps oxps
application/patch-ops-error+xml xer
application/pdf pdf
application/pgp-encrypted pgp
application/pgp-signature asc sig
application/pics-rules prf
application/pkcs10 p10
application/pkcs7-mime p7m p7c
application/pkcs7-signature p7s
application/pkcs8 p8
application/pkix-attr-cert ac
application/pkix-cert cer
application/pkix-crl crl
application/pkix-pkipath pkipath
application/pkixcmp pki
application/pls+xml pls
application/postscript ai eps ps
application/prs.cww cww
application/pskc+xml pskcxml
application/rdf+xml rdf
application/reginfo+xml rif
application/relax-ng-compact-syntax rnc
application/resource-lists+xml rl
application/resource-lists-diff+xml rld
application/rls-services+xml rs
application/rpki-ghostbusters gbr
application/rpki-manifest mft
application/rpki-roa roa
application/rsd+xml rsd
application/rss+xml rss
application/rtf rtf
application/sbml+xml sbml
application/scvp-cv-request scq
application/scvp-cv-response scs
application/scvp-vp-request spq
application/scvp-vp-response spp
application/sdp sdp
application/set-payment-initiation setpay
application/set-registration-initiation setreg
application/shf+xml shf
application/smil+xml smi smil
application/sparql-query rq
application/sparql-results+xml srx
application/srgs gram
application/srgs+xml grxml
application/sru+xml sru
application/ssdl+xml ssdl
application/ssml+xml ssml
application/tei+xml tei teicorpus
application/thraud+xml tfi
application/timestamped-data tsd
application/vnd.3gpp.pic-bw-large plb
application/vnd.3gpp.pic-bw-small psb
application/vnd.3gpp.pic-bw-var pvb
application/vnd.3gpp2.tcap tcap
application/vnd.3m.post-it-notes pwn
application/vnd.accpac.simply.aso aso
application/vnd.accpac.simply.imp imp
application/vnd.acucobol acu
application/vnd.acucorp atc acutc
application/vnd.adobe.air-application-installer-package+zip air
application/vnd.adobe.formscentral.fcdt fcdt
application/vnd.adobe.fxp fxp fxpl
application/vnd.adobe.xdp+xml xdp
application/vnd.adobe.xfdf xfdf
application/vnd.ahead.space ahead
application/vnd.airzip.filesecure.azf azf
application/vnd.airzip.filesecure.azs azs
application/vnd.amazon.ebook azw
application/vnd.americandynamics.acc acc
application/vnd.amiga.ami ami
application/vnd.android.package-archive apk
application/vnd.anser-web-certificate-issue-initiation cii
application/vnd.anser-web-funds-transfer-initiation fti
application/vnd.antix.game-component atx
application/vnd.apple.installer+xml mpkg
application/vnd.apple.mpegurl m3u8
application/vnd.aristanetworks.swi swi
application/vnd.astraea-software.iota iota
application/vnd.audiograph aep
application/vnd.blueice.multipass mpm
application/vnd.bmi bmi
application/vnd.businessobjects rep
application/vnd.chemdraw+xml cdxml
application/vnd.chipnuts.karaoke-mmd mmd
application/vnd.cinderella cdy
application/vnd.claymore cla
application/vnd.cloanto.rp9 rp9
application/vnd.clonk.c4group c4g c4d c4f c4p c4u
application/vnd.cluetrust.cartomobile-config c11amc
application/vnd.cluetrust.cartomobile-config-pkg c11amz
application/vnd.commonspace csp
application/vnd.contact.cmsg cdbcmsg
application/vnd.cosmocaller cmc
application/vnd.crick.clicker clkx
application/vnd.crick.clicker.keyboard clkk
application/vnd.crick.clicker.palette clkp
application/vnd.crick.clicker.template clkt
application/vnd.crick.clicker.wordbank clkw
application/vnd.criticaltools.wbs+xml wbs
application/vnd.ctc-posml pml
application/vnd.cups-ppd ppd
application/vnd.curl.car car
application/vnd.curl.pcurl pcurl
application/vnd.dart dart
application/vnd.data-vision.rdz rdz
application/vnd.dece.data uvf uvvf uvd uvvd
application/vnd.dece.ttml+xml uvt uvvt
application/vnd.dece.unspecified uvx uvvx
application/vnd.dece.zip uvz uvvz
application/vnd.denovo.fcselayout-link fe_launch
application/vnd.dna dna
application/vnd.dolby.mlp mlp
application/vnd.dpgraph dpg
application/vnd.dreamfactory dfac
application/vnd.ds-keypoint kpxx
application/vnd.dvb.ait ait
application/vnd.dvb.service svc
application/vnd.dynageo geo
application/vnd.ecowin.chart mag
application/vnd.enliven nml
application/vnd.epson.esf esf
application/vnd.epson.msf msf
application/vnd.epson.quickanime qam
application/vnd.epson.salt slt
application/vnd.epson.ssf ssf
application/vnd.eszigno3+xml es3 et3
application/vnd.ezpix-album ez2
application/vnd.ezpix-package ez3
application/vnd.fdf fdf
application/vnd.fdsn.mseed mseed
application/vnd.fdsn.seed seed dataless
application/vnd.flographit gph
application/vnd.fluxtime.clip ftc
application/vnd.framemaker fm frame maker book
application/vnd.frogans.fnc fnc
application/vnd.frogans.ltf ltf
application/vnd.fsc.weblaunch fsc
application/vnd.fujitsu.oasys oas
application/vnd.fujitsu.oasys2 oa2
application/vnd.fujitsu.oasys3 oa3
application/vnd.fujitsu.oasysgp fg5
application/vnd.fujitsu.oasysprs bh2
application/vnd.fujixerox.ddd ddd
application/vnd.fujixerox.docuworks xdw
application/vnd.fujixerox.docuworks.binder xbd
application/vnd.fuzzysheet fzs
application/vnd.genomatix.tuxedo txd
application/vnd.geogebra.file ggb
application/vnd.geogebra.tool ggt
application/vnd.geometry-explorer gex gre
application/vnd.geonext gxt
application/vnd.geoplan g2w
application/vnd.geospace g3w
application/vnd.gmx gmx
application/vnd.google-earth.kml+xml kml
application/vnd.google-earth.kmz kmz
application/vnd.grafeq gqf gqs
application/vnd.groove-account gac
application/vnd.groove-help ghf
application/vnd.groove-identity-message gim
application/vnd.groove-injector grv
application/vnd.groove-tool-message gtm
application/vnd.groove-tool-template tpl
application/vnd.groove-vcard vcg
application/vnd.hal+xml hal
application/vnd.handheld-entertainment+xml zmm
application/vnd.hbci hbci
application/vnd.hhe.lesson-player les
application/vnd.hp-hpgl hpgl
application/vnd.hp-hpid hpid
application/vnd.hp-hps hps
application/vnd.hp-jlyt jlt
application/vnd.hp-pcl pcl
application/vnd.hp-pclxl pclxl
application/vnd.hydrostatix.sof-data sfd-hdstx
application/vnd.ibm.minipay mpy
application/vnd.ibm.modcap afp listafp list3820
application/vnd.ibm.rights-management irm
application/vnd.ibm.secure-container sc
application/vnd.iccprofile icc icm
application/vnd.igloader igl
application/vnd.immervision-ivp ivp
application/vnd.immervision-ivu ivu
application/vnd.insors.igm igm
application/vnd.intercon.formnet xpw xpx
application/vnd.intergeo i2g
application/vnd.intu.qbo qbo
application/vnd.intu.qfx qfx
application/vnd.ipunplugged.rcprofile rcprofile
application/vnd.irepository.package+xml irp
application/vnd.is-xpr xpr
application/vnd.isac.fcs fcs
application/vnd.jam jam
application/vnd.jcp.javame.midlet-rms rms
application/vnd.jisp jisp
application/vnd.joost.joda-archive joda
application/vnd.kahootz ktz ktr
application/vnd.kde.karbon karbon
application/vnd.kde.kchart chrt
application/vnd.kde.kformula kfo
application/vnd.kde.kivio flw
application/vnd.kde.kontour kon
application/vnd.kde.kpresenter kpr kpt
application/vnd.kde.kspread ksp
application/vnd.kde.kword kwd kwt
application/vnd.kenameaapp htke
application/vnd.kidspiration kia
application/vnd.kinar kne knp
application/vnd.koan skp skd skt skm
application/vnd.kodak-descriptor sse
application/vnd.las.las+xml lasxml
application/vnd.llamagraphics.life-balance.desktop lbd
application/vnd.llamagraphics.life-balance.exchange+xml lbe
application/vnd.lotus-1-2-3 123
application/vnd.lotus-approach apr
application/vnd.lotus-freelance pre
application/vnd.lotus-notes nsf
application/vnd.lotus-organizer org
application/vnd.lotus-screencam scm
application/vnd.lotus-wordpro lwp
application/vnd.macports.portpkg portpkg
application/vnd.mcd mcd
application/vnd.medcalcdata mc1
application/vnd.mediastation.cdkey cdkey
application/vnd.mfer mwf
application/vnd.mfmp mfm
application/vnd.micrografx.flo flo
application/vnd.micrografx.igx igx
application/vnd.mif mif
application/vnd.mobius.daf daf
application/vnd.mobius.dis dis
application/vnd.mobius.mbk mbk
application/vnd.mobius.mqy mqy
application/vnd.mobius.msl msl
application/vnd.mobius.plc plc
application/vnd.mobius.txf txf
application/vnd.mophun.application mpn
application/vnd.mophun.certificate mpc
application/vnd.mozilla.xul+xml xul
application/vnd.ms-artgalry cil
application/vnd.ms-cab-compressed cab
application/vnd.ms-excel xls xlm xla xlc xlt xlw
application/vnd.ms-excel.addin.macroenabled.12 xlam
application/vnd.ms-excel.sheet.binary.macroenabled.12 xlsb
application/vnd.ms-excel.sheet.macroenabled.12 xlsm
application/vnd.ms-excel.template.macroenabled.12 xltm
application/vnd.ms-fontobject eot
application/vnd.ms-htmlhelp chm
application/vnd.ms-ims ims
application/vnd.ms-lrm lrm
application/vnd.ms-officetheme thmx
application/vnd.ms-pki.seccat cat
application/vnd.ms-pki.stl stl
application/vnd.ms-powerpoint ppt pps pot
application/vnd.ms-powerpoint.addin.macroenabled.12 ppam
application/vnd.ms-powerpoint.presentation.macroenabled.12 pptm
application/vnd.ms-powerpoint.slide.macroenabled.12 sldm
application/vnd.ms-powerpoint.slideshow.macroenabled.12 ppsm
application/vnd.ms-powerpoint.template.macroenabled.12 potm
application/vnd.ms-project mpp mpt
application/vnd.ms-word.document.macroenabled.12 docm
application/vnd.ms-word.template.macroenabled.12 dotm
application/vnd.ms-works wps wks wcm wdb
application/vnd.ms-wpl wpl
application/vnd.ms-xpsdocument xps
application/vnd.mseq mseq
application/vnd.musician mus
application/vnd.muvee.style msty
application/vnd.mynfc taglet
application/vnd.neurolanguage.nlu nlu
application/vnd.nitf ntf nitf
application/vnd.noblenet-directory nnd
application/vnd.noblenet-sealer nns
application/vnd.noblenet-web nnw
application/vnd.nokia.n-gage.data ngdat
application/vnd.nokia.n-gage.symbian.install n-gage
application/vnd.nokia.radio-preset rpst
application/vnd.nokia.radio-presets rpss
application/vnd.novadigm.edm edm
application/vnd.novadigm.edx edx
application/vnd.novadigm.ext ext
application/vnd.oasis.opendocument.chart odc
application/vnd.oasis.opendocument.chart-template otc
application/vnd.oasis.opendocument.database odb
application/vnd.oasis.opendocument.formula odf
application/vnd.oasis.opendocument.formula-template odft
application/vnd.oasis.opendocument.graphics odg
application/vnd.oasis.opendocument.graphics-template otg
application/vnd.oasis.opendocument.image odi
application/vnd.oasis.opendocument.image-template oti
application/vnd.oasis.opendocument.presentation odp
application/vnd.oasis.opendocument.presentation-template otp
application/vnd.oasis.opendocument.spreadsheet ods
application/vnd.oasis.opendocument.spreadsheet-template ots
application/vnd.oasis.opendocument.text odt
application/vnd.oasis.opendocument.text-master odm
application/vnd.oasis.opendocument.text-template ott
application/vnd.oasis.opendocument.text-web oth
application/vnd.olpc-sugar xo
application/vnd.oma.dd2+xml dd2
application/vnd.openofficeorg.extension oxt
application/vnd.openxmlformats-officedocument.presentationml.presentation pptx
application/vnd.openxmlformats-officedocument.presentationml.slide sldx
application/vnd.openxmlformats-officedocument.presentationml.slideshow ppsx
application/vnd.openxmlformats-officedocument.presentationml.template potx
application/vnd.openxmlformats-officedocument.spreadsheetml.sheet xlsx
application/vnd.openxmlformats-officedocument.spreadsheetml.template xltx
application/vnd.openxmlformats-officedocument.wordprocessingml.document docx
application/vnd.openxmlformats-officedocument.wordprocessingml.template dotx
application/vnd.osgeo.mapguide.package mgp
application/vnd.osgi.dp dp
application/vnd.osgi.subsystem esa
application/vnd.palm pdb pqa oprc
application/vnd.pawaafile paw
application/vnd.pg.format str
application/vnd.pg.osasli ei6
application/vnd.picsel efif
application/vnd.pmi.widget wg
application/vnd.pocketlearn plf
application/vnd.powerbuilder6 pbd
application/vnd.previewsystems.box box
application/vnd.proteus.magazine mgz
application/vnd.publishare-delta-tree qps
application/vnd.pvi.ptid1 ptid
application/vnd.quark.quarkxpress qxd qxt qwd qwt qxl qxb
application/vnd.realvnc.bed bed
application/vnd.recordare.musicxml mxl
application/vnd.recordare.musicxml+xml musicxml
application/vnd.rig.cryptonote cryptonote
application/vnd.rim.cod cod
application/vnd.rn-realmedia rm
application/vnd.rn-realmedia-vbr rmvb
application/vnd.route66.link66+xml link66
application/vnd.sailingtracker.track st
application/vnd.seemail see
application/vnd.sema sema
application/vnd.semd semd
application/vnd.semf semf
application/vnd.shana.informed.formdata ifm
application/vnd.shana.informed.formtemplate itp
application/vnd.shana.informed.interchange iif
application/vnd.shana.informed.package ipk
application/vnd.simtech-mindmapper twd twds
application/vnd.smaf mmf
application/vnd.smart.teacher teacher
application/vnd.solent.sdkm+xml sdkm sdkd
application/vnd.spotfire.dxp dxp
application/vnd.spotfire.sfs sfs
application/vnd.stardivision.calc sdc
application/vnd.stardivision.draw sda
application/vnd.stardivision.impress sdd
application/vnd.stardivision.math smf
application/vnd.stardivision.writer sdw vor
application/vnd.stardivision.writer-global sgl
application/vnd.stepmania.package smzip
application/vnd.stepmania.stepchart sm
application/vnd.sun.xml.calc sxc
application/vnd.sun.xml.calc.template stc
application/vnd.sun.xml.draw sxd
application/vnd.sun.xml.draw.template std
application/vnd.sun.xml.impress sxi
application/vnd.sun.xml.impress.template sti
application/vnd.sun.xml.math sxm
application/vnd.sun.xml.writer sxw
application/vnd.sun.xml.writer.global sxg
application/vnd.sun.xml.writer.template stw
application/vnd.sus-calendar sus susp
application/vnd.svd svd
application/vnd.symbian.install sis sisx
application/vnd.syncml+xml xsm
application/vnd.syncml.dm+wbxml bdm
application/vnd.syncml.dm+xml xdm
application/vnd.tao.intent-module-archive tao
application/vnd.tcpdump.pcap pcap cap dmp
application/vnd.tmobile-livetv tmo
application/vnd.trid.tpt tpt
application/vnd.triscape.mxs mxs
application/vnd.trueapp tra
application/vnd.ufdl ufd ufdl
application/vnd.uiq.theme utz
application/vnd.umajin umj
application/vnd.unity unityweb
application/vnd.uoml+xml uoml
application/vnd.vcx vcx
application/vnd.visio vsd vst vss vsw
application/vnd.visionary vis
application/vnd.vsf vsf
application/vnd.wap.wbxml wbxml
application/vnd.wap.wmlc wmlc
application/vnd.wap.wmlscriptc wmlsc
application/vnd.webturbo wtb
application/vnd.wolfram.player nbp
application/vnd.wordperfect wpd
application/vnd.wqd wqd
application/vnd.wt.stf stf
application/vnd.xara xar
application/vnd.xfdl xfdl
application/vnd.yamaha.hv-dic hvd
application/vnd.yamaha.hv-script hvs
application/vnd.yamaha.hv-voice hvp
application/vnd.yamaha.openscoreformat osf
application/vnd.yamaha.openscoreformat.osfpvg+xml osfpvg
application/vnd.yamaha.smaf-audio saf
application/vnd.yamaha.smaf-phrase spf
application/vnd.yellowriver-custom-menu cmp
application/vnd.zul zir zirz
application/vnd.zzazz.deck+xml zaz
application/voicexml+xml vxml
application/widget wgt
application/winhlp hlp
application/wsdl+xml wsdl
application/wspolicy+xml wspolicy
application/x-7z-compressed 7z
application/x-abiword abw
application/x-ace-compressed ace
application/x-apple-diskimage dmg
application/x-authorware-bin aab x32 u32 vox
application/x-authorware-map aam
application/x-authorware-seg aas
application/x-bcpio bcpio
application/x-bittorrent torrent
application/x-blorb blb blorb
application/x-bzip bz
application/x-bzip2 bz2 boz
application/x-cbr cbr cba cbt cbz cb7
application/x-cdlink vcd
application/x-cfs-compressed cfs
application/x-chat chat
application/x-chess-pgn pgn
application/x-conference nsc
application/x-cpio cpio
application/x-csh csh
application/x-debian-package deb udeb
application/x-dgc-compressed dgc
application/x-director dir dcr dxr cst cct cxt w3d fgd swa
application/x-doom wad
application/x-dtbncx+xml ncx
application/x-dtbook+xml dtb
application/x-dtbresource+xml res
application/x-dvi dvi
application/x-envoy evy
application/x-eva eva
application/x-font-bdf bdf
application/x-font-ghostscript gsf
application/x-font-linux-psf psf
application/x-font-otf otf
application/x-font-pcf pcf
application/x-font-snf snf
application/x-font-ttf ttf ttc
application/x-font-type1 pfa pfb pfm afm
application/x-font-woff woff
application/x-freearc arc
application/x-futuresplash spl
application/x-gca-compressed gca
application/x-glulx ulx
application/x-gnumeric gnumeric
application/x-gramps-xml gramps
application/x-gtar gtar
application/x-hdf hdf
application/x-install-instructions install
application/x-iso9660-image iso
application/x-java-jnlp-file jnlp
application/x-latex latex
application/x-lzh-compressed lzh lha
application/x-mie mie
application/x-mobipocket-ebook prc mobi
application/x-ms-application application
application/x-ms-shortcut lnk
application/x-ms-wmd wmd
application/x-ms-wmz wmz
application/x-ms-xbap xbap
application/x-msaccess mdb
application/x-msbinder obd
application/x-mscardfile crd
application/x-msclip clp
application/x-msdownload exe dll com bat msi
application/x-msmediaview mvb m13 m14
application/x-msmetafile wmf wmz emf emz
application/x-msmoney mny
application/x-mspublisher pub
application/x-msschedule scd
application/x-msterminal trm
application/x-mswrite wri
application/x-netcdf nc cdf
application/x-nzb nzb
application/x-pkcs12 p12 pfx
application/x-pkcs7-certificates p7b spc
application/x-pkcs7-certreqresp p7r
application/x-rar-compressed rar
application/x-research-info-systems ris
application/x-sh sh
application/x-shar shar
application/x-shockwave-flash swf
application/x-silverlight-app xap
application/x-sql sql
application/x-stuffit sit
application/x-stuffitx sitx
application/x-subrip srt
application/x-sv4cpio sv4cpio
application/x-sv4crc sv4crc
application/x-t3vm-image t3
application/x-tads gam
application/x-tar tar
application/x-tcl tcl
application/x-tex tex
application/x-tex-tfm tfm
application/x-texinfo texinfo texi
application/x-tgif obj
application/x-tgsticker tgs
application/x-ustar ustar
application/x-wais-source src
application/x-x509-ca-cert der crt
application/x-xfig fig
application/x-xliff+xml xlf
application/x-xpinstall xpi
application/x-xz xz
application/x-zmachine z1 z2 z3 z4 z5 z6 z7 z8
application/xaml+xml xaml
application/xcap-diff+xml xdf
application/xenc+xml xenc
application/xhtml+xml xhtml xht
application/xml xml xsl
application/xml-dtd dtd
application/xop+xml xop
application/xproc+xml xpl
application/xslt+xml xslt
application/xspf+xml xspf
application/xv+xml mxml xhvml xvml xvm
application/yang yang
application/yin+xml yin
application/zip zip
audio/adpcm adp
audio/basic au snd
audio/midi mid midi kar rmi
audio/mp4 m4a mp4a
audio/mpeg mpga mp2 mp2a mp3 m2a m3a
audio/ogg oga ogg spx
audio/s3m s3m
audio/silk sil
audio/vnd.dece.audio uva uvva
audio/vnd.digital-winds eol
audio/vnd.dra dra
audio/vnd.dts dts
audio/vnd.dts.hd dtshd
audio/vnd.lucent.voice lvp
audio/vnd.ms-playready.media.pya pya
audio/vnd.nuera.ecelp4800 ecelp4800
audio/vnd.nuera.ecelp7470 ecelp7470
audio/vnd.nuera.ecelp9600 ecelp9600
audio/vnd.rip rip
audio/webm weba
audio/x-aac aac
audio/x-aiff aif aiff aifc
audio/x-caf caf
audio/x-flac flac
audio/x-matroska mka
audio/x-mpegurl m3u
audio/x-ms-wax wax
audio/x-ms-wma wma
audio/x-pn-realaudio ram ra
audio/x-pn-realaudio-plugin rmp
audio/x-wav wav
audio/xm xm
chemical/x-cdx cdx
chemical/x-cif cif
chemical/x-cmdf cmdf
chemical/x-cml cml
chemical/x-csml csml
chemical/x-xyz xyz
font/collection ttc
font/otf otf
font/ttf ttf
font/woff woff
font/woff2 woff2
image/bmp bmp
image/cgm cgm
image/g3fax g3
image/gif gif
image/heic heic
image/heic-sequence heics
image/heif heif
image/heif-sequence heifs
image/ief ief
image/jpeg jpeg jpg jpe
image/ktx ktx
image/png png
image/prs.btif btif
image/sgi sgi
image/svg+xml svg svgz
image/tiff tiff tif
image/vnd.adobe.photoshop psd
image/vnd.dece.graphic uvi uvvi uvg uvvg
image/vnd.djvu djvu djv
image/vnd.dvb.subtitle sub
image/vnd.dwg dwg
image/vnd.dxf dxf
image/vnd.fastbidsheet fbs
image/vnd.fpx fpx
image/vnd.fst fst
image/vnd.fujixerox.edmics-mmr mmr
image/vnd.fujixerox.edmics-rlc rlc
image/vnd.ms-modi mdi
image/vnd.ms-photo wdp
image/vnd.net-fpx npx
image/vnd.wap.wbmp wbmp
image/vnd.xiff xif
image/webp webp
image/x-3ds 3ds
image/x-cmu-raster ras
image/x-cmx cmx
image/x-freehand fh fhc fh4 fh5 fh7
image/x-icon ico
image/x-mrsid-image sid
image/x-pcx pcx
image/x-pict pic pct
image/x-portable-anymap pnm
image/x-portable-bitmap pbm
image/x-portable-graymap pgm
image/x-portable-pixmap ppm
image/x-rgb rgb
image/x-tga tga
image/x-xbitmap xbm
image/x-xpixmap xpm
image/x-xwindowdump xwd
message/rfc822 eml mime
model/iges igs iges
model/mesh msh mesh silo
model/vnd.collada+xml dae
model/vnd.dwf dwf
model/vnd.gdl gdl
model/vnd.gtw gtw
model/vnd.mts mts
model/vnd.vtu vtu
model/vrml wrl vrml
model/x3d+binary x3db x3dbz
model/x3d+vrml x3dv x3dvz
model/x3d+xml x3d x3dz
text/cache-manifest appcache
text/calendar ics ifb
text/css css
text/csv csv
text/html html htm
text/n3 n3
text/plain txt text conf def list log in
text/prs.lines.tag dsc
text/richtext rtx
text/sgml sgml sgm
text/tab-separated-values tsv
text/troff t tr roff man me ms
text/turtle ttl
text/uri-list uri uris urls
text/vcard vcard
text/vnd.curl curl
text/vnd.curl.dcurl dcurl
text/vnd.curl.mcurl mcurl
text/vnd.curl.scurl scurl
text/vnd.dvb.subtitle sub
text/vnd.fly fly
text/vnd.fmi.flexstor flx
text/vnd.graphviz gv
text/vnd.in3d.3dml 3dml
text/vnd.in3d.spot spot
text/vnd.sun.j2me.app-descriptor jad
text/vnd.wap.wml wml
text/vnd.wap.wmlscript wmls
text/x-asm s asm
text/x-c c cc cxx cpp h hh dic
text/x-fortran f for f77 f90
text/x-java-source java
text/x-nfo nfo
text/x-opml opml
text/x-pascal p pas
text/x-php php
text/x-setext etx
text/x-sfv sfv
text/x-uuencode uu
text/x-vcalendar vcs
text/x-vcard vcf
video/3gpp 3gp
video/3gpp2 3g2
video/h261 h261
video/h263 h263
video/h264 h264
video/h265 h265
video/jpeg jpgv
video/jpm jpm jpgm
video/mj2 mj2 mjp2
video/mp4 mp4 mp4v mpg4
video/mpeg mpeg mpg mpe m1v m2v
video/ogg ogv
video/quicktime qt mov
video/vnd.dece.hd uvh uvvh
video/vnd.dece.mobile uvm uvvm
video/vnd.dece.pd uvp uvvp
video/vnd.dece.sd uvs uvvs
video/vnd.dece.video uvv uvvv
video/vnd.dvb.file dvb
video/vnd.fvt fvt
video/vnd.mpegurl mxu m4u
video/vnd.ms-playready.media.pyv pyv
video/vnd.uvvu.mp4 uvu uvvu
video/vnd.vivo viv
video/webm webm
video/x-f4v f4v
video/x-fli fli
video/x-flv flv
video/x-m4v m4v
video/x-matroska mkv mk3d mks
video/x-mng mng
video/x-ms-asf asf asx
video/x-ms-vob vob
video/x-ms-wm wm
video/x-ms-wmv wmv
video/x-ms-wmx wmx
video/x-ms-wvx wvx
video/x-msvideo avi
video/x-sgi-movie movie
video/x-smv smv
x-conference/x-cooltalk ice

68
tdutils/td/utils/AesCtrByteFlow.h Normal file
View file

@ -0,0 +1,68 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/ByteFlow.h"
#include "td/utils/common.h"
#include "td/utils/crypto.h"
#include "td/utils/Slice.h"
#include "td/utils/Status.h"
#include "td/utils/UInt.h"
namespace td {
#if TD_HAVE_OPENSSL
class AesCtrByteFlow : public ByteFlowInplaceBase {
public:
void init(const UInt256 &key, const UInt128 &iv) {
state_.init(key, iv);
}
void init(AesCtrState &&state) {
state_ = std::move(state);
}
AesCtrState move_aes_ctr_state() {
return std::move(state_);
}
void loop() override {
bool was_updated = false;
while (true) {
auto ready = input_->prepare_read();
if (ready.empty()) {
break;
}
state_.encrypt(ready, MutableSlice(const_cast<char *>(ready.data()), ready.size()));
input_->confirm_read(ready.size());
output_.advance_end(ready.size());
was_updated = true;
}
if (was_updated) {
on_output_updated();
}
if (!is_input_active_) {
finish(Status::OK()); // End of input stream.
}
set_need_size(1);
}
private:
AesCtrState state_;
};
#endif
} // namespace td

321
tdutils/td/utils/BigNum.cpp Normal file
View file

@ -0,0 +1,321 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/utils/BigNum.h"
char disable_linker_warning_about_empty_file_bignum_cpp TD_UNUSED;
#if TD_HAVE_OPENSSL
#include "td/utils/logging.h"
#include "td/utils/misc.h"
#include <openssl/bn.h>
#include <openssl/crypto.h>
namespace td {
class BigNumContext::Impl {
public:
BN_CTX *big_num_context;
Impl() : big_num_context(BN_CTX_new()) {
LOG_IF(FATAL, big_num_context == nullptr);
}
Impl(const Impl &other) = delete;
Impl &operator=(const Impl &other) = delete;
Impl(Impl &&other) = delete;
Impl &operator=(Impl &&other) = delete;
~Impl() {
BN_CTX_free(big_num_context);
}
};
BigNumContext::BigNumContext() : impl_(make_unique<Impl>()) {
}
BigNumContext::BigNumContext(BigNumContext &&other) = default;
BigNumContext &BigNumContext::operator=(BigNumContext &&other) = default;
BigNumContext::~BigNumContext() = default;
class BigNum::Impl {
public:
BIGNUM *big_num;
Impl() : Impl(BN_new()) {
}
explicit Impl(BIGNUM *big_num) : big_num(big_num) {
LOG_IF(FATAL, big_num == nullptr);
}
Impl(const Impl &other) = delete;
Impl &operator=(const Impl &other) = delete;
Impl(Impl &&other) = delete;
Impl &operator=(Impl &&other) = delete;
~Impl() {
BN_clear_free(big_num);
}
};
BigNum::BigNum() : impl_(make_unique<Impl>()) {
}
BigNum::BigNum(const BigNum &other) : BigNum() {
*this = other;
}
BigNum &BigNum::operator=(const BigNum &other) {
CHECK(impl_ != nullptr);
CHECK(other.impl_ != nullptr);
BIGNUM *result = BN_copy(impl_->big_num, other.impl_->big_num);
LOG_IF(FATAL, result == nullptr);
return *this;
}
BigNum::BigNum(BigNum &&other) = default;
BigNum &BigNum::operator=(BigNum &&other) = default;
BigNum::~BigNum() = default;
BigNum BigNum::from_binary(Slice str) {
return BigNum(make_unique<Impl>(BN_bin2bn(str.ubegin(), narrow_cast<int>(str.size()), nullptr)));
}
BigNum BigNum::from_le_binary(Slice str) {
#if OPENSSL_VERSION_NUMBER >= 0x10100000L && !defined(LIBRESSL_VERSION_NUMBER)
return BigNum(make_unique<Impl>(BN_lebin2bn(str.ubegin(), narrow_cast<int>(str.size()), nullptr)));
#else
LOG(FATAL) << "Unsupported from_le_binary";
return BigNum();
#endif
}
Result<BigNum> BigNum::from_decimal(CSlice str) {
BigNum result;
int res = BN_dec2bn(&result.impl_->big_num, str.c_str());
if (res == 0 || static_cast<size_t>(res) != str.size()) {
return Status::Error(PSLICE() << "Failed to parse \"" << str << "\" as BigNum");
}
return result;
}
Result<BigNum> BigNum::from_hex(CSlice str) {
BigNum result;
int res = BN_hex2bn(&result.impl_->big_num, str.c_str());
if (res == 0 || static_cast<size_t>(res) != str.size()) {
return Status::Error(PSLICE() << "Failed to parse \"" << str << "\" as hexadecimal BigNum");
}
return result;
}
BigNum BigNum::from_raw(void *openssl_big_num) {
return BigNum(make_unique<Impl>(static_cast<BIGNUM *>(openssl_big_num)));
}
BigNum::BigNum(unique_ptr<Impl> &&impl) : impl_(std::move(impl)) {
}
void BigNum::ensure_const_time() {
BN_set_flags(impl_->big_num, BN_FLG_CONSTTIME);
}
int BigNum::get_num_bits() const {
return BN_num_bits(impl_->big_num);
}
int BigNum::get_num_bytes() const {
return BN_num_bytes(impl_->big_num);
}
void BigNum::set_bit(int num) {
int result = BN_set_bit(impl_->big_num, num);
LOG_IF(FATAL, result != 1);
}
void BigNum::clear_bit(int num) {
int result = BN_clear_bit(impl_->big_num, num);
LOG_IF(FATAL, result != 1);
}
bool BigNum::is_bit_set(int num) const {
return BN_is_bit_set(impl_->big_num, num) != 0;
}
bool BigNum::is_prime(BigNumContext &context) const {
int result = BN_is_prime_ex(impl_->big_num, BN_prime_checks, context.impl_->big_num_context, nullptr);
LOG_IF(FATAL, result == -1);
return result == 1;
}
void BigNum::operator+=(uint32 value) {
int result = BN_add_word(impl_->big_num, value);
LOG_IF(FATAL, result != 1);
}
void BigNum::operator-=(uint32 value) {
int result = BN_sub_word(impl_->big_num, value);
LOG_IF(FATAL, result != 1);
}
void BigNum::operator*=(uint32 value) {
int result = BN_mul_word(impl_->big_num, value);
LOG_IF(FATAL, result != 1);
}
void BigNum::operator/=(uint32 value) {
BN_ULONG result = BN_div_word(impl_->big_num, value);
LOG_IF(FATAL, result == static_cast<BN_ULONG>(-1));
}
uint32 BigNum::operator%(uint32 value) const {
BN_ULONG result = BN_mod_word(impl_->big_num, value);
LOG_IF(FATAL, result == static_cast<BN_ULONG>(-1));
return narrow_cast<uint32>(result);
}
void BigNum::set_value(uint32 new_value) {
if (new_value == 0) {
BN_zero(impl_->big_num);
} else {
int result = BN_set_word(impl_->big_num, new_value);
LOG_IF(FATAL, result != 1);
}
}
BigNum BigNum::clone() const {
BIGNUM *result = BN_dup(impl_->big_num);
LOG_IF(FATAL, result == nullptr);
return BigNum(make_unique<Impl>(result));
}
string BigNum::to_binary(int exact_size) const {
int num_size = get_num_bytes();
if (exact_size == -1) {
exact_size = num_size;
} else {
CHECK(exact_size >= num_size);
}
string res(exact_size, '\0');
BN_bn2bin(impl_->big_num, MutableSlice(res).ubegin() + (exact_size - num_size));
return res;
}
string BigNum::to_le_binary(int exact_size) const {
#if OPENSSL_VERSION_NUMBER >= 0x10100000L && !defined(LIBRESSL_VERSION_NUMBER)
int num_size = get_num_bytes();
if (exact_size == -1) {
exact_size = num_size;
} else {
CHECK(exact_size >= num_size);
}
string res(exact_size, '\0');
BN_bn2lebinpad(impl_->big_num, MutableSlice(res).ubegin(), exact_size);
return res;
#else
LOG(FATAL) << "Unsupported to_le_binary";
return "";
#endif
}
string BigNum::to_decimal() const {
char *result = BN_bn2dec(impl_->big_num);
CHECK(result != nullptr);
string res(result);
OPENSSL_free(result);
return res;
}
void BigNum::random(BigNum &r, int bits, int top, int bottom) {
int result = BN_rand(r.impl_->big_num, bits, top, bottom);
LOG_IF(FATAL, result != 1);
}
void BigNum::add(BigNum &r, const BigNum &a, const BigNum &b) {
int result = BN_add(r.impl_->big_num, a.impl_->big_num, b.impl_->big_num);
LOG_IF(FATAL, result != 1);
}
void BigNum::sub(BigNum &r, const BigNum &a, const BigNum &b) {
CHECK(r.impl_->big_num != a.impl_->big_num);
CHECK(r.impl_->big_num != b.impl_->big_num);
int result = BN_sub(r.impl_->big_num, a.impl_->big_num, b.impl_->big_num);
LOG_IF(FATAL, result != 1);
}
void BigNum::mul(BigNum &r, BigNum &a, BigNum &b, BigNumContext &context) {
int result = BN_mul(r.impl_->big_num, a.impl_->big_num, b.impl_->big_num, context.impl_->big_num_context);
LOG_IF(FATAL, result != 1);
}
void BigNum::mod_add(BigNum &r, BigNum &a, BigNum &b, const BigNum &m, BigNumContext &context) {
int result = BN_mod_add(r.impl_->big_num, a.impl_->big_num, b.impl_->big_num, m.impl_->big_num,
context.impl_->big_num_context);
LOG_IF(FATAL, result != 1);
}
void BigNum::mod_sub(BigNum &r, BigNum &a, BigNum &b, const BigNum &m, BigNumContext &context) {
int result = BN_mod_sub(r.impl_->big_num, a.impl_->big_num, b.impl_->big_num, m.impl_->big_num,
context.impl_->big_num_context);
LOG_IF(FATAL, result != 1);
}
void BigNum::mod_mul(BigNum &r, BigNum &a, BigNum &b, const BigNum &m, BigNumContext &context) {
int result = BN_mod_mul(r.impl_->big_num, a.impl_->big_num, b.impl_->big_num, m.impl_->big_num,
context.impl_->big_num_context);
LOG_IF(FATAL, result != 1);
}
void BigNum::mod_inverse(BigNum &r, BigNum &a, const BigNum &m, BigNumContext &context) {
auto result = BN_mod_inverse(r.impl_->big_num, a.impl_->big_num, m.impl_->big_num, context.impl_->big_num_context);
LOG_IF(FATAL, result != r.impl_->big_num);
}
void BigNum::div(BigNum *quotient, BigNum *remainder, const BigNum &dividend, const BigNum &divisor,
BigNumContext &context) {
auto q = quotient == nullptr ? nullptr : quotient->impl_->big_num;
auto r = remainder == nullptr ? nullptr : remainder->impl_->big_num;
if (q == nullptr && r == nullptr) {
return;
}
auto result = BN_div(q, r, dividend.impl_->big_num, divisor.impl_->big_num, context.impl_->big_num_context);
LOG_IF(FATAL, result != 1);
}
void BigNum::mod_exp(BigNum &r, const BigNum &a, const BigNum &p, const BigNum &m, BigNumContext &context) {
int result = BN_mod_exp(r.impl_->big_num, a.impl_->big_num, p.impl_->big_num, m.impl_->big_num,
context.impl_->big_num_context);
LOG_IF(FATAL, result != 1);
}
void BigNum::gcd(BigNum &r, BigNum &a, BigNum &b, BigNumContext &context) {
int result = BN_gcd(r.impl_->big_num, a.impl_->big_num, b.impl_->big_num, context.impl_->big_num_context);
LOG_IF(FATAL, result != 1);
}
int BigNum::compare(const BigNum &a, const BigNum &b) {
return BN_cmp(a.impl_->big_num, b.impl_->big_num);
}
StringBuilder &operator<<(StringBuilder &sb, const BigNum &bn) {
return sb << bn.to_decimal();
}
} // namespace td
#endif

138
tdutils/td/utils/BigNum.h Normal file
View file

@ -0,0 +1,138 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#if TD_HAVE_OPENSSL
#include "td/utils/Slice.h"
#include "td/utils/Status.h"
#include "td/utils/StringBuilder.h"
namespace td {
class BigNumContext {
public:
BigNumContext();
BigNumContext(const BigNumContext &other) = delete;
BigNumContext &operator=(const BigNumContext &other) = delete;
BigNumContext(BigNumContext &&other);
BigNumContext &operator=(BigNumContext &&other);
~BigNumContext();
private:
class Impl;
unique_ptr<Impl> impl_;
friend class BigNum;
};
class BigNum {
public:
BigNum();
BigNum(const BigNum &other);
BigNum &operator=(const BigNum &other);
BigNum(BigNum &&other);
BigNum &operator=(BigNum &&other);
~BigNum();
static BigNum from_binary(Slice str);
// Available only if OpenSSL >= 1.1.0
static BigNum from_le_binary(Slice str);
static Result<BigNum> from_decimal(CSlice str);
static Result<BigNum> from_hex(CSlice str);
static BigNum from_raw(void *openssl_big_num);
void set_value(uint32 new_value);
void ensure_const_time();
int get_num_bits() const;
int get_num_bytes() const;
void set_bit(int num);
void clear_bit(int num);
bool is_bit_set(int num) const;
bool is_prime(BigNumContext &context) const;
BigNum clone() const;
string to_binary(int exact_size = -1) const;
// Available only if OpenSSL >= 1.1.0
string to_le_binary(int exact_size = -1) const;
string to_decimal() const;
void operator+=(uint32 value);
void operator-=(uint32 value);
void operator*=(uint32 value);
void operator/=(uint32 value);
uint32 operator%(uint32 value) const;
static void random(BigNum &r, int bits, int top, int bottom);
static void add(BigNum &r, const BigNum &a, const BigNum &b);
static void sub(BigNum &r, const BigNum &a, const BigNum &b);
static void mul(BigNum &r, BigNum &a, BigNum &b, BigNumContext &context);
static void mod_add(BigNum &r, BigNum &a, BigNum &b, const BigNum &m, BigNumContext &context);
static void mod_sub(BigNum &r, BigNum &a, BigNum &b, const BigNum &m, BigNumContext &context);
static void mod_mul(BigNum &r, BigNum &a, BigNum &b, const BigNum &m, BigNumContext &context);
static void mod_inverse(BigNum &r, BigNum &a, const BigNum &m, BigNumContext &context);
static void div(BigNum *quotient, BigNum *remainder, const BigNum &dividend, const BigNum &divisor,
BigNumContext &context);
static void mod_exp(BigNum &r, const BigNum &a, const BigNum &p, const BigNum &m, BigNumContext &context);
static void gcd(BigNum &r, BigNum &a, BigNum &b, BigNumContext &context);
static int compare(const BigNum &a, const BigNum &b);
private:
class Impl;
unique_ptr<Impl> impl_;
explicit BigNum(unique_ptr<Impl> &&impl);
};
StringBuilder &operator<<(StringBuilder &sb, const BigNum &bn);
} // namespace td
#endif

226
tdutils/td/utils/BufferedFd.h Normal file
View file

@ -0,0 +1,226 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/buffer.h"
#include "td/utils/common.h"
#include "td/utils/format.h"
#include "td/utils/logging.h"
#include "td/utils/port/detail/PollableFd.h"
#include "td/utils/port/IoSlice.h"
#include "td/utils/Slice.h"
#include "td/utils/Span.h"
#include "td/utils/Status.h"
#include <limits>
namespace td {
// just reads from given reader and writes to given writer
template <class FdT>
class BufferedFdBase : public FdT {
public:
BufferedFdBase() = default;
explicit BufferedFdBase(FdT &&fd_);
// TODO: make move constructor and move assignment safer
Result<size_t> flush_read(size_t max_read = std::numeric_limits<size_t>::max()) TD_WARN_UNUSED_RESULT;
Result<size_t> flush_write() TD_WARN_UNUSED_RESULT;
bool need_flush_write(size_t at_least = 0) {
CHECK(write_);
write_->sync_with_writer();
return write_->size() > at_least;
}
size_t ready_for_flush_write() {
CHECK(write_);
write_->sync_with_writer();
return write_->size();
}
void set_input_writer(ChainBufferWriter *read) {
read_ = read;
}
void set_output_reader(ChainBufferReader *write) {
write_ = write;
}
private:
ChainBufferWriter *read_ = nullptr;
ChainBufferReader *write_ = nullptr;
};
template <class FdT>
class BufferedFd : public BufferedFdBase<FdT> {
using Parent = BufferedFdBase<FdT>;
ChainBufferWriter input_writer_;
ChainBufferReader input_reader_;
ChainBufferWriter output_writer_;
ChainBufferReader output_reader_;
void init();
void init_ptr();
public:
BufferedFd();
explicit BufferedFd(FdT &&fd_);
BufferedFd(BufferedFd &&);
BufferedFd &operator=(BufferedFd &&);
BufferedFd(const BufferedFd &) = delete;
BufferedFd &operator=(const BufferedFd &) = delete;
~BufferedFd();
void close();
Result<size_t> flush_read(size_t max_read = std::numeric_limits<size_t>::max()) TD_WARN_UNUSED_RESULT;
Result<size_t> flush_write() TD_WARN_UNUSED_RESULT;
// Yep, direct access to buffers. It is IO interface too.
ChainBufferReader &input_buffer();
ChainBufferWriter &output_buffer();
};
// IMPLEMENTATION
/*** BufferedFd ***/
template <class FdT>
BufferedFdBase<FdT>::BufferedFdBase(FdT &&fd_) : FdT(std::move(fd_)) {
}
template <class FdT>
Result<size_t> BufferedFdBase<FdT>::flush_read(size_t max_read) {
CHECK(read_);
size_t result = 0;
while (::td::can_read(*this) && max_read) {
MutableSlice slice = read_->prepare_append().truncate(max_read);
TRY_RESULT(x, FdT::read(slice));
slice.truncate(x);
read_->confirm_append(x);
result += x;
max_read -= x;
}
return result;
}
template <class FdT>
Result<size_t> BufferedFdBase<FdT>::flush_write() {
// TODO: sync on demand
write_->sync_with_writer();
size_t result = 0;
while (!write_->empty() && ::td::can_write(*this)) {
constexpr size_t buf_size = 20;
IoSlice buf[buf_size];
auto it = write_->clone();
size_t buf_i;
for (buf_i = 0; buf_i < buf_size; buf_i++) {
Slice slice = it.prepare_read();
if (slice.empty()) {
break;
}
buf[buf_i] = as_io_slice(slice);
it.confirm_read(slice.size());
}
TRY_RESULT(x, FdT::writev(Span<IoSlice>(buf, buf_i)));
write_->advance(x);
result += x;
}
return result;
}
/*** BufferedFd ***/
template <class FdT>
void BufferedFd<FdT>::init() {
input_reader_ = input_writer_.extract_reader();
output_reader_ = output_writer_.extract_reader();
init_ptr();
}
template <class FdT>
void BufferedFd<FdT>::init_ptr() {
this->set_input_writer(&input_writer_);
this->set_output_reader(&output_reader_);
}
template <class FdT>
BufferedFd<FdT>::BufferedFd() {
init();
}
template <class FdT>
BufferedFd<FdT>::BufferedFd(FdT &&fd_) : Parent(std::move(fd_)) {
init();
}
template <class FdT>
BufferedFd<FdT>::BufferedFd(BufferedFd &&from) {
*this = std::move(from);
}
template <class FdT>
BufferedFd<FdT> &BufferedFd<FdT>::operator=(BufferedFd &&from) {
FdT::operator=(std::move(static_cast<FdT &>(from)));
input_reader_ = std::move(from.input_reader_);
input_writer_ = std::move(from.input_writer_);
output_reader_ = std::move(from.output_reader_);
output_writer_ = std::move(from.output_writer_);
init_ptr();
return *this;
}
template <class FdT>
BufferedFd<FdT>::~BufferedFd() {
close();
}
template <class FdT>
void BufferedFd<FdT>::close() {
FdT::close();
// TODO: clear buffers
}
template <class FdT>
Result<size_t> BufferedFd<FdT>::flush_read(size_t max_read) {
TRY_RESULT(result, Parent::flush_read(max_read));
if (result) {
// TODO: faster sync is possible if you owns writer.
input_reader_.sync_with_writer();
LOG(DEBUG) << "Flush read: +" << format::as_size(result) << tag("total", format::as_size(input_reader_.size()));
}
return result;
}
template <class FdT>
Result<size_t> BufferedFd<FdT>::flush_write() {
TRY_RESULT(result, Parent::flush_write());
if (result) {
LOG(DEBUG) << "Flush write: +" << format::as_size(result) << tag("left", format::as_size(output_reader_.size()));
}
return result;
}
// Yep, direct access to buffers. It is IO interface too.
template <class FdT>
ChainBufferReader &BufferedFd<FdT>::input_buffer() {
return input_reader_;
}
template <class FdT>
ChainBufferWriter &BufferedFd<FdT>::output_buffer() {
return output_writer_;
}
} // namespace td

73
tdutils/td/utils/BufferedReader.h Normal file
View file

@ -0,0 +1,73 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#include "td/utils/port/FileFd.h"
#include "td/utils/Slice.h"
#include "td/utils/Status.h"
namespace td {
class BufferedReader {
public:
explicit BufferedReader(FileFd &file, size_t buff_size = 8152)
: file_(file), buff_(buff_size), begin_pos_(0), end_pos_(0) {
}
Result<size_t> read(MutableSlice slice) TD_WARN_UNUSED_RESULT;
private:
FileFd &file_;
vector<char> buff_;
size_t begin_pos_;
size_t end_pos_;
};
inline Result<size_t> BufferedReader::read(MutableSlice slice) {
size_t available = end_pos_ - begin_pos_;
if (available >= slice.size()) {
// have enough data in buffer
slice.copy_from({&buff_[begin_pos_], slice.size()});
begin_pos_ += slice.size();
return slice.size();
}
if (available) {
slice.copy_from({&buff_[begin_pos_], available});
begin_pos_ += available;
slice.remove_prefix(available);
}
if (slice.size() > buff_.size() / 2) {
TRY_RESULT(result, file_.read(slice));
return result + available;
}
TRY_RESULT(result, file_.read({&buff_[0], buff_.size()}));
begin_pos_ = 0;
end_pos_ = result;
size_t left = min(end_pos_, slice.size());
slice.copy_from({&buff_[0], left});
begin_pos_ = left;
return left + available;
}
} // namespace td

29
tdutils/td/utils/BufferedUdp.cpp Normal file
View file

@ -0,0 +1,29 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/utils/BufferedUdp.h"
char disable_linker_warning_about_empty_file_buffered_udp_cpp TD_UNUSED;
namespace td {
#if TD_PORT_POSIX
TD_THREAD_LOCAL detail::UdpReader *BufferedUdp::udp_reader_;
#endif
} // namespace td

184
tdutils/td/utils/BufferedUdp.h Normal file
View file

@ -0,0 +1,184 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/buffer.h"
#include "td/utils/common.h"
#include "td/utils/logging.h"
#include "td/utils/optional.h"
#include "td/utils/port/detail/PollableFd.h"
#include "td/utils/port/thread_local.h"
#include "td/utils/port/UdpSocketFd.h"
#include "td/utils/Span.h"
#include "td/utils/Status.h"
#include "td/utils/VectorQueue.h"
#include <array>
namespace td {
#if TD_PORT_POSIX
namespace detail {
class UdpWriter {
public:
static Status write_once(UdpSocketFd &fd, VectorQueue<UdpMessage> &queue) TD_WARN_UNUSED_RESULT {
std::array<UdpSocketFd::OutboundMessage, 16> messages;
auto to_send = queue.as_span();
size_t to_send_n = td::min(messages.size(), to_send.size());
to_send.truncate(to_send_n);
for (size_t i = 0; i < to_send_n; i++) {
messages[i].to = &to_send[i].address;
messages[i].data = to_send[i].data.as_slice();
}
size_t cnt;
auto status = fd.send_messages(::td::Span<UdpSocketFd::OutboundMessage>(messages).truncate(to_send_n), cnt);
queue.pop_n(cnt);
return status;
}
};
class UdpReaderHelper {
public:
void init_inbound_message(UdpSocketFd::InboundMessage &message) {
message.from = &message_.address;
message.error = &message_.error;
if (buffer_.size() < MAX_PACKET_SIZE) {
buffer_ = BufferSlice(RESERVED_SIZE);
}
CHECK(buffer_.size() >= MAX_PACKET_SIZE);
message.data = buffer_.as_slice().truncate(MAX_PACKET_SIZE);
}
UdpMessage extract_udp_message(UdpSocketFd::InboundMessage &message) {
message_.data = buffer_.from_slice(message.data);
auto size = message_.data.size();
size = (size + 7) & ~7;
CHECK(size <= MAX_PACKET_SIZE);
buffer_.confirm_read(size);
return std::move(message_);
}
private:
enum : size_t { MAX_PACKET_SIZE = 2048, RESERVED_SIZE = MAX_PACKET_SIZE * 8 };
UdpMessage message_;
BufferSlice buffer_;
};
// One for thread is enough
class UdpReader {
public:
UdpReader() {
for (size_t i = 0; i < messages_.size(); i++) {
helpers_[i].init_inbound_message(messages_[i]);
}
}
Status read_once(UdpSocketFd &fd, VectorQueue<UdpMessage> &queue) TD_WARN_UNUSED_RESULT {
for (size_t i = 0; i < messages_.size(); i++) {
CHECK(messages_[i].data.size() == 2048);
}
size_t cnt = 0;
auto status = fd.receive_messages(messages_, cnt);
for (size_t i = 0; i < cnt; i++) {
queue.push(helpers_[i].extract_udp_message(messages_[i]));
helpers_[i].init_inbound_message(messages_[i]);
}
for (size_t i = cnt; i < messages_.size(); i++) {
LOG_CHECK(messages_[i].data.size() == 2048)
<< " cnt = " << cnt << " i = " << i << " size = " << messages_[i].data.size() << " status = " << status;
}
if (status.is_error() && !UdpSocketFd::is_critical_read_error(status)) {
queue.push(UdpMessage{{}, {}, std::move(status)});
}
return status;
}
private:
enum : size_t { BUFFER_SIZE = 16 };
std::array<UdpSocketFd::InboundMessage, BUFFER_SIZE> messages_;
std::array<UdpReaderHelper, BUFFER_SIZE> helpers_;
};
} // namespace detail
#endif
class BufferedUdp : public UdpSocketFd {
public:
explicit BufferedUdp(UdpSocketFd fd) : UdpSocketFd(std::move(fd)) {
}
#if TD_PORT_POSIX
Result<optional<UdpMessage>> receive() {
if (input_.empty() && can_read(*this)) {
TRY_STATUS(flush_read_once());
}
if (input_.empty()) {
return optional<UdpMessage>();
}
return input_.pop();
}
void send(UdpMessage message) {
output_.push(std::move(message));
}
Status flush_send() {
Status status;
while (status.is_ok() && can_write(*this) && !output_.empty()) {
status = flush_send_once();
}
return status;
}
#endif
UdpSocketFd move_as_udp_socket_fd() {
return std::move(as_fd());
}
UdpSocketFd &as_fd() {
return *static_cast<UdpSocketFd *>(this);
}
private:
#if TD_PORT_POSIX
VectorQueue<UdpMessage> input_;
VectorQueue<UdpMessage> output_;
VectorQueue<UdpMessage> &input() {
return input_;
}
VectorQueue<UdpMessage> &output() {
return output_;
}
Status flush_send_once() TD_WARN_UNUSED_RESULT {
return detail::UdpWriter::write_once(as_fd(), output_);
}
Status flush_read_once() TD_WARN_UNUSED_RESULT {
init_thread_local<detail::UdpReader>(udp_reader_);
return udp_reader_->read_once(as_fd(), input_);
}
static TD_THREAD_LOCAL detail::UdpReader *udp_reader_;
#endif
};
} // namespace td

303
tdutils/td/utils/ByteFlow.h Normal file
View file

@ -0,0 +1,303 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/buffer.h"
#include "td/utils/common.h"
#include "td/utils/Status.h"
namespace td {
class ByteFlowInterface {
public:
virtual void close_input(Status status) = 0;
virtual void wakeup() = 0;
virtual void set_parent(ByteFlowInterface &other) = 0;
virtual void set_input(ChainBufferReader *input) = 0;
virtual size_t get_need_size() = 0;
ByteFlowInterface() = default;
ByteFlowInterface(const ByteFlowInterface &) = delete;
ByteFlowInterface &operator=(const ByteFlowInterface &) = delete;
ByteFlowInterface(ByteFlowInterface &&) = default;
ByteFlowInterface &operator=(ByteFlowInterface &&) = default;
virtual ~ByteFlowInterface() = default;
};
class ByteFlowBaseCommon : public ByteFlowInterface {
public:
ByteFlowBaseCommon() = default;
void close_input(Status status) final {
if (status.is_error()) {
finish(std::move(status));
} else {
is_input_active_ = false;
wakeup();
}
}
void wakeup() final {
if (stop_flag_) {
return;
}
input_->sync_with_writer();
if (waiting_flag_) {
if (!is_input_active_) {
finish(Status::OK());
}
return;
}
if (is_input_active_) {
if (need_size_ != 0 && input_->size() < need_size_) {
return;
}
}
need_size_ = 0;
loop();
}
size_t get_need_size() final {
return need_size_;
}
virtual void loop() = 0;
protected:
bool waiting_flag_ = false;
ChainBufferReader *input_ = nullptr;
bool is_input_active_ = true;
size_t need_size_ = 0;
void finish(Status status) {
stop_flag_ = true;
need_size_ = 0;
if (parent_) {
parent_->close_input(std::move(status));
parent_ = nullptr;
}
}
void set_need_size(size_t need_size) {
need_size_ = need_size;
}
void on_output_updated() {
if (parent_) {
parent_->wakeup();
}
}
void consume_input() {
waiting_flag_ = true;
if (!is_input_active_) {
finish(Status::OK());
}
}
private:
ByteFlowInterface *parent_ = nullptr;
bool stop_flag_ = false;
friend class ByteFlowBase;
friend class ByteFlowInplaceBase;
};
class ByteFlowBase : public ByteFlowBaseCommon {
public:
ByteFlowBase() = default;
void set_input(ChainBufferReader *input) final {
input_ = input;
}
void set_parent(ByteFlowInterface &other) final {
parent_ = &other;
parent_->set_input(&output_reader_);
}
void loop() override = 0;
// ChainBufferWriter &get_output() {
// return output_;
//}
protected:
ChainBufferWriter output_;
ChainBufferReader output_reader_ = output_.extract_reader();
};
class ByteFlowInplaceBase : public ByteFlowBaseCommon {
public:
ByteFlowInplaceBase() = default;
void set_input(ChainBufferReader *input) final {
input_ = input;
output_ = ChainBufferReader(input_->begin().clone(), input_->begin().clone(), false);
}
void set_parent(ByteFlowInterface &other) final {
parent_ = &other;
parent_->set_input(&output_);
}
void loop() override = 0;
ChainBufferReader &get_output() {
return output_;
}
protected:
ChainBufferReader output_;
};
inline ByteFlowInterface &operator>>(ByteFlowInterface &from, ByteFlowInterface &to) {
from.set_parent(to);
return to;
}
class ByteFlowSource : public ByteFlowInterface {
public:
ByteFlowSource() = default;
explicit ByteFlowSource(ChainBufferReader *buffer) : buffer_(buffer) {
}
ByteFlowSource(ByteFlowSource &&other) : buffer_(other.buffer_), parent_(other.parent_) {
other.buffer_ = nullptr;
other.parent_ = nullptr;
}
ByteFlowSource &operator=(ByteFlowSource &&other) {
buffer_ = other.buffer_;
parent_ = other.parent_;
other.buffer_ = nullptr;
other.parent_ = nullptr;
return *this;
}
ByteFlowSource(const ByteFlowSource &) = delete;
ByteFlowSource &operator=(const ByteFlowSource &) = delete;
~ByteFlowSource() override = default;
void set_input(ChainBufferReader *) final {
UNREACHABLE();
}
void set_parent(ByteFlowInterface &parent) final {
CHECK(parent_ == nullptr);
parent_ = &parent;
parent_->set_input(buffer_);
}
void close_input(Status status) final {
CHECK(parent_);
parent_->close_input(std::move(status));
parent_ = nullptr;
}
void wakeup() final {
CHECK(parent_);
parent_->wakeup();
}
size_t get_need_size() final {
if (parent_ == nullptr) {
return 0;
}
return parent_->get_need_size();
}
private:
ChainBufferReader *buffer_ = nullptr;
ByteFlowInterface *parent_ = nullptr;
};
class ByteFlowSink : public ByteFlowInterface {
public:
void set_input(ChainBufferReader *input) final {
CHECK(buffer_ == nullptr);
buffer_ = input;
}
void set_parent(ByteFlowInterface & /*parent*/) final {
UNREACHABLE();
}
void close_input(Status status) final {
CHECK(active_);
active_ = false;
status_ = std::move(status);
buffer_->sync_with_writer();
}
void wakeup() final {
buffer_->sync_with_writer();
}
size_t get_need_size() final {
UNREACHABLE();
return 0;
}
bool is_ready() {
return !active_;
}
Status &status() {
return status_;
}
ChainBufferReader *result() {
CHECK(is_ready() && status().is_ok());
return buffer_;
}
ChainBufferReader *get_output() {
return buffer_;
}
private:
bool active_ = true;
Status status_;
ChainBufferReader *buffer_ = nullptr;
};
class ByteFlowMoveSink : public ByteFlowInterface {
public:
ByteFlowMoveSink() = default;
explicit ByteFlowMoveSink(ChainBufferWriter *output) {
set_output(output);
}
void set_input(ChainBufferReader *input) final {
CHECK(!input_);
input_ = input;
}
void set_parent(ByteFlowInterface & /*parent*/) final {
UNREACHABLE();
}
void close_input(Status status) final {
CHECK(active_);
active_ = false;
status_ = std::move(status);
wakeup();
}
void wakeup() final {
input_->sync_with_writer();
output_->append(*input_);
}
size_t get_need_size() final {
UNREACHABLE();
return 0;
}
void set_output(ChainBufferWriter *output) {
CHECK(!output_);
output_ = output;
}
bool is_ready() {
return !active_;
}
Status &status() {
return status_;
}
private:
bool active_ = true;
Status status_;
ChainBufferReader *input_ = nullptr;
ChainBufferWriter *output_ = nullptr;
};
} // namespace td

78
tdutils/td/utils/CancellationToken.h Normal file
View file

@ -0,0 +1,78 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include <atomic>
#include <memory>
namespace td {
namespace detail {
struct RawCancellationToken {
std::atomic<bool> is_cancelled_{false};
};
} // namespace detail
class CancellationToken {
public:
explicit operator bool() const {
return token_->is_cancelled_.load(std::memory_order_acquire);
}
explicit CancellationToken(std::shared_ptr<detail::RawCancellationToken> token) : token_(std::move(token)) {
}
private:
std::shared_ptr<detail::RawCancellationToken> token_;
};
class CancellationTokenSource {
public:
CancellationTokenSource() = default;
CancellationTokenSource(CancellationTokenSource &&other) : token_(std::move(other.token_)) {
}
CancellationTokenSource &operator=(CancellationTokenSource &&other) {
cancel();
token_ = std::move(other.token_);
return *this;
}
CancellationTokenSource(const CancellationTokenSource &other) = delete;
CancellationTokenSource &operator=(const CancellationTokenSource &other) = delete;
~CancellationTokenSource() {
cancel();
}
CancellationToken get_cancellation_token() {
if (!token_) {
token_ = std::make_shared<detail::RawCancellationToken>();
}
return CancellationToken(token_);
}
void cancel() {
if (!token_) {
return;
}
token_->is_cancelled_.store(true, std::memory_order_release);
token_.reset();
}
private:
std::shared_ptr<detail::RawCancellationToken> token_;
};
} // namespace td

73
tdutils/td/utils/ChangesProcessor.h Normal file
View file

@ -0,0 +1,73 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#include <utility>
namespace td {
// Process changes after they are finished in order of addition
template <class DataT>
class ChangesProcessor {
public:
using Id = uint64;
void clear() {
offset_ += data_array_.size();
ready_i_ = 0;
data_array_.clear();
}
template <class FromDataT>
Id add(FromDataT &&data) {
auto res = offset_ + data_array_.size();
data_array_.emplace_back(std::forward<DataT>(data), false);
return static_cast<Id>(res);
}
template <class F>
void finish(Id token, F &&func) {
size_t pos = static_cast<size_t>(token) - offset_;
if (pos >= data_array_.size()) {
return;
}
data_array_[pos].second = true;
while (ready_i_ < data_array_.size() && data_array_[ready_i_].second == true) {
func(std::move(data_array_[ready_i_].first));
ready_i_++;
}
try_compactify();
}
private:
size_t offset_ = 1;
size_t ready_i_ = 0;
std::vector<std::pair<DataT, bool>> data_array_;
void try_compactify() {
if (ready_i_ > 5 && ready_i_ * 2 > data_array_.size()) {
data_array_.erase(data_array_.begin(), data_array_.begin() + ready_i_);
offset_ += ready_i_;
ready_i_ = 0;
}
}
};
} // namespace td

180
tdutils/td/utils/Closure.h Normal file
View file

@ -0,0 +1,180 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#include "td/utils/invoke.h"
#include "td/utils/logging.h"
#include <cstdlib>
#include <tuple>
#include <type_traits>
#include <utility>
//
// Essentially we have:
// (ActorT::func, arg1, arg2, ..., argn)
// We want to call:
// actor->func(arg1, arg2, ..., argn)
// And in some cases we would like to delay this call.
//
// First attempt would be
// [a1=arg1, a2=arg2, ..., an=argn](ActorT *actor) {
// actor->func(a1, a2, ..., an)
// }
//
// But there are some difficulties with elimitation on unnecessary copies.
// We want to use move constructor when it is possible
//
// We may pass
// Tmp. Temporary / rvalue reference
// Var. Variable / reference
// CnstRef. const reference
//
//
// Function may expect
// Val. Value
// CnstRef. const reference
// Ref. rvalue reverence / reference
//
// TODO:
// Immediate call / Delayed call
// Tmp->Val move / move->move
// Tmp->CnstRef + / move->+
// Tmp->Ref + / move->+
// Var->Val copy / copy->move
// Var->CnstRef + / copy->
// Var->Ref + / copy->+ // khm. It will complile, but won't work
//
// So I will use common idiom: forward references
// If delay is needed, just std::forward data to temporary storage, and std::move them when call is executed.
//
//
// create_immediate_closure(&ActorT::func, arg1, arg2, ..., argn).run(actor)
// to_delayed_closure(std::move(immediate)).run(actor)
namespace td {
template <class ActorT, class FunctionT, class... ArgsT>
class DelayedClosure;
template <class ActorT, class FunctionT, class... ArgsT>
class ImmediateClosure {
public:
using Delayed = DelayedClosure<ActorT, FunctionT, ArgsT...>;
friend Delayed;
using ActorType = ActorT;
// no &&. just save references as references.
explicit ImmediateClosure(FunctionT func, ArgsT... args) : args(func, std::forward<ArgsT>(args)...) {
}
private:
std::tuple<FunctionT, ArgsT...> args;
public:
auto run(ActorT *actor) -> decltype(mem_call_tuple(actor, std::move(args))) {
return mem_call_tuple(actor, std::move(args));
}
};
template <class ActorT, class ResultT, class... DestArgsT, class... SrcArgsT>
ImmediateClosure<ActorT, ResultT (ActorT::*)(DestArgsT...), SrcArgsT &&...> create_immediate_closure(
ResultT (ActorT::*func)(DestArgsT...), SrcArgsT &&... args) {
return ImmediateClosure<ActorT, ResultT (ActorT::*)(DestArgsT...), SrcArgsT &&...>(func,
std::forward<SrcArgsT>(args)...);
}
template <class ActorT, class FunctionT, class... ArgsT>
class DelayedClosure {
public:
using ActorType = ActorT;
using Delayed = DelayedClosure<ActorT, FunctionT, ArgsT...>;
DelayedClosure clone() const {
return do_clone(*this);
}
explicit DelayedClosure(ImmediateClosure<ActorT, FunctionT, ArgsT...> &&other) : args(std::move(other.args)) {
}
explicit DelayedClosure(FunctionT func, ArgsT... args) : args(func, std::forward<ArgsT>(args)...) {
}
template <class F>
void for_each(const F &f) {
tuple_for_each(args, f);
}
private:
using ArgsStorageT = std::tuple<FunctionT, typename std::decay<ArgsT>::type...>;
ArgsStorageT args;
template <class FromActorT, class FromFunctionT, class... FromArgsT>
explicit DelayedClosure(const DelayedClosure<FromActorT, FromFunctionT, FromArgsT...> &other,
std::enable_if_t<LogicAnd<std::is_copy_constructible<FromArgsT>::value...>::value, int> = 0)
: args(other.args) {
}
template <class FromActorT, class FromFunctionT, class... FromArgsT>
explicit DelayedClosure(
const DelayedClosure<FromActorT, FromFunctionT, FromArgsT...> &other,
std::enable_if_t<!LogicAnd<std::is_copy_constructible<FromArgsT>::value...>::value, int> = 0) {
LOG(FATAL) << "Deleted constructor";
std::abort();
}
template <class FromActorT, class FromFunctionT, class... FromArgsT>
std::enable_if_t<!LogicAnd<std::is_copy_constructible<FromArgsT>::value...>::value,
DelayedClosure<FromActorT, FromFunctionT, FromArgsT...>>
do_clone(const DelayedClosure<FromActorT, FromFunctionT, FromArgsT...> &value) const {
LOG(FATAL) << "Trying to clone DelayedClosure that contains noncopyable elements";
std::abort();
}
template <class FromActorT, class FromFunctionT, class... FromArgsT>
std::enable_if_t<LogicAnd<std::is_copy_constructible<FromArgsT>::value...>::value,
DelayedClosure<FromActorT, FromFunctionT, FromArgsT...>>
do_clone(const DelayedClosure<FromActorT, FromFunctionT, FromArgsT...> &value) const {
return DelayedClosure<FromActorT, FromFunctionT, FromArgsT...>(value);
}
public:
auto run(ActorT *actor) -> decltype(mem_call_tuple(actor, std::move(args))) {
return mem_call_tuple(actor, std::move(args));
}
};
template <class... ArgsT>
typename ImmediateClosure<ArgsT...>::Delayed to_delayed_closure(ImmediateClosure<ArgsT...> &&other) {
return typename ImmediateClosure<ArgsT...>::Delayed(std::move(other));
}
template <class... ArgsT>
DelayedClosure<ArgsT...> to_delayed_closure(DelayedClosure<ArgsT...> &&other) {
return std::move(other);
}
template <class ActorT, class ResultT, class... DestArgsT, class... SrcArgsT>
auto create_delayed_closure(ResultT (ActorT::*func)(DestArgsT...), SrcArgsT &&... args) {
return DelayedClosure<ActorT, ResultT (ActorT::*)(DestArgsT...), SrcArgsT &&...>(func,
std::forward<SrcArgsT>(args)...);
}
} // namespace td

334
tdutils/td/utils/ConcurrentHashTable.h Normal file
View file

@ -0,0 +1,334 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#include "td/utils/HazardPointers.h"
#include "td/utils/logging.h"
#include "td/utils/port/thread_local.h"
#include <atomic>
#include <condition_variable>
#include <mutex>
namespace td {
// AtomicHashArray<KeyT, ValueT>
// Building block for other concurrent hash maps
//
// Support one operation:
// template <class F>
// bool with_value(KeyT key, bool should_create, F &&func);
//
// Finds slot for key, and call func(value)
// Creates slot if should_create is true.
// Returns true if func was called.
//
// Concurrent calls with the same key may result in concurrent calls to func(value)
// It is responsibility of the caller to handle such races.
//
// Key should already be random
// It is responsibility of the caller to provide unique random key.
// One may use injective hash function, or handle collisions in some other way.
template <class KeyT, class ValueT>
class AtomicHashArray {
public:
explicit AtomicHashArray(size_t n) : nodes_(n) {
}
struct Node {
std::atomic<KeyT> key{KeyT{}};
ValueT value{};
};
size_t size() const {
return nodes_.size();
}
Node &node_at(size_t i) {
return nodes_[i];
}
static KeyT empty_key() {
return KeyT{};
}
template <class F>
bool with_value(KeyT key, bool should_create, F &&f) {
DCHECK(key != empty_key());
size_t pos = static_cast<size_t>(key) % nodes_.size();
size_t n = td::min(td::max(static_cast<size_t>(300), nodes_.size() / 16 + 2), nodes_.size());
for (size_t i = 0; i < n; i++) {
pos++;
if (pos >= nodes_.size()) {
pos = 0;
}
auto &node = nodes_[pos];
while (true) {
auto node_key = node.key.load(std::memory_order_acquire);
if (node_key == empty_key()) {
if (!should_create) {
return false;
}
KeyT expected_key = empty_key();
if (node.key.compare_exchange_strong(expected_key, key, std::memory_order_relaxed,
std::memory_order_relaxed)) {
f(node.value);
return true;
}
} else if (node_key == key) {
f(node.value);
return true;
} else {
break;
}
}
}
return false;
}
private:
std::vector<Node> nodes_;
};
// Simple concurrent hash map with multiple limitations
template <class KeyT, class ValueT>
class ConcurrentHashMap {
using HashMap = AtomicHashArray<KeyT, std::atomic<ValueT>>;
static HazardPointers<HashMap> hp_;
public:
explicit ConcurrentHashMap(size_t n = 32) {
n = 1;
hash_map_.store(make_unique<HashMap>(n).release());
}
ConcurrentHashMap(const ConcurrentHashMap &) = delete;
ConcurrentHashMap &operator=(const ConcurrentHashMap &) = delete;
ConcurrentHashMap(ConcurrentHashMap &&) = delete;
ConcurrentHashMap &operator=(ConcurrentHashMap &&) = delete;
~ConcurrentHashMap() {
unique_ptr<HashMap>(hash_map_.load());
}
static std::string get_name() {
return "ConcurrrentHashMap";
}
static KeyT empty_key() {
return KeyT{};
}
static ValueT empty_value() {
return ValueT{};
}
static ValueT migrate_value() {
return (ValueT)(1); // c-style conversion because reinterpret_cast<int>(1) is CE in MSVC
}
ValueT insert(KeyT key, ValueT value) {
CHECK(key != empty_key());
CHECK(value != migrate_value());
typename HazardPointers<HashMap>::Holder holder(hp_, get_thread_id(), 0);
while (true) {
auto hash_map = holder.protect(hash_map_);
if (!hash_map) {
do_migrate(nullptr);
continue;
}
bool ok = false;
ValueT inserted_value;
hash_map->with_value(key, true, [&](auto &node_value) {
ValueT expected_value = this->empty_value();
if (node_value.compare_exchange_strong(expected_value, value, std::memory_order_release,
std::memory_order_acquire)) {
ok = true;
inserted_value = value;
} else {
if (expected_value == this->migrate_value()) {
ok = false;
} else {
ok = true;
inserted_value = expected_value;
}
}
});
if (ok) {
return inserted_value;
}
do_migrate(hash_map);
}
}
ValueT find(KeyT key, ValueT value) {
typename HazardPointers<HashMap>::Holder holder(hp_, get_thread_id(), 0);
while (true) {
auto hash_map = holder.protect(hash_map_);
if (!hash_map) {
do_migrate(nullptr);
continue;
}
bool has_value = hash_map->with_value(
key, false, [&](auto &node_value) { value = node_value.load(std::memory_order_acquire); });
if (!has_value || value != migrate_value()) {
return value;
}
do_migrate(hash_map);
}
}
template <class F>
void for_each(F &&f) {
auto hash_map = hash_map_.load();
CHECK(hash_map);
auto size = hash_map->size();
for (size_t i = 0; i < size; i++) {
auto &node = hash_map->node_at(i);
auto key = node.key.load(std::memory_order_relaxed);
auto value = node.value.load(std::memory_order_relaxed);
if (key != empty_key()) {
CHECK(value != migrate_value());
if (value != empty_value()) {
f(key, value);
}
}
}
}
private:
// use no padding intentionally
std::atomic<HashMap *> hash_map_{nullptr};
std::mutex migrate_mutex_;
std::condition_variable migrate_cv_;
int migrate_cnt_{0};
int migrate_generation_{0};
HashMap *migrate_from_hash_map_{nullptr};
HashMap *migrate_to_hash_map_{nullptr};
struct Task {
size_t begin;
size_t end;
bool empty() const {
return begin >= end;
}
size_t size() const {
if (empty()) {
return 0;
}
return end - begin;
}
};
struct TaskCreator {
size_t chunk_size;
size_t size;
std::atomic<size_t> pos{0};
Task create() {
auto i = pos++;
auto begin = i * chunk_size;
auto end = begin + chunk_size;
if (end > size) {
end = size;
}
return {begin, end};
}
};
TaskCreator task_creator;
void do_migrate(HashMap *ptr) {
//LOG(ERROR) << "In do_migrate: " << ptr;
std::unique_lock<std::mutex> lock(migrate_mutex_);
if (hash_map_.load() != ptr) {
return;
}
init_migrate();
CHECK(!ptr || migrate_from_hash_map_ == ptr);
migrate_cnt_++;
auto migrate_generation = migrate_generation_;
lock.unlock();
run_migrate();
lock.lock();
migrate_cnt_--;
if (migrate_cnt_ == 0) {
finish_migrate();
}
migrate_cv_.wait(lock, [&] { return migrate_generation_ != migrate_generation; });
}
void finish_migrate() {
//LOG(ERROR) << "In finish_migrate";
hash_map_.store(migrate_to_hash_map_);
hp_.retire(get_thread_id(), migrate_from_hash_map_);
migrate_from_hash_map_ = nullptr;
migrate_to_hash_map_ = nullptr;
migrate_generation_++;
migrate_cv_.notify_all();
}
void init_migrate() {
if (migrate_from_hash_map_ != nullptr) {
return;
}
//LOG(ERROR) << "In init_migrate";
CHECK(migrate_cnt_ == 0);
migrate_generation_++;
migrate_from_hash_map_ = hash_map_.exchange(nullptr);
auto new_size = migrate_from_hash_map_->size() * 2;
migrate_to_hash_map_ = make_unique<HashMap>(new_size).release();
task_creator.chunk_size = 100;
task_creator.size = migrate_from_hash_map_->size();
task_creator.pos = 0;
}
void run_migrate() {
//LOG(ERROR) << "In run_migrate";
size_t cnt = 0;
while (true) {
auto task = task_creator.create();
cnt += task.size();
if (task.empty()) {
break;
}
run_task(task);
}
//LOG(ERROR) << "In run_migrate " << cnt;
}
void run_task(Task task) {
for (auto i = task.begin; i < task.end; i++) {
auto &node = migrate_from_hash_map_->node_at(i);
auto old_value = node.value.exchange(migrate_value(), std::memory_order_acq_rel);
if (old_value == 0) {
continue;
}
auto node_key = node.key.load(std::memory_order_relaxed);
//LOG(ERROR) << node_key << " " << node_key;
auto ok = migrate_to_hash_map_->with_value(
node_key, true, [&](auto &node_value) { node_value.store(old_value, std::memory_order_relaxed); });
LOG_CHECK(ok) << "Migration overflow";
}
}
};
template <class KeyT, class ValueT>
td::HazardPointers<typename ConcurrentHashMap<KeyT, ValueT>::HashMap> ConcurrentHashMap<KeyT, ValueT>::hp_(64);
} // namespace td

160
tdutils/td/utils/Container.h Normal file
View file

@ -0,0 +1,160 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#include <limits>
namespace td {
// 1. Allocates all objects in vector. (but vector never shrinks)
// 2. Id is safe way to reach this object.
// 3. All ids are unique.
// 4. All ids are non-zero.
template <class DataT>
class Container {
public:
using Id = uint64;
DataT *get(Id id) {
int32 slot_id = decode_id(id);
if (slot_id == -1) {
return nullptr;
}
return &slots_[slot_id].data;
}
void erase(Id id) {
int32 slot_id = decode_id(id);
if (slot_id == -1) {
return;
}
release(slot_id);
}
DataT extract(Id id) {
int32 slot_id = decode_id(id);
CHECK(slot_id != -1);
auto res = std::move(slots_[slot_id].data);
release(slot_id);
return res;
}
Id create(DataT &&data = DataT(), uint8 type = 0) {
int32 id = store(std::move(data), type);
return encode_id(id);
}
Id reset_id(Id id) {
int32 slot_id = decode_id(id);
CHECK(slot_id != -1);
inc_generation(slot_id);
return encode_id(slot_id);
}
static uint8 type_from_id(Id id) {
return static_cast<uint8>(id);
}
vector<Id> ids() {
vector<bool> is_bad(slots_.size(), false);
for (auto id : empty_slots_) {
is_bad[id] = true;
}
vector<Id> res;
for (size_t i = 0, n = slots_.size(); i < n; i++) {
if (!is_bad[i]) {
res.push_back(encode_id(static_cast<int32>(i)));
}
}
return res;
}
template <class F>
void for_each(const F &f) {
auto ids = this->ids();
for (auto id : ids) {
f(id, *get(id));
}
}
size_t size() const {
CHECK(empty_slots_.size() <= slots_.size());
return slots_.size() - empty_slots_.size();
}
bool empty() const {
return size() == 0;
}
void clear() {
*this = Container<DataT>();
}
private:
static constexpr uint32 GENERATION_STEP = 1 << 8;
static constexpr uint32 TYPE_MASK = (1 << 8) - 1;
struct Slot {
uint32 generation;
DataT data;
};
vector<Slot> slots_;
vector<int32> empty_slots_;
Id encode_id(int32 id) const {
return (static_cast<uint64>(id) << 32) | slots_[id].generation;
}
int32 decode_id(Id id) const {
int32 slot_id = static_cast<int32>(id >> 32);
uint32 generation = static_cast<uint32>(id);
if (slot_id < 0 || slot_id >= static_cast<int32>(slots_.size())) {
return -1;
}
if (generation != slots_[slot_id].generation) {
return -1;
}
return slot_id;
}
int32 store(DataT &&data, uint8 type) {
int32 pos;
if (!empty_slots_.empty()) {
pos = empty_slots_.back();
empty_slots_.pop_back();
slots_[pos].data = std::move(data);
slots_[pos].generation ^= (slots_[pos].generation & TYPE_MASK) ^ type;
} else {
CHECK(slots_.size() <= static_cast<size_t>(std::numeric_limits<int32>::max()));
pos = static_cast<int32>(slots_.size());
slots_.push_back(Slot{GENERATION_STEP + type, std::move(data)});
}
return pos;
}
void release(int32 id) {
inc_generation(id);
slots_[id].data = DataT();
if (slots_[id].generation & ~TYPE_MASK) { // generation overflow. Can't use this id anymore
empty_slots_.push_back(id);
}
}
void inc_generation(int32 id) {
slots_[id].generation += GENERATION_STEP;
}
};
} // namespace td

56
tdutils/td/utils/Context.h Normal file
View file

@ -0,0 +1,56 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/port/thread_local.h"
namespace td {
template <class Impl>
class Context {
public:
static Impl *get() {
return context_;
}
class Guard {
public:
explicit Guard(Impl *new_context) {
old_context_ = context_;
context_ = new_context;
}
~Guard() {
context_ = old_context_;
}
Guard(const Guard &) = delete;
Guard &operator=(const Guard &) = delete;
Guard(Guard &&) = delete;
Guard &operator=(Guard &&) = delete;
private:
Impl *old_context_;
};
private:
static TD_THREAD_LOCAL Impl *context_;
};
template <class Impl>
TD_THREAD_LOCAL Impl *Context<Impl>::context_;
} // namespace td

228
tdutils/td/utils/DecTree.h Normal file
View file

@ -0,0 +1,228 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#include "td/utils/Random.h"
#include <functional>
#include <utility>
namespace td {
template <typename KeyType, typename ValueType, typename Compare = std::less<KeyType>>
class DecTree {
struct Node {
unique_ptr<Node> left_;
unique_ptr<Node> right_;
size_t size_;
KeyType key_;
ValueType value_;
uint32 y_;
void relax() {
size_ = 1;
if (left_ != nullptr) {
size_ += left_->size_;
}
if (right_ != nullptr) {
size_ += right_->size_;
}
}
Node(KeyType key, ValueType value, uint32 y) : size_(1), key_(std::move(key)), value_(std::move(value)), y_(y) {
}
};
unique_ptr<Node> root_;
static unique_ptr<Node> create_node(KeyType key, ValueType value, uint32 y) {
return make_unique<Node>(std::move(key), std::move(value), y);
}
static unique_ptr<Node> insert_node(unique_ptr<Node> Tree, KeyType key, ValueType value, uint32 y) {
if (Tree == nullptr) {
return create_node(std::move(key), std::move(value), y);
}
if (Tree->y_ < y) {
auto P = split_node(std::move(Tree), key);
auto T = create_node(std::move(key), std::move(value), y);
T->left_ = std::move(P.first);
T->right_ = std::move(P.second);
T->relax();
return T;
}
if (Compare()(key, Tree->key_)) {
Tree->left_ = insert_node(std::move(Tree->left_), std::move(key), std::move(value), y);
} else if (Compare()(Tree->key_, key)) {
Tree->right_ = insert_node(std::move(Tree->right_), std::move(key), std::move(value), y);
} else {
// ?? assert
}
Tree->relax();
return Tree;
}
static unique_ptr<Node> remove_node(unique_ptr<Node> Tree, const KeyType &key) {
if (Tree == nullptr) {
// ?? assert
return nullptr;
}
if (Compare()(key, Tree->key_)) {
Tree->left_ = remove_node(std::move(Tree->left_), key);
} else if (Compare()(Tree->key_, key)) {
Tree->right_ = remove_node(std::move(Tree->right_), key);
} else {
Tree = merge_node(std::move(Tree->left_), std::move(Tree->right_));
}
if (Tree != nullptr) {
Tree->relax();
}
return Tree;
}
static ValueType *get_node(unique_ptr<Node> &Tree, const KeyType &key) {
if (Tree == nullptr) {
return nullptr;
}
if (Compare()(key, Tree->key_)) {
return get_node(Tree->left_, key);
} else if (Compare()(Tree->key_, key)) {
return get_node(Tree->right_, key);
} else {
return &Tree->value_;
}
}
static ValueType *get_node_by_idx(unique_ptr<Node> &Tree, size_t idx) {
CHECK(Tree != nullptr);
auto s = (Tree->left_ != nullptr) ? Tree->left_->size_ : 0;
if (idx < s) {
return get_node_by_idx(Tree->left_, idx);
} else if (idx == s) {
return &Tree->value_;
} else {
return get_node_by_idx(Tree->right_, idx - s - 1);
}
}
static const ValueType *get_node(const unique_ptr<Node> &Tree, const KeyType &key) {
if (Tree == nullptr) {
return nullptr;
}
if (Compare()(key, Tree->key_)) {
return get_node(Tree->left_, key);
} else if (Compare()(Tree->key_, key)) {
return get_node(Tree->right_, key);
} else {
return &Tree->value_;
}
}
static const ValueType *get_node_by_idx(const unique_ptr<Node> &Tree, size_t idx) {
CHECK(Tree != nullptr);
auto s = (Tree->left_ != nullptr) ? Tree->left_->size_ : 0;
if (idx < s) {
return get_node_by_idx(Tree->left_, idx);
} else if (idx == s) {
return &Tree->value_;
} else {
return get_node_by_idx(Tree->right_, idx - s - 1);
}
}
static std::pair<unique_ptr<Node>, unique_ptr<Node>> split_node(unique_ptr<Node> Tree, const KeyType &key) {
if (Tree == nullptr) {
return {nullptr, nullptr};
}
if (Compare()(key, Tree->key_)) {
auto P = split_node(std::move(Tree->left_), key);
Tree->left_ = std::move(P.second);
Tree->relax();
P.second = std::move(Tree);
return P;
} else {
auto P = split_node(std::move(Tree->right_), key);
Tree->right_ = std::move(P.first);
Tree->relax();
P.first = std::move(Tree);
return P;
}
}
static unique_ptr<Node> merge_node(unique_ptr<Node> left, unique_ptr<Node> right) {
if (left == nullptr) {
return right;
}
if (right == nullptr) {
return left;
}
if (left->y_ < right->y_) {
right->left_ = merge_node(std::move(left), std::move(right->left_));
right->relax();
return right;
} else {
left->right_ = merge_node(std::move(left->right_), std::move(right));
left->relax();
return left;
}
}
public:
size_t size() const {
if (root_ == nullptr) {
return 0;
} else {
return root_->size_;
}
}
void insert(KeyType key, ValueType value) {
root_ = insert_node(std::move(root_), std::move(key), std::move(value), td::Random::fast_uint32());
}
void remove(const KeyType &key) {
root_ = remove_node(std::move(root_), key);
}
void reset() {
root_ = nullptr;
}
ValueType *get(const KeyType &key) {
return get_node(root_, key);
}
ValueType *get_random() {
if (size() == 0) {
return nullptr;
} else {
return get_node_by_idx(root_, td::Random::fast_uint32() % size());
}
}
const ValueType *get(const KeyType &key) const {
return get_node(root_, key);
}
const ValueType *get_random() const {
if (size() == 0) {
return nullptr;
} else {
return get_node_by_idx(root_, td::Random::fast_uint32() % size());
}
}
bool exists(const KeyType &key) const {
return get_node(root_, key) != nullptr;
}
};
} // namespace td

64
tdutils/td/utils/Destructor.h Normal file
View file

@ -0,0 +1,64 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#include <memory>
#include <utility>
namespace td {
class Destructor {
public:
Destructor() = default;
Destructor(const Destructor &other) = delete;
Destructor &operator=(const Destructor &other) = delete;
Destructor(Destructor &&other) = default;
Destructor &operator=(Destructor &&other) = default;
virtual ~Destructor() = default;
};
template <class F>
class LambdaDestructor : public Destructor {
public:
explicit LambdaDestructor(F &&f) : f_(std::move(f)) {
}
LambdaDestructor(const LambdaDestructor &other) = delete;
LambdaDestructor &operator=(const LambdaDestructor &other) = delete;
LambdaDestructor(LambdaDestructor &&other) = default;
LambdaDestructor &operator=(LambdaDestructor &&other) = default;
~LambdaDestructor() override {
f_();
}
private:
F f_;
};
template <class F>
auto create_destructor(F &&f) {
return make_unique<LambdaDestructor<F>>(std::forward<F>(f));
}
template <class F>
auto create_shared_destructor(F &&f) {
return std::make_shared<LambdaDestructor<F>>(std::forward<F>(f));
}
} // namespace td

56
tdutils/td/utils/Enumerator.h Normal file
View file

@ -0,0 +1,56 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#include "td/utils/misc.h"
#include <map>
#include <tuple>
namespace td {
template <class ValueT>
class Enumerator {
public:
using Key = int32;
Key add(ValueT v) {
int32 next_id = narrow_cast<int32>(arr_.size() + 1);
bool was_inserted;
decltype(map_.begin()) it;
std::tie(it, was_inserted) = map_.emplace(std::move(v), next_id);
if (was_inserted) {
arr_.push_back(&it->first);
}
return it->second;
}
const ValueT &get(Key key) const {
auto pos = static_cast<size_t>(key - 1);
CHECK(pos < arr_.size());
return *arr_[pos];
}
private:
std::map<ValueT, int32> map_;
std::vector<const ValueT *> arr_;
};
} // namespace td

213
tdutils/td/utils/EpochBasedMemoryReclamation.h Normal file
View file

@ -0,0 +1,213 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#include "td/utils/logging.h"
#include "td/utils/port/sleep.h"
#include <atomic>
#include <memory>
namespace td {
template <class T>
class EpochBasedMemoryReclamation {
public:
EpochBasedMemoryReclamation(const EpochBasedMemoryReclamation &other) = delete;
EpochBasedMemoryReclamation &operator=(const EpochBasedMemoryReclamation &other) = delete;
EpochBasedMemoryReclamation(EpochBasedMemoryReclamation &&other) = delete;
EpochBasedMemoryReclamation &operator=(EpochBasedMemoryReclamation &&other) = delete;
~EpochBasedMemoryReclamation() = default;
class Locker {
public:
Locker(size_t thread_id, EpochBasedMemoryReclamation *ebmr) : thread_id_(thread_id), ebmr_(ebmr) {
}
Locker(const Locker &other) = delete;
Locker &operator=(const Locker &other) = delete;
Locker(Locker &&other) = default;
Locker &operator=(Locker &&other) = delete;
~Locker() {
if (ebmr_) {
retire_sync();
unlock();
ebmr_.release();
}
}
void lock() {
DCHECK(ebmr_);
ebmr_->lock(thread_id_);
}
void unlock() {
DCHECK(ebmr_);
ebmr_->unlock(thread_id_);
}
void retire_sync() {
ebmr_->retire_sync(thread_id_);
}
void retire() {
ebmr_->retire(thread_id_);
}
void retire(T *ptr) {
ebmr_->retire(thread_id_, ptr);
}
private:
size_t thread_id_;
struct Never {
template <class S>
void operator()(S *) const {
UNREACHABLE();
}
};
std::unique_ptr<EpochBasedMemoryReclamation, Never> ebmr_;
};
explicit EpochBasedMemoryReclamation(size_t threads_n) : threads_(threads_n) {
}
Locker get_locker(size_t thread_id) {
return Locker{thread_id, this};
}
size_t to_delete_size_unsafe() const {
size_t res = 0;
for (auto &thread_data : threads_) {
// LOG(ERROR) << "---" << thread_data.epoch.load() / 2;
for (size_t i = 0; i < MAX_BAGS; i++) {
res += thread_data.to_delete[i].size();
// LOG(ERROR) << thread_data.to_delete[i].size();
}
}
return res;
}
private:
static constexpr size_t MAX_BAGS = 3;
struct ThreadData {
std::atomic<int64> epoch{1};
char pad[TD_CONCURRENCY_PAD - sizeof(epoch)];
size_t to_skip{0};
size_t checked_thread_i{0};
size_t bag_i{0};
std::vector<unique_ptr<T>> to_delete[MAX_BAGS];
char pad2[TD_CONCURRENCY_PAD - sizeof(to_delete)];
void rotate_bags() {
bag_i = (bag_i + 1) % MAX_BAGS;
to_delete[bag_i].clear();
}
void set_epoch(int64 new_epoch) {
//LOG(ERROR) << new_epoch;
if (epoch.load(std::memory_order_relaxed) / 2 != new_epoch) {
checked_thread_i = 0;
to_skip = 0;
rotate_bags();
}
epoch = new_epoch * 2;
}
void idle() {
epoch.store(epoch.load(std::memory_order_relaxed) | 1);
}
size_t undeleted() const {
size_t res = 0;
for (size_t i = 0; i < MAX_BAGS; i++) {
res += to_delete[i].size();
}
return res;
}
};
std::vector<ThreadData> threads_;
char pad[TD_CONCURRENCY_PAD - sizeof(threads_)];
std::atomic<int64> epoch_{1};
char pad2[TD_CONCURRENCY_PAD - sizeof(epoch_)];
void lock(size_t thread_id) {
auto &data = threads_[thread_id];
auto epoch = epoch_.load();
data.set_epoch(epoch);
if (data.to_skip == 0) {
data.to_skip = 30;
step_check(data);
} else {
data.to_skip--;
}
}
void unlock(size_t thread_id) {
//LOG(ERROR) << "UNLOCK";
auto &data = threads_[thread_id];
data.idle();
}
bool step_check(ThreadData &data) {
auto epoch = data.epoch.load(std::memory_order_relaxed) / 2;
auto checked_thread_epoch = threads_[data.checked_thread_i].epoch.load();
if (checked_thread_epoch % 2 == 1 || checked_thread_epoch / 2 == epoch) {
data.checked_thread_i++;
if (data.checked_thread_i == threads_.size()) {
if (epoch_.compare_exchange_strong(epoch, epoch + 1)) {
data.set_epoch(epoch + 1);
} else {
data.set_epoch(epoch);
}
}
return true;
}
return false;
}
void retire_sync(size_t thread_id) {
auto &data = threads_[thread_id];
while (true) {
retire(thread_id);
data.idle();
if (data.undeleted() == 0) {
break;
}
usleep_for(1000);
}
}
void retire(size_t thread_id) {
auto &data = threads_[thread_id];
data.set_epoch(epoch_.load());
while (step_check(data) && data.undeleted() != 0) {
}
}
void retire(size_t thread_id, T *ptr) {
auto &data = threads_[thread_id];
data.to_delete[data.bag_i].push_back(unique_ptr<T>{ptr});
}
};
} // namespace td

131
tdutils/td/utils/FileLog.cpp Normal file
View file

@ -0,0 +1,131 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/utils/FileLog.h"
#include "td/utils/common.h"
#include "td/utils/logging.h"
#include "td/utils/port/FileFd.h"
#include "td/utils/port/path.h"
#include "td/utils/port/StdStreams.h"
#include "td/utils/Slice.h"
#include <limits>
namespace td {
Status FileLog::init(string path, int64 rotate_threshold, bool redirect_stderr) {
if (path == path_) {
set_rotate_threshold(rotate_threshold);
return Status::OK();
}
if (path.empty()) {
return Status::Error("Log file path can't be empty");
}
TRY_RESULT(fd, FileFd::open(path, FileFd::Create | FileFd::Write | FileFd::Append));
fd_.close();
fd_ = std::move(fd);
if (!Stderr().empty() && redirect_stderr) {
fd_.get_native_fd().duplicate(Stderr().get_native_fd()).ignore();
}
auto r_path = realpath(path, true);
if (r_path.is_error()) {
path_ = std::move(path);
} else {
path_ = r_path.move_as_ok();
}
TRY_RESULT(size, fd_.get_size());
size_ = size;
rotate_threshold_ = rotate_threshold;
redirect_stderr_ = redirect_stderr;
return Status::OK();
}
Slice FileLog::get_path() const {
return path_;
}
vector<string> FileLog::get_file_paths() {
vector<string> result;
if (!path_.empty()) {
result.push_back(path_);
result.push_back(PSTRING() << path_ << ".old");
}
return result;
}
void FileLog::set_rotate_threshold(int64 rotate_threshold) {
rotate_threshold_ = rotate_threshold;
}
int64 FileLog::get_rotate_threshold() const {
return rotate_threshold_;
}
void FileLog::append(CSlice cslice, int log_level) {
Slice slice = cslice;
while (!slice.empty()) {
auto r_size = fd_.write(slice);
if (r_size.is_error()) {
process_fatal_error(PSLICE() << r_size.error() << " in " << __FILE__ << " at " << __LINE__);
}
auto written = r_size.ok();
size_ += static_cast<int64>(written);
slice.remove_prefix(written);
}
if (log_level == VERBOSITY_NAME(FATAL)) {
process_fatal_error(cslice);
}
if (size_ > rotate_threshold_) {
auto status = rename(path_, PSLICE() << path_ << ".old");
if (status.is_error()) {
process_fatal_error(PSLICE() << status.error() << " in " << __FILE__ << " at " << __LINE__);
}
do_rotate();
}
}
void FileLog::rotate() {
if (path_.empty()) {
return;
}
do_rotate();
}
void FileLog::do_rotate() {
auto current_verbosity_level = GET_VERBOSITY_LEVEL();
SET_VERBOSITY_LEVEL(std::numeric_limits<int>::min()); // to ensure that nothing will be printed to the closed log
CHECK(!path_.empty());
fd_.close();
auto r_fd = FileFd::open(path_, FileFd::Create | FileFd::Truncate | FileFd::Write);
if (r_fd.is_error()) {
process_fatal_error(PSLICE() << r_fd.error() << " in " << __FILE__ << " at " << __LINE__);
}
fd_ = r_fd.move_as_ok();
if (!Stderr().empty() && redirect_stderr_) {
fd_.get_native_fd().duplicate(Stderr().get_native_fd()).ignore();
}
size_ = 0;
SET_VERBOSITY_LEVEL(current_verbosity_level);
}
} // namespace td

57
tdutils/td/utils/FileLog.h Normal file
View file

@ -0,0 +1,57 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#include "td/utils/logging.h"
#include "td/utils/port/FileFd.h"
#include "td/utils/Slice.h"
#include "td/utils/Status.h"
namespace td {
class FileLog : public LogInterface {
static constexpr int64 DEFAULT_ROTATE_THRESHOLD = 10 * (1 << 20);
public:
Status init(string path, int64 rotate_threshold = DEFAULT_ROTATE_THRESHOLD, bool redirect_stderr = true);
Slice get_path() const;
vector<string> get_file_paths() override;
void set_rotate_threshold(int64 rotate_threshold);
int64 get_rotate_threshold() const;
void append(CSlice cslice, int log_level) override;
void rotate() override;
private:
FileFd fd_;
string path_;
int64 size_ = 0;
int64 rotate_threshold_ = 0;
bool redirect_stderr_;
void do_rotate();
};
} // namespace td

74
tdutils/td/utils/FloodControlFast.h Normal file
View file

@ -0,0 +1,74 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#include "td/utils/TimedStat.h"
namespace td {
class FloodControlFast {
public:
uint32 add_event(int32 now) {
for (auto &limit : limits_) {
limit.stat_.add_event(CounterStat::Event(), now);
if (limit.stat_.get_stat(now).count_ > limit.count_) {
wakeup_at_ = max(wakeup_at_, now + limit.duration_ * 2);
}
}
return wakeup_at_;
}
uint32 get_wakeup_at() {
return wakeup_at_;
}
void add_limit(uint32 duration, int32 count) {
limits_.push_back({TimedStat<CounterStat>(duration, 0), duration, count});
}
void clear_events() {
for (auto &limit : limits_) {
limit.stat_.clear_events();
}
wakeup_at_ = 0;
}
private:
class CounterStat {
public:
struct Event {};
int32 count_ = 0;
void on_event(Event e) {
count_++;
}
void clear() {
count_ = 0;
}
};
uint32 wakeup_at_ = 0;
struct Limit {
TimedStat<CounterStat> stat_;
uint32 duration_;
int32 count_;
};
std::vector<Limit> limits_;
};
} // namespace td

108
tdutils/td/utils/FloodControlStrict.h Normal file
View file

@ -0,0 +1,108 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#include <limits>
namespace td {
// More strict implementaions of flood control than FloodControlFast.
// Should be just fine for small counters.
class FloodControlStrict {
public:
int32 add_event(int32 now) {
events_.push_back(Event{now});
if (without_update_ > 0) {
without_update_--;
} else {
update(now);
}
return wakeup_at_;
}
// no more than count in each duration.
void add_limit(int32 duration, int32 count) {
limits_.push_back(Limit{duration, count, 0});
}
int32 get_wakeup_at() {
return wakeup_at_;
}
void clear_events() {
events_.clear();
for (auto &limit : limits_) {
limit.pos_ = 0;
}
without_update_ = 0;
wakeup_at_ = 0;
}
int32 update(int32 now) {
size_t min_pos = events_.size();
without_update_ = std::numeric_limits<size_t>::max();
for (auto &limit : limits_) {
if (limit.pos_ + limit.count_ < events_.size()) {
limit.pos_ = events_.size() - limit.count_;
}
// binary-search? :D
while (limit.pos_ < events_.size() && events_[limit.pos_].timestamp_ + limit.duration_ < now) {
limit.pos_++;
}
if (limit.count_ + limit.pos_ <= events_.size()) {
CHECK(limit.count_ + limit.pos_ == events_.size());
wakeup_at_ = max(wakeup_at_, events_[limit.pos_].timestamp_ + limit.duration_);
without_update_ = 0;
} else {
without_update_ = min(without_update_, limit.count_ + limit.pos_ - events_.size());
}
min_pos = min(min_pos, limit.pos_);
}
if (min_pos * 2 > events_.size()) {
for (auto &limit : limits_) {
limit.pos_ -= min_pos;
}
events_.erase(events_.begin(), events_.begin() + min_pos);
}
return wakeup_at_;
}
private:
int32 wakeup_at_ = 0;
struct Event {
int32 timestamp_;
};
struct Limit {
int32 duration_;
int32 count_;
size_t pos_;
};
size_t without_update_ = 0;
std::vector<Event> events_;
std::vector<Limit> limits_;
};
} // namespace td

32
tdutils/td/utils/GitInfo.cpp Normal file
View file

@ -0,0 +1,32 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/utils/GitInfo.h"
#include "auto/git_info.h"
namespace td {
CSlice GitInfo::commit() {
return GIT_COMMIT;
}
bool GitInfo::is_dirty() {
return GIT_DIRTY;
}
} // namespace td

31
tdutils/td/utils/GitInfo.h Normal file
View file

@ -0,0 +1,31 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/Slice.h"
namespace td {
class GitInfo {
public:
static CSlice commit();
static bool is_dirty();
};
} // namespace td

220
tdutils/td/utils/Gzip.cpp Normal file
View file

@ -0,0 +1,220 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/utils/Gzip.h"
char disable_linker_warning_about_empty_file_gzip_cpp TD_UNUSED;
#if TD_HAVE_ZLIB
#include "td/utils/logging.h"
#include <cstring>
#include <limits>
#include <utility>
#include <zlib.h>
namespace td {
class Gzip::Impl {
public:
z_stream stream_;
// z_stream is not copyable nor movable
Impl() = default;
Impl(const Impl &other) = delete;
Impl &operator=(const Impl &other) = delete;
Impl(Impl &&other) = delete;
Impl &operator=(Impl &&other) = delete;
~Impl() = default;
};
Status Gzip::init_encode() {
CHECK(mode_ == Empty);
init_common();
mode_ = Encode;
int ret = deflateInit2(&impl_->stream_, 6, Z_DEFLATED, 15, MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY);
if (ret != Z_OK) {
return Status::Error(PSLICE() << "zlib deflate init failed: " << ret);
}
return Status::OK();
}
Status Gzip::init_decode() {
CHECK(mode_ == Empty);
init_common();
mode_ = Decode;
int ret = inflateInit2(&impl_->stream_, MAX_WBITS + 32);
if (ret != Z_OK) {
return Status::Error(PSLICE() << "zlib inflate init failed: " << ret);
}
return Status::OK();
}
void Gzip::set_input(Slice input) {
CHECK(input_size_ == 0);
CHECK(!close_input_flag_);
CHECK(input.size() <= std::numeric_limits<uInt>::max());
CHECK(impl_->stream_.avail_in == 0);
input_size_ = input.size();
impl_->stream_.avail_in = static_cast<uInt>(input.size());
impl_->stream_.next_in = reinterpret_cast<Bytef *>(const_cast<char *>(input.data()));
}
void Gzip::set_output(MutableSlice output) {
CHECK(output_size_ == 0);
CHECK(output.size() <= std::numeric_limits<uInt>::max());
CHECK(impl_->stream_.avail_out == 0);
output_size_ = output.size();
impl_->stream_.avail_out = static_cast<uInt>(output.size());
impl_->stream_.next_out = reinterpret_cast<Bytef *>(output.data());
}
Result<Gzip::State> Gzip::run() {
while (true) {
int ret;
if (mode_ == Decode) {
ret = inflate(&impl_->stream_, Z_NO_FLUSH);
} else {
ret = deflate(&impl_->stream_, close_input_flag_ ? Z_FINISH : Z_NO_FLUSH);
}
if (ret == Z_OK) {
return Running;
}
if (ret == Z_STREAM_END) {
// TODO(now): fail if input is not empty;
clear();
return Done;
}
clear();
return Status::Error(PSLICE() << "zlib error " << ret);
}
}
size_t Gzip::left_input() const {
return impl_->stream_.avail_in;
}
size_t Gzip::left_output() const {
return impl_->stream_.avail_out;
}
void Gzip::init_common() {
std::memset(&impl_->stream_, 0, sizeof(impl_->stream_));
impl_->stream_.zalloc = Z_NULL;
impl_->stream_.zfree = Z_NULL;
impl_->stream_.opaque = Z_NULL;
impl_->stream_.avail_in = 0;
impl_->stream_.next_in = nullptr;
impl_->stream_.avail_out = 0;
impl_->stream_.next_out = nullptr;
input_size_ = 0;
output_size_ = 0;
close_input_flag_ = false;
}
void Gzip::clear() {
if (mode_ == Decode) {
inflateEnd(&impl_->stream_);
} else if (mode_ == Encode) {
deflateEnd(&impl_->stream_);
}
mode_ = Empty;
}
Gzip::Gzip() : impl_(make_unique<Impl>()) {
}
Gzip::Gzip(Gzip &&other) : Gzip() {
swap(other);
}
Gzip &Gzip::operator=(Gzip &&other) {
CHECK(this != &other);
clear();
swap(other);
return *this;
}
void Gzip::swap(Gzip &other) {
using std::swap;
swap(impl_, other.impl_);
swap(input_size_, other.input_size_);
swap(output_size_, other.output_size_);
swap(close_input_flag_, other.close_input_flag_);
swap(mode_, other.mode_);
}
Gzip::~Gzip() {
clear();
}
BufferSlice gzdecode(Slice s) {
Gzip gzip;
gzip.init_decode().ensure();
ChainBufferWriter message;
gzip.set_input(s);
gzip.close_input();
double k = 2;
gzip.set_output(message.prepare_append(static_cast<size_t>(static_cast<double>(s.size()) * k)));
while (true) {
auto r_state = gzip.run();
if (r_state.is_error()) {
return BufferSlice();
}
auto state = r_state.ok();
if (state == Gzip::Done) {
message.confirm_append(gzip.flush_output());
break;
}
if (gzip.need_input()) {
return BufferSlice();
}
if (gzip.need_output()) {
message.confirm_append(gzip.flush_output());
k *= 1.5;
gzip.set_output(message.prepare_append(static_cast<size_t>(static_cast<double>(gzip.left_input()) * k)));
}
}
return message.extract_reader().move_as_buffer_slice();
}
BufferSlice gzencode(Slice s, double k) {
Gzip gzip;
gzip.init_encode().ensure();
gzip.set_input(s);
gzip.close_input();
size_t max_size = static_cast<size_t>(static_cast<double>(s.size()) * k);
BufferWriter message{max_size};
gzip.set_output(message.prepare_append());
auto r_state = gzip.run();
if (r_state.is_error()) {
return BufferSlice();
}
auto state = r_state.ok();
if (state != Gzip::Done) {
return BufferSlice();
}
message.confirm_append(gzip.flush_output());
return message.as_buffer_slice();
}
} // namespace td
#endif

118
tdutils/td/utils/Gzip.h Normal file
View file

@ -0,0 +1,118 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#if TD_HAVE_ZLIB
#include "td/utils/buffer.h"
#include "td/utils/Slice.h"
#include "td/utils/Status.h"
namespace td {
class Gzip {
public:
Gzip();
Gzip(const Gzip &) = delete;
Gzip &operator=(const Gzip &) = delete;
Gzip(Gzip &&other);
Gzip &operator=(Gzip &&other);
~Gzip();
enum Mode { Empty, Encode, Decode };
Status init(Mode mode) TD_WARN_UNUSED_RESULT {
if (mode == Encode) {
return init_encode();
} else if (mode == Decode) {
return init_decode();
}
clear();
return Status::OK();
}
Status init_encode() TD_WARN_UNUSED_RESULT;
Status init_decode() TD_WARN_UNUSED_RESULT;
void set_input(Slice input);
void set_output(MutableSlice output);
void close_input() {
close_input_flag_ = true;
}
bool need_input() const {
return left_input() == 0;
}
bool need_output() const {
return left_output() == 0;
}
size_t left_input() const;
size_t left_output() const;
size_t used_input() const {
return input_size_ - left_input();
}
size_t used_output() const {
return output_size_ - left_output();
}
size_t flush_input() {
auto res = used_input();
input_size_ = left_input();
return res;
}
size_t flush_output() {
auto res = used_output();
output_size_ = left_output();
return res;
}
enum State { Running, Done };
Result<State> run() TD_WARN_UNUSED_RESULT;
private:
class Impl;
unique_ptr<Impl> impl_;
size_t input_size_ = 0;
size_t output_size_ = 0;
bool close_input_flag_ = false;
Mode mode_ = Empty;
void init_common();
void clear();
void swap(Gzip &other);
};
BufferSlice gzdecode(Slice s);
BufferSlice gzencode(Slice s, double k = 0.9);
} // namespace td
#endif

82
tdutils/td/utils/GzipByteFlow.cpp Normal file
View file

@ -0,0 +1,82 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/utils/GzipByteFlow.h"
char disable_linker_warning_about_empty_file_gzipbyteflow_cpp TD_UNUSED;
#if TD_HAVE_ZLIB
#include "td/utils/common.h"
#include "td/utils/Status.h"
namespace td {
void GzipByteFlow::loop() {
while (true) {
if (gzip_.need_input()) {
auto slice = input_->prepare_read();
if (slice.empty()) {
if (!is_input_active_) {
gzip_.close_input();
} else {
break;
}
} else {
gzip_.set_input(input_->prepare_read());
}
}
if (gzip_.need_output()) {
auto slice = output_.prepare_append();
CHECK(!slice.empty());
gzip_.set_output(slice);
}
auto r_state = gzip_.run();
auto output_size = gzip_.flush_output();
if (output_size) {
uncommited_size_ += output_size;
total_output_size_ += output_size;
if (total_output_size_ > max_output_size_) {
return finish(Status::Error("Max output size limit exceeded"));
}
output_.confirm_append(output_size);
}
auto input_size = gzip_.flush_input();
if (input_size) {
input_->confirm_read(input_size);
}
if (r_state.is_error()) {
return finish(r_state.move_as_error());
}
auto state = r_state.ok();
if (state == Gzip::Done) {
on_output_updated();
return consume_input();
}
}
if (uncommited_size_ >= MIN_UPDATE_SIZE) {
uncommited_size_ = 0;
on_output_updated();
}
}
constexpr size_t GzipByteFlow::MIN_UPDATE_SIZE;
} // namespace td
#endif

60
tdutils/td/utils/GzipByteFlow.h Normal file
View file

@ -0,0 +1,60 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/ByteFlow.h"
#include "td/utils/Gzip.h"
#include <limits>
namespace td {
#if TD_HAVE_ZLIB
class GzipByteFlow final : public ByteFlowBase {
public:
GzipByteFlow() = default;
explicit GzipByteFlow(Gzip::Mode mode) {
gzip_.init(mode).ensure();
}
void init_decode() {
gzip_.init_decode().ensure();
}
void init_encode() {
gzip_.init_encode().ensure();
}
void set_max_output_size(size_t max_output_size) {
max_output_size_ = max_output_size;
}
void loop() override;
private:
Gzip gzip_;
size_t uncommited_size_ = 0;
size_t total_output_size_ = 0;
size_t max_output_size_ = std::numeric_limits<size_t>::max();
static constexpr size_t MIN_UPDATE_SIZE = 1 << 14;
};
#endif
} // namespace td

87
tdutils/td/utils/Hash.h Normal file
View file

@ -0,0 +1,87 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#if TD_HAVE_ABSL
#include <absl/hash/hash.h>
#endif
#include <utility>
namespace td {
// A simple wrapper for absl::flat_hash_map, std::unordered_map and probably some our implementaion of hash map in
// the future
// We will introduce out own Hashing utility like an absl one.
class Hasher {
public:
Hasher() = default;
explicit Hasher(size_t init_value) : hash_(init_value) {
}
std::size_t finalize() const {
return hash_;
}
static Hasher combine(Hasher hasher, size_t value) {
hasher.hash_ ^= value;
return hasher;
}
template <class A, class B>
static Hasher combine(Hasher hasher, const std::pair<A, B> &value) {
hasher = AbslHashValue(std::move(hasher), value.first);
hasher = AbslHashValue(std::move(hasher), value.second);
return hasher;
}
private:
std::size_t hash_{0};
};
template <class IgnoreT>
class TdHash {
public:
template <class T>
std::size_t operator()(const T &value) const noexcept {
return AbslHashValue(Hasher(), value).finalize();
}
};
#if TD_HAVE_ABSL
template <class T>
using AbslHash = absl::Hash<T>;
#endif
// default hash implementations
template <class H, class T>
decltype(H::combine(std::declval<H>(), std::declval<T>())) AbslHashValue(H hasher, const T &value) {
return H::combine(std::move(hasher), value);
}
#if TD_HAVE_ABSL
template <class T>
using Hash = AbslHash<T>;
#else
template <class T>
using Hash = TdHash<T>;
#endif
} // namespace td

39
tdutils/td/utils/HashMap.h Normal file
View file

@ -0,0 +1,39 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/Hash.h"
#if TD_HAVE_ABSL
#include <absl/container/flat_hash_map.h>
#else
#include <unordered_map>
#endif
namespace td {
#if TD_HAVE_ABSL
template <class Key, class Value, class H = Hash<Key>>
using HashMap = absl::flat_hash_map<Key, Value, H>;
#else
template <class Key, class Value, class H = Hash<Key>>
using HashMap = std::unordered_map<Key, Value, H>;
#endif
} // namespace td

39
tdutils/td/utils/HashSet.h Normal file
View file

@ -0,0 +1,39 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/Hash.h"
#if TD_HAVE_ABSL
#include <absl/container/flat_hash_set.h>
#else
#include <unordered_set>
#endif
namespace td {
#if TD_HAVE_ABSL
template <class Key, class H = Hash<Key>>
using HashSet = absl::flat_hash_set<Key, H>;
#else
template <class Key, class H = Hash<Key>>
using HashSet = std::unordered_set<Key, H>;
#endif
} // namespace td

153
tdutils/td/utils/HazardPointers.h Normal file
View file

@ -0,0 +1,153 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#include <array>
#include <atomic>
#include <memory>
namespace td {
template <class T, int MaxPointersN = 1, class Deleter = std::default_delete<T>>
class HazardPointers {
public:
explicit HazardPointers(size_t threads_n) : threads_(threads_n) {
for (auto &data : threads_) {
for (auto &ptr : data.hazard_) {
// workaround for https://gcc.gnu.org/bugzilla/show_bug.cgi?id=64658
#if TD_GCC && GCC_VERSION <= 40902
ptr = nullptr;
#else
std::atomic_init(&ptr, static_cast<T *>(nullptr));
#endif
}
}
}
HazardPointers(const HazardPointers &other) = delete;
HazardPointers &operator=(const HazardPointers &other) = delete;
HazardPointers(HazardPointers &&other) = delete;
HazardPointers &operator=(HazardPointers &&other) = delete;
class Holder {
public:
template <class S>
S *protect(std::atomic<S *> &to_protect) {
return do_protect(hazard_ptr_, to_protect);
}
Holder(HazardPointers &hp, size_t thread_id, size_t pos) : Holder(hp.get_hazard_ptr(thread_id, pos)) {
CHECK(hazard_ptr_.load() == 0);
hazard_ptr_.store(reinterpret_cast<T *>(1));
}
Holder(const Holder &other) = delete;
Holder &operator=(const Holder &other) = delete;
Holder(Holder &&other) = delete;
Holder &operator=(Holder &&other) = delete;
~Holder() {
clear();
}
void clear() {
hazard_ptr_.store(nullptr, std::memory_order_release);
}
private:
friend class HazardPointers;
explicit Holder(std::atomic<T *> &ptr) : hazard_ptr_(ptr) {
}
std::atomic<T *> &hazard_ptr_;
};
void retire(size_t thread_id, T *ptr = nullptr) {
CHECK(thread_id < threads_.size());
auto &data = threads_[thread_id];
if (ptr) {
data.to_delete_.push_back(std::unique_ptr<T, Deleter>(ptr));
}
for (auto it = data.to_delete_.begin(); it != data.to_delete_.end();) {
if (!is_protected(it->get())) {
it->reset();
it = data.to_delete_.erase(it);
} else {
++it;
}
}
}
// old inteface
T *protect(size_t thread_id, size_t pos, std::atomic<T *> &ptr) {
return do_protect(get_hazard_ptr(thread_id, pos), ptr);
}
void clear(size_t thread_id, size_t pos) {
do_clear(get_hazard_ptr(thread_id, pos));
}
size_t to_delete_size_unsafe() const {
size_t res = 0;
for (auto &thread : threads_) {
res += thread.to_delete_.size();
}
return res;
}
private:
struct ThreadData {
std::array<std::atomic<T *>, MaxPointersN> hazard_;
char pad[TD_CONCURRENCY_PAD - sizeof(hazard_)];
// stupid gc
std::vector<std::unique_ptr<T, Deleter>> to_delete_;
char pad2[TD_CONCURRENCY_PAD - sizeof(to_delete_)];
};
std::vector<ThreadData> threads_;
char pad2[TD_CONCURRENCY_PAD - sizeof(threads_)];
template <class S>
static S *do_protect(std::atomic<T *> &hazard_ptr, std::atomic<S *> &to_protect) {
T *saved = nullptr;
T *to_save;
while ((to_save = to_protect.load()) != saved) {
hazard_ptr.store(to_save);
saved = to_save;
}
return static_cast<S *>(saved);
}
static void do_clear(std::atomic<T *> &hazard_ptr) {
hazard_ptr.store(nullptr, std::memory_order_release);
}
bool is_protected(T *ptr) {
for (auto &thread : threads_) {
for (auto &hazard_ptr : thread.hazard_) {
if (hazard_ptr.load() == ptr) {
return true;
}
}
}
return false;
}
std::atomic<T *> &get_hazard_ptr(size_t thread_id, size_t pos) {
CHECK(thread_id < threads_.size());
return threads_[thread_id].hazard_[pos];
}
};
} // namespace td

170
tdutils/td/utils/Heap.h Normal file
View file

@ -0,0 +1,170 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
namespace td {
struct HeapNode {
bool in_heap() const {
return pos_ != -1;
}
bool is_top() const {
return pos_ == 0;
}
void remove() {
pos_ = -1;
}
int pos_ = -1;
};
template <class KeyT, int K = 4>
class KHeap {
public:
bool empty() const {
return array_.empty();
}
size_t size() const {
return array_.size();
}
KeyT top_key() const {
return array_[0].key_;
}
HeapNode *pop() {
CHECK(!empty());
HeapNode *result = array_[0].node_;
result->remove();
erase(0);
return result;
}
void insert(KeyT key, HeapNode *node) {
CHECK(!node->in_heap());
array_.push_back({key, node});
fix_up(static_cast<int>(array_.size()) - 1);
}
void fix(KeyT key, HeapNode *node) {
CHECK(node->in_heap());
int pos = node->pos_;
KeyT old_key = array_[pos].key_;
array_[pos].key_ = key;
if (key < old_key) {
fix_up(pos);
} else {
fix_down(pos);
}
}
void erase(HeapNode *node) {
CHECK(node->in_heap());
int pos = node->pos_;
node->remove();
erase(pos);
}
template <class F>
void for_each(F &&f) const {
for (auto &it : array_) {
f(it.key_, it.node_);
}
}
template <class F>
void for_each(F &&f) {
for (auto &it : array_) {
f(it.key_, it.node_);
}
}
void check() const {
for (size_t i = 0; i < array_.size(); i++) {
for (size_t j = i * K + 1; j < i * K + 1 + K && j < array_.size(); j++) {
CHECK(array_[i].key_ <= array_[j].key_);
}
}
}
private:
struct Item {
KeyT key_;
HeapNode *node_;
};
vector<Item> array_;
void fix_up(int pos) {
auto item = array_[pos];
while (pos) {
int parent_pos = (pos - 1) / K;
auto parent_item = array_[parent_pos];
if (parent_item.key_ < item.key_) {
break;
}
parent_item.node_->pos_ = pos;
array_[pos] = parent_item;
pos = parent_pos;
}
item.node_->pos_ = pos;
array_[pos] = item;
}
void fix_down(int pos) {
auto item = array_[pos];
while (true) {
int left_pos = pos * K + 1;
int right_pos = min(left_pos + K, static_cast<int>(array_.size()));
int next_pos = pos;
KeyT next_key = item.key_;
for (int i = left_pos; i < right_pos; i++) {
KeyT i_key = array_[i].key_;
if (i_key < next_key) {
next_key = i_key;
next_pos = i;
}
}
if (next_pos == pos) {
break;
}
array_[pos] = array_[next_pos];
array_[pos].node_->pos_ = pos;
pos = next_pos;
}
item.node_->pos_ = pos;
array_[pos] = item;
}
void erase(int pos) {
array_[pos] = array_.back();
array_.pop_back();
if (pos < static_cast<int>(array_.size())) {
fix_down(pos);
fix_up(pos);
}
}
};
} // namespace td

245
tdutils/td/utils/Hints.cpp Normal file
View file

@ -0,0 +1,245 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/utils/Hints.h"
#include "td/utils/logging.h"
#include "td/utils/misc.h"
#include "td/utils/Slice.h"
#include "td/utils/translit.h"
#include "td/utils/unicode.h"
#include "td/utils/utf8.h"
#include <algorithm>
namespace td {
vector<string> Hints::fix_words(vector<string> words) {
std::sort(words.begin(), words.end());
size_t new_words_size = 0;
for (size_t i = 0; i != words.size(); i++) {
if (i == words.size() - 1 || !begins_with(words[i + 1], words[i])) {
if (i != new_words_size) {
words[new_words_size] = std::move(words[i]);
}
new_words_size++;
}
}
words.resize(new_words_size);
return words;
}
vector<string> Hints::get_words(Slice name, bool is_search) {
bool in_word = false;
string word;
vector<string> words;
auto pos = name.ubegin();
auto end = name.uend();
while (pos != end) {
uint32 code;
pos = next_utf8_unsafe(pos, &code, is_search ? "get_words_search" : "get_words_add");
code = prepare_search_character(code);
if (code == 0) {
continue;
}
if (code == ' ') {
if (in_word) {
words.push_back(std::move(word));
word.clear();
in_word = false;
}
} else {
in_word = true;
code = remove_diacritics(code);
append_utf8_character(word, code);
}
}
if (in_word) {
words.push_back(std::move(word));
}
return fix_words(std::move(words));
}
void Hints::add_word(const string &word, KeyT key, std::map<string, vector<KeyT>> &word_to_keys) {
vector<KeyT> &keys = word_to_keys[word];
CHECK(std::find(keys.begin(), keys.end(), key) == keys.end());
keys.push_back(key);
}
void Hints::delete_word(const string &word, KeyT key, std::map<string, vector<KeyT>> &word_to_keys) {
vector<KeyT> &keys = word_to_keys[word];
auto key_it = std::find(keys.begin(), keys.end(), key);
CHECK(key_it != keys.end());
if (keys.size() == 1) {
word_to_keys.erase(word);
} else {
CHECK(keys.size() > 1);
*key_it = keys.back();
keys.pop_back();
}
}
void Hints::add(KeyT key, Slice name) {
// LOG(ERROR) << "Add " << key << ": " << name;
auto it = key_to_name_.find(key);
if (it != key_to_name_.end()) {
if (it->second == name) {
return;
}
vector<string> old_transliterations;
for (auto &old_word : get_words(it->second, false)) {
delete_word(old_word, key, word_to_keys_);
for (auto &w : get_word_transliterations(old_word, false)) {
if (w != old_word) {
old_transliterations.push_back(std::move(w));
}
}
}
for (auto &word : fix_words(old_transliterations)) {
delete_word(word, key, translit_word_to_keys_);
}
}
if (name.empty()) {
if (it != key_to_name_.end()) {
key_to_name_.erase(it);
}
key_to_rating_.erase(key);
return;
}
vector<string> transliterations;
for (auto &word : get_words(name, false)) {
add_word(word, key, word_to_keys_);
for (auto &w : get_word_transliterations(word, false)) {
if (w != word) {
transliterations.push_back(std::move(w));
}
}
}
for (auto &word : fix_words(transliterations)) {
add_word(word, key, translit_word_to_keys_);
}
key_to_name_[key] = name.str();
}
void Hints::set_rating(KeyT key, RatingT rating) {
// LOG(ERROR) << "Set rating " << key << ": " << rating;
key_to_rating_[key] = rating;
}
void Hints::add_search_results(vector<KeyT> &results, const string &word,
const std::map<string, vector<KeyT>> &word_to_keys) {
LOG(DEBUG) << "Search for word " << word;
auto it = word_to_keys.lower_bound(word);
while (it != word_to_keys.end() && begins_with(it->first, word)) {
results.insert(results.end(), it->second.begin(), it->second.end());
++it;
}
}
vector<Hints::KeyT> Hints::search_word(const string &word) const {
vector<KeyT> results;
add_search_results(results, word, translit_word_to_keys_);
for (auto w : get_word_transliterations(word, true)) {
add_search_results(results, w, word_to_keys_);
}
std::sort(results.begin(), results.end());
results.erase(std::unique(results.begin(), results.end()), results.end());
return results;
}
std::pair<size_t, vector<Hints::KeyT>> Hints::search(Slice query, int32 limit, bool return_all_for_empty_query) const {
// LOG(ERROR) << "Search " << query;
vector<KeyT> results;
if (limit < 0) {
return {key_to_name_.size(), std::move(results)};
}
auto words = get_words(query, true);
if (return_all_for_empty_query && words.empty()) {
results.reserve(key_to_name_.size());
for (auto &it : key_to_name_) {
results.push_back(it.first);
}
}
for (size_t i = 0; i < words.size(); i++) {
vector<KeyT> keys = search_word(words[i]);
if (i == 0) {
results = std::move(keys);
continue;
}
// now need to intersect two lists
size_t results_pos = 0;
size_t keys_pos = 0;
size_t new_results_size = 0;
while (results_pos != results.size() && keys_pos != keys.size()) {
if (results[results_pos] < keys[keys_pos]) {
results_pos++;
} else if (results[results_pos] > keys[keys_pos]) {
keys_pos++;
} else {
results[new_results_size++] = results[results_pos];
results_pos++;
keys_pos++;
}
}
results.resize(new_results_size);
}
auto total_size = results.size();
if (total_size < static_cast<size_t>(limit)) {
std::sort(results.begin(), results.end(), CompareByRating(key_to_rating_));
} else {
std::partial_sort(results.begin(), results.begin() + limit, results.end(), CompareByRating(key_to_rating_));
results.resize(limit);
}
return {total_size, std::move(results)};
}
bool Hints::has_key(KeyT key) const {
return key_to_name_.find(key) != key_to_name_.end();
}
string Hints::key_to_string(KeyT key) const {
auto it = key_to_name_.find(key);
if (it == key_to_name_.end()) {
return string();
}
return it->second;
}
std::pair<size_t, vector<Hints::KeyT>> Hints::search_empty(int32 limit) const {
return search(Slice(), limit, true);
}
size_t Hints::size() const {
return key_to_name_.size();
}
} // namespace td

97
tdutils/td/utils/Hints.h Normal file
View file

@ -0,0 +1,97 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#include "td/utils/Slice.h"
#include <map>
#include <unordered_map>
#include <utility>
namespace td {
// TODO template KeyT
class Hints {
using KeyT = int64;
using RatingT = int64;
public:
void add(KeyT key, Slice name);
void remove(KeyT key) {
add(key, "");
}
void set_rating(KeyT key, RatingT rating);
std::pair<size_t, vector<KeyT>> search(
Slice query, int32 limit,
bool return_all_for_empty_query = false) const; // TODO sort by name instead of sort by rating
bool has_key(KeyT key) const;
string key_to_string(KeyT key) const;
std::pair<size_t, vector<KeyT>> search_empty(int32 limit) const; // == search("", limit, true)
size_t size() const;
private:
std::map<string, vector<KeyT>> word_to_keys_;
std::map<string, vector<KeyT>> translit_word_to_keys_;
std::unordered_map<KeyT, string> key_to_name_;
std::unordered_map<KeyT, RatingT> key_to_rating_;
static void add_word(const string &word, KeyT key, std::map<string, vector<KeyT>> &word_to_keys);
static void delete_word(const string &word, KeyT key, std::map<string, vector<KeyT>> &word_to_keys);
static vector<string> fix_words(vector<string> words);
static vector<string> get_words(Slice name, bool is_search);
static void add_search_results(vector<KeyT> &results, const string &word,
const std::map<string, vector<KeyT>> &word_to_keys);
vector<KeyT> search_word(const string &word) const;
class CompareByRating {
const std::unordered_map<KeyT, RatingT> &key_to_rating_;
RatingT get_rating(const KeyT &key) const {
auto it = key_to_rating_.find(key);
if (it == key_to_rating_.end()) {
return RatingT();
}
return it->second;
}
public:
explicit CompareByRating(const std::unordered_map<KeyT, RatingT> &key_to_rating) : key_to_rating_(key_to_rating) {
}
bool operator()(const KeyT &lhs, const KeyT &rhs) const {
auto lhs_rating = get_rating(lhs);
auto rhs_rating = get_rating(rhs);
return lhs_rating < rhs_rating || (lhs_rating == rhs_rating && lhs < rhs);
}
};
};
} // namespace td

211
tdutils/td/utils/HttpUrl.cpp Normal file
View file

@ -0,0 +1,211 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/utils/HttpUrl.h"
#include "td/utils/format.h"
#include "td/utils/logging.h"
#include "td/utils/misc.h"
#include "td/utils/Parser.h"
namespace td {
string HttpUrl::get_url() const {
string result;
switch (protocol_) {
case Protocol::HTTP:
result += "http://";
break;
case Protocol::HTTPS:
result += "https://";
break;
default:
UNREACHABLE();
}
if (!userinfo_.empty()) {
result += userinfo_;
result += '@';
}
if (is_ipv6_) {
result += '[';
}
result += host_;
if (is_ipv6_) {
result += ']';
}
if (specified_port_ > 0) {
result += ':';
result += to_string(specified_port_);
}
LOG_CHECK(!query_.empty() && query_[0] == '/') << query_;
result += query_;
return result;
}
Result<HttpUrl> parse_url(Slice url, HttpUrl::Protocol default_protocol) {
// url == [https?://][userinfo@]host[:port]
ConstParser parser(url);
string protocol_str = to_lower(parser.read_till_nofail(':'));
HttpUrl::Protocol protocol;
if (parser.start_with("://")) {
parser.advance(3);
if (protocol_str == "http") {
protocol = HttpUrl::Protocol::HTTP;
} else if (protocol_str == "https") {
protocol = HttpUrl::Protocol::HTTPS;
} else {
return Status::Error("Unsupported URL protocol");
}
} else {
parser = ConstParser(url);
protocol = default_protocol;
}
Slice userinfo_host_port = parser.read_till_nofail("/?#");
int port = 0;
const char *colon = userinfo_host_port.end() - 1;
while (colon > userinfo_host_port.begin() && *colon != ':' && *colon != ']' && *colon != '@') {
colon--;
}
Slice userinfo_host;
if (colon > userinfo_host_port.begin() && *colon == ':') {
port = to_integer<int>(Slice(colon + 1, userinfo_host_port.end()));
userinfo_host = Slice(userinfo_host_port.begin(), colon);
} else {
userinfo_host = userinfo_host_port;
}
if (port < 0 || port > 65535) {
return Status::Error("Wrong port number specified in the URL");
}
auto at_pos = userinfo_host.rfind('@');
Slice userinfo = at_pos == static_cast<size_t>(-1) ? "" : userinfo_host.substr(0, at_pos);
Slice host = userinfo_host.substr(at_pos + 1);
bool is_ipv6 = false;
if (!host.empty() && host[0] == '[' && host.back() == ']') {
host.remove_prefix(1);
host.remove_suffix(1);
is_ipv6 = true;
}
if (host.empty()) {
return Status::Error("URL host is empty");
}
if (host == ".") {
return Status::Error("Host is invalid");
}
int specified_port = port;
if (port == 0) {
if (protocol == HttpUrl::Protocol::HTTP) {
port = 80;
} else {
CHECK(protocol == HttpUrl::Protocol::HTTPS);
port = 443;
}
}
Slice query = parser.read_all();
while (!query.empty() && is_space(query.back())) {
query.remove_suffix(1);
}
if (query.empty()) {
query = Slice("/");
}
string query_str;
if (query[0] != '/') {
query_str = '/';
}
for (auto c : query) {
if (static_cast<unsigned char>(c) <= 0x20) {
query_str += '%';
query_str += "0123456789ABCDEF"[c / 16];
query_str += "0123456789ABCDEF"[c % 16];
} else {
query_str += c;
}
}
string host_str = to_lower(host);
for (size_t i = 0; i < host_str.size(); i++) {
char c = host_str[i];
if (is_ipv6) {
if (c == ':' || ('0' <= c && c <= '9') || ('a' <= c && c <= 'f') || c == '.') {
continue;
}
return Status::Error("Wrong IPv6 URL host");
}
if (('a' <= c && c <= 'z') || c == '.' || ('0' <= c && c <= '9') || c == '-' || c == '_' || c == '!' || c == '$' ||
c == ',' || c == '~' || c == '*' || c == '\'' || c == '(' || c == ')' || c == ';' || c == '&' || c == '+' ||
c == '=') {
// symbols allowed by RFC 7230 and RFC 3986
continue;
}
if (c == '%') {
c = host_str[++i];
if (('a' <= c && c <= 'f') || ('0' <= c && c <= '9')) {
c = host_str[++i];
if (('a' <= c && c <= 'f') || ('0' <= c && c <= '9')) {
// percent encoded symbol as allowed by RFC 7230 and RFC 3986
continue;
}
}
return Status::Error("Wrong percent-encoded symbol in URL host");
}
// all other symbols aren't allowed
unsigned char uc = static_cast<unsigned char>(c);
if (uc >= 128) {
// but we allow plain UTF-8 symbols
continue;
}
return Status::Error("Wrong URL host");
}
return HttpUrl{protocol, userinfo.str(), std::move(host_str), is_ipv6, specified_port, port, std::move(query_str)};
}
StringBuilder &operator<<(StringBuilder &sb, const HttpUrl &url) {
sb << tag("protocol", url.protocol_ == HttpUrl::Protocol::HTTP ? "HTTP" : "HTTPS") << tag("userinfo", url.userinfo_)
<< tag("host", url.host_) << tag("port", url.port_) << tag("query", url.query_);
return sb;
}
string get_url_query_file_name(const string &query) {
Slice query_slice = query;
query_slice.truncate(query.find_first_of("?#"));
auto slash_pos = query_slice.rfind('/');
if (slash_pos < query_slice.size()) {
return query_slice.substr(slash_pos + 1).str();
}
return query_slice.str();
}
string get_url_file_name(Slice url) {
auto r_http_url = parse_url(url);
if (r_http_url.is_error()) {
LOG(WARNING) << "Receive wrong URL \"" << url << '"';
return string();
}
return get_url_query_file_name(r_http_url.ok().query_);
}
} // namespace td

60
tdutils/td/utils/HttpUrl.h Normal file
View file

@ -0,0 +1,60 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#include "td/utils/Slice.h"
#include "td/utils/Status.h"
#include "td/utils/StringBuilder.h"
namespace td {
class HttpUrl {
public:
enum class Protocol { HTTP, HTTPS } protocol_ = Protocol::HTTP;
string userinfo_;
string host_;
bool is_ipv6_ = false;
int specified_port_ = 0;
int port_ = 0;
string query_;
string get_url() const;
HttpUrl(Protocol protocol, string userinfo, string host, bool is_ipv6, int specified_port, int port, string query)
: protocol_(protocol)
, userinfo_(std::move(userinfo))
, host_(std::move(host))
, is_ipv6_(is_ipv6)
, specified_port_(specified_port)
, port_(port)
, query_(std::move(query)) {
}
};
Result<HttpUrl> parse_url(Slice url,
HttpUrl::Protocol default_protocol = HttpUrl::Protocol::HTTP) TD_WARN_UNUSED_RESULT;
StringBuilder &operator<<(StringBuilder &sb, const HttpUrl &url);
string get_url_query_file_name(const string &query);
string get_url_file_name(Slice url);
} // namespace td

698
tdutils/td/utils/JsonBuilder.cpp Normal file
View file

@ -0,0 +1,698 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/utils/JsonBuilder.h"
#include "td/utils/misc.h"
#include "td/utils/ScopeGuard.h"
#include <cstring>
namespace td {
StringBuilder &operator<<(StringBuilder &sb, const JsonRawString &val) {
sb << '"';
SCOPE_EXIT {
sb << '"';
};
auto *s = val.value_.begin();
auto len = val.value_.size();
for (size_t pos = 0; pos < len; pos++) {
auto ch = static_cast<unsigned char>(s[pos]);
switch (ch) {
case '"':
sb << '\\' << '"';
break;
case '\\':
sb << '\\' << '\\';
break;
case '\b':
sb << '\\' << 'b';
break;
case '\f':
sb << '\\' << 'f';
break;
case '\n':
sb << '\\' << 'n';
break;
case '\r':
sb << '\\' << 'r';
break;
case '\t':
sb << '\\' << 't';
break;
default:
if (ch <= 31) {
sb << JsonOneChar(s[pos]);
break;
}
sb << s[pos];
break;
}
}
return sb;
}
StringBuilder &operator<<(StringBuilder &sb, const JsonString &val) {
sb << '"';
SCOPE_EXIT {
sb << '"';
};
auto *s = val.str_.begin();
auto len = val.str_.size();
for (size_t pos = 0; pos < len; pos++) {
auto ch = static_cast<unsigned char>(s[pos]);
switch (ch) {
case '"':
sb << '\\' << '"';
break;
case '\\':
sb << '\\' << '\\';
break;
case '\b':
sb << '\\' << 'b';
break;
case '\f':
sb << '\\' << 'f';
break;
case '\n':
sb << '\\' << 'n';
break;
case '\r':
sb << '\\' << 'r';
break;
case '\t':
sb << '\\' << 't';
break;
default:
if (ch <= 31) {
sb << JsonOneChar(s[pos]);
break;
}
if (128 <= ch) {
int a = s[pos];
CHECK((a & 0x40) != 0);
CHECK(pos + 1 < len);
int b = s[++pos];
CHECK((b & 0xc0) == 0x80);
if ((a & 0x20) == 0) {
CHECK((a & 0x1e) > 0);
sb << JsonChar(((a & 0x1f) << 6) | (b & 0x3f));
break;
}
CHECK(pos + 1 < len);
int c = s[++pos];
CHECK((c & 0xc0) == 0x80);
if ((a & 0x10) == 0) {
CHECK(((a & 0x0f) | (b & 0x20)) > 0);
sb << JsonChar(((a & 0x0f) << 12) | ((b & 0x3f) << 6) | (c & 0x3f));
break;
}
CHECK(pos + 1 < len);
int d = s[++pos];
CHECK((d & 0xc0) == 0x80);
if ((a & 0x08) == 0) {
CHECK(((a & 0x07) | (b & 0x30)) > 0);
sb << JsonChar(((a & 0x07) << 18) | ((b & 0x3f) << 12) | ((c & 0x3f) << 6) | (d & 0x3f));
break;
}
UNREACHABLE();
break;
}
sb << s[pos];
break;
}
}
return sb;
}
Result<MutableSlice> json_string_decode(Parser &parser) {
if (!parser.try_skip('"')) {
return Status::Error("Opening '\"' expected");
}
auto *cur_src = parser.data().data();
auto *end_src = parser.data().end();
auto *end = cur_src;
while (end < end_src && end[0] != '"') {
if (end[0] == '\\') {
end++;
}
end++;
}
if (end >= end_src) {
return Status::Error("Closing '\"' not found");
}
parser.advance(end + 1 - cur_src);
end_src = end;
auto *cur_dest = cur_src;
auto *begin_dest = cur_src;
while (cur_src != end_src) {
auto *slash = static_cast<char *>(std::memchr(cur_src, '\\', end_src - cur_src));
if (slash == nullptr) {
slash = end_src;
}
std::memmove(cur_dest, cur_src, slash - cur_src);
cur_dest += slash - cur_src;
cur_src = slash;
if (cur_src != end_src) {
cur_src++;
if (cur_src == end_src) {
// TODO UNREACHABLE();
return Status::Error("Unexpected end of string");
}
switch (*cur_src) {
case '"':
case '\\':
case '/':
*cur_dest++ = *cur_src++;
break;
case 'b':
*cur_dest++ = '\b';
cur_src++;
break;
case 'f':
*cur_dest++ = '\f';
cur_src++;
break;
case 'n':
*cur_dest++ = '\n';
cur_src++;
break;
case 'r':
*cur_dest++ = '\r';
cur_src++;
break;
case 't':
*cur_dest++ = '\t';
cur_src++;
break;
case 'u': {
cur_src++;
if (cur_src + 4 > end_src) {
return Status::Error("\\u has less than 4 symbols");
}
int num = 0;
for (int i = 0; i < 4; i++, cur_src++) {
int d = hex_to_int(*cur_src);
if (d == 16) {
return Status::Error("Invalid \\u -- not hex digit");
}
num = num * 16 + d;
}
if (0xD7FF < num && num < 0xE000) {
if (cur_src + 6 <= end_src && cur_src[0] == '\\' && cur_src[1] == 'u') {
cur_src += 2;
int new_num = 0;
for (int i = 0; i < 4; i++, cur_src++) {
int d = hex_to_int(*cur_src);
if (d == 16) {
return Status::Error("Invalid \\u -- not hex digit");
}
new_num = new_num * 16 + d;
}
if (0xD7FF < new_num && new_num < 0xE000) {
num = (((num & 0x3FF) << 10) | (new_num & 0x3FF)) + 0x10000;
} else {
cur_src -= 6;
}
}
}
if (num < 128) {
*cur_dest++ = static_cast<char>(num);
} else if (num < 0x800) {
*cur_dest++ = static_cast<char>(0xc0 + (num >> 6));
*cur_dest++ = static_cast<char>(0x80 + (num & 63));
} else if (num <= 0xffff) {
*cur_dest++ = static_cast<char>(0xe0 + (num >> 12));
*cur_dest++ = static_cast<char>(0x80 + ((num >> 6) & 63));
*cur_dest++ = static_cast<char>(0x80 + (num & 63));
} else {
*cur_dest++ = static_cast<char>(0xf0 + (num >> 18));
*cur_dest++ = static_cast<char>(0x80 + ((num >> 12) & 63));
*cur_dest++ = static_cast<char>(0x80 + ((num >> 6) & 63));
*cur_dest++ = static_cast<char>(0x80 + (num & 63));
}
break;
}
}
}
}
CHECK(cur_dest <= end_src);
return MutableSlice(begin_dest, cur_dest);
}
Status json_string_skip(Parser &parser) {
if (!parser.try_skip('"')) {
return Status::Error("Opening '\"' expected");
}
auto *begin_src = parser.data().data();
auto *cur_src = begin_src;
auto *end_src = parser.data().end();
auto *end = cur_src;
while (end < end_src && *end != '"') {
if (*end == '\\') {
end++;
}
end++;
}
if (end >= end_src) {
return Status::Error("Closing '\"' not found");
}
parser.advance(end + 1 - cur_src);
end_src = end;
while (cur_src != end_src) {
auto *slash = static_cast<char *>(std::memchr(cur_src, '\\', end_src - cur_src));
if (slash == nullptr) {
slash = end_src;
}
cur_src = slash;
if (cur_src != end_src) {
cur_src++;
if (cur_src == end_src) {
// TODO UNREACHABLE();
return Status::Error("Unexpected end of string");
}
switch (*cur_src) {
case '"':
case '\\':
case '/':
case 'b':
case 'f':
case 'n':
case 'r':
case 't':
cur_src++;
break;
case 'u': {
cur_src++;
if (cur_src + 4 > end_src) {
return Status::Error("\\u has less than 4 symbols");
}
int num = 0;
for (int i = 0; i < 4; i++, cur_src++) {
int d = hex_to_int(*cur_src);
if (d == 16) {
return Status::Error("Invalid \\u -- not hex digit");
}
num = num * 16 + d;
}
if (0xD7FF < num && num < 0xE000) {
if (cur_src + 6 <= end_src && cur_src[0] == '\\' && cur_src[1] == 'u') {
cur_src += 2;
int new_num = 0;
for (int i = 0; i < 4; i++, cur_src++) {
int d = hex_to_int(*cur_src);
if (d == 16) {
return Status::Error("Invalid \\u -- not hex digit");
}
new_num = new_num * 16 + d;
}
if (0xD7FF < new_num && new_num < 0xE000) {
// num = (((num & 0x3FF) << 10) | (new_num & 0x3FF)) + 0x10000;
} else {
cur_src -= 6;
}
}
}
break;
}
}
}
}
return Status::OK();
}
Result<JsonValue> do_json_decode(Parser &parser, int32 max_depth) {
if (max_depth < 0) {
return Status::Error("Too big object depth");
}
parser.skip_whitespaces();
switch (parser.peek_char()) {
case 'f':
if (parser.skip_start_with("false")) {
return JsonValue::create_boolean(false);
}
return Status::Error("Token starts with 'f' -- false expected");
case 't':
if (parser.skip_start_with("true")) {
return JsonValue::create_boolean(true);
}
return Status::Error("Token starts with 't' -- true expected");
case 'n':
if (parser.skip_start_with("null")) {
return JsonValue();
}
return Status::Error("Token starts with 'n' -- null expected");
case '"': {
TRY_RESULT(slice, json_string_decode(parser));
return JsonValue::create_string(slice);
}
case '[': {
parser.skip('[');
parser.skip_whitespaces();
std::vector<JsonValue> res;
if (parser.try_skip(']')) {
return JsonValue::create_array(std::move(res));
}
while (true) {
if (parser.empty()) {
return Status::Error("Unexpected string end");
}
TRY_RESULT(value, do_json_decode(parser, max_depth - 1));
res.emplace_back(std::move(value));
parser.skip_whitespaces();
if (parser.try_skip(']')) {
break;
}
if (parser.try_skip(',')) {
parser.skip_whitespaces();
continue;
}
if (parser.empty()) {
return Status::Error("Unexpected string end");
}
return Status::Error("Unexpected symbol while parsing JSON Array");
}
return JsonValue::create_array(std::move(res));
}
case '{': {
parser.skip('{');
parser.skip_whitespaces();
std::vector<std::pair<MutableSlice, JsonValue> > res;
if (parser.try_skip('}')) {
return JsonValue::make_object(std::move(res));
}
while (true) {
if (parser.empty()) {
return Status::Error("Unexpected string end");
}
TRY_RESULT(key, json_string_decode(parser));
parser.skip_whitespaces();
if (!parser.try_skip(':')) {
return Status::Error("':' expected");
}
TRY_RESULT(value, do_json_decode(parser, max_depth - 1));
res.emplace_back(std::move(key), std::move(value));
parser.skip_whitespaces();
if (parser.try_skip('}')) {
break;
}
if (parser.try_skip(',')) {
parser.skip_whitespaces();
continue;
}
if (parser.empty()) {
return Status::Error("Unexpected string end");
}
return Status::Error("Unexpected symbol while parsing JSON Object");
}
return JsonValue::make_object(std::move(res));
}
case '-':
case '+':
case '.':
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9': {
auto num = parser.read_while(
[](char c) { return c == '-' || ('0' <= c && c <= '9') || c == 'e' || c == 'E' || c == '+' || c == '.'; });
return JsonValue::create_number(num);
}
case 0:
return Status::Error("Unexpected string end");
default: {
char next = parser.peek_char();
if (0 < next && next < 127) {
return Status::Error(PSLICE() << "Unexpected symbol '" << parser.peek_char() << "'");
} else {
return Status::Error("Unexpected symbol");
}
}
}
UNREACHABLE();
}
Status do_json_skip(Parser &parser, int32 max_depth) {
if (max_depth < 0) {
return Status::Error("Too big object depth");
}
parser.skip_whitespaces();
switch (parser.peek_char()) {
case 'f':
if (parser.skip_start_with("false")) {
return Status::OK();
}
return Status::Error("Starts with 'f' -- false expected");
case 't':
if (parser.skip_start_with("true")) {
return Status::OK();
}
return Status::Error("Starts with 't' -- true expected");
case 'n':
if (parser.skip_start_with("null")) {
return Status::OK();
}
return Status::Error("Starts with 'n' -- null expected");
case '"': {
return json_string_skip(parser);
}
case '[': {
parser.skip('[');
parser.skip_whitespaces();
if (parser.try_skip(']')) {
return Status::OK();
}
while (true) {
if (parser.empty()) {
return Status::Error("Unexpected end");
}
TRY_STATUS(do_json_skip(parser, max_depth - 1));
parser.skip_whitespaces();
if (parser.try_skip(']')) {
break;
}
if (parser.try_skip(',')) {
parser.skip_whitespaces();
continue;
}
return Status::Error("Unexpected symbol");
}
return Status::OK();
}
case '{': {
parser.skip('{');
parser.skip_whitespaces();
if (parser.try_skip('}')) {
return Status::OK();
}
while (true) {
if (parser.empty()) {
return Status::Error("Unexpected end");
}
TRY_STATUS(json_string_skip(parser));
parser.skip_whitespaces();
if (!parser.try_skip(':')) {
return Status::Error("':' expected");
}
TRY_STATUS(do_json_skip(parser, max_depth - 1));
parser.skip_whitespaces();
if (parser.try_skip('}')) {
break;
}
if (parser.try_skip(',')) {
parser.skip_whitespaces();
continue;
}
return Status::Error("Unexpected symbol");
}
return Status::OK();
}
case '-':
case '+':
case '.':
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9': {
parser.read_while(
[](char c) { return c == '-' || ('0' <= c && c <= '9') || c == 'e' || c == 'E' || c == '+' || c == '.'; });
return Status::OK();
}
case 0:
return Status::Error("Unexpected end");
default: {
char next = parser.peek_char();
if (0 < next && next < 127) {
return Status::Error(PSLICE() << "Unexpected symbol '" << parser.peek_char() << "'");
} else {
return Status::Error("Unexpected symbol");
}
}
}
return Status::Error("Can't parse");
}
Slice JsonValue::get_type_name(Type type) {
switch (type) {
case Type::Null:
return Slice("Null");
case Type::Number:
return Slice("Number");
case Type::Boolean:
return Slice("Boolean");
case Type::String:
return Slice("String");
case Type::Array:
return Slice("Array");
case Type::Object:
return Slice("Object");
default:
UNREACHABLE();
return Slice("Unknown");
}
}
bool has_json_object_field(const JsonObject &object, Slice name) {
for (auto &field_value : object) {
if (field_value.first == name) {
return true;
}
}
return false;
}
Result<JsonValue> get_json_object_field(JsonObject &object, Slice name, JsonValue::Type type, bool is_optional) {
for (auto &field_value : object) {
if (field_value.first == name) {
if (type != JsonValue::Type::Null && field_value.second.type() != type) {
return Status::Error(400, PSLICE()
<< "Field \"" << name << "\" must be of type " << JsonValue::get_type_name(type));
}
return std::move(field_value.second);
}
}
if (!is_optional) {
return Status::Error(400, PSLICE() << "Can't find field \"" << name << "\"");
}
return JsonValue();
}
Result<bool> get_json_object_bool_field(JsonObject &object, Slice name, bool is_optional, bool default_value) {
TRY_RESULT(value, get_json_object_field(object, name, JsonValue::Type::Boolean, is_optional));
if (value.type() == JsonValue::Type::Null) {
return default_value;
}
return value.get_boolean();
}
Result<int32> get_json_object_int_field(JsonObject &object, Slice name, bool is_optional, int32 default_value) {
for (auto &field_value : object) {
if (field_value.first == name) {
if (field_value.second.type() == JsonValue::Type::String) {
return to_integer_safe<int32>(field_value.second.get_string());
}
if (field_value.second.type() == JsonValue::Type::Number) {
return to_integer_safe<int32>(field_value.second.get_number());
}
return Status::Error(400, PSLICE() << "Field \"" << name << "\" must be of type Number");
}
}
if (is_optional) {
return default_value;
}
return Status::Error(400, PSLICE() << "Can't find field \"" << name << "\"");
}
Result<int64> get_json_object_long_field(JsonObject &object, Slice name, bool is_optional, int64 default_value) {
for (auto &field_value : object) {
if (field_value.first == name) {
if (field_value.second.type() == JsonValue::Type::String) {
return to_integer_safe<int64>(field_value.second.get_string());
}
if (field_value.second.type() == JsonValue::Type::Number) {
return to_integer_safe<int64>(field_value.second.get_number());
}
return Status::Error(400, PSLICE() << "Field \"" << name << "\" must be a Number");
}
}
if (is_optional) {
return default_value;
}
return Status::Error(400, PSLICE() << "Can't find field \"" << name << "\"");
}
Result<double> get_json_object_double_field(JsonObject &object, Slice name, bool is_optional, double default_value) {
TRY_RESULT(value, get_json_object_field(object, name, JsonValue::Type::Number, is_optional));
if (value.type() == JsonValue::Type::Null) {
return default_value;
}
return to_double(value.get_number());
}
Result<string> get_json_object_string_field(JsonObject &object, Slice name, bool is_optional, string default_value) {
for (auto &field_value : object) {
if (field_value.first == name) {
if (field_value.second.type() == JsonValue::Type::String) {
return field_value.second.get_string().str();
}
if (field_value.second.type() == JsonValue::Type::Number) {
return field_value.second.get_number().str();
}
return Status::Error(400, PSLICE() << "Field \"" << name << "\" must be of type String");
}
}
if (is_optional) {
return default_value;
}
return Status::Error(400, PSLICE() << "Can't find field \"" << name << "\"");
}
} // namespace td

902
tdutils/td/utils/JsonBuilder.h Normal file
View file

@ -0,0 +1,902 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#include "td/utils/logging.h"
#include "td/utils/Parser.h"
#include "td/utils/Slice.h"
#include "td/utils/StackAllocator.h"
#include "td/utils/Status.h"
#include "td/utils/StringBuilder.h"
#include <new>
#include <tuple>
#include <type_traits>
#include <utility>
namespace td {
template <class... Args>
std::tuple<const Args &...> ctie(const Args &... args) TD_WARN_UNUSED_RESULT;
template <class... Args>
std::tuple<const Args &...> ctie(const Args &... args) {
return std::tie(args...);
}
class JsonTrue {
public:
friend StringBuilder &operator<<(StringBuilder &sb, const JsonTrue &val) {
return sb << "true";
}
};
class JsonFalse {
public:
friend StringBuilder &operator<<(StringBuilder &sb, const JsonFalse &val) {
return sb << "false";
}
};
class JsonNull {
public:
friend StringBuilder &operator<<(StringBuilder &sb, JsonNull val) {
return sb << "null";
}
};
class JsonBool {
public:
explicit JsonBool(bool value) : value_(value) {
}
friend StringBuilder &operator<<(StringBuilder &sb, const JsonBool &val) {
if (val.value_) {
return sb << JsonTrue();
} else {
return sb << JsonFalse();
}
}
private:
bool value_;
};
class JsonInt {
public:
explicit JsonInt(int32 value) : value_(value) {
}
friend StringBuilder &operator<<(StringBuilder &sb, const JsonInt &val) {
return sb << val.value_;
}
private:
int32 value_;
};
class JsonLong {
public:
explicit JsonLong(int64 value) : value_(value) {
}
friend StringBuilder &operator<<(StringBuilder &sb, const JsonLong &val) {
return sb << val.value_;
}
private:
int64 value_;
};
class JsonFloat {
public:
explicit JsonFloat(double value) : value_(value) {
}
friend StringBuilder &operator<<(StringBuilder &sb, const JsonFloat &val) {
return sb << val.value_;
}
private:
double value_;
};
class JsonOneChar {
public:
explicit JsonOneChar(unsigned int c) : c_(c) {
}
friend StringBuilder &operator<<(StringBuilder &sb, const JsonOneChar &val) {
auto c = val.c_;
return sb << '\\' << 'u' << "0123456789abcdef"[c >> 12] << "0123456789abcdef"[(c >> 8) & 15]
<< "0123456789abcdef"[(c >> 4) & 15] << "0123456789abcdef"[c & 15];
}
private:
unsigned int c_;
};
class JsonChar {
public:
explicit JsonChar(unsigned int c) : c_(c) {
}
friend StringBuilder &operator<<(StringBuilder &sb, const JsonChar &val) {
auto c = val.c_;
if (c < 0x10000) {
if (0xD7FF < c && c < 0xE000) {
// UTF-8 correctness has already been checked
UNREACHABLE();
}
return sb << JsonOneChar(c);
} else if (c <= 0x10ffff) {
return sb << JsonOneChar(0xD7C0 + (c >> 10)) << JsonOneChar(0xDC00 + (c & 0x3FF));
} else {
// UTF-8 correctness has already been checked
UNREACHABLE();
}
}
private:
unsigned int c_;
};
class JsonRaw {
public:
explicit JsonRaw(Slice value) : value_(value) {
}
friend StringBuilder &operator<<(StringBuilder &sb, const JsonRaw &val) {
return sb << val.value_;
}
private:
Slice value_;
};
class JsonRawString {
public:
explicit JsonRawString(Slice value) : value_(value) {
}
friend StringBuilder &operator<<(StringBuilder &sb, const JsonRawString &val);
private:
Slice value_;
};
class JsonString {
public:
explicit JsonString(Slice str) : str_(str) {
}
friend StringBuilder &operator<<(StringBuilder &sb, const JsonString &val);
private:
Slice str_;
};
class JsonScope;
class JsonValueScope;
class JsonArrayScope;
class JsonObjectScope;
class JsonBuilder {
public:
explicit JsonBuilder(StringBuilder &&sb, int32 offset = -1) : sb_(std::move(sb)), offset_(offset) {
}
StringBuilder &string_builder() {
return sb_;
}
friend class JsonScope;
JsonValueScope enter_value() TD_WARN_UNUSED_RESULT;
JsonArrayScope enter_array() TD_WARN_UNUSED_RESULT;
JsonObjectScope enter_object() TD_WARN_UNUSED_RESULT;
int32 offset() const {
return offset_;
}
bool is_pretty() const {
return offset_ >= 0;
}
void print_offset() {
for (int x = 0; x < offset_; x++) {
sb_ << " ";
}
}
void dec_offset() {
if (offset_ >= 0) {
CHECK(offset_ > 0);
offset_--;
}
}
void inc_offset() {
if (offset_ >= 0) {
offset_++;
}
}
private:
StringBuilder sb_;
JsonScope *scope_ = nullptr;
int32 offset_;
};
class Jsonable {};
class JsonScope {
public:
explicit JsonScope(JsonBuilder *jb) : sb_(&jb->sb_), jb_(jb), save_scope_(jb->scope_) {
jb_->scope_ = this;
CHECK(is_active());
}
JsonScope(const JsonScope &other) = delete;
JsonScope(JsonScope &&other) : sb_(other.sb_), jb_(other.jb_), save_scope_(other.save_scope_) {
other.jb_ = nullptr;
}
JsonScope &operator=(const JsonScope &) = delete;
JsonScope &operator=(JsonScope &&) = delete;
~JsonScope() {
if (jb_) {
leave();
}
}
void leave() {
CHECK(is_active());
jb_->scope_ = save_scope_;
}
protected:
StringBuilder *sb_;
// For CHECK
JsonBuilder *jb_;
JsonScope *save_scope_;
bool is_active() const {
return jb_ && jb_->scope_ == this;
}
JsonScope &operator<<(JsonTrue x) {
*sb_ << x;
return *this;
}
JsonScope &operator<<(JsonFalse x) {
*sb_ << x;
return *this;
}
JsonScope &operator<<(JsonNull x) {
*sb_ << x;
return *this;
}
JsonScope &operator<<(const JsonBool &x) {
*sb_ << x;
return *this;
}
JsonScope &operator<<(const JsonInt &x) {
*sb_ << x;
return *this;
}
JsonScope &operator<<(const JsonLong &x) {
*sb_ << x;
return *this;
}
JsonScope &operator<<(const JsonFloat &x) {
*sb_ << x;
return *this;
}
JsonScope &operator<<(const JsonString &x) {
*sb_ << x;
return *this;
}
JsonScope &operator<<(const JsonRawString &x) {
*sb_ << x;
return *this;
}
JsonScope &operator<<(const JsonRaw &x) {
*sb_ << x;
return *this;
}
JsonScope &operator<<(bool x) {
return *this << JsonBool(x);
}
JsonScope &operator<<(int32 x) {
return *this << JsonInt(x);
}
JsonScope &operator<<(int64 x) {
return *this << JsonLong(x);
}
JsonScope &operator<<(double x) {
return *this << JsonFloat(x);
}
template <class T>
JsonScope &operator<<(const T *x); // not implemented
template <size_t N>
JsonScope &operator<<(const char (&x)[N]) {
return *this << JsonString(Slice(x));
}
JsonScope &operator<<(const char *x) {
return *this << JsonString(Slice(x));
}
JsonScope &operator<<(const string &x) {
return *this << JsonString(Slice(x));
}
JsonScope &operator<<(Slice x) {
return *this << JsonString(x);
}
};
class JsonValueScope : public JsonScope {
public:
using JsonScope::JsonScope;
template <class T>
std::enable_if_t<std::is_base_of<Jsonable, typename std::decay<T>::type>::value, JsonValueScope &> operator<<(
const T &x) {
x.store(this);
return *this;
}
template <class T>
std::enable_if_t<!std::is_base_of<Jsonable, typename std::decay<T>::type>::value, JsonValueScope &> operator<<(
const T &x) {
CHECK(!was_);
was_ = true;
JsonScope::operator<<(x);
return *this;
}
JsonArrayScope enter_array() TD_WARN_UNUSED_RESULT;
JsonObjectScope enter_object() TD_WARN_UNUSED_RESULT;
private:
bool was_ = false;
};
class JsonArrayScope : public JsonScope {
public:
explicit JsonArrayScope(JsonBuilder *jb) : JsonScope(jb) {
jb->inc_offset();
*sb_ << "[";
}
JsonArrayScope(JsonArrayScope &&other) = default;
~JsonArrayScope() {
if (jb_) {
leave();
}
}
void leave() {
jb_->dec_offset();
if (jb_->is_pretty()) {
*sb_ << "\n";
jb_->print_offset();
}
*sb_ << "]";
}
template <class T>
JsonArrayScope &operator<<(const T &x) {
return (*this)(x);
}
template <class T>
JsonArrayScope &operator()(const T &x) {
enter_value() << x;
return *this;
}
JsonValueScope enter_value() {
CHECK(is_active());
if (is_first_) {
*sb_ << ",";
} else {
is_first_ = true;
}
if (jb_->is_pretty()) {
*sb_ << "\n";
jb_->print_offset();
}
return jb_->enter_value();
}
private:
bool is_first_ = false;
};
class JsonObjectScope : public JsonScope {
public:
explicit JsonObjectScope(JsonBuilder *jb) : JsonScope(jb) {
jb->inc_offset();
*sb_ << "{";
}
JsonObjectScope(JsonObjectScope &&other) = default;
~JsonObjectScope() {
if (jb_) {
leave();
}
}
void leave() {
jb_->dec_offset();
if (jb_->is_pretty()) {
*sb_ << "\n";
jb_->print_offset();
}
*sb_ << "}";
}
template <class S, class T>
JsonObjectScope &operator<<(std::tuple<S, T> key_value) {
return (*this)(std::get<0>(key_value), std::get<1>(key_value));
}
template <class S, class T>
JsonObjectScope &operator<<(std::pair<S, T> key_value) {
return (*this)(key_value.first, key_value.second);
}
template <class S, class T>
JsonObjectScope &operator()(S &&key, T &&value) {
CHECK(is_active());
if (is_first_) {
*sb_ << ",";
} else {
is_first_ = true;
}
if (jb_->is_pretty()) {
*sb_ << "\n";
jb_->print_offset();
}
jb_->enter_value() << key;
if (jb_->is_pretty()) {
*sb_ << " : ";
} else {
*sb_ << ":";
}
jb_->enter_value() << value;
return *this;
}
JsonObjectScope &operator<<(const JsonRaw &key_value) {
CHECK(is_active());
is_first_ = true;
jb_->enter_value() << key_value;
return *this;
}
private:
bool is_first_ = false;
};
inline JsonArrayScope JsonValueScope::enter_array() {
CHECK(!was_);
was_ = true;
return JsonArrayScope(jb_);
}
inline JsonObjectScope JsonValueScope::enter_object() {
CHECK(!was_);
was_ = true;
return JsonObjectScope(jb_);
}
inline JsonValueScope JsonBuilder::enter_value() {
return JsonValueScope(this);
}
inline JsonObjectScope JsonBuilder::enter_object() {
return JsonObjectScope(this);
}
inline JsonArrayScope JsonBuilder::enter_array() {
return JsonArrayScope(this);
}
class JsonValue;
using JsonObject = vector<std::pair<MutableSlice, JsonValue>>;
using JsonArray = vector<JsonValue>;
class JsonValue : public Jsonable {
public:
enum class Type { Null, Number, Boolean, String, Array, Object };
static Slice get_type_name(Type type);
JsonValue() {
}
~JsonValue() {
destroy();
}
JsonValue(JsonValue &&other) : JsonValue() {
init(std::move(other));
}
JsonValue &operator=(JsonValue &&other) {
if (&other == this) {
return *this;
}
destroy();
init(std::move(other));
return *this;
}
JsonValue(const JsonValue &other) = delete;
JsonValue &operator=(const JsonValue &other) = delete;
Type type() const {
return type_;
}
MutableSlice &get_string() {
CHECK(type_ == Type::String);
return string_;
}
const MutableSlice &get_string() const {
CHECK(type_ == Type::String);
return string_;
}
bool &get_boolean() {
CHECK(type_ == Type::Boolean);
return boolean_;
}
const bool &get_boolean() const {
CHECK(type_ == Type::Boolean);
return boolean_;
}
MutableSlice &get_number() {
CHECK(type_ == Type::Number);
return number_;
}
const MutableSlice &get_number() const {
CHECK(type_ == Type::Number);
return number_;
}
JsonArray &get_array() {
CHECK(type_ == Type::Array);
return array_;
}
const JsonArray &get_array() const {
CHECK(type_ == Type::Array);
return array_;
}
JsonObject &get_object() {
CHECK(type_ == Type::Object);
return object_;
}
const JsonObject &get_object() const {
CHECK(type_ == Type::Object);
return object_;
}
static JsonValue create_boolean(bool val) {
JsonValue res;
res.init_boolean(val);
return res;
}
static JsonValue create_number(MutableSlice number) {
JsonValue res;
res.init_number(number);
return res;
}
static JsonValue create_string(MutableSlice str) {
JsonValue res;
res.init_string(str);
return res;
}
static JsonValue create_array(JsonArray v) {
JsonValue res;
res.init_array(std::move(v));
return res;
}
static JsonValue make_object(JsonObject c) {
JsonValue res;
res.init_object(std::move(c));
return res;
}
void store(JsonValueScope *scope) const {
switch (type_) {
case Type::Null:
*scope << JsonRaw("null");
break;
case Type::Boolean:
if (get_boolean()) {
*scope << JsonRaw("true");
} else {
*scope << JsonRaw("false");
}
break;
case Type::Number:
*scope << JsonRaw(get_number());
break;
case Type::String:
*scope << JsonString(get_string());
break;
case Type::Array: {
auto arr = scope->enter_array();
for (auto &val : get_array()) {
arr << val;
}
break;
}
case Type::Object: {
auto object = scope->enter_object();
for (auto &key_value : get_object()) {
object << ctie(JsonString(key_value.first), key_value.second);
}
break;
}
}
};
private:
Type type_{Type::Null};
union {
MutableSlice number_;
bool boolean_;
MutableSlice string_;
JsonArray array_;
JsonObject object_;
};
void init_null() {
type_ = Type::Null;
}
void init_number(MutableSlice number) {
type_ = Type::Number;
new (&number_) MutableSlice(number);
}
void init_boolean(bool boolean) {
type_ = Type::Boolean;
boolean_ = boolean;
}
void init_string(MutableSlice slice) {
type_ = Type::String;
new (&string_) MutableSlice(slice);
}
void init_array(JsonArray array) {
type_ = Type::Array;
new (&array_) JsonArray(std::move(array));
}
void init_object(JsonObject object) {
type_ = Type::Object;
new (&object_) JsonObject(std::move(object));
}
void init(JsonValue &&other) {
switch (other.type_) {
case Type::Null:
break;
case Type::Number:
init_number(other.number_);
break;
case Type::Boolean:
init_boolean(other.boolean_);
break;
case Type::String:
init_string(other.string_);
break;
case Type::Array:
init_array(std::move(other.array_));
break;
case Type::Object:
init_object(std::move(other.object_));
break;
}
other.destroy();
}
void destroy() {
switch (type_) {
case Type::Null:
case Type::Boolean:
break;
case Type::Number:
number_.~MutableSlice();
break;
case Type::String:
string_.~MutableSlice();
break;
case Type::Array:
array_.~vector<JsonValue>();
break;
case Type::Object:
object_.~vector<std::pair<MutableSlice, JsonValue>>();
break;
}
type_ = Type::Null;
}
};
inline StringBuilder &operator<<(StringBuilder &sb, JsonValue::Type type) {
switch (type) {
case JsonValue::Type::Null:
return sb << "Null";
case JsonValue::Type::Number:
return sb << "Number";
case JsonValue::Type::Boolean:
return sb << "Boolean";
case JsonValue::Type::String:
return sb << "String";
case JsonValue::Type::Array:
return sb << "Array";
case JsonValue::Type::Object:
return sb << "Object";
default:
UNREACHABLE();
return sb;
}
}
class VirtuallyJsonable : public Jsonable {
public:
virtual void store(JsonValueScope *scope) const = 0;
VirtuallyJsonable() = default;
VirtuallyJsonable(const VirtuallyJsonable &) = delete;
VirtuallyJsonable &operator=(const VirtuallyJsonable &) = delete;
VirtuallyJsonable(VirtuallyJsonable &&) = default;
VirtuallyJsonable &operator=(VirtuallyJsonable &&) = default;
virtual ~VirtuallyJsonable() = default;
};
class VirtuallyJsonableInt : public VirtuallyJsonable {
public:
explicit VirtuallyJsonableInt(int32 value) : value_(value) {
}
void store(JsonValueScope *scope) const override {
*scope << JsonInt(value_);
}
private:
int32 value_;
};
class VirtuallyJsonableLong : public VirtuallyJsonable {
public:
explicit VirtuallyJsonableLong(int64 value) : value_(value) {
}
void store(JsonValueScope *scope) const override {
*scope << JsonLong(value_);
}
private:
int64 value_;
};
class VirtuallyJsonableString : public VirtuallyJsonable {
public:
explicit VirtuallyJsonableString(Slice value) : value_(value) {
}
void store(JsonValueScope *scope) const override {
*scope << JsonString(value_);
}
private:
Slice value_;
};
Result<MutableSlice> json_string_decode(Parser &parser) TD_WARN_UNUSED_RESULT;
Status json_string_skip(Parser &parser) TD_WARN_UNUSED_RESULT;
Result<JsonValue> do_json_decode(Parser &parser, int32 max_depth) TD_WARN_UNUSED_RESULT;
Status do_json_skip(Parser &parser, int32 max_depth) TD_WARN_UNUSED_RESULT;
inline Result<JsonValue> json_decode(MutableSlice json) {
Parser parser(json);
const int32 DEFAULT_MAX_DEPTH = 100;
auto result = do_json_decode(parser, DEFAULT_MAX_DEPTH);
if (result.is_ok()) {
parser.skip_whitespaces();
if (!parser.empty()) {
return Status::Error("Expected string end");
}
}
return result;
}
template <class StrT, class ValT>
StrT json_encode(const ValT &val, bool pretty = false) {
auto buf_len = 1 << 18;
auto buf = StackAllocator::alloc(buf_len);
JsonBuilder jb(StringBuilder(buf.as_slice(), true), pretty ? 0 : -1);
jb.enter_value() << val;
if (pretty) {
jb.string_builder() << "\n";
}
LOG_IF(ERROR, jb.string_builder().is_error()) << "JSON buffer overflow";
auto slice = jb.string_builder().as_cslice();
return StrT(slice.begin(), slice.size());
}
template <class T>
class ToJsonImpl : public Jsonable {
public:
explicit ToJsonImpl(const T &value) : value_(value) {
}
void store(JsonValueScope *scope) const {
to_json(*scope, value_);
}
private:
const T &value_;
};
template <class T>
auto ToJson(const T &value) {
return ToJsonImpl<T>(value);
}
template <class T>
void to_json(JsonValueScope &jv, const T &value) {
jv << value;
}
template <class F>
class JsonObjectImpl : Jsonable {
public:
explicit JsonObjectImpl(F &&f) : f_(std::forward<F>(f)) {
}
void store(JsonValueScope *scope) const {
auto object = scope->enter_object();
f_(object);
}
private:
F f_;
};
template <class F>
auto json_object(F &&f) {
return JsonObjectImpl<F>(std::forward<F>(f));
}
template <class F>
class JsonArrayImpl : Jsonable {
public:
explicit JsonArrayImpl(F &&f) : f_(std::forward<F>(f)) {
}
void store(JsonValueScope *scope) const {
auto array = scope->enter_array();
f_(array);
}
private:
F f_;
};
template <class F>
auto json_array(F &&f) {
return JsonArrayImpl<F>(std::forward<F>(f));
}
template <class A, class F>
auto json_array(const A &a, F &&f) {
return json_array([&a, &f](auto &arr) {
for (auto &x : a) {
arr(f(x));
}
});
}
bool has_json_object_field(const JsonObject &object, Slice name);
Result<JsonValue> get_json_object_field(JsonObject &object, Slice name, JsonValue::Type type,
bool is_optional = true) TD_WARN_UNUSED_RESULT;
Result<bool> get_json_object_bool_field(JsonObject &object, Slice name, bool is_optional = true,
bool default_value = false) TD_WARN_UNUSED_RESULT;
Result<int32> get_json_object_int_field(JsonObject &object, Slice name, bool is_optional = true,
int32 default_value = 0) TD_WARN_UNUSED_RESULT;
Result<int64> get_json_object_long_field(JsonObject &object, Slice name, bool is_optional = true,
int64 default_value = 0) TD_WARN_UNUSED_RESULT;
Result<double> get_json_object_double_field(JsonObject &object, Slice name, bool is_optional = true,
double default_value = 0.0) TD_WARN_UNUSED_RESULT;
Result<string> get_json_object_string_field(JsonObject &object, Slice name, bool is_optional = true,
string default_value = "") TD_WARN_UNUSED_RESULT;
} // namespace td

104
tdutils/td/utils/List.h Normal file
View file

@ -0,0 +1,104 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
namespace td {
struct ListNode {
ListNode *next;
ListNode *prev;
ListNode() {
clear();
}
~ListNode() {
remove();
}
ListNode(const ListNode &) = delete;
ListNode &operator=(const ListNode &) = delete;
ListNode(ListNode &&other) {
if (other.empty()) {
clear();
} else {
ListNode *head = other.prev;
other.remove();
head->put(this);
}
}
ListNode &operator=(ListNode &&other) {
this->remove();
if (!other.empty()) {
ListNode *head = other.prev;
other.remove();
head->put(this);
}
return *this;
}
void connect(ListNode *to) {
CHECK(to != nullptr);
next = to;
to->prev = this;
}
void remove() {
prev->connect(next);
clear();
}
void put(ListNode *other) {
other->connect(next);
this->connect(other);
}
void put_back(ListNode *other) {
prev->connect(other);
other->connect(this);
}
ListNode *get() {
ListNode *result = prev;
if (result == this) {
return nullptr;
}
result->prev->connect(this);
result->clear();
// this->connect(result->next);
return result;
}
bool empty() const {
return next == this;
}
private:
void clear() {
next = this;
prev = this;
}
};
} // namespace td

95
tdutils/td/utils/MemoryLog.h Normal file
View file

@ -0,0 +1,95 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#include "td/utils/logging.h"
#include "td/utils/Slice.h"
#include <atomic>
#include <cstdio>
#include <cstring>
namespace td {
template <int buffer_size = 32 * (1 << 10)>
class MemoryLog : public LogInterface {
static constexpr size_t MAX_OUTPUT_SIZE = buffer_size / 16 < (8 << 10) ? buffer_size / 16 : (8 << 10);
public:
MemoryLog() {
std::memset(buffer_, ' ', sizeof(buffer_));
}
void append(CSlice new_slice, int log_level) override {
Slice slice = new_slice;
slice.truncate(MAX_OUTPUT_SIZE);
while (!slice.empty() && slice.back() == '\n') {
slice.remove_suffix(1);
}
size_t slice_size = slice.size();
CHECK(slice_size * 3 < buffer_size);
size_t pad_size = ((slice_size + 15) & ~15) - slice_size;
constexpr size_t magic_size = 16;
uint32 total_size = static_cast<uint32>(slice_size + pad_size + magic_size);
uint32 real_pos = pos_.fetch_add(total_size, std::memory_order_relaxed);
CHECK((total_size & 15) == 0);
uint32 start_pos = real_pos & (buffer_size - 1);
uint32 end_pos = start_pos + total_size;
if (likely(end_pos <= buffer_size)) {
std::memcpy(&buffer_[start_pos + magic_size], slice.data(), slice_size);
std::memcpy(&buffer_[start_pos + magic_size + slice_size], " ", pad_size);
} else {
size_t first = buffer_size - start_pos - magic_size;
size_t second = slice_size - first;
std::memcpy(&buffer_[start_pos + magic_size], slice.data(), first);
std::memcpy(&buffer_[0], slice.data() + first, second);
std::memcpy(&buffer_[second], " ", pad_size);
}
CHECK((start_pos & 15) == 0);
CHECK(start_pos <= buffer_size - magic_size);
buffer_[start_pos] = '\n';
size_t printed = std::snprintf(&buffer_[start_pos + 1], magic_size - 1, "LOG:%08x: ", real_pos);
CHECK(printed == magic_size - 2);
buffer_[start_pos + magic_size - 1] = ' ';
if (log_level == VERBOSITY_NAME(FATAL)) {
process_fatal_error(new_slice);
}
}
void rotate() override {
}
Slice get_buffer() const {
return Slice(buffer_, sizeof(buffer_));
}
size_t get_pos() const {
return pos_ & (buffer_size - 1);
}
private:
char buffer_[buffer_size];
std::atomic<uint32> pos_{0};
};
} // namespace td

56
tdutils/td/utils/MimeType.cpp Normal file
View file

@ -0,0 +1,56 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/utils/MimeType.h"
#include "td/utils/logging.h"
const char *extension_to_mime_type(const char *extension, size_t extension_len); // auto-generated
const char *mime_type_to_extension(const char *mime_type, size_t mime_type_len); // auto-generated
namespace td {
string MimeType::to_extension(Slice mime_type, Slice default_value) {
if (mime_type.empty()) {
return default_value.str();
}
const char *result = ::mime_type_to_extension(mime_type.data(), mime_type.size());
if (result != nullptr) {
return result;
}
LOG(INFO) << "Unknown file MIME type " << mime_type;
return default_value.str();
}
string MimeType::from_extension(Slice extension, Slice default_value) {
if (extension.empty()) {
return default_value.str();
}
const char *result = ::extension_to_mime_type(extension.data(), extension.size());
if (result != nullptr) {
return result;
}
LOG(INFO) << "Unknown file extension " << extension;
return default_value.str();
}
} // namespace td

32
tdutils/td/utils/MimeType.h Normal file
View file

@ -0,0 +1,32 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#include "td/utils/Slice.h"
namespace td {
class MimeType {
public:
static string to_extension(Slice mime_type, Slice default_value = Slice());
static string from_extension(Slice extension, Slice default_value = Slice());
};
} // namespace td

52
tdutils/td/utils/MovableValue.h Normal file
View file

@ -0,0 +1,52 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
namespace td {
template <class T, T empty_val = T()>
class MovableValue {
public:
MovableValue() = default;
MovableValue(T val) : val_(val) {
}
MovableValue(MovableValue &&other) : val_(other.val_) {
other.clear();
}
MovableValue &operator=(MovableValue &&other) {
val_ = other.val_;
other.clear();
return *this;
}
MovableValue(const MovableValue &) = delete;
MovableValue &operator=(const MovableValue &) = delete;
~MovableValue() = default;
void clear() {
val_ = empty_val;
}
const T &get() const {
return val_;
}
private:
T val_ = empty_val;
};
} // namespace td

27
tdutils/td/utils/MpmcQueue.cpp Normal file
View file

@ -0,0 +1,27 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/utils/MpmcQueue.h"
namespace td {
namespace detail {
MpmcStat stat_;
} // namespace detail
} // namespace td

469
tdutils/td/utils/MpmcQueue.h Normal file
View file

@ -0,0 +1,469 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
// MPMC queue
// Simple semaphore protected implementation
// To close queue, one should send as much sentinel elements as there are readers.
// Once there are no readers and writers, one may easily destroy queue
#include "td/utils/format.h"
#include "td/utils/HazardPointers.h"
#include "td/utils/logging.h"
#include "td/utils/port/thread.h"
#include "td/utils/ScopeGuard.h"
#include <array>
#include <atomic>
namespace td {
namespace detail {
struct MpmcStat {
void alloc_ok(size_t thread_id) {
s(thread_id).alloc_ok_cnt++;
}
void alloc_error(size_t thread_id) {
s(thread_id).alloc_error_cnt++;
}
void push_loop_error(size_t thread_id) {
s(thread_id).push_loop_error_cnt++;
}
void push_loop_ok(size_t thread_id) {
s(thread_id).push_loop_ok_cnt++;
}
void dump() {
int alloc_ok_cnt = 0;
int alloc_error_cnt = 0;
int push_loop_error_cnt = 0;
int push_loop_ok_cnt = 0;
for (auto &d : arr) {
alloc_ok_cnt += d.alloc_ok_cnt;
alloc_error_cnt += d.alloc_error_cnt;
push_loop_error_cnt += d.push_loop_error_cnt;
push_loop_ok_cnt += d.push_loop_ok_cnt;
}
LOG(ERROR) << tag("alloc_ok_cnt", alloc_ok_cnt) << tag("alloc_error_cnt", alloc_error_cnt)
<< tag("push_loop_error_cnt", push_loop_error_cnt) << tag("push_loop_ok_cnt", push_loop_ok_cnt);
}
private:
struct ThreadStat {
int alloc_ok_cnt{0};
int alloc_error_cnt{0};
int push_loop_ok_cnt{0};
int push_loop_error_cnt{0};
char pad[TD_CONCURRENCY_PAD - sizeof(int) * 4];
};
std::array<ThreadStat, 1024> arr;
ThreadStat &s(size_t thread_id) {
return arr[thread_id];
}
};
extern MpmcStat stat_;
} // namespace detail
template <class T>
class OneValue {
public:
bool set_value(T &value) {
value_ = std::move(value);
int state = Empty;
if (state_.compare_exchange_strong(state, Value, std::memory_order_acq_rel)) {
return true;
}
value = std::move(value_);
return false;
}
bool get_value(T &value) {
auto old_state = state_.exchange(Taken, std::memory_order_acq_rel);
if (old_state == Value) {
value = std::move(value_);
return true;
}
return false;
}
void reset() {
state_ = Empty;
value_ = T();
}
private:
enum Type : int { Empty = 0, Taken, Value };
std::atomic<int> state_{Empty};
T value_{};
};
template <class T>
class OneValue<T *> {
public:
bool set_value(T *value) {
T *was = nullptr;
return state_.compare_exchange_strong(was, value, std::memory_order_acq_rel);
}
bool get_value(T *&value) {
value = state_.exchange(Taken(), std::memory_order_acq_rel);
return value != nullptr;
}
void reset() {
state_ = nullptr;
}
OneValue() {
}
private:
std::atomic<T *> state_{nullptr};
T *Taken() {
static T xxx;
return &xxx;
}
};
template <class T>
class MpmcQueueBlock {
public:
explicit MpmcQueueBlock(size_t size) : nodes_(size) {
}
enum class PopStatus { Ok, Empty, Closed };
//blocking pop
//returns Ok or Closed
PopStatus pop(T &value) {
while (true) {
auto read_pos = read_pos_.fetch_add(1, std::memory_order_relaxed);
if (read_pos >= nodes_.size()) {
return PopStatus::Closed;
}
//TODO blocking get_value
if (nodes_[static_cast<size_t>(read_pos)].one_value.get_value(value)) {
return PopStatus::Ok;
}
}
}
//nonblocking pop
//returns Ok, Empty or Closed
PopStatus try_pop(T &value) {
while (true) {
// this check slows 1:1 case but prevents writer starvation in 1:N case
if (write_pos_.load(std::memory_order_relaxed) <= read_pos_.load(std::memory_order_relaxed) &&
read_pos_.load(std::memory_order_relaxed) < nodes_.size()) {
return PopStatus::Empty;
}
auto read_pos = read_pos_.fetch_add(1, std::memory_order_relaxed);
if (read_pos >= nodes_.size()) {
return PopStatus::Closed;
}
if (nodes_[static_cast<size_t>(read_pos)].one_value.get_value(value)) {
return PopStatus::Ok;
}
auto write_pos = write_pos_.load(std::memory_order_relaxed);
if (write_pos <= read_pos + 1) {
return PopStatus::Empty;
}
}
}
enum class PushStatus { Ok, Closed };
PushStatus push(T &value) {
while (true) {
auto write_pos = write_pos_.fetch_add(1, std::memory_order_relaxed);
if (write_pos >= nodes_.size()) {
return PushStatus::Closed;
}
if (nodes_[static_cast<size_t>(write_pos)].one_value.set_value(value)) {
//stat_.push_loop_ok(0);
return PushStatus::Ok;
}
//stat_.push_loop_error(0);
}
}
private:
struct Node {
OneValue<T> one_value;
};
std::atomic<uint64> write_pos_{0};
char pad[TD_CONCURRENCY_PAD - sizeof(std::atomic<uint64>)];
std::atomic<uint64> read_pos_{0};
char pad2[TD_CONCURRENCY_PAD - sizeof(std::atomic<uint64>)];
std::vector<Node> nodes_;
char pad3[TD_CONCURRENCY_PAD - sizeof(std::vector<Node>)];
};
template <class T>
class MpmcQueueOld {
public:
explicit MpmcQueueOld(size_t threads_n) : MpmcQueueOld(1024, threads_n) {
}
static std::string get_description() {
return "Mpmc queue (fetch and add array queue)";
}
MpmcQueueOld(size_t block_size, size_t threads_n) : block_size_{block_size}, hazard_pointers_{threads_n} {
auto node = make_unique<Node>(block_size_);
write_pos_ = node.get();
read_pos_ = node.get();
node.release();
}
MpmcQueueOld(const MpmcQueueOld &other) = delete;
MpmcQueueOld &operator=(const MpmcQueueOld &other) = delete;
MpmcQueueOld(MpmcQueueOld &&other) = delete;
MpmcQueueOld &operator=(MpmcQueueOld &&other) = delete;
~MpmcQueueOld() {
auto *ptr = read_pos_.load(std::memory_order_relaxed);
while (ptr) {
auto *to_delete = ptr;
ptr = ptr->next_.load(std::memory_order_relaxed);
delete to_delete;
}
//stat_.dump();
//stat_ = detail::MpmcStat();
}
size_t hazard_pointers_to_delele_size_unsafe() const {
return hazard_pointers_.to_delete_size_unsafe();
}
void gc(size_t thread_id) {
hazard_pointers_.retire(thread_id);
}
using PushStatus = typename MpmcQueueBlock<T>::PushStatus;
using PopStatus = typename MpmcQueueBlock<T>::PopStatus;
void push(T value, size_t thread_id) {
typename decltype(hazard_pointers_)::Holder hazard_ptr_holder(hazard_pointers_, thread_id, 0);
while (true) {
auto node = hazard_ptr_holder.protect(write_pos_);
auto status = node->block.push(value);
switch (status) {
case PushStatus::Ok:
return;
case PushStatus::Closed: {
auto next = node->next_.load(std::memory_order_acquire);
if (next == nullptr) {
auto new_node = new Node(block_size_);
new_node->block.push(value);
if (node->next_.compare_exchange_strong(next, new_node, std::memory_order_acq_rel)) {
//stat_.alloc_ok(thread_id);
write_pos_.compare_exchange_strong(node, new_node, std::memory_order_acq_rel);
return;
} else {
//stat_.alloc_error(thread_id);
new_node->block.pop(value);
//CHECK(status == PopStatus::Ok);
delete new_node;
}
}
//CHECK(next != nullptr);
write_pos_.compare_exchange_strong(node, next, std::memory_order_acq_rel);
break;
}
}
}
}
bool try_pop(T &value, size_t thread_id) {
typename decltype(hazard_pointers_)::Holder hazard_ptr_holder(hazard_pointers_, thread_id, 0);
while (true) {
auto node = hazard_ptr_holder.protect(read_pos_);
auto status = node->block.try_pop(value);
switch (status) {
case PopStatus::Ok:
return true;
case PopStatus::Empty:
return false;
case PopStatus::Closed: {
auto next = node->next_.load(std::memory_order_acquire);
if (!next) {
return false;
}
if (read_pos_.compare_exchange_strong(node, next, std::memory_order_acq_rel)) {
hazard_ptr_holder.clear();
hazard_pointers_.retire(thread_id, node);
}
break;
}
}
}
}
T pop(size_t thread_id) {
T value;
while (true) {
if (try_pop(value, thread_id)) {
return value;
}
td::this_thread::yield();
}
}
private:
struct Node {
explicit Node(size_t block_size) : block{block_size} {
}
std::atomic<Node *> next_{nullptr};
char pad[TD_CONCURRENCY_PAD - sizeof(std::atomic<Node *>)];
MpmcQueueBlock<T> block;
//Got pad in MpmcQueueBlock
};
std::atomic<Node *> write_pos_{nullptr};
char pad[TD_CONCURRENCY_PAD - sizeof(std::atomic<Node *>)];
std::atomic<Node *> read_pos_{nullptr};
char pad2[TD_CONCURRENCY_PAD - sizeof(std::atomic<Node *>)];
size_t block_size_;
HazardPointers<Node, 1> hazard_pointers_;
//Got pad in HazardPointers
};
template <class T>
class MpmcQueue {
public:
explicit MpmcQueue(size_t threads_n) : MpmcQueue(1024, threads_n) {
}
static std::string get_description() {
return "NEW Mpmc queue (fetch and add array queue)";
}
MpmcQueue(size_t block_size, size_t threads_n) : hazard_pointers_{threads_n} {
auto node = make_unique<Node>();
write_pos_ = node.get();
read_pos_ = node.get();
node.release();
}
MpmcQueue(const MpmcQueue &other) = delete;
MpmcQueue &operator=(const MpmcQueue &other) = delete;
MpmcQueue(MpmcQueue &&other) = delete;
MpmcQueue &operator=(MpmcQueue &&other) = delete;
~MpmcQueue() {
auto *ptr = read_pos_.load(std::memory_order_relaxed);
while (ptr) {
auto *to_delete = ptr;
ptr = ptr->next.load(std::memory_order_relaxed);
delete to_delete;
}
}
size_t hazard_pointers_to_delele_size_unsafe() const {
return hazard_pointers_.to_delete_size_unsafe();
}
void gc(size_t thread_id) {
hazard_pointers_.retire(thread_id);
}
void push(T value, size_t thread_id) {
SCOPE_EXIT {
hazard_pointers_.clear(thread_id, 0);
};
while (true) {
auto node = hazard_pointers_.protect(thread_id, 0, write_pos_);
auto &block = node->block;
auto pos = block.write_pos++;
if (pos >= block.data.size()) {
auto next = node->next.load();
if (next == nullptr) {
auto new_node = new Node{};
new_node->block.write_pos++;
new_node->block.data[0].set_value(value);
Node *null = nullptr;
if (node->next.compare_exchange_strong(null, new_node)) {
write_pos_.compare_exchange_strong(node, new_node);
return;
} else {
new_node->block.data[0].get_value(value);
delete new_node;
}
} else {
write_pos_.compare_exchange_strong(node, next);
}
} else {
if (block.data[static_cast<size_t>(pos)].set_value(value)) {
return;
}
}
}
}
bool try_pop(T &value, size_t thread_id) {
SCOPE_EXIT {
hazard_pointers_.clear(thread_id, 0);
};
while (true) {
auto node = hazard_pointers_.protect(thread_id, 0, read_pos_);
auto &block = node->block;
if (block.write_pos <= block.read_pos && node->next.load(std::memory_order_relaxed) == nullptr) {
return false;
}
auto pos = block.read_pos++;
if (pos >= block.data.size()) {
auto next = node->next.load();
if (!next) {
return false;
}
if (read_pos_.compare_exchange_strong(node, next)) {
hazard_pointers_.clear(thread_id, 0);
hazard_pointers_.retire(thread_id, node);
}
} else {
if (block.data[static_cast<size_t>(pos)].get_value(value)) {
return true;
}
}
}
}
T pop(size_t thread_id) {
T value;
while (true) {
if (try_pop(value, thread_id)) {
return value;
}
td::this_thread::yield();
}
}
private:
struct Block {
std::atomic<uint64> write_pos{0};
char pad[TD_CONCURRENCY_PAD - sizeof(std::atomic<uint64>)];
std::atomic<uint64> read_pos{0};
char pad2[TD_CONCURRENCY_PAD - sizeof(std::atomic<uint64>)];
std::array<OneValue<T>, 1024> data;
char pad3[TD_CONCURRENCY_PAD];
};
struct Node {
Node() = default;
Block block;
std::atomic<Node *> next{nullptr};
char pad[TD_CONCURRENCY_PAD - sizeof(std::atomic<Node *>)];
//Got pad in MpmcQueueBlock
};
std::atomic<Node *> write_pos_{nullptr};
char pad[TD_CONCURRENCY_PAD - sizeof(std::atomic<Node *>)];
std::atomic<Node *> read_pos_{nullptr};
char pad2[TD_CONCURRENCY_PAD - sizeof(std::atomic<Node *>)];
HazardPointers<Node, 1> hazard_pointers_;
//Got pad in HazardPointers
};
} // namespace td

120
tdutils/td/utils/MpmcWaiter.h Normal file
View file

@ -0,0 +1,120 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#include "td/utils/port/thread.h"
#include <atomic>
#include <condition_variable>
#include <mutex>
namespace td {
class MpmcWaiter {
public:
int wait(int yields, uint32 worker_id) {
if (yields < RoundsTillSleepy) {
td::this_thread::yield();
return yields + 1;
} else if (yields == RoundsTillSleepy) {
auto state = state_.load(std::memory_order_relaxed);
if (!State::has_worker(state)) {
auto new_state = State::with_worker(state, worker_id);
if (state_.compare_exchange_strong(state, new_state, std::memory_order_acq_rel)) {
td::this_thread::yield();
return yields + 1;
}
if (state == State::awake()) {
return 0;
}
}
td::this_thread::yield();
return 0;
} else if (yields < RoundsTillAsleep) {
auto state = state_.load(std::memory_order_acquire);
if (State::still_sleepy(state, worker_id)) {
td::this_thread::yield();
return yields + 1;
}
return 0;
} else {
auto state = state_.load(std::memory_order_acquire);
if (State::still_sleepy(state, worker_id)) {
std::unique_lock<std::mutex> lock(mutex_);
if (state_.compare_exchange_strong(state, State::asleep(), std::memory_order_acq_rel)) {
condition_variable_.wait(lock);
}
}
return 0;
}
}
int stop_wait(int yields, uint32 worker_id) {
if (yields > RoundsTillSleepy) {
notify_cold();
}
return 0;
}
void notify() {
std::atomic_thread_fence(std::memory_order_seq_cst);
if (state_.load(std::memory_order_acquire) == State::awake()) {
return;
}
notify_cold();
}
private:
struct State {
static constexpr uint32 awake() {
return 0;
}
static constexpr uint32 asleep() {
return 1;
}
static bool is_asleep(uint32 state) {
return (state & 1) != 0;
}
static bool has_worker(uint32 state) {
return (state >> 1) != 0;
}
static int32 with_worker(uint32 state, uint32 worker) {
return state | ((worker + 1) << 1);
}
static bool still_sleepy(uint32 state, uint32 worker) {
return (state >> 1) == (worker + 1);
}
};
//enum { RoundsTillSleepy = 32, RoundsTillAsleep = 64 };
enum { RoundsTillSleepy = 1, RoundsTillAsleep = 2 };
std::atomic<uint32> state_{State::awake()};
std::mutex mutex_;
std::condition_variable condition_variable_;
void notify_cold() {
auto old_state = state_.exchange(State::awake(), std::memory_order_release);
if (State::is_asleep(old_state)) {
std::lock_guard<std::mutex> guard(mutex_);
condition_variable_.notify_all();
}
}
};
} // namespace td

185
tdutils/td/utils/MpscLinkQueue.h Normal file
View file

@ -0,0 +1,185 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#include <atomic>
namespace td {
//NB: holder of the queue holds all responsibility of freeing its nodes
class MpscLinkQueueImpl {
public:
class Node;
class Reader;
void push(Node *node) {
node->next_ = head_.load(std::memory_order_relaxed);
while (!head_.compare_exchange_strong(node->next_, node, std::memory_order_release, std::memory_order_relaxed)) {
}
}
void push_unsafe(Node *node) {
node->next_ = head_.load(std::memory_order_relaxed);
head_.store(node, std::memory_order_relaxed);
}
void pop_all(Reader &reader) {
return reader.add(head_.exchange(nullptr, std::memory_order_acquire));
}
void pop_all_unsafe(Reader &reader) {
return reader.add(head_.exchange(nullptr, std::memory_order_relaxed));
}
class Node {
friend class MpscLinkQueueImpl;
Node *next_{nullptr};
};
class Reader {
public:
Node *read() {
auto old_head = head_;
if (head_) {
head_ = head_->next_;
}
return old_head;
}
void delay(Node *node) {
node->next_ = head_;
if (!head_) {
tail_ = node;
}
head_ = node;
}
size_t calc_size() const {
size_t res = 0;
for (auto it = head_; it != nullptr; it = it->next_, res++) {
}
return res;
}
private:
friend class MpscLinkQueueImpl;
void add(Node *node) {
if (node == nullptr) {
return;
}
// Reverse list
Node *tail = node;
Node *head = nullptr;
while (node) {
auto next = node->next_;
node->next_ = head;
head = node;
node = next;
}
if (head_ == nullptr) {
head_ = head;
} else {
tail_->next_ = head;
}
tail_ = tail;
}
Node *head_{nullptr};
Node *tail_{nullptr};
};
private:
std::atomic<Node *> head_{nullptr};
};
// Uses MpscLinkQueueImpl.
// Node should have to_mpsc_link_queue_node and from_mpsc_link_queue_node functions
template <class Node>
class MpscLinkQueue {
public:
void push(Node node) {
impl_.push(node.to_mpsc_link_queue_node());
}
void push_unsafe(Node node) {
impl_.push_unsafe(node.to_mpsc_link_queue_node());
}
class Reader {
public:
~Reader() {
CHECK(!read());
}
Node read() {
auto node = impl_.read();
if (!node) {
return {};
}
return Node::from_mpsc_link_queue_node(node);
}
void delay(Node node) {
impl_.delay(node.to_mpsc_link_queue_node());
}
size_t calc_size() const {
return impl_.calc_size();
}
private:
friend class MpscLinkQueue;
MpscLinkQueueImpl::Reader impl_;
MpscLinkQueueImpl::Reader &impl() {
return impl_;
}
};
void pop_all(Reader &reader) {
return impl_.pop_all(reader.impl());
}
void pop_all_unsafe(Reader &reader) {
return impl_.pop_all_unsafe(reader.impl());
}
private:
MpscLinkQueueImpl impl_;
};
template <class Value>
class MpscLinkQueueUniquePtrNode {
public:
MpscLinkQueueUniquePtrNode() = default;
explicit MpscLinkQueueUniquePtrNode(unique_ptr<Value> ptr) : ptr_(std::move(ptr)) {
}
MpscLinkQueueImpl::Node *to_mpsc_link_queue_node() {
return ptr_.release()->to_mpsc_link_queue_node();
}
static MpscLinkQueueUniquePtrNode<Value> from_mpsc_link_queue_node(td::MpscLinkQueueImpl::Node *node) {
return MpscLinkQueueUniquePtrNode<Value>(unique_ptr<Value>(Value::from_mpsc_link_queue_node(node)));
}
explicit operator bool() {
return ptr_ != nullptr;
}
Value &value() {
return *ptr_;
}
private:
unique_ptr<Value> ptr_;
};
} // namespace td

164
tdutils/td/utils/MpscPollableQueue.h Normal file
View file

@ -0,0 +1,164 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#include "td/utils/misc.h"
#include "td/utils/port/EventFd.h"
#if !TD_EVENTFD_UNSUPPORTED
#include "td/utils/SpinLock.h"
#include <utility>
namespace td {
// interface like in PollableQueue
template <class T>
class MpscPollableQueue {
public:
using ValueType = T;
int reader_wait_nonblock() {
auto ready = reader_vector_.size() - reader_pos_;
if (ready != 0) {
return narrow_cast<int>(ready);
}
for (int i = 0; i < 2; i++) {
auto guard = lock_.lock();
if (writer_vector_.empty()) {
if (i == 1) {
wait_event_fd_ = true;
return 0;
}
} else {
reader_vector_.clear();
reader_pos_ = 0;
std::swap(writer_vector_, reader_vector_);
return narrow_cast<int>(reader_vector_.size());
}
event_fd_.acquire();
}
UNREACHABLE();
}
ValueType reader_get_unsafe() {
return std::move(reader_vector_[reader_pos_++]);
}
void reader_flush() {
//nop
}
void writer_put(ValueType value) {
auto guard = lock_.lock();
writer_vector_.push_back(std::move(value));
if (wait_event_fd_) {
wait_event_fd_ = false;
guard.reset();
event_fd_.release();
}
}
EventFd &reader_get_event_fd() {
return event_fd_;
}
void writer_flush() {
//nop
}
void init() {
event_fd_.init();
}
void destroy() {
if (!event_fd_.empty()) {
event_fd_.close();
wait_event_fd_ = false;
writer_vector_.clear();
reader_vector_.clear();
reader_pos_ = 0;
}
}
// Just an example of usage
int reader_wait() {
int res;
while ((res = reader_wait_nonblock()) == 0) {
reader_get_event_fd().wait(1000);
}
return res;
}
private:
SpinLock lock_;
bool wait_event_fd_{false};
EventFd event_fd_;
std::vector<ValueType> writer_vector_;
std::vector<ValueType> reader_vector_;
size_t reader_pos_{0};
};
} // namespace td
#else
namespace td {
// dummy implementation which shouldn't be used
template <class T>
class MpscPollableQueue {
public:
using ValueType = T;
void init() {
UNREACHABLE();
}
template <class PutValueType>
void writer_put(PutValueType &&value) {
UNREACHABLE();
}
void writer_flush() {
UNREACHABLE();
}
int reader_wait_nonblock() {
UNREACHABLE();
return 0;
}
ValueType reader_get_unsafe() {
UNREACHABLE();
return ValueType();
}
void reader_flush() {
UNREACHABLE();
}
MpscPollableQueue() = default;
MpscPollableQueue(const MpscPollableQueue &) = delete;
MpscPollableQueue &operator=(const MpscPollableQueue &) = delete;
MpscPollableQueue(MpscPollableQueue &&) = delete;
MpscPollableQueue &operator=(MpscPollableQueue &&) = delete;
~MpscPollableQueue() = default;
};
} // namespace td
#endif

39
tdutils/td/utils/Named.h Normal file
View file

@ -0,0 +1,39 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#include "td/utils/Slice.h"
namespace td {
class Named {
public:
Slice get_name() const {
return name_;
}
void set_name(Slice name) {
name_ = name.str();
}
private:
string name_;
};
} // namespace td

261
tdutils/td/utils/ObjectPool.h Normal file
View file

@ -0,0 +1,261 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#include "td/utils/logging.h"
#include <atomic>
#include <memory>
#include <utility>
namespace td {
// It is draft object pool implementaion
//
// Compared with std::shared_ptr:
// + WeakPtr are much faster. Just pointer copy. No barriers, no atomics.
// - We can't destroy object, because we don't know if it is pointed to by some weak pointer
//
template <class DataT>
class ObjectPool {
struct Storage;
public:
class WeakPtr {
public:
WeakPtr() : generation_(-1), storage_(nullptr) {
}
WeakPtr(int32 generation, Storage *storage) : generation_(generation), storage_(storage) {
}
DataT &operator*() const {
return storage_->data;
}
DataT *operator->() const {
return &**this;
}
// Pattern of usage: 1. Read an object 2. Check if read was valid via is_alive
//
// It is not very usual case of acquire/release use.
// Instead of publishing an object via some flag we do the opposite.
// We publish new generation via destruction of the data.
// In usual case if we see a flag then we are able to use an object.
// In our case if we have used an object and it is already invalid, then generation will mismatch
bool is_alive() const {
if (!storage_) {
return false;
}
std::atomic_thread_fence(std::memory_order_acquire);
return generation_ == storage_->generation.load(std::memory_order_relaxed);
}
// Used for ActorId
bool is_alive_unsafe() const {
if (!storage_) {
return false;
}
return generation_ == storage_->generation.load(std::memory_order_relaxed);
}
bool empty() const {
return storage_ == nullptr;
}
void clear() {
generation_ = -1;
storage_ = nullptr;
}
int32 generation() {
return generation_;
}
private:
int32 generation_;
Storage *storage_;
};
class OwnerPtr {
public:
OwnerPtr() = default;
OwnerPtr(const OwnerPtr &) = delete;
OwnerPtr &operator=(const OwnerPtr &) = delete;
OwnerPtr(OwnerPtr &&other) : storage_(other.storage_), parent_(other.parent_) {
other.storage_ = nullptr;
other.parent_ = nullptr;
}
OwnerPtr &operator=(OwnerPtr &&other) {
if (this != &other) {
storage_ = other.storage_;
parent_ = other.parent_;
other.storage_ = nullptr;
other.parent_ = nullptr;
}
return *this;
}
~OwnerPtr() {
reset();
}
DataT *get() {
return &storage_->data;
}
DataT &operator*() {
return *get();
}
DataT *operator->() {
return get();
}
const DataT *get() const {
return &storage_->data;
}
const DataT &operator*() const {
return *get();
}
const DataT *operator->() const {
return get();
}
WeakPtr get_weak() {
return WeakPtr(storage_->generation.load(std::memory_order_relaxed), storage_);
}
int32 generation() {
return storage_->generation.load(std::memory_order_relaxed);
}
Storage *release() {
auto result = storage_;
storage_ = nullptr;
return result;
}
bool empty() const {
return storage_ == nullptr;
}
void reset() {
if (storage_ != nullptr) {
// for crazy cases when data owns owner pointer to itself.
auto tmp = storage_;
storage_ = nullptr;
parent_->release(OwnerPtr(tmp, parent_));
}
}
private:
friend class ObjectPool;
OwnerPtr(Storage *storage, ObjectPool<DataT> *parent) : storage_(storage), parent_(parent) {
}
Storage *storage_ = nullptr;
ObjectPool<DataT> *parent_ = nullptr;
};
template <class... ArgsT>
OwnerPtr create(ArgsT &&... args) {
Storage *storage = get_storage();
storage->init_data(std::forward<ArgsT>(args)...);
return OwnerPtr(storage, this);
}
OwnerPtr create_empty() {
Storage *storage = get_storage();
return OwnerPtr(storage, this);
}
void set_check_empty(bool flag) {
check_empty_flag_ = flag;
}
void release(OwnerPtr &&owner_ptr) {
Storage *storage = owner_ptr.release();
storage->destroy_data();
release_storage(storage);
}
ObjectPool() = default;
ObjectPool(const ObjectPool &) = delete;
ObjectPool &operator=(const ObjectPool &) = delete;
ObjectPool(ObjectPool &&other) = delete;
ObjectPool &operator=(ObjectPool &&other) = delete;
~ObjectPool() {
while (head_.load()) {
auto to_delete = head_.load();
head_ = to_delete->next;
delete to_delete;
storage_count_--;
}
LOG_CHECK(storage_count_.load() == 0) << storage_count_.load();
}
private:
struct Storage {
// union {
DataT data;
//};
Storage *next = nullptr;
std::atomic<int32> generation{1};
template <class... ArgsT>
void init_data(ArgsT &&... args) {
// new (&data) DataT(std::forward<ArgsT>(args)...);
data = DataT(std::forward<ArgsT>(args)...);
}
void destroy_data() {
generation.fetch_add(1, std::memory_order_relaxed);
std::atomic_thread_fence(std::memory_order_release);
data.clear();
}
};
std::atomic<int32> storage_count_{0};
std::atomic<Storage *> head_{static_cast<Storage *>(nullptr)};
bool check_empty_flag_ = false;
// TODO(perf): allocation Storages in chunks? Anyway we won't be able to release them.
// TODO(perf): memory order
// TODO(perf): use another non lockfree list for release on the same thread
// only one thread, so no aba problem
Storage *get_storage() {
if (head_.load() == nullptr) {
storage_count_++;
return new Storage();
}
Storage *res;
while (true) {
res = head_.load();
auto *next = res->next;
if (head_.compare_exchange_weak(res, next)) {
break;
}
}
return res;
}
// release can be called from other thread
void release_storage(Storage *storage) {
while (true) {
auto *save_head = head_.load();
storage->next = save_head;
if (head_.compare_exchange_weak(save_head, storage)) {
break;
}
}
}
};
} // namespace td

53
tdutils/td/utils/Observer.h Normal file
View file

@ -0,0 +1,53 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
namespace td {
class ObserverBase {
public:
ObserverBase() = default;
ObserverBase(const ObserverBase &) = delete;
ObserverBase &operator=(const ObserverBase &) = delete;
ObserverBase(ObserverBase &&) = delete;
ObserverBase &operator=(ObserverBase &&) = delete;
virtual ~ObserverBase() = default;
virtual void notify() = 0;
};
class Observer : ObserverBase {
public:
Observer() = default;
explicit Observer(unique_ptr<ObserverBase> &&ptr) : observer_ptr_(std::move(ptr)) {
}
void notify() override {
if (observer_ptr_) {
observer_ptr_->notify();
}
}
private:
unique_ptr<ObserverBase> observer_ptr_;
};
} // namespace td

142
tdutils/td/utils/OptionsParser.cpp Normal file
View file

@ -0,0 +1,142 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/utils/OptionsParser.h"
#if TD_HAVE_GETOPT
#include "getopt.h"
#endif
#if !TD_WINDOWS
#include <getopt.h>
#include <unistd.h>
#endif
namespace td {
void OptionsParser::set_description(std::string description) {
description_ = std::move(description);
}
void OptionsParser::add_option(Option::Type type, char short_key, Slice long_key, Slice description,
std::function<Status(Slice)> callback) {
options_.push_back(Option{type, short_key, long_key.str(), description.str(), std::move(callback)});
}
void OptionsParser::add_option(char short_key, Slice long_key, Slice description,
std::function<Status(Slice)> callback) {
add_option(Option::Type::Arg, short_key, long_key, description, std::move(callback));
}
void OptionsParser::add_option(char short_key, Slice long_key, Slice description,
std::function<Status(void)> callback) {
// Ouch. There must be some better way
add_option(Option::Type::NoArg, short_key, long_key, description,
std::bind([](std::function<Status(void)> &func, Slice) { return func(); }, std::move(callback),
std::placeholders::_1));
}
Result<int> OptionsParser::run(int argc, char *argv[]) {
#if TD_HAVE_GETOPT
char buff[1024];
StringBuilder sb(MutableSlice{buff, sizeof(buff)});
for (auto &opt : options_) {
CHECK(opt.type != Option::OptionalArg);
sb << opt.short_key;
if (opt.type == Option::Arg) {
sb << ":";
}
}
if (sb.is_error()) {
return Status::Error("Can't parse options");
}
CSlice short_options = sb.as_cslice();
vector<option> long_options;
for (auto &opt : options_) {
if (opt.long_key.empty()) {
continue;
}
option o;
o.flag = nullptr;
o.val = opt.short_key;
o.has_arg = opt.type == Option::Arg ? required_argument : no_argument;
o.name = opt.long_key.c_str();
long_options.push_back(o);
}
long_options.push_back({nullptr, 0, nullptr, 0});
while (true) {
int opt_i = getopt_long(argc, argv, short_options.c_str(), &long_options[0], nullptr);
if (opt_i == ':') {
return Status::Error("Missing argument");
}
if (opt_i == '?') {
return Status::Error("Unrecognized option");
}
if (opt_i == -1) {
break;
}
bool found = false;
for (auto &opt : options_) {
if (opt.short_key == opt_i) {
Slice arg;
if (opt.type == Option::Arg) {
arg = Slice(optarg);
}
auto status = opt.arg_callback(arg);
if (status.is_error()) {
return std::move(status);
}
found = true;
break;
}
}
if (!found) {
return Status::Error("Unknown argument");
}
}
return optind;
#else
return -1;
#endif
}
StringBuilder &operator<<(StringBuilder &sb, const OptionsParser &o) {
sb << o.description_ << "\n";
for (auto &opt : o.options_) {
sb << "-" << opt.short_key;
if (!opt.long_key.empty()) {
sb << "|--" << opt.long_key;
}
if (opt.type == OptionsParser::Option::OptionalArg) {
sb << "[";
}
if (opt.type != OptionsParser::Option::NoArg) {
sb << "<arg>";
}
if (opt.type == OptionsParser::Option::OptionalArg) {
sb << "]";
}
sb << "\t" << opt.description;
sb << "\n";
}
return sb;
}
} // namespace td

60
tdutils/td/utils/OptionsParser.h Normal file
View file

@ -0,0 +1,60 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#include "td/utils/Slice.h"
#include "td/utils/Status.h"
#include "td/utils/StringBuilder.h"
#include <functional>
namespace td {
class OptionsParser {
class Option {
public:
enum Type { NoArg, Arg, OptionalArg };
Type type;
char short_key;
std::string long_key;
std::string description;
std::function<Status(Slice)> arg_callback;
};
public:
void set_description(std::string description);
void add_option(Option::Type type, char short_key, Slice long_key, Slice description,
std::function<Status(Slice)> callback);
void add_option(char short_key, Slice long_key, Slice description, std::function<Status(Slice)> callback);
void add_option(char short_key, Slice long_key, Slice description, std::function<Status(void)> callback);
Result<int> run(int argc, char *argv[]) TD_WARN_UNUSED_RESULT;
friend StringBuilder &operator<<(StringBuilder &sb, const OptionsParser &o);
private:
std::vector<Option> options_;
std::string description_;
};
} // namespace td

111
tdutils/td/utils/OrderedEventsProcessor.h Normal file
View file

@ -0,0 +1,111 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#include "td/utils/logging.h"
#include <utility>
namespace td {
// Process states in order defined by their Id
template <class DataT>
class OrderedEventsProcessor {
public:
using SeqNo = uint64;
OrderedEventsProcessor() = default;
explicit OrderedEventsProcessor(SeqNo offset) : offset_(offset), begin_(offset_), end_(offset_) {
}
template <class FunctionT>
void clear(FunctionT &&function) {
for (auto &it : data_array_) {
if (it.second) {
function(std::move(it.first));
}
}
*this = OrderedEventsProcessor();
}
void clear() {
*this = OrderedEventsProcessor();
}
template <class FromDataT, class FunctionT>
void add(SeqNo seq_no, FromDataT &&data, FunctionT &&function) {
LOG_CHECK(seq_no >= begin_) << seq_no << ">=" << begin_; // or ignore?
if (seq_no == begin_) { // run now
begin_++;
function(seq_no, std::forward<FromDataT>(data));
while (begin_ < end_) {
auto &data_flag = data_array_[static_cast<size_t>(begin_ - offset_)];
if (!data_flag.second) {
break;
}
function(begin_, std::move(data_flag.first));
data_flag.second = false;
begin_++;
}
if (begin_ > end_) {
end_ = begin_;
}
if (begin_ == end_) {
offset_ = begin_;
}
// try_compactify
auto begin_pos = static_cast<size_t>(begin_ - offset_);
if (begin_pos > 5 && begin_pos * 2 > data_array_.size()) {
data_array_.erase(data_array_.begin(), data_array_.begin() + begin_pos);
offset_ = begin_;
}
} else {
auto pos = static_cast<size_t>(seq_no - offset_);
auto need_size = pos + 1;
if (data_array_.size() < need_size) {
data_array_.resize(need_size);
}
data_array_[pos].first = std::forward<FromDataT>(data);
data_array_[pos].second = true;
if (end_ < seq_no + 1) {
end_ = seq_no + 1;
}
}
}
bool has_events() const {
return begin_ != end_;
}
SeqNo max_unfinished_seq_no() {
return end_ - 1;
}
SeqNo max_finished_seq_no() {
return begin_ - 1;
}
private:
SeqNo offset_ = 1;
SeqNo begin_ = 1;
SeqNo end_ = 1;
std::vector<std::pair<DataT, bool>> data_array_;
};
} // namespace td

209
tdutils/td/utils/Parser.h Normal file
View file

@ -0,0 +1,209 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#include "td/utils/format.h"
#include "td/utils/logging.h"
#include "td/utils/Slice.h"
#include "td/utils/Status.h"
#include <cstring>
#include <utility>
namespace td {
namespace detail {
template <class SliceT>
class ParserImpl {
public:
explicit ParserImpl(SliceT data) : ptr_(data.begin()), end_(data.end()), status_() {
}
ParserImpl(ParserImpl &&other) : ptr_(other.ptr_), end_(other.end_), status_(std::move(other.status_)) {
other.clear();
}
ParserImpl &operator=(ParserImpl &&other) {
if (&other == this) {
return *this;
}
ptr_ = other.ptr_;
end_ = other.end_;
status_ = std::move(other.status_);
other.clear();
return *this;
}
ParserImpl(const ParserImpl &) = delete;
ParserImpl &operator=(const ParserImpl &) = delete;
~ParserImpl() = default;
bool empty() const {
return ptr_ == end_;
}
void clear() {
ptr_ = nullptr;
end_ = ptr_;
status_ = Status::OK();
}
SliceT read_till_nofail(char c) {
if (status_.is_error()) {
return SliceT();
}
auto till = static_cast<decltype(ptr_)>(std::memchr(ptr_, c, end_ - ptr_));
if (till == nullptr) {
till = end_;
}
SliceT result(ptr_, till);
ptr_ = till;
return result;
}
SliceT read_till_nofail(Slice str) {
if (status_.is_error()) {
return SliceT();
}
auto best_till = end_;
for (auto c : str) {
auto till = static_cast<decltype(ptr_)>(std::memchr(ptr_, c, end_ - ptr_));
if (till != nullptr && till < best_till) {
best_till = till;
}
}
SliceT result(ptr_, best_till);
ptr_ = best_till;
return result;
}
template <class F>
SliceT read_while(const F &f) {
auto save_ptr = ptr_;
while (ptr_ != end_ && f(*ptr_)) {
ptr_++;
}
return SliceT(save_ptr, ptr_);
}
SliceT read_all() {
auto save_ptr = ptr_;
ptr_ = end_;
return SliceT(save_ptr, ptr_);
}
SliceT read_till(char c) {
if (status_.is_error()) {
return SliceT();
}
SliceT res = read_till_nofail(c);
if (ptr_ == end_ || ptr_[0] != c) {
status_ = Status::Error(PSLICE() << "Read till " << tag("char", c) << " failed");
return SliceT();
}
return res;
}
char peek_char() {
if (ptr_ == end_) {
return 0;
}
return *ptr_;
}
char *ptr() {
return ptr_;
}
void skip_nofail(char c) {
if (ptr_ != end_ && ptr_[0] == c) {
ptr_++;
}
}
void skip(char c) {
if (status_.is_error()) {
return;
}
if (ptr_ == end_ || ptr_[0] != c) {
status_ = Status::Error(PSLICE() << "Skip " << tag("char", c) << " failed");
return;
}
ptr_++;
}
bool try_skip(char c) {
if (ptr_ != end_ && ptr_[0] == c) {
ptr_++;
return true;
}
return false;
}
void skip_till_not(Slice str) {
while (ptr_ != end_) {
if (std::memchr(str.data(), *ptr_, str.size()) == nullptr) {
break;
}
ptr_++;
}
}
void skip_whitespaces() {
skip_till_not(" \t\r\n");
}
SliceT read_word() {
skip_whitespaces();
return read_till_nofail(" \t\r\n");
}
SliceT data() const {
return SliceT(ptr_, end_);
}
Status &status() {
return status_;
}
bool start_with(Slice prefix) const {
if (prefix.size() > static_cast<size_t>(end_ - ptr_)) {
return false;
}
return prefix == Slice(ptr_, prefix.size());
}
bool skip_start_with(Slice prefix) {
if (start_with(prefix)) {
advance(prefix.size());
return true;
}
return false;
}
void advance(size_t diff) {
ptr_ += diff;
CHECK(ptr_ <= end_);
}
private:
decltype(std::declval<SliceT>().begin()) ptr_;
decltype(std::declval<SliceT>().end()) end_;
Status status_;
};
} // namespace detail
using Parser = detail::ParserImpl<MutableSlice>;
using ConstParser = detail::ParserImpl<Slice>;
} // namespace td

128
tdutils/td/utils/PathView.h Normal file
View file

@ -0,0 +1,128 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/misc.h"
#include "td/utils/Slice.h"
namespace td {
class PathView {
public:
explicit PathView(Slice path) : path_(path) {
last_slash_ = narrow_cast<int32>(path_.size()) - 1;
while (last_slash_ >= 0 && !is_slash(path_[last_slash_])) {
last_slash_--;
}
last_dot_ = static_cast<int32>(path_.size());
for (auto i = last_dot_ - 1; i > last_slash_ + 1; i--) {
if (path_[i] == '.') {
last_dot_ = i;
break;
}
}
}
bool empty() const {
return path_.empty();
}
bool is_dir() const {
if (empty()) {
return false;
}
return is_slash(path_.back());
}
Slice parent_dir() const {
return path_.substr(0, last_slash_ + 1);
}
Slice extension() const {
if (last_dot_ == static_cast<int32>(path_.size())) {
return Slice();
}
return path_.substr(last_dot_ + 1);
}
Slice without_extension() const {
return path_.substr(0, last_dot_);
}
Slice file_stem() const {
return path_.substr(last_slash_ + 1, last_dot_ - last_slash_ - 1);
}
Slice file_name() const {
return path_.substr(last_slash_ + 1);
}
Slice path() const {
return path_;
}
bool is_absolute() const {
return !empty() && (is_slash(path_[0]) || (path_.size() >= 3 && path_[1] == ':' && is_slash(path_[2])));
}
bool is_relative() const {
return !is_absolute();
}
static Slice relative(Slice path, Slice dir, bool force = false) {
if (begins_with(path, dir)) {
path.remove_prefix(dir.size());
return path;
}
if (force) {
return Slice();
}
return path;
}
static Slice dir_and_file(Slice path) {
auto last_slash = static_cast<int32>(path.size()) - 1;
while (last_slash >= 0 && !is_slash(path[last_slash])) {
last_slash--;
}
if (last_slash < 0) {
return Slice();
}
last_slash--;
while (last_slash >= 0 && !is_slash(path[last_slash])) {
last_slash--;
}
if (last_slash < 0) {
return Slice();
}
return path.substr(last_slash + 1);
}
private:
static bool is_slash(char c) {
return c == '/' || c == '\\';
}
Slice path_;
int32 last_slash_;
int32 last_dot_;
};
} // namespace td

187
tdutils/td/utils/Random.cpp Normal file
View file

@ -0,0 +1,187 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/utils/Random.h"
#include "td/utils/logging.h"
#include "td/utils/port/thread_local.h"
#if TD_HAVE_OPENSSL
#include <openssl/rand.h>
#endif
#include <atomic>
#include <cstring>
#include <limits>
#include <random>
namespace td {
#if TD_HAVE_OPENSSL
namespace {
std::atomic<int64> random_seed_generation{0};
} // namespace
void Random::secure_bytes(MutableSlice dest) {
Random::secure_bytes(dest.ubegin(), dest.size());
}
void Random::secure_bytes(unsigned char *ptr, size_t size) {
constexpr size_t buf_size = 512;
static TD_THREAD_LOCAL unsigned char *buf; // static zero-initialized
static TD_THREAD_LOCAL size_t buf_pos;
static TD_THREAD_LOCAL int64 generation;
if (init_thread_local<unsigned char[]>(buf, buf_size)) {
buf_pos = buf_size;
generation = 0;
}
if (generation != random_seed_generation.load(std::memory_order_relaxed)) {
generation = random_seed_generation.load(std::memory_order_acquire);
buf_pos = buf_size;
}
auto ready = min(size, buf_size - buf_pos);
if (ready != 0) {
std::memcpy(ptr, buf + buf_pos, ready);
buf_pos += ready;
ptr += ready;
size -= ready;
if (size == 0) {
return;
}
}
if (size < buf_size) {
int err = RAND_bytes(buf, static_cast<int>(buf_size));
// TODO: it CAN fail
LOG_IF(FATAL, err != 1);
buf_pos = size;
std::memcpy(ptr, buf, size);
return;
}
CHECK(size <= static_cast<size_t>(std::numeric_limits<int>::max()));
int err = RAND_bytes(ptr, static_cast<int>(size));
// TODO: it CAN fail
LOG_IF(FATAL, err != 1);
}
int32 Random::secure_int32() {
int32 res = 0;
secure_bytes(reinterpret_cast<unsigned char *>(&res), sizeof(int32));
return res;
}
int64 Random::secure_int64() {
int64 res = 0;
secure_bytes(reinterpret_cast<unsigned char *>(&res), sizeof(int64));
return res;
}
uint32 Random::secure_uint32() {
uint32 res = 0;
secure_bytes(reinterpret_cast<unsigned char *>(&res), sizeof(uint32));
return res;
}
uint64 Random::secure_uint64() {
uint64 res = 0;
secure_bytes(reinterpret_cast<unsigned char *>(&res), sizeof(uint64));
return res;
}
void Random::add_seed(Slice bytes, double entropy) {
RAND_add(bytes.data(), static_cast<int>(bytes.size()), entropy);
random_seed_generation++;
}
#endif
static unsigned int rand_device_helper() {
static TD_THREAD_LOCAL std::random_device *rd;
init_thread_local<std::random_device>(rd);
return (*rd)();
}
uint32 Random::fast_uint32() {
static TD_THREAD_LOCAL std::mt19937 *gen;
if (!gen) {
auto &rg = rand_device_helper;
std::seed_seq seq{rg(), rg(), rg(), rg(), rg(), rg(), rg(), rg(), rg(), rg(), rg(), rg()};
init_thread_local<std::mt19937>(gen, seq);
}
return static_cast<uint32>((*gen)());
}
uint64 Random::fast_uint64() {
static TD_THREAD_LOCAL std::mt19937_64 *gen;
if (!gen) {
auto &rg = rand_device_helper;
std::seed_seq seq{rg(), rg(), rg(), rg(), rg(), rg(), rg(), rg(), rg(), rg(), rg(), rg()};
init_thread_local<std::mt19937_64>(gen, seq);
}
return static_cast<uint64>((*gen)());
}
int Random::fast(int min, int max) {
if (min == std::numeric_limits<int>::min() && max == std::numeric_limits<int>::max()) {
// to prevent integer overflow and division by zero
min++;
}
DCHECK(min <= max);
return static_cast<int>(min + fast_uint32() % (max - min + 1)); // TODO signed_cast
}
double Random::fast(double min, double max) {
DCHECK(min <= max);
return min +
fast_uint32() * 1.0 /
(static_cast<double>(std::numeric_limits<td::uint32>::max()) - std::numeric_limits<td::uint32>::min()) *
(max - min);
}
Random::Xorshift128plus::Xorshift128plus(uint64 seed) {
auto next = [&]() {
// splitmix64
seed += static_cast<uint64>(0x9E3779B97F4A7C15);
uint64 z = seed;
z = (z ^ (z >> 30)) * static_cast<uint64>(0xBF58476D1CE4E5B9);
z = (z ^ (z >> 27)) * static_cast<uint64>(0x94D049BB133111EB);
return z ^ (z >> 31);
};
seed_[0] = next();
seed_[1] = next();
}
Random::Xorshift128plus::Xorshift128plus(uint64 seed_a, uint64 seed_b) {
seed_[0] = seed_a;
seed_[1] = seed_b;
}
uint64 Random::Xorshift128plus::operator()() {
uint64 x = seed_[0];
const uint64 y = seed_[1];
seed_[0] = y;
x ^= x << 23;
seed_[1] = x ^ y ^ (x >> 17) ^ (y >> 26);
return seed_[1] + y;
}
int Random::Xorshift128plus::fast(int min, int max) {
return static_cast<int>((*this)() % (max - min + 1) + min);
}
} // namespace td

59
tdutils/td/utils/Random.h Normal file
View file

@ -0,0 +1,59 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#include "td/utils/Slice.h"
namespace td {
class Random {
public:
#if TD_HAVE_OPENSSL
static void secure_bytes(MutableSlice dest);
static void secure_bytes(unsigned char *ptr, size_t size);
static int32 secure_int32();
static int64 secure_int64();
static uint32 secure_uint32();
static uint64 secure_uint64();
// works only for current thread
static void add_seed(Slice bytes, double entropy = 0);
#endif
static uint32 fast_uint32();
static uint64 fast_uint64();
// distribution is not uniform, min and max are included
static int fast(int min, int max);
static double fast(double min, double max);
class Xorshift128plus {
public:
explicit Xorshift128plus(uint64 seed);
Xorshift128plus(uint64 seed_a, uint64 seed_b);
uint64 operator()();
int fast(int min, int max);
private:
uint64 seed_[2];
};
};
} // namespace td

89
tdutils/td/utils/ScopeGuard.h Normal file
View file

@ -0,0 +1,89 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#include <cstdlib>
#include <memory>
#include <type_traits>
#include <utility>
namespace td {
class Guard {
public:
Guard() = default;
Guard(const Guard &other) = delete;
Guard &operator=(const Guard &other) = delete;
Guard(Guard &&other) = default;
Guard &operator=(Guard &&other) = default;
virtual ~Guard() = default;
virtual void dismiss() {
std::abort();
}
};
template <class FunctionT>
class LambdaGuard : public Guard {
public:
explicit LambdaGuard(const FunctionT &func) : func_(func) {
}
explicit LambdaGuard(FunctionT &&func) : func_(std::move(func)) {
}
LambdaGuard(const LambdaGuard &other) = delete;
LambdaGuard &operator=(const LambdaGuard &other) = delete;
LambdaGuard(LambdaGuard &&other) : func_(std::move(other.func_)), dismissed_(other.dismissed_) {
other.dismissed_ = true;
}
LambdaGuard &operator=(LambdaGuard &&other) = delete;
void dismiss() {
dismissed_ = true;
}
~LambdaGuard() {
if (!dismissed_) {
func_();
}
}
private:
FunctionT func_;
bool dismissed_ = false;
};
template <class F>
unique_ptr<Guard> create_lambda_guard(F &&f) {
return make_unique<LambdaGuard<F>>(std::forward<F>(f));
}
template <class F>
std::shared_ptr<Guard> create_shared_lambda_guard(F &&f) {
return std::make_shared<LambdaGuard<F>>(std::forward<F>(f));
}
enum class ScopeExit {};
template <class FunctionT>
auto operator+(ScopeExit, FunctionT &&func) {
return LambdaGuard<std::decay_t<FunctionT>>(std::forward<FunctionT>(func));
}
} // namespace td
#define SCOPE_EXIT auto TD_CONCAT(SCOPE_EXIT_VAR_, __LINE__) = ::td::ScopeExit() + [&]()

294
tdutils/td/utils/SharedObjectPool.h Normal file
View file

@ -0,0 +1,294 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#include "td/utils/logging.h"
#include "td/utils/MpscLinkQueue.h"
#include <atomic>
#include <memory>
#include <new>
#include <utility>
namespace td {
namespace detail {
class AtomicRefCnt {
public:
explicit AtomicRefCnt(uint64 cnt) : cnt_(cnt) {
}
void inc() {
cnt_.fetch_add(1, std::memory_order_relaxed);
}
bool dec() {
return cnt_.fetch_sub(1, std::memory_order_acq_rel) == 1;
}
uint64 value() const {
return cnt_.load(std::memory_order_relaxed);
}
private:
std::atomic<uint64> cnt_{0};
};
template <class DataT, class DeleterT>
class SharedPtrRaw
: public DeleterT
, private MpscLinkQueueImpl::Node {
public:
explicit SharedPtrRaw(DeleterT deleter) : DeleterT(std::move(deleter)), ref_cnt_{0}, option_magic_(Magic) {
}
~SharedPtrRaw() {
CHECK(use_cnt() == 0);
CHECK(option_magic_ == Magic);
}
template <class... ArgsT>
void init_data(ArgsT &&... args) {
new (&option_data_) DataT(std::forward<ArgsT>(args)...);
}
void destroy_data() {
option_data_.~DataT();
option_magic_ = Magic;
}
uint64 use_cnt() const {
return ref_cnt_.value();
}
void inc() {
ref_cnt_.inc();
}
bool dec() {
return ref_cnt_.dec();
}
DataT &data() {
return option_data_;
}
static SharedPtrRaw *from_mpsc_link_queue_node(MpscLinkQueueImpl::Node *node) {
return static_cast<SharedPtrRaw<DataT, DeleterT> *>(node);
}
MpscLinkQueueImpl::Node *to_mpsc_link_queue_node() {
return static_cast<MpscLinkQueueImpl::Node *>(this);
}
private:
AtomicRefCnt ref_cnt_;
enum { Magic = 0x732817a2 };
union {
DataT option_data_;
uint32 option_magic_;
};
};
template <class T, class DeleterT = std::default_delete<T>>
class SharedPtr {
public:
using Raw = detail::SharedPtrRaw<T, DeleterT>;
SharedPtr() = default;
~SharedPtr() {
if (!raw_) {
return;
}
reset();
}
explicit SharedPtr(Raw *raw) : raw_(raw) {
if (raw_) {
raw_->inc();
}
}
SharedPtr(const SharedPtr &other) : SharedPtr(other.raw_) {
}
SharedPtr &operator=(const SharedPtr &other) {
if (other.raw_) {
other.raw_->inc();
}
reset(other.raw_);
return *this;
}
SharedPtr(SharedPtr &&other) : raw_(other.raw_) {
other.raw_ = nullptr;
}
SharedPtr &operator=(SharedPtr &&other) {
reset(other.raw_);
other.raw_ = nullptr;
return *this;
}
bool empty() const {
return raw_ == nullptr;
}
explicit operator bool() const {
return !empty();
}
uint64 use_cnt() const {
if (!raw_) {
return 0;
}
return raw_->use_cnt();
}
T &operator*() const {
return raw_->data();
}
T *operator->() const {
return &raw_->data();
}
Raw *release() {
auto res = raw_;
raw_ = nullptr;
return res;
}
void reset(Raw *new_raw = nullptr) {
if (raw_ && raw_->dec()) {
raw_->destroy_data();
auto deleter = std::move(static_cast<DeleterT &>(*raw_));
deleter(raw_);
}
raw_ = new_raw;
}
template <class... ArgsT>
static SharedPtr<T, DeleterT> create(ArgsT &&... args) {
auto raw = make_unique<Raw>(DeleterT());
raw->init_data(std::forward<ArgsT>(args)...);
return SharedPtr<T, DeleterT>(raw.release());
}
template <class D, class... ArgsT>
static SharedPtr<T, DeleterT> create_with_deleter(D &&d, ArgsT &&... args) {
auto raw = make_unique<Raw>(std::forward<D>(d));
raw->init_data(std::forward<ArgsT>(args)...);
return SharedPtr<T, DeleterT>(raw.release());
}
bool operator==(const SharedPtr<T, DeleterT> &other) const {
return raw_ == other.raw_;
}
private:
Raw *raw_{nullptr};
};
} // namespace detail
template <class DataT>
class SharedObjectPool {
class Deleter;
public:
using Ptr = detail::SharedPtr<DataT, Deleter>;
SharedObjectPool() = default;
SharedObjectPool(const SharedObjectPool &other) = delete;
SharedObjectPool &operator=(const SharedObjectPool &other) = delete;
SharedObjectPool(SharedObjectPool &&other) = delete;
SharedObjectPool &operator=(SharedObjectPool &&other) = delete;
~SharedObjectPool() {
free_queue_.pop_all(free_queue_reader_);
size_t free_cnt = 0;
while (free_queue_reader_.read()) {
free_cnt++;
}
LOG_CHECK(free_cnt == allocated_.size()) << free_cnt << " " << allocated_.size();
}
template <class... ArgsT>
Ptr alloc(ArgsT &&... args) {
auto *raw = alloc_raw();
raw->init_data(std::forward<ArgsT>(args)...);
return Ptr(raw);
}
size_t total_size() const {
return allocated_.size();
}
uint64 calc_free_size() {
free_queue_.pop_all(free_queue_reader_);
return free_queue_reader_.calc_size();
}
//non thread safe
template <class F>
void for_each(F &&f) {
for (auto &raw : allocated_) {
if (raw->use_cnt() > 0) {
f(raw->data());
}
}
}
private:
using Raw = typename Ptr::Raw;
Raw *alloc_raw() {
free_queue_.pop_all(free_queue_reader_);
auto *raw = free_queue_reader_.read().get();
if (raw) {
return raw;
}
allocated_.push_back(make_unique<Raw>(deleter()));
return allocated_.back().get();
}
void free_raw(Raw *raw) {
free_queue_.push(Node{raw});
}
class Node {
public:
Node() = default;
explicit Node(Raw *raw) : raw_(raw) {
}
MpscLinkQueueImpl::Node *to_mpsc_link_queue_node() {
return raw_->to_mpsc_link_queue_node();
}
static Node from_mpsc_link_queue_node(MpscLinkQueueImpl::Node *node) {
return Node{Raw::from_mpsc_link_queue_node(node)};
}
Raw *get() const {
return raw_;
}
explicit operator bool() const {
return raw_ != nullptr;
}
private:
Raw *raw_{nullptr};
};
class Deleter {
public:
explicit Deleter(SharedObjectPool<DataT> *pool) : pool_(pool) {
}
void operator()(Raw *raw) {
pool_->free_raw(raw);
};
private:
SharedObjectPool<DataT> *pool_;
};
friend class Deleter;
Deleter deleter() {
return Deleter(this);
}
std::vector<unique_ptr<Raw>> allocated_;
MpscLinkQueue<Node> free_queue_;
typename MpscLinkQueue<Node>::Reader free_queue_reader_;
};
} // namespace td

29
tdutils/td/utils/SharedSlice.cpp Normal file
View file

@ -0,0 +1,29 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/utils/SharedSlice.h"
#include "td/utils/buffer.h"
namespace td {
BufferSlice SharedSlice::clone_as_buffer_slice() const {
return BufferSlice{as_slice().str()};
}
} // namespace td

392
tdutils/td/utils/SharedSlice.h Normal file
View file

@ -0,0 +1,392 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/Slice.h"
#include <atomic>
#include <memory>
namespace td {
namespace detail {
struct SharedSliceHeader {
explicit SharedSliceHeader(size_t size) : refcnt_{1}, size_{size} {
}
void inc() {
refcnt_.fetch_add(1, std::memory_order_relaxed);
}
bool dec() {
return refcnt_.fetch_sub(1, std::memory_order_acq_rel) == 1;
}
bool is_unique() const {
// NB: race if std::memory_order_relaxed is used
// reader may see a change by a new writer
return refcnt_.load(std::memory_order_acquire) == 1;
}
size_t size() const {
return size_;
}
private:
std::atomic<uint64> refcnt_;
size_t size_;
};
struct UniqueSliceHeader {
explicit UniqueSliceHeader(size_t size) : size_{size} {
}
void inc() {
}
bool dec() {
return true;
}
bool is_unique() const {
return true;
}
size_t size() const {
return size_;
}
private:
size_t size_;
};
template <class HeaderT, bool zero_on_destruct = false>
class UnsafeSharedSlice {
public:
UnsafeSharedSlice() = default;
UnsafeSharedSlice clone() const {
if (is_null()) {
return UnsafeSharedSlice();
}
header()->inc();
return UnsafeSharedSlice(ptr_.get());
}
bool is_null() const {
return !ptr_;
}
bool is_unique() const {
if (is_null()) {
return true;
}
return header()->is_unique();
}
MutableSlice as_mutable_slice() {
if (is_null()) {
return MutableSlice();
}
return MutableSlice(ptr_.get() + sizeof(HeaderT), header()->size());
}
Slice as_slice() const {
if (is_null()) {
return Slice();
}
return Slice(ptr_.get() + sizeof(HeaderT), header()->size());
}
size_t size() const {
if (is_null()) {
return 0;
}
return header()->size();
}
static UnsafeSharedSlice create(size_t size) {
static_assert(std::is_standard_layout<HeaderT>::value, "HeaderT must have statdard layout");
auto ptr = std::make_unique<char[]>(size + sizeof(HeaderT));
auto header_ptr = new (ptr.get()) HeaderT(size);
CHECK(reinterpret_cast<char *>(header_ptr) == ptr.get());
return UnsafeSharedSlice(std::move(ptr));
}
static UnsafeSharedSlice create(Slice slice) {
auto res = create(slice.size());
res.as_mutable_slice().copy_from(slice);
return res;
}
void clear() {
ptr_.reset();
}
private:
explicit UnsafeSharedSlice(char *ptr) : ptr_(ptr) {
}
explicit UnsafeSharedSlice(std::unique_ptr<char[]> from) : ptr_(from.release()) {
}
HeaderT *header() const {
return reinterpret_cast<HeaderT *>(ptr_.get());
}
struct Destructor {
void operator()(char *ptr) {
auto header = reinterpret_cast<HeaderT *>(ptr);
if (header->dec()) {
if (zero_on_destruct) {
MutableSlice(ptr, sizeof(HeaderT) + header->size()).fill_zero_secure();
}
std::default_delete<char[]>()(ptr);
}
}
};
std::unique_ptr<char[], Destructor> ptr_;
};
} // namespace detail
class BufferSlice;
class UniqueSharedSlice;
class SharedSlice {
using Impl = detail::UnsafeSharedSlice<detail::SharedSliceHeader>;
public:
SharedSlice() = default;
explicit SharedSlice(Slice slice) : impl_(Impl::create(slice.size())) {
impl_.as_mutable_slice().copy_from(slice);
}
explicit SharedSlice(UniqueSharedSlice from);
SharedSlice(const char *ptr, size_t size) : SharedSlice(Slice(ptr, size)) {
}
SharedSlice clone() const {
return SharedSlice(impl_.clone());
}
Slice as_slice() const {
return impl_.as_slice();
}
BufferSlice clone_as_buffer_slice() const;
operator Slice() const {
return as_slice();
}
// like in std::string
const char *data() const {
return as_slice().data();
}
char operator[](size_t at) const {
return as_slice()[at];
}
bool empty() const {
return size() == 0;
}
size_t size() const {
return impl_.size();
}
// like in std::string
size_t length() const {
return size();
}
void clear() {
impl_.clear();
}
private:
friend class UniqueSharedSlice;
explicit SharedSlice(Impl impl) : impl_(std::move(impl)) {
}
Impl impl_;
};
class UniqueSharedSlice {
using Impl = detail::UnsafeSharedSlice<detail::SharedSliceHeader>;
public:
UniqueSharedSlice() = default;
explicit UniqueSharedSlice(size_t size) : impl_(Impl::create(size)) {
}
explicit UniqueSharedSlice(Slice slice) : impl_(Impl::create(slice)) {
}
UniqueSharedSlice(const char *ptr, size_t size) : UniqueSharedSlice(Slice(ptr, size)) {
}
explicit UniqueSharedSlice(SharedSlice from) : impl_() {
if (from.impl_.is_unique()) {
impl_ = std::move(from.impl_);
} else {
impl_ = Impl::create(from.as_slice());
}
}
UniqueSharedSlice copy() const {
return UniqueSharedSlice(as_slice());
}
Slice as_slice() const {
return impl_.as_slice();
}
MutableSlice as_mutable_slice() {
return impl_.as_mutable_slice();
}
operator Slice() const {
return as_slice();
}
// like in std::string
char *data() {
return as_mutable_slice().data();
}
const char *data() const {
return as_slice().data();
}
char operator[](size_t at) const {
return as_slice()[at];
}
bool empty() const {
return size() == 0;
}
size_t size() const {
return impl_.size();
}
// like in std::string
size_t length() const {
return size();
}
void clear() {
impl_.clear();
}
private:
friend class SharedSlice;
explicit UniqueSharedSlice(Impl impl) : impl_(std::move(impl)) {
}
Impl impl_;
};
inline SharedSlice::SharedSlice(UniqueSharedSlice from) : impl_(std::move(from.impl_)) {
}
template <bool zero_on_destruct>
class UniqueSliceImpl {
using Impl = detail::UnsafeSharedSlice<detail::UniqueSliceHeader, zero_on_destruct>;
public:
UniqueSliceImpl() = default;
explicit UniqueSliceImpl(size_t size) : impl_(Impl::create(size)) {
}
UniqueSliceImpl(size_t size, char c) : impl_(Impl::create(size)) {
std::memset(as_mutable_slice().data(), c, size);
}
explicit UniqueSliceImpl(Slice slice) : impl_(Impl::create(slice)) {
}
UniqueSliceImpl(const char *ptr, size_t size) : UniqueSliceImpl(Slice(ptr, size)) {
}
UniqueSliceImpl copy() const {
return UniqueSliceImpl(as_slice());
}
Slice as_slice() const {
return impl_.as_slice();
}
MutableSlice as_mutable_slice() {
return impl_.as_mutable_slice();
}
operator Slice() const {
return as_slice();
}
// like in std::string
char *data() {
return as_mutable_slice().data();
}
const char *data() const {
return as_slice().data();
}
char operator[](size_t at) const {
return as_slice()[at];
}
bool empty() const {
return size() == 0;
}
size_t size() const {
return impl_.size();
}
// like in std::string
size_t length() const {
return size();
}
void clear() {
impl_.clear();
}
private:
explicit UniqueSliceImpl(Impl impl) : impl_(std::move(impl)) {
}
Impl impl_;
};
using UniqueSlice = UniqueSliceImpl<false>;
using SecureString = UniqueSliceImpl<true>;
inline MutableSlice as_mutable_slice(UniqueSharedSlice &unique_shared_slice) {
return unique_shared_slice.as_mutable_slice();
}
inline MutableSlice as_mutable_slice(UniqueSlice &unique_slice) {
return unique_slice.as_mutable_slice();
}
inline MutableSlice as_mutable_slice(SecureString &secure_string) {
return secure_string.as_mutable_slice();
}
} // namespace td

224
tdutils/td/utils/Slice-decl.h Normal file
View file

@ -0,0 +1,224 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#include <type_traits>
namespace td {
class Slice;
class MutableSlice {
char *s_;
size_t len_;
struct private_tag {};
public:
MutableSlice();
MutableSlice(char *s, size_t len);
MutableSlice(unsigned char *s, size_t len);
MutableSlice(string &s);
template <class T>
explicit MutableSlice(T s, std::enable_if_t<std::is_same<char *, T>::value, private_tag> = {});
MutableSlice(char *s, char *t);
MutableSlice(unsigned char *s, unsigned char *t);
template <size_t N>
constexpr MutableSlice(char (&a)[N]) = delete;
bool empty() const;
size_t size() const;
MutableSlice &remove_prefix(size_t prefix_len);
MutableSlice &remove_suffix(size_t suffix_len);
MutableSlice &truncate(size_t size);
MutableSlice copy() const;
char *data() const;
char *begin() const;
unsigned char *ubegin() const;
char *end() const;
unsigned char *uend() const;
string str() const;
MutableSlice substr(size_t from) const;
MutableSlice substr(size_t from, size_t size) const;
size_t find(char c) const;
size_t rfind(char c) const;
void fill(char c);
void fill_zero();
void fill_zero_secure();
void copy_from(Slice from);
char &back();
char &operator[](size_t i);
static const size_t npos = string::npos;
};
class Slice {
const char *s_;
size_t len_;
struct private_tag {};
public:
Slice();
Slice(const MutableSlice &other);
Slice(const char *s, size_t len);
Slice(const unsigned char *s, size_t len);
Slice(const string &s);
template <class T>
explicit Slice(T s, std::enable_if_t<std::is_same<char *, std::remove_const_t<T>>::value, private_tag> = {});
template <class T>
explicit Slice(T s, std::enable_if_t<std::is_same<const char *, std::remove_const_t<T>>::value, private_tag> = {});
Slice(const char *s, const char *t);
Slice(const unsigned char *s, const unsigned char *t);
template <size_t N>
constexpr Slice(char (&a)[N]) = delete;
template <size_t N>
constexpr Slice(const char (&a)[N]) : s_(a), len_(N - 1) {
}
Slice &operator=(string &&s) = delete;
template <size_t N>
constexpr Slice &operator=(char (&a)[N]) = delete;
template <size_t N>
constexpr Slice &operator=(const char (&a)[N]) {
s_ = a;
len_ = N - 1;
return *this;
}
bool empty() const;
size_t size() const;
Slice &remove_prefix(size_t prefix_len);
Slice &remove_suffix(size_t suffix_len);
Slice &truncate(size_t size);
Slice copy() const;
const char *data() const;
const char *begin() const;
const unsigned char *ubegin() const;
const char *end() const;
const unsigned char *uend() const;
string str() const;
Slice substr(size_t from) const;
Slice substr(size_t from, size_t size) const;
size_t find(char c) const;
size_t rfind(char c) const;
char back() const;
char operator[](size_t i) const;
static const size_t npos = string::npos;
};
bool operator==(const Slice &a, const Slice &b);
bool operator!=(const Slice &a, const Slice &b);
bool operator<(const Slice &a, const Slice &b);
class MutableCSlice : public MutableSlice {
struct private_tag {};
MutableSlice &remove_suffix(size_t suffix_len) = delete;
MutableSlice &truncate(size_t size) = delete;
public:
MutableCSlice() = delete;
MutableCSlice(string &s) : MutableSlice(s) {
}
template <class T>
explicit MutableCSlice(T s, std::enable_if_t<std::is_same<char *, T>::value, private_tag> = {}) : MutableSlice(s) {
}
MutableCSlice(char *s, char *t);
template <size_t N>
constexpr MutableCSlice(char (&a)[N]) = delete;
const char *c_str() const {
return begin();
}
};
class CSlice : public Slice {
struct private_tag {};
Slice &remove_suffix(size_t suffix_len) = delete;
Slice &truncate(size_t size) = delete;
public:
explicit CSlice(const MutableSlice &other) : Slice(other) {
}
CSlice(const MutableCSlice &other) : Slice(other.begin(), other.size()) {
}
CSlice(const string &s) : Slice(s) {
}
template <class T>
explicit CSlice(T s, std::enable_if_t<std::is_same<char *, std::remove_const_t<T>>::value, private_tag> = {})
: Slice(s) {
}
template <class T>
explicit CSlice(T s, std::enable_if_t<std::is_same<const char *, std::remove_const_t<T>>::value, private_tag> = {})
: Slice(s) {
}
CSlice(const char *s, const char *t);
template <size_t N>
constexpr CSlice(char (&a)[N]) = delete;
template <size_t N>
constexpr CSlice(const char (&a)[N]) : Slice(a) {
}
CSlice() : CSlice("") {
}
CSlice &operator=(string &&s) = delete;
template <size_t N>
constexpr CSlice &operator=(char (&a)[N]) = delete;
template <size_t N>
constexpr CSlice &operator=(const char (&a)[N]) {
this->Slice::operator=(a);
return *this;
}
const char *c_str() const {
return begin();
}
};
struct SliceHash {
std::size_t operator()(Slice slice) const;
};
} // namespace td

43
tdutils/td/utils/Slice.cpp Normal file
View file

@ -0,0 +1,43 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/utils/Slice.h"
#if TD_HAVE_OPENSSL
#include <openssl/crypto.h>
#endif
namespace td {
void MutableSlice::fill(char c) {
std::memset(data(), c, size());
}
void MutableSlice::fill_zero() {
fill(0);
}
void MutableSlice::fill_zero_secure() {
#if TD_HAVE_OPENSSL
OPENSSL_cleanse(begin(), size());
#else
volatile char *ptr = begin();
for (size_t i = 0; i < size(); i++) {
ptr[i] = 0;
}
#endif
}
} // namespace td

339
tdutils/td/utils/Slice.h Normal file
View file

@ -0,0 +1,339 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#include "td/utils/Slice-decl.h"
#include <cstring>
#include <type_traits>
namespace td {
inline MutableSlice::MutableSlice() : s_(const_cast<char *>("")), len_(0) {
}
inline MutableSlice::MutableSlice(char *s, size_t len) : s_(s), len_(len) {
CHECK(s_ != nullptr);
}
inline MutableSlice::MutableSlice(unsigned char *s, size_t len) : s_(reinterpret_cast<char *>(s)), len_(len) {
CHECK(s_ != nullptr);
}
inline MutableSlice::MutableSlice(string &s) : s_(&s[0]), len_(s.size()) {
}
template <class T>
MutableSlice::MutableSlice(T s, std::enable_if_t<std::is_same<char *, T>::value, private_tag>) : s_(s) {
CHECK(s_ != nullptr);
len_ = std::strlen(s_);
}
inline MutableSlice::MutableSlice(char *s, char *t) : MutableSlice(s, t - s) {
}
inline MutableSlice::MutableSlice(unsigned char *s, unsigned char *t) : MutableSlice(s, t - s) {
}
inline size_t MutableSlice::size() const {
return len_;
}
inline MutableSlice &MutableSlice::remove_prefix(size_t prefix_len) {
CHECK(prefix_len <= len_);
s_ += prefix_len;
len_ -= prefix_len;
return *this;
}
inline MutableSlice &MutableSlice::remove_suffix(size_t suffix_len) {
CHECK(suffix_len <= len_);
len_ -= suffix_len;
return *this;
}
inline MutableSlice &MutableSlice::truncate(size_t size) {
if (len_ > size) {
len_ = size;
}
return *this;
}
inline MutableSlice MutableSlice::copy() const {
return *this;
}
inline bool MutableSlice::empty() const {
return len_ == 0;
}
inline char *MutableSlice::data() const {
return s_;
}
inline char *MutableSlice::begin() const {
return s_;
}
inline unsigned char *MutableSlice::ubegin() const {
return reinterpret_cast<unsigned char *>(s_);
}
inline char *MutableSlice::end() const {
return s_ + len_;
}
inline unsigned char *MutableSlice::uend() const {
return reinterpret_cast<unsigned char *>(s_) + len_;
}
inline string MutableSlice::str() const {
return string(begin(), size());
}
inline MutableSlice MutableSlice::substr(size_t from) const {
CHECK(from <= len_);
return MutableSlice(s_ + from, len_ - from);
}
inline MutableSlice MutableSlice::substr(size_t from, size_t size) const {
CHECK(from <= len_);
return MutableSlice(s_ + from, min(size, len_ - from));
}
inline size_t MutableSlice::find(char c) const {
for (size_t pos = 0; pos < len_; pos++) {
if (s_[pos] == c) {
return pos;
}
}
return npos;
}
inline size_t MutableSlice::rfind(char c) const {
for (size_t pos = len_; pos-- > 0;) {
if (s_[pos] == c) {
return pos;
}
}
return npos;
}
inline void MutableSlice::copy_from(Slice from) {
CHECK(size() >= from.size());
std::memcpy(ubegin(), from.ubegin(), from.size());
}
inline char &MutableSlice::back() {
CHECK(1 <= len_);
return s_[len_ - 1];
}
inline char &MutableSlice::operator[](size_t i) {
return s_[i];
}
inline Slice::Slice() : s_(""), len_(0) {
}
inline Slice::Slice(const MutableSlice &other) : s_(other.begin()), len_(other.size()) {
}
inline Slice::Slice(const char *s, size_t len) : s_(s), len_(len) {
CHECK(s_ != nullptr);
}
inline Slice::Slice(const unsigned char *s, size_t len) : s_(reinterpret_cast<const char *>(s)), len_(len) {
CHECK(s_ != nullptr);
}
inline Slice::Slice(const string &s) : s_(s.c_str()), len_(s.size()) {
}
template <class T>
Slice::Slice(T s, std::enable_if_t<std::is_same<char *, std::remove_const_t<T>>::value, private_tag>) : s_(s) {
CHECK(s_ != nullptr);
len_ = std::strlen(s_);
}
template <class T>
Slice::Slice(T s, std::enable_if_t<std::is_same<const char *, std::remove_const_t<T>>::value, private_tag>) : s_(s) {
CHECK(s_ != nullptr);
len_ = std::strlen(s_);
}
inline Slice::Slice(const char *s, const char *t) : s_(s), len_(t - s) {
CHECK(s_ != nullptr);
}
inline Slice::Slice(const unsigned char *s, const unsigned char *t)
: s_(reinterpret_cast<const char *>(s)), len_(t - s) {
CHECK(s_ != nullptr);
}
inline size_t Slice::size() const {
return len_;
}
inline Slice &Slice::remove_prefix(size_t prefix_len) {
CHECK(prefix_len <= len_);
s_ += prefix_len;
len_ -= prefix_len;
return *this;
}
inline Slice &Slice::remove_suffix(size_t suffix_len) {
CHECK(suffix_len <= len_);
len_ -= suffix_len;
return *this;
}
inline Slice &Slice::truncate(size_t size) {
if (len_ > size) {
len_ = size;
}
return *this;
}
inline Slice Slice::copy() const {
return *this;
}
inline bool Slice::empty() const {
return len_ == 0;
}
inline const char *Slice::data() const {
return s_;
}
inline const char *Slice::begin() const {
return s_;
}
inline const unsigned char *Slice::ubegin() const {
return reinterpret_cast<const unsigned char *>(s_);
}
inline const char *Slice::end() const {
return s_ + len_;
}
inline const unsigned char *Slice::uend() const {
return reinterpret_cast<const unsigned char *>(s_) + len_;
}
inline string Slice::str() const {
return string(begin(), size());
}
inline Slice Slice::substr(size_t from) const {
CHECK(from <= len_);
return Slice(s_ + from, len_ - from);
}
inline Slice Slice::substr(size_t from, size_t size) const {
CHECK(from <= len_);
return Slice(s_ + from, min(size, len_ - from));
}
inline size_t Slice::find(char c) const {
for (size_t pos = 0; pos < len_; pos++) {
if (s_[pos] == c) {
return pos;
}
}
return npos;
}
inline size_t Slice::rfind(char c) const {
for (size_t pos = len_; pos-- > 0;) {
if (s_[pos] == c) {
return pos;
}
}
return npos;
}
inline char Slice::back() const {
CHECK(1 <= len_);
return s_[len_ - 1];
}
inline char Slice::operator[](size_t i) const {
return s_[i];
}
inline bool operator==(const Slice &a, const Slice &b) {
return a.size() == b.size() && std::memcmp(a.data(), b.data(), a.size()) == 0;
}
inline bool operator!=(const Slice &a, const Slice &b) {
return !(a == b);
}
inline bool operator<(const Slice &a, const Slice &b) {
auto x = std::memcmp(a.data(), b.data(), td::min(a.size(), b.size()));
if (x == 0) {
return a.size() < b.size();
}
return x < 0;
}
inline MutableCSlice::MutableCSlice(char *s, char *t) : MutableSlice(s, t) {
CHECK(*t == '\0');
}
inline CSlice::CSlice(const char *s, const char *t) : Slice(s, t) {
CHECK(*t == '\0');
}
inline std::size_t SliceHash::operator()(Slice slice) const {
// simple string hash
std::size_t result = 0;
constexpr std::size_t MUL = 123456789;
for (auto c : slice) {
result = result * MUL + c;
}
return result;
}
inline Slice as_slice(Slice slice) {
return slice;
}
inline MutableSlice as_slice(MutableSlice slice) {
return slice;
}
inline Slice as_slice(const string &str) {
return str;
}
inline MutableSlice as_slice(string &str) {
return str;
}
inline MutableSlice as_mutable_slice(MutableSlice slice) {
return slice;
}
inline MutableSlice as_mutable_slice(string &str) {
return str;
}
} // namespace td

138
tdutils/td/utils/Span.h Normal file
View file

@ -0,0 +1,138 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#include <array>
namespace td {
namespace detail {
template <class T, class InnerT>
class SpanImpl {
InnerT *data_{nullptr};
size_t size_{0};
public:
SpanImpl() = default;
SpanImpl(InnerT *data, size_t size) : data_(data), size_(size) {
}
SpanImpl(InnerT &data) : SpanImpl(&data, 1) {
}
template <class OtherInnerT>
SpanImpl(const SpanImpl<T, OtherInnerT> &other) : SpanImpl(other.data(), other.size()) {
}
template <size_t N>
SpanImpl(const std::array<T, N> &arr) : SpanImpl(arr.data(), arr.size()) {
}
template <size_t N>
SpanImpl(std::array<T, N> &arr) : SpanImpl(arr.data(), arr.size()) {
}
template <size_t N>
SpanImpl(const T (&arr)[N]) : SpanImpl(arr, N) {
}
template <size_t N>
SpanImpl(T (&arr)[N]) : SpanImpl(arr, N) {
}
SpanImpl(const vector<T> &v) : SpanImpl(v.data(), v.size()) {
}
SpanImpl(vector<T> &v) : SpanImpl(v.data(), v.size()) {
}
template <class OtherInnerT>
SpanImpl &operator=(const SpanImpl<T, OtherInnerT> &other) {
SpanImpl copy{other};
*this = copy;
}
template <class OtherInnerT>
bool operator==(const SpanImpl<T, OtherInnerT> &other) const {
if (size() != other.size()) {
return false;
}
for (size_t i = 0; i < size(); i++) {
if (!((*this)[i] == other[i])) {
return false;
}
}
return true;
}
InnerT &operator[](size_t i) {
DCHECK(i < size());
return data_[i];
}
const InnerT &operator[](size_t i) const {
DCHECK(i < size());
return data_[i];
}
InnerT *data() const {
return data_;
}
InnerT *begin() const {
return data_;
}
InnerT *end() const {
return data_ + size_;
}
size_t size() const {
return size_;
}
bool empty() const {
return size() == 0;
}
SpanImpl &truncate(size_t size) {
CHECK(size <= size_);
size_ = size;
return *this;
}
SpanImpl substr(size_t offset) const {
CHECK(offset <= size_);
return SpanImpl(begin() + offset, size_ - offset);
}
SpanImpl substr(size_t offset, size_t size) const {
CHECK(offset <= size_);
CHECK(size_ - offset >= size);
return SpanImpl(begin() + offset, size);
}
};
} // namespace detail
template <class T>
using Span = detail::SpanImpl<T, const T>;
template <class T>
using MutableSpan = detail::SpanImpl<T, T>;
template <class T>
Span<T> span(const T *ptr, size_t size) {
return Span<T>(ptr, size);
}
template <class T>
MutableSpan<T> mutable_span(T *ptr, size_t size) {
return MutableSpan<T>(ptr, size);
}
} // namespace td

72
tdutils/td/utils/SpinLock.h Normal file
View file

@ -0,0 +1,72 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/port/thread.h"
#include <atomic>
#include <memory>
namespace td {
class SpinLock {
struct Unlock {
void operator()(SpinLock *ptr) {
ptr->unlock();
}
};
class InfBackoff {
int cnt = 0;
public:
bool next() {
cnt++;
if (cnt < 50) {
//TODO pause
return true;
} else {
td::this_thread::yield();
return true;
}
}
};
public:
using Lock = std::unique_ptr<SpinLock, Unlock>;
Lock lock() {
InfBackoff backoff;
while (!try_lock()) {
backoff.next();
}
return Lock(this);
}
bool try_lock() {
return !flag_.test_and_set(std::memory_order_acquire);
}
private:
std::atomic_flag flag_ = ATOMIC_FLAG_INIT;
void unlock() {
flag_.clear(std::memory_order_release);
}
};
} // namespace td

31
tdutils/td/utils/StackAllocator.cpp Normal file
View file

@ -0,0 +1,31 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/utils/StackAllocator.h"
#include "td/utils/port/thread_local.h"
namespace td {
StackAllocator::Impl &StackAllocator::impl() {
static TD_THREAD_LOCAL StackAllocator::Impl *impl; // static zero-initialized
init_thread_local<Impl>(impl);
return *impl;
}
} // namespace td

95
tdutils/td/utils/StackAllocator.h Normal file
View file

@ -0,0 +1,95 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#include "td/utils/MovableValue.h"
#include "td/utils/Slice.h"
#include <array>
#include <cstdlib>
#include <memory>
namespace td {
class StackAllocator {
class Deleter {
public:
void operator()(char *ptr) {
free_ptr(ptr);
}
};
// TODO: alloc memory with mmap and unload unused pages
// memory still can be corrupted, but it is better than explicit free function
// TODO: use pointer that can't be even copied
using PtrImpl = std::unique_ptr<char, Deleter>;
class Ptr {
public:
Ptr(char *ptr, size_t size) : ptr_(ptr), size_(size) {
}
MutableSlice as_slice() const {
return MutableSlice(ptr_.get(), size_.get());
}
private:
PtrImpl ptr_;
MovableValue<size_t> size_;
};
static void free_ptr(char *ptr) {
impl().free_ptr(ptr);
}
struct Impl {
static const size_t MEM_SIZE = 1024 * 1024;
std::array<char, MEM_SIZE> mem;
size_t pos{0};
char *alloc(size_t size) {
if (size == 0) {
size = 1;
}
char *res = mem.data() + pos;
size = (size + 7) & -8;
pos += size;
if (pos > MEM_SIZE) {
std::abort(); // memory is over
}
return res;
}
void free_ptr(char *ptr) {
size_t new_pos = ptr - mem.data();
if (new_pos >= pos) {
std::abort(); // shouldn't happen
}
pos = new_pos;
}
};
static Impl &impl();
public:
static Ptr alloc(size_t size) {
return Ptr(impl().alloc(size), size);
}
};
} // namespace td

66
tdutils/td/utils/Status.cpp Normal file
View file

@ -0,0 +1,66 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/utils/Status.h"
#if TD_PORT_WINDOWS
#include "td/utils/port/wstring_convert.h"
#endif
#if TD_PORT_POSIX
#include "td/utils/port/thread_local.h"
#include <string.h>
#include <cstring>
#endif
namespace td {
#if TD_PORT_POSIX
CSlice strerror_safe(int code) {
const size_t size = 1000;
static TD_THREAD_LOCAL char *buf;
init_thread_local<char[]>(buf, size);
#if !defined(__GLIBC__) || ((_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600) && !_GNU_SOURCE)
strerror_r(code, buf, size);
return CSlice(buf, buf + std::strlen(buf));
#else
return CSlice(strerror_r(code, buf, size));
#endif
}
#endif
#if TD_PORT_WINDOWS
string winerror_to_string(int code) {
const size_t size = 1000;
wchar_t wbuf[size];
auto res_size = FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM, nullptr, code, 0, wbuf, size - 1, nullptr);
if (res_size == 0) {
return "Unknown windows error";
}
while (res_size != 0 && (wbuf[res_size - 1] == '\n' || wbuf[res_size - 1] == '\r')) {
res_size--;
}
return from_wstring(wbuf, res_size).ok();
}
#endif
} // namespace td

557
tdutils/td/utils/Status.h Normal file
View file

@ -0,0 +1,557 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#include "td/utils/logging.h"
#include "td/utils/ScopeGuard.h"
#include "td/utils/Slice.h"
#include "td/utils/StackAllocator.h"
#include "td/utils/StringBuilder.h"
#include <cerrno>
#include <cstring>
#include <memory>
#include <new>
#include <type_traits>
#include <utility>
#define TRY_STATUS(status) \
{ \
auto try_status = (status); \
if (try_status.is_error()) { \
return try_status.move_as_error(); \
} \
}
#define TRY_STATUS_PREFIX(status, prefix) \
{ \
auto try_status = (status); \
if (try_status.is_error()) { \
return try_status.move_as_error_prefix(prefix); \
} \
}
#define TRY_RESULT(name, result) TRY_RESULT_IMPL(TD_CONCAT(TD_CONCAT(r_, name), __LINE__), auto name, result)
#define TRY_RESULT_ASSIGN(name, result) TRY_RESULT_IMPL(TD_CONCAT(TD_CONCAT(r_, name), __LINE__), name, result)
#define TRY_RESULT_PREFIX(name, result, prefix) \
TRY_RESULT_PREFIX_IMPL(TD_CONCAT(TD_CONCAT(r_, name), __LINE__), auto name, result, prefix)
#define TRY_RESULT_PREFIX_ASSIGN(name, result, prefix) \
TRY_RESULT_PREFIX_IMPL(TD_CONCAT(TD_CONCAT(r_, name), __LINE__), name, result, prefix)
#define TRY_RESULT_PROMISE_PREFIX(promise_name, name, result, prefix) \
TRY_RESULT_PROMISE_PREFIX_IMPL(promise_name, TD_CONCAT(TD_CONCAT(r_, name), __LINE__), auto name, result, prefix)
#define TRY_RESULT_PROMISE_PREFIX_ASSIGN(promise_name, name, result, prefix) \
TRY_RESULT_PROMISE_PREFIX_IMPL(promise_name, TD_CONCAT(TD_CONCAT(r_, name), __LINE__), name, result, prefix)
#define TRY_RESULT_IMPL(r_name, name, result) \
auto r_name = (result); \
if (r_name.is_error()) { \
return r_name.move_as_error(); \
} \
name = r_name.move_as_ok();
#define TRY_RESULT_PREFIX_IMPL(r_name, name, result, prefix) \
auto r_name = (result); \
if (r_name.is_error()) { \
return r_name.move_as_error_prefix(prefix); \
} \
name = r_name.move_as_ok();
#define TRY_RESULT_PROMISE_PREFIX_IMPL(promise_name, r_name, name, result, prefix) \
auto r_name = (result); \
if (r_name.is_error()) { \
promise.set_error(r_name.move_as_error()); \
return; \
} \
name = r_name.move_as_ok();
#define LOG_STATUS(status) \
{ \
auto log_status = (status); \
if (log_status.is_error()) { \
LOG(ERROR) << log_status.move_as_error(); \
} \
}
#ifndef TD_STATUS_NO_ENSURE
#define ensure() ensure_impl(__FILE__, __LINE__)
#define ensure_error() ensure_error_impl(__FILE__, __LINE__)
#endif
#if TD_PORT_POSIX
#define OS_ERROR(message) \
[&]() { \
auto saved_errno = errno; \
return ::td::Status::PosixError(saved_errno, (message)); \
}()
#define OS_SOCKET_ERROR(message) OS_ERROR(message)
#elif TD_PORT_WINDOWS
#define OS_ERROR(message) \
[&]() { \
auto saved_error = ::GetLastError(); \
return ::td::Status::WindowsError(saved_error, (message)); \
}()
#define OS_SOCKET_ERROR(message) \
[&]() { \
auto saved_error = ::WSAGetLastError(); \
return ::td::Status::WindowsError(saved_error, (message)); \
}()
#endif
namespace td {
#if TD_PORT_POSIX
CSlice strerror_safe(int code);
#endif
#if TD_PORT_WINDOWS
string winerror_to_string(int code);
#endif
class Status {
enum class ErrorType : int8 { general, os };
public:
Status() = default;
bool operator==(const Status &other) const {
return ptr_ == other.ptr_;
}
Status clone() const TD_WARN_UNUSED_RESULT {
if (is_ok()) {
return Status();
}
auto info = get_info();
if (info.static_flag) {
return clone_static();
}
return Status(false, info.error_type, info.error_code, message());
}
static Status OK() TD_WARN_UNUSED_RESULT {
return Status();
}
static Status Error(int err, Slice message = Slice()) TD_WARN_UNUSED_RESULT {
return Status(false, ErrorType::general, err, message);
}
static Status Error(Slice message) TD_WARN_UNUSED_RESULT {
return Error(0, message);
}
#if TD_PORT_WINDOWS
static Status WindowsError(int saved_error, Slice message) TD_WARN_UNUSED_RESULT {
return Status(false, ErrorType::os, saved_error, message);
}
#endif
#if TD_PORT_POSIX
static Status PosixError(int32 saved_errno, Slice message) TD_WARN_UNUSED_RESULT {
return Status(false, ErrorType::os, saved_errno, message);
}
#endif
static Status Error() TD_WARN_UNUSED_RESULT {
return Error<0>();
}
template <int Code>
static Status Error() {
static Status status(true, ErrorType::general, Code, "");
return status.clone_static();
}
StringBuilder &print(StringBuilder &sb) const {
if (is_ok()) {
return sb << "OK";
}
Info info = get_info();
switch (info.error_type) {
case ErrorType::general:
sb << "[Error";
break;
case ErrorType::os:
#if TD_PORT_POSIX
sb << "[PosixError : " << strerror_safe(info.error_code);
#elif TD_PORT_WINDOWS
sb << "[WindowsError : " << winerror_to_string(info.error_code);
#endif
break;
default:
UNREACHABLE();
break;
}
sb << " : " << code() << " : " << message() << "]";
return sb;
}
string to_string() const {
auto buf = StackAllocator::alloc(4096);
StringBuilder sb(buf.as_slice());
print(sb);
return sb.as_cslice().str();
}
// Default interface
bool is_ok() const TD_WARN_UNUSED_RESULT {
return !is_error();
}
bool is_error() const TD_WARN_UNUSED_RESULT {
return ptr_ != nullptr;
}
#ifdef TD_STATUS_NO_ENSURE
void ensure() const {
if (!is_ok()) {
LOG(FATAL) << "Unexpected Status " << to_string();
}
}
void ensure_error() const {
if (is_ok()) {
LOG(FATAL) << "Unexpected Status::OK";
}
}
#else
void ensure_impl(CSlice file_name, int line) const {
if (!is_ok()) {
LOG(FATAL) << "Unexpected Status " << to_string() << " in file " << file_name << " at line " << line;
}
}
void ensure_error_impl(CSlice file_name, int line) const {
if (is_ok()) {
LOG(FATAL) << "Unexpected Status::OK in file " << file_name << " at line " << line;
}
}
#endif
void ignore() const {
// nop
}
int32 code() const {
if (is_ok()) {
return 0;
}
return get_info().error_code;
}
CSlice message() const {
if (is_ok()) {
return CSlice("OK");
}
return CSlice(ptr_.get() + sizeof(Info));
}
string public_message() const {
if (is_ok()) {
return "OK";
}
Info info = get_info();
switch (info.error_type) {
case ErrorType::general:
return message().str();
case ErrorType::os:
#if TD_PORT_POSIX
return strerror_safe(info.error_code).str();
#elif TD_PORT_WINDOWS
return winerror_to_string(info.error_code);
#endif
default:
UNREACHABLE();
return "";
}
}
const Status &error() const {
return *this;
}
Status move() TD_WARN_UNUSED_RESULT {
return std::move(*this);
}
Status move_as_error() TD_WARN_UNUSED_RESULT {
return std::move(*this);
}
Status move_as_error_prefix(Slice prefix) TD_WARN_UNUSED_RESULT {
CHECK(is_error());
Info info = get_info();
switch (info.error_type) {
case ErrorType::general:
return Error(code(), PSLICE() << prefix << message());
case ErrorType::os:
return Status(false, ErrorType::os, code(), PSLICE() << prefix << message());
default:
UNREACHABLE();
return {};
}
}
private:
struct Info {
bool static_flag : 1;
signed int error_code : 23;
ErrorType error_type;
};
struct Deleter {
void operator()(char *ptr) {
if (!get_info(ptr).static_flag) {
delete[] ptr;
}
}
};
std::unique_ptr<char[], Deleter> ptr_;
Status(Info info, Slice message) {
size_t size = sizeof(Info) + message.size() + 1;
ptr_ = std::unique_ptr<char[], Deleter>(new char[size]);
char *ptr = ptr_.get();
reinterpret_cast<Info *>(ptr)[0] = info;
ptr += sizeof(Info);
std::memcpy(ptr, message.begin(), message.size());
ptr += message.size();
*ptr = 0;
}
Status(bool static_flag, ErrorType error_type, int error_code, Slice message)
: Status(to_info(static_flag, error_type, error_code), message) {
if (static_flag) {
TD_LSAN_IGNORE(ptr_.get());
}
}
Status clone_static() const TD_WARN_UNUSED_RESULT {
CHECK(is_ok() || get_info().static_flag);
Status result;
result.ptr_ = std::unique_ptr<char[], Deleter>(ptr_.get());
return result;
}
static Info to_info(bool static_flag, ErrorType error_type, int error_code) {
const int MIN_ERROR_CODE = -(1 << 22) + 1;
const int MAX_ERROR_CODE = (1 << 22) - 1;
Info tmp;
tmp.static_flag = static_flag;
tmp.error_type = error_type;
if (error_code < MIN_ERROR_CODE) {
LOG(ERROR) << "Error code value is altered from " << error_code;
error_code = MIN_ERROR_CODE;
}
if (error_code > MAX_ERROR_CODE) {
LOG(ERROR) << "Error code value is altered from " << error_code;
error_code = MAX_ERROR_CODE;
}
#if TD_GCC
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wconversion"
#endif
tmp.error_code = error_code;
#if TD_GCC
#pragma GCC diagnostic pop
#endif
CHECK(error_code == tmp.error_code);
return tmp;
}
Info get_info() const {
return get_info(ptr_.get());
}
static Info get_info(char *ptr) {
return reinterpret_cast<Info *>(ptr)[0];
}
};
template <class T = Unit>
class Result {
public:
Result() : status_(Status::Error<-1>()) {
}
template <class S, std::enable_if_t<!std::is_same<std::decay_t<S>, Result>::value, int> = 0>
Result(S &&x) : status_(), value_(std::forward<S>(x)) {
}
struct emplace_t {};
template <class... ArgsT>
Result(emplace_t, ArgsT &&... args) : status_(), value_(std::forward<ArgsT>(args)...) {
}
Result(Status &&status) : status_(std::move(status)) {
CHECK(status_.is_error());
}
Result(const Result &) = delete;
Result &operator=(const Result &) = delete;
Result(Result &&other) : status_(std::move(other.status_)) {
if (status_.is_ok()) {
new (&value_) T(std::move(other.value_));
other.value_.~T();
}
other.status_ = Status::Error<-2>();
}
Result &operator=(Result &&other) {
CHECK(this != &other);
if (status_.is_ok()) {
value_.~T();
}
if (other.status_.is_ok()) {
#if TD_GCC
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
#endif
new (&value_) T(std::move(other.value_));
#if TD_GCC
#pragma GCC diagnostic pop
#endif
other.value_.~T();
}
status_ = std::move(other.status_);
other.status_ = Status::Error<-3>();
return *this;
}
template <class... ArgsT>
void emplace(ArgsT &&... args) {
if (status_.is_ok()) {
value_.~T();
}
new (&value_) T(std::forward<ArgsT>(args)...);
status_ = Status::OK();
}
~Result() {
if (status_.is_ok()) {
value_.~T();
}
}
#ifdef TD_STATUS_NO_ENSURE
void ensure() const {
status_.ensure();
}
void ensure_error() const {
status_.ensure_error();
}
#else
void ensure_impl(CSlice file_name, int line) const {
status_.ensure_impl(file_name, line);
}
void ensure_error_impl(CSlice file_name, int line) const {
status_.ensure_error_impl(file_name, line);
}
#endif
void ignore() const {
status_.ignore();
}
bool is_ok() const {
return status_.is_ok();
}
bool is_error() const {
return status_.is_error();
}
const Status &error() const {
CHECK(status_.is_error());
return status_;
}
Status move_as_error() TD_WARN_UNUSED_RESULT {
CHECK(status_.is_error());
SCOPE_EXIT {
status_ = Status::Error<-4>();
};
return std::move(status_);
}
Status move_as_error_prefix(Slice prefix) TD_WARN_UNUSED_RESULT {
SCOPE_EXIT {
status_ = Status::Error<-5>();
};
return status_.move_as_error_prefix(prefix);
}
const T &ok() const {
LOG_CHECK(status_.is_ok()) << status_;
return value_;
}
T &ok_ref() {
LOG_CHECK(status_.is_ok()) << status_;
return value_;
}
const T &ok_ref() const {
LOG_CHECK(status_.is_ok()) << status_;
return value_;
}
T move_as_ok() {
LOG_CHECK(status_.is_ok()) << status_;
return std::move(value_);
}
Result<T> clone() const TD_WARN_UNUSED_RESULT {
if (is_ok()) {
return Result<T>(ok()); // TODO: return clone(ok());
}
return error().clone();
}
void clear() {
*this = Result<T>();
}
private:
Status status_;
union {
T value_;
};
};
template <>
inline Result<Unit>::Result(Status &&status) : status_(std::move(status)) {
// no assert
}
inline StringBuilder &operator<<(StringBuilder &string_builder, const Status &status) {
return status.print(string_builder);
}
namespace detail {
class SlicifySafe {
public:
Result<CSlice> operator&(Logger &logger) {
if (logger.is_error()) {
return Status::Error("Buffer overflow");
}
return logger.as_cslice();
}
};
class StringifySafe {
public:
Result<string> operator&(Logger &logger) {
if (logger.is_error()) {
return Status::Error("Buffer overflow");
}
return logger.as_cslice().str();
}
};
} // namespace detail
} // namespace td

100
tdutils/td/utils/Storer.h Normal file
View file

@ -0,0 +1,100 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/StorerBase.h"
#include "td/utils/common.h"
#include "td/utils/Slice.h"
#include "td/utils/tl_storers.h"
#include <cstring>
#include <limits>
namespace td {
class SliceStorer : public Storer {
Slice slice;
public:
explicit SliceStorer(Slice slice) : slice(slice) {
}
size_t size() const override {
return slice.size();
}
size_t store(uint8 *ptr) const override {
std::memcpy(ptr, slice.ubegin(), slice.size());
return slice.size();
}
};
inline SliceStorer create_storer(Slice slice) {
return SliceStorer(slice);
}
class ConcatStorer : public Storer {
const Storer &a_;
const Storer &b_;
public:
ConcatStorer(const Storer &a, const Storer &b) : a_(a), b_(b) {
}
size_t size() const override {
return a_.size() + b_.size();
}
size_t store(uint8 *ptr) const override {
uint8 *ptr_save = ptr;
ptr += a_.store(ptr);
ptr += b_.store(ptr);
return ptr - ptr_save;
}
};
inline ConcatStorer create_storer(const Storer &a, const Storer &b) {
return ConcatStorer(a, b);
}
template <class T>
class DefaultStorer : public Storer {
public:
explicit DefaultStorer(const T &object) : object_(object) {
}
size_t size() const override {
if (size_ == std::numeric_limits<size_t>::max()) {
size_ = tl_calc_length(object_);
}
return size_;
}
size_t store(uint8 *ptr) const override {
return tl_store_unsafe(object_, ptr);
}
private:
mutable size_t size_ = std::numeric_limits<size_t>::max();
const T &object_;
};
template <class T>
DefaultStorer<T> create_default_storer(const T &from) {
return DefaultStorer<T>(from);
}
} // namespace td

37
tdutils/td/utils/StorerBase.h Normal file
View file

@ -0,0 +1,37 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/int_types.h"
namespace td {
class Storer {
public:
Storer() = default;
Storer(const Storer &) = delete;
Storer &operator=(const Storer &) = delete;
Storer(Storer &&) = default;
Storer &operator=(Storer &&) = default;
virtual ~Storer() = default;
virtual size_t size() const = 0;
virtual size_t store(uint8 *ptr) const TD_WARN_UNUSED_RESULT = 0;
};
} // namespace td

226
tdutils/td/utils/StringBuilder.cpp Normal file
View file

@ -0,0 +1,226 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/utils/StringBuilder.h"
#include "td/utils/misc.h"
#include "td/utils/port/thread_local.h"
#include "td/utils/Slice.h"
#include <cstdio>
#include <cstring>
#include <limits>
#include <locale>
#include <memory>
#include <sstream>
#include <utility>
namespace td {
StringBuilder::StringBuilder(MutableSlice slice, bool use_buffer)
: begin_ptr_(slice.begin()), current_ptr_(begin_ptr_), use_buffer_(use_buffer) {
if (slice.size() <= reserved_size) {
auto buffer_size = reserved_size + 100;
buffer_ = std::make_unique<char[]>(buffer_size);
begin_ptr_ = buffer_.get();
current_ptr_ = begin_ptr_;
end_ptr_ = begin_ptr_ + buffer_size - reserved_size;
} else {
end_ptr_ = slice.end() - reserved_size;
}
}
StringBuilder &StringBuilder::operator<<(Slice slice) {
size_t size = slice.size();
if (unlikely(!reserve(size))) {
if (end_ptr_ < current_ptr_) {
return on_error();
}
auto available_size = static_cast<size_t>(end_ptr_ + reserved_size - 1 - current_ptr_);
if (size > available_size) {
error_flag_ = true;
size = available_size;
}
}
std::memcpy(current_ptr_, slice.begin(), size);
current_ptr_ += size;
return *this;
}
template <class T>
static char *print_uint(char *current_ptr, T x) {
if (x < 100) {
if (x < 10) {
*current_ptr++ = static_cast<char>('0' + x);
} else {
*current_ptr++ = static_cast<char>('0' + x / 10);
*current_ptr++ = static_cast<char>('0' + x % 10);
}
return current_ptr;
}
auto begin_ptr = current_ptr;
do {
*current_ptr++ = static_cast<char>('0' + x % 10);
x /= 10;
} while (x > 0);
auto end_ptr = current_ptr - 1;
while (begin_ptr < end_ptr) {
std::swap(*begin_ptr++, *end_ptr--);
}
return current_ptr;
}
template <class T>
static char *print_int(char *current_ptr, T x) {
if (x < 0) {
if (x == std::numeric_limits<T>::min()) {
std::stringstream ss;
ss << x;
auto len = narrow_cast<int>(static_cast<std::streamoff>(ss.tellp()));
ss.read(current_ptr, len);
return current_ptr + len;
}
*current_ptr++ = '-';
x = -x;
}
return print_uint(current_ptr, x);
}
bool StringBuilder::reserve_inner(size_t size) {
if (!use_buffer_) {
return false;
}
size_t old_data_size = current_ptr_ - begin_ptr_;
if (size >= std::numeric_limits<size_t>::max() - reserved_size - old_data_size - 1) {
return false;
}
size_t need_data_size = old_data_size + size;
size_t old_buffer_size = end_ptr_ - begin_ptr_;
if (old_buffer_size >= (std::numeric_limits<size_t>::max() - reserved_size) / 2 - 2) {
return false;
}
size_t new_buffer_size = (old_buffer_size + 1) * 2;
if (new_buffer_size < need_data_size) {
new_buffer_size = need_data_size;
}
if (new_buffer_size < 100) {
new_buffer_size = 100;
}
new_buffer_size += reserved_size;
auto new_buffer = std::make_unique<char[]>(new_buffer_size);
std::memcpy(new_buffer.get(), begin_ptr_, old_data_size);
buffer_ = std::move(new_buffer);
begin_ptr_ = buffer_.get();
current_ptr_ = begin_ptr_ + old_data_size;
end_ptr_ = begin_ptr_ + new_buffer_size - reserved_size;
CHECK(end_ptr_ > current_ptr_);
CHECK(static_cast<size_t>(end_ptr_ - current_ptr_) >= size);
return true;
}
StringBuilder &StringBuilder::operator<<(int x) {
if (unlikely(!reserve())) {
return on_error();
}
current_ptr_ = print_int(current_ptr_, x);
return *this;
}
StringBuilder &StringBuilder::operator<<(unsigned int x) {
if (unlikely(!reserve())) {
return on_error();
}
current_ptr_ = print_uint(current_ptr_, x);
return *this;
}
StringBuilder &StringBuilder::operator<<(long int x) {
if (unlikely(!reserve())) {
return on_error();
}
current_ptr_ = print_int(current_ptr_, x);
return *this;
}
StringBuilder &StringBuilder::operator<<(long unsigned int x) {
if (unlikely(!reserve())) {
return on_error();
}
current_ptr_ = print_uint(current_ptr_, x);
return *this;
}
StringBuilder &StringBuilder::operator<<(long long int x) {
if (unlikely(!reserve())) {
return on_error();
}
current_ptr_ = print_int(current_ptr_, x);
return *this;
}
StringBuilder &StringBuilder::operator<<(long long unsigned int x) {
if (unlikely(!reserve())) {
return on_error();
}
current_ptr_ = print_uint(current_ptr_, x);
return *this;
}
StringBuilder &StringBuilder::operator<<(FixedDouble x) {
if (unlikely(!reserve(std::numeric_limits<double>::max_exponent10 + x.precision + 4))) {
return on_error();
}
static TD_THREAD_LOCAL std::stringstream *ss;
if (init_thread_local<std::stringstream>(ss)) {
auto previous_locale = ss->imbue(std::locale::classic());
ss->setf(std::ios_base::fixed, std::ios_base::floatfield);
} else {
ss->str(std::string());
ss->clear();
}
ss->precision(x.precision);
*ss << x.d;
int len = narrow_cast<int>(static_cast<std::streamoff>(ss->tellp()));
auto left = end_ptr_ + reserved_size - current_ptr_;
if (unlikely(len >= left)) {
error_flag_ = true;
len = left ? narrow_cast<int>(left - 1) : 0;
}
ss->read(current_ptr_, len);
current_ptr_ += len;
return *this;
}
StringBuilder &StringBuilder::operator<<(const void *ptr) {
if (unlikely(!reserve())) {
return on_error();
}
current_ptr_ += std::snprintf(current_ptr_, reserved_size, "%p", ptr);
return *this;
}
} // namespace td

150
tdutils/td/utils/StringBuilder.h Normal file
View file

@ -0,0 +1,150 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#include "td/utils/Slice.h"
#include "td/utils/StackAllocator.h"
#include <cstdlib>
#include <memory>
#include <type_traits>
namespace td {
class StringBuilder {
public:
explicit StringBuilder(MutableSlice slice, bool use_buffer = false);
void clear() {
current_ptr_ = begin_ptr_;
error_flag_ = false;
}
MutableCSlice as_cslice() {
if (current_ptr_ >= end_ptr_ + reserved_size) {
std::abort(); // shouldn't happen
}
*current_ptr_ = 0;
return MutableCSlice(begin_ptr_, current_ptr_);
}
bool is_error() const {
return error_flag_;
}
StringBuilder &operator<<(const char *str) {
return *this << Slice(str);
}
StringBuilder &operator<<(const wchar_t *str) = delete;
StringBuilder &operator<<(Slice slice);
StringBuilder &operator<<(bool b) {
return *this << (b ? Slice("true") : Slice("false"));
}
StringBuilder &operator<<(char c) {
if (unlikely(!reserve())) {
return on_error();
}
*current_ptr_++ = c;
return *this;
}
StringBuilder &operator<<(unsigned char c) {
return *this << static_cast<unsigned int>(c);
}
StringBuilder &operator<<(signed char c) {
return *this << static_cast<int>(c);
}
StringBuilder &operator<<(int x);
StringBuilder &operator<<(unsigned int x);
StringBuilder &operator<<(long int x);
StringBuilder &operator<<(long unsigned int x);
StringBuilder &operator<<(long long int x);
StringBuilder &operator<<(long long unsigned int x);
struct FixedDouble {
double d;
int precision;
FixedDouble(double d, int precision) : d(d), precision(precision) {
}
};
StringBuilder &operator<<(FixedDouble x);
StringBuilder &operator<<(double x) {
return *this << FixedDouble(x, 6);
}
StringBuilder &operator<<(const void *ptr);
template <class T>
StringBuilder &operator<<(const T *ptr) {
return *this << static_cast<const void *>(ptr);
}
private:
char *begin_ptr_;
char *current_ptr_;
char *end_ptr_;
bool error_flag_ = false;
bool use_buffer_ = false;
std::unique_ptr<char[]> buffer_;
static constexpr size_t reserved_size = 30;
StringBuilder &on_error() {
error_flag_ = true;
return *this;
}
bool reserve() {
if (end_ptr_ > current_ptr_) {
return true;
}
return reserve_inner(reserved_size);
}
bool reserve(size_t size) {
if (end_ptr_ > current_ptr_ && static_cast<size_t>(end_ptr_ - current_ptr_) >= size) {
return true;
}
return reserve_inner(size);
}
bool reserve_inner(size_t size);
};
template <class T>
std::enable_if_t<std::is_arithmetic<T>::value, string> to_string(const T &x) {
const size_t buf_size = 1000;
auto buf = StackAllocator::alloc(buf_size);
StringBuilder sb(buf.as_slice());
sb << x;
return sb.as_cslice().str();
}
} // namespace td

66
tdutils/td/utils/ThreadLocalStorage.h Normal file
View file

@ -0,0 +1,66 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/port/thread.h"
#include "td/utils/port/thread_local.h"
#include "td/utils/int_types.h"
#include <atomic>
#include <array>
namespace td {
template <class T>
class ThreadLocalStorage {
public:
T& get() {
return thread_local_node().value;
}
template <class F>
void for_each(F&& f) {
int n = max_thread_id_.load();
for (int i = 0; i < n; i++) {
f(nodes_[i].value);
}
}
template <class F>
void for_each(F&& f) const {
int n = max_thread_id_.load();
for (int i = 0; i < n; i++) {
f(nodes_[i].value);
}
}
private:
struct Node {
T value{};
char padding[128];
};
static constexpr int MAX_THREAD_ID = 128;
std::atomic<int> max_thread_id_{MAX_THREAD_ID};
std::array<Node, MAX_THREAD_ID> nodes_;
Node& thread_local_node() {
auto thread_id = get_thread_id();
CHECK(0 <= thread_id && static_cast<size_t>(thread_id) < nodes_.size());
return nodes_[thread_id];
}
};
} // namespace td

123
tdutils/td/utils/ThreadSafeCounter.h Normal file
View file

@ -0,0 +1,123 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/ThreadLocalStorage.h"
#include "td/utils/StringBuilder.h"
#include <array>
#include <mutex>
namespace td {
template <size_t N>
class ThreadSafeMultiCounter {
public:
void add(size_t index, int64 diff) {
CHECK(index < N);
tls_.get()[index].fetch_add(diff, std::memory_order_relaxed);
}
int64 sum(size_t index) const {
CHECK(index < N);
int64 res = 0;
tls_.for_each([&](auto &value) { res += value[index].load(); });
return res;
}
private:
ThreadLocalStorage<std::array<std::atomic<int64>, N>> tls_;
};
class ThreadSafeCounter {
public:
void add(int64 diff) {
counter_.add(0, diff);
}
int64 sum() const {
return counter_.sum(0);
}
private:
ThreadSafeMultiCounter<1> counter_;
};
class NamedThreadSafeCounter {
static constexpr int N = 128;
using Counter = ThreadSafeMultiCounter<N>;
public:
class CounterRef {
public:
CounterRef() = default;
CounterRef(size_t index, Counter *counter) : index_(index), counter_(counter) {
}
void add(int64 diff) {
counter_->add(index_, diff);
}
int64 sum() const {
return counter_->sum(index_);
}
private:
size_t index_{0};
Counter *counter_{nullptr};
};
CounterRef get_counter(Slice name) {
std::unique_lock<std::mutex> guard(mutex_);
for (size_t i = 0; i < names_.size(); i++) {
if (names_[i] == name) {
return get_counter_ref(i);
}
}
CHECK(names_.size() < N);
names_.push_back(name.str());
return get_counter_ref(names_.size() - 1);
}
CounterRef get_counter_ref(size_t index) {
return CounterRef(index, &counter_);
}
static NamedThreadSafeCounter &get_default() {
static NamedThreadSafeCounter res;
return res;
}
template <class F>
void for_each(F &&f) const {
std::unique_lock<std::mutex> guard(mutex_);
for (size_t i = 0; i < names_.size(); i++) {
f(names_[i], counter_.sum(i));
}
}
friend StringBuilder &operator<<(StringBuilder &sb, const NamedThreadSafeCounter &counter) {
counter.for_each([&sb](Slice name, int64 cnt) { sb << name << ": " << cnt << "\n"; });
return sb;
}
private:
mutable std::mutex mutex_;
std::vector<std::string> names_;
Counter counter_;
};
} // namespace td

32
tdutils/td/utils/Time.cpp Normal file
View file

@ -0,0 +1,32 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/utils/Time.h"
#include <cmath>
namespace td {
bool operator==(Timestamp a, Timestamp b) {
return std::abs(a.at() - b.at()) < 1e-6;
}
double Time::now() {
return Clocks::monotonic();
}
} // namespace td

124
tdutils/td/utils/Time.h Normal file
View file

@ -0,0 +1,124 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#include "td/utils/port/Clocks.h"
namespace td {
class Time {
public:
static double now();
static double now_cached() {
// Temporary(?) use now in now_cached
// Problem:
// thread A: check that now() > timestamp and notifies thread B
// thread B: must see that now() > timestamp()
//
// now() and now_cached() must be monotonic
//
// if a=now[_cached]() happens before b=now[_cached] than
// a <= b
//
// As an alternative we may say that now_cached is a thread local copy of now
return now();
}
};
inline void relax_timeout_at(double *timeout, double new_timeout) {
if (new_timeout == 0) {
return;
}
if (*timeout == 0 || new_timeout < *timeout) {
*timeout = new_timeout;
}
}
class Timestamp {
public:
Timestamp() = default;
static Timestamp never() {
return Timestamp{};
}
static Timestamp now() {
return Timestamp{Time::now()};
}
static Timestamp now_cached() {
return Timestamp{Time::now_cached()};
}
static Timestamp at(double timeout) {
return Timestamp{timeout};
}
static Timestamp at_unix(double timeout) {
return Timestamp{timeout - td::Clocks::system() + Time::now()};
}
static Timestamp in(double timeout) {
return Timestamp{Time::now_cached() + timeout};
}
bool is_in_past() const {
return at_ <= Time::now_cached();
}
explicit operator bool() const {
return at_ > 0;
}
double at() const {
return at_;
}
double at_unix() const {
return at_ + Clocks::system() - Time::now();
}
double in() const {
return at_ - Time::now_cached();
}
void relax(const Timestamp &timeout) {
if (!timeout) {
return;
}
if (!*this || at_ > timeout.at_) {
at_ = timeout.at_;
}
}
friend bool operator==(Timestamp a, Timestamp b);
private:
double at_{0};
explicit Timestamp(double timeout) : at_(timeout) {
}
};
template <class StorerT>
void store(const Timestamp &timestamp, StorerT &storer) {
storer.store_binary(timestamp.at() - Time::now() + Clocks::system());
}
template <class ParserT>
void parse(Timestamp &timestamp, ParserT &parser) {
timestamp = Timestamp::in(parser.fetch_double() - Clocks::system());
}
} // namespace td

83
tdutils/td/utils/TimedStat.h Normal file
View file

@ -0,0 +1,83 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#include <utility>
namespace td {
template <class StatT>
class TimedStat {
public:
TimedStat(double duration, double now)
: duration_(duration), current_(), current_timestamp_(now), next_(), next_timestamp_(now) {
}
TimedStat() : TimedStat(0, 0) {
}
template <class EventT>
void add_event(const EventT &e, double now) {
update(now);
current_.on_event(e);
next_.on_event(e);
}
const StatT &get_stat(double now) {
update(now);
return current_;
}
std::pair<StatT, double> stat_duration(double now) {
update(now);
return std::make_pair(current_, now - current_timestamp_);
}
void clear_events() {
current_.clear();
next_.clear();
}
private:
double duration_;
StatT current_;
double current_timestamp_;
StatT next_;
double next_timestamp_;
void update(double &now) {
if (now < next_timestamp_) {
// LOG_CHECK(now >= next_timestamp_ * (1 - 1e-14)) << now << " " << next_timestamp_;
now = next_timestamp_;
}
if (duration_ == 0) {
return;
}
if (next_timestamp_ + 2 * duration_ < now) {
current_ = StatT();
current_timestamp_ = now;
next_ = StatT();
next_timestamp_ = now;
} else if (next_timestamp_ + duration_ < now) {
current_ = next_;
current_timestamp_ = next_timestamp_;
next_ = StatT();
next_timestamp_ = now;
}
}
};
} // namespace td

62
tdutils/td/utils/Timer.cpp Normal file
View file

@ -0,0 +1,62 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/utils/Timer.h"
#include "td/utils/format.h"
#include "td/utils/logging.h"
//#include "td/utils/Slice.h" // TODO move StringBuilder implementation to cpp, remove header
#include "td/utils/Time.h"
namespace td {
Timer::Timer() : start_time_(Time::now()) {
}
double Timer::elapsed() const {
return Time::now() - start_time_;
}
StringBuilder &operator<<(StringBuilder &string_builder, const Timer &timer) {
return string_builder << "in " << Time::now() - timer.start_time_;
}
PerfWarningTimer::PerfWarningTimer(string name, double max_duration)
: name_(std::move(name)), start_at_(Time::now()), max_duration_(max_duration) {
}
PerfWarningTimer::PerfWarningTimer(PerfWarningTimer &&other)
: name_(std::move(other.name_)), start_at_(other.start_at_), max_duration_(other.max_duration_) {
other.start_at_ = 0;
}
PerfWarningTimer::~PerfWarningTimer() {
reset();
}
void PerfWarningTimer::reset() {
if (start_at_ == 0) {
return;
}
double duration = Time::now() - start_at_;
LOG_IF(WARNING, duration > max_duration_)
<< "SLOW: " << tag("name", name_) << tag("duration", format::as_time(duration));
start_at_ = 0;
}
} // namespace td

53
tdutils/td/utils/Timer.h Normal file
View file

@ -0,0 +1,53 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/StringBuilder.h"
namespace td {
class Timer {
public:
Timer();
double elapsed() const;
private:
friend StringBuilder &operator<<(StringBuilder &string_builder, const Timer &timer);
double start_time_;
};
class PerfWarningTimer {
public:
explicit PerfWarningTimer(string name, double max_duration = 0.1);
PerfWarningTimer(const PerfWarningTimer &) = delete;
PerfWarningTimer &operator=(const PerfWarningTimer &) = delete;
PerfWarningTimer(PerfWarningTimer &&other);
PerfWarningTimer &operator=(PerfWarningTimer &&) = delete;
~PerfWarningTimer();
void reset();
private:
string name_;
double start_at_{0};
double max_duration_{0};
};
} // namespace td

92
tdutils/td/utils/TsFileLog.cpp Normal file
View file

@ -0,0 +1,92 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "TsFileLog.h"
#include <limits>
namespace td {
namespace detail {
class TsFileLog : public LogInterface {
public:
Status init(string path) {
path_ = std::move(path);
for (int i = 0; i < (int)logs_.size(); i++) {
logs_[i].id = i;
}
return init_info(&logs_[0]);
}
vector<string> get_file_paths() override {
vector<string> res;
for (auto &log : logs_) {
res.push_back(get_path(&log));
}
return res;
}
void append(CSlice cslice) override {
return append(cslice, -1);
}
void append(CSlice cslice, int log_level) override {
get_current_logger()->append(cslice, log_level);
}
private:
struct Info {
FileLog log;
bool is_inited;
int id;
};
static constexpr int MAX_THREAD_ID = 128;
std::string path_;
std::array<Info, MAX_THREAD_ID> logs_;
LogInterface *get_current_logger() {
auto *info = get_current_info();
if (!info->is_inited) {
CHECK(init_info(info).is_ok());
}
return &info->log;
}
Info *get_current_info() {
return &logs_[get_thread_id()];
}
Status init_info(Info *info) {
TRY_STATUS(info->log.init(get_path(info), std::numeric_limits<int64>::max(), info->id == 0));
info->is_inited = true;
return Status::OK();
}
string get_path(Info *info) {
if (info->id == 0) {
return path_;
}
return PSTRING() << path_ << "." << info->id;
}
};
} // namespace detail
Result<td::unique_ptr<LogInterface>> TsFileLog::create(string path) {
auto res = td::make_unique<detail::TsFileLog>();
TRY_STATUS(res->init(path));
return std::move(res);
}
} // namespace td

28
tdutils/td/utils/TsFileLog.h Normal file
View file

@ -0,0 +1,28 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/FileLog.h"
namespace td {
class TsFileLog {
public:
static Result<td::unique_ptr<LogInterface>> create(string path);
};
} // namespace td

105
tdutils/td/utils/UInt.h Normal file
View file

@ -0,0 +1,105 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#include "td/utils/Slice.h"
#include <cstring>
namespace td {
template <size_t size>
struct UInt {
static_assert(size % 8 == 0, "size should be divisible by 8");
uint8 raw[size / 8];
Slice as_slice() const {
return Slice(raw, size / 8);
}
MutableSlice as_slice() {
return MutableSlice(raw, size / 8);
}
bool is_zero() const {
for (size_t i = 0; i < size / 8; i++) {
if (raw[i] != 0) {
return false;
}
}
return true;
}
void set_zero() {
for (size_t i = 0; i < size / 8; i++) {
raw[i] = 0;
}
}
static UInt zero() {
UInt v;
v.set_zero();
return v;
}
};
template <size_t size>
bool operator==(const UInt<size> &a, const UInt<size> &b) {
return a.as_slice() == b.as_slice();
}
template <size_t size>
bool operator!=(const UInt<size> &a, const UInt<size> &b) {
return !(a == b);
}
template <size_t size>
td::UInt<size> operator^(const UInt<size> &a, const UInt<size> &b) {
td::UInt<size> res;
for (size_t i = 0; i < size / 8; i++) {
res.raw[i] = static_cast<uint8>(a.raw[i] ^ b.raw[i]);
}
return res;
}
template <size_t size>
int get_kth_bit(const UInt<size> &a, uint32 bit) {
uint8 b = a.raw[bit / 8];
bit &= 7;
return (b >> (7 - bit)) & 1;
}
template <size_t size>
Slice as_slice(const UInt<size> &value) {
return value.as_slice();
}
template <size_t size>
MutableSlice as_slice(UInt<size> &value) {
return value.as_slice();
}
template <size_t size>
bool operator<(const UInt<size> &a, const UInt<size> &b) {
return a.as_slice() < b.as_slice();
}
using UInt128 = UInt<128>;
using UInt256 = UInt<256>;
} // namespace td

303
tdutils/td/utils/Variant.h Normal file
View file

@ -0,0 +1,303 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#include "td/utils/logging.h"
#include <new>
#include <type_traits>
#include <utility>
namespace td {
namespace detail {
template <size_t... Args>
class MaxSizeImpl {};
template <class T>
constexpr const T &constexpr_max(const T &a, const T &b) {
return a < b ? b : a;
}
template <size_t Res, size_t X, size_t... Args>
class MaxSizeImpl<Res, X, Args...> {
public:
static constexpr size_t value = MaxSizeImpl<constexpr_max(Res, X), Args...>::value;
};
template <size_t Res>
class MaxSizeImpl<Res> {
public:
static constexpr size_t value = Res;
};
template <class... Args>
class MaxSize {
public:
static constexpr size_t value = MaxSizeImpl<0, sizeof(Args)...>::value;
};
template <size_t to_skip, class... Args>
class IthTypeImpl {};
template <class Res, class... Args>
class IthTypeImpl<0, Res, Args...> {
public:
using type = Res;
};
template <size_t pos, class Skip, class... Args>
class IthTypeImpl<pos, Skip, Args...> : public IthTypeImpl<pos - 1, Args...> {};
class Dummy {};
template <size_t pos, class... Args>
class IthType : public IthTypeImpl<pos, Args..., Dummy> {};
template <bool ok, int offset, class... Types>
class FindTypeOffsetImpl {};
template <int offset, class... Types>
class FindTypeOffsetImpl<true, offset, Types...> {
public:
static constexpr int value = offset;
};
template <int offset, class T, class S, class... Types>
class FindTypeOffsetImpl<false, offset, T, S, Types...>
: public FindTypeOffsetImpl<std::is_same<T, S>::value, offset + 1, T, Types...> {};
template <class T, class... Types>
class FindTypeOffset : public FindTypeOffsetImpl<false, -1, T, Types...> {};
template <int offset, class... Types>
class ForEachTypeImpl {};
template <int offset>
class ForEachTypeImpl<offset, Dummy> {
public:
template <class F>
static void visit(F &&f) {
}
};
template <int offset, class T, class... Types>
class ForEachTypeImpl<offset, T, Types...> {
public:
template <class F>
static void visit(F &&f) {
f(offset, static_cast<T *>(nullptr));
ForEachTypeImpl<offset + 1, Types...>::visit(f);
}
};
template <class... Types>
class ForEachType {
public:
template <class F>
static void visit(F &&f) {
ForEachTypeImpl<0, Types..., Dummy>::visit(f);
}
};
} // namespace detail
template <class... Types>
class Variant {
public:
static constexpr int npos = -1;
Variant() {
}
Variant(Variant &&other) noexcept {
other.visit([&](auto &&value) { this->init_empty(std::forward<decltype(value)>(value)); });
}
Variant(const Variant &other) {
other.visit([&](auto &&value) { this->init_empty(std::forward<decltype(value)>(value)); });
}
Variant &operator=(Variant &&other) {
clear();
other.visit([&](auto &&value) { this->init_empty(std::forward<decltype(value)>(value)); });
return *this;
}
Variant &operator=(const Variant &other) {
clear();
other.visit([&](auto &&value) { this->init_empty(std::forward<decltype(value)>(value)); });
return *this;
}
bool operator==(const Variant &other) const {
if (offset_ != other.offset_) {
return false;
}
bool res = false;
for_each([&](int offset, auto *ptr) {
using T = std::decay_t<decltype(*ptr)>;
if (offset == offset_) {
res = this->get<T>() == other.template get<T>();
}
});
return res;
}
bool operator<(const Variant &other) const {
if (offset_ != other.offset_) {
return offset_ < other.offset_;
}
bool res = false;
for_each([&](int offset, auto *ptr) {
using T = std::decay_t<decltype(*ptr)>;
if (offset == offset_) {
res = this->get<T>() < other.template get<T>();
}
});
return res;
}
template <class T, std::enable_if_t<!std::is_same<std::decay_t<T>, Variant>::value, int> = 0>
Variant(T &&t) {
init_empty(std::forward<T>(t));
}
template <class T, std::enable_if_t<!std::is_same<std::decay_t<T>, Variant>::value, int> = 0>
Variant &operator=(T &&t) {
clear();
init_empty(std::forward<T>(t));
return *this;
}
template <class T>
static constexpr int offset() {
return detail::FindTypeOffset<std::decay_t<T>, Types...>::value;
}
template <class T>
void init_empty(T &&t) {
LOG_CHECK(offset_ == npos) << offset_
#if TD_CLANG || TD_GCC
<< ' ' << __PRETTY_FUNCTION__
#endif
;
offset_ = offset<T>();
new (&get<T>()) std::decay_t<T>(std::forward<T>(t));
}
~Variant() {
clear();
}
template <class F>
void visit(F &&f) {
for_each([&](int offset, auto *ptr) {
using T = std::decay_t<decltype(*ptr)>;
if (offset == offset_) {
f(std::move(*this->get_unsafe<T>()));
}
});
}
template <class F>
void for_each(F &&f) {
detail::ForEachType<Types...>::visit(f);
}
template <class F>
void visit(F &&f) const {
for_each([&](int offset, auto *ptr) {
using T = std::decay_t<decltype(*ptr)>;
if (offset == offset_) {
f(std::move(*this->get_unsafe<T>()));
}
});
}
template <class F>
void for_each(F &&f) const {
detail::ForEachType<Types...>::visit(f);
}
void clear() {
visit([](auto &&value) {
using T = std::decay_t<decltype(value)>;
value.~T();
});
offset_ = npos;
}
template <int offset>
auto &get() {
CHECK(offset == offset_);
return *get_unsafe<offset>();
}
template <class T>
auto &get() {
return get<offset<T>()>();
}
template <int offset>
const auto &get() const {
CHECK(offset == offset_);
return *get_unsafe<offset>();
}
template <class T>
const auto &get() const {
return get<offset<T>()>();
}
int32 get_offset() const {
return offset_;
}
private:
union {
int64 align_;
char data_[detail::MaxSize<Types...>::value];
};
int offset_{npos};
template <class T>
auto *get_unsafe() {
return reinterpret_cast<T *>(data_);
}
template <int offset>
auto *get_unsafe() {
using T = typename detail::IthType<offset, Types...>::type;
return get_unsafe<T>();
}
template <class T>
const auto *get_unsafe() const {
return reinterpret_cast<const T *>(data_);
}
template <int offset>
const auto *get_unsafe() const {
using T = typename detail::IthType<offset, Types...>::type;
return get_unsafe<T>();
}
};
template <class T, class... Types>
auto &get(Variant<Types...> &v) {
return v.template get<T>();
}
template <class T, class... Types>
auto &get(const Variant<Types...> &v) {
return v.template get<T>();
}
template <int T, class... Types>
auto &get(Variant<Types...> &v) {
return v.template get<T>();
}
template <int T, class... Types>
auto &get(const Variant<Types...> &v) {
return v.template get<T>();
}
} // namespace td

84
tdutils/td/utils/VectorQueue.h Normal file
View file

@ -0,0 +1,84 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/Span.h"
#include <utility>
#include <vector>
namespace td {
template <class T>
class VectorQueue {
public:
template <class S>
void push(S &&s) {
vector_.push_back(std::forward<S>(s));
}
template <class... Args>
void emplace(Args &&... args) {
vector_.emplace_back(std::forward<Args>(args)...);
}
T pop() {
try_shrink();
return std::move(vector_[read_pos_++]);
}
void pop_n(size_t n) {
read_pos_ += n;
try_shrink();
}
T &front() {
return vector_[read_pos_];
}
T &back() {
return vector_.back();
}
bool empty() const {
return size() == 0;
}
size_t size() const {
return vector_.size() - read_pos_;
}
T *data() {
return vector_.data() + read_pos_;
}
const T *data() const {
return vector_.data() + read_pos_;
}
Span<T> as_span() const {
return {data(), size()};
}
MutableSpan<T> as_mutable_span() {
return {data(), size()};
}
private:
std::vector<T> vector_;
size_t read_pos_{0};
void try_shrink() {
if (read_pos_ * 2 > vector_.size() && read_pos_ > 4) {
vector_.erase(vector_.begin(), vector_.begin() + read_pos_);
read_pos_ = 0;
}
}
};
} // namespace td

98
tdutils/td/utils/as.h Normal file
View file

@ -0,0 +1,98 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include <cstring>
#include <type_traits>
namespace td {
namespace detail {
template <class T>
class As {
public:
explicit As(void *ptr) : ptr_(ptr) {
}
As(const As &new_value) = delete;
As &operator=(const As &) = delete;
As(As &&) = default;
As &operator=(As &&new_value) && {
std::memcpy(ptr_, new_value.ptr_, sizeof(T));
return *this;
}
~As() = default;
As &operator=(T new_value) && {
std::memcpy(ptr_, &new_value, sizeof(T));
return *this;
}
operator T() const {
T res;
std::memcpy(&res, ptr_, sizeof(T));
return res;
}
bool operator==(const As &other) const {
return this->operator T() == other.operator T();
}
private:
void *ptr_;
};
template <class T>
class ConstAs {
public:
explicit ConstAs(const void *ptr) : ptr_(ptr) {
}
operator T() const {
T res;
std::memcpy(&res, ptr_, sizeof(T));
return res;
}
private:
const void *ptr_;
};
} // namespace detail
// no std::is_trivially_copyable in libstdc++ before 5.0
#if __GLIBCXX__
#define TD_IS_TRIVIALLY_COPYABLE(T) __has_trivial_copy(T)
#else
#define TD_IS_TRIVIALLY_COPYABLE(T) std::is_trivially_copyable<T>::value
#endif
template <class ToT, class FromT,
std::enable_if_t<TD_IS_TRIVIALLY_COPYABLE(ToT) && TD_IS_TRIVIALLY_COPYABLE(FromT), int> = 0>
detail::As<ToT> as(FromT *from) {
return detail::As<ToT>(from);
}
template <class ToT, class FromT,
std::enable_if_t<TD_IS_TRIVIALLY_COPYABLE(ToT) && TD_IS_TRIVIALLY_COPYABLE(FromT), int> = 0>
const detail::ConstAs<ToT> as(const FromT *from) {
return detail::ConstAs<ToT>(from);
}
} // namespace td

302
tdutils/td/utils/base64.cpp Normal file
View file

@ -0,0 +1,302 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/utils/base64.h"
#include "td/utils/common.h"
#include "td/utils/Slice.h"
#include "td/utils/Status.h"
#include "td/utils/format.h"
#include <algorithm>
#include <iterator>
namespace td {
//TODO: fix copypaste
static const char *const symbols64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
string base64_encode(Slice input) {
string base64;
base64.reserve((input.size() + 2) / 3 * 4);
for (size_t i = 0; i < input.size();) {
size_t left = min(input.size() - i, static_cast<size_t>(3));
int c = input.ubegin()[i++] << 16;
base64 += symbols64[c >> 18];
if (left != 1) {
c |= input.ubegin()[i++] << 8;
}
base64 += symbols64[(c >> 12) & 63];
if (left == 3) {
c |= input.ubegin()[i++];
}
if (left != 1) {
base64 += symbols64[(c >> 6) & 63];
} else {
base64 += '=';
}
if (left == 3) {
base64 += symbols64[c & 63];
} else {
base64 += '=';
}
}
return base64;
}
static unsigned char char_to_value[256];
static void init_base64_table() {
static bool is_inited = [] {
std::fill(std::begin(char_to_value), std::end(char_to_value), static_cast<unsigned char>(64));
for (unsigned char i = 0; i < 64; i++) {
char_to_value[static_cast<size_t>(symbols64[i])] = i;
}
return true;
}();
CHECK(is_inited);
}
Result<Slice> base64_drop_padding(Slice base64) {
if ((base64.size() & 3) != 0) {
return Status::Error("Wrong string length");
}
size_t padding_length = 0;
while (!base64.empty() && base64.back() == '=') {
base64.remove_suffix(1);
padding_length++;
}
if (padding_length >= 3) {
return Status::Error("Wrong string padding");
}
return base64;
}
template <class F>
Status base64_do_decode(Slice base64, F &&append) {
for (size_t i = 0; i < base64.size();) {
size_t left = min(base64.size() - i, static_cast<size_t>(4));
int c = 0;
for (size_t t = 0; t < left; t++) {
auto value = char_to_value[base64.ubegin()[i++]];
if (value == 64) {
return Status::Error("Wrong character in the string");
}
c |= value << ((3 - t) * 6);
}
append(static_cast<char>(static_cast<unsigned char>(c >> 16))); // implementation-defined
if (left == 2) {
if ((c & ((1 << 16) - 1)) != 0) {
return Status::Error("Wrong padding in the string");
}
} else {
append(static_cast<char>(static_cast<unsigned char>(c >> 8))); // implementation-defined
if (left == 3) {
if ((c & ((1 << 8) - 1)) != 0) {
return Status::Error("Wrong padding in the string");
}
} else {
append(static_cast<char>(static_cast<unsigned char>(c))); // implementation-defined
}
}
}
return Status::OK();
}
Result<string> base64_decode(Slice base64) {
init_base64_table();
TRY_RESULT(tmp, base64_drop_padding(base64));
base64 = tmp;
string output;
output.reserve(((base64.size() + 3) >> 2) * 3);
TRY_STATUS(base64_do_decode(base64, [&output](char c) { output += c; }));
return output;
}
Result<SecureString> base64_decode_secure(Slice base64) {
init_base64_table();
TRY_RESULT(tmp, base64_drop_padding(base64));
base64 = tmp;
SecureString output(((base64.size() + 3) >> 2) * 3);
char *ptr = output.as_mutable_slice().begin();
TRY_STATUS(base64_do_decode(base64, [&ptr](char c) { *ptr++ = c; }));
size_t size = ptr - output.as_mutable_slice().begin();
if (size == output.size()) {
return std::move(output);
}
return SecureString(output.as_slice().substr(0, size));
}
static const char *const url_symbols64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_";
string base64url_encode(Slice input) {
string base64;
base64.reserve((input.size() + 2) / 3 * 4);
for (size_t i = 0; i < input.size();) {
size_t left = min(input.size() - i, static_cast<size_t>(3));
int c = input.ubegin()[i++] << 16;
base64 += url_symbols64[c >> 18];
if (left != 1) {
c |= input.ubegin()[i++] << 8;
}
base64 += url_symbols64[(c >> 12) & 63];
if (left == 3) {
c |= input.ubegin()[i++];
}
if (left != 1) {
base64 += url_symbols64[(c >> 6) & 63];
}
if (left == 3) {
base64 += url_symbols64[c & 63];
}
}
return base64;
}
static unsigned char url_char_to_value[256];
static void init_base64url_table() {
static bool is_inited = [] {
std::fill(std::begin(url_char_to_value), std::end(url_char_to_value), static_cast<unsigned char>(64));
for (unsigned char i = 0; i < 64; i++) {
url_char_to_value[static_cast<size_t>(url_symbols64[i])] = i;
}
return true;
}();
CHECK(is_inited);
}
Result<string> base64url_decode(Slice base64) {
init_base64url_table();
size_t padding_length = 0;
while (!base64.empty() && base64.back() == '=') {
base64.remove_suffix(1);
padding_length++;
}
if (padding_length >= 3 || (padding_length > 0 && ((base64.size() + padding_length) & 3) != 0)) {
return Status::Error("Wrong string padding");
}
if ((base64.size() & 3) == 1) {
return Status::Error("Wrong string length");
}
string output;
output.reserve(((base64.size() + 3) >> 2) * 3);
for (size_t i = 0; i < base64.size();) {
size_t left = min(base64.size() - i, static_cast<size_t>(4));
int c = 0;
for (size_t t = 0; t < left; t++) {
auto value = url_char_to_value[base64.ubegin()[i++]];
if (value == 64) {
return Status::Error("Wrong character in the string");
}
c |= value << ((3 - t) * 6);
}
output += static_cast<char>(static_cast<unsigned char>(c >> 16)); // implementation-defined
if (left == 2) {
if ((c & ((1 << 16) - 1)) != 0) {
return Status::Error("Wrong padding in the string");
}
} else {
output += static_cast<char>(static_cast<unsigned char>(c >> 8)); // implementation-defined
if (left == 3) {
if ((c & ((1 << 8) - 1)) != 0) {
return Status::Error("Wrong padding in the string");
}
} else {
output += static_cast<char>(static_cast<unsigned char>(c)); // implementation-defined
}
}
}
return output;
}
template <bool is_url>
static bool is_base64_impl(Slice input) {
size_t padding_length = 0;
while (!input.empty() && input.back() == '=') {
input.remove_suffix(1);
padding_length++;
}
if (padding_length >= 3) {
return false;
}
if ((!is_url || padding_length > 0) && ((input.size() + padding_length) & 3) != 0) {
return false;
}
if (is_url && (input.size() & 3) == 1) {
return false;
}
unsigned char *table;
if (is_url) {
init_base64url_table();
table = url_char_to_value;
} else {
init_base64_table();
table = char_to_value;
}
for (auto c : input) {
if (table[static_cast<unsigned char>(c)] == 64) {
return false;
}
}
if ((input.size() & 3) == 2) {
auto value = table[static_cast<int>(input.back())];
if ((value & 15) != 0) {
return false;
}
}
if ((input.size() & 3) == 3) {
auto value = table[static_cast<int>(input.back())];
if ((value & 3) != 0) {
return false;
}
}
return true;
}
bool is_base64(Slice input) {
return is_base64_impl<false>(input);
}
bool is_base64url(Slice input) {
return is_base64_impl<true>(input);
}
string base64_filter(Slice input) {
string res;
res.reserve(input.size());
init_base64_table();
for (auto c : input) {
if (char_to_value[static_cast<unsigned char>(c)] != 64 || c == '=') {
res += c;
}
}
return res;
}
} // namespace td

40
tdutils/td/utils/base64.h Normal file
View file

@ -0,0 +1,40 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#include "td/utils/Slice.h"
#include "td/utils/SharedSlice.h"
#include "td/utils/Status.h"
namespace td {
string base64_encode(Slice input);
Result<string> base64_decode(Slice base64);
Result<SecureString> base64_decode_secure(Slice base64);
string base64url_encode(Slice input);
Result<string> base64url_decode(Slice base64);
bool is_base64(Slice input);
bool is_base64url(Slice input);
string base64_filter(Slice input);
} // namespace td

144
tdutils/td/utils/benchmark.h Normal file
View file

@ -0,0 +1,144 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/format.h"
#include "td/utils/logging.h"
#include "td/utils/port/Clocks.h"
#include "td/utils/StringBuilder.h"
#include <cmath>
#include <tuple>
#include <utility>
#define BENCH(name, desc) \
class name##Bench : public ::td::Benchmark { \
public: \
std::string get_description() const override { \
return (desc); \
} \
void run(int n) override; \
}; \
void name##Bench::run(int n)
namespace td {
#if TD_MSVC
#pragma optimize("", off)
template <class T>
void do_not_optimize_away(T &&datum) {
datum = datum;
}
#pragma optimize("", on)
#else
template <class T>
void do_not_optimize_away(T &&datum) {
asm volatile("" : "+r"(datum));
}
#endif
class Benchmark {
public:
Benchmark() = default;
Benchmark(const Benchmark &) = delete;
Benchmark &operator=(const Benchmark &) = delete;
Benchmark(Benchmark &&) = delete;
Benchmark &operator=(Benchmark &&) = delete;
virtual ~Benchmark() = default;
virtual std::string get_description() const = 0;
virtual void start_up() {
}
virtual void start_up_n(int n) {
start_up();
}
virtual void tear_down() {
}
virtual void run(int n) = 0;
};
inline std::pair<double, double> bench_n(Benchmark &b, int n) {
double total = -Clocks::monotonic();
b.start_up_n(n);
double t = -Clocks::monotonic();
b.run(n);
t += Clocks::monotonic();
b.tear_down();
total += Clocks::monotonic();
return std::make_pair(t, total);
}
inline std::pair<double, double> bench_n(Benchmark &&b, int n) {
return bench_n(b, n);
}
inline void bench(Benchmark &b, double max_time = 1.0) {
int n = 1;
double pass_time = 0;
double total_pass_time = 0;
while (pass_time < max_time && total_pass_time < max_time * 3 && n < (1 << 30)) {
n *= 2;
std::tie(pass_time, total_pass_time) = bench_n(b, n);
}
pass_time = n / pass_time;
int pass_cnt = 2;
double sum = pass_time;
double square_sum = pass_time * pass_time;
double min_pass_time = pass_time;
double max_pass_time = pass_time;
for (int i = 1; i < pass_cnt; i++) {
pass_time = n / bench_n(b, n).first;
sum += pass_time;
square_sum += pass_time * pass_time;
if (pass_time < min_pass_time) {
min_pass_time = pass_time;
}
if (pass_time > max_pass_time) {
max_pass_time = pass_time;
}
}
double average = sum / pass_cnt;
double d = sqrt(square_sum / pass_cnt - average * average);
auto description = b.get_description();
std::string pad;
if (description.size() < 40) {
pad = std::string(40 - description.size(), ' ');
}
LOG(ERROR) << "Bench [" << pad << description << "]: " << StringBuilder::FixedDouble(average, 3) << '['
<< StringBuilder::FixedDouble(min_pass_time, 3) << '-' << StringBuilder::FixedDouble(max_pass_time, 3)
<< "] ops/sec,\t" << format::as_time(1 / average) << " [d = " << StringBuilder::FixedDouble(d, 6) << ']';
}
inline void bench(Benchmark &&b, double max_time = 1.0) {
bench(b, max_time);
}
} // namespace td

267
tdutils/td/utils/bits.h Normal file
View file

@ -0,0 +1,267 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#if TD_MSVC
#include <intrin.h>
#endif
#ifdef bswap32
#undef bswap32
#endif
#ifdef bswap64
#undef bswap64
#endif
namespace td {
int32 count_leading_zeroes32(uint32 x);
int32 count_leading_zeroes64(uint64 x);
int32 count_trailing_zeroes32(uint32 x);
int32 count_trailing_zeroes64(uint64 x);
uint32 bswap32(uint32 x);
uint64 bswap64(uint64 x);
int32 count_bits32(uint32 x);
int32 count_bits64(uint64 x);
inline uint32 bits_negate32(uint32 x) {
return ~x + 1;
}
inline uint64 bits_negate64(uint64 x) {
return ~x + 1;
}
inline uint32 lower_bit32(uint32 x) {
return x & bits_negate32(x);
}
inline uint64 lower_bit64(uint64 x) {
return x & bits_negate64(x);
}
//TODO: optimize
inline int32 count_leading_zeroes_non_zero32(uint32 x) {
DCHECK(x != 0);
return count_leading_zeroes32(x);
}
inline int32 count_leading_zeroes_non_zero64(uint64 x) {
DCHECK(x != 0);
return count_leading_zeroes64(x);
}
inline int32 count_trailing_zeroes_non_zero32(uint32 x) {
DCHECK(x != 0);
return count_trailing_zeroes32(x);
}
inline int32 count_trailing_zeroes_non_zero64(uint64 x) {
DCHECK(x != 0);
return count_trailing_zeroes64(x);
}
//
// Platform specific implementation
//
#if TD_MSVC
inline int32 count_leading_zeroes32(uint32 x) {
unsigned long res = 0;
if (_BitScanReverse(&res, x)) {
return 31 - res;
}
return 32;
}
inline int32 count_leading_zeroes64(uint64 x) {
#if defined(_M_X64)
unsigned long res = 0;
if (_BitScanReverse64(&res, x)) {
return 63 - res;
}
return 64;
#else
if ((x >> 32) == 0) {
return count_leading_zeroes32(static_cast<uint32>(x)) + 32;
} else {
return count_leading_zeroes32(static_cast<uint32>(x >> 32));
}
#endif
}
inline int32 count_trailing_zeroes32(uint32 x) {
unsigned long res = 0;
if (_BitScanForward(&res, x)) {
return res;
}
return 32;
}
inline int32 count_trailing_zeroes64(uint64 x) {
#if defined(_M_X64)
unsigned long res = 0;
if (_BitScanForward64(&res, x)) {
return res;
}
return 64;
#else
if (static_cast<uint32>(x) == 0) {
return count_trailing_zeroes32(static_cast<uint32>(x >> 32)) + 32;
} else {
return count_trailing_zeroes32(static_cast<uint32>(x));
}
#endif
}
inline uint32 bswap32(uint32 x) {
return _byteswap_ulong(x);
}
inline uint64 bswap64(uint64 x) {
return _byteswap_uint64(x);
}
inline int32 count_bits32(uint32 x) {
return __popcnt(x);
}
inline int32 count_bits64(uint64 x) {
#if defined(_M_X64)
return static_cast<int32>(__popcnt64(x));
#else
return count_bits32(static_cast<uint32>(x >> 32)) + count_bits32(static_cast<uint32>(x));
#endif
}
#elif TD_INTEL
inline int32 count_leading_zeroes32(uint32 x) {
unsigned __int32 res = 0;
if (_BitScanReverse(&res, x)) {
return 31 - res;
}
return 32;
}
inline int32 count_leading_zeroes64(uint64 x) {
#if defined(_M_X64) || defined(__x86_64__)
unsigned __int32 res = 0;
if (_BitScanReverse64(&res, x)) {
return 63 - res;
}
return 64;
#else
if ((x >> 32) == 0) {
return count_leading_zeroes32(static_cast<uint32>(x)) + 32;
} else {
return count_leading_zeroes32(static_cast<uint32>(x >> 32));
}
#endif
}
inline int32 count_trailing_zeroes32(uint32 x) {
unsigned __int32 res = 0;
if (_BitScanForward(&res, x)) {
return res;
}
return 32;
}
inline int32 count_trailing_zeroes64(uint64 x) {
#if defined(_M_X64) || defined(__x86_64__)
unsigned __int32 res = 0;
if (_BitScanForward64(&res, x)) {
return res;
}
return 64;
#else
if (static_cast<uint32>(x) == 0) {
return count_trailing_zeroes32(static_cast<uint32>(x >> 32)) + 32;
} else {
return count_trailing_zeroes32(static_cast<uint32>(x));
}
#endif
}
inline uint32 bswap32(uint32 x) {
return _bswap(static_cast<int>(x));
}
inline uint64 bswap64(uint64 x) {
return _bswap64(static_cast<__int64>(x));
}
inline int32 count_bits32(uint32 x) {
return _popcnt32(static_cast<int>(x));
}
inline int32 count_bits64(uint64 x) {
return _popcnt64(static_cast<__int64>(x));
}
#else
inline int32 count_leading_zeroes32(uint32 x) {
if (x == 0) {
return 32;
}
return __builtin_clz(x);
}
inline int32 count_leading_zeroes64(uint64 x) {
if (x == 0) {
return 64;
}
return __builtin_clzll(x);
}
inline int32 count_trailing_zeroes32(uint32 x) {
if (x == 0) {
return 32;
}
return __builtin_ctz(x);
}
inline int32 count_trailing_zeroes64(uint64 x) {
if (x == 0) {
return 64;
}
return __builtin_ctzll(x);
}
inline uint32 bswap32(uint32 x) {
return __builtin_bswap32(x);
}
inline uint64 bswap64(uint64 x) {
return __builtin_bswap64(x);
}
inline int32 count_bits32(uint32 x) {
return __builtin_popcount(x);
}
inline int32 count_bits64(uint64 x) {
return __builtin_popcountll(x);
}
#endif
} // namespace td

196
tdutils/td/utils/buffer.cpp Normal file
View file

@ -0,0 +1,196 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#include "td/utils/buffer.h"
#include "td/utils/port/thread_local.h"
#include <cstddef>
#include <new>
// fixes https://bugs.llvm.org/show_bug.cgi?id=33723 for clang >= 3.6 + c++11 + libc++
#if TD_CLANG && _LIBCPP_VERSION
#define TD_OFFSETOF __builtin_offsetof
#else
#define TD_OFFSETOF offsetof
#endif
namespace td {
TD_THREAD_LOCAL BufferAllocator::BufferRawTls *BufferAllocator::buffer_raw_tls; // static zero-initialized
std::atomic<size_t> BufferAllocator::buffer_mem;
size_t BufferAllocator::get_buffer_mem() {
return buffer_mem;
}
BufferAllocator::WriterPtr BufferAllocator::create_writer(size_t size) {
if (size < 512) {
size = 512;
}
return create_writer_exact(size);
}
BufferAllocator::WriterPtr BufferAllocator::create_writer_exact(size_t size) {
return WriterPtr(create_buffer_raw(size));
}
BufferAllocator::WriterPtr BufferAllocator::create_writer(size_t size, size_t prepend, size_t append) {
auto ptr = create_writer(size + prepend + append);
ptr->begin_ += prepend;
ptr->end_ += prepend + size;
return ptr;
}
BufferAllocator::ReaderPtr BufferAllocator::create_reader(size_t size) {
if (size < 512) {
return create_reader_fast(size);
}
auto ptr = create_writer_exact(size);
ptr->end_ += (size + 7) & -8;
return create_reader(ptr);
}
BufferAllocator::ReaderPtr BufferAllocator::create_reader_fast(size_t size) {
size = (size + 7) & -8;
init_thread_local<BufferRawTls>(buffer_raw_tls);
auto buffer_raw = buffer_raw_tls->buffer_raw.get();
if (buffer_raw == nullptr || buffer_raw->data_size_ - buffer_raw->end_.load(std::memory_order_relaxed) < size) {
buffer_raw = create_buffer_raw(4096 * 4);
buffer_raw_tls->buffer_raw = std::unique_ptr<BufferRaw, BufferAllocator::BufferRawDeleter>(buffer_raw);
}
buffer_raw->end_.fetch_add(size, std::memory_order_relaxed);
buffer_raw->ref_cnt_.fetch_add(1, std::memory_order_acq_rel);
return ReaderPtr(buffer_raw);
}
BufferAllocator::ReaderPtr BufferAllocator::create_reader(const WriterPtr &raw) {
raw->was_reader_ = true;
raw->ref_cnt_.fetch_add(1, std::memory_order_acq_rel);
return ReaderPtr(raw.get());
}
BufferAllocator::ReaderPtr BufferAllocator::create_reader(const ReaderPtr &raw) {
raw->ref_cnt_.fetch_add(1, std::memory_order_acq_rel);
return ReaderPtr(raw.get());
}
void BufferAllocator::dec_ref_cnt(BufferRaw *ptr) {
int left = ptr->ref_cnt_.fetch_sub(1, std::memory_order_acq_rel);
if (left == 1) {
auto buf_size = max(sizeof(BufferRaw), TD_OFFSETOF(BufferRaw, data_) + ptr->data_size_);
buffer_mem -= buf_size;
ptr->~BufferRaw();
delete[] ptr;
}
}
BufferRaw *BufferAllocator::create_buffer_raw(size_t size) {
size = (size + 7) & -8;
auto buf_size = TD_OFFSETOF(BufferRaw, data_) + size;
if (buf_size < sizeof(BufferRaw)) {
buf_size = sizeof(BufferRaw);
}
buffer_mem += buf_size;
auto *buffer_raw = reinterpret_cast<BufferRaw *>(new char[buf_size]);
return new (buffer_raw) BufferRaw(size);
}
void BufferBuilder::append(BufferSlice slice) {
if (append_inplace(slice.as_slice())) {
return;
}
append_slow(std::move(slice));
}
void BufferBuilder::append(Slice slice) {
if (append_inplace(slice)) {
return;
}
append_slow(BufferSlice(slice));
}
void BufferBuilder::prepend(BufferSlice slice) {
if (prepend_inplace(slice.as_slice())) {
return;
}
prepend_slow(std::move(slice));
}
void BufferBuilder::prepend(Slice slice) {
if (prepend_inplace(slice)) {
return;
}
prepend_slow(BufferSlice(slice));
}
BufferSlice BufferBuilder::extract() {
if (to_append_.empty() && to_prepend_.empty()) {
return buffer_writer_.as_buffer_slice();
}
size_t total_size = size();
BufferWriter writer(0, 0, total_size);
std::move(*this).for_each([&](auto &&slice) {
writer.prepare_append().truncate(slice.size()).copy_from(slice.as_slice());
writer.confirm_append(slice.size());
});
*this = {};
return writer.as_buffer_slice();
}
size_t BufferBuilder::size() const {
size_t total_size = 0;
for_each([&](auto &&slice) { total_size += slice.size(); });
return total_size;
}
bool BufferBuilder::append_inplace(Slice slice) {
if (!to_append_.empty()) {
return false;
}
auto dest = buffer_writer_.prepare_append();
if (dest.size() < slice.size()) {
return false;
}
dest.remove_suffix(dest.size() - slice.size());
dest.copy_from(slice);
buffer_writer_.confirm_append(slice.size());
return true;
}
void BufferBuilder::append_slow(BufferSlice slice) {
to_append_.push_back(std::move(slice));
}
bool BufferBuilder::prepend_inplace(Slice slice) {
if (!to_prepend_.empty()) {
return false;
}
auto dest = buffer_writer_.prepare_prepend();
if (dest.size() < slice.size()) {
return false;
}
dest.remove_prefix(dest.size() - slice.size());
dest.copy_from(slice);
buffer_writer_.confirm_prepend(slice.size());
return true;
}
void BufferBuilder::prepend_slow(BufferSlice slice) {
to_prepend_.push_back(std::move(slice));
}
} // namespace td

815
tdutils/td/utils/buffer.h Normal file
View file

@ -0,0 +1,815 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2019 Telegram Systems LLP
*/
#pragma once
#include "td/utils/common.h"
#include "td/utils/logging.h"
#include "td/utils/misc.h"
#include "td/utils/port/thread_local.h"
#include "td/utils/Slice.h"
#include <atomic>
#include <limits>
#include <memory>
namespace td {
struct BufferRaw {
explicit BufferRaw(size_t size) : data_size_(size) {
}
size_t data_size_;
// Constant after first reader is created.
// May be change by writer before it.
// So writer may do prepends till there is no reader created.
size_t begin_ = 0;
// Write by writer.
// Read by reader.
std::atomic<size_t> end_{0};
mutable std::atomic<int32> ref_cnt_{1};
std::atomic<bool> has_writer_{true};
bool was_reader_{false};
alignas(4) unsigned char data_[1];
};
class BufferAllocator {
public:
class DeleteWriterPtr {
public:
void operator()(BufferRaw *ptr) {
ptr->has_writer_.store(false, std::memory_order_release);
dec_ref_cnt(ptr);
}
};
class DeleteReaderPtr {
public:
void operator()(BufferRaw *ptr) {
dec_ref_cnt(ptr);
}
};
using WriterPtr = std::unique_ptr<BufferRaw, DeleteWriterPtr>;
using ReaderPtr = std::unique_ptr<BufferRaw, DeleteReaderPtr>;
static WriterPtr create_writer(size_t size);
static WriterPtr create_writer(size_t size, size_t prepend, size_t append);
static ReaderPtr create_reader(size_t size);
static ReaderPtr create_reader(const WriterPtr &raw);
static ReaderPtr create_reader(const ReaderPtr &raw);
static size_t get_buffer_mem();
static void clear_thread_local();
private:
static ReaderPtr create_reader_fast(size_t size);
static WriterPtr create_writer_exact(size_t size);
struct BufferRawDeleter {
void operator()(BufferRaw *ptr) {
dec_ref_cnt(ptr);
}
};
struct BufferRawTls {
std::unique_ptr<BufferRaw, BufferRawDeleter> buffer_raw;
};
static TD_THREAD_LOCAL BufferRawTls *buffer_raw_tls;
static void dec_ref_cnt(BufferRaw *ptr);
static BufferRaw *create_buffer_raw(size_t size);
static std::atomic<size_t> buffer_mem;
};
using BufferWriterPtr = BufferAllocator::WriterPtr;
using BufferReaderPtr = BufferAllocator::ReaderPtr;
class BufferSlice {
public:
BufferSlice() = default;
explicit BufferSlice(BufferReaderPtr buffer_ptr) : buffer_(std::move(buffer_ptr)) {
if (is_null()) {
return;
}
begin_ = buffer_->begin_;
sync_with_writer();
}
BufferSlice(BufferReaderPtr buffer_ptr, size_t begin, size_t end)
: buffer_(std::move(buffer_ptr)), begin_(begin), end_(end) {
}
explicit BufferSlice(size_t size) : buffer_(BufferAllocator::create_reader(size)) {
end_ = buffer_->end_.load(std::memory_order_relaxed);
begin_ = end_ - ((size + 7) & -8);
end_ = begin_ + size;
}
explicit BufferSlice(Slice slice) : BufferSlice(slice.size()) {
as_slice().copy_from(slice);
}
BufferSlice(const char *ptr, size_t size) : BufferSlice(Slice(ptr, size)) {
}
BufferSlice clone() const {
if (is_null()) {
return BufferSlice(BufferReaderPtr(), begin_, end_);
}
return BufferSlice(BufferAllocator::create_reader(buffer_), begin_, end_);
}
BufferSlice copy() const {
if (is_null()) {
return BufferSlice(BufferReaderPtr(), begin_, end_);
}
return BufferSlice(as_slice());
}
Slice as_slice() const {
if (is_null()) {
return Slice();
}
return Slice(buffer_->data_ + begin_, size());
}
operator Slice() const {
return as_slice();
}
MutableSlice as_slice() {
if (is_null()) {
return MutableSlice();
}
return MutableSlice(buffer_->data_ + begin_, size());
}
Slice prepare_read() const {
return as_slice();
}
Slice after(size_t offset) const {
auto full = as_slice();
full.remove_prefix(offset);
return full;
}
bool confirm_read(size_t size) {
begin_ += size;
CHECK(begin_ <= end_);
return begin_ == end_;
}
void truncate(size_t limit) {
if (size() > limit) {
end_ = begin_ + limit;
}
}
BufferSlice from_slice(Slice slice) const {
auto res = BufferSlice(BufferAllocator::create_reader(buffer_));
res.begin_ = static_cast<size_t>(slice.ubegin() - buffer_->data_);
res.end_ = static_cast<size_t>(slice.uend() - buffer_->data_);
CHECK(buffer_->begin_ <= res.begin_);
CHECK(res.begin_ <= res.end_);
CHECK(res.end_ <= buffer_->end_.load(std::memory_order_relaxed));
return res;
}
// like in std::string
char *data() {
return as_slice().data();
}
const char *data() const {
return as_slice().data();
}
char operator[](size_t at) const {
return as_slice()[at];
}
bool empty() const {
return size() == 0;
}
bool is_null() const {
return !buffer_;
}
size_t size() const {
if (is_null()) {
return 0;
}
return end_ - begin_;
}
// like in std::string
size_t length() const {
return size();
}
// set end_ into writer's end_
size_t sync_with_writer() {
CHECK(!is_null());
auto old_end = end_;
end_ = buffer_->end_.load(std::memory_order_acquire);
return end_ - old_end;
}
bool is_writer_alive() const {
CHECK(!is_null());
return buffer_->has_writer_.load(std::memory_order_acquire);
}
void clear() {
begin_ = 0;
end_ = 0;
buffer_ = nullptr;
}
private:
BufferReaderPtr buffer_;
size_t begin_ = 0;
size_t end_ = 0;
};
template <class StorerT>
void store(const BufferSlice &buffer_slice, StorerT &storer) {
storer.store_string(buffer_slice);
}
template <class ParserT>
void parse(BufferSlice &buffer_slice, ParserT &parser) {
buffer_slice = parser.template fetch_string<BufferSlice>();
}
class BufferWriter {
public:
BufferWriter() = default;
explicit BufferWriter(size_t size) : BufferWriter(BufferAllocator::create_writer(size)) {
}
BufferWriter(size_t size, size_t prepend, size_t append)
: BufferWriter(BufferAllocator::create_writer(size, prepend, append)) {
}
BufferWriter(Slice slice, size_t prepend, size_t append)
: BufferWriter(BufferAllocator::create_writer(slice.size(), prepend, append)) {
as_slice().copy_from(slice);
}
explicit BufferWriter(BufferWriterPtr buffer_ptr) : buffer_(std::move(buffer_ptr)) {
}
BufferSlice as_buffer_slice() const {
return BufferSlice(BufferAllocator::create_reader(buffer_));
}
bool is_null() const {
return !buffer_;
}
bool empty() const {
return size() == 0;
}
size_t size() const {
if (is_null()) {
return 0;
}
return buffer_->end_.load(std::memory_order_relaxed) - buffer_->begin_;
}
MutableSlice as_slice() {
auto end = buffer_->end_.load(std::memory_order_relaxed);
return MutableSlice(buffer_->data_ + buffer_->begin_, buffer_->data_ + end);
}
Slice as_slice() const {
auto end = buffer_->end_.load(std::memory_order_relaxed);
return Slice(buffer_->data_ + buffer_->begin_, buffer_->data_ + end);
}
MutableSlice prepare_prepend() {
if (is_null()) {
return MutableSlice();
}
CHECK(!buffer_->was_reader_);
return MutableSlice(buffer_->data_, buffer_->begin_);
}
MutableSlice prepare_append() {
if (is_null()) {
return MutableSlice();
}
auto end = buffer_->end_.load(std::memory_order_relaxed);
return MutableSlice(buffer_->data_ + end, buffer_->data_size_ - end);
}
void confirm_append(size_t size) {
if (is_null()) {
CHECK(size == 0);
return;
}
auto new_end = buffer_->end_.load(std::memory_order_relaxed) + size;
CHECK(new_end <= buffer_->data_size_);
buffer_->end_.store(new_end, std::memory_order_release);
}
void confirm_prepend(size_t size) {
if (is_null()) {
CHECK(size == 0);
return;
}
CHECK(buffer_->begin_ >= size);
buffer_->begin_ -= size;
}
private:
BufferWriterPtr buffer_;
};
struct ChainBufferNode {
friend struct DeleteWriterPtr;
struct DeleteWriterPtr {
void operator()(ChainBufferNode *ptr) {
ptr->has_writer_.store(false, std::memory_order_release);
dec_ref_cnt(ptr);
}
};
friend struct DeleteReaderPtr;
struct DeleteReaderPtr {
void operator()(ChainBufferNode *ptr) {
dec_ref_cnt(ptr);
}
};
using WriterPtr = std::unique_ptr<ChainBufferNode, DeleteWriterPtr>;
using ReaderPtr = std::unique_ptr<ChainBufferNode, DeleteReaderPtr>;
static WriterPtr make_writer_ptr(ChainBufferNode *ptr) {
ptr->ref_cnt_.store(1, std::memory_order_relaxed);
ptr->has_writer_.store(true, std::memory_order_relaxed);
return WriterPtr(ptr);
}
static ReaderPtr make_reader_ptr(ChainBufferNode *ptr) {
ptr->ref_cnt_.fetch_add(1, std::memory_order_acq_rel);
return ReaderPtr(ptr);
}
bool has_writer() {
return has_writer_.load(std::memory_order_acquire);
}
bool unique() {
return ref_cnt_.load(std::memory_order_acquire) == 1;
}
ChainBufferNode(BufferSlice slice, bool sync_flag) : slice_(std::move(slice)), sync_flag_(sync_flag) {
}
// reader
// There are two options
// 1. Fixed slice of Buffer
// 2. Slice with non-fixed right end
// In each case slice_ is const. Reader should read it and use sync_with_writer on its own copy.
const BufferSlice slice_;
const bool sync_flag_{false}; // should we call slice_.sync_with_writer or not.
// writer
ReaderPtr next_{nullptr};
private:
std::atomic<int> ref_cnt_{0};
std::atomic<bool> has_writer_{false};
static void clear_nonrecursive(ReaderPtr ptr) {
while (ptr && ptr->unique()) {
ptr = std::move(ptr->next_);
}
}
static void dec_ref_cnt(ChainBufferNode *ptr) {
int left = --ptr->ref_cnt_;
if (left == 0) {
clear_nonrecursive(std::move(ptr->next_));
// TODO(refact): move memory management into allocator (?)
delete ptr;
}
}
};
using ChainBufferNodeWriterPtr = ChainBufferNode::WriterPtr;
using ChainBufferNodeReaderPtr = ChainBufferNode::ReaderPtr;
class ChainBufferNodeAllocator {
public:
static ChainBufferNodeWriterPtr create(BufferSlice slice, bool sync_flag) {
auto *ptr = new ChainBufferNode(std::move(slice), sync_flag);
return ChainBufferNode::make_writer_ptr(ptr);
}
static ChainBufferNodeReaderPtr clone(const ChainBufferNodeReaderPtr &ptr) {
if (!ptr) {
return ChainBufferNodeReaderPtr();
}
return ChainBufferNode::make_reader_ptr(ptr.get());
}
static ChainBufferNodeReaderPtr clone(ChainBufferNodeWriterPtr &ptr) {
if (!ptr) {
return ChainBufferNodeReaderPtr();
}
return ChainBufferNode::make_reader_ptr(ptr.get());
}
};
class ChainBufferIterator {
public:
ChainBufferIterator() = default;
explicit ChainBufferIterator(ChainBufferNodeReaderPtr head) : head_(std::move(head)) {
load_head();
}
ChainBufferIterator clone() const {
return ChainBufferIterator(ChainBufferNodeAllocator::clone(head_), reader_.clone(), need_sync_, offset_);
}
size_t offset() const {
return offset_;
}
void clear() {
*this = ChainBufferIterator();
}
Slice prepare_read() {
if (!head_) {
return Slice();
}
while (true) {
auto res = reader_.prepare_read();
if (!res.empty()) {
return res;
}
auto has_writer = head_->has_writer();
if (need_sync_) {
reader_.sync_with_writer();
res = reader_.prepare_read();
if (!res.empty()) {
return res;
}
}
if (has_writer) {
return Slice();
}
head_ = ChainBufferNodeAllocator::clone(head_->next_);
if (!head_) {
return Slice();
}
load_head();
}
}
// returns only head
BufferSlice read_as_buffer_slice(size_t limit) {
prepare_read();
auto res = reader_.clone();
res.truncate(limit);
confirm_read(res.size());
return res;
}
const BufferSlice &head() const {
return reader_;
}
void confirm_read(size_t size) {
offset_ += size;
reader_.confirm_read(size);
}
void advance_till_end() {
while (true) {
auto ready = prepare_read();
if (ready.empty()) {
break;
}
confirm_read(ready.size());
}
}
size_t advance(size_t offset, MutableSlice dest = MutableSlice()) {
size_t skipped = 0;
while (offset != 0) {
auto ready = prepare_read();
if (ready.empty()) {
break;
}
// read no more than offset
ready.truncate(offset);
offset -= ready.size();
skipped += ready.size();
// copy to dest if possible
auto to_dest_size = min(ready.size(), dest.size());
if (to_dest_size != 0) {
dest.copy_from(ready.substr(0, to_dest_size));
dest.remove_prefix(to_dest_size);
}
confirm_read(ready.size());
}
return skipped;
}
private:
ChainBufferNodeReaderPtr head_;
BufferSlice reader_; // copy of head_->slice_
bool need_sync_ = false; // copy of head_->sync_flag_
size_t offset_ = 0; // position in the union of all nodes
ChainBufferIterator(ChainBufferNodeReaderPtr head, BufferSlice reader, bool need_sync, size_t offset)
: head_(std::move(head)), reader_(std::move(reader)), need_sync_(need_sync), offset_(offset) {
}
void load_head() {
reader_ = head_->slice_.clone();
need_sync_ = head_->sync_flag_;
}
};
class ChainBufferReader {
public:
ChainBufferReader() = default;
explicit ChainBufferReader(ChainBufferNodeReaderPtr head)
: begin_(ChainBufferNodeAllocator::clone(head)), end_(std::move(head)) {
end_.advance_till_end();
}
ChainBufferReader(ChainBufferIterator begin, ChainBufferIterator end, bool sync_flag)
: begin_(std::move(begin)), end_(std::move(end)), sync_flag_(sync_flag) {
}
ChainBufferReader(ChainBufferNodeReaderPtr head, size_t size)
: begin_(ChainBufferNodeAllocator::clone(head)), end_(std::move(head)) {
auto advanced = end_.advance(size);
CHECK(advanced == size);
}
ChainBufferReader(ChainBufferReader &&) = default;
ChainBufferReader &operator=(ChainBufferReader &&) = default;
ChainBufferReader(const ChainBufferReader &) = delete;
ChainBufferReader &operator=(const ChainBufferReader &) = delete;
~ChainBufferReader() = default;
ChainBufferReader clone() {
return ChainBufferReader(begin_.clone(), end_.clone(), sync_flag_);
}
Slice prepare_read() {
auto res = begin_.prepare_read();
res.truncate(size());
return res;
}
void confirm_read(size_t size) {
CHECK(size <= this->size());
begin_.confirm_read(size);
}
size_t advance(size_t offset, MutableSlice dest = MutableSlice()) {
CHECK(offset <= size());
return begin_.advance(offset, dest);
}
size_t size() const {
return end_.offset() - begin_.offset();
}
bool empty() const {
return size() == 0;
}
void sync_with_writer() {
if (sync_flag_) {
end_.advance_till_end();
}
}
void advance_end(size_t size) {
end_.advance(size);
}
const ChainBufferIterator &begin() {
return begin_;
}
const ChainBufferIterator &end() {
return end_;
}
// Return [begin_, tail.begin_)
// *this = tail
ChainBufferReader cut_head(ChainBufferIterator pos) TD_WARN_UNUSED_RESULT {
auto tmp = begin_.clone();
begin_ = pos.clone();
return ChainBufferReader(std::move(tmp), std::move(pos), false);
}
ChainBufferReader cut_head(size_t offset) TD_WARN_UNUSED_RESULT {
LOG_CHECK(offset <= size()) << offset << " " << size();
auto it = begin_.clone();
it.advance(offset);
return cut_head(std::move(it));
}
BufferSlice move_as_buffer_slice() {
BufferSlice res;
if (begin_.head().size() >= size()) {
res = begin_.read_as_buffer_slice(size());
} else {
auto save_size = size();
res = BufferSlice{save_size};
advance(save_size, res.as_slice());
}
*this = ChainBufferReader();
return res;
}
BufferSlice read_as_buffer_slice(size_t limit = std::numeric_limits<size_t>::max()) {
return begin_.read_as_buffer_slice(min(limit, size()));
}
private:
ChainBufferIterator begin_; // use it for prepare_read. Fix result with size()
ChainBufferIterator end_; // keep end as far as we can. use it for size()
bool sync_flag_ = true; // auto sync of end_
// 1. We have fixed size. Than end_ is useless.
// 2. No fixed size. One has to sync end_ with end_.advance_till_end() in order to calculate size.
};
class ChainBufferWriter {
public:
ChainBufferWriter() {
init();
}
void init(size_t size = 0) {
writer_ = BufferWriter(size);
tail_ = ChainBufferNodeAllocator::create(writer_.as_buffer_slice(), true);
head_ = ChainBufferNodeAllocator::clone(tail_);
}
MutableSlice prepare_append(size_t hint = 0) {
CHECK(!empty());
auto res = prepare_append_inplace();
if (res.empty()) {
return prepare_append_alloc(hint);
}
return res;
}
MutableSlice prepare_append_at_least(size_t size) {
CHECK(!empty());
auto res = prepare_append_inplace();
if (res.size() < size) {
return prepare_append_alloc(size);
}
return res;
}
MutableSlice prepare_append_inplace() {
CHECK(!empty());
return writer_.prepare_append();
}
MutableSlice prepare_append_alloc(size_t hint = 0) {
CHECK(!empty());
if (hint < (1 << 10)) {
hint = 1 << 12;
}
BufferWriter new_writer(hint);
auto new_tail = ChainBufferNodeAllocator::create(new_writer.as_buffer_slice(), true);
tail_->next_ = ChainBufferNodeAllocator::clone(new_tail);
writer_ = std::move(new_writer);
tail_ = std::move(new_tail); // release tail_
return writer_.prepare_append();
}
void confirm_append(size_t size) {
CHECK(!empty());
writer_.confirm_append(size);
}
void append(Slice slice, size_t hint = 0) {
while (!slice.empty()) {
auto ready = prepare_append(td::max(slice.size(), hint));
auto shift = min(ready.size(), slice.size());
ready.copy_from(slice.substr(0, shift));
confirm_append(shift);
slice.remove_prefix(shift);
}
}
void append(BufferSlice slice) {
auto ready = prepare_append_inplace();
// TODO(perf): we have to store some stats in ChainBufferWriter
// for better append logic
if (slice.size() < (1 << 8) || ready.size() >= slice.size()) {
return append(slice.as_slice());
}
auto new_tail = ChainBufferNodeAllocator::create(std::move(slice), false);
tail_->next_ = ChainBufferNodeAllocator::clone(new_tail);
writer_ = BufferWriter();
tail_ = std::move(new_tail); // release tail_
}
void append(ChainBufferReader &&reader) {
while (!reader.empty()) {
append(reader.read_as_buffer_slice());
}
}
void append(ChainBufferReader &reader) {
while (!reader.empty()) {
append(reader.read_as_buffer_slice());
}
}
ChainBufferReader extract_reader() {
CHECK(head_);
return ChainBufferReader(std::move(head_));
}
private:
bool empty() const {
return !tail_;
}
ChainBufferNodeReaderPtr head_;
ChainBufferNodeWriterPtr tail_;
BufferWriter writer_;
};
class BufferBuilder {
public:
BufferBuilder() = default;
BufferBuilder(Slice slice, size_t prepend_size, size_t append_size)
: buffer_writer_(slice, prepend_size, append_size) {
}
explicit BufferBuilder(BufferWriter &&buffer_writer) : buffer_writer_(std::move(buffer_writer)) {
}
void append(BufferSlice slice);
void append(Slice slice);
void prepend(BufferSlice slice);
void prepend(Slice slice);
template <class F>
void for_each(F &&f) const & {
for (auto &slice : reversed(to_prepend_)) {
f(slice.as_slice());
}
if (!buffer_writer_.empty()) {
f(buffer_writer_.as_slice());
}
for (auto &slice : to_append_) {
f(slice.as_slice());
}
}
template <class F>
void for_each(F &&f) && {
for (auto &slice : reversed(to_prepend_)) {
f(std::move(slice));
}
if (!buffer_writer_.empty()) {
f(buffer_writer_.as_buffer_slice());
}
for (auto &slice : to_append_) {
f(std::move(slice));
}
}
size_t size() const;
BufferSlice extract();
private:
BufferWriter buffer_writer_;
std::vector<BufferSlice> to_append_;
std::vector<BufferSlice> to_prepend_;
bool append_inplace(Slice slice);
void append_slow(BufferSlice slice);
bool prepend_inplace(Slice slice);
void prepend_slow(BufferSlice slice);
};
inline Slice as_slice(const BufferSlice &value) {
return value.as_slice();
}
inline MutableSlice as_slice(BufferSlice &value) {
return value.as_slice();
}
inline MutableSlice as_mutable_slice(BufferSlice &value) {
return value.as_slice();
}
} // namespace td

Some files were not shown because too many files have changed in this diff Show more