dark/zerotiertspu
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'dev' of https://github.com/zerotier/ZeroTierOne into dev

Browse source
This commit is contained in:
Joseph Henry 2020-11-15 19:00:02 -08:00
commit a690ffdb7c
24 changed files with 660 additions and 349 deletions
Download patch file Download diff file Expand all files Collapse all files

22
node/AES_aesni.cpp
View file

@ -26,7 +26,9 @@ namespace {
const __m128i s_sseSwapBytes = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
#ifdef __GNUC__
__attribute__((__target__("ssse3,sse4,sse4.1,sse4.2,pclmul")))
#endif
__m128i p_gmacPCLMUL128(const __m128i h, __m128i y) noexcept
{
y = _mm_shuffle_epi8(y, s_sseSwapBytes);
@ -57,7 +59,9 @@ __m128i p_gmacPCLMUL128(const __m128i h, __m128i y) noexcept
#define ZT_AES_VAES512 1
#ifdef __GNUC__
__attribute__((__target__("sse4,aes,avx,avx2,vaes,avx512f,avx512bw")))
#endif
void p_aesCtrInnerVAES512(unsigned int &len, const uint64_t c0, uint64_t &c1, const uint8_t *&in, uint8_t *&out, const __m128i *const k) noexcept
{
const __m512i kk0 = _mm512_broadcast_i32x4(k[0]);
@ -107,7 +111,9 @@ void p_aesCtrInnerVAES512(unsigned int &len, const uint64_t c0, uint64_t &c1, co
#define ZT_AES_VAES256 1
#ifdef __GNUC__
__attribute__((__target__("sse4,aes,avx,avx2,vaes")))
#endif
void p_aesCtrInnerVAES256(unsigned int &len, const uint64_t c0, uint64_t &c1, const uint8_t *&in, uint8_t *&out, const __m128i *const k) noexcept
{
const __m256i kk0 = _mm256_broadcastsi128_si256(k[0]);
@ -175,7 +181,9 @@ void p_aesCtrInnerVAES256(unsigned int &len, const uint64_t c0, uint64_t &c1, co
#endif // does compiler support AVX2 and AVX512 AES intrinsics?
#ifdef __GNUC__
__attribute__((__target__("ssse3,sse4,sse4.1,sse4.2,aes,pclmul")))
#endif
__m128i p_init256_1_aesni(__m128i a, __m128i b) noexcept
{
__m128i x, y;
@ -190,7 +198,9 @@ __m128i p_init256_1_aesni(__m128i a, __m128i b) noexcept
return x;
}
#ifdef __GNUC__
__attribute__((__target__("ssse3,sse4,sse4.1,sse4.2,aes,pclmul")))
#endif
__m128i p_init256_2_aesni(__m128i a, __m128i b) noexcept
{
__m128i x, y, z;
@ -208,7 +218,9 @@ __m128i p_init256_2_aesni(__m128i a, __m128i b) noexcept
} // anonymous namespace
#ifdef __GNUC__
__attribute__((__target__("ssse3,sse4,sse4.1,sse4.2,pclmul")))
#endif
void AES::GMAC::p_aesNIUpdate(const uint8_t *in, unsigned int len) noexcept
{
__m128i y = _mm_loadu_si128(reinterpret_cast<const __m128i *>(_y));
@ -274,7 +286,9 @@ void AES::GMAC::p_aesNIUpdate(const uint8_t *in, unsigned int len) noexcept
_rp = len; // len is always less than 16 here
}
#ifdef __GNUC__
__attribute__((__target__("ssse3,sse4,sse4.1,sse4.2,pclmul,aes")))
#endif
void AES::GMAC::p_aesNIFinish(uint8_t tag[16]) noexcept
{
__m128i y = _mm_loadu_si128(reinterpret_cast<const __m128i *>(_y));
@ -345,7 +359,9 @@ void AES::GMAC::p_aesNIFinish(uint8_t tag[16]) noexcept
_mm_storeu_si128(reinterpret_cast<__m128i *>(tag), _mm_xor_si128(_mm_shuffle_epi8(t4, s_sseSwapBytes), encIV));
}
#ifdef __GNUC__
__attribute__((__target__("ssse3,sse4,sse4.1,sse4.2,aes")))
#endif
void AES::CTR::p_aesNICrypt(const uint8_t *in, uint8_t *out, unsigned int len) noexcept
{
const __m128i dd = _mm_set_epi64x(0, (long long)_ctr[0]);
@ -542,7 +558,9 @@ void AES::CTR::p_aesNICrypt(const uint8_t *in, uint8_t *out, unsigned int len) n
_ctr[1] = Utils::hton(c1);
}
#ifdef __GNUC__
__attribute__((__target__("ssse3,sse4,sse4.1,sse4.2,aes,pclmul")))
#endif
void AES::p_init_aesni(const uint8_t *key) noexcept
{
__m128i t1, t2, k1, k2, k3, k4, k5, k6, k7, k8, k9, k10, k11, k12, k13;
@ -604,7 +622,9 @@ void AES::p_init_aesni(const uint8_t *key) noexcept
p_k.ni.h2[3] = _mm_xor_si128(_mm_shuffle_epi32(hhhh, 78), hhhh);
}
#ifdef __GNUC__
__attribute__((__target__("ssse3,sse4,sse4.1,sse4.2,aes,pclmul")))
#endif
void AES::p_encrypt_aesni(const void *const in, void *const out) const noexcept
{
__m128i tmp = _mm_loadu_si128((const __m128i *)in);
@ -625,7 +645,9 @@ void AES::p_encrypt_aesni(const void *const in, void *const out) const noexcept
_mm_storeu_si128((__m128i *)out, _mm_aesenclast_si128(tmp, p_k.ni.k[14]));
}
#ifdef __GNUC__
__attribute__((__target__("ssse3,sse4,sse4.1,sse4.2,aes,pclmul")))
#endif
void AES::p_decrypt_aesni(const void *in, void *out) const noexcept
{
__m128i tmp = _mm_loadu_si128((const __m128i *)in);

9
node/Constants.hpp
View file

@ -107,8 +107,6 @@
#include <xmmintrin.h>
#include <emmintrin.h>
#include <immintrin.h>
#include <tmmintrin.h>
#include <mmintrin.h>
#endif
#if (defined(__ARM_NEON) || defined(__ARM_NEON__) || defined(ZT_ARCH_ARM_HAS_NEON))
@ -188,6 +186,9 @@
*/
#define ZT_ADDRESS_LENGTH_HEX 10
/**
* Size of symmetric key (only the first 32 bits are used for some ciphers)
*/
#define ZT_SYMMETRIC_KEY_SIZE 48
/**
@ -223,7 +224,7 @@
/**
* How often Topology::clean() and Network::clean() and similar are called, in ms
*/
#define ZT_HOUSEKEEPING_PERIOD 60000
#define ZT_HOUSEKEEPING_PERIOD 30000
/**
* Delay between WHOIS retries in ms
@ -255,7 +256,7 @@
/**
* Period for multicast LIKE announcements
*/
#define ZT_MULTICAST_ANNOUNCE_PERIOD 120000
#define ZT_MULTICAST_ANNOUNCE_PERIOD 60000
/**
* Delay between explicit MULTICAST_GATHER requests for a given multicast channel

46
node/InetAddress.hpp
View file

@ -453,6 +453,52 @@ struct InetAddress : public sockaddr_storage
*/
bool isNetwork() const;
/**
* Find the total number of prefix bits that match between this IP and another
*
* @param b Second IP to compare with
* @return Number of matching prefix bits or 0 if none match or IPs are of different families (e.g. v4 and v6)
*/
inline unsigned int matchingPrefixBits(const InetAddress &b) const
{
unsigned int c = 0;
if (ss_family == b.ss_family) {
switch(ss_family) {
case AF_INET: {
uint32_t ip0 = Utils::ntoh((uint32_t)reinterpret_cast<const struct sockaddr_in *>(this)->sin_addr.s_addr);
uint32_t ip1 = Utils::ntoh((uint32_t)reinterpret_cast<const struct sockaddr_in *>(&b)->sin_addr.s_addr);
while ((ip0 >> 31) == (ip1 >> 31)) {
ip0 <<= 1;
ip1 <<= 1;
if (++c == 32)
break;
}
} break;
case AF_INET6: {
const uint8_t *ip0 = reinterpret_cast<const uint8_t *>(reinterpret_cast<const struct sockaddr_in6 *>(this)->sin6_addr.s6_addr);
const uint8_t *ip1 = reinterpret_cast<const uint8_t *>(reinterpret_cast<const struct sockaddr_in6 *>(&b)->sin6_addr.s6_addr);
for(unsigned int i=0;i<16;++i) {
if (ip0[i] == ip1[i]) {
c += 8;
} else {
uint8_t ip0b = ip0[i];
uint8_t ip1b = ip1[i];
uint8_t bit = 0x80;
while (bit != 0) {
if ((ip0b & bit) != (ip1b & bit))
break;
++c;
bit >>= 1;
}
break;
}
}
} break;
}
}
return c;
}
/**
* @return 14-bit (0-16383) hash of this IP's first 24 or 48 bits (for V4 or V6) for rate limiting code, or 0 if non-IP
*/

61
node/Multicaster.cpp
View file

@ -262,9 +262,8 @@ void Multicaster::send(
}
}
} else {
if (gs.txQueue.size() >= ZT_TX_QUEUE_SIZE) {
RR->t->outgoingNetworkFrameDropped(tPtr,network,src,mg.mac(),etherType,0,len,"multicast TX queue is full");
return;
while (gs.txQueue.size() >= ZT_TX_QUEUE_SIZE) {
gs.txQueue.pop_front();
}
const unsigned int gatherLimit = (limit - (unsigned int)gs.members.size()) + 1;
@ -371,39 +370,37 @@ void Multicaster::send(
void Multicaster::clean(int64_t now)
{
{
Mutex::Lock _l(_groups_m);
Multicaster::Key *k = (Multicaster::Key *)0;
MulticastGroupStatus *s = (MulticastGroupStatus *)0;
Hashtable<Multicaster::Key,MulticastGroupStatus>::Iterator mm(_groups);
while (mm.next(k,s)) {
for(std::list<OutboundMulticast>::iterator tx(s->txQueue.begin());tx!=s->txQueue.end();) {
if ((tx->expired(now))||(tx->atLimit()))
s->txQueue.erase(tx++);
else ++tx;
}
Mutex::Lock _l(_groups_m);
Multicaster::Key *k = (Multicaster::Key *)0;
MulticastGroupStatus *s = (MulticastGroupStatus *)0;
Hashtable<Multicaster::Key,MulticastGroupStatus>::Iterator mm(_groups);
while (mm.next(k,s)) {
for(std::list<OutboundMulticast>::iterator tx(s->txQueue.begin());tx!=s->txQueue.end();) {
if ((tx->expired(now))||(tx->atLimit()))
s->txQueue.erase(tx++);
else ++tx;
}
unsigned long count = 0;
{
std::vector<MulticastGroupMember>::iterator reader(s->members.begin());
std::vector<MulticastGroupMember>::iterator writer(reader);
while (reader != s->members.end()) {
if ((now - reader->timestamp) < ZT_MULTICAST_LIKE_EXPIRE) {
*writer = *reader;
++writer;
++count;
}
++reader;
unsigned long count = 0;
{
std::vector<MulticastGroupMember>::iterator reader(s->members.begin());
std::vector<MulticastGroupMember>::iterator writer(reader);
while (reader != s->members.end()) {
if ((now - reader->timestamp) < ZT_MULTICAST_LIKE_EXPIRE) {
*writer = *reader;
++writer;
++count;
}
++reader;
}
}
if (count) {
s->members.resize(count);
} else if (s->txQueue.empty()) {
_groups.erase(*k);
} else {
s->members.clear();
}
if (count) {
s->members.resize(count);
} else if (s->txQueue.empty()) {
_groups.erase(*k);
} else {
s->members.clear();
}
}
}

6
node/Multicaster.hpp
View file

@ -138,7 +138,7 @@ public:
unsigned int len);
/**
* Clean up and resort database
* Clean database
*
* @param RR Runtime environment
* @param now Current time
@ -172,14 +172,14 @@ private:
inline bool operator!=(const Address &a) const { return (address != a); }
Address address;
uint64_t timestamp; // time of last notification
int64_t timestamp; // time of last notification
};
struct MulticastGroupStatus
{
MulticastGroupStatus() : lastExplicitGather(0) {}
uint64_t lastExplicitGather;
int64_t lastExplicitGather;
std::list<OutboundMulticast> txQueue; // pending outbound multicasts
std::vector<MulticastGroupMember> members; // members of this group
};

2
node/Peer.cpp
View file

@ -409,7 +409,7 @@ void Peer::sendHELLO(void *tPtr,const int64_t localSocket,const InetAddress &atA
outp.cryptField(_key,startCryptedPortionAt,outp.size() - startCryptedPortionAt);
if (atAddress) {
outp.armor(_key,false,aesKeysIfSupported()); // false == don't encrypt full payload, but add MAC
outp.armor(_key,false,nullptr); // false == don't encrypt full payload, but add MAC
RR->node->expectReplyTo(outp.packetId());
RR->node->putPacket(tPtr,localSocket,atAddress,outp.data(),outp.size());
} else {

37
node/Utils.cpp
View file

@ -50,6 +50,43 @@
#include <asm/hwcap.h>
#endif
#ifdef ZT_ARCH_ARM_HAS_NEON
#ifdef __LINUX__
#include <sys/auxv.h>
#include <asm/hwcap.h>
#endif
#if defined(__FreeBSD__)
#include <elf.h>
#include <sys/auxv.h>
static inline long getauxval(int caps)
{
long hwcaps = 0;
elf_aux_info(caps, &hwcaps, sizeof(hwcaps));
return hwcaps;
}
#endif
// If these are not even defined, obviously they are not supported.
#ifndef HWCAP_AES
#define HWCAP_AES 0
#endif
#ifndef HWCAP_CRC32
#define HWCAP_CRC32 0
#endif
#ifndef HWCAP_PMULL
#define HWCAP_PMULL 0
#endif
#ifndef HWCAP_SHA1
#define HWCAP_SHA1 0
#endif
#ifndef HWCAP_SHA2
#define HWCAP_SHA2 0
#endif
#endif // ZT_ARCH_ARM_HAS_NEON
namespace ZeroTier {
const uint64_t Utils::ZERO256[4] = {0ULL,0ULL,0ULL,0ULL};

16
node/Utils.hpp
View file

@ -94,6 +94,22 @@ public:
static const CPUIDRegisters CPUID;
#endif
/**
* Compute the log2 (most significant bit set) of a 32-bit integer
*
* @param v Integer to compute
* @return log2 or 0 if v is 0
*/
static inline unsigned int log2(uint32_t v)
{
uint32_t r = (v > 0xffff) << 4; v >>= r;
uint32_t shift = (v > 0xff) << 3; v >>= shift; r |= shift;
shift = (v > 0xf) << 2; v >>= shift; r |= shift;
shift = (v > 0x3) << 1; v >>= shift; r |= shift;
r |= (v >> 1);
return (unsigned int)r;
}
/**
* Perform a time-invariant binary comparison
*