Add Bonds, Slaves, and Flows
This commit is contained in:
Joseph Henry
parent
de9cfbe9b0
commit
a50e8e9878
31 changed files with 4898 additions and 1966 deletions
782
node/Peer.cpp
782
node/Peer.cpp
|
|
@ -1,10 +1,10 @@
|
|||
/*
|
||||
* Copyright (c)2019 ZeroTier, Inc.
|
||||
* Copyright (c)2013-2020 ZeroTier, Inc.
|
||||
*
|
||||
* Use of this software is governed by the Business Source License included
|
||||
* in the LICENSE.TXT file in the project's root directory.
|
||||
*
|
||||
* Change Date: 2023-01-01
|
||||
* Change Date: 2024-01-01
|
||||
*
|
||||
* On the date above, in accordance with the Business Source License, use
|
||||
* of this software will be governed by version 2.0 of the Apache License.
|
||||
|
|
@ -14,7 +14,6 @@
|
|||
#include "../version.h"
|
||||
#include "Constants.hpp"
|
||||
#include "Peer.hpp"
|
||||
#include "Node.hpp"
|
||||
#include "Switch.hpp"
|
||||
#include "Network.hpp"
|
||||
#include "SelfAwareness.hpp"
|
||||
|
|
@ -24,8 +23,6 @@
|
|||
#include "RingBuffer.hpp"
|
||||
#include "Utils.hpp"
|
||||
|
||||
#include "../include/ZeroTierDebug.h"
|
||||
|
||||
namespace ZeroTier {
|
||||
|
||||
static unsigned char s_freeRandomByteCounter = 0;
|
||||
|
|
@ -37,20 +34,14 @@ Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Ident
|
|||
_lastTriedMemorizedPath(0),
|
||||
_lastDirectPathPushSent(0),
|
||||
_lastDirectPathPushReceive(0),
|
||||
_lastEchoRequestReceived(0),
|
||||
_lastCredentialRequestSent(0),
|
||||
_lastWhoisRequestReceived(0),
|
||||
_lastEchoRequestReceived(0),
|
||||
_lastCredentialsReceived(0),
|
||||
_lastTrustEstablishedPacketReceived(0),
|
||||
_lastSentFullHello(0),
|
||||
_lastACKWindowReset(0),
|
||||
_lastQoSWindowReset(0),
|
||||
_lastMultipathCompatibilityCheck(0),
|
||||
_lastEchoCheck(0),
|
||||
_freeRandomByte((unsigned char)((uintptr_t)this >> 4) ^ ++s_freeRandomByteCounter),
|
||||
_uniqueAlivePathCount(0),
|
||||
_localMultipathSupported(false),
|
||||
_remoteMultipathSupported(false),
|
||||
_canUseMultipath(false),
|
||||
_vProto(0),
|
||||
_vMajor(0),
|
||||
_vMinor(0),
|
||||
|
|
@ -58,17 +49,17 @@ Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Ident
|
|||
_id(peerIdentity),
|
||||
_directPathPushCutoffCount(0),
|
||||
_credentialsCutoffCount(0),
|
||||
_linkIsBalanced(false),
|
||||
_linkIsRedundant(false),
|
||||
_remotePeerMultipathEnabled(false),
|
||||
_lastAggregateStatsReport(0),
|
||||
_lastAggregateAllocation(0),
|
||||
_virtualPathCount(0),
|
||||
_roundRobinPathAssignmentIdx(0),
|
||||
_pathAssignmentIdx(0)
|
||||
_echoRequestCutoffCount(0),
|
||||
_uniqueAlivePathCount(0),
|
||||
_localMultipathSupported(false),
|
||||
_remoteMultipathSupported(false),
|
||||
_canUseMultipath(false),
|
||||
_shouldCollectPathStatistics(0),
|
||||
_lastComputedAggregateMeanLatency(0)
|
||||
{
|
||||
if (!myIdentity.agree(peerIdentity,_key,ZT_PEER_SECRET_KEY_LENGTH))
|
||||
if (!myIdentity.agree(peerIdentity,_key,ZT_PEER_SECRET_KEY_LENGTH)) {
|
||||
throw ZT_EXCEPTION_INVALID_ARGUMENT;
|
||||
}
|
||||
}
|
||||
|
||||
void Peer::received(
|
||||
|
|
@ -81,7 +72,8 @@ void Peer::received(
|
|||
const uint64_t inRePacketId,
|
||||
const Packet::Verb inReVerb,
|
||||
const bool trustEstablished,
|
||||
const uint64_t networkId)
|
||||
const uint64_t networkId,
|
||||
const int32_t flowId)
|
||||
{
|
||||
const int64_t now = RR->node->now();
|
||||
|
||||
|
|
@ -98,28 +90,13 @@ void Peer::received(
|
|||
break;
|
||||
}
|
||||
|
||||
recordIncomingPacket(tPtr, path, packetId, payloadLength, verb, flowId, now);
|
||||
|
||||
if (trustEstablished) {
|
||||
_lastTrustEstablishedPacketReceived = now;
|
||||
path->trustedPacketReceived(now);
|
||||
}
|
||||
|
||||
{
|
||||
Mutex::Lock _l(_paths_m);
|
||||
|
||||
recordIncomingPacket(tPtr, path, packetId, payloadLength, verb, now);
|
||||
|
||||
if (_canUseMultipath) {
|
||||
if (path->needsToSendQoS(now)) {
|
||||
sendQOS_MEASUREMENT(tPtr, path, path->localSocket(), path->address(), now);
|
||||
}
|
||||
for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
|
||||
if (_paths[i].p) {
|
||||
_paths[i].p->processBackgroundPathMeasurements(now);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (hops == 0) {
|
||||
// If this is a direct packet (no hops), update existing paths or learn new ones
|
||||
bool havePath = false;
|
||||
|
|
@ -137,60 +114,45 @@ void Peer::received(
|
|||
}
|
||||
|
||||
bool attemptToContact = false;
|
||||
|
||||
int replaceIdx = ZT_MAX_PEER_NETWORK_PATHS;
|
||||
if ((!havePath)&&(RR->node->shouldUsePathForZeroTierTraffic(tPtr,_id.address(),path->localSocket(),path->address()))) {
|
||||
Mutex::Lock _l(_paths_m);
|
||||
|
||||
// Paths are redundant if they duplicate an alive path to the same IP or
|
||||
// with the same local socket and address family.
|
||||
bool redundant = false;
|
||||
unsigned int replacePath = ZT_MAX_PEER_NETWORK_PATHS;
|
||||
for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
|
||||
if (_paths[i].p) {
|
||||
if ( (_paths[i].p->alive(now)) && ( ((_paths[i].p->localSocket() == path->localSocket())&&(_paths[i].p->address().ss_family == path->address().ss_family)) || (_paths[i].p->address().ipsEqual2(path->address())) ) ) {
|
||||
redundant = true;
|
||||
break;
|
||||
}
|
||||
// If the path is the same address and port, simply assume this is a replacement
|
||||
if ( (_paths[i].p->address().ipsEqual2(path->address()))) {
|
||||
replacePath = i;
|
||||
break;
|
||||
}
|
||||
} else break;
|
||||
}
|
||||
|
||||
// If the path isn't a duplicate of the same localSocket AND we haven't already determined a replacePath,
|
||||
// then find the worst path and replace it.
|
||||
if (!redundant && replacePath == ZT_MAX_PEER_NETWORK_PATHS) {
|
||||
int replacePathQuality = 0;
|
||||
for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
|
||||
if (_paths[i].p) {
|
||||
const int q = _paths[i].p->quality(now);
|
||||
if (q > replacePathQuality) {
|
||||
replacePathQuality = q;
|
||||
replacePath = i;
|
||||
// match addr
|
||||
if ( (_paths[i].p->alive(now)) && ( ((_paths[i].p->localSocket() == path->localSocket())&&(_paths[i].p->address().ss_family == path->address().ss_family)) && (_paths[i].p->address().ipsEqual2(path->address())) ) ) {
|
||||
// port
|
||||
if (_paths[i].p->address().port() == path->address().port()) {
|
||||
replaceIdx = i;
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
replacePath = i;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (replaceIdx == ZT_MAX_PEER_NETWORK_PATHS) {
|
||||
for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
|
||||
if (!_paths[i].p) {
|
||||
replaceIdx = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (replacePath != ZT_MAX_PEER_NETWORK_PATHS) {
|
||||
if (replaceIdx != ZT_MAX_PEER_NETWORK_PATHS) {
|
||||
if (verb == Packet::VERB_OK) {
|
||||
RR->t->peerLearnedNewPath(tPtr,networkId,*this,path,packetId);
|
||||
_paths[replacePath].lr = now;
|
||||
_paths[replacePath].p = path;
|
||||
_paths[replacePath].priority = 1;
|
||||
performMultipathStateCheck(now);
|
||||
if (_bondToPeer) {
|
||||
_bondToPeer->nominatePath(path, now);
|
||||
}
|
||||
_paths[replaceIdx].lr = now;
|
||||
_paths[replaceIdx].p = path;
|
||||
_paths[replaceIdx].priority = 1;
|
||||
} else {
|
||||
attemptToContact = true;
|
||||
}
|
||||
|
||||
// Every time we learn of new path, rebuild set of virtual paths
|
||||
constructSetOfVirtualPaths();
|
||||
}
|
||||
}
|
||||
|
||||
if (attemptToContact) {
|
||||
attemptToContactAt(tPtr,path->localSocket(),path->address(),now,true);
|
||||
path->sent(now);
|
||||
|
|
@ -203,8 +165,7 @@ void Peer::received(
|
|||
// is done less frequently.
|
||||
if (this->trustEstablished(now)) {
|
||||
const int64_t sinceLastPush = now - _lastDirectPathPushSent;
|
||||
if (sinceLastPush >= ((hops == 0) ? ZT_DIRECT_PATH_PUSH_INTERVAL_HAVEPATH : ZT_DIRECT_PATH_PUSH_INTERVAL)
|
||||
|| (_localMultipathSupported && (sinceLastPush >= (ZT_DIRECT_PATH_PUSH_INTERVAL_MULTIPATH)))) {
|
||||
if (sinceLastPush >= ((hops == 0) ? ZT_DIRECT_PATH_PUSH_INTERVAL_HAVEPATH : ZT_DIRECT_PATH_PUSH_INTERVAL)) {
|
||||
_lastDirectPathPushSent = now;
|
||||
std::vector<InetAddress> pathsToPush(RR->node->directPaths());
|
||||
if (pathsToPush.size() > 0) {
|
||||
|
|
@ -249,189 +210,15 @@ void Peer::received(
|
|||
}
|
||||
}
|
||||
|
||||
void Peer::constructSetOfVirtualPaths()
|
||||
SharedPtr<Path> Peer::getAppropriatePath(int64_t now, bool includeExpired, int32_t flowId)
|
||||
{
|
||||
if (!_remoteMultipathSupported) {
|
||||
return;
|
||||
}
|
||||
Mutex::Lock _l(_virtual_paths_m);
|
||||
|
||||
int64_t now = RR->node->now();
|
||||
_virtualPathCount = 0;
|
||||
for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
|
||||
if (_paths[i].p && _paths[i].p->alive(now)) {
|
||||
for(unsigned int j=0;j<ZT_MAX_PEER_NETWORK_PATHS;++j) {
|
||||
if (_paths[j].p && _paths[j].p->alive(now)) {
|
||||
int64_t localSocket = _paths[j].p->localSocket();
|
||||
bool foundVirtualPath = false;
|
||||
for (int k=0; k<_virtualPaths.size(); k++) {
|
||||
if (_virtualPaths[k]->localSocket == localSocket && _virtualPaths[k]->p == _paths[i].p) {
|
||||
foundVirtualPath = true;
|
||||
}
|
||||
}
|
||||
if (!foundVirtualPath)
|
||||
{
|
||||
VirtualPath *np = new VirtualPath;
|
||||
np->p = _paths[i].p;
|
||||
np->localSocket = localSocket;
|
||||
_virtualPaths.push_back(np);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void Peer::recordOutgoingPacket(const SharedPtr<Path> &path, const uint64_t packetId,
|
||||
uint16_t payloadLength, const Packet::Verb verb, int64_t now)
|
||||
{
|
||||
_freeRandomByte += (unsigned char)(packetId >> 8); // grab entropy to use in path selection logic for multipath
|
||||
if (_canUseMultipath) {
|
||||
path->recordOutgoingPacket(now, packetId, payloadLength, verb);
|
||||
}
|
||||
}
|
||||
|
||||
void Peer::recordIncomingPacket(void *tPtr, const SharedPtr<Path> &path, const uint64_t packetId,
|
||||
uint16_t payloadLength, const Packet::Verb verb, int64_t now)
|
||||
{
|
||||
if (_canUseMultipath) {
|
||||
if (path->needsToSendAck(now)) {
|
||||
sendACK(tPtr, path, path->localSocket(), path->address(), now);
|
||||
}
|
||||
path->recordIncomingPacket(now, packetId, payloadLength, verb);
|
||||
}
|
||||
}
|
||||
|
||||
void Peer::computeAggregateAllocation(int64_t now)
|
||||
{
|
||||
float maxStability = 0;
|
||||
float totalRelativeQuality = 0;
|
||||
float maxThroughput = 1;
|
||||
float maxScope = 0;
|
||||
float relStability[ZT_MAX_PEER_NETWORK_PATHS];
|
||||
float relThroughput[ZT_MAX_PEER_NETWORK_PATHS];
|
||||
memset(&relStability, 0, sizeof(relStability));
|
||||
memset(&relThroughput, 0, sizeof(relThroughput));
|
||||
// Survey all paths
|
||||
for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
|
||||
if (_paths[i].p) {
|
||||
relStability[i] = _paths[i].p->lastComputedStability();
|
||||
relThroughput[i] = (float)_paths[i].p->maxLifetimeThroughput();
|
||||
maxStability = relStability[i] > maxStability ? relStability[i] : maxStability;
|
||||
maxThroughput = relThroughput[i] > maxThroughput ? relThroughput[i] : maxThroughput;
|
||||
maxScope = _paths[i].p->ipScope() > maxScope ? _paths[i].p->ipScope() : maxScope;
|
||||
}
|
||||
}
|
||||
// Convert to relative values
|
||||
for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
|
||||
if (_paths[i].p) {
|
||||
relStability[i] /= maxStability ? maxStability : 1;
|
||||
relThroughput[i] /= maxThroughput ? maxThroughput : 1;
|
||||
float normalized_ma = Utils::normalize((float)_paths[i].p->ackAge(now), 0, ZT_PATH_MAX_AGE, 0, 10);
|
||||
float age_contrib = exp((-1)*normalized_ma);
|
||||
float relScope = ((float)(_paths[i].p->ipScope()+1) / (maxScope + 1));
|
||||
float relQuality =
|
||||
(relStability[i] * (float)ZT_PATH_CONTRIB_STABILITY)
|
||||
+ (fmaxf(1.0f, relThroughput[i]) * (float)ZT_PATH_CONTRIB_THROUGHPUT)
|
||||
+ relScope * (float)ZT_PATH_CONTRIB_SCOPE;
|
||||
relQuality *= age_contrib;
|
||||
// Clamp values
|
||||
relQuality = relQuality > (1.00f / 100.0f) ? relQuality : 0.0f;
|
||||
relQuality = relQuality < (99.0f / 100.0f) ? relQuality : 1.0f;
|
||||
totalRelativeQuality += relQuality;
|
||||
_paths[i].p->updateRelativeQuality(relQuality);
|
||||
}
|
||||
}
|
||||
// Convert set of relative performances into an allocation set
|
||||
for(uint16_t i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
|
||||
if (_paths[i].p) {
|
||||
if (RR->node->getMultipathMode() == ZT_MULTIPATH_BALANCE_RANDOM) {
|
||||
_paths[i].p->updateComponentAllocationOfAggregateLink(((float)_pathChoiceHist.countValue(i) / (float)_pathChoiceHist.count()) * 255);
|
||||
}
|
||||
if (RR->node->getMultipathMode() == ZT_MULTIPATH_BALANCE_DYNAMIC_OPAQUE) {
|
||||
_paths[i].p->updateComponentAllocationOfAggregateLink((unsigned char)((_paths[i].p->relativeQuality() / totalRelativeQuality) * 255));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int Peer::computeAggregateLinkPacketDelayVariance()
|
||||
{
|
||||
float pdv = 0.0;
|
||||
for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
|
||||
if (_paths[i].p) {
|
||||
pdv += _paths[i].p->relativeQuality() * _paths[i].p->packetDelayVariance();
|
||||
}
|
||||
}
|
||||
return (int)pdv;
|
||||
}
|
||||
|
||||
int Peer::computeAggregateLinkMeanLatency()
|
||||
{
|
||||
int ml = 0;
|
||||
int pathCount = 0;
|
||||
for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
|
||||
if (_paths[i].p) {
|
||||
pathCount++;
|
||||
ml += (int)(_paths[i].p->relativeQuality() * _paths[i].p->meanLatency());
|
||||
}
|
||||
}
|
||||
return ml / pathCount;
|
||||
}
|
||||
|
||||
int Peer::aggregateLinkPhysicalPathCount()
|
||||
{
|
||||
std::map<std::string, bool> ifnamemap;
|
||||
int pathCount = 0;
|
||||
int64_t now = RR->node->now();
|
||||
for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
|
||||
if (_paths[i].p && _paths[i].p->alive(now)) {
|
||||
if (!ifnamemap[_paths[i].p->getName()]) {
|
||||
ifnamemap[_paths[i].p->getName()] = true;
|
||||
pathCount++;
|
||||
}
|
||||
}
|
||||
}
|
||||
return pathCount;
|
||||
}
|
||||
|
||||
int Peer::aggregateLinkLogicalPathCount()
|
||||
{
|
||||
int pathCount = 0;
|
||||
int64_t now = RR->node->now();
|
||||
for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
|
||||
if (_paths[i].p && _paths[i].p->alive(now)) {
|
||||
pathCount++;
|
||||
}
|
||||
}
|
||||
return pathCount;
|
||||
}
|
||||
|
||||
std::vector<SharedPtr<Path> > Peer::getAllPaths(int64_t now)
|
||||
{
|
||||
Mutex::Lock _l(_virtual_paths_m); // FIXME: TX can now lock RX
|
||||
std::vector<SharedPtr<Path> > paths;
|
||||
for (int i=0; i<_virtualPaths.size(); i++) {
|
||||
if (_virtualPaths[i]->p) {
|
||||
paths.push_back(_virtualPaths[i]->p);
|
||||
}
|
||||
}
|
||||
return paths;
|
||||
}
|
||||
|
||||
SharedPtr<Path> Peer::getAppropriatePath(int64_t now, bool includeExpired, int64_t flowId)
|
||||
{
|
||||
Mutex::Lock _l(_paths_m);
|
||||
SharedPtr<Path> selectedPath;
|
||||
char curPathStr[128];
|
||||
char newPathStr[128];
|
||||
unsigned int bestPath = ZT_MAX_PEER_NETWORK_PATHS;
|
||||
|
||||
/**
|
||||
* Send traffic across the highest quality path only. This algorithm will still
|
||||
* use the old path quality metric from protocol version 9.
|
||||
*/
|
||||
if (!_canUseMultipath) {
|
||||
if (!_bondToPeer) {
|
||||
Mutex::Lock _l(_paths_m);
|
||||
unsigned int bestPath = ZT_MAX_PEER_NETWORK_PATHS;
|
||||
/**
|
||||
* Send traffic across the highest quality path only. This algorithm will still
|
||||
* use the old path quality metric from protocol version 9.
|
||||
*/
|
||||
long bestPathQuality = 2147483647;
|
||||
for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
|
||||
if (_paths[i].p) {
|
||||
|
|
@ -449,293 +236,7 @@ SharedPtr<Path> Peer::getAppropriatePath(int64_t now, bool includeExpired, int64
|
|||
}
|
||||
return SharedPtr<Path>();
|
||||
}
|
||||
|
||||
// Update path measurements
|
||||
for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
|
||||
if (_paths[i].p) {
|
||||
_paths[i].p->processBackgroundPathMeasurements(now);
|
||||
}
|
||||
}
|
||||
if (RR->sw->isFlowAware()) {
|
||||
// Detect new flows and update existing records
|
||||
if (_flows.count(flowId)) {
|
||||
_flows[flowId]->lastSend = now;
|
||||
}
|
||||
else {
|
||||
fprintf(stderr, "new flow %llx detected between this node and %llx (%lu active flow(s))\n",
|
||||
flowId, this->_id.address().toInt(), (_flows.size()+1));
|
||||
struct Flow *newFlow = new Flow(flowId, now);
|
||||
_flows[flowId] = newFlow;
|
||||
newFlow->assignedPath = nullptr;
|
||||
}
|
||||
}
|
||||
// Construct set of virtual paths if needed
|
||||
if (!_virtualPaths.size()) {
|
||||
constructSetOfVirtualPaths();
|
||||
}
|
||||
if (!_virtualPaths.size()) {
|
||||
fprintf(stderr, "no paths to send packet out on\n");
|
||||
return SharedPtr<Path>();
|
||||
}
|
||||
|
||||
/**
|
||||
* All traffic is sent on all paths.
|
||||
*/
|
||||
if (RR->node->getMultipathMode() == ZT_MULTIPATH_BROADCAST) {
|
||||
// Not handled here. Handled in Switch::_trySend()
|
||||
}
|
||||
|
||||
/**
|
||||
* Only one link is active. Fail-over is immediate.
|
||||
*/
|
||||
if (RR->node->getMultipathMode() == ZT_MULTIPATH_ACTIVE_BACKUP) {
|
||||
bool bFoundHotPath = false;
|
||||
if (!_activeBackupPath) {
|
||||
/* Select the fist path that appears to still be active.
|
||||
* This will eventually be user-configurable */
|
||||
for (int i=0; i<ZT_MAX_PEER_NETWORK_PATHS; i++) {
|
||||
if (_paths[i].p) {
|
||||
if (_activeBackupPath.ptr() == _paths[i].p.ptr()) {
|
||||
continue;
|
||||
}
|
||||
_activeBackupPath = _paths[i].p;
|
||||
if ((now - _paths[i].p->lastIn()) < ZT_MULTIPATH_ACTIVE_BACKUP_RAPID_FAILOVER_PERIOD) {
|
||||
bFoundHotPath = true;
|
||||
_activeBackupPath = _paths[i].p;
|
||||
_pathAssignmentIdx = i;
|
||||
_activeBackupPath->address().toString(curPathStr);
|
||||
fprintf(stderr, "selected %s as the primary active-backup path to %llx (idx=%d)\n",
|
||||
curPathStr, this->_id.address().toInt(), _pathAssignmentIdx);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else {
|
||||
char what[128];
|
||||
if ((now - _activeBackupPath->lastIn()) > ZT_MULTIPATH_ACTIVE_BACKUP_RAPID_FAILOVER_PERIOD) {
|
||||
_activeBackupPath->address().toString(curPathStr); // Record path string for later debug trace
|
||||
int16_t previousIdx = _pathAssignmentIdx;
|
||||
SharedPtr<Path> nextAlternativePath;
|
||||
// Search for a hot path, at the same time find the next path in
|
||||
// a RR sequence that seems viable to use as an alternative
|
||||
int searchCount = 0;
|
||||
while (searchCount < ZT_MAX_PEER_NETWORK_PATHS) {
|
||||
_pathAssignmentIdx++;
|
||||
if (_pathAssignmentIdx == ZT_MAX_PEER_NETWORK_PATHS) {
|
||||
_pathAssignmentIdx = 0;
|
||||
}
|
||||
searchCount++;
|
||||
if (_paths[_pathAssignmentIdx].p) {
|
||||
_paths[_pathAssignmentIdx].p->address().toString(what);
|
||||
if (_activeBackupPath.ptr() == _paths[_pathAssignmentIdx].p.ptr()) {
|
||||
continue;
|
||||
}
|
||||
if (!nextAlternativePath) { // Record the first viable alternative in the RR sequence
|
||||
nextAlternativePath = _paths[_pathAssignmentIdx].p;
|
||||
}
|
||||
if ((now - _paths[_pathAssignmentIdx].p->lastIn()) < ZT_MULTIPATH_ACTIVE_BACKUP_RAPID_FAILOVER_PERIOD) {
|
||||
bFoundHotPath = true;
|
||||
_activeBackupPath = _paths[_pathAssignmentIdx].p;
|
||||
_activeBackupPath->address().toString(newPathStr);
|
||||
fprintf(stderr, "primary active-backup path %s to %llx appears to be dead, switched to %s\n",
|
||||
curPathStr, this->_id.address().toInt(), newPathStr);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!bFoundHotPath) {
|
||||
if (nextAlternativePath) {
|
||||
_activeBackupPath = nextAlternativePath;
|
||||
_activeBackupPath->address().toString(curPathStr);
|
||||
//fprintf(stderr, "no hot paths found to use as active-backup primary to %llx, using next best: %s\n",
|
||||
// this->_id.address().toInt(), curPathStr);
|
||||
}
|
||||
else {
|
||||
// No change
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!_activeBackupPath) {
|
||||
return SharedPtr<Path>();
|
||||
}
|
||||
return _activeBackupPath;
|
||||
}
|
||||
|
||||
/**
|
||||
* Traffic is randomly distributed among all active paths.
|
||||
*/
|
||||
if (RR->node->getMultipathMode() == ZT_MULTIPATH_BALANCE_RANDOM) {
|
||||
int sz = _virtualPaths.size();
|
||||
if (sz) {
|
||||
int idx = _freeRandomByte % sz;
|
||||
_pathChoiceHist.push(idx);
|
||||
_virtualPaths[idx]->p->address().toString(curPathStr);
|
||||
fprintf(stderr, "sending out: (%llx), idx=%d: path=%s, localSocket=%lld\n",
|
||||
this->_id.address().toInt(), idx, curPathStr, _virtualPaths[idx]->localSocket);
|
||||
return _virtualPaths[idx]->p;
|
||||
}
|
||||
// This call is algorithmically inert but gives us a value to show in the status output
|
||||
computeAggregateAllocation(now);
|
||||
}
|
||||
|
||||
/**
|
||||
* Packets are striped across all available paths.
|
||||
*/
|
||||
if (RR->node->getMultipathMode() == ZT_MULTIPATH_BALANCE_RR_OPAQUE) {
|
||||
int16_t previousIdx = _roundRobinPathAssignmentIdx;
|
||||
int cycleCount = 0;
|
||||
int minLastIn = 0;
|
||||
int bestAlternativeIdx = -1;
|
||||
while (cycleCount < ZT_MAX_PEER_NETWORK_PATHS) {
|
||||
if (_roundRobinPathAssignmentIdx < (_virtualPaths.size()-1)) {
|
||||
_roundRobinPathAssignmentIdx++;
|
||||
}
|
||||
else {
|
||||
_roundRobinPathAssignmentIdx = 0;
|
||||
}
|
||||
cycleCount++;
|
||||
if (_virtualPaths[_roundRobinPathAssignmentIdx]->p) {
|
||||
uint64_t lastIn = _virtualPaths[_roundRobinPathAssignmentIdx]->p->lastIn();
|
||||
if (bestAlternativeIdx == -1) {
|
||||
minLastIn = lastIn; // Initialization
|
||||
bestAlternativeIdx = 0;
|
||||
}
|
||||
if (lastIn < minLastIn) {
|
||||
minLastIn = lastIn;
|
||||
bestAlternativeIdx = _roundRobinPathAssignmentIdx;
|
||||
}
|
||||
if ((now - lastIn) < 5000) {
|
||||
selectedPath = _virtualPaths[_roundRobinPathAssignmentIdx]->p;
|
||||
}
|
||||
}
|
||||
}
|
||||
// If we can't find an appropriate path, try the most recently active one
|
||||
if (!selectedPath) {
|
||||
_roundRobinPathAssignmentIdx = bestAlternativeIdx;
|
||||
selectedPath = _virtualPaths[bestAlternativeIdx]->p;
|
||||
selectedPath->address().toString(curPathStr);
|
||||
fprintf(stderr, "could not find good path, settling for next best %s\n",curPathStr);
|
||||
}
|
||||
selectedPath->address().toString(curPathStr);
|
||||
fprintf(stderr, "sending packet out on path %s at index %d\n",
|
||||
curPathStr, _roundRobinPathAssignmentIdx);
|
||||
return selectedPath;
|
||||
}
|
||||
|
||||
/**
|
||||
* Flows are striped across all available paths.
|
||||
*/
|
||||
if (RR->node->getMultipathMode() == ZT_MULTIPATH_BALANCE_RR_FLOW) {
|
||||
// fprintf(stderr, "ZT_MULTIPATH_BALANCE_RR_FLOW\n");
|
||||
}
|
||||
|
||||
/**
|
||||
* Flows are hashed across all available paths.
|
||||
*/
|
||||
if (RR->node->getMultipathMode() == ZT_MULTIPATH_BALANCE_XOR_FLOW) {
|
||||
// fprintf(stderr, "ZT_MULTIPATH_BALANCE_XOR_FLOW (%llx) \n", flowId);
|
||||
struct Flow *currFlow = NULL;
|
||||
if (_flows.count(flowId)) {
|
||||
currFlow = _flows[flowId];
|
||||
if (!currFlow->assignedPath) {
|
||||
int idx = abs((int)(currFlow->flowId % (_virtualPaths.size()-1)));
|
||||
currFlow->assignedPath = _virtualPaths[idx];
|
||||
_virtualPaths[idx]->p->address().toString(curPathStr);
|
||||
fprintf(stderr, "assigning flow %llx between this node and peer %llx to path %s at index %d\n",
|
||||
currFlow->flowId, this->_id.address().toInt(), curPathStr, idx);
|
||||
}
|
||||
else {
|
||||
if (!currFlow->assignedPath->p->alive(now)) {
|
||||
currFlow->assignedPath->p->address().toString(curPathStr);
|
||||
// Re-assign
|
||||
int idx = abs((int)(currFlow->flowId % (_virtualPaths.size()-1)));
|
||||
currFlow->assignedPath = _virtualPaths[idx];
|
||||
_virtualPaths[idx]->p->address().toString(newPathStr);
|
||||
fprintf(stderr, "path %s assigned to flow %llx between this node and %llx appears to be dead, reassigning to path %s\n",
|
||||
curPathStr, currFlow->flowId, this->_id.address().toInt(), newPathStr);
|
||||
}
|
||||
}
|
||||
return currFlow->assignedPath->p;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Proportionally allocate traffic according to dynamic path quality measurements.
|
||||
*/
|
||||
if (RR->node->getMultipathMode() == ZT_MULTIPATH_BALANCE_DYNAMIC_OPAQUE) {
|
||||
if ((now - _lastAggregateAllocation) >= ZT_PATH_QUALITY_COMPUTE_INTERVAL) {
|
||||
_lastAggregateAllocation = now;
|
||||
computeAggregateAllocation(now);
|
||||
}
|
||||
// Randomly choose path according to their allocations
|
||||
float rf = _freeRandomByte;
|
||||
for(int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
|
||||
if (_paths[i].p) {
|
||||
if (rf < _paths[i].p->allocation()) {
|
||||
bestPath = i;
|
||||
_pathChoiceHist.push(bestPath); // Record which path we chose
|
||||
break;
|
||||
}
|
||||
rf -= _paths[i].p->allocation();
|
||||
}
|
||||
}
|
||||
if (bestPath < ZT_MAX_PEER_NETWORK_PATHS) {
|
||||
return _paths[bestPath].p;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Flows are dynamically allocated across paths in proportion to link strength and load.
|
||||
*/
|
||||
if (RR->node->getMultipathMode() == ZT_MULTIPATH_BALANCE_DYNAMIC_FLOW) {
|
||||
}
|
||||
|
||||
return SharedPtr<Path>();
|
||||
}
|
||||
|
||||
char *Peer::interfaceListStr()
|
||||
{
|
||||
std::map<std::string, int> ifnamemap;
|
||||
char tmp[32];
|
||||
const int64_t now = RR->node->now();
|
||||
char *ptr = _interfaceListStr;
|
||||
bool imbalanced = false;
|
||||
memset(_interfaceListStr, 0, sizeof(_interfaceListStr));
|
||||
int alivePathCount = aggregateLinkLogicalPathCount();
|
||||
for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
|
||||
if (_paths[i].p && _paths[i].p->alive(now)) {
|
||||
int ipv = _paths[i].p->address().isV4();
|
||||
// If this is acting as an aggregate link, check allocations
|
||||
float targetAllocation = 1.0f / (float)alivePathCount;
|
||||
float currentAllocation = 1.0f;
|
||||
if (alivePathCount > 1) {
|
||||
currentAllocation = (float)_pathChoiceHist.countValue(i) / (float)_pathChoiceHist.count();
|
||||
if (fabs(targetAllocation - currentAllocation) > ZT_PATH_IMBALANCE_THRESHOLD) {
|
||||
imbalanced = true;
|
||||
}
|
||||
}
|
||||
char *ipvStr = ipv ? (char*)"ipv4" : (char*)"ipv6";
|
||||
sprintf(tmp, "(%s, %s, %.3f)", _paths[i].p->getName(), ipvStr, currentAllocation);
|
||||
// Prevent duplicates
|
||||
if(ifnamemap[_paths[i].p->getName()] != ipv) {
|
||||
memcpy(ptr, tmp, strlen(tmp));
|
||||
ptr += strlen(tmp);
|
||||
*ptr = ' ';
|
||||
ptr++;
|
||||
ifnamemap[_paths[i].p->getName()] = ipv;
|
||||
}
|
||||
}
|
||||
}
|
||||
ptr--; // Overwrite trailing space
|
||||
if (imbalanced) {
|
||||
sprintf(tmp, ", is asymmetrical");
|
||||
memcpy(ptr, tmp, sizeof(tmp));
|
||||
} else {
|
||||
*ptr = '\0';
|
||||
}
|
||||
return _interfaceListStr;
|
||||
return _bondToPeer->getAppropriatePath(now, flowId);
|
||||
}
|
||||
|
||||
void Peer::introduce(void *const tPtr,const int64_t now,const SharedPtr<Peer> &other) const
|
||||
|
|
@ -859,87 +360,6 @@ void Peer::introduce(void *const tPtr,const int64_t now,const SharedPtr<Peer> &o
|
|||
}
|
||||
}
|
||||
|
||||
inline void Peer::processBackgroundPeerTasks(const int64_t now)
|
||||
{
|
||||
// Determine current multipath compatibility with other peer
|
||||
if ((now - _lastMultipathCompatibilityCheck) >= ZT_PATH_QUALITY_COMPUTE_INTERVAL) {
|
||||
//
|
||||
// Cache number of available paths so that we can short-circuit multipath logic elsewhere
|
||||
//
|
||||
// We also take notice of duplicate paths (same IP only) because we may have
|
||||
// recently received a direct path push from a peer and our list might contain
|
||||
// a dead path which hasn't been fully recognized as such. In this case we
|
||||
// don't want the duplicate to trigger execution of multipath code prematurely.
|
||||
//
|
||||
// This is done to support the behavior of auto multipath enable/disable
|
||||
// without user intervention.
|
||||
//
|
||||
int currAlivePathCount = 0;
|
||||
int duplicatePathsFound = 0;
|
||||
for (unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
|
||||
if (_paths[i].p) {
|
||||
currAlivePathCount++;
|
||||
for (unsigned int j=0;j<ZT_MAX_PEER_NETWORK_PATHS;++j) {
|
||||
if (_paths[i].p && _paths[j].p && _paths[i].p->address().ipsEqual2(_paths[j].p->address()) && i != j) {
|
||||
duplicatePathsFound+=1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
_uniqueAlivePathCount = (currAlivePathCount - (duplicatePathsFound / 2));
|
||||
_lastMultipathCompatibilityCheck = now;
|
||||
_localMultipathSupported = ((RR->node->getMultipathMode() != ZT_MULTIPATH_NONE) && (ZT_PROTO_VERSION > 9));
|
||||
_remoteMultipathSupported = _vProto > 9;
|
||||
// If both peers support multipath and more than one path exist, we can use multipath logic
|
||||
_canUseMultipath = _localMultipathSupported && _remoteMultipathSupported && (_uniqueAlivePathCount > 1);
|
||||
}
|
||||
|
||||
// Remove old flows
|
||||
if (RR->sw->isFlowAware()) {
|
||||
std::map<int64_t, struct Flow *>::iterator it = _flows.begin();
|
||||
while (it != _flows.end()) {
|
||||
if ((now - it->second->lastSend) > ZT_MULTIPATH_FLOW_EXPIRATION) {
|
||||
fprintf(stderr, "forgetting flow %llx between this node and %llx (%lu active flow(s))\n",
|
||||
it->first, this->_id.address().toInt(), _flows.size());
|
||||
it = _flows.erase(it);
|
||||
} else {
|
||||
it++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void Peer::sendACK(void *tPtr,const SharedPtr<Path> &path,const int64_t localSocket,const InetAddress &atAddress,int64_t now)
|
||||
{
|
||||
Packet outp(_id.address(),RR->identity.address(),Packet::VERB_ACK);
|
||||
uint32_t bytesToAck = path->bytesToAck();
|
||||
outp.append<uint32_t>(bytesToAck);
|
||||
if (atAddress) {
|
||||
outp.armor(_key,false);
|
||||
RR->node->putPacket(tPtr,localSocket,atAddress,outp.data(),outp.size());
|
||||
} else {
|
||||
RR->sw->send(tPtr,outp,false);
|
||||
}
|
||||
path->sentAck(now);
|
||||
}
|
||||
|
||||
void Peer::sendQOS_MEASUREMENT(void *tPtr,const SharedPtr<Path> &path,const int64_t localSocket,const InetAddress &atAddress,int64_t now)
|
||||
{
|
||||
const int64_t _now = RR->node->now();
|
||||
Packet outp(_id.address(),RR->identity.address(),Packet::VERB_QOS_MEASUREMENT);
|
||||
char qosData[ZT_PATH_MAX_QOS_PACKET_SZ];
|
||||
int16_t len = path->generateQoSPacket(_now,qosData);
|
||||
outp.append(qosData,len);
|
||||
if (atAddress) {
|
||||
outp.armor(_key,false);
|
||||
RR->node->putPacket(tPtr,localSocket,atAddress,outp.data(),outp.size());
|
||||
} else {
|
||||
RR->sw->send(tPtr,outp,false);
|
||||
}
|
||||
path->sentQoS(now);
|
||||
}
|
||||
|
||||
void Peer::sendHELLO(void *tPtr,const int64_t localSocket,const InetAddress &atAddress,int64_t now)
|
||||
{
|
||||
Packet outp(_id.address(),RR->identity.address(),Packet::VERB_HELLO);
|
||||
|
|
@ -1005,29 +425,57 @@ void Peer::tryMemorizedPath(void *tPtr,int64_t now)
|
|||
}
|
||||
}
|
||||
|
||||
void Peer::performMultipathStateCheck(int64_t now)
|
||||
{
|
||||
/**
|
||||
* Check for conditions required for multipath bonding and create a bond
|
||||
* if allowed.
|
||||
*/
|
||||
_localMultipathSupported = ((RR->bc->inUse()) && (ZT_PROTO_VERSION > 9));
|
||||
if (_localMultipathSupported) {
|
||||
int currAlivePathCount = 0;
|
||||
int duplicatePathsFound = 0;
|
||||
for (unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
|
||||
if (_paths[i].p) {
|
||||
currAlivePathCount++;
|
||||
for (unsigned int j=0;j<ZT_MAX_PEER_NETWORK_PATHS;++j) {
|
||||
if (_paths[i].p && _paths[j].p && _paths[i].p->address().ipsEqual2(_paths[j].p->address()) && i != j) {
|
||||
duplicatePathsFound+=1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
_uniqueAlivePathCount = (currAlivePathCount - (duplicatePathsFound / 2));
|
||||
_remoteMultipathSupported = _vProto > 9;
|
||||
_canUseMultipath = _localMultipathSupported && _remoteMultipathSupported && (_uniqueAlivePathCount > 1);
|
||||
}
|
||||
if (_canUseMultipath && !_bondToPeer) {
|
||||
if (RR->bc) {
|
||||
_bondToPeer = RR->bc->createTransportTriggeredBond(RR, this);
|
||||
/**
|
||||
* Allow new bond to retroactively learn all paths known to this peer
|
||||
*/
|
||||
if (_bondToPeer) {
|
||||
for (unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
|
||||
if (_paths[i].p) {
|
||||
_bondToPeer->nominatePath(_paths[i].p, now);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
unsigned int Peer::doPingAndKeepalive(void *tPtr,int64_t now)
|
||||
{
|
||||
unsigned int sent = 0;
|
||||
Mutex::Lock _l(_paths_m);
|
||||
|
||||
processBackgroundPeerTasks(now);
|
||||
performMultipathStateCheck(now);
|
||||
|
||||
// Emit traces regarding aggregate link status
|
||||
if (_canUseMultipath) {
|
||||
int alivePathCount = aggregateLinkPhysicalPathCount();
|
||||
if ((now - _lastAggregateStatsReport) > ZT_PATH_AGGREGATE_STATS_REPORT_INTERVAL) {
|
||||
_lastAggregateStatsReport = now;
|
||||
if (alivePathCount) {
|
||||
RR->t->peerLinkAggregateStatistics(NULL,*this);
|
||||
}
|
||||
} if (alivePathCount < 2 && _linkIsRedundant) {
|
||||
_linkIsRedundant = !_linkIsRedundant;
|
||||
RR->t->peerLinkNoLongerAggregate(NULL,*this);
|
||||
} if (alivePathCount > 1 && !_linkIsRedundant) {
|
||||
_linkIsRedundant = !_linkIsRedundant;
|
||||
RR->t->peerLinkNoLongerAggregate(NULL,*this);
|
||||
}
|
||||
}
|
||||
const bool sendFullHello = ((now - _lastSentFullHello) >= ZT_PEER_PING_PERIOD);
|
||||
_lastSentFullHello = now;
|
||||
|
||||
// Right now we only keep pinging links that have the maximum priority. The
|
||||
// priority is used to track cluster redirections, meaning that when a cluster
|
||||
|
|
@ -1040,15 +488,13 @@ unsigned int Peer::doPingAndKeepalive(void *tPtr,int64_t now)
|
|||
else break;
|
||||
}
|
||||
|
||||
const bool sendFullHello = ((now - _lastSentFullHello) >= ZT_PEER_PING_PERIOD);
|
||||
_lastSentFullHello = now;
|
||||
|
||||
unsigned int j = 0;
|
||||
for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
|
||||
if (_paths[i].p) {
|
||||
// Clean expired and reduced priority paths
|
||||
if ( ((now - _paths[i].lr) < ZT_PEER_PATH_EXPIRATION) && (_paths[i].priority == maxPriority) ) {
|
||||
if ((sendFullHello)||(_paths[i].p->needsHeartbeat(now))) {
|
||||
if ((sendFullHello)||(_paths[i].p->needsHeartbeat(now))
|
||||
|| (_canUseMultipath && _paths[i].p->needsGratuitousHeartbeat(now))) {
|
||||
attemptToContactAt(tPtr,_paths[i].p->localSocket(),_paths[i].p->address(),now,sendFullHello);
|
||||
_paths[i].p->sent(now);
|
||||
sent |= (_paths[i].p->address().ss_family == AF_INET) ? 0x1 : 0x2;
|
||||
|
|
@ -1059,14 +505,6 @@ unsigned int Peer::doPingAndKeepalive(void *tPtr,int64_t now)
|
|||
}
|
||||
} else break;
|
||||
}
|
||||
if (canUseMultipath()) {
|
||||
while(j < ZT_MAX_PEER_NETWORK_PATHS) {
|
||||
_paths[j].lr = 0;
|
||||
_paths[j].p.zero();
|
||||
_paths[j].priority = 1;
|
||||
++j;
|
||||
}
|
||||
}
|
||||
return sent;
|
||||
}
|
||||
|
||||
|
|
@ -1133,4 +571,30 @@ void Peer::resetWithinScope(void *tPtr,InetAddress::IpScope scope,int inetAddres
|
|||
}
|
||||
}
|
||||
|
||||
void Peer::recordOutgoingPacket(const SharedPtr<Path> &path, const uint64_t packetId,
|
||||
uint16_t payloadLength, const Packet::Verb verb, const int32_t flowId, int64_t now)
|
||||
{
|
||||
if (!_shouldCollectPathStatistics || !_bondToPeer) {
|
||||
return;
|
||||
}
|
||||
_bondToPeer->recordOutgoingPacket(path, packetId, payloadLength, verb, flowId, now);
|
||||
}
|
||||
|
||||
void Peer::recordIncomingInvalidPacket(const SharedPtr<Path>& path)
|
||||
{
|
||||
if (!_shouldCollectPathStatistics || !_bondToPeer) {
|
||||
return;
|
||||
}
|
||||
_bondToPeer->recordIncomingInvalidPacket(path);
|
||||
}
|
||||
|
||||
void Peer::recordIncomingPacket(void *tPtr, const SharedPtr<Path> &path, const uint64_t packetId,
|
||||
uint16_t payloadLength, const Packet::Verb verb, const int32_t flowId, int64_t now)
|
||||
{
|
||||
if (!_shouldCollectPathStatistics || !_bondToPeer) {
|
||||
return;
|
||||
}
|
||||
_bondToPeer->recordIncomingPacket(path, packetId, payloadLength, verb, flowId, now);
|
||||
}
|
||||
|
||||
} // namespace ZeroTier
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue