Add Bonds, Slaves, and Flows

This commit is contained in:
Joseph Henry 2020-05-12 01:35:48 -07:00
parent de9cfbe9b0
commit a50e8e9878
31 changed files with 4898 additions and 1966 deletions

View file

@ -1,10 +1,10 @@
/*
* Copyright (c)2019 ZeroTier, Inc.
* Copyright (c)2013-2020 ZeroTier, Inc.
*
* Use of this software is governed by the Business Source License included
* in the LICENSE.TXT file in the project's root directory.
*
* Change Date: 2023-01-01
* Change Date: 2024-01-01
*
* On the date above, in accordance with the Business Source License, use
* of this software will be governed by version 2.0 of the Apache License.
@ -15,8 +15,6 @@
#define ZT_PEER_HPP
#include <vector>
#include <map>
#include <queue>
#include "../include/ZeroTierOne.h"
@ -33,6 +31,8 @@
#include "AtomicCounter.hpp"
#include "Hashtable.hpp"
#include "Mutex.hpp"
#include "Bond.hpp"
#include "BondController.hpp"
#define ZT_PEER_MAX_SERIALIZED_STATE_SIZE (sizeof(Peer) + 32 + (sizeof(Path) * 2))
@ -44,6 +44,9 @@ namespace ZeroTier {
class Peer
{
friend class SharedPtr<Peer>;
friend class SharedPtr<Bond>;
friend class Switch;
friend class Bond;
private:
Peer() {} // disabled to prevent bugs -- should not be constructed uninitialized
@ -97,7 +100,8 @@ public:
const uint64_t inRePacketId,
const Packet::Verb inReVerb,
const bool trustEstablished,
const uint64_t networkId);
const uint64_t networkId,
const int32_t flowId);
/**
* Check whether we have an active path to this peer via the given address
@ -136,94 +140,49 @@ public:
return false;
}
void constructSetOfVirtualPaths();
/**
* Record statistics on outgoing packets
*
* @param path Path over which packet was sent
* @param id Packet ID
* @param len Length of packet payload
* @param verb Packet verb
* @param now Current time
*/
void recordOutgoingPacket(const SharedPtr<Path> &path, const uint64_t packetId, uint16_t payloadLength, const Packet::Verb verb, int64_t now);
/**
* Record statistics on incoming packets
*
* @param path Path over which packet was sent
* @param id Packet ID
* @param len Length of packet payload
* @param verb Packet verb
* @param now Current time
*/
void recordIncomingPacket(void *tPtr, const SharedPtr<Path> &path, const uint64_t packetId, uint16_t payloadLength, const Packet::Verb verb, int64_t now);
/**
* Send an ACK to peer for the most recent packets received
* Record incoming packets to
*
* @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call
* @param localSocket Raw socket the ACK packet will be sent over
* @param atAddress Destination for the ACK packet
* @param path Path over which packet was received
* @param packetId Packet ID
* @param payloadLength Length of packet data payload
* @param verb Packet verb
* @param flowId Flow ID
* @param now Current time
*/
void sendACK(void *tPtr, const SharedPtr<Path> &path, const int64_t localSocket,const InetAddress &atAddress,int64_t now);
void recordIncomingPacket(void *tPtr, const SharedPtr<Path> &path, const uint64_t packetId,
uint16_t payloadLength, const Packet::Verb verb, const int32_t flowId, int64_t now);
/**
* Send a QoS packet to peer so that it can evaluate the quality of this link
*
* @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call
* @param localSocket Raw socket the QoS packet will be sent over
* @param atAddress Destination for the QoS packet
* @param path Path over which packet is being sent
* @param packetId Packet ID
* @param payloadLength Length of packet data payload
* @param verb Packet verb
* @param flowId Flow ID
* @param now Current time
*/
void sendQOS_MEASUREMENT(void *tPtr, const SharedPtr<Path> &path, const int64_t localSocket,const InetAddress &atAddress,int64_t now);
void recordOutgoingPacket(const SharedPtr<Path> &path, const uint64_t packetId,
uint16_t payloadLength, const Packet::Verb verb, const int32_t flowId, int64_t now);
/**
* Compute relative quality values and allocations for the components of the aggregate link
* Record an invalid incoming packet. This packet failed
* MAC/compression/cipher checks and will now contribute to a
* Packet Error Ratio (PER).
*
* @param now Current time
* @param path Path over which packet was received
*/
void computeAggregateAllocation(int64_t now);
/**
* @return The aggregate link Packet Delay Variance (PDV)
*/
int computeAggregateLinkPacketDelayVariance();
/**
* @return The aggregate link mean latency
*/
int computeAggregateLinkMeanLatency();
/**
* @return The number of currently alive "physical" paths in the aggregate link
*/
int aggregateLinkPhysicalPathCount();
/**
* @return The number of currently alive "logical" paths in the aggregate link
*/
int aggregateLinkLogicalPathCount();
std::vector<SharedPtr<Path>> getAllPaths(int64_t now);
void recordIncomingInvalidPacket(const SharedPtr<Path>& path);
/**
* Get the most appropriate direct path based on current multipath and QoS configuration
*
* @param now Current time
* @param flowId Session-specific protocol flow identifier used for path allocation
* @param includeExpired If true, include even expired paths
* @return Best current path or NULL if none
*/
SharedPtr<Path> getAppropriatePath(int64_t now, bool includeExpired, int64_t flowId = -1);
/**
* Generate a human-readable string of interface names making up the aggregate link, also include
* moving allocation and IP version number for each (for tracing)
*/
char *interfaceListStr();
SharedPtr<Path> getAppropriatePath(int64_t now, bool includeExpired, int32_t flowId = -1);
/**
* Send VERB_RENDEZVOUS to this and another peer via the best common IP scope and path
@ -265,6 +224,13 @@ public:
*/
void tryMemorizedPath(void *tPtr,int64_t now);
/**
* A check to be performed periodically which determines whether multipath communication is
* possible with this peer. This check should be performed early in the life-cycle of the peer
* as well as during the process of learning new paths.
*/
void performMultipathStateCheck(int64_t now);
/**
* Send pings or keepalives depending on configured timeouts
*
@ -277,16 +243,6 @@ public:
*/
unsigned int doPingAndKeepalive(void *tPtr,int64_t now);
/**
* Clear paths whose localSocket(s) are in a CLOSED state or have an otherwise INVALID state.
* This should be called frequently so that we can detect and remove unproductive or invalid paths.
*
* Under the hood this is done periodically based on ZT_CLOSED_PATH_PRUNING_INTERVAL.
*
* @return Number of paths that were pruned this round
*/
unsigned int prunePaths();
/**
* Process a cluster redirect sent by this peer
*
@ -348,7 +304,7 @@ public:
inline unsigned int latency(const int64_t now)
{
if (_canUseMultipath) {
return (int)computeAggregateLinkMeanLatency();
return (int)_lastComputedAggregateMeanLatency;
} else {
SharedPtr<Path> bp(getAppropriatePath(now,false));
if (bp)
@ -407,37 +363,6 @@ public:
inline bool remoteVersionKnown() const { return ((_vMajor > 0)||(_vMinor > 0)||(_vRevision > 0)); }
/**
* Periodically update known multipath activation constraints. This is done so that we know when and when
* not to use multipath logic. Doing this once every few seconds is sufficient.
*
* @param now Current time
*/
inline void processBackgroundPeerTasks(const int64_t now);
/**
* Record that the remote peer does have multipath enabled. As is evident by the receipt of a VERB_ACK
* or a VERB_QOS_MEASUREMENT packet at some point in the past. Until this flag is set, the local client
* shall assume that multipath is not enabled and should only use classical Protocol 9 logic.
*/
inline void inferRemoteMultipathEnabled() { _remotePeerMultipathEnabled = true; }
/**
* @return Whether the local client supports and is configured to use multipath
*/
inline bool localMultipathSupport() { return _localMultipathSupported; }
/**
* @return Whether the remote peer supports and is configured to use multipath
*/
inline bool remoteMultipathSupport() { return _remoteMultipathSupported; }
/**
* @return Whether this client can use multipath to communicate with this peer. True if both peers are using
* the correct protocol and if both peers have multipath enabled. False if otherwise.
*/
inline bool canUseMultipath() { return _canUseMultipath; }
/**
* @return True if peer has received a trust established packet (e.g. common network membership) in the past ZT_TRUST_EXPIRATION ms
*/
@ -492,50 +417,35 @@ public:
}
/**
* Rate limit gate for inbound ECHO requests
* Rate limit gate for inbound ECHO requests. This rate limiter works
* by draining a certain number of requests per unit time. Each peer may
* theoretically receive up to ZT_ECHO_CUTOFF_LIMIT requests per second.
*/
inline bool rateGateEchoRequest(const int64_t now)
{
if ((now - _lastEchoRequestReceived) >= ZT_PEER_GENERAL_RATE_LIMIT) {
_lastEchoRequestReceived = now;
return true;
}
return false;
}
/**
* Rate limit gate for VERB_ACK
*/
inline bool rateGateACK(const int64_t now)
{
if ((now - _lastACKWindowReset) >= ZT_PATH_QOS_ACK_CUTOFF_TIME) {
_lastACKWindowReset = now;
_ACKCutoffCount = 0;
/*
// TODO: Rethink this
if (_canUseMultipath) {
_echoRequestCutoffCount++;
int numToDrain = (now - _lastEchoCheck) / ZT_ECHO_DRAINAGE_DIVISOR;
_lastEchoCheck = now;
fprintf(stderr, "ZT_ECHO_CUTOFF_LIMIT=%d, (now - _lastEchoCheck)=%d, numToDrain=%d, ZT_ECHO_DRAINAGE_DIVISOR=%d\n", ZT_ECHO_CUTOFF_LIMIT, (now - _lastEchoCheck), numToDrain, ZT_ECHO_DRAINAGE_DIVISOR);
if (_echoRequestCutoffCount > numToDrain) {
_echoRequestCutoffCount-=numToDrain;
}
else {
_echoRequestCutoffCount = 0;
}
return (_echoRequestCutoffCount < ZT_ECHO_CUTOFF_LIMIT);
} else {
++_ACKCutoffCount;
if ((now - _lastEchoRequestReceived) >= (ZT_PEER_GENERAL_RATE_LIMIT)) {
_lastEchoRequestReceived = now;
return true;
}
return false;
}
return (_ACKCutoffCount < ZT_PATH_QOS_ACK_CUTOFF_LIMIT);
}
/**
* Rate limit gate for VERB_QOS_MEASUREMENT
*/
inline bool rateGateQoS(const int64_t now)
{
if ((now - _lastQoSWindowReset) >= ZT_PATH_QOS_ACK_CUTOFF_TIME) {
_lastQoSWindowReset = now;
_QoSCutoffCount = 0;
} else {
++_QoSCutoffCount;
}
return (_QoSCutoffCount < ZT_PATH_QOS_ACK_CUTOFF_LIMIT);
}
/**
* @return Whether this peer is reachable via an aggregate link
*/
inline bool hasAggregateLink() {
return _localMultipathSupported && _remoteMultipathSupported && _remotePeerMultipathEnabled;
*/
return true;
}
/**
@ -610,6 +520,18 @@ public:
}
}
/**
*
* @return
*/
SharedPtr<Bond> bond() { return _bondToPeer; }
/**
*
* @return
*/
inline int8_t bondingPolicy() { return _bondingPolicy; }
private:
struct _PeerPath
{
@ -628,25 +550,16 @@ private:
int64_t _lastTriedMemorizedPath;
int64_t _lastDirectPathPushSent;
int64_t _lastDirectPathPushReceive;
int64_t _lastEchoRequestReceived;
int64_t _lastCredentialRequestSent;
int64_t _lastWhoisRequestReceived;
int64_t _lastEchoRequestReceived;
int64_t _lastCredentialsReceived;
int64_t _lastTrustEstablishedPacketReceived;
int64_t _lastSentFullHello;
int64_t _lastPathPrune;
int64_t _lastACKWindowReset;
int64_t _lastQoSWindowReset;
int64_t _lastMultipathCompatibilityCheck;
int64_t _lastEchoCheck;
unsigned char _freeRandomByte;
int _uniqueAlivePathCount;
bool _localMultipathSupported;
bool _remoteMultipathSupported;
bool _canUseMultipath;
uint16_t _vProto;
uint16_t _vMajor;
uint16_t _vMinor;
@ -659,62 +572,22 @@ private:
unsigned int _directPathPushCutoffCount;
unsigned int _credentialsCutoffCount;
unsigned int _QoSCutoffCount;
unsigned int _ACKCutoffCount;
unsigned int _echoRequestCutoffCount;
AtomicCounter __refCount;
RingBuffer<int,ZT_MULTIPATH_PROPORTION_WIN_SZ> _pathChoiceHist;
bool _linkIsBalanced;
bool _linkIsRedundant;
bool _remotePeerMultipathEnabled;
int _uniqueAlivePathCount;
bool _localMultipathSupported;
bool _remoteMultipathSupported;
bool _canUseMultipath;
int64_t _lastAggregateStatsReport;
int64_t _lastAggregateAllocation;
volatile bool _shouldCollectPathStatistics;
volatile int8_t _bondingPolicy;
char _interfaceListStr[256]; // 16 characters * 16 paths in a link
int32_t _lastComputedAggregateMeanLatency;
//
struct LinkPerformanceEntry
{
int64_t packetId;
struct VirtualPath *egressVirtualPath;
struct VirtualPath *ingressVirtualPath;
};
// Virtual paths
int _virtualPathCount;
Mutex _virtual_paths_m;
struct VirtualPath
{
SharedPtr<Path> p;
int64_t localSocket;
std::queue<struct LinkPerformanceEntry *> performanceEntries;
};
std::vector<struct VirtualPath*> _virtualPaths;
// Flows
struct Flow
{
Flow(int64_t fid, int64_t ls) :
flowId(fid),
lastSend(ls),
assignedPath(NULL)
{}
int64_t flowId;
int64_t bytesPerSecond;
int64_t lastSend;
struct VirtualPath *assignedPath;
};
std::map<int64_t, struct Flow *> _flows;
int16_t _roundRobinPathAssignmentIdx;
SharedPtr<Path> _activeBackupPath;
int16_t _pathAssignmentIdx;
SharedPtr<Bond> _bondToPeer;
};
} // namespace ZeroTier