Add Bonds, Slaves, and Flows
This commit is contained in:
Joseph Henry
parent
de9cfbe9b0
commit
a50e8e9878
31 changed files with 4898 additions and 1966 deletions
809
node/Path.hpp
809
node/Path.hpp
|
|
@ -1,10 +1,10 @@
|
|||
/*
|
||||
* Copyright (c)2019 ZeroTier, Inc.
|
||||
* Copyright (c)2013-2020 ZeroTier, Inc.
|
||||
*
|
||||
* Use of this software is governed by the Business Source License included
|
||||
* in the LICENSE.TXT file in the project's root directory.
|
||||
*
|
||||
* Change Date: 2023-01-01
|
||||
* Change Date: 2024-01-01
|
||||
*
|
||||
* On the date above, in accordance with the Business Source License, use
|
||||
* of this software will be governed by version 2.0 of the Apache License.
|
||||
|
|
@ -26,12 +26,11 @@
|
|||
#include "SharedPtr.hpp"
|
||||
#include "AtomicCounter.hpp"
|
||||
#include "Utils.hpp"
|
||||
#include "RingBuffer.hpp"
|
||||
#include "Packet.hpp"
|
||||
#include "RingBuffer.hpp"
|
||||
//#include "Bond.hpp"
|
||||
|
||||
#include "../osdep/Phy.hpp"
|
||||
|
||||
#include "../include/ZeroTierDebug.h"
|
||||
#include "../osdep/Slave.hpp"
|
||||
|
||||
/**
|
||||
* Maximum return value of preferenceRank()
|
||||
|
|
@ -48,7 +47,8 @@ class RuntimeEnvironment;
|
|||
class Path
|
||||
{
|
||||
friend class SharedPtr<Path>;
|
||||
Phy<Path *> *_phy;
|
||||
friend class Bond;
|
||||
//friend class SharedPtr<Bond>;
|
||||
|
||||
public:
|
||||
/**
|
||||
|
|
@ -87,77 +87,113 @@ public:
|
|||
_lastOut(0),
|
||||
_lastIn(0),
|
||||
_lastTrustEstablishedPacketReceived(0),
|
||||
_lastPathQualityComputeTime(0),
|
||||
_localSocket(-1),
|
||||
_latency(0xffff),
|
||||
_addr(),
|
||||
_ipScope(InetAddress::IP_SCOPE_NONE),
|
||||
_lastAck(0),
|
||||
_lastThroughputEstimation(0),
|
||||
_lastAckReceived(0),
|
||||
_lastAckSent(0),
|
||||
_lastQoSMeasurement(0),
|
||||
_lastQoSRecordPurge(0),
|
||||
_lastThroughputEstimation(0),
|
||||
_lastRefractoryUpdate(0),
|
||||
_lastAliveToggle(0),
|
||||
_lastEligibilityState(false),
|
||||
_lastTrialBegin(0),
|
||||
_refractoryPeriod(0),
|
||||
_monitorInterval(0),
|
||||
_upDelay(0),
|
||||
_downDelay(0),
|
||||
_ipvPref(0),
|
||||
_mode(0),
|
||||
_onlyPathOnSlave(false),
|
||||
_enabled(false),
|
||||
_bonded(false),
|
||||
_negotiated(false),
|
||||
_deprecated(false),
|
||||
_shouldReallocateFlows(false),
|
||||
_assignedFlowCount(0),
|
||||
_latencyMean(0),
|
||||
_latencyVariance(0),
|
||||
_packetLossRatio(0),
|
||||
_packetErrorRatio(0),
|
||||
_throughputMean(0),
|
||||
_throughputMax(0),
|
||||
_throughputVariance(0),
|
||||
_allocation(0),
|
||||
_byteLoad(0),
|
||||
_relativeByteLoad(0),
|
||||
_affinity(0),
|
||||
_failoverScore(0),
|
||||
_unackedBytes(0),
|
||||
_expectingAckAsOf(0),
|
||||
_packetsReceivedSinceLastAck(0),
|
||||
_packetsReceivedSinceLastQoS(0),
|
||||
_maxLifetimeThroughput(0),
|
||||
_lastComputedMeanThroughput(0),
|
||||
_bytesAckedSinceLastThroughputEstimation(0),
|
||||
_lastComputedMeanLatency(0.0),
|
||||
_lastComputedPacketDelayVariance(0.0),
|
||||
_lastComputedPacketErrorRatio(0.0),
|
||||
_lastComputedPacketLossRatio(0),
|
||||
_lastComputedStability(0.0),
|
||||
_lastComputedRelativeQuality(0),
|
||||
_lastComputedThroughputDistCoeff(0.0),
|
||||
_lastAllocation(0)
|
||||
{
|
||||
memset(_ifname, 0, 16);
|
||||
memset(_addrString, 0, sizeof(_addrString));
|
||||
}
|
||||
_packetsIn(0),
|
||||
_packetsOut(0),
|
||||
_prevEligibility(false)
|
||||
{}
|
||||
|
||||
Path(const int64_t localSocket,const InetAddress &addr) :
|
||||
_lastOut(0),
|
||||
_lastIn(0),
|
||||
_lastTrustEstablishedPacketReceived(0),
|
||||
_lastPathQualityComputeTime(0),
|
||||
_localSocket(localSocket),
|
||||
_latency(0xffff),
|
||||
_addr(addr),
|
||||
_ipScope(addr.ipScope()),
|
||||
_lastAck(0),
|
||||
_lastThroughputEstimation(0),
|
||||
_lastAckReceived(0),
|
||||
_lastAckSent(0),
|
||||
_lastQoSMeasurement(0),
|
||||
_lastQoSRecordPurge(0),
|
||||
_lastThroughputEstimation(0),
|
||||
_lastRefractoryUpdate(0),
|
||||
_lastAliveToggle(0),
|
||||
_lastEligibilityState(false),
|
||||
_lastTrialBegin(0),
|
||||
_refractoryPeriod(0),
|
||||
_monitorInterval(0),
|
||||
_upDelay(0),
|
||||
_downDelay(0),
|
||||
_ipvPref(0),
|
||||
_mode(0),
|
||||
_onlyPathOnSlave(false),
|
||||
_enabled(false),
|
||||
_bonded(false),
|
||||
_negotiated(false),
|
||||
_deprecated(false),
|
||||
_shouldReallocateFlows(false),
|
||||
_assignedFlowCount(0),
|
||||
_latencyMean(0),
|
||||
_latencyVariance(0),
|
||||
_packetLossRatio(0),
|
||||
_packetErrorRatio(0),
|
||||
_throughputMean(0),
|
||||
_throughputMax(0),
|
||||
_throughputVariance(0),
|
||||
_allocation(0),
|
||||
_byteLoad(0),
|
||||
_relativeByteLoad(0),
|
||||
_affinity(0),
|
||||
_failoverScore(0),
|
||||
_unackedBytes(0),
|
||||
_expectingAckAsOf(0),
|
||||
_packetsReceivedSinceLastAck(0),
|
||||
_packetsReceivedSinceLastQoS(0),
|
||||
_maxLifetimeThroughput(0),
|
||||
_lastComputedMeanThroughput(0),
|
||||
_bytesAckedSinceLastThroughputEstimation(0),
|
||||
_lastComputedMeanLatency(0.0),
|
||||
_lastComputedPacketDelayVariance(0.0),
|
||||
_lastComputedPacketErrorRatio(0.0),
|
||||
_lastComputedPacketLossRatio(0),
|
||||
_lastComputedStability(0.0),
|
||||
_lastComputedRelativeQuality(0),
|
||||
_lastComputedThroughputDistCoeff(0.0),
|
||||
_lastAllocation(0)
|
||||
{
|
||||
memset(_ifname, 0, 16);
|
||||
memset(_addrString, 0, sizeof(_addrString));
|
||||
if (_localSocket != -1) {
|
||||
_phy->getIfName((PhySocket *) ((uintptr_t) _localSocket), _ifname, 16);
|
||||
}
|
||||
}
|
||||
_packetsIn(0),
|
||||
_packetsOut(0),
|
||||
_prevEligibility(false)
|
||||
{}
|
||||
|
||||
/**
|
||||
* Called when a packet is received from this remote path, regardless of content
|
||||
*
|
||||
* @param t Time of receive
|
||||
*/
|
||||
inline void received(const uint64_t t) { _lastIn = t; }
|
||||
inline void received(const uint64_t t) {
|
||||
_lastIn = t;
|
||||
if (!_prevEligibility) {
|
||||
_lastAliveToggle = _lastIn;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Set time last trusted packet was received (done in Peer::received())
|
||||
|
|
@ -197,7 +233,6 @@ public:
|
|||
else {
|
||||
_latency = l;
|
||||
}
|
||||
_latencySamples.push(l);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -286,341 +321,32 @@ public:
|
|||
}
|
||||
|
||||
/**
|
||||
* Record statistics on outgoing packets. Used later to estimate QoS metrics.
|
||||
*
|
||||
* @param now Current time
|
||||
* @param packetId ID of packet
|
||||
* @param payloadLength Length of payload
|
||||
* @param verb Packet verb
|
||||
* @param bonded Whether this path is part of a bond.
|
||||
*/
|
||||
inline void recordOutgoingPacket(int64_t now, int64_t packetId, uint16_t payloadLength, Packet::Verb verb)
|
||||
{
|
||||
Mutex::Lock _l(_statistics_m);
|
||||
if (verb != Packet::VERB_ACK && verb != Packet::VERB_QOS_MEASUREMENT) {
|
||||
if ((packetId & (ZT_PATH_QOS_ACK_PROTOCOL_DIVISOR - 1)) == 0) {
|
||||
_unackedBytes += payloadLength;
|
||||
// Take note that we're expecting a VERB_ACK on this path as of a specific time
|
||||
_expectingAckAsOf = ackAge(now) > ZT_PATH_ACK_INTERVAL ? _expectingAckAsOf : now;
|
||||
if (_outQoSRecords.size() < ZT_PATH_MAX_OUTSTANDING_QOS_RECORDS) {
|
||||
_outQoSRecords[packetId] = now;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
inline void setBonded(bool bonded) { _bonded = bonded; }
|
||||
|
||||
/**
|
||||
* Record statistics on incoming packets. Used later to estimate QoS metrics.
|
||||
*
|
||||
* @param now Current time
|
||||
* @param packetId ID of packet
|
||||
* @param payloadLength Length of payload
|
||||
* @param verb Packet verb
|
||||
* @return True if this path is currently part of a bond.
|
||||
*/
|
||||
inline void recordIncomingPacket(int64_t now, int64_t packetId, uint16_t payloadLength, Packet::Verb verb)
|
||||
{
|
||||
Mutex::Lock _l(_statistics_m);
|
||||
if (verb != Packet::VERB_ACK && verb != Packet::VERB_QOS_MEASUREMENT) {
|
||||
if ((packetId & (ZT_PATH_QOS_ACK_PROTOCOL_DIVISOR - 1)) == 0) {
|
||||
_inACKRecords[packetId] = payloadLength;
|
||||
_packetsReceivedSinceLastAck++;
|
||||
_inQoSRecords[packetId] = now;
|
||||
_packetsReceivedSinceLastQoS++;
|
||||
}
|
||||
_packetValiditySamples.push(true);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Record that we've received a VERB_ACK on this path, also compute throughput if required.
|
||||
*
|
||||
* @param now Current time
|
||||
* @param ackedBytes Number of bytes acknowledged by other peer
|
||||
*/
|
||||
inline void receivedAck(int64_t now, int32_t ackedBytes)
|
||||
{
|
||||
_expectingAckAsOf = 0;
|
||||
_unackedBytes = (ackedBytes > _unackedBytes) ? 0 : _unackedBytes - ackedBytes;
|
||||
int64_t timeSinceThroughputEstimate = (now - _lastThroughputEstimation);
|
||||
if (timeSinceThroughputEstimate >= ZT_PATH_THROUGHPUT_MEASUREMENT_INTERVAL) {
|
||||
uint64_t throughput = (uint64_t)((float)(_bytesAckedSinceLastThroughputEstimation * 8) / ((float)timeSinceThroughputEstimate / (float)1000));
|
||||
_throughputSamples.push(throughput);
|
||||
_maxLifetimeThroughput = throughput > _maxLifetimeThroughput ? throughput : _maxLifetimeThroughput;
|
||||
_lastThroughputEstimation = now;
|
||||
_bytesAckedSinceLastThroughputEstimation = 0;
|
||||
} else {
|
||||
_bytesAckedSinceLastThroughputEstimation += ackedBytes;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @return Number of bytes this peer is responsible for ACKing since last ACK
|
||||
*/
|
||||
inline int32_t bytesToAck()
|
||||
{
|
||||
Mutex::Lock _l(_statistics_m);
|
||||
int32_t bytesToAck = 0;
|
||||
std::map<uint64_t,uint16_t>::iterator it = _inACKRecords.begin();
|
||||
while (it != _inACKRecords.end()) {
|
||||
bytesToAck += it->second;
|
||||
it++;
|
||||
}
|
||||
return bytesToAck;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return Number of bytes thus far sent that have not been acknowledged by the remote peer
|
||||
*/
|
||||
inline int64_t unackedSentBytes()
|
||||
{
|
||||
return _unackedBytes;
|
||||
}
|
||||
|
||||
/**
|
||||
* Account for the fact that an ACK was just sent. Reset counters, timers, and clear statistics buffers
|
||||
*
|
||||
* @param Current time
|
||||
*/
|
||||
inline void sentAck(int64_t now)
|
||||
{
|
||||
Mutex::Lock _l(_statistics_m);
|
||||
_inACKRecords.clear();
|
||||
_packetsReceivedSinceLastAck = 0;
|
||||
_lastAck = now;
|
||||
}
|
||||
|
||||
/**
|
||||
* Receive QoS data, match with recorded egress times from this peer, compute latency
|
||||
* estimates.
|
||||
*
|
||||
* @param now Current time
|
||||
* @param count Number of records
|
||||
* @param rx_id table of packet IDs
|
||||
* @param rx_ts table of holding times
|
||||
*/
|
||||
inline void receivedQoS(int64_t now, int count, uint64_t *rx_id, uint16_t *rx_ts)
|
||||
{
|
||||
Mutex::Lock _l(_statistics_m);
|
||||
// Look up egress times and compute latency values for each record
|
||||
std::map<uint64_t,uint64_t>::iterator it;
|
||||
for (int j=0; j<count; j++) {
|
||||
it = _outQoSRecords.find(rx_id[j]);
|
||||
if (it != _outQoSRecords.end()) {
|
||||
uint16_t rtt = (uint16_t)(now - it->second);
|
||||
uint16_t rtt_compensated = rtt - rx_ts[j];
|
||||
uint16_t latency = rtt_compensated / 2;
|
||||
updateLatency(latency, now);
|
||||
_outQoSRecords.erase(it);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate the contents of a VERB_QOS_MEASUREMENT packet.
|
||||
*
|
||||
* @param now Current time
|
||||
* @param qosBuffer destination buffer
|
||||
* @return Size of payload
|
||||
*/
|
||||
inline int32_t generateQoSPacket(int64_t now, char *qosBuffer)
|
||||
{
|
||||
Mutex::Lock _l(_statistics_m);
|
||||
int32_t len = 0;
|
||||
std::map<uint64_t,uint64_t>::iterator it = _inQoSRecords.begin();
|
||||
int i=0;
|
||||
while (i<_packetsReceivedSinceLastQoS && it != _inQoSRecords.end()) {
|
||||
uint64_t id = it->first;
|
||||
memcpy(qosBuffer, &id, sizeof(uint64_t));
|
||||
qosBuffer+=sizeof(uint64_t);
|
||||
uint16_t holdingTime = (uint16_t)(now - it->second);
|
||||
memcpy(qosBuffer, &holdingTime, sizeof(uint16_t));
|
||||
qosBuffer+=sizeof(uint16_t);
|
||||
len+=sizeof(uint64_t)+sizeof(uint16_t);
|
||||
_inQoSRecords.erase(it++);
|
||||
i++;
|
||||
}
|
||||
return len;
|
||||
}
|
||||
|
||||
/**
|
||||
* Account for the fact that a VERB_QOS_MEASUREMENT was just sent. Reset timers.
|
||||
*
|
||||
* @param Current time
|
||||
*/
|
||||
inline void sentQoS(int64_t now) {
|
||||
_packetsReceivedSinceLastQoS = 0;
|
||||
_lastQoSMeasurement = now;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param now Current time
|
||||
* @return Whether an ACK (VERB_ACK) packet needs to be emitted at this time
|
||||
*/
|
||||
inline bool needsToSendAck(int64_t now) {
|
||||
return ((now - _lastAck) >= ZT_PATH_ACK_INTERVAL ||
|
||||
(_packetsReceivedSinceLastAck == ZT_PATH_QOS_TABLE_SIZE)) && _packetsReceivedSinceLastAck;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param now Current time
|
||||
* @return Whether a QoS (VERB_QOS_MEASUREMENT) packet needs to be emitted at this time
|
||||
*/
|
||||
inline bool needsToSendQoS(int64_t now) {
|
||||
return ((_packetsReceivedSinceLastQoS >= ZT_PATH_QOS_TABLE_SIZE) ||
|
||||
((now - _lastQoSMeasurement) > ZT_PATH_QOS_INTERVAL)) && _packetsReceivedSinceLastQoS;
|
||||
}
|
||||
|
||||
/**
|
||||
* How much time has elapsed since we've been expecting a VERB_ACK on this path. This value
|
||||
* is used to determine a more relevant path "age". This lets us penalize paths which are no
|
||||
* longer ACKing, but not those that simple aren't being used to carry traffic at the
|
||||
* current time.
|
||||
*/
|
||||
inline int64_t ackAge(int64_t now) { return _expectingAckAsOf ? now - _expectingAckAsOf : 0; }
|
||||
|
||||
/**
|
||||
* The maximum observed throughput (in bits/s) for this path
|
||||
*/
|
||||
inline uint64_t maxLifetimeThroughput() { return _maxLifetimeThroughput; }
|
||||
|
||||
/**
|
||||
* @return The mean throughput (in bits/s) of this link
|
||||
*/
|
||||
inline uint64_t meanThroughput() { return _lastComputedMeanThroughput; }
|
||||
|
||||
/**
|
||||
* Assign a new relative quality value for this path in the aggregate link
|
||||
*
|
||||
* @param rq Quality of this path in comparison to other paths available to this peer
|
||||
*/
|
||||
inline void updateRelativeQuality(float rq) { _lastComputedRelativeQuality = rq; }
|
||||
|
||||
/**
|
||||
* @return Quality of this path compared to others in the aggregate link
|
||||
*/
|
||||
inline float relativeQuality() { return _lastComputedRelativeQuality; }
|
||||
|
||||
/**
|
||||
* Assign a new allocation value for this path in the aggregate link
|
||||
*
|
||||
* @param allocation Percentage of traffic to be sent over this path to a peer
|
||||
*/
|
||||
inline void updateComponentAllocationOfAggregateLink(unsigned char allocation) { _lastAllocation = allocation; }
|
||||
|
||||
/**
|
||||
* @return Percentage of traffic allocated to this path in the aggregate link
|
||||
*/
|
||||
inline unsigned char allocation() { return _lastAllocation; }
|
||||
|
||||
/**
|
||||
* @return Stability estimates can become expensive to compute, we cache the most recent result.
|
||||
*/
|
||||
inline float lastComputedStability() { return _lastComputedStability; }
|
||||
|
||||
/**
|
||||
* @return A pointer to a cached copy of the human-readable name of the interface this Path's localSocket is bound to
|
||||
*/
|
||||
inline char *getName() { return _ifname; }
|
||||
|
||||
/**
|
||||
* @return Packet delay variance
|
||||
*/
|
||||
inline float packetDelayVariance() { return _lastComputedPacketDelayVariance; }
|
||||
|
||||
/**
|
||||
* @return Previously-computed mean latency
|
||||
*/
|
||||
inline float meanLatency() { return _lastComputedMeanLatency; }
|
||||
|
||||
/**
|
||||
* @return Packet loss rate (PLR)
|
||||
*/
|
||||
inline float packetLossRatio() { return _lastComputedPacketLossRatio; }
|
||||
|
||||
/**
|
||||
* @return Packet error ratio (PER)
|
||||
*/
|
||||
inline float packetErrorRatio() { return _lastComputedPacketErrorRatio; }
|
||||
|
||||
/**
|
||||
* Record an invalid incoming packet. This packet failed MAC/compression/cipher checks and will now
|
||||
* contribute to a Packet Error Ratio (PER).
|
||||
*/
|
||||
inline void recordInvalidPacket() { _packetValiditySamples.push(false); }
|
||||
|
||||
/**
|
||||
* @return A pointer to a cached copy of the address string for this Path (For debugging only)
|
||||
*/
|
||||
inline char *getAddressString() { return _addrString; }
|
||||
|
||||
/**
|
||||
* @return The current throughput disturbance coefficient
|
||||
*/
|
||||
inline float throughputDisturbanceCoefficient() { return _lastComputedThroughputDistCoeff; }
|
||||
|
||||
/**
|
||||
* Compute and cache stability and performance metrics. The resultant stability coefficient is a measure of how "well behaved"
|
||||
* this path is. This figure is substantially different from (but required for the estimation of the path's overall "quality".
|
||||
*
|
||||
* @param now Current time
|
||||
*/
|
||||
inline void processBackgroundPathMeasurements(const int64_t now)
|
||||
{
|
||||
if (now - _lastPathQualityComputeTime > ZT_PATH_QUALITY_COMPUTE_INTERVAL) {
|
||||
Mutex::Lock _l(_statistics_m);
|
||||
_lastPathQualityComputeTime = now;
|
||||
address().toString(_addrString);
|
||||
_lastComputedMeanLatency = _latencySamples.mean();
|
||||
_lastComputedPacketDelayVariance = _latencySamples.stddev(); // Similar to "jitter" (SEE: RFC 3393, RFC 4689)
|
||||
_lastComputedMeanThroughput = (uint64_t)_throughputSamples.mean();
|
||||
|
||||
// If no packet validity samples, assume PER==0
|
||||
_lastComputedPacketErrorRatio = 1 - (_packetValiditySamples.count() ? _packetValiditySamples.mean() : 1);
|
||||
|
||||
// Compute path stability
|
||||
// Normalize measurements with wildly different ranges into a reasonable range
|
||||
float normalized_pdv = Utils::normalize(_lastComputedPacketDelayVariance, 0, ZT_PATH_MAX_PDV, 0, 10);
|
||||
float normalized_la = Utils::normalize(_lastComputedMeanLatency, 0, ZT_PATH_MAX_MEAN_LATENCY, 0, 10);
|
||||
float throughput_cv = _throughputSamples.mean() > 0 ? _throughputSamples.stddev() / _throughputSamples.mean() : 1;
|
||||
|
||||
// Form an exponential cutoff and apply contribution weights
|
||||
float pdv_contrib = expf((-1.0f)*normalized_pdv) * (float)ZT_PATH_CONTRIB_PDV;
|
||||
float latency_contrib = expf((-1.0f)*normalized_la) * (float)ZT_PATH_CONTRIB_LATENCY;
|
||||
|
||||
// Throughput Disturbance Coefficient
|
||||
float throughput_disturbance_contrib = expf((-1.0f)*throughput_cv) * (float)ZT_PATH_CONTRIB_THROUGHPUT_DISTURBANCE;
|
||||
_throughputDisturbanceSamples.push(throughput_cv);
|
||||
_lastComputedThroughputDistCoeff = _throughputDisturbanceSamples.mean();
|
||||
|
||||
// Obey user-defined ignored contributions
|
||||
pdv_contrib = ZT_PATH_CONTRIB_PDV > 0.0 ? pdv_contrib : 1;
|
||||
latency_contrib = ZT_PATH_CONTRIB_LATENCY > 0.0 ? latency_contrib : 1;
|
||||
throughput_disturbance_contrib = ZT_PATH_CONTRIB_THROUGHPUT_DISTURBANCE > 0.0 ? throughput_disturbance_contrib : 1;
|
||||
|
||||
// Stability
|
||||
_lastComputedStability = pdv_contrib + latency_contrib + throughput_disturbance_contrib;
|
||||
_lastComputedStability *= 1 - _lastComputedPacketErrorRatio;
|
||||
|
||||
// Prevent QoS records from sticking around for too long
|
||||
std::map<uint64_t,uint64_t>::iterator it = _outQoSRecords.begin();
|
||||
while (it != _outQoSRecords.end()) {
|
||||
// Time since egress of tracked packet
|
||||
if ((now - it->second) >= ZT_PATH_QOS_TIMEOUT) {
|
||||
_outQoSRecords.erase(it++);
|
||||
} else { it++; }
|
||||
}
|
||||
}
|
||||
}
|
||||
inline bool bonded() { return _bonded; }
|
||||
|
||||
/**
|
||||
* @return True if this path is alive (receiving heartbeats)
|
||||
*/
|
||||
inline bool alive(const int64_t now) const { return ((now - _lastIn) < (ZT_PATH_HEARTBEAT_PERIOD + 5000)); }
|
||||
inline bool alive(const int64_t now, bool bondingEnabled = false) const {
|
||||
return (bondingEnabled && _monitorInterval) ? ((now - _lastIn) < (_monitorInterval * 3)) : ((now - _lastIn) < (ZT_PATH_HEARTBEAT_PERIOD + 5000));
|
||||
}
|
||||
|
||||
/**
|
||||
* @return True if this path needs a heartbeat
|
||||
*/
|
||||
inline bool needsHeartbeat(const int64_t now) const { return ((now - _lastOut) >= ZT_PATH_HEARTBEAT_PERIOD); }
|
||||
|
||||
/**
|
||||
* @return True if this path needs a heartbeat in accordance to the user-specified path monitor frequency
|
||||
*/
|
||||
inline bool needsGratuitousHeartbeat(const int64_t now) { return allowed() && (_monitorInterval > 0) && ((now - _lastOut) >= _monitorInterval); }
|
||||
|
||||
/**
|
||||
* @return Last time we sent something
|
||||
*/
|
||||
|
|
@ -631,62 +357,339 @@ public:
|
|||
*/
|
||||
inline int64_t lastIn() const { return _lastIn; }
|
||||
|
||||
/**
|
||||
* @return the age of the path in terms of receiving packets
|
||||
*/
|
||||
inline int64_t age(int64_t now) { return (now - _lastIn); }
|
||||
|
||||
/**
|
||||
* @return Time last trust-established packet was received
|
||||
*/
|
||||
inline int64_t lastTrustEstablishedPacketReceived() const { return _lastTrustEstablishedPacketReceived; }
|
||||
|
||||
/**
|
||||
* @return Time since last VERB_ACK was received
|
||||
*/
|
||||
inline int64_t ackAge(int64_t now) { return _lastAckReceived ? now - _lastAckReceived : 0; }
|
||||
|
||||
/**
|
||||
* Set or update a refractory period for the path.
|
||||
*
|
||||
* @param punishment How much a path should be punished
|
||||
* @param pathFailure Whether this call is the result of a recent path failure
|
||||
*/
|
||||
inline void adjustRefractoryPeriod(int64_t now, uint32_t punishment, bool pathFailure) {
|
||||
if (pathFailure) {
|
||||
unsigned int suggestedRefractoryPeriod = _refractoryPeriod ? punishment + (_refractoryPeriod * 2) : punishment;
|
||||
_refractoryPeriod = std::min(suggestedRefractoryPeriod, (unsigned int)ZT_MULTIPATH_MAX_REFRACTORY_PERIOD);
|
||||
_lastRefractoryUpdate = 0;
|
||||
} else {
|
||||
uint32_t drainRefractory = 0;
|
||||
if (_lastRefractoryUpdate) {
|
||||
drainRefractory = (now - _lastRefractoryUpdate);
|
||||
} else {
|
||||
drainRefractory = (now - _lastAliveToggle);
|
||||
}
|
||||
_lastRefractoryUpdate = now;
|
||||
if (_refractoryPeriod > drainRefractory) {
|
||||
_refractoryPeriod -= drainRefractory;
|
||||
} else {
|
||||
_refractoryPeriod = 0;
|
||||
_lastRefractoryUpdate = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine the current state of eligibility of the path.
|
||||
*
|
||||
* @param includeRefractoryPeriod Whether current punishment should be taken into consideration
|
||||
* @return True if this path can be used in a bond at the current time
|
||||
*/
|
||||
inline bool eligible(uint64_t now, int ackSendInterval, bool includeRefractoryPeriod = false) {
|
||||
if (includeRefractoryPeriod && _refractoryPeriod) {
|
||||
return false;
|
||||
}
|
||||
bool acceptableAge = age(now) < ((_monitorInterval * 4) + _downDelay); // Simple RX age (driven by packets of any type and gratuitous VERB_HELLOs)
|
||||
bool acceptableAckAge = ackAge(now) < (ackSendInterval); // Whether the remote peer is actually responding to our outgoing traffic or simply sending stuff to us
|
||||
bool notTooEarly = (now - _lastAliveToggle) >= _upDelay; // Whether we've waited long enough since the link last came online
|
||||
bool inTrial = (now - _lastTrialBegin) < _upDelay; // Whether this path is still in its trial period
|
||||
bool currEligibility = allowed() && (((acceptableAge || acceptableAckAge) && notTooEarly) || inTrial);
|
||||
return currEligibility;
|
||||
}
|
||||
|
||||
/**
|
||||
* Record when this path first entered the bond. Each path is given a trial period where it is admitted
|
||||
* to the bond without requiring observations to prove its performance or reliability.
|
||||
*/
|
||||
inline void startTrial(uint64_t now) { _lastTrialBegin = now; }
|
||||
|
||||
/**
|
||||
* @return True if a path is permitted to be used in a bond (according to user pref.)
|
||||
*/
|
||||
inline bool allowed() {
|
||||
return _enabled
|
||||
&& (!_ipvPref
|
||||
|| ((_addr.isV4() && (_ipvPref == 4 || _ipvPref == 46 || _ipvPref == 64))
|
||||
|| ((_addr.isV6() && (_ipvPref == 6 || _ipvPref == 46 || _ipvPref == 64)))));
|
||||
}
|
||||
|
||||
/**
|
||||
* @return True if a path is preferred over another on the same physical slave (according to user pref.)
|
||||
*/
|
||||
inline bool preferred() {
|
||||
return _onlyPathOnSlave
|
||||
|| (_addr.isV4() && (_ipvPref == 4 || _ipvPref == 46))
|
||||
|| (_addr.isV6() && (_ipvPref == 6 || _ipvPref == 64));
|
||||
}
|
||||
|
||||
/**
|
||||
* @param now Current time
|
||||
* @return Whether an ACK (VERB_ACK) packet needs to be emitted at this time
|
||||
*/
|
||||
inline bool needsToSendAck(int64_t now, int ackSendInterval) {
|
||||
return ((now - _lastAckSent) >= ackSendInterval ||
|
||||
(_packetsReceivedSinceLastAck == ZT_QOS_TABLE_SIZE)) && _packetsReceivedSinceLastAck;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param now Current time
|
||||
* @return Whether a QoS (VERB_QOS_MEASUREMENT) packet needs to be emitted at this time
|
||||
*/
|
||||
inline bool needsToSendQoS(int64_t now, int qosSendInterval) {
|
||||
return ((_packetsReceivedSinceLastQoS >= ZT_QOS_TABLE_SIZE) ||
|
||||
((now - _lastQoSMeasurement) > qosSendInterval)) && _packetsReceivedSinceLastQoS;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset packet counters
|
||||
*/
|
||||
inline void resetPacketCounts()
|
||||
{
|
||||
_packetsIn = 0;
|
||||
_packetsOut = 0;
|
||||
}
|
||||
|
||||
private:
|
||||
Mutex _statistics_m;
|
||||
|
||||
volatile int64_t _lastOut;
|
||||
volatile int64_t _lastIn;
|
||||
volatile int64_t _lastTrustEstablishedPacketReceived;
|
||||
volatile int64_t _lastPathQualityComputeTime;
|
||||
int64_t _localSocket;
|
||||
volatile unsigned int _latency;
|
||||
InetAddress _addr;
|
||||
InetAddress::IpScope _ipScope; // memoize this since it's a computed value checked often
|
||||
AtomicCounter __refCount;
|
||||
|
||||
std::map<uint64_t,uint64_t> _outQoSRecords; // id:egress_time
|
||||
std::map<uint64_t,uint64_t> _inQoSRecords; // id:now
|
||||
std::map<uint64_t,uint16_t> _inACKRecords; // id:len
|
||||
std::map<uint64_t,uint64_t> qosStatsOut; // id:egress_time
|
||||
std::map<uint64_t,uint64_t> qosStatsIn; // id:now
|
||||
std::map<uint64_t,uint16_t> ackStatsIn; // id:len
|
||||
|
||||
int64_t _lastAck;
|
||||
int64_t _lastThroughputEstimation;
|
||||
int64_t _lastQoSMeasurement;
|
||||
int64_t _lastQoSRecordPurge;
|
||||
RingBuffer<int,ZT_QOS_SHORTTERM_SAMPLE_WIN_SIZE> qosRecordSize;
|
||||
RingBuffer<float,ZT_QOS_SHORTTERM_SAMPLE_WIN_SIZE> qosRecordLossSamples;
|
||||
RingBuffer<uint64_t,ZT_QOS_SHORTTERM_SAMPLE_WIN_SIZE> throughputSamples;
|
||||
RingBuffer<bool,ZT_QOS_SHORTTERM_SAMPLE_WIN_SIZE> packetValiditySamples;
|
||||
RingBuffer<float,ZT_QOS_SHORTTERM_SAMPLE_WIN_SIZE> _throughputVarianceSamples;
|
||||
RingBuffer<uint16_t,ZT_QOS_SHORTTERM_SAMPLE_WIN_SIZE> latencySamples;
|
||||
|
||||
/**
|
||||
* Last time that a VERB_ACK was received on this path.
|
||||
*/
|
||||
uint64_t _lastAckReceived;
|
||||
|
||||
/**
|
||||
* Last time that a VERB_ACK was sent out on this path.
|
||||
*/
|
||||
uint64_t _lastAckSent;
|
||||
|
||||
/**
|
||||
* Last time that a VERB_QOS_MEASUREMENT was sent out on this path.
|
||||
*/
|
||||
uint64_t _lastQoSMeasurement;
|
||||
|
||||
/**
|
||||
* Last time that a the path's throughput was estimated.
|
||||
*/
|
||||
uint64_t _lastThroughputEstimation;
|
||||
|
||||
/**
|
||||
* The last time that the refractory period was updated.
|
||||
*/
|
||||
uint64_t _lastRefractoryUpdate;
|
||||
|
||||
/**
|
||||
* The last time that the path was marked as "alive".
|
||||
*/
|
||||
uint64_t _lastAliveToggle;
|
||||
|
||||
/**
|
||||
* State of eligibility at last check. Used for determining state changes.
|
||||
*/
|
||||
bool _lastEligibilityState;
|
||||
|
||||
/**
|
||||
* Timestamp indicating when this path's trial period began.
|
||||
*/
|
||||
uint64_t _lastTrialBegin;
|
||||
|
||||
/**
|
||||
* Amount of time that this path is prevented from becoming a member of a bond.
|
||||
*/
|
||||
uint32_t _refractoryPeriod;
|
||||
|
||||
/**
|
||||
* Monitor interval specific to this path or that was inherited from the bond controller.
|
||||
*/
|
||||
int32_t _monitorInterval;
|
||||
|
||||
/**
|
||||
* Up delay interval specific to this path or that was inherited from the bond controller.
|
||||
*/
|
||||
uint32_t _upDelay;
|
||||
|
||||
/**
|
||||
* Down delay interval specific to this path or that was inherited from the bond controller.
|
||||
*/
|
||||
uint32_t _downDelay;
|
||||
|
||||
/**
|
||||
* IP version preference inherited from the physical slave.
|
||||
*/
|
||||
uint8_t _ipvPref;
|
||||
|
||||
/**
|
||||
* Mode inherited from the physical slave.
|
||||
*/
|
||||
uint8_t _mode;
|
||||
|
||||
/**
|
||||
* IP version preference inherited from the physical slave.
|
||||
*/
|
||||
bool _onlyPathOnSlave;
|
||||
|
||||
/**
|
||||
* Enabled state inherited from the physical slave.
|
||||
*/
|
||||
bool _enabled;
|
||||
|
||||
/**
|
||||
* Whether this path is currently part of a bond.
|
||||
*/
|
||||
bool _bonded;
|
||||
|
||||
/**
|
||||
* Whether this path was intentionally _negotiated by either peer.
|
||||
*/
|
||||
bool _negotiated;
|
||||
|
||||
/**
|
||||
* Whether this path has been deprecated due to performance issues. Current traffic flows
|
||||
* will be re-allocated to other paths in the most non-disruptive manner (if possible),
|
||||
* and new traffic will not be allocated to this path.
|
||||
*/
|
||||
bool _deprecated;
|
||||
|
||||
/**
|
||||
* Whether flows should be moved from this path. Current traffic flows will be re-allocated
|
||||
* immediately.
|
||||
*/
|
||||
bool _shouldReallocateFlows;
|
||||
|
||||
/**
|
||||
* The number of flows currently assigned to this path.
|
||||
*/
|
||||
uint16_t _assignedFlowCount;
|
||||
|
||||
/**
|
||||
* The mean latency (computed from a sliding window.)
|
||||
*/
|
||||
float _latencyMean;
|
||||
|
||||
/**
|
||||
* Packet delay variance (computed from a sliding window.)
|
||||
*/
|
||||
float _latencyVariance;
|
||||
|
||||
/**
|
||||
* The ratio of lost packets to received packets.
|
||||
*/
|
||||
float _packetLossRatio;
|
||||
|
||||
/**
|
||||
* The ratio of packets that failed their MAC/CRC checks to those that did not.
|
||||
*/
|
||||
float _packetErrorRatio;
|
||||
|
||||
/**
|
||||
* The estimated mean throughput of this path.
|
||||
*/
|
||||
uint64_t _throughputMean;
|
||||
|
||||
/**
|
||||
* The maximum observed throughput of this path.
|
||||
*/
|
||||
uint64_t _throughputMax;
|
||||
|
||||
/**
|
||||
* The variance in the estimated throughput of this path.
|
||||
*/
|
||||
float _throughputVariance;
|
||||
|
||||
/**
|
||||
* The relative quality of this path to all others in the bond, [0-255].
|
||||
*/
|
||||
uint8_t _allocation;
|
||||
|
||||
/**
|
||||
* How much load this path is under.
|
||||
*/
|
||||
uint64_t _byteLoad;
|
||||
|
||||
/**
|
||||
* How much load this path is under (relative to other paths in the bond.)
|
||||
*/
|
||||
uint8_t _relativeByteLoad;
|
||||
|
||||
/**
|
||||
* Relative value expressing how "deserving" this path is of new traffic.
|
||||
*/
|
||||
uint8_t _affinity;
|
||||
|
||||
/**
|
||||
* Score that indicates to what degree this path is preferred over others that
|
||||
* are available to the bonding policy. (specifically for active-backup)
|
||||
*/
|
||||
uint32_t _failoverScore;
|
||||
|
||||
/**
|
||||
* Number of bytes thus far sent that have not been acknowledged by the remote peer.
|
||||
*/
|
||||
int64_t _unackedBytes;
|
||||
int64_t _expectingAckAsOf;
|
||||
int16_t _packetsReceivedSinceLastAck;
|
||||
int16_t _packetsReceivedSinceLastQoS;
|
||||
|
||||
uint64_t _maxLifetimeThroughput;
|
||||
uint64_t _lastComputedMeanThroughput;
|
||||
/**
|
||||
* Number of packets received since the last VERB_ACK was sent to the remote peer.
|
||||
*/
|
||||
int32_t _packetsReceivedSinceLastAck;
|
||||
|
||||
/**
|
||||
* Number of packets received since the last VERB_QOS_MEASUREMENT was sent to the remote peer.
|
||||
*/
|
||||
int32_t _packetsReceivedSinceLastQoS;
|
||||
|
||||
/**
|
||||
* Bytes acknowledged via incoming VERB_ACK since the last estimation of throughput.
|
||||
*/
|
||||
uint64_t _bytesAckedSinceLastThroughputEstimation;
|
||||
|
||||
float _lastComputedMeanLatency;
|
||||
float _lastComputedPacketDelayVariance;
|
||||
/**
|
||||
* Counters used for tracking path load.
|
||||
*/
|
||||
int _packetsIn;
|
||||
int _packetsOut;
|
||||
|
||||
float _lastComputedPacketErrorRatio;
|
||||
float _lastComputedPacketLossRatio;
|
||||
// TODO: Remove
|
||||
|
||||
// cached estimates
|
||||
float _lastComputedStability;
|
||||
float _lastComputedRelativeQuality;
|
||||
float _lastComputedThroughputDistCoeff;
|
||||
unsigned char _lastAllocation;
|
||||
|
||||
// cached human-readable strings for tracing purposes
|
||||
char _ifname[16];
|
||||
char _addrString[256];
|
||||
|
||||
RingBuffer<uint64_t,ZT_PATH_QUALITY_METRIC_WIN_SZ> _throughputSamples;
|
||||
RingBuffer<uint32_t,ZT_PATH_QUALITY_METRIC_WIN_SZ> _latencySamples;
|
||||
RingBuffer<bool,ZT_PATH_QUALITY_METRIC_WIN_SZ> _packetValiditySamples;
|
||||
RingBuffer<float,ZT_PATH_QUALITY_METRIC_WIN_SZ> _throughputDisturbanceSamples;
|
||||
bool _prevEligibility;
|
||||
};
|
||||
|
||||
} // namespace ZeroTier
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue