/* * Copyright (c)2019 ZeroTier, Inc. * * Use of this software is governed by the Business Source License included * in the LICENSE.TXT file in the project's root directory. * * Change Date: 2026-01-01 * * On the date above, in accordance with the Business Source License, use * of this software will be governed by version 2.0 of the Apache License. */ /****/ #include "../node/Constants.hpp" #ifdef __APPLE__ #include "../node/Dictionary.hpp" #include "../node/Mutex.hpp" #include "../node/Utils.hpp" #include "MacDNSHelper.hpp" #include "MacEthernetTap.hpp" #include "MacEthernetTapAgent.h" #include "OSUtils.hpp" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static const ZeroTier::MulticastGroup _blindWildcardMulticastGroup(ZeroTier::MAC(0xff), 0); #define MACOS_FETH_MAX_MTU_SYSCTL "net.link.fake.max_mtu" namespace ZeroTier { static Mutex globalTapCreateLock; static bool globalTapInitialized = false; static bool fethMaxMtuAdjusted = false; MacEthernetTap::MacEthernetTap( const char* homePath, const MAC& mac, unsigned int mtu, unsigned int metric, uint64_t nwid, const char* friendlyName, void (*handler)(void*, void*, uint64_t, const MAC&, const MAC&, unsigned int, unsigned int, const void* data, unsigned int len), void* arg) : _handler(handler) , _arg(arg) , _nwid(nwid) , _homePath(homePath) , _mtu(mtu) , _metric(metric) , _devNo(0) , _agentStdin(-1) , _agentStdout(-1) , _agentStderr(-1) , _agentStdin2(-1) , _agentStdout2(-1) , _agentStderr2(-1) , _agentPid(-1) , _enabled(true) , _lastIfAddrsUpdate(0) { char ethaddr[64], mtustr[16], devnostr[16], devstr[16], metricstr[16]; OSUtils::ztsnprintf(ethaddr, sizeof(ethaddr), "%.2x:%.2x:%.2x:%.2x:%.2x:%.2x", (int)mac[0], (int)mac[1], (int)mac[2], (int)mac[3], (int)mac[4], (int)mac[5]); OSUtils::ztsnprintf(mtustr, sizeof(mtustr), "%u", mtu); OSUtils::ztsnprintf(metricstr, sizeof(metricstr), "%u", metric); std::string agentPath(homePath); agentPath.push_back(ZT_PATH_SEPARATOR); agentPath.append("MacEthernetTapAgent"); if (! OSUtils::fileExists(agentPath.c_str())) throw std::runtime_error("MacEthernetTapAgent not present in ZeroTier home"); Mutex::Lock _gl(globalTapCreateLock); // only make one at a time if (! fethMaxMtuAdjusted) { fethMaxMtuAdjusted = true; int old_mtu = 0; size_t old_mtu_len = sizeof(old_mtu); int mtu = 10000; sysctlbyname(MACOS_FETH_MAX_MTU_SYSCTL, &old_mtu, &old_mtu_len, &mtu, sizeof(mtu)); } // Destroy all feth devices on first tap start in case ZeroTier did not exit cleanly last time. // We leave interfaces less than feth100 alone in case something else is messing with feth devices. if (! globalTapInitialized) { globalTapInitialized = true; struct ifaddrs* ifa = (struct ifaddrs*)0; std::set deleted; if (! getifaddrs(&ifa)) { struct ifaddrs* p = ifa; while (p) { int nameLen = (int)strlen(p->ifa_name); // Delete feth# from feth0 to feth9999, but don't touch >10000. if ((! strncmp(p->ifa_name, "feth", 4)) && (nameLen >= 5) && (nameLen <= 8) && (deleted.count(std::string(p->ifa_name)) == 0)) { deleted.insert(std::string(p->ifa_name)); const char* args[4]; args[0] = "/sbin/ifconfig"; args[1] = p->ifa_name; args[2] = "destroy"; args[3] = (char*)0; const pid_t pid = vfork(); if (pid == 0) { execv(args[0], const_cast(args)); _exit(-1); } else if (pid > 0) { int rv = 0; waitpid(pid, &rv, 0); } } p = p->ifa_next; } freeifaddrs(ifa); } } unsigned int devNo = 100 + ((nwid ^ (nwid >> 32) ^ (nwid >> 48)) % 4900); for (;;) { OSUtils::ztsnprintf(devnostr, sizeof(devnostr), "%u", devNo); OSUtils::ztsnprintf(devstr, sizeof(devstr), "feth%u", devNo); bool duplicate = false; struct ifaddrs* ifa = (struct ifaddrs*)0; if (! getifaddrs(&ifa)) { struct ifaddrs* p = ifa; while (p) { if (! strcmp(p->ifa_name, devstr)) { duplicate = true; break; } p = p->ifa_next; } freeifaddrs(ifa); } if (duplicate) { devNo = (devNo + 1) % 5000; if (devNo < 100) devNo = 100; } else { _dev = devstr; _devNo = devNo; break; } } if (::pipe(_shutdownSignalPipe)) throw std::runtime_error("pipe creation failed"); int agentStdin[2]; int agentStdout[2]; int agentStderr[2]; if (::pipe(agentStdin)) throw std::runtime_error("pipe creation failed"); if (::pipe(agentStdout)) throw std::runtime_error("pipe creation failed"); if (::pipe(agentStderr)) throw std::runtime_error("pipe creation failed"); _agentStdin = agentStdin[1]; _agentStdout = agentStdout[0]; _agentStderr = agentStderr[0]; _agentStdin2 = agentStdin[0]; _agentStdout2 = agentStdout[1]; _agentStderr2 = agentStderr[1]; long apid = (long)fork(); if (apid < 0) { throw std::runtime_error("fork failed"); } else if (apid == 0) { ::dup2(agentStdin[0], STDIN_FILENO); ::dup2(agentStdout[1], STDOUT_FILENO); ::dup2(agentStderr[1], STDERR_FILENO); ::close(agentStdin[0]); ::close(agentStdin[1]); ::close(agentStdout[0]); ::close(agentStdout[1]); ::close(agentStderr[0]); ::close(agentStderr[1]); ::execl(agentPath.c_str(), agentPath.c_str(), devnostr, ethaddr, mtustr, metricstr, (char*)0); ::_exit(-1); } else { _agentPid = apid; // Wait up to 10 seconds for the subprocess to actually create the device. This prevents // things like routes from being created before the device exists. for (int waitLoops = 0;; ++waitLoops) { struct ifaddrs* ifa = (struct ifaddrs*)0; if (! getifaddrs(&ifa)) { struct ifaddrs* p = ifa; while (p) { if ((p->ifa_name) && (! strcmp(devstr, p->ifa_name))) { waitLoops = -1; break; } p = p->ifa_next; } freeifaddrs(ifa); } if (waitLoops == -1) { break; } else if (waitLoops >= 100) { // 10 seconds throw std::runtime_error("feth device creation timed out"); } Thread::sleep(100); } } _thread = Thread::start(this); } MacEthernetTap::~MacEthernetTap() { char tmp[64]; const char* args[4]; pid_t pid0, pid1; MacDNSHelper::removeDNS(_nwid); MacDNSHelper::removeIps4(_nwid); MacDNSHelper::removeIps6(_nwid); Mutex::Lock _gl(globalTapCreateLock); ::write(_shutdownSignalPipe[1], "\0", 1); // causes thread to exit int ec = 0; ::kill(_agentPid, SIGKILL); ::waitpid(_agentPid, &ec, 0); args[0] = "/sbin/ifconfig"; args[1] = _dev.c_str(); args[2] = "destroy"; args[3] = (char*)0; pid0 = vfork(); if (pid0 == 0) { execv(args[0], const_cast(args)); _exit(-1); } snprintf(tmp, sizeof(tmp), "feth%u", _devNo + 5000); // args[0] = "/sbin/ifconfig"; args[1] = tmp; // args[2] = "destroy"; // args[3] = (char *)0; pid1 = vfork(); if (pid1 == 0) { execv(args[0], const_cast(args)); _exit(-1); } if (pid0 > 0) { int rv = 0; waitpid(pid0, &rv, 0); } if (pid1 > 0) { int rv = 0; waitpid(pid1, &rv, 0); } Thread::join(_thread); } void MacEthernetTap::setEnabled(bool en) { _enabled = en; } bool MacEthernetTap::enabled() const { return _enabled; } bool MacEthernetTap::addIp(const InetAddress& ip) { char tmp[128]; if (! ip) return false; std::string cmd; cmd.push_back((char)ZT_MACETHERNETTAPAGENT_STDIN_CMD_IFCONFIG); cmd.append((ip.ss_family == AF_INET6) ? "inet6" : "inet"); cmd.push_back(0); cmd.append(ip.toString(tmp)); cmd.push_back(0); cmd.append("alias"); cmd.push_back(0); uint16_t l = (uint16_t)cmd.length(); _putLock.lock(); write(_agentStdin, &l, 2); write(_agentStdin, cmd.data(), cmd.length()); _putLock.unlock(); return true; } bool MacEthernetTap::removeIp(const InetAddress& ip) { char tmp[128]; if (! ip) return false; std::string cmd; cmd.push_back((char)ZT_MACETHERNETTAPAGENT_STDIN_CMD_IFCONFIG); cmd.append((ip.ss_family == AF_INET6) ? "inet6" : "inet"); cmd.push_back(0); cmd.append(ip.toString(tmp)); cmd.push_back(0); cmd.append("-alias"); cmd.push_back(0); uint16_t l = (uint16_t)cmd.length(); _putLock.lock(); write(_agentStdin, &l, 2); write(_agentStdin, cmd.data(), cmd.length()); _putLock.unlock(); return true; } std::vector MacEthernetTap::ips() const { uint64_t now = OSUtils::now(); if ((now - _lastIfAddrsUpdate) <= GETIFADDRS_CACHE_TIME) { return _ifaddrs; } _lastIfAddrsUpdate = now; struct ifaddrs* ifa = (struct ifaddrs*)0; std::vector r; if (! getifaddrs(&ifa)) { struct ifaddrs* p = ifa; while (p) { if ((p->ifa_name) && (! strcmp(p->ifa_name, _dev.c_str())) && (p->ifa_addr)) { switch (p->ifa_addr->sa_family) { case AF_INET: { struct sockaddr_in* sin = (struct sockaddr_in*)p->ifa_addr; struct sockaddr_in* nm = (struct sockaddr_in*)p->ifa_netmask; r.push_back(InetAddress(&(sin->sin_addr.s_addr), 4, Utils::countBits((uint32_t)nm->sin_addr.s_addr))); } break; case AF_INET6: { struct sockaddr_in6* sin = (struct sockaddr_in6*)p->ifa_addr; struct sockaddr_in6* nm = (struct sockaddr_in6*)p->ifa_netmask; uint32_t b[4]; memcpy(b, nm->sin6_addr.s6_addr, sizeof(b)); r.push_back(InetAddress(sin->sin6_addr.s6_addr, 16, Utils::countBits(b[0]) + Utils::countBits(b[1]) + Utils::countBits(b[2]) + Utils::countBits(b[3]))); } break; } } p = p->ifa_next; } freeifaddrs(ifa); } std::sort(r.begin(), r.end()); r.erase(std::unique(r.begin(), r.end()), r.end()); _ifaddrs = r; return r; } void MacEthernetTap::put(const MAC& from, const MAC& to, unsigned int etherType, const void* data, unsigned int len) { struct iovec iov[3]; unsigned char hdr[15]; uint16_t l; if ((_agentStdin > 0) && (len <= _mtu) && (_enabled)) { hdr[0] = ZT_MACETHERNETTAPAGENT_STDIN_CMD_PACKET; to.copyTo(hdr + 1, 6); from.copyTo(hdr + 7, 6); hdr[13] = (unsigned char)((etherType >> 8) & 0xff); hdr[14] = (unsigned char)(etherType & 0xff); l = (uint16_t)(len + 15); iov[0].iov_base = &l; iov[0].iov_len = 2; iov[1].iov_base = hdr; iov[1].iov_len = 15; iov[2].iov_base = const_cast(data); iov[2].iov_len = len; _putLock.lock(); writev(_agentStdin, iov, 3); _putLock.unlock(); } } std::string MacEthernetTap::deviceName() const { return _dev; } void MacEthernetTap::setFriendlyName(const char* friendlyName) { } void MacEthernetTap::scanMulticastGroups(std::vector& added, std::vector& removed) { std::vector newGroups; struct ifmaddrs* ifmap = (struct ifmaddrs*)0; if (! getifmaddrs(&ifmap)) { struct ifmaddrs* p = ifmap; while (p) { if (p->ifma_addr->sa_family == AF_LINK) { struct sockaddr_dl* in = (struct sockaddr_dl*)p->ifma_name; struct sockaddr_dl* la = (struct sockaddr_dl*)p->ifma_addr; if ((la->sdl_alen == 6) && (in->sdl_nlen <= _dev.length()) && (! memcmp(_dev.data(), in->sdl_data, in->sdl_nlen))) newGroups.push_back(MulticastGroup(MAC(la->sdl_data + la->sdl_nlen, 6), 0)); } p = p->ifma_next; } freeifmaddrs(ifmap); } std::vector allIps(ips()); for (std::vector::iterator ip(allIps.begin()); ip != allIps.end(); ++ip) newGroups.push_back(MulticastGroup::deriveMulticastGroupForAddressResolution(*ip)); std::sort(newGroups.begin(), newGroups.end()); newGroups.erase(std::unique(newGroups.begin(), newGroups.end()), newGroups.end()); for (std::vector::iterator m(newGroups.begin()); m != newGroups.end(); ++m) { if (! std::binary_search(_multicastGroups.begin(), _multicastGroups.end(), *m)) added.push_back(*m); } for (std::vector::iterator m(_multicastGroups.begin()); m != _multicastGroups.end(); ++m) { if (! std::binary_search(newGroups.begin(), newGroups.end(), *m)) removed.push_back(*m); } _multicastGroups.swap(newGroups); } void MacEthernetTap::setMtu(unsigned int mtu) { if (_mtu != mtu) { char tmp[16]; std::string cmd; cmd.push_back((char)ZT_MACETHERNETTAPAGENT_STDIN_CMD_IFCONFIG); cmd.append("mtu"); cmd.push_back(0); OSUtils::ztsnprintf(tmp, sizeof(tmp), "%u", mtu); cmd.append(tmp); cmd.push_back(0); uint16_t l = (uint16_t)cmd.length(); _putLock.lock(); write(_agentStdin, &l, 2); write(_agentStdin, cmd.data(), cmd.length()); _putLock.unlock(); _mtu = mtu; } } #define ZT_MACETHERNETTAP_AGENT_READ_BUF_SIZE 131072 void MacEthernetTap::threadMain() throw() { char agentReadBuf[ZT_MACETHERNETTAP_AGENT_READ_BUF_SIZE]; char agentStderrBuf[256]; fd_set readfds, nullfds; MAC to, from; Thread::sleep(250); const int nfds = std::max(std::max(_shutdownSignalPipe[0], _agentStdout), _agentStderr) + 1; long agentReadPtr = 0; fcntl(_agentStdout, F_SETFL, fcntl(_agentStdout, F_GETFL) | O_NONBLOCK); fcntl(_agentStderr, F_SETFL, fcntl(_agentStderr, F_GETFL) | O_NONBLOCK); FD_ZERO(&readfds); FD_ZERO(&nullfds); for (;;) { FD_SET(_shutdownSignalPipe[0], &readfds); FD_SET(_agentStdout, &readfds); FD_SET(_agentStderr, &readfds); select(nfds, &readfds, &nullfds, &nullfds, (struct timeval*)0); if (FD_ISSET(_shutdownSignalPipe[0], &readfds)) break; if (FD_ISSET(_agentStdout, &readfds)) { long n = (long)read(_agentStdout, agentReadBuf + agentReadPtr, ZT_MACETHERNETTAP_AGENT_READ_BUF_SIZE - agentReadPtr); if (n > 0) { agentReadPtr += n; while (agentReadPtr >= 2) { long len = *((uint16_t*)agentReadBuf); if (agentReadPtr >= (len + 2)) { char* msg = agentReadBuf + 2; if ((len > 14) && (_enabled)) { to.setTo(msg, 6); from.setTo(msg + 6, 6); _handler(_arg, (void*)0, _nwid, from, to, ntohs(((const uint16_t*)msg)[6]), 0, (const void*)(msg + 14), (unsigned int)len - 14); } if (agentReadPtr > (len + 2)) { memmove(agentReadBuf, agentReadBuf + len + 2, agentReadPtr -= (len + 2)); } else { agentReadPtr = 0; } } else { break; } } } } if (FD_ISSET(_agentStderr, &readfds)) { read(_agentStderr, agentStderrBuf, sizeof(agentStderrBuf)); /* const ssize_t n = read(_agentStderr,agentStderrBuf,sizeof(agentStderrBuf)); if (n > 0) write(STDERR_FILENO,agentStderrBuf,(size_t)n); */ } } ::close(_agentStdin); ::close(_agentStdout); ::close(_agentStderr); ::close(_agentStdin2); ::close(_agentStdout2); ::close(_agentStderr2); ::close(_shutdownSignalPipe[0]); ::close(_shutdownSignalPipe[1]); } void MacEthernetTap::setDns(const char* domain, const std::vector& servers) { MacDNSHelper::setDNS(this->_nwid, domain, servers); } } // namespace ZeroTier #endif // __APPLE__