From feaf98eb69bcdc99ab0bd102582cb0cd2520956d Mon Sep 17 00:00:00 2001 From: winlin Date: Tue, 14 Apr 2020 10:58:53 +0800 Subject: [PATCH] For #307, remove dedicate GSO cache --- trunk/conf/full.conf | 6 +- trunk/src/app/srs_app_config.cpp | 28 +------- trunk/src/app/srs_app_config.hpp | 1 - trunk/src/app/srs_app_listener.hpp | 4 -- trunk/src/app/srs_app_rtc_conn.cpp | 100 ++++------------------------- trunk/src/app/srs_app_rtc_conn.hpp | 10 --- 6 files changed, 13 insertions(+), 136 deletions(-) diff --git a/trunk/conf/full.conf b/trunk/conf/full.conf index 45156da17..2e40a387e 100644 --- a/trunk/conf/full.conf +++ b/trunk/conf/full.conf @@ -440,13 +440,9 @@ rtc_server { # default: on merge_nalus on; # Whether enable GSO to send out RTP packets. - # @remark Linux only, for other OS always disabled. + # @remark Linux 4.18+ only, for other OS always disabled. # default: off gso off; - # Whether GSO uses dedicated cache. Share the same mmsghdr cache if not dedicated. - # For Linux 3.*, we must use dedicated cache for GSO. - # default: off - gso_dedicated off; } vhost rtc.vhost.srs.com { diff --git a/trunk/src/app/srs_app_config.cpp b/trunk/src/app/srs_app_config.cpp index 9693a17fa..f7f69a381 100644 --- a/trunk/src/app/srs_app_config.cpp +++ b/trunk/src/app/srs_app_config.cpp @@ -3614,8 +3614,7 @@ srs_error_t SrsConfig::check_normal_config() for (int i = 0; conf && i < (int)conf->directives.size(); i++) { string n = conf->at(i)->name; if (n != "enabled" && n != "listen" && n != "dir" && n != "candidate" && n != "ecdsa" - && n != "sendmmsg" && n != "encrypt" && n != "reuseport" && n != "gso" && n != "merge_nalus" - && n != "gso_dedicated") { + && n != "sendmmsg" && n != "encrypt" && n != "reuseport" && n != "gso" && n != "merge_nalus") { return srs_error_new(ERROR_SYSTEM_CONFIG_INVALID, "illegal rtc_server.%s", n.c_str()); } } @@ -4815,31 +4814,6 @@ bool SrsConfig::get_rtc_server_gso() return v; } -bool SrsConfig::get_rtc_server_gso_dedicated() -{ - static int DEFAULT = false; - - SrsConfDirective* conf = root->get("rtc_server"); - if (!conf) { - return DEFAULT; - } - - conf = conf->get("gso_dedicated"); - if (!conf || conf->arg0().empty()) { - return DEFAULT; - } - - bool v = SRS_CONF_PERFER_FALSE(conf->arg0()); - -#ifdef SRS_AUTO_OSX - if (v != DEFAULT) { - srs_warn("GSO is for Linux only"); - } -#endif - - return v; -} - SrsConfDirective* SrsConfig::get_rtc(string vhost) { SrsConfDirective* conf = get_vhost(vhost); diff --git a/trunk/src/app/srs_app_config.hpp b/trunk/src/app/srs_app_config.hpp index bfc6f373f..8093a50a4 100644 --- a/trunk/src/app/srs_app_config.hpp +++ b/trunk/src/app/srs_app_config.hpp @@ -530,7 +530,6 @@ public: virtual int get_rtc_server_reuseport(); virtual bool get_rtc_server_merge_nalus(); virtual bool get_rtc_server_gso(); - virtual bool get_rtc_server_gso_dedicated(); SrsConfDirective* get_rtc(std::string vhost); bool get_rtc_enabled(std::string vhost); diff --git a/trunk/src/app/srs_app_listener.hpp b/trunk/src/app/srs_app_listener.hpp index ec503cf6b..8ef384dd9 100644 --- a/trunk/src/app/srs_app_listener.hpp +++ b/trunk/src/app/srs_app_listener.hpp @@ -139,10 +139,6 @@ public: public: // Fetch a mmsghdr from sender's cache. virtual srs_error_t fetch(mmsghdr** pphdr) = 0; - // For Linux kernel 3.*, we must use isolate APIs for GSO, - // that is, sendmmsg does not work with GSO. - // Fetch a mmsghdr from sender's cache. - virtual srs_error_t gso_fetch(mmsghdr** pphdr) = 0; // Notify the sender to send out the msg. virtual srs_error_t sendmmsg(mmsghdr* hdr) = 0; }; diff --git a/trunk/src/app/srs_app_rtc_conn.cpp b/trunk/src/app/srs_app_rtc_conn.cpp index 2d3726043..773ac594c 100644 --- a/trunk/src/app/srs_app_rtc_conn.cpp +++ b/trunk/src/app/srs_app_rtc_conn.cpp @@ -813,6 +813,7 @@ srs_error_t SrsRtcSenderThread::send_packets(SrsUdpMuxSocket* skt, SrsRtcPackets mhdr->msg_hdr.msg_name = (sockaddr_in*)addr; mhdr->msg_hdr.msg_namelen = (socklen_t)addrlen; mhdr->msg_hdr.msg_iov->iov_len = length; + mhdr->msg_hdr.msg_controllen = 0; mhdr->msg_len = 0; if ((err = sender->sendmmsg(mhdr)) != srs_success) { @@ -889,12 +890,7 @@ srs_error_t SrsRtcSenderThread::send_packets2(SrsUdpMuxSocket* skt, SrsRtcPacket if (!mhdr) { // Fetch a cached message from queue. // TODO: FIXME: Maybe encrypt in async, so the state of mhdr maybe not ready. - if (use_gso) { - err = sender->gso_fetch(&mhdr); - } else { - err = sender->fetch(&mhdr); - } - if (err != srs_success) { + if ((err = sender->fetch(&mhdr)) != srs_success) { return srs_error_wrap(err, "fetch msghdr"); } @@ -1689,7 +1685,6 @@ SrsUdpMuxSender::SrsUdpMuxSender(SrsRtcServer* s) trd = new SrsDummyCoroutine(); cache_pos = 0; - gso_cache_pos = 0; _srs_config->subscribe(this); } @@ -1706,12 +1701,6 @@ SrsUdpMuxSender::~SrsUdpMuxSender() free_mhdrs(cache); cache.clear(); - - free_mhdrs(gso_hotspot); - gso_hotspot.clear(); - - free_mhdrs(gso_cache); - gso_cache.clear(); } srs_error_t SrsUdpMuxSender::initialize(srs_netfd_t fd) @@ -1728,9 +1717,7 @@ srs_error_t SrsUdpMuxSender::initialize(srs_netfd_t fd) max_sendmmsg = _srs_config->get_rtc_server_sendmmsg(); bool gso = _srs_config->get_rtc_server_gso(); - gso_dedicated = _srs_config->get_rtc_server_gso_dedicated(); - srs_trace("UDP sender #%d init ok, max_sendmmsg=%d, gso=%d, gso-dedicated=%d", - srs_netfd_fileno(fd), max_sendmmsg, gso, gso_dedicated); + srs_trace("UDP sender #%d init ok, max_sendmmsg=%d, gso=%d", srs_netfd_fileno(fd), max_sendmmsg, gso); return err; } @@ -1775,34 +1762,6 @@ srs_error_t SrsUdpMuxSender::fetch(mmsghdr** pphdr) return srs_success; } -srs_error_t SrsUdpMuxSender::gso_fetch(mmsghdr** pphdr) -{ - // When GSO share cache, we use the same cache with non-GSO. - if (!gso_dedicated) { - return fetch(pphdr); - } - - // TODO: FIXME: Maybe need to shrink? - if (gso_cache_pos >= (int)gso_cache.size()) { - mmsghdr mhdr; - memset(&mhdr, 0, sizeof(mmsghdr)); - - mhdr.msg_hdr.msg_iovlen = 1; - mhdr.msg_hdr.msg_iov = new iovec(); - mhdr.msg_hdr.msg_iov->iov_base = new char[kRtpPacketSize]; - mhdr.msg_hdr.msg_iov->iov_len = kRtpPacketSize; - mhdr.msg_len = 0; - - mhdr.msg_hdr.msg_controllen = CMSG_SPACE(sizeof(uint16_t)); - mhdr.msg_hdr.msg_control = new char[mhdr.msg_hdr.msg_controllen]; - - gso_cache.push_back(mhdr); - } - - *pphdr = &gso_cache[gso_cache_pos++]; - return srs_success; -} - srs_error_t SrsUdpMuxSender::sendmmsg(mmsghdr* hdr) { if (waiting_msgs) { @@ -1834,7 +1793,7 @@ srs_error_t SrsUdpMuxSender::cycle() nn_loop++; int pos = cache_pos; - int gso_pos = gso_cache_pos; + int gso_pos = 0; int gso_iovs = 0; if (pos <= 0 && gso_pos == 0) { waiting_msgs = true; @@ -1847,13 +1806,8 @@ srs_error_t SrsUdpMuxSender::cycle() cache.swap(hotspot); cache_pos = 0; - if (gso_dedicated) { - gso_cache.swap(gso_hotspot); - gso_cache_pos = 0; - } - // Collect informations for GSO - if (!gso_dedicated && pos > 0) { + if (pos > 0) { // For shared GSO cache, stat the messages. mmsghdr* p = &hotspot[0]; mmsghdr* end = p + pos; for (p = &hotspot[0]; p < end; p++) { @@ -1871,7 +1825,7 @@ srs_error_t SrsUdpMuxSender::cycle() } // Send out all messages, may GSO if shared cache. - if (false && pos > 0) { + if (pos > 0) { // Send out all messages. mmsghdr* p = &hotspot[0]; mmsghdr* end = p + pos; for (p = &hotspot[0]; p < end; p += max_sendmmsg) { @@ -1887,32 +1841,9 @@ srs_error_t SrsUdpMuxSender::cycle() } } - // Send out GSO in dedicated queue. - if (gso_dedicated && gso_pos > 0) { - mmsghdr* p = &gso_hotspot[0]; mmsghdr* end = p + gso_pos; - for (; p < end; p++) { - // Private message, use it to store the cursor. - int real_iovs = p->msg_len; - p->msg_len = 0; - - // Send out GSO message. - int r0 = srs_sendmsg(lfd, &p->msg_hdr, 0, SRS_UTIME_NO_TIMEOUT); - if (r0 < 0) { - srs_warn("sendmsg err, r0=%d", r0); - } - - nn_gso_msgs++; nn_gso_iovs += real_iovs; gso_iovs += real_iovs; - stat->perf_gso_on_packets(real_iovs); - } - } - // Increase total messages. - int nn_pos = pos; - if (gso_dedicated) { - nn_pos = pos + gso_pos; - } - nn_msgs += nn_pos; - nn_msgs_max = srs_max(nn_pos, nn_msgs_max); + nn_msgs += pos; + nn_msgs_max = srs_max(pos, nn_msgs_max); nn_gso_msgs_max = srs_max(gso_pos, nn_gso_msgs_max); nn_gso_iovs_max = srs_max(gso_iovs, nn_gso_iovs_max); @@ -1936,9 +1867,9 @@ srs_error_t SrsUdpMuxSender::cycle() pps_unit = "(k)"; pps_last /= 1000; pps_average /= 1000; } - srs_trace("-> RTC #%d SEND %d/%d/%" PRId64 ", gso %d/%d/%" PRId64 ", gso-iovs %d/%d/%" PRId64 ", pps %d/%d%s, schedule %d/%d, sessions %d, cache %d/%d, sendmmsg %d", - srs_netfd_fileno(lfd), nn_pos, nn_msgs_max, nn_msgs, gso_pos, nn_gso_msgs_max, nn_gso_msgs, gso_iovs, nn_gso_iovs_max, nn_gso_iovs, pps_average, pps_last, pps_unit.c_str(), - nn_loop, nn_wait, (int)server->nn_sessions(), (int)cache.size(), (int)gso_cache.size(), max_sendmmsg); + srs_trace("-> RTC #%d SEND %d/%d/%" PRId64 ", gso %d/%d/%" PRId64 ", gso-iovs %d/%d/%" PRId64 ", pps %d/%d%s, schedule %d/%d, sessions %d, cache %d, sendmmsg %d", + srs_netfd_fileno(lfd), pos, nn_msgs_max, nn_msgs, gso_pos, nn_gso_msgs_max, nn_gso_msgs, gso_iovs, nn_gso_iovs_max, nn_gso_iovs, pps_average, pps_last, pps_unit.c_str(), + nn_loop, nn_wait, (int)server->nn_sessions(), (int)cache.size(), max_sendmmsg); nn_msgs_last = nn_msgs; time_last = srs_get_system_time(); nn_loop = nn_wait = nn_msgs_max = 0; nn_gso_msgs_max = 0; nn_gso_iovs_max = 0; @@ -1958,15 +1889,6 @@ srs_error_t SrsUdpMuxSender::on_reload_rtc_server() } } - if (true) { - bool gso = _srs_config->get_rtc_server_gso(); - bool v = _srs_config->get_rtc_server_gso_dedicated(); - if (gso_dedicated != v) { - srs_trace("Reload gso=%d, gso-dedicated %d=>%d", gso, gso_dedicated, v); - gso_dedicated = v; - } - } - return srs_success; } diff --git a/trunk/src/app/srs_app_rtc_conn.hpp b/trunk/src/app/srs_app_rtc_conn.hpp index 263420cd7..748374290 100644 --- a/trunk/src/app/srs_app_rtc_conn.hpp +++ b/trunk/src/app/srs_app_rtc_conn.hpp @@ -273,15 +273,6 @@ private: int cache_pos; // The max number of messages for sendmmsg. If 1, we use sendmsg to send. int max_sendmmsg; - // Whether GSO shares the same mmsghdr cache. - // If not shared, use dedicated cache for GSO. - bool gso_dedicated; -private: - // For Linux kernel 3.*, we must use isolate queue for GSO, - // that is, sendmmsg does not work with GSO. - std::vector gso_hotspot; - std::vector gso_cache; - int gso_cache_pos; public: SrsUdpMuxSender(SrsRtcServer* s); virtual ~SrsUdpMuxSender(); @@ -291,7 +282,6 @@ private: void free_mhdrs(std::vector& mhdrs); public: virtual srs_error_t fetch(mmsghdr** pphdr); - virtual srs_error_t gso_fetch(mmsghdr** pphdr); virtual srs_error_t sendmmsg(mmsghdr* hdr); virtual srs_error_t cycle(); // interface ISrsReloadHandler