diff --git a/README.md b/README.md index 7e94b13bd..c99ca714b 100755 --- a/README.md +++ b/README.md @@ -146,6 +146,7 @@ For previous versions, please read: ## V3 changes +* v3.0, 2020-02-18, For [#1579][bug #1579], support gracefully quit. 3.0.119 * v3.0, 2020-02-17, For [#1601][bug #1601], flush async on_dvr/on_hls events before stop. 3.0.118 * v3.0, 2020-02-14, [3.0 beta1(3.0.117)][r3.0b1] released. 121964 lines. * v3.0, 2020-02-14, For [#1595][bug #1595], migrating streaming from ossrs.net to r.ossrs.net. 3.0.117 @@ -1651,6 +1652,7 @@ Winlin [bug #665]: https://github.com/ossrs/srs/issues/665 [bug #1595]: https://github.com/ossrs/srs/issues/1595 [bug #1601]: https://github.com/ossrs/srs/issues/1601 +[bug #1579]: https://github.com/ossrs/srs/issues/1579 [bug #xxxxxxxxxxxxx]: https://github.com/ossrs/srs/issues/xxxxxxxxxxxxx [exo #828]: https://github.com/google/ExoPlayer/pull/828 diff --git a/trunk/conf/full.conf b/trunk/conf/full.conf index c1e268ad8..137d91884 100644 --- a/trunk/conf/full.conf +++ b/trunk/conf/full.conf @@ -73,6 +73,10 @@ work_dir ./; # default: off asprocess off; +# for gracefully quit, final wait for cleanup in milliseconds. +# default: 3200 +grace_final_wait 3200; + ############################################################################################# # heartbeat/stats sections ############################################################################################# diff --git a/trunk/etc/init.d/srs b/trunk/etc/init.d/srs index 7e749044d..0e2afca4b 100755 --- a/trunk/etc/init.d/srs +++ b/trunk/etc/init.d/srs @@ -178,6 +178,18 @@ logrotate() { return 0 } +grace() { + # not start, exit + load_process_info + if [[ 0 -ne $? ]]; then failed_msg "SRS not start."; return 0; fi + + ok_msg "Gracefully quit for SRS(pid ${srs_pid})..." + kill -s SIGQUIT ${srs_pid} + + ok_msg "Gracefully quit" + return 0 +} + menu() { case "$1" in start) @@ -199,10 +211,14 @@ menu() { rotate) logrotate ;; + grace) + grace + ;; *) - echo "Usage: $0 {start|stop|status|restart|reload|rotate}" + echo "Usage: $0 {start|stop|status|restart|reload|rotate|grace}" echo " reload Apply log file by not restarting SRS" echo " rotate For log rotate, to send SIGUSR1 to SRS to reopen the log file." + echo " grace For gracefully quit, to send SIGQUIT to SRS." return 1 ;; esac diff --git a/trunk/src/app/srs_app_config.cpp b/trunk/src/app/srs_app_config.cpp index 79a5b5caf..363ad358b 100644 --- a/trunk/src/app/srs_app_config.cpp +++ b/trunk/src/app/srs_app_config.cpp @@ -3487,7 +3487,7 @@ srs_error_t SrsConfig::check_normal_config() && n != "http_api" && n != "stats" && n != "vhost" && n != "pithy_print_ms" && n != "http_server" && n != "stream_caster" && n != "utc_time" && n != "work_dir" && n != "asprocess" - && n != "ff_log_level" + && n != "ff_log_level" && n != "grace_final_wait" ) { return srs_error_new(ERROR_SYSTEM_CONFIG_INVALID, "illegal directive %s", n.c_str()); } @@ -4050,6 +4050,18 @@ bool SrsConfig::get_asprocess() return SRS_CONF_PERFER_FALSE(conf->arg0()); } +srs_utime_t SrsConfig::get_grace_final_wait() +{ + static srs_utime_t DEFAULT = 3200 * SRS_UTIME_MILLISECONDS; + + SrsConfDirective* conf = root->get("grace_final_wait"); + if (!conf || conf->arg0().empty()) { + return DEFAULT; + } + + return (srs_utime_t)(::atol(conf->arg0().c_str()) * SRS_UTIME_MILLISECONDS); +} + vector SrsConfig::get_stream_casters() { srs_assert(root); diff --git a/trunk/src/app/srs_app_config.hpp b/trunk/src/app/srs_app_config.hpp index e814b8777..3c04c3bdb 100644 --- a/trunk/src/app/srs_app_config.hpp +++ b/trunk/src/app/srs_app_config.hpp @@ -468,6 +468,8 @@ public: virtual std::string get_work_dir(); // Whether use asprocess mode. virtual bool get_asprocess(); + // Get the final wait in ms for gracefully quit. + virtual srs_utime_t get_grace_final_wait(); // stream_caster section public: // Get all stream_caster in config file. diff --git a/trunk/src/app/srs_app_ffmpeg.cpp b/trunk/src/app/srs_app_ffmpeg.cpp index e9c87a70e..1cde4f5f9 100644 --- a/trunk/src/app/srs_app_ffmpeg.cpp +++ b/trunk/src/app/srs_app_ffmpeg.cpp @@ -230,7 +230,7 @@ srs_error_t SrsFFMPEG::start() params.push_back(ffmpeg); // input params - for (int i = 0; i < iparams.size(); i++) { + for (int i = 0; i < (int)iparams.size(); i++) { string iparam = iparams.at(i); if (!iparam.empty()) { params.push_back(iparam); diff --git a/trunk/src/app/srs_app_server.cpp b/trunk/src/app/srs_app_server.cpp index 71af22927..4c0a4b2fb 100644 --- a/trunk/src/app/srs_app_server.cpp +++ b/trunk/src/app/srs_app_server.cpp @@ -396,6 +396,11 @@ srs_error_t SrsSignalManager::start() sigemptyset(&sa.sa_mask); sa.sa_flags = 0; sigaction(SRS_SIGNAL_FAST_QUIT, &sa, NULL); + + sa.sa_handler = SrsSignalManager::sig_catcher; + sigemptyset(&sa.sa_mask); + sa.sa_flags = 0; + sigaction(SRS_SIGNAL_GRACEFULLY_QUIT, &sa, NULL); sa.sa_handler = SrsSignalManager::sig_catcher; sigemptyset(&sa.sa_mask); @@ -407,8 +412,8 @@ srs_error_t SrsSignalManager::start() sa.sa_flags = 0; sigaction(SRS_SIGNAL_REOPEN_LOG, &sa, NULL); - srs_trace("signal installed, reload=%d, reopen=%d, fast_quit=%d", - SRS_SIGNAL_RELOAD, SRS_SIGNAL_REOPEN_LOG, SRS_SIGNAL_FAST_QUIT); + srs_trace("signal installed, reload=%d, reopen=%d, fast_quit=%d, grace_quit=%d", + SRS_SIGNAL_RELOAD, SRS_SIGNAL_REOPEN_LOG, SRS_SIGNAL_FAST_QUIT, SRS_SIGNAL_GRACEFULLY_QUIT); if ((err = trd->start()) != srs_success) { return srs_error_wrap(err, "signal manager"); @@ -465,6 +470,7 @@ SrsServer::SrsServer() signal_reload = false; signal_persistence_config = false; signal_gmc_stop = false; + signal_fast_quit = false; signal_gracefully_quit = false; pid_fd = -1; @@ -533,6 +539,44 @@ void SrsServer::dispose() #endif } +void SrsServer::gracefully_dispose() +{ + _srs_config->unsubscribe(this); + + // prevent fresh clients. + close_listeners(SrsListenerRtmpStream); + close_listeners(SrsListenerHttpApi); + close_listeners(SrsListenerHttpStream); + close_listeners(SrsListenerMpegTsOverUdp); + close_listeners(SrsListenerRtsp); + close_listeners(SrsListenerFlv); + + // Fast stop to notify FFMPEG to quit, wait for a while then fast kill. + ingester->stop(); + + // Wait for connections to quit. + // While gracefully quiting, user can requires SRS to fast quit. + int wait_step = 1; + while (!conns.empty() && !signal_fast_quit) { + for (int i = 0; i < wait_step && !conns.empty() && !signal_fast_quit; i++) { + srs_usleep(1000 * SRS_UTIME_MILLISECONDS); + } + + wait_step = (wait_step * 2) % 33; + srs_trace("wait for %d conns to quit", conns.size()); + } + + // dispose the source for hls and dvr. + _srs_sources->dispose(); + +#ifdef SRS_AUTO_MEM_WATCH + srs_memory_report(); +#endif + + srs_usleep(_srs_config->get_grace_final_wait()); + srs_trace("final wait for another %dms", srsu2msi(_srs_config->get_grace_final_wait())); +} + srs_error_t SrsServer::initialize(ISrsServerCycle* ch) { srs_error_t err = srs_success; @@ -807,19 +851,33 @@ srs_error_t SrsServer::cycle() srs_warn("sleep a long time for system st-threads to cleanup."); srs_usleep(3 * 1000 * 1000); srs_warn("system quit"); -#else - // normally quit with neccessary cleanup by dispose(). + + return err; +#endif + + // quit normally. srs_warn("main cycle terminated, system quit normally."); - dispose(); + + // fast quit, do some essential cleanup. + if (signal_fast_quit) { + dispose(); + srs_trace("srs disposed"); + } + + // gracefully quit, do carefully cleanup. + if (signal_gracefully_quit) { + gracefully_dispose(); + srs_trace("srs gracefully quit"); + } + srs_trace("srs terminated"); // for valgrind to detect. srs_freep(_srs_config); srs_freep(_srs_log); - + exit(0); -#endif - + return err; } @@ -863,8 +921,14 @@ void SrsServer::on_signal(int signo) #endif } - if ((signo == SIGINT || signo == SRS_SIGNAL_FAST_QUIT) && !signal_gracefully_quit) { + if ((signo == SIGINT || signo == SRS_SIGNAL_FAST_QUIT) && !signal_fast_quit) { srs_trace("sig=%d, user terminate program, fast quit", signo); + signal_fast_quit = true; + return; + } + + if (signo == SRS_SIGNAL_GRACEFULLY_QUIT && !signal_gracefully_quit) { + srs_trace("sig=%d, user start gracefully quit", signo); signal_gracefully_quit = true; return; } @@ -909,9 +973,9 @@ srs_error_t SrsServer::do_cycle() return srs_error_new(ERROR_ASPROCESS_PPID, "asprocess ppid changed from %d to %d", ppid, ::getppid()); } - // gracefully quit for SIGINT or SIGTERM. - if (signal_gracefully_quit) { - srs_trace("cleanup for gracefully terminate."); + // gracefully quit for SIGINT or SIGTERM or SIGQUIT. + if (signal_fast_quit || signal_gracefully_quit) { + srs_trace("cleanup for quit signal fast=%d, grace=%d", signal_fast_quit, signal_gracefully_quit); return err; } diff --git a/trunk/src/app/srs_app_server.hpp b/trunk/src/app/srs_app_server.hpp index 5ab4f011f..28154600e 100644 --- a/trunk/src/app/srs_app_server.hpp +++ b/trunk/src/app/srs_app_server.hpp @@ -227,6 +227,7 @@ private: bool signal_reload; bool signal_persistence_config; bool signal_gmc_stop; + bool signal_fast_quit; bool signal_gracefully_quit; // Parent pid for asprocess. int ppid; @@ -241,6 +242,9 @@ private: // When SIGTERM, SRS should do cleanup, for example, // to stop all ingesters, cleanup HLS and dvr. virtual void dispose(); + // Close listener to stop accepting new connections, + // then wait and quit when all connections finished. + virtual void gracefully_dispose(); // server startup workflow, @see run_master() public: // Initialize server with callback handler ch. diff --git a/trunk/src/core/srs_core_version3.hpp b/trunk/src/core/srs_core_version3.hpp index 4ddcca91c..c27634123 100644 --- a/trunk/src/core/srs_core_version3.hpp +++ b/trunk/src/core/srs_core_version3.hpp @@ -24,6 +24,6 @@ #ifndef SRS_CORE_VERSION3_HPP #define SRS_CORE_VERSION3_HPP -#define SRS_VERSION3_REVISION 118 +#define SRS_VERSION3_REVISION 119 #endif diff --git a/trunk/src/kernel/srs_kernel_consts.hpp b/trunk/src/kernel/srs_kernel_consts.hpp index a1a946e62..208765ae7 100644 --- a/trunk/src/kernel/srs_kernel_consts.hpp +++ b/trunk/src/kernel/srs_kernel_consts.hpp @@ -132,7 +132,6 @@ #define SRS_SIGNAL_FAST_QUIT SIGTERM // The signal for srs to gracefully quit, do carefully dispose then exit. // @see https://github.com/ossrs/srs/issues/1579 -// TODO: FIXME: Not implemented. #define SRS_SIGNAL_GRACEFULLY_QUIT SIGQUIT // The application level signals.