For #1579, support gracefully quit. 3.0.119

README.md

@@ -146,6 +146,7 @@ For previous versions, please read:
 
 ## V3 changes
 
+* v3.0, 2020-02-18, For [#1579][bug #1579], support gracefully quit. 3.0.119
 * v3.0, 2020-02-17, For [#1601][bug #1601], flush async on_dvr/on_hls events before stop. 3.0.118
 * <strong>v3.0, 2020-02-14, [3.0 beta1(3.0.117)][r3.0b1] released. 121964 lines.</strong>
 * v3.0, 2020-02-14, For [#1595][bug #1595], migrating streaming from ossrs.net to r.ossrs.net. 3.0.117
@@ -1651,6 +1652,7 @@ Winlin
 [bug #665]: https://github.com/ossrs/srs/issues/665
 [bug #1595]: https://github.com/ossrs/srs/issues/1595
 [bug #1601]: https://github.com/ossrs/srs/issues/1601
+[bug #1579]: https://github.com/ossrs/srs/issues/1579
 [bug #xxxxxxxxxxxxx]: https://github.com/ossrs/srs/issues/xxxxxxxxxxxxx
 
 [exo #828]: https://github.com/google/ExoPlayer/pull/828

trunk/conf/full.conf

@@ -73,6 +73,10 @@ work_dir ./;
 # default: off
 asprocess off;
 
+# for gracefully quit, final wait for cleanup in milliseconds.
+# default: 3200
+grace_final_wait 3200;
+
 #############################################################################################
 # heartbeat/stats sections
 #############################################################################################

trunk/etc/init.d/srs

@@ -178,6 +178,18 @@ logrotate() {
     return 0
 }
 
+grace() {
+    # not start, exit
+    load_process_info
+    if [[ 0 -ne $? ]]; then failed_msg "SRS not start."; return 0; fi
+
+    ok_msg "Gracefully quit for SRS(pid ${srs_pid})..."
+    kill -s SIGQUIT ${srs_pid}
+
+    ok_msg "Gracefully quit"
+    return 0
+}
+
 menu() {
     case "$1" in
         start)
@@ -199,10 +211,14 @@ menu() {
         rotate)
             logrotate
             ;;
+        grace)
+            grace
+            ;;
         *)
-            echo "Usage: $0 {start|stop|status|restart|reload|rotate}"
+            echo "Usage: $0 {start|stop|status|restart|reload|rotate|grace}"
             echo "    reload    Apply log file by not restarting SRS"
             echo "    rotate    For log rotate, to send SIGUSR1 to SRS to reopen the log file."
+            echo "    grace    For gracefully quit, to send SIGQUIT to SRS."
             return 1
             ;;
     esac

trunk/src/app/srs_app_config.cpp

@@ -3487,7 +3487,7 @@ srs_error_t SrsConfig::check_normal_config()
             && n != "http_api" && n != "stats" && n != "vhost" && n != "pithy_print_ms"
             && n != "http_server" && n != "stream_caster"
             && n != "utc_time" && n != "work_dir" && n != "asprocess"
-            && n != "ff_log_level"
+            && n != "ff_log_level" && n != "grace_final_wait"
             ) {
             return srs_error_new(ERROR_SYSTEM_CONFIG_INVALID, "illegal directive %s", n.c_str());
         }
@@ -4050,6 +4050,18 @@ bool SrsConfig::get_asprocess()
     return SRS_CONF_PERFER_FALSE(conf->arg0());
 }
 
+srs_utime_t SrsConfig::get_grace_final_wait()
+{
+    static srs_utime_t DEFAULT = 3200 * SRS_UTIME_MILLISECONDS;
+
+    SrsConfDirective* conf = root->get("grace_final_wait");
+    if (!conf || conf->arg0().empty()) {
+        return DEFAULT;
+    }
+
+    return (srs_utime_t)(::atol(conf->arg0().c_str()) * SRS_UTIME_MILLISECONDS);
+}
+
 vector<SrsConfDirective*> SrsConfig::get_stream_casters()
 {
     srs_assert(root);

trunk/src/app/srs_app_config.hpp

@@ -468,6 +468,8 @@ public:
     virtual std::string get_work_dir();
     // Whether use asprocess mode.
     virtual bool get_asprocess();
+    // Get the final wait in ms for gracefully quit.
+    virtual srs_utime_t get_grace_final_wait();
 // stream_caster section
 public:
     // Get all stream_caster in config file.

trunk/src/app/srs_app_ffmpeg.cpp

@@ -230,7 +230,7 @@ srs_error_t SrsFFMPEG::start()
     params.push_back(ffmpeg);
     
     // input params
-    for (int i = 0; i < iparams.size(); i++) {
+    for (int i = 0; i < (int)iparams.size(); i++) {
         string iparam = iparams.at(i);
         if (!iparam.empty()) {
             params.push_back(iparam);

trunk/src/app/srs_app_server.cpp

@@ -397,6 +397,11 @@ srs_error_t SrsSignalManager::start()
     sa.sa_flags = 0;
     sigaction(SRS_SIGNAL_FAST_QUIT, &sa, NULL);
 
+    sa.sa_handler = SrsSignalManager::sig_catcher;
+    sigemptyset(&sa.sa_mask);
+    sa.sa_flags = 0;
+    sigaction(SRS_SIGNAL_GRACEFULLY_QUIT, &sa, NULL);
+    
     sa.sa_handler = SrsSignalManager::sig_catcher;
     sigemptyset(&sa.sa_mask);
     sa.sa_flags = 0;
@@ -407,8 +412,8 @@ srs_error_t SrsSignalManager::start()
     sa.sa_flags = 0;
     sigaction(SRS_SIGNAL_REOPEN_LOG, &sa, NULL);
     
-    srs_trace("signal installed, reload=%d, reopen=%d, fast_quit=%d",
+    srs_trace("signal installed, reload=%d, reopen=%d, fast_quit=%d, grace_quit=%d",
-              SRS_SIGNAL_RELOAD, SRS_SIGNAL_REOPEN_LOG, SRS_SIGNAL_FAST_QUIT);
+              SRS_SIGNAL_RELOAD, SRS_SIGNAL_REOPEN_LOG, SRS_SIGNAL_FAST_QUIT, SRS_SIGNAL_GRACEFULLY_QUIT);
     
     if ((err = trd->start()) != srs_success) {
         return srs_error_wrap(err, "signal manager");
@@ -465,6 +470,7 @@ SrsServer::SrsServer()
     signal_reload = false;
     signal_persistence_config = false;
     signal_gmc_stop = false;
+    signal_fast_quit = false;
     signal_gracefully_quit = false;
     pid_fd = -1;
     
@@ -533,6 +539,44 @@ void SrsServer::dispose()
 #endif
 }
 
+void SrsServer::gracefully_dispose()
+{
+    _srs_config->unsubscribe(this);
+
+    // prevent fresh clients.
+    close_listeners(SrsListenerRtmpStream);
+    close_listeners(SrsListenerHttpApi);
+    close_listeners(SrsListenerHttpStream);
+    close_listeners(SrsListenerMpegTsOverUdp);
+    close_listeners(SrsListenerRtsp);
+    close_listeners(SrsListenerFlv);
+
+    // Fast stop to notify FFMPEG to quit, wait for a while then fast kill.
+    ingester->stop();
+
+    // Wait for connections to quit.
+    // While gracefully quiting, user can requires SRS to fast quit.
+    int wait_step = 1;
+    while (!conns.empty() && !signal_fast_quit) {
+        for (int i = 0; i < wait_step && !conns.empty() && !signal_fast_quit; i++) {
+            srs_usleep(1000 * SRS_UTIME_MILLISECONDS);
+        }
+
+        wait_step = (wait_step * 2) % 33;
+        srs_trace("wait for %d conns to quit", conns.size());
+    }
+
+    // dispose the source for hls and dvr.
+    _srs_sources->dispose();
+
+#ifdef SRS_AUTO_MEM_WATCH
+    srs_memory_report();
+#endif
+
+    srs_usleep(_srs_config->get_grace_final_wait());
+    srs_trace("final wait for another %dms", srsu2msi(_srs_config->get_grace_final_wait()));
+}
+
 srs_error_t SrsServer::initialize(ISrsServerCycle* ch)
 {
     srs_error_t err = srs_success;
@@ -807,10 +851,25 @@ srs_error_t SrsServer::cycle()
     srs_warn("sleep a long time for system st-threads to cleanup.");
     srs_usleep(3 * 1000 * 1000);
     srs_warn("system quit");
-#else
+
-    // normally quit with neccessary cleanup by dispose().
+    return err;
+#endif
+
+    // quit normally.
     srs_warn("main cycle terminated, system quit normally.");
+
+    // fast quit, do some essential cleanup.
+    if (signal_fast_quit) {
         dispose();
+        srs_trace("srs disposed");
+    }
+
+    // gracefully quit, do carefully cleanup.
+    if (signal_gracefully_quit) {
+        gracefully_dispose();
+        srs_trace("srs gracefully quit");
+    }
+
     srs_trace("srs terminated");
     
     // for valgrind to detect.
@@ -818,7 +877,6 @@ srs_error_t SrsServer::cycle()
     srs_freep(_srs_log);
 
     exit(0);
-#endif
 
     return err;
 }
@@ -863,8 +921,14 @@ void SrsServer::on_signal(int signo)
 #endif
     }
     
-    if ((signo == SIGINT || signo == SRS_SIGNAL_FAST_QUIT) && !signal_gracefully_quit) {
+    if ((signo == SIGINT || signo == SRS_SIGNAL_FAST_QUIT) && !signal_fast_quit) {
         srs_trace("sig=%d, user terminate program, fast quit", signo);
+        signal_fast_quit = true;
+        return;
+    }
+
+    if (signo == SRS_SIGNAL_GRACEFULLY_QUIT && !signal_gracefully_quit) {
+        srs_trace("sig=%d, user start gracefully quit", signo);
         signal_gracefully_quit = true;
         return;
     }
@@ -909,9 +973,9 @@ srs_error_t SrsServer::do_cycle()
                 return srs_error_new(ERROR_ASPROCESS_PPID, "asprocess ppid changed from %d to %d", ppid, ::getppid());
             }
             
-            // gracefully quit for SIGINT or SIGTERM.
+            // gracefully quit for SIGINT or SIGTERM or SIGQUIT.
-            if (signal_gracefully_quit) {
+            if (signal_fast_quit || signal_gracefully_quit) {
-                srs_trace("cleanup for gracefully terminate.");
+                srs_trace("cleanup for quit signal fast=%d, grace=%d", signal_fast_quit, signal_gracefully_quit);
                 return err;
             }
             

trunk/src/app/srs_app_server.hpp

@@ -227,6 +227,7 @@ private:
     bool signal_reload;
     bool signal_persistence_config;
     bool signal_gmc_stop;
+    bool signal_fast_quit;
     bool signal_gracefully_quit;
     // Parent pid for asprocess.
     int ppid;
@@ -241,6 +242,9 @@ private:
     // When SIGTERM, SRS should do cleanup, for example,
     // to stop all ingesters, cleanup HLS and dvr.
     virtual void dispose();
+    // Close listener to stop accepting new connections,
+    // then wait and quit when all connections finished.
+    virtual void gracefully_dispose();
 // server startup workflow, @see run_master()
 public:
     // Initialize server with callback handler ch.

trunk/src/core/srs_core_version3.hpp

@@ -24,6 +24,6 @@
 #ifndef SRS_CORE_VERSION3_HPP
 #define SRS_CORE_VERSION3_HPP
 
-#define SRS_VERSION3_REVISION 118
+#define SRS_VERSION3_REVISION 119
 
 #endif

trunk/src/kernel/srs_kernel_consts.hpp

@@ -132,7 +132,6 @@
 #define SRS_SIGNAL_FAST_QUIT SIGTERM
 // The signal for srs to gracefully quit, do carefully dispose then exit.
 // @see https://github.com/ossrs/srs/issues/1579
-// TODO: FIXME: Not implemented.
 #define SRS_SIGNAL_GRACEFULLY_QUIT SIGQUIT
 
 // The application level signals.