diff --git a/trunk/auto/auto_headers.sh b/trunk/auto/auto_headers.sh index 80b086182..71402ca8b 100755 --- a/trunk/auto/auto_headers.sh +++ b/trunk/auto/auto_headers.sh @@ -68,6 +68,12 @@ else srs_undefine_macro "SRS_CXX14" $SRS_AUTO_HEADERS_H fi +if [ $SRS_BACKTRACE = YES ]; then + srs_define_macro "SRS_BACKTRACE" $SRS_AUTO_HEADERS_H +else + srs_undefine_macro "SRS_BACKTRACE" $SRS_AUTO_HEADERS_H +fi + if [ $SRS_RTC = YES ]; then srs_define_macro "SRS_RTC" $SRS_AUTO_HEADERS_H else diff --git a/trunk/auto/options.sh b/trunk/auto/options.sh index c84dc7924..6466163e2 100755 --- a/trunk/auto/options.sh +++ b/trunk/auto/options.sh @@ -9,6 +9,7 @@ SRS_RTC=YES SRS_GB28181=NO SRS_CXX11=YES SRS_CXX14=NO +SRS_BACKTRACE=YES SRS_NGINX=NO SRS_UTEST=NO # Always enable the bellow features. @@ -128,6 +129,7 @@ Features: --gb28181=on|off Whether build the GB28181. Default: $(value2switch $SRS_GB28181) --cxx11=on|off Whether enable the C++11. Default: $(value2switch $SRS_CXX11) --cxx14=on|off Whether enable the C++14. Default: $(value2switch $SRS_CXX14) + --backtrace=on|off Whether show backtrace when crashing. Default: $(value2switch $SRS_BACKTRACE) --ffmpeg-fit=on|off Whether enable the FFmpeg fit(source code). Default: $(value2switch $SRS_FFMPEG_FIT) --prefix= The absolute installation path. Default: $SRS_PREFIX @@ -298,6 +300,7 @@ function parse_user_option() { --cxx11) SRS_CXX11=$(switch2value $value) ;; --cxx14) SRS_CXX14=$(switch2value $value) ;; + --backtrace) SRS_BACKTRACE=$(switch2value $value) ;; --with-clean) SRS_CLEAN=YES ;; --without-clean) SRS_CLEAN=NO ;; @@ -525,6 +528,7 @@ function regenerate_options() { SRS_AUTO_CONFIGURE="${SRS_AUTO_CONFIGURE} --simulator=$(value2switch $SRS_SIMULATOR)" SRS_AUTO_CONFIGURE="${SRS_AUTO_CONFIGURE} --cxx11=$(value2switch $SRS_CXX11)" SRS_AUTO_CONFIGURE="${SRS_AUTO_CONFIGURE} --cxx14=$(value2switch $SRS_CXX14)" + SRS_AUTO_CONFIGURE="${SRS_AUTO_CONFIGURE} --backtrace=$(value2switch $SRS_BACKTRACE)" SRS_AUTO_CONFIGURE="${SRS_AUTO_CONFIGURE} --ffmpeg-fit=$(value2switch $SRS_FFMPEG_FIT)" SRS_AUTO_CONFIGURE="${SRS_AUTO_CONFIGURE} --nasm=$(value2switch $SRS_NASM)" SRS_AUTO_CONFIGURE="${SRS_AUTO_CONFIGURE} --srtp-nasm=$(value2switch $SRS_SRTP_ASM)" diff --git a/trunk/configure b/trunk/configure index c24a56c15..95f3d043a 100755 --- a/trunk/configure +++ b/trunk/configure @@ -193,6 +193,11 @@ if [[ $SRS_OSX != YES && $SRS_RTC == YES && $SRS_FFMPEG_FIT == YES ]]; then SrsLinkOptions="${SrsLinkOptions} -lrt"; fi +# The backtrace symbol option. +if [[ $SRS_BACKTRACE == YES ]]; then + SrsLinkOptions="${SrsLinkOptions} -rdynamic"; +fi + ##################################################################################### # Modules, compile each module, then link to binary # diff --git a/trunk/doc/CHANGELOG.md b/trunk/doc/CHANGELOG.md index eb069c5ae..56aaeb296 100644 --- a/trunk/doc/CHANGELOG.md +++ b/trunk/doc/CHANGELOG.md @@ -7,6 +7,7 @@ The changelog for SRS. ## SRS 5.0 Changelog +* v5.0, 2022-10-21, Kernel: Support grab backtrace stack when assert fail. v5.0.80 * v5.0, 2022-10-21, ST: Refine tools and CMakeLists.txt. Add backtrace example. v5.0.79 * v5.0, 2022-10-10, For [#2901](https://github.com/ossrs/srs/issues/2901): Edge: Fast disconnect and reconnect. v5.0.78 * v5.0, 2022-10-09, Fix [#3198](https://github.com/ossrs/srs/issues/3198): SRT: Support PUSH SRT by IP and optional port. v5.0.76 diff --git a/trunk/ide/srs_clion/CMakeLists.txt b/trunk/ide/srs_clion/CMakeLists.txt index f87d6090a..95ebaabbf 100755 --- a/trunk/ide/srs_clion/CMakeLists.txt +++ b/trunk/ide/srs_clion/CMakeLists.txt @@ -87,6 +87,7 @@ ADD_EXECUTABLE(srs ${SRS_SOURCE_FILES}) TARGET_LINK_LIBRARIES(srs dl) TARGET_LINK_LIBRARIES(srs ${DEPS_LIBS}) TARGET_LINK_LIBRARIES(srs -ldl -pthread) +TARGET_LINK_LIBRARIES(srs -rdynamic) ########################################################### # For utest. diff --git a/trunk/src/app/srs_app_rtmp_conn.cpp b/trunk/src/app/srs_app_rtmp_conn.cpp index 096ce983c..cf8c54af8 100644 --- a/trunk/src/app/srs_app_rtmp_conn.cpp +++ b/trunk/src/app/srs_app_rtmp_conn.cpp @@ -66,7 +66,7 @@ SrsSimpleRtmpClient::~SrsSimpleRtmpClient() srs_error_t SrsSimpleRtmpClient::connect_app() { std::vector& ips = srs_get_local_ips(); - assert(_srs_config->get_stats_network() < (int)ips.size()); + srs_assert(_srs_config->get_stats_network() < (int)ips.size()); SrsIPAddress* local_ip = ips[_srs_config->get_stats_network()]; bool debug_srs_upnode = _srs_config->get_debug_srs_upnode(req->vhost); diff --git a/trunk/src/core/srs_core_platform.hpp b/trunk/src/core/srs_core_platform.hpp index f7ca1ac7c..b25304aeb 100644 --- a/trunk/src/core/srs_core_platform.hpp +++ b/trunk/src/core/srs_core_platform.hpp @@ -28,11 +28,6 @@ #include #endif -#include -#ifndef srs_assert -#define srs_assert(expression) assert(expression) -#endif - #include #include diff --git a/trunk/src/core/srs_core_version5.hpp b/trunk/src/core/srs_core_version5.hpp index cd2aeaf39..fb52fbd86 100644 --- a/trunk/src/core/srs_core_version5.hpp +++ b/trunk/src/core/srs_core_version5.hpp @@ -9,6 +9,6 @@ #define VERSION_MAJOR 5 #define VERSION_MINOR 0 -#define VERSION_REVISION 79 +#define VERSION_REVISION 80 #endif diff --git a/trunk/src/kernel/srs_kernel_balance.cpp b/trunk/src/kernel/srs_kernel_balance.cpp index 3574e8e0c..2cfd618bf 100644 --- a/trunk/src/kernel/srs_kernel_balance.cpp +++ b/trunk/src/kernel/srs_kernel_balance.cpp @@ -6,6 +6,8 @@ #include +#include + using namespace std; SrsLbRoundRobin::SrsLbRoundRobin() diff --git a/trunk/src/kernel/srs_kernel_error.cpp b/trunk/src/kernel/srs_kernel_error.cpp index 8e33e07e8..ab4362eb0 100644 --- a/trunk/src/kernel/srs_kernel_error.cpp +++ b/trunk/src/kernel/srs_kernel_error.cpp @@ -12,10 +12,112 @@ #include #include #include +#include #include using namespace std; +#if defined(SRS_BACKTRACE) && defined(__linux) +#include +#include +#include +#include + +void* parse_symbol_offset(char* frame) +{ + char* p = NULL; + char* p_symbol = NULL; + int nn_symbol = 0; + char* p_offset = NULL; + int nn_offset = 0; + + // Read symbol and offset, for example: + // /tools/backtrace(foo+0x1820) [0x555555555820] + for (p = frame; *p; p++) { + if (*p == '(') { + p_symbol = p + 1; + } else if (*p == '+') { + if (p_symbol) nn_symbol = p - p_symbol; + p_offset = p + 1; + } else if (*p == ')') { + if (p_offset) nn_offset = p - p_offset; + } + } + if (!nn_symbol && !nn_offset) { + return NULL; + } + + // Convert offset(0x1820) to pointer, such as 0x1820. + char tmp[128]; + if (!nn_offset || nn_offset >= (int)sizeof(tmp)) { + return NULL; + } + + int r0 = EOF; + void* offset = NULL; + tmp[nn_offset] = 0; + if ((r0 = sscanf(strncpy(tmp, p_offset, nn_offset), "%p", &offset)) == EOF) { + return NULL; + } + + // Covert symbol(foo) to offset, such as 0x2fba. + if (!nn_symbol || nn_symbol >= (int)sizeof(tmp)) { + return offset; + } + + void* object_file; + if ((object_file = dlopen(NULL, RTLD_LAZY)) == NULL) { + return offset; + } + + void* address; + tmp[nn_symbol] = 0; + if ((address = dlsym(object_file, strncpy(tmp, p_symbol, nn_symbol))) == NULL) { + dlclose(object_file); + return offset; + } + + Dl_info symbol_info; + if ((r0 = dladdr(address, &symbol_info)) == 0) { + dlclose(object_file); + return offset; + } + + dlclose(object_file); + return (char*)symbol_info.dli_saddr - (char*)symbol_info.dli_fbase + (char*)offset; +} + +extern const char* _srs_binary; + +char* addr2line_format(void* addr, char* symbol, char* buffer, int nn_buffer) +{ + char cmd[512] = {0}; + int r0 = snprintf(cmd, sizeof(cmd), "addr2line -C -p -s -f -a -e %s %p", _srs_binary, (void*)((char*)addr - 1)); + if (r0 < 0 || r0 >= (int)sizeof(cmd)) return symbol; + + FILE* fp = popen(cmd, "r"); + if (!fp) return symbol; + + char* p = fgets(buffer, nn_buffer, fp); + pclose(fp); + + if (p == NULL) return symbol; + if ((r0 = strlen(p)) == 0) return symbol; + + // Trait the last newline if exists. + if (p[r0 - 1] == '\n') p[r0 - 1] = '\0'; + + // Find symbol not match by addr2line, like + // 0x0000000000021c87: ?? ??:0 + // 0x0000000000002ffa: _start at ??:? + for (p = buffer; p < buffer + r0 - 1; p++) { + if (p[0] == '?' && p[1] == '?') return symbol; + } + + return buffer; +} +#endif + bool srs_is_system_control_error(srs_error_t err) { int error_code = srs_error_code(err); @@ -266,3 +368,30 @@ std::string SrsCplxError::error_code_longstr(SrsCplxError* err) return it->second; } +void SrsCplxError::srs_assert(bool expression) +{ +#if defined(SRS_BACKTRACE) && defined(__linux) + if (!expression) { + void* addresses[64]; + int nn_addresses = backtrace(addresses, sizeof(addresses) / sizeof(void*)); + char** symbols = backtrace_symbols(addresses, nn_addresses); + + // No error code for assert failed. + errno = 0; + + char buffer[128]; + srs_error("backtrace %d frames of %s %s", nn_addresses, _srs_binary, RTMP_SIG_SRS_SERVER); + for (int i = 0; i < nn_addresses; i++) { + void* frame = parse_symbol_offset(symbols[i]); + char* fmt = addr2line_format(frame, symbols[i], buffer, sizeof(buffer)); + int parsed = (fmt == buffer); + srs_error("#%d %p %d %s", i, frame, parsed, fmt); + } + + free(symbols); + } +#endif + + assert(expression); +} + diff --git a/trunk/src/kernel/srs_kernel_error.hpp b/trunk/src/kernel/srs_kernel_error.hpp index 081198852..2a70f229a 100644 --- a/trunk/src/kernel/srs_kernel_error.hpp +++ b/trunk/src/kernel/srs_kernel_error.hpp @@ -439,6 +439,8 @@ public: static int error_code(SrsCplxError* err); static std::string error_code_str(SrsCplxError* err); static std::string error_code_longstr(SrsCplxError* err); +public: + static void srs_assert(bool expression); }; // Error helpers, should use these functions to new or wrap an error. @@ -453,5 +455,9 @@ public: #define srs_error_code_longstr(err) SrsCplxError::error_code_longstr(err) #define srs_error_reset(err) srs_freep(err); err = srs_success +#ifndef srs_assert +#define srs_assert(expression) SrsCplxError::srs_assert(expression) +#endif + #endif diff --git a/trunk/src/kernel/srs_kernel_kbps.cpp b/trunk/src/kernel/srs_kernel_kbps.cpp index 30dcb3f4e..969c5775d 100644 --- a/trunk/src/kernel/srs_kernel_kbps.cpp +++ b/trunk/src/kernel/srs_kernel_kbps.cpp @@ -7,6 +7,7 @@ #include #include +#include SrsRateSample::SrsRateSample() { diff --git a/trunk/src/main/srs_main_ingest_hls.cpp b/trunk/src/main/srs_main_ingest_hls.cpp index a3e82ebf7..650f2d64f 100644 --- a/trunk/src/main/srs_main_ingest_hls.cpp +++ b/trunk/src/main/srs_main_ingest_hls.cpp @@ -44,6 +44,9 @@ SrsConfig* _srs_config = new SrsConfig(); // @global Other variables. bool _srs_in_docker = false; +// The binary name of SRS. +const char* _srs_binary = NULL; + /** * main entrance. */ @@ -51,6 +54,8 @@ int main(int argc, char** argv) { // TODO: support both little and big endian. srs_assert(srs_is_little_endian()); + + _srs_binary = argv[0]; // directly failed when compile limited. #if defined(SRS_GPERF_MP) || defined(SRS_GPERF_MP) \ diff --git a/trunk/src/main/srs_main_mp4_parser.cpp b/trunk/src/main/srs_main_mp4_parser.cpp index 4d1b040d0..841fa2844 100644 --- a/trunk/src/main/srs_main_mp4_parser.cpp +++ b/trunk/src/main/srs_main_mp4_parser.cpp @@ -30,6 +30,9 @@ SrsConfig* _srs_config = new SrsConfig(); // @global Other variables. bool _srs_in_docker = false; +// The binary name of SRS. +const char* _srs_binary = NULL; + srs_error_t parse(std::string mp4_file, bool verbose) { srs_error_t err = srs_success; @@ -87,6 +90,8 @@ int main(int argc, char** argv) { printf("SRS MP4 parser/%d.%d.%d, parse and show the mp4 boxes structure.\n", VERSION_MAJOR, VERSION_MINOR, VERSION_REVISION); + + _srs_binary = argv[0]; if (argc < 2) { printf("Usage: %s [verbose]\n" diff --git a/trunk/src/main/srs_main_server.cpp b/trunk/src/main/srs_main_server.cpp index f687d7b46..59497b6a9 100644 --- a/trunk/src/main/srs_main_server.cpp +++ b/trunk/src/main/srs_main_server.cpp @@ -71,6 +71,9 @@ SrsServer* _srs_server = NULL; // Whether setup config by environment variables, see https://github.com/ossrs/srs/issues/2277 bool _srs_config_by_env = false; +// The binary name of SRS. +const char* _srs_binary = NULL; + /** * main entrance. */ @@ -78,6 +81,9 @@ srs_error_t do_main(int argc, char** argv, char** envp) { srs_error_t err = srs_success; + // TODO: Might fail if change working directory. + _srs_binary = argv[0]; + // Initialize global and thread-local variables. if ((err = srs_global_initialize()) != srs_success) { return srs_error_wrap(err, "global init"); diff --git a/trunk/src/protocol/srs_protocol_conn.cpp b/trunk/src/protocol/srs_protocol_conn.cpp index 9c3ed3dd3..01f2e878b 100644 --- a/trunk/src/protocol/srs_protocol_conn.cpp +++ b/trunk/src/protocol/srs_protocol_conn.cpp @@ -6,6 +6,8 @@ #include +#include + #include using namespace std; diff --git a/trunk/src/protocol/srs_protocol_json.cpp b/trunk/src/protocol/srs_protocol_json.cpp index 7378f3522..9016c0c34 100644 --- a/trunk/src/protocol/srs_protocol_json.cpp +++ b/trunk/src/protocol/srs_protocol_json.cpp @@ -6,6 +6,8 @@ #include +#include + // LCOV_EXCL_START /* vim: set et ts=3 sw=3 sts=3 ft=c: * diff --git a/trunk/src/protocol/srs_protocol_st.hpp b/trunk/src/protocol/srs_protocol_st.hpp index fb0e8f6b3..cd4c8f215 100644 --- a/trunk/src/protocol/srs_protocol_st.hpp +++ b/trunk/src/protocol/srs_protocol_st.hpp @@ -12,6 +12,7 @@ #include #include +#include // Wrap for coroutine. typedef void* srs_netfd_t; diff --git a/trunk/src/utest/srs_utest.cpp b/trunk/src/utest/srs_utest.cpp index e1b468bb9..ea21b05dd 100644 --- a/trunk/src/utest/srs_utest.cpp +++ b/trunk/src/utest/srs_utest.cpp @@ -37,6 +37,9 @@ SrsServer* _srs_server = NULL; bool _srs_in_docker = false; bool _srs_config_by_env = false; +// The binary name of SRS. +const char* _srs_binary = NULL; + #include // Initialize global settings. @@ -83,6 +86,8 @@ srs_error_t prepare_main() { GTEST_API_ int main(int argc, char **argv) { srs_error_t err = srs_success; + _srs_binary = argv[0]; + if ((err = prepare_main()) != srs_success) { fprintf(stderr, "Failed, %s\n", srs_error_desc(err).c_str());