From ff0e03800d306f0cb0db311dd7eb6312f9a1faed Mon Sep 17 00:00:00 2001 From: HuyaJohn Date: Tue, 10 Mar 2020 04:47:49 -0700 Subject: [PATCH] h264 rtp debuging --- trunk/src/app/srs_app_rtc_conn.cpp | 93 +++++++++++- trunk/src/app/srs_app_rtc_conn.hpp | 2 + trunk/src/app/srs_app_rtp.cpp | 221 +++++++++++++++++++++++++++-- 3 files changed, 301 insertions(+), 15 deletions(-) diff --git a/trunk/src/app/srs_app_rtc_conn.cpp b/trunk/src/app/srs_app_rtc_conn.cpp index f60172072..a901efab9 100644 --- a/trunk/src/app/srs_app_rtc_conn.cpp +++ b/trunk/src/app/srs_app_rtc_conn.cpp @@ -86,14 +86,20 @@ static string dump_string_hex(const std::string& str, const int& max_len = 128) static string dump_string_hex(const char* buf, const int nb_buf, const int& max_len = 128) { char tmp_buf[1024*16]; - int len = 0; + tmp_buf[0] = '\n'; + int len = 1; for (int i = 0; i < nb_buf && i < max_len; ++i) { - int nb = snprintf(tmp_buf + len, sizeof(tmp_buf) - len - 1, "%02X ", (uint8_t)buf[i]); + //int nb = snprintf(tmp_buf + len, sizeof(tmp_buf) - len - 2, "(%03d)%02X ", i, (uint8_t)buf[i]); + int nb = snprintf(tmp_buf + len, sizeof(tmp_buf) - len - 2, "%02X ", (uint8_t)buf[i]); if (nb <= 0) break; len += nb; + + if (i % 16 == 15) { + tmp_buf[len++] = '\n'; + } } tmp_buf[len] = '\0'; @@ -540,6 +546,23 @@ srs_error_t SrsDtlsSession::srtp_sender_protect(char* protected_buf, const char* return srs_error_wrap(err, "srtp sender protect failed"); } +srs_error_t SrsDtlsSession::srtp_receiver_unprotect(char* unprotected_buf, const char* ori_buf, int& nb_unprotected_buf) +{ + srs_error_t err = srs_success; + + if (srtp_send) { + memcpy(unprotected_buf, ori_buf, nb_unprotected_buf); + if (srtp_unprotect(srtp_recv, unprotected_buf, &nb_unprotected_buf) != 0) { + srs_error("srtp receiver unprotect failed"); + return srs_error_wrap(err, "srtp receiver unprotect failed"); + } + + return err; + } + + return srs_error_wrap(err, "srtp receiver unprotect failed"); +} + SrsRtcSenderThread::SrsRtcSenderThread(SrsRtcSession* s, SrsUdpRemuxSocket* u, int parent_cid) : ukt(NULL) { @@ -627,6 +650,12 @@ srs_error_t SrsRtcSenderThread::cycle() srs_trace("rtp fragment size=%d, payload=%s", msg->rtp_fragments[i].size, dump_string_hex(msg->rtp_fragments[i].bytes, msg->rtp_fragments[i].size, 1460).c_str()); + SrsBuffer stream(msg->rtp_fragments[i].bytes + 2, 2); + static uint16_t seq = 0; + stream.write_2bytes(++seq); + + srs_trace("seq=%u", seq); + if (rtc_session->dtls_session) { char rtp_send_protected_buf[1500]; int rtp_send_protected_len = msg->rtp_fragments[i].size; @@ -735,6 +764,56 @@ srs_error_t SrsRtcSession::on_dtls(SrsUdpRemuxSocket* udp_remux_socket) return dtls_session->on_dtls(udp_remux_socket); } +srs_error_t SrsRtcSession::on_rtp_or_rtcp(SrsUdpRemuxSocket* udp_remux_socket) +{ + srs_error_t err = srs_success; + if (dtls_session == NULL) { + return srs_error_wrap(err, "recv unexpect rtp/rtcp packet before dtls done"); + } + + char srtp_unprotect_buf[1460]; + int nb_srtp_unprotect_buf = udp_remux_socket->size(); + if (dtls_session->srtp_receiver_unprotect(srtp_unprotect_buf, udp_remux_socket->data(), nb_srtp_unprotect_buf) != srs_success) { + return srs_error_wrap(err, "srtp receiver unprotect failed"); + } + + //srs_trace("srtp unprotect success, %s", dump_string_hex(srtp_unprotect_buf, nb_srtp_unprotect_buf, nb_srtp_unprotect_buf).c_str()); + + SrsBuffer* stream = new SrsBuffer(srtp_unprotect_buf, nb_srtp_unprotect_buf); + uint8_t first = stream->read_1bytes(); + uint8_t second = stream->read_1bytes(); + bool marker = (second & 0x80) == 0x80; + uint8_t payload_type = second &0x7F; + + uint16_t sequence = stream->read_2bytes(); + uint32_t timestamp = stream->read_4bytes(); + uint32_t ssrc = stream->read_4bytes(); + + srs_trace("sequence=%u, timestamp=%u, ssrc=%u, marker=%d, payload_type=%u", sequence, timestamp, ssrc, marker, payload_type); + + if (first & 0x10) { + uint16_t extern_profile = stream->read_2bytes(); + uint16_t extern_length = stream->read_2bytes(); + + srs_trace("extern_profile=%u, extern_length=%u", extern_profile, extern_length); + + stream->read_string(extern_length * 4); + } + + if (payload_type == 102) { + char rtp_send_protected_buf[1500]; + int rtp_send_protected_len = nb_srtp_unprotect_buf; + SrsBuffer stream(srtp_unprotect_buf + 8, 4); + stream.write_4bytes(3233846889); + dtls_session->srtp_sender_protect(rtp_send_protected_buf, srtp_unprotect_buf, rtp_send_protected_len); + udp_remux_socket->sendto(rtp_send_protected_buf, rtp_send_protected_len, 0); + } + + srs_trace("rtp payload, %s", dump_string_hex(stream->data() + stream->pos(), stream->left(), stream->left()).c_str()); + + return err; +} + SrsRtcServer::SrsRtcServer(SrsServer* svr) { server = svr; @@ -828,7 +907,6 @@ srs_error_t SrsRtcServer::on_dtls(SrsUdpRemuxSocket* udp_remux_socket) srs_error_t err = srs_success; srs_trace("on dtls"); - // FIXME SrsRtcSession* rtc_session = find_rtc_session_by_peer_id(udp_remux_socket->get_peer_id()); if (rtc_session == NULL) { @@ -844,6 +922,15 @@ srs_error_t SrsRtcServer::on_rtp_or_rtcp(SrsUdpRemuxSocket* udp_remux_socket) { srs_error_t err = srs_success; srs_trace("on rtp/rtcp"); + + SrsRtcSession* rtc_session = find_rtc_session_by_peer_id(udp_remux_socket->get_peer_id()); + + if (rtc_session == NULL) { + return srs_error_wrap(err, "can not find rtc session by peer_id=%s", udp_remux_socket->get_peer_id().c_str()); + } + + rtc_session->on_rtp_or_rtcp(udp_remux_socket); + return err; } diff --git a/trunk/src/app/srs_app_rtc_conn.hpp b/trunk/src/app/srs_app_rtc_conn.hpp index 4f0606571..fab2afc4c 100644 --- a/trunk/src/app/srs_app_rtc_conn.hpp +++ b/trunk/src/app/srs_app_rtc_conn.hpp @@ -123,6 +123,7 @@ public: void send_client_hello(SrsUdpRemuxSocket* udp_remux_socket); srs_error_t handshake(SrsUdpRemuxSocket* udp_remux_socket); srs_error_t srtp_sender_protect(char* protected_buf, const char* ori_buf, int& nb_protected_buf); + srs_error_t srtp_receiver_unprotect(char* unprotected_buf, const char* ori_buf, int& nb_unprotected_buf); private: srs_error_t srtp_initialize(); @@ -182,6 +183,7 @@ public: public: srs_error_t on_stun(SrsUdpRemuxSocket* udp_remux_socket, SrsStunPacket* stun_req); srs_error_t on_dtls(SrsUdpRemuxSocket* udp_remux_socket); + srs_error_t on_rtp_or_rtcp(SrsUdpRemuxSocket* udp_remux_socket); public: srs_error_t send_client_hello(SrsUdpRemuxSocket* udp_remux_socket); void on_connection_established(SrsUdpRemuxSocket* udp_remux_socket); diff --git a/trunk/src/app/srs_app_rtp.cpp b/trunk/src/app/srs_app_rtp.cpp index 4081f4534..d3bf6534f 100644 --- a/trunk/src/app/srs_app_rtp.cpp +++ b/trunk/src/app/srs_app_rtp.cpp @@ -53,20 +53,34 @@ using namespace std; static string dump_string_hex(const char* buf, const int nb_buf, const int& max_len); static string dump_string_hex(const std::string& str, const int& max_len = 128) { - return dump_string_hex(str.c_str(), str.size(), max_len); + return dump_string_hex(str.c_str(), str.size(), max_len); } static string dump_string_hex(const char* buf, const int nb_buf, const int& max_len = 128) { string ret; - ret.reserve(nb_buf > max_len ? nb_buf * 4 : max_len * 4); + ret.reserve((nb_buf > max_len ? nb_buf : max_len) * 8); + + char tmp_buf[1024*16]; + tmp_buf[0] = '\n'; + int len = 1; - char tmp[64]; for (int i = 0; i < nb_buf && i < max_len; ++i) { - int nb = snprintf(tmp, sizeof(tmp), "%02X ", (uint8_t)buf[i]); - assert(nb == 3); - ret.append(tmp, nb); + //int nb = snprintf(tmp_buf + len, sizeof(tmp_buf) - len - 2, "(%03d)%02X ", i, (uint8_t)buf[i]); + int nb = snprintf(tmp_buf + len, sizeof(tmp_buf) - len - 2, "%02X ", (uint8_t)buf[i]); + if (nb <= 0) + break; + + len += nb; + + if (i % 16 == 15) { + tmp_buf[len++] = '\n'; + ret.append(tmp_buf, len); + len = 0; + } } + tmp_buf[len] = '\0'; + ret.append(tmp_buf, len); return ret; } @@ -80,6 +94,7 @@ SrsRtpMuxer::~SrsRtpMuxer() { } +#if 0 srs_error_t SrsRtpMuxer::frame_to_packet(SrsSharedPtrMessage* shared_frame, SrsFormat* format) { srs_error_t err = srs_success; @@ -87,6 +102,15 @@ srs_error_t SrsRtpMuxer::frame_to_packet(SrsSharedPtrMessage* shared_frame, SrsF int nb_samples = format->video->nb_samples; SrsSample* samples = format->video->samples; + SrsSample* rtp_fragment_samples = new SrsSample[2000]; + int rtp_fragment_samples_index = 0; + + static int debug_fd = -1; + static uint8_t start_code[4] = {0x00, 0x00, 0x00, 0x01}; + if (debug_fd < 0) { + debug_fd = open("./raw.264", O_CREAT|O_TRUNC|O_RDWR, 0664); + } + SrsSample sps_pps_samples[2]; if (format->is_avc_sequence_header()) { sps_pps_samples[0].bytes = format->vcodec->sequenceParameterSetNALUnit.data(); @@ -96,15 +120,62 @@ srs_error_t SrsRtpMuxer::frame_to_packet(SrsSharedPtrMessage* shared_frame, SrsF nb_samples = 2; samples = sps_pps_samples; + + { + char* buf = new char[1460]; + SrsBuffer* stream = new SrsBuffer(buf, 1460); + SrsAutoFree(SrsBuffer, stream); + // write rtp header first + stream->write_1bytes(0x80); + stream->write_1bytes(102); + + stream->write_2bytes(sequence++); + stream->write_4bytes((int32_t)shared_frame->timestamp * 90); + stream->write_4bytes((int32_t)3233846889); + + stream->write_1bytes(24/*STAP-A*/); + // AUD + stream->write_2bytes(2); + stream->write_1bytes(0x09); + stream->write_1bytes(0x10); + + stream->write_2bytes(sps_pps_samples[0].size); + stream->write_bytes(sps_pps_samples[0].bytes, sps_pps_samples[0].size); + stream->write_2bytes(sps_pps_samples[1].size); + stream->write_bytes(sps_pps_samples[1].bytes, sps_pps_samples[1].size); + + if (debug_fd >= 0) { + write(debug_fd, start_code, sizeof(start_code)); + write(debug_fd, sps_pps_samples[0].bytes, sps_pps_samples[0].size); + write(debug_fd, start_code, sizeof(start_code)); + write(debug_fd, sps_pps_samples[1].bytes, sps_pps_samples[1].size); + } + + rtp_fragment_samples[rtp_fragment_samples_index].bytes = stream->data(); + rtp_fragment_samples[rtp_fragment_samples_index].size = stream->pos(); + + ++rtp_fragment_samples_index; + } + shared_frame->set_rtp_fragments(rtp_fragment_samples, rtp_fragment_samples_index); + + return err; } - SrsSample* rtp_fragment_samples = new SrsSample[2000]; - int rtp_fragment_samples_index = 0; for (int i = 0; i < nb_samples; ++i) { SrsSample sample = samples[i]; srs_trace("nal size=%d, dump=%s", sample.size, dump_string_hex(sample.bytes, sample.size, sample.size).c_str()); + if ((sample.bytes[0] & 0x1F) == 0x06) { + srs_trace("ignore SEI"); + continue; + } + + if (debug_fd >= 0) { + write(debug_fd, start_code, sizeof(start_code)); + write(debug_fd, sample.bytes, sample.size); + } + static int max_packet_size = 900; if (sample.size <= max_packet_size) { char* buf = new char[1460]; @@ -112,12 +183,23 @@ srs_error_t SrsRtpMuxer::frame_to_packet(SrsSharedPtrMessage* shared_frame, SrsF SrsAutoFree(SrsBuffer, stream); // write rtp header first stream->write_1bytes(0x80); - stream->write_1bytes((1 << 7) | 102); + if ((sample.bytes[0] & 0x1F) <= 5) { + stream->write_1bytes((1 << 7) | 102); + } else { + stream->write_1bytes(102); + } stream->write_2bytes(sequence++); - stream->write_4bytes((int32_t)shared_frame->timestamp); + stream->write_4bytes((int32_t)shared_frame->timestamp * 90); stream->write_4bytes((int32_t)3233846889); + +#if 0 // single nalu stream->write_bytes(sample.bytes, sample.size); +#else + stream->write_1bytes((sample.bytes[0] & 0xE0) | 24/*STAP-A*/); + stream->write_2bytes(sample.size); + stream->write_bytes(sample.bytes, sample.size); +#endif rtp_fragment_samples[rtp_fragment_samples_index].bytes = stream->data(); rtp_fragment_samples[rtp_fragment_samples_index].size = stream->pos(); @@ -133,14 +215,129 @@ srs_error_t SrsRtpMuxer::frame_to_packet(SrsSharedPtrMessage* shared_frame, SrsF SrsAutoFree(SrsBuffer, stream); // write rtp header first stream->write_1bytes(0x80); - if (n == num_of_packet - 1) { + if ((sample.bytes[0] & 0x1F) <= 5) { stream->write_1bytes((1 << 7) | 102); } else { stream->write_1bytes(102); } stream->write_2bytes(sequence++); - stream->write_4bytes((int32_t)shared_frame->timestamp); + stream->write_4bytes((int32_t)shared_frame->timestamp * 90); + stream->write_4bytes((int32_t)3233846889); + + stream->write_1bytes((sample.bytes[0] & 0xE0) | 28); + if (n == 0) { + stream->write_1bytes(0x80 | (sample.bytes[0] & 0x1F)); + } else if (n == num_of_packet - 1) { + stream->write_1bytes(0x40 | (sample.bytes[0] & 0x1F)); + } else { + stream->write_1bytes(0x00 | (sample.bytes[0] & 0x1F)); + } + + int len = left_bytes > max_packet_size ? max_packet_size : left_bytes; + stream->write_bytes(p, len); + left_bytes -= len; + p += len; + + rtp_fragment_samples[rtp_fragment_samples_index].bytes = stream->data(); + rtp_fragment_samples[rtp_fragment_samples_index].size = stream->pos(); + + ++rtp_fragment_samples_index; + + } + } + } + shared_frame->set_rtp_fragments(rtp_fragment_samples, rtp_fragment_samples_index); + + return err; +} +#endif + +srs_error_t SrsRtpMuxer::frame_to_packet(SrsSharedPtrMessage* shared_frame, SrsFormat* format) +{ + srs_error_t err = srs_success; + + int nb_samples = format->video->nb_samples; + SrsSample* samples = format->video->samples; + + SrsSample* rtp_fragment_samples = new SrsSample[2000]; + int rtp_fragment_samples_index = 0; + + static int debug_fd = -1; + static uint8_t start_code[4] = {0x00, 0x00, 0x00, 0x01}; + if (debug_fd < 0) { + debug_fd = open("./raw.264", O_CREAT|O_TRUNC|O_RDWR, 0664); + } + + SrsSample sps_pps_samples[2]; + if (format->is_avc_sequence_header()) { + sps_pps_samples[0].bytes = format->vcodec->sequenceParameterSetNALUnit.data(); + sps_pps_samples[0].size = format->vcodec->sequenceParameterSetNALUnit.size(); + sps_pps_samples[1].bytes = format->vcodec->pictureParameterSetNALUnit.data(); + sps_pps_samples[1].size = format->vcodec->pictureParameterSetNALUnit.size(); + + nb_samples = 2; + samples = sps_pps_samples; + } + + for (int i = 0; i < nb_samples; ++i) { + SrsSample sample = samples[i]; + + srs_trace("nal size=%d, dump=%s", sample.size, dump_string_hex(sample.bytes, sample.size, sample.size).c_str()); + + if ((sample.bytes[0] & 0x1F) == 0x06) { + srs_trace("ignore SEI"); + continue; + } + + if (debug_fd >= 0) { + write(debug_fd, start_code, sizeof(start_code)); + write(debug_fd, sample.bytes, sample.size); + } + + static int max_packet_size = 900; + if (sample.size <= max_packet_size) { + char* buf = new char[1460]; + SrsBuffer* stream = new SrsBuffer(buf, 1460); + SrsAutoFree(SrsBuffer, stream); + // write rtp header first + stream->write_1bytes(0x80); + if ((sample.bytes[0] & 0x1F) <= 5) { + stream->write_1bytes((1 << 7) | 102); + } else { + stream->write_1bytes(102); + } + + stream->write_2bytes(sequence++); + stream->write_4bytes((int32_t)shared_frame->timestamp * 90); + stream->write_4bytes((int32_t)3233846889); + + stream->write_1bytes((sample.bytes[0] & 0xE0) | 28/*FU-A*/); + stream->write_1bytes(0xC0 | (sample.bytes[0] & 0x1F)); + stream->write_bytes(sample.bytes + 1, sample.size - 1); + + rtp_fragment_samples[rtp_fragment_samples_index].bytes = stream->data(); + rtp_fragment_samples[rtp_fragment_samples_index].size = stream->pos(); + + ++rtp_fragment_samples_index; + } else { + int num_of_packet = (sample.size + max_packet_size) / max_packet_size; + char* p = sample.bytes + 1; + int left_bytes = sample.size - 1; + for (int n = 0; n < num_of_packet; ++n) { + char* buf = new char[1460]; + SrsBuffer* stream = new SrsBuffer(buf, 1460); + SrsAutoFree(SrsBuffer, stream); + // write rtp header first + stream->write_1bytes(0x80); + if ((sample.bytes[0] & 0x1F) <= 5) { + stream->write_1bytes((1 << 7) | 102); + } else { + stream->write_1bytes(102); + } + + stream->write_2bytes(sequence++); + stream->write_4bytes((int32_t)shared_frame->timestamp * 90); stream->write_4bytes((int32_t)3233846889); stream->write_1bytes((sample.bytes[0] & 0xE0) | 28);