h264 rtp debuging

trunk/src/app/srs_app_rtc_conn.cpp

@@ -86,14 +86,20 @@ static string dump_string_hex(const std::string& str, const int& max_len = 128)
 static string dump_string_hex(const char* buf, const int nb_buf, const int& max_len = 128)
 {
     char tmp_buf[1024*16];
-    int len = 0;
+    tmp_buf[0] = '\n';
+    int len = 1;
     
     for (int i = 0; i < nb_buf && i < max_len; ++i) {
-        int nb = snprintf(tmp_buf + len, sizeof(tmp_buf) - len - 1, "%02X ", (uint8_t)buf[i]);
+        //int nb = snprintf(tmp_buf + len, sizeof(tmp_buf) - len - 2, "(%03d)%02X ", i, (uint8_t)buf[i]);
+        int nb = snprintf(tmp_buf + len, sizeof(tmp_buf) - len - 2, "%02X ", (uint8_t)buf[i]);
         if (nb <= 0)
             break;
 
         len += nb; 
+
+        if (i % 16 == 15) {
+            tmp_buf[len++] = '\n';
+        }
     }   
     tmp_buf[len] = '\0';
 
@@ -540,6 +546,23 @@ srs_error_t SrsDtlsSession::srtp_sender_protect(char* protected_buf, const char*
     return srs_error_wrap(err, "srtp sender protect failed");
 }
 
+srs_error_t SrsDtlsSession::srtp_receiver_unprotect(char* unprotected_buf, const char* ori_buf, int& nb_unprotected_buf)
+{
+    srs_error_t err = srs_success;
+
+    if (srtp_send) {
+        memcpy(unprotected_buf, ori_buf, nb_unprotected_buf);
+        if (srtp_unprotect(srtp_recv, unprotected_buf, &nb_unprotected_buf) != 0) {
+            srs_error("srtp receiver unprotect failed");
+            return srs_error_wrap(err, "srtp receiver unprotect failed");
+        }
+
+        return err;
+    }
+
+    return srs_error_wrap(err, "srtp receiver unprotect failed");
+}
+
 SrsRtcSenderThread::SrsRtcSenderThread(SrsRtcSession* s, SrsUdpRemuxSocket* u, int parent_cid)
     : ukt(NULL)
 {
@@ -627,6 +650,12 @@ srs_error_t SrsRtcSenderThread::cycle()
                 srs_trace("rtp fragment size=%d, payload=%s", msg->rtp_fragments[i].size, 
                     dump_string_hex(msg->rtp_fragments[i].bytes, msg->rtp_fragments[i].size, 1460).c_str());
 
+                SrsBuffer stream(msg->rtp_fragments[i].bytes + 2, 2);
+                static uint16_t seq = 0;
+                stream.write_2bytes(++seq);
+
+                srs_trace("seq=%u", seq);
+
                 if (rtc_session->dtls_session) {
                     char rtp_send_protected_buf[1500];
                     int rtp_send_protected_len = msg->rtp_fragments[i].size;
@@ -735,6 +764,56 @@ srs_error_t SrsRtcSession::on_dtls(SrsUdpRemuxSocket* udp_remux_socket)
     return dtls_session->on_dtls(udp_remux_socket);
 }
 
+srs_error_t SrsRtcSession::on_rtp_or_rtcp(SrsUdpRemuxSocket* udp_remux_socket)
+{
+    srs_error_t err = srs_success;
+    if (dtls_session == NULL) {
+        return srs_error_wrap(err, "recv unexpect rtp/rtcp packet before dtls done");
+    }
+
+    char srtp_unprotect_buf[1460];
+    int nb_srtp_unprotect_buf = udp_remux_socket->size();
+    if (dtls_session->srtp_receiver_unprotect(srtp_unprotect_buf, udp_remux_socket->data(), nb_srtp_unprotect_buf) != srs_success) {
+        return srs_error_wrap(err, "srtp receiver unprotect failed");
+    }
+
+    //srs_trace("srtp unprotect success, %s", dump_string_hex(srtp_unprotect_buf, nb_srtp_unprotect_buf, nb_srtp_unprotect_buf).c_str());
+
+    SrsBuffer* stream = new SrsBuffer(srtp_unprotect_buf, nb_srtp_unprotect_buf);
+    uint8_t first = stream->read_1bytes();
+    uint8_t second = stream->read_1bytes();
+    bool marker = (second & 0x80) == 0x80;
+    uint8_t payload_type = second &0x7F;
+
+    uint16_t sequence = stream->read_2bytes();
+    uint32_t timestamp = stream->read_4bytes();
+    uint32_t ssrc = stream->read_4bytes();
+
+    srs_trace("sequence=%u, timestamp=%u, ssrc=%u, marker=%d, payload_type=%u", sequence, timestamp, ssrc, marker, payload_type);
+
+    if (first & 0x10) {
+        uint16_t extern_profile = stream->read_2bytes();
+        uint16_t extern_length = stream->read_2bytes();
+
+        srs_trace("extern_profile=%u, extern_length=%u", extern_profile, extern_length);
+
+        stream->read_string(extern_length * 4);
+    }
+
+    if (payload_type == 102) {
+        char rtp_send_protected_buf[1500];
+        int rtp_send_protected_len = nb_srtp_unprotect_buf;
+        SrsBuffer stream(srtp_unprotect_buf + 8, 4);
+        stream.write_4bytes(3233846889);
+        dtls_session->srtp_sender_protect(rtp_send_protected_buf, srtp_unprotect_buf, rtp_send_protected_len);
+        udp_remux_socket->sendto(rtp_send_protected_buf, rtp_send_protected_len, 0);
+    }
+
+    srs_trace("rtp payload, %s", dump_string_hex(stream->data() + stream->pos(), stream->left(), stream->left()).c_str());
+
+    return err;
+}
+
 SrsRtcServer::SrsRtcServer(SrsServer* svr)
 {
     server = svr;
@@ -828,7 +907,6 @@ srs_error_t SrsRtcServer::on_dtls(SrsUdpRemuxSocket* udp_remux_socket)
     srs_error_t err = srs_success;
     srs_trace("on dtls");
 
-    // FIXME
     SrsRtcSession* rtc_session = find_rtc_session_by_peer_id(udp_remux_socket->get_peer_id());
 
     if (rtc_session == NULL) {
@@ -844,6 +922,15 @@ srs_error_t SrsRtcServer::on_rtp_or_rtcp(SrsUdpRemuxSocket* udp_remux_socket)
 {
     srs_error_t err = srs_success;
     srs_trace("on rtp/rtcp");
+
+    SrsRtcSession* rtc_session = find_rtc_session_by_peer_id(udp_remux_socket->get_peer_id());
+
+    if (rtc_session == NULL) {
+        return srs_error_wrap(err, "can not find rtc session by peer_id=%s", udp_remux_socket->get_peer_id().c_str());
+    }
+
+    rtc_session->on_rtp_or_rtcp(udp_remux_socket);
+
     return err;
 }
 

trunk/src/app/srs_app_rtc_conn.hpp

@@ -123,6 +123,7 @@ public:
     void send_client_hello(SrsUdpRemuxSocket* udp_remux_socket);
     srs_error_t handshake(SrsUdpRemuxSocket* udp_remux_socket);
     srs_error_t srtp_sender_protect(char* protected_buf, const char* ori_buf, int& nb_protected_buf);
+    srs_error_t srtp_receiver_unprotect(char* unprotected_buf, const char* ori_buf, int& nb_unprotected_buf);
  
 private:
     srs_error_t srtp_initialize();
@@ -182,6 +183,7 @@ public:
 public:
     srs_error_t on_stun(SrsUdpRemuxSocket* udp_remux_socket, SrsStunPacket* stun_req);
     srs_error_t on_dtls(SrsUdpRemuxSocket* udp_remux_socket);
+    srs_error_t on_rtp_or_rtcp(SrsUdpRemuxSocket* udp_remux_socket);
 public:
     srs_error_t send_client_hello(SrsUdpRemuxSocket* udp_remux_socket);
     void on_connection_established(SrsUdpRemuxSocket* udp_remux_socket);

trunk/src/app/srs_app_rtp.cpp

@@ -53,20 +53,34 @@ using namespace std;
 static string dump_string_hex(const char* buf, const int nb_buf, const int& max_len);
 static string dump_string_hex(const std::string& str, const int& max_len = 128)
 {
-	return dump_string_hex(str.c_str(), str.size(), max_len);
+    return dump_string_hex(str.c_str(), str.size(), max_len);
 }
 
 static string dump_string_hex(const char* buf, const int nb_buf, const int& max_len = 128)
 {
     string ret;
-    ret.reserve(nb_buf > max_len ? nb_buf * 4 : max_len * 4);
+    ret.reserve((nb_buf > max_len ? nb_buf : max_len) * 8);
+
+    char tmp_buf[1024*16];
+    tmp_buf[0] = '\n';
+    int len = 1;
     
-    char tmp[64];
     for (int i = 0; i < nb_buf && i < max_len; ++i) {
-        int nb = snprintf(tmp, sizeof(tmp), "%02X ", (uint8_t)buf[i]);
-        assert(nb == 3);
-        ret.append(tmp, nb);
+        //int nb = snprintf(tmp_buf + len, sizeof(tmp_buf) - len - 2, "(%03d)%02X ", i, (uint8_t)buf[i]);
+        int nb = snprintf(tmp_buf + len, sizeof(tmp_buf) - len - 2, "%02X ", (uint8_t)buf[i]);
+        if (nb <= 0)
+            break;
+
+        len += nb; 
+
+        if (i % 16 == 15) {
+            tmp_buf[len++] = '\n';
+            ret.append(tmp_buf, len);
+            len = 0;
+        }   
     }   
+    tmp_buf[len] = '\0';
+    ret.append(tmp_buf, len);
 
     return ret;
 }
@@ -80,6 +94,7 @@ SrsRtpMuxer::~SrsRtpMuxer()
 {
 }
 
+#if 0
 srs_error_t SrsRtpMuxer::frame_to_packet(SrsSharedPtrMessage* shared_frame, SrsFormat* format)
 {
     srs_error_t err = srs_success;
@@ -87,6 +102,15 @@ srs_error_t SrsRtpMuxer::frame_to_packet(SrsSharedPtrMessage* shared_frame, SrsF
     int nb_samples = format->video->nb_samples;
     SrsSample* samples = format->video->samples;
 
+    SrsSample* rtp_fragment_samples = new SrsSample[2000];
+    int rtp_fragment_samples_index = 0;
+
+    static int debug_fd = -1;
+    static uint8_t start_code[4] = {0x00, 0x00, 0x00, 0x01};
+    if (debug_fd < 0) {
+        debug_fd = open("./raw.264", O_CREAT|O_TRUNC|O_RDWR, 0664);
+    }
+
     SrsSample sps_pps_samples[2];
     if (format->is_avc_sequence_header()) {
         sps_pps_samples[0].bytes = format->vcodec->sequenceParameterSetNALUnit.data();
@@ -96,15 +120,62 @@ srs_error_t SrsRtpMuxer::frame_to_packet(SrsSharedPtrMessage* shared_frame, SrsF
 
         nb_samples = 2;
         samples = sps_pps_samples;
+
+        {
+            char* buf = new char[1460];
+            SrsBuffer* stream = new SrsBuffer(buf, 1460);
+            SrsAutoFree(SrsBuffer, stream);
+            // write rtp header first
+            stream->write_1bytes(0x80);
+            stream->write_1bytes(102);
+
+            stream->write_2bytes(sequence++);
+            stream->write_4bytes((int32_t)shared_frame->timestamp * 90);
+            stream->write_4bytes((int32_t)3233846889);
+
+            stream->write_1bytes(24/*STAP-A*/);
+            // AUD
+            stream->write_2bytes(2);
+            stream->write_1bytes(0x09);
+            stream->write_1bytes(0x10);
+
+            stream->write_2bytes(sps_pps_samples[0].size);
+            stream->write_bytes(sps_pps_samples[0].bytes, sps_pps_samples[0].size);
+            stream->write_2bytes(sps_pps_samples[1].size);
+            stream->write_bytes(sps_pps_samples[1].bytes, sps_pps_samples[1].size);
+
+            if (debug_fd >= 0) {
+                write(debug_fd, start_code, sizeof(start_code));
+                write(debug_fd, sps_pps_samples[0].bytes, sps_pps_samples[0].size);
+                write(debug_fd, start_code, sizeof(start_code));
+                write(debug_fd, sps_pps_samples[1].bytes, sps_pps_samples[1].size);
+            }
+
+            rtp_fragment_samples[rtp_fragment_samples_index].bytes = stream->data();
+            rtp_fragment_samples[rtp_fragment_samples_index].size = stream->pos();
+
+            ++rtp_fragment_samples_index;
+        }
+        shared_frame->set_rtp_fragments(rtp_fragment_samples, rtp_fragment_samples_index);
+
+        return err;
     }
 
-    SrsSample* rtp_fragment_samples = new SrsSample[2000];
-    int rtp_fragment_samples_index = 0;
     for (int i = 0; i < nb_samples; ++i) {
         SrsSample sample = samples[i];
 
         srs_trace("nal size=%d, dump=%s", sample.size, dump_string_hex(sample.bytes, sample.size, sample.size).c_str());
 
+        if ((sample.bytes[0] & 0x1F) == 0x06) {
+            srs_trace("ignore SEI");
+            continue;
+        }
+
+        if (debug_fd >= 0) {
+            write(debug_fd, start_code, sizeof(start_code));
+            write(debug_fd, sample.bytes, sample.size);
+        }
+
         static int max_packet_size = 900;
         if (sample.size <= max_packet_size) {
             char* buf = new char[1460];
@@ -112,12 +183,23 @@ srs_error_t SrsRtpMuxer::frame_to_packet(SrsSharedPtrMessage* shared_frame, SrsF
             SrsAutoFree(SrsBuffer, stream);
             // write rtp header first
             stream->write_1bytes(0x80);
-            stream->write_1bytes((1 << 7) | 102);
+            if ((sample.bytes[0] & 0x1F) <= 5) {
+                stream->write_1bytes((1 << 7) | 102);
+            } else {
+                stream->write_1bytes(102);
+            }
 
             stream->write_2bytes(sequence++);
-            stream->write_4bytes((int32_t)shared_frame->timestamp);
+            stream->write_4bytes((int32_t)shared_frame->timestamp * 90);
             stream->write_4bytes((int32_t)3233846889);
+
+#if 0 // single nalu
             stream->write_bytes(sample.bytes, sample.size);
+#else
+            stream->write_1bytes((sample.bytes[0] & 0xE0) | 24/*STAP-A*/);
+            stream->write_2bytes(sample.size);
+            stream->write_bytes(sample.bytes, sample.size);
+#endif
 
             rtp_fragment_samples[rtp_fragment_samples_index].bytes = stream->data();
             rtp_fragment_samples[rtp_fragment_samples_index].size = stream->pos();
@@ -133,14 +215,129 @@ srs_error_t SrsRtpMuxer::frame_to_packet(SrsSharedPtrMessage* shared_frame, SrsF
                 SrsAutoFree(SrsBuffer, stream);
                 // write rtp header first
                 stream->write_1bytes(0x80);
-                if (n == num_of_packet - 1) {
+                if ((sample.bytes[0] & 0x1F) <= 5) {
                     stream->write_1bytes((1 << 7) | 102);
                 } else {
                     stream->write_1bytes(102);
                 }
 
                 stream->write_2bytes(sequence++);
-                stream->write_4bytes((int32_t)shared_frame->timestamp);
+                stream->write_4bytes((int32_t)shared_frame->timestamp * 90);
+                stream->write_4bytes((int32_t)3233846889);
+
+                stream->write_1bytes((sample.bytes[0] & 0xE0) | 28);
+                if (n == 0) {
+                    stream->write_1bytes(0x80 | (sample.bytes[0] & 0x1F));
+                } else if (n == num_of_packet - 1) {
+                    stream->write_1bytes(0x40 | (sample.bytes[0] & 0x1F));
+                } else {
+                    stream->write_1bytes(0x00 | (sample.bytes[0] & 0x1F));
+                }
+
+                int len = left_bytes > max_packet_size ? max_packet_size : left_bytes;
+                stream->write_bytes(p, len);
+                left_bytes -= len;
+                p += len;
+
+                rtp_fragment_samples[rtp_fragment_samples_index].bytes = stream->data();
+                rtp_fragment_samples[rtp_fragment_samples_index].size = stream->pos();
+
+                ++rtp_fragment_samples_index;
+
+            }
+        }
+    }
+    shared_frame->set_rtp_fragments(rtp_fragment_samples, rtp_fragment_samples_index);
+
+    return err;
+}
+#endif
+
+srs_error_t SrsRtpMuxer::frame_to_packet(SrsSharedPtrMessage* shared_frame, SrsFormat* format)
+{
+    srs_error_t err = srs_success;
+
+    int nb_samples = format->video->nb_samples;
+    SrsSample* samples = format->video->samples;
+
+    SrsSample* rtp_fragment_samples = new SrsSample[2000];
+    int rtp_fragment_samples_index = 0;
+
+    static int debug_fd = -1;
+    static uint8_t start_code[4] = {0x00, 0x00, 0x00, 0x01};
+    if (debug_fd < 0) {
+        debug_fd = open("./raw.264", O_CREAT|O_TRUNC|O_RDWR, 0664);
+    }
+
+    SrsSample sps_pps_samples[2];
+    if (format->is_avc_sequence_header()) {
+        sps_pps_samples[0].bytes = format->vcodec->sequenceParameterSetNALUnit.data();
+        sps_pps_samples[0].size = format->vcodec->sequenceParameterSetNALUnit.size();
+        sps_pps_samples[1].bytes = format->vcodec->pictureParameterSetNALUnit.data();
+        sps_pps_samples[1].size = format->vcodec->pictureParameterSetNALUnit.size();
+
+        nb_samples = 2;
+        samples = sps_pps_samples;
+    }
+
+    for (int i = 0; i < nb_samples; ++i) {
+        SrsSample sample = samples[i];
+
+        srs_trace("nal size=%d, dump=%s", sample.size, dump_string_hex(sample.bytes, sample.size, sample.size).c_str());
+
+        if ((sample.bytes[0] & 0x1F) == 0x06) {
+            srs_trace("ignore SEI");
+            continue;
+        }
+
+        if (debug_fd >= 0) {
+            write(debug_fd, start_code, sizeof(start_code));
+            write(debug_fd, sample.bytes, sample.size);
+        }
+
+        static int max_packet_size = 900;
+        if (sample.size <= max_packet_size) {
+            char* buf = new char[1460];
+            SrsBuffer* stream = new SrsBuffer(buf, 1460);
+            SrsAutoFree(SrsBuffer, stream);
+            // write rtp header first
+            stream->write_1bytes(0x80);
+            if ((sample.bytes[0] & 0x1F) <= 5) {
+                stream->write_1bytes((1 << 7) | 102);
+            } else {
+                stream->write_1bytes(102);
+            }
+
+            stream->write_2bytes(sequence++);
+            stream->write_4bytes((int32_t)shared_frame->timestamp * 90);
+            stream->write_4bytes((int32_t)3233846889);
+
+            stream->write_1bytes((sample.bytes[0] & 0xE0) | 28/*FU-A*/);
+            stream->write_1bytes(0xC0 | (sample.bytes[0] & 0x1F));
+            stream->write_bytes(sample.bytes + 1, sample.size - 1);
+
+            rtp_fragment_samples[rtp_fragment_samples_index].bytes = stream->data();
+            rtp_fragment_samples[rtp_fragment_samples_index].size = stream->pos();
+
+            ++rtp_fragment_samples_index;
+        } else {
+            int num_of_packet = (sample.size + max_packet_size) / max_packet_size;
+            char* p = sample.bytes + 1;
+            int left_bytes = sample.size - 1;
+            for (int n = 0; n < num_of_packet; ++n) {
+                char* buf = new char[1460];
+                SrsBuffer* stream = new SrsBuffer(buf, 1460);
+                SrsAutoFree(SrsBuffer, stream);
+                // write rtp header first
+                stream->write_1bytes(0x80);
+                if ((sample.bytes[0] & 0x1F) <= 5) {
+                    stream->write_1bytes((1 << 7) | 102);
+                } else {
+                    stream->write_1bytes(102);
+                }
+
+                stream->write_2bytes(sequence++);
+                stream->write_4bytes((int32_t)shared_frame->timestamp * 90);
                 stream->write_4bytes((int32_t)3233846889);
 
                 stream->write_1bytes((sample.bytes[0] & 0xE0) | 28);