For #307, package opus when send it.

trunk/src/app/srs_app_rtc.cpp

@@ -260,17 +260,12 @@ srs_error_t SrsRtpH264Muxer::packet_stap_a(const string &sps, const string& pps,
 
 SrsRtpOpusMuxer::SrsRtpOpusMuxer()
 {
-    sequence = 0;
-    timestamp = 0;
     transcode = NULL;
 }
 
 SrsRtpOpusMuxer::~SrsRtpOpusMuxer()
 {
-    if (transcode) {
+    srs_freep(transcode);
-        delete transcode;
-        transcode = NULL;
-    }
 }
 
 srs_error_t SrsRtpOpusMuxer::initialize()
@@ -309,7 +304,7 @@ srs_error_t SrsRtpOpusMuxer::frame_to_packet(SrsSharedPtrMessage* shared_audio,
         }
     }
 
-    // Transcode aac packet to opus packets.
+    // Transcode an aac packet to many opus packets.
     SrsSample aac;
     aac.bytes = adts_audio;
     aac.size = nn_adts_audio;
@@ -320,37 +315,20 @@ srs_error_t SrsRtpOpusMuxer::frame_to_packet(SrsSharedPtrMessage* shared_audio,
         return srs_error_wrap(err, "recode error");
     }
 
-    // Package opus packets to RTP packets.
+    // Save OPUS packets in shared message.
-    vector<SrsRtpSharedPacket*> rtp_packets;
+    if (nn_opus_packets <= 0) {
+        return err;
+    }
 
+    SrsSample samples[nn_opus_packets];
     for (int i = 0; i < nn_opus_packets; i++) {
-        SrsSample sample;
+        SrsSample* p = samples + i;
-        sample.size = opus_sizes[i];
+        p->size = opus_sizes[i];
-        sample.bytes = opus_payloads[i];
+        p->bytes = new char[p->size];
-        if ((err = packet_opus(shared_audio, &sample, rtp_packets)) != srs_success) {
+        memcpy(p->bytes, opus_payloads[i], p->size);
-            return srs_error_wrap(err, "packet as opus");
-        }
     }
 
-    shared_audio->set_rtp_packets(rtp_packets);
+    shared_audio->set_extra_payloads(samples, nn_opus_packets);
-
-    return err;
-}
-
-srs_error_t SrsRtpOpusMuxer::packet_opus(SrsSharedPtrMessage* shared_frame, SrsSample* sample, std::vector<SrsRtpSharedPacket*>& rtp_packets)
-{
-    srs_error_t err = srs_success;
-
-    SrsRtpSharedPacket* packet = new SrsRtpSharedPacket();
-    packet->rtp_header.set_marker(true);
-    if ((err = packet->create(timestamp, sequence++, kAudioSSRC, kOpusPayloadType, sample->bytes, sample->size)) != srs_success) {
-        return srs_error_wrap(err, "rtp packet encode");
-    }
-
-    // TODO: FIXME: Why 960? Need Refactoring?
-    timestamp += 960;
-
-    rtp_packets.push_back(packet);
 
     return err;
 }

trunk/src/app/srs_app_rtc.hpp

@@ -90,9 +90,6 @@ private:
 class SrsRtpOpusMuxer
 {
 private:
-    // TODO: FIXME: How to handle timestamp overflow?
-    uint32_t timestamp;
-    uint16_t sequence;
     SrsAudioRecode* transcode;
 public:
     SrsRtpOpusMuxer();
@@ -100,8 +97,6 @@ public:
     virtual srs_error_t initialize();
 public:
     srs_error_t frame_to_packet(SrsSharedPtrMessage* shared_audio, SrsFormat* format, char* adts_audio, int nn_adts_audio);
-private:
-    srs_error_t packet_opus(SrsSharedPtrMessage* shared_frame, SrsSample* sample, std::vector<SrsRtpSharedPacket*>& rtp_packets);
 };
 
 class SrsRtc

trunk/src/app/srs_app_rtc_conn.cpp

@@ -442,6 +442,9 @@ SrsRtcSenderThread::SrsRtcSenderThread(SrsRtcSession* s, SrsUdpMuxSocket* u, int
 
     rtc_session = s;
     sendonly_ukt = u->copy_sendonly();
+
+    timestamp = 0;
+    sequence = 0;
 }
 
 SrsRtcSenderThread::~SrsRtcSenderThread()
@@ -575,6 +578,8 @@ void SrsRtcSenderThread::update_sendonly_socket(SrsUdpMuxSocket* skt)
 
 void SrsRtcSenderThread::send_and_free_messages(SrsSharedPtrMessage** msgs, int nb_msgs, SrsUdpMuxSocket* skt, int* pnn, int* pnn_rtp_pkts)
 {
+    srs_error_t err = srs_success;
+
     if (!rtc_session->dtls_session) {
         return;
     }
@@ -584,14 +589,35 @@ void SrsRtcSenderThread::send_and_free_messages(SrsSharedPtrMessage** msgs, int
         bool is_video = msg->is_video();
         bool is_audio = msg->is_audio();
 
-        int nn_rtp_pkts = (int)msg->rtp_packets.size();
+        if (is_audio) {
-        for (int j = 0; j < nn_rtp_pkts; j++) {
+            // Package opus packets to RTP packets.
-            SrsRtpSharedPacket* pkt = msg->rtp_packets[j];
+            vector<SrsRtpSharedPacket*> rtp_packets;
-            send_and_free_message(msg, is_video, is_audio, pkt, skt);
-        }
 
-        *pnn += msg->size;
+            for (int i = 0; i < msg->nn_extra_payloads(); i++) {
-        *pnn_rtp_pkts += nn_rtp_pkts;
+                SrsSample* sample = msg->extra_payloads() + i;
+                if ((err = packet_opus(msg, sample, rtp_packets)) != srs_success) {
+                    srs_warn("packet opus err %s", srs_error_summary(err).c_str()); srs_error_reset(err);
+                }
+            }
+
+            int nn_rtp_pkts = (int)rtp_packets.size();
+            for (int j = 0; j < nn_rtp_pkts; j++) {
+                SrsRtpSharedPacket* pkt = rtp_packets[j];
+                send_and_free_message(msg, is_video, is_audio, pkt, skt);
+            }
+
+            *pnn += msg->size;
+            *pnn_rtp_pkts += nn_rtp_pkts;
+        } else {
+            int nn_rtp_pkts = (int)msg->rtp_packets.size();
+            for (int j = 0; j < nn_rtp_pkts; j++) {
+                SrsRtpSharedPacket* pkt = msg->rtp_packets[j];
+                send_and_free_message(msg, is_video, is_audio, pkt, skt);
+            }
+
+            *pnn += msg->size;
+            *pnn_rtp_pkts += nn_rtp_pkts;
+        }
 
         srs_freep(msg);
     }
@@ -636,6 +662,24 @@ void SrsRtcSenderThread::send_and_free_message(SrsSharedPtrMessage* msg, bool is
     rtc_session->rtc_server->sendmmsg(skt->stfd(), mhdr);
 }
 
+srs_error_t SrsRtcSenderThread::packet_opus(SrsSharedPtrMessage* shared_frame, SrsSample* sample, std::vector<SrsRtpSharedPacket*>& rtp_packets)
+{
+    srs_error_t err = srs_success;
+
+    SrsRtpSharedPacket* packet = new SrsRtpSharedPacket();
+    packet->rtp_header.set_marker(true);
+    if ((err = packet->create(timestamp, sequence++, kAudioSSRC, kOpusPayloadType, sample->bytes, sample->size)) != srs_success) {
+        return srs_error_wrap(err, "rtp packet encode");
+    }
+
+    // TODO: FIXME: Why 960? Need Refactoring?
+    timestamp += 960;
+
+    rtp_packets.push_back(packet);
+
+    return err;
+}
+
 SrsRtcSession::SrsRtcSession(SrsRtcServer* rtc_svr, const SrsRequest& req, const std::string& un, int context_id)
 {
     rtc_server = rtc_svr;

trunk/src/app/srs_app_rtc_conn.hpp

@@ -126,6 +126,10 @@ private:
     uint32_t audio_ssrc;
     uint16_t video_payload_type;
     uint16_t audio_payload_type;
+private:
+    // TODO: FIXME: How to handle timestamp overflow?
+    uint32_t timestamp;
+    uint16_t sequence;
 public:
     SrsUdpMuxSocket* sendonly_ukt;
 public:
@@ -146,6 +150,8 @@ public:
 private:
     void send_and_free_messages(SrsSharedPtrMessage** msgs, int nb_msgs, SrsUdpMuxSocket* skt, int* pnn, int* pnn_rtp_pkts);
     void send_and_free_message(SrsSharedPtrMessage* msg, bool is_video, bool is_audio, SrsRtpSharedPacket* pkt, SrsUdpMuxSocket* skt);
+private:
+    srs_error_t packet_opus(SrsSharedPtrMessage* shared_frame, SrsSample* sample, std::vector<SrsRtpSharedPacket*>& rtp_packets);
 };
 
 class SrsRtcSession

trunk/src/kernel/srs_kernel_codec.hpp

@@ -538,10 +538,10 @@ public:
     bool bframe;
 public:
     SrsSample();
-    virtual ~SrsSample();
+    ~SrsSample();
 public:
     // If we need to know whether sample is bframe, we have to parse the NALU payload.
-    virtual srs_error_t parse_bframe();
+    srs_error_t parse_bframe();
 };
 
 /**

trunk/src/kernel/srs_kernel_flv.cpp

@@ -230,7 +230,8 @@ SrsSharedPtrMessage::SrsSharedPtrPayload::~SrsSharedPtrPayload()
         SrsSample* p = extra_payloads + i;
         srs_freep(p->bytes);
     }
-    srs_freep(extra_payloads);
+    srs_freepa(extra_payloads);
+    nn_extra_payloads = 0;
 }
 
 SrsSharedPtrMessage::SrsSharedPtrMessage() : timestamp(0), stream_id(0), size(0), payload(NULL)
@@ -385,6 +386,17 @@ void SrsSharedPtrMessage::set_rtp_packets(const std::vector<SrsRtpSharedPacket*>
 {
     rtp_packets = pkts;
 }
+
+void SrsSharedPtrMessage::set_extra_payloads(SrsSample* payloads, int nn_payloads)
+{
+    srs_assert(nn_payloads);
+    srs_assert(!ptr->extra_payloads);
+    
+    ptr->nn_extra_payloads = nn_payloads;
+
+    ptr->extra_payloads = new SrsSample[nn_payloads];
+    memcpy(ptr->extra_payloads, payloads, nn_payloads * sizeof(SrsSample));
+}
 #endif
 
 SrsFlvTransmuxer::SrsFlvTransmuxer()

trunk/src/kernel/srs_kernel_flv.hpp

@@ -358,6 +358,12 @@ public:
 public:
 #ifdef SRS_AUTO_RTC
     virtual void set_rtp_packets(const std::vector<SrsRtpSharedPacket*>& pkts);
+    // Set extra samples, for example, when we transcode an AAC audio packet to OPUS,
+    // we may get more than one OPUS packets, we set these OPUS packets in extra payloads.
+    void set_extra_payloads(SrsSample* payloads, int nn_payloads);
+    // Get the extra payloads and the number of it.
+    int nn_extra_payloads() { return ptr->nn_extra_payloads; }
+    SrsSample* extra_payloads() { return ptr->extra_payloads; }
 #endif
 };