From c3219d5431fdb8bea326f90db60214d09854021f Mon Sep 17 00:00:00 2001 From: winlin Date: Mon, 20 Jul 2020 14:15:08 +0800 Subject: [PATCH] RTC: Allow dup play. Parse RED desc. --- trunk/src/app/srs_app_rtc_conn.cpp | 92 ++++++++++++++---------- trunk/src/app/srs_app_rtc_conn.hpp | 2 - trunk/src/app/srs_app_rtc_source.cpp | 104 ++++++++++++++++++++------- trunk/src/app/srs_app_rtc_source.hpp | 19 +++++ 4 files changed, 154 insertions(+), 63 deletions(-) diff --git a/trunk/src/app/srs_app_rtc_conn.cpp b/trunk/src/app/srs_app_rtc_conn.cpp index 71cecd02d..398c8578a 100644 --- a/trunk/src/app/srs_app_rtc_conn.cpp +++ b/trunk/src/app/srs_app_rtc_conn.cpp @@ -295,8 +295,9 @@ srs_error_t SrsRtcPlayStream::start() // If player coroutine allocated, we think the player is started. // To prevent play multiple times for this play stream. + // @remark Allow start multiple times, for DTLS may retransmit the final packet. if (is_started) { - return srs_error_new(ERROR_RTC_STREM_STARTED, "playstream start"); + return err; } srs_freep(trd); @@ -1698,12 +1699,14 @@ srs_error_t SrsRtcConnection::add_player(SrsRequest* req, const SrsSdp& remote_s SrsAutoFree(SrsRtcStreamDescription, stream_desc); std::map::iterator it = play_sub_relations.begin(); while (it != play_sub_relations.end()) { - if (it->second->type_ == "audio" || !stream_desc->audio_track_desc_) { - stream_desc->audio_track_desc_ = it->second->copy(); + SrsRtcTrackDescription* track_desc = it->second; + + if (track_desc->type_ == "audio" || !stream_desc->audio_track_desc_) { + stream_desc->audio_track_desc_ = track_desc->copy(); } - if (it->second->type_ == "video") { - stream_desc->video_track_descs_.push_back(it->second->copy()); + if (track_desc->type_ == "video") { + stream_desc->video_track_descs_.push_back(track_desc->copy()); } ++it; } @@ -1737,12 +1740,14 @@ srs_error_t SrsRtcConnection::add_player2(SrsRequest* req, SrsSdp& local_sdp) std::map::iterator it = play_sub_relations.begin(); while (it != play_sub_relations.end()) { - if (it->second->type_ == "audio" || !stream_desc->audio_track_desc_) { - stream_desc->audio_track_desc_ = it->second->copy(); + SrsRtcTrackDescription* track_desc = it->second; + + if (track_desc->type_ == "audio" || !stream_desc->audio_track_desc_) { + stream_desc->audio_track_desc_ = track_desc->copy(); } - if (it->second->type_ == "video") { - stream_desc->video_track_descs_.push_back(it->second->copy()); + if (track_desc->type_ == "video") { + stream_desc->video_track_descs_.push_back(track_desc->copy()); } ++it; } @@ -2267,7 +2272,6 @@ srs_error_t SrsRtcConnection::negotiate_publish_capability(SrsRequest* req, cons audio_payload->set_opus_param_desc(iter->format_specific_param_); // TODO: FIXME: Only support some transport algorithms. for (int j = 0; j < (int)iter->rtcp_fb_.size(); ++j) { - if (nack_enabled) { if (iter->rtcp_fb_.at(j) == "nack" || iter->rtcp_fb_.at(j) == "nack pli") { audio_payload->rtcp_fbs_.push_back(iter->rtcp_fb_.at(j)); @@ -2375,6 +2379,7 @@ srs_error_t SrsRtcConnection::negotiate_publish_capability(SrsRequest* req, cons SrsRtcTrackDescription* track_desc_copy = track_desc->copy(); track_desc_copy->ssrc_ = ssrc_info.ssrc_; track_desc_copy->id_ = ssrc_info.msid_tracker_; + track_desc_copy->msid_ = ssrc_info.msid_; if (remote_media_desc.is_audio() && !stream_desc->audio_track_desc_) { stream_desc->audio_track_desc_ = track_desc_copy; @@ -2559,22 +2564,24 @@ srs_error_t SrsRtcConnection::negotiate_play_capability(SrsRequest* req, const S track->mid_ = remote_media_desc.mid_; uint32_t publish_ssrc = track->ssrc_; - track->media_->rtcp_fbs_.clear(); - track->extmaps_.clear(); - - for (int j = 0; j < (int)remote_rtcp_fb.size(); j++) { + vector rtcp_fb; + track->media_->rtcp_fbs_.swap(rtcp_fb); + for (int j = 0; j < (int)rtcp_fb.size(); j++) { if (nack_enabled) { - if (remote_rtcp_fb.at(j) == "nack" || remote_rtcp_fb.at(j) == "nack pli" || remote_rtcp_fb.at(j) == "transport-cc") { - track->media_->rtcp_fbs_.push_back(remote_rtcp_fb.at(j)); + if (rtcp_fb.at(j) == "nack" || rtcp_fb.at(j) == "nack pli") { + track->media_->rtcp_fbs_.push_back(rtcp_fb.at(j)); } } + if (twcc_enabled && remote_twcc_id) { + if (rtcp_fb.at(j) == "transport-cc") { + track->media_->rtcp_fbs_.push_back(rtcp_fb.at(j)); + } + track->add_rtp_extension_desc(remote_twcc_id, kTWCCExt); + } } - if (twcc_enabled && remote_twcc_id) { - track->add_rtp_extension_desc(remote_twcc_id, kTWCCExt); - } - track->ssrc_ = SrsRtcSSRCGenerator::instance()->generate_ssrc(); + // TODO: FIXME: set audio_payload rtcp_fbs_, // according by whether downlink is support transport algorithms. // TODO: FIXME: if we support downlink RTX, MUST assign rtx_ssrc_, rtx_pt, rtx_apt @@ -2618,20 +2625,26 @@ srs_error_t SrsRtcConnection::fetch_source_capability(SrsRequest* req, std::map< SrsRtcTrackDescription* track = track_descs[i]->copy(); uint32_t publish_ssrc = track->ssrc_; - track->media_->rtcp_fbs_.clear(); - if (nack_enabled) { - track->media_->rtcp_fbs_.push_back("nack"); - track->media_->rtcp_fbs_.push_back("nack pli"); - } - int local_twcc_id = track->get_rtp_extension_id(kTWCCExt); - track->extmaps_.clear(); - if (twcc_enabled && local_twcc_id) { - track->media_->rtcp_fbs_.push_back("transport-cc"); - track->add_rtp_extension_desc(local_twcc_id, kTWCCExt); + + vector rtcp_fb; + track->media_->rtcp_fbs_.swap(rtcp_fb); + for (int j = 0; j < (int)rtcp_fb.size(); j++) { + if (nack_enabled) { + if (rtcp_fb.at(j) == "nack" || rtcp_fb.at(j) == "nack pli") { + track->media_->rtcp_fbs_.push_back(rtcp_fb.at(j)); + } + } + if (twcc_enabled && local_twcc_id) { + if (rtcp_fb.at(j) == "transport-cc") { + track->media_->rtcp_fbs_.push_back(rtcp_fb.at(j)); + } + track->add_rtp_extension_desc(local_twcc_id, kTWCCExt); + } } track->ssrc_ = SrsRtcSSRCGenerator::instance()->generate_ssrc(); + // TODO: FIXME: set audio_payload rtcp_fbs_, // according by whether downlink is support transport algorithms. // TODO: FIXME: if we support downlink RTX, MUST assign rtx_ssrc_, rtx_pt, rtx_apt @@ -2705,13 +2718,18 @@ srs_error_t SrsRtcConnection::generate_play_local_sdp(SrsRequest* req, SrsSdp& l local_media_desc.inactive_ = true; } + if (audio_track->red_) { + SrsRedPayload* red_payload = (SrsRedPayload*)audio_track->red_; + local_media_desc.payload_types_.push_back(red_payload->generate_media_payload_type()); + } + SrsAudioPayload* payload = (SrsAudioPayload*)audio_track->media_; - local_media_desc.payload_types_.push_back(payload->generate_media_payload_type()); + local_media_desc.payload_types_.push_back(payload->generate_media_payload_type()); //TODO: FIXME: add red, rtx, ulpfec..., payload_types_. //local_media_desc.payload_types_.push_back(payload->generate_media_payload_type()); - local_media_desc.ssrc_infos_.push_back(SrsSSRCInfo(audio_track->ssrc_, cname, stream_id, audio_track->id_)); + local_media_desc.ssrc_infos_.push_back(SrsSSRCInfo(audio_track->ssrc_, cname, audio_track->msid_, audio_track->id_)); if (audio_track->rtx_) { std::vector group_ssrcs; @@ -2719,7 +2737,7 @@ srs_error_t SrsRtcConnection::generate_play_local_sdp(SrsRequest* req, SrsSdp& l group_ssrcs.push_back(audio_track->rtx_ssrc_); local_media_desc.ssrc_groups_.push_back(SrsSSRCGroup("FID", group_ssrcs)); - local_media_desc.ssrc_infos_.push_back(SrsSSRCInfo(audio_track->rtx_ssrc_, cname, stream_id, audio_track->id_)); + local_media_desc.ssrc_infos_.push_back(SrsSSRCInfo(audio_track->rtx_ssrc_, cname, audio_track->msid_, audio_track->id_)); } if (audio_track->ulpfec_) { @@ -2728,7 +2746,7 @@ srs_error_t SrsRtcConnection::generate_play_local_sdp(SrsRequest* req, SrsSdp& l group_ssrcs.push_back(audio_track->fec_ssrc_); local_media_desc.ssrc_groups_.push_back(SrsSSRCGroup("FEC", group_ssrcs)); - local_media_desc.ssrc_infos_.push_back(SrsSSRCInfo(audio_track->fec_ssrc_, cname, stream_id, audio_track->id_)); + local_media_desc.ssrc_infos_.push_back(SrsSSRCInfo(audio_track->fec_ssrc_, cname, audio_track->msid_, audio_track->id_)); } } @@ -2766,7 +2784,7 @@ srs_error_t SrsRtcConnection::generate_play_local_sdp(SrsRequest* req, SrsSdp& l } SrsMediaDesc& local_media_desc = local_sdp.media_descs_.back(); - local_media_desc.ssrc_infos_.push_back(SrsSSRCInfo(track->ssrc_, cname, stream_id, track->id_)); + local_media_desc.ssrc_infos_.push_back(SrsSSRCInfo(track->ssrc_, cname, track->msid_, track->id_)); if (track->rtx_ && track->rtx_ssrc_) { std::vector group_ssrcs; @@ -2774,7 +2792,7 @@ srs_error_t SrsRtcConnection::generate_play_local_sdp(SrsRequest* req, SrsSdp& l group_ssrcs.push_back(track->rtx_ssrc_); local_media_desc.ssrc_groups_.push_back(SrsSSRCGroup("FID", group_ssrcs)); - local_media_desc.ssrc_infos_.push_back(SrsSSRCInfo(track->rtx_ssrc_, cname, stream_id, track->id_)); + local_media_desc.ssrc_infos_.push_back(SrsSSRCInfo(track->rtx_ssrc_, cname, track->msid_, track->id_)); } if (track->ulpfec_ && track->fec_ssrc_) { @@ -2783,7 +2801,7 @@ srs_error_t SrsRtcConnection::generate_play_local_sdp(SrsRequest* req, SrsSdp& l group_ssrcs.push_back(track->fec_ssrc_); local_media_desc.ssrc_groups_.push_back(SrsSSRCGroup("FEC", group_ssrcs)); - local_media_desc.ssrc_infos_.push_back(SrsSSRCInfo(track->fec_ssrc_, cname, stream_id, track->id_)); + local_media_desc.ssrc_infos_.push_back(SrsSSRCInfo(track->fec_ssrc_, cname, track->msid_, track->id_)); } } diff --git a/trunk/src/app/srs_app_rtc_conn.hpp b/trunk/src/app/srs_app_rtc_conn.hpp index b11a205e9..6827c499b 100644 --- a/trunk/src/app/srs_app_rtc_conn.hpp +++ b/trunk/src/app/srs_app_rtc_conn.hpp @@ -300,8 +300,6 @@ class SrsRtcConnection friend class SrsRtcPublishStream; public: bool disposing_; -private: - static uint32_t ssrc_num; private: SrsRtcServer* server_; SrsRtcConnectionStateType state_; diff --git a/trunk/src/app/srs_app_rtc_source.cpp b/trunk/src/app/srs_app_rtc_source.cpp index 73342d385..4eda934a0 100644 --- a/trunk/src/app/srs_app_rtc_source.cpp +++ b/trunk/src/app/srs_app_rtc_source.cpp @@ -40,6 +40,7 @@ #include #include #include +#include #ifdef SRS_FFMPEG_FIT #include @@ -1301,6 +1302,48 @@ srs_error_t SrsAudioPayload::set_opus_param_desc(std::string fmtp) return err; } +SrsRedPayload::SrsRedPayload() +{ +} + +SrsRedPayload::SrsRedPayload(uint8_t pt, std::string encode_name, int sample, int channel) + :SrsCodecPayload(pt, encode_name, sample) +{ + channel_ = channel; +} + +SrsRedPayload::~SrsRedPayload() +{ +} + +SrsRedPayload* SrsRedPayload::copy() +{ + SrsRedPayload* cp = new SrsRedPayload(); + + cp->type_ = type_; + cp->pt_ = pt_; + cp->name_ = name_; + cp->sample_ = sample_; + cp->rtcp_fbs_ = rtcp_fbs_; + cp->channel_ = channel_; + + return cp; +} + +SrsMediaPayloadType SrsRedPayload::generate_media_payload_type() +{ + SrsMediaPayloadType media_payload_type(pt_); + + media_payload_type.encoding_name_ = name_; + media_payload_type.clock_rate_ = sample_; + if (channel_ != 0) { + media_payload_type.encoding_param_ = srs_int2str(channel_); + } + media_payload_type.rtcp_fb_ = rtcp_fbs_; + + return media_payload_type; +} + SrsRtcTrackDescription::SrsRtcTrackDescription() { ssrc_ = 0; @@ -1324,6 +1367,10 @@ SrsRtcTrackDescription::~SrsRtcTrackDescription() bool SrsRtcTrackDescription::has_ssrc(uint32_t ssrc) { + if (!is_active_) { + return false; + } + if (ssrc == ssrc_ || ssrc == rtx_ssrc_ || ssrc == fec_ssrc_) { return true; } @@ -1354,7 +1401,7 @@ void SrsRtcTrackDescription::create_auxiliary_payload(const std::vectorextmaps_ = extmaps_; cp->direction_ = direction_; cp->mid_ = mid_; + cp->msid_ = msid_; cp->is_active_ = is_active_; cp->media_ = media_ ? media_->copy():NULL; cp->red_ = red_ ? red_->copy():NULL; @@ -1572,17 +1620,6 @@ SrsRtcAudioRecvTrack::~SrsRtcAudioRecvTrack() srs_error_t SrsRtcAudioRecvTrack::on_rtp(SrsRtcStream* source, SrsRtpPacket2* pkt) { srs_error_t err = srs_success; - // uint8_t pt = pkt->header.get_payload_type(); - - // SrsRtcTrackDescription track = rtc_stream_desc_->get_audio_tracks(); - // // process red packet. - // if (pt == red_pt) { - - // } else if (pt == rtx_pt) { // process rtx_pt. - // // restore retranmission packet. - // } else if (pt == fec_pt) { - - // } if (source) { if ((err = source->on_rtp(pkt)) != srs_success) { @@ -1614,6 +1651,12 @@ srs_error_t SrsRtcVideoRecvTrack::on_rtp(SrsRtcStream* source, SrsRtpPacket2* pk pkt->frame_type = SrsFrameTypeVideo; + if (source) { + if ((err = source->on_rtp(pkt)) != srs_success) { + return srs_error_wrap(err, "source on rtp"); + } + } + // TODO: FIXME: add rtp process if (request_key_frame_) { // TODO: FIXME: add coroutine to request key frame. @@ -1622,12 +1665,6 @@ srs_error_t SrsRtcVideoRecvTrack::on_rtp(SrsRtcStream* source, SrsRtpPacket2* pk session_->send_rtcp_fb_pli(track_desc_->ssrc_); } - if (source) { - if ((err = source->on_rtp(pkt)) != srs_success) { - return srs_error_wrap(err, "source on rtp"); - } - } - // For NACK to handle packet. if ((err = on_nack(pkt)) != srs_success) { return srs_error_wrap(err, "on nack"); @@ -1666,6 +1703,7 @@ bool SrsRtcSendTrack::has_ssrc(uint32_t ssrc) return false; } + SrsRtpPacket2* SrsRtcSendTrack::fetch_rtp_packet(uint16_t seq) { if (rtp_queue_) { @@ -1675,6 +1713,16 @@ SrsRtpPacket2* SrsRtcSendTrack::fetch_rtp_packet(uint16_t seq) return NULL; } +void SrsRtcSendTrack::set_track_status(bool active) +{ + track_desc_->is_active_ = active; +} + +std::string SrsRtcSendTrack::get_track_id() +{ + return track_desc_->id_; +} + srs_error_t SrsRtcSendTrack::on_rtp(std::vector& send_packets, SrsRtpPacket2* pkt) { return srs_success; @@ -1698,15 +1746,19 @@ srs_error_t SrsRtcAudioSendTrack::on_rtp(std::vector& send_packe { srs_error_t err = srs_success; + if (!track_desc_->is_active_) { + return err; + } + pkt->header.set_ssrc(track_desc_->ssrc_); - pkt->header.set_payload_type(track_desc_->media_->pt_); + + send_packets.push_back(pkt); // Put rtp packet to NACK/ARQ queue if (true) { SrsRtpPacket2* nack = pkt->copy(); rtp_queue_->set(nack->header.get_sequence(), nack); } - send_packets.push_back(pkt); return err; } @@ -1731,17 +1783,20 @@ srs_error_t SrsRtcVideoSendTrack::on_rtp(std::vector& send_packe { srs_error_t err = srs_success; - pkt->header.set_ssrc(track_desc_->ssrc_); - pkt->header.set_payload_type(track_desc_->media_->pt_); + if (!track_desc_->is_active_) { + return err; + } + // TODO: FIXME: rtmp->rtc, need set payload type. + pkt->header.set_ssrc(track_desc_->ssrc_); + + send_packets.push_back(pkt); // Put rtp packet to NACK/ARQ queue if (true) { SrsRtpPacket2* nack = pkt->copy(); rtp_queue_->set(nack->header.get_sequence(), nack); } - send_packets.push_back(pkt); - return err; } @@ -1779,3 +1834,4 @@ uint32_t SrsRtcSSRCGenerator::generate_ssrc() return ++ssrc_num; } + diff --git a/trunk/src/app/srs_app_rtc_source.hpp b/trunk/src/app/srs_app_rtc_source.hpp index f950939e1..aaec5fa36 100644 --- a/trunk/src/app/srs_app_rtc_source.hpp +++ b/trunk/src/app/srs_app_rtc_source.hpp @@ -53,6 +53,7 @@ class SrsRtcTrackDescription; class SrsRtcConnection; class SrsRtpRingBuffer; class SrsRtpNackForReceiver; +class SrsJsonObject; class SrsNtp { @@ -314,6 +315,20 @@ public: srs_error_t set_opus_param_desc(std::string fmtp); }; +// TODO: FIXME: Rename it. +class SrsRedPayload : public SrsCodecPayload +{ +public: + int channel_; +public: + SrsRedPayload(); + SrsRedPayload(uint8_t pt, std::string encode_name, int sample, int channel); + virtual ~SrsRedPayload(); +public: + virtual SrsRedPayload* copy(); + virtual SrsMediaPayloadType generate_media_payload_type(); +}; + class SrsRtcTrackDescription { public: @@ -338,6 +353,8 @@ public: std::string direction_; // TODO: FIXME: whether mid is needed? std::string mid_; + // msid_: track stream id + std::string msid_; // meida payload, such as opus, h264. SrsCodecPayload* media_; @@ -450,6 +467,8 @@ public: public: bool has_ssrc(uint32_t ssrc); SrsRtpPacket2* fetch_rtp_packet(uint16_t seq); + void set_track_status(bool active); + std::string get_track_id(); public: virtual srs_error_t on_rtp(std::vector& send_packets, SrsRtpPacket2* pkt); virtual srs_error_t on_rtcp(SrsRtpPacket2* pkt);