diff --git a/trunk/doc/CHANGELOG.md b/trunk/doc/CHANGELOG.md index 39345a169..415fa2a7f 100644 --- a/trunk/doc/CHANGELOG.md +++ b/trunk/doc/CHANGELOG.md @@ -66,6 +66,7 @@ The changelog for SRS. ## SRS 4.0 Changelog +* v4.0, 2022-09-16, For [#3179](https://github.com/ossrs/srs/issues/3179): WebRTC: Make sure the same m-lines order for offer and answer. v4.0.265 * v4.0, 2022-09-09, For [#3174](https://github.com/ossrs/srs/issues/3174): WebRTC: Support Unity to publish or play stream. v4.0.264 * v4.0, 2022-09-09, Fix [#3093](https://github.com/ossrs/srs/issues/3093): WebRTC: Ignore unknown fmtp for h.264. v4.0.263 * v4.0, 2022-09-06, Fix [#3170](https://github.com/ossrs/srs/issues/3170): WebRTC: Support WHIP(WebRTC-HTTP ingestion protocol). v4.0.262 diff --git a/trunk/research/players/js/srs.sdk.js b/trunk/research/players/js/srs.sdk.js index 29c37178c..f2ea96c9b 100644 --- a/trunk/research/players/js/srs.sdk.js +++ b/trunk/research/players/js/srs.sdk.js @@ -54,6 +54,8 @@ function SrsRtcPublisherAsync() { var conf = self.__internal.prepareUrl(url); self.pc.addTransceiver("audio", {direction: "sendonly"}); self.pc.addTransceiver("video", {direction: "sendonly"}); + //self.pc.addTransceiver("video", {direction: "sendonly"}); + //self.pc.addTransceiver("audio", {direction: "sendonly"}); if (!navigator.mediaDevices && window.location.protocol === 'http:' && window.location.hostname !== 'localhost') { throw new SrsError('HttpsRequiredError', `Please use HTTPS or localhost to publish, read https://github.com/ossrs/srs/issues/2762#issuecomment-983147576`); @@ -300,6 +302,8 @@ function SrsRtcPlayerAsync() { var conf = self.__internal.prepareUrl(url); self.pc.addTransceiver("audio", {direction: "recvonly"}); self.pc.addTransceiver("video", {direction: "recvonly"}); + //self.pc.addTransceiver("video", {direction: "recvonly"}); + //self.pc.addTransceiver("audio", {direction: "recvonly"}); var offer = await self.pc.createOffer(); await self.pc.setLocalDescription(offer); diff --git a/trunk/src/app/srs_app_rtc_conn.cpp b/trunk/src/app/srs_app_rtc_conn.cpp index f3891927a..b109d2642 100644 --- a/trunk/src/app/srs_app_rtc_conn.cpp +++ b/trunk/src/app/srs_app_rtc_conn.cpp @@ -1898,7 +1898,7 @@ srs_error_t SrsRtcConnection::add_publisher(SrsRtcUserConfig* ruc, SrsSdp& local return srs_error_wrap(err, "publish negotiate, offer=%s", ruc->remote_sdp_str_.c_str()); } - if ((err = generate_publish_local_sdp(req, local_sdp, stream_desc, ruc->remote_sdp_.is_unified())) != srs_success) { + if ((err = generate_publish_local_sdp(req, local_sdp, stream_desc, ruc->remote_sdp_.is_unified(), ruc->audio_before_video_)) != srs_success) { return srs_error_wrap(err, "generate local sdp"); } @@ -1958,7 +1958,7 @@ srs_error_t SrsRtcConnection::add_player(SrsRtcUserConfig* ruc, SrsSdp& local_sd ++it; } - if ((err = generate_play_local_sdp(req, local_sdp, stream_desc, ruc->remote_sdp_.is_unified())) != srs_success) { + if ((err = generate_play_local_sdp(req, local_sdp, stream_desc, ruc->remote_sdp_.is_unified(), ruc->audio_before_video_)) != srs_success) { return srs_error_wrap(err, "generate local sdp"); } @@ -2569,9 +2569,14 @@ srs_error_t SrsRtcConnection::negotiate_publish_capability(SrsRtcUserConfig* ruc // TODO: FIME: Should check packetization-mode=1 also. bool has_42e01f = srs_sdp_has_h264_profile(remote_sdp, "42e01f"); + // How many video descriptions we have parsed. + int nn_any_video_parsed = 0; + for (int i = 0; i < (int)remote_sdp.media_descs_.size(); ++i) { const SrsMediaDesc& remote_media_desc = remote_sdp.media_descs_.at(i); + if (remote_media_desc.is_video()) nn_any_video_parsed++; + SrsRtcTrackDescription* track_desc = new SrsRtcTrackDescription(); SrsAutoFree(SrsRtcTrackDescription, track_desc); @@ -2594,6 +2599,9 @@ srs_error_t SrsRtcConnection::negotiate_publish_capability(SrsRtcUserConfig* ruc } if (remote_media_desc.is_audio()) { + // Update the ruc, which is about user specified configuration. + ruc->audio_before_video_ = !nn_any_video_parsed; + // TODO: check opus format specific param std::vector payloads = remote_media_desc.find_media_with_encoding_name("opus"); if (payloads.empty()) { @@ -2807,7 +2815,7 @@ srs_error_t SrsRtcConnection::negotiate_publish_capability(SrsRtcUserConfig* ruc return err; } -srs_error_t SrsRtcConnection::generate_publish_local_sdp(SrsRequest* req, SrsSdp& local_sdp, SrsRtcSourceDescription* stream_desc, bool unified_plan) +srs_error_t SrsRtcConnection::generate_publish_local_sdp(SrsRequest* req, SrsSdp& local_sdp, SrsRtcSourceDescription* stream_desc, bool unified_plan, bool audio_before_video) { srs_error_t err = srs_success; @@ -2832,6 +2840,29 @@ srs_error_t SrsRtcConnection::generate_publish_local_sdp(SrsRequest* req, SrsSdp local_sdp.group_policy_ = "BUNDLE"; + if (audio_before_video) { + if ((err = generate_publish_local_sdp_for_audio(local_sdp, stream_desc)) != srs_success) { + return srs_error_wrap(err, "audio"); + } + if ((err = generate_publish_local_sdp_for_video(local_sdp, stream_desc, unified_plan)) != srs_success) { + return srs_error_wrap(err, "video"); + } + } else { + if ((err = generate_publish_local_sdp_for_video(local_sdp, stream_desc, unified_plan)) != srs_success) { + return srs_error_wrap(err, "video"); + } + if ((err = generate_publish_local_sdp_for_audio(local_sdp, stream_desc)) != srs_success) { + return srs_error_wrap(err, "audio"); + } + } + + return err; +} + +srs_error_t SrsRtcConnection::generate_publish_local_sdp_for_audio(SrsSdp& local_sdp, SrsRtcSourceDescription* stream_desc) +{ + srs_error_t err = srs_success; + // generate audio media desc if (stream_desc->audio_track_desc_) { SrsRtcTrackDescription* audio_track = stream_desc->audio_track_desc_; @@ -2866,6 +2897,13 @@ srs_error_t SrsRtcConnection::generate_publish_local_sdp(SrsRequest* req, SrsSdp local_media_desc.payload_types_.push_back(payload->generate_media_payload_type()); } + return err; +} + +srs_error_t SrsRtcConnection::generate_publish_local_sdp_for_video(SrsSdp& local_sdp, SrsRtcSourceDescription* stream_desc, bool unified_plan) +{ + srs_error_t err = srs_success; + for (int i = 0; i < (int)stream_desc->video_track_descs_.size(); ++i) { SrsRtcTrackDescription* video_track = stream_desc->video_track_descs_.at(i); @@ -2929,9 +2967,14 @@ srs_error_t SrsRtcConnection::negotiate_play_capability(SrsRtcUserConfig* ruc, s return srs_error_wrap(err, "fetch rtc source"); } + // How many video descriptions we have parsed. + int nn_any_video_parsed = 0; + for (int i = 0; i < (int)remote_sdp.media_descs_.size(); ++i) { const SrsMediaDesc& remote_media_desc = remote_sdp.media_descs_.at(i); + if (remote_media_desc.is_video()) nn_any_video_parsed++; + // Whether feature enabled in remote extmap. int remote_twcc_id = 0; if (true) { @@ -2947,6 +2990,9 @@ srs_error_t SrsRtcConnection::negotiate_play_capability(SrsRtcUserConfig* ruc, s std::vector track_descs; SrsMediaPayloadType remote_payload(0); if (remote_media_desc.is_audio()) { + // Update the ruc, which is about user specified configuration. + ruc->audio_before_video_ = !nn_any_video_parsed; + // TODO: check opus format specific param vector payloads = remote_media_desc.find_media_with_encoding_name("opus"); if (payloads.empty()) { @@ -3106,7 +3152,7 @@ void video_track_generate_play_offer(SrsRtcTrackDescription* track, string mid, } } -srs_error_t SrsRtcConnection::generate_play_local_sdp(SrsRequest* req, SrsSdp& local_sdp, SrsRtcSourceDescription* stream_desc, bool unified_plan) +srs_error_t SrsRtcConnection::generate_play_local_sdp(SrsRequest* req, SrsSdp& local_sdp, SrsRtcSourceDescription* stream_desc, bool unified_plan, bool audio_before_video) { srs_error_t err = srs_success; @@ -3133,6 +3179,29 @@ srs_error_t SrsRtcConnection::generate_play_local_sdp(SrsRequest* req, SrsSdp& l std::string cname = srs_random_str(16); + if (audio_before_video) { + if ((err = generate_play_local_sdp_for_audio(local_sdp, stream_desc, cname)) != srs_success) { + return srs_error_wrap(err, "audio"); + } + if ((err = generate_play_local_sdp_for_video(local_sdp, stream_desc, unified_plan, cname)) != srs_success) { + return srs_error_wrap(err, "video"); + } + } else { + if ((err = generate_play_local_sdp_for_video(local_sdp, stream_desc, unified_plan, cname)) != srs_success) { + return srs_error_wrap(err, "video"); + } + if ((err = generate_play_local_sdp_for_audio(local_sdp, stream_desc, cname)) != srs_success) { + return srs_error_wrap(err, "audio"); + } + } + + return err; +} + +srs_error_t SrsRtcConnection::generate_play_local_sdp_for_audio(SrsSdp& local_sdp, SrsRtcSourceDescription* stream_desc, std::string cname) +{ + srs_error_t err = srs_success; + // generate audio media desc if (stream_desc->audio_track_desc_) { SrsRtcTrackDescription* audio_track = stream_desc->audio_track_desc_; @@ -3192,6 +3261,13 @@ srs_error_t SrsRtcConnection::generate_play_local_sdp(SrsRequest* req, SrsSdp& l } } + return err; +} + +srs_error_t SrsRtcConnection::generate_play_local_sdp_for_video(SrsSdp& local_sdp, SrsRtcSourceDescription* stream_desc, bool unified_plan, std::string cname) +{ + srs_error_t err = srs_success; + for (int i = 0; i < (int)stream_desc->video_track_descs_.size(); ++i) { SrsRtcTrackDescription* track = stream_desc->video_track_descs_[i]; diff --git a/trunk/src/app/srs_app_rtc_conn.hpp b/trunk/src/app/srs_app_rtc_conn.hpp index 28021e648..82b47fa4b 100644 --- a/trunk/src/app/srs_app_rtc_conn.hpp +++ b/trunk/src/app/srs_app_rtc_conn.hpp @@ -540,11 +540,15 @@ public: private: // publish media capabilitiy negotiate srs_error_t negotiate_publish_capability(SrsRtcUserConfig* ruc, SrsRtcSourceDescription* stream_desc); - srs_error_t generate_publish_local_sdp(SrsRequest* req, SrsSdp& local_sdp, SrsRtcSourceDescription* stream_desc, bool unified_plan); + srs_error_t generate_publish_local_sdp(SrsRequest* req, SrsSdp& local_sdp, SrsRtcSourceDescription* stream_desc, bool unified_plan, bool audio_before_video); + srs_error_t generate_publish_local_sdp_for_audio(SrsSdp& local_sdp, SrsRtcSourceDescription* stream_desc); + srs_error_t generate_publish_local_sdp_for_video(SrsSdp& local_sdp, SrsRtcSourceDescription* stream_desc, bool unified_plan); // play media capabilitiy negotiate //TODO: Use StreamDescription to negotiate and remove first negotiate_play_capability function srs_error_t negotiate_play_capability(SrsRtcUserConfig* ruc, std::map& sub_relations); - srs_error_t generate_play_local_sdp(SrsRequest* req, SrsSdp& local_sdp, SrsRtcSourceDescription* stream_desc, bool unified_plan); + srs_error_t generate_play_local_sdp(SrsRequest* req, SrsSdp& local_sdp, SrsRtcSourceDescription* stream_desc, bool unified_plan, bool audio_before_video); + srs_error_t generate_play_local_sdp_for_audio(SrsSdp& local_sdp, SrsRtcSourceDescription* stream_desc, std::string cname); + srs_error_t generate_play_local_sdp_for_video(SrsSdp& local_sdp, SrsRtcSourceDescription* stream_desc, bool unified_plan, std::string cname); srs_error_t create_player(SrsRequest* request, std::map sub_relations); srs_error_t create_publisher(SrsRequest* request, SrsRtcSourceDescription* stream_desc); }; diff --git a/trunk/src/app/srs_app_rtc_server.cpp b/trunk/src/app/srs_app_rtc_server.cpp index 824e3e7ff..f95374fe8 100644 --- a/trunk/src/app/srs_app_rtc_server.cpp +++ b/trunk/src/app/srs_app_rtc_server.cpp @@ -275,6 +275,7 @@ SrsRtcUserConfig::SrsRtcUserConfig() req_ = new SrsRequest(); publish_ = false; dtls_ = srtp_ = true; + audio_before_video_ = false; } SrsRtcUserConfig::~SrsRtcUserConfig() diff --git a/trunk/src/app/srs_app_rtc_server.hpp b/trunk/src/app/srs_app_rtc_server.hpp index 180ed9226..c5980b90d 100644 --- a/trunk/src/app/srs_app_rtc_server.hpp +++ b/trunk/src/app/srs_app_rtc_server.hpp @@ -68,6 +68,10 @@ public: bool publish_; bool dtls_; bool srtp_; + + // The order of audio and video, or whether audio is before video. Please make sure the order is match for offer and + // answer, or client might fail at setRemoveDescription(answer). See https://github.com/ossrs/srs/issues/3179 + bool audio_before_video_; public: SrsRtcUserConfig(); virtual ~SrsRtcUserConfig(); diff --git a/trunk/src/core/srs_core_version4.hpp b/trunk/src/core/srs_core_version4.hpp index 9a8cd8ee9..1c1c3a4da 100644 --- a/trunk/src/core/srs_core_version4.hpp +++ b/trunk/src/core/srs_core_version4.hpp @@ -9,6 +9,6 @@ #define VERSION_MAJOR 4 #define VERSION_MINOR 0 -#define VERSION_REVISION 264 +#define VERSION_REVISION 265 #endif