external/srs
1
0
Fork 0
mirror of https://github.com/ossrs/srs.git synced 2025-03-09 15:49:59 +00:00
Code Issues Releases Wiki Activity

rtmp2rtc: support hevc

Browse source
This commit is contained in:
chenhaibo 2025-02-16 09:19:48 +08:00
parent 13597d1b7f
commit 1a224c77de
12 changed files with 933 additions and 139 deletions
Download patch file Download diff file Expand all files Collapse all files

262
trunk/src/app/srs_app_rtc_source.cpp
View file

@ -785,7 +785,15 @@ std::vector<SrsRtcTrackDescription*> SrsRtcSource::get_track_desc(std::string ty
if (type == "video") {
std::vector<SrsRtcTrackDescription*>::iterator it = stream_desc_->video_track_descs_.begin();
while (it != stream_desc_->video_track_descs_.end() ){
track_descs.push_back(*it);
if (media_name.empty()) {
track_descs.push_back(*it);
} else {
string name = (*it)->media_->name_;
std::transform(name.begin(), name.end(), name.begin(), static_cast<int(*)(int)>(std::toupper));
if (name == media_name) {
track_descs.push_back(*it);
}
}
++it;
}
}
@ -1065,13 +1073,6 @@ srs_error_t SrsRtcRtpBuilder::on_video(SrsSharedPtrMessage* msg)
return err;
}
// WebRTC does NOT support HEVC.
#ifdef SRS_H265
if (format->vcodec->id == SrsVideoCodecIdHEVC) {
return err;
}
#endif
bool has_idr = false;
vector<SrsSample*> samples;
if ((err = filter(msg, format, has_idr, samples)) != srs_success) {
@ -1083,6 +1084,10 @@ srs_error_t SrsRtcRtpBuilder::on_video(SrsSharedPtrMessage* msg)
if (has_idr) {
SrsUniquePtr<SrsRtpPacket> pkt(new SrsRtpPacket());
if ((err = bridge_->update_codec(format->vcodec->id)) != srs_success) {
return srs_error_wrap(err, "update codec");
}
if ((err = package_stap_a(msg, pkt.get())) != srs_success) {
return srs_error_wrap(err, "package stap-a");
}
@ -1149,10 +1154,16 @@ srs_error_t SrsRtcRtpBuilder::filter(SrsSharedPtrMessage* msg, SrsFormat* format
// Because RTC does not support B-frame, so we will drop them.
// TODO: Drop B-frame in better way, which not cause picture corruption.
if (!keep_bframe && format->vcodec->id == SrsVideoCodecIdAVC) {
bool is_b_frame;
if ((err = SrsVideoFrame::parse_avc_b_frame(sample, is_b_frame)) != srs_success) {
return srs_error_wrap(err, "parse bframe");
if (!keep_bframe) {
bool is_b_frame = false;
if (format->vcodec->id == SrsVideoCodecIdAVC) {
if ((err = SrsVideoFrame::parse_avc_b_frame(sample, is_b_frame)) != srs_success) {
return srs_error_wrap(err, "parse bframe");
}
} else if (format->vcodec->id == SrsVideoCodecIdHEVC) {
if ((err = SrsVideoFrame::parse_hevc_b_frame(sample, format, is_b_frame)) != srs_success) {
return srs_error_wrap(err, "parse bframe");
}
}
if (is_b_frame) {
continue;
@ -1174,53 +1185,59 @@ srs_error_t SrsRtcRtpBuilder::package_stap_a(SrsSharedPtrMessage* msg, SrsRtpPac
return err;
}
// Note that the sps/pps may change, so we should copy it.
const vector<char>& sps = format->vcodec->sequenceParameterSetNALUnit;
const vector<char>& pps = format->vcodec->pictureParameterSetNALUnit;
if (sps.empty() || pps.empty()) {
return srs_error_new(ERROR_RTC_RTP_MUXER, "sps/pps empty");
}
pkt->header.set_payload_type(video_payload_type_);
pkt->header.set_ssrc(video_ssrc_);
pkt->frame_type = SrsFrameTypeVideo;
pkt->nalu_type = (SrsAvcNaluType)kStapA;
pkt->header.set_marker(false);
pkt->header.set_sequence(video_sequence++);
pkt->header.set_timestamp(msg->timestamp * 90);
SrsRtpSTAPPayload* stap = new SrsRtpSTAPPayload();
pkt->set_payload(stap, SrsRtspPacketPayloadTypeSTAP);
ISrsRtpPayloader* stap = NULL;
vector<vector<char>*> params;
int size = 0;
if (format->vcodec->id == SrsVideoCodecIdHEVC) {
for (int i = 0; i < format->vcodec->hevc_dec_conf_record_.nalu_vec.size(); i++) {
if (format->vcodec->hevc_dec_conf_record_.nalu_vec[i].nal_unit_type == SrsHevcNaluType_VPS
|| format->vcodec->hevc_dec_conf_record_.nalu_vec[i].nal_unit_type == SrsHevcNaluType_SPS
|| format->vcodec->hevc_dec_conf_record_.nalu_vec[i].nal_unit_type == SrsHevcNaluType_PPS) {
vector<char>& nalu = (vector<char>&)format->vcodec->hevc_dec_conf_record_.nalu_vec[i].nal_data_vec[0].nal_unit_data;
params.push_back(&nalu);
size += format->vcodec->hevc_dec_conf_record_.nalu_vec[i].nal_data_vec[0].nal_unit_length;
}
}
uint8_t header = sps[0];
stap->nri = (SrsAvcNaluType)header;
stap = new SrsRtpSTAPPayloadHevc();
pkt->set_payload(stap, SrsRtspPacketPayloadTypeSTAPHevc);
pkt->nalu_type = kStapHevc;
} else if (format->vcodec->id == SrsVideoCodecIdAVC) {
params.push_back(&format->vcodec->sequenceParameterSetNALUnit);
params.push_back(&format->vcodec->pictureParameterSetNALUnit);
size = format->vcodec->sequenceParameterSetNALUnit.size() + format->vcodec->pictureParameterSetNALUnit.size();
// Copy the SPS/PPS bytes, because it may change.
int size = (int)(sps.size() + pps.size());
stap = new SrsRtpSTAPPayload();
pkt->set_payload(stap, SrsRtspPacketPayloadTypeSTAP);
pkt->nalu_type = kStapA;
}
if (size == 0) {
return srs_error_new(ERROR_RTC_RTP_MUXER, "vps/sps/pps empty");
}
char* payload = pkt->wrap(size);
if (true) {
for (vector<char>* param : params) {
SrsSample* sample = new SrsSample();
sample->bytes = payload;
sample->size = (int)sps.size();
stap->nalus.push_back(sample);
sample->size = param->size();
if (format->vcodec->id == SrsVideoCodecIdHEVC) {
static_cast<SrsRtpSTAPPayloadHevc*>(stap)->nalus.push_back(sample);
} else {
static_cast<SrsRtpSTAPPayload*>(stap)->nalus.push_back(sample);
}
memcpy(payload, (char*)&sps[0], sps.size());
payload += (int)sps.size();
memcpy(payload, (char*)param->data(), param->size());
payload += (int)param->size();
}
if (true) {
SrsSample* sample = new SrsSample();
sample->bytes = payload;
sample->size = (int)pps.size();
stap->nalus.push_back(sample);
memcpy(payload, (char*)&pps[0], pps.size());
payload += (int)pps.size();
}
srs_info("RTC STAP-A seq=%u, sps %d, pps %d bytes", pkt->header.get_sequence(), sps.size(), pps.size());
return err;
}
@ -1260,12 +1277,22 @@ srs_error_t SrsRtcRtpBuilder::package_nalus(SrsSharedPtrMessage* msg, const vect
pkt->header.set_payload_type(video_payload_type_);
pkt->header.set_ssrc(video_ssrc_);
pkt->frame_type = SrsFrameTypeVideo;
pkt->nalu_type = (SrsAvcNaluType)first_nalu_type;
pkt->nalu_type = first_nalu_type;
pkt->header.set_sequence(video_sequence++);
pkt->header.set_timestamp(msg->timestamp * 90);
pkt->set_payload(raw_raw, SrsRtspPacketPayloadTypeNALU);
pkt->wrap(msg);
} else {
SrsFormat* format = meta->vsh_format();
if (!format || !format->vcodec) {
return err;
}
bool is_hevc = format->vcodec->id == SrsVideoCodecIdHEVC;
// H264 FU-A header size is 1 @see: https://datatracker.ietf.org/doc/html/rfc6184#section-5.8
// H265 FU-A header size is 2 @see: https://datatracker.ietf.org/doc/html/rfc7798#section-4.4.3
int header_size = is_hevc ? 2 : 1;
// We must free it, should never use RTP packets to free it,
// because more than one RTP packet will refer to it.
SrsUniquePtr<SrsRtpRawNALUs> raw(raw_raw);
@ -1274,36 +1301,48 @@ srs_error_t SrsRtcRtpBuilder::package_nalus(SrsSharedPtrMessage* msg, const vect
int fu_payload_size = kRtpMaxPayloadSize;
// The first byte is store in FU-A header.
uint8_t header = raw->skip_first_byte();
uint8_t nal_type = header & kNalTypeMask;
int nb_left = nn_bytes - 1;
uint8_t header = raw->skip_bytes(header_size);
int nb_left = nn_bytes - header_size;
int num_of_packet = 1 + (nn_bytes - 1) / fu_payload_size;
for (int i = 0; i < num_of_packet; ++i) {
int packet_size = srs_min(nb_left, fu_payload_size);
SrsRtpFUAPayload* fua = new SrsRtpFUAPayload();
if ((err = raw->read_samples(fua->nalus, packet_size)) != srs_success) {
srs_freep(fua);
return srs_error_wrap(err, "read samples %d bytes, left %d, total %d", packet_size, nb_left, nn_bytes);
}
SrsRtpPacket* pkt = new SrsRtpPacket();
pkts.push_back(pkt);
pkt->header.set_payload_type(video_payload_type_);
pkt->header.set_ssrc(video_ssrc_);
pkt->frame_type = SrsFrameTypeVideo;
pkt->nalu_type = (SrsAvcNaluType)kFuA;
pkt->nalu_type = kFuA;
pkt->header.set_sequence(video_sequence++);
pkt->header.set_timestamp(msg->timestamp * 90);
fua->nri = (SrsAvcNaluType)header;
fua->nalu_type = (SrsAvcNaluType)nal_type;
fua->start = bool(i == 0);
fua->end = bool(i == num_of_packet - 1);
if (is_hevc) {
SrsRtpFUAPayloadHevc* fua = new SrsRtpFUAPayloadHevc();
if ((err = raw->read_samples(fua->nalus, packet_size)) != srs_success) {
srs_freep(fua);
return srs_error_wrap(err, "read samples %d bytes, left %d, total %d", packet_size, nb_left, nn_bytes);
}
fua->nalu_type = SrsHevcNaluTypeParse(header);
fua->start = bool(i == 0);
fua->end = bool(i == num_of_packet - 1);
pkt->set_payload(fua, SrsRtspPacketPayloadTypeFUAHevc);
} else {
SrsRtpFUAPayload* fua = new SrsRtpFUAPayload();
if ((err = raw->read_samples(fua->nalus, packet_size)) != srs_success) {
srs_freep(fua);
return srs_error_wrap(err, "read samples %d bytes, left %d, total %d", packet_size, nb_left, nn_bytes);
}
fua->nalu_type = (SrsAvcNaluType)(header & kNalTypeMask);
fua->start = bool(i == 0);
fua->end = bool(i == num_of_packet - 1);
pkt->set_payload(fua, SrsRtspPacketPayloadTypeFUA);
}
pkt->set_payload(fua, SrsRtspPacketPayloadTypeFUA);
pkt->wrap(msg);
nb_left -= packet_size;
@ -1342,11 +1381,19 @@ srs_error_t SrsRtcRtpBuilder::package_fu_a(SrsSharedPtrMessage* msg, SrsSample*
{
srs_error_t err = srs_success;
char* p = sample->bytes + 1;
int nb_left = sample->size - 1;
uint8_t header = sample->bytes[0];
uint8_t nal_type = header & kNalTypeMask;
SrsFormat* format = meta->vsh_format();
if (!format || !format->vcodec) {
return err;
}
bool is_hevc = format->vcodec->id == SrsVideoCodecIdHEVC;
// H264 FU-A header size is 1 @see: https://datatracker.ietf.org/doc/html/rfc6184#section-5.8
// H265 FU-A header size is 2 @see: https://datatracker.ietf.org/doc/html/rfc7798#section-4.4.3
int header_size = is_hevc ? 2 : 1;
char* p = sample->bytes + header_size;
int nb_left = sample->size - header_size;
uint8_t header = sample->bytes[0];
int num_of_packet = 1 + (nb_left - 1) / fu_payload_size;
for (int i = 0; i < num_of_packet; ++i) {
int packet_size = srs_min(nb_left, fu_payload_size);
@ -1359,17 +1406,34 @@ srs_error_t SrsRtcRtpBuilder::package_fu_a(SrsSharedPtrMessage* msg, SrsSample*
pkt->frame_type = SrsFrameTypeVideo;
pkt->header.set_sequence(video_sequence++);
pkt->header.set_timestamp(msg->timestamp * 90);
pkt->nalu_type = is_hevc ? kFuHevc : kFuA;
SrsRtpFUAPayload2* fua = new SrsRtpFUAPayload2();
pkt->set_payload(fua, SrsRtspPacketPayloadTypeFUA2);
if (is_hevc) {
uint8_t nal_type = SrsHevcNaluTypeParse(header);
// H265 FU-A header
SrsRtpFUAPayloadHevc2* fua = new SrsRtpFUAPayloadHevc2();
pkt->set_payload(fua, SrsRtspPacketPayloadTypeFUAHevc);
fua->nri = (SrsAvcNaluType)header;
fua->nalu_type = (SrsAvcNaluType)nal_type;
fua->start = bool(i == 0);
fua->end = bool(i == num_of_packet - 1);
fua->nalu_type = (SrsHevcNaluType)nal_type;
fua->start = bool(i == 0);
fua->end = bool(i == num_of_packet - 1);
fua->payload = p;
fua->size = packet_size;
fua->payload = p;
fua->size = packet_size;
} else {
uint8_t nal_type = header & kNalTypeMask;
// H264 FU-A header
SrsRtpFUAPayload2* fua = new SrsRtpFUAPayload2();
pkt->set_payload(fua, SrsRtspPacketPayloadTypeFUA2);
fua->nri = (SrsAvcNaluType)header;
fua->nalu_type = (SrsAvcNaluType)nal_type;
fua->start = bool(i == 0);
fua->end = bool(i == num_of_packet - 1);
fua->payload = p;
fua->size = packet_size;
}
pkt->wrap(msg);
@ -1568,6 +1632,7 @@ srs_error_t SrsRtcFrameBuilder::packet_video(SrsRtpPacket* src)
SrsRtpPacket* pkt = src->copy();
if (pkt->is_keyframe()) {
// TODO: 处理H265
return packet_video_key_frame(pkt);
}
@ -2042,6 +2107,7 @@ SrsVideoPayload* SrsVideoPayload::copy()
cp->sample_ = sample_;
cp->rtcp_fbs_ = rtcp_fbs_;
cp->h264_param_ = h264_param_;
cp->h265_param_ = h265_param_;
return cp;
}
@ -2070,6 +2136,33 @@ SrsMediaPayloadType SrsVideoPayload::generate_media_payload_type()
return media_payload_type;
}
SrsMediaPayloadType SrsVideoPayload::generate_media_payload_type_h265()
{
SrsMediaPayloadType media_payload_type(pt_);
media_payload_type.encoding_name_ = name_;
media_payload_type.clock_rate_ = sample_;
media_payload_type.rtcp_fb_ = rtcp_fbs_;
std::ostringstream format_specific_param;
if (!h265_param_.level_id.empty()) {
format_specific_param << "level-id=" << h265_param_.level_id;
}
if (!h265_param_.profile_id.empty()) {
format_specific_param << ";profile-id=" << h265_param_.profile_id;
}
if (!h265_param_.tier_flag.empty()) {
format_specific_param << ";tier-flag=" << h265_param_.tier_flag;
}
if (!h265_param_.tx_mode.empty()) {
format_specific_param << ";tx-mode=" << h265_param_.tx_mode;
}
media_payload_type.format_specific_param_ = format_specific_param.str();
return media_payload_type;
}
srs_error_t SrsVideoPayload::set_h264_param_desc(std::string fmtp)
{
srs_error_t err = srs_success;
@ -2107,6 +2200,31 @@ srs_error_t SrsVideoPayload::set_h264_param_desc(std::string fmtp)
return err;
}
// level-id=180;profile-id=1;tier-flag=0;tx-mode=SRST
srs_error_t SrsVideoPayload::set_h265_param_desc(std::string fmtp)
{
std::vector<std::string> attributes = split_str(fmtp, ";");
for (size_t i = 0; i < attributes.size(); ++i) {
std::string attribute = attributes.at(i);
std::vector<std::string> kv = split_str(attribute, "=");
if (kv.size() != 2) {
return srs_error_new(ERROR_RTC_SDP_DECODE, "invalid h265 param=%s", attribute.c_str());
}
if (kv[0] == "level-id") {
h265_param_.level_id = kv[1];
} else if (kv[0] == "profile-id") {
h265_param_.profile_id = kv[1];
} else if (kv[0] == "tier-flag") {
h265_param_.tier_flag = kv[1];
} else if (kv[0] == "tx-mode") {
h265_param_.tx_mode = kv[1];
} else {
return srs_error_new(ERROR_RTC_SDP_DECODE, "invalid h265 param=%s", kv[0].c_str());
}
}
return srs_success;
}
SrsAudioPayload::SrsAudioPayload()
{
channel_ = 0;
@ -2699,7 +2817,7 @@ void SrsRtcVideoRecvTrack::on_before_decode_payload(SrsRtpPacket* pkt, SrsBuffer
}
uint8_t v = (uint8_t)(buf->head()[0] & kNalTypeMask);
pkt->nalu_type = SrsAvcNaluType(v);
pkt->nalu_type = v;
if (v == kStapA) {
*ppayload = new SrsRtpSTAPPayload();