From a0a4337214331f174abc5274e4e31e4cf3b72a6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=83=E6=9B=A6?= Date: Sat, 21 Mar 2020 21:50:06 +0800 Subject: [PATCH] make webrtc audio work --- trunk/configure | 22 +- trunk/src/app/srs_app_audio_recode.cpp | 468 +++++++++++++++++++++++++ trunk/src/app/srs_app_audio_recode.hpp | 126 +++++++ trunk/src/app/srs_app_rtp.cpp | 154 +++++++- trunk/src/app/srs_app_rtp.hpp | 26 ++ trunk/src/app/srs_app_source.cpp | 10 +- 6 files changed, 790 insertions(+), 16 deletions(-) create mode 100644 trunk/src/app/srs_app_audio_recode.cpp create mode 100644 trunk/src/app/srs_app_audio_recode.hpp diff --git a/trunk/configure b/trunk/configure index ab9402506..0abbedc51 100755 --- a/trunk/configure +++ b/trunk/configure @@ -150,6 +150,8 @@ LibSTRoot="${SRS_OBJS_DIR}/st"; LibSTfile="${LibSTRoot}/libst.a" if [[ $SRS_SHARED_ST == YES ]]; then LibSTfile="-lst"; fi # srtp LibSrtpRoot="${SRS_OBJS_DIR}/srtp2/include"; LibSrtpFile="${SRS_OBJS_DIR}/srtp2/lib/libsrtp2.a" +# ffmpeg +LibFfmpegRoot="${SRS_OBJS_DIR}/ffmpeg/include"; LibFfmpegFile="${SRS_OBJS_DIR}/ffmpeg/lib/libavcodec.a ${SRS_OBJS_DIR}/ffmpeg/lib/libswresample.a ${SRS_OBJS_DIR}/ffmpeg/lib/libavutil.a ${SRS_OBJS_DIR}/ffmpeg/lib/libopus.a -lpthread" # openssl-1.1.0e, for the RTMP complex handshake. LibSSLRoot="";LibSSLfile="" if [[ $SRS_SSL == YES && $SRS_USE_SYS_SSL == NO ]]; then @@ -235,7 +237,7 @@ fi if [ $SRS_EXPORT_LIBRTMP_PROJECT = NO ]; then MODULE_ID="SERVICE" MODULE_DEPENDS=("CORE" "KERNEL" "PROTOCOL") - ModuleLibIncs=(${LibSTRoot} ${LibSrtpRoot} ${SRS_OBJS_DIR} ${LibSSLRoot}) + ModuleLibIncs=(${LibSTRoot} ${LibSrtpRoot} ${LibFfmpegRoot} ${SRS_OBJS_DIR} ${LibSSLRoot}) MODULE_FILES=("srs_service_log" "srs_service_st" "srs_service_http_client" "srs_service_http_conn" "srs_service_rtmp_conn" "srs_service_utility" "srs_service_conn") @@ -248,7 +250,7 @@ fi if [ $SRS_EXPORT_LIBRTMP_PROJECT = NO ]; then MODULE_ID="APP" MODULE_DEPENDS=("CORE" "KERNEL" "PROTOCOL" "SERVICE") - ModuleLibIncs=(${LibSTRoot} ${LibSrtpRoot} ${SRS_OBJS_DIR} ${LibSSLRoot}) + ModuleLibIncs=(${LibSTRoot} ${LibSrtpRoot} ${LibFfmpegRoot} ${SRS_OBJS_DIR} ${LibSSLRoot}) MODULE_FILES=("srs_app_server" "srs_app_conn" "srs_app_rtmp_conn" "srs_app_source" "srs_app_refer" "srs_app_hls" "srs_app_forward" "srs_app_encoder" "srs_app_http_stream" "srs_app_thread" "srs_app_bandwidth" "srs_app_st" "srs_app_log" "srs_app_config" @@ -259,7 +261,7 @@ if [ $SRS_EXPORT_LIBRTMP_PROJECT = NO ]; then "srs_app_mpegts_udp" "srs_app_rtp" "srs_app_rtc_conn" "srs_app_dtls" "srs_app_rtsp" "srs_app_listener" "srs_app_async_call" "srs_app_caster_flv" "srs_app_process" "srs_app_ng_exec" "srs_app_hourglass" "srs_app_dash" "srs_app_fragment" "srs_app_dvr" - "srs_app_coworkers" "srs_app_hybrid") + "srs_app_coworkers" "srs_app_hybrid" "srs_app_audio_recode") DEFINES="" # add each modules for app for SRS_MODULE in ${SRS_MODULES[*]}; do @@ -286,7 +288,7 @@ if [ $SRS_EXPORT_LIBRTMP_PROJECT = NO ]; then if [[ $SRS_SRT == YES ]]; then MODULE_DEPENDS+=("SRT") fi - ModuleLibIncs=(${LibSTRoot} ${LibSrtpRoot} ${SRS_OBJS_DIR} ${LibGperfRoot} ${LibSSLRoot}) + ModuleLibIncs=(${LibSTRoot} ${LibSrtpRoot} ${LibFfmpegRoot} ${SRS_OBJS_DIR} ${LibGperfRoot} ${LibSSLRoot}) if [[ $SRS_SRT == YES ]]; then ModuleLibIncs+=("${LibSRTRoot[*]}") fi @@ -299,7 +301,7 @@ fi if [ $SRS_EXPORT_LIBRTMP_PROJECT = NO ]; then MODULE_ID="MAIN" MODULE_DEPENDS=("CORE" "KERNEL" "PROTOCOL" "SERVICE") - ModuleLibIncs=(${LibSTRoot} ${LibSrtpRoot} ${SRS_OBJS_DIR} ${LibGperfRoot} ${LibSSLRoot}) + ModuleLibIncs=(${LibSTRoot} ${LibSrtpRoot} ${LibFfmpegRoot} ${SRS_OBJS_DIR} ${LibGperfRoot} ${LibSSLRoot}) MODULE_FILES=() DEFINES="" # add each modules for main @@ -326,13 +328,13 @@ if [ $SRS_EXPORT_LIBRTMP_PROJECT = NO ]; then done # # all depends libraries - ModuleLibFiles=(${LibSTfile} ${LibSrtpFile} ${LibSSLfile} ${LibGperfFile}) + ModuleLibFiles=(${LibSTfile} ${LibSrtpFile} ${LibFfmpegFile} ${LibSSLfile} ${LibGperfFile}) if [[ $SRS_SRT == YES ]]; then ModuleLibFiles+=("${LibSRTfile[*]}") fi # all depends objects MODULE_OBJS="${CORE_OBJS[@]} ${KERNEL_OBJS[@]} ${PROTOCOL_OBJS[@]} ${SERVICE_OBJS[@]} ${APP_OBJS[@]} ${SERVER_OBJS[@]}" - ModuleLibIncs=(${LibSTRoot} ${LibSrtpRoot} ${SRS_OBJS_DIR} ${LibGperfRoot} ${LibSSLRoot}) + ModuleLibIncs=(${LibSTRoot} ${LibSrtpRoot} ${LibFfmpegRoot} ${SRS_OBJS_DIR} ${LibGperfRoot} ${LibSSLRoot}) if [[ $SRS_SRT == YES ]]; then MODULE_OBJS="${MODULE_OBJS} ${SRT_OBJS[@]}" fi @@ -343,7 +345,7 @@ if [ $SRS_EXPORT_LIBRTMP_PROJECT = NO ]; then # # For modules, without the app module. MODULE_OBJS="${CORE_OBJS[@]} ${KERNEL_OBJS[@]} ${PROTOCOL_OBJS[@]} ${SERVICE_OBJS[@]} ${MAIN_OBJS[@]}" - ModuleLibFiles=(${LibSTfile} ${LibSrtpFile} ${LibSSLfile} ${LibGperfFile}) + ModuleLibFiles=(${LibSTfile} ${LibSrtpFile} ${LibFfmpegFile} ${LibSSLfile} ${LibGperfFile}) # for SRS_MODULE in ${SRS_MODULES[*]}; do . $SRS_MODULE/config @@ -363,11 +365,11 @@ if [ $SRS_UTEST = YES ]; then MODULE_FILES=("srs_utest" "srs_utest_amf0" "srs_utest_protocol" "srs_utest_kernel" "srs_utest_core" "srs_utest_config" "srs_utest_rtmp" "srs_utest_http" "srs_utest_avc" "srs_utest_reload" "srs_utest_mp4" "srs_utest_service" "srs_utest_app") - ModuleLibIncs=(${SRS_OBJS_DIR} ${LibSTRoot} ${LibSrtpRoot} ${LibSSLRoot}) + ModuleLibIncs=(${SRS_OBJS_DIR} ${LibSTRoot} ${LibSrtpRoot} ${LibFfmpegRoot} ${LibSSLRoot}) if [[ $SRS_SRT == YES ]]; then ModuleLibIncs+=("${LibSRTRoot[*]}") fi - ModuleLibFiles=(${LibSTfile} ${LibSrtpFile} ${LibSSLfile}) + ModuleLibFiles=(${LibSTfile} ${LibSrtpFile} ${LibFfmpegFile} ${LibSSLfile}) if [[ $SRS_SRT == YES ]]; then ModuleLibFiles+=("${LibSRTfile[*]}") fi diff --git a/trunk/src/app/srs_app_audio_recode.cpp b/trunk/src/app/srs_app_audio_recode.cpp new file mode 100644 index 000000000..ba6a9cf65 --- /dev/null +++ b/trunk/src/app/srs_app_audio_recode.cpp @@ -0,0 +1,468 @@ + +/** + * The MIT License (MIT) + * + * Copyright (c) 2013-2020 Winlin + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include + +static const int kOpusPacketMs = 20; +static const int kOpusMaxbytes = 8000; +static const int kFrameBufMax = 40960; +static const int kPacketBufMax = 8192; +static const int kPcmBufMax = 4096*4; + +SrsAudioDecoder::SrsAudioDecoder(std::string codec) + : codec_name_(codec) +{ + frame_ = NULL; + packet_ = NULL; + codec_ctx_ = NULL; +} + +SrsAudioDecoder::~SrsAudioDecoder() +{ + if (codec_ctx_) { + avcodec_free_context(&codec_ctx_); + codec_ctx_ = NULL; + } + if (frame_) { + av_frame_free(&frame_); + frame_ = NULL; + } + if (packet_) { + av_packet_free(&packet_); + packet_ = NULL; + } +} + +srs_error_t SrsAudioDecoder::initialize() +{ + srs_error_t err = srs_success; + + if (codec_name_.compare("aac")) { + return srs_error_wrap(err, "Invalid codec name"); + } + + const AVCodec *codec = avcodec_find_decoder_by_name(codec_name_.c_str()); + if (!codec) { + return srs_error_wrap(err, "Codec not found by name"); + } + + codec_ctx_ = avcodec_alloc_context3(codec); + if (!codec_ctx_) { + return srs_error_wrap(err, "Could not allocate audio codec context"); + } + + if (avcodec_open2(codec_ctx_, codec, NULL) < 0) { + return srs_error_wrap(err, "Could not open codec"); + } + + frame_ = av_frame_alloc(); + if (!frame_) { + return srs_error_wrap(err, "Could not allocate audio frame"); + } + + packet_ = av_packet_alloc(); + if (!packet_) { + return srs_error_wrap(err, "Could not allocate audio packet"); + } + + return err; +} + +srs_error_t SrsAudioDecoder::decode(SrsSample *pkt, char *buf, int &size) +{ + srs_error_t err = srs_success; + + packet_->data = (uint8_t *)pkt->bytes; + packet_->size = pkt->size; + + int ret = avcodec_send_packet(codec_ctx_, packet_); + if (ret < 0) { + return srs_error_wrap(err, "Error submitting the packet to the decoder"); + } + + int max = size; + size = 0; + + while (ret >= 0) { + ret = avcodec_receive_frame(codec_ctx_, frame_); + if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) { + return err; + } else if (ret < 0) { + return srs_error_wrap(err, "Error during decoding"); + } + + int pcm_size = av_get_bytes_per_sample(codec_ctx_->sample_fmt); + if (pcm_size < 0) { + return srs_error_wrap(err, "Failed to calculate data size"); + } + + for (int i = 0; i < frame_->nb_samples; i++) { + if (size + pcm_size * codec_ctx_->channels <= max) { + memcpy(buf + size,frame_->data[0] + pcm_size*codec_ctx_->channels * i, pcm_size * codec_ctx_->channels); + size += pcm_size * codec_ctx_->channels; + } + } + } + + return err; +} + +AVCodecContext* SrsAudioDecoder::codec_ctx() +{ + return codec_ctx_; +} + +SrsAudioEncoder::SrsAudioEncoder(int samplerate, int channels, int fec, int complexity) + : inband_fec_(fec), + channels_(channels), + sampling_rate_(samplerate), + complexity_(complexity) +{ + opus_ = NULL; +} + +SrsAudioEncoder::~SrsAudioEncoder() +{ + if (opus_) { + opus_encoder_destroy(opus_); + opus_ = NULL; + } +} + +srs_error_t SrsAudioEncoder::initialize() +{ + srs_error_t err = srs_success; + + int error = 0; + opus_ = opus_encoder_create(sampling_rate_, channels_, OPUS_APPLICATION_VOIP, &error); + if (error != OPUS_OK) { + return srs_error_wrap(err, "Error create Opus encoder"); + } + + switch (sampling_rate_) + { + case 48000: + opus_encoder_ctl(opus_, OPUS_SET_MAX_BANDWIDTH(OPUS_BANDWIDTH_FULLBAND)); + break; + + case 24000: + opus_encoder_ctl(opus_, OPUS_SET_MAX_BANDWIDTH(OPUS_BANDWIDTH_SUPERWIDEBAND)); + + case 16000: + opus_encoder_ctl(opus_, OPUS_SET_MAX_BANDWIDTH(OPUS_BANDWIDTH_WIDEBAND)); + break; + + case 12000: + opus_encoder_ctl(opus_, OPUS_SET_MAX_BANDWIDTH(OPUS_BANDWIDTH_MEDIUMBAND)); + break; + + case 8000: + opus_encoder_ctl(opus_, OPUS_SET_MAX_BANDWIDTH(OPUS_BANDWIDTH_NARROWBAND)); + break; + + default: + sampling_rate_ = 16000; + opus_encoder_ctl(opus_, OPUS_SET_MAX_BANDWIDTH(OPUS_BANDWIDTH_WIDEBAND)); + break; + } + opus_encoder_ctl(opus_, OPUS_SET_INBAND_FEC(inband_fec_)); + opus_encoder_ctl(opus_, OPUS_SET_COMPLEXITY(complexity_)); + + return err; +} + +srs_error_t SrsAudioEncoder::encode(SrsSample *frame, char *buf, int &size) +{ + srs_error_t err = srs_success; + + int nb_samples = sampling_rate_ * kOpusPacketMs / 1000; + if (frame->size != nb_samples * 2 * channels_) { + return srs_error_wrap(err, "invalid frame size %d, should be %d", frame->size, nb_samples * 2 * channels_); + } + + opus_int16 *data = (opus_int16 *)frame->bytes; + size = opus_encode(opus_, data, nb_samples, (unsigned char *)buf, kOpusMaxbytes); + + return err; +} + +SrsAudioResample::SrsAudioResample(int src_rate, int src_layout, enum AVSampleFormat src_fmt, + int src_nb, int dst_rate, int dst_layout, enum AVSampleFormat dst_fmt) + : src_rate_(src_rate), + src_ch_layout_(src_layout), + src_sample_fmt_(src_fmt), + src_nb_samples_(src_nb), + dst_rate_(dst_rate), + dst_ch_layout_(dst_layout), + dst_sample_fmt_(dst_fmt) +{ + src_nb_channels_ = 0; + dst_nb_channels_ = 0; + src_linesize_ = 0; + dst_linesize_ = 0; + dst_nb_samples_ = 0; + src_data_ = NULL; + dst_data_ = 0; + + max_dst_nb_samples_ = 0; + swr_ctx_ = NULL; +} + +SrsAudioResample::~SrsAudioResample() +{ + if (src_data_) { + av_freep(&src_data_[0]); + av_freep(&src_data_); + src_data_ = NULL; + } + if (dst_data_) { + av_freep(&dst_data_[0]); + av_freep(&dst_data_); + dst_data_ = NULL; + } + if (swr_ctx_) { + swr_free(&swr_ctx_); + swr_ctx_ = NULL; + } +} + +srs_error_t SrsAudioResample::initialize() +{ + srs_error_t err = srs_success; + + swr_ctx_ = swr_alloc(); + if (!swr_ctx_) { + return srs_error_wrap(err, "Could not allocate resampler context"); + } + + av_opt_set_int(swr_ctx_, "in_channel_layout", src_ch_layout_, 0); + av_opt_set_int(swr_ctx_, "in_sample_rate", src_rate_, 0); + av_opt_set_sample_fmt(swr_ctx_, "in_sample_fmt", src_sample_fmt_, 0); + + av_opt_set_int(swr_ctx_, "out_channel_layout", dst_ch_layout_, 0); + av_opt_set_int(swr_ctx_, "out_sample_rate", dst_rate_, 0); + av_opt_set_sample_fmt(swr_ctx_, "out_sample_fmt", dst_sample_fmt_, 0); + + int ret; + if ((ret = swr_init(swr_ctx_)) < 0) { + return srs_error_wrap(err, "Failed to initialize the resampling context"); + } + + src_nb_channels_ = av_get_channel_layout_nb_channels(src_ch_layout_); + ret = av_samples_alloc_array_and_samples(&src_data_, &src_linesize_, src_nb_channels_, + src_nb_samples_, src_sample_fmt_, 0); + if (ret < 0) { + return srs_error_wrap(err, "Could not allocate source samples"); + } + + max_dst_nb_samples_ = dst_nb_samples_ = + av_rescale_rnd(src_nb_samples_, dst_rate_, src_rate_, AV_ROUND_UP); + + dst_nb_channels_ = av_get_channel_layout_nb_channels(dst_ch_layout_); + ret = av_samples_alloc_array_and_samples(&dst_data_, &dst_linesize_, dst_nb_channels_, + dst_nb_samples_, dst_sample_fmt_, 0); + if (ret < 0) { + return srs_error_wrap(err, "Could not allocate destination samples"); + } + + return err; +} + +srs_error_t SrsAudioResample::resample(SrsSample *pcm, char *buf, int &size) +{ + srs_error_t err = srs_success; + + int ret, plane = 1; + if (src_sample_fmt_ == AV_SAMPLE_FMT_FLTP) { + plane = 2; + } + if (src_linesize_ * plane < pcm->size || pcm->size < 0) { + return srs_error_wrap(err, "size not ok"); + } + memcpy(src_data_[0], pcm->bytes, pcm->size); + + dst_nb_samples_ = av_rescale_rnd(swr_get_delay(swr_ctx_, src_rate_) + + src_nb_samples_, dst_rate_, src_rate_, AV_ROUND_UP); + if (dst_nb_samples_ > max_dst_nb_samples_) { + av_freep(&dst_data_[0]); + ret = av_samples_alloc(dst_data_, &dst_linesize_, dst_nb_channels_, + dst_nb_samples_, dst_sample_fmt_, 1); + if (ret < 0) { + return srs_error_wrap(err, "alloc error"); + } + max_dst_nb_samples_ = dst_nb_samples_; + } + + ret = swr_convert(swr_ctx_, dst_data_, dst_nb_samples_, (const uint8_t **)src_data_, src_nb_samples_); + if (ret < 0) { + return srs_error_wrap(err, "Error while converting"); + } + + int dst_bufsize = av_samples_get_buffer_size(&dst_linesize_, dst_nb_channels_, + ret, dst_sample_fmt_, 1); + if (dst_bufsize < 0) { + return srs_error_wrap(err, "Could not get sample buffer size"); + } + + int max = size; + size = 0; + if (max > dst_bufsize) { + memcpy(buf, dst_data_[0], dst_bufsize); + size = dst_bufsize; + } + + return err; +} + +SrsAudioRecode::SrsAudioRecode(int channels, int samplerate) + : dst_channels_(channels), + dst_samplerate_(samplerate) +{ + size_ = 0; + data_ = new char[kPcmBufMax]; +} + +SrsAudioRecode::~SrsAudioRecode() +{ + if (dec_) { + delete dec_; + dec_ = NULL; + } + if (enc_) { + delete enc_; + enc_ = NULL; + } + if (resample_) { + delete resample_; + resample_ = NULL; + } + + delete[] data_; +} + +srs_error_t SrsAudioRecode::initialize() +{ + srs_error_t err = srs_success; + + dec_ = new SrsAudioDecoder("aac"); + if (!dec_) { + return srs_error_wrap(err, "SrsAudioDecoder failed"); + } + dec_->initialize(); + + enc_ = new SrsAudioEncoder(dst_samplerate_, dst_channels_, 1, 1); + if (!enc_) { + return srs_error_wrap(err, "SrsAudioEncoder failed"); + } + enc_->initialize(); + + resample_ = NULL; + + return err; +} + +srs_error_t SrsAudioRecode::recode(SrsSample *pkt, char **buf, int *buf_len, int &n) +{ + srs_error_t err = srs_success; + + static char decode_buffer[kPacketBufMax]; + static char resample_buffer[kFrameBufMax]; + static char encode_buffer[kPacketBufMax]; + + if (!dec_) { + return srs_error_wrap(err, "dec_ nullptr"); + } + + int decode_len = kPacketBufMax; + if ((err = dec_->decode(pkt, decode_buffer, decode_len)) != srs_success) { + return srs_error_wrap(err, "decode error"); + } + + if (!resample_) { + int channel_layout = av_get_default_channel_layout(dst_channels_); + AVCodecContext *codec_ctx = dec_->codec_ctx(); + resample_ = new SrsAudioResample(codec_ctx->sample_rate, (int)codec_ctx->channel_layout, \ + codec_ctx->sample_fmt, codec_ctx->frame_size, dst_samplerate_, channel_layout, \ + AV_SAMPLE_FMT_S16); + + if (!resample_) { + return srs_error_wrap(err, "SrsAudioResample failed"); + } + resample_->initialize(); + } + + SrsSample pcm; + pcm.bytes = decode_buffer; + pcm.size = decode_len; + int resample_len = kFrameBufMax; + if ((err = resample_->resample(&pcm, resample_buffer, resample_len)) != srs_success) { + return srs_error_wrap(err, "decode error"); + } + + n = 0; + int data_left = resample_len; + int total; + total = (dst_samplerate_ * kOpusPacketMs / 1000) * 2 * dst_channels_; + + if (size_ + data_left < total) { + memcpy(data_ + size_, resample_buffer, data_left); + size_ += data_left; + } else { + int index = 0; + while (1) { + data_left = data_left - (total - size_); + memcpy(data_ + size_, resample_buffer + index, total - size_); + index += total - size_; + size_ += total - size_; + if (!enc_) { + return srs_error_wrap(err, "enc_ nullptr"); + } + + int encode_len; + pcm.bytes = (char *)data_; + pcm.size = size_; + if ((err = enc_->encode(&pcm, encode_buffer, encode_len)) != srs_success) { + return srs_error_wrap(err, "decode error"); + } + + memcpy(buf[n], encode_buffer, encode_len); + buf_len[n] = encode_len; + n++; + + size_ = 0; + if(!data_left) + break; + + if(data_left < total) { + memcpy(data_ + size_, resample_buffer + index, data_left); + size_ += data_left; + break; + } + } + } + + return err; +} diff --git a/trunk/src/app/srs_app_audio_recode.hpp b/trunk/src/app/srs_app_audio_recode.hpp new file mode 100644 index 000000000..6dac2f07a --- /dev/null +++ b/trunk/src/app/srs_app_audio_recode.hpp @@ -0,0 +1,126 @@ +/** + * The MIT License (MIT) + * + * Copyright (c) 2013-2020 Winlin + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef SRS_APP_AUDIO_RECODE_HPP +#define SRS_APP_AUDIO_RECODE_HPP + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include + +#ifdef __cplusplus +} +#endif + +class SrsSample; + +class SrsAudioDecoder +{ +private: + AVFrame* frame_; + AVPacket* packet_; + AVCodecContext* codec_ctx_; + std::string codec_name_; +public: + SrsAudioDecoder(std::string codec); + virtual ~SrsAudioDecoder(); + srs_error_t initialize(); + virtual srs_error_t decode(SrsSample *pkt, char *buf, int &size); + AVCodecContext* codec_ctx(); +}; + +class SrsAudioEncoder +{ +private: + int inband_fec_; + int channels_; + int sampling_rate_; + int complexity_; + OpusEncoder *opus_; +public: + SrsAudioEncoder(int samplerate, int channels, int fec, int complexity); + virtual ~SrsAudioEncoder(); + srs_error_t initialize(); + virtual srs_error_t encode(SrsSample *frame, char *buf, int &size); +}; + +class SrsAudioResample +{ +private: + int src_rate_; + int src_ch_layout_; + int src_nb_channels_; + enum AVSampleFormat src_sample_fmt_; + int src_linesize_; + int src_nb_samples_; + uint8_t **src_data_; + + int dst_rate_; + int dst_ch_layout_; + int dst_nb_channels_; + enum AVSampleFormat dst_sample_fmt_; + int dst_linesize_; + int dst_nb_samples_; + uint8_t **dst_data_; + + int max_dst_nb_samples_; + struct SwrContext *swr_ctx_; +public: + SrsAudioResample(int src_rate, int src_layout, enum AVSampleFormat src_fmt, + int src_nb, int dst_rate, int dst_layout, enum AVSampleFormat dst_fmt); + virtual ~SrsAudioResample(); + srs_error_t initialize(); + virtual srs_error_t resample(SrsSample *pcm, char *buf, int &size); +}; + +class SrsAudioRecode +{ +private: + SrsAudioDecoder *dec_; + SrsAudioEncoder *enc_; + SrsAudioResample *resample_; + int dst_channels_; + int dst_samplerate_; + int size_; + char *data_; +public: + SrsAudioRecode(int channels, int samplerate); + virtual ~SrsAudioRecode(); + srs_error_t initialize(); + virtual srs_error_t recode(SrsSample *pkt, char **buf, int *buf_len, int &n); +}; + +#endif /* SRS_APP_AUDIO_RECODE_HPP */ diff --git a/trunk/src/app/srs_app_rtp.cpp b/trunk/src/app/srs_app_rtp.cpp index de8932018..b9322ec7d 100644 --- a/trunk/src/app/srs_app_rtp.cpp +++ b/trunk/src/app/srs_app_rtp.cpp @@ -50,6 +50,46 @@ using namespace std; #include #include #include +#include + +// TODO: Add this function into SrsRtpMux class. +srs_error_t aac_raw_append_adts_header(SrsSharedPtrMessage* shared_audio, SrsFormat* format, SrsBuffer** stream_ptr) +{ + srs_error_t err = srs_success; + + if (format->is_aac_sequence_header()) { + return err; + } + + if (stream_ptr == NULL) { + return srs_error_new(ERROR_RTC_RTP_MUXER, "adts"); + } + + srs_verbose("audio samples=%d", format->audio->nb_samples); + + if (format->audio->nb_samples != 1) { + return srs_error_new(ERROR_RTC_RTP_MUXER, "adts"); + } + + int nb_buf = format->audio->samples[0].size + 7; + char* buf = new char[nb_buf]; + SrsBuffer* stream = new SrsBuffer(buf, nb_buf); + + // TODO: Add comment. + stream->write_1bytes(0xFF); + stream->write_1bytes(0xF9); + stream->write_1bytes(((format->acodec->aac_object - 1) << 6) | ((format->acodec->aac_sample_rate & 0x0F) << 2) | ((format->acodec->aac_channels & 0x04) >> 2)); + stream->write_1bytes(((format->acodec->aac_channels & 0x03) << 6) | ((nb_buf >> 11) & 0x03)); + stream->write_1bytes((nb_buf >> 3) & 0xFF); + stream->write_1bytes(((nb_buf & 0x07) << 5) | 0x1F); + stream->write_1bytes(0xFC); + + stream->write_bytes(format->audio->samples[0].bytes, format->audio->samples[0].size); + + *stream_ptr = stream; + + return err; +} SrsRtpMuxer::SrsRtpMuxer() { @@ -288,6 +328,104 @@ srs_error_t SrsRtpMuxer::packet_stap_a(const string &sps, const string& pps, Srs return err; } +SrsRtpOpusMuxer::SrsRtpOpusMuxer() +{ + sequence = 0; + timestamp = 0; + recoder = NULL; +} + +SrsRtpOpusMuxer::~SrsRtpOpusMuxer() +{ + if (recoder) { + delete recoder; + recoder = NULL; + } +} + +srs_error_t SrsRtpOpusMuxer::initialize() +{ + srs_error_t err = srs_success; + + recoder = new SrsAudioRecode(kChannel, kSamplerate); + if (!recoder) { + return srs_error_wrap(err, "SrsAacOpus init failed"); + } + recoder->initialize(); + + return err; +} + +srs_error_t SrsRtpOpusMuxer::frame_to_packet(SrsSharedPtrMessage* shared_audio, SrsFormat* format, SrsBuffer* stream) +{ + srs_error_t err = srs_success; + + vector rtp_packet_vec; + + char* data_ptr[kArrayLength]; + static char data_array[kArrayLength][kArrayBuffer]; + int elen[kArrayLength], number = 0; + + data_ptr[0] = &data_array[0][0]; + for (int i = 1; i < kArrayLength; i++) { + data_ptr[i] = data_array[i]; + } + + SrsSample pkt; + pkt.bytes = stream->data(); + pkt.size = stream->pos(); + + if ((err = recoder->recode(&pkt, data_ptr, elen, number)) != srs_success) { + return srs_error_wrap(err, "recode error"); + } + + for (int i = 0; i < number; i++) { + SrsSample sample; + sample.size = elen[i]; + sample.bytes = data_ptr[i]; + packet_opus(shared_audio, &sample, rtp_packet_vec); + } + + shared_audio->set_rtp_packets(rtp_packet_vec); + + return err; +} + +srs_error_t SrsRtpOpusMuxer::packet_opus(SrsSharedPtrMessage* shared_frame, SrsSample* sample, std::vector& rtp_packet_vec) +{ + srs_error_t err = srs_success; + + char* buf = new char[kRtpPacketSize]; + SrsBuffer* stream = new SrsBuffer(buf, kRtpPacketSize); + SrsAutoFree(SrsBuffer, stream); + + // v=2,p=0,x=0,cc=0 + stream->write_1bytes(0x80); + // marker payloadtype + stream->write_1bytes(kOpusPayloadType); + // sequenct + stream->write_2bytes(sequence); + // timestamp + stream->write_4bytes(int32_t(timestamp)); + timestamp += 960; + // ssrc + stream->write_4bytes(int32_t(kAudioSSRC)); + + stream->write_bytes(sample->bytes, sample->size); + + srs_verbose("sample=%s", srs_string_dumps_hex(sample->bytes, sample->size).c_str()); + srs_verbose("opus, size=%u, seq=%u, timestamp=%lu, ssrc=%u, payloadtype=%u", + sample->size, sequence, timestamp, kAudioSSRC, kOpusPayloadType); + + SrsRtpSharedPacket* rtp_shared_pkt = new SrsRtpSharedPacket(); + rtp_shared_pkt->create(timestamp, sequence++, kAudioSSRC, kOpusPayloadType, stream->data(), stream->pos()); + rtp_shared_pkt->set_marker(true); + + rtp_packet_vec.push_back(rtp_shared_pkt); + + return err; +} + SrsRtp::SrsRtp() { req = NULL; @@ -326,6 +464,11 @@ srs_error_t SrsRtp::initialize(SrsOriginHub* h, SrsRequest* r) req = r; rtp_h264_muxer = new SrsRtpMuxer(); + + rtp_opus_muxer = new SrsRtpOpusMuxer(); + if (rtp_opus_muxer) { + rtp_opus_muxer->initialize(); + } return err; } @@ -387,7 +530,16 @@ srs_error_t SrsRtp::on_audio(SrsSharedPtrMessage* shared_audio, SrsFormat* forma // ignore sequence header srs_assert(format->audio); - // TODO: rtc no support aac + SrsBuffer* stream = NULL; + SrsAutoFree(SrsBuffer, stream); + if ((err = aac_raw_append_adts_header(shared_audio, format, &stream)) != srs_success) { + return srs_error_wrap(err, "aac append header"); + } + + if (stream) { + rtp_opus_muxer->frame_to_packet(shared_audio, format, stream); + } + return err; } diff --git a/trunk/src/app/srs_app_rtp.hpp b/trunk/src/app/srs_app_rtp.hpp index 842d82620..3b64bdf38 100644 --- a/trunk/src/app/srs_app_rtp.hpp +++ b/trunk/src/app/srs_app_rtp.hpp @@ -36,10 +36,13 @@ class SrsSharedPtrMessage; class SrsRtpSharedPacket; class SrsRequest; class SrsOriginHub; +class SrsAudioRecode; +class SrsBuffer; const int max_payload_size = 1200; const int kRtpPacketSize = 1500; +const uint8_t kOpusPayloadType = 111; const uint8_t kH264PayloadType = 102; const uint8_t kNalTypeMask = 0x1F; @@ -50,7 +53,13 @@ const uint8_t kFuA = 28; const uint8_t kStart = 0x80; const uint8_t kEnd = 0x40; +const int kChannel = 2; +const int kSamplerate = 48000; +const int kArrayLength = 8; +const int kArrayBuffer = 4096; + // FIXME: ssrc can relate to source +const uint32_t kAudioSSRC = 3233846890; const uint32_t kVideoSSRC = 3233846889; class SrsRtpMuxer @@ -70,6 +79,22 @@ private: srs_error_t packet_stap_a(const std::string &sps, const std::string& pps, SrsSharedPtrMessage* shared_frame, std::vector& rtp_packet_vec); }; +class SrsRtpOpusMuxer +{ +private: + uint32_t timestamp; + uint16_t sequence; + SrsAudioRecode* recoder; +public: + SrsRtpOpusMuxer(); + virtual ~SrsRtpOpusMuxer(); + virtual srs_error_t initialize(); +public: + srs_error_t frame_to_packet(SrsSharedPtrMessage* shared_audio, SrsFormat* format, SrsBuffer* stream); +private: + srs_error_t packet_opus(SrsSharedPtrMessage* shared_frame, SrsSample* sample, std::vector& rtp_packet_vec); +}; + class SrsRtp { private: @@ -78,6 +103,7 @@ private: bool disposable; srs_utime_t last_update_time; SrsRtpMuxer* rtp_h264_muxer; + SrsRtpOpusMuxer* rtp_opus_muxer; SrsOriginHub* hub; public: SrsRtp(); diff --git a/trunk/src/app/srs_app_source.cpp b/trunk/src/app/srs_app_source.cpp index bccf5aae7..8ecd711e3 100755 --- a/trunk/src/app/srs_app_source.cpp +++ b/trunk/src/app/srs_app_source.cpp @@ -2261,6 +2261,11 @@ srs_error_t SrsSource::on_audio_imp(SrsSharedPtrMessage* msg) } } + // Copy to hub to all utilities. + if ((err = hub->on_audio(msg)) != srs_success) { + return srs_error_wrap(err, "consume audio"); + } + // copy to all consumer if (!drop_for_reduce) { for (int i = 0; i < (int)consumers.size(); i++) { @@ -2271,11 +2276,6 @@ srs_error_t SrsSource::on_audio_imp(SrsSharedPtrMessage* msg) } } - // Copy to hub to all utilities. - if ((err = hub->on_audio(msg)) != srs_success) { - return srs_error_wrap(err, "consume audio"); - } - // cache the sequence header of aac, or first packet of mp3. // for example, the mp3 is used for hls to write the "right" audio codec. // TODO: FIXME: to refine the stream info system.