mirror of
https://github.com/ossrs/srs.git
synced 2025-03-09 15:49:59 +00:00
make webrtc audio work
This commit is contained in:
七曦
parent
68ad006b73
commit
a0a4337214
6 changed files with 790 additions and 16 deletions
22
trunk/configure
vendored
22
trunk/configure
vendored
|
|
@ -150,6 +150,8 @@ LibSTRoot="${SRS_OBJS_DIR}/st"; LibSTfile="${LibSTRoot}/libst.a"
|
|||
if [[ $SRS_SHARED_ST == YES ]]; then LibSTfile="-lst"; fi
|
||||
# srtp
|
||||
LibSrtpRoot="${SRS_OBJS_DIR}/srtp2/include"; LibSrtpFile="${SRS_OBJS_DIR}/srtp2/lib/libsrtp2.a"
|
||||
# ffmpeg
|
||||
LibFfmpegRoot="${SRS_OBJS_DIR}/ffmpeg/include"; LibFfmpegFile="${SRS_OBJS_DIR}/ffmpeg/lib/libavcodec.a ${SRS_OBJS_DIR}/ffmpeg/lib/libswresample.a ${SRS_OBJS_DIR}/ffmpeg/lib/libavutil.a ${SRS_OBJS_DIR}/ffmpeg/lib/libopus.a -lpthread"
|
||||
# openssl-1.1.0e, for the RTMP complex handshake.
|
||||
LibSSLRoot="";LibSSLfile=""
|
||||
if [[ $SRS_SSL == YES && $SRS_USE_SYS_SSL == NO ]]; then
|
||||
|
|
@ -235,7 +237,7 @@ fi
|
|||
if [ $SRS_EXPORT_LIBRTMP_PROJECT = NO ]; then
|
||||
MODULE_ID="SERVICE"
|
||||
MODULE_DEPENDS=("CORE" "KERNEL" "PROTOCOL")
|
||||
ModuleLibIncs=(${LibSTRoot} ${LibSrtpRoot} ${SRS_OBJS_DIR} ${LibSSLRoot})
|
||||
ModuleLibIncs=(${LibSTRoot} ${LibSrtpRoot} ${LibFfmpegRoot} ${SRS_OBJS_DIR} ${LibSSLRoot})
|
||||
MODULE_FILES=("srs_service_log" "srs_service_st" "srs_service_http_client"
|
||||
"srs_service_http_conn" "srs_service_rtmp_conn" "srs_service_utility"
|
||||
"srs_service_conn")
|
||||
|
|
@ -248,7 +250,7 @@ fi
|
|||
if [ $SRS_EXPORT_LIBRTMP_PROJECT = NO ]; then
|
||||
MODULE_ID="APP"
|
||||
MODULE_DEPENDS=("CORE" "KERNEL" "PROTOCOL" "SERVICE")
|
||||
ModuleLibIncs=(${LibSTRoot} ${LibSrtpRoot} ${SRS_OBJS_DIR} ${LibSSLRoot})
|
||||
ModuleLibIncs=(${LibSTRoot} ${LibSrtpRoot} ${LibFfmpegRoot} ${SRS_OBJS_DIR} ${LibSSLRoot})
|
||||
MODULE_FILES=("srs_app_server" "srs_app_conn" "srs_app_rtmp_conn" "srs_app_source"
|
||||
"srs_app_refer" "srs_app_hls" "srs_app_forward" "srs_app_encoder" "srs_app_http_stream"
|
||||
"srs_app_thread" "srs_app_bandwidth" "srs_app_st" "srs_app_log" "srs_app_config"
|
||||
|
|
@ -259,7 +261,7 @@ if [ $SRS_EXPORT_LIBRTMP_PROJECT = NO ]; then
|
|||
"srs_app_mpegts_udp" "srs_app_rtp" "srs_app_rtc_conn" "srs_app_dtls" "srs_app_rtsp" "srs_app_listener" "srs_app_async_call"
|
||||
"srs_app_caster_flv" "srs_app_process" "srs_app_ng_exec"
|
||||
"srs_app_hourglass" "srs_app_dash" "srs_app_fragment" "srs_app_dvr"
|
||||
"srs_app_coworkers" "srs_app_hybrid")
|
||||
"srs_app_coworkers" "srs_app_hybrid" "srs_app_audio_recode")
|
||||
DEFINES=""
|
||||
# add each modules for app
|
||||
for SRS_MODULE in ${SRS_MODULES[*]}; do
|
||||
|
|
@ -286,7 +288,7 @@ if [ $SRS_EXPORT_LIBRTMP_PROJECT = NO ]; then
|
|||
if [[ $SRS_SRT == YES ]]; then
|
||||
MODULE_DEPENDS+=("SRT")
|
||||
fi
|
||||
ModuleLibIncs=(${LibSTRoot} ${LibSrtpRoot} ${SRS_OBJS_DIR} ${LibGperfRoot} ${LibSSLRoot})
|
||||
ModuleLibIncs=(${LibSTRoot} ${LibSrtpRoot} ${LibFfmpegRoot} ${SRS_OBJS_DIR} ${LibGperfRoot} ${LibSSLRoot})
|
||||
if [[ $SRS_SRT == YES ]]; then
|
||||
ModuleLibIncs+=("${LibSRTRoot[*]}")
|
||||
fi
|
||||
|
|
@ -299,7 +301,7 @@ fi
|
|||
if [ $SRS_EXPORT_LIBRTMP_PROJECT = NO ]; then
|
||||
MODULE_ID="MAIN"
|
||||
MODULE_DEPENDS=("CORE" "KERNEL" "PROTOCOL" "SERVICE")
|
||||
ModuleLibIncs=(${LibSTRoot} ${LibSrtpRoot} ${SRS_OBJS_DIR} ${LibGperfRoot} ${LibSSLRoot})
|
||||
ModuleLibIncs=(${LibSTRoot} ${LibSrtpRoot} ${LibFfmpegRoot} ${SRS_OBJS_DIR} ${LibGperfRoot} ${LibSSLRoot})
|
||||
MODULE_FILES=()
|
||||
DEFINES=""
|
||||
# add each modules for main
|
||||
|
|
@ -326,13 +328,13 @@ if [ $SRS_EXPORT_LIBRTMP_PROJECT = NO ]; then
|
|||
done
|
||||
#
|
||||
# all depends libraries
|
||||
ModuleLibFiles=(${LibSTfile} ${LibSrtpFile} ${LibSSLfile} ${LibGperfFile})
|
||||
ModuleLibFiles=(${LibSTfile} ${LibSrtpFile} ${LibFfmpegFile} ${LibSSLfile} ${LibGperfFile})
|
||||
if [[ $SRS_SRT == YES ]]; then
|
||||
ModuleLibFiles+=("${LibSRTfile[*]}")
|
||||
fi
|
||||
# all depends objects
|
||||
MODULE_OBJS="${CORE_OBJS[@]} ${KERNEL_OBJS[@]} ${PROTOCOL_OBJS[@]} ${SERVICE_OBJS[@]} ${APP_OBJS[@]} ${SERVER_OBJS[@]}"
|
||||
ModuleLibIncs=(${LibSTRoot} ${LibSrtpRoot} ${SRS_OBJS_DIR} ${LibGperfRoot} ${LibSSLRoot})
|
||||
ModuleLibIncs=(${LibSTRoot} ${LibSrtpRoot} ${LibFfmpegRoot} ${SRS_OBJS_DIR} ${LibGperfRoot} ${LibSSLRoot})
|
||||
if [[ $SRS_SRT == YES ]]; then
|
||||
MODULE_OBJS="${MODULE_OBJS} ${SRT_OBJS[@]}"
|
||||
fi
|
||||
|
|
@ -343,7 +345,7 @@ if [ $SRS_EXPORT_LIBRTMP_PROJECT = NO ]; then
|
|||
#
|
||||
# For modules, without the app module.
|
||||
MODULE_OBJS="${CORE_OBJS[@]} ${KERNEL_OBJS[@]} ${PROTOCOL_OBJS[@]} ${SERVICE_OBJS[@]} ${MAIN_OBJS[@]}"
|
||||
ModuleLibFiles=(${LibSTfile} ${LibSrtpFile} ${LibSSLfile} ${LibGperfFile})
|
||||
ModuleLibFiles=(${LibSTfile} ${LibSrtpFile} ${LibFfmpegFile} ${LibSSLfile} ${LibGperfFile})
|
||||
#
|
||||
for SRS_MODULE in ${SRS_MODULES[*]}; do
|
||||
. $SRS_MODULE/config
|
||||
|
|
@ -363,11 +365,11 @@ if [ $SRS_UTEST = YES ]; then
|
|||
MODULE_FILES=("srs_utest" "srs_utest_amf0" "srs_utest_protocol" "srs_utest_kernel" "srs_utest_core"
|
||||
"srs_utest_config" "srs_utest_rtmp" "srs_utest_http" "srs_utest_avc" "srs_utest_reload"
|
||||
"srs_utest_mp4" "srs_utest_service" "srs_utest_app")
|
||||
ModuleLibIncs=(${SRS_OBJS_DIR} ${LibSTRoot} ${LibSrtpRoot} ${LibSSLRoot})
|
||||
ModuleLibIncs=(${SRS_OBJS_DIR} ${LibSTRoot} ${LibSrtpRoot} ${LibFfmpegRoot} ${LibSSLRoot})
|
||||
if [[ $SRS_SRT == YES ]]; then
|
||||
ModuleLibIncs+=("${LibSRTRoot[*]}")
|
||||
fi
|
||||
ModuleLibFiles=(${LibSTfile} ${LibSrtpFile} ${LibSSLfile})
|
||||
ModuleLibFiles=(${LibSTfile} ${LibSrtpFile} ${LibFfmpegFile} ${LibSSLfile})
|
||||
if [[ $SRS_SRT == YES ]]; then
|
||||
ModuleLibFiles+=("${LibSRTfile[*]}")
|
||||
fi
|
||||
|
|
|
|||
468
trunk/src/app/srs_app_audio_recode.cpp
Normal file
468
trunk/src/app/srs_app_audio_recode.cpp
Normal file
|
|
@ -0,0 +1,468 @@
|
|||
|
||||
/**
|
||||
* The MIT License (MIT)
|
||||
*
|
||||
* Copyright (c) 2013-2020 Winlin
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
* this software and associated documentation files (the "Software"), to deal in
|
||||
* the Software without restriction, including without limitation the rights to
|
||||
* use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
||||
* the Software, and to permit persons to whom the Software is furnished to do so,
|
||||
* subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
||||
* IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
||||
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include <srs_kernel_codec.hpp>
|
||||
#include <srs_kernel_error.hpp>
|
||||
#include <srs_app_audio_recode.hpp>
|
||||
|
||||
static const int kOpusPacketMs = 20;
|
||||
static const int kOpusMaxbytes = 8000;
|
||||
static const int kFrameBufMax = 40960;
|
||||
static const int kPacketBufMax = 8192;
|
||||
static const int kPcmBufMax = 4096*4;
|
||||
|
||||
SrsAudioDecoder::SrsAudioDecoder(std::string codec)
|
||||
: codec_name_(codec)
|
||||
{
|
||||
frame_ = NULL;
|
||||
packet_ = NULL;
|
||||
codec_ctx_ = NULL;
|
||||
}
|
||||
|
||||
SrsAudioDecoder::~SrsAudioDecoder()
|
||||
{
|
||||
if (codec_ctx_) {
|
||||
avcodec_free_context(&codec_ctx_);
|
||||
codec_ctx_ = NULL;
|
||||
}
|
||||
if (frame_) {
|
||||
av_frame_free(&frame_);
|
||||
frame_ = NULL;
|
||||
}
|
||||
if (packet_) {
|
||||
av_packet_free(&packet_);
|
||||
packet_ = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
srs_error_t SrsAudioDecoder::initialize()
|
||||
{
|
||||
srs_error_t err = srs_success;
|
||||
|
||||
if (codec_name_.compare("aac")) {
|
||||
return srs_error_wrap(err, "Invalid codec name");
|
||||
}
|
||||
|
||||
const AVCodec *codec = avcodec_find_decoder_by_name(codec_name_.c_str());
|
||||
if (!codec) {
|
||||
return srs_error_wrap(err, "Codec not found by name");
|
||||
}
|
||||
|
||||
codec_ctx_ = avcodec_alloc_context3(codec);
|
||||
if (!codec_ctx_) {
|
||||
return srs_error_wrap(err, "Could not allocate audio codec context");
|
||||
}
|
||||
|
||||
if (avcodec_open2(codec_ctx_, codec, NULL) < 0) {
|
||||
return srs_error_wrap(err, "Could not open codec");
|
||||
}
|
||||
|
||||
frame_ = av_frame_alloc();
|
||||
if (!frame_) {
|
||||
return srs_error_wrap(err, "Could not allocate audio frame");
|
||||
}
|
||||
|
||||
packet_ = av_packet_alloc();
|
||||
if (!packet_) {
|
||||
return srs_error_wrap(err, "Could not allocate audio packet");
|
||||
}
|
||||
|
||||
return err;
|
||||
}
|
||||
|
||||
srs_error_t SrsAudioDecoder::decode(SrsSample *pkt, char *buf, int &size)
|
||||
{
|
||||
srs_error_t err = srs_success;
|
||||
|
||||
packet_->data = (uint8_t *)pkt->bytes;
|
||||
packet_->size = pkt->size;
|
||||
|
||||
int ret = avcodec_send_packet(codec_ctx_, packet_);
|
||||
if (ret < 0) {
|
||||
return srs_error_wrap(err, "Error submitting the packet to the decoder");
|
||||
}
|
||||
|
||||
int max = size;
|
||||
size = 0;
|
||||
|
||||
while (ret >= 0) {
|
||||
ret = avcodec_receive_frame(codec_ctx_, frame_);
|
||||
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
|
||||
return err;
|
||||
} else if (ret < 0) {
|
||||
return srs_error_wrap(err, "Error during decoding");
|
||||
}
|
||||
|
||||
int pcm_size = av_get_bytes_per_sample(codec_ctx_->sample_fmt);
|
||||
if (pcm_size < 0) {
|
||||
return srs_error_wrap(err, "Failed to calculate data size");
|
||||
}
|
||||
|
||||
for (int i = 0; i < frame_->nb_samples; i++) {
|
||||
if (size + pcm_size * codec_ctx_->channels <= max) {
|
||||
memcpy(buf + size,frame_->data[0] + pcm_size*codec_ctx_->channels * i, pcm_size * codec_ctx_->channels);
|
||||
size += pcm_size * codec_ctx_->channels;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return err;
|
||||
}
|
||||
|
||||
AVCodecContext* SrsAudioDecoder::codec_ctx()
|
||||
{
|
||||
return codec_ctx_;
|
||||
}
|
||||
|
||||
SrsAudioEncoder::SrsAudioEncoder(int samplerate, int channels, int fec, int complexity)
|
||||
: inband_fec_(fec),
|
||||
channels_(channels),
|
||||
sampling_rate_(samplerate),
|
||||
complexity_(complexity)
|
||||
{
|
||||
opus_ = NULL;
|
||||
}
|
||||
|
||||
SrsAudioEncoder::~SrsAudioEncoder()
|
||||
{
|
||||
if (opus_) {
|
||||
opus_encoder_destroy(opus_);
|
||||
opus_ = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
srs_error_t SrsAudioEncoder::initialize()
|
||||
{
|
||||
srs_error_t err = srs_success;
|
||||
|
||||
int error = 0;
|
||||
opus_ = opus_encoder_create(sampling_rate_, channels_, OPUS_APPLICATION_VOIP, &error);
|
||||
if (error != OPUS_OK) {
|
||||
return srs_error_wrap(err, "Error create Opus encoder");
|
||||
}
|
||||
|
||||
switch (sampling_rate_)
|
||||
{
|
||||
case 48000:
|
||||
opus_encoder_ctl(opus_, OPUS_SET_MAX_BANDWIDTH(OPUS_BANDWIDTH_FULLBAND));
|
||||
break;
|
||||
|
||||
case 24000:
|
||||
opus_encoder_ctl(opus_, OPUS_SET_MAX_BANDWIDTH(OPUS_BANDWIDTH_SUPERWIDEBAND));
|
||||
|
||||
case 16000:
|
||||
opus_encoder_ctl(opus_, OPUS_SET_MAX_BANDWIDTH(OPUS_BANDWIDTH_WIDEBAND));
|
||||
break;
|
||||
|
||||
case 12000:
|
||||
opus_encoder_ctl(opus_, OPUS_SET_MAX_BANDWIDTH(OPUS_BANDWIDTH_MEDIUMBAND));
|
||||
break;
|
||||
|
||||
case 8000:
|
||||
opus_encoder_ctl(opus_, OPUS_SET_MAX_BANDWIDTH(OPUS_BANDWIDTH_NARROWBAND));
|
||||
break;
|
||||
|
||||
default:
|
||||
sampling_rate_ = 16000;
|
||||
opus_encoder_ctl(opus_, OPUS_SET_MAX_BANDWIDTH(OPUS_BANDWIDTH_WIDEBAND));
|
||||
break;
|
||||
}
|
||||
opus_encoder_ctl(opus_, OPUS_SET_INBAND_FEC(inband_fec_));
|
||||
opus_encoder_ctl(opus_, OPUS_SET_COMPLEXITY(complexity_));
|
||||
|
||||
return err;
|
||||
}
|
||||
|
||||
srs_error_t SrsAudioEncoder::encode(SrsSample *frame, char *buf, int &size)
|
||||
{
|
||||
srs_error_t err = srs_success;
|
||||
|
||||
int nb_samples = sampling_rate_ * kOpusPacketMs / 1000;
|
||||
if (frame->size != nb_samples * 2 * channels_) {
|
||||
return srs_error_wrap(err, "invalid frame size %d, should be %d", frame->size, nb_samples * 2 * channels_);
|
||||
}
|
||||
|
||||
opus_int16 *data = (opus_int16 *)frame->bytes;
|
||||
size = opus_encode(opus_, data, nb_samples, (unsigned char *)buf, kOpusMaxbytes);
|
||||
|
||||
return err;
|
||||
}
|
||||
|
||||
SrsAudioResample::SrsAudioResample(int src_rate, int src_layout, enum AVSampleFormat src_fmt,
|
||||
int src_nb, int dst_rate, int dst_layout, enum AVSampleFormat dst_fmt)
|
||||
: src_rate_(src_rate),
|
||||
src_ch_layout_(src_layout),
|
||||
src_sample_fmt_(src_fmt),
|
||||
src_nb_samples_(src_nb),
|
||||
dst_rate_(dst_rate),
|
||||
dst_ch_layout_(dst_layout),
|
||||
dst_sample_fmt_(dst_fmt)
|
||||
{
|
||||
src_nb_channels_ = 0;
|
||||
dst_nb_channels_ = 0;
|
||||
src_linesize_ = 0;
|
||||
dst_linesize_ = 0;
|
||||
dst_nb_samples_ = 0;
|
||||
src_data_ = NULL;
|
||||
dst_data_ = 0;
|
||||
|
||||
max_dst_nb_samples_ = 0;
|
||||
swr_ctx_ = NULL;
|
||||
}
|
||||
|
||||
SrsAudioResample::~SrsAudioResample()
|
||||
{
|
||||
if (src_data_) {
|
||||
av_freep(&src_data_[0]);
|
||||
av_freep(&src_data_);
|
||||
src_data_ = NULL;
|
||||
}
|
||||
if (dst_data_) {
|
||||
av_freep(&dst_data_[0]);
|
||||
av_freep(&dst_data_);
|
||||
dst_data_ = NULL;
|
||||
}
|
||||
if (swr_ctx_) {
|
||||
swr_free(&swr_ctx_);
|
||||
swr_ctx_ = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
srs_error_t SrsAudioResample::initialize()
|
||||
{
|
||||
srs_error_t err = srs_success;
|
||||
|
||||
swr_ctx_ = swr_alloc();
|
||||
if (!swr_ctx_) {
|
||||
return srs_error_wrap(err, "Could not allocate resampler context");
|
||||
}
|
||||
|
||||
av_opt_set_int(swr_ctx_, "in_channel_layout", src_ch_layout_, 0);
|
||||
av_opt_set_int(swr_ctx_, "in_sample_rate", src_rate_, 0);
|
||||
av_opt_set_sample_fmt(swr_ctx_, "in_sample_fmt", src_sample_fmt_, 0);
|
||||
|
||||
av_opt_set_int(swr_ctx_, "out_channel_layout", dst_ch_layout_, 0);
|
||||
av_opt_set_int(swr_ctx_, "out_sample_rate", dst_rate_, 0);
|
||||
av_opt_set_sample_fmt(swr_ctx_, "out_sample_fmt", dst_sample_fmt_, 0);
|
||||
|
||||
int ret;
|
||||
if ((ret = swr_init(swr_ctx_)) < 0) {
|
||||
return srs_error_wrap(err, "Failed to initialize the resampling context");
|
||||
}
|
||||
|
||||
src_nb_channels_ = av_get_channel_layout_nb_channels(src_ch_layout_);
|
||||
ret = av_samples_alloc_array_and_samples(&src_data_, &src_linesize_, src_nb_channels_,
|
||||
src_nb_samples_, src_sample_fmt_, 0);
|
||||
if (ret < 0) {
|
||||
return srs_error_wrap(err, "Could not allocate source samples");
|
||||
}
|
||||
|
||||
max_dst_nb_samples_ = dst_nb_samples_ =
|
||||
av_rescale_rnd(src_nb_samples_, dst_rate_, src_rate_, AV_ROUND_UP);
|
||||
|
||||
dst_nb_channels_ = av_get_channel_layout_nb_channels(dst_ch_layout_);
|
||||
ret = av_samples_alloc_array_and_samples(&dst_data_, &dst_linesize_, dst_nb_channels_,
|
||||
dst_nb_samples_, dst_sample_fmt_, 0);
|
||||
if (ret < 0) {
|
||||
return srs_error_wrap(err, "Could not allocate destination samples");
|
||||
}
|
||||
|
||||
return err;
|
||||
}
|
||||
|
||||
srs_error_t SrsAudioResample::resample(SrsSample *pcm, char *buf, int &size)
|
||||
{
|
||||
srs_error_t err = srs_success;
|
||||
|
||||
int ret, plane = 1;
|
||||
if (src_sample_fmt_ == AV_SAMPLE_FMT_FLTP) {
|
||||
plane = 2;
|
||||
}
|
||||
if (src_linesize_ * plane < pcm->size || pcm->size < 0) {
|
||||
return srs_error_wrap(err, "size not ok");
|
||||
}
|
||||
memcpy(src_data_[0], pcm->bytes, pcm->size);
|
||||
|
||||
dst_nb_samples_ = av_rescale_rnd(swr_get_delay(swr_ctx_, src_rate_) +
|
||||
src_nb_samples_, dst_rate_, src_rate_, AV_ROUND_UP);
|
||||
if (dst_nb_samples_ > max_dst_nb_samples_) {
|
||||
av_freep(&dst_data_[0]);
|
||||
ret = av_samples_alloc(dst_data_, &dst_linesize_, dst_nb_channels_,
|
||||
dst_nb_samples_, dst_sample_fmt_, 1);
|
||||
if (ret < 0) {
|
||||
return srs_error_wrap(err, "alloc error");
|
||||
}
|
||||
max_dst_nb_samples_ = dst_nb_samples_;
|
||||
}
|
||||
|
||||
ret = swr_convert(swr_ctx_, dst_data_, dst_nb_samples_, (const uint8_t **)src_data_, src_nb_samples_);
|
||||
if (ret < 0) {
|
||||
return srs_error_wrap(err, "Error while converting");
|
||||
}
|
||||
|
||||
int dst_bufsize = av_samples_get_buffer_size(&dst_linesize_, dst_nb_channels_,
|
||||
ret, dst_sample_fmt_, 1);
|
||||
if (dst_bufsize < 0) {
|
||||
return srs_error_wrap(err, "Could not get sample buffer size");
|
||||
}
|
||||
|
||||
int max = size;
|
||||
size = 0;
|
||||
if (max > dst_bufsize) {
|
||||
memcpy(buf, dst_data_[0], dst_bufsize);
|
||||
size = dst_bufsize;
|
||||
}
|
||||
|
||||
return err;
|
||||
}
|
||||
|
||||
SrsAudioRecode::SrsAudioRecode(int channels, int samplerate)
|
||||
: dst_channels_(channels),
|
||||
dst_samplerate_(samplerate)
|
||||
{
|
||||
size_ = 0;
|
||||
data_ = new char[kPcmBufMax];
|
||||
}
|
||||
|
||||
SrsAudioRecode::~SrsAudioRecode()
|
||||
{
|
||||
if (dec_) {
|
||||
delete dec_;
|
||||
dec_ = NULL;
|
||||
}
|
||||
if (enc_) {
|
||||
delete enc_;
|
||||
enc_ = NULL;
|
||||
}
|
||||
if (resample_) {
|
||||
delete resample_;
|
||||
resample_ = NULL;
|
||||
}
|
||||
|
||||
delete[] data_;
|
||||
}
|
||||
|
||||
srs_error_t SrsAudioRecode::initialize()
|
||||
{
|
||||
srs_error_t err = srs_success;
|
||||
|
||||
dec_ = new SrsAudioDecoder("aac");
|
||||
if (!dec_) {
|
||||
return srs_error_wrap(err, "SrsAudioDecoder failed");
|
||||
}
|
||||
dec_->initialize();
|
||||
|
||||
enc_ = new SrsAudioEncoder(dst_samplerate_, dst_channels_, 1, 1);
|
||||
if (!enc_) {
|
||||
return srs_error_wrap(err, "SrsAudioEncoder failed");
|
||||
}
|
||||
enc_->initialize();
|
||||
|
||||
resample_ = NULL;
|
||||
|
||||
return err;
|
||||
}
|
||||
|
||||
srs_error_t SrsAudioRecode::recode(SrsSample *pkt, char **buf, int *buf_len, int &n)
|
||||
{
|
||||
srs_error_t err = srs_success;
|
||||
|
||||
static char decode_buffer[kPacketBufMax];
|
||||
static char resample_buffer[kFrameBufMax];
|
||||
static char encode_buffer[kPacketBufMax];
|
||||
|
||||
if (!dec_) {
|
||||
return srs_error_wrap(err, "dec_ nullptr");
|
||||
}
|
||||
|
||||
int decode_len = kPacketBufMax;
|
||||
if ((err = dec_->decode(pkt, decode_buffer, decode_len)) != srs_success) {
|
||||
return srs_error_wrap(err, "decode error");
|
||||
}
|
||||
|
||||
if (!resample_) {
|
||||
int channel_layout = av_get_default_channel_layout(dst_channels_);
|
||||
AVCodecContext *codec_ctx = dec_->codec_ctx();
|
||||
resample_ = new SrsAudioResample(codec_ctx->sample_rate, (int)codec_ctx->channel_layout, \
|
||||
codec_ctx->sample_fmt, codec_ctx->frame_size, dst_samplerate_, channel_layout, \
|
||||
AV_SAMPLE_FMT_S16);
|
||||
|
||||
if (!resample_) {
|
||||
return srs_error_wrap(err, "SrsAudioResample failed");
|
||||
}
|
||||
resample_->initialize();
|
||||
}
|
||||
|
||||
SrsSample pcm;
|
||||
pcm.bytes = decode_buffer;
|
||||
pcm.size = decode_len;
|
||||
int resample_len = kFrameBufMax;
|
||||
if ((err = resample_->resample(&pcm, resample_buffer, resample_len)) != srs_success) {
|
||||
return srs_error_wrap(err, "decode error");
|
||||
}
|
||||
|
||||
n = 0;
|
||||
int data_left = resample_len;
|
||||
int total;
|
||||
total = (dst_samplerate_ * kOpusPacketMs / 1000) * 2 * dst_channels_;
|
||||
|
||||
if (size_ + data_left < total) {
|
||||
memcpy(data_ + size_, resample_buffer, data_left);
|
||||
size_ += data_left;
|
||||
} else {
|
||||
int index = 0;
|
||||
while (1) {
|
||||
data_left = data_left - (total - size_);
|
||||
memcpy(data_ + size_, resample_buffer + index, total - size_);
|
||||
index += total - size_;
|
||||
size_ += total - size_;
|
||||
if (!enc_) {
|
||||
return srs_error_wrap(err, "enc_ nullptr");
|
||||
}
|
||||
|
||||
int encode_len;
|
||||
pcm.bytes = (char *)data_;
|
||||
pcm.size = size_;
|
||||
if ((err = enc_->encode(&pcm, encode_buffer, encode_len)) != srs_success) {
|
||||
return srs_error_wrap(err, "decode error");
|
||||
}
|
||||
|
||||
memcpy(buf[n], encode_buffer, encode_len);
|
||||
buf_len[n] = encode_len;
|
||||
n++;
|
||||
|
||||
size_ = 0;
|
||||
if(!data_left)
|
||||
break;
|
||||
|
||||
if(data_left < total) {
|
||||
memcpy(data_ + size_, resample_buffer + index, data_left);
|
||||
size_ += data_left;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return err;
|
||||
}
|
||||
126
trunk/src/app/srs_app_audio_recode.hpp
Normal file
126
trunk/src/app/srs_app_audio_recode.hpp
Normal file
|
|
@ -0,0 +1,126 @@
|
|||
/**
|
||||
* The MIT License (MIT)
|
||||
*
|
||||
* Copyright (c) 2013-2020 Winlin
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
* this software and associated documentation files (the "Software"), to deal in
|
||||
* the Software without restriction, including without limitation the rights to
|
||||
* use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
||||
* the Software, and to permit persons to whom the Software is furnished to do so,
|
||||
* subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
||||
* IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
||||
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#ifndef SRS_APP_AUDIO_RECODE_HPP
|
||||
#define SRS_APP_AUDIO_RECODE_HPP
|
||||
|
||||
#include <string>
|
||||
#include <srs_core.hpp>
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include <libavutil/frame.h>
|
||||
#include <libavutil/mem.h>
|
||||
#include <libavcodec/avcodec.h>
|
||||
#include <libavutil/opt.h>
|
||||
#include <libavutil/channel_layout.h>
|
||||
#include <libavutil/samplefmt.h>
|
||||
#include <libswresample/swresample.h>
|
||||
|
||||
#include <opus/opus.h>
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
class SrsSample;
|
||||
|
||||
class SrsAudioDecoder
|
||||
{
|
||||
private:
|
||||
AVFrame* frame_;
|
||||
AVPacket* packet_;
|
||||
AVCodecContext* codec_ctx_;
|
||||
std::string codec_name_;
|
||||
public:
|
||||
SrsAudioDecoder(std::string codec);
|
||||
virtual ~SrsAudioDecoder();
|
||||
srs_error_t initialize();
|
||||
virtual srs_error_t decode(SrsSample *pkt, char *buf, int &size);
|
||||
AVCodecContext* codec_ctx();
|
||||
};
|
||||
|
||||
class SrsAudioEncoder
|
||||
{
|
||||
private:
|
||||
int inband_fec_;
|
||||
int channels_;
|
||||
int sampling_rate_;
|
||||
int complexity_;
|
||||
OpusEncoder *opus_;
|
||||
public:
|
||||
SrsAudioEncoder(int samplerate, int channels, int fec, int complexity);
|
||||
virtual ~SrsAudioEncoder();
|
||||
srs_error_t initialize();
|
||||
virtual srs_error_t encode(SrsSample *frame, char *buf, int &size);
|
||||
};
|
||||
|
||||
class SrsAudioResample
|
||||
{
|
||||
private:
|
||||
int src_rate_;
|
||||
int src_ch_layout_;
|
||||
int src_nb_channels_;
|
||||
enum AVSampleFormat src_sample_fmt_;
|
||||
int src_linesize_;
|
||||
int src_nb_samples_;
|
||||
uint8_t **src_data_;
|
||||
|
||||
int dst_rate_;
|
||||
int dst_ch_layout_;
|
||||
int dst_nb_channels_;
|
||||
enum AVSampleFormat dst_sample_fmt_;
|
||||
int dst_linesize_;
|
||||
int dst_nb_samples_;
|
||||
uint8_t **dst_data_;
|
||||
|
||||
int max_dst_nb_samples_;
|
||||
struct SwrContext *swr_ctx_;
|
||||
public:
|
||||
SrsAudioResample(int src_rate, int src_layout, enum AVSampleFormat src_fmt,
|
||||
int src_nb, int dst_rate, int dst_layout, enum AVSampleFormat dst_fmt);
|
||||
virtual ~SrsAudioResample();
|
||||
srs_error_t initialize();
|
||||
virtual srs_error_t resample(SrsSample *pcm, char *buf, int &size);
|
||||
};
|
||||
|
||||
class SrsAudioRecode
|
||||
{
|
||||
private:
|
||||
SrsAudioDecoder *dec_;
|
||||
SrsAudioEncoder *enc_;
|
||||
SrsAudioResample *resample_;
|
||||
int dst_channels_;
|
||||
int dst_samplerate_;
|
||||
int size_;
|
||||
char *data_;
|
||||
public:
|
||||
SrsAudioRecode(int channels, int samplerate);
|
||||
virtual ~SrsAudioRecode();
|
||||
srs_error_t initialize();
|
||||
virtual srs_error_t recode(SrsSample *pkt, char **buf, int *buf_len, int &n);
|
||||
};
|
||||
|
||||
#endif /* SRS_APP_AUDIO_RECODE_HPP */
|
||||
|
|
@ -50,6 +50,46 @@ using namespace std;
|
|||
#include <srs_app_http_hooks.hpp>
|
||||
#include <srs_protocol_format.hpp>
|
||||
#include <openssl/rand.h>
|
||||
#include <srs_app_audio_recode.hpp>
|
||||
|
||||
// TODO: Add this function into SrsRtpMux class.
|
||||
srs_error_t aac_raw_append_adts_header(SrsSharedPtrMessage* shared_audio, SrsFormat* format, SrsBuffer** stream_ptr)
|
||||
{
|
||||
srs_error_t err = srs_success;
|
||||
|
||||
if (format->is_aac_sequence_header()) {
|
||||
return err;
|
||||
}
|
||||
|
||||
if (stream_ptr == NULL) {
|
||||
return srs_error_new(ERROR_RTC_RTP_MUXER, "adts");
|
||||
}
|
||||
|
||||
srs_verbose("audio samples=%d", format->audio->nb_samples);
|
||||
|
||||
if (format->audio->nb_samples != 1) {
|
||||
return srs_error_new(ERROR_RTC_RTP_MUXER, "adts");
|
||||
}
|
||||
|
||||
int nb_buf = format->audio->samples[0].size + 7;
|
||||
char* buf = new char[nb_buf];
|
||||
SrsBuffer* stream = new SrsBuffer(buf, nb_buf);
|
||||
|
||||
// TODO: Add comment.
|
||||
stream->write_1bytes(0xFF);
|
||||
stream->write_1bytes(0xF9);
|
||||
stream->write_1bytes(((format->acodec->aac_object - 1) << 6) | ((format->acodec->aac_sample_rate & 0x0F) << 2) | ((format->acodec->aac_channels & 0x04) >> 2));
|
||||
stream->write_1bytes(((format->acodec->aac_channels & 0x03) << 6) | ((nb_buf >> 11) & 0x03));
|
||||
stream->write_1bytes((nb_buf >> 3) & 0xFF);
|
||||
stream->write_1bytes(((nb_buf & 0x07) << 5) | 0x1F);
|
||||
stream->write_1bytes(0xFC);
|
||||
|
||||
stream->write_bytes(format->audio->samples[0].bytes, format->audio->samples[0].size);
|
||||
|
||||
*stream_ptr = stream;
|
||||
|
||||
return err;
|
||||
}
|
||||
|
||||
SrsRtpMuxer::SrsRtpMuxer()
|
||||
{
|
||||
|
|
@ -288,6 +328,104 @@ srs_error_t SrsRtpMuxer::packet_stap_a(const string &sps, const string& pps, Srs
|
|||
return err;
|
||||
}
|
||||
|
||||
SrsRtpOpusMuxer::SrsRtpOpusMuxer()
|
||||
{
|
||||
sequence = 0;
|
||||
timestamp = 0;
|
||||
recoder = NULL;
|
||||
}
|
||||
|
||||
SrsRtpOpusMuxer::~SrsRtpOpusMuxer()
|
||||
{
|
||||
if (recoder) {
|
||||
delete recoder;
|
||||
recoder = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
srs_error_t SrsRtpOpusMuxer::initialize()
|
||||
{
|
||||
srs_error_t err = srs_success;
|
||||
|
||||
recoder = new SrsAudioRecode(kChannel, kSamplerate);
|
||||
if (!recoder) {
|
||||
return srs_error_wrap(err, "SrsAacOpus init failed");
|
||||
}
|
||||
recoder->initialize();
|
||||
|
||||
return err;
|
||||
}
|
||||
|
||||
srs_error_t SrsRtpOpusMuxer::frame_to_packet(SrsSharedPtrMessage* shared_audio, SrsFormat* format, SrsBuffer* stream)
|
||||
{
|
||||
srs_error_t err = srs_success;
|
||||
|
||||
vector<SrsRtpSharedPacket*> rtp_packet_vec;
|
||||
|
||||
char* data_ptr[kArrayLength];
|
||||
static char data_array[kArrayLength][kArrayBuffer];
|
||||
int elen[kArrayLength], number = 0;
|
||||
|
||||
data_ptr[0] = &data_array[0][0];
|
||||
for (int i = 1; i < kArrayLength; i++) {
|
||||
data_ptr[i] = data_array[i];
|
||||
}
|
||||
|
||||
SrsSample pkt;
|
||||
pkt.bytes = stream->data();
|
||||
pkt.size = stream->pos();
|
||||
|
||||
if ((err = recoder->recode(&pkt, data_ptr, elen, number)) != srs_success) {
|
||||
return srs_error_wrap(err, "recode error");
|
||||
}
|
||||
|
||||
for (int i = 0; i < number; i++) {
|
||||
SrsSample sample;
|
||||
sample.size = elen[i];
|
||||
sample.bytes = data_ptr[i];
|
||||
packet_opus(shared_audio, &sample, rtp_packet_vec);
|
||||
}
|
||||
|
||||
shared_audio->set_rtp_packets(rtp_packet_vec);
|
||||
|
||||
return err;
|
||||
}
|
||||
|
||||
srs_error_t SrsRtpOpusMuxer::packet_opus(SrsSharedPtrMessage* shared_frame, SrsSample* sample, std::vector<SrsRtpSharedPacket*>& rtp_packet_vec)
|
||||
{
|
||||
srs_error_t err = srs_success;
|
||||
|
||||
char* buf = new char[kRtpPacketSize];
|
||||
SrsBuffer* stream = new SrsBuffer(buf, kRtpPacketSize);
|
||||
SrsAutoFree(SrsBuffer, stream);
|
||||
|
||||
// v=2,p=0,x=0,cc=0
|
||||
stream->write_1bytes(0x80);
|
||||
// marker payloadtype
|
||||
stream->write_1bytes(kOpusPayloadType);
|
||||
// sequenct
|
||||
stream->write_2bytes(sequence);
|
||||
// timestamp
|
||||
stream->write_4bytes(int32_t(timestamp));
|
||||
timestamp += 960;
|
||||
// ssrc
|
||||
stream->write_4bytes(int32_t(kAudioSSRC));
|
||||
|
||||
stream->write_bytes(sample->bytes, sample->size);
|
||||
|
||||
srs_verbose("sample=%s", srs_string_dumps_hex(sample->bytes, sample->size).c_str());
|
||||
srs_verbose("opus, size=%u, seq=%u, timestamp=%lu, ssrc=%u, payloadtype=%u",
|
||||
sample->size, sequence, timestamp, kAudioSSRC, kOpusPayloadType);
|
||||
|
||||
SrsRtpSharedPacket* rtp_shared_pkt = new SrsRtpSharedPacket();
|
||||
rtp_shared_pkt->create(timestamp, sequence++, kAudioSSRC, kOpusPayloadType, stream->data(), stream->pos());
|
||||
rtp_shared_pkt->set_marker(true);
|
||||
|
||||
rtp_packet_vec.push_back(rtp_shared_pkt);
|
||||
|
||||
return err;
|
||||
}
|
||||
|
||||
SrsRtp::SrsRtp()
|
||||
{
|
||||
req = NULL;
|
||||
|
|
@ -326,6 +464,11 @@ srs_error_t SrsRtp::initialize(SrsOriginHub* h, SrsRequest* r)
|
|||
req = r;
|
||||
|
||||
rtp_h264_muxer = new SrsRtpMuxer();
|
||||
|
||||
rtp_opus_muxer = new SrsRtpOpusMuxer();
|
||||
if (rtp_opus_muxer) {
|
||||
rtp_opus_muxer->initialize();
|
||||
}
|
||||
|
||||
return err;
|
||||
}
|
||||
|
|
@ -387,7 +530,16 @@ srs_error_t SrsRtp::on_audio(SrsSharedPtrMessage* shared_audio, SrsFormat* forma
|
|||
// ignore sequence header
|
||||
srs_assert(format->audio);
|
||||
|
||||
// TODO: rtc no support aac
|
||||
SrsBuffer* stream = NULL;
|
||||
SrsAutoFree(SrsBuffer, stream);
|
||||
if ((err = aac_raw_append_adts_header(shared_audio, format, &stream)) != srs_success) {
|
||||
return srs_error_wrap(err, "aac append header");
|
||||
}
|
||||
|
||||
if (stream) {
|
||||
rtp_opus_muxer->frame_to_packet(shared_audio, format, stream);
|
||||
}
|
||||
|
||||
return err;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -36,10 +36,13 @@ class SrsSharedPtrMessage;
|
|||
class SrsRtpSharedPacket;
|
||||
class SrsRequest;
|
||||
class SrsOriginHub;
|
||||
class SrsAudioRecode;
|
||||
class SrsBuffer;
|
||||
|
||||
const int max_payload_size = 1200;
|
||||
const int kRtpPacketSize = 1500;
|
||||
|
||||
const uint8_t kOpusPayloadType = 111;
|
||||
const uint8_t kH264PayloadType = 102;
|
||||
|
||||
const uint8_t kNalTypeMask = 0x1F;
|
||||
|
|
@ -50,7 +53,13 @@ const uint8_t kFuA = 28;
|
|||
const uint8_t kStart = 0x80;
|
||||
const uint8_t kEnd = 0x40;
|
||||
|
||||
const int kChannel = 2;
|
||||
const int kSamplerate = 48000;
|
||||
const int kArrayLength = 8;
|
||||
const int kArrayBuffer = 4096;
|
||||
|
||||
// FIXME: ssrc can relate to source
|
||||
const uint32_t kAudioSSRC = 3233846890;
|
||||
const uint32_t kVideoSSRC = 3233846889;
|
||||
|
||||
class SrsRtpMuxer
|
||||
|
|
@ -70,6 +79,22 @@ private:
|
|||
srs_error_t packet_stap_a(const std::string &sps, const std::string& pps, SrsSharedPtrMessage* shared_frame, std::vector<SrsRtpSharedPacket*>& rtp_packet_vec);
|
||||
};
|
||||
|
||||
class SrsRtpOpusMuxer
|
||||
{
|
||||
private:
|
||||
uint32_t timestamp;
|
||||
uint16_t sequence;
|
||||
SrsAudioRecode* recoder;
|
||||
public:
|
||||
SrsRtpOpusMuxer();
|
||||
virtual ~SrsRtpOpusMuxer();
|
||||
virtual srs_error_t initialize();
|
||||
public:
|
||||
srs_error_t frame_to_packet(SrsSharedPtrMessage* shared_audio, SrsFormat* format, SrsBuffer* stream);
|
||||
private:
|
||||
srs_error_t packet_opus(SrsSharedPtrMessage* shared_frame, SrsSample* sample, std::vector<SrsRtpSharedPacket*>& rtp_packet_vec);
|
||||
};
|
||||
|
||||
class SrsRtp
|
||||
{
|
||||
private:
|
||||
|
|
@ -78,6 +103,7 @@ private:
|
|||
bool disposable;
|
||||
srs_utime_t last_update_time;
|
||||
SrsRtpMuxer* rtp_h264_muxer;
|
||||
SrsRtpOpusMuxer* rtp_opus_muxer;
|
||||
SrsOriginHub* hub;
|
||||
public:
|
||||
SrsRtp();
|
||||
|
|
|
|||
|
|
@ -2261,6 +2261,11 @@ srs_error_t SrsSource::on_audio_imp(SrsSharedPtrMessage* msg)
|
|||
}
|
||||
}
|
||||
|
||||
// Copy to hub to all utilities.
|
||||
if ((err = hub->on_audio(msg)) != srs_success) {
|
||||
return srs_error_wrap(err, "consume audio");
|
||||
}
|
||||
|
||||
// copy to all consumer
|
||||
if (!drop_for_reduce) {
|
||||
for (int i = 0; i < (int)consumers.size(); i++) {
|
||||
|
|
@ -2271,11 +2276,6 @@ srs_error_t SrsSource::on_audio_imp(SrsSharedPtrMessage* msg)
|
|||
}
|
||||
}
|
||||
|
||||
// Copy to hub to all utilities.
|
||||
if ((err = hub->on_audio(msg)) != srs_success) {
|
||||
return srs_error_wrap(err, "consume audio");
|
||||
}
|
||||
|
||||
// cache the sequence header of aac, or first packet of mp3.
|
||||
// for example, the mp3 is used for hls to write the "right" audio codec.
|
||||
// TODO: FIXME: to refine the stream info system.
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue