Repo created
This commit is contained in:
parent
81b91f4139
commit
f8c34fa5ee
22732 changed files with 4815320 additions and 2 deletions
|
|
@ -0,0 +1,41 @@
|
|||
/*
|
||||
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include "modules/audio_processing/include/aec_dump.h"
|
||||
|
||||
namespace webrtc {
|
||||
InternalAPMConfig::InternalAPMConfig() = default;
|
||||
InternalAPMConfig::InternalAPMConfig(const InternalAPMConfig&) = default;
|
||||
InternalAPMConfig::InternalAPMConfig(InternalAPMConfig&&) = default;
|
||||
InternalAPMConfig& InternalAPMConfig::operator=(const InternalAPMConfig&) =
|
||||
default;
|
||||
|
||||
bool InternalAPMConfig::operator==(const InternalAPMConfig& other) const {
|
||||
return aec_enabled == other.aec_enabled &&
|
||||
aec_delay_agnostic_enabled == other.aec_delay_agnostic_enabled &&
|
||||
aec_drift_compensation_enabled ==
|
||||
other.aec_drift_compensation_enabled &&
|
||||
aec_extended_filter_enabled == other.aec_extended_filter_enabled &&
|
||||
aec_suppression_level == other.aec_suppression_level &&
|
||||
aecm_enabled == other.aecm_enabled &&
|
||||
aecm_comfort_noise_enabled == other.aecm_comfort_noise_enabled &&
|
||||
aecm_routing_mode == other.aecm_routing_mode &&
|
||||
agc_enabled == other.agc_enabled && agc_mode == other.agc_mode &&
|
||||
agc_limiter_enabled == other.agc_limiter_enabled &&
|
||||
hpf_enabled == other.hpf_enabled && ns_enabled == other.ns_enabled &&
|
||||
ns_level == other.ns_level &&
|
||||
transient_suppression_enabled == other.transient_suppression_enabled &&
|
||||
noise_robust_agc_enabled == other.noise_robust_agc_enabled &&
|
||||
pre_amplifier_enabled == other.pre_amplifier_enabled &&
|
||||
pre_amplifier_fixed_gain_factor ==
|
||||
other.pre_amplifier_fixed_gain_factor &&
|
||||
experiments_description == other.experiments_description;
|
||||
}
|
||||
} // namespace webrtc
|
||||
|
|
@ -0,0 +1,116 @@
|
|||
/*
|
||||
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#ifndef MODULES_AUDIO_PROCESSING_INCLUDE_AEC_DUMP_H_
|
||||
#define MODULES_AUDIO_PROCESSING_INCLUDE_AEC_DUMP_H_
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "absl/base/attributes.h"
|
||||
#include "absl/types/optional.h"
|
||||
#include "modules/audio_processing/include/audio_frame_view.h"
|
||||
#include "modules/audio_processing/include/audio_processing.h"
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
// Struct for passing current config from APM without having to
|
||||
// include protobuf headers.
|
||||
struct InternalAPMConfig {
|
||||
InternalAPMConfig();
|
||||
InternalAPMConfig(const InternalAPMConfig&);
|
||||
InternalAPMConfig(InternalAPMConfig&&);
|
||||
|
||||
InternalAPMConfig& operator=(const InternalAPMConfig&);
|
||||
InternalAPMConfig& operator=(InternalAPMConfig&&) = delete;
|
||||
|
||||
bool operator==(const InternalAPMConfig& other) const;
|
||||
|
||||
bool aec_enabled = false;
|
||||
bool aec_delay_agnostic_enabled = false;
|
||||
bool aec_drift_compensation_enabled = false;
|
||||
bool aec_extended_filter_enabled = false;
|
||||
int aec_suppression_level = 0;
|
||||
bool aecm_enabled = false;
|
||||
bool aecm_comfort_noise_enabled = false;
|
||||
int aecm_routing_mode = 0;
|
||||
bool agc_enabled = false;
|
||||
int agc_mode = 0;
|
||||
bool agc_limiter_enabled = false;
|
||||
bool hpf_enabled = false;
|
||||
bool ns_enabled = false;
|
||||
int ns_level = 0;
|
||||
bool transient_suppression_enabled = false;
|
||||
bool noise_robust_agc_enabled = false;
|
||||
bool pre_amplifier_enabled = false;
|
||||
float pre_amplifier_fixed_gain_factor = 1.f;
|
||||
std::string experiments_description = "";
|
||||
};
|
||||
|
||||
// An interface for recording configuration and input/output streams
|
||||
// of the Audio Processing Module. The recordings are called
|
||||
// 'aec-dumps' and are stored in a protobuf format defined in
|
||||
// debug.proto.
|
||||
// The Write* methods are always safe to call concurrently or
|
||||
// otherwise for all implementing subclasses. The intended mode of
|
||||
// operation is to create a protobuf object from the input, and send
|
||||
// it away to be written to file asynchronously.
|
||||
class AecDump {
|
||||
public:
|
||||
struct AudioProcessingState {
|
||||
int delay;
|
||||
int drift;
|
||||
absl::optional<int> applied_input_volume;
|
||||
bool keypress;
|
||||
};
|
||||
|
||||
virtual ~AecDump() = default;
|
||||
|
||||
// Logs Event::Type INIT message.
|
||||
virtual void WriteInitMessage(const ProcessingConfig& api_format,
|
||||
int64_t time_now_ms) = 0;
|
||||
ABSL_DEPRECATED("")
|
||||
void WriteInitMessage(const ProcessingConfig& api_format) {
|
||||
WriteInitMessage(api_format, 0);
|
||||
}
|
||||
|
||||
// Logs Event::Type STREAM message. To log an input/output pair,
|
||||
// call the AddCapture* and AddAudioProcessingState methods followed
|
||||
// by a WriteCaptureStreamMessage call.
|
||||
virtual void AddCaptureStreamInput(
|
||||
const AudioFrameView<const float>& src) = 0;
|
||||
virtual void AddCaptureStreamOutput(
|
||||
const AudioFrameView<const float>& src) = 0;
|
||||
virtual void AddCaptureStreamInput(const int16_t* const data,
|
||||
int num_channels,
|
||||
int samples_per_channel) = 0;
|
||||
virtual void AddCaptureStreamOutput(const int16_t* const data,
|
||||
int num_channels,
|
||||
int samples_per_channel) = 0;
|
||||
virtual void AddAudioProcessingState(const AudioProcessingState& state) = 0;
|
||||
virtual void WriteCaptureStreamMessage() = 0;
|
||||
|
||||
// Logs Event::Type REVERSE_STREAM message.
|
||||
virtual void WriteRenderStreamMessage(const int16_t* const data,
|
||||
int num_channels,
|
||||
int samples_per_channel) = 0;
|
||||
virtual void WriteRenderStreamMessage(
|
||||
const AudioFrameView<const float>& src) = 0;
|
||||
|
||||
virtual void WriteRuntimeSetting(
|
||||
const AudioProcessing::RuntimeSetting& runtime_setting) = 0;
|
||||
|
||||
// Logs Event::Type CONFIG message.
|
||||
virtual void WriteConfig(const InternalAPMConfig& config) = 0;
|
||||
};
|
||||
} // namespace webrtc
|
||||
|
||||
#endif // MODULES_AUDIO_PROCESSING_INCLUDE_AEC_DUMP_H_
|
||||
|
|
@ -0,0 +1,66 @@
|
|||
/*
|
||||
* Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include "modules/audio_processing/include/audio_frame_proxies.h"
|
||||
|
||||
#include "api/audio/audio_frame.h"
|
||||
#include "modules/audio_processing/include/audio_processing.h"
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
int ProcessAudioFrame(AudioProcessing* ap, AudioFrame* frame) {
|
||||
if (!frame || !ap) {
|
||||
return AudioProcessing::Error::kNullPointerError;
|
||||
}
|
||||
|
||||
StreamConfig input_config(frame->sample_rate_hz_, frame->num_channels_);
|
||||
StreamConfig output_config(frame->sample_rate_hz_, frame->num_channels_);
|
||||
RTC_DCHECK_EQ(frame->samples_per_channel(), input_config.num_frames());
|
||||
|
||||
int result = ap->ProcessStream(frame->data(), input_config, output_config,
|
||||
frame->mutable_data());
|
||||
|
||||
AudioProcessingStats stats = ap->GetStatistics();
|
||||
|
||||
if (stats.voice_detected) {
|
||||
frame->vad_activity_ = *stats.voice_detected
|
||||
? AudioFrame::VADActivity::kVadActive
|
||||
: AudioFrame::VADActivity::kVadPassive;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
int ProcessReverseAudioFrame(AudioProcessing* ap, AudioFrame* frame) {
|
||||
if (!frame || !ap) {
|
||||
return AudioProcessing::Error::kNullPointerError;
|
||||
}
|
||||
|
||||
// Must be a native rate.
|
||||
if (frame->sample_rate_hz_ != AudioProcessing::NativeRate::kSampleRate8kHz &&
|
||||
frame->sample_rate_hz_ != AudioProcessing::NativeRate::kSampleRate16kHz &&
|
||||
frame->sample_rate_hz_ != AudioProcessing::NativeRate::kSampleRate32kHz &&
|
||||
frame->sample_rate_hz_ != AudioProcessing::NativeRate::kSampleRate48kHz) {
|
||||
return AudioProcessing::Error::kBadSampleRateError;
|
||||
}
|
||||
|
||||
if (frame->num_channels_ <= 0) {
|
||||
return AudioProcessing::Error::kBadNumberChannelsError;
|
||||
}
|
||||
|
||||
StreamConfig input_config(frame->sample_rate_hz_, frame->num_channels_);
|
||||
StreamConfig output_config(frame->sample_rate_hz_, frame->num_channels_);
|
||||
|
||||
int result = ap->ProcessReverseStream(frame->data(), input_config,
|
||||
output_config, frame->mutable_data());
|
||||
return result;
|
||||
}
|
||||
|
||||
} // namespace webrtc
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
/*
|
||||
* Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#ifndef MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_FRAME_PROXIES_H_
|
||||
#define MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_FRAME_PROXIES_H_
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
class AudioFrame;
|
||||
class AudioProcessing;
|
||||
|
||||
// Processes a 10 ms `frame` of the primary audio stream using the provided
|
||||
// AudioProcessing object. On the client-side, this is the near-end (or
|
||||
// captured) audio. The `sample_rate_hz_`, `num_channels_`, and
|
||||
// `samples_per_channel_` members of `frame` must be valid. If changed from the
|
||||
// previous call to this function, it will trigger an initialization of the
|
||||
// provided AudioProcessing object.
|
||||
// The function returns any error codes passed from the AudioProcessing
|
||||
// ProcessStream method.
|
||||
int ProcessAudioFrame(AudioProcessing* ap, AudioFrame* frame);
|
||||
|
||||
// Processes a 10 ms `frame` of the reverse direction audio stream using the
|
||||
// provided AudioProcessing object. The frame may be modified. On the
|
||||
// client-side, this is the far-end (or to be rendered) audio. The
|
||||
// `sample_rate_hz_`, `num_channels_`, and `samples_per_channel_` members of
|
||||
// `frame` must be valid. If changed from the previous call to this function, it
|
||||
// will trigger an initialization of the provided AudioProcessing object.
|
||||
// The function returns any error codes passed from the AudioProcessing
|
||||
// ProcessReverseStream method.
|
||||
int ProcessReverseAudioFrame(AudioProcessing* ap, AudioFrame* frame);
|
||||
|
||||
} // namespace webrtc
|
||||
|
||||
#endif // MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_FRAME_PROXIES_H_
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
/*
|
||||
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#ifndef MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_FRAME_VIEW_H_
|
||||
#define MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_FRAME_VIEW_H_
|
||||
|
||||
#include "api/array_view.h"
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
// Class to pass audio data in T** format, where T is a numeric type.
|
||||
template <class T>
|
||||
class AudioFrameView {
|
||||
public:
|
||||
// `num_channels` and `channel_size` describe the T**
|
||||
// `audio_samples`. `audio_samples` is assumed to point to a
|
||||
// two-dimensional |num_channels * channel_size| array of floats.
|
||||
AudioFrameView(T* const* audio_samples, int num_channels, int channel_size)
|
||||
: audio_samples_(audio_samples),
|
||||
num_channels_(num_channels),
|
||||
channel_size_(channel_size) {
|
||||
RTC_DCHECK_GE(num_channels_, 0);
|
||||
RTC_DCHECK_GE(channel_size_, 0);
|
||||
}
|
||||
|
||||
// Implicit cast to allow converting Frame<float> to
|
||||
// Frame<const float>.
|
||||
template <class U>
|
||||
AudioFrameView(AudioFrameView<U> other)
|
||||
: audio_samples_(other.data()),
|
||||
num_channels_(other.num_channels()),
|
||||
channel_size_(other.samples_per_channel()) {}
|
||||
|
||||
AudioFrameView() = delete;
|
||||
|
||||
int num_channels() const { return num_channels_; }
|
||||
|
||||
int samples_per_channel() const { return channel_size_; }
|
||||
|
||||
rtc::ArrayView<T> channel(int idx) {
|
||||
RTC_DCHECK_LE(0, idx);
|
||||
RTC_DCHECK_LE(idx, num_channels_);
|
||||
return rtc::ArrayView<T>(audio_samples_[idx], channel_size_);
|
||||
}
|
||||
|
||||
rtc::ArrayView<const T> channel(int idx) const {
|
||||
RTC_DCHECK_LE(0, idx);
|
||||
RTC_DCHECK_LE(idx, num_channels_);
|
||||
return rtc::ArrayView<const T>(audio_samples_[idx], channel_size_);
|
||||
}
|
||||
|
||||
T* const* data() { return audio_samples_; }
|
||||
|
||||
private:
|
||||
T* const* audio_samples_;
|
||||
int num_channels_;
|
||||
int channel_size_;
|
||||
};
|
||||
} // namespace webrtc
|
||||
|
||||
#endif // MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_FRAME_VIEW_H_
|
||||
|
|
@ -0,0 +1,210 @@
|
|||
/*
|
||||
* Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include "modules/audio_processing/include/audio_processing.h"
|
||||
|
||||
#include "rtc_base/strings/string_builder.h"
|
||||
#include "rtc_base/system/arch.h"
|
||||
|
||||
namespace webrtc {
|
||||
namespace {
|
||||
|
||||
using Agc1Config = AudioProcessing::Config::GainController1;
|
||||
using Agc2Config = AudioProcessing::Config::GainController2;
|
||||
|
||||
std::string NoiseSuppressionLevelToString(
|
||||
const AudioProcessing::Config::NoiseSuppression::Level& level) {
|
||||
switch (level) {
|
||||
case AudioProcessing::Config::NoiseSuppression::Level::kLow:
|
||||
return "Low";
|
||||
case AudioProcessing::Config::NoiseSuppression::Level::kModerate:
|
||||
return "Moderate";
|
||||
case AudioProcessing::Config::NoiseSuppression::Level::kHigh:
|
||||
return "High";
|
||||
case AudioProcessing::Config::NoiseSuppression::Level::kVeryHigh:
|
||||
return "VeryHigh";
|
||||
}
|
||||
RTC_CHECK_NOTREACHED();
|
||||
}
|
||||
|
||||
std::string GainController1ModeToString(const Agc1Config::Mode& mode) {
|
||||
switch (mode) {
|
||||
case Agc1Config::Mode::kAdaptiveAnalog:
|
||||
return "AdaptiveAnalog";
|
||||
case Agc1Config::Mode::kAdaptiveDigital:
|
||||
return "AdaptiveDigital";
|
||||
case Agc1Config::Mode::kFixedDigital:
|
||||
return "FixedDigital";
|
||||
}
|
||||
RTC_CHECK_NOTREACHED();
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
constexpr int AudioProcessing::kNativeSampleRatesHz[];
|
||||
|
||||
void CustomProcessing::SetRuntimeSetting(
|
||||
AudioProcessing::RuntimeSetting setting) {}
|
||||
|
||||
bool Agc1Config::operator==(const Agc1Config& rhs) const {
|
||||
const auto& analog_lhs = analog_gain_controller;
|
||||
const auto& analog_rhs = rhs.analog_gain_controller;
|
||||
return enabled == rhs.enabled && mode == rhs.mode &&
|
||||
target_level_dbfs == rhs.target_level_dbfs &&
|
||||
compression_gain_db == rhs.compression_gain_db &&
|
||||
enable_limiter == rhs.enable_limiter &&
|
||||
analog_lhs.enabled == analog_rhs.enabled &&
|
||||
analog_lhs.startup_min_volume == analog_rhs.startup_min_volume &&
|
||||
analog_lhs.clipped_level_min == analog_rhs.clipped_level_min &&
|
||||
analog_lhs.enable_digital_adaptive ==
|
||||
analog_rhs.enable_digital_adaptive &&
|
||||
analog_lhs.clipped_level_step == analog_rhs.clipped_level_step &&
|
||||
analog_lhs.clipped_ratio_threshold ==
|
||||
analog_rhs.clipped_ratio_threshold &&
|
||||
analog_lhs.clipped_wait_frames == analog_rhs.clipped_wait_frames &&
|
||||
analog_lhs.clipping_predictor.mode ==
|
||||
analog_rhs.clipping_predictor.mode &&
|
||||
analog_lhs.clipping_predictor.window_length ==
|
||||
analog_rhs.clipping_predictor.window_length &&
|
||||
analog_lhs.clipping_predictor.reference_window_length ==
|
||||
analog_rhs.clipping_predictor.reference_window_length &&
|
||||
analog_lhs.clipping_predictor.reference_window_delay ==
|
||||
analog_rhs.clipping_predictor.reference_window_delay &&
|
||||
analog_lhs.clipping_predictor.clipping_threshold ==
|
||||
analog_rhs.clipping_predictor.clipping_threshold &&
|
||||
analog_lhs.clipping_predictor.crest_factor_margin ==
|
||||
analog_rhs.clipping_predictor.crest_factor_margin &&
|
||||
analog_lhs.clipping_predictor.use_predicted_step ==
|
||||
analog_rhs.clipping_predictor.use_predicted_step;
|
||||
}
|
||||
|
||||
bool Agc2Config::AdaptiveDigital::operator==(
|
||||
const Agc2Config::AdaptiveDigital& rhs) const {
|
||||
return enabled == rhs.enabled && headroom_db == rhs.headroom_db &&
|
||||
max_gain_db == rhs.max_gain_db &&
|
||||
initial_gain_db == rhs.initial_gain_db &&
|
||||
max_gain_change_db_per_second == rhs.max_gain_change_db_per_second &&
|
||||
max_output_noise_level_dbfs == rhs.max_output_noise_level_dbfs;
|
||||
}
|
||||
|
||||
bool Agc2Config::InputVolumeController::operator==(
|
||||
const Agc2Config::InputVolumeController& rhs) const {
|
||||
return enabled == rhs.enabled;
|
||||
}
|
||||
|
||||
bool Agc2Config::operator==(const Agc2Config& rhs) const {
|
||||
return enabled == rhs.enabled &&
|
||||
fixed_digital.gain_db == rhs.fixed_digital.gain_db &&
|
||||
adaptive_digital == rhs.adaptive_digital &&
|
||||
input_volume_controller == rhs.input_volume_controller;
|
||||
}
|
||||
|
||||
bool AudioProcessing::Config::CaptureLevelAdjustment::operator==(
|
||||
const AudioProcessing::Config::CaptureLevelAdjustment& rhs) const {
|
||||
return enabled == rhs.enabled && pre_gain_factor == rhs.pre_gain_factor &&
|
||||
post_gain_factor == rhs.post_gain_factor &&
|
||||
analog_mic_gain_emulation == rhs.analog_mic_gain_emulation;
|
||||
}
|
||||
|
||||
bool AudioProcessing::Config::CaptureLevelAdjustment::AnalogMicGainEmulation::
|
||||
operator==(const AudioProcessing::Config::CaptureLevelAdjustment::
|
||||
AnalogMicGainEmulation& rhs) const {
|
||||
return enabled == rhs.enabled && initial_level == rhs.initial_level;
|
||||
}
|
||||
|
||||
std::string AudioProcessing::Config::ToString() const {
|
||||
char buf[2048];
|
||||
rtc::SimpleStringBuilder builder(buf);
|
||||
builder << "AudioProcessing::Config{ "
|
||||
"pipeline: { "
|
||||
"maximum_internal_processing_rate: "
|
||||
<< pipeline.maximum_internal_processing_rate
|
||||
<< ", multi_channel_render: " << pipeline.multi_channel_render
|
||||
<< ", multi_channel_capture: " << pipeline.multi_channel_capture
|
||||
<< " }, pre_amplifier: { enabled: " << pre_amplifier.enabled
|
||||
<< ", fixed_gain_factor: " << pre_amplifier.fixed_gain_factor
|
||||
<< " },capture_level_adjustment: { enabled: "
|
||||
<< capture_level_adjustment.enabled
|
||||
<< ", pre_gain_factor: " << capture_level_adjustment.pre_gain_factor
|
||||
<< ", post_gain_factor: " << capture_level_adjustment.post_gain_factor
|
||||
<< ", analog_mic_gain_emulation: { enabled: "
|
||||
<< capture_level_adjustment.analog_mic_gain_emulation.enabled
|
||||
<< ", initial_level: "
|
||||
<< capture_level_adjustment.analog_mic_gain_emulation.initial_level
|
||||
<< " }}, high_pass_filter: { enabled: " << high_pass_filter.enabled
|
||||
<< " }, echo_canceller: { enabled: " << echo_canceller.enabled
|
||||
<< ", mobile_mode: " << echo_canceller.mobile_mode
|
||||
<< ", enforce_high_pass_filtering: "
|
||||
<< echo_canceller.enforce_high_pass_filtering
|
||||
<< " }, noise_suppression: { enabled: " << noise_suppression.enabled
|
||||
<< ", level: "
|
||||
<< NoiseSuppressionLevelToString(noise_suppression.level)
|
||||
<< " }, transient_suppression: { enabled: "
|
||||
<< transient_suppression.enabled
|
||||
<< " }, gain_controller1: { enabled: " << gain_controller1.enabled
|
||||
<< ", mode: " << GainController1ModeToString(gain_controller1.mode)
|
||||
<< ", target_level_dbfs: " << gain_controller1.target_level_dbfs
|
||||
<< ", compression_gain_db: " << gain_controller1.compression_gain_db
|
||||
<< ", enable_limiter: " << gain_controller1.enable_limiter
|
||||
<< ", analog_gain_controller { enabled: "
|
||||
<< gain_controller1.analog_gain_controller.enabled
|
||||
<< ", startup_min_volume: "
|
||||
<< gain_controller1.analog_gain_controller.startup_min_volume
|
||||
<< ", clipped_level_min: "
|
||||
<< gain_controller1.analog_gain_controller.clipped_level_min
|
||||
<< ", enable_digital_adaptive: "
|
||||
<< gain_controller1.analog_gain_controller.enable_digital_adaptive
|
||||
<< ", clipped_level_step: "
|
||||
<< gain_controller1.analog_gain_controller.clipped_level_step
|
||||
<< ", clipped_ratio_threshold: "
|
||||
<< gain_controller1.analog_gain_controller.clipped_ratio_threshold
|
||||
<< ", clipped_wait_frames: "
|
||||
<< gain_controller1.analog_gain_controller.clipped_wait_frames
|
||||
<< ", clipping_predictor: { enabled: "
|
||||
<< gain_controller1.analog_gain_controller.clipping_predictor.enabled
|
||||
<< ", mode: "
|
||||
<< gain_controller1.analog_gain_controller.clipping_predictor.mode
|
||||
<< ", window_length: "
|
||||
<< gain_controller1.analog_gain_controller.clipping_predictor
|
||||
.window_length
|
||||
<< ", reference_window_length: "
|
||||
<< gain_controller1.analog_gain_controller.clipping_predictor
|
||||
.reference_window_length
|
||||
<< ", reference_window_delay: "
|
||||
<< gain_controller1.analog_gain_controller.clipping_predictor
|
||||
.reference_window_delay
|
||||
<< ", clipping_threshold: "
|
||||
<< gain_controller1.analog_gain_controller.clipping_predictor
|
||||
.clipping_threshold
|
||||
<< ", crest_factor_margin: "
|
||||
<< gain_controller1.analog_gain_controller.clipping_predictor
|
||||
.crest_factor_margin
|
||||
<< ", use_predicted_step: "
|
||||
<< gain_controller1.analog_gain_controller.clipping_predictor
|
||||
.use_predicted_step
|
||||
<< " }}}, gain_controller2: { enabled: " << gain_controller2.enabled
|
||||
<< ", fixed_digital: { gain_db: "
|
||||
<< gain_controller2.fixed_digital.gain_db
|
||||
<< " }, adaptive_digital: { enabled: "
|
||||
<< gain_controller2.adaptive_digital.enabled
|
||||
<< ", headroom_db: " << gain_controller2.adaptive_digital.headroom_db
|
||||
<< ", max_gain_db: " << gain_controller2.adaptive_digital.max_gain_db
|
||||
<< ", initial_gain_db: "
|
||||
<< gain_controller2.adaptive_digital.initial_gain_db
|
||||
<< ", max_gain_change_db_per_second: "
|
||||
<< gain_controller2.adaptive_digital.max_gain_change_db_per_second
|
||||
<< ", max_output_noise_level_dbfs: "
|
||||
<< gain_controller2.adaptive_digital.max_output_noise_level_dbfs
|
||||
<< " }, input_volume_control : { enabled "
|
||||
<< gain_controller2.input_volume_controller.enabled << "}}";
|
||||
return builder.str();
|
||||
}
|
||||
|
||||
} // namespace webrtc
|
||||
|
|
@ -0,0 +1,941 @@
|
|||
/*
|
||||
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#ifndef MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_H_
|
||||
#define MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_H_
|
||||
|
||||
// MSVC++ requires this to be set before any other includes to get M_PI.
|
||||
#ifndef _USE_MATH_DEFINES
|
||||
#define _USE_MATH_DEFINES
|
||||
#endif
|
||||
|
||||
#include <math.h>
|
||||
#include <stddef.h> // size_t
|
||||
#include <stdio.h> // FILE
|
||||
#include <string.h>
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "absl/base/nullability.h"
|
||||
#include "absl/strings/string_view.h"
|
||||
#include "absl/types/optional.h"
|
||||
#include "api/array_view.h"
|
||||
#include "api/audio/echo_canceller3_config.h"
|
||||
#include "api/audio/echo_control.h"
|
||||
#include "api/ref_count.h"
|
||||
#include "api/scoped_refptr.h"
|
||||
#include "api/task_queue/task_queue_base.h"
|
||||
#include "modules/audio_processing/include/audio_processing_statistics.h"
|
||||
#include "rtc_base/arraysize.h"
|
||||
#include "rtc_base/system/file_wrapper.h"
|
||||
#include "rtc_base/system/rtc_export.h"
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
class AecDump;
|
||||
class AudioBuffer;
|
||||
|
||||
class StreamConfig;
|
||||
class ProcessingConfig;
|
||||
|
||||
class EchoDetector;
|
||||
class CustomAudioAnalyzer;
|
||||
class CustomProcessing;
|
||||
|
||||
// The Audio Processing Module (APM) provides a collection of voice processing
|
||||
// components designed for real-time communications software.
|
||||
//
|
||||
// APM operates on two audio streams on a frame-by-frame basis. Frames of the
|
||||
// primary stream, on which all processing is applied, are passed to
|
||||
// `ProcessStream()`. Frames of the reverse direction stream are passed to
|
||||
// `ProcessReverseStream()`. On the client-side, this will typically be the
|
||||
// near-end (capture) and far-end (render) streams, respectively. APM should be
|
||||
// placed in the signal chain as close to the audio hardware abstraction layer
|
||||
// (HAL) as possible.
|
||||
//
|
||||
// On the server-side, the reverse stream will normally not be used, with
|
||||
// processing occurring on each incoming stream.
|
||||
//
|
||||
// Component interfaces follow a similar pattern and are accessed through
|
||||
// corresponding getters in APM. All components are disabled at create-time,
|
||||
// with default settings that are recommended for most situations. New settings
|
||||
// can be applied without enabling a component. Enabling a component triggers
|
||||
// memory allocation and initialization to allow it to start processing the
|
||||
// streams.
|
||||
//
|
||||
// Thread safety is provided with the following assumptions to reduce locking
|
||||
// overhead:
|
||||
// 1. The stream getters and setters are called from the same thread as
|
||||
// ProcessStream(). More precisely, stream functions are never called
|
||||
// concurrently with ProcessStream().
|
||||
// 2. Parameter getters are never called concurrently with the corresponding
|
||||
// setter.
|
||||
//
|
||||
// APM accepts only linear PCM audio data in chunks of ~10 ms (see
|
||||
// AudioProcessing::GetFrameSize() for details) and sample rates ranging from
|
||||
// 8000 Hz to 384000 Hz. The int16 interfaces use interleaved data, while the
|
||||
// float interfaces use deinterleaved data.
|
||||
//
|
||||
// Usage example, omitting error checking:
|
||||
// rtc::scoped_refptr<AudioProcessing> apm = AudioProcessingBuilder().Create();
|
||||
//
|
||||
// AudioProcessing::Config config;
|
||||
// config.echo_canceller.enabled = true;
|
||||
// config.echo_canceller.mobile_mode = false;
|
||||
//
|
||||
// config.gain_controller1.enabled = true;
|
||||
// config.gain_controller1.mode =
|
||||
// AudioProcessing::Config::GainController1::kAdaptiveAnalog;
|
||||
// config.gain_controller1.analog_level_minimum = 0;
|
||||
// config.gain_controller1.analog_level_maximum = 255;
|
||||
//
|
||||
// config.gain_controller2.enabled = true;
|
||||
//
|
||||
// config.high_pass_filter.enabled = true;
|
||||
//
|
||||
// apm->ApplyConfig(config)
|
||||
//
|
||||
// // Start a voice call...
|
||||
//
|
||||
// // ... Render frame arrives bound for the audio HAL ...
|
||||
// apm->ProcessReverseStream(render_frame);
|
||||
//
|
||||
// // ... Capture frame arrives from the audio HAL ...
|
||||
// // Call required set_stream_ functions.
|
||||
// apm->set_stream_delay_ms(delay_ms);
|
||||
// apm->set_stream_analog_level(analog_level);
|
||||
//
|
||||
// apm->ProcessStream(capture_frame);
|
||||
//
|
||||
// // Call required stream_ functions.
|
||||
// analog_level = apm->recommended_stream_analog_level();
|
||||
// has_voice = apm->stream_has_voice();
|
||||
//
|
||||
// // Repeat render and capture processing for the duration of the call...
|
||||
// // Start a new call...
|
||||
// apm->Initialize();
|
||||
//
|
||||
// // Close the application...
|
||||
// apm.reset();
|
||||
//
|
||||
class RTC_EXPORT AudioProcessing : public RefCountInterface {
|
||||
public:
|
||||
// The struct below constitutes the new parameter scheme for the audio
|
||||
// processing. It is being introduced gradually and until it is fully
|
||||
// introduced, it is prone to change.
|
||||
// TODO(peah): Remove this comment once the new config scheme is fully rolled
|
||||
// out.
|
||||
//
|
||||
// The parameters and behavior of the audio processing module are controlled
|
||||
// by changing the default values in the AudioProcessing::Config struct.
|
||||
// The config is applied by passing the struct to the ApplyConfig method.
|
||||
//
|
||||
// This config is intended to be used during setup, and to enable/disable
|
||||
// top-level processing effects. Use during processing may cause undesired
|
||||
// submodule resets, affecting the audio quality. Use the RuntimeSetting
|
||||
// construct for runtime configuration.
|
||||
struct RTC_EXPORT Config {
|
||||
// Sets the properties of the audio processing pipeline.
|
||||
struct RTC_EXPORT Pipeline {
|
||||
// Ways to downmix a multi-channel track to mono.
|
||||
enum class DownmixMethod {
|
||||
kAverageChannels, // Average across channels.
|
||||
kUseFirstChannel // Use the first channel.
|
||||
};
|
||||
|
||||
// Maximum allowed processing rate used internally. May only be set to
|
||||
// 32000 or 48000 and any differing values will be treated as 48000.
|
||||
int maximum_internal_processing_rate = 48000;
|
||||
// Allow multi-channel processing of render audio.
|
||||
bool multi_channel_render = false;
|
||||
// Allow multi-channel processing of capture audio when AEC3 is active
|
||||
// or a custom AEC is injected..
|
||||
bool multi_channel_capture = false;
|
||||
// Indicates how to downmix multi-channel capture audio to mono (when
|
||||
// needed).
|
||||
DownmixMethod capture_downmix_method = DownmixMethod::kAverageChannels;
|
||||
} pipeline;
|
||||
|
||||
// Enabled the pre-amplifier. It amplifies the capture signal
|
||||
// before any other processing is done.
|
||||
// TODO(webrtc:5298): Deprecate and use the pre-gain functionality in
|
||||
// capture_level_adjustment instead.
|
||||
struct PreAmplifier {
|
||||
bool enabled = false;
|
||||
float fixed_gain_factor = 1.0f;
|
||||
} pre_amplifier;
|
||||
|
||||
// Functionality for general level adjustment in the capture pipeline. This
|
||||
// should not be used together with the legacy PreAmplifier functionality.
|
||||
struct CaptureLevelAdjustment {
|
||||
bool operator==(const CaptureLevelAdjustment& rhs) const;
|
||||
bool operator!=(const CaptureLevelAdjustment& rhs) const {
|
||||
return !(*this == rhs);
|
||||
}
|
||||
bool enabled = false;
|
||||
// The `pre_gain_factor` scales the signal before any processing is done.
|
||||
float pre_gain_factor = 1.0f;
|
||||
// The `post_gain_factor` scales the signal after all processing is done.
|
||||
float post_gain_factor = 1.0f;
|
||||
struct AnalogMicGainEmulation {
|
||||
bool operator==(const AnalogMicGainEmulation& rhs) const;
|
||||
bool operator!=(const AnalogMicGainEmulation& rhs) const {
|
||||
return !(*this == rhs);
|
||||
}
|
||||
bool enabled = false;
|
||||
// Initial analog gain level to use for the emulated analog gain. Must
|
||||
// be in the range [0...255].
|
||||
int initial_level = 255;
|
||||
} analog_mic_gain_emulation;
|
||||
} capture_level_adjustment;
|
||||
|
||||
struct HighPassFilter {
|
||||
bool enabled = false;
|
||||
bool apply_in_full_band = true;
|
||||
} high_pass_filter;
|
||||
|
||||
struct EchoCanceller {
|
||||
bool enabled = false;
|
||||
bool mobile_mode = false;
|
||||
bool export_linear_aec_output = false;
|
||||
// Enforce the highpass filter to be on (has no effect for the mobile
|
||||
// mode).
|
||||
bool enforce_high_pass_filtering = true;
|
||||
} echo_canceller;
|
||||
|
||||
// Enables background noise suppression.
|
||||
struct NoiseSuppression {
|
||||
bool enabled = false;
|
||||
enum Level { kLow, kModerate, kHigh, kVeryHigh };
|
||||
Level level = kModerate;
|
||||
bool analyze_linear_aec_output_when_available = false;
|
||||
} noise_suppression;
|
||||
|
||||
// Enables transient suppression.
|
||||
struct TransientSuppression {
|
||||
bool enabled = false;
|
||||
} transient_suppression;
|
||||
|
||||
// Enables automatic gain control (AGC) functionality.
|
||||
// The automatic gain control (AGC) component brings the signal to an
|
||||
// appropriate range. This is done by applying a digital gain directly and,
|
||||
// in the analog mode, prescribing an analog gain to be applied at the audio
|
||||
// HAL.
|
||||
// Recommended to be enabled on the client-side.
|
||||
struct RTC_EXPORT GainController1 {
|
||||
bool operator==(const GainController1& rhs) const;
|
||||
bool operator!=(const GainController1& rhs) const {
|
||||
return !(*this == rhs);
|
||||
}
|
||||
|
||||
bool enabled = false;
|
||||
enum Mode {
|
||||
// Adaptive mode intended for use if an analog volume control is
|
||||
// available on the capture device. It will require the user to provide
|
||||
// coupling between the OS mixer controls and AGC through the
|
||||
// stream_analog_level() functions.
|
||||
// It consists of an analog gain prescription for the audio device and a
|
||||
// digital compression stage.
|
||||
kAdaptiveAnalog,
|
||||
// Adaptive mode intended for situations in which an analog volume
|
||||
// control is unavailable. It operates in a similar fashion to the
|
||||
// adaptive analog mode, but with scaling instead applied in the digital
|
||||
// domain. As with the analog mode, it additionally uses a digital
|
||||
// compression stage.
|
||||
kAdaptiveDigital,
|
||||
// Fixed mode which enables only the digital compression stage also used
|
||||
// by the two adaptive modes.
|
||||
// It is distinguished from the adaptive modes by considering only a
|
||||
// short time-window of the input signal. It applies a fixed gain
|
||||
// through most of the input level range, and compresses (gradually
|
||||
// reduces gain with increasing level) the input signal at higher
|
||||
// levels. This mode is preferred on embedded devices where the capture
|
||||
// signal level is predictable, so that a known gain can be applied.
|
||||
kFixedDigital
|
||||
};
|
||||
Mode mode = kAdaptiveAnalog;
|
||||
// Sets the target peak level (or envelope) of the AGC in dBFs (decibels
|
||||
// from digital full-scale). The convention is to use positive values. For
|
||||
// instance, passing in a value of 3 corresponds to -3 dBFs, or a target
|
||||
// level 3 dB below full-scale. Limited to [0, 31].
|
||||
int target_level_dbfs = 3;
|
||||
// Sets the maximum gain the digital compression stage may apply, in dB. A
|
||||
// higher number corresponds to greater compression, while a value of 0
|
||||
// will leave the signal uncompressed. Limited to [0, 90].
|
||||
// For updates after APM setup, use a RuntimeSetting instead.
|
||||
int compression_gain_db = 9;
|
||||
// When enabled, the compression stage will hard limit the signal to the
|
||||
// target level. Otherwise, the signal will be compressed but not limited
|
||||
// above the target level.
|
||||
bool enable_limiter = true;
|
||||
|
||||
// Enables the analog gain controller functionality.
|
||||
struct AnalogGainController {
|
||||
bool enabled = true;
|
||||
// TODO(bugs.webrtc.org/7494): Deprecated. Stop using and remove.
|
||||
int startup_min_volume = 0;
|
||||
// Lowest analog microphone level that will be applied in response to
|
||||
// clipping.
|
||||
int clipped_level_min = 70;
|
||||
// If true, an adaptive digital gain is applied.
|
||||
bool enable_digital_adaptive = true;
|
||||
// Amount the microphone level is lowered with every clipping event.
|
||||
// Limited to (0, 255].
|
||||
int clipped_level_step = 15;
|
||||
// Proportion of clipped samples required to declare a clipping event.
|
||||
// Limited to (0.f, 1.f).
|
||||
float clipped_ratio_threshold = 0.1f;
|
||||
// Time in frames to wait after a clipping event before checking again.
|
||||
// Limited to values higher than 0.
|
||||
int clipped_wait_frames = 300;
|
||||
|
||||
// Enables clipping prediction functionality.
|
||||
struct ClippingPredictor {
|
||||
bool enabled = false;
|
||||
enum Mode {
|
||||
// Clipping event prediction mode with fixed step estimation.
|
||||
kClippingEventPrediction,
|
||||
// Clipped peak estimation mode with adaptive step estimation.
|
||||
kAdaptiveStepClippingPeakPrediction,
|
||||
// Clipped peak estimation mode with fixed step estimation.
|
||||
kFixedStepClippingPeakPrediction,
|
||||
};
|
||||
Mode mode = kClippingEventPrediction;
|
||||
// Number of frames in the sliding analysis window.
|
||||
int window_length = 5;
|
||||
// Number of frames in the sliding reference window.
|
||||
int reference_window_length = 5;
|
||||
// Reference window delay (unit: number of frames).
|
||||
int reference_window_delay = 5;
|
||||
// Clipping prediction threshold (dBFS).
|
||||
float clipping_threshold = -1.0f;
|
||||
// Crest factor drop threshold (dB).
|
||||
float crest_factor_margin = 3.0f;
|
||||
// If true, the recommended clipped level step is used to modify the
|
||||
// analog gain. Otherwise, the predictor runs without affecting the
|
||||
// analog gain.
|
||||
bool use_predicted_step = true;
|
||||
} clipping_predictor;
|
||||
} analog_gain_controller;
|
||||
} gain_controller1;
|
||||
|
||||
// Parameters for AGC2, an Automatic Gain Control (AGC) sub-module which
|
||||
// replaces the AGC sub-module parametrized by `gain_controller1`.
|
||||
// AGC2 brings the captured audio signal to the desired level by combining
|
||||
// three different controllers (namely, input volume controller, adapative
|
||||
// digital controller and fixed digital controller) and a limiter.
|
||||
// TODO(bugs.webrtc.org:7494): Name `GainController` when AGC1 removed.
|
||||
struct RTC_EXPORT GainController2 {
|
||||
bool operator==(const GainController2& rhs) const;
|
||||
bool operator!=(const GainController2& rhs) const {
|
||||
return !(*this == rhs);
|
||||
}
|
||||
|
||||
// AGC2 must be created if and only if `enabled` is true.
|
||||
bool enabled = false;
|
||||
|
||||
// Parameters for the input volume controller, which adjusts the input
|
||||
// volume applied when the audio is captured (e.g., microphone volume on
|
||||
// a soundcard, input volume on HAL).
|
||||
struct InputVolumeController {
|
||||
bool operator==(const InputVolumeController& rhs) const;
|
||||
bool operator!=(const InputVolumeController& rhs) const {
|
||||
return !(*this == rhs);
|
||||
}
|
||||
bool enabled = false;
|
||||
} input_volume_controller;
|
||||
|
||||
// Parameters for the adaptive digital controller, which adjusts and
|
||||
// applies a digital gain after echo cancellation and after noise
|
||||
// suppression.
|
||||
struct RTC_EXPORT AdaptiveDigital {
|
||||
bool operator==(const AdaptiveDigital& rhs) const;
|
||||
bool operator!=(const AdaptiveDigital& rhs) const {
|
||||
return !(*this == rhs);
|
||||
}
|
||||
bool enabled = false;
|
||||
float headroom_db = 6.0f;
|
||||
float max_gain_db = 30.0f;
|
||||
float initial_gain_db = 8.0f;
|
||||
float max_gain_change_db_per_second = 3.0f;
|
||||
float max_output_noise_level_dbfs = -50.0f;
|
||||
} adaptive_digital;
|
||||
|
||||
// Parameters for the fixed digital controller, which applies a fixed
|
||||
// digital gain after the adaptive digital controller and before the
|
||||
// limiter.
|
||||
struct FixedDigital {
|
||||
// By setting `gain_db` to a value greater than zero, the limiter can be
|
||||
// turned into a compressor that first applies a fixed gain.
|
||||
float gain_db = 0.0f;
|
||||
} fixed_digital;
|
||||
} gain_controller2;
|
||||
|
||||
std::string ToString() const;
|
||||
};
|
||||
|
||||
// Specifies the properties of a setting to be passed to AudioProcessing at
|
||||
// runtime.
|
||||
class RuntimeSetting {
|
||||
public:
|
||||
enum class Type {
|
||||
kNotSpecified,
|
||||
kCapturePreGain,
|
||||
kCaptureCompressionGain,
|
||||
kCaptureFixedPostGain,
|
||||
kPlayoutVolumeChange,
|
||||
kCustomRenderProcessingRuntimeSetting,
|
||||
kPlayoutAudioDeviceChange,
|
||||
kCapturePostGain,
|
||||
kCaptureOutputUsed
|
||||
};
|
||||
|
||||
// Play-out audio device properties.
|
||||
struct PlayoutAudioDeviceInfo {
|
||||
int id; // Identifies the audio device.
|
||||
int max_volume; // Maximum play-out volume.
|
||||
};
|
||||
|
||||
RuntimeSetting() : type_(Type::kNotSpecified), value_(0.0f) {}
|
||||
~RuntimeSetting() = default;
|
||||
|
||||
static RuntimeSetting CreateCapturePreGain(float gain) {
|
||||
return {Type::kCapturePreGain, gain};
|
||||
}
|
||||
|
||||
static RuntimeSetting CreateCapturePostGain(float gain) {
|
||||
return {Type::kCapturePostGain, gain};
|
||||
}
|
||||
|
||||
// Corresponds to Config::GainController1::compression_gain_db, but for
|
||||
// runtime configuration.
|
||||
static RuntimeSetting CreateCompressionGainDb(int gain_db) {
|
||||
RTC_DCHECK_GE(gain_db, 0);
|
||||
RTC_DCHECK_LE(gain_db, 90);
|
||||
return {Type::kCaptureCompressionGain, static_cast<float>(gain_db)};
|
||||
}
|
||||
|
||||
// Corresponds to Config::GainController2::fixed_digital::gain_db, but for
|
||||
// runtime configuration.
|
||||
static RuntimeSetting CreateCaptureFixedPostGain(float gain_db) {
|
||||
RTC_DCHECK_GE(gain_db, 0.0f);
|
||||
RTC_DCHECK_LE(gain_db, 90.0f);
|
||||
return {Type::kCaptureFixedPostGain, gain_db};
|
||||
}
|
||||
|
||||
// Creates a runtime setting to notify play-out (aka render) audio device
|
||||
// changes.
|
||||
static RuntimeSetting CreatePlayoutAudioDeviceChange(
|
||||
PlayoutAudioDeviceInfo audio_device) {
|
||||
return {Type::kPlayoutAudioDeviceChange, audio_device};
|
||||
}
|
||||
|
||||
// Creates a runtime setting to notify play-out (aka render) volume changes.
|
||||
// `volume` is the unnormalized volume, the maximum of which
|
||||
static RuntimeSetting CreatePlayoutVolumeChange(int volume) {
|
||||
return {Type::kPlayoutVolumeChange, volume};
|
||||
}
|
||||
|
||||
static RuntimeSetting CreateCustomRenderSetting(float payload) {
|
||||
return {Type::kCustomRenderProcessingRuntimeSetting, payload};
|
||||
}
|
||||
|
||||
static RuntimeSetting CreateCaptureOutputUsedSetting(
|
||||
bool capture_output_used) {
|
||||
return {Type::kCaptureOutputUsed, capture_output_used};
|
||||
}
|
||||
|
||||
Type type() const { return type_; }
|
||||
// Getters do not return a value but instead modify the argument to protect
|
||||
// from implicit casting.
|
||||
void GetFloat(float* value) const {
|
||||
RTC_DCHECK(value);
|
||||
*value = value_.float_value;
|
||||
}
|
||||
void GetInt(int* value) const {
|
||||
RTC_DCHECK(value);
|
||||
*value = value_.int_value;
|
||||
}
|
||||
void GetBool(bool* value) const {
|
||||
RTC_DCHECK(value);
|
||||
*value = value_.bool_value;
|
||||
}
|
||||
void GetPlayoutAudioDeviceInfo(PlayoutAudioDeviceInfo* value) const {
|
||||
RTC_DCHECK(value);
|
||||
*value = value_.playout_audio_device_info;
|
||||
}
|
||||
|
||||
private:
|
||||
RuntimeSetting(Type id, float value) : type_(id), value_(value) {}
|
||||
RuntimeSetting(Type id, int value) : type_(id), value_(value) {}
|
||||
RuntimeSetting(Type id, PlayoutAudioDeviceInfo value)
|
||||
: type_(id), value_(value) {}
|
||||
Type type_;
|
||||
union U {
|
||||
U() {}
|
||||
U(int value) : int_value(value) {}
|
||||
U(float value) : float_value(value) {}
|
||||
U(PlayoutAudioDeviceInfo value) : playout_audio_device_info(value) {}
|
||||
float float_value;
|
||||
int int_value;
|
||||
bool bool_value;
|
||||
PlayoutAudioDeviceInfo playout_audio_device_info;
|
||||
} value_;
|
||||
};
|
||||
|
||||
~AudioProcessing() override {}
|
||||
|
||||
// Initializes internal states, while retaining all user settings. This
|
||||
// should be called before beginning to process a new audio stream. However,
|
||||
// it is not necessary to call before processing the first stream after
|
||||
// creation.
|
||||
//
|
||||
// It is also not necessary to call if the audio parameters (sample
|
||||
// rate and number of channels) have changed. Passing updated parameters
|
||||
// directly to `ProcessStream()` and `ProcessReverseStream()` is permissible.
|
||||
// If the parameters are known at init-time though, they may be provided.
|
||||
// TODO(webrtc:5298): Change to return void.
|
||||
virtual int Initialize() = 0;
|
||||
|
||||
// The int16 interfaces require:
|
||||
// - only `NativeRate`s be used
|
||||
// - that the input, output and reverse rates must match
|
||||
// - that `processing_config.output_stream()` matches
|
||||
// `processing_config.input_stream()`.
|
||||
//
|
||||
// The float interfaces accept arbitrary rates and support differing input and
|
||||
// output layouts, but the output must have either one channel or the same
|
||||
// number of channels as the input.
|
||||
virtual int Initialize(const ProcessingConfig& processing_config) = 0;
|
||||
|
||||
// TODO(peah): This method is a temporary solution used to take control
|
||||
// over the parameters in the audio processing module and is likely to change.
|
||||
virtual void ApplyConfig(const Config& config) = 0;
|
||||
|
||||
// TODO(ajm): Only intended for internal use. Make private and friend the
|
||||
// necessary classes?
|
||||
virtual int proc_sample_rate_hz() const = 0;
|
||||
virtual int proc_split_sample_rate_hz() const = 0;
|
||||
virtual size_t num_input_channels() const = 0;
|
||||
virtual size_t num_proc_channels() const = 0;
|
||||
virtual size_t num_output_channels() const = 0;
|
||||
virtual size_t num_reverse_channels() const = 0;
|
||||
|
||||
// Set to true when the output of AudioProcessing will be muted or in some
|
||||
// other way not used. Ideally, the captured audio would still be processed,
|
||||
// but some components may change behavior based on this information.
|
||||
// Default false. This method takes a lock. To achieve this in a lock-less
|
||||
// manner the PostRuntimeSetting can instead be used.
|
||||
virtual void set_output_will_be_muted(bool muted) = 0;
|
||||
|
||||
// Enqueues a runtime setting.
|
||||
virtual void SetRuntimeSetting(RuntimeSetting setting) = 0;
|
||||
|
||||
// Enqueues a runtime setting. Returns a bool indicating whether the
|
||||
// enqueueing was successfull.
|
||||
virtual bool PostRuntimeSetting(RuntimeSetting setting) = 0;
|
||||
|
||||
// Accepts and produces a ~10 ms frame of interleaved 16 bit integer audio as
|
||||
// specified in `input_config` and `output_config`. `src` and `dest` may use
|
||||
// the same memory, if desired.
|
||||
virtual int ProcessStream(const int16_t* const src,
|
||||
const StreamConfig& input_config,
|
||||
const StreamConfig& output_config,
|
||||
int16_t* const dest) = 0;
|
||||
|
||||
// Accepts deinterleaved float audio with the range [-1, 1]. Each element of
|
||||
// `src` points to a channel buffer, arranged according to `input_stream`. At
|
||||
// output, the channels will be arranged according to `output_stream` in
|
||||
// `dest`.
|
||||
//
|
||||
// The output must have one channel or as many channels as the input. `src`
|
||||
// and `dest` may use the same memory, if desired.
|
||||
virtual int ProcessStream(const float* const* src,
|
||||
const StreamConfig& input_config,
|
||||
const StreamConfig& output_config,
|
||||
float* const* dest) = 0;
|
||||
|
||||
// Accepts and produces a ~10 ms frame of interleaved 16 bit integer audio for
|
||||
// the reverse direction audio stream as specified in `input_config` and
|
||||
// `output_config`. `src` and `dest` may use the same memory, if desired.
|
||||
virtual int ProcessReverseStream(const int16_t* const src,
|
||||
const StreamConfig& input_config,
|
||||
const StreamConfig& output_config,
|
||||
int16_t* const dest) = 0;
|
||||
|
||||
// Accepts deinterleaved float audio with the range [-1, 1]. Each element of
|
||||
// `data` points to a channel buffer, arranged according to `reverse_config`.
|
||||
virtual int ProcessReverseStream(const float* const* src,
|
||||
const StreamConfig& input_config,
|
||||
const StreamConfig& output_config,
|
||||
float* const* dest) = 0;
|
||||
|
||||
// Accepts deinterleaved float audio with the range [-1, 1]. Each element
|
||||
// of `data` points to a channel buffer, arranged according to
|
||||
// `reverse_config`.
|
||||
virtual int AnalyzeReverseStream(const float* const* data,
|
||||
const StreamConfig& reverse_config) = 0;
|
||||
|
||||
// Returns the most recently produced ~10 ms of the linear AEC output at a
|
||||
// rate of 16 kHz. If there is more than one capture channel, a mono
|
||||
// representation of the input is returned. Returns true/false to indicate
|
||||
// whether an output returned.
|
||||
virtual bool GetLinearAecOutput(
|
||||
rtc::ArrayView<std::array<float, 160>> linear_output) const = 0;
|
||||
|
||||
// This must be called prior to ProcessStream() if and only if adaptive analog
|
||||
// gain control is enabled, to pass the current analog level from the audio
|
||||
// HAL. Must be within the range [0, 255].
|
||||
virtual void set_stream_analog_level(int level) = 0;
|
||||
|
||||
// When an analog mode is set, this should be called after
|
||||
// `set_stream_analog_level()` and `ProcessStream()` to obtain the recommended
|
||||
// new analog level for the audio HAL. It is the user's responsibility to
|
||||
// apply this level.
|
||||
virtual int recommended_stream_analog_level() const = 0;
|
||||
|
||||
// This must be called if and only if echo processing is enabled.
|
||||
//
|
||||
// Sets the `delay` in ms between ProcessReverseStream() receiving a far-end
|
||||
// frame and ProcessStream() receiving a near-end frame containing the
|
||||
// corresponding echo. On the client-side this can be expressed as
|
||||
// delay = (t_render - t_analyze) + (t_process - t_capture)
|
||||
// where,
|
||||
// - t_analyze is the time a frame is passed to ProcessReverseStream() and
|
||||
// t_render is the time the first sample of the same frame is rendered by
|
||||
// the audio hardware.
|
||||
// - t_capture is the time the first sample of a frame is captured by the
|
||||
// audio hardware and t_process is the time the same frame is passed to
|
||||
// ProcessStream().
|
||||
virtual int set_stream_delay_ms(int delay) = 0;
|
||||
virtual int stream_delay_ms() const = 0;
|
||||
|
||||
// Call to signal that a key press occurred (true) or did not occur (false)
|
||||
// with this chunk of audio.
|
||||
virtual void set_stream_key_pressed(bool key_pressed) = 0;
|
||||
|
||||
// Creates and attaches an webrtc::AecDump for recording debugging
|
||||
// information.
|
||||
// The `worker_queue` may not be null and must outlive the created
|
||||
// AecDump instance. |max_log_size_bytes == -1| means the log size
|
||||
// will be unlimited. `handle` may not be null. The AecDump takes
|
||||
// responsibility for `handle` and closes it in the destructor. A
|
||||
// return value of true indicates that the file has been
|
||||
// sucessfully opened, while a value of false indicates that
|
||||
// opening the file failed.
|
||||
virtual bool CreateAndAttachAecDump(
|
||||
absl::string_view file_name,
|
||||
int64_t max_log_size_bytes,
|
||||
absl::Nonnull<TaskQueueBase*> worker_queue) = 0;
|
||||
virtual bool CreateAndAttachAecDump(
|
||||
absl::Nonnull<FILE*> handle,
|
||||
int64_t max_log_size_bytes,
|
||||
absl::Nonnull<TaskQueueBase*> worker_queue) = 0;
|
||||
|
||||
// TODO(webrtc:5298) Deprecated variant.
|
||||
// Attaches provided webrtc::AecDump for recording debugging
|
||||
// information. Log file and maximum file size logic is supposed to
|
||||
// be handled by implementing instance of AecDump. Calling this
|
||||
// method when another AecDump is attached resets the active AecDump
|
||||
// with a new one. This causes the d-tor of the earlier AecDump to
|
||||
// be called. The d-tor call may block until all pending logging
|
||||
// tasks are completed.
|
||||
virtual void AttachAecDump(std::unique_ptr<AecDump> aec_dump) = 0;
|
||||
|
||||
// If no AecDump is attached, this has no effect. If an AecDump is
|
||||
// attached, it's destructor is called. The d-tor may block until
|
||||
// all pending logging tasks are completed.
|
||||
virtual void DetachAecDump() = 0;
|
||||
|
||||
// Get audio processing statistics.
|
||||
virtual AudioProcessingStats GetStatistics() = 0;
|
||||
// TODO(webrtc:5298) Deprecated variant. The `has_remote_tracks` argument
|
||||
// should be set if there are active remote tracks (this would usually be true
|
||||
// during a call). If there are no remote tracks some of the stats will not be
|
||||
// set by AudioProcessing, because they only make sense if there is at least
|
||||
// one remote track.
|
||||
virtual AudioProcessingStats GetStatistics(bool has_remote_tracks) = 0;
|
||||
|
||||
// Returns the last applied configuration.
|
||||
virtual AudioProcessing::Config GetConfig() const = 0;
|
||||
|
||||
enum Error {
|
||||
// Fatal errors.
|
||||
kNoError = 0,
|
||||
kUnspecifiedError = -1,
|
||||
kCreationFailedError = -2,
|
||||
kUnsupportedComponentError = -3,
|
||||
kUnsupportedFunctionError = -4,
|
||||
kNullPointerError = -5,
|
||||
kBadParameterError = -6,
|
||||
kBadSampleRateError = -7,
|
||||
kBadDataLengthError = -8,
|
||||
kBadNumberChannelsError = -9,
|
||||
kFileError = -10,
|
||||
kStreamParameterNotSetError = -11,
|
||||
kNotEnabledError = -12,
|
||||
|
||||
// Warnings are non-fatal.
|
||||
// This results when a set_stream_ parameter is out of range. Processing
|
||||
// will continue, but the parameter may have been truncated.
|
||||
kBadStreamParameterWarning = -13
|
||||
};
|
||||
|
||||
// Native rates supported by the integer interfaces.
|
||||
enum NativeRate {
|
||||
kSampleRate8kHz = 8000,
|
||||
kSampleRate16kHz = 16000,
|
||||
kSampleRate32kHz = 32000,
|
||||
kSampleRate48kHz = 48000
|
||||
};
|
||||
|
||||
// TODO(kwiberg): We currently need to support a compiler (Visual C++) that
|
||||
// complains if we don't explicitly state the size of the array here. Remove
|
||||
// the size when that's no longer the case.
|
||||
static constexpr int kNativeSampleRatesHz[4] = {
|
||||
kSampleRate8kHz, kSampleRate16kHz, kSampleRate32kHz, kSampleRate48kHz};
|
||||
static constexpr size_t kNumNativeSampleRates =
|
||||
arraysize(kNativeSampleRatesHz);
|
||||
static constexpr int kMaxNativeSampleRateHz =
|
||||
kNativeSampleRatesHz[kNumNativeSampleRates - 1];
|
||||
|
||||
// APM processes audio in chunks of about 10 ms. See GetFrameSize() for
|
||||
// details.
|
||||
static constexpr int kChunkSizeMs = 10;
|
||||
|
||||
// Returns floor(sample_rate_hz/100): the number of samples per channel used
|
||||
// as input and output to the audio processing module in calls to
|
||||
// ProcessStream, ProcessReverseStream, AnalyzeReverseStream, and
|
||||
// GetLinearAecOutput.
|
||||
//
|
||||
// This is exactly 10 ms for sample rates divisible by 100. For example:
|
||||
// - 48000 Hz (480 samples per channel),
|
||||
// - 44100 Hz (441 samples per channel),
|
||||
// - 16000 Hz (160 samples per channel).
|
||||
//
|
||||
// Sample rates not divisible by 100 are received/produced in frames of
|
||||
// approximately 10 ms. For example:
|
||||
// - 22050 Hz (220 samples per channel, or ~9.98 ms per frame),
|
||||
// - 11025 Hz (110 samples per channel, or ~9.98 ms per frame).
|
||||
// These nondivisible sample rates yield lower audio quality compared to
|
||||
// multiples of 100. Internal resampling to 10 ms frames causes a simulated
|
||||
// clock drift effect which impacts the performance of (for example) echo
|
||||
// cancellation.
|
||||
static int GetFrameSize(int sample_rate_hz) { return sample_rate_hz / 100; }
|
||||
};
|
||||
|
||||
class RTC_EXPORT AudioProcessingBuilder {
|
||||
public:
|
||||
AudioProcessingBuilder();
|
||||
AudioProcessingBuilder(const AudioProcessingBuilder&) = delete;
|
||||
AudioProcessingBuilder& operator=(const AudioProcessingBuilder&) = delete;
|
||||
~AudioProcessingBuilder();
|
||||
|
||||
// Sets the APM configuration.
|
||||
AudioProcessingBuilder& SetConfig(const AudioProcessing::Config& config) {
|
||||
config_ = config;
|
||||
return *this;
|
||||
}
|
||||
|
||||
// Sets the echo controller factory to inject when APM is created.
|
||||
AudioProcessingBuilder& SetEchoControlFactory(
|
||||
std::unique_ptr<EchoControlFactory> echo_control_factory) {
|
||||
echo_control_factory_ = std::move(echo_control_factory);
|
||||
return *this;
|
||||
}
|
||||
|
||||
// Sets the capture post-processing sub-module to inject when APM is created.
|
||||
AudioProcessingBuilder& SetCapturePostProcessing(
|
||||
std::unique_ptr<CustomProcessing> capture_post_processing) {
|
||||
capture_post_processing_ = std::move(capture_post_processing);
|
||||
return *this;
|
||||
}
|
||||
|
||||
// Sets the render pre-processing sub-module to inject when APM is created.
|
||||
AudioProcessingBuilder& SetRenderPreProcessing(
|
||||
std::unique_ptr<CustomProcessing> render_pre_processing) {
|
||||
render_pre_processing_ = std::move(render_pre_processing);
|
||||
return *this;
|
||||
}
|
||||
|
||||
// Sets the echo detector to inject when APM is created.
|
||||
AudioProcessingBuilder& SetEchoDetector(
|
||||
rtc::scoped_refptr<EchoDetector> echo_detector) {
|
||||
echo_detector_ = std::move(echo_detector);
|
||||
return *this;
|
||||
}
|
||||
|
||||
// Sets the capture analyzer sub-module to inject when APM is created.
|
||||
AudioProcessingBuilder& SetCaptureAnalyzer(
|
||||
std::unique_ptr<CustomAudioAnalyzer> capture_analyzer) {
|
||||
capture_analyzer_ = std::move(capture_analyzer);
|
||||
return *this;
|
||||
}
|
||||
|
||||
// Creates an APM instance with the specified config or the default one if
|
||||
// unspecified. Injects the specified components transferring the ownership
|
||||
// to the newly created APM instance - i.e., except for the config, the
|
||||
// builder is reset to its initial state.
|
||||
rtc::scoped_refptr<AudioProcessing> Create();
|
||||
|
||||
private:
|
||||
AudioProcessing::Config config_;
|
||||
std::unique_ptr<EchoControlFactory> echo_control_factory_;
|
||||
std::unique_ptr<CustomProcessing> capture_post_processing_;
|
||||
std::unique_ptr<CustomProcessing> render_pre_processing_;
|
||||
rtc::scoped_refptr<EchoDetector> echo_detector_;
|
||||
std::unique_ptr<CustomAudioAnalyzer> capture_analyzer_;
|
||||
};
|
||||
|
||||
class StreamConfig {
|
||||
public:
|
||||
// sample_rate_hz: The sampling rate of the stream.
|
||||
// num_channels: The number of audio channels in the stream.
|
||||
StreamConfig(int sample_rate_hz = 0, size_t num_channels = 0)
|
||||
: sample_rate_hz_(sample_rate_hz),
|
||||
num_channels_(num_channels),
|
||||
num_frames_(calculate_frames(sample_rate_hz)) {}
|
||||
|
||||
void set_sample_rate_hz(int value) {
|
||||
sample_rate_hz_ = value;
|
||||
num_frames_ = calculate_frames(value);
|
||||
}
|
||||
void set_num_channels(size_t value) { num_channels_ = value; }
|
||||
|
||||
int sample_rate_hz() const { return sample_rate_hz_; }
|
||||
|
||||
// The number of channels in the stream.
|
||||
size_t num_channels() const { return num_channels_; }
|
||||
|
||||
size_t num_frames() const { return num_frames_; }
|
||||
size_t num_samples() const { return num_channels_ * num_frames_; }
|
||||
|
||||
bool operator==(const StreamConfig& other) const {
|
||||
return sample_rate_hz_ == other.sample_rate_hz_ &&
|
||||
num_channels_ == other.num_channels_;
|
||||
}
|
||||
|
||||
bool operator!=(const StreamConfig& other) const { return !(*this == other); }
|
||||
|
||||
private:
|
||||
static size_t calculate_frames(int sample_rate_hz) {
|
||||
return static_cast<size_t>(AudioProcessing::GetFrameSize(sample_rate_hz));
|
||||
}
|
||||
|
||||
int sample_rate_hz_;
|
||||
size_t num_channels_;
|
||||
size_t num_frames_;
|
||||
};
|
||||
|
||||
class ProcessingConfig {
|
||||
public:
|
||||
enum StreamName {
|
||||
kInputStream,
|
||||
kOutputStream,
|
||||
kReverseInputStream,
|
||||
kReverseOutputStream,
|
||||
kNumStreamNames,
|
||||
};
|
||||
|
||||
const StreamConfig& input_stream() const {
|
||||
return streams[StreamName::kInputStream];
|
||||
}
|
||||
const StreamConfig& output_stream() const {
|
||||
return streams[StreamName::kOutputStream];
|
||||
}
|
||||
const StreamConfig& reverse_input_stream() const {
|
||||
return streams[StreamName::kReverseInputStream];
|
||||
}
|
||||
const StreamConfig& reverse_output_stream() const {
|
||||
return streams[StreamName::kReverseOutputStream];
|
||||
}
|
||||
|
||||
StreamConfig& input_stream() { return streams[StreamName::kInputStream]; }
|
||||
StreamConfig& output_stream() { return streams[StreamName::kOutputStream]; }
|
||||
StreamConfig& reverse_input_stream() {
|
||||
return streams[StreamName::kReverseInputStream];
|
||||
}
|
||||
StreamConfig& reverse_output_stream() {
|
||||
return streams[StreamName::kReverseOutputStream];
|
||||
}
|
||||
|
||||
bool operator==(const ProcessingConfig& other) const {
|
||||
for (int i = 0; i < StreamName::kNumStreamNames; ++i) {
|
||||
if (this->streams[i] != other.streams[i]) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool operator!=(const ProcessingConfig& other) const {
|
||||
return !(*this == other);
|
||||
}
|
||||
|
||||
StreamConfig streams[StreamName::kNumStreamNames];
|
||||
};
|
||||
|
||||
// Experimental interface for a custom analysis submodule.
|
||||
class CustomAudioAnalyzer {
|
||||
public:
|
||||
// (Re-) Initializes the submodule.
|
||||
virtual void Initialize(int sample_rate_hz, int num_channels) = 0;
|
||||
// Analyzes the given capture or render signal.
|
||||
virtual void Analyze(const AudioBuffer* audio) = 0;
|
||||
// Returns a string representation of the module state.
|
||||
virtual std::string ToString() const = 0;
|
||||
|
||||
virtual ~CustomAudioAnalyzer() {}
|
||||
};
|
||||
|
||||
// Interface for a custom processing submodule.
|
||||
class CustomProcessing {
|
||||
public:
|
||||
// (Re-)Initializes the submodule.
|
||||
virtual void Initialize(int sample_rate_hz, int num_channels) = 0;
|
||||
// Processes the given capture or render signal.
|
||||
virtual void Process(AudioBuffer* audio) = 0;
|
||||
// Returns a string representation of the module state.
|
||||
virtual std::string ToString() const = 0;
|
||||
// Handles RuntimeSettings. TODO(webrtc:9262): make pure virtual
|
||||
// after updating dependencies.
|
||||
virtual void SetRuntimeSetting(AudioProcessing::RuntimeSetting setting);
|
||||
|
||||
virtual ~CustomProcessing() {}
|
||||
};
|
||||
|
||||
// Interface for an echo detector submodule.
|
||||
class EchoDetector : public RefCountInterface {
|
||||
public:
|
||||
// (Re-)Initializes the submodule.
|
||||
virtual void Initialize(int capture_sample_rate_hz,
|
||||
int num_capture_channels,
|
||||
int render_sample_rate_hz,
|
||||
int num_render_channels) = 0;
|
||||
|
||||
// Analysis (not changing) of the first channel of the render signal.
|
||||
virtual void AnalyzeRenderAudio(rtc::ArrayView<const float> render_audio) = 0;
|
||||
|
||||
// Analysis (not changing) of the capture signal.
|
||||
virtual void AnalyzeCaptureAudio(
|
||||
rtc::ArrayView<const float> capture_audio) = 0;
|
||||
|
||||
struct Metrics {
|
||||
absl::optional<double> echo_likelihood;
|
||||
absl::optional<double> echo_likelihood_recent_max;
|
||||
};
|
||||
|
||||
// Collect current metrics from the echo detector.
|
||||
virtual Metrics GetMetrics() const = 0;
|
||||
};
|
||||
|
||||
} // namespace webrtc
|
||||
|
||||
#endif // MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_H_
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
/*
|
||||
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include "modules/audio_processing/include/audio_processing_statistics.h"
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
AudioProcessingStats::AudioProcessingStats() = default;
|
||||
|
||||
AudioProcessingStats::AudioProcessingStats(const AudioProcessingStats& other) =
|
||||
default;
|
||||
|
||||
AudioProcessingStats::~AudioProcessingStats() = default;
|
||||
|
||||
} // namespace webrtc
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
/*
|
||||
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#ifndef MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_STATISTICS_H_
|
||||
#define MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_STATISTICS_H_
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#include "absl/types/optional.h"
|
||||
#include "rtc_base/system/rtc_export.h"
|
||||
|
||||
namespace webrtc {
|
||||
// This version of the stats uses Optionals, it will replace the regular
|
||||
// AudioProcessingStatistics struct.
|
||||
struct RTC_EXPORT AudioProcessingStats {
|
||||
AudioProcessingStats();
|
||||
AudioProcessingStats(const AudioProcessingStats& other);
|
||||
~AudioProcessingStats();
|
||||
|
||||
// Deprecated.
|
||||
// TODO(bugs.webrtc.org/11226): Remove.
|
||||
// True if voice is detected in the last capture frame, after processing.
|
||||
// It is conservative in flagging audio as speech, with low likelihood of
|
||||
// incorrectly flagging a frame as voice.
|
||||
// Only reported if voice detection is enabled in AudioProcessing::Config.
|
||||
absl::optional<bool> voice_detected;
|
||||
|
||||
// AEC Statistics.
|
||||
// ERL = 10log_10(P_far / P_echo)
|
||||
absl::optional<double> echo_return_loss;
|
||||
// ERLE = 10log_10(P_echo / P_out)
|
||||
absl::optional<double> echo_return_loss_enhancement;
|
||||
// Fraction of time that the AEC linear filter is divergent, in a 1-second
|
||||
// non-overlapped aggregation window.
|
||||
absl::optional<double> divergent_filter_fraction;
|
||||
|
||||
// The delay metrics consists of the delay median and standard deviation. It
|
||||
// also consists of the fraction of delay estimates that can make the echo
|
||||
// cancellation perform poorly. The values are aggregated until the first
|
||||
// call to `GetStatistics()` and afterwards aggregated and updated every
|
||||
// second. Note that if there are several clients pulling metrics from
|
||||
// `GetStatistics()` during a session the first call from any of them will
|
||||
// change to one second aggregation window for all.
|
||||
absl::optional<int32_t> delay_median_ms;
|
||||
absl::optional<int32_t> delay_standard_deviation_ms;
|
||||
|
||||
// Residual echo detector likelihood.
|
||||
absl::optional<double> residual_echo_likelihood;
|
||||
// Maximum residual echo likelihood from the last time period.
|
||||
absl::optional<double> residual_echo_likelihood_recent_max;
|
||||
|
||||
// The instantaneous delay estimate produced in the AEC. The unit is in
|
||||
// milliseconds and the value is the instantaneous value at the time of the
|
||||
// call to `GetStatistics()`.
|
||||
absl::optional<int32_t> delay_ms;
|
||||
};
|
||||
|
||||
} // namespace webrtc
|
||||
|
||||
#endif // MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_STATISTICS_H_
|
||||
|
|
@ -0,0 +1,180 @@
|
|||
/*
|
||||
* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#ifndef MODULES_AUDIO_PROCESSING_INCLUDE_MOCK_AUDIO_PROCESSING_H_
|
||||
#define MODULES_AUDIO_PROCESSING_INCLUDE_MOCK_AUDIO_PROCESSING_H_
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "absl/base/nullability.h"
|
||||
#include "absl/strings/string_view.h"
|
||||
#include "api/task_queue/task_queue_base.h"
|
||||
#include "modules/audio_processing/include/aec_dump.h"
|
||||
#include "modules/audio_processing/include/audio_processing.h"
|
||||
#include "modules/audio_processing/include/audio_processing_statistics.h"
|
||||
#include "test/gmock.h"
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
namespace test {
|
||||
class MockCustomProcessing : public CustomProcessing {
|
||||
public:
|
||||
virtual ~MockCustomProcessing() {}
|
||||
MOCK_METHOD(void,
|
||||
Initialize,
|
||||
(int sample_rate_hz, int num_channels),
|
||||
(override));
|
||||
MOCK_METHOD(void, Process, (AudioBuffer * audio), (override));
|
||||
MOCK_METHOD(void,
|
||||
SetRuntimeSetting,
|
||||
(AudioProcessing::RuntimeSetting setting),
|
||||
(override));
|
||||
MOCK_METHOD(std::string, ToString, (), (const, override));
|
||||
};
|
||||
|
||||
class MockCustomAudioAnalyzer : public CustomAudioAnalyzer {
|
||||
public:
|
||||
virtual ~MockCustomAudioAnalyzer() {}
|
||||
MOCK_METHOD(void,
|
||||
Initialize,
|
||||
(int sample_rate_hz, int num_channels),
|
||||
(override));
|
||||
MOCK_METHOD(void, Analyze, (const AudioBuffer* audio), (override));
|
||||
MOCK_METHOD(std::string, ToString, (), (const, override));
|
||||
};
|
||||
|
||||
class MockEchoControl : public EchoControl {
|
||||
public:
|
||||
virtual ~MockEchoControl() {}
|
||||
MOCK_METHOD(void, AnalyzeRender, (AudioBuffer * render), (override));
|
||||
MOCK_METHOD(void, AnalyzeCapture, (AudioBuffer * capture), (override));
|
||||
MOCK_METHOD(void,
|
||||
ProcessCapture,
|
||||
(AudioBuffer * capture, bool echo_path_change),
|
||||
(override));
|
||||
MOCK_METHOD(void,
|
||||
ProcessCapture,
|
||||
(AudioBuffer * capture,
|
||||
AudioBuffer* linear_output,
|
||||
bool echo_path_change),
|
||||
(override));
|
||||
MOCK_METHOD(Metrics, GetMetrics, (), (const, override));
|
||||
MOCK_METHOD(void, SetAudioBufferDelay, (int delay_ms), (override));
|
||||
MOCK_METHOD(bool, ActiveProcessing, (), (const, override));
|
||||
};
|
||||
|
||||
class MockEchoDetector : public EchoDetector {
|
||||
public:
|
||||
virtual ~MockEchoDetector() {}
|
||||
MOCK_METHOD(void,
|
||||
Initialize,
|
||||
(int capture_sample_rate_hz,
|
||||
int num_capture_channels,
|
||||
int render_sample_rate_hz,
|
||||
int num_render_channels),
|
||||
(override));
|
||||
MOCK_METHOD(void,
|
||||
AnalyzeRenderAudio,
|
||||
(rtc::ArrayView<const float> render_audio),
|
||||
(override));
|
||||
MOCK_METHOD(void,
|
||||
AnalyzeCaptureAudio,
|
||||
(rtc::ArrayView<const float> capture_audio),
|
||||
(override));
|
||||
MOCK_METHOD(Metrics, GetMetrics, (), (const, override));
|
||||
};
|
||||
|
||||
class MockAudioProcessing : public AudioProcessing {
|
||||
public:
|
||||
MockAudioProcessing() {}
|
||||
|
||||
virtual ~MockAudioProcessing() {}
|
||||
|
||||
MOCK_METHOD(int, Initialize, (), (override));
|
||||
MOCK_METHOD(int,
|
||||
Initialize,
|
||||
(const ProcessingConfig& processing_config),
|
||||
(override));
|
||||
MOCK_METHOD(void, ApplyConfig, (const Config& config), (override));
|
||||
MOCK_METHOD(int, proc_sample_rate_hz, (), (const, override));
|
||||
MOCK_METHOD(int, proc_split_sample_rate_hz, (), (const, override));
|
||||
MOCK_METHOD(size_t, num_input_channels, (), (const, override));
|
||||
MOCK_METHOD(size_t, num_proc_channels, (), (const, override));
|
||||
MOCK_METHOD(size_t, num_output_channels, (), (const, override));
|
||||
MOCK_METHOD(size_t, num_reverse_channels, (), (const, override));
|
||||
MOCK_METHOD(void, set_output_will_be_muted, (bool muted), (override));
|
||||
MOCK_METHOD(void, SetRuntimeSetting, (RuntimeSetting setting), (override));
|
||||
MOCK_METHOD(bool, PostRuntimeSetting, (RuntimeSetting setting), (override));
|
||||
MOCK_METHOD(int,
|
||||
ProcessStream,
|
||||
(const int16_t* const src,
|
||||
const StreamConfig& input_config,
|
||||
const StreamConfig& output_config,
|
||||
int16_t* const dest),
|
||||
(override));
|
||||
MOCK_METHOD(int,
|
||||
ProcessStream,
|
||||
(const float* const* src,
|
||||
const StreamConfig& input_config,
|
||||
const StreamConfig& output_config,
|
||||
float* const* dest),
|
||||
(override));
|
||||
MOCK_METHOD(int,
|
||||
ProcessReverseStream,
|
||||
(const int16_t* const src,
|
||||
const StreamConfig& input_config,
|
||||
const StreamConfig& output_config,
|
||||
int16_t* const dest),
|
||||
(override));
|
||||
MOCK_METHOD(int,
|
||||
AnalyzeReverseStream,
|
||||
(const float* const* data, const StreamConfig& reverse_config),
|
||||
(override));
|
||||
MOCK_METHOD(int,
|
||||
ProcessReverseStream,
|
||||
(const float* const* src,
|
||||
const StreamConfig& input_config,
|
||||
const StreamConfig& output_config,
|
||||
float* const* dest),
|
||||
(override));
|
||||
MOCK_METHOD(bool,
|
||||
GetLinearAecOutput,
|
||||
((rtc::ArrayView<std::array<float, 160>> linear_output)),
|
||||
(const, override));
|
||||
MOCK_METHOD(int, set_stream_delay_ms, (int delay), (override));
|
||||
MOCK_METHOD(int, stream_delay_ms, (), (const, override));
|
||||
MOCK_METHOD(void, set_stream_key_pressed, (bool key_pressed), (override));
|
||||
MOCK_METHOD(void, set_stream_analog_level, (int), (override));
|
||||
MOCK_METHOD(int, recommended_stream_analog_level, (), (const, override));
|
||||
MOCK_METHOD(bool,
|
||||
CreateAndAttachAecDump,
|
||||
(absl::string_view file_name,
|
||||
int64_t max_log_size_bytes,
|
||||
absl::Nonnull<TaskQueueBase*> worker_queue),
|
||||
(override));
|
||||
MOCK_METHOD(bool,
|
||||
CreateAndAttachAecDump,
|
||||
(FILE * handle,
|
||||
int64_t max_log_size_bytes,
|
||||
absl::Nonnull<TaskQueueBase*> worker_queue),
|
||||
(override));
|
||||
MOCK_METHOD(void, AttachAecDump, (std::unique_ptr<AecDump>), (override));
|
||||
MOCK_METHOD(void, DetachAecDump, (), (override));
|
||||
|
||||
MOCK_METHOD(AudioProcessingStats, GetStatistics, (), (override));
|
||||
MOCK_METHOD(AudioProcessingStats, GetStatistics, (bool), (override));
|
||||
|
||||
MOCK_METHOD(AudioProcessing::Config, GetConfig, (), (const, override));
|
||||
};
|
||||
|
||||
} // namespace test
|
||||
} // namespace webrtc
|
||||
|
||||
#endif // MODULES_AUDIO_PROCESSING_INCLUDE_MOCK_AUDIO_PROCESSING_H_
|
||||
Loading…
Add table
Add a link
Reference in a new issue