Enable new getUserMedia audio constraints algorithm behind a flag.

content/renderer/media/media_stream_audio_processor.cc

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/renderer/media/media_stream_audio_processor.h"
 
 #include <stddef.h>
 #include <stdint.h>
+#include <algorithm>
+#include <limits>
+#include <string>
 #include <utility>
+#include <vector>
 
 #include "base/command_line.h"
 #include "base/feature_list.h"
 #include "base/metrics/field_trial.h"
 #include "base/metrics/histogram_macros.h"
 #include "base/optional.h"
 #include "base/single_thread_task_runner.h"
 #include "base/strings/string_number_conversions.h"
 #include "base/threading/thread_task_runner_handle.h"
 #include "base/trace_event/trace_event.h"
 #include "build/build_config.h"
 #include "content/public/common/content_features.h"
 #include "content/public/common/content_switches.h"
-#include "content/renderer/media/media_stream_audio_processor_options.h"
 #include "content/renderer/media/webrtc_audio_device_impl.h"
 #include "media/base/audio_converter.h"
 #include "media/base/audio_fifo.h"
 #include "media/base/audio_parameters.h"
 #include "media/base/channel_layout.h"
-#include "third_party/WebKit/public/platform/WebMediaConstraints.h"
 #include "third_party/webrtc/api/mediaconstraintsinterface.h"
 #include "third_party/webrtc/modules/audio_processing/typing_detection.h"
 
 namespace content {
 
 namespace {
 
 using webrtc::AudioProcessing;
 using webrtc::NoiseSuppression;
 
@@ skipping 81 matching lines @@
   return base::Optional<int>(startup_min_volume);
 }
 
 // Checks if the AEC's refined adaptive filter tuning was enabled on the command
 // line.
 bool UseAecRefinedAdaptiveFilter() {
   return base::CommandLine::ForCurrentProcess()->HasSwitch(
       switches::kAecRefinedAdaptiveFilter);
 }
 
+webrtc::Point WebrtcPointFromMediaPoint(const media::Point& point) {
+  return webrtc::Point(point.x(), point.y(), point.z());
+}
+
+std::vector<webrtc::Point> WebrtcPointsFromMediaPoints(
+    const std::vector<media::Point>& points) {
+  std::vector<webrtc::Point> webrtc_points;
+  webrtc_points.reserve(webrtc_points.size());
+  for (const auto& point : points)
+    webrtc_points.push_back(WebrtcPointFromMediaPoint(point));
+  return webrtc_points;
+}
+
 }  // namespace
 
 // Wraps AudioBus to provide access to the array of channel pointers, since this
 // is the type webrtc::AudioProcessing deals in. The array is refreshed on every
 // channel_ptrs() call, and will be valid until the underlying AudioBus pointers
 // are changed, e.g. through calls to SetChannelData() or SwapChannels().
 //
 // All methods are called on one of the capture or render audio threads
 // exclusively.
 class MediaStreamAudioBus {
@@ skipping 144 matching lines @@
   // consumed next from the FIFO.  When not using |fifo_|, this is the audio
   // delay of the first sample in |destination_|.
   base::TimeDelta next_audio_delay_;
 
   // True when |destination_| contains the data to be returned by the next call
   // to Consume().  Only used when the FIFO is disabled.
   bool data_available_;
 };
 
 MediaStreamAudioProcessor::MediaStreamAudioProcessor(
-    const blink::WebMediaConstraints& constraints,
-    const MediaStreamDevice::AudioDeviceParameters& input_params,
+    const AudioProcessingProperties& properties,
     WebRtcPlayoutDataSource* playout_data_source)
     : render_delay_ms_(0),
       has_echo_cancellation_(false),
       playout_data_source_(playout_data_source),
       main_thread_runner_(base::ThreadTaskRunnerHandle::Get()),
       audio_mirroring_(false),
       typing_detected_(false),
       stopped_(false) {
   DCHECK(main_thread_runner_);
   capture_thread_checker_.DetachFromThread();
   render_thread_checker_.DetachFromThread();
-  InitializeAudioProcessingModule(constraints, input_params);
+  InitializeAudioProcessingModule(properties);
 
   aec_dump_message_filter_ = AecDumpMessageFilter::Get();
   // In unit tests not creating a message filter, |aec_dump_message_filter_|
   // will be NULL. We can just ignore that. Other unit tests and browser tests
   // ensure that we do get the filter when we should.
   if (aec_dump_message_filter_.get())
     aec_dump_message_filter_->AddDelegate(this);
 
   // Create and configure |audio_repetition_detector_|.
 #if ENABLE_AUDIO_REPETITION_DETECTOR
@@ skipping 165 matching lines @@
   }
 }
 
 void MediaStreamAudioProcessor::OnIpcClosing() {
   DCHECK(main_thread_runner_->BelongsToCurrentThread());
   aec_dump_message_filter_ = NULL;
 }
 
 // static
 bool MediaStreamAudioProcessor::WouldModifyAudio(
-    const blink::WebMediaConstraints& constraints,
-    int effects_flags) {
+    const AudioProcessingProperties& properties) {
   // Note: This method should by kept in-sync with any changes to the logic in
   // MediaStreamAudioProcessor::InitializeAudioProcessingModule().
 
-  const MediaAudioConstraints audio_constraints(constraints, effects_flags);
-
-  if (audio_constraints.GetGoogAudioMirroring())
+  if (properties.goog_audio_mirroring)
     return true;
 
 #if !defined(OS_IOS)
-  if (audio_constraints.GetEchoCancellationProperty() ||
-      audio_constraints.GetGoogAutoGainControl()) {
+  if (properties.enable_sw_echo_cancellation ||
+      properties.goog_auto_gain_control) {
     return true;
   }
 #endif
 
 #if !defined(OS_IOS) && !defined(OS_ANDROID)
-  if (audio_constraints.GetGoogExperimentalEchoCancellation() ||
-      audio_constraints.GetGoogTypingNoiseDetection()) {
+  if (properties.goog_experimental_echo_cancellation ||
+      properties.goog_typing_noise_detection) {
     return true;
   }
 #endif
 
-  if (audio_constraints.GetGoogNoiseSuppression() ||
-      audio_constraints.GetGoogExperimentalNoiseSuppression() ||
-      audio_constraints.GetGoogBeamforming() ||
-      audio_constraints.GetGoogHighpassFilter()) {
+  if (properties.goog_noise_suppression ||
+      properties.goog_experimental_noise_suppression ||
+      properties.goog_beamforming || properties.goog_highpass_filter) {
     return true;
   }
 
   return false;
 }
 
 void MediaStreamAudioProcessor::OnPlayoutData(media::AudioBus* audio_bus,
                                               int sample_rate,
                                               int audio_delay_milliseconds) {
   DCHECK(render_thread_checker_.CalledOnValidThread());
@@ skipping 39 matching lines @@
   render_fifo_->ReattachThreadChecker();
 }
 
 void MediaStreamAudioProcessor::GetStats(AudioProcessorStats* stats) {
   stats->typing_noise_detected =
       (base::subtle::Acquire_Load(&typing_detected_) != false);
   GetAudioProcessingStats(audio_processing_.get(), stats);
 }
 
 void MediaStreamAudioProcessor::InitializeAudioProcessingModule(
-    const blink::WebMediaConstraints& constraints,
-    const MediaStreamDevice::AudioDeviceParameters& input_params) {
+    const AudioProcessingProperties& properties) {
   DCHECK(main_thread_runner_->BelongsToCurrentThread());
   DCHECK(!audio_processing_);
 
-  MediaAudioConstraints audio_constraints(constraints, input_params.effects);
-
   // Note: The audio mirroring constraint (i.e., swap left and right channels)
   // is handled within this MediaStreamAudioProcessor and does not, by itself,
   // require webrtc::AudioProcessing.
-  audio_mirroring_ = audio_constraints.GetGoogAudioMirroring();
-
-  const bool echo_cancellation =
-      audio_constraints.GetEchoCancellationProperty();
-  has_echo_cancellation_ = echo_cancellation;
-  const bool goog_agc = audio_constraints.GetGoogAutoGainControl();
+  audio_mirroring_ = properties.goog_audio_mirroring;
+  has_echo_cancellation_ = properties.enable_sw_echo_cancellation;
 
 #if defined(OS_ANDROID)
   const bool goog_experimental_aec = false;
   const bool goog_typing_detection = false;
 #else
   const bool goog_experimental_aec =
-      audio_constraints.GetGoogExperimentalEchoCancellation();
-  const bool goog_typing_detection =
-      audio_constraints.GetGoogTypingNoiseDetection();
+      properties.goog_experimental_echo_cancellation;
+  const bool goog_typing_detection = properties.goog_typing_noise_detection;
 #endif
 
-  const bool goog_ns = audio_constraints.GetGoogNoiseSuppression();
-  const bool goog_experimental_ns =
-      audio_constraints.GetGoogExperimentalNoiseSuppression();
-  const bool goog_beamforming = audio_constraints.GetGoogBeamforming();
-  const bool goog_high_pass_filter = audio_constraints.GetGoogHighpassFilter();
-
   // Return immediately if none of the goog constraints requiring
   // webrtc::AudioProcessing are enabled.
-  if (!echo_cancellation && !goog_experimental_aec && !goog_ns &&
-      !goog_high_pass_filter && !goog_typing_detection &&
-      !goog_agc && !goog_experimental_ns && !goog_beamforming) {
+  if (!properties.enable_sw_echo_cancellation && !goog_experimental_aec &&
+      !properties.goog_noise_suppression && !properties.goog_highpass_filter &&
+      !goog_typing_detection && !properties.goog_auto_gain_control &&
+      !properties.goog_experimental_noise_suppression &&
+      !properties.goog_beamforming) {
     // Sanity-check: WouldModifyAudio() should return true iff
     // |audio_mirroring_| is true.
-    DCHECK_EQ(audio_mirroring_, WouldModifyAudio(constraints,
-                                                 input_params.effects));
+    DCHECK_EQ(audio_mirroring_, WouldModifyAudio(properties));
     RecordProcessingState(AUDIO_PROCESSING_DISABLED);
     return;
   }
 
   // Sanity-check: WouldModifyAudio() should return true because the above logic
   // has determined webrtc::AudioProcessing will be used.
-  DCHECK(WouldModifyAudio(constraints, input_params.effects));
+  DCHECK(WouldModifyAudio(properties));
 
   // Experimental options provided at creation.
   webrtc::Config config;
   config.Set<webrtc::ExtendedFilter>(
       new webrtc::ExtendedFilter(goog_experimental_aec));
-  config.Set<webrtc::ExperimentalNs>(
-      new webrtc::ExperimentalNs(goog_experimental_ns));
+  config.Set<webrtc::ExperimentalNs>(new webrtc::ExperimentalNs(
+      properties.goog_experimental_noise_suppression));
   config.Set<webrtc::DelayAgnostic>(new webrtc::DelayAgnostic(true));
   if (UseAecRefinedAdaptiveFilter()) {
     config.Set<webrtc::RefinedAdaptiveFilter>(
         new webrtc::RefinedAdaptiveFilter(true));
   }
-  if (goog_beamforming) {
-    const auto& geometry =
-        GetArrayGeometryPreferringConstraints(audio_constraints, input_params);
-
+  if (properties.goog_beamforming) {
     // Only enable beamforming if we have at least two mics.
-    config.Set<webrtc::Beamforming>(
-        new webrtc::Beamforming(geometry.size() > 1, geometry));
+    config.Set<webrtc::Beamforming>(new webrtc::Beamforming(
+        properties.goog_array_geometry.size() > 1,
+        WebrtcPointsFromMediaPoints(properties.goog_array_geometry)));
   }
 
   // If the experimental AGC is enabled, check for overridden config params.
-  if (audio_constraints.GetGoogExperimentalAutoGainControl()) {
+  if (properties.goog_experimental_auto_gain_control) {
     auto startup_min_volume = GetStartupMinVolumeForAgc();
     constexpr int kClippingLevelMin = 70;
     // TODO(hlundin) Make this value default in WebRTC and clean up here.
     config.Set<webrtc::ExperimentalAgc>(new webrtc::ExperimentalAgc(
         true, startup_min_volume.value_or(0), kClippingLevelMin));
   }
 
   // Create and configure the webrtc::AudioProcessing.
   audio_processing_.reset(webrtc::AudioProcessing::Create(config));
 
   // Enable the audio processing components.
   webrtc::AudioProcessing::Config apm_config;
 
   if (playout_data_source_) {
     playout_data_source_->AddPlayoutSink(this);
   }
 
-  if (echo_cancellation) {
+  if (properties.enable_sw_echo_cancellation) {
     EnableEchoCancellation(audio_processing_.get());
 
     apm_config.echo_canceller3.enabled = override_aec3_.value_or(
         base::FeatureList::IsEnabled(features::kWebRtcUseEchoCanceller3));
 
     if (!apm_config.echo_canceller3.enabled) {
       // Prepare for logging echo information. If there are data remaining in
       // |echo_information_| we simply discard it.
       echo_information_ = base::MakeUnique<EchoInformation>();
     } else {
       // Do not log any echo information when AEC3 is active, as the echo
       // information then will not be properly updated.
       echo_information_.reset();
     }
   } else {
     apm_config.echo_canceller3.enabled = false;
   }
 
-  if (goog_ns) {
+  if (properties.goog_noise_suppression) {
     // The beamforming postfilter is effective at suppressing stationary noise,
     // so reduce the single-channel NS aggressiveness when enabled.
     const NoiseSuppression::Level ns_level =
         config.Get<webrtc::Beamforming>().enabled ? NoiseSuppression::kLow
                                                   : NoiseSuppression::kHigh;
 
     EnableNoiseSuppression(audio_processing_.get(), ns_level);
   }
 
-  apm_config.high_pass_filter.enabled = goog_high_pass_filter;
+  apm_config.high_pass_filter.enabled = properties.goog_highpass_filter;
 
   if (goog_typing_detection) {
     // TODO(xians): Remove this |typing_detector_| after the typing suppression
     // is enabled by default.
     typing_detector_.reset(new webrtc::TypingDetection());
     EnableTypingDetection(audio_processing_.get(), typing_detector_.get());
   }
 
-  if (goog_agc)
+  if (properties.goog_auto_gain_control)
     EnableAutomaticGainControl(audio_processing_.get());
 
   audio_processing_->ApplyConfig(apm_config);
 
   RecordProcessingState(AUDIO_PROCESSING_ENABLED);
 }
 
 void MediaStreamAudioProcessor::InitializeCaptureFifo(
     const media::AudioParameters& input_format) {
   DCHECK(main_thread_runner_->BelongsToCurrentThread());
@@ skipping 152 matching lines @@
       0 : agc->stream_analog_level();
 }
 
 void MediaStreamAudioProcessor::UpdateAecStats() {
   DCHECK(main_thread_runner_->BelongsToCurrentThread());
   if (echo_information_)
     echo_information_->UpdateAecStats(audio_processing_->echo_cancellation());
 }
 
 }  // namespace content