Merge pull request #2 from webrtc-sdk/listen-only-audio-session

allow listen-only mode in AudioUnit, adjust when category changes

sdk/objc/components/audio/RTCAudioSession+Configuration.mm

@@ -55,7 +55,8 @@ - (BOOL)setConfiguration:(RTC_OBJC_TYPE(RTCAudioSessionConfiguration) *)configur
     if (![self setCategory:configuration.category
                withOptions:configuration.categoryOptions
                      error:&categoryError]) {
-      RTCLogError(@"Failed to set category: %@",
+      RTCLogError(@"Failed to set category to %@: %@",
+                  self.category,
                   categoryError.localizedDescription);
       error = categoryError;
     } else {
@@ -66,7 +67,8 @@ - (BOOL)setConfiguration:(RTC_OBJC_TYPE(RTCAudioSessionConfiguration) *)configur
   if (self.mode != configuration.mode) {
     NSError *modeError = nil;
     if (![self setMode:configuration.mode error:&modeError]) {
-      RTCLogError(@"Failed to set mode: %@",
+      RTCLogError(@"Failed to set mode to %@: %@",
+                  self.mode,
                   modeError.localizedDescription);
       error = modeError;
     } else {

sdk/objc/components/audio/RTCAudioSession+Private.h

@@ -35,6 +35,9 @@ NS_ASSUME_NONNULL_BEGIN
  */
 @property(nonatomic, assign) BOOL isInterrupted;
 
+/** if the current category could allow recording */
+@property(nonatomic, assign) BOOL isRecordingEnabled;
+
 /** Adds the delegate to the list of delegates, and places it at the front of
  *  the list. This delegate will be notified before other delegates of
  *  audio events.

sdk/objc/components/audio/RTCAudioSession.h

@@ -102,6 +102,9 @@ RTC_OBJC_EXPORT
 - (void)audioSession:(RTC_OBJC_TYPE(RTCAudioSession) *)audioSession
     audioUnitStartFailedWithError:(NSError *)error;
 
+/** Called when audio session changed from output-only to input & output */
+- (void)audioSessionWillRecord:(RTC_OBJC_TYPE(RTCAudioSession) *)audioSession;
+
 @end
 
 /** This is a protocol used to inform RTCAudioSession when the audio session

sdk/objc/components/audio/RTCAudioSession.mm

@@ -114,6 +114,8 @@ - (instancetype)initWithAudioSession:(id)audioSession {
                   options:NSKeyValueObservingOptionNew | NSKeyValueObservingOptionOld
                   context:(__bridge void *)RTC_OBJC_TYPE(RTCAudioSession).class];
 
+    self.isRecordingEnabled = [_session.category isEqualToString:AVAudioSessionCategoryPlayAndRecord];
+
     RTCLog(@"RTC_OBJC_TYPE(RTCAudioSession) (%p): init.", self);
   }
   return self;
@@ -542,6 +544,13 @@ - (void)handleRouteChangeNotification:(NSNotification *)notification {
     case AVAudioSessionRouteChangeReasonCategoryChange:
       RTCLog(@"Audio route changed: CategoryChange to :%@",
              self.session.category);
+      if (!self.isRecordingEnabled && [self.session.category isEqualToString:AVAudioSessionCategoryPlayAndRecord]) {
+        self.isRecordingEnabled = true;
+        [self notifyWillRecord];
+      }
+      if (self.isRecordingEnabled && [self.session.category isEqualToString:AVAudioSessionCategoryPlayback]) {
+        self.isRecordingEnabled = false;
+      }
       break;
     case AVAudioSessionRouteChangeReasonOverride:
       RTCLog(@"Audio route changed: Override");
@@ -773,6 +782,7 @@ - (BOOL)unconfigureWebRTCSession:(NSError **)outError {
   }
   RTCLog(@"Unconfiguring audio session for WebRTC.");
   [self setActive:NO error:outError];
+  self.isRecordingEnabled = NO;
 
   return YES;
 }
@@ -997,4 +1007,14 @@ - (void)notifyFailedToSetActive:(BOOL)active error:(NSError *)error {
   }
 }
 
+- (void)notifyWillRecord {
+  for (auto delegate : self.delegates) {
+    SEL sel = @selector(audioSessionWillRecord:);
+    if ([delegate respondsToSelector:sel]) {
+      [delegate audioSessionWillRecord:self];
+    }
+  }
+}
+
+
 @end

sdk/objc/components/audio/RTCNativeAudioSessionDelegateAdapter.mm

@@ -86,4 +86,9 @@ - (void)audioSession:(RTC_OBJC_TYPE(RTCAudioSession) *)audioSession
   _observer->OnChangedOutputVolume();
 }
 
+- (void)audioSessionWillRecord:(RTC_OBJC_TYPE(RTCAudioSession) *)session {
+  // re-trigger audio unit init, by using interrupt ended callback
+  _observer->OnAudioWillRecord();
+}
+
 @end

sdk/objc/native/src/audio/audio_device_ios.h

@@ -147,6 +147,7 @@ class AudioDeviceIOS : public AudioDeviceGeneric,
   void OnValidRouteChange() override;
   void OnCanPlayOrRecordChange(bool can_play_or_record) override;
   void OnChangedOutputVolume() override;
+  void OnAudioWillRecord() override;
 
   // VoiceProcessingAudioUnitObserver methods.
   OSStatus OnDeliverRecordedData(AudioUnitRenderActionFlags* flags,
@@ -171,6 +172,7 @@ class AudioDeviceIOS : public AudioDeviceGeneric,
   void HandleSampleRateChange();
   void HandlePlayoutGlitchDetected();
   void HandleOutputVolumeChange();
+  void HandleAudioWillRecord();
 
   // Uses current `playout_parameters_` and `record_parameters_` to inform the
   // audio device buffer (ADB) about our internal audio parameters.

sdk/objc/native/src/audio/audio_device_ios.mm

@@ -61,6 +61,16 @@
 const UInt16 kFixedPlayoutDelayEstimate = 30;
 const UInt16 kFixedRecordDelayEstimate = 30;
 
+enum AudioDeviceMessageType : uint32_t {
+  kMessageTypeInterruptionBegin,
+  kMessageTypeInterruptionEnd,
+  kMessageTypeValidRouteChange,
+  kMessageTypeCanPlayOrRecordChange,
+  kMessageTypePlayoutGlitchDetected,
+  kMessageOutputVolumeChange,
+  kMessageTypeAudioWillRecord,
+};
+
 using ios::CheckAndLogError;
 
 #if !defined(NDEBUG)
@@ -360,6 +370,11 @@ static void LogDeviceInfo() {
   thread_->PostTask(SafeTask(safety_, [this] { HandleOutputVolumeChange(); }));
 }
 
+void AudioDeviceIOS::OnAudioWillRecord() {
+  RTC_DCHECK(thread_);
+  thread_->Post(RTC_FROM_HERE, this, kMessageTypeAudioWillRecord);
+}
+
 OSStatus AudioDeviceIOS::OnDeliverRecordedData(AudioUnitRenderActionFlags* flags,
                                                const AudioTimeStamp* time_stamp,
                                                UInt32 bus_number,
@@ -445,7 +460,7 @@ static void LogDeviceInfo() {
       // Exclude extreme delta values since they do most likely not correspond
       // to a real glitch. Instead, the most probable cause is that a headset
       // has been plugged in or out. There are more direct ways to detect
-      // audio device changes (see HandleValidRouteChange()) but experiments
+      // audio device changes (see ValidRouteChange()) but experiments
       // show that using it leads to more complex implementations.
       // TODO(henrika): more tests might be needed to come up with an even
       // better upper limit.
@@ -467,6 +482,34 @@ static void LogDeviceInfo() {
   return noErr;
 }
 
+void AudioDeviceIOS::OnMessage(rtc::Message* msg) {
+  switch (msg->message_id) {
+    case kMessageTypeInterruptionBegin:
+      HandleInterruptionBegin();
+      break;
+    case kMessageTypeInterruptionEnd:
+      HandleInterruptionEnd();
+      break;
+    case kMessageTypeValidRouteChange:
+      HandleValidRouteChange();
+      break;
+    case kMessageTypeCanPlayOrRecordChange: {
+      rtc::TypedMessageData<bool>* data = static_cast<rtc::TypedMessageData<bool>*>(msg->pdata);
+      HandleCanPlayOrRecordChange(data->data());
+      delete data;
+      break;
+    }
+    case kMessageTypePlayoutGlitchDetected:
+      HandlePlayoutGlitchDetected();
+      break;
+    case kMessageOutputVolumeChange:
+      HandleOutputVolumeChange();
+      break;
+    case kMessageTypeAudioWillRecord:
+      HandleAudioWillRecord();
+  }
+}
+
 void AudioDeviceIOS::HandleInterruptionBegin() {
   RTC_DCHECK_RUN_ON(thread_);
   RTCLog(@"Interruption begin. IsInterrupted changed from %d to 1.", is_interrupted_);
@@ -633,6 +676,61 @@ static void LogDeviceInfo() {
   last_output_volume_change_time_ = rtc::TimeMillis();
 }
 
+void AudioDeviceIOS::HandleAudioWillRecord() {
+  RTC_DCHECK_RUN_ON(&thread_checker_);
+
+  LOGI() << "HandleAudioWillRecord";
+
+  // If we don't have an audio unit yet, or the audio unit is uninitialized,
+  // there is no work to do.
+  if (!audio_unit_ || audio_unit_->GetState() < VoiceProcessingAudioUnit::kInitialized) {
+    return;
+  }
+
+  // The audio unit is already initialized or started.
+  // Check to see if the sample rate or buffer size has changed.
+  RTC_OBJC_TYPE(RTCAudioSession)* session = [RTC_OBJC_TYPE(RTCAudioSession) sharedInstance];
+  const double session_sample_rate = session.sampleRate;
+
+  // Extra sanity check to ensure that the new sample rate is valid.
+  if (session_sample_rate <= 0.0) {
+    RTCLogError(@"Sample rate is invalid: %f", session_sample_rate);
+    LOGI() << "Sample rate is invalid " << session_sample_rate;
+    return;
+  }
+  // We need to adjust our format and buffer sizes.
+  // The stream format is about to be changed and it requires that we first
+  // stop and uninitialize the audio unit to deallocate its resources.
+  RTCLog(@"Stopping and uninitializing audio unit to adjust buffers.");
+  bool restart_audio_unit = false;
+  if (audio_unit_->GetState() == VoiceProcessingAudioUnit::kStarted) {
+    audio_unit_->Stop();
+    restart_audio_unit = true;
+    PrepareForNewStart();
+  }
+  if (audio_unit_->GetState() == VoiceProcessingAudioUnit::kInitialized) {
+    audio_unit_->Uninitialize();
+  }
+
+  // Allocate new buffers given the new stream format.
+  SetupAudioBuffersForActiveAudioSession();
+
+  // Initialize the audio unit again with the new sample rate.
+  RTC_DCHECK_EQ(playout_parameters_.sample_rate(), session_sample_rate);
+  if (!audio_unit_->Initialize(session_sample_rate)) {
+    RTCLogError(@"Failed to initialize the audio unit with sample rate: %f", session_sample_rate);
+    return;
+  }
+
+  // Restart the audio unit if it was already running.
+  if (restart_audio_unit && !audio_unit_->Start()) {
+    RTCLogError(@"Failed to start audio unit with sample rate: %f", session_sample_rate);
+    return;
+  }
+
+  LOGI() << "Successfully enabled audio unit for recording.";
+}
+
 void AudioDeviceIOS::UpdateAudioDeviceBuffer() {
   LOGI() << "UpdateAudioDevicebuffer";
   // AttachAudioBuffer() is called at construction by the main class but check

sdk/objc/native/src/audio/audio_session_observer.h

@@ -32,6 +32,8 @@ class AudioSessionObserver {
 
   virtual void OnChangedOutputVolume() = 0;
 
+  virtual void OnAudioWillRecord() = 0;
+
  protected:
   virtual ~AudioSessionObserver() {}
 };

sdk/objc/native/src/audio/voice_processing_audio_unit.mm

@@ -111,19 +111,6 @@ static OSStatus GetAGCState(AudioUnit audio_unit, UInt32* enabled) {
     return false;
   }
 
-  // Enable input on the input scope of the input element.
-  UInt32 enable_input = 1;
-  result = AudioUnitSetProperty(vpio_unit_, kAudioOutputUnitProperty_EnableIO,
-                                kAudioUnitScope_Input, kInputBus, &enable_input,
-                                sizeof(enable_input));
-  if (result != noErr) {
-    DisposeAudioUnit();
-    RTCLogError(@"Failed to enable input on input scope of input element. "
-                 "Error=%ld.",
-                (long)result);
-    return false;
-  }
-
   // Enable output on the output scope of the output element.
   UInt32 enable_output = 1;
   result = AudioUnitSetProperty(vpio_unit_, kAudioOutputUnitProperty_EnableIO,
@@ -204,6 +191,27 @@ static OSStatus GetAGCState(AudioUnit audio_unit, UInt32* enabled) {
   LogStreamDescription(format);
 #endif
 
+  // Enable input on the input scope of the input element.
+  // keep it disabled if audio session is configured for playback only
+  AVAudioSession* session = [AVAudioSession sharedInstance];
+  UInt32 enable_input = 0;
+  if ([session.category isEqualToString: AVAudioSessionCategoryPlayAndRecord] ||
+      [session.category isEqualToString: AVAudioSessionCategoryRecord]) {
+    enable_input = 1;
+  }
+  RTCLog(@"Initializing AudioUnit, category=%@, enable_input=%d", session.category, enable_input);
+  // LOGI() << "Initialize" << session.category << ", enable_input=" << enable_input;
+  result = AudioUnitSetProperty(vpio_unit_, kAudioOutputUnitProperty_EnableIO,
+                                kAudioUnitScope_Input, kInputBus, &enable_input,
+                                sizeof(enable_input));
+  if (result != noErr) {
+    DisposeAudioUnit();
+    RTCLogError(@"Failed to enable input on input scope of input element. "
+                 "Error=%ld.",
+                (long)result);
+    return false;
+  }
+
   // Set the format on the output scope of the input element/bus.
   result =
       AudioUnitSetProperty(vpio_unit_, kAudioUnitProperty_StreamFormat,