[video_player_avfoundation, camera_avfoundation] never overwrite but …

…only upgrade audio session category (#7143)

Setting category of `AVAudioSession` was moved into a wrapper which ensures that only changes which do not disable the ability to play in silent mode (`play`) and ability to record (`record`) are considered. If either the new category or old category is `play`/`record` then also the set category will be `play`/`record`. This currently means that the video player will not overwrite `PlayAndRecord` with `Playback` which causes inability to record audio by camera. Options are treated selectively so the video player will no longer overwrite flags set by camera like `DefaultToSpeaker` and camera will not overwrite `MixWithOthers` set by video player `setMixWithOthers`. It will also only change category or options if there is change to prevent lags every time is constructed `VideoPlayerController` with non-null `videoPlayerOptions` which always causes call to `setMixWithOthers`.

Test `testSeekToWhilePausedStartsDisplayLinkTemporarily` is failing on the main branch on the tested device:
``` objc
XCTAssertEqual([player position], 1234); // (([player position]) equal to (1234)) failed: ("0") is not equal to ("1234")
```

Fixes flutter/flutter#131553

packages/camera/camera_avfoundation/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.9.17+7
+
+* Fixes changing global audio session category to be collision free across plugins.
+
 ## 0.9.17+6
 
 * Updates minimum supported SDK version to Flutter 3.22/Dart 3.4.

packages/camera/camera_avfoundation/example/ios/RunnerTests/FLTCamSampleBufferTests.m

@@ -338,4 +338,27 @@ - (void)testStartWritingShouldNotBeCalledBetweenSampleCreationAndAppending {
   CFRelease(videoSample);
 }
 
+- (void)testStartVideoRecordingWithCompletionShouldNotDisableMixWithOthers {
+  FLTCam *cam = FLTCreateCamWithCaptureSessionQueue(dispatch_queue_create("testing", NULL));
+
+  id writerMock = OCMClassMock([AVAssetWriter class]);
+  OCMStub([writerMock alloc]).andReturn(writerMock);
+  OCMStub([writerMock initWithURL:OCMOCK_ANY fileType:OCMOCK_ANY error:[OCMArg setTo:nil]])
+      .andReturn(writerMock);
+
+  [AVAudioSession.sharedInstance setCategory:AVAudioSessionCategoryPlayback
+                                 withOptions:AVAudioSessionCategoryOptionMixWithOthers
+                                       error:nil];
+
+  [cam
+      startVideoRecordingWithCompletion:^(FlutterError *_Nullable error) {
+      }
+                  messengerForStreaming:nil];
+  XCTAssert(
+      AVAudioSession.sharedInstance.categoryOptions & AVAudioSessionCategoryOptionMixWithOthers,
+      @"Flag MixWithOthers was removed.");
+  XCTAssert(AVAudioSession.sharedInstance.category == AVAudioSessionCategoryPlayAndRecord,
+            @"Category should be PlayAndRecord.");
+}
+
 @end

packages/camera/camera_avfoundation/ios/camera_avfoundation/Sources/camera_avfoundation/CameraPlugin.m

@@ -234,7 +234,7 @@ - (void)prepareForVideoRecordingWithCompletion:
     (nonnull void (^)(FlutterError *_Nullable))completion {
   __weak typeof(self) weakSelf = self;
   dispatch_async(self.captureSessionQueue, ^{
-    [weakSelf.camera setUpCaptureSessionForAudio];
+    [weakSelf.camera setUpCaptureSessionForAudioIfNeeded];
     completion(nil);
   });
 }

packages/camera/camera_avfoundation/ios/camera_avfoundation/Sources/camera_avfoundation/FLTCam.m

@@ -237,6 +237,8 @@ - (instancetype)initWithMediaSettings:(FCPPlatformMediaSettings *)mediaSettings
   _videoFormat = kCVPixelFormatType_32BGRA;
   _inProgressSavePhotoDelegates = [NSMutableDictionary dictionary];
   _fileFormat = FCPPlatformImageFileFormatJpeg;
+  _videoCaptureSession.automaticallyConfiguresApplicationAudioSession = NO;
+  _audioCaptureSession.automaticallyConfiguresApplicationAudioSession = NO;
 
   // To limit memory consumption, limit the number of frames pending processing.
   // After some testing, 4 was determined to be the best maximum value.
@@ -725,7 +727,8 @@ - (void)captureOutput:(AVCaptureOutput *)output
     if (_isFirstVideoSample) {
       [_videoWriter startSessionAtSourceTime:currentSampleTime];
       // fix sample times not being numeric when pause/resume happens before first sample buffer
-      // arrives https://github.com/flutter/flutter/issues/132014
+      // arrives
+      // https://github.com/flutter/flutter/issues/132014
       _lastVideoSampleTime = currentSampleTime;
       _lastAudioSampleTime = currentSampleTime;
       _isFirstVideoSample = NO;
@@ -1283,9 +1286,7 @@ - (BOOL)setupWriterForPath:(NSString *)path {
     return NO;
   }
 
-  if (_mediaSettings.enableAudio && !_isAudioSetup) {
-    [self setUpCaptureSessionForAudio];
-  }
+  [self setUpCaptureSessionForAudioIfNeeded];
 
   _videoWriter = [[AVAssetWriter alloc] initWithURL:outputURL
                                            fileType:AVFileTypeMPEG4
@@ -1365,9 +1366,42 @@ - (BOOL)setupWriterForPath:(NSString *)path {
   return YES;
 }
 
-- (void)setUpCaptureSessionForAudio {
+// This function, although slightly modified, is also in video_player_avfoundation.
+// Both need to do the same thing and run on the same thread (for example main thread).
+// Configure application wide audio session manually to prevent overwriting flag
+// MixWithOthers by capture session.
+// Only change category if it is considered an upgrade which means it can only enable
+// ability to play in silent mode or ability to record audio but never disables it,
+// that could affect other plugins which depend on this global state. Only change
+// category or options if there is change to prevent unnecessary lags and silence.
+static void upgradeAudioSessionCategory(AVAudioSessionCategory requestedCategory,
+                                        AVAudioSessionCategoryOptions options) {
+  NSSet *playCategories = [NSSet
+      setWithObjects:AVAudioSessionCategoryPlayback, AVAudioSessionCategoryPlayAndRecord, nil];
+  NSSet *recordCategories =
+      [NSSet setWithObjects:AVAudioSessionCategoryRecord, AVAudioSessionCategoryPlayAndRecord, nil];
+  NSSet *requiredCategories =
+      [NSSet setWithObjects:requestedCategory, AVAudioSession.sharedInstance.category, nil];
+  BOOL requiresPlay = [requiredCategories intersectsSet:playCategories];
+  BOOL requiresRecord = [requiredCategories intersectsSet:recordCategories];
+  if (requiresPlay && requiresRecord) {
+    requestedCategory = AVAudioSessionCategoryPlayAndRecord;
+  } else if (requiresPlay) {
+    requestedCategory = AVAudioSessionCategoryPlayback;
+  } else if (requiresRecord) {
+    requestedCategory = AVAudioSessionCategoryRecord;
+  }
+  options = AVAudioSession.sharedInstance.categoryOptions | options;
+  if ([requestedCategory isEqualToString:AVAudioSession.sharedInstance.category] &&
+      options == AVAudioSession.sharedInstance.categoryOptions) {
+    return;
+  }
+  [AVAudioSession.sharedInstance setCategory:requestedCategory withOptions:options error:nil];
+}
+
+- (void)setUpCaptureSessionForAudioIfNeeded {
   // Don't setup audio twice or we will lose the audio.
-  if (_isAudioSetup) {
+  if (!_mediaSettings.enableAudio || _isAudioSetup) {
     return;
   }
 
@@ -1383,6 +1417,20 @@ - (void)setUpCaptureSessionForAudio {
   // Setup the audio output.
   _audioOutput = [[AVCaptureAudioDataOutput alloc] init];
 
+  dispatch_block_t block = ^{
+    // Set up options implicit to AVAudioSessionCategoryPlayback to avoid conflicts with other
+    // plugins like video_player.
+    upgradeAudioSessionCategory(AVAudioSessionCategoryPlayAndRecord,
+                                AVAudioSessionCategoryOptionDefaultToSpeaker |
+                                    AVAudioSessionCategoryOptionAllowBluetoothA2DP |
+                                    AVAudioSessionCategoryOptionAllowAirPlay);
+  };
+  if (!NSThread.isMainThread) {
+    dispatch_sync(dispatch_get_main_queue(), block);
+  } else {
+    block();
+  }
+
   if ([_audioCaptureSession canAddInput:audioInput]) {
     [_audioCaptureSession addInput:audioInput];
 

packages/camera/camera_avfoundation/ios/camera_avfoundation/Sources/camera_avfoundation/include/camera_avfoundation/FLTCam.h

@@ -113,7 +113,7 @@ NS_ASSUME_NONNULL_BEGIN
 - (void)startImageStreamWithMessenger:(NSObject<FlutterBinaryMessenger> *)messenger;
 - (void)stopImageStream;
 - (void)setZoomLevel:(CGFloat)zoom withCompletion:(void (^)(FlutterError *_Nullable))completion;
-- (void)setUpCaptureSessionForAudio;
+- (void)setUpCaptureSessionForAudioIfNeeded;
 
 @end
 

packages/camera/camera_avfoundation/pubspec.yaml

@@ -2,7 +2,7 @@ name: camera_avfoundation
 description: iOS implementation of the camera plugin.
 repository: https://github.com/flutter/packages/tree/main/packages/camera/camera_avfoundation
 issue_tracker: https://github.com/flutter/flutter/issues?q=is%3Aissue+is%3Aopen+label%3A%22p%3A+camera%22
-version: 0.9.17+6
+version: 0.9.17+7
 
 environment:
   sdk: ^3.4.0

packages/video_player/video_player_avfoundation/CHANGELOG.md

@@ -1,5 +1,6 @@
-## NEXT
+## 2.6.6
 
+* Fixes changing global audio session category to be collision free across plugins.
 * Updates minimum supported SDK version to Flutter 3.22/Dart 3.4.
 
 ## 2.6.5

packages/video_player/video_player_avfoundation/darwin/RunnerTests/VideoPlayerTests.m

@@ -840,6 +840,40 @@ - (void)testFailedToLoadVideoEventShouldBeAlwaysSent {
 }
 
 #if TARGET_OS_IOS
+- (void)testVideoPlayerShouldNotOverwritePlayAndRecordNorDefaultToSpeaker {
+  NSObject<FlutterPluginRegistrar> *registrar = [GetPluginRegistry()
+      registrarForPlugin:@"testVideoPlayerShouldNotOverwritePlayAndRecordNorDefaultToSpeaker"];
+  FVPVideoPlayerPlugin *videoPlayerPlugin =
+      [[FVPVideoPlayerPlugin alloc] initWithRegistrar:registrar];
+  FlutterError *error;
+
+  [AVAudioSession.sharedInstance setCategory:AVAudioSessionCategoryPlayAndRecord
+                                 withOptions:AVAudioSessionCategoryOptionDefaultToSpeaker
+                                       error:nil];
+
+  [videoPlayerPlugin initialize:&error];
+  [videoPlayerPlugin setMixWithOthers:true error:&error];
+  XCTAssert(AVAudioSession.sharedInstance.category == AVAudioSessionCategoryPlayAndRecord,
+            @"Category should be PlayAndRecord.");
+  XCTAssert(
+      AVAudioSession.sharedInstance.categoryOptions & AVAudioSessionCategoryOptionDefaultToSpeaker,
+      @"Flag DefaultToSpeaker was removed.");
+  XCTAssert(
+      AVAudioSession.sharedInstance.categoryOptions & AVAudioSessionCategoryOptionMixWithOthers,
+      @"Flag MixWithOthers should be set.");
+
+  id sessionMock = OCMClassMock([AVAudioSession class]);
+  OCMStub([sessionMock sharedInstance]).andReturn(sessionMock);
+  OCMStub([sessionMock category]).andReturn(AVAudioSessionCategoryPlayAndRecord);
+  OCMStub([sessionMock categoryOptions])
+      .andReturn(AVAudioSessionCategoryOptionMixWithOthers |
+                 AVAudioSessionCategoryOptionDefaultToSpeaker);
+  OCMReject([sessionMock setCategory:OCMOCK_ANY withOptions:0 error:[OCMArg setTo:nil]])
+      .ignoringNonObjectArgs();
+
+  [videoPlayerPlugin setMixWithOthers:true error:&error];
+}
+
 - (void)validateTransformFixForOrientation:(UIImageOrientation)orientation {
   AVAssetTrack *track = [[FakeAVAssetTrack alloc] initWithOrientation:orientation];
   CGAffineTransform t = FVPGetStandardizedTransformForTrack(track);

packages/video_player/video_player_avfoundation/darwin/video_player_avfoundation/Sources/video_player_avfoundation/FVPVideoPlayerPlugin.m

@@ -90,10 +90,46 @@ - (int64_t)onPlayerSetup:(FVPVideoPlayer *)player frameUpdater:(FVPFrameUpdater
   return textureId;
 }
 
+// This function, although slightly modified, is also in camera_avfoundation.
+// Both need to do the same thing and run on the same thread (for example main thread).
+// Do not overwrite PlayAndRecord with Playback which causes inability to record
+// audio, do not overwrite all options.
+// Only change category if it is considered an upgrade which means it can only enable
+// ability to play in silent mode or ability to record audio but never disables it,
+// that could affect other plugins which depend on this global state. Only change
+// category or options if there is change to prevent unnecessary lags and silence.
+#if TARGET_OS_IOS
+static void upgradeAudioSessionCategory(AVAudioSessionCategory requestedCategory,
+                                        AVAudioSessionCategoryOptions options,
+                                        AVAudioSessionCategoryOptions clearOptions) {
+  NSSet *playCategories = [NSSet
+      setWithObjects:AVAudioSessionCategoryPlayback, AVAudioSessionCategoryPlayAndRecord, nil];
+  NSSet *recordCategories =
+      [NSSet setWithObjects:AVAudioSessionCategoryRecord, AVAudioSessionCategoryPlayAndRecord, nil];
+  NSSet *requiredCategories =
+      [NSSet setWithObjects:requestedCategory, AVAudioSession.sharedInstance.category, nil];
+  BOOL requiresPlay = [requiredCategories intersectsSet:playCategories];
+  BOOL requiresRecord = [requiredCategories intersectsSet:recordCategories];
+  if (requiresPlay && requiresRecord) {
+    requestedCategory = AVAudioSessionCategoryPlayAndRecord;
+  } else if (requiresPlay) {
+    requestedCategory = AVAudioSessionCategoryPlayback;
+  } else if (requiresRecord) {
+    requestedCategory = AVAudioSessionCategoryRecord;
+  }
+  options = (AVAudioSession.sharedInstance.categoryOptions & ~clearOptions) | options;
+  if ([requestedCategory isEqualToString:AVAudioSession.sharedInstance.category] &&
+      options == AVAudioSession.sharedInstance.categoryOptions) {
+    return;
+  }
+  [AVAudioSession.sharedInstance setCategory:requestedCategory withOptions:options error:nil];
+}
+#endif
+
 - (void)initialize:(FlutterError *__autoreleasing *)error {
 #if TARGET_OS_IOS
   // Allow audio playback when the Ring/Silent switch is set to silent
-  [[AVAudioSession sharedInstance] setCategory:AVAudioSessionCategoryPlayback error:nil];
+  upgradeAudioSessionCategory(AVAudioSessionCategoryPlayback, 0, 0);
 #endif
 
   [self.playersByTextureId
@@ -204,11 +240,11 @@ - (void)setMixWithOthers:(BOOL)mixWithOthers
   // AVAudioSession doesn't exist on macOS, and audio always mixes, so just no-op.
 #else
   if (mixWithOthers) {
-    [[AVAudioSession sharedInstance] setCategory:AVAudioSessionCategoryPlayback
-                                     withOptions:AVAudioSessionCategoryOptionMixWithOthers
-                                           error:nil];
+    upgradeAudioSessionCategory(AVAudioSession.sharedInstance.category,
+                                AVAudioSessionCategoryOptionMixWithOthers, 0);
   } else {
-    [[AVAudioSession sharedInstance] setCategory:AVAudioSessionCategoryPlayback error:nil];
+    upgradeAudioSessionCategory(AVAudioSession.sharedInstance.category, 0,
+                                AVAudioSessionCategoryOptionMixWithOthers);
   }
 #endif
 }

packages/video_player/video_player_avfoundation/pubspec.yaml

@@ -2,7 +2,7 @@ name: video_player_avfoundation
 description: iOS and macOS implementation of the video_player plugin.
 repository: https://github.com/flutter/packages/tree/main/packages/video_player/video_player_avfoundation
 issue_tracker: https://github.com/flutter/flutter/issues?q=is%3Aissue+is%3Aopen+label%3A%22p%3A+video_player%22
-version: 2.6.5
+version: 2.6.6
 
 environment:
   sdk: ^3.4.0