[camera_android] Support concurrently image capture and image streaming (#4332)

Properly configures surface needed for image capture when image streaming/recording is started to support concurrently still capture and image streaming.

Fixes flutter/flutter#125314.

Apologies for the many commits :(

Author: camsim99

packages/camera/camera_android/CHANGELOG.md

@@ -1,7 +1,8 @@
-## NEXT
+## 0.10.8+3
 
 * Fixes unawaited_futures violations.
 * Removes duplicate line in `MediaRecorderBuilder.java`.
+* Adds support for concurrently capturing images and image streaming/recording.
 
 ## 0.10.8+2
 

packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java

@@ -21,6 +21,7 @@
 import android.hardware.camera2.params.SessionConfiguration;
 import android.media.CamcorderProfile;
 import android.media.EncoderProfiles;
+import android.media.Image;
 import android.media.ImageReader;
 import android.media.MediaRecorder;
 import android.os.Build;
@@ -414,8 +415,14 @@ private void createCaptureSession(
 
     List<Surface> remainingSurfaces = Arrays.asList(surfaces);
     if (templateType != CameraDevice.TEMPLATE_PREVIEW) {
-      // If it is not preview mode, add all surfaces as targets.
+      // If it is not preview mode, add all surfaces as targets
+      // except the surface used for still capture as this should
+      // not be part of a repeating request.
+      Surface pictureImageReaderSurface = pictureImageReader.getSurface();
       for (Surface surface : remainingSurfaces) {
+        if (surface == pictureImageReaderSurface) {
+          continue;
+        }
         previewRequestBuilder.addTarget(surface);
       }
     }
@@ -539,6 +546,10 @@ private void startCapture(boolean record, boolean stream) throws CameraAccessExc
       surfaces.add(imageStreamReader.getSurface());
     }
 
+    // Add pictureImageReader surface to allow for still capture
+    // during recording/image streaming.
+    surfaces.add(pictureImageReader.getSurface());
+
     createCaptureSession(
         CameraDevice.TEMPLATE_RECORD, successCallback, surfaces.toArray(new Surface[0]));
   }
@@ -659,7 +670,6 @@ public void onCaptureCompleted(
         };
 
     try {
-      captureSession.stopRepeating();
       Log.i(TAG, "sending capture request");
       captureSession.capture(stillBuilder.build(), captureCallback, backgroundHandler);
     } catch (CameraAccessException e) {
@@ -1140,10 +1150,15 @@ public void startPreviewWithImageStream(EventChannel imageStreamChannel)
   public void onImageAvailable(ImageReader reader) {
     Log.i(TAG, "onImageAvailable");
 
+    // Use acquireNextImage since image reader is only for one image.
+    Image image = reader.acquireNextImage();
+    if (image == null) {
+      return;
+    }
+
     backgroundHandler.post(
         new ImageSaver(
-            // Use acquireNextImage since image reader is only for one image.
-            reader.acquireNextImage(),
+            image,
             captureFile,
             new ImageSaver.Callback() {
               @Override
@@ -1159,7 +1174,8 @@ public void onError(String errorCode, String errorMessage) {
     cameraCaptureCallback.setCameraState(CameraState.STATE_PREVIEW);
   }
 
-  private void prepareRecording(@NonNull Result result) {
+  @VisibleForTesting
+  void prepareRecording(@NonNull Result result) {
     final File outputDir = applicationContext.getCacheDir();
     try {
       captureFile = File.createTempFile("REC", ".mp4", outputDir);

packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/CameraTest.java

@@ -8,11 +8,14 @@
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
 import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doNothing;
 import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.mockStatic;
 import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
@@ -36,6 +39,7 @@
 import androidx.annotation.Nullable;
 import androidx.lifecycle.LifecycleObserver;
 import io.flutter.embedding.engine.systemchannels.PlatformChannel;
+import io.flutter.plugin.common.EventChannel;
 import io.flutter.plugin.common.MethodChannel;
 import io.flutter.plugins.camera.features.CameraFeatureFactory;
 import io.flutter.plugins.camera.features.CameraFeatures;
@@ -56,8 +60,10 @@
 import io.flutter.plugins.camera.features.sensororientation.DeviceOrientationManager;
 import io.flutter.plugins.camera.features.sensororientation.SensorOrientationFeature;
 import io.flutter.plugins.camera.features.zoomlevel.ZoomLevelFeature;
+import io.flutter.plugins.camera.media.ImageStreamReader;
 import io.flutter.plugins.camera.utils.TestUtils;
 import io.flutter.view.TextureRegistry;
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import org.junit.After;
@@ -638,6 +644,8 @@ public void startPreview_shouldPullStreamFromVideoRenderer()
     TestUtils.setPrivateField(camera, "videoRenderer", mockVideoRenderer);
     CameraDeviceWrapper fakeCamera = new FakeCameraDeviceWrapper(mockRequestBuilders);
     TestUtils.setPrivateField(camera, "cameraDevice", fakeCamera);
+    ImageReader mockPictureImageReader = mock(ImageReader.class);
+    TestUtils.setPrivateField(camera, "pictureImageReader", mockPictureImageReader);
 
     TextureRegistry.SurfaceTextureEntry cameraFlutterTexture =
         (TextureRegistry.SurfaceTextureEntry) TestUtils.getPrivateField(camera, "flutterTexture");
@@ -674,9 +682,10 @@ public void startPreview_shouldPullStreamFromImageReader()
 
     when(cameraFlutterTexture.surfaceTexture()).thenReturn(mockSurfaceTexture);
     when(resolutionFeature.getPreviewSize()).thenReturn(mockSize);
+    when(mockImageReader.getSurface()).thenReturn(mock(Surface.class));
 
     camera.startPreview();
-    verify(mockImageReader, times(1))
+    verify(mockImageReader, times(2)) // we expect two calls to start regular preview.
         .getSurface(); // stream pulled from regular imageReader's surface.
   }
 
@@ -692,6 +701,8 @@ public void startPreview_shouldFlipRotation() throws InterruptedException, Camer
     TestUtils.setPrivateField(camera, "initialCameraFacing", CameraMetadata.LENS_FACING_BACK);
     CameraDeviceWrapper fakeCamera = new FakeCameraDeviceWrapper(mockRequestBuilders);
     TestUtils.setPrivateField(camera, "cameraDevice", fakeCamera);
+    ImageReader mockPictureImageReader = mock(ImageReader.class);
+    TestUtils.setPrivateField(camera, "pictureImageReader", mockPictureImageReader);
 
     TextureRegistry.SurfaceTextureEntry cameraFlutterTexture =
         (TextureRegistry.SurfaceTextureEntry) TestUtils.getPrivateField(camera, "flutterTexture");
@@ -707,6 +718,39 @@ public void startPreview_shouldFlipRotation() throws InterruptedException, Camer
     verify(mockVideoRenderer, times(1)).setRotation(180);
   }
 
+  @Test
+  public void startPreviewWithImageStream_shouldPullStreamsFromImageReaders()
+      throws InterruptedException, CameraAccessException {
+    ArrayList<CaptureRequest.Builder> mockRequestBuilders = new ArrayList<>();
+    mockRequestBuilders.add(mock(CaptureRequest.Builder.class));
+    SurfaceTexture mockSurfaceTexture = mock(SurfaceTexture.class);
+    Size mockSize = mock(Size.class);
+    ImageReader mockPictureImageReader = mock(ImageReader.class);
+    ImageStreamReader mockImageStreamReader = mock(ImageStreamReader.class);
+    TestUtils.setPrivateField(camera, "recordingVideo", false);
+    TestUtils.setPrivateField(camera, "pictureImageReader", mockPictureImageReader);
+    CameraDeviceWrapper fakeCamera = new FakeCameraDeviceWrapper(mockRequestBuilders);
+    TestUtils.setPrivateField(camera, "cameraDevice", fakeCamera);
+    camera.imageStreamReader = mockImageStreamReader;
+
+    TextureRegistry.SurfaceTextureEntry cameraFlutterTexture =
+        (TextureRegistry.SurfaceTextureEntry) TestUtils.getPrivateField(camera, "flutterTexture");
+    ResolutionFeature resolutionFeature =
+        (ResolutionFeature)
+            TestUtils.getPrivateField(mockCameraFeatureFactory, "mockResolutionFeature");
+
+    when(cameraFlutterTexture.surfaceTexture()).thenReturn(mockSurfaceTexture);
+    when(resolutionFeature.getPreviewSize()).thenReturn(mockSize);
+
+    camera.startPreviewWithImageStream(mock(EventChannel.class));
+    verify(mockImageStreamReader, times(1))
+        .getSurface(); // stream pulled from image streaming imageReader's surface.
+    verify(
+            mockPictureImageReader,
+            times(2)) // we expect one call to start the capture, one to create the capture session.
+        .getSurface(); // stream pulled from regular imageReader's surface.
+  }
+
   @Test
   public void setDescriptionWhileRecording_shouldErrorWhenNotRecording() {
     MethodChannel.Result mockResult = mock(MethodChannel.Result.class);
@@ -806,6 +850,43 @@ public void setFocusMode_shouldSendErrorEventOnUnlockAutoFocusCameraAccessExcept
     verify(mockDartMessenger, times(1)).sendCameraErrorEvent(any());
   }
 
+  @Test
+  public void startVideoRecording_shouldPullStreamsFromMediaRecorderAndImageReader()
+      throws InterruptedException, IOException, CameraAccessException {
+    Camera cameraSpy = spy(camera);
+    ArrayList<CaptureRequest.Builder> mockRequestBuilders = new ArrayList<>();
+    mockRequestBuilders.add(mock(CaptureRequest.Builder.class));
+    SurfaceTexture mockSurfaceTexture = mock(SurfaceTexture.class);
+    Size mockSize = mock(Size.class);
+    MediaRecorder mockMediaRecorder = mock(MediaRecorder.class);
+    ImageReader mockPictureImageReader = mock(ImageReader.class);
+    TestUtils.setPrivateField(cameraSpy, "mediaRecorder", mockMediaRecorder);
+    TestUtils.setPrivateField(cameraSpy, "recordingVideo", false);
+    TestUtils.setPrivateField(cameraSpy, "pictureImageReader", mockPictureImageReader);
+    CameraDeviceWrapper fakeCamera = new FakeCameraDeviceWrapper(mockRequestBuilders);
+    TestUtils.setPrivateField(cameraSpy, "cameraDevice", fakeCamera);
+    MethodChannel.Result mockResult = mock(MethodChannel.Result.class);
+
+    TextureRegistry.SurfaceTextureEntry cameraFlutterTexture =
+        (TextureRegistry.SurfaceTextureEntry)
+            TestUtils.getPrivateField(cameraSpy, "flutterTexture");
+    ResolutionFeature resolutionFeature =
+        (ResolutionFeature)
+            TestUtils.getPrivateField(mockCameraFeatureFactory, "mockResolutionFeature");
+
+    when(cameraFlutterTexture.surfaceTexture()).thenReturn(mockSurfaceTexture);
+    when(resolutionFeature.getPreviewSize()).thenReturn(mockSize);
+    doNothing().when(cameraSpy).prepareRecording(mockResult);
+
+    cameraSpy.startVideoRecording(mockResult, null);
+    verify(mockMediaRecorder, times(1))
+        .getSurface(); // stream pulled from media recorder's surface.
+    verify(
+            mockPictureImageReader,
+            times(2)) // we expect one call to start the capture, one to create the capture session.
+        .getSurface(); // stream pulled from image streaming imageReader's surface.
+  }
+
   @Test
   public void setFocusMode_shouldLockAutoFocusForLockedMode() throws CameraAccessException {
     camera.setFocusMode(mock(MethodChannel.Result.class), FocusMode.locked);
@@ -1013,6 +1094,45 @@ public void createCaptureSession_doesNotCloseCaptureSession() throws CameraAcces
     verify(mockCaptureSession, never()).close();
   }
 
+  @Test
+  public void createCaptureSession_shouldNotAddPictureImageSurfaceToPreviewRequest()
+      throws CameraAccessException {
+    Surface mockSurface = mock(Surface.class);
+    Surface mockSecondarySurface = mock(Surface.class);
+    SurfaceTexture mockSurfaceTexture = mock(SurfaceTexture.class);
+    ResolutionFeature mockResolutionFeature = mock(ResolutionFeature.class);
+    Size mockSize = mock(Size.class);
+    ArrayList<CaptureRequest.Builder> mockRequestBuilders = new ArrayList<>();
+    mockRequestBuilders.add(mock(CaptureRequest.Builder.class));
+    CameraDeviceWrapper fakeCamera = spy(new FakeCameraDeviceWrapper(mockRequestBuilders));
+    ImageReader mockPictureImageReader = mock(ImageReader.class);
+    TestUtils.setPrivateField(camera, "cameraDevice", fakeCamera);
+    TestUtils.setPrivateField(camera, "pictureImageReader", mockPictureImageReader);
+    CaptureRequest.Builder mockPreviewRequestBuilder = mock(CaptureRequest.Builder.class);
+
+    TextureRegistry.SurfaceTextureEntry cameraFlutterTexture =
+        (TextureRegistry.SurfaceTextureEntry) TestUtils.getPrivateField(camera, "flutterTexture");
+    CameraFeatures cameraFeatures =
+        (CameraFeatures) TestUtils.getPrivateField(camera, "cameraFeatures");
+    ResolutionFeature resolutionFeature =
+        (ResolutionFeature)
+            TestUtils.getPrivateField(mockCameraFeatureFactory, "mockResolutionFeature");
+
+    when(cameraFlutterTexture.surfaceTexture()).thenReturn(mockSurfaceTexture);
+    when(resolutionFeature.getPreviewSize()).thenReturn(mockSize);
+    when(fakeCamera.createCaptureRequest(anyInt())).thenReturn(mockPreviewRequestBuilder);
+    when(mockPictureImageReader.getSurface()).thenReturn(mockSurface);
+
+    // Test with preview template.
+    camera.createCaptureSession(CameraDevice.TEMPLATE_PREVIEW, mockSurface, mockSecondarySurface);
+    verify(mockPreviewRequestBuilder, times(0)).addTarget(mockSurface);
+
+    // Test with non-preview template.
+    camera.createCaptureSession(CameraDevice.TEMPLATE_RECORD, mockSurface, mockSecondarySurface);
+    verify(mockPreviewRequestBuilder, times(0)).addTarget(mockSurface);
+    verify(mockPreviewRequestBuilder).addTarget(mockSecondarySurface);
+  }
+
   @Test
   public void close_doesCloseCaptureSessionWhenCameraDeviceNull() {
     camera.close();

packages/camera/camera_android/pubspec.yaml

@@ -3,7 +3,7 @@ description: Android implementation of the camera plugin.
 repository: https://github.com/flutter/packages/tree/main/packages/camera/camera_android
 issue_tracker: https://github.com/flutter/flutter/issues?q=is%3Aissue+is%3Aopen+label%3A%22p%3A+camera%22
 
-version: 0.10.8+2
+version: 0.10.8+3
 
 environment:
   sdk: ">=2.18.0 <4.0.0"