Tabletop AR: scene placement

microapps/ArTabletopApp/app/src/main/java/com/arcgismaps/toolkit/artabletopapp/screens/MainScreen.kt

@@ -18,24 +18,33 @@
 
 package com.arcgismaps.toolkit.artabletopapp.screens
 
+import androidx.compose.foundation.background
+import androidx.compose.foundation.layout.Arrangement
 import androidx.compose.foundation.layout.Box
+import androidx.compose.foundation.layout.Column
 import androidx.compose.foundation.layout.fillMaxSize
-import androidx.compose.foundation.layout.height
 import androidx.compose.foundation.layout.wrapContentSize
+import androidx.compose.material3.CircularProgressIndicator
 import androidx.compose.material3.Text
 import androidx.compose.runtime.Composable
 import androidx.compose.runtime.getValue
 import androidx.compose.runtime.mutableStateOf
 import androidx.compose.runtime.remember
 import androidx.compose.runtime.setValue
+import androidx.compose.ui.Alignment
 import androidx.compose.ui.Modifier
 import androidx.compose.ui.graphics.Color
 import androidx.compose.ui.res.stringResource
-import androidx.compose.ui.unit.dp
+import androidx.lifecycle.compose.collectAsStateWithLifecycle
+import com.arcgismaps.LoadStatus
 import com.arcgismaps.geometry.Point
+import com.arcgismaps.geometry.SpatialReference
 import com.arcgismaps.mapping.ArcGISScene
 import com.arcgismaps.mapping.BasemapStyle
-import com.arcgismaps.mapping.Viewpoint
+import com.arcgismaps.mapping.ElevationSource
+import com.arcgismaps.mapping.Surface
+import com.arcgismaps.mapping.layers.ArcGISMapImageLayer
+import com.arcgismaps.mapping.layers.ArcGISSceneLayer
 import com.arcgismaps.toolkit.ar.TableTopSceneView
 import com.arcgismaps.toolkit.ar.TableTopSceneViewProxy
 import com.arcgismaps.toolkit.ar.TableTopSceneViewStatus
@@ -47,7 +56,23 @@ import kotlin.math.roundToInt
 fun MainScreen() {
     val arcGISScene = remember {
         ArcGISScene(BasemapStyle.ArcGISImagery).apply {
-            initialViewpoint = Viewpoint(34.056295, -117.195800, 10000000.0)
+            operationalLayers.addAll(
+                listOf(
+                    // New York Transit Frequency
+                    ArcGISMapImageLayer("https://tiles.arcgis.com/tiles/nGt4QxSblgDfeJn9/arcgis/rest/services/UrbanObservatory_NYC_TransitFrequency/MapServer"),
+                    // New York Industrial Area
+                    ArcGISMapImageLayer("https://tiles.arcgis.com/tiles/nGt4QxSblgDfeJn9/arcgis/rest/services/New_York_Industrial/MapServer"),
+                    // New York Population Density
+                    ArcGISMapImageLayer("https://tiles.arcgis.com/tiles/4yjifSiIG17X0gW4/arcgis/rest/services/NewYorkCity_PopDensity/MapServer"),
+                    // New York Buildings
+                    ArcGISSceneLayer("https://tiles.arcgis.com/tiles/P3ePLMYs2RVChkJx/arcgis/rest/services/Buildings_NewYork_17/SceneServer")
+                )
+            )
+            baseSurface = Surface().apply {
+                elevationSources.add(
+                    ElevationSource.fromTerrain3dService()
+                )
+            }
         }
     }
     val tableTopSceneViewProxy = remember { TableTopSceneViewProxy() }
@@ -56,6 +81,9 @@ fun MainScreen() {
     Box(modifier = Modifier.fillMaxSize()) {
         TableTopSceneView(
             arcGISScene = arcGISScene,
+            arcGISSceneAnchor = Point(-74.0, 40.72, 0.0, SpatialReference.wgs84()),
+            translationFactor = 2000.0,
+            clippingDistance = 750.0,
             modifier = Modifier.fillMaxSize(),
             tableTopSceneViewProxy = tableTopSceneViewProxy,
             onInitializationStatusChanged = {
@@ -70,14 +98,58 @@ fun MainScreen() {
         ) {
             tappedLocation?.let {
                 Callout(location = it, modifier = Modifier.wrapContentSize()) {
-                    Text("Lat: ${it.y.roundToInt()}, Lon: ${it.x.roundToInt()}")
+                    Text(stringResource(R.string.lat_lon, it.y.roundToInt(), it.x.roundToInt()))
+                }
+            }
+        }
+        initializationStatus.let { status ->
+            when (status) {
+                is TableTopSceneViewStatus.Initializing -> TextWithScrim(text = stringResource(R.string.initializing_overlay))
+                is TableTopSceneViewStatus.DetectingPlanes -> TextWithScrim(text = stringResource(R.string.detect_planes_overlay))
+                is TableTopSceneViewStatus.Initialized -> {
+                    val sceneLoadStatus = arcGISScene.loadStatus.collectAsStateWithLifecycle().value
+                    if (sceneLoadStatus is LoadStatus.NotLoaded) {
+                        // Tell the user to tap the screen if the scene has not started loading
+                        TextWithScrim(text = stringResource(R.string.tap_scene_overlay))
+                    } else if (sceneLoadStatus is LoadStatus.Loading) {
+                        // The scene may take a while to load, so show a progress indicator
+                        Column(
+                            modifier = Modifier.fillMaxSize(),
+                            verticalArrangement = Arrangement.Center,
+                            horizontalAlignment = Alignment.CenterHorizontally
+                        ) {
+                            CircularProgressIndicator()
+                        }
+                    }
+                }
+
+                is TableTopSceneViewStatus.FailedToInitialize -> {
+                    TextWithScrim(
+                        text = stringResource(
+                            R.string.failed_to_initialize_overlay,
+                            status.error.message ?: status.error
+                        )
+                    )
                 }
             }
         }
-        Text(
-            text = stringResource(R.string.initialization_status, initializationStatus),
-            modifier = Modifier.height(200.dp),
-            color = Color.Red
-        )
+    }
+}
+
+/**
+ * Displays the provided [text] on top of a half-transparent gray background.
+ *
+ * @since 200.6.0
+ */
+@Composable
+fun TextWithScrim(text: String) {
+    Column(
+        modifier = Modifier
+            .background(Color.Gray.copy(alpha = 0.5f))
+            .fillMaxSize(),
+        verticalArrangement = Arrangement.Center,
+        horizontalAlignment = Alignment.CenterHorizontally
+    ) {
+        Text(text = text)
     }
 }

microapps/ArTabletopApp/app/src/main/res/values/strings.xml

@@ -20,4 +20,9 @@
     <string name="app_name">ArTabletopApp</string>
     <string name="arcore_not_installed_screen_message">Google Play Services for AR must be installed to run this app.</string>
     <string name="initialization_status">Initialization status: %1$s</string>
+    <string name="detect_planes_overlay">Move your phone around to detect planes...</string>
+    <string name="lat_lon">Lat: %1$s, Lon: %2$s</string>
+    <string name="tap_scene_overlay">Tap on a plane to place the scene</string>
+    <string name="initializing_overlay">Setting up AR...</string>
+    <string name="failed_to_initialize_overlay">Failed to initialize: %1$s</string>
 </resources>

toolkit/ar/src/main/java/com/arcgismaps/toolkit/ar/TableTopSceneView.kt

@@ -39,6 +39,7 @@ import androidx.compose.ui.graphics.Color
 import androidx.compose.ui.platform.LocalContext
 import androidx.compose.ui.platform.LocalLifecycleOwner
 import androidx.core.content.ContextCompat
+import com.arcgismaps.geometry.Point
 import com.arcgismaps.geometry.SpatialReference
 import com.arcgismaps.mapping.ArcGISScene
 import com.arcgismaps.mapping.TimeExtent
@@ -48,6 +49,7 @@ import com.arcgismaps.mapping.view.AnalysisOverlay
 import com.arcgismaps.mapping.view.AtmosphereEffect
 import com.arcgismaps.mapping.view.AttributionBarLayoutChangeEvent
 import com.arcgismaps.mapping.view.Camera
+import com.arcgismaps.mapping.view.DeviceOrientation
 import com.arcgismaps.mapping.view.DoubleTapEvent
 import com.arcgismaps.mapping.view.DownEvent
 import com.arcgismaps.mapping.view.DrawStatus
@@ -63,15 +65,18 @@ import com.arcgismaps.mapping.view.SceneViewInteractionOptions
 import com.arcgismaps.mapping.view.SelectionProperties
 import com.arcgismaps.mapping.view.SingleTapConfirmedEvent
 import com.arcgismaps.mapping.view.SpaceEffect
+import com.arcgismaps.mapping.view.TransformationMatrix
+import com.arcgismaps.mapping.view.TransformationMatrixCameraController
 import com.arcgismaps.mapping.view.TwoPointerTapEvent
 import com.arcgismaps.mapping.view.UpEvent
 import com.arcgismaps.mapping.view.ViewLabelProperties
 import com.arcgismaps.toolkit.ar.internal.ArCameraFeed
 import com.arcgismaps.toolkit.ar.internal.rememberArSessionWrapper
 import com.arcgismaps.toolkit.geoviewcompose.SceneView
 import com.arcgismaps.toolkit.geoviewcompose.SceneViewDefaults
+import com.google.ar.core.Anchor
 import com.google.ar.core.ArCoreApk
-import kotlinx.coroutines.delay
+import com.google.ar.core.Pose
 import kotlinx.coroutines.suspendCancellableCoroutine
 import java.time.Instant
 import kotlin.coroutines.resume
@@ -80,6 +85,9 @@ import kotlin.coroutines.resume
  * Displays a [SceneView] in a tabletop AR environment.
  *
  * @param arcGISScene the [ArcGISScene] to be rendered by this TableTopSceneView
+ * @param arcGISSceneAnchor the [Point] in the [ArcGISScene] where the AR scene will initially be centered
+ * @param translationFactor the factor defines how much the scene view translates as the device moves.
+ * @param clippingDistance the clipping distance in meters around the camera. A null means that no data will be clipped.
  * @param modifier Modifier to be applied to the TableTopSceneView
  * @param onInitializationStatusChanged a callback that is invoked when the initialization status of the [TableTopSceneView] changes.
  * @param requestCameraPermissionAutomatically whether to request the camera permission automatically.
@@ -126,6 +134,9 @@ import kotlin.coroutines.resume
 @Composable
 fun TableTopSceneView(
     arcGISScene: ArcGISScene,
+    arcGISSceneAnchor: Point,
+    translationFactor: Double,
+    clippingDistance: Double?,
     modifier: Modifier = Modifier,
     onInitializationStatusChanged: ((TableTopSceneViewStatus) -> Unit)? = null,
     requestCameraPermissionAutomatically: Boolean = true,
@@ -163,7 +174,7 @@ fun TableTopSceneView(
     onDrawStatusChanged: ((DrawStatus) -> Unit)? = null,
     content: (@Composable TableTopSceneViewScope.() -> Unit)? = null
 ) {
-    var initializationStatus = rememberTableTopSceneViewStatus()
+    val initializationStatus = rememberTableTopSceneViewStatus()
 
     val lifecycleOwner = LocalLifecycleOwner.current
     val context = LocalContext.current
@@ -196,13 +207,21 @@ fun TableTopSceneView(
     }
 
     Box(modifier = modifier) {
+        val cameraController = remember {
+            TransformationMatrixCameraController().apply {
+                setOriginCamera(Camera(arcGISSceneAnchor, heading = 0.0, pitch = 90.0, roll = 0.0))
+                setTranslationFactor(translationFactor)
+                this.clippingDistance = clippingDistance
+            }
+        }
+        var arCoreAnchor: Anchor? by remember { mutableStateOf(null) }
         if (cameraPermissionGranted && arCoreInstalled) {
             val arSessionWrapper =
                 rememberArSessionWrapper(applicationContext = context.applicationContext)
             DisposableEffect(Unit) {
                 // We call this from inside DisposableEffect so that we invoke the callback in a side effect
                 initializationStatus.update(
-                    TableTopSceneViewStatus.Initialized,
+                    TableTopSceneViewStatus.DetectingPlanes,
                     onInitializationStatusChanged
                 )
                 lifecycleOwner.lifecycle.addObserver(arSessionWrapper)
@@ -211,51 +230,102 @@ fun TableTopSceneView(
                     arSessionWrapper.onDestroy(lifecycleOwner)
                 }
             }
-            ArCameraFeed(arSessionWrapper = arSessionWrapper, onFrame = {}, onTap = {})
+            val identityMatrix = remember { TransformationMatrix.createIdentityMatrix() }
+            ArCameraFeed(
+                arSessionWrapper = arSessionWrapper,
+                onFrame = { frame, displayRotation ->
+                    arCoreAnchor?.let { anchor ->
+                        val anchorPosition = identityMatrix - anchor.pose.translation.let {
+                            TransformationMatrix.createWithQuaternionAndTranslation(
+                                0.0,
+                                0.0,
+                                0.0,
+                                1.0,
+                                it[0].toDouble(),
+                                it[1].toDouble(),
+                                it[2].toDouble()
+                            )
+                        }
+                        val cameraPosition =
+                            anchorPosition + frame.camera.displayOrientedPose.transformationMatrix
+                        cameraController.transformationMatrix = cameraPosition
+                        val imageIntrinsics = frame.camera.imageIntrinsics
+                        tableTopSceneViewProxy.sceneViewProxy.setFieldOfViewFromLensIntrinsics(
+                            imageIntrinsics.focalLength[0],
+                            imageIntrinsics.focalLength[1],
+                            imageIntrinsics.principalPoint[0],
+                            imageIntrinsics.principalPoint[1],
+                            imageIntrinsics.imageDimensions[0].toFloat(),
+                            imageIntrinsics.imageDimensions[1].toFloat(),
+                            deviceOrientation = when (displayRotation) {
+                                0 -> DeviceOrientation.Portrait
+                                90 -> DeviceOrientation.LandscapeRight
+                                180 -> DeviceOrientation.ReversePortrait
+                                270 -> DeviceOrientation.LandscapeLeft
+                                else -> DeviceOrientation.Portrait
+                            }
+                        )
+                        tableTopSceneViewProxy.sceneViewProxy.renderFrame()
+                    }
+                },
+                onTapWithHitResult = { hit ->
+                    hit?.let { hitResult ->
+                        if (arCoreAnchor == null) {
+                            arCoreAnchor = hitResult.createAnchor()
+                        }
+                    }
+                },
+                onFirstPlaneDetected = {
+                    initializationStatus.update(
+                        TableTopSceneViewStatus.Initialized,
+                        onInitializationStatusChanged
+                    )
+                })
+        }
+        if (initializationStatus.value == TableTopSceneViewStatus.Initialized && arCoreAnchor != null) {
+            SceneView(
+                arcGISScene = arcGISScene,
+                modifier = Modifier.fillMaxSize(),
+                onViewpointChangedForCenterAndScale = onViewpointChangedForCenterAndScale,
+                onViewpointChangedForBoundingGeometry = onViewpointChangedForBoundingGeometry,
+                graphicsOverlays = graphicsOverlays,
+                sceneViewProxy = tableTopSceneViewProxy.sceneViewProxy,
+                sceneViewInteractionOptions = sceneViewInteractionOptions,
+                viewLabelProperties = viewLabelProperties,
+                selectionProperties = selectionProperties,
+                isAttributionBarVisible = isAttributionBarVisible,
+                onAttributionTextChanged = onAttributionTextChanged,
+                onAttributionBarLayoutChanged = onAttributionBarLayoutChanged,
+                cameraController = cameraController,
+                analysisOverlays = analysisOverlays,
+                imageOverlays = imageOverlays,
+                atmosphereEffect = AtmosphereEffect.None,
+                timeExtent = timeExtent,
+                onTimeExtentChanged = onTimeExtentChanged,
+                spaceEffect = SpaceEffect.Transparent,
+                sunTime = sunTime,
+                sunLighting = sunLighting,
+                ambientLightColor = ambientLightColor,
+                onNavigationChanged = onNavigationChanged,
+                onSpatialReferenceChanged = onSpatialReferenceChanged,
+                onLayerViewStateChanged = onLayerViewStateChanged,
+                onInteractingChanged = onInteractingChanged,
+                onCurrentViewpointCameraChanged = onCurrentViewpointCameraChanged,
+                onRotate = onRotate,
+                onScale = onScale,
+                onUp = onUp,
+                onDown = onDown,
+                onSingleTapConfirmed = onSingleTapConfirmed,
+                onDoubleTap = onDoubleTap,
+                onLongPress = onLongPress,
+                onTwoPointerTap = onTwoPointerTap,
+                onPan = onPan,
+                onDrawStatusChanged = onDrawStatusChanged,
+                content = {
+                    content?.invoke(TableTopSceneViewScope(this))
+                }
+            )
         }
-        SceneView(
-            arcGISScene = arcGISScene,
-            modifier = Modifier.fillMaxSize(),
-            onViewpointChangedForCenterAndScale = onViewpointChangedForCenterAndScale,
-            onViewpointChangedForBoundingGeometry = onViewpointChangedForBoundingGeometry,
-            graphicsOverlays = graphicsOverlays,
-            sceneViewProxy = tableTopSceneViewProxy.sceneViewProxy,
-            sceneViewInteractionOptions = sceneViewInteractionOptions,
-            viewLabelProperties = viewLabelProperties,
-            selectionProperties = selectionProperties,
-            isAttributionBarVisible = isAttributionBarVisible,
-            onAttributionTextChanged = onAttributionTextChanged,
-            onAttributionBarLayoutChanged = onAttributionBarLayoutChanged,
-            analysisOverlays = analysisOverlays,
-            imageOverlays = imageOverlays,
-            atmosphereEffect = AtmosphereEffect.None,
-            timeExtent = timeExtent,
-            onTimeExtentChanged = onTimeExtentChanged,
-            spaceEffect = SpaceEffect.Transparent,
-            sunTime = sunTime,
-            sunLighting = sunLighting,
-            ambientLightColor = ambientLightColor,
-            onNavigationChanged = onNavigationChanged,
-            onSpatialReferenceChanged = {
-                onSpatialReferenceChanged?.invoke(it)
-            },
-            onLayerViewStateChanged = onLayerViewStateChanged,
-            onInteractingChanged = onInteractingChanged,
-            onCurrentViewpointCameraChanged = onCurrentViewpointCameraChanged,
-            onRotate = onRotate,
-            onScale = onScale,
-            onUp = onUp,
-            onDown = onDown,
-            onSingleTapConfirmed = onSingleTapConfirmed,
-            onDoubleTap = onDoubleTap,
-            onLongPress = onLongPress,
-            onTwoPointerTap = onTwoPointerTap,
-            onPan = onPan,
-            onDrawStatusChanged = onDrawStatusChanged,
-            content = {
-                content?.invoke(TableTopSceneViewScope(this))
-            }
-        )
     }
 }
 
@@ -317,3 +387,16 @@ private fun MutableState<TableTopSceneViewStatus>.update(
     this.value = newStatus
     callback?.invoke(newStatus)
 }
+
+private val Pose.transformationMatrix: TransformationMatrix
+    get() {
+        return TransformationMatrix.createWithQuaternionAndTranslation(
+            rotationQuaternion[0].toDouble(),
+            rotationQuaternion[1].toDouble(),
+            rotationQuaternion[2].toDouble(),
+            rotationQuaternion[3].toDouble(),
+            translation[0].toDouble(),
+            translation[1].toDouble(),
+            translation[2].toDouble()
+        )
+    }