diff --git a/README.md b/README.md index 1c508281..8c1a47cf 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,9 @@ example code can focus on the Tango-specific aspects of the application. to stay affixed in space. * **java_floor_plan_example** - Create a floor plan by using the depth sensor to detect and mesure walls in a room. + * **java_marker_detection_example** - Build an augmented reality appliction + to detect AR markers in real world, and to render the boundaries and poses of + markers. * **java_model_correspondence_example** - Create a mapping between a virtual 3D object and selected points in the real world. * **java_motion_tracking_example** - Use Tango motion diff --git a/TangoReleaseLibs/aar/tango_support_java_lib.aar b/TangoReleaseLibs/aar/tango_support_java_lib.aar new file mode 100755 index 00000000..22f63e45 Binary files /dev/null and b/TangoReleaseLibs/aar/tango_support_java_lib.aar differ diff --git a/TangoReleaseLibs/aar/tango_ux_support_library.aar b/TangoReleaseLibs/aar/tango_ux_support_library.aar new file mode 100755 index 00000000..db9aa818 Binary files /dev/null and b/TangoReleaseLibs/aar/tango_ux_support_library.aar differ diff --git a/TangoReleaseLibs/jar/tango_java_lib.jar b/TangoReleaseLibs/jar/tango_java_lib.jar new file mode 100755 index 00000000..61abee71 Binary files /dev/null and b/TangoReleaseLibs/jar/tango_java_lib.jar differ diff --git a/java_augmented_reality_example/app/src/main/java/com/projecttango/examples/java/augmentedreality/AugmentedRealityActivity.java b/java_augmented_reality_example/app/src/main/java/com/projecttango/examples/java/augmentedreality/AugmentedRealityActivity.java index 59d47fde..bd417365 100644 --- a/java_augmented_reality_example/app/src/main/java/com/projecttango/examples/java/augmentedreality/AugmentedRealityActivity.java +++ b/java_augmented_reality_example/app/src/main/java/com/projecttango/examples/java/augmentedreality/AugmentedRealityActivity.java @@ -182,10 +182,10 @@ public void run() { // OpenGL thread or in the UI thread. synchronized (AugmentedRealityActivity.this) { try { - TangoSupport.initialize(); mConfig = setupTangoConfig(mTango); mTango.connect(mConfig); startupTango(); + TangoSupport.initialize(mTango); mIsConnected = true; setDisplayRotation(); } catch (TangoOutOfDateException e) { diff --git a/java_basic_examples/hello_motion_tracking/src/main/AndroidManifest.xml b/java_basic_examples/hello_motion_tracking/src/main/AndroidManifest.xml index 5f509d33..3447f366 100644 --- a/java_basic_examples/hello_motion_tracking/src/main/AndroidManifest.xml +++ b/java_basic_examples/hello_motion_tracking/src/main/AndroidManifest.xml @@ -20,6 +20,7 @@ android:required="true" /> diff --git a/java_basic_examples/hello_video/src/main/java/com/projecttango/examples/java/hellovideo/HelloVideoActivity.java b/java_basic_examples/hello_video/src/main/java/com/projecttango/examples/java/hellovideo/HelloVideoActivity.java index e870ab04..58bbb052 100644 --- a/java_basic_examples/hello_video/src/main/java/com/projecttango/examples/java/hellovideo/HelloVideoActivity.java +++ b/java_basic_examples/hello_video/src/main/java/com/projecttango/examples/java/hellovideo/HelloVideoActivity.java @@ -129,10 +129,10 @@ public void run() { // the OpenGL thread or in the UI thread. synchronized (HelloVideoActivity.this) { try { - TangoSupport.initialize(); mConfig = setupTangoConfig(mTango); mTango.connect(mConfig); startupTango(); + TangoSupport.initialize(mTango); mIsConnected = true; setDisplayRotation(); } catch (TangoOutOfDateException e) { diff --git a/java_floor_plan_reconstruction_example/app/src/main/java/com/projecttango/examples/java/floorplanreconstruction/FloorPlanReconstructionActivity.java b/java_floor_plan_reconstruction_example/app/src/main/java/com/projecttango/examples/java/floorplanreconstruction/FloorPlanReconstructionActivity.java index 488826e5..79516157 100644 --- a/java_floor_plan_reconstruction_example/app/src/main/java/com/projecttango/examples/java/floorplanreconstruction/FloorPlanReconstructionActivity.java +++ b/java_floor_plan_reconstruction_example/app/src/main/java/com/projecttango/examples/java/floorplanreconstruction/FloorPlanReconstructionActivity.java @@ -170,10 +170,10 @@ private void bindTangoService() { public void run() { synchronized (FloorPlanReconstructionActivity.this) { try { - TangoSupport.initialize(); mConfig = setupTangoConfig(mTango); mTango.connect(mConfig); startupTango(); + TangoSupport.initialize(mTango); mIsConnected = true; mIsPaused = false; runOnUiThread(new Runnable() { diff --git a/java_green_screen_example/app/src/main/java/com/projecttango/examples/java/greenscreen/GreenScreenActivity.java b/java_green_screen_example/app/src/main/java/com/projecttango/examples/java/greenscreen/GreenScreenActivity.java index 5aea0ba2..28468090 100644 --- a/java_green_screen_example/app/src/main/java/com/projecttango/examples/java/greenscreen/GreenScreenActivity.java +++ b/java_green_screen_example/app/src/main/java/com/projecttango/examples/java/greenscreen/GreenScreenActivity.java @@ -210,10 +210,10 @@ private void bindTangoService() { public void run() { synchronized (GreenScreenActivity.this) { try { - TangoSupport.initialize(); mConfig = setupTangoConfig(mTango); mTango.connect(mConfig); startupTango(); + TangoSupport.initialize(mTango); mIsConnected = true; setDisplayRotation(); } catch (TangoOutOfDateException e) { diff --git a/java_marker_detection_example/.gitignore b/java_marker_detection_example/.gitignore new file mode 100644 index 00000000..758d6a59 --- /dev/null +++ b/java_marker_detection_example/.gitignore @@ -0,0 +1,7 @@ +.gradle +local.properties +.idea +*.iml +.DS_Store +build +captures diff --git a/java_marker_detection_example/README.md b/java_marker_detection_example/README.md new file mode 100644 index 00000000..bd237880 --- /dev/null +++ b/java_marker_detection_example/README.md @@ -0,0 +1,14 @@ +API example that demonstrates marker detection API. + +Please follow the instructions below to run the example: + 1. Build and install the app onto your device. + 2. Print one or a few markers from the attached ar_markers.pdf file. Please + print the marker on a Letter size paper, without using any zooming or scaling + factor. Please measure every side of the marker using a ruler to make sure + it is 14cm. + 3. Place the printed marker(s) on a flat surface, tables or walls. + 4. Run the example app from your device, and point the device camera to the + printed markers. + 5. For every marker detected, the boundary and the axes of the local frame will + be displayed. For the X, Y and Z axes of the local frame, RED, GREEN and + BLUE lines will be displayed. diff --git a/java_marker_detection_example/app/.gitignore b/java_marker_detection_example/app/.gitignore new file mode 100644 index 00000000..796b96d1 --- /dev/null +++ b/java_marker_detection_example/app/.gitignore @@ -0,0 +1 @@ +/build diff --git a/java_marker_detection_example/app/build.gradle b/java_marker_detection_example/app/build.gradle new file mode 100644 index 00000000..25873cb8 --- /dev/null +++ b/java_marker_detection_example/app/build.gradle @@ -0,0 +1,33 @@ +apply plugin: 'com.android.application' + +android { + compileSdkVersion 23 + buildToolsVersion "21.1.2" + + defaultConfig { + applicationId "com.projecttango.experiments.markerdetectionsample" + minSdkVersion 19 + targetSdkVersion 23 + versionCode 1 + versionName "1.0" + } + buildTypes { + release { + minifyEnabled false + proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro' + } + + lintOptions { + abortOnError false + } + } +} + + +dependencies { + apply from: '../../version.gradle' + compile "com.google.tango:sdk-base:${release_version}" + compile "com.google.tango:support-base:${release_version}" + compile "org.rajawali3d:rajawali:1.1.668@aar" + compile "com.android.support:appcompat-v7:23.0.0" +} diff --git a/java_marker_detection_example/app/proguard-rules.pro b/java_marker_detection_example/app/proguard-rules.pro new file mode 100644 index 00000000..45dc58a5 --- /dev/null +++ b/java_marker_detection_example/app/proguard-rules.pro @@ -0,0 +1,17 @@ +# Add project specific ProGuard rules here. +# By default, the flags in this file are appended to flags specified +# in /opt/android-sdk/tools/proguard/proguard-android.txt +# You can edit the include path and order by changing the proguardFiles +# directive in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# Add any project specific keep options here: + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} diff --git a/java_marker_detection_example/app/src/main/AndroidManifest.xml b/java_marker_detection_example/app/src/main/AndroidManifest.xml new file mode 100644 index 00000000..32f10203 --- /dev/null +++ b/java_marker_detection_example/app/src/main/AndroidManifest.xml @@ -0,0 +1,47 @@ + + + + + + + + + + + + + + + + + + + diff --git a/java_marker_detection_example/app/src/main/java/com/projecttango/examples/java/markerdetection/MarkerDetectionActivity.java b/java_marker_detection_example/app/src/main/java/com/projecttango/examples/java/markerdetection/MarkerDetectionActivity.java new file mode 100644 index 00000000..5eb5bb2d --- /dev/null +++ b/java_marker_detection_example/app/src/main/java/com/projecttango/examples/java/markerdetection/MarkerDetectionActivity.java @@ -0,0 +1,563 @@ +/* + * Copyright 2017 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.projecttango.examples.java.markerdetection; + +import com.google.atap.tangoservice.Tango; +import com.google.atap.tangoservice.Tango.OnTangoUpdateListener; +import com.google.atap.tangoservice.TangoCameraIntrinsics; +import com.google.atap.tangoservice.TangoConfig; +import com.google.atap.tangoservice.TangoCoordinateFramePair; +import com.google.atap.tangoservice.TangoErrorException; +import com.google.atap.tangoservice.TangoEvent; +import com.google.atap.tangoservice.TangoException; +import com.google.atap.tangoservice.TangoInvalidException; +import com.google.atap.tangoservice.TangoOutOfDateException; +import com.google.atap.tangoservice.TangoPointCloudData; +import com.google.atap.tangoservice.TangoPoseData; +import com.google.atap.tangoservice.TangoXyzIjData; +import com.google.atap.tangoservice.experimental.TangoImageBuffer; + +import android.Manifest; +import android.app.Activity; +import android.app.AlertDialog; +import android.content.DialogInterface; +import android.content.pm.PackageManager; +import android.hardware.display.DisplayManager; +import android.opengl.GLSurfaceView; +import android.opengl.Matrix; +import android.os.Bundle; +import android.support.v4.app.ActivityCompat; +import android.support.v4.content.ContextCompat; +import android.util.Log; +import android.view.Display; +import android.widget.Toast; + +import org.rajawali3d.scene.ASceneFrameCallback; +import org.rajawali3d.view.SurfaceView; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +import com.projecttango.tangosupport.TangoSupport; + +/** + * This is a simple example that shows how to use the Tango APIs to detect markers within camera + * images. + *

+ * Note that it is important to include the KEY_BOOLEAN_LOWLATENCYIMUINTEGRATION configuration + * parameter in order to achieve best results synchronizing the Rajawali virtual world with the + * RGB camera. + *

+ * If you're looking for a more stripped down example that doesn't use a rendering library like + * Rajawali, see java_hello_video_example. + */ +public class MarkerDetectionActivity extends Activity { + private static final String TAG = MarkerDetectionActivity.class.getSimpleName(); + private static final int INVALID_TEXTURE_ID = 0; + + private static final String CAMERA_PERMISSION = Manifest.permission.CAMERA; + private static final int CAMERA_PERMISSION_CODE = 0; + + private SurfaceView mSurfaceView; + private MarkerDetectionRenderer mRenderer; + private Tango mTango; + private TangoConfig mConfig; + private boolean mIsConnected = false; + private double mCameraPoseTimestamp = 0; + private volatile TangoImageBuffer mCurrentImageBuffer; + + // Texture rendering related fields. + // NOTE: Naming indicates which thread is in charge of updating this variable. + private int mConnectedTextureIdGlThread = INVALID_TEXTURE_ID; + private AtomicBoolean mIsFrameAvailableTangoThread = new AtomicBoolean(false); + private double mRgbTimestampGlThread; + + private int mDisplayRotation = 0; + + @Override + protected void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + setContentView(R.layout.activity_main); + mSurfaceView = (SurfaceView) findViewById(R.id.surfaceview); + mRenderer = new MarkerDetectionRenderer(this); + + DisplayManager displayManager = (DisplayManager) getSystemService(DISPLAY_SERVICE); + if (displayManager != null) { + displayManager.registerDisplayListener(new DisplayManager.DisplayListener() { + @Override + public void onDisplayAdded(int displayId) { + } + + @Override + public void onDisplayChanged(int displayId) { + synchronized (this) { + setDisplayRotation(); + } + } + + @Override + public void onDisplayRemoved(int displayId) { + } + }, null); + } + + setupRenderer(); + } + + @Override + protected void onStart() { + super.onStart(); + + // Set render mode to RENDERMODE_CONTINUOUSLY to force getting onDraw callbacks until + // the Tango service is properly set up and we start getting onFrameAvailable callbacks. + mSurfaceView.setRenderMode(GLSurfaceView.RENDERMODE_CONTINUOUSLY); + // Check and request camera permission at run time. + if (checkAndRequestPermissions()) { + bindTangoService(); + } + } + + @Override + public void onStop() { + super.onStop(); + + // Synchronize against disconnecting while the service is being used in the OpenGL thread or + // in the UI thread. + // NOTE: DO NOT lock against this same object in the Tango callback thread. Tango.disconnect + // will block here until all Tango callback calls are finished. If you lock against this + // object in a Tango callback thread it will cause a deadlock. + synchronized (this) { + try { + // mTango may be null if the app is closed before permissions are granted. + if (mTango != null) { + mTango.disconnectCamera(TangoCameraIntrinsics.TANGO_CAMERA_COLOR); + mTango.disconnect(); + } + // We need to invalidate the connected texture ID so that we cause a + // re-connection in the OpenGL thread after resume. + mConnectedTextureIdGlThread = INVALID_TEXTURE_ID; + mTango = null; + mIsConnected = false; + } catch (TangoErrorException e) { + Log.e(TAG, getString(R.string.exception_tango_error), e); + } + } + } + + /** + * Initialize Tango Service as a normal Android Service. + */ + private void bindTangoService() { + // Since we call mTango.disconnect() in onStop, this will unbind Tango Service, so every + // time onStart gets called we should create a new Tango object. + mTango = new Tango(MarkerDetectionActivity.this, new Runnable() { + // Pass in a Runnable to be called from UI thread when Tango is ready. This Runnable + // will be running on a new thread. + // When Tango is ready, we can call Tango functions safely here only when there + // are no UI thread changes involved. + @Override + public void run() { + // Synchronize against disconnecting while the service is being used in the + // OpenGL thread or in the UI thread. + synchronized (MarkerDetectionActivity.this) { + try { + mConfig = setupTangoConfig(mTango); + mTango.connect(mConfig); + startupTango(); + TangoSupport.initialize(mTango); + mIsConnected = true; + setDisplayRotation(); + } catch (TangoOutOfDateException e) { + Log.e(TAG, getString(R.string.exception_out_of_date), e); + showsToastAndFinishOnUiThread(R.string.exception_out_of_date); + } catch (TangoErrorException e) { + Log.e(TAG, getString(R.string.exception_tango_error), e); + showsToastAndFinishOnUiThread(R.string.exception_tango_error); + } catch (TangoInvalidException e) { + Log.e(TAG, getString(R.string.exception_tango_invalid), e); + showsToastAndFinishOnUiThread(R.string.exception_tango_invalid); + } + } + } + }); + } + + /** + * Sets up the Tango configuration object. Make sure mTango object is initialized before + * making this call. + */ + private TangoConfig setupTangoConfig(Tango tango) { + // Use default configuration for Tango Service, plus color camera, low latency + // IMU integration and drift correction. + TangoConfig config = tango.getConfig(TangoConfig.CONFIG_TYPE_DEFAULT); + config.putBoolean(TangoConfig.KEY_BOOLEAN_COLORCAMERA, true); + // NOTE: Low latency integration is necessary to achieve a precise alignment of + // virtual objects with the RBG image and produce a good AR effect. + config.putBoolean(TangoConfig.KEY_BOOLEAN_LOWLATENCYIMUINTEGRATION, true); + // Drift correction allows motion tracking to recover after it loses tracking. + // The drift-corrected pose is available through the frame pair with + // base frame AREA_DESCRIPTION and target frame DEVICE. + config.putBoolean(TangoConfig.KEY_BOOLEAN_DRIFT_CORRECTION, true); + return config; + } + + /** + * Set up the callback listeners for the Tango Service and obtain other parameters required + * after Tango connection. + * Listen to updates from the RGB camera. + */ + private void startupTango() { + // No need to add any coordinate frame pairs since we aren't using pose data from callbacks. + ArrayList framePairs = new ArrayList(); + + mTango.connectListener(framePairs, new OnTangoUpdateListener() { + @Override + public void onPoseAvailable(TangoPoseData pose) { + // We are not using onPoseAvailable for this app. + } + + @Override + public void onXyzIjAvailable(TangoXyzIjData xyzIj) { + // We are not using onXyzIjAvailable for this app. + } + + @Override + public void onPointCloudAvailable(TangoPointCloudData pointCloud) { + // We are not using onPointCloudAvailable for this app. + } + + @Override + public void onTangoEvent(TangoEvent event) { + // We are not using onTangoEvent for this app. + } + + @Override + public void onFrameAvailable(int cameraId) { + // Check if the frame available is for the camera we want and update its frame + // on the view. + if (cameraId == TangoCameraIntrinsics.TANGO_CAMERA_COLOR) { + // Now that we are receiving onFrameAvailable callbacks, we can switch + // to RENDERMODE_WHEN_DIRTY to drive the render loop from this callback. + // This will result in a frame rate of approximately 30FPS, in synchrony with + // the RGB camera driver. + // If you need to render at a higher rate (i.e., if you want to render complex + // animations smoothly) you can use RENDERMODE_CONTINUOUSLY throughout the + // application lifecycle. + if (mSurfaceView.getRenderMode() != GLSurfaceView.RENDERMODE_WHEN_DIRTY) { + mSurfaceView.setRenderMode(GLSurfaceView.RENDERMODE_WHEN_DIRTY); + } + + // Mark a camera frame as available for rendering in the OpenGL thread. + mIsFrameAvailableTangoThread.set(true); + // Trigger a Rajawali render to update the scene with the new RGB data. + mSurfaceView.requestRender(); + } + } + }); + mTango.experimentalConnectOnFrameListener( + TangoCameraIntrinsics.TANGO_CAMERA_COLOR, new Tango.OnFrameAvailableListener() { + @Override + public void onFrameAvailable(TangoImageBuffer tangoImageBuffer, int i) { + mCurrentImageBuffer = copyImageBuffer(tangoImageBuffer); + } + + TangoImageBuffer copyImageBuffer(TangoImageBuffer imageBuffer) { + ByteBuffer clone = ByteBuffer.allocateDirect(imageBuffer.data.capacity()); + imageBuffer.data.rewind(); + clone.put(imageBuffer.data); + imageBuffer.data.rewind(); + clone.flip(); + return new TangoImageBuffer(imageBuffer.width, imageBuffer.height, + imageBuffer.stride, imageBuffer.frameNumber, imageBuffer.timestamp, + imageBuffer.format, clone); + } + }); + } + + /** + * Connects the view and renderer to the color camara and callbacks. + */ + private void setupRenderer() { + // Register a Rajawali Scene Frame Callback to update the scene camera pose whenever a new + // RGB frame is rendered. + // (@see https://github.com/Rajawali/Rajawali/wiki/Scene-Frame-Callbacks) + mRenderer.getCurrentScene().registerFrameCallback(new ASceneFrameCallback() { + @Override + public void onPreFrame(long sceneTime, double deltaTime) { + // NOTE: This is called from the OpenGL render thread after all the renderer + // onRender callbacks have a chance to run and before scene objects are rendered + // into the scene. + + // Prevent concurrent access to {@code mIsFrameAvailableTangoThread} from the Tango + // callback thread and service disconnection from an onPause event. + try { + synchronized (MarkerDetectionActivity.this) { + // Don't execute tango API actions if we're not connected to the service. + if (!mIsConnected) { + return; + } + + // Set up scene camera projection to match RGB camera intrinsics. + if (!mRenderer.isSceneCameraConfigured()) { + TangoCameraIntrinsics intrinsics = + TangoSupport.getCameraIntrinsicsBasedOnDisplayRotation( + TangoCameraIntrinsics.TANGO_CAMERA_COLOR, + mDisplayRotation); + mRenderer.setProjectionMatrix( + projectionMatrixFromCameraIntrinsics(intrinsics)); + } + // Connect the camera texture to the OpenGL Texture if necessary + // NOTE: When the OpenGL context is recycled, Rajawali may regenerate the + // texture with a different ID. + if (mConnectedTextureIdGlThread != mRenderer.getTextureId()) { + mTango.connectTextureId(TangoCameraIntrinsics.TANGO_CAMERA_COLOR, + mRenderer.getTextureId()); + mConnectedTextureIdGlThread = mRenderer.getTextureId(); + Log.d(TAG, "connected to texture id: " + mRenderer.getTextureId()); + } + + // If there is a new RGB camera frame available, update the texture + // with it. + if (mIsFrameAvailableTangoThread.compareAndSet(true, false)) { + mRgbTimestampGlThread = + mTango.updateTexture(TangoCameraIntrinsics.TANGO_CAMERA_COLOR); + } + + // If a new RGB frame has been rendered, update the camera pose to match. + if (mRgbTimestampGlThread > mCameraPoseTimestamp) { + // Calculate the camera color pose at the camera frame update time in + // OpenGL engine. + // + // When drift correction mode is enabled in config file, we must query + // the device with respect to Area Description pose in order to use the + // drift corrected pose. + // + // Note that if you don't want to use the drift corrected pose, the + // normal device with respect to start of service pose is available. + TangoPoseData lastFramePose = TangoSupport.getPoseAtTime( + mRgbTimestampGlThread, + TangoPoseData.COORDINATE_FRAME_AREA_DESCRIPTION, + TangoPoseData.COORDINATE_FRAME_CAMERA_COLOR, + TangoSupport.TANGO_SUPPORT_ENGINE_OPENGL, + TangoSupport.TANGO_SUPPORT_ENGINE_OPENGL, + mDisplayRotation); + if (lastFramePose.statusCode == TangoPoseData.POSE_VALID) { + // Update the camera pose from the renderer + mRenderer.updateRenderCameraPose(lastFramePose); + mCameraPoseTimestamp = lastFramePose.timestamp; + + // Detect markers within the current image buffer. + TangoSupport.MarkerParam param = new TangoSupport.MarkerParam(); + param.type = TangoSupport.TANGO_MARKER_ARTAG; + param.markerSize = 0.1395; + + TangoPoseData worldTcamera = TangoSupport.getPoseAtTime( + mCurrentImageBuffer.timestamp, + TangoPoseData.COORDINATE_FRAME_AREA_DESCRIPTION, + TangoPoseData.COORDINATE_FRAME_CAMERA_COLOR, + TangoSupport.TANGO_SUPPORT_ENGINE_OPENGL, + TangoSupport.TANGO_SUPPORT_ENGINE_TANGO, + TangoSupport.ROTATION_IGNORED); + try { + List markerList = + TangoSupport.detectMarkers( + mCurrentImageBuffer, + TangoCameraIntrinsics.TANGO_CAMERA_COLOR, + worldTcamera.translation, + worldTcamera.rotation, + param); + + mRenderer.updateMarkers(markerList); + } catch (TangoException e) { + e.printStackTrace(); + } + } else { + // When the pose status is not valid, it indicates the tracking has + // been lost. In this case, we simply stop rendering. + // + // This is also the place to display UI to suggest the user walk + // to recover tracking. + Log.w(TAG, "Can't get device pose at time: " + + mRgbTimestampGlThread); + } + } + } + + // Avoid crashing the application due to unhandled exceptions. + } catch (TangoErrorException e) { + Log.e(TAG, "Tango API call error within the OpenGL render thread", e); + } catch (Throwable t) { + Log.e(TAG, "Exception on the OpenGL thread", t); + } + } + + @Override + public void onPreDraw(long sceneTime, double deltaTime) { + + } + + @Override + public void onPostFrame(long sceneTime, double deltaTime) { + + } + + @Override + public boolean callPreFrame() { + return true; + } + }); + + mSurfaceView.setSurfaceRenderer(mRenderer); + } + + /** + * Use Tango camera intrinsics to calculate the projection matrix for the Rajawali scene. + * + * @param intrinsics camera instrinsics for computing the project matrix. + */ + private static float[] projectionMatrixFromCameraIntrinsics(TangoCameraIntrinsics intrinsics) { + float cx = (float) intrinsics.cx; + float cy = (float) intrinsics.cy; + float width = (float) intrinsics.width; + float height = (float) intrinsics.height; + float fx = (float) intrinsics.fx; + float fy = (float) intrinsics.fy; + + // Uses frustumM to create a projection matrix taking into account calibrated camera + // intrinsic parameter. + // Reference: http://ksimek.github.io/2013/06/03/calibrated_cameras_in_opengl/ + float near = 0.1f; + float far = 100; + + float xScale = near / fx; + float yScale = near / fy; + float xOffset = (cx - (width / 2.0f)) * xScale; + // Color camera's coordinates has y pointing downwards so we negate this term. + float yOffset = -(cy - (height / 2.0f)) * yScale; + + float m[] = new float[16]; + Matrix.frustumM(m, 0, + xScale * (float) -width / 2.0f - xOffset, + xScale * (float) width / 2.0f - xOffset, + yScale * (float) -height / 2.0f - yOffset, + yScale * (float) height / 2.0f - yOffset, + near, far); + return m; + } + + /** + * Set the color camera background texture rotation and save the camera to display rotation. + */ + private void setDisplayRotation() { + Display display = getWindowManager().getDefaultDisplay(); + mDisplayRotation = display.getRotation(); + + // We also need to update the camera texture UV coordinates. This must be run in the OpenGL + // thread. + mSurfaceView.queueEvent(new Runnable() { + @Override + public void run() { + if (mIsConnected) { + mRenderer.updateColorCameraTextureUvGlThread(mDisplayRotation); + } + } + }); + } + + /** + * Check to see we have the necessary permissions for this app, and ask for them if we don't. + * + * @return True if we have the necessary permissions, false if we haven't. + */ + private boolean checkAndRequestPermissions() { + if (!hasCameraPermission()) { + requestCameraPermission(); + return false; + } + return true; + } + + /** + * Check to see we have the necessary permissions for this app. + */ + private boolean hasCameraPermission() { + return ContextCompat.checkSelfPermission(this, CAMERA_PERMISSION) == + PackageManager.PERMISSION_GRANTED; + } + + /** + * Request the necessary permissions for this app. + */ + private void requestCameraPermission() { + if (ActivityCompat.shouldShowRequestPermissionRationale(this, CAMERA_PERMISSION)) { + showRequestPermissionRationale(); + } else { + ActivityCompat.requestPermissions(this, new String[]{CAMERA_PERMISSION}, + CAMERA_PERMISSION_CODE); + } + } + + /** + * If the user has declined the permission before, we have to explain that the app needs this + * permission. + */ + private void showRequestPermissionRationale() { + final AlertDialog dialog = new AlertDialog.Builder(this) + .setMessage("Java Marker Detection Example requires camera permission") + .setPositiveButton("Ok", new DialogInterface.OnClickListener() { + @Override + public void onClick(DialogInterface dialogInterface, int i) { + ActivityCompat.requestPermissions(MarkerDetectionActivity.this, + new String[]{CAMERA_PERMISSION}, CAMERA_PERMISSION_CODE); + } + }) + .create(); + dialog.show(); + } + + /** + * Result for requesting camera permission. + */ + @Override + public void onRequestPermissionsResult(int requestCode, String[] permissions, + int[] grantResults) { + if (hasCameraPermission()) { + bindTangoService(); + } else { + Toast.makeText(this, "Java Marker Detection Example requires camera permission", + Toast.LENGTH_LONG).show(); + } + } + + /** + * Display toast on UI thread. + * + * @param resId The resource id of the string resource to use. Can be formatted text. + */ + private void showsToastAndFinishOnUiThread(final int resId) { + runOnUiThread(new Runnable() { + @Override + public void run() { + Toast.makeText(MarkerDetectionActivity.this, + getString(resId), Toast.LENGTH_LONG).show(); + finish(); + } + }); + } +} diff --git a/java_marker_detection_example/app/src/main/java/com/projecttango/examples/java/markerdetection/MarkerDetectionRenderer.java b/java_marker_detection_example/app/src/main/java/com/projecttango/examples/java/markerdetection/MarkerDetectionRenderer.java new file mode 100644 index 00000000..b3068cc5 --- /dev/null +++ b/java_marker_detection_example/app/src/main/java/com/projecttango/examples/java/markerdetection/MarkerDetectionRenderer.java @@ -0,0 +1,191 @@ +/* + * Copyright 2017 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.projecttango.examples.java.markerdetection; + +import com.google.atap.tangoservice.TangoPoseData; + +import android.content.Context; + +import android.util.Log; +import android.view.MotionEvent; +import org.rajawali3d.materials.Material; +import org.rajawali3d.materials.textures.ATexture; +import org.rajawali3d.materials.textures.StreamingTexture; +import org.rajawali3d.math.Matrix4; +import org.rajawali3d.math.Quaternion; +import org.rajawali3d.primitives.ScreenQuad; +import org.rajawali3d.renderer.Renderer; +import org.rajawali3d.scene.Scene; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.microedition.khronos.opengles.GL10; + +import com.projecttango.tangosupport.TangoSupport; + +/** + * Renderer that implements a basic augmented reality scene using Rajawali. + * It creates a scene with a background using color camera image, and renders the bounding box and + * three axes of any marker that has been detected in the image. + */ +public class MarkerDetectionRenderer extends Renderer { + private static final String TAG = MarkerDetectionRenderer.class.getSimpleName(); + + private float[] textureCoords0 = new float[] {0.0F, 1.0F, 0.0F, 0.0F, 1.0F, 1.0F, 1.0F, 0.0F}; + + // Rajawali texture used to render the Tango color camera. + private ATexture mTangoCameraTexture; + + // Keeps track of whether the scene camera has been configured. + private boolean mSceneCameraConfigured; + + private ScreenQuad mBackgroundQuad; + + // All markers + private Map mMarkerObjects; + + public MarkerDetectionRenderer(Context context) { + super(context); + } + + @Override + protected void initScene() { + if (mMarkerObjects == null) { + mMarkerObjects = new HashMap(); + } + + // Create a quad covering the whole background and assign a texture to it where the + // Tango color camera contents will be rendered. + Material tangoCameraMaterial = new Material(); + tangoCameraMaterial.setColorInfluence(0); + + if (mBackgroundQuad == null) { + mBackgroundQuad = new ScreenQuad(); + mBackgroundQuad.getGeometry().setTextureCoords(textureCoords0); + } + // We need to use Rajawali's {@code StreamingTexture} since it sets up the texture + // for GL_TEXTURE_EXTERNAL_OES rendering. + mTangoCameraTexture = + new StreamingTexture("camera", (StreamingTexture.ISurfaceListener) null); + + try { + tangoCameraMaterial.addTexture(mTangoCameraTexture); + mBackgroundQuad.setMaterial(tangoCameraMaterial); + } catch (ATexture.TextureException e) { + Log.e(TAG, "Exception creating texture for RGB camera contents", e); + } + getCurrentScene().addChildAt(mBackgroundQuad, 0); + } + + /** + * Update marker objects and scene. + */ + public void updateMarkers(List markerList) { + if (markerList.size() > 0) { + Scene scene = getCurrentScene(); + + // Create objects based on new markers + for (int i = 0; i < markerList.size(); i++) { + TangoSupport.Marker marker = markerList.get(i); + Log.w(TAG, "Marker detected[" + i + "] = " + marker.content); + + // Remove the marker object from scene if it exists. + MarkerObject existingObject = mMarkerObjects.get(marker.content); + if (existingObject != null) { + existingObject.removeFromScene(scene); + } + + // Create a new marker object and add it to scene. + MarkerObject newObject = new MarkerObject(marker); + mMarkerObjects.put(marker.content, newObject); + newObject.addToScene(scene); + } + } + } + + /** + * Update background texture's UV coordinates when device orientation is changed (i.e., change + * between landscape and portrait mode). + * This must be run in the OpenGL thread. + */ + public void updateColorCameraTextureUvGlThread(int rotation) { + if (mBackgroundQuad == null) { + mBackgroundQuad = new ScreenQuad(); + } + + float[] textureCoords = + TangoSupport.getVideoOverlayUVBasedOnDisplayRotation(textureCoords0, rotation); + mBackgroundQuad.getGeometry().setTextureCoords(textureCoords, true); + mBackgroundQuad.getGeometry().reload(); + } + + /** + * Update the scene camera based on the provided pose in Tango start of service frame. + * The camera pose should match the pose of the camera color at the time of the last rendered + * RGB frame, which can be retrieved with this.getTimestamp(); + *

+ * NOTE: This must be called from the OpenGL render thread; it is not thread-safe. + */ + public void updateRenderCameraPose(TangoPoseData cameraPose) { + float[] rotation = cameraPose.getRotationAsFloats(); + float[] translation = cameraPose.getTranslationAsFloats(); + Quaternion quaternion = new Quaternion(rotation[3], rotation[0], rotation[1], rotation[2]); + // Conjugating the Quaternion is needed because Rajawali uses left-handed convention for + // quaternions. + getCurrentCamera().setRotation(quaternion.conjugate()); + getCurrentCamera().setPosition(translation[0], translation[1], translation[2]); + } + + /** + * It returns the ID currently assigned to the texture where the Tango color camera contents + * should be rendered. + * NOTE: This must be called from the OpenGL render thread; it is not thread-safe. + */ + public int getTextureId() { + return mTangoCameraTexture == null ? -1 : mTangoCameraTexture.getTextureId(); + } + + /** + * We need to override this method to mark the camera for re-configuration (set proper + * projection matrix) since it will be reset by Rajawali on surface changes. + */ + @Override + public void onRenderSurfaceSizeChanged(GL10 gl, int width, int height) { + super.onRenderSurfaceSizeChanged(gl, width, height); + mSceneCameraConfigured = false; + } + + public boolean isSceneCameraConfigured() { + return mSceneCameraConfigured; + } + + /** + * Sets the projection matrix for the scene camera to match the parameters of the color camera, + * provided by the {@code TangoCameraIntrinsics}. + */ + public void setProjectionMatrix(float[] matrixFloats) { + getCurrentCamera().setProjectionMatrix(new Matrix4(matrixFloats)); + } + + @Override + public void onOffsetsChanged(float xOffset, float yOffset, float xOffsetStep, float yOffsetStep, + int xPixelOffset, int yPixelOffset) {} + + @Override + public void onTouchEvent(MotionEvent event) {} +} diff --git a/java_marker_detection_example/app/src/main/java/com/projecttango/examples/java/markerdetection/MarkerObject.java b/java_marker_detection_example/app/src/main/java/com/projecttango/examples/java/markerdetection/MarkerObject.java new file mode 100644 index 00000000..92301d6a --- /dev/null +++ b/java_marker_detection_example/app/src/main/java/com/projecttango/examples/java/markerdetection/MarkerObject.java @@ -0,0 +1,127 @@ +/* + * Copyright 2017 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.projecttango.examples.java.markerdetection; + +import android.graphics.Color; + +import org.rajawali3d.materials.Material; +import org.rajawali3d.math.Quaternion; +import org.rajawali3d.math.vector.Vector3; +import org.rajawali3d.primitives.Line3D; +import org.rajawali3d.scene.Scene; + +import java.util.Stack; + +import com.projecttango.tangosupport.TangoSupport; +/** + * Rajawali object which represents a marker. + */ +public class MarkerObject { + // 3D object for bounding box of the marker. + private Line3D mRect; + + // 3D object for three axes of the marker local frame. + private Line3D mAxisX; + private Line3D mAxisY; + private Line3D mAxisZ; + + /** + * Construct marker object from a marker. + */ + public MarkerObject(TangoSupport.Marker marker) { + // Create marker center and four corners. + Vector3 center = + new Vector3(marker.translation[0], marker.translation[1], marker.translation[2]); + Vector3 cornerBottomLeft = + new Vector3(marker.corners3d[0][0], marker.corners3d[0][1], marker.corners3d[0][2]); + Vector3 cornerBottomRight = + new Vector3(marker.corners3d[1][0], marker.corners3d[1][1], marker.corners3d[1][2]); + Vector3 cornerTopRight = + new Vector3(marker.corners3d[2][0], marker.corners3d[2][1], marker.corners3d[2][2]); + Vector3 cornerTopLeft = + new Vector3(marker.corners3d[3][0], marker.corners3d[3][1], marker.corners3d[3][2]); + + // Create a quaternion from marker orientation. + Quaternion q = new Quaternion(marker.orientation[3], marker.orientation[0], + marker.orientation[1], marker.orientation[2]); + + // Calculate marker size in meters, assuming square-shape markers. + double markerSize = cornerTopLeft.distanceTo(cornerTopRight); + + // Line width for drawing the axes and bounding rectangle. + final float axisLineWidth = 10.f; + final float rectLineWidth = 5.f; + + Stack points = new Stack(); + Material material = new Material(); + + // X axis + Vector3 xAxis = q.multiply(new Vector3(markerSize / 3, 0, 0)); + points.add(center); + points.add(Vector3.addAndCreate(center, xAxis)); + mAxisX = new Line3D(points, axisLineWidth, Color.RED); + mAxisX.setMaterial(material); + + // Y axis + points.clear(); + Vector3 yAxis = q.multiply(new Vector3(0, markerSize / 3, 0)); + points.add(center); + points.add(Vector3.addAndCreate(center, yAxis)); + mAxisY = new Line3D(points, axisLineWidth, Color.GREEN); + mAxisY.setMaterial(material); + + // Z axis + points.clear(); + Vector3 zAxis = q.multiply(new Vector3(0, 0, markerSize / 3)); + points.add(center); + points.add(Vector3.addAndCreate(center, zAxis)); + mAxisZ = new Line3D(points, axisLineWidth, Color.BLUE); + mAxisZ.setMaterial(material); + + // Rect + points.clear(); + points.add(cornerBottomLeft); + points.add(cornerBottomRight); + points.add(cornerBottomRight); + points.add(cornerTopRight); + points.add(cornerTopRight); + points.add(cornerTopLeft); + points.add(cornerTopLeft); + points.add(cornerBottomLeft); + mRect = new Line3D(points, rectLineWidth, Color.CYAN); + mRect.setMaterial(material); + } + + /** + * Add all 3D objects of a marker as children of the input scene. + */ + public void addToScene(Scene scene) { + scene.addChild(mAxisX); + scene.addChild(mAxisY); + scene.addChild(mAxisZ); + scene.addChild(mRect); + } + + /** + * Remove all 3D objects of a marker from the input scene. + */ + public void removeFromScene(Scene scene) { + scene.removeChild(mAxisX); + scene.removeChild(mAxisY); + scene.removeChild(mAxisZ); + scene.removeChild(mRect); + } +} diff --git a/java_marker_detection_example/app/src/main/java/com/projecttango/examples/java/markerdetection/package-info.java b/java_marker_detection_example/app/src/main/java/com/projecttango/examples/java/markerdetection/package-info.java new file mode 100644 index 00000000..527516b9 --- /dev/null +++ b/java_marker_detection_example/app/src/main/java/com/projecttango/examples/java/markerdetection/package-info.java @@ -0,0 +1,19 @@ +/* + * Copyright 2017 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * A sample showing how to build marker detection app using Tango and Rajawali. + */ +package com.projecttango.examples.java.markerdetection; diff --git a/java_marker_detection_example/app/src/main/res/drawable-xxhdpi/earth.jpeg b/java_marker_detection_example/app/src/main/res/drawable-xxhdpi/earth.jpeg new file mode 100644 index 00000000..959b435a Binary files /dev/null and b/java_marker_detection_example/app/src/main/res/drawable-xxhdpi/earth.jpeg differ diff --git a/java_marker_detection_example/app/src/main/res/drawable-xxhdpi/ic_launcher.png b/java_marker_detection_example/app/src/main/res/drawable-xxhdpi/ic_launcher.png new file mode 100644 index 00000000..2a715aca Binary files /dev/null and b/java_marker_detection_example/app/src/main/res/drawable-xxhdpi/ic_launcher.png differ diff --git a/java_marker_detection_example/app/src/main/res/drawable-xxhdpi/moon.jpg b/java_marker_detection_example/app/src/main/res/drawable-xxhdpi/moon.jpg new file mode 100644 index 00000000..f4a512d8 Binary files /dev/null and b/java_marker_detection_example/app/src/main/res/drawable-xxhdpi/moon.jpg differ diff --git a/java_marker_detection_example/app/src/main/res/layout/activity_main.xml b/java_marker_detection_example/app/src/main/res/layout/activity_main.xml new file mode 100644 index 00000000..914ea51c --- /dev/null +++ b/java_marker_detection_example/app/src/main/res/layout/activity_main.xml @@ -0,0 +1,28 @@ + + + + + + diff --git a/java_marker_detection_example/app/src/main/res/values/strings.xml b/java_marker_detection_example/app/src/main/res/values/strings.xml new file mode 100644 index 00000000..a8609e64 --- /dev/null +++ b/java_marker_detection_example/app/src/main/res/values/strings.xml @@ -0,0 +1,25 @@ + + + + Java Marker Detection + Java Marker Detection Example + + "Tango exception! Try again!" + "Tango invalid exception! Try again!" + "Tango service outdated!" + "Camera permission needed!" + diff --git a/java_marker_detection_example/app/src/main/res/values/styles.xml b/java_marker_detection_example/app/src/main/res/values/styles.xml new file mode 100644 index 00000000..0ecfbc53 --- /dev/null +++ b/java_marker_detection_example/app/src/main/res/values/styles.xml @@ -0,0 +1,35 @@ + + + + + + + + + + diff --git a/java_marker_detection_example/ar_markers.pdf b/java_marker_detection_example/ar_markers.pdf new file mode 100644 index 00000000..bbe8655d Binary files /dev/null and b/java_marker_detection_example/ar_markers.pdf differ diff --git a/java_marker_detection_example/build.gradle b/java_marker_detection_example/build.gradle new file mode 100644 index 00000000..695fe3fc --- /dev/null +++ b/java_marker_detection_example/build.gradle @@ -0,0 +1,19 @@ +// Top-level build file where you can add configuration options common to all sub-projects/modules. +buildscript { + repositories { + jcenter() + } + dependencies { + classpath 'com.android.tools.build:gradle:2.2.2' + + // NOTE: Do not place your application dependencies here; they belong + // in the individual module build.gradle files + } +} + +allprojects { + repositories { + mavenLocal() + jcenter() + } +} diff --git a/java_marker_detection_example/gradle.properties b/java_marker_detection_example/gradle.properties new file mode 100644 index 00000000..89e0d99e --- /dev/null +++ b/java_marker_detection_example/gradle.properties @@ -0,0 +1,18 @@ +# Project-wide Gradle settings. + +# IDE (e.g. Android Studio) users: +# Gradle settings configured through the IDE *will override* +# any settings specified in this file. + +# For more details on how to configure your build environment visit +# http://www.gradle.org/docs/current/userguide/build_environment.html + +# Specifies the JVM arguments used for the daemon process. +# The setting is particularly useful for tweaking memory settings. +# Default value: -Xmx10248m -XX:MaxPermSize=256m +# org.gradle.jvmargs=-Xmx2048m -XX:MaxPermSize=512m -XX:+HeapDumpOnOutOfMemoryError -Dfile.encoding=UTF-8 + +# When configured, Gradle will run in incubating parallel mode. +# This option should only be used with decoupled projects. More details, visit +# http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects +# org.gradle.parallel=true diff --git a/java_marker_detection_example/gradle/wrapper/gradle-wrapper.jar b/java_marker_detection_example/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 00000000..c97a8bdb Binary files /dev/null and b/java_marker_detection_example/gradle/wrapper/gradle-wrapper.jar differ diff --git a/java_marker_detection_example/gradle/wrapper/gradle-wrapper.properties b/java_marker_detection_example/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 00000000..53a7f3ca --- /dev/null +++ b/java_marker_detection_example/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +#Mon Oct 26 08:02:00 ART 2015 +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-2.14.1-bin.zip diff --git a/java_marker_detection_example/gradlew b/java_marker_detection_example/gradlew new file mode 100755 index 00000000..91a7e269 --- /dev/null +++ b/java_marker_detection_example/gradlew @@ -0,0 +1,164 @@ +#!/usr/bin/env bash + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS="" + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn ( ) { + echo "$*" +} + +die ( ) { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; +esac + +# For Cygwin, ensure paths are in UNIX format before anything is touched. +if $cygwin ; then + [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"` +fi + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >&- +APP_HOME="`pwd -P`" +cd "$SAVED" >&- + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin, switch paths to Windows format before running java +if $cygwin ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=$((i+1)) + done + case $i in + (0) set -- ;; + (1) set -- "$args0" ;; + (2) set -- "$args0" "$args1" ;; + (3) set -- "$args0" "$args1" "$args2" ;; + (4) set -- "$args0" "$args1" "$args2" "$args3" ;; + (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules +function splitJvmOpts() { + JVM_OPTS=("$@") +} +eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS +JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME" + +exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@" diff --git a/java_marker_detection_example/gradlew.bat b/java_marker_detection_example/gradlew.bat new file mode 100644 index 00000000..aec99730 --- /dev/null +++ b/java_marker_detection_example/gradlew.bat @@ -0,0 +1,90 @@ +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS= + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +@rem Get command-line arguments, handling Windowz variants + +if not "%OS%" == "Windows_NT" goto win9xME_args +if "%@eval[2+2]" == "4" goto 4NT_args + +:win9xME_args +@rem Slurp the command line arguments. +set CMD_LINE_ARGS= +set _SKIP=2 + +:win9xME_args_slurp +if "x%~1" == "x" goto execute + +set CMD_LINE_ARGS=%* +goto execute + +:4NT_args +@rem Get arguments from the 4NT Shell from JP Software +set CMD_LINE_ARGS=%$ + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/java_marker_detection_example/settings.gradle b/java_marker_detection_example/settings.gradle new file mode 100644 index 00000000..e7b4def4 --- /dev/null +++ b/java_marker_detection_example/settings.gradle @@ -0,0 +1 @@ +include ':app' diff --git a/java_mesh_builder_example/app/src/main/java/com/projecttango/examples/java/meshbuilder/MeshBuilderActivity.java b/java_mesh_builder_example/app/src/main/java/com/projecttango/examples/java/meshbuilder/MeshBuilderActivity.java index 87bd8b04..d8a9bfd6 100644 --- a/java_mesh_builder_example/app/src/main/java/com/projecttango/examples/java/meshbuilder/MeshBuilderActivity.java +++ b/java_mesh_builder_example/app/src/main/java/com/projecttango/examples/java/meshbuilder/MeshBuilderActivity.java @@ -141,10 +141,10 @@ private void bindTangoService() { public void run() { synchronized (MeshBuilderActivity.this) { try { - TangoSupport.initialize(); mConfig = setupTangoConfig(mTango); mTango.connect(mConfig); startupTango(); + TangoSupport.initialize(mTango); mIsConnected = true; mIsPaused = false; runOnUiThread(new Runnable() { diff --git a/java_mesh_builder_example/app/src/main/java/com/projecttango/examples/java/meshbuilder/TangoMesher.java b/java_mesh_builder_example/app/src/main/java/com/projecttango/examples/java/meshbuilder/TangoMesher.java index b9afa5a3..b61ca114 100644 --- a/java_mesh_builder_example/app/src/main/java/com/projecttango/examples/java/meshbuilder/TangoMesher.java +++ b/java_mesh_builder_example/app/src/main/java/com/projecttango/examples/java/meshbuilder/TangoMesher.java @@ -23,6 +23,7 @@ import com.google.atap.tangoservice.Tango; import com.google.atap.tangoservice.TangoCameraIntrinsics; import com.google.atap.tangoservice.TangoEvent; +import com.google.atap.tangoservice.TangoInvalidException; import com.google.atap.tangoservice.TangoPointCloudData; import com.google.atap.tangoservice.TangoPoseData; import com.google.atap.tangoservice.TangoXyzIjData; @@ -97,15 +98,25 @@ public void run() { } TangoPointCloudData cloudData = mPointCloudBuffer.getLatestPointCloud(); - TangoPoseData depthPose = TangoSupport.getPoseAtTime(cloudData.timestamp, - TangoPoseData.COORDINATE_FRAME_START_OF_SERVICE, - TangoPoseData.COORDINATE_FRAME_CAMERA_DEPTH, - TangoSupport.TANGO_SUPPORT_ENGINE_TANGO, - TangoSupport.TANGO_SUPPORT_ENGINE_TANGO, - TangoSupport.ROTATION_IGNORED); - if (depthPose.statusCode != TangoPoseData.POSE_VALID) { - Log.e(TAG, "couldn't extract a valid depth pose"); - return; + TangoPoseData depthPose = null; + try { + depthPose = TangoSupport.getPoseAtTime(cloudData.timestamp, + TangoPoseData.COORDINATE_FRAME_START_OF_SERVICE, + TangoPoseData.COORDINATE_FRAME_CAMERA_DEPTH, + TangoSupport.TANGO_SUPPORT_ENGINE_TANGO, + TangoSupport.TANGO_SUPPORT_ENGINE_TANGO, + TangoSupport.ROTATION_IGNORED); + if (depthPose.statusCode != TangoPoseData.POSE_VALID) { + Log.e(TAG, "couldn't extract a valid depth pose"); + return; + } + } catch (TangoInvalidException e) { + e.printStackTrace(); + depthPose = null; + } + if (depthPose == null) { + Log.e(TAG, "couldn't extract a valid depth pose"); + return; } TangoImageBuffer imageBuffer = null; diff --git a/java_model_correspondence_example/app/src/main/java/com/projecttango/examples/java/modelcorrespondence/ModelCorrespondenceActivity.java b/java_model_correspondence_example/app/src/main/java/com/projecttango/examples/java/modelcorrespondence/ModelCorrespondenceActivity.java index 3e649688..dde6b1f0 100644 --- a/java_model_correspondence_example/app/src/main/java/com/projecttango/examples/java/modelcorrespondence/ModelCorrespondenceActivity.java +++ b/java_model_correspondence_example/app/src/main/java/com/projecttango/examples/java/modelcorrespondence/ModelCorrespondenceActivity.java @@ -206,10 +206,10 @@ public void run() { // thread or in the UI thread. synchronized (ModelCorrespondenceActivity.this) { try { - TangoSupport.initialize(); mConfig = setupTangoConfig(mTango); mTango.connect(mConfig); startupTango(); + TangoSupport.initialize(mTango); connectRenderer(); mIsConnected = true; setDisplayRotation(); diff --git a/java_motion_tracking_example/app/src/main/AndroidManifest.xml b/java_motion_tracking_example/app/src/main/AndroidManifest.xml index e993d2b5..b44310b4 100644 --- a/java_motion_tracking_example/app/src/main/AndroidManifest.xml +++ b/java_motion_tracking_example/app/src/main/AndroidManifest.xml @@ -22,6 +22,7 @@ android:name=".MotionTrackingActivity" android:label="@string/app_name_long" android:screenOrientation="fullSensor" + android:configChanges="orientation|screenSize" android:icon="@drawable/ic_launcher"> diff --git a/java_motion_tracking_example/app/src/main/java/com/projecttango/examples/java/motiontracking/MotionTrackingActivity.java b/java_motion_tracking_example/app/src/main/java/com/projecttango/examples/java/motiontracking/MotionTrackingActivity.java index ac120636..dbdef796 100644 --- a/java_motion_tracking_example/app/src/main/java/com/projecttango/examples/java/motiontracking/MotionTrackingActivity.java +++ b/java_motion_tracking_example/app/src/main/java/com/projecttango/examples/java/motiontracking/MotionTrackingActivity.java @@ -28,6 +28,7 @@ import com.google.atap.tangoservice.TangoXyzIjData; import android.app.Activity; +import android.hardware.display.DisplayManager; import android.opengl.GLSurfaceView; import android.os.Bundle; import android.util.Log; @@ -69,11 +70,28 @@ protected void onCreate(Bundle savedInstanceState) { mSurfaceView = (RajawaliSurfaceView) findViewById(R.id.gl_surface_view); mRenderer = new MotionTrackingRajawaliRenderer(this); - // Get current display orientation. Note that each time display orientation - // changes, the onCreate function will be called again. - WindowManager mWindowManager = (WindowManager) getSystemService(WINDOW_SERVICE); - Display mDisplay = mWindowManager.getDefaultDisplay(); - mDisplayRotation = mDisplay.getRotation(); + // Register a listener to get the current display orientation. If we don't manage this + // event, the activity will be destroyed and recreated on each rotation, forcing the + // disconnection from the Tango Service. + DisplayManager displayManager = (DisplayManager) getSystemService(DISPLAY_SERVICE); + if (displayManager != null) { + displayManager.registerDisplayListener(new DisplayManager.DisplayListener() { + @Override + public void onDisplayAdded(int displayId) { + } + + @Override + public void onDisplayChanged(int displayId) { + synchronized (this) { + setDisplayRotation(); + } + } + + @Override + public void onDisplayRemoved(int displayId) { + } + }, null); + } // Configure OpenGL renderer. setupRenderer(); @@ -97,10 +115,11 @@ public void run() { // thread or in the UI thread. synchronized (MotionTrackingActivity.this) { try { - TangoSupport.initialize(); mConfig = setupTangoConfig(mTango); mTango.connect(mConfig); startupTango(); + TangoSupport.initialize(mTango); + setDisplayRotation(); } catch (TangoOutOfDateException e) { Log.e(TAG, getString(R.string.exception_out_of_date), e); showsToastAndFinishOnUiThread(R.string.exception_out_of_date); @@ -164,11 +183,10 @@ private void startupTango() { mTango.connectListener(framePairs, new Tango.OnTangoUpdateListener() { @Override public void onPoseAvailable(final TangoPoseData pose) { - synchronized (MotionTrackingActivity.this) { - // When we receive the first onPoseAvailable callback, we know the device has - // located itself. - mIsTangoPoseReady.compareAndSet(false, true); - } + // As noted above, do not synchronize against MotionTrackingActivity.this. + // When we receive the first onPoseAvailable callback, we know the device has + // located itself. + mIsTangoPoseReady.compareAndSet(false, true); } @Override @@ -252,6 +270,14 @@ public boolean callPreFrame() { mSurfaceView.setRenderMode(GLSurfaceView.RENDERMODE_CONTINUOUSLY); } + /** + * Set the current display rotation. + */ + private void setDisplayRotation() { + Display display = getWindowManager().getDefaultDisplay(); + mDisplayRotation = display.getRotation(); + } + /** * Display toast on UI thread. * diff --git a/java_occlusion_example/app/src/main/java/com/projecttango/examples/java/occlusion/OcclusionActivity.java b/java_occlusion_example/app/src/main/java/com/projecttango/examples/java/occlusion/OcclusionActivity.java index 65368731..f6783313 100644 --- a/java_occlusion_example/app/src/main/java/com/projecttango/examples/java/occlusion/OcclusionActivity.java +++ b/java_occlusion_example/app/src/main/java/com/projecttango/examples/java/occlusion/OcclusionActivity.java @@ -178,10 +178,10 @@ private void bindTangoService() { public void run() { try { synchronized (OcclusionActivity.this) { - TangoSupport.initialize(); mConfig = setupTangoConfig(mTango); mTango.connect(mConfig); startupTango(); + TangoSupport.initialize(mTango); mIsConnected = true; setDisplayRotation(); } diff --git a/java_opengl_augmented_reality_example/opengl_ar/src/main/java/com/projecttango/examples/java/openglar/OpenGlAugmentedRealityActivity.java b/java_opengl_augmented_reality_example/opengl_ar/src/main/java/com/projecttango/examples/java/openglar/OpenGlAugmentedRealityActivity.java index 5584869a..89424e68 100644 --- a/java_opengl_augmented_reality_example/opengl_ar/src/main/java/com/projecttango/examples/java/openglar/OpenGlAugmentedRealityActivity.java +++ b/java_opengl_augmented_reality_example/opengl_ar/src/main/java/com/projecttango/examples/java/openglar/OpenGlAugmentedRealityActivity.java @@ -187,10 +187,10 @@ public void run() { // thread or in the UI thread. synchronized (OpenGlAugmentedRealityActivity.this) { try { - TangoSupport.initialize(); mConfig = setupTangoConfig(mTango); mTango.connect(mConfig); startupTango(); + TangoSupport.initialize(mTango); mIsConnected = true; setDisplayRotation(); } catch (TangoOutOfDateException e) { diff --git a/java_plane_fitting_example/app/src/main/java/com/projecttango/examples/java/planefitting/PlaneFittingActivity.java b/java_plane_fitting_example/app/src/main/java/com/projecttango/examples/java/planefitting/PlaneFittingActivity.java index e8da7e6f..4de32864 100644 --- a/java_plane_fitting_example/app/src/main/java/com/projecttango/examples/java/planefitting/PlaneFittingActivity.java +++ b/java_plane_fitting_example/app/src/main/java/com/projecttango/examples/java/planefitting/PlaneFittingActivity.java @@ -47,6 +47,7 @@ import android.view.View; import android.widget.Toast; +import org.rajawali3d.math.Matrix4; import org.rajawali3d.scene.ASceneFrameCallback; import org.rajawali3d.view.SurfaceView; @@ -101,6 +102,9 @@ public class PlaneFittingActivity extends Activity implements View.OnTouchListen private int mDisplayRotation; + private float[] mDepthTPlane; + private double mPlanePlacedTimestamp; + @Override protected void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); @@ -183,10 +187,10 @@ public void run() { // thread or in the UI thread. synchronized (PlaneFittingActivity.this) { try { - TangoSupport.initialize(); mConfig = setupTangoConfig(mTango); mTango.connect(mConfig); startupTango(); + TangoSupport.initialize(mTango); connectRenderer(); mIsConnected = true; setDisplayRotation(); @@ -351,6 +355,29 @@ public void onPreFrame(long sceneTime, double deltaTime) { Log.w(TAG, "Can't get device pose at time: " + mRgbTimestampGlThread); } + + if (mDepthTPlane != null) { + // Update the position of the rendered cube to the pose of the + // detected plane. This update is made thread-safe by the renderer. + // + // To make sure drift corrected pose is applied to the virtual + // object we need to re-query the Area Description to Depth camera + // at the time when the corresponding plane fitting + // measurement was acquired. + TangoSupport.TangoMatrixTransformData openglTDepthArr = + TangoSupport.getMatrixTransformAtTime( + mPlanePlacedTimestamp, + TangoPoseData.COORDINATE_FRAME_AREA_DESCRIPTION, + TangoPoseData.COORDINATE_FRAME_CAMERA_DEPTH, + TangoSupport.TANGO_SUPPORT_ENGINE_OPENGL, + TangoSupport.TANGO_SUPPORT_ENGINE_TANGO, + TangoSupport.ROTATION_IGNORED); + + if (openglTDepthArr.statusCode == TangoPoseData.POSE_VALID) { + mRenderer.updateObjectPose(openglTDepthArr.matrix, + mDepthTPlane); + } + } } } // Avoid crashing the application due to unhandled exceptions. @@ -422,15 +449,8 @@ public boolean onTouch(View view, MotionEvent motionEvent) { // Fit a plane on the clicked point using the latest point cloud data. // Synchronize against concurrent access to the RGB timestamp in the OpenGL thread // and a possible service disconnection due to an onPause event. - float[] planeFitTransform; synchronized (this) { - planeFitTransform = doFitPlane(u, v, mRgbTimestampGlThread); - } - - if (planeFitTransform != null) { - // Update the position of the rendered cube to the pose of the detected plane. - // This update is made thread-safe by the renderer. - mRenderer.updateObjectPose(planeFitTransform); + mDepthTPlane = doFitPlane(u, v, mRgbTimestampGlThread); } } catch (TangoException t) { @@ -460,50 +480,44 @@ private float[] doFitPlane(float u, float v, double rgbTimestamp) { return null; } - // Get pose transforms for depth/color cameras from world frame in - // OpenGL engine space. - TangoPoseData openglToDepthPose = TangoSupport.getPoseAtTime( - pointCloud.timestamp, - TangoPoseData.COORDINATE_FRAME_AREA_DESCRIPTION, - TangoPoseData.COORDINATE_FRAME_CAMERA_DEPTH, - TangoSupport.TANGO_SUPPORT_ENGINE_OPENGL, - TangoSupport.TANGO_SUPPORT_ENGINE_TANGO, - TangoSupport.ROTATION_IGNORED); - if (openglToDepthPose.statusCode != TangoPoseData.POSE_VALID) { - Log.d(TAG, "Could not get a valid pose from area description " - + "to depth camera at time " + pointCloud.timestamp); - return null; - } - - TangoPoseData openglToColorPose = TangoSupport.getPoseAtTime( + TangoPoseData depthToColorPose = TangoSupport.getPoseAtTime( rgbTimestamp, - TangoPoseData.COORDINATE_FRAME_AREA_DESCRIPTION, + TangoPoseData.COORDINATE_FRAME_CAMERA_DEPTH, TangoPoseData.COORDINATE_FRAME_CAMERA_COLOR, - TangoSupport.TANGO_SUPPORT_ENGINE_OPENGL, + TangoSupport.TANGO_SUPPORT_ENGINE_TANGO, TangoSupport.TANGO_SUPPORT_ENGINE_TANGO, TangoSupport.ROTATION_IGNORED); - if (openglToDepthPose.statusCode != TangoPoseData.POSE_VALID) { - Log.d(TAG, "Could not get a valid pose from area description " + if (depthToColorPose.statusCode != TangoPoseData.POSE_VALID) { + Log.d(TAG, "Could not get a valid pose from depth camera" + "to color camera at time " + rgbTimestamp); return null; } - // Plane model is in OpenGL space due to input poses. + // Plane model is in depth camera space due to input poses. IntersectionPointPlaneModelPair intersectionPointPlaneModelPair = TangoSupport.fitPlaneModelNearPoint(pointCloud, - openglToDepthPose.translation, - openglToDepthPose.rotation, u, v, + new double[] {0.0, 0.0, 0.0}, + new double[] {0.0, 0.0, 0.0, 1.0}, + u, v, mDisplayRotation, - openglToColorPose.translation, - openglToColorPose.rotation); - // Convert plane model into 4x4 matrix. The first 3 elements of - // the plane model make up the normal vector. - float[] openglUp = new float[]{0, 1, 0, 0}; - float[] openglToPlaneMatrix = matrixFromPointNormalUp( - intersectionPointPlaneModelPair.intersectionPoint, - intersectionPointPlaneModelPair.planeModel, - openglUp); - return openglToPlaneMatrix; + depthToColorPose.translation, + depthToColorPose.rotation); + + mPlanePlacedTimestamp = mRgbTimestampGlThread; + return convertPlaneModelToMatrix(intersectionPointPlaneModelPair); + } + + private float[] convertPlaneModelToMatrix(IntersectionPointPlaneModelPair planeModel) { + // Note that depth camera's space is: + // X - right + // Y - down + // Z - forward + float[] up = new float[]{0, 1, 0, 0}; + float[] depthTPlane = matrixFromPointNormalUp( + planeModel.intersectionPoint, + planeModel.planeModel, + up); + return depthTPlane; } /** diff --git a/java_plane_fitting_example/app/src/main/java/com/projecttango/examples/java/planefitting/PlaneFittingRenderer.java b/java_plane_fitting_example/app/src/main/java/com/projecttango/examples/java/planefitting/PlaneFittingRenderer.java index 83a868fe..51edd01c 100644 --- a/java_plane_fitting_example/app/src/main/java/com/projecttango/examples/java/planefitting/PlaneFittingRenderer.java +++ b/java_plane_fitting_example/app/src/main/java/com/projecttango/examples/java/planefitting/PlaneFittingRenderer.java @@ -15,14 +15,12 @@ */ package com.projecttango.examples.java.planefitting; -import com.google.atap.tangoservice.TangoCameraIntrinsics; import com.google.atap.tangoservice.TangoPoseData; import android.content.Context; - +import android.graphics.Color; import android.util.Log; import android.view.MotionEvent; -import android.view.Surface; import org.rajawali3d.Object3D; import org.rajawali3d.lights.DirectionalLight; @@ -33,10 +31,13 @@ import org.rajawali3d.materials.textures.Texture; import org.rajawali3d.math.Matrix4; import org.rajawali3d.math.Quaternion; +import org.rajawali3d.math.vector.Vector3; import org.rajawali3d.primitives.Cube; +import org.rajawali3d.primitives.Line3D; import org.rajawali3d.primitives.ScreenQuad; import org.rajawali3d.renderer.Renderer; +import java.util.Stack; import javax.microedition.khronos.opengles.GL10; import com.projecttango.tangosupport.TangoSupport; @@ -47,8 +48,16 @@ * method. */ public class PlaneFittingRenderer extends Renderer { - private static final float CUBE_SIDE_LENGTH = 0.5f; private static final String TAG = PlaneFittingRenderer.class.getSimpleName(); + private static final float CUBE_SIDE_LENGTH = 0.5f; + private static final float AXIS_THICKNESS = 20.0f; + + private static final Matrix4 DEPTH_T_OPENGL = new Matrix4(new float[] { + 1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 1.0f, 0.0f, + 0.0f, -1.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 1.0f + }); private float[] textureCoords0 = new float[]{0.0F, 1.0F, 0.0F, 0.0F, 1.0F, 1.0F, 1.0F, 0.0F}; @@ -95,23 +104,54 @@ protected void initScene() { light.setPosition(3, 2, 4); getCurrentScene().addLight(light); - // Set up a material: green with application of the light and - // instructions. - Material material = new Material(); - material.setColor(0xff009900); + // Build a Axis and a cube to represent plane's transformation. + + Cube cube = new Cube(CUBE_SIDE_LENGTH); + Material cubeMaterial = new Material(); + cubeMaterial.setColor(0xff009900); + cubeMaterial.setColorInfluence(0.2f); + cubeMaterial.enableLighting(true); + cubeMaterial.setDiffuseMethod(new DiffuseMethod.Lambert()); try { Texture t = new Texture("instructions", R.drawable.instructions); - material.addTexture(t); + cubeMaterial.addTexture(t); } catch (ATexture.TextureException e) { - e.printStackTrace(); + e.printStackTrace();; } - material.setColorInfluence(0.1f); - material.enableLighting(true); - material.setDiffuseMethod(new DiffuseMethod.Lambert()); + cube.setMaterial(cubeMaterial); + + Line3D xLine3d, yLine3d, zLine3d; + Material lineMaterial = new Material(); + + Stack xpoints = new Stack(); + xpoints.push(new Vector3(CUBE_SIDE_LENGTH, 0.0f, 0.0f)); + xpoints.push(new Vector3(0.0f, 0.0f, 0.0f)); - // Build a Cube and place it initially three meters forward from the origin. - mObject = new Cube(CUBE_SIDE_LENGTH); - mObject.setMaterial(material); + Stack ypoints = new Stack(); + ypoints.push(new Vector3(0.0f, 0.0f, 0.0f)); + ypoints.push(new Vector3(0.0f, CUBE_SIDE_LENGTH, 0.0f)); + + Stack zpoints = new Stack(); + zpoints.push(new Vector3(0.0f, 0.0f, 0.0f)); + zpoints.push(new Vector3(0.0f, 0.0f, CUBE_SIDE_LENGTH)); + + xLine3d = new Line3D(xpoints, AXIS_THICKNESS, Color.RED); + xLine3d.setMaterial(lineMaterial); + + yLine3d = new Line3D(ypoints, AXIS_THICKNESS, Color.GREEN); + yLine3d.setMaterial(lineMaterial); + + zLine3d = new Line3D(zpoints, AXIS_THICKNESS, Color.BLUE); + zLine3d.setMaterial(lineMaterial); + + mObject = new Object3D(); + mObject.addChild(xLine3d); + mObject.addChild(yLine3d); + mObject.addChild(zLine3d); + mObject.addChild(cube); + cube.moveUp(CUBE_SIDE_LENGTH / 2.0); + cube.moveForward(CUBE_SIDE_LENGTH / 2.0); + cube.moveRight(CUBE_SIDE_LENGTH / 2.0); mObject.setPosition(0, 0, -3); getCurrentScene().addChild(mObject); } @@ -145,7 +185,6 @@ protected void onRender(long elapsedRealTime, double deltaTime) { mObject.setOrientation(new Quaternion().fromMatrix(mObjectTransform)); // Move it forward by half of the size of the cube to make it // flush with the plane surface. - mObject.moveForward(CUBE_SIDE_LENGTH / 2.0f); mObjectPoseUpdated = false; } } @@ -157,8 +196,14 @@ protected void onRender(long elapsedRealTime, double deltaTime) { * Save the updated plane fit pose to update the AR object on the next render pass. * This is synchronized against concurrent access in the render loop above. */ - public synchronized void updateObjectPose(float[] planeFitTransform) { - mObjectTransform = new Matrix4(planeFitTransform); + public synchronized void updateObjectPose( + float[] openglTDepthArr, + float[] mDepthTPlaneArr) { + Matrix4 openglTDepth = new Matrix4(openglTDepthArr); + Matrix4 openglTPlane = + openglTDepth.multiply(new Matrix4(mDepthTPlaneArr)); + + mObjectTransform = openglTPlane.multiply(DEPTH_T_OPENGL);; mObjectPoseUpdated = true; } diff --git a/java_point_cloud_example/app/src/main/java/com/projecttango/examples/java/pointcloud/PointCloudActivity.java b/java_point_cloud_example/app/src/main/java/com/projecttango/examples/java/pointcloud/PointCloudActivity.java index 96f70992..b9858978 100644 --- a/java_point_cloud_example/app/src/main/java/com/projecttango/examples/java/pointcloud/PointCloudActivity.java +++ b/java_point_cloud_example/app/src/main/java/com/projecttango/examples/java/pointcloud/PointCloudActivity.java @@ -163,10 +163,10 @@ public void run() { // thread or in the UI thread. synchronized (PointCloudActivity.this) { try { - TangoSupport.initialize(); mConfig = setupTangoConfig(mTango); mTango.connect(mConfig); startupTango(); + TangoSupport.initialize(mTango); mIsConnected = true; setDisplayRotation(); } catch (TangoOutOfDateException e) { diff --git a/java_point_to_point_example/app/src/main/java/com/projecttango/examples/java/pointtopoint/PointToPointActivity.java b/java_point_to_point_example/app/src/main/java/com/projecttango/examples/java/pointtopoint/PointToPointActivity.java index 0af828c0..269dfdd6 100644 --- a/java_point_to_point_example/app/src/main/java/com/projecttango/examples/java/pointtopoint/PointToPointActivity.java +++ b/java_point_to_point_example/app/src/main/java/com/projecttango/examples/java/pointtopoint/PointToPointActivity.java @@ -78,6 +78,16 @@ * see java_augmented_reality_example or java_hello_video_example. */ public class PointToPointActivity extends Activity implements View.OnTouchListener { + private class MeasuredPoint { + public double mTimestamp; + public float[] mDepthTPoint; + + public MeasuredPoint(double timestamp, float[] depthTPoint) { + mTimestamp = timestamp; + mDepthTPoint = depthTPoint; + } + } + private static final String TAG = PointToPointActivity.class.getSimpleName(); private static final String CAMERA_PERMISSION = Manifest.permission.CAMERA; @@ -96,7 +106,7 @@ public class PointToPointActivity extends Activity implements View.OnTouchListen private TangoConfig mConfig; private boolean mIsConnected = false; private double mCameraPoseTimestamp = 0; - private TextView mDistanceMeasure; + private TextView mDistanceTextview; private CheckBox mBilateralBox; private volatile TangoImageBuffer mCurrentImageBuffer; @@ -106,9 +116,17 @@ public class PointToPointActivity extends Activity implements View.OnTouchListen private AtomicBoolean mIsFrameAvailableTangoThread = new AtomicBoolean(false); private double mRgbTimestampGlThread; - private float[][] mLinePoints = new float[2][3]; private boolean mPointSwitch = true; + // Two measured points in Depth Camera space. + private MeasuredPoint[] mMeasuredPoints = new MeasuredPoint[2]; + + // Two measured points in OpenGL space, we used a stack to hold the data is because rajawalli + // LineRenderer expects a stack of points to be passed in. This is render ready data format from + // Rajawalli's perspective. + private Stack mMeasurePoitnsInOpenGLSpace = new Stack(); + private float mMeasuredDistance = 0.0f; + // Handles the debug text UI update loop. private Handler mHandler = new Handler(); @@ -123,10 +141,10 @@ protected void onCreate(Bundle savedInstanceState) { mSurfaceView.setSurfaceRenderer(mRenderer); mSurfaceView.setOnTouchListener(this); mPointCloudManager = new TangoPointCloudManager(); - mDistanceMeasure = (TextView) findViewById(R.id.distance_textview); + mDistanceTextview = (TextView) findViewById(R.id.distance_textview); mBilateralBox = (CheckBox) findViewById(R.id.check_box); - mLinePoints[0] = null; - mLinePoints[1] = null; + mMeasuredPoints[0] = null; + mMeasuredPoints[1] = null; DisplayManager displayManager = (DisplayManager) getSystemService(DISPLAY_SERVICE); if (displayManager != null) { @@ -203,10 +221,10 @@ public void run() { // thread or in the UI thread. synchronized (PointToPointActivity.this) { try { - TangoSupport.initialize(); mConfig = setupTangoConfig(mTango); mTango.connect(mConfig); startupTango(); + TangoSupport.initialize(mTango); connectRenderer(); mIsConnected = true; setDisplayRotation(); @@ -401,6 +419,56 @@ public void onPreFrame(long sceneTime, double deltaTime) { Log.w(TAG, "Can't get device pose at time: " + mRgbTimestampGlThread); } + + // If both points have been measured, we transform the points to OpenGL + // space, and send it to mRenderer to render. + if (mMeasuredPoints[0] != null && mMeasuredPoints[1] != null) { + // To make sure drift correct pose is also applied to virtual + // object (measured points). + // We need to re-query the Area Description to Depth camera + // pose every frame. Note that you will need to use the timestamp + // at the time when the points were measured to query the pose. + TangoSupport.TangoMatrixTransformData openglTDepthArr0 = + TangoSupport.getMatrixTransformAtTime( + mMeasuredPoints[0].mTimestamp, + TangoPoseData.COORDINATE_FRAME_AREA_DESCRIPTION, + TangoPoseData.COORDINATE_FRAME_CAMERA_DEPTH, + TangoSupport.TANGO_SUPPORT_ENGINE_OPENGL, + TangoSupport.TANGO_SUPPORT_ENGINE_TANGO, + TangoSupport.ROTATION_IGNORED); + + TangoSupport.TangoMatrixTransformData openglTDepthArr1 = + TangoSupport.getMatrixTransformAtTime( + mMeasuredPoints[1].mTimestamp, + TangoPoseData.COORDINATE_FRAME_AREA_DESCRIPTION, + TangoPoseData.COORDINATE_FRAME_CAMERA_DEPTH, + TangoSupport.TANGO_SUPPORT_ENGINE_OPENGL, + TangoSupport.TANGO_SUPPORT_ENGINE_TANGO, + TangoSupport.ROTATION_IGNORED); + + if (openglTDepthArr0.statusCode == TangoPoseData.POSE_VALID && + openglTDepthArr1.statusCode == TangoPoseData.POSE_VALID) { + mMeasurePoitnsInOpenGLSpace.clear(); + float[] p0 = TangoSupport.transformPoint( + openglTDepthArr0.matrix, + mMeasuredPoints[0].mDepthTPoint); + float[] p1 = TangoSupport.transformPoint( + openglTDepthArr1.matrix, + mMeasuredPoints[1].mDepthTPoint); + + mMeasurePoitnsInOpenGLSpace.push( + new Vector3(p0[0], p0[1], p0[2])); + mMeasurePoitnsInOpenGLSpace.push( + new Vector3(p1[0], p1[1], p1[2])); + + mMeasuredDistance = (float) Math.sqrt( + Math.pow(p0[0] - p1[0], 2) + + Math.pow(p0[1] - p1[1], 2) + + Math.pow(p0[2] - p1[2], 2)); + } + } + + mRenderer.setLine(mMeasurePoitnsInOpenGLSpace); } } // Avoid crashing the application due to unhandled exceptions. @@ -472,15 +540,14 @@ public boolean onTouch(View view, MotionEvent motionEvent) { // Place point near the clicked point using the latest point cloud data. // Synchronize against concurrent access to the RGB timestamp in the OpenGL thread // and a possible service disconnection due to an onPause event. - float[] rgbPoint; + MeasuredPoint newMeasuredPoint; synchronized (this) { - rgbPoint = getDepthAtTouchPosition(u, v); + newMeasuredPoint = getDepthAtTouchPosition(u, v); } - if (rgbPoint != null) { + if (newMeasuredPoint != null) { // Update a line endpoint to the touch location. // This update is made thread-safe by the renderer. - updateLine(rgbPoint); - mRenderer.setLine(generateEndpoints()); + updateLine(newMeasuredPoint); } else { Log.w(TAG, "Point was null."); } @@ -505,7 +572,7 @@ public boolean onTouch(View view, MotionEvent motionEvent) { * of the point closest to where the user touches the screen. It returns a * Vector3 in OpenGL world space. */ - private float[] getDepthAtTouchPosition(float u, float v) { + private MeasuredPoint getDepthAtTouchPosition(float u, float v) { TangoPointCloudData pointCloud = mPointCloudManager.getLatestPointCloud(); if (pointCloud == null) { return null; @@ -519,123 +586,84 @@ private float[] getDepthAtTouchPosition(float u, float v) { rgbTimestamp = mRgbTimestampGlThread; // GPU. } - // Get pose transforms for openGL to depth/color cameras. - TangoPoseData oglTdepthPose = TangoSupport.getPoseAtTime( - pointCloud.timestamp, - TangoPoseData.COORDINATE_FRAME_AREA_DESCRIPTION, - TangoPoseData.COORDINATE_FRAME_CAMERA_DEPTH, - TangoSupport.TANGO_SUPPORT_ENGINE_OPENGL, - TangoSupport.TANGO_SUPPORT_ENGINE_TANGO, - TangoSupport.ROTATION_IGNORED); - if (oglTdepthPose.statusCode != TangoPoseData.POSE_VALID) { - Log.w(TAG, "Could not get depth camera transform at time " - + pointCloud.timestamp); - return null; - } - TangoPoseData oglTcolorPose = TangoSupport.getPoseAtTime( - rgbTimestamp, - TangoPoseData.COORDINATE_FRAME_AREA_DESCRIPTION, - TangoPoseData.COORDINATE_FRAME_CAMERA_COLOR, - TangoSupport.TANGO_SUPPORT_ENGINE_OPENGL, - TangoSupport.TANGO_SUPPORT_ENGINE_TANGO, - TangoSupport.ROTATION_IGNORED); - if (oglTcolorPose.statusCode != TangoPoseData.POSE_VALID) { + TangoPoseData depthlTcolorPose = TangoSupport.getPoseAtTime( + rgbTimestamp, + TangoPoseData.COORDINATE_FRAME_CAMERA_DEPTH, + TangoPoseData.COORDINATE_FRAME_CAMERA_COLOR, + TangoSupport.TANGO_SUPPORT_ENGINE_TANGO, + TangoSupport.TANGO_SUPPORT_ENGINE_TANGO, + TangoSupport.ROTATION_IGNORED); + if (depthlTcolorPose.statusCode != TangoPoseData.POSE_VALID) { Log.w(TAG, "Could not get color camera transform at time " + rgbTimestamp); return null; } - float[] openglPoint; + float[] depthPoint; if (mBilateralBox.isChecked()) { - openglPoint = TangoSupport.getDepthAtPointBilateral(pointCloud, - oglTdepthPose.translation, oglTdepthPose.rotation, - imageBuffer, u, v, mDisplayRotation, oglTcolorPose.translation, - oglTcolorPose.rotation); + depthPoint = TangoSupport.getDepthAtPointBilateral( + pointCloud, + new double[] {0.0, 0.0, 0.0}, + new double[] {0.0, 0.0, 0.0, 1.0}, + imageBuffer, + u, v, + mDisplayRotation, + depthlTcolorPose.translation, + depthlTcolorPose.rotation); } else { - openglPoint = TangoSupport.getDepthAtPointNearestNeighbor( - pointCloud, oglTdepthPose.translation, oglTdepthPose.rotation, - u, v, mDisplayRotation, oglTcolorPose.translation, - oglTcolorPose.rotation); + depthPoint = TangoSupport.getDepthAtPointNearestNeighbor( + pointCloud, + new double[] {0.0, 0.0, 0.0}, + new double[] {0.0, 0.0, 0.0, 1.0}, + u, v, + mDisplayRotation, + depthlTcolorPose.translation, + depthlTcolorPose.rotation); } - if (openglPoint == null) { + + if (depthPoint == null) { return null; } - return openglPoint; + + return new MeasuredPoint(rgbTimestamp, depthPoint); } /** * Update the oldest line endpoint to the value passed into this function. * This will also flag the line for update on the next render pass. */ - private synchronized void updateLine(float[] worldPoint) { + private synchronized void updateLine(MeasuredPoint newPoint) { if (mPointSwitch) { mPointSwitch = !mPointSwitch; - mLinePoints[0] = worldPoint; + mMeasuredPoints[0] = newPoint; return; } mPointSwitch = !mPointSwitch; - mLinePoints[1] = worldPoint; - } - - /** - * Return the endpoints of the line as a Stack of Vector3 objects. Returns - * null if the line is not visible. - */ - private synchronized Stack generateEndpoints() { - - // Place the line based on the two points. - if (mLinePoints[0] != null && mLinePoints[1] != null) { - Stack points = new Stack(); - points.push(new Vector3(mLinePoints[0][0], mLinePoints[0][1], mLinePoints[0][2])); - points.push(new Vector3(mLinePoints[1][0], mLinePoints[1][1], mLinePoints[1][2])); - return points; - } - return null; + mMeasuredPoints[1] = newPoint; } /* * Remove all the points from the Scene. */ private synchronized void clearLine() { - mLinePoints[0] = null; - mLinePoints[1] = null; + mMeasuredPoints[0] = null; + mMeasuredPoints[1] = null; mPointSwitch = true; mRenderer.setLine(null); } - /** - * Create a String containing a human-readable description of the - * distance between endpoints. - */ - private synchronized String getPointSeparation() { - if (mLinePoints[0] == null || mLinePoints[1] == null) { - return "Null"; - } - float[] p1 = mLinePoints[0]; - float[] p2 = mLinePoints[1]; - double separation = Math.sqrt( - Math.pow(p1[0] - p2[0], 2) + - Math.pow(p1[1] - p2[1], 2) + - Math.pow(p1[2] - p2[2], 2)); - return String.format("%.2f", separation) + " meters"; - } - // Debug text UI update loop, updating at 10Hz. private Runnable mUpdateUiLoopRunnable = new Runnable() { public void run() { - updateUi(); + try { + mDistanceTextview.setText(String.format("%.2f", mMeasuredDistance) + " meters"); + } catch (Exception e) { + e.printStackTrace(); + } mHandler.postDelayed(this, UPDATE_UI_INTERVAL_MS); } }; - private synchronized void updateUi() { - try { - mDistanceMeasure.setText(getPointSeparation()); - } catch (Exception e) { - e.printStackTrace(); - } - } - /** * Set the color camera background texture rotation and save the display rotation. */ diff --git a/version.gradle b/version.gradle index 7cd163b8..8fdd72f3 100644 --- a/version.gradle +++ b/version.gradle @@ -1,5 +1,5 @@ allprojects { ext { - release_version = '1.53' + release_version = '1.54' } }