diff --git a/tfjs-react-native/src/camera/camera.ts b/tfjs-react-native/src/camera/camera.ts index 23164bb2582..47801c7d956 100644 --- a/tfjs-react-native/src/camera/camera.ts +++ b/tfjs-react-native/src/camera/camera.ts @@ -18,6 +18,7 @@ import * as tf from '@tensorflow/tfjs-core'; import {downloadTextureData, drawTexture, runResizeProgram, uploadTextureData} from './camera_webgl_util'; +import {Rotation} from './types'; interface Dimensions { width: number; height: number; @@ -32,6 +33,7 @@ interface Size { interface FromTextureOptions { alignCorners?: boolean; interpolation?: 'nearest_neighbor'|'bilinear'; + rotation?: Rotation; } const glCapabilities = { @@ -184,15 +186,21 @@ export function fromTexture( options.alignCorners != null ? options.alignCorners : false; const interpolation = options.interpolation != null ? options.interpolation : 'bilinear'; + const rotation = options.rotation != null ? options.rotation : 0; tf.util.assert( interpolation === 'bilinear' || interpolation === 'nearest_neighbor', () => 'fromTexture Error: interpolation must be one of' + ' "bilinear" or "nearest_neighbor"'); + tf.util.assert( + [0, 90, 180, 270, 360, -90, -180, -270].includes(rotation), + () => 'fromTexture Error: rotation must be ' + + '0, +/- 90, +/- 180, +/- 270 or 360'); + const resizedTexture = runResizeProgram( gl, texture, sourceDims, targetShape, alignCorners, - useCustomShadersToResize, interpolation); + useCustomShadersToResize, interpolation, rotation); const downloadedTextureData = downloadTextureData(gl, resizedTexture, targetShape); @@ -231,10 +239,15 @@ export function fromTexture( */ export function renderToGLView( gl: WebGL2RenderingContext, texture: WebGLTexture, size: Size, - flipHorizontal = true) { + flipHorizontal = true, rotation: Rotation = 0) { + tf.util.assert( + [0, 90, 180, 270, 360, -90, -180, -270].includes(rotation), + () => 'renderToGLView Error: rotation must be ' + + '0, +/- 90, +/- 180, +/- 270 or 360'); + size = { width: Math.floor(size.width), height: Math.floor(size.height), }; - drawTexture(gl, texture, size, flipHorizontal); + drawTexture(gl, texture, size, flipHorizontal, rotation); } diff --git a/tfjs-react-native/src/camera/camera_stream.tsx b/tfjs-react-native/src/camera/camera_stream.tsx index ea02dd55cc4..6e616989642 100644 --- a/tfjs-react-native/src/camera/camera_stream.tsx +++ b/tfjs-react-native/src/camera/camera_stream.tsx @@ -21,11 +21,12 @@ import { StyleSheet, PixelRatio, LayoutChangeEvent, - Platform + Platform, } from 'react-native'; import { Camera } from 'expo-camera'; import { GLView, ExpoWebGLRenderingContext } from 'expo-gl'; import { fromTexture, renderToGLView, detectGLCapabilities } from './camera'; +import { Rotation } from './types'; interface WrappedComponentProps { onLayout?: (event: LayoutChangeEvent) => void; @@ -41,11 +42,12 @@ interface Props { resizeHeight: number; resizeDepth: number; autorender: boolean; + rotation?: Rotation; onReady: ( images: IterableIterator, updateCameraPreview: () => void, gl: ExpoWebGLRenderingContext, - cameraTexture: WebGLTexture, + cameraTexture: WebGLTexture ) => void; } @@ -99,6 +101,9 @@ const DEFAULT_USE_CUSTOM_SHADERS_TO_RESIZE = false; * - __autorender__: boolean — if true the view will be automatically updated * with the contents of the camera. Set this to false if you want more direct * control on when rendering happens. + * - __rotation__: number — the degrees that the internal camera texture and + * preview will be rotated. Accepted values: 0, +/- 90, +/- 180, +/- 270 or + * 360. * - __onReady__: ( * images: IterableIterator, * updateCameraPreview: () => void, @@ -167,10 +172,12 @@ const DEFAULT_USE_CUSTOM_SHADERS_TO_RESIZE = false; /** @doc {heading: 'Media', subheading: 'Camera'} */ export function cameraWithTensors( // tslint:disable-next-line: variable-name - CameraComponent: React.ComponentType, + CameraComponent: React.ComponentType ) { - return class CameraWithTensorStream - extends React.Component { + return class CameraWithTensorStream extends React.Component< + T & Props, + State + > { camera: Camera; glView: GLView; glContext: ExpoWebGLRenderingContext; @@ -188,7 +195,7 @@ export function cameraWithTensors( componentWillUnmount() { cancelAnimationFrame(this.rafID); - if(this.glContext) { + if (this.glContext) { GLView.destroyContextAsync(this.glContext); } this.camera = null; @@ -246,11 +253,7 @@ export function cameraWithTensors( renderLoop(); } - const { - resizeHeight, - resizeWidth, - resizeDepth, - } = this.props; + const { resizeDepth } = this.props; // cameraTextureHeight and cameraTextureWidth props can be omitted when // useCustomShadersToResize is set to false. Setting a default value to @@ -281,19 +284,20 @@ export function cameraWithTensors( depth: RGBA_DEPTH, }; - const targetDims = { - height: resizeHeight, - width: resizeWidth, - depth: resizeDepth || DEFAULT_RESIZE_DEPTH, - }; - while (cameraStreamView.glContext != null) { + const targetDims = { + height: cameraStreamView.props.resizeHeight, + width: cameraStreamView.props.resizeWidth, + depth: resizeDepth || DEFAULT_RESIZE_DEPTH, + }; + const imageTensor = fromTexture( gl, cameraTexture, textureDims, targetDims, useCustomShadersToResize, + { rotation: cameraStreamView.props.rotation } ); yield imageTensor; } @@ -316,6 +320,7 @@ export function cameraWithTensors( ) { const renderFunc = () => { const { cameraLayout } = this.state; + const { rotation } = this.props; const width = PixelRatio.getPixelSizeForLayoutSize(cameraLayout.width); const height = PixelRatio.getPixelSizeForLayoutSize( cameraLayout.height @@ -325,7 +330,13 @@ export function cameraWithTensors( const flipHorizontal = Platform.OS === 'ios' && isFrontCamera ? false : true; - renderToGLView(gl, cameraTexture, { width, height }, flipHorizontal); + renderToGLView( + gl, + cameraTexture, + { width, height }, + flipHorizontal, + rotation + ); }; return renderFunc.bind(this); @@ -351,6 +362,7 @@ export function cameraWithTensors( resizeDepth: null, autorender: null, onReady: null, + rotation: 0, }; const tensorCameraPropKeys = Object.keys(tensorCameraPropMap); @@ -364,10 +376,12 @@ export function cameraWithTensors( } // Set up an on layout handler - const onlayout = this.props.onLayout ? (e: LayoutChangeEvent) => { - this.props.onLayout(e); - this.onCameraLayout(e); - } : this.onCameraLayout; + const onlayout = this.props.onLayout + ? (e: LayoutChangeEvent) => { + this.props.onLayout(e); + this.onCameraLayout(e); + } + : this.onCameraLayout; cameraProps.onLayout = onlayout; @@ -375,7 +389,7 @@ export function cameraWithTensors( //@ts-ignore see https://github.com/microsoft/TypeScript/issues/30650 (this.camera = ref)} /> ); @@ -390,10 +404,10 @@ export function cameraWithTensors( top: cameraLayout.y, width: cameraLayout.width, height: cameraLayout.height, - zIndex: this.props.style.zIndex ? - parseInt(this.props.style.zIndex, 10) + 10 : 10, - } - + zIndex: this.props.style.zIndex + ? parseInt(this.props.style.zIndex, 10) + 10 + : 10, + }, }); glViewComponent = ( @@ -401,7 +415,7 @@ export function cameraWithTensors( key='camera-with-tensor-gl-view' style={styles.glView} onContextCreate={this.onGLContextCreate} - ref={ref => (this.glView = ref)} + ref={(ref) => (this.glView = ref)} /> ); } diff --git a/tfjs-react-native/src/camera/camera_webgl_util.ts b/tfjs-react-native/src/camera/camera_webgl_util.ts index 8ef1272a34d..0cf62f8aeab 100644 --- a/tfjs-react-native/src/camera/camera_webgl_util.ts +++ b/tfjs-react-native/src/camera/camera_webgl_util.ts @@ -21,6 +21,7 @@ import * as tf from '@tensorflow/tfjs-core'; import * as drawTextureProgramInfo from './draw_texture_program_info'; import * as resizeBilinearProgramInfo from './resize_bilinear_program_info'; import * as resizeNNProgramInfo from './resize_nearest_neigbor_program_info'; +import {Rotation} from './types'; interface Dimensions { width: number; @@ -149,9 +150,10 @@ export function uploadTextureData( */ export function drawTexture( gl: WebGL2RenderingContext, texture: WebGLTexture, - dims: {width: number, height: number}, flipHorizontal: boolean) { + dims: {width: number, height: number}, flipHorizontal: boolean, + rotation: Rotation) { const {program, vao, vertices, uniformLocations} = - drawTextureProgram(gl, flipHorizontal, false); + drawTextureProgram(gl, flipHorizontal, false, rotation); gl.useProgram(program); gl.bindVertexArray(vao); @@ -177,10 +179,10 @@ export function runResizeProgram( gl: WebGL2RenderingContext, inputTexture: WebGLTexture, inputDims: Dimensions, outputDims: Dimensions, alignCorners: boolean, useCustomShadersToResize: boolean, - interpolation: 'nearest_neighbor'|'bilinear') { + interpolation: 'nearest_neighbor'|'bilinear', rotation: Rotation) { const {program, vao, vertices, uniformLocations} = useCustomShadersToResize ? resizeProgram(gl, inputDims, outputDims, alignCorners, interpolation) : - drawTextureProgram(gl, false, true); + drawTextureProgram(gl, false, true, rotation); gl.useProgram(program); // Set up geometry webgl_util.callAndCheck(gl, () => { @@ -304,17 +306,17 @@ function createFrameBuffer(gl: WebGL2RenderingContext): WebGLFramebuffer { } export function drawTextureProgram( - gl: WebGL2RenderingContext, flipHorizontal: boolean, - flipVertical: boolean): ProgramObjects { + gl: WebGL2RenderingContext, flipHorizontal: boolean, flipVertical: boolean, + rotation: Rotation): ProgramObjects { if (!programCacheByContext.has(gl)) { programCacheByContext.set(gl, new Map()); } const programCache = programCacheByContext.get(gl); - const cacheKey = `drawTexture_${flipHorizontal}_${flipVertical}`; + const cacheKey = `drawTexture_${flipHorizontal}_${flipVertical}_${rotation}`; if (!programCache.has(cacheKey)) { - const vertSource = - drawTextureProgramInfo.vertexShaderSource(flipHorizontal, flipVertical); + const vertSource = drawTextureProgramInfo.vertexShaderSource( + flipHorizontal, flipVertical, rotation); const fragSource = drawTextureProgramInfo.fragmentShaderSource(); const vertices = drawTextureProgramInfo.vertices(); diff --git a/tfjs-react-native/src/camera/draw_texture_program_info.ts b/tfjs-react-native/src/camera/draw_texture_program_info.ts index cc854b26cde..df804fed3e9 100644 --- a/tfjs-react-native/src/camera/draw_texture_program_info.ts +++ b/tfjs-react-native/src/camera/draw_texture_program_info.ts @@ -15,10 +15,13 @@ * ============================================================================= */ +import {Rotation} from './types'; + export function vertexShaderSource( - flipHorizontal: boolean, flipVertical: boolean) { + flipHorizontal: boolean, flipVertical: boolean, rotation: Rotation) { const horizontalScale = flipHorizontal ? -1 : 1; const verticalScale = flipVertical ? -1 : 1; + const rotateAngle = rotation === 0 ? '0.' : rotation * (Math.PI / 180); return `#version 300 es precision highp float; @@ -27,11 +30,22 @@ in vec2 texCoords; out vec2 uv; +vec2 rotate(vec2 uvCoods, vec2 pivot, float rotation) { + float cosa = cos(rotation); + float sina = sin(rotation); + uvCoods -= pivot; + return vec2( + cosa * uvCoods.x - sina * uvCoods.y, + cosa * uvCoods.y + sina * uvCoods.x + ) + pivot; +} + void main() { + uv = rotate(texCoords, vec2(0.5), ${rotateAngle}); + // Invert geometry to match the image orientation from the camera. gl_Position = vec4(position * vec2(${horizontalScale}., ${ verticalScale}. * -1.), 0, 1); - uv = texCoords; }`; } diff --git a/tfjs-react-native/src/camera/types.ts b/tfjs-react-native/src/camera/types.ts new file mode 100644 index 00000000000..77aac909105 --- /dev/null +++ b/tfjs-react-native/src/camera/types.ts @@ -0,0 +1,2 @@ +// Rotation in degrees. +export type Rotation = 0|90|180|270|360|- 80|- 180|- 270;