Skip to content

Commit

Permalink
[react-native] Add "rotation" property to TensorCamera to allow users…
Browse files Browse the repository at this point in the history 
… to rotate camera view and internal texture (#5804)

* [react-native] Add option to TensorCamera to allow users to rotate camera view and internal texture

* update

* address comments
  • Loading branch information
@jinjingforever
jinjingforever authored Nov 3, 2021
1 parent c16a217 commit d89b7ed
Show file tree
Hide file tree
Showing 5 changed files with 87 additions and 42 deletions.
19 changes: 16 additions & 3 deletions tfjs-react-native/src/camera/camera.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import * as tf from '@tensorflow/tfjs-core';

import {downloadTextureData, drawTexture, runResizeProgram, uploadTextureData} from './camera_webgl_util';
import {Rotation} from './types';
interface Dimensions {
width: number;
height: number;
Expand All @@ -32,6 +33,7 @@ interface Size {
interface FromTextureOptions {
alignCorners?: boolean;
interpolation?: 'nearest_neighbor'|'bilinear';
rotation?: Rotation;
}

const glCapabilities = {
Expand Down Expand Up @@ -184,15 +186,21 @@ export function fromTexture(
options.alignCorners != null ? options.alignCorners : false;
const interpolation =
options.interpolation != null ? options.interpolation : 'bilinear';
const rotation = options.rotation != null ? options.rotation : 0;

tf.util.assert(
interpolation === 'bilinear' || interpolation === 'nearest_neighbor',
() => 'fromTexture Error: interpolation must be one of' +
' "bilinear" or "nearest_neighbor"');

tf.util.assert(
[0, 90, 180, 270, 360, -90, -180, -270].includes(rotation),
() => 'fromTexture Error: rotation must be ' +
'0, +/- 90, +/- 180, +/- 270 or 360');

const resizedTexture = runResizeProgram(
gl, texture, sourceDims, targetShape, alignCorners,
useCustomShadersToResize, interpolation);
useCustomShadersToResize, interpolation, rotation);
const downloadedTextureData =
downloadTextureData(gl, resizedTexture, targetShape);

Expand Down Expand Up @@ -231,10 +239,15 @@ export function fromTexture(
*/
export function renderToGLView(
gl: WebGL2RenderingContext, texture: WebGLTexture, size: Size,
flipHorizontal = true) {
flipHorizontal = true, rotation: Rotation = 0) {
tf.util.assert(
[0, 90, 180, 270, 360, -90, -180, -270].includes(rotation),
() => 'renderToGLView Error: rotation must be ' +
'0, +/- 90, +/- 180, +/- 270 or 360');

size = {
width: Math.floor(size.width),
height: Math.floor(size.height),
};
drawTexture(gl, texture, size, flipHorizontal);
drawTexture(gl, texture, size, flipHorizontal, rotation);
}
70 changes: 42 additions & 28 deletions tfjs-react-native/src/camera/camera_stream.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,12 @@ import {
StyleSheet,
PixelRatio,
LayoutChangeEvent,
Platform
Platform,
} from 'react-native';
import { Camera } from 'expo-camera';
import { GLView, ExpoWebGLRenderingContext } from 'expo-gl';
import { fromTexture, renderToGLView, detectGLCapabilities } from './camera';
import { Rotation } from './types';

interface WrappedComponentProps {
onLayout?: (event: LayoutChangeEvent) => void;
Expand All @@ -41,11 +42,12 @@ interface Props {
resizeHeight: number;
resizeDepth: number;
autorender: boolean;
rotation?: Rotation;
onReady: (
images: IterableIterator<tf.Tensor3D>,
updateCameraPreview: () => void,
gl: ExpoWebGLRenderingContext,
cameraTexture: WebGLTexture,
cameraTexture: WebGLTexture
) => void;
}

Expand Down Expand Up @@ -99,6 +101,9 @@ const DEFAULT_USE_CUSTOM_SHADERS_TO_RESIZE = false;
* - __autorender__: boolean — if true the view will be automatically updated
* with the contents of the camera. Set this to false if you want more direct
* control on when rendering happens.
* - __rotation__: number — the degrees that the internal camera texture and
* preview will be rotated. Accepted values: 0, +/- 90, +/- 180, +/- 270 or
* 360.
* - __onReady__: (
* images: IterableIterator<tf.Tensor3D>,
* updateCameraPreview: () => void,
Expand Down Expand Up @@ -167,10 +172,12 @@ const DEFAULT_USE_CUSTOM_SHADERS_TO_RESIZE = false;
/** @doc {heading: 'Media', subheading: 'Camera'} */
export function cameraWithTensors<T extends WrappedComponentProps>(
// tslint:disable-next-line: variable-name
CameraComponent: React.ComponentType<T>,
CameraComponent: React.ComponentType<T>
) {
return class CameraWithTensorStream
extends React.Component<T & Props, State> {
return class CameraWithTensorStream extends React.Component<
T & Props,
State
> {
camera: Camera;
glView: GLView;
glContext: ExpoWebGLRenderingContext;
Expand All @@ -188,7 +195,7 @@ export function cameraWithTensors<T extends WrappedComponentProps>(

componentWillUnmount() {
cancelAnimationFrame(this.rafID);
if(this.glContext) {
if (this.glContext) {
GLView.destroyContextAsync(this.glContext);
}
this.camera = null;
Expand Down Expand Up @@ -246,11 +253,7 @@ export function cameraWithTensors<T extends WrappedComponentProps>(
renderLoop();
}

const {
resizeHeight,
resizeWidth,
resizeDepth,
} = this.props;
const { resizeDepth } = this.props;

// cameraTextureHeight and cameraTextureWidth props can be omitted when
// useCustomShadersToResize is set to false. Setting a default value to
Expand Down Expand Up @@ -281,19 +284,20 @@ export function cameraWithTensors<T extends WrappedComponentProps>(
depth: RGBA_DEPTH,
};

const targetDims = {
height: resizeHeight,
width: resizeWidth,
depth: resizeDepth || DEFAULT_RESIZE_DEPTH,
};

while (cameraStreamView.glContext != null) {
const targetDims = {
height: cameraStreamView.props.resizeHeight,
width: cameraStreamView.props.resizeWidth,
depth: resizeDepth || DEFAULT_RESIZE_DEPTH,
};

const imageTensor = fromTexture(
gl,
cameraTexture,
textureDims,
targetDims,
useCustomShadersToResize,
{ rotation: cameraStreamView.props.rotation }
);
yield imageTensor;
}
Expand All @@ -316,6 +320,7 @@ export function cameraWithTensors<T extends WrappedComponentProps>(
) {
const renderFunc = () => {
const { cameraLayout } = this.state;
const { rotation } = this.props;
const width = PixelRatio.getPixelSizeForLayoutSize(cameraLayout.width);
const height = PixelRatio.getPixelSizeForLayoutSize(
cameraLayout.height
Expand All @@ -325,7 +330,13 @@ export function cameraWithTensors<T extends WrappedComponentProps>(
const flipHorizontal =
Platform.OS === 'ios' && isFrontCamera ? false : true;

renderToGLView(gl, cameraTexture, { width, height }, flipHorizontal);
renderToGLView(
gl,
cameraTexture,
{ width, height },
flipHorizontal,
rotation
);
};

return renderFunc.bind(this);
Expand All @@ -351,6 +362,7 @@ export function cameraWithTensors<T extends WrappedComponentProps>(
resizeDepth: null,
autorender: null,
onReady: null,
rotation: 0,
};
const tensorCameraPropKeys = Object.keys(tensorCameraPropMap);

Expand All @@ -364,18 +376,20 @@ export function cameraWithTensors<T extends WrappedComponentProps>(
}

// Set up an on layout handler
const onlayout = this.props.onLayout ? (e: LayoutChangeEvent) => {
this.props.onLayout(e);
this.onCameraLayout(e);
} : this.onCameraLayout;
const onlayout = this.props.onLayout
? (e: LayoutChangeEvent) => {
this.props.onLayout(e);
this.onCameraLayout(e);
}
: this.onCameraLayout;

cameraProps.onLayout = onlayout;

const cameraComp = (
//@ts-ignore see https://github.com/microsoft/TypeScript/issues/30650
<CameraComponent
key='camera-with-tensor-camera-view'
{...(cameraProps)}
{...cameraProps}
ref={(ref: Camera) => (this.camera = ref)}
/>
);
Expand All @@ -390,18 +404,18 @@ export function cameraWithTensors<T extends WrappedComponentProps>(
top: cameraLayout.y,
width: cameraLayout.width,
height: cameraLayout.height,
zIndex: this.props.style.zIndex ?
parseInt(this.props.style.zIndex, 10) + 10 : 10,
}

zIndex: this.props.style.zIndex
? parseInt(this.props.style.zIndex, 10) + 10
: 10,
},
});

glViewComponent = (
<GLView
key='camera-with-tensor-gl-view'
style={styles.glView}
onContextCreate={this.onGLContextCreate}
ref={ref => (this.glView = ref)}
ref={(ref) => (this.glView = ref)}
/>
);
}
Expand Down
20 changes: 11 additions & 9 deletions tfjs-react-native/src/camera/camera_webgl_util.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import * as tf from '@tensorflow/tfjs-core';
import * as drawTextureProgramInfo from './draw_texture_program_info';
import * as resizeBilinearProgramInfo from './resize_bilinear_program_info';
import * as resizeNNProgramInfo from './resize_nearest_neigbor_program_info';
import {Rotation} from './types';

interface Dimensions {
width: number;
Expand Down Expand Up @@ -149,9 +150,10 @@ export function uploadTextureData(
*/
export function drawTexture(
gl: WebGL2RenderingContext, texture: WebGLTexture,
dims: {width: number, height: number}, flipHorizontal: boolean) {
dims: {width: number, height: number}, flipHorizontal: boolean,
rotation: Rotation) {
const {program, vao, vertices, uniformLocations} =
drawTextureProgram(gl, flipHorizontal, false);
drawTextureProgram(gl, flipHorizontal, false, rotation);
gl.useProgram(program);
gl.bindVertexArray(vao);

Expand All @@ -177,10 +179,10 @@ export function runResizeProgram(
gl: WebGL2RenderingContext, inputTexture: WebGLTexture,
inputDims: Dimensions, outputDims: Dimensions, alignCorners: boolean,
useCustomShadersToResize: boolean,
interpolation: 'nearest_neighbor'|'bilinear') {
interpolation: 'nearest_neighbor'|'bilinear', rotation: Rotation) {
const {program, vao, vertices, uniformLocations} = useCustomShadersToResize ?
resizeProgram(gl, inputDims, outputDims, alignCorners, interpolation) :
drawTextureProgram(gl, false, true);
drawTextureProgram(gl, false, true, rotation);
gl.useProgram(program);
// Set up geometry
webgl_util.callAndCheck(gl, () => {
Expand Down Expand Up @@ -304,17 +306,17 @@ function createFrameBuffer(gl: WebGL2RenderingContext): WebGLFramebuffer {
}

export function drawTextureProgram(
gl: WebGL2RenderingContext, flipHorizontal: boolean,
flipVertical: boolean): ProgramObjects {
gl: WebGL2RenderingContext, flipHorizontal: boolean, flipVertical: boolean,
rotation: Rotation): ProgramObjects {
if (!programCacheByContext.has(gl)) {
programCacheByContext.set(gl, new Map());
}
const programCache = programCacheByContext.get(gl);

const cacheKey = `drawTexture_${flipHorizontal}_${flipVertical}`;
const cacheKey = `drawTexture_${flipHorizontal}_${flipVertical}_${rotation}`;
if (!programCache.has(cacheKey)) {
const vertSource =
drawTextureProgramInfo.vertexShaderSource(flipHorizontal, flipVertical);
const vertSource = drawTextureProgramInfo.vertexShaderSource(
flipHorizontal, flipVertical, rotation);
const fragSource = drawTextureProgramInfo.fragmentShaderSource();

const vertices = drawTextureProgramInfo.vertices();
Expand Down
18 changes: 16 additions & 2 deletions tfjs-react-native/src/camera/draw_texture_program_info.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,13 @@
* =============================================================================
*/

import {Rotation} from './types';

export function vertexShaderSource(
flipHorizontal: boolean, flipVertical: boolean) {
flipHorizontal: boolean, flipVertical: boolean, rotation: Rotation) {
const horizontalScale = flipHorizontal ? -1 : 1;
const verticalScale = flipVertical ? -1 : 1;
const rotateAngle = rotation === 0 ? '0.' : rotation * (Math.PI / 180);
return `#version 300 es
precision highp float;
Expand All @@ -27,11 +30,22 @@ in vec2 texCoords;
out vec2 uv;
vec2 rotate(vec2 uvCoods, vec2 pivot, float rotation) {
float cosa = cos(rotation);
float sina = sin(rotation);
uvCoods -= pivot;
return vec2(
cosa * uvCoods.x - sina * uvCoods.y,
cosa * uvCoods.y + sina * uvCoods.x
) + pivot;
}
void main() {
uv = rotate(texCoords, vec2(0.5), ${rotateAngle});
// Invert geometry to match the image orientation from the camera.
gl_Position = vec4(position * vec2(${horizontalScale}., ${
verticalScale}. * -1.), 0, 1);
uv = texCoords;
}`;
}

Expand Down
2 changes: 2 additions & 0 deletions tfjs-react-native/src/camera/types.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
// Rotation in degrees.
export type Rotation = 0|90|180|270|360|- 80|- 180|- 270;

0 comments on commit d89b7ed

Please sign in to comment.