Update pose-detection demo to support orientation change (#629)

* Update pose-detection demo to support orientation change

* update

* fix

* fix

Author: jinjingforever

cloudbuild.yml

@@ -1,19 +1,19 @@
 steps:
-- name: 'node:10'
-  entrypoint: 'yarn'
-  id: 'yarn'
-  args: ['install']
-- name: 'node:10'
-  entrypoint: 'yarn'
-  id: 'test'
-  args: ['presubmit']
-  waitFor: ['yarn']
-  env: ['BROWSERSTACK_USERNAME=deeplearnjs1']
-  secretEnv: ['BROWSERSTACK_KEY']
+  - name: 'node:12'
+    entrypoint: 'yarn'
+    id: 'yarn'
+    args: ['install']
+  - name: 'node:12'
+    entrypoint: 'yarn'
+    id: 'test'
+    args: ['presubmit']
+    waitFor: ['yarn']
+    env: ['BROWSERSTACK_USERNAME=deeplearnjs1']
+    secretEnv: ['BROWSERSTACK_KEY']
 secrets:
-- kmsKeyName: projects/learnjs-174218/locations/global/keyRings/tfjs/cryptoKeys/enc
-  secretEnv:
-    BROWSERSTACK_KEY: CiQAkwyoIW0LcnxymzotLwaH4udVTQFBEN4AEA5CA+a3+yflL2ASPQAD8BdZnGARf78MhH5T9rQqyz9HNODwVjVIj64CTkFlUCGrP1B2HX9LXHWHLmtKutEGTeFFX9XhuBzNExA=
+  - kmsKeyName: projects/learnjs-174218/locations/global/keyRings/tfjs/cryptoKeys/enc
+    secretEnv:
+      BROWSERSTACK_KEY: CiQAkwyoIW0LcnxymzotLwaH4udVTQFBEN4AEA5CA+a3+yflL2ASPQAD8BdZnGARf78MhH5T9rQqyz9HNODwVjVIj64CTkFlUCGrP1B2HX9LXHWHLmtKutEGTeFFX9XhuBzNExA=
 timeout: 1800s
 logsBucket: 'gs://tfjs-build-logs'
 options:

presubmit.ts

@@ -38,7 +38,7 @@ dirs.forEach(dir => {
     return;
   }
 
-  const packageJSON: {} =
+  const packageJSON: {'scripts': {[key: string]: string}} =
       JSON.parse(fs.readFileSync('./package.json', {encoding: 'utf-8'}));
   if (packageJSON['scripts']['test'] != null) {
     console.log(`~~~~~~~~~~~~ Testing ${dir} ~~~~~~~~~~~~`);

react-native/pose-detection/App.tsx

@@ -5,13 +5,17 @@ import { Camera } from 'expo-camera';
 
 import * as tf from '@tensorflow/tfjs';
 import * as posedetection from '@tensorflow-models/pose-detection';
+import * as ScreenOrientation from 'expo-screen-orientation';
 import { cameraWithTensors } from '@tensorflow/tfjs-react-native';
 import Svg, { Circle } from 'react-native-svg';
 import { ExpoWebGLRenderingContext } from 'expo-gl';
 
 // tslint:disable-next-line: variable-name
 const TensorCamera = cameraWithTensors(Camera);
 
+const IS_ANDROID = Platform.OS === 'android';
+const IS_IOS = Platform.OS === 'ios';
+
 // Camera preview size.
 //
 // From experiments, to render camera feed without distortion, 16:9 ratio
@@ -20,18 +24,18 @@ const TensorCamera = cameraWithTensors(Camera);
 //
 // This might not cover all cases.
 const CAM_PREVIEW_WIDTH = Dimensions.get('window').width;
-const CAM_PREVIEW_HEIGHT =
-  CAM_PREVIEW_WIDTH / (Platform.OS === 'ios' ? 9 / 16 : 3 / 4);
+const CAM_PREVIEW_HEIGHT = CAM_PREVIEW_WIDTH / (IS_IOS ? 9 / 16 : 3 / 4);
 
 // The score threshold for pose detection results.
 const MIN_KEYPOINT_SCORE = 0.3;
 
 // The size of the resized output from TensorCamera.
 //
 // For movenet, the size here doesn't matter too much because the model will
-// preprocess the input (crop, resize, etc).
-const OUTPUT_TENSOR_WIDTH = 240;
-const OUTPUT_TENSOR_HEIGHT = 320;
+// preprocess the input (crop, resize, etc). For best result, use the size that
+// doesn't distort the image.
+const OUTPUT_TENSOR_WIDTH = 180;
+const OUTPUT_TENSOR_HEIGHT = OUTPUT_TENSOR_WIDTH / (IS_IOS ? 9 / 16 : 3 / 4);
 
 // Whether to auto-render TensorCamera preview.
 const AUTO_RENDER = false;
@@ -42,9 +46,20 @@ export default function App() {
   const [model, setModel] = useState<posedetection.PoseDetector>();
   const [poses, setPoses] = useState<posedetection.Pose[]>();
   const [fps, setFps] = useState(0);
+  const [orientation, setOrientation] =
+    useState<ScreenOrientation.Orientation>();
 
   useEffect(() => {
     async function prepare() {
+      // Set initial orientation.
+      const curOrientation = await ScreenOrientation.getOrientationAsync();
+      setOrientation(curOrientation);
+
+      // Listens to orientation change.
+      ScreenOrientation.addOrientationChangeListener((event) => {
+        setOrientation(event.orientationInfo.orientation);
+      });
+
       // Camera permission.
       await Camera.requestPermissionsAsync();
 
@@ -107,13 +122,19 @@ export default function App() {
         .filter((k) => (k.score ?? 0) > MIN_KEYPOINT_SCORE)
         .map((k) => {
           // Flip horizontally on android.
-          const x = Platform.OS === 'android' ? OUTPUT_TENSOR_WIDTH - k.x : k.x;
+          const x = IS_ANDROID ? OUTPUT_TENSOR_WIDTH - k.x : k.x;
           const y = k.y;
+          const cx =
+            (x / getOutputTensorWidth()) *
+            (isPortrait() ? CAM_PREVIEW_WIDTH : CAM_PREVIEW_HEIGHT);
+          const cy =
+            (y / getOutputTensorHeight()) *
+            (isPortrait() ? CAM_PREVIEW_HEIGHT : CAM_PREVIEW_WIDTH);
           return (
             <Circle
               key={`skeletonkp_${k.name}`}
-              cx={(x / OUTPUT_TENSOR_WIDTH) * CAM_PREVIEW_WIDTH}
-              cy={(y / OUTPUT_TENSOR_HEIGHT) * CAM_PREVIEW_HEIGHT}
+              cx={cx}
+              cy={cy}
               r='4'
               strokeWidth='2'
               fill='#00AA00'
@@ -136,6 +157,53 @@ export default function App() {
     );
   };
 
+  const isPortrait = () => {
+    return (
+      orientation === ScreenOrientation.Orientation.PORTRAIT_UP ||
+      orientation === ScreenOrientation.Orientation.PORTRAIT_DOWN
+    );
+  };
+
+  const getOutputTensorWidth = () => {
+    // On iOS landscape mode, switch width and height of the output tensor to
+    // get better result. Without this, the image stored in the output tensor
+    // would be stretched too much.
+    //
+    // Same for getOutputTensorHeight below.
+    return isPortrait() || IS_ANDROID
+      ? OUTPUT_TENSOR_WIDTH
+      : OUTPUT_TENSOR_HEIGHT;
+  };
+
+  const getOutputTensorHeight = () => {
+    return isPortrait() || IS_ANDROID
+      ? OUTPUT_TENSOR_HEIGHT
+      : OUTPUT_TENSOR_WIDTH;
+  };
+
+  const getTextureRotationAngleInDegrees = () => {
+    // On Android, the camera texture will rotate behind the scene as the phone
+    // changes orientation, so we don't need to rotate it in TensorCamera.
+    if (IS_ANDROID) {
+      return 0;
+    }
+
+    // For iOS, the camera texture won't rotate automatically. Calculate the
+    // rotation angles here which will be passed to TensorCamera to rotate it
+    // internally.
+    switch (orientation) {
+      // Not supported on iOS as of 11/2021, but add it here just in case.
+      case ScreenOrientation.Orientation.PORTRAIT_DOWN:
+        return 180;
+      case ScreenOrientation.Orientation.LANDSCAPE_LEFT:
+        return 270;
+      case ScreenOrientation.Orientation.LANDSCAPE_RIGHT:
+        return 90;
+      default:
+        return 0;
+    }
+  };
+
   if (!tfReady) {
     return (
       <View style={styles.loadingMsg}>
@@ -146,16 +214,21 @@ export default function App() {
     return (
       // Note that you don't need to specify `cameraTextureWidth` and
       // `cameraTextureHeight` prop in `TensorCamera` below.
-      <View style={styles.container}>
+      <View
+        style={
+          isPortrait() ? styles.containerPortrait : styles.containerLandscape
+        }
+      >
         <TensorCamera
           ref={cameraRef}
           style={styles.camera}
           autorender={AUTO_RENDER}
           type={Camera.Constants.Type.front}
           // tensor related props
-          resizeWidth={OUTPUT_TENSOR_WIDTH}
-          resizeHeight={OUTPUT_TENSOR_HEIGHT}
+          resizeWidth={getOutputTensorWidth()}
+          resizeHeight={getOutputTensorHeight()}
           resizeDepth={3}
+          rotation={getTextureRotationAngleInDegrees()}
           onReady={handleCameraStream}
         />
         {renderPose()}
@@ -166,8 +239,17 @@ export default function App() {
 }
 
 const styles = StyleSheet.create({
-  container: {
+  containerPortrait: {
+    position: 'relative',
+    width: CAM_PREVIEW_WIDTH,
+    height: CAM_PREVIEW_HEIGHT,
+    marginTop: Dimensions.get('window').height / 2 - CAM_PREVIEW_HEIGHT / 2,
+  },
+  containerLandscape: {
     position: 'relative',
+    width: CAM_PREVIEW_HEIGHT,
+    height: CAM_PREVIEW_WIDTH,
+    marginLeft: Dimensions.get('window').height / 2 - CAM_PREVIEW_HEIGHT / 2,
   },
   loadingMsg: {
     position: 'absolute',
@@ -177,24 +259,19 @@ const styles = StyleSheet.create({
     justifyContent: 'center',
   },
   camera: {
-    position: 'absolute',
-    top: Dimensions.get('window').height / 2 - CAM_PREVIEW_HEIGHT / 2,
-    left: Dimensions.get('window').width / 2 - CAM_PREVIEW_WIDTH / 2,
-    width: CAM_PREVIEW_WIDTH,
-    height: CAM_PREVIEW_HEIGHT,
+    width: '100%',
+    height: '100%',
     zIndex: 1,
   },
   svg: {
-    top: Dimensions.get('window').height / 2 - CAM_PREVIEW_HEIGHT / 2,
-    left: Dimensions.get('window').width / 2 - CAM_PREVIEW_WIDTH / 2,
-    width: CAM_PREVIEW_WIDTH,
-    height: CAM_PREVIEW_HEIGHT,
+    width: '100%',
+    height: '100%',
     position: 'absolute',
     zIndex: 30,
   },
   fpsContainer: {
     position: 'absolute',
-    top: 80,
+    top: 10,
     left: 10,
     width: 80,
     alignItems: 'center',

react-native/pose-detection/README.md

@@ -2,8 +2,8 @@
 
 A quick demo for running [TFJS Pose Detection][posedetection] model
 ([MoveNet.SinglePose.Ligntning][tfhub]) using
-[TFJS React Native][tfjs-react-native] in an Expo project. Only the keypoints
-are rendered in the demo.
+[TFJS React Native][tfjs-react-native] in an Expo project. It supports both
+portrait and landscape mode. Only the keypoints are rendered in the demo.
 
 To run it locally:
 
@@ -15,7 +15,9 @@ $ yarn start
 Then scan the QR code to open it in the `Expo Go` app.
 
 Tested on Pixel 2, iPhone 12 Pro Max, iPad, and iPad Pro.
-<img src="screenshot.jpg" width="300">
+
+<img src="screenshot_portrait.jpg" width="250">
+<img src="screenshot_landscape.jpg" width="500">
 
 [posedetection]: https://github.com/tensorflow/tfjs-models/tree/master/pose-detection
 [tfhub]: https://tfhub.dev/google/tfjs-model/movenet/singlepose/lightning/4

react-native/pose-detection/ios/Podfile.lock

@@ -3,13 +3,9 @@ PODS:
   - DoubleConversion (1.1.6)
   - EXApplication (3.1.2):
     - UMCore
-  - EXCamera (7.0.0):
-    - UMBarCodeScannerInterface
+  - EXCamera (11.2.2):
+    - ExpoModulesCore
     - UMCore
-    - UMFaceDetectorInterface
-    - UMFileSystemInterface
-    - UMImageLoaderInterface
-    - UMPermissionsInterface
   - EXConstants (10.1.3):
     - UMConstantsInterface
     - UMCore
@@ -42,6 +38,18 @@ PODS:
   - EXPermissions (12.0.1):
     - UMCore
     - UMPermissionsInterface
+  - ExpoModulesCore (0.2.0):
+    - ExpoModulesCore/Core (= 0.2.0)
+    - ExpoModulesCore/Interfaces (= 0.2.0)
+    - UMCore
+  - ExpoModulesCore/Core (0.2.0):
+    - UMCore
+  - ExpoModulesCore/Interfaces (0.2.0):
+    - ExpoModulesCore/Core
+    - UMCore
+  - EXScreenOrientation (3.1.0):
+    - React-Core
+    - UMCore
   - EXSplashScreen (0.10.3):
     - React-Core
     - UMCore
@@ -372,6 +380,8 @@ DEPENDENCIES:
   - EXImageManipulator (from `../node_modules/expo-image-manipulator/ios`)
   - EXKeepAwake (from `../node_modules/expo-keep-awake/ios`)
   - EXPermissions (from `../node_modules/expo-permissions/ios`)
+  - ExpoModulesCore (from `../node_modules/expo-modules-core/ios`)
+  - EXScreenOrientation (from `../node_modules/expo-screen-orientation/ios`)
   - EXSplashScreen (from `../node_modules/expo-splash-screen/ios`)
   - EXStructuredHeaders (from `../node_modules/expo-structured-headers/ios`)
   - EXUpdates (from `../node_modules/expo-updates/ios`)
@@ -453,6 +463,10 @@ EXTERNAL SOURCES:
     :path: "../node_modules/expo-keep-awake/ios"
   EXPermissions:
     :path: "../node_modules/expo-permissions/ios"
+  ExpoModulesCore:
+    :path: "../node_modules/expo-modules-core/ios"
+  EXScreenOrientation:
+    :path: "../node_modules/expo-screen-orientation/ios"
   EXSplashScreen:
     :path: "../node_modules/expo-splash-screen/ios"
   EXStructuredHeaders:
@@ -552,7 +566,7 @@ SPEC CHECKSUMS:
   boost-for-react-native: 39c7adb57c4e60d6c5479dd8623128eb5b3f0f2c
   DoubleConversion: cde416483dac037923206447da6e1454df403714
   EXApplication: 4797b8b37f0b0470f587fdccf6407f44b50d18b5
-  EXCamera: 8ad7012d4e0fef09ff35de14fcb41b410357e9be
+  EXCamera: 042b334f6ee9b918855fe4c4f155d4c372300a5f
   EXConstants: c4dd28acc12039c999612507a5f935556f2c86ce
   EXErrorRecovery: 720641265b8cf95e6cdeb1884ac38e794a352488
   EXFileSystem: dcf2273f49431e5037347c733a2dc5d08e0d0a9e
@@ -563,6 +577,8 @@ SPEC CHECKSUMS:
   EXImageManipulator: b2e17bdbd0872cc59b43b76756f76fdb8a849831
   EXKeepAwake: d4e4a3ed8c1c4fd940dd62fc5a8be2a190371fd4
   EXPermissions: 8f8c1c05580c4e02d4ee2c8dd74bfe173ff6a723
+  ExpoModulesCore: 2734852616127a6c1fc23012197890a6f3763dc7
+  EXScreenOrientation: 4d9ec57c73055c8a4dbf9dd85def4db29b690a91
   EXSplashScreen: 6d5b4274537d2b9f343a9ec2347dbb61f3d40ab7
   EXStructuredHeaders: be834496a4d9fd0069cd12ec1cc57b31c6d3b256
   EXUpdates: efe0e8c514dcff06a8fd0b63be6019a6365fb9c7

react-native/pose-detection/ios/posenet/AppDelegate.m

@@ -18,6 +18,7 @@
 #import <FlipperKitNetworkPlugin/FlipperKitNetworkPlugin.h>
 #import <SKIOSNetworkPlugin/SKIOSNetworkAdapter.h>
 #import <FlipperKitReactPlugin/FlipperKitReactPlugin.h>
+#import <EXScreenOrientation/EXScreenOrientationViewController.h>
 
 static void InitializeFlipper(UIApplication *application) {
   FlipperClient *client = [FlipperClient sharedClient];
@@ -67,7 +68,7 @@ - (RCTBridge *)initializeReactNativeApp
   RCTRootView *rootView = [[RCTRootView alloc] initWithBridge:bridge moduleName:@"main" initialProperties:nil];
   rootView.backgroundColor = [[UIColor alloc] initWithRed:1.0f green:1.0f blue:1.0f alpha:1];
 
-  UIViewController *rootViewController = [UIViewController new];
+  UIViewController *rootViewController = [[EXScreenOrientationViewController alloc] init]; // The default screen orientation will be set to `portrait`.
   rootViewController.view = rootView;
   self.window.rootViewController = rootViewController;
   [self.window makeKeyAndVisible];

react-native/pose-detection/package.json

@@ -4,20 +4,23 @@
   "version": "0.0.1",
   "license": "Apache-2.0",
   "scripts": {
+    "build-dep": "cd ../../../tfjs/tfjs-react-native && yarn build-npm",
+    "copy-dep": "rm -rf node_modules/@tensorflow/tfjs-react-native/dist/* && cp -rf ../../../tfjs/tfjs-react-native/dist/* node_modules/@tensorflow/tfjs-react-native/dist/",
     "start": "expo start"
   },
   "dependencies": {
     "@mediapipe/pose": "~0.4.0",
     "@react-native-async-storage/async-storage": "1.15.7",
     "@tensorflow-models/pose-detection": "0.0.6",
     "@tensorflow/tfjs": "3.9.0",
-    "@tensorflow/tfjs-react-native": "0.7.0",
+    "@tensorflow/tfjs-react-native": "0.8.0",
     "expo": "~41.0.1",
     "expo-camera": "^11.2.2",
     "expo-file-system": "^10.0.0",
     "expo-gl": "^7.0.0",
     "expo-gl-cpp": "^7.0.0",
     "expo-image-manipulator": "^6.0.0",
+    "expo-screen-orientation": "~3.1.0",
     "expo-splash-screen": "~0.10.2",
     "expo-updates": "~0.5.4",
     "react": "16.13.1",