Skip to content

Commit c317640

Browse files
ahmedsabieahmedsabie
authored
Add PortraitDepth API and infrastructure (#1005)
* Add PortraitDepth API and infrastructure * Fix minDepth and maxDepth description * Rename PortraitDepth to ARPortraitDepth * Fix readme
1 parent 8045b51 commit c317640

15 files changed

+702
-5
lines changed

‎depth-estimation/.npmignore

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
.DS_Store
2+
.yalc/
3+
.vscode/
4+
.rpt2_cache/
5+
demos/
6+
scripts/
7+
src/
8+
test_data/
9+
coverage/
10+
node_modules/
11+
karma.conf.js
12+
*.tgz
13+
.travis.yml
14+
.npmignore
15+
tslint.json
16+
yarn.lock
17+
yalc.lock
18+
cloudbuild.yml
19+
dist/*_test.js
20+
dist/*_test.js.map
21+
dist/test_util*

‎depth-estimation/README.md

+79
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
# Depth Estimation
2+
3+
This package provides models for running depth estimation in TensorFlow.js.
4+
5+
Currently, we provide 1 model option:
6+
7+
## AR Portrait Depth API
8+
9+
This AR portrait depth model estimates per-pixel depth (the distance to the camera center) for a single portrait image, which can be further used for creative applications.
10+
(See [DepthLab](https://augmentedperception.github.io/depthlab/) for potential
11+
applications).
12+
13+
For example, the following demo transforms a single 2D RGB image into a 3D Portrait:
14+
[3D Photo Demo](https://storage.googleapis.com/tfjs-models/demos/3dphoto/index.html)
15+
16+
-------------------------------------------------------------------------------
17+
18+
## Table of Contents
19+
20+
- [How to Run It](#how-to-run-it)
21+
- [Example Code and Demos](#example-code-and-demos)
22+
23+
-------------------------------------------------------------------------------
24+
25+
## How to Run It
26+
27+
There are two steps to run the AR portrait depth API:
28+
29+
First, you create an estimator by choosing one of the models from
30+
`SupportedModels`.
31+
32+
For example:
33+
34+
```javascript
35+
const model = depthEstimation.SupportedModels.ARPortraitDepth;
36+
const estimatorConfig = {
37+
minDepth: 0,
38+
maxDepth: 1,
39+
}
40+
const estimator = await depthEstimation.createEstimator(model, estimatorConfig);
41+
```
42+
43+
Next, you can use the estimator to estimate depth.
44+
45+
```javascript
46+
const depthMap = await estimator.estimateDepth(image);
47+
```
48+
49+
The returned depth map contains depth values for each pixel in the image.
50+
51+
Example output:
52+
53+
```javascript
54+
{
55+
toCanvasImageSource(): ...
56+
toArray(): ...
57+
toTensor(): ...
58+
getUnderlyingType(): ...
59+
}
60+
```
61+
62+
The output provides access to the underlying depth values using the conversion
63+
functions toCanvasImageSource, toArray, and toTensor depending on the desired
64+
output type. Note that getUnderlyingType can be queried to determine what is the
65+
type being used underneath the hood to avoid expensive conversions (such as from
66+
tensor to image data).
67+
68+
Refer to each model's documentation for specific configurations for the model
69+
and their performance.
70+
71+
[ARPortraitDepth Documentation](https://github.com/tensorflow/tfjs-models/tree/master/depth-estimation/src/ar_portrait_depth)
72+
73+
-------------------------------------------------------------------------------
74+
75+
## Example Code and Demos
76+
77+
You may reference the demos for code examples.
78+
Details for how to run the demos are included in the `demos/`
79+
[folder](https://github.com/tensorflow/tfjs-models/tree/master/depth-estimation/demos).

‎depth-estimation/cloudbuild.yml

+54
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
steps:
2+
3+
# Install common dependencies.
4+
- name: 'node:12'
5+
id: 'yarn-common'
6+
entrypoint: 'yarn'
7+
args: ['install']
8+
9+
# Install depth-estimation dependencies.
10+
- name: 'node:12'
11+
dir: 'depth-estimation'
12+
entrypoint: 'yarn'
13+
id: 'yarn'
14+
args: ['install']
15+
waitFor: ['yarn-common']
16+
17+
# Lint.
18+
- name: 'node:12'
19+
dir: 'depth-estimation'
20+
entrypoint: 'yarn'
21+
id: 'lint'
22+
args: ['lint']
23+
waitFor: ['yarn']
24+
25+
# Build.
26+
- name: 'node:12'
27+
dir: 'depth-estimation'
28+
entrypoint: 'yarn'
29+
id: 'build'
30+
args: ['build']
31+
waitFor: ['yarn']
32+
33+
# Run browser tests.
34+
- name: 'node:12'
35+
dir: 'depth-estimation'
36+
entrypoint: 'yarn'
37+
id: 'test'
38+
args: ['test-ci']
39+
env: ['BROWSERSTACK_USERNAME=deeplearnjs1']
40+
secretEnv: ['BROWSERSTACK_KEY']
41+
waitFor: ['build']
42+
43+
# General configuration
44+
secrets:
45+
- kmsKeyName: projects/learnjs-174218/locations/global/keyRings/tfjs/cryptoKeys/enc
46+
secretEnv:
47+
BROWSERSTACK_KEY: CiQAkwyoIW0LcnxymzotLwaH4udVTQFBEN4AEA5CA+a3+yflL2ASPQAD8BdZnGARf78MhH5T9rQqyz9HNODwVjVIj64CTkFlUCGrP1B2HX9LXHWHLmtKutEGTeFFX9XhuBzNExA=
48+
timeout: 1800s
49+
logsBucket: 'gs://tfjs-build-logs'
50+
substitutions:
51+
_NIGHTLY: ''
52+
options:
53+
logStreamingOption: 'STREAM_ON'
54+
substitution_option: 'ALLOW_LOOSE'

‎depth-estimation/karma.conf.js

+157
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
/**
2+
* @license
3+
* Copyright 2022 Google LLC. All Rights Reserved.
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
* =============================================================================
16+
*/
17+
18+
const karmaTypescriptConfig = {
19+
tsconfig: 'tsconfig.test.json',
20+
// Disable coverage reports and instrumentation by default for tests.
21+
coverageOptions: {instrumentation: false},
22+
reports: {},
23+
bundlerOptions: {
24+
sourceMap: true,
25+
// Process any non ES5 code through karma-typescript-es6-transform (babel).
26+
acornOptions: {ecmaVersion: 8},
27+
transforms: [
28+
require('karma-typescript-es6-transform')({
29+
presets: [
30+
// Ensure we get ES5 by adding IE 11 as a target.
31+
['@babel/env', {'targets': {'ie': '11'}, 'loose': true}]
32+
]
33+
}),
34+
]
35+
}
36+
};
37+
38+
const devConfig = {
39+
frameworks: ['jasmine', 'karma-typescript'],
40+
files: [
41+
{pattern: './node_modules/@babel/polyfill/dist/polyfill.js'},
42+
{
43+
// Serve test data as static resources.
44+
pattern: 'test_data/**',
45+
watched: true,
46+
included: false,
47+
served: true,
48+
nocache: true
49+
},
50+
'src/setup_test.ts',
51+
{pattern: 'src/**/*.ts'},
52+
],
53+
preprocessors: {'**/*.ts': ['karma-typescript']},
54+
karmaTypescriptConfig,
55+
reporters: ['dots', 'karma-typescript']
56+
};
57+
58+
const browserstackConfig = {
59+
...devConfig,
60+
hostname: 'bs-local.com',
61+
port: 9886
62+
};
63+
64+
module.exports = function(config) {
65+
const args = [];
66+
if (config.testEnv) {
67+
args.push('--testEnv', config.testEnv);
68+
}
69+
if (config.grep) {
70+
args.push('--grep', config.grep);
71+
}
72+
if (config.flags) {
73+
args.push('--flags', config.flags);
74+
}
75+
76+
let extraConfig = null;
77+
78+
if (config.browserstack) {
79+
extraConfig = browserstackConfig;
80+
} else {
81+
extraConfig = devConfig;
82+
}
83+
84+
let exclude = [];
85+
if (config.excludeTest != null) {
86+
exclude.push(config.excludeTest);
87+
}
88+
89+
config.set({
90+
...extraConfig,
91+
singleRun: true,
92+
exclude,
93+
browsers: ['Chrome'],
94+
browserStack: {
95+
username: process.env.BROWSERSTACK_USERNAME,
96+
accessKey: process.env.BROWSERSTACK_KEY,
97+
timeout: 1800,
98+
tunnelIdentifier:
99+
`depth_estimation_${Date.now()}_${Math.floor(Math.random() * 1000)}`
100+
},
101+
captureTimeout: 3e5,
102+
reportSlowerThan: 500,
103+
browserNoActivityTimeout: 3e5,
104+
browserDisconnectTimeout: 3e5,
105+
browserDisconnectTolerance: 3,
106+
browserSocketTimeout: 1.2e5,
107+
customLaunchers: {
108+
// For browserstack configs see:
109+
// https://www.browserstack.com/automate/node
110+
bs_chrome_mac: {
111+
base: 'BrowserStack',
112+
browser: 'chrome',
113+
browser_version: 'latest',
114+
os: 'OS X',
115+
os_version: 'High Sierra'
116+
},
117+
bs_firefox_mac: {
118+
base: 'BrowserStack',
119+
browser: 'firefox',
120+
browser_version: 'latest',
121+
os: 'OS X',
122+
os_version: 'High Sierra'
123+
},
124+
bs_safari_mac: {
125+
base: 'BrowserStack',
126+
browser: 'safari',
127+
browser_version: 'latest',
128+
os: 'OS X',
129+
os_version: 'High Sierra'
130+
},
131+
bs_ios_11: {
132+
base: 'BrowserStack',
133+
device: 'iPhone X',
134+
os: 'iOS',
135+
os_version: '11.0',
136+
real_mobile: true
137+
},
138+
bs_android_9: {
139+
base: 'BrowserStack',
140+
device: 'Google Pixel 3 XL',
141+
os: 'android',
142+
os_version: '9.0',
143+
real_mobile: true
144+
},
145+
win_10_chrome: {
146+
base: 'BrowserStack',
147+
browser: 'chrome',
148+
// Latest Chrome on Windows has WebGL problems:
149+
// https://github.com/tensorflow/tfjs/issues/2272
150+
browser_version: '77.0',
151+
os: 'Windows',
152+
os_version: '10'
153+
},
154+
},
155+
client: {jasmine: {random: false}, args: args}
156+
})
157+
}

‎depth-estimation/package.json

+64
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
{
2+
"name": "@tensorflow-models/depth-estimation",
3+
"version": "0.0.1",
4+
"description": "Pretrained depth model",
5+
"main": "dist/index.js",
6+
"jsnext:main": "dist/depth-estimation.esm.js",
7+
"module": "dist/depth-estimation.esm.js",
8+
"unpkg": "dist/depth-estimation.min.js",
9+
"jsdelivr": "dist/depth-estimation.min.js",
10+
"types": "dist/index.d.ts",
11+
"repository": {
12+
"type": "git",
13+
"url": "https://github.com/tensorflow/tfjs-models.git"
14+
},
15+
"peerDependencies": {
16+
"@tensorflow/tfjs-backend-webgl": "^3.15.0",
17+
"@tensorflow/tfjs-converter": "^3.15.0",
18+
"@tensorflow/tfjs-core": "^3.15.0"
19+
},
20+
"devDependencies": {
21+
"@babel/polyfill": "^7.10.4",
22+
"@rollup/plugin-commonjs": "^11.0.2",
23+
"@rollup/plugin-node-resolve": "^7.1.1",
24+
"@rollup/plugin-typescript": "^3.0.0",
25+
"@tensorflow/tfjs-backend-cpu": "^3.15.0",
26+
"@tensorflow/tfjs-backend-webgl": "^3.15.0",
27+
"@tensorflow/tfjs-converter": "^3.15.0",
28+
"@tensorflow/tfjs-core": "^3.15.0",
29+
"@types/jasmine": "~2.8.8",
30+
"babel-core": "~6.26.0",
31+
"babel-plugin-transform-runtime": "~6.23.0",
32+
"jasmine": "~3.1.0",
33+
"jasmine-core": "~3.1.0",
34+
"karma": "~6.3.1",
35+
"karma-browserstack-launcher": "~1.6.0",
36+
"karma-chrome-launcher": "~2.2.0",
37+
"karma-jasmine": "~1.1.0",
38+
"karma-spec-reporter": "~0.0.32",
39+
"karma-typescript": "~5.5.1",
40+
"karma-typescript-es6-transform": "^5.1.0",
41+
"rollup": "~2.3.2",
42+
"rollup-plugin-terser": "~7.0.2",
43+
"rollup-plugin-visualizer": "~3.3.2",
44+
"ts-node": "~8.8.2",
45+
"tslint": "~5.18.0",
46+
"typescript": "~3.9.9",
47+
"yalc": "~1.0.0-pre.21"
48+
},
49+
"scripts": {
50+
"bundle": "rollup -c",
51+
"build": "rimraf dist && tsc && yarn bundle",
52+
"publish-local": "yarn build && rollup -c && yalc publish",
53+
"run-browserstack": "karma start --browserstack",
54+
"test-node": "NODE_PRESERVE_SYMLINKS=1 ts-node --skip-ignore --project tsconfig.test.json run_tests.ts",
55+
"test": "yarn && karma start",
56+
"test-ci": "yarn run-browserstack --browsers=bs_chrome_mac",
57+
"build-npm": "yarn build && yarn bundle",
58+
"lint": "tslint -p . -t verbose"
59+
},
60+
"license": "Apache-2.0",
61+
"dependencies": {
62+
"rimraf": "^3.0.2"
63+
}
64+
}

0 commit comments

Comments
 (0)