model.json support in custom bundle tool & converter

Adds support for parsing model.json(s) for ops to be included in the custom bundle.

Allows for modifying ops available to tfjs-converter's op executors at build time of an end user program along with a custom build. 

DEV

Author: tafsiri

e2e/karma.conf.js

@@ -17,12 +17,24 @@
 
 const karmaTypescriptConfig = {
   tsconfig: 'tsconfig.json',
-  coverageOptions: {instrumentation: false}
+  coverageOptions: {instrumentation: false},
+  bundlerOptions: {
+    acornOptions: {ecmaVersion: 8},
+    transforms: [
+      require('karma-typescript-es6-transform')({
+        presets: [
+          // ensure we get es5 by adding IE 11 as a target
+          ['@babel/env', {targets: {browsers: ['defaults', 'IE 11']}}]
+        ]
+      }),
+    ]
+  }
 };
 
 const devConfig = {
   frameworks: ['jasmine', 'karma-typescript'],
   files: [
+    {pattern: './node_modules/@babel/polyfill/dist/polyfill.js'},
     'integration_tests/setup_test.ts',
     {pattern: 'integration_tests/**/*.ts'},
     {

e2e/package.json

@@ -8,13 +8,14 @@
     "url": "https://github.com/tensorflow/tfjs"
   },
   "devDependencies": {
-    "@tensorflow/tfjs-core": "link:../tfjs-core",
-    "@tensorflow/tfjs-converter": "link:../tfjs-converter",
-    "@tensorflow/tfjs-layers": "link:../tfjs-layers",
+    "@babel/polyfill": "^7.10.4",
+    "@tensorflow/tfjs": "link:../tfjs",
     "@tensorflow/tfjs-backend-cpu": "link:../tfjs-backend-cpu",
     "@tensorflow/tfjs-backend-webgl": "link:../tfjs-backend-webgl",
+    "@tensorflow/tfjs-converter": "link:../tfjs-converter",
+    "@tensorflow/tfjs-core": "link:../tfjs-core",
     "@tensorflow/tfjs-data": "link:../tfjs-data",
-    "@tensorflow/tfjs": "link:../tfjs",
+    "@tensorflow/tfjs-layers": "link:../tfjs-layers",
     "@tensorflow/tfjs-node": "link:../tfjs-node",
     "@types/detect-browser": "^4.0.0",
     "@types/jasmine": "~3.0.0",
@@ -28,6 +29,7 @@
     "karma-chrome-launcher": "~2.2.0",
     "karma-jasmine": "~1.1.0",
     "karma-typescript": "~4.1.1",
+    "karma-typescript-es6-transform": "^5.1.0",
     "mathjs": "^5.7.0",
     "request": "^2.88.0",
     "ts-node": "~8.8.2",

e2e/yarn.lock

@@ -74,5 +74,11 @@
     "typescript": "3.5.3",
     "yalc": "~1.0.0-pre.21"
   },
-  "license": "Apache-2.0"
+  "license": "Apache-2.0",
+  "sideEffects": [
+    "./dist/register_all_kernels.js",
+    "./dist/flags_wasm.js",
+    "./dist/base.js",
+    "./dist/index.js"
+  ]
 }

tfjs-backend-wasm/package.json

@@ -102,7 +102,7 @@ function depthwiseConv2d(args: {
   return out;
 }
 
-export const depthwiseConv2DNativeConfig: KernelConfig = {
+export const depthwiseConv2dNativeConfig: KernelConfig = {
   kernelName: DepthwiseConv2dNative,
   backendName: 'wasm',
   setupFunc: setup,

tfjs-backend-wasm/src/kernels/DepthwiseConv2dNative.ts

@@ -34,7 +34,7 @@ import {conv2DBackpropInputConfig} from './kernels/Conv2DBackpropInput';
 import {cosConfig} from './kernels/Cos';
 import {cropAndResizeConfig} from './kernels/CropAndResize';
 import {cumsumConfig} from './kernels/Cumsum';
-import {depthwiseConv2DNativeConfig} from './kernels/DepthwiseConv2dNative';
+import {depthwiseConv2dNativeConfig} from './kernels/DepthwiseConv2dNative';
 import {divConfig} from './kernels/Div';
 import {equalConfig} from './kernels/Equal';
 import {expConfig} from './kernels/Exp';
@@ -110,7 +110,7 @@ const kernelConfigs: KernelConfig[] = [
   cosConfig,
   cropAndResizeConfig,
   cumsumConfig,
-  depthwiseConv2DNativeConfig,
+  depthwiseConv2dNativeConfig,
   divConfig,
   equalConfig,
   expConfig,

tfjs-backend-wasm/src/register_all_kernels.ts

@@ -315,9 +315,7 @@
     "onesLike"
   ],
   "Pack": [
-    "tidy",
     "squeeze",
-    "util.arraysEqual",
     "reshape",
     "stack"
   ],

tfjs-converter/metadata/kernel2op.json

@@ -20,6 +20,7 @@
   "devDependencies": {
     "@rollup/plugin-commonjs": "^11.0.2",
     "@rollup/plugin-node-resolve": "^7.1.1",
+    "@rollup/plugin-replace": "^2.3.3",
     "@rollup/plugin-typescript": "^3.0.0",
     "@tensorflow/tfjs-backend-cpu": "link:../tfjs-backend-cpu",
     "@tensorflow/tfjs-core": "link:../tfjs-core",

tfjs-converter/package.json

@@ -17,6 +17,7 @@
 
 import commonjs from '@rollup/plugin-commonjs';
 import resolve from '@rollup/plugin-node-resolve';
+import replace from '@rollup/plugin-replace';
 import typescript from '@rollup/plugin-typescript';
 import {terser} from 'rollup-plugin-terser';
 import visualizer from 'rollup-plugin-visualizer';
@@ -74,10 +75,16 @@ function config({
     output: {
       banner: PREAMBLE,
       sourcemap: true,
-      globals: {'@tensorflow/tfjs-core': 'tf'},
+      globals: {
+        '@tensorflow/tfjs-core': 'tf',
+        '@tensorflow/tfjs-core/dist/ops/ops_for_converter': 'tf'
+      },
       ...output
     },
-    external: ['@tensorflow/tfjs-core', ...external],
+    external: [
+      '@tensorflow/tfjs-core',
+      '@tensorflow/tfjs-core/dist/ops/ops_for_converter', ...external
+    ],
     onwarn: warning => {
       let {code} = warning;
       if (code === 'CIRCULAR_DEPENDENCY' || code === 'CIRCULAR' ||
@@ -161,6 +168,29 @@ module.exports = cmdOptions => {
       },
       tsCompilerOptions: {target: 'es2017'}
     }));
+
+    // Miniprogram entry (minified es5)
+    bundles.push(config({
+      plugins: [
+        // replace dist import with tfjs-core because miniprogram build
+        // systems modify the package structure of our npm package, and only
+        // mirrors the 'main' entries.
+        replace({
+          '@tensorflow/tfjs-core/dist/ops/ops_for_converter':
+              '@tensorflow/tfjs-core',
+          delimiters: ['', '']
+        }),
+        terserPlugin
+      ],
+      output: {
+        format: browserFormat,
+        name,
+        extend,
+        file: `dist/miniprogram/index.js`,
+        freeze: false
+      },
+      tsCompilerOptions: {target: 'es5'},
+    }));
   }
 
   return bundles;

tfjs-converter/rollup.config.js

@@ -22,10 +22,5 @@ yarn
 yarn build-ci
 yarn rollup -c --visualize --npm
 
-# Use minified files for miniprogram
-mkdir dist/miniprogram
-cp dist/tf-converter.min.js dist/miniprogram/index.js
-cp dist/tf-converter.min.js.map dist/miniprogram/index.js.map
-
 echo "Stored standalone library at dist/tf-converter(.min).js"
 npm pack