Build improvements (#2076)

* [WIP] Build improvements.

* Fix production build.

* Extract static assets to config/build.js and use them in both dev and prod scripts.

* Cleanup.

* Update yarn.lock.

* Update standalone-packages/vscode-textmate/package-lock.json.

* Bump CircleCI cache keys version.

* Revert "Update standalone-packages/vscode-textmate/package-lock.json."

* Small improvements.

* Update webpack to latest 4.35.0.

* CodeMirror: remove auto-complete, so we're not bundling tern.

* Remove unneeded gulp tasks.

* Update webpack-dev-server to latest 3.7.2 and remove unused webpack-dev-middleware.

* Show the updated module which triggers re-compiling.

* Don't use thread-loader while linting.

* Extract tern dep to codesandbox-deps and re-add CodeMirror bits.

* Disable console clearing on Windows by default and
- use CLEAR environment variable to override default behavior.

Author: lbogdan

Diff for: .circleci/config.yml

@@ -6,21 +6,21 @@ version: 2.1
 
 aliases:
   - &restore_repo_cache
-    key: v20-repo-{{ .Environment.CIRCLE_SHA1 }}
+    key: v18-repo-{{ .Environment.CIRCLE_SHA1 }}
 
   - &save_repo_cache
-    key: v20-repo-{{ .Environment.CIRCLE_SHA1 }}
+    key: v18-repo-{{ .Environment.CIRCLE_SHA1 }}
     paths:
       - ~/codesandbox-client
 
   - &restore_deps_cache
     keys:
-      - v20-dependency-cache-{{ .Branch }}-{{ checksum "yarn.lock" }}
-      - v20-dependency-cache-{{ .Branch }}
-      - v20-dependency-cache
+      - v18-dependency-cache-{{ .Branch }}-{{ checksum "yarn.lock" }}
+      - v18-dependency-cache-{{ .Branch }}
+      - v18-dependency-cache
 
   - &save_deps_cache
-    key: v20-dependency-cache-{{ .Branch }}-{{ checksum "yarn.lock" }}
+    key: v18-dependency-cache-{{ .Branch }}-{{ checksum "yarn.lock" }}
     paths:
       - node_modules
       - packages/app/node_modules
@@ -39,29 +39,29 @@ aliases:
 
   - &restore_standalone_deps_cache
     keys:
-      - v20-standalone-dependency-cache-{{ .Branch }}-{{ checksum "standalone-packages/codesandbox-browserfs/yarn.lock" }}
-      - v20-standalone-dependency-cache-{{ .Branch }}
-      - v20-standalone-dependency-cache
+      - v18-standalone-dependency-cache-{{ .Branch }}-{{ checksum "standalone-packages/codesandbox-browserfs/yarn.lock" }}
+      - v18-standalone-dependency-cache-{{ .Branch }}
+      - v18-standalone-dependency-cache
 
   - &save_standalone_deps_cache
-    key: v20-standalone-dependency-cache-{{ .Branch }}-{{ checksum "standalone-packages/codesandbox-browserfs/yarn.lock" }}
+    key: v18-standalone-dependency-cache-{{ .Branch }}-{{ checksum "standalone-packages/codesandbox-browserfs/yarn.lock" }}
     paths:
       - standalone-packages/codesandbox-browserfs/node_modules
 
   - &restore_prod_build_cache
-    key: v20-prod-app-build-cache-master
+    key: v18-prod-app-build-cache-master
 
   - &restore_prod_cache
-    key: v20-prod-build-cache-{{ .Environment.CIRCLE_BRANCH }}-{{ .Environment.CIRCLE_SHA1 }}
+    key: v18-prod-build-cache-{{ .Environment.CIRCLE_BRANCH }}-{{ .Environment.CIRCLE_SHA1 }}
 
   - &save_prod_app_cache
-    key: v20-prod-app-build-cache-{{ .Environment.CIRCLE_BRANCH }}-{{ .Environment.CIRCLE_SHA1 }}
+    key: v18-prod-app-build-cache-{{ .Environment.CIRCLE_BRANCH }}-{{ .Environment.CIRCLE_SHA1 }}
     paths:
       - ./packages/app/www
       - ./packages/homepage/.cache
 
   - &save_prod_cache
-    key: v20-prod-build-cache-{{ .Environment.CIRCLE_BRANCH }}-{{ .Environment.CIRCLE_SHA1 }}
+    key: v18-prod-build-cache-{{ .Environment.CIRCLE_BRANCH }}-{{ .Environment.CIRCLE_SHA1 }}
     paths:
       - ./www
 

Diff for: Gulpfile.js

@@ -21,46 +21,6 @@ function rimraf(dir) {
   return cb => retry(cb);
 }
 
-gulp.task('app', () =>
-  gulp.src('packages/app/www/**/*').pipe(gulp.dest('www'))
-);
-
-gulp.task('homepage', () =>
-  gulp.src('packages/homepage/public/**/*').pipe(gulp.dest('www'))
-);
-
-gulp.task('monaco', () =>
-  gulp
-    .src('standalone-packages/monaco-editor/release/min/vs/**/*')
-    .pipe(gulp.dest('www/public/14/vs'))
-);
-
-// Used for cases where HTML is cached
-gulp.task('old-browserfs', () =>
-  gulp
-    .src('standalone-packages/codesandbox-browserfs/dist/**/*')
-    .pipe(gulp.dest('www/static/browserfs2'))
-);
-
-gulp.task('old-vscode', () =>
-  gulp
-    .src('standalone-packages/vscode-editor/release/min/vs/**/*')
-    .pipe(gulp.dest('www/public/vscode8/vs'))
-);
-
-gulp.task('statics', () =>
-  gulp.src('packages/app/public/**/*').pipe(gulp.dest('www'))
-);
-
-gulp.task('default', [
-  'app',
-  'homepage',
-  'statics',
-  'monaco',
-  'old-browserfs',
-  'old-vscode',
-]);
-
 gulp.task('clean-vscode', rimraf('standalone-packages/monaco-editor-core'));
 
 gulp.task('prepare-vscode', ['clean-vscode'], () =>

Diff for: package.json

@@ -4,26 +4,26 @@
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
-    "build": "NODE_OPTIONS=\"--max-old-space-size=4096\" yarn build:deps && yarn build:prod",
-    "build:prod": "NODE_OPTIONS=\"--max-old-space-size=4096\" lerna run build  --scope homepage --stream && lerna run build --scope app --stream && gulp",
+    "build": "cross-env NODE_OPTIONS=\"--max-old-space-size=4096\" yarn build:deps && yarn build:prod",
+    "build:prod": "cross-env NODE_OPTIONS=\"--max-old-space-size=4096\" lerna run build  --scope homepage --stream && lerna run build --scope app --stream && lerna run copy-assets --scope app --stream",
     "build:embed": "lerna run build:embed --scope app --stream && gulp",
     "build:clean": "lerna run build:clean --scope app --scope homepage && rimraf www",
-    "build:deps": "lerna run build:dev --scope codesandbox-api --scope @codesandbox/notifications && lerna run build:dev --scope @codesandbox/common && lerna run build:dev --scope vscode-textmate --scope codesandbox-browserfs --scope node-services && lerna run build:dev --scope sse-hooks",
+    "build:deps": "lerna run build:dev --scope codesandbox-api --scope @codesandbox/notifications && lerna run build:dev --scope @codesandbox/common && lerna run build:dev --scope vscode-textmate --scope codesandbox-browserfs --scope node-services && lerna run build:dev --scope sse-hooks && lerna run build --scope codesandbox-deps",
     "build:common": "lerna run build:dev --scope @codesandbox/common --stream",
     "build:dynamic": "lerna run build --scope dynamic-pages --stream",
-    "start": "yarn build:deps && lerna run start --scope app --stream",
+    "start": "yarn build:deps && yarn start:fast",
     "start:overmind": "yarn build:deps && concurrently \"lerna run start --scope app --stream\" \"overmind-devtools\"",
-    "start:fast": "lerna run start --scope app --stream",
-    "start:vscode": "VSCODE=1 yarn start:fast & cd standalone-packages/monaco-editor && yarn simpleserver & cd standalone-packages/vscode && yarn watch",
+    "start:fast": "cd packages/app && yarn start",
+    "start:vscode": "cross-env VSCODE=1 yarn start:fast & cd standalone-packages/monaco-editor && yarn simpleserver & cd standalone-packages/vscode && yarn watch",
     "start:dynamic": "lerna run dev --scope dynamic-pages --stream",
     "start:common": "lerna run start --scope common --stream",
-    "start:home": "lerna run start --scope homepage --stream",
+    "start:home": "cd packages/homepage && yarn start",
     "start:test": "lerna run start:test --scope app --stream",
     "start:dev_api": "lerna run start:dev_api --scope app --stream",
     "test": "lerna run test --ignore codesandbox-browserfs",
     "test:integrations": "lerna exec --scope app --stream -- yarn test:integrations",
     "test:jest-lite": "lerna exec --scope app --stream -- yarn run test jest-lite --watch --coverage",
-    "lint": "lerna run lint",
+    "lint": "lerna run lint --stream",
     "add-contributor": "all-contributors add",
     "generate": "all-contributors generate",
     "postinstall": "rimraf node_modules/@types/react-native && rimraf packages/app/node_modules/styled-components node_modules/react-split-pane/node_modules/react* && yarn lerna run install-dependencies --scope vscode-textmate --scope codesandbox-browserfs --scope sse-loading-screen --stream && opencollective postinstall",
@@ -42,11 +42,15 @@
       "packages/sse-hooks",
       "packages/react-embed",
       "packages/dynamic-pages",
-      "packages/notifications"
+      "packages/notifications",
+      "packages/deps"
     ],
     "nohoist": [
       "**/react-codemirror2",
-      "**/react-codemirror2/**"
+      "**/react-codemirror2/**",
+      "app/webpack",
+      "app/acorn-dynamic-import",
+      "codesandbox-deps/tern"
     ]
   },
   "resolutions": {

Diff for: packages/app/config/build.js

@@ -0,0 +1,42 @@
+const isDev = process.env.NODE_ENV === 'development';
+const staticAssets = [
+  {
+    from: 'standalone-packages/vscode-editor/release/min/vs',
+    to: 'public/vscode22/vs',
+  },
+  {
+    from: 'standalone-packages/vscode-extensions/out',
+    to: 'public/vscode-extensions/v8',
+  },
+  {
+    from: 'node_modules/onigasm/lib/onigasm.wasm',
+    to: 'public/onigasm/2.2.1/onigasm.wasm',
+  },
+  {
+    from:
+      'standalone-packages/vscode-textmate/node_modules/onigasm/lib/onigasm.wasm',
+    to: 'public/onigasm/2.1.0/onigasm.wasm',
+  },
+  {
+    from: 'node_modules/monaco-vue/release/min',
+    to: 'public/14/vs/language/vue',
+  },
+  {
+    from: 'standalone-packages/monaco-editor/release/min/vs',
+    to: 'public/14/vs',
+  },
+  {
+    from: 'packages/app/static',
+    to: 'static',
+  },
+  {
+    from: isDev
+      ? 'standalone-packages/codesandbox-browserfs/build'
+      : 'standalone-packages/codesandbox-browserfs/dist',
+    to: 'static/browserfs3',
+  },
+];
+
+module.exports = {
+  staticAssets,
+};

Diff for: packages/app/config/webpack.common.js

@@ -4,8 +4,7 @@ const fs = require('fs');
 const paths = require('./paths');
 const HtmlWebpackPlugin = require('html-webpack-plugin');
 const CaseSensitivePathsPlugin = require('case-sensitive-paths-webpack-plugin');
-const CopyWebpackPlugin = require('copy-webpack-plugin');
-const HappyPack = require('happypack');
+const threadLoader = require('thread-loader');
 const WatchMissingNodeModulesPlugin = require('../scripts/utils/WatchMissingNodeModulesPlugin');
 const env = require('@codesandbox/common/lib/config/env');
 const getHost = require('@codesandbox/common/lib/utils/host');
@@ -19,20 +18,8 @@ const __DEV__ = NODE_ENV === 'development'; // eslint-disable-line no-underscore
 const __PROD__ = NODE_ENV === 'production'; // eslint-disable-line no-underscore-dangle
 // const __TEST__ = NODE_ENV === 'test'; // eslint-disable-line no-underscore-dangle
 const babelConfig = __DEV__ && !SANDBOX_ONLY ? babelDev : babelProd;
-
 const publicPath = SANDBOX_ONLY || __DEV__ ? '/' : getHost.default() + '/';
-
-let threads = 1;
-
-try {
-  if (process.env.CIRCLECI) {
-    threads = 2;
-  } else {
-    threads = Math.max(1, require('os').cpus().length - 1);
-  }
-} catch (e) {
-  threads = 3;
-}
+const isLint = 'LINT' in process.env;
 
 // Shim for `eslint-plugin-vue/lib/index.js`
 const ESLINT_PLUGIN_VUE_INDEX = `module.exports = {
@@ -63,6 +50,14 @@ const ESLINT_PLUGIN_VUE_INDEX = `module.exports = {
 
 const sepRe = `\\${path.sep}`; // path separator regex
 
+const threadPoolConfig = {
+  workers: 2,
+};
+
+if (!isLint) {
+  threadLoader.warmup(threadPoolConfig, ['babel-loader']);
+}
+
 module.exports = {
   entry: SANDBOX_ONLY
     ? {
@@ -101,6 +96,7 @@ module.exports = {
     publicPath,
     globalObject: 'this',
     jsonpFunction: 'csbJsonP', // So we don't conflict with webpack generated libraries in the sandbox
+    pathinfo: false,
   },
 
   module: {
@@ -172,13 +168,18 @@ module.exports = {
           /typescriptServices\.js$/,
           /\.no-webpack\./,
         ],
-        loader: 'happypack/loader',
-      },
-
-      {
-        test: /\.tsx?$/,
-        exclude: [/node_modules/],
-        loader: 'happypack/loader',
+        use: [
+          !isLint
+            ? {
+                loader: 'thread-loader',
+                options: threadPoolConfig,
+              }
+            : false,
+          {
+            loader: 'babel-loader',
+            options: babelConfig,
+          },
+        ].filter(Boolean),
       },
 
       // `eslint-plugin-vue/lib/index.js` depends on `fs` module we cannot use in browsers, so needs shimming.
@@ -333,15 +334,6 @@ module.exports = {
   },
 
   plugins: [
-    new HappyPack({
-      loaders: [
-        {
-          path: 'babel-loader',
-          query: babelConfig,
-        },
-      ],
-      threads,
-    }),
     ...(SANDBOX_ONLY
       ? [
           new HtmlWebpackPlugin({
@@ -462,51 +454,5 @@ module.exports = {
     // makes the discovery automatic so you don't have to restart.
     // See https://github.com/facebookincubator/create-react-app/issues/186
     new WatchMissingNodeModulesPlugin(paths.appNodeModules),
-    // Make the monaco editor work
-    new CopyWebpackPlugin(
-      [
-        {
-          from: '../../standalone-packages/vscode-editor/release/min/vs',
-          to: 'public/vscode22/vs',
-          force: true,
-        },
-        {
-          from: '../../standalone-packages/vscode-extensions/out',
-          to: 'public/vscode-extensions/v8',
-          force: true,
-        },
-        {
-          from: '../../node_modules/onigasm/lib/onigasm.wasm',
-          to: 'public/onigasm/2.2.1/onigasm.wasm',
-        },
-        {
-          from:
-            '../../standalone-packages/vscode-textmate/node_modules/onigasm/lib/onigasm.wasm',
-          to: 'public/onigasm/2.1.0/onigasm.wasm',
-        },
-        {
-          from: '../../node_modules/monaco-vue/release/min',
-          to: 'public/14/vs/language/vue',
-        },
-        {
-          from: '../../standalone-packages/monaco-editor/release/min/vs',
-          to: 'public/14/vs',
-        },
-        {
-          from: '../sse-hooks/dist',
-          to: 'public/sse-hooks',
-        },
-        {
-          from: 'static',
-          to: 'static',
-        },
-        {
-          from: __DEV__
-            ? '../../standalone-packages/codesandbox-browserfs/build'
-            : '../../standalone-packages/codesandbox-browserfs/dist',
-          to: 'static/browserfs3',
-        },
-      ].filter(x => x)
-    ),
   ].filter(Boolean),
 };