chore: move to patches (#4997)

* Move integration types into code-server

This will be easier to maintain than to have it as a patch.

* Disable connection token

Using a flag means we will not need to patch it out.  I think this is
new from 1.64?

* Add product.json to build process

This way we do not have to patch it.

* Ship with remote agent package.json

Instead of the root one.  This contains fewer dependencies.

* Let Code handle errors

This way we will not have to patch Code to make this work and I think it
makes sense to let Code handle the request.

If we do want to handle errors we can do it cleanly by patching their
error handler to throw instead.

* Move manifest override into code-server

This way we will not have to patch it.

* Move to patches

- Switch submodule to track upstream
- Add quilt to the process
- Add patches

The node-* ignore was ignoring one of the diffs so I removed it.  This
was added when we were curling Node as node-v{version}-darwin-x64 for
the macOS build but this no longer happens (we use the Node action to
install a specific version now so we just use the system-wide Node).

* Use pre-packaged Code

Author: code-asher

.github/workflows/ci.yaml

@@ -110,6 +110,12 @@ jobs:
           fetch-depth: 0
           submodules: true
 
+      - name: Install quilt
+        run: sudo apt update && sudo apt install quilt
+
+      - name: Patch Code
+        run: quilt push -a
+
       - name: Install Node.js v14
         uses: actions/setup-node@v3
         with:
@@ -132,25 +138,22 @@ jobs:
         run: yarn build
 
       # Get Code's git hash.  When this changes it means the content is
-      # different and we need to rebuild.  Use VSCODE_CACHE_VERSION to force a
-      # rebuild.
+      # different and we need to rebuild.
       - name: Get latest lib/vscode rev
         id: vscode-rev
         run: echo "::set-output name=rev::$(git rev-parse HEAD:./lib/vscode)"
 
-      - name: Fetch Code build from cache
-        id: cache-vscode-2
+        # We need to rebuild when we have a new version of Code or when any of
+        # the patches changed.  Use VSCODE_CACHE_VERSION to force a rebuild.
+      - name: Fetch prebuilt Code package from cache
+        id: cache-vscode
         uses: actions/cache@v3
         with:
-          path: |
-            lib/vscode/.build
-            lib/vscode/out-build
-            lib/vscode/out-vscode-reh-web
-            lib/vscode/out-vscode-reh-web-min
-          key: vscode-reh-build-${{ secrets.VSCODE_CACHE_VERSION }}-${{ steps.vscode-rev.outputs.rev }}
+          path: lib/vscode-reh-web-*
+          key: vscode-reh-package-${{ secrets.VSCODE_CACHE_VERSION }}-${{ steps.vscode-rev.outputs.rev }}-${{ hashFiles('patches/*.diff') }}
 
       - name: Build vscode
-        if: steps.cache-vscode-2.outputs.cache-hit != 'true'
+        if: steps.cache-vscode.outputs.cache-hit != 'true'
         run: yarn build:vscode
 
       # Our code imports code from VS Code's `out` directory meaning VS Code

.gitignore

@@ -8,12 +8,17 @@ release-packages/
 release-gcp/
 release-images/
 node_modules
-vendor/modules
-node-*
 /plugins
 /lib/coder-cloud-agent
 .home
 coverage
 **/.DS_Store
+
+# Code packages itself here.
+/lib/vscode-reh-web-*
+
 # Failed e2e test videos are saved here
 test/test-results
+
+# Quilt's internal data.
+/.pc

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "lib/vscode"]
 	path = lib/vscode
-	url = https://github.com/coder/vscode
+	url = https://github.com/microsoft/vscode

ci/build/build-packages.sh

@@ -50,6 +50,11 @@ release_nfpm() {
 
   export NFPM_ARCH
 
+  # Code deletes some files from the extension node_modules directory which
+  # leaves broken symlinks in the corresponding .bin directory.  nfpm will fail
+  # on these broken symlinks so clean them up.
+  rm -fr "./release-standalone/lib/vscode/extensions/node_modules/.bin"
+
   PKG_FORMAT="deb"
   NFPM_ARCH="$(get_nfpm_arch $PKG_FORMAT "$ARCH")"
   nfpm_config="$(envsubst < ./ci/build/nfpm.yaml)"

ci/build/build-release.sh

@@ -66,47 +66,59 @@ EOF
 
 bundle_vscode() {
   mkdir -p "$VSCODE_OUT_PATH"
-  rsync "$VSCODE_SRC_PATH/yarn.lock" "$VSCODE_OUT_PATH"
-  rsync "$VSCODE_SRC_PATH/out-vscode-reh-web${MINIFY:+-min}/" "$VSCODE_OUT_PATH/out"
+  rsync ./lib/vscode-reh-web-*/ "$VSCODE_OUT_PATH"
 
-  rsync "$VSCODE_SRC_PATH/.build/extensions/" "$VSCODE_OUT_PATH/extensions"
-  if [ "$KEEP_MODULES" = 0 ]; then
-    rm -Rf "$VSCODE_OUT_PATH/extensions/node_modules"
-  else
-    rsync "$VSCODE_SRC_PATH/node_modules/" "$VSCODE_OUT_PATH/node_modules"
-  fi
-  rsync "$VSCODE_SRC_PATH/extensions/package.json" "$VSCODE_OUT_PATH/extensions"
-  rsync "$VSCODE_SRC_PATH/extensions/yarn.lock" "$VSCODE_OUT_PATH/extensions"
-  rsync "$VSCODE_SRC_PATH/extensions/postinstall.js" "$VSCODE_OUT_PATH/extensions"
-
-  mkdir -p "$VSCODE_OUT_PATH/resources/"
-  rsync "$VSCODE_SRC_PATH/resources/" "$VSCODE_OUT_PATH/resources/"
-
-  # TODO: We should look into using VS Code's packaging task (see
-  # gulpfile.reh.js).  For now copy this directory into the right spot (for some
-  # reason VS Code uses a different path in production).
-  mkdir -p "$VSCODE_OUT_PATH/bin/helpers"
-  rsync "$VSCODE_SRC_PATH/resources/server/bin/helpers/" "$VSCODE_OUT_PATH/bin/helpers"
-  chmod +x "$VSCODE_OUT_PATH/bin/helpers/browser.sh"
-
-  # Add the commit and date and enable telemetry. This just makes telemetry
-  # available; telemetry can still be disabled by flag or setting.
+  # Add the commit, date, our name, links, and enable telemetry. This just makes
+  # telemetry available; telemetry can still be disabled by flag or setting.
   jq --slurp '.[0] * .[1]' "$VSCODE_SRC_PATH/product.json" <(
     cat << EOF
   {
     "enableTelemetry": true,
     "commit": "$(cd "$VSCODE_SRC_PATH" && git rev-parse HEAD)",
     "quality": "stable",
     "date": $(jq -n 'now | todate'),
-    "codeServerVersion": "$VERSION"
+    "codeServerVersion": "$VERSION",
+    "nameShort": "code-server",
+    "nameLong": "code-server",
+    "applicationName": "code-server",
+    "dataFolderName": ".code-server",
+    "win32MutexName": "codeserver",
+    "licenseUrl": "https://github.com/coder/code-server/blob/main/LICENSE.txt",
+    "win32DirName": "code-server",
+    "win32NameVersion": "code-server",
+    "win32AppUserModelId": "coder.code-server",
+    "win32ShellNameShort": "c&ode-server",
+    "darwinBundleIdentifier": "com.coder.code.server",
+    "linuxIconName": "com.coder.code.server",
+    "reportIssueUrl": "https://github.com/coder/code-server/issues/new",
+    "documentationUrl": "https://go.microsoft.com/fwlink/?LinkID=533484#vscode",
+    "keyboardShortcutsUrlMac": "https://go.microsoft.com/fwlink/?linkid=832143",
+    "keyboardShortcutsUrlLinux": "https://go.microsoft.com/fwlink/?linkid=832144",
+    "keyboardShortcutsUrlWin": "https://go.microsoft.com/fwlink/?linkid=832145",
+    "introductoryVideosUrl": "https://go.microsoft.com/fwlink/?linkid=832146",
+    "tipsAndTricksUrl": "https://go.microsoft.com/fwlink/?linkid=852118",
+    "newsletterSignupUrl": "https://www.research.net/r/vsc-newsletter",
+    "linkProtectionTrustedDomains": [
+      "https://open-vsx.org"
+    ]
   }
 EOF
   ) > "$VSCODE_OUT_PATH/product.json"
 
-  # We remove the scripts field so that later on we can run
-  # yarn to fetch node_modules if necessary without build scripts running.
-  # We cannot use --no-scripts because we still want dependent package scripts to run.
-  jq 'del(.scripts)' < "$VSCODE_SRC_PATH/package.json" > "$VSCODE_OUT_PATH/package.json"
+  # Use the package.json for the web/remote server.  It does not have the right
+  # version though so pull that from the main package.json.  Also remove keytar
+  # since the web does not rely on it and that removes the dependency on
+  # libsecret.
+  jq --slurp '.[0] * {version: .[1].version} | del(.dependencies.keytar)' \
+    "$VSCODE_SRC_PATH/remote/package.json" \
+    "$VSCODE_SRC_PATH/package.json" > "$VSCODE_OUT_PATH/package.json"
+
+  rsync "$VSCODE_SRC_PATH/remote/yarn.lock" "$VSCODE_OUT_PATH/yarn.lock"
+
+  if [ "$KEEP_MODULES" = 0 ]; then
+    rm -Rf "$VSCODE_OUT_PATH/extensions/node_modules"
+    rm -Rf "$VSCODE_OUT_PATH/node_modules"
+  fi
 
   pushd "$VSCODE_OUT_PATH"
   symlink_asar

ci/build/build-standalone-release.sh

@@ -29,11 +29,6 @@ main() {
 
   cd "$RELEASE_PATH"
   yarn --production --frozen-lockfile
-
-  # HACK: the version of Typescript vscode 1.57 uses in extensions/
-  # leaves a few stray symlinks. Clean them up so nfpm does not fail.
-  # Remove this line when its no longer needed.
-  rm -fr "$RELEASE_PATH/lib/vscode/extensions/node_modules/.bin"
 }
 
 main "$@"

ci/dev/test-unit.sh

@@ -14,11 +14,17 @@ main() {
   # Our code imports from `out` in order to work during development but if you
   # have only built for production you will have not have this directory.  In
   # that case symlink `out` to a production build directory.
-  local vscode="lib/vscode"
-  local link="$vscode/out"
-  local target="out-build"
-  if [[ ! -e $link ]] && [[ -d $vscode/$target ]]; then
-    ln -s "$target" "$link"
+  if [[ ! -e lib/vscode/out ]]; then
+    pushd lib
+    local out=(vscode-reh-web-*)
+    if [[ -d "${out[0]}" ]]; then
+      ln -s "../${out[0]}/out" ./vscode/out
+    else
+      echo "Could not find lib/vscode/out or lib/vscode-reh-web-*"
+      echo "Code must be built before running unit tests"
+      exit 1
+    fi
+    popd
   fi
 
   # We must keep jest in a sub-directory. See ../../test/package.json for more