Adopt the MSAL broker to talk to the OS for Microsoft auth (#233739)

This adopts the `NativeBrokerPlugin` provided by `@azure/msal-node-extensions` to provide the ability to use auth state from the OS, and show native auth dialogs instead of going to the browser.

This has several pieces:
* The adoption of the broker in the microsoft-authentication extension:
  * Adding `NativeBrokerPlugin` to our PCAs
  * Using the proposed handle API to pass the native window handle down to MSAL calls (btw, this API will change in a follow up PR)
  * Adopting an AccountAccess layer to handle:
    * giving the user control of which accounts VS Code uses
    * an eventing layer so that auth state can be updated across multiple windows
* Getting the extension to build properly and only build what it really needs. This required several package.json/webpack hacks:
  * Use a fake keytar since we don't use the feature in `@azure/msal-node-extensions` that uses keytar
  * Use a fake dpapi layer since we don't use the feature in  `@azure/msal-node-extensions` that uses it
  * Ensure the msal runtime `.node` and `.dll` files are included in the bundle
* Get the VS Code build to allow a native node module in an extension: by having a list of native extensions that will be built in the "ci" part of the build - in other words when VS Code is building on the target platform

There are a couple of followups:
* Refactor the `handle` API to handle (heh) Auxiliary Windows #233106
* Separate the call to `acquireTokenSilent` and `acquireTokenInteractive` and all the usage of this native node module into a separate process or maybe in Core... we'll see. Something to experiment with after we have something working. NEEDS FOLLOW UP ISSUE

Fixes #229431

build/darwin/create-universal-app.ts

@@ -28,6 +28,8 @@ async function main(buildDir?: string) {
 	const filesToSkip = [
 		'**/CodeResources',
 		'**/Credits.rtf',
+		// TODO: Should we consider expanding this to other files in this area?
+		'**/node_modules/@parcel/node-addon-api/nothing.target.mk'
 	];
 
 	await makeUniversalApp({

build/gulpfile.extensions.js

@@ -230,27 +230,61 @@ exports.compileExtensionMediaBuildTask = compileExtensionMediaBuildTask;
 
 //#region Azure Pipelines
 
+/**
+ * Cleans the build directory for extensions
+ */
 const cleanExtensionsBuildTask = task.define('clean-extensions-build', util.rimraf('.build/extensions'));
-const compileExtensionsBuildTask = task.define('compile-extensions-build', task.series(
+exports.cleanExtensionsBuildTask = cleanExtensionsBuildTask;
+
+/**
+ * brings in the marketplace extensions for the build
+ */
+const bundleMarketplaceExtensionsBuildTask = task.define('bundle-marketplace-extensions-build', () => ext.packageMarketplaceExtensionsStream(false).pipe(gulp.dest('.build')));
+
+/**
+ * Compiles the non-native extensions for the build
+ * @note this does not clean the directory ahead of it. See {@link cleanExtensionsBuildTask} for that.
+ */
+const compileNonNativeExtensionsBuildTask = task.define('compile-non-native-extensions-build', task.series(
+	bundleMarketplaceExtensionsBuildTask,
+	task.define('bundle-non-native-extensions-build', () => ext.packageNonNativeLocalExtensionsStream().pipe(gulp.dest('.build')))
+));
+gulp.task(compileNonNativeExtensionsBuildTask);
+exports.compileNonNativeExtensionsBuildTask = compileNonNativeExtensionsBuildTask;
+
+/**
+ * Compiles the native extensions for the build
+ * @note this does not clean the directory ahead of it. See {@link cleanExtensionsBuildTask} for that.
+ */
+const compileNativeExtensionsBuildTask = task.define('compile-native-extensions-build', () => ext.packageNativeLocalExtensionsStream().pipe(gulp.dest('.build')));
+gulp.task(compileNativeExtensionsBuildTask);
+exports.compileNativeExtensionsBuildTask = compileNativeExtensionsBuildTask;
+
+/**
+ * Compiles the extensions for the build.
+ * This is essentially a helper task that combines {@link cleanExtensionsBuildTask}, {@link compileNonNativeExtensionsBuildTask} and {@link compileNativeExtensionsBuildTask}
+ */
+const compileAllExtensionsBuildTask = task.define('compile-extensions-build', task.series(
 	cleanExtensionsBuildTask,
-	task.define('bundle-marketplace-extensions-build', () => ext.packageMarketplaceExtensionsStream(false).pipe(gulp.dest('.build'))),
-	task.define('bundle-extensions-build', () => ext.packageLocalExtensionsStream(false, false).pipe(gulp.dest('.build'))),
+	bundleMarketplaceExtensionsBuildTask,
+	task.define('bundle-extensions-build', () => ext.packageAllLocalExtensionsStream(false, false).pipe(gulp.dest('.build'))),
 ));
+gulp.task(compileAllExtensionsBuildTask);
+exports.compileAllExtensionsBuildTask = compileAllExtensionsBuildTask;
 
-gulp.task(compileExtensionsBuildTask);
-gulp.task(task.define('extensions-ci', task.series(compileExtensionsBuildTask, compileExtensionMediaBuildTask)));
+// This task is run in the compilation stage of the CI pipeline. We only compile the non-native extensions since those can be fully built regardless of platform.
+// This defers the native extensions to the platform specific stage of the CI pipeline.
+gulp.task(task.define('extensions-ci', task.series(compileNonNativeExtensionsBuildTask, compileExtensionMediaBuildTask)));
 
 const compileExtensionsBuildPullRequestTask = task.define('compile-extensions-build-pr', task.series(
 	cleanExtensionsBuildTask,
-	task.define('bundle-marketplace-extensions-build', () => ext.packageMarketplaceExtensionsStream(false).pipe(gulp.dest('.build'))),
-	task.define('bundle-extensions-build-pr', () => ext.packageLocalExtensionsStream(false, true).pipe(gulp.dest('.build'))),
+	bundleMarketplaceExtensionsBuildTask,
+	task.define('bundle-extensions-build-pr', () => ext.packageAllLocalExtensionsStream(false, true).pipe(gulp.dest('.build'))),
 ));
-
 gulp.task(compileExtensionsBuildPullRequestTask);
-gulp.task(task.define('extensions-ci-pr', task.series(compileExtensionsBuildPullRequestTask, compileExtensionMediaBuildTask)));
 
-
-exports.compileExtensionsBuildTask = compileExtensionsBuildTask;
+// This task is run in the compilation stage of the PR pipeline. We compile all extensions in it to verify compilation.
+gulp.task(task.define('extensions-ci-pr', task.series(compileExtensionsBuildPullRequestTask, compileExtensionMediaBuildTask)));
 
 //#endregion
 

build/gulpfile.reh.js

@@ -27,7 +27,7 @@ const File = require('vinyl');
 const fs = require('fs');
 const glob = require('glob');
 const { compileBuildTask } = require('./gulpfile.compile');
-const { compileExtensionsBuildTask, compileExtensionMediaBuildTask } = require('./gulpfile.extensions');
+const { cleanExtensionsBuildTask, compileNonNativeExtensionsBuildTask, compileNativeExtensionsBuildTask, compileExtensionMediaBuildTask } = require('./gulpfile.extensions');
 const { vscodeWebResourceIncludes, createVSCodeWebFileContentMapper } = require('./gulpfile.vscode.web');
 const cp = require('child_process');
 const log = require('fancy-log');
@@ -468,6 +468,7 @@ function tweakProductForServerWeb(product) {
 			const destinationFolderName = `vscode-${type}${dashed(platform)}${dashed(arch)}`;
 
 			const serverTaskCI = task.define(`vscode-${type}${dashed(platform)}${dashed(arch)}${dashed(minified)}-ci`, task.series(
+				compileNativeExtensionsBuildTask,
 				gulp.task(`node-${platform}-${arch}`),
 				util.rimraf(path.join(BUILD_ROOT, destinationFolderName)),
 				packageTask(type, platform, arch, sourceFolderName, destinationFolderName)
@@ -476,7 +477,8 @@ function tweakProductForServerWeb(product) {
 
 			const serverTask = task.define(`vscode-${type}${dashed(platform)}${dashed(arch)}${dashed(minified)}`, task.series(
 				compileBuildTask,
-				compileExtensionsBuildTask,
+				cleanExtensionsBuildTask,
+				compileNonNativeExtensionsBuildTask,
 				compileExtensionMediaBuildTask,
 				minified ? minifyTask : bundleTask,
 				serverTaskCI

build/gulpfile.vscode.js

@@ -31,7 +31,7 @@ const { config } = require('./lib/electron');
 const createAsar = require('./lib/asar').createAsar;
 const minimist = require('minimist');
 const { compileBuildTask } = require('./gulpfile.compile');
-const { compileExtensionsBuildTask, compileExtensionMediaBuildTask } = require('./gulpfile.extensions');
+const { compileNonNativeExtensionsBuildTask, compileNativeExtensionsBuildTask, compileAllExtensionsBuildTask, compileExtensionMediaBuildTask, cleanExtensionsBuildTask } = require('./gulpfile.extensions');
 const { promisify } = require('util');
 const glob = promisify(require('glob'));
 const rcedit = promisify(require('rcedit'));
@@ -487,6 +487,7 @@ BUILD_TARGETS.forEach(buildTarget => {
 		const destinationFolderName = `VSCode${dashed(platform)}${dashed(arch)}`;
 
 		const tasks = [
+			compileNativeExtensionsBuildTask,
 			util.rimraf(path.join(buildRoot, destinationFolderName)),
 			packageTask(platform, arch, sourceFolderName, destinationFolderName, opts)
 		];
@@ -500,7 +501,8 @@ BUILD_TARGETS.forEach(buildTarget => {
 
 		const vscodeTask = task.define(`vscode${dashed(platform)}${dashed(arch)}${dashed(minified)}`, task.series(
 			compileBuildTask,
-			compileExtensionsBuildTask,
+			cleanExtensionsBuildTask,
+			compileNonNativeExtensionsBuildTask,
 			compileExtensionMediaBuildTask,
 			minified ? minifyVSCodeTask : bundleVSCodeTask,
 			vscodeTaskCI
@@ -537,7 +539,7 @@ gulp.task(task.define(
 	'vscode-translations-export',
 	task.series(
 		core,
-		compileExtensionsBuildTask,
+		compileAllExtensionsBuildTask,
 		function () {
 			const pathToMetadata = './out-build/nls.metadata.json';
 			const pathToExtensions = '.build/extensions/*';

build/gulpfile.vscode.web.js

@@ -202,7 +202,7 @@ function packageTask(sourceFolderName, destinationFolderName) {
 
 const compileWebExtensionsBuildTask = task.define('compile-web-extensions-build', task.series(
 	task.define('clean-web-extensions-build', util.rimraf('.build/web/extensions')),
-	task.define('bundle-web-extensions-build', () => extensions.packageLocalExtensionsStream(true, false).pipe(gulp.dest('.build/web'))),
+	task.define('bundle-web-extensions-build', () => extensions.packageAllLocalExtensionsStream(true, false).pipe(gulp.dest('.build/web'))),
 	task.define('bundle-marketplace-web-extensions-build', () => extensions.packageMarketplaceExtensionsStream(true).pipe(gulp.dest('.build/web'))),
 	task.define('bundle-web-extension-media-build', () => extensions.buildExtensionMedia(false, '.build/web/extensions')),
 ));

build/lib/extensions.ts

@@ -277,6 +277,14 @@ export function fromGithub({ name, version, repo, sha256, metadata }: IExtension
 		.pipe(packageJsonFilter.restore);
 }
 
+/**
+ * All extensions that are known to have some native component and thus must be built on the
+ * platform that is being built.
+ */
+const nativeExtensions = [
+	'microsoft-authentication',
+];
+
 const excludedExtensions = [
 	'vscode-api-tests',
 	'vscode-colorize-tests',
@@ -334,7 +342,49 @@ function isWebExtension(manifest: IExtensionManifest): boolean {
 	return true;
 }
 
-export function packageLocalExtensionsStream(forWeb: boolean, disableMangle: boolean): Stream {
+/**
+ * Package local extensions that are known to not have native dependencies. Mutually exclusive to {@link packageNativeLocalExtensionsStream}.
+ * @param forWeb build the extensions that have web targets
+ * @param disableMangle disable the mangler
+ * @returns a stream
+ */
+export function packageNonNativeLocalExtensionsStream(forWeb: boolean, disableMangle: boolean): Stream {
+	return doPackageLocalExtensionsStream(forWeb, disableMangle, false);
+}
+
+/**
+ * Package local extensions that are known to have native dependencies. Mutually exclusive to {@link packageNonNativeLocalExtensionsStream}.
+ * @note it's possible that the extension does not have native dependencies for the current platform, especially if building for the web,
+ * but we simplify the logic here by having a flat list of extensions (See {@link nativeExtensions}) that are known to have native
+ * dependencies on some platform and thus should be packaged on the platform that they are building for.
+ * @param forWeb build the extensions that have web targets
+ * @param disableMangle disable the mangler
+ * @returns a stream
+ */
+export function packageNativeLocalExtensionsStream(forWeb: boolean, disableMangle: boolean): Stream {
+	return doPackageLocalExtensionsStream(forWeb, disableMangle, true);
+}
+
+/**
+ * Package all the local extensions... both those that are known to have native dependencies and those that are not.
+ * @param forWeb build the extensions that have web targets
+ * @param disableMangle disable the mangler
+ * @returns a stream
+ */
+export function packageAllLocalExtensionsStream(forWeb: boolean, disableMangle: boolean): Stream {
+	return es.merge([
+		packageNonNativeLocalExtensionsStream(forWeb, disableMangle),
+		packageNativeLocalExtensionsStream(forWeb, disableMangle)
+	]);
+}
+
+/**
+ * @param forWeb build the extensions that have web targets
+ * @param disableMangle disable the mangler
+ * @param native build the extensions that are marked as having native dependencies
+ */
+function doPackageLocalExtensionsStream(forWeb: boolean, disableMangle: boolean, native: boolean): Stream {
+	const nativeExtensionsSet = new Set(nativeExtensions);
 	const localExtensionsDescriptions = (
 		(<string[]>glob.sync('extensions/*/package.json'))
 			.map(manifestPath => {
@@ -343,6 +393,7 @@ export function packageLocalExtensionsStream(forWeb: boolean, disableMangle: boo
 				const extensionName = path.basename(extensionPath);
 				return { name: extensionName, path: extensionPath, manifestPath: absoluteManifestPath };
 			})
+			.filter(({ name }) => native ? nativeExtensionsSet.has(name) : !nativeExtensionsSet.has(name))
 			.filter(({ name }) => excludedExtensions.indexOf(name) === -1)
 			.filter(({ name }) => builtInExtensions.every(b => b.name !== name))
 			.filter(({ manifestPath }) => (forWeb ? isWebExtension(require(manifestPath)) : true))

extensions/microsoft-authentication/.vscodeignore

@@ -12,3 +12,4 @@ vsc-extension-quickstart.md
 **/tslint.json
 **/*.map
 **/*.ts
+packageMocks/

extensions/microsoft-authentication/extension.webpack.config.js

@@ -8,10 +8,42 @@
 'use strict';
 
 const withDefaults = require('../shared.webpack.config');
+const CopyWebpackPlugin = require('copy-webpack-plugin');
+const path = require('path');
+const { NormalModuleReplacementPlugin } = require('webpack');
+
+const isWindows = process.platform === 'win32';
 
 module.exports = withDefaults({
 	context: __dirname,
 	entry: {
 		extension: './src/extension.ts'
-	}
+	},
+	externals: {
+		// The @azure/msal-node-runtime package requires this native node module (.node).
+		// It is currently only included on Windows, but the package handles unsupported platforms
+		// gracefully.
+		'./msal-node-runtime': 'commonjs ./msal-node-runtime'
+	},
+	plugins: [
+		...withDefaults.nodePlugins(__dirname),
+		new CopyWebpackPlugin({
+			patterns: [
+				{
+					// The native files we need to ship with the extension
+					from: '**/dist/msal*.(node|dll)',
+					to: '[name][ext]',
+					// These will only be present on Windows for now
+					noErrorOnMissing: !isWindows
+				}
+			]
+		}),
+		// We don't use the feature that uses Dpapi, so we can just replace it with a mock.
+		// This is a bit of a hack, but it's the easiest way to do it. Really, msal should
+		// handle when this native node module is not available.
+		new NormalModuleReplacementPlugin(
+			/\.\.\/Dpapi\.mjs/,
+			path.resolve(__dirname, 'packageMocks', 'dpapi', 'dpapi.js')
+		)
+	]
 });