From 26241cea29986e591ce47c1f8874a7afe598a46a Mon Sep 17 00:00:00 2001 From: Joe Hanley Date: Thu, 10 Oct 2024 15:49:32 -0700 Subject: [PATCH 01/24] Added config --- src/firebaseConfig.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/firebaseConfig.ts b/src/firebaseConfig.ts index 3fc4fb855a6..03118aaed61 100644 --- a/src/firebaseConfig.ts +++ b/src/firebaseConfig.ts @@ -239,6 +239,7 @@ export type EmulatorsConfig = { port?: number; postgresHost?: string; postgresPort?: number; + postgresDataDirectory?: string; }; tasks?: { host?: string; From 6e03ec1da4dd61c8279eb9604b559a0108350de5 Mon Sep 17 00:00:00 2001 From: Joe Hanley Date: Tue, 15 Oct 2024 10:24:41 -0700 Subject: [PATCH 02/24] Adding basic persistence support --- src/emulator/dataconnect/pgliteServer.ts | 8 +++++--- src/emulator/dataconnectEmulator.ts | 4 +++- src/firebaseConfig.ts | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/emulator/dataconnect/pgliteServer.ts b/src/emulator/dataconnect/pgliteServer.ts index f4c7b786a92..7f659a187ab 100644 --- a/src/emulator/dataconnect/pgliteServer.ts +++ b/src/emulator/dataconnect/pgliteServer.ts @@ -19,6 +19,7 @@ import { logger } from "../../logger"; export class PostgresServer { private username: string; private database: string; + private dataDirectory?: string; public db: PGlite | undefined; public async createPGServer(host: string = "127.0.0.1", port: number): Promise { @@ -77,14 +78,15 @@ export class PostgresServer { uuidOssp, }, // TODO: Use dataDir + loadDataDir to implement import/export. - // dataDir?: string; - // loadDataDir?: Blob | File; + dataDir: this.dataDirectory, + // loadDataDir?: Blob | File; // This will be used with .dumpDataDir() for import/export }); } - constructor(database: string, username: string) { + constructor(database: string, username: string, dataDirectory?: string) { this.username = username; this.database = database; + this.dataDirectory = dataDirectory; } } diff --git a/src/emulator/dataconnectEmulator.ts b/src/emulator/dataconnectEmulator.ts index c1acbfc66b1..2144355688d 100644 --- a/src/emulator/dataconnectEmulator.ts +++ b/src/emulator/dataconnectEmulator.ts @@ -101,7 +101,8 @@ export class DataConnectEmulator implements EmulatorInstance { `FIREBASE_DATACONNECT_POSTGRESQL_STRING is set to ${dataConnectLocalConnString()} - using that instead of starting a new database`, ); } else if (pgHost && pgPort) { - const pgServer = new PostgresServer(dbId, "postgres"); + const dataDirectory = this.args.config.get("emulators.dataconnect.dataDirectory"); + const pgServer = new PostgresServer(dbId, "postgres", dataDirectory); const server = await pgServer.createPGServer(pgHost, pgPort); const connectableHost = connectableHostname(pgHost); connStr = `postgres://${connectableHost}:${pgPort}/${dbId}?sslmode=disable`; @@ -109,6 +110,7 @@ export class DataConnectEmulator implements EmulatorInstance { if (err instanceof FirebaseError) { this.logger.logLabeled("ERROR", "Data Connect", `${err}`); } else { + console this.logger.logLabeled( "ERROR", "Data Connect", diff --git a/src/firebaseConfig.ts b/src/firebaseConfig.ts index 89e3d92de5c..9a9d674df1e 100644 --- a/src/firebaseConfig.ts +++ b/src/firebaseConfig.ts @@ -240,7 +240,7 @@ export type EmulatorsConfig = { port?: number; postgresHost?: string; postgresPort?: number; - postgresDataDirectory?: string; + dataDirectory?: string; }; tasks?: { host?: string; From 6c9834d0b23909dfe10f73dee489459274b52650 Mon Sep 17 00:00:00 2001 From: Joe Hanley Date: Tue, 15 Oct 2024 11:29:39 -0700 Subject: [PATCH 03/24] formats --- src/emulator/dataconnect/pgliteServer.ts | 2 +- src/emulator/dataconnectEmulator.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/emulator/dataconnect/pgliteServer.ts b/src/emulator/dataconnect/pgliteServer.ts index 7f659a187ab..897f8139041 100644 --- a/src/emulator/dataconnect/pgliteServer.ts +++ b/src/emulator/dataconnect/pgliteServer.ts @@ -78,7 +78,7 @@ export class PostgresServer { uuidOssp, }, // TODO: Use dataDir + loadDataDir to implement import/export. - dataDir: this.dataDirectory, + dataDir: this.dataDirectory, // loadDataDir?: Blob | File; // This will be used with .dumpDataDir() for import/export }); } diff --git a/src/emulator/dataconnectEmulator.ts b/src/emulator/dataconnectEmulator.ts index 2144355688d..839a10a13d8 100644 --- a/src/emulator/dataconnectEmulator.ts +++ b/src/emulator/dataconnectEmulator.ts @@ -110,7 +110,7 @@ export class DataConnectEmulator implements EmulatorInstance { if (err instanceof FirebaseError) { this.logger.logLabeled("ERROR", "Data Connect", `${err}`); } else { - console + console; this.logger.logLabeled( "ERROR", "Data Connect", From 31560d96fde2229cb8cda417b97b1e4231cf24ac Mon Sep 17 00:00:00 2001 From: Joe Hanley Date: Mon, 28 Oct 2024 14:45:39 -0700 Subject: [PATCH 04/24] Starting on clearData --- src/emulator/dataconnect/pgliteServer.ts | 58 +++++++++++++++--------- src/emulator/dataconnectEmulator.ts | 13 ++++-- src/emulator/hub.ts | 24 ++++++++++ 3 files changed, 70 insertions(+), 25 deletions(-) diff --git a/src/emulator/dataconnect/pgliteServer.ts b/src/emulator/dataconnect/pgliteServer.ts index 897f8139041..2072e99f82f 100644 --- a/src/emulator/dataconnect/pgliteServer.ts +++ b/src/emulator/dataconnect/pgliteServer.ts @@ -7,6 +7,8 @@ import { PGlite } from "@electric-sql/pglite"; // during module resolution. const { dynamicImport } = require(true && "../../dynamicImport"); import * as net from "node:net"; +import { rmSync } from "node:fs"; + import { getMessages, type PostgresConnection, @@ -21,10 +23,11 @@ export class PostgresServer { private database: string; private dataDirectory?: string; - public db: PGlite | undefined; + public db: PGlite | undefined = undefined; public async createPGServer(host: string = "127.0.0.1", port: number): Promise { - const db: PGlite = await this.getDb(); - await db.waitReady; + const getDb: () => Promise = this.getDb; + await getDb(); + const server = net.createServer(async (socket) => { const connection: PostgresConnection = await fromNodeSocket(socket, { serverVersion: "16.3 (PGlite 0.2.0)", @@ -35,6 +38,7 @@ export class PostgresServer { if (!isAuthenticated) { return; } + const db = await getDb(); const result = await db.execProtocolRaw(data); // Extended query patch removes the extra Ready for Query messages that // pglite wrongly sends. @@ -51,36 +55,46 @@ export class PostgresServer { server.emit("error", err); }); }); + const listeningPromise = new Promise((resolve) => { server.listen(port, host, () => { resolve(); }); }); - await db.waitReady; await listeningPromise; return server; } async getDb(): Promise { - if (this.db) { - return this.db; + if (!this.db) { + // Not all schemas will need vector installed, but we don't have an good way + // to swap extensions after starting PGLite, so we always include it. + const vector = (await dynamicImport("@electric-sql/pglite/vector")).vector; + const uuidOssp = (await dynamicImport("@electric-sql/pglite/contrib/uuid_ossp")).uuid_ossp; + this.db = await PGlite.create({ + username: this.username, + database: this.database, + debug: 0, + extensions: { + vector, + uuidOssp, + }, + // TODO: Use dataDir + loadDataDir to implement import/export. + dataDir: this.dataDirectory, + // loadDataDir?: Blob | File; // This will be used with .dumpDataDir() for import/export + }); + // await this.db.waitReady; } - // Not all schemas will need vector installed, but we don't have an good way - // to swap extensions after starting PGLite, so we always include it. - const vector = (await dynamicImport("@electric-sql/pglite/vector")).vector; - const uuidOssp = (await dynamicImport("@electric-sql/pglite/contrib/uuid_ossp")).uuid_ossp; - return PGlite.create({ - username: this.username, - database: this.database, - debug: 0, - extensions: { - vector, - uuidOssp, - }, - // TODO: Use dataDir + loadDataDir to implement import/export. - dataDir: this.dataDirectory, - // loadDataDir?: Blob | File; // This will be used with .dumpDataDir() for import/export - }); + return this.db; + } + + public async clearDb(): Promise { + if (this.dataDirectory) { + rmSync(this.dataDirectory, { recursive: true }); + } + await this.db?.close(); + this.db = undefined; + return this.getDb(); } constructor(database: string, username: string, dataDirectory?: string) { diff --git a/src/emulator/dataconnectEmulator.ts b/src/emulator/dataconnectEmulator.ts index 839a10a13d8..a43ee8c41a0 100644 --- a/src/emulator/dataconnectEmulator.ts +++ b/src/emulator/dataconnectEmulator.ts @@ -48,6 +48,7 @@ export const dataConnectEmulatorEvents = new EventEmitter(); export class DataConnectEmulator implements EmulatorInstance { private emulatorClient: DataConnectEmulatorClient; private usingExistingEmulator: boolean = false; + private postgresServer: PostgresServer | undefined; constructor(private args: DataConnectEmulatorArgs) { this.emulatorClient = new DataConnectEmulatorClient(); @@ -102,11 +103,11 @@ export class DataConnectEmulator implements EmulatorInstance { ); } else if (pgHost && pgPort) { const dataDirectory = this.args.config.get("emulators.dataconnect.dataDirectory"); - const pgServer = new PostgresServer(dbId, "postgres", dataDirectory); - const server = await pgServer.createPGServer(pgHost, pgPort); + this.postgresServer = new PostgresServer(dbId, "postgres", dataDirectory); + const server = await this.postgresServer.createPGServer(pgHost, pgPort); const connectableHost = connectableHostname(pgHost); connStr = `postgres://${connectableHost}:${pgPort}/${dbId}?sslmode=disable`; - server.on("error", (err) => { + server.on("error", (err: any) => { if (err instanceof FirebaseError) { this.logger.logLabeled("ERROR", "Data Connect", `${err}`); } else { @@ -171,6 +172,12 @@ export class DataConnectEmulator implements EmulatorInstance { return Emulators.DATACONNECT; } + async clearData(): Promise { + if (this.postgresServer) { + await this.postgresServer.clearDb(); + } + } + static async generate(args: DataConnectGenerateArgs): Promise { const commandInfo = await downloadIfNecessary(Emulators.DATACONNECT); const cmd = [ diff --git a/src/emulator/hub.ts b/src/emulator/hub.ts index 792aee63a1b..8ddf1656513 100644 --- a/src/emulator/hub.ts +++ b/src/emulator/hub.ts @@ -12,6 +12,7 @@ import { FunctionsEmulator } from "./functionsEmulator"; import { ExpressBasedEmulator } from "./ExpressBasedEmulator"; import { PortName } from "./portUtils"; import { isVSCodeExtension } from "../utils"; +import { DataConnectEmulator } from "./dataconnectEmulator"; // We use the CLI version from package.json const pkg = require("../../package.json"); @@ -36,6 +37,7 @@ export class EmulatorHub extends ExpressBasedEmulator { static PATH_DISABLE_FUNCTIONS = "/functions/disableBackgroundTriggers"; static PATH_ENABLE_FUNCTIONS = "/functions/enableBackgroundTriggers"; static PATH_EMULATORS = "/emulators"; + static PATH_CLEAR_DATA_CONNECT = "/dataconnect/clearData"; /** * Given a project ID, find and read the Locator file for the emulator hub. @@ -160,6 +162,28 @@ export class EmulatorHub extends ExpressBasedEmulator { res.status(200).json({ enabled: true }); }); + app.post(EmulatorHub.PATH_CLEAR_DATA_CONNECT, async (req, res) => { + // TODO: Sanity check that this is needed. + // if (req.headers.origin) { + // res.status(403).json({ + // message: `Clear Data Connect cannot be triggered by external callers.`, + // }); + // } + utils.logLabeledBullet( + "emulators", + `Clearing data from Data Connect data sources.`, + ); + + const instance = EmulatorRegistry.get(Emulators.DATACONNECT) as DataConnectEmulator; + if (!instance) { + res.status(400).json({ error: "The Data Connect emulator is not running." }); + return; + } + + await instance.clearData(); + + }); + return app; } From f510a73b7307f718a9f8d0cbf79d3a883b667c5f Mon Sep 17 00:00:00 2001 From: Joe Hanley Date: Thu, 14 Nov 2024 11:40:09 -0800 Subject: [PATCH 05/24] Clear data now working! --- src/emulator/dataconnect/pgliteServer.ts | 31 +++++++++++++++++------- src/emulator/hub.ts | 7 ++---- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/src/emulator/dataconnect/pgliteServer.ts b/src/emulator/dataconnect/pgliteServer.ts index 2072e99f82f..4d4af6f5441 100644 --- a/src/emulator/dataconnect/pgliteServer.ts +++ b/src/emulator/dataconnect/pgliteServer.ts @@ -7,7 +7,6 @@ import { PGlite } from "@electric-sql/pglite"; // during module resolution. const { dynamicImport } = require(true && "../../dynamicImport"); import * as net from "node:net"; -import { rmSync } from "node:fs"; import { getMessages, @@ -25,7 +24,9 @@ export class PostgresServer { public db: PGlite | undefined = undefined; public async createPGServer(host: string = "127.0.0.1", port: number): Promise { - const getDb: () => Promise = this.getDb; + const getDb: () => Promise = () => { + return this.getDb(); + }; await getDb(); const server = net.createServer(async (socket) => { @@ -88,13 +89,25 @@ export class PostgresServer { return this.db; } - public async clearDb(): Promise { - if (this.dataDirectory) { - rmSync(this.dataDirectory, { recursive: true }); - } - await this.db?.close(); - this.db = undefined; - return this.getDb(); + public async clearDb(): Promise { + // if (this.dataDirectory) { + // rmSync(this.dataDirectory, { recursive: true }); + // } + // await this.db?.close(); + // this.db = undefined; + // return this.getDb(); + const db = await this.getDb(); + await db.query(` +DO $do$ +BEGIN + EXECUTE + (SELECT 'TRUNCATE TABLE ' || string_agg(oid::regclass::text, ', ') || ' CASCADE' + FROM pg_class + WHERE relkind = 'r' + AND relnamespace = 'public'::regnamespace + ); +END +$do$;`); } constructor(database: string, username: string, dataDirectory?: string) { diff --git a/src/emulator/hub.ts b/src/emulator/hub.ts index 62b9c3fb4e8..a95f009295e 100644 --- a/src/emulator/hub.ts +++ b/src/emulator/hub.ts @@ -173,10 +173,7 @@ export class EmulatorHub extends ExpressBasedEmulator { // message: `Clear Data Connect cannot be triggered by external callers.`, // }); // } - utils.logLabeledBullet( - "emulators", - `Clearing data from Data Connect data sources.`, - ); + utils.logLabeledBullet("emulators", `Clearing data from Data Connect data sources.`); const instance = EmulatorRegistry.get(Emulators.DATACONNECT) as DataConnectEmulator; if (!instance) { @@ -185,7 +182,7 @@ export class EmulatorHub extends ExpressBasedEmulator { } await instance.clearData(); - + res.status(200).send("Data cleared"); }); return app; From 5d046cf0f6032a65ef80a6790833a952f753ca9e Mon Sep 17 00:00:00 2001 From: Joe Hanley Date: Thu, 14 Nov 2024 17:42:29 -0800 Subject: [PATCH 06/24] Added import/export support --- src/emulator/controller.ts | 32 ++++++++++++++++++---- src/emulator/dataconnect/pgliteServer.ts | 27 +++++++++++++++---- src/emulator/dataconnectEmulator.ts | 18 ++++++++++++- src/emulator/hubExport.ts | 34 ++++++++++++++++++++++++ src/emulator/types.ts | 1 + 5 files changed, 101 insertions(+), 11 deletions(-) diff --git a/src/emulator/controller.ts b/src/emulator/controller.ts index 130b7e7e4b9..0a78473ac58 100755 --- a/src/emulator/controller.ts +++ b/src/emulator/controller.ts @@ -51,7 +51,7 @@ import { Runtime, isRuntime } from "../deploy/functions/runtimes/supported"; import { AuthEmulator, SingleProjectMode } from "./auth"; import { DatabaseEmulator, DatabaseEmulatorArgs } from "./databaseEmulator"; import { EventarcEmulator } from "./eventarcEmulator"; -import { DataConnectEmulator } from "./dataconnectEmulator"; +import { DataConnectEmulator, DataConnectEmulatorArgs } from "./dataconnectEmulator"; import { FirestoreEmulator, FirestoreEmulatorArgs } from "./firestoreEmulator"; import { HostingEmulator } from "./hostingEmulator"; import { PubsubEmulator } from "./pubsubEmulator"; @@ -871,19 +871,41 @@ export async function startAll( `TODO: Add support for multiple services in the Data Connect emulator. Currently emulating first service ${config[0].source}`, ); } - const configDir = config[0].source; - const dataConnectEmulator = new DataConnectEmulator({ + + const args: DataConnectEmulatorArgs = { listen: listenForEmulator.dataconnect, projectId, auto_download: true, - configDir, + configDir: config[0].source, rc: options.rc, config: options.config, autoconnectToPostgres: true, postgresListen: listenForEmulator["dataconnect.postgres"], enable_output_generated_sdk: true, // TODO: source from arguments enable_output_schema_extensions: true, - }); + }; + + if (exportMetadata.dataconnect) { + utils.assertIsString(options.import); + const importDirAbsPath = path.resolve(options.import); + const exportMetadataFilePath = path.resolve( + importDirAbsPath, + exportMetadata.dataconnect.path, + ); + + EmulatorLogger.forEmulator(Emulators.DATACONNECT).logLabeled( + "BULLET", + "dataconnect", + `Importing data from ${exportMetadataFilePath}`, + ); + args.importPath = exportMetadataFilePath; + void trackEmulator("emulator_import", { + initiated_by: "start", + emulator_name: Emulators.DATACONNECT, + }); + } + + const dataConnectEmulator = new DataConnectEmulator(args); await startEmulator(dataConnectEmulator); } diff --git a/src/emulator/dataconnect/pgliteServer.ts b/src/emulator/dataconnect/pgliteServer.ts index 4d4af6f5441..e4cf1071824 100644 --- a/src/emulator/dataconnect/pgliteServer.ts +++ b/src/emulator/dataconnect/pgliteServer.ts @@ -1,12 +1,13 @@ // https://github.com/supabase-community/pg-gateway -import { PGlite } from "@electric-sql/pglite"; +import { PGlite, PGliteOptions } from "@electric-sql/pglite"; // Unfortunately, we need to dynamically import the Postgres extensions. // They are only available as ESM, and if we import them normally, // our tsconfig will convert them to requires, which will cause errors // during module resolution. const { dynamicImport } = require(true && "../../dynamicImport"); import * as net from "node:net"; +import * as fs from "fs"; import { getMessages, @@ -21,6 +22,7 @@ export class PostgresServer { private username: string; private database: string; private dataDirectory?: string; + private importPath?: string; public db: PGlite | undefined = undefined; public async createPGServer(host: string = "127.0.0.1", port: number): Promise { @@ -72,7 +74,7 @@ export class PostgresServer { // to swap extensions after starting PGLite, so we always include it. const vector = (await dynamicImport("@electric-sql/pglite/vector")).vector; const uuidOssp = (await dynamicImport("@electric-sql/pglite/contrib/uuid_ossp")).uuid_ossp; - this.db = await PGlite.create({ + const pgliteArgs: PGliteOptions = { username: this.username, database: this.database, debug: 0, @@ -83,8 +85,15 @@ export class PostgresServer { // TODO: Use dataDir + loadDataDir to implement import/export. dataDir: this.dataDirectory, // loadDataDir?: Blob | File; // This will be used with .dumpDataDir() for import/export - }); - // await this.db.waitReady; + }; + if (this.importPath) { + logger.debug(`Importing from ${this.importPath}`); + const rf = fs.readFileSync(this.importPath); + const file = new File([rf.buffer], this.importPath); + pgliteArgs.loadDataDir = file; + } + this.db = await PGlite.create(pgliteArgs); + await this.db.waitReady; } return this.db; } @@ -110,10 +119,18 @@ END $do$;`); } - constructor(database: string, username: string, dataDirectory?: string) { + public async exportData(exportPath: string): Promise { + const db = await this.getDb(); + const dump = await db.dumpDataDir(); + const arrayBuff = await dump.arrayBuffer(); + fs.writeFileSync(exportPath, new Uint8Array(arrayBuff)); + } + + constructor(database: string, username: string, dataDirectory?: string, importPath?: string) { this.username = username; this.database = database; this.dataDirectory = dataDirectory; + this.importPath = importPath; } } diff --git a/src/emulator/dataconnectEmulator.ts b/src/emulator/dataconnectEmulator.ts index f2b5d240735..ec22fb8d74f 100644 --- a/src/emulator/dataconnectEmulator.ts +++ b/src/emulator/dataconnectEmulator.ts @@ -31,6 +31,7 @@ export interface DataConnectEmulatorArgs { postgresListen?: ListenSpec[]; enable_output_schema_extensions: boolean; enable_output_generated_sdk: boolean; + importPath?: string; } export interface DataConnectGenerateArgs { @@ -104,7 +105,12 @@ export class DataConnectEmulator implements EmulatorInstance { ); } else if (pgHost && pgPort) { const dataDirectory = this.args.config.get("emulators.dataconnect.dataDirectory"); - this.postgresServer = new PostgresServer(dbId, "postgres", dataDirectory); + this.postgresServer = new PostgresServer( + dbId, + "postgres", + dataDirectory, + this.args.importPath, + ); const server = await this.postgresServer.createPGServer(pgHost, pgPort); const connectableHost = connectableHostname(pgHost); connStr = `postgres://${connectableHost}:${pgPort}/${dbId}?sslmode=disable`; @@ -179,6 +185,16 @@ export class DataConnectEmulator implements EmulatorInstance { } } + async exportData(path: string): Promise { + if (this.postgresServer) { + await this.postgresServer.exportData(path); + } else { + throw new FirebaseError( + "The Data Connect emulator is currently connected to a separate Postgres instance. Export is not supported.", + ); + } + } + static async generate(args: DataConnectGenerateArgs): Promise { const commandInfo = await downloadIfNecessary(Emulators.DATACONNECT); const cmd = [ diff --git a/src/emulator/hubExport.ts b/src/emulator/hubExport.ts index d9c0fd1b8fa..d975c11b936 100644 --- a/src/emulator/hubExport.ts +++ b/src/emulator/hubExport.ts @@ -10,6 +10,7 @@ import { FirebaseError } from "../error"; import { EmulatorHub } from "./hub"; import { getDownloadDetails } from "./downloadableEmulators"; import { DatabaseEmulator } from "./databaseEmulator"; +import { DataConnectEmulator } from "./dataconnectEmulator"; import { rmSync } from "node:fs"; import { trackEmulator } from "../track"; @@ -34,12 +35,18 @@ export interface StorageExportMetadata { path: string; } +export interface DataConnectExportMetadata { + version: string; + path: string; +} + export interface ExportMetadata { version: string; firestore?: FirestoreExportMetadata; database?: DatabaseExportMetadata; auth?: AuthExportMetadata; storage?: StorageExportMetadata; + dataconnect?: DataConnectExportMetadata; } export interface ExportOptions { @@ -122,6 +129,14 @@ export class HubExport { await this.exportStorage(metadata); } + if (shouldExport(Emulators.DATACONNECT)) { + metadata.dataconnect = { + version: EmulatorHub.CLI_VERSION, + path: "dataconnect_export.tar.gz", + }; + await this.exportDataConnect(metadata); + } + // Make sure the export directory exists if (!fs.existsSync(this.exportPath)) { fs.mkdirSync(this.exportPath); @@ -289,6 +304,25 @@ export class HubExport { throw new FirebaseError(`Failed to export storage: ${await res.response.text()}`); } } + + private async exportDataConnect(metadata: ExportMetadata): Promise { + void trackEmulator("emulator_export", { + initiated_by: this.options.initiatedBy, + emulator_name: Emulators.DATACONNECT, + }); + const dataconnectExportPath = path.join(this.tmpDir, metadata.dataconnect!.path); + if (fs.existsSync(dataconnectExportPath)) { + fse.removeSync(dataconnectExportPath); + } + + const instance = EmulatorRegistry.get(Emulators.DATACONNECT) as DataConnectEmulator; + if (!instance) { + throw new FirebaseError( + "Unable to export Data Connect emulator data: the Data Connect emulator is not running.", + ); + } + await instance.exportData(dataconnectExportPath); + } } function fetchToFile(options: http.RequestOptions, path: fs.PathLike): Promise { diff --git a/src/emulator/types.ts b/src/emulator/types.ts index 2e519fcdfdc..8492026cf85 100644 --- a/src/emulator/types.ts +++ b/src/emulator/types.ts @@ -43,6 +43,7 @@ export const IMPORT_EXPORT_EMULATORS = [ Emulators.DATABASE, Emulators.AUTH, Emulators.STORAGE, + Emulators.DATACONNECT, ]; export const ALL_SERVICE_EMULATORS = [ From 1a13dfa84b000825d3ada06bf4cbf8f75cbe4723 Mon Sep 17 00:00:00 2001 From: Joe Hanley Date: Fri, 15 Nov 2024 15:17:42 -0800 Subject: [PATCH 07/24] PR fixes + fidelity to approved API' --- CHANGELOG.md | 2 ++ src/emulator/controller.ts | 9 +++++---- src/emulator/dataconnect/pgliteServer.ts | 13 +------------ src/emulator/dataconnectEmulator.ts | 18 ++++++++---------- src/emulator/hub.ts | 11 +++++------ src/emulator/hubExport.ts | 13 ++++++++----- src/firebaseConfig.ts | 2 +- 7 files changed, 30 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8372252acd5..993b0d981ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,3 +2,5 @@ - Moved firebase-tools-ui server.js logic to fireabse-tools to run it in-memory. (#7897) - Updates `superstatic` to `9.1.0` (#7929). - Added the appdistribution:group:list and appdistribution:testers:list commands. +- Added `--import` and `emulators:export` support to the Data Connect emulator. +- Added `firebase.json#emulators.dataconnect.dataDir`. When set, Data Connect data will be persisted to the configured directory between emulator runs. diff --git a/src/emulator/controller.ts b/src/emulator/controller.ts index 0a78473ac58..350ff90f8b8 100755 --- a/src/emulator/controller.ts +++ b/src/emulator/controller.ts @@ -72,8 +72,9 @@ const START_LOGGING_EMULATOR = utils.envOverride( * Exports emulator data on clean exit (SIGINT or process end) * @param options */ -export async function exportOnExit(options: any) { - const exportOnExitDir = options.exportOnExit; +export async function exportOnExit(options: Options): Promise { + // Note: options.exportOnExit is coerced to a string before this point in commandUtils.ts#setExportOnExitOptions + const exportOnExitDir = options.exportOnExit as string; if (exportOnExitDir) { try { utils.logBullet( @@ -81,8 +82,8 @@ export async function exportOnExit(options: any) { "please wait for the export to finish...", ); await exportEmulatorData(exportOnExitDir, options, /* initiatedBy= */ "exit"); - } catch (e: any) { - utils.logWarning(e); + } catch (e: unknown) { + utils.logWarning(`${e}`); utils.logWarning(`Automatic export to "${exportOnExitDir}" failed, going to exit now...`); } } diff --git a/src/emulator/dataconnect/pgliteServer.ts b/src/emulator/dataconnect/pgliteServer.ts index e4cf1071824..b889b99ab50 100644 --- a/src/emulator/dataconnect/pgliteServer.ts +++ b/src/emulator/dataconnect/pgliteServer.ts @@ -26,10 +26,7 @@ export class PostgresServer { public db: PGlite | undefined = undefined; public async createPGServer(host: string = "127.0.0.1", port: number): Promise { - const getDb: () => Promise = () => { - return this.getDb(); - }; - await getDb(); + const getDb = this.getDb.bind(this); const server = net.createServer(async (socket) => { const connection: PostgresConnection = await fromNodeSocket(socket, { @@ -82,9 +79,7 @@ export class PostgresServer { vector, uuidOssp, }, - // TODO: Use dataDir + loadDataDir to implement import/export. dataDir: this.dataDirectory, - // loadDataDir?: Blob | File; // This will be used with .dumpDataDir() for import/export }; if (this.importPath) { logger.debug(`Importing from ${this.importPath}`); @@ -99,12 +94,6 @@ export class PostgresServer { } public async clearDb(): Promise { - // if (this.dataDirectory) { - // rmSync(this.dataDirectory, { recursive: true }); - // } - // await this.db?.close(); - // this.db = undefined; - // return this.getDb(); const db = await this.getDb(); await db.query(` DO $do$ diff --git a/src/emulator/dataconnectEmulator.ts b/src/emulator/dataconnectEmulator.ts index ec22fb8d74f..4a8aa1d900d 100644 --- a/src/emulator/dataconnectEmulator.ts +++ b/src/emulator/dataconnectEmulator.ts @@ -1,6 +1,7 @@ import * as childProcess from "child_process"; import { EventEmitter } from "events"; import * as clc from "colorette"; +import * as path from "path"; import { dataConnectLocalConnString } from "../api"; import { Constants } from "./constants"; @@ -104,13 +105,11 @@ export class DataConnectEmulator implements EmulatorInstance { `FIREBASE_DATACONNECT_POSTGRESQL_STRING is set to ${clc.bold(connStr)} - using that instead of starting a new database`, ); } else if (pgHost && pgPort) { - const dataDirectory = this.args.config.get("emulators.dataconnect.dataDirectory"); - this.postgresServer = new PostgresServer( - dbId, - "postgres", - dataDirectory, - this.args.importPath, - ); + const dataDirectory = this.args.config.get("emulators.dataconnect.dataDir"); + const postgresDumpPath = this.args.importPath + ? path.join(this.args.importPath, "postgres.tar.gz") + : undefined; + this.postgresServer = new PostgresServer(dbId, "postgres", dataDirectory, postgresDumpPath); const server = await this.postgresServer.createPGServer(pgHost, pgPort); const connectableHost = connectableHostname(pgHost); connStr = `postgres://${connectableHost}:${pgPort}/${dbId}?sslmode=disable`; @@ -118,7 +117,6 @@ export class DataConnectEmulator implements EmulatorInstance { if (err instanceof FirebaseError) { this.logger.logLabeled("ERROR", "Data Connect", `${err}`); } else { - console; this.logger.logLabeled( "ERROR", "dataconnect", @@ -185,9 +183,9 @@ export class DataConnectEmulator implements EmulatorInstance { } } - async exportData(path: string): Promise { + async exportData(exportPath: string): Promise { if (this.postgresServer) { - await this.postgresServer.exportData(path); + await this.postgresServer.exportData(path.join(exportPath, "postgres.tar.gz")); } else { throw new FirebaseError( "The Data Connect emulator is currently connected to a separate Postgres instance. Export is not supported.", diff --git a/src/emulator/hub.ts b/src/emulator/hub.ts index a95f009295e..23cd1bac679 100644 --- a/src/emulator/hub.ts +++ b/src/emulator/hub.ts @@ -167,12 +167,11 @@ export class EmulatorHub extends ExpressBasedEmulator { }); app.post(EmulatorHub.PATH_CLEAR_DATA_CONNECT, async (req, res) => { - // TODO: Sanity check that this is needed. - // if (req.headers.origin) { - // res.status(403).json({ - // message: `Clear Data Connect cannot be triggered by external callers.`, - // }); - // } + if (req.headers.origin) { + res.status(403).json({ + message: `Clear Data Connect cannot be triggered by external callers.`, + }); + } utils.logLabeledBullet("emulators", `Clearing data from Data Connect data sources.`); const instance = EmulatorRegistry.get(Emulators.DATACONNECT) as DataConnectEmulator; diff --git a/src/emulator/hubExport.ts b/src/emulator/hubExport.ts index d975c11b936..c61cd9b81bc 100644 --- a/src/emulator/hubExport.ts +++ b/src/emulator/hubExport.ts @@ -132,7 +132,7 @@ export class HubExport { if (shouldExport(Emulators.DATACONNECT)) { metadata.dataconnect = { version: EmulatorHub.CLI_VERSION, - path: "dataconnect_export.tar.gz", + path: "dataconnect_export", }; await this.exportDataConnect(metadata); } @@ -310,10 +310,6 @@ export class HubExport { initiated_by: this.options.initiatedBy, emulator_name: Emulators.DATACONNECT, }); - const dataconnectExportPath = path.join(this.tmpDir, metadata.dataconnect!.path); - if (fs.existsSync(dataconnectExportPath)) { - fse.removeSync(dataconnectExportPath); - } const instance = EmulatorRegistry.get(Emulators.DATACONNECT) as DataConnectEmulator; if (!instance) { @@ -321,6 +317,13 @@ export class HubExport { "Unable to export Data Connect emulator data: the Data Connect emulator is not running.", ); } + + const dataconnectExportPath = path.join(this.tmpDir, metadata.dataconnect!.path); + if (fs.existsSync(dataconnectExportPath)) { + fse.removeSync(dataconnectExportPath); + } + fs.mkdirSync(dataconnectExportPath); + await instance.exportData(dataconnectExportPath); } } diff --git a/src/firebaseConfig.ts b/src/firebaseConfig.ts index 9a9d674df1e..5918e33063a 100644 --- a/src/firebaseConfig.ts +++ b/src/firebaseConfig.ts @@ -240,7 +240,7 @@ export type EmulatorsConfig = { port?: number; postgresHost?: string; postgresPort?: number; - dataDirectory?: string; + dataDir?: string; }; tasks?: { host?: string; From 0fe172f1b50d57716ec77b532a6c3ef7df029266 Mon Sep 17 00:00:00 2001 From: Joe Hanley Date: Mon, 18 Nov 2024 09:27:09 -0800 Subject: [PATCH 08/24] Trying to fix vsce compile --- src/emulator/dataconnect/pgliteServer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/emulator/dataconnect/pgliteServer.ts b/src/emulator/dataconnect/pgliteServer.ts index b889b99ab50..642b6b578a3 100644 --- a/src/emulator/dataconnect/pgliteServer.ts +++ b/src/emulator/dataconnect/pgliteServer.ts @@ -84,7 +84,7 @@ export class PostgresServer { if (this.importPath) { logger.debug(`Importing from ${this.importPath}`); const rf = fs.readFileSync(this.importPath); - const file = new File([rf.buffer], this.importPath); + const file = new File([rf], this.importPath); pgliteArgs.loadDataDir = file; } this.db = await PGlite.create(pgliteArgs); From 6aa6e9872e03e61a117c07cca76f31bba04a98f2 Mon Sep 17 00:00:00 2001 From: Joe Hanley Date: Tue, 19 Nov 2024 15:30:29 -0800 Subject: [PATCH 09/24] Progress on vsce import/export/clearData UI --- firebase-vscode/common/messaging/protocol.ts | 8 ++- firebase-vscode/src/analytics.ts | 1 + firebase-vscode/src/core/emulators.ts | 50 ++++++++++++++++--- .../webviews/components/EmulatorPanel.tsx | 25 ++++++++-- src/emulator/hubClient.ts | 8 +++ src/emulator/types.ts | 2 +- 6 files changed, 81 insertions(+), 13 deletions(-) diff --git a/firebase-vscode/common/messaging/protocol.ts b/firebase-vscode/common/messaging/protocol.ts index 68b5edbff54..1dccd3de4ae 100644 --- a/firebase-vscode/common/messaging/protocol.ts +++ b/firebase-vscode/common/messaging/protocol.ts @@ -55,9 +55,12 @@ export interface WebviewToExtensionParamsMap { /** Calls the `firebase init` CLI */ runFirebaseInit: void; - /** Calls the `firebase init` CLI */ + /** Calls the `firebase emulators:start` CLI */ runStartEmulators: void; + /** Calls the `firebase emulators:export` CLI */ + runEmulatorsExport: void; + /** * Show a UI message using the vscode interface */ @@ -100,6 +103,9 @@ export interface WebviewToExtensionParamsMap { /** Opens generated docs */ "fdc.open-docs": void; + /** Clears data from a running data connect emulator */ + "fdc.clear-emulator-data": void; + // Initialize "result" tab. getDataConnectResults: void; diff --git a/firebase-vscode/src/analytics.ts b/firebase-vscode/src/analytics.ts index 4956742e46a..80e01bc60fa 100644 --- a/firebase-vscode/src/analytics.ts +++ b/firebase-vscode/src/analytics.ts @@ -31,6 +31,7 @@ export enum DATA_CONNECT_EVENT_NAME { START_EMULATORS = "start_emulators", AUTO_COMPLETE = "auto_complete", SESSION_CHAR_COUNT = "session_char_count", + EMULATOR_EXPORT ="emulator_export" } export class AnalyticsLogger { diff --git a/firebase-vscode/src/core/emulators.ts b/firebase-vscode/src/core/emulators.ts index 410f057924f..c5c3029f1fd 100644 --- a/firebase-vscode/src/core/emulators.ts +++ b/firebase-vscode/src/core/emulators.ts @@ -21,6 +21,19 @@ export class EmulatorsController implements Disposable { this.setEmulatorsStarting(); }), ); + + // called by emulator UI + this.subscriptions.push( + broker.on("fdc.clear-emulator-data", () => { + this.clearDataConnectData(); + }), + ); + + this.subscriptions.push(broker.on("runEmulatorsExport", () => { + // TODO: optional debug mode + // TODO: Let users choose a export directory + this.exportEmulatorData(); + })) } readonly emulatorStatusItem = vscode.window.createStatusBarItem("emulators"); @@ -107,11 +120,8 @@ export class EmulatorsController implements Disposable { async findRunningCliEmulators(): Promise< { status: EmulatorsStatus; infos?: RunningEmulatorInfo } | undefined > { - const projectId = firebaseRC.value?.tryReadValue?.projects?.default; - // TODO: think about what to without projectID, in potentially a logged out mode - const hubClient = new EmulatorHubClient(projectId!); - - if (hubClient.foundHub()) { + const hubClient = this.getHubClient(); + if (hubClient) { const response: GetEmulatorsResponse = await hubClient.getEmulators(); if (Object.values(response)) { @@ -119,11 +129,37 @@ export class EmulatorsController implements Disposable { } else { this.setEmulatorsStopped(); } + } + return this.emulators; + } + + async clearDataConnectData(): Promise { + const hubClient = this.getHubClient(); + if (hubClient) { + await hubClient.clearDataConnectData(); + vscode.window.showInformationMessage(`Data Connect emulator data has been cleared.`); + } + } + + async exportEmulatorData(): Promise { + const hubClient = this.getHubClient(); + if (hubClient) { + const exportDir = "./exportedData"; + // TODO: Make exportDir configurable + await hubClient.postExport({path: exportDir, initiatedBy: "Data Connect VSCode extension"}); + vscode.window.showInformationMessage(`Emulator Data exported to ${exportDir}`); + } + } + + private getHubClient(): EmulatorHubClient | undefined { + const projectId = firebaseRC.value?.tryReadValue?.projects?.default; + // TODO: think about what to without projectID, in potentially a logged out mode + const hubClient = new EmulatorHubClient(projectId!); + if (hubClient.foundHub()) { + return hubClient; } else { this.setEmulatorsStopped(); } - - return this.emulators; } public areEmulatorsRunning() { diff --git a/firebase-vscode/webviews/components/EmulatorPanel.tsx b/firebase-vscode/webviews/components/EmulatorPanel.tsx index 11d549513e3..e3328132f4c 100644 --- a/firebase-vscode/webviews/components/EmulatorPanel.tsx +++ b/firebase-vscode/webviews/components/EmulatorPanel.tsx @@ -1,10 +1,10 @@ import { VSCodeButton, VSCodeLink } from "@vscode/webview-ui-toolkit/react"; -import React, { useEffect } from "react"; +import React from "react"; import { Spacer } from "./ui/Spacer"; -import { PanelSection } from "./ui/PanelSection"; -import { EmulatorInfo } from "../../../src/emulator/types"; +import { EmulatorInfo, Emulators, IMPORT_EXPORT_EMULATORS } from "../../../src/emulator/types"; import { RunningEmulatorInfo } from "../messaging/types"; import { Body, Label } from "./ui/Text"; +import { broker } from "../globals/html-broker"; import styles from "./EmulatorPanel.scss"; import { ExternalLink } from "./ui/ExternalLink"; import { Icon } from "./ui/Icon"; @@ -31,7 +31,7 @@ export function EmulatorPanel({ )} - + ); } + +function RunningEmulatorControlButtons({ infos }: { infos: EmulatorInfo[] }) { + return ( + + {!!infos.some(e => e.name === Emulators.DATACONNECT) && ( + broker.send("fdc.clear-emulator-data")}> + Clear Data Connect Data + + )} + {!!infos.some(e => IMPORT_EXPORT_EMULATORS.includes(e.name)) && ( + broker.send("runEmulatorsExport")}> + Export Emulator Data + + )} + + ); +} diff --git a/src/emulator/hubClient.ts b/src/emulator/hubClient.ts index 5c411171e29..bd8ee7cd248 100644 --- a/src/emulator/hubClient.ts +++ b/src/emulator/hubClient.ts @@ -47,6 +47,14 @@ export class EmulatorHubClient { return res.body; } + async clearDataConnectData(): Promise { + // This is a POST operation that should not be retried / multicast, so we + // will try to find the right origin first via GET. + const origin = await this.getStatus(); + const apiClient = new Client({ urlPrefix: origin, auth: false }); + await apiClient.post(EmulatorHub.PATH_CLEAR_DATA_CONNECT); + } + async postExport(options: ExportOptions): Promise { // This is a POST operation that should not be retried / multicast, so we // will try to find the right origin first via GET. diff --git a/src/emulator/types.ts b/src/emulator/types.ts index 8492026cf85..516d63c3ed9 100644 --- a/src/emulator/types.ts +++ b/src/emulator/types.ts @@ -37,7 +37,7 @@ export const DOWNLOADABLE_EMULATORS = [ Emulators.DATACONNECT, ]; -export type ImportExportEmulators = Emulators.FIRESTORE | Emulators.DATABASE | Emulators.AUTH; +export type ImportExportEmulators = Emulators.FIRESTORE | Emulators.DATABASE | Emulators.AUTH | Emulators.STORAGE | Emulators.DATACONNECT; export const IMPORT_EXPORT_EMULATORS = [ Emulators.FIRESTORE, Emulators.DATABASE, From c6f7ada531bb21bbeced22a17843c32427b86871 Mon Sep 17 00:00:00 2001 From: Joe Hanley Date: Thu, 21 Nov 2024 09:58:55 -0800 Subject: [PATCH 10/24] fixed build issue --- .../webviews/components/EmulatorPanel.scss | 6 ++++++ .../webviews/components/EmulatorPanel.tsx | 19 ++++++++----------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/firebase-vscode/webviews/components/EmulatorPanel.scss b/firebase-vscode/webviews/components/EmulatorPanel.scss index 62553956193..be56cdcef9a 100644 --- a/firebase-vscode/webviews/components/EmulatorPanel.scss +++ b/firebase-vscode/webviews/components/EmulatorPanel.scss @@ -1,7 +1,13 @@ +@import "../globals/index.scss"; + .list { list-style: none; } +.fullWidth { + width: 100%; +} + .list-item { align-items: center; display: flex; diff --git a/firebase-vscode/webviews/components/EmulatorPanel.tsx b/firebase-vscode/webviews/components/EmulatorPanel.tsx index e3328132f4c..14e42d57b5c 100644 --- a/firebase-vscode/webviews/components/EmulatorPanel.tsx +++ b/firebase-vscode/webviews/components/EmulatorPanel.tsx @@ -1,7 +1,7 @@ import { VSCodeButton, VSCodeLink } from "@vscode/webview-ui-toolkit/react"; import React from "react"; import { Spacer } from "./ui/Spacer"; -import { EmulatorInfo, Emulators, IMPORT_EXPORT_EMULATORS } from "../../../src/emulator/types"; +import type { EmulatorInfo } from "../../../src/emulator/types"; import { RunningEmulatorInfo } from "../messaging/types"; import { Body, Label } from "./ui/Text"; import { broker } from "../globals/html-broker"; @@ -70,16 +70,13 @@ function FormatEmulatorRunningInfo({ infos }: { infos: EmulatorInfo[] }) { function RunningEmulatorControlButtons({ infos }: { infos: EmulatorInfo[] }) { return ( - {!!infos.some(e => e.name === Emulators.DATACONNECT) && ( - broker.send("fdc.clear-emulator-data")}> - Clear Data Connect Data - - )} - {!!infos.some(e => IMPORT_EXPORT_EMULATORS.includes(e.name)) && ( - broker.send("runEmulatorsExport")}> - Export Emulator Data - - )} + broker.send("fdc.clear-emulator-data")}> + Clear Data Connect Data + + + broker.send("runEmulatorsExport")}> + Export Emulator Data + ); } From 4f725b64260308f3c9991eb40f931e109ea9d535 Mon Sep 17 00:00:00 2001 From: Joe Hanley Date: Tue, 10 Dec 2024 11:00:54 -0800 Subject: [PATCH 11/24] Adding config options for importPath/exportPath/exportOnExit --- firebase-vscode/common/messaging/protocol.ts | 3 +++ firebase-vscode/package.json | 14 ++++++++++++++ firebase-vscode/package.nls.json | 8 ++++++-- firebase-vscode/src/core/emulators.ts | 13 ++++++++++--- firebase-vscode/src/data-connect/terminal.ts | 12 +++++++++--- firebase-vscode/src/utils/settings.ts | 6 ++++++ firebase-vscode/webviews/SidebarApp.tsx | 7 +++++++ 7 files changed, 55 insertions(+), 8 deletions(-) diff --git a/firebase-vscode/common/messaging/protocol.ts b/firebase-vscode/common/messaging/protocol.ts index 1dccd3de4ae..a3dfa3bf98c 100644 --- a/firebase-vscode/common/messaging/protocol.ts +++ b/firebase-vscode/common/messaging/protocol.ts @@ -103,6 +103,9 @@ export interface WebviewToExtensionParamsMap { /** Opens generated docs */ "fdc.open-docs": void; + /** Opens settings page searching for Data Connect emualtor settings */ + "fdc.open-emulator-settings": void; + /** Clears data from a running data connect emulator */ "fdc.clear-emulator-data": void; diff --git a/firebase-vscode/package.json b/firebase-vscode/package.json index db57bfc9f27..279f4a534d9 100644 --- a/firebase-vscode/package.json +++ b/firebase-vscode/package.json @@ -77,6 +77,20 @@ "default": true, "markdownDescription": "%ext.config.idx.viewMetricNotice%", "scope": "application" + }, + "firebase.emulators.importPath": { + "type": "string", + "markdownDescription": "%ext.config.emulators.importPath%" + }, + "firebase.emulators.exportPath": { + "type": "string", + "default": "./exportedData", + "markdownDescription": "%ext.config.emulators.exportPath%" + }, + "firebase.dataConnect.emulators.exportOnExit":{ + "type": "boolean", + "default": false, + "markdownDescription": "%ext.config.emulators.exportOnExit%" } } }, diff --git a/firebase-vscode/package.nls.json b/firebase-vscode/package.nls.json index 2fd89eb524d..209372943ec 100644 --- a/firebase-vscode/package.nls.json +++ b/firebase-vscode/package.nls.json @@ -4,6 +4,10 @@ "ext.config.firebasePath": "Path to the `Firebase` module, e.g. `./node_modules/firebase`", "ext.config.hosting.useFrameworks": "Enable web frameworks", "ext.config.npmPath": "Path to NPM executable in local environment", - "ext.config.title": "Prettier", - "ext.config.idx.viewMetricNotice": "Show data collection notice on next startup (IDX Only)" + "ext.config.title": "Firebase Data Connect", + "ext.config.idx.viewMetricNotice": "Show data collection notice on next startup (IDX Only)", + "ext.config.emulators.importPath": "Path to import emulator data from", + "ext.config.emulators.exportPath": "Path to export emulator data to", + "ext.config.emulators.exportOnExit": "If true, data will be exported to exportPath when the emulator shuts down" + } diff --git a/firebase-vscode/src/core/emulators.ts b/firebase-vscode/src/core/emulators.ts index c5c3029f1fd..7bb203a413f 100644 --- a/firebase-vscode/src/core/emulators.ts +++ b/firebase-vscode/src/core/emulators.ts @@ -6,6 +6,7 @@ import { EmulatorsStatus, RunningEmulatorInfo } from "../messaging/types"; import { EmulatorHubClient } from "../../../src/emulator/hubClient"; import { GetEmulatorsResponse } from "../../../src/emulator/hub"; import { EmulatorInfo } from "../emulator/types"; +import { getSettings } from "../utils/settings"; export class EmulatorsController implements Disposable { constructor(private broker: ExtensionBrokerImpl) { this.emulatorStatusItem.command = "firebase.openFirebaseRc"; @@ -22,6 +23,12 @@ export class EmulatorsController implements Disposable { }), ); + this.subscriptions.push( + broker.on("fdc.open-emulator-settings", () => { + vscode.commands.executeCommand( 'workbench.action.openSettings', 'firebase.emulators' ); + }) + ) + // called by emulator UI this.subscriptions.push( broker.on("fdc.clear-emulator-data", () => { @@ -32,7 +39,8 @@ export class EmulatorsController implements Disposable { this.subscriptions.push(broker.on("runEmulatorsExport", () => { // TODO: optional debug mode // TODO: Let users choose a export directory - this.exportEmulatorData(); + const settings = getSettings(); + this.exportEmulatorData(settings.exportPath); })) } @@ -141,10 +149,9 @@ export class EmulatorsController implements Disposable { } } - async exportEmulatorData(): Promise { + async exportEmulatorData(exportDir: string): Promise { const hubClient = this.getHubClient(); if (hubClient) { - const exportDir = "./exportedData"; // TODO: Make exportDir configurable await hubClient.postExport({path: exportDir, initiatedBy: "Data Connect VSCode extension"}); vscode.window.showInformationMessage(`Emulator Data exported to ${exportDir}`); diff --git a/firebase-vscode/src/data-connect/terminal.ts b/firebase-vscode/src/data-connect/terminal.ts index 9c836760cc0..ceac98e4778 100644 --- a/firebase-vscode/src/data-connect/terminal.ts +++ b/firebase-vscode/src/data-connect/terminal.ts @@ -89,12 +89,18 @@ export function registerTerminalTasks( analyticsLogger.logger.logUsage(DATA_CONNECT_EVENT_NAME.START_EMULATORS, { firebase_binary_kind: settings.firebaseBinaryKind, }); + let cmd = `${settings.firebasePath} emulators:start --project ${currentProjectId.value}`; + if (settings.importPath) { + cmd += ` --import ${settings.importPath}`; + } + if (settings.exportOnExit) { + cmd += ` --export-on-exit ${settings.exportPath}`; + } // TODO: optional debug mode runTerminalTask( "firebase emulators", - `${settings.firebasePath} emulators:start --project ${currentProjectId.value}`, - // emulators:start almost never ask interactive questions. - { focus: false }, + cmd, + { focus: true }, ); }); diff --git a/firebase-vscode/src/utils/settings.ts b/firebase-vscode/src/utils/settings.ts index a1ba94a1568..19fe1963b56 100644 --- a/firebase-vscode/src/utils/settings.ts +++ b/firebase-vscode/src/utils/settings.ts @@ -7,6 +7,9 @@ export interface Settings { readonly npmPath: string; readonly useFrameworks: boolean; readonly shouldShowIdxMetricNotice: boolean; + readonly importPath?: string; + readonly exportPath: string; + readonly exportOnExit: boolean; } // TODO: Temporary fallback for bashing, this should probably point to the global firebase binary on the system @@ -37,6 +40,9 @@ export function getSettings(): Settings { "idx.viewMetricNotice", true, ), + importPath: config.get("emulators.importPath"), + exportPath: config.get("emulators.exportPath", "./exportedData"), + exportOnExit: config.get("emulators.exporOnExit", false), }; } diff --git a/firebase-vscode/webviews/SidebarApp.tsx b/firebase-vscode/webviews/SidebarApp.tsx index c04f5150e59..0abd8c02f95 100644 --- a/firebase-vscode/webviews/SidebarApp.tsx +++ b/firebase-vscode/webviews/SidebarApp.tsx @@ -105,6 +105,13 @@ function EmulatorsPanel() { Start emulators +