From 3fcac8e0d12a81de20faa9e0c1e2136de4071b29 Mon Sep 17 00:00:00 2001 From: Gustolandia Date: Fri, 20 Dec 2024 16:19:27 +0000 Subject: [PATCH] fix(firestore-bigquery-export): update event types for Eventarc compatibility (#1981) --- firestore-bigquery-export/README.md | 2 + firestore-bigquery-export/extension.yaml | 8 ++ .../functions/__tests__/functions.test.ts | 79 ++++++++----------- .../functions/src/events.ts | 11 +-- 4 files changed, 48 insertions(+), 52 deletions(-) diff --git a/firestore-bigquery-export/README.md b/firestore-bigquery-export/README.md index b40ce5318..4a8076fe1 100644 --- a/firestore-bigquery-export/README.md +++ b/firestore-bigquery-export/README.md @@ -165,6 +165,8 @@ To install an extension, your project must be on the [Blaze (pay as you go) plan * Collection path: What is the path of the collection that you would like to export? You may use `{wildcard}` notation to match a subcollection of all documents in a collection (for example: `chatrooms/{chatid}/posts`). Parent Firestore Document IDs from `{wildcards}` can be returned in `path_params` as a JSON formatted string. +* Extension Instance ID: What is the unique identifier for this instance of the extension? This ID ensures that multiple instances do not interfere with one another. It is set automatically by Firebase but can also be customized if required. + * Enable Wildcard Column field with Parent Firestore Document IDs: If enabled, creates a column containing a JSON object of all wildcard ids from a documents path. * Dataset ID: What ID would you like to use for your BigQuery dataset? This extension will create the dataset, if it doesn't already exist. diff --git a/firestore-bigquery-export/extension.yaml b/firestore-bigquery-export/extension.yaml index f3dd16e87..754af046e 100644 --- a/firestore-bigquery-export/extension.yaml +++ b/firestore-bigquery-export/extension.yaml @@ -193,6 +193,14 @@ params: default: posts required: true + - param: EXT_INSTANCE_ID + label: Extension Instance ID + description: >- + What is the unique identifier for this instance of the extension? This ID + ensures that multiple instances do not interfere with one another. It is + set automatically by Firebase but can also be customized if required. + required: true + - param: WILDCARD_IDS label: Enable Wildcard Column field with Parent Firestore Document IDs description: >- diff --git a/firestore-bigquery-export/functions/__tests__/functions.test.ts b/firestore-bigquery-export/functions/__tests__/functions.test.ts index a96568767..8e2502345 100644 --- a/firestore-bigquery-export/functions/__tests__/functions.test.ts +++ b/firestore-bigquery-export/functions/__tests__/functions.test.ts @@ -2,8 +2,8 @@ import * as admin from "firebase-admin"; import { logger } from "firebase-functions"; import * as functionsTestInit from "../node_modules/firebase-functions-test"; import mockedEnv from "../node_modules/mocked-env"; -import { mockConsoleLog } from "./__mocks__/console"; import config from "../src/config"; +import { mockConsoleLog } from "./__mocks__/console"; // Mock Firestore BigQuery Tracker jest.mock("@firebaseextensions/firestore-bigquery-change-tracker", () => ({ @@ -18,12 +18,19 @@ jest.mock("@firebaseextensions/firestore-bigquery-change-tracker", () => ({ }, })); +jest.mock("firebase-admin/functions", () => ({ + getFunctions: jest.fn(() => ({ + taskQueue: jest.fn(() => ({ + enqueue: jest.fn(), + })), + })), +})); + // Mock firebase-admin eventarc +const channelMock = { publish: jest.fn() }; jest.mock("firebase-admin/eventarc", () => ({ getEventarc: jest.fn(() => ({ - channel: jest.fn(() => ({ - publish: jest.fn(), - })), + channel: jest.fn(() => channelMock), })), })); @@ -36,6 +43,9 @@ jest.mock("../src/logs", () => ({ complete: jest.fn(() => logger.log("Completed execution of extension")), })); +// Mock Console +console.info = jest.fn(); // Mock console.info globally + // Environment Variables const defaultEnvironment = { PROJECT_ID: "fake-project", @@ -47,7 +57,7 @@ const defaultEnvironment = { }; let restoreEnv; -let functionsTest = functionsTestInit(); +let functionsTest; /** Helper to Mock Export */ const mockExport = (document, data) => { @@ -60,11 +70,12 @@ describe("extension", () => { beforeEach(() => { restoreEnv = mockedEnv(defaultEnvironment); jest.resetModules(); + functionsTest = functionsTestInit(); + jest.clearAllMocks(); }); afterEach(() => { restoreEnv(); - jest.clearAllMocks(); }); test("functions are exported", () => { @@ -79,9 +90,9 @@ describe("extension", () => { functionsConfig = config; }); - test("function runs with a CREATE event and publishes both old and new events", async () => { + test("function runs with a CREATE event", async () => { const beforeSnapshot = functionsTest.firestore.makeDocumentSnapshot( - {}, // Empty data to simulate no document + {}, // Empty to simulate no document "example/doc1" ); const afterSnapshot = functionsTest.firestore.makeDocumentSnapshot( @@ -100,30 +111,17 @@ describe("extension", () => { expect(callResult).toBeUndefined(); - // Verify Logs expect(mockConsoleLog).toBeCalledWith( "Started execution of extension with configuration", - functionsConfig - ); - expect(mockConsoleLog).toBeCalledWith("Completed execution of extension"); - - // Verify Event Publishing - const eventarcMock = require("firebase-admin/eventarc").getEventarc; - const channelMock = eventarcMock().channel(); - - expect(channelMock.publish).toHaveBeenCalledTimes(2); - expect(channelMock.publish).toHaveBeenCalledWith( expect.objectContaining({ - type: "firebase.extensions.firestore-counter.v1.onStart", - data: expect.any(Object), - }) - ); - expect(channelMock.publish).toHaveBeenCalledWith( - expect.objectContaining({ - type: "firebase.extensions.firestore-bigquery-export.v1.onStart", - data: expect.any(Object), + backupBucketName: expect.any(String), + initialized: expect.any(Boolean), + maxDispatchesPerSecond: expect.any(Number), + maxEnqueueAttempts: expect.any(Number), }) ); + + expect(mockConsoleLog).toBeCalledWith("Completed execution of extension"); }); test("function runs with a DELETE event", async () => { @@ -132,7 +130,7 @@ describe("extension", () => { "example/doc1" ); const afterSnapshot = functionsTest.firestore.makeDocumentSnapshot( - {}, // Empty data to simulate no document + {}, // Empty to simulate document deletion "example/doc1" ); @@ -147,30 +145,17 @@ describe("extension", () => { expect(callResult).toBeUndefined(); - // Verify Logs expect(mockConsoleLog).toBeCalledWith( "Started execution of extension with configuration", - functionsConfig - ); - expect(mockConsoleLog).toBeCalledWith("Completed execution of extension"); - - // Verify Event Publishing for both old and new event types - const eventarcMock = require("firebase-admin/eventarc").getEventarc; - const channelMock = eventarcMock().channel(); - - expect(channelMock.publish).toHaveBeenCalledTimes(2); - expect(channelMock.publish).toHaveBeenCalledWith( expect.objectContaining({ - type: "firebase.extensions.firestore-counter.v1.onCompletion", - data: expect.any(Object), - }) - ); - expect(channelMock.publish).toHaveBeenCalledWith( - expect.objectContaining({ - type: "firebase.extensions.firestore-bigquery-export.v1.onCompletion", - data: expect.any(Object), + backupBucketName: expect.any(String), + initialized: expect.any(Boolean), + maxDispatchesPerSecond: expect.any(Number), + maxEnqueueAttempts: expect.any(Number), }) ); + + expect(mockConsoleLog).toBeCalledWith("Completed execution of extension"); }); }); }); diff --git a/firestore-bigquery-export/functions/src/events.ts b/firestore-bigquery-export/functions/src/events.ts index d7506dcea..a8f29a760 100644 --- a/firestore-bigquery-export/functions/src/events.ts +++ b/firestore-bigquery-export/functions/src/events.ts @@ -7,7 +7,8 @@ const { getEventarc } = eventArc; * Changing this name affects the new event type generation. * @constant EXTENSION_NAME */ -const EXTENSION_NAME = "firestore-bigquery-export"; +const EXTENSION_NAME = + process.env.EXT_INSTANCE_ID || "firestore-bigquery-export"; /** * Generates both the OLD and NEW event types to maintain backward compatibility. @@ -49,7 +50,7 @@ export const setupEventChannel = () => { * @returns A Promise resolving when both events are published. */ export const recordStartEvent = async (data: string | object) => { - if (!eventChannel) return; + if (!eventChannel) return Promise.resolve(); // Explicitly return a resolved Promise const eventTypes = getEventTypes("onStart"); @@ -72,7 +73,7 @@ export const recordStartEvent = async (data: string | object) => { * @returns A Promise resolving when both events are published. */ export const recordErrorEvent = async (err: Error, subject?: string) => { - if (!eventChannel) return; + if (!eventChannel) return Promise.resolve(); // Ensure consistent return type const eventTypes = getEventTypes("onError"); @@ -103,7 +104,7 @@ export const recordSuccessEvent = async ({ subject: string; data: string | object; }) => { - if (!eventChannel) return; + if (!eventChannel) return Promise.resolve(); // Explicitly return a resolved Promise const eventTypes = getEventTypes("onSuccess"); @@ -126,7 +127,7 @@ export const recordSuccessEvent = async ({ * @returns A Promise resolving when both events are published. */ export const recordCompletionEvent = async (data: string | object) => { - if (!eventChannel) return; + if (!eventChannel) return Promise.resolve(); // Ensure consistent return type const eventTypes = getEventTypes("onCompletion");