From 816ed6221e15befbbab9a92d673e0cb09667dc4d Mon Sep 17 00:00:00 2001 From: Erik Eldridge Date: Thu, 1 May 2025 11:44:53 -0700 Subject: [PATCH 1/2] Log a warning when input is excluded --- packages/vertexai/src/methods/chrome-adapter.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/vertexai/src/methods/chrome-adapter.ts b/packages/vertexai/src/methods/chrome-adapter.ts index 9ac8f350a02..245a0da9898 100644 --- a/packages/vertexai/src/methods/chrome-adapter.ts +++ b/packages/vertexai/src/methods/chrome-adapter.ts @@ -16,6 +16,7 @@ */ import { AIError } from '../errors'; +import { logger } from '../logger'; import { CountTokensRequest, GenerateContentRequest, @@ -97,6 +98,7 @@ export class ChromeAdapter { const contents = await Promise.all( request.contents[0].parts.map(ChromeAdapter.toLanguageModelMessageContent) ); + logger.warn('Only generating content from first item in "contents" array.'); const text = await session.prompt(contents); return ChromeAdapter.toResponse(text); } @@ -118,6 +120,7 @@ export class ChromeAdapter { const contents = await Promise.all( request.contents[0].parts.map(ChromeAdapter.toLanguageModelMessageContent) ); + logger.warn('Only generating content from first item in "contents" array.'); const stream = await session.promptStreaming(contents); return ChromeAdapter.toStreamResponse(stream); } @@ -135,6 +138,7 @@ export class ChromeAdapter { private static isOnDeviceRequest(request: GenerateContentRequest): boolean { // Returns false if the prompt is empty. if (request.contents.length === 0) { + logger.debug('Empty prompt rejected for on-device inference.'); return false; } @@ -142,6 +146,7 @@ export class ChromeAdapter { // Returns false if the request contains multiple roles, eg a chat history. // TODO: remove this guard once LanguageModelMessage is supported. if (content.role !== 'user') { + logger.debug('Non-user role "${content.role}" rejected for on-device inference.'); return false; } @@ -153,6 +158,9 @@ export class ChromeAdapter { part.inlineData.mimeType ) === -1 ) { + logger.debug( + `Unsupported mime type "${part.inlineData.mimeType}" rejected for on-device inference.` + ); return false; } } From 78d1cc25b880fd5c114a6a839ef1bdf1a6950d76 Mon Sep 17 00:00:00 2001 From: Erik Eldridge Date: Thu, 1 May 2025 14:03:39 -0700 Subject: [PATCH 2/2] Log availability checks --- e2e/sample-apps/modular.js | 17 +++++++++--- .../vertexai/src/methods/chrome-adapter.ts | 27 ++++++++++++++----- 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/e2e/sample-apps/modular.js b/e2e/sample-apps/modular.js index aeebe19a4b1..abcf829856b 100644 --- a/e2e/sample-apps/modular.js +++ b/e2e/sample-apps/modular.js @@ -314,13 +314,22 @@ async function callVertexAI(app) { console.log('[VERTEXAI] start'); const vertexAI = getVertexAI(app); const model = getGenerativeModel(vertexAI, { - mode: 'only_on_device' + mode: 'prefer_on_device' }); const singleResult = await model.generateContent([ - { text: 'describe the following:' }, - { text: 'the mojave desert' } + { text: 'describe this 20 x 20 px image in two words' }, + { + inlineData: { + mimeType: 'image/heic', + data: 'AAAAGGZ0eXBoZWljAAAAAGhlaWNtaWYxAAAB7G1ldGEAAAAAAAAAIWhkbHIAAAAAAAAAAHBpY3QAAAAAAAAAAAAAAAAAAAAAJGRpbmYAAAAcZHJlZgAAAAAAAAABAAAADHVybCAAAAABAAAADnBpdG0AAAAAAAEAAAA4aWluZgAAAAAAAgAAABVpbmZlAgAAAAABAABodmMxAAAAABVpbmZlAgAAAQACAABFeGlmAAAAABppcmVmAAAAAAAAAA5jZHNjAAIAAQABAAABD2lwcnAAAADtaXBjbwAAABNjb2xybmNseAACAAIABoAAAAAMY2xsaQDLAEAAAAAUaXNwZQAAAAAAAAAUAAAADgAAAChjbGFwAAAAFAAAAAEAAAANAAAAAQAAAAAAAAAB/8AAAACAAAAAAAAJaXJvdAAAAAAQcGl4aQAAAAADCAgIAAAAcWh2Y0MBA3AAAACwAAAAAAAe8AD8/fj4AAALA6AAAQAXQAEMAf//A3AAAAMAsAAAAwAAAwAecCShAAEAI0IBAQNwAAADALAAAAMAAAMAHqAUIEHAjw1iHuRZVNwICBgCogABAAlEAcBhcshAUyQAAAAaaXBtYQAAAAAAAAABAAEHgQIDhIUGhwAAACxpbG9jAAAAAEQAAAIAAQAAAAEAAAJsAAABDAACAAAAAQAAAhQAAABYAAAAAW1kYXQAAAAAAAABdAAAAAZFeGlmAABNTQAqAAAACAAEARIAAwAAAAEAAQAAARoABQAAAAEAAAA+ARsABQAAAAEAAABGASgAAwAAAAEAAgAAAAAAAAAAAEgAAAABAAAASAAAAAEAAAEIKAGvoR8wDimTiRYUbALiHkU3ZdZ8DXAcSrRB9GARtVQHvnCE0LEyBGAyb5P4eYr6JAK5UxNX10WNlARq3ZpcGeVD+Xom6LodYasuZKKtDHCz/xnswOtC/ksZzVKhtWQqGvkXcsJnLYqWevNkacnccQ95jbHJBg9nXub69jAAN3xhNOXxjGSxaG9QvES5R7sYICEojRjLF5OB5K3v+okQAwfgWpz/u21ayideOgOZQLAyBkKOv7ymLNCagiPWTlHAuy/3qR1Q7m2ERFaxKIAbLSkIVO/P8m8+anKxhzhC//L8NMAUoF+Sf3aEH9O41fwLc+PlcbrDrjgY2EboD3cn9DyN32Rum2Ym' + } + } ]); console.log(`Generated text: ${singleResult.response.text()}`); + const chat = model.startChat(); + let chatResult = await chat.sendMessage('describe red in two words'); + chatResult = await chat.sendMessage('describe blue'); + console.log('Chat history:', await chat.getHistory()); console.log(`[VERTEXAI] end`); } @@ -345,7 +354,7 @@ function callDataConnect(app) { async function main() { console.log('FIREBASE VERSION', SDK_VERSION); const app = initializeApp(config); - setLogLevel('warn'); + setLogLevel('debug'); // callAppCheck(app); // await authLogin(app); diff --git a/packages/vertexai/src/methods/chrome-adapter.ts b/packages/vertexai/src/methods/chrome-adapter.ts index 245a0da9898..9ba674937a8 100644 --- a/packages/vertexai/src/methods/chrome-adapter.ts +++ b/packages/vertexai/src/methods/chrome-adapter.ts @@ -66,6 +66,9 @@ export class ChromeAdapter { */ async isAvailable(request: GenerateContentRequest): Promise { if (this.mode === 'only_in_cloud') { + logger.debug( + `On-device inference unavailable because mode is "only_in_cloud".` + ); return false; } @@ -77,10 +80,20 @@ export class ChromeAdapter { } // Applies prefer_on_device logic. - return ( - availability === Availability.available && - ChromeAdapter.isOnDeviceRequest(request) - ); + if (availability !== Availability.available) { + logger.debug( + `On-device inference unavailable because availability is "${availability}".` + ); + return false; + } + if (!ChromeAdapter.isOnDeviceRequest(request)) { + logger.debug( + `On-device inference unavailable because request is incompatible.` + ); + return false; + } + + return true; } /** @@ -98,7 +111,6 @@ export class ChromeAdapter { const contents = await Promise.all( request.contents[0].parts.map(ChromeAdapter.toLanguageModelMessageContent) ); - logger.warn('Only generating content from first item in "contents" array.'); const text = await session.prompt(contents); return ChromeAdapter.toResponse(text); } @@ -120,7 +132,6 @@ export class ChromeAdapter { const contents = await Promise.all( request.contents[0].parts.map(ChromeAdapter.toLanguageModelMessageContent) ); - logger.warn('Only generating content from first item in "contents" array.'); const stream = await session.promptStreaming(contents); return ChromeAdapter.toStreamResponse(stream); } @@ -146,7 +157,9 @@ export class ChromeAdapter { // Returns false if the request contains multiple roles, eg a chat history. // TODO: remove this guard once LanguageModelMessage is supported. if (content.role !== 'user') { - logger.debug('Non-user role "${content.role}" rejected for on-device inference.'); + logger.debug( + `Non-user role "${content.role}" rejected for on-device inference.` + ); return false; }