diff --git a/genkit-tools/cli/src/cli.ts b/genkit-tools/cli/src/cli.ts index c6f258c2d7..b45ed7054d 100644 --- a/genkit-tools/cli/src/cli.ts +++ b/genkit-tools/cli/src/cli.ts @@ -138,10 +138,11 @@ export async function startCLI(): Promise { logger.info(program.help()); }) ); - // Default action to catch unknown commands. - program.action(() => { - // print help + // Handle unknown commands. + program.on('command:*', (operands) => { + logger.error(`error: unknown command '${operands[0]}'`); logger.info(program.help()); + process.exit(1); }); await program.parseAsync(); diff --git a/genkit-tools/common/src/eval/evaluate.ts b/genkit-tools/common/src/eval/evaluate.ts index ddf99e3361..8bbc6607ba 100644 --- a/genkit-tools/common/src/eval/evaluate.ts +++ b/genkit-tools/common/src/eval/evaluate.ts @@ -193,6 +193,9 @@ export async function runEvaluation(params: { }, }); scores[name] = response.result; + logger.info( + `Finished evaluator '${action.name}'. Trace ID: ${response.telemetry?.traceId}` + ); } const scoredResults = enrichResultsWithScoring(scores, evalDataset); diff --git a/js/testapps/evals/src/genkit.ts b/js/testapps/evals/src/genkit.ts index e59f0f13b6..f08f688169 100644 --- a/js/testapps/evals/src/genkit.ts +++ b/js/testapps/evals/src/genkit.ts @@ -16,12 +16,7 @@ import { devLocalVectorstore } from '@genkit-ai/dev-local-vectorstore'; import { GenkitMetric, genkitEval } from '@genkit-ai/evaluator'; -import { - gemini15Flash, - gemini15Pro, - googleAI, - textEmbeddingGecko001, -} from '@genkit-ai/googleai'; +import { googleAI } from '@genkit-ai/googleai'; import { genkit } from 'genkit'; import { langchain } from 'genkitx-langchain'; @@ -55,12 +50,12 @@ export const ai = genkit({ metrics: [ { type: GenkitMetric.MALICIOUSNESS, - judge: gemini15Pro, + judge: googleAI.model('gemini-2.5-pro'), judgeConfig: PERMISSIVE_SAFETY_SETTINGS, }, { type: GenkitMetric.ANSWER_ACCURACY, - judge: gemini15Pro, + judge: googleAI.model('gemini-2.5-pro'), judgeConfig: PERMISSIVE_SAFETY_SETTINGS, }, ], @@ -68,14 +63,14 @@ export const ai = genkit({ devLocalVectorstore([ { indexName: 'pdfQA', - embedder: textEmbeddingGecko001, + embedder: googleAI.embedder('text-embedding-004'), }, ]), langchain({ evaluators: { criteria: ['coherence'], labeledCriteria: ['correctness'], - judge: gemini15Flash, + judge: googleAI.model('gemini-2.5-pro'), }, }), ], diff --git a/js/testapps/evals/src/pdf-rag.ts b/js/testapps/evals/src/pdf-rag.ts index d564822bad..59ac6d3629 100644 --- a/js/testapps/evals/src/pdf-rag.ts +++ b/js/testapps/evals/src/pdf-rag.ts @@ -18,7 +18,7 @@ import { devLocalIndexerRef, devLocalRetrieverRef, } from '@genkit-ai/dev-local-vectorstore'; -import { gemini15Flash } from '@genkit-ai/googleai'; +import googleAI from '@genkit-ai/googleai'; import { z } from 'genkit'; import { EvalStatusEnum, type BaseEvalDataPoint } from 'genkit/evaluator'; import { Document } from 'genkit/retriever'; @@ -70,7 +70,7 @@ export const pdfQA = ai.defineFlow( context: docs.map((d) => d.text).join('\n\n'), }); const llmResponse = await ai.generate({ - model: gemini15Flash, + model: googleAI.model('gemini-2.5-flash'), prompt: augmentedPrompt, }); return llmResponse.text; @@ -86,7 +86,7 @@ export const simpleStructured = ai.defineFlow( }, async (i) => { const llmResponse = await ai.generate({ - model: gemini15Flash, + model: googleAI.model('gemini-2.5-flash'), prompt: i.query, }); return { response: llmResponse.text }; @@ -102,7 +102,7 @@ export const simpleEcho = ai.defineFlow( }, async (i) => { const llmResponse = await ai.generate({ - model: gemini15Flash, + model: googleAI.model('gemini-2.5-flash'), prompt: i, }); return llmResponse.text;