diff --git a/src/lib/assertions/javascript.ts b/src/lib/assertions/javascript.ts index e918516..4b63206 100644 --- a/src/lib/assertions/javascript.ts +++ b/src/lib/assertions/javascript.ts @@ -20,6 +20,7 @@ const jsResultSchema = assertionResultSchema.extend({ outputs: z.record(z.string(), z.union([z.boolean(), z.number()])).optional(), }); type JsResult = z.infer; +export type JavascriptAssertionArgs = z.infer; export function createJavascriptAssertion( args: unknown, diff --git a/src/lib/documentation/codeReferenceTypes.compat.ts b/src/lib/documentation/codeReferenceTypes.compat.ts new file mode 100644 index 0000000..fb8e25e --- /dev/null +++ b/src/lib/documentation/codeReferenceTypes.compat.ts @@ -0,0 +1,244 @@ +import type { JavascriptAssertionArgs } from '$lib/assertions/javascript'; +import type { CodeReference, Executable, ModuleExecutable } from '$lib/storage/CodeReference'; +import type { FileReference } from '$lib/storage/FileReference'; +import type { + AssertionResult, + ConversationPrompt, + MultiPartPrompt, + NormalizedPipelineStep, + NormalizedPrompt, + ProviderOutput, + ProviderOutputPart, + PromptPart, + Provider, + RolePromptPart, + TestOutput, + TransformOutput, +} from '$lib/types'; +import type * as DocTypes from './codeReferenceTypes'; + +type Assert = T; +type IsAssignable = [A] extends [B] ? true : false; +type IsMutuallyAssignable = + IsAssignable extends true ? (IsAssignable extends true ? true : false) : false; + +interface CodeReferencePublic { + uri: string; + file: File; + type: 'file' | 'image' | 'code'; + bind(): Promise; + bindModule(): Promise; + getCode(): Promise; +} + +interface CodeReferenceMarker { + readonly __codeReference: unique symbol; +} +interface FileReferenceMarker { + readonly __fileReference: unique symbol; +} +type ReplaceCodeReference = T extends CodeReference | DocTypes.CodeReference + ? CodeReferenceMarker + : T extends FileReference | DocTypes.FileReference + ? FileReferenceMarker + : T extends File + ? File + : T extends (infer U)[] + ? ReplaceCodeReference[] + : T extends object + ? { [K in keyof T]: ReplaceCodeReference } + : T; + +type _CodeReferenceActual = Assert>; +type _CodeReferenceDoc = Assert>; +type _Executable = Assert>; +type _ModuleExecutable = Assert>; +type PromptPartShape = + | { text: string } + | { file: File } + | { + type: 'function-call'; + name: string; + args: unknown; + meta: unknown; + } + | { + type: 'function-response'; + call: { + type: 'function-call'; + name: string; + args: unknown; + meta: unknown; + }; + response: unknown; + }; + +type _PromptPartDocText = Assert>; +type _PromptPartActualText = Assert>; +type _PromptPartDocFile = Assert>; +type _PromptPartActualFile = Assert>; +type _PromptPartDocCall = Assert< + IsAssignable< + { type: 'function-call'; name: string; args: unknown; meta: unknown }, + DocTypes.PromptPart + > +>; +type _PromptPartActualCall = Assert< + IsAssignable<{ type: 'function-call'; name: string; args: unknown; meta: unknown }, PromptPart> +>; +type _PromptPartDocResponse = Assert< + IsAssignable< + { + type: 'function-response'; + call: { type: 'function-call'; name: string; args: unknown; meta: unknown }; + response: unknown; + }, + DocTypes.PromptPart + > +>; +type _PromptPartActualResponse = Assert< + IsAssignable< + { + type: 'function-response'; + call: { type: 'function-call'; name: string; args: unknown; meta: unknown }; + response: unknown; + }, + PromptPart + > +>; +type _MultiPartPromptDoc = Assert< + IsMutuallyAssignable +>; +type _MultiPartPromptActual = Assert>; +type _RolePromptPartDoc = Assert< + IsMutuallyAssignable< + DocTypes.RolePromptPart, + { role: RolePromptPart['role']; content: DocTypes.MultiPartPrompt } + > +>; +type _RolePromptPartActual = Assert< + IsMutuallyAssignable +>; +type _ConversationPromptDoc = Assert< + IsMutuallyAssignable +>; +type _ConversationPromptActual = Assert>; + +type _PromptPartShape = Assert>; +type _Provider = Assert< + IsMutuallyAssignable, ReplaceCodeReference> +>; +type _JavascriptAssertionArgs = Assert< + IsMutuallyAssignable< + ReplaceCodeReference, + ReplaceCodeReference + > +>; +interface ExpectedJavascriptAssertionResult { + pass: boolean; + message?: string; + visuals?: (string | Blob)[]; + outputs?: Record; +} +type _JavascriptAssertionResult = Assert< + IsMutuallyAssignable +>; +type _ProviderOutput = Assert< + IsAssignable, ReplaceCodeReference> +>; +type _ProviderOutputPart = Assert< + IsAssignable< + ReplaceCodeReference, + ReplaceCodeReference + > +>; +type _TransformOutput = Assert< + IsAssignable< + ReplaceCodeReference, + ReplaceCodeReference + > +>; +type _TestOutput = Assert< + IsAssignable, ReplaceCodeReference> +>; +type _NormalizedPrompt = Assert< + IsAssignable< + ReplaceCodeReference, + ReplaceCodeReference + > +>; +type _AssertionResult = Assert< + IsAssignable< + ReplaceCodeReference, + ReplaceCodeReference + > +>; +type _PipelineIf = Assert< + IsMutuallyAssignable< + ReplaceCodeReference, + ReplaceCodeReference + > +>; +type _PipelineTransform = Assert< + IsMutuallyAssignable< + ReplaceCodeReference, + ReplaceCodeReference + > +>; + +type ExpectedPromptTransformHandler = ( + output: DocTypes.ProviderOutput, + context: { vars: DocTypes.PipelineVars }, +) => + | DocTypes.TransformOutput + | DocTypes.TransformResult + | Promise; + +type ExpectedPromptIfHandler = (vars: DocTypes.PipelineVars) => boolean | Promise; + +type ExpectedProviderPrepareHandler = ( + prompt: DocTypes.ConversationPrompt, + context: DocTypes.ProviderConfigContext, +) => DocTypes.MaybePromise; + +type ExpectedProviderRunHandler = ( + key: unknown, + context: DocTypes.ProviderConfigContext, +) => + | string + | (string | Blob | DocTypes.MetaProviderOutputPart)[] + | DocTypes.MaybePromise; + +type ExpectedProviderEnvHandler = () => DocTypes.MaybePromise; + +type ExpectedAssertionCellHandler = ( + output: DocTypes.ProviderOutput, + context: DocTypes.AssertionCellContext, +) => DocTypes.MaybePromise; + +type ExpectedAssertionRowHandler = ( + results: DocTypes.TestOutput[], + context: DocTypes.AssertionRowContext, +) => DocTypes.MaybePromise; + +type _PromptTransformHandler = Assert< + IsMutuallyAssignable +>; +type _PromptIfHandler = Assert< + IsMutuallyAssignable +>; +type _ProviderPrepareHandler = Assert< + IsMutuallyAssignable +>; +type _ProviderRunHandler = Assert< + IsMutuallyAssignable +>; +type _ProviderEnvHandler = Assert< + IsMutuallyAssignable +>; +type _AssertionCellHandler = Assert< + IsMutuallyAssignable +>; +type _AssertionRowHandler = Assert< + IsMutuallyAssignable +>; diff --git a/src/lib/documentation/codeReferenceTypes.ts b/src/lib/documentation/codeReferenceTypes.ts new file mode 100644 index 0000000..fea0dfa --- /dev/null +++ b/src/lib/documentation/codeReferenceTypes.ts @@ -0,0 +1,187 @@ +// Self-contained CodeReference-related type definitions for docs/downloads. +// This file must not import from $lib or any other local modules. + +export type Executable = (...args: unknown[]) => Promise; +export type ModuleExecutable = (name: string | undefined, ...args: unknown[]) => Promise; + +export declare class FileReference { + readonly uri: string; + readonly file: File; + readonly type: 'file' | 'image' | 'code'; + constructor(uri: string, file: File, type?: 'file' | 'image' | 'code'); +} + +export declare class CodeReference extends FileReference { + bind(): Promise; + bindModule(): Promise; + getCode(): Promise; +} + +export type CodeReferenceInput = string | CodeReference; +export type PipelineIf = string | CodeReference | undefined; +export type PipelineTransform = string | CodeReference | undefined; + +export interface FunctionCall { + type: 'function-call'; + name: string; + args: unknown; + meta: unknown; +} +export interface FunctionResponse { + type: 'function-response'; + call: FunctionCall; + response: unknown; +} +export type FunctionTool = FunctionCall | FunctionResponse; + +export interface MetaMessagePart { + type: 'meta'; + title: string; + icon: 'thinking' | 'search' | 'code' | 'other'; + message: string; + data?: unknown; +} +export type MetaProviderOutputPart = MetaMessagePart | FunctionCall | FunctionResponse; + +export type PromptPart = { text: string } | { file: File } | FunctionTool; +export type MultiPartPrompt = PromptPart[]; +export interface RolePromptPart { + role: 'user' | 'assistant' | 'system'; + content: MultiPartPrompt; +} +export type ConversationPrompt = RolePromptPart[]; +export type ProviderOutputPart = string | FileReference | MetaProviderOutputPart; +export type ProviderOutput = string | ProviderOutputPart[]; + +export interface JavascriptAssertionArgs { + code: string | CodeReference; + row?: boolean; +} +export interface JavascriptAssertionResult { + pass: boolean; + message?: string; + visuals?: (string | Blob)[]; + outputs?: Record; +} + +export type ProviderId = string | CodeReference; +export interface NormalizedProviderConfig extends Record { + mimeTypes?: string[]; +} +export interface ProviderDefinition { + id: ProviderId; + labels?: string[]; + config?: NormalizedProviderConfig; + env?: string[]; + prompts?: string[]; +} +export type Provider = string | CodeReference | ProviderDefinition; + +export type Var = unknown; +export type VarSet = Record; + +export interface AssertionResult { + pass: boolean; + message?: string; + visuals?: (string | FileReference)[]; + outputs?: Record; + id?: string; +} + +export interface TokenUsage { + inputTokens?: number; + outputTokens?: number; + totalTokens?: number; + costDollars?: number; +} + +export interface TestOutput { + rawPrompt: unknown; + rawOutput?: unknown; + output?: ProviderOutput; + latencyMillis?: number; + tokenUsage?: TokenUsage; + error?: string; +} + +export interface NormalizedPipelineStep { + id: string; + prompt?: string; + outputAs?: string; + if?: string | CodeReference; + deps?: string[]; + providerLabel?: string; + session?: string | boolean; + functionCalls?: 'loop' | 'once' | 'never'; + transform?: string | CodeReference; + state?: string[]; +} +export interface NormalizedPipelinePrompt { + $pipeline: NormalizedPipelineStep[]; +} +export type NormalizedPrompt = + | string + | { prompt: string; providerLabel?: string } + | NormalizedPipelinePrompt; + +export interface HistoryItem { + id: string; + prompt: ConversationPrompt; + output: ProviderOutput; +} +export interface PipelineVars extends VarSet { + $output: ProviderOutput | null; + $history: HistoryItem[]; + $state?: VarSet; + $args?: unknown; +} + +export type TransformOutput = string | (string | Blob | MetaProviderOutputPart)[]; +export interface TransformResult { + vars?: VarSet; + output?: TransformOutput; +} + +export interface ProviderConfigContext { + env: Record; + config: Record; +} + +export type MaybePromise = T | Promise; + +export type PromptTransformHandler = ( + output: ProviderOutput, + context: { vars: PipelineVars }, +) => MaybePromise; + +export type PromptIfHandler = (vars: PipelineVars) => MaybePromise; + +export type ProviderPrepareHandler = ( + prompt: ConversationPrompt, + context: ProviderConfigContext, +) => MaybePromise; + +export type ProviderRunOutput = string | (string | Blob | MetaProviderOutputPart)[]; + +export type ProviderRunHandler = ( + key: unknown, + context: ProviderConfigContext, +) => MaybePromise; + +export type ProviderEnvHandler = () => MaybePromise; + +export interface AssertionCellContext { + provider: { id: string | CodeReference | null; labeled?: Record }; + prompt: NormalizedPrompt; +} +export interface AssertionRowContext { + prompts: NormalizedPrompt[]; +} +export type AssertionCellHandler = ( + output: ProviderOutput, + context: AssertionCellContext, +) => MaybePromise; +export type AssertionRowHandler = ( + results: TestOutput[], + context: AssertionRowContext, +) => MaybePromise; diff --git a/src/lib/storage/runGenerators.ts b/src/lib/storage/runGenerators.ts index c007613..a9b8f94 100644 --- a/src/lib/storage/runGenerators.ts +++ b/src/lib/storage/runGenerators.ts @@ -4,7 +4,7 @@ import { getFileExtension } from '$lib/utils/path'; import { CodeReference, toCodeReference } from './CodeReference'; import { FileReference } from './FileReference'; -interface Generator { +export interface Generator { '=gen': string | CodeReference; args?: unknown[]; } diff --git a/src/lib/storage/types.ts b/src/lib/storage/types.ts index c5db971..23a503e 100644 --- a/src/lib/storage/types.ts +++ b/src/lib/storage/types.ts @@ -35,6 +35,7 @@ export const fsProviderSchema = z.union([ z.instanceof(CodeReference), fsExpandedProviderSchema, ]); +export type FsProvider = z.infer; export const fsConvoPromptSchema = z.array( z.union([ diff --git a/src/routes/documentation/+page.md b/src/routes/documentation/+page.md index b3ac19f..b9023dd 100644 --- a/src/routes/documentation/+page.md +++ b/src/routes/documentation/+page.md @@ -10,6 +10,8 @@ _**Want AI to write your evals?** Check out [our llms.txt](/llms.txt). You can c _**Looking for a reference for configuration files?** Check [here](/documentation/config)._ +_**Need the full CodeReference TypeScript definitions?** Grab the copy/download-ready file [here](/documentation/code-reference-types)._ + First, create a folder where your evaluation data and output will be stored. Create an `evals.yaml` file inside it and add the following: ```yaml diff --git a/src/routes/documentation/code-reference-types/+page.svelte b/src/routes/documentation/code-reference-types/+page.svelte new file mode 100644 index 0000000..f762205 --- /dev/null +++ b/src/routes/documentation/code-reference-types/+page.svelte @@ -0,0 +1,21 @@ + + + + CodeReference Types + + +
+

CodeReference Types

+

+ Copy the TypeScript definitions below or + download the file + to keep all CodeReference-related types in sync with your project. This file is generated directly + from the source types, so it stays aligned as we evolve the API. +

+ +
{codeReferenceTypes}
+
diff --git a/src/routes/documentation/code-reference-types/types/+server.ts b/src/routes/documentation/code-reference-types/types/+server.ts new file mode 100644 index 0000000..ddb7e08 --- /dev/null +++ b/src/routes/documentation/code-reference-types/types/+server.ts @@ -0,0 +1,11 @@ +import type { RequestHandler } from '@sveltejs/kit'; +import codeReferenceTypes from '$lib/documentation/codeReferenceTypes.ts?raw'; + +export const GET: RequestHandler = () => { + return new Response(codeReferenceTypes, { + headers: { + 'Content-Type': 'text/plain; charset=utf-8', + 'Content-Disposition': 'attachment; filename="code-reference-types.ts"', + }, + }); +};