diff --git a/.changeset/fal-typed-video-duration.md b/.changeset/fal-typed-video-duration.md new file mode 100644 index 0000000000..a96db2d19e --- /dev/null +++ b/.changeset/fal-typed-video-duration.md @@ -0,0 +1,14 @@ +--- +'@tanstack/ai-fal': minor +--- + +Add per-model typed durations for fal video generation. + +`generateVideo({ duration })` is now typed from `@fal-ai/client`'s +`EndpointTypeMap` for the selected model (e.g. `'5' | '10'` on Kling, +`'4s' | '6s' | '8s'` on Veo3). Popular models also implement +`availableDurations()` / `snapDuration()`. + +**Breaking:** callers passing `duration: ` to fal video models must +either pass the model's duration union directly or call +`adapter.snapDuration(seconds)`. diff --git a/docs/adapters/fal.md b/docs/adapters/fal.md index 46ccc246c2..8c7d3ed9ce 100644 --- a/docs/adapters/fal.md +++ b/docs/adapters/fal.md @@ -160,7 +160,7 @@ size: "landscape_16_9" ## Video Generation (Experimental) -> **Note:** Video generation is an experimental feature and may change in future releases. In particular, this version of the adapter does not map the duration paramater +> **Note:** Video generation is an experimental feature and may change in future releases. Video generation uses a queue-based workflow: submit a job, poll for status, then retrieve the video URL when complete. @@ -184,9 +184,7 @@ const job = await generateVideo({ adapter, prompt: "A timelapse of a flower blooming", size: "16:9", - modelOptions: { - duration: "5", - }, + duration: "5", }); // 2. Poll for status @@ -207,14 +205,43 @@ import { falVideo } from "@tanstack/ai-fal"; const job = await generateVideo({ adapter: falVideo("fal-ai/kling-video/v2.6/pro/image-to-video"), prompt: "Animate this scene with gentle wind", + duration: "5", modelOptions: { start_image_url: "https://example.com/image.jpg", generate_audio: true, - duration: "5", }, }); ``` +`duration` is typed per model from `@fal-ai/client`'s `EndpointTypeMap`. Popular models also implement `availableDurations()` / `snapDuration()` for UI sliders: + +| Model | `duration` type | `availableDurations()` | +| --- | --- | --- | +| `fal-ai/kling-video/v1.6/{standard,pro}/text-to-video` | `'5' \| '10'` | discrete | +| `fal-ai/pika/v2.2/text-to-video` | `'5' \| '10'` | discrete | +| `fal-ai/luma-dream-machine/ray-2` | `'5s' \| '9s'` | discrete | +| `fal-ai/veo3` / `fal-ai/veo3/image-to-video` | `'4s' \| '6s' \| '8s'` | discrete | +| `fal-ai/wan-25-preview/text-to-video` | `'2'` … `'15'` | discrete | +| `fal-ai/minimax/video-01` | not accepted | `{ kind: 'none' }` | +| `fal-ai/hunyuan-video-v1.5/text-to-video` | not accepted (`num_frames`) | `{ kind: 'none' }` | + +```typescript +import { generateVideo } from "@tanstack/ai"; +import { falVideo } from "@tanstack/ai-fal"; + +const adapter = falVideo("fal-ai/veo3"); +adapter.availableDurations(); // { kind: 'discrete', values: ['4s', '6s', '8s'] } +adapter.snapDuration(7); // '6s' + +await generateVideo({ + adapter, + prompt: "A timelapse of a city skyline at dusk", + duration: adapter.snapDuration(7), +}); +``` + +Uncurated models still type `duration` from the SDK when the endpoint declares the field, but `availableDurations()` returns `{ kind: 'none' }` until they are added to the runtime map. + ## Text-to-Speech Text-to-speech uses `falSpeech()` with the `generateSpeech()` activity. The adapter fetches the generated audio from fal's CDN and returns it as base64-encoded data to match the `TTSResult` contract. diff --git a/docs/media/video-generation.md b/docs/media/video-generation.md index f4b36b6898..24e3e7c647 100644 --- a/docs/media/video-generation.md +++ b/docs/media/video-generation.md @@ -590,6 +590,8 @@ await generateVideo({ Adapters that haven't declared a per-model duration map keep the plain `duration?: number` typing, return `{ kind: 'none' }` from `availableDurations()`, and return `undefined` from `snapDuration()`. +fal is the exception: `duration` is typed from `@fal-ai/client`'s +`EndpointTypeMap` even when the runtime map has no entry. > **Note:** The video URL returned for Veo jobs is served by the Gemini > Files API and requires your API key to download (send it as an @@ -775,6 +777,28 @@ A bare `size: '720p'` is valid on fal and throws on BytePlus, which follows the The mode also moves: fal encodes it in the endpoint id (`fal-ai/bytedance/seedance/v1/pro/image-to-video` vs `.../reference-to-video`), while BytePlus takes one model id and infers the mode from the prompt parts you attach. Neither is configurable — it follows each provider's own API. +#### fal.ai Model Options + +`duration` is typed per endpoint from `@fal-ai/client`. Popular models also +implement `availableDurations()` / `snapDuration()` (Kling/Pika `'5' | '10'`, +Luma `'5s' | '9s'`, Veo3 `'4s' | '6s' | '8s'`, WAN `'2'`…`'15'`). Models with +no duration field (Minimax, Hunyuan) reject the option. See the +[fal adapter](../adapters/fal) for the full table. + +```typescript ignore +import { generateVideo } from '@tanstack/ai' +import { falVideo } from '@tanstack/ai-fal' + +const adapter = falVideo('fal-ai/veo3') +adapter.availableDurations() // { kind: 'discrete', values: ['4s', '6s', '8s'] } + +await generateVideo({ + adapter, + prompt: 'A timelapse of a city skyline at dusk', + duration: adapter.snapDuration(7), // '6s' +}) +``` + ### Response Types > **Note:** The interfaces below are the underlying adapter-level types. The `getVideoJobStatus()` helper returns a single merged object, `{ status, progress?, url?, error?, usage? }` — it does not return `jobId` or `expiresAt`. diff --git a/examples/ts-react-media/src/lib/server-functions.ts b/examples/ts-react-media/src/lib/server-functions.ts index c1e08dc933..b406cbf91c 100644 --- a/examples/ts-react-media/src/lib/server-functions.ts +++ b/examples/ts-react-media/src/lib/server-functions.ts @@ -293,23 +293,17 @@ function videoStreamForModel(data: VideoRequest): AsyncIterable { adapter: falVideo('fal-ai/kling-video/v3/pro/text-to-video'), prompt: asTextPrompt(data.prompt), size: '16:9', - modelOptions: { - duration: '5', - }, + duration: '5', }) } case 'fal-ai/veo3.1': { - // NOTE pass aspect ratio, resolution, and duration in model options - // This makes use of existing types and avoids type errors return generateVideo({ stream: true, pollingInterval: VIDEO_POLL_INTERVAL_MS, adapter: falVideo('fal-ai/veo3.1'), prompt: asTextPrompt(data.prompt), size: '16:9_1080p', - modelOptions: { - duration: '4s', - }, + duration: '4s', }) } case 'xai/grok-imagine-video/text-to-video': { @@ -369,9 +363,9 @@ function videoStreamForModel(data: VideoRequest): AsyncIterable { pollingInterval: VIDEO_POLL_INTERVAL_MS, adapter: falVideo('fal-ai/kling-video/v3/pro/image-to-video'), prompt: asImageToVideoPrompt(data.prompt), + duration: '5', modelOptions: { generate_audio: true, - duration: '5', }, }) } @@ -382,9 +376,7 @@ function videoStreamForModel(data: VideoRequest): AsyncIterable { adapter: falVideo('fal-ai/veo3.1/image-to-video'), prompt: asImageToVideoPrompt(data.prompt), size: '16:9_1080p', - modelOptions: { - duration: '4s', - }, + duration: '4s', }) } case 'xai/grok-imagine-video/image-to-video': { diff --git a/packages/ai-fal/src/adapters/video.ts b/packages/ai-fal/src/adapters/video.ts index ec0f75bf97..dd17ef3318 100644 --- a/packages/ai-fal/src/adapters/video.ts +++ b/packages/ai-fal/src/adapters/video.ts @@ -1,13 +1,17 @@ import { fal } from '@fal-ai/client' import { resolveMediaPrompt } from '@tanstack/ai' -import { BaseVideoAdapter } from '@tanstack/ai/adapters' +import { BaseVideoAdapter, snapToDurationOption } from '@tanstack/ai/adapters' import { configureFalClient, generateId as utilGenerateId, } from '../utils/client' import { buildFalUsage, takeBillableUnits } from '../utils/billing' -import { mapVideoSizeToFalFormat } from '../video/video-provider-options' +import { + getFalVideoDurationOptions, + mapVideoSizeToFalFormat, +} from '../video/video-provider-options' import { mapImageInputsToFalVideoFields } from '../image/image-inputs' +import type { DurationOptions } from '@tanstack/ai/adapters' import type { AudioPart, MediaInputMetadata, @@ -20,6 +24,7 @@ import type { import type { FalModel, FalModelInput, + FalModelVideoDuration, FalModelVideoSize, FalVideoPromptModalitiesFor, FalVideoProviderOptions, @@ -132,7 +137,8 @@ export class FalVideoAdapter extends BaseVideoAdapter< FalVideoProviderOptions, Record>, Record>, - Record> + Record>, + Record> > { override readonly kind = 'video' as const readonly name = 'fal' as const @@ -145,7 +151,8 @@ export class FalVideoAdapter extends BaseVideoAdapter< async createVideoJob( options: VideoGenerationOptions< FalVideoProviderOptions, - FalModelVideoSize + FalModelVideoSize, + FalModelVideoDuration >, ): Promise { const { size, duration, modelOptions, logger } = options @@ -176,7 +183,7 @@ export class FalVideoAdapter extends BaseVideoAdapter< // Media-only prompts omit the prompt field rather than sending an // empty string (e.g. pure image-to-video endpoints). ...(resolved.text ? { prompt: resolved.text } : {}), - ...(duration ? { duration } : {}), + ...(duration !== undefined ? { duration } : {}), } as FalModelInput // Submit to queue and get request ID. Request-specific abortSignal only — @@ -199,6 +206,22 @@ export class FalVideoAdapter extends BaseVideoAdapter< } } + override availableDurations(): DurationOptions< + FalModelVideoDuration + > { + return getFalVideoDurationOptions(this.model) as DurationOptions< + FalModelVideoDuration + > + } + + override snapDuration( + seconds: number, + ): FalModelVideoDuration | undefined { + return snapToDurationOption(seconds, this.availableDurations()) as + | FalModelVideoDuration + | undefined + } + async getVideoStatus(jobId: string): Promise { const statusResponse = (await fal.queue.status(this.model, { requestId: jobId, diff --git a/packages/ai-fal/src/index.ts b/packages/ai-fal/src/index.ts index d4a73058f0..7351ef11db 100644 --- a/packages/ai-fal/src/index.ts +++ b/packages/ai-fal/src/index.ts @@ -46,6 +46,7 @@ export { type FalModelOutput, type FalModelImageSize, type FalModelVideoSize, + type FalModelVideoDuration, } from './model-meta' // ============================================================================ // Utils diff --git a/packages/ai-fal/src/model-meta.ts b/packages/ai-fal/src/model-meta.ts index c6f4083566..04a888a29f 100644 --- a/packages/ai-fal/src/model-meta.ts +++ b/packages/ai-fal/src/model-meta.ts @@ -144,6 +144,31 @@ export type FalModelVideoSizeInput = : never : { aspect_ratio?: string; resolution?: string } +/** + * Extract the `duration` field type from a fal video model's input. + * Falls back to `string | number | undefined` for unknown models. + * + * Shapes seen in the wild: + * - `'5' | '10'` (Kling, Pika): discrete numeric strings + * - `'5s' | '9s'` (Luma): keyword strings with unit + * - `'4s' | '6s' | '8s'` (Veo3 via FAL): keyword strings + * - `'2' | … | '15'` (WAN-25): discrete-range numeric strings + * - never (Minimax, Hunyuan): no duration field + */ +export type FalModelVideoDuration = + TModel extends keyof EndpointTypeMap + ? 'duration' extends keyof EndpointTypeMap[TModel]['input'] + ? Extract< + NonNullable< + EndpointTypeMap[TModel]['input'] extends { duration?: infer D } + ? D + : never + >, + string | number + > + : undefined + : string | number | undefined + /** * Prompt input modalities for a fal image endpoint, derived from the SDK's * endpoint input type: an endpoint accepts image prompt parts exactly when diff --git a/packages/ai-fal/src/video/video-provider-options.ts b/packages/ai-fal/src/video/video-provider-options.ts index 7991684469..8f79b94e06 100644 --- a/packages/ai-fal/src/video/video-provider-options.ts +++ b/packages/ai-fal/src/video/video-provider-options.ts @@ -1,3 +1,4 @@ +import type { DurationOptions } from '@tanstack/ai/adapters' import type { FalModelVideoSize, FalModelVideoSizeInput } from '../model-meta' export function mapVideoSizeToFalFormat( @@ -22,3 +23,67 @@ export function mapVideoSizeToFalFormat( return { resolution: size } as FalModelVideoSizeInput } + +/** + * Curated map of per-model duration options for popular fal.ai video models. + * Values were sourced from `@fal-ai/client`'s `EndpointTypeMap` input types. + * + * Models not listed here fall back to `{ kind: 'none' }` — honest "we don't + * know" rather than guessing. The type-level `FalModelVideoDuration` + * still derives from the SDK types so autocomplete works for unknown models. + */ +const FAL_VIDEO_DURATIONS: Readonly< + Record> +> = { + 'fal-ai/kling-video/v1.6/standard/text-to-video': { + kind: 'discrete', + values: ['5', '10'], + }, + 'fal-ai/kling-video/v1.6/pro/text-to-video': { + kind: 'discrete', + values: ['5', '10'], + }, + 'fal-ai/pika/v2.2/text-to-video': { + kind: 'discrete', + values: ['5', '10'], + }, + 'fal-ai/luma-dream-machine/ray-2': { + kind: 'discrete', + values: ['5s', '9s'], + }, + 'fal-ai/veo3': { + kind: 'discrete', + values: ['4s', '6s', '8s'], + }, + 'fal-ai/veo3/image-to-video': { + kind: 'discrete', + values: ['4s', '6s', '8s'], + }, + 'fal-ai/wan-25-preview/text-to-video': { + kind: 'discrete', + values: [ + '2', + '3', + '4', + '5', + '6', + '7', + '8', + '9', + '10', + '11', + '12', + '13', + '14', + '15', + ], + }, + 'fal-ai/minimax/video-01': { kind: 'none' }, + 'fal-ai/hunyuan-video-v1.5/text-to-video': { kind: 'none' }, +} + +export function getFalVideoDurationOptions( + model: string, +): DurationOptions { + return FAL_VIDEO_DURATIONS[model] ?? { kind: 'none' } +} diff --git a/packages/ai-fal/tests/video-adapter.test.ts b/packages/ai-fal/tests/video-adapter.test.ts index b3778406f5..fa1ac9097b 100644 --- a/packages/ai-fal/tests/video-adapter.test.ts +++ b/packages/ai-fal/tests/video-adapter.test.ts @@ -1,4 +1,4 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { beforeEach, describe, expect, expectTypeOf, it, vi } from 'vitest' import { generateVideo } from '@tanstack/ai' import { resolveDebugOption } from '@tanstack/ai/adapter-internals' @@ -109,7 +109,7 @@ describe('Fal Video Adapter', () => { }) }) - it('includes duration option', async () => { + it('includes duration option using the model-typed keyword', async () => { mockQueueSubmit.mockResolvedValueOnce({ request_id: 'job-789', }) @@ -119,7 +119,8 @@ describe('Fal Video Adapter', () => { await generateVideo({ adapter: adapter, prompt: 'A time lapse of a sunset', - duration: 10, + // veo3/image-to-video accepts '4s' | '6s' | '8s' + duration: '8s', modelOptions: { image_url: 'https://example.com/sunset.jpg', }, @@ -127,10 +128,26 @@ describe('Fal Video Adapter', () => { const [, options] = mockQueueSubmit.mock.calls[0]! expect(options.input).toMatchObject({ - duration: 10, + duration: '8s', }) }) + it('omits duration for kind: none models (Minimax)', async () => { + mockQueueSubmit.mockResolvedValueOnce({ + request_id: 'job-mini', + }) + + const adapter = falVideo('fal-ai/minimax/video-01', { apiKey: 'test' }) + + await generateVideo({ + adapter, + prompt: 'A fox running through snow', + }) + + const [, options] = mockQueueSubmit.mock.calls[0]! + expect(options.input).not.toHaveProperty('duration') + }) + it('converts size with aspect_ratio and resolution', async () => { mockQueueSubmit.mockResolvedValueOnce({ request_id: 'job-ar', @@ -259,6 +276,51 @@ describe('Fal Video Adapter', () => { }) }) + describe('availableDurations / snapDuration', () => { + it('returns discrete keyword durations for Veo3', () => { + const adapter = falVideo('fal-ai/veo3', { apiKey: 'test' }) + expect(adapter.availableDurations()).toEqual({ + kind: 'discrete', + values: ['4s', '6s', '8s'], + }) + expect(adapter.snapDuration(7)).toBe('6s') + expect(adapter.snapDuration(9)).toBe('8s') + }) + + it('returns discrete numeric durations for Kling', () => { + const adapter = falVideo( + 'fal-ai/kling-video/v1.6/standard/text-to-video', + { apiKey: 'test' }, + ) + expect(adapter.availableDurations()).toEqual({ + kind: 'discrete', + values: ['5', '10'], + }) + expect(adapter.snapDuration(7)).toBe('5') + expect(adapter.snapDuration(8)).toBe('10') + }) + + it('returns kind: none for Minimax (no duration field)', () => { + const adapter = falVideo('fal-ai/minimax/video-01', { apiKey: 'test' }) + expect(adapter.availableDurations()).toEqual({ kind: 'none' }) + expect(adapter.snapDuration(7)).toBeUndefined() + }) + + it('falls back to kind: none for uncurated models', () => { + const adapter = falVideo('fal-ai/some-unknown-video-model', { + apiKey: 'test', + }) + expect(adapter.availableDurations()).toEqual({ kind: 'none' }) + }) + + it('types duration as the model-specific union at compile time', () => { + const veo3 = falVideo('fal-ai/veo3', { apiKey: 'test' }) + expectTypeOf(veo3.snapDuration).returns.toEqualTypeOf< + '4s' | '6s' | '8s' | undefined + >() + }) + }) + describe('getVideoStatus', () => { it('returns pending status for queued jobs', async () => { mockQueueStatus.mockResolvedValueOnce({ diff --git a/packages/ai/skills/ai-core/media-generation/SKILL.md b/packages/ai/skills/ai-core/media-generation/SKILL.md index e464a5f787..5fbcaf28cd 100644 --- a/packages/ai/skills/ai-core/media-generation/SKILL.md +++ b/packages/ai/skills/ai-core/media-generation/SKILL.md @@ -542,7 +542,10 @@ aspect-ratio size template like `'16:9_720p'`, integer durations 1-15s, reports `usage.unitsBilled` seconds and exact `usage.cost`), `byteplusVideo(...)` (Seedance — aspect-ratio size template like `'16:9_720p'`, durations 4-15s on the 2.0 family, 4-12s on 1.5-pro, 2-12s on the 1.0-pro models; reads `ARK_API_KEY`), and -`falVideo(...)` (hosted models, see cost tracking below). +`falVideo(...)` (hosted models; `duration` typed from `@fal-ai/client`'s +`EndpointTypeMap` — `'5' | '10'` on Kling/Pika, `'4s' | '6s' | '8s'` on Veo3, +`'5s' | '9s'` on Luma; `availableDurations()` / `snapDuration()` on the curated +set; see cost tracking below). > **Seedance option applicability is per model and enforced server-side** — > Ark returns a 400 for an inapplicable field rather than ignoring it.