From a7bd8a49c1f92de968327b27d6c6b9b6b37c9b18 Mon Sep 17 00:00:00 2001 From: Tom Beckenham <34339192+tombeckenham@users.noreply.github.com> Date: Thu, 13 Aug 2026 16:38:57 +1000 Subject: [PATCH 1/3] feat(ai-fal): per-model typed durations for video generation (#534) Port the FAL duration map onto current main. generateVideo({ duration }) is now typed from EndpointTypeMap, and popular models implement availableDurations() / snapDuration(). --- .changeset/fal-typed-video-duration.md | 14 ++++ docs/adapters/fal.md | 34 +++++++-- docs/media/video-generation.md | 24 +++++++ .../src/lib/server-functions.ts | 16 ++--- packages/ai-fal/src/adapters/video.ts | 33 +++++++-- packages/ai-fal/src/index.ts | 1 + packages/ai-fal/src/model-meta.ts | 25 +++++++ .../src/video/video-provider-options.ts | 65 +++++++++++++++++ packages/ai-fal/tests/video-adapter.test.ts | 70 +++++++++++++++++-- .../skills/ai-core/media-generation/SKILL.md | 5 ++ 10 files changed, 261 insertions(+), 26 deletions(-) create mode 100644 .changeset/fal-typed-video-duration.md diff --git a/.changeset/fal-typed-video-duration.md b/.changeset/fal-typed-video-duration.md new file mode 100644 index 0000000000..a96db2d19e --- /dev/null +++ b/.changeset/fal-typed-video-duration.md @@ -0,0 +1,14 @@ +--- +'@tanstack/ai-fal': minor +--- + +Add per-model typed durations for fal video generation. + +`generateVideo({ duration })` is now typed from `@fal-ai/client`'s +`EndpointTypeMap` for the selected model (e.g. `'5' | '10'` on Kling, +`'4s' | '6s' | '8s'` on Veo3). Popular models also implement +`availableDurations()` / `snapDuration()`. + +**Breaking:** callers passing `duration: ` to fal video models must +either pass the model's duration union directly or call +`adapter.snapDuration(seconds)`. diff --git a/docs/adapters/fal.md b/docs/adapters/fal.md index 46ccc246c2..655052979d 100644 --- a/docs/adapters/fal.md +++ b/docs/adapters/fal.md @@ -160,7 +160,7 @@ size: "landscape_16_9" ## Video Generation (Experimental) -> **Note:** Video generation is an experimental feature and may change in future releases. In particular, this version of the adapter does not map the duration paramater +> **Note:** Video generation is an experimental feature and may change in future releases. Video generation uses a queue-based workflow: submit a job, poll for status, then retrieve the video URL when complete. @@ -184,9 +184,7 @@ const job = await generateVideo({ adapter, prompt: "A timelapse of a flower blooming", size: "16:9", - modelOptions: { - duration: "5", - }, + duration: "5", }); // 2. Poll for status @@ -207,14 +205,40 @@ import { falVideo } from "@tanstack/ai-fal"; const job = await generateVideo({ adapter: falVideo("fal-ai/kling-video/v2.6/pro/image-to-video"), prompt: "Animate this scene with gentle wind", + duration: "5", modelOptions: { start_image_url: "https://example.com/image.jpg", generate_audio: true, - duration: "5", }, }); ``` +`duration` is typed per model from `@fal-ai/client`'s `EndpointTypeMap`. Popular models also implement `availableDurations()` / `snapDuration()` for UI sliders: + +| Model | `duration` type | `availableDurations()` | +| --- | --- | --- | +| `fal-ai/kling-video/v1.6/{standard,pro}/text-to-video` | `'5' \| '10'` | discrete | +| `fal-ai/pika/v2.2/text-to-video` | `'5' \| '10'` | discrete | +| `fal-ai/luma-dream-machine/ray-2` | `'5s' \| '9s'` | discrete | +| `fal-ai/veo3` / `fal-ai/veo3/image-to-video` | `'4s' \| '6s' \| '8s'` | discrete | +| `fal-ai/wan-25-preview/text-to-video` | `'2'` … `'15'` | discrete | +| `fal-ai/minimax/video-01` | not accepted | `{ kind: 'none' }` | +| `fal-ai/hunyuan-video-v1.5/text-to-video` | not accepted (`num_frames`) | `{ kind: 'none' }` | + +```typescript +const adapter = falVideo("fal-ai/veo3"); +adapter.availableDurations(); // { kind: 'discrete', values: ['4s', '6s', '8s'] } +adapter.snapDuration(7); // '6s' + +await generateVideo({ + adapter, + prompt: "A timelapse of a city skyline at dusk", + duration: adapter.snapDuration(7), +}); +``` + +Uncurated models still type `duration` from the SDK when the endpoint declares the field, but `availableDurations()` returns `{ kind: 'none' }` until they are added to the runtime map. + ## Text-to-Speech Text-to-speech uses `falSpeech()` with the `generateSpeech()` activity. The adapter fetches the generated audio from fal's CDN and returns it as base64-encoded data to match the `TTSResult` contract. diff --git a/docs/media/video-generation.md b/docs/media/video-generation.md index f2865b1c08..973ff1bc16 100644 --- a/docs/media/video-generation.md +++ b/docs/media/video-generation.md @@ -600,6 +600,8 @@ await generateVideo({ Adapters that haven't declared a per-model duration map keep the plain `duration?: number` typing, return `{ kind: 'none' }` from `availableDurations()`, and return `undefined` from `snapDuration()`. +fal is the exception: `duration` is typed from `@fal-ai/client`'s +`EndpointTypeMap` even when the runtime map has no entry. > **Note:** The video URL returned for Veo jobs is served by the Gemini > Files API and requires your API key to download (send it as an @@ -888,6 +890,28 @@ Two OpenRouter-specific behaviors to know about: - **Cost is reported on completion.** The gateway reports the real billed cost for the job; it's surfaced as `usage.cost` on the completed result. +#### fal.ai Model Options + +`duration` is typed per endpoint from `@fal-ai/client`. Popular models also +implement `availableDurations()` / `snapDuration()` (Kling/Pika `'5' | '10'`, +Luma `'5s' | '9s'`, Veo3 `'4s' | '6s' | '8s'`, WAN `'2'`…`'15'`). Models with +no duration field (Minimax, Hunyuan) reject the option. See the +[fal adapter](../adapters/fal) for the full table. + +```typescript ignore +import { generateVideo } from '@tanstack/ai' +import { falVideo } from '@tanstack/ai-fal' + +const adapter = falVideo('fal-ai/veo3') +adapter.availableDurations() // { kind: 'discrete', values: ['4s', '6s', '8s'] } + +await generateVideo({ + adapter, + prompt: 'A timelapse of a city skyline at dusk', + duration: adapter.snapDuration(7), // '6s' +}) +``` + ### Response Types > **Note:** The interfaces below are the underlying adapter-level types. The `getVideoJobStatus()` helper returns a single merged object, `{ status, progress?, url?, error?, usage? }` — it does not return `jobId` or `expiresAt`. diff --git a/examples/ts-react-media/src/lib/server-functions.ts b/examples/ts-react-media/src/lib/server-functions.ts index c01164f597..40680cd0d2 100644 --- a/examples/ts-react-media/src/lib/server-functions.ts +++ b/examples/ts-react-media/src/lib/server-functions.ts @@ -305,23 +305,17 @@ function videoStreamForModel(data: VideoRequest): AsyncIterable { adapter: falVideo('fal-ai/kling-video/v3/pro/text-to-video'), prompt: asTextPrompt(data.prompt), size: '16:9', - modelOptions: { - duration: '5', - }, + duration: '5', }) } case 'fal-ai/veo3.1': { - // NOTE pass aspect ratio, resolution, and duration in model options - // This makes use of existing types and avoids type errors return generateVideo({ stream: true, pollingInterval: VIDEO_POLL_INTERVAL_MS, adapter: falVideo('fal-ai/veo3.1'), prompt: asTextPrompt(data.prompt), size: '16:9_1080p', - modelOptions: { - duration: '4s', - }, + duration: '4s', }) } case 'xai/grok-imagine-video/text-to-video': { @@ -393,9 +387,9 @@ function videoStreamForModel(data: VideoRequest): AsyncIterable { pollingInterval: VIDEO_POLL_INTERVAL_MS, adapter: falVideo('fal-ai/kling-video/v3/pro/image-to-video'), prompt: asImageToVideoPrompt(data.prompt), + duration: '5', modelOptions: { generate_audio: true, - duration: '5', }, }) } @@ -406,9 +400,7 @@ function videoStreamForModel(data: VideoRequest): AsyncIterable { adapter: falVideo('fal-ai/veo3.1/image-to-video'), prompt: asImageToVideoPrompt(data.prompt), size: '16:9_1080p', - modelOptions: { - duration: '4s', - }, + duration: '4s', }) } case 'xai/grok-imagine-video/image-to-video': { diff --git a/packages/ai-fal/src/adapters/video.ts b/packages/ai-fal/src/adapters/video.ts index ec0f75bf97..dd17ef3318 100644 --- a/packages/ai-fal/src/adapters/video.ts +++ b/packages/ai-fal/src/adapters/video.ts @@ -1,13 +1,17 @@ import { fal } from '@fal-ai/client' import { resolveMediaPrompt } from '@tanstack/ai' -import { BaseVideoAdapter } from '@tanstack/ai/adapters' +import { BaseVideoAdapter, snapToDurationOption } from '@tanstack/ai/adapters' import { configureFalClient, generateId as utilGenerateId, } from '../utils/client' import { buildFalUsage, takeBillableUnits } from '../utils/billing' -import { mapVideoSizeToFalFormat } from '../video/video-provider-options' +import { + getFalVideoDurationOptions, + mapVideoSizeToFalFormat, +} from '../video/video-provider-options' import { mapImageInputsToFalVideoFields } from '../image/image-inputs' +import type { DurationOptions } from '@tanstack/ai/adapters' import type { AudioPart, MediaInputMetadata, @@ -20,6 +24,7 @@ import type { import type { FalModel, FalModelInput, + FalModelVideoDuration, FalModelVideoSize, FalVideoPromptModalitiesFor, FalVideoProviderOptions, @@ -132,7 +137,8 @@ export class FalVideoAdapter extends BaseVideoAdapter< FalVideoProviderOptions, Record>, Record>, - Record> + Record>, + Record> > { override readonly kind = 'video' as const readonly name = 'fal' as const @@ -145,7 +151,8 @@ export class FalVideoAdapter extends BaseVideoAdapter< async createVideoJob( options: VideoGenerationOptions< FalVideoProviderOptions, - FalModelVideoSize + FalModelVideoSize, + FalModelVideoDuration >, ): Promise { const { size, duration, modelOptions, logger } = options @@ -176,7 +183,7 @@ export class FalVideoAdapter extends BaseVideoAdapter< // Media-only prompts omit the prompt field rather than sending an // empty string (e.g. pure image-to-video endpoints). ...(resolved.text ? { prompt: resolved.text } : {}), - ...(duration ? { duration } : {}), + ...(duration !== undefined ? { duration } : {}), } as FalModelInput // Submit to queue and get request ID. Request-specific abortSignal only — @@ -199,6 +206,22 @@ export class FalVideoAdapter extends BaseVideoAdapter< } } + override availableDurations(): DurationOptions< + FalModelVideoDuration + > { + return getFalVideoDurationOptions(this.model) as DurationOptions< + FalModelVideoDuration + > + } + + override snapDuration( + seconds: number, + ): FalModelVideoDuration | undefined { + return snapToDurationOption(seconds, this.availableDurations()) as + | FalModelVideoDuration + | undefined + } + async getVideoStatus(jobId: string): Promise { const statusResponse = (await fal.queue.status(this.model, { requestId: jobId, diff --git a/packages/ai-fal/src/index.ts b/packages/ai-fal/src/index.ts index d4a73058f0..7351ef11db 100644 --- a/packages/ai-fal/src/index.ts +++ b/packages/ai-fal/src/index.ts @@ -46,6 +46,7 @@ export { type FalModelOutput, type FalModelImageSize, type FalModelVideoSize, + type FalModelVideoDuration, } from './model-meta' // ============================================================================ // Utils diff --git a/packages/ai-fal/src/model-meta.ts b/packages/ai-fal/src/model-meta.ts index c6f4083566..04a888a29f 100644 --- a/packages/ai-fal/src/model-meta.ts +++ b/packages/ai-fal/src/model-meta.ts @@ -144,6 +144,31 @@ export type FalModelVideoSizeInput = : never : { aspect_ratio?: string; resolution?: string } +/** + * Extract the `duration` field type from a fal video model's input. + * Falls back to `string | number | undefined` for unknown models. + * + * Shapes seen in the wild: + * - `'5' | '10'` (Kling, Pika): discrete numeric strings + * - `'5s' | '9s'` (Luma): keyword strings with unit + * - `'4s' | '6s' | '8s'` (Veo3 via FAL): keyword strings + * - `'2' | … | '15'` (WAN-25): discrete-range numeric strings + * - never (Minimax, Hunyuan): no duration field + */ +export type FalModelVideoDuration = + TModel extends keyof EndpointTypeMap + ? 'duration' extends keyof EndpointTypeMap[TModel]['input'] + ? Extract< + NonNullable< + EndpointTypeMap[TModel]['input'] extends { duration?: infer D } + ? D + : never + >, + string | number + > + : undefined + : string | number | undefined + /** * Prompt input modalities for a fal image endpoint, derived from the SDK's * endpoint input type: an endpoint accepts image prompt parts exactly when diff --git a/packages/ai-fal/src/video/video-provider-options.ts b/packages/ai-fal/src/video/video-provider-options.ts index 7991684469..8f79b94e06 100644 --- a/packages/ai-fal/src/video/video-provider-options.ts +++ b/packages/ai-fal/src/video/video-provider-options.ts @@ -1,3 +1,4 @@ +import type { DurationOptions } from '@tanstack/ai/adapters' import type { FalModelVideoSize, FalModelVideoSizeInput } from '../model-meta' export function mapVideoSizeToFalFormat( @@ -22,3 +23,67 @@ export function mapVideoSizeToFalFormat( return { resolution: size } as FalModelVideoSizeInput } + +/** + * Curated map of per-model duration options for popular fal.ai video models. + * Values were sourced from `@fal-ai/client`'s `EndpointTypeMap` input types. + * + * Models not listed here fall back to `{ kind: 'none' }` — honest "we don't + * know" rather than guessing. The type-level `FalModelVideoDuration` + * still derives from the SDK types so autocomplete works for unknown models. + */ +const FAL_VIDEO_DURATIONS: Readonly< + Record> +> = { + 'fal-ai/kling-video/v1.6/standard/text-to-video': { + kind: 'discrete', + values: ['5', '10'], + }, + 'fal-ai/kling-video/v1.6/pro/text-to-video': { + kind: 'discrete', + values: ['5', '10'], + }, + 'fal-ai/pika/v2.2/text-to-video': { + kind: 'discrete', + values: ['5', '10'], + }, + 'fal-ai/luma-dream-machine/ray-2': { + kind: 'discrete', + values: ['5s', '9s'], + }, + 'fal-ai/veo3': { + kind: 'discrete', + values: ['4s', '6s', '8s'], + }, + 'fal-ai/veo3/image-to-video': { + kind: 'discrete', + values: ['4s', '6s', '8s'], + }, + 'fal-ai/wan-25-preview/text-to-video': { + kind: 'discrete', + values: [ + '2', + '3', + '4', + '5', + '6', + '7', + '8', + '9', + '10', + '11', + '12', + '13', + '14', + '15', + ], + }, + 'fal-ai/minimax/video-01': { kind: 'none' }, + 'fal-ai/hunyuan-video-v1.5/text-to-video': { kind: 'none' }, +} + +export function getFalVideoDurationOptions( + model: string, +): DurationOptions { + return FAL_VIDEO_DURATIONS[model] ?? { kind: 'none' } +} diff --git a/packages/ai-fal/tests/video-adapter.test.ts b/packages/ai-fal/tests/video-adapter.test.ts index 79e4969283..8ad47dcb6f 100644 --- a/packages/ai-fal/tests/video-adapter.test.ts +++ b/packages/ai-fal/tests/video-adapter.test.ts @@ -1,4 +1,4 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { beforeEach, describe, expect, expectTypeOf, it, vi } from 'vitest' import { generateVideo } from '@tanstack/ai' import { resolveDebugOption } from '@tanstack/ai/adapter-internals' @@ -109,7 +109,7 @@ describe('Fal Video Adapter', () => { }) }) - it('includes duration option', async () => { + it('includes duration option using the model-typed keyword', async () => { mockQueueSubmit.mockResolvedValueOnce({ request_id: 'job-789', }) @@ -119,7 +119,8 @@ describe('Fal Video Adapter', () => { await generateVideo({ adapter: adapter, prompt: 'A time lapse of a sunset', - duration: 10, + // veo3/image-to-video accepts '4s' | '6s' | '8s' + duration: '8s', modelOptions: { image_url: 'https://example.com/sunset.jpg', }, @@ -127,10 +128,26 @@ describe('Fal Video Adapter', () => { const [, options] = mockQueueSubmit.mock.calls[0]! expect(options.input).toMatchObject({ - duration: 10, + duration: '8s', }) }) + it('omits duration for kind: none models (Minimax)', async () => { + mockQueueSubmit.mockResolvedValueOnce({ + request_id: 'job-mini', + }) + + const adapter = falVideo('fal-ai/minimax/video-01', { apiKey: 'test' }) + + await generateVideo({ + adapter, + prompt: 'A fox running through snow', + }) + + const [, options] = mockQueueSubmit.mock.calls[0]! + expect(options.input).not.toHaveProperty('duration') + }) + it('converts size with aspect_ratio and resolution', async () => { mockQueueSubmit.mockResolvedValueOnce({ request_id: 'job-ar', @@ -259,6 +276,51 @@ describe('Fal Video Adapter', () => { }) }) + describe('availableDurations / snapDuration', () => { + it('returns discrete keyword durations for Veo3', () => { + const adapter = falVideo('fal-ai/veo3', { apiKey: 'test' }) + expect(adapter.availableDurations()).toEqual({ + kind: 'discrete', + values: ['4s', '6s', '8s'], + }) + expect(adapter.snapDuration(7)).toBe('6s') + expect(adapter.snapDuration(9)).toBe('8s') + }) + + it('returns discrete numeric durations for Kling', () => { + const adapter = falVideo( + 'fal-ai/kling-video/v1.6/standard/text-to-video', + { apiKey: 'test' }, + ) + expect(adapter.availableDurations()).toEqual({ + kind: 'discrete', + values: ['5', '10'], + }) + expect(adapter.snapDuration(7)).toBe('5') + expect(adapter.snapDuration(8)).toBe('10') + }) + + it('returns kind: none for Minimax (no duration field)', () => { + const adapter = falVideo('fal-ai/minimax/video-01', { apiKey: 'test' }) + expect(adapter.availableDurations()).toEqual({ kind: 'none' }) + expect(adapter.snapDuration(7)).toBeUndefined() + }) + + it('falls back to kind: none for uncurated models', () => { + const adapter = falVideo('fal-ai/some-unknown-video-model', { + apiKey: 'test', + }) + expect(adapter.availableDurations()).toEqual({ kind: 'none' }) + }) + + it('types duration as the model-specific union at compile time', () => { + const veo3 = falVideo('fal-ai/veo3', { apiKey: 'test' }) + expectTypeOf(veo3.snapDuration).returns.toEqualTypeOf< + '4s' | '6s' | '8s' | undefined + >() + }) + }) + describe('getVideoStatus', () => { it('returns pending status for queued jobs', async () => { mockQueueStatus.mockResolvedValueOnce({ diff --git a/packages/ai/skills/ai-core/media-generation/SKILL.md b/packages/ai/skills/ai-core/media-generation/SKILL.md index 3bd1f6d937..10d94faa6d 100644 --- a/packages/ai/skills/ai-core/media-generation/SKILL.md +++ b/packages/ai/skills/ai-core/media-generation/SKILL.md @@ -566,6 +566,11 @@ aspect-ratio size template like `'16:9_720p'`, durations 4-15s on the 2.0 family 4-12s on 1.5-pro, 2-12s on the 1.0-pro models; reads `ARK_API_KEY`), `openRouterVideo(...)` (OpenRouter's dedicated `POST /api/v1/videos` gateway), and `falVideo(...)` (hosted models, see cost tracking below). +4-12s on 1.5-pro, 2-12s on the 1.0-pro models; reads `ARK_API_KEY`), and +`falVideo(...)` (hosted models; `duration` typed from `@fal-ai/client`'s +`EndpointTypeMap` — `'5' | '10'` on Kling/Pika, `'4s' | '6s' | '8s'` on Veo3, +`'5s' | '9s'` on Luma; `availableDurations()` / `snapDuration()` on the curated +set; see cost tracking below). > **Seedance option applicability is per model and enforced server-side** — > Ark returns a 400 for an inapplicable field rather than ignoring it. From e3973f2813016fd475552df5d95885b37a896362 Mon Sep 17 00:00:00 2001 From: Tom Beckenham <34339192+tombeckenham@users.noreply.github.com> Date: Thu, 13 Aug 2026 16:48:30 +1000 Subject: [PATCH 2/3] docs(fal): make the snapDuration example self-contained for kiira --- docs/adapters/fal.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/adapters/fal.md b/docs/adapters/fal.md index 655052979d..8c7d3ed9ce 100644 --- a/docs/adapters/fal.md +++ b/docs/adapters/fal.md @@ -226,6 +226,9 @@ const job = await generateVideo({ | `fal-ai/hunyuan-video-v1.5/text-to-video` | not accepted (`num_frames`) | `{ kind: 'none' }` | ```typescript +import { generateVideo } from "@tanstack/ai"; +import { falVideo } from "@tanstack/ai-fal"; + const adapter = falVideo("fal-ai/veo3"); adapter.availableDurations(); // { kind: 'discrete', values: ['4s', '6s', '8s'] } adapter.snapDuration(7); // '6s' From 242f7cc9c9fb809789de35f6a3f8abace8a4a026 Mon Sep 17 00:00:00 2001 From: Tom Beckenham <34339192+tombeckenham@users.noreply.github.com> Date: Thu, 20 Aug 2026 19:57:29 +1000 Subject: [PATCH 3/3] fix(ai-fal): type-check the curated duration map and cover the example models - durationMap() identity helper checks each entry against the SDK's duration union for its key; drops both casts in the adapter - curate kling v2.6/v3, veo3.1 (+fast), ltx-2.3 so availableDurations() / snapDuration() work for the models the docs and example use - Omit 'duration' from FalVideoProviderOptions: one way in (top-level) - example: drive fal durations through adapter.snapDuration() - tests: @ts-expect-error at the generateVideo call site, type fallbacks, numeric passthrough for non-SDK models - docs: veo3.1 in samples, compile-time wording, config.json updatedAt --- .changeset/fal-typed-video-duration.md | 2 +- docs/adapters/fal.md | 8 +- docs/config.json | 5 +- docs/media/video-generation.md | 9 +- .../src/lib/server-functions.ts | 48 +++---- packages/ai-fal/src/adapters/video.ts | 8 +- packages/ai-fal/src/model-meta.ts | 9 +- .../src/video/video-provider-options.ts | 117 ++++++++++++------ packages/ai-fal/tests/video-adapter.test.ts | 69 ++++++++++- .../skills/ai-core/media-generation/SKILL.md | 10 +- 10 files changed, 199 insertions(+), 86 deletions(-) diff --git a/.changeset/fal-typed-video-duration.md b/.changeset/fal-typed-video-duration.md index a96db2d19e..523fd83cd4 100644 --- a/.changeset/fal-typed-video-duration.md +++ b/.changeset/fal-typed-video-duration.md @@ -6,7 +6,7 @@ Add per-model typed durations for fal video generation. `generateVideo({ duration })` is now typed from `@fal-ai/client`'s `EndpointTypeMap` for the selected model (e.g. `'5' | '10'` on Kling, -`'4s' | '6s' | '8s'` on Veo3). Popular models also implement +`'4s' | '6s' | '8s'` on Veo 3.1). Popular models also implement `availableDurations()` / `snapDuration()`. **Breaking:** callers passing `duration: ` to fal video models must diff --git a/docs/adapters/fal.md b/docs/adapters/fal.md index 8c7d3ed9ce..ac8f8457d1 100644 --- a/docs/adapters/fal.md +++ b/docs/adapters/fal.md @@ -217,10 +217,12 @@ const job = await generateVideo({ | Model | `duration` type | `availableDurations()` | | --- | --- | --- | -| `fal-ai/kling-video/v1.6/{standard,pro}/text-to-video` | `'5' \| '10'` | discrete | +| `fal-ai/kling-video/v1.6/{standard,pro}/text-to-video`, `fal-ai/kling-video/v2.6/pro/{text,image}-to-video` | `'5' \| '10'` | discrete | +| `fal-ai/kling-video/v3/pro/{text,image}-to-video` | `'3'` … `'15'` | discrete | | `fal-ai/pika/v2.2/text-to-video` | `'5' \| '10'` | discrete | +| `fal-ai/ltx-2.3/{text,image}-to-video` (+ `/fast`) | `'6' \| '8' \| '10'` | discrete | | `fal-ai/luma-dream-machine/ray-2` | `'5s' \| '9s'` | discrete | -| `fal-ai/veo3` / `fal-ai/veo3/image-to-video` | `'4s' \| '6s' \| '8s'` | discrete | +| `fal-ai/veo3.1`, `fal-ai/veo3.1/fast` (+ `/image-to-video`), `fal-ai/veo3` (+ `/image-to-video`) | `'4s' \| '6s' \| '8s'` | discrete | | `fal-ai/wan-25-preview/text-to-video` | `'2'` … `'15'` | discrete | | `fal-ai/minimax/video-01` | not accepted | `{ kind: 'none' }` | | `fal-ai/hunyuan-video-v1.5/text-to-video` | not accepted (`num_frames`) | `{ kind: 'none' }` | @@ -229,7 +231,7 @@ const job = await generateVideo({ import { generateVideo } from "@tanstack/ai"; import { falVideo } from "@tanstack/ai-fal"; -const adapter = falVideo("fal-ai/veo3"); +const adapter = falVideo("fal-ai/veo3.1"); adapter.availableDurations(); // { kind: 'discrete', values: ['4s', '6s', '8s'] } adapter.snapDuration(7); // '6s' diff --git a/docs/config.json b/docs/config.json index 25dd02b10c..a9d2ce4ee0 100644 --- a/docs/config.json +++ b/docs/config.json @@ -478,7 +478,7 @@ "label": "Video Generation", "to": "media/video-generation", "addedAt": "2026-04-15", - "updatedAt": "2026-08-19" + "updatedAt": "2026-08-20" }, { "label": "Generation Hooks", @@ -878,7 +878,8 @@ { "label": "fal.ai", "to": "adapters/fal", - "addedAt": "2026-04-15" + "addedAt": "2026-04-15", + "updatedAt": "2026-08-20" }, { "label": "OpenRouter Adapter", diff --git a/docs/media/video-generation.md b/docs/media/video-generation.md index 973ff1bc16..5a78ffe781 100644 --- a/docs/media/video-generation.md +++ b/docs/media/video-generation.md @@ -893,16 +893,17 @@ Two OpenRouter-specific behaviors to know about: #### fal.ai Model Options `duration` is typed per endpoint from `@fal-ai/client`. Popular models also -implement `availableDurations()` / `snapDuration()` (Kling/Pika `'5' | '10'`, -Luma `'5s' | '9s'`, Veo3 `'4s' | '6s' | '8s'`, WAN `'2'`…`'15'`). Models with -no duration field (Minimax, Hunyuan) reject the option. See the +implement `availableDurations()` / `snapDuration()` (Kling 2.6/Pika `'5' | '10'`, +Kling 3 `'3'`…`'15'`, Luma `'5s' | '9s'`, Veo 3.1 `'4s' | '6s' | '8s'`, WAN +`'2'`…`'15'`). Models with no duration field (Minimax, Hunyuan) type `duration` +as `undefined`, so passing one is a compile error. See the [fal adapter](../adapters/fal) for the full table. ```typescript ignore import { generateVideo } from '@tanstack/ai' import { falVideo } from '@tanstack/ai-fal' -const adapter = falVideo('fal-ai/veo3') +const adapter = falVideo('fal-ai/veo3.1') adapter.availableDurations() // { kind: 'discrete', values: ['4s', '6s', '8s'] } await generateVideo({ diff --git a/examples/ts-react-media/src/lib/server-functions.ts b/examples/ts-react-media/src/lib/server-functions.ts index 40680cd0d2..b72faed030 100644 --- a/examples/ts-react-media/src/lib/server-functions.ts +++ b/examples/ts-react-media/src/lib/server-functions.ts @@ -296,26 +296,31 @@ function videoStreamForModel(data: VideoRequest): AsyncIterable { // Image-to-video models receive the start frame as a prompt part // (role: 'start_frame') — the fal adapter routes it to the endpoint's // start-image field. Text-to-video models take the text prompt only. + // `duration` is typed per endpoint ('5' | '10' on Kling 2.6, '3'…'15' on + // Kling 3, '4s' | '6s' | '8s' on Veo 3.1); `snapDuration(seconds)` picks the + // nearest supported value so a UI slider never has to know the literal. switch (data.model) { // Text-to-video models case 'fal-ai/kling-video/v3/pro/text-to-video': { + const adapter = falVideo('fal-ai/kling-video/v3/pro/text-to-video') return generateVideo({ stream: true, pollingInterval: VIDEO_POLL_INTERVAL_MS, - adapter: falVideo('fal-ai/kling-video/v3/pro/text-to-video'), + adapter, prompt: asTextPrompt(data.prompt), size: '16:9', - duration: '5', + duration: adapter.snapDuration(5), }) } case 'fal-ai/veo3.1': { + const adapter = falVideo('fal-ai/veo3.1') return generateVideo({ stream: true, pollingInterval: VIDEO_POLL_INTERVAL_MS, - adapter: falVideo('fal-ai/veo3.1'), + adapter, prompt: asTextPrompt(data.prompt), size: '16:9_1080p', - duration: '4s', + duration: adapter.snapDuration(4), }) } case 'xai/grok-imagine-video/text-to-video': { @@ -325,9 +330,7 @@ function videoStreamForModel(data: VideoRequest): AsyncIterable { adapter: falVideo('xai/grok-imagine-video/text-to-video'), prompt: asTextPrompt(data.prompt), size: '16:9_720p', - modelOptions: { - duration: 5, - }, + duration: 5, }) } case 'grok-imagine-video': { @@ -372,35 +375,39 @@ function videoStreamForModel(data: VideoRequest): AsyncIterable { }) } case 'fal-ai/ltx-2.3/text-to-video/fast': { + const adapter = falVideo('fal-ai/ltx-2.3/text-to-video/fast') return generateVideo({ stream: true, pollingInterval: VIDEO_POLL_INTERVAL_MS, - adapter: falVideo('fal-ai/ltx-2.3/text-to-video/fast'), + adapter, prompt: asTextPrompt(data.prompt), size: '16:9_2160p', + duration: adapter.snapDuration(6), }) } // Image-to-video models case 'fal-ai/kling-video/v3/pro/image-to-video': { + const adapter = falVideo('fal-ai/kling-video/v3/pro/image-to-video') return generateVideo({ stream: true, pollingInterval: VIDEO_POLL_INTERVAL_MS, - adapter: falVideo('fal-ai/kling-video/v3/pro/image-to-video'), + adapter, prompt: asImageToVideoPrompt(data.prompt), - duration: '5', + duration: adapter.snapDuration(5), modelOptions: { generate_audio: true, }, }) } case 'fal-ai/veo3.1/image-to-video': { + const adapter = falVideo('fal-ai/veo3.1/image-to-video') return generateVideo({ stream: true, pollingInterval: VIDEO_POLL_INTERVAL_MS, - adapter: falVideo('fal-ai/veo3.1/image-to-video'), + adapter, prompt: asImageToVideoPrompt(data.prompt), size: '16:9_1080p', - duration: '4s', + duration: adapter.snapDuration(4), }) } case 'xai/grok-imagine-video/image-to-video': { @@ -410,9 +417,7 @@ function videoStreamForModel(data: VideoRequest): AsyncIterable { adapter: falVideo('xai/grok-imagine-video/image-to-video'), prompt: asImageToVideoPrompt(data.prompt), size: '16:9_720p', - modelOptions: { - duration: 5, - }, + duration: 5, }) } case 'grok-imagine-video-1.5/image-to-video': { @@ -429,12 +434,14 @@ function videoStreamForModel(data: VideoRequest): AsyncIterable { }) } case 'fal-ai/ltx-2.3/image-to-video/fast': { + const adapter = falVideo('fal-ai/ltx-2.3/image-to-video/fast') return generateVideo({ stream: true, pollingInterval: VIDEO_POLL_INTERVAL_MS, - adapter: falVideo('fal-ai/ltx-2.3/image-to-video/fast'), + adapter, prompt: asImageToVideoPrompt(data.prompt), size: '16:9_2160p', + duration: adapter.snapDuration(6), }) } // Gemini Omni Flash (Interactions API, GEMINI_API_KEY). One model @@ -477,11 +484,10 @@ function videoStreamForModel(data: VideoRequest): AsyncIterable { : {}), }) } - // OpenRouter's dedicated async video API (`POST /api/v1/videos`). Unlike - // fal (which takes duration in `modelOptions`), OpenRouter types the - // top-level `duration` per model from its published metadata, and the - // adapter exposes `snapDuration()` to coerce a raw UI seconds value to - // the model's nearest supported duration. + // OpenRouter's dedicated async video API (`POST /api/v1/videos`). Like + // fal, it types the top-level `duration` per model and exposes + // `snapDuration()` to coerce a raw UI seconds value to the model's + // nearest supported duration. case 'bytedance/seedance-2.0': { const adapter = openRouterVideo('bytedance/seedance-2.0') return generateVideo({ diff --git a/packages/ai-fal/src/adapters/video.ts b/packages/ai-fal/src/adapters/video.ts index dd17ef3318..dab045e6c5 100644 --- a/packages/ai-fal/src/adapters/video.ts +++ b/packages/ai-fal/src/adapters/video.ts @@ -209,17 +209,13 @@ export class FalVideoAdapter extends BaseVideoAdapter< override availableDurations(): DurationOptions< FalModelVideoDuration > { - return getFalVideoDurationOptions(this.model) as DurationOptions< - FalModelVideoDuration - > + return getFalVideoDurationOptions(this.model) } override snapDuration( seconds: number, ): FalModelVideoDuration | undefined { - return snapToDurationOption(seconds, this.availableDurations()) as - | FalModelVideoDuration - | undefined + return snapToDurationOption(seconds, this.availableDurations()) } async getVideoStatus(jobId: string): Promise { diff --git a/packages/ai-fal/src/model-meta.ts b/packages/ai-fal/src/model-meta.ts index 04a888a29f..8c6c850632 100644 --- a/packages/ai-fal/src/model-meta.ts +++ b/packages/ai-fal/src/model-meta.ts @@ -146,14 +146,15 @@ export type FalModelVideoSizeInput = /** * Extract the `duration` field type from a fal video model's input. - * Falls back to `string | number | undefined` for unknown models. + * Falls back to `string | number | undefined` for models not in the SDK's + * `EndpointTypeMap`. * * Shapes seen in the wild: * - `'5' | '10'` (Kling, Pika): discrete numeric strings * - `'5s' | '9s'` (Luma): keyword strings with unit * - `'4s' | '6s' | '8s'` (Veo3 via FAL): keyword strings * - `'2' | … | '15'` (WAN-25): discrete-range numeric strings - * - never (Minimax, Hunyuan): no duration field + * - undefined (Minimax, Hunyuan): no duration field; passing one is a type error */ export type FalModelVideoDuration = TModel extends keyof EndpointTypeMap @@ -219,7 +220,9 @@ export type FalVideoPromptModalitiesFor = */ export type FalVideoProviderOptions = TModel extends keyof EndpointTypeMap - ? WithOptionalMediaInputFields, 'prompt'>> + ? WithOptionalMediaInputFields< + Omit, 'prompt' | 'duration'> + > : Record /** diff --git a/packages/ai-fal/src/video/video-provider-options.ts b/packages/ai-fal/src/video/video-provider-options.ts index 8f79b94e06..5e1daa70f0 100644 --- a/packages/ai-fal/src/video/video-provider-options.ts +++ b/packages/ai-fal/src/video/video-provider-options.ts @@ -1,18 +1,21 @@ import type { DurationOptions } from '@tanstack/ai/adapters' -import type { FalModelVideoSize, FalModelVideoSizeInput } from '../model-meta' +import type { + FalModelVideoDuration, + FalModelVideoSize, + FalModelVideoSizeInput, +} from '../model-meta' export function mapVideoSizeToFalFormat( - size: FalModelVideoSize | undefined, -): FalModelVideoSizeInput | undefined { - if (!size) return undefined + size?: FalModelVideoSize, +): FalModelVideoSizeInput { + if (!size) { + return {} as FalModelVideoSizeInput + } - // "16:9_720p" → { aspect_ratio, resolution } - // "16:9" → { aspect_ratio } - // "720p" → { resolution } if (size.includes('_')) { - const [aspect_ratio, resolution] = size.split('_') + const [aspectRatio, resolution] = size.split('_') return { - aspect_ratio, + aspect_ratio: aspectRatio, resolution, } as FalModelVideoSizeInput } @@ -24,41 +27,72 @@ export function mapVideoSizeToFalFormat( return { resolution: size } as FalModelVideoSizeInput } +/** + * Identity helper that checks each entry against the SDK's `duration` union + * for its own key, so a typo'd endpoint id or a stale union fails to compile. + */ +function durationMap< + const T extends { + [K in keyof T & string]: DurationOptions> + }, +>(map: T): T { + return map +} + +const KLING_5_10 = { kind: 'discrete', values: ['5', '10'] } as const +const VEO_4_6_8 = { kind: 'discrete', values: ['4s', '6s', '8s'] } as const +const LTX_6_8_10 = { kind: 'discrete', values: ['6', '8', '10'] } as const +const KLING_3_15 = { + kind: 'discrete', + values: [ + '3', + '4', + '5', + '6', + '7', + '8', + '9', + '10', + '11', + '12', + '13', + '14', + '15', + ], +} as const + /** * Curated map of per-model duration options for popular fal.ai video models. - * Values were sourced from `@fal-ai/client`'s `EndpointTypeMap` input types. + * Values are checked at compile time against `@fal-ai/client`'s + * `EndpointTypeMap` input types via `durationMap`. * * Models not listed here fall back to `{ kind: 'none' }` — honest "we don't * know" rather than guessing. The type-level `FalModelVideoDuration` - * still derives from the SDK types so autocomplete works for unknown models. + * still derives from the SDK types, so autocomplete works for SDK-known + * models that aren't curated here. */ -const FAL_VIDEO_DURATIONS: Readonly< - Record> -> = { - 'fal-ai/kling-video/v1.6/standard/text-to-video': { - kind: 'discrete', - values: ['5', '10'], - }, - 'fal-ai/kling-video/v1.6/pro/text-to-video': { - kind: 'discrete', - values: ['5', '10'], - }, - 'fal-ai/pika/v2.2/text-to-video': { - kind: 'discrete', - values: ['5', '10'], - }, +const FAL_VIDEO_DURATIONS = durationMap({ + 'fal-ai/kling-video/v1.6/standard/text-to-video': KLING_5_10, + 'fal-ai/kling-video/v1.6/pro/text-to-video': KLING_5_10, + 'fal-ai/kling-video/v2.6/pro/text-to-video': KLING_5_10, + 'fal-ai/kling-video/v2.6/pro/image-to-video': KLING_5_10, + 'fal-ai/kling-video/v3/pro/text-to-video': KLING_3_15, + 'fal-ai/kling-video/v3/pro/image-to-video': KLING_3_15, + 'fal-ai/pika/v2.2/text-to-video': KLING_5_10, 'fal-ai/luma-dream-machine/ray-2': { kind: 'discrete', values: ['5s', '9s'], }, - 'fal-ai/veo3': { - kind: 'discrete', - values: ['4s', '6s', '8s'], - }, - 'fal-ai/veo3/image-to-video': { - kind: 'discrete', - values: ['4s', '6s', '8s'], - }, + 'fal-ai/veo3': VEO_4_6_8, + 'fal-ai/veo3/image-to-video': VEO_4_6_8, + 'fal-ai/veo3.1': VEO_4_6_8, + 'fal-ai/veo3.1/image-to-video': VEO_4_6_8, + 'fal-ai/veo3.1/fast': VEO_4_6_8, + 'fal-ai/veo3.1/fast/image-to-video': VEO_4_6_8, + 'fal-ai/ltx-2.3/text-to-video': LTX_6_8_10, + 'fal-ai/ltx-2.3/text-to-video/fast': LTX_6_8_10, + 'fal-ai/ltx-2.3/image-to-video': LTX_6_8_10, + 'fal-ai/ltx-2.3/image-to-video/fast': LTX_6_8_10, 'fal-ai/wan-25-preview/text-to-video': { kind: 'discrete', values: [ @@ -80,10 +114,15 @@ const FAL_VIDEO_DURATIONS: Readonly< }, 'fal-ai/minimax/video-01': { kind: 'none' }, 'fal-ai/hunyuan-video-v1.5/text-to-video': { kind: 'none' }, -} +}) -export function getFalVideoDurationOptions( - model: string, -): DurationOptions { - return FAL_VIDEO_DURATIONS[model] ?? { kind: 'none' } +export function getFalVideoDurationOptions( + model: TModel, +): DurationOptions> { + const entry = (FAL_VIDEO_DURATIONS as Record)[model] + // The map is keyed by literal ids but the adapter's TModel is an open string; + // the lookup result can't be correlated by TS, hence the one cast here. + return (entry ?? { kind: 'none' }) as DurationOptions< + FalModelVideoDuration + > } diff --git a/packages/ai-fal/tests/video-adapter.test.ts b/packages/ai-fal/tests/video-adapter.test.ts index 8ad47dcb6f..f7bc66a410 100644 --- a/packages/ai-fal/tests/video-adapter.test.ts +++ b/packages/ai-fal/tests/video-adapter.test.ts @@ -4,7 +4,10 @@ import { resolveDebugOption } from '@tanstack/ai/adapter-internals' import { falVideo } from '../src/adapters/video' import { recordBillableUnitsFromResponse } from '../src/utils/billing' -import type { FalVideoProviderOptions } from '../src/model-meta' +import type { + FalModelVideoDuration, + FalVideoProviderOptions, +} from '../src/model-meta' function seedBillableUnits(requestId: string, units: string) { recordBillableUnitsFromResponse( @@ -132,6 +135,19 @@ describe('Fal Video Adapter', () => { }) }) + it('passes a numeric duration through for models not in the SDK', async () => { + mockQueueSubmit.mockResolvedValueOnce({ request_id: 'job-unk' }) + + await generateVideo({ + adapter: falVideo('fal-ai/not-in-the-sdk', { apiKey: 'test' }), + prompt: 'A fox running through snow', + duration: 10, + }) + + const [, options] = mockQueueSubmit.mock.calls[0]! + expect(options.input).toMatchObject({ duration: 10 }) + }) + it('omits duration for kind: none models (Minimax)', async () => { mockQueueSubmit.mockResolvedValueOnce({ request_id: 'job-mini', @@ -318,6 +334,57 @@ describe('Fal Video Adapter', () => { expectTypeOf(veo3.snapDuration).returns.toEqualTypeOf< '4s' | '6s' | '8s' | undefined >() + expectTypeOf< + FalModelVideoDuration<'fal-ai/minimax/video-01'> + >().toEqualTypeOf() + expectTypeOf< + FalModelVideoDuration<'fal-ai/not-in-the-sdk'> + >().toEqualTypeOf() + }) + + it('rejects out-of-union durations at the generateVideo call site', () => { + // Type-only: never invoked, so nothing hits the mocked queue. + const typeOnly = () => { + const veo = falVideo('fal-ai/veo3.1', { apiKey: 'test' }) + // @ts-expect-error 7 is not '4s' | '6s' | '8s' + void generateVideo({ adapter: veo, prompt: 'x', duration: 7 }) + // @ts-expect-error '5' is not '4s' | '6s' | '8s' + void generateVideo({ adapter: veo, prompt: 'x', duration: '5' }) + + const mini = falVideo('fal-ai/minimax/video-01', { apiKey: 'test' }) + // @ts-expect-error minimax has no duration field + void generateVideo({ adapter: mini, prompt: 'x', duration: 5 }) + + const kling = falVideo('fal-ai/kling-video/v3/pro/text-to-video', { + apiKey: 'test', + }) + void generateVideo({ + adapter: kling, + prompt: 'x', + // @ts-expect-error duration moved out of modelOptions + modelOptions: { duration: '5' }, + }) + } + expect(typeOnly).toBeTypeOf('function') + }) + + it('curates the models used by the docs and example app', () => { + expect( + falVideo('fal-ai/veo3.1', { apiKey: 'test' }).snapDuration(7), + ).toBe('6s') + expect( + falVideo('fal-ai/kling-video/v3/pro/text-to-video', { + apiKey: 'test', + }).availableDurations(), + ).toMatchObject({ + kind: 'discrete', + values: expect.arrayContaining(['3', '15']), + }) + expect( + falVideo('fal-ai/kling-video/v2.6/pro/image-to-video', { + apiKey: 'test', + }).snapDuration(8), + ).toBe('10') }) }) diff --git a/packages/ai/skills/ai-core/media-generation/SKILL.md b/packages/ai/skills/ai-core/media-generation/SKILL.md index 10d94faa6d..65ecaddb4e 100644 --- a/packages/ai/skills/ai-core/media-generation/SKILL.md +++ b/packages/ai/skills/ai-core/media-generation/SKILL.md @@ -565,12 +565,10 @@ integer durations 1-15s, reports `usage.billed` seconds ({ quantity, unit: 'seco aspect-ratio size template like `'16:9_720p'`, durations 4-15s on the 2.0 family, 4-12s on 1.5-pro, 2-12s on the 1.0-pro models; reads `ARK_API_KEY`), `openRouterVideo(...)` (OpenRouter's dedicated `POST /api/v1/videos` gateway), -and `falVideo(...)` (hosted models, see cost tracking below). -4-12s on 1.5-pro, 2-12s on the 1.0-pro models; reads `ARK_API_KEY`), and -`falVideo(...)` (hosted models; `duration` typed from `@fal-ai/client`'s -`EndpointTypeMap` — `'5' | '10'` on Kling/Pika, `'4s' | '6s' | '8s'` on Veo3, -`'5s' | '9s'` on Luma; `availableDurations()` / `snapDuration()` on the curated -set; see cost tracking below). +and `falVideo(...)` (hosted models; `duration` typed from `@fal-ai/client`'s +`EndpointTypeMap` — `'5' | '10'` on Kling 2.6, `'3'`…`'15'` on Kling 3, +`'4s' | '6s' | '8s'` on Veo 3.1, `'5s' | '9s'` on Luma; `availableDurations()` / +`snapDuration()` on the curated set; see cost tracking below). > **Seedance option applicability is per model and enforced server-side** — > Ark returns a 400 for an inapplicable field rather than ignoring it.