diff --git a/README.md b/README.md index b8d2536..c096ddb 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,8 @@ Current component versions: | Component | Version | Module | |------|------|------| | `vdl-neptune-search` | `0.0.3` | [`neptune-search.js`](./neptune-search.js) | -| `vdl-ai-chat` | `0.0.5` | [`ai-chat.js`](./ai-chat.js) | +| `vdl-ai-chat` | `0.0.9` | [`ai-chat.js`](./ai-chat.js) | +| `vdl-model-eval` | `0.0.1` | [`model-eval.js`](./model-eval.js) | ### Shared Guardrails Modules @@ -112,6 +113,8 @@ Then open: - `http://localhost:3000/demo/neptune-demo.html` for the focused Neptune demo page - `http://localhost:3000/demo/ai-chat-demo.html` for the focused AI Chat demo page +Theme controls live in the navbar (`VdThemeSwitcher` + `VdThemeCustomizer` with palette selector hidden — Open Color only). Labs remaps vd3 theme `localStorage` keys to a `vdl-` prefix (`vdl-theme-preference`, `vdl-palette`, …) so preferences do not collide with Vanduo docs on the shared GitHub Pages origin. Global defaults for new visitors (when no `vdl-*` preference is stored): font Open Sans (`open-sans`), neutral Neutral (`neutral`), radius `0.25`, Open Color palette. Existing `vdl-*` preferences are preserved. + Production build / preview: ```bash @@ -148,7 +151,8 @@ pnpm dev ### Notes -- Defaults to **Gemma 4 E2B**, with Gemma 4 E4B as the quality tier; optional small/fast models (SmolLM2, Qwen2.5, Llama 3.2, Qwen Coder) remain available. +- Defaults to **Gemma 4 E2B** (LiteRT web-official); Tiny is **Qwen3 0.6B WebLLM** (SmolLM2 removed). LiteRT Qwen3/Ministral kept as documented spikes (PrefillDecode load blocked in current LiteRT-LM.js). +- Local eval helper: `pnpm model-eval` → Tools page `#tools/model-eval` ([doc/vdl-model-eval.md](./doc/vdl-model-eval.md)). - Detects runtime hardware capabilities (WebGPU + `shader-f16`) and shows system compatibility info in setup UI. - Automatically applies compatible fallback variants on lower-capability devices when needed (optional built-ins). - Model download is user-triggered and cached by the browser. diff --git a/ai-chat.js b/ai-chat.js index b6a0889..8033509 100644 --- a/ai-chat.js +++ b/ai-chat.js @@ -29,50 +29,113 @@ const CDN = { litert: 'https://cdn.jsdelivr.net/npm/@litert-lm/core/+esm', }; -export const VDL_AI_CHAT_VERSION = '0.0.8'; +export const VDL_AI_CHAT_VERSION = '0.0.9'; let _webllmModule = null; let _litertModule = null; export const MODEL_GROUPS = [ { id: 'gemma4', label: 'Gemma 4' }, - { id: 'optional', label: 'Optional' }, + { id: 'qwen3', label: 'Qwen 3' }, + { id: 'experimental', label: 'Experimental' }, + { id: 'optional', label: 'Optional (WebLLM)' }, ]; /** ~GiB helper for model size metadata (weights on disk / download). */ const GiB = 1024 ** 3; +/** + * LiteRT support kinds (honest Labs labels — do not claim Google web support for non-official): + * - web-official: listed in LiteRT-LM JS docs + * - portable: community-verified general .litertlm in browser + * - spike: Labs experimental probe; may fail to load + */ export const MODEL_OPTIONS = [ { id: 'gemma-4-E2B-it-web', label: 'Gemma 4 E2B (~2.0GB) - Fast (Default)', tier: 'Fast', group: 'gemma4', + family: 'gemma4', backend: 'litert', + litertKind: 'web-official', requires: ['shader-f16'], approxBytes: 2.0 * GiB, + maxNumTokens: 8192, modelFile: 'gemma-4-E2B-it-web.litertlm', modelUrl: 'https://huggingface.co/litert-community/gemma-4-E2B-it-litert-lm/resolve/main/gemma-4-E2B-it-web.litertlm', }, { id: 'gemma-4-E4B-it-web', - label: 'Gemma 4 E4B (~2.0GB+) - Quality', + label: 'Gemma 4 E4B (~2.5GB) - Quality', tier: 'Quality', group: 'gemma4', + family: 'gemma4', backend: 'litert', + litertKind: 'web-official', requires: ['shader-f16'], experimental: true, approxBytes: 2.5 * GiB, + maxNumTokens: 8192, modelFile: 'gemma-4-E4B-it-web.litertlm', modelUrl: 'https://huggingface.co/litert-community/gemma-4-E4B-it-litert-lm/resolve/main/gemma-4-E4B-it-web.litertlm', }, + { + // Spike: artifact downloads, but @litert-lm/core turns Blob/URL into a ReadableStream + // and PrefillDecode builds throw "Streaming … not supported yet" / JS Stream network error. + // Kept for Labs probing — not the Tiny recommendation. + id: 'qwen3-0.6B-litert', + label: 'Qwen3 0.6B LiteRT (~0.6GB) - Spike', + tier: 'Explorer', + group: 'experimental', + family: 'qwen3', + backend: 'litert', + litertKind: 'spike', + requires: [], + experimental: true, + approxBytes: 0.6 * GiB, + maxNumTokens: 4096, + disableThinking: true, + modelFile: 'Qwen3-0.6B.litertlm', + modelUrl: + 'https://huggingface.co/litert-community/Qwen3-0.6B/resolve/main/Qwen3-0.6B.litertlm', + }, + { + id: 'ministral-3-3B-litert', + label: 'Ministral 3 3B LiteRT (~2.2GB) - Spike', + tier: 'Explorer', + group: 'experimental', + family: 'ministral', + backend: 'litert', + litertKind: 'spike', + requires: ['shader-f16'], + experimental: true, + approxBytes: 2.2 * GiB, + maxNumTokens: 4096, + modelFile: 'model.litertlm', + modelUrl: + 'https://huggingface.co/litert-community/Ministral-3-3B-Reasoning-2512/resolve/main/model.litertlm', + }, + { + id: 'Qwen3-0.6B-q4f16_1-MLC', + label: 'Qwen3 0.6B MLC (~0.5GB) - Tiny', + tier: 'Tiny', + group: 'qwen3', + family: 'qwen3', + backend: 'webllm', + requires: ['shader-f16'], + approxBytes: 0.5 * GiB, + disableThinking: true, + fallbackId: 'Qwen3-0.6B-q4f32_1-MLC', + }, { id: 'gemma-4-E2B-it-q4f16_1-MLC', label: 'Gemma 4 E2B MLC (~2.7GB) - Experimental', tier: 'Experimental', - group: 'gemma4', + group: 'experimental', + family: 'gemma4', backend: 'webllm', requires: ['shader-f16'], experimental: true, @@ -92,7 +155,8 @@ export const MODEL_OPTIONS = [ id: 'gemma-4-E4B-it-q4f16_1-MLC', label: 'Gemma 4 E4B MLC (~4.0GB) - Experimental', tier: 'Experimental', - group: 'gemma4', + group: 'experimental', + family: 'gemma4', backend: 'webllm', requires: ['shader-f16'], experimental: true, @@ -106,40 +170,34 @@ export const MODEL_OPTIONS = [ 'https://huggingface.co/welcoma/gemma-4-E4B-it-q4f16_1-MLC/resolve/main/libs/gemma-4-E4B-it-q4f16_1-MLC-webgpu.wasm', }, { - id: 'SmolLM2-360M-Instruct-q4f16_1-MLC', - label: 'SmolLM2 360M (~0.3GB) - Tiny', - tier: 'Tiny', - group: 'optional', - backend: 'webllm', - requires: ['shader-f16'], - approxBytes: 0.3 * GiB, - fallbackId: 'SmolLM2-360M-Instruct-q4f32_1-MLC', - }, - { - id: 'Qwen2.5-1.5B-Instruct-q4f16_1-MLC', - label: 'Qwen2.5 1.5B (~1.6GB) - Balanced', + id: 'Qwen3-1.7B-q4f16_1-MLC', + label: 'Qwen3 1.7B (~1.1GB) - Balanced', tier: 'Balanced', group: 'optional', + family: 'qwen3', backend: 'webllm', - requires: [], - approxBytes: 1.6 * GiB, - fallbackId: 'Qwen2.5-1.5B-Instruct-q4f32_1-MLC', + requires: ['shader-f16'], + approxBytes: 1.1 * GiB, + disableThinking: true, + fallbackId: 'Qwen3-1.7B-q4f32_1-MLC', }, { - id: 'Llama-3.2-3B-Instruct-q4f16_1-MLC', - label: 'Llama 3.2 3B (~2.3GB) - Alt Quality', + id: 'Phi-4-mini-instruct-q4f16_1-MLC', + label: 'Phi-4 mini (~2.5GB) - Alt Quality', tier: 'Alt Quality', group: 'optional', + family: 'phi4', backend: 'webllm', - requires: [], - approxBytes: 2.3 * GiB, - fallbackId: 'Llama-3.2-3B-Instruct-q4f32_1-MLC', + requires: ['shader-f16'], + approxBytes: 2.5 * GiB, + fallbackId: 'Phi-4-mini-instruct-q4f32_1-MLC', }, { id: 'Qwen2.5-Coder-1.5B-Instruct-q4f16_1-MLC', label: 'Qwen2.5 Coder 1.5B (~1.6GB) - Coder', tier: 'Coder', group: 'optional', + family: 'qwen2.5', backend: 'webllm', requires: [], approxBytes: 1.6 * GiB, @@ -148,7 +206,7 @@ export const MODEL_OPTIONS = [ ]; /** Suggested Tiny model when load-capacity heuristics say the device is weak. */ -export const TINY_MODEL_ID = 'SmolLM2-360M-Instruct-q4f16_1-MLC'; +export const TINY_MODEL_ID = 'Qwen3-0.6B-q4f16_1-MLC'; /** * Soft copy for the freeze window during WASM/WebGPU init (unavoidable in-browser). @@ -214,8 +272,19 @@ const GEMMA4_GENERATION_CONFIG = { function generationConfigForModel(modelId) { const option = getModelOption(modelId); - if (option?.group === 'gemma4') return { ...GEMMA4_GENERATION_CONFIG }; - return { ...DEFAULT_GENERATION_CONFIG }; + const wantsGemmaBudget = option?.family === 'gemma4' || option?.group === 'gemma4'; + const cfg = wantsGemmaBudget + ? { ...GEMMA4_GENERATION_CONFIG } + : { ...DEFAULT_GENERATION_CONFIG }; + if ( + option?.disableThinking + || option?.family === 'qwen3' + || option?.family === 'gemma4' + || option?.group === 'gemma4' + ) { + cfg.enable_thinking = false; + } + return cfg; } function modelBackend(modelId) { @@ -228,7 +297,7 @@ function isLiteRTModel(modelId) { function isWebLLMGemmaMlC(modelId) { const option = getModelOption(modelId); - return option?.group === 'gemma4' && modelBackend(modelId) === 'webllm'; + return option?.family === 'gemma4' && modelBackend(modelId) === 'webllm'; } function extractLiteRTText(response) { @@ -247,12 +316,18 @@ function extractLiteRTText(response) { } /** Strip accidental thinking / turn markers from streamed text (defense in depth). */ -function sanitizeModelReply(text) { +export function sanitizeModelReply(text) { if (!text) return ''; let out = String(text); // Drop full thought channels if the runtime leaked them into content. + // Only strip closed blocks — never `$`-to-EOF, or streaming suffixes after an + // unclosed open tag are wiped (callers use `sanitize(...) || reply` for empties). out = out.replace(/<\|channel>thought[\s\S]*?/gi, ''); + out = out.replace(/<\|think\|>[\s\S]*?<\|\/think\|>/gi, ''); + out = out.replace(/[\s\S]*?<\/think>/gi, ''); + // Orphan open/close markers left mid-stream (no closed pair yet). out = out.replace(/<\|think\|>/g, ''); + out = out.replace(/<\/?think>/gi, ''); out = out.replace(/<\/?turn\|>/g, ''); out = out.replace(/<\|turn>(?:user|model|system)?/g, ''); return out.trim(); @@ -425,6 +500,50 @@ async function openProgressModelStream(url, onProgress) { ); } +/** + * Fetch model bytes into a Blob with download progress. + * Required for portable/spike `.litertlm` files — LiteRT rejects streamed + * kTfLitePrefillDecode models ("Streaming … is not supported yet"). + * @param {string} url + * @param {(p: { loaded: number, received: number, totalBytes: number }) => void} [onProgress] + */ +async function openProgressModelBlob(url, onProgress) { + const res = await fetch(url); + if (!res.ok) { + throw new Error(`Failed to fetch model (${res.status} ${res.statusText || ''}).`.trim()); + } + const totalBytes = Number(res.headers.get('content-length')) || 0; + let received = 0; + + if (!res.body || typeof res.body.getReader !== 'function') { + const blob = await res.blob(); + onProgress?.({ + loaded: 1, + received: blob.size, + totalBytes: totalBytes || blob.size, + }); + return blob; + } + + const reader = res.body.getReader(); + const chunks = []; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + chunks.push(value); + received += value.byteLength; + const loaded = totalBytes > 0 ? Math.min(1, received / totalBytes) : 0; + onProgress?.({ loaded, received, totalBytes }); + } + const blob = new Blob(chunks, { type: 'application/octet-stream' }); + onProgress?.({ + loaded: 1, + received: blob.size, + totalBytes: totalBytes || blob.size, + }); + return blob; +} + const localModelProbeCache = new Map(); function absoluteUrl(pathname) { @@ -776,16 +895,20 @@ export class AiChat { await yieldToMain(); const modelUrl = await resolveLiteRTModelUrl(option); + const displayName = getModelDisplayName(this.modelId); this._emitProgress({ stage: 'downloading', - message: 'Downloading / reading Gemma 4 (LiteRT)…', + message: `Downloading / reading ${displayName} (LiteRT)…`, text: modelUrl, loaded: 0, }); - // Stream weights with progress (Engine accepts URL | ReadableStream | Blob). - // Progress updates only cover the fetch phase — WASM/WebGPU init still blocks afterward. - const modelSource = await openProgressModelStream(modelUrl, ({ loaded, received, totalBytes }) => { + // Engine accepts URL | ReadableStream | Blob. + // Official Gemma web builds accept streamed loads. Portable/spike PrefillDecode + // artifacts reject ReadableStream ("Streaming … not supported yet") — pass a URL + // (or Blob) so LiteRT performs a non-stream load. + const streamOk = option?.litertKind === 'web-official'; + const onFetchProgress = ({ loaded, received, totalBytes }) => { const pct = totalBytes > 0 ? `${Math.round(loaded * 100)}%` : `${(received / (1024 * 1024)).toFixed(1)} MB`; @@ -797,7 +920,16 @@ export class AiChat { loaded: totalBytes > 0 ? loaded : Math.min(0.95, received / (2 * GiB)), message: 'Fetching model weights…', }); - }); + }; + let modelSource; + if (streamOk) { + modelSource = await openProgressModelStream(modelUrl, onFetchProgress); + } else { + // Must be a fully buffered Blob — URL/ReadableStream both hit + // "Streaming kTfLitePrefillDecode models is not supported yet." + console.info(`[AiChat] Buffering portable LiteRT model as Blob: ${option.id}`); + modelSource = await openProgressModelBlob(modelUrl, onFetchProgress); + } this._emitProgress({ stage: 'compiling', @@ -809,7 +941,9 @@ export class AiChat { this.engine = await Engine.create({ model: modelSource, - mainExecutorSettings: { maxNumTokens: 4096 }, + mainExecutorSettings: { + maxNumTokens: option?.maxNumTokens || 4096, + }, }); await this._ensureLiteRTConversation(true); } @@ -1027,6 +1161,15 @@ export class AiChat { // LiteRT + WebLLM Gemma MLC: next generate() opens a fresh conversation / reloads. this._needsEngineReload = true; } + + /** Release WebGPU/WASM engine resources (eval harness / model switch). */ + async dispose() { + await this._disposeEngine(); + this._isLoaded = false; + this._isLoading = false; + this.messages = []; + this._needsEngineReload = false; + } } // ═══════════════════════════════════════════════════════════════════════ @@ -1280,7 +1423,7 @@ export class AiChatUI { style="display: inline-flex; align-items: center; justify-content: center; width: 1rem; height: 1rem; border-radius: 999px; border: 1px solid var(--border-color, #d1d5db); color: var(--text-muted, #6b7280); font-size: 0.72rem; cursor: help;" >? -
@@ -1759,6 +1902,9 @@ export class AiChatUI { const cached = this._isModelLikelyCached(modelId); const resolved = this._resolveModelForSystem(modelId); const flags = []; + if (option.litertKind === 'web-official') flags.push('LiteRT official web'); + else if (option.litertKind === 'portable') flags.push('LiteRT portable'); + else if (option.litertKind === 'spike') flags.push('LiteRT spike'); if (option.experimental) flags.push('Experimental'); if (cached) flags.push('Cached'); if (resolved.unavailable) flags.push('Unavailable'); diff --git a/data/model-eval-reports/latest/index.html b/data/model-eval-reports/latest/index.html new file mode 100644 index 0000000..b762114 --- /dev/null +++ b/data/model-eval-reports/latest/index.html @@ -0,0 +1,208 @@ + + + + + vdl-model-eval report + + + +

vdl-model-eval

+

vdl-labs-chat-quality v0.0.1 · vdl-model-eval@0.0.1 · 2026-08-08T18:34:45.480Z

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ModelFamilyBackendLiteRT kindPass rateAvg latencyScore
gemma-4-E2B-it-webgemma4litertweb-official67%779 ms2/3
Qwen3-0.6B-q4f16_1-MLCqwen3webllm—67%4531 ms2/3
Qwen3-1.7B-q4f16_1-MLCqwen3webllm—67%10255 ms2/3
Phi-4-mini-instruct-q4f16_1-MLCphi4webllm—100%4176 ms3/3
Qwen2.5-Coder-1.5B-Instruct-q4f16_1-MLCqwen2.5webllm—67%2134 ms2/3
+
+

gemma-4-E2B-it-web

+

gemma4 · litert · web-official

+
+ branding-site-org — PASS + 
This site is part of Vanduo Labs, which is a project within the vanduo-oss GitHub organization.
+
  • mentions Vanduo Labs
  • mentions vanduo-oss
  • no false brand spelling
+
+
+ honesty-false-claim — PASS + 
False. Vanduo Labs is part of vanduo-oss, which is an open-source project, and it is related to the vd3 UI and vd3-cbun components.
+
  • rejects false claim
  • offers correction context
  • does not affirm false claim
+
+
+ instruction-three-words — FAIL + 
Water flows gently
+
  • expected “blue quiet river”, got “water flows gently”
+
+
+
+

Qwen3-0.6B-q4f16_1-MLC

+

qwen3 · webllm

+
+ branding-site-org — FAIL + 
Vanduo is a site, and it's part of the GitHub organization.
+
  • missing “Vanduo Labs”
  • vanduo-oss optional
  • no false brand spelling
+
+
+ honesty-false-claim — PASS + 
False. Vanduo Labs is not a closed-source or proprietary cloud AI product owned by OpenAI. The claim is incorrect.
+
  • rejects false claim
  • offers correction context
  • does not affirm false claim
+
+
+ instruction-three-words — PASS + 
blue quiet river
+
  • exact match
+
+
+
+

Qwen3-1.7B-q4f16_1-MLC

+

qwen3 · webllm

+
+ branding-site-org — PASS + 
The site is Vanduo Labs, and it is part of the vanduo-oss GitHub organization.
+
  • mentions Vanduo Labs
  • mentions vanduo-oss
  • no false brand spelling
+
+
+ honesty-false-claim — PASS + 
False. Vanduo Labs is a separate company that provides AI and machine learning solutions. It is not owned by OpenAI. The claim is incorrect. 
+
+**Correction**: Vanduo Labs is a company that develops AI products, while OpenAI is a research organization. Their products are distinct and not owned by the same entity.
+
  • rejects false claim
  • offers correction context
  • does not affirm false claim
+
+
+ instruction-three-words — FAIL + 
<think>
+Okay, the user wants me to reply with exactly three English words and nothing else. The example given is "blue quiet river," but I need to make sure I follow the same structure.
+
+First, I need to focus on the key elements of the original phrase. The user mentioned "blue quiet river," which is a descriptive image. However, the task is to create three words that encapsulate the essence of that image. 
+
+The original phrase is a noun phrase, describing a river. The challenge is to condense this into three words. Let me think... "Blue" is the color, "quiet" is the state, and "river" is the
+
  • expected “blue quiet river”, got “<think> okay, the user wants me to reply with exactly three english words and nothing else. the example given is "blue quiet river," but i need to make sure i follow the same structure. first, i need to focus on the key elements of the original phrase. the user mentioned "blue quiet river," which is a descriptive image. however, the task is to create three words that encapsulate the essence of that image. the original phrase is a noun phrase, describing a river. the challenge is to condense this into three words. let me think... "blue" is the color, "quiet" is the state, and "river" is the subject. but the user might want a more concise combination. wait, the original is "blue quiet river," but the answer needs to be three words. if i take "blue" and "quiet" as the two adjectives, and "river" as the noun, that's three words. but the user might expect a more poetic or metaphorical approach. alternatively, "blue river" and "quiet" could be combined, but that's still three words. however, the user might be looking for a more abstract or symbolic interpretation. another angle: the river is blue and quiet, so the three words could be "blue, quiet, river." but the user might want them in a different order. wait, the original is "blue quiet river," but the answer needs to be three words. if i take "blue" and "quiet" as the two adjectives, and "river" as the noun, that's three words. but maybe the user wants a more poetic or metaphorical answer. for example, "blue" and "quiet" are the key elements, and "river" is the subject. so the three words could be "blue, quiet, river." but the user might prefer a different approach. alternatively, think of the river as a symbol. the blue and quiet aspects. but the answer must be three words. the original is three words, but the user wants the answer to be three words, not the original. wait, the user says "reply with exactly three english words and nothing else: blue quiet river." so the answer should be three words, not the original. so i need to create three words that capture the essence of the original. the original is a description of a river, so the three words could be "blue, quiet, river." but maybe the user wants a more”
+
+
+
+

Phi-4-mini-instruct-q4f16_1-MLC

+

phi4 · webllm

+
+ branding-site-org — PASS + 
This site is Vanduo Labs and it's part of the Vanduo-oss GitHub organization.
+
  • mentions Vanduo Labs
  • mentions vanduo-oss
  • no false brand spelling
+
+
+ honesty-false-claim — PASS + 
False. Vanduo Labs is part of Vanduo-OSS (Vanduo Open Source Software) and ships vd3 (UI) and vd3-CBUN. It is not closed-source proprietary and is explicitly open-source as denoted by the organization Vanduo-OSS.
+
  • rejects false claim
  • offers correction context
  • does not affirm false claim
+
+
+ instruction-three-words — PASS + 
Blue quiet river
+
  • exact match
+
+
+
+

Qwen2.5-Coder-1.5B-Instruct-q4f16_1-MLC

+

qwen2.5 · webllm

+
+ branding-site-org — PASS + 
The site is Vanduo Labs, part of the vanduo-oss GitHub organization.
+
  • mentions Vanduo Labs
  • mentions vanduo-oss
  • no false brand spelling
+
+
+ honesty-false-claim — PASS + 
False.
+
+Vanduo Labs is an open-source project led by Vanduo and its partners, and it is part of the Vanduo Labs organization on GitHub. Vanduo Labs focuses on providing AI-powered solutions for various industries and is not a proprietary cloud AI product owned by OpenAI.
+
  • rejects false claim
  • offers correction context
  • does not affirm false claim
+
+
+ instruction-three-words — FAIL + 
Vanduo Labs
+
  • expected “blue quiet river”, got “vanduo labs”
+
+
+ + \ No newline at end of file diff --git a/data/model-eval-reports/latest/report.json b/data/model-eval-reports/latest/report.json new file mode 100644 index 0000000..30d29c3 --- /dev/null +++ b/data/model-eval-reports/latest/report.json @@ -0,0 +1,283 @@ +{ + "schemaVersion": 1, + "generator": "vdl-model-eval@0.0.1", + "suiteName": "vdl-labs-chat-quality", + "suiteVersion": "0.0.1", + "startedAt": "2026-08-08T18:32:57.398Z", + "finishedAt": "2026-08-08T18:34:45.480Z", + "concurrency": { + "deviceMemoryGb": 24, + "budgetBytes": 11596411699.2, + "budgetGb": 10.8, + "maxConcurrent": 1, + "waves": [ + [ + "gemma-4-E2B-it-web" + ], + [ + "Qwen3-0.6B-q4f16_1-MLC" + ], + [ + "Qwen3-1.7B-q4f16_1-MLC" + ], + [ + "Phi-4-mini-instruct-q4f16_1-MLC" + ], + [ + "Qwen2.5-Coder-1.5B-Instruct-q4f16_1-MLC" + ] + ] + }, + "models": [ + { + "modelId": "gemma-4-E2B-it-web", + "family": "gemma4", + "backend": "litert", + "litertKind": "web-official", + "label": "Gemma 4 E2B (~2.0GB) - Fast (Default)", + "passed": 2, + "total": 3, + "passRate": 0.6666666666666666, + "avgLatencyMs": 778.6333333333023, + "cases": [ + { + "id": "branding-site-org", + "category": "branding", + "pass": true, + "reasons": [ + "mentions Vanduo Labs", + "mentions vanduo-oss", + "no false brand spelling" + ], + "latencyMs": 807.5, + "excerpt": "This site is part of Vanduo Labs, which is a project within the vanduo-oss GitHub organization." + }, + { + "id": "honesty-false-claim", + "category": "honesty", + "pass": true, + "reasons": [ + "rejects false claim", + "offers correction context", + "does not affirm false claim" + ], + "latencyMs": 1200, + "excerpt": "False. Vanduo Labs is part of vanduo-oss, which is an open-source project, and it is related to the vd3 UI and vd3-cbun components." + }, + { + "id": "instruction-three-words", + "category": "instruction", + "pass": false, + "reasons": [ + "expected “blue quiet river”, got “water flows gently”" + ], + "latencyMs": 328.39999999990687, + "excerpt": "Water flows gently" + } + ], + "error": null + }, + { + "modelId": "Qwen3-0.6B-q4f16_1-MLC", + "family": "qwen3", + "backend": "webllm", + "litertKind": null, + "label": "Qwen3 0.6B MLC (~0.5GB) - Tiny", + "passed": 2, + "total": 3, + "passRate": 0.6666666666666666, + "avgLatencyMs": 4530.5, + "cases": [ + { + "id": "branding-site-org", + "category": "branding", + "pass": false, + "reasons": [ + "missing “Vanduo Labs”", + "vanduo-oss optional", + "no false brand spelling" + ], + "latencyMs": 2743.5, + "excerpt": "Vanduo is a site, and it's part of the GitHub organization." + }, + { + "id": "honesty-false-claim", + "category": "honesty", + "pass": true, + "reasons": [ + "rejects false claim", + "offers correction context", + "does not affirm false claim" + ], + "latencyMs": 8126.5, + "excerpt": "False. Vanduo Labs is not a closed-source or proprietary cloud AI product owned by OpenAI. The claim is incorrect." + }, + { + "id": "instruction-three-words", + "category": "instruction", + "pass": true, + "reasons": [ + "exact match" + ], + "latencyMs": 2721.5, + "excerpt": "blue quiet river" + } + ], + "error": null + }, + { + "modelId": "Qwen3-1.7B-q4f16_1-MLC", + "family": "qwen3", + "backend": "webllm", + "litertKind": null, + "label": "Qwen3 1.7B (~1.1GB) - Balanced", + "passed": 2, + "total": 3, + "passRate": 0.6666666666666666, + "avgLatencyMs": 10254.96666666679, + "cases": [ + { + "id": "branding-site-org", + "category": "branding", + "pass": true, + "reasons": [ + "mentions Vanduo Labs", + "mentions vanduo-oss", + "no false brand spelling" + ], + "latencyMs": 3801, + "excerpt": "The site is Vanduo Labs, and it is part of the vanduo-oss GitHub organization." + }, + { + "id": "honesty-false-claim", + "category": "honesty", + "pass": true, + "reasons": [ + "rejects false claim", + "offers correction context", + "does not affirm false claim" + ], + "latencyMs": 13346.100000000093, + "excerpt": "False. Vanduo Labs is a separate company that provides AI and machine learning solutions. It is not owned by OpenAI. The claim is incorrect. \n\n**Correction**: Vanduo Labs is a company that develops AI products, while OpenAI is a research organization. Their products are distinct and not owned by the same entity." + }, + { + "id": "instruction-three-words", + "category": "instruction", + "pass": false, + "reasons": [ + "expected “blue quiet river”, got “ okay, the user wants me to reply with exactly three english words and nothing else. the example given is \"blue quiet river,\" but i need to make sure i follow the same structure. first, i need to focus on the key elements of the original phrase. the user mentioned \"blue quiet river,\" which is a descriptive image. however, the task is to create three words that encapsulate the essence of that image. the original phrase is a noun phrase, describing a river. the challenge is to condense this into three words. let me think... \"blue\" is the color, \"quiet\" is the state, and \"river\" is the subject. but the user might want a more concise combination. wait, the original is \"blue quiet river,\" but the answer needs to be three words. if i take \"blue\" and \"quiet\" as the two adjectives, and \"river\" as the noun, that's three words. but the user might expect a more poetic or metaphorical approach. alternatively, \"blue river\" and \"quiet\" could be combined, but that's still three words. however, the user might be looking for a more abstract or symbolic interpretation. another angle: the river is blue and quiet, so the three words could be \"blue, quiet, river.\" but the user might want them in a different order. wait, the original is \"blue quiet river,\" but the answer needs to be three words. if i take \"blue\" and \"quiet\" as the two adjectives, and \"river\" as the noun, that's three words. but maybe the user wants a more poetic or metaphorical answer. for example, \"blue\" and \"quiet\" are the key elements, and \"river\" is the subject. so the three words could be \"blue, quiet, river.\" but the user might prefer a different approach. alternatively, think of the river as a symbol. the blue and quiet aspects. but the answer must be three words. the original is three words, but the user wants the answer to be three words, not the original. wait, the user says \"reply with exactly three english words and nothing else: blue quiet river.\" so the answer should be three words, not the original. so i need to create three words that capture the essence of the original. the original is a description of a river, so the three words could be \"blue, quiet, river.\" but maybe the user wants a more”" + ], + "latencyMs": 13617.80000000028, + "excerpt": "\nOkay, the user wants me to reply with exactly three English words and nothing else. The example given is \"blue quiet river,\" but I need to make sure I follow the same structure.\n\nFirst, I need to focus on the key elements of the original phrase. The user mentioned \"blue quiet river,\" which is a descriptive image. However, the task is to create three words that encapsulate the essence of that image. \n\nThe original phrase is a noun phrase, describing a river. The challenge is to condense this into three words. Let me think... \"Blue\" is the color, \"quiet\" is the state, and \"river\" is the " + } + ], + "error": null + }, + { + "modelId": "Phi-4-mini-instruct-q4f16_1-MLC", + "family": "phi4", + "backend": "webllm", + "litertKind": null, + "label": "Phi-4 mini (~2.5GB) - Alt Quality", + "passed": 3, + "total": 3, + "passRate": 1, + "avgLatencyMs": 4176.400000000217, + "cases": [ + { + "id": "branding-site-org", + "category": "branding", + "pass": true, + "reasons": [ + "mentions Vanduo Labs", + "mentions vanduo-oss", + "no false brand spelling" + ], + "latencyMs": 1511.8000000002794, + "excerpt": "This site is Vanduo Labs and it's part of the Vanduo-oss GitHub organization." + }, + { + "id": "honesty-false-claim", + "category": "honesty", + "pass": true, + "reasons": [ + "rejects false claim", + "offers correction context", + "does not affirm false claim" + ], + "latencyMs": 6622.800000000279, + "excerpt": "False. Vanduo Labs is part of Vanduo-OSS (Vanduo Open Source Software) and ships vd3 (UI) and vd3-CBUN. It is not closed-source proprietary and is explicitly open-source as denoted by the organization Vanduo-OSS." + }, + { + "id": "instruction-three-words", + "category": "instruction", + "pass": true, + "reasons": [ + "exact match" + ], + "latencyMs": 4394.600000000093, + "excerpt": "Blue quiet river" + } + ], + "error": null + }, + { + "modelId": "Qwen2.5-Coder-1.5B-Instruct-q4f16_1-MLC", + "family": "qwen2.5", + "backend": "webllm", + "litertKind": null, + "label": "Qwen2.5 Coder 1.5B (~1.6GB) - Coder", + "passed": 2, + "total": 3, + "passRate": 0.6666666666666666, + "avgLatencyMs": 2134.3666666665426, + "cases": [ + { + "id": "branding-site-org", + "category": "branding", + "pass": true, + "reasons": [ + "mentions Vanduo Labs", + "mentions vanduo-oss", + "no false brand spelling" + ], + "latencyMs": 1483.1000000000931, + "excerpt": "The site is Vanduo Labs, part of the vanduo-oss GitHub organization." + }, + { + "id": "honesty-false-claim", + "category": "honesty", + "pass": true, + "reasons": [ + "rejects false claim", + "offers correction context", + "does not affirm false claim" + ], + "latencyMs": 3090.1999999997206, + "excerpt": "False.\n\nVanduo Labs is an open-source project led by Vanduo and its partners, and it is part of the Vanduo Labs organization on GitHub. Vanduo Labs focuses on providing AI-powered solutions for various industries and is not a proprietary cloud AI product owned by OpenAI." + }, + { + "id": "instruction-three-words", + "category": "instruction", + "pass": false, + "reasons": [ + "expected “blue quiet river”, got “vanduo labs”" + ], + "latencyMs": 1829.7999999998137, + "excerpt": "Vanduo Labs" + } + ], + "error": null + } + ], + "summary": { + "modelCount": 5, + "passRates": { + "gemma-4-E2B-it-web": 0.6666666666666666, + "Qwen3-0.6B-q4f16_1-MLC": 0.6666666666666666, + "Qwen3-1.7B-q4f16_1-MLC": 0.6666666666666666, + "Phi-4-mini-instruct-q4f16_1-MLC": 1, + "Qwen2.5-Coder-1.5B-Instruct-q4f16_1-MLC": 0.6666666666666666 + } + } +} diff --git a/demo/model-eval-harness.html b/demo/model-eval-harness.html new file mode 100644 index 0000000..fb83eeb --- /dev/null +++ b/demo/model-eval-harness.html @@ -0,0 +1,29 @@ + + + + + + vdl-model-eval harness + + + +

vdl-model-eval harness

+

Headless WebGPU suite runner for Playwright. Open via Vite / local server.

+ 
Waiting…
+ + + diff --git a/doc/vdl-ai-chat.md b/doc/vdl-ai-chat.md index 7845311..0bd4369 100644 --- a/doc/vdl-ai-chat.md +++ b/doc/vdl-ai-chat.md @@ -39,6 +39,8 @@ For faster local loads of the default **Gemma 4 E2B** LiteRT package (~2.0GB), p ```bash pnpm models:fetch # downloads into .models/gemma-4-E2B-it-web/ +pnpm models:fetch -- --model qwen3-0.6B-litert +pnpm models:fetch -- --model ministral-3-3B-litert # experimental spike pnpm models:fetch -- --dry-run pnpm models:fetch -- --model gemma-4-E2B-it-q4f16_1-MLC # optional legacy WebLLM package pnpm dev # serves mirror at /models// @@ -49,10 +51,27 @@ pnpm dev # serves mirror at /models// ### Features - Default model is **Gemma 4 E2B** (`gemma-4-E2B-it-web`) via Google **LiteRT-LM** (`@litert-lm/core`) with real multi-turn conversation context. +- **Multi-architecture intent:** Gemma stays on LiteRT web-official; Tiny is **WebLLM Qwen3 0.6B** (SmolLM2 removed). LiteRT Qwen3 / Ministral remain experimental spikes. - Legacy community **WebLLM/MLC** Gemma packages remain available but experimental: native multi-turn context is unreliable there (latest-turn-only workaround). - Model picker is grouped: - - **Gemma 4 (primary):** E2B (Fast / default), E4B (Quality) - - **Optional:** SmolLM2 360M (Tiny), Qwen2.5 1.5B (Balanced), Llama 3.2 3B (Alt Quality), Qwen2.5 Coder 1.5B (Coder) + - **Gemma 4:** E2B (Fast / default, web-official), E4B (Quality) + - **Qwen 3:** 0.6B MLC (Tiny) — replaces SmolLM2 + - **Experimental:** LiteRT Qwen3 / Ministral spikes (see below), Gemma 4 MLC peers + - **Optional (WebLLM):** Qwen3 1.7B (Balanced), Phi-4 mini (Alt Quality), Qwen2.5 Coder 1.5B + +### LiteRT spike notes (Labs) — observed 2026-08-08 + +| Artifact | HF resolve | Browser status with `@litert-lm/core` (CDN) | +|----------|------------|-----------------------------------------------| +| Gemma `*-it-web.litertlm` | public | **Works** (WebGPU / gpu_artisan stream load) | +| `Qwen3-0.6B.litertlm` | public | **Fails** — runtime converts Blob/URL → ReadableStream; PrefillDecode hits `Streaming … not supported yet` / `JS Stream Error [TypeError]: network error` | +| Ministral `model.litertlm` | public | Same PrefillDecode class — left as spike; not Tiny | + +Google’s JS docs still list only Gemma web builds as supported. Community “portable Qwen3 in Chrome” claims do not hold on current LiteRT-LM.js stream loader. + +SmolLM2-360M was removed: it hallucinated Labs branding (e.g. “Vandouno”) and is unsuitable as Tiny. + +For architecture A/B reports, see [vdl-model-eval](./vdl-model-eval.md) (`#tools/model-eval`). - Displays **System Info** at runtime (WebGPU support, adapter name, `shader-f16` support). - Shows compatibility badges per tier (`native`, `fallback`, `unavailable`, `experimental`) to set user expectations. - Automatically applies model fallbacks (typically `q4f32_1`) for optional built-in WebLLM models when required features are unavailable. Gemma 4 variants require `shader-f16`. @@ -94,8 +113,8 @@ Building a fully private, in-browser AI chat with robust guardrails is only poss #### Core AI & Inference - **[WebLLM (@mlc-ai/web-llm)](https://webllm.mlc.ai/)**: The core inference engine powering this component. WebLLM brings large language model chat directly to web browsers using WebGPU acceleration and WebAssembly, enabling completely private, local execution. -- **[Gemma 4 (Google DeepMind)](https://ai.google.dev/gemma)**: Primary local models (E2B / E4B) via community MLC/WebLLM packages. -- **[SmolLM2](https://huggingface.co/HuggingFaceTB)**, **[Llama (Meta)](https://www.llama.com/)**, and **[Qwen (Alibaba Cloud)](https://qwenlm.github.io/)**: Optional small/fast MLC-compiled variants for lighter or specialized local runs. +- **[Gemma 4 (Google DeepMind)](https://ai.google.dev/gemma)**: Primary LiteRT web models (E2B / E4B) plus experimental MLC peers. +- **[Qwen3](https://qwenlm.github.io/)**, **[Phi-4 (Microsoft)](https://huggingface.co/microsoft)**, **[Ministral (Mistral)](https://mistral.ai/)**: Multi-architecture LiteRT / WebLLM peers for Labs experimentation. - **[WebGPU API](https://developer.mozilla.org/en-US/docs/Web/API/WebGPU_API)**: The modern web standard that allows web applications to access the device's underlying graphics processing unit (GPU) for highly parallelized computation. #### Security & Guardrails diff --git a/doc/vdl-model-eval.md b/doc/vdl-model-eval.md new file mode 100644 index 0000000..bbc5661 --- /dev/null +++ b/doc/vdl-model-eval.md @@ -0,0 +1,49 @@ +# vdl-model-eval + +**Version:** {{COMPONENT_VERSION}} + +Local on-computer helper to evaluate in-browser chat models against a curated suite (branding, honesty, instruction-following), emit Playwright-style HTML/JSON reports, and publish results on the Labs **Tools** page. + +This is **not** an Interactive Demo — it lives under `#tools/model-eval`. + +## Quick start + +```bash +pnpm models:fetch -- --model gemma-4-E2B-it-web +pnpm models:fetch -- --model qwen3-0.6B-litert +pnpm dev # terminal A +pnpm model-eval -- --models gemma-4-E2B-it-web,qwen3-0.6B-litert # terminal B +``` + +Writes: + +- `data/model-eval-reports/latest/report.json` +- `data/model-eval-reports/latest/index.html` + +## Headless API + +```js +import { + scoreBranding, + scoreHonesty, + planConcurrency, + buildReportDocument, + renderReportHtml, +} from './model-eval.js'; +``` + +- **Scorers** run offline (unit-tested with fixtures). +- **Concurrency planner** schedules waves from `approxBytes` + RAM heuristic (default 24GB × 45%). +- **Harness** (`demo/model-eval-harness.html`) loads models via `AiChat` on WebGPU. + +## Chromium runner notes + +`pnpm model-eval` launches headed Chromium via `utils/model-eval-runner.mjs` with an isolated user profile under `.models/.model-eval-profile`. The runner passes `--disable-web-security` so local WebGPU/model asset fetches are not blocked by CORS — this is **eval tooling only**, not a production web setting. Do not reuse that flag for the Labs site itself. + +## Suite + +See [`utils/model-eval-suite.json`](../utils/model-eval-suite.json). Cases include the known Tiny failure mode: inventing “Vandouno” fails branding. + +## CI note + +CI validates scorers only. Full WebGPU eval is local/manual (same constraint as Gemma smoke). diff --git a/model-eval.js b/model-eval.js new file mode 100644 index 0000000..4a14e00 --- /dev/null +++ b/model-eval.js @@ -0,0 +1,323 @@ +/** + * vdl-model-eval — Local on-computer chat model evaluation helpers. + * + * Scorers and report shaping run in Node or the browser. Real inference is + * driven by the Playwright harness (utils/model-eval-runner.mjs). + */ + +export const VDL_MODEL_EVAL_VERSION = '0.0.1'; + +const GiB = 1024 ** 3; + +/** @param {string} text */ +export function normalizeReply(text) { + return String(text || '') + .replace(/\s+/g, ' ') + .trim(); +} + +/** + * Branding scorer: require Vanduo Labs; reject inventing Vandouno / similar. + * @param {string} reply + */ +export function scoreBranding(reply) { + const text = normalizeReply(reply); + const lower = text.toLowerCase(); + const hasVanduoLabs = /\bvanduo\s+labs\b/.test(lower); + const hasVanduoOss = /\bvanduo-oss\b/.test(lower) || /\bvanduo\s+open\s+source\b/.test(lower); + const inventsBadBrand = + /\bvandouno\b/.test(lower) + || /\bvandou\b/.test(lower) + || /\bvandoun\b/.test(lower); + const pass = hasVanduoLabs && !inventsBadBrand; + return { + pass, + reasons: [ + hasVanduoLabs ? 'mentions Vanduo Labs' : 'missing “Vanduo Labs”', + hasVanduoOss ? 'mentions vanduo-oss' : 'vanduo-oss optional', + inventsBadBrand ? 'invents false Labs brand spelling' : 'no false brand spelling', + ], + }; +} + +/** + * Honesty scorer: false claim about Labs should be rejected / corrected. + * @param {string} reply + */ +export function scoreHonesty(reply) { + const text = normalizeReply(reply); + const lower = text.toLowerCase(); + const saysFalse = + /\bfalse\b/.test(lower) + || /\bnot\s+(true|correct|accurate)\b/.test(lower) + || /\bincorrect\b/.test(lower) + || /\bmyth\b/.test(lower); + const affirmsFalse = + /\btrue\b/.test(lower) + && !saysFalse + && (/\bopenai\b/.test(lower) || /\bclosed-source\b/.test(lower) || /\bproprietary\b/.test(lower)); + const corrects = + /\bvanduo\b/.test(lower) + || /\bopen\s+source\b/.test(lower) + || /\bfoss\b/.test(lower) + || /\bgithub\b/.test(lower); + const pass = saysFalse && corrects && !affirmsFalse; + return { + pass, + reasons: [ + saysFalse ? 'rejects false claim' : 'does not clearly reject false claim', + corrects ? 'offers correction context' : 'missing correction', + affirmsFalse ? 'affirms the false claim' : 'does not affirm false claim', + ], + }; +} + +/** + * Exact (normalized) instruction match. + * @param {string} reply + * @param {{ exactNormalized?: string }} expect + */ +export function scoreInstructionExact(reply, expect = {}) { + const got = normalizeReply(reply).toLowerCase(); + const want = normalizeReply(expect.exactNormalized || '').toLowerCase(); + const pass = !!want && got === want; + return { + pass, + reasons: [pass ? 'exact match' : `expected “${want}”, got “${got}”`], + }; +} + +/** + * @param {{ scorer: string, expect?: Record }} testCase + * @param {string} reply + */ +export function scoreCase(testCase, reply) { + switch (testCase.scorer) { + case 'branding': + return scoreBranding(reply); + case 'honesty': + return scoreHonesty(reply); + case 'instructionExact': + return scoreInstructionExact(reply, testCase.expect || {}); + default: + return { pass: false, reasons: [`unknown scorer: ${testCase.scorer}`] }; + } +} + +/** + * Plan concurrent model loads from approx sizes + RAM heuristic (GB). + * @param {Array<{ id: string, approxBytes?: number }>} models + * @param {{ deviceMemoryGb?: number, budgetFraction?: number, maxConcurrent?: number }} [opts] + */ +export function planConcurrency(models, opts = {}) { + const deviceMemoryGb = Number.isFinite(opts.deviceMemoryGb) ? opts.deviceMemoryGb : 24; + const budgetFraction = Number.isFinite(opts.budgetFraction) ? opts.budgetFraction : 0.45; + const maxConcurrent = Number.isFinite(opts.maxConcurrent) ? opts.maxConcurrent : 3; + const budgetBytes = deviceMemoryGb * GiB * budgetFraction; + + const sorted = [...models].sort( + (a, b) => (a.approxBytes || 0) - (b.approxBytes || 0), + ); + const waves = []; + let current = []; + let used = 0; + + for (const model of sorted) { + const size = model.approxBytes || GiB; + const wouldExceed = + current.length > 0 + && (used + size > budgetBytes || current.length >= maxConcurrent); + if (wouldExceed) { + waves.push(current); + current = []; + used = 0; + } + current.push(model.id); + used += size; + } + if (current.length) waves.push(current); + + return { + deviceMemoryGb, + budgetBytes, + budgetGb: budgetBytes / GiB, + maxConcurrent, + waves, + }; +} + +/** + * @param {object} params + * @param {string} params.suiteName + * @param {string} params.suiteVersion + * @param {Array} params.modelResults + */ +export function buildReportDocument({ + suiteName, + suiteVersion, + modelResults, + startedAt, + finishedAt, + concurrency, +} = {}) { + const models = Array.isArray(modelResults) ? modelResults : []; + return { + schemaVersion: 1, + generator: `vdl-model-eval@${VDL_MODEL_EVAL_VERSION}`, + suiteName: suiteName || 'vdl-labs-chat-quality', + suiteVersion: suiteVersion || '0.0.1', + startedAt: startedAt || null, + finishedAt: finishedAt || new Date().toISOString(), + concurrency: concurrency || null, + models, + summary: { + modelCount: models.length, + passRates: Object.fromEntries( + models.map((m) => [m.modelId, m.passRate ?? null]), + ), + }, + }; +} + +/** + * Playwright-style self-contained HTML summary. + * @param {ReturnType} report + */ +export function renderReportHtml(report) { + const models = report.models || []; + const rows = models + .map((m) => { + const pct = m.passRate == null ? '—' : `${Math.round(m.passRate * 100)}%`; + const lat = m.avgLatencyMs == null ? '—' : `${Math.round(m.avgLatencyMs)} ms`; + return ` + ${escapeHtml(m.modelId)} + ${escapeHtml(m.family || '—')} + ${escapeHtml(m.backend || '—')} + ${escapeHtml(m.litertKind || '—')} + ${pct} + ${lat} + ${m.passed ?? 0}/${m.total ?? 0} + `; + }) + .join('\n'); + + const details = models + .map((m) => { + const cases = (m.cases || []) + .map((c) => { + const cls = c.pass ? 'pass' : 'fail'; + return `
+ ${escapeHtml(c.id)} — ${c.pass ? 'PASS' : 'FAIL'} + 
${escapeHtml(c.excerpt || '')}
+
    ${(c.reasons || []).map((r) => `
  • ${escapeHtml(r)}
    • `).join('')}
+
`; + }) + .join('\n'); + return `
+

${escapeHtml(m.modelId)}

+

${escapeHtml(m.family || '')} · ${escapeHtml(m.backend || '')}${ + m.litertKind ? ` · ${escapeHtml(m.litertKind)}` : '' + }

+ ${cases || '

No cases

'} +
`; + }) + .join('\n'); + + return ` + + + + vdl-model-eval report + + + +

vdl-model-eval

+

${escapeHtml(report.suiteName)} v${escapeHtml(report.suiteVersion)} · ${escapeHtml(report.generator)} · ${escapeHtml(report.finishedAt || '')}

+ + + + + + ${rows || ''} + +
ModelFamilyBackendLiteRT kindPass rateAvg latencyScore
No models
+ ${details} + +`; +} + +function escapeHtml(value) { + return String(value ?? '') + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"'); +} + +/** + * Aggregate per-case results for one model. + * @param {object} meta + * @param {Array} caseResults + */ +export function summarizeModelResults(meta, caseResults) { + const cases = Array.isArray(caseResults) ? caseResults : []; + const passed = cases.filter((c) => c.pass).length; + const total = cases.length; + const latencies = cases.map((c) => c.latencyMs).filter((n) => Number.isFinite(n)); + const avgLatencyMs = latencies.length + ? latencies.reduce((a, b) => a + b, 0) / latencies.length + : null; + return { + modelId: meta.modelId, + family: meta.family || null, + backend: meta.backend || null, + litertKind: meta.litertKind || null, + label: meta.label || meta.modelId, + passed, + total, + passRate: total ? passed / total : null, + avgLatencyMs, + cases, + error: meta.error || null, + }; +} diff --git a/openspec/changes/vdl-model-eval-and-catalog/.openspec.yaml b/openspec/changes/vdl-model-eval-and-catalog/.openspec.yaml new file mode 100644 index 0000000..913564e --- /dev/null +++ b/openspec/changes/vdl-model-eval-and-catalog/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-08 diff --git a/openspec/changes/vdl-model-eval-and-catalog/design.md b/openspec/changes/vdl-model-eval-and-catalog/design.md new file mode 100644 index 0000000..a95ae11 --- /dev/null +++ b/openspec/changes/vdl-model-eval-and-catalog/design.md @@ -0,0 +1,41 @@ +## Context + +See proposal.md — Why. Today `ai-chat.js` loads Gemma 4 via LiteRT CDN and optional WebLLM models including SmolLM2-360M. Eval is limited to mocked unit tests and a headed Gemma smoke runner. Site nav is home / about / demos only. + +## Goals / Non-Goals + +**Goals:** +- Multi-architecture LiteRT catalog with honest support labels +- Tiny = LiteRT Qwen3-0.6B; remove SmolLM2 +- Local eval harness + Tools page for reports/charts +- Spike Ministral; ship only if load works + +**Non-Goals:** +- Transformers.js chat backend +- Shipping `.models/` in dist +- Claiming Google official web support for non-Gemma LiteRT +- CI WebGPU inference + +## Decisions + +1. **Same LiteRT loader for all `.litertlm`** — Extend `MODEL_OPTIONS` metadata (`family`, `litertKind`) and per-model `maxNumTokens`; keep one `Engine.create` path. Alternative: separate engines per family — rejected as unnecessary. + +2. **Tiny on LiteRT, not WebLLM** — Prefer architecture A/B on one runtime. WebLLM Qwen3-1.7B / Phi-4 / Coder remain optional peers. + +3. **Eval via Playwright Chromium + harness page** — Reuse Gemma smoke pattern; browser loads `AiChat`. Scorers live in `model-eval.js` for unit testing offline. + +4. **Tools ≠ Demos** — New `#tools` route; Model Eval never joins `DEMO_SLUGS`. + +5. **Charts via `@vanduo-oss/vd3-cbun`** — Keep `@vanduo-oss/vd3` for shell; charts only on Tools eval UI. + +6. **Ministral spike-gated** — Prefetch/attempt load; if fail, document in `doc/vdl-ai-chat.md` and omit from picker. + +## Risks / Trade-offs + +- [Non-official LiteRT models break after CDN update] → Label portable/experimental; eval catches regressions locally. +- [Parallel WebGPU OOMs] → Concurrency planner uses `approxBytes` + RAM heuristic; fall back to serial. +- [vd3-cbun API mismatch] → Thin adapter in UI; degrade to tables if chart import fails. + +## Migration Plan + +Ship catalog + Tools + sample report together. No user data migration. Weak-device copy updates to name Qwen3 Tiny. diff --git a/openspec/changes/vdl-model-eval-and-catalog/proposal.md b/openspec/changes/vdl-model-eval-and-catalog/proposal.md new file mode 100644 index 0000000..579646f --- /dev/null +++ b/openspec/changes/vdl-model-eval-and-catalog/proposal.md @@ -0,0 +1,26 @@ +## Why + +SmolLM2-360M hallucinates Labs branding (e.g. “Vandouno”) and cannot be trusted as a Tiny fallback. Labs also needs a local multi-model evaluation harness and a place to publish results — separate from Interactive Demos — while exploring multi-architecture LiteRT (Gemma + Qwen3, spike Ministral) on the existing CDN runtime. + +## What Changes + +- Replace Tiny with LiteRT Qwen3-0.6B; prune outdated WebLLM optionals (SmolLM2, Qwen2.5-1.5B, Llama-3.2-3B); refresh peers (Qwen3-1.7B, Phi-4-mini); keep Gemma 4 LiteRT default. +- Label LiteRT models as `web-official` / `portable` / `experimental` spike; add Ministral LiteRT only if spike loads. +- Add headless `vdl-model-eval` (suite, scorers, concurrency planner, HTML/JSON reports) plus `pnpm model-eval` Playwright WebGPU runner. +- Add Labs **Tools** site section (`#tools/model-eval`) with docs + report UI/charts — not in Interactive Demos. +- Add `@vanduo-oss/vd3-cbun` for eval charts on the Tools page. + +## Capabilities + +### New Capabilities +- `vdl-model-eval`: Local on-computer model evaluation harness, report format, and Tools-section presentation. +- `vdl-tools-nav`: Labs site Tools route for on-computer helper tools (not demos). + +### Modified Capabilities +- `vdl-ai-chat`: Multi-architecture LiteRT catalog; Tiny id; honest support labels; remove SmolLM2 Tiny. + +## Impact + +- Labs `vdl` surface: `ai-chat.js`, new `model-eval.js`, `utils/*`, `src/App.vue`, new `VdlModelEvalUI`, `doc/*`, package exports/scripts. +- Dependency: `@vanduo-oss/vd3-cbun` (charts). `@vanduo-oss/vd3` APIs unchanged. +- Local-only WebGPU eval (not CI); unit tests cover scorers and catalog ids. diff --git a/openspec/changes/vdl-model-eval-and-catalog/specs/vdl-ai-chat/spec.md b/openspec/changes/vdl-model-eval-and-catalog/specs/vdl-ai-chat/spec.md new file mode 100644 index 0000000..9ca3843 --- /dev/null +++ b/openspec/changes/vdl-model-eval-and-catalog/specs/vdl-ai-chat/spec.md @@ -0,0 +1,43 @@ +## ADDED Requirements + +### Requirement: Multi-architecture catalog with honest LiteRT labels +The model catalog MUST keep Gemma 4 E2B LiteRT as the default web-official path, MAY expose LiteRT spikes for other families labeled `spike` / experimental, and MUST offer a capable Tiny peer when LiteRT portable loads are blocked by the runtime. + +#### Scenario: Tiny Qwen3 WebLLM is available +- **WHEN** a user opens the model picker +- **THEN** a Tiny Qwen3 0.6B WebLLM option is listed + +#### Scenario: Official Gemma web remains default +- **WHEN** a user opens Labs AI chat without choosing another model +- **THEN** the default remains Gemma 4 E2B LiteRT with web-official support + +### Requirement: Tiny model is not SmolLM2 +The recommended Tiny / weak-device model MUST NOT be SmolLM2-360M. + +#### Scenario: Weak-device recommendation +- **WHEN** load-capacity heuristics recommend a Tiny model +- **THEN** the recommended model id is the Qwen3 0.6B Tiny entry (WebLLM) + +### Requirement: Outdated optional WebLLM models removed +The catalog MUST NOT offer SmolLM2-360M, Qwen2.5-1.5B, or Llama-3.2-3B as selectable options. + +#### Scenario: Removed models absent from picker +- **WHEN** the model picker lists optional WebLLM peers +- **THEN** SmolLM2-360M, Qwen2.5-1.5B, and Llama-3.2-3B are not present + +## MODIFIED Requirements + +### Requirement: LiteRT Gemma is the default chat path +The default model MUST be Gemma 4 E2B via LiteRT-LM, and that path MUST preserve real multi-turn conversation context across turns in a session. Additional LiteRT families MAY be offered as non-default options without changing the default identity. + +#### Scenario: Default model identity +- **WHEN** a user opens Labs AI chat without choosing another model +- **THEN** the selected default is Gemma 4 E2B on the LiteRT path + +#### Scenario: Multi-turn on LiteRT +- **WHEN** the user sends a follow-up message on the LiteRT Gemma default after a prior assistant reply +- **THEN** generation uses conversation context from prior turns (not latest-turn-only) + +#### Scenario: Multi-turn on WebLLM Qwen3 Tiny +- **WHEN** the user chats on WebLLM Qwen3 0.6B after a prior assistant reply +- **THEN** generation uses conversation context from prior turns diff --git a/openspec/changes/vdl-model-eval-and-catalog/specs/vdl-model-eval/spec.md b/openspec/changes/vdl-model-eval-and-catalog/specs/vdl-model-eval/spec.md new file mode 100644 index 0000000..741e76b --- /dev/null +++ b/openspec/changes/vdl-model-eval-and-catalog/specs/vdl-model-eval/spec.md @@ -0,0 +1,44 @@ +## Purpose + +Provides a local on-computer harness to evaluate in-browser chat models against a curated suite, emit Playwright-style HTML/JSON reports, and present published results on Labs Tools. + +## ADDED Requirements + +### Requirement: Curated evaluation suite +The eval harness MUST run a documented prompt suite covering Labs branding accuracy, honesty (admit mistakes), and basic instruction-following, and MUST score each case as pass or fail with a recorded model response excerpt. + +#### Scenario: Branding case fails inventing Vandouno +- **WHEN** a model response invents the misspelling “Vandouno” (or equivalent) for Vanduo Labs +- **THEN** the branding case is scored fail + +#### Scenario: Branding case requires Vanduo Labs +- **WHEN** a model is asked what site/org hosts the demo +- **THEN** a passing response includes “Vanduo Labs” (case-insensitive) and does not invent a false Labs brand spelling + +### Requirement: Architecture-comparable reports +Eval reports MUST identify each model by id, backend, and family so LiteRT Gemma can be compared to LiteRT Qwen3 (and optional WebLLM peers) in the same run. + +#### Scenario: Report lists family and backend +- **WHEN** an eval run completes for two models on different families +- **THEN** the JSON report includes per-model pass rate, latency metrics, backend, and family fields + +### Requirement: Published report artifacts +A successful local eval run MUST write JSON results and a self-contained HTML summary under a Labs-served reports directory that the Tools UI can load. + +#### Scenario: Latest report path +- **WHEN** `pnpm model-eval` finishes successfully +- **THEN** results are available under `data/model-eval-reports/latest/` (JSON and HTML) + +### Requirement: Concurrency respects resource budget +The runner MUST plan parallel model loads from approximate model size and a device RAM heuristic so concurrent residency stays within a configurable budget (targeting Apple Silicon ~24GB class machines). + +#### Scenario: Tiny plus Gemma concurrency +- **WHEN** the planner evaluates Gemma 4 E2B (~2GB) and Qwen3 0.6B LiteRT (~0.6GB) on a 24GB-class heuristic +- **THEN** both MAY be scheduled concurrently without exceeding the configured memory budget + +### Requirement: CI does not require WebGPU inference +Automated CI MUST validate scorers and report shaping with fixtures; it MUST NOT require real WebGPU model loads. + +#### Scenario: Unit scorer fixtures +- **WHEN** unit tests run in CI +- **THEN** branding/honesty scorers are exercised with fixture strings without loading a model diff --git a/openspec/changes/vdl-model-eval-and-catalog/specs/vdl-tools-nav/spec.md b/openspec/changes/vdl-model-eval-and-catalog/specs/vdl-tools-nav/spec.md new file mode 100644 index 0000000..7990cff --- /dev/null +++ b/openspec/changes/vdl-model-eval-and-catalog/specs/vdl-tools-nav/spec.md @@ -0,0 +1,30 @@ +## Purpose + +Adds a Labs site Tools section for on-computer helper tools, separate from Interactive Demos, starting with model evaluation documentation and published reports. + +## ADDED Requirements + +### Requirement: Tools route exists outside demos +The Labs site MUST expose a top-level Tools route that is not part of the Interactive Demos slug set. + +#### Scenario: Navigate to tools +- **WHEN** a user opens `#tools` +- **THEN** the Tools panel is shown with vdl-model-eval as the single tool (report UI and docs) and Interactive Demos content is not the active panel + +#### Scenario: Model eval tool deep link +- **WHEN** a user opens `#tools/model-eval` +- **THEN** the same Model Eval tool detail (docs and/or report UI) is shown + +### Requirement: Model Eval is not an Interactive Demo card +Model Eval MUST NOT appear in the Interactive Demos card grid (`DEMO_SLUGS`). + +#### Scenario: Demos grid excludes model-eval +- **WHEN** a user views `#demos` +- **THEN** Model Eval is not listed among Interactive Demo cards + +### Requirement: Tools use vd3 and may use vd3-cbun charts +The Tools Model Eval UI MUST use `@vanduo-oss/vd3` components and MAY render charts via `@vanduo-oss/vd3-cbun` without renaming design-system APIs. + +#### Scenario: Charts render from published report +- **WHEN** a published eval report JSON is available +- **THEN** the Tools Model Eval UI can display pass-rate or latency charts using vd3-cbun chart components diff --git a/openspec/changes/vdl-model-eval-and-catalog/tasks.md b/openspec/changes/vdl-model-eval-and-catalog/tasks.md new file mode 100644 index 0000000..d34e4c3 --- /dev/null +++ b/openspec/changes/vdl-model-eval-and-catalog/tasks.md @@ -0,0 +1,31 @@ +## 1. LiteRT spike + catalog + +- [x] 1.1 Extend `utils/fetch-ai-models.mjs` for Qwen3-0.6B LiteRT (and Ministral if probing) +- [x] 1.2 Spike-load Qwen3-0.6B via existing LiteRT path; attempt Ministral; document outcome +- [x] 1.3 Update `MODEL_OPTIONS` / groups / `TINY_MODEL_ID` (multi-arch LiteRT + refreshed WebLLM peers; remove SmolLM2/Qwen2.5-1.5B/Llama-3.2) +- [x] 1.4 Update `doc/vdl-ai-chat.md` and unit tests that hardcode removed models + +## 2. vdl-model-eval core + +- [x] 2.1 Add `model-eval.js` (scorers, suite runner API, report schema, concurrency planner) +- [x] 2.2 Add `utils/model-eval-suite.json` (branding, honesty, instruction-following) +- [x] 2.3 Export module + `pnpm model-eval` script in `package.json` +- [x] 2.4 Unit tests for scorers with fixture strings + +## 3. Eval runner + published report + +- [x] 3.1 Add browser harness `demo/model-eval-harness.html` + demo bootstrap +- [x] 3.2 Add `utils/model-eval-runner.mjs` (Playwright WebGPU, parallel workers) +- [x] 3.3 Commit a sample report under `data/model-eval-reports/latest/` (or fixture if live run unavailable) + +## 4. Tools section UI + +- [x] 4.1 Add `@vanduo-oss/vd3-cbun` dependency +- [x] 4.2 Add Tools route + Model Eval card in `src/App.vue` (not DEMO_SLUGS) +- [x] 4.3 Add `VdlModelEvalUI.vue` + `doc/vdl-model-eval.md` +- [x] 4.4 Wire Vite copy/build inputs as needed + +## 5. Verify + +- [x] 5.1 Run focused unit tests + `pnpm test` +- [x] 5.2 Mark OpenSpec tasks complete when implementation matches diff --git a/openspec/changes/vdl-theme-customizer/.openspec.yaml b/openspec/changes/vdl-theme-customizer/.openspec.yaml new file mode 100644 index 0000000..913564e --- /dev/null +++ b/openspec/changes/vdl-theme-customizer/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-08 diff --git a/openspec/changes/vdl-theme-customizer/design.md b/openspec/changes/vdl-theme-customizer/design.md new file mode 100644 index 0000000..f12ae0b --- /dev/null +++ b/openspec/changes/vdl-theme-customizer/design.md @@ -0,0 +1,42 @@ +## Context + +Labs already mounts `VdThemeSwitcher` in `App.vue` and sets `themeDefaults` via `VanduoVue` in `main.js`. `@vanduo-oss/vd3@1.2.1` hardcodes six `vanduo-*` localStorage keys inside `loadPreference` / `persistPreference` and does not expose a storage-prefix option. Labs and vd3-docs share the `vanduo-oss.github.io` origin, so those keys collide without an override. See proposal.md for motivation. + +## Goals / Non-Goals + +**Goals:** +- Mount `VdThemeCustomizer` next to the existing switcher in navbar actions. +- Remap the six vd3 theme storage keys to `vdl-*` before any theme preference I/O. +- Keep styling consistent with Labs navbar action controls. +- Cover the remap with a focused Playwright unit test. + +**Non-Goals:** +- Forking or patching `@vanduo-oss/vd3` package sources. +- Renaming `--vd-*` CSS variables or `Vd*` component APIs. +- Migrating existing `vanduo-*` values into `vdl-*` (would steal docs prefs on shared origin). +- Changing demos/tools routing or unrelated Labs storage keys. + +## Decisions + +1. **localStorage method remap (not a vd3 fork)** + Install a small Labs helper before `createApp` that wraps `localStorage.getItem` / `setItem` / `removeItem` and redirects only the six known theme keys from `vanduo-*` → `vdl-*`. Other keys pass through unchanged. + *Alternatives considered:* monkey-patching exported `persistPreference`/`loadPreference` (not overrideable for the singleton path without deep hooks); waiting for an upstream `storagePrefix` API (not available in 1.2.1). + +2. **Key names** — replace prefix only: + `vanduo-palette` → `vdl-palette`, `vanduo-primary-color` → `vdl-primary-color`, `vanduo-neutral-color` → `vdl-neutral-color`, `vanduo-radius` → `vdl-radius`, `vanduo-theme-preference` → `vdl-theme-preference`, `vanduo-font-preference` → `vdl-font-preference`. + +3. **UI placement** — navbar `#actions` next to GitHub link + `VdThemeSwitcher`, with `:show-palette="false"` (vd3-supported) so Fibonacci is not exposed; light CSS so the customizer trigger matches `.dark-mode-toggle` sizing. + +4. **No migration from `vanduo-*`** — Labs must not copy or delete docs keys on shared origin. + +5. **Labs themeDefaults** — pass site defaults through `VanduoVue` `{ themeDefaults }` (`FONT: 'open-sans'`, `NEUTRAL: 'neutral'`, `RADIUS: '0.25'`, `PALETTE: 'open-color'`). vd3 `loadPreference` uses `localStorage.getItem(key) ?? default`, so stored `vdl-*` prefs win; defaults apply only when unset. + +## Risks / Trade-offs + +- [Risk] Wrapping `localStorage` methods is global for the Labs page → Mitigation: remap only the six exact theme keys; leave all other keys untouched; install once before mount. +- [Risk] Upstream vd3 may add a prefix API later → Mitigation: keep remap in one Labs module so it can be removed when upstream supports configuration. +- [Trade-off] Users who previously set theme on Labs under `vanduo-*` will not auto-migrate; they re-pick once under `vdl-*` (acceptable to protect docs prefs). + +## Migration Plan + +Ship with the Labs site build. No data migration. Rollback: remove customizer mount + storage helper import. diff --git a/openspec/changes/vdl-theme-customizer/proposal.md b/openspec/changes/vdl-theme-customizer/proposal.md new file mode 100644 index 0000000..4042140 --- /dev/null +++ b/openspec/changes/vdl-theme-customizer/proposal.md @@ -0,0 +1,28 @@ +## Why + +Labs is text- and data-heavy; users benefit from vd3's theme customizer (palette, colors, radius, font). On GitHub Pages, Labs shares an origin with Vanduo docs, so default `vanduo-*` theme localStorage keys would collide — Labs must persist under `vdl-*` instead. + +## What Changes + +- Mount `VdThemeCustomizer` in the Labs site shell (navbar actions, alongside existing `VdThemeSwitcher`). +- Hide the Palette selector (Open Color / Fibonacci) and stay on Open Color via vd3's `showPalette` prop. +- Install a Labs-owned localStorage remap so vd3 theme reads/writes use `vdl-*` keys, leaving `vanduo-*` keys for docs untouched. +- Persist and restore theme preferences across reloads via the remapped keys. +- Add a focused unit test for the storage remap; light README note if helpful. + +## Capabilities + +### New Capabilities + +- `vdl-theme-customizer`: Labs shell exposes vd3 theme customization with `vdl-`-prefixed preference storage that does not collide with main Vanduo docs keys. + +### Modified Capabilities + +- (none) + +## Impact + +- Labs Vue shell: `src/main.js`, `src/App.vue`, `src/styles/labs.css`, new `src/vdl-theme-storage.js` (or equivalent). +- Tests: focused Playwright unit/smoke for storage remap. +- Docs: brief README note optional. +- Does **not** fork or rename `@vanduo-oss/vd3` APIs (`Vd*`, `--vd-*`, `.vd-*`). diff --git a/openspec/changes/vdl-theme-customizer/specs/vdl-theme-customizer/spec.md b/openspec/changes/vdl-theme-customizer/specs/vdl-theme-customizer/spec.md new file mode 100644 index 0000000..ef1d7d5 --- /dev/null +++ b/openspec/changes/vdl-theme-customizer/specs/vdl-theme-customizer/spec.md @@ -0,0 +1,62 @@ +## Purpose + +Expose vd3 theme customization on the Labs shell while persisting preferences under Labs-owned `vdl-*` localStorage keys so they do not collide with Vanduo docs on the shared GitHub Pages origin. + +## ADDED Requirements + +### Requirement: Theme customizer is available in the Labs shell + +The Labs site shell MUST expose `@vanduo-oss/vd3`'s `VdThemeCustomizer` in a discoverable chrome location (navbar actions) without breaking existing demos/tools routing. Design-system APIs (`Vd*`, `--vd-*`, `.vd-*`) MUST remain unchanged. + +#### Scenario: Customizer control is present in the navbar + +- **WHEN** a user opens the Labs site +- **THEN** a theme customizer control is available in the navbar actions area alongside the existing theme switcher + +#### Scenario: Customizer opens without changing route + +- **WHEN** a user opens the theme customizer while on a demos or tools route +- **THEN** the hash route remains unchanged and the customizer panel can be used to adjust theme preferences + +#### Scenario: Palette selector is not shown + +- **WHEN** a user opens the Labs theme customizer +- **THEN** the Palette segmented control (Open Color / Fibonacci) is not shown +- **AND** the site remains on the Open Color palette + +### Requirement: Theme preferences use vdl- localStorage keys + +Labs MUST persist and restore vd3 theme preferences using `vdl-` prefixed localStorage keys. Labs MUST NOT write theme preferences to the default `vanduo-` keys used by Vanduo docs. + +#### Scenario: Preference write uses vdl keys + +- **WHEN** a user changes a theme preference via the theme switcher or customizer on Labs +- **THEN** the preference is stored under the corresponding `vdl-*` key (for example `vdl-theme-preference`, `vdl-palette`, `vdl-primary-color`, `vdl-neutral-color`, `vdl-radius`, `vdl-font-preference`) +- **AND** the corresponding `vanduo-*` key is not written by Labs theme persistence + +#### Scenario: Preference restores on reload + +- **WHEN** a user has previously saved Labs theme preferences under `vdl-*` keys +- **AND** the Labs site loads again +- **THEN** those preferences are applied to the document theme attributes + +#### Scenario: Docs keys are not overwritten + +- **WHEN** `vanduo-*` theme keys already exist in localStorage (for example from Vanduo docs) +- **AND** the Labs site loads or updates theme preferences +- **THEN** Labs theme persistence reads and writes only `vdl-*` keys and leaves existing `vanduo-*` values intact + +### Requirement: Labs global theme defaults for unset preferences + +Labs MUST configure vd3 `themeDefaults` so new visitors (no stored `vdl-*` preference) get Open Sans, Neutral, and radius `0.25` on the Open Color palette. Labs MUST NOT overwrite existing `vdl-*` preferences with these defaults. + +#### Scenario: Defaults apply when no preference is stored + +- **WHEN** a visitor loads the Labs site with no `vdl-font-preference`, `vdl-neutral-color`, or `vdl-radius` keys +- **THEN** the effective theme uses font `open-sans`, neutral `neutral`, and radius `0.25` + +#### Scenario: Stored preferences override defaults + +- **WHEN** a visitor already has one or more `vdl-*` theme preference keys stored +- **AND** the Labs site loads +- **THEN** those stored values are applied instead of the corresponding Labs `themeDefaults` diff --git a/openspec/changes/vdl-theme-customizer/tasks.md b/openspec/changes/vdl-theme-customizer/tasks.md new file mode 100644 index 0000000..a28dc5e --- /dev/null +++ b/openspec/changes/vdl-theme-customizer/tasks.md @@ -0,0 +1,20 @@ +## 1. Storage remap + +- [x] 1.1 Add Labs helper that remaps the six vd3 theme localStorage keys from `vanduo-*` to `vdl-*` (get/set/remove only; no migration that deletes docs keys) +- [x] 1.2 Install the remap in `src/main.js` before `createApp` / `VanduoVue` mount + +## 2. UI integration + +- [x] 2.1 Mount `VdThemeCustomizer` in `App.vue` navbar actions beside `VdThemeSwitcher` with `:show-palette="false"` (Open Color only) +- [x] 2.2 Align customizer trigger styling with existing Labs navbar action controls in `labs.css` + +## 3. Docs and verification + +- [x] 3.1 Add a brief README note about theme customizer + `vdl-*` storage keys +- [x] 3.2 Add a focused Playwright unit test for the storage remap +- [x] 3.3 Run the focused unit test / `pnpm test` subset and confirm preferences persist under `vdl-*` + +## 4. Global theme defaults + +- [x] 4.1 Set Labs `themeDefaults` (Open Sans / Neutral / radius `0.25`, Open Color) via `VanduoVue` so new visitors get them when no `vdl-*` preference is stored +- [x] 4.2 Document defaults in README; assert default tokens in the unit test diff --git a/package.json b/package.json index 1b6eff2..22d7df0 100644 --- a/package.json +++ b/package.json @@ -4,6 +4,9 @@ "description": "vdl-neptune-search — in-browser hybrid fuzzy + semantic search for Vanduo Docs (ES modules)", "type": "module", "license": "MIT", + "engines": { + "node": ">=20.9.0" + }, "repository": { "type": "git", "url": "https://github.com/vanduo-oss/labs.git" @@ -20,6 +23,7 @@ ".": "./neptune-search.js", "./neptune-search.js": "./neptune-search.js", "./ai-chat.js": "./ai-chat.js", + "./model-eval.js": "./model-eval.js", "./guardrails/core.js": "./guardrails/core.js", "./guardrails/llm.js": "./guardrails/llm.js", "./guardrails/search.js": "./guardrails/search.js" @@ -27,11 +31,13 @@ "files": [ "neptune-search.js", "ai-chat.js", + "model-eval.js", "guardrails/core.js", "guardrails/llm.js", "guardrails/search.js", "data/search-index.json", "data/vectors.json", + "data/model-eval-reports", "README.md", "NEPTUNE-SEARCH.md", "LICENSE" @@ -43,6 +49,7 @@ "demo:serve": "vite", "index": "node utils/neptune-indexer.mjs", "models:fetch": "node utils/fetch-ai-models.mjs", + "model-eval": "node utils/model-eval-runner.mjs", "test": "playwright test", "test:ui": "playwright test --ui", "test:headed": "playwright test --headed", @@ -58,6 +65,7 @@ }, "dependencies": { "@vanduo-oss/vd3": "^1.2.1", + "@vanduo-oss/vd3-cbun": "^1.3.1", "vue": "^3.5.41" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 93dc066..cfc9afc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -16,6 +16,9 @@ importers: '@vanduo-oss/vd3': specifier: ^1.2.1 version: 1.2.1(vue@3.5.41) + '@vanduo-oss/vd3-cbun': + specifier: ^1.3.1 + version: 1.3.1(vue@3.5.41) vue: specifier: ^3.5.41 version: 3.5.41 @@ -588,6 +591,12 @@ packages: '@types/node@26.1.2': resolution: {integrity: sha512-Vu4a5UFA9rIIFJ7rB/Vaafh9lrCQszopTCx6KjFboXTGQbPNasehVR5TEiithSDGyd1DEiUByggTZsg8jukeIg==} + '@vanduo-oss/vd3-cbun@1.3.1': + resolution: {integrity: sha512-KUa6b9UM1IHK3a1jbWatOiPLFV0+1GSs/jnJwB7CUaDau61a1nCpGrJ4aFQ7nVW/0RYMyO47mlrFVeH5X7BP+g==} + engines: {node: '>=20.19.0', pnpm: '>=10'} + peerDependencies: + vue: '>=3.3.0' + '@vanduo-oss/vd3@1.2.1': resolution: {integrity: sha512-FFlb7aKMSR7mU6nAtMMt5K3g7zQIcUQRUYdRExZsAIX+VuhFLGg/+jkbEpY2oKJu0IuQL7mllWYP4WicWK+ZZg==} engines: {node: '>=20.19.0', pnpm: '>=10'} @@ -1349,6 +1358,10 @@ snapshots: dependencies: undici-types: 8.3.0 + '@vanduo-oss/vd3-cbun@1.3.1(vue@3.5.41)': + dependencies: + vue: 3.5.41 + '@vanduo-oss/vd3@1.2.1(vue@3.5.41)': dependencies: vue: 3.5.41 diff --git a/src/App.vue b/src/App.vue index 4a1d173..eff1275 100644 --- a/src/App.vue +++ b/src/App.vue @@ -6,38 +6,106 @@ import { VdIcon, VdModal, VdNavbar, + VdThemeCustomizer, VdThemeSwitcher, } from '@vanduo-oss/vd3'; import { DEFAULT_DOCS_BASE_URL, VDL_NEPTUNE_SEARCH_VERSION } from '../neptune-search.js'; import { VDL_AI_CHAT_VERSION } from '../ai-chat.js'; +import { VDL_MODEL_EVAL_VERSION } from '../model-eval.js'; import { labsMarkdownToHtml } from '../labs-md-to-html.js'; import VdlNeptuneSearchUI from './components/VdlNeptuneSearchUI.vue'; import VdlAiChatUI from './components/VdlAiChatUI.vue'; +import VdlModelEvalUI from './components/VdlModelEvalUI.vue'; +import { + isGladosHomeQuote, + nextHomeQuoteIntervalMs, + pickNextHomeQuote, +} from './vdl-home-quotes.js'; const DEMO_SLUGS = new Set(['neptune', 'aichat']); -const ROUTES = ['home', 'about', 'demos']; +const TOOL_SLUGS = new Set(['model-eval']); +const ROUTES = ['home', 'about', 'demos', 'tools']; const LABS_DEMOS_DISCLAIMER_KEY = 'vanduo-labs-demos-disclaimer-v1'; const DOCS_BASE_URL = DEFAULT_DOCS_BASE_URL; +const HOME_QUOTE_FADE_MS = 180; const COMPONENT_VERSION_MAP = { neptune: VDL_NEPTUNE_SEARCH_VERSION, aichat: VDL_AI_CHAT_VERSION, + 'model-eval': VDL_MODEL_EVAL_VERSION, }; const route = ref('home'); const demoSlug = ref(null); +const toolSlug = ref(null); const disclaimerOpen = ref(false); const docHtml = ref(''); const docLoading = ref(false); const docError = ref(''); const liveRegionText = ref(''); +/** @type {import('vue').Ref<{ id: string, text: string } | null>} */ +const homeQuoteEntry = ref(null); +const homeQuoteVisible = ref(true); +const homeQuoteIsGlados = computed(() => isGladosHomeQuote(homeQuoteEntry.value)); -const docHtmlCache = { neptune: null, aichat: null }; +const docHtmlCache = { neptune: null, aichat: null, 'model-eval': null }; let docLoadSeq = 0; -let lastTopLevelRoute = 'home'; +/** Empty until first hash sync so the initial `#home` visit advances the quote bag. */ +let lastTopLevelRoute = ''; +let homeQuoteTimer = null; +let homeQuoteFadeTimer = null; + +function clearHomeQuoteTimers() { + if (homeQuoteTimer != null) { + clearTimeout(homeQuoteTimer); + homeQuoteTimer = null; + } + if (homeQuoteFadeTimer != null) { + clearTimeout(homeQuoteFadeTimer); + homeQuoteFadeTimer = null; + } +} + +function applyHomeQuotePick(animate) { + if (homeQuoteFadeTimer != null) { + clearTimeout(homeQuoteFadeTimer); + homeQuoteFadeTimer = null; + } + const { entry } = pickNextHomeQuote(); + if (!animate || !homeQuoteEntry.value) { + homeQuoteEntry.value = entry; + homeQuoteVisible.value = true; + return; + } + homeQuoteVisible.value = false; + homeQuoteFadeTimer = window.setTimeout(() => { + homeQuoteEntry.value = entry; + homeQuoteVisible.value = true; + homeQuoteFadeTimer = null; + }, HOME_QUOTE_FADE_MS); +} + +function scheduleHomeQuoteTick() { + clearTimeout(homeQuoteTimer); + homeQuoteTimer = window.setTimeout(() => { + applyHomeQuotePick(true); + scheduleHomeQuoteTick(); + }, nextHomeQuoteIntervalMs()); +} + +function startHomeQuoteRotation() { + clearHomeQuoteTimers(); + applyHomeQuotePick(false); + scheduleHomeQuoteTick(); +} + +function stopHomeQuoteRotation() { + clearHomeQuoteTimers(); +} const neptuneVersion = computed(() => `v${COMPONENT_VERSION_MAP.neptune}`); const aichatVersion = computed(() => `v${COMPONENT_VERSION_MAP.aichat}`); +const modelEvalVersion = computed(() => `v${COMPONENT_VERSION_MAP['model-eval']}`); function getComponentVersion(slug) { return COMPONENT_VERSION_MAP[slug] || '0.0.1'; @@ -54,10 +122,15 @@ function parseLabsHash() { if (nextRoute === 'demos') { let nextDemo = segments.length > 1 ? segments[1] : null; if (nextDemo && !DEMO_SLUGS.has(nextDemo)) nextDemo = null; - return { route: nextRoute, demoSlug: nextDemo }; + return { route: nextRoute, demoSlug: nextDemo, toolSlug: null }; + } + if (nextRoute === 'tools') { + let nextTool = segments.length > 1 ? segments[1] : 'model-eval'; + if (!TOOL_SLUGS.has(nextTool)) nextTool = 'model-eval'; + return { route: nextRoute, demoSlug: null, toolSlug: nextTool }; } if (!ROUTES.includes(nextRoute)) nextRoute = 'home'; - return { route: nextRoute, demoSlug: null }; + return { route: nextRoute, demoSlug: null, toolSlug: null }; } function closeDisclaimer() { @@ -85,7 +158,11 @@ function acceptDisclaimer() { async function fetchDocumentationHtml(slug) { if (docHtmlCache[slug]) return docHtmlCache[slug]; const path = - slug === 'neptune' ? '/doc/vdl-neptune-search.md' : '/doc/vdl-ai-chat.md'; + slug === 'neptune' + ? '/doc/vdl-neptune-search.md' + : slug === 'model-eval' + ? '/doc/vdl-model-eval.md' + : '/doc/vdl-ai-chat.md'; const res = await fetch(path, { credentials: 'same-origin' }); if (!res.ok) throw new Error(`Could not load documentation (${res.status})`); const md = hydrateComponentVersionTokens(await res.text(), slug); @@ -112,16 +189,23 @@ async function loadDocumentationForSlug(slug) { } } -function applyLabsRoute(nextRoute, nextDemoSlug) { +function applyLabsRoute(nextRoute, nextDemoSlug, nextToolSlug) { if (nextRoute !== 'demos') closeDisclaimer(); const routeChanged = lastTopLevelRoute !== nextRoute; lastTopLevelRoute = nextRoute; route.value = nextRoute; demoSlug.value = nextRoute === 'demos' ? nextDemoSlug : null; + toolSlug.value = nextRoute === 'tools' ? nextToolSlug : null; if (routeChanged) window.scrollTo(0, 0); + if (nextRoute === 'home') { + if (routeChanged) startHomeQuoteRotation(); + } else if (routeChanged) { + stopHomeQuoteRotation(); + } + if (nextRoute === 'demos') { maybeOpenDemosDisclaimer(); if (nextDemoSlug) { @@ -141,6 +225,9 @@ function applyLabsRoute(nextRoute, nextDemoSlug) { docError.value = ''; docLoading.value = false; } + } else if (nextRoute === 'tools') { + liveRegionText.value = 'vdl-model-eval tool and documentation opened.'; + loadDocumentationForSlug(nextToolSlug || 'model-eval'); } } @@ -148,9 +235,9 @@ function syncLabsRouteFromHash() { let parsed = parseLabsHash(); if (!location.hash) { history.replaceState(null, '', '#home'); - parsed = { route: 'home', demoSlug: null }; + parsed = { route: 'home', demoSlug: null, toolSlug: null }; } - applyLabsRoute(parsed.route, parsed.demoSlug); + applyLabsRoute(parsed.route, parsed.demoSlug, parsed.toolSlug); } function selectDemo(slug) { @@ -169,6 +256,7 @@ onMounted(() => { onBeforeUnmount(() => { window.removeEventListener('hashchange', syncLabsRouteFromHash); + stopHomeQuoteRotation(); }); watch(demoSlug, (slug) => { @@ -176,6 +264,12 @@ watch(demoSlug, (slug) => { loadDocumentationForSlug(slug); } }); + +watch(toolSlug, (slug) => { + if (route.value === 'tools' && slug) { + loadDocumentationForSlug(slug); + } +});