|
| 1 | +#!/usr/bin/env node |
| 2 | +/** |
| 3 | + * Translates data.js (EN) → data-cn.js (CN) using an LLM API. |
| 4 | + * Only re-translates fields whose content hash has changed. |
| 5 | + * |
| 6 | + * Usage: |
| 7 | + * TRANSLATE_API_KEY=sk-... node translate.js |
| 8 | + * node translate.js --dry-run |
| 9 | + * |
| 10 | + * Env vars: |
| 11 | + * TRANSLATE_API_KEY — OpenAI API key (required unless --dry-run) |
| 12 | + * TRANSLATE_MODEL — model name (default: gpt-4o-mini) |
| 13 | + * TRANSLATE_BASE_URL — custom API base URL (optional, for compatible providers) |
| 14 | + */ |
| 15 | + |
| 16 | +import { createHash } from "node:crypto"; |
| 17 | +import { readFileSync, writeFileSync, existsSync } from "node:fs"; |
| 18 | +import { resolve, dirname } from "node:path"; |
| 19 | +import { fileURLToPath } from "node:url"; |
| 20 | + |
| 21 | +const __dirname = dirname(fileURLToPath(import.meta.url)); |
| 22 | +const SITE_DIR = resolve(__dirname, "../site"); |
| 23 | +const DATA_PATH = resolve(SITE_DIR, "data.js"); |
| 24 | +const DATA_CN_PATH = resolve(SITE_DIR, "data-cn.js"); |
| 25 | +const HASH_PATH = resolve(__dirname, ".translate-hashes.json"); |
| 26 | + |
| 27 | +const DRY_RUN = process.argv.includes("--dry-run"); |
| 28 | +const MODEL = process.env.TRANSLATE_MODEL || "gpt-4o-mini"; |
| 29 | +const API_KEY = process.env.TRANSLATE_API_KEY; |
| 30 | +const BASE_URL = process.env.TRANSLATE_BASE_URL || undefined; |
| 31 | + |
| 32 | +if (!DRY_RUN && !API_KEY) { |
| 33 | + console.error("Error: TRANSLATE_API_KEY env var is required (or use --dry-run)"); |
| 34 | + process.exit(1); |
| 35 | +} |
| 36 | + |
| 37 | +function md5(str) { |
| 38 | + return createHash("md5").update(str).digest("hex"); |
| 39 | +} |
| 40 | + |
| 41 | +function loadWiki() { |
| 42 | + const src = readFileSync(DATA_PATH, "utf-8"); |
| 43 | + const fn = new Function(src + "\n; return WIKI;"); |
| 44 | + return fn(); |
| 45 | +} |
| 46 | + |
| 47 | +function loadExistingCN() { |
| 48 | + if (!existsSync(DATA_CN_PATH)) return { pages: {}, concepts: {}, taxonomy: [] }; |
| 49 | + const src = readFileSync(DATA_CN_PATH, "utf-8"); |
| 50 | + const fn = new Function(src + "\n; return WIKI_CN;"); |
| 51 | + try { return fn(); } catch { return { pages: {}, concepts: {}, taxonomy: [] }; } |
| 52 | +} |
| 53 | + |
| 54 | +function loadHashes() { |
| 55 | + if (!existsSync(HASH_PATH)) return {}; |
| 56 | + try { return JSON.parse(readFileSync(HASH_PATH, "utf-8")); } catch { return {}; } |
| 57 | +} |
| 58 | + |
| 59 | +function saveHashes(hashes) { |
| 60 | + writeFileSync(HASH_PATH, JSON.stringify(hashes, null, 2)); |
| 61 | +} |
| 62 | + |
| 63 | +function extractTranslatableFields(wiki) { |
| 64 | + const fields = []; |
| 65 | + |
| 66 | + for (const domain of wiki.taxonomy) { |
| 67 | + fields.push({ key: `taxonomy.${domain.id}.label`, value: domain.label }); |
| 68 | + for (const child of domain.children) { |
| 69 | + fields.push({ key: `taxonomy.${domain.id}.${child.id}.label`, value: child.label }); |
| 70 | + } |
| 71 | + } |
| 72 | + |
| 73 | + for (const [id, page] of Object.entries(wiki.pages)) { |
| 74 | + if (page.title) fields.push({ key: `pages.${id}.title`, value: page.title }); |
| 75 | + if (page.subtitle) fields.push({ key: `pages.${id}.subtitle`, value: page.subtitle }); |
| 76 | + if (page.body) fields.push({ key: `pages.${id}.body`, value: page.body }); |
| 77 | + } |
| 78 | + |
| 79 | + if (wiki.concepts) { |
| 80 | + for (const [id, concept] of Object.entries(wiki.concepts)) { |
| 81 | + if (concept.name) fields.push({ key: `concepts.${id}.name`, value: concept.name }); |
| 82 | + if (concept.role) fields.push({ key: `concepts.${id}.role`, value: concept.role }); |
| 83 | + if (concept.summary) fields.push({ key: `concepts.${id}.summary`, value: concept.summary }); |
| 84 | + if (concept.definition) fields.push({ key: `concepts.${id}.definition`, value: concept.definition }); |
| 85 | + if (concept.examples) fields.push({ key: `concepts.${id}.examples`, value: concept.examples }); |
| 86 | + } |
| 87 | + } |
| 88 | + |
| 89 | + return fields; |
| 90 | +} |
| 91 | + |
| 92 | +function findChangedFields(fields, hashes) { |
| 93 | + const changed = []; |
| 94 | + for (const f of fields) { |
| 95 | + const hash = md5(f.value); |
| 96 | + if (hashes[f.key] !== hash) { |
| 97 | + changed.push({ ...f, hash }); |
| 98 | + } |
| 99 | + } |
| 100 | + return changed; |
| 101 | +} |
| 102 | + |
| 103 | +const SYSTEM_PROMPT = `You are a professional technical translator for a personal knowledge wiki about compilers, AI infrastructure, high-performance computing, and programming language theory. |
| 104 | +
|
| 105 | +Rules: |
| 106 | +- Translate English to natural, idiomatic Simplified Chinese suitable for a Chinese software engineer audience. |
| 107 | +- Preserve ALL HTML tags, attributes, code blocks (<pre>, <code>), and their contents EXACTLY unchanged. |
| 108 | +- Preserve technical terms that are universally used in English: MLIR, LLVM, SSA, IR, TensorFlow, XLA, HLO, CUDA, GPU, CPU, API, etc. |
| 109 | +- For terms that have well-known Chinese translations, use them: compiler=编译器, optimization=优化, abstraction=抽象, etc. |
| 110 | +- Keep proper nouns (project names, people names) unchanged. |
| 111 | +- Output ONLY the translated text, no explanations or markdown wrappers.`; |
| 112 | + |
| 113 | +async function translateBatch(fields) { |
| 114 | + const baseURL = BASE_URL || "https://api.openai.com/v1"; |
| 115 | + const results = []; |
| 116 | + const BATCH_SIZE = 5; |
| 117 | + |
| 118 | + for (let i = 0; i < fields.length; i += BATCH_SIZE) { |
| 119 | + const batch = fields.slice(i, i + BATCH_SIZE); |
| 120 | + const prompt = batch.map((f, idx) => |
| 121 | + `--- FIELD ${idx + 1} [${f.key}] ---\n${f.value}` |
| 122 | + ).join("\n\n"); |
| 123 | + |
| 124 | + const userMsg = `Translate each field below to Simplified Chinese. Return the translations in the same order, separated by the same "--- FIELD N [key] ---" delimiters. Keep code blocks and HTML tags unchanged.\n\n${prompt}`; |
| 125 | + |
| 126 | + console.log(` Translating batch ${Math.floor(i / BATCH_SIZE) + 1}/${Math.ceil(fields.length / BATCH_SIZE)} (${batch.length} fields)...`); |
| 127 | + |
| 128 | + const resp = await fetch(`${baseURL}/chat/completions`, { |
| 129 | + method: "POST", |
| 130 | + headers: { |
| 131 | + "Content-Type": "application/json", |
| 132 | + "Authorization": `Bearer ${API_KEY}` |
| 133 | + }, |
| 134 | + body: JSON.stringify({ |
| 135 | + model: MODEL, |
| 136 | + messages: [ |
| 137 | + { role: "system", content: SYSTEM_PROMPT }, |
| 138 | + { role: "user", content: userMsg } |
| 139 | + ], |
| 140 | + temperature: 0.3, |
| 141 | + max_tokens: 16000 |
| 142 | + }) |
| 143 | + }); |
| 144 | + |
| 145 | + if (!resp.ok) { |
| 146 | + const errBody = await resp.text(); |
| 147 | + throw new Error(`API request failed (${resp.status}): ${errBody}`); |
| 148 | + } |
| 149 | + |
| 150 | + const data = await resp.json(); |
| 151 | + const text = data.choices[0].message.content; |
| 152 | + const parts = text.split(/---\s*FIELD\s+\d+\s*\[.*?\]\s*---/).filter(s => s.trim()); |
| 153 | + |
| 154 | + for (let j = 0; j < batch.length; j++) { |
| 155 | + results.push({ |
| 156 | + key: batch[j].key, |
| 157 | + hash: batch[j].hash, |
| 158 | + translated: (parts[j] || "").trim() |
| 159 | + }); |
| 160 | + } |
| 161 | + } |
| 162 | + |
| 163 | + return results; |
| 164 | +} |
| 165 | + |
| 166 | +function buildCNObject(wiki, translations) { |
| 167 | + const existing = loadExistingCN(); |
| 168 | + const cn = { |
| 169 | + taxonomy: existing.taxonomy || [], |
| 170 | + pages: { ...existing.pages }, |
| 171 | + concepts: { ...existing.concepts } |
| 172 | + }; |
| 173 | + |
| 174 | + const txMap = {}; |
| 175 | + for (const t of translations) { txMap[t.key] = t.translated; } |
| 176 | + |
| 177 | + // Rebuild taxonomy translations |
| 178 | + const taxonomyCN = []; |
| 179 | + for (const domain of wiki.taxonomy) { |
| 180 | + const domainCN = { |
| 181 | + id: domain.id, |
| 182 | + label: txMap[`taxonomy.${domain.id}.label`] || existing.taxonomy?.find(d => d.id === domain.id)?.label || domain.label, |
| 183 | + children: [] |
| 184 | + }; |
| 185 | + for (const child of domain.children) { |
| 186 | + domainCN.children.push({ |
| 187 | + id: child.id, |
| 188 | + label: txMap[`taxonomy.${domain.id}.${child.id}.label`] || |
| 189 | + existing.taxonomy?.find(d => d.id === domain.id)?.children?.find(c => c.id === child.id)?.label || |
| 190 | + child.label, |
| 191 | + type: child.type |
| 192 | + }); |
| 193 | + } |
| 194 | + taxonomyCN.push(domainCN); |
| 195 | + } |
| 196 | + cn.taxonomy = taxonomyCN; |
| 197 | + |
| 198 | + for (const [id, page] of Object.entries(wiki.pages)) { |
| 199 | + if (!cn.pages[id]) cn.pages[id] = {}; |
| 200 | + if (txMap[`pages.${id}.title`]) cn.pages[id].title = txMap[`pages.${id}.title`]; |
| 201 | + if (txMap[`pages.${id}.subtitle`]) cn.pages[id].subtitle = txMap[`pages.${id}.subtitle`]; |
| 202 | + if (txMap[`pages.${id}.body`]) cn.pages[id].body = txMap[`pages.${id}.body`]; |
| 203 | + } |
| 204 | + |
| 205 | + if (wiki.concepts) { |
| 206 | + for (const [id, concept] of Object.entries(wiki.concepts)) { |
| 207 | + if (!cn.concepts[id]) cn.concepts[id] = {}; |
| 208 | + if (txMap[`concepts.${id}.name`]) cn.concepts[id].name = txMap[`concepts.${id}.name`]; |
| 209 | + if (txMap[`concepts.${id}.role`]) cn.concepts[id].role = txMap[`concepts.${id}.role`]; |
| 210 | + if (txMap[`concepts.${id}.summary`]) cn.concepts[id].summary = txMap[`concepts.${id}.summary`]; |
| 211 | + if (txMap[`concepts.${id}.definition`]) cn.concepts[id].definition = txMap[`concepts.${id}.definition`]; |
| 212 | + if (txMap[`concepts.${id}.examples`]) cn.concepts[id].examples = txMap[`concepts.${id}.examples`]; |
| 213 | + } |
| 214 | + } |
| 215 | + |
| 216 | + return cn; |
| 217 | +} |
| 218 | + |
| 219 | +function serializeCN(cn) { |
| 220 | + let out = "const WIKI_CN = "; |
| 221 | + out += JSON.stringify(cn, null, 2); |
| 222 | + out += ";\n"; |
| 223 | + return out; |
| 224 | +} |
| 225 | + |
| 226 | +async function main() { |
| 227 | + console.log("🔍 Loading data.js..."); |
| 228 | + const wiki = loadWiki(); |
| 229 | + const hashes = loadHashes(); |
| 230 | + |
| 231 | + console.log("📝 Extracting translatable fields..."); |
| 232 | + const fields = extractTranslatableFields(wiki); |
| 233 | + console.log(` Found ${fields.length} translatable fields.`); |
| 234 | + |
| 235 | + const changed = findChangedFields(fields, hashes); |
| 236 | + console.log(` ${changed.length} field(s) need translation.`); |
| 237 | + |
| 238 | + if (changed.length === 0) { |
| 239 | + console.log("✅ Nothing to translate — data-cn.js is up to date."); |
| 240 | + return; |
| 241 | + } |
| 242 | + |
| 243 | + if (DRY_RUN) { |
| 244 | + console.log("\n[DRY RUN] Would translate these fields:"); |
| 245 | + for (const f of changed) { |
| 246 | + console.log(` - ${f.key} (${f.value.length} chars)`); |
| 247 | + } |
| 248 | + return; |
| 249 | + } |
| 250 | + |
| 251 | + console.log("\n🌐 Translating via LLM..."); |
| 252 | + const translations = await translateBatch(changed); |
| 253 | + |
| 254 | + console.log("\n📦 Building data-cn.js..."); |
| 255 | + const cn = buildCNObject(wiki, translations); |
| 256 | + const output = serializeCN(cn); |
| 257 | + writeFileSync(DATA_CN_PATH, output, "utf-8"); |
| 258 | + console.log(` Written to ${DATA_CN_PATH} (${output.length} bytes)`); |
| 259 | + |
| 260 | + // Update hashes |
| 261 | + for (const t of translations) { |
| 262 | + if (t.translated) hashes[t.key] = t.hash; |
| 263 | + } |
| 264 | + saveHashes(hashes); |
| 265 | + console.log("✅ Done."); |
| 266 | +} |
| 267 | + |
| 268 | +main().catch(err => { |
| 269 | + console.error("Translation failed:", err.message); |
| 270 | + process.exit(1); |
| 271 | +}); |
0 commit comments