feat(generator): 优化生成器上传和错误处理

- 改进生成器上传功能，使用原始二进制流读取文件，避免UTF-8编码损坏
- 更新错误处理，增强服务器错误日志，包含堆栈信息
- 修改请求处理逻辑，支持二进制流读取
- 新增生成器端到端集成测试，验证上传和市场交互功能

Author: Geniusay

content-forest-backend/src/api/generators.ts

@@ -103,7 +103,22 @@ export function createGeneratorRoutes(service: GeneratorService): Hono {
       return c.json({ code: 400, message: `platform must be one of: ${validPlatforms.join(", ")}` }, 400)
     }
 
-    const zipBuffer = Buffer.from(await file.arrayBuffer())
+    // 用 file.stream() 读取原始二进制，避免 formData 解析时 UTF-8 编码损坏二进制内容
+    const chunks: Uint8Array[] = []
+    const reader = file.stream().getReader()
+    while (true) {
+      const { done, value } = await reader.read()
+      if (done) break
+      if (value) chunks.push(value)
+    }
+    const totalLen = chunks.reduce((s, c) => s + c.length, 0)
+    const zipBuf = Buffer.allocUnsafe(totalLen)
+    let pos = 0
+    for (const chunk of chunks) { zipBuf.set(chunk, pos); pos += chunk.length }
+    process.stderr.write(`[upload] zip size=${zipBuf.length} first4: ${zipBuf.slice(0,4).toString('hex')}\n`)
+    if (zipBuf.length === 0) {
+      return c.json({ code: 400, message: "file is empty" }, 400)
+    }
     const safeJson = (raw: FormDataEntryValue | null): string[] => {
       if (!raw) return []
       try { return JSON.parse(String(raw)) } catch { return [] }
@@ -120,7 +135,7 @@ export function createGeneratorRoutes(service: GeneratorService): Hono {
       price: Number(formData.get("price") ?? 0),
     }
 
-    const result = await service.upload(userId, dto, zipBuffer)
+    const result = await service.upload(userId, dto, zipBuf)
     return c.json({ code: 0, data: result }, 201)
   })
 

content-forest-backend/src/api/server.ts

@@ -72,7 +72,7 @@ app.onError((err, c) => {
   if (err instanceof InvalidSkillPackageError) {
     return c.json({ code: 400, message: err.message }, 400)
   }
-  process.stderr.write(`[Server] unhandled error: ${err.message}\n`)
+  process.stderr.write(`[Server] unhandled error: ${err.message}\n${err.stack ?? ""}\n`)
   return c.json({ code: 500, message: "Internal Server Error" }, 500)
 })
 
@@ -105,7 +105,7 @@ export async function handleRequest(
     ),
     body: ["GET", "HEAD"].includes(request.method ?? "GET")
       ? undefined
-      : await readStream(request),
+      : await readStreamBinary(request),
   })
 
   const res = await app.fetch(req)
@@ -120,11 +120,12 @@ export async function handleRequest(
 // 内部工具
 // ---------------------------------------------------------------------------
 
-async function readStream(stream: IncomingMessage): Promise<string> {
+async function readStreamBinary(stream: IncomingMessage): Promise<Buffer> {
   return new Promise((resolve, reject) => {
     const chunks: Buffer[] = []
     stream.on("data", (chunk: Buffer) => chunks.push(chunk))
-    stream.on("end", () => resolve(Buffer.concat(chunks).toString("utf-8")))
+    stream.on("end", () => resolve(Buffer.concat(chunks)))
     stream.on("error", reject)
   })
 }
+

content-forest-backend/src/storage/generator-fs.ts

@@ -14,8 +14,11 @@
 
 import fs from "node:fs/promises"
 import path from "node:path"
+import { createRequire } from "node:module"
 import { CF_DATA_ROOT } from "../config.js"
 
+const require = createRequire(import.meta.url)
+
 /** 平台生成器 Skill 存储根目录 */
 export const CF_PLATFORM_ROOT =
   process.env.CF_PLATFORM_ROOT ??
@@ -147,34 +150,60 @@ export class InvalidSkillPackageError extends Error {
 }
 
 /**
- * 解压 zip Buffer，返回文件内容 Map<相对路径, Buffer>。
- * 校验根目录必须包含 SKILL.md，否则抛 InvalidSkillPackageError。
- *
- * 依赖：unzipper（需在 package.json 中声明）
+ * 纯 Node.js zip 解析（无外部依赖，兼容 Node v23）。
+ * 支持 STORED (0) 和 DEFLATED (8) 压缩方式。
  */
 export async function extractAndValidateZip(
   zipBuffer: Buffer
 ): Promise<Map<string, Buffer>> {
-  const unzipper = await import("unzipper").catch(() => {
-    throw new Error(
-      "unzipper is not installed. Run: npm install unzipper @types/unzipper"
-    )
-  })
-
-  const directory = await unzipper.Open.buffer(zipBuffer)
+  const { inflateRawSync } = require("node:zlib") as typeof import("node:zlib")
   const files = new Map<string, Buffer>()
 
-  // 收集所有文件
-  for (const file of directory.files) {
-    if (file.type === "Directory") continue
-    const content = await file.buffer()
-    // 去除 zip 根目录前缀（如 my-skill/SKILL.md → SKILL.md）
-    const parts = file.path.split("/")
-    const relPath = parts.length > 1 ? parts.slice(1).join("/") : parts[0]
-    if (relPath) files.set(relPath, content)
+  // 强制拷贝一份独立 Buffer，彻底避免内存池引用问题
+  const buf = Buffer.allocUnsafe(zipBuffer.length)
+  zipBuffer.copy(buf)
+
+  process.stderr.write(`[zip] parse start, len=${buf.length}, sig=${buf.readUInt32LE(0).toString(16)}\n`)
+
+  let offset = 0
+  while (offset < buf.length - 4) {
+    const sig = buf.readUInt32LE(offset)
+    if (sig !== 0x04034b50) break // Local file header signature
+
+    const compression = buf.readUInt16LE(offset + 8)
+    const compSize    = buf.readUInt32LE(offset + 18)
+    const nameLen     = buf.readUInt16LE(offset + 26)
+    const extraLen    = buf.readUInt16LE(offset + 28)
+    const name        = buf.slice(offset + 30, offset + 30 + nameLen).toString("utf-8")
+    const dataOffset  = offset + 30 + nameLen + extraLen
+    const compData    = buf.slice(dataOffset, dataOffset + compSize)
+
+    process.stderr.write(`[zip] entry name=${name} comp=${compression} compSize=${compSize} dataOffset=${dataOffset} compDataLen=${compData.length}\n`)
+
+    if (!name.endsWith("/")) {
+      let content: Buffer
+      if (compression === 0) {
+        content = Buffer.from(compData) // 显式拷贝
+      } else if (compression === 8) {
+        content = inflateRawSync(compData)
+      } else {
+        throw new InvalidSkillPackageError(`Unsupported compression method ${compression} for: ${name}`)
+      }
+      const parts = name.split("/")
+      const relPath = parts.length > 1 ? parts.slice(1).join("/") : parts[0]!
+      if (relPath) files.set(relPath, content)
+      process.stderr.write(`[zip] stored relPath=${relPath} contentLen=${content.length}\n`)
+    }
+
+    offset = dataOffset + compSize
+  }
+
+  process.stderr.write(`[zip] total files: ${files.size} [${[...files.keys()].join(', ')}]\n`)
+
+  if (files.size === 0) {
+    throw new InvalidSkillPackageError("Invalid skill package: zip appears empty or corrupt")
   }
 
-  // 校验 SKILL.md 存在
   const hasSkillMd = [...files.keys()].some(
     (p) => p === "SKILL.md" || p.toLowerCase() === "skill.md"
   )

content-forest-backend/src/storage/redis-client.ts

@@ -11,10 +11,15 @@ import { Redis } from "ioredis"
 import { REDIS_URL } from "../config.js"
 
 const redis = new Redis(REDIS_URL, {
-  // 连接失败时自动重试，最多 3 次，避免启动时因 Redis 未就绪崩溃
-  maxRetriesPerRequest: 3,
-  enableReadyCheck: true,
+  maxRetriesPerRequest: 1,       // 快速失败，不长时间等待
+  enableReadyCheck: false,       // 不等 ready 再发命令
   lazyConnect: false,
+  retryStrategy: (times: number) => {
+    if (times > 10) return null   // 超过 10 次停止重试
+    return Math.min(times * 200, 2000) // 200ms 起步，最多 2s
+  },
+  reconnectOnError: () => true,   // 任何错误都尝试重连
+  commandTimeout: 5000,          // 单个命令 5s 超时
 })
 
 redis.on("error", (err: Error) => {