diff --git a/electron/mcpLocalServer.js b/electron/mcpLocalServer.js index afed4420..5d48690c 100644 --- a/electron/mcpLocalServer.js +++ b/electron/mcpLocalServer.js @@ -23,6 +23,7 @@ function performBasicTextSearch(repos, query) { ...(repo.ai_platforms || []), ...(repo.custom_tags || []), repo.custom_category || '', + normalizeLicense(repo.license), ] .join(' ') .toLowerCase(); @@ -30,6 +31,26 @@ function performBasicTextSearch(repos, query) { }); } +// License 归一化镜像(与 src/utils/licenseFilter.ts 一致) +const NO_LICENSE_SENTINEL = '__NO_LICENSE__'; +const NOASSERTION_KEYS = new Set(['', 'noassertion', 'other', 'none', 'no-license']); +function normalizeLicense(v) { + if (v == null || v === '') return NO_LICENSE_SENTINEL; + if (typeof v === 'object') { + const spdx = typeof v.spdx_id === 'string' ? v.spdx_id.trim() : ''; + const key = typeof v.key === 'string' ? v.key.trim() : ''; + const resolved = spdx || key; + if (!resolved) return NO_LICENSE_SENTINEL; + return NOASSERTION_KEYS.has(resolved.toLowerCase()) ? NO_LICENSE_SENTINEL : resolved; + } + if (typeof v !== 'string') return NO_LICENSE_SENTINEL; + // 直接字符串路径也需 trim:避免 " Other " / " NOASSERTION " 等空白变体逃过哨兵归并 + const normalized = v.trim(); + return !normalized || NOASSERTION_KEYS.has(normalized.toLowerCase()) + ? NO_LICENSE_SENTINEL + : normalized; +} + function projectRepo(repo, max = 400) { const summary = repo.ai_summary || repo.custom_description || repo.description || null; const truncated = @@ -54,6 +75,7 @@ function projectRepo(repo, max = 400) { starred_at: repo.starred_at, updated_at: repo.updated_at, pushed_at: repo.pushed_at, + license: repo.license ?? null, }; } @@ -265,6 +287,11 @@ function getTools(vectorAvailable) { query: { type: 'string' }, languages: { type: 'array', items: { type: 'string' } }, tags: { type: 'array', items: { type: 'string' } }, + licenses: { + type: 'array', + items: { type: 'string' }, + description: 'SPDX id list (e.g. ["MIT","Apache-2.0"]); use "__NO_LICENSE__" for repos with no license', + }, category: { type: 'string' }, minStars: { type: 'number' }, maxStars: { type: 'number' }, @@ -362,6 +389,9 @@ async function callTool(name, args, snapshot) { return args.tags.some((t) => tags.includes(t)); }); } + if (args?.licenses?.length) { + list = list.filter((r) => args.licenses.includes(normalizeLicense(r.license))); + } if (args?.category) { list = list.filter((r) => r.custom_category === args.category); } @@ -400,11 +430,14 @@ async function callTool(name, args, snapshot) { } case 'gsm_stats': { const byLanguage = {}; + const byLicense = {}; let analyzed = 0; let subscribed = 0; for (const r of repos) { const lang = r.language || 'Unknown'; byLanguage[lang] = (byLanguage[lang] || 0) + 1; + const lic = normalizeLicense(r.license); + byLicense[lic] = (byLicense[lic] || 0) + 1; if (r.analyzed_at && !r.analysis_failed) analyzed += 1; if (r.subscribed_to_releases) subscribed += 1; } @@ -413,6 +446,7 @@ async function callTool(name, args, snapshot) { analyzed, subscribedToReleases: subscribed, byLanguage, + byLicense, }); } case 'gsm_vector_search': { diff --git a/server/src/db/schema.ts b/server/src/db/schema.ts index e45b5931..caa02568 100644 --- a/server/src/db/schema.ts +++ b/server/src/db/schema.ts @@ -150,4 +150,8 @@ export function initializeSchema(db: Database.Database): void { addColumnIfMissing(db, 'vector_search_configs', 'index_mode', "TEXT NOT NULL DEFAULT 'readme'"); addColumnIfMissing(db, 'vector_search_configs', 'readme_max_chars', 'INTEGER NOT NULL DEFAULT 6000'); addColumnIfMissing(db, 'repositories', 'vector_indexed_at', 'TEXT'); + addColumnIfMissing(db, 'repositories', 'license', 'TEXT'); + // 上一次向量索引时采用的 license 值(SPDX id / null)。用于增量谓词判断 license 是否 + // 变化:当期 license 与此值不一致时需重新索引,保证 license 变更能使向量元数据失效。 + addColumnIfMissing(db, 'repositories', 'vector_indexed_license', 'TEXT'); } diff --git a/server/src/mcp/provider.ts b/server/src/mcp/provider.ts index e8e7924c..3a986378 100644 --- a/server/src/mcp/provider.ts +++ b/server/src/mcp/provider.ts @@ -5,6 +5,7 @@ import { logger } from '../services/logger.js'; import { type McpRepository, type McpSearchFilters, + normalizeLicense, projectRepoForAgent, searchRepositories, } from './repoSearch.js'; @@ -47,6 +48,7 @@ export function transformRepoRow(row: Record): McpRepository { custom_category: (row.custom_category as string | null) ?? null, category_locked: !!row.category_locked, subscribed_to_releases: !!row.subscribed_to_releases, + license: (row.license as string | null) ?? null, }; } @@ -107,6 +109,7 @@ export function searchRepos(filters: McpSearchFilters) { export function getStats() { const repos = loadAllRepositories(); const byLanguage: Record = {}; + const byLicense: Record = {}; const tagCounts: Record = {}; let analyzed = 0; let subscribed = 0; @@ -115,6 +118,8 @@ export function getStats() { for (const r of repos) { const lang = r.language || 'Unknown'; byLanguage[lang] = (byLanguage[lang] || 0) + 1; + const lic = normalizeLicense(r.license); + byLicense[lic] = (byLicense[lic] || 0) + 1; if (r.analyzed_at && !r.analysis_failed) analyzed += 1; if (r.analyzed_at && r.analysis_failed) failed += 1; if (r.subscribed_to_releases) subscribed += 1; @@ -135,6 +140,7 @@ export function getStats() { unanalyzed: repos.length - analyzed - failed, subscribedToReleases: subscribed, byLanguage, + byLicense, topTags, }; } diff --git a/server/src/mcp/repoSearch.ts b/server/src/mcp/repoSearch.ts index ddc6a5ea..cf889ca6 100644 --- a/server/src/mcp/repoSearch.ts +++ b/server/src/mcp/repoSearch.ts @@ -3,6 +3,30 @@ * Kept server-local to avoid coupling the Express package to the Vite app tree. */ +/** + * License 归一化的服务端镜像(与 src/utils/licenseFilter.ts 保持一致)。 + * 服务端 MCP 无法 import src/ 树,故此处保留一份相同实现;改前端那份时请一并同步。 + */ +export const NO_LICENSE_SENTINEL = '__NO_LICENSE__'; +const NOASSERTION_KEYS = new Set(['', 'noassertion', 'other', 'none', 'no-license']); +export function normalizeLicense(v: unknown): string { + if (v == null || v === '') return NO_LICENSE_SENTINEL; + if (typeof v === 'object') { + const l = v as { spdx_id?: unknown; key?: unknown }; + const spdx = typeof l.spdx_id === 'string' ? l.spdx_id.trim() : ''; + const key = typeof l.key === 'string' ? l.key.trim() : ''; + const resolved = spdx || key; + if (!resolved) return NO_LICENSE_SENTINEL; + return NOASSERTION_KEYS.has(resolved.toLowerCase()) ? NO_LICENSE_SENTINEL : resolved; + } + if (typeof v !== 'string') return NO_LICENSE_SENTINEL; + // 直接字符串路径也需 trim:避免 " Other " / " NOASSERTION " 等空白变体逃过哨兵归并 + const normalized = v.trim(); + return !normalized || NOASSERTION_KEYS.has(normalized.toLowerCase()) + ? NO_LICENSE_SENTINEL + : normalized; +} + export interface McpRepository { id: number; name: string; @@ -27,6 +51,7 @@ export interface McpRepository { category_locked?: boolean; subscribed_to_releases?: boolean; owner?: { login: string; avatar_url?: string }; + license?: string | null; } export interface McpSearchFilters { @@ -43,6 +68,8 @@ export interface McpSearchFilters { isCategoryLocked?: boolean; analysisFailed?: boolean; category?: string; + /** SPDX id 过滤;含 {@link NO_LICENSE_SENTINEL} 表示「无/未声明 license」。 */ + licenses?: string[]; limit?: number; offset?: number; } @@ -65,6 +92,7 @@ export function performBasicTextSearch(repos: T[], quer ...(repo.ai_platforms || []), ...(repo.custom_tags || []), repo.custom_category || '', + normalizeLicense(repo.license), ] .join(' ') .toLowerCase(); @@ -108,6 +136,11 @@ export function applyRepoFilters( return filters.platforms!.some((p) => platforms.includes(p)); }); } + if (filters.licenses?.length) { + filtered = filtered.filter((r) => + filters.licenses!.includes(normalizeLicense(r.license)) + ); + } if (filters.isAnalyzed !== undefined && filters.analysisFailed === undefined) { filtered = filtered.filter((r) => filters.isAnalyzed ? !!r.analyzed_at && !r.analysis_failed : !r.analyzed_at @@ -196,5 +229,6 @@ export function projectRepoForAgent( starred_at: repo.starred_at, updated_at: repo.updated_at, pushed_at: repo.pushed_at, + license: repo.license ?? null, }; } diff --git a/server/src/mcp/tools.ts b/server/src/mcp/tools.ts index 7b334c9d..850c4129 100644 --- a/server/src/mcp/tools.ts +++ b/server/src/mcp/tools.ts @@ -54,6 +54,10 @@ export function registerMcpTools(server: McpServer): void { languages: z.array(z.string()).optional(), tags: z.array(z.string()).optional(), platforms: z.array(z.string()).optional(), + licenses: z + .array(z.string()) + .optional() + .describe('SPDX id list (e.g. ["MIT","Apache-2.0"]); use "__NO_LICENSE__" for repos with no license'), category: z.string().optional().describe('custom_category exact match'), minStars: z.number().optional(), maxStars: z.number().optional(), @@ -71,6 +75,7 @@ export function registerMcpTools(server: McpServer): void { languages: args.languages, tags: args.tags, platforms: args.platforms, + licenses: args.licenses, category: args.category, minStars: args.minStars, maxStars: args.maxStars, diff --git a/server/src/routes/repositories.ts b/server/src/routes/repositories.ts index dbf6779f..93eb669e 100644 --- a/server/src/routes/repositories.ts +++ b/server/src/routes/repositories.ts @@ -12,6 +12,24 @@ function parseJsonColumn(value: unknown): unknown[] { } catch { return []; } } +/** + * 把 GitHub 的 license 值统一为 SPDX id 字符串或 null。 + * 接受三种形态:GitHub 原始对象 `{ key, spdx_id, name, url }`、已规范化的字符串、null。 + * 优先取 spdx_id(如 "MIT"),无则回退 key(如 "Other" → "NOASSERTION" 由前端归一化处理)。 + */ +function toLicenseSpdxId(license: unknown): string | null { + if (license == null) return null; + if (typeof license === 'string') return license.trim() || null; + if (typeof license === 'object') { + // 运行时校验:malformed 备份/第三方源可能把 spdx_id/key 写成非字符串 + const l = license as { spdx_id?: unknown; key?: unknown }; + const spdx = typeof l.spdx_id === 'string' ? l.spdx_id.trim() : ''; + const key = typeof l.key === 'string' ? l.key.trim() : ''; + return spdx || key || null; + } + return null; +} + /** Transform a database row into the API response shape, parsing JSON columns. */ function transformRepo(row: Record) { return { @@ -40,6 +58,8 @@ function transformRepo(row: Record) { last_edited: row.last_edited, subscribed_to_releases: !!row.subscribed_to_releases, vector_indexed_at: row.vector_indexed_at ?? undefined, + license: row.license ?? null, + vector_indexed_license: row.vector_indexed_license ?? null, }; } @@ -134,8 +154,8 @@ router.put('/api/repositories', (req, res) => { owner_login, owner_avatar_url, topics, ai_summary, ai_tags, ai_platforms, analyzed_at, analysis_failed, custom_description, custom_tags, custom_category, category_locked, last_edited, - subscribed_to_releases, vector_indexed_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + subscribed_to_releases, vector_indexed_at, license, vector_indexed_license + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(id) DO UPDATE SET name = excluded.name, full_name = excluded.full_name, @@ -161,7 +181,15 @@ router.put('/api/repositories', (req, res) => { category_locked = excluded.category_locked, last_edited = CASE WHEN excluded.last_edited IS NOT NULL AND excluded.last_edited != '' THEN excluded.last_edited ELSE repositories.last_edited END, subscribed_to_releases = excluded.subscribed_to_releases, - vector_indexed_at = excluded.vector_indexed_at + vector_indexed_at = excluded.vector_indexed_at, + -- vector_indexed_license 是「上次向量索引时采用的 license」的照实记录, + -- 仅由索引流程(PATCH)写入;批量 upsert/sync 一律保留已存储值,避免被 + -- 同步流写入的当前 license 覆盖而破坏增量变更检测。 + vector_indexed_license = COALESCE(repositories.vector_indexed_license, excluded.vector_indexed_license), + -- 区分「省略 license 字段」与「显式提供 null/对象」: + -- 旧客户端/旧备份不含 license 字段时(@licenseProvided = 0)保留已存储值; + -- 显式提供时(@licenseProvided = 1)采用归一化后的 excluded.license(含 null 清空)。 + license = CASE WHEN @licenseProvided IS 1 THEN excluded.license ELSE repositories.license END `); const deleteAllReleases = db.prepare('DELETE FROM releases'); @@ -193,6 +221,8 @@ router.put('/api/repositories', (req, res) => { let count = 0; for (const repo of repositories) { const owner = repo.owner as { login?: string; avatar_url?: string } | undefined; + // 仅当 payload 显式提供 license 字段时才覆盖已存储值;省略(旧客户端/旧备份)则保留。 + const licenseProvided = Object.prototype.hasOwnProperty.call(repo, 'license') ? 1 : 0; stmt.run( repo.id, repo.name, repo.full_name, repo.description ?? null, repo.html_url, repo.stargazers_count ?? 0, repo.language ?? null, @@ -208,7 +238,12 @@ router.put('/api/repositories', (req, res) => { JSON.stringify(Array.isArray(repo.custom_tags) ? repo.custom_tags : []), repo.custom_category ?? null, (repo.category_locked === true || repo.category_locked === 1) ? 1 : 0, repo.last_edited ?? null, (repo.subscribed_to_releases === true || repo.subscribed_to_releases === 1) ? 1 : 0, - repo.vector_indexed_at ?? null + repo.vector_indexed_at ?? null, + toLicenseSpdxId(repo.license), + // 备份中 vector_indexed_license 已是规范化的 SPDX id 字符串或 null; + // 非 string 一律清空,避免奇怪类型破坏增量谓词的字符串比较。 + typeof repo.vector_indexed_license === 'string' ? repo.vector_indexed_license || null : null, + { licenseProvided } ); count++; } @@ -245,6 +280,9 @@ router.patch('/api/repositories/:id', (req, res) => { // 规范化:null/undefined/空字符串 → null;仅接受字符串(ISO 时间戳) vector_indexed_at: (v) => (v === null || v === undefined || v === '') ? null : v, + // vector_indexed_license:规范化为 SPDX id 字符串或 null;非字符串一律清空。 + vector_indexed_license: (v) => + (v === null || v === undefined || v === '' || typeof v !== 'string') ? null : v, description: (v) => v, name: (v) => v, }; diff --git a/server/src/routes/sync.ts b/server/src/routes/sync.ts index 265a0fa3..e4f5b408 100644 --- a/server/src/routes/sync.ts +++ b/server/src/routes/sync.ts @@ -5,6 +5,19 @@ import { config } from '../config.js'; const router = Router(); +/** 与前端 NOASSERTION_KEYS 对齐:空白 / NOASSERTION / Other / none 等落 null。 */ +const NO_LICENSE_KEYS = new Set(['', 'noassertion', 'other', 'none', 'no-license']); + +/** + * 把导入的 license 字符串规范为 SPDX id 或 null。 + * trim 后若为空或落入「无 license」集合则返回 null,否则返回 trim 后的原值。 + */ +function canonicalizeLicenseString(value: string): string | null { + const trimmed = value.trim(); + if (!trimmed || NO_LICENSE_KEYS.has(trimmed.toLowerCase())) return null; + return trimmed; +} + function maskApiKey(key: string | null | undefined): string { if (!key || typeof key !== 'string') return ''; if (key.length <= 4) return '****'; @@ -108,14 +121,33 @@ router.post('/api/sync/import', (req, res) => { owner_login, owner_avatar_url, topics, ai_summary, ai_tags, ai_platforms, analyzed_at, analysis_failed, custom_description, custom_tags, custom_category, category_locked, last_edited, - subscribed_to_releases, vector_indexed_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + subscribed_to_releases, vector_indexed_at, license, vector_indexed_license + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `); for (const r of repos) { // 验证必需的字段 if (!r.id || typeof r.id !== 'number') { throw new Error(`Invalid repository data: missing or invalid id`); } + // license:兼容旧备份(无该列→null)、GitHub 对象形态、已规范化的 SPDX 字符串。 + // 候选字符串 trim 后,空白 / NOASSERTION / Other / none 统一落 null(与 + // 前端 normalizeLicense 的「无 license」语义对齐),保证 DB 只存 SPDX-or-null。 + const rawLicense = (r as Record).license; + let licenseValue: string | null = null; + if (typeof rawLicense === 'string') { + licenseValue = canonicalizeLicenseString(rawLicense); + } else if (rawLicense && typeof rawLicense === 'object') { + const obj = rawLicense as { spdx_id?: unknown; key?: unknown }; + const spdx = typeof obj.spdx_id === 'string' ? obj.spdx_id.trim() : ''; + const key = typeof obj.key === 'string' ? obj.key.trim() : ''; + licenseValue = canonicalizeLicenseString(spdx || key); + } + // vector_indexed_license 与 license 同一套 SPDX-or-null 规则,避免指纹与当前 + // license 语义分裂(例如 "NOASSERTION" 原样入库后增量谓词永远判定为变更)。 + const rawVectorLicense = (r as Record).vector_indexed_license; + const vectorIndexedLicense = typeof rawVectorLicense === 'string' + ? canonicalizeLicenseString(rawVectorLicense) + : null; repoStmt.run( r.id, r.name, r.full_name, r.description ?? null, r.html_url, r.stargazers_count ?? 0, r.language ?? null, @@ -131,7 +163,11 @@ router.post('/api/sync/import', (req, res) => { typeof r.custom_tags === 'string' ? r.custom_tags : JSON.stringify(r.custom_tags ?? []), r.custom_category ?? null, (r.category_locked === true || r.category_locked === 1) ? 1 : 0, r.last_edited ?? null, r.subscribed_to_releases ? 1 : 0, - r.vector_indexed_at ?? null + r.vector_indexed_at ?? null, + licenseValue, + // INSERT OR REPLACE 会整行替换,故备份无此列时落 null(影响:增量谓词会 + // 触发一次重索引回填指纹),合预期。 + vectorIndexedLicense ); } counts.repositories = repos.length; diff --git a/src/components/RepositoryCard.tsx b/src/components/RepositoryCard.tsx index 8b04c3ba..74c25f1a 100644 --- a/src/components/RepositoryCard.tsx +++ b/src/components/RepositoryCard.tsx @@ -1,6 +1,6 @@ import React, { useState, useRef, useEffect, useMemo, useCallback } from 'react'; import { createPortal } from 'react-dom'; -import { GripVertical, Star, StarOff, ExternalLink, Calendar, Bell, BellOff, Bot, Sparkles, Monitor, Smartphone, Globe, Terminal, Package, Edit3, BookOpen, Apple, Square, CheckSquare, Loader2, HelpCircle, Search } from 'lucide-react'; +import { GripVertical, Star, StarOff, ExternalLink, Calendar, Bell, BellOff, Bot, Sparkles, Monitor, Smartphone, Globe, Terminal, Package, Edit3, BookOpen, Apple, Square, CheckSquare, Loader2, HelpCircle, Search, Scale } from 'lucide-react'; import { Repository, Category } from '../types'; import { useAppStore } from '../store/useAppStore'; import { EmbeddingClient, VectorSearchService, findSimilarRepositories } from '../services/vectorSearchService'; @@ -12,6 +12,7 @@ import { formatDistanceToNow } from 'date-fns'; import { RepositoryEditModal } from './RepositoryEditModal'; import { ReadmeModal } from './ReadmeModal'; import { FloatingTooltip } from './FloatingTooltip'; +import { NO_LICENSE_SENTINEL, normalizeLicense } from '../utils/licenseFilter'; import { shallow } from 'zustand/shallow'; import { useDialog } from '../hooks/useDialog'; import { logger } from '../services/logger'; @@ -1122,20 +1123,31 @@ const RepositoryCardComponent: React.FC = ({ {/* Stats */}
{/* Language and Stars */} -
+
{repository.language && ( -
+
{repository.language}
)} -
- - {formatNumber(repository.stargazers_count)} +
+ + {formatNumber(repository.stargazers_count)}
+ {(() => { + // license:归一化后展示 SPDX id;无 license 不渲染 + const lic = normalizeLicense(repository.license); + if (lic === NO_LICENSE_SENTINEL) return null; + return ( +
+ + {lic} +
+ ); + })()}
{/* Update Time / 查找相似仓库 - 悬停时时间淡出,显示高亮按钮 */} @@ -1232,6 +1244,7 @@ export const RepositoryCard = React.memo(RepositoryCardComponent, (prevProps, ne prevProps.repository.category_locked === nextProps.repository.category_locked && prevProps.repository.description === nextProps.repository.description && prevProps.repository.topics === nextProps.repository.topics && + prevProps.repository.license === nextProps.repository.license && prevProps.repository.stargazers_count === nextProps.repository.stargazers_count && prevProps.repository.pushed_at === nextProps.repository.pushed_at && prevProps.repository.updated_at === nextProps.repository.updated_at && diff --git a/src/components/SearchBar.test.tsx b/src/components/SearchBar.test.tsx index 4bbef5cf..fde9054d 100644 --- a/src/components/SearchBar.test.tsx +++ b/src/components/SearchBar.test.tsx @@ -48,6 +48,7 @@ const defaultSearchFilters: SearchFilters = { tags: [], languages: [], platforms: [], + licenses: [], sortBy: 'stars', sortOrder: 'desc', }; diff --git a/src/components/SearchBar.tsx b/src/components/SearchBar.tsx index a0b4a417..c8f8a38a 100644 --- a/src/components/SearchBar.tsx +++ b/src/components/SearchBar.tsx @@ -9,6 +9,7 @@ import { useSearchShortcuts } from '../hooks/useSearchShortcuts'; import { useDialog } from '../hooks/useDialog'; import { isRepoCustomized } from '../utils/repoUtils'; import { applyRepoFilters, performBasicTextSearch as basicTextSearch, sortRepositories } from '../utils/repoSearch'; +import { NO_LICENSE_SENTINEL, normalizeLicense } from '../utils/licenseFilter'; import { NumberInput } from './ui/NumberInput'; type SortBy = 'stars' | 'updated' | 'name' | 'starred'; @@ -107,6 +108,7 @@ export const SearchBar: React.FC = () => { const [availableLanguages, setAvailableLanguages] = useState([]); const [availableTags, setAvailableTags] = useState([]); const [availablePlatforms, setAvailablePlatforms] = useState([]); + const [availableLicenses, setAvailableLicenses] = useState([]); const [isRealTimeSearch, setIsRealTimeSearch] = useState(false); const [isComposing, setIsComposing] = useState(false); @@ -186,10 +188,17 @@ export const SearchBar: React.FC = () => { ...repositories.flatMap(r => r.custom_tags || []) ])]; const platforms = [...new Set(repositories.flatMap(r => r.ai_platforms || []))] as string[]; + // 开源许可:归一化为 SPDX id 或 NO_LICENSE_SENTINEL,排序并把「无」项放最后 + const licenses = [...new Set(repositories.map(r => normalizeLicense(r.license)))].sort((a, b) => { + if (a === NO_LICENSE_SENTINEL) return 1; + if (b === NO_LICENSE_SENTINEL) return -1; + return a.localeCompare(b); + }); setAvailableLanguages(languages); setAvailableTags(tags); setAvailablePlatforms(platforms); + setAvailableLicenses(licenses); // Generate search suggestions from available data const suggestions = [ @@ -249,7 +258,7 @@ export const SearchBar: React.FC = () => { // Search helpers are intentionally kept as local closures; the explicit deps below // cover the state they read without causing a search loop on every render. // eslint-disable-next-line react-hooks/exhaustive-deps - }, [searchFilters.languages, searchFilters.tags, searchFilters.platforms, searchFilters.isAnalyzed, searchFilters.isSubscribed, searchFilters.isEdited, searchFilters.isCategoryLocked, searchFilters.analysisFailed, searchFilters.minStars, searchFilters.maxStars, searchFilters.sortBy, searchFilters.sortOrder, searchFilters.query, repositories, releaseSubscriptions, allCategories]); + }, [searchFilters.languages, searchFilters.tags, searchFilters.platforms, searchFilters.licenses, searchFilters.isAnalyzed, searchFilters.isSubscribed, searchFilters.isEdited, searchFilters.isCategoryLocked, searchFilters.analysisFailed, searchFilters.minStars, searchFilters.maxStars, searchFilters.sortBy, searchFilters.sortOrder, searchFilters.query, repositories, releaseSubscriptions, allCategories]); // Real-time search effect for repository name matching useEffect(() => { @@ -640,6 +649,14 @@ export const SearchBar: React.FC = () => { setSearchFilters({ platforms: newPlatforms }); }; + const handleLicenseToggle = (license: string) => { + const current = searchFilters.licenses ?? []; + const newLicenses = current.includes(license) + ? current.filter(l => l !== license) + : [...current, license]; + setSearchFilters({ licenses: newLicenses }); + }; + const clearFilters = () => { setSearchQuery(''); setIsRealTimeSearch(false); @@ -648,6 +665,7 @@ export const SearchBar: React.FC = () => { tags: [], languages: [], platforms: [], + licenses: [], sortBy: 'stars', sortOrder: 'desc', minStars: undefined, @@ -664,6 +682,7 @@ export const SearchBar: React.FC = () => { searchFilters.languages.length + searchFilters.tags.length + searchFilters.platforms.length + + (searchFilters.licenses?.length ?? 0) + (searchFilters.minStars !== undefined ? 1 : 0) + (searchFilters.maxStars !== undefined ? 1 : 0) + (searchFilters.isAnalyzed !== undefined ? 1 : 0) + @@ -748,6 +767,8 @@ export const SearchBar: React.FC = () => { starred_at: newRepo.starred_at, owner: newRepo.owner, topics: newRepo.topics, + // 回填历史仓库缺失的 license 字段(GitHub 源元数据,跟随 newRepo) + license: newRepo.license ?? null, }; } return newRepo; @@ -1236,6 +1257,32 @@ export const SearchBar: React.FC = () => {
)} + {/* Licenses */} + {availableLicenses.length > 0 && ( +
+

+ {t('开源许可', 'License')} +

+
+ {availableLicenses.map(license => ( + + ))} +
+
+ )} + {/* Tags */} {availableTags.length > 0 && (
diff --git a/src/components/settings/DataManagementPanel.tsx b/src/components/settings/DataManagementPanel.tsx index 8b922d56..e3fdb9a2 100644 --- a/src/components/settings/DataManagementPanel.tsx +++ b/src/components/settings/DataManagementPanel.tsx @@ -442,6 +442,7 @@ export const DataManagementPanel: React.FC = ({ t }) = tags: [], languages: [], platforms: [], + licenses: [], sortBy: 'stars', sortOrder: 'desc', isAnalyzed: undefined, @@ -669,7 +670,14 @@ export const DataManagementPanel: React.FC = ({ t }) = useAppStore.setState({ readReleases: new Set(importedData.readReleases || []) }); } if (selectedTypes.includes('searchFilters') && importedData.searchFilters) { - useAppStore.setState({ searchFilters: importedData.searchFilters }); + // 旧备份可能缺少新增的 licenses 字段,导入时默认 [] 保持 SearchFilters 契约 + const importedFilters = importedData.searchFilters; + useAppStore.setState({ + searchFilters: { + ...importedFilters, + licenses: importedFilters.licenses ?? [], + }, + }); } if (selectedTypes.includes('uiSettings')) { if (importedData.theme === 'light' || importedData.theme === 'dark') { @@ -968,6 +976,7 @@ export const DataManagementPanel: React.FC = ({ t }) = tags: [], languages: [], platforms: [], + licenses: [], sortBy: 'stars', sortOrder: 'desc', isAnalyzed: undefined, diff --git a/src/components/settings/VectorSearchSettings.tsx b/src/components/settings/VectorSearchSettings.tsx index 3857d22d..12e4817f 100644 --- a/src/components/settings/VectorSearchSettings.tsx +++ b/src/components/settings/VectorSearchSettings.tsx @@ -21,10 +21,12 @@ import { EmbeddingClient, VectorSearchService, indexAllRepos, + needsReindex, EMBEDDING_FORMAT_VERSION, } from '../../services/vectorSearchService'; import { GitHubApiService } from '../../services/githubApi'; import type { EmbeddingApiType, EmbeddingConfig } from '../../types'; +import { normalizeLicense } from '../../utils/licenseFilter'; interface VectorSearchSettingsProps { t: (zh: string, en: string) => string; @@ -224,15 +226,11 @@ export const VectorSearchSettings: React.FC = ({ t }) } }, [formWorkerUrl, formAuthToken, setVectorSearchStatus]); - // 未索引数量(已分析、未失败、未向量索引或内容已更新) + // 未索引数量(已分析、未失败、未向量索引或内容已更新)。仅计算内容/license 维度, + // 不含格式版本升级——后者由 incrementalTargetCount 在 formatVersionNeedsReindex 时另行叠加。 const unindexedCount = repositories.filter((r) => { if (!r.analyzed_at || r.analysis_failed) return false; - if (!r.vector_indexed_at) return true; - const contentTime = [r.last_edited, r.analyzed_at] - .filter((t): t is string => !!t) - .sort() - .pop() || ''; - return contentTime > r.vector_indexed_at; + return needsReindex(r, false); }).length; const indexableCount = repositories.filter((r) => r.analyzed_at && !r.analysis_failed).length; @@ -279,15 +277,26 @@ export const VectorSearchSettings: React.FC = ({ t }) try { // 每次点击时读取最新的 repositories,避免闭包捕获过期数据 const currentRepos = useAppStore.getState().repositories; + const now = new Date().toISOString(); + // 按 id 取 license,用于在 stamp vector_indexed_at 的同时记录本次采用的 + // 归一化 license(向量增量谓词据此判断 license 变更触发重索引)。 + const licenseById = new Map(currentRepos.map(r => [r.id, r.license ?? null])); + const stampRepo = (id: number) => ({ + id, + patch: { vector_indexed_at: now, vector_indexed_license: normalizeLicense(licenseById.get(id) ?? null) }, + }); // 1. 清除所有 vector_indexed_at(包括之前失败/不可索引的 repo 的残留值) // 用 updateRepositoriesMetadata 避免重置当前过滤的 searchResults + // 同步清除 vector_indexed_license,使 license 指纹与 stamp 同进退。 updateRepositoriesMetadata( - currentRepos.filter(r => r.vector_indexed_at).map(r => ({ id: r.id, patch: { vector_indexed_at: undefined } })) + currentRepos.filter(r => r.vector_indexed_at).map(r => ({ + id: r.id, + patch: { vector_indexed_at: undefined, vector_indexed_license: undefined }, + })) ); // 2. 全量索引,逐批确认后立即 stamp(中断不丢失已确认进度) - const now = new Date().toISOString(); const stampedRepoIds: number[] = []; const result = await indexAllRepos(currentRepos, clients.embeddingClient, clients.vectorService, { onProgress: (progress) => setVectorIndexingState({ @@ -305,14 +314,14 @@ export const VectorSearchSettings: React.FC = ({ t }) // 批量 stamp:每 32 个(一个 batch)刷新一次,减少 UI 刷新频率 if (stampedRepoIds.length % 32 === 0) { const batch = stampedRepoIds.splice(0, stampedRepoIds.length); - updateRepositoriesMetadata(batch.map(id => ({ id, patch: { vector_indexed_at: now } }))); + updateRepositoriesMetadata(batch.map(stampRepo)); } }, }); // stamp 剩余未刷新的 if (stampedRepoIds.length > 0) { - updateRepositoriesMetadata(stampedRepoIds.map(id => ({ id, patch: { vector_indexed_at: now } }))); + updateRepositoriesMetadata(stampedRepoIds.map(stampRepo)); } // 3. cleanup:全量重建后只保留本次成功重建的向量 @@ -368,6 +377,13 @@ export const VectorSearchSettings: React.FC = ({ t }) ); const now = new Date().toISOString(); + // 与全量重建一致:stamp 时同步记录本次索引采用的归一化 license, + // 供增量谓词下次判断 license 是否变化。 + const licenseById = new Map(currentRepos.map(r => [r.id, r.license ?? null])); + const stampRepo = (id: number) => ({ + id, + patch: { vector_indexed_at: now, vector_indexed_license: normalizeLicense(licenseById.get(id) ?? null) }, + }); const stampedRepoIds: number[] = []; const result = await indexAllRepos(currentRepos, clients.embeddingClient, clients.vectorService, { onProgress: (progress) => setVectorIndexingState({ @@ -386,14 +402,14 @@ export const VectorSearchSettings: React.FC = ({ t }) stampedRepoIds.push(repoId); if (stampedRepoIds.length % 32 === 0) { const batch = stampedRepoIds.splice(0, stampedRepoIds.length); - updateRepositoriesMetadata(batch.map(id => ({ id, patch: { vector_indexed_at: now } }))); + updateRepositoriesMetadata(batch.map(stampRepo)); } }, }); // stamp 剩余未刷新的 if (stampedRepoIds.length > 0) { - updateRepositoriesMetadata(stampedRepoIds.map(id => ({ id, patch: { vector_indexed_at: now } }))); + updateRepositoriesMetadata(stampedRepoIds.map(stampRepo)); } setVectorIndexingState({ result, isIndexing: false, phase: null }); @@ -420,14 +436,11 @@ export const VectorSearchSettings: React.FC = ({ t }) } else { const msg = err instanceof Error ? err.message : String(err); const currentRepos = useAppStore.getState().repositories; + // 与 indexAllRepos 增量谓词保持一致(含格式版本升级判定),避免计数漂移。 + const formatVersionChanged = currentEmbeddingFormatVersion < EMBEDDING_FORMAT_VERSION; const attemptedCount = currentRepos.filter((r) => { if (!r.analyzed_at || r.analysis_failed) return false; - if (!r.vector_indexed_at) return true; - const contentTime = [r.last_edited, r.analyzed_at] - .filter((t): t is string => !!t) - .sort() - .pop() || ''; - return contentTime > r.vector_indexed_at; + return needsReindex(r, formatVersionChanged); }).length; const skippedCount = currentRepos.length - attemptedCount; setVectorIndexingState({ diff --git a/src/services/aiService.test.ts b/src/services/aiService.test.ts new file mode 100644 index 00000000..0a8ca2d5 --- /dev/null +++ b/src/services/aiService.test.ts @@ -0,0 +1,86 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import type { Repository } from '../types'; +import { AIService } from './aiService'; + +// Minimal AIConfig that lets AIService construct without a real token. +const makeConfig = () => ({ + id: 'test', + name: 'test', + apiType: 'openai' as const, + baseUrl: 'http://localhost:0', + apiKey: '', + model: 'gpt-test', + isActive: true, +}); + +function makeRepo(partial: Partial & Pick): Repository { + return { + description: null, + html_url: `https://github.com/${partial.full_name}`, + stargazers_count: 0, + forks_count: 0, + forks: 0, + language: 'TypeScript', + created_at: '2024-01-01T00:00:00Z', + updated_at: '2024-06-01T00:00:00Z', + pushed_at: '2024-06-01T00:00:00Z', + owner: { login: 'owner', avatar_url: '' }, + topics: [], + ...partial, + }; +} + +describe('AIService.searchRepositoriesWithReranking — enhanced basic search fallback', () => { + beforeEach(() => { + // Force the AI request path to fail so we fall back to performEnhancedBasicSearch. + (window.fetch as ReturnType).mockImplementation(() => { + throw new Error('network disabled in test'); + }); + }); + + it('ranks a license-matching repo above a higher-star non-matching repo when the query mixes license + other terms', async () => { + // A matches both the name term ("react") and the license term ("mit"). + const repoA = makeRepo({ + id: 1, + name: 'react-app', + full_name: 'acme/react-app', + stargazers_count: 500, + license: 'MIT', + }); + // B has far more stars and matches the name term, but NOT the license term. + const repoB = makeRepo({ + id: 2, + name: 'react-lib', + full_name: 'acme/react-lib', + stargazers_count: 1000, + license: 'Apache-2.0', + }); + + const service = new AIService(makeConfig() as never, 'en'); + const results = await service.searchRepositoriesWithReranking([repoA, repoB], 'react mit'); + + const ids = results.map((r) => r.id); + // With the license weight, A's license match outweighs B's popularity edge. + expect(ids).toEqual([1, 2]); + expect(results).toHaveLength(2); + }); + + it('does not crash when a repo carries a raw GitHub license object (toLowerCase defensive)', async () => { + // Regression for "e.toLowerCase is not a function": a repo whose license never passed + // through toLicenseSpdxId (legacy persisted store / third-party import) keeps a raw + // GitHub object `{ key, spdx_id, ... }`. performEnhancedBasicSearch must reduce it via + // normalizeLicense rather than (repo.license || '').toLowerCase(). + const repoA = makeRepo({ + id: 3, + name: 'react-legacy', + full_name: 'acme/react-legacy', + stargazers_count: 10, + license: { spdx_id: 'MIT', key: 'MIT', name: 'MIT License', url: 'https://api.github.com/licenses/mit' } as never, + }); + + const service = new AIService(makeConfig() as never, 'en'); + // Should resolve the license object to 'MIT' and rank the repo — not throw. + const results = await service.searchRepositoriesWithReranking([repoA], 'mit'); + expect(results.map((r) => r.id)).toEqual([3]); + }); +}); diff --git a/src/services/aiService.ts b/src/services/aiService.ts index 45f1df29..0b68e98f 100644 --- a/src/services/aiService.ts +++ b/src/services/aiService.ts @@ -1,6 +1,7 @@ import { Repository, Gist, AIConfig, AIApiType } from '../types'; import { backend } from './backendAdapter'; import { buildApiUrl, buildFinalApiUrl } from '../utils/apiUrlBuilder'; +import { NO_LICENSE_SENTINEL, normalizeLicense } from '../utils/licenseFilter'; import { logger } from './logger'; interface OpenAIResponseContentPart { @@ -738,6 +739,11 @@ AI Summary: ${gist.ai_summary || 'None'}`; const parts = [`${index + 1}. ID: ${repo.id} | ${repo.full_name}`]; if (desc) parts.push(` ${desc}`); const meta = [repo.language, `★${stars}`]; + // 与 embedding 一致:归一化后、非哨兵才写入,避免 raw 对象变成 "[object Object]" + { + const lic = normalizeLicense(repo.license); + if (lic !== NO_LICENSE_SENTINEL) meta.push(`License: ${lic}`); + } if (tags) meta.push(`Tags: ${tags}`); parts.push(` ${meta.join(' | ')}`); return parts.join('\n'); @@ -1315,7 +1321,8 @@ ${repoInfo} aiTags: (repo.ai_tags || []).join(' ').toLowerCase(), aiPlatforms: (repo.ai_platforms || []).join(' ').toLowerCase(), customDescription: (repo.custom_description || '').toLowerCase(), - customTags: (repo.custom_tags || []).join(' ').toLowerCase() + customTags: (repo.custom_tags || []).join(' ').toLowerCase(), + license: normalizeLicense(repo.license).toLowerCase(), }; // Check if any query word matches any field @@ -1348,6 +1355,9 @@ ${repoInfo} // Platform and language matches if (searchableFields.aiPlatforms.includes(word)) score += 0.18; if (searchableFields.language.includes(word)) score += 0.12; + + // License matches + if (searchableFields.license.includes(word)) score += 0.2; }); // Boost for exact matches @@ -1434,8 +1444,9 @@ Reply in JSON format: repo.ai_summary || '', ...(repo.ai_tags || []), ...(repo.ai_platforms || []), + normalizeLicense(repo.license), ].join(' ').toLowerCase(); - + // Check if any of the AI-enhanced terms match return allSearchTerms.some(term => { const normalizedTerm = term.toLowerCase(); @@ -1459,8 +1470,9 @@ Reply in JSON format: repo.ai_summary || '', ...(repo.ai_tags || []), ...(repo.ai_platforms || []), + normalizeLicense(repo.license), ].join(' ').toLowerCase(); - + // Split query into words and check if all words are present const queryWords = normalizedQuery.split(/\s+/); return queryWords.every(word => searchableText.includes(word)); @@ -1483,8 +1495,9 @@ Reply in JSON format: repo.ai_summary || '', ...(repo.ai_tags || []), ...(repo.ai_platforms || []), + normalizeLicense(repo.license), ].join(' ').toLowerCase(); - + // Split query into words and check if all words are present const queryWords = normalizedQuery.split(/\s+/); return queryWords.every(word => searchableText.includes(word)); diff --git a/src/services/githubApi.ts b/src/services/githubApi.ts index 02c33bd6..58978251 100644 --- a/src/services/githubApi.ts +++ b/src/services/githubApi.ts @@ -40,6 +40,24 @@ interface GitHubStarredItem { [key: string]: unknown; } +/** + * 把 GitHub 返回的 license 值统一为 SPDX id 字符串或 null。 + * 接受 GitHub 原始对象 `{ key, spdx_id, name, url }`、已规范化的字符串、null。 + * 优先取 spdx_id(如 "MIT"),无则回退 key(如 "Other")。 + */ +function toLicenseSpdxId(license: unknown): string | null { + if (license == null) return null; + if (typeof license === 'string') return license.trim() || null; + if (typeof license === 'object') { + // 运行时校验:malformed 备份/第三方源可能把 spdx_id/key 写成非字符串 + const l = license as { spdx_id?: unknown; key?: unknown }; + const spdx = typeof l.spdx_id === 'string' ? l.spdx_id.trim() : ''; + const key = typeof l.key === 'string' ? l.key.trim() : ''; + return spdx || key || null; + } + return null; +} + interface GitHubRateLimitResponse { rate: { remaining: number; @@ -301,10 +319,15 @@ export class GitHubApiService { if (item.starred_at && item.repo) { return { ...item.repo, + // 归一化 GitHub 的 license 值(对象/字符串/null)为 SPDX id 字符串或 null + license: toLicenseSpdxId(item.repo.license), starred_at: item.starred_at }; } - return item; + return { + ...item, + license: toLicenseSpdxId(item.license), + }; }) as T; } @@ -581,7 +604,10 @@ export class GitHubApiService { const endpoint = username ? `/users/${encodeURIComponent(username)}/subscriptions?page=${page}&per_page=${perPage}` : `/user/subscriptions?page=${page}&per_page=${perPage}`; - return this.makeRequest(endpoint); + // GitHub 对 watched repos 同样返回原始 license 对象,这里统一归一化为 SPDX id / null, + // 与 /user/starred 路径保持一致,避免下游 normalizeLicense 拿到对象时崩溃。 + const repos = await this.makeRequest(endpoint); + return repos.map((repo) => ({ ...repo, license: toLicenseSpdxId(repo.license) })); } async getAllWatchedRepositories(username?: string): Promise { diff --git a/src/services/vectorSearchService.test.ts b/src/services/vectorSearchService.test.ts index d0ff9f13..f717bf4d 100644 --- a/src/services/vectorSearchService.test.ts +++ b/src/services/vectorSearchService.test.ts @@ -5,6 +5,7 @@ import { truncateForRetry, embedWithFallback, indexAllRepos, + needsReindex, EMBEDDING_FORMAT_VERSION, } from './vectorSearchService'; @@ -51,6 +52,49 @@ const makeVectorService = () => ({ const LENGTH_ERROR = () => new Error('Embedding API error 400: {"code":20015,"message":"The parameter is invalid.","data":null}'); +describe('needsReindex', () => { + // 共享:未索引 / 内容更新 / license 变化 / 格式升级——权威谓词,UI 与 indexAllRepos 共用。 + const indexed = (overrides: Partial = {}): Repository => makeRepository(1, { + vector_indexed_at: '2026-01-10T00:00:00.000Z', + vector_indexed_license: 'MIT', + license: 'MIT', + analyzed_at: '2026-01-04T00:00:00.000Z', + ...overrides, + }); + + it('needs reindex when a repo was never indexed', () => { + expect(needsReindex({ ...indexed(), vector_indexed_at: undefined }, false)).toBe(true); + }); + + it('needs reindex when content time is newer than vector_indexed_at', () => { + expect(needsReindex(indexed({ last_edited: '2026-01-12T00:00:00.000Z' }), false)).toBe(true); + }); + + it('needs reindex when license changed (normalized drift)', () => { + expect(needsReindex(indexed({ license: 'Apache-2.0' }), false)).toBe(true); + }); + + it('needs reindex when format version changed regardless of content', () => { + expect(needsReindex(indexed(), true)).toBe(true); + }); + + it('does NOT need reindex when content and license are unchanged', () => { + expect(needsReindex(indexed(), false)).toBe(false); + }); + + it('treats null vs NOASSERTION sentinel as equal (no reindex)', () => { + // Both normalize to NO_LICENSE_SENTINEL, so null fingerprint ↔ no-assertion license is a match. + expect(needsReindex( + indexed({ vector_indexed_license: null, license: null }), + false, + )).toBe(false); + expect(needsReindex( + indexed({ vector_indexed_license: null, license: 'NOASSERTION' }), + false, + )).toBe(false); + }); +}); + describe('looksLikeLengthError', () => { it('detects SiliconFlow 20015 length errors', () => { expect(looksLikeLengthError(LENGTH_ERROR())).toBe(true); @@ -138,6 +182,94 @@ describe('indexAllRepos incremental filtering', () => { }); }); +describe('indexAllRepos incremental filtering — license change invalidation', () => { + it('reindexes an indexed repo whose license changed even when content timestamps are unchanged', async () => { + // Repo was indexed under MIT; its vector_indexed_at is newer than content, so without + // the license fingerprint check it would be skipped. GitHub sync then changes the + // license to Apache-2.0 — the incremental predicate must reindex to refresh metadata. + const repos = [ + makeRepository(1, { + vector_indexed_at: '2026-01-10T00:00:00.000Z', + vector_indexed_license: 'MIT', + license: 'Apache-2.0', + analyzed_at: '2026-01-04T00:00:00.000Z', + // No last_edited / newer analyzed_at => content time predates vector_indexed_at. + }), + ]; + + const client = makeIndexClient(); + const vectorService = makeVectorService(); + const indexedRepoIds: number[] = []; + const result = await indexAllRepos(repos, client, vectorService, { + incremental: true, + formatVersion: EMBEDDING_FORMAT_VERSION, + currentFormatVersion: EMBEDDING_FORMAT_VERSION, + indexMode: 'description', + onRepoIndexed: (repoId) => indexedRepoIds.push(repoId), + }); + + // License differs from the indexed fingerprint => reindex, despite no content-time change. + expect(indexedRepoIds).toEqual([1]); + expect(result.indexedRepoIds).toEqual([1]); + }); + + it('treats a null-vs-no-assertion drift as a license change and reindexes', async () => { + // normalizeLicense collapses both null and 'NOASSERTION' to the sentinel, so an indexed + // repo whose stored license was null (fingerprint not yet captured) must reindex once to + // capture the fingerprint, then settle on subsequent runs. + const repos = [ + makeRepository(1, { + vector_indexed_at: '2026-01-10T00:00:00.000Z', + vector_indexed_license: null, + license: 'MIT', + analyzed_at: '2026-01-04T00:00:00.000Z', + }), + ]; + + const client = makeIndexClient(); + const vectorService = makeVectorService(); + const indexedRepoIds: number[] = []; + const result = await indexAllRepos(repos, client, vectorService, { + incremental: true, + formatVersion: EMBEDDING_FORMAT_VERSION, + currentFormatVersion: EMBEDDING_FORMAT_VERSION, + indexMode: 'description', + onRepoIndexed: (repoId) => indexedRepoIds.push(repoId), + }); + + // null fingerprint vs MIT license => mismatch => reindex. + expect(indexedRepoIds).toEqual([1]); + expect(result.indexedRepoIds).toEqual([1]); + }); + + it('skips an indexed repo whose license is unchanged', async () => { + const repos = [ + makeRepository(1, { + vector_indexed_at: '2026-01-10T00:00:00.000Z', + vector_indexed_license: 'MIT', + license: 'MIT', + analyzed_at: '2026-01-04T00:00:00.000Z', + }), + ]; + + const client = makeIndexClient(); + const vectorService = makeVectorService(); + const indexedRepoIds: number[] = []; + const result = await indexAllRepos(repos, client, vectorService, { + incremental: true, + formatVersion: EMBEDDING_FORMAT_VERSION, + currentFormatVersion: EMBEDDING_FORMAT_VERSION, + indexMode: 'description', + onRepoIndexed: (repoId) => indexedRepoIds.push(repoId), + }); + + // No content change and license fingerprint matches => nothing to index. + expect(indexedRepoIds).toEqual([]); + expect(result.indexedRepoIds).toEqual([]); + expect(result.indexed).toBe(0); + }); +}); + describe('embedWithFallback', () => { it('uses fast batch path on success', async () => { const client = makeClient(async (texts) => texts.map((_, i) => vec(i))); diff --git a/src/services/vectorSearchService.ts b/src/services/vectorSearchService.ts index 8ff6dafa..37786d57 100644 --- a/src/services/vectorSearchService.ts +++ b/src/services/vectorSearchService.ts @@ -6,6 +6,7 @@ */ import type { EmbeddingConfig, VectorSearchConfig, Repository } from '../types'; +import { NO_LICENSE_SENTINEL, normalizeLicense } from '../utils/licenseFilter'; // ============================================================ // EmbeddingClient @@ -187,6 +188,9 @@ export interface VectorizeVector { language: string; stars: number; tags: string[]; + // 可选:此 PR 之前(embedding v2)索引的向量不含 license 键,重索引前以 undefined 出现。 + // indexAllRepos 写入时会填入字符串;消费方请按可选处理。 + license?: string; }; } @@ -199,6 +203,8 @@ export interface VectorQueryResult { language: string; stars: number; tags: string[]; + // 同上:旧向量查询返回时缺该键,按可选处理。 + license?: string; }; } @@ -323,7 +329,7 @@ export class VectorSearchService { * buildEmbeddingText 的输出格式变化时必须递增, * 使增量索引能检测到格式变化并强制重新索引所有向量。 */ -export const EMBEDDING_FORMAT_VERSION = 2; +export const EMBEDDING_FORMAT_VERSION = 3; /** * 拼接仓库文本用于 embedding @@ -357,6 +363,12 @@ export function buildEmbeddingText(repo: Repository, readmeContent?: string, max if (allTopics.length > 0) parts.push(`Topics: ${allTopics.join(', ')}`); if (repo.language) parts.push(`Language: ${repo.language}`); + // 开源许可:先归一化,避免 raw GitHub 对象变成 "[object Object]" 污染 embedding; + // 哨兵(无/未声明)不写入,与 null/缺失保持同一语义。 + { + const lic = normalizeLicense(repo.license); + if (lic !== NO_LICENSE_SENTINEL) parts.push(`License: ${lic}`); + } // README 内容提供最丰富的语义信息 if (readmeContent) { @@ -372,6 +384,32 @@ export function buildEmbeddingText(repo: Repository, readmeContent?: string, max return parts.join('\n'); } +/** + * 判断单个仓库是否需要重新向量索引(增量谓词的权威实现)。 + * + * 以下任一成立即视为需要重索引: + * - 从未索引(`vector_indexed_at` 缺失); + * - embedding 格式版本升级(由调用方据 `formatVersionChanged` 传入); + * - 内容时间(`last_edited` 与 `analyzed_at` 中较新者)晚于上次索引时间; + * - license 变化(归一化后比对 `vector_indexed_license` 与 `license`)。 + * + * 该模块与 `VectorSearchSettings.tsx` 的 `unindexedCount` / `attemptedCount` 谓词 + * 必须保持一致,故统一抽取为可选导出供复用,避免三处副本分别漂移。 + */ +export function needsReindex(repo: Pick, formatVersionChanged: boolean): boolean { + if (!repo.vector_indexed_at) return true; // 从未索引 + if (formatVersionChanged) return true; // 格式版本升级,需要重新索引 + // 取 last_edited 与 analyzed_at 中较新者作为内容时间,更新后需要重新索引 + const contentTime = [repo.last_edited, repo.analyzed_at] + .filter((t): t is string => !!t) + .sort() + .pop() || ''; + if (contentTime > repo.vector_indexed_at) return true; + // license 变化(归一化后比对):GitHub 同步可能仅更新 license 而 last_edited 不变, + // 此时向量元数据与嵌入文本中的 License 字段会过时,故需重新索引。 + return normalizeLicense(repo.vector_indexed_license ?? null) !== normalizeLicense(repo.license ?? null); +} + /** * 全量/增量重建向量索引 * 遍历已分析仓库,分批生成 embedding 并 upsert 到 Worker @@ -510,16 +548,7 @@ export async function indexAllRepos( // 嵌入文本格式版本变化时,强制重新索引所有向量以避免混合格式 // 缺失版本号视为 v1(旧格式),仍需触发升级 const formatVersionChanged = (options.formatVersion ?? 1) < (options.currentFormatVersion ?? EMBEDDING_FORMAT_VERSION); - indexable = indexable.filter((r) => { - if (!r.vector_indexed_at) return true; // 从未索引 - if (formatVersionChanged) return true; // 格式版本升级,需要重新索引 - // 取 last_edited 与 analyzed_at 中较新者作为内容时间,更新后需要重新索引 - const contentTime = [r.last_edited, r.analyzed_at] - .filter((t): t is string => !!t) - .sort() - .pop() || ''; - return contentTime > r.vector_indexed_at; - }); + indexable = indexable.filter((r) => needsReindex(r, formatVersionChanged)); } let indexed = 0; let errors = 0; @@ -585,6 +614,9 @@ export async function indexAllRepos( language: batch[j].language || '', stars: batch[j].stargazers_count || 0, tags: batch[j].ai_tags || [], + // 历史/导入数据 license 可能是 GitHub 对象或数字;统一经 normalizeLicense + // 降为 SPDX id / 哨兵,避免把 "[object Object]" 写入向量元数据。 + license: normalizeLicense(batch[j].license), }, }); } else { diff --git a/src/store/useAppStore.ts b/src/store/useAppStore.ts index 282e8e6c..dee69ac2 100644 --- a/src/store/useAppStore.ts +++ b/src/store/useAppStore.ts @@ -470,6 +470,7 @@ const initialSearchFilters: SearchFilters = { tags: [], languages: [], platforms: [], + licenses: [], sortBy: 'stars', sortOrder: 'desc', isAnalyzed: undefined, diff --git a/src/types/index.ts b/src/types/index.ts index ca25533d..a6bb959d 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -30,8 +30,12 @@ export interface Repository { category_locked?: boolean; last_edited?: string; vector_indexed_at?: string; // ISO timestamp of last successful vector indexing + /** 上次向量索引时采用的 license(SPDX id / null)。增量谓词据此判断 license 是否变化以触发重索引。 */ + vector_indexed_license?: string | null; last_release_fetch_time?: string; // ISO timestamp, for incremental sync has_fetched_releases?: boolean; // whether this repo has been synced for releases + /** SPDX id(如 'MIT'、'Apache-2.0');无许可证/未识别为 null。AI/搜索/过滤均以此为准。 */ + license?: string | null; } export interface ReleaseAsset { @@ -344,6 +348,8 @@ export interface SearchFilters { isEdited?: boolean; // 新增:是否已编辑 isCategoryLocked?: boolean; // 新增:分类是否已锁定 analysisFailed?: boolean; // 新增:分析是否失败 + /** SPDX id 过滤;过滤面板可采用 `NO_LICENSE_SENTINEL` 表示「无/未声明 license」。 */ + licenses: string[]; // 新增:开源许可过滤 } export interface Category { diff --git a/src/utils/licenseFilter.test.ts b/src/utils/licenseFilter.test.ts new file mode 100644 index 00000000..7842243c --- /dev/null +++ b/src/utils/licenseFilter.test.ts @@ -0,0 +1,107 @@ +import { describe, expect, it } from 'vitest'; +import { buildEmbeddingText } from '../services/vectorSearchService'; +import { NO_LICENSE_SENTINEL, normalizeLicense } from './licenseFilter'; +import type { Repository } from '../types'; + +const baseRepo = (overrides: Partial = {}): Repository => ({ + id: 1, + name: 'repo-1', + full_name: 'owner/repo-1', + description: 'A test repo', + html_url: 'https://github.com/owner/repo-1', + stargazers_count: 10, + forks_count: 1, + forks: 1, + language: 'TypeScript', + created_at: '2026-01-01T00:00:00.000Z', + updated_at: '2026-01-02T00:00:00.000Z', + pushed_at: '2026-01-03T00:00:00.000Z', + owner: { login: 'owner', avatar_url: 'https://github.com/a.png' }, + topics: ['test'], + ...overrides, +}); + +describe('normalizeLicense', () => { + it('passes through SPDX ids unchanged', () => { + expect(normalizeLicense('MIT')).toBe('MIT'); + expect(normalizeLicense('Apache-2.0')).toBe('Apache-2.0'); + expect(normalizeLicense('GPL-3.0')).toBe('GPL-3.0'); + }); + + it('collapses null/undefined/empty to the no-license sentinel', () => { + expect(normalizeLicense(null)).toBe(NO_LICENSE_SENTINEL); + expect(normalizeLicense(undefined)).toBe(NO_LICENSE_SENTINEL); + expect(normalizeLicense('')).toBe(NO_LICENSE_SENTINEL); + }); + + it('collapses GitHub "no assertion" forms to the sentinel', () => { + expect(normalizeLicense('NOASSERTION')).toBe(NO_LICENSE_SENTINEL); + expect(normalizeLicense('Other')).toBe(NO_LICENSE_SENTINEL); + expect(normalizeLicense('NONE')).toBe(NO_LICENSE_SENTINEL); + expect(normalizeLicense('no-license')).toBe(NO_LICENSE_SENTINEL); + }); + + it('collapses lowercase variants (legacy backup / third-party sources)', () => { + expect(normalizeLicense('noassertion')).toBe(NO_LICENSE_SENTINEL); + expect(normalizeLicense('other')).toBe(NO_LICENSE_SENTINEL); + expect(normalizeLicense('none')).toBe(NO_LICENSE_SENTINEL); + }); + + it('trims scalar strings before sentinel matching and SPDX pass-through', () => { + // 直接字符串路径此前未 trim," Other " 会成为独立 facet,拆分过滤/统计。 + expect(normalizeLicense(' Other ')).toBe(NO_LICENSE_SENTINEL); + expect(normalizeLicense(' NOASSERTION ')).toBe(NO_LICENSE_SENTINEL); + expect(normalizeLicense(' none ')).toBe(NO_LICENSE_SENTINEL); + expect(normalizeLicense(' ')).toBe(NO_LICENSE_SENTINEL); + expect(normalizeLicense(' MIT ')).toBe('MIT'); + }); + + it('never throws on a non-string license (GitHub object / number / legacy data)', () => { + // Regression: performBasicTextSearch / performEnhancedBasicSearch / RepositoryCard call + // normalizeLicense(repo.license) during render. A repo that never passed through + // toLicenseSpdxId (legacy persisted store, third-party import, or a return path not + // yet normalized) can carry a raw GitHub license object { key, spdx_id, ... } or a + // number. normalizeLicense must reduce these without calling .toLowerCase() on a + // non-string and crashing the client render ("e.toLowerCase is not a function"). + // GitHub object → resolve SPDX id (spdx_id preferred over key): + expect(normalizeLicense({ spdx_id: 'MIT', key: 'MIT', name: 'MIT License', url: 'u' } as unknown)) + .toBe('MIT'); + // GitHub "Other" object (no SPDX): key 'Other' → sentinel. + expect(normalizeLicense({ key: 'Other', spdx_id: 'NOASSERTION', name: 'Other' } as unknown)) + .toBe(NO_LICENSE_SENTINEL); + // Object missing both string fields → sentinel. + expect(normalizeLicense({ name: 'Custom' } as unknown)).toBe(NO_LICENSE_SENTINEL); + // Non-string truthy values (number/boolean) → sentinel, no crash. + expect(normalizeLicense(123 as unknown)).toBe(NO_LICENSE_SENTINEL); + expect(normalizeLicense(true as unknown)).toBe(NO_LICENSE_SENTINEL); + // String already carrying the sentinel stays. + expect(normalizeLicense(NO_LICENSE_SENTINEL)).toBe(NO_LICENSE_SENTINEL); + }); + + it('falls back from blank/whitespace spdx_id to a valid key', () => { + // Regression: `spdx_id ?? key` retained an empty-string spdx_id and wrongly returned + // the no-license sentinel, discarding a valid key. Trim-first + `||` must select the + // first non-empty field (spdx_id preferred, else key). + expect(normalizeLicense({ spdx_id: '', key: 'MIT' } as unknown)).toBe('MIT'); + expect(normalizeLicense({ spdx_id: ' ', key: 'apache-2.0', name: 'Apache' } as unknown)) + .toBe('apache-2.0'); + // Both blank → sentinel. + expect(normalizeLicense({ spdx_id: '', key: ' ' } as unknown)).toBe(NO_LICENSE_SENTINEL); + // key 'Other' still collapses to sentinel once spdx_id is absent/blank. + expect(normalizeLicense({ spdx_id: '', key: 'Other' } as unknown)).toBe(NO_LICENSE_SENTINEL); + }); +}); + +describe('buildEmbeddingText license', () => { + it('includes a License: line when license is set', () => { + const text = buildEmbeddingText(baseRepo({ license: 'MIT' })); + expect(text).toContain('License: MIT'); + }); + + it('omits the License line when license is null/missing', () => { + const textNull = buildEmbeddingText(baseRepo({ license: null })); + const textMissing = buildEmbeddingText(baseRepo()); + expect(textNull).not.toContain('License:'); + expect(textMissing).not.toContain('License:'); + }); +}); diff --git a/src/utils/licenseFilter.ts b/src/utils/licenseFilter.ts new file mode 100644 index 00000000..fddacf42 --- /dev/null +++ b/src/utils/licenseFilter.ts @@ -0,0 +1,56 @@ +/** + * License 过滤与归一化工具。 + * + * 仓库元数据中 license 存为 SPDX id 字符串(如 "MIT"、"Apache-2.0")或 null。 + * GitHub 对未声明 / 无法识别许可证的仓库返回 `{ key: 'Other', spdx_id: 'NOASSERTION' }`, + * 我们存 spdx_id 即 `'NOASSERTION'`。为让过滤面板提供一个「无/未声明」聚合项, + * 统一把这些情形归一化为 {@link NO_LICENSE_SENTINEL}。 + * + * 本模块同时被前端 UI、过滤求值、MCP 等多处复用;服务端 MCP 因无法 import src/ 树, + * 在 server/src/mcp/repoSearch.ts 内保留一份相同实现,改这里时请一并同步。 + */ + +/** 「无 license」聚合哨兵:用于过滤器把 null / NOASSERTION / Other 等归并为一项。 */ +export const NO_LICENSE_SENTINEL = '__NO_LICENSE__'; + +/** + * 视作「无/未声明 license」的值集合(大小写不敏感比对,覆盖常见 GitHub/SPDX 写法)。 + * - `''` 空串 + * - `'noassertion'` GitHub「无 SPDX 断言」的 spdx_id(NOASSERTION) + * - `'other'` GitHub license.key(Other,无 SPDX 时) + * - `'none'` SPDX「无 license」(NONE) + * - `'no-license'` 兜底串 + */ +const NOASSERTION_KEYS = new Set(['', 'noassertion', 'other', 'none', 'no-license']); + +/** + * 把仓库的 license 值归一化为「SPDX id」或「无 license 哨兵」。 + * 比对大小写不敏感,以收敛历史备份/第三方源写入的小写变体(如 'other'、'none')。 + * + * 防御:`v` 可能并非字符串——历史持久化数据、第三方备份导入,或尚未走 {@link toLicenseSpdxId} + * 的 GitHub 原始对象 `{ key, spdx_id, name, url }` 都可能携带非字符串 license。此处不再假设字符串: + * 对象形态先还原为 `spdx_id ?? key`(再字符串比对),其余非字符串一律按「无 license」归并, + * 避免对对象/数字调用 `.toLowerCase()` 导致客户端渲染崩溃。 + * @param v 原始 license 值(SPDX id / GitHub 对象 / 数字 / null / undefined / 空串) + * @returns 归一化后的字符串;无 license 时返回 {@link NO_LICENSE_SENTINEL} + */ +export function normalizeLicense(v: unknown): string { + if (v == null || v === '') return NO_LICENSE_SENTINEL; + if (typeof v === 'object') { + // GitHub license 对象:优先 spdx_id(如 'MIT'),回退 key(如 'Other')。 + // 注意 trimmed-first + `||`:若 spdx_id 为空白串(≠ null),不可用 `??` 否则会保留空串 + // 并错误归入「无 license」,应回退到非空 key。 + const l = v as { spdx_id?: unknown; key?: unknown }; + const spdx = typeof l.spdx_id === 'string' ? l.spdx_id.trim() : ''; + const key = typeof l.key === 'string' ? l.key.trim() : ''; + const resolved = spdx || key; + if (!resolved) return NO_LICENSE_SENTINEL; + return NOASSERTION_KEYS.has(resolved.toLowerCase()) ? NO_LICENSE_SENTINEL : resolved; + } + if (typeof v !== 'string') return NO_LICENSE_SENTINEL; // 数字等为非合法 license + // 直接字符串路径也需 trim:避免 " Other " / " NOASSERTION " 等空白变体逃过哨兵归并 + const normalized = v.trim(); + return !normalized || NOASSERTION_KEYS.has(normalized.toLowerCase()) + ? NO_LICENSE_SENTINEL + : normalized; +} diff --git a/src/utils/repoSearch.test.ts b/src/utils/repoSearch.test.ts index 841696f3..109543d4 100644 --- a/src/utils/repoSearch.test.ts +++ b/src/utils/repoSearch.test.ts @@ -6,6 +6,7 @@ import { searchRepositories, projectRepoForAgent, } from './repoSearch'; +import { NO_LICENSE_SENTINEL, normalizeLicense } from './licenseFilter'; function makeRepo(partial: Partial & Pick): Repository { return { @@ -36,6 +37,7 @@ const sample: Repository[] = [ ai_tags: ['crdt', 'sync'], ai_platforms: ['cli'], topics: ['database'], + license: 'MIT', }), makeRepo({ id: 2, @@ -46,6 +48,7 @@ const sample: Repository[] = [ stargazers_count: 50, ai_tags: ['webdav'], custom_category: 'tools', + license: 'GPL-3.0', }), makeRepo({ id: 3, @@ -56,6 +59,7 @@ const sample: Repository[] = [ stargazers_count: 200, analyzed_at: '2024-01-01', analysis_failed: true, + license: 'NOASSERTION', }), ]; @@ -90,6 +94,30 @@ describe('applyRepoFilters', () => { const hits = applyRepoFilters(sample, { sortBy: 'stars', sortOrder: 'desc' }); expect(hits.map((r) => r.id)).toEqual([1, 3, 2]); }); + + it('filters by SPDX id license', () => { + const hits = applyRepoFilters(sample, { licenses: ['MIT'] }); + expect(hits.map((r) => r.id)).toEqual([1]); + }); + + it('aggregates no-license repos via the sentinel', () => { + // normalizeLicense collapses every no-license shape to the sentinel, so the + // "no license" UI bucket is a single stable key regardless of how GitHub + // reports the absence: null/'Other' (key-only licenses)/'NOASSERTION'/ + // 'none'/''/and mixed-case 'noassertion' all collapse to NO_LICENSE_SENTINEL. + expect(normalizeLicense(null)).toBe(NO_LICENSE_SENTINEL); + expect(normalizeLicense('NOASSERTION')).toBe(NO_LICENSE_SENTINEL); + expect(normalizeLicense('Other')).toBe(NO_LICENSE_SENTINEL); + expect(normalizeLicense('none')).toBe(NO_LICENSE_SENTINEL); + expect(normalizeLicense('')).toBe(NO_LICENSE_SENTINEL); + expect(normalizeLicense('noassertion')).toBe(NO_LICENSE_SENTINEL); + expect(normalizeLicense('NoAssertion')).toBe(NO_LICENSE_SENTINEL); + + // A repo whose stored license collapses to the sentinel is matched by the + // "no license" filter. (sample[2] carries license: 'NOASSERTION' → sentinel.) + const hits = applyRepoFilters(sample, { licenses: [NO_LICENSE_SENTINEL] }); + expect(hits.map((r) => r.id)).toEqual([3]); + }); }); describe('searchRepositories', () => { diff --git a/src/utils/repoSearch.ts b/src/utils/repoSearch.ts index 64eeafb6..afe99f34 100644 --- a/src/utils/repoSearch.ts +++ b/src/utils/repoSearch.ts @@ -1,5 +1,6 @@ import type { Category, Repository, SearchFilters } from '../types'; import { isRepoCustomized } from './repoUtils'; +import { normalizeLicense } from './licenseFilter'; /** Partial filters used by MCP and UI search (all fields optional except when provided). */ export type RepoSearchFilterInput = Partial & { @@ -27,6 +28,7 @@ export function performBasicTextSearch(repos: T[], query: ...(repo.ai_platforms || []), ...(repo.custom_tags || []), repo.custom_category || '', + normalizeLicense(repo.license), ] .join(' ') .toLowerCase(); @@ -117,6 +119,14 @@ export function applyRepoFilters( }); } + const licenses = searchFilters.licenses ?? []; + if (licenses.length > 0) { + // 归一化后比对:SPDX id 精确匹配,无 license(含 NOASSERTION/Other/null)落入 NO_LICENSE_SENTINEL + filtered = filtered.filter((repo) => + licenses.includes(normalizeLicense(repo.license)) + ); + } + if (searchFilters.isAnalyzed !== undefined && searchFilters.analysisFailed === undefined) { filtered = filtered.filter((repo) => searchFilters.isAnalyzed