Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 34 additions & 0 deletions electron/mcpLocalServer.js
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,34 @@ function performBasicTextSearch(repos, query) {
...(repo.ai_platforms || []),
...(repo.custom_tags || []),
repo.custom_category || '',
normalizeLicense(repo.license),
]
.join(' ')
.toLowerCase();
return words.every((w) => text.includes(w));
});
}

// License 归一化镜像(与 src/utils/licenseFilter.ts 一致)
const NO_LICENSE_SENTINEL = '__NO_LICENSE__';
const NOASSERTION_KEYS = new Set(['', 'noassertion', 'other', 'none', 'no-license']);
function normalizeLicense(v) {
if (v == null || v === '') return NO_LICENSE_SENTINEL;
if (typeof v === 'object') {
const spdx = typeof v.spdx_id === 'string' ? v.spdx_id.trim() : '';
const key = typeof v.key === 'string' ? v.key.trim() : '';
const resolved = spdx || key;
if (!resolved) return NO_LICENSE_SENTINEL;
return NOASSERTION_KEYS.has(resolved.toLowerCase()) ? NO_LICENSE_SENTINEL : resolved;
}
if (typeof v !== 'string') return NO_LICENSE_SENTINEL;
// 直接字符串路径也需 trim:避免 " Other " / " NOASSERTION " 等空白变体逃过哨兵归并
const normalized = v.trim();
return !normalized || NOASSERTION_KEYS.has(normalized.toLowerCase())
? NO_LICENSE_SENTINEL
: normalized;
}

function projectRepo(repo, max = 400) {
const summary = repo.ai_summary || repo.custom_description || repo.description || null;
const truncated =
Expand All @@ -54,6 +75,7 @@ function projectRepo(repo, max = 400) {
starred_at: repo.starred_at,
updated_at: repo.updated_at,
pushed_at: repo.pushed_at,
license: repo.license ?? null,
};
}

Expand Down Expand Up @@ -265,6 +287,11 @@ function getTools(vectorAvailable) {
query: { type: 'string' },
languages: { type: 'array', items: { type: 'string' } },
tags: { type: 'array', items: { type: 'string' } },
licenses: {
type: 'array',
items: { type: 'string' },
description: 'SPDX id list (e.g. ["MIT","Apache-2.0"]); use "__NO_LICENSE__" for repos with no license',
},
Comment thread
coderabbitai[bot] marked this conversation as resolved.
category: { type: 'string' },
minStars: { type: 'number' },
maxStars: { type: 'number' },
Expand Down Expand Up @@ -362,6 +389,9 @@ async function callTool(name, args, snapshot) {
return args.tags.some((t) => tags.includes(t));
});
}
if (args?.licenses?.length) {
list = list.filter((r) => args.licenses.includes(normalizeLicense(r.license)));
}
if (args?.category) {
list = list.filter((r) => r.custom_category === args.category);
}
Expand Down Expand Up @@ -400,11 +430,14 @@ async function callTool(name, args, snapshot) {
}
case 'gsm_stats': {
const byLanguage = {};
const byLicense = {};
let analyzed = 0;
let subscribed = 0;
for (const r of repos) {
const lang = r.language || 'Unknown';
byLanguage[lang] = (byLanguage[lang] || 0) + 1;
const lic = normalizeLicense(r.license);
byLicense[lic] = (byLicense[lic] || 0) + 1;
if (r.analyzed_at && !r.analysis_failed) analyzed += 1;
if (r.subscribed_to_releases) subscribed += 1;
}
Expand All @@ -413,6 +446,7 @@ async function callTool(name, args, snapshot) {
analyzed,
subscribedToReleases: subscribed,
byLanguage,
byLicense,
});
}
case 'gsm_vector_search': {
Expand Down
4 changes: 4 additions & 0 deletions server/src/db/schema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -150,4 +150,8 @@ export function initializeSchema(db: Database.Database): void {
addColumnIfMissing(db, 'vector_search_configs', 'index_mode', "TEXT NOT NULL DEFAULT 'readme'");
addColumnIfMissing(db, 'vector_search_configs', 'readme_max_chars', 'INTEGER NOT NULL DEFAULT 6000');
addColumnIfMissing(db, 'repositories', 'vector_indexed_at', 'TEXT');
addColumnIfMissing(db, 'repositories', 'license', 'TEXT');
// 上一次向量索引时采用的 license 值(SPDX id / null)。用于增量谓词判断 license 是否
// 变化:当期 license 与此值不一致时需重新索引,保证 license 变更能使向量元数据失效。
addColumnIfMissing(db, 'repositories', 'vector_indexed_license', 'TEXT');
}
6 changes: 6 additions & 0 deletions server/src/mcp/provider.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { logger } from '../services/logger.js';
import {
type McpRepository,
type McpSearchFilters,
normalizeLicense,
projectRepoForAgent,
searchRepositories,
} from './repoSearch.js';
Expand Down Expand Up @@ -47,6 +48,7 @@ export function transformRepoRow(row: Record<string, unknown>): McpRepository {
custom_category: (row.custom_category as string | null) ?? null,
category_locked: !!row.category_locked,
subscribed_to_releases: !!row.subscribed_to_releases,
license: (row.license as string | null) ?? null,
};
}

Expand Down Expand Up @@ -107,6 +109,7 @@ export function searchRepos(filters: McpSearchFilters) {
export function getStats() {
const repos = loadAllRepositories();
const byLanguage: Record<string, number> = {};
const byLicense: Record<string, number> = {};
const tagCounts: Record<string, number> = {};
let analyzed = 0;
let subscribed = 0;
Expand All @@ -115,6 +118,8 @@ export function getStats() {
for (const r of repos) {
const lang = r.language || 'Unknown';
byLanguage[lang] = (byLanguage[lang] || 0) + 1;
const lic = normalizeLicense(r.license);
byLicense[lic] = (byLicense[lic] || 0) + 1;
if (r.analyzed_at && !r.analysis_failed) analyzed += 1;
if (r.analyzed_at && r.analysis_failed) failed += 1;
if (r.subscribed_to_releases) subscribed += 1;
Expand All @@ -135,6 +140,7 @@ export function getStats() {
unanalyzed: repos.length - analyzed - failed,
subscribedToReleases: subscribed,
byLanguage,
byLicense,
topTags,
};
}
Expand Down
34 changes: 34 additions & 0 deletions server/src/mcp/repoSearch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,30 @@
* Kept server-local to avoid coupling the Express package to the Vite app tree.
*/

/**
* License 归一化的服务端镜像(与 src/utils/licenseFilter.ts 保持一致)。
* 服务端 MCP 无法 import src/ 树,故此处保留一份相同实现;改前端那份时请一并同步。
*/
export const NO_LICENSE_SENTINEL = '__NO_LICENSE__';
const NOASSERTION_KEYS = new Set(['', 'noassertion', 'other', 'none', 'no-license']);
export function normalizeLicense(v: unknown): string {
if (v == null || v === '') return NO_LICENSE_SENTINEL;
if (typeof v === 'object') {
const l = v as { spdx_id?: unknown; key?: unknown };
const spdx = typeof l.spdx_id === 'string' ? l.spdx_id.trim() : '';
const key = typeof l.key === 'string' ? l.key.trim() : '';
const resolved = spdx || key;
if (!resolved) return NO_LICENSE_SENTINEL;
return NOASSERTION_KEYS.has(resolved.toLowerCase()) ? NO_LICENSE_SENTINEL : resolved;
}
if (typeof v !== 'string') return NO_LICENSE_SENTINEL;
// 直接字符串路径也需 trim:避免 " Other " / " NOASSERTION " 等空白变体逃过哨兵归并
const normalized = v.trim();
return !normalized || NOASSERTION_KEYS.has(normalized.toLowerCase())
? NO_LICENSE_SENTINEL
: normalized;
}

export interface McpRepository {
id: number;
name: string;
Expand All @@ -27,6 +51,7 @@ export interface McpRepository {
category_locked?: boolean;
subscribed_to_releases?: boolean;
owner?: { login: string; avatar_url?: string };
license?: string | null;
}

export interface McpSearchFilters {
Expand All @@ -43,6 +68,8 @@ export interface McpSearchFilters {
isCategoryLocked?: boolean;
analysisFailed?: boolean;
category?: string;
/** SPDX id 过滤;含 {@link NO_LICENSE_SENTINEL} 表示「无/未声明 license」。 */
licenses?: string[];
limit?: number;
offset?: number;
}
Expand All @@ -65,6 +92,7 @@ export function performBasicTextSearch<T extends McpRepository>(repos: T[], quer
...(repo.ai_platforms || []),
...(repo.custom_tags || []),
repo.custom_category || '',
normalizeLicense(repo.license),
]
.join(' ')
.toLowerCase();
Expand Down Expand Up @@ -108,6 +136,11 @@ export function applyRepoFilters<T extends McpRepository>(
return filters.platforms!.some((p) => platforms.includes(p));
});
}
if (filters.licenses?.length) {
filtered = filtered.filter((r) =>
filters.licenses!.includes(normalizeLicense(r.license))
);
}
if (filters.isAnalyzed !== undefined && filters.analysisFailed === undefined) {
filtered = filtered.filter((r) =>
filters.isAnalyzed ? !!r.analyzed_at && !r.analysis_failed : !r.analyzed_at
Expand Down Expand Up @@ -196,5 +229,6 @@ export function projectRepoForAgent(
starred_at: repo.starred_at,
updated_at: repo.updated_at,
pushed_at: repo.pushed_at,
license: repo.license ?? null,
};
}
5 changes: 5 additions & 0 deletions server/src/mcp/tools.ts
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,10 @@ export function registerMcpTools(server: McpServer): void {
languages: z.array(z.string()).optional(),
tags: z.array(z.string()).optional(),
platforms: z.array(z.string()).optional(),
licenses: z
.array(z.string())
.optional()
.describe('SPDX id list (e.g. ["MIT","Apache-2.0"]); use "__NO_LICENSE__" for repos with no license'),
category: z.string().optional().describe('custom_category exact match'),
minStars: z.number().optional(),
maxStars: z.number().optional(),
Expand All @@ -71,6 +75,7 @@ export function registerMcpTools(server: McpServer): void {
languages: args.languages,
tags: args.tags,
platforms: args.platforms,
licenses: args.licenses,
category: args.category,
minStars: args.minStars,
maxStars: args.maxStars,
Expand Down
46 changes: 42 additions & 4 deletions server/src/routes/repositories.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,24 @@ function parseJsonColumn(value: unknown): unknown[] {
} catch { return []; }
}

/**
* 把 GitHub 的 license 值统一为 SPDX id 字符串或 null。
* 接受三种形态:GitHub 原始对象 `{ key, spdx_id, name, url }`、已规范化的字符串、null。
* 优先取 spdx_id(如 "MIT"),无则回退 key(如 "Other" → "NOASSERTION" 由前端归一化处理)。
*/
function toLicenseSpdxId(license: unknown): string | null {
if (license == null) return null;
if (typeof license === 'string') return license.trim() || null;
if (typeof license === 'object') {
// 运行时校验:malformed 备份/第三方源可能把 spdx_id/key 写成非字符串
const l = license as { spdx_id?: unknown; key?: unknown };
const spdx = typeof l.spdx_id === 'string' ? l.spdx_id.trim() : '';
const key = typeof l.key === 'string' ? l.key.trim() : '';
return spdx || key || null;
}
return null;
}

/** Transform a database row into the API response shape, parsing JSON columns. */
function transformRepo(row: Record<string, unknown>) {
return {
Expand Down Expand Up @@ -40,6 +58,8 @@ function transformRepo(row: Record<string, unknown>) {
last_edited: row.last_edited,
subscribed_to_releases: !!row.subscribed_to_releases,
vector_indexed_at: row.vector_indexed_at ?? undefined,
license: row.license ?? null,
vector_indexed_license: row.vector_indexed_license ?? null,
};
}

Expand Down Expand Up @@ -134,8 +154,8 @@ router.put('/api/repositories', (req, res) => {
owner_login, owner_avatar_url, topics,
ai_summary, ai_tags, ai_platforms, analyzed_at, analysis_failed,
custom_description, custom_tags, custom_category, category_locked, last_edited,
subscribed_to_releases, vector_indexed_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
subscribed_to_releases, vector_indexed_at, license, vector_indexed_license
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
name = excluded.name,
full_name = excluded.full_name,
Expand All @@ -161,7 +181,15 @@ router.put('/api/repositories', (req, res) => {
category_locked = excluded.category_locked,
last_edited = CASE WHEN excluded.last_edited IS NOT NULL AND excluded.last_edited != '' THEN excluded.last_edited ELSE repositories.last_edited END,
subscribed_to_releases = excluded.subscribed_to_releases,
vector_indexed_at = excluded.vector_indexed_at
vector_indexed_at = excluded.vector_indexed_at,
-- vector_indexed_license 是「上次向量索引时采用的 license」的照实记录,
-- 仅由索引流程(PATCH)写入;批量 upsert/sync 一律保留已存储值,避免被
-- 同步流写入的当前 license 覆盖而破坏增量变更检测。
vector_indexed_license = COALESCE(repositories.vector_indexed_license, excluded.vector_indexed_license),
-- 区分「省略 license 字段」与「显式提供 null/对象」:
-- 旧客户端/旧备份不含 license 字段时(@licenseProvided = 0)保留已存储值;
-- 显式提供时(@licenseProvided = 1)采用归一化后的 excluded.license(含 null 清空)。
license = CASE WHEN @licenseProvided IS 1 THEN excluded.license ELSE repositories.license END
`);

const deleteAllReleases = db.prepare('DELETE FROM releases');
Expand Down Expand Up @@ -193,6 +221,8 @@ router.put('/api/repositories', (req, res) => {
let count = 0;
for (const repo of repositories) {
const owner = repo.owner as { login?: string; avatar_url?: string } | undefined;
// 仅当 payload 显式提供 license 字段时才覆盖已存储值;省略(旧客户端/旧备份)则保留。
const licenseProvided = Object.prototype.hasOwnProperty.call(repo, 'license') ? 1 : 0;
stmt.run(
repo.id, repo.name, repo.full_name, repo.description ?? null,
repo.html_url, repo.stargazers_count ?? 0, repo.language ?? null,
Expand All @@ -208,7 +238,12 @@ router.put('/api/repositories', (req, res) => {
JSON.stringify(Array.isArray(repo.custom_tags) ? repo.custom_tags : []),
repo.custom_category ?? null, (repo.category_locked === true || repo.category_locked === 1) ? 1 : 0, repo.last_edited ?? null,
(repo.subscribed_to_releases === true || repo.subscribed_to_releases === 1) ? 1 : 0,
repo.vector_indexed_at ?? null
repo.vector_indexed_at ?? null,
toLicenseSpdxId(repo.license),
// 备份中 vector_indexed_license 已是规范化的 SPDX id 字符串或 null;
// 非 string 一律清空,避免奇怪类型破坏增量谓词的字符串比较。
typeof repo.vector_indexed_license === 'string' ? repo.vector_indexed_license || null : null,
{ licenseProvided }
);
count++;
}
Expand Down Expand Up @@ -245,6 +280,9 @@ router.patch('/api/repositories/:id', (req, res) => {
// 规范化:null/undefined/空字符串 → null;仅接受字符串(ISO 时间戳)
vector_indexed_at: (v) =>
(v === null || v === undefined || v === '') ? null : v,
// vector_indexed_license:规范化为 SPDX id 字符串或 null;非字符串一律清空。
vector_indexed_license: (v) =>
(v === null || v === undefined || v === '' || typeof v !== 'string') ? null : v,
description: (v) => v,
name: (v) => v,
};
Expand Down
42 changes: 39 additions & 3 deletions server/src/routes/sync.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,19 @@ import { config } from '../config.js';

const router = Router();

/** 与前端 NOASSERTION_KEYS 对齐:空白 / NOASSERTION / Other / none 等落 null。 */
const NO_LICENSE_KEYS = new Set(['', 'noassertion', 'other', 'none', 'no-license']);

/**
* 把导入的 license 字符串规范为 SPDX id 或 null。
* trim 后若为空或落入「无 license」集合则返回 null,否则返回 trim 后的原值。
*/
function canonicalizeLicenseString(value: string): string | null {
const trimmed = value.trim();
if (!trimmed || NO_LICENSE_KEYS.has(trimmed.toLowerCase())) return null;
return trimmed;
}

function maskApiKey(key: string | null | undefined): string {
if (!key || typeof key !== 'string') return '';
if (key.length <= 4) return '****';
Expand Down Expand Up @@ -108,14 +121,33 @@ router.post('/api/sync/import', (req, res) => {
owner_login, owner_avatar_url, topics,
ai_summary, ai_tags, ai_platforms, analyzed_at, analysis_failed,
custom_description, custom_tags, custom_category, category_locked, last_edited,
subscribed_to_releases, vector_indexed_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
subscribed_to_releases, vector_indexed_at, license, vector_indexed_license
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`);
Comment thread
coderabbitai[bot] marked this conversation as resolved.
for (const r of repos) {
// 验证必需的字段
if (!r.id || typeof r.id !== 'number') {
throw new Error(`Invalid repository data: missing or invalid id`);
}
// license:兼容旧备份(无该列→null)、GitHub 对象形态、已规范化的 SPDX 字符串。
// 候选字符串 trim 后,空白 / NOASSERTION / Other / none 统一落 null(与
// 前端 normalizeLicense 的「无 license」语义对齐),保证 DB 只存 SPDX-or-null。
const rawLicense = (r as Record<string, unknown>).license;
let licenseValue: string | null = null;
if (typeof rawLicense === 'string') {
licenseValue = canonicalizeLicenseString(rawLicense);
} else if (rawLicense && typeof rawLicense === 'object') {
const obj = rawLicense as { spdx_id?: unknown; key?: unknown };
const spdx = typeof obj.spdx_id === 'string' ? obj.spdx_id.trim() : '';
const key = typeof obj.key === 'string' ? obj.key.trim() : '';
licenseValue = canonicalizeLicenseString(spdx || key);
}
// vector_indexed_license 与 license 同一套 SPDX-or-null 规则,避免指纹与当前
// license 语义分裂(例如 "NOASSERTION" 原样入库后增量谓词永远判定为变更)。
const rawVectorLicense = (r as Record<string, unknown>).vector_indexed_license;
const vectorIndexedLicense = typeof rawVectorLicense === 'string'
? canonicalizeLicenseString(rawVectorLicense)
: null;
repoStmt.run(
r.id, r.name, r.full_name, r.description ?? null,
r.html_url, r.stargazers_count ?? 0, r.language ?? null,
Expand All @@ -131,7 +163,11 @@ router.post('/api/sync/import', (req, res) => {
typeof r.custom_tags === 'string' ? r.custom_tags : JSON.stringify(r.custom_tags ?? []),
r.custom_category ?? null, (r.category_locked === true || r.category_locked === 1) ? 1 : 0, r.last_edited ?? null,
r.subscribed_to_releases ? 1 : 0,
r.vector_indexed_at ?? null
r.vector_indexed_at ?? null,
licenseValue,
// INSERT OR REPLACE 会整行替换,故备份无此列时落 null(影响:增量谓词会
// 触发一次重索引回填指纹),合预期。
vectorIndexedLicense
);
}
counts.repositories = repos.length;
Expand Down
Loading