diff --git a/.vscode/nextapi.code-snippets b/.vscode/nextapi.code-snippets index 145b57c96520..19864409b03a 100644 --- a/.vscode/nextapi.code-snippets +++ b/.vscode/nextapi.code-snippets @@ -10,14 +10,19 @@ "scope": "javascript,typescript", "prefix": "nextapi", "body": [ - "import type { NextApiRequest, NextApiResponse } from 'next';", + "import type { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next';", "import { NextAPI } from '@/service/middle/entry';", "", - "type Props = {};", + "export type ${TM_FILENAME_BASE}Query = {};", "", - "type Response = {};", + "export type ${TM_FILENAME_BASE}Body = {};", "", - "async function handler(req: NextApiRequest, res: NextApiResponse): Promise {", + "export type ${TM_FILENAME_BASE}Response = {};", + "", + "async function handler(", + " req: ApiRequestProps,", + " res: ApiResponseType", + "): Promise {", " $1", " return {}", "}", @@ -25,5 +30,30 @@ "export default NextAPI(handler);" ], "description": "FastGPT Next API template" + }, + "use context template": { + "scope": "typescriptreact", + "prefix": "context", + "body": [ + "import { ReactNode } from 'react';", + "import { createContext } from 'use-context-selector';", + "", + "type ContextType = {$1};", + "", + "type ContextValueType = {};", + "", + "export const Context = createContext({});", + "", + "export const ContextProvider = ({", + " children,", + " value", + "}: {", + " children: ReactNode;", + " value: ContextValueType;", + "}) => {", + " return {children};", + "};", + ], + "description": "FastGPT usecontext template" } } \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index 003a9b700290..92400ddd1aec 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -11,5 +11,6 @@ "i18n-ally.sortKeys": true, "i18n-ally.keepFulfilled": false, "i18n-ally.sourceLanguage": "zh", // 根据此语言文件翻译其他语言文件的变量和内容 - "i18n-ally.displayLanguage": "zh" // 显示语言 + "i18n-ally.displayLanguage": "zh", // 显示语言 + "i18n-ally.extract.targetPickingStrategy": "most-similar-by-key" } \ No newline at end of file diff --git a/docSite/content/docs/development/upgrading/48.md b/docSite/content/docs/development/upgrading/48.md index 80aea7b35031..8717addd4a8b 100644 --- a/docSite/content/docs/development/upgrading/48.md +++ b/docSite/content/docs/development/upgrading/48.md @@ -1,5 +1,5 @@ --- -title: 'V4.8(开发中)' +title: 'V4.8' description: 'FastGPT V4.8 更新说明' icon: 'upgrade' draft: false diff --git a/docSite/content/docs/development/upgrading/481.md b/docSite/content/docs/development/upgrading/481.md new file mode 100644 index 000000000000..26dfea4d6b70 --- /dev/null +++ b/docSite/content/docs/development/upgrading/481.md @@ -0,0 +1,38 @@ +--- +title: 'V4.8.1(进行中)' +description: 'FastGPT V4.8.1 更新说明' +icon: 'upgrade' +draft: false +toc: true +weight: 825 +--- + +## 初始化脚本 + +从任意终端，发起 1 个 HTTP 请求。其中 {{rootkey}} 替换成环境变量里的 `rootkey`；{{host}} 替换成FastGPT的域名。 + +```bash +curl --location --request POST 'https://{{host}}/api/admin/initv481' \ +--header 'rootkey: {{rootkey}}' \ +--header 'Content-Type: application/json' +``` + +由于之前集合名不规范，该初始化会重置表名。请在初始化前，确保 dataset.trainings 表没有数据。 +最好更新该版本时，暂停所有进行中业务，再进行初始化，避免数据冲突。 + +## 执行脏数据清理 + +从任意终端，发起 1 个 HTTP 请求。其中 {{rootkey}} 替换成环境变量里的 `rootkey`；{{host}} 替换成FastGPT的域名。 + +```bash +curl --location --request POST 'https://{{host}}/api/admin/clearInvalidData' \ +--header 'rootkey: {{rootkey}}' \ +--header 'Content-Type: application/json' +``` + +初始化完后，可以执行这个命令。之前定时清理的定时器有些问题，部分数据没被清理，可以手动执行清理。 + +## V4.8.1 更新说明 + +1. 新增 - 知识库重新选择向量模型重建 +2. 修复 - 定时器清理脏数据任务 \ No newline at end of file diff --git a/packages/global/core/dataset/type.d.ts b/packages/global/core/dataset/type.d.ts index 3b84a1f7f610..7d89b322c2f2 100644 --- a/packages/global/core/dataset/type.d.ts +++ b/packages/global/core/dataset/type.d.ts @@ -80,6 +80,7 @@ export type DatasetDataSchemaType = { a: string; // answer or custom content fullTextToken: string; indexes: DatasetDataIndexItemType[]; + rebuilding?: boolean; }; export type DatasetTrainingSchemaType = { @@ -95,6 +96,7 @@ export type DatasetTrainingSchemaType = { mode: `${TrainingModeEnum}`; model: string; prompt: string; + dataId?: string; q: string; a: string; chunkIndex: number; diff --git a/packages/global/support/user/team/constant.ts b/packages/global/support/user/team/constant.ts index 024e869c970b..cc4a921587fb 100644 --- a/packages/global/support/user/team/constant.ts +++ b/packages/global/support/user/team/constant.ts @@ -1,6 +1,6 @@ export const TeamCollectionName = 'teams'; -export const TeamMemberCollectionName = 'team.members'; -export const TeamTagsCollectionName = 'team.tags'; +export const TeamMemberCollectionName = 'team_members'; +export const TeamTagsCollectionName = 'team_tags'; export enum TeamMemberRoleEnum { owner = 'owner', diff --git a/packages/service/common/buffer/rawText/schema.ts b/packages/service/common/buffer/rawText/schema.ts index d86d486ad744..64b37f5b0a45 100644 --- a/packages/service/common/buffer/rawText/schema.ts +++ b/packages/service/common/buffer/rawText/schema.ts @@ -2,7 +2,7 @@ import { connectionMongo, type Model } from '../../mongo'; const { Schema, model, models } = connectionMongo; import { RawTextBufferSchemaType } from './type'; -export const collectionName = 'buffer.rawText'; +export const collectionName = 'buffer_rawtexts'; const RawTextBufferSchema = new Schema({ sourceId: { diff --git a/packages/service/common/buffer/tts/schema.ts b/packages/service/common/buffer/tts/schema.ts index 670d1ed15793..1ffdadc5fcdb 100644 --- a/packages/service/common/buffer/tts/schema.ts +++ b/packages/service/common/buffer/tts/schema.ts @@ -2,7 +2,7 @@ import { connectionMongo, type Model } from '../../../common/mongo'; const { Schema, model, models } = connectionMongo; import { TTSBufferSchemaType } from './type.d'; -export const collectionName = 'buffer.tts'; +export const collectionName = 'buffer_tts'; const TTSBufferSchema = new Schema({ bufferId: { diff --git a/packages/service/common/mongo/sessionRun.ts b/packages/service/common/mongo/sessionRun.ts index c20219cd9301..6bfa29992969 100644 --- a/packages/service/common/mongo/sessionRun.ts +++ b/packages/service/common/mongo/sessionRun.ts @@ -12,8 +12,6 @@ export const mongoSessionRun = async (fn: (session: ClientSession) return result as T; } catch (error) { - console.log(error); - await session.abortTransaction(); await session.endSession(); return Promise.reject(error); diff --git a/packages/service/common/vectorStore/pg/controller.ts b/packages/service/common/vectorStore/pg/controller.ts index c789236420e6..3aad5b411ef0 100644 --- a/packages/service/common/vectorStore/pg/controller.ts +++ b/packages/service/common/vectorStore/pg/controller.ts @@ -98,12 +98,15 @@ export const deleteDatasetDataVector = async ( return `${teamIdWhere} ${datasetIdWhere}`; } - if ('idList' in props && props.idList) { + if ('idList' in props && Array.isArray(props.idList)) { + if (props.idList.length === 0) return; return `${teamIdWhere} id IN (${props.idList.map((id) => `'${String(id)}'`).join(',')})`; } return Promise.reject('deleteDatasetData: no where'); })(); + if (!where) return; + try { await PgClient.delete(PgDatasetTableName, { where: [where] diff --git a/packages/service/core/app/versionSchema.ts b/packages/service/core/app/versionSchema.ts index 907b44ceb88d..c573078266eb 100644 --- a/packages/service/core/app/versionSchema.ts +++ b/packages/service/core/app/versionSchema.ts @@ -2,7 +2,7 @@ import { connectionMongo, type Model } from '../../common/mongo'; const { Schema, model, models } = connectionMongo; import { AppVersionSchemaType } from '@fastgpt/global/core/app/version'; -export const AppVersionCollectionName = 'app.versions'; +export const AppVersionCollectionName = 'app_versions'; const AppVersionSchema = new Schema({ appId: { diff --git a/packages/service/core/dataset/collection/schema.ts b/packages/service/core/dataset/collection/schema.ts index 4b3d4ca8d170..e8cb6e53acc6 100644 --- a/packages/service/core/dataset/collection/schema.ts +++ b/packages/service/core/dataset/collection/schema.ts @@ -8,7 +8,7 @@ import { TeamMemberCollectionName } from '@fastgpt/global/support/user/team/constant'; -export const DatasetColCollectionName = 'dataset.collections'; +export const DatasetColCollectionName = 'dataset_collections'; const DatasetCollectionSchema = new Schema({ parentId: { diff --git a/packages/service/core/dataset/data/schema.ts b/packages/service/core/dataset/data/schema.ts index cf4fba0057f3..0a80b23a6df4 100644 --- a/packages/service/core/dataset/data/schema.ts +++ b/packages/service/core/dataset/data/schema.ts @@ -8,7 +8,7 @@ import { import { DatasetCollectionName } from '../schema'; import { DatasetColCollectionName } from '../collection/schema'; -export const DatasetDataCollectionName = 'dataset.datas'; +export const DatasetDataCollectionName = 'dataset_datas'; const DatasetDataSchema = new Schema({ teamId: { @@ -73,7 +73,8 @@ const DatasetDataSchema = new Schema({ }, inited: { type: Boolean - } + }, + rebuilding: Boolean }); try { @@ -90,10 +91,13 @@ try { { background: true } ); DatasetDataSchema.index({ updateTime: 1 }, { background: true }); + // rebuild data + DatasetDataSchema.index({ rebuilding: 1, teamId: 1, datasetId: 1 }, { background: true }); } catch (error) { console.log(error); } export const MongoDatasetData: Model = models[DatasetDataCollectionName] || model(DatasetDataCollectionName, DatasetDataSchema); + MongoDatasetData.syncIndexes(); diff --git a/packages/service/core/dataset/training/schema.ts b/packages/service/core/dataset/training/schema.ts index 0ff24ac9c951..db1602eae8c4 100644 --- a/packages/service/core/dataset/training/schema.ts +++ b/packages/service/core/dataset/training/schema.ts @@ -10,7 +10,7 @@ import { TeamMemberCollectionName } from '@fastgpt/global/support/user/team/constant'; -export const DatasetTrainingCollectionName = 'dataset.trainings'; +export const DatasetTrainingCollectionName = 'dataset_trainings'; const TrainingDataSchema = new Schema({ teamId: { @@ -35,8 +35,7 @@ const TrainingDataSchema = new Schema({ }, billId: { // concat bill - type: String, - default: '' + type: Schema.Types.ObjectId }, mode: { type: String, @@ -78,6 +77,9 @@ const TrainingDataSchema = new Schema({ type: Number, default: 0 }, + dataId: { + type: Schema.Types.ObjectId + }, indexes: { type: [ { diff --git a/packages/service/type/next.d.ts b/packages/service/type/next.d.ts new file mode 100644 index 000000000000..dadfa81d3e2c --- /dev/null +++ b/packages/service/type/next.d.ts @@ -0,0 +1,8 @@ +import type { NextApiRequest, NextApiResponse } from 'next'; + +export type ApiRequestProps = Omit & { + query: Query; + body: Body; +}; + +export type { NextApiResponse as ApiResponseType } from 'next'; diff --git a/packages/web/components/common/MyDivider/index.tsx b/packages/web/components/common/MyDivider/index.tsx new file mode 100644 index 000000000000..42a68617c445 --- /dev/null +++ b/packages/web/components/common/MyDivider/index.tsx @@ -0,0 +1,9 @@ +import React from 'react'; +import { Divider, type DividerProps } from '@chakra-ui/react'; + +const MyDivider = (props: DividerProps) => { + const { h } = props; + return ; +}; + +export default MyDivider; diff --git a/packages/web/components/common/MyTooltip/index.tsx b/packages/web/components/common/MyTooltip/index.tsx index 69e9bf3fa7fb..25982258d4b0 100644 --- a/packages/web/components/common/MyTooltip/index.tsx +++ b/packages/web/components/common/MyTooltip/index.tsx @@ -1,5 +1,5 @@ import React from 'react'; -import { Tooltip, TooltipProps, css, useMediaQuery } from '@chakra-ui/react'; +import { Box, Tooltip, TooltipProps, css, useMediaQuery } from '@chakra-ui/react'; interface Props extends TooltipProps { forceShow?: boolean; @@ -9,24 +9,32 @@ const MyTooltip = ({ children, forceShow = false, shouldWrapChildren = true, ... const [isPc] = useMediaQuery('(min-width: 900px)'); return isPc || forceShow ? ( - - {children} - + + {children} + + ) : ( <>{children} ); diff --git a/projects/app/.eslintrc.json b/projects/app/.eslintrc.json index be661eb5c4df..3a5d07f926c9 100644 --- a/projects/app/.eslintrc.json +++ b/projects/app/.eslintrc.json @@ -1,6 +1,12 @@ + { + "parser": "@typescript-eslint/parser", // 确保使用了 TypeScript 解析器 + "plugins": ["@typescript-eslint"], // 引入 TypeScript 插件 + "extends": "next/core-web-vitals", "rules": { - "react-hooks/rules-of-hooks": 0 + "react-hooks/rules-of-hooks": 0, + "@typescript-eslint/consistent-type-imports": "warn" // 或者 "error" 来强制执行 + } } diff --git a/projects/app/i18n/en/dataset.json b/projects/app/i18n/en/dataset.json index e69de29bb2d1..0c2bc01d815e 100644 --- a/projects/app/i18n/en/dataset.json +++ b/projects/app/i18n/en/dataset.json @@ -0,0 +1,6 @@ +{ + "Confirm to rebuild embedding tip": "Are you sure to switch the knowledge base index? Switching index is a very heavy operation that requires re-indexing all the data in your knowledge base, which may take a long time. Please ensure that the remaining points in your account are sufficient.", + "Rebuild embedding start tip": "The task of switching index models has begun", + "Rebuilding index count": "Rebuilding count: {{count}}", + "The knowledge base has indexes that are being trained or being rebuilt": "The knowledge base has indexes that are being trained or being rebuilt" +} diff --git a/projects/app/i18n/zh/dataset.json b/projects/app/i18n/zh/dataset.json index e69de29bb2d1..4289048f7cba 100644 --- a/projects/app/i18n/zh/dataset.json +++ b/projects/app/i18n/zh/dataset.json @@ -0,0 +1,6 @@ +{ + "Confirm to rebuild embedding tip": "确认为知识库切换索引？\n切换索引是一个非常重量的操作，需要对您知识库内所有数据进行重新索引，时间可能较长，请确保账号内剩余积分充足。", + "Rebuild embedding start tip": "切换索引模型任务已开始", + "Rebuilding index count": "重建中索引数量: {{count}}", + "The knowledge base has indexes that are being trained or being rebuilt": "知识库有训练中或正在重建的索引" +} diff --git a/projects/app/src/components/Select/AIModelSelector.tsx b/projects/app/src/components/Select/AIModelSelector.tsx index 8d19102fb81a..fe779a1f9d73 100644 --- a/projects/app/src/components/Select/AIModelSelector.tsx +++ b/projects/app/src/components/Select/AIModelSelector.tsx @@ -8,8 +8,13 @@ import MySelect, { SelectProps } from '@fastgpt/web/components/common/MySelect'; import { HUGGING_FACE_ICON, LOGO_ICON } from '@fastgpt/global/common/system/constants'; import { Box, Flex } from '@chakra-ui/react'; import Avatar from '../Avatar'; +import MyTooltip from '@fastgpt/web/components/common/MyTooltip'; -const AIModelSelector = ({ list, onchange, ...props }: SelectProps) => { +type Props = SelectProps & { + disableTip?: string; +}; + +const AIModelSelector = ({ list, onchange, disableTip, ...props }: Props) => { const { t } = useTranslation(); const { feConfigs, llmModelList, vectorModelList } = useSystemStore(); const router = useRouter(); @@ -62,9 +67,9 @@ const AIModelSelector = ({ list, onchange, ...props }: SelectProps) => { ); return ( - <> - - + + + ); }; diff --git a/projects/app/src/global/core/dataset/api.d.ts b/projects/app/src/global/core/dataset/api.d.ts index 240f92e4e4b5..69ef8c9ecf9e 100644 --- a/projects/app/src/global/core/dataset/api.d.ts +++ b/projects/app/src/global/core/dataset/api.d.ts @@ -22,6 +22,11 @@ export type CreateDatasetParams = { agentModel?: string; }; +export type RebuildEmbeddingProps = { + datasetId: string; + vectorModel: string; +}; + /* ================= collection ===================== */ /* ================= data ===================== */ diff --git a/projects/app/src/pages/api/admin/initv481.ts b/projects/app/src/pages/api/admin/initv481.ts new file mode 100644 index 000000000000..3ef36186871a --- /dev/null +++ b/projects/app/src/pages/api/admin/initv481.ts @@ -0,0 +1,178 @@ +import type { NextApiRequest, NextApiResponse } from 'next'; +import { jsonRes } from '@fastgpt/service/common/response'; +import { connectToDatabase } from '@/service/mongo'; +import { authCert } from '@fastgpt/service/support/permission/auth/common'; +import { PgClient } from '@fastgpt/service/common/vectorStore/pg'; +import { NextAPI } from '@/service/middle/entry'; +import { PgDatasetTableName } from '@fastgpt/global/common/vectorStore/constants'; +import { connectionMongo } from '@fastgpt/service/common/mongo'; +import { addLog } from '@fastgpt/service/common/system/log'; + +/* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */ +async function handler(req: NextApiRequest, res: NextApiResponse) { + await authCert({ req, authRoot: true }); + + // 重命名 dataset.trainigns -> dataset_trainings + try { + const collections = await connectionMongo.connection.db + .listCollections({ name: 'dataset.trainings' }) + .toArray(); + if (collections.length > 0) { + const sourceCol = connectionMongo.connection.db.collection('dataset.trainings'); + const targetCol = connectionMongo.connection.db.collection('dataset_trainings'); + + if ((await targetCol.countDocuments()) > 0) { + console.log( + 'dataset_trainings 中有数据，无法自动将 dataset.trainings 迁移到 dataset_trainings，请手动操作' + ); + } else { + await sourceCol.rename('dataset_trainings', { dropTarget: true }); + console.log('success rename dataset.trainings -> dataset_trainings'); + } + } + } catch (error) { + console.log('error： rename dataset.trainings -> dataset_trainings', error); + } + + try { + const collections = await connectionMongo.connection.db + .listCollections({ name: 'dataset.collections' }) + .toArray(); + if (collections.length > 0) { + const sourceCol = connectionMongo.connection.db.collection('dataset.collections'); + const targetCol = connectionMongo.connection.db.collection('dataset_collections'); + + if ((await targetCol.countDocuments()) > 0) { + console.log( + 'dataset_collections 中有数据，无法自动将 dataset.collections 迁移到 dataset_collections，请手动操作' + ); + } else { + await sourceCol.rename('dataset_collections', { dropTarget: true }); + console.log('success rename dataset.collections -> dataset_collections'); + } + } + } catch (error) { + console.log('error： rename dataset.collections -> dataset_collections', error); + } + + try { + const collections = await connectionMongo.connection.db + .listCollections({ name: 'dataset.datas' }) + .toArray(); + if (collections.length > 0) { + const sourceCol = connectionMongo.connection.db.collection('dataset.datas'); + const targetCol = connectionMongo.connection.db.collection('dataset_datas'); + + if ((await targetCol.countDocuments()) > 0) { + console.log( + 'dataset_datas 中有数据，无法自动将 dataset.datas 迁移到 dataset_datas，请手动操作' + ); + } else { + await sourceCol.rename('dataset_datas', { dropTarget: true }); + console.log('success rename dataset.datas -> dataset_datas'); + } + } + } catch (error) { + console.log('error： rename dataset.datas -> dataset_datas', error); + } + + try { + const collections = await connectionMongo.connection.db + .listCollections({ name: 'app.versions' }) + .toArray(); + if (collections.length > 0) { + const sourceCol = connectionMongo.connection.db.collection('app.versions'); + const targetCol = connectionMongo.connection.db.collection('app_versions'); + + if ((await targetCol.countDocuments()) > 0) { + console.log( + 'app_versions 中有数据，无法自动将 app.versions 迁移到 app_versions，请手动操作' + ); + } else { + await sourceCol.rename('app_versions', { dropTarget: true }); + console.log('success rename app.versions -> app_versions'); + } + } + } catch (error) { + console.log('error： rename app.versions -> app_versions', error); + } + + try { + const collections = await connectionMongo.connection.db + .listCollections({ name: 'buffer.rawtexts' }) + .toArray(); + if (collections.length > 0) { + const sourceCol = connectionMongo.connection.db.collection('buffer.rawtexts'); + const targetCol = connectionMongo.connection.db.collection('buffer_rawtexts'); + + if ((await targetCol.countDocuments()) > 0) { + console.log( + 'buffer_rawtexts 中有数据，无法自动将 buffer.rawtexts 迁移到 buffer_rawtexts，请手动操作' + ); + } else { + await sourceCol.rename('buffer_rawtexts', { dropTarget: true }); + console.log('success rename buffer.rawtexts -> buffer_rawtexts'); + } + } + } catch (error) { + console.log('error： rename buffer.rawtext -> buffer_rawtext', error); + } + + try { + const collections = await connectionMongo.connection.db + .listCollections({ name: 'buffer.tts' }) + .toArray(); + if (collections.length > 0) { + const sourceCol = connectionMongo.connection.db.collection('buffer.tts'); + const targetCol = connectionMongo.connection.db.collection('buffer_tts'); + + if ((await targetCol.countDocuments()) > 0) { + console.log('buffer_tts 中有数据，无法自动将 buffer.tts 迁移到 buffer_tts，请手动操作'); + } else { + await sourceCol.rename('buffer_tts', { dropTarget: true }); + console.log('success rename buffer.tts -> buffer_tts'); + } + } + } catch (error) { + console.log('error： rename buffer.tts -> buffer_tts', error); + } + + try { + const collections = await connectionMongo.connection.db + .listCollections({ name: 'team.members' }) + .toArray(); + if (collections.length > 0) { + const sourceCol = connectionMongo.connection.db.collection('team.members'); + + await sourceCol.rename('team_members', { dropTarget: true }); + console.log('success rename team.members -> team_members'); + } + } catch (error) { + console.log('error： rename team.members -> team_members', error); + } + + try { + const collections = await connectionMongo.connection.db + .listCollections({ name: 'team.tags' }) + .toArray(); + if (collections.length > 0) { + const sourceCol = connectionMongo.connection.db.collection('team.tags'); + const targetCol = connectionMongo.connection.db.collection('team_tags'); + + if ((await targetCol.countDocuments()) > 0) { + console.log('team_tags 中有数据，无法自动将 team.tags 迁移到 team_tags，请手动操作'); + } else { + await sourceCol.rename('team_tags', { dropTarget: true }); + console.log('success rename team.tags -> team_tags'); + } + } + } catch (error) { + console.log('error： rename team.tags -> team_tags', error); + } + + jsonRes(res, { + message: 'success' + }); +} + +export default NextAPI(handler); diff --git a/projects/app/src/pages/api/core/dataset/training/getDatasetTrainingQueue.ts b/projects/app/src/pages/api/core/dataset/training/getDatasetTrainingQueue.ts new file mode 100644 index 000000000000..b52f5a59a833 --- /dev/null +++ b/projects/app/src/pages/api/core/dataset/training/getDatasetTrainingQueue.ts @@ -0,0 +1,39 @@ +import type { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next'; +import { NextAPI } from '@/service/middle/entry'; +import { authDataset } from '@fastgpt/service/support/permission/auth/dataset'; +import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema'; +import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema'; + +type Props = {}; + +export type getDatasetTrainingQueueResponse = { + rebuildingCount: number; + trainingCount: number; +}; + +async function handler( + req: ApiRequestProps, + res: ApiResponseType +): Promise { + const { datasetId } = req.query; + + const { teamId } = await authDataset({ + req, + authToken: true, + authApiKey: true, + datasetId, + per: 'r' + }); + + const [rebuildingCount, trainingCount] = await Promise.all([ + MongoDatasetData.countDocuments({ teamId, datasetId, rebuilding: true }), + MongoDatasetTraining.countDocuments({ teamId, datasetId }) + ]); + + return { + rebuildingCount, + trainingCount + }; +} + +export default NextAPI(handler); diff --git a/projects/app/src/pages/api/core/dataset/training/rebuildEmbedding.ts b/projects/app/src/pages/api/core/dataset/training/rebuildEmbedding.ts new file mode 100644 index 000000000000..5d29104824a7 --- /dev/null +++ b/projects/app/src/pages/api/core/dataset/training/rebuildEmbedding.ts @@ -0,0 +1,133 @@ +import { NextAPI } from '@/service/middle/entry'; +import { authDataset } from '@fastgpt/service/support/permission/auth/dataset'; +import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun'; +import { MongoDataset } from '@fastgpt/service/core/dataset/schema'; +import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema'; +import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema'; +import { createTrainingUsage } from '@fastgpt/service/support/wallet/usage/controller'; +import { UsageSourceEnum } from '@fastgpt/global/support/wallet/usage/constants'; +import { getLLMModel, getVectorModel } from '@fastgpt/service/core/ai/model'; +import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants'; +import { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next'; + +export type rebuildEmbeddingBody = { + datasetId: string; + vectorModel: string; +}; + +export type Response = {}; + +async function handler( + req: ApiRequestProps, + res: ApiResponseType +): Promise { + const { datasetId, vectorModel } = req.body; + + const { teamId, tmbId, dataset } = await authDataset({ + req, + authToken: true, + authApiKey: true, + datasetId, + per: 'owner' + }); + + // check vector model + if (!vectorModel || dataset.vectorModel === vectorModel) { + return Promise.reject('vectorModel 不合法'); + } + + // check rebuilding or training + const [rebuilding, training] = await Promise.all([ + MongoDatasetData.findOne({ teamId, datasetId, rebuilding: true }), + MongoDatasetTraining.findOne({ teamId, datasetId }) + ]); + + if (rebuilding || training) { + return Promise.reject('数据集正在训练或者重建中，请稍后再试'); + } + + const { billId } = await createTrainingUsage({ + teamId, + tmbId, + appName: '切换索引模型', + billSource: UsageSourceEnum.training, + vectorModel: getVectorModel(dataset.vectorModel)?.name, + agentModel: getLLMModel(dataset.agentModel)?.name + }); + + // update vector model and dataset.data rebuild field + await mongoSessionRun(async (session) => { + await MongoDataset.findByIdAndUpdate( + datasetId, + { + vectorModel + }, + { session } + ); + await MongoDatasetData.updateMany( + { + teamId, + datasetId + }, + { + $set: { + rebuilding: true + } + }, + { + session + } + ); + }); + + // get 10 init dataset.data + const arr = new Array(10).fill(0); + for await (const _ of arr) { + await mongoSessionRun(async (session) => { + const data = await MongoDatasetData.findOneAndUpdate( + { + teamId, + datasetId, + rebuilding: true + }, + { + $unset: { + rebuilding: null + }, + updateTime: new Date() + }, + { + session + } + ).select({ + _id: 1, + collectionId: 1 + }); + + if (data) { + await MongoDatasetTraining.create( + [ + { + teamId, + tmbId, + datasetId, + collectionId: data.collectionId, + billId, + mode: TrainingModeEnum.chunk, + model: vectorModel, + q: '1', + dataId: data._id + } + ], + { + session + } + ); + } + }); + } + + return {}; +} + +export default NextAPI(handler); diff --git a/projects/app/src/pages/dataset/detail/components/Info.tsx b/projects/app/src/pages/dataset/detail/components/Info.tsx index ed586b8fb74f..b26bbd76cbc1 100644 --- a/projects/app/src/pages/dataset/detail/components/Info.tsx +++ b/projects/app/src/pages/dataset/detail/components/Info.tsx @@ -16,25 +16,47 @@ import PermissionRadio from '@/components/support/permission/Radio'; import { useSystemStore } from '@/web/common/system/useSystemStore'; import { useRequest } from '@fastgpt/web/hooks/useRequest'; import { MongoImageTypeEnum } from '@fastgpt/global/common/file/image/constants'; -import MySelect from '@fastgpt/web/components/common/MySelect'; import AIModelSelector from '@/components/Select/AIModelSelector'; +import { postRebuildEmbedding } from '@/web/core/dataset/api'; +import { useI18n } from '@/web/context/I18n'; +import type { VectorModelItemType } from '@fastgpt/global/core/ai/model.d'; +import { useContextSelector } from 'use-context-selector'; +import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; +import MyDivider from '@fastgpt/web/components/common/MyDivider/index'; const Info = ({ datasetId }: { datasetId: string }) => { const { t } = useTranslation(); - const { datasetDetail, loadDatasets, updateDataset } = useDatasetStore(); - const { getValues, setValue, register, handleSubmit } = useForm({ + const { datasetT } = useI18n(); + const { datasetDetail, loadDatasetDetail, loadDatasets, updateDataset } = useDatasetStore(); + const rebuildingCount = useContextSelector(DatasetPageContext, (v) => v.rebuildingCount); + const trainingCount = useContextSelector(DatasetPageContext, (v) => v.trainingCount); + const refetchDatasetTraining = useContextSelector( + DatasetPageContext, + (v) => v.refetchDatasetTraining + ); + + const { setValue, register, handleSubmit, watch } = useForm({ defaultValues: datasetDetail }); + + const avatar = watch('avatar'); + const vectorModel = watch('vectorModel'); + const agentModel = watch('agentModel'); + const permission = watch('permission'); + const { datasetModelList, vectorModelList } = useSystemStore(); const router = useRouter(); - const [refresh, setRefresh] = useState(false); - - const { openConfirm, ConfirmModal } = useConfirm({ + const { openConfirm: onOpenConfirmDel, ConfirmModal: ConfirmDelModal } = useConfirm({ content: t('core.dataset.Delete Confirm'), type: 'delete' }); + const { openConfirm: onOpenConfirmRebuild, ConfirmModal: ConfirmRebuildModal } = useConfirm({ + title: t('common.confirm.Common Tip'), + content: datasetT('Confirm to rebuild embedding tip'), + type: 'delete' + }); const { File, onOpen: onOpenSelectFile } = useSelectFile({ fileType: '.jpg,.png', @@ -81,13 +103,27 @@ const Info = ({ datasetId }: { datasetId: string }) => { onSuccess(src: string | null) { if (src) { setValue('avatar', src); - setRefresh((state) => !state); } }, errorToast: t('common.avatar.Select Failed') }); - const btnLoading = useMemo(() => isDeleting || isSaving, [isDeleting, isSaving]); + const { mutate: onRebuilding, isLoading: isRebuilding } = useRequest({ + mutationFn: (vectorModel: VectorModelItemType) => { + return postRebuildEmbedding({ + datasetId, + vectorModel: vectorModel.model + }); + }, + onSuccess() { + refetchDatasetTraining(); + loadDatasetDetail(datasetId, true); + }, + successToast: datasetT('Rebuild embedding start tip'), + errorToast: t('common.Update Failed') + }); + + const btnLoading = isSelecting || isDeleting || isSaving || isRebuilding; return ( @@ -97,50 +133,48 @@ const Info = ({ datasetId }: { datasetId: string }) => { {datasetDetail._id} - - + - {t('core.dataset.Avatar')} + {t('core.ai.model.Vector Model')} - - - - - - - - {t('core.dataset.Name')} - - - - - - {t('core.ai.model.Vector Model')} + 0 || trainingCount > 0 + ? datasetT('The knowledge base has indexes that are being trained or being rebuilt') + : undefined + } + list={vectorModelList.map((item) => ({ + label: item.name, + value: item.model + }))} + onchange={(e) => { + const vectorModel = vectorModelList.find((item) => item.model === e); + if (!vectorModel) return; + onOpenConfirmRebuild(() => { + setValue('vectorModel', vectorModel); + onRebuilding(vectorModel); + })(); + }} + /> - {getValues('vectorModel').name} {t('core.Max Token')} - {getValues('vectorModel').maxToken} + {vectorModel.maxToken} - + {t('core.ai.model.Dataset Agent Model')} ({ label: item.name, value: item.model @@ -149,12 +183,36 @@ const Info = ({ datasetId }: { datasetId: string }) => { const agentModel = datasetModelList.find((item) => item.model === e); if (!agentModel) return; setValue('agentModel', agentModel); - setRefresh((state) => !state); }} /> + + + + + {t('core.dataset.Avatar')} + + + + + + + + + + {t('core.dataset.Name')} + + + {t('common.Intro')}