diff --git a/docSite/content/zh-cn/docs/development/openapi/dataset.md b/docSite/content/zh-cn/docs/development/openapi/dataset.md
index d6ca1ba284fa..d0fc0b508126 100644
--- a/docSite/content/zh-cn/docs/development/openapi/dataset.md
+++ b/docSite/content/zh-cn/docs/development/openapi/dataset.md
@@ -531,6 +531,8 @@ data 为集合的 ID。
{{< tab tabName="请求示例" >}}
{{< markdownify >}}
+使用代码上传时,请注意中文 filename 需要进行 encode 处理,否则容易乱码。
+
```bash
curl --location --request POST 'http://localhost:3000/api/core/dataset/collection/create/localFile' \
--header 'Authorization: Bearer {{authorization}}' \
@@ -1173,7 +1175,7 @@ curl --location --request PUT 'http://localhost:3000/api/core/dataset/data/updat
--header 'Authorization: Bearer {{authorization}}' \
--header 'Content-Type: application/json' \
--data-raw '{
- "id":"65abd4b29d1448617cba61db",
+ "dataId":"65abd4b29d1448617cba61db",
"q":"测试111",
"a":"sss",
"indexes":[
@@ -1196,7 +1198,7 @@ curl --location --request PUT 'http://localhost:3000/api/core/dataset/data/updat
{{< markdownify >}}
{{% alert icon=" " context="success" %}}
-- id: 数据的id
+- dataId: 数据的id
- q: 主要数据(选填)
- a: 辅助数据(选填)
- indexes: 自定义索引(选填),类型参考`为集合批量添加添加数据`。如果创建时候有自定义索引,
diff --git a/docSite/content/zh-cn/docs/development/upgrading/489.md b/docSite/content/zh-cn/docs/development/upgrading/489.md
index 95f470ebe88b..46ea842a5cd1 100644
--- a/docSite/content/zh-cn/docs/development/upgrading/489.md
+++ b/docSite/content/zh-cn/docs/development/upgrading/489.md
@@ -38,12 +38,14 @@ curl --location --request POST 'https://{{host}}/api/admin/init/489' \
6. 商业版新增 - 知识库搜索节点支持标签过滤和创建时间过滤。
7. 商业版新增 - 转移 App owner 权限。
8. 新增 - 删除所有对话引导内容。
-9. 优化 - 对话框信息懒加载,减少网络传输。
-10. 优化 - 清除选文件缓存,支持重复选择同一个文件。
-11. 修复 - 知识库上传文件,网络不稳定或文件较多情况下,进度无法到 100%。
-12. 修复 - 删除应用后回到聊天选择最后一次对话的应用为删除的应用时提示无该应用问题。
-13. 修复 - 插件动态变量配置默认值时,无法正常显示默认值。
-14. 修复 - 工具调用温度和最大回复值未生效。
-15. 修复 - 函数调用模式,assistant role 中,GPT 模型必须传入 content 参数。(不影响大部分模型,目前基本都改用用 ToolChoice 模式,FC 模式已弃用)。
-16. 修复 - 知识库文件上传进度更新可能异常。
-17. 修复 - 知识库 rebuilding 时候,页面总是刷新到第一页。
+9. 新增 - QA 拆分支持自定义 chunk 大小,并优化 gpt4o-mini 拆分时,chunk 太大导致生成内容很少的问题。
+10. 优化 - 对话框信息懒加载,减少网络传输。
+11. 优化 - 清除选文件缓存,支持重复选择同一个文件。
+12. 修复 - 知识库上传文件,网络不稳定或文件较多情况下,进度无法到 100%。
+13. 修复 - 删除应用后回到聊天选择最后一次对话的应用为删除的应用时提示无该应用问题。
+14. 修复 - 插件动态变量配置默认值时,无法正常显示默认值。
+15. 修复 - 工具调用温度和最大回复值未生效。
+16. 修复 - 函数调用模式,assistant role 中,GPT 模型必须传入 content 参数。(不影响大部分模型,目前基本都改用用 ToolChoice 模式,FC 模式已弃用)。
+17. 修复 - 知识库文件上传进度更新可能异常。
+18. 修复 - 知识库 rebuilding 时候,页面总是刷新到第一页。
+19. 修复 - 知识库 list openapi 鉴权问题。
diff --git a/packages/global/common/error/utils.ts b/packages/global/common/error/utils.ts
index 73f0ec646749..54a47ed01c3e 100644
--- a/packages/global/common/error/utils.ts
+++ b/packages/global/common/error/utils.ts
@@ -1,6 +1,6 @@
import { replaceSensitiveText } from '../string/tools';
-export const getErrText = (err: any, def = '') => {
+export const getErrText = (err: any, def = ''): any => {
const msg: string =
typeof err === 'string'
? err
diff --git a/packages/global/common/file/constants.ts b/packages/global/common/file/constants.ts
index e1f7eac1d77e..d92144c37719 100644
--- a/packages/global/common/file/constants.ts
+++ b/packages/global/common/file/constants.ts
@@ -7,10 +7,12 @@ export enum BucketNameEnum {
}
export const bucketNameMap = {
[BucketNameEnum.dataset]: {
- label: i18nT('file:bucket_file')
+ label: i18nT('file:bucket_file'),
+ previewExpireMinutes: 30 // 30 minutes
},
[BucketNameEnum.chat]: {
- label: i18nT('file:bucket_chat')
+ label: i18nT('file:bucket_chat'),
+ previewExpireMinutes: 7 * 24 * 60 // 7 days
}
};
diff --git a/packages/global/common/file/type.d.ts b/packages/global/common/file/type.d.ts
index 56fe0876e541..dfe8b21c6dfe 100644
--- a/packages/global/common/file/type.d.ts
+++ b/packages/global/common/file/type.d.ts
@@ -5,5 +5,4 @@ export type FileTokenQuery = {
teamId: string;
tmbId: string;
fileId: string;
- expiredTime?: number;
};
diff --git a/packages/global/common/frequenctLimit/type.d.ts b/packages/global/common/frequenctLimit/type.d.ts
new file mode 100644
index 000000000000..3f326a40cd1e
--- /dev/null
+++ b/packages/global/common/frequenctLimit/type.d.ts
@@ -0,0 +1,5 @@
+export type AuthFrequencyLimitProps = {
+ eventId: string;
+ maxAmount: number;
+ expiredTime: Date;
+};
diff --git a/packages/global/core/ai/prompt/agent.ts b/packages/global/core/ai/prompt/agent.ts
index 2477696e18be..3c6a4fb12b0c 100644
--- a/packages/global/core/ai/prompt/agent.ts
+++ b/packages/global/core/ai/prompt/agent.ts
@@ -1,9 +1,9 @@
export const Prompt_AgentQA = {
description: ` 标记中是一段文本,学习和分析它,并整理学习成果:
- 提出问题并给出每个问题的答案。
-- 答案需详细完整,尽可能保留原文描述。
+- 答案需详细完整,尽可能保留原文描述,可以适当扩展答案描述。
- 答案可以包含普通文字、链接、代码、表格、公示、媒体链接等 Markdown 元素。
-- 最多提出 30 个问题。
+- 最多提出 50 个问题。
`,
fixedText: `请按以下格式整理学习成果:
diff --git a/packages/global/core/workflow/template/system/readFiles/index.tsx b/packages/global/core/workflow/template/system/readFiles/index.tsx
index eb2ea9d767d9..cbff1d219409 100644
--- a/packages/global/core/workflow/template/system/readFiles/index.tsx
+++ b/packages/global/core/workflow/template/system/readFiles/index.tsx
@@ -19,7 +19,7 @@ export const ReadFilesNodes: FlowNodeTemplateType = {
flowNodeType: FlowNodeTypeEnum.readFiles,
sourceHandle: getHandleConfig(true, true, true, true),
targetHandle: getHandleConfig(true, true, true, true),
- avatar: 'core/app/simpleMode/file',
+ avatar: 'core/workflow/template/readFiles',
name: i18nT('app:workflow.read_files'),
intro: i18nT('app:workflow.read_files_tip'),
showStatus: true,
diff --git a/packages/global/support/wallet/usage/constants.ts b/packages/global/support/wallet/usage/constants.ts
index 13dae786e95e..9e7a1b92a578 100644
--- a/packages/global/support/wallet/usage/constants.ts
+++ b/packages/global/support/wallet/usage/constants.ts
@@ -1,8 +1,11 @@
+import { i18nT } from '../../../../web/i18n/utils';
+
export enum UsageSourceEnum {
fastgpt = 'fastgpt',
api = 'api',
shareLink = 'shareLink',
- training = 'training'
+ training = 'training',
+ cronJob = 'cronJob'
}
export const UsageSourceMap = {
@@ -17,5 +20,8 @@ export const UsageSourceMap = {
},
[UsageSourceEnum.training]: {
label: 'dataset.Training Name'
+ },
+ [UsageSourceEnum.cronJob]: {
+ label: i18nT('common:cron_job_run_app')
}
};
diff --git a/packages/service/common/file/gridfs/utils.ts b/packages/service/common/file/gridfs/utils.ts
index 231a5622d031..8f27a07187cc 100644
--- a/packages/service/common/file/gridfs/utils.ts
+++ b/packages/service/common/file/gridfs/utils.ts
@@ -45,7 +45,7 @@ export const stream2Encoding = async (stream: NodeJS.ReadableStream) => {
})();
const enc = detectFileEncoding(buffer);
- console.log('Get encoding time', Date.now() - start, enc);
+
return {
encoding: enc,
stream: copyStream
diff --git a/packages/service/common/file/multer.ts b/packages/service/common/file/multer.ts
index 344bb5c8988e..a1c531d1a46d 100644
--- a/packages/service/common/file/multer.ts
+++ b/packages/service/common/file/multer.ts
@@ -3,7 +3,6 @@ import multer from 'multer';
import path from 'path';
import { BucketNameEnum, bucketNameMap } from '@fastgpt/global/common/file/constants';
import { getNanoid } from '@fastgpt/global/common/string/tools';
-import { tmpFileDirPath } from './constants';
type FileType = {
fieldname: string;
diff --git a/packages/service/core/workflow/dispatch/agent/extract.ts b/packages/service/core/workflow/dispatch/agent/extract.ts
index e8004f7590bb..a37540920778 100644
--- a/packages/service/core/workflow/dispatch/agent/extract.ts
+++ b/packages/service/core/workflow/dispatch/agent/extract.ts
@@ -228,7 +228,7 @@ const toolChoice = async (props: ActionProps) => {
const response = await ai.chat.completions.create({
model: extractModel.model,
- temperature: 0,
+ temperature: 0.01,
messages: filterMessages,
tools,
tool_choice: { type: 'function', function: { name: agentFunName } }
@@ -273,7 +273,7 @@ const functionCall = async (props: ActionProps) => {
const response = await ai.chat.completions.create({
model: extractModel.model,
- temperature: 0,
+ temperature: 0.01,
messages: filterMessages,
function_call: {
name: agentFunName
diff --git a/packages/service/core/workflow/dispatch/tools/readFiles.ts b/packages/service/core/workflow/dispatch/tools/readFiles.ts
index 6f4bffce3fe0..3a2b85a72be2 100644
--- a/packages/service/core/workflow/dispatch/tools/readFiles.ts
+++ b/packages/service/core/workflow/dispatch/tools/readFiles.ts
@@ -49,7 +49,7 @@ export const dispatchReadFiles = async (props: Props): Promise => {
chatConfig,
params: { fileUrlList = [] }
} = props;
- const maxFiles = chatConfig?.fileSelectConfig?.maxFiles || 0;
+ const maxFiles = chatConfig?.fileSelectConfig?.maxFiles || 20;
// Get files from histories
const filesFromHistories = histories
@@ -70,29 +70,35 @@ export const dispatchReadFiles = async (props: Props): Promise => {
})
.flat();
- const parseUrlList = [...fileUrlList, ...filesFromHistories].slice(0, maxFiles);
-
- const readFilesResult = await Promise.all(
- parseUrlList
- .map(async (url) => {
- // System file
- if (url.startsWith('/') || (requestOrigin && url.startsWith(requestOrigin))) {
- // Parse url, get filename query. Keep only documents that can be parsed
- const parseUrl = new URL(url);
- const filenameQuery = parseUrl.searchParams.get('filename');
- if (filenameQuery) {
- const extensionQuery = filenameQuery.split('.').pop()?.toLowerCase() || '';
- if (!documentFileType.includes(extensionQuery)) {
- return;
- }
+ // Concat fileUrlList and filesFromHistories; remove not supported files
+ const parseUrlList = [...fileUrlList, ...filesFromHistories]
+ .map((url) => {
+ // System file
+ if (url.startsWith('/') || (requestOrigin && url.startsWith(requestOrigin))) {
+ // Parse url, get filename query. Keep only documents that can be parsed
+ const parseUrl = new URL(url);
+ const filenameQuery = parseUrl.searchParams.get('filename');
+ if (filenameQuery) {
+ const extensionQuery = filenameQuery.split('.').pop()?.toLowerCase() || '';
+ if (!documentFileType.includes(extensionQuery)) {
+ return '';
}
+ }
- // Remove the origin(Make intranet requests directly)
- if (requestOrigin && url.startsWith(requestOrigin)) {
- url = url.replace(requestOrigin, '');
- }
+ // Remove the origin(Make intranet requests directly)
+ if (requestOrigin && url.startsWith(requestOrigin)) {
+ url = url.replace(requestOrigin, '');
}
+ }
+ return url;
+ })
+ .filter(Boolean)
+ .slice(0, maxFiles);
+ console.log(parseUrlList);
+ const readFilesResult = await Promise.all(
+ parseUrlList
+ .map(async (url) => {
// Get from buffer
const fileBuffer = await MongoRawTextBuffer.findOne({ sourceId: url }, undefined, {
...readFromSecondary
diff --git a/packages/service/core/workflow/dispatchV1/agent/classifyQuestion.ts b/packages/service/core/workflow/dispatchV1/agent/classifyQuestion.ts
index 7b60303e2b13..376e16389b1a 100644
--- a/packages/service/core/workflow/dispatchV1/agent/classifyQuestion.ts
+++ b/packages/service/core/workflow/dispatchV1/agent/classifyQuestion.ts
@@ -188,7 +188,7 @@ const toolChoice = async (props: ActionProps) => {
const response = await ai.chat.completions.create({
model: cqModel.model,
- temperature: 0,
+ temperature: 0.01,
messages: filterMessages,
tools,
tool_choice: { type: 'function', function: { name: agentFunName } }
@@ -235,7 +235,7 @@ const functionCall = async (props: ActionProps) => {
const response = await ai.chat.completions.create({
model: cqModel.model,
- temperature: 0,
+ temperature: 0.01,
messages: filterMessages,
function_call: {
name: agentFunName
diff --git a/packages/service/support/permission/controller.ts b/packages/service/support/permission/controller.ts
index 9be1f868f6ee..f96b5ffdaf25 100644
--- a/packages/service/support/permission/controller.ts
+++ b/packages/service/support/permission/controller.ts
@@ -10,6 +10,8 @@ import { MongoResourcePermission } from './schema';
import { ClientSession } from 'mongoose';
import { ParentIdType } from '@fastgpt/global/common/parentFolder/type';
import { ResourcePermissionType } from '@fastgpt/global/support/permission/type';
+import { bucketNameMap } from '@fastgpt/global/common/file/constants';
+import { addMinutes } from 'date-fns';
export const getResourcePermission = async ({
resourceType,
@@ -250,15 +252,15 @@ export const clearCookie = (res: NextApiResponse) => {
};
/* file permission */
-export const createFileToken = ({
- expiredTime = Math.floor(Date.now() / 1000) + 60 * 30,
- ...data
-}: FileTokenQuery) => {
+export const createFileToken = (data: FileTokenQuery) => {
if (!process.env.FILE_TOKEN_KEY) {
return Promise.reject('System unset FILE_TOKEN_KEY');
}
- const key = process.env.FILE_TOKEN_KEY as string;
+ const expireMinutes = bucketNameMap[data.bucketName].previewExpireMinutes;
+ const expiredTime = Math.floor(addMinutes(new Date(), expireMinutes).getTime() / 1000);
+
+ const key = (process.env.FILE_TOKEN_KEY as string) ?? 'filetoken';
const token = jwt.sign(
{
...data,
@@ -274,7 +276,7 @@ export const authFileToken = (token?: string) =>
if (!token) {
return reject(ERROR_ENUM.unAuthFile);
}
- const key = process.env.FILE_TOKEN_KEY as string;
+ const key = (process.env.FILE_TOKEN_KEY as string) ?? 'filetoken';
jwt.verify(token, key, function (err, decoded: any) {
if (err || !decoded.bucketName || !decoded?.teamId || !decoded?.tmbId || !decoded?.fileId) {
diff --git a/packages/web/components/common/Avatar/index.tsx b/packages/web/components/common/Avatar/index.tsx
index 11b69e4ae744..141f20e47161 100644
--- a/packages/web/components/common/Avatar/index.tsx
+++ b/packages/web/components/common/Avatar/index.tsx
@@ -10,7 +10,7 @@ const Avatar = ({ w = '30px', src, ...props }: ImageProps) => {
const isIcon = !!iconPaths[src as any];
return isIcon ? (
-
+
) : (
diff --git a/packages/web/components/common/Icon/constants.ts b/packages/web/components/common/Icon/constants.ts
index ffa3cf15bd07..4587b979bfbc 100644
--- a/packages/web/components/common/Icon/constants.ts
+++ b/packages/web/components/common/Icon/constants.ts
@@ -199,6 +199,7 @@ export const iconPaths = {
import('./icons/core/workflow/template/queryExtension.svg'),
'core/workflow/template/questionClassify': () =>
import('./icons/core/workflow/template/questionClassify.svg'),
+ 'core/workflow/template/readFiles': () => import('./icons/core/workflow/template/readFiles.svg'),
'core/workflow/template/reply': () => import('./icons/core/workflow/template/reply.svg'),
'core/workflow/template/runApp': () => import('./icons/core/workflow/template/runApp.svg'),
'core/workflow/template/stopTool': () => import('./icons/core/workflow/template/stopTool.svg'),
diff --git a/packages/web/components/common/Icon/icons/core/app/simpleMode/tts.svg b/packages/web/components/common/Icon/icons/core/app/simpleMode/tts.svg
index 4b9be223ca3e..4f8fae23600a 100644
--- a/packages/web/components/common/Icon/icons/core/app/simpleMode/tts.svg
+++ b/packages/web/components/common/Icon/icons/core/app/simpleMode/tts.svg
@@ -1,4 +1,4 @@
-