fix: chat variable update (#3156)

* perf: file encoding * fix: chat variable update
labring · Nov 14, 2024 · 710fa37 · 710fa37
1 parent e22031c
commit 710fa37
Show file tree

Hide file tree

Showing 7 changed files with 23 additions and 11 deletions.
diff --git a/packages/service/common/file/gridfs/controller.ts b/packages/service/common/file/gridfs/controller.ts
@@ -36,6 +36,7 @@ export async function uploadFile({
   path,
   filename,
   contentType,
+  encoding,
   metadata = {}
 }: {
   bucketName: `${BucketNameEnum}`;
@@ -44,6 +45,7 @@ export async function uploadFile({
   path: string;
   filename: string;
   contentType?: string;
+  encoding: string;
   metadata?: Record<string, any>;
 }) {
   if (!path) return Promise.reject(`filePath is empty`);
@@ -52,7 +54,7 @@ export async function uploadFile({
   const stats = await fsp.stat(path);
   if (!stats.isFile()) return Promise.reject(`${path} is not a file`);
 
-  const { stream: readStream, encoding } = await stream2Encoding(fs.createReadStream(path));
+  const readStream = fs.createReadStream(path);
 
   // Add default metadata
   metadata.teamId = teamId;

diff --git a/packages/service/common/file/read/utils.ts b/packages/service/common/file/read/utils.ts
@@ -14,6 +14,7 @@ import { addHours } from 'date-fns';
 export type readRawTextByLocalFileParams = {
   teamId: string;
   path: string;
+  encoding: string;
   metadata?: Record<string, any>;
 };
 export const readRawTextByLocalFile = async (params: readRawTextByLocalFileParams) => {
@@ -22,13 +23,12 @@ export const readRawTextByLocalFile = async (params: readRawTextByLocalFileParam
   const extension = path?.split('.')?.pop()?.toLowerCase() || '';
 
   const buffer = fs.readFileSync(path);
-  const encoding = detectFileEncoding(buffer);
 
   const { rawText } = await readRawContentByFileBuffer({
     extension,
     isQAImport: false,
     teamId: params.teamId,
-    encoding,
+    encoding: params.encoding,
     buffer,
     metadata: params.metadata
   });
@@ -53,6 +53,7 @@ export const readRawContentByFileBuffer = async ({
   encoding: string;
   metadata?: Record<string, any>;
 }) => {
+  // Custom read file service
   const customReadfileUrl = process.env.CUSTOM_READ_FILE_URL;
   const customReadFileExtension = process.env.CUSTOM_READ_FILE_EXTENSION || '';
   const ocrParse = process.env.CUSTOM_READ_FILE_OCR || 'false';

diff --git a/packages/service/worker/readFile/extension/rawText.ts b/packages/service/worker/readFile/extension/rawText.ts
@@ -18,9 +18,17 @@ const rawEncodingList = [
 
 // 加载源文件内容
 export const readFileRawText = ({ buffer, encoding }: ReadRawTextByBuffer): ReadFileResponse => {
-  const content = rawEncodingList.includes(encoding)
-    ? buffer.toString(encoding as BufferEncoding)
-    : iconv.decode(buffer, 'gbk');
+  const content = (() => {
+    try {
+      if (rawEncodingList.includes(encoding)) {
+        return buffer.toString(encoding as BufferEncoding);
+      }
+
+      return iconv.decode(buffer, encoding);
+    } catch (error) {
+      return buffer.toString('utf-8');
+    }
+  })();
 
   return {
     rawText: content

diff --git a/projects/app/src/components/core/chat/ChatContainer/ChatBox/index.tsx b/projects/app/src/components/core/chat/ChatContainer/ChatBox/index.tsx
@@ -323,7 +323,7 @@ const ChatBox = (
               })
             };
           } else if (event === SseResponseEventEnum.updateVariables && variables) {
-            variablesForm.reset(variables);
+            variablesForm.setValue('variables', variables);
           } else if (event === SseResponseEventEnum.interactive) {
             const val: AIChatItemValueItemType = {
               type: ChatItemValueTypeEnum.interactive,
@@ -408,7 +408,7 @@ const ChatBox = (
       isInteractivePrompt = false
     }) => {
       variablesForm.handleSubmit(
-        async ({ variables }) => {
+        async ({ variables = {} }) => {
           if (!onStartChat) return;
           if (isChatting) {
             toast({
@@ -435,7 +435,7 @@ const ChatBox = (
           // Only declared variables are kept
           const requestVariables: Record<string, any> = {};
           allVariableList?.forEach((item) => {
-            requestVariables[item.key] = variables[item.key] || '';
+            requestVariables[item.key] = variables[item.key];
           });
 
           const responseChatId = getNanoid(24);

diff --git a/projects/app/src/pages/api/common/file/upload.ts b/projects/app/src/pages/api/common/file/upload.ts
@@ -51,6 +51,7 @@ async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
       path: file.path,
       filename: file.originalname,
       contentType: file.mimetype,
+      encoding: file.encoding,
       metadata: metadata
     });
 

diff --git a/projects/app/src/pages/api/core/dataset/collection/create/localFile.ts b/projects/app/src/pages/api/core/dataset/collection/create/localFile.ts
@@ -67,6 +67,7 @@ async function handler(req: NextApiRequest, res: NextApiResponse<any>): CreateCo
     const { rawText } = await readRawTextByLocalFile({
       teamId,
       path: file.path,
+      encoding: file.encoding,
       metadata: {
         ...fileMetadata,
         relatedId: relatedImgId
@@ -81,6 +82,7 @@ async function handler(req: NextApiRequest, res: NextApiResponse<any>): CreateCo
       path: file.path,
       filename: file.originalname,
       contentType: file.mimetype,
+      encoding: file.encoding,
       metadata: fileMetadata
     });
 

diff --git a/projects/app/src/pages/dataset/detail/components/Import/Context.tsx b/projects/app/src/pages/dataset/detail/components/Import/Context.tsx
@@ -254,8 +254,6 @@ const DatasetImportContextProvider = ({ children }: { children: React.ReactNode
               icon={<MyIcon name={'common/backFill'} w={'14px'} />}
               aria-label={''}
               size={'smSquare'}
-              w={'26px'}
-              h={'26px'}
               borderRadius={'50%'}
               variant={'whiteBase'}
               mr={2}