From 8386f707cd39b50b76ba6e607518e1f286984fd2 Mon Sep 17 00:00:00 2001 From: Archer <545436317@qq.com> Date: Wed, 15 May 2024 16:17:43 +0800 Subject: [PATCH] Perf workflow (#1492) * perf: handle edge check * search model * feat: plugin input can render all input; fix: plugin default value * fix ts * feat: plugin input support required --- .vscode/i18n-ally-custom-framework.yml | 1 + .../content/docs/development/openapi/share.md | 2 +- .../content/docs/development/upgrading/481.md | 7 +- packages/global/core/chat/type.d.ts | 2 +- packages/global/core/workflow/api.d.ts | 2 +- .../workflow/template/system/datasetSearch.ts | 1 - packages/global/core/workflow/type/index.d.ts | 2 +- packages/global/core/workflow/utils.ts | 12 +- .../core/workflow/dispatch/dataset/search.ts | 5 +- .../service/core/workflow/dispatch/index.ts | 2 +- .../common/Input/NumberInput/index.tsx | 36 +++++ .../common/MyDrawer/MyRightDrawer.tsx | 2 +- projects/app/i18n/en/dataset.json | 2 +- projects/app/i18n/en/workflow.json | 3 + projects/app/i18n/zh/dataset.json | 2 +- projects/app/i18n/zh/workflow.json | 3 + .../core/app/DatasetSelectModal.tsx | 11 +- .../core/workflow/Flow/hooks/useDebug.tsx | 150 ++++++++++-------- .../components/core/workflow/Flow/index.tsx | 2 +- .../workflow/Flow/nodes/NodePluginInput.tsx | 22 +-- .../Flow/nodes/render/FieldEditModal.tsx | 39 ++++- .../workflow/Flow/nodes/render/NodeCard.tsx | 1 - .../Flow/nodes/render/RenderInput/Label.tsx | 8 +- .../Flow/nodes/render/VariableTable.tsx | 1 - .../src/components/core/workflow/context.tsx | 2 - .../app/src/components/core/workflow/utils.ts | 15 +- .../app/detail/components/FlowEdit/Header.tsx | 2 +- projects/app/src/pages/app/detail/index.tsx | 5 +- projects/app/src/pages/plugin/edit/index.tsx | 2 +- .../pages/plugin/list/component/EditModal.tsx | 5 +- projects/app/src/types/i18n.d.ts | 2 + projects/app/src/web/context/I18n.tsx | 5 +- projects/app/src/web/core/app/templates.ts | 37 +++-- projects/app/src/web/core/app/utils.ts | 12 +- .../web/core/workflow/constants/dataType.ts | 4 +- projects/app/src/web/core/workflow/utils.ts | 7 +- 36 files changed, 256 insertions(+), 160 deletions(-) create mode 100644 packages/web/components/common/Input/NumberInput/index.tsx create mode 100644 projects/app/i18n/en/workflow.json create mode 100644 projects/app/i18n/zh/workflow.json diff --git a/.vscode/i18n-ally-custom-framework.yml b/.vscode/i18n-ally-custom-framework.yml index 5582ce8dc19e..ad841db50861 100644 --- a/.vscode/i18n-ally-custom-framework.yml +++ b/.vscode/i18n-ally-custom-framework.yml @@ -23,6 +23,7 @@ usageMatchRegex: - "[^\\w\\d]datasetT\\(['\"`]({key})['\"`]" - "[^\\w\\d]fileT\\(['\"`]({key})['\"`]" - "[^\\w\\d]publishT\\(['\"`]({key})['\"`]" + - "[^\\w\\d]workflowT\\(['\"`]({key})['\"`]" # A RegEx to set a custom scope range. This scope will be used as a prefix when detecting keys # and works like how the i18next framework identifies the namespace scope from the diff --git a/docSite/content/docs/development/openapi/share.md b/docSite/content/docs/development/openapi/share.md index cc865b616b2e..f65ddcd0d8d9 100644 --- a/docSite/content/docs/development/openapi/share.md +++ b/docSite/content/docs/development/openapi/share.md @@ -247,7 +247,7 @@ curl --location --request POST '{{host}}/shareAuth/finish' \ ```ts type ResponseType = { - moduleType: `${FlowNodeTypeEnum}`; // 模块类型 + moduleType: FlowNodeTypeEnum; // 模块类型 moduleName: string; // 模块名 moduleLogo?: string; // logo runningTime?: number; // 运行时间 diff --git a/docSite/content/docs/development/upgrading/481.md b/docSite/content/docs/development/upgrading/481.md index 21a391593079..bd069c3a8055 100644 --- a/docSite/content/docs/development/upgrading/481.md +++ b/docSite/content/docs/development/upgrading/481.md @@ -35,5 +35,8 @@ curl --location --request POST 'https://{{host}}/api/admin/clearInvalidData' \ ## V4.8.1 更新说明 1. 新增 - 知识库重新选择向量模型重建 -2. 修复 - 工作流删除节点的动态输入和输出时候，没有正确的删除连接线，导致可能出现逻辑异常。 -3. 修复 - 定时器清理脏数据任务 \ No newline at end of file +2. 新增 - 工作流节点版本变更提示，并可以同步最新版本。 +3. 优化 - 插件输入的 debug 模式，支持全量参数输入渲染。 +4. 修复 - 插件输入默认值被清空问题。 +5. 修复 - 工作流删除节点的动态输入和输出时候，没有正确的删除连接线，导致可能出现逻辑异常。 +6. 修复 - 定时器清理脏数据任务 \ No newline at end of file diff --git a/packages/global/core/chat/type.d.ts b/packages/global/core/chat/type.d.ts index 59b70683fbc2..f8e5f6d7de7d 100644 --- a/packages/global/core/chat/type.d.ts +++ b/packages/global/core/chat/type.d.ts @@ -139,7 +139,7 @@ export type ChatHistoryItemType = HistoryItemType & { /* ------- response data ------------ */ export type ChatHistoryItemResType = DispatchNodeResponseType & { nodeId: string; - moduleType: `${FlowNodeTypeEnum}`; + moduleType: FlowNodeTypeEnum; moduleName: string; }; diff --git a/packages/global/core/workflow/api.d.ts b/packages/global/core/workflow/api.d.ts index be573e22caa4..30524fd5f89f 100644 --- a/packages/global/core/workflow/api.d.ts +++ b/packages/global/core/workflow/api.d.ts @@ -1,7 +1,7 @@ import { VectorModelItemType } from '../ai/model.d'; import { NodeInputKeyEnum } from './constants'; -export type SelectedDatasetType = { datasetId: string; vectorModel: VectorModelItemType }[]; +export type SelectedDatasetType = { datasetId: string }[]; export type HttpBodyType> = { [NodeInputKeyEnum.addInputParam]: Record; diff --git a/packages/global/core/workflow/template/system/datasetSearch.ts b/packages/global/core/workflow/template/system/datasetSearch.ts index b8cf918e2ba8..07cfd5682e59 100644 --- a/packages/global/core/workflow/template/system/datasetSearch.ts +++ b/packages/global/core/workflow/template/system/datasetSearch.ts @@ -36,7 +36,6 @@ export const DatasetSearchModule: FlowNodeTemplateType = { label: 'core.module.input.label.Select dataset', value: [], valueType: WorkflowIOValueTypeEnum.selectDataset, - list: [], required: true }, { diff --git a/packages/global/core/workflow/type/index.d.ts b/packages/global/core/workflow/type/index.d.ts index f24fd07ebd81..2862a6269e8d 100644 --- a/packages/global/core/workflow/type/index.d.ts +++ b/packages/global/core/workflow/type/index.d.ts @@ -22,7 +22,7 @@ import { RuntimeEdgeItemType, StoreEdgeItemType } from './edge'; import { NextApiResponse } from 'next'; export type FlowNodeCommonType = { - flowNodeType: `${FlowNodeTypeEnum}`; // render node card + flowNodeType: FlowNodeTypeEnum; // render node card avatar?: string; name: string; diff --git a/packages/global/core/workflow/utils.ts b/packages/global/core/workflow/utils.ts index 7a672d41f824..2e72f7d24bc9 100644 --- a/packages/global/core/workflow/utils.ts +++ b/packages/global/core/workflow/utils.ts @@ -1,4 +1,4 @@ -import { FlowNodeOutputTypeEnum, FlowNodeTypeEnum } from './node/constant'; +import { FlowNodeInputTypeEnum, FlowNodeOutputTypeEnum, FlowNodeTypeEnum } from './node/constant'; import { WorkflowIOValueTypeEnum, NodeInputKeyEnum, @@ -22,15 +22,9 @@ export const getHandleId = (nodeId: string, type: 'source' | 'target', key: stri }; export const checkInputIsReference = (input: FlowNodeInputItemType) => { - const value = input.value; - if ( - Array.isArray(value) && - value.length === 2 && - typeof value[0] === 'string' && - typeof value[1] === 'string' - ) { + if (input.renderTypeList?.[input?.selectedTypeIndex || 0] === FlowNodeInputTypeEnum.reference) return true; - } + return false; }; diff --git a/packages/service/core/workflow/dispatch/dataset/search.ts b/packages/service/core/workflow/dispatch/dataset/search.ts index bd8368d58fa7..0db4dd147f6b 100644 --- a/packages/service/core/workflow/dispatch/dataset/search.ts +++ b/packages/service/core/workflow/dispatch/dataset/search.ts @@ -15,6 +15,7 @@ import { getHistories } from '../utils'; import { datasetSearchQueryExtension } from '../../../dataset/search/utils'; import { ChatNodeUsageType } from '@fastgpt/global/support/wallet/bill/type'; import { checkTeamReRankPermission } from '../../../../support/permission/teamLimit'; +import { MongoDataset } from '../../../dataset/schema'; type DatasetSearchProps = ModuleDispatchProps<{ [NodeInputKeyEnum.datasetSelectList]: SelectedDatasetType; @@ -79,7 +80,9 @@ export async function dispatchDatasetSearch( // console.log(concatQueries, rewriteQuery, aiExtensionResult); // get vector - const vectorModel = getVectorModel(datasets[0]?.vectorModel?.model); + const vectorModel = getVectorModel( + (await MongoDataset.findById(datasets[0].datasetId, 'vectorModel').lean())?.vectorModel + ); // start search const { diff --git a/packages/service/core/workflow/dispatch/index.ts b/packages/service/core/workflow/dispatch/index.ts index bb4a78385cec..b40a13c2dc7d 100644 --- a/packages/service/core/workflow/dispatch/index.ts +++ b/packages/service/core/workflow/dispatch/index.ts @@ -45,7 +45,7 @@ import { getReferenceVariableValue } from '@fastgpt/global/core/workflow/runtime import { dispatchSystemConfig } from './init/systemConfig'; import { dispatchUpdateVariable } from './tools/runUpdateVar'; -const callbackMap: Record<`${FlowNodeTypeEnum}`, Function> = { +const callbackMap: Record = { [FlowNodeTypeEnum.workflowStart]: dispatchWorkflowStart, [FlowNodeTypeEnum.answerNode]: dispatchAnswer, [FlowNodeTypeEnum.chatNode]: dispatchChatCompletion, diff --git a/packages/web/components/common/Input/NumberInput/index.tsx b/packages/web/components/common/Input/NumberInput/index.tsx new file mode 100644 index 000000000000..8f8909fce5c4 --- /dev/null +++ b/packages/web/components/common/Input/NumberInput/index.tsx @@ -0,0 +1,36 @@ +import { + NumberInput, + NumberIncrementStepper, + NumberInputField, + NumberInputStepper, + NumberDecrementStepper, + NumberInputProps +} from '@chakra-ui/react'; +import React from 'react'; + +type Props = Omit & { + onChange: (e: number | '') => any; +}; + +const MyNumberInput = (props: Props) => { + return ( + { + if (isNaN(Number(e))) { + props?.onChange(''); + } else { + props?.onChange(Number(e)); + } + }} + > + + + + + + + ); +}; + +export default MyNumberInput; diff --git a/packages/web/components/common/MyDrawer/MyRightDrawer.tsx b/packages/web/components/common/MyDrawer/MyRightDrawer.tsx index 6b8ab4de445c..9f6130ac5f00 100644 --- a/packages/web/components/common/MyDrawer/MyRightDrawer.tsx +++ b/packages/web/components/common/MyDrawer/MyRightDrawer.tsx @@ -67,7 +67,7 @@ const MyRightDrawer = ({ - + {children} diff --git a/projects/app/i18n/en/dataset.json b/projects/app/i18n/en/dataset.json index 510955b1c4c1..cce429907467 100644 --- a/projects/app/i18n/en/dataset.json +++ b/projects/app/i18n/en/dataset.json @@ -1,7 +1,7 @@ { "Common Dataset": "Common dataset", "Common Dataset Desc": "Can be built by importing files, web links, or manual entry", - "Confirm to rebuild embedding tip": "Are you sure to switch the knowledge base index? Switching index is a very heavy operation that requires re-indexing all the data in your knowledge base, which may take a long time. Please ensure that the remaining points in your account are sufficient.", + "Confirm to rebuild embedding tip": "Are you sure to switch the knowledge base index?\nSwitching index is a very heavy operation that requires re-indexing all the data in your knowledge base, which may take a long time. Please ensure that the remaining points in your account are sufficient.\n\nIn addition, you need to be careful to modify the applications that select this knowledge base to avoid mixing them with other index model knowledge bases.", "External file": "External file", "External file Dataset Desc": "You can import files from an external file library to build a knowledge base. Files are not stored twice", "External id": "File id", diff --git a/projects/app/i18n/en/workflow.json b/projects/app/i18n/en/workflow.json new file mode 100644 index 000000000000..41187fc5be99 --- /dev/null +++ b/projects/app/i18n/en/workflow.json @@ -0,0 +1,3 @@ +{ + "Field required": "Required" +} diff --git a/projects/app/i18n/zh/dataset.json b/projects/app/i18n/zh/dataset.json index d07a16bbe4ea..cf350ba27346 100644 --- a/projects/app/i18n/zh/dataset.json +++ b/projects/app/i18n/zh/dataset.json @@ -1,7 +1,7 @@ { "Common Dataset": "通用知识库", "Common Dataset Desc": "可通过导入文件、网页链接或手动录入形式构建知识库", - "Confirm to rebuild embedding tip": "确认为知识库切换索引？\n切换索引是一个非常重量的操作，需要对您知识库内所有数据进行重新索引，时间可能较长，请确保账号内剩余积分充足。", + "Confirm to rebuild embedding tip": "确认为知识库切换索引？\n切换索引是一个非常重量的操作，需要对您知识库内所有数据进行重新索引，时间可能较长，请确保账号内剩余积分充足。\n\n此外，你还需要注意修改选择该知识库的应用，避免它们与其他索引模型知识库混用。", "External File": "外部文件库", "External file Dataset Desc": "可以从外部文件库导入文件构建知识库，文件不会进行二次存储", "External id": "文件阅读ID", diff --git a/projects/app/i18n/zh/workflow.json b/projects/app/i18n/zh/workflow.json new file mode 100644 index 000000000000..ef72acbe2226 --- /dev/null +++ b/projects/app/i18n/zh/workflow.json @@ -0,0 +1,3 @@ +{ + "Field required": "必填" +} diff --git a/projects/app/src/components/core/app/DatasetSelectModal.tsx b/projects/app/src/components/core/app/DatasetSelectModal.tsx index 983b4f8cf347..ebe00353b150 100644 --- a/projects/app/src/components/core/app/DatasetSelectModal.tsx +++ b/projects/app/src/components/core/app/DatasetSelectModal.tsx @@ -96,7 +96,7 @@ export const DatasetSelectModal = ({ _hover={{ color: 'red.500' }} onClick={() => { setSelectedDatasets((state) => - state.filter((kb) => kb.datasetId !== item._id) + state.filter((dataset) => dataset.datasetId !== item._id) ); }} /> @@ -141,7 +141,9 @@ export const DatasetSelectModal = ({ if (item.type === DatasetTypeEnum.folder) { setParentId(item._id); } else { - const vectorModel = selectedDatasets[0]?.vectorModel?.model; + const vectorModel = datasets.find( + (dataset) => dataset._id === selectedDatasets[0]?.datasetId + )?.vectorModel?.model; if (vectorModel && vectorModel !== item.vectorModel.model) { return toast({ @@ -149,10 +151,7 @@ export const DatasetSelectModal = ({ title: t('dataset.Select Dataset Tips') }); } - setSelectedDatasets((state) => [ - ...state, - { datasetId: item._id, vectorModel: item.vectorModel } - ]); + setSelectedDatasets((state) => [...state, { datasetId: item._id }]); } }} > diff --git a/projects/app/src/components/core/workflow/Flow/hooks/useDebug.tsx b/projects/app/src/components/core/workflow/Flow/hooks/useDebug.tsx index 14afa8879fa6..276a45d4d9a0 100644 --- a/projects/app/src/components/core/workflow/Flow/hooks/useDebug.tsx +++ b/projects/app/src/components/core/workflow/Flow/hooks/useDebug.tsx @@ -26,6 +26,8 @@ import { WorkflowIOValueTypeEnum } from '@fastgpt/global/core/workflow/constants import { checkInputIsReference } from '@fastgpt/global/core/workflow/utils'; import { useContextSelector } from 'use-context-selector'; import { WorkflowContext, getWorkflowStore } from '../../context'; +import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip'; +import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant'; const MyRightDrawer = dynamic( () => import('@fastgpt/web/components/common/MyDrawer/MyRightDrawer') @@ -109,15 +111,23 @@ export const useDebug = () => { const runtimeNode = runtimeNodes.find((node) => node.nodeId === runtimeNodeId); if (!runtimeNode) return <>; - const referenceInputs = runtimeNode.inputs.filter((input) => { + const renderInputs = runtimeNode.inputs.filter((input) => { + if (runtimeNode.flowNodeType === FlowNodeTypeEnum.pluginInput) return true; if (checkInputIsReference(input)) return true; if (input.required && !input.value) return true; }); const { register, getValues, setValue, handleSubmit } = useForm>({ - defaultValues: referenceInputs.reduce((acc, input) => { - //@ts-ignore - acc[input.key] = undefined; + defaultValues: renderInputs.reduce((acc: Record, input) => { + const isReference = checkInputIsReference(input); + if (isReference) { + acc[input.key] = undefined; + } else if (typeof input.value === 'object') { + acc[input.key] = JSON.stringify(input.value, null, 2); + } else { + acc[input.key] = input.value; + } + return acc; }, {}) }); @@ -153,73 +163,81 @@ export const useDebug = () => { iconSrc="core/workflow/debugBlue" title={t('core.workflow.Debug Node')} maxW={['90vw', '35vw']} + px={0} > - - {referenceInputs.map((input) => { + + {renderInputs.map((input) => { const required = input.required || false; - return ( - - - {required && ( - - * - - )} - {t(input.debugLabel || input.label)} - - {(() => { - if (input.valueType === WorkflowIOValueTypeEnum.string) { - return ( -