diff --git a/src/background/first_workflow.ts b/src/background/first_workflow.ts new file mode 100644 index 0000000..6c494c5 --- /dev/null +++ b/src/background/first_workflow.ts @@ -0,0 +1,18 @@ +import { Eko } from "@eko-ai/eko"; +import { EkoConfig } from "@eko-ai/eko/types"; +import { tools, getLLMConfig } from "@eko-ai/eko/extension"; + +export async function main() { + // Load LLM model configuration + // the current browser plugin project provides a page for configuring LLM parameters + let config = await getLLMConfig(); + if (!config && !config.apiKey) { + throw Error("Please configure apiKey"); + } + + // Initialize eko + let eko = new Eko(config as EkoConfig); + + // TODO workflow code + +} \ No newline at end of file diff --git a/src/background/index.ts b/src/background/index.ts index b442463..be91e13 100644 --- a/src/background/index.ts +++ b/src/background/index.ts @@ -1,17 +1,9 @@ -import { testTools } from "./test_tools"; -import { testWebSearchWithLLM } from "./test_llm_search"; -import { testWebSearchWithComputer } from "./test_llm_computer"; -import { testWebSearchWithWorkflow } from "./test_llm_workflow"; +import { main } from "./first_workflow"; -chrome.runtime.onMessage.addListener(async function ( - request, - sender, - sendResponse -) { +// Listen to messages from the browser extension +chrome.runtime.onMessage.addListener(async function (request, sender, sendResponse) { if (request.type == "run") { - // await testTools(); - // await testWebSearchWithLLM(); - // await testWebSearchWithComputer(); - await testWebSearchWithWorkflow(); + // Click the RUN button to execute the main function (workflow) + await main(); } -}); +}); \ No newline at end of file diff --git a/src/background/test_llm_computer.ts b/src/background/test_llm_computer.ts deleted file mode 100644 index 9ac7d7e..0000000 --- a/src/background/test_llm_computer.ts +++ /dev/null @@ -1,149 +0,0 @@ -import { ClaudeProvider, OpenaiProvider } from "@eko-ai/eko"; -import { - ExecutionContext, - Message, - LLMParameters, - ToolDefinition, - Tool, -} from "@eko-ai/eko/types"; -import { tools, getLLMConfig } from "@eko-ai/eko/extension"; - -export async function testWebSearchWithComputer() { - let config = await getLLMConfig(); - if (!config && !config.apiKey) { - throw Error("Please configure apiKey"); - } - let llmProvider = config.llm == "openai" - ? new OpenaiProvider(config.apiKey, config.modelName, config.options) - : new ClaudeProvider(config.apiKey, config.modelName, config.options); - let context = { - llmProvider, - variables: new Map(), - tools: new Map>(), - } as ExecutionContext; - - let messages: Message[] = [ - { - role: "user", - content: "Search Elon Musk information and summarize it into markdown format for export", - }, - ]; - let params: LLMParameters = { - maxTokens: 4096, - toolChoice: { - type: "auto", - }, - tools: [ - new tools.TabManagement() as ToolDefinition, - new tools.BrowserUse() as ToolDefinition, - new tools.ExportFile() as ToolDefinition, - ], - }; - let toolMap: { [key: string]: Tool } = {}; - for (let i = 0; i < params.tools.length; i++) { - let tool = params.tools[i]; - toolMap[tool.name] = tool as Tool; - } - do { - printLog(messages[messages.length - 1]); - console.log("Requesting..."); - let result = await llmProvider.generateText(messages, params); - messages.push({ - role: "assistant", - content: result.content, - }); - printLog(messages[messages.length - 1]); - let toolCalls = result.toolCalls; - if (!toolCalls || toolCalls.length == 0) { - break; - } - let user_content = []; - for (let i = 0; i < toolCalls.length; i++) { - let toolCall = toolCalls[i]; - let tool = toolMap[toolCall.name]; - let result = (await tool.execute(context, toolCall.input)) as any; - if (result.image && result.image.type) { - let content: any[] = [{ - type: "image", - source: result.image, - }] - if (result.text) { - content.push({ - type: 'text', - text: result.text - }) - } - user_content.push({ - type: "tool_result", - tool_use_id: toolCall.id, - content: content, - }); - } else { - user_content.push({ - type: "tool_result", - tool_use_id: toolCall.id, - content: [ - { - type: "text", - text: JSON.stringify(result), - }, - ], - }); - } - } - messages.push({ - role: "user", - content: user_content, - }); - } while (true); -} - -function printLog(message: Message) { - let sb = message.role + ":\n"; - let content = message.content; - if (typeof content == "string") { - sb += content; - } else { - for (let i = 0; i < content.length; i++) { - let _content = content[i] as any; - if (_content.type == "text") { - sb += _content.text + "\n"; - } else if (_content.type == "tool_use") { - sb += - "tool_use: [" + - _content.name + - "] > " + - JSON.stringify(_content.input) + - "\n"; - } else if (_content.type == "tool_result") { - let tool_content = []; - if (typeof _content.content == "string") { - tool_content.push({ type: "text", text: _content.content }); - } else { - tool_content = _content.content; - } - sb += "tool_result: "; - for (let j = 0; j < tool_content.length; j++) { - let __content = tool_content[j]; - if (__content.type == "text") { - sb += "\n" + __content.text; - } else { - let source = __content.source; - sb += - __content.type + - " > data:" + - source.media_type + - ";" + - source.type + - "," + - source.data.substring(0, 10) + - "...(" + - source.data.length + - ")"; - } - } - } - } - } - console.log(sb.trim()); -} diff --git a/src/background/test_llm_search.ts b/src/background/test_llm_search.ts deleted file mode 100644 index 3994865..0000000 --- a/src/background/test_llm_search.ts +++ /dev/null @@ -1,143 +0,0 @@ -import { ClaudeProvider, OpenaiProvider } from "@eko-ai/eko"; -import { - ExecutionContext, - Message, - LLMParameters, - ToolDefinition, - Tool, -} from "@eko-ai/eko/types"; -import { tools, getLLMConfig } from "@eko-ai/eko/extension"; - -export async function testWebSearchWithLLM() { - let config = await getLLMConfig(); - if (!config && !config.apiKey) { - throw Error("Please configure apiKey"); - } - let llmProvider = config.llm == "openai" - ? new OpenaiProvider(config.apiKey, config.modelName, config.options) - : new ClaudeProvider(config.apiKey, config.modelName, config.options); - let context = { - llmProvider, - variables: new Map(), - tools: new Map>(), - } as ExecutionContext; - - let messages: Message[] = [ - { - role: "user", - content: "Search Elon Musk information and summarize it into markdown format for export", - }, - ]; - let params: LLMParameters = { - maxTokens: 4096, - toolChoice: { - type: "auto", - }, - tools: [ - new tools.WebSearch() as ToolDefinition, - new tools.ExportFile() as ToolDefinition, - ], - }; - let toolMap: { [key: string]: Tool } = {}; - for (let i = 0; i < params.tools.length; i++) { - let tool = params.tools[i]; - toolMap[tool.name] = tool as Tool; - } - do { - printLog(messages[messages.length - 1]); - console.log("Requesting..."); - let result = await llmProvider.generateText(messages, params); - messages.push({ - role: "assistant", - content: result.content, - }); - printLog(messages[messages.length - 1]); - let toolCalls = result.toolCalls; - if (!toolCalls || toolCalls.length == 0) { - break; - } - let user_content = []; - for (let i = 0; i < toolCalls.length; i++) { - let toolCall = toolCalls[i]; - let tool = toolMap[toolCall.name]; - let result = (await tool.execute(context, toolCall.input)) as any; - if (result.image && result.image.type) { - user_content.push({ - type: "tool_result", - tool_use_id: toolCall.id, - content: [ - { - type: "image", - source: result.image, - }, - ], - }); - } else { - user_content.push({ - type: "tool_result", - tool_use_id: toolCall.id, - content: [ - { - type: "text", - text: JSON.stringify(result), - }, - ], - }); - } - } - messages.push({ - role: "user", - content: user_content, - }); - } while (true); -} - -function printLog(message: Message) { - let sb = message.role + ":\n"; - let content = message.content; - if (typeof content == "string") { - sb += content; - } else { - for (let i = 0; i < content.length; i++) { - let _content = content[i] as any; - if (_content.type == "text") { - sb += _content.text + "\n"; - } else if (_content.type == "tool_use") { - sb += - "tool_use: [" + - _content.name + - "] > " + - JSON.stringify(_content.input) + - "\n"; - } else if (_content.type == "tool_result") { - let tool_content = []; - if (typeof _content.content == "string") { - tool_content.push({ type: "text", text: _content.content }); - } else { - tool_content = _content.content; - } - sb += "tool_result: "; - for (let j = 0; j < tool_content.length; j++) { - let __content = tool_content[j]; - if (__content.type == "text") { - sb += "\n" + __content.text; - } else { - let source = __content.source; - sb += - __content.type + - " > data:" + - source.media_type + - ";" + - source.type + - "," + - source.data.substring(0, 10) + - "...(" + - source.data.length + - ")"; - } - } - } - } - } - console.log(sb.trim()); -} diff --git a/src/background/test_llm_workflow.ts b/src/background/test_llm_workflow.ts deleted file mode 100644 index 39679b4..0000000 --- a/src/background/test_llm_workflow.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { Eko, WorkflowParser } from "@eko-ai/eko"; -import { EkoConfig } from "@eko-ai/eko/types"; -import { fellou } from "@eko-ai/eko/fellou"; -import { tools, getLLMConfig } from "@eko-ai/eko/extension"; - -export async function testWebSearchWithWorkflow() { - let config = await getLLMConfig(); - fellou.computer.screenshot() - fellou.computer.mouse_move([200, 400]); - fellou.computer.type("Elon Musk"); - if (!config && !config.apiKey) { - throw Error("Please configure apiKey"); - } - - let eko = new Eko(config as EkoConfig); - - eko.registerTool(new tools.WebSearch()); - eko.registerTool(new tools.ExportFile()); - - const workflow = await eko.generateWorkflow("Search Elon Musk information and summarize it into markdown format for export"); - console.log("dsl", WorkflowParser.serialize(workflow)); - await eko.executeWorkflow(workflow); -} diff --git a/src/background/test_tools.ts b/src/background/test_tools.ts deleted file mode 100644 index 2e4fe9f..0000000 --- a/src/background/test_tools.ts +++ /dev/null @@ -1,144 +0,0 @@ -import { ElementRect, ExecutionContext, Tool } from "@eko-ai/eko/types"; -import { getLLMConfig, tools, browser, utils } from "@eko-ai/eko/extension"; -import { ClaudeProvider } from "@eko-ai/eko"; - -var context: ExecutionContext; - -async function testWebSearch() { - let webSearch = new tools.WebSearch(); - let result = await webSearch.execute(context, { query: "Elon Musk" }); - console.log("result: ", result); - return result; -} - -async function testTabManagement(commond: string) { - let tabManagement = new tools.TabManagement(); - let result = await tabManagement.execute(context, { commond }); - console.log("result: ", commond, result); - return result; -} - -async function exportFile() { - let exportFile = new tools.ExportFile(); - let result = await exportFile.execute(context, { - fileType: "csv", - content: "Name, Price\niPhone, $1000\nnotebook, $1.02", - }); - console.log("result: ", result); - return result; -} - -async function extractContent() { - let extract = new tools.ExtractContent(); - let result = await extract.execute(context, {}); - console.log("result: ", result); - return result; -} - -async function openUrl(url: string, newWindow: boolean) { - let openUrl = new tools.OpenUrl(); - let result = await openUrl.execute(context, { url, newWindow }); - console.log("result: ", result); - return result; -} - -async function findElementPosition( - task_prompt: string -): Promise { - let find_element_position = new tools.FindElementPosition(); - let result = await find_element_position.execute(context, { - task_prompt: task_prompt, - }); - console.log("find_element_position", result); - return result; -} - -async function elementClick(task_prompt: string) { - let element_click = new tools.ElementClick(); - let result = await element_click.execute(context, { - task_prompt: task_prompt, - }); - console.log("element_click", result); - return result; -} - -async function type(rect: ElementRect, text: string) { - let tabId = await utils.getCurrentTabId(); - let result = await browser.type(tabId, text, [rect.left, rect.top]); - console.log("type", result); - return result; -} - -async function screenshot() { - let screenshot = new tools.Screenshot(); - let result = await screenshot.execute(context, {}); - console.log("result: ", result); - return result; -} - -async function computerWeb( - action: string, - index?: number, - text?: string -) { - let tabId = await utils.getCurrentTabId(); - context.variables.set("tabId", tabId); - let computer = new tools.BrowserUse(); - let result = await computer.execute(context, { action, index, text }); - console.log("result: ", result); - return result; -} - -async function computer() { - await openUrl("https://www.baidu.com", true); - let result = await computerWeb("screenshot_extract_element"); - let elements = result.text.split("\n"); - let inputIdx; - for (let i = 0; i < elements.length; i++) { - let str = elements[i]; - if (str.indexOf('id="kw"') > -1) { - inputIdx = str.split(":")[0].replace("[", "").replace("]", ""); - } - } - await computerWeb("input_text", +inputIdx, "Elon Musk"); - result = await computerWeb("screenshot_extract_element"); - elements = result.text.split("\n"); - let butIdx; - for (let i = 0; i < elements.length; i++) { - let str = elements[i]; - if (str.indexOf('id="su"') > -1) { - butIdx = str.split(":")[0].replace("[", "").replace("]", ""); - } - } - await computerWeb("click", +butIdx); -} - -async function test_task_prompt() { - // open new tab - await openUrl("https://www.baidu.com", true); - let rect = await findElementPosition("Find the search input box"); - if (rect) { - // input -> text - await type(rect, "Elon Musk"); - await utils.sleep(500); - // click the search button - await elementClick("click the search button"); - } -} - -export async function testTools() { - let apiKey = (await getLLMConfig()).apiKey; - context = { - llmProvider: new ClaudeProvider(apiKey), - variables: new Map(), - tools: new Map>(), - }; - // test - // await testWebSearch(); - // await testTabManagement("tab_all"); - // await exportFile(); - // await extractContent(); - // await openUrl("https://www.google.com", true); - // await screenshot(); - await computer(); -} diff --git a/src/popup/index.tsx b/src/popup/index.tsx index 417acc5..156744a 100644 --- a/src/popup/index.tsx +++ b/src/popup/index.tsx @@ -1,7 +1,6 @@ import { createRoot } from "react-dom/client"; import React from "react"; import { Button } from "antd"; -import { CaretRightOutlined } from "@ant-design/icons"; const AppRun = () => { const handleClick = () => { @@ -19,7 +18,6 @@ const AppRun = () => {

Click to test