Added integration tests

metagpt/utils/parse_html.py

@@ -4,7 +4,6 @@
 from typing import Generator, Optional
 from urllib.parse import urljoin, urlparse
 
-import htmlmin
 from bs4 import BeautifulSoup
 from pydantic import BaseModel, PrivateAttr
 
@@ -73,4 +72,4 @@ def _get_soup(page: str):
 
 def simplify_html(html: str, url: str, keep_links: bool = False):
     html = WebPage(inner_text="", html=html, url=url).get_slim_soup(keep_links).decode()
-    return htmlmin.minify(html, remove_comments=True, remove_empty_space=True)
+    return html

pyproject.toml

@@ -0,0 +1,93 @@
+[project]
+name = "metagpt"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.12,<4.0"
+dependencies = [
+    "aioboto3>=7.0.0",
+    "aiofiles>=25.1.0",
+    "aiohttp>=3.13.2",
+    "anthropic>=0.75.0",
+    "anytree>=2.13.0",
+    "azure-cognitiveservices-speech>=1.47.0",
+    "beautifulsoup4>=4.14.2",
+    "boto3>=1.41.5",
+    "channels>=4.3.2",
+    "chromadb>=1.3.5",
+    "curl-cffi>=0.13.0",
+    "dashscope>=1.25.2",
+    "faiss-cpu>=1.13.0",
+    "fire>=0.7.1",
+    "fsspec>=2025.10.0",
+    "gitignore-parser>=0.1.13",
+    "gitpython>=3.1.45",
+    "google-generativeai>=0.8.5",
+    "grep-ast>=0.9.0",
+    "grpcio>=1.76.0",
+    "grpcio-status>=1.71.2",
+    "grpcio-tools>=1.71.2",
+    "gymnasium>=1.2.2",
+    "httplib2>=0.31.0",
+    "httpx>=0.28.1",
+    "imap-tools>=1.11.0",
+    "jieba>=0.42.1",
+    "lancedb>=0.25.3",
+    "libcst>=1.8.6",
+    "llama-index>=0.12.42",
+    "llama-index-retrievers-bm25>=0.5.2",
+    "llama-index-vector-stores-chroma>=0.4.2",
+    "llama-index-vector-stores-elasticsearch>=0.4.3",
+    "llama-index-vector-stores-faiss>=0.4.0",
+    "loguru>=0.7.3",
+    "matplotlib>=3.10.7",
+    "meilisearch>=0.38.0",
+    "nbclient>=0.10.2",
+    "nbformat>=5.10.4",
+    "networkx>=3.6",
+    "numpy>=2.3.5",
+    "openai>=1.109.1",
+    "openpyxl>=3.1.5",
+    "pandas>=2.3.3",
+    "pillow>=12.0.0",
+    "playwright>=1.26",
+    "pydantic>=2.10.6",
+    "pygithub>=2.8.1",
+    "pylint>=4.0.3",
+    "python-docx>=1.2.0",
+    "pyyaml>=6.0.3",
+    "qdrant-client>=1.16.1",
+    "qianfan>=0.4.12.3",
+    "rank-bm25>=0.2.2",
+    "redis>=7.1.0",
+    "rich>=14.2.0",
+    "scikit-learn>=1.7.2",
+    "semantic-kernel>=1.23.0",
+    "socksio>=1.0.0",
+    "spark-ai-python>=0.4.5",
+    "ta>=0.11.0",
+    "tenacity>=8.5.0",
+    "tiktoken>=0.12.0",
+    "tqdm>=4.67.1",
+    "tree-sitter>=0.25.2",
+    "tree-sitter-languages>=1.10.2",
+    "tree-sitter-python>=0.25.0",
+    "typer>=0.20.0",
+    "typing-extensions>=4.15.0",
+    "typing-inspect>=0.9.0",
+    "unidiff>=0.7.5",
+    "volcengine-python-sdk[ark]>=4.0.36",
+    "websocket-client>=1.9.0",
+    "websockets>=10.0,<12.0",
+    "zhipuai>=2.1.5.20250825",
+]
+
+[dependency-groups]
+dev = [
+    "ipykernel>=7.1.0",
+    "ipywidgets>=8.1.8",
+    "pytest>=9.0.1",
+    "pytest-asyncio>=1.3.0",
+    "pytest-cov>=7.0.0",
+    "pytest-mock>=3.15.1",
+]

requirements.txt

@@ -1,90 +1,70 @@
-aiohttp==3.8.6
-#azure_storage==0.37.0
-channels==4.0.0
-# Django==4.1.5
-# docx==0.2.4
-#faiss==1.5.3
-faiss_cpu==1.7.4
-fire==0.4.0
-typer==0.9.0
-# godot==0.1.1
-# google_api_python_client==2.93.0  # Used by search_engine.py
-lancedb==0.4.0
-loguru==0.6.0
-meilisearch==0.21.0
-numpy~=1.26.4
-openai~=1.64.0
-openpyxl~=3.1.5
-beautifulsoup4==4.12.3
-pandas==2.1.1
-pydantic>=2.5.3
-#pygame==2.1.3
-# pymilvus==2.4.6
-# pytest==7.2.2 # test extras require
-python_docx==0.8.11
-PyYAML==6.0.1
-# sentence_transformers==2.2.2
-setuptools>=65.6.3
-tenacity==8.2.3
-tiktoken==0.7.0
-tqdm==4.66.2
-#unstructured[local-inference]
-# selenium>4
-# webdriver_manager<3.9
-anthropic==0.47.2
-typing-inspect==0.8.0
-libcst==1.0.1
-qdrant-client==1.7.0
-grpcio~=1.67.0
-grpcio-tools~=1.62.3
-grpcio-status~=1.62.3
-# pytest-mock==3.11.1  # test extras require
-# open-interpreter==0.1.7; python_version>"3.9" # Conflict with openai 1.x
-ta==0.10.2
-semantic-kernel==0.4.3.dev0
-wrapt==1.15.0
-#aiohttp_jinja2
-# azure-cognitiveservices-speech~=1.31.0 # Used by metagpt/tools/azure_tts.py
-#aioboto3~=12.4.0  # Used by metagpt/utils/s3.py
-redis~=5.0.0 # Used by metagpt/utils/redis.py
-curl-cffi~=0.7.0
-httplib2~=0.22.0
-websocket-client~=1.8.0
-aiofiles==23.2.1
-gitpython==3.1.40
-zhipuai~=2.1.5
-rich==13.6.0
-nbclient==0.9.0
-nbformat==5.9.2
-ipython==8.17.2
-ipykernel==6.27.1
-scikit_learn==1.3.2
-typing-extensions==4.11.0
-socksio~=1.0.0
-gitignore-parser==0.1.9
-# connexion[uvicorn]~=3.0.5 # Used by metagpt/tools/openapi_v3_hello.py
+aiohttp
+channels
+faiss_cpu
+fire
+typer
+lancedb
+loguru
+meilisearch
+numpy
+openai
+openpyxl
+beautifulsoup4
+pandas
+pydantic
+python_docx
+PyYAML
+setuptools
+tenacity
+tiktoken
+tqdm
+anthropic
+typing-inspect
+libcst
+qdrant-client
+grpcio
+grpcio-tools
+grpcio-status
+ta
+semantic-kernel
+wrapt
+redis
+curl-cffi
+httplib2
+websocket-client
+aiofiles
+gitpython
+zhipuai
+rich
+nbclient
+nbformat
+ipython
+ipykernel
+scikit_learn
+typing-extensions
+socksio
+gitignore-parser
 websockets>=10.0,<12.0
-networkx~=3.2.1
-google-generativeai==0.4.1
-playwright>=1.26  # used at metagpt/tools/libs/web_scraping.py
+networkx
+google-generativeai
+playwright>=1.26 
 anytree
-ipywidgets==8.1.1
+ipywidgets
 Pillow
-imap_tools==1.5.0  # Used by metagpt/tools/libs/email_login.py
-pylint~=3.0.3
-pygithub~=2.3
-htmlmin
+imap_tools
+pylint
+pygithub
 fsspec
-grep-ast~=0.3.3  # linter
-unidiff==0.7.5 # used at metagpt/tools/libs/cr.py
-qianfan~=0.4.4
-dashscope~=1.19.3
-rank-bm25==0.2.2  # for tool recommendation
-jieba==0.42.1  # for tool recommendation
-volcengine-python-sdk[ark]~=1.0.94 # Solution for installation error in Windows: https://github.com/volcengine/volcengine-python-sdk/issues/5
-gymnasium==0.29.1
-boto3~=1.34.69
-spark_ai_python~=0.3.30
-tree_sitter~=0.23.2
-tree_sitter_python~=0.23.2
-httpx==0.28.1
+grep-ast
+unidiff
+qianfan
+dashscope
+rank-bm25
+jieba
+volcengine-python-sdk[ark]
+gymnasium
+boto3
+spark_ai_python
+tree_sitter
+tree_sitter_python
+httpx