diff --git a/.env.backup b/.env.backup new file mode 100644 index 0000000..0160dc7 --- /dev/null +++ b/.env.backup @@ -0,0 +1,26 @@ +# Development Environment Configuration + +# API Keys (Development) +OPENAI_API_KEY=your_dev_openai_key +ANTHROPIC_API_KEY=your_dev_anthropic_key +GOOGLE_API_KEY=your_dev_google_key + +# Database Configuration (Development) +DB_HOST=localhost +DB_PORT=5432 +DB_NAME=mcp_dev +DB_USER=dev_user +DB_PASSWORD=dev_password + +# Development Settings +DEBUG=true +LOG_LEVEL=DEBUG +ENVIRONMENT=development + +# Development Tools Configuration +PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=0 +PYTEST_ADDOPTS="--color=yes" + +# Development URLs +API_BASE_URL=http://localhost:8000 +FRONTEND_URL=http://localhost:3000 \ No newline at end of file diff --git a/.gitignore b/.gitignore index 33c71ca..7c6fd9f 100644 --- a/.gitignore +++ b/.gitignore @@ -129,6 +129,7 @@ celerybeat.pid # Environments .env +.env.dev .venv env/ venv/ diff --git a/dev/scripts/__init__.py b/dev/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dev/tests/__init__.py b/dev/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dev/tests/mcp/__init__.py b/dev/tests/mcp/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dev/tests/mcp/unit/__init__.py b/dev/tests/mcp/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dev/tests/mcp/unit/test_dev_tools.py b/dev/tests/mcp/unit/test_dev_tools.py new file mode 100644 index 0000000..909f936 --- /dev/null +++ b/dev/tests/mcp/unit/test_dev_tools.py @@ -0,0 +1,62 @@ +import os +import sys +import pytest +from pathlib import Path + +def test_dev_tools_structure(): + """Test that development tools are properly organized.""" + # Test that dev directory exists + dev_dir = Path('dev') + assert dev_dir.exists() + assert dev_dir.is_dir() + + # Test that tools directory exists + tools_dir = dev_dir / 'tools' + assert tools_dir.exists() + assert tools_dir.is_dir() + + # Test that scripts directory exists + scripts_dir = dev_dir / 'scripts' + assert scripts_dir.exists() + assert scripts_dir.is_dir() + +def test_dev_tools_imports(): + """Test that development tools can be imported.""" + # Add dev directory to Python path + dev_tools_path = os.path.abspath('dev/tools') + if dev_tools_path not in sys.path: + sys.path.append(dev_tools_path) + + try: + # Test importing tools + import llm_api + import web_scraper + import search_engine + import screenshot_utils + + # Test basic attributes + assert hasattr(llm_api, 'query_llm') + assert hasattr(web_scraper, 'scrape_urls') + assert hasattr(search_engine, 'search') + assert hasattr(screenshot_utils, 'take_screenshot_sync') + except ImportError as e: + pytest.fail(f"Failed to import development tools: {e}") + +def test_dev_requirements(): + """Test that development requirements are installed.""" + import pkg_resources + + with open('requirements-dev.txt') as f: + requirements = [ + line.strip() + for line in f + if line.strip() and not line.startswith('#') and not line.startswith('-r') + ] + + for requirement in requirements: + try: + pkg_resources.require(requirement) + except pkg_resources.DistributionNotFound: + pytest.fail(f"Required package not found: {requirement}") + except pkg_resources.VersionConflict: + pytest.fail(f"Version conflict for package: {requirement}") \ No newline at end of file diff --git a/dev/tests/mcp/unit/test_env_manager.py b/dev/tests/mcp/unit/test_env_manager.py new file mode 100644 index 0000000..9c7703a --- /dev/null +++ b/dev/tests/mcp/unit/test_env_manager.py @@ -0,0 +1,54 @@ +import os +import pytest +from pathlib import Path +from dev.tools.env_manager import backup_env, switch_environment, list_environments + +@pytest.fixture +def temp_env_files(tmp_path): + """Create temporary environment files for testing.""" + # Create test environment files + (tmp_path / '.env.dev').write_text('DEV=true') + (tmp_path / '.env.prod').write_text('PROD=true') + (tmp_path / '.env').write_text('CURRENT=true') + + # Change to temp directory for tests + original_dir = os.getcwd() + os.chdir(tmp_path) + + yield tmp_path + + # Cleanup and restore original directory + os.chdir(original_dir) + +def test_backup_env(temp_env_files): + """Test environment file backup functionality.""" + env_path = temp_env_files / '.env' + backup_path = backup_env(env_path) + + assert backup_path.exists() + assert backup_path.name == '.env.backup' + assert backup_path.read_text() == 'CURRENT=true' + +def test_switch_environment(temp_env_files): + """Test switching between environments.""" + # Switch to dev environment + switch_environment('dev') + + env_path = temp_env_files / '.env' + assert env_path.exists() + if os.name != 'nt': # Skip symlink check on Windows + assert env_path.is_symlink() + assert env_path.resolve() == temp_env_files / '.env.dev' + +def test_switch_to_nonexistent_environment(temp_env_files): + """Test switching to a non-existent environment.""" + with pytest.raises(FileNotFoundError): + switch_environment('nonexistent') + +def test_list_environments(temp_env_files, capsys): + """Test listing available environments.""" + list_environments() + captured = capsys.readouterr() + + assert 'dev' in captured.out + assert 'prod' in captured.out \ No newline at end of file diff --git a/dev/tests/mcp/unit/test_environment.py b/dev/tests/mcp/unit/test_environment.py new file mode 100644 index 0000000..c37bedb --- /dev/null +++ b/dev/tests/mcp/unit/test_environment.py @@ -0,0 +1,53 @@ +import os +import pytest +from pathlib import Path +from dotenv import load_dotenv +from dev.tools.env_manager import switch_environment + +@pytest.fixture(scope='module') +def dev_environment(): + """Set up development environment for tests.""" + # Store current environment + original_env = {} + env_vars = ['ENVIRONMENT', 'DEBUG', 'LOG_LEVEL', 'API_BASE_URL', 'FRONTEND_URL'] + for var in env_vars: + original_env[var] = os.getenv(var) + + # Switch to dev environment + switch_environment('dev') + load_dotenv() + + yield + + # Restore original environment variables + for var, value in original_env.items(): + if value is None: + os.unsetenv(var) + else: + os.environ[var] = value + +def test_environment_variables(dev_environment): + """Test that required environment variables are set.""" + # Test environment type + assert os.getenv('ENVIRONMENT') == 'development' + + # Test debug mode + assert os.getenv('DEBUG') == 'true' + + # Test log level + assert os.getenv('LOG_LEVEL') == 'DEBUG' + +def test_python_environment(dev_environment): + """Test Python environment setup.""" + # Test that we're running in a virtual environment + assert os.getenv('VIRTUAL_ENV') is not None + + # Test that pytest is installed + import pytest + assert pytest.__version__ >= '8.3.4' + +def test_api_configuration(dev_environment): + """Test API configuration.""" + # Test API URLs are set + assert os.getenv('API_BASE_URL') == 'http://localhost:8000' + assert os.getenv('FRONTEND_URL') == 'http://localhost:3000' \ No newline at end of file diff --git a/dev/tests/mcp/unit/test_llm.py b/dev/tests/mcp/unit/test_llm.py new file mode 100644 index 0000000..a3f8df9 --- /dev/null +++ b/dev/tests/mcp/unit/test_llm.py @@ -0,0 +1,56 @@ +import os +import pytest +from pathlib import Path +from dotenv import load_dotenv +from dev.tools.env_manager import switch_environment + +@pytest.fixture(scope='module') +def dev_environment(): + """Set up development environment for tests.""" + # Store current environment + original_env = {} + env_vars = ['OPENAI_API_KEY', 'ANTHROPIC_API_KEY', 'GOOGLE_API_KEY'] + for var in env_vars: + original_env[var] = os.getenv(var) + + # Switch to dev environment + switch_environment('dev') + load_dotenv() + + yield + + # Restore original environment variables + for var, value in original_env.items(): + if value is None: + os.unsetenv(var) + else: + os.environ[var] = value + +def test_llm_api_keys_exist(dev_environment): + """Test that LLM API keys are set.""" + # Test OpenAI API key + openai_key = os.getenv('OPENAI_API_KEY') + assert openai_key is not None, "OpenAI API key not found" + + # Test Anthropic API key + anthropic_key = os.getenv('ANTHROPIC_API_KEY') + assert anthropic_key is not None, "Anthropic API key not found" + + # Test Google API key + google_key = os.getenv('GOOGLE_API_KEY') + assert google_key is not None, "Google API key not found" + +@pytest.mark.asyncio +async def test_llm_imports(dev_environment): + """Test that LLM libraries are properly installed.""" + # Test OpenAI + import openai + assert openai.__version__ >= '1.63.2' + + # Test Anthropic + import anthropic + assert anthropic.__version__ >= '0.46.0' + + # Test Google Generative AI + import google.generativeai as genai + assert genai.__package__ == 'google.generativeai' \ No newline at end of file diff --git a/dev/tools/__init__.py b/dev/tools/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dev/tools/env_manager.py b/dev/tools/env_manager.py new file mode 100644 index 0000000..258a922 --- /dev/null +++ b/dev/tools/env_manager.py @@ -0,0 +1,85 @@ +import os +import shutil +import argparse +from pathlib import Path +from typing import Optional + +def backup_env(env_path: Path, backup_suffix: str = 'backup') -> Optional[Path]: + """Backup existing .env file if it exists.""" + if env_path.exists(): + backup_path = env_path.with_suffix(f'.{backup_suffix}') + shutil.copy2(env_path, backup_path) + return backup_path + return None + +def switch_environment(env_type: str) -> None: + """Switch to a different environment configuration.""" + root_dir = Path.cwd() + env_path = root_dir / '.env' + target_env = root_dir / f'.env.{env_type}' + + if not target_env.exists(): + raise FileNotFoundError(f"Environment file .env.{env_type} not found") + + # Create backup of current .env if it exists + if env_path.exists(): + backup_path = backup_env(env_path) + print(f"Backed up current .env to {backup_path}") + + # Create symlink to target environment + if os.name == 'nt': # Windows + import ctypes + if not ctypes.windll.shell32.IsUserAnAdmin(): + print("Warning: On Windows, you may need admin privileges to create symlinks") + os.system(f'mklink {env_path} {target_env}') + else: # Unix-like + if env_path.exists(): + env_path.unlink() + env_path.symlink_to(target_env) + + print(f"Switched to {env_type} environment") + +def list_environments() -> None: + """List all available environment configurations.""" + root_dir = Path.cwd() + env_files = list(root_dir.glob('.env.*')) + + if not env_files: + print("No environment configurations found") + return + + print("\nAvailable environments:") + for env_file in env_files: + env_type = env_file.suffix[1:] # Remove the leading dot + if env_type != 'backup': + print(f"- {env_type}") + + current_env = root_dir / '.env' + if current_env.exists() and current_env.is_symlink(): + current = current_env.resolve().name.replace('.env.', '') + print(f"\nCurrent environment: {current}") + else: + print("\nCurrent environment: not linked to any environment file") + +def main(): + parser = argparse.ArgumentParser(description='Manage environment configurations') + parser.add_argument('action', choices=['switch', 'list'], help='Action to perform') + parser.add_argument('--env', help='Environment to switch to (e.g., dev, prod)') + + args = parser.parse_args() + + try: + if args.action == 'list': + list_environments() + elif args.action == 'switch': + if not args.env: + parser.error("--env is required when using 'switch'") + switch_environment(args.env) + except Exception as e: + print(f"Error: {e}") + return 1 + + return 0 + +if __name__ == '__main__': + exit(main()) \ No newline at end of file diff --git a/dev/tools/llm_api.py b/dev/tools/llm_api.py new file mode 100644 index 0000000..3be6a54 --- /dev/null +++ b/dev/tools/llm_api.py @@ -0,0 +1,79 @@ +"""LLM API integration module.""" +from typing import Optional, Dict, Any +import os +import openai +import anthropic +import google.generativeai as genai +from dotenv import load_dotenv + +load_dotenv() + +def query_llm( + prompt: str, + provider: str = "openai", + model: Optional[str] = None, + **kwargs: Any +) -> str: + """Query an LLM provider with the given prompt.""" + if provider == "openai": + return _query_openai(prompt, model, **kwargs) + elif provider == "anthropic": + return _query_anthropic(prompt, model, **kwargs) + elif provider == "google": + return _query_google(prompt, model, **kwargs) + else: + raise ValueError(f"Unsupported provider: {provider}") + +def _query_openai( + prompt: str, + model: Optional[str] = None, + **kwargs: Any +) -> str: + """Query OpenAI's API.""" + openai.api_key = os.getenv("OPENAI_API_KEY") + if not openai.api_key: + raise ValueError("OpenAI API key not found in environment") + + model = model or "gpt-4o" + response = openai.ChatCompletion.create( + model=model, + messages=[{"role": "user", "content": prompt}], + **kwargs + ) + return response.choices[0].message.content + +def _query_anthropic( + prompt: str, + model: Optional[str] = None, + **kwargs: Any +) -> str: + """Query Anthropic's API.""" + api_key = os.getenv("ANTHROPIC_API_KEY") + if not api_key: + raise ValueError("Anthropic API key not found in environment") + + client = anthropic.Client(api_key=api_key) + model = model or "claude-3-sonnet-20240229" + + response = client.messages.create( + model=model, + messages=[{"role": "user", "content": prompt}], + **kwargs + ) + return response.content[0].text + +def _query_google( + prompt: str, + model: Optional[str] = None, + **kwargs: Any +) -> str: + """Query Google's Generative AI API.""" + api_key = os.getenv("GOOGLE_API_KEY") + if not api_key: + raise ValueError("Google API key not found in environment") + + genai.configure(api_key=api_key) + model = genai.GenerativeModel(model or "gemini-pro") + + response = model.generate_content(prompt, **kwargs) + return response.text \ No newline at end of file diff --git a/dev/tools/screenshot_utils.py b/dev/tools/screenshot_utils.py new file mode 100644 index 0000000..74ebd24 --- /dev/null +++ b/dev/tools/screenshot_utils.py @@ -0,0 +1,56 @@ +"""Screenshot utility module.""" +import os +import asyncio +from typing import Optional +from pathlib import Path +from playwright.async_api import async_playwright + +async def take_screenshot( + url: str, + output_path: Optional[str] = None, + width: int = 1920, + height: int = 1080, + full_page: bool = False, + wait_for_load: bool = True, + wait_for_network_idle: bool = True +) -> str: + """Take a screenshot of a webpage using Playwright.""" + if output_path is None: + output_path = f"screenshot_{hash(url)}.png" + + output_path = Path(output_path) + output_path.parent.mkdir(parents=True, exist_ok=True) + + async with async_playwright() as p: + browser = await p.chromium.launch() + page = await browser.new_page() + + # Set viewport size + await page.set_viewport_size({"width": width, "height": height}) + + # Navigate to the page + await page.goto(url) + + # Wait for the page to load + if wait_for_load: + await page.wait_for_load_state("load") + if wait_for_network_idle: + await page.wait_for_load_state("networkidle") + + # Take the screenshot + await page.screenshot( + path=str(output_path), + full_page=full_page + ) + + await browser.close() + + return str(output_path) + +def take_screenshot_sync( + url: str, + output_path: Optional[str] = None, + **kwargs +) -> str: + """Synchronous wrapper for take_screenshot.""" + return asyncio.run(take_screenshot(url, output_path, **kwargs)) \ No newline at end of file diff --git a/dev/tools/search_engine.py b/dev/tools/search_engine.py new file mode 100644 index 0000000..57b6e28 --- /dev/null +++ b/dev/tools/search_engine.py @@ -0,0 +1,91 @@ +"""Search engine integration module.""" +from typing import List, Dict, Any, Optional +from duckduckgo_search import DDGS + +def search( + query: str, + max_results: int = 10, + region: str = "wt-wt", + safesearch: str = "moderate", + **kwargs: Any +) -> List[Dict[str, str]]: + """Search the web using DuckDuckGo.""" + with DDGS() as ddgs: + results = list(ddgs.text( + query, + region=region, + safesearch=safesearch, + max_results=max_results + )) + + # Format results + formatted_results = [] + for result in results: + formatted_results.append({ + "title": result["title"], + "link": result["link"], + "snippet": result["body"] + }) + + return formatted_results + +def search_news( + query: str, + max_results: int = 10, + region: str = "wt-wt", + **kwargs: Any +) -> List[Dict[str, str]]: + """Search news articles using DuckDuckGo.""" + with DDGS() as ddgs: + results = list(ddgs.news( + query, + region=region, + max_results=max_results + )) + + # Format results + formatted_results = [] + for result in results: + formatted_results.append({ + "title": result["title"], + "link": result["link"], + "snippet": result["body"], + "date": result.get("date", ""), + "source": result.get("source", "") + }) + + return formatted_results + +def search_images( + query: str, + max_results: int = 10, + size: Optional[str] = None, + color: Optional[str] = None, + type_image: Optional[str] = None, + layout: Optional[str] = None, + **kwargs: Any +) -> List[Dict[str, str]]: + """Search images using DuckDuckGo.""" + with DDGS() as ddgs: + results = list(ddgs.images( + query, + max_results=max_results, + size=size, + color=color, + type_image=type_image, + layout=layout + )) + + # Format results + formatted_results = [] + for result in results: + formatted_results.append({ + "title": result["title"], + "image": result["image"], + "thumbnail": result.get("thumbnail", ""), + "source": result["source"], + "width": result.get("width", ""), + "height": result.get("height", "") + }) + + return formatted_results \ No newline at end of file diff --git a/dev/tools/web_scraper.py b/dev/tools/web_scraper.py new file mode 100644 index 0000000..21cbcf0 --- /dev/null +++ b/dev/tools/web_scraper.py @@ -0,0 +1,126 @@ +"""Web scraping utility module.""" +import asyncio +from typing import List, Dict, Any, Optional +import aiohttp +from bs4 import BeautifulSoup +from playwright.async_api import async_playwright +from tqdm import tqdm + +async def scrape_urls( + urls: List[str], + max_concurrent: int = 5, + use_playwright: bool = False, + **kwargs: Any +) -> Dict[str, str]: + """Scrape multiple URLs concurrently.""" + if use_playwright: + return await _scrape_with_playwright(urls, max_concurrent, **kwargs) + else: + return await _scrape_with_aiohttp(urls, max_concurrent, **kwargs) + +async def _scrape_with_aiohttp( + urls: List[str], + max_concurrent: int = 5, + timeout: int = 30, + **kwargs: Any +) -> Dict[str, str]: + """Scrape URLs using aiohttp.""" + results = {} + semaphore = asyncio.Semaphore(max_concurrent) + + async with aiohttp.ClientSession() as session: + tasks = [] + for url in urls: + task = asyncio.create_task( + _fetch_url(session, url, semaphore, timeout) + ) + tasks.append(task) + + for coro in tqdm( + asyncio.as_completed(tasks), + total=len(tasks), + desc="Scraping URLs" + ): + url, content = await coro + if content: + results[url] = content + + return results + +async def _fetch_url( + session: aiohttp.ClientSession, + url: str, + semaphore: asyncio.Semaphore, + timeout: int +) -> tuple[str, Optional[str]]: + """Fetch a single URL using aiohttp.""" + async with semaphore: + try: + async with session.get(url, timeout=timeout) as response: + if response.status == 200: + content = await response.text() + soup = BeautifulSoup(content, 'lxml') + return url, soup.get_text() + return url, None + except Exception as e: + print(f"Error fetching {url}: {e}") + return url, None + +async def _scrape_with_playwright( + urls: List[str], + max_concurrent: int = 5, + **kwargs: Any +) -> Dict[str, str]: + """Scrape URLs using Playwright.""" + results = {} + semaphore = asyncio.Semaphore(max_concurrent) + + async with async_playwright() as p: + browser = await p.chromium.launch() + + tasks = [] + for url in urls: + task = asyncio.create_task( + _fetch_with_playwright(browser, url, semaphore) + ) + tasks.append(task) + + for coro in tqdm( + asyncio.as_completed(tasks), + total=len(tasks), + desc="Scraping URLs with Playwright" + ): + url, content = await coro + if content: + results[url] = content + + await browser.close() + + return results + +async def _fetch_with_playwright( + browser: Any, + url: str, + semaphore: asyncio.Semaphore +) -> tuple[str, Optional[str]]: + """Fetch a single URL using Playwright.""" + async with semaphore: + try: + page = await browser.new_page() + await page.goto(url) + content = await page.content() + soup = BeautifulSoup(content, 'lxml') + text = soup.get_text() + await page.close() + return url, text + except Exception as e: + print(f"Error fetching {url} with Playwright: {e}") + return url, None + +def scrape_urls_sync( + urls: List[str], + max_concurrent: int = 5, + **kwargs: Any +) -> Dict[str, str]: + """Synchronous wrapper for scrape_urls.""" + return asyncio.run(scrape_urls(urls, max_concurrent, **kwargs)) \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..386c051 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,21 @@ +# Development Dependencies +pytest>=8.3.4 +pytest-asyncio>=0.25.3 +pytest-cov>=4.1.0 # Coverage reports +matplotlib>=3.10.0 # Visualization for development +seaborn>=0.13.2 # Additional plotting +tabulate>=0.9.0 # Development data formatting +tqdm>=4.67.1 # Progress bars for development scripts + +# Development Tools +python-dotenv>=1.0.1 # Environment management +playwright>=1.50.0 # Web automation for testing +duckduckgo-search>=7.4.3 # Search capabilities for development + +# Linting and Type Checking +mypy>=1.8.0 +pylint>=3.0.3 +black>=24.1.1 +isort>=5.13.2 + +-r requirements.txt # Include base requirements \ No newline at end of file