Implement comprehensive LLMOps (evaluating, testing, and monitoring) with Opik integration

.gitignore

@@ -12,7 +12,6 @@ build/
 develop-eggs/
 dist/
 downloads/
-examples/
 history/
 eggs/
 .eggs/

browser_ai/agent/service.py

@@ -29,6 +29,7 @@
 from pydantic import BaseModel, ValidationError
 
 from browser_ai.agent.message_manager.service import MessageManager
+from browser_ai.llmops import OpikConfig, OpikLLMOps
 from browser_ai.agent.prompts import AgentMessagePrompt, PlannerPrompt, SystemPrompt
 from browser_ai.agent.views import (
 	ActionResult,
@@ -100,6 +101,9 @@ def __init__(
 		page_extraction_llm: Optional[BaseChatModel] = None,
 		planner_llm: Optional[BaseChatModel] = None,
 		planner_interval: int = 1,  # Run planner every N steps
+		# Opik LLMOps integration
+		opik_config: Optional[OpikConfig] = None,
+		enable_opik_llmops: bool = True,
 	):
 		self.agent_id = str(uuid.uuid4())  # unique identifier for the agent
 		self.sensitive_data = sensitive_data
@@ -191,6 +195,20 @@ def __init__(
 		self._stopped = False
 
 		self.action_descriptions = self.controller.registry.get_prompt_description()
+
+		# Initialize Opik LLMOps integration
+		if enable_opik_llmops:
+			if opik_config is None:
+				opik_config = OpikConfig(
+					project_name=f"browser-ai-{self.task[:30]}",
+					enabled=True,
+					tags=["browser-ai", "agent", getattr(llm, 'model_name', 'unknown-model')]
+				)
+			self.opik_llmops = OpikLLMOps(opik_config)
+			logger.info("Opik LLMOps integration enabled")
+		else:
+			self.opik_llmops = None
+			logger.info("Opik LLMOps integration disabled")
 
 	def _set_version_and_source(self) -> None:
 		try:
@@ -265,6 +283,16 @@ def _check_if_stopped_or_paused(self) -> bool:
 	@time_execution_async('--step')
 	async def step(self, step_info: Optional[AgentStepInfo] = None) -> None:
 		"""Execute one step of the task"""
+
+		# Apply Opik tracing if enabled
+		if self.opik_llmops:
+			decorated_step = self.opik_llmops.trace_action_execution(self._step_impl)
+			return await decorated_step(self, step_info)
+		else:
+			return await self._step_impl(step_info)
+
+	async def _step_impl(self, step_info: Optional[AgentStepInfo] = None) -> None:
+		"""Internal implementation of step method"""
 		logger.info(f'📍 Step {self.n_steps}')
 		state = None
 		model_output = None
@@ -507,6 +535,16 @@ def _log_agent_run(self) -> None:
 	@observe(name='agent.run', ignore_output=True)
 	async def run(self, max_steps: int = 100) -> AgentHistoryList:
 		"""Execute the task with maximum number of steps"""
+
+		# Apply Opik tracing if enabled
+		if self.opik_llmops:
+			decorated_run = self.opik_llmops.trace_agent_execution(self._run_impl)
+			return await decorated_run(self, max_steps)
+		else:
+			return await self._run_impl(max_steps)
+
+	async def _run_impl(self, max_steps: int = 100) -> AgentHistoryList:
+		"""Internal implementation of run method"""
 		try:
 			self._log_agent_run()
 

browser_ai/controller/service.py

@@ -10,6 +10,7 @@
 from browser_ai.agent.views import ActionModel, ActionResult
 from browser_ai.browser.context import BrowserContext
 from browser_ai.controller.registry.service import Registry
+from browser_ai.llmops import OpikConfig, OpikLLMOps
 from browser_ai.controller.views import (
 	ClickElementAction,
 	DoneAction,
@@ -33,10 +34,12 @@ def __init__(
 		self,
 		exclude_actions: list[str] = [],
 		output_model: Optional[Type[BaseModel]] = None,
+		opik_llmops: Optional[OpikLLMOps] = None,
 	):
 		self.exclude_actions = exclude_actions
 		self.output_model = output_model
 		self.registry = Registry(exclude_actions)
+		self.opik_llmops = opik_llmops
 		self._register_default_actions()
 
 	def _register_default_actions(self):

browser_ai/llmops/__init__.py

@@ -0,0 +1,24 @@
+"""
+LLMOps module for Browser.AI
+
+This module provides comprehensive LLMOps capabilities including:
+- Evaluation of LLM performance and task completion
+- Testing workflows and automation scenarios  
+- Monitoring of agent execution and metrics
+- Integration with observability platforms (LMNR, Opik)
+"""
+
+from .opik_integration import OpikConfig, OpikLLMOps, OpikTracer, OpikEvaluator, OpikMonitor
+from .test_framework import BrowserAITestSuite, TestScenario, TestResult, create_sample_scenarios
+
+__all__ = [
+    'OpikConfig',
+    'OpikLLMOps', 
+    'OpikTracer',
+    'OpikEvaluator',
+    'OpikMonitor',
+    'BrowserAITestSuite',
+    'TestScenario',
+    'TestResult',
+    'create_sample_scenarios'
+]