Add LLM-as-a-Judge evaluator using MLflow and OpenAI-compatible endpoints

ai4rag/core/experiment/experiment.py

@@ -200,14 +200,19 @@ def optimization_metric(self) -> str:
     @optimization_metric.setter
     def optimization_metric(self, val: str) -> None:
         """Validate and set optimization metrics"""
-        if val not in MetricType:
+        if val not in MetricType and not self._is_custom_evaluator_metric(val):
             raise RAGExperimentError(
-                f"Provided optimization metric: '{val}' is not supported. "
-                f"Available metrics: ['answer_correctness', 'faithfulness', 'context_correctness']."
+                f"Provided optimization metric: '{val}' is not supported. " f"Available metrics: {list(MetricType)}."
             )
 
         self._optimization_metric = val
 
+    def _is_custom_evaluator_metric(self, metric_name: str) -> bool:
+        """Check if a metric is supported by a custom evaluator."""
+        if hasattr(self, "evaluator") and hasattr(self.evaluator, "get_supported_metrics"):
+            return metric_name in self.evaluator.get_supported_metrics()
+        return False
+
     @property
     def benchmark_data(self) -> BenchmarkData:
         """Get benchmark data."""

ai4rag/evaluator/__init__.py

@@ -4,3 +4,11 @@
 # -----------------------------------------------------------------------------
 from ai4rag.evaluator.base_evaluator import BaseEvaluator
 from ai4rag.evaluator.unitxt_evaluator import UnitxtEvaluator
+
+try:
+    from ai4rag.evaluator.mlflow_llm_judge_evaluator import (
+        LLMJudgeConfig,
+        MlflowLLMJudgeEvaluator,
+    )
+except ImportError:
+    pass

ai4rag/evaluator/base_evaluator.py

@@ -77,6 +77,7 @@ class MetricType(metaclass=ConstantMeta):
     ANSWER_CORRECTNESS = "answer_correctness"
     FAITHFULNESS = "faithfulness"
     CONTEXT_CORRECTNESS = "context_correctness"
+    ANSWER_RELEVANCE = "answer_relevance"
 
 
 class BaseEvaluator(ABC):