merged with main

Author: jjmachan

.github/workflows/ci.yaml

@@ -96,6 +96,9 @@ jobs:
           fi
           # Now run the unit tests
           pytest tests/unit "${OPTS[@]}"
+        env:
+          __RAGAS_DEBUG_TRACKING: true
+          RAGAS_DO_NOT_TRACK: true
 
   codestyle_check:
     runs-on: ubuntu-latest

pyproject.toml

@@ -9,6 +9,7 @@ dependencies = [
     "openai>1",
     "pysbd>=0.3.4",
     "nest-asyncio",
+    "appdirs",
 ]
 dynamic = ["version", "readme"]
 

src/ragas/_analytics.py

@@ -1,12 +1,15 @@
 from __future__ import annotations
 
+import json
 import logging
 import os
 import typing as t
-from dataclasses import asdict, dataclass
+import uuid
 from functools import lru_cache, wraps
 
 import requests
+from appdirs import user_data_dir
+from langchain_core.pydantic_v1 import BaseModel, Field
 
 from ragas.utils import get_debug_mode
 
@@ -19,9 +22,11 @@
 
 
 USAGE_TRACKING_URL = "https://t.explodinggradients.com"
+USAGE_REQUESTS_TIMEOUT_SEC = 1
+USER_DATA_DIR_NAME = "ragas"
+# Any chance you chance this also change the variable in our ci.yaml file
 RAGAS_DO_NOT_TRACK = "RAGAS_DO_NOT_TRACK"
 RAGAS_DEBUG_TRACKING = "__RAGAS_DEBUG_TRACKING"
-USAGE_REQUESTS_TIMEOUT_SEC = 1
 
 
 @lru_cache(maxsize=1)
@@ -33,7 +38,7 @@ def do_not_track() -> bool:  # pragma: no cover
 
 @lru_cache(maxsize=1)
 def _usage_event_debugging() -> bool:
-    # For BentoML developers only - debug and print event payload if turned on
+    # For Ragas developers only - debug and print event payload if turned on
     return os.environ.get(RAGAS_DEBUG_TRACKING, str(False)).lower() == "true"
 
 
@@ -49,6 +54,7 @@ def wrapper(*args: P.args, **kwargs: P.kwargs) -> t.Any:
                     logger.error(
                         "Tracking Error: %s", err, stack_info=True, stacklevel=3
                     )
+                    raise err
                 else:
                     logger.info("Tracking Error: %s", err)
             else:
@@ -57,14 +63,28 @@ def wrapper(*args: P.args, **kwargs: P.kwargs) -> t.Any:
     return wrapper
 
 
-@dataclass
-class BaseEvent:
+@lru_cache(maxsize=1)
+@silent
+def get_userid() -> str:
+    user_id_path = user_data_dir(appname=USER_DATA_DIR_NAME)
+    uuid_filepath = os.path.join(user_id_path, "uuid.json")
+    if os.path.exists(uuid_filepath):
+        user_id = json.load(open(uuid_filepath))["userid"]
+    else:
+        user_id = "a-" + uuid.uuid4().hex
+        os.makedirs(user_id_path)
+        with open(uuid_filepath, "w") as f:
+            json.dump({"userid": user_id}, f)
+    return user_id
+
+
+class BaseEvent(BaseModel):
     event_type: str
+    user_id: str = Field(default_factory=get_userid)
 
 
-@dataclass
 class EvaluationEvent(BaseEvent):
-    metrics: list[str]
+    metrics: t.List[str]
     evaluation_mode: str
     num_rows: int
 
@@ -74,8 +94,7 @@ def track(event_properties: BaseEvent):
     if do_not_track():
         return
 
-    payload = asdict(event_properties)
-
+    payload = dict(event_properties)
     if _usage_event_debugging():
         # For internal debugging purpose
         logger.info("Tracking Payload: %s", payload)

src/ragas/evaluation.py

@@ -144,7 +144,12 @@ def evaluate(
         row_run_managers.append((row_rm, row_group_cm))
 
         if is_async:
-            [executor.submit(metric.ascore, row, row_group_cm) for metric in metrics]
+            [
+                executor.submit(
+                    metric.ascore, row, row_group_cm, name=f"{metric.name}-{i}"
+                )
+                for metric in metrics
+            ]
         else:
             [executor.submit(metric.score, row, row_group_cm) for metric in metrics]
 

src/ragas/executor.py

@@ -43,7 +43,9 @@ async def wrapped_callable_async(*args, **kwargs):
         else:
             return wrapped_callable
 
-    def submit(self, callable: t.Callable, *args, **kwargs):
+    def submit(
+        self, callable: t.Callable, *args, name: t.Optional[str] = None, **kwargs
+    ):
         if self.is_async:
             self.executor = t.cast(asyncio.AbstractEventLoop, self.executor)
             callable_with_index = self.wrap_callable_with_index(
@@ -52,7 +54,9 @@ def submit(self, callable: t.Callable, *args, **kwargs):
             # is type correct?
             callable_with_index = t.cast(t.Callable, callable_with_index)
             self.futures.append(
-                self.executor.create_task(callable_with_index(*args, **kwargs))
+                self.executor.create_task(
+                    callable_with_index(*args, **kwargs), name=name
+                )
             )
         else:
             self.executor = t.cast(ThreadPoolExecutor, self.executor)

src/ragas/llms/base.py

@@ -9,12 +9,13 @@
 from langchain_core.language_models import BaseLanguageModel
 from langchain_core.outputs import LLMResult
 
-from ragas.llms.prompt import PromptValue
-
 if t.TYPE_CHECKING:
     from langchain_core.callbacks import Callbacks
     from langchain_core.prompts import ChatPromptTemplate
 
+    from ragas.llms.prompt import PromptValue
+
+
 MULTIPLE_COMPLETION_SUPPORTED = [
     OpenAI,
     ChatOpenAI,
@@ -66,6 +67,8 @@ def generate_text_with_hmpt(
         stop: t.Optional[t.List[str]] = None,
         callbacks: Callbacks = [],
     ) -> LLMResult:
+        from ragas.llms.prompt import PromptValue
+
         prompt = PromptValue(prompt_str=prompts[0].format())
         return self.generate_text(prompt, n, temperature, stop, callbacks)
 

src/ragas/llms/json_load.py

@@ -82,21 +82,38 @@ def safe_load(self, text: str, llm: BaseRagasLLM, callbacks: Callbacks = None):
                 start, end = self._find_outermost_json(text)
                 return json.loads(text[start:end])
             except ValueError:
-                text = self._fix_to_json(text, llm, callbacks)
+                from ragas.llms.prompt import PromptValue
+
+                results = llm.generate_text(
+                    PromptValue(prompt_str=JSON_PROMPT.format(input=text)),
+                    n=1,
+                    callbacks=callbacks,
+                )
+                text = results.generations[0][0].text
             retry += 1
 
         return {}
 
-    def _fix_to_json(self, text: str, llm: BaseRagasLLM, callbacks: Callbacks):
-        from ragas.llms.prompt import PromptValue
+    async def asafe_load(
+        self, text: str, llm: BaseRagasLLM, callbacks: Callbacks = None
+    ):
+        retry = 0
+        while retry <= self.max_retries:
+            try:
+                start, end = self._find_outermost_json(text)
+                return json.loads(text[start:end])
+            except ValueError:
+                from ragas.llms.prompt import PromptValue
+
+                results = await llm.agenerate_text(
+                    PromptValue(prompt_str=JSON_PROMPT.format(input=text)),
+                    n=1,
+                    callbacks=callbacks,
+                )
+                text = results.generations[0][0].text
+            retry += 1
 
-        # TODO (executor)
-        results = llm.generate_text(
-            PromptValue(prompt_str=JSON_PROMPT.format(input=text)),
-            n=1,
-            callbacks=callbacks,
-        )
-        return results.generations[0][0].text
+        return {}
 
     def _find_outermost_json(self, text):
         stack = []

src/ragas/llms/prompt.py

@@ -9,12 +9,10 @@
 from langchain_core.prompt_values import PromptValue as BasePromptValue
 from langchain_core.pydantic_v1 import BaseModel, root_validator
 
+from ragas.llms import BaseRagasLLM
 from ragas.llms.json_load import json_loader
 from ragas.utils import get_cache_dir
 
-if t.TYPE_CHECKING:
-    from ragas.llms import BaseRagasLLM
-
 Example = t.Dict[str, t.Any]
 
 

src/ragas/metrics/_answer_correctness.py

@@ -15,7 +15,6 @@
 
 if t.TYPE_CHECKING:
     from langchain_core.callbacks import Callbacks
-    from langchain_core.outputs import LLMResult
 
 CORRECTNESS_PROMPT = Prompt(
     name="answer_correctness",
@@ -110,7 +109,7 @@ def __post_init__(self: t.Self):
                 llm=self.llm, batch_size=self.batch_size
             )
 
-    def _compute_statement_presence(self, result: LLMResult) -> float:
+    def _compute_statement_presence(self, prediction: t.Any) -> float:
         assert self.llm is not None, "LLM must be set"
 
         key_map = {
@@ -120,7 +119,6 @@ def _compute_statement_presence(self, result: LLMResult) -> float:
         }
         outputs = result.generations[0]
 
-        prediction = json_loader.safe_load(outputs[0].text, self.llm)
         prediction = prediction if isinstance(prediction, list) else [prediction]
         if prediction:
             prediction = [
@@ -146,7 +144,10 @@ def _score(self, row: t.Dict, callbacks: Callbacks) -> float:
         p_value = self.correctness_prompt.format(question=q, ground_truth=g, answer=a)
         is_statement_present = self.llm.generate_text(p_value, callbacks=callbacks)
 
-        f1_score = self._compute_statement_presence(is_statement_present)
+        prediction = json_loader.safe_load(
+            is_statement_present.generations[0][0].text, self.llm
+        )
+        f1_score = self._compute_statement_presence(prediction)
 
         if self.weights[1] == 0:
             similarity_score = 0
@@ -169,7 +170,10 @@ async def _ascore(self, row: t.Dict, callbacks: Callbacks) -> float:
             p_value, callbacks=callbacks
         )
 
-        f1_score = self._compute_statement_presence(is_statement_present)
+        prediction = await json_loader.asafe_load(
+            is_statement_present.generations[0][0].text, self.llm
+        )
+        f1_score = self._compute_statement_presence(prediction)
 
         if self.weights[1] == 0:
             similarity_score = 0

src/ragas/metrics/_answer_relevance.py

@@ -5,10 +5,8 @@
 from dataclasses import dataclass, field
 
 import numpy as np
-from langchain.embeddings import OpenAIEmbeddings
 
 from ragas.embeddings.base import embedding_factory
-from ragas.exceptions import OpenAIKeyNotFound
 from ragas.llms.json_load import json_loader
 from ragas.llms.prompt import Prompt
 from ragas.metrics.base import EvaluationMode, MetricWithLLM
@@ -83,10 +81,6 @@ class AnswerRelevancy(MetricWithLLM):
     def init_model(self):
         super().init_model()
 
-        if isinstance(self.embeddings, OpenAIEmbeddings):
-            if self.embeddings.openai_api_key == "no-key":
-                raise OpenAIKeyNotFound
-
     def calculate_similarity(
         self: t.Self, question: str, generated_questions: list[str]
     ):
@@ -143,7 +137,8 @@ async def _ascore(self, row: t.Dict, callbacks: Callbacks) -> float:
             callbacks=callbacks,
         )
         response = [
-            json_loader.safe_load(r.text, self.llm) for r in result.generations[0]
+            await json_loader.asafe_load(r.text, self.llm)
+            for r in result.generations[0]
         ]
 
         return self._calculate_score(response, row)

src/ragas/metrics/_context_precision.py

@@ -143,7 +143,9 @@ async def _ascore(
             )
             responses.append(result.generations[0][0].text)
 
-        json_responses = [json_loader.safe_load(item, self.llm) for item in responses]
+        json_responses = [
+            await json_loader.asafe_load(item, self.llm) for item in responses
+        ]
         score = self._calculate_average_precision(json_responses)
         return score
 

src/ragas/metrics/_context_recall.py

@@ -123,7 +123,7 @@ async def _ascore(self, row: t.Dict, callbacks: Callbacks) -> float:
         result = await self.llm.agenerate_text(
             self._create_context_recall_prompt(row), callbacks=callbacks
         )
-        response = json_loader.safe_load(result.generations[0][0].text, self.llm)
+        response = await json_loader.asafe_load(result.generations[0][0].text, self.llm)
 
         return self._compute_score(response)