add rag judges that use mistral-large-instruct

Signed-off-by: lilacheden <[email protected]>
IBM · Feb 11, 2025 · 56347a8 · 56347a8
1 parent 16ebe42
commit 56347a8
Show file tree

Hide file tree

Showing 23 changed files with 272 additions and 0 deletions.
diff --git a/prepare/metrics/llm_as_judge/rag_judge.py b/prepare/metrics/llm_as_judge/rag_judge.py
@@ -102,6 +102,8 @@ def get_prediction_field(metric_type):
     "llama_3_3_70b_instruct_watsonx": "engines.classification.llama_3_3_70b_instruct_watsonx",
     "llama_3_3_70b_instruct_rits": "engines.classification.llama_3_3_70b_instruct_rits",
     "gpt_4o_azure": "engines.classification.gpt_4o_2024_08_06_azure_openai",
+    "mistral_large_instruct_watsonx": "engines.classification.mistral_large_watsonx",
+    "mistral_large_instruct_rits": "engines.classification.mistral_large_instruct_2407_rits",
     generic_engine_label: GenericInferenceEngine(),
 }
 

diff --git a/.../catalog/metrics/rag/end_to_end/answer_correctness/mistral_large_instruct_rits_judge.json b/.../catalog/metrics/rag/end_to_end/answer_correctness/mistral_large_instruct_rits_judge.json
@@ -0,0 +1,13 @@
+{
+    "__type__": "task_based_ll_mas_judge",
+    "inference_model": "engines.classification.mistral_large_instruct_2407_rits",
+    "template": "templates.rag_eval.answer_correctness.judge_loose_match_no_context_numeric",
+    "task": "tasks.rag_eval.answer_correctness.binary",
+    "format": null,
+    "main_score": "answer_correctness_judge",
+    "prediction_field": "answer",
+    "infer_log_probs": false,
+    "judge_to_generator_fields_mapping": {
+        "ground_truths": "reference_answers"
+    }
+}
diff --git a/...talog/metrics/rag/end_to_end/answer_correctness/mistral_large_instruct_watsonx_judge.json b/...talog/metrics/rag/end_to_end/answer_correctness/mistral_large_instruct_watsonx_judge.json
@@ -0,0 +1,13 @@
+{
+    "__type__": "task_based_ll_mas_judge",
+    "inference_model": "engines.classification.mistral_large_watsonx",
+    "template": "templates.rag_eval.answer_correctness.judge_loose_match_no_context_numeric",
+    "task": "tasks.rag_eval.answer_correctness.binary",
+    "format": null,
+    "main_score": "answer_correctness_judge",
+    "prediction_field": "answer",
+    "infer_log_probs": false,
+    "judge_to_generator_fields_mapping": {
+        "ground_truths": "reference_answers"
+    }
+}
diff --git a/...xt/catalog/metrics/rag/end_to_end/answer_relevance/mistral_large_instruct_rits_judge.json b/...xt/catalog/metrics/rag/end_to_end/answer_relevance/mistral_large_instruct_rits_judge.json
@@ -0,0 +1,13 @@
+{
+    "__type__": "task_based_ll_mas_judge",
+    "inference_model": "engines.classification.mistral_large_instruct_2407_rits",
+    "template": "templates.rag_eval.answer_relevance.judge_answer_relevance_numeric",
+    "task": "tasks.rag_eval.answer_relevance.binary",
+    "format": null,
+    "main_score": "answer_relevance_judge",
+    "prediction_field": "answer",
+    "infer_log_probs": false,
+    "judge_to_generator_fields_mapping": {
+        "ground_truths": "reference_answers"
+    }
+}
diff --git a/...catalog/metrics/rag/end_to_end/answer_relevance/mistral_large_instruct_watsonx_judge.json b/...catalog/metrics/rag/end_to_end/answer_relevance/mistral_large_instruct_watsonx_judge.json
@@ -0,0 +1,13 @@
+{
+    "__type__": "task_based_ll_mas_judge",
+    "inference_model": "engines.classification.mistral_large_watsonx",
+    "template": "templates.rag_eval.answer_relevance.judge_answer_relevance_numeric",
+    "task": "tasks.rag_eval.answer_relevance.binary",
+    "format": null,
+    "main_score": "answer_relevance_judge",
+    "prediction_field": "answer",
+    "infer_log_probs": false,
+    "judge_to_generator_fields_mapping": {
+        "ground_truths": "reference_answers"
+    }
+}
diff --git a/...t/catalog/metrics/rag/end_to_end/context_relevance/mistral_large_instruct_rits_judge.json b/...t/catalog/metrics/rag/end_to_end/context_relevance/mistral_large_instruct_rits_judge.json
@@ -0,0 +1,13 @@
+{
+    "__type__": "task_based_ll_mas_judge",
+    "inference_model": "engines.classification.mistral_large_instruct_2407_rits",
+    "template": "templates.rag_eval.context_relevance.judge_context_relevance_ares_numeric",
+    "task": "tasks.rag_eval.context_relevance.binary",
+    "format": null,
+    "main_score": "context_relevance_judge",
+    "prediction_field": "contexts",
+    "infer_log_probs": false,
+    "judge_to_generator_fields_mapping": {
+        "ground_truths": "reference_answers"
+    }
+}
diff --git a/...atalog/metrics/rag/end_to_end/context_relevance/mistral_large_instruct_watsonx_judge.json b/...atalog/metrics/rag/end_to_end/context_relevance/mistral_large_instruct_watsonx_judge.json
@@ -0,0 +1,13 @@
+{
+    "__type__": "task_based_ll_mas_judge",
+    "inference_model": "engines.classification.mistral_large_watsonx",
+    "template": "templates.rag_eval.context_relevance.judge_context_relevance_ares_numeric",
+    "task": "tasks.rag_eval.context_relevance.binary",
+    "format": null,
+    "main_score": "context_relevance_judge",
+    "prediction_field": "contexts",
+    "infer_log_probs": false,
+    "judge_to_generator_fields_mapping": {
+        "ground_truths": "reference_answers"
+    }
+}
diff --git a/...unitxt/catalog/metrics/rag/end_to_end/faithfulness/mistral_large_instruct_rits_judge.json b/...unitxt/catalog/metrics/rag/end_to_end/faithfulness/mistral_large_instruct_rits_judge.json
@@ -0,0 +1,13 @@
+{
+    "__type__": "task_based_ll_mas_judge",
+    "inference_model": "engines.classification.mistral_large_instruct_2407_rits",
+    "template": "templates.rag_eval.faithfulness.judge_with_question_simplified_verbal",
+    "task": "tasks.rag_eval.faithfulness.binary",
+    "format": null,
+    "main_score": "faithfulness_judge",
+    "prediction_field": "answer",
+    "infer_log_probs": false,
+    "judge_to_generator_fields_mapping": {
+        "ground_truths": "reference_answers"
+    }
+}
diff --git a/...txt/catalog/metrics/rag/end_to_end/faithfulness/mistral_large_instruct_watsonx_judge.json b/...txt/catalog/metrics/rag/end_to_end/faithfulness/mistral_large_instruct_watsonx_judge.json
@@ -0,0 +1,13 @@
+{
+    "__type__": "task_based_ll_mas_judge",
+    "inference_model": "engines.classification.mistral_large_watsonx",
+    "template": "templates.rag_eval.faithfulness.judge_with_question_simplified_verbal",
+    "task": "tasks.rag_eval.faithfulness.binary",
+    "format": null,
+    "main_score": "faithfulness_judge",
+    "prediction_field": "answer",
+    "infer_log_probs": false,
+    "judge_to_generator_fields_mapping": {
+        "ground_truths": "reference_answers"
+    }
+}
diff --git a/...atalog/metrics/rag/external_rag/answer_correctness/mistral_large_instruct_rits_judge.json b/...atalog/metrics/rag/external_rag/answer_correctness/mistral_large_instruct_rits_judge.json
@@ -0,0 +1,11 @@
+{
+    "__type__": "task_based_ll_mas_judge",
+    "inference_model": "engines.classification.mistral_large_instruct_2407_rits",
+    "template": "templates.rag_eval.answer_correctness.judge_loose_match_no_context_numeric",
+    "task": "tasks.rag_eval.answer_correctness.binary",
+    "format": null,
+    "main_score": "answer_correctness_judge",
+    "prediction_field": "answer",
+    "infer_log_probs": false,
+    "judge_to_generator_fields_mapping": {}
+}
diff --git a/...log/metrics/rag/external_rag/answer_correctness/mistral_large_instruct_watsonx_judge.json b/...log/metrics/rag/external_rag/answer_correctness/mistral_large_instruct_watsonx_judge.json
@@ -0,0 +1,11 @@
+{
+    "__type__": "task_based_ll_mas_judge",
+    "inference_model": "engines.classification.mistral_large_watsonx",
+    "template": "templates.rag_eval.answer_correctness.judge_loose_match_no_context_numeric",
+    "task": "tasks.rag_eval.answer_correctness.binary",
+    "format": null,
+    "main_score": "answer_correctness_judge",
+    "prediction_field": "answer",
+    "infer_log_probs": false,
+    "judge_to_generator_fields_mapping": {}
+}
diff --git a/.../catalog/metrics/rag/external_rag/answer_relevance/mistral_large_instruct_rits_judge.json b/.../catalog/metrics/rag/external_rag/answer_relevance/mistral_large_instruct_rits_judge.json
@@ -0,0 +1,11 @@
+{
+    "__type__": "task_based_ll_mas_judge",
+    "inference_model": "engines.classification.mistral_large_instruct_2407_rits",
+    "template": "templates.rag_eval.answer_relevance.judge_answer_relevance_numeric",
+    "task": "tasks.rag_eval.answer_relevance.binary",
+    "format": null,
+    "main_score": "answer_relevance_judge",
+    "prediction_field": "answer",
+    "infer_log_probs": false,
+    "judge_to_generator_fields_mapping": {}
+}
diff --git a/...talog/metrics/rag/external_rag/answer_relevance/mistral_large_instruct_watsonx_judge.json b/...talog/metrics/rag/external_rag/answer_relevance/mistral_large_instruct_watsonx_judge.json
@@ -0,0 +1,11 @@
+{
+    "__type__": "task_based_ll_mas_judge",
+    "inference_model": "engines.classification.mistral_large_watsonx",
+    "template": "templates.rag_eval.answer_relevance.judge_answer_relevance_numeric",
+    "task": "tasks.rag_eval.answer_relevance.binary",
+    "format": null,
+    "main_score": "answer_relevance_judge",
+    "prediction_field": "answer",
+    "infer_log_probs": false,
+    "judge_to_generator_fields_mapping": {}
+}
diff --git a/...catalog/metrics/rag/external_rag/context_relevance/mistral_large_instruct_rits_judge.json b/...catalog/metrics/rag/external_rag/context_relevance/mistral_large_instruct_rits_judge.json
@@ -0,0 +1,11 @@
+{
+    "__type__": "task_based_ll_mas_judge",
+    "inference_model": "engines.classification.mistral_large_instruct_2407_rits",
+    "template": "templates.rag_eval.context_relevance.judge_context_relevance_ares_numeric",
+    "task": "tasks.rag_eval.context_relevance.binary",
+    "format": null,
+    "main_score": "context_relevance_judge",
+    "prediction_field": "contexts",
+    "infer_log_probs": false,
+    "judge_to_generator_fields_mapping": {}
+}
diff --git a/...alog/metrics/rag/external_rag/context_relevance/mistral_large_instruct_watsonx_judge.json b/...alog/metrics/rag/external_rag/context_relevance/mistral_large_instruct_watsonx_judge.json
@@ -0,0 +1,11 @@
+{
+    "__type__": "task_based_ll_mas_judge",
+    "inference_model": "engines.classification.mistral_large_watsonx",
+    "template": "templates.rag_eval.context_relevance.judge_context_relevance_ares_numeric",
+    "task": "tasks.rag_eval.context_relevance.binary",
+    "format": null,
+    "main_score": "context_relevance_judge",
+    "prediction_field": "contexts",
+    "infer_log_probs": false,
+    "judge_to_generator_fields_mapping": {}
+}
diff --git a/...itxt/catalog/metrics/rag/external_rag/faithfulness/mistral_large_instruct_rits_judge.json b/...itxt/catalog/metrics/rag/external_rag/faithfulness/mistral_large_instruct_rits_judge.json
@@ -0,0 +1,11 @@
+{
+    "__type__": "task_based_ll_mas_judge",
+    "inference_model": "engines.classification.mistral_large_instruct_2407_rits",
+    "template": "templates.rag_eval.faithfulness.judge_with_question_simplified_verbal",
+    "task": "tasks.rag_eval.faithfulness.binary",
+    "format": null,
+    "main_score": "faithfulness_judge",
+    "prediction_field": "answer",
+    "infer_log_probs": false,
+    "judge_to_generator_fields_mapping": {}
+}
diff --git a/...t/catalog/metrics/rag/external_rag/faithfulness/mistral_large_instruct_watsonx_judge.json b/...t/catalog/metrics/rag/external_rag/faithfulness/mistral_large_instruct_watsonx_judge.json
@@ -0,0 +1,11 @@
+{
+    "__type__": "task_based_ll_mas_judge",
+    "inference_model": "engines.classification.mistral_large_watsonx",
+    "template": "templates.rag_eval.faithfulness.judge_with_question_simplified_verbal",
+    "task": "tasks.rag_eval.faithfulness.binary",
+    "format": null,
+    "main_score": "faithfulness_judge",
+    "prediction_field": "answer",
+    "infer_log_probs": false,
+    "judge_to_generator_fields_mapping": {}
+}
diff --git a/...metrics/rag/response_generation/answer_correctness/mistral_large_instruct_rits_judge.json b/...metrics/rag/response_generation/answer_correctness/mistral_large_instruct_rits_judge.json
@@ -0,0 +1,13 @@
+{
+    "__type__": "task_based_ll_mas_judge",
+    "inference_model": "engines.classification.mistral_large_instruct_2407_rits",
+    "template": "templates.rag_eval.answer_correctness.judge_loose_match_no_context_numeric",
+    "task": "tasks.rag_eval.answer_correctness.binary",
+    "format": null,
+    "main_score": "answer_correctness_judge",
+    "prediction_field": "answer",
+    "infer_log_probs": false,
+    "judge_to_generator_fields_mapping": {
+        "ground_truths": "reference_answers"
+    }
+}
diff --git a/...rics/rag/response_generation/answer_correctness/mistral_large_instruct_watsonx_judge.json b/...rics/rag/response_generation/answer_correctness/mistral_large_instruct_watsonx_judge.json
@@ -0,0 +1,13 @@
+{
+    "__type__": "task_based_ll_mas_judge",
+    "inference_model": "engines.classification.mistral_large_watsonx",
+    "template": "templates.rag_eval.answer_correctness.judge_loose_match_no_context_numeric",
+    "task": "tasks.rag_eval.answer_correctness.binary",
+    "format": null,
+    "main_score": "answer_correctness_judge",
+    "prediction_field": "answer",
+    "infer_log_probs": false,
+    "judge_to_generator_fields_mapping": {
+        "ground_truths": "reference_answers"
+    }
+}
diff --git a/...g/metrics/rag/response_generation/answer_relevance/mistral_large_instruct_rits_judge.json b/...g/metrics/rag/response_generation/answer_relevance/mistral_large_instruct_rits_judge.json
@@ -0,0 +1,13 @@
+{
+    "__type__": "task_based_ll_mas_judge",
+    "inference_model": "engines.classification.mistral_large_instruct_2407_rits",
+    "template": "templates.rag_eval.answer_relevance.judge_answer_relevance_numeric",
+    "task": "tasks.rag_eval.answer_relevance.binary",
+    "format": null,
+    "main_score": "answer_relevance_judge",
+    "prediction_field": "answer",
+    "infer_log_probs": false,
+    "judge_to_generator_fields_mapping": {
+        "ground_truths": "reference_answers"
+    }
+}
diff --git a/...etrics/rag/response_generation/answer_relevance/mistral_large_instruct_watsonx_judge.json b/...etrics/rag/response_generation/answer_relevance/mistral_large_instruct_watsonx_judge.json
@@ -0,0 +1,13 @@
+{
+    "__type__": "task_based_ll_mas_judge",
+    "inference_model": "engines.classification.mistral_large_watsonx",
+    "template": "templates.rag_eval.answer_relevance.judge_answer_relevance_numeric",
+    "task": "tasks.rag_eval.answer_relevance.binary",
+    "format": null,
+    "main_score": "answer_relevance_judge",
+    "prediction_field": "answer",
+    "infer_log_probs": false,
+    "judge_to_generator_fields_mapping": {
+        "ground_truths": "reference_answers"
+    }
+}
diff --git a/...talog/metrics/rag/response_generation/faithfulness/mistral_large_instruct_rits_judge.json b/...talog/metrics/rag/response_generation/faithfulness/mistral_large_instruct_rits_judge.json
@@ -0,0 +1,13 @@
+{
+    "__type__": "task_based_ll_mas_judge",
+    "inference_model": "engines.classification.mistral_large_instruct_2407_rits",
+    "template": "templates.rag_eval.faithfulness.judge_with_question_simplified_verbal",
+    "task": "tasks.rag_eval.faithfulness.binary",
+    "format": null,
+    "main_score": "faithfulness_judge",
+    "prediction_field": "answer",
+    "infer_log_probs": false,
+    "judge_to_generator_fields_mapping": {
+        "ground_truths": "reference_answers"
+    }
+}
diff --git a/...og/metrics/rag/response_generation/faithfulness/mistral_large_instruct_watsonx_judge.json b/...og/metrics/rag/response_generation/faithfulness/mistral_large_instruct_watsonx_judge.json
@@ -0,0 +1,13 @@
+{
+    "__type__": "task_based_ll_mas_judge",
+    "inference_model": "engines.classification.mistral_large_watsonx",
+    "template": "templates.rag_eval.faithfulness.judge_with_question_simplified_verbal",
+    "task": "tasks.rag_eval.faithfulness.binary",
+    "format": null,
+    "main_score": "faithfulness_judge",
+    "prediction_field": "answer",
+    "infer_log_probs": false,
+    "judge_to_generator_fields_mapping": {
+        "ground_truths": "reference_answers"
+    }
+}