Merge rule and prompt Result Info.

README.md

@@ -388,6 +388,11 @@ After evaluation, Dingo generates:
 1. **Summary Report** (`summary.json`): Overall metrics and scores
 2. **Detailed Reports**: Specific issues for each rule violation
 
+Report Description:
+1. **score**: `num_good` / `total`
+2. **type_ratio**: The count of type / total, such as: `QUALITY_BAD_COMPLETENESS` / `total`
+3. **name_ratio**: The count of name / total, such as: `QUALITY_BAD_COMPLETENESS-RuleColonEnd` / `total`
+
 Example summary:
 ```json
 {

README_ja.md

@@ -386,6 +386,11 @@ result = executor.execute()
 1. **サマリーレポート** (`summary.json`): 全体的なメトリクスとスコア
 2. **詳細レポート**: 各ルール違反の具体的な問題
 
+レポートの説明：
+1. **score**: `num_good` / `total`
+2. **type_ratio**: タイプの数 / 総数, 例: `QUALITY_BAD_COMPLETENESS` / `total`
+3. **name_ratio**: 名前の数 / 総数, 例: `QUALITY_BAD_COMPLETENESS-RuleColonEnd` / `total`
+
 サマリー例：
 ```json
 {

README_zh-CN.md

@@ -387,6 +387,11 @@ result = executor.execute()
 1. **概要报告**（`summary.json`）：总体指标和分数
 2. **详细报告**：每个规则违反的具体问题
 
+报告说明：
+1. **score**: `num_good` / `total`
+2. **type_ratio**: 类型的数量 / 总数, 例如: `QUALITY_BAD_COMPLETENESS` / `total`
+3. **name_ratio**: 名称的数量 / 总数, 例如: `QUALITY_BAD_COMPLETENESS-RuleColonEnd` / `total`
+
 概要示例：
 ```json
 {

dingo/exec/local.py

@@ -75,6 +75,22 @@ def execute(self) -> SummaryModel:
 
         return self.summary
 
+    def merge_result_info(self, existing_list: List[ResultInfo], new_item: ResultInfo) -> List[ResultInfo]:
+        existing_item = next((item for item in existing_list if item.data_id == new_item.data_id), None)
+
+        if existing_item:
+            existing_item.error_status = existing_item.error_status or new_item.error_status
+
+            existing_item.type_list = list(set(existing_item.type_list + new_item.type_list))
+            existing_item.name_list = list(set(existing_item.name_list + new_item.name_list))
+            existing_item.reason_list = list(set(existing_item.reason_list + new_item.reason_list))
+
+            existing_item.raw_data = new_item.raw_data
+        else:
+            existing_list.append(new_item)
+
+        return existing_list
+
     def evaluate(self):
         """
         get score (main progres).
@@ -95,38 +111,44 @@ def evaluate(self):
             )
             pbar = tqdm(total=None, unit="items")
 
-            def process_batch(batch: List):
+            while True:
+                batch = list(itertools.islice(data_iter, self.input_args.batch_size))
+                if not batch:
+                    break
+
                 futures = []
-                for group_type, group in Model.get_group(
-                    self.input_args.eval_group
-                ).items():
-                    if group_type == "rule":
-                        if os.environ.get("LOCAL_DEPLOYMENT_MODE") == "true":
+                futures_results = []
+                for data in batch:
+                    for group_type, group in Model.get_group(
+                            self.input_args.eval_group
+                    ).items():
+                        if group_type == "rule":
+                            if os.environ.get("LOCAL_DEPLOYMENT_MODE") == "true":
+                                futures += [
+                                    thread_executor.submit(
+                                        self.evaluate_single_data, group_type, group, data
+                                    )
+                                ]
+                            else:
+                                futures += [
+                                    process_executor.submit(
+                                        self.evaluate_single_data, group_type, group, data
+                                    )
+                                ]
+                        elif group_type == "prompt":
                             futures += [
                                 thread_executor.submit(
                                     self.evaluate_single_data, group_type, group, data
                                 )
-                                for data in batch
                             ]
                         else:
-                            futures += [
-                                process_executor.submit(
-                                    self.evaluate_single_data, group_type, group, data
-                                )
-                                for data in batch
-                            ]
-                    elif group_type == "prompt":
-                        futures += [
-                            thread_executor.submit(
-                                self.evaluate_single_data, group_type, group, data
-                            )
-                            for data in batch
-                        ]
-                    else:
-                        raise RuntimeError(f"Unsupported group type: {group_type}")
+                            raise RuntimeError(f"Unsupported group type: {group_type}")
 
                 for future in concurrent.futures.as_completed(futures):
                     result_info = future.result()
+                    futures_results = self.merge_result_info(futures_results, result_info)
+
+                for result_info in futures_results:
                     for t in result_info.type_list:
                         self.summary.type_ratio[t] += 1
                     for n in result_info.name_list:
@@ -147,12 +169,6 @@ def process_batch(batch: List):
                     self.summarize(self.summary),
                 )
 
-            while True:
-                batch = list(itertools.islice(data_iter, self.input_args.batch_size))
-                if not batch:
-                    break
-                process_batch(batch)
-
         log.debug("[Summary]: " + str(self.summary))
 
     def evaluate_single_data(self, group_type, group, data: Data):

test/scripts/exec/test_local.py

@@ -0,0 +1,45 @@
+import pytest
+from dingo.exec import LocalExecutor
+from dingo.io import ResultInfo
+
+
+class TestLocal:
+    def test_merge_result_info(self):
+        existing_list = []
+        new_item1 = ResultInfo(
+            data_id = "1",
+            prompt = "",
+            content = "�I am 8 years old. ^I love apple because:",
+            error_status = True,
+            type_list = ["QUALITY_BAD_EFFECTIVENESS"],
+            name_list = ["QUALITY_BAD_EFFECTIVENESS-RuleColonEnd"],
+            reason_list = ["�I am 8 years old. ^I love apple because:"],
+            raw_data = {}
+        )
+        new_item2 = ResultInfo(
+            data_id = "1",
+            prompt = "",
+            content = "�I am 8 years old. ^I love apple because:",
+            error_status = True,
+            type_list = ["QUALITY_BAD_EFFECTIVENESS"],
+            name_list = ["QUALITY_BAD_EFFECTIVENESS-PromptContentChaos"],
+            reason_list = ["文本中包含不可见字符或乱码（如�和^），可能影响阅读理解。"],
+            raw_data = {}
+        )
+
+        localexecutor = LocalExecutor({})
+
+        new_existing_list = localexecutor.merge_result_info(existing_list, new_item1)
+        assert new_existing_list[0] == new_item1
+
+        new_existing_list = localexecutor.merge_result_info(existing_list, new_item1)
+        new_existing_list = localexecutor.merge_result_info(new_existing_list, new_item2)
+        assert len(new_existing_list) == 1
+        assert len(new_existing_list[0].type_list) == 1
+        assert len(new_existing_list[0].name_list) == 2
+        assert len(new_existing_list[0].reason_list) == 2
+        assert "QUALITY_BAD_EFFECTIVENESS" in new_existing_list[0].type_list
+        assert "QUALITY_BAD_EFFECTIVENESS-RuleColonEnd" in new_existing_list[0].name_list
+        assert "QUALITY_BAD_EFFECTIVENESS-PromptContentChaos" in new_existing_list[0].name_list
+        assert "�I am 8 years old. ^I love apple because:" in new_existing_list[0].reason_list
+        assert "文本中包含不可见字符或乱码（如�和^），可能影响阅读理解。" in new_existing_list[0].reason_list