From 76c2e534221bf9d76408e38af180b4b2369d8f82 Mon Sep 17 00:00:00 2001 From: shijinpjlab Date: Mon, 16 Jun 2025 15:43:07 +0800 Subject: [PATCH 1/7] feat: change process_batch place --- dingo/exec/local.py | 105 ++++++++++++++++++++++---------------------- 1 file changed, 53 insertions(+), 52 deletions(-) diff --git a/dingo/exec/local.py b/dingo/exec/local.py index 941db330..64c3d41f 100644 --- a/dingo/exec/local.py +++ b/dingo/exec/local.py @@ -82,6 +82,59 @@ def evaluate(self): group (Any): _description_ group_type (str): _description_ """ + + def process_batch(batch: List): + futures = [] + for group_type, group in Model.get_group( + self.input_args.eval_group + ).items(): + if group_type == "rule": + if os.environ.get("LOCAL_DEPLOYMENT_MODE") == "true": + futures += [ + thread_executor.submit( + self.evaluate_single_data, group_type, group, data + ) + for data in batch + ] + else: + futures += [ + process_executor.submit( + self.evaluate_single_data, group_type, group, data + ) + for data in batch + ] + elif group_type == "prompt": + futures += [ + thread_executor.submit( + self.evaluate_single_data, group_type, group, data + ) + for data in batch + ] + else: + raise RuntimeError(f"Unsupported group type: {group_type}") + + for future in concurrent.futures.as_completed(futures): + result_info = future.result() + for t in result_info.type_list: + self.summary.type_ratio[t] += 1 + for n in result_info.name_list: + self.summary.name_ratio[n] += 1 + if result_info.error_status: + self.summary.num_bad += 1 + else: + self.summary.num_good += 1 + self.summary.total += 1 + + self.write_single_data( + self.summary.output_path, self.input_args, result_info + ) + pbar.update() + self.write_summary( + self.summary.output_path, + self.input_args, + self.summarize(self.summary), + ) + with concurrent.futures.ThreadPoolExecutor( max_workers=self.input_args.max_workers ) as thread_executor, concurrent.futures.ProcessPoolExecutor( @@ -95,58 +148,6 @@ def evaluate(self): ) pbar = tqdm(total=None, unit="items") - def process_batch(batch: List): - futures = [] - for group_type, group in Model.get_group( - self.input_args.eval_group - ).items(): - if group_type == "rule": - if os.environ.get("LOCAL_DEPLOYMENT_MODE") == "true": - futures += [ - thread_executor.submit( - self.evaluate_single_data, group_type, group, data - ) - for data in batch - ] - else: - futures += [ - process_executor.submit( - self.evaluate_single_data, group_type, group, data - ) - for data in batch - ] - elif group_type == "prompt": - futures += [ - thread_executor.submit( - self.evaluate_single_data, group_type, group, data - ) - for data in batch - ] - else: - raise RuntimeError(f"Unsupported group type: {group_type}") - - for future in concurrent.futures.as_completed(futures): - result_info = future.result() - for t in result_info.type_list: - self.summary.type_ratio[t] += 1 - for n in result_info.name_list: - self.summary.name_ratio[n] += 1 - if result_info.error_status: - self.summary.num_bad += 1 - else: - self.summary.num_good += 1 - self.summary.total += 1 - - self.write_single_data( - self.summary.output_path, self.input_args, result_info - ) - pbar.update() - self.write_summary( - self.summary.output_path, - self.input_args, - self.summarize(self.summary), - ) - while True: batch = list(itertools.islice(data_iter, self.input_args.batch_size)) if not batch: From c30d42d34d3f17f58f40ae9393e0ba794f6a8581 Mon Sep 17 00:00:00 2001 From: shijinpjlab Date: Mon, 16 Jun 2025 16:23:06 +0800 Subject: [PATCH 2/7] feat: merge rule and prompt result info --- dingo/exec/local.py | 60 +++++++++++++++++++++++++++++---------------- 1 file changed, 39 insertions(+), 21 deletions(-) diff --git a/dingo/exec/local.py b/dingo/exec/local.py index 64c3d41f..391aef54 100644 --- a/dingo/exec/local.py +++ b/dingo/exec/local.py @@ -83,38 +83,56 @@ def evaluate(self): group_type (str): _description_ """ + def merge_result_info(existing_list: List[ResultInfo], new_item: ResultInfo) -> List[ResultInfo]: + existing_item = next((item for item in existing_list if item.data_id == new_item.data_id), None) + + if existing_item: + existing_item.error_status = existing_item.error_status or new_item.error_status + + existing_item.type_list = list(set(existing_item.type_list + new_item.type_list)) + existing_item.name_list = list(set(existing_item.name_list + new_item.name_list)) + existing_item.reason_list = list(set(existing_item.reason_list + new_item.reason_list)) + + existing_item.raw_data = new_item.raw_data + else: + existing_list.append(new_item) + + return existing_list + def process_batch(batch: List): futures = [] - for group_type, group in Model.get_group( - self.input_args.eval_group - ).items(): - if group_type == "rule": - if os.environ.get("LOCAL_DEPLOYMENT_MODE") == "true": + futures_results = [] + for data in batch: + for group_type, group in Model.get_group( + self.input_args.eval_group + ).items(): + if group_type == "rule": + if os.environ.get("LOCAL_DEPLOYMENT_MODE") == "true": + futures += [ + thread_executor.submit( + self.evaluate_single_data, group_type, group, data + ) + ] + else: + futures += [ + process_executor.submit( + self.evaluate_single_data, group_type, group, data + ) + ] + elif group_type == "prompt": futures += [ thread_executor.submit( self.evaluate_single_data, group_type, group, data ) - for data in batch ] else: - futures += [ - process_executor.submit( - self.evaluate_single_data, group_type, group, data - ) - for data in batch - ] - elif group_type == "prompt": - futures += [ - thread_executor.submit( - self.evaluate_single_data, group_type, group, data - ) - for data in batch - ] - else: - raise RuntimeError(f"Unsupported group type: {group_type}") + raise RuntimeError(f"Unsupported group type: {group_type}") for future in concurrent.futures.as_completed(futures): result_info = future.result() + futures_results = merge_result_info(futures_results, result_info) + + for result_info in futures_results: for t in result_info.type_list: self.summary.type_ratio[t] += 1 for n in result_info.name_list: From c09821b660fabd16fd8f14f336fd88f9c86631dd Mon Sep 17 00:00:00 2001 From: shijinpjlab Date: Mon, 16 Jun 2025 16:56:41 +0800 Subject: [PATCH 3/7] feat: describe the way of calculate ratio in md --- README.md | 5 +++++ README_ja.md | 5 +++++ README_zh-CN.md | 5 +++++ 3 files changed, 15 insertions(+) diff --git a/README.md b/README.md index fd7dd745..f558b460 100644 --- a/README.md +++ b/README.md @@ -388,6 +388,11 @@ After evaluation, Dingo generates: 1. **Summary Report** (`summary.json`): Overall metrics and scores 2. **Detailed Reports**: Specific issues for each rule violation +Warning: +1. **score**: `num_good` / `total` +2. **type_ratio**: The count of type / total, such as: `QUALITY_BAD_COMPLETENESS` / `total` +3. **name_ratio**: The count of name / total, such as: `QUALITY_BAD_COMPLETENESS-RuleColonEnd` / `total` + Example summary: ```json { diff --git a/README_ja.md b/README_ja.md index 8b9c7a46..54e0df8c 100644 --- a/README_ja.md +++ b/README_ja.md @@ -386,6 +386,11 @@ result = executor.execute() 1. **サマリーレポート** (`summary.json`): 全体的なメトリクスとスコア 2. **詳細レポート**: 各ルール違反の具体的な問題 +に注意： +1. **score**: `num_good` / `total` +2. **type_ratio**: タイプの数 / 総数, 例: `QUALITY_BAD_COMPLETENESS` / `total` +3. **name_ratio**: 名前の数 / 総数, 例: `QUALITY_BAD_COMPLETENESS-RuleColonEnd` / `total` + サマリー例： ```json { diff --git a/README_zh-CN.md b/README_zh-CN.md index d9004310..68627c61 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -387,6 +387,11 @@ result = executor.execute() 1. **概要报告**（`summary.json`）：总体指标和分数 2. **详细报告**：每个规则违反的具体问题 +注意： +1. **score**: `num_good` / `total` +2. **type_ratio**: 类型的数量 / 总数, 例如: `QUALITY_BAD_COMPLETENESS` / `total` +3. **name_ratio**: 名称的数量 / 总数, 例如: `QUALITY_BAD_COMPLETENESS-RuleColonEnd` / `total` + 概要示例： ```json { From 3dc3bbbbbac5270cfa22e4747c56063f42c80068 Mon Sep 17 00:00:00 2001 From: shijinpjlab Date: Mon, 16 Jun 2025 17:06:05 +0800 Subject: [PATCH 4/7] feat: update readme --- README.md | 2 +- README_ja.md | 2 +- README_zh-CN.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f558b460..363062ba 100644 --- a/README.md +++ b/README.md @@ -388,7 +388,7 @@ After evaluation, Dingo generates: 1. **Summary Report** (`summary.json`): Overall metrics and scores 2. **Detailed Reports**: Specific issues for each rule violation -Warning: +Report Description: 1. **score**: `num_good` / `total` 2. **type_ratio**: The count of type / total, such as: `QUALITY_BAD_COMPLETENESS` / `total` 3. **name_ratio**: The count of name / total, such as: `QUALITY_BAD_COMPLETENESS-RuleColonEnd` / `total` diff --git a/README_ja.md b/README_ja.md index 54e0df8c..bf111e98 100644 --- a/README_ja.md +++ b/README_ja.md @@ -386,7 +386,7 @@ result = executor.execute() 1. **サマリーレポート** (`summary.json`): 全体的なメトリクスとスコア 2. **詳細レポート**: 各ルール違反の具体的な問題 -に注意： +レポートの説明： 1. **score**: `num_good` / `total` 2. **type_ratio**: タイプの数 / 総数, 例: `QUALITY_BAD_COMPLETENESS` / `total` 3. **name_ratio**: 名前の数 / 総数, 例: `QUALITY_BAD_COMPLETENESS-RuleColonEnd` / `total` diff --git a/README_zh-CN.md b/README_zh-CN.md index 68627c61..203f526e 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -387,7 +387,7 @@ result = executor.execute() 1. **概要报告**（`summary.json`）：总体指标和分数 2. **详细报告**：每个规则违反的具体问题 -注意： +报告说明： 1. **score**: `num_good` / `total` 2. **type_ratio**: 类型的数量 / 总数, 例如: `QUALITY_BAD_COMPLETENESS` / `total` 3. **name_ratio**: 名称的数量 / 总数, 例如: `QUALITY_BAD_COMPLETENESS-RuleColonEnd` / `total` From 9cbc8b797f1a5985dc9ffe530a9d5163cb941b41 Mon Sep 17 00:00:00 2001 From: shijinpjlab Date: Mon, 16 Jun 2025 17:58:45 +0800 Subject: [PATCH 5/7] feat: 1. change `merge_result_info` to class func 2. add init in ResultInfo 3. add test local --- dingo/exec/local.py | 141 ++++++++++++++++---------------- dingo/io/output/ResultInfo.py | 6 ++ test/scripts/exec/test_local.py | 45 ++++++++++ 3 files changed, 120 insertions(+), 72 deletions(-) create mode 100644 test/scripts/exec/test_local.py diff --git a/dingo/exec/local.py b/dingo/exec/local.py index 391aef54..d55bb9c3 100644 --- a/dingo/exec/local.py +++ b/dingo/exec/local.py @@ -75,6 +75,22 @@ def execute(self) -> SummaryModel: return self.summary + def merge_result_info(self, existing_list: List[ResultInfo], new_item: ResultInfo) -> List[ResultInfo]: + existing_item = next((item for item in existing_list if item.data_id == new_item.data_id), None) + + if existing_item: + existing_item.error_status = existing_item.error_status or new_item.error_status + + existing_item.type_list = list(set(existing_item.type_list + new_item.type_list)) + existing_item.name_list = list(set(existing_item.name_list + new_item.name_list)) + existing_item.reason_list = list(set(existing_item.reason_list + new_item.reason_list)) + + existing_item.raw_data = new_item.raw_data + else: + existing_list.append(new_item) + + return existing_list + def evaluate(self): """ get score (main progres). @@ -82,77 +98,6 @@ def evaluate(self): group (Any): _description_ group_type (str): _description_ """ - - def merge_result_info(existing_list: List[ResultInfo], new_item: ResultInfo) -> List[ResultInfo]: - existing_item = next((item for item in existing_list if item.data_id == new_item.data_id), None) - - if existing_item: - existing_item.error_status = existing_item.error_status or new_item.error_status - - existing_item.type_list = list(set(existing_item.type_list + new_item.type_list)) - existing_item.name_list = list(set(existing_item.name_list + new_item.name_list)) - existing_item.reason_list = list(set(existing_item.reason_list + new_item.reason_list)) - - existing_item.raw_data = new_item.raw_data - else: - existing_list.append(new_item) - - return existing_list - - def process_batch(batch: List): - futures = [] - futures_results = [] - for data in batch: - for group_type, group in Model.get_group( - self.input_args.eval_group - ).items(): - if group_type == "rule": - if os.environ.get("LOCAL_DEPLOYMENT_MODE") == "true": - futures += [ - thread_executor.submit( - self.evaluate_single_data, group_type, group, data - ) - ] - else: - futures += [ - process_executor.submit( - self.evaluate_single_data, group_type, group, data - ) - ] - elif group_type == "prompt": - futures += [ - thread_executor.submit( - self.evaluate_single_data, group_type, group, data - ) - ] - else: - raise RuntimeError(f"Unsupported group type: {group_type}") - - for future in concurrent.futures.as_completed(futures): - result_info = future.result() - futures_results = merge_result_info(futures_results, result_info) - - for result_info in futures_results: - for t in result_info.type_list: - self.summary.type_ratio[t] += 1 - for n in result_info.name_list: - self.summary.name_ratio[n] += 1 - if result_info.error_status: - self.summary.num_bad += 1 - else: - self.summary.num_good += 1 - self.summary.total += 1 - - self.write_single_data( - self.summary.output_path, self.input_args, result_info - ) - pbar.update() - self.write_summary( - self.summary.output_path, - self.input_args, - self.summarize(self.summary), - ) - with concurrent.futures.ThreadPoolExecutor( max_workers=self.input_args.max_workers ) as thread_executor, concurrent.futures.ProcessPoolExecutor( @@ -170,7 +115,59 @@ def process_batch(batch: List): batch = list(itertools.islice(data_iter, self.input_args.batch_size)) if not batch: break - process_batch(batch) + + futures = [] + futures_results = [] + for data in batch: + for group_type, group in Model.get_group( + self.input_args.eval_group + ).items(): + if group_type == "rule": + if os.environ.get("LOCAL_DEPLOYMENT_MODE") == "true": + futures += [ + thread_executor.submit( + self.evaluate_single_data, group_type, group, data + ) + ] + else: + futures += [ + process_executor.submit( + self.evaluate_single_data, group_type, group, data + ) + ] + elif group_type == "prompt": + futures += [ + thread_executor.submit( + self.evaluate_single_data, group_type, group, data + ) + ] + else: + raise RuntimeError(f"Unsupported group type: {group_type}") + + for future in concurrent.futures.as_completed(futures): + result_info = future.result() + futures_results = self.merge_result_info(futures_results, result_info) + + for result_info in futures_results: + for t in result_info.type_list: + self.summary.type_ratio[t] += 1 + for n in result_info.name_list: + self.summary.name_ratio[n] += 1 + if result_info.error_status: + self.summary.num_bad += 1 + else: + self.summary.num_good += 1 + self.summary.total += 1 + + self.write_single_data( + self.summary.output_path, self.input_args, result_info + ) + pbar.update() + self.write_summary( + self.summary.output_path, + self.input_args, + self.summarize(self.summary), + ) log.debug("[Summary]: " + str(self.summary)) diff --git a/dingo/io/output/ResultInfo.py b/dingo/io/output/ResultInfo.py index 7b68f103..c6c2527f 100644 --- a/dingo/io/output/ResultInfo.py +++ b/dingo/io/output/ResultInfo.py @@ -13,6 +13,12 @@ class ResultInfo(BaseModel): reason_list: List[str] = [] raw_data: Dict = {} + def __init__(self, data: Dict): + super().__init__(**{ + **self.__class__.model_fields, + **data + }) + def to_dict(self): return { 'data_id': self.data_id, diff --git a/test/scripts/exec/test_local.py b/test/scripts/exec/test_local.py new file mode 100644 index 00000000..8c86da0f --- /dev/null +++ b/test/scripts/exec/test_local.py @@ -0,0 +1,45 @@ +import pytest + +from dingo.exec import LocalExecutor +from dingo.io import ResultInfo + +class TestLocal: + def test_merge_result_info(self): + existing_list = [] + new_item1 = ResultInfo({ + "data_id": "1", + "prompt": "", + "content": "�I am 8 years old. ^I love apple because:", + "error_status": True, + "type_list": ["QUALITY_BAD_EFFECTIVENESS"], + "name_list": ["QUALITY_BAD_EFFECTIVENESS-RuleColonEnd"], + "reason_list": ["�I am 8 years old. ^I love apple because:"], + "raw_data": {} + }) + new_item2 = ResultInfo({ + "data_id": "1", + "prompt": "", + "content": "�I am 8 years old. ^I love apple because:", + "error_status": True, + "type_list": ["QUALITY_BAD_EFFECTIVENESS"], + "name_list": ["QUALITY_BAD_EFFECTIVENESS-PromptContentChaos"], + "reason_list": ["文本中包含不可见字符或乱码（如�和^），可能影响阅读理解。"], + "raw_data": {} + }) + + localexecutor = LocalExecutor({}) + + new_existing_list = localexecutor.merge_result_info(existing_list, new_item1) + assert new_existing_list[0] == new_item1 + + new_existing_list = localexecutor.merge_result_info(existing_list, new_item1) + new_existing_list = localexecutor.merge_result_info(new_existing_list, new_item2) + assert len(new_existing_list) == 1 + assert len(new_existing_list[0].type_list) == 1 + assert len(new_existing_list[0].name_list) == 2 + assert len(new_existing_list[0].reason_list) == 2 + assert "QUALITY_BAD_EFFECTIVENESS" in new_existing_list[0].type_list + assert "QUALITY_BAD_EFFECTIVENESS-RuleColonEnd" in new_existing_list[0].name_list + assert "QUALITY_BAD_EFFECTIVENESS-PromptContentChaos" in new_existing_list[0].name_list + assert "�I am 8 years old. ^I love apple because:" in new_existing_list[0].reason_list + assert "文本中包含不可见字符或乱码（如�和^），可能影响阅读理解。" in new_existing_list[0].reason_list From 14d37bcb343c0e1eff276a257ec5a46e997c0b6b Mon Sep 17 00:00:00 2001 From: shijinpjlab Date: Mon, 16 Jun 2025 18:02:51 +0800 Subject: [PATCH 6/7] feat: fix lint --- test/scripts/exec/test_local.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/scripts/exec/test_local.py b/test/scripts/exec/test_local.py index 8c86da0f..da8fca38 100644 --- a/test/scripts/exec/test_local.py +++ b/test/scripts/exec/test_local.py @@ -1,8 +1,8 @@ import pytest - from dingo.exec import LocalExecutor from dingo.io import ResultInfo + class TestLocal: def test_merge_result_info(self): existing_list = [] From 409c10b9115286b39fa41ce001a4b411f5d696fd Mon Sep 17 00:00:00 2001 From: shijinpjlab Date: Mon, 16 Jun 2025 18:15:40 +0800 Subject: [PATCH 7/7] feat: remove init --- dingo/io/output/ResultInfo.py | 6 ----- test/scripts/exec/test_local.py | 40 ++++++++++++++++----------------- 2 files changed, 20 insertions(+), 26 deletions(-) diff --git a/dingo/io/output/ResultInfo.py b/dingo/io/output/ResultInfo.py index c6c2527f..7b68f103 100644 --- a/dingo/io/output/ResultInfo.py +++ b/dingo/io/output/ResultInfo.py @@ -13,12 +13,6 @@ class ResultInfo(BaseModel): reason_list: List[str] = [] raw_data: Dict = {} - def __init__(self, data: Dict): - super().__init__(**{ - **self.__class__.model_fields, - **data - }) - def to_dict(self): return { 'data_id': self.data_id, diff --git a/test/scripts/exec/test_local.py b/test/scripts/exec/test_local.py index da8fca38..fcd31710 100644 --- a/test/scripts/exec/test_local.py +++ b/test/scripts/exec/test_local.py @@ -6,26 +6,26 @@ class TestLocal: def test_merge_result_info(self): existing_list = [] - new_item1 = ResultInfo({ - "data_id": "1", - "prompt": "", - "content": "�I am 8 years old. ^I love apple because:", - "error_status": True, - "type_list": ["QUALITY_BAD_EFFECTIVENESS"], - "name_list": ["QUALITY_BAD_EFFECTIVENESS-RuleColonEnd"], - "reason_list": ["�I am 8 years old. ^I love apple because:"], - "raw_data": {} - }) - new_item2 = ResultInfo({ - "data_id": "1", - "prompt": "", - "content": "�I am 8 years old. ^I love apple because:", - "error_status": True, - "type_list": ["QUALITY_BAD_EFFECTIVENESS"], - "name_list": ["QUALITY_BAD_EFFECTIVENESS-PromptContentChaos"], - "reason_list": ["文本中包含不可见字符或乱码（如�和^），可能影响阅读理解。"], - "raw_data": {} - }) + new_item1 = ResultInfo( + data_id = "1", + prompt = "", + content = "�I am 8 years old. ^I love apple because:", + error_status = True, + type_list = ["QUALITY_BAD_EFFECTIVENESS"], + name_list = ["QUALITY_BAD_EFFECTIVENESS-RuleColonEnd"], + reason_list = ["�I am 8 years old. ^I love apple because:"], + raw_data = {} + ) + new_item2 = ResultInfo( + data_id = "1", + prompt = "", + content = "�I am 8 years old. ^I love apple because:", + error_status = True, + type_list = ["QUALITY_BAD_EFFECTIVENESS"], + name_list = ["QUALITY_BAD_EFFECTIVENESS-PromptContentChaos"], + reason_list = ["文本中包含不可见字符或乱码（如�和^），可能影响阅读理解。"], + raw_data = {} + ) localexecutor = LocalExecutor({})