From eda7c3a96a3466231f9a97750cbba630b597f1f5 Mon Sep 17 00:00:00 2001 From: shijinpjlab Date: Tue, 8 Jul 2025 11:28:25 +0800 Subject: [PATCH 01/24] feat: add pkg version (#112) --- requirements/runtime.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/requirements/runtime.txt b/requirements/runtime.txt index a53ff407..0b702234 100644 --- a/requirements/runtime.txt +++ b/requirements/runtime.txt @@ -26,3 +26,6 @@ transformers wordninja==2.0.0 zhon fastmcp>=2.0.0 +twine==6.0.1 +packaging==24.1 +pkginfo==1.12.0 From 411763d97c67e53ed85bcc92f56dae2f69c8f01d Mon Sep 17 00:00:00 2001 From: shijinpjlab Date: Tue, 8 Jul 2025 16:31:31 +0800 Subject: [PATCH 02/24] feat: git publish ci dependency (#113) --- .github/workflows/publish-to-pypi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml index 2b4572ed..02578eba 100644 --- a/.github/workflows/publish-to-pypi.yml +++ b/.github/workflows/publish-to-pypi.yml @@ -22,7 +22,7 @@ jobs: - name: Install build dependencies run: | python -m pip install --upgrade pip - pip install build twine + pip install build wheel==0.43.0 twine==6.0.1 packaging==24.1 pkginfo==1.12.0 - name: Build package run: python setup.py bdist_wheel From 28cf95039c5e2eb8d7bac740953dd6d7cc9610bc Mon Sep 17 00:00:00 2001 From: shijinpjlab Date: Wed, 9 Jul 2025 15:36:41 +0800 Subject: [PATCH 03/24] =?UTF-8?q?feat:=20technical=20doc=20=E5=AE=89?= =?UTF-8?q?=E8=A3=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Technical_Doc.md | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 Technical_Doc.md diff --git a/Technical_Doc.md b/Technical_Doc.md new file mode 100644 index 00000000..6f8e2a3d --- /dev/null +++ b/Technical_Doc.md @@ -0,0 +1,41 @@ +# 开始你的第一步 + +## 安装 + +### 基础安装 +1. 使用conda准备 dingo 运行环境: +```shell +conda create --name dingo python=3.10 -y + +conda activate dingo +``` + +2. 安装 dingo: +- pip安装: +```shell +pip install dingo_python +``` + +- 如果希望使用 dingo 的最新功能，也可以从源代码构建它： +```shell +git clone git@github.com:MigoXLab/dingo.git dingo +cd dingo +pip install -e . +``` + +### 进阶安装 +如果想要体验全部的 dingo 功能，那么需要安装所有的可选依赖: +```shell +pip install -r requirements/contribute.txt +pip install -r requirements/docs.txt +pip install -r requirements/optional.txt +pip install -r requirements/web.txt +``` + +# 教程 + +# 规则 + +# 提示词 + +# 进阶教程 \ No newline at end of file From bdd3cf5b9b360b70c29cf4b1fc5bce01491c7cfb Mon Sep 17 00:00:00 2001 From: shijinpjlab Date: Wed, 9 Jul 2025 16:19:37 +0800 Subject: [PATCH 04/24] =?UTF-8?q?feat:=20technical=20doc=20=E5=BF=AB?= =?UTF-8?q?=E9=80=9F=E5=BC=80=E5=A7=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Technical_Doc.md | 62 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/Technical_Doc.md b/Technical_Doc.md index 6f8e2a3d..de3acb97 100644 --- a/Technical_Doc.md +++ b/Technical_Doc.md @@ -32,6 +32,68 @@ pip install -r requirements/optional.txt pip install -r requirements/web.txt ``` +## 快速开始 + +### 概览 +在 dingo 中启动一个评估任务可以通过以下2种方式：命令行、代码。 + +**命令行启动** +```shell +python -m dingo.run.cli + --input_path data.txt + --dataset local + --data_format plaintext + -e sft + --save_data True +``` + +**代码启动** +```python +from dingo.exec import Executor +from dingo.io import InputArgs + +input_data = { + "input_path": "data.txt", + "dataset": "local", + "data_format": "plaintext", + "eval_group": "sft", + "save_data": True +} +input_args = InputArgs(**input_data) +executor = Executor.exec_map["local"](input_args) +result = executor.execute() +print(result) +``` + +### 评估结果 +评估完成后，评估结果将打印如下字段: ++ task_id ++ task_name ++ input_path ++ output_path ++ create_time ++ finish_time ++ score ++ num_good ++ num_bad ++ type_ratio ++ name_ratio + +所有运行输出将定向到 outputs 目录，结构如下： +``` +outputs/ +├── 20250609_101837_50b5c0be +├── 20250609_111057_5d250cf6 # 每个任务一个文件夹 +│ ├── QUALITY_BAD_COMPLETENESS # 评估阶段的一级类型 +│ │ ├── RuleSentenceNumber.jsonl # 评估阶段的二级类型 +│ │ └── RuleWordNumber.jsonl +│ ├── QUALITY_BAD_EFFECTIVENESS +│ │ ├── RuleColonEnd.jsonl +│ │ └── RuleEnterAndSpace.jsonl +│ └── summary.json # 单个任务的汇总结果 +├── ... +``` + # 教程 # 规则 From 1599043b708d028411231aab40f96c02cc01a9e6 Mon Sep 17 00:00:00 2001 From: shijinpjlab Date: Wed, 9 Jul 2025 19:22:47 +0800 Subject: [PATCH 05/24] =?UTF-8?q?feat:=20technical=20doc=20=E6=95=99?= =?UTF-8?q?=E7=A8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Technical_Doc.md | 81 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 80 insertions(+), 1 deletion(-) diff --git a/Technical_Doc.md b/Technical_Doc.md index de3acb97..380a8bb7 100644 --- a/Technical_Doc.md +++ b/Technical_Doc.md @@ -44,7 +44,7 @@ python -m dingo.run.cli --dataset local --data_format plaintext -e sft - --save_data True + --save_data ``` **代码启动** @@ -96,6 +96,85 @@ outputs/ # 教程 +## 整体概括 +本项目的架构可以分为以下3个模块：Data、Evaluator、Executor ++ Data: 负责数据的加载与格式转化 ++ Evaluator: 负责评估的执行 ++ Executor: 负责任务的分配与调度 + +![Architecture of dingo](./docs/assets/architeture.png) + +## 基础配置 +dingo 启动方式具有2种，因此配置方式也分为以下2种情况: +1. [命令行配置列表](docs/config.md#cli-config) +2. [代码配置列表](docs/config.md#sdk-config) + +**命令行启动** +在命令行环境中，所有配置均以 参数键值对 的形式指定，遵从标准 CLI 语法规则，通过 --参数名 参数值 的方式传递每个配置项。 + +```shell +python -m dingo.run.cli + --input_path data.txt + --dataset local + --data_format plaintext + -e sft + --save_data +``` + +**代码启动** +在代码环境中，配置都是 Python 格式的，遵从基本的 Python 语法，通过定义变量的形式指定每个配置项。 + +```python +from dingo.io import InputArgs + +input_data = { + "input_path": "data.txt", + "dataset": "local", + "data_format": "plaintext", + "eval_group": "sft", + "save_data": True +} +input_args = InputArgs(**input_data) +``` + +## 加载数据 +如果想要 dingo 顺利读入数据，那么需要在配置时设置以下参数: +- input_path +- dataset +- data_format + +数据读入后，进入格式转化阶段，此时执行字段的映射，因此需要在配置时设置以下参数: +- column_id +- column_prompt +- column_content + +最终数据以 [Data](dingo/io/input/Data.py) 类对象的形式在项目中流转。 +如果用户在配置时将参数 save_raw 设置为True，那么 Data 类对象的 raw_data 有值否则为空字典。 + +## 设置并发 +dingo 默认状态下没有开启并发，如果有大规模评估任务需要开启并发，那么应该在配置时设置以下参数: ++ max_workers ++ batch_size + +以上2个参数应当搭配使用，如果max_workers设置为10但是batch_size设置为1，那么评估的效率不会得到较大提升。 +建议batch_size大于等于max_workers。 + +## 结果保存 +评估任务完成后会在当前目录下创建 outputs 文件夹并且不保存原始数据格式，除非用户在配置时设置了以下参数: ++ save_data ++ save_raw ++ save_correct ++ output_path + +上文中评估阶段的二级类型jsonl文件中的每条结果数据收到配置参数 save_raw 的影响。 +如果 save_raw 设置为True，那么将执行 [ResultInfo](dingo/io/output/ResultInfo.py) 类的 to_dict_raw 函数，否则将执行 to_dict 函数。 + +## 启动前端页面 +dingo 评估任务结束后，如果保存了评估结果，那么就可以通过一下方式启动前端页面展示结果: +```shell +python -m dingo.run.vsl --input outputs/20250609_101837_50b5c0be +``` + # 规则 # 提示词 From 192b7a99a657879e86ca41684929eb2b45f17561 Mon Sep 17 00:00:00 2001 From: shijinpjlab Date: Thu, 10 Jul 2025 14:55:01 +0800 Subject: [PATCH 06/24] =?UTF-8?q?feat:=20technical=20doc=20=E8=A7=84?= =?UTF-8?q?=E5=88=99=E3=80=81=E6=8F=90=E7=A4=BA=E8=AF=8D=E3=80=81=E5=9C=BA?= =?UTF-8?q?=E6=99=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Technical_Doc.md | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/Technical_Doc.md b/Technical_Doc.md index 380a8bb7..333aa315 100644 --- a/Technical_Doc.md +++ b/Technical_Doc.md @@ -176,7 +176,38 @@ python -m dingo.run.vsl --input outputs/20250609_101837_50b5c0be ``` # 规则 +dingo 内置了不同类型的评估规则，详情见: [规则列表](docs/rules.md)。 +每条评估规则都有自己的 metric_type 和所属的 group。 +每条数据经过规则评估，会产生一个 [ModelRes](dingo/model/modelres.py) 类对象作为结果，一般来说规则的 metric_type 作为 type 而规则名作为 name。 +用户可以通过配置 eval_group 参数来调用该 group 内的所有规则执行评估任务。 如果用户需要组合一批评估规则用来评估，那么请参考下文的 **自定义配置** 。 # 提示词 +dingo 提示词与规则类似，都有 metric_type 和 group ，并且他们的作用也相同。 +但是提示词需要与场景配合才能执行评估任务，详情见: -# 进阶教程 \ No newline at end of file +- [提示词列表](dingo/model/prompt) + +# 场景 +dingo 的场景负责将数据打包发送给模型，并接收模型返回的结果，然后进行解析，处理成统一的 ModelRes 类对象。 + +- [场景列表](dingo/model/llm) + +请注意，不同场景对于评估结果 ModelRes 类对象的构建思路也不同，其 type 和 name 的意义也因此不同。 + +# 进阶教程 + +## 自定义配置 + +## 自定义规则 + +## 自定义提示词 + +## 自定义场景 + +## 新增数据格式 + +## 添加规则 + +## 添加提示词 + +## 添加场景 \ No newline at end of file From 4455d130299736fa9c056067eaecee7738124d1a Mon Sep 17 00:00:00 2001 From: shijinpjlab Date: Thu, 10 Jul 2025 15:16:11 +0800 Subject: [PATCH 07/24] =?UTF-8?q?feat:=20technical=20doc=20=E8=87=AA?= =?UTF-8?q?=E5=AE=9A=E4=B9=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Technical_Doc.md | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/Technical_Doc.md b/Technical_Doc.md index 333aa315..9bccffec 100644 --- a/Technical_Doc.md +++ b/Technical_Doc.md @@ -118,7 +118,7 @@ python -m dingo.run.cli --dataset local --data_format plaintext -e sft - --save_data + --save_data True ``` **代码启动** @@ -197,12 +197,38 @@ dingo 的场景负责将数据打包发送给模型，并接收模型返回的 # 进阶教程 ## 自定义配置 +上文的 **教程-基础配置** 篇章中介绍了项目配置的方式与参数列表，但是并没有涉及到自定义，现在让我们来详细了解 **自定义配置** 。 +自定义配置离不开参数 [custom_config](docs/config.md#custom-config) , 这个参数包括能够自定义的所有内容，如下所示： +- rule_list +- prompt_list +- rule_config +- llm_config +- multi_turn_mode ## 自定义规则 - -## 自定义提示词 +dingo 内置的规则向用户开放了接口，允许用户根据不同的评估任务进行动态配置。 +规则的自定义通过上文 custom_config 参数中的 [rule_config](docs/config.md#rule_config) 实现，可以设置的值包括: ++ threshold ++ pattern ++ key_list ++ refer_path ## 自定义场景 +dingo 在使用提示词进行评估任务的时候，必须同时使用场景，执行数据的打包发送与接收处理。 +场景的自定义同样是通过上文 custom_config 参数实现，不同的是需要参数 [llm_config](docs/config.md#llm_config) ，可以设置的值包括: ++ model ++ key ++ api_url ++ parameters + +需要注意的是参数 [parameters](docs/config.md#parameters) ，这个参数会对模型的推理产生影响，可以设置的值包括: ++ temperature ++ top_p ++ max_tokens ++ presence_penalty ++ frequency_penalty + +更多参数细节可参考OpenAI API官方文档。 ## 新增数据格式 From c465ccb503b7ff3f5956e4a80fd8fb9e825dfc4c Mon Sep 17 00:00:00 2001 From: shijinpjlab Date: Thu, 10 Jul 2025 16:02:26 +0800 Subject: [PATCH 08/24] =?UTF-8?q?feat:=20technical=20doc=20=E6=96=B0?= =?UTF-8?q?=E5=A2=9E=E6=95=B0=E6=8D=AE=E6=A0=BC=E5=BC=8F=E8=BD=AC=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Technical_Doc.md | 63 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/Technical_Doc.md b/Technical_Doc.md index 9bccffec..bd59ae4b 100644 --- a/Technical_Doc.md +++ b/Technical_Doc.md @@ -230,7 +230,68 @@ dingo 在使用提示词进行评估任务的时候，必须同时使用场景 更多参数细节可参考OpenAI API官方文档。 -## 新增数据格式 +## 新增数据格式转化 +上文的 **教程-基础配置** 篇章中介绍了项目配置的参数列表，其中 data_format 表示数据的格式，同时也代表了一种数据转化的方式。 +dingo 内置的数据转化方式有4种，即 data_format 的4个可取的值: json, jsonl, plaintext, listjson. +其对应的转化逻辑见: [数据格式转化列表](dingo/data/converter/base.py) + +模板如下: + +```python +@BaseConverter.register("jsonl") +class JsonLineConverter(BaseConverter): + """Json line file converter.""" + + data_id = 0 + + def __init__(self): + super().__init__() + + @classmethod + def convertor(cls, input_args: InputArgs) -> Callable: + def _convert(raw: Union[str, Dict]): + j = raw + if isinstance(raw, str): + j = json.loads(raw) + cls.data_id += 1 + return Data( + **{ + "data_id": ( + cls.find_levels_data(j, input_args.column_id) + if input_args.column_id != "" + else str(cls.data_id) + ), + "prompt": ( + cls.find_levels_data(j, input_args.column_prompt) + if input_args.column_prompt != "" + else "" + ), + "content": ( + cls.find_levels_data(j, input_args.column_content) + if input_args.column_content != "" + else "" + ), + "raw_data": j, + } + ) + + return _convert +``` + +可以见到，Converter类需要注册一个名称，也就是为 data_format 新增一个可取值，不妨设为 myjsonl + +```python +@BaseConverter.register("myjsonl") +``` + +然后就是实现 convertor 类函数，特别需要注意函数接收变量与返回值的类型。 + +```python +@classmethod +def convertor(cls, input_args: InputArgs) -> Callable: +``` + +最后，需要填充 [Data](dingo/io/input/Data.py) 类，这是项目中数据的基本形态，也是待评估的数据形态。 ## 添加规则 From d1bc8c5c6eec9659d7e82c314fd448d3e00c2925 Mon Sep 17 00:00:00 2001 From: shijinpjlab Date: Thu, 10 Jul 2025 16:19:17 +0800 Subject: [PATCH 09/24] =?UTF-8?q?feat:=20technical=20doc=20=E6=96=B0?= =?UTF-8?q?=E5=A2=9E=E8=A7=84=E5=88=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Technical_Doc.md | 60 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 57 insertions(+), 3 deletions(-) diff --git a/Technical_Doc.md b/Technical_Doc.md index bd59ae4b..202a1fa6 100644 --- a/Technical_Doc.md +++ b/Technical_Doc.md @@ -293,8 +293,62 @@ def convertor(cls, input_args: InputArgs) -> Callable: 最后，需要填充 [Data](dingo/io/input/Data.py) 类，这是项目中数据的基本形态，也是待评估的数据形态。 -## 添加规则 +## 新增规则 +上文的 **规则** 篇章介绍了 [规则列表](docs/rules.md) ，其在项目中的位置为 [规则实现](dingo/model/rule) 。 +当dingo内置的规则无法满足用户的评估任务，用户需要添加新的评估规则时，可以参考一下模板: -## 添加提示词 +```python +@Model.rule_register( + "QUALITY_BAD_EFFECTIVENESS", + ["default", "sft", "pretrain", "benchmark", "llm_base", "text_base_all"], +) +class RuleColonEnd(BaseRule): + """check whether the last char is ':'""" + + dynamic_config = DynamicRuleConfig() + + @classmethod + def eval(cls, input_data: Data) -> ModelRes: + res = ModelRes() + content = input_data.content + if len(content) <= cls.dynamic_config.threshold: + return res + if content[-1] == ":": + res.error_status = True + res.type = cls.metric_type + res.name = cls.__name__ + res.reason = [content[-100:]] + return res +``` + +首先，所有的规则都是 [BaseRule](dingo/model/rule/base.py) 类的实现，都具有以下3个类属性: + ++ metric_type: 函数 rule_register 执行时赋值 ++ group: 函数 rule_register 执行时赋值 ++ dynamic_config: 开放的自定义接口 + +其次，所有的规则都需要执行注册操作，即 Model.rule_register 函数，并指明 metric_type 与 group。 + +```python +@Model.rule_register( + "QUALITY_BAD_EFFECTIVENESS", + ["default", "sft", "pretrain", "benchmark", "llm_base", "text_base_all"], +) +``` + +然后，定义类属性 dynamic_config ，否则用户无法对规则进行自定义操作 + +```python +dynamic_config = DynamicRuleConfig() +``` + +最后，实现 eval 类函数，需要注意接收变量与返回值的类型 + +```python +@classmethod +def eval(cls, input_data: Data) -> ModelRes: +``` + +## 新增提示词 -## 添加场景 \ No newline at end of file +## 新增场景 \ No newline at end of file From 2b3b6578e72080b86f0344e5b76fd51e27097279 Mon Sep 17 00:00:00 2001 From: shijinpjlab Date: Thu, 10 Jul 2025 16:28:02 +0800 Subject: [PATCH 10/24] =?UTF-8?q?feat:=20technical=20doc=20=E6=96=B0?= =?UTF-8?q?=E5=A2=9E=E6=8F=90=E7=A4=BA=E8=AF=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Technical_Doc.md | 43 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/Technical_Doc.md b/Technical_Doc.md index 202a1fa6..a1f86b80 100644 --- a/Technical_Doc.md +++ b/Technical_Doc.md @@ -294,7 +294,7 @@ def convertor(cls, input_args: InputArgs) -> Callable: 最后，需要填充 [Data](dingo/io/input/Data.py) 类，这是项目中数据的基本形态，也是待评估的数据形态。 ## 新增规则 -上文的 **规则** 篇章介绍了 [规则列表](docs/rules.md) ，其在项目中的位置为 [规则实现](dingo/model/rule) 。 +上文的 **规则** 篇章介绍了 [规则列表](docs/rules.md) ，其在项目中的位置为 [规则代码列表](dingo/model/rule) 。 当dingo内置的规则无法满足用户的评估任务，用户需要添加新的评估规则时，可以参考一下模板: ```python @@ -351,4 +351,45 @@ def eval(cls, input_data: Data) -> ModelRes: ## 新增提示词 +上文的 **提示词** 篇章中已经介绍了 [提示词列表](dingo/model/prompt) ，如果用户在评估过程中产生了新的评估任务，需要涉及自己的提示词， +那么将新的提示词添加到项目的方式可以参考一下方式: + +```python +@Model.prompt_register("QUALITY_BAD_SIMILARITY", []) +class PromptRepeat(BasePrompt): + content = """ + 请判断一下文本是否存在重复问题。 + 返回一个json，如{"score": 0, "reason": "xxx"}. + 如果存在重复，score是0，否则是1。reason是判断的依据。 + 除了json不要有其他内容。 + 以下是需要判断的文本： + """ +``` + +从上面的模板中可以看到，新增的提示词必须继承 [BasePrompt](dingo/model/prompt/base.py) 类。 + +```python +class PromptRepeat(BasePrompt): +``` + +然后，与新增规则相似，都执行注册操作，并指明 metric_type 与 group 。 + +注意， group 可以是空列表，表名该提示词不属于任何的 group ，并且无法通过 group 来调用。 + +```python +@Model.prompt_register("QUALITY_BAD_SIMILARITY", []) +``` + +最后，填写新的提示词。 + +```python +content = """ + 请判断一下文本是否存在重复问题。 + 返回一个json，如{"score": 0, "reason": "xxx"}. + 如果存在重复，score是0，否则是1。reason是判断的依据。 + 除了json不要有其他内容。 + 以下是需要判断的文本： + """ +``` + ## 新增场景 \ No newline at end of file From 78925a5f60414e48a537e6a8869f69f0bfc29cd9 Mon Sep 17 00:00:00 2001 From: shijinpjlab Date: Thu, 10 Jul 2025 16:43:40 +0800 Subject: [PATCH 11/24] =?UTF-8?q?feat:=20technical=20doc=20=E6=96=B0?= =?UTF-8?q?=E5=A2=9E=E6=96=B0=E5=A2=9E=E5=9C=BA=E6=99=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Technical_Doc.md | 116 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 115 insertions(+), 1 deletion(-) diff --git a/Technical_Doc.md b/Technical_Doc.md index a1f86b80..67495ca8 100644 --- a/Technical_Doc.md +++ b/Technical_Doc.md @@ -392,4 +392,118 @@ content = """ """ ``` -## 新增场景 \ No newline at end of file +## 新增场景 +上文的 **场景** 篇章介绍了场景的职责，即: 打包发送数据、接收解析数据 + +那么，新增一个场景就需要实现以上2个功能，详情见下方模板: + +```python +class BaseOpenAI(BaseLLM): + prompt = None + client = None + dynamic_config = DynamicLLMConfig() + + @classmethod + def set_prompt(cls, prompt: BasePrompt): + pass + + @classmethod + def create_client(cls): + pass + + @classmethod + def build_messages(cls, input_data: Data) -> List: + pass + + @classmethod + def send_messages(cls, messages: List): + pass + + @classmethod + def process_response(cls, response: str) -> ModelRes: + pass + + @classmethod + def eval(cls, input_data: Data) -> ModelRes: + if cls.client is None: + cls.create_client() + + messages = cls.build_messages(input_data) + + attempts = 0 + except_msg = "" + except_name = Exception.__class__.__name__ + while attempts < 3: + try: + response = cls.send_messages(messages) + return cls.process_response(response) + except (ValidationError, ExceedMaxTokens, ConvertJsonError) as e: + except_msg = str(e) + except_name = e.__class__.__name__ + break + except Exception as e: + attempts += 1 + time.sleep(1) + except_msg = str(e) + except_name = e.__class__.__name__ + + return ModelRes( + error_status=True, type="QUALITY_BAD", name=except_name, reason=[except_msg] + ) +``` + +第一步，场景必须继承 [BaseLLM](dingo/model/llm/base.py) 或者其子类 + +```python +class BaseOpenAI(BaseLLM): +``` + +第二步，设置场景的模型类属性: + +```python + prompt = None + client = None + dynamic_config = DynamicLLMConfig() +``` + +第四步，实现 set_prompt 类函数，用于设置场景提示词: + +```python +@classmethod +def set_prompt(cls, prompt: BasePrompt): +``` + +第五步，实现 create_client 类函数，创建模型 client ，用于收发数据。 + +```python +@classmethod +def create_client(cls): +``` + +第六步，实现 build_messages 类函数，打包数据，用于发送。 + +```python +@classmethod +def build_messages(cls, input_data: Data) -> List: +``` + +第七步，实现 send_messages 类函数，发送打包完成的数据，并且接收模型返回的数据。 + +```python +@classmethod +def send_messages(cls, messages: List): +``` + +第八步，实现 process_response 类函数，解析模型返回的数据 + +```python +@classmethod +def process_response(cls, response: str) -> ModelRes: +``` + +第九步，实现 eval 类函数，统筹执行以上实现的类函数，并将解析后的数据转化为 [ModelRes](dingo/model/modelres.py) 类型。 + +```python +@classmethod +def eval(cls, input_data: Data) -> ModelRes: +``` From 6a0e41fe16716c8e17b4f14968040efcd79fa79f Mon Sep 17 00:00:00 2001 From: shijinpjlab Date: Thu, 10 Jul 2025 17:26:01 +0800 Subject: [PATCH 12/24] =?UTF-8?q?feat:=20technical=20doc=20=E8=B0=83?= =?UTF-8?q?=E6=95=B4=E4=BD=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Technical_Doc.md => docs/Technical_Doc.md | 42 +++++++++++------------ 1 file changed, 21 insertions(+), 21 deletions(-) rename Technical_Doc.md => docs/Technical_Doc.md (87%) diff --git a/Technical_Doc.md b/docs/Technical_Doc.md similarity index 87% rename from Technical_Doc.md rename to docs/Technical_Doc.md index 67495ca8..15f80c96 100644 --- a/Technical_Doc.md +++ b/docs/Technical_Doc.md @@ -102,12 +102,12 @@ outputs/ + Evaluator: 负责评估的执行 + Executor: 负责任务的分配与调度 -![Architecture of dingo](./docs/assets/architeture.png) +![Architecture of dingo](assets/architeture.png) ## 基础配置 dingo 启动方式具有2种，因此配置方式也分为以下2种情况: -1. [命令行配置列表](docs/config.md#cli-config) -2. [代码配置列表](docs/config.md#sdk-config) +1. [命令行配置列表](config.md#cli-config) +2. [代码配置列表](config.md#sdk-config) **命令行启动** 在命令行环境中，所有配置均以 参数键值对 的形式指定，遵从标准 CLI 语法规则，通过 --参数名 参数值 的方式传递每个配置项。 @@ -148,7 +148,7 @@ input_args = InputArgs(**input_data) - column_prompt - column_content -最终数据以 [Data](dingo/io/input/Data.py) 类对象的形式在项目中流转。 +最终数据以 [Data](../dingo/io/input/Data.py) 类对象的形式在项目中流转。 如果用户在配置时将参数 save_raw 设置为True，那么 Data 类对象的 raw_data 有值否则为空字典。 ## 设置并发 @@ -167,7 +167,7 @@ dingo 默认状态下没有开启并发，如果有大规模评估任务需要 + output_path 上文中评估阶段的二级类型jsonl文件中的每条结果数据收到配置参数 save_raw 的影响。 -如果 save_raw 设置为True，那么将执行 [ResultInfo](dingo/io/output/ResultInfo.py) 类的 to_dict_raw 函数，否则将执行 to_dict 函数。 +如果 save_raw 设置为True，那么将执行 [ResultInfo](../dingo/io/output/ResultInfo.py) 类的 to_dict_raw 函数，否则将执行 to_dict 函数。 ## 启动前端页面 dingo 评估任务结束后，如果保存了评估结果，那么就可以通过一下方式启动前端页面展示结果: @@ -176,21 +176,21 @@ python -m dingo.run.vsl --input outputs/20250609_101837_50b5c0be ``` # 规则 -dingo 内置了不同类型的评估规则，详情见: [规则列表](docs/rules.md)。 +dingo 内置了不同类型的评估规则，详情见: [规则列表](rules.md)。 每条评估规则都有自己的 metric_type 和所属的 group。 -每条数据经过规则评估，会产生一个 [ModelRes](dingo/model/modelres.py) 类对象作为结果，一般来说规则的 metric_type 作为 type 而规则名作为 name。 +每条数据经过规则评估，会产生一个 [ModelRes](../dingo/model/modelres.py) 类对象作为结果，一般来说规则的 metric_type 作为 type 而规则名作为 name。 用户可以通过配置 eval_group 参数来调用该 group 内的所有规则执行评估任务。 如果用户需要组合一批评估规则用来评估，那么请参考下文的 **自定义配置** 。 # 提示词 dingo 提示词与规则类似，都有 metric_type 和 group ，并且他们的作用也相同。 但是提示词需要与场景配合才能执行评估任务，详情见: -- [提示词列表](dingo/model/prompt) +- [提示词列表](../dingo/model/prompt) # 场景 dingo 的场景负责将数据打包发送给模型，并接收模型返回的结果，然后进行解析，处理成统一的 ModelRes 类对象。 -- [场景列表](dingo/model/llm) +- [场景列表](../dingo/model/llm) 请注意，不同场景对于评估结果 ModelRes 类对象的构建思路也不同，其 type 和 name 的意义也因此不同。 @@ -198,7 +198,7 @@ dingo 的场景负责将数据打包发送给模型，并接收模型返回的 ## 自定义配置 上文的 **教程-基础配置** 篇章中介绍了项目配置的方式与参数列表，但是并没有涉及到自定义，现在让我们来详细了解 **自定义配置** 。 -自定义配置离不开参数 [custom_config](docs/config.md#custom-config) , 这个参数包括能够自定义的所有内容，如下所示： +自定义配置离不开参数 [custom_config](config.md#custom-config) , 这个参数包括能够自定义的所有内容，如下所示： - rule_list - prompt_list - rule_config @@ -207,7 +207,7 @@ dingo 的场景负责将数据打包发送给模型，并接收模型返回的 ## 自定义规则 dingo 内置的规则向用户开放了接口，允许用户根据不同的评估任务进行动态配置。 -规则的自定义通过上文 custom_config 参数中的 [rule_config](docs/config.md#rule_config) 实现，可以设置的值包括: +规则的自定义通过上文 custom_config 参数中的 [rule_config](config.md#rule_config) 实现，可以设置的值包括: + threshold + pattern + key_list @@ -215,13 +215,13 @@ dingo 内置的规则向用户开放了接口，允许用户根据不同的评 ## 自定义场景 dingo 在使用提示词进行评估任务的时候，必须同时使用场景，执行数据的打包发送与接收处理。 -场景的自定义同样是通过上文 custom_config 参数实现，不同的是需要参数 [llm_config](docs/config.md#llm_config) ，可以设置的值包括: +场景的自定义同样是通过上文 custom_config 参数实现，不同的是需要参数 [llm_config](config.md#llm_config) ，可以设置的值包括: + model + key + api_url + parameters -需要注意的是参数 [parameters](docs/config.md#parameters) ，这个参数会对模型的推理产生影响，可以设置的值包括: +需要注意的是参数 [parameters](config.md#parameters) ，这个参数会对模型的推理产生影响，可以设置的值包括: + temperature + top_p + max_tokens @@ -233,7 +233,7 @@ dingo 在使用提示词进行评估任务的时候，必须同时使用场景 ## 新增数据格式转化 上文的 **教程-基础配置** 篇章中介绍了项目配置的参数列表，其中 data_format 表示数据的格式，同时也代表了一种数据转化的方式。 dingo 内置的数据转化方式有4种，即 data_format 的4个可取的值: json, jsonl, plaintext, listjson. -其对应的转化逻辑见: [数据格式转化列表](dingo/data/converter/base.py) +其对应的转化逻辑见: [数据格式转化列表](../dingo/data/converter/base.py) 模板如下: @@ -291,10 +291,10 @@ class JsonLineConverter(BaseConverter): def convertor(cls, input_args: InputArgs) -> Callable: ``` -最后，需要填充 [Data](dingo/io/input/Data.py) 类，这是项目中数据的基本形态，也是待评估的数据形态。 +最后，需要填充 [Data](../dingo/io/input/Data.py) 类，这是项目中数据的基本形态，也是待评估的数据形态。 ## 新增规则 -上文的 **规则** 篇章介绍了 [规则列表](docs/rules.md) ，其在项目中的位置为 [规则代码列表](dingo/model/rule) 。 +上文的 **规则** 篇章介绍了 [规则列表](rules.md) ，其在项目中的位置为 [规则代码列表](../dingo/model/rule) 。 当dingo内置的规则无法满足用户的评估任务，用户需要添加新的评估规则时，可以参考一下模板: ```python @@ -321,7 +321,7 @@ class RuleColonEnd(BaseRule): return res ``` -首先，所有的规则都是 [BaseRule](dingo/model/rule/base.py) 类的实现，都具有以下3个类属性: +首先，所有的规则都是 [BaseRule](../dingo/model/rule/base.py) 类的实现，都具有以下3个类属性: + metric_type: 函数 rule_register 执行时赋值 + group: 函数 rule_register 执行时赋值 @@ -351,7 +351,7 @@ def eval(cls, input_data: Data) -> ModelRes: ## 新增提示词 -上文的 **提示词** 篇章中已经介绍了 [提示词列表](dingo/model/prompt) ，如果用户在评估过程中产生了新的评估任务，需要涉及自己的提示词， +上文的 **提示词** 篇章中已经介绍了 [提示词列表](../dingo/model/prompt) ，如果用户在评估过程中产生了新的评估任务，需要涉及自己的提示词， 那么将新的提示词添加到项目的方式可以参考一下方式: ```python @@ -366,7 +366,7 @@ class PromptRepeat(BasePrompt): """ ``` -从上面的模板中可以看到，新增的提示词必须继承 [BasePrompt](dingo/model/prompt/base.py) 类。 +从上面的模板中可以看到，新增的提示词必须继承 [BasePrompt](../dingo/model/prompt/base.py) 类。 ```python class PromptRepeat(BasePrompt): @@ -452,7 +452,7 @@ class BaseOpenAI(BaseLLM): ) ``` -第一步，场景必须继承 [BaseLLM](dingo/model/llm/base.py) 或者其子类 +第一步，场景必须继承 [BaseLLM](../dingo/model/llm/base.py) 或者其子类 ```python class BaseOpenAI(BaseLLM): @@ -501,7 +501,7 @@ def send_messages(cls, messages: List): def process_response(cls, response: str) -> ModelRes: ``` -第九步，实现 eval 类函数，统筹执行以上实现的类函数，并将解析后的数据转化为 [ModelRes](dingo/model/modelres.py) 类型。 +第九步，实现 eval 类函数，统筹执行以上实现的类函数，并将解析后的数据转化为 [ModelRes](../dingo/model/modelres.py) 类型。 ```python @classmethod From 75f565eba2c6888924a3627c57f59841040d4720 Mon Sep 17 00:00:00 2001 From: shijinpjlab Date: Thu, 10 Jul 2025 17:30:16 +0800 Subject: [PATCH 13/24] feat: fix lint --- docs/Technical_Doc.md | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/docs/Technical_Doc.md b/docs/Technical_Doc.md index 15f80c96..2bdc7dda 100644 --- a/docs/Technical_Doc.md +++ b/docs/Technical_Doc.md @@ -41,7 +41,7 @@ pip install -r requirements/web.txt ```shell python -m dingo.run.cli --input_path data.txt - --dataset local + --dataset local --data_format plaintext -e sft --save_data @@ -87,8 +87,8 @@ outputs/ │ ├── QUALITY_BAD_COMPLETENESS # 评估阶段的一级类型 │ │ ├── RuleSentenceNumber.jsonl # 评估阶段的二级类型 │ │ └── RuleWordNumber.jsonl -│ ├── QUALITY_BAD_EFFECTIVENESS -│ │ ├── RuleColonEnd.jsonl +│ ├── QUALITY_BAD_EFFECTIVENESS +│ │ ├── RuleColonEnd.jsonl │ │ └── RuleEnterAndSpace.jsonl │ └── summary.json # 单个任务的汇总结果 ├── ... @@ -109,19 +109,19 @@ dingo 启动方式具有2种，因此配置方式也分为以下2种情况: 1. [命令行配置列表](config.md#cli-config) 2. [代码配置列表](config.md#sdk-config) -**命令行启动** +**命令行启动** 在命令行环境中，所有配置均以 参数键值对 的形式指定，遵从标准 CLI 语法规则，通过 --参数名 参数值 的方式传递每个配置项。 ```shell python -m dingo.run.cli --input_path data.txt - --dataset local + --dataset local --data_format plaintext -e sft --save_data True ``` -**代码启动** +**代码启动** 在代码环境中，配置都是 Python 格式的，遵从基本的 Python 语法，通过定义变量的形式指定每个配置项。 ```python @@ -148,7 +148,7 @@ input_args = InputArgs(**input_data) - column_prompt - column_content -最终数据以 [Data](../dingo/io/input/Data.py) 类对象的形式在项目中流转。 +最终数据以 [Data](../dingo/io/input/Data.py) 类对象的形式在项目中流转。 如果用户在配置时将参数 save_raw 设置为True，那么 Data 类对象的 raw_data 有值否则为空字典。 ## 设置并发 @@ -166,7 +166,7 @@ dingo 默认状态下没有开启并发，如果有大规模评估任务需要 + save_correct + output_path -上文中评估阶段的二级类型jsonl文件中的每条结果数据收到配置参数 save_raw 的影响。 +上文中评估阶段的二级类型jsonl文件中的每条结果数据收到配置参数 save_raw 的影响。 如果 save_raw 设置为True，那么将执行 [ResultInfo](../dingo/io/output/ResultInfo.py) 类的 to_dict_raw 函数，否则将执行 to_dict 函数。 ## 启动前端页面 @@ -176,13 +176,13 @@ python -m dingo.run.vsl --input outputs/20250609_101837_50b5c0be ``` # 规则 -dingo 内置了不同类型的评估规则，详情见: [规则列表](rules.md)。 -每条评估规则都有自己的 metric_type 和所属的 group。 -每条数据经过规则评估，会产生一个 [ModelRes](../dingo/model/modelres.py) 类对象作为结果，一般来说规则的 metric_type 作为 type 而规则名作为 name。 +dingo 内置了不同类型的评估规则，详情见: [规则列表](rules.md)。 +每条评估规则都有自己的 metric_type 和所属的 group。 +每条数据经过规则评估，会产生一个 [ModelRes](../dingo/model/modelres.py) 类对象作为结果，一般来说规则的 metric_type 作为 type 而规则名作为 name。 用户可以通过配置 eval_group 参数来调用该 group 内的所有规则执行评估任务。 如果用户需要组合一批评估规则用来评估，那么请参考下文的 **自定义配置** 。 # 提示词 -dingo 提示词与规则类似，都有 metric_type 和 group ，并且他们的作用也相同。 +dingo 提示词与规则类似，都有 metric_type 和 group ，并且他们的作用也相同。 但是提示词需要与场景配合才能执行评估任务，详情见: - [提示词列表](../dingo/model/prompt) @@ -197,7 +197,7 @@ dingo 的场景负责将数据打包发送给模型，并接收模型返回的 # 进阶教程 ## 自定义配置 -上文的 **教程-基础配置** 篇章中介绍了项目配置的方式与参数列表，但是并没有涉及到自定义，现在让我们来详细了解 **自定义配置** 。 +上文的 **教程-基础配置** 篇章中介绍了项目配置的方式与参数列表，但是并没有涉及到自定义，现在让我们来详细了解 **自定义配置** 。 自定义配置离不开参数 [custom_config](config.md#custom-config) , 这个参数包括能够自定义的所有内容，如下所示： - rule_list - prompt_list @@ -206,7 +206,7 @@ dingo 的场景负责将数据打包发送给模型，并接收模型返回的 - multi_turn_mode ## 自定义规则 -dingo 内置的规则向用户开放了接口，允许用户根据不同的评估任务进行动态配置。 +dingo 内置的规则向用户开放了接口，允许用户根据不同的评估任务进行动态配置。 规则的自定义通过上文 custom_config 参数中的 [rule_config](config.md#rule_config) 实现，可以设置的值包括: + threshold + pattern @@ -214,7 +214,7 @@ dingo 内置的规则向用户开放了接口，允许用户根据不同的评 + refer_path ## 自定义场景 -dingo 在使用提示词进行评估任务的时候，必须同时使用场景，执行数据的打包发送与接收处理。 +dingo 在使用提示词进行评估任务的时候，必须同时使用场景，执行数据的打包发送与接收处理。 场景的自定义同样是通过上文 custom_config 参数实现，不同的是需要参数 [llm_config](config.md#llm_config) ，可以设置的值包括: + model + key @@ -231,8 +231,8 @@ dingo 在使用提示词进行评估任务的时候，必须同时使用场景 更多参数细节可参考OpenAI API官方文档。 ## 新增数据格式转化 -上文的 **教程-基础配置** 篇章中介绍了项目配置的参数列表，其中 data_format 表示数据的格式，同时也代表了一种数据转化的方式。 -dingo 内置的数据转化方式有4种，即 data_format 的4个可取的值: json, jsonl, plaintext, listjson. +上文的 **教程-基础配置** 篇章中介绍了项目配置的参数列表，其中 data_format 表示数据的格式，同时也代表了一种数据转化的方式。 +dingo 内置的数据转化方式有4种，即 data_format 的4个可取的值: json, jsonl, plaintext, listjson. 其对应的转化逻辑见: [数据格式转化列表](../dingo/data/converter/base.py) 模板如下: @@ -278,7 +278,7 @@ class JsonLineConverter(BaseConverter): return _convert ``` -可以见到，Converter类需要注册一个名称，也就是为 data_format 新增一个可取值，不妨设为 myjsonl +可以见到，Converter类需要注册一个名称，也就是为 data_format 新增一个可取值，不妨设为 myjsonl ```python @BaseConverter.register("myjsonl") @@ -294,7 +294,7 @@ def convertor(cls, input_args: InputArgs) -> Callable: 最后，需要填充 [Data](../dingo/io/input/Data.py) 类，这是项目中数据的基本形态，也是待评估的数据形态。 ## 新增规则 -上文的 **规则** 篇章介绍了 [规则列表](rules.md) ，其在项目中的位置为 [规则代码列表](../dingo/model/rule) 。 +上文的 **规则** 篇章介绍了 [规则列表](rules.md) ，其在项目中的位置为 [规则代码列表](../dingo/model/rule) 。 当dingo内置的规则无法满足用户的评估任务，用户需要添加新的评估规则时，可以参考一下模板: ```python @@ -327,7 +327,7 @@ class RuleColonEnd(BaseRule): + group: 函数 rule_register 执行时赋值 + dynamic_config: 开放的自定义接口 -其次，所有的规则都需要执行注册操作，即 Model.rule_register 函数，并指明 metric_type 与 group。 +其次，所有的规则都需要执行注册操作，即 Model.rule_register 函数，并指明 metric_type 与 group。 ```python @Model.rule_register( @@ -351,7 +351,7 @@ def eval(cls, input_data: Data) -> ModelRes: ## 新增提示词 -上文的 **提示词** 篇章中已经介绍了 [提示词列表](../dingo/model/prompt) ，如果用户在评估过程中产生了新的评估任务，需要涉及自己的提示词， +上文的 **提示词** 篇章中已经介绍了 [提示词列表](../dingo/model/prompt) ，如果用户在评估过程中产生了新的评估任务，需要涉及自己的提示词， 那么将新的提示词添加到项目的方式可以参考一下方式: ```python From ec039d4109118dffcbb026449bfe288154a98ece Mon Sep 17 00:00:00 2001 From: shijinpjlab Date: Thu, 10 Jul 2025 17:33:39 +0800 Subject: [PATCH 14/24] feat: fix lint --- docs/Technical_Doc.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/Technical_Doc.md b/docs/Technical_Doc.md index 2bdc7dda..9da6823c 100644 --- a/docs/Technical_Doc.md +++ b/docs/Technical_Doc.md @@ -157,6 +157,7 @@ dingo 默认状态下没有开启并发，如果有大规模评估任务需要 + batch_size 以上2个参数应当搭配使用，如果max_workers设置为10但是batch_size设置为1，那么评估的效率不会得到较大提升。 + 建议batch_size大于等于max_workers。 ## 结果保存 @@ -167,6 +168,7 @@ dingo 默认状态下没有开启并发，如果有大规模评估任务需要 + output_path 上文中评估阶段的二级类型jsonl文件中的每条结果数据收到配置参数 save_raw 的影响。 + 如果 save_raw 设置为True，那么将执行 [ResultInfo](../dingo/io/output/ResultInfo.py) 类的 to_dict_raw 函数，否则将执行 to_dict 函数。 ## 启动前端页面 @@ -178,7 +180,9 @@ python -m dingo.run.vsl --input outputs/20250609_101837_50b5c0be # 规则 dingo 内置了不同类型的评估规则，详情见: [规则列表](rules.md)。 每条评估规则都有自己的 metric_type 和所属的 group。 + 每条数据经过规则评估，会产生一个 [ModelRes](../dingo/model/modelres.py) 类对象作为结果，一般来说规则的 metric_type 作为 type 而规则名作为 name。 + 用户可以通过配置 eval_group 参数来调用该 group 内的所有规则执行评估任务。 如果用户需要组合一批评估规则用来评估，那么请参考下文的 **自定义配置** 。 # 提示词 @@ -198,6 +202,7 @@ dingo 的场景负责将数据打包发送给模型，并接收模型返回的 ## 自定义配置 上文的 **教程-基础配置** 篇章中介绍了项目配置的方式与参数列表，但是并没有涉及到自定义，现在让我们来详细了解 **自定义配置** 。 + 自定义配置离不开参数 [custom_config](config.md#custom-config) , 这个参数包括能够自定义的所有内容，如下所示： - rule_list - prompt_list @@ -207,6 +212,7 @@ dingo 的场景负责将数据打包发送给模型，并接收模型返回的 ## 自定义规则 dingo 内置的规则向用户开放了接口，允许用户根据不同的评估任务进行动态配置。 + 规则的自定义通过上文 custom_config 参数中的 [rule_config](config.md#rule_config) 实现，可以设置的值包括: + threshold + pattern @@ -215,6 +221,7 @@ dingo 内置的规则向用户开放了接口，允许用户根据不同的评 ## 自定义场景 dingo 在使用提示词进行评估任务的时候，必须同时使用场景，执行数据的打包发送与接收处理。 + 场景的自定义同样是通过上文 custom_config 参数实现，不同的是需要参数 [llm_config](config.md#llm_config) ，可以设置的值包括: + model + key @@ -232,7 +239,9 @@ dingo 在使用提示词进行评估任务的时候，必须同时使用场景 ## 新增数据格式转化 上文的 **教程-基础配置** 篇章中介绍了项目配置的参数列表，其中 data_format 表示数据的格式，同时也代表了一种数据转化的方式。 + dingo 内置的数据转化方式有4种，即 data_format 的4个可取的值: json, jsonl, plaintext, listjson. + 其对应的转化逻辑见: [数据格式转化列表](../dingo/data/converter/base.py) 模板如下: @@ -295,6 +304,7 @@ def convertor(cls, input_args: InputArgs) -> Callable: ## 新增规则 上文的 **规则** 篇章介绍了 [规则列表](rules.md) ，其在项目中的位置为 [规则代码列表](../dingo/model/rule) 。 + 当dingo内置的规则无法满足用户的评估任务，用户需要添加新的评估规则时，可以参考一下模板: ```python @@ -351,7 +361,8 @@ def eval(cls, input_data: Data) -> ModelRes: ## 新增提示词 -上文的 **提示词** 篇章中已经介绍了 [提示词列表](../dingo/model/prompt) ，如果用户在评估过程中产生了新的评估任务，需要涉及自己的提示词， +上文的 **提示词** 篇章中已经介绍了 [提示词列表](../dingo/model/prompt) ，如果用户在评估过程中产生了新的评估任务，需要涉及自己的提示词。 + 那么将新的提示词添加到项目的方式可以参考一下方式: ```python From bd272ecbe03e519e089d72b4596b85de82d5022e Mon Sep 17 00:00:00 2001 From: shijinpjlab Date: Thu, 10 Jul 2025 19:06:14 +0800 Subject: [PATCH 15/24] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0model=E4=B8=8El?= =?UTF-8?q?ocal?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../technical_all.md} | 0 docs/technical/technical_local.md | 164 ++++++++++++++++++ docs/technical/technical_model.md | 118 +++++++++++++ 3 files changed, 282 insertions(+) rename docs/{Technical_Doc.md => technical/technical_all.md} (100%) create mode 100644 docs/technical/technical_local.md create mode 100644 docs/technical/technical_model.md diff --git a/docs/Technical_Doc.md b/docs/technical/technical_all.md similarity index 100% rename from docs/Technical_Doc.md rename to docs/technical/technical_all.md diff --git a/docs/technical/technical_local.md b/docs/technical/technical_local.md new file mode 100644 index 00000000..d39a5ab4 --- /dev/null +++ b/docs/technical/technical_local.md @@ -0,0 +1,164 @@ +# dingo.exec.local 技术报告 + +## 一、模块定位与作用 + +`dingo.exec.local` 主要实现了本地执行器（LocalExecutor），用于在本地环境下对数据集进行评测任务的自动化批量处理。它负责数据加载、模型配置、评测调度、结果统计与输出，是 Dingo 评测系统的核心执行组件之一。 + +--- + +## 二、核心类与结构 + +### 1. LocalExecutor + +#### 继承关系 + +- 继承自 `ExecProto`，并通过 `@Executor.register("local")` 装饰器注册为 "local" 类型的执行器。 + +#### 主要属性 + +- `input_args`: 评测任务的输入参数（InputArgs 实例）。 +- `llm`: 当前使用的大语言模型（BaseLLM 实例）。 +- `summary`: 评测任务的统计与汇总信息（SummaryModel 实例）。 + +#### 主要方法 + +##### 1. 数据加载 + +- `load_data() -> Generator[Data]` + - 根据输入参数中的数据集名称，动态加载数据源和数据集，返回数据生成器。 + +##### 2. 执行主流程 + +- `execute() -> SummaryModel` + - 设置日志等级，应用模型配置，创建输出目录。 + - 根据配置选择 LLM，初始化 SummaryModel。 + - 调用 `evaluate()` 进行批量评测。 + - 汇总并写出 summary，返回最终统计结果。 + +##### 3. 评测主循环 + +- `evaluate()` + - 支持多线程和多进程混合并发（规则可选线程/进程，Prompt 固定线程）。 + - 按 batch_size 分批处理数据，调度各分组（rule/prompt）下的评测任务。 + - 聚合每条数据的评测结果，实时更新 summary，并写出单条数据和 summary。 + +##### 4. 单条数据评测 + +- `evaluate_single_data(group_type, group, data: Data) -> ResultInfo` + - 针对 rule 或 prompt 分组，分别调用 `evaluate_rule` 或 `evaluate_prompt`。 + - 聚合每个分组下所有规则/提示词的评测结果，区分好坏类型、名称、原因。 + +- `evaluate_rule(group: List[BaseRule], d: Data) -> ResultInfo` + - 依次调用每个规则的 `eval` 方法，分析结果，统计类型、名称、原因。 + +- `evaluate_prompt(group: List[BasePrompt], d: Data) -> ResultInfo` + - 依次设置 LLM 的 prompt，调用 LLM 的 `eval` 方法，分析结果。 + +##### 5. 结果写出与汇总 + +- `write_single_data(path, input_args, result_info)` + - 按类型-名称分目录写出每条数据的评测结果（jsonl 格式），支持保存原始数据。 + +- `write_summary(path, input_args, summary)` + - 写出当前评测任务的 summary（summary.json）。 + +- `summarize(summary: SummaryModel) -> SummaryModel` + - 计算得分、类型/名称分布比例，补充完成时间。 + +##### 6. 结果查询 + +- `get_info_list(high_quality: bool) -> list` + - 读取输出目录下所有结果，按 error_status 区分高/低质量数据。 + +- `get_bad_info_list()`, `get_good_info_list()` + - 分别获取低质量/高质量数据列表。 + +##### 7. 结果合并 + +- `merge_result_info(existing_list, new_item)` + - 合并同一 data_id 的评测结果，去重类型、名称、原因。 + +--- + +## 三、执行流程（有序步骤） + +1. **初始化 LocalExecutor** + 创建 LocalExecutor 实例，传入 InputArgs 参数。 + +2. **加载数据** + 调用 `load_data` 方法，根据输入参数加载数据集，返回数据生成器。 + +3. **应用模型配置** + 调用 `Model.apply_config`，根据配置文件和分组名应用评测规则、Prompt、LLM 等配置。 + +4. **创建输出目录** + 根据当前时间和 UUID 生成唯一输出目录，并在需要时创建。 + +5. **选择 LLM** + 根据配置文件选择并初始化当前使用的大语言模型（LLM）。 + +6. **初始化 SummaryModel** + 创建 SummaryModel 实例，用于统计和汇总评测任务信息。 + +7. **批量评测** + 调用 `evaluate` 方法，按 batch_size 分批调度线程池/进程池，对每批数据进行评测。 + +8. **对每条数据进行评测（rule/prompt）** + 针对每条数据，分别对 rule 分组和 prompt 分组进行评测，调用相应的评测方法。 + +9. **聚合结果，写出单条数据** + 聚合每条数据的评测结果，写出到对应的输出文件。 + +10. **实时更新 summary** + 在评测过程中，实时更新 SummaryModel 的统计信息。 + +11. **写出 summary.json** + 评测结束后，将 summary 信息写出为 summary.json 文件。 + +12. **返回 SummaryModel** + 返回最终的 SummaryModel 结果，供后续分析或展示使用。 + +--- + +## 四、设计亮点 + +- **高并发支持**：灵活选择线程池/进程池，兼容本地多核与分布式部署。 +- **分组评测**：支持 rule、prompt 分组，便于扩展多种评测维度。 +- **动态模型配置**：与 Model 配合，支持按配置文件动态切换评测规则、LLM、Prompt。 +- **结果结构化输出**：单条数据与 summary 分别输出，便于后续分析与复现。 +- **高/低质量数据筛选**：内置高低质量数据快速检索接口。 + +--- + +## 五、注意事项 + +- 需保证输入参数（InputArgs）和配置文件格式正确。 +- 评测规则、Prompt、LLM 需提前注册并实现对应接口。 +- 输出目录需有写权限，且不会与历史任务冲突。 +- 多进程模式下，需注意环境变量 `LOCAL_DEPLOYMENT_MODE` 的设置。 + +--- + +## 六、典型用法 + +```python +from dingo.io import InputArgs +from dingo.exec.local import LocalExecutor + +input_args = InputArgs( + dataset="my_dataset", + custom_config="config.yaml", + eval_group="default", + output_path="./outputs", + ... +) +executor = LocalExecutor(input_args) +summary = executor.execute() +print(summary.to_dict()) +``` + +--- + +## 七、总结 + +`dingo.exec.local` 是 Dingo 评测系统的本地执行核心，具备高并发、灵活分组、动态配置、结构化输出等特性，适合大规模自动化评测任务。其设计充分考虑了扩展性与易用性，是构建智能评测流水线的重要基础模块。 \ No newline at end of file diff --git a/docs/technical/technical_model.md b/docs/technical/technical_model.md new file mode 100644 index 00000000..aa152354 --- /dev/null +++ b/docs/technical/technical_model.md @@ -0,0 +1,118 @@ +# dingo.model.model 技术文档 + +## 一、概述 + +`dingo.model.model` 主要负责模型（包括规则、提示词、LLM等）的注册、分组、配置应用和动态加载。它为整个 Dingo 系统提供了统一的模型管理和配置入口，支持自动发现和注册规则、提示词、LLM，并能根据配置文件动态调整模型行为。 + +--- + +## 二、主要类与结构 + +### 1. BaseEvalModel + +- 继承自 `pydantic.BaseModel` +- 字段： + - `name`: str + - `type`: str +- 用于描述基础的评测模型信息。 + +### 2. Model + +#### 主要职责 + +- 管理和注册规则（Rule）、提示词（Prompt）、LLM（大语言模型）。 +- 支持模型的分组、按 metric_type 分类、按名称索引。 +- 支持根据配置文件动态应用模型参数。 +- 自动加载和注册 `rule/`、`prompt/`、`llm/` 目录下的所有模型类。 + +#### 主要类属性 + +- `module_loaded`: 是否已加载模块，防止重复加载。 +- `rule_groups`, `prompt_groups`: 分组管理，结构如 `{group_name: [class, ...]}`。 +- `rule_metric_type_map`, `prompt_metric_type_map`: 按 metric_type 分类的映射。 +- `rule_name_map`, `prompt_name_map`, `llm_name_map`: 名称到类的映射。 + +#### 关键方法 + +##### 注册相关 + +- `rule_register(metric_type, group)`: 装饰器，用于注册规则类，指定其 metric_type 和分组。 +- `llm_register(llm_id)`: 装饰器，用于注册 LLM 类，指定其唯一 ID。 +- `prompt_register(metric_type, group)`: 装饰器，用于注册提示词类，指定其 metric_type 和分组。 + +##### 获取与查询 + +- `get_group(group_name)`: 获取指定分组下的规则和/或提示词。 +- `get_rule_metric_type_map()`: 获取所有规则的 metric_type 映射。 +- `get_metric_type_by_rule_name(rule_name)`: 通过规则名获取其 metric_type。 +- `get_rule_group(rule_group_name)`: 获取指定规则分组的所有规则类。 +- `get_rule_groups()`: 获取所有规则分组。 +- `get_rules_by_group(group_name)`: 获取指定分组下所有规则的名称（含 metric_type）。 +- `get_rule_by_name(name)`: 通过名称获取规则类。 +- `get_llm_name_map()`: 获取所有 LLM 的名称映射。 +- `get_llm(llm_name)`: 通过名称获取 LLM 类。 +- `print_rule_list()`: 打印所有已注册规则的名称。 + +##### 配置应用 + +- `apply_config_rule()`: 应用全局配置中的规则参数到已注册规则。 +- `apply_config_llm()`: 应用全局配置中的 LLM 参数到已注册 LLM。 +- `apply_config_rule_list(eval_group)`: 根据配置文件中的 rule_list，生成分组。 +- `apply_config_prompt_list(eval_group)`: 根据配置文件中的 prompt_list，生成分组。 +- `apply_config(custom_config, eval_group)`: 读取配置文件并应用所有相关配置。 +- `apply_config_for_spark_driver(custom_config, eval_group)`: 专为 Spark Driver 场景设计的配置应用。 + +##### 自动加载 + +- `load_model()`: 自动加载 `rule/`、`prompt/`、`llm/` 目录下的所有 Python 文件，实现自动注册。 + +--- + +## 三、使用示例 + +### 1. 注册规则/LLM/Prompt + +```python +@Model.rule_register(metric_type="QUALITY", group=["default"]) +class MyRule(BaseRule): + ... +``` + +```python +@Model.llm_register(llm_id="gpt3") +class GPT3LLM(BaseLLM): + ... +``` + +### 2. 应用配置 + +```python +Model.apply_config("path/to/config.yaml", eval_group="my_eval_group") +``` + +### 3. 获取分组信息 + +```python +group_info = Model.get_group("default") +rules = group_info.get("rule", []) +prompts = group_info.get("prompt", []) +``` + +--- + +## 四、设计亮点 + +- **自动注册**：通过装饰器和自动扫描目录，极大简化了模型的注册流程。 +- **灵活分组**：支持规则、提示词的多分组和多类型映射，便于扩展和管理。 +- **动态配置**：可通过配置文件动态调整模型参数和分组，适应不同评测场景。 +- **统一接口**：所有模型的获取、注册、配置应用均通过统一接口完成，便于维护。 + +--- + +## 五、注意事项 + +- 注册的类必须继承自对应的基类（如 `BaseRule`, `BasePrompt`, `BaseLLM`）。 +- 配置文件需符合 `GlobalConfig` 的格式要求。 +- 自动加载依赖于目录结构，需保证 `rule/`、`prompt/`、`llm/` 目录下的 Python 文件可被正确导入。 + +--- From d2d0daf81f7b0e17e95091c04a1d3b1a6920b532 Mon Sep 17 00:00:00 2001 From: shijinpjlab Date: Thu, 10 Jul 2025 19:16:48 +0800 Subject: [PATCH 16/24] feat: fix lint --- docs/technical/technical_local.md | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/technical/technical_local.md b/docs/technical/technical_local.md index d39a5ab4..ae649d58 100644 --- a/docs/technical/technical_local.md +++ b/docs/technical/technical_local.md @@ -82,40 +82,40 @@ ## 三、执行流程（有序步骤） -1. **初始化 LocalExecutor** +1. **初始化 LocalExecutor** 创建 LocalExecutor 实例，传入 InputArgs 参数。 -2. **加载数据** +2. **加载数据** 调用 `load_data` 方法，根据输入参数加载数据集，返回数据生成器。 -3. **应用模型配置** +3. **应用模型配置** 调用 `Model.apply_config`，根据配置文件和分组名应用评测规则、Prompt、LLM 等配置。 -4. **创建输出目录** +4. **创建输出目录** 根据当前时间和 UUID 生成唯一输出目录，并在需要时创建。 -5. **选择 LLM** +5. **选择 LLM** 根据配置文件选择并初始化当前使用的大语言模型（LLM）。 -6. **初始化 SummaryModel** +6. **初始化 SummaryModel** 创建 SummaryModel 实例，用于统计和汇总评测任务信息。 -7. **批量评测** +7. **批量评测** 调用 `evaluate` 方法，按 batch_size 分批调度线程池/进程池，对每批数据进行评测。 -8. **对每条数据进行评测（rule/prompt）** +8. **对每条数据进行评测（rule/prompt）** 针对每条数据，分别对 rule 分组和 prompt 分组进行评测，调用相应的评测方法。 -9. **聚合结果，写出单条数据** +9. **聚合结果，写出单条数据** 聚合每条数据的评测结果，写出到对应的输出文件。 -10. **实时更新 summary** +10. **实时更新 summary** 在评测过程中，实时更新 SummaryModel 的统计信息。 -11. **写出 summary.json** +11. **写出 summary.json** 评测结束后，将 summary 信息写出为 summary.json 文件。 -12. **返回 SummaryModel** +12. **返回 SummaryModel** 返回最终的 SummaryModel 结果，供后续分析或展示使用。 --- @@ -161,4 +161,4 @@ print(summary.to_dict()) ## 七、总结 -`dingo.exec.local` 是 Dingo 评测系统的本地执行核心，具备高并发、灵活分组、动态配置、结构化输出等特性，适合大规模自动化评测任务。其设计充分考虑了扩展性与易用性，是构建智能评测流水线的重要基础模块。 \ No newline at end of file +`dingo.exec.local` 是 Dingo 评测系统的本地执行核心，具备高并发、灵活分组、动态配置、结构化输出等特性，适合大规模自动化评测任务。其设计充分考虑了扩展性与易用性，是构建智能评测流水线的重要基础模块。 From 4f7a56f84732dcdf11c183af353be80982d3f08e Mon Sep 17 00:00:00 2001 From: chupei Date: Thu, 10 Jul 2025 19:46:08 +0800 Subject: [PATCH 17/24] feat: add auto metrics gen method (#114) * feat: add auto metrics gen method * fix: isort formatting in rule_common.py --- .github/workflows/metrics-validation.yml | 53 ++ .pre-commit-config.yaml | 2 + README.md | 75 +-- README_ja.md | 85 +--- README_zh-CN.md | 76 +-- app_gradio/app.py | 1 + dingo/config/config.py | 3 +- dingo/data/converter/img_utils.py | 3 +- dingo/data/dataset/huggingface.py | 1 + dingo/data/datasource/huggingface.py | 1 + dingo/data/datasource/s3.py | 1 + dingo/data/utils/digit.py | 3 +- dingo/exec/local.py | 3 +- dingo/exec/spark.py | 7 +- dingo/model/llm/base_lmdeploy_apiclient.py | 3 +- dingo/model/llm/base_openai.py | 3 +- dingo/model/model.py | 3 +- dingo/model/prompt/prompt_classify_qr.py | 9 + dingo/model/prompt/prompt_classify_topic.py | 13 + .../model/prompt/prompt_dataman_assessment.py | 12 + dingo/model/prompt/prompt_image_relevant.py | 9 + dingo/model/prompt/prompt_politics.py | 14 +- dingo/model/prompt/prompt_text_3h.py | 36 ++ dingo/model/prompt/prompt_text_quality.py | 11 + .../prompt/prompt_text_quality_multilan.py | 3 +- dingo/model/rule/rule_common.py | 455 +++++++++++++++++- dingo/model/rule/rule_image.py | 78 ++- dingo/model/rule/utils/detect_lang.py | 3 +- dingo/run/cli.py | 1 + dingo/run/web.py | 5 +- dingo/utils/log_util/__init__.py | 3 +- docs/metrics.md | 65 ++- examples/spark/sdk_spark.py | 3 +- mcp_server.py | 3 +- scripts/generate_metrics.py | 270 +++++++++++ setup.cfg | 4 +- test/scripts/data/dataset/test_hf_dataset.py | 1 + .../data/datasource/test_hf_datasource.py | 1 + test/scripts/exec/test_local.py | 1 + test/scripts/io/input/test_continue.py | 1 + test/scripts/io/input/test_write.py | 1 + .../model/rule/utils/test_rule_utils.py | 1 + 42 files changed, 1082 insertions(+), 244 deletions(-) create mode 100644 .github/workflows/metrics-validation.yml create mode 100644 scripts/generate_metrics.py diff --git a/.github/workflows/metrics-validation.yml b/.github/workflows/metrics-validation.yml new file mode 100644 index 00000000..32c630c2 --- /dev/null +++ b/.github/workflows/metrics-validation.yml @@ -0,0 +1,53 @@ +name: Metrics Documentation Auto-Update + +on: + push: + branches: [ main, dev ] + paths: + - 'dingo/model/prompt/**' + - 'scripts/generate_metrics.py' + pull_request: + branches: [ main ] + paths: + - 'dingo/model/prompt/**' + - 'scripts/generate_metrics.py' + workflow_dispatch: + +jobs: + update-metrics-docs: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e . + + - name: Generate metrics documentation + run: | + python scripts/generate_metrics.py + + - name: Check if documentation changed + id: check_changes + run: | + if git diff --quiet docs/metrics.md; then + echo "changed=false" >> $GITHUB_OUTPUT + else + echo "changed=true" >> $GITHUB_OUTPUT + fi + + - name: Commit updated documentation + if: steps.check_changes.outputs.changed == 'true' && github.event_name == 'push' + run: | + git config --local user.email "action@github.com" + git config --local user.name "GitHub Action" + git add docs/metrics.md + git commit -m "📚 Auto-update metrics documentation" + git push diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 36ad15b2..50fda2cc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,12 +6,14 @@ repos: hooks: - id: trailing-whitespace - id: end-of-file-fixer + exclude: 'docs/metrics\.md' - id: check-yaml - id: check-added-large-files - repo: https://github.com/PyCQA/isort rev: 6.0.1 hooks: - id: isort + args: [ "-l", "200", "-m", "0", "-p", "dingo" ] - repo: https://github.com/PyCQA/flake8 rev: 7.2.0 hooks: diff --git a/README.md b/README.md index 363062ba..6f275f10 100644 --- a/README.md +++ b/README.md @@ -179,61 +179,19 @@ This video demonstrates step-by-step how to use Dingo MCP server with Cursor. # Data Quality Metrics -Dingo classifies data quality issues into 7 dimensions of Quality Metrics. Each dimension can be evaluated using both rule-based methods and LLM-based prompts: +Dingo provides comprehensive data quality assessment through both rule-based and prompt-based evaluation metrics. These metrics cover multiple quality dimensions including effectiveness, completeness, similarity, security, and more. -| Quality Metric | Description | Rule Examples | LLM Prompt Examples | -|-------------------|-------------|---------------|---------------------| -| **COMPLETENESS** | Checks if data is incomplete or missing | `RuleColonEnd`, `RuleContentNull` | Evaluates if text abruptly ends with a colon or ellipsis, has mismatched parentheses, or missing critical components | -| **EFFECTIVENESS** | Checks if data is meaningful and properly formatted | `RuleAbnormalChar`, `RuleHtmlEntity`, `RuleSpecialCharacter` | Detects garbled text, words stuck together without spaces, and text lacking proper punctuation | -| **FLUENCY** | Checks if text is grammatically correct and reads naturally | `RuleAbnormalNumber`, `RuleNoPunc`, `RuleWordStuck` | Identifies excessively long words, text fragments without punctuation, or content with chaotic reading order | -| **RELEVANCE** | Detects irrelevant content within the data | `RuleHeadWord` variants for different languages | Examines for irrelevant information like citation details, headers/footers, entity markers, HTML tags | -| **SECURITY** | Identifies sensitive information or value conflicts | `RuleIDCard`, `RuleUnsafeWords` | Checks for personal information, and content related to gambling, pornography, political issues | -| **SIMILARITY** | Detects repetitive or highly similar content | `RuleDocRepeat` | Evaluates text for consecutive repeated content or multiple occurrences of special characters | -| **UNDERSTANDABILITY** | Assesses how easily data can be interpreted | `RuleCapitalWords` | Ensures LaTeX formulas and Markdown are correctly formatted, with proper segmentation and line breaks | +📊 **[View Complete Metrics Documentation →](docs/metrics.md)** -## LLM Quality Assessment +Our evaluation system includes: +- **Text Quality Assessment Metrics**: Pre-training data quality evaluation using DataMan methodology and enhanced multi-dimensional assessment +- **SFT Data Assessment Metrics**: Honest, Helpful, Harmless evaluation for supervised fine-tuning data +- **Classification Metrics**: Topic categorization and content classification +- **Multimodality Assessment Metrics**: Image classification and relevance evaluation +- **Rule-Based Quality Metrics**: Automated quality checks using heuristic rules for effectiveness and similarity detection +- etc -Dingo provides several LLM-based assessment methods defined by prompts in the `dingo/model/prompt` directory. These prompts are registered using the `prompt_register` decorator and can be combined with LLM models for quality evaluation: - -### Text Quality Assessment Prompts - -| Prompt Type | Metric | Description | -|-------------|--------|-------------| -| `TEXT_QUALITY_V2`, `TEXT_QUALITY_V3` | Various quality dimensions | Comprehensive text quality evaluation covering effectiveness, relevance, completeness, understandability, similarity, fluency, and security | -| `QUALITY_BAD_EFFECTIVENESS` | Effectiveness | Detects garbled text and anti-crawling content | -| `QUALITY_BAD_SIMILARITY` | Similarity | Identifies text repetition issues | -| `WORD_STICK` | Fluency | Checks for words stuck together without proper spacing | -| `CODE_LIST_ISSUE` | Completeness | Evaluates code blocks and list formatting issues | -| `UNREAD_ISSUE` | Effectiveness | Detects unreadable characters due to encoding issues | - -### 3H Assessment Prompts (Honest, Helpful, Harmless) - -| Prompt Type | Metric | Description | -|-------------|--------|-------------| -| `QUALITY_HONEST` | Honesty | Evaluates if responses provide accurate information without fabrication or deception | -| `QUALITY_HELPFUL` | Helpfulness | Assesses if responses address questions directly and follow instructions appropriately | -| `QUALITY_HARMLESS` | Harmlessness | Checks if responses avoid harmful content, discriminatory language, and dangerous assistance | - -### Domain-Specific Assessment Prompts - -| Prompt Type | Metric | Description | -|-------------|--------|-------------| -| `TEXT_QUALITY_KAOTI` | Exam question quality | Specialized assessment for evaluating the quality of exam questions, focusing on formula rendering, table formatting, paragraph structure, and answer formatting | -| `Html_Abstract` | HTML extraction quality | Compares different methods of extracting Markdown from HTML, evaluating completeness, formatting accuracy, and semantic coherence | -| `DATAMAN_ASSESSMENT` | Data Quality & Domain | Evaluates pre-training data quality using the DataMan methodology (14 standards, 15 domains). Assigns a score (0/1), domain type, quality status, and reason. | - -### Classification Prompts - -| Prompt Type | Metric | Description | -|-------------|--------|-------------| -| `CLASSIFY_TOPIC` | Topic Categorization | Classifies text into categories like language processing, writing, code, mathematics, role-play, or knowledge Q&A | -| `CLASSIFY_QR` | Image Classification | Identifies images as CAPTCHA, QR code, or normal images | - -### Image Assessment Prompts - -| Prompt Type | Metric | Description | -|-------------|--------|-------------| -| `IMAGE_RELEVANCE` | Image Relevance | Evaluates if an image matches reference image in terms of face count, feature details, and visual elements | +Most metrics are backed by academic sources to ensure objectivity and scientific rigor. ### Using LLM Assessment in Evaluation @@ -417,19 +375,6 @@ Example summary: } ``` - -# Research & Publications - -## Research Powered by Dingo -- **WanJuanSiLu**: [A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/pdf/2501.14506) - *Uses Dingo for comprehensive data quality assessment of multilingual web data* - -## Methodologies Implemented in Dingo -- **DataMan Methodology**: [DataMan: Data Manager for Pre-training Large Language Models](https://openreview.net/pdf?id=eNbA8Fqir4) - *Dingo implements the DataMan methodology for pre-training data quality assessment* -- **RedPajama-Data-v2**: [RedPajama-Data](https://github.com/togethercomputer/RedPajama-Data) - *Dingo implements parts of the RedPajama-Data-v2 methodology for web text quality assessment and filtering* - # Future Plans - [ ] Richer graphic and text evaluation indicators diff --git a/README_ja.md b/README_ja.md index bf111e98..57ee7e3a 100644 --- a/README_ja.md +++ b/README_ja.md @@ -177,61 +177,19 @@ https://github.com/user-attachments/assets/aca26f4c-3f2e-445e-9ef9-9331c4d7a37b # データ品質メトリクス -Dingoはデータ品質問題を7つの品質メトリクス次元に分類します。各次元は、ルールベース手法とLLMベースプロンプトの両方で評価できます： +Dingoはルールベースおよびプロンプトベースの評価メトリクスを通じて包括的なデータ品質評価を提供します。これらのメトリクスは、効果性、完全性、類似性、セキュリティなどの複数の品質次元をカバーしています。 -| 品質メトリクス | 説明 | ルール例 | LLMプロンプト例 | -|----------------|------|----------|-----------------| -| **COMPLETENESS** | データが不完全または欠落していないかをチェック | `RuleColonEnd`, `RuleContentNull` | テキストがコロンや省略記号で突然終わっているか、括弧が不一致か、重要な要素が欠落していないかを評価 | -| **EFFECTIVENESS** | データが意味があり適切にフォーマットされているかをチェック | `RuleAbnormalChar`, `RuleHtmlEntity`, `RuleSpecialCharacter` | 文字化けテキスト、スペースなしで結合された単語、適切な句読点のないテキストを検出 | -| **FLUENCY** | テキストが文法的に正しく自然に読めるかをチェック | `RuleAbnormalNumber`, `RuleNoPunc`, `RuleWordStuck` | 過度に長い単語、句読点のないテキスト断片、読み順が混乱したコンテンツを識別 | -| **RELEVANCE** | データ内の無関係なコンテンツを検出 | 異なる言語用の`RuleHeadWord`バリアント | 引用詳細、ヘッダー/フッター、エンティティマーカー、HTMLタグなどの無関係な情報を検査 | -| **SECURITY** | 機密情報や価値観の対立を識別 | `RuleIDCard`, `RuleUnsafeWords` | 個人情報、ギャンブル、ポルノ、政治問題に関連するコンテンツをチェック | -| **SIMILARITY** | 反復的または非常に類似したコンテンツを検出 | `RuleDocRepeat` | 連続した反復コンテンツや特殊文字の複数出現についてテキストを評価 | -| **UNDERSTANDABILITY** | データがどれだけ容易に解釈できるかを評価 | `RuleCapitalWords` | LaTeX数式とMarkdownが正しくフォーマットされ、適切なセグメンテーションと改行があることを確認 | +📊 **[完全なメトリクス文書を表示 →](docs/metrics.md)** -## LLM品質評価 +評価システムには以下が含まれます： +- **テキスト品質評価メトリクス**: DataMan手法と拡張された多次元評価を使用した事前学習データの品質評価 +- **SFTデータ評価メトリクス**: 教師ありファインチューニングデータの正直、有用、無害評価 +- **分類メトリクス**: トピック分類とコンテンツ分類 +- **マルチモーダル評価メトリクス**: 画像分類と関連性評価 +- **ルールベース品質メトリクス**: ヒューリスティックルールによる効果性と類似性検出を用いた自動品質チェック +- など -Dingoは`dingo/model/prompt`ディレクトリ内のプロンプトで定義された複数のLLMベース評価手法を提供します。これらのプロンプトは`prompt_register`デコレータを使用して登録され、品質評価のためにLLMモデルと組み合わせることができます： - -### テキスト品質評価プロンプト - -| プロンプトタイプ | メトリクス | 説明 | -|------------------|------------|------| -| `TEXT_QUALITY_V2`, `TEXT_QUALITY_V3` | 様々な品質次元 | 効果性、関連性、完全性、理解しやすさ、類似性、流暢性、セキュリティを含む包括的なテキスト品質評価 | -| `QUALITY_BAD_EFFECTIVENESS` | 効果性 | 文字化けテキストとアンチクローリングコンテンツを検出 | -| `QUALITY_BAD_SIMILARITY` | 類似性 | テキスト反復問題を識別 | -| `WORD_STICK` | 流暢性 | 適切なスペースなしで結合された単語をチェック | -| `CODE_LIST_ISSUE` | 完全性 | コードブロックとリストフォーマット問題を評価 | -| `UNREAD_ISSUE` | 効果性 | エンコーディング問題による読み取り不可能な文字を検出 | - -### 3H評価プロンプト（正直、有用、無害） - -| プロンプトタイプ | メトリクス | 説明 | -|------------------|------------|------| -| `QUALITY_HONEST` | 正直性 | 捏造や欺瞞なしに正確な情報を提供するかどうかを評価 | -| `QUALITY_HELPFUL` | 有用性 | 質問に直接答え、指示に適切に従うかどうかを評価 | -| `QUALITY_HARMLESS` | 無害性 | 有害なコンテンツ、差別的言語、危険な支援を避けるかどうかをチェック | - -### ドメイン固有評価プロンプト - -| プロンプトタイプ | メトリクス | 説明 | -|------------------|------------|------| -| `TEXT_QUALITY_KAOTI` | 試験問題品質 | 数式レンダリング、表フォーマット、段落構造、回答フォーマットに焦点を当てた試験問題品質の専門評価 | -| `Html_Abstract` | HTML抽出品質 | HTMLからMarkdownを抽出する異なる手法を比較し、完全性、フォーマット精度、意味的一貫性を評価 | -| `DATAMAN_ASSESSMENT` | データ品質とドメイン | DataMan手法（14基準、15ドメイン）を使用して事前学習データ品質を評価。スコア（0/1）、ドメインタイプ、品質状態、理由を割り当て | - -### 分類プロンプト - -| プロンプトタイプ | メトリクス | 説明 | -|------------------|------------|------| -| `CLASSIFY_TOPIC` | トピック分類 | 言語処理、執筆、コード、数学、ロールプレイ、知識Q&Aなどのカテゴリにテキストを分類 | -| `CLASSIFY_QR` | 画像分類 | 画像をCAPTCHA、QRコード、または通常の画像として識別 | - -### 画像評価プロンプト - -| プロンプトタイプ | メトリクス | 説明 | -|------------------|------------|------| -| `IMAGE_RELEVANCE` | 画像関連性 | 顔の数、特徴の詳細、視覚的要素の観点から、画像が参照画像と一致するかを評価 | +大部分のメトリクスは学術的なソースによって支持されており、客観性と科学的厳密性を保証しています。 ### 評価でのLLM評価の使用 @@ -284,10 +242,12 @@ input_data = { ## ルールベース・モデルベース評価 -- **組み込みルール**: 20以上の一般的なヒューリスティック評価ルール -- **LLM統合**: OpenAI、Kimi、ローカルモデル（例：Llama3） -- **カスタムルール**: 独自のルールとモデルで簡単に拡張 -- **セキュリティ評価**: Perspective API統合 +評価システムには以下が含まれます： +- **テキスト品質評価メトリクス**: DataMan手法と拡張された多次元評価を使用した事前学習データの品質評価 +- **SFTデータ評価メトリクス**: 教師ありファインチューニングデータの正直、有用、無害評価 +- **分類メトリクス**: トピック分類とコンテンツ分類 +- **マルチモーダル評価メトリクス**: 画像分類と関連性評価 +- **ルールベース品質メトリクス**: ヒューリスティックルールによる効果性と類似性検出を用いた自動品質チェック ## 柔軟な使用方法 @@ -415,19 +375,6 @@ result = executor.execute() } ``` - -# 研究・出版物 - -## Dingoを活用した研究 -- **WanJuanSiLu**: [A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/pdf/2501.14506) - *多言語ウェブデータの包括的なデータ品質評価にDingoを使用* - -## Dingoに実装された手法 -- **DataMan手法**: [DataMan: Data Manager for Pre-training Large Language Models](https://openreview.net/pdf?id=eNbA8Fqir4) - *Dingoは事前学習データ品質評価のためのDataMan手法を実装* -- **RedPajama-Data-v2**: [RedPajama-Data](https://github.com/togethercomputer/RedPajama-Data) - *Dingoはウェブテキストの品質評価とフィルタリングのためのRedPajama-Data-v2手法の一部を実装* - # 今後の計画 - [ ] より豊富なグラフィックとテキスト評価指標 diff --git a/README_zh-CN.md b/README_zh-CN.md index 203f526e..ea496d4c 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -176,61 +176,19 @@ https://github.com/user-attachments/assets/aca26f4c-3f2e-445e-9ef9-9331c4d7a37b # 数据质量指标 -Dingo将数据质量问题分为7个维度的质量指标。每个维度可以通过基于规则的方法和基于LLM的prompt进行评估： +Dingo通过基于规则和基于提示的评估指标提供全面的数据质量评估。这些指标涵盖多个质量维度，包括有效性、完整性、相似性、安全性等。 -| 质量指标 | 描述 | 规则示例 | LLM Prompt示例 | -|-------------------|-------------|---------------|---------------------| -| **完整性(COMPLETENESS)** | 检查数据是否不完整或缺失 | `RuleColonEnd`, `RuleContentNull` | 评估文本是否突然以冒号或省略号结束，是否有不匹配的括号，或缺少关键组件 | -| **有效性(EFFECTIVENESS)** | 检查数据是否有意义且格式正确 | `RuleAbnormalChar`, `RuleHtmlEntity`, `RuleSpecialCharacter` | 检测乱码文本、没有空格的粘连单词和缺少适当标点的文本 | -| **流畅性(FLUENCY)** | 检查文本是否语法正确且自然易读 | `RuleAbnormalNumber`, `RuleNoPunc`, `RuleWordStuck` | 识别过长的单词、无标点的文本片段或阅读顺序混乱的内容 | -| **相关性(RELEVANCE)** | 检测数据中的不相关内容 | 不同语言的`RuleHeadWord`变体 | 检查引用详情、页眉/页脚、实体标记、HTML标签等不相关信息 | -| **安全性(SECURITY)** | 识别敏感信息或价值冲突 | `RuleIDCard`, `RuleUnsafeWords` | 检查个人信息和与赌博、色情、政治问题相关的内容 | -| **相似性(SIMILARITY)** | 检测重复或高度相似的内容 | `RuleDocRepeat` | 评估文本中连续重复的内容或特殊字符的多次出现 | -| **可理解性(UNDERSTANDABILITY)** | 评估数据解释的容易程度 | `RuleCapitalWords` | 确保LaTeX公式和Markdown格式正确，具有适当的分段和换行 | +📊 **[查看完整指标文档 →](docs/metrics.md)** -## LLM质量评估 +我们的评估系统包括： +- **文本质量评估指标**：使用DataMan方法论和增强的多维评估进行预训练数据质量评估 +- **SFT数据评估指标**：针对监督微调数据的诚实、有帮助、无害评估 +- **分类指标**：主题分类和内容分类 +- **多模态评估指标**：图像分类和相关性评估 +- **基于规则的质量指标**：使用启发式规则进行效果性和相似性检测的自动化质量检查 +- 等等 -Dingo在`dingo/model/prompt`目录下提供了多种基于LLM的评估方法。这些prompt使用`prompt_register`装饰器注册，可以与LLM模型结合进行质量评估： - -### 文本质量评估Prompt - -| Prompt类型 | 指标 | 描述 | -|-------------|--------|-------------| -| `TEXT_QUALITY_V2`, `TEXT_QUALITY_V3` | 多种质量维度 | 全面的文本质量评估，涵盖有效性、相关性、完整性、可理解性、相似性、流畅性和安全性 | -| `QUALITY_BAD_EFFECTIVENESS` | 有效性 | 检测乱码文本和反爬虫内容 | -| `QUALITY_BAD_SIMILARITY` | 相似性 | 识别文本重复问题 | -| `WORD_STICK` | 流畅性 | 检查单词是否缺少适当间距而粘连在一起 | -| `CODE_LIST_ISSUE` | 完整性 | 评估代码块和列表格式问题 | -| `UNREAD_ISSUE` | 有效性 | 检测由编码问题导致的不可读字符 | - -### 3H评估Prompt (诚实、有帮助、无害) - -| Prompt类型 | 指标 | 描述 | -|-------------|--------|-------------| -| `QUALITY_HONEST` | 诚实性 | 评估回答是否提供准确信息，不含虚构或欺骗内容 | -| `QUALITY_HELPFUL` | 有帮助性 | 评估回答是否直接解决问题并适当遵循指令 | -| `QUALITY_HARMLESS` | 无害性 | 检查回答是否避免有害内容、歧视性语言和危险指导 | - -### 领域专用评估Prompt - -| Prompt类型 | 指标 | 描述 | -|-------------|--------|-------------| -| `TEXT_QUALITY_KAOTI` | 考题质量 | 专门评估考试题目的质量，关注公式渲染、表格格式、段落结构和答案格式 | -| `Html_Abstract` | HTML提取质量 | 比较从HTML提取Markdown的不同方法，评估完整性、格式准确性和语义连贯性 | -| `DATAMAN_ASSESSMENT` | 数据质量与领域 | 使用DataMan方法论（14个标准，15个领域）评估预训练数据质量。分配分数（0/1）、领域类型、质量状态和原因。 | - -### 分类Prompt - -| Prompt类型 | 指标 | 描述 | -|-------------|--------|-------------| -| `CLASSIFY_TOPIC` | 主题分类 | 将文本分类为语言处理、写作、代码、数学、角色扮演或知识问答等类别 | -| `CLASSIFY_QR` | 图像分类 | 识别图像为验证码、二维码或普通图像 | - -### 图像评估Prompt - -| Prompt类型 | 指标 | 描述 | -|-------------|--------|-------------| -| `IMAGE_RELEVANCE` | 图像相关性 | 评估图像是否在面部数量、特征细节和视觉元素方面与参考图像匹配 | +大部分指标都由学术来源支持，以确保客观性和科学严谨性。 ### 在评估中使用LLM评估 @@ -254,8 +212,6 @@ input_data = { 您可以自定义这些prompt，以关注特定的质量维度或适应特定的领域需求。当与适当的LLM模型结合时，这些prompt能够在多个维度上对数据质量进行全面评估。 -每条规则都针对文本质量的特定方面进行检查，并映射到这些指标之一。运行评估时，Dingo将提供每个维度的分数并识别触发了哪些规则。 - # 规则组 Dingo为不同类型的数据集提供预配置的规则组： @@ -416,18 +372,6 @@ result = executor.execute() } ``` -# 研究与学术成果 - -## Dingo驱动的研究 -- **WanJuanSiLu**: [A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/pdf/2501.14506) - *使用Dingo对多语言网页数据进行全面的数据质量评估* - -## Dingo实现的方法论 -- **DataMan方法论**: [DataMan: Data Manager for Pre-training Large Language Models](https://openreview.net/pdf?id=eNbA8Fqir4) - *Dingo实现了DataMan方法论用于预训练数据质量评估* -- **RedPajama-Data-v2**: [RedPajama-Data](https://github.com/togethercomputer/RedPajama-Data) - *Dingo实现了部分RedPajama-Data-v2方法论用于网页文本质量评估和过滤* - # 未来计划 - [ ] 更丰富的图文评测指标 diff --git a/app_gradio/app.py b/app_gradio/app.py index ca1d94c2..4426fd4b 100644 --- a/app_gradio/app.py +++ b/app_gradio/app.py @@ -4,6 +4,7 @@ from pathlib import Path import gradio as gr + from dingo.exec import Executor from dingo.io import InputArgs diff --git a/dingo/config/config.py b/dingo/config/config.py index 623b488a..2b4638b6 100644 --- a/dingo/config/config.py +++ b/dingo/config/config.py @@ -1,9 +1,10 @@ import json from typing import Dict, List, Optional -from dingo.utils import log from pydantic import BaseModel +from dingo.utils import log + class DynamicRuleConfig(BaseModel): threshold: Optional[float] = None diff --git a/dingo/data/converter/img_utils.py b/dingo/data/converter/img_utils.py index b3289144..b4d141d8 100644 --- a/dingo/data/converter/img_utils.py +++ b/dingo/data/converter/img_utils.py @@ -6,9 +6,10 @@ from botocore.exceptions import ClientError from botocore.response import StreamingBody +from PIL import Image + from dingo.data.datasource import S3DataSource from dingo.io import InputArgs -from PIL import Image def try_close(obj): diff --git a/dingo/data/dataset/huggingface.py b/dingo/data/dataset/huggingface.py index 329d8021..af68382a 100644 --- a/dingo/data/dataset/huggingface.py +++ b/dingo/data/dataset/huggingface.py @@ -2,6 +2,7 @@ from typing import Any, Dict, Generator, Mapping, Optional, Sequence, Union import datasets + from dingo.data.dataset.base import Dataset from dingo.data.datasource import DataSource from dingo.data.datasource.huggingface import HuggingFaceSource diff --git a/dingo/data/datasource/huggingface.py b/dingo/data/datasource/huggingface.py index 612230df..dbd66cd2 100644 --- a/dingo/data/datasource/huggingface.py +++ b/dingo/data/datasource/huggingface.py @@ -1,6 +1,7 @@ from typing import Any, Dict, Mapping, Optional, Sequence, Union import datasets + from dingo.data.datasource.base import DataSource from dingo.io import InputArgs diff --git a/dingo/data/datasource/s3.py b/dingo/data/datasource/s3.py index 36dfcf06..6879bb5d 100644 --- a/dingo/data/datasource/s3.py +++ b/dingo/data/datasource/s3.py @@ -3,6 +3,7 @@ import boto3 import boto3.s3 from botocore.config import Config + from dingo.data.datasource.base import DataSource from dingo.io import InputArgs diff --git a/dingo/data/utils/digit.py b/dingo/data/utils/digit.py index d37b830d..0cf3637a 100644 --- a/dingo/data/utils/digit.py +++ b/dingo/data/utils/digit.py @@ -18,9 +18,10 @@ import logging from typing import Any, List -from dingo.data.utils import insecure_hash from packaging.version import Version +from dingo.data.utils import insecure_hash + logger = logging.getLogger(__name__) logger.setLevel("ERROR") MAX_ROWS = 10000 diff --git a/dingo/exec/local.py b/dingo/exec/local.py index d55bb9c3..437da125 100644 --- a/dingo/exec/local.py +++ b/dingo/exec/local.py @@ -7,6 +7,8 @@ import uuid from typing import Generator, List, Optional +from tqdm import tqdm + from dingo.config import GlobalConfig from dingo.data import Dataset, DataSource, dataset_map, datasource_map from dingo.exec.base import ExecProto, Executor @@ -17,7 +19,6 @@ from dingo.model.prompt.base import BasePrompt from dingo.model.rule.base import BaseRule from dingo.utils import log -from tqdm import tqdm @Executor.register("local") diff --git a/dingo/exec/spark.py b/dingo/exec/spark.py index f8ac5a15..d5445f0f 100644 --- a/dingo/exec/spark.py +++ b/dingo/exec/spark.py @@ -3,6 +3,10 @@ import uuid from typing import Any, Dict, List, Optional +from pyspark import SparkConf +from pyspark.rdd import RDD +from pyspark.sql import SparkSession + from dingo.config import GlobalConfig from dingo.exec.base import ExecProto, Executor from dingo.io import Data, InputArgs, ResultInfo, SummaryModel @@ -11,9 +15,6 @@ from dingo.model.modelres import ModelRes from dingo.model.prompt.base import BasePrompt from dingo.model.rule.base import BaseRule -from pyspark import SparkConf -from pyspark.rdd import RDD -from pyspark.sql import SparkSession @Executor.register("spark") diff --git a/dingo/model/llm/base_lmdeploy_apiclient.py b/dingo/model/llm/base_lmdeploy_apiclient.py index 6bb2b62e..fa37dc8a 100644 --- a/dingo/model/llm/base_lmdeploy_apiclient.py +++ b/dingo/model/llm/base_lmdeploy_apiclient.py @@ -2,6 +2,8 @@ import time from typing import List +from pydantic import ValidationError + from dingo.config.config import DynamicLLMConfig from dingo.io import Data from dingo.model.llm.base import BaseLLM @@ -10,7 +12,6 @@ from dingo.model.response.response_class import ResponseScoreReason from dingo.utils import log from dingo.utils.exception import ConvertJsonError, ExceedMaxTokens -from pydantic import ValidationError class BaseLmdeployApiClient(BaseLLM): diff --git a/dingo/model/llm/base_openai.py b/dingo/model/llm/base_openai.py index 52d50864..6f8cbc7d 100644 --- a/dingo/model/llm/base_openai.py +++ b/dingo/model/llm/base_openai.py @@ -2,6 +2,8 @@ import time from typing import Dict, List +from pydantic import ValidationError + from dingo.config.config import DynamicLLMConfig from dingo.io import Data from dingo.model.llm.base import BaseLLM @@ -10,7 +12,6 @@ from dingo.model.response.response_class import ResponseScoreReason from dingo.utils import log from dingo.utils.exception import ConvertJsonError, ExceedMaxTokens -from pydantic import ValidationError class BaseOpenAI(BaseLLM): diff --git a/dingo/model/model.py b/dingo/model/model.py index d937ffa3..49297fb9 100644 --- a/dingo/model/model.py +++ b/dingo/model/model.py @@ -3,12 +3,13 @@ import os from typing import Callable, Dict, List, Optional +from pydantic import BaseModel + from dingo.config import GlobalConfig from dingo.model.llm.base import BaseLLM from dingo.model.prompt.base import BasePrompt from dingo.model.rule.base import BaseRule from dingo.utils import log -from pydantic import BaseModel class BaseEvalModel(BaseModel): diff --git a/dingo/model/prompt/prompt_classify_qr.py b/dingo/model/prompt/prompt_classify_qr.py index ba9bb865..6ccfee1b 100644 --- a/dingo/model/prompt/prompt_classify_qr.py +++ b/dingo/model/prompt/prompt_classify_qr.py @@ -4,6 +4,15 @@ @Model.prompt_register("CLASSIFY_QR", []) class PromptClassifyQR(BasePrompt): + + # Metadata for documentation generation + _metric_info = { + "category": "Multimodality Assessment Metrics", + "metric_name": "Image Classification", + "description": "Identifies images as CAPTCHA, QR code, or normal images", + "evaluation_results": "" + } + content = """ 'Classify the image into one of the following categories: "CAPTCHA", "QR code", or "Normal image". ' 'Return the type as the image category (CAPTCHA or QR code or Normal image) and the reason as the specific type of CAPTCHA or QR code. ' diff --git a/dingo/model/prompt/prompt_classify_topic.py b/dingo/model/prompt/prompt_classify_topic.py index a89bb411..ea6001cb 100644 --- a/dingo/model/prompt/prompt_classify_topic.py +++ b/dingo/model/prompt/prompt_classify_topic.py @@ -4,6 +4,19 @@ @Model.prompt_register("CLASSIFY_TOPIC", []) class PromptClassifyTopic(BasePrompt): + + # Metadata for documentation generation + _metric_info = { + "category": "Classification Metrics", + "metric_name": "Topic Categorization", + "description": "Classifies text into categories like language processing, writing, code, mathematics, role-play, or knowledge Q&A. Based on BERTopic and INSTAG methodologies", + "paper_title": "BERTopic & INSTAG", + "paper_url": "https://maartengr.github.io/BERTopic/index.html#quick-start, https://arxiv.org/pdf/2308.07074", + "paper_authors": "Grootendorst, 2022; Wei et al., 2023", + "evaluation_results": "docs/eval/prompt/text_data_classified_by_topic.md", + "validation_dataset": "AlignBench (https://github.com/THUDM/AlignBench)" + } + content = """ Assume you are a topic classifier, and your task is to categorize user-provided instructions. There are six options in the list provided. You are required to select one category from the following list: ["Language Understanding and Processing", "Writing Ability", "Code", "Mathematics & Reasoning", "Task-oriented Role Play", "Knowledge-based Question and Answering"]. diff --git a/dingo/model/prompt/prompt_dataman_assessment.py b/dingo/model/prompt/prompt_dataman_assessment.py index 016a6f4b..930dde77 100644 --- a/dingo/model/prompt/prompt_dataman_assessment.py +++ b/dingo/model/prompt/prompt_dataman_assessment.py @@ -84,4 +84,16 @@ @Model.prompt_register("DATAMAN_ASSESSMENT", []) class PromptDataManAssessment(BasePrompt): + + # Metadata for documentation generation + _metric_info = { + "category": "Text Quality Assessment Metrics", + "metric_name": "DATAMAN", + "description": "Evaluates pre-training data quality using the DataMan methodology (14 standards, 15 domains). Assigns a score (0/1), domain type, quality status, and reason.", + "paper_title": "DataMan: Data Manager for Pre-training Large Language Models", + "paper_url": "https://arxiv.org/abs/2502.19363", + "paper_authors": "Peng et al., 2025", + "evaluation_results": "" + } + content = ROLE + DATAMAN_ASSESSMENT diff --git a/dingo/model/prompt/prompt_image_relevant.py b/dingo/model/prompt/prompt_image_relevant.py index b1a11818..ff37566c 100644 --- a/dingo/model/prompt/prompt_image_relevant.py +++ b/dingo/model/prompt/prompt_image_relevant.py @@ -4,6 +4,15 @@ @Model.prompt_register("IMAGE_RELEVANT", []) class PromptImageRelevant(BasePrompt): + + # Metadata for documentation generation + _metric_info = { + "category": "Multimodality Assessment Metrics", + "metric_name": "Image Relevance", + "description": "Evaluates if an image matches reference image in terms of face count, feature details, and visual elements", + "evaluation_results": "" + } + content = """ 作为一款专业的图片检测AI工具，请结合第一张图评估第二张图片是否符合标准。请先分析第一张图片，包括背景信息、人脸数量、以及每个人物的脸部和手部特征。 然后根据以下标准对第二张图片进行评分：\n diff --git a/dingo/model/prompt/prompt_politics.py b/dingo/model/prompt/prompt_politics.py index 27a5b1b0..b69e2321 100644 --- a/dingo/model/prompt/prompt_politics.py +++ b/dingo/model/prompt/prompt_politics.py @@ -2,8 +2,20 @@ from dingo.model.prompt.base import BasePrompt -@Model.prompt_register("TEXT_POLITICS", []) +@Model.prompt_register("QUALITY_BAD_SECURITY", []) class PromptPolitics(BasePrompt): + + # Metadata for documentation generation + _metric_info = { + "category": "Text Quality Assessment Metrics", + "metric_name": "Politics", + "description": "Evaluates whether the text contains politics-related content", + "paper_title": "", + "paper_url": "", + "paper_authors": "", + "evaluation_results": "" + } + content = """ 你是一名内容安全审查员，可以用英语流利沟通，会站在中国人的角度思考问题。我将提供一段文字，需要你判断其内容是否属于政治敏感内容，并且立场是否为负面。政治敏感内容包括以下五个方面，以下内容解释了对应方面的负面立场： diff --git a/dingo/model/prompt/prompt_text_3h.py b/dingo/model/prompt/prompt_text_3h.py index 8d92c89e..6c7ccfc4 100644 --- a/dingo/model/prompt/prompt_text_3h.py +++ b/dingo/model/prompt/prompt_text_3h.py @@ -4,6 +4,18 @@ @Model.prompt_register("QUALITY_HELPFUL", []) class PromptTextHelpful(BasePrompt): + + # Metadata for documentation generation + _metric_info = { + "category": "SFT Data Assessment Metrics", + "metric_name": "Helpfulness", + "description": "Assesses if responses address questions directly and follow instructions appropriately", + "paper_title": "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback", + "paper_url": "https://arxiv.org/pdf/2204.05862", + "paper_authors": "Bai et al., 2022", + "evaluation_results": "docs/eval/prompt/qa_data_evaluated_by_3h.md" + } + content = """ Please act as an impartial judge and evaluate the quality of the responses provided by the AI assistant to the user question displayed below. Determine whether the response is helpful. @@ -30,6 +42,18 @@ class PromptTextHelpful(BasePrompt): @Model.prompt_register("QUALITY_HARMLESS", []) class PromptTextHarmless(BasePrompt): + + # Metadata for documentation generation + _metric_info = { + "category": "SFT Data Assessment Metrics", + "metric_name": "Harmlessness", + "description": "Checks if responses avoid harmful content, discriminatory language, and dangerous assistance", + "paper_title": "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback", + "paper_url": "https://arxiv.org/pdf/2204.05862", + "paper_authors": "Bai et al., 2022", + "evaluation_results": "docs/eval/prompt/qa_data_evaluated_by_3h.md" + } + content = """ Please act as an impartial judge and evaluate the quality of the responses provided by the AI assistant to the user question displayed below. Determine whether the response is harmless. @@ -55,6 +79,18 @@ class PromptTextHarmless(BasePrompt): @Model.prompt_register("QUALITY_HONEST", []) class PromptTextHonest(BasePrompt): + + # Metadata for documentation generation + _metric_info = { + "category": "SFT Data Assessment Metrics", + "metric_name": "Honesty", + "description": "Evaluates if responses provide accurate information without fabrication or deception", + "paper_title": "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback", + "paper_url": "https://arxiv.org/pdf/2204.05862", + "paper_authors": "Bai et al., 2022", + "evaluation_results": "docs/eval/prompt/qa_data_evaluated_by_3h.md" + } + content = """ Please act as an impartial judge and evaluate the quality of the responses provided by the AI assistant to the user question displayed below. Determine whether the response is honest. diff --git a/dingo/model/prompt/prompt_text_quality.py b/dingo/model/prompt/prompt_text_quality.py index 5421d296..31f24291 100644 --- a/dingo/model/prompt/prompt_text_quality.py +++ b/dingo/model/prompt/prompt_text_quality.py @@ -80,6 +80,17 @@ class PromptTextQualityV3(BasePrompt): @Model.prompt_register("TEXT_QUALITY_V4", []) class PromptTextQualityV4(BasePrompt): + + # Metadata for documentation generation + _metric_info = { + "category": "Text Quality Assessment Metrics", + "metric_name": "Pretrain Text Quality Assessment V4", + "description": "Enhanced text quality evaluation covering completeness (formulas, tables, code), effectiveness (garbled text, spacing), similarity (duplicates), and security (politics, prohibited content)", + "paper_title": "WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages", + "paper_url": "https://arxiv.org/abs/2501.14506", + "paper_authors": "Yu et al., 2025", + "evaluation_results": "docs/eval/prompt/redpajama_data_evaluated_by_prompt.md" + } content = """ # Role You are an expert in language model evaluation. diff --git a/dingo/model/prompt/prompt_text_quality_multilan.py b/dingo/model/prompt/prompt_text_quality_multilan.py index 43f20b2d..b1fbb96f 100644 --- a/dingo/model/prompt/prompt_text_quality_multilan.py +++ b/dingo/model/prompt/prompt_text_quality_multilan.py @@ -1,7 +1,6 @@ from dingo.model.model import Model from dingo.model.prompt.base import BasePrompt -from dingo.model.prompt.prompt_text_quality_v2 import \ - TEXT_QUALITY_WITHOUT_ROLE_V2 +from dingo.model.prompt.prompt_text_quality_v2 import TEXT_QUALITY_WITHOUT_ROLE_V2 AR_ROLE = """ ### Role diff --git a/dingo/model/rule/rule_common.py b/dingo/model/rule/rule_common.py index 4aa29711..75758bf6 100644 --- a/dingo/model/rule/rule_common.py +++ b/dingo/model/rule/rule_common.py @@ -13,6 +13,18 @@ class RuleAbnormalChar(BaseRule): # consist of [RuleSpecialCharacter, RuleInvisibleChar] + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "EFFECTIVENESS", + "metric_name": "Abnormal Character Detection", + "description": "Detects garbled text and anti-crawling characters by combining special character and invisible character detection", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + @classmethod def eval(cls, input_data: Data) -> ModelRes: res = ModelRes() @@ -30,6 +42,17 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleAbnormalHtml(BaseRule): # consist of [RuleHtmlEntity, RuleHtmlTag] + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "EFFECTIVENESS", + "metric_name": "Abnormal HTML Detection", + "description": "Detects abnormal HTML content by combining HTML entity and tag detection", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + @classmethod def eval(cls, input_data: Data) -> ModelRes: res = ModelRes() @@ -47,6 +70,17 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleAbnormalNumber(BaseRule): """check pdf content abnormal book page or index number.""" + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "FLUENCY", + "metric_name": "Abnormal Number Detection", + "description": "Checks PDF content for abnormal book page or index numbers that disrupt text flow", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig(pattern=r"\n{4}\d+\n{4}") @classmethod @@ -66,6 +100,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleAlphaWords(BaseRule): """check whether the ratio of words that contain at least one alphabetic character > 0.6""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "EFFECTIVENESS", + "metric_name": "Alphabetic Word Ratio", + "description": "Checks whether the ratio of words containing at least one alphabetic character is above threshold", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig(threshold=0.6) @classmethod @@ -98,6 +144,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleCapitalWords(BaseRule): """check whether capital words ratio > 0.2""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "UNDERSTANDABILITY", + "metric_name": "Capital Words Ratio", + "description": "Checks whether the ratio of capital words is above threshold, indicating poor readability", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig(threshold=0.2) @classmethod @@ -124,6 +182,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleCharNumber(BaseRule): """check whether the number of char > 100""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "EFFECTIVENESS", + "metric_name": "Character Count", + "description": "Checks whether the number of characters is above minimum threshold", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig(threshold=100) @classmethod @@ -147,6 +217,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleCharSplit(BaseRule): """check pdf content char split.""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "FLUENCY", + "metric_name": "Character Split Detection", + "description": "Checks PDF content for abnormal character splitting that disrupts readability", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig( pattern=r"(?:(?:[a-zA-Z]\s){5}[a-zA-Z])", threshold=3 ) @@ -172,6 +254,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleColonEnd(BaseRule): """check whether the last char is ':'""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "EFFECTIVENESS", + "metric_name": "Colon End Detection", + "description": "Checks if text abruptly ends with a colon, indicating incomplete content", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig() @classmethod @@ -212,6 +306,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleContentNull(BaseRule): """check whether content is null""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "EFFECTIVENESS", + "metric_name": "Content Null Detection", + "description": "Checks whether content is empty or null", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig() @classmethod @@ -230,6 +336,20 @@ def eval(cls, input_data: Data) -> ModelRes: "QUALITY_BAD_EFFECTIVENESS", ["text_base_all", "qa_standard_v1", "pdf"] ) class RuleContentShort(BaseRule): + """check whether content is too short""" + + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "EFFECTIVENESS", + "metric_name": "Content Short Detection", + "description": "Checks whether content is too short to be meaningful", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig(threshold=20) @classmethod @@ -260,6 +380,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleContentShortMultiLan(BaseRule): """check whether content is too short.""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "EFFECTIVENESS", + "metric_name": "Multi-Language Content Short Detection", + "description": "Checks whether multi-language content is too short to be meaningful", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig(threshold=20) @classmethod @@ -282,6 +414,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleCurlyBracket(BaseRule): """check whether the ratio of the number of {,} and the number of characters < 0.025""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "UNDERSTANDABILITY", + "metric_name": "Curly Bracket Ratio", + "description": "Checks whether the ratio of curly brackets to total characters is below threshold", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig(threshold=0.025) @classmethod @@ -326,12 +470,23 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleDocRepeat(BaseRule): """check whether content repeats""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "SIMILARITY", + "metric_name": "Document Repetition Detection", + "description": "Evaluates text for consecutive repeated content and multiple occurrences of special characters", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig(threshold=80) @classmethod def eval(cls, input_data: Data) -> ModelRes: - from dingo.model.rule.utils.util import \ - base_rps_frac_chars_in_dupe_ngrams + from dingo.model.rule.utils.util import base_rps_frac_chars_in_dupe_ngrams res = ModelRes() repeat_score = base_rps_frac_chars_in_dupe_ngrams(6, input_data.content) @@ -349,6 +504,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleEnterAndSpace(BaseRule): # consist of [RuleEnterMore, RuleEnterRatioMore, RuleSpaceMore] + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "EFFECTIVENESS", + "metric_name": "Enter and Space Detection", + "description": "Composite rule checking for excessive carriage returns and spaces", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + @classmethod def eval(cls, input_data: Data) -> ModelRes: res = ModelRes() @@ -381,6 +548,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleEnterMore(BaseRule): """check whether content has 8 consecutive carriage returns.""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "EFFECTIVENESS", + "metric_name": "Excessive Carriage Returns", + "description": "Checks whether content has 8 consecutive carriage returns", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig(key_list=[r"\n{8,}", r"\r\n{8,}"]) @classmethod @@ -418,6 +597,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleEnterRatioMore(BaseRule): """check whether the number of enter / the number of content > 25%""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "EFFECTIVENESS", + "metric_name": "Enter Ratio Detection", + "description": "Checks whether the ratio of enter characters to total content is above 25%", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig() @classmethod @@ -440,6 +631,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleHeadWordAr(BaseRule): """check whether ar content contains irrelevance tail source info.""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "RELEVANCE", + "metric_name": "Arabic Head Word Detection", + "description": "Checks whether Arabic content contains irrelevant tail source information", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig() @classmethod @@ -634,6 +837,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleHtmlEntity(BaseRule): """check whether content has html entity""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "EFFECTIVENESS", + "metric_name": "HTML Entity Detection", + "description": "Checks whether content contains HTML entities indicating web scraping artifacts", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig( key_list=[ "nbsp", @@ -707,6 +922,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleHtmlTag(BaseRule): """check whether content has image links or html tags.""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "EFFECTIVENESS", + "metric_name": "HTML Tag Detection", + "description": "Checks whether content contains HTML tags or image links indicating web scraping artifacts", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig( key_list=["", "

", ""] ) @@ -769,6 +996,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleInvisibleChar(BaseRule): """check whether content has invisible chars.""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "EFFECTIVENESS", + "metric_name": "Invisible Character Detection", + "description": "Checks whether content contains invisible characters that may cause display issues", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig( pattern=r"[\u2000-\u200F\u202F\u205F\u3000\uFEFF\u00A0\u2060-\u206F\uFEFF\xa0]" ) @@ -813,6 +1052,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleLineEndWithEllipsis(BaseRule): """check whether the ratio of line ends with ellipsis < 0.3""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "COMPLETENESS", + "metric_name": "Line End With Ellipsis", + "description": "Checks whether the ratio of lines ending with ellipsis is below threshold", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig(threshold=0.3, key_list=["...", "…"]) @classmethod @@ -847,6 +1098,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleLineEndWithTerminal(BaseRule): """check whether the ratio of line ends with terminal punctuation mark > 0.6""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "COMPLETENESS", + "metric_name": "Line End With Terminal", + "description": "Checks whether the ratio of lines ending with terminal punctuation is above threshold", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig( threshold=0.6, key_list=[".", "!", "?", '"', '"'] ) @@ -888,6 +1151,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleLineStartWithBulletpoint(BaseRule): """check whether the ratio of line starts with bullet points < 0.9""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "UNDERSTANDABILITY", + "metric_name": "Line Start With Bulletpoint", + "description": "Checks whether the ratio of lines starting with bullet points is below threshold", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig( threshold=0.9, key_list=[ @@ -936,12 +1211,23 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleLineJavascriptCount(BaseRule): """check whether line with the word Javascript.""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "EFFECTIVENESS", + "metric_name": "Javascript Count Detection", + "description": "Checks whether content contains excessive Javascript-related text", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig(threshold=3) @classmethod def eval(cls, input_data: Data) -> ModelRes: - from dingo.model.rule.utils.util import (TextSlice, normalize, - split_paragraphs) + from dingo.model.rule.utils.util import TextSlice, normalize, split_paragraphs res = ModelRes() raw_content = input_data.content @@ -968,6 +1254,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleLoremIpsum(BaseRule): """check whether the ratio of lorem ipsum < 3e-08""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "EFFECTIVENESS", + "metric_name": "Lorem Ipsum Detection", + "description": "Checks whether content contains lorem ipsum placeholder text", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig(threshold=3e-08) @classmethod @@ -995,6 +1293,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleMeanWordLength(BaseRule): """check whether the mean length of word in [3, 10]""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "EFFECTIVENESS", + "metric_name": "Mean Word Length", + "description": "Checks whether the mean length of words is within acceptable range", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig(key_list=["3", "10"]) @classmethod @@ -1045,6 +1355,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleNoPunc(BaseRule): """check whether paragraph has no punctuation.""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "FLUENCY", + "metric_name": "No Punctuation Detection", + "description": "Checks whether paragraphs lack punctuation marks, indicating poor text quality", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig(threshold=112) @classmethod @@ -1095,6 +1417,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleSentenceNumber(BaseRule): """check whether the number of sentence in [3, 7500]""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "COMPLETENESS", + "metric_name": "Sentence Number", + "description": "Checks whether the number of sentences is within acceptable range", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig(key_list=["3", "7500"]) @classmethod @@ -1133,6 +1467,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleSpaceMore(BaseRule): """check whether content has 500 spaces.""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "EFFECTIVENESS", + "metric_name": "Excessive Space Detection", + "description": "Checks whether content contains excessive consecutive spaces", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig(pattern=" {500,}") @classmethod @@ -1172,6 +1518,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleSpecialCharacter(BaseRule): """check whether content has special characters.""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "EFFECTIVENESS", + "metric_name": "Special Character Detection", + "description": "Checks if data is meaningful and properly formatted by detecting excessive special characters", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig( key_list=[ r"u200e", @@ -1208,13 +1566,26 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleStopWord(BaseRule): """check whether the ratio of stop word > 0.06""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "EFFECTIVENESS", + "metric_name": "Stop Word Ratio", + "description": "Checks whether the ratio of stop words is above threshold", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig(threshold=0.06) @classmethod def eval(cls, input_data: Data) -> ModelRes: - from dingo.model.rule.utils.util import get_stop_words from nltk.tokenize import WordPunctTokenizer + from dingo.model.rule.utils.util import get_stop_words + res = ModelRes() raw_content = input_data.content raw_words = list(WordPunctTokenizer().tokenize(raw_content)) @@ -1238,6 +1609,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleSymbolWordRatio(BaseRule): """check whether the ratio of symbol and word is > 0.4""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "UNDERSTANDABILITY", + "metric_name": "Symbol Word Ratio", + "description": "Checks whether the ratio of symbols to words is above threshold", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig(threshold=0.4, key_list=["#", "...", "…"]) @classmethod @@ -1269,6 +1652,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleUniqueWords(BaseRule): """check whether the ratio of unique words > 0.1""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "UNDERSTANDABILITY", + "metric_name": "Unique Words Ratio", + "description": "Checks whether the ratio of unique words is above threshold", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig(threshold=0.1) @classmethod @@ -1304,6 +1699,7 @@ class RuleUnsafeWords(BaseRule): @classmethod def eval(cls, input_data: Data) -> ModelRes: import ahocorasick + from dingo.model.rule.utils.util import get_unsafe_words res = ModelRes() @@ -1348,6 +1744,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleOnlyUrl(BaseRule): """check whether content is only an url link.""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "EFFECTIVENESS", + "metric_name": "URL-Only Content Detection", + "description": "Checks whether content consists only of URLs without meaningful text", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig( pattern=r"http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+" ) @@ -1390,6 +1798,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleWordNumber(BaseRule): """check whether the number of word in [20, 100000]""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "COMPLETENESS", + "metric_name": "Word Number", + "description": "Checks whether the number of words is within acceptable range", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig(key_list=["20", "100000"]) @classmethod @@ -1416,6 +1836,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleWordSplit(BaseRule): """check pdf word abnormal split such as "ca- se".""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "FLUENCY", + "metric_name": "Word Split Detection", + "description": "Checks for abnormal word splits in PDF content that disrupt readability", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig(pattern=r"[A-Za-z]+-\s*$") @classmethod @@ -1449,6 +1881,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleWordStuck(BaseRule): """check whether words are stuck.""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "FLUENCY", + "metric_name": "Word Stuck Detection", + "description": "Checks whether words are stuck together without proper spacing", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig( key_list=[ r"https?://[^\s]+|www.(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+", @@ -1462,6 +1906,7 @@ class RuleWordStuck(BaseRule): @classmethod def eval(cls, input_data: Data) -> ModelRes: import wordninja + from dingo.model.rule.utils.detect_lang import decide_language_by_str from dingo.model.rule.utils.util import is_sha256 diff --git a/dingo/model/rule/rule_image.py b/dingo/model/rule/rule_image.py index 21c36414..6f541cc6 100644 --- a/dingo/model/rule/rule_image.py +++ b/dingo/model/rule/rule_image.py @@ -1,18 +1,31 @@ import os import numpy as np +from PIL import Image + from dingo.config.config import DynamicRuleConfig from dingo.io import Data from dingo.model.model import Model from dingo.model.modelres import ModelRes from dingo.model.rule.base import BaseRule -from PIL import Image -@Model.rule_register("QUALITY_BAD_EFFECTIVENESS", ["img"]) +@Model.rule_register("QUALITY_BAD_IMG_EFFECTIVENESS", ["img"]) class RuleImageValid(BaseRule): """check whether image is not all white or black""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based IMG Quality Metrics", + "quality_dimension": "IMG_EFFECTIVENESS", + "metric_name": "Image Valid Detection", + "description": "Checks whether image is not all white or black, ensuring visual content validity", + "paper_title": "", + "paper_url": "", + "paper_authors": "", + "evaluation_results": "" + } + dynamic_config = DynamicRuleConfig() @classmethod @@ -32,10 +45,22 @@ def eval(cls, input_data: Data) -> ModelRes: return res -@Model.rule_register("QUALITY_BAD_EFFECTIVENESS", ["img"]) +@Model.rule_register("QUALITY_BAD_IMG_EFFECTIVENESS", ["img"]) class RuleImageSizeValid(BaseRule): """check whether image ratio of width to height is valid""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based IMG Quality Metrics", + "quality_dimension": "IMG_EFFECTIVENESS", + "metric_name": "Image Size Valid Detection", + "description": "Checks whether image ratio of width to height is within valid range", + "paper_title": "", + "paper_url": "", + "paper_authors": "", + "evaluation_results": "" + } + dynamic_config = DynamicRuleConfig() @classmethod @@ -58,10 +83,22 @@ def eval(cls, input_data: Data) -> ModelRes: return res -@Model.rule_register("QUALITY_BAD_EFFECTIVENESS", ["img"]) +@Model.rule_register("QUALITY_BAD_IMG_EFFECTIVENESS", ["img"]) class RuleImageQuality(BaseRule): """check whether image quality is good.""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based IMG Quality Metrics", + "quality_dimension": "IMG_EFFECTIVENESS", + "metric_name": "Image Quality Assessment", + "description": "Evaluates image quality using NIMA (Neural Image Assessment) metrics", + "paper_title": "NIMA: Neural Image Assessment", + "paper_url": "https://arxiv.org/abs/1709.05424", + "paper_authors": "Talebi & Milanfar, 2018", + "evaluation_results": "" + } + dynamic_config = DynamicRuleConfig(threshold=5.5) @classmethod @@ -88,10 +125,22 @@ def eval(cls, input_data: Data) -> ModelRes: return res -@Model.rule_register("QUALITY_BAD_EFFECTIVENESS", []) +@Model.rule_register("QUALITY_BAD_IMG_SIMILARITY", []) class RuleImageRepeat(BaseRule): """Check for duplicate images using PHash and CNN methods.""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based IMG Quality Metrics", + "quality_dimension": "IMG_SIMILARITY", + "metric_name": "Image Duplicate Detection", + "description": "Detects duplicate images using PHash and CNN methods to ensure data diversity", + "paper_title": "ImageNet Classification with Deep Convolutional Neural Networks", + "paper_url": "https://papers.nips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks.pdf", + "paper_authors": "Krizhevsky et al., 2012", + "evaluation_results": "" + } + dynamic_config = DynamicRuleConfig() @classmethod @@ -132,8 +181,22 @@ def eval(cls, input_data: Data) -> ModelRes: return res -@Model.rule_register("QUALITY_BAD_EFFECTIVENESS", []) +@Model.rule_register("QUALITY_BAD_IMG_RELEVANCE", []) class RuleImageTextSimilarity(BaseRule): + """Check similarity between image and text content""" + + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based IMG Quality Metrics", + "quality_dimension": "IMG_RELEVANCE", + "metric_name": "Image-Text Similarity", + "description": "Evaluates semantic similarity between image and text content using CLIP model", + "paper_title": "Learning Transferable Visual Representations with Natural Language Supervision", + "paper_url": "https://arxiv.org/abs/2103.00020", + "paper_authors": "Radford et al., 2021", + "evaluation_results": "" + } + dynamic_config = DynamicRuleConfig(threshold=0.17) @classmethod @@ -141,10 +204,11 @@ def eval(cls, input_data: Data) -> ModelRes: import nltk nltk.download("punkt_tab") - from dingo.model.rule.utils.image_util import download_similar_tool from nltk.tokenize import word_tokenize from similarities import ClipSimilarity + from dingo.model.rule.utils.image_util import download_similar_tool + res = ModelRes() if not input_data.image or not input_data.content: return res diff --git a/dingo/model/rule/utils/detect_lang.py b/dingo/model/rule/utils/detect_lang.py index 70baee36..2c1dc066 100644 --- a/dingo/model/rule/utils/detect_lang.py +++ b/dingo/model/rule/utils/detect_lang.py @@ -4,9 +4,10 @@ import fasttext import requests -from dingo.utils import log from tqdm import tqdm +from dingo.utils import log + _global_lang_detect = [] _fasttext_path = "" diff --git a/dingo/run/cli.py b/dingo/run/cli.py index ec40ae9a..b5996dc6 100644 --- a/dingo/run/cli.py +++ b/dingo/run/cli.py @@ -2,6 +2,7 @@ import os import prettytable as pt + from dingo.exec import Executor from dingo.io import InputArgs from dingo.model import Model diff --git a/dingo/run/web.py b/dingo/run/web.py index 2f19cbf1..39fd0af1 100644 --- a/dingo/run/web.py +++ b/dingo/run/web.py @@ -3,11 +3,12 @@ from zipfile import ZIP_DEFLATED, ZipFile import uvicorn +from fastapi import FastAPI, HTTPException, status +from fastapi.responses import StreamingResponse + from dingo.exec import ExecProto, Executor from dingo.io import InputArgs from dingo.model import Model -from fastapi import FastAPI, HTTPException, status -from fastapi.responses import StreamingResponse app = FastAPI(title="dingo: Tool for detect language quality") diff --git a/dingo/utils/log_util/__init__.py b/dingo/utils/log_util/__init__.py index 4309f5d5..fa3bc9c9 100644 --- a/dingo/utils/log_util/__init__.py +++ b/dingo/utils/log_util/__init__.py @@ -1,8 +1,9 @@ from typing import Optional -from dingo.utils.log_util.logger import Logger from pydantic import BaseModel +from dingo.utils.log_util.logger import Logger + class LogConfig(BaseModel): """ diff --git a/docs/metrics.md b/docs/metrics.md index f3255dbf..7c03198f 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -1,11 +1,54 @@ -We classify data quality issues into 7 Quality Metrics, with the following definitions: - -| Quality Metric | Description | -|-------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| COMPLETENESS | Refers to data that is incomplete or completely missing. For example, whether some text data is truncated or the content is empty. | -| EFFECTIVENESS | Refers to whether the data is meaningful, suitable for a specific task, and conforms to the expected format or standard. For example, whether the text content contains garbled characters. | -| FLUENCY | Refers to whether the data is fluent, grammatically correct, and can be read naturally. For example, whether sentences conform to the grammatical rules. | -| RELEVANCE | Refers to data that contains data that is irrelevant to the task. For example, some texts describe medical knowledge, but insert irrelevant advertising content. | -| SECURITY | Refers to whether the data contains sensitive or private information and whether it conforms to the culture and values of various countries (the other party's values & our values). | -| SIMILARITY | Refers to whether the data content is repeated or there is very similar content. | -| UNDERSTANDABILITY | Refers to whether the data is easy to understand and interpret. For example, whether the data is clear, unambiguous, and meaningful in context. | +# Data Quality Metrics + +This document provides comprehensive information about all quality metrics used in Dingo. + +**Note**: All metrics are backed by academic sources to ensure objectivity and scientific rigor. + +### Text Quality Assessment Metrics + +| Type | Metric | Description | Paper Source | Evaluation Results | +|------|--------|-------------|--------------|-------------------| +| `DATAMAN_ASSESSMENT` | DATAMAN | Evaluates pre-training data quality using the DataMan methodology (14 standar... | [DataMan: Data Manager for Pre-training Large Language Models](https://arxiv.org/abs/2502.19363) (Peng et al., 2025) | N/A | +| `QUALITY_BAD_SECURITY` | Politics | Evaluates whether the text contains politics-related content | Internal Implementation | N/A | +| `TEXT_QUALITY_V4` | Pretrain Text Quality Assessment V4 | Enhanced text quality evaluation covering completeness (formulas, tables, cod... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [📊 See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | + +### SFT Data Assessment Metrics + +| Type | Metric | Description | Paper Source | Evaluation Results | +|------|--------|-------------|--------------|-------------------| +| `QUALITY_HARMLESS` | Harmlessness | Checks if responses avoid harmful content, discriminatory language, and dange... | [Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback](https://arxiv.org/pdf/2204.05862) (Bai et al., 2022) | [📊 See Results](eval/prompt/qa_data_evaluated_by_3h.md) | +| `QUALITY_HELPFUL` | Helpfulness | Assesses if responses address questions directly and follow instructions appr... | [Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback](https://arxiv.org/pdf/2204.05862) (Bai et al., 2022) | [📊 See Results](eval/prompt/qa_data_evaluated_by_3h.md) | +| `QUALITY_HONEST` | Honesty | Evaluates if responses provide accurate information without fabrication or de... | [Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback](https://arxiv.org/pdf/2204.05862) (Bai et al., 2022) | [📊 See Results](eval/prompt/qa_data_evaluated_by_3h.md) | + +### Classification Metrics + +| Type | Metric | Description | Paper Source | Evaluation Results | +|------|--------|-------------|--------------|-------------------| +| `CLASSIFY_TOPIC` | Topic Categorization | Classifies text into categories like language processing, writing, code, math... | [BERTopic](https://maartengr.github.io/BERTopic/index.html#quick-start) & [INSTAG](https://arxiv.org/pdf/2308.07074) (Grootendorst, 2022; Wei et al., 2023) | [📊 See Results](eval/prompt/text_data_classified_by_topic.md) | + +### Multimodality Assessment Metrics + +| Type | Metric | Description | Paper Source | Evaluation Results | +|------|--------|-------------|--------------|-------------------| +| `CLASSIFY_QR` | Image Classification | Identifies images as CAPTCHA, QR code, or normal images | Internal Implementation | N/A | +| `IMAGE_RELEVANT` | Image Relevance | Evaluates if an image matches reference image in terms of face count, feature... | Internal Implementation | N/A | + +### Rule-Based TEXT Quality Metrics + +| Type | Metric | Description | Paper Source | Evaluation Results | +|------|--------|-------------|--------------|-------------------| +| `QUALITY_BAD_COMPLETENESS` | RuleLineEndWithEllipsis, RuleLineEndWithTerminal, RuleSentenceNumber, RuleWordNumber | Checks whether the ratio of lines ending with ellipsis is below threshold; Ch... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [📊 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | +| `QUALITY_BAD_EFFECTIVENESS` | RuleAbnormalChar, RuleAbnormalHtml, RuleAlphaWords, RuleCharNumber, RuleColonEnd, RuleContentNull, RuleContentShort, RuleContentShortMultiLan, RuleEnterAndSpace, RuleEnterMore, RuleEnterRatioMore, RuleHtmlEntity, RuleHtmlTag, RuleInvisibleChar, RuleLineJavascriptCount, RuleLoremIpsum, RuleMeanWordLength, RuleSpaceMore, RuleSpecialCharacter, RuleStopWord, RuleSymbolWordRatio, RuleOnlyUrl | Detects garbled text and anti-crawling characters by combining special charac... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [📊 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | +| `QUALITY_BAD_FLUENCY` | RuleAbnormalNumber, RuleCharSplit, RuleNoPunc, RuleWordSplit, RuleWordStuck | Checks PDF content for abnormal book page or index numbers that disrupt text ... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [📊 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | +| `QUALITY_BAD_RELEVANCE` | RuleHeadWordAr | Checks whether Arabic content contains irrelevant tail source information | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [📊 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | +| `QUALITY_BAD_SIMILARITY` | RuleDocRepeat | Evaluates text for consecutive repeated content and multiple occurrences of s... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [📊 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | +| `QUALITY_BAD_UNDERSTANDABILITY` | RuleCapitalWords, RuleCurlyBracket, RuleLineStartWithBulletpoint, RuleUniqueWords | Checks whether the ratio of capital words is above threshold, indicating poor... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [📊 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | + +### Rule-Based IMG Quality Metrics + +| Type | Metric | Description | Paper Source | Evaluation Results | +|------|--------|-------------|--------------|-------------------| +| `QUALITY_BAD_IMG_EFFECTIVENESS` | RuleImageValid, RuleImageSizeValid, RuleImageQuality | Checks whether image is not all white or black, ensuring visual content valid... | Internal Implementation | N/A | +| `QUALITY_BAD_IMG_RELEVANCE` | RuleImageTextSimilarity | Evaluates semantic similarity between image and text content using CLIP model | [Learning Transferable Visual Representations with Natural Language Supervision](https://arxiv.org/abs/2103.00020) (Radford et al., 2021) | N/A | +| `QUALITY_BAD_IMG_SIMILARITY` | RuleImageRepeat | Detects duplicate images using PHash and CNN methods to ensure data diversity | [ImageNet Classification with Deep Convolutional Neural Networks](https://papers.nips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks.pdf) (Krizhevsky et al., 2012) | N/A | + diff --git a/examples/spark/sdk_spark.py b/examples/spark/sdk_spark.py index 6e9210b4..11924a84 100644 --- a/examples/spark/sdk_spark.py +++ b/examples/spark/sdk_spark.py @@ -1,8 +1,9 @@ import json +from pyspark.sql import DataFrame, SparkSession + from dingo.exec import Executor from dingo.io import Data, InputArgs -from pyspark.sql import DataFrame, SparkSession ################## # please prepare # diff --git a/mcp_server.py b/mcp_server.py index 8e10460b..cceb6224 100644 --- a/mcp_server.py +++ b/mcp_server.py @@ -3,11 +3,12 @@ import uuid from typing import Any, Dict, List, Literal, Optional, Tuple +from fastmcp import FastMCP + from dingo.exec import Executor from dingo.io import InputArgs from dingo.model import Model from dingo.utils import log -from fastmcp import FastMCP # Configure logging based on environment variable log_level = os.environ.get("LOG_LEVEL", "info").upper() diff --git a/scripts/generate_metrics.py b/scripts/generate_metrics.py new file mode 100644 index 00000000..d06a1ed0 --- /dev/null +++ b/scripts/generate_metrics.py @@ -0,0 +1,270 @@ +#!/usr/bin/env python3 +""" +自动生成 Metrics 文档的脚本 +按照 3H Assessment Prompts 表格的格式生成文档 +""" + +import sys +from pathlib import Path +from typing import Any, Dict, List + +# Add project root to path +project_root = Path(__file__).parent.parent +sys.path.insert(0, str(project_root)) + +from dingo.model.model import Model # noqa: E402 + + +def scan_prompt_classes() -> List[Dict[str, Any]]: + """扫描所有 prompt 类，提取 _metric_info 信息""" + # 先加载模型 + Model.load_model() + + metrics_info = [] + + # 直接从 prompt_metric_type_map 中获取信息 + for metric_type, prompt_classes in Model.prompt_metric_type_map.items(): + for prompt_class in prompt_classes: + if hasattr(prompt_class, '_metric_info'): + info = prompt_class._metric_info.copy() + info['prompt_type'] = metric_type + info['class_name'] = prompt_class.__name__ + info['type'] = 'prompt' + metrics_info.append(info) + + return metrics_info + + +def scan_rule_classes() -> List[Dict[str, Any]]: + """扫描所有 rule 类，提取 _metric_info 信息""" + # 先加载模型 + Model.load_model() + + metrics_info = [] + + # 直接从 rule_metric_type_map 中获取信息 + for metric_type, rule_classes in Model.rule_metric_type_map.items(): + for rule_class in rule_classes: + if hasattr(rule_class, '_metric_info'): + info = rule_class._metric_info.copy() + info['rule_type'] = metric_type + info['class_name'] = rule_class.__name__ + info['type'] = 'rule' + + # 如果 _metric_info 中没有设置 category，则根据类型设置默认值 + if 'category' not in info or not info['category']: + info['category'] = 'Rule-Based Quality Metrics' + + metrics_info.append(info) + + return metrics_info + + +def truncate_description(description: str, max_length: int = 80) -> str: + """截断description到指定长度""" + if len(description) <= max_length: + return description + return description[:max_length - 3] + "..." + + +def generate_table_section(title: str, metrics: List[Dict[str, Any]]) -> str: + """生成表格部分""" + if not metrics: + return "" + + # 表格头部 + table = f"### {title}\n\n" + table += "| Type | Metric | Description | Paper Source | Evaluation Results |\n" + table += "|------|--------|-------------|--------------|-------------------|\n" + + # 对于rule类，按type分组合并；对于prompt类，保持原有逻辑 + if title.startswith("Rule-Based") and "Quality Metrics" in title: + # 按type分组 + type_groups = {} + for metric in metrics: + if metric.get('type') == 'rule': + rule_type = metric.get('rule_type', '') + if rule_type not in type_groups: + type_groups[rule_type] = [] + type_groups[rule_type].append(metric) + + # 为每个type生成一行 + for rule_type in sorted(type_groups.keys()): + group_metrics = type_groups[rule_type] + type_name = f"`{rule_type}`" + + # 合并同一type的metric名称 + metric_names = [m['class_name'] for m in group_metrics] + combined_metrics = ", ".join(metric_names) + + # 合并描述（取第一个作为代表，或者合并所有描述） + descriptions = [m['description'] for m in group_metrics] + combined_description = "; ".join(descriptions) + combined_description = truncate_description(combined_description) + + # 取第一个metric的论文信息（因为都是相同的） + first_metric = group_metrics[0] + + # 处理论文来源 + if first_metric.get('paper_url') and first_metric.get('paper_title'): + paper_urls = [url.strip() for url in first_metric['paper_url'].split(',')] + paper_titles = [title.strip() for title in first_metric['paper_title'].split('&')] + + # 如果有多个URL和标题，为每个创建单独的链接 + if len(paper_urls) > 1 and len(paper_titles) > 1: + links = [] + for i, (title, url) in enumerate(zip(paper_titles, paper_urls)): + links.append(f"[{title}]({url})") + paper_source = " & ".join(links) + else: + paper_source = f"[{first_metric['paper_title']}](" \ + f"{first_metric['paper_url']})" + + if first_metric.get('paper_authors'): + paper_source += f" ({first_metric['paper_authors']})" + else: + paper_source = "Internal Implementation" + + # 处理评测结果 + if first_metric.get('evaluation_results'): + # 修正相对路径：从 docs/metrics.md 到 docs/eval/prompt/xxx.md + eval_path = first_metric['evaluation_results'] + if eval_path.startswith('docs/'): + eval_path = eval_path[5:] # 去掉 'docs/' 前缀 + eval_results = f"[📊 See Results]({eval_path})" + else: + eval_results = "N/A" + + table += f"| {type_name} | {combined_metrics} | " \ + f"{combined_description} | {paper_source} | {eval_results} |\n" + else: + # 对于prompt类，保持原有逻辑 + sort_key = lambda x: x.get('prompt_type', x.get('rule_type', '')) # noqa: E731 + for metric in sorted(metrics, key=sort_key): + # 处理type列 + if metric.get('type') == 'prompt': + type_name = f"`{metric['prompt_type']}`" + elif metric.get('type') == 'rule': + type_name = f"`{metric['rule_type']}`" + else: + type_name = "N/A" + + # 对于rule类，使用类名作为metric名称；对于prompt类，使用描述名称 + if metric.get('type') == 'rule': + metric_name = metric['class_name'] + else: + metric_name = metric['metric_name'] + description = truncate_description(metric['description']) + + # 处理论文来源 + if metric.get('paper_url') and metric.get('paper_title'): + paper_urls = [url.strip() for url in metric['paper_url'].split(',')] + paper_titles = [title.strip() for title in metric['paper_title'].split('&')] + + # 如果有多个URL和标题，为每个创建单独的链接 + if len(paper_urls) > 1 and len(paper_titles) > 1: + links = [] + for i, (title, url) in enumerate(zip(paper_titles, paper_urls)): + links.append(f"[{title}]({url})") + paper_source = " & ".join(links) + else: + paper_source = f"[{metric['paper_title']}](" \ + f"{metric['paper_url']})" + + if metric.get('paper_authors'): + paper_source += f" ({metric['paper_authors']})" + else: + paper_source = "Internal Implementation" + + # 处理评测结果 + if metric.get('evaluation_results'): + # 修正相对路径：从 docs/metrics.md 到 docs/eval/prompt/xxx.md + eval_path = metric['evaluation_results'] + if eval_path.startswith('docs/'): + eval_path = eval_path[5:] # 去掉 'docs/' 前缀 + eval_results = f"[📊 See Results]({eval_path})" + else: + eval_results = "N/A" + + table += f"| {type_name} | {metric_name} | {description} | " \ + f"{paper_source} | {eval_results} |\n" + + table += "\n" + return table + + +def generate_metrics_documentation() -> str: + """生成完整的 metrics 文档""" + # 扫描所有类 + prompt_metrics = scan_prompt_classes() + rule_metrics = scan_rule_classes() + + # 合并所有metrics + all_metrics = prompt_metrics + rule_metrics + + # 按类别分组 + categories = {} + for metric in all_metrics: + category = metric.get('category', 'other') + if category not in categories: + categories[category] = [] + categories[category].append(metric) + + # 生成文档 + doc = "# Data Quality Metrics\n\n" + doc += "This document provides comprehensive information about " \ + "all quality metrics used in Dingo.\n\n" + doc += "**Note**: All metrics are backed by academic sources to " \ + "ensure objectivity and scientific rigor.\n\n" + + # 按预定义顺序生成各个类别 + category_order = ["Text Quality Assessment Metrics", "SFT Data Assessment Metrics", + "Classification Metrics", "Multimodality Assessment Metrics", + "Rule-Based TEXT Quality Metrics", "Rule-Based IMG Quality Metrics", + "other"] + for category in category_order: + if category in categories: + doc += generate_table_section(category, categories[category]) + + return doc + + +def main(): + """主函数""" + try: + documentation = generate_metrics_documentation() + + # 写入文档文件 + output_file = project_root / "docs" / "metrics.md" + output_file.parent.mkdir(exist_ok=True) + + with open(output_file, 'w', encoding='utf-8') as f: + f.write(documentation) + + print(f"✅ Metrics documentation generated successfully: {output_file}") + + # 打印统计信息 + prompt_metrics = scan_prompt_classes() + rule_metrics = scan_rule_classes() + all_metrics = prompt_metrics + rule_metrics + + print(f"📊 Total metrics found: {len(all_metrics)}") + print(f" - Prompt-based: {len(prompt_metrics)}") + print(f" - Rule-based: {len(rule_metrics)}") + + categories = {} + for metric in all_metrics: + category = metric.get('category', 'other') + categories[category] = categories.get(category, 0) + 1 + + print("📈 Metrics by category:") + for category, count in sorted(categories.items()): + print(f" - {category}: {count}") + + except Exception as e: + print(f"❌ Error generating documentation: {e}") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/setup.cfg b/setup.cfg index 8ebd72cb..2e96d8fc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,10 +4,10 @@ blank_line_before_nested_class_or_def = true split_before_expression_after_opening_paren = true [isort] -line_length = 79 +line_length = 200 multi_line_output = 0 extra_standard_library = pkg_resources,setuptools -known_first_party = omniplat +known_first_party = dingo no_lines_before = STDLIB,LOCALFOLDER default_section = THIRDPARTY diff --git a/test/scripts/data/dataset/test_hf_dataset.py b/test/scripts/data/dataset/test_hf_dataset.py index 0446ab10..f2de3943 100644 --- a/test/scripts/data/dataset/test_hf_dataset.py +++ b/test/scripts/data/dataset/test_hf_dataset.py @@ -1,4 +1,5 @@ import pytest + from dingo.data.dataset.huggingface import HuggingFaceDataset from dingo.data.datasource.huggingface import HuggingFaceSource from dingo.io import InputArgs diff --git a/test/scripts/data/datasource/test_hf_datasource.py b/test/scripts/data/datasource/test_hf_datasource.py index 525658af..23be277c 100644 --- a/test/scripts/data/datasource/test_hf_datasource.py +++ b/test/scripts/data/datasource/test_hf_datasource.py @@ -1,4 +1,5 @@ import pytest + from dingo.data.datasource.huggingface import HuggingFaceSource from dingo.io import InputArgs diff --git a/test/scripts/exec/test_local.py b/test/scripts/exec/test_local.py index fcd31710..234063ce 100644 --- a/test/scripts/exec/test_local.py +++ b/test/scripts/exec/test_local.py @@ -1,4 +1,5 @@ import pytest + from dingo.exec import LocalExecutor from dingo.io import ResultInfo diff --git a/test/scripts/io/input/test_continue.py b/test/scripts/io/input/test_continue.py index 7b3e3e0a..da0f7683 100644 --- a/test/scripts/io/input/test_continue.py +++ b/test/scripts/io/input/test_continue.py @@ -2,6 +2,7 @@ import os.path import pytest + from dingo.exec import Executor from dingo.io import InputArgs diff --git a/test/scripts/io/input/test_write.py b/test/scripts/io/input/test_write.py index fe336301..c3eeeda9 100644 --- a/test/scripts/io/input/test_write.py +++ b/test/scripts/io/input/test_write.py @@ -2,6 +2,7 @@ import shutil import pytest + from dingo.exec import Executor from dingo.io import InputArgs diff --git a/test/scripts/model/rule/utils/test_rule_utils.py b/test/scripts/model/rule/utils/test_rule_utils.py index eec904df..039f0508 100644 --- a/test/scripts/model/rule/utils/test_rule_utils.py +++ b/test/scripts/model/rule/utils/test_rule_utils.py @@ -1,6 +1,7 @@ import time import pytest + from dingo.model.rule.utils.detect_lang import calculate_md5, download_fasttext From 12a4ed3fa7673ed1a22a9d30b588db33ee0e3f6d Mon Sep 17 00:00:00 2001 From: shijinpjlab Date: Fri, 11 Jul 2025 11:48:12 +0800 Subject: [PATCH 18/24] update technical doc and extend terminal args compatibility (#117) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: 修改标题等级 * feat: 修改标题 * feat: model介绍细化 * feat: 命令行的bool型参数兼容性支持 * feat: fix lint --- dingo/run/cli.py | 40 ++- docs/technical/technical_all.md | 40 +-- docs/technical/technical_local.md | 2 +- docs/technical/technical_model.md | 460 +++++++++++++++++++++++++----- 4 files changed, 448 insertions(+), 94 deletions(-) diff --git a/dingo/run/cli.py b/dingo/run/cli.py index b5996dc6..4c3abed7 100644 --- a/dingo/run/cli.py +++ b/dingo/run/cli.py @@ -33,21 +33,39 @@ def parse_args(): ) parser.add_argument( "--save_data", - action="store_true", + nargs='?', + const=True, default=False, - help="Save data in output path", + type=lambda x: ( + True if x.lower() in ('true', '1') else + False if x.lower() in ('false', '0') else + argparse.ArgumentTypeError(f"Invalid boolean value: {x}") + ), + help="Save data in output path (default: False, use --save_data or --save_data True to enable)" ) parser.add_argument( "--save_correct", - action="store_true", + nargs='?', + const=True, default=False, - help="Save correct data in output path", + type=lambda x: ( + True if x.lower() in ('true', '1') else + False if x.lower() in ('false', '0') else + argparse.ArgumentTypeError(f"Invalid boolean value: {x}") + ), + help="Save correct data in output path (default: False, use --save_correct or --save_correct True to enable)", ) parser.add_argument( "--save_raw", - action="store_true", + nargs='?', + const=True, default=False, - help="Save raw data in output path", + type=lambda x: ( + True if x.lower() in ('true', '1') else + False if x.lower() in ('false', '0') else + argparse.ArgumentTypeError(f"Invalid boolean value: {x}") + ), + help="Save raw data in output path (default: False, use --save_raw or --save_raw True to enable)", ) parser.add_argument( "--start_index", @@ -136,9 +154,15 @@ def parse_args(): ) parser.add_argument( "--use_browser", - action="store_true", + nargs='?', + const=True, default=False, - help="Open browser to display result after evaluation.", + type=lambda x: ( + True if x.lower() in ('true', '1') else + False if x.lower() in ('false', '0') else + argparse.ArgumentTypeError(f"Invalid boolean value: {x}") + ), + help="Open browser to display result (default: False, use --use_browser or --use_browser True to enable)", ) return parser.parse_args() diff --git a/docs/technical/technical_all.md b/docs/technical/technical_all.md index 9da6823c..0a2dd94c 100644 --- a/docs/technical/technical_all.md +++ b/docs/technical/technical_all.md @@ -1,6 +1,6 @@ -# 开始你的第一步 +# Dingo: 小白也能理解的技术文档 -## 安装 +## 一、安装 ### 基础安装 1. 使用conda准备 dingo 运行环境: @@ -32,7 +32,7 @@ pip install -r requirements/optional.txt pip install -r requirements/web.txt ``` -## 快速开始 +## 二、快速开始 ### 概览 在 dingo 中启动一个评估任务可以通过以下2种方式：命令行、代码。 @@ -94,9 +94,9 @@ outputs/ ├── ... ``` -# 教程 +## 三、教程 -## 整体概括 +### 整体概括 本项目的架构可以分为以下3个模块：Data、Evaluator、Executor + Data: 负责数据的加载与格式转化 + Evaluator: 负责评估的执行 @@ -104,7 +104,7 @@ outputs/ ![Architecture of dingo](assets/architeture.png) -## 基础配置 +### 基础配置 dingo 启动方式具有2种，因此配置方式也分为以下2种情况: 1. [命令行配置列表](config.md#cli-config) 2. [代码配置列表](config.md#sdk-config) @@ -137,7 +137,7 @@ input_data = { input_args = InputArgs(**input_data) ``` -## 加载数据 +### 加载数据 如果想要 dingo 顺利读入数据，那么需要在配置时设置以下参数: - input_path - dataset @@ -151,7 +151,7 @@ input_args = InputArgs(**input_data) 最终数据以 [Data](../dingo/io/input/Data.py) 类对象的形式在项目中流转。 如果用户在配置时将参数 save_raw 设置为True，那么 Data 类对象的 raw_data 有值否则为空字典。 -## 设置并发 +### 设置并发 dingo 默认状态下没有开启并发，如果有大规模评估任务需要开启并发，那么应该在配置时设置以下参数: + max_workers + batch_size @@ -160,7 +160,7 @@ dingo 默认状态下没有开启并发，如果有大规模评估任务需要 建议batch_size大于等于max_workers。 -## 结果保存 +### 结果保存 评估任务完成后会在当前目录下创建 outputs 文件夹并且不保存原始数据格式，除非用户在配置时设置了以下参数: + save_data + save_raw @@ -171,13 +171,13 @@ dingo 默认状态下没有开启并发，如果有大规模评估任务需要 如果 save_raw 设置为True，那么将执行 [ResultInfo](../dingo/io/output/ResultInfo.py) 类的 to_dict_raw 函数，否则将执行 to_dict 函数。 -## 启动前端页面 +### 启动前端页面 dingo 评估任务结束后，如果保存了评估结果，那么就可以通过一下方式启动前端页面展示结果: ```shell python -m dingo.run.vsl --input outputs/20250609_101837_50b5c0be ``` -# 规则 +## 四、规则 dingo 内置了不同类型的评估规则，详情见: [规则列表](rules.md)。 每条评估规则都有自己的 metric_type 和所属的 group。 @@ -185,22 +185,22 @@ dingo 内置了不同类型的评估规则，详情见: [规则列表](rules.md) 用户可以通过配置 eval_group 参数来调用该 group 内的所有规则执行评估任务。 如果用户需要组合一批评估规则用来评估，那么请参考下文的 **自定义配置** 。 -# 提示词 +## 五、提示词 dingo 提示词与规则类似，都有 metric_type 和 group ，并且他们的作用也相同。 但是提示词需要与场景配合才能执行评估任务，详情见: - [提示词列表](../dingo/model/prompt) -# 场景 +## 六、场景 dingo 的场景负责将数据打包发送给模型，并接收模型返回的结果，然后进行解析，处理成统一的 ModelRes 类对象。 - [场景列表](../dingo/model/llm) 请注意，不同场景对于评估结果 ModelRes 类对象的构建思路也不同，其 type 和 name 的意义也因此不同。 -# 进阶教程 +## 七、进阶教程 -## 自定义配置 +### 自定义配置 上文的 **教程-基础配置** 篇章中介绍了项目配置的方式与参数列表，但是并没有涉及到自定义，现在让我们来详细了解 **自定义配置** 。 自定义配置离不开参数 [custom_config](config.md#custom-config) , 这个参数包括能够自定义的所有内容，如下所示： @@ -210,7 +210,7 @@ dingo 的场景负责将数据打包发送给模型，并接收模型返回的 - llm_config - multi_turn_mode -## 自定义规则 +### 自定义规则 dingo 内置的规则向用户开放了接口，允许用户根据不同的评估任务进行动态配置。 规则的自定义通过上文 custom_config 参数中的 [rule_config](config.md#rule_config) 实现，可以设置的值包括: @@ -219,7 +219,7 @@ dingo 内置的规则向用户开放了接口，允许用户根据不同的评 + key_list + refer_path -## 自定义场景 +### 自定义场景 dingo 在使用提示词进行评估任务的时候，必须同时使用场景，执行数据的打包发送与接收处理。 场景的自定义同样是通过上文 custom_config 参数实现，不同的是需要参数 [llm_config](config.md#llm_config) ，可以设置的值包括: @@ -237,7 +237,7 @@ dingo 在使用提示词进行评估任务的时候，必须同时使用场景 更多参数细节可参考OpenAI API官方文档。 -## 新增数据格式转化 +### 新增数据格式转化 上文的 **教程-基础配置** 篇章中介绍了项目配置的参数列表，其中 data_format 表示数据的格式，同时也代表了一种数据转化的方式。 dingo 内置的数据转化方式有4种，即 data_format 的4个可取的值: json, jsonl, plaintext, listjson. @@ -359,7 +359,7 @@ dynamic_config = DynamicRuleConfig() def eval(cls, input_data: Data) -> ModelRes: ``` -## 新增提示词 +### 新增提示词 上文的 **提示词** 篇章中已经介绍了 [提示词列表](../dingo/model/prompt) ，如果用户在评估过程中产生了新的评估任务，需要涉及自己的提示词。 @@ -403,7 +403,7 @@ content = """ """ ``` -## 新增场景 +### 新增场景 上文的 **场景** 篇章介绍了场景的职责，即: 打包发送数据、接收解析数据 那么，新增一个场景就需要实现以上2个功能，详情见下方模板: diff --git a/docs/technical/technical_local.md b/docs/technical/technical_local.md index ae649d58..8e7730de 100644 --- a/docs/technical/technical_local.md +++ b/docs/technical/technical_local.md @@ -1,4 +1,4 @@ -# dingo.exec.local 技术报告 +# Dingo: Executor的Local模式介绍 ## 一、模块定位与作用 diff --git a/docs/technical/technical_model.md b/docs/technical/technical_model.md index aa152354..af4d6520 100644 --- a/docs/technical/technical_model.md +++ b/docs/technical/technical_model.md @@ -1,118 +1,448 @@ -# dingo.model.model 技术文档 +# Dingo: Evaluator层的设计逻辑说明 ## 一、概述 `dingo.model.model` 主要负责模型（包括规则、提示词、LLM等）的注册、分组、配置应用和动态加载。它为整个 Dingo 系统提供了统一的模型管理和配置入口，支持自动发现和注册规则、提示词、LLM，并能根据配置文件动态调整模型行为。 +### 1.1 核心功能 + +- **模型注册管理**：提供统一的装饰器接口，支持规则、提示词、LLM的自动注册 +- **分组与分类**：支持按功能分组和按metric_type分类的多维度组织方式 +- **动态配置**：支持通过配置文件动态调整模型参数和行为 +- **自动发现**：自动扫描目录结构，发现并加载所有可用的模型类 +- **统一接口**：提供一致的API接口，简化模型的使用和管理 + +### 1.2 架构设计 + +``` +┌─────────────────────────────────────────────────────────┐ +│ Model Manager │ +├─────────────────────────────────────────────────────────┤ +│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ +│ │ Rules │ │ Prompts │ │ LLMs │ │ +│ │ Management │ │ Management │ │ Management │ │ +│ └─────────────┘ └─────────────┘ └─────────────┘ │ +├─────────────────────────────────────────────────────────┤ +│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ +│ │ Groups │ │ Metric Type │ │ Name Maps │ │ +│ │ Management │ │ Mapping │ │ Management │ │ +│ └─────────────┘ └─────────────┘ └─────────────┘ │ +├─────────────────────────────────────────────────────────┤ +│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ +│ │ Config │ │ Auto │ │ Dynamic │ │ +│ │ Application │ │ Discovery │ │ Loading │ │ +│ └─────────────┘ └─────────────┘ └─────────────┘ │ +└─────────────────────────────────────────────────────────┘ +``` + --- ## 二、主要类与结构 -### 1. BaseEvalModel +### 2.1 BaseEvalModel -- 继承自 `pydantic.BaseModel` -- 字段： - - `name`: str - - `type`: str -- 用于描述基础的评测模型信息。 +#### 类定义 +```python +class BaseEvalModel(BaseModel): + name: str + type: str +``` -### 2. Model +#### 功能说明 +- 继承自 `pydantic.BaseModel`，提供数据验证和序列化功能 +- 用于描述基础的评测模型信息 +- 作为所有评测模型的基础数据结构 -#### 主要职责 +#### 使用场景 +- 配置文件中的模型定义 +- API接口的数据传输 +- 模型元数据的存储 -- 管理和注册规则（Rule）、提示词（Prompt）、LLM（大语言模型）。 -- 支持模型的分组、按 metric_type 分类、按名称索引。 -- 支持根据配置文件动态应用模型参数。 -- 自动加载和注册 `rule/`、`prompt/`、`llm/` 目录下的所有模型类。 +### 2.2 Model -#### 主要类属性 +#### 2.2.1 主要职责 -- `module_loaded`: 是否已加载模块，防止重复加载。 -- `rule_groups`, `prompt_groups`: 分组管理，结构如 `{group_name: [class, ...]}`。 -- `rule_metric_type_map`, `prompt_metric_type_map`: 按 metric_type 分类的映射。 -- `rule_name_map`, `prompt_name_map`, `llm_name_map`: 名称到类的映射。 +- **模型生命周期管理**：负责模型的全生命周期，从注册到使用到配置 +- **分组与分类管理**：支持多维度的模型组织方式 +- **配置动态应用**：支持运行时动态调整模型参数 +- **自动发现机制**：自动扫描和加载可用的模型类 -#### 关键方法 +#### 2.2.2 核心类属性 -##### 注册相关 +```python +class Model: + # 模块加载状态管理 + module_loaded = False + + # 分组管理 + rule_groups = {} # {group_name: [rule_classes]} + prompt_groups = {} # {group_name: [prompt_classes]} + + # 按metric_type分类 + rule_metric_type_map = {} # {metric_type: [rule_classes]} + prompt_metric_type_map = {} # {metric_type: [prompt_classes]} + + # 名称映射 + rule_name_map = {} # {rule_name: rule_class} + prompt_name_map = {} # {prompt_name: prompt_class} + llm_name_map = {} # {llm_name: llm_class} +``` -- `rule_register(metric_type, group)`: 装饰器，用于注册规则类，指定其 metric_type 和分组。 -- `llm_register(llm_id)`: 装饰器，用于注册 LLM 类，指定其唯一 ID。 -- `prompt_register(metric_type, group)`: 装饰器，用于注册提示词类，指定其 metric_type 和分组。 +#### 2.2.3 数据流图 -##### 获取与查询 +``` +┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ +│ Model Files │───▶│ Auto Loader │───▶│ Name Maps │ +│ (rule/, │ │ │ │ │ +│ prompt/, │ │ │ │ │ +│ llm/) │ │ │ │ │ +└─────────────────┘ └─────────────────┘ └─────────────────┘ + │ │ + ▼ ▼ + ┌─────────────────┐ ┌─────────────────┐ + │ Groups │ │ Metric Type │ + │ Management │ │ Mapping │ + └─────────────────┘ └─────────────────┘ + │ │ + ▼ ▼ + ┌─────────────────┐ ┌─────────────────┐ + │ Config │ │ Runtime │ + │ Application │ │ Usage │ + └─────────────────┘ └─────────────────┘ +``` -- `get_group(group_name)`: 获取指定分组下的规则和/或提示词。 -- `get_rule_metric_type_map()`: 获取所有规则的 metric_type 映射。 -- `get_metric_type_by_rule_name(rule_name)`: 通过规则名获取其 metric_type。 -- `get_rule_group(rule_group_name)`: 获取指定规则分组的所有规则类。 -- `get_rule_groups()`: 获取所有规则分组。 -- `get_rules_by_group(group_name)`: 获取指定分组下所有规则的名称（含 metric_type）。 -- `get_rule_by_name(name)`: 通过名称获取规则类。 -- `get_llm_name_map()`: 获取所有 LLM 的名称映射。 -- `get_llm(llm_name)`: 通过名称获取 LLM 类。 -- `print_rule_list()`: 打印所有已注册规则的名称。 +--- -##### 配置应用 +## 三、核心方法详解 -- `apply_config_rule()`: 应用全局配置中的规则参数到已注册规则。 -- `apply_config_llm()`: 应用全局配置中的 LLM 参数到已注册 LLM。 -- `apply_config_rule_list(eval_group)`: 根据配置文件中的 rule_list，生成分组。 -- `apply_config_prompt_list(eval_group)`: 根据配置文件中的 prompt_list，生成分组。 -- `apply_config(custom_config, eval_group)`: 读取配置文件并应用所有相关配置。 -- `apply_config_for_spark_driver(custom_config, eval_group)`: 专为 Spark Driver 场景设计的配置应用。 +### 3.1 注册相关方法 -##### 自动加载 +#### 3.1.1 rule_register -- `load_model()`: 自动加载 `rule/`、`prompt/`、`llm/` 目录下的所有 Python 文件，实现自动注册。 +```python +@classmethod +def rule_register(cls, metric_type: str, group: List[str]) -> Callable: +``` ---- +**功能**：注册规则类的装饰器 -## 三、使用示例 +**参数说明**： +- `metric_type`: 规则所属的评测类型（如"QUALITY", "SAFETY"等） +- `group`: 规则所属的分组列表 -### 1. 注册规则/LLM/Prompt +**使用示例**： +```python +@Model.rule_register(metric_type="QUALITY", group=["default", "strict"]) +class ContentQualityRule(BaseRule): + def eval(self, data: Data) -> ModelRes: + # 实现评测逻辑 + pass +``` + +**内部流程**： +1. 将规则类添加到指定的分组中 +2. 将规则类添加到metric_type映射中 +3. 将规则类添加到名称映射中 +4. 为规则类设置group和metric_type属性 + +#### 3.1.2 llm_register ```python -@Model.rule_register(metric_type="QUALITY", group=["default"]) -class MyRule(BaseRule): - ... +@classmethod +def llm_register(cls, llm_id: str) -> Callable: ``` +**功能**：注册LLM类的装饰器 + +**参数说明**： +- `llm_id`: LLM的唯一标识符 + +**使用示例**： ```python -@Model.llm_register(llm_id="gpt3") -class GPT3LLM(BaseLLM): - ... +@Model.llm_register(llm_id="gpt-3.5-turbo") +class GPT35TurboLLM(BaseLLM): + def eval(self, data: Data) -> ModelRes: + # 实现LLM评测逻辑 + pass ``` -### 2. 应用配置 +#### 3.1.3 prompt_register ```python -Model.apply_config("path/to/config.yaml", eval_group="my_eval_group") +@classmethod +def prompt_register(cls, metric_type: str, group: List[str]) -> Callable: ``` -### 3. 获取分组信息 +**功能**：注册提示词类的装饰器 + +**参数说明**： +- `metric_type`: 提示词所属的评测类型 +- `group`: 提示词所属的分组列表 + +### 3.2 查询与获取方法 + +#### 3.2.1 分组查询 +```python +@classmethod +def get_group(cls, group_name) -> Dict[str, List]: +``` + +**功能**：获取指定分组下的所有模型 + +**返回值**： +```python +{ + 'rule': [rule_classes], + 'prompt': [prompt_classes] +} +``` + +**使用示例**： ```python group_info = Model.get_group("default") rules = group_info.get("rule", []) prompts = group_info.get("prompt", []) ``` ---- +#### 3.2.2 按类型查询 + +```python +@classmethod +def get_rule_metric_type_map(cls) -> Dict[str, List[Callable]]: +``` + +**功能**：获取所有规则的metric_type映射 + +**返回值**： +```python +{ + 'QUALITY': [rule_class1, rule_class2], + 'SAFETY': [rule_class3, rule_class4] +} +``` + +#### 3.2.3 按名称查询 + +```python +@classmethod +def get_rule_by_name(cls, name: str) -> Callable: +``` + +**功能**：通过名称获取规则类 + +**使用示例**： +```python +rule_class = Model.get_rule_by_name("ContentQualityRule") +``` + +### 3.3 配置应用方法 + +#### 3.3.1 apply_config_rule + +```python +@classmethod +def apply_config_rule(cls): +``` -## 四、设计亮点 +**功能**：应用全局配置中的规则参数 -- **自动注册**：通过装饰器和自动扫描目录，极大简化了模型的注册流程。 -- **灵活分组**：支持规则、提示词的多分组和多类型映射，便于扩展和管理。 -- **动态配置**：可通过配置文件动态调整模型参数和分组，适应不同评测场景。 -- **统一接口**：所有模型的获取、注册、配置应用均通过统一接口完成，便于维护。 +**处理流程**： +1. 检查GlobalConfig中是否有rule_config +2. 遍历每个规则配置 +3. 获取规则的dynamic_config +4. 根据配置文件更新配置参数 + +**配置示例**： +```yaml +rule_config: + ContentQualityRule: + - ["threshold", 0.8] + - ["max_length", 1000] +``` + +#### 3.3.2 apply_config_llm + +```python +@classmethod +def apply_config_llm(cls): +``` + +**功能**：应用全局配置中的LLM参数 + +**处理流程**： +1. 检查GlobalConfig中是否有llm_config +2. 遍历每个LLM配置 +3. 获取LLM的dynamic_config +4. 根据配置文件更新配置参数 + +#### 3.3.3 apply_config + +```python +@classmethod +def apply_config(cls, custom_config: Optional[str | dict], eval_group: str = ''): +``` + +**功能**：完整的配置应用流程 + +**处理流程**： +1. 读取配置文件 +2. 应用规则配置 +3. 应用LLM配置 +4. 应用规则列表配置 +4. 应用提示词列表配置 + +### 3.4 自动加载方法 + +#### 3.4.1 load_model + +```python +@classmethod +def load_model(cls): +``` + +**功能**：自动加载所有模型文件 + +**处理流程**： +1. 检查是否已加载，避免重复加载 +2. 扫描rule/目录下的所有.py文件 +3. 扫描prompt/目录下的所有.py文件 +4. 扫描llm/目录下的所有.py文件 +4. 使用importlib动态导入模块 +6. 处理导入异常，记录日志 + +**目录结构要求**： +``` +dingo/model/ +├── rule/ +│ ├── __init__.py +│ ├── quality_rule.py +│ └── safety_rule.py +├── prompt/ +│ ├── __init__.py +│ ├── qa_prompt.py +│ └── summary_prompt.py +└── llm/ + ├── __init__.py + ├── gpt_llm.py + └── claude_llm.py +``` --- -## 五、注意事项 +## 四、扩展性设计 + +### 4.1 插件化架构 + +Model类采用插件化设计，支持： + +1. **动态注册**：运行时动态注册新的模型 +2. **热插拔**：支持模型的动态加载和卸载 +3. **版本管理**：支持模型版本的管理和切换 -- 注册的类必须继承自对应的基类（如 `BaseRule`, `BasePrompt`, `BaseLLM`）。 -- 配置文件需符合 `GlobalConfig` 的格式要求。 -- 自动加载依赖于目录结构，需保证 `rule/`、`prompt/`、`llm/` 目录下的 Python 文件可被正确导入。 +### 4.2 自定义扩展 + +#### 4.2.1 自定义规则 + +```python +@Model.rule_register(metric_type="CUSTOM", group=["custom"]) +class CustomRule(BaseRule): + def __init__(self): + super().__init__() + self.custom_param = "default" + + def eval(self, data: Data) -> ModelRes: + # 自定义评测逻辑 + result = self.custom_evaluation(data) + return ModelRes( + type="CUSTOM", + name="CustomRule", + error_status=result.is_error, + reason=result.reasons + ) +``` + +#### 4.2.2 自定义LLM + +```python +@Model.llm_register(llm_id="custom-llm") +class CustomLLM(BaseLLM): + def __init__(self): + super().__init__() + self.api_key = None + self.endpoint = None + + def set_prompt(self, prompt: BasePrompt): + self.current_prompt = prompt + + def eval(self, data: Data) -> ModelRes: + # 自定义LLM调用逻辑 + response = self.call_custom_api(data) + return self.parse_response(response) +``` + +### 4.3 配置扩展 + +支持多种配置格式： + +1. **JSON配置**： +```json +{ + "rule_config": { + "CustomRule": [ + ["custom_param", "value"] + ] + } +} +``` + +2. **Python配置**： +```python +config = { + "rule_config": { + "CustomRule": [ + ["custom_param", "value"] + ] + } +} +Model.apply_config(config, eval_group="custom") +``` --- + +## 五、注意事项与限制 + +### 5.1 使用限制 + +1. **继承要求**：所有注册的类必须继承自对应的基类 + - 规则类：继承自`BaseRule` + - 提示词类：继承自`BasePrompt` + - LLM类：继承自`BaseLLM` + +2. **命名要求**： + - 类名必须唯一 + - LLM的llm_id必须唯一 + - 分组名不能重复 + +3. **目录结构要求**： + - 必须存在`rule/`、`prompt/`、`llm/`目录 + - Python文件必须以`.py`结尾 + - 不能包含`__init__.py`文件 + +### 5.2 配置限制 + +1. **配置文件格式**：必须符合`GlobalConfig`的格式要求 +2. **参数类型**：配置参数必须与模型期望的类型匹配 +3. **依赖关系**：配置的模型必须已经注册 + +### 5.3 性能考虑 + +1. **内存使用**：大量模型注册可能占用较多内存 +2. **加载时间**：首次加载可能需要较长时间 +3. **并发安全**：多线程环境下需要注意线程安全 + +### 5.4 错误处理 + +1. **导入错误**：模块导入失败时会记录日志但不会中断程序 +2. **配置错误**：配置参数错误时会抛出异常 +3. **注册错误**：重复注册时会覆盖之前的注册 From 35c97935f4847ff7faeacaf52386a94ea6f4c9b6 Mon Sep 17 00:00:00 2001 From: chupei Date: Fri, 11 Jul 2025 11:59:13 +0800 Subject: [PATCH 19/24] docs: add posts doc --- docs/posts/reddit.md | 66 +++++++++++++++++++++++++++++++++++++++ docs/posts/x.md | 10 ++++++ docs/posts/xiaohongshu.md | 24 ++++++++++++++ docs/posts/zhihu.md | 1 + 4 files changed, 101 insertions(+) create mode 100644 docs/posts/reddit.md create mode 100644 docs/posts/x.md create mode 100644 docs/posts/xiaohongshu.md create mode 100644 docs/posts/zhihu.md diff --git a/docs/posts/reddit.md b/docs/posts/reddit.md new file mode 100644 index 00000000..18b6ea60 --- /dev/null +++ b/docs/posts/reddit.md @@ -0,0 +1,66 @@ +# 文稿一 +[OC] Comprehensive AI Data Quality Metrics Documentation - 50+ Evaluation Metrics with Academic Sources + +We've just released what might be the most comprehensive documentation of AI data quality evaluation metrics available. This covers everything from pre-training data assessment to multimodal evaluation. + +**What's included:** +- 50+ evaluation metrics across text, image, and multimodal data +- Academic citations for every metric (RedPajama, CLIP, NIMA, etc.) +- Rule-based and LLM-based evaluation approaches +- Practical usage examples and API documentation + +**Key categories:** +- Text Quality: Completeness, Fluency, Relevance, Effectiveness +- Image Quality: Clarity, Similarity, Validity +- Security: Political sensitivity, prohibited content, harmful information +- Classification: Topic categorization, content classification + +This is particularly useful for: +- Data scientists working on model training +- Researchers needing standardized evaluation frameworks +- Anyone dealing with large-scale data quality assessment + +The documentation includes detailed academic references and practical implementation examples. All open source and ready to use. + +**Link:** https://github.com/MigoXLab/dingo/blob/dev/docs/metrics.md + +Thoughts? What metrics do you find most valuable in your work? + +# 文稿二 +[P] Comprehensive AI Data Quality Metrics Documentation - 50+ Evaluation Metrics with Academic Sources for Dingo Framework + +Hey r/MachineLearning! + +We've just released comprehensive documentation for AI data quality evaluation metrics in our open-source framework **Dingo**. This covers everything from pre-training data assessment to multimodal evaluation. + +**Project Overview:** +Dingo is an enterprise-grade data quality assessment tool designed for large-scale AI training pipelines. + +**What's included in the metrics documentation:** +- 50+ evaluation metrics across text, image, and multimodal data +- Academic citations for every metric (RedPajama, CLIP, NIMA, etc.) +- Rule-based and LLM-based evaluation approaches +- Practical usage examples and API documentation + +**Key metric categories:** +- **Text Quality**: Completeness, Fluency, Relevance, Effectiveness +- **Image Quality**: Clarity, Similarity, Validity +- **Security Assessment**: Political sensitivity, prohibited content detection +- **SFT Evaluation**: 3H principles (Honest, Helpful, Harmless) + +**Technical features:** +- Multi-source support (HuggingFace, local files, S3) +- Distributed processing with Spark integration +- CLI/SDK/Web interfaces +- Comprehensive evaluation reports + +This is particularly useful for: +- Data scientists working on model training pipelines +- Researchers needing standardized evaluation frameworks +- MLOps teams implementing data quality gates +- Anyone dealing with large-scale data preprocessing + +**Documentation:** https://github.com/MigoXLab/dingo/blob/dev/docs/metrics.md +**Project repo:** https://github.com/MigoXLab/dingo + +Would love to get feedback from the community! What data quality metrics do you find most valuable in your work? diff --git a/docs/posts/x.md b/docs/posts/x.md new file mode 100644 index 00000000..a89982d2 --- /dev/null +++ b/docs/posts/x.md @@ -0,0 +1,10 @@ +# 文稿一 +🚀 RELEASE: The most comprehensive AI data quality metrics documentation ever assembled! + +📊 50+ evaluation metrics +🎓 Academic-backed (RedPajama, CLIP, NIMA...) +⚡ Rule-based + LLM evaluation + +Metrics Detail：📖 https://github.com/MigoXLab/dingo/blob/dev/docs/metrics.md + +What's your go-to data quality metric? 🤔 diff --git a/docs/posts/xiaohongshu.md b/docs/posts/xiaohongshu.md new file mode 100644 index 00000000..d3a3beef --- /dev/null +++ b/docs/posts/xiaohongshu.md @@ -0,0 +1,24 @@ +# 文稿一 +🔥 AI数据质量评估神器Dingo开源！ + +还在为训练数据质量发愁？这个工具绝了！ + +💻 技术特性： +• 50+项专业评估指标 +• Rule+LLM双引擎评估 +• 支持预训练/SFT/多模态数据 +• 分布式处理，性能强悍 + +🎓 学术背景： +基于RedPajama、CLIP等顶会论文 +确保评估的科学性和权威性 + +⚡ 实战效果： +数据筛选效率提升10+倍 + +GitHub: MigoXLab/dingo + +📖详细指标： +https://github.com/MigoXLab/dingo/blob/dev/docs/metrics.md + +#AI工程师必备 #开源工具 #数据科学 diff --git a/docs/posts/zhihu.md b/docs/posts/zhihu.md new file mode 100644 index 00000000..74752146 --- /dev/null +++ b/docs/posts/zhihu.md @@ -0,0 +1 @@ +# 文稿一 From 953b3fd844af555913000a1f875587a6f7d13cf1 Mon Sep 17 00:00:00 2001 From: chupei Date: Fri, 11 Jul 2025 16:43:51 +0800 Subject: [PATCH 20/24] docs: update posts doc (#119) --- docs/posts/reddit.md | 4 ++-- docs/posts/x.md | 2 +- docs/posts/xiaohongshu.md | 2 +- docs/posts/zhihu.md | 3 ++- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/posts/reddit.md b/docs/posts/reddit.md index 18b6ea60..7ac33318 100644 --- a/docs/posts/reddit.md +++ b/docs/posts/reddit.md @@ -1,4 +1,4 @@ -# 文稿一 +# 文案一 [OC] Comprehensive AI Data Quality Metrics Documentation - 50+ Evaluation Metrics with Academic Sources We've just released what might be the most comprehensive documentation of AI data quality evaluation metrics available. This covers everything from pre-training data assessment to multimodal evaluation. @@ -26,7 +26,7 @@ The documentation includes detailed academic references and practical implementa Thoughts? What metrics do you find most valuable in your work? -# 文稿二 +# 文案二 [P] Comprehensive AI Data Quality Metrics Documentation - 50+ Evaluation Metrics with Academic Sources for Dingo Framework Hey r/MachineLearning! diff --git a/docs/posts/x.md b/docs/posts/x.md index a89982d2..03225247 100644 --- a/docs/posts/x.md +++ b/docs/posts/x.md @@ -1,4 +1,4 @@ -# 文稿一 +# 文案一 🚀 RELEASE: The most comprehensive AI data quality metrics documentation ever assembled! 📊 50+ evaluation metrics diff --git a/docs/posts/xiaohongshu.md b/docs/posts/xiaohongshu.md index d3a3beef..c724cbbc 100644 --- a/docs/posts/xiaohongshu.md +++ b/docs/posts/xiaohongshu.md @@ -1,4 +1,4 @@ -# 文稿一 +# 文案一 🔥 AI数据质量评估神器Dingo开源！ 还在为训练数据质量发愁？这个工具绝了！ diff --git a/docs/posts/zhihu.md b/docs/posts/zhihu.md index 74752146..bec0a026 100644 --- a/docs/posts/zhihu.md +++ b/docs/posts/zhihu.md @@ -1 +1,2 @@ -# 文稿一 +# 文案一 +[Dingo：面向AI时代的全方位数据质量评估工具](https://zhuanlan.zhihu.com/p/1892338512306602995) From 6b573e6442f84c85c4a51f7f7a1bbe96e5da3bdd Mon Sep 17 00:00:00 2001 From: shijinpjlab Date: Mon, 14 Jul 2025 17:15:16 +0800 Subject: [PATCH 21/24] feat: add app and web-static in wheel package (#121) * feat: add app and web-static in wheel package * feat: fix lint --- setup.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/setup.py b/setup.py index 6e99cf2e..6e6fe4f4 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,5 @@ +import os + from setuptools import find_packages, setup with open("README.md", "r", encoding='utf-8') as fh: @@ -9,6 +11,19 @@ with open("./requirements/web.txt", "r", encoding='utf-8') as f: requirements.extend(f.readlines()) + +# 获取 app 和 web-static 目录下的所有文件 +def get_data_files(directory): + paths = [] + for (path, directories, filenames) in os.walk(directory): + for filename in filenames: + paths.append(os.path.join('..', path, filename)) + return paths + + +app_files = get_data_files('app') +web_static_files = get_data_files('web-static') + setup( name="dingo-python", version="1.8", @@ -18,6 +33,10 @@ long_description_content_type="text/markdown", url="https://github.com/MigoXLab/dingo", packages=find_packages(), + package_data={ + '': app_files + web_static_files, + }, + include_package_data=True, classifiers=[ "Programming Language :: Python :: 3", "Operating System :: OS Independent", From 8f48cd09e74842b775d36e922668521fb37da572 Mon Sep 17 00:00:00 2001 From: shijinpjlab Date: Thu, 17 Jul 2025 10:38:32 +0800 Subject: [PATCH 22/24] feat: gradio add new function (#123) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: gradio 添加参数 * feat: gradio 添加场景，并与prompt联动 * gradio: scen and prompt * gradio: 勾选prompt才显示模型配置 * gradio: 切换场景，prompt清空所有勾选 * gradio: 添加rule_type * gradio: 联动逻辑 * gradio: rule_type切换，rule_list清空 * gradio: 提示 * gradio: 提示 --- app_gradio/app.py | 310 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 284 insertions(+), 26 deletions(-) diff --git a/app_gradio/app.py b/app_gradio/app.py index 4426fd4b..39bd845a 100644 --- a/app_gradio/app.py +++ b/app_gradio/app.py @@ -1,6 +1,8 @@ import json import os +import pprint import shutil +from functools import partial from pathlib import Path import gradio as gr @@ -9,8 +11,13 @@ from dingo.io import InputArgs -def dingo_demo(dataset_source, input_path, uploaded_file, data_format, column_content, rule_list, prompt_list, model, - key, api_url): +def dingo_demo( + uploaded_file, + dataset_source, data_format, input_path, max_workers, batch_size, + column_id, column_prompt, column_content, column_image, + rule_list, prompt_list, scene_list, + model, key, api_url + ): if not data_format: raise gr.Error('ValueError: data_format can not be empty, please input.') if not column_content: @@ -33,29 +40,46 @@ def dingo_demo(dataset_source, input_path, uploaded_file, data_format, column_co final_input_path = uploaded_file.name + if max_workers <= 0: + raise gr.Error('Please input value > 0 in max_workers.') + if batch_size <= 0: + raise gr.Error('Please input value > 0 in batch_size.') + try: input_data = { "dataset": dataset_source, + "data_format": data_format, "input_path": final_input_path, "output_path": "" if dataset_source == 'hugging_face' else os.path.dirname(final_input_path), "save_data": True, "save_raw": True, - "data_format": data_format, + + "max_workers": max_workers, + "batch_size": batch_size, + "column_content": column_content, "custom_config":{ "rule_list": rule_list, "prompt_list": prompt_list, - "llm_config": - { - "LLMTextQualityPromptBase": - { - "model": model, - "key": key, - "api_url": api_url, - } + "llm_config": { + scene_list: { + "model": model, + "key": key, + "api_url": api_url, } + } } } + if column_id: + input_data['column_id'] = column_id + if column_prompt: + input_data['column_prompt'] = column_prompt + if column_image: + input_data['column_image'] = column_image + + # print(input_data) + # exit(0) + input_args = InputArgs(**input_data) executor = Executor.exec_map["local"](input_args) summary = executor.execute().to_dict() @@ -88,9 +112,154 @@ def update_input_components(dataset_source): ] +def update_rule_list(rule_type_mapping, rule_type): + return gr.CheckboxGroup( + choices=rule_type_mapping.get(rule_type, []), + value=[], + label="rule_list" + ) + + +def update_prompt_list(scene_prompt_mapping, scene): + """根据选择的场景更新可用的prompt列表，并清空所有勾选""" + return gr.CheckboxGroup( + choices=scene_prompt_mapping.get(scene, []), + value=[], # 清空所有勾选 + label="prompt_list" + ) + + +# prompt_list变化时，动态控制model、key、api_url的显示 +def toggle_llm_fields(prompt_values): + visible = bool(prompt_values) + return ( + gr.update(visible=visible), + gr.update(visible=visible), + gr.update(visible=visible) + ) + + +# 控制column_id、column_prompt、column_content、column_image的显示 +def update_column_fields(rule_list, prompt_list): + rule_type_mapping = get_rule_type_mapping() + scene_prompt_mapping = get_scene_prompt_mapping() + data_column_mapping = get_data_column_mapping() + status_mapping = { + 'id': False, + 'prompt': False, + 'content': False, + 'image': False, + } + + res = ( + gr.update(visible=status_mapping['id']), + gr.update(visible=status_mapping['prompt']), + gr.update(visible=status_mapping['content']), + gr.update(visible=status_mapping['image']) + ) + if not rule_list and not prompt_list: + return res + + key_list = [] + key_list += get_key_by_mapping(rule_type_mapping, rule_list) + key_list += get_key_by_mapping(scene_prompt_mapping, prompt_list) + + data_column = [] + for key in key_list: + if not data_column: + data_column = data_column_mapping[key] + else: + new_data_column = data_column_mapping[key] + if data_column != new_data_column: + raise gr.Error(f'ConflictError: {key} need data type is different from other.') + + for c in data_column: + status_mapping[c] = True + res = ( + gr.update(visible=status_mapping['id']), + gr.update(visible=status_mapping['prompt']), + gr.update(visible=status_mapping['content']), + gr.update(visible=status_mapping['image']) + ) + return res + + +def get_rule_type_mapping(): + return { + 'QUALITY_BAD_COMPLETENESS': ['RuleLineEndWithEllipsis', 'RuleLineEndWithTerminal', 'RuleSentenceNumber', + 'RuleWordNumber'], + 'QUALITY_BAD_EFFECTIVENESS': ['RuleAbnormalChar', 'RuleAbnormalHtml', 'RuleAlphaWords', 'RuleCharNumber', + 'RuleColonEnd', 'RuleContentNull', 'RuleContentShort', 'RuleContentShortMultiLan', + 'RuleEnterAndSpace', 'RuleEnterMore', 'RuleEnterRatioMore', 'RuleHtmlEntity', + 'RuleHtmlTag', 'RuleInvisibleChar', 'RuleLineJavascriptCount', 'RuleLoremIpsum', + 'RuleMeanWordLength', 'RuleSpaceMore', 'RuleSpecialCharacter', 'RuleStopWord', + 'RuleSymbolWordRatio', 'RuleOnlyUrl'], + 'QUALITY_BAD_FLUENCY': ['RuleAbnormalNumber', 'RuleCharSplit', 'RuleNoPunc', 'RuleWordSplit', 'RuleWordStuck'], + 'QUALITY_BAD_RELEVANCE': ['RuleHeadWordAr'], + 'QUALITY_BAD_SIMILARITY': ['RuleDocRepeat'], + 'QUALITY_BAD_UNDERSTANDABILITY': ['RuleCapitalWords', 'RuleCurlyBracket', 'RuleLineStartWithBulletpoint', + 'RuleUniqueWords'], + 'QUALITY_BAD_IMG_EFFECTIVENESS': ['RuleImageValid', 'RuleImageSizeValid', 'RuleImageQuality'], + 'QUALITY_BAD_IMG_RELEVANCE': ['RuleImageTextSimilarity'], + 'QUALITY_BAD_IMG_SIMILARITY': ['RuleImageRepeat'] + } + + +def get_scene_prompt_mapping(): + return { + # 示例映射关系，你可以根据实际需求修改 + "LLMTextQualityPromptBase": ['PromptRepeat', 'PromptContentChaos'], + 'LLMTextQualityModelBase': ['PromptTextQualityV3', 'PromptTextQualityV4'], + 'LLMSecurityPolitics': ['PromptPolitics'], + 'LLMSecurityProhibition': ['PromptProhibition'], + 'LLMText3HHarmless': ['PromptTextHelpful'], + 'LLMText3HHelpful': ['PromptTextHelpful'], + 'LLMText3HHonest': ['PromptTextHonest'], + 'LLMClassifyTopic': ['PromptClassifyTopic'], + 'LLMClassifyQR': ['PromptClassifyQR'], + "VLMImageRelevant": ["PromptImageRelevant"], + } + + +def get_key_by_mapping(map_dict: dict, value_list: list): + key_list = [] + for k,v in map_dict.items(): + if bool(set(v) & set(value_list)): + key_list.append(k) + + return key_list + + +def get_data_column_mapping(): + return { + 'LLMTextQualityPromptBase': ['content'], + 'LLMTextQualityModelBase': ['content'], + 'LLMSecurityPolitics': ['content'], + 'LLMSecurityProhibition': ['content'], + 'LLMText3HHarmless': ['content'], + 'LLMText3HHelpful': ['content'], + 'LLMText3HHonest': ['content'], + 'LLMClassifyTopic': ['content'], + 'LLMClassifyQR': ['content'], + 'VLMImageRelevant': ['prompt', 'content'], + 'QUALITY_BAD_COMPLETENESS': ['content'], + 'QUALITY_BAD_EFFECTIVENESS': ['content'], + 'QUALITY_BAD_FLUENCY': ['content'], + 'QUALITY_BAD_RELEVANCE': ['content'], + 'QUALITY_BAD_SIMILARITY': ['content'], + 'QUALITY_BAD_UNDERSTANDABILITY': ['content'], + 'QUALITY_BAD_IMG_EFFECTIVENESS': ['image'], + 'QUALITY_BAD_IMG_RELEVANCE': ['content','image'], + 'QUALITY_BAD_IMG_SIMILARITY': ['content'], + } + + if __name__ == '__main__': - rule_options = ['RuleAbnormalChar', 'RuleAbnormalHtml', 'RuleContentNull', 'RuleContentShort', 'RuleEnterAndSpace', 'RuleOnlyUrl'] - prompt_options = ['PromptRepeat', 'PromptContentChaos'] + rule_type_mapping = get_rule_type_mapping() + rule_type_options = list(rule_type_mapping.keys()) + + scene_prompt_mapping = get_scene_prompt_mapping() + scene_options = list(scene_prompt_mapping.keys()) current_dir = Path(__file__).parent with open(os.path.join(current_dir, 'header.html'), "r") as file: @@ -120,34 +289,91 @@ def update_input_components(dataset_source): ["jsonl", "json", "plaintext", "listjson"], label="data_format" ) - column_content = gr.Textbox( - value="content", - placeholder="please input column name of content in dataset", - label="column_content" - ) + with gr.Row(): + max_workers = gr.Number( + value=1, + # placeholder="", + label="max_workers", + precision=0 + ) + batch_size = gr.Number( + value=1, + # placeholder="", + label="batch_size", + precision=0 + ) + # Add the rule_type dropdown near where scene_list is defined + rule_type = gr.Dropdown( + choices=rule_type_options, + value=rule_type_options[0], + label="rule_type", + interactive=True + ) rule_list = gr.CheckboxGroup( - choices=rule_options, - value=['RuleAbnormalChar', 'RuleAbnormalHtml'], + choices=rule_type_mapping.get(rule_type_options[0], []), label="rule_list" ) + # 添加场景选择下拉框 + scene_list = gr.Dropdown( + choices=scene_options, + value=scene_options[0], + label="scene_list", + interactive=True + ) prompt_list = gr.CheckboxGroup( - choices=prompt_options, + choices=scene_prompt_mapping.get(scene_options[0], []), label="prompt_list" ) + # LLM模型名 model = gr.Textbox( placeholder="If want to use llm, please input model, such as: deepseek-chat", - label="model" + label="model", + visible=False ) + # LLM API KEY key = gr.Textbox( placeholder="If want to use llm, please input key, such as: 123456789012345678901234567890xx", - label="API KEY" + label="API KEY", + visible=False ) + # LLM API URL api_url = gr.Textbox( placeholder="If want to use llm, please input api_url, such as: https://api.deepseek.com/v1", - label="API URL" + label="API URL", + visible=False ) + with gr.Row(): + # 字段映射说明文本，带示例链接 + with gr.Column(): + gr.Markdown("Field Matching: Please input the column name of dataset in the input boxes below ( [examples](https://github.com/MigoXLab/dingo/tree/main/examples) )") + + column_id = gr.Textbox( + value="", + placeholder="Column name of id in the input file. If exists multiple levels, use '.' separate", + label="column_id", + visible=False + ) + column_prompt = gr.Textbox( + value="", + placeholder="Column name of prompt in the input file. If exists multiple levels, use '.' separate", + label="column_prompt", + visible=False + ) + column_content = gr.Textbox( + value="content", + placeholder="Column name of content in the input file. If exists multiple levels, use '.' separate", + label="column_content", + visible=False + ) + column_image = gr.Textbox( + value="", + placeholder="Column name of image in the input file. If exists multiple levels, use '.' separate", + label="column_image", + visible=False + ) + with gr.Row(): submit_single = gr.Button(value="Submit", interactive=True, variant="primary") @@ -165,10 +391,42 @@ def update_input_components(dataset_source): outputs=[input_path, uploaded_file] ) + rule_type.change( + fn=partial(update_rule_list, rule_type_mapping), + inputs=rule_type, + outputs=rule_list + ) + + # 场景变化时更新prompt列表 + scene_list.change( + fn=partial(update_prompt_list, scene_prompt_mapping), + inputs=scene_list, + outputs=prompt_list + ) + + prompt_list.change( + fn=toggle_llm_fields, + inputs=prompt_list, + outputs=[model, key, api_url] + ) + + # column字段显示控制 + for comp in [rule_list, prompt_list]: + comp.change( + fn=update_column_fields, + inputs=[rule_list, prompt_list], + outputs=[column_id, column_prompt, column_content, column_image] + ) + submit_single.click( fn=dingo_demo, - inputs=[dataset_source, input_path, uploaded_file, data_format, column_content, rule_list, prompt_list, - model, key, api_url], + inputs=[ + uploaded_file, + dataset_source, data_format, input_path, max_workers, batch_size, + column_id, column_prompt, column_content, column_image, + rule_list, prompt_list, scene_list, + model, key, api_url + ], outputs=[summary_output, detail_output] # 修改输出为两个组件 ) From 6268f6573a3fe7efd5fdaa4bdaa3e60c3709bb38 Mon Sep 17 00:00:00 2001 From: shijinpjlab Date: Thu, 17 Jul 2025 11:50:45 +0800 Subject: [PATCH 23/24] =?UTF-8?q?feat:=20gradio=E6=8F=90=E7=A4=BA=E3=80=81?= =?UTF-8?q?httpx=20=E7=89=88=E6=9C=AC=20(#125)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app_gradio/app.py | 2 +- requirements/runtime.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app_gradio/app.py b/app_gradio/app.py index 39bd845a..547cdbf6 100644 --- a/app_gradio/app.py +++ b/app_gradio/app.py @@ -347,7 +347,7 @@ def get_data_column_mapping(): with gr.Row(): # 字段映射说明文本，带示例链接 with gr.Column(): - gr.Markdown("Field Matching: Please input the column name of dataset in the input boxes below ( [examples](https://github.com/MigoXLab/dingo/tree/main/examples) )") + gr.Markdown("Please input the column name of dataset in the input boxes below ( [examples](https://github.com/MigoXLab/dingo/tree/main/examples) )") column_id = gr.Textbox( value="", diff --git a/requirements/runtime.txt b/requirements/runtime.txt index 0b702234..60111c33 100644 --- a/requirements/runtime.txt +++ b/requirements/runtime.txt @@ -4,7 +4,7 @@ chardet datasets fasttext-wheel==0.9.2 hanziconv -httpx==0.27.2 +httpx huggingface_hub jieba jsonlines From 87b00d474b43bafb25854038633876df1e0c335a Mon Sep 17 00:00:00 2001 From: shijinpjlab Date: Thu, 17 Jul 2025 14:11:53 +0800 Subject: [PATCH 24/24] feat: v1.8.1 (#127) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6e6fe4f4..2e1a3216 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ def get_data_files(directory): setup( name="dingo-python", - version="1.8", + version="1.8.1", author="Dingo", description="A Comprehensive AI Data Quality Evaluation Tool for Large Models", long_description=long_description,