diff --git a/.github/workflows/metrics-validation.yml b/.github/workflows/metrics-validation.yml new file mode 100644 index 00000000..32c630c2 --- /dev/null +++ b/.github/workflows/metrics-validation.yml @@ -0,0 +1,53 @@ +name: Metrics Documentation Auto-Update + +on: + push: + branches: [ main, dev ] + paths: + - 'dingo/model/prompt/**' + - 'scripts/generate_metrics.py' + pull_request: + branches: [ main ] + paths: + - 'dingo/model/prompt/**' + - 'scripts/generate_metrics.py' + workflow_dispatch: + +jobs: + update-metrics-docs: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e . + + - name: Generate metrics documentation + run: | + python scripts/generate_metrics.py + + - name: Check if documentation changed + id: check_changes + run: | + if git diff --quiet docs/metrics.md; then + echo "changed=false" >> $GITHUB_OUTPUT + else + echo "changed=true" >> $GITHUB_OUTPUT + fi + + - name: Commit updated documentation + if: steps.check_changes.outputs.changed == 'true' && github.event_name == 'push' + run: | + git config --local user.email "action@github.com" + git config --local user.name "GitHub Action" + git add docs/metrics.md + git commit -m "📚 Auto-update metrics documentation" + git push diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8b57ecd4..acb15b4d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,12 +6,14 @@ repos: hooks: - id: trailing-whitespace - id: end-of-file-fixer + exclude: 'docs/metrics\.md' - id: check-yaml - id: check-added-large-files - repo: https://github.com/PyCQA/isort rev: 6.0.1 hooks: - id: isort + args: [ "-l", "200", "-m", "0", "-p", "dingo" ] - repo: https://github.com/PyCQA/flake8 rev: 6.1.0 hooks: diff --git a/README.md b/README.md index 363062ba..6f275f10 100644 --- a/README.md +++ b/README.md @@ -179,61 +179,19 @@ This video demonstrates step-by-step how to use Dingo MCP server with Cursor. # Data Quality Metrics -Dingo classifies data quality issues into 7 dimensions of Quality Metrics. Each dimension can be evaluated using both rule-based methods and LLM-based prompts: +Dingo provides comprehensive data quality assessment through both rule-based and prompt-based evaluation metrics. These metrics cover multiple quality dimensions including effectiveness, completeness, similarity, security, and more. -| Quality Metric | Description | Rule Examples | LLM Prompt Examples | -|-------------------|-------------|---------------|---------------------| -| **COMPLETENESS** | Checks if data is incomplete or missing | `RuleColonEnd`, `RuleContentNull` | Evaluates if text abruptly ends with a colon or ellipsis, has mismatched parentheses, or missing critical components | -| **EFFECTIVENESS** | Checks if data is meaningful and properly formatted | `RuleAbnormalChar`, `RuleHtmlEntity`, `RuleSpecialCharacter` | Detects garbled text, words stuck together without spaces, and text lacking proper punctuation | -| **FLUENCY** | Checks if text is grammatically correct and reads naturally | `RuleAbnormalNumber`, `RuleNoPunc`, `RuleWordStuck` | Identifies excessively long words, text fragments without punctuation, or content with chaotic reading order | -| **RELEVANCE** | Detects irrelevant content within the data | `RuleHeadWord` variants for different languages | Examines for irrelevant information like citation details, headers/footers, entity markers, HTML tags | -| **SECURITY** | Identifies sensitive information or value conflicts | `RuleIDCard`, `RuleUnsafeWords` | Checks for personal information, and content related to gambling, pornography, political issues | -| **SIMILARITY** | Detects repetitive or highly similar content | `RuleDocRepeat` | Evaluates text for consecutive repeated content or multiple occurrences of special characters | -| **UNDERSTANDABILITY** | Assesses how easily data can be interpreted | `RuleCapitalWords` | Ensures LaTeX formulas and Markdown are correctly formatted, with proper segmentation and line breaks | +📊 **[View Complete Metrics Documentation →](docs/metrics.md)** -## LLM Quality Assessment +Our evaluation system includes: +- **Text Quality Assessment Metrics**: Pre-training data quality evaluation using DataMan methodology and enhanced multi-dimensional assessment +- **SFT Data Assessment Metrics**: Honest, Helpful, Harmless evaluation for supervised fine-tuning data +- **Classification Metrics**: Topic categorization and content classification +- **Multimodality Assessment Metrics**: Image classification and relevance evaluation +- **Rule-Based Quality Metrics**: Automated quality checks using heuristic rules for effectiveness and similarity detection +- etc -Dingo provides several LLM-based assessment methods defined by prompts in the `dingo/model/prompt` directory. These prompts are registered using the `prompt_register` decorator and can be combined with LLM models for quality evaluation: - -### Text Quality Assessment Prompts - -| Prompt Type | Metric | Description | -|-------------|--------|-------------| -| `TEXT_QUALITY_V2`, `TEXT_QUALITY_V3` | Various quality dimensions | Comprehensive text quality evaluation covering effectiveness, relevance, completeness, understandability, similarity, fluency, and security | -| `QUALITY_BAD_EFFECTIVENESS` | Effectiveness | Detects garbled text and anti-crawling content | -| `QUALITY_BAD_SIMILARITY` | Similarity | Identifies text repetition issues | -| `WORD_STICK` | Fluency | Checks for words stuck together without proper spacing | -| `CODE_LIST_ISSUE` | Completeness | Evaluates code blocks and list formatting issues | -| `UNREAD_ISSUE` | Effectiveness | Detects unreadable characters due to encoding issues | - -### 3H Assessment Prompts (Honest, Helpful, Harmless) - -| Prompt Type | Metric | Description | -|-------------|--------|-------------| -| `QUALITY_HONEST` | Honesty | Evaluates if responses provide accurate information without fabrication or deception | -| `QUALITY_HELPFUL` | Helpfulness | Assesses if responses address questions directly and follow instructions appropriately | -| `QUALITY_HARMLESS` | Harmlessness | Checks if responses avoid harmful content, discriminatory language, and dangerous assistance | - -### Domain-Specific Assessment Prompts - -| Prompt Type | Metric | Description | -|-------------|--------|-------------| -| `TEXT_QUALITY_KAOTI` | Exam question quality | Specialized assessment for evaluating the quality of exam questions, focusing on formula rendering, table formatting, paragraph structure, and answer formatting | -| `Html_Abstract` | HTML extraction quality | Compares different methods of extracting Markdown from HTML, evaluating completeness, formatting accuracy, and semantic coherence | -| `DATAMAN_ASSESSMENT` | Data Quality & Domain | Evaluates pre-training data quality using the DataMan methodology (14 standards, 15 domains). Assigns a score (0/1), domain type, quality status, and reason. | - -### Classification Prompts - -| Prompt Type | Metric | Description | -|-------------|--------|-------------| -| `CLASSIFY_TOPIC` | Topic Categorization | Classifies text into categories like language processing, writing, code, mathematics, role-play, or knowledge Q&A | -| `CLASSIFY_QR` | Image Classification | Identifies images as CAPTCHA, QR code, or normal images | - -### Image Assessment Prompts - -| Prompt Type | Metric | Description | -|-------------|--------|-------------| -| `IMAGE_RELEVANCE` | Image Relevance | Evaluates if an image matches reference image in terms of face count, feature details, and visual elements | +Most metrics are backed by academic sources to ensure objectivity and scientific rigor. ### Using LLM Assessment in Evaluation @@ -417,19 +375,6 @@ Example summary: } ``` - -# Research & Publications - -## Research Powered by Dingo -- **WanJuanSiLu**: [A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/pdf/2501.14506) - *Uses Dingo for comprehensive data quality assessment of multilingual web data* - -## Methodologies Implemented in Dingo -- **DataMan Methodology**: [DataMan: Data Manager for Pre-training Large Language Models](https://openreview.net/pdf?id=eNbA8Fqir4) - *Dingo implements the DataMan methodology for pre-training data quality assessment* -- **RedPajama-Data-v2**: [RedPajama-Data](https://github.com/togethercomputer/RedPajama-Data) - *Dingo implements parts of the RedPajama-Data-v2 methodology for web text quality assessment and filtering* - # Future Plans - [ ] Richer graphic and text evaluation indicators diff --git a/README_ja.md b/README_ja.md index bf111e98..57ee7e3a 100644 --- a/README_ja.md +++ b/README_ja.md @@ -177,61 +177,19 @@ https://github.com/user-attachments/assets/aca26f4c-3f2e-445e-9ef9-9331c4d7a37b # データ品質メトリクス -Dingoはデータ品質問題を7つの品質メトリクス次元に分類します。各次元は、ルールベース手法とLLMベースプロンプトの両方で評価できます： +Dingoはルールベースおよびプロンプトベースの評価メトリクスを通じて包括的なデータ品質評価を提供します。これらのメトリクスは、効果性、完全性、類似性、セキュリティなどの複数の品質次元をカバーしています。 -| 品質メトリクス | 説明 | ルール例 | LLMプロンプト例 | -|----------------|------|----------|-----------------| -| **COMPLETENESS** | データが不完全または欠落していないかをチェック | `RuleColonEnd`, `RuleContentNull` | テキストがコロンや省略記号で突然終わっているか、括弧が不一致か、重要な要素が欠落していないかを評価 | -| **EFFECTIVENESS** | データが意味があり適切にフォーマットされているかをチェック | `RuleAbnormalChar`, `RuleHtmlEntity`, `RuleSpecialCharacter` | 文字化けテキスト、スペースなしで結合された単語、適切な句読点のないテキストを検出 | -| **FLUENCY** | テキストが文法的に正しく自然に読めるかをチェック | `RuleAbnormalNumber`, `RuleNoPunc`, `RuleWordStuck` | 過度に長い単語、句読点のないテキスト断片、読み順が混乱したコンテンツを識別 | -| **RELEVANCE** | データ内の無関係なコンテンツを検出 | 異なる言語用の`RuleHeadWord`バリアント | 引用詳細、ヘッダー/フッター、エンティティマーカー、HTMLタグなどの無関係な情報を検査 | -| **SECURITY** | 機密情報や価値観の対立を識別 | `RuleIDCard`, `RuleUnsafeWords` | 個人情報、ギャンブル、ポルノ、政治問題に関連するコンテンツをチェック | -| **SIMILARITY** | 反復的または非常に類似したコンテンツを検出 | `RuleDocRepeat` | 連続した反復コンテンツや特殊文字の複数出現についてテキストを評価 | -| **UNDERSTANDABILITY** | データがどれだけ容易に解釈できるかを評価 | `RuleCapitalWords` | LaTeX数式とMarkdownが正しくフォーマットされ、適切なセグメンテーションと改行があることを確認 | +📊 **[完全なメトリクス文書を表示 →](docs/metrics.md)** -## LLM品質評価 +評価システムには以下が含まれます： +- **テキスト品質評価メトリクス**: DataMan手法と拡張された多次元評価を使用した事前学習データの品質評価 +- **SFTデータ評価メトリクス**: 教師ありファインチューニングデータの正直、有用、無害評価 +- **分類メトリクス**: トピック分類とコンテンツ分類 +- **マルチモーダル評価メトリクス**: 画像分類と関連性評価 +- **ルールベース品質メトリクス**: ヒューリスティックルールによる効果性と類似性検出を用いた自動品質チェック +- など -Dingoは`dingo/model/prompt`ディレクトリ内のプロンプトで定義された複数のLLMベース評価手法を提供します。これらのプロンプトは`prompt_register`デコレータを使用して登録され、品質評価のためにLLMモデルと組み合わせることができます： - -### テキスト品質評価プロンプト - -| プロンプトタイプ | メトリクス | 説明 | -|------------------|------------|------| -| `TEXT_QUALITY_V2`, `TEXT_QUALITY_V3` | 様々な品質次元 | 効果性、関連性、完全性、理解しやすさ、類似性、流暢性、セキュリティを含む包括的なテキスト品質評価 | -| `QUALITY_BAD_EFFECTIVENESS` | 効果性 | 文字化けテキストとアンチクローリングコンテンツを検出 | -| `QUALITY_BAD_SIMILARITY` | 類似性 | テキスト反復問題を識別 | -| `WORD_STICK` | 流暢性 | 適切なスペースなしで結合された単語をチェック | -| `CODE_LIST_ISSUE` | 完全性 | コードブロックとリストフォーマット問題を評価 | -| `UNREAD_ISSUE` | 効果性 | エンコーディング問題による読み取り不可能な文字を検出 | - -### 3H評価プロンプト（正直、有用、無害） - -| プロンプトタイプ | メトリクス | 説明 | -|------------------|------------|------| -| `QUALITY_HONEST` | 正直性 | 捏造や欺瞞なしに正確な情報を提供するかどうかを評価 | -| `QUALITY_HELPFUL` | 有用性 | 質問に直接答え、指示に適切に従うかどうかを評価 | -| `QUALITY_HARMLESS` | 無害性 | 有害なコンテンツ、差別的言語、危険な支援を避けるかどうかをチェック | - -### ドメイン固有評価プロンプト - -| プロンプトタイプ | メトリクス | 説明 | -|------------------|------------|------| -| `TEXT_QUALITY_KAOTI` | 試験問題品質 | 数式レンダリング、表フォーマット、段落構造、回答フォーマットに焦点を当てた試験問題品質の専門評価 | -| `Html_Abstract` | HTML抽出品質 | HTMLからMarkdownを抽出する異なる手法を比較し、完全性、フォーマット精度、意味的一貫性を評価 | -| `DATAMAN_ASSESSMENT` | データ品質とドメイン | DataMan手法（14基準、15ドメイン）を使用して事前学習データ品質を評価。スコア（0/1）、ドメインタイプ、品質状態、理由を割り当て | - -### 分類プロンプト - -| プロンプトタイプ | メトリクス | 説明 | -|------------------|------------|------| -| `CLASSIFY_TOPIC` | トピック分類 | 言語処理、執筆、コード、数学、ロールプレイ、知識Q&Aなどのカテゴリにテキストを分類 | -| `CLASSIFY_QR` | 画像分類 | 画像をCAPTCHA、QRコード、または通常の画像として識別 | - -### 画像評価プロンプト - -| プロンプトタイプ | メトリクス | 説明 | -|------------------|------------|------| -| `IMAGE_RELEVANCE` | 画像関連性 | 顔の数、特徴の詳細、視覚的要素の観点から、画像が参照画像と一致するかを評価 | +大部分のメトリクスは学術的なソースによって支持されており、客観性と科学的厳密性を保証しています。 ### 評価でのLLM評価の使用 @@ -284,10 +242,12 @@ input_data = { ## ルールベース・モデルベース評価 -- **組み込みルール**: 20以上の一般的なヒューリスティック評価ルール -- **LLM統合**: OpenAI、Kimi、ローカルモデル（例：Llama3） -- **カスタムルール**: 独自のルールとモデルで簡単に拡張 -- **セキュリティ評価**: Perspective API統合 +評価システムには以下が含まれます： +- **テキスト品質評価メトリクス**: DataMan手法と拡張された多次元評価を使用した事前学習データの品質評価 +- **SFTデータ評価メトリクス**: 教師ありファインチューニングデータの正直、有用、無害評価 +- **分類メトリクス**: トピック分類とコンテンツ分類 +- **マルチモーダル評価メトリクス**: 画像分類と関連性評価 +- **ルールベース品質メトリクス**: ヒューリスティックルールによる効果性と類似性検出を用いた自動品質チェック ## 柔軟な使用方法 @@ -415,19 +375,6 @@ result = executor.execute() } ``` - -# 研究・出版物 - -## Dingoを活用した研究 -- **WanJuanSiLu**: [A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/pdf/2501.14506) - *多言語ウェブデータの包括的なデータ品質評価にDingoを使用* - -## Dingoに実装された手法 -- **DataMan手法**: [DataMan: Data Manager for Pre-training Large Language Models](https://openreview.net/pdf?id=eNbA8Fqir4) - *Dingoは事前学習データ品質評価のためのDataMan手法を実装* -- **RedPajama-Data-v2**: [RedPajama-Data](https://github.com/togethercomputer/RedPajama-Data) - *Dingoはウェブテキストの品質評価とフィルタリングのためのRedPajama-Data-v2手法の一部を実装* - # 今後の計画 - [ ] より豊富なグラフィックとテキスト評価指標 diff --git a/README_zh-CN.md b/README_zh-CN.md index 203f526e..ea496d4c 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -176,61 +176,19 @@ https://github.com/user-attachments/assets/aca26f4c-3f2e-445e-9ef9-9331c4d7a37b # 数据质量指标 -Dingo将数据质量问题分为7个维度的质量指标。每个维度可以通过基于规则的方法和基于LLM的prompt进行评估： +Dingo通过基于规则和基于提示的评估指标提供全面的数据质量评估。这些指标涵盖多个质量维度，包括有效性、完整性、相似性、安全性等。 -| 质量指标 | 描述 | 规则示例 | LLM Prompt示例 | -|-------------------|-------------|---------------|---------------------| -| **完整性(COMPLETENESS)** | 检查数据是否不完整或缺失 | `RuleColonEnd`, `RuleContentNull` | 评估文本是否突然以冒号或省略号结束，是否有不匹配的括号，或缺少关键组件 | -| **有效性(EFFECTIVENESS)** | 检查数据是否有意义且格式正确 | `RuleAbnormalChar`, `RuleHtmlEntity`, `RuleSpecialCharacter` | 检测乱码文本、没有空格的粘连单词和缺少适当标点的文本 | -| **流畅性(FLUENCY)** | 检查文本是否语法正确且自然易读 | `RuleAbnormalNumber`, `RuleNoPunc`, `RuleWordStuck` | 识别过长的单词、无标点的文本片段或阅读顺序混乱的内容 | -| **相关性(RELEVANCE)** | 检测数据中的不相关内容 | 不同语言的`RuleHeadWord`变体 | 检查引用详情、页眉/页脚、实体标记、HTML标签等不相关信息 | -| **安全性(SECURITY)** | 识别敏感信息或价值冲突 | `RuleIDCard`, `RuleUnsafeWords` | 检查个人信息和与赌博、色情、政治问题相关的内容 | -| **相似性(SIMILARITY)** | 检测重复或高度相似的内容 | `RuleDocRepeat` | 评估文本中连续重复的内容或特殊字符的多次出现 | -| **可理解性(UNDERSTANDABILITY)** | 评估数据解释的容易程度 | `RuleCapitalWords` | 确保LaTeX公式和Markdown格式正确，具有适当的分段和换行 | +📊 **[查看完整指标文档 →](docs/metrics.md)** -## LLM质量评估 +我们的评估系统包括： +- **文本质量评估指标**：使用DataMan方法论和增强的多维评估进行预训练数据质量评估 +- **SFT数据评估指标**：针对监督微调数据的诚实、有帮助、无害评估 +- **分类指标**：主题分类和内容分类 +- **多模态评估指标**：图像分类和相关性评估 +- **基于规则的质量指标**：使用启发式规则进行效果性和相似性检测的自动化质量检查 +- 等等 -Dingo在`dingo/model/prompt`目录下提供了多种基于LLM的评估方法。这些prompt使用`prompt_register`装饰器注册，可以与LLM模型结合进行质量评估： - -### 文本质量评估Prompt - -| Prompt类型 | 指标 | 描述 | -|-------------|--------|-------------| -| `TEXT_QUALITY_V2`, `TEXT_QUALITY_V3` | 多种质量维度 | 全面的文本质量评估，涵盖有效性、相关性、完整性、可理解性、相似性、流畅性和安全性 | -| `QUALITY_BAD_EFFECTIVENESS` | 有效性 | 检测乱码文本和反爬虫内容 | -| `QUALITY_BAD_SIMILARITY` | 相似性 | 识别文本重复问题 | -| `WORD_STICK` | 流畅性 | 检查单词是否缺少适当间距而粘连在一起 | -| `CODE_LIST_ISSUE` | 完整性 | 评估代码块和列表格式问题 | -| `UNREAD_ISSUE` | 有效性 | 检测由编码问题导致的不可读字符 | - -### 3H评估Prompt (诚实、有帮助、无害) - -| Prompt类型 | 指标 | 描述 | -|-------------|--------|-------------| -| `QUALITY_HONEST` | 诚实性 | 评估回答是否提供准确信息，不含虚构或欺骗内容 | -| `QUALITY_HELPFUL` | 有帮助性 | 评估回答是否直接解决问题并适当遵循指令 | -| `QUALITY_HARMLESS` | 无害性 | 检查回答是否避免有害内容、歧视性语言和危险指导 | - -### 领域专用评估Prompt - -| Prompt类型 | 指标 | 描述 | -|-------------|--------|-------------| -| `TEXT_QUALITY_KAOTI` | 考题质量 | 专门评估考试题目的质量，关注公式渲染、表格格式、段落结构和答案格式 | -| `Html_Abstract` | HTML提取质量 | 比较从HTML提取Markdown的不同方法，评估完整性、格式准确性和语义连贯性 | -| `DATAMAN_ASSESSMENT` | 数据质量与领域 | 使用DataMan方法论（14个标准，15个领域）评估预训练数据质量。分配分数（0/1）、领域类型、质量状态和原因。 | - -### 分类Prompt - -| Prompt类型 | 指标 | 描述 | -|-------------|--------|-------------| -| `CLASSIFY_TOPIC` | 主题分类 | 将文本分类为语言处理、写作、代码、数学、角色扮演或知识问答等类别 | -| `CLASSIFY_QR` | 图像分类 | 识别图像为验证码、二维码或普通图像 | - -### 图像评估Prompt - -| Prompt类型 | 指标 | 描述 | -|-------------|--------|-------------| -| `IMAGE_RELEVANCE` | 图像相关性 | 评估图像是否在面部数量、特征细节和视觉元素方面与参考图像匹配 | +大部分指标都由学术来源支持，以确保客观性和科学严谨性。 ### 在评估中使用LLM评估 @@ -254,8 +212,6 @@ input_data = { 您可以自定义这些prompt，以关注特定的质量维度或适应特定的领域需求。当与适当的LLM模型结合时，这些prompt能够在多个维度上对数据质量进行全面评估。 -每条规则都针对文本质量的特定方面进行检查，并映射到这些指标之一。运行评估时，Dingo将提供每个维度的分数并识别触发了哪些规则。 - # 规则组 Dingo为不同类型的数据集提供预配置的规则组： @@ -416,18 +372,6 @@ result = executor.execute() } ``` -# 研究与学术成果 - -## Dingo驱动的研究 -- **WanJuanSiLu**: [A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/pdf/2501.14506) - *使用Dingo对多语言网页数据进行全面的数据质量评估* - -## Dingo实现的方法论 -- **DataMan方法论**: [DataMan: Data Manager for Pre-training Large Language Models](https://openreview.net/pdf?id=eNbA8Fqir4) - *Dingo实现了DataMan方法论用于预训练数据质量评估* -- **RedPajama-Data-v2**: [RedPajama-Data](https://github.com/togethercomputer/RedPajama-Data) - *Dingo实现了部分RedPajama-Data-v2方法论用于网页文本质量评估和过滤* - # 未来计划 - [ ] 更丰富的图文评测指标 diff --git a/app_gradio/app.py b/app_gradio/app.py index ca1d94c2..4426fd4b 100644 --- a/app_gradio/app.py +++ b/app_gradio/app.py @@ -4,6 +4,7 @@ from pathlib import Path import gradio as gr + from dingo.exec import Executor from dingo.io import InputArgs diff --git a/dingo/config/config.py b/dingo/config/config.py index 623b488a..2b4638b6 100644 --- a/dingo/config/config.py +++ b/dingo/config/config.py @@ -1,9 +1,10 @@ import json from typing import Dict, List, Optional -from dingo.utils import log from pydantic import BaseModel +from dingo.utils import log + class DynamicRuleConfig(BaseModel): threshold: Optional[float] = None diff --git a/dingo/data/converter/img_utils.py b/dingo/data/converter/img_utils.py index b3289144..b4d141d8 100644 --- a/dingo/data/converter/img_utils.py +++ b/dingo/data/converter/img_utils.py @@ -6,9 +6,10 @@ from botocore.exceptions import ClientError from botocore.response import StreamingBody +from PIL import Image + from dingo.data.datasource import S3DataSource from dingo.io import InputArgs -from PIL import Image def try_close(obj): diff --git a/dingo/data/dataset/huggingface.py b/dingo/data/dataset/huggingface.py index 329d8021..af68382a 100644 --- a/dingo/data/dataset/huggingface.py +++ b/dingo/data/dataset/huggingface.py @@ -2,6 +2,7 @@ from typing import Any, Dict, Generator, Mapping, Optional, Sequence, Union import datasets + from dingo.data.dataset.base import Dataset from dingo.data.datasource import DataSource from dingo.data.datasource.huggingface import HuggingFaceSource diff --git a/dingo/data/datasource/huggingface.py b/dingo/data/datasource/huggingface.py index 612230df..dbd66cd2 100644 --- a/dingo/data/datasource/huggingface.py +++ b/dingo/data/datasource/huggingface.py @@ -1,6 +1,7 @@ from typing import Any, Dict, Mapping, Optional, Sequence, Union import datasets + from dingo.data.datasource.base import DataSource from dingo.io import InputArgs diff --git a/dingo/data/datasource/s3.py b/dingo/data/datasource/s3.py index 36dfcf06..6879bb5d 100644 --- a/dingo/data/datasource/s3.py +++ b/dingo/data/datasource/s3.py @@ -3,6 +3,7 @@ import boto3 import boto3.s3 from botocore.config import Config + from dingo.data.datasource.base import DataSource from dingo.io import InputArgs diff --git a/dingo/data/utils/digit.py b/dingo/data/utils/digit.py index d37b830d..0cf3637a 100644 --- a/dingo/data/utils/digit.py +++ b/dingo/data/utils/digit.py @@ -18,9 +18,10 @@ import logging from typing import Any, List -from dingo.data.utils import insecure_hash from packaging.version import Version +from dingo.data.utils import insecure_hash + logger = logging.getLogger(__name__) logger.setLevel("ERROR") MAX_ROWS = 10000 diff --git a/dingo/exec/local.py b/dingo/exec/local.py index d55bb9c3..437da125 100644 --- a/dingo/exec/local.py +++ b/dingo/exec/local.py @@ -7,6 +7,8 @@ import uuid from typing import Generator, List, Optional +from tqdm import tqdm + from dingo.config import GlobalConfig from dingo.data import Dataset, DataSource, dataset_map, datasource_map from dingo.exec.base import ExecProto, Executor @@ -17,7 +19,6 @@ from dingo.model.prompt.base import BasePrompt from dingo.model.rule.base import BaseRule from dingo.utils import log -from tqdm import tqdm @Executor.register("local") diff --git a/dingo/exec/spark.py b/dingo/exec/spark.py index f8ac5a15..d5445f0f 100644 --- a/dingo/exec/spark.py +++ b/dingo/exec/spark.py @@ -3,6 +3,10 @@ import uuid from typing import Any, Dict, List, Optional +from pyspark import SparkConf +from pyspark.rdd import RDD +from pyspark.sql import SparkSession + from dingo.config import GlobalConfig from dingo.exec.base import ExecProto, Executor from dingo.io import Data, InputArgs, ResultInfo, SummaryModel @@ -11,9 +15,6 @@ from dingo.model.modelres import ModelRes from dingo.model.prompt.base import BasePrompt from dingo.model.rule.base import BaseRule -from pyspark import SparkConf -from pyspark.rdd import RDD -from pyspark.sql import SparkSession @Executor.register("spark") diff --git a/dingo/model/llm/base_lmdeploy_apiclient.py b/dingo/model/llm/base_lmdeploy_apiclient.py index 6bb2b62e..fa37dc8a 100644 --- a/dingo/model/llm/base_lmdeploy_apiclient.py +++ b/dingo/model/llm/base_lmdeploy_apiclient.py @@ -2,6 +2,8 @@ import time from typing import List +from pydantic import ValidationError + from dingo.config.config import DynamicLLMConfig from dingo.io import Data from dingo.model.llm.base import BaseLLM @@ -10,7 +12,6 @@ from dingo.model.response.response_class import ResponseScoreReason from dingo.utils import log from dingo.utils.exception import ConvertJsonError, ExceedMaxTokens -from pydantic import ValidationError class BaseLmdeployApiClient(BaseLLM): diff --git a/dingo/model/llm/base_openai.py b/dingo/model/llm/base_openai.py index 52d50864..6f8cbc7d 100644 --- a/dingo/model/llm/base_openai.py +++ b/dingo/model/llm/base_openai.py @@ -2,6 +2,8 @@ import time from typing import Dict, List +from pydantic import ValidationError + from dingo.config.config import DynamicLLMConfig from dingo.io import Data from dingo.model.llm.base import BaseLLM @@ -10,7 +12,6 @@ from dingo.model.response.response_class import ResponseScoreReason from dingo.utils import log from dingo.utils.exception import ConvertJsonError, ExceedMaxTokens -from pydantic import ValidationError class BaseOpenAI(BaseLLM): diff --git a/dingo/model/model.py b/dingo/model/model.py index d937ffa3..49297fb9 100644 --- a/dingo/model/model.py +++ b/dingo/model/model.py @@ -3,12 +3,13 @@ import os from typing import Callable, Dict, List, Optional +from pydantic import BaseModel + from dingo.config import GlobalConfig from dingo.model.llm.base import BaseLLM from dingo.model.prompt.base import BasePrompt from dingo.model.rule.base import BaseRule from dingo.utils import log -from pydantic import BaseModel class BaseEvalModel(BaseModel): diff --git a/dingo/model/prompt/prompt_classify_qr.py b/dingo/model/prompt/prompt_classify_qr.py index ba9bb865..6ccfee1b 100644 --- a/dingo/model/prompt/prompt_classify_qr.py +++ b/dingo/model/prompt/prompt_classify_qr.py @@ -4,6 +4,15 @@ @Model.prompt_register("CLASSIFY_QR", []) class PromptClassifyQR(BasePrompt): + + # Metadata for documentation generation + _metric_info = { + "category": "Multimodality Assessment Metrics", + "metric_name": "Image Classification", + "description": "Identifies images as CAPTCHA, QR code, or normal images", + "evaluation_results": "" + } + content = """ 'Classify the image into one of the following categories: "CAPTCHA", "QR code", or "Normal image". ' 'Return the type as the image category (CAPTCHA or QR code or Normal image) and the reason as the specific type of CAPTCHA or QR code. ' diff --git a/dingo/model/prompt/prompt_classify_topic.py b/dingo/model/prompt/prompt_classify_topic.py index a89bb411..ea6001cb 100644 --- a/dingo/model/prompt/prompt_classify_topic.py +++ b/dingo/model/prompt/prompt_classify_topic.py @@ -4,6 +4,19 @@ @Model.prompt_register("CLASSIFY_TOPIC", []) class PromptClassifyTopic(BasePrompt): + + # Metadata for documentation generation + _metric_info = { + "category": "Classification Metrics", + "metric_name": "Topic Categorization", + "description": "Classifies text into categories like language processing, writing, code, mathematics, role-play, or knowledge Q&A. Based on BERTopic and INSTAG methodologies", + "paper_title": "BERTopic & INSTAG", + "paper_url": "https://maartengr.github.io/BERTopic/index.html#quick-start, https://arxiv.org/pdf/2308.07074", + "paper_authors": "Grootendorst, 2022; Wei et al., 2023", + "evaluation_results": "docs/eval/prompt/text_data_classified_by_topic.md", + "validation_dataset": "AlignBench (https://github.com/THUDM/AlignBench)" + } + content = """ Assume you are a topic classifier, and your task is to categorize user-provided instructions. There are six options in the list provided. You are required to select one category from the following list: ["Language Understanding and Processing", "Writing Ability", "Code", "Mathematics & Reasoning", "Task-oriented Role Play", "Knowledge-based Question and Answering"]. diff --git a/dingo/model/prompt/prompt_dataman_assessment.py b/dingo/model/prompt/prompt_dataman_assessment.py index 016a6f4b..930dde77 100644 --- a/dingo/model/prompt/prompt_dataman_assessment.py +++ b/dingo/model/prompt/prompt_dataman_assessment.py @@ -84,4 +84,16 @@ @Model.prompt_register("DATAMAN_ASSESSMENT", []) class PromptDataManAssessment(BasePrompt): + + # Metadata for documentation generation + _metric_info = { + "category": "Text Quality Assessment Metrics", + "metric_name": "DATAMAN", + "description": "Evaluates pre-training data quality using the DataMan methodology (14 standards, 15 domains). Assigns a score (0/1), domain type, quality status, and reason.", + "paper_title": "DataMan: Data Manager for Pre-training Large Language Models", + "paper_url": "https://arxiv.org/abs/2502.19363", + "paper_authors": "Peng et al., 2025", + "evaluation_results": "" + } + content = ROLE + DATAMAN_ASSESSMENT diff --git a/dingo/model/prompt/prompt_image_relevant.py b/dingo/model/prompt/prompt_image_relevant.py index b1a11818..ff37566c 100644 --- a/dingo/model/prompt/prompt_image_relevant.py +++ b/dingo/model/prompt/prompt_image_relevant.py @@ -4,6 +4,15 @@ @Model.prompt_register("IMAGE_RELEVANT", []) class PromptImageRelevant(BasePrompt): + + # Metadata for documentation generation + _metric_info = { + "category": "Multimodality Assessment Metrics", + "metric_name": "Image Relevance", + "description": "Evaluates if an image matches reference image in terms of face count, feature details, and visual elements", + "evaluation_results": "" + } + content = """ 作为一款专业的图片检测AI工具，请结合第一张图评估第二张图片是否符合标准。请先分析第一张图片，包括背景信息、人脸数量、以及每个人物的脸部和手部特征。 然后根据以下标准对第二张图片进行评分：\n diff --git a/dingo/model/prompt/prompt_politics.py b/dingo/model/prompt/prompt_politics.py index 27a5b1b0..b69e2321 100644 --- a/dingo/model/prompt/prompt_politics.py +++ b/dingo/model/prompt/prompt_politics.py @@ -2,8 +2,20 @@ from dingo.model.prompt.base import BasePrompt -@Model.prompt_register("TEXT_POLITICS", []) +@Model.prompt_register("QUALITY_BAD_SECURITY", []) class PromptPolitics(BasePrompt): + + # Metadata for documentation generation + _metric_info = { + "category": "Text Quality Assessment Metrics", + "metric_name": "Politics", + "description": "Evaluates whether the text contains politics-related content", + "paper_title": "", + "paper_url": "", + "paper_authors": "", + "evaluation_results": "" + } + content = """ 你是一名内容安全审查员，可以用英语流利沟通，会站在中国人的角度思考问题。我将提供一段文字，需要你判断其内容是否属于政治敏感内容，并且立场是否为负面。政治敏感内容包括以下五个方面，以下内容解释了对应方面的负面立场： diff --git a/dingo/model/prompt/prompt_text_3h.py b/dingo/model/prompt/prompt_text_3h.py index 8d92c89e..6c7ccfc4 100644 --- a/dingo/model/prompt/prompt_text_3h.py +++ b/dingo/model/prompt/prompt_text_3h.py @@ -4,6 +4,18 @@ @Model.prompt_register("QUALITY_HELPFUL", []) class PromptTextHelpful(BasePrompt): + + # Metadata for documentation generation + _metric_info = { + "category": "SFT Data Assessment Metrics", + "metric_name": "Helpfulness", + "description": "Assesses if responses address questions directly and follow instructions appropriately", + "paper_title": "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback", + "paper_url": "https://arxiv.org/pdf/2204.05862", + "paper_authors": "Bai et al., 2022", + "evaluation_results": "docs/eval/prompt/qa_data_evaluated_by_3h.md" + } + content = """ Please act as an impartial judge and evaluate the quality of the responses provided by the AI assistant to the user question displayed below. Determine whether the response is helpful. @@ -30,6 +42,18 @@ class PromptTextHelpful(BasePrompt): @Model.prompt_register("QUALITY_HARMLESS", []) class PromptTextHarmless(BasePrompt): + + # Metadata for documentation generation + _metric_info = { + "category": "SFT Data Assessment Metrics", + "metric_name": "Harmlessness", + "description": "Checks if responses avoid harmful content, discriminatory language, and dangerous assistance", + "paper_title": "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback", + "paper_url": "https://arxiv.org/pdf/2204.05862", + "paper_authors": "Bai et al., 2022", + "evaluation_results": "docs/eval/prompt/qa_data_evaluated_by_3h.md" + } + content = """ Please act as an impartial judge and evaluate the quality of the responses provided by the AI assistant to the user question displayed below. Determine whether the response is harmless. @@ -55,6 +79,18 @@ class PromptTextHarmless(BasePrompt): @Model.prompt_register("QUALITY_HONEST", []) class PromptTextHonest(BasePrompt): + + # Metadata for documentation generation + _metric_info = { + "category": "SFT Data Assessment Metrics", + "metric_name": "Honesty", + "description": "Evaluates if responses provide accurate information without fabrication or deception", + "paper_title": "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback", + "paper_url": "https://arxiv.org/pdf/2204.05862", + "paper_authors": "Bai et al., 2022", + "evaluation_results": "docs/eval/prompt/qa_data_evaluated_by_3h.md" + } + content = """ Please act as an impartial judge and evaluate the quality of the responses provided by the AI assistant to the user question displayed below. Determine whether the response is honest. diff --git a/dingo/model/prompt/prompt_text_quality.py b/dingo/model/prompt/prompt_text_quality.py index 5421d296..31f24291 100644 --- a/dingo/model/prompt/prompt_text_quality.py +++ b/dingo/model/prompt/prompt_text_quality.py @@ -80,6 +80,17 @@ class PromptTextQualityV3(BasePrompt): @Model.prompt_register("TEXT_QUALITY_V4", []) class PromptTextQualityV4(BasePrompt): + + # Metadata for documentation generation + _metric_info = { + "category": "Text Quality Assessment Metrics", + "metric_name": "Pretrain Text Quality Assessment V4", + "description": "Enhanced text quality evaluation covering completeness (formulas, tables, code), effectiveness (garbled text, spacing), similarity (duplicates), and security (politics, prohibited content)", + "paper_title": "WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages", + "paper_url": "https://arxiv.org/abs/2501.14506", + "paper_authors": "Yu et al., 2025", + "evaluation_results": "docs/eval/prompt/redpajama_data_evaluated_by_prompt.md" + } content = """ # Role You are an expert in language model evaluation. diff --git a/dingo/model/prompt/prompt_text_quality_multilan.py b/dingo/model/prompt/prompt_text_quality_multilan.py index 43f20b2d..b1fbb96f 100644 --- a/dingo/model/prompt/prompt_text_quality_multilan.py +++ b/dingo/model/prompt/prompt_text_quality_multilan.py @@ -1,7 +1,6 @@ from dingo.model.model import Model from dingo.model.prompt.base import BasePrompt -from dingo.model.prompt.prompt_text_quality_v2 import \ - TEXT_QUALITY_WITHOUT_ROLE_V2 +from dingo.model.prompt.prompt_text_quality_v2 import TEXT_QUALITY_WITHOUT_ROLE_V2 AR_ROLE = """ ### Role diff --git a/dingo/model/rule/rule_common.py b/dingo/model/rule/rule_common.py index 4aa29711..75758bf6 100644 --- a/dingo/model/rule/rule_common.py +++ b/dingo/model/rule/rule_common.py @@ -13,6 +13,18 @@ class RuleAbnormalChar(BaseRule): # consist of [RuleSpecialCharacter, RuleInvisibleChar] + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "EFFECTIVENESS", + "metric_name": "Abnormal Character Detection", + "description": "Detects garbled text and anti-crawling characters by combining special character and invisible character detection", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + @classmethod def eval(cls, input_data: Data) -> ModelRes: res = ModelRes() @@ -30,6 +42,17 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleAbnormalHtml(BaseRule): # consist of [RuleHtmlEntity, RuleHtmlTag] + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "EFFECTIVENESS", + "metric_name": "Abnormal HTML Detection", + "description": "Detects abnormal HTML content by combining HTML entity and tag detection", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + @classmethod def eval(cls, input_data: Data) -> ModelRes: res = ModelRes() @@ -47,6 +70,17 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleAbnormalNumber(BaseRule): """check pdf content abnormal book page or index number.""" + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "FLUENCY", + "metric_name": "Abnormal Number Detection", + "description": "Checks PDF content for abnormal book page or index numbers that disrupt text flow", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig(pattern=r"\n{4}\d+\n{4}") @classmethod @@ -66,6 +100,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleAlphaWords(BaseRule): """check whether the ratio of words that contain at least one alphabetic character > 0.6""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "EFFECTIVENESS", + "metric_name": "Alphabetic Word Ratio", + "description": "Checks whether the ratio of words containing at least one alphabetic character is above threshold", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig(threshold=0.6) @classmethod @@ -98,6 +144,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleCapitalWords(BaseRule): """check whether capital words ratio > 0.2""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "UNDERSTANDABILITY", + "metric_name": "Capital Words Ratio", + "description": "Checks whether the ratio of capital words is above threshold, indicating poor readability", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig(threshold=0.2) @classmethod @@ -124,6 +182,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleCharNumber(BaseRule): """check whether the number of char > 100""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "EFFECTIVENESS", + "metric_name": "Character Count", + "description": "Checks whether the number of characters is above minimum threshold", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig(threshold=100) @classmethod @@ -147,6 +217,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleCharSplit(BaseRule): """check pdf content char split.""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "FLUENCY", + "metric_name": "Character Split Detection", + "description": "Checks PDF content for abnormal character splitting that disrupts readability", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig( pattern=r"(?:(?:[a-zA-Z]\s){5}[a-zA-Z])", threshold=3 ) @@ -172,6 +254,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleColonEnd(BaseRule): """check whether the last char is ':'""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "EFFECTIVENESS", + "metric_name": "Colon End Detection", + "description": "Checks if text abruptly ends with a colon, indicating incomplete content", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig() @classmethod @@ -212,6 +306,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleContentNull(BaseRule): """check whether content is null""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "EFFECTIVENESS", + "metric_name": "Content Null Detection", + "description": "Checks whether content is empty or null", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig() @classmethod @@ -230,6 +336,20 @@ def eval(cls, input_data: Data) -> ModelRes: "QUALITY_BAD_EFFECTIVENESS", ["text_base_all", "qa_standard_v1", "pdf"] ) class RuleContentShort(BaseRule): + """check whether content is too short""" + + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "EFFECTIVENESS", + "metric_name": "Content Short Detection", + "description": "Checks whether content is too short to be meaningful", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig(threshold=20) @classmethod @@ -260,6 +380,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleContentShortMultiLan(BaseRule): """check whether content is too short.""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "EFFECTIVENESS", + "metric_name": "Multi-Language Content Short Detection", + "description": "Checks whether multi-language content is too short to be meaningful", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig(threshold=20) @classmethod @@ -282,6 +414,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleCurlyBracket(BaseRule): """check whether the ratio of the number of {,} and the number of characters < 0.025""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "UNDERSTANDABILITY", + "metric_name": "Curly Bracket Ratio", + "description": "Checks whether the ratio of curly brackets to total characters is below threshold", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig(threshold=0.025) @classmethod @@ -326,12 +470,23 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleDocRepeat(BaseRule): """check whether content repeats""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "SIMILARITY", + "metric_name": "Document Repetition Detection", + "description": "Evaluates text for consecutive repeated content and multiple occurrences of special characters", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig(threshold=80) @classmethod def eval(cls, input_data: Data) -> ModelRes: - from dingo.model.rule.utils.util import \ - base_rps_frac_chars_in_dupe_ngrams + from dingo.model.rule.utils.util import base_rps_frac_chars_in_dupe_ngrams res = ModelRes() repeat_score = base_rps_frac_chars_in_dupe_ngrams(6, input_data.content) @@ -349,6 +504,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleEnterAndSpace(BaseRule): # consist of [RuleEnterMore, RuleEnterRatioMore, RuleSpaceMore] + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "EFFECTIVENESS", + "metric_name": "Enter and Space Detection", + "description": "Composite rule checking for excessive carriage returns and spaces", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + @classmethod def eval(cls, input_data: Data) -> ModelRes: res = ModelRes() @@ -381,6 +548,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleEnterMore(BaseRule): """check whether content has 8 consecutive carriage returns.""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "EFFECTIVENESS", + "metric_name": "Excessive Carriage Returns", + "description": "Checks whether content has 8 consecutive carriage returns", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig(key_list=[r"\n{8,}", r"\r\n{8,}"]) @classmethod @@ -418,6 +597,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleEnterRatioMore(BaseRule): """check whether the number of enter / the number of content > 25%""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "EFFECTIVENESS", + "metric_name": "Enter Ratio Detection", + "description": "Checks whether the ratio of enter characters to total content is above 25%", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig() @classmethod @@ -440,6 +631,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleHeadWordAr(BaseRule): """check whether ar content contains irrelevance tail source info.""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "RELEVANCE", + "metric_name": "Arabic Head Word Detection", + "description": "Checks whether Arabic content contains irrelevant tail source information", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig() @classmethod @@ -634,6 +837,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleHtmlEntity(BaseRule): """check whether content has html entity""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "EFFECTIVENESS", + "metric_name": "HTML Entity Detection", + "description": "Checks whether content contains HTML entities indicating web scraping artifacts", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig( key_list=[ "nbsp", @@ -707,6 +922,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleHtmlTag(BaseRule): """check whether content has image links or html tags.""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "EFFECTIVENESS", + "metric_name": "HTML Tag Detection", + "description": "Checks whether content contains HTML tags or image links indicating web scraping artifacts", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig( key_list=["", "

", ""] ) @@ -769,6 +996,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleInvisibleChar(BaseRule): """check whether content has invisible chars.""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "EFFECTIVENESS", + "metric_name": "Invisible Character Detection", + "description": "Checks whether content contains invisible characters that may cause display issues", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig( pattern=r"[\u2000-\u200F\u202F\u205F\u3000\uFEFF\u00A0\u2060-\u206F\uFEFF\xa0]" ) @@ -813,6 +1052,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleLineEndWithEllipsis(BaseRule): """check whether the ratio of line ends with ellipsis < 0.3""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "COMPLETENESS", + "metric_name": "Line End With Ellipsis", + "description": "Checks whether the ratio of lines ending with ellipsis is below threshold", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig(threshold=0.3, key_list=["...", "…"]) @classmethod @@ -847,6 +1098,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleLineEndWithTerminal(BaseRule): """check whether the ratio of line ends with terminal punctuation mark > 0.6""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "COMPLETENESS", + "metric_name": "Line End With Terminal", + "description": "Checks whether the ratio of lines ending with terminal punctuation is above threshold", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig( threshold=0.6, key_list=[".", "!", "?", '"', '"'] ) @@ -888,6 +1151,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleLineStartWithBulletpoint(BaseRule): """check whether the ratio of line starts with bullet points < 0.9""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "UNDERSTANDABILITY", + "metric_name": "Line Start With Bulletpoint", + "description": "Checks whether the ratio of lines starting with bullet points is below threshold", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig( threshold=0.9, key_list=[ @@ -936,12 +1211,23 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleLineJavascriptCount(BaseRule): """check whether line with the word Javascript.""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "EFFECTIVENESS", + "metric_name": "Javascript Count Detection", + "description": "Checks whether content contains excessive Javascript-related text", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig(threshold=3) @classmethod def eval(cls, input_data: Data) -> ModelRes: - from dingo.model.rule.utils.util import (TextSlice, normalize, - split_paragraphs) + from dingo.model.rule.utils.util import TextSlice, normalize, split_paragraphs res = ModelRes() raw_content = input_data.content @@ -968,6 +1254,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleLoremIpsum(BaseRule): """check whether the ratio of lorem ipsum < 3e-08""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "EFFECTIVENESS", + "metric_name": "Lorem Ipsum Detection", + "description": "Checks whether content contains lorem ipsum placeholder text", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig(threshold=3e-08) @classmethod @@ -995,6 +1293,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleMeanWordLength(BaseRule): """check whether the mean length of word in [3, 10]""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "EFFECTIVENESS", + "metric_name": "Mean Word Length", + "description": "Checks whether the mean length of words is within acceptable range", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig(key_list=["3", "10"]) @classmethod @@ -1045,6 +1355,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleNoPunc(BaseRule): """check whether paragraph has no punctuation.""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "FLUENCY", + "metric_name": "No Punctuation Detection", + "description": "Checks whether paragraphs lack punctuation marks, indicating poor text quality", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig(threshold=112) @classmethod @@ -1095,6 +1417,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleSentenceNumber(BaseRule): """check whether the number of sentence in [3, 7500]""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "COMPLETENESS", + "metric_name": "Sentence Number", + "description": "Checks whether the number of sentences is within acceptable range", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig(key_list=["3", "7500"]) @classmethod @@ -1133,6 +1467,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleSpaceMore(BaseRule): """check whether content has 500 spaces.""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "EFFECTIVENESS", + "metric_name": "Excessive Space Detection", + "description": "Checks whether content contains excessive consecutive spaces", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig(pattern=" {500,}") @classmethod @@ -1172,6 +1518,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleSpecialCharacter(BaseRule): """check whether content has special characters.""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "EFFECTIVENESS", + "metric_name": "Special Character Detection", + "description": "Checks if data is meaningful and properly formatted by detecting excessive special characters", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig( key_list=[ r"u200e", @@ -1208,13 +1566,26 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleStopWord(BaseRule): """check whether the ratio of stop word > 0.06""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "EFFECTIVENESS", + "metric_name": "Stop Word Ratio", + "description": "Checks whether the ratio of stop words is above threshold", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig(threshold=0.06) @classmethod def eval(cls, input_data: Data) -> ModelRes: - from dingo.model.rule.utils.util import get_stop_words from nltk.tokenize import WordPunctTokenizer + from dingo.model.rule.utils.util import get_stop_words + res = ModelRes() raw_content = input_data.content raw_words = list(WordPunctTokenizer().tokenize(raw_content)) @@ -1238,6 +1609,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleSymbolWordRatio(BaseRule): """check whether the ratio of symbol and word is > 0.4""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "UNDERSTANDABILITY", + "metric_name": "Symbol Word Ratio", + "description": "Checks whether the ratio of symbols to words is above threshold", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig(threshold=0.4, key_list=["#", "...", "…"]) @classmethod @@ -1269,6 +1652,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleUniqueWords(BaseRule): """check whether the ratio of unique words > 0.1""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "UNDERSTANDABILITY", + "metric_name": "Unique Words Ratio", + "description": "Checks whether the ratio of unique words is above threshold", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig(threshold=0.1) @classmethod @@ -1304,6 +1699,7 @@ class RuleUnsafeWords(BaseRule): @classmethod def eval(cls, input_data: Data) -> ModelRes: import ahocorasick + from dingo.model.rule.utils.util import get_unsafe_words res = ModelRes() @@ -1348,6 +1744,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleOnlyUrl(BaseRule): """check whether content is only an url link.""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "EFFECTIVENESS", + "metric_name": "URL-Only Content Detection", + "description": "Checks whether content consists only of URLs without meaningful text", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig( pattern=r"http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+" ) @@ -1390,6 +1798,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleWordNumber(BaseRule): """check whether the number of word in [20, 100000]""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "COMPLETENESS", + "metric_name": "Word Number", + "description": "Checks whether the number of words is within acceptable range", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig(key_list=["20", "100000"]) @classmethod @@ -1416,6 +1836,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleWordSplit(BaseRule): """check pdf word abnormal split such as "ca- se".""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "FLUENCY", + "metric_name": "Word Split Detection", + "description": "Checks for abnormal word splits in PDF content that disrupt readability", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig(pattern=r"[A-Za-z]+-\s*$") @classmethod @@ -1449,6 +1881,18 @@ def eval(cls, input_data: Data) -> ModelRes: class RuleWordStuck(BaseRule): """check whether words are stuck.""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "FLUENCY", + "metric_name": "Word Stuck Detection", + "description": "Checks whether words are stuck together without proper spacing", + "paper_title": "RedPajama: an Open Dataset for Training Large Language Models", + "paper_url": "https://github.com/togethercomputer/RedPajama-Data", + "paper_authors": "Together Computer, 2023", + "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" + } + dynamic_config = DynamicRuleConfig( key_list=[ r"https?://[^\s]+|www.(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+", @@ -1462,6 +1906,7 @@ class RuleWordStuck(BaseRule): @classmethod def eval(cls, input_data: Data) -> ModelRes: import wordninja + from dingo.model.rule.utils.detect_lang import decide_language_by_str from dingo.model.rule.utils.util import is_sha256 diff --git a/dingo/model/rule/rule_image.py b/dingo/model/rule/rule_image.py index 21c36414..6f541cc6 100644 --- a/dingo/model/rule/rule_image.py +++ b/dingo/model/rule/rule_image.py @@ -1,18 +1,31 @@ import os import numpy as np +from PIL import Image + from dingo.config.config import DynamicRuleConfig from dingo.io import Data from dingo.model.model import Model from dingo.model.modelres import ModelRes from dingo.model.rule.base import BaseRule -from PIL import Image -@Model.rule_register("QUALITY_BAD_EFFECTIVENESS", ["img"]) +@Model.rule_register("QUALITY_BAD_IMG_EFFECTIVENESS", ["img"]) class RuleImageValid(BaseRule): """check whether image is not all white or black""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based IMG Quality Metrics", + "quality_dimension": "IMG_EFFECTIVENESS", + "metric_name": "Image Valid Detection", + "description": "Checks whether image is not all white or black, ensuring visual content validity", + "paper_title": "", + "paper_url": "", + "paper_authors": "", + "evaluation_results": "" + } + dynamic_config = DynamicRuleConfig() @classmethod @@ -32,10 +45,22 @@ def eval(cls, input_data: Data) -> ModelRes: return res -@Model.rule_register("QUALITY_BAD_EFFECTIVENESS", ["img"]) +@Model.rule_register("QUALITY_BAD_IMG_EFFECTIVENESS", ["img"]) class RuleImageSizeValid(BaseRule): """check whether image ratio of width to height is valid""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based IMG Quality Metrics", + "quality_dimension": "IMG_EFFECTIVENESS", + "metric_name": "Image Size Valid Detection", + "description": "Checks whether image ratio of width to height is within valid range", + "paper_title": "", + "paper_url": "", + "paper_authors": "", + "evaluation_results": "" + } + dynamic_config = DynamicRuleConfig() @classmethod @@ -58,10 +83,22 @@ def eval(cls, input_data: Data) -> ModelRes: return res -@Model.rule_register("QUALITY_BAD_EFFECTIVENESS", ["img"]) +@Model.rule_register("QUALITY_BAD_IMG_EFFECTIVENESS", ["img"]) class RuleImageQuality(BaseRule): """check whether image quality is good.""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based IMG Quality Metrics", + "quality_dimension": "IMG_EFFECTIVENESS", + "metric_name": "Image Quality Assessment", + "description": "Evaluates image quality using NIMA (Neural Image Assessment) metrics", + "paper_title": "NIMA: Neural Image Assessment", + "paper_url": "https://arxiv.org/abs/1709.05424", + "paper_authors": "Talebi & Milanfar, 2018", + "evaluation_results": "" + } + dynamic_config = DynamicRuleConfig(threshold=5.5) @classmethod @@ -88,10 +125,22 @@ def eval(cls, input_data: Data) -> ModelRes: return res -@Model.rule_register("QUALITY_BAD_EFFECTIVENESS", []) +@Model.rule_register("QUALITY_BAD_IMG_SIMILARITY", []) class RuleImageRepeat(BaseRule): """Check for duplicate images using PHash and CNN methods.""" + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based IMG Quality Metrics", + "quality_dimension": "IMG_SIMILARITY", + "metric_name": "Image Duplicate Detection", + "description": "Detects duplicate images using PHash and CNN methods to ensure data diversity", + "paper_title": "ImageNet Classification with Deep Convolutional Neural Networks", + "paper_url": "https://papers.nips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks.pdf", + "paper_authors": "Krizhevsky et al., 2012", + "evaluation_results": "" + } + dynamic_config = DynamicRuleConfig() @classmethod @@ -132,8 +181,22 @@ def eval(cls, input_data: Data) -> ModelRes: return res -@Model.rule_register("QUALITY_BAD_EFFECTIVENESS", []) +@Model.rule_register("QUALITY_BAD_IMG_RELEVANCE", []) class RuleImageTextSimilarity(BaseRule): + """Check similarity between image and text content""" + + # Metadata for documentation generation + _metric_info = { + "category": "Rule-Based IMG Quality Metrics", + "quality_dimension": "IMG_RELEVANCE", + "metric_name": "Image-Text Similarity", + "description": "Evaluates semantic similarity between image and text content using CLIP model", + "paper_title": "Learning Transferable Visual Representations with Natural Language Supervision", + "paper_url": "https://arxiv.org/abs/2103.00020", + "paper_authors": "Radford et al., 2021", + "evaluation_results": "" + } + dynamic_config = DynamicRuleConfig(threshold=0.17) @classmethod @@ -141,10 +204,11 @@ def eval(cls, input_data: Data) -> ModelRes: import nltk nltk.download("punkt_tab") - from dingo.model.rule.utils.image_util import download_similar_tool from nltk.tokenize import word_tokenize from similarities import ClipSimilarity + from dingo.model.rule.utils.image_util import download_similar_tool + res = ModelRes() if not input_data.image or not input_data.content: return res diff --git a/dingo/model/rule/utils/detect_lang.py b/dingo/model/rule/utils/detect_lang.py index 70baee36..2c1dc066 100644 --- a/dingo/model/rule/utils/detect_lang.py +++ b/dingo/model/rule/utils/detect_lang.py @@ -4,9 +4,10 @@ import fasttext import requests -from dingo.utils import log from tqdm import tqdm +from dingo.utils import log + _global_lang_detect = [] _fasttext_path = "" diff --git a/dingo/run/cli.py b/dingo/run/cli.py index ec40ae9a..b5996dc6 100644 --- a/dingo/run/cli.py +++ b/dingo/run/cli.py @@ -2,6 +2,7 @@ import os import prettytable as pt + from dingo.exec import Executor from dingo.io import InputArgs from dingo.model import Model diff --git a/dingo/run/web.py b/dingo/run/web.py index 2f19cbf1..39fd0af1 100644 --- a/dingo/run/web.py +++ b/dingo/run/web.py @@ -3,11 +3,12 @@ from zipfile import ZIP_DEFLATED, ZipFile import uvicorn +from fastapi import FastAPI, HTTPException, status +from fastapi.responses import StreamingResponse + from dingo.exec import ExecProto, Executor from dingo.io import InputArgs from dingo.model import Model -from fastapi import FastAPI, HTTPException, status -from fastapi.responses import StreamingResponse app = FastAPI(title="dingo: Tool for detect language quality") diff --git a/dingo/utils/log_util/__init__.py b/dingo/utils/log_util/__init__.py index 4309f5d5..fa3bc9c9 100644 --- a/dingo/utils/log_util/__init__.py +++ b/dingo/utils/log_util/__init__.py @@ -1,8 +1,9 @@ from typing import Optional -from dingo.utils.log_util.logger import Logger from pydantic import BaseModel +from dingo.utils.log_util.logger import Logger + class LogConfig(BaseModel): """ diff --git a/docs/metrics.md b/docs/metrics.md index f3255dbf..7c03198f 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -1,11 +1,54 @@ -We classify data quality issues into 7 Quality Metrics, with the following definitions: - -| Quality Metric | Description | -|-------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| COMPLETENESS | Refers to data that is incomplete or completely missing. For example, whether some text data is truncated or the content is empty. | -| EFFECTIVENESS | Refers to whether the data is meaningful, suitable for a specific task, and conforms to the expected format or standard. For example, whether the text content contains garbled characters. | -| FLUENCY | Refers to whether the data is fluent, grammatically correct, and can be read naturally. For example, whether sentences conform to the grammatical rules. | -| RELEVANCE | Refers to data that contains data that is irrelevant to the task. For example, some texts describe medical knowledge, but insert irrelevant advertising content. | -| SECURITY | Refers to whether the data contains sensitive or private information and whether it conforms to the culture and values of various countries (the other party's values & our values). | -| SIMILARITY | Refers to whether the data content is repeated or there is very similar content. | -| UNDERSTANDABILITY | Refers to whether the data is easy to understand and interpret. For example, whether the data is clear, unambiguous, and meaningful in context. | +# Data Quality Metrics + +This document provides comprehensive information about all quality metrics used in Dingo. + +**Note**: All metrics are backed by academic sources to ensure objectivity and scientific rigor. + +### Text Quality Assessment Metrics + +| Type | Metric | Description | Paper Source | Evaluation Results | +|------|--------|-------------|--------------|-------------------| +| `DATAMAN_ASSESSMENT` | DATAMAN | Evaluates pre-training data quality using the DataMan methodology (14 standar... | [DataMan: Data Manager for Pre-training Large Language Models](https://arxiv.org/abs/2502.19363) (Peng et al., 2025) | N/A | +| `QUALITY_BAD_SECURITY` | Politics | Evaluates whether the text contains politics-related content | Internal Implementation | N/A | +| `TEXT_QUALITY_V4` | Pretrain Text Quality Assessment V4 | Enhanced text quality evaluation covering completeness (formulas, tables, cod... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [📊 See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | + +### SFT Data Assessment Metrics + +| Type | Metric | Description | Paper Source | Evaluation Results | +|------|--------|-------------|--------------|-------------------| +| `QUALITY_HARMLESS` | Harmlessness | Checks if responses avoid harmful content, discriminatory language, and dange... | [Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback](https://arxiv.org/pdf/2204.05862) (Bai et al., 2022) | [📊 See Results](eval/prompt/qa_data_evaluated_by_3h.md) | +| `QUALITY_HELPFUL` | Helpfulness | Assesses if responses address questions directly and follow instructions appr... | [Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback](https://arxiv.org/pdf/2204.05862) (Bai et al., 2022) | [📊 See Results](eval/prompt/qa_data_evaluated_by_3h.md) | +| `QUALITY_HONEST` | Honesty | Evaluates if responses provide accurate information without fabrication or de... | [Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback](https://arxiv.org/pdf/2204.05862) (Bai et al., 2022) | [📊 See Results](eval/prompt/qa_data_evaluated_by_3h.md) | + +### Classification Metrics + +| Type | Metric | Description | Paper Source | Evaluation Results | +|------|--------|-------------|--------------|-------------------| +| `CLASSIFY_TOPIC` | Topic Categorization | Classifies text into categories like language processing, writing, code, math... | [BERTopic](https://maartengr.github.io/BERTopic/index.html#quick-start) & [INSTAG](https://arxiv.org/pdf/2308.07074) (Grootendorst, 2022; Wei et al., 2023) | [📊 See Results](eval/prompt/text_data_classified_by_topic.md) | + +### Multimodality Assessment Metrics + +| Type | Metric | Description | Paper Source | Evaluation Results | +|------|--------|-------------|--------------|-------------------| +| `CLASSIFY_QR` | Image Classification | Identifies images as CAPTCHA, QR code, or normal images | Internal Implementation | N/A | +| `IMAGE_RELEVANT` | Image Relevance | Evaluates if an image matches reference image in terms of face count, feature... | Internal Implementation | N/A | + +### Rule-Based TEXT Quality Metrics + +| Type | Metric | Description | Paper Source | Evaluation Results | +|------|--------|-------------|--------------|-------------------| +| `QUALITY_BAD_COMPLETENESS` | RuleLineEndWithEllipsis, RuleLineEndWithTerminal, RuleSentenceNumber, RuleWordNumber | Checks whether the ratio of lines ending with ellipsis is below threshold; Ch... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [📊 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | +| `QUALITY_BAD_EFFECTIVENESS` | RuleAbnormalChar, RuleAbnormalHtml, RuleAlphaWords, RuleCharNumber, RuleColonEnd, RuleContentNull, RuleContentShort, RuleContentShortMultiLan, RuleEnterAndSpace, RuleEnterMore, RuleEnterRatioMore, RuleHtmlEntity, RuleHtmlTag, RuleInvisibleChar, RuleLineJavascriptCount, RuleLoremIpsum, RuleMeanWordLength, RuleSpaceMore, RuleSpecialCharacter, RuleStopWord, RuleSymbolWordRatio, RuleOnlyUrl | Detects garbled text and anti-crawling characters by combining special charac... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [📊 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | +| `QUALITY_BAD_FLUENCY` | RuleAbnormalNumber, RuleCharSplit, RuleNoPunc, RuleWordSplit, RuleWordStuck | Checks PDF content for abnormal book page or index numbers that disrupt text ... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [📊 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | +| `QUALITY_BAD_RELEVANCE` | RuleHeadWordAr | Checks whether Arabic content contains irrelevant tail source information | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [📊 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | +| `QUALITY_BAD_SIMILARITY` | RuleDocRepeat | Evaluates text for consecutive repeated content and multiple occurrences of s... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [📊 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | +| `QUALITY_BAD_UNDERSTANDABILITY` | RuleCapitalWords, RuleCurlyBracket, RuleLineStartWithBulletpoint, RuleUniqueWords | Checks whether the ratio of capital words is above threshold, indicating poor... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [📊 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | + +### Rule-Based IMG Quality Metrics + +| Type | Metric | Description | Paper Source | Evaluation Results | +|------|--------|-------------|--------------|-------------------| +| `QUALITY_BAD_IMG_EFFECTIVENESS` | RuleImageValid, RuleImageSizeValid, RuleImageQuality | Checks whether image is not all white or black, ensuring visual content valid... | Internal Implementation | N/A | +| `QUALITY_BAD_IMG_RELEVANCE` | RuleImageTextSimilarity | Evaluates semantic similarity between image and text content using CLIP model | [Learning Transferable Visual Representations with Natural Language Supervision](https://arxiv.org/abs/2103.00020) (Radford et al., 2021) | N/A | +| `QUALITY_BAD_IMG_SIMILARITY` | RuleImageRepeat | Detects duplicate images using PHash and CNN methods to ensure data diversity | [ImageNet Classification with Deep Convolutional Neural Networks](https://papers.nips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks.pdf) (Krizhevsky et al., 2012) | N/A | + diff --git a/examples/spark/sdk_spark.py b/examples/spark/sdk_spark.py index 6e9210b4..11924a84 100644 --- a/examples/spark/sdk_spark.py +++ b/examples/spark/sdk_spark.py @@ -1,8 +1,9 @@ import json +from pyspark.sql import DataFrame, SparkSession + from dingo.exec import Executor from dingo.io import Data, InputArgs -from pyspark.sql import DataFrame, SparkSession ################## # please prepare # diff --git a/mcp_server.py b/mcp_server.py index 8e10460b..cceb6224 100644 --- a/mcp_server.py +++ b/mcp_server.py @@ -3,11 +3,12 @@ import uuid from typing import Any, Dict, List, Literal, Optional, Tuple +from fastmcp import FastMCP + from dingo.exec import Executor from dingo.io import InputArgs from dingo.model import Model from dingo.utils import log -from fastmcp import FastMCP # Configure logging based on environment variable log_level = os.environ.get("LOG_LEVEL", "info").upper() diff --git a/scripts/generate_metrics.py b/scripts/generate_metrics.py new file mode 100644 index 00000000..d06a1ed0 --- /dev/null +++ b/scripts/generate_metrics.py @@ -0,0 +1,270 @@ +#!/usr/bin/env python3 +""" +自动生成 Metrics 文档的脚本 +按照 3H Assessment Prompts 表格的格式生成文档 +""" + +import sys +from pathlib import Path +from typing import Any, Dict, List + +# Add project root to path +project_root = Path(__file__).parent.parent +sys.path.insert(0, str(project_root)) + +from dingo.model.model import Model # noqa: E402 + + +def scan_prompt_classes() -> List[Dict[str, Any]]: + """扫描所有 prompt 类，提取 _metric_info 信息""" + # 先加载模型 + Model.load_model() + + metrics_info = [] + + # 直接从 prompt_metric_type_map 中获取信息 + for metric_type, prompt_classes in Model.prompt_metric_type_map.items(): + for prompt_class in prompt_classes: + if hasattr(prompt_class, '_metric_info'): + info = prompt_class._metric_info.copy() + info['prompt_type'] = metric_type + info['class_name'] = prompt_class.__name__ + info['type'] = 'prompt' + metrics_info.append(info) + + return metrics_info + + +def scan_rule_classes() -> List[Dict[str, Any]]: + """扫描所有 rule 类，提取 _metric_info 信息""" + # 先加载模型 + Model.load_model() + + metrics_info = [] + + # 直接从 rule_metric_type_map 中获取信息 + for metric_type, rule_classes in Model.rule_metric_type_map.items(): + for rule_class in rule_classes: + if hasattr(rule_class, '_metric_info'): + info = rule_class._metric_info.copy() + info['rule_type'] = metric_type + info['class_name'] = rule_class.__name__ + info['type'] = 'rule' + + # 如果 _metric_info 中没有设置 category，则根据类型设置默认值 + if 'category' not in info or not info['category']: + info['category'] = 'Rule-Based Quality Metrics' + + metrics_info.append(info) + + return metrics_info + + +def truncate_description(description: str, max_length: int = 80) -> str: + """截断description到指定长度""" + if len(description) <= max_length: + return description + return description[:max_length - 3] + "..." + + +def generate_table_section(title: str, metrics: List[Dict[str, Any]]) -> str: + """生成表格部分""" + if not metrics: + return "" + + # 表格头部 + table = f"### {title}\n\n" + table += "| Type | Metric | Description | Paper Source | Evaluation Results |\n" + table += "|------|--------|-------------|--------------|-------------------|\n" + + # 对于rule类，按type分组合并；对于prompt类，保持原有逻辑 + if title.startswith("Rule-Based") and "Quality Metrics" in title: + # 按type分组 + type_groups = {} + for metric in metrics: + if metric.get('type') == 'rule': + rule_type = metric.get('rule_type', '') + if rule_type not in type_groups: + type_groups[rule_type] = [] + type_groups[rule_type].append(metric) + + # 为每个type生成一行 + for rule_type in sorted(type_groups.keys()): + group_metrics = type_groups[rule_type] + type_name = f"`{rule_type}`" + + # 合并同一type的metric名称 + metric_names = [m['class_name'] for m in group_metrics] + combined_metrics = ", ".join(metric_names) + + # 合并描述（取第一个作为代表，或者合并所有描述） + descriptions = [m['description'] for m in group_metrics] + combined_description = "; ".join(descriptions) + combined_description = truncate_description(combined_description) + + # 取第一个metric的论文信息（因为都是相同的） + first_metric = group_metrics[0] + + # 处理论文来源 + if first_metric.get('paper_url') and first_metric.get('paper_title'): + paper_urls = [url.strip() for url in first_metric['paper_url'].split(',')] + paper_titles = [title.strip() for title in first_metric['paper_title'].split('&')] + + # 如果有多个URL和标题，为每个创建单独的链接 + if len(paper_urls) > 1 and len(paper_titles) > 1: + links = [] + for i, (title, url) in enumerate(zip(paper_titles, paper_urls)): + links.append(f"[{title}]({url})") + paper_source = " & ".join(links) + else: + paper_source = f"[{first_metric['paper_title']}](" \ + f"{first_metric['paper_url']})" + + if first_metric.get('paper_authors'): + paper_source += f" ({first_metric['paper_authors']})" + else: + paper_source = "Internal Implementation" + + # 处理评测结果 + if first_metric.get('evaluation_results'): + # 修正相对路径：从 docs/metrics.md 到 docs/eval/prompt/xxx.md + eval_path = first_metric['evaluation_results'] + if eval_path.startswith('docs/'): + eval_path = eval_path[5:] # 去掉 'docs/' 前缀 + eval_results = f"[📊 See Results]({eval_path})" + else: + eval_results = "N/A" + + table += f"| {type_name} | {combined_metrics} | " \ + f"{combined_description} | {paper_source} | {eval_results} |\n" + else: + # 对于prompt类，保持原有逻辑 + sort_key = lambda x: x.get('prompt_type', x.get('rule_type', '')) # noqa: E731 + for metric in sorted(metrics, key=sort_key): + # 处理type列 + if metric.get('type') == 'prompt': + type_name = f"`{metric['prompt_type']}`" + elif metric.get('type') == 'rule': + type_name = f"`{metric['rule_type']}`" + else: + type_name = "N/A" + + # 对于rule类，使用类名作为metric名称；对于prompt类，使用描述名称 + if metric.get('type') == 'rule': + metric_name = metric['class_name'] + else: + metric_name = metric['metric_name'] + description = truncate_description(metric['description']) + + # 处理论文来源 + if metric.get('paper_url') and metric.get('paper_title'): + paper_urls = [url.strip() for url in metric['paper_url'].split(',')] + paper_titles = [title.strip() for title in metric['paper_title'].split('&')] + + # 如果有多个URL和标题，为每个创建单独的链接 + if len(paper_urls) > 1 and len(paper_titles) > 1: + links = [] + for i, (title, url) in enumerate(zip(paper_titles, paper_urls)): + links.append(f"[{title}]({url})") + paper_source = " & ".join(links) + else: + paper_source = f"[{metric['paper_title']}](" \ + f"{metric['paper_url']})" + + if metric.get('paper_authors'): + paper_source += f" ({metric['paper_authors']})" + else: + paper_source = "Internal Implementation" + + # 处理评测结果 + if metric.get('evaluation_results'): + # 修正相对路径：从 docs/metrics.md 到 docs/eval/prompt/xxx.md + eval_path = metric['evaluation_results'] + if eval_path.startswith('docs/'): + eval_path = eval_path[5:] # 去掉 'docs/' 前缀 + eval_results = f"[📊 See Results]({eval_path})" + else: + eval_results = "N/A" + + table += f"| {type_name} | {metric_name} | {description} | " \ + f"{paper_source} | {eval_results} |\n" + + table += "\n" + return table + + +def generate_metrics_documentation() -> str: + """生成完整的 metrics 文档""" + # 扫描所有类 + prompt_metrics = scan_prompt_classes() + rule_metrics = scan_rule_classes() + + # 合并所有metrics + all_metrics = prompt_metrics + rule_metrics + + # 按类别分组 + categories = {} + for metric in all_metrics: + category = metric.get('category', 'other') + if category not in categories: + categories[category] = [] + categories[category].append(metric) + + # 生成文档 + doc = "# Data Quality Metrics\n\n" + doc += "This document provides comprehensive information about " \ + "all quality metrics used in Dingo.\n\n" + doc += "**Note**: All metrics are backed by academic sources to " \ + "ensure objectivity and scientific rigor.\n\n" + + # 按预定义顺序生成各个类别 + category_order = ["Text Quality Assessment Metrics", "SFT Data Assessment Metrics", + "Classification Metrics", "Multimodality Assessment Metrics", + "Rule-Based TEXT Quality Metrics", "Rule-Based IMG Quality Metrics", + "other"] + for category in category_order: + if category in categories: + doc += generate_table_section(category, categories[category]) + + return doc + + +def main(): + """主函数""" + try: + documentation = generate_metrics_documentation() + + # 写入文档文件 + output_file = project_root / "docs" / "metrics.md" + output_file.parent.mkdir(exist_ok=True) + + with open(output_file, 'w', encoding='utf-8') as f: + f.write(documentation) + + print(f"✅ Metrics documentation generated successfully: {output_file}") + + # 打印统计信息 + prompt_metrics = scan_prompt_classes() + rule_metrics = scan_rule_classes() + all_metrics = prompt_metrics + rule_metrics + + print(f"📊 Total metrics found: {len(all_metrics)}") + print(f" - Prompt-based: {len(prompt_metrics)}") + print(f" - Rule-based: {len(rule_metrics)}") + + categories = {} + for metric in all_metrics: + category = metric.get('category', 'other') + categories[category] = categories.get(category, 0) + 1 + + print("📈 Metrics by category:") + for category, count in sorted(categories.items()): + print(f" - {category}: {count}") + + except Exception as e: + print(f"❌ Error generating documentation: {e}") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/setup.cfg b/setup.cfg index 8ebd72cb..2e96d8fc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,10 +4,10 @@ blank_line_before_nested_class_or_def = true split_before_expression_after_opening_paren = true [isort] -line_length = 79 +line_length = 200 multi_line_output = 0 extra_standard_library = pkg_resources,setuptools -known_first_party = omniplat +known_first_party = dingo no_lines_before = STDLIB,LOCALFOLDER default_section = THIRDPARTY diff --git a/test/scripts/data/dataset/test_hf_dataset.py b/test/scripts/data/dataset/test_hf_dataset.py index 0446ab10..f2de3943 100644 --- a/test/scripts/data/dataset/test_hf_dataset.py +++ b/test/scripts/data/dataset/test_hf_dataset.py @@ -1,4 +1,5 @@ import pytest + from dingo.data.dataset.huggingface import HuggingFaceDataset from dingo.data.datasource.huggingface import HuggingFaceSource from dingo.io import InputArgs diff --git a/test/scripts/data/datasource/test_hf_datasource.py b/test/scripts/data/datasource/test_hf_datasource.py index 525658af..23be277c 100644 --- a/test/scripts/data/datasource/test_hf_datasource.py +++ b/test/scripts/data/datasource/test_hf_datasource.py @@ -1,4 +1,5 @@ import pytest + from dingo.data.datasource.huggingface import HuggingFaceSource from dingo.io import InputArgs diff --git a/test/scripts/exec/test_local.py b/test/scripts/exec/test_local.py index fcd31710..234063ce 100644 --- a/test/scripts/exec/test_local.py +++ b/test/scripts/exec/test_local.py @@ -1,4 +1,5 @@ import pytest + from dingo.exec import LocalExecutor from dingo.io import ResultInfo diff --git a/test/scripts/io/input/test_continue.py b/test/scripts/io/input/test_continue.py index 7b3e3e0a..da0f7683 100644 --- a/test/scripts/io/input/test_continue.py +++ b/test/scripts/io/input/test_continue.py @@ -2,6 +2,7 @@ import os.path import pytest + from dingo.exec import Executor from dingo.io import InputArgs diff --git a/test/scripts/io/input/test_write.py b/test/scripts/io/input/test_write.py index fe336301..c3eeeda9 100644 --- a/test/scripts/io/input/test_write.py +++ b/test/scripts/io/input/test_write.py @@ -2,6 +2,7 @@ import shutil import pytest + from dingo.exec import Executor from dingo.io import InputArgs diff --git a/test/scripts/model/rule/utils/test_rule_utils.py b/test/scripts/model/rule/utils/test_rule_utils.py index eec904df..039f0508 100644 --- a/test/scripts/model/rule/utils/test_rule_utils.py +++ b/test/scripts/model/rule/utils/test_rule_utils.py @@ -1,6 +1,7 @@ import time import pytest + from dingo.model.rule.utils.detect_lang import calculate_md5, download_fasttext