Update from task d7a22460-dbef-4e7a-b30b-b713551bc119

.github/workflows/kb-pipeline.yml

@@ -0,0 +1,140 @@
+name: KB Pipeline - End-to-End ICD-11 Knowledge Base
+
+on:
+  schedule:
+    # Run every 6 hours to allow iterative LLM batch processing (20 cards x 4 runs = 80 cards/day)
+    - cron: '0 */6 * * *'
+  workflow_dispatch:
+    inputs:
+      force_full_rebuild:
+        description: 'Force full rebuild of all artifacts'
+        required: false
+        default: 'false'
+
+concurrency:
+  group: kb-pipeline
+  cancel-in-progress: true
+
+permissions:
+  contents: write
+
+jobs:
+  pipeline:
+    runs-on: ubuntu-latest
+    timeout-minutes: 45  # Allow enough time for all stages including LLM batch
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # Full history for proper diff detection
+
+      - name: Setup Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          cache: 'pip'
+
+      - name: Install dependencies
+        run: |
+          pip install -e ".[dev]"
+          pip install openai  # For LLM enrichment
+
+      # ============================================
+      # STAGE 1: Raw Data Ingestion & Base YAML Generation
+      # ============================================
+      - name: Stage 1 - Fetch Raw Data from WHO API
+        env:
+          ICD_CLIENT_ID: ${{ secrets.ICD_CLIENT_ID }}
+          ICD_CLIENT_SECRET: ${{ secrets.ICD_CLIENT_SECRET }}
+        run: |
+          echo "=== Stage 1: Raw Data Ingestion ==="
+          python src/fetchers/who_client.py --data-dir data || echo "WHO fetch completed (may have processed 0 nodes if queue empty)"
+
+      - name: Stage 1 - Generate Base YAML Files
+        run: |
+          python scripts/generate_base_yaml.py --data-dir data
+
+      - name: Commit Stage 1 Artifacts
+        uses: stefanzweifel/git-auto-commit-action@v5
+        with:
+          commit_message: "chore(stage1): raw ingestion and base yaml generation"
+          file_pattern: "data/db/sync_state.db data/mms/*.yaml"
+          branch: ${{ github.ref_name }}
+
+      # ============================================
+      # STAGE 2: External / Rule-Based Enrichment
+      # ============================================
+      - name: Stage 2 - Rule-based Enrichment
+        run: |
+          echo "=== Stage 2: External Enrichment ==="
+          python scripts/enrich_external.py --data-dir data
+
+      - name: Commit Stage 2 Artifacts
+        uses: stefanzweifel/git-auto-commit-action@v5
+        with:
+          commit_message: "chore(stage2): external rule-based enrichment"
+          file_pattern: "data/mms/*.yaml"
+          branch: ${{ github.ref_name }}
+
+      # ============================================
+      # STAGE 3: Iterative LLM Enrichment (Batch Processing)
+      # ============================================
+      - name: Stage 3 - LLM Enrichment (Batch of 20)
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          BATCH_SIZE: 20
+        run: |
+          echo "=== Stage 3: LLM Enrichment (Batch Processing) ==="
+          if [ -z "$OPENAI_API_KEY" ]; then
+            echo "Warning: OPENAI_API_KEY not set. Skipping LLM enrichment."
+          else
+            python scripts/enrich_llm.py --data-dir data --batch-size $BATCH_SIZE
+          fi
+
+      - name: Commit Stage 3 Artifacts
+        uses: stefanzweifel/git-auto-commit-action@v5
+        with:
+          commit_message: "chore(stage3): llm enrichment batch"
+          file_pattern: "data/mms/*.yaml"
+          branch: ${{ github.ref_name }}
+
+      # ============================================
+      # STAGE 4: Vectorization & Indexing
+      # ============================================
+      - name: Stage 4 - Build Vector Database
+        run: |
+          echo "=== Stage 4: Vectorization ==="
+          python scripts/build_vectors.py --data-dir data
+
+      - name: Stage 4 - Build Symptom Links
+        run: |
+          python scripts/link_symptoms.py --data-dir data
+
+      - name: Stage 4 - Validate All Artifacts
+        run: |
+          python scripts/validate.py
+
+      - name: Commit Stage 4 Artifacts
+        uses: stefanzweifel/git-auto-commit-action@v5
+        with:
+          commit_message: "chore(stage4): vector db and indices rebuilt"
+          file_pattern: "data/generated/**"
+          branch: ${{ github.ref_name }}
+
+      # ============================================
+      # Summary
+      # ============================================
+      - name: Pipeline Summary
+        run: |
+          echo "=== Pipeline Complete ==="
+          echo ""
+          echo "Artifacts generated:"
+          echo "  - data/db/sync_state.db (SQLite raw data)"
+          echo "  - data/mms/*.yaml (Enriched disease cards)"
+          echo "  - data/foundation/*.yaml (Symptom definitions)"
+          echo "  - data/generated/lancedb_store/ (Vector database)"
+          echo "  - data/generated/links.yaml (Symptom reverse index)"
+          echo ""
+          echo "Disease cards in mms/: $(ls -1 data/mms/*.yaml 2>/dev/null | wc -l)"
+          echo "Foundation symptoms: $(ls -1 data/foundation/*.yaml 2>/dev/null | wc -l)"

.gitignore

@@ -4,35 +4,105 @@ __pycache__/
 *.pyc
 *.pyo
 *.pyd
+.Python
+env/
+venv/
+.venv/
+pip-log.txt
+pip-delete-this-directory.txt
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.log
+*.pot
+*.po
+*.mo
+build/
+develop-eggs/
+.installed.cfg
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
 
 # Dependencies
-.venv/
-venv/
-env/
-requirements.txt.bak
+node_modules/
 
-# Logs and temp files
+# Logs
 *.log
-*.tmp
 
-# Environment
+# Environment variables
 .env
 .env.local
-*.env.*
+.env.*
 
-# Editors
+# IDE
 .vscode/
 .idea/
-*.swp
-*.swo
 
-# Coverage
-.coverage
-coverage/
+# OS generated files
+.DS_Store
+.DS_Store?
+._*
+.Spotlight-V100
+.Trashes
+ehthumbs.db
+Thumbs.db
+
+# Testing
+.pytest_cache/
+.mypy_cache/
 htmlcov/
+.coverage
 
 # Distribution / packaging
 dist/
 build/
 *.egg-info/
+*.dist-info/
+
+# Compiled Binary files
+*.class
+*.o
+*.so
+*.a
+*.obj
+*.out
+*.exe
+*.dll
+
+# Compressed files
+*.zip
+*.gz
+*.tar
+*.tgz
+*.bz2
+*.xz
+*.7z
+*.rar
+*.zst
+*.lz4
+*.lzh
+*.cab
+*.arj
+*.rpm
+*.deb
+*.Z
+*.lz
+*.lzo
+*.tar.gz
+*.tar.bz2
+*.tar.xz
+*.tar.zst
+
 ```

data/mms/1B21.0.yaml

@@ -1,31 +1,22 @@
 entity_uri: https://icd.who.int/browse11/l-m/en#/http%3a%2f%2fid.who.int%2ficd%2fentity%2f1B21.0
 code: 1B21.0
 title_en: Influenza due to identified novel influenza A virus
-definition_en: Acute respiratory infection caused by a novel influenza A virus subtype that has not previously circulated in humans.
+definition_en: Acute respiratory infection caused by a novel influenza A virus subtype
+  that has not previously circulated in humans.
 parent_code: 1B21
 children_codes: []
-pathophysiology_en: Viral infection of the respiratory tract with systemic symptoms due to cytokine release.
-symptoms:
-  - id: fever
-    grade: ALWAYS
-    probability: 1.0
-    note: High fever typically present at onset
-  - id: headache
-    grade: VERY_COMMON
-    probability: 0.85
-    note: Often severe and frontal
+pathophysiology_en: Viral infection of the respiratory tract with systemic symptoms
+  due to cytokine release.
+symptoms: []
 differential_diagnosis:
-  - Common influenza
-  - COVID-19
-  - Respiratory syncytial virus
-risk_factors:
-  - Close contact with infected individuals
-  - Travel to endemic areas
-  - Compromised immune system
+- Common influenza
+- COVID-19
+- Respiratory syncytial virus
+risk_factors: []
 drugs:
-  - Oseltamivir
-  - Zanamivir
-  - Peramivir
+- Oseltamivir
+- Zanamivir
+- Peramivir
 vector_text_en: Novel influenza A virus respiratory infection with systemic symptoms
 stats:
   mortality_global_annual: null
@@ -35,4 +26,5 @@ stats:
   child_code_count: 0
   research_link_count: 0
 ai_enriched: false
-last_updated: "2024-01-15"
+last_updated: '2024-01-15'
+externally_enriched: true

data/mms/Abnormal.yaml

@@ -0,0 +1,29 @@
+entity_uri: https://id.who.int/icd/release/11/2026-01/mms/1324937831
+code: Abnormal
+title_en: Abnormal uterine or vaginal bleeding
+definition_en: ''
+parent_code: null
+children_codes:
+- '213603738'
+- '1015594514'
+- '537271184'
+- '1001041854'
+- other
+- unspecified
+raw_paragraphs: []
+pathophysiology_en: ''
+symptoms: []
+differential_diagnosis: []
+risk_factors: []
+drugs: []
+vector_text_en: ''
+stats:
+  mortality_global_annual: null
+  dalys_global: null
+  incidence_rate_per_100k: null
+  active_clinical_trials: null
+  child_code_count: 6
+  research_link_count: 0
+externally_enriched: false
+ai_enriched: false
+last_updated: '2026-07-01'

data/mms/Abnormalities.yaml

@@ -0,0 +1,30 @@
+entity_uri: https://id.who.int/icd/release/11/2026-01/mms/1730591881
+code: Abnormalities
+title_en: Abnormalities of forces of labour
+definition_en: ''
+parent_code: null
+children_codes:
+- '773832481'
+- '1263825589'
+- '1073291850'
+- '1521676256'
+- '693926988'
+- other
+- unspecified
+raw_paragraphs: []
+pathophysiology_en: ''
+symptoms: []
+differential_diagnosis: []
+risk_factors: []
+drugs: []
+vector_text_en: ''
+stats:
+  mortality_global_annual: null
+  dalys_global: null
+  incidence_rate_per_100k: null
+  active_clinical_trials: null
+  child_code_count: 7
+  research_link_count: 0
+externally_enriched: false
+ai_enriched: false
+last_updated: '2026-07-01'