diff --git a/.gitignore b/.gitignore index 920f71c..69d93fc 100644 --- a/.gitignore +++ b/.gitignore @@ -12,17 +12,17 @@ dist/ build/ # Runtime data — raw fetches, enriched outputs, spine manifests are regenerable -data/raw/ -data/out/ -data/cache/ -data/inline/ -data/explore/ -data/bootstrap/ -data/spine/* +apps/pipeline/data/raw/ +apps/pipeline/data/out/ +apps/pipeline/data/cache/ +apps/pipeline/data/inline/ +apps/pipeline/data/explore/ +apps/pipeline/data/bootstrap/ +apps/pipeline/data/spine/* # …except the Ed-Fi domain map — canonical snapshot committed so fresh clones # can build WI/MN spines without re-fetching AZ's swagger first. Regenerate # via `mc spine domain-map --source AZ`. -!data/spine/edfi_domain_map.json +!apps/pipeline/data/spine/edfi_domain_map.json # Excel lockfiles ~$*.xlsx @@ -44,10 +44,10 @@ docs/human-scored-files/ # regime (ADR 0010, issue #213 item 4). Untracked WITHOUT history rewrite; # prior revisions remain recoverable from git history. Exact paths only — # the methodology spec + data/bootstrap/ source docs stay tracked. -data/bootstrap/NACHOS[ ]Arizona_20260310.xlsx -data/bootstrap/NACHOS_Arizona_No_scoring.xlsx -data/bootstrap/SourceData_Agent_training_20pct_sample[ ]-[ ]origin.xlsx -data/bootstrap/SourceData_Agent_training_file.xlsx +apps/pipeline/data/bootstrap/NACHOS[ ]Arizona_20260310.xlsx +apps/pipeline/data/bootstrap/NACHOS_Arizona_No_scoring.xlsx +apps/pipeline/data/bootstrap/SourceData_Agent_training_20pct_sample[ ]-[ ]origin.xlsx +apps/pipeline/data/bootstrap/SourceData_Agent_training_file.xlsx # Calibration JSON join artifacts — regenerated by scripts/issue108_az_compare.py # and similar diagnostic scripts. The writeup .md next to them stays committed. diff --git a/apps/pipeline/.env.example b/apps/pipeline/.env.example new file mode 100644 index 0000000..0c5e07e --- /dev/null +++ b/apps/pipeline/.env.example @@ -0,0 +1,28 @@ +# Copy to `.env` (gitignored) and fill in real values before running +# commands that need them. Loaded at CLI entry via python-dotenv — +# `mc ...` picks it up automatically; `python -c "..."` one-offs still +# need `export ...` or `source .env`. Exported shell vars always win +# over .env (override=False). + +# Anthropic API key — required for `mc score extract` without --dry-run. +# Get one at https://console.anthropic.com/settings/keys. +ANTHROPIC_API_KEY= + +# Fill in only one provider block at a time. The runtime uses the first +# matching configured lane, so keeping a single block populated avoids +# ambiguity. + +# Ed-Fi Azure Anthropic-compatible route for the main scoring pipeline. +# When both of these are set, src/score/azure_client.py selects the +# Azure Anthropic Haiku client instead of the default Anthropic client. +ANTHROPIC_API_HAIKU_ENDPOINT= +ANTHROPIC_API_HAIKU_KEY= + +# Optional Sonnet 4.6 Azure Anthropic-compatible route. +# If both Sonnet vars are set, they take precedence over the Haiku pair. +ANTHROPIC_API_SONNET_ENDPOINT= +ANTHROPIC_API_SONNET_KEY= + +# Ed-Fi Azure OpenAI Responses route (parity / alternate client use). +OPENAPI_API_GPT54= +OPENAPI_API_GPT54_KEY= diff --git a/apps/pipeline/data/spine/edfi_domain_map.json b/apps/pipeline/data/spine/edfi_domain_map.json new file mode 100644 index 0000000..78f6ee8 --- /dev/null +++ b/apps/pipeline/data/spine/edfi_domain_map.json @@ -0,0 +1,719 @@ +{ + "AcademicWeek": [ + "SchoolCalendar" + ], + "AccountabilityRating": [ + "EducationOrganization", + "Enrollment" + ], + "ApplicantProfile": [ + "RecruitingAndStaffing" + ], + "Application": [ + "RecruitingAndStaffing" + ], + "ApplicationEvent": [ + "RecruitingAndStaffing" + ], + "Assessment": [ + "Assessment", + "AssessmentMetadata" + ], + "AssessmentAdministration": [ + "AssessmentRegistration" + ], + "AssessmentAdministrationParticipation": [ + "AssessmentRegistration" + ], + "AssessmentBatteryPart": [ + "AssessmentRegistration" + ], + "AssessmentItem": [ + "Assessment", + "AssessmentMetadata" + ], + "AssessmentScoreRangeLearningStandard": [ + "Assessment" + ], + "BalanceSheetDimension": [ + "Finance" + ], + "BellSchedule": [ + "BellSchedule" + ], + "Calendar": [ + "SchoolCalendar" + ], + "CalendarDate": [ + "SchoolCalendar" + ], + "Candidate": [ + "EducatorPreparationProgram" + ], + "CandidateEducatorPreparationProgramAssociation": [ + "EducatorPreparationProgram" + ], + "CandidateIdentificationCode": [ + "EducatorPreparationProgram" + ], + "CandidateRelationshipToStaffAssociation": [ + "EducatorPreparationProgram" + ], + "Certification": [ + "Credential" + ], + "CertificationExam": [ + "Credential" + ], + "CertificationExamResult": [ + "Credential" + ], + "ChartOfAccount": [ + "Finance" + ], + "ClassPeriod": [ + "BellSchedule", + "TeachingAndLearning", + "SectionsAndPrograms" + ], + "Cohort": [ + "Intervention", + "StudentCohort" + ], + "CommunityOrganization": [ + "EducationOrganization" + ], + "CommunityProvider": [ + "EducationOrganization" + ], + "CommunityProviderLicense": [ + "EducationOrganization" + ], + "CompetencyObjective": [ + "StudentAcademicRecord", + "ReportCard" + ], + "Contact": [ + "StudentIdentificationAndDemographics", + "Survey" + ], + "ContactIdentificationCode": [ + "StudentIdentificationAndDemographics" + ], + "Course": [ + "StudentAcademicRecord", + "Survey", + "TeachingAndLearning", + "CourseCatalog", + "Gradebook", + "SectionsAndPrograms", + "StudentTranscript" + ], + "CourseOffering": [ + "StudentAcademicRecord", + "TeachingAndLearning", + "CourseCatalog", + "SectionsAndPrograms" + ], + "CourseTranscript": [ + "StudentAcademicRecord", + "TeachingAndLearning", + "SectionsAndPrograms", + "StudentTranscript" + ], + "Credential": [ + "Credential", + "Staff" + ], + "CredentialEvent": [ + "Credential" + ], + "CrisisEvent": [ + "Enrollment" + ], + "DescriptorMapping": [ + "EducationOrganization" + ], + "DisciplineAction": [ + "Discipline" + ], + "DisciplineIncident": [ + "Discipline" + ], + "EducationContent": [ + "Intervention" + ], + "EducationOrganizationIdentificationCode": [ + "EducationOrganization" + ], + "EducationOrganizationInterventionPrescriptionAssociation": [ + "Intervention" + ], + "EducationOrganizationNetwork": [ + "EducationOrganization" + ], + "EducationOrganizationNetworkAssociation": [ + "EducationOrganization" + ], + "EducationOrganizationPeerAssociation": [ + "EducationOrganization" + ], + "EducationServiceCenter": [ + "EducationOrganization", + "Staff" + ], + "EducatorPreparationProgram": [ + "EducatorPreparationProgram" + ], + "Evaluation": [ + "PerformanceEvaluation" + ], + "EvaluationElement": [ + "PerformanceEvaluation" + ], + "EvaluationElementRating": [ + "PerformanceEvaluation" + ], + "EvaluationObjective": [ + "PerformanceEvaluation" + ], + "EvaluationObjectiveRating": [ + "PerformanceEvaluation" + ], + "EvaluationRating": [ + "PerformanceEvaluation" + ], + "EvaluationRubricDimension": [ + "TeachingAndLearning", + "StudentProgramEvaluation" + ], + "FeederSchoolAssociation": [ + "EducationOrganization" + ], + "FieldworkExperience": [ + "EducatorPreparationProgram" + ], + "FieldworkExperienceSectionAssociation": [ + "EducatorPreparationProgram" + ], + "FinancialAid": [ + "Enrollment" + ], + "FunctionDimension": [ + "Finance" + ], + "FundDimension": [ + "Finance" + ], + "Goal": [ + "PerformanceEvaluation" + ], + "Grade": [ + "StudentAcademicRecord", + "ReportCard" + ], + "GradebookEntry": [ + "StudentAcademicRecord", + "Gradebook" + ], + "GradingPeriod": [ + "SchoolCalendar", + "StudentAcademicRecord", + "ReportCard" + ], + "GraduationPlan": [ + "Enrollment", + "Graduation" + ], + "Intervention": [ + "Intervention", + "StudentCohort" + ], + "InterventionPrescription": [ + "Intervention" + ], + "InterventionStudy": [ + "Intervention" + ], + "LearningStandard": [ + "Assessment", + "StudentAcademicRecord", + "TeachingAndLearning", + "CourseCatalog", + "Gradebook", + "ReportCard", + "Standards" + ], + "LearningStandardEquivalenceAssociation": [ + "Assessment" + ], + "LocalAccount": [ + "Finance" + ], + "LocalActual": [ + "Finance" + ], + "LocalBudget": [ + "Finance" + ], + "LocalContractedStaff": [ + "Finance" + ], + "LocalEducationAgency": [ + "EducationOrganization", + "Enrollment", + "Staff" + ], + "LocalEncumbrance": [ + "Finance" + ], + "LocalPayroll": [ + "Finance" + ], + "Location": [ + "TeachingAndLearning", + "SectionsAndPrograms" + ], + "ObjectDimension": [ + "Finance" + ], + "ObjectiveAssessment": [ + "Assessment", + "AssessmentMetadata" + ], + "OpenStaffPosition": [ + "Staff" + ], + "OpenStaffPositionEvent": [ + "Staff" + ], + "OperationalUnitDimension": [ + "Finance" + ], + "OrganizationDepartment": [ + "EducationOrganization" + ], + "Path": [ + "Path" + ], + "PathMilestone": [ + "Path" + ], + "PathPhase": [ + "Path" + ], + "PerformanceEvaluation": [ + "PerformanceEvaluation" + ], + "PerformanceEvaluationRating": [ + "PerformanceEvaluation" + ], + "Person": [ + "Staff", + "StudentIdentificationAndDemographics" + ], + "PostSecondaryEvent": [ + "Graduation" + ], + "PostSecondaryInstitution": [ + "EducationOrganization", + "Graduation" + ], + "ProfessionalDevelopmentEvent": [ + "RecruitingAndStaffing" + ], + "ProfessionalDevelopmentEventAttendance": [ + "RecruitingAndStaffing" + ], + "Program": [ + "AlternativeAndSupplementalServices", + "Assessment", + "SpecialEducation", + "StudentAcademicRecord", + "StudentAttendance", + "StudentCohort", + "Survey", + "TeachingAndLearning", + "AssessmentMetadata", + "ReportCard", + "SectionsAndPrograms", + "StudentProgramEvaluation" + ], + "ProgramDimension": [ + "Finance" + ], + "ProgramEvaluation": [ + "TeachingAndLearning", + "StudentProgramEvaluation" + ], + "ProgramEvaluationElement": [ + "TeachingAndLearning", + "StudentProgramEvaluation" + ], + "ProgramEvaluationObjective": [ + "TeachingAndLearning", + "StudentProgramEvaluation" + ], + "ProjectDimension": [ + "Finance" + ], + "QuantitativeMeasure": [ + "PerformanceEvaluation" + ], + "QuantitativeMeasureScore": [ + "PerformanceEvaluation" + ], + "RecruitmentEvent": [ + "RecruitingAndStaffing" + ], + "RecruitmentEventAttendance": [ + "RecruitingAndStaffing" + ], + "ReportCard": [ + "StudentAcademicRecord", + "ReportCard" + ], + "RestraintEvent": [ + "SpecialEducation" + ], + "RubricDimension": [ + "PerformanceEvaluation" + ], + "School": [ + "AlternativeAndSupplementalServices", + "BellSchedule", + "Discipline", + "EducationOrganization", + "Enrollment", + "Graduation", + "SchoolCalendar", + "SpecialEducation", + "Staff", + "StudentAcademicRecord", + "StudentAttendance", + "TeachingAndLearning", + "CourseCatalog", + "SectionsAndPrograms" + ], + "Section": [ + "AlternativeAndSupplementalServices", + "Assessment", + "BellSchedule", + "SpecialEducation", + "StudentAcademicRecord", + "StudentAttendance", + "StudentCohort", + "Survey", + "TeachingAndLearning", + "AssessmentMetadata", + "Gradebook", + "ReportCard", + "SectionsAndPrograms" + ], + "SectionAttendanceTakenEvent": [ + "StudentAttendance" + ], + "Session": [ + "BellSchedule", + "SchoolCalendar", + "StudentAcademicRecord", + "StudentAttendance", + "Survey", + "TeachingAndLearning", + "CourseCatalog", + "SectionsAndPrograms" + ], + "SourceDimension": [ + "Finance" + ], + "Staff": [ + "AlternativeAndSupplementalServices", + "Discipline", + "Finance", + "Intervention", + "SpecialEducation", + "Staff", + "StudentAttendance", + "StudentCohort", + "Survey", + "TeachingAndLearning", + "SectionsAndPrograms" + ], + "StaffAbsenceEvent": [ + "Staff" + ], + "StaffCohortAssociation": [ + "StudentCohort" + ], + "StaffDemographic": [ + "Staff" + ], + "StaffDirectory": [ + "Staff" + ], + "StaffDisciplineIncidentAssociation": [ + "Discipline" + ], + "StaffEducationOrganizationAssignmentAssociation": [ + "Staff" + ], + "StaffEducationOrganizationEmploymentAssociation": [ + "Staff" + ], + "StaffEducatorPreparationProgramAssociation": [ + "EducatorPreparationProgram" + ], + "StaffIdentificationCode": [ + "Staff" + ], + "StaffLeave": [ + "Staff" + ], + "StaffProgramAssociation": [ + "AlternativeAndSupplementalServices", + "SpecialEducation", + "TeachingAndLearning", + "SectionsAndPrograms" + ], + "StaffSchoolAssociation": [ + "Staff", + "TeachingAndLearning", + "SectionsAndPrograms" + ], + "StaffSectionAssociation": [ + "SpecialEducation", + "StudentCohort", + "TeachingAndLearning", + "SectionsAndPrograms" + ], + "StateEducationAgency": [ + "EducationOrganization", + "Staff" + ], + "Student": [ + "AlternativeAndSupplementalServices", + "Assessment", + "Discipline", + "Enrollment", + "Graduation", + "Intervention", + "SchoolCalendar", + "SpecialEducation", + "StudentAcademicRecord", + "StudentAttendance", + "StudentCohort", + "StudentHealth", + "StudentIdentificationAndDemographics", + "Survey", + "TeachingAndLearning", + "Gradebook", + "ReportCard", + "SectionsAndPrograms", + "StudentAssessment", + "StudentTranscript" + ], + "StudentAcademicRecord": [ + "Graduation", + "StudentAcademicRecord", + "TeachingAndLearning", + "SectionsAndPrograms", + "StudentTranscript" + ], + "StudentAssessment": [ + "Assessment", + "StudentAssessment" + ], + "StudentAssessmentRegistration": [ + "AssessmentRegistration" + ], + "StudentAssessmentRegistrationBatteryPartAssociation": [ + "AssessmentRegistration" + ], + "StudentCTEProgramAssociation": [ + "AlternativeAndSupplementalServices" + ], + "StudentCohortAssociation": [ + "Intervention", + "StudentCohort" + ], + "StudentCompetencyObjective": [ + "StudentAcademicRecord", + "ReportCard" + ], + "StudentContactAssociation": [ + "StudentIdentificationAndDemographics" + ], + "StudentDemographic": [ + "StudentIdentificationAndDemographics" + ], + "StudentDirectory": [ + "StudentIdentificationAndDemographics" + ], + "StudentDisciplineIncidentBehaviorAssociation": [ + "Discipline" + ], + "StudentDisciplineIncidentNonOffenderAssociation": [ + "Discipline" + ], + "StudentEducationOrganizationAssessmentAccommodation": [ + "AssessmentRegistration" + ], + "StudentEducationOrganizationAssociation": [ + "Enrollment", + "StudentIdentificationAndDemographics" + ], + "StudentEducationOrganizationResponsibilityAssociation": [ + "Enrollment" + ], + "StudentGradebookEntry": [ + "StudentAcademicRecord", + "Gradebook" + ], + "StudentHealth": [ + "StudentHealth" + ], + "StudentHomelessProgramAssociation": [ + "AlternativeAndSupplementalServices" + ], + "StudentIdentificationCode": [ + "StudentIdentificationAndDemographics" + ], + "StudentInterventionAssociation": [ + "Intervention", + "StudentCohort" + ], + "StudentInterventionAttendanceEvent": [ + "Intervention" + ], + "StudentLanguageInstructionProgramAssociation": [ + "AlternativeAndSupplementalServices" + ], + "StudentMigrantEducationProgramAssociation": [ + "AlternativeAndSupplementalServices" + ], + "StudentNeglectedOrDelinquentProgramAssociation": [ + "AlternativeAndSupplementalServices" + ], + "StudentPath": [ + "Path" + ], + "StudentPathMilestoneStatus": [ + "Path" + ], + "StudentPathPhaseStatus": [ + "Path" + ], + "StudentProgramAssociation": [ + "AlternativeAndSupplementalServices", + "SpecialEducation", + "StudentAcademicRecord", + "StudentAttendance", + "TeachingAndLearning", + "ReportCard", + "SectionsAndPrograms" + ], + "StudentProgramAttendanceEvent": [ + "AlternativeAndSupplementalServices", + "SpecialEducation", + "StudentAttendance" + ], + "StudentProgramEvaluation": [ + "TeachingAndLearning", + "StudentProgramEvaluation" + ], + "StudentSchoolAssociation": [ + "Enrollment", + "Graduation", + "SchoolCalendar", + "TeachingAndLearning", + "SectionsAndPrograms" + ], + "StudentSchoolAttendanceEvent": [ + "AlternativeAndSupplementalServices", + "SpecialEducation", + "StudentAttendance" + ], + "StudentSchoolFoodServiceProgramAssociation": [ + "AlternativeAndSupplementalServices" + ], + "StudentSection504ProgramAssociation": [ + "AlternativeAndSupplementalServices" + ], + "StudentSectionAssociation": [ + "AlternativeAndSupplementalServices", + "SpecialEducation", + "StudentAcademicRecord", + "StudentAttendance", + "StudentCohort", + "TeachingAndLearning", + "Gradebook", + "ReportCard", + "SectionsAndPrograms" + ], + "StudentSectionAttendanceEvent": [ + "AlternativeAndSupplementalServices", + "SpecialEducation", + "StudentAttendance" + ], + "StudentSpecialEducationProgramAssociation": [ + "AlternativeAndSupplementalServices", + "SpecialEducation" + ], + "StudentSpecialEducationProgramEligibilityAssociation": [ + "AlternativeAndSupplementalServices" + ], + "StudentTitleIPartAProgramAssociation": [ + "AlternativeAndSupplementalServices" + ], + "StudentTransportation": [ + "Enrollment" + ], + "Survey": [ + "Survey" + ], + "SurveyCourseAssociation": [ + "Survey" + ], + "SurveyProgramAssociation": [ + "Survey" + ], + "SurveyQuestion": [ + "Survey" + ], + "SurveyQuestionResponse": [ + "Survey" + ], + "SurveyResponse": [ + "Survey" + ], + "SurveyResponseEducationOrganizationTargetAssociation": [ + "Survey" + ], + "SurveyResponsePersonTargetAssociation": [ + "Survey" + ], + "SurveyResponseStaffTargetAssociation": [ + "Survey" + ], + "SurveySection": [ + "Survey" + ], + "SurveySectionAggregateResponse": [ + "PerformanceEvaluation" + ], + "SurveySectionAssociation": [ + "Survey" + ], + "SurveySectionResponse": [ + "Survey" + ], + "SurveySectionResponseEducationOrganizationTargetAssociation": [ + "Survey" + ], + "SurveySectionResponsePersonTargetAssociation": [ + "Survey" + ], + "SurveySectionResponseStaffTargetAssociation": [ + "Survey" + ] +} diff --git a/apps/pipeline/pyproject.toml b/apps/pipeline/pyproject.toml new file mode 100644 index 0000000..fb2d2f3 --- /dev/null +++ b/apps/pipeline/pyproject.toml @@ -0,0 +1,131 @@ +[project] +name = "metadata-catalog" +version = "0.1.0" +description = "Data Standard Metadata Collection and Usage tools for Ed-Fi metadata analysis" +readme = "README.md" +requires-python = ">=3.12" +dependencies = [ + "anthropic>=0.96.0", + "beautifulsoup4>=4.14.0", + "click>=8.3.0", + "httpx>=0.28.0", + "jsonschema>=4.24.0", + "lxml>=6.0.0", + "openpyxl>=3.1.0", + "pdfplumber>=0.11.0", + "pydantic>=2.12.0", + "python-docx>=1.2.0", + "python-dotenv>=1.0.0", + "pyyaml>=6.0.0", +] + +[project.optional-dependencies] +# playwright is only needed for the TX TWEDS Chromium scraper +# (ingest/tx_tweds.py, one lazy import site). Everything else — including +# the full test suite — runs without it. Install with: +# uv pip install -e '.[scrape]' +scrape = [ + "playwright>=1.48.0", +] + +[dependency-groups] +dev = [ + "pytest>=9.0.0", + "ruff>=0.9.0", + "mypy>=1.14.0", +] + +[project.scripts] +metadata-catalog = "src.cli:main" +mc = "src.cli:main" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src"] + +[tool.ruff] +target-version = "py312" + +[tool.ruff.lint] +# Small, high-signal baseline (issue #213 item 4). Deliberately configured to +# be GREEN on the current tree — lint adoption must not force churn in the +# scoring/report/ingest modules (sidecar byte-identity guardrail). What was +# relaxed, and why: +# E501 — long lines are pervasive in this prose-heavy tree; enforcing it is +# pure formatting churn. +# E402 — late module-level imports are deliberate in adapters, spine/build, +# and tests (docstring/config setup before import); revisit as #213 +# items 1–3 refactor those modules. +# E741 — `l` loop variables in diagnostic scripts/tests; rename churn only. +# F541 — placeholder-less f-strings in CLI/script print scaffolding; cosmetic. +# F841 — unused locals sit in modules item 4 must not touch (report/score/ +# ingest); clean up opportunistically as #213 items 1–3 land. +# isort (I) deliberately NOT enabled: it would reorder imports across most of +# the tree for zero behavior benefit. +select = ["E", "F", "W"] +ignore = ["E501", "E402", "E741", "F541", "F841"] + +[tool.mypy] +# Pragmatic green-today baseline (issue #213 item 4): the tree is ~98% +# annotated, but adoption is deliberately loose so no source refactors were +# needed. What was relaxed, and why: +# ignore_missing_imports — openpyxl/pdfplumber/docx et al. ship no stubs. +# Scoped to src — tests/scripts monkeypatch heavily; type them later. +# Tighten incrementally (e.g. per-module strict = true) as #213 items 1–3 +# touch each module. +python_version = "3.12" +files = ["src"] +ignore_missing_imports = true + +# Baseline module exclusions — these 24 modules carry pre-existing type +# errors (dominated by arg-type on the dict-row workbook-rendering seam in +# workbook_spec/analyst, plus the openpyxl/anthropic library boundaries). +# `ignore_errors` silences THEIR errors while they still contribute inferred +# signatures to consumers; the other 53 src modules are fully checked. +# RATCHET RULE: when #213 items 1–3 refactor one of these modules, fix its +# errors and DELETE its entry here. Do not add new modules to this list. +[[tool.mypy.overrides]] +module = [ + "src.ingest.az_integrity_rules", + "src.ingest.gap_surfacer", + "src.ingest.idoe_confluence", + "src.ingest.shared", + "src.ingest.swagger_backfill", + "src.ingest.tx_tweds", + "src.ingest.wisconsin", + "src.publish.orchestrator", + "src.publish.stages", + "src.report.analyst", + # Split out of analyst.py by issue #213 item 1 — carries two of its + # relocated pre-existing dict-row-seam errors (same ratchet rule). + "src.report.score_card", + "src.report.coverage", + "src.report.human_score_backfill", + "src.report.workbook_render", + "src.report.workbook_spec", + "src.score.aggregate", + "src.score.aggregate_gap", + "src.score.batch_client", + "src.score.client", + "src.score.deterministic", + "src.score.extract", + "src.score.inline", + "src.score.review_comparison", + "src.score.rules", + "src.score.runner", +] +ignore_errors = true + +[tool.pytest.ini_options] +# `pythonpath` makes bare `pytest` work from the repo root regardless of +# editable-install state (kills the PYTHONPATH=src incantation and the +# macOS hidden-.pth / shadow-install traps for testing). +pythonpath = ["src"] +addopts = "-p no:anyio" +testpaths = ["tests"] +markers = [ + "realdata: needs locally generated data/ artifacts (deselected in CI via -m 'not realdata')", +] diff --git a/apps/pipeline/src/__init__.py b/apps/pipeline/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/pipeline/src/__main__.py b/apps/pipeline/src/__main__.py new file mode 100644 index 0000000..f48f4d9 --- /dev/null +++ b/apps/pipeline/src/__main__.py @@ -0,0 +1,6 @@ +"""Run the Metadata-Catalog CLI via `python -m mc`.""" + +from src.cli import main + +if __name__ == "__main__": + main() diff --git a/apps/pipeline/src/cli.py b/apps/pipeline/src/cli.py new file mode 100644 index 0000000..d8032c5 --- /dev/null +++ b/apps/pipeline/src/cli.py @@ -0,0 +1,1705 @@ +"""POC-3 CLI entry point. Subcommands: spine, ingest, report, score, review.""" + +from __future__ import annotations + +from pathlib import Path + +import click + +from src.ingest import INGEST_MODULES +from src.publish.stages import DEFAULT_COST_CAP, DEFAULT_TX_BASE_URL, STAGE_NAMES +from src.states import SUPPORTED_STATES + +# One source of truth for the state roster (src.states) — these shared +# Choice objects replace the previously duplicated per-command literals. +_STATE_CHOICES = click.Choice(list(SUPPORTED_STATES), case_sensitive=False) +_STATE_CHOICES_WITH_ALL = click.Choice( + [*SUPPORTED_STATES, "all"], case_sensitive=False +) + + +def _expand_states(state_opt: str) -> tuple[str, ...]: + """Expand the shared ``--state all`` idiom to the canonical roster. + + One helper for the ~8 copies of ``SUPPORTED_STATES if state.lower() + == "all" else (state.upper(),)`` (issue #213 item 3). + """ + if state_opt.lower() == "all": + return tuple(SUPPORTED_STATES) + return (state_opt.upper(),) + + +# --- shared option factories (issue #213 item 3) ----------------------------- +# Repeated option groups built once; per-command help/defaults stay +# parameters so every command's --help output is byte-identical. + + +def _state_all_option(*, default: str, help: str): + return click.option( + "--state", + "state_opt", + type=_STATE_CHOICES_WITH_ALL, + default=default, + show_default=True, + help=help, + ) + + +def _lens_option( + *, + default: str | None = "source", + help: str, + required: bool = False, + choices: tuple[str, ...] = ("source", "spine"), +): + kwargs: dict = { + "type": click.Choice(list(choices), case_sensitive=False), + "help": help, + } + if required: + kwargs["required"] = True + else: + kwargs["default"] = default + kwargs["show_default"] = True + return click.option("--lens", **kwargs) + + +def _allow_stale_option( + help: str = ( + "Proceed on inputs the publish manifest marks stale/missing " + "(degraded numbers — see PR #182)." + ), +): + return click.option("--allow-stale", is_flag=True, help=help) + + +def _model_option( + help: str = "Anthropic model ID (defaults to the harness's current Sonnet).", +): + return click.option("--model", type=str, default=None, help=help) + + +@click.group() +def cli() -> None: + """NACHOS POC-3 — ingestion-first pipeline.""" + + +@cli.group() +def spine() -> None: + """Fetch and build the Ed-Fi Swagger/API model (internal name: spine).""" + + +@spine.command("fetch") +@click.option("--state", type=_STATE_CHOICES, required=True) +@click.option("--school-year", type=int, default=2026, show_default=True, + help="Used for WI (fills {schoolYearFromRoute}) and IN (year-prefixed path " + "/{school_year}/metadata/...). IN currently hosts /2026/ + /2027/; " + "/2027/ is the active deployment as of 2026-05.") +@click.option("--base-url", type=str, default=None, + help="Override the sandbox base URL (up through '.../data/v3'). " + "Intended for TX (local Docker); ignored when the default is correct.") +def spine_fetch(state: str, school_year: int, base_url: str | None) -> None: + """Fetch resources + descriptors Swagger JSON from a state Ed-Fi sandbox.""" + from src.spine.fetch import fetch_state_swagger + fetch_state_swagger(state.upper(), school_year=school_year, base_url=base_url) + + +@spine.command("build") +@click.option("--state", type=_STATE_CHOICES, required=True) +def spine_build(state: str) -> None: + """Build per-state spine manifest from cached Swagger.""" + from src.spine.build import build_state_spine + build_state_spine(state.upper()) + + +@spine.command("domain-map") +@click.option("--source", type=click.Choice(["AZ", "WI", "MN"], case_sensitive=False), + default="AZ", show_default=True, + help="State whose cached swagger carries x-Ed-Fi-domains tags.") +def spine_domain_map(source: str) -> None: + """(Re)generate data/spine/edfi_domain_map.json from a domain-rich state swagger.""" + from src.spine.build import regenerate_domain_map + regenerate_domain_map(source.upper()) + + +@cli.group() +def ingest() -> None: + """Per-state ingestion adapters (AZ, WI, MN, TX, IN).""" + + +# The five per-state ingest commands are generated from the one +# state→module registry (`src.ingest.INGEST_MODULES` — issue #213 +# item 3). Help text stays per-state; behavior (lazy import + call the +# module-level plain `run()`) is identical to the former hand-written +# commands. +_INGEST_HELP: dict[str, str] = { + "AZ": "Ingest Arizona (XLSX + PDFs) and enrich spine.", + "WI": "Ingest Wisconsin (Confluence) and enrich spine.", + "MN": "Ingest Minnesota (GitHub / MetaEd) and enrich spine.", + "TX": "Ingest Texas — TWEDS v33 source, spine-enriched via local TSDS SDK.", + "IN": "Ingest Indiana — IDOE Vendor Documentation XLSX, spine-enriched.", +} + + +def _make_ingest_command(code: str) -> click.Command: + @ingest.command(code.lower(), help=_INGEST_HELP[code]) + def _ingest_state() -> None: + import importlib + + module = importlib.import_module(f"src.ingest.{INGEST_MODULES[code]}") + module.run() + + return _ingest_state + + +for _code in SUPPORTED_STATES: + _make_ingest_command(_code) + + +@ingest.command("swagger-backfill") +@click.option( + "--state", + type=_STATE_CHOICES_WITH_ALL, + required=True, + help="State code, or 'all' to backfill AZ + WI + MN + TX + IN in one pass.", +) +def ingest_swagger_backfill(state: str) -> None: + """Append swagger-as-source rows for entities the state's source doc is silent on. + + Reads ``{state}_elements_source.json`` + ``{state}_spine.json`` and rewrites + both lens artifacts: appends synthetic source-lens rows carrying + ``documentation_source="swagger"`` for spine-only entities, and flips the + matching spine-lens rows from ``documented=False`` to ``documented=True``. + Issue #70 — swagger publication counts as state documentation when the + primary source is silent on a whole entity. + """ + from src.ingest.swagger_backfill import run as run_backfill + + for code in _expand_states(state): + run_backfill(code) + + +@ingest.command("gap") +@click.option( + "--state", + type=_STATE_CHOICES_WITH_ALL, + required=True, + help="State code, or 'all' to surface gaps for AZ + WI + MN + TX + IN in one pass.", +) +def ingest_gap(state: str) -> None: + """Surface spine-anchored coverage gaps to data/out/{state}_elements_gap.json. + + Reads {state}_elements_source.json + {state}_spine.json and writes a + sibling gap artifact enumerating spine (entity, element) pairs the + state's source doc is silent on. Source-lens output is byte-unchanged + (the gap is a separate artifact). Issue #66 Layer 2. + """ + from src.ingest.gap_surfacer import run as run_gap + + for code in _expand_states(state): + run_gap(code) + + +@cli.group() +def report() -> None: + """Cross-state coverage and analyst exports.""" + + +@report.command("coverage") +@_lens_option( + help=( + "Which lens to report on. 'source' reads {state}_elements_source.json " + "(source-driven shape). 'spine' selects the API-model lens " + "(flag value kept for operator stability) — reads " + "{state}_elements_spine.json, the Ed-Fi Swagger/API-model-" + "enumerated, source-enriched shape." + ), +) +@_allow_stale_option() +def report_coverage(lens: str, allow_stale: bool) -> None: + """Write data/out/coverage_report{_lens}.{json,md}.""" + from src.report.coverage import run as run_coverage + run_coverage(lens=lens.lower(), allow_stale=allow_stale) + + +@report.command("analyst") +@click.option("--state", type=_STATE_CHOICES) +@click.option( + "--all", "all_states", is_flag=True, + help="Produce per-state workbooks + combined coverage workbook " + "(this is already the default when --state is omitted; flag " + "kept for invocation compatibility).", +) +@_lens_option( + help=( + "Analyst workbook lens. 'source' (default) produces " + "{state}_analyst.xlsx. 'spine' selects the API-model lens (flag " + "value kept for operator stability) — produces " + "{state}_analyst_spine.xlsx with an AI: Documented column, " + "per-state 'Documented only' sheets, and the API Model Gaps " + "sheet." + ), +) +@click.option( + "--with-human", + type=click.Path(exists=True, dir_okay=False, path_type=Path), + default=None, + help=( + "Human-scored workbook to compare against: appends the Cmp: " + "Status / Base Δ / Adj Δ / Why columns to the Details sheet " + "(the June follow-up comparison format) and writes SEPARATE " + "*_with_human.xlsx copies — the pipeline deliverables on disk " + "are never overwritten. Columns are located by header name; " + "pass --human-config for a known per-state file." + ), +) +@click.option( + "--human-config", + type=click.Choice( + ["arizona", "wisconsin", "minnesota", "texas", "indiana"], + case_sensitive=False, + ), + default=None, + help="Use a known per-state workbook column mapping " + "(docs/human-scored-files/ basis) instead of header-name " + "autodetection.", +) +@_allow_stale_option() +def report_analyst( + state: str | None, + all_states: bool, + lens: str, + with_human: Path | None, + human_config: str | None, + allow_stale: bool, +) -> None: + """Produce analyst XLSX matching NACHOS Template shape.""" + # `all_states` is accepted for invocation compat but not forwarded: + # `run()` always produces per-state + combined when `state` is None + # (the old parameter was never read — issue #213 item 1 dead surface). + del all_states + from src.report.analyst import run as run_analyst + run_analyst( + state=state.upper() if state else None, + lens=lens.lower(), + with_human=with_human, + human_config=human_config.lower() if human_config else None, + allow_stale=allow_stale, + ) + + +@report.command("audit") +@click.option("--state", type=_STATE_CHOICES, required=True) +@_lens_option( + help="Which lens's rows/facts to audit ('spine' = API-model lens).", +) +def report_audit(state: str, lens: str) -> None: + """Write the on-demand audit workbook {state}_audit{_spine}.xlsx. + + The full Audit Trail surface (every row, every fact/span/dim/rule + column) — removed from the analyst deliverables under Option D and + kept one command away here. Row # remains the shared address into + the analyst workbook's Details sheet. + """ + from src.report.audit import run as run_audit + + path = run_audit(state=state.upper(), lens=lens.lower()) + click.echo(f"wrote {path}") + + +@report.command("scoring") +@_lens_option( + default="spine", + help="Lens whose sidecars feed the rollup (reads data/out/{state}_scores_{lens}.json).", +) +@_state_all_option( + default="all", + help="State(s) to include. Use 'all' for AZ+WI+MN+TX+IN.", +) +@_allow_stale_option() +def report_scoring(lens: str, state_opt: str, allow_stale: bool) -> None: + """Write data/out/scoring_report_{lens}.{json,md} — Phase D cross-state rollup.""" + from src.report.scoring import run as run_scoring + + states = _expand_states(state_opt) + report = run_scoring( + lens=lens.lower(), states=states, allow_stale=allow_stale + ) + cs = report["cross_state"] + csm = cs["cross_state_mean_quality"] + csm_str = "n/a" if csm is None else f"{csm:.2f}" + click.echo( + f"scoring rollup ({lens}): {cs['state_count']} states · " + f"{cs['record_total']:,} records · " + f"{cs['review_total']:,} review flags · " + f"mean quality {csm_str}" + ) + + +@report.command("review-queue") +@_lens_option( + default="spine", + help="Lens whose sidecars feed the review queue.", +) +@click.option( + "--top-n", + type=int, + default=30, + show_default=True, + help="Max flagged rows rendered in the MD listing (JSON carries every row).", +) +@_allow_stale_option() +def report_review_queue(lens: str, top_n: int, allow_stale: bool) -> None: + """Write data/out/review_queue_{lens}.{json,md} — Phase D review-queue routing.""" + from src.report.review_queue import run as run_review_queue + + queue = run_review_queue( + lens=lens.lower(), top_n=top_n, allow_stale=allow_stale + ) + totals = queue["route_totals"] + click.echo( + f"review queue ({lens}): " + + ", ".join(f"{k}={v:,}" for k, v in totals.items()) + + f" — {len(queue['entries']):,} flagged total" + ) + + +@report.command("review-digest") +@_lens_option( + required=True, + help="Lens whose sidecars the reviewer file is compared against.", +) +@click.option( + "--top-n", + type=int, + default=10, + show_default=True, + help="Max divergence patterns rendered in the MD digest.", +) +@_allow_stale_option() +def report_review_digest(lens: str, top_n: int, allow_stale: bool) -> None: + """Phase E: write data/out/review_digest_{lens}.{json,md}. + + Compares the five per-state human-scored workbooks + (docs/human-scored-files/ — hand-placed, gitignored) against POC-3 + per-record sidecars. Human-scored framing — never labels either + side correct. + """ + from src.report.review_digest import run as run_review_digest + + digest = run_review_digest( + lens=lens.lower(), top_n=top_n, allow_stale=allow_stale + ) + overall = digest["overall"] + click.echo( + f"review-digest ({lens}): " + f"{overall['matched']:,}/{overall['reviewer_rows']:,} matched " + f"({overall['match_pct']:.1f}%) · " + f"top_patterns={len(digest['top_patterns'])}" + ) + + +@report.command("reviewer-comparison") +def report_reviewer_comparison() -> None: + """Refresh ``docs/reviewer-comparison.md`` from current review-digest output. + + Reads ``data/out/review_digest_{source,spine}.json`` plus one source- + lens sidecar per state for the ``scoring_plan_version`` stamp and + writes the committed ``docs/reviewer-comparison.md`` summary doc. + Run after ``mc report review-digest --lens source`` and ``--lens + spine`` complete; commit the regenerated doc on the same PR that + bumps ``SCORING_PLAN_VERSION`` (CLAUDE.md operator playbook). + """ + from src.report.reviewer_comparison_summary import run as run_summary + + summary = run_summary() + plan_v = summary.get("scoring_plan_version") or "unknown" + lenses = ", ".join(summary.get("lenses") or []) + click.echo( + f"reviewer-comparison: docs/reviewer-comparison.md refreshed " + f"(scoring_plan_version={plan_v}, lenses={lenses})" + ) + + +@report.command("divergence") +@_allow_stale_option() +def report_divergence(allow_stale: bool) -> None: + """Write data/out/lens_divergence.{json,md} — cross-lens comparison per state. + + Requires both {state}_elements_source.json and {state}_elements_spine.json + for each state (regenerate with `mc ingest ` if missing). + """ + from src.report.divergence import run as run_divergence + run_divergence(allow_stale=allow_stale) + + +def _run_human_score_backfill(config_name: str, output_path: Path | None) -> None: + from src.report.human_score_backfill import run as run_backfill + + out = run_backfill(config_name, output_path=output_path) + click.echo(f"{config_name}: workbook written to {out}") + + +@report.command("human-overlay") +@click.option( + "--config", + "config_name", + required=True, + # Hardcoded to honor cli.py's lazy-import convention (importing + # `human_score_backfill.CONFIGS` at decoration time would defeat it). + # `tests/test_report_human_score_backfill.py` pins this Choice list + # against `set(CONFIGS)` so the two cannot drift. + type=click.Choice( + ["arizona", "wisconsin", "minnesota", "texas", "indiana"], + case_sensitive=False, + ), + help=( + "Which per-state human-scored workbook to overlay (see " + "human_score_backfill.CONFIGS — derived from " + "review_loader.REVIEWER_SOURCES, the docs/human-scored-files/ " + "basis). Each writes data/out/{name}_with_mc_scores.xlsx." + ), +) +@click.option( + "--output-path", + type=click.Path(dir_okay=False, path_type=Path), + default=None, + help="Override output xlsx (defaults to the config's data/out path).", +) +def report_human_overlay(config_name: str, output_path: Path | None) -> None: + """Issue #136: overlay a human-scored workbook with POC-3 ai- columns. + + One engine (`human_score_backfill.build_workbook`), one command, + five per-state configs (2026-07-07 basis — the former + training-20pct/training-file/nachos-arizona configs retired with + the single training file). Copies the origin columns verbatim and + appends the ai-prefixed source-lens Reviewer View cells for every + row whose (entity, element) resolves to a POC-3 sidecar + (gap-artifact fallthrough included). Read-only against existing + sidecars. + """ + _run_human_score_backfill(config_name.lower(), output_path) + + +@report.command("recommendations") +@click.option( + "--state", + type=_STATE_CHOICES, + default="AZ", + show_default=True, + help="State whose scored sidecar feeds the recommendations.", +) +@_lens_option( + choices=("source", "spine", "gap"), + help=( + "Lens whose sidecar drives the recommendation templates. " + "``source`` and ``spine`` walk the per-dimension rule cascade; " + "``gap`` (issue #73) emits one record-level rec per spine-anchored " + "gap row plus an optional structural-depth callout." + ), +) +@click.option( + "--hero-n", + type=int, + default=4, + show_default=True, + help="Number of hero examples to surface at the top of the MD digest.", +) +@click.option( + "--emit-md", + is_flag=True, + default=False, + help=( + "Also write the markdown digest (1-5 MB per state/lens; no " + "downstream consumer — R4 artifact diet made JSON-only the " + "default)." + ), +) +@_allow_stale_option() +def report_recommendations( + state: str, lens: str, hero_n: int, emit_md: bool, allow_stale: bool +) -> None: + """State-facing recommendations — deterministic template layer (Track C).""" + from src.report.recommendations import run as run_recommendations + + result = run_recommendations( + state=state.upper(), + lens=lens.lower(), # type: ignore[arg-type] + hero_n=hero_n, + emit_md=emit_md, + allow_stale=allow_stale, + ) + click.echo( + f"recommendations ({result['state']}/{result['lens']}): " + f"{result['rows_with_recommendations']:,} rows with recs · " + f"{result['recommendations_total']:,} total · " + f"{result['rows_at_target']:,} already at target" + ) + + +@cli.group() +def score() -> None: + """Scoring pipeline — extract binary facts, apply rules, aggregate per-record.""" + + +@score.command("extract") +@click.option( + "--fact", + type=str, + default="has_conditional_logic", + show_default=True, + help=( + "Binary or count-int fact to extract. Phase B LLM facts: " + "has_conditional_logic, definition_is_implementable, " + "required_when_stated, conditional_reporting_stated, " + "populations_or_scope_stated, has_cross_entity_logic, " + "has_aggregation, cross_entity_targets. " + "Deterministic facts: definition_present, business_rules_present, " + "data_type_canonical." + ), +) +@click.option( + "--state", + type=_STATE_CHOICES, + default="AZ", + show_default=True, + help="State to score (Phase B fans out to all four).", +) +@_lens_option( + default="spine", + help="Lens to score ('spine' = API-model lens; both lenses supported).", +) +@click.option( + "--limit", + type=int, + default=200, + show_default=True, + help="Max documented records to extract (sorted by entity,element_name).", +) +@click.option( + "--dry-run", + is_flag=True, + default=False, + help="Render prompts + manifest without hitting the LLM. No API key required.", +) +@click.option( + "--cost-cap", + type=float, + default=1.0, + show_default=True, + help="Hard USD cap on this run; halts before the next LLM call once exceeded.", +) +@_model_option() +@click.option( + "--validate-only", + type=int, + default=None, + metavar="N", + help=( + "Sanity-run N records and print a downgrade-reason histogram. " + "Writes the artifact to data/out/scoring/phase_a/validate_only/ " + "so the committed per-state JSONL is untouched. Useful before " + "fanning out a new fact/prompt across all 4 states. " + "Bounded cost — N rows worth of API calls." + ), +) +def score_extract( + fact: str, + state: str, + lens: str, + limit: int, + dry_run: bool, + cost_cap: float, + model: str | None, + validate_only: int | None, +) -> None: + """Extract one binary fact for one state/lens (Phase A harness). + + Exit codes: 0 success, 2 usage error, 3 schema drift, 4 API failure + after retries, 5 cost cap hit. + """ + from src.score.extract import DEFAULT_MODEL, run as run_extract + from src.score.schema import CostCapExceeded, ScoringSchemaError + + try: + import anthropic # noqa: F401 (imported lazily — only needed for non-dry-run API calls) + except ImportError: + if not dry_run: + click.echo("error: anthropic SDK not installed — run `uv sync`", err=True) + raise SystemExit(4) + + resolved_model = model or DEFAULT_MODEL + + from src.cli_render import echo_extract_result, render_extract_progress + + try: + header = run_extract( + fact=fact, + state=state.upper(), + lens=lens.lower(), + limit=limit, + dry_run=dry_run, + cost_cap=cost_cap, + model=resolved_model, + progress_callback=None if dry_run else render_extract_progress, + validate_only=validate_only, + ) + except ScoringSchemaError as exc: + click.echo(f"error: LLM response failed schema validation: {exc}", err=True) + raise SystemExit(3) + except CostCapExceeded as exc: + click.echo(f"error: {exc}", err=True) + raise SystemExit(5) + + echo_extract_result(header, fact=fact, state=state, lens=lens.lower()) + + +@score.command("estimate") +@_state_all_option( + default="AZ", + help="State(s) to estimate. Use 'all' for AZ+WI+MN+TX+IN.", +) +@_lens_option( + default="spine", + choices=("spine", "source"), + help="Lens to estimate ('spine' = API-model lens; both lenses supported).", +) +@click.option( + "--fact", + "fact_opt", + type=str, + default=None, + help=( + "Comma-separated fact names, or 'all' for every LLM fact in the " + "selected lens's roster. " + "Mutually exclusive with --facts. Facts without an authored prompt " + "use has_conditional_logic.md as a proxy template (marked '*')." + ), +) +@click.option( + "--facts", + "facts_opt", + type=str, + default=None, + help="Alias for --fact (e.g., --facts all). Mutually exclusive with --fact.", +) +@click.option( + "--limit", + type=int, + default=None, + help="Max documented records per state (sorted by entity,element_name).", +) +@_model_option() +@click.option( + "--band/--no-band", + default=True, + show_default=True, + help="Print the ±40% uncertainty range band alongside the point estimate.", +) +@click.option( + "--write/--no-write", + default=False, + show_default=True, + help="Also write the markdown to data/out/scoring/phase_b/estimate.md.", +) +def score_estimate( + state_opt: str, + lens: str, + fact_opt: str | None, + facts_opt: str | None, + limit: int | None, + model: str | None, + band: bool, + write: bool, +) -> None: + """Estimate LLM cost per (state, fact) without spending (budget-gating). + + Uses `anthropic.messages.count_tokens()` (non-billable) when + ANTHROPIC_API_KEY is set; else falls back to a chars/4 heuristic. + Output tokens are projected from Phase A's 18.6% out/in ratio with a + ±40% range band. + """ + from src.score.estimate import ( + DEFAULT_MODEL, + facts_llm_for_lens, + format_markdown, + run as run_estimate, + ) + from src.utils.paths import project_root, scoring_phase_b_estimate_path + + if fact_opt and facts_opt: + click.echo("error: --fact and --facts are mutually exclusive", err=True) + raise SystemExit(2) + lens_facts = facts_llm_for_lens(lens.lower()) + raw_facts = fact_opt or facts_opt or "has_conditional_logic" + if raw_facts.strip().lower() == "all": + selected_facts: list[str] = list(lens_facts) + else: + selected_facts = [f.strip() for f in raw_facts.split(",") if f.strip()] + unknown = [f for f in selected_facts if f not in lens_facts] + if unknown: + click.echo( + f"error: unknown fact(s) {unknown}; {lens.lower()}-lens LLM facts " + f"are {list(lens_facts)}", + err=True, + ) + raise SystemExit(2) + + states = list(_expand_states(state_opt)) + + result = run_estimate( + states=states, + facts=selected_facts, + lens=lens.lower(), + limit=limit, + model=model or DEFAULT_MODEL, + ) + markdown = format_markdown(result, show_band=band) + click.echo(markdown, nl=False) + + if write: + path = scoring_phase_b_estimate_path() + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(markdown, encoding="utf-8") + click.echo(f"\nwrote {path.relative_to(project_root())}") + + +@score.command("run-all") +@_state_all_option( + default="all", + help="State(s) to run. Use 'all' for AZ+WI+MN+TX+IN.", +) +@click.option( + "--lens", + type=click.Choice(["spine", "source"], case_sensitive=False), + default="spine", + show_default=True, + help=( + "Lens to score. Spine fans out the Phase B roster; source adds " + "the Phase C2 deterministic + LLM facts so `aggregate --lens " + "source` finds every artifact it needs." + ), +) +@click.option( + "--facts", + "facts_opt", + type=str, + default="all", + show_default=True, + help=( + "Comma-separated fact names, or 'all' for every fact in the " + "current lens's roster (spine = Phase B; source = Phase B + " + "source-lens deterministic + Phase C2)." + ), +) +@click.option( + "--limit", + type=int, + default=None, + help="Max documented records per state (sorted by entity,element_name).", +) +@click.option( + "--cost-cap", + type=float, + default=50.0, + show_default=True, + help="Global USD cap across all (state, fact) pairs.", +) +@_model_option() +@click.option( + "--checkpoint-after", + type=_STATE_CHOICES, + default=None, + help=( + "Pause after the named state's fact fanout completes and prompt " + "for y/n to continue. Use with a new fact's first cold run " + "(e.g., --checkpoint-after AZ) so a polarity / prompt bug that " + "fires mid-fanout stops after the cheapest state instead of " + "walking through WI/MN/TX at full spend." + ), +) +@click.option( + "--yes", + "assume_yes", + is_flag=True, + default=False, + help=( + "Skip the interactive prompt at --checkpoint-after (proceed " + "automatically). Also OKs the --batch cost gate when used with " + "--batch. Useful in CI or scripted runs where you want the " + "checkpoint summary in the log but can't respond to the " + "confirmation." + ), +) +@click.option( + "--batch", + "batch_mode", + is_flag=True, + default=False, + help=( + "Submit cache-miss prompts to the Anthropic Messages Batch API " + "(50%% off, 24h SLA) instead of the synchronous API path. Prints " + "the batch_id(s) and exits — collect with `mc score batches " + "collect ` once Anthropic finishes. No artifacts " + "are written by submit; the next normal `score run-all` builds " + "sidecars from the (now cached) responses at $0 LLM." + ), +) +@click.option( + "--max-cost", + type=float, + default=10.0, + show_default=True, + help=( + "Estimated-cost gate for --batch. Submission aborts if the " + "estimated batch-tier USD exceeds this value unless --yes is " + "also set. Ignored without --batch." + ), +) +def score_run_all( + state_opt: str, + lens: str, + facts_opt: str, + limit: int | None, + cost_cap: float, + model: str | None, + checkpoint_after: str | None, + assume_yes: bool, + batch_mode: bool, + max_cost: float, +) -> None: + """Run every (state, fact) pair with skip-if-complete + streaming manifest. + + Idempotent: rerun after a crash or deliberate kill and only + unfinished pairs execute; completed pairs read their cached + artifact headers and are skipped. Cache replay in the extractor + makes partial pairs cheap to resume. + + With ``--batch``, this becomes a Batch API submission instead. + Renders every prompt the sync path would call, filters cache hits, + submits the misses to ``messages.batches.create``, prints the + ``batch_id``(s), and exits. Collect with ``score batches collect + `` once the batch ends; then re-run ``score run-all`` + (no flag) to build sidecars at $0 from the populated cache. + """ + from src.score.runner import ( + phase_b_facts_for_lens, + run_all as run_all_runner, + ) + + lens_facts = phase_b_facts_for_lens(lens.lower()) + if facts_opt.strip().lower() == "all": + selected_facts = list(lens_facts) + else: + selected_facts = [f.strip() for f in facts_opt.split(",") if f.strip()] + unknown = [f for f in selected_facts if f not in lens_facts] + if unknown: + click.echo( + f"error: unknown fact(s) {unknown}; currently available: " + f"{list(lens_facts)}", + err=True, + ) + raise SystemExit(2) + + states = list(_expand_states(state_opt)) + + from src.cli_render import ( + echo_batch_submission, + render_run_all_checkpoint, + render_run_all_pair, + ) + + # One home for the model ID (issue #213 item 3): extract.DEFAULT_MODEL + # (re-exported from score.client). Imported lazily per the CLI + # convention. + from src.score.extract import DEFAULT_MODEL + + resolved_model = model or DEFAULT_MODEL + + if batch_mode: + if checkpoint_after: + click.echo( + "error: --checkpoint-after is incompatible with --batch (no per-pair " + "progress to checkpoint in batch submission)", + err=True, + ) + raise SystemExit(2) + from src.score.batch_runner import BatchCostGateError, submit_run_all + try: + manifests = submit_run_all( + states=states, + lens=lens.lower(), + facts=selected_facts, + limit=limit, + model=resolved_model, + max_cost_usd=max_cost, + confirm=assume_yes, + ) + except BatchCostGateError as exc: + click.echo(f"error: {exc}", err=True) + raise SystemExit(2) + if not manifests: + click.echo( + "All prompts already cached — no batch submitted. Run " + "`mc score run-all` (no --batch) to build sidecars from cache." + ) + return + echo_batch_submission(manifests) + return + + def _checkpoint(state: str, manifest) -> bool: + return render_run_all_checkpoint(state, manifest, assume_yes=assume_yes) + + manifest = run_all_runner( + states=states, + facts=selected_facts, + lens=lens.lower(), + limit=limit, + model=resolved_model, + cost_cap=cost_cap, + progress=render_run_all_pair, + checkpoint_after=checkpoint_after, + checkpoint_callback=_checkpoint if checkpoint_after else None, + ) + click.echo( + f"\nrun-all: {len(manifest.pairs)} pairs · " + f"${manifest.total_usd:.4f} total · cap ${cost_cap:.2f}" + ) + + +@score.command("aggregate") +@_state_all_option( + default="all", + help="State(s) to aggregate. Use 'all' for AZ+WI+MN+TX+IN.", +) +@_lens_option( + default="spine", + choices=("spine", "source"), + help="Lens to aggregate — `spine` (Phase C1) or `source` (Phase C2).", +) +@click.option( + "--model", + type=str, + default="claude-sonnet-4-6", + show_default=True, + help="Model-id recorded in the sidecar header (metadata only).", +) +@click.option( + "--prompt-version", + type=str, + default="phase-a.v1", + show_default=True, + help="Prompt-version recorded in the sidecar header (metadata only).", +) +@click.option( + "--allow-stale", + is_flag=True, + help="Proceed when the publish manifest marks the source-lens " + "sidecar stale/missing (spine extension adjustments degrade " + "to the conservative unresolved fallback).", +) +def score_aggregate( + state_opt: str, lens: str, model: str, prompt_version: str, + allow_stale: bool, +) -> None: + """Score every record: join facts → rules → per-record sidecar. + + Reads per-fact JSONL artifacts from data/out/scoring/phase_a/ and + writes data/out/{state}_scores_{lens}.json per plan §8.1. + """ + from src.score.aggregate import run as run_aggregate, run_all as run_all_aggregate + + if state_opt.lower() == "all": + headers = run_all_aggregate( + lens=lens.lower(), model=model, prompt_version=prompt_version, + allow_stale=allow_stale, + ) + else: + headers = [ + run_aggregate( + state=state_opt.upper(), + lens=lens.lower(), + model=model, + prompt_version=prompt_version, + allow_stale=allow_stale, + ) + ] + + for h in headers: + quality = h.get("mean_quality_score") + quality_str = "n/a" if quality is None else f"{quality:.2f}" + click.echo( + f" [{h['state']}] records={h['record_count']} " + f"mean_quality={quality_str} " + f"review_flags={h['needs_review_count']}" + ) + + +@score.command("aggregate-gap") +@_state_all_option( + default="all", + help="State(s) to aggregate. Use 'all' for AZ+WI+MN+TX+IN.", +) +@click.option( + "--model", + type=str, + default="deterministic", + show_default=True, + help="Model-id recorded in the sidecar header (metadata only).", +) +@click.option( + "--prompt-version", + type=str, + default="step1.det.v1", + show_default=True, + help="Prompt-version recorded in the sidecar header (metadata only).", +) +@click.option( + "--with-llm", + is_flag=True, + default=False, + help=( + "Read LLM-fact artifacts from data/out/scoring/phase_a_gap/ in " + "addition to deterministic facts. Step 2 / Step 3 callers set " + "this after running `score gap-extract`. Default is Step 1 " + "deterministic-only." + ), +) +def score_aggregate_gap( + state_opt: str, model: str, prompt_version: str, with_llm: bool +) -> None: + """Score spine-anchored gap rows; emit data/out/{state}_scores_gap.json. + + Issue #73 Step 1: deterministic-only pass over `{state}_elements_gap.json`. + Every emitted record carries `discovery_lens="spine_anchored"`. + LLM-dependent dimensions surface at tier 0 with low confidence — + Step 2 / Step 3 (LLM extraction) re-runs through the same pipeline + with `--with-llm` and lifts those dimensions onto real values. + """ + from src.score.aggregate_gap import ( + _phase_a_gap_dir, + run as run_gap, + run_all as run_all_gap, + ) + + llm_dir = _phase_a_gap_dir() if with_llm else None + + if state_opt.lower() == "all": + headers = run_all_gap( + model=model, + prompt_version=prompt_version, + llm_artifact_dir=llm_dir, + ) + else: + headers = [ + run_gap( + state=state_opt.upper(), + model=model, + prompt_version=prompt_version, + llm_artifact_dir=llm_dir, + ) + ] + + for h in headers: + quality = h.get("mean_quality_score") + quality_str = "n/a" if quality is None else f"{quality:.2f}" + click.echo( + f" [{h['state']}] gap_records={h['record_count']} " + f"mean_quality={quality_str} " + f"review_flags={h['needs_review_count']} " + f"in_scope={h['in_scope_count']}" + ) + + +@score.command("gap-extract") +@_state_all_option( + default="all", + help="State(s) to extract gap-row LLM facts for.", +) +@click.option( + "--sample-pct", + type=float, + default=0.10, + show_default=True, + help=( + "Fraction of gap rows to sample per state. Step 2 default: 0.10 " + "(10 % stratified). Step 3 / full extract: 1.0." + ), +) +@click.option( + "--seed", + type=int, + default=73, + show_default=True, + help="PRNG seed for sample selection. Default 73 = issue number.", +) +@click.option( + "--cost-cap", + type=float, + default=25.0, + show_default=True, + help=( + "Per-state cost ceiling across all 12 LLM facts. Step 2 default " + "$25 gives ~75 %% headroom over the ~$14 expected spend on a " + "10 %% sample of one state." + ), +) +@click.option( + "--dry-run", + is_flag=True, + default=False, + help="Render prompts without hitting the LLM. No API key required.", +) +def score_gap_extract( + state_opt: str, + sample_pct: float, + seed: int, + cost_cap: float, + dry_run: bool, +) -> None: + """Run the 12 spine-lens LLM facts over sampled gap rows (issue #73 Step 2/3). + + Sampled rows feed through the existing `extract.run()` pipeline; + artifacts land under `data/out/scoring/phase_a_gap/` so they cannot + collide with the source/spine artifacts under `phase_a/`. Re-run + `mc score aggregate-gap --with-llm` afterward to fold the new LLM + values into the gap sidecars. + """ + from src.score.gap_extract import run_all as run_all_gap_extract + + from src.cli_render import echo_gap_extract_results + + states = list(_expand_states(state_opt)) + results = run_all_gap_extract( + states=states, + sample_pct=sample_pct, + seed=seed, + cost_cap=cost_cap, + dry_run=dry_run, + ) + echo_gap_extract_results(results) + + +@score.command("gap-downgrade-summary") +@_state_all_option( + default="all", + help="State(s) to summarize.", +) +def score_gap_downgrade_summary(state_opt: str) -> None: + """Print per-fact downgrade rates across the gap-extract artifacts. + + Issue #73 Step 2 verification: halt before Step 3 if any fact's + downgrade rate exceeds 30 % on the sample. + """ + from src.cli_render import echo_gap_downgrade_summary + from src.score.gap_extract import downgrade_summary + + states = list(_expand_states(state_opt)) + summary = downgrade_summary(states=states) + + if not summary["states_loaded"]: + click.echo("No gap-extract artifacts found.") + click.echo(" Run `mc score gap-extract` first.") + return + + echo_gap_downgrade_summary(summary) + + +@score.command("peer-gap") +@click.option( + "--dry-run", + is_flag=True, + default=False, + help="Render prompts + manifest without hitting the LLM. No API key required.", +) +@click.option( + "--cost-cap", + type=float, + default=15.0, + show_default=True, + help=( + "Hard USD cap on this run; halts before the next slot once " + "exceeded. Full multi-state fanout at ~230 slots costs ~$7 at " + "the prompt's typical $0.03/slot; default gives ~2x headroom." + ), +) +@click.option( + "--slot-set", + type=click.Choice(["all-multi-state", "seed"], case_sensitive=False), + default="all-multi-state", + show_default=True, + help=( + "'all-multi-state' fans out to every (entity, element) spine " + "slot documented in ≥ 2 states (production default). 'seed' " + "replays the historical 10-slot pilot — useful for " + "reproducibility checks; cache replay makes it $0 after the " + "original 2026-04-24 run." + ), +) +@_model_option() +def score_peer_gap( + dry_run: bool, cost_cap: float, slot_set: str, model: str | None, +) -> None: + """Run the peer-state triangulation fanout — Wave 2 Mitigation 4. + + Reads each multi-state (entity, element) bundle's per-state + narratives from the spine-lens elements files + structural tier + from the spine-lens scoring sidecars, prompts the LLM for one + cross-state synthesis per slot, and writes + ``data/out/scoring/phase_a/peer_gap.jsonl``. + + Default slot set is ``all-multi-state`` — every spine slot + documented in ≥ 2 states. The ``seed`` set replays the original + 10-slot pilot for reproducibility. + """ + from src.score.peer_gap import ( + DEFAULT_MODEL, + SEED_SLOTS, + all_multi_state_slots, + run as run_peer_gap, + ) + from src.score.schema import CostCapExceeded, ScoringSchemaError + + try: + import anthropic # noqa: F401 + except ImportError: + if not dry_run: + click.echo("error: anthropic SDK not installed — run `uv sync`", err=True) + raise SystemExit(4) + + resolved_model = model or DEFAULT_MODEL + + from src.cli_render import echo_peer_gap_result, render_peer_gap_progress + + if slot_set.lower() == "seed": + selected_slots = SEED_SLOTS + else: + selected_slots = all_multi_state_slots() + + try: + header = run_peer_gap( + slots=selected_slots, + model=resolved_model, + cost_cap=cost_cap, + dry_run=dry_run, + progress_callback=None if dry_run else render_peer_gap_progress, + ) + except ScoringSchemaError as exc: + click.echo(f"error: peer-gap response failed schema validation: {exc}", err=True) + raise SystemExit(3) + except CostCapExceeded as exc: + click.echo(f"error: cost cap exceeded: {exc}", err=True) + raise SystemExit(5) + + echo_peer_gap_result(header, dry_run=dry_run, cost_cap=cost_cap) + + +@score.group("batches") +def score_batches() -> None: + """Anthropic Messages Batch API helpers (list / status / collect). + + Use ``mc score run-all --batch`` to submit. The commands here + manage the lifecycle of a batch once Anthropic has accepted it: + + \b + mc score batches list # show submitted manifests + mc score batches status # fetch live processing_status + mc score batches collect # write results through cache + """ + + +@score_batches.command("list") +def score_batches_list() -> None: + """List submitted batch manifests on disk.""" + from src.score.batch_manifest import list_manifests + + manifests = list_manifests() + if not manifests: + click.echo("No batch manifests on disk.") + return + click.echo(f"{'batch_id':<48} {'submitted_at':<22} {'items':>6} {'est_usd':>9} model") + for m in manifests: + click.echo( + f"{m.batch_id:<48} {m.submitted_at:<22} {len(m.items):>6} " + f"${m.estimated_cost_usd:>7.2f} {m.model}" + ) + + +@score_batches.command("status") +@click.argument("batch_id") +def score_batches_status(batch_id: str) -> None: + """Fetch live processing_status for ``batch_id`` from Anthropic.""" + from src.score.batch_client import BatchClient + + client = BatchClient() + status = client.status(batch_id) + click.echo(f"batch_id: {status.batch_id}") + click.echo(f"processing_status: {status.processing_status}") + if status.ended_at: + click.echo(f"ended_at: {status.ended_at}") + if status.request_counts: + click.echo("request_counts:") + for key in ("processing", "succeeded", "errored", "canceled", "expired"): + if key in status.request_counts: + click.echo(f" {key:<12} {status.request_counts[key]}") + + +@score_batches.command("collect") +@click.argument("batch_id") +@_model_option() +def score_batches_collect(batch_id: str, model: str | None) -> None: + """Fetch results for ``batch_id`` and write through the cache. + + After this command, run ``mc score run-all`` (no --batch) to + build sidecars from the populated cache at $0 LLM. Failed items + (errored / parse_failed / canceled / expired) are not cached; + a sync re-run picks them up automatically. + """ + from src.score.batch_runner import collect_batch + + # One home for the model ID (issue #213 item 3): extract.DEFAULT_MODEL + # (re-exported from score.client). Imported lazily per the CLI + # convention. + from src.score.extract import DEFAULT_MODEL + + resolved_model = model or DEFAULT_MODEL + summary = collect_batch(batch_id, model=resolved_model) + click.echo( + f"batch {summary.batch_id}: {summary.succeeded_count} succeeded " + f"({summary.cache_writes} written to cache now)" + ) + if summary.parse_failed_count: + click.echo(f" parse_failed: {summary.parse_failed_count}") + if summary.errored_count: + click.echo(f" errored: {summary.errored_count}") + if summary.canceled_count: + click.echo(f" canceled: {summary.canceled_count}") + if summary.expired_count: + click.echo(f" expired: {summary.expired_count}") + click.echo(f" actual cost: ${summary.actual_cost_usd:.4f} (batch tier)") + + +@cli.command("publish") +@click.option("--state", "state", multiple=True, + type=_STATE_CHOICES_WITH_ALL, default=("all",), + show_default=True, + help="Restrict per-state stages to the named state(s); " + "repeatable. Cross-state stages then run only when " + "the other states' inputs are manifest-fresh.") +@click.option("--from", "from_stage", type=click.Choice(STAGE_NAMES), + default=None, + help="Start at this stage (forces it to run; later stages " + "follow normal freshness rules). See --dry-run for " + "stage names.") +@click.option("--skip", multiple=True, + help="Skip the named stage(s). Repeatable.") +@click.option("--reports-only", is_flag=True, + help="Shorthand for --from report-coverage.") +@click.option("--dry-run", is_flag=True, + help="Print the resolved plan (would_run / skipped and why) " + "without executing or writing anything.") +@click.option("--cost-cap", type=float, default=DEFAULT_COST_CAP, + show_default=True, + help="Cumulative LLM spend ceiling for this run (USD).") +@click.option("--yes", "assume_yes", is_flag=True, + help="Skip the LLM-stage confirmation gate.") +@click.option("--refresh-spine", is_flag=True, + help="Enable the spine-fetch stage (default: pinned " + "artifacts — the WS-3 pattern; TX needs the local " + "TSDS Docker stack).") +@click.option("--tx-base-url", default=DEFAULT_TX_BASE_URL, + show_default=True, + help="TX sandbox base URL (local TSDS stack) for " + "--refresh-spine.") +@click.option("--with-gap-llm", is_flag=True, + help="Enable the optional LLM gap-extract stage.") +@click.option("--with-spine-workbooks", is_flag=True, + help="Also produce the internal/QA API-model-lens " + "workbooks.") +@click.option("--lite", is_flag=True, + help="POC-Lite profile (docs/pipeline-lite.md): the " + "score-only path — ingest, source-lens extraction + " + "scoring, and the deliverable workbooks. Cuts the " + "API-model-lens scoring pass and the coverage/" + "divergence/reviewer-comparison analysis reports; the " + "free deterministic gap layer stays in so the " + "workbooks' Documentation Gaps sheet can't go stale. " + "Conflicts with --with-gap-llm/--with-spine-workbooks.") +def publish( + state: tuple[str, ...], + from_stage: str | None, + skip: tuple[str, ...], + reports_only: bool, + dry_run: bool, + cost_cap: float, + assume_yes: bool, + refresh_spine: bool, + tx_base_url: str, + with_gap_llm: bool, + with_spine_workbooks: bool, + lite: bool, +) -> None: + """Run the full refresh chain with ordering + a freshness manifest. + + Encodes the ~35-invocation operator playbook — including its four + silent ordering constraints — as one command. Warm reruns skip + every fresh stage; a changed input re-runs exactly its downstream + cone. See CLAUDE.md "Operator playbook" for the background. + """ + import sys + + from src.publish import run as run_publish + + # --state is repeatable (issue #213 item 3): "all" anywhere wins; + # otherwise dedupe in first-seen order. The orchestrator already + # takes an arbitrary states tuple (partial-roster handling included). + if any(s.lower() == "all" for s in state): + states = SUPPORTED_STATES + else: + states = tuple(dict.fromkeys(s.upper() for s in state)) + try: + result = run_publish( + states=states, + from_stage=from_stage, + skip=tuple(skip), + reports_only=reports_only, + dry_run=dry_run, + cost_cap=cost_cap, + assume_yes=assume_yes, + refresh_spine=refresh_spine, + tx_base_url=tx_base_url, + with_gap_llm=with_gap_llm, + with_spine_workbooks=with_spine_workbooks, + lite=lite, + ) + except ValueError as exc: + raise click.UsageError(str(exc)) from exc + click.echo(result.render_table()) + if result.failed: + sys.exit(1) + + +@cli.group() +def review() -> None: + """Analyst round-trip — ingest workbook edits back into curation.""" + + +@review.command("ingest") +@click.argument( + "workbook", + type=click.Path(exists=True, dir_okay=False, path_type=Path), +) +@click.option( + "--author", + type=str, + default=None, + help="Who made these edits (recorded on every captured value; " + "xlsx carries no cell-level authorship).", +) +@click.option( + "--dry-run", + is_flag=True, + help="Scan and report what would be captured without writing.", +) +def review_ingest(workbook: Path, author: str | None, dry_run: bool) -> None: + """Read the analyst-input band back into data/curation/{state}.json. + + Reads the green analyst-input columns from the workbook's Details + sheet, keys rows by (State, Entity Name, Data Element), validates + them against the elements artifact, and merges non-blank values into + the per-state curation sidecar (newest-wins per column, prior values + kept in history; blank cells never clear stored values). Every + ``mc report analyst`` regeneration re-applies the sidecar, so + workbook regeneration stops destroying analyst work (issue #186 + Option C / design §8.4). + """ + from src.report.curation import run as run_ingest + + report = run_ingest(workbook, author=author, dry_run=dry_run) + click.echo(report.render_text()) + + +@review.command("adjudicate") +@click.option("--state", required=True, help="State code (e.g. tx).") +@click.option("--entity", required=True, help="Entity Name, verbatim.") +@click.option("--element", required=True, help="Data Element, verbatim.") +@click.option( + "--value", + type=float, + required=True, + help="The team-consensus adjusted score.", +) +@click.option( + "--lens", + type=click.Choice(["source", "spine"]), + default="source", + show_default=True, + help="Which lens's engine score the consensus is about.", +) +@click.option( + "--agreed-by", + "agreed_by", + multiple=True, + required=True, + help="Repeatable — one per analyst in the consensus.", +) +@click.option( + "--rationale", + required=True, + help="Why the team's score differs from the engine's.", +) +@click.option( + "--allow-stale", + is_flag=True, + help="Stamp the engine score even if the publish manifest says the " + "scores sidecar is stale.", +) +@click.option( + "--dry-run", + is_flag=True, + help="Report what would be recorded without writing.", +) +def review_adjudicate( + state: str, + entity: str, + element: str, + value: float, + lens: str, + agreed_by: tuple[str, ...], + rationale: str, + allow_stale: bool, + dry_run: bool, +) -> None: + """Record team consensus for one row's adjusted score (issue #248). + + Writes an `adjudication` block into data/curation/{state}.json with + full provenance (who agreed, when, why, the engine score and plan + version at decision time). The engine score is NEVER modified — + the consensus renders as `Effective Score (adjudicated)` beside it, + and adjudications never feed prompts, rules, or tests (the GT + boundary applies to our own consensus too). If the engine score or + plan version later moves, the adjudication goes stale: it renders + blank and the row re-enters the Review Queue as RE-ADJUDICATE. + """ + from src.report.curation import adjudicate + + report = adjudicate( + state, + entity, + element, + value=value, + agreed_by=agreed_by, + rationale=rationale, + lens=lens, + allow_stale=allow_stale, + dry_run=dry_run, + ) + click.echo(report.render_text()) + + +@review.command("correct-fact") +@click.option("--state", required=True, help="State code (e.g. tx).") +@click.option("--entity", required=True, help="Entity Name, verbatim.") +@click.option("--element", required=True, help="Data Element, verbatim.") +@click.option( + "--fact", + required=True, + help="The LLM-extracted fact to correct (e.g. has_conditional_logic).", +) +@click.option( + "--value", + required=True, + help="The corrected value — true/false for bool facts, an integer " + "for count facts, an allowed token for enum facts.", +) +@click.option( + "--lens", + type=click.Choice(["source", "spine"]), + default="source", + show_default=True, + help="Which lens's extraction the correction is about.", +) +@click.option( + "--author", + required=True, + help="Who is asserting the corrected value.", +) +@click.option( + "--rationale", + required=True, + help="Why the extracted value is wrong.", +) +@click.option( + "--allow-stale", + is_flag=True, + help="Stamp the prior value even if the publish manifest says the " + "scores sidecar is stale.", +) +@click.option( + "--dry-run", + is_flag=True, + help="Report what would be recorded without writing.", +) +def review_correct_fact( + state: str, + entity: str, + element: str, + fact: str, + value: str, + lens: str, + author: str, + rationale: str, + allow_stale: bool, + dry_run: bool, +) -> None: + """Correct one LLM-extracted fact on one record (issue #249). + + Writes a `facts` block into data/curation/{state}.json with full + provenance (author, rationale, the prior extracted value and plan + version at correction time). The next `mc score aggregate` run + overlays the corrected value onto the fact pool BEFORE the rule + cascade runs, so the score recomputes for a reason the audit trail + fully explains — the fact's sidecar provenance becomes + `human_corrected`. The prompt cache is never written (it stays the + immutable record of what the model said). Deterministic facts are + rejected: a wrong deterministic fact is a code bug, not a curation + entry. Corrections never feed prompts, rules, or tests (GT + boundary); a correction pattern across many rows is a prompt + weakness — route it to the prompt-version-bump path. + """ + from src.report.curation import correct_fact + + report = correct_fact( + state, + entity, + element, + fact=fact, + value=value, + rationale=rationale, + author=author, + lens=lens, + allow_stale=allow_stale, + dry_run=dry_run, + ) + click.echo(report.render_text()) + + +def main() -> None: + """Console-script entry point for `mc` and `python -m mc`. + + Loads a `.env` (if present) before dispatching so that + `ANTHROPIC_API_KEY` and any future secrets don't need to be exported + into the shell. Already-set env vars win — `.env` is a default, not + an override. `.env` is gitignored; see `.env.example` for the + expected shape. + """ + from dotenv import load_dotenv + + # Search from the current working directory upward so both source + # runs and packaged exe runs can pick up the active .env. + load_dotenv(override=False) + cli(prog_name="mc") + + +if __name__ == "__main__": + main() diff --git a/apps/pipeline/src/cli_render.py b/apps/pipeline/src/cli_render.py new file mode 100644 index 0000000..12c8d18 --- /dev/null +++ b/apps/pipeline/src/cli_render.py @@ -0,0 +1,283 @@ +"""CLI progress/summary renderers (issue #213 item 3). + +The twin per-batch progress renderers and the post-run summary echo +blocks for ``mc score extract`` / ``mc score peer-gap`` lived +inline in ``cli.py``. They are pure presentation — every string here is +byte-identical to the pre-extraction ``cli.py`` output. ``cli.py`` +keeps only option parsing + dispatch. +""" + +from __future__ import annotations + +import click + + +# --------------------------------------------------------------------------- +# `mc score extract` +# --------------------------------------------------------------------------- + + +def render_extract_progress(event: dict) -> None: + """Per-entity-batch progress line for ``score extract``.""" + # Dry-run path never invokes the callback; keep output crisp. + tag = "cache" if event["cache_hit"] else "llm " + click.echo( + f" [{event['index'] + 1:>2}/{event['total']}] {tag} {event['entity']:<52s} " + f"elems={event['element_count']:>2} " + f"in={event['tokens_in']:>5} out={event['tokens_out']:>4} " + f"${event['usd']:.4f} (run ${event['running_usd']:.4f})" + + (f" downgrades={event['downgrades_in_batch']}" if event["downgrades_in_batch"] else "") + ) + + +def echo_extract_result( + header: dict, *, fact: str, state: str, lens: str +) -> None: + """Post-run summary for ``score extract`` (all modes).""" + mode = header.get("mode") + if mode == "dry-run": + click.echo( + f"dry-run: {header['entities_processed']} entity batches, " + f"{header['record_count']} records, manifest at " + f"{header['dry_run_manifest']}" + ) + elif mode == "deterministic": + true_count = header.get("true_count", 0) + total = header.get("record_count", 0) + pct = (true_count / total * 100) if total else 0.0 + click.echo( + f"deterministic {fact}: {true_count}/{total} true ({pct:.1f}%) · " + f"{header['entities_processed']} entities · $0.0000" + ) + elif mode == "validate-only": + print_validate_only_summary(header, fact=fact, state=state, lens=lens) + else: + click.echo( + f"extracted {header['scored_count']}/{header['record_count']} records " + f"across {header['entities_processed']} entities · " + f"${header['total_usd']:.4f} · " + f"{header['cache_hit_count']} cache hits · " + f"{header['downgrade_count']} downgrades" + ) + + +def print_validate_only_summary( + header: dict, *, fact: str, state: str, lens: str +) -> None: + """Surface downgrade-reason histogram for a ``--validate-only`` run. + + Reads the scratch artifact the harness wrote to + ``phase_a/validate_only/{state}_{lens}_{fact}.jsonl`` and emits a + per-reason count plus up to five sample rows per reason so a + polarity / schema bug is obvious on screen (Phase D carryover #3). + """ + import json as _json + from collections import Counter + + from src.utils.paths import scoring_phase_a_dir + + artifact = ( + scoring_phase_a_dir() + / "validate_only" + / f"{state.upper()}_{lens}_{fact}.jsonl" + ) + click.echo( + f"validate-only: {header['scored_count']}/{header['record_count']} records · " + f"${header['total_usd']:.4f} · " + f"{header['cache_hit_count']} cache hits · " + f"{header['downgrade_count']} downgrades" + ) + if not artifact.exists(): + click.echo(f" (no artifact at {artifact} — skipping sample list)") + return + reason_counter: Counter = Counter() + samples: dict[str, list[dict]] = {} + with artifact.open("r", encoding="utf-8") as fh: + for idx, line in enumerate(fh): + if idx == 0: + continue # header row + row = _json.loads(line) + reason = row.get("downgrade_reason") + if reason is None: + continue + reason_counter[reason] += 1 + samples.setdefault(reason, []).append(row) + if not reason_counter: + click.echo(" no downgrades — sanity clean") + return + click.echo(" downgrade-reason histogram:") + for reason, count in reason_counter.most_common(): + click.echo(f" {count:>3} {reason}") + click.echo(" samples (first 2 per reason):") + for reason, rows in samples.items(): + for row in rows[:2]: + click.echo( + f" [{reason}] {row.get('record_key', '?')} — " + f"llm_value={row.get('llm_value')!r} " + f"confidence={row.get('confidence')!r} " + f"spans={len(row.get('spans', []))}" + ) + + +# --------------------------------------------------------------------------- +# `mc score peer-gap` +# --------------------------------------------------------------------------- + + +def render_peer_gap_progress(event: dict) -> None: + """Per-slot progress line for ``score peer-gap``.""" + tag = "cache" if event["cache_hit"] else "llm " + confidence = event.get("confidence") or "?" + states = ",".join(event["states"]) + click.echo( + f" [{event['index'] + 1:>2}/{event['total']}] {tag} " + f"{event['slot_key']:<60s} states=[{states}] " + f"in={event['tokens_in']:>5} out={event['tokens_out']:>4} " + f"${event['usd']:.4f} (run ${event['running_usd']:.4f}) " + f"conf={confidence}" + ) + + +def echo_peer_gap_result( + header: dict, *, dry_run: bool, cost_cap: float +) -> None: + """Post-run summary for ``score peer-gap`` (all modes).""" + if dry_run: + click.echo( + f"peer-gap dry-run: {header['bundle_count']} bundles → " + f"{header['dry_run_manifest']}" + ) + else: + click.echo( + f"peer-gap complete: {header['scored_count']}/{header['bundle_count']} slots · " + f"cache hits {header['cache_hit_count']} · " + f"${header['total_usd']:.4f} total · cap ${cost_cap:.2f}" + ) + + +# --------------------------------------------------------------------------- +# `mc score run-all` +# --------------------------------------------------------------------------- + + +def render_run_all_pair(pair) -> None: + """Per-(state, fact) progress line for ``score run-all``.""" + tag = pair.status.upper() + line = ( + f" [{tag:<16}] {pair.state} · {pair.fact:<32} " + f"records={pair.scored_count}/{pair.record_count} " + f"${pair.total_usd:.4f} " + f"cache={pair.cache_hit_count} downgrades={pair.downgrade_count}" + ) + if pair.error: + line += f" err={pair.error}" + click.echo(line) + + +def render_run_all_checkpoint(state: str, manifest, *, assume_yes: bool) -> bool: + """The ``--checkpoint-after`` summary + confirm gate.""" + state_pairs = [p for p in manifest.pairs if p.state == state] + total_downgrades = sum(p.downgrade_count for p in state_pairs) + total_spend = sum(p.total_usd for p in state_pairs) + total_records = sum(p.scored_count for p in state_pairs) + click.echo( + f"\n--- checkpoint after {state} ---\n" + f" {len(state_pairs)} pairs · " + f"${total_spend:.4f} · {total_downgrades} downgrades · " + f"{total_records} records scored\n" + f" per-pair downgrade detail:" + ) + for p in state_pairs: + tag = "HIGH" if p.record_count and p.downgrade_count / max(p.record_count, 1) > 0.15 else "ok " + click.echo( + f" [{tag}] {p.fact:<32} dg={p.downgrade_count:>3}/{p.record_count:<3} " + f"${p.total_usd:.4f}" + ) + if assume_yes: + click.echo(" --yes set — continuing without prompt\n") + return True + return click.confirm("continue fanout to remaining states?", default=True) + + +def echo_batch_submission(manifests) -> None: + """Post-submit summary for ``score run-all --batch``.""" + plural = "es" if len(manifests) > 1 else "" + click.echo(f"Submitted {len(manifests)} batch{plural}:") + for m in manifests: + click.echo( + f" {m.batch_id} items={len(m.items)} est=${m.estimated_cost_usd:.2f}" + ) + click.echo("\nWait ~5–60 min, then collect:") + for m in manifests: + click.echo(f" mc score batches collect {m.batch_id}") + click.echo( + "\nAfter collect, run `mc score run-all` (no --batch) to build " + "sidecars from the populated cache." + ) + + +# --------------------------------------------------------------------------- +# `mc score gap-extract` / `gap-downgrade-summary` +# --------------------------------------------------------------------------- + + +def echo_gap_extract_results(results) -> None: + """Per-state + grand-total summary for ``score gap-extract``.""" + grand_usd = 0.0 + grand_downgrades = 0 + grand_records = 0 + for r in results: + click.echo( + f" [{r.state}] sample={r.sample_count} " + f"facts_complete={sum(1 for f in r.fact_results if f['status'] == 'complete')}/{len(r.fact_results)} " + f"usd=${r.total_usd:.4f} " + f"downgrades={r.total_downgrades}/{r.total_records}" + ) + grand_usd += r.total_usd + grand_downgrades += r.total_downgrades + grand_records += r.total_records + + click.echo( + f"\nTotal: ${grand_usd:.2f} across {grand_records} fact-rows " + f"(downgrade_count={grand_downgrades}, " + f"rate={grand_downgrades / grand_records if grand_records else 0:.4f})" + ) + click.echo( + "\nNext: `mc score aggregate-gap --with-llm` to fold these " + "values into the gap sidecars, then " + "`mc score gap-downgrade-summary` for the halt-criterion check." + ) + + +def echo_gap_downgrade_summary(summary: dict) -> None: + """Per-fact downgrade-rate table for ``score gap-downgrade-summary``.""" + click.echo( + f"States loaded: {', '.join(summary['states_loaded'])}" + ) + click.echo( + f"Total scored: {summary['total_scored']} · " + f"downgrades: {summary['total_downgrades']} · " + f"overall rate: {summary['overall_rate']:.4f}" + ) + click.echo(f"\nPer-fact downgrade rates (halt threshold > 0.30):") + rows = sorted( + summary["per_fact"].items(), + key=lambda kv: kv[1]["rate"], + reverse=True, + ) + for fact, stats in rows: + flag = " ⚠ HALT" if stats["rate"] > 0.30 else "" + click.echo( + f" {fact:<40} {stats['downgrades']:>4}/{stats['scored']:<5} " + f"= {stats['rate']:.4f}{flag}" + ) + + if summary["halt_recommended"]: + click.echo( + "\nHALT RECOMMENDED: at least one fact > 30 % downgrade rate. " + "Investigate before committing Step 3 spend." + ) + else: + click.echo( + "\nAll facts at or below 30 % — Step 3 path is clean to authorize." + ) diff --git a/apps/pipeline/src/ingest/__init__.py b/apps/pipeline/src/ingest/__init__.py new file mode 100644 index 0000000..c41993b --- /dev/null +++ b/apps/pipeline/src/ingest/__init__.py @@ -0,0 +1,24 @@ +"""Per-state ingestion adapters. + +``INGEST_MODULES`` is the one state→adapter-module registry (issue #213 +item 3): ``cli.py`` generates the five ``mc ingest `` commands +from it and ``publish/stages`` derives its stage fanout from it, so a +sixth state is registered exactly once. Keys are the canonical state +codes from :data:`src.states.SUPPORTED_STATES`; values are module +names under ``src.ingest`` whose module-level ``run()`` is the +adapter entry point (plain function — never ``@click.command``; see +CLAUDE.md). + +This module deliberately does NOT import the adapters — the CLI's +lazy-import convention keeps ``mc --help`` fast. +""" + +from __future__ import annotations + +INGEST_MODULES: dict[str, str] = { + "AZ": "arizona", + "WI": "wisconsin", + "MN": "minnesota", + "TX": "texas", + "IN": "indiana", +} diff --git a/apps/pipeline/src/ingest/arizona.py b/apps/pipeline/src/ingest/arizona.py new file mode 100644 index 0000000..c8d2e5b --- /dev/null +++ b/apps/pipeline/src/ingest/arizona.py @@ -0,0 +1,706 @@ +"""Parse Arizona Use Case 12.0 Excel into pipeline data structures. + +Reads domain sheets from the AzEDS Use Case Excel file and extracts all data +element definitions organized by Ed-Fi entity. Entities are namespaced as +edfi.* (core) or az.* (extension), giving us structural classification for free. + +Produces: +- StateElements (Stage 1 output) from all ~642 elements + +Known source-side limitations (Phase B.8 documentation, 2026-04-09 — see +docs/ingest-layer-audit.md §9.13): + +- **`business_rules_text` is always None for AZ elements.** The Use Case Excel + has no business rules / formula column — only Property Name, Data Type, + Codes, and Description. `build_element_records` sets + `business_rules_text=None`. This is a + source-data limitation, not a parser bug. WI populates this field from + its Confluence source; AZ has only definition text to work from. + +- **Internal-whitespace entity names are preserved verbatim.** The Use Case + source has at least one entity with an internal space — + `StudentDropOut RecoveryProgramMonthlyUpdates`. `_clean_entity_name` + intentionally does not collapse internal whitespace so the canonical AZ + output stays faithful to the source. + +- **One element has a truncated `definition_text`:** + `SectionExternalProviderTeacher.BeginDate` carries + `definition_text = "The first date the SectionExternalProviderTeacher"`, + which is incomplete mid-phrase. Verified to be a source-side issue: the + Description cell in the `Master Schedule` sheet literally contains that + exact truncated string with no continuation. The parser is innocent. +""" + +import logging +import re +from dataclasses import dataclass, field +from pathlib import Path + +import openpyxl + +from src.models.edfi_catalog import EdFiCatalog +from src.models.element import ElementRecord +from src.ingest.normalize import normalize_data_type, normalize_entity + +logger = logging.getLogger(__name__) + + +# Sheets that don't contain Ed-Fi data element tables. +# +# Excluded by design: +# - Code Values, NeedCategoryMapping, Tribal Affiliations: lookup/mapping +# tables, not entity definitions. +# - Change Log: revision history. +# - Identity: AzEDS /identities API (vendor identity-resolution service). +# Not part of the Ed-Fi data standard. Schools call these endpoints to +# look up UniqueIDs; they don't populate them as part of submitting data. +# The bare "Identities" entity header doesn't match _ENTITY_RE so the +# parser already drops this sheet — this list documents that as +# intentional rather than accidental. See docs/ingest-layer-audit.md §9.8. +# +# Previously excluded but reinstated by Phase B.2 (2026-04-09): +# - IDEA Part C: contains 3 az.PartC* extension entity tables (PartCAZEIP, +# PartCNotification, PartCTransition) with ~17 valid IDEA federal- +# compliance fields. The skip was incorrect. +_SKIP_SHEETS = frozenset([ + "Code Values", + "NeedCategoryMapping", + "Change Log", + "Tribal Affiliations", + "Identity", +]) + +# Parse floors (issue #213 item 2). MN/IN raise loudly on source-format +# drift, but the hardcoded column scan in `detect_entity_tables` DROPS +# unrecognized tables silently — a renamed "Column Name" header or shifted +# entity columns would quietly shrink the corpus while every downstream +# stage reports success. `parse_az_excel` raises when the end-of-parse +# corpus falls below these floors. Today's actual counts (2026-07-09, +# data/bootstrap/az/Use_Case_12.0_20260227.xlsm): 107 entity tables / +# 663 raw element rows — each floor is cleared >2x. Tests monkeypatch +# these module constants for small synthetic workbooks. +_MIN_ENTITY_TABLES = 50 # today: 107 (2026-07-09) +_MIN_ELEMENT_ROWS = 300 # today: 663 (2026-07-09) + +# Regex to match entity names like edfi.StudentSchoolAssociation or az.SectionExtension +_ENTITY_RE = re.compile(r"^(edfi|az)\.+\s*\w+", re.IGNORECASE) + +# Regex to extract optionality suffix from element names: (R), (C), (O) +# Also handles variants: "( R)", "(O)*", "(O) *" +_OPTIONALITY_RE = re.compile(r"\s*\(\s*([RCO])\s*\)\s*\*?\s*$") + + +@dataclass +class AZElementRow: + """A single data element extracted from the Use Case Excel.""" + + raw_name: str + clean_name: str + optionality: str | None # R, C, O, or None + data_type: str + codes_ref: str | None + description: str + + +@dataclass +class AZEntityTable: + """A group of elements belonging to one entity within a domain sheet.""" + + sheet_name: str + entity_name: str + is_extension: bool + requirement_level: str # "Required", "Optional*", etc. + elements: list[AZElementRow] = field(default_factory=list) + + +def _str_or_none(val: object) -> str | None: + """Convert cell value to stripped string or None.""" + if val is None: + return None + s = str(val).strip() + return s if s else None + + +def _clean_entity_name(raw: str) -> str: + """Normalize entity name: trim outer whitespace, fix double dots. + + INTENTIONAL: internal whitespace is preserved verbatim. The AZ Use Case + source has at least one entity name with an internal space — + `az.StudentDropOut RecoveryProgramMonthlyUpdates`. We preserve it so the + canonical AZ output stays faithful to the source. A future engineer who + "fixes" this by collapsing internal whitespace will break round-trip + fidelity to the source spreadsheet. + """ + name = raw.strip() + # Fix double dots: az..Foo -> az.Foo + name = re.sub(r"\.{2,}", ".", name) + # Internal whitespace deliberately preserved — see docstring above. + return name + + +def _normalize_descriptor_type_in_place(records: list[ElementRecord]) -> None: + """Overwrite `data_type` to `"Descriptor"` when the element name has a + descriptor suffix. + + Analyst review flagged that the AZ source XLSX stores descriptor fields + with the underlying serialization type (Integer for `*DescriptorId`, or + occasionally Date — row 107 PartCTransition.DelayDescriptorId, where the + source type is `Date` but the definition clearly describes a descriptor). + For analyst review and scoring, `Descriptor` is the meaningful canonical + form. Primitive types on non-descriptor fields are preserved verbatim. + """ + for i, r in enumerate(records): + low = r.element_name.lower() + if low.endswith("descriptor") or low.endswith("descriptorid"): + if r.data_type != "Descriptor": + records[i] = r.model_copy(update={"data_type": "Descriptor"}) + + +def _canonicalize_az_extension_name( + raw: str, known_extension_keys: set[str] | None = None +) -> str: + """Canonicalize the analyst-facing extension_name for AZ records. + + Applied separately from `_clean_entity_name` (which preserves verbatim + source for `raw_entity` / `entity`). Fixes four source-side quirks flagged + by analyst review: + - `AZ.` (uppercase) normalized to `az.` (the dominant convention) + - `Extention` typo corrected to `Extension` + - Internal whitespace collapsed, e.g. + `az.StudentDropOut RecoveryProgramMonthlyUpdates` + -> `az.StudentDropOutRecoveryProgramMonthlyUpdates` + - Trailing-`s` pluralization is stripped when the spine catalog has + the singular form as a known extension schema. Round-2 reviewer B + flagged `az.StudentDropOutRecoveryProgramMonthlyUpdates` as an + XLSX plural that should render as the singular spine-extension + form (`az.StudentDropOutRecoveryProgramMonthlyUpdate`). + + `known_extension_keys` is the set of canonical AZ extension catalog keys + (from `spine.catalog.extensions`) — e.g. `{"az_studentDropOutRecoveryProgramMonthlyUpdate", + "az_calendarExtension", ...}`. When provided and the singularized form's + camelCase projection matches a key, we strip the trailing `s`. When None, + we skip singularization entirely — preserves backward-compat for callers + that don't have spine access. + """ + name = raw.strip() + # Lowercase namespace prefix. + if name.startswith("AZ."): + name = "az." + name[3:] + # Fix trailing `Extention` typo; reuse the matching regex behavior used + # in `normalize_entity` (normalize.py:_TYPO_EXTENTION). + name = re.sub(r"Extention$", "Extension", name) + # Collapse internal whitespace (NOT applied to raw_entity / entity — + # those preserve verbatim per _clean_entity_name). + name = re.sub(r"\s+", "", name) + + # Spine-validated trailing-`s` singularization. The display form + # `az.XxxYyys` projects to spine key `az_xxxYyys` (lowercase first segment + # letter); singular projects to `az_xxxYyy`. Only strip when the singular + # form is a known extension schema — avoids mangling names that legitimately + # end in `s` (none currently, but defensive). + if ( + known_extension_keys + and name.lower().startswith("az.") + and name.endswith("s") + and not name.endswith("ss") + and not name.endswith("us") + and not name.endswith("is") + and len(name) > len("az.") + 3 + ): + singular = name[:-1] + # Project "az.FooBar" -> "az_fooBar" (preserve all characters past the + # first one after the prefix). + pascal_body = singular[len("az."):] + if pascal_body: + camel_body = pascal_body[0].lower() + pascal_body[1:] + if f"az_{camel_body}" in known_extension_keys: + name = singular + return name + + +def _parse_element_name(raw: str) -> tuple[str, str | None]: + """Extract clean name and optionality suffix from raw element name. + + Examples: + "EntryDate (R)" -> ("EntryDate", "R") + "ExitWithdrawDate (C)" -> ("ExitWithdrawDate", "C") + "CalendarCode (R )" -> ("CalendarCode", "R") + "StudentUniqueID" -> ("StudentUniqueID", None) + """ + raw = raw.strip() + m = _OPTIONALITY_RE.search(raw) + if m: + clean = raw[: m.start()].strip() + return clean, m.group(1).strip() + return raw, None + + +def _is_entity_cell(val: object) -> bool: + """Check if a cell value looks like an entity name (edfi.* or az.*).""" + if not val or not isinstance(val, str): + return False + return bool(_ENTITY_RE.match(val.strip())) + + +def _is_element_row(row: tuple, name_col: int, type_col: int) -> bool: + """Check if a row contains an element definition (has name and data type).""" + if name_col >= len(row) or type_col >= len(row): + return False + name = row[name_col] + dtype = row[type_col] + if not name or not isinstance(name, str): + return False + name_s = name.strip() + # Skip header rows + if name_s == "Column Name": + return False + # Must have a data type + if not dtype or not isinstance(dtype, str): + return False + return True + + +def detect_entity_tables(rows: list[tuple], sheet_name: str) -> list[AZEntityTable]: + """Detect all entity table boundaries in a sheet. + + Scans for entity names (edfi.*/az.*) in columns 5-7 and identifies + the element rows that follow. Handles three structural variants: + + 1. Standard: entity header row, then "Column Name" header row, then elements + 2. Same-row: entity + "Column Name" on same row, elements on next row + 3. No header: entity row, elements start directly (e.g., CalendarDateCalendarEvent) + + Also handles the special case where entity header has the first element + on the same row (e.g., az.CourseTranscriptExtention). + """ + tables: list[AZEntityTable] = [] + + i = 0 + while i < len(rows): + row = rows[i] + entity_name = None + entity_col = None + req_level = "" + + # Look for entity name in columns 5-7 only (skip col 0-4 which are API sections) + for col in (7, 6, 5): + if col < len(row) and _is_entity_cell(row[col]): + entity_name = _clean_entity_name(str(row[col])) + entity_col = col + # Check for requirement level in col 6 + if col == 7 and 6 < len(row) and row[6] and isinstance(row[6], str): + req_level = str(row[6]).strip() + break + + if not entity_name: + i += 1 + continue + + is_ext = entity_name.lower().startswith("az.") + + # Determine element column layout based on entity column position + # When entity is in col 5 (Learning Modality variant): name=6, type=7, codes=8, desc=9 + # When entity is in col 6 or 7 (standard): name=8, type=9, codes=10, desc=11 + if entity_col is not None and entity_col <= 5: + name_col, type_col, codes_col, desc_col = 6, 7, 8, 9 + else: + name_col, type_col, codes_col, desc_col = 8, 9, 10, 11 + + # Refine column offsets by finding where "Column Name" actually appears + # Check if "Column Name" is on the same row + same_row_cn = any( + c == "Column Name" for c in row if c and isinstance(c, str) + ) + # If same-row Column Name found, use its position to set offsets + if same_row_cn: + cn_pos = next( + (k for k, c in enumerate(row) if c == "Column Name"), name_col + ) + name_col = cn_pos + type_col = cn_pos + 1 + codes_col = cn_pos + 2 + desc_col = cn_pos + 3 + + # Check if "Column Name" is on the next row + next_row_cn = ( + i + 1 < len(rows) + and any( + c == "Column Name" + for c in rows[i + 1] + if c and isinstance(c, str) + ) + ) + # If next-row Column Name found, use its position + if next_row_cn and not same_row_cn: + cn_pos = next( + (k for k, c in enumerate(rows[i + 1]) if c == "Column Name"), + name_col, + ) + name_col = cn_pos + type_col = cn_pos + 1 + codes_col = cn_pos + 2 + desc_col = cn_pos + 3 + + # Check if entity row also has element data (e.g., az.CourseTranscriptExtention) + has_element_on_entity_row = ( + not same_row_cn + and not next_row_cn + and _is_element_row(row, name_col, type_col) + ) + + # Determine where elements start + if has_element_on_entity_row: + elem_start = i # Elements start on this row + elif same_row_cn: + elem_start = i + 1 # Elements start on next row + elif next_row_cn: + elem_start = i + 2 # Skip header row + else: + # No Column Name found -- elements start on next row + elem_start = i + 1 + + # Extract elements + table = AZEntityTable( + sheet_name=sheet_name, + entity_name=entity_name, + is_extension=is_ext, + requirement_level=req_level, + ) + + j = elem_start + while j < len(rows): + erow = rows[j] + + # Stop conditions: empty row, or next entity header + if all(c is None for c in erow): + break + # Check if this row is a new entity header + if any( + col < len(erow) and _is_entity_cell(erow[col]) + for col in (5, 6, 7) + ): + # But not the current entity row itself + if j != i: + break + + if _is_element_row(erow, name_col, type_col): + raw_name = str(erow[name_col]).strip() + clean_name, optionality = _parse_element_name(raw_name) + data_type = str(erow[type_col]).strip() + codes_ref = _str_or_none(erow[codes_col]) if codes_col < len(erow) else None + description = _str_or_none(erow[desc_col]) if desc_col < len(erow) else None + + table.elements.append(AZElementRow( + raw_name=raw_name, + clean_name=clean_name, + optionality=optionality, + data_type=data_type, + codes_ref=codes_ref, + description=description or "", + )) + + j += 1 + + if table.elements: + tables.append(table) + + # Move past the elements we just processed + i = max(i + 1, j) + + return tables + + +def parse_az_excel(workbook_path: Path) -> list[AZEntityTable]: + """Parse all entity tables from the AZ Use Case Excel file. + + Args: + workbook_path: Path to the Use Case 12.0 .xlsm file. + + Returns: + List of AZEntityTable with extracted elements. + """ + wb = openpyxl.load_workbook( + workbook_path, read_only=True, data_only=True, keep_links=False + ) + + all_tables: list[AZEntityTable] = [] + + for sheet_name in wb.sheetnames: + if sheet_name.strip() in _SKIP_SHEETS: + continue + + ws = wb[sheet_name] + rows = list(ws.iter_rows(values_only=True)) + tables = detect_entity_tables(rows, sheet_name.strip()) + all_tables.extend(tables) + + wb.close() + + # End-of-parse corpus floor (issue #213 item 2): the column scan drops + # unrecognized tables silently, so a structural workbook change would + # otherwise shrink the corpus without any failure signal. + total_rows = sum(len(t.elements) for t in all_tables) + if len(all_tables) < _MIN_ENTITY_TABLES or total_rows < _MIN_ELEMENT_ROWS: + raise ValueError( + f"AZ source format may have changed: parsed {len(all_tables)} entity " + f"tables / {total_rows} element rows from {workbook_path.name}, below " + f"the floor of {_MIN_ENTITY_TABLES} tables / {_MIN_ELEMENT_ROWS} rows " + "(2026-07-09 actual: 107 tables / 663 rows). detect_entity_tables " + "drops unrecognized tables silently — inspect the workbook layout " + "(entity columns 5-7, 'Column Name' headers) before lowering " + "_MIN_ENTITY_TABLES/_MIN_ELEMENT_ROWS." + ) + return all_tables + + +# Known typos in AZ Use Case Excel -- corrected at ingestion +_AZ_ELEMENT_NAME_CORRECTIONS = { + "StudenUniqueID": "StudentUniqueID", + "StudentUniqeID": "StudentUniqueID", + "ProgramTypeDescriptionId": "ProgramTypeDescriptorId", + "staffUniqueId": "StaffUniqueID", +} + + +def build_element_records( + tables: list[AZEntityTable], + edfi_version: str = "5.2", + source_document: str = "Use_Case_12.0_20260227.xlsm", + catalog: EdFiCatalog | None = None, +) -> list[ElementRecord]: + """Convert parsed AZ entity tables to ElementRecord instances.""" + records: list[ElementRecord] = [] + + for table in tables: + normalized = normalize_entity(table.entity_name, catalog) + for elem in table.elements: + element_name = _AZ_ELEMENT_NAME_CORRECTIONS.get( + elem.clean_name, elem.clean_name + ) + records.append(ElementRecord( + state="AZ", + edfi_version=edfi_version, + domain=table.sheet_name, + entity=normalized, + raw_entity=table.entity_name, + element_name=element_name, + data_type=normalize_data_type(elem.data_type), + definition_text=elem.description, + # AZ source has no business rules / formula column — see + # module docstring "Known source-side limitations". Always None. + business_rules_text=None, + source="extension" if table.is_extension else "core", + extension_name=( + _canonicalize_az_extension_name(table.entity_name) + if table.is_extension else None + ), + source_document=source_document, + source_page_or_section=table.sheet_name, + documented=True, + )) + + # NOTE: Dedup intentionally NOT called here — it used to run pre-unflatten, + # but that missed the 53 (entity, element_name) collisions introduced by + # the unflatten loop in `run()` (flagged by analyst review). The final + # dedup pass runs in `run()` AFTER unflatten. See utils/dedup.py. + return records + + +# POC-3: the authoritative catalog is the live-fetched Swagger spine at +# data/spine/az_spine.json. Entity normalization during ingest uses +# spine.catalog so AZ names align with core Ed-Fi forms. +from src.utils.paths import ( + state_elements_path, + state_gap_log_path, + state_spine_path, +) + +_MC_ROOT = Path(__file__).resolve().parents[2] +_AZ_SPINE_PATH = state_spine_path("AZ") +_AZ_XLSX_PATH = _MC_ROOT / "data" / "raw" / "az" / "Use_Case_12.0_20260227.xlsm" +# Fresh-clone bootstrap: the workbook lives under `data/bootstrap/az/` so +# `mc ingest az` works out of the box. A copy in `data/raw/az/` still +# wins when present — devs re-downloading from ADE don't have to touch +# `data/bootstrap/`. +_AZ_XLSX_BOOTSTRAP_PATH = ( + _MC_ROOT / "data" / "bootstrap" / "az" / "Use_Case_12.0_20260227.xlsm" +) +_AZ_RULES_PATH = _MC_ROOT / "data" / "raw" / "az" / "az_integrity_rules.json" +# Fresh-clone bootstrap: the parsed integrity-rules JSON lives under +# `data/bootstrap/az/` so `mc ingest az` enriches business_rules_text +# out of the box. A copy in `data/raw/az/` still wins when present — +# devs who re-run `python -m src.ingest.az_integrity_rules` against an +# updated PDF set don't have to touch `data/bootstrap/`. +_AZ_RULES_BOOTSTRAP_PATH = ( + _MC_ROOT / "data" / "bootstrap" / "az" / "az_integrity_rules.json" +) +_AZ_ELEMENTS_OUT = state_elements_path("AZ", "source") +_AZ_ELEMENTS_SPINE_OUT = state_elements_path("AZ", "spine") +_AZ_GAP_OUT = state_gap_log_path("AZ") + + +def run() -> None: + """POC-3 AZ ingestion: parse XLSX, enrich with integrity rules, match to spine. + + Writes `data/out/az_elements_source.json` (StateElements) and + `data/out/az_gap_log.json` (spine-match diagnostics). + """ + from src.ingest.az_enrich import enrich_records_with_rules + from src.ingest.shared import ( + compute_coverage, + demote_unmatched_to_unknown, + populate_data_types_from_spine, + populate_edfi_standard_definition_from_spine, + run_unflatten_pass, + stamp_edfi_domains, + write_dual_lens_artifacts, + ) + from src.models.spine import StateSpine + from src.utils.dedup import dedup_records + + if not _AZ_SPINE_PATH.exists(): + raise FileNotFoundError( + f"No AZ spine at {_AZ_SPINE_PATH}. " + f"Run `mc spine fetch --state AZ` + `mc spine build --state AZ` first." + ) + spine = StateSpine.model_validate_json(_AZ_SPINE_PATH.read_text(encoding="utf-8")) + logger.info( + "Loaded AZ spine: %d core entities, %d extensions (Ed-Fi %s)", + spine.entity_count, spine.extension_count, spine.edfi_version, + ) + + if _AZ_XLSX_PATH.exists(): + xlsx_path = _AZ_XLSX_PATH + elif _AZ_XLSX_BOOTSTRAP_PATH.exists(): + xlsx_path = _AZ_XLSX_BOOTSTRAP_PATH + logger.info( + "Using bundled AZ workbook at %s (no copy in data/raw/az/)", + xlsx_path, + ) + else: + raise FileNotFoundError( + f"AZ Use Case XLSX not found at {_AZ_XLSX_PATH} or " + f"{_AZ_XLSX_BOOTSTRAP_PATH}" + ) + + tables = parse_az_excel(xlsx_path) + total_raw = sum(len(t.elements) for t in tables) + logger.info( + "Parsed %s: %d entity tables (%d core, %d extension), %d raw elements", + xlsx_path.name, + len(tables), + sum(1 for t in tables if not t.is_extension), + sum(1 for t in tables if t.is_extension), + total_raw, + ) + + records = build_element_records( + tables, + edfi_version=spine.edfi_version, + source_document=_AZ_XLSX_PATH.name, + catalog=spine.catalog, + ) + logger.info("Built %d ElementRecords (from %d raw rows)", len(records), total_raw) + + # Round 2.2: re-canonicalize extension_name against the spine's extension + # catalog so XLSX plural forms display as the singular spine form. + known_ext_keys = set(spine.catalog.extensions.keys()) + for i, r in enumerate(records): + if r.extension_name: + canonical = _canonicalize_az_extension_name(r.extension_name, known_ext_keys) + if canonical != r.extension_name: + records[i] = r.model_copy(update={"extension_name": canonical}) + + _normalize_descriptor_type_in_place(records) + + rules_path = _AZ_RULES_PATH if _AZ_RULES_PATH.exists() else _AZ_RULES_BOOTSTRAP_PATH + if rules_path.exists(): + records = enrich_records_with_rules(records, rules_path) + enriched = sum(1 for r in records if r.business_rules_text) + logger.info("Integrity-rules enrichment: %d/%d records carry business_rules_text (source=%s)", enriched, len(records), rules_path.name) + else: + logger.warning("No az_integrity_rules.json at %s or %s — skipping rules enrichment", _AZ_RULES_PATH, _AZ_RULES_BOOTSTRAP_PATH) + + # AZ-bespoke pipeline: unflatten → cross-attribution hint fix → dedup → + # demote-to-unknown → spine-type contract. The shared orchestrator + # `assemble_source_driven` does not fit AZ because AZ pre-sets `source` + # from namespace at record-creation time (no `attribute_record_source` + # call) and needs the cross-attribution fix between unflatten and dedup. + recovered = run_unflatten_pass(records, spine) + + # Round 2.2: narrow fix for reviewer B's row 42 cross-attribution. + # The AZ `StudentDropOutRecoveryProgramMonthlyUpdate` extension declares + # its own properties but not the entity's inherited identity, so the + # spine's parent-propagation loop walks them up to `Student` — wrong + # semantic parent. Target ONLY rows that were rewritten from a + # StudentDropOut...MonthlyUpdate(s) raw entity to `Student`. + _AZ_CROSS_ATTRIBUTION_HINTS: tuple[str, ...] = ( + "StudentDropOutRecoveryProgramMonthlyUpdate", + ) + for i, r in enumerate(records): + if r.entity != "Student": + continue + raw_collapsed = (r.raw_entity or "").replace(" ", "").replace(".", "") + for hint in _AZ_CROSS_ATTRIBUTION_HINTS: + if hint.lower() in raw_collapsed.lower(): + records[i] = r.model_copy(update={ + "entity": hint, + "source": "unknown", + "extension_name": None, + }) + break + + pre_dedup = len(records) + records = dedup_records(records) + if len(records) < pre_dedup: + logger.info( + "Post-unflatten dedup: %d -> %d records (collapsed %d duplicate pairs)", + pre_dedup, len(records), pre_dedup - len(records), + ) + + # AZ sets `source` at creation time from the XLSX namespace (`edfi.*` → + # core, `az.*` → extension). Rows that don't actually match the spine + # get demoted to `unknown` so Match Status reconciles with the gap log. + demote_unmatched_to_unknown(records, spine) + + # Canonical-type contract: for matched records, `data_type` is spine- + # derived. Runs AFTER demote_unmatched_to_unknown so unresolved rows + # (source=="unknown") keep source-verbatim types as an audit trail. + populate_data_types_from_spine(records, spine) + populate_edfi_standard_definition_from_spine(records, spine) + + # Issue #184: stamp the Ed-Fi domain (distinct from `domain` = Source + # Area) on every source-lens record. AZ orchestrates the helpers inline + # rather than via assemble_source_driven, so the stamp is explicit here. + stamp_edfi_domains(records, spine) + + assembly = compute_coverage(records, spine) + + # Shared adapter tail: dual-lens writes → gap log → swagger backfill + # (issue #213 item 3 — the ~80-line sequence lives once in shared.py). + write_dual_lens_artifacts( + state="AZ", + spine=spine, + records=records, + assembly=assembly, + recovered=recovered, + source_document=_AZ_XLSX_PATH.name, + source_out=_AZ_ELEMENTS_OUT, + spine_out=_AZ_ELEMENTS_SPINE_OUT, + gap_out=_AZ_GAP_OUT, + spine_path=_AZ_SPINE_PATH, + spine_source_rel=str(_AZ_SPINE_PATH.relative_to(_MC_ROOT)), + source_coverage_note=( + "Of our AZ doc elements, how many match a spine element " + "(case-insensitive, FK+descriptor aliases)." + ), + spine_coverage_note=( + "Of the spine's authoritative element slots, how many are " + "represented in AZ docs." + ), + logger=logger, + ) + + +if __name__ == "__main__": + run() diff --git a/apps/pipeline/src/ingest/az_enrich.py b/apps/pipeline/src/ingest/az_enrich.py new file mode 100644 index 0000000..5594650 --- /dev/null +++ b/apps/pipeline/src/ingest/az_enrich.py @@ -0,0 +1,149 @@ +"""Enrich AZ elements with business rules from Integrity Rules. + +Maps parsed integrity rules to elements by domain affinity and keyword matching. +Populates the business_rules_text field on each ElementRecord. + +Critical difference from prior POC: rules that don't match a specific element +are DROPPED, not broadcast to all elements in the domain. This trades coverage +for precision — better to have fewer, relevant rules than irrelevant ones +polluting the evidence extraction prompt. +""" + +import json +from pathlib import Path + +from src.models.element import ElementRecord + +# Map integrity rule domains to Use Case sheet names (= element.domain values) +_RULE_DOMAIN_TO_SHEETS = { + "ADM": [ + "Student Enrollment", "Student Withdrawal", "Student Demographics", + "Student Attendance", "Learning Modality", "School Calendar", + ], + "Accountability": ["Student Enrollment", "Student Demographics"], + "ELL": ["ELL Student Program"], + "SPED": ["SPED Program"], + "Support Programs": ["Support Program"], + "Data Quality": [ + "Student Enrollment", "Student Demographics", "Staff", + "Student Attendance", "Student Transcript", + ], + "Discipline": ["Discipline"], + "Gifted": ["Support Program"], + "Food Service": ["Food Service Program (NSLP)"], + "Homeless": ["Homeless Program"], + "STC": ["Student Transcript", "Master Schedule", "Staff"], + "FRPL": ["Food Service Program (NSLP)"], + "DRP": ["Dropout Recovery Program"], + "PCCP": ["Student Transcript"], + "Calendar": ["School Calendar"], +} + +# Keywords that map to specific element names (AZ-specific abbreviations) +_KEYWORD_TO_ELEMENTS = { + "DOA": ["SchoolID", "EducationOrganizationId"], + "DOR": ["LocalEducationAgencyID", "ResidentDistrictID"], + "FTE": ["MembershipFTEDescriptorID", "FTEStartDate", "FTEEndDate"], + "grade": ["EntryGradeLevelDescriptorID", "GradeLevelDescriptorId"], + "enrollment": ["EntryDate", "ExitWithdrawDate", "EntryTypeDescriptorID"], + "withdrawal": ["ExitWithdrawDate", "ExitWithdrawTypeDescriptorID"], + "attendance": ["AttendanceEventCategoryDescriptorID", "EventDate", "EventDuration"], + "calendar": ["CalendarCode", "CalendarTypeDescriptorId"], + "membership": ["MembershipTypeDescriptorID"], + "tuition": ["TuitionPayerDescriptorID"], + "special enrollment": ["SpecialEnrollmentDescriptorID"], + "CEC": ["SpecialEnrollmentDescriptorID"], + "birth": ["BirthDate", "BirthCountryDescriptorId", "BirthStateAbbreviationDescriptorId"], + "discipline": ["IncidentIdentifier", "BehaviorDescriptorId"], + "course": ["CourseCode", "LocalCourseCode"], + "section": ["SectionIdentifier", "LocalCourseCode"], + "staff": ["StaffUniqueId", "StaffClassificationDescriptorId"], + "session": ["SessionName", "TermDescriptorID"], + "program": ["ProgramName", "ProgramTypeDescriptorId"], + "need": ["NeedDescriptorId"], + "service": ["ServiceDescriptorId"], +} + + +def _find_matching_elements( + rule_desc: str, + domain_elements: list[ElementRecord], +) -> list[str]: + """Find element names that match keywords in a rule description.""" + desc_lower = rule_desc.lower() + matched: set[str] = set() + + # Direct element name matching + for elem in domain_elements: + if elem.element_name.lower() in desc_lower: + matched.add(elem.element_name) + + # Keyword-based matching + for keyword, element_names in _KEYWORD_TO_ELEMENTS.items(): + if keyword.lower() in desc_lower: + domain_names = {e.element_name for e in domain_elements} + for ename in element_names: + if ename in domain_names: + matched.add(ename) + + return sorted(matched) + + +def enrich_records_with_rules( + records: list[ElementRecord], + rules_path: Path, +) -> list[ElementRecord]: + """Populate business_rules_text from AZ integrity rules JSON. + + Only attaches rules that match a specific element by name or keyword. + Rules that don't match any element are dropped (no broadcast fallback). + + Returns the same list of records (mutated in place) with business_rules_text + populated where integrity rules matched. + """ + if not rules_path.exists(): + return records + + rules = json.loads(rules_path.read_text(encoding="utf-8")) + + # Build domain index: domain -> list of records + domain_index: dict[str, list[ElementRecord]] = {} + for rec in records: + domain_index.setdefault(rec.domain, []).append(rec) + + # Accumulate rule texts per (domain, element_name) + rules_by_element: dict[tuple[str, str], list[str]] = {} + + for rule in rules: + rule_domain = rule["domain_category"] + sheets = _RULE_DOMAIN_TO_SHEETS.get(rule_domain, []) + + for sheet in sheets: + domain_elements = domain_index.get(sheet, []) + if not domain_elements: + continue + + matched_elements = _find_matching_elements( + rule["description"], domain_elements + ) + + if matched_elements: + rule_text = f"[Rule {rule['error_code']}] {rule['description']}" + for ename in matched_elements: + key = (sheet, ename) + rules_by_element.setdefault(key, []).append(rule_text) + # No else branch — unmatched rules are dropped, NOT broadcast + + # Apply enrichment + for rec in records: + key = (rec.domain, rec.element_name) + if key in rules_by_element: + rule_texts = rules_by_element[key] + unique_rules = list(dict.fromkeys(rule_texts)) + existing = rec.business_rules_text or "" + if existing: + rec.business_rules_text = existing + "\n" + "\n".join(unique_rules) + else: + rec.business_rules_text = "\n".join(unique_rules) + + return records diff --git a/apps/pipeline/src/ingest/az_integrity_rules.py b/apps/pipeline/src/ingest/az_integrity_rules.py new file mode 100644 index 0000000..33cf609 --- /dev/null +++ b/apps/pipeline/src/ingest/az_integrity_rules.py @@ -0,0 +1,241 @@ +"""Parse Arizona Integrity Rules PDFs into structured rule records. + +Reads tabular PDFs from azed.gov that define business validation rules +applied to AzEDS data submissions. Each PDF covers a domain area +(ADM, ELL, SPED, etc.) identified by error code prefix. + +Also handles the 58XXX Food Service Program rules which are in .docx format. +""" + +import json +import re +from dataclasses import asdict, dataclass +from pathlib import Path + +import click +import pdfplumber + + +# Error code prefix -> domain mapping +_DOMAIN_MAP = { + "10": "ADM", + "20": "ADM", + "21": "Accountability", + "30": "ELL", + "40": "SPED", + "50": "Support Programs", + "51": "Data Quality", + "52": "Discipline", + "57": "Gifted", + "58": "Food Service", + "59": "Homeless", + "60": "STC", + "70": "FRPL", + "80": "DRP", + "81": "PCCP", + "90": "Calendar", +} + + +@dataclass +class IntegrityRule: + """A single integrity validation rule from the AzEDS rules documents.""" + + error_code: str + description: str + message: str + severity: str + active_current_year: bool + source_document: str + domain_category: str + + +def _infer_domain(error_code: str) -> str: + """Map error code to domain category.""" + code = error_code.strip() + # Try 2-char prefix first, then 1-char + for prefix_len in (2, 1): + prefix = code[:prefix_len] + if prefix in _DOMAIN_MAP: + return _DOMAIN_MAP[prefix] + return "Unknown" + + +def _clean_text(text: str | None) -> str: + """Clean extracted PDF text: normalize whitespace, strip.""" + if not text: + return "" + return re.sub(r"\s+", " ", text).strip() + + +def parse_integrity_rules_pdf(pdf_path: Path) -> list[IntegrityRule]: + """Parse a single Integrity Rules PDF into IntegrityRule records. + + Each page contains a table with columns: + - Col 0: Error Code + - Col 1: Description (the business rule) + - Col 2: Message + - Col 3: Severity (Error/Warning) + - Col 4: Comments + - Col 5+: Active status per fiscal year + """ + rules: list[IntegrityRule] = [] + source = pdf_path.name + + with pdfplumber.open(pdf_path) as pdf: + for page in pdf.pages: + tables = page.extract_tables() + for table in tables: + for row in table: + if not row or len(row) < 5: + continue + + error_code = _clean_text(row[0]) + + # Skip header rows and non-data rows + if not error_code or not error_code[0].isdigit(): + continue + + description = _clean_text(row[1]) + message = _clean_text(row[2]) + severity = _clean_text(row[3]) + # Active status: col 5 contains "Active" or "Inactive" for current year + active_text = _clean_text(row[5]) if len(row) > 5 else "" + active = active_text.lower() == "active" + + if description: # Skip rows with empty descriptions + rules.append(IntegrityRule( + error_code=error_code, + description=description, + message=message, + severity=severity, + active_current_year=active, + source_document=source, + domain_category=_infer_domain(error_code), + )) + + return rules + + +def parse_integrity_rules_docx(docx_path: Path) -> list[IntegrityRule]: + """Parse the Food Service (58XXX) rules from .docx format.""" + from docx import Document + + rules: list[IntegrityRule] = [] + source = docx_path.name + + doc = Document(docx_path) + for table in doc.tables: + for row in table.rows: + cells = [c.text.strip() for c in row.cells] + if len(cells) < 4: + continue + + error_code = cells[0] + if not error_code or not error_code[0].isdigit(): + continue + + description = _clean_text(cells[1]) + message = _clean_text(cells[2]) if len(cells) > 2 else "" + severity = _clean_text(cells[3]) if len(cells) > 3 else "" + active_text = _clean_text(cells[5]) if len(cells) > 5 else "" + active = active_text.lower() == "active" + + if description: + rules.append(IntegrityRule( + error_code=error_code, + description=description, + message=message, + severity=severity, + active_current_year=active, + source_document=source, + domain_category=_infer_domain(error_code), + )) + + return rules + + +def parse_all_integrity_rules(input_dir: Path) -> list[IntegrityRule]: + """Parse all integrity rules files from the input directory. + + Looks for PDFs and DOCX files matching integrity rules naming patterns. + """ + all_rules: list[IntegrityRule] = [] + + # Find integrity rules files (they contain "xxx" or "XXX" or "Integrity" in name) + for f in sorted(input_dir.iterdir()): + if not f.is_file(): + continue + + name_lower = f.name.lower() + is_integrity = ( + "xxx" in name_lower + or "integrity" in name_lower + or "rules" in name_lower + ) + # Skip non-integrity files + if not is_integrity: + continue + # Skip the vendor cert package and other non-rule files + if "vendor" in name_lower or "use case" in name_lower: + continue + + if f.suffix.lower() == ".pdf": + rules = parse_integrity_rules_pdf(f) + all_rules.extend(rules) + elif f.suffix.lower() == ".docx": + rules = parse_integrity_rules_docx(f) + all_rules.extend(rules) + + return all_rules + + +def save_rules(rules: list[IntegrityRule], output_path: Path) -> None: + """Save parsed rules to JSON.""" + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text( + json.dumps([asdict(r) for r in rules], indent=2), + encoding="utf-8", + ) + + +@click.command() +@click.option( + "--input-dir", + required=True, + type=click.Path(exists=True, file_okay=False, path_type=Path), + help="Directory containing integrity rules PDFs/DOCX files", +) +@click.option( + "--output", + default="data/input/arizona/az_integrity_rules.json", + type=click.Path(path_type=Path), + help="Output JSON file path", +) +def main(input_dir: Path, output: Path) -> None: + """Parse all AZ Integrity Rules and produce a JSON catalog.""" + click.echo(f"Parsing integrity rules from: {input_dir}") + + rules = parse_all_integrity_rules(input_dir) + + click.echo(f" Total rules parsed: {len(rules)}") + + # Summary by domain + domains: dict[str, int] = {} + for r in rules: + domains[r.domain_category] = domains.get(r.domain_category, 0) + 1 + for domain, count in sorted(domains.items()): + active = sum(1 for r in rules if r.domain_category == domain and r.active_current_year) + click.echo(f" {domain}: {count} rules ({active} active)") + + # Severity breakdown + errors = sum(1 for r in rules if r.severity.lower() == "error") + warnings = sum(1 for r in rules if r.severity.lower() == "warning") + click.echo(f" Errors: {errors}, Warnings: {warnings}") + + save_rules(rules, output) + click.echo(f" Saved: {output}") + + +if __name__ == "__main__": + main() diff --git a/apps/pipeline/src/ingest/domain_filter.py b/apps/pipeline/src/ingest/domain_filter.py new file mode 100644 index 0000000..9c1fef3 --- /dev/null +++ b/apps/pipeline/src/ingest/domain_filter.py @@ -0,0 +1,215 @@ +"""SIS-never-populated domain filter for the spine lens. + +SIS vendors typically do not wire up the following Ed-Fi domains in real +deployments. The reporting surface exists in Ed-Fi, but scoring the +(entity, element) positions inside these domains for SIS coverage is noise +— no vendor populates them. Drop them from the spine-lens artifact and +replace each with a single placeholder row per (state, domain) so +reviewers can see we intentionally ignored the domain (not a scraper bug). + +Source-lens artifacts are unaffected — they remain the literal audit trail +of what each state actually documented. + +Filter posture: + +- **Primary (domain-map):** an entity whose declared Ed-Fi domains are a + subset of `FILTERED_DOMAINS` is dropped. Shared entities (Student, + Staff, Program, Section, Course — which also participate in Enrollment, + Discipline, etc.) stay because their domain set is not a pure subset. +- **Explicit concept-anchor list:** a small hand-curated set of entities + whose name and purpose clearly belong to a filtered domain even though + Ed-Fi's domain tags spread them across 2+ domains (e.g., LearningStandard + is tagged with `Standards`+`Assessment`+`CourseCatalog`+...; GradebookEntry + with `StudentAcademicRecord`+`Gradebook`). The all-subset rule wouldn't + catch these; the explicit list does. +- **Fallback (entity-name prefix):** state extensions named e.g. + `tx_AssessmentExt` are not present in the committed domain map. + When the domain lookup returns no domains, match the entity name + against `_FALLBACK_PREFIXES`. +""" + +from __future__ import annotations + +from src.ingest.domain_scope import is_domain_enabled +from src.models.spine import StateSpine + +FILTERED_DOMAINS: frozenset[str] = frozenset({ + "Assessment", + "AssessmentMetadata", + "AssessmentRegistration", + "StudentAssessment", + "Survey", + "Standards", + "Gradebook", + "Intervention", +}) + +# Reviewer-facing collapsed labels — one placeholder per (state, label). +_ASSESSMENT_LABEL = "Assessment" +_SURVEY_LABEL = "Survey" +_LEARNING_STANDARD_LABEL = "LearningStandard" +_GRADEBOOK_LABEL = "Gradebook" +_INTERVENTION_LABEL = "Intervention" + +PLACEHOLDER_LABELS: tuple[str, ...] = ( + _ASSESSMENT_LABEL, + _SURVEY_LABEL, + _LEARNING_STANDARD_LABEL, + _GRADEBOOK_LABEL, + _INTERVENTION_LABEL, +) + +_EXPLICIT_ENTITY_FILTERS: dict[str, str] = { + # Standards-concept entities — tagged with many adjacent domains but + # SIS vendors do not populate learning-standard structure. + "LearningStandard": _LEARNING_STANDARD_LABEL, + "LearningStandardEquivalenceAssociation": _LEARNING_STANDARD_LABEL, + # Gradebook-concept entities — tagged with StudentAcademicRecord as + # well, but the gradebook reporting stream itself is SIS-unpopulated. + "GradebookEntry": _GRADEBOOK_LABEL, + "StudentGradebookEntry": _GRADEBOOK_LABEL, + # Intervention-concept entities — tagged with StudentCohort as a + # secondary domain; stakeholder view is that SIS vendors do not + # populate the intervention reporting surface. + "Cohort": _INTERVENTION_LABEL, + "Intervention": _INTERVENTION_LABEL, + "StudentCohortAssociation": _INTERVENTION_LABEL, + "StudentInterventionAssociation": _INTERVENTION_LABEL, +} + + +_FALLBACK_PREFIXES: tuple[tuple[str, str], ...] = ( + # Order matters — longer prefixes first so `ObjectiveAssessment` + # matches before the bare `Assessment` prefix would. + ("ObjectiveAssessment", _ASSESSMENT_LABEL), + ("StudentAssessment", _ASSESSMENT_LABEL), + ("AssessmentItem", _ASSESSMENT_LABEL), + ("Assessment", _ASSESSMENT_LABEL), + ("Survey", _SURVEY_LABEL), + ("LearningStandard", _LEARNING_STANDARD_LABEL), + ("Gradebook", _GRADEBOOK_LABEL), + ("Intervention", _INTERVENTION_LABEL), +) + + +def _collapse_label(domains: list[str]) -> str: + s = set(domains) + if s & {"Assessment", "AssessmentMetadata", "AssessmentRegistration", "StudentAssessment"}: + return _ASSESSMENT_LABEL + if "Survey" in s: + return _SURVEY_LABEL + if "Standards" in s: + return _LEARNING_STANDARD_LABEL + if "Gradebook" in s: + return _GRADEBOOK_LABEL + if "Intervention" in s: + return _INTERVENTION_LABEL + return domains[0] + + +# Map a filtered entity's evidence (declared Ed-Fi domains, or — for +# domain-less extension entities — its name prefix) to the `domain_scope` +# registry keys under which it can be re-enabled. Only the Assessment family +# is registry-gateable; Survey / Standards / Gradebook / Intervention are not +# in `domain_scope.NEW_DOMAINS`, so they are never un-filtered. +_ASSESSMENT_REGISTRY_DOMAINS: frozenset[str] = frozenset( + {"Assessment", "AssessmentMetadata", "StudentAssessment"} +) +_ASSESSMENT_REGISTRY_PREFIXES: tuple[str, ...] = ( + "ObjectiveAssessment", + "StudentAssessment", + "AssessmentItem", + "Assessment", +) + + +def _enablement_keys(entity_name: str, spine: StateSpine) -> set[str]: + """Registry domain keys (`domain_scope.NEW_DOMAINS`) this entity could be + re-enabled under. AssessmentRegistration is kept distinct from the broader + Assessment family so a state can enable one without the other. + + Eligibility mirrors `_raw_filter_label`'s decision path so enablement only + affects entities that were filtered *as* Assessment: + + - Explicit concept-anchor entities (LearningStandard, GradebookEntry, + Cohort, Intervention, …) are filtered by identity, not domain. They are + never registry-enableable — even though some (e.g. LearningStandard) + carry an `Assessment` domain tag among several non-filtered domains, a + tag that must NOT leak them back into the spine lens when Assessment is + enabled. + - Domain-tagged entities are eligible only when the domain-subset rule + would have filtered them (`domains ⊆ FILTERED_DOMAINS`). + - Domain-less extension entities fall back to the assessment name prefixes. + """ + if entity_name in _EXPLICIT_ENTITY_FILTERS: + return set() + entity = spine.catalog.entities.get(entity_name) + domains = set(entity.domains) if entity and entity.domains else set() + keys: set[str] = set() + if domains: + if domains <= FILTERED_DOMAINS: + if domains & _ASSESSMENT_REGISTRY_DOMAINS: + keys.add("Assessment") + if "AssessmentRegistration" in domains: + keys.add("AssessmentRegistration") + elif entity_name.startswith(_ASSESSMENT_REGISTRY_PREFIXES): + keys.add("Assessment") + return keys + + +def _raw_filter_label(entity_name: str, spine: StateSpine) -> str | None: + """The unconditional collapse label for `entity_name`, ignoring per-state + enablement. + + Lookup order: + + 1. Explicit concept-anchor list — for entities Ed-Fi spreads across + several domains but whose name and purpose belong to a filtered + concept (LearningStandard, GradebookEntry, Intervention, etc.). + 2. Domain-subset rule — the entity's declared domains are all in + `FILTERED_DOMAINS`. + 3. Entity-name prefix fallback — for extension entities that have no + registered domains in the committed domain map. + """ + if entity_name in _EXPLICIT_ENTITY_FILTERS: + return _EXPLICIT_ENTITY_FILTERS[entity_name] + entity = spine.catalog.entities.get(entity_name) + domains = list(entity.domains) if entity and entity.domains else [] + if domains and set(domains) <= FILTERED_DOMAINS: + return _collapse_label(domains) + if not domains: + for prefix, label in _FALLBACK_PREFIXES: + if entity_name.startswith(prefix): + return label + return None + + +def entity_filter_domain(entity_name: str, spine: StateSpine) -> str | None: + """Return the reviewer-facing domain label if `entity_name` should be + collapsed out of the spine lens, else None. + + Computes the unconditional collapse label (`_raw_filter_label`) and then + applies the per-state un-filter: if any registry key the entity could be + enabled under is registered for `spine.state` in `domain_scope`, the + entity stays in the spine lens (return None). With an empty + `domain_scope.DOMAIN_SOURCES` this is always a no-op, so behavior is + byte-identical to the pre-registry filter. + """ + label = _raw_filter_label(entity_name, spine) + if label is None: + return None + if any(is_domain_enabled(spine.state, key) for key in _enablement_keys(entity_name, spine)): + return None + return label + + +def placeholder_note(domain_label: str) -> str: + return ( + f"SIS vendors typically do not populate the {domain_label} domain " + "in real deployments. The spine-lens artifact collapses undocumented " + f"{domain_label} spine positions into this single placeholder row; " + f"any rows the state explicitly documented in the {domain_label} " + "domain are retained inline above as real signals of state intent. " + "Source-lens artifacts also keep state-documented rows verbatim as " + "an audit trail." + ) diff --git a/apps/pipeline/src/ingest/domain_scope.py b/apps/pipeline/src/ingest/domain_scope.py new file mode 100644 index 0000000..6b0a2c6 --- /dev/null +++ b/apps/pipeline/src/ingest/domain_scope.py @@ -0,0 +1,211 @@ +"""Per-(state, domain) source-documentation registry. + +Single source of truth for the Ed-Fi domains POC-3 has *additional* state +source documentation for, and where that documentation lives. Adding a new +state-resource link is one ``DomainSource`` entry in ``DOMAIN_SOURCES`` — the +domain filter, the state adapters, and the analyst scope narrative all read +this registry rather than carrying their own hard-coded per-domain maps. + +The four domains this registry governs (`NEW_DOMAINS`): + +- ``Staff`` / ``Finance`` — already flow through both lenses today (they are + not in ``domain_filter.FILTERED_DOMAINS``). A registry entry here only + records *where* a state's new Staff/Finance documentation lives so the + relevant adapter can ingest it; it does not change any filter behavior. +- ``Assessment`` / ``AssessmentRegistration`` — collapsed out of the spine + lens by ``domain_filter`` today. A registry entry for one of these lifts + the placeholder collapse **for that (state, domain) pair only**, so the + domain joins that state's spine coverage denominator. The two are separate + keys so a state can enable one without the other. + +**Empty-registry invariant.** With ``DOMAIN_SOURCES`` empty, every consumer +behaves exactly as it did before this module existed: ``domain_filter`` still +collapses Assessment/AssessmentRegistration, and adapters ingest only their +default source documents. The registry is purely additive. + +``AssessmentRegistration`` is a newer Ed-Fi domain and is **not present in +every state's deployed swagger** — it is in the AZ/WI/IN spines but absent +from MN/TX. Enabling it for a state whose spine lacks those entities yields +source-lens rows only (no spine coverage to populate); see the working +journal / ADR 0007. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +#: Domains this registry governs. ``domain`` on every ``DomainSource`` must be +#: one of these. Assessment-family keys gate the spine-lens filter; Staff and +#: Finance are already unfiltered and are listed here only so their source +#: documentation can be registered uniformly. +NEW_DOMAINS: tuple[str, ...] = ( + "Assessment", + "AssessmentRegistration", + "Staff", + "Finance", +) + + +@dataclass(frozen=True) +class DomainSource: + """One state's source documentation for one newly-in-scope domain. + + Only the adapter-specific pointer fields relevant to the state's source + type are populated; the rest stay empty. All collection fields are tuples + (not lists/dicts) so the dataclass stays frozen/hashable and safe as a + module-level default. + + ``enabled`` decouples *registered/documented* from *spine-lens enabled*. + A registered-but-disabled (``enabled=False``) entry records that a state's + documentation for a domain exists and where it lives, without lifting the + spine-lens placeholder collapse — used to stage the Assessment / + AssessmentRegistration un-filter behind the methodology sign-off + (``is_domain_enabled`` returns False until it is flipped). Staff / Finance + are not filtered, so ``enabled`` is immaterial for them (kept ``True``). + + Fields by adapter: + + - WI (Confluence): ``confluence_page_ids`` (extra crawl roots beyond the + default top page) and ``page_title_entity_map`` (``(page_title, entity)`` + pairs that extend ``wisconsin._LEAF_DOMAIN_ENTITY_MAP``). + - AZ / MN / IN (XLSX): ``xlsx_path`` (a new workbook, else the state's + default file) and ``xlsx_sheets`` (sheet names to ingest — extends MN's + ``_ELEMENT_SHEETS`` allowlist; informational for AZ/IN which auto-detect). + - TX (TWEDS): ``tweds_entities`` (TEDS entity names to scrape). + - ``url`` / ``notes``: provenance of the supplied link and free-text context. + """ + + state: str + domain: str + enabled: bool = True + url: str = "" + confluence_page_ids: tuple[str, ...] = () + page_title_entity_map: tuple[tuple[str, str], ...] = () + xlsx_path: str = "" + xlsx_sheets: tuple[str, ...] = () + tweds_entities: tuple[str, ...] = () + notes: str = "" + + def __post_init__(self) -> None: + if self.domain not in NEW_DOMAINS: + raise ValueError( + f"DomainSource.domain={self.domain!r} is not one of " + f"NEW_DOMAINS={NEW_DOMAINS!r}" + ) + if not self.state: + raise ValueError("DomainSource.state must be non-empty") + + +#: The registry, seeded with the domain documentation already wired into the +#: state adapters (mined from code, 2026-06-29). Each entry records where a +#: state's source documentation for a domain lives. +#: +#: ``enabled`` tier: +#: - **Staff / Finance** — not in ``domain_filter.FILTERED_DOMAINS``; they +#: already flow through both lenses today. ``enabled=True`` is immaterial to +#: the filter (it never consults the registry for them) — these entries are +#: pure provenance. +#: - **Assessment / AssessmentRegistration** — collapsed out of the spine lens +#: today. Seeded with ``enabled=False`` so artifacts stay byte-identical; +#: flipping any of these to ``enabled=True`` lifts that (state, domain) +#: placeholder collapse and is the methodology change gated on the +#: ``SCORING_PLAN_VERSION`` bump + ADR 0007 + Doug/Maria sign-off. +DOMAIN_SOURCES: tuple[DomainSource, ...] = ( + # ---- Staff (already flows; documentation-only) ---- + DomainSource( + state="AZ", domain="Staff", + notes="Use Case 12.0 .xlsm; Staff sheets via az_enrich._RULE_DOMAIN_TO_SHEETS. 38 documented rows.", + ), + DomainSource( + state="IN", domain="Staff", + url="https://idoe.atlassian.net/wiki/spaces/IKHTV/", + notes="IDOE Confluence (space IKHTV) + Vendor Docs xlsx; 6 Staff page maps in idoe_confluence. 42 documented rows.", + ), + DomainSource( + state="MN", domain="Staff", + url="https://raw.githubusercontent.com/mn-mde-edfi/MDE-EdFi-Documentation/master/2026-27%20MDE%20Ed-Fi%20Documentation/2026-2027%20Data%20Mapping%20Matrix%20Ed-Fi%20Suite%203%20v%206.2.xlsx", + notes="MDE Data Mapping Matrix; Staff rows arrive via FK refs in the Student Enrollment / MCCC sheets. 5 documented rows.", + ), + DomainSource( + state="TX", domain="Staff", + notes="TWEDS v33 (TEDS) scrape (data/raw/tx/tweds/). 82 documented rows.", + ), + DomainSource( + state="WI", domain="Staff", + url="https://wisconsindpi.atlassian.net/wiki/spaces/widpiedfi/pages/2294032", + notes="DPI Confluence crawl (TOP_PAGE_ID=2294032); Staff demographics inline. 58 documented rows.", + ), + # ---- Finance (already flows; documentation-only) ---- + DomainSource( + state="WI", domain="Finance", + url="https://wisconsindpi.atlassian.net/wiki/spaces/widpiedfi/pages/2294032", + notes="DPI Confluence — SAFR pages (chartOfAccount / LocalActual). 15 documented rows.", + ), + # ---- Assessment ---- + # TX + IN enabled under SCORING_PLAN_VERSION 27 (ADR 0007, 2026-06-29). + DomainSource( + state="TX", domain="Assessment", enabled=True, + notes="TWEDS v33 (TEDS) assessment entities. 31 documented rows. " + "Enabled v27 — lifts the TX spine-lens Assessment placeholder.", + ), + DomainSource( + state="IN", domain="Assessment", enabled=True, + url="https://idoe.atlassian.net/wiki/spaces/IKHTV/", + notes="IDOE Confluence assessment-accommodation page. 3 documented rows. " + "Enabled v27 — lifts the IN spine-lens Assessment placeholder.", + ), + DomainSource( + state="WI", domain="Assessment", enabled=False, + url="https://wisconsindpi.atlassian.net/wiki/spaces/widpiedfi/pages/2294032", + page_title_entity_map=(("Assessment (Public LEAs Only)", "StudentAssessment"),), + notes="DPI Confluence assessment page is mapped (wisconsin._LEAF_DOMAIN_ENTITY_MAP) " + "but yields 0 documented pure-assessment rows today — investigate the page parse " + "before enabling. Deliberately left disabled in the v27 enablement.", + ), + # ---- AssessmentRegistration ---- + DomainSource( + state="IN", domain="AssessmentRegistration", enabled=True, + url="https://idoe.atlassian.net/wiki/spaces/IKHTV/", + notes="IDOE Confluence. 5 documented rows. AR entities present in the IN spine. " + "Enabled v27 — lifts the IN spine-lens AssessmentRegistration collapse. " + "(AR entities are absent from MN/TX spines, so AR is not enabled there.)", + ), +) + + +#: States that deliberately carry NO ``DomainSource`` entries. Every +#: ``src.states.SUPPORTED_STATES`` member must either appear in +#: ``DOMAIN_SOURCES`` or be listed here explicitly — +#: ``tests/test_state_roster.py`` fails otherwise (issue #213 item 2), so a +#: sixth state can't ship in the "nobody decided about its domain +#: documentation" limbo silently. Empty today: all five states register at +#: least a Staff entry. +NO_DOMAIN_SOURCES: frozenset[str] = frozenset() + + +def sources_for(state: str) -> list[DomainSource]: + """All registered ``DomainSource`` entries for ``state`` (case-insensitive), + enabled or not — adapters use this to see provenance regardless of state.""" + s = state.upper() + return [d for d in DOMAIN_SOURCES if d.state.upper() == s] + + +def is_domain_enabled(state: str, domain: str) -> bool: + """True iff an **enabled** source link is registered for this (state, + domain) pair. + + This is the gate ``domain_filter`` consults to decide whether to lift the + spine-lens placeholder collapse for a filtered domain. Case-insensitive on + state; ``domain`` is matched exactly against ``NEW_DOMAINS`` keys. A + registered-but-disabled (``enabled=False``) entry returns False here. + """ + s = state.upper() + return any( + d.state.upper() == s and d.domain == domain and d.enabled + for d in DOMAIN_SOURCES + ) + + +def enabled_domains(state: str) -> frozenset[str]: + """The set of ``NEW_DOMAINS`` keys **enabled** for ``state``.""" + return frozenset(d.domain for d in sources_for(state) if d.enabled) diff --git a/apps/pipeline/src/ingest/gap_surfacer.py b/apps/pipeline/src/ingest/gap_surfacer.py new file mode 100644 index 0000000..418c2a9 --- /dev/null +++ b/apps/pipeline/src/ingest/gap_surfacer.py @@ -0,0 +1,529 @@ +"""Spine-anchored coverage-gap surfacer (issue #66 Layer 2). + +Walks ``{state}_elements_source.json`` + ``{state}_spine.json`` and emits a +sibling artifact ``{state}_elements_gap.json`` enumerating every spine +``(entity, element)`` pair the state's source doc is silent on. Two +``discovery`` classes: + +- ``spine_within_documented_entity`` — entity DOES appear in the source + doc but this specific element is missing from the source's enumeration. +- ``spine_only_full_entity`` — entity does not appear in the source doc at + all (typically TEA / WI / MN extension entities the state's source-doc + scrape didn't reach). + +Methodology discipline (CLAUDE.md "Scope guardrails"): + +- The surfacer is **a separate artifact**. It does NOT mutate + ``{state}_elements_source.json`` — the source-lens contract ("the + state's source doc enumerates what's in scope; spine enriches") is + preserved by construction. +- The surfacer never reads the reviewer xlsx. The reviewer pass + oracled this gap shape into focus (issue #62 → #66) but is not a + data input — gap output depends only on spine + source-lens output. +- A consumer that treats the gap file as in-scope is making an + explicit choice. Source-lens percentages stay comparable across + states because the gap rows live elsewhere. + +Output schema (one record per gap pair): + +```jsonc +{ + "state": "TX", + "entity": "tx_priorYearLeaver", + "element_name": "DiplomaType", + "discovery": "spine_only_full_entity", + "documented_in_source": false, + "spine_data_type": "Descriptor", + "spine_extension_name": "tx_priorYearLeaver", + "rationale": "Entity not enumerated in source doc; spine catalog confirms TEA extension entity" +} +``` + +CLI: ``mc ingest gap --state STATE`` (or ``--state all``). Outputs land +at ``data/out/{state}_elements_gap.json``. +""" + +from __future__ import annotations + +import json +import logging +from datetime import datetime, timezone +from pathlib import Path +from typing import Iterable + +from src.ingest.domain_filter import entity_filter_domain +from src.ingest.shared import canonical_type +from src.models.alias_grammar import ( + capitalize_first, + fk_prefixed_alias, + reference_prefix, +) +from src.models.element import StateElements +from src.models.spine import StateSpine +from src.utils.matching import ( + match_key, + entity_match_form, + record_match_keys, +) +from src.utils.paths import ( + state_elements_gap_path, + state_elements_path, + state_spine_path, +) + +logger = logging.getLogger(__name__) + +# discovery classes — keep short, machine-readable; consumers may switch on +# these directly. +DISCOVERY_WITHIN = "spine_within_documented_entity" +DISCOVERY_FULL_ENTITY = "spine_only_full_entity" + + +# `schoolId` → `SchoolId` — one rule home (alias_grammar; issue #213 +# item 2). Kept under the historical local name for the concat call sites. +_capitalize_first = capitalize_first + + +def _source_keys(records: Iterable) -> set[tuple[str, str]]: + """Build the alias-expanded source-side membership set. + + For each source record ``r``, expand ``record_match_keys(r.entity, + r.element_name)`` into ``(normalized_entity, lowered_alias)`` pairs. + Used to test whether a spine pair is already documented under any + naming convention the state's source doc emitted. + """ + keys: set[tuple[str, str]] = set() + for r in records: + keys |= record_match_keys(r.entity, r.element_name) + return keys + + +def _source_entities(records: Iterable) -> set[str]: + """Set of normalized entity names the source doc touched at all. + + Drives the discovery classification: if the entity appears here, gap + rows for that entity are ``spine_within_documented_entity``; + otherwise ``spine_only_full_entity``. + """ + return {entity_match_form(r.entity) for r in records} + + +def _swagger_backfilled_entities(records: Iterable) -> set[str]: + """Set of normalized entity names whose source-lens presence is + exclusively via swagger backfill (issue #70). + + The swagger-backfill module appends rows only for entities the source + doc was silent on; a partial-coverage entity never triggers backfill. + So the membership test reduces to "any row carries + documentation_source='swagger'." Used by ``surface_gaps()`` to skip + these entities entirely — their full property set is now claimed as + documented (low-quality, but documented), so emitting them as + ``spine_only_full_entity`` would double-count and emitting their + Reference/Collection slots as ``spine_within_documented_entity`` + would mis-classify swagger-only entities as authored-prose entities. + """ + return { + entity_match_form(r.entity) + for r in records + if getattr(r, "documentation_source", "source_doc") == "swagger" + } + + +def _is_documented( + entity: str, + element_candidates: Iterable[str], + source_keys: set[tuple[str, str]], +) -> bool: + """Return True if any candidate (entity, element) form is in source_keys. + + ``element_candidates`` is an ordered set of equivalent source-side + representations the spine primary may map to (bare leaf, FK concat, + sub-collection concat). Any one match marks the spine slot + documented. + """ + for cand in element_candidates: + e_norm, n_norm = match_key(entity, cand) + if (e_norm, n_norm.lower()) in source_keys: + return True + # Also test record_match_keys-expanded aliases — covers + # path-tail / arrow-nav variants the source might have emitted. + for ent_n, alias_lower in record_match_keys(entity, cand): + if (ent_n, alias_lower) in source_keys: + return True + return False + + +def _emit_gap( + state: str, + entity: str, + element_name: str, + *, + discovery: str, + spine_data_type: str, + spine_extension_name: str | None, + rationale: str, + sub_collection: str | None = None, + leaf_name: str | None = None, +) -> dict: + """Build a gap record dict. + + ``element_name`` is the canonical source-side form (concat for + sub-collection elements). ``sub_collection`` + ``leaf_name`` carry + the deconstructed parts so the reviewer-comparison lookup can + register both the concat and bare-leaf aliases — without that, + reviewer rows like ``PriorYearLeaver / DiplomaType`` would never + resolve to the gap row whose concat name is + ``graduationSetDiplomaTypeDescriptor``. + """ + record = { + "state": state, + "entity": entity, + "element_name": element_name, + "discovery": discovery, + "documented_in_source": False, + "spine_data_type": spine_data_type, + "spine_extension_name": spine_extension_name, + "rationale": rationale, + } + if sub_collection: + record["sub_collection"] = sub_collection + if leaf_name: + record["leaf_name"] = leaf_name + return record + + +def _classify_discovery( + entity: str, source_entities: set[str] +) -> tuple[str, str]: + """Return ``(discovery_class, rationale_prefix)``.""" + if entity_match_form(entity) in source_entities: + return ( + DISCOVERY_WITHIN, + "Entity enumerated in source doc but element missing from its row set", + ) + return ( + DISCOVERY_FULL_ENTITY, + "Entity not enumerated in source doc; spine catalog confirms canonical position", + ) + + +def surface_gaps( + state: str, + elements_path: Path | None = None, + spine_path: Path | None = None, +) -> list[dict]: + """Walk spine catalog vs source-lens output; return gap records (newest API). + + Pure function — does not write to disk. Public so tests can drive it + with synthetic spines / fixtures without involving the filesystem. + The CLI wrapper ``run()`` handles disk IO and aggregates per-state + results into ``state_elements_gap_path(state)``. + """ + elements_p = elements_path or state_elements_path(state, "source") + spine_p = spine_path or state_spine_path(state) + + elements = StateElements.model_validate_json( + elements_p.read_text(encoding="utf-8") + ) + spine = StateSpine.model_validate_json(spine_p.read_text(encoding="utf-8")) + + src_keys = _source_keys(elements.elements) + src_ents = _source_entities(elements.elements) + swagger_ents = _swagger_backfilled_entities(elements.elements) + + gaps: list[dict] = [] + seen: set[tuple[str, str]] = set() # (entity_lower, element_lower) + + # Apply the same SIS-never-populated domain filter as the spine-lens + # placeholder collapse — entities in Assessment / Survey / Standards / + # Gradebook / Intervention are not authentic gap signal because no + # SIS vendor populates them. Treat them as out-of-scope here so the + # gap artifact stays focused on the ingestion surface that matters + # to scoring. + def _filtered(entity_name: str) -> bool: + return entity_filter_domain(entity_name, spine) is not None + + def _swagger_only(entity_name: str) -> bool: + """True when the entity's source-lens presence is exclusively + swagger-backfilled (issue #70). Skip it in the gap surface — the + backfill claims the whole entity, so no within-entity gap rows + should be emitted. + """ + return entity_match_form(entity_name) in swagger_ents + + def _track(entity: str, element_name: str) -> bool: + """Return True if this (entity, element) was not previously emitted.""" + key = (entity.lower(), element_name.lower()) + if key in seen: + return False + seen.add(key) + return True + + def _document_check( + entity: str, + element_name: str, + also: list[str] | None = None, + ) -> bool: + candidates = [element_name] + if also: + candidates.extend(also) + return _is_documented(entity, candidates, src_keys) + + # -- core entities -------------------------------------------------- + for entity_name, entity in spine.catalog.entities.items(): + if _filtered(entity_name): + continue + if _swagger_only(entity_name): + continue + # Top-level scalar/descriptor properties. + for prop_name, prop in entity.properties.items(): + if _document_check(entity_name, prop_name): + continue + if not _track(entity_name, prop_name): + continue + disc, rat = _classify_discovery(entity_name, src_ents) + gaps.append( + _emit_gap( + state, + entity_name, + prop_name, + discovery=disc, + spine_data_type=canonical_type( + prop_name, prop.type, prop.format + ), + spine_extension_name=None, + rationale=rat, + ) + ) + + # References + their key properties. We treat the reference as + # documented if the source touched any of its FK projections — + # source docs commonly enumerate the leaf FK rather than the + # reference object itself. + for ref_name, ref in entity.references.items(): + ref_concat_aliases: list[str] = [] + prefix = reference_prefix(ref_name) + # Deliberately NARROWER than `element_keys` (issue #213 item + # 2, documented divergence): no camel-collapse, no + # UniqueId→Id, no EdOrg subtypes, no descriptor variants — + # so the gap artifact can classify a slot undocumented that + # coverage counted matched. Widening is a measured follow-up + # (see the alias-tier histogram in the gap log). The + # `ref_name + CapKey` form below is this consumer's OWN + # extra (`schoolReferenceSchoolId`) — no other consumer + # emits it. + for kp in ref.key_properties: + ref_concat_aliases.append(kp) + if prefix and prefix != kp: + ref_concat_aliases.append(fk_prefixed_alias(prefix, kp)) + ref_concat_aliases.append(fk_prefixed_alias(ref_name, kp)) + + if _document_check(entity_name, ref_name, also=ref_concat_aliases): + continue + if not _track(entity_name, ref_name): + continue + disc, rat = _classify_discovery(entity_name, src_ents) + gaps.append( + _emit_gap( + state, + entity_name, + ref_name, + discovery=disc, + spine_data_type="Reference", + spine_extension_name=None, + rationale=rat, + ) + ) + + # Sub-collections — emit one gap per sub-collection property + # (most analyst-relevant grain) using the source-side concat form + # as the canonical element name so it lines up with how + # state-adapter ingest emits sub-collection rows. + for sub_name, sub in entity.sub_collections.items(): + for sp_name, sp in sub.properties.items(): + concat = sub_name + _capitalize_first(sp_name) + if _document_check( + entity_name, sp_name, also=[concat, sub_name] + ): + continue + if not _track(entity_name, concat): + continue + disc, rat = _classify_discovery(entity_name, src_ents) + gaps.append( + _emit_gap( + state, + entity_name, + concat, + discovery=disc, + spine_data_type=canonical_type( + sp_name, sp.type, sp.format + ), + spine_extension_name=None, + rationale=rat, + sub_collection=sub_name, + leaf_name=sp_name, + ) + ) + + # -- extensions ----------------------------------------------------- + # Extensions attribute back to ``ext.extends_entity``. The discovery + # classification uses ``extends_entity`` for the "is the entity in + # source?" test, but the emitted record carries ``extends_entity`` as + # the entity name (matching the source-lens convention) and + # ``ext_key`` as the spine_extension_name attribution. + for ext_key, ext in spine.catalog.extensions.items(): + target = ext.extends_entity + if _filtered(target): + continue + if _swagger_only(target): + continue + for prop_name, prop in ext.properties.items(): + if _document_check(target, prop_name): + continue + if not _track(target, prop_name): + continue + disc, rat = _classify_discovery(target, src_ents) + # Extension attribution upgrades the rationale prefix — + # surfaces the extension key explicitly so analysts know + # which schema contributed. + rat_ext = ( + f"{rat}; contributed by extension {ext_key}" + if disc == DISCOVERY_WITHIN + else ( + "Entity not enumerated in source doc; spine catalog " + f"confirms extension entity {ext_key}" + ) + ) + gaps.append( + _emit_gap( + state, + target, + prop_name, + discovery=disc, + spine_data_type=canonical_type( + prop_name, prop.type, prop.format + ), + spine_extension_name=ext_key, + rationale=rat_ext, + ) + ) + + # Extension references (e.g. TEA `tx_priorYearLeaver` redeclares + # `studentReference`). Treat documented if the FK leaf is in + # source; otherwise emit. + for ref_name, ref in ext.references.items(): + ref_concat_aliases: list[str] = [] + prefix = reference_prefix(ref_name) + # Deliberately NARROWER than `element_keys` (issue #213 item + # 2, documented divergence): no camel-collapse, no + # UniqueId→Id, no EdOrg subtypes, no descriptor variants — + # so the gap artifact can classify a slot undocumented that + # coverage counted matched. Widening is a measured follow-up + # (see the alias-tier histogram in the gap log). The + # `ref_name + CapKey` form below is this consumer's OWN + # extra (`schoolReferenceSchoolId`) — no other consumer + # emits it. + for kp in ref.key_properties: + ref_concat_aliases.append(kp) + if prefix and prefix != kp: + ref_concat_aliases.append(fk_prefixed_alias(prefix, kp)) + ref_concat_aliases.append(fk_prefixed_alias(ref_name, kp)) + + if _document_check(target, ref_name, also=ref_concat_aliases): + continue + if not _track(target, ref_name): + continue + disc, rat = _classify_discovery(target, src_ents) + rat_ext = ( + f"{rat}; contributed by extension {ext_key}" + if disc == DISCOVERY_WITHIN + else ( + "Entity not enumerated in source doc; spine catalog " + f"confirms extension entity {ext_key}" + ) + ) + gaps.append( + _emit_gap( + state, + target, + ref_name, + discovery=disc, + spine_data_type="Reference", + spine_extension_name=ext_key, + rationale=rat_ext, + ) + ) + + # Extension sub-collections. + for sub_name, sub in ext.sub_collections.items(): + for sp_name, sp in sub.properties.items(): + concat = sub_name + _capitalize_first(sp_name) + if _document_check( + target, sp_name, also=[concat, sub_name] + ): + continue + if not _track(target, concat): + continue + disc, rat = _classify_discovery(target, src_ents) + rat_ext = ( + f"{rat}; contributed by extension {ext_key}" + if disc == DISCOVERY_WITHIN + else ( + "Entity not enumerated in source doc; spine " + f"catalog confirms extension entity {ext_key}" + ) + ) + gaps.append( + _emit_gap( + state, + target, + concat, + discovery=disc, + spine_data_type=canonical_type( + sp_name, sp.type, sp.format + ), + spine_extension_name=ext_key, + rationale=rat_ext, + sub_collection=sub_name, + leaf_name=sp_name, + ) + ) + + # Stable order: by (entity, element) so re-runs produce byte-identical + # output across processes (avoids hash-order churn in PYTHONHASHSEED). + gaps.sort(key=lambda g: (g["entity"], g["element_name"])) + return gaps + + +def write_gap_artifact(state: str, gaps: list[dict]) -> Path: + """Write ``{state}_elements_gap.json`` with the standard envelope.""" + out_path = state_elements_gap_path(state) + out_path.parent.mkdir(parents=True, exist_ok=True) + discovery_counts: dict[str, int] = {} + for g in gaps: + discovery_counts[g["discovery"]] = ( + discovery_counts.get(g["discovery"], 0) + 1 + ) + payload = { + "state": state.upper(), + "generated_at": datetime.now(timezone.utc).isoformat(), + "gap_count": len(gaps), + "discovery_counts": discovery_counts, + "gaps": gaps, + } + out_path.write_text( + json.dumps(payload, indent=2) + "\n", encoding="utf-8" + ) + logger.info("Wrote %d gap rows to %s", len(gaps), out_path) + return out_path + + +def run(state: str) -> Path: + """CLI entry — surface gaps for a single state and write the artifact. + + Plain function (NOT @click.command) per the CLI-wiring convention in + CLAUDE.md / `tests/test_cli_e2e.py::TestCliWiring`. The Click command + in `cli.py` calls this. + """ + gaps = surface_gaps(state.upper()) + return write_gap_artifact(state.upper(), gaps) diff --git a/apps/pipeline/src/ingest/idoe_confluence.py b/apps/pipeline/src/ingest/idoe_confluence.py new file mode 100644 index 0000000..fef5cd5 --- /dev/null +++ b/apps/pipeline/src/ingest/idoe_confluence.py @@ -0,0 +1,634 @@ +"""Harvest IDOE Knowledge Hub Confluence prose into per-element enrichment data. + +Source: https://idoe.atlassian.net/wiki/spaces/IKHTV/ — public Ed-Fi data +exchange knowledge hub. Anonymous Confluence REST API works. + +Two prose-bearing page-type families: + +1. ``{Domain}: Descriptors`` and ``{Domain}: Types`` — per-descriptor tables of + ``(code_value, [short_description], description)``. Joins to (entity, + element) via the IDOE ``Enumerations`` column from the API Datastructure + sheet (descriptor name match). Yields ``definition_text`` for descriptor- + typed elements. + +2. ``{Domain}: General Reporting Info[rmation]`` and ``Reporting Guide: + {Domain}`` — free-form narrative with overview / business-purpose / + regulatory-citation prose. Joins to elements via a Confluence-domain → + API-resource keymap (``_DOMAIN_RESOURCE_KEYMAP``). Yields + ``business_rules_text``. + +Caching: every fetched page is written to +``data/raw/in/confluence/pages/{pageId}.json``. The harvested digest is written +to ``data/raw/in/confluence/harvested.json`` and is what ``indiana.run()`` +reads at ingest time. + +Entry point: ``harvest()`` (or run ``mc ingest in`` — the IN adapter +delegates to ``ensure_harvested()``). +""" + +from __future__ import annotations + +import json +import logging +import re +import time +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path + +import httpx +from bs4 import BeautifulSoup, Tag + +logger = logging.getLogger(__name__) + +CONFLUENCE_BASE = "https://idoe.atlassian.net" +SPACE_KEY = "IKHTV" +API_V1_CONTENT = f"{CONFLUENCE_BASE}/wiki/rest/api/content" +API_V1_SPACE_PAGES = f"{CONFLUENCE_BASE}/wiki/rest/api/space/{SPACE_KEY}/content/page" + +# Page-title patterns we harvest. Anything not matching is skipped — keeps the +# fetched corpus narrow and the regen cycle cheap. +_DESCRIPTOR_TITLE_RE = re.compile(r":\s*(Descriptors|Types)$", re.IGNORECASE) +_NARRATIVE_TITLE_RE = re.compile( + r"(?:: General Reporting Info(?:rmation)?$|^Reporting Guide:\s)", + re.IGNORECASE, +) + +# Confluence Knowledge Hub uses a "Domain Name" prefix on each prose page +# (e.g. "Attendance: Descriptors"). Map those domain names to the IDOE API +# resources that the prose narrative covers, so the per-resource business- +# rules backfill in ``indiana.py`` can find the right narrative. +# +# Many-to-many is normal: ``Calendar`` covers calendars + calendarDates + +# gradingPeriods because the Reporting Guide blends them. Resource names use +# the camelCase form that appears in the IDOE API Datastructure sheet's +# ``API Resource`` column. +_DOMAIN_RESOURCE_KEYMAP: dict[str, tuple[str, ...]] = { + "Attendance": ("studentSchoolAttendanceEvents",), + "Calendar": ("calendars", "calendarDates"), + "Cohorts (Grouping)": ("StudentCohortAssociation",), + "Curricular Materials Assistance": ( + "studentCurricularMaterialProgramAssociations", + ), + "Membership": ( + "studentEducationOrganizationResponsibilityAssociation", + ), + "Pupil Enrollment": ( + "studentEducationOrganizationResponsibilityAssociation", + ), + "Enrollment": ("studentSchoolAssociations",), + "Student Demographics": ( + "studentEducationOrganizationAssociations", + "students", + "parents", + "studentParentAssociations", + ), + "Discipline": ( + "disciplineActions", + "disciplineIncidents", + "disciplineincidents", + "studentDisciplineIncidentBehaviorAssociations", + ), + "Student Academic Record (Course Outcomes)": ( + "studentSectionAssociations", + "courseTranscripts", + "studentAcademicRecords", + "sections", + "sessions", + "gradingPeriods", + "courseOfferings", + "courses", + ), + "Staff Assignment": ( + "staffEducationOrganizationAssignmentAssociation", + "staffs", + ), + "Staff Contact": ("staffEducationOrganizationContactAssociation",), + "Staff Employment": ( + "staffEducationOrganizationEmploymentAssociation", + ), + "Staff Injury": ("staffEducationOrganizationEmploymentAssociation",), + "Staff Other Personnel": ("educationOrganizationOtherPersonnels",), + "Staff Section": ("staffSectionAssociation",), + "Special Education": ("studentSpecialEducationProgramAssociations",), + "Special Education Evaluation": ( + "studentSpecialEducationProgramAssociations", + ), + "Special Education Termination": ( + "studentSpecialEducationProgramAssociations", + ), + "Title I": ("studentTitleIPartAProgramAssociations",), + "Alternative Education": ( + "studentAlternativeEducationProgramAssociations", + ), + "Homebound and Hospitalized": ("studentProgramAssociations",), + "Additional Student Programs": ( + "studentProgramAssociations", + "programs", + "StudentCTEProgramAssociations", + ), + "Multilingual Learners": ("studentProgramAssociations",), + "Student Accommodations": ( + "studentEducationOrganizationAssessmentAccommodations", + "assessmentAccommodations", + "Surveys", + "SurveyQuestions", + "surveyQuestionResponses", + "SurveyResponse", + "surveyResponseEducationOrganizationTargetAssociations", + ), + "Graduate": ("studentAcademicRecords",), +} + +# Resources that aren't covered by any IDOE domain narrative — listed here so +# the keymap-coverage smoke test in tests can assert intentional omission +# rather than silently failing. ``schools`` / ``localEducationAgencies`` in +# particular are EdOrg foundation rows that IDOE doesn't author per-domain +# prose for; the elements still get descriptor-level prose for descriptor- +# typed columns. +_NO_NARRATIVE_RESOURCES = frozenset({ + "schools", + "localEducationAgencies", + "Calendars", # capitalized variant for state Calendar Events sub-collection + "postSecondaryInstitutions", + "communityProviders", +}) + + +# --------------------------------------------------------------------------- +# Confluence client +# --------------------------------------------------------------------------- + + +def _new_client() -> httpx.Client: + return httpx.Client( + headers={"User-Agent": "Mozilla/5.0 (POC-3 IDOE harvester)"}, + timeout=30.0, + follow_redirects=True, + ) + + +def list_space_pages(client: httpx.Client) -> list[dict]: + """Return all page stubs in the IKHTV space (id, title).""" + pages: list[dict] = [] + start = 0 + limit = 200 + while True: + resp = client.get(API_V1_SPACE_PAGES, params={"limit": limit, "start": start}) + resp.raise_for_status() + data = resp.json() + results = data.get("results") or data.get("page", {}).get("results", []) or [] + if not results: + break + for r in results: + pages.append({"id": r["id"], "title": r["title"]}) + size = data.get("size", len(results)) + if size < limit: + break + start += size + time.sleep(0.2) + return pages + + +def fetch_page_body(client: httpx.Client, page_id: str) -> dict: + """Fetch a single page with storage-format body.""" + url = f"{API_V1_CONTENT}/{page_id}" + resp = client.get(url, params={"expand": "body.storage"}) + resp.raise_for_status() + return resp.json() + + +# --------------------------------------------------------------------------- +# Page filtering +# --------------------------------------------------------------------------- + + +def is_descriptor_page(title: str) -> bool: + return bool(_DESCRIPTOR_TITLE_RE.search(title)) + + +def is_narrative_page(title: str) -> bool: + return bool(_NARRATIVE_TITLE_RE.search(title)) + + +def page_domain(title: str) -> str | None: + """Extract the domain prefix from a page title. + + ``"Calendar: Descriptors"`` → ``"Calendar"``. + ``"Reporting Guide: Membership"`` → ``"Membership"``. + ``"Enrollment: General Reporting Info"`` → ``"Enrollment"``. + """ + if title.startswith("Reporting Guide:"): + return title[len("Reporting Guide:"):].strip() + if ":" in title: + return title.split(":", 1)[0].strip() + return None + + +# --------------------------------------------------------------------------- +# Body parsing +# --------------------------------------------------------------------------- + + +_SKIP_MACROS = { + "appanvil-karma-designer", # The header / banner layout JSON blob. + "html-canvas", + "iframe", +} + + +def _clean_storage_xml(body_xml: str) -> BeautifulSoup: + """Parse Confluence storage XML and drop layout-macro JSON blobs. + + Wraps the body in a synthetic root element so BeautifulSoup parses the + Confluence ``ac:`` namespaced tags as regular tags. Drops Karma-designer + macros (the ones that embed long JSON parameters) so subsequent text- + extraction doesn't pick up the JSON noise. + """ + wrapped = ( + '' + body_xml + "" + ) + soup = BeautifulSoup(wrapped, "html.parser") + for macro in soup.find_all("ac:structured-macro"): + name = macro.get("ac:name", "") + if name in _SKIP_MACROS: + macro.decompose() + return soup + + +_DESCRIPTOR_NAME_RE = re.compile( + r"^[A-Z][A-Za-z0-9 ]*Descriptors?$" +) + + +def canonical_descriptor_key(name: str) -> str: + """Canonicalize a descriptor name for lookup. + + Confluence labels descriptors in three flavours that all need to collide + on a single key for matching against the XLSX ``Enumerations`` column: + + - ``CalendarEventDescriptor`` (PascalCase singular — the XLSX form) + - ``Calendar Type Descriptors`` (spaced plural — Confluence heading form) + - ``calendar event descriptor`` (any-case fallback) + + Strategy: lowercase, drop non-alphanumerics, strip a trailing ``s`` after + ``descriptor``. + """ + cleaned = re.sub(r"[^a-z0-9]+", "", name.lower()) + if cleaned.endswith("descriptors"): + cleaned = cleaned[:-1] # plural → singular + return cleaned + + +@dataclass +class DescriptorTable: + descriptor_name: str + rows: list[dict[str, str]] = field(default_factory=list) + + +def _heading_text_above(table: Tag) -> str | None: + """Walk back from a table to find the nearest preceding text that names a + descriptor (e.g. ``CalendarTypeDescriptors``). + + Confluence storage XML places descriptor labels in any of: an ``

`` / + ``

`` / ```` / a leading paragraph just before the table. We + walk previous siblings of the table (and its ancestors) until we find a + text node that matches the descriptor pattern. + """ + candidates: list[Tag] = [] + cur = table + # Walk up at most 3 ancestors and gather their preceding siblings + for _ in range(4): + if cur is None: + break + candidates.extend(cur.find_all_previous( + ["h1", "h2", "h3", "h4", "p", "strong", "b"], limit=20, + )) + cur = cur.parent + for cand in candidates: + text = cand.get_text(strip=True) + if not text: + continue + # Strip non-word noise (e.g. trailing emoji, parens) + m = re.search(r"([A-Za-z][A-Za-z0-9 ]*Descriptors?)", text) + if m: + normalized = m.group(1).strip() + # Single-word "Descriptor"/"Descriptors" bare label is too generic + if normalized.lower() in ("descriptor", "descriptors"): + continue + return normalized + return None + + +def _row_cells(row: Tag) -> list[str]: + cells = row.find_all(["td", "th"]) + out: list[str] = [] + for c in cells: + text = c.get_text(separator=" ", strip=True) + text = re.sub(r"\s+", " ", text) + out.append(text) + return out + + +def extract_descriptor_tables(soup: BeautifulSoup) -> list[DescriptorTable]: + """Walk all tables in a Descriptors page and emit per-descriptor rows. + + A table qualifies if its preceding heading names a descriptor, OR its + header row contains both a code-value column and a description column. + """ + out: list[DescriptorTable] = [] + for table in soup.find_all("table"): + rows = table.find_all("tr") + if len(rows) < 2: + continue + header_cells = _row_cells(rows[0]) + header_text = " ".join(header_cells).lower() + # Must look like a descriptor table (code/value + description columns) + # OR have a descriptor-named header above it. + looks_like_descriptor = ( + ("code" in header_text or "value" in header_text or "short description" in header_text) + and ("description" in header_text or "definition" in header_text or "uses" in header_text) + ) + descriptor_name = _heading_text_above(table) + if not (looks_like_descriptor or descriptor_name): + continue + # Map header columns to canonical roles + col_map: dict[str, int] = {} + for i, h in enumerate(header_cells): + hl = h.lower() + if "code" in hl or hl in ("value", "values", "code values", "code value"): + col_map.setdefault("code", i) + elif "short description" in hl: + col_map.setdefault("short", i) + elif "description" in hl or "definition" in hl or "uses" in hl: + col_map.setdefault("desc", i) + if "desc" not in col_map and len(header_cells) >= 2: + # Fallback: treat last column as the description. + col_map["desc"] = len(header_cells) - 1 + if "code" not in col_map and len(header_cells) >= 1: + col_map["code"] = 0 + + if not descriptor_name: + descriptor_name = "UNKNOWN_DESCRIPTOR" + + table_rec = DescriptorTable(descriptor_name=descriptor_name) + seen_codes: set[str] = set() + for row in rows[1:]: + cells = _row_cells(row) + if not cells: + continue + code = cells[col_map["code"]] if col_map.get("code") is not None and col_map["code"] < len(cells) else "" + desc = cells[col_map["desc"]] if col_map.get("desc") is not None and col_map["desc"] < len(cells) else "" + short = cells[col_map["short"]] if col_map.get("short") is not None and col_map["short"] < len(cells) else "" + code = code.strip() + desc = desc.strip() + short = short.strip() + if not code: + continue + if code in seen_codes: + continue + seen_codes.add(code) + table_rec.rows.append({ + "code_value": code, + "short_description": short, + "description": desc, + }) + if table_rec.rows: + out.append(table_rec) + return out + + +_NARRATIVE_SKIP_TOKENS = ( + "click here to view", + "was this documentation helpful", + "submit a ticket", + "powered by", + "feedback", + "did this documentation help", + "submit feedback", +) + + +def extract_narrative(soup: BeautifulSoup) -> str: + """Pull free-form narrative paragraphs / list items from a page. + + Used for ``Reporting Guide:`` and ``: General Reporting Info`` pages, where + the prose explains business purpose / regulatory citation / scope. + """ + parts: list[str] = [] + for el in soup.find_all(["p", "li", "blockquote"]): + if el.find_parent("table"): + continue + text = el.get_text(separator=" ", strip=True) + text = re.sub(r"\s+", " ", text) + if len(text) < 20: + continue + lower = text.lower() + if any(skip in lower for skip in _NARRATIVE_SKIP_TOKENS): + continue + # Skip the residual Karma JSON ("name":"page" / "templateId":"..."), + # which sneaks in via paragraphs that hold layout config. + if text.count('":"') > 2 or text.count('","') > 4: + continue + parts.append(text) + return "\n".join(parts).strip() + + +# --------------------------------------------------------------------------- +# Cache + harvest orchestration +# --------------------------------------------------------------------------- + + +def _cache_dir() -> Path: + """``data/raw/in/confluence/`` — gitignored under ``data/raw/``.""" + project_root = Path(__file__).resolve().parents[2] + return project_root / "data" / "raw" / "in" / "confluence" + + +def _pages_cache_dir() -> Path: + return _cache_dir() / "pages" + + +def _list_cache_path() -> Path: + return _cache_dir() / "page_index.json" + + +def _digest_path() -> Path: + return _cache_dir() / "harvested.json" + + +def _load_or_fetch_page(client: httpx.Client, page_id: str) -> dict: + pages_dir = _pages_cache_dir() + cached = pages_dir / f"{page_id}.json" + if cached.exists(): + return json.loads(cached.read_text(encoding="utf-8")) + data = fetch_page_body(client, page_id) + pages_dir.mkdir(parents=True, exist_ok=True) + cached.write_text(json.dumps(data), encoding="utf-8") + return data + + +def _load_or_fetch_index(client: httpx.Client) -> list[dict]: + cache = _list_cache_path() + if cache.exists(): + return json.loads(cache.read_text(encoding="utf-8")) + pages = list_space_pages(client) + cache.parent.mkdir(parents=True, exist_ok=True) + cache.write_text(json.dumps(pages, indent=2), encoding="utf-8") + return pages + + +def harvest(force: bool = False) -> dict: + """Run the full Confluence harvest. Returns the digest dict and writes it + to ``data/raw/in/confluence/harvested.json``. + + Subsequent calls are cache-served unless ``force=True``. + """ + digest_path = _digest_path() + if digest_path.exists() and not force: + return json.loads(digest_path.read_text(encoding="utf-8")) + + descriptors: dict[str, dict[str, str]] = {} + descriptor_short: dict[str, dict[str, str]] = {} + domain_narratives: dict[str, str] = {} + descriptor_pages: list[str] = [] + narrative_pages: list[str] = [] + + with _new_client() as client: + pages = _load_or_fetch_index(client) + logger.info("IDOE Confluence: %d pages in space %s", len(pages), SPACE_KEY) + + # Filter to prose-bearing titles + descriptor_titles = [p for p in pages if is_descriptor_page(p["title"])] + narrative_titles = [p for p in pages if is_narrative_page(p["title"])] + logger.info( + " %d descriptor pages + %d narrative pages will be harvested", + len(descriptor_titles), len(narrative_titles), + ) + + for p in descriptor_titles: + try: + page = _load_or_fetch_page(client, p["id"]) + except Exception as exc: + logger.warning("Failed to fetch %s (%s): %s", p["title"], p["id"], exc) + continue + body_xml = page.get("body", {}).get("storage", {}).get("value", "") + if not body_xml: + continue + descriptor_pages.append(p["title"]) + soup = _clean_storage_xml(body_xml) + for table in extract_descriptor_tables(soup): + key = canonical_descriptor_key(table.descriptor_name) + if not key or not table.rows: + continue + bucket = descriptors.setdefault(key, {}) + short_bucket = descriptor_short.setdefault(key, {}) + for row in table.rows: + code = row["code_value"] + desc = row["description"] + short = row["short_description"] + if desc and code not in bucket: + bucket[code] = desc + if short and code not in short_bucket: + short_bucket[code] = short + time.sleep(0.15) + + for p in narrative_titles: + try: + page = _load_or_fetch_page(client, p["id"]) + except Exception as exc: + logger.warning("Failed to fetch %s (%s): %s", p["title"], p["id"], exc) + continue + body_xml = page.get("body", {}).get("storage", {}).get("value", "") + if not body_xml: + continue + narrative_pages.append(p["title"]) + soup = _clean_storage_xml(body_xml) + text = extract_narrative(soup) + if not text: + continue + domain = page_domain(p["title"]) + if not domain: + continue + # Concatenate narratives across pages for the same domain (Reporting + # Guide + General Reporting Info both contribute). + existing = domain_narratives.get(domain) + if existing: + domain_narratives[domain] = existing + "\n\n" + text + else: + domain_narratives[domain] = text + + digest = { + "fetched_at": datetime.now(timezone.utc).isoformat(), + "source_url": f"{CONFLUENCE_BASE}/wiki/spaces/{SPACE_KEY}/", + "page_count": len(descriptor_pages) + len(narrative_pages), + "descriptor_pages": sorted(descriptor_pages), + "narrative_pages": sorted(narrative_pages), + "descriptors": descriptors, + "descriptor_short": descriptor_short, + "domain_narratives": domain_narratives, + "domain_resource_keymap": _DOMAIN_RESOURCE_KEYMAP, + } + digest_path.parent.mkdir(parents=True, exist_ok=True) + digest_path.write_text(json.dumps(digest, indent=2, sort_keys=True), encoding="utf-8") + logger.info( + "Harvested IDOE Confluence: %d descriptors, %d domain narratives → %s", + len(descriptors), len(domain_narratives), digest_path, + ) + return digest + + +def ensure_harvested(force: bool = False) -> dict: + """Public alias for ``harvest`` — used by ``indiana.run()`` at ingest time.""" + return harvest(force=force) + + +# --------------------------------------------------------------------------- +# Per-element prose lookup helpers (consumed by indiana.py) +# --------------------------------------------------------------------------- + + +def descriptor_definition( + digest: dict, + descriptor_name: str | None, + *, + code_value: str | None = None, +) -> str: + """Return ``definition_text`` for an element backed by a descriptor. + + If ``code_value`` is supplied and matches, return that single value's + description. Otherwise return a digest of all known code values for the + descriptor (one per line, ``": "``), capped at the + first ~12 entries to keep the field readable. + """ + if not descriptor_name: + return "" + descriptors_map = digest.get("descriptors") or {} + bucket = descriptors_map.get(canonical_descriptor_key(descriptor_name)) or {} + if not bucket: + return "" + if code_value and code_value in bucket: + return bucket[code_value] + # Render up to 12 code-value definitions; if more, append "(...)" footer. + items = list(bucket.items()) + rendered = "\n".join(f"{c}: {d}" for c, d in items[:12]) + if len(items) > 12: + rendered += f"\n(+{len(items) - 12} additional values)" + return f"Descriptor {descriptor_name} permits:\n{rendered}" + + +def domain_business_rules(digest: dict, api_resource: str | None) -> str | None: + """Return ``business_rules_text`` for an element under ``api_resource``. + + Looks up the Confluence-domain → API-resource keymap in reverse: which + domain narrative covers this resource? Returns the first match (the + keymap is roughly 1:1 from resource to domain). + """ + if not api_resource: + return None + keymap = digest.get("domain_resource_keymap") or _DOMAIN_RESOURCE_KEYMAP + narratives = digest.get("domain_narratives") or {} + for domain, resources in keymap.items(): + if api_resource in resources and domain in narratives: + return narratives[domain] + return None diff --git a/apps/pipeline/src/ingest/indiana.py b/apps/pipeline/src/ingest/indiana.py new file mode 100644 index 0000000..dcd10c5 --- /dev/null +++ b/apps/pipeline/src/ingest/indiana.py @@ -0,0 +1,447 @@ +"""Parse Indiana IDOE Vendor Documentation Excel into pipeline data structures. + +Source: `data/bootstrap/in/idoe_vendor_documentation_2024_v6.1.xlsx`, primary +sheet `API Datastructure`. The sheet is a flat row-per-(table, column) catalog +with table-name forward-fill (top-most filled `Table` cell carries down through +blank rows within the same sub-table). Schema-prefixed entity names: `edfi.X` +for core, `idoe.X` for IDOE 1.0.0 extensions. + +Layout (cols 0-7, after the title row + header row): + 0: API Type — `/ed-fi/`, `/idoe/identities/`, etc. (forward-filled) + 1: API Resource — `localEducationAgencies`, `students` (forward-filled) + 2: Table — `edfi.EducationOrganization`, `idoe.SchoolExtension` + (forward-filled within sub-collection groups) + 3: Column — element name (`EducationOrganizationId`) + 4: Enumerations — descriptor reference if applicable (`AddressTypeDescriptor`) + 5: Data Type — SQL types: `int`, `nvarchar(75)`, `date` + 6: Req/Opt — `Required`, `Required *`, `Optional` + 7: Unique ID — `Y` (PK indicator) or blank + +Headline cohort: 487 element rows across 91 unique tables (74 `edfi.*` core ++ 17 `idoe.*` extension), 448 under `/ed-fi/` API + 39 under `/idoe/`. + +Prose enrichment (issue #156, 2026-05-04): + +The XLSX itself carries no `definition_text` / `business_rules_text` columns — +IDOE delegates per-element prose to its public Confluence Knowledge Hub at +``https://idoe.atlassian.net/wiki/spaces/IKHTV/``. The harvester in +``src.ingest.idoe_confluence`` fetches the prose-bearing pages, extracts +descriptor code-value definitions from ``: Descriptors`` / ``: Types`` pages, +and pulls domain-narrative prose from ``: General Reporting Info`` / +``Reporting Guide:`` pages. ``run()`` wires those lookups into per-row +`definition_text` (descriptor-typed columns) and `business_rules_text` +(domain narrative attached via the API-resource-to-domain keymap). + +Swagger-description fallback (issue #162 follow-on, 2026-05-04): + +Confluence covers descriptor enumerations and per-domain narrative but +NOT per-property prose for non-descriptor IDOE-extension fields. The IN +swagger DOES carry IDOE-authored descriptions on ``idoe_*`` schemas +(212 of 288 IDOE-extension properties). For documented ``source="extension"`` +rows where the Confluence pass yielded empty ``definition_text``, +``_swagger_extension_description()`` falls back to the spine catalog's +extension-property description. Gated to ``source="extension"`` so the +edFi_* rejection-as-Ed-Fi-prose discipline (ADR 0006) stays intact. + +CRITICAL: module-level `run()` is a PLAIN function. Do NOT decorate it with +`@click.command()` — that makes `cli.py`'s `run_in()` invocation call +`Click.Command.__call__`, re-parse `sys.argv`, and die. See +`tests/test_ingest_in.py::TestCliWiring` for the regression guard. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass +from pathlib import Path + +import openpyxl + +from src.ingest.normalize import normalize_data_type, normalize_entity +from src.models.element import ElementRecord +from src.models.spine import StateSpine +from src.utils.paths import ( + state_elements_path, + state_gap_log_path, + state_spine_path, +) + +logger = logging.getLogger(__name__) + +_PROJECT_ROOT = Path(__file__).resolve().parents[2] +_IN_SPINE_PATH = state_spine_path("IN") +_IN_XLSX_NAME = "idoe_vendor_documentation_2024_v6.1.xlsx" +_IN_XLSX_BOOTSTRAP_PATH = _PROJECT_ROOT / "data" / "bootstrap" / "in" / _IN_XLSX_NAME +_IN_XLSX_PATH = _PROJECT_ROOT / "data" / "raw" / "in" / _IN_XLSX_NAME +_IN_ELEMENTS_OUT = state_elements_path("IN", "source") +_IN_ELEMENTS_SPINE_OUT = state_elements_path("IN", "spine") +_IN_GAP_OUT = state_gap_log_path("IN") + +# Sheet that carries the row-per-element catalog. Other sheets in the workbook +# describe vendor-cert workflow scenarios (Student Demographics, Student +# Discipline, etc.) that aren't per-element — out of scope for source-lens row +# enumeration; see data/bootstrap/in/README.md. +_API_DATASTRUCTURE_SHEET = "API Datastructure" + + +@dataclass +class INElementRow: + """One row from the API Datastructure sheet, after forward-fill.""" + + sheet_row: int + api_type: str | None + api_resource: str | None + table_raw: str # `edfi.EducationOrganization` / `idoe.SchoolExtension` + column: str + enumeration: str | None + data_type_raw: str | None + req_opt: str | None + is_unique: bool + + +def _s(val: object) -> str | None: + if val is None: + return None + s = str(val).strip().replace("\xa0", "") + return s if s else None + + +def parse_in_api_datastructure(workbook_path: Path) -> list[INElementRow]: + """Parse the API Datastructure sheet into flat INElementRow records. + + Forward-fills API Type / API Resource / Table across rows where the cell + is blank. Skips the title row (row 1) and header row (row 2); element rows + start at row 3. + """ + wb = openpyxl.load_workbook(workbook_path, read_only=True, data_only=True) + if _API_DATASTRUCTURE_SHEET not in wb.sheetnames: + wb.close() + raise ValueError( + f"Expected sheet {_API_DATASTRUCTURE_SHEET!r} not found in {workbook_path.name}; " + f"sheets present: {wb.sheetnames}" + ) + + ws = wb[_API_DATASTRUCTURE_SHEET] + rows: list[INElementRow] = [] + + last_api_type: str | None = None + last_resource: str | None = None + last_table: str | None = None + + for row_idx, raw in enumerate(ws.iter_rows(min_row=3, values_only=True), start=3): + if not raw or all(c is None for c in raw): + continue + # pad to at least 8 cols + cells = list(raw) + [None] * (8 - len(raw)) if len(raw) < 8 else list(raw) + + api_type = _s(cells[0]) + resource = _s(cells[1]) + table = _s(cells[2]) + column = _s(cells[3]) + enumeration = _s(cells[4]) + data_type_raw = _s(cells[5]) + req_opt = _s(cells[6]) + unique_flag = _s(cells[7]) + + # Forward-fill context cells + if api_type: + last_api_type = api_type + if resource: + last_resource = resource + if table: + last_table = table + + if not column: + # Spacer row, sub-collection separator, or pure context-rotation row + continue + if not last_table: + logger.warning("IN row %d has Column %r but no preceding Table — skipping", row_idx, column) + continue + + rows.append(INElementRow( + sheet_row=row_idx, + api_type=last_api_type, + api_resource=last_resource, + table_raw=last_table, + column=column, + enumeration=enumeration, + data_type_raw=data_type_raw, + req_opt=req_opt, + is_unique=(unique_flag is not None and unique_flag.upper().startswith("Y")), + )) + + wb.close() + return rows + + +def _table_is_extension(table_raw: str) -> bool: + """`idoe.X` is an IDOE 1.0.0 extension; `edfi.X` is core.""" + return table_raw.lower().startswith("idoe.") + + +def _canonical_extension_name(table_raw: str) -> str: + """Project `idoe.SchoolExtension` (XLSX form) → `idoe_schoolExtension` (spine form). + + The IN spine catalog stores extension keys in camelCase with underscore + namespace (e.g., `idoe_schoolExtension`, `idoe_assessmentAccommodation`). + The XLSX uses dot + PascalCase (`idoe.SchoolExtension`). For the + `extension_name` audit field we keep a spine-aligned projection so analyst + workbooks read consistently with the spine-lens artifact. + """ + if not table_raw.lower().startswith("idoe."): + return table_raw + body = table_raw[len("idoe."):] + if not body: + return table_raw + return f"idoe_{body[0].lower() + body[1:]}" + + +def _swagger_extension_description( + catalog, + extension_name: str | None, + element_name: str, +) -> str: + """Look up an IDOE-extension property's swagger description in the spine catalog. + + Issue #162 follow-on. The IDOE Confluence Knowledge Hub carries + descriptor-code-value enumerations and per-domain narrative, but does + NOT carry per-property prose for non-descriptor IDOE-extension fields + (e.g. ``School.ChoiceIndicator``, ``Calendar.InstructionalTimeInMinutesIndicator``). + For those rows the IN swagger DOES carry IDOE-authored descriptions — + e.g. ``idoe_schoolExtension.choiceIndicator`` says *"Indicator of + whether or not the school is a Choice School"*. Pull those into + ``definition_text`` as a last-resort fallback after the Confluence + lookup so the LLM extraction pool has more signal on these rows. + + GATED to ``source="extension"`` at the call-site: for ``edFi_*`` core + rows the swagger description IS the upstream Ed-Fi prose the user + already rejected as ``definition_text``, so widening to core rows + would silently undo that rejection. ADR 0006 framed this swagger- + description path as the "Option B Phase F fallback"; this lands + Option B for IDOE-extension rows only. + + Resolution rules (case-insensitive, all on the spine catalog — + same data source the dual-lens artifacts already trust): + 1. Locate the spine extension record by ``extension_name`` + (e.g. ``idoe_schoolExtension``). + 2. Try the element name verbatim, then first-letter-lowered, then + (if the name ends in ``Id``) the bare-stem variants — bridging the + XLSX's PascalCase + DescriptorId convention to the swagger's + camelCase + Descriptor form. + 3. Search the extension's direct properties first, then its + sub-collection properties (so sub-entity leaves like + ``contract.contractDays`` resolve too). + + Returns the description string when one is found, empty string + otherwise. Empty result is the honest "swagger doesn't carry prose + for this row either" signal — typically FK key columns whose + description lives on the FK target entity, not the extension. + """ + if catalog is None or not extension_name or not element_name: + return "" + ext = catalog.extensions.get(extension_name) + if ext is None: + return "" + + candidates: set[str] = {element_name.lower()} + candidates.add((element_name[0].lower() + element_name[1:]).lower()) + if element_name.endswith("Id") and len(element_name) > 2: + stem = element_name[:-2] + candidates.add(stem.lower()) + candidates.add((stem[0].lower() + stem[1:]).lower()) + + def _description(prop) -> str: + desc = getattr(prop, "description", None) + return (desc or "").strip() + + for prop_name, prop in ext.properties.items(): + if prop_name.lower() in candidates: + return _description(prop) + for sub in ext.sub_collections.values(): + sub_props = getattr(sub, "properties", {}) or {} + for prop_name, prop in sub_props.items(): + if prop_name.lower() in candidates: + return _description(prop) + return "" + + +def build_element_records( + rows: list[INElementRow], + *, + edfi_version: str, + source_document: str, + catalog, + confluence_digest: dict | None = None, +) -> list[ElementRecord]: + """Convert parsed IN rows to ElementRecord instances. + + When ``confluence_digest`` is supplied (the harvested IDOE Knowledge Hub + output from ``src.ingest.idoe_confluence.harvest``), descriptor-typed + rows pick up ``definition_text`` from the per-code-value Confluence prose + and every row picks up ``business_rules_text`` from the matching domain + narrative (via the ``api_resource`` → Confluence-domain keymap). When + omitted, both fields fall back to empty strings — matches pre-issue-156 + behaviour for tests that don't run the harvester. + """ + from src.ingest.idoe_confluence import ( + descriptor_definition, + domain_business_rules, + ) + + records: list[ElementRecord] = [] + for r in rows: + is_ext = _table_is_extension(r.table_raw) + normalized_entity = normalize_entity(r.table_raw, catalog) + element_name = r.column + # The XLSX often appends `Id` to a column name that's a descriptor FK; + # the matching `Enumerations` cell carries the bare descriptor name. + # Spine matching already handles `*DescriptorId` → `*Descriptor` aliasing, + # so we leave the column name verbatim. + + if confluence_digest is not None: + definition_text = descriptor_definition( + confluence_digest, r.enumeration + ) + business_rules_text = domain_business_rules( + confluence_digest, r.api_resource + ) + else: + definition_text = "" + business_rules_text = None + + # Issue #162 follow-on — for IDOE-extension rows where Confluence + # didn't carry per-element prose, fall back to the IN swagger + # description (which IS IDOE-authored on `idoe_*` schemas, unlike + # `edFi_*` core rows where the description is upstream Ed-Fi prose + # the user rejected as `definition_text`). Gated to + # ``source="extension"`` so the rejection-as-Ed-Fi-prose holds. + if is_ext and not definition_text: + definition_text = _swagger_extension_description( + catalog, + _canonical_extension_name(r.table_raw), + element_name, + ) + + records.append(ElementRecord( + state="IN", + edfi_version=edfi_version, + domain=r.api_resource or r.table_raw, + entity=normalized_entity, + raw_entity=r.table_raw, + element_name=element_name, + data_type=normalize_data_type(r.data_type_raw), + definition_text=definition_text, + business_rules_text=business_rules_text, + source="extension" if is_ext else "core", + extension_name=_canonical_extension_name(r.table_raw) if is_ext else None, + source_document=source_document, + source_page_or_section=f"{_API_DATASTRUCTURE_SHEET}#row{r.sheet_row}", + documented=True, + )) + return records + + +def run() -> None: + """POC-3 IN ingestion: parse IDOE Vendor Documentation, match to spine. + + Writes: + - `data/out/in_elements_source.json` (source-lens StateElements) + - `data/out/in_elements_spine.json` (spine-lens dual artifact) + - `data/out/in_gap_log.json` (spine-match diagnostics) + """ + from src.ingest.shared import ( + assemble_source_driven, + write_dual_lens_artifacts, + ) + + if not _IN_SPINE_PATH.exists(): + raise FileNotFoundError( + f"No IN spine at {_IN_SPINE_PATH}. " + f"Run `mc spine fetch --state IN --school-year 2027` + " + f"`mc spine build --state IN` first." + ) + spine = StateSpine.model_validate_json(_IN_SPINE_PATH.read_text(encoding="utf-8")) + logger.info( + "Loaded IN spine: %d core entities, %d extensions (Ed-Fi %s)", + spine.entity_count, spine.extension_count, spine.edfi_version, + ) + + if _IN_XLSX_PATH.exists(): + xlsx_path = _IN_XLSX_PATH + elif _IN_XLSX_BOOTSTRAP_PATH.exists(): + xlsx_path = _IN_XLSX_BOOTSTRAP_PATH + logger.info( + "Using bundled IN workbook at %s (no copy in data/raw/in/)", + xlsx_path, + ) + else: + raise FileNotFoundError( + f"IN Vendor Documentation XLSX not found at {_IN_XLSX_PATH} or " + f"{_IN_XLSX_BOOTSTRAP_PATH}" + ) + + rows = parse_in_api_datastructure(xlsx_path) + logger.info( + "Parsed %s::%s: %d element rows across %d unique tables (%d core, %d idoe)", + xlsx_path.name, _API_DATASTRUCTURE_SHEET, len(rows), + len({r.table_raw for r in rows}), + len({r.table_raw for r in rows if not _table_is_extension(r.table_raw)}), + len({r.table_raw for r in rows if _table_is_extension(r.table_raw)}), + ) + + # Issue #156 — harvest IDOE Confluence Knowledge Hub for prose. + from src.ingest.idoe_confluence import ensure_harvested + confluence_digest = ensure_harvested() + logger.info( + "Loaded IDOE Confluence digest: %d descriptors, %d domain narratives", + len(confluence_digest.get("descriptors") or {}), + len(confluence_digest.get("domain_narratives") or {}), + ) + + records = build_element_records( + rows, + edfi_version=spine.edfi_version, + source_document=xlsx_path.name, + catalog=spine.catalog, + confluence_digest=confluence_digest, + ) + n_with_def = sum(1 for r in records if r.definition_text) + n_with_rules = sum(1 for r in records if r.business_rules_text) + logger.info( + "Built %d IN ElementRecords (def_text: %d/%d = %.1f%%, " + "business_rules_text: %d/%d = %.1f%%)", + len(records), + n_with_def, len(records), 100 * n_with_def / max(1, len(records)), + n_with_rules, len(records), 100 * n_with_rules / max(1, len(records)), + ) + + records, recovered, assembly = assemble_source_driven(records, spine) + + # Shared adapter tail: dual-lens writes → gap log → swagger backfill + # (issue #213 item 3 — the ~80-line sequence lives once in shared.py). + write_dual_lens_artifacts( + state="IN", + spine=spine, + records=records, + assembly=assembly, + recovered=recovered, + source_document=xlsx_path.name, + source_out=_IN_ELEMENTS_OUT, + spine_out=_IN_ELEMENTS_SPINE_OUT, + gap_out=_IN_GAP_OUT, + spine_path=_IN_SPINE_PATH, + spine_source_rel=str(_IN_SPINE_PATH.relative_to(_PROJECT_ROOT)), + source_coverage_note=( + "Of our IN API Datastructure rows, how many match a spine element " + "(case-insensitive, FK+descriptor aliases)." + ), + spine_coverage_note=( + "Of the spine's authoritative element slots, how many are " + "represented in IN docs." + ), + logger=logger, + ) + + +if __name__ == "__main__": + run() diff --git a/apps/pipeline/src/ingest/minnesota.py b/apps/pipeline/src/ingest/minnesota.py new file mode 100644 index 0000000..424126c --- /dev/null +++ b/apps/pipeline/src/ingest/minnesota.py @@ -0,0 +1,716 @@ +"""Parse Minnesota MDE Ed-Fi Data Mapping Matrix into pipeline data structures. + +Unlike AZ (a per-entity element tree) or WI (Confluence wiki pages), MN's +authoritative state source is a **mapping matrix** XLSX: each row declares +"MDE field X maps to Ed-Fi entity Y, element Z". That means spine alignment +is already 1:1 per row — there's no entity-table detection, no HTML scraping. + +Source: `mn-mde-edfi/MDE-EdFi-Documentation` GitHub repo, current-year + `2026-27 MDE Ed-Fi Documentation/2026-2027 Data Mapping Matrix Ed-Fi + Suite 3 v 6.2.xlsx`. Cached locally at `data/raw/mn/github/` with auto- + download when missing, mirroring the WI Confluence auto-cache pattern. + +Sheets processed: + - `Student Enrollment Elements`: 199 data rows, 8 columns. + - `MCCC Elements`: 156 data rows, 10 columns. + - `List of Custom Descriptors`: skipped (descriptor URI catalog, not + per-element mappings — useful only for descriptor value enrichment). + +Known source-side wrinkles (handled, not parser bugs): + +- **`Ed-Fi Element Name` is sometimes a multi-line business rule**, e.g. + `"To indicate Kindergarten Schedule\\nCore.Calendar.CalendarCode = …"`. + These rows are retained but the rule text goes into `business_rules_text` + and `element_name` falls back to the MDE element name so the row stays + comparable. +- **`Ed-Fi Entity` may be a reference path** like + `"Course.EducationOrganizationReference"`. We split on the first `.` so + spine matching uses the root entity (`Course`); the full reference path + is preserved in `raw_entity`. +- **`Ed-Fi Element Name` in MCCC rows sometimes holds a foreign key dotted + name** like `Course.EducationOrganizationReference.postSecondaryInstitutionId`. + `record_match_keys` already emits path-tail aliases, so those resolve. + +CRITICAL: module-level `run()` is a PLAIN function. Do NOT decorate it with +`@click.command()` — that makes `cli.py`'s `run_mn()` invocation call +`Click.Command.__call__`, re-parse `sys.argv`, and die. See +`tests/test_ingest_mn.py::TestCliWiring` for the regression guard. +""" + +from __future__ import annotations + +import logging +import re +from dataclasses import dataclass +from pathlib import Path + +import httpx +import openpyxl + +from src.ingest.normalize import normalize_data_type, normalize_entity +from src.models.element import ElementRecord +from src.models.spine import StateSpine + +logger = logging.getLogger(__name__) + + +from src.utils.paths import ( + state_elements_path, + state_gap_log_path, + state_spine_path, +) + +_PROJECT_ROOT = Path(__file__).resolve().parents[2] +_MN_SPINE_PATH = state_spine_path("MN") +_MN_GITHUB_DIR = _PROJECT_ROOT / "data" / "raw" / "mn" / "github" +_MN_XLSX_NAME = "data_mapping_matrix_2026-27.xlsx" +_MN_XLSX_PATH = _MN_GITHUB_DIR / _MN_XLSX_NAME +_MN_XLSX_URL = ( + "https://raw.githubusercontent.com/mn-mde-edfi/MDE-EdFi-Documentation/master/" + "2026-27%20MDE%20Ed-Fi%20Documentation/" + "2026-2027%20Data%20Mapping%20Matrix%20Ed-Fi%20Suite%203%20v%206.2.xlsx" +) +_MN_ELEMENTS_OUT = state_elements_path("MN", "source") +_MN_ELEMENTS_SPINE_OUT = state_elements_path("MN", "spine") +_MN_GAP_OUT = state_gap_log_path("MN") + +# Sheets in the Data Mapping Matrix that carry row-per-element mappings. +_ELEMENT_SHEETS = ("Student Enrollment Elements", "MCCC Elements") + +# Heuristic: the Ed-Fi Element Name column sometimes holds a multi-line +# business rule instead of a clean identifier. Detect with newline or +# '=' (comparison) — identifiers never contain either. +def _looks_like_rule(value: str) -> bool: + return "\n" in value or "=" in value + + +@dataclass +class MNMatrixRow: + """One row from a mapping-matrix sheet, pre-canonicalization.""" + + sheet: str + row_id: str + collection: str | None + mde_entity: str | None # present only on MCCC sheet + mde_group: str | None + mde_element: str | None + edfi_entity_raw: str | None + edfi_element_raw: str | None + mapping_method: str | None + enumeration: str | None + notes: str | None + + +def _s(val: object) -> str | None: + if val is None: + return None + s = str(val).strip() + return s if s else None + + +def _download_xlsx_if_missing() -> None: + if _MN_XLSX_PATH.exists(): + return + _MN_GITHUB_DIR.mkdir(parents=True, exist_ok=True) + logger.info("Downloading MN mapping matrix from %s", _MN_XLSX_URL) + with httpx.Client(timeout=60.0, follow_redirects=True) as client: + resp = client.get(_MN_XLSX_URL) + resp.raise_for_status() + _MN_XLSX_PATH.write_bytes(resp.content) + logger.info(" wrote %s (%d bytes)", _MN_XLSX_PATH.name, _MN_XLSX_PATH.stat().st_size) + + +def parse_mn_matrix(workbook_path: Path) -> list[MNMatrixRow]: + """Parse the two element-mapping sheets into flat MNMatrixRow records. + + Column detection is header-driven so small layout changes don't break + parsing silently. Each target column is required to appear in the + header; missing columns raise immediately rather than silently emit + Nones. + """ + wb = openpyxl.load_workbook(workbook_path, read_only=True, data_only=True) + rows: list[MNMatrixRow] = [] + + # Accepted header variants (key -> set of lowered header strings). + header_aliases: dict[str, set[str]] = { + "row_id": {"row id"}, + "collection": {"collection"}, + "mde_entity": {"mde entity"}, + "mde_group": {"mde element group"}, + "mde_element": {"mde element name"}, + "edfi_entity": {"ed-fi entity"}, + "edfi_element": {"ed-fi element name"}, + "mapping_method": {"element mapping method"}, + "enumeration": {"ed-fi enumeration descriptor/type"}, + "notes": {"notes for discussion"}, + } + + for sheet_name in _ELEMENT_SHEETS: + if sheet_name not in wb.sheetnames: + logger.warning("Expected sheet %r missing from %s", sheet_name, workbook_path.name) + continue + ws = wb[sheet_name] + sheet_rows = list(ws.iter_rows(values_only=True)) + if not sheet_rows: + continue + + header = [(_s(c) or "").lower() for c in sheet_rows[0]] + cols: dict[str, int] = {} + for key, aliases in header_aliases.items(): + idx = next((i for i, h in enumerate(header) if h in aliases), None) + if idx is not None: + cols[key] = idx + + for required in ("row_id", "edfi_entity", "edfi_element"): + if required not in cols: + raise ValueError( + f"Sheet {sheet_name!r}: missing required header for {required!r}. " + f"Found headers: {header}" + ) + + for erow in sheet_rows[1:]: + row_id = _s(erow[cols["row_id"]]) if cols["row_id"] < len(erow) else None + if row_id is None: + continue # blank/spacer row + + def _cell(key: str) -> str | None: + idx = cols.get(key) + if idx is None or idx >= len(erow): + return None + return _s(erow[idx]) + + # Skip header-style narrative rows where edfi_entity is empty. + edfi_entity_raw = _cell("edfi_entity") + if edfi_entity_raw is None: + continue + + rows.append(MNMatrixRow( + sheet=sheet_name, + row_id=row_id, + collection=_cell("collection"), + mde_entity=_cell("mde_entity"), + mde_group=_cell("mde_group"), + mde_element=_cell("mde_element"), + edfi_entity_raw=edfi_entity_raw, + edfi_element_raw=_cell("edfi_element"), + mapping_method=_cell("mapping_method"), + enumeration=_cell("enumeration"), + notes=_cell("notes"), + )) + + wb.close() + return rows + + +_ENTITY_PATH_RE = re.compile(r"\s*>\s*|\.") + + +def _split_root_entity(raw: str) -> tuple[str, str | None]: + """Split an Ed-Fi entity reference path into (concat_entity, suffix_info). + + MN uses three entity-path conventions in the same column: + - Plain name: "Calendar" + - Dot reference: "Course.EducationOrganizationReference" + - Angle-bracket navigation: "Course > AssessmentTool", + "Grade > StudentSectionAssociation > CollegeCourseReference > Course" + + For angle-bracket navigations, concatenating PascalCase segments produces + a flattened sub-collection name (e.g. `CourseAssessmentTool`) that + `spine.unflatten` can resolve back to the parent entity. For dot + references, treat everything after the first dot as a reference suffix + rather than part of the entity name, since those describe a reference + path (`Course` has a reference called `EducationOrganizationReference`), + not a flattened sub-collection. + """ + raw = raw.strip() + if ">" in raw: + parts = [p.strip() for p in re.split(r"\s*>\s*", raw) if p.strip()] + # Normalize internal whitespace in each segment before concatenating + # so "LevelCharacteristics" style single-word segments still PascalCase. + parts = [re.sub(r"\s+", "", p) for p in parts] + if not parts: + return raw, None + concat = "".join(p[0].upper() + p[1:] if p else p for p in parts) + return concat, " > ".join(parts[1:]) if len(parts) > 1 else None + if "." in raw: + root, suffix = raw.split(".", 1) + return root.strip(), suffix.strip() or None + return raw, None + + +# A cell whose element-name value is actually a comma-separated list of +# multiple identifiers. We only expand when ALL comma-separated tokens look +# like identifiers (alphanumeric, no spaces within a token) to avoid chopping +# up human-written descriptions. +_IDENT_TOKEN_RE = re.compile(r"^[A-Za-z][A-Za-z0-9_]*$") + +# MN matrix uses Arabic numeral 1 in places Ed-Fi uses Roman I +# (`Title1PartAParticipant` vs spine's `titleIPartAParticipant`). Apply the +# substitution to element names too — the entity-name path is handled in +# `normalize._ENTITY_NAME_FIXES`. +_TITLE1_RE = re.compile(r"\bTitle1(?=[A-Za-z])") + + +def _collapse_whitespace_to_camel(name: str) -> str: + """Collapse internal whitespace in a label like `First Name` to camelCase. + + MN matrix authors sometimes write humanized labels inside FK-navigation + paths (`StudentReference>StudentUniqueId > First Name`). After + `_strip_mn_nav_path` reduces that to `First Name`, we still need to + produce the camelCase identifier `firstName` for display consistency. + + Preserves the input unchanged when there's no internal whitespace. + """ + if " " not in name.strip(): + return name + tokens = [t for t in re.split(r"\s+", name.strip()) if t] + if not tokens: + return name + pascal = "".join(t[0].upper() + t[1:] if t else t for t in tokens) + if pascal: + return pascal[0].lower() + pascal[1:] + return name + + +def _strip_mn_nav_path(name: str) -> str: + """Strip FK-navigation path prefixes from an MN element name. + + The MDE mapping matrix encodes reference navigation as `>`-separated + segments — e.g., `StudentReference>StudentUniqueId > LastSurname`. Spine + matching already handles these via path-tail alias emission in + `utils.matching.element_aliases`. For display, we want the terminal + segment so the workbook reads cleanly. Examples: + + `StudentReference>StudentUniqueId` -> `studentUniqueId` + `StudentReference>StudentUniqueId > LastSurname` -> `lastSurname` + `StudentReference>StudentUniqueId > First Name` -> `firstName` + `CollegeCourseReference >` -> `collegeCourseReference` + + Only fires when `>` is present — dotted FK paths are still handled by + `_strip_known_source_aliases` / `_strip_entity_prefix`. Trailing + whitespace / collapsed-internal-space is normalized via + `_collapse_whitespace_to_camel` so humanized labels become camelCase. + + Truncated source paths like `CollegeCourseReference >` (trailing `>` with + no following segment) fall back to the preceding named segment — analyst- + review round-3 P2 flagged these as unclean artifacts. A naive + `rsplit(">", 1)[-1]` returns an empty string in that case, so we do a + segment-based split + empty-filter instead to pick the last non-empty + segment reliably. + """ + if ">" not in name: + return name + segments = [s.strip() for s in name.split(">")] + segments = [s for s in segments if s] + if not segments: + return name + tail = segments[-1] + tail = _collapse_whitespace_to_camel(tail) + # Normalize first-letter case to match Ed-Fi camelCase element convention + # when the tail is a single identifier (no dots, no existing lowercase). + if tail and "." not in tail and tail[0].isupper(): + tail = tail[0].lower() + tail[1:] + return tail + + +def _clean_element_name(name: str) -> str: + # Title1 → TitleI (Roman-numeral normalization; see _TITLE1_RE docstring). + cleaned = _TITLE1_RE.sub("TitleI", name) + # Strip trailing dots that the MDE matrix occasionally leaves (e.g., + # `GenderIdentities.`) — these are authorial typos, not path separators. + cleaned = cleaned.rstrip(".") + # Strip FK-navigation path prefixes (analyst review round-2 reviewer B). + cleaned = _strip_mn_nav_path(cleaned) + return cleaned + + +# MN matrix authors occasionally append a role hint in parentheses to the +# Ed-Fi Element Name column — e.g. `AcademicHonorCategory (Part of Identity)`, +# `Language (Optional Collection)`, `HonorDescription (Part of Identity)`. The +# bare identifier is needed for spine matching; the hint belongs in the +# definition text where analysts can still see it. +_ANNOTATION_RE = re.compile(r"\s*\(([^)]+)\)\s*$") + + +def _strip_element_annotations(name: str) -> tuple[str, str | None]: + """Return (bare_name, annotation_text). Annotation is None if absent.""" + m = _ANNOTATION_RE.search(name) + if m: + bare = name[: m.start()].strip() + return bare, m.group(1).strip() + return name, None + + +def _strip_entity_prefix(name: str, root_entity: str | None) -> str: + """Strip a leading `.` prefix from the element name. + + Flagged by analyst review: MN row with entity `Student` sometimes carries + element `Student.StudentUniqueId` where every other row uses the bare + identifier. Compare case-insensitively so `student.` stripping works even + when the root_entity form differs slightly in case. + """ + if not root_entity: + return name + prefix = root_entity + "." + if name.startswith(prefix): + return name[len(prefix):] + if name.lower().startswith(prefix.lower()): + return name[len(prefix):] + return name + + +# MN matrix authors sometimes prefix element names with a source alias that +# isn't the row's Ed-Fi Entity — e.g., `SEOA.FirstName` on a Student row, +# `BirthData.BirthDate`, `Core.Student.StudentUniqueId`, `Name.LastSurname`. +# Round-2 analyst review flagged 46 rows across 17 such prefixes. Strip a +# closed list of known source aliases so the element name is the bare +# identifier the spine can match on. `memsberships` is a source-side typo +# (should be `memberships`) — we strip it as a prefix but leave the typo +# itself documented in Known Limitations. +_MN_DOTTED_SOURCE_ALIASES: tuple[str, ...] = ( + "Core.Student", + "SEOA", + "BirthData", + "Name", + "OtherName", + "Membership", + "memsberships", + "Transportation", + "MeetingTimes", + "CourseIdentificationCode", + "CourseIndentificationCode", # source-side typo; document in KL + "EducationOrganizationIndicator", + "NeglectedOrDelinquentProgramService", + "postsecondaryinstitution", + "staffReference", + # Round 2.2 additions — bare entity-name and FK-reference prefixes the + # MN matrix uses in `Ed-Fi Element Name` to disambiguate path semantics. + # The spine-match step already resolves these via path-tail aliases, but + # the display-side `element_name` was retaining the prefix. Reviewer B + # flagged this as a clarity problem (workbook readability, not matching). + "Student", + "School", + "EducationOrganization", + "TransportingLocalEducationAgencyReference", + "addresses", +) + + +def _strip_known_source_aliases(name: str) -> str: + """Strip any of a closed list of MN source-alias prefixes from element name. + + Applied AFTER `_strip_entity_prefix` so entity-matched prefixes take + precedence. Case-insensitive match on prefix + `.`; preserves the tail + exactly. Iterates to a fixed point — round-2.2 reviewer flagged rows + like `Transportation.TransportingLocalEducationAgencyReference.Local + EducationAgencyId` where stripping just `Transportation.` leaves a + residual chain that also starts with an aliased prefix. Fixed-point + iteration (bounded by len(aliases) iterations as a safety guard) + ensures full stripping without ordering sensitivity. + """ + if "." not in name: + return name + prefixes = sorted(_MN_DOTTED_SOURCE_ALIASES, key=len, reverse=True) + max_iter = len(prefixes) + 1 + for _ in range(max_iter): + stripped = None + for prefix in prefixes: + pfx = prefix + "." + if name.lower().startswith(pfx.lower()): + stripped = name[len(pfx):] + break + if stripped is None or not stripped or "." not in stripped: + if stripped is not None: + name = stripped + break + name = stripped + return name + + +# Shared spine-type helpers were promoted to `src.ingest.shared` in Round 2.2 +# so AZ and WI can apply the same canonical-type contract. Re-exported here as +# module-local names for backward compat with existing tests / callers. +from src.ingest.shared import ( # noqa: E402,F401 — deliberate re-exports + build_spine_type_index as _build_spine_type_index, + canonical_type as _canonical_type, + populate_data_types_from_spine as _populate_data_types_from_spine, +) + + +def _expand_comma_list(name: str) -> list[str]: + """Split a comma-separated element cell when every token is identifier-shaped. + + MN mapping-matrix authors occasionally pack two distinct Ed-Fi elements + into a single cell, e.g. `CourseIdentificationSystem, CourseIndentificationCode. + IdentificationCode`. Tokens may be bare identifiers OR dotted paths — + analyst-review P2 flagged that the narrower ident-only check left composite + rows like the above unsplit, rendering a matched-core row with an unclean + visible label (and a source typo in the prefix). Using `_PATH_TOKEN_RE` + accepts dotted tokens so downstream `_strip_known_source_aliases` + + `_strip_entity_prefix` normalize each split token independently. + + Human-prose cells like `Course Reference (CourseIdentificationCode, CourseCode)` + still stay unsplit — spaces and parens reject both regexes. + """ + if "," not in name: + return [name] + tokens = [t.strip() for t in name.split(",")] + if len(tokens) < 2: + return [name] + if all(_PATH_TOKEN_RE.match(t) for t in tokens): + return tokens + return [name] + + +# Multi-target splitters. Used only when the Ed-Fi Entity cell lists 2+ +# entities (comma or `&` separated) — analyst feedback flagged MN rows like +# `Student, StudentEducationOrganizationAssociation` / `BirthData.BirthDate, +# SEOA.BirthDate` that violate the one-row-per-element premise. +_MULTI_ENTITY_SEP_RE = re.compile(r"\s*[,&]\s*") +_ENTITY_TOKEN_RE = re.compile(r"^[A-Z][A-Za-z0-9]*$") +_PATH_TOKEN_RE = re.compile(r"^[A-Za-z][A-Za-z0-9_]*(?:\.[A-Za-z][A-Za-z0-9_]*)*$") + + +def _split_multi_entity(entity_raw: str) -> list[str] | None: + """Return a list of entity names when the cell is a multi-target list. + + Returns None if the cell is a single entity or the list fails quality + checks (contains non-PascalCase tokens). Recognizes both `,` and `&` + separators — MN row 295 uses `StudentEducationOrganizationAssociation & + Student`, everywhere else uses a comma. + """ + parts = [p.strip() for p in _MULTI_ENTITY_SEP_RE.split(entity_raw) if p.strip()] + if len(parts) < 2: + return None + if not all(_ENTITY_TOKEN_RE.match(p) for p in parts): + return None + return parts + + +def _expand_multi_target_elements(name: str) -> list[str] | None: + """Aggressive comma-split for element cells in multi-target rows. + + Looser than `_expand_comma_list` — accepts dotted paths like + `BirthData.BirthDate` so `BirthData.BirthDate, SEOA.BirthDate` splits + correctly. Returns None when any token is obviously non-identifier (has + embedded spaces, parenthetical hints, etc.) so ambiguous cells fall back + to the single-token path. + """ + if "," not in name: + return [name] + tokens = [t.strip() for t in name.split(",")] + if len(tokens) < 2: + return None + if all(_PATH_TOKEN_RE.match(t) for t in tokens): + return tokens + return None + + +def _expand_multi_target_rows(rows: list[MNMatrixRow]) -> list[MNMatrixRow]: + """Fan out rows whose `Ed-Fi Entity` cell names multiple target entities. + + Analyst review flagged MN rows with entity cells like + `Student, StudentEducationOrganizationAssociation` paired with element + cells like `BirthData.BirthDate, SEOA.BirthDate`. These violate the + one-row-per-element premise. We emit one pseudo-row per (entity, element) + pair so the downstream single-target logic handles them uniformly. + + Strategy: + - If entity_count == element_count, pair 1:1 (parallel lists). + - Else, emit the full cross-product (N*M rows). + + Rows without multi-entity signals pass through unchanged. + """ + import dataclasses + + expanded: list[MNMatrixRow] = [] + for r in rows: + if r.edfi_entity_raw is None: + expanded.append(r) + continue + entity_list = _split_multi_entity(r.edfi_entity_raw) + if entity_list is None: + expanded.append(r) + continue + primary = r.edfi_element_raw or "" + element_list = _expand_multi_target_elements(primary) + if element_list is None: + # Elements can't be split confidently — fall back to a single + # composite element so the row still lands in the catalog as an + # unresolved multi-target (spine match will likely miss). + element_list = [primary] + if len(entity_list) == len(element_list): + pairs = list(zip(entity_list, element_list)) + else: + pairs = [(e, el) for e in entity_list for el in element_list] + for ent, elem in pairs: + expanded.append(dataclasses.replace( + r, + edfi_entity_raw=ent, + edfi_element_raw=elem, + )) + return expanded + + +def build_element_records( + rows: list[MNMatrixRow], + edfi_version: str, + source_document: str, + catalog=None, +) -> list[ElementRecord]: + """Convert parsed MN matrix rows to ElementRecord instances.""" + records: list[ElementRecord] = [] + + rows = _expand_multi_target_rows(rows) + + for r in rows: + if r.edfi_entity_raw is None: + continue + # MN matrix uses literal "-" in `Ed-Fi Entity` and/or `Ed-Fi Element + # Name` to mark MDE fields with no Ed-Fi mapping (entity OR element + # left blank). These rows carry no resolvable target and shouldn't + # pollute the source-coverage denominator. + if r.edfi_entity_raw.strip() == "-": + continue + if (r.edfi_element_raw or "").strip() == "-": + continue + root_entity, ref_suffix = _split_root_entity(r.edfi_entity_raw) + normalized_entity = normalize_entity(root_entity, catalog) + + edfi_element = r.edfi_element_raw or "" + rule_text: str | None = None + if edfi_element and _looks_like_rule(edfi_element): + # Cell holds a multi-line business rule, not an identifier. + rule_text = edfi_element + element_names = [r.mde_element or "(unmapped rule)"] + else: + primary = edfi_element or r.mde_element or "(unnamed)" + element_names = _expand_comma_list(primary) + + definition_parts: list[str] = [] + if r.mde_group and r.mde_element: + definition_parts.append(f"MDE mapping: {r.mde_group}.{r.mde_element}") + elif r.mde_element: + definition_parts.append(f"MDE mapping: {r.mde_element}") + if r.mde_entity: + definition_parts.append(f"MDE entity: {r.mde_entity}") + if r.enumeration and r.enumeration.lower() != "n/a": + definition_parts.append(f"Enumeration: {r.enumeration}") + if ref_suffix: + definition_parts.append(f"Via reference: {ref_suffix}") + definition_text = "; ".join(definition_parts) + + combined_rules: list[str] = [] + if rule_text: + combined_rules.append(rule_text) + if r.notes: + combined_rules.append(f"Notes: {r.notes}") + business_rules_text = "\n".join(combined_rules) if combined_rules else None + + raw_entity = r.edfi_entity_raw + if r.mde_group: + raw_entity = f"mn:{r.mde_group}/{r.edfi_entity_raw}" + + for raw_element_name in element_names: + bare, annotation = _strip_element_annotations(raw_element_name) + bare = _strip_entity_prefix(bare, root_entity) + bare = _strip_entity_prefix(bare, normalized_entity) + bare = _strip_known_source_aliases(bare) + bare = _clean_element_name(bare) + + row_definition = definition_text + if annotation: + role_note = f"role: {annotation}" + row_definition = ( + f"{row_definition}; {role_note}" if row_definition else role_note + ) + + records.append(ElementRecord( + state="MN", + edfi_version=edfi_version, + domain=r.collection or r.sheet, + entity=normalized_entity, + raw_entity=raw_entity, + element_name=bare, + data_type=normalize_data_type(None), + definition_text=row_definition, + business_rules_text=business_rules_text, + source_document=source_document, + source_page_or_section=f"{r.sheet}#row{r.row_id}", + documented=True, + )) + + return records + + +def run() -> None: + """POC-3 MN ingestion: parse Data Mapping Matrix, match to spine. + + Writes `data/out/mn_elements_source.json` (StateElements) and + `data/out/mn_gap_log.json` (spine-match diagnostics). + """ + from src.ingest.shared import ( + assemble_source_driven, + write_dual_lens_artifacts, + ) + + if not _MN_SPINE_PATH.exists(): + raise FileNotFoundError( + f"No MN spine at {_MN_SPINE_PATH}. " + f"Run `mc spine fetch --state MN` + `mc spine build --state MN` first." + ) + spine = StateSpine.model_validate_json(_MN_SPINE_PATH.read_text(encoding="utf-8")) + logger.info( + "Loaded MN spine: %d core entities, %d extensions (Ed-Fi %s)", + spine.entity_count, spine.extension_count, spine.edfi_version, + ) + + _download_xlsx_if_missing() + if not _MN_XLSX_PATH.exists(): + raise FileNotFoundError(f"MN mapping matrix not found at {_MN_XLSX_PATH}") + + rows = parse_mn_matrix(_MN_XLSX_PATH) + logger.info( + "Parsed %s: %d rows across %d sheets", + _MN_XLSX_PATH.name, len(rows), + len({r.sheet for r in rows}), + ) + + records = build_element_records( + rows, + edfi_version=spine.edfi_version, + source_document=_MN_XLSX_PATH.name, + catalog=spine.catalog, + ) + logger.info("Built %d MN ElementRecords", len(records)) + + records, recovered, assembly = assemble_source_driven(records, spine) + + # Shared adapter tail: dual-lens writes → gap log → swagger backfill + # (issue #213 item 3 — the ~80-line sequence lives once in shared.py). + write_dual_lens_artifacts( + state="MN", + spine=spine, + records=records, + assembly=assembly, + recovered=recovered, + source_document=_MN_XLSX_PATH.name, + source_out=_MN_ELEMENTS_OUT, + spine_out=_MN_ELEMENTS_SPINE_OUT, + gap_out=_MN_GAP_OUT, + spine_path=_MN_SPINE_PATH, + spine_source_rel=str(_MN_SPINE_PATH.relative_to(_PROJECT_ROOT)), + source_coverage_note=( + "Of our MN mapping-matrix rows, how many match a spine element " + "(case-insensitive, FK+descriptor aliases)." + ), + spine_coverage_note=( + "Of the spine's authoritative element slots, how many are " + "represented in MN docs." + ), + logger=logger, + ) + + +if __name__ == "__main__": + run() diff --git a/apps/pipeline/src/ingest/normalize.py b/apps/pipeline/src/ingest/normalize.py new file mode 100644 index 0000000..f36ec1d --- /dev/null +++ b/apps/pipeline/src/ingest/normalize.py @@ -0,0 +1,207 @@ +"""Normalize entity names and data types to canonical forms. + +Entity normalization ensures consistent naming across states so cross-state +comparison and Ed-Fi catalog lookup both work. + +Data type normalization maps SQL/vendor-specific types (bigint, nvarchar) +to a canonical semantic vocabulary (Integer, String, Date, etc.). + +Entity examples: + edfi.Course -> Course + az.SectionExtension -> Section (resolves via catalog) + az.CourseTranscriptExtention -> CourseTranscript (typo fixed, resolved) + CourseTranscriptExt -> CourseTranscript (Ext alias in catalog) + wi_studentApplication -> StudentApplication (prefix stripped, PascalCase) + az.StudentNeed -> StudentNeed (no catalog match, kept as-is) + +Data type examples: + bigint -> Integer + nvarchar(255) -> String + decimal(5,2) -> Decimal + bit -> Boolean + Descriptor -> Descriptor (already canonical) +""" + +import re + +from src.models.edfi_catalog import EdFiCatalog +from src.states import STATE_INFO + +# Namespace prefixes: edfi., az., AZ., or two-letter state prefix with underscore. +# `idoe` accepted for Indiana — the IDOE Vendor Documentation XLSX uses +# `idoe.SchoolExtension` style table names; the spine extension catalog uses +# the underscore form `idoe_schoolExtension`. Both flavors strip cleanly here. +# +# Issue #213 item 2: the generic `[A-Za-z]{2}` class already covers every +# two-letter state prefix, but longer agency prefixes (IN's `idoe`) were a +# hidden literal a sixth state would silently miss. They now DERIVE from +# `src.states.STATE_INFO`; `edfi` is the core namespace, kept as an explicit +# extra token. The derived pattern string is pinned byte-for-byte to the +# pre-derivation literal `^(?:edfi|idoe|[A-Za-z]{2})[._/]` in +# tests/test_normalize.py. +_LONG_STATE_PREFIXES: tuple[str, ...] = tuple( + dict.fromkeys( + p + for info in STATE_INFO.values() + for p in info.ext_prefixes + if len(p) > 2 # two-letter prefixes ride the generic class below + ) +) +_NS_PREFIX_RE = re.compile( + "^(?:" + "|".join(("edfi", *_LONG_STATE_PREFIXES, "[A-Za-z]{2}")) + ")[._/]" +) + +# Common typo in AZ data +_TYPO_EXTENTION = re.compile(r"Extention$") + +# Common typo in WI Confluence: `/calenderDates` instead of `/calendarDates`. +# Applied as a whole-word substring replacement so `CalenderDate`, +# `CalenderDates`, `CalenderEvent` etc. all resolve to the correct stem. +_TYPO_CALENDER = re.compile(r"Calender", re.IGNORECASE) + +# Explicit entity-name corrections for state-source authoring slips that +# don't fit a generic regex. Each maps a state-source spelling to its +# canonical Ed-Fi entity. Applied after namespace-prefix stripping but +# before catalog lookup so the catalog hit succeeds. Currently: +# - MN matrix consistently spells `StudentSection504Placement…` where the +# Ed-Fi 4.0 entity is `StudentSection504Plan…`. +_ENTITY_NAME_FIXES: dict[str, str] = { + "StudentSection504PlacementProgramAssociation": + "StudentSection504PlanProgramAssociation", + "StudentSection504PlacementProgramAssociationExtension": + "StudentSection504PlanProgramAssociationExtension", + # MN matrix consistently uses Arabic numeral `1` where Ed-Fi entities + # use Roman numeral `I` (`StudentTitle1PartA…` -> `StudentTitleIPartA…`). + "StudentTitle1PartAProgramAssociation": + "StudentTitleIPartAProgramAssociation", + "StudentTitle1PartAProgramAssociationExtension": + "StudentTitleIPartAProgramAssociationExtension", +} + + +def normalize_entity( + raw_name: str, + catalog: EdFiCatalog | None = None, +) -> str: + """Normalize a raw entity name to canonical PascalCase form. + + Steps: + 1. Strip namespace prefix (edfi., az., wi_, etc.) + 2. Fix known typos (Extention -> Extension, Calender -> Calendar) + 3. Ensure first character is uppercase (PascalCase) + 4. Apply explicit entity-name corrections (e.g. MN's `Section504Placement…` + -> Ed-Fi `Section504Plan…`) + 5. Resolve through Ed-Fi catalog lookup index if available + 6. If no catalog match, try stripping Extension/Ext suffix and re-check + 7. Try depluralization (Ed-Fi API endpoints are plural, entities singular) + 8. Fall back to cleaned name + """ + name = raw_name.strip() + + name = _NS_PREFIX_RE.sub("", name) + name = _TYPO_EXTENTION.sub("Extension", name) + name = _TYPO_CALENDER.sub("Calendar", name) + + # Collapse internal whitespace — the AZ Use Case XLSX has at least one + # entity with an internal space (`StudentDropOut RecoveryProgramMonthly + # Updates`) that otherwise can't match a catalog key (which are all + # space-free PascalCase). `raw_entity` preserves the source form for + # round-trip audit; only the display/lookup form is collapsed here. + name = re.sub(r"\s+", "", name) + + if name and name[0].islower(): + name = name[0].upper() + name[1:] + + name = _ENTITY_NAME_FIXES.get(name, name) + + if catalog is None: + return name + + canonical = catalog.lookup_index.get(name.lower()) + if canonical: + return canonical + + for suffix in ("Extension", "Ext"): + if name.endswith(suffix) and len(name) > len(suffix): + base = name[: -len(suffix)] + canonical = catalog.lookup_index.get(base.lower()) + if canonical: + return canonical + + for suffix, repl in (("ies", "y"), ("ses", "s"), ("s", "")): + if name.endswith(suffix) and len(name) > len(suffix): + singular = name[: -len(suffix)] + repl + canonical = catalog.lookup_index.get(singular.lower()) + if canonical: + return canonical + + # No catalog hit. For extension-only entities (where no core catalog entry + # exists), fall back to the depluralized form as the display entity — Ed-Fi + # convention is singular entity names and the reviewer flagged + # `StudentDropOut RecoveryProgramMonthlyUpdates` (plural) when the spine + # uses `StudentDropOutRecoveryProgramMonthlyUpdate` (singular). Only the + # simple trailing-`s` strip; more aggressive depluralization risks + # mangling entities like `Students` that should stay intact. + if ( + name.endswith("s") + and not name.endswith("ss") + and not name.endswith("us") + and not name.endswith("is") + and len(name) > 4 + ): + return name[:-1] + + return name + + +# --------------------------------------------------------------------------- +# Data type normalization +# --------------------------------------------------------------------------- + +# WI Confluence authors write types like "big integer" (space-separated), +# "Big integer", and the known typo "big nteger"; collections are written as +# "array". Match those BEFORE the bare `int|integer|bigint` pattern so +# "big integer" doesn't fall through to a stale pass-through when the +# leading word matches the generic pattern. Order matters. +_TYPE_PATTERNS: list[tuple[re.Pattern, str]] = [ + (re.compile(r"^(?:n?varchar|text|string)", re.IGNORECASE), "String"), + (re.compile(r"^big\s*integer", re.IGNORECASE), "Integer"), + (re.compile(r"^big\s*nteger", re.IGNORECASE), "Integer"), # known WI typo + (re.compile(r"^array\b", re.IGNORECASE), "Collection"), + (re.compile(r"^(?:int|integer|bigint|smallint|tinyint)\b", re.IGNORECASE), "Integer"), + (re.compile(r"^(?:decimal|dec|numeric)\b", re.IGNORECASE), "Decimal"), + (re.compile(r"^(?:number)\b", re.IGNORECASE), "Number"), + # TODO(#213 item 2, deferred): `datetime` deliberately folds into "Date". + # The canonical vocabulary predates a DateTime bucket, and every + # committed element golden/artifact was normalized under this fold — the + # WI/MN/TX raw source caches aren't committed, so a `datetime` → + # "DateTime" split can't be proven byte-inert against the live corpora + # from a fresh clone. Splitting it requires a deliberate golden regen in + # the same PR; don't change the fold casually. + (re.compile(r"^(?:date|datetime)\b", re.IGNORECASE), "Date"), + (re.compile(r"^(?:time)\b", re.IGNORECASE), "Time"), + (re.compile(r"^(?:bit|boolean)\b", re.IGNORECASE), "Boolean"), + (re.compile(r"^(?:descriptor)\b", re.IGNORECASE), "Descriptor"), + (re.compile(r"^(?:reference)\b", re.IGNORECASE), "Reference"), +] + + +def normalize_data_type(raw_type: str | None) -> str | None: + """Normalize a data type string to a canonical semantic form. + + Maps SQL/vendor types (bigint, nvarchar(255), dec(7,2)) to canonical + Ed-Fi semantic types (Integer, String, Decimal). Unrecognized types + are returned as-is after stripping whitespace. + """ + if raw_type is None: + return None + + cleaned = raw_type.strip() + if not cleaned: + return None + + for pattern, canonical in _TYPE_PATTERNS: + if pattern.match(cleaned): + return canonical + + return cleaned diff --git a/apps/pipeline/src/ingest/shared.py b/apps/pipeline/src/ingest/shared.py new file mode 100644 index 0000000..afdfffc --- /dev/null +++ b/apps/pipeline/src/ingest/shared.py @@ -0,0 +1,1483 @@ +"""Cross-state ingest helpers shared by AZ / WI / MN / TX adapters. + +Originally this logic lived in `minnesota.py` (because MN was the first adapter +to need spine-driven type inference). Round 2.2 promotes it to a shared module +so AZ and WI can apply the same canonical-type contract: **for matched records, +`data_type` is spine-derived**. Unmatched rows keep source-verbatim types as an +audit-trail fallback. + +Phase 2 (Option 3b dual-lens) adds the pipeline helpers that consolidate the +post-parse plumbing the four adapters used to duplicate: + +- `run_unflatten_pass(records, spine)` — identity-preserving sub-entity + collapse loop (returns the `recovered` list, modifies records in place). +- `compute_coverage(records, spine)` — builds the SourceAssembly struct with + record_keys, matched/unmatched lists, missing_from_docs, unmatched_by_entity. +- `demote_unmatched_to_unknown(records, spine)` — AZ-specific helper that + treats pre-set `source` values as authoritative while still marking rows + whose (entity, element) doesn't land on the spine as unknown. +- `assemble_source_driven(records, spine, *, dedup=True, attribute=True)` — + one-shot orchestrator for WI/MN/TX (AZ keeps bespoke ordering around its + cross-attribution fix and uses the helpers individually). + +Other public API: + +- `build_spine_type_index(spine)` — (entity_norm, element_lower) -> canonical type +- `canonical_type(prop_name, raw_type)` — single-property type resolution +- `populate_data_types_from_spine(records, spine)` — apply spine types to matched records +""" + +from __future__ import annotations + +import json +import logging +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from src.models.element import ElementRecord, StateElements +from src.models.alias_grammar import ( + SPA_TEMPLATE_ENTITY, + capitalize_first, + collapse_camel_overlap, + descriptor_variants, + edorg_subtype_aliases, + fk_prefixed_alias, + id_stripped_alias, + is_spa_template_target, + naive_plural_forms, + parent_stripped_tail, + prefix_parent_of, + reference_prefix, + reference_qualifier, + sub_entity_name_forms, + uniqueid_id_alias, +) +from src.models.spine import StateSpine + + +def canonical_type( + prop_name: str, + raw_type: str | None, + raw_format: str | None = None, +) -> str: + """Return a workbook-friendly data type for a spine property. + + Descriptor-suffixed names resolve to `"Descriptor"` regardless of the raw + swagger type — analysts reason about these as descriptor values, not the + underlying serialization (usually integer or string). + + Swagger models temporal types as `string` with `format: date` / `date-time`; + we promote these to canonical `Date` / `DateTime` so reviewer-facing type + cells read correctly (Round 2.2 fix — reviewer B flagged `BirthDate` typed + `String` when the format-derived canonical is `Date`). + + Swagger models integer-ish types as `integer` with `format: int32` / + `int64`; we don't split those further — both collapse to `Integer`. + """ + if prop_name.endswith("Descriptor") or prop_name.endswith("DescriptorId"): + return "Descriptor" + if not raw_type: + return "" + if raw_type == "string" and raw_format: + fmt = raw_format.lower() + if fmt == "date": + return "Date" + if fmt == "date-time": + return "DateTime" + if fmt == "time": + return "Time" + return raw_type[0].upper() + raw_type[1:] + + +def build_spine_type_index(spine: StateSpine) -> dict[tuple[str, str], str]: + """Build a (entity_norm, element_lower) -> canonical-type index from the spine. + + Mirrors the alias keying used by `utils.matching.match_key` so records + that have been normalized to lowercase entity forms can look up types + directly. Covers the same FK-alias forms that `StateSpine.element_keys()` + emits — otherwise records whose spine-match succeeded via an FK alias + (e.g., source cell `SchoolYear` resolving against `schoolYearTypeReference`'s + key property `schoolYear`) get no type back. + + Reference properties (both the `*Reference` form and the bare-prefix alias) + type as `"Reference"`. The round-2 analyst review flagged WI matched-core + rows like `schoolReference` with blank Data Type — populating them here + as `Reference` closes that gap. + """ + from src.utils.matching import match_key + + idx: dict[tuple[str, str], str] = {} + + def _key(entity: str, name: str) -> tuple[str, str]: + e, n = match_key(entity, name) + return (e, n.lower()) + + def _add(entity: str, name: str, type_str: str) -> None: + idx.setdefault(_key(entity, name), type_str) + + def _add_with_descriptor_variants(entity: str, name: str, type_str: str) -> None: + _add(entity, name, type_str) + # Alias FORMS from alias_grammar; the variant TYPE is this + # consumer's own policy (always "Descriptor"). + for variant in descriptor_variants(name): + _add(entity, variant, "Descriptor") + + for entity_name, entity in spine.catalog.entities.items(): + for prop_name, prop in entity.properties.items(): + _add_with_descriptor_variants( + entity_name, + prop_name, + canonical_type(prop_name, prop.type, prop.format), + ) + for ref_name, ref in entity.references.items(): + _add(entity_name, ref_name, "Reference") + prefix = reference_prefix(ref_name) + if prefix and prefix != ref_name: + _add(entity_name, prefix, "Reference") + # Deliberately NARROWER than `element_keys` (issue #213 item + # 2, documented divergence): no camel-collapse, no EdOrg + # subtypes, no qualifier tier here — widening the type index + # is a measured follow-up, not a refactor side effect. + for kp_name, kp in ref.key_properties.items(): + kp_type = canonical_type(kp_name, kp.type, kp.format) + _add_with_descriptor_variants(entity_name, kp_name, kp_type) + uid = uniqueid_id_alias(kp_name) + if uid: + _add(entity_name, uid, kp_type) + if prefix and prefix != kp_name: + cap = capitalize_first(kp_name) + _add_with_descriptor_variants( + entity_name, fk_prefixed_alias(prefix, kp_name), kp_type + ) + # NOTE the guard runs on the BARE cap key (pre-#213 + # behavior), not the full prefixed alias like + # element_keys — same rule, different operand. + stripped_cap = id_stripped_alias(cap) + if stripped_cap: + _add(entity_name, prefix + stripped_cap, kp_type) + for sub_name, sub in entity.sub_collections.items(): + _add(entity_name, sub_name, "Collection") + for sp_name, sp in sub.properties.items(): + _add_with_descriptor_variants( + entity_name, + sp_name, + canonical_type(sp_name, sp.type, sp.format), + ) + + for ext in spine.catalog.extensions.values(): + for prop_name, prop in ext.properties.items(): + _add_with_descriptor_variants( + ext.extends_entity, + prop_name, + canonical_type(prop_name, prop.type, prop.format), + ) + # Extension references: flatten FK key properties onto the + # extends_entity the same way `canonical_spine_emit_keys` does, + # so source rows that land on a ref-flattened slot pick up a + # type even when the extension has no direct properties. + for ref_name, ref in ext.references.items(): + _add(ext.extends_entity, ref_name, "Reference") + prefix = reference_prefix(ref_name) + if prefix and prefix != ref_name: + _add(ext.extends_entity, prefix, "Reference") + for kp_name, kp in ref.key_properties.items(): + kp_type = canonical_type(kp_name, kp.type, kp.format) + _add_with_descriptor_variants(ext.extends_entity, kp_name, kp_type) + uid = uniqueid_id_alias(kp_name) + if uid: + _add(ext.extends_entity, uid, kp_type) + if prefix and prefix != kp_name: + cap = capitalize_first(kp_name) + _add_with_descriptor_variants( + ext.extends_entity, + fk_prefixed_alias(prefix, kp_name), + kp_type, + ) + stripped_cap = id_stripped_alias(cap) + if stripped_cap: + _add(ext.extends_entity, prefix + stripped_cap, kp_type) + # Extension sub-collections: flatten sub-props onto the + # extends_entity (matches `_emit_sub_collection` in + # `canonical_spine_emit_keys`). + # + # Sub-props are registered BEFORE the sub-coll wrapper so a + # scalar sub-prop wins over a colliding wrapper name under + # setdefault's first-writer-wins rule — source docs naming + # such a collision (e.g., MN's `Membership`) typically mean + # the scalar count, not the collection wrapper. + for sub_name, sub in ext.sub_collections.items(): + for sp_name, sp in sub.properties.items(): + _add_with_descriptor_variants( + ext.extends_entity, + sp_name, + canonical_type(sp_name, sp.type, sp.format), + ) + _add(ext.extends_entity, sub_name, "Collection") + + # Mirror the parent-propagation in `StateSpine.extension_element_keys()`. + # Skip when `extends_entity` is itself a catalog entity (no cross-up + # propagation to unrelated parents — matches the rule in spine.py). + extended = ext.extends_entity + if extended in spine.catalog.entities: + continue + parent_name = prefix_parent_of(extended, spine.catalog.entities) + if parent_name is not None: + for prop_name, prop in ext.properties.items(): + _add_with_descriptor_variants( + parent_name, + prop_name, + canonical_type(prop_name, prop.type, prop.format), + ) + sub_tail = extended[len(parent_name):] + if sub_tail and sub_tail[0].isupper(): + _add(parent_name, sub_tail[0].lower() + sub_tail[1:], "Collection") + _add(parent_name, sub_tail, "Collection") + plural_forms = naive_plural_forms(sub_tail) + if plural_forms: + plural_cap, plural_lower = plural_forms + _add(parent_name, plural_cap, "Collection") + _add(parent_name, plural_lower, "Collection") + + # Inherited-identity propagation for concrete Student*ProgramAssociation + # entities — see note in `StateSpine.element_keys`. + template = spine.catalog.entities.get(SPA_TEMPLATE_ENTITY) + if template is not None: + template_entries: list[tuple[tuple[str, str], str]] = [] + tpl_entity_norm, _ = match_key(SPA_TEMPLATE_ENTITY, "") + for (e_norm, n_lower), type_str in list(idx.items()): + if e_norm == tpl_entity_norm: + template_entries.append(((e_norm, n_lower), type_str)) + all_entity_names: set[str] = set(spine.catalog.entities.keys()) + for ext in spine.catalog.extensions.values(): + all_entity_names.add(ext.extends_entity) + # Sort so setdefault-based first-writer-wins picks a deterministic + # propagation target across processes. Without the sort, two entities + # (e.g., a core Student*ProgramAssociation and a + # StudentSpecialEducationProgramAssociationExtension) racing to write + # the same (entity, element) key pick different winners per hash seed. + for entity_name in sorted(all_entity_names): + if not is_spa_template_target(entity_name): + continue + for (_, n_lower), type_str in template_entries: + _add(entity_name, n_lower, type_str) + + return idx + + +def build_spine_description_index( + spine: StateSpine, +) -> dict[tuple[str, str], str]: + """Build (entity_lower, element_lower) → Ed-Fi description lookup. + + Wraps ``canonical_spine_emit_keys`` with its alias sink so every + alias a source row might use (FK prefix, descriptor -Id, subtype + expansion, etc.) points back at the canonical emit's description. + Returned strings are never empty — emits without a description are + omitted so callers can safely test by key presence. + """ + from src.utils.matching import match_key + + alias_sink: dict[tuple[str, str], set[tuple[str, str]]] = {} + emits = canonical_spine_emit_keys(spine, _alias_sink=alias_sink) + + def _key(entity: str, name: str) -> tuple[str, str]: + e, n = match_key(entity, name) + return (e, n.lower()) + + idx: dict[tuple[str, str], str] = {} + + # Primary alias set — record-match forms for each emit's own slot. + for emit in emits: + desc = (emit.description or "").strip() + if not desc: + continue + idx.setdefault(_key(emit.entity, emit.element_name), desc) + + # Secondary alias sink — EducationOrganization subtype expansion, + # descriptor -Id / bare forms, FK-prefix + collapsed + strip-Id aliases. + slot_to_desc = { + _key(emit.entity, emit.element_name): (emit.description or "").strip() + for emit in emits + } + for slot, aliases in alias_sink.items(): + desc = slot_to_desc.get(slot, "") + if not desc: + continue + for alias_key in aliases: + idx.setdefault(alias_key, desc) + + return idx + + +def populate_edfi_standard_definition_from_spine( + records: list[ElementRecord], spine: StateSpine +) -> None: + """Fill ``edfi_standard_definition`` on matched source-lens records. + + Source-lens records arrive from state adapters with only the state's + text — the Ed-Fi baseline lives in the spine catalog's property + descriptions and was never propagated. Phase C2's + ``definition_adds_detail_beyond_edfi`` LLM fact needs both to + compare, so propagate here after ``attribute_record_source``. + + Only fires on rows with ``source in {"core", "extension"}`` — those + resolved to a spine slot and have a canonical Ed-Fi description to + copy. ``source='unknown'`` rows have no spine anchor and keep their + ``edfi_standard_definition`` unchanged (typically None). + + Does NOT overwrite a pre-existing non-empty value — state adapters + that populate the field from another source (none today) stay + authoritative. + """ + from src.utils.matching import record_match_keys + + idx = build_spine_description_index(spine) + + for i, r in enumerate(records): + if r.source == "unknown": + continue + if (r.edfi_standard_definition or "").strip(): + continue + for key in sorted(record_match_keys(r.entity, r.element_name)): + desc = idx.get(key) + if desc: + records[i] = r.model_copy( + update={"edfi_standard_definition": desc} + ) + break + + +def populate_data_types_from_spine( + records: list[ElementRecord], spine: StateSpine +) -> None: + """Fill per-record `data_type` from the spine's property metadata. + + **Canonical-type contract:** for matched records (source in `{"core", + "extension"}`), the spine is authoritative — we OVERWRITE any source- + inferred type with the spine-derived canonical form. This resolves: + + - WI descriptor rows typed as `String` from raw Confluence cells. + - Source typos like WI's `lastQualifyingMove: Bloolean`. + - AZ `*DescriptorId` rows typed as Integer by the XLSX. + - MN rows whose source cell has no type column at all. + + For unmatched (`source="unknown"`) records, we leave `data_type` untouched + so the source-verbatim form remains as an audit trail. + + Runs AFTER `attribute_record_source` so the record's `entity` is already + the canonical post-unflatten form. Uses `record_match_keys` aliases so + descriptor-suffix variants and path-tail splits resolve the way the + spine-match step does. + + Alias keys are sorted before iteration so the first-hit `break` picks a + deterministic winner across processes — Python `set` iteration order + depends on `PYTHONHASHSEED`, which would otherwise cause byte-unstable + artifacts when ambiguous aliases hit different types. + """ + from src.utils.matching import record_match_keys + + idx = build_spine_type_index(spine) + + for i, r in enumerate(records): + if r.source == "unknown": + continue + for key in sorted(record_match_keys(r.entity, r.element_name)): + type_str = idx.get(key) + if type_str: + records[i] = r.model_copy(update={"data_type": type_str}) + break + + +# ------------------------------------------------------------------------ +# Phase 2 — shared post-parse pipeline helpers +# ------------------------------------------------------------------------ + + +def _spine_match_helpers(spine: StateSpine): + """Build (spine_keys, _matches) closures used across pipeline helpers. + + `spine_keys` is the lowercased (entity, element) set derived from + `spine.element_keys()`; `_matches` tests a (entity, element_name) + record via `record_match_keys` intersection. + """ + from src.utils.matching import match_key, record_match_keys + + def _key(entity: str, name: str) -> tuple[str, str]: + e, n = match_key(entity, name) + return (e, n.lower()) + + spine_keys = {_key(e, n) for (e, n) in spine.element_keys()} + + def _matches(entity: str, name: str) -> bool: + return bool(record_match_keys(entity, name) & spine_keys) + + return spine_keys, _matches, _key + + +def run_unflatten_pass( + records: list[ElementRecord], spine: StateSpine +) -> list[dict[str, str]]: + """Identity-preserving unflatten pass used by all four adapters. + + For each record whose (entity, element_name) doesn't match a spine key, + try to resolve the entity as a concatenated sub-entity of a core parent + (e.g., `CalendarDateCalendarEvent` → parent `CalendarDate`, sub-collection + `calendarEvents`). If the re-keyed pair matches, rewrite the record's + entity in place and record the rewrite in the returned `recovered` list. + + This was duplicated verbatim in arizona.py, wisconsin.py, minnesota.py, + and texas.py before Phase 2 consolidated it here. + """ + from src.spine.unflatten import build_unflatten_map, resolve_parent + + _, _matches, _ = _spine_match_helpers(spine) + unflatten_map = build_unflatten_map(spine) + spine_entity_names = spine.entity_keys() + recovered: list[dict[str, str]] = [] + + for i, r in enumerate(records): + if _matches(r.entity, r.element_name): + continue + parent_info = resolve_parent(r.entity, unflatten_map, spine_entity_names) + if parent_info is None: + continue + parent, sub_coll_name = parent_info + if not _matches(parent, r.element_name): + continue + original_entity = r.entity + original_section = r.source_page_or_section or "" + records[i] = r.model_copy(update={ + "entity": parent, + "raw_entity": original_entity, + "source_page_or_section": ( + f"{sub_coll_name} / {original_section}" + if original_section else sub_coll_name + ), + }) + recovered.append({ + "from_entity": original_entity, + "to_entity": parent, + "sub_collection": sub_coll_name, + "element_name": r.element_name, + }) + + return recovered + + +def demote_unmatched_to_unknown( + records: list[ElementRecord], spine: StateSpine +) -> None: + """AZ-specific: demote rows that don't land on the spine to source=unknown. + + AZ uniquely sets `source` at record-creation time from the XLSX namespace + (`edfi.*` → core, `az.*` → extension). Rows whose (entity, element_name) + don't actually match the spine shouldn't read as "Matched (core)" in the + analyst workbook — they're genuinely unresolved. Demote them here while + preserving pre-set `source` values on matched rows. + """ + _, _matches, _ = _spine_match_helpers(spine) + for i, r in enumerate(records): + if not _matches(r.entity, r.element_name): + records[i] = r.model_copy(update={ + "source": "unknown", + "extension_name": None, + }) + + +@dataclass +class SourceAssembly: + """Coverage book-keeping for a source-driven adapter run. + + Populated by `compute_coverage` after the final record list is in hand + (post-unflatten, post-attribute, post-types, post-dedup). + """ + + records: list[ElementRecord] + spine_keys: set[tuple[str, str]] + record_keys: set[tuple[str, str]] + matched_records: list[ElementRecord] + unmatched_records: list[ElementRecord] + missing_from_docs: set[tuple[str, str]] + spine_extension_pairs: set[tuple[str, str]] + unmatched_by_entity: dict[str, dict] = field(default_factory=dict) + # Spine-lens only (issue #213 item 2): documented-match counts per + # alias TIER — how much of the match rate rides on the fuzziest + # grammar tiers (`fk_camel_collapsed`, `sub_entity_tail`, ...) vs + # the primary name. Filled by `assemble_spine_driven`; empty on + # source-driven assemblies. + alias_tier_histogram: dict[str, int] = field(default_factory=dict) + + +def compute_coverage( + records: list[ElementRecord], spine: StateSpine +) -> SourceAssembly: + """Build per-state coverage book-keeping from a finalized record list.""" + from src.spine.unflatten import ABSTRACT_BASES + from src.utils.matching import record_match_keys + + spine_keys, _matches, _key = _spine_match_helpers(spine) + + record_keys: set[tuple[str, str]] = set() + for r in records: + record_keys |= record_match_keys(r.entity, r.element_name) + matched_records = [r for r in records if _matches(r.entity, r.element_name)] + unmatched_records = [ + r for r in records if not _matches(r.entity, r.element_name) + ] + missing_from_docs = spine_keys - record_keys + spine_extension_pairs = { + _key(ext.extends_entity, prop) + for ext in spine.catalog.extensions.values() + for prop in ext.properties + } + + unmatched_by_entity: dict[str, dict] = {} + for r in unmatched_records: + bucket = unmatched_by_entity.setdefault( + r.entity, + { + "tag": "abstract_base" if r.entity in ABSTRACT_BASES else "unknown", + "elements": [], + }, + ) + bucket["elements"].append(r.element_name) + for bucket in unmatched_by_entity.values(): + bucket["elements"] = sorted(set(bucket["elements"])) + unmatched_by_entity = dict(sorted(unmatched_by_entity.items())) + + return SourceAssembly( + records=records, + spine_keys=spine_keys, + record_keys=record_keys, + matched_records=matched_records, + unmatched_records=unmatched_records, + missing_from_docs=missing_from_docs, + spine_extension_pairs=spine_extension_pairs, + unmatched_by_entity=unmatched_by_entity, + ) + + +def assemble_source_driven( + records: list[ElementRecord], + spine: StateSpine, + *, + dedup: bool = True, + attribute: bool = True, +) -> tuple[list[ElementRecord], list[dict[str, str]], SourceAssembly]: + """Shared WI/MN/TX pipeline: unflatten → attribute → types → dedup → coverage. + + AZ's adapter has bespoke interleaving (cross-attribution hint fix + + demote-to-unknown around its pre-set namespace-driven `source`), so it + calls the underlying helpers (`run_unflatten_pass`, `compute_coverage`, + `demote_unmatched_to_unknown`) directly rather than this orchestrator. + + `dedup=False` preserves WI's historical no-dedup behavior (WI's + Confluence scrape doesn't produce duplicates and the adapter has never + called `dedup_records`). + `attribute=False` is for AZ, which pre-sets `source` from namespace. + + Returns `(records, recovered, assembly)`. `records` may be a new list + (post-dedup) distinct from the input. + """ + from src.utils.dedup import dedup_records + from src.utils.matching import attribute_record_source + + recovered = run_unflatten_pass(records, spine) + if attribute: + attribute_record_source(records, spine) + populate_data_types_from_spine(records, spine) + populate_edfi_standard_definition_from_spine(records, spine) + if dedup: + pre_dedup = len(records) + records = dedup_records(records) + if len(records) < pre_dedup: + import logging + logging.getLogger(__name__).info( + "Post-unflatten dedup: %d -> %d records (collapsed %d duplicate pairs)", + pre_dedup, len(records), pre_dedup - len(records), + ) + stamp_edfi_domains(records, spine) + assembly = compute_coverage(records, spine) + return records, recovered, assembly + + +def build_gap_log( + state: str, + spine: StateSpine, + spine_source_rel: str, + assembly: SourceAssembly, + recovered: list[dict[str, str]], + source_coverage_note: str, + spine_coverage_note: str, + extra: dict | None = None, + alias_tier_histogram: dict[str, int] | None = None, +) -> dict: + """Build the canonical gap-log dict from a SourceAssembly. + + The `extra` dict is merged verbatim (e.g., TX adds `source_attribution` + and `provenance`). + """ + total = len(assembly.records) + matched = len(assembly.matched_records) + record_keys_on_spine = len(assembly.record_keys & assembly.spine_keys) + gap_log = { + "state": state, + "spine_source": spine_source_rel, + "spine_unique_element_keys": len(assembly.spine_keys), + "spine_extension_element_count": len(assembly.spine_extension_pairs), + "source_element_count": total, + "source_coverage": { + "matched": matched, + "total": total, + "pct": round(matched / total * 100, 1) if total else 0.0, + "note": source_coverage_note, + }, + "spine_coverage": { + "matched_unique_keys": record_keys_on_spine, + "total_spine_keys": len(assembly.spine_keys), + "pct": ( + round(record_keys_on_spine / len(assembly.spine_keys) * 100, 1) + if assembly.spine_keys else 0.0 + ), + "note": spine_coverage_note, + }, + "unmatched_source_count": len(assembly.unmatched_records), + "unflatten_recovered_count": len(recovered), + "unflatten_recovered": sorted( + recovered, key=lambda d: (d["from_entity"], d["element_name"]) + ), + "unmatched_by_entity": assembly.unmatched_by_entity, + "missing_from_docs_count": len(assembly.missing_from_docs), + "missing_from_docs_samples": sorted(assembly.missing_from_docs)[:50], + } + if alias_tier_histogram is not None: + # Issue #213 item 2 — spine-lens documented matches per alias + # tier ("primary" = the slot's own name/direct expansion; the + # rest are the grammar's fuzzy tiers). Audits how much match + # rate rides on the fuzziest tiers. + gap_log["alias_tier_histogram"] = alias_tier_histogram + if extra: + gap_log.update(extra) + return gap_log + + +# ------------------------------------------------------------------------ +# Phase 3 — spine-driven assembler +# ------------------------------------------------------------------------ + + +@dataclass +class SpineEmit: + """One canonical (entity, element) emission from the spine catalog. + + `canonical_spine_emit_keys()` collapses `element_keys()` alias + expansions (e.g., `schoolReference.schoolId` AND `schoolId`) into one + logical emit per (entity, element_lower) key. + """ + + entity: str + element_name: str + data_type: str + source: str # "core" | "extension" + extension_name: str | None = None + domain: str = "" + description: str = "" + + +@dataclass +class SourceFacts: + """Source-document enrichment for a single (entity, element) slot. + + Populated by `build_source_index()` and looked up by + `assemble_spine_driven()` to enrich spine-origin records with the + state's source-doc text and descriptor/collection metadata. + """ + + definition_text: str = "" + business_rules_text: str | None = None + element_specific_rules: str | None = None + regulatory_citations: list = field(default_factory=list) + related_entities: list = field(default_factory=list) + descriptor_table_code: str | None = None + descriptor_table_values: list = field(default_factory=list) + collections_text: str | None = None + source_document: str | None = None + source_page_or_section: str | None = None + # Issue #184: the source row's Source Area, carried through so the + # spine-driven assembler can stamp the real Source Area onto documented + # spine-lens rows (instead of the Ed-Fi domain that lived in `domain`). + domain: str = "" + + +def _entity_domain(entity_name: str, spine: StateSpine) -> str: + """Return the primary domain for an entity, or empty string. + + Resolution order: + + 1. Direct catalog hit on ``entity_name``. + 2. Longest strict-prefix catalog parent — sub-collection names like + ``StudentSchoolAssociationLocalEducationAgency`` inherit from + ``StudentSchoolAssociation``. + 3. Longest-common-prefix catalog sibling. Abstract bases like + ``EducationOrganization`` and ``GeneralStudentProgramAssociation`` + are not materialized as concrete catalog entities, but their + sibling implementations (e.g., ``EducationOrganizationNetwork``) + carry the convention-named domain. Pick the sibling that shares + the longest CamelCase-aligned prefix and has non-empty domains. + + Mirrors the workbook-side helper at + ``src/report/analyst.py::_edfi_domain_for`` and extends it with + the sibling fallback so swagger-backfilled rows always surface with a + real domain rather than an empty string. + """ + entities = spine.catalog.entities + entity = entities.get(entity_name) + if entity and entity.domains: + return entity.domains[0] + + # Step 2: longest strict-prefix catalog parent. + best_parent: str | None = None + for candidate in entities: + if ( + entity_name != candidate + and len(entity_name) > len(candidate) + and entity_name.startswith(candidate) + and entity_name[len(candidate)].isupper() + and entities[candidate].domains + ): + if best_parent is None or len(candidate) > len(best_parent): + best_parent = candidate + if best_parent is not None: + return entities[best_parent].domains[0] + + # Step 3: longest CamelCase-aligned common-prefix sibling. Some + # abstract bases (``EducationOrganization``, + # ``GeneralStudentProgramAssociation``) are never materialized as + # concrete catalog entities, but their concrete siblings carry the + # right domain. Pick the sibling whose shared prefix is longest AND + # ends at a CamelCase boundary in both names. + best_overlap = 0 + best_sibling_domain: str | None = None + for candidate in entities: + if candidate == entity_name or not entities[candidate].domains: + continue + common_len = 0 + for i in range(min(len(entity_name), len(candidate))): + if entity_name[i] != candidate[i]: + break + common_len = i + 1 + # Require a meaningful overlap (≥6 chars to avoid noisy matches + # like ``Course*`` vs ``CourseTranscript``) AND a CamelCase + # boundary on both sides — i.e., either we hit the end of one + # name or the next char of each name is uppercase. + if common_len < 6: + continue + boundary_self = ( + common_len == len(entity_name) or entity_name[common_len].isupper() + ) + boundary_cand = ( + common_len == len(candidate) or candidate[common_len].isupper() + ) + if not (boundary_self and boundary_cand): + continue + if common_len > best_overlap: + best_overlap = common_len + best_sibling_domain = entities[candidate].domains[0] + if best_sibling_domain is not None: + return best_sibling_domain + + return "" + + +def edfi_domain_for_entity(entity_name: str, spine: StateSpine) -> str | None: + """Canonical entity -> Ed-Fi domain resolver for the ``edfi_domain`` field. + + Single source of truth (issue #184) unifying the two resolvers that + previously disagreed: ``_entity_domain`` (single ``domains[0]``, with + sibling fallback) and the workbook-side ``analyst._edfi_domain_for`` + (joined, no sibling fallback). This returns the **joined** form + (``"; ".join(domains)``) AND keeps the full **3-step** fallback + (direct -> longest-prefix parent -> longest CamelCase sibling), so + every record's ``edfi_domain`` is populated consistently regardless of + provenance and the analyst "Ed-Fi Domain" column can read it directly. + + Returns ``None`` only when neither the entity, any prefix parent, nor + any CamelCase sibling carries domain data (Ed-Fi abstract bases that + are never materialized as concrete catalog entities). + """ + entities = spine.catalog.entities + ent = entities.get(entity_name) + if ent is not None and ent.domains: + return "; ".join(ent.domains) + + # Step 2: longest strict-prefix catalog parent. Sub-collection / + # concatenated names (``StudentSchoolAssociationLocalEducationAgency``) + # inherit their parent's domains. + best_parent: str | None = None + for candidate in entities: + if ( + entity_name != candidate + and len(entity_name) > len(candidate) + and entity_name.startswith(candidate) + and entity_name[len(candidate)].isupper() + and entities[candidate].domains + ): + if best_parent is None or len(candidate) > len(best_parent): + best_parent = candidate + if best_parent is not None: + return "; ".join(entities[best_parent].domains) + + # Step 3: longest CamelCase-aligned common-prefix sibling. Abstract + # bases (``EducationOrganization``, ``GeneralStudentProgramAssociation``) + # aren't concrete catalog entities, but their concrete siblings carry + # the right domains. Mirrors ``_entity_domain``'s step 3 but returns the + # joined form. + best_overlap = 0 + best_sibling: str | None = None + for candidate in entities: + if candidate == entity_name or not entities[candidate].domains: + continue + common_len = 0 + for i in range(min(len(entity_name), len(candidate))): + if entity_name[i] != candidate[i]: + break + common_len = i + 1 + if common_len < 6: + continue + boundary_self = ( + common_len == len(entity_name) or entity_name[common_len].isupper() + ) + boundary_cand = ( + common_len == len(candidate) or candidate[common_len].isupper() + ) + if not (boundary_self and boundary_cand): + continue + if common_len > best_overlap: + best_overlap = common_len + best_sibling = candidate + if best_sibling is not None: + return "; ".join(entities[best_sibling].domains) + + return None + + +def stamp_edfi_domains(records: list[ElementRecord], spine: StateSpine) -> None: + """Set ``edfi_domain`` on every record from its entity via the canonical + resolver (issue #184). Mutates in place. + + Filtered-domain placeholder rows (``source=="filtered"``) are skipped — + they are constructed with ``edfi_domain`` already set to their + PLACEHOLDER_LABEL (the collapsed Ed-Fi domain they represent), and the + canonical resolver would not recognize the synthetic + ``"