Metrics matrix creation logic from database

changelog_entry.yaml

@@ -0,0 +1,7 @@
+- bump: minor
+  changes:
+    added:
+    - Logic to create estimate matrix for calibration from a database.
+    - Conversion functions between dataset classes to enable stacking datasets.
+    - Logic to calibrate for multiple geographic levels with two different routines.
+    - Calibration documentation.

docs/_toc.yml

@@ -2,3 +2,5 @@ format: jb-book
 root: intro
 chapters:
   - file: dataset.ipynb
+  - file: normalise_keys.md
+  - file: calibration.ipynb

docs/dataset.ipynb

@@ -5,7 +5,7 @@
    "id": "gq2207gugn",
    "metadata": {},
    "source": [
-    "# PolicyEngine Dataset classes documentation\n",
+    "# PolicyEngine Dataset classes\n",
     "\n",
     "This notebook provides documentation for the `SingleYearDataset` and `MultiYearDataset` classes in PolicyEngine Data. These classes are designed to handle structured data for policy analysis and microsimulation.\n",
     "\n",

docs/intro.md

@@ -1,3 +1,3 @@
-## PolicyEngine Data
+## PolicyEngine Data
 
 This is the documentation for PolicyEngine Data, the open-source Python package powering PolicyEngine's data processing and storing functionality. It is used by PolicyEngine UK Data and PolicyEngine US Data, which each define the custom logic specific to processing UK and US data sources.

docs/normalise_keys.md

@@ -21,32 +21,32 @@ Normalises primary and foreign keys across multiple related tables.
 import pandas as pd
 from policyengine_data import normalise_table_keys
 
-users = pd.DataFrame({
-    'user_id': [101, 105, 103],
+person = pd.DataFrame({
+    'person_id': [101, 105, 103],
     'name': ['Alice', 'Bob', 'Carol']
 })
 
-orders = pd.DataFrame({
-    'order_id': [201, 205, 207], 
-    'user_id': [105, 101, 105],
-    'amount': [25.99, 15.50, 42.00]
+household = pd.DataFrame({
+    'household_id': [201, 205, 207], 
+    'person_id': [105, 101, 105],
+    'income': [25000, 15000, 42000]
 })
 
-tables = {'users': users, 'orders': orders}
-primary_keys = {'users': 'user_id', 'orders': 'order_id'}
+tables = {'person': person, 'household': household}
+primary_keys = {'person': 'person_id', 'household': 'household_id'}
 
 # Auto-detect foreign keys
 normalised = normalise_table_keys(tables, primary_keys)
 
 # Or specify foreign keys explicitly
-foreign_keys = {'orders': {'user_id': 'users'}}
+foreign_keys = {'household': {'person_id': 'persons'}}
 normalised = normalise_table_keys(tables, primary_keys, foreign_keys)
 ```
 
 After normalisation:
-- User IDs become 0, 1, 2 (instead of 101, 105, 103)
-- Order IDs become 0, 1, 2 (instead of 201, 205, 207)  
-- Foreign key relationships are preserved (Bob's orders still reference Bob's new ID)
+- Person IDs become 0, 1, 2 (instead of 101, 105, 103)
+- Household IDs become 0, 1, 2 (instead of 201, 205, 207)  
+- Foreign key relationships are preserved (Bob's household still reference Bob's new ID)
 
 ### `normalise_single_table_keys(df, key_column, start_index=0)`
 
@@ -66,12 +66,12 @@ import pandas as pd
 from policyengine_data import normalise_single_table_keys
 
 df = pd.DataFrame({
-    'id': [101, 105, 103],
-    'value': ['A', 'B', 'C'] 
+    'person_id': [101, 105, 103],
+    'age': [25, 30, 35] 
 })
 
-normalised = normalise_single_table_keys(df, 'id')
-# Result: IDs become 0, 1, 2
+normalised = normalise_single_table_keys(df, 'person_id')
+# Result: person_ids become 0, 1, 2
 ```
 
 ## Key features

pyproject.toml

@@ -14,9 +14,11 @@ dependencies = [
     "huggingface_hub>=0.25.1",
     "tables",
     "policyengine-core>=3.6.4",
+    "policyengine-us", # remove as soon as we fix UCGID
     "microdf-python",
+    "microcalibrate",
     "sqlalchemy",
-    "huggingface_hub",
+    "torch",
 ]
 
 [project.optional-dependencies]
@@ -30,6 +32,7 @@ dev = [
     "build",
     "linecheck",
     "yaml-changelog>=0.1.7",
+    "policyengine-us>=1.366.0",
 ]
 
 docs = [

src/policyengine_data/calibration/__init__.py

@@ -1 +1,12 @@
+from .calibrate import calibrate_all_levels, calibrate_single_geography_level
+from .dataset_duplication import (
+    load_dataset_for_geography_legacy,
+    minimize_calibrated_dataset_legacy,
+)
+from .metrics_matrix_creation import (
+    create_metrics_matrix,
+    validate_metrics_matrix,
+)
 from .target_rescaling import download_database, rescale_calibration_targets
+from .target_uprating import uprate_calibration_targets
+from .utils import create_geographic_normalization_factor