From 89d0f7986ee66155378e64b143f05addf3d70336 Mon Sep 17 00:00:00 2001 From: juaristi22 Date: Fri, 11 Jul 2025 13:55:22 +0200 Subject: [PATCH 1/9] enabling analytical assessment of targets --- changelog_entry.yaml | 4 + src/microcalibrate/calibration.py | 66 +++++++++++++++ tests/test_calibration.py | 76 ----------------- tests/test_diagnostics.py | 135 ++++++++++++++++++++++++++++++ 4 files changed, 205 insertions(+), 76 deletions(-) create mode 100644 tests/test_diagnostics.py diff --git a/changelog_entry.yaml b/changelog_entry.yaml index e69de29..9fcbc61 100644 --- a/changelog_entry.yaml +++ b/changelog_entry.yaml @@ -0,0 +1,4 @@ +- bump: minor + changes: + added: + - Adding analytical assessment of targets to Calibration class. diff --git a/src/microcalibrate/calibration.py b/src/microcalibrate/calibration.py index 44b1da1..730251d 100644 --- a/src/microcalibrate/calibration.py +++ b/src/microcalibrate/calibration.py @@ -196,6 +196,72 @@ def _assess_targets( f"of records in the loss matrix. This may make calibration unstable or ineffective." ) + def assess_analytical_solution( + self, use_sparse: Optional[bool] = False + ) -> None: + """Assess analytically which targets complicate achieving calibration accuracy as an optimization problem. + + Uses the Moore-Penrose inverse for least squares solution to relax the assumption that weights need be positive and measure by how much loss increases when trying to solve for a set of equations (the more targets, the larger the number of equations, the harder the optimization problem). + + Args: + use_sparse (bool): Whether to use sparse matrix methods for the analytical solution. Defaults to False. + """ + if self.estimate_matrix is None: + raise ValueError( + "Estimate matrix is not provided. Cannot assess analytical solution from the estimate function alone." + ) + + if logger.level != logging.INFO: + previous_level = logger.level + logger.setLevel(logging.INFO) + + def _get_linear_loss(metrics_matrix, target_vector, sparse=False): + """Gets the mean squared error loss of X.T @ w wrt y for least squares solution""" + X = metrics_matrix + y = target_vector + if not sparse: + X_inv_mp = np.linalg.pinv(X) # Moore-Penrose inverse + w_mp = X_inv_mp.T @ y + y_hat = X.T @ w_mp + + else: + from scipy.sparse import csr_matrix + from scipy.sparse.linalg import lsqr + + X_sparse = csr_matrix(X) + result = lsqr( + X_sparse.T, y + ) # iterative method for sparse matrices + w_sparse = result[0] + y_hat = X_sparse.T @ w_sparse + + return round(np.mean((y - y_hat) ** 2), 3) # mostly for display + + X = self.estimate_matrix_tensor.cpu().numpy() + y = self.targets + + slices = [] + iterative_losses = [] + idx_dict = { + self.estimate_matrix.columns.to_list()[i]: i + for i in range(len(self.estimate_matrix.columns)) + } + + logger.info( + "Assessing analytical solution to the optimization problem for each target... \n" + "This evaluates how much each target complicates achieving calibration accuracy. The loss reported is the mean squared error of the least squares solution." + ) + + for target_name, index_list in idx_dict.items(): + slices.append(index_list) + loss = _get_linear_loss(X[:, slices], y[slices], use_sparse) + iterative_losses.append(loss) + logger.info( + f"Adding: {target_name} to the above, Loss: {loss}, Change in loss: {iterative_losses[-1] - iterative_losses[-2] if len(iterative_losses) > 1 else 'N/A'}" + ) + + logger.setLevel(previous_level) + def summary( self, ) -> str: diff --git a/tests/test_calibration.py b/tests/test_calibration.py index 7b0719e..e5ab6ca 100644 --- a/tests/test_calibration.py +++ b/tests/test_calibration.py @@ -117,79 +117,3 @@ def test_calibration_harder_targets() -> None: rtol=0.01, # relative tolerance err_msg="Calibrated totals do not match target values", ) - - -def test_calibration_warnings_system(caplog) -> None: - """Test the calibration process raises the expected warnings in response to certain inputs.""" - - # Create a sample dataset for testing - random_generator = np.random.default_rng(0) - data = pd.DataFrame( - { - "age": np.append(random_generator.integers(18, 70, size=120), 71), - "income": random_generator.normal(40000, 10000, size=121), - } - ) - - weights = np.ones(len(data)) - - # Calculate target values: - targets_matrix = pd.DataFrame( - { - "income_aged_20_30": ( - (data["age"] >= 20) & (data["age"] <= 30) - ).astype(float) - * data["income"], - "income_aged_40_50": ( - (data["age"] >= 40) & (data["age"] <= 50) - ).astype(float) - * data["income"], - "income_aged_71": (data["age"] == 71).astype(float) - * data["income"], - "income_aged_72": (data["age"] == 72).astype(float) - * data["income"], - } - ) - - # Add specific characteristics to the targets to trigger warnings - targets = np.array( - [ - (targets_matrix["income_aged_20_30"] * weights * 1000).sum(), - (targets_matrix["income_aged_40_50"] * weights * 1.15).sum(), - (targets_matrix["income_aged_71"] * weights * 1.15).sum(), - (targets_matrix["income_aged_71"] * weights * -1.15).sum(), - ] - ) - - calibrator = Calibration( - estimate_matrix=targets_matrix, - weights=weights, - targets=targets, - noise_level=0.05, - epochs=128, - learning_rate=0.01, - dropout_rate=0, - ) - - with caplog.at_level(logging.WARNING, logger="microcalibrate.calibration"): - performance_df = calibrator.calibrate() - - log_text = "\n".join(record.getMessage() for record in caplog.records) - - # Expected fragments - assert ( - "Target income_aged_20_30" in log_text - and "orders of magnitude" in log_text - ), "Magnitude-mismatch warning not emitted." - - assert ( - "Target income_aged_71 is supported by only" in log_text - ), "Low-support warning not emitted." - - assert ( - "Column income_aged_72 has a zero estimate sum" in log_text - ), "Zero estimate sum warning not emitted." - - assert ( - "Some targets are negative" in log_text - ), "Negative target warning not emitted." diff --git a/tests/test_diagnostics.py b/tests/test_diagnostics.py new file mode 100644 index 0000000..95ce76c --- /dev/null +++ b/tests/test_diagnostics.py @@ -0,0 +1,135 @@ +""" +Test the different calibration diagnostics and user warnings. +""" + +from src.microcalibrate.calibration import Calibration +import logging +import numpy as np +import pandas as pd + + +def test_calibration_warnings_system(caplog) -> None: + """Test the calibration process raises the expected warnings in response to certain inputs.""" + + # Create a sample dataset for testing + random_generator = np.random.default_rng(0) + data = pd.DataFrame( + { + "age": np.append(random_generator.integers(18, 70, size=120), 71), + "income": random_generator.normal(40000, 10000, size=121), + } + ) + + weights = np.ones(len(data)) + + # Calculate target values: + targets_matrix = pd.DataFrame( + { + "income_aged_20_30": ( + (data["age"] >= 20) & (data["age"] <= 30) + ).astype(float) + * data["income"], + "income_aged_40_50": ( + (data["age"] >= 40) & (data["age"] <= 50) + ).astype(float) + * data["income"], + "income_aged_71": (data["age"] == 71).astype(float) + * data["income"], + "income_aged_72": (data["age"] == 72).astype(float) + * data["income"], + } + ) + + # Add specific characteristics to the targets to trigger warnings + targets = np.array( + [ + (targets_matrix["income_aged_20_30"] * weights * 1000).sum(), + (targets_matrix["income_aged_40_50"] * weights * 1.15).sum(), + (targets_matrix["income_aged_71"] * weights * 1.15).sum(), + (targets_matrix["income_aged_71"] * weights * -1.15).sum(), + ] + ) + + calibrator = Calibration( + estimate_matrix=targets_matrix, + weights=weights, + targets=targets, + noise_level=0.05, + epochs=128, + learning_rate=0.01, + dropout_rate=0, + ) + + with caplog.at_level(logging.WARNING, logger="microcalibrate.calibration"): + performance_df = calibrator.calibrate() + + log_text = "\n".join(record.getMessage() for record in caplog.records) + + # Expected fragments + assert ( + "Target income_aged_20_30" in log_text + and "orders of magnitude" in log_text + ), "Magnitude-mismatch warning not emitted." + + assert ( + "Target income_aged_71 is supported by only" in log_text + ), "Low-support warning not emitted." + + assert ( + "Column income_aged_72 has a zero estimate sum" in log_text + ), "Zero estimate sum warning not emitted." + + assert ( + "Some targets are negative" in log_text + ), "Negative target warning not emitted." + + +def test_calibration_analytical_solution(caplog) -> None: + """Test the calibration process produces the expected analytical target evaluation reporting.""" + + # Create a mock dataset with age and income + random_generator = np.random.default_rng(0) + data = pd.DataFrame( + { + "age": random_generator.integers(18, 70, size=100), + "income": random_generator.normal(40000, 50000, size=100), + } + ) + weights = np.ones(len(data)) + targets_matrix = pd.DataFrame( + { + "income_aged_20_30": ( + (data["age"] >= 20) & (data["age"] <= 30) + ).astype(float) + * data["income"], + "income_aged_40_50": ( + (data["age"] >= 40) & (data["age"] <= 50) + ).astype(float) + * data["income"], + } + ) + targets = np.array( + [ + (targets_matrix["income_aged_20_30"] * weights).sum(), + (targets_matrix["income_aged_40_50"] * weights).sum(), + ] + ) + + calibrator = Calibration( + estimate_matrix=targets_matrix, + weights=weights, + targets=targets, + noise_level=0.05, + epochs=528, + learning_rate=0.01, + dropout_rate=0, + ) + + with caplog.at_level(logging.INFO, logger="microcalibrate.calibration"): + calibrator.assess_analytical_solution() + + log_text = "\n".join(record.getMessage() for record in caplog.records) + + assert ( + "Change in loss:" in log_text + ), "Analytical solution diagnostics not reported." From 82a6de1615a4ab336d15dc5be6d386376ce202b9 Mon Sep 17 00:00:00 2001 From: juaristi22 Date: Mon, 14 Jul 2025 14:18:29 +0200 Subject: [PATCH 2/9] dashboard can now load csvs with infinite values --- changelog_entry.yaml | 1 + microcalibration-dashboard/src/utils/csvParser.ts | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/changelog_entry.yaml b/changelog_entry.yaml index 9fcbc61..b7fab3d 100644 --- a/changelog_entry.yaml +++ b/changelog_entry.yaml @@ -2,3 +2,4 @@ changes: added: - Adding analytical assessment of targets to Calibration class. + - Supporting loading a csv with infinite values in the dashboard. diff --git a/microcalibration-dashboard/src/utils/csvParser.ts b/microcalibration-dashboard/src/utils/csvParser.ts index ab139aa..62a89b6 100644 --- a/microcalibration-dashboard/src/utils/csvParser.ts +++ b/microcalibration-dashboard/src/utils/csvParser.ts @@ -16,7 +16,14 @@ export function parseCalibrationCSV(csvContent: string): CalibrationDataPoint[] row.epoch !== undefined && row.loss !== undefined && row.target_name !== undefined - ); + ).map(row => ({ + ...row, + // Replace infinite values with a large finite number for better handling + loss: isFinite(row.loss) ? row.loss : (row.loss > 0 ? 1e10 : -1e10), + error: isFinite(row.error) ? row.error : (row.error > 0 ? 1e10 : -1e10), + abs_error: isFinite(row.abs_error) ? row.abs_error : 1e10, + rel_abs_error: isFinite(row.rel_abs_error) ? row.rel_abs_error : 1e10, + })); } export function getCalibrationMetrics(data: CalibrationDataPoint[]): CalibrationMetrics { @@ -43,6 +50,12 @@ export function getCalibrationMetrics(data: CalibrationDataPoint[]): Calibration for (let i = 1; i < lossByEpoch.length; i++) { const currentLoss = lossByEpoch[i].loss; const prevLoss = lossByEpoch[i - 1].loss; + + // Handle cases where loss values might be very large or infinite + if (!isFinite(currentLoss) || !isFinite(prevLoss) || prevLoss === 0) { + continue; + } + const improvement = (prevLoss - currentLoss) / prevLoss; if (improvement < 0.001) { // Less than 0.1% improvement From 1e76552234a5a69e79aed74ee7eab2a83890d2e7 Mon Sep 17 00:00:00 2001 From: juaristi22 Date: Tue, 15 Jul 2025 12:58:45 +0200 Subject: [PATCH 3/9] change assessment to original metrics matrix --- src/microcalibrate/calibration.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/microcalibrate/calibration.py b/src/microcalibrate/calibration.py index 790a987..76a8500 100644 --- a/src/microcalibrate/calibration.py +++ b/src/microcalibrate/calibration.py @@ -389,14 +389,14 @@ def _get_linear_loss(metrics_matrix, target_vector, sparse=False): return round(np.mean((y - y_hat) ** 2), 3) # mostly for display - X = self.estimate_matrix_tensor.cpu().numpy() + X = self.original_estimate_matrix.values y = self.targets slices = [] iterative_losses = [] idx_dict = { - self.estimate_matrix.columns.to_list()[i]: i - for i in range(len(self.estimate_matrix.columns)) + self.original_estimate_matrix.columns.to_list()[i]: i + for i in range(len(self.original_estimate_matrix.columns)) } logger.info( From 2e985cb9885edb7c6081e1ac7d5d7b1e8b19f136 Mon Sep 17 00:00:00 2001 From: juaristi22 Date: Mon, 21 Jul 2025 11:18:45 +0200 Subject: [PATCH 4/9] change font to roboto --- .../src/app/globals.css | 19 +++++++++++++++---- microcalibration-dashboard/src/app/layout.tsx | 14 ++++++++------ 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/microcalibration-dashboard/src/app/globals.css b/microcalibration-dashboard/src/app/globals.css index b67f3da..6b85e72 100644 --- a/microcalibration-dashboard/src/app/globals.css +++ b/microcalibration-dashboard/src/app/globals.css @@ -1,15 +1,26 @@ -@import url('https://fonts.googleapis.com/css2?family=JetBrains+Mono:wght@300;400;500;600;700&display=swap'); @import "tailwindcss"; -/* Apply JetBrains Mono everywhere */ +/* Apply Roboto Serif as default font */ * { - font-family: 'JetBrains Mono', 'Courier New', monospace; + font-family: var(--font-roboto-serif), serif; +} + +/* Apply Roboto Mono to results elements */ +.font-mono, +.results-text, +[class*="metric"], +[class*="value"], +[class*="number"], +code, +pre, +.data-cell { + font-family: var(--font-roboto-mono), monospace !important; } /* Base page styling */ body { background: #f8fafc; /* slate-50 */ color: #334155; /* slate-700 */ - font-family: 'JetBrains Mono', 'Courier New', monospace; + font-family: var(--font-roboto-serif), serif; line-height: 1.5; } diff --git a/microcalibration-dashboard/src/app/layout.tsx b/microcalibration-dashboard/src/app/layout.tsx index f7fa87e..48445d3 100644 --- a/microcalibration-dashboard/src/app/layout.tsx +++ b/microcalibration-dashboard/src/app/layout.tsx @@ -1,15 +1,17 @@ import type { Metadata } from "next"; -import { Geist, Geist_Mono } from "next/font/google"; +import { Roboto_Serif, Roboto_Mono } from "next/font/google"; import "./globals.css"; -const geistSans = Geist({ - variable: "--font-geist-sans", +const robotoSerif = Roboto_Serif({ + variable: "--font-roboto-serif", subsets: ["latin"], + weight: ["300", "400", "500", "600", "700"], }); -const geistMono = Geist_Mono({ - variable: "--font-geist-mono", +const robotoMono = Roboto_Mono({ + variable: "--font-roboto-mono", subsets: ["latin"], + weight: ["300", "400", "500", "600", "700"], }); export const metadata: Metadata = { @@ -25,7 +27,7 @@ export default function RootLayout({ return ( {children} From 8a80dcfaac4969390a6ba83a2582689e2b0fcc19 Mon Sep 17 00:00:00 2001 From: juaristi22 Date: Mon, 21 Jul 2025 11:59:58 +0200 Subject: [PATCH 5/9] add table view --- microcalibration-dashboard/src/app/page.tsx | 7 + .../src/components/ComparisonDataTable.tsx | 475 ++++++++++++++++++ 2 files changed, 482 insertions(+) create mode 100644 microcalibration-dashboard/src/components/ComparisonDataTable.tsx diff --git a/microcalibration-dashboard/src/app/page.tsx b/microcalibration-dashboard/src/app/page.tsx index 336e41e..80818e3 100644 --- a/microcalibration-dashboard/src/app/page.tsx +++ b/microcalibration-dashboard/src/app/page.tsx @@ -13,6 +13,7 @@ import ComparisonQualitySummary from '@/components/ComparisonQualitySummary'; import RegressionAnalysis from '@/components/RegressionAnalysis'; import TargetConvergenceComparison from '@/components/TargetConvergenceComparison'; import DataTable from '@/components/DataTable'; +import ComparisonDataTable from '@/components/ComparisonDataTable'; import { CalibrationDataPoint } from '@/types/calibration'; import { parseCalibrationCSV } from '@/utils/csvParser'; import { getCurrentDeeplinkParams, generateShareableUrl, DeeplinkParams } from '@/utils/deeplinks'; @@ -246,6 +247,12 @@ export default function Dashboard() { firstName={filename} secondName={secondFilename} /> + ) : ( // Regular Single Dataset Dashboard diff --git a/microcalibration-dashboard/src/components/ComparisonDataTable.tsx b/microcalibration-dashboard/src/components/ComparisonDataTable.tsx new file mode 100644 index 0000000..cb69f05 --- /dev/null +++ b/microcalibration-dashboard/src/components/ComparisonDataTable.tsx @@ -0,0 +1,475 @@ +'use client'; + +import { CalibrationDataPoint } from '@/types/calibration'; +import { useState, useMemo } from 'react'; +import { compareTargetNames } from '@/utils/targetOrdering'; + +interface ComparisonDataTableProps { + firstData: CalibrationDataPoint[]; + secondData: CalibrationDataPoint[]; + firstName: string; + secondName: string; +} + +interface ComparisonRow { + targetName: string; + epoch: number; + first?: CalibrationDataPoint; + second?: CalibrationDataPoint; +} + +type SortField = keyof CalibrationDataPoint | 'random'; +type SortDataset = 'first' | 'second' | null; +type SortDirection = 'asc' | 'desc'; + +export default function ComparisonDataTable({ + firstData, + secondData, + firstName, + secondName +}: ComparisonDataTableProps) { + // Get all unique epochs from both datasets + const allEpochs = useMemo(() => { + const epochs1 = firstData.map(item => item.epoch); + const epochs2 = secondData.map(item => item.epoch); + return Array.from(new Set([...epochs1, ...epochs2])).sort((a, b) => a - b); + }, [firstData, secondData]); + + // Get the maximum epoch for default selection + const maxEpoch = allEpochs.length > 0 ? Math.max(...allEpochs) : 0; + + const [sortField, setSortField] = useState('target_name'); + const [sortDirection, setSortDirection] = useState('asc'); + const [sortDataset, setSortDataset] = useState(null); + const [filter, setFilter] = useState(''); + const [epochFilter, setEpochFilter] = useState(maxEpoch); + const [currentPage, setCurrentPage] = useState(1); + const itemsPerPage = 50; + + // Create comparison rows by merging data from both datasets + const comparisonRows = useMemo(() => { + // Get all unique target names from both datasets + const targetNames1 = new Set(firstData.map(item => item.target_name)); + const targetNames2 = new Set(secondData.map(item => item.target_name)); + const allTargetNames = Array.from(new Set([...targetNames1, ...targetNames2])); + + // Create lookup maps for efficient data retrieval + const firstDataMap = new Map(); + const secondDataMap = new Map(); + + firstData.forEach(item => { + if (item.epoch === epochFilter) { + firstDataMap.set(item.target_name, item); + } + }); + + secondData.forEach(item => { + if (item.epoch === epochFilter) { + secondDataMap.set(item.target_name, item); + } + }); + + // Create comparison rows + const rows: ComparisonRow[] = allTargetNames.map(targetName => ({ + targetName, + epoch: epochFilter, + first: firstDataMap.get(targetName), + second: secondDataMap.get(targetName), + })); + + // Filter by search term + return rows.filter(row => + row.targetName.toLowerCase().includes(filter.toLowerCase()) + ); + }, [firstData, secondData, epochFilter, filter]); + + const sortedData = useMemo(() => { + if (sortField === 'random') { + return [...comparisonRows].sort(() => { + const seed = comparisonRows.length; + return (seed * 9301 + 49297) % 233280 / 233280 - 0.5; + }); + } + + return [...comparisonRows].sort((a, b) => { + if (sortField === 'target_name') { + const result = compareTargetNames(a.targetName, b.targetName); + return sortDirection === 'asc' ? result : -result; + } + + // Sort by values based on selected dataset + let aVal, bVal; + if (sortDataset === 'first') { + aVal = a.first?.[sortField as keyof CalibrationDataPoint]; + bVal = b.first?.[sortField as keyof CalibrationDataPoint]; + } else if (sortDataset === 'second') { + aVal = a.second?.[sortField as keyof CalibrationDataPoint]; + bVal = b.second?.[sortField as keyof CalibrationDataPoint]; + } else { + // Default behavior: use first dataset, fallback to second + aVal = a.first?.[sortField as keyof CalibrationDataPoint] ?? a.second?.[sortField as keyof CalibrationDataPoint]; + bVal = b.first?.[sortField as keyof CalibrationDataPoint] ?? b.second?.[sortField as keyof CalibrationDataPoint]; + } + + // Handle undefined/null values - put them at the end regardless of sort direction + if ((aVal === undefined || aVal === null) && (bVal === undefined || bVal === null)) { + return 0; + } + if (aVal === undefined || aVal === null) { + return 1; + } + if (bVal === undefined || bVal === null) { + return -1; + } + + if (typeof aVal === 'number' && typeof bVal === 'number') { + return sortDirection === 'asc' ? aVal - bVal : bVal - aVal; + } + + const aStr = String(aVal || '').toLowerCase(); + const bStr = String(bVal || '').toLowerCase(); + return sortDirection === 'asc' + ? aStr.localeCompare(bStr) + : bStr.localeCompare(aStr); + }); + }, [comparisonRows, sortField, sortDirection, sortDataset]); + + const paginatedData = useMemo(() => { + const start = (currentPage - 1) * itemsPerPage; + return sortedData.slice(start, start + itemsPerPage); + }, [sortedData, currentPage]); + + if (firstData.length === 0 && secondData.length === 0) { + return ( +
+

Detailed comparison results

+

No data available

+
+ ); + } + + const totalPages = Math.ceil(sortedData.length / itemsPerPage); + + const handleSort = (field: keyof CalibrationDataPoint, dataset?: 'first' | 'second') => { + // For target_name, don't use dataset-specific sorting + if (field === 'target_name') { + if (sortField === field && sortDataset === null) { + setSortDirection(sortDirection === 'asc' ? 'desc' : 'asc'); + } else { + setSortField(field); + setSortDataset(null); + setSortDirection('asc'); + } + return; + } + + // For other fields, use dataset-specific sorting + // Ensure dataset is always defined for non-target_name fields + const targetDataset = dataset || 'first'; + if (sortField === field && sortDataset === targetDataset) { + setSortDirection(sortDirection === 'asc' ? 'desc' : 'asc'); + } else { + setSortField(field); + setSortDataset(targetDataset); + setSortDirection('desc'); + } + }; + + const SortButton = ({ field, children, dataset }: { + field: keyof CalibrationDataPoint, + children: React.ReactNode, + dataset?: 'first' | 'second' + }) => { + const isActive = field === 'target_name' + ? (sortField === field && sortDataset === null) + : (sortField === field && sortDataset === dataset); + + return ( +
+ {children} + +
+ ); + }; + + const DatasetSortButton = ({ field, dataset, children }: { + field: keyof CalibrationDataPoint, + dataset: 'first' | 'second', + children: React.ReactNode + }) => { + const isActive = sortField === field && sortDataset === dataset; + const isFirst = dataset === 'first'; + const colorClass = isFirst ? 'text-blue-600' : 'text-purple-600'; + + return ( +
+ {children} + +
+ ); + }; + + const formatValue = (value: number | undefined | null) => { + if (value === undefined || value === null || isNaN(value)) { + return 'N/A'; + } + + const numValue = Number(value); + if (isNaN(numValue)) { + return 'N/A'; + } + + if (Math.abs(numValue) >= 1000000) { + return (numValue / 1000000).toFixed(2) + 'M'; + } else if (Math.abs(numValue) >= 1000) { + return (numValue / 1000).toFixed(1) + 'K'; + } + return numValue.toFixed(2); + }; + + const getErrorClass = (relError: number | undefined | null) => { + if (relError === undefined || relError === null || isNaN(relError)) { + return 'text-gray-400'; + } + return relError < 0.05 ? 'text-green-600' : + relError < 0.20 ? 'text-yellow-600' : 'text-red-600'; + }; + + const renderDataCell = (value: number | undefined | null, isFirst: boolean, isError: boolean = false) => { + if (value === undefined || value === null) { + return —; + } + + const colorClass = isFirst ? 'text-blue-600' : 'text-purple-600'; + let finalClass = colorClass; + let displayValue = formatValue(value); + + if (isError && typeof value === 'number' && !isNaN(value)) { + displayValue = `${(value * 100).toFixed(2)}%`; + finalClass = getErrorClass(value); + } else if (typeof value === 'number' && value >= 0 && !isError) { + displayValue = '+' + displayValue; + } + + return ( + + {displayValue} + + ); + }; + + return ( +
+
+

Detailed comparison results

+
+ + { + setFilter(e.target.value); + setCurrentPage(1); + }} + className="bg-white border border-gray-300 text-gray-900 px-3 py-2 rounded font-mono text-sm w-64 focus:outline-none focus:ring-2 focus:ring-blue-500" + /> +
+ {sortedData.length} entries +
+
+
+ +
+
+
+
+ {firstName} +
+
+
+ {secondName} +
+
+
+ 💡 Click the ▲▼ arrows next to column headers to sort. Target name sorts alphabetically, A/B arrows under "Rel abs error %" sort by dataset performance. +
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {paginatedData.map((row, i) => ( + + + + {/* Target value columns */} + + + + {/* Estimate columns */} + + + + {/* Error columns */} + + + + {/* Abs error columns */} + + + + {/* Rel abs error columns */} + + + + ))} + +
+ Target name + + Target value + + Estimate + + Error + + Abs error + + Rel abs error % +
ABABABAB + + A + + + + B + +
+
+ {row.targetName} +
+ 		+ {renderDataCell(row.first?.target, true)} + + {renderDataCell(row.second?.target, false)} + + {renderDataCell(row.first?.estimate, true)} + + {renderDataCell(row.second?.estimate, false)} + + {renderDataCell(row.first?.error, true)} + + {renderDataCell(row.second?.error, false)} + + {renderDataCell(row.first?.abs_error, true)} + + {renderDataCell(row.second?.abs_error, false)} + + {renderDataCell(row.first?.rel_abs_error, true, true)} + + {renderDataCell(row.second?.rel_abs_error, false, true)} +
+
+ + {totalPages > 1 && ( +
+ +
+ Page {currentPage} of {totalPages} +
+ +
+ )} +
+ ); +} \ No newline at end of file From 37de89b8d8ee2501eb59ff8b33ca48645081f09f Mon Sep 17 00:00:00 2001 From: juaristi22 Date: Mon, 21 Jul 2025 12:14:37 +0200 Subject: [PATCH 6/9] ensure all targets show even if they do not overlap in comparison view --- .../src/components/ComparisonDataTable.tsx | 25 ++++++-- .../TargetConvergenceComparison.tsx | 63 +++++++++++++++---- 2 files changed, 73 insertions(+), 15 deletions(-) diff --git a/microcalibration-dashboard/src/components/ComparisonDataTable.tsx b/microcalibration-dashboard/src/components/ComparisonDataTable.tsx index cb69f05..64e6a36 100644 --- a/microcalibration-dashboard/src/components/ComparisonDataTable.tsx +++ b/microcalibration-dashboard/src/components/ComparisonDataTable.tsx @@ -327,8 +327,9 @@ export default function ComparisonDataTable({ {secondName} -
- 💡 Click the ▲▼ arrows next to column headers to sort. Target name sorts alphabetically, A/B arrows under "Rel abs error %" sort by dataset performance. +
+
💡 Click the ▲▼ arrows next to column headers to sort. Target name sorts alphabetically, A/B arrows under "Rel abs error %" sort by dataset performance.
+
* indicates targets that exist in only one dataset (hover for details)
@@ -394,13 +395,29 @@ export default function ComparisonDataTable({
- {row.targetName} + {row.targetName} + {!row.first && row.second && ( + + * + + )} + {row.first && !row.second && ( + + * + + )}
diff --git a/microcalibration-dashboard/src/components/TargetConvergenceComparison.tsx b/microcalibration-dashboard/src/components/TargetConvergenceComparison.tsx index 6a4ffe5..a25471b 100644 --- a/microcalibration-dashboard/src/components/TargetConvergenceComparison.tsx +++ b/microcalibration-dashboard/src/components/TargetConvergenceComparison.tsx @@ -38,12 +38,12 @@ export default function TargetConvergenceComparison({ firstName, secondName }: TargetConvergenceComparisonProps) { - // Find overlapping targets + // Find all targets (union of both datasets) const firstTargets = new Set(firstData.map(d => d.target_name)); const secondTargets = new Set(secondData.map(d => d.target_name)); - const commonTargets = sortTargetNames(Array.from(firstTargets).filter(target => secondTargets.has(target))); + const allTargets = sortTargetNames(Array.from(new Set([...firstTargets, ...secondTargets]))); - const [selectedTarget, setSelectedTarget] = useState(commonTargets[0] || ''); + const [selectedTarget, setSelectedTarget] = useState(allTargets[0] || ''); const [targetSearchQuery, setTargetSearchQuery] = useState(''); const [showTargetDropdown, setShowTargetDropdown] = useState(false); const [lineOpacity, setLineOpacity] = useState({ @@ -73,14 +73,14 @@ export default function TargetConvergenceComparison({ } // No need to set default targets anymore - handled by search/pagination - }, [firstData, secondData, commonTargets, selectedEpoch]); + }, [firstData, secondData, allTargets, selectedEpoch]); // Reset page when search changes useEffect(() => { setCurrentPage(0); }, [searchQuery]); - if (commonTargets.length === 0) { + if (allTargets.length === 0) { return (

@@ -89,7 +89,7 @@ export default function TargetConvergenceComparison({

- No common targets found between datasets for convergence comparison. + No targets found in either dataset for convergence comparison.

@@ -131,7 +131,7 @@ export default function TargetConvergenceComparison({ // Filter targets based on search query for target selection dropdown const searchFilteredTargets = sortTargetsWithRelevance( - commonTargets.filter(target => + allTargets.filter(target => target.toLowerCase().includes(targetSearchQuery.toLowerCase()) ), targetSearchQuery @@ -142,7 +142,7 @@ export default function TargetConvergenceComparison({ // Filter and paginate targets based on search const getFilteredTargets = () => { - const filtered = commonTargets.filter(target => + const filtered = allTargets.filter(target => target.toLowerCase().includes(searchQuery.toLowerCase()) ); return sortTargetsWithRelevance(filtered, searchQuery); @@ -338,8 +338,24 @@ export default function TargetConvergenceComparison({ className="w-full border border-gray-300 rounded-md px-3 py-1 text-sm text-gray-900 bg-white focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500" /> {selectedTarget && ( -
- Selected: {selectedTarget} +
+ Selected: {selectedTarget} + {!firstTargets.has(selectedTarget) && secondTargets.has(selectedTarget) && ( + + * + + )} + {firstTargets.has(selectedTarget) && !secondTargets.has(selectedTarget) && ( + + * + + )}
)} @@ -359,7 +375,25 @@ export default function TargetConvergenceComparison({ }`} title={target} > -
{target}
+
+ {target} + {!firstTargets.has(target) && secondTargets.has(target) && ( + + * + + )} + {firstTargets.has(target) && !secondTargets.has(target) && ( + + * + + )} +
))} {searchFilteredTargets.length > 10 && ( @@ -373,6 +407,13 @@ export default function TargetConvergenceComparison({ + {/* Legend for asterisks */} +
+
+ * indicates targets that exist in only one dataset (hover for details) +
+
+ {/* Summary statistics */}
From 20a56c86d52724165ee955fc8902d0bab28dc836 Mon Sep 17 00:00:00 2001 From: juaristi22 Date: Mon, 21 Jul 2025 12:16:31 +0200 Subject: [PATCH 7/9] update changelog entry --- changelog_entry.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog_entry.yaml b/changelog_entry.yaml index b7fab3d..dd9d7f8 100644 --- a/changelog_entry.yaml +++ b/changelog_entry.yaml @@ -2,4 +2,4 @@ changes: added: - Adding analytical assessment of targets to Calibration class. - - Supporting loading a csv with infinite values in the dashboard. + - Enhance dashboard to show all targets even if not overlapping and better font / table view. From 37db0cfd5b088d5e6ea0ad6ee4c7fafe15fcadb3 Mon Sep 17 00:00:00 2001 From: juaristi22 Date: Mon, 21 Jul 2025 17:54:38 +0200 Subject: [PATCH 8/9] make analytical assessment return a dataframe with results --- src/microcalibrate/calibration.py | 26 ++++++++++++++++---------- tests/test_diagnostics.py | 15 ++++++++------- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/src/microcalibrate/calibration.py b/src/microcalibrate/calibration.py index 76a8500..4deb179 100644 --- a/src/microcalibrate/calibration.py +++ b/src/microcalibrate/calibration.py @@ -363,14 +363,15 @@ def assess_analytical_solution( "Estimate matrix is not provided. Cannot assess analytical solution from the estimate function alone." ) - if logger.level != logging.INFO: - previous_level = logger.level - logger.setLevel(logging.INFO) - def _get_linear_loss(metrics_matrix, target_vector, sparse=False): """Gets the mean squared error loss of X.T @ w wrt y for least squares solution""" X = metrics_matrix y = target_vector + normalization_factor = ( + self.normalization_factor + if self.normalization_factor is not None + else 1 + ) if not sparse: X_inv_mp = np.linalg.pinv(X) # Moore-Penrose inverse w_mp = X_inv_mp.T @ y @@ -387,13 +388,13 @@ def _get_linear_loss(metrics_matrix, target_vector, sparse=False): w_sparse = result[0] y_hat = X_sparse.T @ w_sparse - return round(np.mean((y - y_hat) ** 2), 3) # mostly for display + return np.mean(((y - y_hat) ** 2) * normalization_factor) X = self.original_estimate_matrix.values y = self.targets + results = [] slices = [] - iterative_losses = [] idx_dict = { self.original_estimate_matrix.columns.to_list()[i]: i for i in range(len(self.original_estimate_matrix.columns)) @@ -407,12 +408,17 @@ def _get_linear_loss(metrics_matrix, target_vector, sparse=False): for target_name, index_list in idx_dict.items(): slices.append(index_list) loss = _get_linear_loss(X[:, slices], y[slices], use_sparse) - iterative_losses.append(loss) - logger.info( - f"Adding: {target_name} to the above, Loss: {loss}, Change in loss: {iterative_losses[-1] - iterative_losses[-2] if len(iterative_losses) > 1 else 'N/A'}" + delta = loss - results[-1]["loss"] if results else None + + results.append( + { + "target_added": target_name, + "loss": loss, + "delta_loss": delta, + } ) - logger.setLevel(previous_level) + return pd.DataFrame(results) def summary( self, diff --git a/tests/test_diagnostics.py b/tests/test_diagnostics.py index 95ce76c..5bdc584 100644 --- a/tests/test_diagnostics.py +++ b/tests/test_diagnostics.py @@ -110,8 +110,8 @@ def test_calibration_analytical_solution(caplog) -> None: ) targets = np.array( [ - (targets_matrix["income_aged_20_30"] * weights).sum(), - (targets_matrix["income_aged_40_50"] * weights).sum(), + (targets_matrix["income_aged_20_30"] * weights).sum() * 2, + (targets_matrix["income_aged_40_50"] * weights).sum() * 2, ] ) @@ -125,11 +125,12 @@ def test_calibration_analytical_solution(caplog) -> None: dropout_rate=0, ) - with caplog.at_level(logging.INFO, logger="microcalibrate.calibration"): - calibrator.assess_analytical_solution() + analytical_assessment = calibrator.assess_analytical_solution() - log_text = "\n".join(record.getMessage() for record in caplog.records) + assert set(analytical_assessment["target_added"]) == set( + list(targets_matrix.columns) + ), "Not all targets were added to the assessment." assert ( - "Change in loss:" in log_text - ), "Analytical solution diagnostics not reported." + analytical_assessment["loss"].all() != 0 + ), "Loss values should not be zero." From 2faf86891c92a758910bcaa702e876a5ff8c400a Mon Sep 17 00:00:00 2001 From: juaristi22 Date: Mon, 21 Jul 2025 17:57:17 +0200 Subject: [PATCH 9/9] update test --- tests/test_diagnostics.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/test_diagnostics.py b/tests/test_diagnostics.py index 5bdc584..3ada995 100644 --- a/tests/test_diagnostics.py +++ b/tests/test_diagnostics.py @@ -130,7 +130,3 @@ def test_calibration_analytical_solution(caplog) -> None: assert set(analytical_assessment["target_added"]) == set( list(targets_matrix.columns) ), "Not all targets were added to the assessment." - - assert ( - analytical_assessment["loss"].all() != 0 - ), "Loss values should not be zero."