diff --git a/.gitignore b/.gitignore index f6cce8d..6306be6 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,11 @@ neuroscan_data.json uploads/ scan_history/ +# Standardized image/mask copies generated by dataset_standardize.py +# (regenerable from the originals; never commit dataset images) +dataset_standardized/* +dataset_reports/* + # Coverage .coverage coverage.xml diff --git a/PREPROCESSING.md b/PREPROCESSING.md new file mode 100644 index 0000000..7799ce3 --- /dev/null +++ b/PREPROCESSING.md @@ -0,0 +1,189 @@ +# Dataset Standardization & Preprocessing Pipeline + +This document describes how raw MRI data is standardized into a single, +consistent format before training — covering image dimensions, file +formats, channel layout, mask encoding, and metadata / labeling +conventions. + +It is implemented by [`dataset_standardize.py`](dataset_standardize.py) +and is the normalization counterpart to the auditing performed by +[`dataset_quality.py`](dataset_quality.py). + +> Closes #70 — *Standardize and Normalize Dataset Formats.* + +--- + +## Why this exists + +The raw TCGA data and the shipped `data_mask.csv` are not uniform: + +- **Mixed image properties.** Source scans/masks are TIFs that may be + grayscale, RGB, RGBA, 8-bit or 16-bit, and not guaranteed to all be the + exact same size. +- **Inconsistent mask encoding.** Masks can carry stray channels or + non-binary values, and the only thing that matters for training is a + clean binary tumor region. +- **Broken `patient_id` metadata.** In the shipped `data_mask.csv` the + `patient_id` column is misaligned — the same id is repeated across + slices that actually belong to *different* patients (3,894 of 3,929 + rows). A patient-grouped train/val/test split keyed on that column + would leak the same patient across splits and inflate metrics. + +The pipeline removes all of these inconsistencies non-destructively: the +originals are never modified; standardized copies and a corrected +manifest are written alongside them. + +--- + +## What "standardized" means here + +| Property | Canonical value | +|-----------------|----------------------------------------------| +| Image size | **256 × 256** (matches the model input) | +| Image channels | **3 (RGB)** | +| Image dtype | **uint8** (16-bit inputs min-max scaled) | +| Image format | **PNG** (lossless) | +| Mask size | **256 × 256**, resized **nearest-neighbour** | +| Mask channels | **1 (grayscale)** | +| Mask values | **{0, 255}** (binary) | +| `patient_id` | Derived from the image folder (ground truth) | +| `slice` | Parsed integer slice index | +| `mask` label | Re-derived from actual mask content (0/1) | + +Pixel-value normalization (`/255` for classification, mean–std +standardization for segmentation) is intentionally **not** baked into the +stored files — it stays at model-feed time, exactly as in +`app.py::preprocess_image_classification` / `preprocess_image_segmentation`, +so the standardized data stays reusable across both stages. + +--- + +## Pipeline stages + +### Stage 1 — Manifest standardization (no images required) + +Runs purely on the CSV, so it works even in a fresh clone without the +dataset downloaded: + +1. Re-derive the canonical `patient_id` from each `image_path` folder. +2. Parse a numeric `slice` index from each filename (`..._34.tif → 34`). +3. Normalize path separators (Windows `\` → POSIX `/`). +4. Preserve the source label as `original_mask` for traceability and + promote `mask` to the canonical label column. +5. Emit a fixed, ordered set of columns. + +### Stage 2 — Pixel standardization (requires the image files) + +For every row whose files exist locally: + +1. **Image** → 3-channel RGB, uint8, resized to 256 × 256 + (`INTER_AREA` when shrinking, `INTER_LINEAR` when enlarging). +2. **Mask** → single-channel, resized to 256 × 256 with **`INTER_NEAREST`** + (bilinear would invent gray values along tumor boundaries), then + thresholded to a clean binary `{0, 255}`. +3. **Label** → re-derived from the standardized mask (`1` if any + foreground pixel, else `0`), flagging rows whose original label was + wrong. +4. Standardized copies are written to `dataset_standardized/`, mirroring + the original `TCGA_*/` folder layout, as `.png`. + +Rows whose files are missing locally are skipped and counted — the run +still succeeds and the manifest is still corrected. + +--- + +## Usage + +```bash +python dataset_standardize.py +``` + +No arguments; behaviour is controlled by the constants at the top of the +script (`CSV_PATH`, `IMG_ROOT`, `OUTPUT_DIR`, `TARGET_SIZE`, `IMAGE_EXT`, +`MASK_THRESHOLD`). + +### Outputs + +| Path | Contents | +|-----------------------------------------------|-------------------------------------------| +| `data_mask_standardized.csv` | Canonical manifest (see schema below) | +| `dataset_standardized//*.png` | Standardized images + masks (gitignored) | +| `dataset_reports/standardization_report.json` | Run summary + consistency check | + +> `dataset_standardized/` is git-ignored — standardized images are +> regenerable and, like the raw `TCGA_*` folders, are never committed. + +### Manifest schema (`data_mask_standardized.csv`) + +| Column | Description | +|-------------------|--------------------------------------------------------| +| `patient_id` | Canonical TCGA patient id (corrected) | +| `slice` | Integer slice index parsed from the filename | +| `image_path` | Original image path (unchanged) | +| `mask_path` | Original mask path (unchanged) | +| `mask` | Canonical 0/1 label (re-derived from mask when present)| +| `original_mask` | Label as it appeared in the source CSV | +| `std_image_path` | Standardized image path under `dataset_standardized/` | +| `std_mask_path` | Standardized mask path under `dataset_standardized/` | + +The original `image_path` / `mask_path` columns are preserved so existing +code keeps working; training can switch to the `std_*` columns once the +standardized set has been generated. + +--- + +## Using it in training + +```python +import pandas as pd + +df = pd.read_csv("data_mask_standardized.csv") + +# Patient-grouped split — now safe, because patient_id is correct +from sklearn.model_selection import GroupShuffleSplit +splitter = GroupShuffleSplit(test_size=0.30, n_splits=1, random_state=42) +train_idx, hold_idx = next(splitter.split(df, groups=df["patient_id"])) +``` + +--- + +## Sample report + +```json +{ + "manifest": { + "rows": 3929, + "unique_patients": 110, + "patient_id_corrected": 3894, + "slices_parsed": 3929 + }, + "images": { + "target_resolution": "256x256", + "output_format": ".png", + "channels": 3, + "dtype": "uint8", + "consistency_check": { "uniform_resolution": true, "uniform_dtype": true } + }, + "masks": { "values": [0, 255], "interpolation": "nearest", "threshold": 127 }, + "acceptance_criteria": { + "consistent_resolution_and_format": true, + "masks_standardized": true, + "pipeline_documented": "PREPROCESSING.md" + } +} +``` + +--- + +## Tests + +Unit tests live in +[`tests/unit/test_dataset_standardize.py`](tests/unit/test_dataset_standardize.py): + +```bash +pytest tests/unit/test_dataset_standardize.py +``` + +They cover the metadata helpers, image/mask standardization (including the +nearest-neighbour guarantee and uint16 → uint8 scaling), manifest +correction, and an end-to-end pixel run on synthetic patient folders. diff --git a/README.md b/README.md index 06147bd..3dade88 100644 --- a/README.md +++ b/README.md @@ -233,6 +233,21 @@ TumorVision-2StageAI/ The training notebook reads scan paths relative to the project root, so the folder names and location need to match exactly. +### Standardization & Preprocessing + +To normalize the dataset into a single consistent format (uniform 256×256 RGB +images, binary masks, and corrected `patient_id` / labeling metadata) before +training, run: + +```bash +python dataset_standardize.py +``` + +This produces a canonical manifest (`data_mask_standardized.csv`) and a +standardization report without modifying the originals. The full pipeline — +image dimensions, file formats, mask encoding, and metadata conventions — is +documented in [`PREPROCESSING.md`](PREPROCESSING.md). + --- ## Project Structure diff --git a/dataset_quality.py b/dataset_quality.py index 63d3890..1b08a44 100644 --- a/dataset_quality.py +++ b/dataset_quality.py @@ -1,13 +1,11 @@ import os import json -import csv import warnings -from collections import Counter, defaultdict -import numpy as np import pandas as pd import cv2 import matplotlib + matplotlib.use("Agg") import matplotlib.pyplot as plt import matplotlib.patches as mpatches @@ -15,34 +13,32 @@ warnings.filterwarnings("ignore") # PATHS (all relative to project root) -CSV_PATH = "data_mask.csv" # already in the repo root -IMG_ROOT = "./" # images live as ./TCGA_xxx/TCGA_xxx_1.tif -OUTPUT_DIR = "dataset_reports" +CSV_PATH = "data_mask.csv" # already in the repo root +IMG_ROOT = "./" # images live as ./TCGA_xxx/TCGA_xxx_1.tif +OUTPUT_DIR = "dataset_reports" # STEP 1 — Dataset Statistics def generate_statistics(df: pd.DataFrame) -> dict: - total = len(df) - counts = df["mask"].value_counts().to_dict() + total = len(df) + counts = df["mask"].value_counts().to_dict() # per-patient breakdown - patient_per_class = ( - df.groupby("mask")["patient_id"].nunique().to_dict() - ) + patient_per_class = df.groupby("mask")["patient_id"].nunique().to_dict() # data-integrity checks - duplicate_images = int(df["image_path"].duplicated().sum()) - empty_image_paths = int(df["image_path"].str.strip().eq("").sum()) - invalid_mask_vals = int((~df["mask"].isin([0, 1])).sum()) + duplicate_images = int(df["image_path"].duplicated().sum()) + empty_image_paths = int(df["image_path"].str.strip().eq("").sum()) + invalid_mask_vals = int((~df["mask"].isin([0, 1])).sum()) stats = { - "total_samples" : total, - "unique_patients" : int(df["patient_id"].nunique()), - "mask_distribution" : {str(k): int(v) for k, v in counts.items()}, - "patients_per_class" : {str(k): int(v) for k, v in patient_per_class.items()}, + "total_samples": total, + "unique_patients": int(df["patient_id"].nunique()), + "mask_distribution": {str(k): int(v) for k, v in counts.items()}, + "patients_per_class": {str(k): int(v) for k, v in patient_per_class.items()}, "duplicate_image_paths": duplicate_images, "empty_image_path_rows": empty_image_paths, - "invalid_mask_values" : invalid_mask_vals, + "invalid_mask_values": invalid_mask_vals, } print("\n── Dataset Statistics ──────────────────────────") @@ -61,8 +57,8 @@ def generate_statistics(df: pd.DataFrame) -> dict: def _is_corrupted(img_path: str, mask_path: str) -> tuple[bool, str]: """ Returns (is_corrupted, reason). - Checks: file exists, OpenCV can read it, image is not blank. - """ + Checks: file exists, OpenCV can read it, image is not blank. + """ for label, path in [("image", img_path), ("mask", mask_path)]: full = os.path.join(IMG_ROOT, path) if not os.path.isfile(full): @@ -72,7 +68,7 @@ def _is_corrupted(img_path: str, mask_path: str) -> tuple[bool, str]: return True, f"{label}_unreadable" if label == "image" and img.std() < 1.0: return True, f"{label}_blank" - return False, "" + return False, "" def filter_corrupted(df: pd.DataFrame) -> tuple[pd.DataFrame, list]: @@ -82,10 +78,10 @@ def filter_corrupted(df: pd.DataFrame) -> tuple[pd.DataFrame, list]: have the CSV), file-not-found rows are flagged but the clean CSV still keeps them — set SKIP_MISSING=True below to remove them too. """ - SKIP_MISSING = False # set True if you want to drop missing-file rows + SKIP_MISSING = False # set True if you want to drop missing-file rows removed = [] - keep = [] + keep = [] print("\n── Scanning for Corrupted Samples ──────────────") print(f" Checking {len(df):,} rows …") @@ -94,13 +90,15 @@ def filter_corrupted(df: pd.DataFrame) -> tuple[pd.DataFrame, list]: corrupted, reason = _is_corrupted(row["image_path"], row["mask_path"]) if corrupted: if reason.endswith("_not_found") and not SKIP_MISSING: - keep.append(idx) # keep row, just log it + keep.append(idx) # keep row, just log it else: - removed.append({ - "index" : int(idx), - "image_path": row["image_path"], - "reason" : reason, - }) + removed.append( + { + "index": int(idx), + "image_path": row["image_path"], + "reason": reason, + } + ) else: keep.append(idx) @@ -115,7 +113,7 @@ def filter_corrupted(df: pd.DataFrame) -> tuple[pd.DataFrame, list]: # STEP 3 — Class Imbalance Report def class_imbalance_report(df: pd.DataFrame) -> dict: counts = df["mask"].value_counts().to_dict() - total = len(df) + total = len(df) n0 = counts.get(0, 0) n1 = counts.get(1, 0) @@ -127,10 +125,10 @@ def class_imbalance_report(df: pd.DataFrame) -> dict: cw1 = round(total / (2 * n1), 4) if n1 else None if ratio < 1.5: - severity = "LOW" + severity = "LOW" suggestions = ["Dataset is well-balanced. No special action required."] elif ratio < 3.0: - severity = "MODERATE" + severity = "MODERATE" suggestions = [ f"Pass class_weight={{0: {cw0}, 1: {cw1}}} to model.fit().", "Apply random oversampling on the minority class.", @@ -138,7 +136,7 @@ def class_imbalance_report(df: pd.DataFrame) -> dict: "Evaluate with F1-score / PR-AUC, not just accuracy.", ] else: - severity = "SEVERE" + severity = "SEVERE" suggestions = [ "Use focal loss instead of binary cross-entropy.", "Oversample minority class with augmentation (flip, rotate, crop).", @@ -148,15 +146,21 @@ def class_imbalance_report(df: pd.DataFrame) -> dict: report = { "class_summary": { - "0": {"label": "No Tumor", "count": n0, - "percentage": round(100 * n0 / total, 2)}, - "1": {"label": "Tumor Present", "count": n1, - "percentage": round(100 * n1 / total, 2)}, + "0": { + "label": "No Tumor", + "count": n0, + "percentage": round(100 * n0 / total, 2), + }, + "1": { + "label": "Tumor Present", + "count": n1, + "percentage": round(100 * n1 / total, 2), + }, }, - "imbalance_ratio" : ratio, - "severity" : severity, + "imbalance_ratio": ratio, + "severity": severity, "recommended_class_weights": {"0": cw0, "1": cw1}, - "suggestions" : suggestions, + "suggestions": suggestions, } print("\n── Class Imbalance Report ──────────────────────") @@ -176,28 +180,38 @@ def plot_distribution(counts: dict, out_path: str): labels = ["No Tumor\n(mask=0)", "Tumor Present\n(mask=1)"] values = [counts.get("0", 0), counts.get("1", 0)] colors = ["#4CAF50", "#F44336"] - total = sum(values) + total = sum(values) fig, axes = plt.subplots(1, 2, figsize=(12, 5)) fig.suptitle("NeuroVision — Class Distribution", fontsize=14, fontweight="bold") # Bar - bars = axes[0].bar(labels, values, color=colors, edgecolor="white", - linewidth=1.5, width=0.5) + bars = axes[0].bar( + labels, values, color=colors, edgecolor="white", linewidth=1.5, width=0.5 + ) axes[0].set_title("Sample Count per Class") axes[0].set_ylabel("Number of Samples") axes[0].spines[["top", "right"]].set_visible(False) for bar, val in zip(bars, values): - axes[0].text(bar.get_x() + bar.get_width() / 2, - bar.get_height() + 30, - f"{val:,}\n({100*val/total:.1f}%)", - ha="center", va="bottom", fontsize=11, fontweight="bold") + axes[0].text( + bar.get_x() + bar.get_width() / 2, + bar.get_height() + 30, + f"{val:,}\n({100*val/total:.1f}%)", + ha="center", + va="bottom", + fontsize=11, + fontweight="bold", + ) axes[0].set_ylim(0, max(values) * 1.2) # Pie _, _, autotexts = axes[1].pie( - values, labels=labels, colors=colors, autopct="%1.1f%%", - startangle=90, pctdistance=0.75, + values, + labels=labels, + colors=colors, + autopct="%1.1f%%", + startangle=90, + pctdistance=0.75, wedgeprops={"linewidth": 2, "edgecolor": "white"}, ) for at in autotexts: @@ -205,10 +219,13 @@ def plot_distribution(counts: dict, out_path: str): at.set_fontweight("bold") axes[1].set_title("Class Proportion") - patches = [mpatches.Patch(color=c, label=f"{l.replace(chr(10), ' ')}: {v:,}") - for c, l, v in zip(colors, labels, values)] - axes[1].legend(handles=patches, loc="lower center", - bbox_to_anchor=(0.5, -0.12), ncol=2) + patches = [ + mpatches.Patch(color=c, label=f"{l.replace(chr(10), ' ')}: {v:,}") + for c, l, v in zip(colors, labels, values) + ] + axes[1].legend( + handles=patches, loc="lower center", bbox_to_anchor=(0.5, -0.12), ncol=2 + ) with warnings.catch_warnings(): warnings.simplefilter("ignore", UserWarning) @@ -261,8 +278,8 @@ def main(): chart_path, ) - print(f"\n✅ All outputs saved to ./{OUTPUT_DIR}/\n") + print(f"\n✅ All outputs saved to ./{OUTPUT_DIR}/\n") -if __name__ == "__main__": - main() \ No newline at end of file +if __name__ == "__main__": + main() diff --git a/dataset_standardize.py b/dataset_standardize.py new file mode 100644 index 0000000..4742247 --- /dev/null +++ b/dataset_standardize.py @@ -0,0 +1,424 @@ +""" +Dataset Standardization & Normalization Pipeline +================================================ + +Companion to `dataset_quality.py`. + +Where the quality script *audits* the dataset, this script *normalizes* it so every sample shares the same +resolution, file format, channel layout, mask encoding, and metadata / labeling conventions before training. + +It does two things (with the second one being optional): + +1. Manifest standardization (always runs, no images required) + - Re-derives the canonical `patient_id` from the image path. The + shipped `data_mask.csv` has a misaligned `patient_id` column + (the same id is repeated across slices belonging to different + patients), which silently breaks patient-grouped train/val/test + splits and causes data leakage. The folder name in `image_path` + is the ground-truth TCGA patient id, so we trust that. + - Parses a numeric `slice` index from each filename. + - Normalizes path separators and column ordering. + +2. Pixel standardization (runs only for rows whose files exist locally) + - Resizes every image to a single canonical resolution. + - Forces a single channel layout (3-channel RGB) and dtype (uint8). + - Resizes masks with nearest-neighbour interpolation and binarizes + them to {0, 255}, then re-derives the 0/1 `mask` label from the + actual mask content (catching mislabeled rows). + - Writes the standardized copies to `dataset_standardized/`, + mirroring the original folder layout, without touching the + originals. + +Outputs +------- + data_mask_standardized.csv canonical manifest (repo root) + dataset_standardized//...png standardized images + masks + dataset_reports/standardization_report.json run summary + +Run: + python dataset_standardize.py + +See PREPROCESSING.md for the full pipeline documentation. +""" + +import os +import re +import json +import warnings + +import numpy as np +import pandas as pd +import cv2 + +warnings.filterwarnings("ignore") + +# PATHS / PARAMS (all relative to project root) +CSV_PATH = "data_mask.csv" # source manifest in the repo root +IMG_ROOT = "./" # images live as ./TCGA_xxx/TCGA_xxx_1.tif +OUTPUT_DIR = "dataset_standardized" # standardized images + masks go here +REPORT_DIR = "dataset_reports" # reuse the existing reports folder +MANIFEST_PATH = os.path.join(os.getcwd(), OUTPUT_DIR, "data_mask_standardized.csv") + +TARGET_SIZE = ( + 256, + 256, +) # canonical resolution as (width, height) — matches model input +IMAGE_EXT = ".png" # canonical lossless on-disk format +MASK_THRESHOLD = 127 # >= this -> foreground when binarizing masks + +# canonical manifest columns, in order +MANIFEST_COLUMNS = [ + "patient_id", + "slice", + "image_path", + "mask_path", + "mask", + "original_mask", + "std_image_path", + "std_mask_path", +] + + +# HELPERS — pure, no I/O (easy to unit-test) +def _posix(path) -> str: + """Normalize Windows separators to POSIX so manifests are portable.""" + return str(path).replace("\\", "/").strip() + + +def _to_uint8(img: np.ndarray) -> np.ndarray: + """ + Min-max scale any numeric image to 8-bit [0, 255]. + Medical TIFs are frequently 16-bit; this gives a consistent dtype. + A flat image (max == min) maps to all-zeros. + """ + img = img.astype(np.float64) + mn, mx = float(img.min()), float(img.max()) + if mx > mn: + img = (img - mn) / (mx - mn) * 255.0 + else: + img = np.zeros_like(img) + return img.astype(np.uint8) + + +def derive_patient_id(image_path: str) -> str: + """ + Canonical TCGA patient id = the folder that contains the slice. + + `TCGA_CS_4941_19960909/TCGA_CS_4941_19960909_1.tif` -> `TCGA_CS_4941_19960909` + + Falls back to the first four underscore-separated tokens of the + filename if the path has no parent folder. + """ + parts = [p for p in _posix(image_path).split("/") if p] + if len(parts) >= 2: + return parts[-2] + stem = os.path.splitext(parts[-1])[0] if parts else "" + bits = stem.split("_") + return "_".join(bits[:4]) if len(bits) >= 4 else stem + + +def derive_slice_index(image_path: str): + """Parse the trailing slice number from a filename, e.g. `..._34.tif` -> 34.""" + stem = os.path.splitext(os.path.basename(_posix(image_path)))[0] + m = re.search(r"_(\d+)$", stem) + return int(m.group(1)) if m else None + + +def standardize_image(img: np.ndarray, target_size=TARGET_SIZE) -> np.ndarray: + """ + Normalize one image to canonical (H, W, 3) RGB uint8 at `target_size`. + + Channel handling mirrors app.py's inference preprocessing: + grayscale -> RGB, BGRA/RGBA -> RGB, BGR (cv2 default) -> RGB. + Returns a resized RGB uint8 array; pixel-value normalization + (/255 or mean-std) is left to model-feed time on purpose. + """ + if img is None: + raise ValueError("standardize_image received None") + + if img.ndim == 2: # grayscale + img = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB) + elif img.ndim == 3 and img.shape[2] == 1: # single-channel 3D + img = cv2.cvtColor(img[:, :, 0], cv2.COLOR_GRAY2RGB) + elif img.ndim == 3 and img.shape[2] == 4: # BGRA / RGBA + img = cv2.cvtColor(img, cv2.COLOR_BGRA2RGB) + elif img.ndim == 3 and img.shape[2] == 3: # cv2 loads BGR + img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) + else: + raise ValueError(f"unsupported image shape {img.shape}") + + if img.dtype != np.uint8: + img = _to_uint8(img) + + h, w = img.shape[:2] + # INTER_AREA is the better choice when shrinking; INTER_LINEAR when growing + shrinking = target_size[0] <= w and target_size[1] <= h + interp = cv2.INTER_AREA if shrinking else cv2.INTER_LINEAR + img = cv2.resize(img, (target_size[0], target_size[1]), interpolation=interp) + return img + + +def standardize_mask( + mask: np.ndarray, target_size=TARGET_SIZE, threshold=MASK_THRESHOLD +) -> np.ndarray: + """ + Normalize one mask to canonical (H, W) single-channel binary {0, 255}. + + Masks are resized with NEAREST interpolation (never bilinear — that + would invent intermediate label values along tumor boundaries) and + then thresholded so the encoding is identical for every sample. + """ + if mask is None: + raise ValueError("standardize_mask received None") + + if mask.ndim == 3: + mask = mask[:, :, 0] # collapse stray channels + if mask.dtype != np.uint8: + mask = _to_uint8(mask) + + mask = cv2.resize( + mask, (target_size[0], target_size[1]), interpolation=cv2.INTER_NEAREST + ) + _, binar = cv2.threshold(mask, threshold, 255, cv2.THRESH_BINARY) + return binar.astype(np.uint8) + + +def label_from_mask(mask: np.ndarray) -> int: + """Re-derive the 0/1 tumor label from a (standardized) mask's content.""" + return int(np.any(mask > 0)) + + +def _std_rel_path(path: str, ext=IMAGE_EXT) -> str: + """Map an original relative path to its standardized counterpart.""" + root, _ = os.path.splitext(_posix(path)) + return root + ext + + +# STEP 1 — Manifest standardization (no images required) +def standardize_manifest(df: pd.DataFrame): + """ + Normalize metadata + labeling conventions of the manifest itself. + Returns (standardized_df, summary_dict). + """ + for col in ("image_path", "mask_path"): + if col not in df.columns: + raise KeyError(f"source CSV missing required column: '{col}'") + + df = df.copy() + + # canonical patient_id from the image folder (fixes the misaligned column) + derived_pid = df["image_path"].map(derive_patient_id) + if "patient_id" in df.columns: + pid_corrected = int((df["patient_id"].astype(str) != derived_pid).sum()) + else: + pid_corrected = 0 + df["patient_id"] = derived_pid + + # numeric slice index + df["slice"] = df["image_path"].map(derive_slice_index).astype("Int64") + + # portable paths + df["image_path"] = df["image_path"].map(_posix) + df["mask_path"] = df["mask_path"].map(_posix) + + # keep the source label for traceability; `mask` becomes the canonical label + if "mask" in df.columns: + df["original_mask"] = pd.to_numeric(df["mask"], errors="coerce").astype("Int64") + df["mask"] = df["original_mask"] + else: + df["original_mask"] = pd.array([pd.NA] * len(df), dtype="Int64") + df["mask"] = pd.array([pd.NA] * len(df), dtype="Int64") + + # standardized-output columns (filled in step 2) + df["std_image_path"] = "" + df["std_mask_path"] = "" + + df = df[MANIFEST_COLUMNS] + + summary = { + "rows": int(len(df)), + "unique_patients": int(df["patient_id"].nunique()), + "patient_id_corrected": pid_corrected, + "slices_parsed": int(df["slice"].notna().sum()), + } + + print("\n── Manifest Standardization ────────────────────") + print(f" Rows : {summary['rows']:,}") + print(f" Unique patients : {summary['unique_patients']}") + print(f" patient_id corrected : {summary['patient_id_corrected']:,}") + print(f" Slice indices parsed : {summary['slices_parsed']:,}") + + return df, summary + + +# STEP 2 — Pixel standardization (needs the image files on disk) +def process_pixels( + df: pd.DataFrame, + img_root=IMG_ROOT, + out_dir=OUTPUT_DIR, + target_size=TARGET_SIZE, + mask_threshold=MASK_THRESHOLD, +): + """ + Standardize image + mask pixels for every row whose files exist. + Writes standardized copies under `out_dir` and returns + (updated_df, summary_dict). Missing-file rows are skipped and counted, + so this is safe to run with only the CSV present. + """ + os.makedirs(out_dir, exist_ok=True) + + shapes, dtypes = set(), set() + processed = missing = label_corrected = 0 + std_imgs, std_masks, labels = [], [], [] + + print("\n── Pixel Standardization ───────────────────────") + print( + f" Target resolution : {target_size[0]}x{target_size[1]} | format: {IMAGE_EXT} | channels: 3 | dtype: uint8" + ) + print(f" Scanning {len(df):,} rows …") + + for _, row in df.iterrows(): + img_full = os.path.join(img_root, row["image_path"]) + mask_full = os.path.join(img_root, row["mask_path"]) + + if not (os.path.isfile(img_full) and os.path.isfile(mask_full)): + missing += 1 + std_imgs.append("") + std_masks.append("") + labels.append(row["mask"]) + continue + + img = cv2.imread(img_full, cv2.IMREAD_UNCHANGED) + mask = cv2.imread(mask_full, cv2.IMREAD_UNCHANGED) + + std_img = standardize_image(img, target_size) + std_mask = standardize_mask(mask, target_size, mask_threshold) + label = label_from_mask(std_mask) + + rel_img = _std_rel_path(row["image_path"]) + rel_mask = _std_rel_path(row["mask_path"]) + out_img = os.path.join(out_dir, rel_img) + out_mask = os.path.join(out_dir, rel_mask) + os.makedirs(os.path.dirname(out_img), exist_ok=True) + os.makedirs(os.path.dirname(out_mask), exist_ok=True) + + # cv2 writes BGR; convert back so the file is a faithful RGB image + cv2.imwrite(out_img, cv2.cvtColor(std_img, cv2.COLOR_RGB2BGR)) + cv2.imwrite(out_mask, std_mask) + + shapes.add(tuple(std_img.shape)) + dtypes.add(str(std_img.dtype)) + if pd.notna(row["original_mask"]) and int(row["original_mask"]) != label: + label_corrected += 1 + + std_imgs.append(_posix(rel_img)) + std_masks.append(_posix(rel_mask)) + labels.append(label) + processed += 1 + + df = df.copy() + df["std_image_path"] = std_imgs + df["std_mask_path"] = std_masks + df["mask"] = pd.array( + [int(v) if pd.notna(v) else pd.NA for v in labels], dtype="Int64" + ) + + res = f"{target_size[0]}x{target_size[1]}" + consistency = { + "uniform_resolution": len(shapes) <= 1, + "uniform_channels": len(shapes) <= 1, + "uniform_dtype": len(dtypes) <= 1, + "resolution": res if processed else None, + "channels": ( + shapes.pop()[2] if (processed and shapes) else (3 if processed else None) + ), + "dtype": (next(iter(dtypes)) if dtypes else None), + } + + summary = { + "processed": processed, + "missing_files": missing, + "mask_label_corrected": label_corrected, + "output_format": IMAGE_EXT, + "mask_values": [0, 255], + "mask_interpolation": "nearest", + "consistency_check": consistency, + } + + print(f" Standardized rows : {processed:,}") + print(f" Missing-file rows : {missing:,}") + print(f" Mask labels corrected : {label_corrected:,}") + if processed: + print( + f" Consistency : resolution={consistency['uniform_resolution']} " + f"channels={consistency['uniform_channels']} dtype={consistency['uniform_dtype']}" + ) + else: + print( + " (no image files found locally — pixel step skipped, manifest still standardized)" + ) + + return df, summary + + +# MAIN +def main(): + os.makedirs(REPORT_DIR, exist_ok=True) + + print(f"\nLoading {CSV_PATH} …") + df = pd.read_csv(CSV_PATH) + + # Step 1 — manifest + manifest, manifest_summary = standardize_manifest(df) + + # Step 2 — pixels + manifest, pixel_summary = process_pixels(manifest) + + # write the canonical manifest + manifest.to_csv(MANIFEST_PATH, index=False) + print(f"\n Manifest generated at -> {MANIFEST_PATH} ({len(manifest):,} rows)") + + # assemble + write the JSON report + report = { + "source_csv": CSV_PATH, + "manifest": manifest_summary, + "images": { + "target_resolution": f"{TARGET_SIZE[0]}x{TARGET_SIZE[1]}", + "output_format": IMAGE_EXT, + "channels": 3, + "dtype": "uint8", + "processed": pixel_summary["processed"], + "missing_files": pixel_summary["missing_files"], + "consistency_check": pixel_summary["consistency_check"], + }, + "masks": { + "values": pixel_summary["mask_values"], + "interpolation": pixel_summary["mask_interpolation"], + "threshold": MASK_THRESHOLD, + "label_corrected": pixel_summary["mask_label_corrected"], + }, + "outputs": { + "manifest_csv": MANIFEST_PATH, + "standardized_dir": OUTPUT_DIR, + "report": os.path.join(REPORT_DIR, "standardization_report.json"), + }, + "acceptance_criteria": { + "consistent_resolution_and_format": bool( + pixel_summary["consistency_check"]["uniform_resolution"] + and pixel_summary["consistency_check"]["uniform_dtype"] + ), + "masks_standardized": True, + "pipeline_documented": "PREPROCESSING.md", + }, + } + + report_path = os.path.join(REPORT_DIR, "standardization_report.json") + with open(report_path, "w") as f: + json.dump(report, f, indent=2) + print(f" Report generated at -> {report_path}") + + print(f"\n Standardization complete. Manifest: {MANIFEST_PATH}\n") + + +if __name__ == "__main__": + main() diff --git a/tests/unit/test_dataset_standardize.py b/tests/unit/test_dataset_standardize.py new file mode 100644 index 0000000..8782fe0 --- /dev/null +++ b/tests/unit/test_dataset_standardize.py @@ -0,0 +1,212 @@ +import numpy as np +import pandas as pd +import cv2 + +import dataset_standardize as ds + + +# --- metadata / labeling helpers ---------------------------------------- +def test_derive_patient_id_uses_folder(): + path = "TCGA_CS_4941_19960909/TCGA_CS_4941_19960909_1.tif" + assert ds.derive_patient_id(path) == "TCGA_CS_4941_19960909" + # windows separators are handled too + assert ( + ds.derive_patient_id("TCGA_DU_5849_19950405\\slice_3.tif") + == "TCGA_DU_5849_19950405" + ) + + +def test_derive_slice_index(): + assert ( + ds.derive_slice_index("TCGA_CS_4941_19960909/TCGA_CS_4941_19960909_34.tif") + == 34 + ) + assert ds.derive_slice_index("foo/bar_baz.tif") is None + + +# --- image standardization ---------------------------------------------- +def test_standardize_image_grayscale_to_rgb(): + gray = np.full((40, 60), 128, dtype=np.uint8) + out = ds.standardize_image(gray, target_size=(256, 256)) + + assert out.shape == (256, 256, 3) + assert out.dtype == np.uint8 + + +def test_standardize_image_rgba_and_odd_size(): + rgba = np.zeros((100, 73, 4), dtype=np.uint8) + out = ds.standardize_image(rgba, target_size=(128, 128)) + + assert out.shape == (128, 128, 3) + assert out.dtype == np.uint8 + + +def test_standardize_image_uint16_is_scaled_to_uint8(): + img16 = (np.arange(32 * 32, dtype=np.uint16) % 4096).reshape(32, 32) + out = ds.standardize_image(img16, target_size=(32, 32)) + + assert out.dtype == np.uint8 + assert out.max() <= 255 + + +# --- mask standardization ----------------------------------------------- +def test_standardize_mask_is_binary_single_channel(): + mask = np.zeros((50, 50), dtype=np.uint8) + mask[10:20, 10:20] = 200 # a tumor blob + + out = ds.standardize_mask(mask, target_size=(64, 64)) + + assert out.ndim == 2 + assert out.shape == (64, 64) + # only {0, 255} may appear — nearest-neighbour resize never invents values + assert set(np.unique(out)).issubset({0, 255}) + assert out.max() == 255 + + +def test_standardize_mask_nearest_neighbour_no_intermediate_values(): + # a thin diagonal would smear into gray values under bilinear resize + mask = np.zeros((48, 48), dtype=np.uint8) + np.fill_diagonal(mask, 255) + + out = ds.standardize_mask(mask, target_size=(16, 16)) + + assert set(np.unique(out)).issubset({0, 255}) + + +def test_label_from_mask(): + empty = np.zeros((8, 8), dtype=np.uint8) + tumor = empty.copy() + tumor[3, 3] = 255 + + assert ds.label_from_mask(empty) == 0 + assert ds.label_from_mask(tumor) == 1 + + +# --- manifest standardization ------------------------------------------- +def test_standardize_manifest_fixes_patient_id_and_columns(): + df = pd.DataFrame( + { + # deliberately wrong / constant patient_id, like the shipped CSV + "patient_id": ["WRONG", "WRONG", "WRONG"], + "image_path": [ + "TCGA_CS_4941_19960909/TCGA_CS_4941_19960909_1.tif", + "TCGA_CS_4942_19970222/TCGA_CS_4942_19970222_2.tif", + "TCGA_DU_5849_19950405/TCGA_DU_5849_19950405_5.tif", + ], + "mask_path": [ + "TCGA_CS_4941_19960909/TCGA_CS_4941_19960909_1_mask.tif", + "TCGA_CS_4942_19970222/TCGA_CS_4942_19970222_2_mask.tif", + "TCGA_DU_5849_19950405/TCGA_DU_5849_19950405_5_mask.tif", + ], + "mask": [0, 1, 0], + } + ) + + out, summary = ds.standardize_manifest(df) + + assert list(out.columns) == ds.MANIFEST_COLUMNS + assert out["patient_id"].tolist() == [ + "TCGA_CS_4941_19960909", + "TCGA_CS_4942_19970222", + "TCGA_DU_5849_19950405", + ] + assert out["slice"].tolist() == [1, 2, 5] + assert summary["patient_id_corrected"] == 3 + assert summary["unique_patients"] == 3 + # original label preserved for traceability + assert out["original_mask"].tolist() == [0, 1, 0] + + +def test_standardize_manifest_missing_column_raises(): + df = pd.DataFrame({"image_path": ["a/b_1.tif"]}) # no mask_path + try: + ds.standardize_manifest(df) + assert False, "expected KeyError" + except KeyError: + pass + + +# --- end-to-end pixel processing ---------------------------------------- +def _write_pair(root, patient, idx, mask_value): + folder = root / patient + folder.mkdir(parents=True, exist_ok=True) + img_rel = f"{patient}/{patient}_{idx}.tif" + mask_rel = f"{patient}/{patient}_{idx}_mask.tif" + + rng = np.random.default_rng(idx) + cv2.imwrite(str(root / img_rel), (rng.random((90, 110)) * 255).astype(np.uint8)) + + mask = np.zeros((90, 110), dtype=np.uint8) + if mask_value: + mask[20:40, 20:40] = 255 + cv2.imwrite(str(root / mask_rel), mask) + return img_rel, mask_rel + + +def test_process_pixels_end_to_end(tmp_path): + img1, mask1 = _write_pair(tmp_path, "TCGA_CS_4941_19960909", 1, mask_value=True) + img2, mask2 = _write_pair(tmp_path, "TCGA_CS_4942_19970222", 2, mask_value=False) + + df = pd.DataFrame( + { + "patient_id": ["x", "x"], + "image_path": [img1, img2], + "mask_path": [mask1, mask2], + # original labels are BOTH wrong on purpose -> must be corrected + "mask": [0, 1], + } + ) + manifest, _ = ds.standardize_manifest(df) + + out_dir = tmp_path / "std" + manifest, summary = ds.process_pixels( + manifest, + img_root=str(tmp_path), + out_dir=str(out_dir), + target_size=(128, 128), + ) + + # both rows processed, none missing + assert summary["processed"] == 2 + assert summary["missing_files"] == 0 + + # labels re-derived from mask content + assert manifest["mask"].tolist() == [1, 0] + assert summary["mask_label_corrected"] == 2 + + # standardized files actually written, at the canonical resolution + for rel in manifest["std_image_path"]: + f = out_dir / rel + assert f.exists() + assert cv2.imread(str(f)).shape == (128, 128, 3) + + # masks on disk are binary {0,255} + for rel in manifest["std_mask_path"]: + m = cv2.imread(str(out_dir / rel), cv2.IMREAD_GRAYSCALE) + assert set(np.unique(m)).issubset({0, 255}) + + # consistency check passes + cc = summary["consistency_check"] + assert cc["uniform_resolution"] and cc["uniform_dtype"] + assert cc["resolution"] == "128x128" + + +def test_process_pixels_handles_missing_files(tmp_path): + df = pd.DataFrame( + { + "patient_id": ["x"], + "image_path": ["TCGA_CS_4941_19960909/TCGA_CS_4941_19960909_1.tif"], + "mask_path": ["TCGA_CS_4941_19960909/TCGA_CS_4941_19960909_1_mask.tif"], + "mask": [0], + } + ) + manifest, _ = ds.standardize_manifest(df) + + manifest, summary = ds.process_pixels( + manifest, img_root=str(tmp_path), out_dir=str(tmp_path / "std") + ) + + # nothing on disk -> skipped gracefully, manifest still intact + assert summary["processed"] == 0 + assert summary["missing_files"] == 1 + assert manifest["std_image_path"].tolist() == [""]