diff --git a/DashAI/alembic/versions/b7e4d2a19c63_merge_reports_and_develop_heads.py b/DashAI/alembic/versions/b7e4d2a19c63_merge_reports_and_develop_heads.py new file mode 100644 index 000000000..b864ac619 --- /dev/null +++ b/DashAI/alembic/versions/b7e4d2a19c63_merge_reports_and_develop_heads.py @@ -0,0 +1,23 @@ +"""merge reports and develop heads + +Revision ID: b7e4d2a19c63 +Revises: a5f2c71e9d40, e6c3b91a7d48 +Create Date: 2026-09-07 12:00:00.000000 + +""" + +from typing import Sequence, Union + +# revision identifiers, used by Alembic. +revision: str = "b7e4d2a19c63" +down_revision: Union[str, Sequence[str], None] = ("a5f2c71e9d40", "e6c3b91a7d48") +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + pass + + +def downgrade() -> None: + pass diff --git a/DashAI/alembic/versions/e6c3b91a7d48_add_report_table.py b/DashAI/alembic/versions/e6c3b91a7d48_add_report_table.py new file mode 100644 index 000000000..84f5a82fb --- /dev/null +++ b/DashAI/alembic/versions/e6c3b91a7d48_add_report_table.py @@ -0,0 +1,57 @@ +"""add report table + +Revision ID: e6c3b91a7d48 +Revises: c4e8a1d20f3b +Create Date: 2026-07-30 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "e6c3b91a7d48" +down_revision: Union[str, None] = "c4e8a1d20f3b" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Create the table holding evaluation reports of a run.""" + op.create_table( + "report", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("run_id", sa.Integer(), nullable=False), + sa.Column("huey_id", sa.String(), nullable=True), + sa.Column("report_name", sa.String(), nullable=False), + sa.Column("parameters", sa.JSON(), nullable=True), + sa.Column("artifacts_path", sa.String(), nullable=True), + sa.Column("plot_overrides", sa.JSON(), nullable=True), + sa.Column("created", sa.DateTime(), nullable=True), + sa.Column( + "status", + sa.Enum( + "NOT_STARTED", + "DELIVERED", + "STARTED", + "FINISHED", + "ERROR", + name="reportstatus", + ), + nullable=False, + ), + sa.ForeignKeyConstraint( + ["run_id"], + ["run.id"], + name=op.f("fk_report_run_id_run"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_report")), + ) + + +def downgrade() -> None: + """Drop the report table.""" + op.drop_table("report") diff --git a/DashAI/back/api/api_v1/api.py b/DashAI/back/api/api_v1/api.py index 3ae6829b3..b2642d59a 100644 --- a/DashAI/back/api/api_v1/api.py +++ b/DashAI/back/api/api_v1/api.py @@ -26,6 +26,7 @@ from DashAI.back.api.api_v1.endpoints.predict import router as predict from DashAI.back.api.api_v1.endpoints.prompts import router as prompts from DashAI.back.api.api_v1.endpoints.rag import router as rag +from DashAI.back.api.api_v1.endpoints.reports import router as reports from DashAI.back.api.api_v1.endpoints.runs import router as runs from DashAI.back.api.api_v1.endpoints.statistical_tests import ( router as statistical_tests, @@ -38,6 +39,7 @@ api_router_v1.include_router(documents, prefix="/document") api_router_v1.include_router(model_sessions, prefix="/model-session") api_router_v1.include_router(explainers, prefix="/explainer") +api_router_v1.include_router(reports, prefix="/report") api_router_v1.include_router(explorers, prefix="/explorer") api_router_v1.include_router(jobs, prefix="/job") api_router_v1.include_router(runs, prefix="/run") diff --git a/DashAI/back/api/api_v1/endpoints/explainers.py b/DashAI/back/api/api_v1/endpoints/explainers.py old mode 100644 new mode 100755 diff --git a/DashAI/back/api/api_v1/endpoints/reports.py b/DashAI/back/api/api_v1/endpoints/reports.py new file mode 100644 index 000000000..655f00e67 --- /dev/null +++ b/DashAI/back/api/api_v1/endpoints/reports.py @@ -0,0 +1,316 @@ +"""REST endpoints for evaluation reports.""" + +import logging +from typing import TYPE_CHECKING + +from fastapi import APIRouter, Depends, status +from fastapi.exceptions import HTTPException +from kink import di, inject +from sqlalchemy import exc, select + +from DashAI.back.api.api_v1.schemas.reports_params import ReportParams +from DashAI.back.core.artifacts import ( + PlotOverrideBody, + apply_plot_overrides, + normalize_artifacts, +) +from DashAI.back.dependencies.database.models import Report, Run + +if TYPE_CHECKING: + from sqlalchemy.orm import sessionmaker + +logging.basicConfig(level=logging.DEBUG) +log = logging.getLogger(__name__) + +router = APIRouter() + + +@router.get("/") +@inject +async def get_reports( + run_id: int = None, + session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), +): + """Return the reports stored for a run. + + Parameters + ---------- + run_id : int, optional + Run whose reports are requested. All reports when omitted. + session_factory : Callable[..., ContextManager[Session]] + A factory that creates a context manager that handles a SQLAlchemy + session. + + Returns + ------- + List[Report] + The matching report rows. + + Raises + ------ + HTTPException + If the database cannot be read. + """ + with session_factory() as db: + try: + statement = select(Report) + if run_id is not None: + statement = statement.where(Report.run_id == run_id) + return db.scalars(statement).all() + except exc.SQLAlchemyError as e: + log.exception(e) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Internal database error", + ) from e + + +@router.get("/{report_id}/artifacts") +@inject +async def get_report_artifacts( + report_id: int, + session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), +): + """Return the computed artifacts of a report. + + Parameters + ---------- + report_id : int + Id of the report whose artifacts are requested. + session_factory : Callable[..., ContextManager[Session]] + A factory that creates a context manager that handles a SQLAlchemy + session. + + Returns + ------- + List[dict] + Artifact wire dicts, empty when the report has not run yet. + + Raises + ------ + HTTPException + If the report does not exist or its file cannot be read. + """ + import pickle + + with session_factory() as db: + try: + report = db.get(Report, report_id) + except exc.SQLAlchemyError as e: + log.exception(e) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Internal database error", + ) from e + + if not report: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Report not found", + ) + + if not report.artifacts_path: + return [] + + try: + with open(report.artifacts_path, "rb") as file: + stored = pickle.load(file) + except OSError as e: + log.exception(e) + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Report artifacts file not found", + ) from e + + plot_overrides = report.plot_overrides + + # Re-normalized on read for the same reason the explainer plot endpoints + # do it: artifacts pickled by an older version still come back current. + # Overrides are applied last so a user's saved edits win over the computed + # figure and survive a reload. + return apply_plot_overrides(normalize_artifacts(stored), plot_overrides) + + +@router.post("/", status_code=status.HTTP_201_CREATED) +@inject +async def upload_report( + params: ReportParams, + session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), +): + """Create a report row for a run. + + Parameters + ---------- + params : ReportParams + Run id, report component name and parameters. + session_factory : Callable[..., ContextManager[Session]] + A factory that creates a context manager that handles a SQLAlchemy + session. + + Returns + ------- + Report + The created row. + + Raises + ------ + HTTPException + If the run does not exist or the row cannot be stored. + """ + with session_factory() as db: + try: + run: Run = db.get(Run, params.run_id) + if not run: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Run not found" + ) + + report = Report( + run_id=params.run_id, + report_name=params.report_name, + parameters=params.parameters, + ) + db.add(report) + db.commit() + db.refresh(report) + return report + + except exc.SQLAlchemyError as e: + log.exception(e) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Internal database error", + ) from e + + +@router.put("/{report_id}/override") +@inject +async def save_plot_override( + report_id: int, + body: PlotOverrideBody, + session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), +): + """Persist an edited plotly figure for one artifact of a report. + + Parameters + ---------- + report_id : int + Id of the report whose plot is being edited. + body : PlotOverrideBody + The artifact index and the edited plotly figure. + session_factory : Callable[..., ContextManager[Session]] + Factory yielding a SQLAlchemy session. + + Returns + ------- + dict + ``{"status": "ok"}`` on success. + + Raises + ------ + HTTPException + If the report does not exist. + """ + import json + + with session_factory() as db: + report = db.get(Report, report_id) + if report is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Report not found" + ) + overrides = dict(report.plot_overrides or {}) + figure = body.figure + overrides[str(body.index)] = ( + figure if isinstance(figure, str) else json.dumps(figure) + ) + report.plot_overrides = overrides + db.commit() + return {"status": "ok"} + + +@router.delete("/{report_id}/override/{index}") +@inject +async def delete_plot_override( + report_id: int, + index: int, + session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), +): + """Remove a stored plot override, reverting to the computed figure. + + Parameters + ---------- + report_id : int + Id of the report. + index : int + Artifact index whose override is removed. + session_factory : Callable[..., ContextManager[Session]] + Factory yielding a SQLAlchemy session. + + Returns + ------- + dict + ``{"status": "ok"}``. + + Raises + ------ + HTTPException + If the report does not exist. + """ + with session_factory() as db: + report = db.get(Report, report_id) + if report is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Report not found" + ) + overrides = dict(report.plot_overrides or {}) + overrides.pop(str(index), None) + report.plot_overrides = overrides or None + db.commit() + return {"status": "ok"} + + +@router.delete("/{report_id}") +@inject +async def delete_report( + report_id: int, + session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), +): + """Delete a report and its stored artifacts. + + Parameters + ---------- + report_id : int + Id of the report to delete. + session_factory : Callable[..., ContextManager[Session]] + A factory that creates a context manager that handles a SQLAlchemy + session. + + Raises + ------ + HTTPException + If the report does not exist or the deletion fails. + """ + import os + + with session_factory() as db: + try: + report = db.get(Report, report_id) + if not report: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Report not found", + ) + + if report.artifacts_path and os.path.exists(report.artifacts_path): + os.remove(report.artifacts_path) + + db.delete(report) + db.commit() + + except exc.SQLAlchemyError as e: + log.exception(e) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Internal database error", + ) from e diff --git a/DashAI/back/api/api_v1/endpoints/runs.py b/DashAI/back/api/api_v1/endpoints/runs.py index 18c54ae04..fa847f3e8 100644 --- a/DashAI/back/api/api_v1/endpoints/runs.py +++ b/DashAI/back/api/api_v1/endpoints/runs.py @@ -16,6 +16,7 @@ Metric, ModelSession, Prediction, + Report, Run, RunStatus, ) @@ -575,9 +576,13 @@ async def get_run_operations_count( db.query(Prediction).filter(Prediction.run_id == run_id).count() ) + # Count reports + reports_count = db.query(Report).filter(Report.run_id == run_id).count() + return { "explainers": global_explainers_count + local_explainers_count, "predictions": predictions_count, + "reports": reports_count, } except exc.SQLAlchemyError as e: log.exception(e) @@ -627,8 +632,21 @@ async def delete_run_operations( "global_explainers": 0, "local_explainers": 0, "predictions": 0, + "reports": 0, } + # Delete reports: they describe the predictions of the fit + # being replaced, so a retrain must not leave them behind. + reports = db.query(Report).filter(Report.run_id == run_id).all() + for report in reports: + if report.artifacts_path and os.path.exists(report.artifacts_path): + try: + remove_path(report.artifacts_path) + except Exception as e: + log.warning(f"Failed to delete report file: {e}") + db.delete(report) + deleted_count["reports"] += 1 + # Delete global explainers global_explainers = ( db.query(GlobalExplainer).filter(GlobalExplainer.run_id == run_id).all() diff --git a/DashAI/back/api/api_v1/schemas/job_params.py b/DashAI/back/api/api_v1/schemas/job_params.py index 816e0f22e..1b4dee218 100644 --- a/DashAI/back/api/api_v1/schemas/job_params.py +++ b/DashAI/back/api/api_v1/schemas/job_params.py @@ -9,6 +9,7 @@ class JobParams(BaseModel): job_type: Literal[ "ModelJob", "ExplainerJob", + "ReportJob", "PredictJob", "DatasetJob", "ExplorerJob", diff --git a/DashAI/back/api/api_v1/schemas/reports_params.py b/DashAI/back/api/api_v1/schemas/reports_params.py new file mode 100644 index 000000000..40e436a11 --- /dev/null +++ b/DashAI/back/api/api_v1/schemas/reports_params.py @@ -0,0 +1,14 @@ +from pydantic import BaseModel + + +class ReportParams(BaseModel): + """Body of a report creation request. + + A report covers every evaluation partition of the run, so it carries + neither a split nor a user supplied name: its component and its run + identify it. + """ + + run_id: int + report_name: str + parameters: dict = {} diff --git a/DashAI/back/core/enums/status.py b/DashAI/back/core/enums/status.py index a21973878..20ec6f22e 100644 --- a/DashAI/back/core/enums/status.py +++ b/DashAI/back/core/enums/status.py @@ -9,6 +9,14 @@ class ExplainerStatus(Enum): ERROR = 4 +class ReportStatus(Enum): + NOT_STARTED = 0 + DELIVERED = 1 + STARTED = 2 + FINISHED = 3 + ERROR = 4 + + class RunStatus(Enum): NOT_STARTED = 0 DELIVERED = 1 diff --git a/DashAI/back/dependencies/database/models.py b/DashAI/back/dependencies/database/models.py index fa7b2fe6e..1b41d8d5e 100644 --- a/DashAI/back/dependencies/database/models.py +++ b/DashAI/back/dependencies/database/models.py @@ -30,6 +30,7 @@ ExplorerStatus, PluginStatus, PredictionStatus, + ReportStatus, RunStatus, ) @@ -384,6 +385,45 @@ def set_status_as_error(self) -> None: self.status = ExplainerStatus.ERROR +class Report(Base): + __tablename__ = "report" + """ + Table to store an evaluation report of a run. One row covers every + evaluation partition; the artifacts carry one group per partition. + """ + id: Mapped[int] = mapped_column(primary_key=True) + run_id: Mapped[int] = mapped_column( + ForeignKey("run.id", ondelete="CASCADE"), nullable=False + ) + huey_id: Mapped[str] = mapped_column(String, nullable=True) + report_name: Mapped[str] = mapped_column(String, nullable=False) + parameters: Mapped[JSON] = mapped_column(JSON, nullable=True) + artifacts_path: Mapped[str] = mapped_column(String, nullable=True) + plot_overrides: Mapped[JSON] = mapped_column(JSON, nullable=True) + created: Mapped[DateTime] = mapped_column( + DateTime, default=datetime.now, nullable=True + ) + status: Mapped[Enum] = mapped_column( + Enum(ReportStatus), nullable=False, default=ReportStatus.NOT_STARTED + ) + + def set_status_as_delivered(self) -> None: + """Update the status of the report to delivered.""" + self.status = ReportStatus.DELIVERED + + def set_status_as_started(self) -> None: + """Update the status of the report to started.""" + self.status = ReportStatus.STARTED + + def set_status_as_finished(self) -> None: + """Update the status of the report to finished.""" + self.status = ReportStatus.FINISHED + + def set_status_as_error(self) -> None: + """Update the status of the report to error.""" + self.status = ReportStatus.ERROR + + class LocalExplainer(Base): __tablename__ = "local_explainer" """ diff --git a/DashAI/back/initial_components.py b/DashAI/back/initial_components.py index 27bc9ed23..d2ed352c0 100644 --- a/DashAI/back/initial_components.py +++ b/DashAI/back/initial_components.py @@ -163,6 +163,7 @@ from DashAI.back.job.pipeline_job import PipelineJob from DashAI.back.job.predict_job import PredictJob from DashAI.back.job.RAG_job import RAGJob +from DashAI.back.job.report_job import ReportJob # Metrics from DashAI.back.metrics.classification.accuracy import Accuracy @@ -426,6 +427,31 @@ # Plugins from DashAI.back.plugins.utils import get_available_plugins +from DashAI.back.reports.classification.confusion_matrix import ConfusionMatrix + +# Reports +from DashAI.back.reports.classification.per_class_breakdown import ( + PerClassBreakdown, +) +from DashAI.back.reports.classification.precision_recall_curve import ( + PrecisionRecallCurve, +) +from DashAI.back.reports.classification.roc_curve import RocCurve +from DashAI.back.reports.forecasting.forecast_vs_actual import ForecastVsActual +from DashAI.back.reports.forecasting.residual_autocorrelation import ( + ResidualAutocorrelation, +) +from DashAI.back.reports.forecasting.residuals_over_time import ResidualsOverTime +from DashAI.back.reports.regression.predicted_vs_actual import PredictedVsActual +from DashAI.back.reports.regression.residual_histogram import ResidualHistogram +from DashAI.back.reports.regression.residual_plot import ResidualPlot +from DashAI.back.reports.translation.length_comparison import LengthComparison +from DashAI.back.reports.translation.per_segment_comparison import ( + PerSegmentComparison, +) +from DashAI.back.reports.translation.segment_score_distribution import ( + SegmentScoreDistribution, +) from DashAI.back.splitters.group_k_fold import GroupKFoldSplitter # Splitters @@ -640,6 +666,7 @@ def get_initial_components(): DatafileJob, ExplainerJob, ModelJob, + ReportJob, ExplorerJob, PredictJob, ConverterJob, @@ -647,6 +674,20 @@ def get_initial_components(): GenerativeJob, PipelineJob, RAGJob, + # Reports + ConfusionMatrix, + RocCurve, + PrecisionRecallCurve, + PerClassBreakdown, + PredictedVsActual, + ResidualPlot, + ResidualHistogram, + ForecastVsActual, + ResidualsOverTime, + ResidualAutocorrelation, + PerSegmentComparison, + SegmentScoreDistribution, + LengthComparison, # Explainers ContrastiveShap, DiceCounterfactual, diff --git a/DashAI/back/job/report_job.py b/DashAI/back/job/report_job.py new file mode 100644 index 000000000..7d2c05e94 --- /dev/null +++ b/DashAI/back/job/report_job.py @@ -0,0 +1,409 @@ +"""Job that turns a run's predictions into one report per evaluation partition. + +A report covers every partition the run exposes rather than one chosen at +creation, and which partitions those are is decided by the splitter that +produced the run rather than by this module: a holdout run yields train, test +and validation, while a cross validated one yields the rows it reserved as a +test set and the rest the final model was refit on. A task that predicts +forward only, such as forecasting, can never be scored on the rows its model +was fitted on, so those partitions are dropped exactly as the prediction flow +drops them: a report must not ask a model for a value it refuses to give. The +set is read through the same helpers the prediction and local explainer flows +use, so a report can never cover a different set than the one the user was +offered. No report class changes to support a new splitter, because none of +them know what a partition is. +""" + +import logging +from typing import TYPE_CHECKING, List, Optional + +from kink import inject +from sqlalchemy import exc + +from DashAI.back.dependencies.database.models import ( + Dataset, + ModelSession, + Report, + Run, +) +from DashAI.back.job.base_job import BaseJob, JobError +from DashAI.back.models.base_model import BaseModel +from DashAI.back.splitters.splits_payload import predictable_splits, run_split_indexes + +if TYPE_CHECKING: + from sqlalchemy.orm import sessionmaker + +logging.basicConfig(level=logging.DEBUG) +log = logging.getLogger(__name__) + + +PARTITION_LABELS = { + "train": "Train", + "test": "Test", + "val": "Validation", + "validation": "Validation", + "all": "Whole dataset", +} + + +def partition_label(name: str) -> str: + """Build the selector entry title for one partition. + + Names outside :data:`PARTITION_LABELS` fall back to their own titled form, + so a splitter added later needs no change here. + + Parameters + ---------- + name : str + Partition name as reported by the run's splitter. + + Returns + ------- + str + The mapped label, or the name in title case when it is not mapped. + """ + return PARTITION_LABELS.get(name, name.replace("_", " ").title()) + + +class ReportJob(BaseJob): + """Compute one evaluation report for a run over one split. + + Rebuilds the requested split, predicts with the trained model, and hands + the truth and the predictions to the report. The model's inputs are + never passed on: a report compares predictions against the truth and + nothing else, which is what separates it from an explainer. + """ + + @inject + def set_status_as_delivered( + self, session_factory: "sessionmaker" = lambda di: di["session_factory"] + ) -> None: + """Mark the report as queued. + + Parameters + ---------- + session_factory : sessionmaker + Factory producing a SQLAlchemy session. + + Raises + ------ + JobError + If the report does not exist or the database rejects the update. + """ + report_id: int = self.kwargs["report_id"] + with session_factory() as db: + report: Report = db.get(Report, report_id) + if not report: + raise JobError(f"Report with id {report_id} does not exist in DB.") + try: + report.set_status_as_delivered() + db.commit() + except exc.SQLAlchemyError as e: + log.exception(e) + raise JobError("Internal database error") from e + + @inject + def set_status_as_error( + self, session_factory: "sessionmaker" = lambda di: di["session_factory"] + ) -> None: + """Mark the report as failed. + + Parameters + ---------- + session_factory : sessionmaker + Factory producing a SQLAlchemy session. + """ + report_id: Optional[int] = self.kwargs.get("report_id") + if report_id is None: + return + with session_factory() as db: + try: + report: Report = db.get(Report, report_id) + if report: + report.set_status_as_error() + db.commit() + except exc.SQLAlchemyError as e: + log.exception(e) + + def get_job_name(self) -> str: + """Build a descriptive name for this job. + + Returns + ------- + str + The report's name when it can be read, a generic label + otherwise. + """ + report_id = self.kwargs.get("report_id") + if not report_id: + return "Report" + + from kink import di + + try: + with di["session_factory"]() as db: + report: Report = db.get(Report, report_id) + if report: + return f"Report: {report.report_name}" + except Exception: + pass + return f"Report ({report_id})" + + @staticmethod + def _class_names(model: BaseModel) -> Optional[List[str]]: + """Recover the human readable class labels of a fitted classifier. + + Parameters + ---------- + model : BaseModel + The trained model, which may carry output encodings. + + Returns + ------- + Optional[List[str]] + Class labels in encoded order, or None for a regressor. + """ + encodings = getattr(model, "output_encodings", None) + mapping = next(iter(encodings.values()), None) if encodings else None + if mapping: + return [ + str(label) for label, _ in sorted(mapping.items(), key=lambda p: p[1]) + ] + classes = getattr(model, "classes_", None) + if classes is None: + return None + return [str(label) for label in classes] + + @inject + def run(self) -> None: + """Compute and persist the report's artifacts. + + The dataset is prepared once over every row and then indexed per + partition, because the row indexes a splitter reports are indexes into + the dataset as stored. Inputs reach the model unprepared, exactly as + the prediction job feeds them, since the model applies its own + preprocessing; targets are encoded so they line up with the class + indexes the model predicts. + + A partition that the report cannot describe, such as one holding a + single class, is skipped and named in the saved output rather than + costing the user the partitions that did compute. The job only fails + when no partition produced anything. + + Raises + ------ + JobError + If any stage of the reconstruction or computation fails. + """ + import os + import pickle + + from kink import di + + from DashAI.back.core.artifacts import ( + ArtifactGroup, + GroupedArtifacts, + TextArtifact, + normalize_artifacts, + ) + from DashAI.back.dataloaders.classes.dashai_dataset import ( + load_dataset, + select_columns, + ) + from DashAI.back.reports.base_report import ReportError + from DashAI.back.tasks.base_task import BaseTask + + component_registry = di["component_registry"] + session_factory = di["session_factory"] + config = di["config"] + report_id: int = self.kwargs["report_id"] + + with session_factory() as db: + report: Report = db.get(Report, report_id) + if not report: + raise JobError(f"Report with id {report_id} does not exist in DB.") + + try: + run: Run = db.get(Run, report.run_id) + if not run: + raise JobError(f"Run {report.run_id} does not exist in DB.") + model_session: ModelSession = db.get(ModelSession, run.model_session_id) + if not model_session: + raise JobError( + f"Model session {run.model_session_id} does not exist in DB." + ) + dataset: Dataset = db.get(Dataset, model_session.dataset_id) + if not dataset: + raise JobError( + f"Dataset {model_session.dataset_id} does not exist in DB." + ) + + try: + report.set_status_as_started() + db.commit() + except exc.SQLAlchemyError as e: + log.exception(e) + raise JobError("Connection with the database failed") from e + + try: + model_class = component_registry[run.model_name]["class"] + model: BaseModel = model_class(**run.parameters) + trained_model = model.load(run.run_path) + except Exception as e: + log.exception(e) + raise JobError( + f"Can not load model {run.model_name} from {run.run_path}" + ) from e + + try: + report_class = component_registry[report.report_name]["class"] + instance = report_class(**(report.parameters or {})) + except Exception as e: + log.exception(e) + raise JobError( + f"Unable to instantiate report {report.report_name}." + ) from e + + self.report_progress(0.3, "Resolving the run's partitions") + try: + splits = predictable_splits( + model_session.splits, + run.split_indexes, + component_registry, + task_name=model_session.task_name, + evaluation_strategy=model_session.evaluation_strategy, + ) + except ValueError as e: + log.exception(e) + raise JobError(str(e)) from e + + if not splits: + raise JobError( + "The run has no partition a report can be computed on: " + "every row went into fitting the model." + ) + + try: + partition_indexes = { + split["name"]: run_split_indexes( + model_session.splits, + run.split_indexes, + component_registry, + split["name"], + ) + for split in splits + } + except ValueError as e: + log.exception(e) + raise JobError(str(e)) from e + + try: + loaded_dataset = load_dataset(f"{dataset.file_path}/dataset") + task: BaseTask = component_registry[model_session.task_name][ + "class" + ]() + prepared_dataset = task.prepare_for_task( + dataset=loaded_dataset, + input_columns=model_session.input_columns, + output_columns=model_session.output_columns, + ) + data_x, data_y = select_columns( + prepared_dataset, + model_session.input_columns, + model_session.output_columns, + ) + except Exception as e: + log.exception(e) + raise JobError( + f"Can not prepare dataset {dataset.id} for the report" + ) from e + + self.report_progress(0.6, "Computing every partition") + class_names = self._class_names(trained_model) + groups = [] + skipped = [] + last_error = None + + for partition, row_indexes in partition_indexes.items(): + partition_x = ( + data_x if row_indexes is None else data_x.select(row_indexes) + ) + partition_y = ( + data_y if row_indexes is None else data_y.select(row_indexes) + ) + if partition_x.num_rows == 0: + continue + try: + y_pred = trained_model.predict(partition_x) + y_true = ( + trained_model.prepare_output(partition_y, is_fit=False) + .to_pandas() + .to_numpy() + .ravel() + ) + except Exception as e: + log.exception(e) + raise JobError( + f"Failed to predict the {partition} partition" + ) from e + + try: + leaves = instance.compute(y_true, y_pred, class_names) + except ReportError as e: + log.warning("Skipping %s partition: %s", partition, e) + skipped.append(f"{partition_label(partition)}: {e}") + last_error = e + continue + except Exception as e: + log.exception(e) + raise JobError("Failed to compute the report") from e + + if leaves: + groups.append( + ArtifactGroup( + title=partition_label(partition), + artifacts=leaves, + ) + ) + + if not groups: + raise JobError( + str(last_error) + if last_error + else "The report produced no output for any partition." + ) + + self.report_progress(0.8, "Saving the report") + items = [GroupedArtifacts(title=None, groups=groups)] + if skipped: + items.append( + TextArtifact( + payload="\n".join(skipped), + title="Partitions not shown", + ) + ) + artifacts = normalize_artifacts(items) + + try: + path = os.path.join( + config["RUNS_PATH"], f"report_{report_id}.pickle" + ) + with open(path, "wb") as file: + pickle.dump(artifacts, file) + except Exception as e: + log.exception(e) + raise JobError("Report file saving failed") from e + + try: + report.artifacts_path = path + report.plot_overrides = None + report.set_status_as_finished() + db.commit() + except Exception as e: + log.exception(e) + raise JobError("Report path saving failed") from e + + except Exception as e: + report.set_status_as_error() + db.commit() + raise e diff --git a/DashAI/back/reports/__init__.py b/DashAI/back/reports/__init__.py new file mode 100644 index 000000000..9c0fa90a1 --- /dev/null +++ b/DashAI/back/reports/__init__.py @@ -0,0 +1 @@ +# flake8: noqa diff --git a/DashAI/back/reports/base_report.py b/DashAI/back/reports/base_report.py new file mode 100644 index 000000000..d8657cd9f --- /dev/null +++ b/DashAI/back/reports/base_report.py @@ -0,0 +1,148 @@ +"""Base Report abstract class.""" + +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Any, Dict, Final, List, Optional + +from DashAI.back.config_object import ConfigObject +from DashAI.back.core.artifacts import Artifact + +if TYPE_CHECKING: + from numpy import ndarray + + +class ReportError(Exception): + """Raised when a report cannot be computed from the inputs it got.""" + + +class BaseReport(ConfigObject, ABC): + """Abstract base class for evaluation reports. + + A report is the artifact-valued sibling of a metric. Both are computed + from the same inputs at the same moment, comparing a model's predictions + over a split against the truth; they differ only in codomain:: + + Metric: (y_true, y_pred) -> float (rankable, optimizable) + Report: (y_true, y_pred) -> artifact (structured, not rankable) + + That is why a confusion matrix or an ROC curve cannot be a ``Metric``: a + K x K grid does not fit a float column, cannot be ranked for model + selection and cannot be handed to a hyperparameter optimizer. The scalar + summaries of those shapes (accuracy, ROC AUC) already exist as metrics; a + report renders the shape those numbers condense. + + A report is also not an explainer. An explainer probes how the model + responds to *features*; a report never looks at the inputs at all, only + at predictions against the truth. + + Class attributes + ---------------- + TYPE : str + Always ``"Report"``; used by the DashAI component registry. + REQUIRES_PROBABILITIES : bool + ``True`` when ``compute`` needs the full class probability matrix + rather than hard labels. The job checks the actual prediction shape + against this before running, so an incompatible model fails with a + clear message instead of a shape error deep inside a plot call. + DISPLAY_NAME, DESCRIPTION, COLOR, ICON + UI metadata, matching the conventions used by models and explainers. + """ + + TYPE: Final[str] = "Report" + REQUIRES_PROBABILITIES: bool = False + DISPLAY_NAME: str = "" + DESCRIPTION: str = "" + COLOR: str = "#5C6BC0" + ICON: str = "Insights" + + @classmethod + def get_metadata(cls) -> Dict[str, Any]: + """Get metadata values for the current report. + + Returns + ------- + Dict[str, Any] + UI metadata, including whether the report needs a model that + outputs class probabilities. + """ + metadata: Dict[str, Any] = {} + metadata["icon"] = cls.ICON if cls.ICON else "Insights" + metadata["requires_probabilities"] = cls.REQUIRES_PROBABILITIES + return metadata + + @abstractmethod + def compute( + self, + y_true: "ndarray", + y_pred: "ndarray", + class_names: Optional[List[str]] = None, + ) -> List[Artifact]: + """Build renderable artifacts comparing predictions against the truth. + + Parameters + ---------- + y_true : ndarray + Ground truth for the split, encoded the way the model was trained: + class indexes for classification, raw values for regression. + y_pred : ndarray + What the model's ``predict`` returned for the same rows. DashAI + classifiers return a ``(n_samples, n_classes)`` probability matrix + and regressors return a 1D array of values, so a report that + needs hard labels takes the argmax itself. + class_names : Optional[List[str]] + Class labels in encoded order, or None for regression. + + Returns + ------- + List[Artifact] + The artifacts to render, in display order. Leaves only: the job + calls this once per evaluation partition and wraps each result in + a group, and a group cannot contain another group. + + Raises + ------ + NotImplementedError + If the subclass does not provide an implementation. + """ + raise NotImplementedError + + +def as_labels(y_pred: "ndarray") -> "ndarray": + """Reduce a prediction array to hard class labels. + + Parameters + ---------- + y_pred : ndarray + Either a ``(n_samples, n_classes)`` probability matrix or a 1D array of + labels. + + Returns + ------- + ndarray + A 1D array of class indexes. + """ + import numpy as np + + predictions = np.asarray(y_pred) + return predictions.argmax(axis=1) if predictions.ndim == 2 else predictions + + +def resolve_class_names(class_names: Optional[List[str]], n_classes: int) -> List[str]: + """Fill in class labels when the run carries none. + + Parameters + ---------- + class_names : Optional[List[str]] + Labels in encoded order, possibly None or shorter than ``n_classes``. + n_classes : int + How many classes the model predicts. + + Returns + ------- + List[str] + Exactly ``n_classes`` labels, falling back to the class index. + """ + names = list(class_names or []) + return [ + str(names[index]) if index < len(names) else str(index) + for index in range(n_classes) + ] diff --git a/DashAI/back/reports/classification/__init__.py b/DashAI/back/reports/classification/__init__.py new file mode 100644 index 000000000..9c0fa90a1 --- /dev/null +++ b/DashAI/back/reports/classification/__init__.py @@ -0,0 +1 @@ +# flake8: noqa diff --git a/DashAI/back/reports/classification/confusion_matrix.py b/DashAI/back/reports/classification/confusion_matrix.py new file mode 100644 index 000000000..7f657029f --- /dev/null +++ b/DashAI/back/reports/classification/confusion_matrix.py @@ -0,0 +1,189 @@ +"""Confusion matrix report.""" + +from typing import List, Optional + +from DashAI.back.core.artifacts import Artifact, PlotlyArtifact +from DashAI.back.core.schema_fields import BaseSchema, enum_field, schema_field +from DashAI.back.core.utils import MultilingualString +from DashAI.back.reports.base_report import ( + BaseReport, + as_labels, + resolve_class_names, +) + + +class ConfusionMatrixSchema(BaseSchema): + """Schema that configures the confusion matrix.""" + + normalize: schema_field( + enum_field(enum=["none", "true", "pred"]), + placeholder="none", + description=MultilingualString( + en=( + "How to normalise the counts. 'none' shows raw counts, 'true' " + "divides each row by its true class total (recall per class), " + "'pred' divides each column by its predicted total (precision " + "per class)." + ), + es=( + "Cómo normalizar los conteos. 'none' muestra conteos crudos, " + "'true' divide cada fila por el total de su clase real " + "(exhaustividad por clase), 'pred' divide cada columna por su " + "total predicho (precisión por clase)." + ), + pt=( + "Como normalizar as contagens. 'none' mostra contagens brutas, " + "'true' divide cada linha pelo total da sua classe real " + "(revocação por classe), 'pred' divide cada coluna pelo seu " + "total previsto (precisão por classe)." + ), + de=( + "Wie die Zählungen normalisiert werden. 'none' zeigt Rohwerte, " + "'true' teilt jede Zeile durch die Gesamtzahl ihrer echten " + "Klasse (Trefferquote je Klasse), 'pred' teilt jede Spalte " + "durch ihre vorhergesagte Gesamtzahl (Genauigkeit je Klasse)." + ), + zh=( + "如何归一化计数。'none' 显示原始计数，'true' 将每行除以其真实" + "类别总数（每类召回率），'pred' 将每列除以其预测总数" + "（每类精确率）。" + ), + ), + alias=MultilingualString( + en="Normalize", + es="Normalizar", + pt="Normalizar", + de="Normalisieren", + zh="归一化", + ), + ) # type: ignore + + +class ConfusionMatrix(BaseReport): + """K x K grid of true class against predicted class. + + Reading the grid tells you *which* classes a model confuses, which the + scalar accuracy summarising it cannot: two models with identical accuracy + can fail in completely different places. Off diagonal mass concentrated in + one cell means a systematic confusion between that pair of classes; mass + spread evenly across a row means the model has no signal for that class. + + References + ---------- + - [1] https://scikit-learn.org/stable/modules/generated/sklearn.metrics.confusion_matrix.html + """ + + SCHEMA = ConfusionMatrixSchema + COMPATIBLE_COMPONENTS = [ + "TabularClassificationTask", + "TextClassificationTask", + "ImageClassificationTask", + ] + DISPLAY_NAME: str = MultilingualString( + en="Confusion Matrix", + es="Matriz de Confusión", + pt="Matriz de Confusão", + de="Konfusionsmatrix", + zh="混淆矩阵", + ) + DESCRIPTION: str = MultilingualString( + en="Which classes the model confuses, as a true against predicted grid.", + es=("Qué clases confunde el modelo, como una grilla de real contra predicho."), + pt=("Quais classes o modelo confunde, como uma grade de real contra previsto."), + de=( + "Welche Klassen das Modell verwechselt, als Gitter aus echt gegen " + "vorhergesagt." + ), + zh="模型混淆了哪些类别，以真实类别与预测类别的网格呈现。", + ) + COLOR: str = "#5C6BC0" + ICON: str = "GridOn" + + def __init__(self, normalize: str = "none", **kwargs) -> None: + """Initialise the report. + + Parameters + ---------- + normalize : str + One of ``"none"``, ``"true"`` or ``"pred"``. + **kwargs : dict + Ignored; accepted so unknown stored parameters do not break loading. + """ + self.normalize = normalize + + def compute( + self, + y_true, + y_pred, + class_names: Optional[List[str]] = None, + ) -> List[Artifact]: + """Build the confusion matrix heatmap. + + Parameters + ---------- + y_true : ndarray + Encoded true class indexes. + y_pred : ndarray + Model predictions, probabilities or hard labels. + class_names : Optional[List[str]] + Class labels in encoded order. + + Returns + ------- + List[Artifact] + A single heatmap artifact. + """ + import numpy as np + import plotly.graph_objects as go + from sklearn.metrics import confusion_matrix + + labels = as_labels(y_pred) + truth = np.asarray(y_true).ravel() + n_classes = int(max(truth.max(), labels.max())) + 1 + names = resolve_class_names(class_names, n_classes) + + matrix = confusion_matrix(truth, labels, labels=list(range(n_classes))).astype( + float + ) + + if self.normalize == "true": + totals = matrix.sum(axis=1, keepdims=True) + title_suffix = " (row normalized)" + elif self.normalize == "pred": + totals = matrix.sum(axis=0, keepdims=True) + title_suffix = " (column normalized)" + else: + totals = None + title_suffix = "" + + if totals is not None: + # An unpredicted class leaves a zero total; leave those cells at 0 + # rather than emitting NaN, which plotly renders as a hole. + with np.errstate(divide="ignore", invalid="ignore"): + matrix = np.divide( + matrix, totals, out=np.zeros_like(matrix), where=totals != 0 + ) + + text_format = "{:.0f}" if totals is None else "{:.2f}" + figure = go.Figure( + go.Heatmap( + z=matrix.tolist(), + x=names, + y=names, + colorscale="Blues", + text=[[text_format.format(value) for value in row] for row in matrix], + texttemplate="%{text}", + hovertemplate=( + "true: %{y}
predicted: %{x}
value: %{z}" + ), + ) + ) + figure.update_layout( + title=f"Confusion matrix{title_suffix}", + xaxis_title="Predicted", + yaxis_title="True", + # Read top-left to bottom-right like the printed convention. + yaxis={"autorange": "reversed"}, + margin={"l": 20, "r": 20, "t": 50, "b": 40}, + ) + return [PlotlyArtifact(payload=figure, title="Confusion matrix")] diff --git a/DashAI/back/reports/classification/per_class_breakdown.py b/DashAI/back/reports/classification/per_class_breakdown.py new file mode 100644 index 000000000..b80c32242 --- /dev/null +++ b/DashAI/back/reports/classification/per_class_breakdown.py @@ -0,0 +1,124 @@ +"""Per class precision, recall, F1 and support as a table.""" + +from typing import List, Optional + +from DashAI.back.core.artifacts import ( + Artifact, + TableArtifact, + TablePayload, +) +from DashAI.back.core.utils import MultilingualString +from DashAI.back.reports.base_report import ( + BaseReport, + as_labels, + resolve_class_names, +) + + +class PerClassBreakdown(BaseReport): + """Precision, recall, F1 and support broken down per class. + + The aggregate precision and recall metrics average over classes and so hide + the case that matters most: a model that scores well overall while being + useless on a small class. This table is that breakdown, with support + included so a weak row can be read against how many samples back it. + + References + ---------- + - [1] https://scikit-learn.org/stable/modules/generated/sklearn.metrics.per_class_breakdown.html + """ + + COMPATIBLE_COMPONENTS = [ + "TabularClassificationTask", + "TextClassificationTask", + "ImageClassificationTask", + ] + DISPLAY_NAME: str = MultilingualString( + en="Per Class Breakdown", + es="Desglose por Clase", + pt="Detalhamento por Classe", + de="Aufschlüsselung je Klasse", + zh="分类别明细", + ) + DESCRIPTION: str = MultilingualString( + en="Precision, recall, F1 and support for every class.", + es="Precisión, exhaustividad, F1 y soporte para cada clase.", + pt="Precisão, revocação, F1 e suporte para cada classe.", + de="Genauigkeit, Trefferquote, F1 und Support für jede Klasse.", + zh="每个类别的精确率、召回率、F1 和支持度。", + ) + COLOR: str = "#66BB6A" + ICON: str = "TableChart" + + def __init__(self, **kwargs) -> None: + """Initialise the report. It takes no parameters.""" + + def compute( + self, + y_true, + y_pred, + class_names: Optional[List[str]] = None, + ) -> List[Artifact]: + """Build the per class report table. + + Parameters + ---------- + y_true : ndarray + Encoded true class indexes. + y_pred : ndarray + Model predictions, probabilities or hard labels. + class_names : Optional[List[str]] + Class labels in encoded order. + + Returns + ------- + List[Artifact] + A single table artifact, one row per class plus the averages. + """ + import numpy as np + from sklearn.metrics import precision_recall_fscore_support + + labels = as_labels(y_pred) + truth = np.asarray(y_true).ravel() + n_classes = int(max(truth.max(), labels.max())) + 1 + names = resolve_class_names(class_names, n_classes) + indexes = list(range(n_classes)) + + precision, recall, f1, support = precision_recall_fscore_support( + truth, labels, labels=indexes, zero_division=0 + ) + + rows = [ + [ + names[index], + round(float(precision[index]), 4), + round(float(recall[index]), 4), + round(float(f1[index]), 4), + int(support[index]), + ] + for index in indexes + ] + + for average in ("macro", "weighted"): + avg_precision, avg_recall, avg_f1, _ = precision_recall_fscore_support( + truth, labels, labels=indexes, average=average, zero_division=0 + ) + rows.append( + [ + f"{average} avg", + round(float(avg_precision), 4), + round(float(avg_recall), 4), + round(float(avg_f1), 4), + int(support.sum()), + ] + ) + + return [ + TableArtifact( + payload=TablePayload( + columns=["Class", "Precision", "Recall", "F1", "Support"], + rows=rows, + ), + title="Per class breakdown", + ) + ] diff --git a/DashAI/back/reports/classification/precision_recall_curve.py b/DashAI/back/reports/classification/precision_recall_curve.py new file mode 100644 index 000000000..0f47fc97a --- /dev/null +++ b/DashAI/back/reports/classification/precision_recall_curve.py @@ -0,0 +1,130 @@ +"""Precision recall curve report.""" + +from typing import List, Optional + +import numpy as np + +from DashAI.back.core.artifacts import Artifact, PlotlyArtifact +from DashAI.back.core.utils import MultilingualString +from DashAI.back.reports.base_report import ( + BaseReport, + ReportError, + resolve_class_names, +) +from DashAI.back.reports.classification.roc_curve import probability_matrix + + +class PrecisionRecallCurve(BaseReport): + """One vs rest precision against recall, with average precision annotated. + + Preferred over ROC under class imbalance: the false positive rate that ROC + plots is divided by the (large) number of true negatives, so a rare-positive + problem can show an excellent ROC curve while the model is mostly wrong + whenever it does predict the positive class. Precision has no such + denominator, so this curve stays honest as the classes skew. + + References + ---------- + - [1] https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_curve.html + """ + + REQUIRES_PROBABILITIES: bool = True + COMPATIBLE_COMPONENTS = [ + "TabularClassificationTask", + "TextClassificationTask", + "ImageClassificationTask", + ] + DISPLAY_NAME: str = MultilingualString( + en="Precision Recall Curve", + es="Curva Precisión-Exhaustividad", + pt="Curva Precisão-Revocação", + de="Precision-Recall-Kurve", + zh="精确率召回率曲线", + ) + DESCRIPTION: str = MultilingualString( + en=( + "Precision against recall per class; more honest than ROC when " + "classes are imbalanced." + ), + es=( + "Precisión contra exhaustividad por clase; más honesta que ROC " + "cuando las clases están desbalanceadas." + ), + pt=( + "Precisão contra revocação por classe; mais honesta que ROC quando " + "as classes estão desbalanceadas." + ), + de=( + "Genauigkeit gegen Trefferquote je Klasse; ehrlicher als ROC bei " + "unausgeglichenen Klassen." + ), + zh="每个类别的精确率与召回率；在类别不平衡时比 ROC 更可靠。", + ) + COLOR: str = "#AB47BC" + ICON: str = "Timeline" + + def __init__(self, **kwargs) -> None: + """Initialise the report. It takes no parameters.""" + + def compute( + self, + y_true, + y_pred, + class_names: Optional[List[str]] = None, + ) -> List[Artifact]: + """Build the one vs rest precision recall curves. + + Parameters + ---------- + y_true : ndarray + Encoded true class indexes. + y_pred : ndarray + Class probability matrix. + class_names : Optional[List[str]] + Class labels in encoded order. + + Returns + ------- + List[Artifact] + A single figure holding one curve per class. + """ + import plotly.graph_objects as go + from sklearn.metrics import average_precision_score, precision_recall_curve + + probabilities = probability_matrix(y_pred) + truth = np.asarray(y_true).ravel() + names = resolve_class_names(class_names, probabilities.shape[1]) + + figure = go.Figure() + target_classes = [1] if probabilities.shape[1] == 2 else range(len(names)) + for class_index in target_classes: + positives = (truth == class_index).astype(int) + if positives.sum() == 0 or positives.sum() == len(positives): + continue + precision, recall, _ = precision_recall_curve( + positives, probabilities[:, class_index] + ) + average = average_precision_score(positives, probabilities[:, class_index]) + figure.add_trace( + go.Scatter( + x=recall.tolist(), + y=precision.tolist(), + mode="lines", + name=f"{names[class_index]} (AP {average:.3f})", + line={"width": 2}, + ) + ) + + if not figure.data: + raise ReportError( + "The split holds a single class, so no precision recall curve " + "is defined." + ) + + figure.update_layout( + title="Precision recall curve (one vs rest)", + xaxis_title="Recall", + yaxis_title="Precision", + margin={"l": 20, "r": 20, "t": 50, "b": 40}, + ) + return [PlotlyArtifact(payload=figure, title="Precision recall curve")] diff --git a/DashAI/back/reports/classification/roc_curve.py b/DashAI/back/reports/classification/roc_curve.py new file mode 100644 index 000000000..2b972d9d6 --- /dev/null +++ b/DashAI/back/reports/classification/roc_curve.py @@ -0,0 +1,161 @@ +"""ROC curve report.""" + +from typing import List, Optional + +import numpy as np + +from DashAI.back.core.artifacts import Artifact, PlotlyArtifact +from DashAI.back.core.utils import MultilingualString +from DashAI.back.reports.base_report import ( + BaseReport, + ReportError, + resolve_class_names, +) + + +def probability_matrix(y_pred) -> "np.ndarray": + """Return the prediction as a probability matrix, or fail loudly. + + Parameters + ---------- + y_pred : ndarray + What the model's ``predict`` returned. + + Returns + ------- + np.ndarray + A ``(n_samples, n_classes)`` matrix. + + Raises + ------ + ReportError + If the model returned hard labels, which carry no ranking information + and so cannot produce a curve. + """ + predictions = np.asarray(y_pred, dtype=float) + if predictions.ndim != 2 or predictions.shape[1] < 2: + raise ReportError( + "This report needs class probabilities, but the model returned " + "hard labels. Pick a model that outputs probabilities." + ) + return predictions + + +class RocCurve(BaseReport): + """One vs rest ROC curve per class, with the AUC annotated. + + ROC AUC already exists as a metric because it is a single number. The curve + is the shape that number condenses: it shows *where* along the operating + range the model trades false positives for true positives, so two models + with equal AUC can be told apart by which end of the range they are good at. + + References + ---------- + - [1] https://scikit-learn.org/stable/modules/generated/sklearn.metrics.roc_curve.html + """ + + REQUIRES_PROBABILITIES: bool = True + COMPATIBLE_COMPONENTS = [ + "TabularClassificationTask", + "TextClassificationTask", + "ImageClassificationTask", + ] + DISPLAY_NAME: str = MultilingualString( + en="ROC Curve", + es="Curva ROC", + pt="Curva ROC", + de="ROC-Kurve", + zh="ROC 曲线", + ) + DESCRIPTION: str = MultilingualString( + en="True positive rate against false positive rate, one curve per class.", + es=( + "Tasa de verdaderos positivos contra falsos positivos, una curva por clase." + ), + pt=( + "Taxa de verdadeiros positivos contra falsos positivos, uma curva " + "por classe." + ), + de=("Richtig-Positiv-Rate gegen Falsch-Positiv-Rate, eine Kurve je Klasse."), + zh="真正例率与假正例率的关系，每个类别一条曲线。", + ) + COLOR: str = "#26A69A" + ICON: str = "ShowChart" + + def __init__(self, **kwargs) -> None: + """Initialise the report. It takes no parameters.""" + + def compute( + self, + y_true, + y_pred, + class_names: Optional[List[str]] = None, + ) -> List[Artifact]: + """Build the one vs rest ROC curves. + + Parameters + ---------- + y_true : ndarray + Encoded true class indexes. + y_pred : ndarray + Class probability matrix. + class_names : Optional[List[str]] + Class labels in encoded order. + + Returns + ------- + List[Artifact] + A single figure holding one curve per class plus the chance line. + """ + import plotly.graph_objects as go + from sklearn.metrics import auc, roc_curve + + probabilities = probability_matrix(y_pred) + truth = np.asarray(y_true).ravel() + names = resolve_class_names(class_names, probabilities.shape[1]) + + figure = go.Figure() + figure.add_trace( + go.Scatter( + x=[0, 1], + y=[0, 1], + mode="lines", + name="Chance", + line={"dash": "dash", "width": 1, "color": "#9e9e9e"}, + hoverinfo="skip", + ) + ) + + # A binary problem is one curve, not two mirror images of each other. + target_classes = [1] if probabilities.shape[1] == 2 else range(len(names)) + for class_index in target_classes: + positives = (truth == class_index).astype(int) + if positives.sum() == 0 or positives.sum() == len(positives): + # Only one class present in this split: the curve is undefined. + continue + false_positive, true_positive, _ = roc_curve( + positives, probabilities[:, class_index] + ) + area = auc(false_positive, true_positive) + figure.add_trace( + go.Scatter( + x=false_positive.tolist(), + y=true_positive.tolist(), + mode="lines", + name=f"{names[class_index]} (AUC {area:.3f})", + line={"width": 2}, + ) + ) + + if len(figure.data) == 1: + raise ReportError( + "The split holds a single class, so no ROC curve is defined." + ) + + figure.update_layout( + title="ROC curve (one vs rest)", + xaxis_title="False positive rate", + yaxis_title="True positive rate", + margin={"l": 20, "r": 20, "t": 50, "b": 40}, + ) + return [PlotlyArtifact(payload=figure, title="ROC curve")] diff --git a/DashAI/back/reports/forecasting/__init__.py b/DashAI/back/reports/forecasting/__init__.py new file mode 100644 index 000000000..9c0fa90a1 --- /dev/null +++ b/DashAI/back/reports/forecasting/__init__.py @@ -0,0 +1 @@ +# flake8: noqa diff --git a/DashAI/back/reports/forecasting/forecast_vs_actual.py b/DashAI/back/reports/forecasting/forecast_vs_actual.py new file mode 100644 index 000000000..ffcc2b7f8 --- /dev/null +++ b/DashAI/back/reports/forecasting/forecast_vs_actual.py @@ -0,0 +1,98 @@ +"""Forecast against actual values over the observation index.""" + +from typing import List, Optional + +from DashAI.back.core.artifacts import Artifact, PlotlyArtifact +from DashAI.back.core.utils import MultilingualString +from DashAI.back.reports.base_report import BaseReport +from DashAI.back.reports.regression.predicted_vs_actual import flat_predictions + + +class ForecastVsActual(BaseReport): + """Forecast and truth as two lines over the observation index. + + An error scalar says how far off the forecast is on average; this says + *where*. A forecast that tracks well early and drifts later shows a model + whose error compounds over the horizon, while a line that runs parallel + but offset reveals a level bias that RMSE folds into the rest of the + error. Row order is time order for a forecasting run, because the task + sorts its rows by date before anything downstream reads them. + """ + + COMPATIBLE_COMPONENTS = ["ForecastingTask"] + DISPLAY_NAME: str = MultilingualString( + en="Forecast vs Actual", + es="Pronóstico vs Real", + pt="Previsão vs Real", + de="Prognose gegen Tatsächlich", + zh="预测值与实际值", + ) + DESCRIPTION: str = MultilingualString( + en="Forecast and truth as lines over time; reveals drift and bias.", + es="Pronóstico y verdad como líneas en el tiempo; revela deriva y sesgo.", + pt=("Previsão e verdade como linhas ao longo do tempo; revela deriva e viés."), + de=( + "Prognose und Wahrheit als Linien über die Zeit; zeigt Drift und " + "Verzerrung." + ), + zh="随时间变化的预测值与真实值曲线；可揭示漂移和偏差。", + ) + COLOR: str = "#42A5F5" + ICON: str = "ShowChart" + + def __init__(self, **kwargs) -> None: + """Initialise the report. It takes no parameters.""" + + def compute( + self, + y_true, + y_pred, + class_names: Optional[List[str]] = None, + ) -> List[Artifact]: + """Build the forecast against actual line plot. + + Parameters + ---------- + y_true : ndarray + Ground truth values of the series. + y_pred : ndarray + The model's forecast for the same points. + class_names : Optional[List[str]] + Unused; always None for forecasting. + + Returns + ------- + List[Artifact] + A single figure holding the truth and the forecast lines. + """ + import plotly.graph_objects as go + + truth, predictions = flat_predictions(y_true, y_pred) + observations = list(range(len(truth))) + + figure = go.Figure() + figure.add_trace( + go.Scatter( + x=observations, + y=truth.tolist(), + mode="lines", + name="Actual", + line={"width": 2, "color": "#42a5f5"}, + ) + ) + figure.add_trace( + go.Scatter( + x=observations, + y=predictions.tolist(), + mode="lines", + name="Forecast", + line={"width": 2, "color": "#ffa726", "dash": "dash"}, + ) + ) + figure.update_layout( + title="Forecast vs actual", + xaxis_title="Observation", + yaxis_title="Value", + margin={"l": 20, "r": 20, "t": 50, "b": 40}, + ) + return [PlotlyArtifact(payload=figure, title="Forecast vs actual")] diff --git a/DashAI/back/reports/forecasting/residual_autocorrelation.py b/DashAI/back/reports/forecasting/residual_autocorrelation.py new file mode 100644 index 000000000..1c6732422 --- /dev/null +++ b/DashAI/back/reports/forecasting/residual_autocorrelation.py @@ -0,0 +1,164 @@ +"""Residual autocorrelation report.""" + +from typing import List, Optional + +from DashAI.back.core.artifacts import Artifact, PlotlyArtifact +from DashAI.back.core.schema_fields import BaseSchema, int_field, schema_field +from DashAI.back.core.utils import MultilingualString +from DashAI.back.reports.base_report import BaseReport, ReportError +from DashAI.back.reports.regression.predicted_vs_actual import flat_predictions + + +class ResidualAutocorrelationSchema(BaseSchema): + """Schema that configures the residual autocorrelation report.""" + + max_lag: schema_field( + int_field(ge=1, le=200), + placeholder=20, + description=MultilingualString( + en="Number of lags to plot the autocorrelation for.", + es="Número de rezagos para los cuales graficar la autocorrelación.", + pt="Número de defasagens para as quais plotar a autocorrelação.", + de="Anzahl der Lags, für die die Autokorrelation dargestellt wird.", + zh="要绘制自相关的滞后阶数。", + ), + alias=MultilingualString( + en="Max lag", + es="Rezago máximo", + pt="Defasagem máxima", + de="Maximale Verzögerung", + zh="最大滞后阶数", + ), + ) # type: ignore + + +class ResidualAutocorrelation(BaseReport): + """Autocorrelation of the residuals against the number of lags. + + A good forecast leaves residuals that look like noise: each error carries + no information about the next. Autocorrelation measures exactly that, so + a bar above the confidence band means the model left structure behind — + neighbouring periods are wrong in the same direction, which a tuned + model would have learned. Lags rising and falling smoothly usually mean + a missed trend or seasonality, while a single spike means a specific + lag that was never modelled. + """ + + SCHEMA = ResidualAutocorrelationSchema + COMPATIBLE_COMPONENTS = ["ForecastingTask"] + DISPLAY_NAME: str = MultilingualString( + en="Residual Autocorrelation", + es="Autocorrelación de Residuos", + pt="Autocorrelação dos Resíduos", + de="Residuen-Autokorrelation", + zh="残差自相关", + ) + DESCRIPTION: str = MultilingualString( + en="Autocorrelation of the residuals by lag, with confidence band.", + es="Autocorrelación de los residuos por rezago, con banda de confianza.", + pt=("Autocorrelação dos resíduos por defasagem, com banda de confiança."), + de=("Autokorrelation der Residuen nach Lag, mit Konfidenzband."), + zh="按滞后阶数计算的残差自相关，附置信带。", + ) + COLOR: str = "#26A69A" + ICON: str = "BarChart" + + def __init__(self, max_lag: int = 20, **kwargs) -> None: + """Initialise the report. + + Parameters + ---------- + max_lag : int + Number of lags to plot the autocorrelation for. + **kwargs : dict + Ignored; accepted so unknown stored parameters do not break loading. + """ + self.max_lag = max_lag + + def compute( + self, + y_true, + y_pred, + class_names: Optional[List[str]] = None, + ) -> List[Artifact]: + """Build the residual autocorrelation bar chart. + + Parameters + ---------- + y_true : ndarray + Ground truth values of the series. + y_pred : ndarray + The model's forecast for the same points. + class_names : Optional[List[str]] + Unused; always None for forecasting. + + Returns + ------- + List[Artifact] + A single bar figure with a confidence band around zero. + + Raises + ------ + ReportError + If the residuals are constant or there are not enough of them to + estimate a single lag. + """ + import numpy as np + import plotly.graph_objects as go + + truth, predictions = flat_predictions(y_true, y_pred) + residuals = truth - predictions + + if len(residuals) < 2: + raise ReportError( + "The partition holds too few rows to estimate autocorrelation." + ) + + centered = residuals - residuals.mean() + denom = centered @ centered + if denom == 0: + raise ReportError( + "The residuals are constant, so autocorrelation is undefined." + ) + + lags = range(1, min(int(self.max_lag), len(residuals) - 1) + 1) + values = [(centered[k:] @ centered[:-k]) / denom for k in lags] + + figure = go.Figure() + figure.add_trace( + go.Bar( + x=list(lags), + y=values, + name="Autocorrelation", + marker={"color": "#26a69a"}, + ) + ) + + band = 1.96 / np.sqrt(len(residuals)) + figure.add_trace( + go.Scatter( + x=[lags[0], lags[-1]], + y=[band, band], + mode="lines", + name="Confidence band", + line={"dash": "dash", "width": 1, "color": "#9e9e9e"}, + hoverinfo="skip", + ) + ) + figure.add_trace( + go.Scatter( + x=[lags[0], lags[-1]], + y=[-band, -band], + mode="lines", + showlegend=False, + line={"dash": "dash", "width": 1, "color": "#9e9e9e"}, + hoverinfo="skip", + ) + ) + figure.update_layout( + title="Residual autocorrelation", + xaxis_title="Lag", + yaxis_title="Autocorrelation", + margin={"l": 20, "r": 20, "t": 50, "b": 40}, + ) + return [PlotlyArtifact(payload=figure, title="Residual autocorrelation")] diff --git a/DashAI/back/reports/forecasting/residuals_over_time.py b/DashAI/back/reports/forecasting/residuals_over_time.py new file mode 100644 index 000000000..1c071070d --- /dev/null +++ b/DashAI/back/reports/forecasting/residuals_over_time.py @@ -0,0 +1,104 @@ +"""Residuals against the observation index report.""" + +from typing import List, Optional + +from DashAI.back.core.artifacts import Artifact, PlotlyArtifact +from DashAI.back.core.utils import MultilingualString +from DashAI.back.reports.base_report import BaseReport +from DashAI.back.reports.regression.predicted_vs_actual import flat_predictions + + +class ResidualsOverTime(BaseReport): + """Residual against observation index, with the zero line. + + The residual plot used by regression scatters residuals against the + predicted value; forecasting keeps that diagnostic but plots against + time, because time is where the error pattern lives. A band that widens + to the right means the forecast error compounds over the horizon, a + band that drifts off zero means the model is systematically behind or + ahead of the series, and a repeating pattern means a missed seasonality. + """ + + COMPATIBLE_COMPONENTS = ["ForecastingTask"] + DISPLAY_NAME: str = MultilingualString( + en="Residuals Over Time", + es="Residuos en el Tiempo", + pt="Resíduos ao Longo do Tempo", + de="Residuen über die Zeit", + zh="残差随时间变化", + ) + DESCRIPTION: str = MultilingualString( + en="Residual against observation index; reveals drift and missed seasonality.", + es=( + "Residuo contra índice de observación; revela deriva y estacionalidad " + "omitida." + ), + pt=( + "Resíduo contra índice de observação; revela deriva e sazonalidade perdida." + ), + de=( + "Residuum gegen Beobachtungsindex; zeigt Drift und übersehene Saisonalität." + ), + zh="残差与观测序号的对应关系；可揭示漂移与遗漏的季节性。", + ) + COLOR: str = "#EF5350" + ICON: str = "Timeline" + + def __init__(self, **kwargs) -> None: + """Initialise the report. It takes no parameters.""" + + def compute( + self, + y_true, + y_pred, + class_names: Optional[List[str]] = None, + ) -> List[Artifact]: + """Build the residuals against observation index scatter. + + Parameters + ---------- + y_true : ndarray + Ground truth values of the series. + y_pred : ndarray + The model's forecast for the same points. + class_names : Optional[List[str]] + Unused; always None for forecasting. + + Returns + ------- + List[Artifact] + A single scatter figure with the zero reference line. + """ + import plotly.graph_objects as go + + truth, predictions = flat_predictions(y_true, y_pred) + residuals = truth - predictions + observations = list(range(len(truth))) + + figure = go.Figure() + figure.add_trace( + go.Scatter( + x=[observations[0], observations[-1]], + y=[0, 0], + mode="lines", + name="Zero error", + line={"dash": "dash", "width": 1, "color": "#9e9e9e"}, + hoverinfo="skip", + ) + ) + figure.add_trace( + go.Scatter( + x=observations, + y=residuals.tolist(), + mode="markers", + name="Residuals", + marker={"size": 6, "opacity": 0.7, "color": "#ef5350"}, + ) + ) + figure.update_layout( + title="Residuals over time", + xaxis_title="Observation", + yaxis_title="Residual (actual minus forecast)", + margin={"l": 20, "r": 20, "t": 50, "b": 40}, + ) + return [PlotlyArtifact(payload=figure, title="Residuals over time")] diff --git a/DashAI/back/reports/regression/__init__.py b/DashAI/back/reports/regression/__init__.py new file mode 100644 index 000000000..9c0fa90a1 --- /dev/null +++ b/DashAI/back/reports/regression/__init__.py @@ -0,0 +1 @@ +# flake8: noqa diff --git a/DashAI/back/reports/regression/predicted_vs_actual.py b/DashAI/back/reports/regression/predicted_vs_actual.py new file mode 100644 index 000000000..3eb608257 --- /dev/null +++ b/DashAI/back/reports/regression/predicted_vs_actual.py @@ -0,0 +1,117 @@ +"""Predicted against actual report.""" + +from typing import List, Optional + +from DashAI.back.core.artifacts import Artifact, PlotlyArtifact +from DashAI.back.core.utils import MultilingualString +from DashAI.back.reports.base_report import BaseReport + + +def flat_predictions(y_true, y_pred): + """Flatten the truth and predictions into aligned 1D float arrays. + + Parameters + ---------- + y_true : ndarray + Ground truth values. + y_pred : ndarray + Model predictions. + + Returns + ------- + Tuple[np.ndarray, np.ndarray] + The truth and the predictions, both 1D and float typed. + """ + import numpy as np + + return ( + np.asarray(y_true, dtype=float).ravel(), + np.asarray(y_pred, dtype=float).ravel(), + ) + + +class PredictedVsActual(BaseReport): + """Scatter of predicted against true values with the identity line. + + R squared and RMSE say how far off the model is on average; this says + *where*. Points bending away from the identity line at one end reveal a + model that is accurate in the middle of the range and biased at the + extremes, which no single error scalar can express. + """ + + COMPATIBLE_COMPONENTS = ["RegressionTask"] + DISPLAY_NAME: str = MultilingualString( + en="Predicted vs Actual", + es="Predicho vs Real", + pt="Previsto vs Real", + de="Vorhergesagt gegen Tatsächlich", + zh="预测值与实际值", + ) + DESCRIPTION: str = MultilingualString( + en="Predictions against the truth, with the perfect prediction line.", + es="Predicciones contra la verdad, con la línea de predicción perfecta.", + pt="Previsões contra a verdade, com a linha de previsão perfeita.", + de="Vorhersagen gegen die Wahrheit, mit der perfekten Vorhersagelinie.", + zh="预测值与真实值的对比，附完美预测参考线。", + ) + COLOR: str = "#42A5F5" + ICON: str = "ScatterPlot" + + def __init__(self, **kwargs) -> None: + """Initialise the report. It takes no parameters.""" + + def compute( + self, + y_true, + y_pred, + class_names: Optional[List[str]] = None, + ) -> List[Artifact]: + """Build the predicted against actual scatter. + + Parameters + ---------- + y_true : ndarray + Ground truth values. + y_pred : ndarray + Model predictions. + class_names : Optional[List[str]] + Unused; always None for regression. + + Returns + ------- + List[Artifact] + A single scatter figure with the identity reference line. + """ + import plotly.graph_objects as go + + truth, predictions = flat_predictions(y_true, y_pred) + low = float(min(truth.min(), predictions.min())) + high = float(max(truth.max(), predictions.max())) + + figure = go.Figure() + figure.add_trace( + go.Scatter( + x=[low, high], + y=[low, high], + mode="lines", + name="Perfect prediction", + line={"dash": "dash", "width": 1, "color": "#9e9e9e"}, + hoverinfo="skip", + ) + ) + figure.add_trace( + go.Scatter( + x=truth.tolist(), + y=predictions.tolist(), + mode="markers", + name="Predictions", + marker={"size": 6, "opacity": 0.7, "color": "#42a5f5"}, + ) + ) + figure.update_layout( + title="Predicted vs actual", + xaxis_title="Actual", + yaxis_title="Predicted", + margin={"l": 20, "r": 20, "t": 50, "b": 40}, + ) + return [PlotlyArtifact(payload=figure, title="Predicted vs actual")] diff --git a/DashAI/back/reports/regression/residual_histogram.py b/DashAI/back/reports/regression/residual_histogram.py new file mode 100644 index 000000000..5c21eb051 --- /dev/null +++ b/DashAI/back/reports/regression/residual_histogram.py @@ -0,0 +1,118 @@ +"""Residual distribution report.""" + +from typing import List, Optional + +from DashAI.back.core.artifacts import Artifact, PlotlyArtifact +from DashAI.back.core.schema_fields import BaseSchema, int_field, schema_field +from DashAI.back.core.utils import MultilingualString +from DashAI.back.reports.base_report import BaseReport +from DashAI.back.reports.regression.predicted_vs_actual import flat_predictions + + +class ResidualHistogramSchema(BaseSchema): + """Schema that configures the residual histogram.""" + + bins: schema_field( + int_field(ge=5, le=200), + placeholder=30, + description=MultilingualString( + en="Number of bins used to bucket the residuals.", + es="Número de contenedores usados para agrupar los residuos.", + pt="Número de compartimentos usados para agrupar os resíduos.", + de="Anzahl der Klassen zur Gruppierung der Residuen.", + zh="用于划分残差的分箱数量。", + ), + alias=MultilingualString( + en="Bins", es="Contenedores", pt="Compartimentos", de="Klassen", zh="分箱数" + ), + ) # type: ignore + + +class ResidualHistogram(BaseReport): + """Distribution of the residuals, with the zero line marked. + + Shows the shape of the error rather than its magnitude. Residuals should + centre on zero and fall away symmetrically; a shifted centre means constant + bias, a long tail on one side means the model fails asymmetrically, and two + peaks usually mean a subpopulation the model treats as one group. + """ + + SCHEMA = ResidualHistogramSchema + COMPATIBLE_COMPONENTS = ["RegressionTask"] + DISPLAY_NAME: str = MultilingualString( + en="Residual Histogram", + es="Histograma de Residuos", + pt="Histograma de Resíduos", + de="Residuen-Histogramm", + zh="残差直方图", + ) + DESCRIPTION: str = MultilingualString( + en="Distribution of the errors; should centre on zero and look symmetric.", + es=( + "Distribución de los errores; debería centrarse en cero y verse simétrica." + ), + pt=("Distribuição dos erros; deve centrar-se em zero e parecer simétrica."), + de=("Verteilung der Fehler; sollte bei null zentriert und symmetrisch sein."), + zh="误差的分布；应以零为中心且大致对称。", + ) + COLOR: str = "#FFA726" + ICON: str = "BarChart" + + def __init__(self, bins: int = 30, **kwargs) -> None: + """Initialise the report. + + Parameters + ---------- + bins : int + Number of histogram bins. + **kwargs : dict + Ignored; accepted so unknown stored parameters do not break loading. + """ + self.bins = bins + + def compute( + self, + y_true, + y_pred, + class_names: Optional[List[str]] = None, + ) -> List[Artifact]: + """Build the residual histogram. + + Parameters + ---------- + y_true : ndarray + Ground truth values. + y_pred : ndarray + Model predictions. + class_names : Optional[List[str]] + Unused; always None for regression. + + Returns + ------- + List[Artifact] + A single histogram figure with a zero reference line. + """ + import plotly.graph_objects as go + + truth, predictions = flat_predictions(y_true, y_pred) + residuals = truth - predictions + + figure = go.Figure( + go.Histogram( + x=residuals.tolist(), + nbinsx=int(self.bins), + marker={"color": "#ffa726"}, + name="Residuals", + ) + ) + figure.add_vline(x=0, line_dash="dash", line_color="#9e9e9e") + figure.update_layout( + title=( + f"Residual distribution (mean {residuals.mean():.4g}, " + f"std {residuals.std():.4g})" + ), + xaxis_title="Residual (actual minus predicted)", + yaxis_title="Count", + margin={"l": 20, "r": 20, "t": 50, "b": 40}, + ) + return [PlotlyArtifact(payload=figure, title="Residual histogram")] diff --git a/DashAI/back/reports/regression/residual_plot.py b/DashAI/back/reports/regression/residual_plot.py new file mode 100644 index 000000000..7730071b1 --- /dev/null +++ b/DashAI/back/reports/regression/residual_plot.py @@ -0,0 +1,98 @@ +"""Residual against predicted report.""" + +from typing import List, Optional + +from DashAI.back.core.artifacts import Artifact, PlotlyArtifact +from DashAI.back.core.utils import MultilingualString +from DashAI.back.reports.base_report import BaseReport +from DashAI.back.reports.regression.predicted_vs_actual import flat_predictions + + +class ResidualPlot(BaseReport): + """Residual against predicted value, with the zero line. + + A well specified model leaves residuals scattered as a formless band around + zero. Structure in this plot is a diagnosis: curvature means a missing + nonlinear term, a widening fan means heteroscedasticity, and a residual + band that drifts off zero means systematic bias over part of the range. + An error scalar reports the size of these problems but not their shape. + """ + + COMPATIBLE_COMPONENTS = ["RegressionTask"] + DISPLAY_NAME: str = MultilingualString( + en="Residual Plot", + es="Gráfico de Residuos", + pt="Gráfico de Resíduos", + de="Residuendiagramm", + zh="残差图", + ) + DESCRIPTION: str = MultilingualString( + en="Residual against predicted value; reveals bias and heteroscedasticity.", + es=("Residuo contra valor predicho; revela sesgo y heterocedasticidad."), + pt=("Resíduo contra valor previsto; revela viés e heterocedasticidade."), + de=( + "Residuum gegen vorhergesagten Wert; zeigt Verzerrung und " + "Heteroskedastizität." + ), + zh="残差与预测值的关系；可揭示偏差与异方差性。", + ) + COLOR: str = "#EF5350" + ICON: str = "BubbleChart" + + def __init__(self, **kwargs) -> None: + """Initialise the report. It takes no parameters.""" + + def compute( + self, + y_true, + y_pred, + class_names: Optional[List[str]] = None, + ) -> List[Artifact]: + """Build the residual against predicted scatter. + + Parameters + ---------- + y_true : ndarray + Ground truth values. + y_pred : ndarray + Model predictions. + class_names : Optional[List[str]] + Unused; always None for regression. + + Returns + ------- + List[Artifact] + A single scatter figure with the zero reference line. + """ + import plotly.graph_objects as go + + truth, predictions = flat_predictions(y_true, y_pred) + residuals = truth - predictions + + figure = go.Figure() + figure.add_trace( + go.Scatter( + x=[float(predictions.min()), float(predictions.max())], + y=[0, 0], + mode="lines", + name="Zero error", + line={"dash": "dash", "width": 1, "color": "#9e9e9e"}, + hoverinfo="skip", + ) + ) + figure.add_trace( + go.Scatter( + x=predictions.tolist(), + y=residuals.tolist(), + mode="markers", + name="Residuals", + marker={"size": 6, "opacity": 0.7, "color": "#ef5350"}, + ) + ) + figure.update_layout( + title="Residuals vs predicted", + xaxis_title="Predicted", + yaxis_title="Residual (actual minus predicted)", + margin={"l": 20, "r": 20, "t": 50, "b": 40}, + ) + return [PlotlyArtifact(payload=figure, title="Residual plot")] diff --git a/DashAI/back/reports/translation/__init__.py b/DashAI/back/reports/translation/__init__.py new file mode 100644 index 000000000..9c0fa90a1 --- /dev/null +++ b/DashAI/back/reports/translation/__init__.py @@ -0,0 +1 @@ +# flake8: noqa diff --git a/DashAI/back/reports/translation/length_comparison.py b/DashAI/back/reports/translation/length_comparison.py new file mode 100644 index 000000000..1425a1cd0 --- /dev/null +++ b/DashAI/back/reports/translation/length_comparison.py @@ -0,0 +1,99 @@ +"""Reference against translation length comparison report.""" + +from typing import List, Optional + +from DashAI.back.core.artifacts import Artifact, PlotlyArtifact +from DashAI.back.core.utils import MultilingualString +from DashAI.back.reports.base_report import BaseReport +from DashAI.back.reports.translation.per_segment_comparison import as_text_pairs + + +class LengthComparison(BaseReport): + """Reference length against translation length, with the identity line. + + Translation metrics score content, not length, so a model can earn good + BLEU while systematically truncating or padding its output. This plot + shows that shape directly: points hugging the identity line translate + with faithful lengths, a cloud consistently below the line means the + model shortens its output, and a horizontal band at one length means it + produces roughly the same size of translation regardless of input. + """ + + COMPATIBLE_COMPONENTS = ["TranslationTask"] + DISPLAY_NAME: str = MultilingualString( + en="Length Comparison", + es="Comparación de Longitudes", + pt="Comparação de Comprimentos", + de="Längenvergleich", + zh="长度对比", + ) + DESCRIPTION: str = MultilingualString( + en="Reference length against translation length; reveals truncation.", + es="Longitud de referencia contra traducción; revela truncamiento.", + pt=("Comprimento de referência contra tradução; revela truncamento."), + de=("Referenzlänge gegen Übersetzungslänge; zeigt Kürzung."), + zh="参考文本长度与译文长度的对比；可揭示截断问题。", + ) + COLOR: str = "#FFA726" + ICON: str = "ScatterPlot" + + def __init__(self, **kwargs) -> None: + """Initialise the report. It takes no parameters.""" + + def compute( + self, + y_true, + y_pred, + class_names: Optional[List[str]] = None, + ) -> List[Artifact]: + """Build the reference against translation length scatter. + + Parameters + ---------- + y_true : array_like + Reference translations, one per row. + y_pred : array_like + The model's translations for the same rows. + class_names : Optional[List[str]] + Unused; always None for translation. + + Returns + ------- + List[Artifact] + A single scatter figure with the identity reference line. + """ + import plotly.graph_objects as go + + truth, hypothesis = as_text_pairs(y_true, y_pred) + reference_lengths = [len(text) for text in truth] + hypothesis_lengths = [len(text) for text in hypothesis] + low = min(min(reference_lengths), min(hypothesis_lengths)) + high = max(max(reference_lengths), max(hypothesis_lengths)) + + figure = go.Figure() + figure.add_trace( + go.Scatter( + x=[low, high], + y=[low, high], + mode="lines", + name="Perfect match", + line={"dash": "dash", "width": 1, "color": "#9e9e9e"}, + hoverinfo="skip", + ) + ) + figure.add_trace( + go.Scatter( + x=reference_lengths, + y=hypothesis_lengths, + mode="markers", + name="Segments", + marker={"size": 7, "opacity": 0.7, "color": "#ffa726"}, + ) + ) + figure.update_layout( + title="Reference vs translation length", + xaxis_title="Reference length (characters)", + yaxis_title="Translation length (characters)", + margin={"l": 20, "r": 20, "t": 50, "b": 40}, + ) + return [PlotlyArtifact(payload=figure, title="Length comparison")] diff --git a/DashAI/back/reports/translation/per_segment_comparison.py b/DashAI/back/reports/translation/per_segment_comparison.py new file mode 100644 index 000000000..c74778d32 --- /dev/null +++ b/DashAI/back/reports/translation/per_segment_comparison.py @@ -0,0 +1,197 @@ +"""Per segment reference against translation comparison table.""" + +from typing import List, Optional + +import numpy as np + +from DashAI.back.core.artifacts import ( + Artifact, + TableArtifact, + TableCell, + TablePayload, +) +from DashAI.back.core.schema_fields import BaseSchema, int_field, schema_field +from DashAI.back.core.utils import MultilingualString +from DashAI.back.reports.base_report import BaseReport, ReportError + + +def as_text_pairs(y_true, y_pred): + """Flatten the truth and predictions into aligned lists of strings. + + Parameters + ---------- + y_true : array_like + Reference translations, one per row. + y_pred : array_like + The model's translations for the same rows. + + Returns + ------- + tuple of list of str + The references and the hypotheses, both as flat string lists. + + Raises + ------ + ReportError + If the two inputs hold a different number of segments. + """ + truth = [str(value) for value in np.asarray(y_true).ravel()] + hypothesis = [str(value) for value in np.asarray(y_pred).ravel()] + if len(truth) != len(hypothesis): + raise ReportError( + "The number of reference and translated segments must match, " + f"given {len(truth)} and {len(hypothesis)}." + ) + return truth, hypothesis + + +def per_segment_scores(y_true, y_pred) -> List[float]: + """Score every segment with sentence level BLEU. + + Sentence BLEU on short strings is degenerate without smoothing, so the + score uses exponential smoothing and the effective order, which makes an + exact match a 100 and a complete miss a 0. It is a diagnostic, not the + corpus level BLEU the metric reports, so the two are free to disagree on + any one split. + + Parameters + ---------- + y_true : array_like + Reference translations, one per row. + y_pred : array_like + The model's translations for the same rows. + + Returns + ------- + list of float + One score in the 0-100 range per segment. + """ + from sacrebleu.metrics import BLEU + + truth, hypothesis = as_text_pairs(y_true, y_pred) + bleu = BLEU(smooth_method="exp", effective_order=True) + scores = [ + float(bleu.sentence_score(hypothesis[index], [truth[index]]).score) + for index in range(len(truth)) + ] + return [min(100.0, max(0.0, score)) for score in scores] + + +class PerSegmentComparisonSchema(BaseSchema): + """Schema that configures the per segment comparison report.""" + + highlight_count: schema_field( + int_field(ge=0, le=100), + placeholder=5, + description=MultilingualString( + en="How many of the lowest scoring segments to highlight.", + es="Cuántos de los segmentos con menor puntuación resaltar.", + pt="Quantos dos segmentos com menor pontuação destacar.", + de="Wie viele der am schlechtesten bewerteten Segmente hervorheben.", + zh="要突出显示的最低评分片段数量。", + ), + alias=MultilingualString( + en="Highlight count", + es="Cantidad a resaltar", + pt="Quantidade a destacar", + de="Hervorzuhebende Anzahl", + zh="突出显示数量", + ), + ) # type: ignore + + +class PerSegmentComparison(BaseReport): + """Reference against translation for every segment, worst ones highlighted. + + The corpus level BLEU, CHRF and TER metrics condense a whole split into + one number. This table is the split itself: it pairs each reference with + its translation and a sentence level score, and highlights the lowest + scoring rows so the segments dragging the aggregate down are the first + thing seen. A concentration of low scores in long or short segments, or + around a particular topic, is exactly what the average cannot say. + """ + + SCHEMA = PerSegmentComparisonSchema + COMPATIBLE_COMPONENTS = ["TranslationTask"] + DISPLAY_NAME: str = MultilingualString( + en="Per Segment Comparison", + es="Comparación por Segmento", + pt="Comparação por Segmento", + de="Vergleich je Segment", + zh="按片段比较", + ) + DESCRIPTION: str = MultilingualString( + en="Reference against translation per segment, with the worst highlighted.", + es=("Referencia contra traducción por segmento, con los peores resaltados."), + pt=("Referência contra tradução por segmento, com os piores destacados."), + de=( + "Referenz gegen Übersetzung je Segment, mit hervorgehobenen Schlechtesten." + ), + zh="逐个片段对比参考译文与翻译结果，并突出显示最差片段。", + ) + COLOR: str = "#66BB6A" + ICON: str = "TableChart" + + def __init__(self, highlight_count: int = 5, **kwargs) -> None: + """Initialise the report. + + Parameters + ---------- + highlight_count : int + How many of the lowest scoring segments to highlight. + **kwargs : dict + Ignored; accepted so unknown stored parameters do not break loading. + """ + self.highlight_count = highlight_count + + def compute( + self, + y_true, + y_pred, + class_names: Optional[List[str]] = None, + ) -> List[Artifact]: + """Build the per segment comparison table. + + Parameters + ---------- + y_true : array_like + Reference translations, one per row. + y_pred : array_like + The model's translations for the same rows. + class_names : Optional[List[str]] + Unused; always None for translation. + + Returns + ------- + List[Artifact] + A single table, one row per segment plus the average row. + """ + truth, hypothesis = as_text_pairs(y_true, y_pred) + scores = per_segment_scores(y_true, y_pred) + + rows = [ + [ + index + 1, + truth[index], + hypothesis[index], + round(scores[index], 2), + ] + for index in range(len(truth)) + ] + average = sum(scores) / len(scores) if scores else 0.0 + rows.append(["average", "", "", round(float(average), 2)]) + + count = min(int(self.highlight_count), len(scores)) + worst = sorted(range(len(scores)), key=lambda i: scores[i])[:count] + highlight = [TableCell(row=index, column=3) for index in worst] + + return [ + TableArtifact( + payload=TablePayload( + columns=["#", "Reference", "Translation", "Score"], + rows=rows, + highlight=highlight, + ), + title="Per segment comparison", + ) + ] diff --git a/DashAI/back/reports/translation/segment_score_distribution.py b/DashAI/back/reports/translation/segment_score_distribution.py new file mode 100644 index 000000000..0a33c9c44 --- /dev/null +++ b/DashAI/back/reports/translation/segment_score_distribution.py @@ -0,0 +1,127 @@ +"""Distribution of per segment translation scores report.""" + +from typing import List, Optional + +from DashAI.back.core.artifacts import Artifact, PlotlyArtifact +from DashAI.back.core.schema_fields import BaseSchema, int_field, schema_field +from DashAI.back.core.utils import MultilingualString +from DashAI.back.reports.base_report import BaseReport +from DashAI.back.reports.translation.per_segment_comparison import ( + per_segment_scores, +) + + +class SegmentScoreDistributionSchema(BaseSchema): + """Schema that configures the segment score distribution report.""" + + bins: schema_field( + int_field(ge=5, le=200), + placeholder=20, + description=MultilingualString( + en="Number of bins used to bucket the per segment scores.", + es=( + "Número de contenedores usados para agrupar las puntuaciones " + "por segmento." + ), + pt=( + "Número de compartimentos usados para agrupar as pontuações " + "por segmento." + ), + de="Anzahl der Klassen zur Gruppierung der Segment-Scores.", + zh="用于划分每片段分数的分箱数量。", + ), + alias=MultilingualString( + en="Bins", es="Contenedores", pt="Compartimentos", de="Klassen", zh="分箱数" + ), + ) # type: ignore + + +class SegmentScoreDistribution(BaseReport): + """Distribution of the sentence level BLEU scores across segments. + + The corpus BLEU metric is a single point on this histogram; the histogram + is the shape of the quality. A split whose scores cluster high with a + handful near zero contains a few broken translations among otherwise good + ones, while a wide flat distribution means quality is uniformly mediocre — + a different problem to solve, and one the aggregate cannot tell apart. + """ + + SCHEMA = SegmentScoreDistributionSchema + COMPATIBLE_COMPONENTS = ["TranslationTask"] + DISPLAY_NAME: str = MultilingualString( + en="Segment Score Distribution", + es="Distribución de Puntuaciones por Segmento", + pt="Distribuição de Pontuações por Segmento", + de="Verteilung der Segment-Scores", + zh="片段分数分布", + ) + DESCRIPTION: str = MultilingualString( + en="Histogram of the per segment BLEU scores, with the mean marked.", + es=("Histograma de las puntuaciones BLEU por segmento, con la media marcada."), + pt=("Histograma das pontuações BLEU por segmento, com a média marcada."), + de=("Histogramm der BLEU-Scores je Segment, mit markiertem Mittelwert."), + zh="每个片段的 BLEU 分数直方图，并标注平均值。", + ) + COLOR: str = "#AB47BC" + ICON: str = "BarChart" + + def __init__(self, bins: int = 20, **kwargs) -> None: + """Initialise the report. + + Parameters + ---------- + bins : int + Number of histogram bins. + **kwargs : dict + Ignored; accepted so unknown stored parameters do not break loading. + """ + self.bins = bins + + def compute( + self, + y_true, + y_pred, + class_names: Optional[List[str]] = None, + ) -> List[Artifact]: + """Build the segment score distribution histogram. + + Parameters + ---------- + y_true : array_like + Reference translations, one per row. + y_pred : array_like + The model's translations for the same rows. + class_names : Optional[List[str]] + Unused; always None for translation. + + Returns + ------- + List[Artifact] + A single histogram figure with a mean reference line. + """ + import plotly.graph_objects as go + + scores = per_segment_scores(y_true, y_pred) + + figure = go.Figure( + go.Histogram( + x=scores, + nbinsx=int(self.bins), + marker={"color": "#ab47bc"}, + name="Scores", + ) + ) + figure.add_vline( + x=sum(scores) / len(scores), + line_dash="dash", + line_color="#9e9e9e", + ) + figure.update_layout( + title=( + f"Segment score distribution (mean {sum(scores) / len(scores):.4g})" + ), + xaxis_title="Sentence BLEU", + yaxis_title="Count", + margin={"l": 20, "r": 20, "t": 50, "b": 40}, + ) + return [PlotlyArtifact(payload=figure, title="Segment score distribution")] diff --git a/DashAI/front/src/api/explainer.ts b/DashAI/front/src/api/explainer.ts index 623f78e26..3ba9445cc 100644 --- a/DashAI/front/src/api/explainer.ts +++ b/DashAI/front/src/api/explainer.ts @@ -121,7 +121,7 @@ export const saveExplainerPlotOverride = async ( return response.data; }; -export const resetExplainerPlotOverride = async ( +export const deleteExplainerPlotOverride = async ( scope: string, explainerId: number, index: number, diff --git a/DashAI/front/src/api/job.ts b/DashAI/front/src/api/job.ts index b3cd69da4..9907722c4 100644 --- a/DashAI/front/src/api/job.ts +++ b/DashAI/front/src/api/job.ts @@ -51,6 +51,18 @@ export const enqueueRunnerJob = async (runId: number): Promise => { return response.data; }; +export const enqueueReportJob = async (reportId: number): Promise => { + const formData = new FormData(); + formData.append("job_type", "ReportJob"); + formData.append("kwargs", JSON.stringify({ report_id: reportId })); + const response = await api.post("/v1/job/", formData, { + headers: { + "Content-Type": "multipart/form-data", + }, + }); + return response.data; +}; + export const enqueueDatasetJob = async ( dataset_id: number, file: File | null, diff --git a/DashAI/front/src/api/report.ts b/DashAI/front/src/api/report.ts new file mode 100644 index 000000000..9ebe7a32f --- /dev/null +++ b/DashAI/front/src/api/report.ts @@ -0,0 +1,57 @@ +import api from "./api"; + +export interface IReport { + id: number; + run_id: number; + report_name: string; + parameters: object; + artifacts_path: string | null; + status: number; + created: string; +} + +export const getReports = async (runId: number): Promise => { + const response = await api.get("/v1/report/", { + params: { run_id: runId }, + }); + return response.data; +}; + +export const getReportArtifacts = async (reportId: number): Promise => { + const response = await api.get(`/v1/report/${reportId}/artifacts`); + return response.data; +}; + +export const createReport = async ( + runId: number, + reportName: string, + parameters: object = {}, +): Promise => { + const response = await api.post("/v1/report/", { + run_id: runId, + report_name: reportName, + parameters, + }); + return response.data; +}; + +export const deleteReport = async (reportId: number): Promise => { + await api.delete(`/v1/report/${reportId}`); +}; + +/** Persist an edited plotly figure so it survives a reload. */ +export const saveReportPlotOverride = async ( + reportId: number, + index: number, + figure: unknown, +): Promise => { + await api.put(`/v1/report/${reportId}/override`, { index, figure }); +}; + +/** Drop a stored edit so the computed figure comes back. */ +export const deleteReportPlotOverride = async ( + reportId: number, + index: number, +): Promise => { + await api.delete(`/v1/report/${reportId}/override/${index}`); +}; diff --git a/DashAI/front/src/components/explainers/ExplainerInstanceTable.jsx b/DashAI/front/src/components/explainers/ExplainerInstanceTable.jsx index 8ab89c2a2..26b4fda05 100644 --- a/DashAI/front/src/components/explainers/ExplainerInstanceTable.jsx +++ b/DashAI/front/src/components/explainers/ExplainerInstanceTable.jsx @@ -1,21 +1,16 @@ -import React, { useState } from "react"; +import React from "react"; import PropTypes from "prop-types"; -import { Box, Divider, TablePagination } from "@mui/material"; -import { useTheme } from "@mui/material/styles"; import { getDatasetFile } from "../../api/datasets"; +import ArtifactGroupSelector from "../shared/ArtifactGroupSelector"; import LeanDatasetTable from "../shared/leanDatasetTable/LeanDatasetTable"; -import "../shared/leanDatasetTable/leanDatasetTable.css"; - -const ROWS_PER_PAGE = 10; /** * Instance picker for a local explainer's explained rows. When the explainer * stored its input rows as a dataset (datasetPath), it renders the shared * dataset table (feature values, image thumbnails, pagination). Otherwise, for - * explainers computed before input rows were persisted, it falls back to a - * list of instance labels styled like the dataset table. Selecting a row calls - * onSelect with the instance index. + * explainers computed before input rows were persisted, it falls back to the + * shared title list. Selecting a row calls onSelect with the instance index. */ export default function ExplainerInstanceTable({ datasetPath = null, @@ -23,88 +18,31 @@ export default function ExplainerInstanceTable({ selectedIndex, onSelect, }) { - const theme = useTheme(); - const [page, setPage] = useState(0); - - if (datasetPath) { + if (!datasetPath) { return ( - - getDatasetFile(datasetPath, fetchPageIndex, pageSize) - } - datasetPath={datasetPath} - initialPageSize={10} - enableFilters={false} - enableSearch={false} - enableColumnVisibility={false} - enableRowsPerPage={false} - showExportButton={false} - selectedRowIndex={selectedIndex} - onRowClick={(row, globalIndex) => onSelect(globalIndex)} + ); } - const pageStart = page * ROWS_PER_PAGE; - const pageTitles = titles.slice(pageStart, pageStart + ROWS_PER_PAGE); - return ( - -
- - - {pageTitles.map((title, i) => { - const globalIndex = pageStart + i; - const isSelected = globalIndex === selectedIndex; - return ( - onSelect(globalIndex)} - style={{ - backgroundColor: isSelected - ? theme.palette.action.selected - : undefined, - }} - > - - - ); - })} - -
- {title} -
-
- - setPage(newPage)} - rowsPerPageOptions={[ROWS_PER_PAGE]} - labelRowsPerPage="" - slotProps={{ select: { sx: { display: "none" } } }} - /> - + + getDatasetFile(datasetPath, fetchPageIndex, pageSize) + } + datasetPath={datasetPath} + initialPageSize={10} + enableFilters={false} + enableSearch={false} + enableColumnVisibility={false} + enableRowsPerPage={false} + showExportButton={false} + selectedRowIndex={selectedIndex} + onRowClick={(row, globalIndex) => onSelect(globalIndex)} + /> ); } diff --git a/DashAI/front/src/components/explainers/ExplainersCard.jsx b/DashAI/front/src/components/explainers/ExplainersCard.jsx index a9600660d..270e9afc3 100644 --- a/DashAI/front/src/components/explainers/ExplainersCard.jsx +++ b/DashAI/front/src/components/explainers/ExplainersCard.jsx @@ -17,8 +17,8 @@ import ExplainersPlot from "./ExplainersPlot"; import { useNavigate } from "react-router-dom"; import { deleteExplainer, + deleteExplainerPlotOverride, saveExplainerPlotOverride, - resetExplainerPlotOverride, } from "../../api/explainer"; import { useTranslation } from "react-i18next"; @@ -41,10 +41,6 @@ export default function ExplainersCard({ }) { const theme = useTheme(); const [open, setOpen] = useState(false); - const [localOverriddenIndexes, setLocalOverriddenIndexes] = useState([]); - const overriddenIndexes = cacheEntry - ? (cacheEntry.overriddenIndexes ?? []) - : localOverriddenIndexes; const { t } = useTranslation(["explainers"]); const isRunning = RUNNING_STATUSES.includes(explainer.status); @@ -74,18 +70,10 @@ export default function ExplainersCard({ const handleSaveOverride = async (index, figure) => { await saveExplainerPlotOverride(scope, explainer.id, index, figure); - const next = overriddenIndexes.includes(index) - ? overriddenIndexes - : [...overriddenIndexes, index]; - if (onCacheUpdate) onCacheUpdate({ overriddenIndexes: next }); - else setLocalOverriddenIndexes(next); }; const handleResetOverride = async (index) => { - await resetExplainerPlotOverride(scope, explainer.id, index); - const next = overriddenIndexes.filter((i) => i !== index); - if (onCacheUpdate) onCacheUpdate({ overriddenIndexes: next }); - else setLocalOverriddenIndexes(next); + await deleteExplainerPlotOverride(scope, explainer.id, index); }; if (compact) { @@ -103,7 +91,10 @@ export default function ExplainersCard({ "@keyframes newItemHighlight": { "0%": { boxShadow: "none" }, "20%": { - boxShadow: `0 0 0 3px ${alpha(theme.palette.primary.main, 0.65)}, 0 0 24px 8px ${alpha(theme.palette.primary.main, 0.2)}`, + boxShadow: `0 0 0 3px ${alpha( + theme.palette.primary.main, + 0.65, + )}, 0 0 24px 8px ${alpha(theme.palette.primary.main, 0.2)}`, }, "100%": { boxShadow: "none" }, }, @@ -171,7 +162,6 @@ export default function ExplainersCard({ scope={scope} onSaveOverride={handleSaveOverride} onResetOverride={handleResetOverride} - overriddenIndexes={overriddenIndexes} cacheEntry={cacheEntry} onCacheUpdate={onCacheUpdate} /> @@ -265,7 +255,6 @@ ExplainersCard.propTypes = { displayName: PropTypes.string, cacheEntry: PropTypes.shape({ items: PropTypes.array, - overriddenIndexes: PropTypes.arrayOf(PropTypes.number), selectedGroups: PropTypes.object, }), onCacheUpdate: PropTypes.func, diff --git a/DashAI/front/src/components/explainers/ExplainersPlot.jsx b/DashAI/front/src/components/explainers/ExplainersPlot.jsx index b1049ae4e..2e962bbb8 100644 --- a/DashAI/front/src/components/explainers/ExplainersPlot.jsx +++ b/DashAI/front/src/components/explainers/ExplainersPlot.jsx @@ -5,7 +5,8 @@ import { useSnackbar } from "notistack"; import { getExplainerPlot as getExplainerPlotRequest } from "../../api/explainer"; import { useTranslation } from "react-i18next"; -import ArtifactViewer from "../shared/ArtifactViewer"; +import ArtifactList from "../shared/ArtifactList"; +import { patchArtifactPayload } from "../../utils/artifactOverrides"; import ExplainerInstanceTable from "./ExplainerInstanceTable"; import StoryBox from "./StoryBox"; @@ -18,202 +19,11 @@ function parseExplanationArtifacts(items) { ); } -/** Build the onSaveEdit/onResetEdit/canReset props shared by every leaf. */ -function leafProps( - artifact, - { onSaveOverride, onResetOverride, overriddenIndexes }, -) { - return { - canReset: overriddenIndexes.includes(artifact.index), - onSaveEdit: onSaveOverride - ? (figure) => onSaveOverride(artifact.index, figure) - : null, - onResetEdit: onResetOverride ? () => onResetOverride(artifact.index) : null, - }; -} - -/** - * Lay out a batch of leaf artifacts: the first artifact fills the row beside - * whatever `leading` element is passed (a selector, or nothing); any further - * artifacts stack below at full width, most recent first. `siblings` is the - * full artifact list of the batch so the fullscreen viewer can navigate - * between them. - */ -function ArtifactBatch({ - artifacts, - siblings, - ctx, - leading = null, - leadingFlex, - leadingMinWidth = 0, - siblingOffset = 0, -}) { - // Key by position within the batch, not by artifact.index: switching the - // selected group then reuses the same viewer/Plot instance at each slot and - // updates it in place (Plotly diffs) instead of unmounting the tall old plot - // and mounting a new one, which briefly collapses page height and makes the - // window scroll up. - // - // siblingIndex maps this leaf into `siblings` (which may span every group, - // not just this batch) via siblingOffset, so the fullscreen viewer can page - // across groups even when each group has a single artifact. - const renderLeaf = (artifact, i) => ( - - ); - - const [firstArtifact, ...rest] = artifacts; - const stacked = rest.map((artifact, i) => ({ artifact, i: i + 1 })).reverse(); - - return ( - - - {leading && ( - - {leading} - - )} - {renderLeaf(firstArtifact, 0)} - - {stacked.map(({ artifact, i }) => renderLeaf(artifact, i))} - - ); -} - -ArtifactBatch.propTypes = { - artifacts: PropTypes.array.isRequired, - siblings: PropTypes.array.isRequired, - ctx: PropTypes.object.isRequired, - leading: PropTypes.node, - leadingFlex: PropTypes.string, - leadingMinWidth: PropTypes.number, - siblingOffset: PropTypes.number, -}; - -/** - * Render a GroupedArtifacts item: a selector listing every group, beside the - * selected group's first artifact (with the rest stacked below). Holds its own - * selection state, so multiple selectors on one card are independent. - * - * The selector widget depends on `datasetPath`: local explainers pass the - * explained rows dataset path so the picker shows the actual instance feature - * values (the row index selects the group); global explainers omit it and get - * a plain title list. - */ -function GroupedArtifactsView({ - grouped, - ctx, - datasetPath = null, - selected: selectedProp = null, - onSelect = null, -}) { - const { t } = useTranslation(["explainers"]); - const [localSelected, setLocalSelected] = useState(0); - const selected = selectedProp ?? localSelected; - const setSelected = onSelect ?? setLocalSelected; - const groups = grouped.groups ?? []; - if (groups.length === 0) return null; - - const group = groups[selected] ?? groups[0]; - const titles = groups.map( - (g, i) => - g.title ?? t("explainers:label.instanceNumber", { number: i + 1 }), - ); - const wide = Boolean(datasetPath); - - // Fullscreen navigation spans every group's artifacts (flattened), so the - // viewer can page across groups even when each group has a single artifact. - // The selected group's artifacts occupy the slice starting at `offset`. - const allArtifacts = groups.flatMap((g) => g.artifacts); - const offset = groups - .slice(0, selected) - .reduce((n, g) => n + g.artifacts.length, 0); - - // Rendered directly (no height cap): ExplainerInstanceTable's root is - // height:100%, so it fills the stretched batch cell and matches the height - // of the first artifact beside it, scrolling internally when long. - const selector = ( - - ); - - return ( - - - - - ); -} - -GroupedArtifactsView.propTypes = { - grouped: PropTypes.object.isRequired, - ctx: PropTypes.object.isRequired, - datasetPath: PropTypes.string, - selected: PropTypes.number, - onSelect: PropTypes.func, -}; - -/** - * Render one top level response item: a "grouped" selector - * (`GroupedArtifactsView`) or a plain leaf artifact (shown alone at full - * width). `datasetPath` is forwarded to grouped items so local explainers get - * the dataset row picker. - */ -function renderItem(item, ctx, datasetPath = null, selection = null) { - if (item.type === "grouped") { - return ( - - ); - } - return ( - - - - - ); -} - export default function ExplainersPlot({ explainer, scope, onSaveOverride = null, onResetOverride = null, - overriddenIndexes = [], cacheEntry = null, onCacheUpdate = null, }) { @@ -277,33 +87,63 @@ export default function ExplainersPlot({ return {t("explainers:error.noData")}; } - const ctx = { onSaveOverride, onResetOverride, overriddenIndexes }; + // A grouped selector swaps which artifact sits in each slot, which drops the + // viewer's local copy of an edit. Writing the saved figure back into the + // fetched list means switching instance and back still shows it. + const handleSaveOverride = onSaveOverride + ? async (index, figure) => { + await onSaveOverride(index, figure); + const patched = patchArtifactPayload( + items, + index, + JSON.stringify(figure), + ); + setItems(patched); + if (onCacheUpdate) onCacheUpdate({ items: patched }); + } + : null; + + const handleResetOverride = onResetOverride + ? async (index) => { + await onResetOverride(index); + await getExplainerPlot(); + } + : null; - // Every top level item renders continuously: a plain artifact at full width, - // a "grouped" item as its own self contained selector. Local explainers pass - // the explained rows dataset path so their grouped selector shows the - // instance feature values instead of plain labels. + // Local explainers pass the explained rows dataset path so their grouped + // selector shows the instance feature values instead of plain labels. return ( - - {items.map((item, i) => ( - - {renderItem(item, ctx, datasetPath, { - selected: cacheEntry ? (cacheEntry.selectedGroups?.[i] ?? 0) : null, - onSelect: onCacheUpdate - ? (value) => - onCacheUpdate({ - selectedGroups: { - ...(cacheEntry?.selectedGroups ?? {}), - [i]: value, - }, - }) - : null, - })} - - ))} - + ( + + )} + wideSelector={Boolean(datasetPath)} + renderStory={(entry) => ( + + )} + fallbackGroupTitle={(index) => + t("explainers:label.instanceNumber", { number: index + 1 }) + } + selection={ + onCacheUpdate + ? { + selectedFor: (index) => cacheEntry?.selectedGroups?.[index] ?? 0, + onSelect: (index, value) => + onCacheUpdate({ + selectedGroups: { + ...(cacheEntry?.selectedGroups ?? {}), + [index]: value, + }, + }), + } + : null + } + /> ); } @@ -316,7 +156,6 @@ ExplainersPlot.propTypes = { scope: PropTypes.string.isRequired, onSaveOverride: PropTypes.func, onResetOverride: PropTypes.func, - overriddenIndexes: PropTypes.arrayOf(PropTypes.number), cacheEntry: PropTypes.shape({ items: PropTypes.array, selectedGroups: PropTypes.object, diff --git a/DashAI/front/src/components/explainers/explainerCache.js b/DashAI/front/src/components/explainers/explainerCache.js index cf382b75c..0810cb898 100644 --- a/DashAI/front/src/components/explainers/explainerCache.js +++ b/DashAI/front/src/components/explainers/explainerCache.js @@ -1,7 +1,6 @@ // Stable reference returned for cache misses. export const EMPTY_EXPLAINER_ENTRY = { items: null, - overriddenIndexes: [], selectedGroups: {}, }; diff --git a/DashAI/front/src/components/models/ModelsContext.jsx b/DashAI/front/src/components/models/ModelsContext.jsx index 3c0578a7d..8b6f3b2e7 100644 --- a/DashAI/front/src/components/models/ModelsContext.jsx +++ b/DashAI/front/src/components/models/ModelsContext.jsx @@ -98,6 +98,8 @@ export function ModelsProvider({ children }) { const [runDetailTab, setRunDetailTab] = useState(null); const [explainerRefreshTrigger, setExplainerRefreshTrigger] = useState(0); const [explainerToCreate, setExplainerToCreate] = useState(null); + const [reportRefreshTrigger, setReportRefreshTrigger] = useState(0); + const [reportToCreate, setReportToCreate] = useState(null); const [openSections, setOpenSections] = useState({}); const [datasetRowCount, setDatasetRowCount] = useState(null); const [selectedStatisticalTest, setSelectedStatisticalTest] = useState(null); @@ -108,6 +110,20 @@ export function ModelsProvider({ children }) { setExplainerRefreshTrigger((prev) => prev + 1); }, []); + const triggerReportRefresh = useCallback(() => { + setReportRefreshTrigger((prev) => prev + 1); + }, []); + + // Open the report creation dialog for a given component, mirroring how + // openExplainerCreator drives the explainer stepper from the sidebar. + const openReportCreator = useCallback((report) => { + setReportToCreate(report); + }, []); + + const closeReportCreator = useCallback(() => { + setReportToCreate(null); + }, []); + // Open the explainer creation dialog for a given {scope, name}. Shared so both // the sidebar (click) and the central view (drag and drop) can trigger it, // mirroring how selectModel opens the add model dialog. @@ -235,6 +251,11 @@ export function ModelsProvider({ children }) { explainerToCreate, openExplainerCreator, closeExplainerCreator, + reportRefreshTrigger, + triggerReportRefresh, + reportToCreate, + openReportCreator, + closeReportCreator, openSections, setOpenSections, openFolderIds, @@ -308,6 +329,11 @@ export function ModelsProvider({ children }) { explainerToCreate, openExplainerCreator, closeExplainerCreator, + reportRefreshTrigger, + triggerReportRefresh, + reportToCreate, + openReportCreator, + closeReportCreator, openSections, openFolderIds, selectedStatisticalTest, diff --git a/DashAI/front/src/components/models/ModelsRightBar.jsx b/DashAI/front/src/components/models/ModelsRightBar.jsx index b96830bb2..f15246288 100644 --- a/DashAI/front/src/components/models/ModelsRightBar.jsx +++ b/DashAI/front/src/components/models/ModelsRightBar.jsx @@ -109,8 +109,10 @@ import AddModelDialog from "./AddModelDialog"; import ColumnInsights from "../notebooks/dataset/ColumnInsights"; import RunInfoSidebar from "./RunInfoSidebar"; import ExplainersSidebar from "../explainers/ExplainersSidebar"; +import ReportsSidebar from "../reports/ReportsSidebar"; import StatisticalTestsList from "./StatisticalTestsList"; import StatisticalTestsModal from "./StatisticalTestsModal"; +import { REPORTS_TAB } from "./runResults/ResultsTabsHeader"; const EXPLAINERS_TAB = 1; @@ -140,6 +142,7 @@ export default function ModelsRightBar({ onToggle }) { sessionRightContent, runDetailTab, triggerExplainerRefresh, + triggerReportRefresh, datasets, tasks, openStatisticalTest, @@ -272,6 +275,17 @@ export default function ModelsRightBar({ onToggle }) { /> ); } + // Same idea on the reports tab: offer the reports compatible with + // the session's task, one click away from being computed. + if (runDetailTab === REPORTS_TAB && activeRun.status === 3) { + return ( + + ); + } const activeModel = models.find((m) => m.name === activeRun.model_name); const datasetName = datasets.find( (d) => d.id === session?.dataset_id, diff --git a/DashAI/front/src/components/models/RunResults.jsx b/DashAI/front/src/components/models/RunResults.jsx index 89ad21f43..bd0b463e7 100644 --- a/DashAI/front/src/components/models/RunResults.jsx +++ b/DashAI/front/src/components/models/RunResults.jsx @@ -9,11 +9,13 @@ import { checkHowManyOptimazers } from "../../utils/schema"; import { isRunActive } from "../../utils/runStatus"; import { useModels } from "./ModelsContext"; import useRunResultsData from "./runResults/useRunResultsData"; -import ResultsTabsHeader from "./runResults/ResultsTabsHeader"; +import ResultsTabsHeader, { REPORTS_TAB } from "./runResults/ResultsTabsHeader"; import ExplainerResultsTab from "./runResults/ExplainerResultsTab"; import PredictionResultsTab from "./runResults/PredictionResultsTab"; +import ReportResultsTab from "./runResults/ReportResultsTab"; import FoldMetricsChart from "./FoldMetricsChart"; import OuterFoldMetricsTable from "./OuterFoldMetricsTable"; +import { getReports } from "../../api/report"; /** * Shows a run's results as two tab groups (metrics: live/hyperparameters, @@ -79,6 +81,23 @@ export default function RunResults({ const [explainerScrollParent, setExplainerScrollParent] = useState(null); const [showDatasetPanel, setShowDatasetPanel] = useState(false); + const modelsContext = useModels(); + + // Only the count is held here, for the tab chip; the tab body owns the rows. + const [reportCount, setReportCount] = useState(0); + const reportRefreshTrigger = modelsContext?.reportRefreshTrigger; + useEffect(() => { + let cancelled = false; + getReports(run.id) + .then((rows) => { + if (!cancelled) setReportCount(rows.length); + }) + .catch((error) => console.error("Error counting reports:", error)); + return () => { + cancelled = true; + }; + }, [run.id, reportRefreshTrigger]); + const optimizables = checkHowManyOptimazers({ params: run.parameters }); const isFinished = run.status === 3; const isRunning = isRunActive(run.status); @@ -114,7 +133,6 @@ export default function RunResults({ // Expose the active tab while this run is shown full screen, so the right // sidebar can swap its content (e.g. list explainers on the explainers tab). const params = useParams(); - const modelsContext = useModels(); const setRunDetailTab = modelsContext?.setRunDetailTab; const isDetailView = String(params.runId ?? "") === String(run.id); useEffect(() => { @@ -131,6 +149,7 @@ export default function RunResults({ optimizables={optimizables} explainerCount={globalExplainers.length + localExplainers.length} predictionCount={predictions.length} + reportCount={reportCount} run={run} /> ); @@ -195,6 +214,14 @@ export default function RunResults({ )} + + {activeTab === REPORTS_TAB && isFinished && ( + + )} ); diff --git a/DashAI/front/src/components/models/SessionVisualization.jsx b/DashAI/front/src/components/models/SessionVisualization.jsx index 2bedbe66d..63d54c407 100644 --- a/DashAI/front/src/components/models/SessionVisualization.jsx +++ b/DashAI/front/src/components/models/SessionVisualization.jsx @@ -20,6 +20,10 @@ import ModelsBreadcrumbs from "./ModelsBreadcrumbs"; import PillToggleButtonGroup from "../shared/PillToggleButtonGroup"; import { useTranslation } from "react-i18next"; import { useSnackbar } from "notistack"; +import { + createAndRunReport, + hasConfigurableParameters, +} from "../reports/createAndRunReport"; import { useModels } from "./ModelsContext"; import { useTourContext } from "../tour/TourProvider"; @@ -53,6 +57,8 @@ export default function SessionVisualization() { clearLastAddedRunId, selectModel, openExplainerCreator, + openReportCreator, + triggerReportRefresh, explainerRefreshTrigger, triggerExplainerRefresh, openStatisticalTest, @@ -77,7 +83,8 @@ export default function SessionVisualization() { const types = e.dataTransfer.types; if ( types.includes("application/x-dashai-model") || - types.includes("application/x-dashai-explainer") + types.includes("application/x-dashai-explainer") || + types.includes("application/x-dashai-report") ) { setIsDragging(true); } @@ -302,6 +309,7 @@ export default function SessionVisualization() { if ( !e.dataTransfer.types.includes("application/x-dashai-model") && !e.dataTransfer.types.includes("application/x-dashai-explainer") && + !e.dataTransfer.types.includes("application/x-dashai-report") && !e.dataTransfer.types.includes( "application/x-dashai-statistical-test", ) @@ -315,6 +323,7 @@ export default function SessionVisualization() { if ( !e.dataTransfer.types.includes("application/x-dashai-model") && !e.dataTransfer.types.includes("application/x-dashai-explainer") && + !e.dataTransfer.types.includes("application/x-dashai-report") && !e.dataTransfer.types.includes( "application/x-dashai-statistical-test", ) @@ -336,7 +345,9 @@ export default function SessionVisualization() { const isStatisticalTest = types.includes( "application/x-dashai-statistical-test", ); - if (!isModel && !isExplainer && !isStatisticalTest) return; + const isReport = types.includes("application/x-dashai-report"); + if (!isModel && !isExplainer && !isStatisticalTest && !isReport) + return; e.preventDefault(); setIsDragOver(false); try { @@ -352,6 +363,30 @@ export default function SessionVisualization() { if (test?.name) { openStatisticalTest(test); } + } else if (isReport) { + const report = JSON.parse( + e.dataTransfer.getData("application/x-dashai-report"), + ); + if (report?.name) { + // Same rule as clicking the row in the sidebar: nothing to + // configure means nothing to ask. + if (hasConfigurableParameters(report) || !activeRun) { + openReportCreator(report); + } else { + createAndRunReport({ + runId: activeRun.id, + reportName: report.name, + t, + enqueueSnackbar, + onCreated: triggerReportRefresh, + }).catch((error) => { + console.error("Error creating report:", error); + enqueueSnackbar(t("reports:error.create"), { + variant: "error", + }); + }); + } + } } else { const model = JSON.parse( e.dataTransfer.getData("application/x-dashai-model"), diff --git a/DashAI/front/src/components/models/runResults/ReportResultsTab.jsx b/DashAI/front/src/components/models/runResults/ReportResultsTab.jsx new file mode 100644 index 000000000..9a0926126 --- /dev/null +++ b/DashAI/front/src/components/models/runResults/ReportResultsTab.jsx @@ -0,0 +1,185 @@ +import React, { useCallback, useEffect, useRef, useState } from "react"; +import PropTypes from "prop-types"; +import { Box, CircularProgress, Typography } from "@mui/material"; +import { useSnackbar } from "notistack"; +import { useTranslation } from "react-i18next"; + +import { getReports, deleteReport } from "../../../api/report"; +import { getComponents } from "../../../api/component"; +import ReportCard from "../../reports/ReportCard"; + +/** Statuses that mean a report job is still outstanding. */ +const IN_FLIGHT = [1, 2]; + +/** How often the list refreshes while any report is still computing. */ +const POLL_INTERVAL_MS = 3000; + +/** + * Lists the evaluation reports created for a run, oldest first so the newest + * lands at the bottom. New ones are added from the right sidebar, mirroring + * how explainers are added and ordered. + */ +export default function ReportResultsTab({ run, session, refreshTrigger }) { + const { t } = useTranslation(["reports"]); + const { enqueueSnackbar } = useSnackbar(); + + const [reports, setReports] = useState([]); + const [displayNames, setDisplayNames] = useState({}); + const [loading, setLoading] = useState(true); + // A just added report: pendingScroll brings it into view, highlightedId + // drives the ring. Both are keyed by id so a poll that returns the same rows + // cannot replay either. + const [pendingScroll, setPendingScroll] = useState(null); + const [highlightedId, setHighlightedId] = useState(null); + // null until the first fetch lands, so the first render can tell "opened the + // tab" (jump to the bottom) apart from "a report was added" (glide to it). + const seenIdsRef = useRef(null); + + const fetchReports = useCallback(async () => { + try { + const response = await getReports(run.id); + // Oldest first, so a newly added report lands at the bottom of the list + // the way a newly added explainer does. + const ordered = [...response].sort((a, b) => a.id - b.id); + setReports(ordered); + + const ids = ordered.map((item) => item.id); + const newest = ids[ids.length - 1] ?? null; + if (seenIdsRef.current === null) { + // Opening the tab lands at the bottom, matching the explainer list. + if (newest !== null) setPendingScroll({ id: newest, smooth: false }); + } else { + const added = ids.filter((id) => !seenIdsRef.current.has(id)); + if (added.length > 0) { + const addedNewest = added[added.length - 1]; + setPendingScroll({ id: addedNewest, smooth: true }); + setHighlightedId(addedNewest); + } + } + seenIdsRef.current = new Set(ids); + } catch (error) { + console.error("Error fetching reports:", error); + enqueueSnackbar(t("reports:error.fetch"), { variant: "error" }); + } + }, [run.id, enqueueSnackbar, t]); + + // Only the very first load shows a spinner. Flipping it back on for every + // refresh would unmount the whole list and replace it with a spinner each + // time a report is added and again when its job lands, which reads as lag + // rather than as progress. Later fetches swap the rows in place. + useEffect(() => { + fetchReports().finally(() => setLoading(false)); + }, [fetchReports, refreshTrigger]); + + // Component display names are resolved once per task so each card can show a + // human readable title instead of the registry key. + useEffect(() => { + if (!session?.task_name) return; + getComponents({ + selectTypes: ["Report"], + relatedComponent: session.task_name, + }) + .then((components) => + setDisplayNames( + Object.fromEntries( + components.map((item) => [ + item.name, + item.display_name || item.name, + ]), + ), + ), + ) + .catch((error) => console.error("Error fetching report names:", error)); + }, [session?.task_name]); + + // The card has to exist before it can be scrolled to, so this waits a beat + // after the list renders, the way the explainer tab does. + useEffect(() => { + if (!pendingScroll) return undefined; + const timer = setTimeout(() => { + const element = document.getElementById( + `report-card-${pendingScroll.id}`, + ); + if (element) { + element.scrollIntoView({ + block: "end", + behavior: pendingScroll.smooth ? "smooth" : "auto", + }); + } + setPendingScroll(null); + }, 100); + return () => clearTimeout(timer); + }, [pendingScroll, reports]); + + // Clear the highlight after the animation, in its own effect so nothing else + // cancels the timer and leaves the card flagged, replaying the ring on every + // remount. + useEffect(() => { + if (!highlightedId) return undefined; + const timer = setTimeout(() => setHighlightedId(null), 4000); + return () => clearTimeout(timer); + }, [highlightedId]); + + const anyRunning = reports.some((item) => IN_FLIGHT.includes(item.status)); + + useEffect(() => { + if (!anyRunning) return undefined; + const handle = setInterval(fetchReports, POLL_INTERVAL_MS); + return () => clearInterval(handle); + }, [anyRunning, fetchReports]); + + const handleDelete = async (report) => { + try { + await deleteReport(report.id); + setReports((prev) => prev.filter((item) => item.id !== report.id)); + } catch (error) { + console.error("Error deleting report:", error); + enqueueSnackbar(t("reports:error.delete"), { variant: "error" }); + } + }; + + if (loading) { + return ( + + + + ); + } + + if (reports.length === 0) { + return ( + + + {t("reports:message.empty")} + + + ); + } + + return ( + // px gives the highlight ring room so the scroller does not clip its sides. + + {reports.map((report) => ( + + ))} + + ); +} + +ReportResultsTab.propTypes = { + run: PropTypes.shape({ + id: PropTypes.number.isRequired, + }).isRequired, + session: PropTypes.shape({ + task_name: PropTypes.string, + }), + refreshTrigger: PropTypes.number, +}; diff --git a/DashAI/front/src/components/models/runResults/ResultsTabsHeader.jsx b/DashAI/front/src/components/models/runResults/ResultsTabsHeader.jsx index 55a36a156..8dba17130 100644 --- a/DashAI/front/src/components/models/runResults/ResultsTabsHeader.jsx +++ b/DashAI/front/src/components/models/runResults/ResultsTabsHeader.jsx @@ -7,6 +7,16 @@ import { useTranslation } from "react-i18next"; import PillTabs from "../../shared/PillTabs"; import { useModels } from "../ModelsContext"; +/** + * Tab identity for the reports tab, shared by the tab bar, the results + * body and the right sidebar so the three cannot drift apart. + * + * Values 0 to 3 are the live metrics, explainability, predictions and + * hyperparameter tabs, and 4 and 5 the cross validation fold and nested + * results tabs, so this one takes the next free value. + */ +export const REPORTS_TAB = 6; + const groupLabelSx = { textTransform: "uppercase", letterSpacing: 0.5, @@ -33,8 +43,8 @@ const tabLabelRowSx = { /** * The two grouped pill tab bars (Metrics: Live/Hyperparameters, Operations: - * Explainability/Predictions) shown above a run's results, with a vertical - * rule between the groups. Purely presentational. + * Explainability/Predictions/Reports) shown above a run's results, with a + * vertical rule between the groups. Purely presentational. */ export default function ResultsTabsHeader({ activeTab, @@ -43,6 +53,7 @@ export default function ResultsTabsHeader({ optimizables, explainerCount, predictionCount, + reportCount = 0, run, }) { const { t } = useTranslation(["models"]); @@ -175,7 +186,7 @@ export default function ResultsTabsHeader({ {t("models:label.operations")} onTabChange(newValue)} aria-label="Result operations tabs" > @@ -211,6 +222,20 @@ export default function ResultsTabsHeader({ } disabled={!isFinished} /> + + + {t("models:label.reports")} + {isFinished && ( + + )} + + + } + disabled={!isFinished} + /> @@ -224,4 +249,5 @@ ResultsTabsHeader.propTypes = { optimizables: PropTypes.number, explainerCount: PropTypes.number, predictionCount: PropTypes.number, + reportCount: PropTypes.number, }; diff --git a/DashAI/front/src/components/notebooks/explorer/plotLayout/ColorscaleSelector.jsx b/DashAI/front/src/components/notebooks/explorer/plotLayout/ColorscaleSelector.jsx index 7f3c1ab98..5d5640823 100644 --- a/DashAI/front/src/components/notebooks/explorer/plotLayout/ColorscaleSelector.jsx +++ b/DashAI/front/src/components/notebooks/explorer/plotLayout/ColorscaleSelector.jsx @@ -12,6 +12,8 @@ import { ToggleButtonGroup, IconButton, Divider, + FormControlLabel, + Switch, alpha, Alert, useTheme, @@ -42,7 +44,12 @@ const COLORMAPS = [ "YlOrRd", ]; -export default function ColorscaleSelector({ value, onChange }) { +export default function ColorscaleSelector({ + value, + onChange, + reversed = false, + onReversedChange = null, +}) { const theme = useTheme(); const isArrayMode = Array.isArray(value); @@ -143,6 +150,27 @@ export default function ColorscaleSelector({ value, onChange }) { + {/* Reversing is a property of the scale itself rather than of either + mode, so it stays visible in both. Hiding it in array mode would + leave a flag silently reversing a hand built set of stops. */} + {onReversedChange && ( + onReversedChange(event.target.checked)} + /> + } + label={ + + {t("datasets:label.reverseColorscale")} + + } + /> + )} + { + if (usesSharedColorAxis) { + handleChange("coloraxis", { + ...layout.coloraxis, + colorscale: newScale, + }); + } else { + handleTraceChange(index, "colorscale", newScale); + } + }; + + // Plotly reverses a scale with a flag rather than by rewriting its stops, so + // this works for a named preset and a hand built array alike. + const setReversescale = (isReversed) => { + if (usesSharedColorAxis) { + handleChange("coloraxis", { + ...layout.coloraxis, + reversescale: isReversed, + }); + } else { + handleTraceChange(index, "reversescale", isReversed); + } + }; const setColorbarField = (field, value) => { - if (colorbarInLayout) { + if (usesSharedColorAxis) { handleChange("coloraxis", { ...layout.coloraxis, colorbar: { ...layout.coloraxis?.colorbar, [field]: value }, @@ -126,13 +158,10 @@ export default function TraceForm({ {usesColormap(trace) && ( <> - handleChange("coloraxis", { - ...layout.coloraxis, - colorscale: newScale, - }) - } + value={colorscaleSrc} + onChange={setColorscale} + reversed={reversescaleSrc} + onReversedChange={setReversescale} /> { + if (isParamsEmpty && Boolean(defaultValues)) { + setNewReport((prev) => ({ ...prev, parameters: defaultValues })); + } + }, [isParamsEmpty, defaultValues, setNewReport]); + + useEffect(() => { + setNextEnabled(!error); + }, [error, setNextEnabled]); + + return ( + + + {t("reports:label.parameters")} + + + + setNewReport((prev) => ({ ...prev, parameters: values })) + } + setError={setError} + formSubmitRef={formSubmitRef} + /> + + + ); +} + +ConfigureReportStep.propTypes = { + newReport: PropTypes.shape({ + report_name: PropTypes.string, + parameters: PropTypes.object, + }).isRequired, + setNewReport: PropTypes.func.isRequired, + setNextEnabled: PropTypes.func.isRequired, + formSubmitRef: PropTypes.shape({ current: PropTypes.any }).isRequired, + defaultValues: PropTypes.object, +}; diff --git a/DashAI/front/src/components/reports/InlineReportCreator.jsx b/DashAI/front/src/components/reports/InlineReportCreator.jsx new file mode 100644 index 000000000..e85faa8ba --- /dev/null +++ b/DashAI/front/src/components/reports/InlineReportCreator.jsx @@ -0,0 +1,149 @@ +import React, { useEffect, useMemo, useRef, useState } from "react"; +import PropTypes from "prop-types"; +import { + Box, + Dialog, + DialogActions, + DialogContent, + DialogTitle, + IconButton, + Typography, +} from "@mui/material"; +import { Close as CloseIcon } from "@mui/icons-material"; +import { LoadingButton } from "@mui/lab"; +import { useSnackbar } from "notistack"; +import { useTranslation } from "react-i18next"; + +import useSchema from "../../hooks/useSchema"; +import ConfigureReportStep from "./ConfigureReportStep"; +import { createAndRunReport } from "./createAndRunReport"; + +const SNACKBAR_AUTO_HIDE_MS = 5000; + +/** + * Creation dialog for one evaluation report. + * + * A report covers every evaluation partition of the run, so there is nothing + * left to choose but its parameters, and most reports have none. What remains + * is a single confirm rather than a wizard. + */ +export default function InlineReportCreator({ + open, + runId, + reportName, + displayName, + onCreated, + onCancel, +}) { + const { enqueueSnackbar } = useSnackbar(); + const { t } = useTranslation(["reports", "common"]); + const formSubmitRef = useRef(null); + + const { defaultValues, loading: schemaLoading } = useSchema({ + modelName: reportName, + }); + const hasParameters = + Boolean(defaultValues) && Object.keys(defaultValues).length > 0; + + const defaultNewReport = useMemo( + () => ({ run_id: runId, report_name: reportName, parameters: null }), + [runId, reportName], + ); + + const [newReport, setNewReport] = useState(defaultNewReport); + const [valid, setValid] = useState(true); + const [isLoading, setIsLoading] = useState(false); + + useEffect(() => { + if (!open) { + setNewReport(defaultNewReport); + setValid(true); + } + }, [open, defaultNewReport]); + + const handleCreate = async () => { + setIsLoading(true); + try { + await createAndRunReport({ + runId: newReport.run_id, + reportName: newReport.report_name, + parameters: newReport.parameters ?? {}, + t, + enqueueSnackbar, + onCreated, + }); + onCancel(); + } catch (error) { + enqueueSnackbar(t("reports:error.create"), { + variant: "error", + autoHideDuration: SNACKBAR_AUTO_HIDE_MS, + }); + console.error("Error details:", error); + } finally { + setIsLoading(false); + } + }; + + return ( + + + + + {t("reports:label.newReport")} + {`: ${displayName || reportName}`} + + + + + + + + + {hasParameters ? ( + + ) : ( + + {t("reports:message.coversEveryPartition")} + + )} + + + + + {t("reports:button.create")} + + + + ); +} + +InlineReportCreator.propTypes = { + open: PropTypes.bool.isRequired, + runId: PropTypes.number.isRequired, + reportName: PropTypes.string.isRequired, + displayName: PropTypes.string, + onCreated: PropTypes.func, + onCancel: PropTypes.func.isRequired, +}; diff --git a/DashAI/front/src/components/reports/ReportCard.jsx b/DashAI/front/src/components/reports/ReportCard.jsx new file mode 100644 index 000000000..7cdcb276e --- /dev/null +++ b/DashAI/front/src/components/reports/ReportCard.jsx @@ -0,0 +1,212 @@ +import React, { useCallback, useEffect, useState } from "react"; +import PropTypes from "prop-types"; +import { + Box, + Card, + CardContent, + CircularProgress, + IconButton, + Tooltip, + Typography, +} from "@mui/material"; +import { useTheme, alpha } from "@mui/material/styles"; +import DeleteIcon from "@mui/icons-material/Delete"; +import { useTranslation } from "react-i18next"; + +import { + deleteReportPlotOverride, + getReportArtifacts, + saveReportPlotOverride, +} from "../../api/report"; +import ArtifactList from "../shared/ArtifactList"; +import RunStatusDot from "../shared/RunStatusDot"; +import DeleteConfirmationModal from "../threeSectionLayout/DeleteConfirmationModal"; +import { patchArtifactPayload } from "../../utils/artifactOverrides"; + +/** Status codes shared with the backend ReportStatus enum. */ +const STATUS = { + NOT_STARTED: 0, + DELIVERED: 1, + STARTED: 2, + FINISHED: 3, + ERROR: 4, +}; + +/** + * One computed report: its name, its job status, and its artifacts. The + * artifacts arrive as one selector entry per evaluation partition, so the + * partition is chosen inside the card rather than at creation. + */ +export default function ReportCard({ + report, + displayName, + onDelete, + isHighlighted = false, +}) { + const theme = useTheme(); + const { t } = useTranslation(["reports", "common"]); + const [confirmOpen, setConfirmOpen] = useState(false); + const [artifacts, setArtifacts] = useState([]); + const [loading, setLoading] = useState(report.status === STATUS.FINISHED); + const [status, setStatus] = useState(report.status); + + const fetchArtifacts = useCallback(async () => { + try { + const response = await getReportArtifacts(report.id); + setArtifacts(response ?? []); + } catch (error) { + console.error("Error fetching report artifacts:", error); + } + }, [report.id]); + + useEffect(() => { + setStatus(report.status); + }, [report.status]); + + useEffect(() => { + if (status !== STATUS.FINISHED) return; + setLoading(true); + fetchArtifacts().finally(() => setLoading(false)); + }, [status, fetchArtifacts]); + + const running = status === STATUS.DELIVERED || status === STATUS.STARTED; + // No poll here on purpose. A running report has no artifacts to fetch yet, + // and the parent already polls the list while any report is running, so a + // second timer would only double the requests during the window the user is + // waiting through. The status arriving from the parent is what flips this + // card, and that triggers the fetch above. + + const handleSaveOverride = async (index, figure) => { + try { + await saveReportPlotOverride(report.id, index, figure); + // Keep the fetched list in step with what was just persisted, so a + // grouped selector switching entry and back still shows the edit. + setArtifacts((prev) => + patchArtifactPayload(prev, index, JSON.stringify(figure)), + ); + } catch (error) { + console.error("Error saving report plot override:", error); + } + }; + + const handleResetOverride = async (index) => { + try { + await deleteReportPlotOverride(report.id, index); + await fetchArtifacts(); + } catch (error) { + console.error("Error resetting report plot override:", error); + } + }; + + // Same surface the explainer cards use, so the two operation tabs read as + // one family rather than two. + return ( + + + + + {displayName || report.report_name} + {/* ReportStatus mirrors the explainer and run status codes, so the + shared dot maps them without a second colour table. */} + + + + + setConfirmOpen(true)} + > + + + + + + {status === STATUS.ERROR ? ( + + {t("reports:message.failed")} + + ) : running || loading ? ( + + + + {t("reports:message.computing")} + + + ) : artifacts.length === 0 ? ( + + {t("reports:message.noData")} + + ) : ( + + )} + + + setConfirmOpen(false)} + onConfirm={() => { + setConfirmOpen(false); + onDelete(report); + }} + content={t("reports:message.confirmDelete")} + /> + + ); +} + +ReportCard.propTypes = { + report: PropTypes.shape({ + id: PropTypes.number.isRequired, + report_name: PropTypes.string, + status: PropTypes.number, + }).isRequired, + displayName: PropTypes.string, + onDelete: PropTypes.func.isRequired, + isHighlighted: PropTypes.bool, +}; diff --git a/DashAI/front/src/components/reports/ReportsSidebar.jsx b/DashAI/front/src/components/reports/ReportsSidebar.jsx new file mode 100644 index 000000000..a3c6ce057 --- /dev/null +++ b/DashAI/front/src/components/reports/ReportsSidebar.jsx @@ -0,0 +1,188 @@ +import React, { useState, useEffect, useCallback } from "react"; +import PropTypes from "prop-types"; +import { Box, Typography, TextField, CircularProgress } from "@mui/material"; +import { useTheme } from "@mui/material/styles"; +import { Search as SearchIcon } from "@mui/icons-material"; +import { useSnackbar } from "notistack"; +import { useTranslation } from "react-i18next"; + +import SideBar from "../threeSectionLayout/panelContainers/SideBar"; +import { getComponents } from "../../api/component"; +import ModelListItem from "../models/model/ModelListItem"; +import InlineReportCreator from "./InlineReportCreator"; +import { + createAndRunReport, + hasConfigurableParameters, +} from "./createAndRunReport"; +import { useModels } from "../models/ModelsContext"; + +const matchesQuery = (component, query) => + (component.display_name || component.name).toLowerCase().includes(query) || + (component.metadata?.description || "").toLowerCase().includes(query); + +/** + * Right-side panel shown while the model detail view is on its Reports + * tab. Lists the reports compatible with the session's task, mirroring the + * add-explainers sidebar. Clicking one adds it outright, or opens the + * parameter dialog when the report has something to configure. + */ +export default function ReportsSidebar({ run, session, onCreated }) { + const theme = useTheme(); + const { enqueueSnackbar } = useSnackbar(); + const { t } = useTranslation(["models", "reports"]); + + const [reports, setReports] = useState([]); + const [searchQuery, setSearchQuery] = useState(""); + const [loading, setLoading] = useState(false); + const { reportToCreate, openReportCreator, closeReportCreator } = useModels(); + + const taskName = session?.task_name; + + const fetchReports = useCallback(async () => { + if (!taskName) return; + try { + setLoading(true); + const response = await getComponents({ + selectTypes: ["Report"], + relatedComponent: taskName, + }); + setReports(response); + } catch (error) { + console.error("Error fetching reports:", error); + enqueueSnackbar(t("reports:error.fetch"), { variant: "error" }); + } finally { + setLoading(false); + } + }, [taskName, enqueueSnackbar, t]); + + useEffect(() => { + fetchReports(); + }, [fetchReports]); + + // A report with nothing to configure is added on the click. Opening a + // dialog whose only control is a Create button would be pure friction. + const handleSelect = async (report) => { + if (hasConfigurableParameters(report)) { + openReportCreator(report); + return; + } + try { + await createAndRunReport({ + runId: run.id, + reportName: report.name, + t, + enqueueSnackbar, + onCreated, + }); + } catch (error) { + console.error("Error creating report:", error); + enqueueSnackbar(t("reports:error.create"), { variant: "error" }); + } + }; + + const query = searchQuery.trim().toLowerCase(); + const filtered = query + ? reports.filter((item) => matchesQuery(item, query)) + : reports; + + return ( + + + + {t("reports:label.availableReports")} + + + + + setSearchQuery(event.target.value)} + slotProps={{ + input: { + startAdornment: ( + + ), + }, + }} + /> + + + + {loading ? ( + + + + ) : filtered.length === 0 ? ( + + {searchQuery + ? t("reports:label.noReportsMatchSearch") + : t("reports:label.noCompatibleReportsFound")} + + ) : ( + + {filtered.map((report) => ( + handleSelect(report)} + /> + ))} + + )} + + + {reportToCreate && ( + + )} + + ); +} + +ReportsSidebar.propTypes = { + run: PropTypes.shape({ + id: PropTypes.number.isRequired, + }).isRequired, + session: PropTypes.shape({ + task_name: PropTypes.string, + }), + onCreated: PropTypes.func, +}; diff --git a/DashAI/front/src/components/reports/createAndRunReport.js b/DashAI/front/src/components/reports/createAndRunReport.js new file mode 100644 index 000000000..d0ac2c269 --- /dev/null +++ b/DashAI/front/src/components/reports/createAndRunReport.js @@ -0,0 +1,73 @@ +import { createReport } from "../../api/report"; +import { enqueueReportJob } from "../../api/job"; +import { startJobPolling } from "../../utils/jobPoller"; + +const SNACKBAR_AUTO_HIDE_MS = 5000; + +/** + * True when a report has parameters worth asking the user about. + * + * The component list already carries each schema, so this needs no request: + * a report with no properties has nothing to configure and can be added on + * the click itself rather than through a dialog with one button. + * + * @param {object} component A report component as listed by getComponents. + * @returns {boolean} + */ +export function hasConfigurableParameters(component) { + return Object.keys(component?.schema?.properties ?? {}).length > 0; +} + +/** + * Create a report and enqueue its job, reporting both outcomes to the user. + * + * Shared so adding a report straight from the sidebar and adding one through + * the parameter dialog behave identically. + * + * @param {object} options + * @param {number} options.runId + * @param {string} options.reportName + * @param {object} [options.parameters] + * @param {function} options.t translation function + * @param {function} options.enqueueSnackbar notistack enqueue + * @param {function} [options.onCreated] called on create and on job end + * @returns {Promise} the created report row + */ +export async function createAndRunReport({ + runId, + reportName, + parameters = {}, + t, + enqueueSnackbar, + onCreated, +}) { + const created = await createReport(runId, reportName, parameters); + const job = await enqueueReportJob(created.id); + + enqueueSnackbar(t("reports:message.created"), { + variant: "success", + autoHideDuration: SNACKBAR_AUTO_HIDE_MS, + }); + + if (job && job.id) { + startJobPolling( + job.id, + () => { + if (onCreated) onCreated(); + }, + (result) => { + console.error("Report job failed:", result); + enqueueSnackbar(t("reports:message.failed"), { + variant: "error", + autoHideDuration: SNACKBAR_AUTO_HIDE_MS, + }); + if (onCreated) onCreated(); + }, + ); + } + + // Fires before the job finishes so the card shows up computing rather than + // appearing only once the work is done. + if (onCreated) onCreated(); + return created; +} diff --git a/DashAI/front/src/components/shared/ArtifactGroupSelector.jsx b/DashAI/front/src/components/shared/ArtifactGroupSelector.jsx new file mode 100644 index 000000000..d24eaa17d --- /dev/null +++ b/DashAI/front/src/components/shared/ArtifactGroupSelector.jsx @@ -0,0 +1,95 @@ +import React, { useState } from "react"; +import PropTypes from "prop-types"; +import { Box, Divider, TablePagination } from "@mui/material"; +import { useTheme } from "@mui/material/styles"; + +import "./leanDatasetTable/leanDatasetTable.css"; + +const ROWS_PER_PAGE = 10; + +/** + * Default picker for the entries of a grouped artifact: a paginated list of + * group titles styled like the shared dataset table. Selecting a row calls + * onSelect with the group index. + * + * Deliberately free of any data fetching, so rendering a grouped artifact never + * drags the dataset API in behind it. Callers wanting a richer picker (a local + * explainer showing each instance's feature values) supply their own through + * ArtifactList's `renderGroupSelector`. + */ +export default function ArtifactGroupSelector({ + titles, + selectedIndex, + onSelect, +}) { + const theme = useTheme(); + const [page, setPage] = useState(0); + + const pageStart = page * ROWS_PER_PAGE; + const pageTitles = titles.slice(pageStart, pageStart + ROWS_PER_PAGE); + + return ( + +
+ + + {pageTitles.map((title, i) => { + const globalIndex = pageStart + i; + const isSelected = globalIndex === selectedIndex; + return ( + onSelect(globalIndex)} + style={{ + backgroundColor: isSelected + ? theme.palette.action.selected + : undefined, + }} + > + + + ); + })} + +
+ {title} +
+
+ + setPage(newPage)} + rowsPerPageOptions={[ROWS_PER_PAGE]} + labelRowsPerPage="" + slotProps={{ select: { sx: { display: "none" } } }} + /> + + ); +} + +ArtifactGroupSelector.propTypes = { + titles: PropTypes.arrayOf(PropTypes.string).isRequired, + selectedIndex: PropTypes.number, + onSelect: PropTypes.func.isRequired, +}; diff --git a/DashAI/front/src/components/shared/ArtifactList.jsx b/DashAI/front/src/components/shared/ArtifactList.jsx new file mode 100644 index 000000000..1b61bff5e --- /dev/null +++ b/DashAI/front/src/components/shared/ArtifactList.jsx @@ -0,0 +1,245 @@ +import React, { useState } from "react"; +import PropTypes from "prop-types"; +import { Box } from "@mui/material"; + +import ArtifactViewer from "./ArtifactViewer"; +import ArtifactGroupSelector from "./ArtifactGroupSelector"; + +/** + * Build the edit props shared by every leaf. + * + * Whether a leaf can be reset is read off the artifact itself: the backend + * stamps `overridden` on any leaf whose stored edit it just applied, so the + * undo button appears exactly on the plots that have something to undo, + * without the caller tracking indexes of its own. + */ +function leafProps(artifact, { onSaveOverride, onResetOverride } = {}) { + return { + canReset: Boolean(artifact.overridden), + onSaveEdit: onSaveOverride + ? (figure) => onSaveOverride(artifact.index, figure) + : null, + onResetEdit: onResetOverride ? () => onResetOverride(artifact.index) : null, + }; +} + +/** + * Lay out a batch of leaf artifacts: the first artifact fills the row beside + * whatever `leading` element is passed (a selector, or nothing); any further + * artifacts stack below at full width, most recent first. `siblings` is the + * full artifact list of the batch so the fullscreen viewer can navigate + * between them. + */ +function ArtifactBatch({ + artifacts, + siblings, + ctx, + leading = null, + leadingFlex, + leadingMinWidth = 0, + siblingOffset = 0, +}) { + const renderLeaf = (artifact, i) => ( + + ); + + const [firstArtifact, ...rest] = artifacts; + const stacked = rest.map((artifact, i) => ({ artifact, i: i + 1 })).reverse(); + + return ( + + + {leading && ( + + {leading} + + )} + {renderLeaf(firstArtifact, 0)} + + {stacked.map(({ artifact, i }) => renderLeaf(artifact, i))} + + ); +} + +ArtifactBatch.propTypes = { + artifacts: PropTypes.array.isRequired, + siblings: PropTypes.array.isRequired, + ctx: PropTypes.object.isRequired, + leading: PropTypes.node, + leadingFlex: PropTypes.string, + leadingMinWidth: PropTypes.number, + siblingOffset: PropTypes.number, +}; + +/** + * Render a GroupedArtifacts item: a selector listing every group, beside the + * selected group's first artifact (with the rest stacked below). Holds its own + * selection state, so multiple selectors on one card are independent. + * + * The selector defaults to a plain title list. A caller with something richer + * to show (a local explainer listing each explained instance's feature values) + * passes `renderGroupSelector` and, when that widget needs the extra room, + * `wideSelector`. `renderStory` appends a caller supplied element below the + * selected group's artifacts, given the group itself. + */ +function GroupedArtifactsView({ + grouped, + ctx, + renderGroupSelector = null, + wideSelector = false, + fallbackGroupTitle = null, + renderStory = null, + selected: selectedProp = null, + onSelect = null, +}) { + const [localSelected, setLocalSelected] = useState(0); + const selected = selectedProp ?? localSelected; + const setSelected = onSelect ?? setLocalSelected; + const groups = grouped.groups ?? []; + if (groups.length === 0) return null; + + const group = groups[selected] ?? groups[0]; + const titles = groups.map( + (g, i) => + g.title ?? + (fallbackGroupTitle ? fallbackGroupTitle(i) : `Group ${i + 1}`), + ); + + // Fullscreen navigation spans every group's artifacts (flattened), so the + // viewer can page across groups even when each group has a single artifact. + // The selected group's artifacts occupy the slice starting at `offset`. + const allArtifacts = groups.flatMap((g) => g.artifacts); + const offset = groups + .slice(0, selected) + .reduce((n, g) => n + g.artifacts.length, 0); + + // Rendered directly (no height cap): the selector's root is height:100%, so + // it fills the stretched batch cell and matches the height of the first + // artifact beside it, scrolling internally when long. + const selectorProps = { + titles, + selectedIndex: selected, + onSelect: setSelected, + }; + const selector = renderGroupSelector ? ( + renderGroupSelector(selectorProps) + ) : ( + + ); + + const batch = ( + + ); + + if (!renderStory) return batch; + + return ( + + {batch} + {renderStory(group)} + + ); +} + +GroupedArtifactsView.propTypes = { + grouped: PropTypes.object.isRequired, + ctx: PropTypes.object.isRequired, + renderGroupSelector: PropTypes.func, + wideSelector: PropTypes.bool, + fallbackGroupTitle: PropTypes.func, + renderStory: PropTypes.func, + selected: PropTypes.number, + onSelect: PropTypes.func, +}; + +/** + * Render a backend artifact response: every top level item is either a + * "grouped" selector (`GroupedArtifactsView`) or a plain leaf artifact shown + * alone at full width. + * + * `renderGroupSelector` swaps the group picker for a caller supplied one. + * `renderStory` appends a caller supplied element below each item (below the + * selected group, for a grouped one), given that item. + * `selection` lets the caller own the per item selected group (used to keep it + * across remounts); omitting it leaves each selector holding its own state. + */ +export default function ArtifactList({ + items, + ctx = {}, + renderGroupSelector = null, + wideSelector = false, + fallbackGroupTitle = null, + renderStory = null, + selection = null, +}) { + return ( + + {items.map((item, i) => ( + + {item.type === "grouped" ? ( + selection.onSelect(i, value) : null + } + /> + ) : renderStory ? ( + + + {renderStory(item)} + + ) : ( + + )} + + ))} + + ); +} + +ArtifactList.propTypes = { + items: PropTypes.array.isRequired, + ctx: PropTypes.object, + renderGroupSelector: PropTypes.func, + wideSelector: PropTypes.bool, + fallbackGroupTitle: PropTypes.func, + renderStory: PropTypes.func, + selection: PropTypes.shape({ + selectedFor: PropTypes.func.isRequired, + onSelect: PropTypes.func.isRequired, + }), +}; diff --git a/DashAI/front/src/components/shared/ArtifactRenderer.jsx b/DashAI/front/src/components/shared/ArtifactRenderer.jsx index 6bd20fbf6..a16ec2ee4 100644 --- a/DashAI/front/src/components/shared/ArtifactRenderer.jsx +++ b/DashAI/front/src/components/shared/ArtifactRenderer.jsx @@ -17,7 +17,7 @@ import TableArtifact from "./TableArtifact"; * The optional height sets the plot height and caps image/table height; it * lets callers render larger (for example a fullscreen view). */ -export default function ArtifactRenderer({ artifact, height = 380 }) { +function ArtifactRenderer({ artifact, height = 380 }) { const theme = useTheme(); const { t } = useTranslation(["common"]); @@ -51,6 +51,15 @@ export default function ArtifactRenderer({ artifact, height = 380 }) { return new Set(cells.map((cell) => `${cell.row}-${cell.column}`)); }, [artifact]); + // react-plotly.js diffs by reference, so a fresh layout object on every + // render sends Plotly through a full relayout. Ancestors re-render often + // (starting a drag is enough), and each one would otherwise relayout every + // mounted plot at once. + const plotLayout = useMemo( + () => ({ ...themedLayout, height, autosize: true }), + [themedLayout, height], + ); + const renderContent = () => { switch (artifact.type) { case "plotly": @@ -58,7 +67,7 @@ export default function ArtifactRenderer({ artifact, height = 380 }) { return ( { setLocalPayload(null); }, [artifact.payload]); - const shownArtifact = - localPayload != null - ? { ...artifact, payload: localPayload, overridden: true } - : artifact; + // Memoized so the renderer below can bail out on an ancestor re-render: a + // fresh object here would defeat its memo and relayout the plot. + const shownArtifact = useMemo( + () => + localPayload != null + ? { ...artifact, payload: localPayload, overridden: true } + : artifact, + [artifact, localPayload], + ); + const cardArtifact = useMemo( + () => ({ ...shownArtifact, title: null }), + [shownArtifact], + ); const fullscreenArtifact = hasSiblings ? siblingArtifacts[fullscreenIndex] @@ -284,7 +293,7 @@ export default function ArtifactViewer({ explicit height so the figure fits its box instead of overflowing it; everyone else gets the renderer's own default. */} @@ -386,7 +395,6 @@ export default function ArtifactViewer({ fullScreen onClose={() => setFullscreen(false)} transitionDuration={0} - keepMounted PaperProps={{ elevation: 0, sx: { diff --git a/DashAI/front/src/utils/artifactOverrides.js b/DashAI/front/src/utils/artifactOverrides.js new file mode 100644 index 000000000..23556bcce --- /dev/null +++ b/DashAI/front/src/utils/artifactOverrides.js @@ -0,0 +1,34 @@ +/** + * Replace one leaf artifact's payload in a fetched artifact list. + * + * Saving a plot edit persists it on the backend, but the list the frontend + * already holds still carries the computed figure. Anything that re-reads that + * list, such as a grouped selector switching instance and back, would show the + * pre-edit plot. Folding the saved figure in keeps the two in step without a + * refetch. + * + * Leaves nested inside a "grouped" selector are matched by their stamped index + * exactly like top level ones, which is how the backend keys overrides too. + * + * @param {Array} items Artifact/grouped dicts as returned by the backend. + * @param {number} index Stamped index of the leaf to replace. + * @param {string} payload The edited plotly figure, JSON stringified. + * @returns {Array} A new list; inputs are not mutated. + */ +export function patchArtifactPayload(items, index, payload) { + const patchLeaf = (leaf) => + leaf.index === index ? { ...leaf, payload, overridden: true } : leaf; + + return (items ?? []).map((item) => { + if (item.type === "grouped") { + return { + ...item, + groups: (item.groups ?? []).map((group) => ({ + ...group, + artifacts: (group.artifacts ?? []).map(patchLeaf), + })), + }; + } + return patchLeaf(item); + }); +} diff --git a/DashAI/front/src/utils/i18n/index.js b/DashAI/front/src/utils/i18n/index.js index 493f4fdbe..172a09029 100644 --- a/DashAI/front/src/utils/i18n/index.js +++ b/DashAI/front/src/utils/i18n/index.js @@ -9,7 +9,9 @@ import customEN from "./locales/en/custom.json"; import customES from "./locales/es/custom.json"; import experimentsEN from "./locales/en/experiments.json"; import experimentsES from "./locales/es/experiments.json"; +import reportsEN from "./locales/en/reports.json"; import explainersEN from "./locales/en/explainers.json"; +import reportsES from "./locales/es/reports.json"; import explainersES from "./locales/es/explainers.json"; import generativeEN from "./locales/en/generative.json"; import generativeES from "./locales/es/generative.json"; @@ -46,6 +48,7 @@ import configurableObjectPT from "./locales/pt/configurableObject.json"; import commonPT from "./locales/pt/common.json"; import customPT from "./locales/pt/custom.json"; import experimentsPT from "./locales/pt/experiments.json"; +import reportsPT from "./locales/pt/reports.json"; import explainersPT from "./locales/pt/explainers.json"; import generativePT from "./locales/pt/generative.json"; import modelsPT from "./locales/pt/models.json"; @@ -63,6 +66,7 @@ import configurableObjectDE from "./locales/de/configurableObject.json"; import commonDE from "./locales/de/common.json"; import customDE from "./locales/de/custom.json"; import experimentsDE from "./locales/de/experiments.json"; +import reportsDE from "./locales/de/reports.json"; import explainersDE from "./locales/de/explainers.json"; import generativeDE from "./locales/de/generative.json"; import modelsDE from "./locales/de/models.json"; @@ -80,6 +84,7 @@ import configurableObjectZH from "./locales/zh/configurableObject.json"; import commonZH from "./locales/zh/common.json"; import customZH from "./locales/zh/custom.json"; import experimentsZH from "./locales/zh/experiments.json"; +import reportsZH from "./locales/zh/reports.json"; import explainersZH from "./locales/zh/explainers.json"; import generativeZH from "./locales/zh/generative.json"; import modelsZH from "./locales/zh/models.json"; @@ -105,6 +110,7 @@ const resources = { custom: customEN, experiments: experimentsEN, explainers: explainersEN, + reports: reportsEN, generative: generativeEN, models: modelsEN, datasets: datasetsEN, @@ -126,6 +132,7 @@ const resources = { custom: customES, experiments: experimentsES, explainers: explainersES, + reports: reportsES, generative: generativeES, models: modelsES, datasets: datasetsES, @@ -147,6 +154,7 @@ const resources = { custom: customPT, experiments: experimentsPT, explainers: explainersPT, + reports: reportsPT, generative: generativePT, models: modelsPT, datasets: datasetsPT, @@ -167,6 +175,7 @@ const resources = { custom: customDE, experiments: experimentsDE, explainers: explainersDE, + reports: reportsDE, generative: generativeDE, models: modelsDE, datasets: datasetsDE, @@ -187,6 +196,7 @@ const resources = { custom: customZH, experiments: experimentsZH, explainers: explainersZH, + reports: reportsZH, generative: generativeZH, models: modelsZH, datasets: datasetsZH, @@ -219,6 +229,7 @@ i18n "configurableObject", "experiments", "explainers", + "reports", "generative", "models", "datasets", diff --git a/DashAI/front/src/utils/i18n/locales/de/datasets.json b/DashAI/front/src/utils/i18n/locales/de/datasets.json index 7e422e5be..6005aa137 100644 --- a/DashAI/front/src/utils/i18n/locales/de/datasets.json +++ b/DashAI/front/src/utils/i18n/locales/de/datasets.json @@ -415,7 +415,8 @@ "previewRows": "Vorschauzeilen", "previewRowsDescription": "Anzahl der Zeilen für die Vorschau (Minimum 2).", "useNativeTypes": "Native Typen verwenden", - "useNativeTypesDescription": "Spaltentypen aus der Datei verwenden statt statistische Inferenz. Schneller und exakt für selbstbeschreibende Formate." + "useNativeTypesDescription": "Spaltentypen aus der Datei verwenden statt statistische Inferenz. Schneller und exakt für selbstbeschreibende Formate.", + "reverseColorscale": "Farbskala umkehren" }, "message": { "columnTypesUpdated": "Spaltentypen erfolgreich aktualisiert", diff --git a/DashAI/front/src/utils/i18n/locales/de/models.json b/DashAI/front/src/utils/i18n/locales/de/models.json index 4538ee7f6..47bab16f2 100644 --- a/DashAI/front/src/utils/i18n/locales/de/models.json +++ b/DashAI/front/src/utils/i18n/locales/de/models.json @@ -278,7 +278,8 @@ "validation": "Validierung", "validationMetrics": "Validierungsmetriken", "validationSet": "Validierungsmenge", - "viewResultsAs": "Ergebnisse als Spalten oder Graphen anzeigen" + "viewResultsAs": "Ergebnisse als Spalten oder Graphen anzeigen", + "reports": "Berichte" }, "message": { "allRunsCompleted": "{{experiment}} hat alle Durchläufe abgeschlossen.", diff --git a/DashAI/front/src/utils/i18n/locales/de/reports.json b/DashAI/front/src/utils/i18n/locales/de/reports.json new file mode 100644 index 000000000..0e7f931c4 --- /dev/null +++ b/DashAI/front/src/utils/i18n/locales/de/reports.json @@ -0,0 +1,30 @@ +{ + "label": { + "availableReports": "Verfügbare Berichte", + "searchReports": "Berichte suchen", + "noReportsMatchSearch": "Kein Bericht passt zur Suche", + "noCompatibleReportsFound": "Keine kompatiblen Berichte gefunden", + "parameters": "Parameter", + "newReport": "Neuer Bericht", + "configureParameters": "Parameter konfigurieren" + }, + "button": { + "create": "Erstellen", + "delete": "Bericht löschen" + }, + "message": { + "created": "Bericht erstellt", + "computing": "Wird berechnet", + "failed": "Dieser Bericht konnte nicht berechnet werden", + "noData": "Dieser Bericht hat keine Ausgabe erzeugt", + "empty": "Noch keine Berichte. Fügen Sie einen über das rechte Panel hinzu.", + "noParameters": "Dieser Bericht hat keine konfigurierbaren Parameter.", + "confirmDelete": "Möchten Sie diesen Bericht wirklich löschen? Dies kann nicht rückgängig gemacht werden.", + "coversEveryPartition": "Dieser Bericht deckt alle Auswertungspartitionen des Laufs ab." + }, + "error": { + "fetch": "Die Berichte konnten nicht geladen werden", + "create": "Der Bericht konnte nicht erstellt werden", + "delete": "Der Bericht konnte nicht gelöscht werden" + } +} diff --git a/DashAI/front/src/utils/i18n/locales/en/datasets.json b/DashAI/front/src/utils/i18n/locales/en/datasets.json index 6de5912f5..8145624e7 100644 --- a/DashAI/front/src/utils/i18n/locales/en/datasets.json +++ b/DashAI/front/src/utils/i18n/locales/en/datasets.json @@ -415,7 +415,8 @@ "valueType": "Value Type", "viewMode": "View mode", "xAxis": "X Axis", - "yAxis": "Y Axis" + "yAxis": "Y Axis", + "reverseColorscale": "Reverse colorscale" }, "message": { "columnTypesUpdated": "Column types updated successfully", diff --git a/DashAI/front/src/utils/i18n/locales/en/models.json b/DashAI/front/src/utils/i18n/locales/en/models.json index db12745b8..f52b952bf 100644 --- a/DashAI/front/src/utils/i18n/locales/en/models.json +++ b/DashAI/front/src/utils/i18n/locales/en/models.json @@ -278,7 +278,8 @@ "validation": "Validation", "validationMetrics": "Validation Metrics", "validationSet": "validation set", - "viewResultsAs": "View results as columns or graphs" + "viewResultsAs": "View results as columns or graphs", + "reports": "Reports" }, "message": { "allRunsCompleted": "{{experiment}} has completed all its runs.", diff --git a/DashAI/front/src/utils/i18n/locales/en/reports.json b/DashAI/front/src/utils/i18n/locales/en/reports.json new file mode 100644 index 000000000..ec6a5fbdf --- /dev/null +++ b/DashAI/front/src/utils/i18n/locales/en/reports.json @@ -0,0 +1,30 @@ +{ + "label": { + "availableReports": "Available Reports", + "searchReports": "Search reports", + "noReportsMatchSearch": "No reports match your search", + "noCompatibleReportsFound": "No compatible reports found", + "parameters": "Parameters", + "newReport": "New Report", + "configureParameters": "Configure parameters" + }, + "button": { + "create": "Create", + "delete": "Delete report" + }, + "message": { + "created": "Report created", + "computing": "Computing", + "failed": "This report failed to compute", + "noData": "This report produced no output", + "empty": "No reports yet. Add one from the panel on the right.", + "noParameters": "This report has no parameters to configure.", + "confirmDelete": "Are you sure you want to delete this report? This cannot be undone.", + "coversEveryPartition": "This report covers every evaluation partition of the run." + }, + "error": { + "fetch": "Could not load the reports", + "create": "Could not create the report", + "delete": "Could not delete the report" + } +} diff --git a/DashAI/front/src/utils/i18n/locales/es/datasets.json b/DashAI/front/src/utils/i18n/locales/es/datasets.json index 1c636ae09..a37e405ac 100644 --- a/DashAI/front/src/utils/i18n/locales/es/datasets.json +++ b/DashAI/front/src/utils/i18n/locales/es/datasets.json @@ -426,7 +426,8 @@ "valueType": "Tipo de Valor", "viewMode": "Modo de Vista", "xAxis": "Eje X", - "yAxis": "Eje Y" + "yAxis": "Eje Y", + "reverseColorscale": "Invertir escala de colores" }, "message": { "columnTypesUpdated": "Tipos de columnas actualizados exitosamente", diff --git a/DashAI/front/src/utils/i18n/locales/es/models.json b/DashAI/front/src/utils/i18n/locales/es/models.json index 63802b062..38bbac05e 100644 --- a/DashAI/front/src/utils/i18n/locales/es/models.json +++ b/DashAI/front/src/utils/i18n/locales/es/models.json @@ -284,7 +284,8 @@ "validation": "Validación", "validationMetrics": "Métricas de Validación", "validationSet": "conjunto de validación", - "viewResultsAs": "Ver resultados como columnas o gráficos" + "viewResultsAs": "Ver resultados como columnas o gráficos", + "reports": "Reportes" }, "message": { "allRunsCompleted": "{{experiment}} ha completado todas sus ejecuciones.", diff --git a/DashAI/front/src/utils/i18n/locales/es/reports.json b/DashAI/front/src/utils/i18n/locales/es/reports.json new file mode 100644 index 000000000..4b4c5765d --- /dev/null +++ b/DashAI/front/src/utils/i18n/locales/es/reports.json @@ -0,0 +1,30 @@ +{ + "label": { + "availableReports": "Reportes Disponibles", + "searchReports": "Buscar reportes", + "noReportsMatchSearch": "Ningún reporte coincide con la búsqueda", + "noCompatibleReportsFound": "No se encontraron reportes compatibles", + "parameters": "Parámetros", + "newReport": "Nuevo Reporte", + "configureParameters": "Configurar parámetros" + }, + "button": { + "create": "Crear", + "delete": "Eliminar reporte" + }, + "message": { + "created": "Reporte creado", + "computing": "Calculando", + "failed": "Este reporte no se pudo calcular", + "noData": "Este reporte no produjo resultados", + "empty": "Aún no hay reportes. Agregue uno desde el panel de la derecha.", + "noParameters": "Este reporte no tiene parámetros para configurar.", + "confirmDelete": "¿Seguro que desea eliminar este reporte? Esta acción no se puede deshacer.", + "coversEveryPartition": "Este reporte cubre todas las particiones de evaluación de la ejecución." + }, + "error": { + "fetch": "No se pudieron cargar los reportes", + "create": "No se pudo crear el reporte", + "delete": "No se pudo eliminar el reporte" + } +} diff --git a/DashAI/front/src/utils/i18n/locales/pt/datasets.json b/DashAI/front/src/utils/i18n/locales/pt/datasets.json index b671ad4ac..e38b9ee3a 100644 --- a/DashAI/front/src/utils/i18n/locales/pt/datasets.json +++ b/DashAI/front/src/utils/i18n/locales/pt/datasets.json @@ -426,7 +426,8 @@ "valueType": "Tipo de Valor", "viewMode": "Modo de Visualização", "xAxis": "Eixo X", - "yAxis": "Eixo Y" + "yAxis": "Eixo Y", + "reverseColorscale": "Inverter escala de cores" }, "message": { "columnTypesUpdated": "Tipos de colunas atualizados com sucesso", diff --git a/DashAI/front/src/utils/i18n/locales/pt/models.json b/DashAI/front/src/utils/i18n/locales/pt/models.json index 521d38d20..65c474522 100644 --- a/DashAI/front/src/utils/i18n/locales/pt/models.json +++ b/DashAI/front/src/utils/i18n/locales/pt/models.json @@ -284,7 +284,8 @@ "validation": "Validação", "validationMetrics": "Métricas de Validação", "validationSet": "conjunto de validação", - "viewResultsAs": "Ver resultados como colunas ou gráficos" + "viewResultsAs": "Ver resultados como colunas ou gráficos", + "reports": "Relatórios" }, "message": { "allRunsCompleted": "{{experiment}} concluiu todas as suas execuções.", diff --git a/DashAI/front/src/utils/i18n/locales/pt/reports.json b/DashAI/front/src/utils/i18n/locales/pt/reports.json new file mode 100644 index 000000000..04e3b7202 --- /dev/null +++ b/DashAI/front/src/utils/i18n/locales/pt/reports.json @@ -0,0 +1,30 @@ +{ + "label": { + "availableReports": "Relatórios Disponíveis", + "searchReports": "Buscar relatórios", + "noReportsMatchSearch": "Nenhum relatório corresponde à busca", + "noCompatibleReportsFound": "Nenhum relatório compatível encontrado", + "parameters": "Parâmetros", + "newReport": "Novo Relatório", + "configureParameters": "Configurar parâmetros" + }, + "button": { + "create": "Criar", + "delete": "Excluir relatório" + }, + "message": { + "created": "Relatório criado", + "computing": "Calculando", + "failed": "Este relatório falhou ao calcular", + "noData": "Este relatório não produziu resultados", + "empty": "Ainda não há relatórios. Adicione um pelo painel à direita.", + "noParameters": "Este relatório não tem parâmetros para configurar.", + "confirmDelete": "Tem certeza de que deseja excluir este relatório? Esta ação não pode ser desfeita.", + "coversEveryPartition": "Este relatório cobre todas as partições de avaliação da execução." + }, + "error": { + "fetch": "Não foi possível carregar os relatórios", + "create": "Não foi possível criar o relatório", + "delete": "Não foi possível excluir o relatório" + } +} diff --git a/DashAI/front/src/utils/i18n/locales/zh/datasets.json b/DashAI/front/src/utils/i18n/locales/zh/datasets.json index fd4db7f53..b2c4a291d 100644 --- a/DashAI/front/src/utils/i18n/locales/zh/datasets.json +++ b/DashAI/front/src/utils/i18n/locales/zh/datasets.json @@ -413,7 +413,8 @@ "valueType": "值类型", "viewMode": "视图模式", "xAxis": "X 轴", - "yAxis": "Y 轴" + "yAxis": "Y 轴", + "reverseColorscale": "反转色阶" }, "computeMetadata": { "label": "计算扩展元数据（EDA）", diff --git a/DashAI/front/src/utils/i18n/locales/zh/models.json b/DashAI/front/src/utils/i18n/locales/zh/models.json index e5a2aac54..0abfc3a6e 100644 --- a/DashAI/front/src/utils/i18n/locales/zh/models.json +++ b/DashAI/front/src/utils/i18n/locales/zh/models.json @@ -277,7 +277,8 @@ "validation": "验证", "validationMetrics": "验证集指标", "validationSet": "验证集", - "viewResultsAs": "以列或图表方式查看结果" + "viewResultsAs": "以列或图表方式查看结果", + "reports": "报告" }, "message": { "allRunsCompleted": "{{experiment}} 已完成所有运行。", diff --git a/DashAI/front/src/utils/i18n/locales/zh/reports.json b/DashAI/front/src/utils/i18n/locales/zh/reports.json new file mode 100644 index 000000000..cf990a23e --- /dev/null +++ b/DashAI/front/src/utils/i18n/locales/zh/reports.json @@ -0,0 +1,30 @@ +{ + "label": { + "availableReports": "可用报告", + "searchReports": "搜索报告", + "noReportsMatchSearch": "没有匹配的报告", + "noCompatibleReportsFound": "未找到兼容的报告", + "parameters": "参数", + "newReport": "新建报告", + "configureParameters": "配置参数" + }, + "button": { + "create": "创建", + "delete": "删除报告" + }, + "message": { + "created": "报告已创建", + "computing": "正在计算", + "failed": "该报告计算失败", + "noData": "该报告未产生任何输出", + "empty": "暂无报告。请从右侧面板添加。", + "noParameters": "该报告没有可配置的参数。", + "confirmDelete": "确定要删除该报告吗？此操作无法撤销。", + "coversEveryPartition": "该报告涵盖此次运行的所有评估划分。" + }, + "error": { + "fetch": "无法加载报告", + "create": "无法创建报告", + "delete": "无法删除报告" + } +} diff --git a/tests/back/api/test_reports_api.py b/tests/back/api/test_reports_api.py new file mode 100644 index 000000000..1bb967663 --- /dev/null +++ b/tests/back/api/test_reports_api.py @@ -0,0 +1,460 @@ +"""End to end tests for reports: creation, job, retrieval and cleanup. + +Trains a real decision tree through the real ``ModelJob`` so the report +receives what a genuine run produces: a probability matrix from ``predict`` and +encoded targets from ``prepare_output``. +""" + +import json + +import pytest +from fastapi.testclient import TestClient + +from DashAI.back.core.enums.status import ReportStatus, RunStatus +from DashAI.back.dataloaders.classes.csv_dataloader import CSVDataLoader +from DashAI.back.dependencies.database.models import ( + Dataset, + ModelSession, + Report, + Run, +) +from DashAI.back.dependencies.registry import ComponentRegistry +from DashAI.back.evaluation.cv import CrossValidationEvaluationStrategy +from DashAI.back.evaluation.holdout import HoldoutEvaluationStrategy +from DashAI.back.job.base_job import JobError +from DashAI.back.job.model_job import ModelJob +from DashAI.back.job.report_job import ReportJob +from DashAI.back.metrics.classification.accuracy import Accuracy +from DashAI.back.models.scikit_learn.decision_tree_classifier import ( + DecisionTreeClassifier, +) +from DashAI.back.optimizers.optuna_optimizer import OptunaOptimizer +from DashAI.back.reports.classification.confusion_matrix import ConfusionMatrix +from DashAI.back.reports.classification.per_class_breakdown import ( + PerClassBreakdown, +) +from DashAI.back.reports.classification.roc_curve import RocCurve +from DashAI.back.reports.regression.residual_plot import ResidualPlot +from DashAI.back.splitters.holdout import HoldoutSplitter +from DashAI.back.splitters.k_fold import KFoldSplitter +from DashAI.back.tasks.tabular_classification_task import TabularClassificationTask + +INPUT_COLUMNS = ["SepalLengthCm", "SepalWidthCm", "PetalLengthCm", "PetalWidthCm"] +OUTPUT_COLUMNS = ["Species"] + + +@pytest.fixture(scope="module", name="test_registry", autouse=True) +def setup_test_registry(client): + container = client.app.container + sentinel = object() + services = container._services + old = services.get("component_registry", sentinel) + + services["component_registry"] = ComponentRegistry( + initial_components=[ + TabularClassificationTask, + DecisionTreeClassifier, + Accuracy, + CSVDataLoader, + ModelJob, + ReportJob, + OptunaOptimizer, + ConfusionMatrix, + RocCurve, + PerClassBreakdown, + ResidualPlot, + HoldoutSplitter, + HoldoutEvaluationStrategy, + KFoldSplitter, + CrossValidationEvaluationStrategy, + ] + ) + yield services["component_registry"] + if old is sentinel: + del services["component_registry"] + else: + services["component_registry"] = old + + +@pytest.fixture(scope="module", name="model_session_id") +def create_model_session(client: TestClient, dataset_1: Dataset, test_registry): + session_factory = client.app.container["session_factory"] + with session_factory() as db: + model_session = ModelSession( + dataset_id=dataset_1.id, + name="ReportsSession", + task_name="TabularClassificationTask", + input_columns=INPUT_COLUMNS, + output_columns=OUTPUT_COLUMNS, + train_metrics=[], + validation_metrics=[], + test_metrics=[], + evaluation_strategy="HoldoutEvaluationStrategy", + splits=json.dumps( + { + "train": 0.5, + "test": 0.2, + "validation": 0.3, + "is_random": True, + "has_changed": True, + "seed": 42, + "shuffle": True, + "stratify": False, + "splitType": "random", + "splitter_name": "HoldoutSplitter", + } + ), + ) + db.add(model_session) + db.commit() + db.refresh(model_session) + + yield model_session.id + + db.delete(model_session) + db.commit() + + +@pytest.fixture(scope="module", name="trained_run_id") +def train_a_real_model(client: TestClient, model_session_id: int, test_registry): + response = client.post( + "/api/v1/run/", + json={ + "model_session_id": model_session_id, + "model_name": "DecisionTreeClassifier", + "name": "ReportsRun", + "parameters": { + "criterion": "gini", + "max_depth": 3, + "min_samples_split": 2, + "min_samples_leaf": 1, + "max_features": None, + "class_weight": None, + }, + "optimizer_name": "", + "optimizer_parameters": { + "n_trials": 1, + "sampler": "TPESampler", + "pruner": "None", + }, + "goal_metric": "", + "description": "Run under report", + "plot_history_path": "path/to/history.png", + "plot_slice_path": "path/to/slice.png", + "plot_contour_path": "path/to/contour.png", + "plot_importance_path": "path/to/importance.png", + }, + ) + assert response.status_code == 201, response.text + run_id = response.json()["id"] + + ModelJob(run_id=run_id).run() + with client.app.container["session_factory"]() as db: + assert db.get(Run, run_id).status == RunStatus.FINISHED + + yield run_id + + client.delete(f"/api/v1/run/{run_id}") + + +def _create(client: TestClient, run_id: int, name: str, **overrides) -> int: + body = { + "run_id": run_id, + "report_name": name, + "parameters": {}, + **overrides, + } + response = client.post("/api/v1/report/", json=body) + assert response.status_code == 201, response.text + return response.json()["id"] + + +def test_create_lists_and_computes_a_confusion_matrix( + client: TestClient, trained_run_id: int +): + report_id = _create(client, trained_run_id, "ConfusionMatrix") + + listed = client.get(f"/api/v1/report/?run_id={trained_run_id}").json() + assert any(item["id"] == report_id for item in listed) + + # Before the job runs there is nothing to show, but the row exists. + assert client.get(f"/api/v1/report/{report_id}/artifacts").json() == [] + + ReportJob(report_id=report_id).run() + + with client.app.container["session_factory"]() as db: + assert db.get(Report, report_id).status == ReportStatus.FINISHED + + artifacts = client.get(f"/api/v1/report/{report_id}/artifacts").json() + assert len(artifacts) == 1 + assert artifacts[0]["type"] == "grouped" + leaf = artifacts[0]["groups"][0]["artifacts"][0] + assert leaf["type"] == "plotly" + figure = json.loads(leaf["payload"]) + # Class names come from the model's own output encoder. + assert "Iris-setosa" in figure["data"][0]["x"] + + client.delete(f"/api/v1/report/{report_id}") + + +def test_roc_curve_runs_on_real_probabilities(client: TestClient, trained_run_id: int): + """DashAI classifiers return a probability matrix from predict. + + That is what makes an ROC curve computable without any new model contract. + """ + report_id = _create(client, trained_run_id, "RocCurve") + + ReportJob(report_id=report_id).run() + + artifacts = client.get(f"/api/v1/report/{report_id}/artifacts").json() + # Every partition produced a curve, so each is its own selector entry. + assert len(artifacts[0]["groups"]) == 4 + for group in artifacts[0]["groups"]: + figure = json.loads(group["artifacts"][0]["payload"]) + assert any("AUC" in trace.get("name", "") for trace in figure["data"]) + + client.delete(f"/api/v1/report/{report_id}") + + +def test_one_report_covers_every_partition(client: TestClient, trained_run_id: int): + """Partitions are selector entries of one report, not separate reports.""" + report_id = _create(client, trained_run_id, "PerClassBreakdown") + ReportJob(report_id=report_id).run() + + artifacts = client.get(f"/api/v1/report/{report_id}/artifacts").json() + assert len(artifacts) == 1 + assert artifacts[0]["type"] == "grouped" + + titles = [group["title"] for group in artifacts[0]["groups"]] + assert titles == ["Train", "Test", "Validation", "Whole dataset"] + + # Support is the row count of the partition, so the groups must disagree. + supports = [ + group["artifacts"][0]["payload"]["rows"][-1][-1] + for group in artifacts[0]["groups"] + ] + assert len(set(supports)) > 1 + + # Indexes are stamped flat across groups so an edit can address any leaf. + indexes = [group["artifacts"][0]["index"] for group in artifacts[0]["groups"]] + assert indexes == [0, 1, 2, 3] + + client.delete(f"/api/v1/report/{report_id}") + + +def test_an_incompatible_report_fails_with_its_own_message( + client: TestClient, trained_run_id: int +): + """A regression report over a classifier must not produce a plot.""" + report_id = _create(client, trained_run_id, "ResidualPlot") + + with pytest.raises(JobError, match="Failed to compute the report"): + ReportJob(report_id=report_id).run() + + with client.app.container["session_factory"]() as db: + assert db.get(Report, report_id).status == ReportStatus.ERROR + + client.delete(f"/api/v1/report/{report_id}") + + +def test_plot_edits_survive_a_reload(client: TestClient, trained_run_id: int): + """A saved edit replaces the computed figure on every later read.""" + report_id = _create(client, trained_run_id, "ConfusionMatrix") + ReportJob(report_id=report_id).run() + + edited = {"data": [{"type": "heatmap", "z": [[1]]}], "layout": {"title": "mine"}} + response = client.put( + f"/api/v1/report/{report_id}/override", + json={"index": 0, "figure": edited}, + ) + assert response.status_code == 200, response.text + + edited_leaf = client.get(f"/api/v1/report/{report_id}/artifacts").json()[0][ + "groups" + ][0]["artifacts"][0] + assert json.loads(edited_leaf["payload"])["layout"]["title"] == "mine" + + assert edited_leaf["overridden"] is True + + client.delete(f"/api/v1/report/{report_id}") + + +def test_resetting_an_edit_restores_the_computed_figure( + client: TestClient, trained_run_id: int +): + report_id = _create(client, trained_run_id, "ConfusionMatrix") + ReportJob(report_id=report_id).run() + + client.put( + f"/api/v1/report/{report_id}/override", + json={"index": 0, "figure": {"data": [], "layout": {"title": "mine"}}}, + ) + response = client.delete(f"/api/v1/report/{report_id}/override/0") + assert response.status_code == 200, response.text + + leaf = client.get(f"/api/v1/report/{report_id}/artifacts").json()[0]["groups"][0][ + "artifacts" + ][0] + figure = json.loads(leaf["payload"]) + assert figure["layout"]["title"]["text"].startswith("Confusion matrix") + assert "overridden" not in leaf + + client.delete(f"/api/v1/report/{report_id}") + + +def test_overriding_an_unknown_report_is_404(client: TestClient): + response = client.put( + "/api/v1/report/99999/override", + json={"index": 0, "figure": {"data": []}}, + ) + assert response.status_code == 404 + + +def test_creating_for_an_unknown_run_is_404(client: TestClient): + response = client.post( + "/api/v1/report/", + json={ + "run_id": 99999, + "report_name": "ConfusionMatrix", + "parameters": {}, + }, + ) + assert response.status_code == 404 + + +def test_retraining_deletes_the_reports(client: TestClient, trained_run_id: int): + """Reports describe the predictions of the fit being replaced.""" + report_id = _create(client, trained_run_id, "ConfusionMatrix") + ReportJob(report_id=report_id).run() + + counts = client.get(f"/api/v1/run/{trained_run_id}/operations/count").json() + assert counts["reports"] == 1 + + response = client.delete(f"/api/v1/run/{trained_run_id}/operations") + assert response.status_code in (200, 204), response.text + + remaining = client.get(f"/api/v1/report/?run_id={trained_run_id}").json() + assert remaining == [] + + +@pytest.fixture(scope="module", name="cv_run_id") +def train_a_cross_validated_model( + client: TestClient, dataset_1: Dataset, test_registry +): + session_factory = client.app.container["session_factory"] + with session_factory() as db: + model_session = ModelSession( + dataset_id=dataset_1.id, + name="ReportsCVSession", + task_name="TabularClassificationTask", + input_columns=INPUT_COLUMNS, + output_columns=OUTPUT_COLUMNS, + train_metrics=[], + validation_metrics=[], + test_metrics=[], + evaluation_strategy="CrossValidationEvaluationStrategy", + splits=json.dumps( + { + "splitter_name": "KFoldSplitter", + "splitType": "cv", + "n_splits": 2, + "shuffle": True, + "random_state": 42, + "test_size": 0.3, + } + ), + ) + db.add(model_session) + db.commit() + db.refresh(model_session) + model_session_id = model_session.id + + response = client.post( + "/api/v1/run/", + json={ + "model_session_id": model_session_id, + "model_name": "DecisionTreeClassifier", + "name": "ReportsCVRun", + "parameters": { + "criterion": "gini", + "max_depth": 3, + "min_samples_split": 2, + "min_samples_leaf": 1, + "max_features": None, + "class_weight": None, + }, + "optimizer_name": "", + "optimizer_parameters": { + "n_trials": 1, + "sampler": "TPESampler", + "pruner": "None", + }, + "goal_metric": "", + "description": "Cross validated run under report", + "plot_history_path": "path/to/history.png", + "plot_slice_path": "path/to/slice.png", + "plot_contour_path": "path/to/contour.png", + "plot_importance_path": "path/to/importance.png", + }, + ) + assert response.status_code == 201, response.text + run_id = response.json()["id"] + + ModelJob(run_id=run_id).run() + with client.app.container["session_factory"]() as db: + assert db.get(Run, run_id).status == RunStatus.FINISHED + + yield run_id + + client.delete(f"/api/v1/run/{run_id}") + with session_factory() as db: + session = db.get(ModelSession, model_session_id) + if session: + db.delete(session) + db.commit() + + +def test_a_report_covers_a_cross_validated_run(client: TestClient, cv_run_id: int): + report_id = _create(client, cv_run_id, "ConfusionMatrix") + ReportJob(report_id=report_id).run() + + with client.app.container["session_factory"]() as db: + assert db.get(Report, report_id).status == ReportStatus.FINISHED + + artifacts = client.get(f"/api/v1/report/{report_id}/artifacts").json() + assert len(artifacts) == 1 + assert artifacts[0]["type"] == "grouped" + + titles = [group["title"] for group in artifacts[0]["groups"]] + assert titles == ["Train", "Test", "Whole dataset"] + + for group in artifacts[0]["groups"]: + leaf = group["artifacts"][0] + assert leaf["type"] == "plotly" + figure = json.loads(leaf["payload"]) + assert "Iris-setosa" in figure["data"][0]["x"] + + client.delete(f"/api/v1/report/{report_id}") + + +def test_cross_validated_partitions_match_the_explainer_flow( + client: TestClient, cv_run_id: int +): + offered = client.get(f"/api/v1/explainer/explainable-splits/{cv_run_id}").json() + offered_titles = [split["name"] for split in offered["splits"]] + assert offered_titles == ["train", "test", "all"] + + report_id = _create(client, cv_run_id, "PerClassBreakdown") + ReportJob(report_id=report_id).run() + + artifacts = client.get(f"/api/v1/report/{report_id}/artifacts").json() + titles = [group["title"] for group in artifacts[0]["groups"]] + assert titles == ["Train", "Test", "Whole dataset"] + + supports = [ + group["artifacts"][0]["payload"]["rows"][-1][-1] + for group in artifacts[0]["groups"] + ] + assert supports[0] + supports[1] == supports[2] + + client.delete(f"/api/v1/report/{report_id}") diff --git a/tests/back/api/test_reports_forecasting_api.py b/tests/back/api/test_reports_forecasting_api.py new file mode 100644 index 000000000..ec8f6d870 --- /dev/null +++ b/tests/back/api/test_reports_forecasting_api.py @@ -0,0 +1,212 @@ +"""End to end test for reports on a forecasting run. + +A forecasting model refuses to return a value for dates inside its training +window, so the report job must not ask it to predict the train partition. It +should behave exactly like the prediction flow and cover only the partitions +that lie after the fit. +""" + +import json +from pathlib import Path + +import pandas as pd +import pytest +from fastapi.testclient import TestClient + +from DashAI.back.core.enums.status import ReportStatus, RunStatus +from DashAI.back.dataloaders.classes.csv_dataloader import CSVDataLoader +from DashAI.back.dependencies.database.models import ( + Dataset, + ModelSession, + Report, + Run, +) +from DashAI.back.dependencies.registry import ComponentRegistry +from DashAI.back.evaluation.forecasting_holdout import ( + ForecastingHoldoutEvaluationStrategy, +) +from DashAI.back.job.dataset_job import DatasetJob +from DashAI.back.job.model_job import ModelJob +from DashAI.back.job.report_job import ReportJob +from DashAI.back.models.forecasting.naive import NaiveForecaster +from DashAI.back.optimizers.optuna_optimizer import OptunaOptimizer +from DashAI.back.reports.forecasting.forecast_vs_actual import ForecastVsActual +from DashAI.back.splitters.temporal_holdout import TemporalHoldoutSplitter +from DashAI.back.tasks.forecasting_task import ForecastingTask + + +@pytest.fixture(scope="module", name="test_registry", autouse=True) +def setup_test_registry(client): + container = client.app.container + sentinel = object() + services = container._services + old = services.get("component_registry", sentinel) + + services["component_registry"] = ComponentRegistry( + initial_components=[ + ForecastingTask, + NaiveForecaster, + CSVDataLoader, + ModelJob, + ReportJob, + OptunaOptimizer, + TemporalHoldoutSplitter, + ForecastingHoldoutEvaluationStrategy, + ForecastVsActual, + ] + ) + yield services["component_registry"] + if old is sentinel: + del services["component_registry"] + else: + services["component_registry"] = old + + +@pytest.fixture(scope="module", name="forecasting_dataset") +def create_forecasting_dataset(client: TestClient, test_path: Path): + """Create a daily linear series dataset through the real DatasetJob.""" + dates = pd.date_range("2024-01-01", periods=60, freq="D").strftime("%Y-%m-%d") + csv_path = Path(test_path) / "forecasting_report.csv" + pd.DataFrame({"date": dates, "value": [float(i) for i in range(60)]}).to_csv( + csv_path, index=False + ) + + session_factory = client.app.container["session_factory"] + with session_factory() as db: + entry = Dataset(name="forecasting_report", file_path="") + db.add(entry) + db.commit() + db.refresh(entry) + + kwargs = { + "dataset_id": entry.id, + "url": "", + "params": { + "dataloader": "CSVDataLoader", + "separator": ",", + "name": entry.name, + "schema": { + "date": {"type": "Date", "dtype": "%Y-%m-%d"}, + "value": {"type": "Float", "dtype": "float64"}, + }, + }, + "file_path": csv_path, + } + DatasetJob(job_type="DatasetJob", kwargs=kwargs, db=db).run() + db.refresh(entry) + + yield entry.id + + with session_factory() as cleanup: + entry = cleanup.get(Dataset, entry.id) + if entry: + cleanup.delete(entry) + cleanup.commit() + + +@pytest.fixture(scope="module", name="model_session_id") +def create_model_session(client: TestClient, forecasting_dataset: int): + session_factory = client.app.container["session_factory"] + with session_factory() as db: + model_session = ModelSession( + dataset_id=forecasting_dataset, + name="ForecastingReportsSession", + task_name="ForecastingTask", + input_columns=["date"], + output_columns=["value"], + train_metrics=[], + validation_metrics=[], + test_metrics=[], + evaluation_strategy="ForecastingHoldoutEvaluationStrategy", + splits=json.dumps( + { + "train": 0.6, + "test": 0.2, + "validation": 0.2, + "splitter_name": "TemporalHoldoutSplitter", + "splitType": "temporal", + } + ), + ) + db.add(model_session) + db.commit() + db.refresh(model_session) + + yield model_session.id + + db.delete(model_session) + db.commit() + + +@pytest.fixture(scope="module", name="trained_forecast_run_id") +def train_a_real_forecaster(client: TestClient, model_session_id: int, test_registry): + response = client.post( + "/api/v1/run/", + json={ + "model_session_id": model_session_id, + "model_name": "NaiveForecaster", + "name": "ForecastingReportsRun", + "parameters": {}, + "optimizer_name": "", + "optimizer_parameters": { + "n_trials": 1, + "sampler": "TPESampler", + "pruner": "None", + }, + "goal_metric": "", + "description": "Forecasting run under report", + "plot_history_path": "path/to/history.png", + "plot_slice_path": "path/to/slice.png", + "plot_contour_path": "path/to/contour.png", + "plot_importance_path": "path/to/importance.png", + }, + ) + assert response.status_code == 201, response.text + run_id = response.json()["id"] + + ModelJob(run_id=run_id).run() + with client.app.container["session_factory"]() as db: + assert db.get(Run, run_id).status == RunStatus.FINISHED + + yield run_id + + client.delete(f"/api/v1/run/{run_id}") + + +def test_a_forecasting_report_only_covers_forecastable_partitions( + client: TestClient, trained_forecast_run_id: int +): + """The train partition is a fit, not a forecast, so it must be skipped.""" + response = client.post( + "/api/v1/report/", + json={ + "run_id": trained_forecast_run_id, + "report_name": "ForecastVsActual", + "parameters": {}, + }, + ) + assert response.status_code == 201, response.text + report_id = response.json()["id"] + + ReportJob(report_id=report_id).run() + + with client.app.container["session_factory"]() as db: + assert db.get(Report, report_id).status == ReportStatus.FINISHED + + artifacts = client.get(f"/api/v1/report/{report_id}/artifacts").json() + assert len(artifacts) == 1 + assert artifacts[0]["type"] == "grouped" + + titles = [group["title"] for group in artifacts[0]["groups"]] + assert titles == ["Test", "Validation"] + + for group in artifacts[0]["groups"]: + leaf = group["artifacts"][0] + assert leaf["type"] == "plotly" + figure = json.loads(leaf["payload"]) + assert [trace["name"] for trace in figure["data"]] == [ + "Actual", + "Forecast", + ] + + client.delete(f"/api/v1/report/{report_id}") diff --git a/tests/back/reports/__init__.py b/tests/back/reports/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/back/reports/test_forecasting_reports.py b/tests/back/reports/test_forecasting_reports.py new file mode 100644 index 000000000..50e54f976 --- /dev/null +++ b/tests/back/reports/test_forecasting_reports.py @@ -0,0 +1,91 @@ +"""Unit tests for the forecasting evaluation reports.""" + +import json + +import numpy as np +import pytest + +from DashAI.back.core.artifacts import normalize_artifacts +from DashAI.back.reports.base_report import BaseReport +from DashAI.back.reports.forecasting.forecast_vs_actual import ForecastVsActual +from DashAI.back.reports.forecasting.residual_autocorrelation import ( + ResidualAutocorrelation, +) +from DashAI.back.reports.forecasting.residuals_over_time import ResidualsOverTime + + +@pytest.fixture +def forecasting_data(): + """A trending series and a forecast that tracks it with noise.""" + rng = np.random.default_rng(0) + steps = np.arange(30) + y_true = 5 + 2 * steps + rng.normal(0, 1, 30) + y_pred = y_true + rng.normal(0, 0.5, 30) + return y_true, y_pred + + +def _figure(artifact): + assert artifact.type == "plotly" + return json.loads(artifact.payload) + + +def test_forecast_vs_actual_has_actual_and_forecast_traces(forecasting_data): + y_true, y_pred = forecasting_data + + figure = _figure(ForecastVsActual().compute(y_true, y_pred)[0]) + + names = [trace["name"] for trace in figure["data"]] + assert names == ["Actual", "Forecast"] + # The observation axis is row order, since reports never see the dates. + assert list(figure["data"][0]["x"]) == list(range(len(y_true))) + assert list(figure["data"][0]["y"]) == pytest.approx(list(y_true)) + assert list(figure["data"][1]["y"]) == pytest.approx(list(y_pred)) + + +def test_residuals_over_time_centers_on_truth_minus_forecast(forecasting_data): + y_true, y_pred = forecasting_data + + figure = _figure(ResidualsOverTime().compute(y_true, y_pred)[0]) + + residual_trace = figure["data"][1] + assert residual_trace["mode"] == "markers" + assert np.mean(residual_trace["y"]) == pytest.approx( + np.mean(y_true - y_pred), abs=1e-9 + ) + assert len(residual_trace["x"]) == len(y_true) + + +def test_residual_autocorrelation_reports_each_lag(forecasting_data): + y_true, y_pred = forecasting_data + + figure = _figure(ResidualAutocorrelation(max_lag=5).compute(y_true, y_pred)[0]) + + bar = figure["data"][0] + assert bar["type"] == "bar" + assert list(bar["x"]) == [1, 2, 3, 4, 5] + + residuals = np.asarray(y_true - y_pred, dtype=float) + centered = residuals - residuals.mean() + denom = centered @ centered + expected = [(centered[k:] @ centered[:-k]) / denom for k in range(1, 6)] + assert list(bar["y"]) == pytest.approx(expected, abs=1e-9) + + +def test_every_forecasting_report_output_normalizes(forecasting_data): + y_true, y_pred = forecasting_data + + outputs = [ + ForecastVsActual().compute(y_true, y_pred), + ResidualsOverTime().compute(y_true, y_pred), + ResidualAutocorrelation().compute(y_true, y_pred), + ] + for output in outputs: + for item in normalize_artifacts(output): + assert item["type"] in {"plotly", "table", "text", "image", "grouped"} + + +def test_forecasting_reports_are_registered_for_the_task(): + for report in (ForecastVsActual, ResidualsOverTime, ResidualAutocorrelation): + assert report.TYPE == "Report" + assert issubclass(report, BaseReport) + assert report.COMPATIBLE_COMPONENTS == ["ForecastingTask"] diff --git a/tests/back/reports/test_reports.py b/tests/back/reports/test_reports.py new file mode 100644 index 000000000..f7a6789bf --- /dev/null +++ b/tests/back/reports/test_reports.py @@ -0,0 +1,222 @@ +"""Unit tests for the concrete evaluation reports.""" + +import json + +import numpy as np +import pytest + +from DashAI.back.core.artifacts import normalize_artifacts +from DashAI.back.reports.base_report import ( + BaseReport, + ReportError, + as_labels, + resolve_class_names, +) +from DashAI.back.reports.classification.confusion_matrix import ConfusionMatrix +from DashAI.back.reports.classification.per_class_breakdown import ( + PerClassBreakdown, +) +from DashAI.back.reports.classification.precision_recall_curve import ( + PrecisionRecallCurve, +) +from DashAI.back.reports.classification.roc_curve import RocCurve +from DashAI.back.reports.regression.predicted_vs_actual import PredictedVsActual +from DashAI.back.reports.regression.residual_histogram import ResidualHistogram +from DashAI.back.reports.regression.residual_plot import ResidualPlot + +CLASS_NAMES = ["setosa", "versicolor", "virginica"] + + +@pytest.fixture +def classification_data(): + """Three classes, probabilities that mostly agree with the truth.""" + rng = np.random.default_rng(0) + y_true = np.array([0, 0, 1, 1, 2, 2, 0, 1, 2, 1]) + probabilities = rng.random((len(y_true), 3)) + # Bias each row toward its true class so the curves are non degenerate. + probabilities[np.arange(len(y_true)), y_true] += 2.0 + probabilities /= probabilities.sum(axis=1, keepdims=True) + return y_true, probabilities + + +@pytest.fixture +def regression_data(): + rng = np.random.default_rng(0) + y_true = rng.normal(10, 3, 50) + return y_true, y_true + rng.normal(0, 1, 50) + + +def _figure(artifact): + assert artifact.type == "plotly" + return json.loads(artifact.payload) + + +def test_as_labels_argmaxes_a_probability_matrix(): + assert as_labels(np.array([[0.1, 0.9], [0.8, 0.2]])).tolist() == [1, 0] + assert as_labels(np.array([1, 0])).tolist() == [1, 0] + + +def test_resolve_class_names_fills_gaps(): + assert resolve_class_names(["a"], 3) == ["a", "1", "2"] + assert resolve_class_names(None, 2) == ["0", "1"] + + +def test_confusion_matrix_counts_every_pair(classification_data): + y_true, probabilities = classification_data + + artifacts = ConfusionMatrix().compute(y_true, probabilities, CLASS_NAMES) + figure = _figure(artifacts[0]) + + assert figure["data"][0]["type"] == "heatmap" + assert list(figure["data"][0]["x"]) == CLASS_NAMES + # Raw counts must add up to the number of samples. + total = sum(sum(row) for row in figure["data"][0]["z"]) + assert total == len(y_true) + + +def test_confusion_matrix_row_normalizes(classification_data): + y_true, probabilities = classification_data + + figure = _figure( + ConfusionMatrix(normalize="true").compute(y_true, probabilities, CLASS_NAMES)[0] + ) + + for row in figure["data"][0]["z"]: + assert sum(row) == pytest.approx(1.0) + + +def test_roc_curve_has_a_trace_per_class_plus_chance(classification_data): + y_true, probabilities = classification_data + + figure = _figure(RocCurve().compute(y_true, probabilities, CLASS_NAMES)[0]) + + assert len(figure["data"]) == len(CLASS_NAMES) + 1 + assert "AUC" in figure["data"][1]["name"] + + +def test_roc_curve_draws_one_curve_for_a_binary_problem(): + y_true = np.array([0, 1, 0, 1]) + probabilities = np.array([[0.8, 0.2], [0.3, 0.7], [0.6, 0.4], [0.2, 0.8]]) + + figure = _figure(RocCurve().compute(y_true, probabilities, ["no", "yes"])[0]) + + # Chance line plus a single curve, not two mirrored ones. + assert len(figure["data"]) == 2 + + +def test_roc_curve_refuses_hard_labels(): + with pytest.raises(ReportError, match="class probabilities"): + RocCurve().compute(np.array([0, 1]), np.array([0, 1]), ["no", "yes"]) + + +def test_roc_curve_refuses_a_single_class_split(): + probabilities = np.array([[0.9, 0.1], [0.8, 0.2]]) + with pytest.raises(ReportError, match="single class"): + RocCurve().compute(np.array([0, 0]), probabilities, ["no", "yes"]) + + +def test_precision_recall_curve_annotates_average_precision(classification_data): + y_true, probabilities = classification_data + + figure = _figure( + PrecisionRecallCurve().compute(y_true, probabilities, CLASS_NAMES)[0] + ) + + assert len(figure["data"]) == len(CLASS_NAMES) + assert "AP" in figure["data"][0]["name"] + + +def test_per_class_breakdown_has_a_row_per_class_and_averages(classification_data): + y_true, probabilities = classification_data + + artifacts = PerClassBreakdown().compute(y_true, probabilities, CLASS_NAMES) + + assert artifacts[0].type == "table" + payload = artifacts[0].payload + assert payload.columns == ["Class", "Precision", "Recall", "F1", "Support"] + assert len(payload.rows) == len(CLASS_NAMES) + 2 + assert payload.rows[-2][0] == "macro avg" + assert payload.rows[-1][0] == "weighted avg" + + +def test_predicted_vs_actual_draws_the_identity_line(regression_data): + y_true, y_pred = regression_data + + figure = _figure(PredictedVsActual().compute(y_true, y_pred)[0]) + + modes = [trace.get("mode") for trace in figure["data"]] + assert "lines" in modes + assert "markers" in modes + + +def test_residual_plot_centers_on_zero(regression_data): + y_true, y_pred = regression_data + + figure = _figure(ResidualPlot().compute(y_true, y_pred)[0]) + + residual_trace = figure["data"][1] + assert np.mean(residual_trace["y"]) == pytest.approx( + np.mean(y_true - y_pred), abs=1e-9 + ) + + +def test_residual_histogram_honours_its_bin_count(regression_data): + y_true, y_pred = regression_data + + figure = _figure(ResidualHistogram(bins=12).compute(y_true, y_pred)[0]) + + assert figure["data"][0]["type"] == "histogram" + assert figure["data"][0]["nbinsx"] == 12 + + +def test_every_report_output_normalizes(classification_data, regression_data): + y_true, probabilities = classification_data + reg_true, reg_pred = regression_data + + outputs = [ + ConfusionMatrix().compute(y_true, probabilities, CLASS_NAMES), + RocCurve().compute(y_true, probabilities, CLASS_NAMES), + PrecisionRecallCurve().compute(y_true, probabilities, CLASS_NAMES), + PerClassBreakdown().compute(y_true, probabilities, CLASS_NAMES), + PredictedVsActual().compute(reg_true, reg_pred), + ResidualPlot().compute(reg_true, reg_pred), + ResidualHistogram().compute(reg_true, reg_pred), + ] + for output in outputs: + for item in normalize_artifacts(output): + assert item["type"] in {"plotly", "table", "text", "image", "grouped"} + + +def test_reports_declare_their_probability_requirement(): + assert RocCurve.REQUIRES_PROBABILITIES is True + assert PrecisionRecallCurve.REQUIRES_PROBABILITIES is True + assert ConfusionMatrix.REQUIRES_PROBABILITIES is False + assert ResidualPlot.REQUIRES_PROBABILITIES is False + assert RocCurve.get_metadata()["requires_probabilities"] is True + + +def test_reports_are_registered_under_one_type(): + """The registry keys every report under the same component type.""" + for report in ( + ConfusionMatrix, + RocCurve, + PrecisionRecallCurve, + PerClassBreakdown, + PredictedVsActual, + ResidualPlot, + ResidualHistogram, + ): + assert report.TYPE == "Report" + assert issubclass(report, BaseReport) + assert report.COMPATIBLE_COMPONENTS + + +def test_a_report_never_receives_model_inputs(): + """Reports compare predictions against the truth and nothing else. + + Taking X would make it an explainer, so the contract must not offer one. + """ + import inspect + + parameters = inspect.signature(BaseReport.compute).parameters + assert list(parameters) == ["self", "y_true", "y_pred", "class_names"] diff --git a/tests/back/reports/test_translation_reports.py b/tests/back/reports/test_translation_reports.py new file mode 100644 index 000000000..b78221570 --- /dev/null +++ b/tests/back/reports/test_translation_reports.py @@ -0,0 +1,131 @@ +"""Unit tests for the translation evaluation reports.""" + +import json + +import numpy as np +import pytest + +from DashAI.back.core.artifacts import normalize_artifacts +from DashAI.back.reports.base_report import BaseReport +from DashAI.back.reports.translation.length_comparison import LengthComparison +from DashAI.back.reports.translation.per_segment_comparison import ( + PerSegmentComparison, +) +from DashAI.back.reports.translation.segment_score_distribution import ( + SegmentScoreDistribution, +) + +REFERENCE = [ + "the cat sat on the mat", + "good morning everyone", + "the quick brown fox jumps over the lazy dog", + "please close the window", + "i would like a cup of coffee", +] +HYPOTHESIS = [ + "the cat sat on the mat", + "good morning everyone", + "the quick brown fox jumps over the lazy dog", + "please open the window", + "i would like a cup of tea", +] + + +@pytest.fixture +def translation_data(): + """Two exact matches, two imperfect ones and one nearly unrelated.""" + return np.array(REFERENCE), np.array(HYPOTHESIS) + + +def _figure(artifact): + assert artifact.type == "plotly" + return json.loads(artifact.payload) + + +def test_per_segment_comparison_has_a_row_per_segment_plus_average( + translation_data, +): + y_true, y_pred = translation_data + + artifacts = PerSegmentComparison(highlight_count=2).compute(y_true, y_pred) + + assert artifacts[0].type == "table" + payload = artifacts[0].payload + assert payload.columns == ["#", "Reference", "Translation", "Score"] + assert len(payload.rows) == len(y_true) + 1 + assert payload.rows[-1][0] == "average" + + +def test_per_segment_comparison_scores_an_exact_match_100(translation_data): + y_true, y_pred = translation_data + + artifacts = PerSegmentComparison().compute(y_true, y_pred) + rows = artifacts[0].payload.rows + + exact = [row for row in rows[: len(y_true)] if row[1] == row[2]] + assert exact + for row in exact: + assert row[3] == pytest.approx(100.0, abs=1e-6) + + +def test_per_segment_comparison_highlights_the_worst_segments(translation_data): + y_true, y_pred = translation_data + + artifacts = PerSegmentComparison(highlight_count=2).compute(y_true, y_pred) + payload = artifacts[0].payload + + highlighted_rows = [cell.row for cell in payload.highlight] + assert len(highlighted_rows) == 2 + assert all(cell.column == 3 for cell in payload.highlight) + + segment_rows = payload.rows[: len(y_true)] + worst = sorted(range(len(segment_rows)), key=lambda i: segment_rows[i][3])[:2] + assert sorted(highlighted_rows) == sorted(worst) + + +def test_segment_score_distribution_is_a_histogram(translation_data): + y_true, y_pred = translation_data + + figure = _figure(SegmentScoreDistribution(bins=8).compute(y_true, y_pred)[0]) + + assert figure["data"][0]["type"] == "histogram" + assert figure["data"][0]["nbinsx"] == 8 + # Scores sit on the 0-100 scale. + assert min(figure["data"][0]["x"]) >= 0 + assert max(figure["data"][0]["x"]) <= 100 + + +def test_length_comparison_has_identity_line_and_points(translation_data): + y_true, y_pred = translation_data + + figure = _figure(LengthComparison().compute(y_true, y_pred)[0]) + + modes = [trace.get("mode") for trace in figure["data"]] + assert "lines" in modes + assert "markers" in modes + assert list(figure["data"][1]["x"]) == [len(str(sample)) for sample in y_true] + assert list(figure["data"][1]["y"]) == [len(str(sample)) for sample in y_pred] + + +def test_every_translation_report_output_normalizes(translation_data): + y_true, y_pred = translation_data + + outputs = [ + PerSegmentComparison().compute(y_true, y_pred), + SegmentScoreDistribution().compute(y_true, y_pred), + LengthComparison().compute(y_true, y_pred), + ] + for output in outputs: + for item in normalize_artifacts(output): + assert item["type"] in {"plotly", "table", "text", "image", "grouped"} + + +def test_translation_reports_are_registered_for_the_task(): + for report in ( + PerSegmentComparison, + SegmentScoreDistribution, + LengthComparison, + ): + assert report.TYPE == "Report" + assert issubclass(report, BaseReport) + assert report.COMPATIBLE_COMPONENTS == ["TranslationTask"]