diff --git a/README.md b/README.md index f543aec5..323fee13 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,47 @@ Both endpoints are public (no `X-Internal-Secret` required). A coverage test (`backend/tests/test_openapi_coverage.py`) asserts every registered route is documented, so the spec never drifts from the code. +--- +## 🧾 Model Registry & Provenance + +The Flask ML API records **which** model bytes it is serving, so a deployed or +hot-reloaded model can be identified and a prediction can be traced back to the +exact artifacts that produced it (issue #1007). Fingerprints are computed by +`backend/model_registry.py` (`build_metadata`) over the classifier `model`, +`vectorizer` and `label_encoder`: each artifact's SHA-256, size and mtime, plus +the optional human-authored fields (`trained_at`, `metrics`, `labels`) read from +a `model_card.json` sitting next to the model when present. + +### `GET /model-info` + +Public (no `X-Internal-Secret` required). Reports the live model set — its +`version` (mirrors `/model-status` and increments on every `/reload-model`), the +per-artifact `checksums`, and the full `metadata`: + +```json +{ + "version": 3, + "checksums": { + "model": "9f2b…", + "vectorizer": "1c7a…", + "label_encoder": "e004…" + }, + "metadata": { + "model": {"path": "…/linear_svm_model.pkl", "sha256": "9f2b…", "size_bytes": 24576, "mtime": 1753000000.0}, + "vectorizer": {"path": "…/tfidf_vectorizer.pkl", "sha256": "1c7a…", "size_bytes": 81920, "mtime": 1753000000.0}, + "label_encoder": {"path": "…/label_encoder.pkl", "sha256": "e004…", "size_bytes": 512, "mtime": 1753000000.0}, + "short_checksum": "9f2b1a0c4d5e", + "trained_at": "2026-07-20T12:00:00Z", + "metrics": {"accuracy": 0.98}, + "labels": ["ham", "spam", "smishing"] + } +} +``` + +`metadata` is `null` when no provenance is available (e.g. a test harness that +installs bare fakes). Dropping a `model_card.json` next to the model artifact is +the only step needed to populate `trained_at` / `metrics` / `labels`. + --- ## System Stability & Environment Fixes This update addresses critical runtime issues that prevented the system from executing in the local development environment: diff --git a/backend/api.py b/backend/api.py index 7cca6f00..4494ac30 100644 --- a/backend/api.py +++ b/backend/api.py @@ -106,6 +106,7 @@ "/openapi.json", "/docs", "/metrics", + "/model-info", } @@ -395,9 +396,19 @@ def handle_internal_error(e): # shared, thread-safe holder so POST /reload-model can atomically hot-swap in a # freshly retrained model without a restart (issue #973); handlers read from # serving_state.STATE rather than these module globals so the swap is visible. +import model_registry import serving_state +def _build_model_metadata(): + """Fingerprint the currently-on-disk classifier artifacts (issue #1007).""" + return model_registry.build_metadata( + model_path=str(MODEL_PATH), + vectorizer_path=str(VECTORIZER_PATH), + label_encoder_path=str(LABEL_ENCODER_PATH), + ) + + def _load_serving_objects(): """Reload the serving object set from disk (used by /reload-model).""" fresh_model = joblib.load(MODEL_PATH) @@ -413,6 +424,7 @@ def _load_serving_objects(): "vectorizer": fresh_vectorizer, "label_encoder": fresh_label_encoder, "xai_service": fresh_xai_service, + "metadata": _build_model_metadata(), } @@ -422,6 +434,7 @@ def _load_serving_objects(): label_encoder=label_encoder, xai_service=xai_service, loader=_load_serving_objects, + metadata=_build_model_metadata(), ) @@ -825,6 +838,26 @@ def swagger_ui(): return _SWAGGER_UI_HTML, 200, {"Content-Type": "text/html; charset=utf-8"} +@app.route("/model-info", methods=["GET"]) +@validate_request +def model_info(): + """Provenance for the currently served model set (issue #1007). + + Public (see PUBLIC_PATHS): reports only artifact checksums, sizes and the + optional model-card fields, never any secret or message content. The + ``version`` mirrors ``/model-status`` and increments on each ``/reload-model``. + """ + snapshot = serving_state.STATE.snapshot() + metadata = snapshot.metadata + return jsonify( + { + "version": snapshot.version, + "checksums": metadata.checksums if metadata is not None else {}, + "metadata": metadata.to_dict() if metadata is not None else None, + } + ) + + # ============================================ # PREDICT ROUTE (Protected) # ============================================ diff --git a/backend/model_registry.py b/backend/model_registry.py new file mode 100644 index 00000000..1b39b796 --- /dev/null +++ b/backend/model_registry.py @@ -0,0 +1,180 @@ +"""Model registry & provenance metadata for the Flask ML API (issue #1007). + +The API serves a triple of artifacts -- the classifier ``model``, its +``vectorizer`` and the ``label_encoder`` -- that ``retrain.py`` overwrites and +``/reload-model`` hot-swaps. Until now nothing recorded *which* bytes were +loaded, so an operator could not tell one deployed model from another or prove +that a reload actually changed anything. + +This module fingerprints those artifacts. :func:`build_metadata` reads each +``.pkl`` and captures its SHA-256, size and mtime, and -- when a +``model_card.json`` sits next to the model -- folds in the human-authored +provenance fields (``trained_at``, ``metrics``, ``labels``). The immutable +:class:`ModelMetadata` it returns is stored alongside the serving objects in +``serving_state`` and surfaced at ``GET /model-info``; its +:attr:`ModelMetadata.short_checksum` tags predictions and reload audit logs. + +>>> a = ArtifactInfo(path="m.pkl", sha256="aa" * 32, size_bytes=1, mtime=0.0) +>>> b = ArtifactInfo(path="v.pkl", sha256="bb" * 32, size_bytes=1, mtime=0.0) +>>> c = ArtifactInfo(path="l.pkl", sha256="cc" * 32, size_bytes=1, mtime=0.0) +>>> meta = ModelMetadata(model=a, vectorizer=b, label_encoder=c) +>>> meta.short_checksum +'aaaaaaaaaaaa' +>>> meta.checksums["vectorizer"] == "bb" * 32 +True +""" + +from __future__ import annotations + +from dataclasses import dataclass +import hashlib +import json +from pathlib import Path + +__all__ = ["ArtifactInfo", "ModelMetadata", "build_metadata", "MODEL_CARD_FILENAME"] + +# Sidecar looked for next to the model artifact; absence is not an error. +MODEL_CARD_FILENAME = "model_card.json" + +# A 12-hex-char prefix is enough to tell deployed models apart in logs and +# prediction payloads without carrying the full 64-char digest everywhere. +SHORT_CHECKSUM_LENGTH = 12 + +# Hash artifacts incrementally so a large .pkl never has to be held in memory. +_HASH_CHUNK_SIZE = 1 << 20 + + +@dataclass(frozen=True, slots=True) +class ArtifactInfo: + """Content fingerprint of one on-disk model artifact. + + >>> info = ArtifactInfo(path="m.pkl", sha256="ab" * 32, size_bytes=10, mtime=1.5) + >>> info.short_sha256 + 'abababababab' + """ + + path: str + sha256: str + size_bytes: int + mtime: float + + @property + def short_sha256(self) -> str: + return self.sha256[:SHORT_CHECKSUM_LENGTH] + + def to_dict(self) -> dict: + return { + "path": self.path, + "sha256": self.sha256, + "size_bytes": self.size_bytes, + "mtime": self.mtime, + } + + +@dataclass(frozen=True, slots=True) +class ModelMetadata: + """Provenance snapshot of the served artifact triple plus optional card. + + The three :class:`ArtifactInfo` members are always present; the sidecar + fields (``trained_at``, ``metrics``, ``labels``) are ``None`` when no + ``model_card.json`` accompanies the model, and otherwise carry whatever that + file supplied. + """ + + model: ArtifactInfo + vectorizer: ArtifactInfo + label_encoder: ArtifactInfo + trained_at: str | None = None + metrics: dict | None = None + labels: list | None = None + + @property + def short_checksum(self) -> str: + """Abbreviated model hash used to tag predictions and reload audit logs.""" + return self.model.short_sha256 + + @property + def checksums(self) -> dict: + """Full SHA-256 per artifact, keyed by role.""" + return { + "model": self.model.sha256, + "vectorizer": self.vectorizer.sha256, + "label_encoder": self.label_encoder.sha256, + } + + def to_dict(self) -> dict: + return { + "model": self.model.to_dict(), + "vectorizer": self.vectorizer.to_dict(), + "label_encoder": self.label_encoder.to_dict(), + "short_checksum": self.short_checksum, + "trained_at": self.trained_at, + "metrics": self.metrics, + "labels": self.labels, + } + + +def build_metadata( + *, + model_path: str, + vectorizer_path: str, + label_encoder_path: str, + model_card_path: str | None = None, +) -> ModelMetadata: + """Fingerprint the three artifacts and fold in the sidecar model card. + + ``model_card_path`` defaults to a ``model_card.json`` next to the model; + when the card is absent or unreadable the provenance fields stay ``None``. + The artifact files themselves must exist -- a missing ``.pkl`` is a + misconfiguration and surfaces as the usual ``OSError``. + """ + card_path = ( + Path(model_card_path) + if model_card_path is not None + else Path(model_path).parent / MODEL_CARD_FILENAME + ) + card = _read_model_card(card_path) + + return ModelMetadata( + model=_describe_artifact(model_path), + vectorizer=_describe_artifact(vectorizer_path), + label_encoder=_describe_artifact(label_encoder_path), + trained_at=card.get("trained_at"), + metrics=card.get("metrics"), + labels=card.get("labels"), + ) + + +def _describe_artifact(path: str) -> ArtifactInfo: + p = Path(path) + stat = p.stat() + return ArtifactInfo( + path=str(p), + sha256=_sha256(p), + size_bytes=stat.st_size, + mtime=stat.st_mtime, + ) + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(_HASH_CHUNK_SIZE), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _read_model_card(path: Path) -> dict: + """Return the sidecar card as a dict, or ``{}`` when it is absent/unreadable. + + A malformed or non-object card is treated as absent rather than fatal: model + provenance is advisory metadata and must never stop the model from serving. + """ + if not path.exists(): + return {} + try: + with open(path, "r", encoding="utf-8") as fh: + data = json.load(fh) + except (OSError, ValueError): + return {} + return data if isinstance(data, dict) else {} diff --git a/backend/openapi_spec.py b/backend/openapi_spec.py index fd315644..4edde0c6 100644 --- a/backend/openapi_spec.py +++ b/backend/openapi_spec.py @@ -901,11 +901,78 @@ def _extended_paths(): }, } }, + "/model-info": { + "get": { + "summary": "Provenance for the currently served model", + "operationId": "getModelInfo", + "tags": ["System"], + "security": [], + "responses": { + "200": _json_response( + "Live model version, per-artifact checksums and the " + "optional model-card metadata.", + {"$ref": "#/components/schemas/ModelInfo"}, + ) + }, + } + }, } def _extended_schemas(): return { + "ArtifactInfo": { + "type": "object", + "description": "Content fingerprint of one model artifact.", + "properties": { + "path": {"type": "string"}, + "sha256": {"type": "string"}, + "size_bytes": {"type": "integer"}, + "mtime": {"type": "number"}, + }, + }, + "ModelInfo": { + "type": "object", + "description": ( + "Provenance of the served model set (issue #1007). `version` " + "increments on each /reload-model; `checksums` maps each " + "artifact role to its SHA-256; `metadata` carries per-artifact " + "fingerprints plus the optional model-card fields, and is null " + "when no provenance is available." + ), + "properties": { + "version": {"type": "integer"}, + "checksums": { + "type": "object", + "properties": { + "model": {"type": "string"}, + "vectorizer": {"type": "string"}, + "label_encoder": {"type": "string"}, + }, + }, + "metadata": { + "type": "object", + "nullable": True, + "properties": { + "model": {"$ref": "#/components/schemas/ArtifactInfo"}, + "vectorizer": {"$ref": "#/components/schemas/ArtifactInfo"}, + "label_encoder": {"$ref": "#/components/schemas/ArtifactInfo"}, + "short_checksum": {"type": "string"}, + "trained_at": {"type": "string", "nullable": True}, + "metrics": { + "type": "object", + "nullable": True, + "additionalProperties": True, + }, + "labels": { + "type": "array", + "nullable": True, + "items": {"type": "string"}, + }, + }, + }, + }, + }, "AuthUrlResponse": { "type": "object", "properties": {"auth_url": {"type": "string"}}, diff --git a/backend/serving_state.py b/backend/serving_state.py index 6cfebd53..c118a1e7 100644 --- a/backend/serving_state.py +++ b/backend/serving_state.py @@ -13,24 +13,34 @@ under that same lock, so a concurrent reload can never expose a half-updated set (e.g. a new model paired with the old vectorizer). +Each snapshot also carries the :class:`~model_registry.ModelMetadata` describing +the artifacts currently loaded, refreshed on every reload so ``/model-info`` and +the per-prediction provenance fields report the live model. + >>> state = ServingState( ... model="m1", vectorizer="v1", label_encoder="l1", xai_service="x1", +... metadata="meta1", ... loader=lambda: {"model": "m2", "vectorizer": "v2", -... "label_encoder": "l2", "xai_service": "x2"}, +... "label_encoder": "l2", "xai_service": "x2", +... "metadata": "meta2"}, ... ) >>> state.snapshot().version 1 +>>> state.snapshot().metadata +'meta1' >>> state.reload().version 2 >>> state.snapshot().model 'm2' +>>> state.snapshot().metadata +'meta2' """ from __future__ import annotations +from dataclasses import dataclass import threading -from dataclasses import dataclass -from typing import Any, Callable +from typing import Any, Callable __all__ = ["ServingSnapshot", "ServingState", "STATE", "init_state"] @@ -44,10 +54,14 @@ class ServingSnapshot: label_encoder: Any xai_service: Any version: int + # Provenance for the loaded artifacts (model_registry.ModelMetadata). Defaults + # to None so callers/tests that build a snapshot without provenance still work. + metadata: Any = None # A loader returns the freshly loaded objects (from disk) as a mapping with the -# keys "model", "vectorizer", "label_encoder" and "xai_service". +# keys "model", "vectorizer", "label_encoder", "xai_service" and, optionally, +# "metadata" (the recomputed model_registry.ModelMetadata for the fresh set). Loader = Callable[[], dict] @@ -62,6 +76,7 @@ def __init__( label_encoder: Any, xai_service: Any, loader: Loader, + metadata: Any = None, ) -> None: self._lock = threading.RLock() self._model = model @@ -69,6 +84,7 @@ def __init__( self._label_encoder = label_encoder self._xai_service = xai_service self._loader = loader + self._metadata = metadata self._version = 1 def snapshot(self) -> ServingSnapshot: @@ -79,6 +95,7 @@ def snapshot(self) -> ServingSnapshot: self._label_encoder, self._xai_service, self._version, + self._metadata, ) def reload(self) -> ServingSnapshot: @@ -95,6 +112,9 @@ def reload(self) -> ServingSnapshot: self._vectorizer = fresh["vectorizer"] self._label_encoder = fresh["label_encoder"] self._xai_service = fresh["xai_service"] + # Loaders may omit "metadata" (e.g. lightweight test fakes); fall back + # to None rather than requiring every loader to supply provenance. + self._metadata = fresh.get("metadata") self._version += 1 return ServingSnapshot( self._model, @@ -102,6 +122,7 @@ def reload(self) -> ServingSnapshot: self._label_encoder, self._xai_service, self._version, + self._metadata, ) @property @@ -123,6 +144,7 @@ def init_state( label_encoder: Any, xai_service: Any, loader: Loader, + metadata: Any = None, ) -> ServingState: """Install the process-wide serving state and return it.""" global STATE @@ -132,5 +154,6 @@ def init_state( label_encoder=label_encoder, xai_service=xai_service, loader=loader, + metadata=metadata, ) return STATE diff --git a/backend/tests/test_model_registry.py b/backend/tests/test_model_registry.py new file mode 100644 index 00000000..0213d1f5 --- /dev/null +++ b/backend/tests/test_model_registry.py @@ -0,0 +1,142 @@ +"""Coverage for the model registry and /model-info (issue #1007, part 1). + +The pure ``build_metadata`` tests fingerprint temp files and need no ML deps. +The /model-info shape test imports ``api`` lazily and skips when the app can't +be imported (e.g. an environment missing an optional model/threat-intel dep). +""" + +import hashlib +import json +import os +from pathlib import Path +import sys + +import pytest + +BASE_DIR = Path(__file__).resolve().parents[2] +BACKEND_DIR = BASE_DIR / "backend" + +os.environ.setdefault("MODEL_PATH", str(BASE_DIR / "linear_svm_model.pkl")) +os.environ.setdefault("VECTORIZER_PATH", str(BACKEND_DIR / "tfidf_vectorizer.pkl")) +os.environ.setdefault("LABEL_ENCODER_PATH", str(BASE_DIR / "label_encoder.pkl")) +os.environ.setdefault("URL_MODEL_PATH", str(BACKEND_DIR / "url_detector.pkl")) +os.environ.setdefault("URL_VECTORIZER_PATH", str(BACKEND_DIR / "url_vectorizer.pkl")) + +sys.path.insert(0, str(BACKEND_DIR)) + +import model_registry # noqa: E402 + + +def _make_artifacts(root, *, model=b"model-v1", vec=b"vec-v1", le=b"le-v1"): + """Write the three artifact files under ``root`` and return their paths.""" + model_path = root / "linear_svm_model.pkl" + vec_path = root / "tfidf_vectorizer.pkl" + le_path = root / "label_encoder.pkl" + model_path.write_bytes(model) + vec_path.write_bytes(vec) + le_path.write_bytes(le) + return model_path, vec_path, le_path + + +def _build(model_path, vec_path, le_path): + return model_registry.build_metadata( + model_path=str(model_path), + vectorizer_path=str(vec_path), + label_encoder_path=str(le_path), + ) + + +class TestBuildMetadata: + def test_checksums_match_sha256_and_are_stable(self, tmp_path): + model_path, vec_path, le_path = _make_artifacts(tmp_path) + + first = _build(model_path, vec_path, le_path) + second = _build(model_path, vec_path, le_path) + + assert first.model.sha256 == hashlib.sha256(b"model-v1").hexdigest() + # Identical bytes -> identical checksums across independent builds. + assert first.checksums == second.checksums + assert first.short_checksum == first.model.sha256[:12] + assert first.model.size_bytes == len(b"model-v1") + + def test_checksum_varies_when_bytes_change(self, tmp_path): + model_path, vec_path, le_path = _make_artifacts(tmp_path) + before = _build(model_path, vec_path, le_path) + + model_path.write_bytes(b"model-v2-retrained") + after = _build(model_path, vec_path, le_path) + + assert after.model.sha256 != before.model.sha256 + assert after.short_checksum != before.short_checksum + # Untouched artifacts keep their fingerprints. + assert after.vectorizer.sha256 == before.vectorizer.sha256 + + def test_reads_model_card_sidecar_when_present(self, tmp_path): + model_path, vec_path, le_path = _make_artifacts(tmp_path) + card = { + "trained_at": "2026-07-20T12:00:00Z", + "metrics": {"accuracy": 0.98}, + "labels": ["ham", "spam", "smishing"], + } + (tmp_path / model_registry.MODEL_CARD_FILENAME).write_text(json.dumps(card)) + + meta = _build(model_path, vec_path, le_path) + + assert meta.trained_at == "2026-07-20T12:00:00Z" + assert meta.metrics == {"accuracy": 0.98} + assert meta.labels == ["ham", "spam", "smishing"] + + def test_missing_model_card_leaves_fields_none(self, tmp_path): + meta = _build(*_make_artifacts(tmp_path)) + + assert meta.trained_at is None + assert meta.metrics is None + assert meta.labels is None + + def test_malformed_model_card_is_tolerated(self, tmp_path): + model_path, vec_path, le_path = _make_artifacts(tmp_path) + (tmp_path / model_registry.MODEL_CARD_FILENAME).write_text("{ not json") + + meta = _build(model_path, vec_path, le_path) + + # Provenance is advisory; a broken card must not break fingerprinting. + assert meta.trained_at is None + assert meta.checksums["model"] == hashlib.sha256(b"model-v1").hexdigest() + + def test_to_dict_is_json_serialisable_and_complete(self, tmp_path): + meta = _build(*_make_artifacts(tmp_path)) + as_dict = meta.to_dict() + + json.dumps(as_dict) # must not raise + assert set(as_dict) >= { + "model", + "vectorizer", + "label_encoder", + "short_checksum", + "trained_at", + "metrics", + "labels", + } + + +@pytest.fixture +def client(): + try: + import api as api_module # noqa: E402 + except Exception as exc: # pragma: no cover - env without ML deps/models + pytest.skip(f"api import unavailable: {exc}") + api_module.app.config["TESTING"] = True + with api_module.app.test_client() as c: + yield c + + +class TestModelInfoEndpoint: + def test_model_info_is_public_and_well_shaped(self, client): + res = client.get("/model-info") + assert res.status_code == 200 + + body = res.get_json() + assert set(body) == {"version", "checksums", "metadata"} + assert isinstance(body["version"], int) + assert set(body["checksums"]) == {"model", "vectorizer", "label_encoder"} + assert body["metadata"]["short_checksum"] == body["checksums"]["model"][:12]