feat(ai-208): run new AI review logic in shadow mode before broad rollout

apps/api/prisma/schema.prisma

@@ -331,6 +331,25 @@ model PointReservation {
   @@map("point_reservations")
 }
 
+/// AI-201: Versioned registry entry for prompts, policies, thresholds, and model versions used in AI-assisted appeal decisions.
+model PromptPolicyEntry {
+  id          String   @id @default(cuid())
+  /// Logical key, e.g. "appeal_evaluation_prompt", "rejection_threshold"
+  key         String
+  version     Int
+  kind        String   /// prompt | policy | threshold | model_version
+  content     String   /// Raw prompt text or serialised JSON value
+  isActive    Boolean  @default(false) @map("is_active")
+  publishedBy String?  @map("published_by")
+  notes       String?
+  createdAt   DateTime @default(now()) @map("created_at")
+
+  @@unique([key, version])
+  @@index([key, isActive])
+  @@index([kind])
+  @@map("prompt_policy_entries")
+}
+
 /// BE-204: Budget-impacting event log for analytics and dashboards.
 model BudgetEvent {
   id              String    @id @default(cuid())

apps/api/src/app.module.ts

@@ -17,6 +17,9 @@ import { ReviewContextModule } from "./modules/review-context/review-context.mod
 import { BackgroundJobModule } from "./modules/jobs/background-job.module.js";
 import { WaveModule } from "./modules/wave/wave.module.js";
 import { BudgetModule } from "./modules/budget/budget.module.js";
+import { PromptPolicyModule } from "./modules/prompt-policy/prompt-policy.module.js";
+import { RetentionModule } from "./modules/retention/retention.module.js";
+import { WaveConfigModule } from "./modules/wave-config/wave-config.module.js";
 
 @Module({
   imports: [
@@ -40,6 +43,10 @@ import { BudgetModule } from "./modules/budget/budget.module.js";
     ReviewContextModule,
     BackgroundJobModule,
     WaveModule,
+    BudgetModule,
+    PromptPolicyModule,
+    RetentionModule,
+    WaveConfigModule,
   ]
 })
 export class AppModule {}

apps/api/src/modules/appeal-decisions/appeal-decisions.module.ts

@@ -4,10 +4,11 @@ import { AuditService } from "../../lib/audit.service.js";
 import { AppealDecisionsController } from "./appeal-decisions.controller.js";
 import { AppealDecisionsService } from "./appeal-decisions.service.js";
 import { AppealDecisionsRepository } from "./appeal-decisions.repository.js";
+import { ShadowModeService } from "./shadow-mode.service.js";
 
 @Module({
   controllers: [AppealDecisionsController],
-  providers: [AppealDecisionsService, AppealDecisionsRepository, PrismaService, AuditService],
-  exports: [AppealDecisionsService],
+  providers: [AppealDecisionsService, AppealDecisionsRepository, PrismaService, AuditService, ShadowModeService],
+  exports: [AppealDecisionsService, ShadowModeService],
 })
 export class AppealDecisionsModule {}

apps/api/src/modules/appeal-decisions/model-rollout.service.spec.ts

@@ -0,0 +1,82 @@
+import { ModelRolloutService } from "./model-rollout.service.js";
+
+describe("ModelRolloutService", () => {
+  let svc: ModelRolloutService;
+
+  beforeEach(() => {
+    svc = new ModelRolloutService();
+  });
+
+  describe("pinned mode", () => {
+    it("always routes to stableVersion", () => {
+      svc.setRollout({ activeVersion: "v2", stableVersion: "v1", mode: "pinned" });
+      const r = svc.resolveModel("any-request-id");
+      expect(r.modelVersion).toBe("v1");
+      expect(r.mode).toBe("pinned");
+    });
+  });
+
+  describe("full mode", () => {
+    it("always routes to activeVersion", () => {
+      svc.setRollout({ activeVersion: "v2", stableVersion: "v1", mode: "full" });
+      const r = svc.resolveModel("any-request-id");
+      expect(r.modelVersion).toBe("v2");
+    });
+  });
+
+  describe("canary mode", () => {
+    it("routes 0% canary to stable every time", () => {
+      svc.setRollout({ activeVersion: "v2", stableVersion: "v1", mode: "canary", canaryPercent: 0 });
+      for (let i = 0; i < 20; i++) {
+        expect(svc.resolveModel(`id-${i}`).modelVersion).toBe("v1");
+      }
+    });
+
+    it("routes 100% canary to active every time", () => {
+      svc.setRollout({ activeVersion: "v2", stableVersion: "v1", mode: "canary", canaryPercent: 100 });
+      for (let i = 0; i < 20; i++) {
+        expect(svc.resolveModel(`id-${i}`).modelVersion).toBe("v2");
+      }
+    });
+
+    it("is deterministic — same requestId yields same routing", () => {
+      svc.setRollout({ activeVersion: "v2", stableVersion: "v1", mode: "canary", canaryPercent: 50 });
+      const first = svc.resolveModel("stable-request-id");
+      const second = svc.resolveModel("stable-request-id");
+      expect(first.modelVersion).toBe(second.modelVersion);
+    });
+  });
+
+  describe("rollback", () => {
+    it("restores the previous config", () => {
+      svc.setRollout({ activeVersion: "v2", stableVersion: "v1", mode: "full" });
+      svc.rollback();
+      const r = svc.resolveModel("x");
+      // should be back to default (pinned to v1)
+      expect(r.modelVersion).toBe("v1");
+      expect(r.mode).toBe("pinned");
+    });
+
+    it("is a no-op when there is no previous config", () => {
+      const before = svc.getState();
+      svc.rollback();
+      const after = svc.getState();
+      expect(after.current).toEqual(before.current);
+    });
+
+    it("clears previous after rollback (single-level undo)", () => {
+      svc.setRollout({ activeVersion: "v2", stableVersion: "v1", mode: "full" });
+      svc.rollback();
+      expect(svc.getState().previous).toBeNull();
+    });
+  });
+
+  describe("getState", () => {
+    it("exposes current and previous configs", () => {
+      svc.setRollout({ activeVersion: "v2", stableVersion: "v1", mode: "full" });
+      const state = svc.getState();
+      expect(state.current.mode).toBe("full");
+      expect(state.previous!.mode).toBe("pinned");
+    });
+  });
+});

apps/api/src/modules/appeal-decisions/model-rollout.service.ts

@@ -0,0 +1,164 @@
+/**
+ * AI-209: Model rollback and safe rollout controls for appeal automation.
+ *
+ * Provides guarded rollout modes (pinned, canary, full) so AI behaviour can be
+ * promoted incrementally and rolled back instantly if fairness or quality metrics
+ * drift. All transitions are explicit, logged, and reversible.
+ *
+ * Explicit inputs  → setRollout(config) / resolveModel(context)
+ * Explicit outputs → RolloutResolution (modelVersion, mode, reason)
+ * Review boundary  → canary mode routes only the configured % of traffic to the
+ *                    candidate; humans monitor before promoting to "full"
+ */
+
+import { Injectable, Logger } from "@nestjs/common";
+
+export type RolloutMode = "pinned" | "canary" | "full";
+
+export interface RolloutConfig {
+  /** The model version to use in "pinned" and "full" modes, and as canary candidate. */
+  activeVersion: string;
+  /** The stable fallback version used when not in "full" mode or when canary misses. */
+  stableVersion: string;
+  mode: RolloutMode;
+  /**
+   * In "canary" mode: percentage of requests routed to activeVersion (0–100).
+   * Ignored in other modes.
+   */
+  canaryPercent?: number;
+}
+
+export interface RolloutResolution {
+  modelVersion: string;
+  mode: RolloutMode;
+  /** Human-readable reason for the routing decision (for logs / operator inspection). */
+  reason: string;
+}
+
+export interface ModelRolloutState {
+  current: RolloutConfig;
+  previous: RolloutConfig | null;
+  updatedAt: string;
+}
+
+const DEFAULT_CONFIG: RolloutConfig = {
+  activeVersion: "v1",
+  stableVersion: "v1",
+  mode: "pinned",
+  canaryPercent: 0,
+};
+
+@Injectable()
+export class ModelRolloutService {
+  private readonly logger = new Logger(ModelRolloutService.name);
+
+  private state: ModelRolloutState = {
+    current: { ...DEFAULT_CONFIG },
+    previous: null,
+    updatedAt: new Date().toISOString(),
+  };
+
+  /** Inspect the current rollout state (for operator dashboards / runbooks). */
+  getState(): ModelRolloutState {
+    return { ...this.state };
+  }
+
+  /**
+   * Update the rollout configuration.
+   * The previous config is preserved so rollback() can restore it instantly.
+   */
+  setRollout(config: RolloutConfig): ModelRolloutState {
+    const previous = this.state.current;
+    this.state = { current: config, previous, updatedAt: new Date().toISOString() };
+
+    this.logger.log(
+      JSON.stringify({
+        event: "rollout_config_updated",
+        mode: config.mode,
+        activeVersion: config.activeVersion,
+        stableVersion: config.stableVersion,
+        canaryPercent: config.canaryPercent ?? 0,
+      }),
+    );
+
+    return this.getState();
+  }
+
+  /**
+   * Roll back to the previous configuration immediately.
+   * No-op if there is no previous config (already at stable baseline).
+   */
+  rollback(): ModelRolloutState {
+    if (!this.state.previous) {
+      this.logger.warn("rollback requested but no previous config exists — no-op");
+      return this.getState();
+    }
+
+    const restored = this.state.previous;
+    this.state = {
+      current: restored,
+      previous: null,
+      updatedAt: new Date().toISOString(),
+    };
+
+    this.logger.warn(
+      JSON.stringify({
+        event: "rollout_rolled_back",
+        restoredVersion: restored.activeVersion,
+        restoredMode: restored.mode,
+      }),
+    );
+
+    return this.getState();
+  }
+
+  /**
+   * Resolve which model version to use for a given request context.
+   *
+   * @param requestId  Used as entropy source for canary bucketing (deterministic per request).
+   */
+  resolveModel(requestId: string): RolloutResolution {
+    const { mode, activeVersion, stableVersion, canaryPercent = 0 } = this.state.current;
+
+    if (mode === "pinned") {
+      return {
+        modelVersion: stableVersion,
+        mode,
+        reason: `pinned to stable version ${stableVersion}`,
+      };
+    }
+
+    if (mode === "full") {
+      return {
+        modelVersion: activeVersion,
+        mode,
+        reason: `full rollout of ${activeVersion}`,
+      };
+    }
+
+    // canary — deterministic bucketing via simple hash of requestId
+    const bucket = this.hashBucket(requestId, 100);
+    if (bucket < canaryPercent) {
+      return {
+        modelVersion: activeVersion,
+        mode,
+        reason: `canary bucket ${bucket} < ${canaryPercent}% → candidate ${activeVersion}`,
+      };
+    }
+
+    return {
+      modelVersion: stableVersion,
+      mode,
+      reason: `canary bucket ${bucket} ≥ ${canaryPercent}% → stable ${stableVersion}`,
+    };
+  }
+
+  /** Deterministic integer bucket [0, modulus) derived from a string. */
+  private hashBucket(input: string, modulus: number): number {
+    let hash = 0;
+    for (let i = 0; i < input.length; i++) {
+      hash = (hash * 31 + input.charCodeAt(i)) >>> 0;
+    }
+    return hash % modulus;
+  }
+}

apps/api/src/modules/appeal-decisions/score-calibration.service.spec.ts

@@ -0,0 +1,62 @@
+import { ScoreCalibrationService, DEFAULT_CALIBRATION_POLICY } from "./score-calibration.service.js";
+
+describe("ScoreCalibrationService", () => {
+  let svc: ScoreCalibrationService;
+
+  beforeEach(() => {
+    svc = new ScoreCalibrationService();
+  });
+
+  it("auto-approves high-confidence scores", () => {
+    const result = svc.calibrate(0.9);
+    expect(result.band).toBe("auto_approve");
+    expect(result.action).toBe("approve");
+    expect(result.needsHumanReview).toBe(false);
+  });
+
+  it("escalates mid-range scores and requires human review below threshold", () => {
+    const result = svc.calibrate(0.5);
+    expect(result.band).toBe("review");
+    expect(result.action).toBe("escalate");
+    expect(result.needsHumanReview).toBe(true);
+  });
+
+  it("auto-rejects low-confidence scores", () => {
+    const result = svc.calibrate(0.1);
+    expect(result.band).toBe("auto_reject");
+    expect(result.action).toBe("reject");
+    expect(result.needsHumanReview).toBe(true);
+  });
+
+  it("applies bias correction factor to shift scores upward", () => {
+    // raw 0.7 * factor 1.2 = 0.84 → auto_approve
+    const result = svc.calibrate(0.7, { biasCorrectionFactor: 1.2 });
+    expect(result.band).toBe("auto_approve");
+    expect(result.confidence).toBeCloseTo(0.84);
+  });
+
+  it("clamps corrected confidence to [0, 1]", () => {
+    const result = svc.calibrate(0.95, { biasCorrectionFactor: 2.0 });
+    expect(result.confidence).toBe(1);
+  });
+
+  it("preserves rawScore in output for audit trail", () => {
+    const result = svc.calibrate(0.65, { biasCorrectionFactor: 1.1 });
+    expect(result.rawScore).toBe(0.65);
+  });
+
+  it("includes the resolved policy in output for traceability", () => {
+    const result = svc.calibrate(0.5);
+    expect(result.appliedPolicy).toMatchObject(DEFAULT_CALIBRATION_POLICY);
+  });
+
+  it("respects custom thresholds", () => {
+    const result = svc.calibrate(0.6, { approveThreshold: 0.55 });
+    expect(result.band).toBe("auto_approve");
+  });
+
+  it("marks high-confidence mid-range score as not needing human review when threshold is low", () => {
+    const result = svc.calibrate(0.75, { humanReviewThreshold: 0.5 });
+    expect(result.needsHumanReview).toBe(false);
+  });
+});