Feature/gh 284 stories 1 8 moved

.github/workflows/code-quality.yml

@@ -87,6 +87,15 @@ jobs:
           dotnet build tests/F1.Api.Tests/F1.Api.Tests.csproj --configuration Release --no-restore /warnaserror
           dotnet build tests/F1.Web.Tests/F1.Web.Tests.csproj --configuration Release --no-restore /warnaserror
 
+      - name: Verify EF Migrations Are Up To Date
+        run: |
+          dotnet tool install --global dotnet-ef --version 9.* || dotnet tool update --global dotnet-ef --version 9.*
+          dotnet ef migrations has-pending-model-changes \
+            --project src/F1.Infrastructure/F1.Infrastructure.csproj \
+            --startup-project src/F1.Api/F1.Api.csproj \
+            --configuration Release \
+            --no-build
+
   unit-tests:
     runs-on: ubuntu-latest
     needs: build

build.sh

@@ -16,6 +16,7 @@ DATA_SYNC_PROJECT="src/F1.DataSyncWorker/F1.DataSyncWorker.csproj"
 API_TEST_PROJECT="tests/F1.Api.Tests/F1.Api.Tests.csproj"
 WEB_TEST_PROJECT="tests/F1.Web.Tests/F1.Web.Tests.csproj"
 INFRA_TEST_PROJECT="tests/F1.Infrastructure.Tests/F1.Infrastructure.Tests.csproj"
+INFRA_DATA_PROJECT="src/F1.Infrastructure/F1.Infrastructure.csproj"
 
 FORMAT_INCLUDE_PATHS=(
   "src/F1.Api"
@@ -70,6 +71,12 @@ run_quality_gate() {
     if ! CI=true dotnet build "$WEB_TEST_PROJECT" --configuration Release --no-restore; then return 1; fi
     if ! CI=true dotnet build "$INFRA_TEST_PROJECT" --configuration Release --no-restore; then return 1; fi
 
+    if ! dotnet tool update --global dotnet-ef --version 9.* && ! dotnet tool install --global dotnet-ef --version 9.*; then return 1; fi
+    if ! dotnet ef migrations has-pending-model-changes --project "$INFRA_DATA_PROJECT" --startup-project "$API_PROJECT" --no-build; then
+        printf "\033[0;31m❌ EF model has pending changes. Add a migration before continuing.\033[0m\n"
+        return 1
+    fi
+
     echo "✅ Quality gate passed."
 }
 

docs/epics/gh-284-migration-write-correctness-and-non-empty-db/epic-migration-write-correctness-and-non-empty-db.md

@@ -46,6 +46,7 @@ Acceptance criteria:
 Test notes:
 - Add a characterization test that reproduces current failed/no-op write behavior before fix.
 - Add traceability artifact reference in docs/runbook so future regressions can be triaged quickly.
+  - Artifact: `docs/runbooks/migration-write-pipeline-trace.md`
 
 ### Story 2: Implement transactional write path to canonical tables
 As an operator, I want write runs to persist all intended entities atomically so partial writes cannot corrupt state.

docs/runbooks/migration-write-pipeline-trace.md

@@ -0,0 +1,68 @@
+# Migration Write Pipeline Trace and Gap Report
+
+## Scope
+This trace documents the current migration import execution path from kickoff to persistence, including dry-run mapping/enrichment and write-mode canonical materialization.
+
+## End-to-End Pipeline Map
+1. Run kickoff
+- Entry points: worker orchestrator `MigrationImportOrchestrator.RunOnceAsync` and queued mode `RunNextQueuedAsync`.
+- Run metadata row is created/claimed in `MigrationImportRunService`.
+
+2. Raw row staging
+- CSV rows are classified and persisted to `MigrationImportRawRows`.
+
+3. Parsing and normalization
+- Race picks are parsed into `MigrationImportRaceSelections`.
+- Generic/preseason question inputs are parsed into:
+  - `MigrationImportPreseasonAnswers`
+  - `QuestionAnswers`
+  - `QuestionActuals`
+
+4. Mapping and enrichment (dry-run and write mode)
+- Race sequence mapping persists into:
+  - `MigrationImportJolpicaRaceSnapshots`
+  - `MigrationImportRaceRoundMappings`
+- Race codes in staged selections are rewritten to mapped circuit ids.
+
+5. Scoring
+- Race pick scoring persists to `MigrationImportCalculatedScores`.
+- Imported legacy totals persist to `MigrationImportLegacyPickScores` and related totals tables.
+- Generic question scoring persists to `QuestionScores`.
+
+6. Reconciliation
+- Diff and summary outputs persist to migration reconciliation tables:
+  - `MigrationImportPickDiffs`
+  - `MigrationImportRaceDiffs`
+  - `MigrationImportParticipantDeltaSummaries`
+  - `MigrationImportReasonCategorySummaries`
+  - preseason diff/summary companion tables
+
+7. Completion
+- Run status and metadata are written to `MigrationImportRuns`.
+
+8. Canonical race-domain writes (write mode only)
+- Canonical persistence runs after reconciliation in write mode via `MigrationCanonicalWriteService`.
+- Races must be pre-seeded for the target season; the migration writer looks them up by circuit id or round and does not create new Race rows.
+- Entities created/updated where applicable:
+  - `Drivers` (created when missing)
+  - `Selections` (created or reused per conflict policy)
+  - `SelectionPositions` (replaced per selection)
+
+## Intended Canonical Targets (Epic Contract)
+For write mode, canonical race-domain entities are created or updated where applicable:
+- `Drivers` (created when not already present)
+- `Races` (pre-seeded required; looked up by circuit id or round — not created by migration writer)
+- `Selections` (created or reused per conflict policy)
+- `SelectionPositions` (replaced per selection)
+
+Question-domain tables already receive run-scoped writes via parser/scoring (`QuestionAnswers`, `QuestionActuals`, `QuestionScores`).
+
+## Current Notes (Observed)
+1. Mapping/enrichment executes for both dry-run and write runs.
+2. Canonical race-domain persistence is write-mode only and is now implemented.
+3. Rollback paths and conflict diagnostics are available for canonical-write operations.
+
+## Follow-On Implementation Work
+- Continue hardening rollback scope and non-empty DB safeguards.
+- Expand operational runbooks for conflict policies and post-write verification.
+- Keep migration and schema drift checks enforced in CI for rollout safety.

scripts/clear-canonical-tables.sh

@@ -0,0 +1,115 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Clears canonical domain tables used by the migration write path.
+#
+# Usage:
+#   scripts/clear-canonical-tables.sh
+#   scripts/clear-canonical-tables.sh --yes
+#
+# Connection resolution (first match wins):
+#   1) DATABASE_URL
+#   2) ConnectionStrings__Postgres
+#   3) PGHOST/PGPORT/PGDATABASE/PGUSER/PGPASSWORD via psql defaults
+
+force='false'
+include_competitions='false'
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --yes|-y)
+      force='true'
+      ;;
+    --include-competitions)
+      include_competitions='true'
+      ;;
+    --help|-h)
+      cat <<'USAGE'
+Clear canonical tables in Postgres.
+
+Options:
+  --yes, -y                 Skip confirmation prompt
+  --include-competitions    Also delete rows from public."Competitions"
+  --help, -h                Show this help text
+USAGE
+      exit 0
+      ;;
+    *)
+      echo "Unknown argument: $1" >&2
+      exit 2
+      ;;
+  esac
+  shift
+done
+
+if [[ "$force" != 'true' ]]; then
+  cat <<'WARN'
+This will permanently delete data from canonical tables:
+  - public."QuestionScores"
+  - public."QuestionAnswers"
+  - public."QuestionActuals"
+  - public."QuestionTemplates"
+  - public."SelectionPositions"
+  - public."Selections"
+  - public."RaceMetadata"
+  - public."Drivers"
+WARN
+
+  if [[ "$include_competitions" == 'true' ]]; then
+    echo '  - public."Competitions"'
+  fi
+
+  read -r -p 'Type CLEAR to continue: ' confirm
+  if [[ "$confirm" != 'CLEAR' ]]; then
+    echo 'Aborted.'
+    exit 1
+  fi
+fi
+
+conn=''
+if [[ -n "${DATABASE_URL:-}" ]]; then
+  conn="$DATABASE_URL"
+elif [[ -n "${ConnectionStrings__Postgres:-}" ]]; then
+  conn="$ConnectionStrings__Postgres"
+fi
+
+sql=$(cat <<'SQL'
+BEGIN;
+DELETE FROM public."QuestionScores";
+DELETE FROM public."QuestionAnswers";
+DELETE FROM public."QuestionActuals";
+DELETE FROM public."QuestionTemplates";
+DELETE FROM public."SelectionPositions";
+DELETE FROM public."Selections";
+SQL
+)
+
+if [[ "$include_competitions" == 'true' ]]; then
+  sql+=$'\nDELETE FROM public."Competitions";'
+fi
+
+sql+=$'\nCOMMIT;\n'
+
+sql_file=$(mktemp)
+cleanup() {
+  rm -f "$sql_file"
+}
+trap cleanup EXIT
+
+printf '%s' "$sql" > "$sql_file"
+
+if command -v psql >/dev/null 2>&1; then
+  if [[ -n "$conn" ]]; then
+    psql "$conn" -v ON_ERROR_STOP=1 -f "$sql_file"
+  else
+    psql -v ON_ERROR_STOP=1 -f "$sql_file"
+  fi
+else
+  if [[ -n "$conn" ]]; then
+    DATABASE_URL="$conn" docker compose exec -T postgres sh -lc 'psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -f /dev/stdin' < "$sql_file"
+  else
+    PGPASSWORD="${POSTGRES_PASSWORD:-f1}" docker compose exec -T postgres psql -U "${POSTGRES_USER:-f1}" -d "${POSTGRES_DB:-f1competition}" -v ON_ERROR_STOP=1 -f /dev/stdin < "$sql_file"
+  fi
+fi
+
+echo 'Canonical table clear-down complete.'

src/F1.Api/Controllers/MigrationRunsController.cs

@@ -206,7 +206,8 @@ public async Task<IActionResult> KickoffRun(
             new MigrationRunKickoffCommand(
                 request.SourceFilePath,
                 request.Mode,
-                ResolveActor()),
+                ResolveActor(),
+                request.ConfirmNonEmptyStrategy),
             cancellationToken);
 
         if (!result.Success)
@@ -283,7 +284,8 @@ public async Task<IActionResult> KickoffRunFromUpload(
             new MigrationRunKickoffCommand(
                 persistedPath,
                 request.Mode,
-                ResolveActor()),
+                ResolveActor(),
+                request.ConfirmNonEmptyStrategy),
             cancellationToken);
 
         if (!result.Success)
@@ -308,6 +310,37 @@ public async Task<IActionResult> KickoffRunFromUpload(
         return CreatedAtAction(nameof(GetRunDetail), new { runId = result.Run!.RunId }, result.Run);
     }
 
+    [HttpPost("{runId:guid}/rollback")]
+    public async Task<IActionResult> RollbackRun(
+        Guid runId,
+        [FromBody] AdminMigrationRollbackRequestDto request,
+        CancellationToken cancellationToken = default)
+    {
+        if (string.IsNullOrWhiteSpace(request.Reason))
+        {
+            return BadRequest(new
+            {
+                message = "Rollback reason is required.",
+                code = "rollback_invalid_request"
+            });
+        }
+
+        var result = await _migrationRunAdminService.RollbackRunAsync(
+            new MigrationRunRollbackCommand(runId, ResolveActor(), request.Reason.Trim()),
+            cancellationToken);
+
+        if (!result.Success)
+        {
+            return BadRequest(new
+            {
+                message = result.Error ?? "Unable to rollback migration run.",
+                code = "rollback_failed"
+            });
+        }
+
+        return Ok(result.Rollback);
+    }
+
     private static string ResolveWritableUploadRoot()
     {
         var primaryRoot = Path.GetFullPath(UploadDirectory, Directory.GetCurrentDirectory());

src/F1.Api/Dtos/AdminMigrationRunDtos.cs

@@ -46,7 +46,40 @@ public sealed record AdminMigrationRunDetailResponseDto(
     IReadOnlyList<AdminMigrationPreseasonQuestionDiffDto> PreseasonQuestionDiffs,
     IReadOnlyList<AdminMigrationPreseasonReasonCategorySummaryDto> PreseasonReasonCategorySummaries,
     IReadOnlyList<AdminMigrationRaceDiffDto> RaceDiffs,
-    IReadOnlyList<AdminMigrationPickDiffDto> PickDiffs);
+    IReadOnlyList<AdminMigrationPickDiffDto> PickDiffs,
+    IReadOnlyList<AdminMigrationConflictDiagnosticDto>? ConflictDiagnostics = null,
+    IReadOnlyList<AdminMigrationRollbackAuditDto>? RollbackAudits = null);
+
+public sealed record AdminMigrationRollbackRequestDto(
+    string Reason);
+
+public sealed record AdminMigrationRollbackResponseDto(
+    Guid RunId,
+    string Status,
+    DateTime RequestedAtUtc,
+    string RequestedBy,
+    string Outcome,
+    int AffectedRaceCount,
+    int AffectedSelectionCount,
+    int AffectedSelectionPositionCount);
+
+public sealed record AdminMigrationRollbackAuditDto(
+    DateTime RequestedAtUtc,
+    string Actor,
+    string Reason,
+    string Outcome,
+    int AffectedRaceCount,
+    int AffectedSelectionCount,
+    int AffectedSelectionPositionCount);
+
+public sealed record AdminMigrationConflictDiagnosticDto(
+    string EntityType,
+    string ConflictType,
+    string KeyFields,
+    string SourceReference,
+    string PolicyOutcome,
+    string RecommendedAction,
+    DateTime CreatedAtUtc);
 
 public sealed record AdminMigrationUnresolvedTokenSummaryDto(
     string RawToken,
@@ -119,11 +152,13 @@ public sealed record AdminMigrationPickDiffDto(
 
 public sealed record AdminMigrationRunKickoffRequestDto(
     string? SourceFilePath,
-    string Mode);
+    string Mode,
+    bool ConfirmNonEmptyStrategy = false);
 
 public sealed record AdminMigrationRunKickoffUploadRequestDto(
     IFormFile? SourceFile,
-    string Mode);
+    string Mode,
+    bool ConfirmNonEmptyStrategy = false);
 
 public sealed record AdminMigrationRunKickoffResponseDto(
     Guid RunId,
@@ -133,7 +168,15 @@ public sealed record AdminMigrationRunKickoffResponseDto(
     string SourceFilePath,
     string SourceFileChecksum,
     DateTime TriggeredAtUtc,
-    string RequestedBy);
+    string RequestedBy,
+    string NonEmptyDbStrategy = "merge_upsert_active_records",
+    bool CanonicalDataPresent = false,
+    int ExistingDriverCount = 0,
+    int ExistingRaceCount = 0,
+    int ExistingSelectionCount = 0,
+    int EstimatedAffectedRaceCount = 0,
+    int EstimatedAffectedParticipantCount = 0,
+    int EstimatedAffectedSelectionCount = 0);
 
 public sealed record AdminMigrationQuestionDiffListResponseDto(
     int Page,
@@ -148,8 +191,7 @@ public sealed record AdminMigrationQuestionDiffDto(
     string Participant,
     int? ImportedPoints,
     int CalculatedPoints,
-    int DeltaPoints,
-    string ReasonCode);
+    int DeltaPoints);
 
 public sealed record AdminMigrationQuestionDiffSummaryResponseDto(
     int TotalCount,