Bugfix/#275 migration preseason issues

README.md

@@ -168,6 +168,8 @@ These `DATA_SYNC_*` values are consumed by `docker-compose.yml` and mapped to `D
 - `MIGRATION_IMPORT_UNRESOLVED_TOKEN_FAIL_THRESHOLD`: mapped to `MigrationImport__UnresolvedTokenFailThreshold`; `0` disables fail-fast and values greater than `0` fail the run when unresolved token count is greater than or equal to the threshold.
 - `MIGRATION_EXPECTED_VARIANCE_ENABLED`: mapped to `MigrationExpectedVariance__Enabled`; enables expected variance ruleset loading.
 - `MIGRATION_EXPECTED_VARIANCE_RULE_MANIFEST_PATH`: mapped to `MigrationExpectedVariance__RuleManifestPath`; default is `data/imports/phil-2025/expected-variance-rules.json`.
+  - `f1-data-sync-worker` image bundles `data/imports/phil-2025` under `/app`, so the default relative paths for migration source CSV and expected variance manifest resolve inside the container without manual host file edits.
+  - To use host-managed files instead, keep `HOST_MIGRATION_UPLOAD_PATH` mounted and point paths to `/tmp/f1-imports/<file-name>`.
 
 Optional API values in `.env`:
 

src/F1.DataSyncWorker/Dockerfile

@@ -17,6 +17,7 @@ RUN dotnet publish -c $CONFIGURATION -o /app/publish
 FROM mcr.microsoft.com/dotnet/runtime:8.0 AS final
 WORKDIR /app
 COPY --from=build /app/publish .
+COPY --from=build /app/data/imports/phil-2025 /app/data/imports/phil-2025
 
 USER app
 ENTRYPOINT ["dotnet", "F1.DataSyncWorker.dll"]

src/F1.DataSyncWorker/Services/MigrationImportOrchestrator.cs

@@ -139,8 +139,9 @@ private async Task ExecuteRunAsync(MigrationImportRunContext run, CancellationTo
                     run.RunId);
             }
 
-            var scoreResult = await _scoreRecalculator.RecalculateAndPersistAsync(run.RunId, cancellationToken);
+            // Import legacy/preseason source tallies first so scoring can read preseason policy values (M2).
             var legacyResult = await _legacyScoreImporter.ImportAndPersistAsync(run.RunId, cancellationToken);
+            var scoreResult = await _scoreRecalculator.RecalculateAndPersistAsync(run.RunId, cancellationToken);
             await EnsurePreseasonRaceIsolationAsync(run.RunId, cancellationToken);
             var reconciliationResult = await _reconciliationService.ReconcileAndPersistAsync(run.RunId, cancellationToken);
 

src/F1.DataSyncWorker/Services/MigrationLegacyScoreImporter.cs

@@ -308,7 +308,7 @@ private List<MigrationImportPreseasonImportedTallyEntity> ParsePreseasonImported
             }
 
             var questionText = columns[0].Trim();
-            var questionKey = $"PRE-{row.RowNumber:D3}";
+            var questionKey = ResolvePreseasonQuestionKey(row.RowNumber, usePhil2025Contract);
 
             for (var participantIndex = 0; participantIndex < participants.Count; participantIndex++)
             {
@@ -343,6 +343,19 @@ private List<MigrationImportPreseasonImportedTallyEntity> ParsePreseasonImported
         return parsed;
     }
 
+    private static string ResolvePreseasonQuestionKey(int rowNumber, bool usePhil2025Contract)
+    {
+        var normalizedRowNumber = rowNumber;
+        if (usePhil2025Contract)
+        {
+            var contractOffset = MigrationPhil2025CsvContractPolicy.PreseasonPointsStartRow -
+                MigrationPhil2025CsvContractPolicy.PreseasonQuestionStartRow;
+            normalizedRowNumber = Math.Max(1, rowNumber - contractOffset);
+        }
+
+        return $"PRE-{normalizedRowNumber:D3}";
+    }
+
     private void HandlePreseasonPolicyParseIssue(string message, int? rowNumber = null)
     {
         if (_importOptions.FailOnPreseasonPolicyParseError)

src/F1.DataSyncWorker/Services/MigrationReconciliationService.cs

@@ -264,45 +264,56 @@ public async Task<MigrationReconciliationResult> ReconcileAndPersistAsync(Guid r
             .ToList();
 
         var preseasonImportedByKey = preseasonImported
-            .GroupBy(x => new PreseasonQuestionDiffKey(x.RowNumber, x.QuestionKey, x.Subject), PreseasonQuestionDiffKeyComparer.Instance)
+            .GroupBy(x => new PreseasonQuestionDiffKey(x.QuestionKey, x.Subject), PreseasonQuestionDiffKeyComparer.Instance)
             .ToDictionary(
                 x => x.Key,
                 x => x.Any(y => y.ImportedPoints is null) ? (int?)null : x.Sum(y => y.ImportedPoints ?? 0),
                 PreseasonQuestionDiffKeyComparer.Instance);
 
         var preseasonImportedRowsByKey = preseasonImported
-            .GroupBy(x => new PreseasonQuestionDiffKey(x.RowNumber, x.QuestionKey, x.Subject), PreseasonQuestionDiffKeyComparer.Instance)
+            .GroupBy(x => new PreseasonQuestionDiffKey(x.QuestionKey, x.Subject), PreseasonQuestionDiffKeyComparer.Instance)
             .ToDictionary(
                 x => x.Key,
                 x => x.Select(y => y.RowNumber).Distinct().OrderBy(y => y).ToArray(),
                 PreseasonQuestionDiffKeyComparer.Instance);
 
         var preseasonCalculatedByKey = preseasonCalculated
-            .GroupBy(x => new PreseasonQuestionDiffKey(x.RowNumber, x.QuestionKey, x.Subject), PreseasonQuestionDiffKeyComparer.Instance)
+            .GroupBy(x => new PreseasonQuestionDiffKey(x.QuestionKey, x.Subject), PreseasonQuestionDiffKeyComparer.Instance)
             .ToDictionary(
                 x => x.Key,
                 x => x.Sum(y => y.Points),
                 PreseasonQuestionDiffKeyComparer.Instance);
 
         var preseasonCalculatedRowsByKey = preseasonCalculated
-            .GroupBy(x => new PreseasonQuestionDiffKey(x.RowNumber, x.QuestionKey, x.Subject), PreseasonQuestionDiffKeyComparer.Instance)
+            .GroupBy(x => new PreseasonQuestionDiffKey(x.QuestionKey, x.Subject), PreseasonQuestionDiffKeyComparer.Instance)
             .ToDictionary(
                 x => x.Key,
                 x => x.Select(y => y.RowNumber).Distinct().OrderBy(y => y).ToArray(),
                 PreseasonQuestionDiffKeyComparer.Instance);
 
         var preseasonQuestionTextByKey = preseasonImported
-            .Select(x => new { x.RowNumber, x.QuestionKey, x.QuestionText })
-            .Concat(preseasonCalculated.Select(x => new { x.RowNumber, x.QuestionKey, x.QuestionText }))
-            .GroupBy(x => (x.RowNumber, x.QuestionKey))
+            .Select(x => new { x.QuestionKey, x.QuestionText })
+            .Concat(preseasonCalculated.Select(x => new { x.QuestionKey, x.QuestionText }))
+            .GroupBy(x => x.QuestionKey, StringComparer.OrdinalIgnoreCase)
             .ToDictionary(
-                x => (x.Key.RowNumber, x.Key.QuestionKey),
-                x => x.Select(y => y.QuestionText).FirstOrDefault(text => !string.IsNullOrWhiteSpace(text)) ?? x.Key.QuestionKey);
+                x => x.Key,
+                x => x.Select(y => y.QuestionText).FirstOrDefault(text => !string.IsNullOrWhiteSpace(text)) ?? x.Key,
+                StringComparer.OrdinalIgnoreCase);
+
+        var preseasonQuestionOrderByKey = preseasonImported
+            .Select(x => new { x.QuestionKey, x.RowNumber })
+            .Concat(preseasonCalculated.Select(x => new { x.QuestionKey, x.RowNumber }))
+            .GroupBy(x => x.QuestionKey, StringComparer.OrdinalIgnoreCase)
+            .ToDictionary(
+                x => x.Key,
+                x => x.Min(y => y.RowNumber),
+                StringComparer.OrdinalIgnoreCase);
 
         var allPreseasonKeys = preseasonImportedByKey.Keys
             .Concat(preseasonCalculatedByKey.Keys)
             .Distinct(PreseasonQuestionDiffKeyComparer.Instance)
-            .OrderBy(x => x.RowNumber)
+            .OrderBy(x => preseasonQuestionOrderByKey.GetValueOrDefault(x.QuestionKey, int.MaxValue))
+            .ThenBy(x => x.QuestionKey, StringComparer.OrdinalIgnoreCase)
             .ThenBy(x => x.Subject, StringComparer.OrdinalIgnoreCase)
             .ToList();
 
@@ -319,8 +330,13 @@ public async Task<MigrationReconciliationResult> ReconcileAndPersistAsync(Guid r
             var importedRows = preseasonImportedRowsByKey.GetValueOrDefault(key, []);
             var calculatedRows = preseasonCalculatedRowsByKey.GetValueOrDefault(key, []);
             participantColumnBySubject.TryGetValue(key.Subject, out var participantColumn);
+            var rowNumber = importedRows.FirstOrDefault();
+            if (rowNumber == 0)
+            {
+                rowNumber = calculatedRows.FirstOrDefault();
+            }
 
-            var questionText = preseasonQuestionTextByKey.GetValueOrDefault((key.RowNumber, key.QuestionKey), key.QuestionKey);
+            var questionText = preseasonQuestionTextByKey.GetValueOrDefault(key.QuestionKey, key.QuestionKey);
             var reasonCode = ResolvePreseasonReasonCode(imported, calculatedValue, delta);
             var explanation = BuildPreseasonQuestionExplanation(
                 key,
@@ -336,7 +352,7 @@ public async Task<MigrationReconciliationResult> ReconcileAndPersistAsync(Guid r
             preseasonQuestionDiffs.Add(new MigrationImportPreseasonQuestionDiffEntity
             {
                 ImportRunId = runId,
-                RowNumber = key.RowNumber,
+                RowNumber = rowNumber,
                 QuestionKey = key.QuestionKey,
                 QuestionText = questionText,
                 Subject = key.Subject,
@@ -684,7 +700,7 @@ public int GetHashCode(PickDiffKey obj)
 
     private sealed record RaceDiffKey(string RaceCode, string Subject);
 
-    private sealed record PreseasonQuestionDiffKey(int RowNumber, string QuestionKey, string Subject);
+    private sealed record PreseasonQuestionDiffKey(string QuestionKey, string Subject);
 
     private sealed class PreseasonQuestionDiffKeyComparer : IEqualityComparer<PreseasonQuestionDiffKey>
     {
@@ -702,15 +718,13 @@ public bool Equals(PreseasonQuestionDiffKey? x, PreseasonQuestionDiffKey? y)
                 return false;
             }
 
-            return x.RowNumber == y.RowNumber
-                && string.Equals(x.QuestionKey, y.QuestionKey, StringComparison.OrdinalIgnoreCase)
+            return string.Equals(x.QuestionKey, y.QuestionKey, StringComparison.OrdinalIgnoreCase)
                 && string.Equals(x.Subject, y.Subject, StringComparison.OrdinalIgnoreCase);
         }
 
         public int GetHashCode(PreseasonQuestionDiffKey obj)
         {
             return HashCode.Combine(
-                obj.RowNumber,
                 StringComparer.OrdinalIgnoreCase.GetHashCode(obj.QuestionKey),
                 StringComparer.OrdinalIgnoreCase.GetHashCode(obj.Subject));
         }

tests/F1.E2E.Tests/Pages/MigrationRunsPage.cs

@@ -113,54 +113,62 @@ public void SetNonZeroOnly(bool enabled)
 
     public IReadOnlyList<MigrationParticipantRow> GetParticipantRows()
     {
-        return GetRowsAfterSection("participant-comparisons")
+        EnsureTabSelected("tab-race-participants", "pane-race-participants");
+        return GetRowsInPane("pane-race-participants")
             .Select(ParseParticipantRow)
             .ToList();
     }
 
     public IReadOnlyList<MigrationRaceRow> GetRaceRows()
     {
-        return GetRowsAfterSection("race-comparisons")
+        EnsureTabSelected("tab-race-diffs", "pane-race-diffs");
+        return GetRowsInPane("pane-race-diffs")
             .Select(ParseRaceRow)
             .ToList();
     }
 
     public IReadOnlyList<MigrationPickRow> GetPickRows()
     {
-        return GetRowsAfterSection("pick-comparisons")
+        EnsureTabSelected("tab-pick-diffs", "pane-pick-diffs");
+        return GetRowsInPane("pane-pick-diffs")
             .Select(ParsePickRow)
             .ToList();
     }
 
     public bool WaitForParticipantComparisonSection()
     {
-        return WaitForComparisonSection("participant-comparisons", "No participant deltas available for this run.");
+        EnsureTabSelected("tab-race-participants", "pane-race-participants");
+        return WaitForPaneComparisonSection("pane-race-participants", "No participant deltas available for this run.");
     }
 
     public bool WaitForPreseasonComparisonSection()
     {
+        EnsureTabSelected("tab-preseason", "pane-preseason");
         return _wait.Until(driver =>
-            driver.FindElements(By.Id("preseason-comparisons")).Count > 0 &&
+            driver.FindElements(By.Id("pane-preseason")).Count > 0 &&
             driver.FindElements(By.Id("preseason-participant-filter")).Count > 0);
     }
 
     public bool WaitForPreseasonQuestionDiffSection()
     {
+        EnsureTabSelected("tab-preseason", "pane-preseason");
         var sectionSelector = By.XPath(
-            "//h4[normalize-space()='Preseason Question Diffs']" +
+            "//*[@id='pane-preseason']//h4[normalize-space()='Preseason Question Diffs']" +
             "/following-sibling::*[1]" +
             "[self::div[.//tbody/tr] or self::p[normalize-space()='No preseason question diffs available for this run.']]");
         return _wait.Until(driver => driver.FindElements(sectionSelector).Count > 0);
     }
 
     public bool WaitForRaceComparisonSection()
     {
-        return WaitForComparisonSection("race-comparisons", "No race diffs available for this run.");
+        EnsureTabSelected("tab-race-diffs", "pane-race-diffs");
+        return WaitForPaneComparisonSection("pane-race-diffs", "No race diffs available for this run.");
     }
 
     public bool WaitForPickComparisonSection()
     {
-        return WaitForComparisonSection("pick-comparisons", "No pick diffs available for this run.");
+        EnsureTabSelected("tab-pick-diffs", "pane-pick-diffs");
+        return WaitForPaneComparisonSection("pane-pick-diffs", "No pick diffs available for this run.");
     }
 
     public void WaitUntil(Func<bool> condition)
@@ -181,17 +189,45 @@ private void SetInputValue(string inputId, string value)
         input.SendKeys(Keys.Tab);
     }
 
-    private IReadOnlyList<IWebElement> GetRowsAfterSection(string sectionId)
+    private IReadOnlyList<IWebElement> GetRowsInPane(string paneId)
     {
-        var rows = _driver.FindElements(By.XPath($"//*[@id='{sectionId}']/following-sibling::*[1][self::div]//tbody/tr"));
+        var rows = _driver.FindElements(By.XPath($"//*[@id='{paneId}']//tbody/tr"));
         return rows;
     }
 
-    private bool WaitForComparisonSection(string sectionId, string emptyStateMessage)
+    private bool WaitForPaneComparisonSection(string paneId, string emptyStateMessage)
     {
-        var sectionSelector = By.XPath(
-            $"//*[@id='{sectionId}']/following-sibling::*[1][self::div[.//table] or self::p[normalize-space()='{emptyStateMessage}']]");
-        return _wait.Until(driver => driver.FindElements(sectionSelector).Count > 0);
+        return _wait.Until(driver =>
+        {
+            var pane = driver.FindElements(By.Id(paneId)).FirstOrDefault();
+            if (pane is null || !pane.Displayed)
+            {
+                return false;
+            }
+
+            var hasTable = pane.FindElements(By.XPath(".//table")).Count > 0;
+            var hasEmptyState = pane.FindElements(By.XPath($".//p[normalize-space()='{emptyStateMessage}']")).Count > 0;
+            return hasTable || hasEmptyState;
+        });
+    }
+
+    private void EnsureTabSelected(string tabId, string paneId)
+    {
+        var tabButton = _wait.Until(driver => driver.FindElement(By.Id(tabId)));
+        if (!string.Equals(tabButton.GetAttribute("aria-selected"), "true", StringComparison.OrdinalIgnoreCase))
+        {
+            tabButton.Click();
+        }
+
+        _wait.Until(driver =>
+        {
+            var currentTabButton = driver.FindElements(By.Id(tabId)).FirstOrDefault();
+            var pane = driver.FindElements(By.Id(paneId)).FirstOrDefault();
+            return currentTabButton is not null
+                && pane is not null
+                && string.Equals(currentTabButton.GetAttribute("aria-selected"), "true", StringComparison.OrdinalIgnoreCase)
+                && pane.Displayed;
+        });
     }
 
     private static MigrationParticipantRow ParseParticipantRow(IWebElement row)

tests/F1.Infrastructure.Tests/Contracts/MigrationLegacyScoreImporterTests.cs

@@ -67,7 +67,7 @@ public async Task ImportAndPersistAsync_WhenPhilPreseasonPolicyAndTalliesPresent
             .ToListAsync();
 
         Assert.Equal(10, tallies.Count);
-        Assert.Equal("PRE-022", tallies[0].QuestionKey);
+        Assert.Equal("PRE-002", tallies[0].QuestionKey);
         Assert.Contains(tallies, x => x.Subject == "Philip" && x.ImportedPoints == 0);
         Assert.Contains(tallies, x => x.Subject == "New Sexy Ayrton" && x.ImportedPoints == 20);
         Assert.Contains(tallies, x => x.Subject == "Veronica" && x.ImportedPoints == 20);

tests/F1.Infrastructure.Tests/Contracts/MigrationRaceSelectionParserTests.cs

@@ -64,6 +64,9 @@ public async Task ParseAndPersistAsync_WhenPreseasonQuestionRowsExist_PersistsPa
         var claireRow2 = preseasonAnswers.Single(x => x.RowNumber == 2 && x.Subject == "Claire" && !x.IsActualOutcome);
         Assert.Null(claireRow2.NormalizedAnswer);
 
+        var daveRow2 = preseasonAnswers.Single(x => x.RowNumber == 2 && x.Subject == "Dave" && !x.IsActualOutcome);
+        Assert.Equal("N", daveRow2.NormalizedAnswer);
+
         var actualRow2 = preseasonAnswers.Single(x => x.RowNumber == 2 && x.Subject == "ACTUAL" && x.IsActualOutcome);
         Assert.Equal("N", actualRow2.NormalizedAnswer);
 

tests/F1.Infrastructure.Tests/Contracts/MigrationReconciliationServiceTests.cs

@@ -488,6 +488,71 @@ public async Task ReconcileAndPersistAsync_ProducesPreseasonQuestionAndParticipa
         Assert.Equal(-40, reasonSummary.TotalDeltaPoints);
     }
 
+    [Fact]
+    public async Task ReconcileAndPersistAsync_PreseasonRowsWithDifferentSourceOffsets_AreMatchedByQuestionKeyAndSubject()
+    {
+        var runId = Guid.NewGuid();
+        var options = CreateOptions();
+        await using var dbContext = new F1DbContext(options);
+
+        dbContext.MigrationImportRuns.Add(new MigrationImportRunEntity
+        {
+            Id = runId,
+            SourceFilePath = "/tmp/PhilMigratedSelectionsAndScores.csv",
+            SourceFileChecksum = "abc",
+            IsDryRun = true,
+            Status = "Started",
+            StartedAtUtc = DateTime.UtcNow
+        });
+
+        dbContext.MigrationImportRawRows.Add(new MigrationImportRawRowEntity
+        {
+            ImportRunId = runId,
+            RowNumber = 1,
+            SectionType = "Header",
+            RawPayload = "Question,Philip,Dave,"
+        });
+
+        dbContext.MigrationImportPreseasonImportedTallies.Add(new MigrationImportPreseasonImportedTallyEntity
+        {
+            ImportRunId = runId,
+            RowNumber = 22,
+            QuestionKey = "PRE-002",
+            QuestionText = "Q1",
+            Subject = "Dave",
+            ImportedPoints = 20,
+            RawPoints = "20"
+        });
+
+        dbContext.MigrationImportPreseasonCalculatedScores.Add(new MigrationImportPreseasonCalculatedScoreEntity
+        {
+            ImportRunId = runId,
+            RowNumber = 2,
+            QuestionKey = "PRE-002",
+            QuestionText = "Q1",
+            Subject = "Dave",
+            Points = 20,
+            ReasonCode = "PRESEASON_EXACT"
+        });
+
+        await dbContext.SaveChangesAsync();
+
+        var service = new MigrationReconciliationService(new TestDbContextFactory(options));
+        await service.ReconcileAndPersistAsync(runId, CancellationToken.None);
+
+        var diff = await dbContext.MigrationImportPreseasonQuestionDiffs
+            .SingleAsync(x => x.ImportRunId == runId && x.Subject == "Dave" && x.QuestionKey == "PRE-002");
+
+        Assert.Equal(22, diff.RowNumber);
+        Assert.Equal(20, diff.ImportedPoints);
+        Assert.Equal(20, diff.CalculatedPoints);
+        Assert.Equal(0, diff.DeltaPoints);
+        Assert.Equal("PRESEASON_POINTS_MATCH", diff.ReasonCode);
+        Assert.DoesNotContain("PRESEASON_CALCULATED_MISSING", diff.Explanation, StringComparison.Ordinal);
+        Assert.Contains("preseason-points row 22, column C", diff.Explanation);
+        Assert.Contains("preseason-calculated row 2, column C", diff.Explanation);
+    }
+
     private static string GetGoldenFilePath(string fileName)
     {
         var directory = new DirectoryInfo(AppContext.BaseDirectory);