Merge pull request #2644 from wabbajack-tools/search-index-cleanup

Clean up the SearchIndex validation code I wrote last night a bit, add it to WabbajackClient

Wabbajack.CLI/Verbs/ValidateLists.cs

@@ -38,13 +38,6 @@
 
 namespace Wabbajack.CLI.Verbs;
 
-public struct SearchIndexJson
-{
-
-    public HashSet<string> SearchIndex { get; set; }
-    public Dictionary<string, HashSet<string>> PerListSearchIndex { get; set; }
-}
-
 public class ValidateLists
 {
     private static readonly Uri MirrorPrefix = new("https://mirror.wabbajack.org");
@@ -123,9 +116,9 @@ public async Task<int> Run(AbsolutePath reports, AbsolutePath otherArchives)
         }
 
         // MachineURL - HashSet of mods per list
-        ConcurrentDictionary<string, HashSet<string>> PerModListSearchIndex = new();
+        ConcurrentDictionary<string, HashSet<string>> modsPerList = new();
         // HashSet of all searchable mods
-        HashSet<string> ModListSearchIndex = new();
+        HashSet<string> allMods = new();
 
         var validatedLists = await listData.PMapAll(async modList =>
         {
@@ -181,11 +174,11 @@ public async Task<int> Run(AbsolutePath reports, AbsolutePath otherArchives)
                 {
                     if (archive.State is not Nexus n) continue;
                     if (string.IsNullOrWhiteSpace(n.Name)) continue;
-                    ModListSearchIndex.Add(n.Name);
+                    allMods.Add(n.Name);
                     modListSearchableMods.Add(n.Name);
                 }
 
-                PerModListSearchIndex.TryAdd(modList.Links.MachineURL, modListSearchableMods);
+                modsPerList.TryAdd(modList.Links.MachineURL, modListSearchableMods);
             }
             catch (Exception ex)
             {
@@ -259,7 +252,7 @@ public async Task<int> Run(AbsolutePath reports, AbsolutePath otherArchives)
         {
             await using var searchIndexFileName = reports.Combine("searchIndex.json")
                 .Open(FileMode.Create, FileAccess.Write, FileShare.None);
-            await _dtos.Serialize(new SearchIndexJson { SearchIndex = ModListSearchIndex, PerListSearchIndex = PerModListSearchIndex.ToDictionary() }, searchIndexFileName, true);
+            await _dtos.Serialize(new SearchIndex() { AllMods = allMods, ModsPerList = modsPerList.ToDictionary() }, searchIndexFileName, true);
         }
 
         var allArchives = validatedLists.SelectMany(l => l.Archives).ToList();

Wabbajack.DTOs/SearchIndex.cs

@@ -0,0 +1,16 @@
+using System.Collections.Generic;
+
+namespace Wabbajack.DTOs;
+
+public class SearchIndex
+{
+    /// <summary>
+    /// All unique mods across all modlists
+    /// </summary>
+    public HashSet<string> AllMods { get; set; }
+
+    /// <summary>
+    /// All mods included per modlist (key: machineURL)
+    /// </summary>
+    public Dictionary<string, HashSet<string>> ModsPerList { get; set; }
+}

Wabbajack.Networking.WabbajackClientApi/Client.cs

@@ -259,6 +259,14 @@ public async Task<Dictionary<string, Uri>> LoadRepositories()
         return repositories!;
     }
 
+    public async Task<SearchIndex> LoadSearchIndex()
+    {
+        return await _client.GetFromJsonAsync<SearchIndex>(_limiter,
+            new HttpRequestMessage(HttpMethod.Get, 
+                "https://raw.githubusercontent.com/wabbajack-tools/mod-lists/refs/heads/master/reports/searchIndex.json"),
+                _dtos.Options);
+    }
+
     public Uri GetPatchUrl(Hash upgradeHash, Hash archiveHash)
     {
         return new Uri($"{_configuration.PatchBaseAddress}{upgradeHash.ToHex()}_{archiveHash.ToHex()}");