Feature/refactor database stuff

.gitignore

@@ -8,3 +8,4 @@ Robust.Cdn/content.db*
 Robust.Cdn/manifest.db*
 *.user
 testData/
+/.vs/**

Robust.Cdn/Config/CdnOptions.cs

@@ -1,8 +1,8 @@
-using Robust.Cdn.Services;
+using Robust.Cdn.Services;
 
 namespace Robust.Cdn.Config;
 
-public sealed class CdnOptions
+public sealed class CdnOptions : IDatabaseOptions
 {
     public const string Position = "Cdn";
 

Robust.Cdn/Config/IDatabaseOptions.cs

@@ -0,0 +1,12 @@
+namespace Robust.Cdn.Config;
+
+/// <summary>
+/// Options that contain database config settings.
+/// </summary>
+public interface IDatabaseOptions
+{
+    /// <summary>
+    /// File to be used as SQLite database file. If the file does not exist, it will be created.
+    /// </summary>
+    public string DatabaseFileName { get; }
+}

Robust.Cdn/Config/ManifestOptions.cs

@@ -1,6 +1,6 @@
-namespace Robust.Cdn.Config;
+namespace Robust.Cdn.Config;
 
-public sealed class ManifestOptions
+public sealed class ManifestOptions : IDatabaseOptions
 {
     public const string Position = "Manifest";
 

Robust.Cdn/Controllers/DownloadCompatibilityController.cs

@@ -1,6 +1,7 @@
-using Microsoft.AspNetCore.Mvc;
+using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Options;
 using Robust.Cdn.Config;
+using Robust.Cdn.DataAccessLayer;
 using Robust.Cdn.Services;
 
 namespace Robust.Cdn.Controllers;

Robust.Cdn/Controllers/ForkBuildPageController.cs

@@ -1,8 +1,9 @@
-using System.Diagnostics.CodeAnalysis;
-using Dapper;
+using System.Diagnostics.CodeAnalysis;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Options;
 using Robust.Cdn.Config;
+using Robust.Cdn.DataAccessLayer;
+using Robust.Cdn.DataAccessLayer.Models;
 
 namespace Robust.Cdn.Controllers;
 
@@ -19,38 +20,9 @@ public IActionResult Index(string fork)
         if (!TryCheckBasicAuth(fork, out var errorResult))
             return errorResult;
 
-        var versions = new List<Version>();
+        database.StartTransaction();
 
-        using var tx = database.Connection.BeginTransaction();
-
-        var dbVersions = database.Connection.Query<DbVersion>(
-            """
-            SELECT FV.Id, FV.Name, PublishedTime, EngineVersion
-            FROM ForkVersion FV
-            INNER JOIN main.Fork F ON FV.ForkId = F.Id
-            WHERE F.Name = @Fork
-              AND FV.Available
-            ORDER BY PublishedTime DESC
-            LIMIT 50
-            """, new { Fork = fork });
-
-        foreach (var dbVersion in dbVersions)
-        {
-            var servers = database.Connection.Query<VersionServer>("""
-                SELECT Platform, FileName, FileSize
-                FROM ForkVersionServerBuild
-                WHERE ForkVersionId = @ForkVersionId
-                ORDER BY Platform
-                """, new { ForkVersionId = dbVersion.Id });
-
-            versions.Add(new Version
-            {
-                Name = dbVersion.Name,
-                EngineVersion = dbVersion.EngineVersion,
-                PublishedTime = DateTime.SpecifyKind(dbVersion.PublishedTime, DateTimeKind.Utc),
-                Servers = servers.ToArray()
-            });
-        }
+        var versions = database.ListForkVersions(fork, limit: 50);
 
         return View(new Model
         {
@@ -71,29 +43,6 @@ public sealed class Model
     {
         public required string Fork;
         public required ManifestForkOptions Options;
-        public required List<Version> Versions;
-    }
-
-    public sealed class Version
-    {
-        public required string Name;
-        public required DateTime PublishedTime;
-        public required string? EngineVersion;
-        public required VersionServer[] Servers;
-    }
-
-    public sealed class VersionServer
-    {
-        public required string Platform { get; set; }
-        public required string FileName { get; set; }
-        public required long? FileSize { get; set; }
-    }
-
-    private sealed class DbVersion
-    {
-        public required int Id { get; set; }
-        public required string Name { get; set; }
-        public required DateTime PublishedTime { get; set; }
-        public required string? EngineVersion { get; set; }
+        public required List<BuildVersionArtifactInfo> Versions;
     }
 }

Robust.Cdn/Controllers/ForkDownloadController.cs

@@ -1,17 +1,16 @@
-using System.Buffers.Binary;
+using System.Buffers.Binary;
 using System.Collections;
 using System.Diagnostics;
-using Dapper;
 using Microsoft.AspNetCore.Http.Features;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Options;
 using Robust.Cdn.Config;
+using Robust.Cdn.DataAccessLayer;
 using Robust.Cdn.Helpers;
 using Robust.Cdn.Lib;
 using Robust.Cdn.Services;
 using SharpZstd;
 using SharpZstd.Interop;
-using SQLitePCL;
 
 namespace Robust.Cdn.Controllers;
 
@@ -33,39 +32,24 @@ public sealed class DownloadController(
     [HttpGet("manifest")]
     public IActionResult GetManifest(string fork, string version)
     {
-        var con = db.Connection;
-        con.BeginTransaction(deferred: true);
-
-        var (row, hash) = con.QuerySingleOrDefault<(long, byte[])>(
-            """
-            SELECT CV.Id, CV.ManifestHash
-            FROM ContentVersion CV
-            INNER JOIN main.Fork F on F.Id = CV.ForkId
-            WHERE F.Name = @Fork AND Version = @Version
-            """,
-            new
-            {
-                Fork = fork,
-                Version = version
-            });
-
-        if (row == 0)
+        db.StartTransaction(true);
+        var manifestBlob = db.FindManifestDataBlob(fork, version);
+        if (manifestBlob == null)
             return NotFound();
 
         // I'll be honest I'm not sure how useful this is.
         // I just wanted to make that SELECT less lonely.
-        Response.Headers["X-Manifest-Hash"] = Convert.ToHexString(hash);
+        Response.Headers["X-Manifest-Hash"] = manifestBlob.ManifestHash;
 
-        var blob = SqliteBlobStream.Open(con.Handle!, "main", "ContentVersion", "ManifestData", row, false);
 
         if (AcceptsZStd)
         {
             Response.Headers.ContentEncoding = "zstd";
 
-            return File(blob, "text/plain; charset=utf-8");
+            return File(manifestBlob.Blob, "text/plain; charset=utf-8");
         }
 
-        var decompress = new ZstdDecodeStream(blob, leaveOpen: false);
+        var decompress = new ZstdDecodeStream(manifestBlob.Blob, leaveOpen: false);
 
         return File(decompress, "text/plain; charset=utf-8");
     }
@@ -96,31 +80,12 @@ public async Task<IActionResult> Download(string fork, string version)
         // TODO: this request limiting logic is pretty bad.
         HttpContext.Features.Get<IHttpMaxRequestBodySizeFeature>()!.MaxRequestBodySize = MaxDownloadRequestSize;
 
-        var con = db.Connection;
-        con.BeginTransaction(deferred: true);
-
-        var (versionId, countDistinctBlobs) = con.QuerySingleOrDefault<(long, int)>(
-            """
-            SELECT CV.Id, CV.CountDistinctBlobs
-            FROM ContentVersion CV
-            INNER JOIN main.Fork F on F.Id = CV.ForkId
-            WHERE F.Name = @Fork AND Version = @Version
-            """,
-            new
-            {
-                Fork = fork,
-                Version = version
-            });
+        db.StartTransaction(deferred: true);
 
-        if (versionId == 0)
+        var result = db.GetDistinctBlobsAndManifestEntriesCounts(fork, version);
+        if (result == null)
             return NotFound();
-
-        var entriesCount = con.ExecuteScalar<int>(
-            "SELECT COUNT(*) FROM ContentManifestEntry WHERE VersionId = @VersionId",
-            new
-            {
-                VersionId = versionId
-            });
+        var (versionId, countDistinctBlobs, entriesCount) = result.Value;
 
         var buffer = new MemoryStream();
         await Request.Body.CopyToAsync(buffer);
@@ -157,7 +122,7 @@ FROM ContentVersion CV
 
         if (optAutoStreamCompressRatio > 0)
         {
-            var requestRatio = countFilesRequested / (float) countDistinctBlobs;
+            var requestRatio = countFilesRequested / (float)countDistinctBlobs;
             logger.LogTrace("Auto stream compression ratio: {RequestRatio}", requestRatio);
             if (requestRatio > optAutoStreamCompressRatio)
             {
@@ -216,20 +181,10 @@ FROM ContentVersion CV
 
             await outStream.WriteAsync(streamHeader);
 
-            SqliteBlobStream? blob = null;
             ZStdDecompressStream? decompress = null;
 
             try
             {
-                using var stmt =
-                    con.Handle!.Prepare(
-                        "SELECT c.Compression, c.Size, c.Id " +
-                        "FROM ContentManifestEntry cme " +
-                        "INNER JOIN Content c on c.Id = cme.ContentId " +
-                        "WHERE cme.VersionId = @VersionId AND cme.ManifestIdx = @ManifestIdx");
-
-                stmt.BindInt64(1, versionId); // @VersionId
-
                 offset = 0;
                 var swSqlite = new Stopwatch();
                 var count = 0;
@@ -238,32 +193,16 @@ FROM ContentVersion CV
                     var index = BinaryPrimitives.ReadInt32LittleEndian(buf.Slice(offset, 4).Span);
 
                     swSqlite.Start();
-                    stmt.BindInt(2, index);
 
-                    if (stmt.Step() != raw.SQLITE_ROW)
-                        throw new InvalidOperationException("Unable to find manifest row??");
+                    var (compression, size, rowId) = db.ListContentMetadata(versionId, index);
 
-                    var compression = (ContentCompression)stmt.ColumnInt(0);
-                    var size = stmt.ColumnInt(1);
-                    var rowId = stmt.ColumnInt64(2);
-
-                    stmt.Reset();
                     swSqlite.Stop();
 
-                    // _aczSawmill.Debug($"{index:D5}: {blobLength:D8} {dataOffset:D8} {dataLength:D8}");
-
                     BinaryPrimitives.WriteInt32LittleEndian(fileHeader, size);
 
-                    if (blob == null)
-                    {
-                        blob = SqliteBlobStream.Open(con.Handle!, "main", "Content", "Data", rowId, false);
-                        if (!preCompressed)
-                            decompress = new ZStdDecompressStream(blob, ownStream: false);
-                    }
-                    else
-                    {
-                        blob.Reopen(rowId);
-                    }
+                    var blob = db.OpenContentBlobForRead(rowId);
+                    if (!preCompressed)
+                        decompress = new ZStdDecompressStream(blob, ownStream: false);
 
                     Stream copyFromStream = blob;
                     if (preCompressed)
@@ -294,7 +233,6 @@ FROM ContentVersion CV
             }
             finally
             {
-                blob?.Dispose();
                 decompress?.Dispose();
             }
         }

Robust.Cdn/Controllers/ForkManifestController.cs

@@ -1,9 +1,9 @@
-using System.Diagnostics.CodeAnalysis;
+using System.Diagnostics.CodeAnalysis;
 using System.Net.Mime;
-using Dapper;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Options;
 using Robust.Cdn.Config;
+using Robust.Cdn.DataAccessLayer;
 using Robust.Cdn.Helpers;
 
 namespace Robust.Cdn.Controllers;
@@ -15,7 +15,7 @@ namespace Robust.Cdn.Controllers;
 [ApiController]
 [Route("/fork/{fork}")]
 public sealed class ForkManifestController(
-    ManifestDatabase database,
+    ManifestDatabase manifestDatabase,
     BuildDirectoryManager buildDirectoryManager,
     IOptions<ManifestOptions> manifestOptions)
     : ControllerBase
@@ -26,21 +26,10 @@ public IActionResult GetManifest(string fork)
         if (!TryCheckBasicAuth(fork, out var errorResult))
             return errorResult;
 
-        var rowId = database.Connection.QuerySingleOrDefault<long>(
-            "SELECT ROWID FROM Fork WHERE Name == @Fork AND ServerManifestCache IS NOT NULL",
-            new { Fork = fork });
-
-        if (rowId == 0)
+        var stream = manifestDatabase.FindManifestCache(fork);
+        if (stream == null)
             return NotFound();
 
-        var stream = SqliteBlobStream.Open(
-            database.Connection.Handle!,
-            "main",
-            "Fork",
-            "ServerManifestCache",
-            rowId,
-            false);
-
         return File(stream, MediaTypeNames.Application.Json);
     }
 
@@ -57,14 +46,7 @@ public IActionResult GetFile(
         if (!TryCheckBasicAuth(fork, out var errorResult))
             return errorResult;
 
-        var versionExists = database.Connection.QuerySingleOrDefault<bool>("""
-            SELECT 1
-            FROM ForkVersion, Fork
-            WHERE ForkVersion.Name = @Version
-              AND Fork.Name = @Fork
-              AND Fork.Id = ForkVersion.ForkId
-            """, new { Fork = fork, Version = version });
-
+        var versionExists = manifestDatabase.IsVersionExists(fork, version);
         if (!versionExists)
             return NotFound();