diff --git a/.gitignore b/.gitignore index 03b6e2d..1ecb8e3 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ Robust.Cdn/content.db* Robust.Cdn/manifest.db* *.user testData/ +/.vs/** diff --git a/Robust.Cdn/Config/CdnOptions.cs b/Robust.Cdn/Config/CdnOptions.cs index 53c407e..b7de750 100644 --- a/Robust.Cdn/Config/CdnOptions.cs +++ b/Robust.Cdn/Config/CdnOptions.cs @@ -1,8 +1,8 @@ -using Robust.Cdn.Services; +using Robust.Cdn.Services; namespace Robust.Cdn.Config; -public sealed class CdnOptions +public sealed class CdnOptions : IDatabaseOptions { public const string Position = "Cdn"; diff --git a/Robust.Cdn/Config/IDatabaseOptions.cs b/Robust.Cdn/Config/IDatabaseOptions.cs new file mode 100644 index 0000000..68b7ee0 --- /dev/null +++ b/Robust.Cdn/Config/IDatabaseOptions.cs @@ -0,0 +1,12 @@ +namespace Robust.Cdn.Config; + +/// +/// Options that contain database config settings. +/// +public interface IDatabaseOptions +{ + /// + /// File to be used as SQLite database file. If the file does not exist, it will be created. + /// + public string DatabaseFileName { get; } +} diff --git a/Robust.Cdn/Config/ManifestOptions.cs b/Robust.Cdn/Config/ManifestOptions.cs index b85a5e7..f41ea54 100644 --- a/Robust.Cdn/Config/ManifestOptions.cs +++ b/Robust.Cdn/Config/ManifestOptions.cs @@ -1,6 +1,6 @@ -namespace Robust.Cdn.Config; +namespace Robust.Cdn.Config; -public sealed class ManifestOptions +public sealed class ManifestOptions : IDatabaseOptions { public const string Position = "Manifest"; diff --git a/Robust.Cdn/Controllers/DownloadCompatibilityController.cs b/Robust.Cdn/Controllers/DownloadCompatibilityController.cs index e763be4..0fea5e8 100644 --- a/Robust.Cdn/Controllers/DownloadCompatibilityController.cs +++ b/Robust.Cdn/Controllers/DownloadCompatibilityController.cs @@ -1,6 +1,7 @@ -using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Options; using Robust.Cdn.Config; +using Robust.Cdn.DataAccessLayer; using Robust.Cdn.Services; namespace Robust.Cdn.Controllers; diff --git a/Robust.Cdn/Controllers/ForkBuildPageController.cs b/Robust.Cdn/Controllers/ForkBuildPageController.cs index 0b3b030..799ba22 100644 --- a/Robust.Cdn/Controllers/ForkBuildPageController.cs +++ b/Robust.Cdn/Controllers/ForkBuildPageController.cs @@ -1,8 +1,9 @@ -using System.Diagnostics.CodeAnalysis; -using Dapper; +using System.Diagnostics.CodeAnalysis; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Options; using Robust.Cdn.Config; +using Robust.Cdn.DataAccessLayer; +using Robust.Cdn.DataAccessLayer.Models; namespace Robust.Cdn.Controllers; @@ -19,38 +20,9 @@ public IActionResult Index(string fork) if (!TryCheckBasicAuth(fork, out var errorResult)) return errorResult; - var versions = new List(); + database.StartTransaction(); - using var tx = database.Connection.BeginTransaction(); - - var dbVersions = database.Connection.Query( - """ - SELECT FV.Id, FV.Name, PublishedTime, EngineVersion - FROM ForkVersion FV - INNER JOIN main.Fork F ON FV.ForkId = F.Id - WHERE F.Name = @Fork - AND FV.Available - ORDER BY PublishedTime DESC - LIMIT 50 - """, new { Fork = fork }); - - foreach (var dbVersion in dbVersions) - { - var servers = database.Connection.Query(""" - SELECT Platform, FileName, FileSize - FROM ForkVersionServerBuild - WHERE ForkVersionId = @ForkVersionId - ORDER BY Platform - """, new { ForkVersionId = dbVersion.Id }); - - versions.Add(new Version - { - Name = dbVersion.Name, - EngineVersion = dbVersion.EngineVersion, - PublishedTime = DateTime.SpecifyKind(dbVersion.PublishedTime, DateTimeKind.Utc), - Servers = servers.ToArray() - }); - } + var versions = database.ListForkVersions(fork, limit: 50); return View(new Model { @@ -71,29 +43,6 @@ public sealed class Model { public required string Fork; public required ManifestForkOptions Options; - public required List Versions; - } - - public sealed class Version - { - public required string Name; - public required DateTime PublishedTime; - public required string? EngineVersion; - public required VersionServer[] Servers; - } - - public sealed class VersionServer - { - public required string Platform { get; set; } - public required string FileName { get; set; } - public required long? FileSize { get; set; } - } - - private sealed class DbVersion - { - public required int Id { get; set; } - public required string Name { get; set; } - public required DateTime PublishedTime { get; set; } - public required string? EngineVersion { get; set; } + public required List Versions; } } diff --git a/Robust.Cdn/Controllers/ForkDownloadController.cs b/Robust.Cdn/Controllers/ForkDownloadController.cs index 9001003..0c63fe3 100644 --- a/Robust.Cdn/Controllers/ForkDownloadController.cs +++ b/Robust.Cdn/Controllers/ForkDownloadController.cs @@ -1,17 +1,16 @@ -using System.Buffers.Binary; +using System.Buffers.Binary; using System.Collections; using System.Diagnostics; -using Dapper; using Microsoft.AspNetCore.Http.Features; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Options; using Robust.Cdn.Config; +using Robust.Cdn.DataAccessLayer; using Robust.Cdn.Helpers; using Robust.Cdn.Lib; using Robust.Cdn.Services; using SharpZstd; using SharpZstd.Interop; -using SQLitePCL; namespace Robust.Cdn.Controllers; @@ -33,39 +32,24 @@ public sealed class DownloadController( [HttpGet("manifest")] public IActionResult GetManifest(string fork, string version) { - var con = db.Connection; - con.BeginTransaction(deferred: true); - - var (row, hash) = con.QuerySingleOrDefault<(long, byte[])>( - """ - SELECT CV.Id, CV.ManifestHash - FROM ContentVersion CV - INNER JOIN main.Fork F on F.Id = CV.ForkId - WHERE F.Name = @Fork AND Version = @Version - """, - new - { - Fork = fork, - Version = version - }); - - if (row == 0) + db.StartTransaction(true); + var manifestBlob = db.FindManifestDataBlob(fork, version); + if (manifestBlob == null) return NotFound(); // I'll be honest I'm not sure how useful this is. // I just wanted to make that SELECT less lonely. - Response.Headers["X-Manifest-Hash"] = Convert.ToHexString(hash); + Response.Headers["X-Manifest-Hash"] = manifestBlob.ManifestHash; - var blob = SqliteBlobStream.Open(con.Handle!, "main", "ContentVersion", "ManifestData", row, false); if (AcceptsZStd) { Response.Headers.ContentEncoding = "zstd"; - return File(blob, "text/plain; charset=utf-8"); + return File(manifestBlob.Blob, "text/plain; charset=utf-8"); } - var decompress = new ZstdDecodeStream(blob, leaveOpen: false); + var decompress = new ZstdDecodeStream(manifestBlob.Blob, leaveOpen: false); return File(decompress, "text/plain; charset=utf-8"); } @@ -96,31 +80,12 @@ public async Task Download(string fork, string version) // TODO: this request limiting logic is pretty bad. HttpContext.Features.Get()!.MaxRequestBodySize = MaxDownloadRequestSize; - var con = db.Connection; - con.BeginTransaction(deferred: true); - - var (versionId, countDistinctBlobs) = con.QuerySingleOrDefault<(long, int)>( - """ - SELECT CV.Id, CV.CountDistinctBlobs - FROM ContentVersion CV - INNER JOIN main.Fork F on F.Id = CV.ForkId - WHERE F.Name = @Fork AND Version = @Version - """, - new - { - Fork = fork, - Version = version - }); + db.StartTransaction(deferred: true); - if (versionId == 0) + var result = db.GetDistinctBlobsAndManifestEntriesCounts(fork, version); + if (result == null) return NotFound(); - - var entriesCount = con.ExecuteScalar( - "SELECT COUNT(*) FROM ContentManifestEntry WHERE VersionId = @VersionId", - new - { - VersionId = versionId - }); + var (versionId, countDistinctBlobs, entriesCount) = result.Value; var buffer = new MemoryStream(); await Request.Body.CopyToAsync(buffer); @@ -157,7 +122,7 @@ FROM ContentVersion CV if (optAutoStreamCompressRatio > 0) { - var requestRatio = countFilesRequested / (float) countDistinctBlobs; + var requestRatio = countFilesRequested / (float)countDistinctBlobs; logger.LogTrace("Auto stream compression ratio: {RequestRatio}", requestRatio); if (requestRatio > optAutoStreamCompressRatio) { @@ -216,20 +181,10 @@ FROM ContentVersion CV await outStream.WriteAsync(streamHeader); - SqliteBlobStream? blob = null; ZStdDecompressStream? decompress = null; try { - using var stmt = - con.Handle!.Prepare( - "SELECT c.Compression, c.Size, c.Id " + - "FROM ContentManifestEntry cme " + - "INNER JOIN Content c on c.Id = cme.ContentId " + - "WHERE cme.VersionId = @VersionId AND cme.ManifestIdx = @ManifestIdx"); - - stmt.BindInt64(1, versionId); // @VersionId - offset = 0; var swSqlite = new Stopwatch(); var count = 0; @@ -238,32 +193,16 @@ FROM ContentVersion CV var index = BinaryPrimitives.ReadInt32LittleEndian(buf.Slice(offset, 4).Span); swSqlite.Start(); - stmt.BindInt(2, index); - if (stmt.Step() != raw.SQLITE_ROW) - throw new InvalidOperationException("Unable to find manifest row??"); + var (compression, size, rowId) = db.ListContentMetadata(versionId, index); - var compression = (ContentCompression)stmt.ColumnInt(0); - var size = stmt.ColumnInt(1); - var rowId = stmt.ColumnInt64(2); - - stmt.Reset(); swSqlite.Stop(); - // _aczSawmill.Debug($"{index:D5}: {blobLength:D8} {dataOffset:D8} {dataLength:D8}"); - BinaryPrimitives.WriteInt32LittleEndian(fileHeader, size); - if (blob == null) - { - blob = SqliteBlobStream.Open(con.Handle!, "main", "Content", "Data", rowId, false); - if (!preCompressed) - decompress = new ZStdDecompressStream(blob, ownStream: false); - } - else - { - blob.Reopen(rowId); - } + var blob = db.OpenContentBlobForRead(rowId); + if (!preCompressed) + decompress = new ZStdDecompressStream(blob, ownStream: false); Stream copyFromStream = blob; if (preCompressed) @@ -294,7 +233,6 @@ FROM ContentVersion CV } finally { - blob?.Dispose(); decompress?.Dispose(); } } diff --git a/Robust.Cdn/Controllers/ForkManifestController.cs b/Robust.Cdn/Controllers/ForkManifestController.cs index 355555d..5005750 100644 --- a/Robust.Cdn/Controllers/ForkManifestController.cs +++ b/Robust.Cdn/Controllers/ForkManifestController.cs @@ -1,9 +1,9 @@ -using System.Diagnostics.CodeAnalysis; +using System.Diagnostics.CodeAnalysis; using System.Net.Mime; -using Dapper; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Options; using Robust.Cdn.Config; +using Robust.Cdn.DataAccessLayer; using Robust.Cdn.Helpers; namespace Robust.Cdn.Controllers; @@ -15,7 +15,7 @@ namespace Robust.Cdn.Controllers; [ApiController] [Route("/fork/{fork}")] public sealed class ForkManifestController( - ManifestDatabase database, + ManifestDatabase manifestDatabase, BuildDirectoryManager buildDirectoryManager, IOptions manifestOptions) : ControllerBase @@ -26,21 +26,10 @@ public IActionResult GetManifest(string fork) if (!TryCheckBasicAuth(fork, out var errorResult)) return errorResult; - var rowId = database.Connection.QuerySingleOrDefault( - "SELECT ROWID FROM Fork WHERE Name == @Fork AND ServerManifestCache IS NOT NULL", - new { Fork = fork }); - - if (rowId == 0) + var stream = manifestDatabase.FindManifestCache(fork); + if (stream == null) return NotFound(); - var stream = SqliteBlobStream.Open( - database.Connection.Handle!, - "main", - "Fork", - "ServerManifestCache", - rowId, - false); - return File(stream, MediaTypeNames.Application.Json); } @@ -57,14 +46,7 @@ public IActionResult GetFile( if (!TryCheckBasicAuth(fork, out var errorResult)) return errorResult; - var versionExists = database.Connection.QuerySingleOrDefault(""" - SELECT 1 - FROM ForkVersion, Fork - WHERE ForkVersion.Name = @Version - AND Fork.Name = @Fork - AND Fork.Id = ForkVersion.ForkId - """, new { Fork = fork, Version = version }); - + var versionExists = manifestDatabase.IsVersionExists(fork, version); if (!versionExists) return NotFound(); diff --git a/Robust.Cdn/Controllers/ForkPublishController.Multi.cs b/Robust.Cdn/Controllers/ForkPublishController.Multi.cs index 935a9dc..0b2531b 100644 --- a/Robust.Cdn/Controllers/ForkPublishController.Multi.cs +++ b/Robust.Cdn/Controllers/ForkPublishController.Multi.cs @@ -1,5 +1,5 @@ -using Dapper; using Microsoft.AspNetCore.Mvc; +using Robust.Cdn.DataAccessLayer.Models; using Robust.Cdn.Helpers; namespace Robust.Cdn.Controllers; @@ -23,44 +23,36 @@ public async Task MultiPublishStart( if (!ValidVersionRegex.IsMatch(request.Version)) return BadRequest("Invalid version name"); - if (VersionAlreadyExists(fork, request.Version)) - return Conflict("Version already exists"); - - var dbCon = manifestDatabase.Connection; + manifestDatabase.StartTransaction(); - await using var tx = await dbCon.BeginTransactionAsync(cancel); + if (manifestDatabase.IsVersionExists(fork, request.Version)) + return Conflict("Version already exists"); logger.LogInformation("Starting multi publish for fork {Fork} version {Version}", fork, request.Version); - var forkId = dbCon.QuerySingle("SELECT Id FROM Fork WHERE Name = @Name", new { Name = fork }); - var hasExistingPublish = dbCon.QuerySingleOrDefault( - "SELECT 1 FROM PublishInProgress WHERE Version = @Version AND ForkId = @ForkId", - new { request.Version, ForkId = forkId }); - if (hasExistingPublish) + var forkId = manifestDatabase.GetForkIdByName(fork); + var isPublishInProgress = manifestDatabase.IsPublishInProgress(forkId, request.Version); + if (isPublishInProgress) { // If a publish with this name already exists we abort it and start again. // We do this so you can "just" retry a mid-way-failed publish without an extra API call required. logger.LogWarning("Already had an in-progress publish for this version, aborting it and restarting."); - publishManager.AbortMultiPublish(fork, request.Version, tx, commit: false); + publishManager.AbortMultiPublish(fork, request.Version); } - await dbCon.ExecuteAsync(""" - INSERT INTO PublishInProgress (Version, ForkId, StartTime, EngineVersion) - VALUES (@Version, @ForkId, @StartTime, @EngineVersion) - """, - new - { - request.Version, - request.EngineVersion, - ForkId = forkId, - StartTime = DateTime.UtcNow - }); + manifestDatabase.InsertPublishInProgress( + request.Version, + request.EngineVersion, + forkId, + new SourceVersionInfo(request.SourceUrl, request.SourceCommitId, request.SourceBranchName), + new SourceVersionInfo(request.EngineSourceUrl, request.EngineSourceCommitId, request.EngineSourceBranchName) + ); var versionDir = buildDirectoryManager.GetBuildVersionPath(fork, request.Version); Directory.CreateDirectory(versionDir); - await tx.CommitAsync(cancel); + manifestDatabase.Commit(); logger.LogInformation("Multi publish initiated. Waiting for subsequent API requests..."); @@ -83,18 +75,11 @@ public async Task MultiPublishFile( if (!ValidFileRegex.IsMatch(fileName)) return BadRequest("Invalid artifact file name"); - var dbCon = manifestDatabase.Connection; - await using var tx = await dbCon.BeginTransactionAsync(cancel); - - var forkId = dbCon.QuerySingle("SELECT Id FROM Fork WHERE Name = @Name", new { Name = fork }); - var versionId = dbCon.QuerySingleOrDefault(""" - SELECT Id - FROM PublishInProgress - WHERE Version = @Name AND ForkId = @Fork - """, - new { Name = version, Fork = forkId }); + manifestDatabase.StartTransaction(); - if (versionId == null) + var forkId = manifestDatabase.GetForkIdByName(fork); + var isPublishInProgress = manifestDatabase.IsPublishInProgress(forkId, version); + if (!isPublishInProgress) return NotFound("Unknown in-progress version"); var versionDir = buildDirectoryManager.GetBuildVersionPath(fork, version); @@ -123,17 +108,10 @@ public async Task MultiPublishFinish( if (!authHelper.IsAuthValid(fork, out var forkConfig, out var failureResult)) return failureResult; - var dbCon = manifestDatabase.Connection; - await using var tx = await dbCon.BeginTransactionAsync(cancel); - - var forkId = dbCon.QuerySingle("SELECT Id FROM Fork WHERE Name = @Name", new { Name = fork }); - var versionMetadata = dbCon.QuerySingleOrDefault(""" - SELECT Version, EngineVersion - FROM PublishInProgress - WHERE Version = @Name AND ForkId = @Fork - """, - new { Name = request.Version, Fork = forkId }); + manifestDatabase.StartTransaction(); + var forkId = manifestDatabase.GetForkIdByName(fork); + var versionMetadata = manifestDatabase.GetVersionMetadata(forkId, request.Version); if (versionMetadata == null) return NotFound("Unknown in-progress version"); @@ -151,7 +129,8 @@ FROM PublishInProgress var clientArtifact = artifacts.SingleOrNull(art => art.artifact.Type == ArtifactType.Client); if (clientArtifact == null) { - publishManager.AbortMultiPublish(fork, request.Version, tx, commit: true); + publishManager.AbortMultiPublish(fork, request.Version); + manifestDatabase.Commit(); return UnprocessableEntity("Publish failed: no client zip was provided"); } @@ -160,13 +139,11 @@ FROM PublishInProgress var buildJson = GenerateBuildJson(diskFiles, clientArtifact.Value.artifact, versionMetadata, fork); InjectBuildJsonIntoServers(diskFiles, buildJson); - AddVersionToDatabase(clientArtifact.Value.artifact, diskFiles, fork, versionMetadata); + manifestDatabase.AddVersionsToDatabase(clientArtifact.Value.artifact, diskFiles, fork, versionMetadata); - dbCon.Execute( - "DELETE FROM PublishInProgress WHERE Version = @Name AND ForkId = @Fork", - new { Name = request.Version, Fork = forkId }); + manifestDatabase.DeleteVersionByVersionName(fork, request.Version); - tx.Commit(); + manifestDatabase.Commit(); await QueueIngestJobAsync(fork); @@ -175,11 +152,10 @@ FROM PublishInProgress return NoContent(); } - public sealed class PublishMultiRequest - { - public required string Version { get; set; } - public required string EngineVersion { get; set; } - } + /// + /// Request for start of multi-step publishing process of a new version. + /// + public sealed class PublishMultiRequest : PublishStartRequestBase; public sealed class PublishFinishRequest { diff --git a/Robust.Cdn/Controllers/ForkPublishController.OneShot.cs b/Robust.Cdn/Controllers/ForkPublishController.OneShot.cs index 75bcf74..cfc3311 100644 --- a/Robust.Cdn/Controllers/ForkPublishController.OneShot.cs +++ b/Robust.Cdn/Controllers/ForkPublishController.OneShot.cs @@ -1,5 +1,6 @@ -using System.IO.Compression; +using System.IO.Compression; using Microsoft.AspNetCore.Mvc; +using Robust.Cdn.DataAccessLayer.Models; using Robust.Cdn.Helpers; namespace Robust.Cdn.Controllers; @@ -25,7 +26,7 @@ public async Task PostPublish( if (!ValidVersionRegex.IsMatch(request.Version)) return BadRequest("Invalid version name"); - if (VersionAlreadyExists(fork, request.Version)) + if (manifestDatabase.IsVersionExists(fork, request.Version)) return Conflict("Version already exists"); logger.LogInformation("Starting one-shot publish for fork {Fork} version {Version}", fork, request.Version); @@ -51,7 +52,12 @@ public async Task PostPublish( var versionDir = buildDirectoryManager.GetBuildVersionPath(fork, request.Version); - var metadata = new VersionMetadata { Version = request.Version, EngineVersion = request.EngineVersion }; + var metadata = new VersionMetadata( + request.Version, + request.EngineVersion, + new(request.SourceUrl, request.SourceCommitId, request.SourceBranchName), + new(request.EngineSourceUrl, request.EngineSourceCommitId, request.EngineSourceBranchName) + ); try { @@ -61,15 +67,11 @@ public async Task PostPublish( var buildJson = GenerateBuildJson(diskFiles, clientArtifact.Value.artifact, metadata, fork); InjectBuildJsonIntoServers(diskFiles, buildJson); - using var tx = manifestDatabase.Connection.BeginTransaction(); + logger.LogDebug("Adding new version to database"); - AddVersionToDatabase( - clientArtifact.Value.artifact, - diskFiles, - fork, - metadata); - - tx.Commit(); + manifestDatabase.StartTransaction(); + manifestDatabase.AddVersionsToDatabase(clientArtifact.Value.artifact, diskFiles, fork, metadata); + manifestDatabase.Commit(); await QueueIngestJobAsync(fork); @@ -86,13 +88,13 @@ public async Task PostPublish( } } - private Dictionary ExtractZipToVersionDir( - List<(ZipArchiveEntry entry, Artifact artifact)> artifacts, + private Dictionary ExtractZipToVersionDir( + List<(ZipArchiveEntry entry, ArtifactBriefInfo artifact)> artifacts, string versionDir) { logger.LogDebug("Extracting artifacts to directory {Directory}", versionDir); - var dict = new Dictionary(); + var dict = new Dictionary(); foreach (var (entry, artifact) in artifacts) { diff --git a/Robust.Cdn/Controllers/ForkPublishController.cs b/Robust.Cdn/Controllers/ForkPublishController.cs index 17bd747..2650ca4 100644 --- a/Robust.Cdn/Controllers/ForkPublishController.cs +++ b/Robust.Cdn/Controllers/ForkPublishController.cs @@ -1,12 +1,13 @@ -using System.IO.Compression; +using System.IO.Compression; using System.Security.Cryptography; using System.Text; using System.Text.Json; using System.Text.RegularExpressions; -using Dapper; using Microsoft.AspNetCore.Mvc; using Quartz; using Robust.Cdn.Config; +using Robust.Cdn.DataAccessLayer; +using Robust.Cdn.DataAccessLayer.Models; using Robust.Cdn.Helpers; using Robust.Cdn.Jobs; using Robust.Cdn.Services; @@ -50,28 +51,12 @@ public sealed partial class ForkPublishController( public const string PublishFetchHttpClient = "PublishFetch"; - private bool VersionAlreadyExists(string fork, string version) - { - return manifestDatabase.Connection.QuerySingleOrDefault( - """ - SELECT 1 - FROM Fork, ForkVersion - WHERE Fork.Id = ForkVersion.ForkId - AND Fork.Name = @ForkName - AND ForkVersion.Name = @ForkVersion - """, new - { - ForkName = fork, - ForkVersion = version - }); - } - - private List<(T key, Artifact artifact)> ClassifyEntries( + private List<(T key, ArtifactBriefInfo artifact)> ClassifyEntries( ManifestForkOptions forkConfig, IEnumerable items, Func getName) { - var list = new List<(T, Artifact)>(); + var list = new List<(T, ArtifactBriefInfo)>(); foreach (var item in items) { @@ -93,15 +78,15 @@ SELECT 1 return list; } - private static Artifact? ClassifyEntry(ManifestForkOptions forkConfig, string name) + private static ArtifactBriefInfo? ClassifyEntry(ManifestForkOptions forkConfig, string name) { if (name == $"{forkConfig.ClientZipName}.zip") - return new Artifact { Type = ArtifactType.Client }; + return new ArtifactBriefInfo { Type = ArtifactType.Client }; if (name.StartsWith(forkConfig.ServerZipName) && name.EndsWith(".zip")) { var rid = name[forkConfig.ServerZipName.Length..^".zip".Length]; - return new Artifact + return new ArtifactBriefInfo { Platform = rid, Type = ArtifactType.Server @@ -112,14 +97,14 @@ SELECT 1 } private MemoryStream GenerateBuildJson( - Dictionary diskFiles, - Artifact clientArtifact, + Dictionary diskFiles, + ArtifactBriefInfo clientArtifactBriefInfo, VersionMetadata metadata, string forkName) { logger.LogDebug("Generating build.json contents"); - var diskPath = diskFiles[clientArtifact]; + var diskPath = diskFiles[clientArtifactBriefInfo]; var diskFileName = Path.GetFileName(diskPath); using var file = System.IO.File.OpenRead(diskPath); @@ -132,7 +117,7 @@ private MemoryStream GenerateBuildJson( logger.LogDebug("Client zip hash is {ZipHash}, manifest hash is {ManifestHash}", hash, manifestHash); - var data = new Dictionary + var data = new Dictionary { { "download", baseUrlManager.MakeBuildInfoUrl($"fork/{{FORK_ID}}/version/{{FORK_VERSION}}/file/{diskFileName}") }, { "version", metadata.Version }, @@ -141,7 +126,13 @@ private MemoryStream GenerateBuildJson( { "engine_version", metadata.EngineVersion }, { "manifest_url", baseUrlManager.MakeBuildInfoUrl("fork/{FORK_ID}/version/{FORK_VERSION}/manifest") }, { "manifest_download_url", baseUrlManager.MakeBuildInfoUrl("fork/{FORK_ID}/version/{FORK_VERSION}/download") }, - { "manifest_hash", manifestHash } + { "manifest_hash", manifestHash }, + { "built_on_source_url", metadata.BuildVersionInfo.SourceUrl }, + { "built_on_commit_id", metadata.BuildVersionInfo.CommitId }, + { "built_on_branch_name", metadata.BuildVersionInfo.BranchName}, + { "built_on_engine_url", metadata.EngineSourceVersionInfo.SourceUrl }, + { "built_on_engine_commit_id", metadata.EngineSourceVersionInfo.CommitId }, + { "built_on_engine_branch_name", metadata.EngineSourceVersionInfo.BranchName }, }; var stream = new MemoryStream(); @@ -185,7 +176,7 @@ private static byte[] GetZipEntryBlake2B(ZipArchiveEntry entry) return HashHelper.HashBlake2B(stream); } - private void InjectBuildJsonIntoServers(Dictionary diskFiles, MemoryStream buildJson) + private void InjectBuildJsonIntoServers(Dictionary diskFiles, MemoryStream buildJson) { logger.LogDebug("Adding build.json to server builds"); @@ -213,63 +204,7 @@ private void InjectBuildJsonIntoServers(Dictionary diskFiles, } } - private void AddVersionToDatabase( - Artifact clientArtifact, - Dictionary diskFiles, - string fork, - VersionMetadata metadata) - { - logger.LogDebug("Adding new version to database"); - - var dbCon = manifestDatabase.Connection; - - var forkId = dbCon.QuerySingle("SELECT Id FROM Fork WHERE Name = @Name", new { Name = fork }); - - var (clientName, clientSha256, _) = GetFileNameSha256Pair(diskFiles[clientArtifact]); - - var versionId = dbCon.QuerySingle(""" - INSERT INTO ForkVersion (Name, ForkId, PublishedTime, ClientFileName, ClientSha256, EngineVersion) - VALUES (@Name, @ForkId, @PublishTime, @ClientName, @ClientSha256, @EngineVersion) - RETURNING Id - """, - new - { - Name = metadata.Version, - ForkId = forkId, - ClientName = clientName, - ClientSha256 = clientSha256, - metadata.EngineVersion, - PublishTime = DateTime.UtcNow - }); - - foreach (var (artifact, diskPath) in diskFiles) - { - if (artifact.Type != ArtifactType.Server) - continue; - - var (serverName, serverSha256, fileSize) = GetFileNameSha256Pair(diskPath); - - dbCon.Execute(""" - INSERT INTO ForkVersionServerBuild (ForkVersionId, Platform, FileName, Sha256, FileSize) - VALUES (@ForkVersion, @Platform, @ServerName, @ServerSha256, @FileSize) - """, - new - { - ForkVersion = versionId, - artifact.Platform, - ServerName = serverName, - ServerSha256 = serverSha256, - FileSize = fileSize - }); - } - } - - private static (string name, byte[] hash, long size) GetFileNameSha256Pair(string diskPath) - { - using var file = System.IO.File.OpenRead(diskPath); - return (Path.GetFileName(diskPath), SHA256.HashData(file), file.Length); - } private async Task QueueIngestJobAsync(string fork) { @@ -290,19 +225,66 @@ private static FileStream CreateTempFile() FileOptions.DeleteOnClose); } - public sealed class PublishRequest + /// + /// Base type with publish start info. + /// + public abstract class PublishStartRequestBase { + /// + /// Human-readable version of the build. This is used to identify the build in the CDN and in the game client. + /// public required string Version { get; set; } + + /// + /// Human-readable version of the engine used to build this version. + /// public required string EngineVersion { get; set; } - public required string Archive { get; set; } + + /// + /// URL of the fork repository. Optional, but useful for debugging. + /// + public string? SourceUrl { get; set; } + + /// + /// Commit ID on which version was built. Optional, but useful for debugging. + /// + public string? SourceCommitId { get; set; } + + /// + /// Branch on which version was built. Optional, but useful for debugging. + /// + public string? SourceBranchName { get; set; } + + /// + /// Url for RobustToolbox repository (or its fork), used for this version. Optional, but useful for debugging. + /// + public string? EngineSourceUrl { get; set; } + + /// + /// Branch on which RobustToolbox for this version was built. Optional, but useful for debugging. + /// + public string? EngineSourceBranchName { get; set; } + + /// + /// Commit ID of RobustToolbox, used for this version. Optional, but useful for debugging. + /// + public string? EngineSourceCommitId { get; set; } } - private sealed class VersionMetadata + /// + /// Request for one-shot publishing of a new version. + /// + /// + public sealed class PublishRequest : PublishStartRequestBase { - public required string Version { get; init; } - public required string EngineVersion { get; set; } + /// + /// Uri for new build artifact. + /// + public required string Archive { get; set; } } + + // File cannot start with a dot but otherwise most shit is fair game. [GeneratedRegex(@"[a-zA-Z0-9\-_][a-zA-Z0-9\-_.]*")] private static partial Regex ValidVersionRegexBuilder(); @@ -310,15 +292,4 @@ private sealed class VersionMetadata [GeneratedRegex(@"[a-zA-Z0-9\-_][a-zA-Z0-9\-_.]*")] private static partial Regex ValidFileRegexBuilder(); - private sealed class Artifact - { - public ArtifactType Type { get; set; } - public string? Platform { get; set; } - } - - private enum ArtifactType - { - Server, - Client - } } diff --git a/Robust.Cdn/Controllers/StatusController.cs b/Robust.Cdn/Controllers/StatusController.cs index ca7310d..1b1eda9 100644 --- a/Robust.Cdn/Controllers/StatusController.cs +++ b/Robust.Cdn/Controllers/StatusController.cs @@ -1,26 +1,18 @@ -using System.Reflection; -using Dapper; +using System.Reflection; using Microsoft.AspNetCore.Mvc; +using Robust.Cdn.DataAccessLayer; namespace Robust.Cdn.Controllers; [ApiController] -public class StatusController : ControllerBase +public class StatusController(Database db) : ControllerBase { - private readonly Database _db; - - public StatusController(Database db) - { - _db = db; - } - [HttpGet("control/status")] - public IActionResult GetControlStatus() + public IActionResult GetControlStatus(CancellationToken ct) { try { - var con = _db.Connection; - var versionCount = con.QuerySingleOrDefault("SELECT COUNT(Id) FROM ContentVersion"); + var versionCount = db.VersionCount(); var assemblyVersion = Assembly.GetExecutingAssembly().GetName().Version; diff --git a/Robust.Cdn/DataAccessLayer/Database.cs b/Robust.Cdn/DataAccessLayer/Database.cs new file mode 100644 index 0000000..64cc1b0 --- /dev/null +++ b/Robust.Cdn/DataAccessLayer/Database.cs @@ -0,0 +1,290 @@ +using Dapper; +using Microsoft.Extensions.Options; +using Robust.Cdn.Config; +using Robust.Cdn.DataAccessLayer.Models; +using Robust.Cdn.DataAccessLayer.Sqlite.Blob; +using Robust.Cdn.DataAccessLayer.Sqlite.Commands; + +namespace Robust.Cdn.DataAccessLayer; + +/// +/// Database service for CDN functionality. +/// +public sealed class Database(IOptions options) : ScopedSqliteRepositoryBase(options.Value) +{ + // Cached blob handle for efficient BLOB writes during ingest. + // Reused across multiple row writes via Reopen() to avoid open/close overhead. + private SqliteBlobStream? _contentBlob; + + // Cached blob handle for efficient BLOB reads during download. + // Reused across multiple row reads via Reopen(). + private SqliteBlobStream? _readBlob; + + /// Count existing content versions. + public int VersionCount() + { + return Connection.QuerySingleOrDefault("SELECT COUNT(Id) FROM ContentVersion"); + } + + /// + /// Ensures that fork record is created in cdn database. + /// + public int EnsureForkCreated(string forkName) + { + var id = Connection.QuerySingleOrDefault( + "SELECT Id FROM Fork WHERE Name = @Name", + new { Name = forkName }); + + id ??= Connection.QuerySingle( + "INSERT INTO Fork (Name) VALUES (@Name) RETURNING Id", + new { Name = forkName }); + + return id.Value; + } + + /// + /// Get distinct blob count and manifest entries count for specified fork version. + /// + /// Fork name. + /// Human-readable build version number. + /// Null if no records with provided fork-version found, aggregate counts otherwise. + public (long versionId, int countDistinctBlobs, int entriesCount)? GetDistinctBlobsAndManifestEntriesCounts(string forkName, string versionNumber) + { + var (versionId, countDistinctBlobs) = Connection.QuerySingleOrDefault<(long, int)>( + """ + SELECT CV.Id, CV.CountDistinctBlobs + FROM ContentVersion CV + INNER JOIN main.Fork F on F.Id = CV.ForkId + WHERE F.Name = @Fork AND Version = @Version + """, + new + { + Fork = forkName, + Version = versionNumber + }); + + if (versionId == 0) + return null; + + var entriesCount = Connection.ExecuteScalar( + "SELECT COUNT(*) FROM ContentManifestEntry WHERE VersionId = @VersionId", + new { VersionId = versionId } + ); + + return (versionId, countDistinctBlobs, entriesCount); + } + + /// + /// Updates counter for distinct blobs of content version for specified version. + /// + public void RefreshCountDistinctBlobs(long versionId) + { + Connection.Execute( + """ + UPDATE ContentVersion AS cv + SET CountDistinctBlobs = + (SELECT COUNT(DISTINCT cme.ContentId) + FROM ContentManifestEntry cme + WHERE cme.VersionId = cv.Id) + WHERE cv.Id = @VersionId + """, + new { VersionId = versionId } + ); + } + + /// + /// Update content version data. + /// + public void UpdateContentVersionData(long versionId, byte[] manifestHash, int compressedLength) + { + Connection.Execute( + """ + UPDATE ContentVersion + SET ManifestHash = @ManifestHash, ManifestData = zeroblob(@ManifestDataSize) + WHERE Id = @VersionId + """, + new + { + VersionId = versionId, + ManifestHash = manifestHash, + ManifestDataSize = compressedLength + } + ); + } + + /// + /// Insert content version data and get generated id. + /// + /// Id of fork. + /// Human-readable build version number. + /// Id of newly created content version record. + public long InsertContentVersionAndGetId(int forkId, string versionNumber) + { + return Connection.ExecuteScalar( + """ + INSERT INTO ContentVersion (ForkId, Version, TimeAdded, ManifestHash, ManifestData, CountDistinctBlobs) + VALUES (@ForkId, @Version, datetime('now'), zeroblob(0), zeroblob(0), 0) + RETURNING Id + """, + new { Version = versionNumber, ForkId = forkId } + ); + } + + #region Working with blobs in database + + /// + /// Write data into the Content.Data BLOB-typed column for a given content row. + /// + /// + /// Internally caches and reuses the SqliteBlobStream handle across calls via Reopen(). + /// + public void WriteContentBlob(long contentId, ReadOnlySpan data) + { + if (_contentBlob == null) + { + _contentBlob = SqliteBlobStream.Open(Connection.Handle!, "main", "Content", "Data", contentId, true); + } + else + { + _contentBlob.Reopen(contentId); + } + + _contentBlob.Write(data); + } + + /// + /// Write compressed manifest data into the ContentVersion.ManifestData BLOB-typed column. + /// + public void WriteManifestBlob(long versionId, ReadOnlySpan data) + { + using var manifestBlob = SqliteBlobStream.Open( + Connection.Handle!, "main", "ContentVersion", "ManifestData", versionId, true); + + manifestBlob.Write(data); + } + + /// + /// Open a read-only stream for content data on the given row. + /// Internally caches and reuses the SqliteBlobStream handle across calls via Reopen(). + /// + /// The id in Content table. + /// A read-only backed by a . + public Stream OpenContentBlobForRead(long rowId) + { + if (_readBlob == null) + { + _readBlob = SqliteBlobStream.Open( + Connection.Handle!, "main", "Content", "Data", rowId, canWrite: false); + } + else + { + _readBlob.Reopen(rowId); + } + + return _readBlob; + } + + /// + /// Attempts to find Content version for provided version number, then returns stream of its ManifestData column. + /// + /// Fork name. + /// Human-readable build version number. + /// Stream with ManifestData and hash, null if no row with provided version number exists. + public ManifestBlob? FindManifestDataBlob(string forkName, string versionNumber) + { + if (Transaction == null) + { + StartTransaction(); + } + + var (row, manifestHash) = Connection.QuerySingleOrDefault<(long, byte[])>( + """ + SELECT CV.Id, CV.ManifestHash + FROM ContentVersion CV + INNER JOIN main.Fork F on F.Id = CV.ForkId + WHERE F.Name = @Fork AND Version = @Version + """, + new + { + Fork = forkName, + Version = versionNumber + }); + + if (row == 0) + return null; + + var blob = SqliteBlobStream.Open(Connection.Handle!, "main", "ContentVersion", "ManifestData", row, false); + return new(blob, Convert.ToHexString(manifestHash)); + } + + /// + /// Release the cached content blob handle. + /// + public void ReleaseContentBlob() + { + _contentBlob?.Dispose(); + _contentBlob = null; + } + + /// + /// Release the cached read blob handle. + /// + public void ReleaseReadBlob() + { + _readBlob?.Dispose(); + _readBlob = null; + } + + #endregion + + #region sqlite wrapped prepared commands + + public void InsertContentManifestEntry(long versionId, int idx, long contentId) + { + var command = GetPreparedCommand(() => new InsertContentManifestEntryCommand(Connection)); + command.Execute(versionId, idx, contentId); + } + + /// Checks if ContentVersion exists. + /// Human-readable build version number. + public bool IsVersionExisting(string versionNumber) + { + var command = GetPreparedCommand(() => new IsVersionExistingCheckCommand(Connection)); + return command.Execute(versionNumber); + } + + /// Try to find content record by provided hash. + public long? FindContentByHash(byte[] hash) + { + var command = GetPreparedCommand(() => new FindContentByHashCommand(Connection)); + return command.Execute(hash); + } + + public (ContentCompression Compression, int Size, long RowId) ListContentMetadata(long versionId, int index) + { + var command = GetPreparedCommand(() => new ListContentMetadataCommand(Connection)); + return command.Execute(versionId, index); + } + + public long InsertContent(byte[] hash, int dataLength, ContentCompression compression, int writeDataLength) + { + var command = GetPreparedCommand(()=> new InsertContentSqlitePreparedCommand(Connection)); + return command.Execute(hash, dataLength, compression, writeDataLength); + } + + #endregion + + public override void Dispose() + { + ReleaseContentBlob(); + ReleaseReadBlob(); + base.Dispose(); + } + + private T GetPreparedCommand(Func factory) where T : SqlitePreparedCommandWrapperBase + { + var command = PreparedCommands.GetOrAdd(typeof(T), + _ => factory()); + return (T)command; + } +} diff --git a/Robust.Cdn/DataAccessLayer/ManifestDatabase.cs b/Robust.Cdn/DataAccessLayer/ManifestDatabase.cs new file mode 100644 index 0000000..9f3efe5 --- /dev/null +++ b/Robust.Cdn/DataAccessLayer/ManifestDatabase.cs @@ -0,0 +1,443 @@ +using Dapper; +using Microsoft.Extensions.Options; +using Robust.Cdn.Config; +using Robust.Cdn.DataAccessLayer.Models; +using Robust.Cdn.DataAccessLayer.Sqlite.Blob; +using Robust.Cdn.Helpers; +using System.Security.Cryptography; +using System.Text.Json; + +namespace Robust.Cdn.DataAccessLayer; + +/// +/// Database service for server manifest functionality. +/// +public sealed class ManifestDatabase(IOptions options) : ScopedSqliteRepositoryBase(options.Value) +{ + private static readonly JsonSerializerOptions ManifestCacheContext = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase + }; + + /// Gets fork id by fork name. + public int GetForkIdByName(string forkName) + { + return Connection.QuerySingle("SELECT Id FROM Fork WHERE Name = @Name", new { Name = forkName }); + } + + /// + /// Ensure all provided forks exists in manifest database. + /// + public void EnsureForksCreated(IEnumerable forkNames) + { + Connection.Execute( + "INSERT INTO Fork (Name) VALUES (@Name) ON CONFLICT DO NOTHING", + forkNames.Select(x => new { Name = x }), + Transaction + ); + } + + /// + /// Checks if there is a record of publish for build versin of fork already in progress. + /// + /// id of a fork. + /// Human-readable fork build version. + public bool IsPublishInProgress(int forkId, string requestVersion) + { + return Connection.QuerySingleOrDefault( + "SELECT 1 FROM PublishInProgress WHERE Version = @Version AND ForkId = @ForkId", + new { requestVersion, ForkId = forkId } + ); + } + + /// + /// Lists fork versions that were published before the specified date. + /// + public IReadOnlyList QueryVersionOlderThen(string forkName, DateTime olderThen) + { + return Connection.Query( + """ + SELECT FV.Id, FV.Name + FROM ForkVersion FV, Fork + WHERE FV.ForkId = Fork.Id + AND Fork.Name = @ForkName + AND FV.PublishedTime < @PruneFrom + """, + new { ForkName = forkName, PruneFrom = olderThen }, + Transaction + ).AsList(); + } + + /// + /// Deletes fork versions by id. + /// + public int DeleteVersion(int id) + { + return Connection.Execute("DELETE FROM ForkVersion WHERE Id = @Id", id, Transaction); + } + + /// + /// Sets all mentioned versions of a fork as 'available'. + /// + /// Fork name. + /// Human-readable fork build version number. + public void SetForkVersionsAvailable(string forkName, IEnumerable versions) + { + var versionsArray = versions.ToArray(); + if(versionsArray.Length == 0) + return; + + // sqlite does not, by default, support arrays inlining of more than 999 items. + if (versionsArray.Length > 999) + throw new ArgumentException("Cannot handle enumeration of more then 999 version.", nameof(versions)); + + Connection.Execute( + """ + UPDATE ForkVersion + SET Available = TRUE + WHERE Name IN @Versions + AND ForkId = (SELECT Id FROM Fork WHERE Name = @ForkName) + """, + new + { + Versions = versionsArray, + ForkName = forkName + }, + Transaction + ); + } + + /// + /// Gets list of fork versions by fork name. + /// + public List ListForkVersions(string forkName, int limit) + { + var dbVersions = Connection.Query<(int Id, string Name, DateTime PublishedTime, string EngineVersion)>( + $""" + SELECT FV.Id, FV.Name, PublishedTime, EngineVersion + FROM ForkVersion FV + INNER JOIN main.Fork F ON FV.ForkId = F.Id + WHERE F.Name = @Fork + AND FV.Available + ORDER BY PublishedTime DESC + LIMIT {limit} + """, + new { Fork = forkName }, + Transaction + ); + + var versions = new List(); + foreach (var dbVersion in dbVersions) + { + var servers = Connection.Query( + """ + SELECT Platform, FileName, FileSize + FROM ForkVersionServerBuild + WHERE ForkVersionId = @ForkVersionId + ORDER BY Platform + """, + new { ForkVersionId = dbVersion.Id }, + Transaction + ); + + var versionInfo = new BuildVersionArtifactInfo + { + Name = dbVersion.Name, + EngineVersion = dbVersion.EngineVersion, + PublishedTime = DateTime.SpecifyKind(dbVersion.PublishedTime, DateTimeKind.Utc), + Servers = servers.ToArray() + }; + versions.Add(versionInfo); + } + return versions; + } + + /// + /// Checks if fork version exists. + /// + /// Fork name. + /// Human-readable fork build version. + public bool IsVersionExists(string forkName, string versionNumber) + { + return Connection.QuerySingleOrDefault( + """ + SELECT 1 + FROM ForkVersion, Fork + WHERE ForkVersion.Name = @Version + AND Fork.Name = @Fork + AND Fork.Id = ForkVersion.ForkId + """, + new { Fork = forkName, Version = versionNumber }, + Transaction + ); + } + + /// + /// Insert build version info for provided files. + /// + /// Artifact info for client side of artifacts. + /// New version build files. + /// Fork name for which versions are added. + /// Version metadata. + public void AddVersionsToDatabase( + ArtifactBriefInfo clientArtifactBriefInfo, + Dictionary diskFiles, + string forkName, + VersionMetadata metadata + ) + { + var dbCon = Connection; + + var forkId = GetForkIdByName(forkName); + + var (clientName, clientSha256, _) = GetFileNameSha256Pair(diskFiles[clientArtifactBriefInfo]); + + var versionId = dbCon.QuerySingle( + """ + INSERT INTO ForkVersion (Name, ForkId, PublishedTime, ClientFileName, ClientSha256, EngineVersion) + VALUES (@Name, @ForkId, @PublishTime, @ClientName, @ClientSha256, @EngineVersion) + RETURNING Id + """, + new + { + Name = metadata.Version, + ForkId = forkId, + ClientName = clientName, + ClientSha256 = clientSha256, + metadata.EngineVersion, + PublishTime = DateTime.UtcNow + }, + Transaction + ); + + foreach (var (artifact, diskPath) in diskFiles) + { + if (artifact.Type != ArtifactType.Server) + continue; + + var (serverName, serverSha256, fileSize) = GetFileNameSha256Pair(diskPath); + + dbCon.Execute( + """ + INSERT INTO ForkVersionServerBuild (ForkVersionId, Platform, FileName, Sha256, FileSize) + VALUES (@ForkVersion, @Platform, @ServerName, @ServerSha256, @FileSize) + """, + new + { + ForkVersion = versionId, + artifact.Platform, + ServerName = serverName, + ServerSha256 = serverSha256, + FileSize = fileSize + }, + Transaction + ); + } + } + + /// + /// List all publishes in progress. + /// + public IReadOnlyList ListPublishInProgress() + { + return Connection.Query( + """ + SELECT PublishInProgress.Id, Version, Fork.Name, StartTime + FROM PublishInProgress + INNER JOIN Fork ON Fork.Id = PublishInProgress.ForkId + """, + transaction: Transaction + ).AsList(); + } + + /// + /// Delete build version by fork name and human-readable build version number. + /// + public void DeleteVersionByVersionName(string forkName, string versionNumber) + { + Connection.Execute( + """ + DELETE FROM PublishInProgress + WHERE Version = @Version + AND ForkId IN ( + SELECT Id FROM Fork WHERE Name = @Fork + ) + """, + new { Version = versionNumber, Fork = forkName }, + Transaction + ); + } + + /// + /// Inserts publish in progress record with all required version metadata. + /// + public int InsertPublishInProgress( + string versionNumber, + string engineVersion, + int forkId, + SourceVersionInfo build, + SourceVersionInfo engine + ) + { + + return Connection.Execute( + """ + INSERT INTO PublishInProgress ( + Version, + ForkId, + StartTime, + EngineVersion, + SourceUrl, + SourceCommitId, + SourceBranchName, + EngineSourceUrl, + EngineSourceCommitId, + EngineSourceBranchName + ) + VALUES ( + @Version, + @ForkId, + @StartTime, + @EngineVersion, + @SourceUrl, + @SourceCommitId, + @SourceBranchName, + @EngineSourceUrl, + @EngineSourceCommitId, + @EngineSourceBranchName + ) + """, + new + { + Version = versionNumber, + EngineVersion = engineVersion, + ForkId = forkId, + StartTime = DateTime.UtcNow, + SourceUrl = build.SourceUrl, + SourceCommitId = build.CommitId, + SourceBranchName = build.BranchName, + EngineSourceUrl = engine.SourceUrl, + EngineSourceCommitId = engine.CommitId, + EngineSourceBranchName = engine.BranchName, + }, + Transaction + ); + } + + /// + /// Gets fork version full metadata. + /// + /// Id of a fork. + /// Human-readable fork version number. + /// + public VersionMetadata? GetVersionMetadata(int forkId, string requestVersion) + { + return Connection.QuerySingleOrDefault( + """ + SELECT Version, EngineVersion, SourceUrl, SourceCommitId, SourceBranchName, EngineSourceUrl, EngineSourceCommitId, EngineSourceBranchName + FROM PublishInProgress + WHERE Version = @Name AND ForkId = @Fork + """, + new { Name = requestVersion, Fork = forkId } + ); + } + + /// + /// Updates server manifest cache for a fork. + /// + /// Fork name. + /// Fork id. + /// Helper for getting urls according to app current base url. + /// Number of affected rows. + public int UpdateManifestCache(string forkName, int forkId, BaseUrlManager baseUrlManager) + { + + var builds = new Dictionary(); + + var versions = Connection + .Query<(int id, string name, DateTime publishedTime, string clientFileName, byte[] clientSha256)>( + """ + SELECT Id, Name, PublishedTime, ClientFileName, ClientSha256 + FROM ForkVersion + WHERE Available AND ForkId = @ForkId + """, + new { ForkId = forkId } + ); + + foreach (var version in versions) + { + var buildData = new ManifestBuildData + { + PublishTime = DateTime.SpecifyKind(version.publishedTime, DateTimeKind.Utc), + Client = new ManifestArtifact + { + Url = baseUrlManager.MakeBuildInfoUrl($"fork/{forkName}/version/{version.name}/file/{version.clientFileName}"), + Sha256 = Convert.ToHexString(version.clientSha256) + }, + Server = new Dictionary() + }; + + var servers = Connection.Query<(string platform, string fileName, byte[] sha256, long? size)>( + """ + SELECT Platform, FileName, Sha256, FileSize + FROM ForkVersionServerBuild + WHERE ForkVersionId = @ForkVersionId + """, + new { ForkVersionId = version.id } + ); + + foreach (var (platform, fileName, sha256, size) in servers) + { + var manifestArtifact = new ManifestArtifact + { + Url = baseUrlManager.MakeBuildInfoUrl($"fork/{forkName}/version/{version.name}/file/{fileName}"), + Sha256 = Convert.ToHexString(sha256), + Size = size + }; + buildData.Server.Add(platform, manifestArtifact); + } + + builds.Add(version.name, buildData); + } + + var data = new ManifestData { Builds = builds }; + var bytes = JsonSerializer.SerializeToUtf8Bytes(data, ManifestCacheContext); + return Connection.Execute( + "UPDATE Fork SET ServerManifestCache = @Data WHERE Id = @ForkId", + new + { + Data = bytes, + ForkId = forkId + } + ); + } + + /// + /// Try to find fork by name, and return its cached server manifest data as stream, if it exists. + /// Returns null if fails to find fork. + /// + public Stream? FindManifestCache(string forkName) + { + var rowId = Connection.QuerySingleOrDefault( + "SELECT ROWID FROM Fork WHERE Name == @Fork AND ServerManifestCache IS NOT NULL", + new { Fork = forkName } + ); + + if (rowId == 0) + return null; + + return SqliteBlobStream.Open(Connection.Handle!, "main", "Fork", "ServerManifestCache", rowId, false); + } + + private static (string name, byte[] hash, long size) GetFileNameSha256Pair(string diskPath) + { + using var file = File.OpenRead(diskPath); + + return (Path.GetFileName(diskPath), SHA256.HashData(file), file.Length); + } + + private sealed class ManifestData + { + public required IReadOnlyDictionary Builds { get; set; } + } +} diff --git a/Robust.Cdn/DataAccessLayer/Models/ArtifactBriefInfo.cs b/Robust.Cdn/DataAccessLayer/Models/ArtifactBriefInfo.cs new file mode 100644 index 0000000..1aaee32 --- /dev/null +++ b/Robust.Cdn/DataAccessLayer/Models/ArtifactBriefInfo.cs @@ -0,0 +1,14 @@ +namespace Robust.Cdn.DataAccessLayer.Models; + +public sealed class ArtifactBriefInfo +{ + /// + /// Marker, if artifact is client or server part. + /// + public ArtifactType Type { get; set; } + + /// + /// Target platform for the artifact. + /// + public string? Platform { get; set; } +} diff --git a/Robust.Cdn/DataAccessLayer/Models/ArtifactType.cs b/Robust.Cdn/DataAccessLayer/Models/ArtifactType.cs new file mode 100644 index 0000000..813caef --- /dev/null +++ b/Robust.Cdn/DataAccessLayer/Models/ArtifactType.cs @@ -0,0 +1,10 @@ +namespace Robust.Cdn.DataAccessLayer.Models; + +/// +/// Marker for which part of application (server or client) artifact is. +/// +public enum ArtifactType +{ + Server, + Client +} diff --git a/Robust.Cdn/DataAccessLayer/Models/BuildVersionArtifactInfo.cs b/Robust.Cdn/DataAccessLayer/Models/BuildVersionArtifactInfo.cs new file mode 100644 index 0000000..56c5558 --- /dev/null +++ b/Robust.Cdn/DataAccessLayer/Models/BuildVersionArtifactInfo.cs @@ -0,0 +1,27 @@ +namespace Robust.Cdn.DataAccessLayer.Models; + +/// +/// Build version info for artifacts. +/// +public sealed class BuildVersionArtifactInfo +{ + /// + /// Human-readable build version number. + /// + public required string Name { get; init; } + + /// + /// Publish time of this build version. + /// + public required DateTime PublishedTime { get; init; } + + /// + /// Engine version, used for this build version. + /// + public required string? EngineVersion { get; init; } + + /// + /// Array of server artifact info for this build version. + /// + public required BuildVersionServerArtifactInfo[] Servers { get; init; } +} diff --git a/Robust.Cdn/DataAccessLayer/Models/BuildVersionServerArtifactInfo.cs b/Robust.Cdn/DataAccessLayer/Models/BuildVersionServerArtifactInfo.cs new file mode 100644 index 0000000..78a4b18 --- /dev/null +++ b/Robust.Cdn/DataAccessLayer/Models/BuildVersionServerArtifactInfo.cs @@ -0,0 +1,22 @@ +namespace Robust.Cdn.DataAccessLayer.Models; + +/// +/// Server information for a specific build version artifact. +/// +public sealed class BuildVersionServerArtifactInfo +{ + /// + /// Target platform. + /// + public required string Platform { get; init; } + + /// + /// File name of artifact. + /// + public required string FileName { get; init; } + + /// + /// File size in bytes. + /// + public required long? FileSize { get; init; } +} diff --git a/Robust.Cdn/DataAccessLayer/Models/ManifestArtifact.cs b/Robust.Cdn/DataAccessLayer/Models/ManifestArtifact.cs new file mode 100644 index 0000000..ace1000 --- /dev/null +++ b/Robust.Cdn/DataAccessLayer/Models/ManifestArtifact.cs @@ -0,0 +1,25 @@ +using System.Text.Json.Serialization; + +namespace Robust.Cdn.DataAccessLayer.Models; + +/// +/// Container for build artifact manifest data. +/// +public sealed class ManifestArtifact +{ + /// + /// Url for artifact download. + /// + public required string Url { get; set; } + + /// + /// Hash of artifact file. + /// + public required string Sha256 { get; set; } + + /// + /// Artifact file size in bytes. + /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? Size { get; set; } +} diff --git a/Robust.Cdn/DataAccessLayer/Models/ManifestBlob.cs b/Robust.Cdn/DataAccessLayer/Models/ManifestBlob.cs new file mode 100644 index 0000000..f1a40bc --- /dev/null +++ b/Robust.Cdn/DataAccessLayer/Models/ManifestBlob.cs @@ -0,0 +1,22 @@ +namespace Robust.Cdn.DataAccessLayer.Models; + +/// +/// Container for manifest stream info. +/// +public class ManifestBlob(Stream blob, string manifestHash) : IDisposable +{ + /// + /// Blob of manifest data. + /// + public Stream Blob { get; } = blob; + + /// + /// Hash of version manifest. + /// + public string ManifestHash { get; } = manifestHash; + + public void Dispose() + { + Blob.Dispose(); + } +} diff --git a/Robust.Cdn/DataAccessLayer/Models/ManifestBuildData.cs b/Robust.Cdn/DataAccessLayer/Models/ManifestBuildData.cs new file mode 100644 index 0000000..e5ad381 --- /dev/null +++ b/Robust.Cdn/DataAccessLayer/Models/ManifestBuildData.cs @@ -0,0 +1,26 @@ +using System.Text.Json.Serialization; + +namespace Robust.Cdn.DataAccessLayer.Models; + +/// +/// Container for build manifest data. +/// +public sealed class ManifestBuildData +{ + /// + /// Version publish time in UTC. + /// + [JsonPropertyName("Time")] + public DateTime PublishTime { get; set; } + + /// + /// Artifact manifest data for client side of build. + /// + public required ManifestArtifact Client { get; set; } + + /// + /// Collection of manifest data for server side of build. + /// Key is the server name, value is the manifest artifact for that server. + /// + public required Dictionary Server { get; set; } +} diff --git a/Robust.Cdn/DataAccessLayer/Models/PublishInProgressBriefInfo.cs b/Robust.Cdn/DataAccessLayer/Models/PublishInProgressBriefInfo.cs new file mode 100644 index 0000000..e5b824e --- /dev/null +++ b/Robust.Cdn/DataAccessLayer/Models/PublishInProgressBriefInfo.cs @@ -0,0 +1,10 @@ +namespace Robust.Cdn.DataAccessLayer.Models; + +/// +/// Container for publish-in-progress info. +/// +/// Publish process id. +/// Human-readable fork build version number. +/// Fork name. +/// Publish start time (utc-based). +public sealed record PublishInProgressBriefInfo(int Id, string Version, string ForkName, DateTime StartTime); diff --git a/Robust.Cdn/DataAccessLayer/Models/SourceVersionInfo.cs b/Robust.Cdn/DataAccessLayer/Models/SourceVersionInfo.cs new file mode 100644 index 0000000..b6288ab --- /dev/null +++ b/Robust.Cdn/DataAccessLayer/Models/SourceVersionInfo.cs @@ -0,0 +1,9 @@ +namespace Robust.Cdn.DataAccessLayer.Models; + +/// +/// Detailed info on sources used for building version. +/// +/// URL for repository that holds sources. +/// Commit ID used for building sources. +/// Branch name or tag, used for building sources. +public record SourceVersionInfo(string? SourceUrl, string? CommitId, string? BranchName); \ No newline at end of file diff --git a/Robust.Cdn/DataAccessLayer/Models/VersionBriefData.cs b/Robust.Cdn/DataAccessLayer/Models/VersionBriefData.cs new file mode 100644 index 0000000..e8a4d75 --- /dev/null +++ b/Robust.Cdn/DataAccessLayer/Models/VersionBriefData.cs @@ -0,0 +1,17 @@ +namespace Robust.Cdn.DataAccessLayer.Models; + +/// +/// Short data for build version. +/// +public sealed class VersionBriefData +{ + /// + /// Build version id. + /// + public required int Id { get; set; } + + /// + /// Human-readable build version number. + /// + public required string Name { get; set; } +} diff --git a/Robust.Cdn/DataAccessLayer/Models/VersionMetadata.cs b/Robust.Cdn/DataAccessLayer/Models/VersionMetadata.cs new file mode 100644 index 0000000..22ebcfe --- /dev/null +++ b/Robust.Cdn/DataAccessLayer/Models/VersionMetadata.cs @@ -0,0 +1,52 @@ +namespace Robust.Cdn.DataAccessLayer.Models; + +/// +/// Build version metadata. +/// +public sealed class VersionMetadata +{ + public VersionMetadata(string version, string engineVersion, SourceVersionInfo buildVersionInfo, SourceVersionInfo engineSourceVersionInfo) + { + Version = version; + EngineVersion = engineVersion; + BuildVersionInfo = buildVersionInfo; + EngineSourceVersionInfo = engineSourceVersionInfo; + } + + public VersionMetadata( + string version, + string engineVersion, + string? sourceUrl, + string? sourceCommitId, + string? sourceBranchName, + string? engineSourceUrl, + string? engineSourceCommitId, + string? engineSourceBranchName + ) + { + Version = version; + EngineVersion = engineVersion; + BuildVersionInfo = new SourceVersionInfo(sourceUrl, sourceCommitId, sourceBranchName); + EngineSourceVersionInfo = new SourceVersionInfo(engineSourceUrl, engineSourceCommitId, engineSourceBranchName); + } + + /// + /// Human-readable version of the build. This is used to identify the build in the CDN and in the game client. + /// + public string Version { get; } + + /// + /// Human-readable version of the engine used to build this version. + /// + public string EngineVersion { get; } + + /// + /// Version info for sources, used for build. + /// + public SourceVersionInfo BuildVersionInfo { get; } + + /// + /// Version info for sources of engine, used for build. + /// + public SourceVersionInfo EngineSourceVersionInfo { get; } +} \ No newline at end of file diff --git a/Robust.Cdn/DataAccessLayer/ScopedSqliteRepositoryBase.cs b/Robust.Cdn/DataAccessLayer/ScopedSqliteRepositoryBase.cs new file mode 100644 index 0000000..f861123 --- /dev/null +++ b/Robust.Cdn/DataAccessLayer/ScopedSqliteRepositoryBase.cs @@ -0,0 +1,68 @@ +using System.Collections.Concurrent; +using Dapper; +using Microsoft.Data.Sqlite; +using Robust.Cdn.Config; +using System.Data.Common; +using System.Diagnostics.CodeAnalysis; +using Robust.Cdn.DataAccessLayer.Sqlite.Commands; + +namespace Robust.Cdn.DataAccessLayer; + +/// +/// Base type for sqlite repository, scoped to a single request. +/// +public abstract class ScopedSqliteRepositoryBase(IDatabaseOptions options) : IDisposable +{ + private SqliteConnection? _connection; + protected DbTransaction? Transaction; + + // cache for pre-compiled sqlite commands, to avoid re-preparing them on every request. + protected readonly ConcurrentDictionary PreparedCommands = new(); + + private readonly string _fileName = options.DatabaseFileName; + + protected SqliteConnection Connection => _connection ??= OpenConnection(); + + private SqliteConnection OpenConnection() + { + var con = new SqliteConnection(GetConnectionString()); + con.Open(); + con.Execute("PRAGMA journal_mode=WAL"); + return con; + } + + // deferred transaction is sqlite-specific, so we should change it to proper async when moving to Postgres + [MemberNotNull(nameof(Transaction))] + public void StartTransaction(bool deferred = false) + { + Transaction = Connection.BeginTransaction(deferred); + } + + // sqlite does not support child transactions, so no point in tracking multiple transactions. + public void Commit() + { + if (Transaction == null) + { + return; + } + + Transaction.Commit(); + Transaction = null; + } + +#pragma warning disable CA1816 + public virtual void Dispose() + { + Transaction?.Rollback(); + foreach (var wrapper in PreparedCommands) + { + wrapper.Value.Dispose(); + } + + _connection?.Dispose(); + } +#pragma warning restore CA1816 + + private string GetConnectionString() + => $"Data Source={_fileName};Mode=ReadWriteCreate;Pooling=True;Foreign Keys=True"; +} diff --git a/Robust.Cdn/Helpers/SqliteBlobStream.cs b/Robust.Cdn/DataAccessLayer/Sqlite/Blob/SqliteBlobStream.cs similarity index 98% rename from Robust.Cdn/Helpers/SqliteBlobStream.cs rename to Robust.Cdn/DataAccessLayer/Sqlite/Blob/SqliteBlobStream.cs index fc2f8d0..0726fb0 100644 --- a/Robust.Cdn/Helpers/SqliteBlobStream.cs +++ b/Robust.Cdn/DataAccessLayer/Sqlite/Blob/SqliteBlobStream.cs @@ -1,8 +1,8 @@ -using Microsoft.Data.Sqlite; +using Microsoft.Data.Sqlite; using SQLitePCL; using static SQLitePCL.raw; -namespace Robust.Cdn.Helpers; +namespace Robust.Cdn.DataAccessLayer.Sqlite.Blob; /// /// Expecting Microsoft top engineers to understand basic API design principles is too much to ask for, diff --git a/Robust.Cdn/DataAccessLayer/Sqlite/Commands/FindContentByHashCommand.cs b/Robust.Cdn/DataAccessLayer/Sqlite/Commands/FindContentByHashCommand.cs new file mode 100644 index 0000000..8d145f6 --- /dev/null +++ b/Robust.Cdn/DataAccessLayer/Sqlite/Commands/FindContentByHashCommand.cs @@ -0,0 +1,18 @@ +using Microsoft.Data.Sqlite; +using SQLitePCL; + +namespace Robust.Cdn.DataAccessLayer.Sqlite.Commands; + +public class FindContentByHashCommand(SqliteConnection connection) + : SqlitePreparedCommandWrapperBase(connection, "SELECT Id FROM Content WHERE Hash = ?") +{ + public long? Execute(byte[] hash) + { + Reset(); + BindBlob(1, hash); + if (Step() == raw.SQLITE_DONE) + return null; + + return ColumnInt64(0); + } +} diff --git a/Robust.Cdn/DataAccessLayer/Sqlite/Commands/InsertContentManifestEntryCommand.cs b/Robust.Cdn/DataAccessLayer/Sqlite/Commands/InsertContentManifestEntryCommand.cs new file mode 100644 index 0000000..7afbe69 --- /dev/null +++ b/Robust.Cdn/DataAccessLayer/Sqlite/Commands/InsertContentManifestEntryCommand.cs @@ -0,0 +1,22 @@ +using Microsoft.Data.Sqlite; + +namespace Robust.Cdn.DataAccessLayer.Sqlite.Commands; + +public class InsertContentManifestEntryCommand(SqliteConnection connection) : SqlitePreparedCommandWrapperBase( + connection, + """ + INSERT INTO ContentManifestEntry (VersionId, ManifestIdx, ContentId) + VALUES (@VersionId, @ManifestIdx, @ContentId) + """ +) +{ + public void Execute(long versionId, int idx, long contentId) + { + BindInt64(1, versionId); + BindInt64(2, idx); // @ManifestIdx + BindInt64(3, contentId); // @ContentId + + Step(); + Reset(); + } +} diff --git a/Robust.Cdn/DataAccessLayer/Sqlite/Commands/InsertContentSqlitePreparedCommand.cs b/Robust.Cdn/DataAccessLayer/Sqlite/Commands/InsertContentSqlitePreparedCommand.cs new file mode 100644 index 0000000..0436532 --- /dev/null +++ b/Robust.Cdn/DataAccessLayer/Sqlite/Commands/InsertContentSqlitePreparedCommand.cs @@ -0,0 +1,28 @@ +using Microsoft.Data.Sqlite; + +namespace Robust.Cdn.DataAccessLayer.Sqlite.Commands; + +public class InsertContentSqlitePreparedCommand(SqliteConnection connection) + : SqlitePreparedCommandWrapperBase( + connection, + """ + INSERT INTO Content (Hash, Size, Compression, Data) + VALUES (@Hash, @Size, @Compression, @Data) + RETURNING Id + """ + ) +{ + public long Execute(byte[] hash, int dataLength, ContentCompression compression, int contentLength) + { + BindBlob(1, hash); // @Hash + BindInt(2, dataLength); // @Size + BindInt(3, (int)compression); // @Compression + BindZeroBlob(4, contentLength); // @Data + + Step(); + var contentId = ColumnInt64(0); + + Reset(); + return contentId; + } +} diff --git a/Robust.Cdn/DataAccessLayer/Sqlite/Commands/IsVersionExistingCheckCommand.cs b/Robust.Cdn/DataAccessLayer/Sqlite/Commands/IsVersionExistingCheckCommand.cs new file mode 100644 index 0000000..365f98b --- /dev/null +++ b/Robust.Cdn/DataAccessLayer/Sqlite/Commands/IsVersionExistingCheckCommand.cs @@ -0,0 +1,19 @@ +using Microsoft.Data.Sqlite; +using SQLitePCL; + +namespace Robust.Cdn.DataAccessLayer.Sqlite.Commands; + +public class IsVersionExistingCheckCommand(SqliteConnection connection) + : SqlitePreparedCommandWrapperBase(connection, "SELECT 1 FROM ContentVersion WHERE Version = ?") +{ + public bool Execute(string version) + { + Reset(); + BindString(1, version); + + if (Step() == raw.SQLITE_ROW) + return true; + + return false; + } +} diff --git a/Robust.Cdn/DataAccessLayer/Sqlite/Commands/ListContentMetadataCommand.cs b/Robust.Cdn/DataAccessLayer/Sqlite/Commands/ListContentMetadataCommand.cs new file mode 100644 index 0000000..feea470 --- /dev/null +++ b/Robust.Cdn/DataAccessLayer/Sqlite/Commands/ListContentMetadataCommand.cs @@ -0,0 +1,32 @@ +using Microsoft.Data.Sqlite; +using SQLitePCL; + +namespace Robust.Cdn.DataAccessLayer.Sqlite.Commands; + +public class ListContentMetadataCommand(SqliteConnection connection) + : SqlitePreparedCommandWrapperBase( + connection, + """ + SELECT c.Compression, c.Size, c.Id + FROM ContentManifestEntry cme + INNER JOIN Content c on c.Id = cme.ContentId + WHERE cme.VersionId = @VersionId AND cme.ManifestIdx = @ManifestIdx + """ + ) +{ + public (ContentCompression Compression, int Size, long RowId) Execute(long versionId, int index) + { + Reset(); + BindInt64(1, versionId);// @VersionId + BindInt(2, index); + if (Step() != raw.SQLITE_ROW) + { + throw new InvalidOperationException("Unable to find manifest row??"); + } + var compression = (ContentCompression)ColumnInt(0); + var size = ColumnInt(1); + var rowId = ColumnInt64(2); + + return (compression, size, rowId); + } +} diff --git a/Robust.Cdn/DataAccessLayer/Sqlite/Commands/SqlitePreparedCommandWrapperBase.cs b/Robust.Cdn/DataAccessLayer/Sqlite/Commands/SqlitePreparedCommandWrapperBase.cs new file mode 100644 index 0000000..3092040 --- /dev/null +++ b/Robust.Cdn/DataAccessLayer/Sqlite/Commands/SqlitePreparedCommandWrapperBase.cs @@ -0,0 +1,117 @@ +using Microsoft.Data.Sqlite; +using SQLitePCL; +using static SQLitePCL.raw; + +namespace Robust.Cdn.DataAccessLayer.Sqlite.Commands; + +/// +/// Base type for prepared command wrappers that handle the lifecycle of a prepared sqlite +/// statement and provide helper methods for binding parameters and retrieving column values. +/// Works as optimization. +/// +public abstract class SqlitePreparedCommandWrapperBase : IDisposable +{ + protected readonly sqlite3_stmt Prepared; + + protected SqlitePreparedCommandWrapperBase(SqliteConnection connection, string commandText) + { + if (connection.Handle == null) + throw new ArgumentException("Expected SqliteConnection to have properly initialized Handle but it is null"); + + Prepared = Prepare(connection.Handle,commandText); + } + + public void Dispose() + { + Prepared.Dispose(); + } + + /// + /// Create sqlite prepared statement from command text. If same command was already processed - it will be reused. + /// + private static sqlite3_stmt Prepare(sqlite3 con, string command) + { + CheckErr(sqlite3_prepare_v2(con, command, out var stmt), con); + + return stmt; + } + + /// + /// Bind argument on index to provided value. + /// + protected void BindString(int index, ReadOnlySpan data) + { + CheckErr(sqlite3_bind_text16(Prepared, index, data)); + } + + /// + /// Bind argument on index to provided value. + /// + protected void BindBlob(int index, ReadOnlySpan data) + { + CheckErr(sqlite3_bind_blob(Prepared, index, data)); + } + + /// + /// Bind argument on index to provided value. + /// + protected void BindInt(int index, int value) + { + CheckErr(sqlite3_bind_int(Prepared, index, value)); + } + + /// + /// Bind argument on index to provided value. + /// + protected void BindInt64(int index, long value) + { + CheckErr(sqlite3_bind_int64(Prepared, index, value)); + } + + /// + /// Bind argument on index to provided value. + /// + protected void BindZeroBlob(int index, int length) + { + CheckErr(sqlite3_bind_zeroblob(Prepared, index, length)); + } + + /// + /// Extracts value from column number response of command execution as long. + /// + protected long ColumnInt64(int index) + { + return sqlite3_column_int64(Prepared, index); + } + + /// + /// Extracts value from column number response of command execution as int. + /// + protected int ColumnInt(int index) + { + return sqlite3_column_int(Prepared, index); + } + + /// + /// Attempts to execute one step of command (attempt to extract row). + /// + /// + protected int Step() + { + return CheckErr(sqlite3_step(Prepared)); + } + + /// + /// Resets command to be reused without reconstruction. Drops results. + /// + protected void Reset() + { + CheckErr(sqlite3_reset(Prepared)); + } + + protected static int CheckErr(int err, sqlite3? db = null) + { + SqliteException.ThrowExceptionForRC(err, db); + return err; + } +} diff --git a/Robust.Cdn/Database.cs b/Robust.Cdn/Database.cs deleted file mode 100644 index f7fcba2..0000000 --- a/Robust.Cdn/Database.cs +++ /dev/null @@ -1,69 +0,0 @@ -using Dapper; -using Microsoft.Data.Sqlite; -using Microsoft.Extensions.Options; -using Robust.Cdn.Config; - -namespace Robust.Cdn; - -public abstract class BaseScopedDatabase : IDisposable -{ - private SqliteConnection? _connection; - public SqliteConnection Connection => _connection ??= OpenConnection(); - - private SqliteConnection OpenConnection() - { - var con = new SqliteConnection(GetConnectionString()); - con.Open(); - con.Execute("PRAGMA journal_mode=WAL"); - return con; - } - -#pragma warning disable CA1816 - public void Dispose() - { - _connection?.Dispose(); - } -#pragma warning restore CA1816 - - protected abstract string GetConnectionString(); - - protected string GetConnectionStringForFile(string fileName) - { - return $"Data Source={fileName};Mode=ReadWriteCreate;Pooling=True;Foreign Keys=True"; - } -} - -/// -/// Database service for CDN functionality. -/// -public sealed class Database(IOptions options) : BaseScopedDatabase -{ - protected override string GetConnectionString() - { - return GetConnectionStringForFile(options.Value.DatabaseFileName); - } -} - -/// -/// Database service for server manifest functionality. -/// -public sealed class ManifestDatabase(IOptions options) : BaseScopedDatabase -{ - protected override string GetConnectionString() - { - return GetConnectionStringForFile(options.Value.DatabaseFileName); - } - - public void EnsureForksCreated() - { - var con = Connection; - using var tx = con.BeginTransaction(); - - foreach (var forkName in options.Value.Forks.Keys) - { - con.Execute("INSERT INTO Fork (Name) VALUES (@Name) ON CONFLICT DO NOTHING", new { Name = forkName }); - } - - tx.Commit(); - } -} diff --git a/Robust.Cdn/Helpers/SqliteExt.cs b/Robust.Cdn/Helpers/SqliteExt.cs deleted file mode 100644 index da6d6ca..0000000 --- a/Robust.Cdn/Helpers/SqliteExt.cs +++ /dev/null @@ -1,66 +0,0 @@ -using Microsoft.Data.Sqlite; -using SQLitePCL; -using static SQLitePCL.raw; - -namespace Robust.Cdn.Helpers; - -public static class SqliteExt -{ - public static sqlite3_stmt Prepare(this sqlite3 con, string command) - { - CheckErr(sqlite3_prepare_v2(con, command, out var stmt), con); - - return stmt; - } - - public static void BindString(this sqlite3_stmt stmt, int index, ReadOnlySpan data) - { - CheckErr(sqlite3_bind_text16(stmt, index, data)); - } - - public static void BindBlob(this sqlite3_stmt stmt, int index, ReadOnlySpan data) - { - CheckErr(sqlite3_bind_blob(stmt, index, data)); - } - - public static void BindInt(this sqlite3_stmt stmt, int index, int value) - { - CheckErr(sqlite3_bind_int(stmt, index, value)); - } - - public static void BindInt64(this sqlite3_stmt stmt, int index, long value) - { - CheckErr(sqlite3_bind_int64(stmt, index, value)); - } - - public static void BindZeroBlob(this sqlite3_stmt stmt, int index, int length) - { - CheckErr(sqlite3_bind_zeroblob(stmt, index, length)); - } - - public static long ColumnInt64(this sqlite3_stmt stmt, int index) - { - return sqlite3_column_int64(stmt, index); - } - - public static int ColumnInt(this sqlite3_stmt stmt, int index) - { - return sqlite3_column_int(stmt, index); - } - - public static int Step(this sqlite3_stmt stmt) - { - return CheckErr(sqlite3_step(stmt)); - } - - public static void Reset(this sqlite3_stmt stmt) - { - CheckErr(sqlite3_reset(stmt)); - } - - public static int CheckErr(int err, sqlite3? db = null) - { - SqliteException.ThrowExceptionForRC(err, db); - return err; - } -} diff --git a/Robust.Cdn/Helpers/StreamHelpers.cs b/Robust.Cdn/Helpers/StreamHelpers.cs index fa3a99f..5f0cf19 100644 --- a/Robust.Cdn/Helpers/StreamHelpers.cs +++ b/Robust.Cdn/Helpers/StreamHelpers.cs @@ -1,4 +1,4 @@ -namespace Robust.Cdn.Helpers; +namespace Robust.Cdn.Helpers; public static class StreamHelpers { @@ -13,5 +13,4 @@ public static void ReadExact(this Stream stream, Span buffer) buffer = buffer[cRead..]; } } - } diff --git a/Robust.Cdn/Jobs/DeleteTimedOutInProgressPublishesJob.cs b/Robust.Cdn/Jobs/DeleteTimedOutInProgressPublishesJob.cs index 1183fa4..cc35a6f 100644 --- a/Robust.Cdn/Jobs/DeleteTimedOutInProgressPublishesJob.cs +++ b/Robust.Cdn/Jobs/DeleteTimedOutInProgressPublishesJob.cs @@ -1,7 +1,7 @@ -using Dapper; using Microsoft.Extensions.Options; using Quartz; using Robust.Cdn.Config; +using Robust.Cdn.DataAccessLayer; using Robust.Cdn.Services; namespace Robust.Cdn.Jobs; @@ -19,40 +19,36 @@ public sealed class DeleteInProgressPublishesJob( ManifestDatabase manifestDatabase, TimeProvider timeProvider, IOptions options, - ILogger logger) : IJob + ILogger logger +) : IJob { public Task Execute(IJobExecutionContext context) { + var ct = context.CancellationToken; var opts = options.Value; logger.LogTrace("Checking for timed out in-progress publishes"); - var db = manifestDatabase.Connection; - using var tx = db.BeginTransaction(); + manifestDatabase.StartTransaction(); var deleteBefore = timeProvider.GetUtcNow() - TimeSpan.FromMinutes(opts.InProgressPublishTimeoutMinutes); var totalDeleted = 0; - var inProgress = db.Query<(int, string, string, DateTime)>(""" - SELECT PublishInProgress.Id, Version, Fork.Name, StartTime - FROM PublishInProgress - INNER JOIN Fork ON Fork.Id = PublishInProgress.ForkId - """); - - foreach (var (_, name, forkName, startTime) in inProgress) + var publishesInProgress = manifestDatabase.ListPublishInProgress(); + foreach (var (_, name, forkName, startTime) in publishesInProgress) { if (startTime >= deleteBefore) continue; logger.LogInformation("Deleting timed out publish for fork {Fork} version {Version}", forkName, name); - publishManager.AbortMultiPublish(forkName, name, tx, commit: false); + publishManager.AbortMultiPublish(forkName, name); totalDeleted += 1; } - tx.Commit(); + manifestDatabase.Commit(); logger.LogInformation("Deleted {TotalDeleted} timed out publishes", totalDeleted); diff --git a/Robust.Cdn/Jobs/IngestNewCdnContentJob.cs b/Robust.Cdn/Jobs/IngestNewCdnContentJob.cs index 9b38cbb..ac5b6d2 100644 --- a/Robust.Cdn/Jobs/IngestNewCdnContentJob.cs +++ b/Robust.Cdn/Jobs/IngestNewCdnContentJob.cs @@ -1,15 +1,13 @@ -using System.Buffers; +using System.Buffers; using System.IO.Compression; using System.Text; -using Dapper; -using Microsoft.Data.Sqlite; using Microsoft.Extensions.Options; using Quartz; using Robust.Cdn.Config; +using Robust.Cdn.DataAccessLayer; using Robust.Cdn.Helpers; using Robust.Cdn.Lib; using SpaceWizards.Sodium; -using SQLitePCL; namespace Robust.Cdn.Jobs; @@ -38,39 +36,34 @@ public async Task Execute(IJobExecutionContext context) var forkConfig = manifestOptions.Value.Forks[fork]; - var connection = cdnDatabase.Connection; - var transaction = connection.BeginTransaction(); + cdnDatabase.StartTransaction(); List newVersions; try { - newVersions = FindNewVersions(fork, connection); + newVersions = FindNewVersions(fork); if (newVersions.Count == 0) return; IngestNewVersions( fork, - connection, newVersions, - ref transaction, forkConfig, - context.CancellationToken); + context.CancellationToken + ); logger.LogDebug("Committing database"); - transaction.Commit(); + cdnDatabase.ReleaseContentBlob(); + cdnDatabase.Commit(); } finally { - transaction.Dispose(); + cdnDatabase.Dispose(); } - await QueueManifestAvailable(fork, newVersions); - } - - private async Task QueueManifestAvailable(string fork, IEnumerable newVersions) - { + // Queue manifest available job var scheduler = await schedulerFactory.GetScheduler(); await scheduler.TriggerJob( MakeNewManifestVersionsAvailableJob.Key, @@ -79,26 +72,13 @@ await scheduler.TriggerJob( private void IngestNewVersions( string fork, - SqliteConnection connection, List newVersions, - ref SqliteTransaction transaction, ManifestForkOptions forkConfig, CancellationToken cancel) { var cdnOpts = cdnOptions.Value; - var manifestOpts = manifestOptions.Value; - - var forkId = EnsureForkCreated(fork, connection); - - using var stmtLookupContent = connection.Handle!.Prepare("SELECT Id FROM Content WHERE Hash = ?"); - using var stmtInsertContent = connection.Handle!.Prepare( - "INSERT INTO Content (Hash, Size, Compression, Data) " + - "VALUES (@Hash, @Size, @Compression, @Data) " + - "RETURNING Id"); - using var stmtInsertContentManifestEntry = connection.Handle!.Prepare( - "INSERT INTO ContentManifestEntry (VersionId, ManifestIdx, ContentId) " + - "VALUES (@VersionId, @ManifestIdx, @ContentId) "); + var forkId = cdnDatabase.EnsureForkCreated(fork); var hash = new byte[32]; @@ -106,7 +86,6 @@ private void IngestNewVersions( var compressBuffer = ArrayPool.Shared.Rent(1024); using var compressor = new ZStdCompressionContext(); - SqliteBlobStream? blob = null; try { @@ -117,24 +96,17 @@ private void IngestNewVersions( { logger.LogDebug("Doing interim commit"); - blob?.Dispose(); - blob = null; - - transaction.Commit(); - transaction = connection.BeginTransaction(); + cdnDatabase.ReleaseContentBlob(); + cdnDatabase.Commit(); + cdnDatabase.StartTransaction(); } cancel.ThrowIfCancellationRequested(); logger.LogInformation("Ingesting new version: {Version}", version); - var versionId = connection.ExecuteScalar( - "INSERT INTO ContentVersion (ForkId, Version, TimeAdded, ManifestHash, ManifestData, CountDistinctBlobs) " + - "VALUES (@ForkId, @Version, datetime('now'), zeroblob(0), zeroblob(0), 0) " + - "RETURNING Id", - new { Version = version, ForkId = forkId }); + var versionId = cdnDatabase.InsertContentVersionAndGetId(forkId, version); - stmtInsertContentManifestEntry.BindInt64(1, versionId); var zipFilePath = buildDirectoryManager.GetBuildVersionFilePath( fork, @@ -173,13 +145,9 @@ private void IngestNewVersions( CryptoGenericHashBlake2B.Hash(hash, readData, ReadOnlySpan.Empty); // Look up if we already have this blob. - stmtLookupContent.BindBlob(1, hash); - - long contentId; - if (stmtLookupContent.Step() == raw.SQLITE_DONE) + var contentId = cdnDatabase.FindContentByHash(hash); + if (contentId == null) { - stmtLookupContent.Reset(); - // Don't have this blob yet, add a new one! newBlobCount += 1; @@ -214,49 +182,13 @@ private void IngestNewVersions( writeData = readData; } - // Insert blob database. - - stmtInsertContent.BindBlob(1, hash); // @Hash - stmtInsertContent.BindInt(2, dataLength); // @Size - stmtInsertContent.BindInt(3, (int)compression); // @Compression - stmtInsertContent.BindZeroBlob(4, writeData.Length); // @Data - - stmtInsertContent.Step(); - - contentId = stmtInsertContent.ColumnInt64(0); - - stmtInsertContent.Reset(); - - if (blob == null) - { - blob = SqliteBlobStream.Open( - connection.Handle!, - "main", - "Content", - "Data", - contentId, - true); - } - else - { - blob.Reopen(contentId); - } - - blob.Write(writeData); - } - else - { - contentId = stmtLookupContent.ColumnInt64(0); - - stmtLookupContent.Reset(); + // Insert blob database and write its data. + contentId = cdnDatabase.InsertContent(hash, dataLength, compression, writeData.Length); + cdnDatabase.WriteContentBlob(contentId.Value, writeData); } // Insert into ContentManifestEntry - stmtInsertContentManifestEntry.BindInt64(2, idx); // @ManifestIdx - stmtInsertContentManifestEntry.BindInt64(3, contentId); // @ContentId - - stmtInsertContentManifestEntry.Step(); - stmtInsertContentManifestEntry.Reset(); + cdnDatabase.InsertContentManifestEntry(versionId, idx, contentId.Value); // Write manifest entry. manifestWriter.Write($"{Convert.ToHexString(hash)} {entry.FullName}\n"); @@ -289,54 +221,25 @@ private void IngestNewVersions( var compressedData = compressBuffer.AsSpan(0, compressedLength); - connection.Execute( - "UPDATE ContentVersion " + - "SET ManifestHash = @ManifestHash, ManifestData = zeroblob(@ManifestDataSize) " + - "WHERE Id = @VersionId", - new - { - VersionId = versionId, - ManifestHash = manifestHash, - ManifestDataSize = compressedLength - }); - - using var manifestBlob = SqliteBlobStream.Open( - connection.Handle!, - "main", - "ContentVersion", - "ManifestData", - versionId, - true); - - manifestBlob.Write(compressedData); + cdnDatabase.UpdateContentVersionData(versionId, manifestHash, compressedLength); + cdnDatabase.WriteManifestBlob(versionId, compressedData); } // Calculate CountBlobsDeduplicated on ContentVersion - - connection.Execute( - "UPDATE ContentVersion AS cv " + - "SET CountDistinctBlobs = " + - " (SELECT COUNT(DISTINCT cme.ContentId) FROM ContentManifestEntry cme WHERE cme.VersionId = cv.Id) " + - "WHERE cv.Id = @VersionId", - new { VersionId = versionId } - ); + cdnDatabase.RefreshCountDistinctBlobs(versionId); versionIdx += 1; } } finally { - blob?.Dispose(); - ArrayPool.Shared.Return(readBuffer); ArrayPool.Shared.Return(compressBuffer); } } - private List FindNewVersions(string fork, SqliteConnection con) + private List FindNewVersions(string fork) { - using var stmtCheckVersion = con.Handle!.Prepare("SELECT 1 FROM ContentVersion WHERE Version = ?"); - var newVersions = new List<(string, DateTime)>(); var dir = buildDirectoryManager.GetForkPath(fork); @@ -351,10 +254,7 @@ private List FindNewVersions(string fork, SqliteConnection con) logger.LogTrace("Found version directory: {VersionDir}, write time: {WriteTime}", versionDirectory, createdTime); - stmtCheckVersion.Reset(); - stmtCheckVersion.BindString(1, version); - - if (stmtCheckVersion.Step() == raw.SQLITE_ROW) + if (cdnDatabase.IsVersionExisting(version)) { // Already have version, skip. logger.LogTrace("Already have version: {Version}", version); @@ -375,17 +275,4 @@ private List FindNewVersions(string fork, SqliteConnection con) return newVersions.OrderByDescending(x => x.Item2).Select(x => x.Item1).ToList(); } - - private static int EnsureForkCreated(string fork, SqliteConnection connection) - { - var id = connection.QuerySingleOrDefault( - "SELECT Id FROM Fork WHERE Name = @Name", - new { Name = fork }); - - id ??= connection.QuerySingle( - "INSERT INTO Fork (Name) VALUES (@Name) RETURNING Id", - new { Name = fork }); - - return id.Value; - } } diff --git a/Robust.Cdn/Jobs/MakeNewManifestVersionsAvailableJob.cs b/Robust.Cdn/Jobs/MakeNewManifestVersionsAvailableJob.cs index 3463d82..aec4af9 100644 --- a/Robust.Cdn/Jobs/MakeNewManifestVersionsAvailableJob.cs +++ b/Robust.Cdn/Jobs/MakeNewManifestVersionsAvailableJob.cs @@ -1,7 +1,6 @@ -using System.Text.Json; -using Dapper; +using System.Text.Json; using Quartz; -using Robust.Cdn.Helpers; +using Robust.Cdn.DataAccessLayer; namespace Robust.Cdn.Jobs; @@ -36,15 +35,11 @@ public async Task Execute(IJobExecutionContext context) fork, versions.Length); - using var tx = database.Connection.BeginTransaction(); + database.StartTransaction(); + database.SetForkVersionsAvailable(fork, versions); + database.Commit(); - var forkId = database.Connection.QuerySingle( - "SELECT Id FROM Fork WHERE Name = @ForkName", - new { ForkName = fork }); - - MakeVersionsAvailable(forkId, versions); - - tx.Commit(); + logger.LogInformation("New available versions: {Version}", string.Join(" , ", versions)); var scheduler = await factory.GetScheduler(); await scheduler.TriggerJob( @@ -52,23 +47,4 @@ await scheduler.TriggerJob( UpdateForkManifestJob.Data(fork, notifyUpdate: true)); } - private void MakeVersionsAvailable(int forkId, IEnumerable versions) - { - foreach (var version in versions) - { - logger.LogInformation("New available version: {Version}", version); - - database.Connection.Execute(""" - UPDATE ForkVersion - SET Available = TRUE - WHERE Name = @Name - AND ForkId = @ForkId - """, - new - { - Name = version, - ForkId = forkId - }); - } - } } diff --git a/Robust.Cdn/Jobs/PruneOldManifestBuilds.cs b/Robust.Cdn/Jobs/PruneOldManifestBuilds.cs index 0ba4e06..3e98111 100644 --- a/Robust.Cdn/Jobs/PruneOldManifestBuilds.cs +++ b/Robust.Cdn/Jobs/PruneOldManifestBuilds.cs @@ -1,7 +1,8 @@ -using Dapper; +using Dapper; using Microsoft.Extensions.Options; using Quartz; using Robust.Cdn.Config; +using Robust.Cdn.DataAccessLayer; namespace Robust.Cdn.Jobs; @@ -58,13 +59,7 @@ private int PruneFork(string forkName, ManifestForkOptions forkConfig, Cancellat var pruneFrom = DateTime.UtcNow - TimeSpan.FromDays(forkConfig.PruneBuildsDays); - var builds = manifestDatabase.Connection.Query(""" - SELECT FV.Id, FV.Name - FROM ForkVersion FV, Fork - WHERE FV.ForkId = Fork.Id - AND Fork.Name = @ForkName - AND FV.PublishedTime < @PruneFrom - """, new { ForkName = forkName, PruneFrom = pruneFrom }); + var builds = manifestDatabase.QueryVersionOlderThen(forkName, pruneFrom); var total = 0; foreach (var versionData in builds) @@ -84,16 +79,11 @@ AND FV.PublishedTime < @PruneFrom logger.LogTrace("Version directory didn't exist when cleaning it up ({Directory})", directory); } - manifestDatabase.Connection.Execute("DELETE FROM ForkVersion WHERE Id = @Id", versionData); + manifestDatabase.DeleteVersion(versionData.Id); total += 1; } return total; } - private sealed class VersionData - { - public required int Id { get; set; } - public required string Name { get; set; } - } } diff --git a/Robust.Cdn/Jobs/UpdateForkManifestJob.cs b/Robust.Cdn/Jobs/UpdateForkManifestJob.cs index 0d90bed..7288089 100644 --- a/Robust.Cdn/Jobs/UpdateForkManifestJob.cs +++ b/Robust.Cdn/Jobs/UpdateForkManifestJob.cs @@ -1,7 +1,5 @@ -using System.Text.Json; -using System.Text.Json.Serialization; -using Dapper; using Quartz; +using Robust.Cdn.DataAccessLayer; using Robust.Cdn.Helpers; namespace Robust.Cdn.Jobs; @@ -15,11 +13,6 @@ public sealed class UpdateForkManifestJob( ISchedulerFactory schedulerFactory, ILogger logger) : IJob { - private static readonly JsonSerializerOptions ManifestCacheContext = new() - { - PropertyNamingPolicy = JsonNamingPolicy.CamelCase - }; - public static readonly JobKey Key = new(nameof(UpdateForkManifestJob)); public const string KeyForkName = "ForkName"; @@ -36,105 +29,23 @@ public async Task Execute(IJobExecutionContext context) var fork = context.MergedJobDataMap.GetString(KeyForkName) ?? throw new InvalidDataException(); var notifyUpdate = context.MergedJobDataMap.GetBooleanValue(KeyNotifyUpdate); - var forkId = database.Connection.QuerySingle( - "SELECT Id FROM Fork WHERE Name = @ForkName", - new { ForkName = fork }); + var forkId = database.GetForkIdByName(fork); logger.LogInformation("Updating manifest cache for fork {Fork}", fork); - UpdateServerManifestCache(fork, forkId); + database.UpdateManifestCache(fork, forkId, baseUrlManager); if (notifyUpdate) await QueueNotifyWatchdogUpdate(fork); } - private void UpdateServerManifestCache(string fork, int forkId) - { - var data = CollectManifestData(fork, forkId); - var bytes = JsonSerializer.SerializeToUtf8Bytes(data, ManifestCacheContext); - - database.Connection.Execute("UPDATE Fork SET ServerManifestCache = @Data WHERE Id = @ForkId", - new - { - Data = bytes, - ForkId = forkId - }); - } - - private ManifestData CollectManifestData(string fork, int forkId) - { - var data = new ManifestData { Builds = new Dictionary() }; - - var versions = database.Connection - .Query<(int id, string name, DateTime time, string clientFileName, byte[] clientSha256)>( - """ - SELECT Id, Name, PublishedTime, ClientFileName, ClientSha256 - FROM ForkVersion - WHERE Available AND ForkId = @ForkId - """, - new { ForkId = forkId }); - - foreach (var version in versions) - { - var buildData = new ManifestBuildData - { - Time = DateTime.SpecifyKind(version.time, DateTimeKind.Utc), - Client = new ManifestArtifact - { - Url = baseUrlManager.MakeBuildInfoUrl( - $"fork/{fork}/version/{version.name}/file/{version.clientFileName}"), - Sha256 = Convert.ToHexString(version.clientSha256) - }, - Server = new Dictionary() - }; - - var servers = database.Connection.Query<(string platform, string fileName, byte[] sha256, long? size)>(""" - SELECT Platform, FileName, Sha256, FileSize - FROM ForkVersionServerBuild - WHERE ForkVersionId = @ForkVersionId - """, new { ForkVersionId = version.id }); - - foreach (var (platform, fileName, sha256, size) in servers) - { - buildData.Server.Add(platform, new ManifestArtifact - { - Url = baseUrlManager.MakeBuildInfoUrl($"fork/{fork}/version/{version.name}/file/{fileName}"), - Sha256 = Convert.ToHexString(sha256), - Size = size - }); - } - - data.Builds.Add(version.name, buildData); - } - - return data; - } - private async Task QueueNotifyWatchdogUpdate(string fork) { var scheduler = await schedulerFactory.GetScheduler(); await scheduler.TriggerJob( NotifyWatchdogUpdateJob.Key, - NotifyWatchdogUpdateJob.Data(fork)); - } - - private sealed class ManifestData - { - public required Dictionary Builds { get; set; } - } - - private sealed class ManifestBuildData - { - public DateTime Time { get; set; } - public required ManifestArtifact Client { get; set; } - public required Dictionary Server { get; set; } + NotifyWatchdogUpdateJob.Data(fork) + ); } - private sealed class ManifestArtifact - { - public required string Url { get; set; } - public required string Sha256 { get; set; } - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public long? Size { get; set; } - } } diff --git a/Robust.Cdn/ManifestMigrations/Script0004_AlterPublishInProgress_AddCommitColumns.sql b/Robust.Cdn/ManifestMigrations/Script0004_AlterPublishInProgress_AddCommitColumns.sql new file mode 100644 index 0000000..e5a573c --- /dev/null +++ b/Robust.Cdn/ManifestMigrations/Script0004_AlterPublishInProgress_AddCommitColumns.sql @@ -0,0 +1,7 @@ +-- Add columns that can represent sources used to build version that should be published +ALTER TABLE PublishInProgress ADD COLUMN SourceUrl TEXT NULL; +ALTER TABLE PublishInProgress ADD COLUMN SourceCommitId TEXT NULL; +ALTER TABLE PublishInProgress ADD COLUMN SourceBranchName TEXT NULL; +ALTER TABLE PublishInProgress ADD COLUMN EngineSourceUrl TEXT NULL; +ALTER TABLE PublishInProgress ADD COLUMN EngineSourceCommitId TEXT NULL; +ALTER TABLE PublishInProgress ADD COLUMN EngineSourceBranchName TEXT NULL; diff --git a/Robust.Cdn/Migrator.cs b/Robust.Cdn/Migrator.cs index 133bbe8..0f327d8 100644 --- a/Robust.Cdn/Migrator.cs +++ b/Robust.Cdn/Migrator.cs @@ -1,28 +1,30 @@ -using System.Reflection; using Dapper; using Microsoft.Data.Sqlite; +using System.Reflection; +using Robust.Cdn.Config; +using Robust.Cdn.DataAccessLayer; namespace Robust.Cdn; /// /// Utility class to do SQLite database migrations. /// -public sealed class Migrator +public class Migrator(IDatabaseOptions options) : ScopedSqliteRepositoryBase(options) { - internal static bool Migrate(IServiceProvider services, ILogger logger, SqliteConnection connection, string prefix) + public bool Migrate(IServiceProvider services, ILogger logger, string prefix) { logger.LogDebug("Migrating with prefix {Prefix}", prefix); - using var transaction = connection.BeginTransaction(deferred: true); + StartTransaction(true); - connection.Execute(@" + Connection.Execute(@" CREATE TABLE IF NOT EXISTS SchemaVersions( SchemaVersionID INTEGER PRIMARY KEY, ScriptName TEXT NOT NULL, Applied DATETIME NOT NULL - );"); + );", transaction: Transaction); - var appliedScripts = connection.Query("SELECT ScriptName FROM main.SchemaVersions"); + var appliedScripts = Connection.Query("SELECT ScriptName FROM main.SchemaVersions", transaction: Transaction); // ReSharper disable once InvokeAsExtensionMethod var scriptsToApply = Enumerable.Concat( @@ -34,32 +36,32 @@ Applied DATETIME NOT NULL foreach (var (name, script) in scriptsToApply) { logger.LogInformation("Applying migration {Transaction}!", name); - transaction.Save(name); + Transaction.Save(name); try { - var code = script.Up(services, connection); + var code = script.Up(services, Connection); if (!string.IsNullOrWhiteSpace(code)) - connection.Execute(code); + Connection.Execute(code); - connection.Execute( + Connection.Execute( "INSERT INTO SchemaVersions(ScriptName, Applied) VALUES (@Script, datetime('now'))", new { Script = name }); - transaction.Release(name); + Transaction.Release(name); } catch (Exception e) { logger.LogError(e, "Exception during migration {Transaction}, rolling back...!", name); - transaction.Rollback(name); + Transaction.Rollback(name); success = false; break; } } logger.LogInformation("Committing migrations"); - transaction.Commit(); + Transaction.Commit(); return success; } diff --git a/Robust.Cdn/Program.cs b/Robust.Cdn/Program.cs index 53815b5..e9adb5b 100644 --- a/Robust.Cdn/Program.cs +++ b/Robust.Cdn/Program.cs @@ -5,6 +5,7 @@ using Robust.Cdn; using Robust.Cdn.Config; using Robust.Cdn.Controllers; +using Robust.Cdn.DataAccessLayer; using Robust.Cdn.Helpers; using Robust.Cdn.Jobs; using Robust.Cdn.Services; @@ -88,7 +89,7 @@ var logFactory = services.GetRequiredService(); var loggerStartup = logFactory.CreateLogger("Robust.Cdn.Program"); var manifestOptions = services.GetRequiredService>().Value; - var db = services.GetRequiredService(); + var cdnOptions = services.GetRequiredService>().Value; var manifestDb = services.GetRequiredService(); if (string.IsNullOrEmpty(manifestOptions.FileDiskPath)) @@ -106,15 +107,19 @@ loggerStartup.LogDebug("Running migrations!"); var loggerMigrator = logFactory.CreateLogger(); - var success = Migrator.Migrate(services, loggerMigrator, db.Connection, "Robust.Cdn.Migrations"); - success &= Migrator.Migrate(services, loggerMigrator, manifestDb.Connection, "Robust.Cdn.ManifestMigrations"); + var mainMigrator = new Migrator(cdnOptions); + var manifestMigrator = new Migrator(manifestOptions); + var success = mainMigrator.Migrate(services, loggerMigrator, "Robust.Cdn.Migrations"); + success &= manifestMigrator.Migrate(services, loggerMigrator, "Robust.Cdn.ManifestMigrations"); if (!success) return 1; loggerStartup.LogDebug("Done running migrations!"); loggerStartup.LogDebug("Ensuring forks created in manifest DB"); - manifestDb.EnsureForksCreated(); + manifestDb.StartTransaction(); + manifestDb.EnsureForksCreated(manifestOptions.Forks.Keys); + manifestDb.Commit(); loggerStartup.LogDebug("Done creating forks in manifest DB!"); var scheduler = await initScope.ServiceProvider.GetRequiredService().GetScheduler(); diff --git a/Robust.Cdn/Services/PublishManager.cs b/Robust.Cdn/Services/PublishManager.cs index d417c31..fa3c5c4 100644 --- a/Robust.Cdn/Services/PublishManager.cs +++ b/Robust.Cdn/Services/PublishManager.cs @@ -1,5 +1,4 @@ -using System.Data.Common; -using Dapper; +using Robust.Cdn.DataAccessLayer; namespace Robust.Cdn.Services; @@ -8,26 +7,16 @@ public sealed class PublishManager( BuildDirectoryManager buildDirectoryManager, ILogger logger) { - public void AbortMultiPublish(string fork, string version, DbTransaction tx, bool commit) + public void AbortMultiPublish(string fork, string version) { logger.LogDebug("Aborting publish for fork {Fork}, version {version}", fork, version); // Drop record from database. - var dbCon = manifestDatabase.Connection; - dbCon.Execute(""" - DELETE FROM PublishInProgress - WHERE Version = @Version - AND ForkId IN ( - SELECT Id FROM Fork WHERE Name = @Fork - ) - """, new { Version = version, Fork = fork }, tx); + manifestDatabase.DeleteVersionByVersionName(fork, version); // Delete directory on disk. var versionDir = buildDirectoryManager.GetBuildVersionPath(fork, version); if (Directory.Exists(versionDir)) Directory.Delete(versionDir, recursive: true); - - if (commit) - tx.Commit(); } } diff --git a/Robust.Cdn/Views/ForkBuildPage/Index.cshtml b/Robust.Cdn/Views/ForkBuildPage/Index.cshtml index de6533e..4b75330 100644 --- a/Robust.Cdn/Views/ForkBuildPage/Index.cshtml +++ b/Robust.Cdn/Views/ForkBuildPage/Index.cshtml @@ -1,4 +1,4 @@ -@using Robust.Cdn.Controllers +@using Robust.Cdn.DataAccessLayer.Models @using Robust.Cdn.Helpers @model Robust.Cdn.Controllers.ForkBuildPageController.Model @addTagHelper *, Microsoft.AspNetCore.Mvc.TagHelpers @@ -27,15 +27,15 @@ }; } - async Task ShowBuild(ForkBuildPageController.Version version) + async Task ShowBuild(BuildVersionArtifactInfo buildVersion) {
Version:
-
@version.Name
+
@buildVersion.Name
Date:
-
- @if (version.EngineVersion is { } engineVersion) +
+ @if (buildVersion.EngineVersion is { } engineVersion) {
Engine Version:
@engineVersion
@@ -43,11 +43,11 @@

Download

    - @foreach (var versionServer in version.Servers) + @foreach (var versionServer in buildVersion.Servers) {
  • @ShowRid(versionServer.Platform) @if (versionServer.FileSize is { } fileSize)