jamesmoore · jamesmoore · Feb 13, 2023
diff --git a/SDMetaTool/Cache/IPngFileDataSource.cs b/SDMetaTool/Cache/IPngFileDataSource.cs
@@ -1,14 +1,15 @@
 using System;
 using System.Collections.Generic;
+using System.Threading.Tasks;
 
 namespace SDMetaTool.Cache
 {
-    public interface IPngFileDataSource : IDisposable
+    public interface IPngFileDataSource : IAsyncDisposable
     {
-        IEnumerable<PngFile> GetAll();
-        PngFile ReadPngFile(string realFileName);
-        void WritePngFile(PngFile info);
-		void BeginTransaction();
-		void CommitTransaction();
+        Task<IEnumerable<PngFile>> GetAll();
+        Task<PngFile> ReadPngFile(string realFileName);
+        Task WritePngFile(PngFile info);
+		Task BeginTransaction();
+		Task CommitTransaction();
 	}
 }
diff --git a/SDMetaTool/Cache/JsonDataSource.cs b/SDMetaTool/Cache/JsonDataSource.cs
@@ -3,7 +3,9 @@
 using System.Collections.Generic;
 using System.IO.Abstractions;
 using System.Linq;
+using System.Runtime.CompilerServices;
 using System.Text.Json;
+using System.Threading.Tasks;
 
 namespace SDMetaTool.Cache
 {
@@ -21,7 +23,7 @@ public JsonDataSource(IFileSystem fileSystem)
 			cache = this.InitialGetAll().ToDictionary(p => p.FileName, p => p);
 		}
 
-		public IEnumerable<PngFile> GetAll()
+		public async Task<IEnumerable<PngFile>> GetAll()
 		{
 			return cache.Values;
 		}
@@ -43,7 +45,7 @@ private IEnumerable<PngFile> InitialGetAll()
 			}
 		}
 
-		private void WriteCache(IEnumerable<PngFile> cache)
+		private async Task WriteCache(IEnumerable<PngFile> cache)
 		{
 			var path = cachePath.GetPath();
 
@@ -62,28 +64,23 @@ private void WriteCache(IEnumerable<PngFile> cache)
 			fileSystem.File.WriteAllText(path, serialized);
 		}
 
-		public PngFile ReadPngFile(string realFileName)
+		public async Task<PngFile> ReadPngFile(string realFileName)
 		{
 			cache.TryGetValue(realFileName, out PngFile value);
 			return value;
 		}
 
-		public void WritePngFile(PngFile info)
+		public async Task WritePngFile(PngFile info)
 		{
 			if (info != null)
 			{
 				cache[info.FileName] = info;
 			}
 		}
 
-		public void Dispose()
+		public async ValueTask DisposeAsync()
 		{
-			Flush();
-		}
-
-		public void Flush()
-		{
-			this.WriteCache(cache.Values);
+			await this.WriteCache(cache.Values);
 		}
 
 		private static PngFile PngFileDTOToPngFile(PngFileDTO trackDTO)
@@ -110,11 +107,11 @@ private static PngFileDTO PngFileToPngFileDTO(PngFile track)
 			};
 		}
 
-		public void BeginTransaction()
+		public async Task BeginTransaction()
 		{
 		}
 
-		public void CommitTransaction()
+		public async Task CommitTransaction()
 		{
 		}
 

diff --git a/SDMetaTool/Cache/SqliteDataSource.cs b/SDMetaTool/Cache/SqliteDataSource.cs
@@ -3,16 +3,18 @@
 using NLog;
 using System;
 using System.Collections.Generic;
+using System.Data.Common;
 using System.IO.Abstractions;
 using System.Linq;
+using System.Threading.Tasks;
 
 namespace SDMetaTool.Cache
 {
 	public class SqliteDataSource : IPngFileDataSource
 	{
 		const string TableName = "PngFiles";
 		private readonly SqliteConnection connection;
-		private SqliteTransaction transaction;
+		private DbTransaction transaction;
 		private static readonly Logger logger = LogManager.GetCurrentClassLogger();
 
 		private readonly string[] columns = new string[]
@@ -224,25 +226,25 @@ internal GenerationParams ToGenerationParams()
 			}
 		}
 
-		public void Dispose()
+		public async ValueTask DisposeAsync()
 		{
-			this.CommitTransaction();
-			connection.Dispose();
+			await this.CommitTransaction();
+			await connection.DisposeAsync();
 		}
 
-		public IEnumerable<PngFile> GetAll()
+		public async Task<IEnumerable<PngFile>> GetAll()
 		{
-			var reader = connection.Query<DataRow>(
+			var reader = await connection.QueryAsync<DataRow>(
 				$@"SELECT *
 				FROM {TableName}"
 				);
 
 			return reader.Select(p => p.ToModel());
 		}
 
-		public PngFile ReadPngFile(string realFileName)
+		public async Task<PngFile> ReadPngFile(string realFileName)
 		{
-			var reader = connection.QueryFirstOrDefault<DataRow>(
+			var reader = await connection.QueryFirstOrDefaultAsync<DataRow>(
 			$@"SELECT *
 				FROM {TableName}
 				WHERE FileName = @FileName
@@ -258,29 +260,26 @@ public PngFile ReadPngFile(string realFileName)
 			}
 		}
 
-		public void WritePngFile(PngFile info)
+		public async Task WritePngFile(PngFile info)
 		{
-			connection.Execute(
+			await connection.ExecuteAsync(
 				insertSql,
 				DataRow.FromModel(info),
 				this.transaction
 			);
 		}
 
-		public void BeginTransaction()
+		public async Task BeginTransaction()
 		{
-			if (this.transaction == null)
-			{
-				this.transaction = this.connection.BeginTransaction();
-			}
+			this.transaction ??= await this.connection.BeginTransactionAsync();
 		}
 
-		public void CommitTransaction()
+		public async Task CommitTransaction()
 		{
 			if (this.transaction != null)
 			{
-				this.transaction.Commit();
-				this.transaction.Dispose();
+				await this.transaction.CommitAsync();
+				await this.transaction.DisposeAsync();
 				this.transaction = null;
 			}
 		}

diff --git a/SDMetaTool/CachedPngFileLoader.cs b/SDMetaTool/CachedPngFileLoader.cs
@@ -1,5 +1,6 @@
 using SDMetaTool.Cache;
 using System.IO.Abstractions;
+using System.Threading.Tasks;
 
 namespace SDMetaTool
 {
@@ -19,19 +20,19 @@ public CachedPngFileLoader(
 			this.fileSystem = fileSystem;
 		}
 
-		public PngFile GetPngFile(string filename)
+		public async Task<PngFile> GetPngFile(string filename)
 		{
 			var fileInfo = fileSystem.FileInfo.New(filename);
-			var pngFile = pngFileDataSource.ReadPngFile(filename);
+			var pngFile = await pngFileDataSource.ReadPngFile(filename);
 			if (pngFile != null && pngFile.LastUpdated == fileInfo.LastWriteTime)
 			{
 				return pngFile;
 			}
 			else
 			{
-				pngFile = inner.GetPngFile(filename);
+				pngFile = await inner.GetPngFile(filename);
 				pngFile.Exists = true;
-				pngFileDataSource.WritePngFile(pngFile);
+				await pngFileDataSource.WritePngFile(pngFile);
 				return pngFile;
 			}
 		}

diff --git a/SDMetaTool/IPngFileListProcessor.cs b/SDMetaTool/IPngFileListProcessor.cs
@@ -1,9 +1,10 @@
 using System.Collections.Generic;
+using System.Threading.Tasks;
 
 namespace SDMetaTool
 {
     public interface IPngFileListProcessor
     {
-        void ProcessPngFiles(string root);
+        Task ProcessPngFiles(string root);
     }
 }
diff --git a/SDMetaTool/IPngFileLoader.cs b/SDMetaTool/IPngFileLoader.cs
@@ -1,7 +1,9 @@
-namespace SDMetaTool
+using System.Threading.Tasks;
+
+namespace SDMetaTool
 {
     public interface IPngFileLoader
     {
-        PngFile GetPngFile(string filename);
+        Task<PngFile> GetPngFile(string filename);
     }
 }
diff --git a/SDMetaTool/PngFileLoader.cs b/SDMetaTool/PngFileLoader.cs
@@ -5,6 +5,7 @@
 using System;
 using System.IO.Abstractions;
 using System.Linq;
+using System.Threading.Tasks;
 
 namespace SDMetaTool
 {
@@ -20,13 +21,13 @@ public PngFileLoader(IFileSystem fileSystem)
 
         }
 
-        public PngFile GetPngFile(string filename)
+        public async Task<PngFile> GetPngFile(string filename)
         {
             logger.Info($"Indexing: {filename}");
 
             try
             {
-                return ReadPngFile(fileSystem, filename);
+                return await ReadPngFile(fileSystem, filename);
             }
             catch (Exception ex)
             {
@@ -35,7 +36,7 @@ public PngFile GetPngFile(string filename)
             }
         }
 
-        public PngFile ReadPngFile(IFileSystem fileSystem, string filename)
+        private async Task<PngFile> ReadPngFile(IFileSystem fileSystem, string filename)
         {
             var fileInfo = fileSystem.FileInfo.New(filename);
 

diff --git a/SDMetaTool/Processors/CSVPngFileLister.cs b/SDMetaTool/Processors/CSVPngFileLister.cs
@@ -4,6 +4,7 @@
 using System.Globalization;
 using System.IO;
 using System.Linq;
+using System.Threading.Tasks;
 
 namespace SDMetaTool.Processors
 {
@@ -22,13 +23,16 @@ public CSVPngFileLister(IFileLister fileLister, IPngFileLoader pngFileLoader, st
 			this.distinct = distinct;
 		}
 
-		public void ProcessPngFiles(string root)
+		public async Task ProcessPngFiles(string root)
 		{
 			using var writer = new StreamWriter(outfile);
 			using var csv = new CsvWriter(writer, CultureInfo.InvariantCulture);
 
 			var fileNames = fileLister.GetList(root);
-			var pngFiles = fileNames.Select(p => pngFileLoader.GetPngFile(p)).Where(p => p != null).OrderBy(p => p.FileName).ToList();
+			var tasks = fileNames.Select(p => pngFileLoader.GetPngFile(p)).ToList();
+			await Task.WhenAll(tasks.ToArray());
+
+			var pngFiles = tasks.Select(p => p.Result).Where(p => p != null).OrderBy(p => p.FileName).ToList();
 
 			var csvs = distinct ? GetCSVDistinct(pngFiles) : GetCSVPerItem(pngFiles);
 			csv.WriteRecords(csvs);

diff --git a/SDMetaTool/Processors/Rescan.cs b/SDMetaTool/Processors/Rescan.cs
@@ -1,6 +1,7 @@
 using NLog;
 using SDMetaTool.Cache;
 using System.Linq;
+using System.Threading.Tasks;
 
 namespace SDMetaTool.Processors
 {
@@ -21,13 +22,13 @@ public Rescan(
 			this.pngFileLoader = pngFileLoader;
 		}
 
-		public void ProcessPngFiles(string root)
+		public async Task ProcessPngFiles(string root)
 		{
 			logger.Info("Rescan started");
-			pngFileDataSource.BeginTransaction();
+			await pngFileDataSource.BeginTransaction();
 			var fileNames = fileLister.GetList(root);
 
-			var knownFiles = pngFileDataSource.GetAll();
+			var knownFiles = await pngFileDataSource.GetAll();
 
 			var deleted = knownFiles.Where(p => p.Exists).Select(p => p.FileName).Except(fileNames);
 
@@ -36,18 +37,22 @@ public void ProcessPngFiles(string root)
 			{
 				var fileToDelete = knownFilesLookup[file].Single();
 				fileToDelete.Exists = false;
-				pngFileDataSource.WritePngFile(fileToDelete);
+				await pngFileDataSource.WritePngFile(fileToDelete);
 				logger.Info("Removing " + file);
 			}
 
-			var newFiles = fileNames.Except(knownFiles.Where(p => p.Exists).Select(p => p.FileName)).Select(p => pngFileLoader.GetPngFile(p)).Where(p => p != null).ToList(); ;
-			foreach (var file in newFiles.Where(p => p.Exists == false))
+			var newFileNames = fileNames.Except(knownFiles.Where(p => p.Exists).Select(p => p.FileName));
+			var newFileTasks = newFileNames.Select( p => pngFileLoader.GetPngFile(p)).ToList();
+
+			await Task.WhenAll(newFileTasks.ToArray());
+
+			foreach (var file in newFileTasks.Select(p => p.Result).Where(p => p != null && p.Exists == false))
 			{
 				file.Exists = true;
-				pngFileDataSource.WritePngFile(file);
+				await pngFileDataSource.WritePngFile(file);
 				logger.Info("Adding " + file.FileName);
 			}
-			pngFileDataSource.CommitTransaction();
+			await pngFileDataSource.CommitTransaction();
 			logger.Info("Rescan finished");
 		}
 	}

diff --git a/SDMetaTool/Processors/SummaryInfo.cs b/SDMetaTool/Processors/SummaryInfo.cs
@@ -1,6 +1,7 @@
 using BetterConsoleTables;
 using System;
 using System.Linq;
+using System.Threading.Tasks;
 
 namespace SDMetaTool.Processors
 {
@@ -15,10 +16,13 @@ public SummaryInfo(IFileLister fileLister, IPngFileLoader pngFileLoader)
 			this.pngFileLoader = pngFileLoader;
 		}
 
-		public void ProcessPngFiles(string root)
+		public async Task ProcessPngFiles(string root)
 		{
 			var fileNames = fileLister.GetList(root);
-			var pngFiles = fileNames.Select(p => pngFileLoader.GetPngFile(p)).Where(p => p != null).OrderBy(p => p.FileName).ToList();
+			var tasks = fileNames.Select(p => pngFileLoader.GetPngFile(p)).ToList();
+			await Task.WhenAll(tasks.ToArray());
+
+			var pngFiles = tasks.Select(p => p.Result).Where(p => p != null).OrderBy(p => p.FileName).ToList();
 
 			var distinctPrompts = pngFiles.Select(p => p.Parameters?.PromptHash).Distinct().ToList();
 			var distinctFullPrompts = pngFiles.Select(p => new

diff --git a/SDMetaTool/Program.cs b/SDMetaTool/Program.cs
@@ -12,7 +12,7 @@ public class Program
         static async Task<int> Main(string[] args)
         {
 			var fileSystem = new FileSystem();
-			using var sqliteDataSource = new SqliteDataSource(fileSystem);
+			await using var sqliteDataSource = new SqliteDataSource(fileSystem);
             var datasource = sqliteDataSource;
 			var loader = new CachedPngFileLoader(fileSystem, new PngFileLoader(fileSystem), datasource);
 			var fileLister = new FileLister(fileSystem);