diff --git a/.vscode/launch.json b/.vscode/launch.json index 33ae28ee..b6060323 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,15 +1,25 @@ { - "version": "0.2.0", - "configurations": [ - { - "name": "Debug NineChroniclesUtilBackend", - "type": "coreclr", - "request": "launch", - "program": "${workspaceFolder}/NineChroniclesUtilBackend/bin/Debug/net8.0/NineChroniclesUtilBackend.dll", - "cwd": "${workspaceFolder}/NineChroniclesUtilBackend", - "stopAtEntry": false, - "console": "internalConsole", - "preLaunchTask": "build", - } - ] -} + "version": "0.2.0", + "configurations": [ + { + "name": "Debug NineChroniclesUtilBackend", + "type": "coreclr", + "request": "launch", + "program": "${workspaceFolder}/NineChroniclesUtilBackend/bin/Debug/net8.0/NineChroniclesUtilBackend.dll", + "cwd": "${workspaceFolder}/NineChroniclesUtilBackend", + "stopAtEntry": false, + "console": "internalConsole", + "preLaunchTask": "build-backend" + }, + { + "name": "Debug NineChroniclesUtilBackend.Store", + "type": "coreclr", + "request": "launch", + "program": "${workspaceFolder}/NineChroniclesUtilBackend.Store/bin/Debug/net8.0/NineChroniclesUtilBackend.Store.dll", + "cwd": "${workspaceFolder}/NineChroniclesUtilBackend.Store", + "stopAtEntry": false, + "console": "internalConsole", + "preLaunchTask": "build-store" + } + ] +} \ No newline at end of file diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 02d7950b..2eff8588 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -2,7 +2,7 @@ "version": "2.0.0", "tasks": [ { - "label": "build", + "label": "build-backend", "command": "dotnet", "type": "process", "args": [ @@ -14,7 +14,7 @@ "problemMatcher": "$msCompile" }, { - "label": "publish", + "label": "publish-backend", "command": "dotnet", "type": "process", "args": [ @@ -26,7 +26,7 @@ "problemMatcher": "$msCompile" }, { - "label": "watch", + "label": "watch-backend", "command": "dotnet", "type": "process", "args": [ @@ -36,6 +36,42 @@ "${workspaceFolder}/NineChroniclesUtilBackend/NineChroniclesUtilBackend.csproj" ], "problemMatcher": "$msCompile" + }, + { + "label": "build-store", + "command": "dotnet", + "type": "process", + "args": [ + "build", + "${workspaceFolder}/NineChroniclesUtilBackend.Store/NineChroniclesUtilBackend.Store.csproj", + "/property:GenerateFullPaths=true", + "/consoleloggerparameters:NoSummary" + ], + "problemMatcher": "$msCompile" + }, + { + "label": "publish-store", + "command": "dotnet", + "type": "process", + "args": [ + "publish", + "${workspaceFolder}/NineChroniclesUtilBackend.Store/NineChroniclesUtilBackend.Store.csproj", + "/property:GenerateFullPaths=true", + "/consoleloggerparameters:NoSummary" + ], + "problemMatcher": "$msCompile" + }, + { + "label": "watch-store", + "command": "dotnet", + "type": "process", + "args": [ + "watch", + "run", + "--project", + "${workspaceFolder}/NineChroniclesUtilBackend.Store/NineChroniclesUtilBackend.Store.csproj" + ], + "problemMatcher": "$msCompile" } ] } \ No newline at end of file diff --git a/NineChroniclesUtilBackend.Store/Client/EmptyChronicleClient.cs b/NineChroniclesUtilBackend.Store/Client/EmptyChronicleClient.cs new file mode 100644 index 00000000..d31d4bb8 --- /dev/null +++ b/NineChroniclesUtilBackend.Store/Client/EmptyChronicleClient.cs @@ -0,0 +1,63 @@ +using Newtonsoft.Json; +using NineChroniclesUtilBackend.Store.Models; + +namespace NineChroniclesUtilBackend.Store.Client; + +public class EmptyChronicleClient +{ + private readonly HttpClient _httpClient; + private readonly string _baseUrl; + + public EmptyChronicleClient(string baseUrl) + { + _baseUrl = baseUrl; + _httpClient = new HttpClient(); + } + + public async Task GetStateByAddressAsync(string address, string? accountAddress = null) + { + var url = $"{_baseUrl}/api/states/{address}/raw"; + if (accountAddress != null) + { + url += $"?account={Uri.EscapeDataString(accountAddress)}"; + } + + var response = await _httpClient.GetAsync(url); + + response.EnsureSuccessStatusCode(); + + var content = await response.Content.ReadAsStringAsync(); + + var stateResponse = JsonConvert.DeserializeObject(content); + if (stateResponse == null) + { + throw new InvalidOperationException("StateResponse is null."); + } + + return stateResponse; + } + + public async Task GetLatestBlock() + { + var url = $"{_baseUrl}/api/blocks/latest"; + + var response = await _httpClient.GetAsync(url); + + response.EnsureSuccessStatusCode(); + + var content = await response.Content.ReadAsStringAsync(); + + var stateResponse = JsonConvert.DeserializeObject(content); + if (stateResponse == null) + { + throw new InvalidOperationException("StateResponse is null."); + } + + return stateResponse; + } + + // public async Task GetBlock(int index) + // { + // return stateResponse; + // } +} diff --git a/NineChroniclesUtilBackend.Store/Configuration.cs b/NineChroniclesUtilBackend.Store/Configuration.cs new file mode 100644 index 00000000..d7964967 --- /dev/null +++ b/NineChroniclesUtilBackend.Store/Configuration.cs @@ -0,0 +1,8 @@ +namespace NineChroniclesUtilBackend.Store; + +public class Configuration +{ + public string EmptyChronicleBaseUrl { get; init; } + public string MongoDbConnectionString { get; init; } + public string DatabaseName { get; set; } +} diff --git a/NineChroniclesUtilBackend.Store/Events/ArenaDataCollectedEventArgs.cs b/NineChroniclesUtilBackend.Store/Events/ArenaDataCollectedEventArgs.cs new file mode 100644 index 00000000..cb21a085 --- /dev/null +++ b/NineChroniclesUtilBackend.Store/Events/ArenaDataCollectedEventArgs.cs @@ -0,0 +1,15 @@ +using NineChroniclesUtilBackend.Store.Models; + +namespace NineChroniclesUtilBackend.Store.Events; + +public class ArenaDataCollectedEventArgs : EventArgs +{ + public ArenaData ArenaData { get; set; } + public AvatarData AvatarData { get; set; } + + public ArenaDataCollectedEventArgs(ArenaData arenaData, AvatarData avatarData) + { + ArenaData = arenaData; + AvatarData = avatarData; + } +} diff --git a/NineChroniclesUtilBackend.Store/Models/BlockResponse.cs b/NineChroniclesUtilBackend.Store/Models/BlockResponse.cs new file mode 100644 index 00000000..e29c5f90 --- /dev/null +++ b/NineChroniclesUtilBackend.Store/Models/BlockResponse.cs @@ -0,0 +1,13 @@ +namespace NineChroniclesUtilBackend.Store.Models; + +public class BlockResponse +{ + public string Hash { get; } + public long Index { get; } + + public BlockResponse(string hash, long index) + { + Hash = hash; + Index = index; + } +} \ No newline at end of file diff --git a/NineChroniclesUtilBackend.Store/Models/ScrapperResult.cs b/NineChroniclesUtilBackend.Store/Models/ScrapperResult.cs new file mode 100644 index 00000000..c9ea22df --- /dev/null +++ b/NineChroniclesUtilBackend.Store/Models/ScrapperResult.cs @@ -0,0 +1,26 @@ +using Libplanet.Crypto; + +namespace NineChroniclesUtilBackend.Store.Models; + +public class ScrapperResult +{ + public DateTime StartTime { get; set; } + public int TotalElapsedMinutes { get; set; } + public int AvatarScrappedCount { get; set; } + public int ArenaScrappedCount { get; set; } + public List
FailedAvatarAddresses { get; } = new List
(); + public List
FailedArenaAddresses { get; } = new List
(); + + public override string ToString() + { + var failedAvatarAddresses = string.Join(", ", FailedAvatarAddresses.Select(a => a.ToString())); + var failedArenaAddresses = string.Join(", ", FailedArenaAddresses.Select(a => a.ToString())); + + return $"StartTime: {StartTime}, " + + $"TotalElapsedMinutes: {TotalElapsedMinutes}, " + + $"AvatarScrappedCount: {AvatarScrappedCount}, " + + $"ArenaScrappedCount: {ArenaScrappedCount}, " + + $"FailedAvatarAddresses: [{failedAvatarAddresses}], " + + $"FailedArenaAddresses: [{failedArenaAddresses}]"; + } +} diff --git a/NineChroniclesUtilBackend.Store/Models/State/ArenaData.cs b/NineChroniclesUtilBackend.Store/Models/State/ArenaData.cs new file mode 100644 index 00000000..cca415b0 --- /dev/null +++ b/NineChroniclesUtilBackend.Store/Models/State/ArenaData.cs @@ -0,0 +1,21 @@ +using Nekoyume.Model.Arena; +using Nekoyume.TableData; +using Libplanet.Crypto; + +namespace NineChroniclesUtilBackend.Store.Models; + +public class ArenaData : BaseData +{ + public ArenaScore Score { get; } + public ArenaInformation Information { get; } + public ArenaSheet.RoundData RoundData { get; } + public Address AvatarAddress { get; } + + public ArenaData(ArenaScore score, ArenaInformation information, ArenaSheet.RoundData roundData, Address avatarAddress) + { + Score = score; + Information = information; + RoundData = roundData; + AvatarAddress = avatarAddress; + } +} \ No newline at end of file diff --git a/NineChroniclesUtilBackend.Store/Models/State/AvataData.cs b/NineChroniclesUtilBackend.Store/Models/State/AvataData.cs new file mode 100644 index 00000000..eeac763e --- /dev/null +++ b/NineChroniclesUtilBackend.Store/Models/State/AvataData.cs @@ -0,0 +1,17 @@ +using Nekoyume.Model.State; + +namespace NineChroniclesUtilBackend.Store.Models; + +public class AvatarData : BaseData +{ + public AvatarState Avatar { get; } + public ItemSlotState ItemSlot { get; } + public List RuneSlot { get; } + + public AvatarData(AvatarState avatar, ItemSlotState itemSlot, List runeSlot) + { + Avatar = avatar; + ItemSlot = itemSlot; + RuneSlot = runeSlot; + } +} \ No newline at end of file diff --git a/NineChroniclesUtilBackend.Store/Models/State/BaseData.cs b/NineChroniclesUtilBackend.Store/Models/State/BaseData.cs new file mode 100644 index 00000000..bb006448 --- /dev/null +++ b/NineChroniclesUtilBackend.Store/Models/State/BaseData.cs @@ -0,0 +1,20 @@ +using Newtonsoft.Json; +using NineChroniclesUtilBackend.Store.Util; + +namespace NineChroniclesUtilBackend.Store.Models; + +public class BaseData +{ + protected static JsonSerializerSettings JsonSerializerSettings => new JsonSerializerSettings + { + Converters = new[] { new BigIntegerToStringConverter() }, + Formatting = Formatting.Indented, + // ContractResolver = new Newtonsoft.Json.Serialization.CamelCasePropertyNamesContractResolver(), + NullValueHandling = NullValueHandling.Ignore + }; + + public string ToJson() + { + return JsonConvert.SerializeObject(this, JsonSerializerSettings); + } +} diff --git a/NineChroniclesUtilBackend.Store/Models/StateResponse.cs b/NineChroniclesUtilBackend.Store/Models/StateResponse.cs new file mode 100644 index 00000000..bd63d494 --- /dev/null +++ b/NineChroniclesUtilBackend.Store/Models/StateResponse.cs @@ -0,0 +1,15 @@ +namespace NineChroniclesUtilBackend.Store.Models; + +public class StateResponse +{ + public string Address { get; } + public string AccountAddress { get; } + public string Value { get; } + + public StateResponse(string address, string accountAddress, string value) + { + Address = address; + AccountAddress = accountAddress; + Value = value; + } +} \ No newline at end of file diff --git a/NineChroniclesUtilBackend.Store/Models/StoreResult.cs b/NineChroniclesUtilBackend.Store/Models/StoreResult.cs new file mode 100644 index 00000000..acf7155a --- /dev/null +++ b/NineChroniclesUtilBackend.Store/Models/StoreResult.cs @@ -0,0 +1,30 @@ +using Libplanet.Crypto; + +namespace NineChroniclesUtilBackend.Store.Models; + +public class StoreResult +{ + public DateTime StartTime { get; set; } + public int TotalElapsedMinutes { get; set; } + public int StoreArenaRequestCount { get; set; } + public int StoreAvatarRequestCount { get; set; } + public int AvatarStoredCount { get; set; } + public int ArenaStoredCount { get; set; } + public List
FailedAvatarAddresses { get; } = new List
(); + public List
FailedArenaAddresses { get; } = new List
(); + + public override string ToString() + { + var failedAvatarAddresses = string.Join(", ", FailedAvatarAddresses.Select(a => a.ToString())); + var failedArenaAddresses = string.Join(", ", FailedArenaAddresses.Select(a => a.ToString())); + + return $"StartTime: {StartTime}, " + + $"TotalElapsedMinutes: {TotalElapsedMinutes}, " + + $"StoreArenaRequestCount: {StoreArenaRequestCount}, " + + $"StoreAvatarRequestCount: {StoreAvatarRequestCount}, " + + $"AvatarStoredCount: {AvatarStoredCount}, " + + $"ArenaStoredCount: {ArenaStoredCount}, " + + $"FailedAvatarAddresses: [{failedAvatarAddresses}], " + + $"FailedArenaAddresses: [{failedArenaAddresses}]"; + } +} diff --git a/NineChroniclesUtilBackend.Store/NineChroniclesUtilBackend.Store.csproj b/NineChroniclesUtilBackend.Store/NineChroniclesUtilBackend.Store.csproj new file mode 100644 index 00000000..76a82d5c --- /dev/null +++ b/NineChroniclesUtilBackend.Store/NineChroniclesUtilBackend.Store.csproj @@ -0,0 +1,17 @@ + + + + net8.0 + enable + enable + dotnet-NineChroniclesUtilBackend.Store-bccda56f-4d38-484b-ab03-ebb26065c837 + + + + + + + + + + diff --git a/NineChroniclesUtilBackend.Store/Program.cs b/NineChroniclesUtilBackend.Store/Program.cs new file mode 100644 index 00000000..bdbb34fe --- /dev/null +++ b/NineChroniclesUtilBackend.Store/Program.cs @@ -0,0 +1,33 @@ +using NineChroniclesUtilBackend.Store; +using NineChroniclesUtilBackend.Store.Client; +using NineChroniclesUtilBackend.Store.Services; +using Microsoft.Extensions.Options; + +var builder = Host.CreateApplicationBuilder(args); + +string configPath = Environment.GetEnvironmentVariable("STORE_CONFIG_FILE") ?? "appsettings.json"; +builder.Configuration + .AddJsonFile(configPath, optional: true, reloadOnChange: true) + .AddEnvironmentVariables("STORE_"); + +builder.Services.Configure(builder.Configuration.GetSection("Configuration")); + +builder.Services.AddSingleton(serviceProvider => +{ + var config = serviceProvider.GetRequiredService>().Value; + return new EmptyChronicleClient(config.EmptyChronicleBaseUrl); +}); + +builder.Services.AddSingleton(serviceProvider => +{ + var config = serviceProvider.GetRequiredService>().Value; + var logger = serviceProvider.GetRequiredService>(); + return new MongoDbStore(logger, config.MongoDbConnectionString, config.DatabaseName); +}); + +builder.Services.AddSingleton(); + +builder.Services.AddHostedService(); + +var host = builder.Build(); +host.Run(); diff --git a/NineChroniclesUtilBackend.Store/Properties/launchSettings.json b/NineChroniclesUtilBackend.Store/Properties/launchSettings.json new file mode 100644 index 00000000..8efeb70e --- /dev/null +++ b/NineChroniclesUtilBackend.Store/Properties/launchSettings.json @@ -0,0 +1,12 @@ +{ + "$schema": "http://json.schemastore.org/launchsettings.json", + "profiles": { + "NineChroniclesUtilBackend.Store": { + "commandName": "Project", + "dotnetRunMessages": true, + "environmentVariables": { + "DOTNET_ENVIRONMENT": "Development" + } + } + } +} diff --git a/NineChroniclesUtilBackend.Store/Scrapper/ArenaScrapper.cs b/NineChroniclesUtilBackend.Store/Scrapper/ArenaScrapper.cs new file mode 100644 index 00000000..3d5fbad6 --- /dev/null +++ b/NineChroniclesUtilBackend.Store/Scrapper/ArenaScrapper.cs @@ -0,0 +1,98 @@ +using NineChroniclesUtilBackend.Store.Events; +using NineChroniclesUtilBackend.Store.Services; +using NineChroniclesUtilBackend.Store.Models; +using NineChroniclesUtilBackend.Store.Client; +using Nekoyume.TableData; +using Libplanet.Crypto; + +namespace NineChroniclesUtilBackend.Store.Scrapper; + +public class ArenaScrapper +{ + private readonly ILogger _logger; + public readonly ScrapperResult Result = new ScrapperResult(); + + private StateGetter _stateGetter; + private EmptyChronicleClient _client; + public event EventHandler OnDataCollected; + + public ArenaScrapper(ILogger logger, IStateService service, EmptyChronicleClient client) + { + _stateGetter = new StateGetter(service); + _logger = logger; + _client = client; + } + + protected virtual void RaiseDataCollected(ArenaData arenaData, AvatarData avatarData) + { + OnDataCollected?.Invoke(this, new ArenaDataCollectedEventArgs(arenaData, avatarData)); + } + + public async Task ExecuteAsync() + { + Result.StartTime = DateTime.UtcNow; + var latestBlock = await _client.GetLatestBlock(); + + var roundData = await GetArenaRoundData(latestBlock.Index); + + var arenaParticipants = await _stateGetter.GetArenaParticipantsState(roundData.ChampionshipId, roundData.Round); + + foreach(var avatarAddress in arenaParticipants.AvatarAddresses) + { + var arenaData = await GetArenaData(roundData, avatarAddress); + var avatarData = await GetAvatarData(avatarAddress); + + if (arenaData != null && avatarData != null) + { + RaiseDataCollected(arenaData, avatarData); + } + } + + Result.TotalElapsedMinutes = DateTime.UtcNow.Subtract(Result.StartTime).Minutes; + } + + public async Task GetArenaRoundData(long index) + { + var arenaSheet = await _stateGetter.GetSheet(); + var roundData = arenaSheet.GetRoundByBlockIndex(index); + + return roundData; + } + + public async Task GetArenaData(ArenaSheet.RoundData roundData, Address avatarAddress) + { + try + { + var arenaScore = await _stateGetter.GetArenaScoreState(avatarAddress, roundData.ChampionshipId, roundData.Round); + var arenaInfo = await _stateGetter.GetArenaInfoState(avatarAddress, roundData.ChampionshipId, roundData.Round); + + Result.ArenaScrappedCount += 1; + return new ArenaData(arenaScore, arenaInfo, roundData, avatarAddress); + } + catch (Exception ex) + { + _logger.LogError($"An error occurred during GetArenaData: {ex.Message}"); + Result.FailedArenaAddresses.Add(avatarAddress); + return null; + } + } + + public async Task GetAvatarData(Address avatarAddress) + { + try + { + var avatarState = await _stateGetter.GetAvatarState(avatarAddress); + var avatarItemSlotState = await _stateGetter.GetItemSlotState(avatarAddress); + var avatarRuneStates = await _stateGetter.GetRuneStates(avatarAddress); + + Result.AvatarScrappedCount += 1; + return new AvatarData(avatarState, avatarItemSlotState, avatarRuneStates); + } + catch (Exception ex) + { + _logger.LogError($"An error occurred during GetAvatarData: {ex.Message}"); + Result.FailedAvatarAddresses.Add(avatarAddress); + return null; + } + } +} diff --git a/NineChroniclesUtilBackend.Store/Scrapper/StateGetter.cs b/NineChroniclesUtilBackend.Store/Scrapper/StateGetter.cs new file mode 100644 index 00000000..a2bbecaf --- /dev/null +++ b/NineChroniclesUtilBackend.Store/Scrapper/StateGetter.cs @@ -0,0 +1,132 @@ +using NineChroniclesUtilBackend.Store.Services; +using Libplanet.Crypto; +using Bencodex.Types; +using Nekoyume.TableData; +using Nekoyume; +using Nekoyume.Action; +using Nekoyume.Model.EnumType; +using Nekoyume.Model.Item; +using Nekoyume.Model.State; +using Nekoyume.Model.Arena; + +namespace NineChroniclesUtilBackend.Store.Scrapper; + +public class StateGetter +{ + private readonly IStateService _service; + + public StateGetter(IStateService service) + { + _service = service; + } + + public async Task GetSheet() + where T : ISheet, new() + { + var sheetState = await _service.GetState(Addresses.TableSheet.Derive(typeof(T).Name)); + if (sheetState is not Text sheetValue) + { + throw new ArgumentException(nameof(T)); + } + + var sheet = new T(); + sheet.Set(sheetValue.Value); + return sheet; + } + + public async Task GetArenaParticipantsState(int championshipId, int roundId) + { + var arenaParticipantsAddress = + ArenaParticipants.DeriveAddress(championshipId, roundId); + var state = await _service.GetState(arenaParticipantsAddress); + return state switch + { + List list => new ArenaParticipants(list), + _ => throw new ArgumentException(nameof(arenaParticipantsAddress)) + }; + } + + public async Task GetArenaScoreState(Address avatarAddress, int championshipId, int roundId) + { + var arenaScoreAddress = + ArenaScore.DeriveAddress(avatarAddress, championshipId, roundId); + var state = await _service.GetState(arenaScoreAddress); + return state switch + { + List list => new ArenaScore(list), + _ => throw new ArgumentException(nameof(arenaScoreAddress)) + }; + } + + public async Task GetArenaInfoState(Address avatarAddress, int championshipId, int roundId) + { + var arenaInfoAddress = + ArenaInformation.DeriveAddress(avatarAddress, championshipId, roundId); + var state = await _service.GetState(arenaInfoAddress); + return state switch + { + List list => new ArenaInformation(list), + _ => throw new ArgumentException(nameof(arenaInfoAddress)) + }; + } + + public async Task GetAvatarState(Address avatarAddress) + { + var state = await _service.GetState(avatarAddress); + if (state is not Dictionary dictionary) + { + throw new ArgumentException(nameof(avatarAddress)); + } + + var inventoryAddress = avatarAddress.Derive("inventory"); + var inventoryState = await _service.GetState(inventoryAddress); + if (inventoryState is not List list) + { + throw new ArgumentException(nameof(avatarAddress)); + } + + var inventory = new Inventory(list); + + var avatarState = new AvatarState(dictionary) + { + inventory = inventory + }; + + return avatarState; + } + + public async Task GetItemSlotState(Address avatarAddress) + { + var state = await _service.GetState( + ItemSlotState.DeriveAddress(avatarAddress, BattleType.Arena)); + return state switch + { + List list => new ItemSlotState(list), + null => new ItemSlotState(BattleType.Arena), + _ => throw new ArgumentException(nameof(avatarAddress)) + }; + } + + public async Task> GetRuneStates(Address avatarAddress) + { + var state = await _service.GetState( + RuneSlotState.DeriveAddress(avatarAddress, BattleType.Arena)); + var runeSlotState = state switch + { + List list => new RuneSlotState(list), + null => new RuneSlotState(BattleType.Arena), + _ => throw new ArgumentException(nameof(avatarAddress)) + }; + + var runes = new List(); + foreach (var runeStateAddress in runeSlotState.GetEquippedRuneSlotInfos().Select(info => RuneState.DeriveAddress(avatarAddress, info.RuneId))) + { + if (await _service.GetState(runeStateAddress) is List list) + { + runes.Add(new RuneState(list)); + } + } + + return runes; + } +} diff --git a/NineChroniclesUtilBackend.Store/Services/EmptyChronicleStateService.cs b/NineChroniclesUtilBackend.Store/Services/EmptyChronicleStateService.cs new file mode 100644 index 00000000..c00688ee --- /dev/null +++ b/NineChroniclesUtilBackend.Store/Services/EmptyChronicleStateService.cs @@ -0,0 +1,36 @@ +using Bencodex; +using Bencodex.Types; +using Libplanet.Crypto; +using Libplanet.Action.State; +using Libplanet.Types.Blocks; +using NineChroniclesUtilBackend.Store.Client; + +namespace NineChroniclesUtilBackend.Store.Services; + +public class EmptyChronicleStateService : IStateService +{ + private readonly EmptyChronicleClient client; + private static readonly Codec Codec = new(); + + public EmptyChronicleStateService(EmptyChronicleClient client) + { + this.client = client; + } + + public Task GetStates(Address[] addresses) + { + return Task.WhenAll(addresses.Select(GetState)); + } + + public async Task GetState(Address address) + { + var result = await client.GetStateByAddressAsync(address.ToString(), ReservedAddresses.LegacyAccount.ToString()); + + if (result.Value is null) + { + return null; + } + + return Codec.Decode(Convert.FromHexString(result.Value)); + } +} diff --git a/NineChroniclesUtilBackend.Store/Services/IStateService.cs b/NineChroniclesUtilBackend.Store/Services/IStateService.cs new file mode 100644 index 00000000..197556a9 --- /dev/null +++ b/NineChroniclesUtilBackend.Store/Services/IStateService.cs @@ -0,0 +1,12 @@ +using System.Threading.Tasks; +using Bencodex.Types; +using Libplanet.Crypto; + +namespace NineChroniclesUtilBackend.Store.Services; + + +public interface IStateService +{ + Task GetStates(Address[] addresses); + Task GetState(Address address); +} diff --git a/NineChroniclesUtilBackend.Store/Services/MongoDbStore.cs b/NineChroniclesUtilBackend.Store/Services/MongoDbStore.cs new file mode 100644 index 00000000..03c570e7 --- /dev/null +++ b/NineChroniclesUtilBackend.Store/Services/MongoDbStore.cs @@ -0,0 +1,207 @@ +using Libplanet.Crypto; +using MongoDB.Bson; +using MongoDB.Driver; +using NineChroniclesUtilBackend.Store.Models; +using System.Collections.Concurrent; + +namespace NineChroniclesUtilBackend.Store.Services; + +public class MongoDbStore +{ + private readonly ILogger _logger; + private readonly IMongoCollection _arenaCollection; + private readonly IMongoCollection _avatarCollection; + private BlockingCollection _arenaDataQueue = new BlockingCollection(); + private BlockingCollection _avatarDataQueue = new BlockingCollection(); + private List
_avatarAddresses = new List
(); + private readonly int _batchSize = 100; + + public readonly StoreResult Result = new StoreResult(); + + public MongoDbStore(ILogger logger, string connectionString, string databaseName) + { + _logger = logger; + + var client = new MongoClient(connectionString); + var database = client.GetDatabase(databaseName); + + _arenaCollection = database.GetCollection("arena"); + _avatarCollection = database.GetCollection("avatars"); + + Task.Run(() => ProcessArenaDataAsync()); + Task.Run(() => ProcessAvatarDataAsync()); + + Result.StartTime = DateTime.UtcNow; + } + + public void AddArenaData(ArenaData arenaData) + { + _arenaDataQueue.Add(arenaData); + + Result.StoreArenaRequestCount += 1; + } + + public void AddAvatarData(AvatarData avatarData) + { + _avatarDataQueue.Add(avatarData); + _avatarAddresses.Add(avatarData.Avatar.address); + + Result.StoreAvatarRequestCount += 1; + } + + private async Task ProcessArenaDataAsync() + { + List batch = new List(); + foreach (var arenaData in _arenaDataQueue.GetConsumingEnumerable()) + { + batch.Add(arenaData); + if (batch.Count >= _batchSize) + { + await BulkUpsertArenaDataAsync(batch); + batch.Clear(); + } + } + + if (batch.Count > 0) + { + await BulkUpsertArenaDataAsync(batch); + } + } + + private async Task ProcessAvatarDataAsync() + { + List batch = new List(); + foreach (var avatarData in _avatarDataQueue.GetConsumingEnumerable()) + { + batch.Add(avatarData); + if (batch.Count >= _batchSize) + { + await BulkUpsertAvatarDataAsync(batch); + batch.Clear(); + } + } + + if (batch.Count > 0) + { + await BulkUpsertAvatarDataAsync(batch); + } + } + + private async Task BulkUpsertArenaDataAsync(List arenaDatas) + { + var bulkOps = new List>(); + + try + { + foreach (var arenaData in arenaDatas) + { + var filter = Builders.Filter.Eq("avatarAddress", arenaData.AvatarAddress.ToHex()); + var bsonDocument = BsonDocument.Parse(arenaData.ToJson()); + var upsertOne = new ReplaceOneModel(filter, bsonDocument) { IsUpsert = true }; + bulkOps.Add(upsertOne); + } + if (bulkOps.Count > 0) + { + await _arenaCollection.BulkWriteAsync(bulkOps); + } + + Result.ArenaStoredCount += bulkOps.Count; + _logger.LogInformation($"Stored {bulkOps.Count} arena data"); + } + catch(Exception ex) + { + _logger.LogError($"An error occurred during BulkUpsertArenaDataAsync: {ex.Message}"); + Result.FailedArenaAddresses.AddRange(arenaDatas.Select(d => d.AvatarAddress) ); + } + } + + private async Task BulkUpsertAvatarDataAsync(List avatarDatas) + { + var bulkOps = new List>(); + + try + { + foreach (var avatarData in avatarDatas) + { + var filter = Builders.Filter.Eq("avatar.address", avatarData.Avatar.address.ToHex()); + var bsonDocument = BsonDocument.Parse(avatarData.ToJson()); + var upsertOne = new ReplaceOneModel(filter, bsonDocument) { IsUpsert = true }; + bulkOps.Add(upsertOne); + } + if (bulkOps.Count > 0) + { + await _avatarCollection.BulkWriteAsync(bulkOps); + } + + Result.AvatarStoredCount += bulkOps.Count; + _logger.LogInformation($"Stored {bulkOps.Count} avatar data"); + } + catch(Exception ex) + { + _logger.LogError($"An error occurred during BulkUpsertAvatarDataAsync: {ex.Message}"); + Result.FailedAvatarAddresses.AddRange(avatarDatas.Select(d => d.Avatar.address) ); + } + } + + public async Task LinkAvatarsToArenaAsync() + { + const int batchSize = 500; + + _logger.LogInformation($"Start Link Avatars"); + + for (int i = 0; i < _avatarAddresses.Count; i += batchSize) + { + var batchAddresses = _avatarAddresses.Skip(i).Take(batchSize).ToList(); + var bulkOps = new List>(); + + foreach (var address in batchAddresses) + { + var avatarFilter = Builders.Filter.Eq("Avatar.address", address.ToHex()); + var avatar = await _avatarCollection.Find(avatarFilter).FirstOrDefaultAsync(); + if (avatar != null) + { + var objectId = avatar["_id"].AsObjectId; + var arenaFilter = Builders.Filter.Eq("AvatarAddress", address.ToHex()); + var update = Builders.Update.Set("avatarObjectId", objectId); + var updateModel = new UpdateOneModel(arenaFilter, update) { IsUpsert = false }; + bulkOps.Add(updateModel); + } + } + + if (bulkOps.Count > 0) + { + await _arenaCollection.BulkWriteAsync(bulkOps); + } + } + + _logger.LogInformation($"Finish Link Avatars"); + } + + public async Task FlushAsync() + { + _arenaDataQueue.CompleteAdding(); + _avatarDataQueue.CompleteAdding(); + + var remainingArenaDatas = new List(); + while (_arenaDataQueue.TryTake(out var arenaData)) + { + remainingArenaDatas.Add(arenaData); + } + if (remainingArenaDatas.Any()) + { + await BulkUpsertArenaDataAsync(remainingArenaDatas); + } + + var remainingAvatarDatas = new List(); + while (_avatarDataQueue.TryTake(out var avatarData)) + { + remainingAvatarDatas.Add(avatarData); + } + if (remainingAvatarDatas.Any()) + { + await BulkUpsertAvatarDataAsync(remainingAvatarDatas); + } + + _logger.LogInformation($"Finish Flushing"); + } +} \ No newline at end of file diff --git a/NineChroniclesUtilBackend.Store/Util/JsonConverter.cs b/NineChroniclesUtilBackend.Store/Util/JsonConverter.cs new file mode 100644 index 00000000..bd351e10 --- /dev/null +++ b/NineChroniclesUtilBackend.Store/Util/JsonConverter.cs @@ -0,0 +1,22 @@ +using System.Numerics; +using Newtonsoft.Json; + +namespace NineChroniclesUtilBackend.Store.Util; + +public class BigIntegerToStringConverter : JsonConverter +{ + public override bool CanConvert(Type objectType) + { + return objectType == typeof(BigInteger); + } + + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + throw new NotImplementedException(); + } + + public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + { + writer.WriteValue(value.ToString()); + } +} diff --git a/NineChroniclesUtilBackend.Store/Worker.cs b/NineChroniclesUtilBackend.Store/Worker.cs new file mode 100644 index 00000000..1a4be43c --- /dev/null +++ b/NineChroniclesUtilBackend.Store/Worker.cs @@ -0,0 +1,48 @@ +using NineChroniclesUtilBackend.Store.Events; +using NineChroniclesUtilBackend.Store.Scrapper; +using NineChroniclesUtilBackend.Store.Client; +using NineChroniclesUtilBackend.Store.Services; + +namespace NineChroniclesUtilBackend.Store; + +public class Worker : BackgroundService +{ + private readonly MongoDbStore _store; + private readonly ArenaScrapper _scrapper; + private readonly ILogger _logger; + private readonly IStateService _stateService; + + public Worker( + ILogger logger, + ILogger scrapperLogger, + IStateService stateService, + MongoDbStore store, + EmptyChronicleClient client) + { + _logger = logger; + _stateService = stateService; + _store = store; + _scrapper = new ArenaScrapper(scrapperLogger, _stateService, client); + _scrapper.OnDataCollected += HandleDataCollected; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + await _scrapper.ExecuteAsync(); + await _store.FlushAsync(); + await _store.LinkAvatarsToArenaAsync(); + + _store.Result.TotalElapsedMinutes = DateTime.UtcNow.Subtract(_store.Result.StartTime).Minutes; + + _logger.LogInformation($"Scrapper Result: {_scrapper.Result}"); + _logger.LogInformation($"Store Result: {_store.Result}"); + } + + private async void HandleDataCollected(object sender, ArenaDataCollectedEventArgs e) + { + _store.AddArenaData(e.ArenaData); + _store.AddAvatarData(e.AvatarData); + + _logger.LogInformation("{avatarAddress} Data Collected", e.AvatarData.Avatar.address); + } +} diff --git a/NineChroniclesUtilBackend.Store/appsettings.Development.json b/NineChroniclesUtilBackend.Store/appsettings.Development.json new file mode 100644 index 00000000..ba2b11fd --- /dev/null +++ b/NineChroniclesUtilBackend.Store/appsettings.Development.json @@ -0,0 +1,9 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.Hosting.Lifetime": "Information" + } + }, + "EmptyChronicleBaseUrl": "http://localhost:5009" +} diff --git a/NineChroniclesUtilBackend.Store/appsettings.json b/NineChroniclesUtilBackend.Store/appsettings.json new file mode 100644 index 00000000..b6d18490 --- /dev/null +++ b/NineChroniclesUtilBackend.Store/appsettings.json @@ -0,0 +1,13 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.Hosting.Lifetime": "Information" + } + }, + "Configuration": { + "EmptyChronicleBaseUrl": "http://localhost:5009", + "MongoDbConnectionString": "mongodb://rootuser:rootpass@localhost:27017", + "DatabaseName": "9cutil_backend" + } +} \ No newline at end of file diff --git a/NineChroniclesUtilBackend.sln b/NineChroniclesUtilBackend.sln index 08c079b7..2d78a670 100644 --- a/NineChroniclesUtilBackend.sln +++ b/NineChroniclesUtilBackend.sln @@ -1,10 +1,12 @@ - + Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 VisualStudioVersion = 17.0.31903.59 MinimumVisualStudioVersion = 10.0.40219.1 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NineChroniclesUtilBackend", "NineChroniclesUtilBackend\NineChroniclesUtilBackend.csproj", "{1909E492-D010-4FB5-BC4B-DEF73025C41E}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NineChroniclesUtilBackend.Store", "NineChroniclesUtilBackend.Store\NineChroniclesUtilBackend.Store.csproj", "{462CDFD3-650B-46A7-A937-8E3BCE7E527C}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -18,5 +20,9 @@ Global {1909E492-D010-4FB5-BC4B-DEF73025C41E}.Debug|Any CPU.Build.0 = Debug|Any CPU {1909E492-D010-4FB5-BC4B-DEF73025C41E}.Release|Any CPU.ActiveCfg = Release|Any CPU {1909E492-D010-4FB5-BC4B-DEF73025C41E}.Release|Any CPU.Build.0 = Release|Any CPU + {462CDFD3-650B-46A7-A937-8E3BCE7E527C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {462CDFD3-650B-46A7-A937-8E3BCE7E527C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {462CDFD3-650B-46A7-A937-8E3BCE7E527C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {462CDFD3-650B-46A7-A937-8E3BCE7E527C}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection EndGlobal