Merge remote-tracking branch 'upstream/main' into store-all-avatar-fr…

…om-every-transactions

# Conflicts:
#	Mimir.Worker/BlockPoller.cs
#	Mimir.Worker/Initializer.cs

Mimir.Worker/BlockPoller.cs

@@ -10,14 +10,14 @@
 
 namespace Mimir.Worker;
 
-public class BlockPoller(IStateService stateService, IHeadlessGQLClient headlessGqlClient, MongoDbWorker mongoDbWorker)
+public class BlockPoller(IStateService stateService, IHeadlessGQLClient headlessGqlClient, MongoDbStore mongoDbStore)
 {
     public async Task RunAsync(CancellationToken cancellationToken)
     {
         var stateGetter = new StateGetter(stateService);
         while (!cancellationToken.IsCancellationRequested)
         {
-            var syncedBlockIndex = await mongoDbWorker.GetLatestBlockIndex();
+            var syncedBlockIndex = await mongoDbStore.GetLatestBlockIndex();
             var currentBlockIndex = await stateService.GetLatestIndex();
             var processBlockIndex = syncedBlockIndex + 1;
             if (processBlockIndex >= currentBlockIndex)
@@ -28,7 +28,7 @@ public async Task RunAsync(CancellationToken cancellationToken)
 
             await EveryAvatarAsync(processBlockIndex, stateGetter, cancellationToken);
             await BattleArenaAsync(processBlockIndex, stateGetter, cancellationToken);
-            await mongoDbWorker.UpdateLatestBlockIndex(processBlockIndex);
+            await mongoDbStore.UpdateLatestBlockIndex(processBlockIndex);
         }
     }
 
@@ -63,7 +63,7 @@ private async Task EveryAvatarAsync(
             var avatarAddresses = Enumerable.Range(0, GameConfig.SlotCount)
                 .Select(e => Addresses.GetAvatarAddress(agentAddress, e));
             var avatarDataArray = await Task.WhenAll(avatarAddresses.Select(stateGetter.GetAvatarData));
-            await mongoDbWorker.BulkUpsertAvatarDataAsync(
+            await mongoDbStore.BulkUpsertAvatarDataAsync(
                 avatarDataArray
                     .Where(e => e is not null)
                     .OfType<AvatarData>()
@@ -108,12 +108,12 @@ private async Task BattleArenaAsync(
             var myArenaData = await stateGetter.GetArenaData(roundData, myAvatarAddress);
             var enemyArenaData = await stateGetter.GetArenaData(roundData, enemyAvatarAddress);
 
-            await mongoDbWorker.BulkUpsertAvatarDataAsync(
+            await mongoDbStore.BulkUpsertAvatarDataAsync(
                 new[] { myAvatarData, enemyAvatarData }
                     .Where(e => e is not null)
                     .OfType<AvatarData>()
                     .ToList());
-            await mongoDbWorker.BulkUpsertArenaDataAsync(
+            await mongoDbStore.BulkUpsertArenaDataAsync(
                 new[] { myArenaData, enemyArenaData }
                     .Where(e => e is not null)
                     .OfType<ArenaData>()

Mimir.Worker/Initializer.cs

@@ -6,35 +6,39 @@ namespace Mimir.Worker;
 
 public class Initializer : BackgroundService
 {
-    private readonly MongoDbWorker _worker;
-    private readonly ArenaScrapper _scrapper;
+    private readonly MongoDbStore _store;
+    private readonly ArenaScrapper _arenaScrapper;
+    private readonly TableSheetScrapper _tableSheetScrapper;
     private readonly ILogger<Initializer> _logger;
     private readonly HeadlessGQLClient _headlessGqlClient;
     private readonly IStateService _stateService;
 
     public Initializer(
         ILogger<Initializer> logger,
-        ILogger<ArenaScrapper> scrapperLogger,
+        ILogger<ArenaScrapper> arenaScrapperLogger,
+        ILogger<TableSheetScrapper> tableSheetScrapperLogger,
         HeadlessGQLClient headlessGqlClient,
         IStateService stateService,
-        MongoDbWorker worker)
+        MongoDbStore store)
     {
         _logger = logger;
         _stateService = stateService;
-        _worker = worker;
+        _store = store;
         _headlessGqlClient = headlessGqlClient;
-        _scrapper = new ArenaScrapper(scrapperLogger, _stateService, _worker);
+        _arenaScrapper = new ArenaScrapper(arenaScrapperLogger, _stateService, _store);
+        _tableSheetScrapper = new TableSheetScrapper(tableSheetScrapperLogger, _stateService, _store);
     }
 
     protected override async Task ExecuteAsync(CancellationToken stoppingToken)
     {
         var started = DateTime.UtcNow;
-        await _scrapper.ExecuteAsync(stoppingToken);   
+        await _tableSheetScrapper.ExecuteAsync(stoppingToken);   
+        await _arenaScrapper.ExecuteAsync(stoppingToken);   
         _logger.LogInformation(
-            "Finished Initializer background service. Elapsed {TotalElapsedMinutes} minutes",
-            DateTime.UtcNow.Subtract(started).Minutes);
+                "Finished Initializer background service. Elapsed {TotalElapsedMinutes} minutes",
+                DateTime.UtcNow.Subtract(started).Minutes);
 
-        var poller = new BlockPoller(_stateService, _headlessGqlClient, _worker);
+        var poller = new BlockPoller(_stateService, _headlessGqlClient, _store);
         await poller.RunAsync(stoppingToken);
     }
 }

Mimir.Worker/Models/State/TableSheetData.cs

@@ -0,0 +1,20 @@
+using Libplanet.Crypto;
+using Nekoyume.TableData;
+
+namespace Mimir.Worker.Models;
+
+public class TableSheetData : BaseData
+{
+    public Address Address { get; }
+    public string Name { get; }
+    public ISheet Sheet { get; }
+    public string Raw { get; }
+
+    public TableSheetData(Address address, string name, ISheet sheet, string raw)
+    {
+        Address = address;
+        Name = name;
+        Sheet = sheet;
+        Raw = raw;
+    }
+}

Mimir.Worker/Program.cs

@@ -40,8 +40,8 @@
 builder.Services.AddSingleton(serviceProvider =>
 {
     var config = serviceProvider.GetRequiredService<IOptions<Configuration>>().Value;
-    var logger = serviceProvider.GetRequiredService<ILogger<MongoDbWorker>>();
-    return new MongoDbWorker(logger, config.MongoDbConnectionString, config.DatabaseName);
+    var logger = serviceProvider.GetRequiredService<ILogger<MongoDbStore>>();
+    return new MongoDbStore(logger, config.MongoDbConnectionString, config.DatabaseName);
 });
 builder.Services.AddHostedService<Initializer>();
 

Mimir.Worker/Scrapper/ArenaScrapper.cs

@@ -1,17 +1,15 @@
-using Mimir.Worker.Events;
 using Mimir.Worker.Services;
 using Mimir.Worker.Models;
-using Nekoyume.TableData;
 using Libplanet.Crypto;
 
 namespace Mimir.Worker.Scrapper;
 
-public class ArenaScrapper(ILogger<ArenaScrapper> logger, IStateService service, MongoDbWorker worker)
+public class ArenaScrapper(ILogger<ArenaScrapper> logger, IStateService service, MongoDbStore store)
 {
     private readonly ILogger<ArenaScrapper> _logger = logger;
 
     private readonly IStateService _stateService = service;
-    private readonly MongoDbWorker _worker = worker;
+    private readonly MongoDbStore _store = store;
 
     public async Task ExecuteAsync(CancellationToken cancellationToken)
     {
@@ -24,17 +22,17 @@ public async Task ExecuteAsync(CancellationToken cancellationToken)
         const int maxBufferSize = 10;
         async Task FlushBufferAsync()
         {
-            await _worker.BulkUpsertArenaDataAsync(buffer.Select(x => x.Arena).ToList());
-            await _worker.BulkUpsertAvatarDataAsync(buffer.Select(x => x.Avatar).ToList());
+            await _store.BulkUpsertArenaDataAsync(buffer.Select(x => x.Arena).ToList());
+            await _store.BulkUpsertAvatarDataAsync(buffer.Select(x => x.Avatar).ToList());
             foreach (var pair in buffer)
             {
-                await _worker.LinkAvatarWithArenaAsync(pair.AvatarAddress);
+                await _store.LinkAvatarWithArenaAsync(pair.AvatarAddress);
             }
 
             buffer.Clear();
         }
 
-        await _worker.UpdateLatestBlockIndex(latestBlockIndex);
+        await _store.UpdateLatestBlockIndex(latestBlockIndex);
 
         foreach (var avatarAddress in arenaParticipants.AvatarAddresses)
         {

Mimir.Worker/Scrapper/TableSheetScrapper.cs

@@ -0,0 +1,74 @@
+using Bencodex;
+using Bencodex.Types;
+using Libplanet.Common;
+using Mimir.Worker.Models;
+using Mimir.Worker.Services;
+using Nekoyume;
+using Nekoyume.Action;
+using Nekoyume.TableData;
+
+namespace Mimir.Worker.Scrapper;
+
+public class TableSheetScrapper(
+    ILogger<TableSheetScrapper> logger,
+    IStateService service,
+    MongoDbStore store
+)
+{
+    private readonly ILogger<TableSheetScrapper> _logger = logger;
+
+    private readonly IStateService _stateService = service;
+    private readonly MongoDbStore _store = store;
+
+    public async Task ExecuteAsync(CancellationToken cancellationToken)
+    {
+        var latestBlockIndex = await service.GetLatestIndex();
+        var stateGetter = _stateService.At();
+
+        var sheetTypes = typeof(ISheet)
+            .Assembly.GetTypes()
+            .Where(type =>
+                type.Namespace is { } @namespace
+                && @namespace.StartsWith($"{nameof(Nekoyume)}.{nameof(Nekoyume.TableData)}")
+                && !type.IsAbstract
+                && typeof(ISheet).IsAssignableFrom(type)
+            );
+
+        foreach (var sheetType in sheetTypes)
+        {
+            if (sheetType == typeof(ItemSheet) || sheetType == typeof(QuestSheet))
+            {
+                continue;
+            }
+
+            if (sheetType == typeof(WorldBossKillRewardSheet) || sheetType == typeof(WorldBossBattleRewardSheet))
+            {
+                // Handle later;
+                continue;
+            }
+
+            var sheetAddress = Addresses.TableSheet.Derive(sheetType.Name);
+            var sheetState = await _stateService.GetState(sheetAddress);
+            if (sheetState is not Text sheetValue)
+            {
+                throw new ArgumentException(nameof(sheetType));
+            }
+
+            var sheetInstance = Activator.CreateInstance(sheetType);
+            if (sheetInstance is not ISheet sheet)
+            {
+                throw new InvalidCastException($"Type {sheetType.Name} cannot be cast to ISheet.");
+            }
+
+            sheet.Set(sheetValue.Value);
+
+            var sheetData = new TableSheetData(
+                sheetAddress,
+                sheetType.Name,
+                sheet,
+                ByteUtil.Hex(new Codec().Encode(sheetState))
+            );
+            await _store.InsertTableSheets(sheetData);
+        }
+    }
+}