Add table sheet csv store & handle api csv response (#49)

* Add sheet csv store & handle api response * Handle large csv data * Remove file format querystring * Handle not found sheet * Implement enum extension method
planetarium · May 13, 2024 · 1c77004 · 1c77004
1 parent 37a8a74
commit 1c77004
Show file tree

Hide file tree

Showing 10 changed files with 135 additions and 36 deletions.
diff --git a/Mimir.Worker/Mimir.Worker.csproj b/Mimir.Worker/Mimir.Worker.csproj
@@ -12,7 +12,8 @@
     <PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
     <PackageReference Include="Lib9c" Version="1.8.0-dev.c6a17ceb8856f4c7c6bdc7a7288be58479d9d006" />
     <PackageReference Include="Libplanet" Version="4.1.0-dev.20242745721" />
-    <PackageReference Include="MongoDB.Driver" Version="2.24.0" />
+    <PackageReference Include="MongoDB.Driver" Version="2.25.0" />
+    <PackageReference Include="MongoDB.Driver.GridFS" Version="2.25.0" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageReference Include="StrawberryShake" Version="13.8.1" />
     <PackageReference Include="StrawberryShake.Transport.Http" Version="13.8.1" />

diff --git a/Mimir.Worker/Models/State/TableSheetData.cs b/Mimir.Worker/Models/State/TableSheetData.cs
@@ -7,14 +7,16 @@ public class TableSheetData : BaseData
 {
     public Address Address { get; }
     public string Name { get; }
-    public ISheet Sheet { get; }
+    public ISheet SheetJson { get; }
+    public string SheetCsv { get; }
     public string Raw { get; }
 
-    public TableSheetData(Address address, string name, ISheet sheet, string raw)
+    public TableSheetData(Address address, string name, ISheet sheetJson, string sheetCsv, string raw)
     {
         Address = address;
         Name = name;
-        Sheet = sheet;
+        SheetJson = sheetJson;
+        SheetCsv = sheetCsv;
         Raw = raw;
     }
 }
diff --git a/Mimir.Worker/Scrapper/TableSheetScrapper.cs b/Mimir.Worker/Scrapper/TableSheetScrapper.cs
@@ -5,6 +5,7 @@
 using Mimir.Worker.Services;
 using Nekoyume;
 using Nekoyume.Action;
+using Nekoyume.Model.State;
 using Nekoyume.TableData;
 
 namespace Mimir.Worker.Scrapper;
@@ -66,8 +67,10 @@ type.Namespace is { } @namespace
                 sheetAddress,
                 sheetType.Name,
                 sheet,
+                sheetState.ToDotnetString(),
                 ByteUtil.Hex(new Codec().Encode(sheetState))
             );
+
             await _store.InsertTableSheets(sheetData);
         }
     }

diff --git a/Mimir.Worker/Services/MongoDbStore.cs b/Mimir.Worker/Services/MongoDbStore.cs
@@ -1,10 +1,9 @@
+using System.Text;
 using Libplanet.Crypto;
 using Mimir.Worker.Models;
-using Mimir.Worker.Scrapper;
 using MongoDB.Bson;
 using MongoDB.Driver;
-using MongoDB.Driver;
-using Nekoyume.TableData;
+using MongoDB.Driver.GridFS;
 
 namespace Mimir.Worker.Services;
 
@@ -18,6 +17,8 @@ public class MongoDbStore
 
     private readonly string _databaseName;
 
+    private readonly GridFSBucket _gridFs;
+
     private IMongoCollection<BsonDocument> ArenaCollection =>
         _database.GetCollection<BsonDocument>("arena");
 
@@ -36,6 +37,7 @@ public MongoDbStore(ILogger<MongoDbStore> logger, string connectionString, strin
         _database = _client.GetDatabase(databaseName);
         _logger = logger;
         _databaseName = databaseName;
+        _gridFs = new GridFSBucket(_database);
     }
 
     public async Task LinkAvatarWithArenaAsync(Address address)
@@ -141,10 +143,22 @@ public async Task BulkUpsertAvatarDataAsync(List<AvatarData> avatarDatas)
     public async Task InsertTableSheets(TableSheetData sheetData)
     {
         var filter = Builders<BsonDocument>.Filter.Eq("Address", sheetData.Address.ToHex());
-        var bsonDocument = BsonDocument.Parse(sheetData.ToJson());
+
+        var sheetCsvBytes = Encoding.UTF8.GetBytes(sheetData.SheetCsv);
+        var sheetCsvId = await _gridFs.UploadFromBytesAsync($"{sheetData.Name}-csv", sheetCsvBytes);
+        var sheetRawBytes = Encoding.UTF8.GetBytes(sheetData.Raw);
+        var sheetRawId = await _gridFs.UploadFromBytesAsync($"{sheetData.Name}-raw", sheetRawBytes);
+
+        var document = BsonDocument.Parse(sheetData.ToJson());
+
+        document.Remove("SheetCsv");
+        document.Add("SheetCsvFileId", sheetCsvId);
+        document.Remove("Raw");
+        document.Add("SheetRawFileId", sheetRawId);
+
         await TableSheetsCollection.ReplaceOneAsync(
             filter,
-            bsonDocument,
+            document,
             new ReplaceOptions { IsUpsert = true }
         );
     }

diff --git a/Mimir/Controllers/TableSheetsController.cs b/Mimir/Controllers/TableSheetsController.cs
@@ -1,9 +1,6 @@
-using Libplanet.Crypto;
 using Microsoft.AspNetCore.Mvc;
-using Mimir.Models.Agent;
+using Mimir.Enums;
 using Mimir.Repositories;
-using Mimir.Services;
-using Mimir.Util;
 
 namespace Mimir.Controllers;
 
@@ -12,23 +9,38 @@ namespace Mimir.Controllers;
 public class TableSheetsController(TableSheetsRepository tableSheetsRepository) : ControllerBase
 {
     [HttpGet("names")]
-    public async Task<string[]> GetSheetNames(
-        string network
-    )
+    public async Task<string[]> GetSheetNames(string network)
     {
         var sheetNames = tableSheetsRepository.GetSheetNames(network);
 
         return sheetNames;
     }
 
     [HttpGet("{sheetName}")]
-    public async Task<ContentResult> GetSheet(
-        string network,
-        string sheetName
-    )
+    [Produces("application/json", "text/csv")]
+    public async Task<IActionResult> GetSheet(string network, string sheetName)
     {
-        var sheet = tableSheetsRepository.GetSheet(network, sheetName);
+        var acceptHeader = Request.Headers["Accept"].ToString();
+        SheetFormat? sheetFormatNullable = acceptHeader switch
+        {
+            "text/csv" => SheetFormat.Csv,
+            "application/json" => SheetFormat.Json,
+            _ => null,
+        };
 
-        return Content(sheet, "application/json");;
+        if (sheetFormatNullable is not { } sheetFormat)
+        {
+            return StatusCode(StatusCodes.Status406NotAcceptable);
+        }
+
+        try
+        {
+            var sheet = await tableSheetsRepository.GetSheet(network, sheetName, sheetFormat);
+            return Content(sheet, sheetFormat.ToMimeType());
+        }
+        catch (KeyNotFoundException ex)
+        {
+            return NotFound(new { message = ex.Message });
+        }
     }
 }
diff --git a/Mimir/Enums/SheetFormat.cs b/Mimir/Enums/SheetFormat.cs
@@ -0,0 +1,24 @@
+namespace Mimir.Enums
+{
+    public enum SheetFormat
+    {
+        Json,
+        Csv
+    }
+
+    static class EnumExtension
+    {
+        public static string ToMimeType(this SheetFormat format)
+        {
+            switch (format)
+            {
+                case SheetFormat.Csv:
+                    return "text/csv";
+                case SheetFormat.Json:
+                    return "application/json";
+                default:
+                    throw new KeyNotFoundException(format.ToString());
+            }
+        }
+    }
+}
diff --git a/Mimir/Mimir.csproj b/Mimir/Mimir.csproj
@@ -11,7 +11,8 @@
   <ItemGroup>
     <PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="8.0.1" />
     <PackageReference Include="Microsoft.IdentityModel.Tokens" Version="7.3.1" />
-    <PackageReference Include="MongoDB.Driver" Version="2.24.0" />
+    <PackageReference Include="MongoDB.Driver" Version="2.25.0" />
+    <PackageReference Include="MongoDB.Driver.GridFS" Version="2.25.0" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
     <PackageReference Include="Lib9c" Version="1.8.0-dev.c6a17ceb8856f4c7c6bdc7a7288be58479d9d006" />

diff --git a/Mimir/Repositories/BaseRepositories.cs b/Mimir/Repositories/BaseRepositories.cs
@@ -1,6 +1,4 @@
-using System.Collections.Generic;
 using Mimir.Services;
-using MongoDB.Bson;
 using MongoDB.Driver;
 
 namespace Mimir.Repositories;
@@ -9,6 +7,7 @@ public abstract class BaseRepository<T>
 {
     private readonly MongoDBCollectionService _mongoDBCollectionService;
     private readonly Dictionary<string, IMongoCollection<T>> _collections = new();
+    private readonly Dictionary<string, IMongoDatabase> _databases = new();
 
     protected BaseRepository(MongoDBCollectionService mongoDBCollectionService)
     {
@@ -31,6 +30,9 @@ protected BaseRepository(MongoDBCollectionService mongoDBCollectionService)
                 databaseName
             );
             _collections[network] = collection;
+
+            var database = _mongoDBCollectionService.GetDatabase(databaseName);
+            _databases[network] = database;
         }
     }
 
@@ -45,4 +47,14 @@ protected IMongoCollection<T> GetCollection(string network)
 
         throw new ArgumentException("Invalid network name", nameof(network));
     }
+
+    protected IMongoDatabase GetDatabase(string network)
+    {
+        if (_databases.TryGetValue(network, out var database))
+        {
+            return database;
+        }
+
+        throw new ArgumentException("Invalid network name", nameof(network));
+    }
 }
diff --git a/Mimir/Repositories/TableSheetsRepository.cs b/Mimir/Repositories/TableSheetsRepository.cs
@@ -1,10 +1,11 @@
-using Mimir.Models.Avatar;
+using System.Collections.Generic;
+using System.Text;
+using Mimir.Enums;
 using Mimir.Services;
 using MongoDB.Bson;
 using MongoDB.Bson.IO;
 using MongoDB.Driver;
-using Newtonsoft.Json;
-using System.Collections.Generic;
+using MongoDB.Driver.GridFS;
 
 namespace Mimir.Repositories;
 
@@ -37,20 +38,44 @@ public string[] GetSheetNames(string network)
         return names.ToArray();
     }
 
-    public string GetSheet(string network, string sheetName)
+    public async Task<string> GetSheet(string network, string sheetName, SheetFormat sheetFormat)
     {
         var collection = GetCollection(network);
+        var gridFs = new GridFSBucket(GetDatabase(network));
 
-        var projection = Builders<BsonDocument>.Projection.Include("Sheet").Exclude("_id");
+        string fieldToInclude = sheetFormat switch
+        {
+            SheetFormat.Csv => "SheetCsvFileId",
+            SheetFormat.Json => "SheetJson",
+            _ => "SheetJson"
+        };
 
+        var projection = Builders<BsonDocument>.Projection.Include(fieldToInclude).Exclude("_id");
         var filter = Builders<BsonDocument>.Filter.Eq("Name", sheetName);
         var document = collection.Find(filter).Project(projection).FirstOrDefault();
 
-        if (document == null)
+        if (
+            document == null
+            || !document.Contains(fieldToInclude)
+            || document[fieldToInclude].IsBsonNull
+        )
         {
-            return "{}";
+            throw new KeyNotFoundException(sheetName);
         }
 
-        return document["Sheet"].ToJson(new JsonWriterSettings { OutputMode = JsonOutputMode.Strict });
+        return sheetFormat switch
+        {
+            SheetFormat.Csv
+                => await RetrieveFromGridFs(gridFs, document[fieldToInclude].AsObjectId),
+            _
+                => document[fieldToInclude]
+                    .ToJson(new JsonWriterSettings { OutputMode = JsonOutputMode.Strict })
+        };
+    }
+
+    private async Task<string> RetrieveFromGridFs(GridFSBucket gridFs, ObjectId fileId)
+    {
+        var fileBytes = await gridFs.DownloadAsBytesAsync(fileId);
+        return Encoding.UTF8.GetString(fileBytes);
     }
 }
diff --git a/Mimir/Services/MongoDBCollectionService.cs b/Mimir/Services/MongoDBCollectionService.cs
@@ -1,6 +1,6 @@
 using Microsoft.Extensions.Options;
-using MongoDB.Driver;
 using Mimir.Options;
+using MongoDB.Driver;
 
 namespace Mimir.Services;
 
@@ -10,8 +10,13 @@ public class MongoDBCollectionService(IOptions<DatabaseOption> databaseOption)
 
     public IMongoCollection<T> GetCollection<T>(string collectionName, string databaseName)
     {
-        var client = new MongoClient(_databaseOption.Value.ConnectionString);
-        var database = client.GetDatabase(databaseName);
+        var database = GetDatabase(databaseName);
         return database.GetCollection<T>(collectionName);
     }
+
+    public IMongoDatabase GetDatabase(string databaseName)
+    {
+        var client = new MongoClient(_databaseOption.Value.ConnectionString);
+        return client.GetDatabase(databaseName);
+    }
 }