feat(Tech: Assets): Split assets over subfolders

CHANGELOG.md

@@ -43,13 +43,18 @@ tech changes will usually be stripped from release notes for the public
     -   Note popouts for clients without edit access now show 'view source' instead of 'edit'
 -   I18n:
     -   Added 95% i18n for zh (except diceTool)
+-   [server] Assets:
+    -   Assets are no longer stored in a flat folder structure, but instead use a subpath based structure
+        -   An asset with hash `35eaef2e9a116aa152f7f161f1281411cb1e1375` is now stored as `assets/35/ea/35eaef2e9a116aa152f7f161f1281411cb1e1375`
 
 ### Removed
 
 -   Labels:
     -   As mentioned in the last 2 releases these were going to be removed
     -   I wasn't happy with the current implementation and they were causing more confusion than they were useful
     -   This also removes the Filter Tool
+-   .paa asset handling
+    -   This was no longer really maintained and the current frontend doesn't offer any support for it
 
 ### Fixed
 
@@ -78,7 +83,6 @@ When run as is, it will loop through all shapes and print out the ones with brok
 
 When run with the `delete` argument, it will remove the broken shapes from the DB. (i.e. `python remove-broken-shape-links.py delete`)
 
-
 ## [2024.3.0] - 2024-10-13
 
 ### Removed

client/src/assetManager/utils.ts

@@ -7,6 +7,12 @@ export function showIdName(dir: AssetId): string {
     return assetState.raw.idMap.get(dir)?.name ?? "";
 }
 
-export function getIdImageSrc(file: AssetId): string {
-    return baseAdjust("/static/assets/" + (assetState.raw.idMap.get(file)!.fileHash ?? ""));
+export function getImageSrcFromAssetId(file: AssetId, addBaseUrl = true): string {
+    const fileHash = assetState.raw.idMap.get(file)!.fileHash ?? "";
+    return getImageSrcFromHash(fileHash, addBaseUrl);
+}
+
+export function getImageSrcFromHash(fileHash: string, addBaseUrl = true): string {
+    const path = `/static/assets/${fileHash.slice(0, 2)}/${fileHash.slice(2, 4)}/${fileHash}`;
+    return addBaseUrl ? baseAdjust(path) : path;
 }

client/src/core/components/modals/AssetPicker.vue

@@ -6,7 +6,7 @@ import { assetSystem } from "../../../assetManager";
 import type { AssetId } from "../../../assetManager/models";
 import { socket } from "../../../assetManager/socket";
 import { assetState } from "../../../assetManager/state";
-import { getIdImageSrc, showIdName } from "../../../assetManager/utils";
+import { getImageSrcFromAssetId, showIdName } from "../../../assetManager/utils";
 import { i18n } from "../../../i18n";
 
 import Modal from "./Modal.vue";
@@ -82,7 +82,7 @@ function select(event: MouseEvent, inode: AssetId): void {
                         :class="{ 'inode-selected': assetState.reactive.selected.includes(file) }"
                         @click="select($event, file)"
                     >
-                        <img :src="getIdImageSrc(file)" width="50" alt="" />
+                        <img :src="getImageSrcFromAssetId(file)" width="50" alt="" />
                         <div class="title">{{ showIdName(file) }}</div>
                     </div>
                 </div>

client/src/dashboard/Assets.vue

@@ -14,7 +14,7 @@ import { sendCreateFolder, sendFolderGetByPath } from "../assetManager/emits";
 import type { AssetId } from "../assetManager/models";
 import { socket } from "../assetManager/socket";
 import { assetState } from "../assetManager/state";
-import { getIdImageSrc } from "../assetManager/utils";
+import { getImageSrcFromAssetId } from "../assetManager/utils";
 import { baseAdjust } from "../core/http";
 import { map } from "../core/iter";
 import { useModal } from "../core/plugins/modals/plugin";
@@ -415,7 +415,7 @@ async function showRenameUI(id: AssetId) : Promise<void> {
                 @dragstart="startDrag($event, file.id)"
             >
                 <font-awesome-icon v-if="isShared(file)" icon="user-tag" class="asset-link" />
-                <img :src="getIdImageSrc(file.id)" width="50" alt="" />
+                <img :src="getImageSrcFromAssetId(file.id)" width="50" alt="" />
                 <div
                      :contenteditable="file.id === currentRenameAsset"
                      class="title"

client/src/game/dropAsset.ts

@@ -1,5 +1,6 @@
 import { assetSystem } from "../assetManager";
 import { assetState } from "../assetManager/state";
+import { getImageSrcFromHash } from "../assetManager/utils";
 import { l2gx, l2gy, l2gz } from "../core/conversions";
 import { type GlobalPoint, toGP, Vector } from "../core/geometry";
 import { DEFAULT_GRID_SIZE, snapPointToGrid } from "../core/grid";
@@ -94,7 +95,10 @@ async function dropHelper(
 
         return;
     }
-    await dropAsset({ assetId: assetInfo.assetId, imageSource: `/static/assets/${assetInfo.assetHash}` }, location);
+    await dropAsset(
+        { assetId: assetInfo.assetId, imageSource: getImageSrcFromHash(assetInfo.assetHash, false) },
+        location,
+    );
 }
 
 export async function dropAsset(

client/src/game/layers/variants/map.ts

@@ -1,4 +1,4 @@
-import { baseAdjust } from "../../../core/http";
+import { getImageSrcFromHash } from "../../../assetManager/utils";
 import { FloorType } from "../../models/floor";
 import { floorSystem } from "../../systems/floors";
 import { positionState } from "../../systems/position/state";
@@ -56,7 +56,7 @@ export class MapLayer extends Layer {
         if (patternImage === undefined) {
             const img = new Image();
             patternImages[hash] = img;
-            img.src = baseAdjust("/static/assets/" + hash);
+            img.src = getImageSrcFromHash(hash);
             img.onload = () => {
                 this.invalidate(true);
             };

client/src/game/ui/menu/AssetNode.vue

@@ -1,7 +1,7 @@
 <script setup lang="ts">
 import { computed, reactive } from "vue";
 
-import { baseAdjust } from "../../../core/http";
+import { getImageSrcFromHash } from "../../../assetManager/utils";
 import { filter } from "../../../core/iter";
 import type { AssetFile, AssetListMap } from "../../../core/models/types";
 
@@ -64,7 +64,7 @@ function dragStart(event: DragEvent, assetHash: string, assetId: number): void {
         >
             {{ file.name }}
             <div v-if="state.hoveredHash == file.hash" class="preview">
-                <img class="asset-preview-image" :src="baseAdjust('/static/assets/' + file.hash)" alt="" />
+                <img class="asset-preview-image" :src="getImageSrcFromHash(file.hash)" alt="" />
             </div>
         </li>
     </ul>

client/src/game/ui/menu/Characters.vue

@@ -2,7 +2,7 @@
 import { computed, ref } from "vue";
 import { useI18n } from "vue-i18n";
 
-import { baseAdjust } from "../../../core/http";
+import { getImageSrcFromHash } from "../../../assetManager/utils";
 import { useModal } from "../../../core/plugins/modals/plugin";
 import { setCenterPosition } from "../../position";
 import { characterSystem } from "../../systems/characters";
@@ -71,7 +71,7 @@ async function remove(characterId: CharacterId): Promise<void> {
             </div>
             <div v-if="!characterState.reactive.characterIds.size">{{ t('game.ui.menu.MenuBar.no_characters') }}</div>
             <div v-if="charAsset !== undefined" class="preview">
-                <img class="asset-preview-image" :src="baseAdjust('/static/assets/' + charAsset.assetHash)" alt="" />
+                <img class="asset-preview-image" :src="getImageSrcFromHash(charAsset.assetHash)" alt="" />
             </div>
         </div>
     </div>

client/src/game/ui/settings/floor/PatternSettings.vue

@@ -2,7 +2,7 @@
 import { computed } from "vue";
 import { useI18n } from "vue-i18n";
 
-import { baseAdjust } from "../../../../core/http";
+import { getImageSrcFromHash } from "../../../../assetManager/utils";
 import { useModal } from "../../../../core/plugins/modals/plugin";
 import { getValue } from "../../../../core/utils";
 import { getPattern, patternToString } from "../../../layers/floor";
@@ -51,7 +51,7 @@ function setPatternData(data: { offsetX?: Event; offsetY?: Event; scaleX?: Event
         <img
             v-if="backgroundPattern.hash !== ''"
             alt="Pattern image preview"
-            :src="baseAdjust('/static/assets/' + backgroundPattern.hash)"
+            :src="getImageSrcFromHash(backgroundPattern.hash)"
             class="pattern-preview"
         />
         <font-awesome-icon id="set-pattern" icon="plus-square" title="Set a pattern" @click="setPatternImage" />

client/src/game/ui/settings/shape/PropertySettings.vue

@@ -2,6 +2,7 @@
 import { computed } from "vue";
 import { useI18n } from "vue-i18n";
 
+import { getImageSrcFromHash } from "../../../../assetManager/utils";
 import ColourPicker from "../../../../core/components/ColourPicker.vue";
 import ToggleGroup from "../../../../core/components/ToggleGroup.vue";
 import { NO_SYNC, SERVER_SYNC, SyncMode } from "../../../../core/models/types";
@@ -121,7 +122,7 @@ async function changeAsset(): Promise<void> {
     if (data === undefined || data.fileHash === undefined) return;
     const shape = getShape(activeShapeStore.state.id);
     if (shape === undefined || shape.type !== "assetrect") return;
-    (shape as Asset).setImage(`/static/assets/${data.fileHash}`, true);
+    (shape as Asset).setImage(getImageSrcFromHash(data.fileHash, false), true);
 }
 </script>
 

client/src/game/ui/settings/shape/VariantSwitcher.vue

@@ -2,6 +2,7 @@
 import { computed, toRef } from "vue";
 import { useToast } from "vue-toastification";
 
+import { getImageSrcFromHash } from "../../../../assetManager/utils";
 import { cloneP } from "../../../../core/geometry";
 import { InvalidationMode, SERVER_SYNC, SyncMode } from "../../../../core/models/types";
 import { useModal } from "../../../../core/plugins/modals/plugin";
@@ -71,7 +72,7 @@ async function addVariant(): Promise<void> {
     if (name === undefined) return;
 
     const newShape = await dropAsset(
-        { imageSource: `/static/assets/${asset.fileHash}`, assetId: asset.id },
+        { imageSource: getImageSrcFromHash(asset.fileHash, false), assetId: asset.id },
         shape.refPoint,
     );
     if (newShape === undefined) {

server/src/api/socket/asset_manager/core.py

@@ -1,15 +1,6 @@
 import hashlib
-import io
-import json
-import os
-import shutil
 import tarfile
-import tempfile
-import time
-from collections import defaultdict
-from pathlib import Path
-from typing import Any, Dict, List, Union
-from uuid import uuid4
+from typing import Any, List
 
 from aiohttp import web
 
@@ -23,7 +14,7 @@
 from ....state.asset import asset_state
 from ....state.game import game_state
 from ....transform.to_api.asset import transform_asset
-from ....utils import ASSETS_DIR, TEMP_DIR
+from ....utils import ASSETS_DIR, get_asset_hash_subpath
 from ...models.asset import (
     ApiAsset,
     ApiAssetAdd,
@@ -35,7 +26,6 @@
 )
 from ..constants import ASSET_NS, GAME_NS
 from .ddraft import handle_ddraft_file
-from .types import AssetDict, AssetExport
 
 
 async def update_live_game(user: User):
@@ -241,12 +231,13 @@ async def assetmgmt_rm(sid: str, data: int):
 
 
 def clean_filehash(file_hash: str):
-    if (ASSETS_DIR / file_hash).exists():
+    full_hash_path = get_asset_hash_subpath(file_hash)
+    if (ASSETS_DIR / full_hash_path).exists():
         no_assets = Asset.get_or_none(file_hash=file_hash) is None
-        no_shapes = AssetRect.get_or_none(src=f"/static/assets/{file_hash}") is None
+        no_shapes = AssetRect.get_or_none(src=full_hash_path) is None
         if no_assets and no_shapes:
             logger.info(f"No asset maps to file {file_hash}, removing from server")
-            (ASSETS_DIR / file_hash).unlink()
+            (ASSETS_DIR / full_hash_path).unlink()
 
 
 def cleanup_assets(assets: list[ApiAsset]):
@@ -272,47 +263,14 @@ def get_safe_members(members: List[tarfile.TarInfo]) -> List[tarfile.TarInfo]:
     return safe_members
 
 
-async def handle_paa_file(upload_data: ApiAssetUpload, data: bytes, sid: str):
-    with tempfile.TemporaryDirectory() as tmpdir:
-        with tarfile.open(fileobj=io.BytesIO(data), mode="r:bz2") as tar:
-            files = tarfile.TarInfo("files")
-            files.type = tarfile.DIRTYPE
-            # We need to explicitly list our members for security reasons
-            # this is upload data so people could upload malicious stuff that breaks out of the path etc
-            tar.extractall(path=tmpdir, members=get_safe_members(tar.getmembers()))
-
-        tmp_path = Path(tmpdir)
-        for asset in os.listdir(tmp_path / "files"):
-            if not (ASSETS_DIR / asset).exists():
-                shutil.move(str(tmp_path / "files" / asset), str(ASSETS_DIR / asset))
-
-        with open(tmp_path / "data") as json_data:
-            raw_assets: list[AssetDict] = json.load(json_data)
-
-    user = asset_state.get_user(sid)
-    parent_map: Dict[int, int] = defaultdict(lambda: upload_data.directory)
-
-    for raw_asset in raw_assets:
-        new_asset = Asset.create(
-            name=raw_asset["name"],
-            file_hash=raw_asset["file_hash"],
-            owner=user,
-            parent=parent_map[raw_asset["parent"]],
-            options=raw_asset["options"],
-        )
-        parent_map[raw_asset["id"]] = new_asset.id
-
-    await sio.emit(
-        "Asset.Import.Finish", upload_data.name, room=sid, namespace=ASSET_NS
-    )
-
-
 async def handle_regular_file(upload_data: ApiAssetUpload, data: bytes, sid: str):
     sh = hashlib.sha1(data)
     hashname = sh.hexdigest()
 
-    if not (ASSETS_DIR / hashname).exists():
-        with open(ASSETS_DIR / hashname, "wb") as f:
+    full_hash_path = get_asset_hash_subpath(hashname)
+
+    if not (ASSETS_DIR / full_hash_path).exists():
+        with open(ASSETS_DIR / full_hash_path, "wb") as f:
             f.write(data)
 
     user = asset_state.get_user(sid)
@@ -404,79 +362,11 @@ async def assetmgmt_upload(sid: str, raw_data: Any):
 
     return_data = None
     file_name = upload_data.name
-    if file_name.endswith(".paa"):
-        await handle_paa_file(upload_data, data, sid)
-    elif file_name.endswith(".dd2vtt"):
+    if file_name.endswith(".dd2vtt"):
         return_data = await handle_ddraft_file(upload_data, data, sid)
     else:
         return_data = await handle_regular_file(upload_data, data, sid)
 
     await update_live_game(user)
 
     return return_data
-
-
-def export_asset(asset: Union[ApiAsset, List[ApiAsset]], parent=-1) -> AssetExport:
-    file_hashes: List[str] = []
-    asset_info: List[ApiAsset] = []
-
-    if not isinstance(asset, list):
-        asset_dict = asset.copy(exclude={"children"})
-        asset_info.append(asset_dict)
-        if asset.fileHash is not None:
-            file_hashes.append(asset.fileHash)
-
-        children = asset.children or []
-        parent = asset.id
-    else:
-        children = asset
-
-    for child in children:
-        child_data = export_asset(child, parent)
-        file_hashes.extend(child_data["file_hashes"])
-        asset_info.extend(child_data["data"])
-    return {"file_hashes": file_hashes, "data": asset_info}
-
-
-@sio.on("Asset.Export", namespace=ASSET_NS)
-@auth.login_required(app, sio, "asset")
-async def assetmgmt_export(sid: str, selection: List[int]):
-    user = asset_state.get_user(sid)
-
-    full_selection = [
-        transform_asset(Asset.get_by_id(asset), user, children=True, recursive=True)
-        for asset in selection
-    ]
-
-    asset_data = export_asset(full_selection)
-    json_data = json.dumps(asset_data["data"])
-
-    data_tar_info = tarfile.TarInfo(
-        "data",
-    )
-    data_tar_info.size = len(json_data)
-    data_tar_info.mode = 0o755
-    data_tar_info.mtime = time.time()  # type: ignore
-
-    files_tar_info = tarfile.TarInfo("files")
-    files_tar_info.type = tarfile.DIRTYPE
-    files_tar_info.mode = 0o755
-    files_tar_info.mtime = time.time()  # type: ignore
-
-    uuid = uuid4()
-    os.makedirs(TEMP_DIR, exist_ok=True)
-    with tarfile.open(TEMP_DIR / f"{uuid}.paa", "w:bz2") as tar:
-        tar.addfile(data_tar_info, io.BytesIO(json_data.encode("utf-8")))
-        tar.addfile(files_tar_info)
-        for file_hash in asset_data["file_hashes"]:
-            try:
-                file_path = ASSETS_DIR / file_hash
-                info = tar.gettarinfo(str(file_path))
-                info.name = f"files/{file_hash}"
-                info.mtime = time.time()  # type: ignore
-                info.mode = 0o755
-                tar.addfile(info, open(file_path, "rb"))  # type: ignore
-            except FileNotFoundError:
-                pass
-
-    await sio.emit("Asset.Export.Finish", str(uuid), room=sid, namespace=ASSET_NS)