From b1958dd73db966c81da0acb8267e5aa12c88fdfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Sat, 17 Aug 2024 17:53:14 +0200 Subject: [PATCH 01/86] Add Nightly builds to README --- README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/README.md b/README.md index 66ffaf566..679fa085f 100644 --- a/README.md +++ b/README.md @@ -587,6 +587,29 @@ In Visual Studio, you can just use Build Solution and everything should build. J In .NET CLI, run `dotnet build` on the solution (`.sln`) level. +### Nightly builds + +Every **5AM UTC**, there is a new build from the `dev` branch published in GitHub Packages of the base GBX.NET package (no other packages are considered at the moment). + +GitHub Packages require you to provide access tokens to be able to read the packages, which is currently the unfortunate part, and I may migrate the nightly builds somewhere else due to this in the future. + +In GitHub settings -> Developer Settings at the bottom -> Personal access tokens -> Tokens (classic) -> Generate new token (classic), you can generate the access token. Setting Expiration to **No expiration** should be the best option, as you only have to enable **`read:packages`**, which if this token would leak, would not cause much harm anyway. Then go to `%appdata%/NuGet` and modify the `NuGet.Config` file to include that secret: +```xml + + + + + + + + + + + + +``` +Then you can select the GitHub package source and fetch nightly builds from there. More details [here](). + ## License GBX.NET 2 is licensed under multiple licenses, depending on the part of the project. Here are the licenses and their directories: From a3ddbd713421282bb8a8394f762f25251f536250 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Sat, 17 Aug 2024 17:54:00 +0200 Subject: [PATCH 02/86] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 679fa085f..860fd1227 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,7 @@ For any questions, open an issue, join the [GameBox Sandbox Discord server](http - [Asynchronous](#asynchronous) - [Benchmarks](#benchmarks) - [Build](#build) + - [Nightly builds](#nightly-builds) - [License](#license) - [Special thanks](#special-thanks) - [Alternative Gbx parsers](#alternative-gbx-parsers) From 3be63f4ea1e8a6b23dbae0ba0be428404583f6be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Mon, 19 Aug 2024 14:49:18 +0200 Subject: [PATCH 03/86] Fix nightly.yml using master hash --- .github/workflows/nightly.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 6fb2165b2..0c6fe7504 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -31,7 +31,8 @@ jobs: - name: Build run: | cd Src/GBX.NET - dotnet build -c Release --no-restore /p:ContinuousIntegrationBuild=true --version-suffix nightly.$(date +'%Y%m%d').${GITHUB_SHA::7} + git_hash=$(git rev-parse HEAD) + dotnet build -c Release --no-restore /p:ContinuousIntegrationBuild=true --version-suffix nightly.$(date +'%Y%m%d').${git_hash::7} - name: Publish nightly ${{ matrix.lib }} nupkg to github.com run: dotnet nuget push Src/GBX.NET/bin/Release/*.nupkg -k ${{ secrets.GITHUB_TOKEN }} -s https://nuget.pkg.github.com/bigbang1112/index.json --skip-duplicate From aed0171e642f485bfdf4149f252557a75d114741 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Wed, 4 Sep 2024 01:17:44 +0200 Subject: [PATCH 04/86] Use latest v4 artifact --- .github/workflows/publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index afdb1360b..cf6f4e271 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -50,7 +50,7 @@ jobs: run: dotnet test -c Release --no-build --verbosity normal - name: Upload a Build Artifact - uses: actions/upload-artifact@v4.3.1 + uses: actions/upload-artifact@v4 with: name: build path: Src/GBX.NET*/bin/Release/*.*nupkg @@ -83,7 +83,7 @@ jobs: - uses: actions/checkout@v4 - name: Download a Build Artifact - uses: actions/download-artifact@v4.1.4 + uses: actions/download-artifact@v4 with: name: build From ddf6b691ab965e480958e2297bdbffea2a23b2ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Sat, 7 Sep 2024 14:25:51 +0200 Subject: [PATCH 05/86] Update readme to July 2024 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 860fd1227..a5c310907 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ For any questions, open an issue, join the [GameBox Sandbox Discord server](http Many *essential* Gbx files from many games are supported: -- **Trackmania (2020)**, April 2024 update +- **Trackmania (2020)**, July 2024 update - **ManiaPlanet 4**(.1), TM2/SM - **Trackmania Turbo** - ManiaPlanet 3, TM2/SM From ae0a669289704a13f6d048fd9fc72c815634d8dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Thu, 12 Sep 2024 19:42:07 +0200 Subject: [PATCH 06/86] Update nightly.yml --- .github/workflows/nightly.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 0c6fe7504..4d6faa98a 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -34,5 +34,5 @@ jobs: git_hash=$(git rev-parse HEAD) dotnet build -c Release --no-restore /p:ContinuousIntegrationBuild=true --version-suffix nightly.$(date +'%Y%m%d').${git_hash::7} - - name: Publish nightly ${{ matrix.lib }} nupkg to github.com - run: dotnet nuget push Src/GBX.NET/bin/Release/*.nupkg -k ${{ secrets.GITHUB_TOKEN }} -s https://nuget.pkg.github.com/bigbang1112/index.json --skip-duplicate + - name: Publish nightly GBX.NET nupkg to nuget.gbx.tools + run: dotnet nuget push Src/GBX.NET/bin/Release/*.nupkg -k ${{ secrets.NUGET_GBXTOOLS_API_KEY }} -s https://nuget.gbx.tools/v3/index.json --skip-duplicate From 24b60c502713739d64ca5b2d708d735ce3a04dbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Thu, 12 Sep 2024 19:46:28 +0200 Subject: [PATCH 07/86] Update nightly build readme --- README.md | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index a5c310907..d4642c2d0 100644 --- a/README.md +++ b/README.md @@ -590,26 +590,20 @@ In .NET CLI, run `dotnet build` on the solution (`.sln`) level. ### Nightly builds -Every **5AM UTC**, there is a new build from the `dev` branch published in GitHub Packages of the base GBX.NET package (no other packages are considered at the moment). +Every **5AM UTC**, there is a new build from the `dev` branch published on https://nuget.gbx.tools/ of the base GBX.NET package (soon also GBX.NET.Tool*). -GitHub Packages require you to provide access tokens to be able to read the packages, which is currently the unfortunate part, and I may migrate the nightly builds somewhere else due to this in the future. - -In GitHub settings -> Developer Settings at the bottom -> Personal access tokens -> Tokens (classic) -> Generate new token (classic), you can generate the access token. Setting Expiration to **No expiration** should be the best option, as you only have to enable **`read:packages`**, which if this token would leak, would not cause much harm anyway. Then go to `%appdata%/NuGet` and modify the `NuGet.Config` file to include that secret: +Go to `%appdata%/NuGet` and modify the `NuGet.Config` file to include the package source: ```xml - + - - - - - - ``` -Then you can select the GitHub package source and fetch nightly builds from there. More details [here](). +Then you can select the `nuget.gbx.tools` package source and fetch nightly builds from there. + +> In the past, nightly builds were pushed to GitHub Packages which required you to provide access tokens to be able to read the packages. Nightly builds are no longer pushed to GitHub Packages. ## License From 61442c3b60252811a9ebf65ab7bac6623a0ae3b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Fri, 13 Sep 2024 04:14:02 +0200 Subject: [PATCH 08/86] Update nightly.yml --- .github/workflows/nightly.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 4d6faa98a..fee2ebe20 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -30,9 +30,16 @@ jobs: - name: Build run: | - cd Src/GBX.NET git_hash=$(git rev-parse HEAD) - dotnet build -c Release --no-restore /p:ContinuousIntegrationBuild=true --version-suffix nightly.$(date +'%Y%m%d').${git_hash::7} + dotnet build Src/GBX.NET -c Release --no-restore /p:ContinuousIntegrationBuild=true --version-suffix nightly.$(date +'%Y%m%d').${git_hash::7} + dotnet build Src/GBX.NET.Tool -c Release --no-restore /p:ContinuousIntegrationBuild=true --version-suffix nightly.$(date +'%Y%m%d').${git_hash::7} + dotnet build Src/GBX.NET.Tool.CLI -c Release --no-restore /p:ContinuousIntegrationBuild=true --version-suffix nightly.$(date +'%Y%m%d').${git_hash::7} - name: Publish nightly GBX.NET nupkg to nuget.gbx.tools run: dotnet nuget push Src/GBX.NET/bin/Release/*.nupkg -k ${{ secrets.NUGET_GBXTOOLS_API_KEY }} -s https://nuget.gbx.tools/v3/index.json --skip-duplicate + + - name: Publish nightly GBX.NET.Tool nupkg to nuget.gbx.tools + run: dotnet nuget push Src/GBX.NET.Tool/bin/Release/*.nupkg -k ${{ secrets.NUGET_GBXTOOLS_API_KEY }} -s https://nuget.gbx.tools/v3/index.json --skip-duplicate + + - name: Publish nightly GBX.NET.Tool.CLI nupkg to nuget.gbx.tools + run: dotnet nuget push Src/GBX.NET.Tool.CLI/bin/Release/*.nupkg -k ${{ secrets.NUGET_GBXTOOLS_API_KEY }} -s https://nuget.gbx.tools/v3/index.json --skip-duplicate From d16247248d24c81412b224d8cc3418bc22fa93e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Sat, 5 Oct 2024 01:37:55 +0200 Subject: [PATCH 09/86] Add missing lzo link in navigation --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index d4642c2d0..767fc99e2 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ For any questions, open an issue, join the [GameBox Sandbox Discord server](http - [Create a new GBX.NET project (lightweight)](#create-a-new-gbxnet-project-lightweight) - [Create a new GBX.NET project (Visual Studio Code)](#create-a-new-gbxnet-project-visual-studio-code) - [Create a new GBX.NET project (Visual Studio)](#create-a-new-gbxnet-project-visual-studio) +- **[IMPORTANT INFO about the LZO and Gbx compression](#important-info-about-the-lzo-and-gbx-compression)** - **[Usage (simple examples)](#usage-simple-examples)** - [Load a map and display block count per block name](#load-a-map-and-display-block-count-per-block-name) - [Modify and save a map](#modify-and-save-a-map) From a259821e9680fcdcd7abc9f3baca0898fe11fbfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Mon, 4 Nov 2024 23:16:10 +0100 Subject: [PATCH 10/86] Update nightly.yml --- .github/workflows/nightly.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index fee2ebe20..dfd984faa 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -24,6 +24,9 @@ jobs: dotnet-version: | 8.0.x 6.0.x + + - name: Install wasm-tools + run: dotnet workload install wasm-tools - name: Restore dependencies run: dotnet restore From 8477567ebeb9678ecde16215e213e18c1896ebfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Fri, 8 Nov 2024 04:15:06 +0100 Subject: [PATCH 11/86] Split Surf Triangle U02 into 4 bytes --- Src/GBX.NET/Engines/Plug/CPlugSurface.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Src/GBX.NET/Engines/Plug/CPlugSurface.cs b/Src/GBX.NET/Engines/Plug/CPlugSurface.cs index d6c7ef1f4..d3eca831e 100644 --- a/Src/GBX.NET/Engines/Plug/CPlugSurface.cs +++ b/Src/GBX.NET/Engines/Plug/CPlugSurface.cs @@ -242,7 +242,7 @@ public void Write(GbxWriter w, int version = 0) public readonly record struct CookedTriangle(Vec4 U01, Int3 U02, ushort U03, byte U04, byte U05); public readonly record struct OctreeCell(int U01, Vec3 U02, Vec3 U03, int U04); - public readonly record struct Triangle(Int3 U01, int U02); + public readonly record struct Triangle(Int3 U01, byte U02, byte U03, byte U04, byte U05); public readonly record struct AABBTreeCell(Vec3 U01, Vec3 U02, int U03); } From f14bdc1e37ef4e7087b51fc8c33c0fab5e5fde75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Fri, 8 Nov 2024 19:32:19 +0100 Subject: [PATCH 12/86] Add missing chunks to CPlugMaterialCustom --- .../Engines/Plug/CPlugMaterialCustom.chunkl | 43 ++++++++++++++++++- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/Src/GBX.NET/Engines/Plug/CPlugMaterialCustom.chunkl b/Src/GBX.NET/Engines/Plug/CPlugMaterialCustom.chunkl index 796fb00b4..15a453dc9 100644 --- a/Src/GBX.NET/Engines/Plug/CPlugMaterialCustom.chunkl +++ b/Src/GBX.NET/Engines/Plug/CPlugMaterialCustom.chunkl @@ -21,10 +21,41 @@ CPlugMaterialCustom 0x0903A000 short U03 short U04 +0x00F (skippable) + version + float + float + v1+ + float + v2+ + DefineNat[] + +0x011 (skippable) + int + +0x012 + CMwNod + +0x013 + version + Bitmap[] Textures (version: 1) + +0x014 + version + CBuffer[] + +0x015 + version + v1+ + int U01 + if U01 == 0 + string + string + archive Bitmap id int - int + CMwNod (external) v1+ int int @@ -33,4 +64,12 @@ archive GpuFx archive BitmapSkip id Name - bool \ No newline at end of file + bool + +archive DefineNat + id + int + +archive CBuffer + int + data \ No newline at end of file From 0f48966db45260b6e8d23193dcb2ec5d0da43297 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Fri, 8 Nov 2024 19:45:08 +0100 Subject: [PATCH 13/86] Display concrete gbx exception --- Src/GBX.NET/Serialization/GbxBodyReader.cs | 2 +- Tools/GbxExplorerOld/Client/Services/GbxService.cs | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/Src/GBX.NET/Serialization/GbxBodyReader.cs b/Src/GBX.NET/Serialization/GbxBodyReader.cs index d260cba71..b6d75b8bf 100644 --- a/Src/GBX.NET/Serialization/GbxBodyReader.cs +++ b/Src/GBX.NET/Serialization/GbxBodyReader.cs @@ -263,7 +263,7 @@ private void ReadMainNode(T node, GbxBody body, GbxReaderWriter rw) where T : throw; } - logger?.LogError(ex, "Failed to read main node (EXPLICIT). Exception was internally ignored and the node returns with partial data."); + logger?.LogWarning(ex, "Failed to read main node (EXPLICIT). Exception was internally ignored and the node returns with partial data."); } } } diff --git a/Tools/GbxExplorerOld/Client/Services/GbxService.cs b/Tools/GbxExplorerOld/Client/Services/GbxService.cs index 501b82157..c9acd99e6 100644 --- a/Tools/GbxExplorerOld/Client/Services/GbxService.cs +++ b/Tools/GbxExplorerOld/Client/Services/GbxService.cs @@ -84,9 +84,21 @@ public GbxService(IJSRuntime js, ISettingsService settings, ILogger logger) IgnoreExceptionsInBody = true, SafeSkippableChunks = true }, cancellationToken: cancellationToken); + + if (gbx.Body.Exception is not null) + { + _logger.LogError(gbx.Body.Exception, "{msg}", gbx.Body.Exception.Message); + } } - _logger.LogInformation("Gbx successfully imported!"); + if (gbx.Body.Exception is null) + { + _logger.LogInformation("Gbx successfully imported!"); + } + else + { + _logger.LogInformation("Gbx imported with exceptions."); + } return new GbxModel(file.Name, file.LastModified, gbxStream.ToArray(), sha256, gbx); } From d2b5a52cd7dc093bfd438aeaeda8c6f336ef04db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Fri, 8 Nov 2024 20:19:29 +0100 Subject: [PATCH 14/86] Add TM2 CGameCtnZone... features --- Src/GBX.NET/Engines/Game/CGameCtnZone.chunkl | 14 +++++++++++++- Src/GBX.NET/Engines/Game/CGameCtnZoneFlat.chunkl | 3 +++ .../Engines/Game/CGameCtnZoneFrontier.chunkl | 16 ++++++++++++++++ .../Engines/Game/CGameCtnZoneFusionInfo.chunkl | 9 +++++++++ 4 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 Src/GBX.NET/Engines/Game/CGameCtnZoneFusionInfo.chunkl diff --git a/Src/GBX.NET/Engines/Game/CGameCtnZone.chunkl b/Src/GBX.NET/Engines/Game/CGameCtnZone.chunkl index 82d449bbf..b9c3e6be2 100644 --- a/Src/GBX.NET/Engines/Game/CGameCtnZone.chunkl +++ b/Src/GBX.NET/Engines/Game/CGameCtnZone.chunkl @@ -12,4 +12,16 @@ CGameCtnZone 0x0305C000 bool 0x006 - bool + bool IsLargeZone + +0x007 + float VisualTopGroundHeight + +0x008 + int Height + +0x00B + id WaterId + +0x00C + id ForcedParentZoneFrontierId \ No newline at end of file diff --git a/Src/GBX.NET/Engines/Game/CGameCtnZoneFlat.chunkl b/Src/GBX.NET/Engines/Game/CGameCtnZoneFlat.chunkl index fe3cc8f78..3cad315db 100644 --- a/Src/GBX.NET/Engines/Game/CGameCtnZoneFlat.chunkl +++ b/Src/GBX.NET/Engines/Game/CGameCtnZoneFlat.chunkl @@ -11,3 +11,6 @@ CGameCtnZoneFlat 0x0305D000 int bool GroundOnly bool + +0x003 (skippable) + bool AutoSimplifyGenealogy \ No newline at end of file diff --git a/Src/GBX.NET/Engines/Game/CGameCtnZoneFrontier.chunkl b/Src/GBX.NET/Engines/Game/CGameCtnZoneFrontier.chunkl index 9977c3148..23b4cb220 100644 --- a/Src/GBX.NET/Engines/Game/CGameCtnZoneFrontier.chunkl +++ b/Src/GBX.NET/Engines/Game/CGameCtnZoneFrontier.chunkl @@ -5,3 +5,19 @@ CGameCtnZoneFrontier 0x0305E000 CGameCtnBlockInfoFrontier BlockInfoFrontier (external) id ParentZoneId id ChildZoneId + +0x002 + int + int BlockYOffsetFromParent + +0x003 + bool FrontierParentBorder_AcceptPylons + bool FrontierChildBorder_AcceptPylons + +0x004 (base: 0x003) + base + bool FrontierTransitionMiddle_AcceptPylons + bool FrontierStraightMiddle_AcceptPylons + +0x005 (skippable) + CGameCtnZoneFusionInfo[]_deprec CompatibleZones \ No newline at end of file diff --git a/Src/GBX.NET/Engines/Game/CGameCtnZoneFusionInfo.chunkl b/Src/GBX.NET/Engines/Game/CGameCtnZoneFusionInfo.chunkl new file mode 100644 index 000000000..c3f4deeaa --- /dev/null +++ b/Src/GBX.NET/Engines/Game/CGameCtnZoneFusionInfo.chunkl @@ -0,0 +1,9 @@ +CGameCtnZoneFusionInfo 0x03151000 + +0x000 + id CompatibleZoneId + int FusionType + +0x001 + version + id MergedZoneId \ No newline at end of file From 8ba35d0d5ccaebb11a70c1e05d1974b83df13baa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Fri, 8 Nov 2024 20:31:40 +0100 Subject: [PATCH 15/86] Implement CGameCtnReplayRecord 0x008 old cut keys --- Src/GBX.NET/Engines/Game/CGameCtnReplayRecord.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Src/GBX.NET/Engines/Game/CGameCtnReplayRecord.cs b/Src/GBX.NET/Engines/Game/CGameCtnReplayRecord.cs index 7cbdf7834..f93187b00 100644 --- a/Src/GBX.NET/Engines/Game/CGameCtnReplayRecord.cs +++ b/Src/GBX.NET/Engines/Game/CGameCtnReplayRecord.cs @@ -339,12 +339,16 @@ public override void Read(CGameCtnReplayRecord n, GbxReader r) public partial class Chunk03093008 { - public int U01; + public byte[][]? U01; public override void Read(CGameCtnReplayRecord n, GbxReader r) { n.Game = r.ReadString(); - U01 = r.ReadInt32(); // SOldCutKey2 + U01 = new byte[r.ReadInt32()][]; // SOldCutKey2 + for (var i = 0; i < U01.Length; i++) + { + U01[i] = r.ReadBytes(72); + } } } From 1c2a8a79c1697c0ee7f3e6327cb503bf348f64a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Fri, 8 Nov 2024 22:08:23 +0100 Subject: [PATCH 16/86] Support CGameCtnDecorationMood in TM2 --- .../Engines/Function/CFuncCloudsSolids.chunkl | 2 + .../Game/CGameCtnDecorationMood.chunkl | 44 +++++++++++++++++++ .../Plug/CPlugFxHdrScales_Tech3.chunkl | 9 ++++ .../Engines/Plug/CPlugFxLightning.chunkl | 2 + .../Engines/Plug/CPlugFxWindOnDecal.chunkl | 2 + .../Plug/CPlugFxWindOnTreeSprite.chunkl | 2 + .../Engines/Plug/CPlugMoodBlender.chunkl | 1 + 7 files changed, 62 insertions(+) create mode 100644 Src/GBX.NET/Engines/Function/CFuncCloudsSolids.chunkl create mode 100644 Src/GBX.NET/Engines/Plug/CPlugFxHdrScales_Tech3.chunkl create mode 100644 Src/GBX.NET/Engines/Plug/CPlugFxLightning.chunkl create mode 100644 Src/GBX.NET/Engines/Plug/CPlugFxWindOnDecal.chunkl create mode 100644 Src/GBX.NET/Engines/Plug/CPlugFxWindOnTreeSprite.chunkl create mode 100644 Src/GBX.NET/Engines/Plug/CPlugMoodBlender.chunkl diff --git a/Src/GBX.NET/Engines/Function/CFuncCloudsSolids.chunkl b/Src/GBX.NET/Engines/Function/CFuncCloudsSolids.chunkl new file mode 100644 index 000000000..440fa6a17 --- /dev/null +++ b/Src/GBX.NET/Engines/Function/CFuncCloudsSolids.chunkl @@ -0,0 +1,2 @@ +CFuncCloudsSolids 0x05047000 +- inherits: CFunc \ No newline at end of file diff --git a/Src/GBX.NET/Engines/Game/CGameCtnDecorationMood.chunkl b/Src/GBX.NET/Engines/Game/CGameCtnDecorationMood.chunkl index 057fc9c13..13a5d0502 100644 --- a/Src/GBX.NET/Engines/Game/CGameCtnDecorationMood.chunkl +++ b/Src/GBX.NET/Engines/Game/CGameCtnDecorationMood.chunkl @@ -27,3 +27,47 @@ CGameCtnDecorationMood 0x0303A000 0x005 CHmsAmbientOcc HmsAmbientOcc (external) + +0x006 + float SunMoonIntensity + +0x007 + float SunMoonIntensity + float LocalLightScale + +0x00C + vec3 + float ToneMapExposureStaticBase + int ToneMapFilmCurve + CFuncKeysReal ToneMapAutoExp_FidAvgLumiToKeyValue + +0x00F + version + vec3 Tech3SpecularFake_ExpScaleMax + vec2 Tech3SpecularLocal + vec3 Tech3Bloom + vec4 Tech3ToneMapAutoExp + CFuncKeysReal FxBloom_FidFuncIntensAtHdrNorm + v1+ + bool WaterReflectFakeCube + v2+ + CPlugFxHdrScales_Tech3 FxHdrScalesT3 + v3+ + CPlugMoodBlender MoodBlender (external) + +0x012 + version + float RemappedStartDayTime + bool IsNight + CPlugGameSkin Remapping (external) + string RemapFolder + bool EnableStars + CFuncCloudsSolids CloudsSolids + CPlugFxLightning FxLightning + CPlugFxWindOnDecal FxWindOnDecal + v1+ + CPlugFxWindOnTreeSprite FxWindOnTreeSprite + +0x013 + version + float EditorHelperHdrScale \ No newline at end of file diff --git a/Src/GBX.NET/Engines/Plug/CPlugFxHdrScales_Tech3.chunkl b/Src/GBX.NET/Engines/Plug/CPlugFxHdrScales_Tech3.chunkl new file mode 100644 index 000000000..b15a50fb4 --- /dev/null +++ b/Src/GBX.NET/Engines/Plug/CPlugFxHdrScales_Tech3.chunkl @@ -0,0 +1,9 @@ +CPlugFxHdrScales_Tech3 0x090F5000 + +0x000 + version + float + v1+ + float + float + float \ No newline at end of file diff --git a/Src/GBX.NET/Engines/Plug/CPlugFxLightning.chunkl b/Src/GBX.NET/Engines/Plug/CPlugFxLightning.chunkl new file mode 100644 index 000000000..7852ec3bb --- /dev/null +++ b/Src/GBX.NET/Engines/Plug/CPlugFxLightning.chunkl @@ -0,0 +1,2 @@ +CPlugFxLightning 0x090D8000 +- inherits: CPlug \ No newline at end of file diff --git a/Src/GBX.NET/Engines/Plug/CPlugFxWindOnDecal.chunkl b/Src/GBX.NET/Engines/Plug/CPlugFxWindOnDecal.chunkl new file mode 100644 index 000000000..24500be32 --- /dev/null +++ b/Src/GBX.NET/Engines/Plug/CPlugFxWindOnDecal.chunkl @@ -0,0 +1,2 @@ +CPlugFxWindOnDecal 0x090D9000 +- inherits: CPlug \ No newline at end of file diff --git a/Src/GBX.NET/Engines/Plug/CPlugFxWindOnTreeSprite.chunkl b/Src/GBX.NET/Engines/Plug/CPlugFxWindOnTreeSprite.chunkl new file mode 100644 index 000000000..9bbdc491b --- /dev/null +++ b/Src/GBX.NET/Engines/Plug/CPlugFxWindOnTreeSprite.chunkl @@ -0,0 +1,2 @@ +CPlugFxWindOnTreeSprite 0x090E1000 +- inherits: CPlug \ No newline at end of file diff --git a/Src/GBX.NET/Engines/Plug/CPlugMoodBlender.chunkl b/Src/GBX.NET/Engines/Plug/CPlugMoodBlender.chunkl new file mode 100644 index 000000000..ef3d621ed --- /dev/null +++ b/Src/GBX.NET/Engines/Plug/CPlugMoodBlender.chunkl @@ -0,0 +1 @@ +CPlugMoodBlender 0x0911A000 \ No newline at end of file From 430ede457adbc8d9cb9268e8f5cfa2e4edbf5754 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Fri, 8 Nov 2024 22:12:15 +0100 Subject: [PATCH 17/86] Fix log text wrap issue --- Tools/GbxExplorerOld/Client/wwwroot/css/layout.css | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Tools/GbxExplorerOld/Client/wwwroot/css/layout.css b/Tools/GbxExplorerOld/Client/wwwroot/css/layout.css index 54f15056e..3070af63c 100644 --- a/Tools/GbxExplorerOld/Client/wwwroot/css/layout.css +++ b/Tools/GbxExplorerOld/Client/wwwroot/css/layout.css @@ -139,6 +139,8 @@ main { .section-log { animation-delay: 0.3s; align-items: center; + white-space: nowrap; + overflow: hidden; } .section-copyright { From 6a0eed150d1058bdd7fc8f82036e58fc0d8669d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Fri, 8 Nov 2024 22:13:10 +0100 Subject: [PATCH 18/86] Fix externals of CGameCtnDecoration --- Src/GBX.NET/Engines/Game/CGameCtnDecoration.chunkl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Src/GBX.NET/Engines/Game/CGameCtnDecoration.chunkl b/Src/GBX.NET/Engines/Game/CGameCtnDecoration.chunkl index 0ecd27b8b..edf6f2a4b 100644 --- a/Src/GBX.NET/Engines/Game/CGameCtnDecoration.chunkl +++ b/Src/GBX.NET/Engines/Game/CGameCtnDecoration.chunkl @@ -31,14 +31,14 @@ CGameCtnDecoration 0x03038000 0x018 version - CPlugGameSkin VehicleFxSkin + CPlugGameSkin VehicleFxSkin (external) string VehicleFxFolder 0x019 version CGameCtnDecorationAudio DecoAudio (external) v1+ - CPlugSound DecoAudioAmbient + CPlugSound DecoAudioAmbient (external) 0x01A version From bdb65263a16da8aa5cc511ee42e8a4b5e50ff345 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Fri, 8 Nov 2024 22:33:21 +0100 Subject: [PATCH 19/86] Add some externals --- Src/GBX.NET/Engines/Game/CGameCtnDecorationMood.chunkl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Src/GBX.NET/Engines/Game/CGameCtnDecorationMood.chunkl b/Src/GBX.NET/Engines/Game/CGameCtnDecorationMood.chunkl index 13a5d0502..ea14f1f44 100644 --- a/Src/GBX.NET/Engines/Game/CGameCtnDecorationMood.chunkl +++ b/Src/GBX.NET/Engines/Game/CGameCtnDecorationMood.chunkl @@ -64,9 +64,9 @@ CGameCtnDecorationMood 0x0303A000 bool EnableStars CFuncCloudsSolids CloudsSolids CPlugFxLightning FxLightning - CPlugFxWindOnDecal FxWindOnDecal + CPlugFxWindOnDecal FxWindOnDecal (external) v1+ - CPlugFxWindOnTreeSprite FxWindOnTreeSprite + CPlugFxWindOnTreeSprite FxWindOnTreeSprite (external) 0x013 version From e8ee7e002c34a882fc6ff63b463592f4bf43e41c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Sat, 9 Nov 2024 04:41:43 +0100 Subject: [PATCH 20/86] Add TM2 write support for CGameCtnBlockUnitInfo --- .../Engines/Game/CGameCtnBlockUnitInfo.cs | 76 +++++++++++++++++-- 1 file changed, 68 insertions(+), 8 deletions(-) diff --git a/Src/GBX.NET/Engines/Game/CGameCtnBlockUnitInfo.cs b/Src/GBX.NET/Engines/Game/CGameCtnBlockUnitInfo.cs index 29f0254b9..654608323 100644 --- a/Src/GBX.NET/Engines/Game/CGameCtnBlockUnitInfo.cs +++ b/Src/GBX.NET/Engines/Game/CGameCtnBlockUnitInfo.cs @@ -42,13 +42,19 @@ public override void Read(CGameCtnBlockUnitInfo n, GbxReader r) U01[i] = new(node, file); } } + + public override void Write(CGameCtnBlockUnitInfo n, GbxWriter w) + { + foreach (var e in U01!) + { + w.WriteNodeRef(e.Node, e.File); + } + } } - public partial class Chunk0303600C + public partial class Chunk0303600C : IVersionable { - private int version; - - public int Version { get => version; set => version = value; } + public int Version { get; set; } public short? U01; public short? U02; @@ -57,12 +63,12 @@ public partial class Chunk0303600C public override void Read(CGameCtnBlockUnitInfo n, GbxReader r) { - var version = r.ReadInt32(); + Version = r.ReadInt32(); - if (version == 0) + if (Version == 0) { //rw.Int16(); - throw new ChunkVersionNotSupportedException(version); + throw new ChunkVersionNotSupportedException(Version); } var clipCountBits = r.ReadInt32(); @@ -81,7 +87,7 @@ public override void Read(CGameCtnBlockUnitInfo n, GbxReader r) n.ClipsTop = r.ReadArrayNodeRef(clipCountTop)!; n.ClipsBottom = r.ReadArrayNodeRef(clipCountBottom)!; - if (version >= 2) + if (Version >= 2) { U01 = r.ReadInt16(); U02 = r.ReadInt16(); @@ -92,6 +98,60 @@ public override void Read(CGameCtnBlockUnitInfo n, GbxReader r) U04 = r.ReadInt32(); } } + + public override void Write(CGameCtnBlockUnitInfo n, GbxWriter w) + { + w.Write(Version); + + var clipCountBits = (n.ClipsNorth?.Length ?? 0) + | (n.ClipsEast?.Length ?? 0) << 3 + | (n.ClipsSouth?.Length ?? 0) << 6 + | (n.ClipsWest?.Length ?? 0) << 9 + | (n.ClipsTop?.Length ?? 0) << 12 + | (n.ClipsBottom?.Length ?? 0) << 15; + w.Write(clipCountBits); + + foreach (var clip in n.ClipsNorth ?? []) + { + w.WriteNodeRef(clip); + } + + foreach (var clip in n.ClipsEast ?? []) + { + w.WriteNodeRef(clip); + } + + foreach (var clip in n.ClipsSouth ?? []) + { + w.WriteNodeRef(clip); + } + + foreach (var clip in n.ClipsWest ?? []) + { + w.WriteNodeRef(clip); + } + + foreach (var clip in n.ClipsTop ?? []) + { + w.WriteNodeRef(clip); + } + + foreach (var clip in n.ClipsBottom ?? []) + { + w.WriteNodeRef(clip); + } + + if (Version >= 2) + { + w.Write(U01.GetValueOrDefault()); + w.Write(U02.GetValueOrDefault()); + } + else + { + w.Write(U03.GetValueOrDefault()); + w.Write(U04.GetValueOrDefault()); + } + } } } From 0f1165d33a6e9e93913fc173481b2e5b5d0d1179 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Sun, 10 Nov 2024 18:33:58 +0100 Subject: [PATCH 21/86] Fix NbLaps in header, remove IsMultilap --- .../Engines/Game/CGameCtnChallenge.chunkl | 2 +- Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs | 69 +++++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/Src/GBX.NET/Engines/Game/CGameCtnChallenge.chunkl b/Src/GBX.NET/Engines/Game/CGameCtnChallenge.chunkl index 086931e24..7a696c6e4 100644 --- a/Src/GBX.NET/Engines/Game/CGameCtnChallenge.chunkl +++ b/Src/GBX.NET/Engines/Game/CGameCtnChallenge.chunkl @@ -18,7 +18,7 @@ CGameCtnChallenge 0x03043000 // A map. v5+ bool IsLapRace v6= - bool IsMultilap + bool v7+ int Mode v9+ diff --git a/Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs b/Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs index aac1dd690..8952ab859 100644 --- a/Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs +++ b/Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs @@ -771,6 +771,75 @@ public void RemoveAll() int IGameCtnChallengeMP4.RemoveBlock(Predicate match) => RemoveBlocks(match); int IGameCtnChallengeTM2020.RemoveBlock(Predicate match) => RemoveBlocks(match); + public partial class HeaderChunk03043002 + { + public override void ReadWrite(CGameCtnChallenge n, GbxReaderWriter rw) + { + rw.VersionByte(this); + if (Version <= 2) + { + rw.Ident(ref n.mapInfo); + rw.String(ref n.mapName); + } + rw.Boolean(ref U01); + if (Version >= 1) + { + rw.TimeInt32Nullable(ref n.bronzeTime); + rw.TimeInt32Nullable(ref n.silverTime); + rw.TimeInt32Nullable(ref n.goldTime); + rw.TimeInt32Nullable(ref n.authorTime); + if (Version == 2) + { + rw.Byte(ref U02); + } + if (Version >= 4) + { + rw.Int32(ref n.cost); + if (Version >= 5) + { + rw.Boolean(ref n.isLapRace); + if (Version == 6) + { + rw.Boolean(ref U03); + } + if (Version >= 7) + { + rw.EnumInt32(ref n.mode); + if (Version >= 9) + { + rw.Int32(ref U04); + if (Version >= 10) + { + rw.Int32(ref n.authorScore); + if (Version >= 11) + { + rw.EnumInt32(ref n.editor); + if (Version >= 12) + { + rw.Int32(ref U05); + if (Version >= 13) + { + rw.Int32(ref n.nbCheckpoints); + if (n.isLapRace) + { + rw.Int32(ref n.nbLaps); + } + else + { + n.nbLaps = rw.Int32(1); + } + } + } + } + } + } + } + } + } + } + } + } + [ChunkGenerationOptions(StructureKind = StructureKind.SeparateReadAndWrite)] public partial class HeaderChunk03043007 : IVersionable { From 87554746ca0e7b890a6684bd16a8a719db38613d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Sun, 10 Nov 2024 18:52:07 +0100 Subject: [PATCH 22/86] Fix writing changing state --- Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs b/Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs index 8952ab859..5be4c7d85 100644 --- a/Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs +++ b/Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs @@ -820,13 +820,13 @@ public override void ReadWrite(CGameCtnChallenge n, GbxReaderWriter rw) if (Version >= 13) { rw.Int32(ref n.nbCheckpoints); - if (n.isLapRace) + if (n.isLapRace || rw.Reader is not null) { rw.Int32(ref n.nbLaps); } else { - n.nbLaps = rw.Int32(1); + rw.Int32(1); } } } From e8cbffcb09000aca984a59dfc6b0a6a52a1f9650 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Sun, 10 Nov 2024 21:13:57 +0100 Subject: [PATCH 23/86] Modernize AdditionalMath class --- Src/GBX.NET/AdditionalMath.cs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/Src/GBX.NET/AdditionalMath.cs b/Src/GBX.NET/AdditionalMath.cs index 995e6a5c4..fb6bcec42 100644 --- a/Src/GBX.NET/AdditionalMath.cs +++ b/Src/GBX.NET/AdditionalMath.cs @@ -4,12 +4,20 @@ public static class AdditionalMath { public static float ToRadians(float degree) { +#if NET5_0_OR_GREATER || NETSTANDARD2_1_OR_GREATER + return degree / 180 * MathF.PI; +#else return (float)(degree / 180 * Math.PI); +#endif } public static float ToRadians(Direction direction) { +#if NET5_0_OR_GREATER || NETSTANDARD2_1_OR_GREATER + return (int)direction * MathF.PI / 2; +#else return (float)((int)direction * Math.PI / 2); +#endif } public static Vec3 ToRadians(Vec3 pitchYawRoll) @@ -19,7 +27,11 @@ public static Vec3 ToRadians(Vec3 pitchYawRoll) public static float ToDegrees(float radians) { +#if NET5_0_OR_GREATER || NETSTANDARD2_1_OR_GREATER + return radians / MathF.PI * 180; +#else return (float)(radians / Math.PI * 180); +#endif } public static Vec3 ToDegrees(Vec3 pitchYawRoll) @@ -29,11 +41,19 @@ public static Vec3 ToDegrees(Vec3 pitchYawRoll) public static Vec3 RotateAroundCenter(Vec3 point, Vec3 center, float radians) { +#if NET5_0_OR_GREATER || NETSTANDARD2_1_OR_GREATER + return ( + MathF.Cos(radians) * (point.X - center.X) - MathF.Sin(radians) * (point.Z - center.Z) + center.X, + point.Y, + MathF.Sin(radians) * (point.X - center.X) + MathF.Cos(radians) * (point.Z - center.Z) + center.Z + ); +#else return ( (float)(Math.Cos(radians) * (point.X - center.X) - Math.Sin(radians) * (point.Z - center.Z) + center.X), point.Y, (float)(Math.Sin(radians) * (point.X - center.X) + Math.Cos(radians) * (point.Z - center.Z) + center.Z) ); +#endif } public static float Lerp(float a, float b, float t) From 780e01d9fff8e51e0a0c7bbe294bf2c1e09d24fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Mon, 11 Nov 2024 03:40:01 +0100 Subject: [PATCH 24/86] Support replacing generated ReadWrite method --- .../SubGenerators/ClassDataSubGenerator.cs | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/Generators/GBX.NET.Generators/SubGenerators/ClassDataSubGenerator.cs b/Generators/GBX.NET.Generators/SubGenerators/ClassDataSubGenerator.cs index 3bad3814d..acbdc8fe9 100644 --- a/Generators/GBX.NET.Generators/SubGenerators/ClassDataSubGenerator.cs +++ b/Generators/GBX.NET.Generators/SubGenerators/ClassDataSubGenerator.cs @@ -1194,17 +1194,23 @@ private static void AppendChunkLine( } else if (structureKind.GetValueOrDefault() == 0) { - sb.AppendLine(); - sb.Append(" public override void ReadWrite("); - sb.Append(classInfo.Name); - sb.AppendLine(" n, GbxReaderWriter rw)"); - sb.AppendLine(" {"); + var doReadWriteMethod = !existingChunkMembers.OfType() + .Any(x => x.Name == "ReadWrite" && x.Parameters.Length == 2 && x.Parameters[0].Type.Name == classInfo.Name && x.Parameters[1].Type.Name == "GbxReaderWriter"); + + if (doReadWriteMethod) + { + sb.AppendLine(); + sb.Append(" public override void ReadWrite("); + sb.Append(classInfo.Name); + sb.AppendLine(" n, GbxReaderWriter rw)"); + sb.AppendLine(" {"); - var memberWriter = new MemberSerializationWriter( - sb, SerializationType.ReadWrite, archive: null, existingFields, existingProps, classInfo, classInfos, archiveInfos, autoProperty: false, context); - memberWriter.Append(indent: 3, chunk.ChunkLDefinition.Members); + var memberWriter = new MemberSerializationWriter( + sb, SerializationType.ReadWrite, archive: null, existingFields, existingProps, classInfo, classInfos, archiveInfos, autoProperty: false, context); + memberWriter.Append(indent: 3, chunk.ChunkLDefinition.Members); - sb.AppendLine(" }"); + sb.AppendLine(" }"); + } } } From 248d1cdf43b64d686ea46bd0276c6b866203af74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Mon, 11 Nov 2024 03:40:25 +0100 Subject: [PATCH 25/86] Add CPlugMaterialCustom 0x010 --- Src/GBX.NET/Engines/Plug/CPlugMaterialCustom.chunkl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Src/GBX.NET/Engines/Plug/CPlugMaterialCustom.chunkl b/Src/GBX.NET/Engines/Plug/CPlugMaterialCustom.chunkl index 15a453dc9..b0233ffd7 100644 --- a/Src/GBX.NET/Engines/Plug/CPlugMaterialCustom.chunkl +++ b/Src/GBX.NET/Engines/Plug/CPlugMaterialCustom.chunkl @@ -30,6 +30,9 @@ CPlugMaterialCustom 0x0903A000 v2+ DefineNat[] +0x010 + CMwNod + 0x011 (skippable) int From c8f1f000ce469109508985c6fed8fad57962b0ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Tue, 12 Nov 2024 17:01:52 +0100 Subject: [PATCH 26/86] Allow nulls --- Tools/GbxExplorerOld/Client/MemoryLogger.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/GbxExplorerOld/Client/MemoryLogger.cs b/Tools/GbxExplorerOld/Client/MemoryLogger.cs index d8d238a81..ecf73b4e9 100644 --- a/Tools/GbxExplorerOld/Client/MemoryLogger.cs +++ b/Tools/GbxExplorerOld/Client/MemoryLogger.cs @@ -34,7 +34,7 @@ public IDisposable BeginScope(TState state) continue; } - str = str.Replace($"{{{key}}}", val.ToString()); + str = str.Replace($"{{{key}}}", val?.ToString()); } return CurrentScope = new LogScopeModel(str, CurrentScope, this); From dcb6be648ac814762d1c79588ca6f1b0aa7f1157 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Tue, 12 Nov 2024 21:14:42 +0100 Subject: [PATCH 27/86] Update to .NET 9 --- .../GBX.NET.Benchmarks/GBX.NET.Benchmarks.csproj | 2 +- README.md | 8 +++++--- Samples/Advanced/ItemPacker/ItemPacker.csproj | 6 +++--- Src/GBX.NET.Crypto/GBX.NET.Crypto.csproj | 2 +- .../GBX.NET.Extensions.Hosting.csproj | 2 +- Src/GBX.NET.Hashing/GBX.NET.Hashing.csproj | 4 ++-- .../GBX.NET.Imaging.ImageSharp.csproj | 2 +- .../GBX.NET.Imaging.SkiaSharp.csproj | 4 ++-- Src/GBX.NET.Imaging/GBX.NET.Imaging.csproj | 4 ++-- Src/GBX.NET.Json/GBX.NET.Json.csproj | 2 +- Src/GBX.NET.LZO/GBX.NET.LZO.csproj | 2 +- Src/GBX.NET.Lua/GBX.NET.Lua.csproj | 2 +- .../GBX.NET.NewtonsoftJson.csproj | 2 +- Src/GBX.NET.PAK/GBX.NET.PAK.csproj | 2 +- Src/GBX.NET.Tool.CLI/GBX.NET.Tool.CLI.csproj | 4 ++-- Src/GBX.NET.Tool.Razor/GBX.NET.Tool.Razor.csproj | 2 +- .../GBX.NET.Tool.StandardIO.csproj | 2 +- Src/GBX.NET.Tool/GBX.NET.Tool.csproj | 2 +- Src/GBX.NET.ZLib/GBX.NET.ZLib.csproj | 2 +- Src/GBX.NET/GBX.NET.csproj | 6 +++--- Tests/GBX.NET.Tests/GBX.NET.Tests.csproj | 7 ++++--- Tools/GbxDiscordBot/GbxDiscordBot.csproj | 14 +++++++------- Tools/GbxExplorer/Client/GbxExplorer.Client.csproj | 8 ++++---- .../Component/GbxExplorer.Component.csproj | 4 ++-- .../GbxExplorer/Photino/GbxExplorer.Photino.csproj | 4 ++-- Tools/GbxExplorer/Server/GbxExplorer.Server.csproj | 4 ++-- Tools/GbxExplorer/Shared/GbxExplorer.Shared.csproj | 2 +- .../Client/GbxExplorerOld.Client.csproj | 11 ++++++----- .../Server/GbxExplorerOld.Server.csproj | 5 +++-- Tools/ParseRepeater/ParseRepeater.csproj | 6 +++--- 30 files changed, 66 insertions(+), 61 deletions(-) diff --git a/Benchmarks/GBX.NET.Benchmarks/GBX.NET.Benchmarks.csproj b/Benchmarks/GBX.NET.Benchmarks/GBX.NET.Benchmarks.csproj index 21b591d01..cf27cb617 100644 --- a/Benchmarks/GBX.NET.Benchmarks/GBX.NET.Benchmarks.csproj +++ b/Benchmarks/GBX.NET.Benchmarks/GBX.NET.Benchmarks.csproj @@ -2,7 +2,7 @@ Exe - net8.0 + net9.0 enable enable diff --git a/README.md b/README.md index 66ffaf566..74678c337 100644 --- a/README.md +++ b/README.md @@ -106,6 +106,7 @@ The goal is to also make it viable for NativeAOT. Due to the recently paced evolution of .NET, framework support has been limited only to a few ones compared to GBX.NET 1: +- **.NET 9** - .NET 8 - .NET 6 - .NET Standard 2.0 @@ -118,8 +119,8 @@ Using the NuGet packages is recommended. ### Create a new GBX.NET project (lightweight) -1. Install [.NET SDK 8](https://dotnet.microsoft.com/en-us/download/dotnet/8.0). - - Windows: [here](https://dotnet.microsoft.com/en-us/download) or `winget install Microsoft.DotNet.SDK.8` (make sure you have WinGet installed) +1. Install [.NET SDK 9](https://dotnet.microsoft.com/en-us/download/dotnet/9.0). + - Windows: [here](https://dotnet.microsoft.com/en-us/download) or `winget install Microsoft.DotNet.SDK.9` (make sure you have WinGet installed) - [Linux](https://learn.microsoft.com/en-us/dotnet/core/install/linux) (just SDK) 2. Create directory for your project (anywhere), **go inside it**. 3. Create new console project: `dotnet new console` @@ -575,11 +576,12 @@ TODO Make sure you have these framework SDKs available: +- **.NET 9** - .NET 8 - .NET 6 - .NET Standard 2.0 -**Visual Studio 2022** should be able to install those with default installation settings. Using Visual Studio 2019 will likely not work. +**Visual Studio 2022** should be able to install those with default installation settings. Using Visual Studio 2019 will not work. You should also have **.NET WebAssembly Build Tools** installed additionally to build the full solution. It is required for Gbx Explorer to work properly, as it uses native LZO implementation compiled into WebAssembly. diff --git a/Samples/Advanced/ItemPacker/ItemPacker.csproj b/Samples/Advanced/ItemPacker/ItemPacker.csproj index 49dfcf49d..ad4af522c 100644 --- a/Samples/Advanced/ItemPacker/ItemPacker.csproj +++ b/Samples/Advanced/ItemPacker/ItemPacker.csproj @@ -2,14 +2,14 @@ Exe - net8.0 + net9.0 enable enable - - + + diff --git a/Src/GBX.NET.Crypto/GBX.NET.Crypto.csproj b/Src/GBX.NET.Crypto/GBX.NET.Crypto.csproj index 869f33c73..a70bb7480 100644 --- a/Src/GBX.NET.Crypto/GBX.NET.Crypto.csproj +++ b/Src/GBX.NET.Crypto/GBX.NET.Crypto.csproj @@ -15,7 +15,7 @@ - net8.0;net6.0;netstandard2.0 + net9.0;net8.0;net6.0;netstandard2.0 12 enable enable diff --git a/Src/GBX.NET.Extensions.Hosting/GBX.NET.Extensions.Hosting.csproj b/Src/GBX.NET.Extensions.Hosting/GBX.NET.Extensions.Hosting.csproj index 721f666e5..ec2c2aaff 100644 --- a/Src/GBX.NET.Extensions.Hosting/GBX.NET.Extensions.Hosting.csproj +++ b/Src/GBX.NET.Extensions.Hosting/GBX.NET.Extensions.Hosting.csproj @@ -15,7 +15,7 @@ - net8.0;net6.0;netstandard2.0 + net9.0;net8.0;net6.0;netstandard2.0 12 enable enable diff --git a/Src/GBX.NET.Hashing/GBX.NET.Hashing.csproj b/Src/GBX.NET.Hashing/GBX.NET.Hashing.csproj index 1fa6d717f..278ff05f9 100644 --- a/Src/GBX.NET.Hashing/GBX.NET.Hashing.csproj +++ b/Src/GBX.NET.Hashing/GBX.NET.Hashing.csproj @@ -15,7 +15,7 @@ - net8.0;net6.0;netstandard2.0 + net9.0;net8.0;net6.0;netstandard2.0 12 enable enable @@ -39,7 +39,7 @@ - + all runtime; build; native; contentfiles; analyzers diff --git a/Src/GBX.NET.Imaging.ImageSharp/GBX.NET.Imaging.ImageSharp.csproj b/Src/GBX.NET.Imaging.ImageSharp/GBX.NET.Imaging.ImageSharp.csproj index c6705b878..60114f3e3 100644 --- a/Src/GBX.NET.Imaging.ImageSharp/GBX.NET.Imaging.ImageSharp.csproj +++ b/Src/GBX.NET.Imaging.ImageSharp/GBX.NET.Imaging.ImageSharp.csproj @@ -15,7 +15,7 @@ - net8.0;net6.0 + net9.0;net8.0;net6.0 12 enable enable diff --git a/Src/GBX.NET.Imaging.SkiaSharp/GBX.NET.Imaging.SkiaSharp.csproj b/Src/GBX.NET.Imaging.SkiaSharp/GBX.NET.Imaging.SkiaSharp.csproj index 6e3c6239d..bb98f46f1 100644 --- a/Src/GBX.NET.Imaging.SkiaSharp/GBX.NET.Imaging.SkiaSharp.csproj +++ b/Src/GBX.NET.Imaging.SkiaSharp/GBX.NET.Imaging.SkiaSharp.csproj @@ -15,7 +15,7 @@ - net8.0;net6.0;netstandard2.0 + net9.0;net8.0;net6.0;netstandard2.0 12 enable enable @@ -37,7 +37,7 @@ - + all runtime; build; native; contentfiles; analyzers diff --git a/Src/GBX.NET.Imaging/GBX.NET.Imaging.csproj b/Src/GBX.NET.Imaging/GBX.NET.Imaging.csproj index 434cb104c..493513b56 100644 --- a/Src/GBX.NET.Imaging/GBX.NET.Imaging.csproj +++ b/Src/GBX.NET.Imaging/GBX.NET.Imaging.csproj @@ -15,7 +15,7 @@ - net8.0-windows;net6.0-windows;netstandard2.0 + net9.0-windows;net8.0-windows;net6.0-windows;netstandard2.0 12 enable enable @@ -39,7 +39,7 @@ - + all runtime; build; native; contentfiles; analyzers diff --git a/Src/GBX.NET.Json/GBX.NET.Json.csproj b/Src/GBX.NET.Json/GBX.NET.Json.csproj index fd22ed041..b843a3fcf 100644 --- a/Src/GBX.NET.Json/GBX.NET.Json.csproj +++ b/Src/GBX.NET.Json/GBX.NET.Json.csproj @@ -15,7 +15,7 @@ - net8.0;net6.0 + net9.0;net8.0;net6.0 12 enable enable diff --git a/Src/GBX.NET.LZO/GBX.NET.LZO.csproj b/Src/GBX.NET.LZO/GBX.NET.LZO.csproj index b5c605908..baf9185f5 100644 --- a/Src/GBX.NET.LZO/GBX.NET.LZO.csproj +++ b/Src/GBX.NET.LZO/GBX.NET.LZO.csproj @@ -16,7 +16,7 @@ - net8.0;net6.0;netstandard2.0 + net9.0;net8.0;net6.0;netstandard2.0 12 enable enable diff --git a/Src/GBX.NET.Lua/GBX.NET.Lua.csproj b/Src/GBX.NET.Lua/GBX.NET.Lua.csproj index ca4153795..9d584c11b 100644 --- a/Src/GBX.NET.Lua/GBX.NET.Lua.csproj +++ b/Src/GBX.NET.Lua/GBX.NET.Lua.csproj @@ -15,7 +15,7 @@ - net8.0 + net9.0;net8.0 12 enable enable diff --git a/Src/GBX.NET.NewtonsoftJson/GBX.NET.NewtonsoftJson.csproj b/Src/GBX.NET.NewtonsoftJson/GBX.NET.NewtonsoftJson.csproj index fbb4cd8e6..051c0deb8 100644 --- a/Src/GBX.NET.NewtonsoftJson/GBX.NET.NewtonsoftJson.csproj +++ b/Src/GBX.NET.NewtonsoftJson/GBX.NET.NewtonsoftJson.csproj @@ -15,7 +15,7 @@ - net8.0;net6.0;netstandard2.0 + net9.0;net8.0;net6.0;netstandard2.0 12 enable enable diff --git a/Src/GBX.NET.PAK/GBX.NET.PAK.csproj b/Src/GBX.NET.PAK/GBX.NET.PAK.csproj index 32e428f8a..c5aba2dd4 100644 --- a/Src/GBX.NET.PAK/GBX.NET.PAK.csproj +++ b/Src/GBX.NET.PAK/GBX.NET.PAK.csproj @@ -15,7 +15,7 @@ - net8.0;net6.0;netstandard2.0 + net9.0;net8.0;net6.0;netstandard2.0 12 enable enable diff --git a/Src/GBX.NET.Tool.CLI/GBX.NET.Tool.CLI.csproj b/Src/GBX.NET.Tool.CLI/GBX.NET.Tool.CLI.csproj index 31239935a..cf6dca4d0 100644 --- a/Src/GBX.NET.Tool.CLI/GBX.NET.Tool.CLI.csproj +++ b/Src/GBX.NET.Tool.CLI/GBX.NET.Tool.CLI.csproj @@ -16,7 +16,7 @@ - net8.0 + net9.0;net8.0 12 enable enable @@ -48,7 +48,7 @@ - + diff --git a/Src/GBX.NET.Tool.Razor/GBX.NET.Tool.Razor.csproj b/Src/GBX.NET.Tool.Razor/GBX.NET.Tool.Razor.csproj index d326bae10..3298cb7ce 100644 --- a/Src/GBX.NET.Tool.Razor/GBX.NET.Tool.Razor.csproj +++ b/Src/GBX.NET.Tool.Razor/GBX.NET.Tool.Razor.csproj @@ -1,7 +1,7 @@ - net8.0 + net9.0;net8.0 enable enable diff --git a/Src/GBX.NET.Tool.StandardIO/GBX.NET.Tool.StandardIO.csproj b/Src/GBX.NET.Tool.StandardIO/GBX.NET.Tool.StandardIO.csproj index fa71b7ae6..5ed19f382 100644 --- a/Src/GBX.NET.Tool.StandardIO/GBX.NET.Tool.StandardIO.csproj +++ b/Src/GBX.NET.Tool.StandardIO/GBX.NET.Tool.StandardIO.csproj @@ -1,7 +1,7 @@  - net8.0 + net9.0;net8.0 enable enable diff --git a/Src/GBX.NET.Tool/GBX.NET.Tool.csproj b/Src/GBX.NET.Tool/GBX.NET.Tool.csproj index 404fc2fdb..494f43b56 100644 --- a/Src/GBX.NET.Tool/GBX.NET.Tool.csproj +++ b/Src/GBX.NET.Tool/GBX.NET.Tool.csproj @@ -15,7 +15,7 @@ - net8.0 + net9.0;net8.0 12 enable enable diff --git a/Src/GBX.NET.ZLib/GBX.NET.ZLib.csproj b/Src/GBX.NET.ZLib/GBX.NET.ZLib.csproj index aabc07fc7..5d6c14cb7 100644 --- a/Src/GBX.NET.ZLib/GBX.NET.ZLib.csproj +++ b/Src/GBX.NET.ZLib/GBX.NET.ZLib.csproj @@ -15,7 +15,7 @@ - net8.0;net6.0;netstandard2.0 + net9.0;net8.0;net6.0;netstandard2.0 12 enable enable diff --git a/Src/GBX.NET/GBX.NET.csproj b/Src/GBX.NET/GBX.NET.csproj index ce980bfc6..bbfd814e8 100644 --- a/Src/GBX.NET/GBX.NET.csproj +++ b/Src/GBX.NET/GBX.NET.csproj @@ -15,7 +15,7 @@ - net8.0;net6.0;netstandard2.0 + net9.0;net8.0;net6.0;netstandard2.0 12 enable enable @@ -40,7 +40,7 @@ - + all @@ -54,7 +54,7 @@ - + diff --git a/Tests/GBX.NET.Tests/GBX.NET.Tests.csproj b/Tests/GBX.NET.Tests/GBX.NET.Tests.csproj index bc3e50593..8f94ec6f9 100644 --- a/Tests/GBX.NET.Tests/GBX.NET.Tests.csproj +++ b/Tests/GBX.NET.Tests/GBX.NET.Tests.csproj @@ -1,7 +1,7 @@  - net8.0;net6.0 + net9.0;net8.0;net6.0 12 enable enable @@ -18,9 +18,10 @@ + - - + + runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Tools/GbxDiscordBot/GbxDiscordBot.csproj b/Tools/GbxDiscordBot/GbxDiscordBot.csproj index 72c8d9a82..fbe15b08f 100644 --- a/Tools/GbxDiscordBot/GbxDiscordBot.csproj +++ b/Tools/GbxDiscordBot/GbxDiscordBot.csproj @@ -2,7 +2,7 @@ Exe - net8.0 + net9.0 enable enable Linux @@ -12,13 +12,13 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive - - - + + + @@ -27,8 +27,8 @@ - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Tools/GbxExplorer/Client/GbxExplorer.Client.csproj b/Tools/GbxExplorer/Client/GbxExplorer.Client.csproj index 03d4efee4..60717da51 100644 --- a/Tools/GbxExplorer/Client/GbxExplorer.Client.csproj +++ b/Tools/GbxExplorer/Client/GbxExplorer.Client.csproj @@ -1,16 +1,16 @@  - net8.0 + net9.0 enable enable service-worker-assets.js - - - + + + diff --git a/Tools/GbxExplorer/Component/GbxExplorer.Component.csproj b/Tools/GbxExplorer/Component/GbxExplorer.Component.csproj index 375c7e06d..00822e1a3 100644 --- a/Tools/GbxExplorer/Component/GbxExplorer.Component.csproj +++ b/Tools/GbxExplorer/Component/GbxExplorer.Component.csproj @@ -1,7 +1,7 @@  - net8.0 + net9.0 enable enable true @@ -14,7 +14,7 @@ - + diff --git a/Tools/GbxExplorer/Photino/GbxExplorer.Photino.csproj b/Tools/GbxExplorer/Photino/GbxExplorer.Photino.csproj index 362ac3e47..df545df8a 100644 --- a/Tools/GbxExplorer/Photino/GbxExplorer.Photino.csproj +++ b/Tools/GbxExplorer/Photino/GbxExplorer.Photino.csproj @@ -3,7 +3,7 @@ GbxExplorer Exe - net8.0 + net9.0 enable enable true @@ -20,7 +20,7 @@ - + false diff --git a/Tools/GbxExplorer/Server/GbxExplorer.Server.csproj b/Tools/GbxExplorer/Server/GbxExplorer.Server.csproj index 64c7fd964..b9b328150 100644 --- a/Tools/GbxExplorer/Server/GbxExplorer.Server.csproj +++ b/Tools/GbxExplorer/Server/GbxExplorer.Server.csproj @@ -1,7 +1,7 @@ - net8.0 + net9.0 enable enable true @@ -20,7 +20,7 @@ - + diff --git a/Tools/GbxExplorer/Shared/GbxExplorer.Shared.csproj b/Tools/GbxExplorer/Shared/GbxExplorer.Shared.csproj index 9af12db85..165d6aa56 100644 --- a/Tools/GbxExplorer/Shared/GbxExplorer.Shared.csproj +++ b/Tools/GbxExplorer/Shared/GbxExplorer.Shared.csproj @@ -1,7 +1,7 @@ - net8.0 + net9.0 enable enable true diff --git a/Tools/GbxExplorerOld/Client/GbxExplorerOld.Client.csproj b/Tools/GbxExplorerOld/Client/GbxExplorerOld.Client.csproj index d3c3b4cd1..9f66b5c99 100644 --- a/Tools/GbxExplorerOld/Client/GbxExplorerOld.Client.csproj +++ b/Tools/GbxExplorerOld/Client/GbxExplorerOld.Client.csproj @@ -1,7 +1,7 @@  - net8.0 + net9.0 enable enable service-worker-assets.js @@ -21,13 +21,14 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive - - - + + + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Tools/GbxExplorerOld/Server/GbxExplorerOld.Server.csproj b/Tools/GbxExplorerOld/Server/GbxExplorerOld.Server.csproj index 9abde7343..e67d629a3 100644 --- a/Tools/GbxExplorerOld/Server/GbxExplorerOld.Server.csproj +++ b/Tools/GbxExplorerOld/Server/GbxExplorerOld.Server.csproj @@ -1,7 +1,7 @@  - net8.0 + net9.0 enable enable 14bef366-38a3-418f-93d8-e19bd6a41e2a @@ -11,7 +11,8 @@ - + + diff --git a/Tools/ParseRepeater/ParseRepeater.csproj b/Tools/ParseRepeater/ParseRepeater.csproj index 0f4e6ddc0..9e932642f 100644 --- a/Tools/ParseRepeater/ParseRepeater.csproj +++ b/Tools/ParseRepeater/ParseRepeater.csproj @@ -2,7 +2,7 @@ Exe - net8.0 + net9.0 enable enable true @@ -10,8 +10,8 @@ - - + + From d3e360177458ce36bfab450570b5d5d20b622e6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Tue, 12 Nov 2024 21:17:28 +0100 Subject: [PATCH 28/86] Update workflows to acknowledge .NET 9 --- .github/workflows/ci.yml | 3 ++- .github/workflows/deploy-explorer-pages.yml | 3 ++- .github/workflows/nightly.yml | 1 + .github/workflows/publish.yml | 1 + 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c572ac26c..05c8f810f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,8 +25,9 @@ jobs: uses: actions/setup-dotnet@v4 with: dotnet-version: | - 6.0.x + 9.0.x 8.0.x + 6.0.x - name: Restore dependencies run: dotnet restore diff --git a/.github/workflows/deploy-explorer-pages.yml b/.github/workflows/deploy-explorer-pages.yml index a891922fe..3f0e91015 100644 --- a/.github/workflows/deploy-explorer-pages.yml +++ b/.github/workflows/deploy-explorer-pages.yml @@ -35,8 +35,9 @@ jobs: uses: actions/setup-dotnet@v4 with: dotnet-version: | - 6.0.x + 9.0.x 8.0.x + 6.0.x - name: Install wasm-tools run: dotnet workload install wasm-tools diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index dfd984faa..75989aa62 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -22,6 +22,7 @@ jobs: uses: actions/setup-dotnet@v4 with: dotnet-version: | + 9.0.x 8.0.x 6.0.x diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index efb421877..aa06e5ba5 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -37,6 +37,7 @@ jobs: uses: actions/setup-dotnet@v4 with: dotnet-version: | + 9.0.x 8.0.x 6.0.x From 7a2ad85e790d0d5e12afb86662323372cbbe4253 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Tue, 12 Nov 2024 21:20:05 +0100 Subject: [PATCH 29/86] Update docker image versions --- Tools/GbxDiscordBot/Dockerfile | 4 ++-- Tools/GbxExplorer/Server/Dockerfile | 4 ++-- Tools/GbxExplorerOld/Server/Dockerfile | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Tools/GbxDiscordBot/Dockerfile b/Tools/GbxDiscordBot/Dockerfile index 6a3503b98..5405ed4fa 100644 --- a/Tools/GbxDiscordBot/Dockerfile +++ b/Tools/GbxDiscordBot/Dockerfile @@ -1,4 +1,4 @@ -FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:8.0-alpine AS build +FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:9.0-alpine AS build ARG BUILD_CONFIGURATION=Release ARG TARGETARCH=x64 ARG APPNAME=GbxDiscordBot @@ -27,7 +27,7 @@ RUN dotnet publish -c $BUILD_CONFIGURATION -a $TARGETARCH -o /app --no-restore # final stage/image -FROM mcr.microsoft.com/dotnet/runtime:8.0-alpine +FROM mcr.microsoft.com/dotnet/runtime:9.0-alpine ENV \ DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=false \ LC_ALL=en_US.UTF-8 \ diff --git a/Tools/GbxExplorer/Server/Dockerfile b/Tools/GbxExplorer/Server/Dockerfile index 6f66e676e..b91e067d2 100644 --- a/Tools/GbxExplorer/Server/Dockerfile +++ b/Tools/GbxExplorer/Server/Dockerfile @@ -1,4 +1,4 @@ -FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:8.0-alpine AS build +FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:9.0-alpine AS build ARG BUILD_CONFIGURATION=Release ARG TARGETARCH=x64 WORKDIR /src @@ -33,7 +33,7 @@ RUN dotnet publish -c $BUILD_CONFIGURATION -a $TARGETARCH -o /app --no-restore - # final stage/image -FROM mcr.microsoft.com/dotnet/runtime-deps:8.0-alpine +FROM mcr.microsoft.com/dotnet/runtime-deps:9.0-alpine EXPOSE 8080 EXPOSE 8081 WORKDIR /app diff --git a/Tools/GbxExplorerOld/Server/Dockerfile b/Tools/GbxExplorerOld/Server/Dockerfile index 689878821..54ff26e00 100644 --- a/Tools/GbxExplorerOld/Server/Dockerfile +++ b/Tools/GbxExplorerOld/Server/Dockerfile @@ -1,4 +1,4 @@ -FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:8.0 AS build +FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:9.0 AS build ARG BUILD_CONFIGURATION=Release ARG TARGETARCH=x64 RUN apt-get update && apt-get install -y git python3 @@ -31,7 +31,7 @@ RUN dotnet publish Tools/GbxExplorerOld/Server -c $BUILD_CONFIGURATION -a $TARGE # final stage/image -FROM mcr.microsoft.com/dotnet/aspnet:8.0 +FROM mcr.microsoft.com/dotnet/aspnet:9.0 EXPOSE 8080 EXPOSE 8081 WORKDIR /app From 73141257cf87f36731cffaec8f85282e5539cbb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Tue, 12 Nov 2024 21:24:56 +0100 Subject: [PATCH 30/86] July to October --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9d9edad3b..ed8a81686 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ For any questions, open an issue, join the [GameBox Sandbox Discord server](http Many *essential* Gbx files from many games are supported: -- **Trackmania (2020)**, July 2024 update +- **Trackmania (2020)**, October 2024 update - **ManiaPlanet 4**(.1), TM2/SM - **Trackmania Turbo** - ManiaPlanet 3, TM2/SM From 83c0a9efb6fa17f313ac31ba9fe55af61cfadea5 Mon Sep 17 00:00:00 2001 From: GreffMASTER <57036833+GreffMASTER@users.noreply.github.com> Date: Tue, 12 Nov 2024 21:55:54 +0100 Subject: [PATCH 31/86] Filled in some of the Vehicle Tunings parameters --- .../Plug/CPlugVehicleCarPhyTuning.chunkl | 427 ++++++++++-------- 1 file changed, 250 insertions(+), 177 deletions(-) diff --git a/Src/GBX.NET/Engines/Plug/CPlugVehicleCarPhyTuning.chunkl b/Src/GBX.NET/Engines/Plug/CPlugVehicleCarPhyTuning.chunkl index 6a84fe879..5042d28b8 100644 --- a/Src/GBX.NET/Engines/Plug/CPlugVehicleCarPhyTuning.chunkl +++ b/Src/GBX.NET/Engines/Plug/CPlugVehicleCarPhyTuning.chunkl @@ -3,58 +3,70 @@ CPlugVehicleCarPhyTuning 0x090ED000 0x000 float - float - float - float - float - float - float - float - float - float - float - float + float SteerRadiusMin + float SteerRadiusCoef + float MaxSpeed + float AbsorbingValKi + float AbsorbingValKa + float AbsorbingValMin + float AbsorbingValMax + float AbsorbingValRest + float RelSpeedMultCoef + float + float DebugAbsorbCoef 0x001 id Name 0x002 - float - float - float + float Mass + float CMAftForce + float CMDownUp 0x004 - bool + bool IsFakeEngine 0x005 + float GravityCoef + +0x006 + float + float + float float 0x007 - float + float SteerSpeed + +0x008 + float ReverseMaxSpeed + float TurboBoost 0x009 - float + float BrakeBase + +0x00A + float LinearFluidFrictionCoef + float AngularFluidFrictionCoef1 0x00B - float - float - float - float + float InertiaMass + Vec3 InertiaHalfDiag + +0x00C + bool 0x00D - int + int TireMaterial 0x00E - int + int ShockModel 0x010 - int - float + int SteerModel + float SteerGroundTorque -0x013 - float - float - float +0x011 float float float @@ -65,15 +77,14 @@ CPlugVehicleCarPhyTuning 0x090ED000 float float -0x014 - float - float +0x012 + float SoundEngineVolume + float SoundSkidSandVolume + float SoundImpactVolume + float SoundBodyImpact_Hard_Threshold -0x016 - float +0x013 float - -0x017 float float float @@ -82,22 +93,50 @@ CPlugVehicleCarPhyTuning 0x090ED000 float float float - -0x018 float float float -0x019 - float +0x014 float float + +0x015 + float SoundEngineVolume + float SoundSkidSandVolume + float SoundSkidConcreteVolume + float SoundImpactVolume + float SoundBodyImpact_Hard_Threshold + +0x016 + float DisplaySteerRadiusMin + float DisplaySteerRadiusCoef + +0x017 + float DecrepitudeImpactVal + float DecrepitudeCoef + float VibrationPeriodSpeedCoef float float float float float +0x018 + float LinearFluidFrictionCoef + float AngularFluidFrictionCoef1 + float AngularFluidFrictionCoef2 + +0x019 + float BodyFrictionCoef + float BodyFrictionCoef_Metal + float BodyRestCoef_Metal + float BodyRestCoef + float WheelFrictionCoef_Concrete + float WheelRestCoef_Concrete + float WheelFrictionCoef_Metal + float WheelRestCoef_Metal + 0x01A float @@ -105,11 +144,11 @@ CPlugVehicleCarPhyTuning 0x090ED000 float 0x01D - float - float + float GravityCoefAir + float RolloverAxial 0x01E - int + int AirControlDuration 0x01F float @@ -117,38 +156,45 @@ CPlugVehicleCarPhyTuning 0x090ED000 float 0x020 + float SteerAngleMax + float SlipAngleForceMax + float SlipAngleForceCoef1 + float SlipAngleForceCoef2 + +0x021 float - float + +0x022 float float 0x023 - float - float + float AngularSpeedYImpulseScale + float SteerLowSpeed 0x024 - CFuncKeysReal Accel + CFuncKeysReal AccelCurve 0x026 - float - float - float - float + float LateralSlopeAdherenceMin + float LateralSlopeAdherenceMax + float AxialSlopeAdherenceMin + float AxialSlopeAdherenceMax 0x027 - float - float + float SteerGroundTorqueSlippingCoef + float MaxSideFrictionBlendCoef 0x028 float CFuncKeysReal MaxSideFriction + float MaxSideFrictionSliding float - float - float - float + float GroundSlowDownBase + float AbsorbTension 0x029 - float + float SideFriction1 CFuncKeysReal RolloverLateral 0x02A @@ -156,222 +202,236 @@ CPlugVehicleCarPhyTuning 0x090ED000 0x02B CFuncKeysReal SteerSlowDown - float - float - int + float SideFriction2 + float RubberBallElasticity + int SteerSlowDownFadeInDuration 0x02C - CFuncKeysReal - float - float - float - float - float + CFuncKeysReal RolloverLateralFromAngle + float MaxAngularSpeedYAirControl + float BrakeCoef + float BrakeMax + float BrakeMaxDynamic + float GroundSlowDownCoef 0x02D - int + int SteerSlowDownFadeOutDuration 0x02E - float + float SteerSlowDownCoef float +0x02F + int TurboDuration + 0x030 CFuncKeysReal SteerDriveTorque 0x031 - float - float + float LimitToMaxSpeedForce + float SlopeSpeedGainLimit 0x032 - float - float - float - float - float - float - float - float + float SoundEngineVolume + float SoundSkidSandVolume + float SoundSkidConcreteVolume + float SoundImpactVolume + float SoundWheelImpact_Hard_Threshold + float SoundWheelImpact_Soft_Threshold + float SoundBodyImpact_Hard_Threshold + float SoundBodyImpact_Soft_Threshold 0x033 - float + float AngularSpeedClamp 0x034 - float + float LinearSpeed2PositiveDeltaMax 0x035 - bool + bool NoSteerSlowDownWhenSlipping 0x036 - float - float - float - CFuncKeysReal SteerRadius + float M4LateralFrictionSquareForce + float M4LateralFrictionTorque + float M4LateralFrictionForce + CFuncKeysReal M4SteerRadiusFromSpeed 0x037 - CFuncKeysReal MaxFrictionTorque + CFuncKeysReal M4MaxFrictionTorqueFromSpeed 0x038 - float - CFuncKeysReal MaxFrictionForce - float - float + float M4MaxFrictionTorqueWhenSlippingCoef + CFuncKeysReal M4MaxFrictionForceFromSpeed + float M4MaxFrictionForceWhenSlipping + float M4SteerRadiusWhenSlippingCoef 0x039 - float + float M4LateralFrictionSquareTorque float 0x03A - float - float + float M4SlipAngleSpeed + float M4SteerRadiusCoefFromSlipAngle 0x03B - float + float M4LeaveSlippingSpeed 0x03C - float + float M4SteerAngleWhenSlippingMax 0x03D - CFuncKeysReal SlippingAccel + CFuncKeysReal M5SlippingAccelCurve 0x03E - int + int M5LateralConstantSlowDownDuration 0x03F - CFuncKeysReal SteerCoef + CFuncKeysReal M5SteerCoefFromSpeed 0x040 - CFuncKeysReal SmoothInputSteerDuration + CFuncKeysReal M5SmoothInputSteerDurationFromSpeed 0x041 - float - int + float M5MaxAxialRolloverTorque + int M5KeepSlidingAccelDuration 0x042 - int + int M5KeepSteerSlowDownDurarion 0x043 - int + int M5KeepNoSteerSlowDownWhenSlippingDuration 0x044 - float + float M5AccelSlipCoefMax -0x046 +0x045 + float WaterGravity float float + float WaterReboundMinHSpeed float - CFuncKeysReal - CFuncKeysReal - CFuncKeysReal + +0x046 + float WaterGravity + float WaterReboundMinHSpeed + float WaterBumpMinSpeed + CFuncKeysReal WaterBumpSlowDownFromSpeedRatio + CFuncKeysReal WaterFrictionFromSpeed + CFuncKeysReal WaterReboundFromSpeedRatio 0x047 - float + float WaterAngularFriction 0x048 - CFuncKeysReal + CFuncKeysReal WaterSplashFromSpeed 0x049 - CFuncKeysReal Modulation + CFuncKeysReal ModulationFromWheelCompression -0x04D +0x04A float + float M6InertialMass + +0x04D + float M6ForceEpsilon 0x04E - float - float + float M6InertialTorqueModualtionX + float M6InertialTorqueModulationZ 0x04F - float - CFuncKeysReal BurnoutRadius + float M6MaxSpeed4Burnout + CFuncKeysReal M6BurnoutRadius 0x051 - float + float M6BurnoutFricMod 0x052 - CFuncKeysReal RolloverLateralRatio + CFuncKeysReal M6RolloverLateralFromSpeedRatio 0x053 - float + float M6BurnoutCenterForceCoeff2 0x056 - float - float - float + float M6BurnoutRpmAcc + float M6AirRpmAcc + float M6AirRpmDeadening 0x057 - float[] + float[] M6RpmWantedOnGearUp 0x058 - float + float M6RpmLossCoefOnGearUp 0x059 - float - float - float - float + float M6RpmGainCoefOnGearDown + float M6SpeedLimitPositiveForTakeOffFront + float M6RpmGainOnTakeOff + float M6RpmLossOnTakeOffFinished 0x05A - float - float - float + float M6SpeedLimitPositiveForTakeOffRear + float M6SpeedLimitNegForTakeOffFront + float M6SpeedLimitNegTakeOffRear 0x05B - CFuncKeysReal - float + CFuncKeysReal BrakeHeatSpeedFromFBrake + float BrakeCoolingSpeed 0x05C - float - float + float M6BrakeMaxRear + float M6BrakeMaxDynamicRear 0x05D - float - float - CFuncKeysReal RearGearAccel - CFuncKeysReal LateralSpeed - float - float - float - float - float - float - float - float - float - int - float - float - int - float - float - float - float - CFuncKeysReal BurnoutRollover - CFuncKeysReal DonutRollover - float - float - float - float - float - float - float[] - float[] - float[] - int + float M6InertialMass + float M6MaxDiffBtwnPropulsionAndSpeed + CFuncKeysReal AccelCurveRearGear + CFuncKeysReal M6BurnoutLateralSpeed + float M6BurnoutLateralSpeedCoeff + float M6MinSpeed4Burnout + float M6BurnoutSteerCoeff + float M6BurnoutCenterForceCoeff + float M6BurnoutRadiusMax + float M6BurnoutLateralSpeedMax + float M6BurnoutSteerCoeff2 + float M6BurnoutSteerCoeff3 + float M6BrakeModulationWhenSlipping + int M6BurnoutDuration + float M6BurnoutAccMod + float M6AfterBurnoutAccMod + int M6AfterBurnoutDuration + float M6FrictionModulationWhenSlipNBrake + float M6BrakeSmokeIntensity + float M6BurnoutSmokeVelocity + float M6BurnoutSmokeIntensity + CFuncKeysReal M6BurnoutRolloverFromSpeed + CFuncKeysReal M6DonutRolloverFromSpeed + float M6MaxDiffBtwGroundNormal + float M6BurnoutWheelAngularRotation + float M6MaxPosAngle4Burnout + float M6MaxNegAngle4Burnout + float M6AfterBurnoutImpulse + float M6MaxRpm + float[] M6GearRatio + float[] M6MaxRPM + float[] M6MinRPM + int SteerDurationBeforeSteerSlowDown 0x05E - CFuncKeysReal + CFuncKeysReal AirControlZCoefFromAngularSpeed 0x05F - float + float WaterAngularFrictionSq 0x060 - float - float - float - int - int + float ReverseMaxSpeed + float TurboBoost + float Turbo2Boost + int TurboDuration + int Turbo2Duration 0x061 - CMwNod + CFuncKeysReal VisualSteerAngleFromSpeed 0x062 float @@ -979,4 +1039,17 @@ CPlugVehicleCarPhyTuning 0x090ED000 int? int? -archive Keys \ No newline at end of file +enum EShockModel + Demo01 + Demo02 + Demo03 + +enum ESteerModel + Steer01 + Steer02 + Steer03 + Steer04 + Steer05 + Steer06 + +archive Keys From 54d02cd0c52f6fcbb597e44bf3ae57b20d2c945f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Tue, 12 Nov 2024 22:35:49 +0100 Subject: [PATCH 32/86] Add CFuncKeysReal 0x000 --- Src/GBX.NET/Engines/Function/CFuncKeysReal.chunkl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Src/GBX.NET/Engines/Function/CFuncKeysReal.chunkl b/Src/GBX.NET/Engines/Function/CFuncKeysReal.chunkl index d16540e97..c8055a641 100644 --- a/Src/GBX.NET/Engines/Function/CFuncKeysReal.chunkl +++ b/Src/GBX.NET/Engines/Function/CFuncKeysReal.chunkl @@ -1,6 +1,9 @@ CFuncKeysReal 0x0501A000 - inherits: CFuncKeys +0x000 + float[] Ys + 0x001 float[] Ys int RealInterp From 5e94fbba356e017ddcfbde31708a6fb375f7cd99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Tue, 12 Nov 2024 22:55:46 +0100 Subject: [PATCH 33/86] Generalize questionable node --- Src/GBX.NET/Engines/Plug/CPlugVehicleCarPhyTuning.chunkl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Src/GBX.NET/Engines/Plug/CPlugVehicleCarPhyTuning.chunkl b/Src/GBX.NET/Engines/Plug/CPlugVehicleCarPhyTuning.chunkl index 5042d28b8..c569d956a 100644 --- a/Src/GBX.NET/Engines/Plug/CPlugVehicleCarPhyTuning.chunkl +++ b/Src/GBX.NET/Engines/Plug/CPlugVehicleCarPhyTuning.chunkl @@ -806,7 +806,7 @@ CPlugVehicleCarPhyTuning 0x090ED000 0x099 version - CFuncKeysReal + CMwNod 0x09A version From 8cc8da25d45826a40ba69b813f3ced7a8901bb16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Tue, 12 Nov 2024 22:59:03 +0100 Subject: [PATCH 34/86] Generealize another questionable node --- Src/GBX.NET/Engines/Plug/CPlugVehicleCarPhyTuning.chunkl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Src/GBX.NET/Engines/Plug/CPlugVehicleCarPhyTuning.chunkl b/Src/GBX.NET/Engines/Plug/CPlugVehicleCarPhyTuning.chunkl index c569d956a..ec04aac8a 100644 --- a/Src/GBX.NET/Engines/Plug/CPlugVehicleCarPhyTuning.chunkl +++ b/Src/GBX.NET/Engines/Plug/CPlugVehicleCarPhyTuning.chunkl @@ -789,7 +789,7 @@ CPlugVehicleCarPhyTuning 0x090ED000 float bool v2+ - CFuncKeysReal + CMwNod 0x098 version From 93c22eb1de222a95c8865c20c08011d6d91abbb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Wed, 13 Nov 2024 01:16:23 +0100 Subject: [PATCH 35/86] Add note of from which context class ID is checked --- Src/GBX.NET/Serialization/GbxReader.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Src/GBX.NET/Serialization/GbxReader.cs b/Src/GBX.NET/Serialization/GbxReader.cs index d50990522..647d74b9f 100644 --- a/Src/GBX.NET/Serialization/GbxReader.cs +++ b/Src/GBX.NET/Serialization/GbxReader.cs @@ -1116,7 +1116,7 @@ public PackDesc ReadPackDesc() } #if NET8_0_OR_GREATER - var node = T.New(classId) ?? throw new Exception($"Unknown class ID: 0x{classId:X8} ({ClassManager.GetName(classId) ?? "unknown class name"})"); + var node = T.New(classId) ?? throw new Exception($"Unknown class ID (within {typeof(T).Name}): 0x{classId:X8} ({ClassManager.GetName(classId) ?? "unknown class name"})"); #else var node = ClassManager.New(classId) ?? throw new Exception($"Unknown class ID: 0x{classId:X8} ({ClassManager.GetName(classId) ?? "unknown class name"})"); #endif @@ -1279,7 +1279,7 @@ private IClass ReadNode(IClass node) var classId = ClassManager.Wrap(rawClassId); #if NET8_0_OR_GREATER - var node = T.New(classId) ?? throw new Exception($"Unknown class ID: 0x{classId:X8} ({ClassManager.GetName(classId) ?? "unknown class name"})"); + var node = T.New(classId) ?? throw new Exception($"Unknown class ID (within {typeof(T).Name}): 0x{classId:X8} ({ClassManager.GetName(classId) ?? "unknown class name"})"); #else var node = ClassManager.New(classId) ?? throw new Exception($"Unknown class ID: 0x{classId:X8} ({ClassManager.GetName(classId) ?? "unknown class name"})"); #endif From 8b1a087d82c66bb57a2136282ae22cce9f1363ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Tue, 19 Nov 2024 23:48:18 +0100 Subject: [PATCH 36/86] Fix old profile chunk 0x011 being written incorrectly (issue with bool arrays) --- Src/GBX.NET/Engines/Game/CGamePlayerProfile.chunkl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Src/GBX.NET/Engines/Game/CGamePlayerProfile.chunkl b/Src/GBX.NET/Engines/Game/CGamePlayerProfile.chunkl index e8d265688..4ba866a59 100644 --- a/Src/GBX.NET/Engines/Game/CGamePlayerProfile.chunkl +++ b/Src/GBX.NET/Engines/Game/CGamePlayerProfile.chunkl @@ -228,7 +228,7 @@ bool 0x240B5011 (not-remapped) - bool[] + int[] 0x240B5012 (not-remapped) bool From 16ed3bfef1edbc5c2f356e3e674d04845dc4a80c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Wed, 20 Nov 2024 10:38:27 +0100 Subject: [PATCH 37/86] Swap remap --- Resources/Wrap.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Resources/Wrap.txt b/Resources/Wrap.txt index eb8f0f632..ba0a381bc 100644 --- a/Resources/Wrap.txt +++ b/Resources/Wrap.txt @@ -98,6 +98,7 @@ 24047000 03047000 24048000 030AF000 24049000 030E0000 +240B5000 0308C000 2404A000 0308C000 2404D000 0308A000 2404E000 03002000 @@ -165,7 +166,6 @@ 240AD000 03096000 240AE000 03097000 240B2000 030D7000 -240B5000 0308C000 240B8000 03098000 240B9000 030B6000 240BA000 030B7000 From 46c8b000c38d88f93b930f48927849ae75980083 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Wed, 20 Nov 2024 10:47:40 +0100 Subject: [PATCH 38/86] Fix Unwrap of profile --- Resources/Unwrap.txt | 2 +- Resources/Wrap.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Resources/Unwrap.txt b/Resources/Unwrap.txt index 7d4975b50..80e5fdc24 100644 --- a/Resources/Unwrap.txt +++ b/Resources/Unwrap.txt @@ -54,7 +54,7 @@ 03088000 240C7000 03089000 240C0000 0308A000 2404D000 -0308C000 2404A000 +0308C000 240B5000 0308D000 24034000 0308E000 240A0000 0308F000 24039000 diff --git a/Resources/Wrap.txt b/Resources/Wrap.txt index ba0a381bc..eb8f0f632 100644 --- a/Resources/Wrap.txt +++ b/Resources/Wrap.txt @@ -98,7 +98,6 @@ 24047000 03047000 24048000 030AF000 24049000 030E0000 -240B5000 0308C000 2404A000 0308C000 2404D000 0308A000 2404E000 03002000 @@ -166,6 +165,7 @@ 240AD000 03096000 240AE000 03097000 240B2000 030D7000 +240B5000 0308C000 240B8000 03098000 240B9000 030B6000 240BA000 030B7000 From 45b0fc72eb6b5ead6d986cd83b75f1ae601500c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Wed, 20 Nov 2024 18:54:11 +0100 Subject: [PATCH 39/86] Start porting GBX.NET.PAK --- GBX.NET.sln | 7 + Src/GBX.NET.PAK/Blowfish.cs | 423 ++++++++++++++++++ Src/GBX.NET.PAK/BlowfishStream.cs | 112 +++++ .../Exceptions/NotAPakException.cs | 9 + Src/GBX.NET.PAK/GBX.NET.PAK.csproj | 12 + Src/GBX.NET.PAK/GbxReaderExtensions.cs | 32 ++ Src/GBX.NET.PAK/Pak.cs | 200 ++++++++- Src/GBX.NET.PAK/PakFile.cs | 37 ++ Src/GBX.NET.PAK/PakFolder.cs | 3 + Src/GBX.NET.PAK/PakList.cs | 2 +- Src/GBX.NET.ZLib/ZLib.cs | 5 + Src/GBX.NET/Engines/MwFoundations/CMwNod.cs | 27 ++ Src/GBX.NET/Extensions/IZLib.cs | 1 + Src/GBX.NET/Serialization/GbxReader.cs | 2 +- Src/GBX.NET/Serialization/IEncryptedStream.cs | 12 + Tools/PakToZip/PakToZip.csproj | 16 + Tools/PakToZip/Program.cs | 21 + 17 files changed, 914 insertions(+), 7 deletions(-) create mode 100644 Src/GBX.NET.PAK/Blowfish.cs create mode 100644 Src/GBX.NET.PAK/BlowfishStream.cs create mode 100644 Src/GBX.NET.PAK/Exceptions/NotAPakException.cs create mode 100644 Src/GBX.NET.PAK/GbxReaderExtensions.cs create mode 100644 Src/GBX.NET.PAK/PakFile.cs create mode 100644 Src/GBX.NET.PAK/PakFolder.cs create mode 100644 Src/GBX.NET/Serialization/IEncryptedStream.cs create mode 100644 Tools/PakToZip/PakToZip.csproj create mode 100644 Tools/PakToZip/Program.cs diff --git a/GBX.NET.sln b/GBX.NET.sln index 0612dc811..0e0214bb2 100644 --- a/GBX.NET.sln +++ b/GBX.NET.sln @@ -174,6 +174,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MuxCryptSwitch", "Tools\Mux EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GBX.NET.Tool.StandardIO", "Src\GBX.NET.Tool.StandardIO\GBX.NET.Tool.StandardIO.csproj", "{55BD1330-7431-41A8-90EB-DEB20D9EB981}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PakToZip", "Tools\PakToZip\PakToZip.csproj", "{47A58E42-EBBC-48D0-B4B1-C25DFD074506}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -412,6 +414,10 @@ Global {55BD1330-7431-41A8-90EB-DEB20D9EB981}.Debug|Any CPU.Build.0 = Debug|Any CPU {55BD1330-7431-41A8-90EB-DEB20D9EB981}.Release|Any CPU.ActiveCfg = Release|Any CPU {55BD1330-7431-41A8-90EB-DEB20D9EB981}.Release|Any CPU.Build.0 = Release|Any CPU + {47A58E42-EBBC-48D0-B4B1-C25DFD074506}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {47A58E42-EBBC-48D0-B4B1-C25DFD074506}.Debug|Any CPU.Build.0 = Debug|Any CPU + {47A58E42-EBBC-48D0-B4B1-C25DFD074506}.Release|Any CPU.ActiveCfg = Release|Any CPU + {47A58E42-EBBC-48D0-B4B1-C25DFD074506}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -481,6 +487,7 @@ Global {89F6FFE5-E32A-4713-8D63-98D070A17DC1} = {80DCE6B7-4BD9-415C-B053-92B059D7C938} {0B5609C6-7B10-4E8D-B41C-35CC1A2532A4} = {F3336145-FDA9-4517-AEDC-7F4C4D526ECB} {55BD1330-7431-41A8-90EB-DEB20D9EB981} = {80DCE6B7-4BD9-415C-B053-92B059D7C938} + {47A58E42-EBBC-48D0-B4B1-C25DFD074506} = {F3336145-FDA9-4517-AEDC-7F4C4D526ECB} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {8EA2F0DE-BA72-486D-AB3A-9320C0CE5CFD} diff --git a/Src/GBX.NET.PAK/Blowfish.cs b/Src/GBX.NET.PAK/Blowfish.cs new file mode 100644 index 000000000..bdf1237f8 --- /dev/null +++ b/Src/GBX.NET.PAK/Blowfish.cs @@ -0,0 +1,423 @@ +/**************************************************************************** + | + | Copyright (c) 2007 Novell, Inc. + | All Rights Reserved. + | + | This program is free software; you can redistribute it and/or + | modify it under the terms of version 2 of the GNU General Public License as + | published by the Free Software Foundation. + | + | This program is distributed in the hope that it will be useful, + | but WITHOUT ANY WARRANTY; without even the implied warranty of + | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + | GNU General Public License for more details. + | + | You should have received a copy of the GNU General Public License + | along with this program; if not, contact Novell, Inc. + | + | To contact Novell about this file by physical or electronic mail, + | you may find current contact information at www.novell.com + | + | Author: Russ Young + | Thanks to: Bruce Schneier / Counterpane Labs + | for the Blowfish encryption algorithm and + | reference implementation. http://www.schneier.com/blowfish.html + |***************************************************************************/ + +namespace GBX.NET.PAK; + +/// +/// Class that provides blowfish encryption. +/// +internal sealed class Blowfish +{ + private const int N = 16; + + private readonly uint[] P = [ + 0x243f6a88, 0x85a308d3, 0x13198a2e, 0x03707344, 0xa4093822, 0x299f31d0, + 0x082efa98, 0xec4e6c89, 0x452821e6, 0x38d01377, 0xbe5466cf, 0x34e90c6c, + 0xc0ac29b7, 0xc97c50dd, 0x3f84d5b5, 0xb5470917, 0x9216d5d9, 0x8979fb1b + ]; + private readonly uint[,] S = + { + { + 0xd1310ba6, 0x98dfb5ac, 0x2ffd72db, 0xd01adfb7, 0xb8e1afed, 0x6a267e96, + 0xba7c9045, 0xf12c7f99, 0x24a19947, 0xb3916cf7, 0x0801f2e2, 0x858efc16, + 0x636920d8, 0x71574e69, 0xa458fea3, 0xf4933d7e, 0x0d95748f, 0x728eb658, + 0x718bcd58, 0x82154aee, 0x7b54a41d, 0xc25a59b5, 0x9c30d539, 0x2af26013, + 0xc5d1b023, 0x286085f0, 0xca417918, 0xb8db38ef, 0x8e79dcb0, 0x603a180e, + 0x6c9e0e8b, 0xb01e8a3e, 0xd71577c1, 0xbd314b27, 0x78af2fda, 0x55605c60, + 0xe65525f3, 0xaa55ab94, 0x57489862, 0x63e81440, 0x55ca396a, 0x2aab10b6, + 0xb4cc5c34, 0x1141e8ce, 0xa15486af, 0x7c72e993, 0xb3ee1411, 0x636fbc2a, + 0x2ba9c55d, 0x741831f6, 0xce5c3e16, 0x9b87931e, 0xafd6ba33, 0x6c24cf5c, + 0x7a325381, 0x28958677, 0x3b8f4898, 0x6b4bb9af, 0xc4bfe81b, 0x66282193, + 0x61d809cc, 0xfb21a991, 0x487cac60, 0x5dec8032, 0xef845d5d, 0xe98575b1, + 0xdc262302, 0xeb651b88, 0x23893e81, 0xd396acc5, 0x0f6d6ff3, 0x83f44239, + 0x2e0b4482, 0xa4842004, 0x69c8f04a, 0x9e1f9b5e, 0x21c66842, 0xf6e96c9a, + 0x670c9c61, 0xabd388f0, 0x6a51a0d2, 0xd8542f68, 0x960fa728, 0xab5133a3, + 0x6eef0b6c, 0x137a3be4, 0xba3bf050, 0x7efb2a98, 0xa1f1651d, 0x39af0176, + 0x66ca593e, 0x82430e88, 0x8cee8619, 0x456f9fb4, 0x7d84a5c3, 0x3b8b5ebe, + 0xe06f75d8, 0x85c12073, 0x401a449f, 0x56c16aa6, 0x4ed3aa62, 0x363f7706, + 0x1bfedf72, 0x429b023d, 0x37d0d724, 0xd00a1248, 0xdb0fead3, 0x49f1c09b, + 0x075372c9, 0x80991b7b, 0x25d479d8, 0xf6e8def7, 0xe3fe501a, 0xb6794c3b, + 0x976ce0bd, 0x04c006ba, 0xc1a94fb6, 0x409f60c4, 0x5e5c9ec2, 0x196a2463, + 0x68fb6faf, 0x3e6c53b5, 0x1339b2eb, 0x3b52ec6f, 0x6dfc511f, 0x9b30952c, + 0xcc814544, 0xaf5ebd09, 0xbee3d004, 0xde334afd, 0x660f2807, 0x192e4bb3, + 0xc0cba857, 0x45c8740f, 0xd20b5f39, 0xb9d3fbdb, 0x5579c0bd, 0x1a60320a, + 0xd6a100c6, 0x402c7279, 0x679f25fe, 0xfb1fa3cc, 0x8ea5e9f8, 0xdb3222f8, + 0x3c7516df, 0xfd616b15, 0x2f501ec8, 0xad0552ab, 0x323db5fa, 0xfd238760, + 0x53317b48, 0x3e00df82, 0x9e5c57bb, 0xca6f8ca0, 0x1a87562e, 0xdf1769db, + 0xd542a8f6, 0x287effc3, 0xac6732c6, 0x8c4f5573, 0x695b27b0, 0xbbca58c8, + 0xe1ffa35d, 0xb8f011a0, 0x10fa3d98, 0xfd2183b8, 0x4afcb56c, 0x2dd1d35b, + 0x9a53e479, 0xb6f84565, 0xd28e49bc, 0x4bfb9790, 0xe1ddf2da, 0xa4cb7e33, + 0x62fb1341, 0xcee4c6e8, 0xef20cada, 0x36774c01, 0xd07e9efe, 0x2bf11fb4, + 0x95dbda4d, 0xae909198, 0xeaad8e71, 0x6b93d5a0, 0xd08ed1d0, 0xafc725e0, + 0x8e3c5b2f, 0x8e7594b7, 0x8ff6e2fb, 0xf2122b64, 0x8888b812, 0x900df01c, + 0x4fad5ea0, 0x688fc31c, 0xd1cff191, 0xb3a8c1ad, 0x2f2f2218, 0xbe0e1777, + 0xea752dfe, 0x8b021fa1, 0xe5a0cc0f, 0xb56f74e8, 0x18acf3d6, 0xce89e299, + 0xb4a84fe0, 0xfd13e0b7, 0x7cc43b81, 0xd2ada8d9, 0x165fa266, 0x80957705, + 0x93cc7314, 0x211a1477, 0xe6ad2065, 0x77b5fa86, 0xc75442f5, 0xfb9d35cf, + 0xebcdaf0c, 0x7b3e89a0, 0xd6411bd3, 0xae1e7e49, 0x00250e2d, 0x2071b35e, + 0x226800bb, 0x57b8e0af, 0x2464369b, 0xf009b91e, 0x5563911d, 0x59dfa6aa, + 0x78c14389, 0xd95a537f, 0x207d5ba2, 0x02e5b9c5, 0x83260376, 0x6295cfa9, + 0x11c81968, 0x4e734a41, 0xb3472dca, 0x7b14a94a, 0x1b510052, 0x9a532915, + 0xd60f573f, 0xbc9bc6e4, 0x2b60a476, 0x81e67400, 0x08ba6fb5, 0x571be91f, + 0xf296ec6b, 0x2a0dd915, 0xb6636521, 0xe7b9f9b6, 0xff34052e, 0xc5855664, + 0x53b02d5d, 0xa99f8fa1, 0x08ba4799, 0x6e85076a + }, + { + 0x4b7a70e9, 0xb5b32944, 0xdb75092e, 0xc4192623, 0xad6ea6b0, 0x49a7df7d, + 0x9cee60b8, 0x8fedb266, 0xecaa8c71, 0x699a17ff, 0x5664526c, 0xc2b19ee1, + 0x193602a5, 0x75094c29, 0xa0591340, 0xe4183a3e, 0x3f54989a, 0x5b429d65, + 0x6b8fe4d6, 0x99f73fd6, 0xa1d29c07, 0xefe830f5, 0x4d2d38e6, 0xf0255dc1, + 0x4cdd2086, 0x8470eb26, 0x6382e9c6, 0x021ecc5e, 0x09686b3f, 0x3ebaefc9, + 0x3c971814, 0x6b6a70a1, 0x687f3584, 0x52a0e286, 0xb79c5305, 0xaa500737, + 0x3e07841c, 0x7fdeae5c, 0x8e7d44ec, 0x5716f2b8, 0xb03ada37, 0xf0500c0d, + 0xf01c1f04, 0x0200b3ff, 0xae0cf51a, 0x3cb574b2, 0x25837a58, 0xdc0921bd, + 0xd19113f9, 0x7ca92ff6, 0x94324773, 0x22f54701, 0x3ae5e581, 0x37c2dadc, + 0xc8b57634, 0x9af3dda7, 0xa9446146, 0x0fd0030e, 0xecc8c73e, 0xa4751e41, + 0xe238cd99, 0x3bea0e2f, 0x3280bba1, 0x183eb331, 0x4e548b38, 0x4f6db908, + 0x6f420d03, 0xf60a04bf, 0x2cb81290, 0x24977c79, 0x5679b072, 0xbcaf89af, + 0xde9a771f, 0xd9930810, 0xb38bae12, 0xdccf3f2e, 0x5512721f, 0x2e6b7124, + 0x501adde6, 0x9f84cd87, 0x7a584718, 0x7408da17, 0xbc9f9abc, 0xe94b7d8c, + 0xec7aec3a, 0xdb851dfa, 0x63094366, 0xc464c3d2, 0xef1c1847, 0x3215d908, + 0xdd433b37, 0x24c2ba16, 0x12a14d43, 0x2a65c451, 0x50940002, 0x133ae4dd, + 0x71dff89e, 0x10314e55, 0x81ac77d6, 0x5f11199b, 0x043556f1, 0xd7a3c76b, + 0x3c11183b, 0x5924a509, 0xf28fe6ed, 0x97f1fbfa, 0x9ebabf2c, 0x1e153c6e, + 0x86e34570, 0xeae96fb1, 0x860e5e0a, 0x5a3e2ab3, 0x771fe71c, 0x4e3d06fa, + 0x2965dcb9, 0x99e71d0f, 0x803e89d6, 0x5266c825, 0x2e4cc978, 0x9c10b36a, + 0xc6150eba, 0x94e2ea78, 0xa5fc3c53, 0x1e0a2df4, 0xf2f74ea7, 0x361d2b3d, + 0x1939260f, 0x19c27960, 0x5223a708, 0xf71312b6, 0xebadfe6e, 0xeac31f66, + 0xe3bc4595, 0xa67bc883, 0xb17f37d1, 0x018cff28, 0xc332ddef, 0xbe6c5aa5, + 0x65582185, 0x68ab9802, 0xeecea50f, 0xdb2f953b, 0x2aef7dad, 0x5b6e2f84, + 0x1521b628, 0x29076170, 0xecdd4775, 0x619f1510, 0x13cca830, 0xeb61bd96, + 0x0334fe1e, 0xaa0363cf, 0xb5735c90, 0x4c70a239, 0xd59e9e0b, 0xcbaade14, + 0xeecc86bc, 0x60622ca7, 0x9cab5cab, 0xb2f3846e, 0x648b1eaf, 0x19bdf0ca, + 0xa02369b9, 0x655abb50, 0x40685a32, 0x3c2ab4b3, 0x319ee9d5, 0xc021b8f7, + 0x9b540b19, 0x875fa099, 0x95f7997e, 0x623d7da8, 0xf837889a, 0x97e32d77, + 0x11ed935f, 0x16681281, 0x0e358829, 0xc7e61fd6, 0x96dedfa1, 0x7858ba99, + 0x57f584a5, 0x1b227263, 0x9b83c3ff, 0x1ac24696, 0xcdb30aeb, 0x532e3054, + 0x8fd948e4, 0x6dbc3128, 0x58ebf2ef, 0x34c6ffea, 0xfe28ed61, 0xee7c3c73, + 0x5d4a14d9, 0xe864b7e3, 0x42105d14, 0x203e13e0, 0x45eee2b6, 0xa3aaabea, + 0xdb6c4f15, 0xfacb4fd0, 0xc742f442, 0xef6abbb5, 0x654f3b1d, 0x41cd2105, + 0xd81e799e, 0x86854dc7, 0xe44b476a, 0x3d816250, 0xcf62a1f2, 0x5b8d2646, + 0xfc8883a0, 0xc1c7b6a3, 0x7f1524c3, 0x69cb7492, 0x47848a0b, 0x5692b285, + 0x095bbf00, 0xad19489d, 0x1462b174, 0x23820e00, 0x58428d2a, 0x0c55f5ea, + 0x1dadf43e, 0x233f7061, 0x3372f092, 0x8d937e41, 0xd65fecf1, 0x6c223bdb, + 0x7cde3759, 0xcbee7460, 0x4085f2a7, 0xce77326e, 0xa6078084, 0x19f8509e, + 0xe8efd855, 0x61d99735, 0xa969a7aa, 0xc50c06c2, 0x5a04abfc, 0x800bcadc, + 0x9e447a2e, 0xc3453484, 0xfdd56705, 0x0e1e9ec9, 0xdb73dbd3, 0x105588cd, + 0x675fda79, 0xe3674340, 0xc5c43465, 0x713e38d8, 0x3d28f89e, 0xf16dff20, + 0x153e21e7, 0x8fb03d4a, 0xe6e39f2b, 0xdb83adf7 + }, + { + 0xe93d5a68, 0x948140f7, 0xf64c261c, 0x94692934, 0x411520f7, 0x7602d4f7, + 0xbcf46b2e, 0xd4a20068, 0xd4082471, 0x3320f46a, 0x43b7d4b7, 0x500061af, + 0x1e39f62e, 0x97244546, 0x14214f74, 0xbf8b8840, 0x4d95fc1d, 0x96b591af, + 0x70f4ddd3, 0x66a02f45, 0xbfbc09ec, 0x03bd9785, 0x7fac6dd0, 0x31cb8504, + 0x96eb27b3, 0x55fd3941, 0xda2547e6, 0xabca0a9a, 0x28507825, 0x530429f4, + 0x0a2c86da, 0xe9b66dfb, 0x68dc1462, 0xd7486900, 0x680ec0a4, 0x27a18dee, + 0x4f3ffea2, 0xe887ad8c, 0xb58ce006, 0x7af4d6b6, 0xaace1e7c, 0xd3375fec, + 0xce78a399, 0x406b2a42, 0x20fe9e35, 0xd9f385b9, 0xee39d7ab, 0x3b124e8b, + 0x1dc9faf7, 0x4b6d1856, 0x26a36631, 0xeae397b2, 0x3a6efa74, 0xdd5b4332, + 0x6841e7f7, 0xca7820fb, 0xfb0af54e, 0xd8feb397, 0x454056ac, 0xba489527, + 0x55533a3a, 0x20838d87, 0xfe6ba9b7, 0xd096954b, 0x55a867bc, 0xa1159a58, + 0xcca92963, 0x99e1db33, 0xa62a4a56, 0x3f3125f9, 0x5ef47e1c, 0x9029317c, + 0xfdf8e802, 0x04272f70, 0x80bb155c, 0x05282ce3, 0x95c11548, 0xe4c66d22, + 0x48c1133f, 0xc70f86dc, 0x07f9c9ee, 0x41041f0f, 0x404779a4, 0x5d886e17, + 0x325f51eb, 0xd59bc0d1, 0xf2bcc18f, 0x41113564, 0x257b7834, 0x602a9c60, + 0xdff8e8a3, 0x1f636c1b, 0x0e12b4c2, 0x02e1329e, 0xaf664fd1, 0xcad18115, + 0x6b2395e0, 0x333e92e1, 0x3b240b62, 0xeebeb922, 0x85b2a20e, 0xe6ba0d99, + 0xde720c8c, 0x2da2f728, 0xd0127845, 0x95b794fd, 0x647d0862, 0xe7ccf5f0, + 0x5449a36f, 0x877d48fa, 0xc39dfd27, 0xf33e8d1e, 0x0a476341, 0x992eff74, + 0x3a6f6eab, 0xf4f8fd37, 0xa812dc60, 0xa1ebddf8, 0x991be14c, 0xdb6e6b0d, + 0xc67b5510, 0x6d672c37, 0x2765d43b, 0xdcd0e804, 0xf1290dc7, 0xcc00ffa3, + 0xb5390f92, 0x690fed0b, 0x667b9ffb, 0xcedb7d9c, 0xa091cf0b, 0xd9155ea3, + 0xbb132f88, 0x515bad24, 0x7b9479bf, 0x763bd6eb, 0x37392eb3, 0xcc115979, + 0x8026e297, 0xf42e312d, 0x6842ada7, 0xc66a2b3b, 0x12754ccc, 0x782ef11c, + 0x6a124237, 0xb79251e7, 0x06a1bbe6, 0x4bfb6350, 0x1a6b1018, 0x11caedfa, + 0x3d25bdd8, 0xe2e1c3c9, 0x44421659, 0x0a121386, 0xd90cec6e, 0xd5abea2a, + 0x64af674e, 0xda86a85f, 0xbebfe988, 0x64e4c3fe, 0x9dbc8057, 0xf0f7c086, + 0x60787bf8, 0x6003604d, 0xd1fd8346, 0xf6381fb0, 0x7745ae04, 0xd736fccc, + 0x83426b33, 0xf01eab71, 0xb0804187, 0x3c005e5f, 0x77a057be, 0xbde8ae24, + 0x55464299, 0xbf582e61, 0x4e58f48f, 0xf2ddfda2, 0xf474ef38, 0x8789bdc2, + 0x5366f9c3, 0xc8b38e74, 0xb475f255, 0x46fcd9b9, 0x7aeb2661, 0x8b1ddf84, + 0x846a0e79, 0x915f95e2, 0x466e598e, 0x20b45770, 0x8cd55591, 0xc902de4c, + 0xb90bace1, 0xbb8205d0, 0x11a86248, 0x7574a99e, 0xb77f19b6, 0xe0a9dc09, + 0x662d09a1, 0xc4324633, 0xe85a1f02, 0x09f0be8c, 0x4a99a025, 0x1d6efe10, + 0x1ab93d1d, 0x0ba5a4df, 0xa186f20f, 0x2868f169, 0xdcb7da83, 0x573906fe, + 0xa1e2ce9b, 0x4fcd7f52, 0x50115e01, 0xa70683fa, 0xa002b5c4, 0x0de6d027, + 0x9af88c27, 0x773f8641, 0xc3604c06, 0x61a806b5, 0xf0177a28, 0xc0f586e0, + 0x006058aa, 0x30dc7d62, 0x11e69ed7, 0x2338ea63, 0x53c2dd94, 0xc2c21634, + 0xbbcbee56, 0x90bcb6de, 0xebfc7da1, 0xce591d76, 0x6f05e409, 0x4b7c0188, + 0x39720a3d, 0x7c927c24, 0x86e3725f, 0x724d9db9, 0x1ac15bb4, 0xd39eb8fc, + 0xed545578, 0x08fca5b5, 0xd83d7cd3, 0x4dad0fc4, 0x1e50ef5e, 0xb161e6f8, + 0xa28514d9, 0x6c51133c, 0x6fd5c7e7, 0x56e14ec4, 0x362abfce, 0xddc6c837, + 0xd79a3234, 0x92638212, 0x670efa8e, 0x406000e0 + }, + { + 0x3a39ce37, 0xd3faf5cf, 0xabc27737, 0x5ac52d1b, 0x5cb0679e, 0x4fa33742, + 0xd3822740, 0x99bc9bbe, 0xd5118e9d, 0xbf0f7315, 0xd62d1c7e, 0xc700c47b, + 0xb78c1b6b, 0x21a19045, 0xb26eb1be, 0x6a366eb4, 0x5748ab2f, 0xbc946e79, + 0xc6a376d2, 0x6549c2c8, 0x530ff8ee, 0x468dde7d, 0xd5730a1d, 0x4cd04dc6, + 0x2939bbdb, 0xa9ba4650, 0xac9526e8, 0xbe5ee304, 0xa1fad5f0, 0x6a2d519a, + 0x63ef8ce2, 0x9a86ee22, 0xc089c2b8, 0x43242ef6, 0xa51e03aa, 0x9cf2d0a4, + 0x83c061ba, 0x9be96a4d, 0x8fe51550, 0xba645bd6, 0x2826a2f9, 0xa73a3ae1, + 0x4ba99586, 0xef5562e9, 0xc72fefd3, 0xf752f7da, 0x3f046f69, 0x77fa0a59, + 0x80e4a915, 0x87b08601, 0x9b09e6ad, 0x3b3ee593, 0xe990fd5a, 0x9e34d797, + 0x2cf0b7d9, 0x022b8b51, 0x96d5ac3a, 0x017da67d, 0xd1cf3ed6, 0x7c7d2d28, + 0x1f9f25cf, 0xadf2b89b, 0x5ad6b472, 0x5a88f54c, 0xe029ac71, 0xe019a5e6, + 0x47b0acfd, 0xed93fa9b, 0xe8d3c48d, 0x283b57cc, 0xf8d56629, 0x79132e28, + 0x785f0191, 0xed756055, 0xf7960e44, 0xe3d35e8c, 0x15056dd4, 0x88f46dba, + 0x03a16125, 0x0564f0bd, 0xc3eb9e15, 0x3c9057a2, 0x97271aec, 0xa93a072a, + 0x1b3f6d9b, 0x1e6321f5, 0xf59c66fb, 0x26dcf319, 0x7533d928, 0xb155fdf5, + 0x03563482, 0x8aba3cbb, 0x28517711, 0xc20ad9f8, 0xabcc5167, 0xccad925f, + 0x4de81751, 0x3830dc8e, 0x379d5862, 0x9320f991, 0xea7a90c2, 0xfb3e7bce, + 0x5121ce64, 0x774fbe32, 0xa8b6e37e, 0xc3293d46, 0x48de5369, 0x6413e680, + 0xa2ae0810, 0xdd6db224, 0x69852dfd, 0x09072166, 0xb39a460a, 0x6445c0dd, + 0x586cdecf, 0x1c20c8ae, 0x5bbef7dd, 0x1b588d40, 0xccd2017f, 0x6bb4e3bb, + 0xdda26a7e, 0x3a59ff45, 0x3e350a44, 0xbcb4cdd5, 0x72eacea8, 0xfa6484bb, + 0x8d6612ae, 0xbf3c6f47, 0xd29be463, 0x542f5d9e, 0xaec2771b, 0xf64e6370, + 0x740e0d8d, 0xe75b1357, 0xf8721671, 0xaf537d5d, 0x4040cb08, 0x4eb4e2cc, + 0x34d2466a, 0x0115af84, 0xe1b00428, 0x95983a1d, 0x06b89fb4, 0xce6ea048, + 0x6f3f3b82, 0x3520ab82, 0x011a1d4b, 0x277227f8, 0x611560b1, 0xe7933fdc, + 0xbb3a792b, 0x344525bd, 0xa08839e1, 0x51ce794b, 0x2f32c9b7, 0xa01fbac9, + 0xe01cc87e, 0xbcc7d1f6, 0xcf0111c3, 0xa1e8aac7, 0x1a908749, 0xd44fbd9a, + 0xd0dadecb, 0xd50ada38, 0x0339c32a, 0xc6913667, 0x8df9317c, 0xe0b12b4f, + 0xf79e59b7, 0x43f5bb3a, 0xf2d519ff, 0x27d9459c, 0xbf97222c, 0x15e6fc2a, + 0x0f91fc71, 0x9b941525, 0xfae59361, 0xceb69ceb, 0xc2a86459, 0x12baa8d1, + 0xb6c1075e, 0xe3056a0c, 0x10d25065, 0xcb03a442, 0xe0ec6e0e, 0x1698db3b, + 0x4c98a0be, 0x3278e964, 0x9f1f9532, 0xe0d392df, 0xd3a0342b, 0x8971f21e, + 0x1b0a7441, 0x4ba3348c, 0xc5be7120, 0xc37632d8, 0xdf359f8d, 0x9b992f2e, + 0xe60b6f47, 0x0fe3f11d, 0xe54cda54, 0x1edad891, 0xce6279cf, 0xcd3e7e6f, + 0x1618b166, 0xfd2c1d05, 0x848fd2c5, 0xf6fb2299, 0xf523f357, 0xa6327623, + 0x93a83531, 0x56cccd02, 0xacf08162, 0x5a75ebb5, 0x6e163697, 0x88d273cc, + 0xde966292, 0x81b949d0, 0x4c50901b, 0x71c65614, 0xe6c6c7bd, 0x327a140a, + 0x45e1d006, 0xc3f27b9a, 0xc9aa53fd, 0x62a80f00, 0xbb25bfe2, 0x35bdd2f6, + 0x71126905, 0xb2040222, 0xb6cbcf7c, 0xcd769c2b, 0x53113ec0, 0x1640e3d3, + 0x38abbd60, 0x2547adf0, 0xba38209c, 0xf746ce76, 0x77afa1c5, 0x20756060, + 0x85cbfe4e, 0x8ae88dd8, 0x7aaaf9b0, 0x4cf9aa7e, 0x1948c25c, 0x02fb8a8c, + 0x01c36ae4, 0xd6ebe1f9, 0x90d4f869, 0xa65cdea0, 0x3f09252d, 0xc208e69f, + 0xb74e6132, 0xce77e25b, 0x578fdfe3, 0x3ac372e6 + } + }; + + /// + /// Constructs and initializes a blowfish instance with the supplied key. + /// + /// The key to cipher with. + public Blowfish(byte[] key) + { + short i; + short j; + short k; + uint data; + uint datal; + uint datar; + + j = 0; + for (i = 0; i < N + 2; ++i) + { + data = 0x00000000; + for (k = 0; k < 4; ++k) + { + data |= (uint)(key[j] << k * 8); + j++; + if (j >= key.Length) + { + j = 0; + } + } + P[i] = P[i] ^ data; + } + + datal = 0x00000000; + datar = 0x00000000; + + for (i = 0; i < N + 2; i += 2) + { + Encipher(ref datal, ref datar); + P[i] = datal; + P[i + 1] = datar; + } + + for (i = 0; i < 4; ++i) + { + for (j = 0; j < 256; j += 2) + { + Encipher(ref datal, ref datar); + + S[i, j] = datal; + S[i, j + 1] = datar; + } + } + } + + /// + /// + /// + /// + /// + private uint F(uint x) + { + ushort a; + ushort b; + ushort c; + ushort d; + uint y; + + d = (ushort)(x & 0x00FF); + x >>= 8; + c = (ushort)(x & 0x00FF); + x >>= 8; + b = (ushort)(x & 0x00FF); + x >>= 8; + a = (ushort)(x & 0x00FF); + y = S[0, a] + S[1, b]; + y ^= S[2, c]; + y += S[3, d]; + + return y; + } + + + /// + /// Encrypts 8 bytes of data (1 block) + /// + /// The left part of the 8 bytes. + /// The right part of the 8 bytes. + private void Encipher(ref uint xl, ref uint xr) + { + uint Xl; + uint Xr; + uint temp; + short i; + + Xl = xl; + Xr = xr; + + for (i = 0; i < N; ++i) + { + Xl ^= P[i]; + Xr = F(Xl) ^ Xr; + + temp = Xl; + Xl = Xr; + Xr = temp; + } + + temp = Xl; + Xl = Xr; + Xr = temp; + + Xr ^= P[N]; + Xl ^= P[N + 1]; + + xl = Xl; + xr = Xr; + } + + /// + /// Decrypts a byte array in place. + /// + /// Data to decrypt. + public void Decipher(byte[] data) + { + if (data.Length % 8 != 0) + { + throw new Exception("Invalid Length"); + } + + for (int i = 0; i < data.Length; i += 8) + { + // Encode the data in 8 byte blocks. + var xl = BitConverter.ToUInt32(data, i); + var xr = BitConverter.ToUInt32(data, i + 4); + Decipher(ref xl, ref xr); + Array.Copy(BitConverter.GetBytes(xl), 0, data, i, 4); + Array.Copy(BitConverter.GetBytes(xr), 0, data, i + 4, 4); + } + } + +#if NET5_0_OR_GREATER + /// + /// Decrypts a byte span in place. + /// + /// Data to decrypt. + public void Decipher(Span data) + { + if (data.Length % 8 != 0) + { + throw new Exception("Invalid Length"); + } + + for (int i = 0; i < data.Length; i += 8) + { + // Encode the data in 8 byte blocks. + var xl = BitConverter.ToUInt32(data.Slice(i)); + var xr = BitConverter.ToUInt32(data.Slice(i + 4)); + Decipher(ref xl, ref xr); + BitConverter.TryWriteBytes(data.Slice(i), xl); + BitConverter.TryWriteBytes(data.Slice(i + 4), xr); + } + } +#endif + + /// + /// Decrypts 8 bytes of data (1 block) + /// + /// The left part of the 8 bytes. + /// The right part of the 8 bytes. + private void Decipher(ref uint xl, ref uint xr) + { + uint Xl; + uint Xr; + uint temp; + short i; + + Xl = xl; + Xr = xr; + + for (i = N + 1; i > 1; --i) + { + Xl ^= P[i]; + Xr = F(Xl) ^ Xr; + + /* Exchange Xl and Xr */ + temp = Xl; + Xl = Xr; + Xr = temp; + } + + /* Exchange Xl and Xr */ + temp = Xl; + Xl = Xr; + Xr = temp; + + Xr ^= P[1]; + Xl ^= P[0]; + + xl = Xl; + xr = Xr; + } +} \ No newline at end of file diff --git a/Src/GBX.NET.PAK/BlowfishStream.cs b/Src/GBX.NET.PAK/BlowfishStream.cs new file mode 100644 index 000000000..42ba0eb3f --- /dev/null +++ b/Src/GBX.NET.PAK/BlowfishStream.cs @@ -0,0 +1,112 @@ +using GBX.NET.Serialization; + +namespace GBX.NET.PAK; + +public class BlowfishStream : Stream, IEncryptedStream +{ + private readonly Stream stream; + private readonly Blowfish blowfish; + + private ulong iv; + private ulong ivXor; +#if NET5_0_OR_GREATER + private readonly byte[] memoryBuffer; +#else + private byte[] memoryBuffer; +#endif + private int bufferIndex; + private int totalIndex; + + public BlowfishStream(Stream stream, byte[] key, ulong iv) + { + this.stream = stream ?? throw new ArgumentNullException(nameof(BlowfishStream.stream)); + blowfish = new Blowfish(key); + this.iv = iv; + memoryBuffer = new byte[8]; + } + + public override bool CanRead => stream.CanRead; + public override bool CanWrite => false; + public override bool CanSeek => false; + public override long Length => throw new NotSupportedException(); + public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); } + + public override void Flush() => stream.Flush(); + + public void Initialize(byte[] data, uint offset, uint count) + { + for (int i = 0; i < count; i++) + { + uint lopart = (uint)(ivXor & 0xFFFFFFFF); + uint hipart = (uint)(ivXor >> 32); + lopart = (uint)(data[offset + i] | 0xAA) ^ (uint)((lopart << 13) | (hipart >> 19)); + hipart = (uint)((ivXor << 13) >> 32); + ivXor = ((ulong)hipart << 32) | lopart; + } + } + + public override int Read(byte[] buffer, int offset, int count) + { + if (!CanRead) + { + throw new NotSupportedException("Stream is not readable."); + } + + if (totalIndex == 0) + { + iv ^= ivXor; + ivXor = 0; + } + +#if NET5_0_OR_GREATER + var memorySpan = memoryBuffer.AsSpan(); +#endif + + for (int i = 0; i < count; i++) + { + if (bufferIndex % 8 == 0) + { + // Trick #1 + if (bufferIndex == 0x100) + { + iv ^= ivXor; + ivXor = 0; + bufferIndex = 0; + } + +#if NET5_0_OR_GREATER + var read = stream.Read(memorySpan); + var nextIV = BitConverter.ToUInt64(memorySpan); + blowfish.Decipher(memorySpan); + var block = BitConverter.ToUInt64(memorySpan); + block ^= iv; + BitConverter.TryWriteBytes(memorySpan, block); +#else + var read = stream.Read(memoryBuffer, 0, 8); + ulong nextIV = BitConverter.ToUInt64(memoryBuffer, 0); + blowfish.Decipher(memoryBuffer); + ulong block = BitConverter.ToUInt64(memoryBuffer, 0); + block ^= iv; + memoryBuffer = BitConverter.GetBytes(block); +#endif + iv = nextIV; + } +#if NET5_0_OR_GREATER + buffer[offset + i] = memorySpan[bufferIndex % 8]; +#else + buffer[offset + i] = memoryBuffer[bufferIndex % 8]; +#endif + bufferIndex++; + totalIndex++; + } + return count; + } + + public override void Write(byte[] inputBuffer, int offset, int count) + { + throw new NotSupportedException("Stream is not writable."); + } + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); +} \ No newline at end of file diff --git a/Src/GBX.NET.PAK/Exceptions/NotAPakException.cs b/Src/GBX.NET.PAK/Exceptions/NotAPakException.cs new file mode 100644 index 000000000..e4a2abef6 --- /dev/null +++ b/Src/GBX.NET.PAK/Exceptions/NotAPakException.cs @@ -0,0 +1,9 @@ +namespace GBX.NET.Exceptions; + +[Serializable] +public class NotAPakException : Exception +{ + public NotAPakException() : base("Pak data stream was not identified.") { } + public NotAPakException(string message) : base(message) { } + public NotAPakException(string message, Exception? innerException) : base(message, innerException) { } +} diff --git a/Src/GBX.NET.PAK/GBX.NET.PAK.csproj b/Src/GBX.NET.PAK/GBX.NET.PAK.csproj index c5aba2dd4..15eb39469 100644 --- a/Src/GBX.NET.PAK/GBX.NET.PAK.csproj +++ b/Src/GBX.NET.PAK/GBX.NET.PAK.csproj @@ -54,4 +54,16 @@ + + + 1.0.1 + + + + + + 1.0.1 + + + diff --git a/Src/GBX.NET.PAK/GbxReaderExtensions.cs b/Src/GBX.NET.PAK/GbxReaderExtensions.cs new file mode 100644 index 000000000..f1c06577e --- /dev/null +++ b/Src/GBX.NET.PAK/GbxReaderExtensions.cs @@ -0,0 +1,32 @@ +using GBX.NET.Serialization; + +namespace GBX.NET.PAK; + +public static class GbxReaderExtensions +{ + private static readonly byte[] nadeoPakMagic = [(byte)'N', (byte)'a', (byte)'d', (byte)'e', (byte)'o', (byte)'P', (byte)'a', (byte)'k']; + + public static bool ReadPakMagic(this GbxReader reader) + { + var magicLength = nadeoPakMagic.Length; +#if NETSTANDARD2_0 + var magic = reader.ReadBytes(magicLength); +#else + Span magic = stackalloc byte[magicLength]; + if (reader.Read(magic) != magicLength) + { + throw new EndOfStreamException("Failed to read NadeoPak magic bytes."); + } +#endif + + for (var i = 0; i < magicLength; i++) + { + if (magic[i] != nadeoPakMagic[i]) + { + return false; + } + } + + return true; + } +} diff --git a/Src/GBX.NET.PAK/Pak.cs b/Src/GBX.NET.PAK/Pak.cs index 41a39c6da..fe58ad8f6 100644 --- a/Src/GBX.NET.PAK/Pak.cs +++ b/Src/GBX.NET.PAK/Pak.cs @@ -1,9 +1,199 @@ -namespace GBX.NET.PAK; +using GBX.NET.Exceptions; +using GBX.NET.Serialization; +using System.Collections.Immutable; +using System.IO.Compression; +using System.Text; -public sealed class Pak +namespace GBX.NET.PAK; + +public sealed partial class Pak : IDisposable +#if NET5_0_OR_GREATER + , IAsyncDisposable +#endif { - public static Task ParseAsync(Stream stream, byte[] key, CancellationToken cancellationToken = default) + /// + /// Magic (intial binary letters) for Pak files. + /// + public const string Magic = "NadeoPak"; + + private readonly Stream stream; + private readonly byte[] key; + private readonly int metadataStart; + private readonly int dataStart; + + public int Version { get; } + public int Flags { get; } + public ImmutableDictionary Files { get; } + + private Pak(Stream stream, byte[] key, int version, int metadataStart, int dataStart, int flags, ImmutableDictionary files) + { + this.stream = stream; + this.key = key; + Version = version; + this.metadataStart = metadataStart; + this.dataStart = dataStart; + Flags = flags; + Files = files; + } + + public static async Task ParseAsync(Stream stream, byte[] key, CancellationToken cancellationToken = default) + { + var r = new GbxReader(stream); + + if (!r.ReadPakMagic()) + { + throw new NotAPakException(); + } + + var version = r.ReadInt32(); + var headerIV = r.ReadUInt64(); + + var decryptStream = new BlowfishStream(stream, key, headerIV); + var decryptReader = new GbxReader(decryptStream); + + return await ParseEncryptedAsync(decryptReader, stream, version, key, cancellationToken); + } + + private static async Task ParseEncryptedAsync( + GbxReader r, + Stream originalStream, + int version, + byte[] key, + CancellationToken cancellationToken) + { + var headerMD5 = await r.ReadBytesAsync(16, cancellationToken); + var metadataStart = r.ReadInt32(); // offset to metadata section + var dataStart = r.ReadInt32(); + + if (version >= 2) + { + var gbxHeadersSize = r.ReadInt32(); + var gbxHeadersComprSize = r.ReadInt32(); + } + + if (version >= 3) + { + r.SkipData(16); // unused + } + + var flags = r.ReadInt32(); + + var allFolders = ReadAllFolders(r); + + if (allFolders.Length > 2 && allFolders[2].Name.Length > 4) + { + var nameBytes = Encoding.Unicode.GetBytes(allFolders[2].Name); + ((IEncryptedStream)r.BaseStream).Initialize(nameBytes, 4, 4); + } + + var files = ReadAllFiles(r, allFolders); + + return new Pak(originalStream, key, version, metadataStart, dataStart, flags, files); + } + + private static PakFolder[] ReadAllFolders(GbxReader r) + { + var numFolders = r.ReadInt32(); + var allFolders = new PakFolder[numFolders]; + + for (var i = 0; i < numFolders; i++) + { + var parentFolderIndex = r.ReadInt32(); // index into folders; -1 if this is a root folder + var name = r.ReadString(); + + allFolders[i] = new PakFolder(name, parentFolderIndex == -1 ? null : parentFolderIndex); + } + + return allFolders; + } + + private static ImmutableDictionary ReadAllFiles(GbxReader r, PakFolder[] allFolders) + { + var files = ImmutableDictionary.CreateBuilder(); + + var numFiles = r.ReadInt32(); + for (var i = 0; i < numFiles; i++) + { + var folderIndex = r.ReadInt32(); // index into folders + var name = r.ReadString(); + var u01 = r.ReadInt32(); + var uncompressedSize = r.ReadInt32(); + var compressedSize = r.ReadInt32(); + var offset = r.ReadInt32(); + var classId = r.ReadUInt32(); // indicates the type of the file + var fileFlags = r.ReadUInt64(); + + var folderPath = string.Concat(RecurseFoldersToParent(folderIndex, allFolders) + .Reverse() + .Select(f => f.Name)); + var filePath = string.Concat(folderPath, name); + + var file = new PakFile(name, folderPath, classId, offset, uncompressedSize, compressedSize, fileFlags); + files[filePath] = file; + } + + return files.ToImmutable(); + } + + private static IEnumerable RecurseFoldersToParent(int folderIndex, PakFolder[] allFolders) + { + if (folderIndex == -1) + { + yield break; + } + + var folder = allFolders[folderIndex]; + + yield return folder; + + if (folder.ParentIndex is null) + { + yield break; + } + + foreach (var f in RecurseFoldersToParent(folder.ParentIndex.Value, allFolders)) + { + yield return f; + } + } + + public Stream OpenFile(PakFile file) + { + stream.Position = dataStart + file.Offset; + + var ivBuffer = new byte[8]; + stream.Read(ivBuffer, 0, 8); + var iv = BitConverter.ToUInt64(ivBuffer, 0); + + var data = new byte[file.CompressedSize]; + stream.Read(data, 0, data.Length); + + var ms = new MemoryStream(data); + + var blowfish = new BlowfishStream(ms, key, iv); + + if (!file.IsCompressed) + { + return blowfish; + } + + if (Gbx.ZLib is null) + { + throw new ZLibNotDefinedException(); + } + + return Gbx.ZLib.Decompress(blowfish); + } + + public void Dispose() + { + stream.Dispose(); + } + +#if NET5_0_OR_GREATER + public async ValueTask DisposeAsync() { - return Task.FromResult(new Pak()); + await stream.DisposeAsync(); } -} +#endif +} \ No newline at end of file diff --git a/Src/GBX.NET.PAK/PakFile.cs b/Src/GBX.NET.PAK/PakFile.cs new file mode 100644 index 000000000..d7eff5bc2 --- /dev/null +++ b/Src/GBX.NET.PAK/PakFile.cs @@ -0,0 +1,37 @@ +namespace GBX.NET.PAK; + +public sealed class PakFile +{ + public string Name { get; } + public string FolderPath { get; } + public uint ClassId { get; } + public int Offset { get; } + public int UncompressedSize { get; } + public int CompressedSize { get; } + public ulong Flags { get; } + + public bool IsCompressed => (Flags & 0x7C) != 0; + + public PakFile( + string name, + string folderPath, + uint classId, + int offset, + int uncompressedSize, + int compressedSize, + ulong flags) + { + Name = name; + FolderPath = folderPath; + ClassId = classId; + Offset = offset; + UncompressedSize = uncompressedSize; + CompressedSize = compressedSize; + Flags = flags; + } + + public override string ToString() + { + return $"{Name}, Folder: {FolderPath}"; + } +} \ No newline at end of file diff --git a/Src/GBX.NET.PAK/PakFolder.cs b/Src/GBX.NET.PAK/PakFolder.cs new file mode 100644 index 000000000..3be618fe3 --- /dev/null +++ b/Src/GBX.NET.PAK/PakFolder.cs @@ -0,0 +1,3 @@ +namespace GBX.NET.PAK; + +internal sealed record PakFolder(string Name, int? ParentIndex); \ No newline at end of file diff --git a/Src/GBX.NET.PAK/PakList.cs b/Src/GBX.NET.PAK/PakList.cs index 1deb97bcd..15e552365 100644 --- a/Src/GBX.NET.PAK/PakList.cs +++ b/Src/GBX.NET.PAK/PakList.cs @@ -73,7 +73,7 @@ public static async Task ParseAsync(Stream stream, CancellationToken ca encryptedKeyString[j] ^= keyStringKey[j % keyStringKey.Length]; } - var key = await MD5.ComputeAsync(Encoding.ASCII.GetString(encryptedKeyString) + "NadeoPak", cancellationToken); + var key = await MD5.ComputeAsync(Encoding.ASCII.GetString(encryptedKeyString) + Pak.Magic, cancellationToken); packs[name] = new PakListItem(key, flags); } diff --git a/Src/GBX.NET.ZLib/ZLib.cs b/Src/GBX.NET.ZLib/ZLib.cs index 71aac5599..dfe7b1a40 100644 --- a/Src/GBX.NET.ZLib/ZLib.cs +++ b/Src/GBX.NET.ZLib/ZLib.cs @@ -16,4 +16,9 @@ public void Decompress(Stream input, Stream output) using var zlib = new ZlibStream(input, CompressionMode.Decompress, leaveOpen: true); zlib.CopyTo(output); } + + public Stream Decompress(Stream input) + { + return new ZlibStream(input, CompressionMode.Decompress); + } } diff --git a/Src/GBX.NET/Engines/MwFoundations/CMwNod.cs b/Src/GBX.NET/Engines/MwFoundations/CMwNod.cs index 7d16eaa23..8209ed927 100644 --- a/Src/GBX.NET/Engines/MwFoundations/CMwNod.cs +++ b/Src/GBX.NET/Engines/MwFoundations/CMwNod.cs @@ -3,6 +3,7 @@ using Microsoft.Extensions.Logging; using System.Diagnostics; using System.Text; +using System.Xml.Linq; namespace GBX.NET.Engines.MwFoundations; @@ -322,6 +323,32 @@ internal virtual void Read(GbxReaderWriter rw) { var r = rw.Reader ?? throw new Exception("Reader is required but not available."); + /*if (r.BaseStream is IEncryptedStream encryptedStream) + { + var baseType = GetType().BaseType ?? throw new ThisShouldNotHappenException(); + + var parentClassId = ClassManager.ClassIds[baseType]; + + if (parentClassId == 0x07031000) + { + parentClassId = 0x07001000; + } + + if (this is CPlugSurfaceGeom) + { + parentClassId = 0x0902B000; + } + + if (baseType == typeof(CGameCtnBlockInfo)) + { + parentClassId = 0x24005000; + } + + var parentClassIDBytes = BitConverter.GetBytes(parentClassId); + + encryptedStream.Initialize(parentClassIDBytes, 0, 4); + }*/ + var prevChunkId = default(uint?); while (true) diff --git a/Src/GBX.NET/Extensions/IZLib.cs b/Src/GBX.NET/Extensions/IZLib.cs index 231a2b34e..da40b943b 100644 --- a/Src/GBX.NET/Extensions/IZLib.cs +++ b/Src/GBX.NET/Extensions/IZLib.cs @@ -4,4 +4,5 @@ public interface IZLib { void Compress(Stream input, Stream output); void Decompress(Stream input, Stream output); + Stream Decompress(Stream input); } diff --git a/Src/GBX.NET/Serialization/GbxReader.cs b/Src/GBX.NET/Serialization/GbxReader.cs index 647d74b9f..e5f066fcc 100644 --- a/Src/GBX.NET/Serialization/GbxReader.cs +++ b/Src/GBX.NET/Serialization/GbxReader.cs @@ -263,7 +263,7 @@ public bool ReadGbxMagic() if (BaseStream.Read(buffer) != 3) { - return false; + throw new EndOfStreamException("Failed to read GBX magic bytes."); } return buffer[0] == 'G' && buffer[1] == 'B' && buffer[2] == 'X'; diff --git a/Src/GBX.NET/Serialization/IEncryptedStream.cs b/Src/GBX.NET/Serialization/IEncryptedStream.cs new file mode 100644 index 000000000..9361b4743 --- /dev/null +++ b/Src/GBX.NET/Serialization/IEncryptedStream.cs @@ -0,0 +1,12 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace GBX.NET.Serialization; + +public interface IEncryptedStream +{ + void Initialize(byte[] data, uint offset, uint count); +} diff --git a/Tools/PakToZip/PakToZip.csproj b/Tools/PakToZip/PakToZip.csproj new file mode 100644 index 000000000..dbab18fac --- /dev/null +++ b/Tools/PakToZip/PakToZip.csproj @@ -0,0 +1,16 @@ + + + + Exe + net9.0 + enable + enable + true + + + + + + + + diff --git a/Tools/PakToZip/Program.cs b/Tools/PakToZip/Program.cs new file mode 100644 index 000000000..47ea826df --- /dev/null +++ b/Tools/PakToZip/Program.cs @@ -0,0 +1,21 @@ +using GBX.NET; +using GBX.NET.PAK; +using GBX.NET.ZLib; + +Gbx.ZLib = new ZLib(); + +var fileName = args[0]; + +var packlistFileName = Path.Combine(Path.GetDirectoryName(fileName)!, "packlist.dat"); +var packlist = await PakList.ParseAsync(packlistFileName); + +var key = packlist[Path.GetFileNameWithoutExtension(fileName).ToLowerInvariant()].Key; + +using var fs = File.OpenRead(fileName); +using var pak = await Pak.ParseAsync(fs, key); + +using var stream = pak.OpenFile(pak.Files.Values.First(x => x.FolderPath.EndsWith("Solid\\"))); + +var ok = Gbx.Parse(stream, new() { IgnoreExceptionsInBody = true }); + +Console.WriteLine(); \ No newline at end of file From b583824957faa897622298486355d90e832eea70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Wed, 20 Nov 2024 18:58:42 +0100 Subject: [PATCH 40/86] Switch PAK to GPL license (due to only functioning blowfish impl) --- README.md | 1 + Src/GBX.NET.LZO/GBX.NET.LZO.csproj | 1 + Src/GBX.NET.PAK/GBX.NET.PAK.csproj | 4 +- Src/GBX.NET.PAK/LICENSE | 674 +++++++++++++++++++++++++++++ 4 files changed, 679 insertions(+), 1 deletion(-) create mode 100644 Src/GBX.NET.PAK/LICENSE diff --git a/README.md b/README.md index ed8a81686..b0a626df7 100644 --- a/README.md +++ b/README.md @@ -628,6 +628,7 @@ GBX.NET 2 is licensed under multiple licenses, depending on the part of the proj - GNU GPL v3 License - **Src/GBX.NET.LZO** - Src/GBX.NET.Tool.CLI + - **Src/GBX.NET.PAK** - Samples - Tools - The Unlicense diff --git a/Src/GBX.NET.LZO/GBX.NET.LZO.csproj b/Src/GBX.NET.LZO/GBX.NET.LZO.csproj index baf9185f5..fc6db30a4 100644 --- a/Src/GBX.NET.LZO/GBX.NET.LZO.csproj +++ b/Src/GBX.NET.LZO/GBX.NET.LZO.csproj @@ -38,6 +38,7 @@ + diff --git a/Src/GBX.NET.PAK/GBX.NET.PAK.csproj b/Src/GBX.NET.PAK/GBX.NET.PAK.csproj index 15eb39469..0b5793a12 100644 --- a/Src/GBX.NET.PAK/GBX.NET.PAK.csproj +++ b/Src/GBX.NET.PAK/GBX.NET.PAK.csproj @@ -11,7 +11,8 @@ README.md gbx, trackmania, shootmania, maniaplanet, gamebox, net, chunk, file - MIT + GPL-3.0-or-later + true @@ -36,6 +37,7 @@ + diff --git a/Src/GBX.NET.PAK/LICENSE b/Src/GBX.NET.PAK/LICENSE new file mode 100644 index 000000000..f288702d2 --- /dev/null +++ b/Src/GBX.NET.PAK/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. From 02d4acd088f05d2276408d6a9cde25b3e673001c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Wed, 20 Nov 2024 19:24:24 +0100 Subject: [PATCH 41/86] Add EncryptionInitializer --- Src/GBX.NET.PAK/BlowfishStream.cs | 6 +-- Src/GBX.NET.PAK/EncryptionInitializer.cs | 16 ++++++++ Src/GBX.NET.PAK/Pak.cs | 6 ++- Src/GBX.NET/Engines/MwFoundations/CMwNod.cs | 26 ------------- Src/GBX.NET/GbxReadSettings.cs | 2 + Src/GBX.NET/IEncryptionInitializer.cs | 6 +++ Src/GBX.NET/Serialization/GbxReader.cs | 37 +++++++++++++++++++ Src/GBX.NET/Serialization/IEncryptedStream.cs | 12 ------ Tools/PakToZip/Program.cs | 6 ++- 9 files changed, 71 insertions(+), 46 deletions(-) create mode 100644 Src/GBX.NET.PAK/EncryptionInitializer.cs create mode 100644 Src/GBX.NET/IEncryptionInitializer.cs delete mode 100644 Src/GBX.NET/Serialization/IEncryptedStream.cs diff --git a/Src/GBX.NET.PAK/BlowfishStream.cs b/Src/GBX.NET.PAK/BlowfishStream.cs index 42ba0eb3f..c9a6b7bdb 100644 --- a/Src/GBX.NET.PAK/BlowfishStream.cs +++ b/Src/GBX.NET.PAK/BlowfishStream.cs @@ -1,8 +1,6 @@ -using GBX.NET.Serialization; +namespace GBX.NET.PAK; -namespace GBX.NET.PAK; - -public class BlowfishStream : Stream, IEncryptedStream +public class BlowfishStream : Stream, IEncryptionInitializer { private readonly Stream stream; private readonly Blowfish blowfish; diff --git a/Src/GBX.NET.PAK/EncryptionInitializer.cs b/Src/GBX.NET.PAK/EncryptionInitializer.cs new file mode 100644 index 000000000..7bfc26f4d --- /dev/null +++ b/Src/GBX.NET.PAK/EncryptionInitializer.cs @@ -0,0 +1,16 @@ +namespace GBX.NET.PAK; + +public sealed class EncryptionInitializer : IEncryptionInitializer +{ + private readonly BlowfishStream stream; + + internal EncryptionInitializer(BlowfishStream stream) + { + this.stream = stream; + } + + public void Initialize(byte[] data, uint offset, uint count) + { + stream.Initialize(data, offset, count); + } +} diff --git a/Src/GBX.NET.PAK/Pak.cs b/Src/GBX.NET.PAK/Pak.cs index fe58ad8f6..9a77c87b3 100644 --- a/Src/GBX.NET.PAK/Pak.cs +++ b/Src/GBX.NET.PAK/Pak.cs @@ -83,7 +83,7 @@ private static async Task ParseEncryptedAsync( if (allFolders.Length > 2 && allFolders[2].Name.Length > 4) { var nameBytes = Encoding.Unicode.GetBytes(allFolders[2].Name); - ((IEncryptedStream)r.BaseStream).Initialize(nameBytes, 4, 4); + ((IEncryptionInitializer)r.BaseStream).Initialize(nameBytes, 4, 4); } var files = ReadAllFiles(r, allFolders); @@ -157,7 +157,7 @@ private static IEnumerable RecurseFoldersToParent(int folderIndex, Pa } } - public Stream OpenFile(PakFile file) + public Stream OpenFile(PakFile file, out EncryptionInitializer encryptionInitializer) { stream.Position = dataStart + file.Offset; @@ -172,6 +172,8 @@ public Stream OpenFile(PakFile file) var blowfish = new BlowfishStream(ms, key, iv); + encryptionInitializer = new EncryptionInitializer(blowfish); + if (!file.IsCompressed) { return blowfish; diff --git a/Src/GBX.NET/Engines/MwFoundations/CMwNod.cs b/Src/GBX.NET/Engines/MwFoundations/CMwNod.cs index 8209ed927..a519dd0f9 100644 --- a/Src/GBX.NET/Engines/MwFoundations/CMwNod.cs +++ b/Src/GBX.NET/Engines/MwFoundations/CMwNod.cs @@ -323,32 +323,6 @@ internal virtual void Read(GbxReaderWriter rw) { var r = rw.Reader ?? throw new Exception("Reader is required but not available."); - /*if (r.BaseStream is IEncryptedStream encryptedStream) - { - var baseType = GetType().BaseType ?? throw new ThisShouldNotHappenException(); - - var parentClassId = ClassManager.ClassIds[baseType]; - - if (parentClassId == 0x07031000) - { - parentClassId = 0x07001000; - } - - if (this is CPlugSurfaceGeom) - { - parentClassId = 0x0902B000; - } - - if (baseType == typeof(CGameCtnBlockInfo)) - { - parentClassId = 0x24005000; - } - - var parentClassIDBytes = BitConverter.GetBytes(parentClassId); - - encryptedStream.Initialize(parentClassIDBytes, 0, 4); - }*/ - var prevChunkId = default(uint?); while (true) diff --git a/Src/GBX.NET/GbxReadSettings.cs b/Src/GBX.NET/GbxReadSettings.cs index aac95021e..c19956ecd 100644 --- a/Src/GBX.NET/GbxReadSettings.cs +++ b/Src/GBX.NET/GbxReadSettings.cs @@ -36,4 +36,6 @@ public readonly record struct GbxReadSettings /// Skippable chunks will be read less efficiently, but they will be ignored if the read will fail, with (usually) no corruption once saved. /// public bool SafeSkippableChunks { get; init; } + + public IEncryptionInitializer EncryptionInitializer { get; init; } } \ No newline at end of file diff --git a/Src/GBX.NET/IEncryptionInitializer.cs b/Src/GBX.NET/IEncryptionInitializer.cs new file mode 100644 index 000000000..6af15bfde --- /dev/null +++ b/Src/GBX.NET/IEncryptionInitializer.cs @@ -0,0 +1,6 @@ +namespace GBX.NET; + +public interface IEncryptionInitializer +{ + void Initialize(byte[] data, uint offset, uint count); +} diff --git a/Src/GBX.NET/Serialization/GbxReader.cs b/Src/GBX.NET/Serialization/GbxReader.cs index e5f066fcc..7d4354c3e 100644 --- a/Src/GBX.NET/Serialization/GbxReader.cs +++ b/Src/GBX.NET/Serialization/GbxReader.cs @@ -1242,6 +1242,8 @@ public PackDesc ReadPackDesc() private T ReadNode(T node) where T : IClass { + TryInitializeDecryption(node); + rw ??= new GbxReaderWriter(this); using var _ = logger?.BeginScope("{ClassName} (aux)", ClassManager.GetName(node.GetType())); @@ -1258,6 +1260,8 @@ private T ReadNode(T node) where T : IClass [IgnoreForCodeGeneration] private IClass ReadNode(IClass node) { + TryInitializeDecryption(node); + rw ??= new GbxReaderWriter(this); using var _ = logger?.BeginScope("{ClassName} (aux)", ClassManager.GetName(node.GetType())); @@ -2036,4 +2040,37 @@ public void Dispose() reader.Format = format; } } + + private bool TryInitializeDecryption(IClass node) + { + if (Settings.EncryptionInitializer is null) + { + return false; + } + + var baseType = node.GetType().BaseType ?? throw new ThisShouldNotHappenException(); + + var parentClassId = ClassManager.ClassIds[baseType]; + + if (parentClassId == 0x07031000) + { + parentClassId = 0x07001000; + } + + if (node is CPlugSurfaceGeom) + { + parentClassId = 0x0902B000; + } + + if (baseType == typeof(CGameCtnBlockInfo)) + { + parentClassId = 0x24005000; + } + + var parentClassIDBytes = BitConverter.GetBytes(parentClassId); + + Settings.EncryptionInitializer.Initialize(parentClassIDBytes, 0, 4); + + return true; + } } \ No newline at end of file diff --git a/Src/GBX.NET/Serialization/IEncryptedStream.cs b/Src/GBX.NET/Serialization/IEncryptedStream.cs deleted file mode 100644 index 9361b4743..000000000 --- a/Src/GBX.NET/Serialization/IEncryptedStream.cs +++ /dev/null @@ -1,12 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace GBX.NET.Serialization; - -public interface IEncryptedStream -{ - void Initialize(byte[] data, uint offset, uint count); -} diff --git a/Tools/PakToZip/Program.cs b/Tools/PakToZip/Program.cs index 47ea826df..07d7764ee 100644 --- a/Tools/PakToZip/Program.cs +++ b/Tools/PakToZip/Program.cs @@ -14,8 +14,10 @@ using var fs = File.OpenRead(fileName); using var pak = await Pak.ParseAsync(fs, key); -using var stream = pak.OpenFile(pak.Files.Values.First(x => x.FolderPath.EndsWith("Solid\\"))); +var file = pak.Files.Values.First(x => x.FolderPath.EndsWith("Solid\\")); -var ok = Gbx.Parse(stream, new() { IgnoreExceptionsInBody = true }); +using var stream = pak.OpenFile(file, out var encryptionInitializer); + +var ok = Gbx.Parse(stream, new() { EncryptionInitializer = encryptionInitializer }); Console.WriteLine(); \ No newline at end of file From 8b5f23b5d83f6da7e54e8b088dc3b6849a547af2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Wed, 20 Nov 2024 23:42:48 +0100 Subject: [PATCH 42/86] Add good old zlibstream --- Src/GBX.NET.PAK/BlowfishStream.cs | 4 +- Src/GBX.NET.PAK/Pak.cs | 19 +- Src/GBX.NET.PAK/ZlibDeflateStream.cs | 346 +++++++++++++++++++ Src/GBX.NET/Engines/MwFoundations/CMwNod.cs | 5 +- Src/GBX.NET/Engines/Plug/CPlugSurfaceGeom.cs | 2 + Src/GBX.NET/Serialization/GbxReader.cs | 6 +- Tools/PakToZip/Program.cs | 6 +- 7 files changed, 365 insertions(+), 23 deletions(-) create mode 100644 Src/GBX.NET.PAK/ZlibDeflateStream.cs diff --git a/Src/GBX.NET.PAK/BlowfishStream.cs b/Src/GBX.NET.PAK/BlowfishStream.cs index c9a6b7bdb..40fd5f5d1 100644 --- a/Src/GBX.NET.PAK/BlowfishStream.cs +++ b/Src/GBX.NET.PAK/BlowfishStream.cs @@ -26,8 +26,8 @@ public BlowfishStream(Stream stream, byte[] key, ulong iv) public override bool CanRead => stream.CanRead; public override bool CanWrite => false; public override bool CanSeek => false; - public override long Length => throw new NotSupportedException(); - public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); } + public override long Length => stream.Length; + public override long Position { get => totalIndex; set => throw new NotSupportedException(); } public override void Flush() => stream.Flush(); diff --git a/Src/GBX.NET.PAK/Pak.cs b/Src/GBX.NET.PAK/Pak.cs index 9a77c87b3..6f6fbb0a5 100644 --- a/Src/GBX.NET.PAK/Pak.cs +++ b/Src/GBX.NET.PAK/Pak.cs @@ -165,12 +165,7 @@ public Stream OpenFile(PakFile file, out EncryptionInitializer encryptionInitial stream.Read(ivBuffer, 0, 8); var iv = BitConverter.ToUInt64(ivBuffer, 0); - var data = new byte[file.CompressedSize]; - stream.Read(data, 0, data.Length); - - var ms = new MemoryStream(data); - - var blowfish = new BlowfishStream(ms, key, iv); + var blowfish = new BlowfishStream(stream, key, iv); encryptionInitializer = new EncryptionInitializer(blowfish); @@ -179,12 +174,14 @@ public Stream OpenFile(PakFile file, out EncryptionInitializer encryptionInitial return blowfish; } - if (Gbx.ZLib is null) - { - throw new ZLibNotDefinedException(); - } + return new ZlibDeflateStream(blowfish, false); + } - return Gbx.ZLib.Decompress(blowfish); + [Zomp.SyncMethodGenerator.CreateSyncVersion] + public async Task OpenGbxFileAsync(PakFile file, GbxReadSettings settings = default, CancellationToken cancellationToken = default) + { + using var stream = OpenFile(file, out var encryptionInitializer); + return await Gbx.ParseAsync(stream, settings with { EncryptionInitializer = encryptionInitializer }, cancellationToken); } public void Dispose() diff --git a/Src/GBX.NET.PAK/ZlibDeflateStream.cs b/Src/GBX.NET.PAK/ZlibDeflateStream.cs new file mode 100644 index 000000000..ce61c796a --- /dev/null +++ b/Src/GBX.NET.PAK/ZlibDeflateStream.cs @@ -0,0 +1,346 @@ +using System.Runtime.InteropServices; + +namespace GBX.NET.PAK; + +internal class ZlibDeflateStream : Stream +{ + #region zlib interop + + [StructLayout(LayoutKind.Sequential)] + private class z_stream + { + public IntPtr next_in; + public int avail_in; + public int total_in; + + public IntPtr next_out; + public int avail_out; + public int total_out; + + public IntPtr msg; + public IntPtr state; + + public IntPtr zalloc; + public IntPtr zfree; + public IntPtr opaque; + + public int data_type; + public uint adler; + public uint reserved; + } + + private enum ZFlushType + { + Z_NO_FLUSH = 0, + Z_PARTIAL_FLUSH = 1, + Z_SYNC_FLUSH = 2, + Z_FULL_FLUSH = 3, + Z_FINISH = 4, + Z_BLOCK = 5, + Z_TREES = 6 + } + + private enum ZStatus + { + Z_OK = 0, + Z_STREAM_END = 1, + Z_NEED_DICT = 2, + Z_ERRNO = -1, + Z_STREAM_ERROR = -2, + Z_DATA_ERROR = -3, + Z_MEM_ERROR = -4, + Z_BUF_ERROR = -5, + Z_VERSION_ERROR = -6 + } + + [DllImport("zlib1.dll")] + private static extern ZStatus deflateInit_(z_stream stream, int level, string version, int stream_size); + + [DllImport("zlib1.dll")] + private static extern ZStatus deflate(z_stream stream, ZFlushType flush); + + [DllImport("zlib1.dll")] + private static extern ZStatus deflateEnd(z_stream stream); + + [DllImport("zlib1.dll")] + private static extern ZStatus inflateInit_(z_stream stream, string version, int stream_size); + + [DllImport("zlib1.dll")] + private static extern ZStatus inflate(z_stream stream, ZFlushType flush); + + [DllImport("zlib1.dll")] + private static extern ZStatus inflateEnd(z_stream stream); + + #endregion + + private const int COMPRESSED_BLOCK_SIZE = 0x100; + private const int UNCOMPRESSED_BLOCK_SIZE = 0x400; + + private Stream _innerStream; + private bool _compressing; + private long _position; + + private z_stream _zlibStream; + private byte[] _compressedBuffer; + private byte[] _uncompressedBuffer; + private IntPtr _zlibCompressedMem; + private IntPtr _zlibUncompressedMem; + private int _compressedIndex; + private int _uncompressedIndex; + private int _uncompressedSize; + + private bool _disposed; + + public ZlibDeflateStream(Stream innerStream, bool compressing) + { + _innerStream = innerStream; + _compressing = compressing; + _position = 0; + + _zlibStream = new z_stream(); + _compressedBuffer = new byte[COMPRESSED_BLOCK_SIZE]; + _uncompressedBuffer = new byte[UNCOMPRESSED_BLOCK_SIZE]; + _zlibCompressedMem = Marshal.AllocHGlobal(COMPRESSED_BLOCK_SIZE); + _zlibUncompressedMem = Marshal.AllocHGlobal(UNCOMPRESSED_BLOCK_SIZE); + if (compressing) + { + VerifyStatus(deflateInit_(_zlibStream, 8, "1.2.5", Marshal.SizeOf(_zlibStream))); + } + else + { + VerifyStatus(inflateInit_(_zlibStream, "1.2.5", Marshal.SizeOf(_zlibStream))); + } + } + + public override bool CanRead + { + get { return !_compressing; } + } + + public override bool CanSeek + { + get { return false; } + } + + public override bool CanWrite + { + get { return _compressing; } + } + + public override void Flush() + { + + } + + public override long Length + { + get { throw new NotSupportedException(); } + } + + public override long Position + { + get + { + return _position; + } + set + { + throw new NotSupportedException(); + } + } + + public override int Read(byte[] buffer, int offset, int count) + { + if (_compressing) + throw new NotSupportedException(); + + int left = count; + while (left > 0) + { + if (_uncompressedIndex == _uncompressedSize) + { + _uncompressedIndex = 0; + _uncompressedSize = 0; + while (_uncompressedSize < UNCOMPRESSED_BLOCK_SIZE) + { + if (_zlibStream.avail_in == 0) + { + int inChunkSize = _innerStream.Read(_compressedBuffer, 0, COMPRESSED_BLOCK_SIZE); + SetInflateInput(_compressedBuffer, 0, inChunkSize); + } + bool streamEnd; + _uncompressedSize += Inflate(_uncompressedBuffer, _uncompressedSize, + UNCOMPRESSED_BLOCK_SIZE - _uncompressedSize, out streamEnd); + if (streamEnd) + break; + } + } + int outChunkSize = Math.Min(_uncompressedSize - _uncompressedIndex, left); + Array.Copy(_uncompressedBuffer, _uncompressedIndex, buffer, offset, outChunkSize); + _uncompressedIndex += outChunkSize; + _position += outChunkSize; + offset += outChunkSize; + left -= outChunkSize; + } + + return count - left; + } + + public override long Seek(long offset, SeekOrigin origin) + { + throw new NotSupportedException(); + } + + public override void SetLength(long value) + { + throw new NotSupportedException(); + } + + public override void Write(byte[] buffer, int offset, int count) + { + if (!_compressing) + throw new NotSupportedException(); + + int left = count; + while (left > 0) + { + int inChunkSize = Math.Min(UNCOMPRESSED_BLOCK_SIZE - _uncompressedIndex, left); + Array.Copy(buffer, offset, _uncompressedBuffer, _uncompressedIndex, inChunkSize); + _uncompressedIndex += inChunkSize; + _position += inChunkSize; + offset += inChunkSize; + left -= inChunkSize; + + if (_uncompressedIndex == UNCOMPRESSED_BLOCK_SIZE) + { + SetDeflateInput(_uncompressedBuffer, 0, UNCOMPRESSED_BLOCK_SIZE); + FlushDeflateOutput(false); + _uncompressedIndex = 0; + } + } + } + + private void FlushDeflateOutput(bool finish) + { + bool finished = false; + while (!NeedInput() || (finish && !finished)) + { + int deflated = Deflate(_compressedBuffer, 0, COMPRESSED_BLOCK_SIZE, finish, out finished); + _innerStream.Write(_compressedBuffer, 0, deflated); + } + } + + private bool NeedInput() + { + return _zlibStream.avail_in == 0; + } + + private void SetInflateInput(byte[] buffer, int offset, int length) + { + _zlibStream.avail_in = length; + _zlibStream.next_in = _zlibCompressedMem; + Marshal.Copy(buffer, offset, _zlibCompressedMem, length); + } + + private int Inflate(byte[] buffer, int offset, int length, out bool streamEnd) + { + streamEnd = false; + + if (_zlibStream.avail_in == 0) + return 0; + + _zlibStream.avail_out = length; + _zlibStream.next_out = _zlibUncompressedMem; + ZStatus status = VerifyStatus(inflate(_zlibStream, ZFlushType.Z_SYNC_FLUSH)); + Marshal.Copy(_zlibUncompressedMem, buffer, offset, length); + streamEnd = (status == ZStatus.Z_STREAM_END); + return (int)((ulong)_zlibStream.next_out - (ulong)_zlibUncompressedMem); + } + + private void SetDeflateInput(byte[] buffer, int offset, int length) + { + _zlibStream.avail_in = length; + _zlibStream.next_in = _zlibUncompressedMem; + Marshal.Copy(buffer, offset, _zlibUncompressedMem, length); + } + + private int Deflate(byte[] buffer, int offset, int length, bool finish, out bool finished) + { + finished = false; + _zlibStream.avail_out = length; + _zlibStream.next_out = _zlibCompressedMem; + if (finish) + { + ZStatus status = VerifyStatus(deflate(_zlibStream, ZFlushType.Z_FINISH)); + finished = (status == ZStatus.Z_STREAM_END); + } + else + { + if (_zlibStream.avail_in == 0) + return 0; + + VerifyStatus(deflate(_zlibStream, ZFlushType.Z_SYNC_FLUSH)); + } + int deflated = (int)_zlibStream.next_out - (int)_zlibCompressedMem; + Marshal.Copy(_zlibCompressedMem, buffer, offset, deflated); + return deflated; + } + + private static ZStatus VerifyStatus(ZStatus status) + { + switch (status) + { + case ZStatus.Z_BUF_ERROR: + throw new Exception("Buffer error"); + + case ZStatus.Z_DATA_ERROR: + throw new Exception("Data error"); + + case ZStatus.Z_ERRNO: + throw new Exception("Filesystem error"); + + case ZStatus.Z_MEM_ERROR: + throw new Exception("Memory error"); + + case ZStatus.Z_STREAM_ERROR: + throw new Exception("Stream error"); + + case ZStatus.Z_VERSION_ERROR: + throw new Exception("Version error"); + } + return status; + } + + protected override void Dispose(bool disposing) + { + if (!_disposed) + { + if (disposing) + { + if (_compressing) + { + if (_uncompressedIndex > 0) + { + SetDeflateInput(_uncompressedBuffer, 0, _uncompressedIndex); + FlushDeflateOutput(false); + } + FlushDeflateOutput(true); + } + _innerStream.Dispose(); + } + + if (_compressing) + VerifyStatus(deflateEnd(_zlibStream)); + else + VerifyStatus(inflateEnd(_zlibStream)); + + Marshal.FreeHGlobal(_zlibCompressedMem); + Marshal.FreeHGlobal(_zlibUncompressedMem); + _zlibCompressedMem = IntPtr.Zero; + _zlibUncompressedMem = IntPtr.Zero; + + _disposed = true; + } + base.Dispose(disposing); + } +} \ No newline at end of file diff --git a/Src/GBX.NET/Engines/MwFoundations/CMwNod.cs b/Src/GBX.NET/Engines/MwFoundations/CMwNod.cs index a519dd0f9..f21d0da17 100644 --- a/Src/GBX.NET/Engines/MwFoundations/CMwNod.cs +++ b/Src/GBX.NET/Engines/MwFoundations/CMwNod.cs @@ -3,7 +3,6 @@ using Microsoft.Extensions.Logging; using System.Diagnostics; using System.Text; -using System.Xml.Linq; namespace GBX.NET.Engines.MwFoundations; @@ -22,6 +21,8 @@ static void IClass.Read(T node, GbxReaderWriter rw) { var r = rw.Reader ?? throw new Exception("Reader is required but not available."); + r.TryInitializeDecryption(node); + var prevChunkId = default(uint?); while (true) @@ -323,6 +324,8 @@ internal virtual void Read(GbxReaderWriter rw) { var r = rw.Reader ?? throw new Exception("Reader is required but not available."); + r.TryInitializeDecryption(this); + var prevChunkId = default(uint?); while (true) diff --git a/Src/GBX.NET/Engines/Plug/CPlugSurfaceGeom.cs b/Src/GBX.NET/Engines/Plug/CPlugSurfaceGeom.cs index 4980d2abe..165537b81 100644 --- a/Src/GBX.NET/Engines/Plug/CPlugSurfaceGeom.cs +++ b/Src/GBX.NET/Engines/Plug/CPlugSurfaceGeom.cs @@ -57,6 +57,8 @@ public override void ReadWrite(CPlugSurfaceGeom n, GbxReaderWriter rw) if (rw.Reader is not null) { + rw.Reader.Settings.EncryptionInitializer?.Initialize(BitConverter.GetBytes(U02.X - U02.X2), 0, 4); + n.Surf = CPlugSurface.ReadSurf(rw.Reader, version: 0); } diff --git a/Src/GBX.NET/Serialization/GbxReader.cs b/Src/GBX.NET/Serialization/GbxReader.cs index 7d4354c3e..01ec07b05 100644 --- a/Src/GBX.NET/Serialization/GbxReader.cs +++ b/Src/GBX.NET/Serialization/GbxReader.cs @@ -1242,8 +1242,6 @@ public PackDesc ReadPackDesc() private T ReadNode(T node) where T : IClass { - TryInitializeDecryption(node); - rw ??= new GbxReaderWriter(this); using var _ = logger?.BeginScope("{ClassName} (aux)", ClassManager.GetName(node.GetType())); @@ -1260,8 +1258,6 @@ private T ReadNode(T node) where T : IClass [IgnoreForCodeGeneration] private IClass ReadNode(IClass node) { - TryInitializeDecryption(node); - rw ??= new GbxReaderWriter(this); using var _ = logger?.BeginScope("{ClassName} (aux)", ClassManager.GetName(node.GetType())); @@ -2041,7 +2037,7 @@ public void Dispose() } } - private bool TryInitializeDecryption(IClass node) + internal bool TryInitializeDecryption(IClass node) { if (Settings.EncryptionInitializer is null) { diff --git a/Tools/PakToZip/Program.cs b/Tools/PakToZip/Program.cs index 07d7764ee..38e14173e 100644 --- a/Tools/PakToZip/Program.cs +++ b/Tools/PakToZip/Program.cs @@ -14,10 +14,8 @@ using var fs = File.OpenRead(fileName); using var pak = await Pak.ParseAsync(fs, key); -var file = pak.Files.Values.First(x => x.FolderPath.EndsWith("Solid\\")); +var file = pak.Files.Values.First(x => x.Name == "B2FC497BF7F81AB02D01FE8FB2F707CD8F"); -using var stream = pak.OpenFile(file, out var encryptionInitializer); - -var ok = Gbx.Parse(stream, new() { EncryptionInitializer = encryptionInitializer }); +var gbx = await pak.OpenGbxFileAsync(file); Console.WriteLine(); \ No newline at end of file From 8c6b466aa5262f5049f25b91c4674bc739b98836 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Thu, 21 Nov 2024 00:20:54 +0100 Subject: [PATCH 43/86] Fix casing --- Tools/PakToZip/PakToZip.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/PakToZip/PakToZip.csproj b/Tools/PakToZip/PakToZip.csproj index dbab18fac..c70d85c54 100644 --- a/Tools/PakToZip/PakToZip.csproj +++ b/Tools/PakToZip/PakToZip.csproj @@ -9,7 +9,7 @@ - + From c5f1962724f8759b055b2a68e054c465d1d0f870 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Sun, 24 Nov 2024 01:30:14 +0100 Subject: [PATCH 44/86] Link CPlugCrystal 0x006 (lightmaps) with vertices --- Src/GBX.NET/Engines/Plug/CPlugCrystal.chunkl | 7 - Src/GBX.NET/Engines/Plug/CPlugCrystal.cs | 135 ++++++++++++++++++- 2 files changed, 131 insertions(+), 11 deletions(-) diff --git a/Src/GBX.NET/Engines/Plug/CPlugCrystal.chunkl b/Src/GBX.NET/Engines/Plug/CPlugCrystal.chunkl index 9989a055b..c9b9be0aa 100644 --- a/Src/GBX.NET/Engines/Plug/CPlugCrystal.chunkl +++ b/Src/GBX.NET/Engines/Plug/CPlugCrystal.chunkl @@ -16,13 +16,6 @@ CPlugCrystal 0x09003000 0x005 [MP4, TM2020] // layers 0x006 [MP4.v0, TM2020.v1] // lightmap UVs - version - v0= - vec2[] - v1+ - uint[] // two Int16 technically - v2+ - optimizedint[] 0x007 [MP4, TM2020] // smoothing groups version diff --git a/Src/GBX.NET/Engines/Plug/CPlugCrystal.cs b/Src/GBX.NET/Engines/Plug/CPlugCrystal.cs index 825b49abd..1bb66f61a 100644 --- a/Src/GBX.NET/Engines/Plug/CPlugCrystal.cs +++ b/Src/GBX.NET/Engines/Plug/CPlugCrystal.cs @@ -25,6 +25,26 @@ public void ExportToObj(string objFilePath, string mtlFilePath, int? mergeVertic ExportToObj(objWriter, mtlWriter, mergeVerticesDigitThreshold); } + public Vec2[] GetLightmapCoords() + { + return Layers.OfType() + .Select(x => x.Crystal) + .OfType() + .SelectMany(c => c.Faces) + .SelectMany(f => f.Vertices.Select(v => v.LightmapCoord)) + .ToArray(); + } + + public Vec2[][] GetLightmapCoordFaces() + { + return Layers.OfType() + .Select(x => x.Crystal) + .OfType() + .SelectMany(c => c.Faces) + .Select(f => f.Vertices.Select(v => v.LightmapCoord).ToArray()) + .ToArray(); + } + public partial class Chunk09003000 : IVersionable { public int Version { get; set; } @@ -130,6 +150,113 @@ public override void Write(CPlugCrystal n, GbxWriter w) } } + public partial class Chunk09003006 : IVersionable + { + public int Version { get; set; } + + public override void Read(CPlugCrystal n, GbxReader r) + { + Version = r.ReadInt32(); + + var faces = n.Layers.OfType() + .Select(x => x.Crystal) + .OfType() + .SelectMany(c => c.Faces); + + var lightmapCoordCount = r.ReadInt32(); + + if (Version == 0) + { + var counter = 0; + foreach (var face in faces) + { + for (int i = 0; i < face.Vertices.Length; i++) + { + face.Vertices[i] = face.Vertices[i] with { LightmapCoord = r.ReadVec2() }; + counter++; + + if (counter > lightmapCoordCount) + { + throw new Exception("LightmapCoord count exceeded"); + } + } + } + + if (counter != lightmapCoordCount) + { + throw new Exception("LightmapCoord count mismatch"); + } + } + + if (Version >= 1) + { + var lightmaps = new Vec2[lightmapCoordCount]; + + for (int i = 0; i < lightmapCoordCount; i++) + { + lightmaps[i] = (r.ReadUInt16() / (float)ushort.MaxValue, r.ReadUInt16() / (float)ushort.MaxValue); + } + + var indices = Version >= 2 ? r.ReadArrayOptimizedInt() : null; + + var counter = 0; + foreach (var face in faces) + { + for (int i = 0; i < face.Vertices.Length; i++) + { + face.Vertices[i] = face.Vertices[i] with { LightmapCoord = lightmaps[indices?[counter++] ?? counter] }; + + if (counter > lightmapCoordCount) + { + throw new Exception("LightmapCoord count exceeded"); + } + } + } + + if (counter != lightmapCoordCount) + { + throw new Exception("LightmapCoord count mismatch"); + } + } + } + + public override void Write(CPlugCrystal n, GbxWriter w) + { + w.Write(Version); + + var faces = n.Layers.OfType() + .Select(x => x.Crystal) + .OfType() + .SelectMany(c => c.Faces); + + var lightmaps = faces.SelectMany(f => f.Vertices.Select(v => v.LightmapCoord)).ToArray(); + + w.Write(lightmaps.Length); + + if (Version == 0) + { + foreach (var lightmap in lightmaps) + { + w.Write(lightmap); + } + } + + if (Version >= 1) + { + foreach (var lightmap in lightmaps) + { + w.Write((ushort)(lightmap.X * ushort.MaxValue)); + w.Write((ushort)(lightmap.Y * ushort.MaxValue)); + } + + // subject to optimization due to IndexOf complexity + var indices = faces.SelectMany(f => f.Vertices.Select(v => Array.IndexOf(lightmaps, v.LightmapCoord))).ToArray(); + + w.WriteArrayOptimizedInt(indices); + } + } + } + [ArchiveGenerationOptions(StructureKind = StructureKind.SeparateReadAndWrite)] public abstract partial class Layer; @@ -343,7 +470,7 @@ public void Read(GbxReader r, CPlugCrystal n, int v = 0) // this doesnt sound right for (var j = 0; j < uvCount; j++) { - vertices[j] = new Vertex(inds[j], TexCoord: r.ReadVec2()); + vertices[j] = new Vertex(inds[j], TexCoord: r.ReadVec2(), default); } u01 = r.ReadVec3(); // normal? @@ -352,14 +479,14 @@ public void Read(GbxReader r, CPlugCrystal n, int v = 0) { for (var j = 0; j < vertices.Length; j++) { - vertices[j] = new Vertex(inds[j], TexCoord: r.ReadVec2()); + vertices[j] = new Vertex(inds[j], TexCoord: r.ReadVec2(), default); } } else { for (var j = 0; j < vertCount; j++) { - vertices[j] = new Vertex(inds[j], TexCoord: texCoords[texCoordIndices[faceVertexIndex++]]); + vertices[j] = new Vertex(inds[j], TexCoord: texCoords[texCoordIndices[faceVertexIndex++]], default); } } @@ -651,5 +778,5 @@ public void Write(GbxWriter w, CPlugCrystal n, int v = 0) public sealed record Face(Vertex[] Vertices, Part Group, Material? Material, Vec3? U01); - public readonly record struct Vertex(int Index, Vec2 TexCoord); + public readonly record struct Vertex(int Index, Vec2 TexCoord, Vec2 LightmapCoord); } From 5c87152619609c4356995f1a81f5db735bef938a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Sun, 24 Nov 2024 01:32:23 +0100 Subject: [PATCH 45/86] Fix versioning when writing --- Src/GBX.NET/Engines/Plug/CPlugCrystal.cs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Src/GBX.NET/Engines/Plug/CPlugCrystal.cs b/Src/GBX.NET/Engines/Plug/CPlugCrystal.cs index 1bb66f61a..c783493f2 100644 --- a/Src/GBX.NET/Engines/Plug/CPlugCrystal.cs +++ b/Src/GBX.NET/Engines/Plug/CPlugCrystal.cs @@ -249,10 +249,13 @@ public override void Write(CPlugCrystal n, GbxWriter w) w.Write((ushort)(lightmap.Y * ushort.MaxValue)); } - // subject to optimization due to IndexOf complexity - var indices = faces.SelectMany(f => f.Vertices.Select(v => Array.IndexOf(lightmaps, v.LightmapCoord))).ToArray(); + if (Version >= 2) + { + // subject to optimization due to IndexOf complexity + var indices = faces.SelectMany(f => f.Vertices.Select(v => Array.IndexOf(lightmaps, v.LightmapCoord))).ToArray(); - w.WriteArrayOptimizedInt(indices); + w.WriteArrayOptimizedInt(indices); + } } } } From 961d48b608a4e33a3e4b3ca3984830f0251f2cc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Sun, 24 Nov 2024 02:13:48 +0100 Subject: [PATCH 46/86] Fixes of invisible meshes and repeating coords --- Src/GBX.NET/Engines/Plug/CPlugCrystal.cs | 47 +++++++++++++++++------- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/Src/GBX.NET/Engines/Plug/CPlugCrystal.cs b/Src/GBX.NET/Engines/Plug/CPlugCrystal.cs index c783493f2..a5fbfd616 100644 --- a/Src/GBX.NET/Engines/Plug/CPlugCrystal.cs +++ b/Src/GBX.NET/Engines/Plug/CPlugCrystal.cs @@ -159,6 +159,7 @@ public override void Read(CPlugCrystal n, GbxReader r) Version = r.ReadInt32(); var faces = n.Layers.OfType() + .Where(x => x.IsVisible) .Select(x => x.Crystal) .OfType() .SelectMany(c => c.Faces); @@ -190,30 +191,41 @@ public override void Read(CPlugCrystal n, GbxReader r) if (Version >= 1) { - var lightmaps = new Vec2[lightmapCoordCount]; + var lightmapCoords = new Vec2[lightmapCoordCount]; for (int i = 0; i < lightmapCoordCount; i++) { - lightmaps[i] = (r.ReadUInt16() / (float)ushort.MaxValue, r.ReadUInt16() / (float)ushort.MaxValue); + lightmapCoords[i] = (r.ReadUInt16() / (float)ushort.MaxValue, r.ReadUInt16() / (float)ushort.MaxValue); } + // indices of lightmap coords var indices = Version >= 2 ? r.ReadArrayOptimizedInt() : null; + var lightmapCount = indices?.Length ?? lightmapCoordCount; + var counter = 0; foreach (var face in faces) { for (int i = 0; i < face.Vertices.Length; i++) { - face.Vertices[i] = face.Vertices[i] with { LightmapCoord = lightmaps[indices?[counter++] ?? counter] }; + try + { + var index = indices?[counter++] ?? counter; + face.Vertices[i] = face.Vertices[i] with { LightmapCoord = lightmapCoords[index] }; - if (counter > lightmapCoordCount) + if (counter > lightmapCount) + { + throw new Exception("LightmapCoord count exceeded"); + } + } + catch { - throw new Exception("LightmapCoord count exceeded"); + } } } - if (counter != lightmapCoordCount) + if (counter != lightmapCount) { throw new Exception("LightmapCoord count mismatch"); } @@ -225,17 +237,18 @@ public override void Write(CPlugCrystal n, GbxWriter w) w.Write(Version); var faces = n.Layers.OfType() + .Where(x => x.IsVisible) .Select(x => x.Crystal) .OfType() .SelectMany(c => c.Faces); - var lightmaps = faces.SelectMany(f => f.Vertices.Select(v => v.LightmapCoord)).ToArray(); + var lightmapCoords = faces.SelectMany(f => f.Vertices.Select(v => v.LightmapCoord)).ToArray(); - w.Write(lightmaps.Length); + w.Write(lightmapCoords.Length); if (Version == 0) { - foreach (var lightmap in lightmaps) + foreach (var lightmap in lightmapCoords) { w.Write(lightmap); } @@ -243,7 +256,7 @@ public override void Write(CPlugCrystal n, GbxWriter w) if (Version >= 1) { - foreach (var lightmap in lightmaps) + foreach (var lightmap in lightmapCoords) { w.Write((ushort)(lightmap.X * ushort.MaxValue)); w.Write((ushort)(lightmap.Y * ushort.MaxValue)); @@ -251,8 +264,10 @@ public override void Write(CPlugCrystal n, GbxWriter w) if (Version >= 2) { - // subject to optimization due to IndexOf complexity - var indices = faces.SelectMany(f => f.Vertices.Select(v => Array.IndexOf(lightmaps, v.LightmapCoord))).ToArray(); + var lightmapCoordIndices = lightmapCoords.Distinct() + .Select((x, i) => (x, i)) + .ToDictionary(x => x.x, x => x.i); + var indices = faces.SelectMany(f => f.Vertices.Select(v => lightmapCoordIndices[v.LightmapCoord])).ToArray(); w.WriteArrayOptimizedInt(indices); } @@ -779,7 +794,13 @@ public void Write(GbxWriter w, CPlugCrystal n, int v = 0) } } - public sealed record Face(Vertex[] Vertices, Part Group, Material? Material, Vec3? U01); + public sealed record Face(Vertex[] Vertices, Part Group, Material? Material, Vec3? U01) + { + public override string ToString() + { + return $"{Vertices.Length} vertices, material: {Material?.MaterialUserInst?.Link ?? Material?.MaterialName ?? "none"}"; + } + } public readonly record struct Vertex(int Index, Vec2 TexCoord, Vec2 LightmapCoord); } From 9a18c7c74da017c7af753e5c84898ec38b848fcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Sun, 24 Nov 2024 02:25:24 +0100 Subject: [PATCH 47/86] Fix write --- Src/GBX.NET/Engines/Plug/CPlugCrystal.cs | 34 ++++++++++-------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/Src/GBX.NET/Engines/Plug/CPlugCrystal.cs b/Src/GBX.NET/Engines/Plug/CPlugCrystal.cs index a5fbfd616..37064a2b8 100644 --- a/Src/GBX.NET/Engines/Plug/CPlugCrystal.cs +++ b/Src/GBX.NET/Engines/Plug/CPlugCrystal.cs @@ -208,19 +208,12 @@ public override void Read(CPlugCrystal n, GbxReader r) { for (int i = 0; i < face.Vertices.Length; i++) { - try - { - var index = indices?[counter++] ?? counter; - face.Vertices[i] = face.Vertices[i] with { LightmapCoord = lightmapCoords[index] }; + var index = indices?[counter++] ?? counter; + face.Vertices[i] = face.Vertices[i] with { LightmapCoord = lightmapCoords[index] }; - if (counter > lightmapCount) - { - throw new Exception("LightmapCoord count exceeded"); - } - } - catch + if (counter > lightmapCount) { - + throw new Exception("LightmapCoord count exceeded"); } } } @@ -242,21 +235,22 @@ public override void Write(CPlugCrystal n, GbxWriter w) .OfType() .SelectMany(c => c.Faces); - var lightmapCoords = faces.SelectMany(f => f.Vertices.Select(v => v.LightmapCoord)).ToArray(); - - w.Write(lightmapCoords.Length); + var lightmapCoords = faces.SelectMany(f => f.Vertices.Select(v => v.LightmapCoord)); if (Version == 0) { - foreach (var lightmap in lightmapCoords) - { - w.Write(lightmap); - } + w.WriteArray(lightmapCoords.ToArray()); } if (Version >= 1) { - foreach (var lightmap in lightmapCoords) + var lightmapCoordArray = Version == 1 + ? lightmapCoords.ToArray() + : lightmapCoords.Distinct().ToArray(); + + w.Write(lightmapCoordArray.Length); + + foreach (var lightmap in lightmapCoordArray) { w.Write((ushort)(lightmap.X * ushort.MaxValue)); w.Write((ushort)(lightmap.Y * ushort.MaxValue)); @@ -264,7 +258,7 @@ public override void Write(CPlugCrystal n, GbxWriter w) if (Version >= 2) { - var lightmapCoordIndices = lightmapCoords.Distinct() + var lightmapCoordIndices = lightmapCoordArray .Select((x, i) => (x, i)) .ToDictionary(x => x.x, x => x.i); var indices = faces.SelectMany(f => f.Vertices.Select(v => lightmapCoordIndices[v.LightmapCoord])).ToArray(); From b7e379a1f1d631ba1be2abcd17bd89a2d12db1d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Sun, 24 Nov 2024 02:28:40 +0100 Subject: [PATCH 48/86] Fix ReadGbxMagic test --- Tests/GBX.NET.Tests/Unit/Serialization/GbxReaderTests.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Tests/GBX.NET.Tests/Unit/Serialization/GbxReaderTests.cs b/Tests/GBX.NET.Tests/Unit/Serialization/GbxReaderTests.cs index 5cf6889a2..1db72bbc6 100644 --- a/Tests/GBX.NET.Tests/Unit/Serialization/GbxReaderTests.cs +++ b/Tests/GBX.NET.Tests/Unit/Serialization/GbxReaderTests.cs @@ -79,10 +79,8 @@ public void ReadGbxMagic_MissingData_ThrowsEndOfStream() ms.Write([(byte)'G', (byte)'B']); ms.Position = 0; - // Act - var result = r.ReadGbxMagic(); - - Assert.False(result); + // Act & Assert + Assert.Throws(() => r.ReadGbxMagic()); } [Fact] From 3e15b2b9f9638878f06b398779ce9861568870a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Sun, 24 Nov 2024 03:44:59 +0100 Subject: [PATCH 49/86] Add zlib info to readme --- README.md | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/README.md b/README.md index b0a626df7..8a64b26c1 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ For any questions, open an issue, join the [GameBox Sandbox Discord server](http - [Processing multiple Gbx types](#processing-multiple-gbx-types) - [Read a large amount of replay metadata quickly](#read-a-large-amount-of-replay-metadata-quickly) - [Tool framework](#tool-framework) +- [Zlib compression in Gbx](#zlib-compression-in-gbx) - [Clarity](#clarity) - [Differences between `Gbx.Parse/Header/Node`](#differences-between-gbxparseheadernode) - [Do not repeat `gbx.Node.[any]` too often!](#do-not-repeat-gbxnodeany-too-often) @@ -376,6 +377,45 @@ Tool class accepts input through constructors (best one is picked according to i Samples are available [here](Samples/Tool/). +## Zlib compression in Gbx + +There are a few places where Gbx includes additional zlib-compressed data: + +- Ghost samples + - `CGameGhost.SampleData` +- Record data (general replay data or TM2020 ghost samples) + - `CGameCtnReplayRecord.RecordData`, `CGameCtnGhost.RecordData`, `CGameCtnMediaBlockEntity.RecordData` +- Lightmap data of a map + - `CGameCtnChallenge.LightmapCache`, additional lightmap parameters + +GBX.NET does not include zlib algorithm by default to read this data. Properties will often return `null` silently, but you can hook onto more details about this if you specify `Logger` in the parse methods. + +To "unlock" this data, you need to specify zlib implementation: + +Command line: + +``` +dotnet add package GBX.NET.ZLib +``` + +C# code: + +```cs +using GBX.NET; +using GBX.NET.ZLib; + +Gbx.ZLib = new ZLib(); +``` + +> [!NOTE] +> Zlib-compressed data is **currently read-only for ghost samples and record data**, lightmap data is the only zlib data that can be modified with GBX.NET. Write support for ghost samples and record data is planned for 2.1 or 2.2. + +The data is often stored in properties of type `CompressedData` which are byte arrays with additional uncompressed data size for validation. If there's a property with this type (`CGameCtnChallenge.LightmapCacheData` for example), zlib data will be stored there no matter if the zlib implementation is included, so read/write consistency is guaranteed. + +> The current reason why the zlib implementation is split similarly to LZO (even when zlib license is perfectly fine) is that there aren't any good official solutions by Microsoft that would work for .NET Standard 2 (new `ZlibStream` is promising but .NET 6+ only, still not tested enough if reliable), and third party solutions are also of a questionable quality. But this topic is continously under attention and the zlib implementation is slowly improving. + +GBX.NET.PAK uses a separate zlib solution due to very specific patterns to follow during decryption + decompression in .pak data. + ## Clarity This section describes best practices to keep your projects clean when using GBX.NET 2. From 82b53c8498c0ddc72656433f6ec01b607c6669cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Sun, 24 Nov 2024 03:51:57 +0100 Subject: [PATCH 50/86] Correct the gnu gpl version --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8a64b26c1..d3ea688c6 100644 --- a/README.md +++ b/README.md @@ -182,7 +182,7 @@ Gbx.LZO = new Lzo(); You should run this line of code **only once** for the whole program lifetime. -The compression logic is split up from the read/write logic to **allow GBX.NET 2 library to be distributed under the MIT license**, as Oberhumer distributes the open source version of LZO under the GNU GPL v3. Therefore, using GBX.NET.LZO 2 requires you to license your project under the GNU GPL v3, see [License](#license). +The compression logic is split up from the read/write logic to **allow GBX.NET 2 library to be distributed under the MIT license**, as Oberhumer distributes the open source version of LZO under the GNU GPL v2+. Therefore, using GBX.NET.LZO 2 requires you to license your project under the GNU GPL v3, see [License](#license). **Gbx header is not compressed** and can contain useful information (icon data, replay time, ...), and also many of the **internal Gbx files from Pak files are not compressed**, so you can avoid LZO for these purposes. From ca7e51aa0be9c9360596177b1a7e4b86b9622423 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Sun, 8 Dec 2024 01:15:16 +0100 Subject: [PATCH 51/86] Fix embedded items being defined incorrectly --- Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs b/Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs index 5be4c7d85..8da322861 100644 --- a/Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs +++ b/Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs @@ -1753,6 +1753,11 @@ public override void Write(CGameCtnChallenge n, GbxWriter w) foreach (var entry in zip.Entries) { + if (!entry.FullName.StartsWith("Items/")) + { + continue; + } + using var entryStream = entry.Open(); try @@ -1761,12 +1766,19 @@ public override void Write(CGameCtnChallenge n, GbxWriter w) if (nodeHeader is CGameItemModel { Ident: not null } itemModel) { - itemModelList.Add(itemModel.Ident); + itemModelList.Add(itemModel.Ident with + { + Id = entry.FullName.Replace('/', '\\').Substring("Items/".Length) + }); } + + // CGameItemModel.Ident is also often renamed inside the Gbx file + // so if this is an issue to match the Ident, read the gbx fully, change Ident, and save + // do so only if it doesn't match with entry file name, to optimize the process } catch { - + // TODO: log } } From c4358698a39fc3701844e6f185003c4325aaa31b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Sun, 8 Dec 2024 01:44:03 +0100 Subject: [PATCH 52/86] Fix club items case --- Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs | 38 +++++++++++++++---- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs b/Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs index 8da322861..eb3f82d79 100644 --- a/Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs +++ b/Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs @@ -1753,10 +1753,9 @@ public override void Write(CGameCtnChallenge n, GbxWriter w) foreach (var entry in zip.Entries) { - if (!entry.FullName.StartsWith("Items/")) - { - continue; - } + const string itemsPrefix = "Items/"; + const string clubItemsPrefix = "ClubItems/"; + //const string favoriteClubItemsPrefix = "/MemoryTemp/FavoriteClubItems"; using var entryStream = entry.Open(); @@ -1764,14 +1763,37 @@ public override void Write(CGameCtnChallenge n, GbxWriter w) { var nodeHeader = Gbx.ParseHeaderNode(entryStream); - if (nodeHeader is CGameItemModel { Ident: not null } itemModel) + if (nodeHeader is not CGameItemModel itemModel) { - itemModelList.Add(itemModel.Ident with + continue; + } + + if (itemModel.Ident is null || itemModel.ItemType == CGameItemModel.EItemType.Block) + { + continue; + } + + var ident = itemModel.Ident; + + if (entry.FullName.StartsWith(itemsPrefix)) + { + ident = ident with { Id = entry.FullName.Replace('/', '\\').Substring(itemsPrefix.Length) }; + } + + if (entry.FullName.StartsWith(clubItemsPrefix)) + { + ident = ident with { - Id = entry.FullName.Replace('/', '\\').Substring("Items/".Length) - }); +#if NET6_0_OR_GREATER + Id = string.Concat("club:", entry.FullName.Replace('/', '\\').AsSpan(clubItemsPrefix.Length)) +#else + Id = "club:" + entry.FullName.Replace('/', '\\').Substring(clubItemsPrefix.Length) +#endif + }; } + itemModelList.Add(ident); + // CGameItemModel.Ident is also often renamed inside the Gbx file // so if this is an issue to match the Ident, read the gbx fully, change Ident, and save // do so only if it doesn't match with entry file name, to optimize the process From 3eb67a095162eec334820d5e488939cf70334d53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Sun, 8 Dec 2024 20:50:22 +0100 Subject: [PATCH 53/86] Add more MD5 methods and Compute136 --- Src/GBX.NET.Crypto/MD5.cs | 64 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/Src/GBX.NET.Crypto/MD5.cs b/Src/GBX.NET.Crypto/MD5.cs index 4bbe8abd4..68ec38505 100644 --- a/Src/GBX.NET.Crypto/MD5.cs +++ b/Src/GBX.NET.Crypto/MD5.cs @@ -14,6 +14,18 @@ public static byte[] Compute(byte[] data) #endif } +#if NET6_0_OR_GREATER + public static byte[] Compute(Span data) + { + return System.Security.Cryptography.MD5.HashData(data); + } + + public static int Compute(Span data, Span destination) + { + return System.Security.Cryptography.MD5.HashData(data, destination); + } +#endif + public static byte[] Compute(string data) { return Compute(Encoding.ASCII.GetBytes(data)); @@ -42,4 +54,56 @@ public static async Task ComputeAsync(string data, CancellationToken can { return await ComputeAsync(Encoding.ASCII.GetBytes(data), cancellationToken); } + + public static string Compute136(string text) + { +#if NET6_0_OR_GREATER + var charSpan = text.AsSpan(); + Span loweredCharSpan = stackalloc char[text.Length]; + if (charSpan.ToLowerInvariant(loweredCharSpan) != text.Length) + { + loweredCharSpan = text.ToLowerInvariant().ToArray(); + } + Span bytes = stackalloc byte[text.Length * 2]; + if (Encoding.UTF8.GetBytes(loweredCharSpan, bytes) != bytes.Length) + { + bytes = Encoding.UTF8.GetBytes(text.ToLowerInvariant()); + } + Span hash = stackalloc byte[17]; + if (Compute(bytes, hash.Slice(1)) != 16) + { + throw new InvalidOperationException(); + } +#else + var lowered = text.ToLower(); + var bytes = Encoding.UTF8.GetBytes(lowered); + var hashWithoutLength = Compute(bytes); + var hash = new byte[17]; + Buffer.BlockCopy(hashWithoutLength, 0, hash, 1, hashWithoutLength.Length); +#endif + hash[0] = (byte)bytes.Length; + + return ToHex(hash).ToString(); + } + + private static Span ToHex(Span value) + { + var str = new char[value.Length * 2]; + + for (var i = 0; i < value.Length; i++) + { + var hex1 = HexIntToChar((byte)(value[i] % 16)); + var hex2 = HexIntToChar((byte)(value[i] / 16)); + + str[i * 2] = hex1; + str[i * 2 + 1] = hex2; + } + + return str; + } + + private static char HexIntToChar(byte v) + { + return v < 10 ? (char)(v + 48) : (char)(v + 55); + } } From 836e46c4d0d4ff49acced946dd0afad12c4b0cce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Sun, 8 Dec 2024 20:55:05 +0100 Subject: [PATCH 54/86] Add BruteforceHashFileNamesAsync --- Src/GBX.NET.PAK/Pak.cs | 102 ++++++++++++++++++++++++++++++++++++-- Tools/PakToZip/Program.cs | 6 ++- 2 files changed, 103 insertions(+), 5 deletions(-) diff --git a/Src/GBX.NET.PAK/Pak.cs b/Src/GBX.NET.PAK/Pak.cs index 6f6fbb0a5..3da6b9471 100644 --- a/Src/GBX.NET.PAK/Pak.cs +++ b/Src/GBX.NET.PAK/Pak.cs @@ -1,4 +1,5 @@ -using GBX.NET.Exceptions; +using GBX.NET.Crypto; +using GBX.NET.Exceptions; using GBX.NET.Serialization; using System.Collections.Immutable; using System.IO.Compression; @@ -18,6 +19,7 @@ public sealed partial class Pak : IDisposable private readonly Stream stream; private readonly byte[] key; + private readonly byte[] headerMD5; private readonly int metadataStart; private readonly int dataStart; @@ -25,11 +27,19 @@ public sealed partial class Pak : IDisposable public int Flags { get; } public ImmutableDictionary Files { get; } - private Pak(Stream stream, byte[] key, int version, int metadataStart, int dataStart, int flags, ImmutableDictionary files) + private Pak(Stream stream, + byte[] key, + int version, + byte[] headerMD5, + int metadataStart, + int dataStart, + int flags, + ImmutableDictionary files) { this.stream = stream; this.key = key; Version = version; + this.headerMD5 = headerMD5; this.metadataStart = metadataStart; this.dataStart = dataStart; Flags = flags; @@ -54,6 +64,12 @@ public static async Task ParseAsync(Stream stream, byte[] key, Cancellation return await ParseEncryptedAsync(decryptReader, stream, version, key, cancellationToken); } + public static async Task ParseAsync(string filePath, byte[] key, CancellationToken cancellationToken = default) + { + var fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: 4096, useAsync: true); + return await ParseAsync(fs, key, cancellationToken); + } + private static async Task ParseEncryptedAsync( GbxReader r, Stream originalStream, @@ -88,7 +104,7 @@ private static async Task ParseEncryptedAsync( var files = ReadAllFiles(r, allFolders); - return new Pak(originalStream, key, version, metadataStart, dataStart, flags, files); + return new Pak(originalStream, key, version, headerMD5, metadataStart, dataStart, flags, files); } private static PakFolder[] ReadAllFolders(GbxReader r) @@ -162,7 +178,10 @@ public Stream OpenFile(PakFile file, out EncryptionInitializer encryptionInitial stream.Position = dataStart + file.Offset; var ivBuffer = new byte[8]; - stream.Read(ivBuffer, 0, 8); + if (stream.Read(ivBuffer, 0, 8) != 8) + { + throw new EndOfStreamException("Could not read IV from file."); + } var iv = BitConverter.ToUInt64(ivBuffer, 0); var blowfish = new BlowfishStream(stream, key, iv); @@ -184,6 +203,13 @@ public async Task OpenGbxFileAsync(PakFile file, GbxReadSettings settings = return await Gbx.ParseAsync(stream, settings with { EncryptionInitializer = encryptionInitializer }, cancellationToken); } + [Zomp.SyncMethodGenerator.CreateSyncVersion] + public Gbx OpenGbxFileHeader(PakFile file, GbxReadSettings settings = default) + { + using var stream = OpenFile(file, out var encryptionInitializer); + return Gbx.ParseHeader(stream, settings with { EncryptionInitializer = encryptionInitializer }); + } + public void Dispose() { stream.Dispose(); @@ -195,4 +221,72 @@ public async ValueTask DisposeAsync() await stream.DisposeAsync(); } #endif + + public static async Task> BruteforceHashFileNamesAsync( + string directoryPath, + IProgress>? progress = null, + CancellationToken cancellationToken = default) + { + var pakList = PakList.Parse(Path.Combine(directoryPath, "packlist.dat")); + + var hashFileNames = new Dictionary(); + + foreach (var pakInfo in pakList) + { + var fileName = $"{char.ToUpperInvariant(pakInfo.Key[0])}{pakInfo.Key.Substring(1)}.pak"; + var fullFileName = Path.Combine(directoryPath, fileName); + + if (!File.Exists(fullFileName)) + { + continue; + } + + using var pak = await ParseAsync(fullFileName, pakInfo.Value.Key, cancellationToken); + + foreach (var file in pak.Files.Values) + { + if (!file.Name.EndsWith(".gbx", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + cancellationToken.ThrowIfCancellationRequested(); + + Gbx gbx; + try + { + gbx = pak.OpenGbxFileHeader(file); + } + catch (NotAGbxException) + { + continue; + } + + var refTable = gbx.RefTable; + + if (refTable is null) + { + continue; + } + + foreach (var refTableFile in refTable.Files) + { + var filePath = refTableFile.FilePath; + var hash = MD5.Compute136(filePath); + progress?.Report(new KeyValuePair(hash, filePath)); + hashFileNames[hash] = filePath; + + while (filePath.Contains('\\')) + { + filePath = filePath.Substring(filePath.IndexOf('\\') + 1); + hash = MD5.Compute136(filePath); + progress?.Report(new KeyValuePair(hash, filePath)); + hashFileNames[hash] = filePath; + } + } + } + } + + return hashFileNames; + } } \ No newline at end of file diff --git a/Tools/PakToZip/Program.cs b/Tools/PakToZip/Program.cs index 38e14173e..0b5ead955 100644 --- a/Tools/PakToZip/Program.cs +++ b/Tools/PakToZip/Program.cs @@ -5,8 +5,11 @@ Gbx.ZLib = new ZLib(); var fileName = args[0]; +var directoryPath = Path.GetDirectoryName(fileName)!; -var packlistFileName = Path.Combine(Path.GetDirectoryName(fileName)!, "packlist.dat"); +var hashes = await Pak.BruteforceHashFileNamesAsync(directoryPath); + +var packlistFileName = Path.Combine(directoryPath, "packlist.dat"); var packlist = await PakList.ParseAsync(packlistFileName); var key = packlist[Path.GetFileNameWithoutExtension(fileName).ToLowerInvariant()].Key; @@ -15,6 +18,7 @@ using var pak = await Pak.ParseAsync(fs, key); var file = pak.Files.Values.First(x => x.Name == "B2FC497BF7F81AB02D01FE8FB2F707CD8F"); +var fileItemName = hashes[file.Name]; var gbx = await pak.OpenGbxFileAsync(file); From 0a95e06ba755e4d8f6713d38da589c486138e9e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Sun, 8 Dec 2024 21:16:09 +0100 Subject: [PATCH 55/86] Acknowledge backslash entry fullnames --- Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs b/Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs index eb3f82d79..9f778cee2 100644 --- a/Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs +++ b/Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs @@ -1775,12 +1775,13 @@ public override void Write(CGameCtnChallenge n, GbxWriter w) var ident = itemModel.Ident; - if (entry.FullName.StartsWith(itemsPrefix)) + var fullName = entry.FullName.Replace('\\', '/'); + if (fullName.StartsWith(itemsPrefix)) { ident = ident with { Id = entry.FullName.Replace('/', '\\').Substring(itemsPrefix.Length) }; } - if (entry.FullName.StartsWith(clubItemsPrefix)) + if (fullName.StartsWith(clubItemsPrefix)) { ident = ident with { From 69b3821d721d20adbd4ef3525856081b5ca856a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Sun, 8 Dec 2024 21:20:13 +0100 Subject: [PATCH 56/86] Prefer backslash work --- Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs b/Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs index 9f778cee2..e723fbb2a 100644 --- a/Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs +++ b/Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs @@ -1753,9 +1753,8 @@ public override void Write(CGameCtnChallenge n, GbxWriter w) foreach (var entry in zip.Entries) { - const string itemsPrefix = "Items/"; - const string clubItemsPrefix = "ClubItems/"; - //const string favoriteClubItemsPrefix = "/MemoryTemp/FavoriteClubItems"; + const string itemsPrefix = "Items\\"; + const string clubItemsPrefix = "ClubItems\\"; using var entryStream = entry.Open(); @@ -1775,10 +1774,10 @@ public override void Write(CGameCtnChallenge n, GbxWriter w) var ident = itemModel.Ident; - var fullName = entry.FullName.Replace('\\', '/'); + var fullName = entry.FullName.Replace('/', '\\'); if (fullName.StartsWith(itemsPrefix)) { - ident = ident with { Id = entry.FullName.Replace('/', '\\').Substring(itemsPrefix.Length) }; + ident = ident with { Id = fullName.Substring(itemsPrefix.Length) }; } if (fullName.StartsWith(clubItemsPrefix)) @@ -1786,9 +1785,9 @@ public override void Write(CGameCtnChallenge n, GbxWriter w) ident = ident with { #if NET6_0_OR_GREATER - Id = string.Concat("club:", entry.FullName.Replace('/', '\\').AsSpan(clubItemsPrefix.Length)) + Id = string.Concat("club:", fullName.AsSpan(clubItemsPrefix.Length)) #else - Id = "club:" + entry.FullName.Replace('/', '\\').Substring(clubItemsPrefix.Length) + Id = "club:" + fullName.Substring(clubItemsPrefix.Length) #endif }; } From e7cee1491f40b836fcdde32afe2676f4c44765ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Mon, 9 Dec 2024 23:02:25 +0100 Subject: [PATCH 57/86] Add Blocks/ fix to embedding write --- Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs b/Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs index e723fbb2a..a829e8dc9 100644 --- a/Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs +++ b/Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs @@ -1754,6 +1754,7 @@ public override void Write(CGameCtnChallenge n, GbxWriter w) foreach (var entry in zip.Entries) { const string itemsPrefix = "Items\\"; + const string blocksPrefix = "Blocks\\"; const string clubItemsPrefix = "ClubItems\\"; using var entryStream = entry.Open(); @@ -1780,6 +1781,11 @@ public override void Write(CGameCtnChallenge n, GbxWriter w) ident = ident with { Id = fullName.Substring(itemsPrefix.Length) }; } + if (fullName.StartsWith(blocksPrefix)) + { + ident = ident with { Id = fullName.Substring(blocksPrefix.Length) }; + } + if (fullName.StartsWith(clubItemsPrefix)) { ident = ident with From ba792c492ec06eb777438b4c99088fb560d56b85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Mon, 9 Dec 2024 23:03:43 +0100 Subject: [PATCH 58/86] Add else --- Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs b/Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs index a829e8dc9..445a21e27 100644 --- a/Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs +++ b/Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs @@ -1780,13 +1780,11 @@ public override void Write(CGameCtnChallenge n, GbxWriter w) { ident = ident with { Id = fullName.Substring(itemsPrefix.Length) }; } - - if (fullName.StartsWith(blocksPrefix)) + else if (fullName.StartsWith(blocksPrefix)) { ident = ident with { Id = fullName.Substring(blocksPrefix.Length) }; } - - if (fullName.StartsWith(clubItemsPrefix)) + else if (fullName.StartsWith(clubItemsPrefix)) { ident = ident with { From d4d4df7cddd7ccaf9dbacef67ab339d0c7327860 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Tue, 10 Dec 2024 18:04:20 +0100 Subject: [PATCH 59/86] Remove Block filter --- Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs b/Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs index 445a21e27..cca3bc897 100644 --- a/Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs +++ b/Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs @@ -1768,7 +1768,7 @@ public override void Write(CGameCtnChallenge n, GbxWriter w) continue; } - if (itemModel.Ident is null || itemModel.ItemType == CGameItemModel.EItemType.Block) + if (itemModel.Ident is null) { continue; } From 9ff62e8875f8359ff46076990f68bf5c22423afc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Tue, 10 Dec 2024 23:25:12 +0100 Subject: [PATCH 60/86] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d3ea688c6..023d14ab6 100644 --- a/README.md +++ b/README.md @@ -408,7 +408,7 @@ Gbx.ZLib = new ZLib(); ``` > [!NOTE] -> Zlib-compressed data is **currently read-only for ghost samples and record data**, lightmap data is the only zlib data that can be modified with GBX.NET. Write support for ghost samples and record data is planned for 2.1 or 2.2. +> Zlib-compressed data is **currently read-only for ghost samples and record data**, lightmap data is the only zlib data that can be modified with GBX.NET. Write support for ghost samples and record data is planned for 2.2. The data is often stored in properties of type `CompressedData` which are byte arrays with additional uncompressed data size for validation. If there's a property with this type (`CGameCtnChallenge.LightmapCacheData` for example), zlib data will be stored there no matter if the zlib implementation is included, so read/write consistency is guaranteed. From 1279c7bbeb315d00e4aa786cc8693cf0daa49cf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Tue, 17 Dec 2024 02:39:38 +0100 Subject: [PATCH 61/86] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 023d14ab6..a231d4916 100644 --- a/README.md +++ b/README.md @@ -180,7 +180,7 @@ using GBX.NET.LZO; Gbx.LZO = new Lzo(); ``` -You should run this line of code **only once** for the whole program lifetime. +You should run this line of code **only once** at the start of the program. The compression logic is split up from the read/write logic to **allow GBX.NET 2 library to be distributed under the MIT license**, as Oberhumer distributes the open source version of LZO under the GNU GPL v2+. Therefore, using GBX.NET.LZO 2 requires you to license your project under the GNU GPL v3, see [License](#license). From d0119f181f981dd713ff3fa1edb85189390036b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Wed, 18 Dec 2024 02:22:22 +0100 Subject: [PATCH 62/86] Fix authorLogin being set to empty string (wtf?) --- Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs b/Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs index cca3bc897..fcf08713b 100644 --- a/Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs +++ b/Src/GBX.NET/Engines/Game/CGameCtnChallenge.cs @@ -12,7 +12,7 @@ public partial class CGameCtnChallenge : IGameCtnChallengeMP4, IGameCtnChallengeTM2020 { - private string authorLogin = string.Empty; + private string? authorLogin; private TimeInt32? bronzeTime; // Only used if ChallengeParameters is null private TimeInt32? silverTime; // Only used if ChallengeParameters is null private TimeInt32? goldTime; // Only used if ChallengeParameters is null From 326c4f42c205f3176d3b9eaf169af41454f8526e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Wed, 18 Dec 2024 15:21:28 +0100 Subject: [PATCH 63/86] Fix repo request --- Src/GBX.NET.Tool.CLI/ToolConsole.cs | 2 +- Src/GBX.NET.Tool.CLI/ToolConsoleOptions.cs | 5 +++++ Src/GBX.NET.Tool.CLI/ToolUpdateChecker.cs | 9 +++++++-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/Src/GBX.NET.Tool.CLI/ToolConsole.cs b/Src/GBX.NET.Tool.CLI/ToolConsole.cs index 5c13fba90..ed58b225f 100644 --- a/Src/GBX.NET.Tool.CLI/ToolConsole.cs +++ b/Src/GBX.NET.Tool.CLI/ToolConsole.cs @@ -140,7 +140,7 @@ public async Task RunAsync(ExecuteLogic executeLogic, CancellationToken cancella // Request update info and additional stuff var updateChecker = toolSettings.ConsoleSettings.DisableUpdateCheck ? null - : ToolUpdateChecker.Check(http, cancellationToken); + : ToolUpdateChecker.Check(http, options.GitHubRepo, cancellationToken); if (introWriterTask is not null) { diff --git a/Src/GBX.NET.Tool.CLI/ToolConsoleOptions.cs b/Src/GBX.NET.Tool.CLI/ToolConsoleOptions.cs index 4c616303c..4e5059324 100644 --- a/Src/GBX.NET.Tool.CLI/ToolConsoleOptions.cs +++ b/Src/GBX.NET.Tool.CLI/ToolConsoleOptions.cs @@ -14,4 +14,9 @@ public sealed record ToolConsoleOptions public YamlDotNet.Serialization.DeserializerBuilder? YmlDeserializer { get; init; } public YamlDotNet.Serialization.SerializerBuilder? YmlSerializer { get; init; } public YamlDotNet.Serialization.StaticContext? YmlContext { get; init; } + + /// + /// GitHub repository identifier where the releases are located. Format: user/repo + /// + public string? GitHubRepo { get; set; } } diff --git a/Src/GBX.NET.Tool.CLI/ToolUpdateChecker.cs b/Src/GBX.NET.Tool.CLI/ToolUpdateChecker.cs index dfec2ad1d..e9ee1086c 100644 --- a/Src/GBX.NET.Tool.CLI/ToolUpdateChecker.cs +++ b/Src/GBX.NET.Tool.CLI/ToolUpdateChecker.cs @@ -11,9 +11,14 @@ public ToolUpdateChecker(Task updateInfoResponseTask) this.updateInfoResponseTask = updateInfoResponseTask; } - public static ToolUpdateChecker Check(HttpClient client, CancellationToken cancellationToken) + public static ToolUpdateChecker? Check(HttpClient client, string? githubRepo, CancellationToken cancellationToken) { - var responseTask = client.GetAsync("https://api.github.com/repos/GBX.NET/GBX.NET.Tool/releases/latest", cancellationToken); + if (githubRepo is null) + { + return null; + } + + var responseTask = client.GetAsync($"https://api.github.com/repos/{githubRepo}/releases/latest", cancellationToken); return new ToolUpdateChecker(responseTask); } From e0c54b922a090d92c5ba207f9704f7eecebad6d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Thu, 19 Dec 2024 01:25:08 +0100 Subject: [PATCH 64/86] Switch to enumerate files --- Src/GBX.NET.Tool.CLI/InputArguments/DirectoryInputArgument.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Src/GBX.NET.Tool.CLI/InputArguments/DirectoryInputArgument.cs b/Src/GBX.NET.Tool.CLI/InputArguments/DirectoryInputArgument.cs index 306296f2c..35c364c6a 100644 --- a/Src/GBX.NET.Tool.CLI/InputArguments/DirectoryInputArgument.cs +++ b/Src/GBX.NET.Tool.CLI/InputArguments/DirectoryInputArgument.cs @@ -7,7 +7,7 @@ public sealed record DirectoryInputArgument(string DirectoryPath) : InputArgumen { public override async Task ResolveAsync(CancellationToken cancellationToken) { - var files = Directory.GetFiles(DirectoryPath, "*.*", SearchOption.AllDirectories); + var files = Directory.EnumerateFiles(DirectoryPath, "*.*", SearchOption.AllDirectories); var tasks = files.Select>(async file => { From c45aaa804e4bdaf8cd5800853dc2083af9545fc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Fri, 20 Dec 2024 14:43:53 +0100 Subject: [PATCH 65/86] Add GitHub release info at the end --- Src/GBX.NET.Tool.CLI/GitHubJsonContext.cs | 9 +++++++ Src/GBX.NET.Tool.CLI/ToolConsole.cs | 16 ++++-------- Src/GBX.NET.Tool.CLI/ToolUpdateChecker.cs | 30 +++++++++++++++-------- Src/GBX.NET.Tool.CLI/UpdateInfo.cs | 8 ++++++ 4 files changed, 42 insertions(+), 21 deletions(-) create mode 100644 Src/GBX.NET.Tool.CLI/GitHubJsonContext.cs create mode 100644 Src/GBX.NET.Tool.CLI/UpdateInfo.cs diff --git a/Src/GBX.NET.Tool.CLI/GitHubJsonContext.cs b/Src/GBX.NET.Tool.CLI/GitHubJsonContext.cs new file mode 100644 index 000000000..536ccdd57 --- /dev/null +++ b/Src/GBX.NET.Tool.CLI/GitHubJsonContext.cs @@ -0,0 +1,9 @@ +using GBX.NET.Tool.CLI; +using System.Text.Json.Serialization; + +namespace NationsConverterWeb; + +[JsonSerializable(typeof(UpdateInfo))] +[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.SnakeCaseLower, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)] +internal sealed partial class GitHubJsonContext : JsonSerializerContext; diff --git a/Src/GBX.NET.Tool.CLI/ToolConsole.cs b/Src/GBX.NET.Tool.CLI/ToolConsole.cs index ed58b225f..6d8e5b6be 100644 --- a/Src/GBX.NET.Tool.CLI/ToolConsole.cs +++ b/Src/GBX.NET.Tool.CLI/ToolConsole.cs @@ -148,12 +148,6 @@ public async Task RunAsync(ExecuteLogic executeLogic, CancellationToken cancella logger.LogTrace("Intro finished."); } - // Check for updates here if received. If not, check at the end of the tool execution - var updateCheckCompleted = updateChecker is null - || await updateChecker.TryCompareVersionAsync(); - - logger.LogDebug("Update check completed: {UpdateCheckCompleted}", updateCheckCompleted); - AnsiConsole.WriteLine(); if (toolSettings.InputArguments.Count == 0) @@ -165,9 +159,9 @@ public async Task RunAsync(ExecuteLogic executeLogic, CancellationToken cancella AnsiConsole.WriteLine(options.IntroText); } - if (!updateCheckCompleted && updateChecker is not null) + if (updateChecker is not null) { - await updateChecker.CompareVersionAsync(); + await updateChecker.CompareVersionAsync(cancellationToken); } AnsiConsole.WriteLine(); @@ -194,10 +188,10 @@ public async Task RunAsync(ExecuteLogic executeLogic, CancellationToken cancella logger.LogInformation("Completed!"); - // Check again for updates if not done before - if (!updateCheckCompleted && updateChecker is not null) + // Check for updates if not done before + if (updateChecker is not null) { - await updateChecker.CompareVersionAsync(); + await updateChecker.CompareVersionAsync(cancellationToken); } } diff --git a/Src/GBX.NET.Tool.CLI/ToolUpdateChecker.cs b/Src/GBX.NET.Tool.CLI/ToolUpdateChecker.cs index e9ee1086c..05e9fed1a 100644 --- a/Src/GBX.NET.Tool.CLI/ToolUpdateChecker.cs +++ b/Src/GBX.NET.Tool.CLI/ToolUpdateChecker.cs @@ -1,4 +1,6 @@ -using Spectre.Console; +using NationsConverterWeb; +using Spectre.Console; +using System.Net.Http.Json; namespace GBX.NET.Tool.CLI; @@ -22,19 +24,19 @@ public ToolUpdateChecker(Task updateInfoResponseTask) return new ToolUpdateChecker(responseTask); } - public async ValueTask TryCompareVersionAsync() + public async ValueTask TryCompareVersionAsync(CancellationToken cancellationToken) { if (!updateInfoResponseTask.IsCompleted) { return false; } - await CompareVersionAsync(); + await CompareVersionAsync(cancellationToken); return true; } - public async Task CompareVersionAsync() + public async Task CompareVersionAsync(CancellationToken cancellationToken) { AnsiConsole.WriteLine(); @@ -45,13 +47,21 @@ public async Task CompareVersionAsync() AnsiConsole.Write(new Rule("Check for updates / auto-updater").LeftJustified().RuleStyle("yellow")); AnsiConsole.WriteLine(); - //var updateInfo = await updateInfoResponse.Content.ReadFromJsonAsync(cancellationToken); + try + { + var updateInfo = await updateInfoResponse.Content.ReadFromJsonAsync(GitHubJsonContext.Default.UpdateInfo, cancellationToken); - //if (updateInfo is not null) - //{ - AnsiConsole.MarkupLine($"[yellow]New version available:[/] [green]tag[/]"); - AnsiConsole.MarkupLine($"[yellow]Release notes:[/] [green]url[/]"); - //} + if (updateInfo is not null) + { + AnsiConsole.MarkupLine($"[yellow]Latest version available:[/] [green]{updateInfo.TagName?.TrimStart('v')}[/]"); + AnsiConsole.MarkupLine($"[yellow]Release notes:[/] [green]{updateInfo.HtmlUrl}[/]"); + } + } + catch (Exception ex) + { + AnsiConsole.MarkupLine("[red]Failed to parse update information.[/]"); + AnsiConsole.WriteException(ex); + } AnsiConsole.WriteLine(); AnsiConsole.Write(new Rule().RuleStyle("yellow")); diff --git a/Src/GBX.NET.Tool.CLI/UpdateInfo.cs b/Src/GBX.NET.Tool.CLI/UpdateInfo.cs new file mode 100644 index 000000000..4069b1cfb --- /dev/null +++ b/Src/GBX.NET.Tool.CLI/UpdateInfo.cs @@ -0,0 +1,8 @@ +namespace GBX.NET.Tool.CLI; + +internal sealed class UpdateInfo +{ + public string? TagName { get; set; } + public string? HtmlUrl { get; set; } + public bool IsPrerelease { get; set; } +} From e293b64454399e018b5198cbe2e7ddf0003f2085 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Fri, 20 Dec 2024 15:03:09 +0100 Subject: [PATCH 66/86] Make sure also prereleases are retrieved --- Src/GBX.NET.Tool.CLI/ToolUpdateChecker.cs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/Src/GBX.NET.Tool.CLI/ToolUpdateChecker.cs b/Src/GBX.NET.Tool.CLI/ToolUpdateChecker.cs index 05e9fed1a..83ba17c0e 100644 --- a/Src/GBX.NET.Tool.CLI/ToolUpdateChecker.cs +++ b/Src/GBX.NET.Tool.CLI/ToolUpdateChecker.cs @@ -20,7 +20,7 @@ public ToolUpdateChecker(Task updateInfoResponseTask) return null; } - var responseTask = client.GetAsync($"https://api.github.com/repos/{githubRepo}/releases/latest", cancellationToken); + var responseTask = client.GetAsync($"https://api.github.com/repos/{githubRepo}/releases", cancellationToken); return new ToolUpdateChecker(responseTask); } @@ -49,13 +49,17 @@ public async Task CompareVersionAsync(CancellationToken cancellationToken) try { - var updateInfo = await updateInfoResponse.Content.ReadFromJsonAsync(GitHubJsonContext.Default.UpdateInfo, cancellationToken); - - if (updateInfo is not null) + await foreach (var updateInfo in updateInfoResponse.Content.ReadFromJsonAsAsyncEnumerable(GitHubJsonContext.Default.UpdateInfo, cancellationToken)) { - AnsiConsole.MarkupLine($"[yellow]Latest version available:[/] [green]{updateInfo.TagName?.TrimStart('v')}[/]"); - AnsiConsole.MarkupLine($"[yellow]Release notes:[/] [green]{updateInfo.HtmlUrl}[/]"); + if (updateInfo is not null) + { + AnsiConsole.MarkupLine($"[yellow]Latest version available:[/] [green]{updateInfo.TagName?.TrimStart('v')}[/]"); + AnsiConsole.MarkupLine($"[yellow]Release notes:[/] [green]{updateInfo.HtmlUrl}[/]"); + } + + break; } + } catch (Exception ex) { From 7549b21d9d69c57ddd86501550303391c05298d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Wed, 25 Dec 2024 03:14:16 +0100 Subject: [PATCH 67/86] Update to latest NativeSharpZlib --- Src/GBX.NET.PAK/GBX.NET.PAK.csproj | 3 +- Src/GBX.NET.PAK/Pak.cs | 3 +- Src/GBX.NET.PAK/ZlibDeflateStream.cs | 346 --------------------------- Tools/PakToZip/PakToZip.csproj | 2 +- Tools/PakToZip/Program.cs | 47 +++- 5 files changed, 42 insertions(+), 359 deletions(-) delete mode 100644 Src/GBX.NET.PAK/ZlibDeflateStream.cs diff --git a/Src/GBX.NET.PAK/GBX.NET.PAK.csproj b/Src/GBX.NET.PAK/GBX.NET.PAK.csproj index 0b5793a12..b1bd46449 100644 --- a/Src/GBX.NET.PAK/GBX.NET.PAK.csproj +++ b/Src/GBX.NET.PAK/GBX.NET.PAK.csproj @@ -16,7 +16,7 @@ - net9.0;net8.0;net6.0;netstandard2.0 + net9.0;net8.0;net6.0 12 enable enable @@ -45,6 +45,7 @@ all runtime; build; native; contentfiles; analyzers + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Src/GBX.NET.PAK/Pak.cs b/Src/GBX.NET.PAK/Pak.cs index 3da6b9471..059992ee5 100644 --- a/Src/GBX.NET.PAK/Pak.cs +++ b/Src/GBX.NET.PAK/Pak.cs @@ -1,6 +1,7 @@ using GBX.NET.Crypto; using GBX.NET.Exceptions; using GBX.NET.Serialization; +using NativeSharpZlib; using System.Collections.Immutable; using System.IO.Compression; using System.Text; @@ -193,7 +194,7 @@ public Stream OpenFile(PakFile file, out EncryptionInitializer encryptionInitial return blowfish; } - return new ZlibDeflateStream(blowfish, false); + return new NativeZlibStream(blowfish, CompressionMode.Decompress); } [Zomp.SyncMethodGenerator.CreateSyncVersion] diff --git a/Src/GBX.NET.PAK/ZlibDeflateStream.cs b/Src/GBX.NET.PAK/ZlibDeflateStream.cs deleted file mode 100644 index ce61c796a..000000000 --- a/Src/GBX.NET.PAK/ZlibDeflateStream.cs +++ /dev/null @@ -1,346 +0,0 @@ -using System.Runtime.InteropServices; - -namespace GBX.NET.PAK; - -internal class ZlibDeflateStream : Stream -{ - #region zlib interop - - [StructLayout(LayoutKind.Sequential)] - private class z_stream - { - public IntPtr next_in; - public int avail_in; - public int total_in; - - public IntPtr next_out; - public int avail_out; - public int total_out; - - public IntPtr msg; - public IntPtr state; - - public IntPtr zalloc; - public IntPtr zfree; - public IntPtr opaque; - - public int data_type; - public uint adler; - public uint reserved; - } - - private enum ZFlushType - { - Z_NO_FLUSH = 0, - Z_PARTIAL_FLUSH = 1, - Z_SYNC_FLUSH = 2, - Z_FULL_FLUSH = 3, - Z_FINISH = 4, - Z_BLOCK = 5, - Z_TREES = 6 - } - - private enum ZStatus - { - Z_OK = 0, - Z_STREAM_END = 1, - Z_NEED_DICT = 2, - Z_ERRNO = -1, - Z_STREAM_ERROR = -2, - Z_DATA_ERROR = -3, - Z_MEM_ERROR = -4, - Z_BUF_ERROR = -5, - Z_VERSION_ERROR = -6 - } - - [DllImport("zlib1.dll")] - private static extern ZStatus deflateInit_(z_stream stream, int level, string version, int stream_size); - - [DllImport("zlib1.dll")] - private static extern ZStatus deflate(z_stream stream, ZFlushType flush); - - [DllImport("zlib1.dll")] - private static extern ZStatus deflateEnd(z_stream stream); - - [DllImport("zlib1.dll")] - private static extern ZStatus inflateInit_(z_stream stream, string version, int stream_size); - - [DllImport("zlib1.dll")] - private static extern ZStatus inflate(z_stream stream, ZFlushType flush); - - [DllImport("zlib1.dll")] - private static extern ZStatus inflateEnd(z_stream stream); - - #endregion - - private const int COMPRESSED_BLOCK_SIZE = 0x100; - private const int UNCOMPRESSED_BLOCK_SIZE = 0x400; - - private Stream _innerStream; - private bool _compressing; - private long _position; - - private z_stream _zlibStream; - private byte[] _compressedBuffer; - private byte[] _uncompressedBuffer; - private IntPtr _zlibCompressedMem; - private IntPtr _zlibUncompressedMem; - private int _compressedIndex; - private int _uncompressedIndex; - private int _uncompressedSize; - - private bool _disposed; - - public ZlibDeflateStream(Stream innerStream, bool compressing) - { - _innerStream = innerStream; - _compressing = compressing; - _position = 0; - - _zlibStream = new z_stream(); - _compressedBuffer = new byte[COMPRESSED_BLOCK_SIZE]; - _uncompressedBuffer = new byte[UNCOMPRESSED_BLOCK_SIZE]; - _zlibCompressedMem = Marshal.AllocHGlobal(COMPRESSED_BLOCK_SIZE); - _zlibUncompressedMem = Marshal.AllocHGlobal(UNCOMPRESSED_BLOCK_SIZE); - if (compressing) - { - VerifyStatus(deflateInit_(_zlibStream, 8, "1.2.5", Marshal.SizeOf(_zlibStream))); - } - else - { - VerifyStatus(inflateInit_(_zlibStream, "1.2.5", Marshal.SizeOf(_zlibStream))); - } - } - - public override bool CanRead - { - get { return !_compressing; } - } - - public override bool CanSeek - { - get { return false; } - } - - public override bool CanWrite - { - get { return _compressing; } - } - - public override void Flush() - { - - } - - public override long Length - { - get { throw new NotSupportedException(); } - } - - public override long Position - { - get - { - return _position; - } - set - { - throw new NotSupportedException(); - } - } - - public override int Read(byte[] buffer, int offset, int count) - { - if (_compressing) - throw new NotSupportedException(); - - int left = count; - while (left > 0) - { - if (_uncompressedIndex == _uncompressedSize) - { - _uncompressedIndex = 0; - _uncompressedSize = 0; - while (_uncompressedSize < UNCOMPRESSED_BLOCK_SIZE) - { - if (_zlibStream.avail_in == 0) - { - int inChunkSize = _innerStream.Read(_compressedBuffer, 0, COMPRESSED_BLOCK_SIZE); - SetInflateInput(_compressedBuffer, 0, inChunkSize); - } - bool streamEnd; - _uncompressedSize += Inflate(_uncompressedBuffer, _uncompressedSize, - UNCOMPRESSED_BLOCK_SIZE - _uncompressedSize, out streamEnd); - if (streamEnd) - break; - } - } - int outChunkSize = Math.Min(_uncompressedSize - _uncompressedIndex, left); - Array.Copy(_uncompressedBuffer, _uncompressedIndex, buffer, offset, outChunkSize); - _uncompressedIndex += outChunkSize; - _position += outChunkSize; - offset += outChunkSize; - left -= outChunkSize; - } - - return count - left; - } - - public override long Seek(long offset, SeekOrigin origin) - { - throw new NotSupportedException(); - } - - public override void SetLength(long value) - { - throw new NotSupportedException(); - } - - public override void Write(byte[] buffer, int offset, int count) - { - if (!_compressing) - throw new NotSupportedException(); - - int left = count; - while (left > 0) - { - int inChunkSize = Math.Min(UNCOMPRESSED_BLOCK_SIZE - _uncompressedIndex, left); - Array.Copy(buffer, offset, _uncompressedBuffer, _uncompressedIndex, inChunkSize); - _uncompressedIndex += inChunkSize; - _position += inChunkSize; - offset += inChunkSize; - left -= inChunkSize; - - if (_uncompressedIndex == UNCOMPRESSED_BLOCK_SIZE) - { - SetDeflateInput(_uncompressedBuffer, 0, UNCOMPRESSED_BLOCK_SIZE); - FlushDeflateOutput(false); - _uncompressedIndex = 0; - } - } - } - - private void FlushDeflateOutput(bool finish) - { - bool finished = false; - while (!NeedInput() || (finish && !finished)) - { - int deflated = Deflate(_compressedBuffer, 0, COMPRESSED_BLOCK_SIZE, finish, out finished); - _innerStream.Write(_compressedBuffer, 0, deflated); - } - } - - private bool NeedInput() - { - return _zlibStream.avail_in == 0; - } - - private void SetInflateInput(byte[] buffer, int offset, int length) - { - _zlibStream.avail_in = length; - _zlibStream.next_in = _zlibCompressedMem; - Marshal.Copy(buffer, offset, _zlibCompressedMem, length); - } - - private int Inflate(byte[] buffer, int offset, int length, out bool streamEnd) - { - streamEnd = false; - - if (_zlibStream.avail_in == 0) - return 0; - - _zlibStream.avail_out = length; - _zlibStream.next_out = _zlibUncompressedMem; - ZStatus status = VerifyStatus(inflate(_zlibStream, ZFlushType.Z_SYNC_FLUSH)); - Marshal.Copy(_zlibUncompressedMem, buffer, offset, length); - streamEnd = (status == ZStatus.Z_STREAM_END); - return (int)((ulong)_zlibStream.next_out - (ulong)_zlibUncompressedMem); - } - - private void SetDeflateInput(byte[] buffer, int offset, int length) - { - _zlibStream.avail_in = length; - _zlibStream.next_in = _zlibUncompressedMem; - Marshal.Copy(buffer, offset, _zlibUncompressedMem, length); - } - - private int Deflate(byte[] buffer, int offset, int length, bool finish, out bool finished) - { - finished = false; - _zlibStream.avail_out = length; - _zlibStream.next_out = _zlibCompressedMem; - if (finish) - { - ZStatus status = VerifyStatus(deflate(_zlibStream, ZFlushType.Z_FINISH)); - finished = (status == ZStatus.Z_STREAM_END); - } - else - { - if (_zlibStream.avail_in == 0) - return 0; - - VerifyStatus(deflate(_zlibStream, ZFlushType.Z_SYNC_FLUSH)); - } - int deflated = (int)_zlibStream.next_out - (int)_zlibCompressedMem; - Marshal.Copy(_zlibCompressedMem, buffer, offset, deflated); - return deflated; - } - - private static ZStatus VerifyStatus(ZStatus status) - { - switch (status) - { - case ZStatus.Z_BUF_ERROR: - throw new Exception("Buffer error"); - - case ZStatus.Z_DATA_ERROR: - throw new Exception("Data error"); - - case ZStatus.Z_ERRNO: - throw new Exception("Filesystem error"); - - case ZStatus.Z_MEM_ERROR: - throw new Exception("Memory error"); - - case ZStatus.Z_STREAM_ERROR: - throw new Exception("Stream error"); - - case ZStatus.Z_VERSION_ERROR: - throw new Exception("Version error"); - } - return status; - } - - protected override void Dispose(bool disposing) - { - if (!_disposed) - { - if (disposing) - { - if (_compressing) - { - if (_uncompressedIndex > 0) - { - SetDeflateInput(_uncompressedBuffer, 0, _uncompressedIndex); - FlushDeflateOutput(false); - } - FlushDeflateOutput(true); - } - _innerStream.Dispose(); - } - - if (_compressing) - VerifyStatus(deflateEnd(_zlibStream)); - else - VerifyStatus(inflateEnd(_zlibStream)); - - Marshal.FreeHGlobal(_zlibCompressedMem); - Marshal.FreeHGlobal(_zlibUncompressedMem); - _zlibCompressedMem = IntPtr.Zero; - _zlibUncompressedMem = IntPtr.Zero; - - _disposed = true; - } - base.Dispose(disposing); - } -} \ No newline at end of file diff --git a/Tools/PakToZip/PakToZip.csproj b/Tools/PakToZip/PakToZip.csproj index c70d85c54..a8aa655a2 100644 --- a/Tools/PakToZip/PakToZip.csproj +++ b/Tools/PakToZip/PakToZip.csproj @@ -2,7 +2,7 @@ Exe - net9.0 + net9.0-windows;net9.0 enable enable true diff --git a/Tools/PakToZip/Program.cs b/Tools/PakToZip/Program.cs index 0b5ead955..754a08b48 100644 --- a/Tools/PakToZip/Program.cs +++ b/Tools/PakToZip/Program.cs @@ -1,25 +1,52 @@ using GBX.NET; +using GBX.NET.Exceptions; using GBX.NET.PAK; using GBX.NET.ZLib; +using System.IO.Compression; Gbx.ZLib = new ZLib(); -var fileName = args[0]; -var directoryPath = Path.GetDirectoryName(fileName)!; +var pakFileName = args[0]; +var directoryPath = Path.GetDirectoryName(pakFileName)!; var hashes = await Pak.BruteforceHashFileNamesAsync(directoryPath); var packlistFileName = Path.Combine(directoryPath, "packlist.dat"); var packlist = await PakList.ParseAsync(packlistFileName); -var key = packlist[Path.GetFileNameWithoutExtension(fileName).ToLowerInvariant()].Key; +var key = packlist[Path.GetFileNameWithoutExtension(pakFileName).ToLowerInvariant()].Key; -using var fs = File.OpenRead(fileName); +using var fs = File.OpenRead(pakFileName); using var pak = await Pak.ParseAsync(fs, key); -var file = pak.Files.Values.First(x => x.Name == "B2FC497BF7F81AB02D01FE8FB2F707CD8F"); -var fileItemName = hashes[file.Name]; - -var gbx = await pak.OpenGbxFileAsync(file); - -Console.WriteLine(); \ No newline at end of file +using var zip = ZipFile.Open(Path.ChangeExtension(pakFileName, ".zip"), ZipArchiveMode.Create); + +foreach (var file in pak.Files.Values) +{ + var fileName = hashes.GetValueOrDefault(file.Name) ?? file.Name; + var fullPath = Path.Combine(file.FolderPath, fileName); + + Console.WriteLine(fullPath); + + try + { + var gbx = await pak.OpenGbxFileAsync(file); + + var entry = zip.CreateEntry(fullPath); + using var stream = entry.Open(); + + gbx.Save(stream); + } + catch (NotAGbxException) + { + var entry = zip.CreateEntry(fullPath); + using var stream = entry.Open(); + + using var pakItemFileStream = pak.OpenFile(file, out _); + //await pakItemFileStream.CopyToAsync(stream); + } + catch (Exception ex) + { + Console.WriteLine(ex); + } +} \ No newline at end of file From 72fb3eb244d634710ed1d20f1a08a31b0f3b1b04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Wed, 25 Dec 2024 03:59:46 +0100 Subject: [PATCH 68/86] Fix bruteforce skipping many files --- Src/GBX.NET.PAK/Pak.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Src/GBX.NET.PAK/Pak.cs b/Src/GBX.NET.PAK/Pak.cs index 059992ee5..8f88155f2 100644 --- a/Src/GBX.NET.PAK/Pak.cs +++ b/Src/GBX.NET.PAK/Pak.cs @@ -246,11 +246,6 @@ public static async Task> BruteforceHashFileNamesAsyn foreach (var file in pak.Files.Values) { - if (!file.Name.EndsWith(".gbx", StringComparison.OrdinalIgnoreCase)) - { - continue; - } - cancellationToken.ThrowIfCancellationRequested(); Gbx gbx; From ed084031781de11a3644948d74bff0e3fd4eadde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Wed, 25 Dec 2024 04:00:23 +0100 Subject: [PATCH 69/86] Also copy non-gbx files --- Tools/PakToZip/Program.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Tools/PakToZip/Program.cs b/Tools/PakToZip/Program.cs index 754a08b48..7e0bab4a0 100644 --- a/Tools/PakToZip/Program.cs +++ b/Tools/PakToZip/Program.cs @@ -9,6 +9,8 @@ var pakFileName = args[0]; var directoryPath = Path.GetDirectoryName(pakFileName)!; +Console.WriteLine("Bruteforcing possible file names from hashes..."); + var hashes = await Pak.BruteforceHashFileNamesAsync(directoryPath); var packlistFileName = Path.Combine(directoryPath, "packlist.dat"); @@ -43,7 +45,9 @@ using var stream = entry.Open(); using var pakItemFileStream = pak.OpenFile(file, out _); - //await pakItemFileStream.CopyToAsync(stream); + var data = new byte[file.UncompressedSize]; + var count = pakItemFileStream.Read(data); + stream.Write(data, 0, count); } catch (Exception ex) { From e7a99b615382090434b2a9ba4af645f8b981f717 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Wed, 25 Dec 2024 04:12:23 +0100 Subject: [PATCH 70/86] Copy file in unknown gbx files --- Tools/PakToZip/Program.cs | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/Tools/PakToZip/Program.cs b/Tools/PakToZip/Program.cs index 7e0bab4a0..5022d3b6a 100644 --- a/Tools/PakToZip/Program.cs +++ b/Tools/PakToZip/Program.cs @@ -1,4 +1,5 @@ using GBX.NET; +using GBX.NET.Components; using GBX.NET.Exceptions; using GBX.NET.PAK; using GBX.NET.ZLib; @@ -37,20 +38,31 @@ var entry = zip.CreateEntry(fullPath); using var stream = entry.Open(); - gbx.Save(stream); + if (gbx.Header is GbxHeaderUnknown) + { + CopyFileToStream(pak, file, stream); + } + else + { + gbx.Save(stream); + } } catch (NotAGbxException) { var entry = zip.CreateEntry(fullPath); using var stream = entry.Open(); - - using var pakItemFileStream = pak.OpenFile(file, out _); - var data = new byte[file.UncompressedSize]; - var count = pakItemFileStream.Read(data); - stream.Write(data, 0, count); + CopyFileToStream(pak, file, stream); } catch (Exception ex) { Console.WriteLine(ex); } +} + +static void CopyFileToStream(Pak pak, PakFile file, Stream stream) +{ + var pakItemFileStream = pak.OpenFile(file, out _); + var data = new byte[file.UncompressedSize]; + var count = pakItemFileStream.Read(data); + stream.Write(data, 0, count); } \ No newline at end of file From 385868a4ad80141b2bcb715ef7a1ff9982714ee0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Thu, 26 Dec 2024 01:19:52 +0100 Subject: [PATCH 71/86] Add filter for only used hashes --- Src/GBX.NET.PAK/Pak.cs | 107 +++++++++++++++++++++++++------------- Tools/PakToZip/Program.cs | 2 +- 2 files changed, 71 insertions(+), 38 deletions(-) diff --git a/Src/GBX.NET.PAK/Pak.cs b/Src/GBX.NET.PAK/Pak.cs index 8f88155f2..e81b94b58 100644 --- a/Src/GBX.NET.PAK/Pak.cs +++ b/Src/GBX.NET.PAK/Pak.cs @@ -4,6 +4,7 @@ using NativeSharpZlib; using System.Collections.Immutable; using System.IO.Compression; +using System.Runtime.CompilerServices; using System.Text; namespace GBX.NET.PAK; @@ -223,66 +224,98 @@ public async ValueTask DisposeAsync() } #endif - public static async Task> BruteforceHashFileNamesAsync( + /// + /// + /// + /// + /// + /// + /// + /// Dictionary where the key is the hash (file name) and value is the true resolved file name. + public static async Task> BruteforceFileHashesAsync( string directoryPath, IProgress>? progress = null, + bool onlyUsedHashes = true, CancellationToken cancellationToken = default) { var pakList = PakList.Parse(Path.Combine(directoryPath, "packlist.dat")); - var hashFileNames = new Dictionary(); + var allPossibleFileHashes = new Dictionary(); - foreach (var pakInfo in pakList) + await foreach (var (pak, file) in EnumeratePakFilesAsync(directoryPath, pakList, cancellationToken)) { - var fileName = $"{char.ToUpperInvariant(pakInfo.Key[0])}{pakInfo.Key.Substring(1)}.pak"; - var fullFileName = Path.Combine(directoryPath, fileName); + cancellationToken.ThrowIfCancellationRequested(); - if (!File.Exists(fullFileName)) + Gbx gbx; + try + { + gbx = pak.OpenGbxFileHeader(file); + } + catch (NotAGbxException) { continue; } - using var pak = await ParseAsync(fullFileName, pakInfo.Value.Key, cancellationToken); + var refTable = gbx.RefTable; - foreach (var file in pak.Files.Values) + if (refTable is null) + { + continue; + } + + foreach (var refTableFile in refTable.Files) { - cancellationToken.ThrowIfCancellationRequested(); + var filePath = refTableFile.FilePath; + var hash = MD5.Compute136(filePath); + progress?.Report(new KeyValuePair(hash, filePath)); + allPossibleFileHashes[hash] = filePath; - Gbx gbx; - try - { - gbx = pak.OpenGbxFileHeader(file); - } - catch (NotAGbxException) + while (filePath.Contains('\\')) { - continue; + filePath = filePath.Substring(filePath.IndexOf('\\') + 1); + hash = MD5.Compute136(filePath); + progress?.Report(new KeyValuePair(hash, filePath)); + allPossibleFileHashes[hash] = filePath; } + } + } - var refTable = gbx.RefTable; + if (!onlyUsedHashes) + { + return allPossibleFileHashes; + } - if (refTable is null) - { - continue; - } + var usedHashes = new Dictionary(); - foreach (var refTableFile in refTable.Files) - { - var filePath = refTableFile.FilePath; - var hash = MD5.Compute136(filePath); - progress?.Report(new KeyValuePair(hash, filePath)); - hashFileNames[hash] = filePath; - - while (filePath.Contains('\\')) - { - filePath = filePath.Substring(filePath.IndexOf('\\') + 1); - hash = MD5.Compute136(filePath); - progress?.Report(new KeyValuePair(hash, filePath)); - hashFileNames[hash] = filePath; - } - } + await foreach (var (_, file) in EnumeratePakFilesAsync(directoryPath, pakList, cancellationToken)) + { + if (allPossibleFileHashes.TryGetValue(file.Name, out var name)) + { + usedHashes[file.Name] = name; } } - return hashFileNames; + return usedHashes; + } + + private static async IAsyncEnumerable<(Pak, PakFile)> EnumeratePakFilesAsync(string directoryPath, PakList pakList, [EnumeratorCancellation] CancellationToken cancellationToken) + { + foreach (var pakInfo in pakList) + { + var fileName = $"{char.ToUpperInvariant(pakInfo.Key[0])}{pakInfo.Key.Substring(1)}.pak"; + var fullFileName = Path.Combine(directoryPath, fileName); + + if (!File.Exists(fullFileName)) + { + continue; + } + + using var pak = await ParseAsync(fullFileName, pakInfo.Value.Key, cancellationToken); + + foreach (var file in pak.Files.Values) + { + yield return (pak, file); + } + } } } \ No newline at end of file diff --git a/Tools/PakToZip/Program.cs b/Tools/PakToZip/Program.cs index 5022d3b6a..fcfe7c8e5 100644 --- a/Tools/PakToZip/Program.cs +++ b/Tools/PakToZip/Program.cs @@ -12,7 +12,7 @@ Console.WriteLine("Bruteforcing possible file names from hashes..."); -var hashes = await Pak.BruteforceHashFileNamesAsync(directoryPath); +var hashes = await Pak.BruteforceFileHashesAsync(directoryPath, onlyUsedHashes: false); var packlistFileName = Path.Combine(directoryPath, "packlist.dat"); var packlist = await PakList.ParseAsync(packlistFileName); From 88f9910aae20ccc2e45e29ebaae593167d0ec253 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Thu, 26 Dec 2024 01:54:43 +0100 Subject: [PATCH 72/86] Add version <6 constraint, add PAK to nightly and publish --- .github/workflows/nightly.yml | 4 ++++ .github/workflows/publish.yml | 1 + Src/GBX.NET.PAK/GBX.NET.PAK.csproj | 14 +------------- Src/GBX.NET.PAK/Pak.cs | 10 +++++++++- Tools/PakToZip/PakToZip.csproj | 1 - Tools/PakToZip/Program.cs | 3 --- 6 files changed, 15 insertions(+), 18 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 75989aa62..18e840949 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -38,6 +38,7 @@ jobs: dotnet build Src/GBX.NET -c Release --no-restore /p:ContinuousIntegrationBuild=true --version-suffix nightly.$(date +'%Y%m%d').${git_hash::7} dotnet build Src/GBX.NET.Tool -c Release --no-restore /p:ContinuousIntegrationBuild=true --version-suffix nightly.$(date +'%Y%m%d').${git_hash::7} dotnet build Src/GBX.NET.Tool.CLI -c Release --no-restore /p:ContinuousIntegrationBuild=true --version-suffix nightly.$(date +'%Y%m%d').${git_hash::7} + dotnet build Src/GBX.NET.PAK -c Release --no-restore /p:ContinuousIntegrationBuild=true --version-suffix nightly.$(date +'%Y%m%d').${git_hash::7} - name: Publish nightly GBX.NET nupkg to nuget.gbx.tools run: dotnet nuget push Src/GBX.NET/bin/Release/*.nupkg -k ${{ secrets.NUGET_GBXTOOLS_API_KEY }} -s https://nuget.gbx.tools/v3/index.json --skip-duplicate @@ -47,3 +48,6 @@ jobs: - name: Publish nightly GBX.NET.Tool.CLI nupkg to nuget.gbx.tools run: dotnet nuget push Src/GBX.NET.Tool.CLI/bin/Release/*.nupkg -k ${{ secrets.NUGET_GBXTOOLS_API_KEY }} -s https://nuget.gbx.tools/v3/index.json --skip-duplicate + + - name: Publish nightly GBX.NET.PAK nupkg to nuget.gbx.tools + run: dotnet nuget push Src/GBX.NET.PAK/bin/Release/*.nupkg -k ${{ secrets.NUGET_GBXTOOLS_API_KEY }} -s https://nuget.gbx.tools/v3/index.json --skip-duplicate diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index aa06e5ba5..520889692 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -73,6 +73,7 @@ jobs: - GBX.NET.Tool - GBX.NET.Tool.CLI - GBX.NET.Crypto + - GBX.NET.PAK name: Publish ${{ matrix.lib }} runs-on: ubuntu-latest diff --git a/Src/GBX.NET.PAK/GBX.NET.PAK.csproj b/Src/GBX.NET.PAK/GBX.NET.PAK.csproj index b1bd46449..ec20119ff 100644 --- a/Src/GBX.NET.PAK/GBX.NET.PAK.csproj +++ b/Src/GBX.NET.PAK/GBX.NET.PAK.csproj @@ -2,7 +2,7 @@ GBX.NET.PAK - 2.0.0-alpha1 + 2.0.0 BigBang1112 Support for Pak (NadeoPak) package files, integrated with GBX.NET. Copyright (c) 2024 Petr Pivoňka @@ -57,16 +57,4 @@ - - - 1.0.1 - - - - - - 1.0.1 - - - diff --git a/Src/GBX.NET.PAK/Pak.cs b/Src/GBX.NET.PAK/Pak.cs index e81b94b58..89eefe775 100644 --- a/Src/GBX.NET.PAK/Pak.cs +++ b/Src/GBX.NET.PAK/Pak.cs @@ -19,6 +19,8 @@ public sealed partial class Pak : IDisposable /// public const string Magic = "NadeoPak"; + private const string PakListFileName = "packlist.dat"; + private readonly Stream stream; private readonly byte[] key; private readonly byte[] headerMD5; @@ -58,6 +60,12 @@ public static async Task ParseAsync(Stream stream, byte[] key, Cancellation } var version = r.ReadInt32(); + + if (version > 5) + { + throw new Exception($"Version >5 (actual: {version}) is not supported. Use Pak6 for this file."); + } + var headerIV = r.ReadUInt64(); var decryptStream = new BlowfishStream(stream, key, headerIV); @@ -238,7 +246,7 @@ public static async Task> BruteforceFileHashesAsync( bool onlyUsedHashes = true, CancellationToken cancellationToken = default) { - var pakList = PakList.Parse(Path.Combine(directoryPath, "packlist.dat")); + var pakList = PakList.Parse(Path.Combine(directoryPath, PakListFileName)); var allPossibleFileHashes = new Dictionary(); diff --git a/Tools/PakToZip/PakToZip.csproj b/Tools/PakToZip/PakToZip.csproj index a8aa655a2..4cffda106 100644 --- a/Tools/PakToZip/PakToZip.csproj +++ b/Tools/PakToZip/PakToZip.csproj @@ -10,7 +10,6 @@ - diff --git a/Tools/PakToZip/Program.cs b/Tools/PakToZip/Program.cs index fcfe7c8e5..fbc02dbb7 100644 --- a/Tools/PakToZip/Program.cs +++ b/Tools/PakToZip/Program.cs @@ -2,11 +2,8 @@ using GBX.NET.Components; using GBX.NET.Exceptions; using GBX.NET.PAK; -using GBX.NET.ZLib; using System.IO.Compression; -Gbx.ZLib = new ZLib(); - var pakFileName = args[0]; var directoryPath = Path.GetDirectoryName(pakFileName)!; From 9245a588562d513adaffc7eaf88c60dfcdc1ec37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Thu, 26 Dec 2024 03:45:41 +0100 Subject: [PATCH 73/86] Add slash to TMU paks and sync methods --- GBX.NET.sln | 7 +++++++ Src/GBX.NET.PAK/Pak.cs | 15 ++++++++++++++- .../ExtractPakFileHashes.csproj | 15 +++++++++++++++ Tools/ExtractPakFileHashes/Program.cs | 13 +++++++++++++ .../Properties/launchSettings.json | 8 ++++++++ 5 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 Tools/ExtractPakFileHashes/ExtractPakFileHashes.csproj create mode 100644 Tools/ExtractPakFileHashes/Program.cs create mode 100644 Tools/ExtractPakFileHashes/Properties/launchSettings.json diff --git a/GBX.NET.sln b/GBX.NET.sln index 0e0214bb2..c5920482c 100644 --- a/GBX.NET.sln +++ b/GBX.NET.sln @@ -176,6 +176,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GBX.NET.Tool.StandardIO", " EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PakToZip", "Tools\PakToZip\PakToZip.csproj", "{47A58E42-EBBC-48D0-B4B1-C25DFD074506}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExtractPakFileHashes", "Tools\ExtractPakFileHashes\ExtractPakFileHashes.csproj", "{B3C5D493-3172-4780-95EC-CBEE73BE4850}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -418,6 +420,10 @@ Global {47A58E42-EBBC-48D0-B4B1-C25DFD074506}.Debug|Any CPU.Build.0 = Debug|Any CPU {47A58E42-EBBC-48D0-B4B1-C25DFD074506}.Release|Any CPU.ActiveCfg = Release|Any CPU {47A58E42-EBBC-48D0-B4B1-C25DFD074506}.Release|Any CPU.Build.0 = Release|Any CPU + {B3C5D493-3172-4780-95EC-CBEE73BE4850}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B3C5D493-3172-4780-95EC-CBEE73BE4850}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B3C5D493-3172-4780-95EC-CBEE73BE4850}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B3C5D493-3172-4780-95EC-CBEE73BE4850}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -488,6 +494,7 @@ Global {0B5609C6-7B10-4E8D-B41C-35CC1A2532A4} = {F3336145-FDA9-4517-AEDC-7F4C4D526ECB} {55BD1330-7431-41A8-90EB-DEB20D9EB981} = {80DCE6B7-4BD9-415C-B053-92B059D7C938} {47A58E42-EBBC-48D0-B4B1-C25DFD074506} = {F3336145-FDA9-4517-AEDC-7F4C4D526ECB} + {B3C5D493-3172-4780-95EC-CBEE73BE4850} = {F3336145-FDA9-4517-AEDC-7F4C4D526ECB} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {8EA2F0DE-BA72-486D-AB3A-9320C0CE5CFD} diff --git a/Src/GBX.NET.PAK/Pak.cs b/Src/GBX.NET.PAK/Pak.cs index 89eefe775..7c66d0d33 100644 --- a/Src/GBX.NET.PAK/Pak.cs +++ b/Src/GBX.NET.PAK/Pak.cs @@ -50,6 +50,7 @@ private Pak(Stream stream, Files = files; } + [Zomp.SyncMethodGenerator.CreateSyncVersion] public static async Task ParseAsync(Stream stream, byte[] key, CancellationToken cancellationToken = default) { var r = new GbxReader(stream); @@ -80,6 +81,13 @@ public static async Task ParseAsync(string filePath, byte[] key, Cancellati return await ParseAsync(fs, key, cancellationToken); } + public static Pak Parse(string filePath, byte[] key) + { + var fs = File.OpenRead(filePath); + return Parse(fs, key); + } + + [Zomp.SyncMethodGenerator.CreateSyncVersion] private static async Task ParseEncryptedAsync( GbxReader r, Stream originalStream, @@ -127,6 +135,11 @@ private static PakFolder[] ReadAllFolders(GbxReader r) var parentFolderIndex = r.ReadInt32(); // index into folders; -1 if this is a root folder var name = r.ReadString(); + if (!name.EndsWith('\\') && !name.EndsWith('/')) + { + name += '\\'; + } + allFolders[i] = new PakFolder(name, parentFolderIndex == -1 ? null : parentFolderIndex); } @@ -246,7 +259,7 @@ public static async Task> BruteforceFileHashesAsync( bool onlyUsedHashes = true, CancellationToken cancellationToken = default) { - var pakList = PakList.Parse(Path.Combine(directoryPath, PakListFileName)); + var pakList = await PakList.ParseAsync(Path.Combine(directoryPath, PakListFileName), cancellationToken); var allPossibleFileHashes = new Dictionary(); diff --git a/Tools/ExtractPakFileHashes/ExtractPakFileHashes.csproj b/Tools/ExtractPakFileHashes/ExtractPakFileHashes.csproj new file mode 100644 index 000000000..4cffda106 --- /dev/null +++ b/Tools/ExtractPakFileHashes/ExtractPakFileHashes.csproj @@ -0,0 +1,15 @@ + + + + Exe + net9.0-windows;net9.0 + enable + enable + true + + + + + + + diff --git a/Tools/ExtractPakFileHashes/Program.cs b/Tools/ExtractPakFileHashes/Program.cs new file mode 100644 index 000000000..1fc2ac6d1 --- /dev/null +++ b/Tools/ExtractPakFileHashes/Program.cs @@ -0,0 +1,13 @@ +using GBX.NET.PAK; + +var directoryPath = args[0]; + +Console.WriteLine("Bruteforcing possible file names from hashes..."); + +var hashes = await Pak.BruteforceFileHashesAsync(directoryPath); + +using var writer = new StreamWriter("hashes.txt"); +foreach (var (hash, name) in hashes.OrderBy(x => x.Value)) +{ + await writer.WriteLineAsync($"{hash:X16} {name}"); +} \ No newline at end of file diff --git a/Tools/ExtractPakFileHashes/Properties/launchSettings.json b/Tools/ExtractPakFileHashes/Properties/launchSettings.json new file mode 100644 index 000000000..4dc2f4033 --- /dev/null +++ b/Tools/ExtractPakFileHashes/Properties/launchSettings.json @@ -0,0 +1,8 @@ +{ + "profiles": { + "ExtractPakFileHashes": { + "commandName": "Project", + "commandLineArgs": "\"E:\\Games\\Vsk5Online\\Packs\"" + } + } +} \ No newline at end of file From 28dd46e62204006897f46fc00e901ecb6c60ad5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Thu, 26 Dec 2024 04:53:16 +0100 Subject: [PATCH 74/86] Implement CPlugParticleEmitterSubModel for TMUF --- .../Plug/CPlugParticleEmitterSubModel.chunkl | 75 ++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-) diff --git a/Src/GBX.NET/Engines/Plug/CPlugParticleEmitterSubModel.chunkl b/Src/GBX.NET/Engines/Plug/CPlugParticleEmitterSubModel.chunkl index 1915584ff..b241cf6b5 100644 --- a/Src/GBX.NET/Engines/Plug/CPlugParticleEmitterSubModel.chunkl +++ b/Src/GBX.NET/Engines/Plug/CPlugParticleEmitterSubModel.chunkl @@ -2,7 +2,7 @@ CPlugParticleEmitterSubModel 0x090B2000 0x011 CPlugMaterial Material (external) - if Material == null + if Material == null && MaterialFile == null CPlugShader Shader (external) float SizeBirthRatioXY vec2 SpritePivotPoint @@ -48,6 +48,79 @@ CPlugParticleEmitterSubModel 0x090B2000 int CircularTrailVertPerPartCount int +0x013 + int + float + float + float + float + float + float + float + float + +0x015 + bool SizeBirthUseIntensity + bool ColorBirthUseIntensity + bool TransparencyBirthUseIntensity + float + float + float + +0x016 + float SizeBirthEmissionZoneScale + int + +0x017 + float FluidFrictionBirthIntensityBase + bool FluidFrictionBirthUseIntensity + +0x018 + CFuncEnvelope IntensityFilter + +0x019 + id + +0x01A + CMwNod + +0x01B + bool MultiState_IsAsyncLink + bool + +0x01C + int TextureAtlas + int TextureAtlasDimX + int TextureAtlasDimY + int TextureAtlasFixedIndex + +0x01D + float + float + +0x01E + float + +0x01F + float BeamLengthSpeedScale + +0x020 + bool PrecalcEnabled + int PrecalcPartCount + int PrecalcSampleRate + bool CollisionEnabled + float CollisionBounce + float CollisionRadius + +0x021 + float CollisionDamper + +0x022 + bool SortSprites + +0x023 + bool + 0x02D version int From d8af4e8e3005b0ea523863690409dbc1881c931e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Thu, 26 Dec 2024 04:53:44 +0100 Subject: [PATCH 75/86] Implement CPlugDecoratorSolid --- .../Engines/Plug/CPlugDecoratorSolid.chunkl | 5 +++- .../Engines/Plug/CPlugDecoratorTree.chunkl | 30 +++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 Src/GBX.NET/Engines/Plug/CPlugDecoratorTree.chunkl diff --git a/Src/GBX.NET/Engines/Plug/CPlugDecoratorSolid.chunkl b/Src/GBX.NET/Engines/Plug/CPlugDecoratorSolid.chunkl index 3c5ffad64..e969d93d6 100644 --- a/Src/GBX.NET/Engines/Plug/CPlugDecoratorSolid.chunkl +++ b/Src/GBX.NET/Engines/Plug/CPlugDecoratorSolid.chunkl @@ -1 +1,4 @@ -CPlugDecoratorSolid 0x090A3000 \ No newline at end of file +CPlugDecoratorSolid 0x090A3000 + +0x000 + CPlugDecoratorTree[]_deprec TreeDecorators \ No newline at end of file diff --git a/Src/GBX.NET/Engines/Plug/CPlugDecoratorTree.chunkl b/Src/GBX.NET/Engines/Plug/CPlugDecoratorTree.chunkl new file mode 100644 index 000000000..89aebbb6e --- /dev/null +++ b/Src/GBX.NET/Engines/Plug/CPlugDecoratorTree.chunkl @@ -0,0 +1,30 @@ +CPlugDecoratorTree 0x090A2000 + +0x006 + id TreeId + CPlugMaterial Material + CPlugTreeLight TreeLight + int VisibleCond + bool VisibleApplyOnChilds + int ShadowCasterCond + bool ShadowCasterApplyOnChilds + bool TransformVisualToSurface + +0x007 + id TreeId + CPlugMaterial Material + CPlugTreeLight TreeLight + int ExistCond + int VisibleCond + bool VisibleApplyOnChilds + int ShadowCasterCond + bool ShadowCasterApplyOnChilds + bool TransformVisualToSurface + +0x008 (base: 0x007) + base + bool NoLocation + +0x009 (base: 0x008) + base + int CollidableCond \ No newline at end of file From 0fc7d1fb5c42ff683cf6b905d509683d0f9a9e48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Thu, 26 Dec 2024 04:54:16 +0100 Subject: [PATCH 76/86] Various chunk additions --- .../Engines/Function/CFuncColorGradient.chunkl | 5 ++++- Src/GBX.NET/Engines/Hms/CHmsLightMap.chunkl | 11 ++++++++++- Src/GBX.NET/Engines/Hms/CHmsLightMapMood.chunkl | 1 + Src/GBX.NET/Engines/Plug/CPlugAudio.chunkl | 5 ++++- Src/GBX.NET/Engines/Plug/CPlugAudioEnvironment.chunkl | 5 ++++- Src/GBX.NET/Engines/Plug/CPlugBitmap.chunkl | 3 +++ Src/GBX.NET/Engines/Plug/CPlugBitmapSampler.chunkl | 2 ++ Src/GBX.NET/Engines/Plug/CPlugMaterialCustom.chunkl | 7 +++++++ .../Engines/Plug/CPlugPointsInSphereOpt.chunkl | 1 + Src/GBX.NET/Engines/Plug/CPlugSound.chunkl | 5 ++++- 10 files changed, 40 insertions(+), 5 deletions(-) create mode 100644 Src/GBX.NET/Engines/Hms/CHmsLightMapMood.chunkl create mode 100644 Src/GBX.NET/Engines/Plug/CPlugPointsInSphereOpt.chunkl diff --git a/Src/GBX.NET/Engines/Function/CFuncColorGradient.chunkl b/Src/GBX.NET/Engines/Function/CFuncColorGradient.chunkl index cc8b9a5d6..4c314c412 100644 --- a/Src/GBX.NET/Engines/Function/CFuncColorGradient.chunkl +++ b/Src/GBX.NET/Engines/Function/CFuncColorGradient.chunkl @@ -1,13 +1,16 @@ CFuncColorGradient 0x05038000 - inherits: CFunc -0x001 +0x000 vec3 KeyFrameValue0 vec3 KeyFrameValue1 vec3 KeyFrameValue2 vec3 KeyFrameValue3 float KeyFramePos1 float KeyFramePos2 + +0x001 (base: 0x000) + base int ColorSpace enum EColorSpace diff --git a/Src/GBX.NET/Engines/Hms/CHmsLightMap.chunkl b/Src/GBX.NET/Engines/Hms/CHmsLightMap.chunkl index 6804c8b20..6ca56dd3e 100644 --- a/Src/GBX.NET/Engines/Hms/CHmsLightMap.chunkl +++ b/Src/GBX.NET/Engines/Hms/CHmsLightMap.chunkl @@ -1 +1,10 @@ -CHmsLightMap 0x06021000 \ No newline at end of file +CHmsLightMap 0x06021000 + +0x001 + CPlugPointsInSphereOpt (external) + +0x002 + CMwNod (external) + +0x003 + CHmsLightMapMood (external) \ No newline at end of file diff --git a/Src/GBX.NET/Engines/Hms/CHmsLightMapMood.chunkl b/Src/GBX.NET/Engines/Hms/CHmsLightMapMood.chunkl new file mode 100644 index 000000000..1b5e967d3 --- /dev/null +++ b/Src/GBX.NET/Engines/Hms/CHmsLightMapMood.chunkl @@ -0,0 +1 @@ +CHmsLightMapMood 0x06023000 \ No newline at end of file diff --git a/Src/GBX.NET/Engines/Plug/CPlugAudio.chunkl b/Src/GBX.NET/Engines/Plug/CPlugAudio.chunkl index 79d6b258d..e103d6569 100644 --- a/Src/GBX.NET/Engines/Plug/CPlugAudio.chunkl +++ b/Src/GBX.NET/Engines/Plug/CPlugAudio.chunkl @@ -1,3 +1,6 @@ CPlugAudio 0x09001000 - inherits: CPlug -- abstract \ No newline at end of file +- abstract + +0x001 + id \ No newline at end of file diff --git a/Src/GBX.NET/Engines/Plug/CPlugAudioEnvironment.chunkl b/Src/GBX.NET/Engines/Plug/CPlugAudioEnvironment.chunkl index 439c6f2fe..19f5e9d62 100644 --- a/Src/GBX.NET/Engines/Plug/CPlugAudioEnvironment.chunkl +++ b/Src/GBX.NET/Engines/Plug/CPlugAudioEnvironment.chunkl @@ -1,2 +1,5 @@ CPlugAudioEnvironment 0x09039000 -- inherits: CPlugAudio \ No newline at end of file +- inherits: CPlugAudio + +0x001 + float \ No newline at end of file diff --git a/Src/GBX.NET/Engines/Plug/CPlugBitmap.chunkl b/Src/GBX.NET/Engines/Plug/CPlugBitmap.chunkl index e08b44559..bc39b9df1 100644 --- a/Src/GBX.NET/Engines/Plug/CPlugBitmap.chunkl +++ b/Src/GBX.NET/Engines/Plug/CPlugBitmap.chunkl @@ -44,6 +44,9 @@ CPlugBitmap 0x09011000 0x020 float[] MipMapFadeAlphas +0x021 + uint + 0x022 (base: 0x018) 0x023 diff --git a/Src/GBX.NET/Engines/Plug/CPlugBitmapSampler.chunkl b/Src/GBX.NET/Engines/Plug/CPlugBitmapSampler.chunkl index 0b80db0db..41dad4586 100644 --- a/Src/GBX.NET/Engines/Plug/CPlugBitmapSampler.chunkl +++ b/Src/GBX.NET/Engines/Plug/CPlugBitmapSampler.chunkl @@ -11,4 +11,6 @@ CPlugBitmapSampler 0x0907E000 0x006 (base: 0x002) +0x007 (base: 0x002) + 0x008 (base: 0x002) diff --git a/Src/GBX.NET/Engines/Plug/CPlugMaterialCustom.chunkl b/Src/GBX.NET/Engines/Plug/CPlugMaterialCustom.chunkl index b0233ffd7..edc8aa1b6 100644 --- a/Src/GBX.NET/Engines/Plug/CPlugMaterialCustom.chunkl +++ b/Src/GBX.NET/Engines/Plug/CPlugMaterialCustom.chunkl @@ -11,6 +11,13 @@ CPlugMaterialCustom 0x0903A000 GpuFx[] GpuFxs1 GpuFx[] GpuFxs2 +0x00B + uint U01 + ulong U02 + if (U01 & 1) != 0 + short U03 + short U04 + 0x00C BitmapSkip[] SkipSamplers diff --git a/Src/GBX.NET/Engines/Plug/CPlugPointsInSphereOpt.chunkl b/Src/GBX.NET/Engines/Plug/CPlugPointsInSphereOpt.chunkl new file mode 100644 index 000000000..555001436 --- /dev/null +++ b/Src/GBX.NET/Engines/Plug/CPlugPointsInSphereOpt.chunkl @@ -0,0 +1 @@ +CPlugPointsInSphereOpt 0x09066000 diff --git a/Src/GBX.NET/Engines/Plug/CPlugSound.chunkl b/Src/GBX.NET/Engines/Plug/CPlugSound.chunkl index 8c79de841..85c7f0e4f 100644 --- a/Src/GBX.NET/Engines/Plug/CPlugSound.chunkl +++ b/Src/GBX.NET/Engines/Plug/CPlugSound.chunkl @@ -1,2 +1,5 @@ CPlugSound 0x0901A000 -- inherits: CPlugAudio \ No newline at end of file +- inherits: CPlugAudio + +0x000 + CMwNod (external) \ No newline at end of file From 7c81acbe14cf3253fc6920cd0fa45883af80ae31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Thu, 26 Dec 2024 16:11:50 +0100 Subject: [PATCH 77/86] Update GBX.NET.PAK.csproj --- Src/GBX.NET.PAK/GBX.NET.PAK.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Src/GBX.NET.PAK/GBX.NET.PAK.csproj b/Src/GBX.NET.PAK/GBX.NET.PAK.csproj index ec20119ff..d99ab9cdc 100644 --- a/Src/GBX.NET.PAK/GBX.NET.PAK.csproj +++ b/Src/GBX.NET.PAK/GBX.NET.PAK.csproj @@ -2,7 +2,7 @@ GBX.NET.PAK - 2.0.0 + 2.0.0 BigBang1112 Support for Pak (NadeoPak) package files, integrated with GBX.NET. Copyright (c) 2024 Petr Pivoňka From eba120e2df6bf9c9df0ff57ade1aaadc5c3df1f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Thu, 26 Dec 2024 19:40:47 +0100 Subject: [PATCH 78/86] Add booltext (not sure if needed) --- .../ChunkL/MemberSerializationWriter.cs | 12 +++++++++++- .../ChunkL/PropertyTypeExtensions.cs | 2 +- Src/GBX.NET/Serialization/BoolType.cs | 8 ++++++++ Src/GBX.NET/Serialization/GbxReader.cs | 17 +++++++++++++++++ Src/GBX.NET/Serialization/GbxWriter.cs | 19 +++++++++++++++++++ 5 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 Src/GBX.NET/Serialization/BoolType.cs diff --git a/Generators/GBX.NET.Generators/ChunkL/MemberSerializationWriter.cs b/Generators/GBX.NET.Generators/ChunkL/MemberSerializationWriter.cs index da40bf895..ac9d054e3 100644 --- a/Generators/GBX.NET.Generators/ChunkL/MemberSerializationWriter.cs +++ b/Generators/GBX.NET.Generators/ChunkL/MemberSerializationWriter.cs @@ -463,6 +463,11 @@ private void AppendWrite(int indent, ChunkProperty chunkProperty) sb.Append(", asByte: true"); } + if (chunkProperty.Type.PrimaryType.ToLowerInvariant() == "booltext") + { + sb.Append(", type: BoolType.Text"); + } + if (chunkProperty.Properties?.TryGetValue("prefix", out var prefix) == true) { if (prefix == "byte") @@ -617,6 +622,11 @@ private void AppendReadWrite(int indent, ChunkProperty chunkProperty) sb.Append(", asByte: true"); } + if (chunkProperty.Type.PrimaryType.ToLowerInvariant() == "booltext") + { + sb.Append(", type: BoolType.Text"); + } + if (chunkProperty.Properties?.TryGetValue("prefix", out var prefix) == true) { if (prefix == "byte") @@ -834,7 +844,7 @@ public static string MapType(string type, out bool noMatch) "uint8" or "byte" => nameof(Byte), "int8" or "sbyte" => nameof(SByte), "float" => nameof(Single), - "bool" or "boolbyte" => nameof(Boolean), + "bool" or "boolbyte" or "booltext" => nameof(Boolean), "string" => nameof(String), "vec2" => "Vec2", "vec3" => "Vec3", diff --git a/Generators/GBX.NET.Generators/ChunkL/PropertyTypeExtensions.cs b/Generators/GBX.NET.Generators/ChunkL/PropertyTypeExtensions.cs index 418a8c184..43b5dd254 100644 --- a/Generators/GBX.NET.Generators/ChunkL/PropertyTypeExtensions.cs +++ b/Generators/GBX.NET.Generators/ChunkL/PropertyTypeExtensions.cs @@ -151,7 +151,7 @@ public static string MapType(string type) "dataint" => "int", "dataulong" => "ulong", "datalong" => "long", - "boolbyte" => "bool", + "boolbyte" or "booltext" => "bool", "optimizedint" => "int", "node" => "CMwNod", _ => type diff --git a/Src/GBX.NET/Serialization/BoolType.cs b/Src/GBX.NET/Serialization/BoolType.cs new file mode 100644 index 000000000..e8861ac03 --- /dev/null +++ b/Src/GBX.NET/Serialization/BoolType.cs @@ -0,0 +1,8 @@ +namespace GBX.NET.Serialization; + +public enum BoolType +{ + Int32, + Byte, + Text +} diff --git a/Src/GBX.NET/Serialization/GbxReader.cs b/Src/GBX.NET/Serialization/GbxReader.cs index 01ec07b05..100186a85 100644 --- a/Src/GBX.NET/Serialization/GbxReader.cs +++ b/Src/GBX.NET/Serialization/GbxReader.cs @@ -63,6 +63,7 @@ public partial interface IGbxReader : IDisposable TransQuat ReadTransQuat(); bool ReadBoolean(); bool ReadBoolean(bool asByte); + bool ReadBoolean(BoolType type); byte[] ReadData(); Task ReadDataAsync(CancellationToken cancellationToken = default); byte[] ReadData(int length); @@ -782,6 +783,22 @@ public bool ReadBoolean(bool asByte) return booleanAsByte != 0; } + public bool ReadBoolean(BoolType type) + { + switch (type) + { + case BoolType.Int32: + return ReadBoolean(); + case BoolType.Byte: + return ReadBoolean(asByte: true); + case BoolType.Text: + var bytes = ReadString(20); + return bool.Parse(ReadToCRLF()); + default: + throw new ArgumentException("Invalid boolean type.", nameof(type)); + } + } + public override string ReadString() { limiter?.ThrowIfLimitExceeded(sizeof(int)); diff --git a/Src/GBX.NET/Serialization/GbxWriter.cs b/Src/GBX.NET/Serialization/GbxWriter.cs index 44780ff94..a6cb1cb92 100644 --- a/Src/GBX.NET/Serialization/GbxWriter.cs +++ b/Src/GBX.NET/Serialization/GbxWriter.cs @@ -42,6 +42,7 @@ public partial interface IGbxWriter : IDisposable void WriteDataUInt64(ulong value); void Write(bool value); void Write(bool value, bool asByte); + void Write(bool value, BoolType type); void Write(string? value); void Write(string? value, StringLengthPrefix lengthPrefix); void WriteGbxMagic(); @@ -267,6 +268,24 @@ public void Write(bool value, bool asByte) } } + public void Write(bool value, BoolType type) + { + switch (type) + { + case BoolType.Int32: + Write(value); + return; + case BoolType.Byte: + Write(value, asByte: true); + return; + case BoolType.Text: + Write(value ? "True\r\n" : "False\r\n"); + return; + default: + throw new ArgumentException("Invalid boolean type.", nameof(type)); + } + } + public void Write(string? value, StringLengthPrefix lengthPrefix) { switch (lengthPrefix) From 402ef16eeddac70e018c69c5e29b566337e7fb08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Fri, 27 Dec 2024 03:37:45 +0100 Subject: [PATCH 79/86] Tweak default intro text --- Src/GBX.NET.Tool.CLI/ToolConsole.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Src/GBX.NET.Tool.CLI/ToolConsole.cs b/Src/GBX.NET.Tool.CLI/ToolConsole.cs index 6d8e5b6be..337b6874a 100644 --- a/Src/GBX.NET.Tool.CLI/ToolConsole.cs +++ b/Src/GBX.NET.Tool.CLI/ToolConsole.cs @@ -166,7 +166,7 @@ public async Task RunAsync(ExecuteLogic executeLogic, CancellationToken cancella AnsiConsole.WriteLine(); - throw new ConsoleProblemException("No files were passed to the tool.\nPlease drag and drop files onto the executable, or include the input paths as the command line arguments.\nFile paths, directory paths, or URLs are supported in any order."); + throw new ConsoleProblemException("No files were passed to the tool.\nPlease drag and drop files onto the executable (not this window), or provide the input paths as command line arguments.\nFile paths, directory paths, or URLs are supported in any order."); } var configName = string.IsNullOrWhiteSpace(toolSettings.ConsoleSettings.ConfigName) ? "Default" From 0a759ded308e2b94b2db2e97146eea3c5004a9f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Sat, 28 Dec 2024 02:16:16 +0100 Subject: [PATCH 80/86] Add Zlib to nightly --- .github/workflows/nightly.yml | 4 ++++ Src/GBX.NET.ZLib/GBX.NET.ZLib.csproj | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 18e840949..1a106f378 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -39,6 +39,7 @@ jobs: dotnet build Src/GBX.NET.Tool -c Release --no-restore /p:ContinuousIntegrationBuild=true --version-suffix nightly.$(date +'%Y%m%d').${git_hash::7} dotnet build Src/GBX.NET.Tool.CLI -c Release --no-restore /p:ContinuousIntegrationBuild=true --version-suffix nightly.$(date +'%Y%m%d').${git_hash::7} dotnet build Src/GBX.NET.PAK -c Release --no-restore /p:ContinuousIntegrationBuild=true --version-suffix nightly.$(date +'%Y%m%d').${git_hash::7} + dotnet build Src/GBX.NET.ZLib -c Release --no-restore /p:ContinuousIntegrationBuild=true --version-suffix nightly.$(date +'%Y%m%d').${git_hash::7} - name: Publish nightly GBX.NET nupkg to nuget.gbx.tools run: dotnet nuget push Src/GBX.NET/bin/Release/*.nupkg -k ${{ secrets.NUGET_GBXTOOLS_API_KEY }} -s https://nuget.gbx.tools/v3/index.json --skip-duplicate @@ -51,3 +52,6 @@ jobs: - name: Publish nightly GBX.NET.PAK nupkg to nuget.gbx.tools run: dotnet nuget push Src/GBX.NET.PAK/bin/Release/*.nupkg -k ${{ secrets.NUGET_GBXTOOLS_API_KEY }} -s https://nuget.gbx.tools/v3/index.json --skip-duplicate + + - name: Publish nightly GBX.NET.ZLib nupkg to nuget.gbx.tools + run: dotnet nuget push Src/GBX.NET.ZLib/bin/Release/*.nupkg -k ${{ secrets.NUGET_GBXTOOLS_API_KEY }} -s https://nuget.gbx.tools/v3/index.json --skip-duplicate diff --git a/Src/GBX.NET.ZLib/GBX.NET.ZLib.csproj b/Src/GBX.NET.ZLib/GBX.NET.ZLib.csproj index 5d6c14cb7..43040eeea 100644 --- a/Src/GBX.NET.ZLib/GBX.NET.ZLib.csproj +++ b/Src/GBX.NET.ZLib/GBX.NET.ZLib.csproj @@ -2,7 +2,7 @@ GBX.NET.ZLib - 1.1.0 + 1.1.1 BigBang1112 Support for zlib compressed parts of Gbx, integrated with GBX.NET. This official implementation uses Iconic.Zlib.Netstandard. Copyright (c) 2024 Petr Pivoňka From 6e9e9d28b724a5c253beef9d1afd972d4dd6dce5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Sat, 28 Dec 2024 22:18:28 +0100 Subject: [PATCH 81/86] Update NativeSharpZlib to 0.2.5 --- Src/GBX.NET.PAK/GBX.NET.PAK.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Src/GBX.NET.PAK/GBX.NET.PAK.csproj b/Src/GBX.NET.PAK/GBX.NET.PAK.csproj index d99ab9cdc..ec7995a53 100644 --- a/Src/GBX.NET.PAK/GBX.NET.PAK.csproj +++ b/Src/GBX.NET.PAK/GBX.NET.PAK.csproj @@ -45,7 +45,7 @@ all runtime; build; native; contentfiles; analyzers - + all runtime; build; native; contentfiles; analyzers; buildtransitive From fde2747880a80fd5af3572288a28510bc4e47a82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Sun, 29 Dec 2024 00:23:42 +0100 Subject: [PATCH 82/86] Fix ArchetypeBlockInfoCollectionId issues with numeric collections --- Src/GBX.NET/Engines/GameData/CGameBlockItem.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Src/GBX.NET/Engines/GameData/CGameBlockItem.cs b/Src/GBX.NET/Engines/GameData/CGameBlockItem.cs index e78d45b3f..182a369b3 100644 --- a/Src/GBX.NET/Engines/GameData/CGameBlockItem.cs +++ b/Src/GBX.NET/Engines/GameData/CGameBlockItem.cs @@ -6,9 +6,9 @@ public partial class CGameBlockItem [AppliedWithChunk] public string ArchetypeBlockInfoId { get => archetypeBlockInfoId; set => archetypeBlockInfoId = value; } - private string archetypeBlockInfoCollectionId = string.Empty; + private Id archetypeBlockInfoCollectionId = new(); [AppliedWithChunk] - public string ArchetypeBlockInfoCollectionId { get => archetypeBlockInfoCollectionId; set => archetypeBlockInfoCollectionId = value; } + public Id ArchetypeBlockInfoCollectionId { get => archetypeBlockInfoCollectionId; set => archetypeBlockInfoCollectionId = value; } private List customizedVariants = []; [AppliedWithChunk] From 866dc5d39fadffef3b46d267f9cc98f324e3943e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Sun, 29 Dec 2024 00:28:18 +0100 Subject: [PATCH 83/86] Fix paths on linux --- Tools/PakToZip/Program.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/PakToZip/Program.cs b/Tools/PakToZip/Program.cs index fbc02dbb7..d148c0156 100644 --- a/Tools/PakToZip/Program.cs +++ b/Tools/PakToZip/Program.cs @@ -24,7 +24,7 @@ foreach (var file in pak.Files.Values) { var fileName = hashes.GetValueOrDefault(file.Name) ?? file.Name; - var fullPath = Path.Combine(file.FolderPath, fileName); + var fullPath = Path.Combine(file.FolderPath, fileName).Replace('\\', '/'); Console.WriteLine(fullPath); From 083876e26814c23306eaa1aaa43a40725251e76f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Sun, 29 Dec 2024 00:39:46 +0100 Subject: [PATCH 84/86] Make paths better crossplatform --- Src/GBX.NET.PAK/Pak.cs | 4 ++-- Tools/PakToZip/Program.cs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Src/GBX.NET.PAK/Pak.cs b/Src/GBX.NET.PAK/Pak.cs index 7c66d0d33..de701f916 100644 --- a/Src/GBX.NET.PAK/Pak.cs +++ b/Src/GBX.NET.PAK/Pak.cs @@ -162,9 +162,9 @@ private static ImmutableDictionary ReadAllFiles(GbxReader r, Pa var classId = r.ReadUInt32(); // indicates the type of the file var fileFlags = r.ReadUInt64(); - var folderPath = string.Concat(RecurseFoldersToParent(folderIndex, allFolders) + var folderPath = string.Join(Path.DirectorySeparatorChar, RecurseFoldersToParent(folderIndex, allFolders) .Reverse() - .Select(f => f.Name)); + .Select(f => f.Name.TrimEnd('\\'))); var filePath = string.Concat(folderPath, name); var file = new PakFile(name, folderPath, classId, offset, uncompressedSize, compressedSize, fileFlags); diff --git a/Tools/PakToZip/Program.cs b/Tools/PakToZip/Program.cs index d148c0156..fbc02dbb7 100644 --- a/Tools/PakToZip/Program.cs +++ b/Tools/PakToZip/Program.cs @@ -24,7 +24,7 @@ foreach (var file in pak.Files.Values) { var fileName = hashes.GetValueOrDefault(file.Name) ?? file.Name; - var fullPath = Path.Combine(file.FolderPath, fileName).Replace('\\', '/'); + var fullPath = Path.Combine(file.FolderPath, fileName); Console.WriteLine(fullPath); From 3a07eb19ddd67c97837c378b99b106ef33ea55e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Sun, 29 Dec 2024 00:44:22 +0100 Subject: [PATCH 85/86] Fix directory char --- Src/GBX.NET.PAK/Pak.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Src/GBX.NET.PAK/Pak.cs b/Src/GBX.NET.PAK/Pak.cs index de701f916..fd2c1b297 100644 --- a/Src/GBX.NET.PAK/Pak.cs +++ b/Src/GBX.NET.PAK/Pak.cs @@ -291,9 +291,9 @@ public static async Task> BruteforceFileHashesAsync( progress?.Report(new KeyValuePair(hash, filePath)); allPossibleFileHashes[hash] = filePath; - while (filePath.Contains('\\')) + while (filePath.Contains(Path.DirectorySeparatorChar)) { - filePath = filePath.Substring(filePath.IndexOf('\\') + 1); + filePath = filePath.Substring(filePath.IndexOf(Path.DirectorySeparatorChar) + 1); hash = MD5.Compute136(filePath); progress?.Report(new KeyValuePair(hash, filePath)); allPossibleFileHashes[hash] = filePath; From 54be67b378c12b0c67516d296f5e4b519f39c1a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Pivo=C5=88ka?= Date: Sun, 29 Dec 2024 01:05:40 +0100 Subject: [PATCH 86/86] Fix various crossplatform slash issues --- Src/GBX.NET.PAK/Pak.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Src/GBX.NET.PAK/Pak.cs b/Src/GBX.NET.PAK/Pak.cs index fd2c1b297..a9bb897c3 100644 --- a/Src/GBX.NET.PAK/Pak.cs +++ b/Src/GBX.NET.PAK/Pak.cs @@ -154,7 +154,7 @@ private static ImmutableDictionary ReadAllFiles(GbxReader r, Pa for (var i = 0; i < numFiles; i++) { var folderIndex = r.ReadInt32(); // index into folders - var name = r.ReadString(); + var name = r.ReadString().Replace('\\', Path.DirectorySeparatorChar); var u01 = r.ReadInt32(); var uncompressedSize = r.ReadInt32(); var compressedSize = r.ReadInt32(); @@ -165,7 +165,7 @@ private static ImmutableDictionary ReadAllFiles(GbxReader r, Pa var folderPath = string.Join(Path.DirectorySeparatorChar, RecurseFoldersToParent(folderIndex, allFolders) .Reverse() .Select(f => f.Name.TrimEnd('\\'))); - var filePath = string.Concat(folderPath, name); + var filePath = Path.Combine(folderPath, name); var file = new PakFile(name, folderPath, classId, offset, uncompressedSize, compressedSize, fileFlags); files[filePath] = file; @@ -286,14 +286,14 @@ public static async Task> BruteforceFileHashesAsync( foreach (var refTableFile in refTable.Files) { - var filePath = refTableFile.FilePath; + var filePath = refTableFile.FilePath.Replace('/', '\\'); var hash = MD5.Compute136(filePath); progress?.Report(new KeyValuePair(hash, filePath)); allPossibleFileHashes[hash] = filePath; - while (filePath.Contains(Path.DirectorySeparatorChar)) + while (filePath.Contains('\\')) { - filePath = filePath.Substring(filePath.IndexOf(Path.DirectorySeparatorChar) + 1); + filePath = filePath.Substring(filePath.IndexOf('\\') + 1); hash = MD5.Compute136(filePath); progress?.Report(new KeyValuePair(hash, filePath)); allPossibleFileHashes[hash] = filePath;