Merge branch 'dev' into tmunlimiter

.github/workflows/ci.yml

@@ -25,8 +25,9 @@ jobs:
       uses: actions/setup-dotnet@v4
       with:
         dotnet-version: |
-          6.0.x
+          9.0.x
           8.0.x
+          6.0.x
           
     - name: Install wasm-tools
       run: dotnet workload install wasm-tools

.github/workflows/deploy-explorer-pages.yml

@@ -35,8 +35,9 @@ jobs:
         uses: actions/setup-dotnet@v4
         with:
           dotnet-version: |
-            6.0.x
+            9.0.x
             8.0.x
+            6.0.x
       
       - name: Install wasm-tools
         run: dotnet workload install wasm-tools

.github/workflows/nightly.yml

@@ -22,8 +22,12 @@ jobs:
       uses: actions/setup-dotnet@v4
       with:
         dotnet-version: |
+          9.0.x
           8.0.x
           6.0.x
+      
+    - name: Install wasm-tools
+      run: dotnet workload install wasm-tools
 
     - name: Restore dependencies
       run: dotnet restore
@@ -34,6 +38,8 @@ jobs:
         dotnet build Src/GBX.NET -c Release --no-restore /p:ContinuousIntegrationBuild=true --version-suffix nightly.$(date +'%Y%m%d').${git_hash::7}
         dotnet build Src/GBX.NET.Tool -c Release --no-restore /p:ContinuousIntegrationBuild=true --version-suffix nightly.$(date +'%Y%m%d').${git_hash::7}
         dotnet build Src/GBX.NET.Tool.CLI -c Release --no-restore /p:ContinuousIntegrationBuild=true --version-suffix nightly.$(date +'%Y%m%d').${git_hash::7}
+        dotnet build Src/GBX.NET.PAK -c Release --no-restore /p:ContinuousIntegrationBuild=true --version-suffix nightly.$(date +'%Y%m%d').${git_hash::7}
+        dotnet build Src/GBX.NET.ZLib -c Release --no-restore /p:ContinuousIntegrationBuild=true --version-suffix nightly.$(date +'%Y%m%d').${git_hash::7}
 
     - name: Publish nightly GBX.NET nupkg to nuget.gbx.tools
       run: dotnet nuget push Src/GBX.NET/bin/Release/*.nupkg -k ${{ secrets.NUGET_GBXTOOLS_API_KEY }} -s https://nuget.gbx.tools/v3/index.json --skip-duplicate
@@ -43,3 +49,9 @@ jobs:
 
     - name: Publish nightly GBX.NET.Tool.CLI nupkg to nuget.gbx.tools
       run: dotnet nuget push Src/GBX.NET.Tool.CLI/bin/Release/*.nupkg -k ${{ secrets.NUGET_GBXTOOLS_API_KEY }} -s https://nuget.gbx.tools/v3/index.json --skip-duplicate
+
+    - name: Publish nightly GBX.NET.PAK nupkg to nuget.gbx.tools
+      run: dotnet nuget push Src/GBX.NET.PAK/bin/Release/*.nupkg -k ${{ secrets.NUGET_GBXTOOLS_API_KEY }} -s https://nuget.gbx.tools/v3/index.json --skip-duplicate
+
+    - name: Publish nightly GBX.NET.ZLib nupkg to nuget.gbx.tools
+      run: dotnet nuget push Src/GBX.NET.ZLib/bin/Release/*.nupkg -k ${{ secrets.NUGET_GBXTOOLS_API_KEY }} -s https://nuget.gbx.tools/v3/index.json --skip-duplicate

.github/workflows/publish.yml

@@ -37,6 +37,7 @@ jobs:
       uses: actions/setup-dotnet@v4
       with:
         dotnet-version: |
+          9.0.x
           8.0.x
           6.0.x
           
@@ -72,6 +73,7 @@ jobs:
         - GBX.NET.Tool
         - GBX.NET.Tool.CLI
         - GBX.NET.Crypto
+        - GBX.NET.PAK
 
     name: Publish ${{ matrix.lib }}
     runs-on: ubuntu-latest

Benchmarks/GBX.NET.Benchmarks/GBX.NET.Benchmarks.csproj

@@ -2,7 +2,7 @@
 
 	<PropertyGroup>
 		<OutputType>Exe</OutputType>
-		<TargetFramework>net8.0</TargetFramework>
+		<TargetFramework>net9.0</TargetFramework>
 		<ImplicitUsings>enable</ImplicitUsings>
 		<Nullable>enable</Nullable>
 	</PropertyGroup>

GBX.NET.sln

@@ -174,6 +174,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MuxCryptSwitch", "Tools\Mux
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GBX.NET.Tool.StandardIO", "Src\GBX.NET.Tool.StandardIO\GBX.NET.Tool.StandardIO.csproj", "{55BD1330-7431-41A8-90EB-DEB20D9EB981}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PakToZip", "Tools\PakToZip\PakToZip.csproj", "{47A58E42-EBBC-48D0-B4B1-C25DFD074506}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExtractPakFileHashes", "Tools\ExtractPakFileHashes\ExtractPakFileHashes.csproj", "{B3C5D493-3172-4780-95EC-CBEE73BE4850}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -412,6 +416,14 @@ Global
 		{55BD1330-7431-41A8-90EB-DEB20D9EB981}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{55BD1330-7431-41A8-90EB-DEB20D9EB981}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{55BD1330-7431-41A8-90EB-DEB20D9EB981}.Release|Any CPU.Build.0 = Release|Any CPU
+		{47A58E42-EBBC-48D0-B4B1-C25DFD074506}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{47A58E42-EBBC-48D0-B4B1-C25DFD074506}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{47A58E42-EBBC-48D0-B4B1-C25DFD074506}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{47A58E42-EBBC-48D0-B4B1-C25DFD074506}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B3C5D493-3172-4780-95EC-CBEE73BE4850}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B3C5D493-3172-4780-95EC-CBEE73BE4850}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B3C5D493-3172-4780-95EC-CBEE73BE4850}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B3C5D493-3172-4780-95EC-CBEE73BE4850}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -481,6 +493,8 @@ Global
 		{89F6FFE5-E32A-4713-8D63-98D070A17DC1} = {80DCE6B7-4BD9-415C-B053-92B059D7C938}
 		{0B5609C6-7B10-4E8D-B41C-35CC1A2532A4} = {F3336145-FDA9-4517-AEDC-7F4C4D526ECB}
 		{55BD1330-7431-41A8-90EB-DEB20D9EB981} = {80DCE6B7-4BD9-415C-B053-92B059D7C938}
+		{47A58E42-EBBC-48D0-B4B1-C25DFD074506} = {F3336145-FDA9-4517-AEDC-7F4C4D526ECB}
+		{B3C5D493-3172-4780-95EC-CBEE73BE4850} = {F3336145-FDA9-4517-AEDC-7F4C4D526ECB}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {8EA2F0DE-BA72-486D-AB3A-9320C0CE5CFD}

Generators/GBX.NET.Generators/ChunkL/MemberSerializationWriter.cs

@@ -463,6 +463,11 @@ private void AppendWrite(int indent, ChunkProperty chunkProperty)
             sb.Append(", asByte: true");
         }
 
+        if (chunkProperty.Type.PrimaryType.ToLowerInvariant() == "booltext")
+        {
+            sb.Append(", type: BoolType.Text");
+        }
+
         if (chunkProperty.Properties?.TryGetValue("prefix", out var prefix) == true)
         {
             if (prefix == "byte")
@@ -617,6 +622,11 @@ private void AppendReadWrite(int indent, ChunkProperty chunkProperty)
             sb.Append(", asByte: true");
         }
 
+        if (chunkProperty.Type.PrimaryType.ToLowerInvariant() == "booltext")
+        {
+            sb.Append(", type: BoolType.Text");
+        }
+
         if (chunkProperty.Properties?.TryGetValue("prefix", out var prefix) == true)
         {
             if (prefix == "byte")
@@ -834,7 +844,7 @@ public static string MapType(string type, out bool noMatch)
             "uint8" or "byte" => nameof(Byte),
             "int8" or "sbyte" => nameof(SByte),
             "float" => nameof(Single),
-            "bool" or "boolbyte" => nameof(Boolean),
+            "bool" or "boolbyte" or "booltext" => nameof(Boolean),
             "string" => nameof(String),
             "vec2" => "Vec2",
             "vec3" => "Vec3",

Generators/GBX.NET.Generators/ChunkL/PropertyTypeExtensions.cs

@@ -151,7 +151,7 @@ public static string MapType(string type)
             "dataint" => "int",
             "dataulong" => "ulong",
             "datalong" => "long",
-            "boolbyte" => "bool",
+            "boolbyte" or "booltext" => "bool",
             "optimizedint" => "int",
             "node" => "CMwNod",
             _ => type

Generators/GBX.NET.Generators/SubGenerators/ClassDataSubGenerator.cs

@@ -1194,17 +1194,23 @@ private static void AppendChunkLine(
             }
             else if (structureKind.GetValueOrDefault() == 0)
             {
-                sb.AppendLine();
-                sb.Append("        public override void ReadWrite(");
-                sb.Append(classInfo.Name);
-                sb.AppendLine(" n, GbxReaderWriter rw)");
-                sb.AppendLine("        {");
+                var doReadWriteMethod = !existingChunkMembers.OfType<IMethodSymbol>()
+                    .Any(x => x.Name == "ReadWrite" && x.Parameters.Length == 2 && x.Parameters[0].Type.Name == classInfo.Name && x.Parameters[1].Type.Name == "GbxReaderWriter");
+
+                if (doReadWriteMethod)
+                {
+                    sb.AppendLine();
+                    sb.Append("        public override void ReadWrite(");
+                    sb.Append(classInfo.Name);
+                    sb.AppendLine(" n, GbxReaderWriter rw)");
+                    sb.AppendLine("        {");
 
-                var memberWriter = new MemberSerializationWriter(
-                    sb, SerializationType.ReadWrite, archive: null, existingFields, existingProps, classInfo, classInfos, archiveInfos, autoProperty: false, context);
-                memberWriter.Append(indent: 3, chunk.ChunkLDefinition.Members);
+                    var memberWriter = new MemberSerializationWriter(
+                        sb, SerializationType.ReadWrite, archive: null, existingFields, existingProps, classInfo, classInfos, archiveInfos, autoProperty: false, context);
+                    memberWriter.Append(indent: 3, chunk.ChunkLDefinition.Members);
 
-                sb.AppendLine("        }");
+                    sb.AppendLine("        }");
+                }
             }
         }
 

README.md

@@ -24,12 +24,14 @@ For any questions, open an issue, join the [GameBox Sandbox Discord server](http
   - [Create a new GBX.NET project (lightweight)](#create-a-new-gbxnet-project-lightweight)
   - [Create a new GBX.NET project (Visual Studio Code)](#create-a-new-gbxnet-project-visual-studio-code)
   - [Create a new GBX.NET project (Visual Studio)](#create-a-new-gbxnet-project-visual-studio)
+- **[IMPORTANT INFO about the LZO and Gbx compression](#important-info-about-the-lzo-and-gbx-compression)**
 - **[Usage (simple examples)](#usage-simple-examples)**
   - [Load a map and display block count per block name](#load-a-map-and-display-block-count-per-block-name)
   - [Modify and save a map](#modify-and-save-a-map)
   - [Processing multiple Gbx types](#processing-multiple-gbx-types)
   - [Read a large amount of replay metadata quickly](#read-a-large-amount-of-replay-metadata-quickly)
 - [Tool framework](#tool-framework)
+- [Zlib compression in Gbx](#zlib-compression-in-gbx)
 - [Clarity](#clarity)
   - [Differences between `Gbx.Parse/Header/Node`](#differences-between-gbxparseheadernode)
   - [Do not repeat `gbx.Node.[any]` too often!](#do-not-repeat-gbxnodeany-too-often)
@@ -42,6 +44,7 @@ For any questions, open an issue, join the [GameBox Sandbox Discord server](http
   - [Asynchronous](#asynchronous)
 - [Benchmarks](#benchmarks)
 - [Build](#build)
+  - [Nightly builds](#nightly-builds)
 - [License](#license)
 - [Special thanks](#special-thanks)
 - [Alternative Gbx parsers](#alternative-gbx-parsers)
@@ -50,7 +53,7 @@ For any questions, open an issue, join the [GameBox Sandbox Discord server](http
 
 Many *essential* Gbx files from many games are supported:
 
-- **Trackmania (2020)**, April 2024 update
+- **Trackmania (2020)**, October 2024 update
 - **ManiaPlanet 4**(.1), TM2/SM
 - **Trackmania Turbo**
 - ManiaPlanet 3, TM2/SM
@@ -106,6 +109,7 @@ The goal is to also make it viable for NativeAOT.
 
 Due to the recently paced evolution of .NET, framework support has been limited only to a few ones compared to GBX.NET 1:
 
+- **.NET 9**
 - .NET 8
 - .NET 6
 - .NET Standard 2.0
@@ -118,8 +122,8 @@ Using the NuGet packages is recommended.
 
 ### Create a new GBX.NET project (lightweight)
 
-1. Install [.NET SDK 8](https://dotnet.microsoft.com/en-us/download/dotnet/8.0).
-    - Windows: [here](https://dotnet.microsoft.com/en-us/download) or `winget install Microsoft.DotNet.SDK.8` (make sure you have WinGet installed)
+1. Install [.NET SDK 9](https://dotnet.microsoft.com/en-us/download/dotnet/9.0).
+    - Windows: [here](https://dotnet.microsoft.com/en-us/download) or `winget install Microsoft.DotNet.SDK.9` (make sure you have WinGet installed)
     - [Linux](https://learn.microsoft.com/en-us/dotnet/core/install/linux) (just SDK)
 2. Create directory for your project (anywhere), **go inside it**.
 3. Create new console project: `dotnet new console`
@@ -176,9 +180,9 @@ using GBX.NET.LZO;
 Gbx.LZO = new Lzo();
 ```
 
-You should run this line of code **only once** for the whole program lifetime.
+You should run this line of code **only once** at the start of the program.
 
-The compression logic is split up from the read/write logic to **allow GBX.NET 2 library to be distributed under the MIT license**, as Oberhumer distributes the open source version of LZO under the GNU GPL v3. Therefore, using GBX.NET.LZO 2 requires you to license your project under the GNU GPL v3, see [License](#license).
+The compression logic is split up from the read/write logic to **allow GBX.NET 2 library to be distributed under the MIT license**, as Oberhumer distributes the open source version of LZO under the GNU GPL v2+. Therefore, using GBX.NET.LZO 2 requires you to license your project under the GNU GPL v3, see [License](#license).
 
 **Gbx header is not compressed** and can contain useful information (icon data, replay time, ...), and also many of the **internal Gbx files from Pak files are not compressed**, so you can avoid LZO for these purposes.
 
@@ -373,6 +377,45 @@ Tool class accepts input through constructors (best one is picked according to i
 
 Samples are available [here](Samples/Tool/).
 
+## Zlib compression in Gbx
+
+There are a few places where Gbx includes additional zlib-compressed data:
+
+- Ghost samples
+  - `CGameGhost.SampleData`
+- Record data (general replay data or TM2020 ghost samples)
+  - `CGameCtnReplayRecord.RecordData`, `CGameCtnGhost.RecordData`, `CGameCtnMediaBlockEntity.RecordData`
+- Lightmap data of a map
+  - `CGameCtnChallenge.LightmapCache`, additional lightmap parameters
+
+GBX.NET does not include zlib algorithm by default to read this data. Properties will often return `null` silently, but you can hook onto more details about this if you specify `Logger` in the parse methods.
+
+To "unlock" this data, you need to specify zlib implementation:
+
+Command line:
+
+```
+dotnet add package GBX.NET.ZLib
+```
+
+C# code:
+
+```cs
+using GBX.NET;
+using GBX.NET.ZLib;
+
+Gbx.ZLib = new ZLib();
+```
+
+> [!NOTE]
+> Zlib-compressed data is **currently read-only for ghost samples and record data**, lightmap data is the only zlib data that can be modified with GBX.NET. Write support for ghost samples and record data is planned for 2.2.
+
+The data is often stored in properties of type `CompressedData` which are byte arrays with additional uncompressed data size for validation. If there's a property with this type (`CGameCtnChallenge.LightmapCacheData` for example), zlib data will be stored there no matter if the zlib implementation is included, so read/write consistency is guaranteed.
+
+> The current reason why the zlib implementation is split similarly to LZO (even when zlib license is perfectly fine) is that there aren't any good official solutions by Microsoft that would work for .NET Standard 2 (new `ZlibStream` is promising but .NET 6+ only, still not tested enough if reliable), and third party solutions are also of a questionable quality. But this topic is continously under attention and the zlib implementation is slowly improving.
+
+GBX.NET.PAK uses a separate zlib solution due to very specific patterns to follow during decryption + decompression in .pak data.
+
 ## Clarity
 
 This section describes best practices to keep your projects clean when using GBX.NET 2.
@@ -575,18 +618,36 @@ TODO
 
 Make sure you have these framework SDKs available:
 
+- **.NET 9**
 - .NET 8
 - .NET 6
 - .NET Standard 2.0
 
-**Visual Studio 2022** should be able to install those with default installation settings. Using Visual Studio 2019 will likely not work.
+**Visual Studio 2022** should be able to install those with default installation settings. Using Visual Studio 2019 will not work.
 
 You should also have **.NET WebAssembly Build Tools** installed additionally to build the full solution. It is required for Gbx Explorer to work properly, as it uses native LZO implementation compiled into WebAssembly.
 
 In Visual Studio, you can just use Build Solution and everything should build. JetBrains Rider has been tested and also works.
 
 In .NET CLI, run `dotnet build` on the solution (`.sln`) level.
 
+### Nightly builds
+
+Every **5AM UTC**, there is a new build from the `dev` branch published on https://nuget.gbx.tools/ of the base GBX.NET package (soon also GBX.NET.Tool*).
+
+Go to `%appdata%/NuGet` and modify the `NuGet.Config` file to include the package source:
+```xml
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <packageSources>
+    <add key="nuget.gbx.tools" value="https://nuget.gbx.tools/v3/index.json" /> <!-- add this -->
+  </packageSources>
+</configuration>
+```
+Then you can select the `nuget.gbx.tools` package source and fetch nightly builds from there.
+
+> In the past, nightly builds were pushed to GitHub Packages which required you to provide access tokens to be able to read the packages. Nightly builds are no longer pushed to GitHub Packages.
+
 ## License
 
 GBX.NET 2 is licensed under multiple licenses, depending on the part of the project. Here are the licenses and their directories:
@@ -607,6 +668,7 @@ GBX.NET 2 is licensed under multiple licenses, depending on the part of the proj
 - GNU GPL v3 License
   - **Src/GBX.NET.LZO**
   - Src/GBX.NET.Tool.CLI
+  - **Src/GBX.NET.PAK**
   - Samples
   - Tools
 - The Unlicense

Resources/Unwrap.txt

@@ -54,7 +54,7 @@
 03088000 240C7000
 03089000 240C0000
 0308A000 2404D000
-0308C000 2404A000
+0308C000 240B5000
 0308D000 24034000
 0308E000 240A0000
 0308F000 24039000

Samples/Advanced/ItemPacker/ItemPacker.csproj

@@ -2,14 +2,14 @@
 
     <PropertyGroup>
         <OutputType>Exe</OutputType>
-        <TargetFramework>net8.0</TargetFramework>
+        <TargetFramework>net9.0</TargetFramework>
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
     </PropertyGroup>
 
     <ItemGroup>
-        <PackageReference Include="Microsoft.Extensions.Logging" Version="8.0.1" />
-        <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="8.0.1" />
+        <PackageReference Include="Microsoft.Extensions.Logging" Version="9.0.0" />
+        <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="9.0.0" />
     </ItemGroup>
 
     <ItemGroup>

Src/GBX.NET.Crypto/GBX.NET.Crypto.csproj

@@ -15,7 +15,7 @@
     </PropertyGroup>
 
     <PropertyGroup>
-        <TargetFrameworks>net8.0;net6.0;netstandard2.0</TargetFrameworks>
+        <TargetFrameworks>net9.0;net8.0;net6.0;netstandard2.0</TargetFrameworks>
         <LangVersion>12</LangVersion>
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>