From 0a102a4ecd6d55c7320263dddfecf5606956b8e3 Mon Sep 17 00:00:00 2001 From: Giorgi Date: Wed, 11 Oct 2023 17:59:08 +0400 Subject: [PATCH 1/8] Basic support for structs --- .../NativeMethods.LogicalType.cs | 9 ++ DuckDB.NET.Data/DuckDBDataReader.cs | 12 +- DuckDB.NET.Data/VectorDataReader.cs | 82 ++++++++++++ .../DuckDBDataReaderStructTests.cs | 120 ++++++++++++++++++ 4 files changed, 213 insertions(+), 10 deletions(-) create mode 100644 DuckDB.NET.Test/DuckDBDataReaderStructTests.cs diff --git a/DuckDB.NET.Bindings/NativeMethods/NativeMethods.LogicalType.cs b/DuckDB.NET.Bindings/NativeMethods/NativeMethods.LogicalType.cs index 561a0764..83e6e9bf 100644 --- a/DuckDB.NET.Bindings/NativeMethods/NativeMethods.LogicalType.cs +++ b/DuckDB.NET.Bindings/NativeMethods/NativeMethods.LogicalType.cs @@ -28,6 +28,15 @@ public static class LogicalType [DllImport(DuckDbLibrary, CallingConvention = CallingConvention.Cdecl, EntryPoint = "duckdb_list_type_child_type")] public static extern DuckDBLogicalType DuckDBListTypeChildType(DuckDBLogicalType type); + [DllImport(DuckDbLibrary, CallingConvention = CallingConvention.Cdecl, EntryPoint = "duckdb_struct_type_child_count")] + public static extern long DuckDBStructTypeChildCount(DuckDBLogicalType type); + + [DllImport(DuckDbLibrary, CallingConvention = CallingConvention.Cdecl, EntryPoint = "duckdb_struct_type_child_name")] + public static extern IntPtr DuckDBStructTypeChildName(DuckDBLogicalType type, long index); + + [DllImport(DuckDbLibrary, CallingConvention = CallingConvention.Cdecl, EntryPoint = "duckdb_struct_type_child_type")] + public static extern DuckDBLogicalType DuckDBStructTypeChildType(DuckDBLogicalType type, long index); + [DllImport(DuckDbLibrary, CallingConvention = CallingConvention.Cdecl, EntryPoint = "duckdb_destroy_logical_type")] public static extern void DuckDBDestroyLogicalType(out IntPtr type); } diff --git a/DuckDB.NET.Data/DuckDBDataReader.cs b/DuckDB.NET.Data/DuckDBDataReader.cs index fc8dd3bf..d8c1ae8f 100644 --- a/DuckDB.NET.Data/DuckDBDataReader.cs +++ b/DuckDB.NET.Data/DuckDBDataReader.cs @@ -195,19 +195,11 @@ public override T GetFieldValue(int ordinal) { DuckDBType.List => (T)vectorReaders[ordinal].GetList(rowsReadFromCurrentChunk - 1, typeof(T)), DuckDBType.Enum => (T)vectorReaders[ordinal].GetEnum(rowsReadFromCurrentChunk - 1, typeof(T)), + DuckDBType.Struct => (T)vectorReaders[ordinal].GetStruct(rowsReadFromCurrentChunk - 1, typeof(T)), _ => (T)vectorReaders[ordinal].GetValue(rowsReadFromCurrentChunk - 1) }; - if (value is not null) - return (T)value; - - if (default(T) is null && typeof(T).IsValueType) - return default!; - - if (typeof(T) == typeof(object)) - return (T)(object)DBNull.Value; - - throw new InvalidCastException(); + return value; } public override object GetValue(int ordinal) diff --git a/DuckDB.NET.Data/VectorDataReader.cs b/DuckDB.NET.Data/VectorDataReader.cs index a1d8e423..52d901a4 100644 --- a/DuckDB.NET.Data/VectorDataReader.cs +++ b/DuckDB.NET.Data/VectorDataReader.cs @@ -21,6 +21,7 @@ internal class VectorDataReader : IDisposable internal DuckDBType DuckDBType { get; } private readonly VectorDataReader? listDataReader; + private readonly Dictionary? structDataReaders; private readonly byte scale; private readonly DuckDBType decimalType; @@ -60,6 +61,7 @@ internal unsafe VectorDataReader(IntPtr vector, void* dataPointer, ulong* validi DuckDBType.Blob => typeof(Stream), DuckDBType.Enum => typeof(Enum), DuckDBType.List => typeof(List<>), + DuckDBType.Struct => typeof(Dictionary), var type => throw new ArgumentException($"Unrecognised type {type} ({(int)type})") }; @@ -85,6 +87,26 @@ internal unsafe VectorDataReader(IntPtr vector, void* dataPointer, ulong* validi listDataReader = new VectorDataReader(childVector, childVectorData, childVectorValidity, type); break; } + case DuckDBType.Struct: + { + var memberCount = NativeMethods.LogicalType.DuckDBStructTypeChildCount(logicalType); + structDataReaders = new Dictionary(StringComparer.OrdinalIgnoreCase); + + for (int index = 0; index < memberCount; index++) + { + var name = NativeMethods.LogicalType.DuckDBStructTypeChildName(logicalType, index).ToManagedString(); + var childVector = NativeMethods.DataChunks.DuckDBStructVectorGetChild(vector, index); + + var childVectorData = NativeMethods.DataChunks.DuckDBVectorGetData(childVector); + var childVectorValidity = NativeMethods.DataChunks.DuckDBVectorGetValidity(childVector); + + using var childType = NativeMethods.LogicalType.DuckDBStructTypeChildType(logicalType, index); + var type = NativeMethods.LogicalType.DuckDBGetTypeId(childType); + + structDataReaders[name] = new VectorDataReader(childVector, childVectorData, childVectorValidity, type); + } + break; + } } } @@ -268,6 +290,7 @@ internal object GetValue(ulong offset, Type? targetType = null) DuckDBType.Blob => GetStream(offset), DuckDBType.List => GetList(offset, targetType ?? typeof(List<>).MakeGenericType(listDataReader!.ClrType)), DuckDBType.Enum => GetEnum(offset, targetType ?? typeof(string)), + DuckDBType.Struct => GetStruct(offset, targetType ?? typeof(Dictionary)), var type => throw new ArgumentException($"Unrecognised type {type} ({(int)type})") }; } @@ -323,4 +346,63 @@ public void Dispose() listDataReader?.Dispose(); logicalType?.Dispose(); } + + internal object GetStruct(ulong offset, Type type) + { + if (structDataReaders is null) + { + throw new InvalidOperationException("Can't get a struct from a non-struct vector."); + } + + var result = Activator.CreateInstance(type); + + if (result is Dictionary dictionary) + { + //var dictionary = result as Dictionary; + foreach (var reader in structDataReaders) + { + var value = reader.Value.IsValid(offset) ? reader.Value.GetValue(offset) : null; + dictionary.Add(reader.Key, value); + } + + return result; + } + + var properties = type.GetProperties(); + foreach (var propertyInfo in properties) + { + if (propertyInfo.SetMethod == null) + { + continue; + } + + structDataReaders.TryGetValue(propertyInfo.Name, out var reader); + var isNotNullable = propertyInfo.PropertyType.IsValueType && Nullable.GetUnderlyingType(propertyInfo.PropertyType) == null; + + if (reader == null) + { + if (isNotNullable) + { + throw new NullReferenceException($"Property {propertyInfo.Name} not found in struct"); + } + + continue; + } + + if (reader.IsValid(offset)) + { + var value = reader.GetValue(offset, propertyInfo.PropertyType); + propertyInfo.SetValue(result, value); + } + else + { + if (isNotNullable) + { + throw new NullReferenceException(""); + } + } + } + + return result!; + } } \ No newline at end of file diff --git a/DuckDB.NET.Test/DuckDBDataReaderStructTests.cs b/DuckDB.NET.Test/DuckDBDataReaderStructTests.cs new file mode 100644 index 00000000..46141b01 --- /dev/null +++ b/DuckDB.NET.Test/DuckDBDataReaderStructTests.cs @@ -0,0 +1,120 @@ +using System.Collections.Generic; +using FluentAssertions; +using Xunit; + +namespace DuckDB.NET.Test; + +public class DuckDBDataReaderStructTests : DuckDBTestBase +{ + public DuckDBDataReaderStructTests(DuckDBDatabaseFixture db) : base(db) + { + } + + [Fact] + public void ReadBasicStruct() + { + Command.CommandText = "SELECT {'x': 1, 'y': 2, 'z': 'test'};"; + using var reader = Command.ExecuteReader(); + reader.Read(); + + var value = reader.GetFieldValue(0); + value.Should().BeEquivalentTo(new Struct1 + { + X = 1, + Y = 2, + Z = "test" + }); + } + + [Fact] + public void ReadBasicStructWithGetValue() + { + Command.CommandText = "SELECT {'x': 1, 'y': 2, 'z': 'test'};"; + using var reader = Command.ExecuteReader(); + reader.Read(); + + var value = reader.GetValue(0); + value.Should().BeEquivalentTo(new Dictionary { { "x", 1 }, { "y", 2 }, { "z", "test" } }); + } + + [Fact] + public void ReadBasicStructList() + { + Command.CommandText = "SELECT [{'x': 1, 'y': 2, 'z': 'test'}, {'x': 4, 'y': 3, 'z': 'tset'}, NULL];"; + using var reader = Command.ExecuteReader(); + reader.Read(); + + var value = reader.GetFieldValue>(0); + value.Should().BeEquivalentTo(new List + { + new() { X = 1, Y = 2, Z = "test" }, + new() { X = 4, Y = 3, Z = "tset" }, + null + }); + } + + [Fact] + public void ReadBasicStructListWithGetValue() + { + Command.CommandText = "SELECT [{'x': 1, 'y': 2, 'z': 'test'}, {'x': 4, 'y': 3, 'z': 'tset'}, NULL];"; + using var reader = Command.ExecuteReader(); + reader.Read(); + + var value = reader.GetValue(0); + value.Should().BeEquivalentTo(new List> + { + new() { { "x", 1 }, { "y", 2 }, { "z", "test" } }, + new() { { "x", 4 }, { "y", 3 }, { "z", "tset" } }, + null + }); + } + + [Fact] + public void ReadStructWithNull() + { + Command.CommandText = "SELECT {'yes': 'duck', 'maybe': 'goose', 'huh': NULL, 'no': 'heron', 'type': 0} Union All " + + "SELECT {'yes': 'duck', 'maybe': 'goose', 'huh': 'bird', 'no': 'heron', 'type':1};"; + using var reader = Command.ExecuteReader(); + + reader.Read(); + var value = reader.GetFieldValue(0); + + value.Should().BeEquivalentTo(new Struct2 + { + Huh = null, + Maybe = "goose", + No = "heron", + Yes = "duck", + Type = 0 + }); + + reader.Read(); + value = reader.GetFieldValue(0); + + value.Should().BeEquivalentTo(new Struct2 + { + Huh = "bird", + Maybe = "goose", + No = "heron", + Yes = "duck", + Type = 1 + }); + } + + class Struct1 + { + public int X { get; set; } + public int Y { get; set; } + public string Z { get; set; } + public int XY { get; } + } + + class Struct2 + { + public string Yes { get; set; } + public string Maybe { get; set; } + public string Huh { get; set; } + public string No { get; set; } + public int Type { get; set; } + } +} \ No newline at end of file From 7ddc497a30451afb2d402f9439c4790be6a75557 Mon Sep 17 00:00:00 2001 From: Giorgi Date: Wed, 11 Oct 2023 23:35:04 +0400 Subject: [PATCH 2/8] Nested struct test --- DuckDB.NET.Data/VectorDataReader.cs | 2 +- .../DuckDBDataReaderStructTests.cs | 40 +++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/DuckDB.NET.Data/VectorDataReader.cs b/DuckDB.NET.Data/VectorDataReader.cs index 52d901a4..1243e918 100644 --- a/DuckDB.NET.Data/VectorDataReader.cs +++ b/DuckDB.NET.Data/VectorDataReader.cs @@ -383,7 +383,7 @@ internal object GetStruct(ulong offset, Type type) { if (isNotNullable) { - throw new NullReferenceException($"Property {propertyInfo.Name} not found in struct"); + throw new NullReferenceException($"Property '{propertyInfo.Name}' not found in struct"); } continue; diff --git a/DuckDB.NET.Test/DuckDBDataReaderStructTests.cs b/DuckDB.NET.Test/DuckDBDataReaderStructTests.cs index 46141b01..87ed04d7 100644 --- a/DuckDB.NET.Test/DuckDBDataReaderStructTests.cs +++ b/DuckDB.NET.Test/DuckDBDataReaderStructTests.cs @@ -100,6 +100,39 @@ public void ReadStructWithNull() Type = 1 }); } + + [Fact] + public void ReadStructWithNestedStruct() + { + Command.CommandText = + "SELECT {'birds': {'yes': 'duck', 'maybe': 'goose', 'huh': NULL, 'no': 'heron', 'type': 0}, " + + "'aliens': NULL, " + + "'amphibians': {'yes':'frog', 'maybe': 'salamander', 'huh': 'dragon', 'no':'toad', 'type':1} };"; + using var reader = Command.ExecuteReader(); + + reader.Read(); + var value = reader.GetFieldValue(0); + + value.Aliens.Should().BeNull(); + + value.Birds.Should().BeEquivalentTo(new Struct2 + { + Huh = null, + Maybe = "goose", + No = "heron", + Yes = "duck", + Type = 0 + }); + + value.Amphibians.Should().BeEquivalentTo(new Struct2 + { + Huh = "dragon", + Maybe = "salamander", + No = "toad", + Yes = "frog", + Type = 1 + }); + } class Struct1 { @@ -117,4 +150,11 @@ class Struct2 public string No { get; set; } public int Type { get; set; } } + + class Struct3 + { + public Struct2 Birds { get; set; } + public Struct2 Aliens { get; set; } + public Struct2 Amphibians { get; set; } + } } \ No newline at end of file From 6435799cefda23bde373831bf7a120895aea963e Mon Sep 17 00:00:00 2001 From: Giorgi Date: Thu, 12 Oct 2023 00:12:24 +0400 Subject: [PATCH 3/8] Add tests --- DuckDB.NET.Data/VectorDataReader.cs | 2 +- DuckDB.NET.Test/DuckDBDataReaderEnumTests.cs | 16 ++++------- .../DuckDBDataReaderStructTests.cs | 28 +++++++++++++++---- 3 files changed, 30 insertions(+), 16 deletions(-) diff --git a/DuckDB.NET.Data/VectorDataReader.cs b/DuckDB.NET.Data/VectorDataReader.cs index 1243e918..ed6919b4 100644 --- a/DuckDB.NET.Data/VectorDataReader.cs +++ b/DuckDB.NET.Data/VectorDataReader.cs @@ -398,7 +398,7 @@ internal object GetStruct(ulong offset, Type type) { if (isNotNullable) { - throw new NullReferenceException(""); + throw new NullReferenceException($"Property '{propertyInfo.Name}' is not nullable but struct contains null"); } } } diff --git a/DuckDB.NET.Test/DuckDBDataReaderEnumTests.cs b/DuckDB.NET.Test/DuckDBDataReaderEnumTests.cs index e44dc276..25c938fe 100644 --- a/DuckDB.NET.Test/DuckDBDataReaderEnumTests.cs +++ b/DuckDB.NET.Test/DuckDBDataReaderEnumTests.cs @@ -1,5 +1,4 @@ -using System; -using System.Collections.Generic; +using System.Collections.Generic; using FluentAssertions; using Xunit; @@ -23,8 +22,7 @@ public DuckDBDataReaderEnumTests(DuckDBDatabaseFixture db) : base(db) public void SelectEnumValues() { Command.CommandText = "Select * from person order by name desc"; - var reader = Command.ExecuteReader(); - + using var reader = Command.ExecuteReader(); reader.Read(); reader.GetFieldValue(1).Should().Be(Mood.Happy); @@ -42,10 +40,10 @@ public void SelectEnumValues() public void SelectEnumList() { Command.CommandText = "Select ['happy'::mood, 'ok'::mood]"; - var reader = Command.ExecuteReader(); - + using var reader = Command.ExecuteReader(); reader.Read(); var list = reader.GetFieldValue>(0); + list.Should().BeEquivalentTo(new List { Mood.Happy, Mood.Ok }); } @@ -53,8 +51,7 @@ public void SelectEnumList() public void SelectEnumValuesAsNullable() { Command.CommandText = "Select * from person order by name desc"; - var reader = Command.ExecuteReader(); - + using var reader = Command.ExecuteReader(); reader.Read(); reader.GetFieldValue(1).Should().Be(Mood.Happy); @@ -75,8 +72,7 @@ public void SelectEnumValuesAsNullable() public void SelectEnumValuesAsString() { Command.CommandText = "Select * from person order by name desc"; - var reader = Command.ExecuteReader(); - + using var reader = Command.ExecuteReader(); reader.Read(); reader.GetFieldValue(1).Should().BeEquivalentTo(Mood.Happy.ToString()); diff --git a/DuckDB.NET.Test/DuckDBDataReaderStructTests.cs b/DuckDB.NET.Test/DuckDBDataReaderStructTests.cs index 87ed04d7..fc191296 100644 --- a/DuckDB.NET.Test/DuckDBDataReaderStructTests.cs +++ b/DuckDB.NET.Test/DuckDBDataReaderStructTests.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using FluentAssertions; using Xunit; @@ -104,10 +105,9 @@ public void ReadStructWithNull() [Fact] public void ReadStructWithNestedStruct() { - Command.CommandText = - "SELECT {'birds': {'yes': 'duck', 'maybe': 'goose', 'huh': NULL, 'no': 'heron', 'type': 0}, " + - "'aliens': NULL, " + - "'amphibians': {'yes':'frog', 'maybe': 'salamander', 'huh': 'dragon', 'no':'toad', 'type':1} };"; + Command.CommandText = "SELECT {'birds': {'yes': 'duck', 'maybe': 'goose', 'huh': NULL, 'no': 'heron', 'type': 0}, " + + "'aliens': NULL, " + + "'amphibians': {'yes':'frog', 'maybe': 'salamander', 'huh': 'dragon', 'no':'toad', 'type':1} };"; using var reader = Command.ExecuteReader(); reader.Read(); @@ -134,6 +134,18 @@ public void ReadStructWithNestedStruct() }); } + [Theory] + [InlineData("SELECT {'b': null};", $"Property '{nameof(Struct4.A)}' not found in struct")] + [InlineData("SELECT {'b': null, 'a': 4};", $"Property '{nameof(Struct4.B)}' is not nullable but struct contains null")] + public void ReadStructWithMissingDataThrowsException(string query, string error) + { + Command.CommandText = query; + using var reader = Command.ExecuteReader(); + reader.Read(); + + reader.Invoking(r => r.GetFieldValue(0)).Should().Throw().WithMessage(error); + } + class Struct1 { public int X { get; set; } @@ -157,4 +169,10 @@ class Struct3 public Struct2 Aliens { get; set; } public Struct2 Amphibians { get; set; } } + + class Struct4 + { + public int A { get; set; } + public int B { get; set; } + } } \ No newline at end of file From 4b4be4711970bd8eaab56fbf78bfc45dfd5f9942 Mon Sep 17 00:00:00 2001 From: Giorgi Date: Thu, 12 Oct 2023 00:22:44 +0400 Subject: [PATCH 4/8] Dispose struct readers. --- DuckDB.NET.Data/VectorDataReader.cs | 116 +++++++++++++++------------- 1 file changed, 62 insertions(+), 54 deletions(-) diff --git a/DuckDB.NET.Data/VectorDataReader.cs b/DuckDB.NET.Data/VectorDataReader.cs index ed6919b4..be7712e6 100644 --- a/DuckDB.NET.Data/VectorDataReader.cs +++ b/DuckDB.NET.Data/VectorDataReader.cs @@ -221,6 +221,65 @@ internal object GetEnum(ulong offset, Type returnType) return enumItem; } + internal object GetStruct(ulong offset, Type returnType) + { + if (structDataReaders is null) + { + throw new InvalidOperationException("Can't get a struct from a non-struct vector."); + } + + var result = Activator.CreateInstance(returnType); + + if (result is Dictionary dictionary) + { + //var dictionary = result as Dictionary; + foreach (var reader in structDataReaders) + { + var value = reader.Value.IsValid(offset) ? reader.Value.GetValue(offset) : null; + dictionary.Add(reader.Key, value); + } + + return result; + } + + var properties = returnType.GetProperties(); + foreach (var propertyInfo in properties) + { + if (propertyInfo.SetMethod == null) + { + continue; + } + + structDataReaders.TryGetValue(propertyInfo.Name, out var reader); + var isNotNullable = propertyInfo.PropertyType.IsValueType && Nullable.GetUnderlyingType(propertyInfo.PropertyType) == null; + + if (reader == null) + { + if (isNotNullable) + { + throw new NullReferenceException($"Property '{propertyInfo.Name}' not found in struct"); + } + + continue; + } + + if (reader.IsValid(offset)) + { + var value = reader.GetValue(offset, propertyInfo.PropertyType); + propertyInfo.SetValue(result, value); + } + else + { + if (isNotNullable) + { + throw new NullReferenceException($"Property '{propertyInfo.Name}' is not nullable but struct contains null"); + } + } + } + + return result!; + } + internal unsafe object GetList(ulong offset, Type returnType) { if (listDataReader is null) @@ -345,64 +404,13 @@ public void Dispose() { listDataReader?.Dispose(); logicalType?.Dispose(); - } - - internal object GetStruct(ulong offset, Type type) - { - if (structDataReaders is null) - { - throw new InvalidOperationException("Can't get a struct from a non-struct vector."); - } - var result = Activator.CreateInstance(type); - - if (result is Dictionary dictionary) + if (structDataReaders != null) { - //var dictionary = result as Dictionary; - foreach (var reader in structDataReaders) + foreach (var item in structDataReaders) { - var value = reader.Value.IsValid(offset) ? reader.Value.GetValue(offset) : null; - dictionary.Add(reader.Key, value); + item.Value.Dispose(); } - - return result; } - - var properties = type.GetProperties(); - foreach (var propertyInfo in properties) - { - if (propertyInfo.SetMethod == null) - { - continue; - } - - structDataReaders.TryGetValue(propertyInfo.Name, out var reader); - var isNotNullable = propertyInfo.PropertyType.IsValueType && Nullable.GetUnderlyingType(propertyInfo.PropertyType) == null; - - if (reader == null) - { - if (isNotNullable) - { - throw new NullReferenceException($"Property '{propertyInfo.Name}' not found in struct"); - } - - continue; - } - - if (reader.IsValid(offset)) - { - var value = reader.GetValue(offset, propertyInfo.PropertyType); - propertyInfo.SetValue(result, value); - } - else - { - if (isNotNullable) - { - throw new NullReferenceException($"Property '{propertyInfo.Name}' is not nullable but struct contains null"); - } - } - } - - return result!; } } \ No newline at end of file From 12c66f90230479a1d718e6d74ade61517c0ccebd Mon Sep 17 00:00:00 2001 From: Giorgi Date: Thu, 12 Oct 2023 00:25:23 +0400 Subject: [PATCH 5/8] Add one more test case --- DuckDB.NET.Test/DuckDBDataReaderStructTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DuckDB.NET.Test/DuckDBDataReaderStructTests.cs b/DuckDB.NET.Test/DuckDBDataReaderStructTests.cs index fc191296..27487647 100644 --- a/DuckDB.NET.Test/DuckDBDataReaderStructTests.cs +++ b/DuckDB.NET.Test/DuckDBDataReaderStructTests.cs @@ -30,12 +30,12 @@ public void ReadBasicStruct() [Fact] public void ReadBasicStructWithGetValue() { - Command.CommandText = "SELECT {'x': 1, 'y': 2, 'z': 'test'};"; + Command.CommandText = "SELECT {'x': 1, 'y': 2, 'z': 'test', 'xy': null};"; using var reader = Command.ExecuteReader(); reader.Read(); var value = reader.GetValue(0); - value.Should().BeEquivalentTo(new Dictionary { { "x", 1 }, { "y", 2 }, { "z", "test" } }); + value.Should().BeEquivalentTo(new Dictionary { { "x", 1 }, { "y", 2 }, { "z", "test" }, {"xy", null} }); } [Fact] From e01bad426f0e76823cbfbf8046220766e15924b7 Mon Sep 17 00:00:00 2001 From: Giorgi Date: Thu, 26 Oct 2023 22:55:37 +0400 Subject: [PATCH 6/8] Use expression trees to set properties when reading Struct --- DuckDB.NET.Data/Internal/IsExternalInit.cs | 4 ++ DuckDB.NET.Data/Internal/TypeDetails.cs | 11 ++++ DuckDB.NET.Data/VectorDataReader.cs | 53 ++++++++++++++----- .../DuckDBDataReaderStructTests.cs | 2 +- 4 files changed, 56 insertions(+), 14 deletions(-) create mode 100644 DuckDB.NET.Data/Internal/IsExternalInit.cs create mode 100644 DuckDB.NET.Data/Internal/TypeDetails.cs diff --git a/DuckDB.NET.Data/Internal/IsExternalInit.cs b/DuckDB.NET.Data/Internal/IsExternalInit.cs new file mode 100644 index 00000000..ef904e6c --- /dev/null +++ b/DuckDB.NET.Data/Internal/IsExternalInit.cs @@ -0,0 +1,4 @@ +namespace System.Runtime.CompilerServices +{ + internal static class IsExternalInit { } +} \ No newline at end of file diff --git a/DuckDB.NET.Data/Internal/TypeDetails.cs b/DuckDB.NET.Data/Internal/TypeDetails.cs new file mode 100644 index 00000000..95c8039f --- /dev/null +++ b/DuckDB.NET.Data/Internal/TypeDetails.cs @@ -0,0 +1,11 @@ +using System; +using System.Collections.Generic; + +namespace DuckDB.NET.Data.Internal; + +class TypeDetails +{ + public Dictionary Properties { get; set; } = new(); +} + +record PropertyDetails(Type PropertyType, bool Nullable, Action Setter); diff --git a/DuckDB.NET.Data/VectorDataReader.cs b/DuckDB.NET.Data/VectorDataReader.cs index 193a8a7e..c94e9b2e 100644 --- a/DuckDB.NET.Data/VectorDataReader.cs +++ b/DuckDB.NET.Data/VectorDataReader.cs @@ -1,8 +1,11 @@ -using System; +using DuckDB.NET.Data.Internal; +using System; using System.Collections; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Globalization; using System.IO; +using System.Linq.Expressions; using System.Numerics; using System.Text; @@ -27,6 +30,8 @@ internal class VectorDataReader : IDisposable private readonly DuckDBType decimalType; private readonly DuckDBType enumType; + private static readonly ConcurrentDictionary TypeCache = new(); + internal unsafe VectorDataReader(IntPtr vector, void* dataPointer, ulong* validityMaskPointer, DuckDBType columnType) { this.vector = vector; @@ -265,22 +270,44 @@ internal object GetStruct(ulong offset, Type returnType) return result; } - var properties = returnType.GetProperties(); - foreach (var propertyInfo in properties) + var typeDetails = TypeCache.GetOrAdd(returnType, type => { - if (propertyInfo.SetMethod == null) + var propertyInfos = returnType.GetProperties(); + var details = new TypeDetails(); + + foreach (var propertyInfo in propertyInfos) { - continue; + if (propertyInfo.SetMethod == null) + { + continue; + } + + var isNullable = !propertyInfo.PropertyType.IsValueType || Nullable.GetUnderlyingType(propertyInfo.PropertyType) != null; + + var instanceParam = Expression.Parameter(typeof(object)); + var argumentParam = Expression.Parameter(typeof(object)); + + var setAction = Expression.Lambda>( + Expression.Call(Expression.Convert(instanceParam, type), propertyInfo.SetMethod, Expression.Convert(argumentParam, propertyInfo.PropertyType)), + instanceParam, argumentParam + ).Compile(); + + details.Properties.Add(propertyInfo.Name, new PropertyDetails(propertyInfo.PropertyType, isNullable, setAction)); } - structDataReaders.TryGetValue(propertyInfo.Name, out var reader); - var isNotNullable = propertyInfo.PropertyType.IsValueType && Nullable.GetUnderlyingType(propertyInfo.PropertyType) == null; + return details; + }); + + foreach (var properties in typeDetails.Properties) + { + structDataReaders.TryGetValue(properties.Key, out var reader); + var isNullable = properties.Value.Nullable; if (reader == null) { - if (isNotNullable) + if (!isNullable) { - throw new NullReferenceException($"Property '{propertyInfo.Name}' not found in struct"); + throw new NullReferenceException($"Property '{properties.Key}' not found in struct"); } continue; @@ -288,14 +315,14 @@ internal object GetStruct(ulong offset, Type returnType) if (reader.IsValid(offset)) { - var value = reader.GetValue(offset, propertyInfo.PropertyType); - propertyInfo.SetValue(result, value); + var value = reader.GetValue(offset, properties.Value.PropertyType); + properties.Value.Setter(result!, value); } else { - if (isNotNullable) + if (!isNullable) { - throw new NullReferenceException($"Property '{propertyInfo.Name}' is not nullable but struct contains null"); + throw new NullReferenceException($"Property '{properties.Key}' is not nullable but struct contains null value"); } } } diff --git a/DuckDB.NET.Test/DuckDBDataReaderStructTests.cs b/DuckDB.NET.Test/DuckDBDataReaderStructTests.cs index 27487647..0d197b92 100644 --- a/DuckDB.NET.Test/DuckDBDataReaderStructTests.cs +++ b/DuckDB.NET.Test/DuckDBDataReaderStructTests.cs @@ -136,7 +136,7 @@ public void ReadStructWithNestedStruct() [Theory] [InlineData("SELECT {'b': null};", $"Property '{nameof(Struct4.A)}' not found in struct")] - [InlineData("SELECT {'b': null, 'a': 4};", $"Property '{nameof(Struct4.B)}' is not nullable but struct contains null")] + [InlineData("SELECT {'b': null, 'a': 4};", $"Property '{nameof(Struct4.B)}' is not nullable but struct contains null value")] public void ReadStructWithMissingDataThrowsException(string query, string error) { Command.CommandText = query; From 80706ba6e5ee0f9514a35b778735c0bf5c490608 Mon Sep 17 00:00:00 2001 From: Giorgi Date: Thu, 26 Oct 2023 23:25:51 +0400 Subject: [PATCH 7/8] Fix merge --- DuckDB.NET.Data/VectorDataReader.cs | 44 ----------------------------- 1 file changed, 44 deletions(-) diff --git a/DuckDB.NET.Data/VectorDataReader.cs b/DuckDB.NET.Data/VectorDataReader.cs index c94e9b2e..121f238b 100644 --- a/DuckDB.NET.Data/VectorDataReader.cs +++ b/DuckDB.NET.Data/VectorDataReader.cs @@ -93,50 +93,6 @@ internal unsafe VectorDataReader(IntPtr vector, void* dataPointer, ulong* validi DuckDBType.Struct => typeof(Dictionary), var type => throw new ArgumentException($"Unrecognised type {type} ({(int)type})") }; - - switch (DuckDBType) - { - case DuckDBType.Enum: - enumType = NativeMethods.LogicalType.DuckDBEnumInternalType(logicalType); - break; - case DuckDBType.Decimal: - scale = NativeMethods.LogicalType.DuckDBDecimalScale(logicalType); - decimalType = NativeMethods.LogicalType.DuckDBDecimalInternalType(logicalType); - break; - case DuckDBType.List: - { - using var childType = NativeMethods.LogicalType.DuckDBListTypeChildType(logicalType); - var type = NativeMethods.LogicalType.DuckDBGetTypeId(childType); - - var childVector = NativeMethods.DataChunks.DuckDBListVectorGetChild(vector); - - var childVectorData = NativeMethods.DataChunks.DuckDBVectorGetData(childVector); - var childVectorValidity = NativeMethods.DataChunks.DuckDBVectorGetValidity(childVector); - - listDataReader = new VectorDataReader(childVector, childVectorData, childVectorValidity, type); - break; - } - case DuckDBType.Struct: - { - var memberCount = NativeMethods.LogicalType.DuckDBStructTypeChildCount(logicalType); - structDataReaders = new Dictionary(StringComparer.OrdinalIgnoreCase); - - for (int index = 0; index < memberCount; index++) - { - var name = NativeMethods.LogicalType.DuckDBStructTypeChildName(logicalType, index).ToManagedString(); - var childVector = NativeMethods.DataChunks.DuckDBStructVectorGetChild(vector, index); - - var childVectorData = NativeMethods.DataChunks.DuckDBVectorGetData(childVector); - var childVectorValidity = NativeMethods.DataChunks.DuckDBVectorGetValidity(childVector); - - using var childType = NativeMethods.LogicalType.DuckDBStructTypeChildType(logicalType, index); - var type = NativeMethods.LogicalType.DuckDBGetTypeId(childType); - - structDataReaders[name] = new VectorDataReader(childVector, childVectorData, childVectorValidity, type); - } - break; - } - } } internal unsafe bool IsValid(ulong offset) From 33654a32272d31149df18172fd96053b3c9f8dab Mon Sep 17 00:00:00 2001 From: Giorgi Date: Thu, 26 Oct 2023 23:29:06 +0400 Subject: [PATCH 8/8] One more fix --- DuckDB.NET.Data/VectorDataReader.cs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/DuckDB.NET.Data/VectorDataReader.cs b/DuckDB.NET.Data/VectorDataReader.cs index 121f238b..09de5cf3 100644 --- a/DuckDB.NET.Data/VectorDataReader.cs +++ b/DuckDB.NET.Data/VectorDataReader.cs @@ -64,6 +64,26 @@ internal unsafe VectorDataReader(IntPtr vector, void* dataPointer, ulong* validi listDataReader = new VectorDataReader(childVector, childVectorData, childVectorValidity, type); break; } + case DuckDBType.Struct: + { + var memberCount = NativeMethods.LogicalType.DuckDBStructTypeChildCount(logicalType); + structDataReaders = new Dictionary(StringComparer.OrdinalIgnoreCase); + + for (int index = 0; index < memberCount; index++) + { + var name = NativeMethods.LogicalType.DuckDBStructTypeChildName(logicalType, index).ToManagedString(); + var childVector = NativeMethods.DataChunks.DuckDBStructVectorGetChild(vector, index); + + var childVectorData = NativeMethods.DataChunks.DuckDBVectorGetData(childVector); + var childVectorValidity = NativeMethods.DataChunks.DuckDBVectorGetValidity(childVector); + + using var childType = NativeMethods.LogicalType.DuckDBStructTypeChildType(logicalType, index); + var type = NativeMethods.LogicalType.DuckDBGetTypeId(childType); + + structDataReaders[name] = new VectorDataReader(childVector, childVectorData, childVectorValidity, type); + } + break; + } } ClrType = DuckDBType switch