From 4e208de477a7110578926285eeda9368162ace09 Mon Sep 17 00:00:00 2001 From: David Grossman Date: Fri, 3 Nov 2023 12:24:15 -0700 Subject: [PATCH 1/6] fix complex defaults --- .../compatibility/avro19/FieldBuilder19.java | 24 ++++++++++++++++-- .../avro19/Avro19FieldBuilderTest.java | 11 ++++++++ .../FieldWithArrayOfEnumDefaultValue.avsc | 25 +++++++++++++++++++ 3 files changed, 58 insertions(+), 2 deletions(-) create mode 100644 helper/tests/helper-tests-19/src/test/resources/FieldWithArrayOfEnumDefaultValue.avsc diff --git a/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/FieldBuilder19.java b/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/FieldBuilder19.java index ea5f6548f..8f8d0d6b9 100644 --- a/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/FieldBuilder19.java +++ b/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/FieldBuilder19.java @@ -10,12 +10,13 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.linkedin.avroutil1.compatibility.AvroSchemaUtil; import com.linkedin.avroutil1.compatibility.FieldBuilder; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; +import java.util.Map; import org.apache.avro.JsonProperties; import org.apache.avro.Schema; import org.apache.avro.Schema.Field.Order; - -import java.util.Map; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericFixed; import org.apache.avro.generic.IndexedRecord; @@ -170,6 +171,25 @@ public FieldBuilder removeProp(String propName) { * @return a representation of the input that avro likes for use as a field default value */ private static Object avroFriendlyDefaultValue(Object mightNotBeFriendly) throws Exception { + // handle default values that are lists + if (mightNotBeFriendly instanceof List) { + List list = (List) mightNotBeFriendly; + List result = new ArrayList<>(list.size()); + for (Object element : list) { + result.add(avroFriendlyDefaultValue(element)); + } + return result; + } + + // handle default values that are maps + if (mightNotBeFriendly instanceof Map) { + Map map = (Map) mightNotBeFriendly; + Map result = new HashMap<>(map.size()); + for (Map.Entry entry : map.entrySet()) { + result.put(entry.getKey(), avroFriendlyDefaultValue(entry.getValue())); + } + return result; + } //generic enums we turn to strings if (mightNotBeFriendly instanceof GenericData.EnumSymbol) { diff --git a/helper/tests/helper-tests-19/src/test/java/com/linkedin/avroutil1/compatibility/avro19/Avro19FieldBuilderTest.java b/helper/tests/helper-tests-19/src/test/java/com/linkedin/avroutil1/compatibility/avro19/Avro19FieldBuilderTest.java index b53524a83..b20e529a1 100644 --- a/helper/tests/helper-tests-19/src/test/java/com/linkedin/avroutil1/compatibility/avro19/Avro19FieldBuilderTest.java +++ b/helper/tests/helper-tests-19/src/test/java/com/linkedin/avroutil1/compatibility/avro19/Avro19FieldBuilderTest.java @@ -10,6 +10,7 @@ import com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper; import com.linkedin.avroutil1.compatibility.FieldBuilder; import com.linkedin.avroutil1.testcommon.TestUtil; +import java.io.IOException; import java.util.HashMap; import java.util.Map; import org.apache.avro.AvroRuntimeException; @@ -159,6 +160,16 @@ public void testNullDefaultForBoolField() throws Exception { } } + @Test + public void testArrayOfEnumDefaultValue() throws IOException { + Schema schema = Schema.parse(TestUtil.load("FieldWithArrayOfEnumDefaultValue.avsc")); + Schema.Field field = schema.getField("arrayOfEnum"); + Object defaultValue = AvroCompatibilityHelper.getGenericDefaultValue(field); + FieldBuilder builder = AvroCompatibilityHelper.newField(field); + builder.setDefault(defaultValue); + Schema.Field resField = builder.build(); + Assert.assertNotNull(resField.defaultVal()); + } @Test public void testAddPropsFields() { diff --git a/helper/tests/helper-tests-19/src/test/resources/FieldWithArrayOfEnumDefaultValue.avsc b/helper/tests/helper-tests-19/src/test/resources/FieldWithArrayOfEnumDefaultValue.avsc new file mode 100644 index 000000000..b0fbd2f16 --- /dev/null +++ b/helper/tests/helper-tests-19/src/test/resources/FieldWithArrayOfEnumDefaultValue.avsc @@ -0,0 +1,25 @@ +{ + "type": "record", + "name": "FieldWithArrayOfEnumDefaultValue", + "fields": [ + { + "name": "arrayOfEnum", + "type": { + "type": "array", + "items": { + "type": "enum", + "name": "EnumDefaultValue", + "symbols": [ + "A", + "B", + "C" + ] + } + }, + "default": [ + "A", + "B" + ] + } + ] +} \ No newline at end of file From 093b66404e0dcf6dfbba3b772657fba411af85d5 Mon Sep 17 00:00:00 2001 From: David Grossman Date: Fri, 3 Nov 2023 12:49:17 -0700 Subject: [PATCH 2/6] update 1.10 and 1.11 --- .../avro110/FieldBuilder110.java | 21 +++++++++++++++++ .../avro111/FieldBuilder111.java | 21 +++++++++++++++++ .../avro110/Avro110FieldBuilderTest.java | 12 ++++++++++ .../avro111/Avro111FieldBuilderTest.java | 23 +++++++++++++++++++ 4 files changed, 77 insertions(+) create mode 100644 helper/tests/helper-tests-111/src/test/java/com/linkedin/avroutil1/compatibility/avro111/Avro111FieldBuilderTest.java diff --git a/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/FieldBuilder110.java b/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/FieldBuilder110.java index 72752b32f..92b4ad1fe 100644 --- a/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/FieldBuilder110.java +++ b/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/FieldBuilder110.java @@ -10,7 +10,9 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.linkedin.avroutil1.compatibility.AvroSchemaUtil; import com.linkedin.avroutil1.compatibility.FieldBuilder; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import org.apache.avro.JsonProperties; import org.apache.avro.Schema; @@ -169,6 +171,25 @@ public Schema.Field build() { * @return a representation of the input that avro likes for use as a field default value */ private static Object avroFriendlyDefaultValue(Object mightNotBeFriendly) throws Exception { + // handle default values that are lists + if (mightNotBeFriendly instanceof List) { + List list = (List) mightNotBeFriendly; + List result = new ArrayList<>(list.size()); + for (Object element : list) { + result.add(avroFriendlyDefaultValue(element)); + } + return result; + } + + // handle default values that are maps + if (mightNotBeFriendly instanceof Map) { + Map map = (Map) mightNotBeFriendly; + Map result = new HashMap<>(map.size()); + for (Map.Entry entry : map.entrySet()) { + result.put(entry.getKey(), avroFriendlyDefaultValue(entry.getValue())); + } + return result; + } //generic enums we turn to strings if (mightNotBeFriendly instanceof GenericData.EnumSymbol) { diff --git a/helper/impls/helper-impl-111/src/main/java/com/linkedin/avroutil1/compatibility/avro111/FieldBuilder111.java b/helper/impls/helper-impl-111/src/main/java/com/linkedin/avroutil1/compatibility/avro111/FieldBuilder111.java index a5519632a..cb47fdc91 100644 --- a/helper/impls/helper-impl-111/src/main/java/com/linkedin/avroutil1/compatibility/avro111/FieldBuilder111.java +++ b/helper/impls/helper-impl-111/src/main/java/com/linkedin/avroutil1/compatibility/avro111/FieldBuilder111.java @@ -10,7 +10,9 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.linkedin.avroutil1.compatibility.AvroSchemaUtil; import com.linkedin.avroutil1.compatibility.FieldBuilder; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import org.apache.avro.JsonProperties; import org.apache.avro.Schema; @@ -168,6 +170,25 @@ public FieldBuilder removeProp(String propName) { * @return a representation of the input that avro likes for use as a field default value */ private static Object avroFriendlyDefaultValue(Object mightNotBeFriendly) throws Exception { + // handle default values that are lists + if (mightNotBeFriendly instanceof List) { + List list = (List) mightNotBeFriendly; + List result = new ArrayList<>(list.size()); + for (Object element : list) { + result.add(avroFriendlyDefaultValue(element)); + } + return result; + } + + // handle default values that are maps + if (mightNotBeFriendly instanceof Map) { + Map map = (Map) mightNotBeFriendly; + Map result = new HashMap<>(map.size()); + for (Map.Entry entry : map.entrySet()) { + result.put(entry.getKey(), avroFriendlyDefaultValue(entry.getValue())); + } + return result; + } //generic enums we turn to strings if (mightNotBeFriendly instanceof GenericData.EnumSymbol) { diff --git a/helper/tests/helper-tests-110/src/test/java/com/linkedin/avroutil1/compatibility/avro110/Avro110FieldBuilderTest.java b/helper/tests/helper-tests-110/src/test/java/com/linkedin/avroutil1/compatibility/avro110/Avro110FieldBuilderTest.java index 4966f84ca..4cafe30e4 100644 --- a/helper/tests/helper-tests-110/src/test/java/com/linkedin/avroutil1/compatibility/avro110/Avro110FieldBuilderTest.java +++ b/helper/tests/helper-tests-110/src/test/java/com/linkedin/avroutil1/compatibility/avro110/Avro110FieldBuilderTest.java @@ -10,6 +10,7 @@ import com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper; import com.linkedin.avroutil1.compatibility.FieldBuilder; import com.linkedin.avroutil1.testcommon.TestUtil; +import java.io.IOException; import java.util.HashMap; import java.util.Map; import org.apache.avro.AvroRuntimeException; @@ -159,6 +160,17 @@ public void testNullDefaultForBoolField() throws Exception { } } + @Test + public void testArrayOfEnumDefaultValue() throws IOException { + Schema schema = Schema.parse(TestUtil.load("FieldWithArrayOfEnumDefaultValue.avsc")); + Schema.Field field = schema.getField("arrayOfEnum"); + Object defaultValue = AvroCompatibilityHelper.getGenericDefaultValue(field); + FieldBuilder builder = AvroCompatibilityHelper.newField(field); + builder.setDefault(defaultValue); + Schema.Field resField = builder.build(); + Assert.assertNotNull(resField.defaultVal()); + } + @Test public void testAddPropsFields() { // default (no order specified). diff --git a/helper/tests/helper-tests-111/src/test/java/com/linkedin/avroutil1/compatibility/avro111/Avro111FieldBuilderTest.java b/helper/tests/helper-tests-111/src/test/java/com/linkedin/avroutil1/compatibility/avro111/Avro111FieldBuilderTest.java new file mode 100644 index 000000000..ab8d7cf0b --- /dev/null +++ b/helper/tests/helper-tests-111/src/test/java/com/linkedin/avroutil1/compatibility/avro111/Avro111FieldBuilderTest.java @@ -0,0 +1,23 @@ +package com.linkedin.avroutil1.compatibility.avro111; + +import com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper; +import com.linkedin.avroutil1.compatibility.FieldBuilder; +import com.linkedin.avroutil1.testcommon.TestUtil; +import java.io.IOException; +import org.apache.avro.Schema; +import org.testng.Assert; +import org.testng.annotations.Test; + + +public class Avro111FieldBuilderTest { + @Test + public void testArrayOfEnumDefaultValue() throws IOException { + Schema schema = Schema.parse(TestUtil.load("FieldWithArrayOfEnumDefaultValue.avsc")); + Schema.Field field = schema.getField("arrayOfEnum"); + Object defaultValue = AvroCompatibilityHelper.getGenericDefaultValue(field); + FieldBuilder builder = AvroCompatibilityHelper.newField(field); + builder.setDefault(defaultValue); + Schema.Field resField = builder.build(); + Assert.assertNotNull(resField.defaultVal()); + } +} From b43139bce7a5e07aaa1cf608eee3a7745418de46 Mon Sep 17 00:00:00 2001 From: David Grossman Date: Fri, 3 Nov 2023 13:00:45 -0700 Subject: [PATCH 3/6] rm handling of maps --- .../compatibility/avro110/FieldBuilder110.java | 10 ---------- .../compatibility/avro111/FieldBuilder111.java | 10 ---------- .../avroutil1/compatibility/avro19/FieldBuilder19.java | 10 ---------- 3 files changed, 30 deletions(-) diff --git a/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/FieldBuilder110.java b/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/FieldBuilder110.java index 92b4ad1fe..fe4fc5c23 100644 --- a/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/FieldBuilder110.java +++ b/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/FieldBuilder110.java @@ -181,16 +181,6 @@ private static Object avroFriendlyDefaultValue(Object mightNotBeFriendly) throws return result; } - // handle default values that are maps - if (mightNotBeFriendly instanceof Map) { - Map map = (Map) mightNotBeFriendly; - Map result = new HashMap<>(map.size()); - for (Map.Entry entry : map.entrySet()) { - result.put(entry.getKey(), avroFriendlyDefaultValue(entry.getValue())); - } - return result; - } - //generic enums we turn to strings if (mightNotBeFriendly instanceof GenericData.EnumSymbol) { return mightNotBeFriendly.toString(); // == symbol string diff --git a/helper/impls/helper-impl-111/src/main/java/com/linkedin/avroutil1/compatibility/avro111/FieldBuilder111.java b/helper/impls/helper-impl-111/src/main/java/com/linkedin/avroutil1/compatibility/avro111/FieldBuilder111.java index cb47fdc91..432f2d54d 100644 --- a/helper/impls/helper-impl-111/src/main/java/com/linkedin/avroutil1/compatibility/avro111/FieldBuilder111.java +++ b/helper/impls/helper-impl-111/src/main/java/com/linkedin/avroutil1/compatibility/avro111/FieldBuilder111.java @@ -180,16 +180,6 @@ private static Object avroFriendlyDefaultValue(Object mightNotBeFriendly) throws return result; } - // handle default values that are maps - if (mightNotBeFriendly instanceof Map) { - Map map = (Map) mightNotBeFriendly; - Map result = new HashMap<>(map.size()); - for (Map.Entry entry : map.entrySet()) { - result.put(entry.getKey(), avroFriendlyDefaultValue(entry.getValue())); - } - return result; - } - //generic enums we turn to strings if (mightNotBeFriendly instanceof GenericData.EnumSymbol) { return mightNotBeFriendly.toString(); // == symbol string diff --git a/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/FieldBuilder19.java b/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/FieldBuilder19.java index 8f8d0d6b9..1bbfc68a7 100644 --- a/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/FieldBuilder19.java +++ b/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/FieldBuilder19.java @@ -181,16 +181,6 @@ private static Object avroFriendlyDefaultValue(Object mightNotBeFriendly) throws return result; } - // handle default values that are maps - if (mightNotBeFriendly instanceof Map) { - Map map = (Map) mightNotBeFriendly; - Map result = new HashMap<>(map.size()); - for (Map.Entry entry : map.entrySet()) { - result.put(entry.getKey(), avroFriendlyDefaultValue(entry.getValue())); - } - return result; - } - //generic enums we turn to strings if (mightNotBeFriendly instanceof GenericData.EnumSymbol) { return mightNotBeFriendly.toString(); // == symbol string From ecceda612f862ad9ded5c67f6721a14ac7f1ab98 Mon Sep 17 00:00:00 2001 From: David Grossman Date: Fri, 3 Nov 2023 13:29:55 -0700 Subject: [PATCH 4/6] mv --- .../src/main}/resources/FieldWithArrayOfEnumDefaultValue.avsc | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename helper/tests/{helper-tests-19/src/test => helper-tests-common/src/main}/resources/FieldWithArrayOfEnumDefaultValue.avsc (100%) diff --git a/helper/tests/helper-tests-19/src/test/resources/FieldWithArrayOfEnumDefaultValue.avsc b/helper/tests/helper-tests-common/src/main/resources/FieldWithArrayOfEnumDefaultValue.avsc similarity index 100% rename from helper/tests/helper-tests-19/src/test/resources/FieldWithArrayOfEnumDefaultValue.avsc rename to helper/tests/helper-tests-common/src/main/resources/FieldWithArrayOfEnumDefaultValue.avsc From ee748f5229e6bff30c5f5c04e3eb0b8251e1dc59 Mon Sep 17 00:00:00 2001 From: David Grossman Date: Fri, 3 Nov 2023 13:40:27 -0700 Subject: [PATCH 5/6] copyright --- .../compatibility/avro111/Avro111FieldBuilderTest.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/helper/tests/helper-tests-111/src/test/java/com/linkedin/avroutil1/compatibility/avro111/Avro111FieldBuilderTest.java b/helper/tests/helper-tests-111/src/test/java/com/linkedin/avroutil1/compatibility/avro111/Avro111FieldBuilderTest.java index ab8d7cf0b..78ee9f251 100644 --- a/helper/tests/helper-tests-111/src/test/java/com/linkedin/avroutil1/compatibility/avro111/Avro111FieldBuilderTest.java +++ b/helper/tests/helper-tests-111/src/test/java/com/linkedin/avroutil1/compatibility/avro111/Avro111FieldBuilderTest.java @@ -1,3 +1,9 @@ +/* + * Copyright 2023 LinkedIn Corp. + * Licensed under the BSD 2-Clause License (the "License"). + * See License in the project root for license information. + */ + package com.linkedin.avroutil1.compatibility.avro111; import com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper; From 0e048dec0f1feb6784db3ff18075d5931950bafc Mon Sep 17 00:00:00 2001 From: David Grossman Date: Sat, 4 Nov 2023 21:22:00 -0700 Subject: [PATCH 6/6] add comment --- .../compatibility/avro110/Avro110FieldBuilderTest.java | 2 ++ .../compatibility/avro111/Avro111FieldBuilderTest.java | 4 +++- .../compatibility/avro19/Avro19FieldBuilderTest.java | 2 ++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/helper/tests/helper-tests-110/src/test/java/com/linkedin/avroutil1/compatibility/avro110/Avro110FieldBuilderTest.java b/helper/tests/helper-tests-110/src/test/java/com/linkedin/avroutil1/compatibility/avro110/Avro110FieldBuilderTest.java index 4cafe30e4..abe1fe409 100644 --- a/helper/tests/helper-tests-110/src/test/java/com/linkedin/avroutil1/compatibility/avro110/Avro110FieldBuilderTest.java +++ b/helper/tests/helper-tests-110/src/test/java/com/linkedin/avroutil1/compatibility/avro110/Avro110FieldBuilderTest.java @@ -167,6 +167,8 @@ public void testArrayOfEnumDefaultValue() throws IOException { Object defaultValue = AvroCompatibilityHelper.getGenericDefaultValue(field); FieldBuilder builder = AvroCompatibilityHelper.newField(field); builder.setDefault(defaultValue); + + // Test that .build() should not throw an exception. Schema.Field resField = builder.build(); Assert.assertNotNull(resField.defaultVal()); } diff --git a/helper/tests/helper-tests-111/src/test/java/com/linkedin/avroutil1/compatibility/avro111/Avro111FieldBuilderTest.java b/helper/tests/helper-tests-111/src/test/java/com/linkedin/avroutil1/compatibility/avro111/Avro111FieldBuilderTest.java index 78ee9f251..ca199889b 100644 --- a/helper/tests/helper-tests-111/src/test/java/com/linkedin/avroutil1/compatibility/avro111/Avro111FieldBuilderTest.java +++ b/helper/tests/helper-tests-111/src/test/java/com/linkedin/avroutil1/compatibility/avro111/Avro111FieldBuilderTest.java @@ -23,7 +23,9 @@ public void testArrayOfEnumDefaultValue() throws IOException { Object defaultValue = AvroCompatibilityHelper.getGenericDefaultValue(field); FieldBuilder builder = AvroCompatibilityHelper.newField(field); builder.setDefault(defaultValue); + + // Test that .build() should not throw an exception. Schema.Field resField = builder.build(); - Assert.assertNotNull(resField.defaultVal()); + Assert.assertNotNull(resField); } } diff --git a/helper/tests/helper-tests-19/src/test/java/com/linkedin/avroutil1/compatibility/avro19/Avro19FieldBuilderTest.java b/helper/tests/helper-tests-19/src/test/java/com/linkedin/avroutil1/compatibility/avro19/Avro19FieldBuilderTest.java index b20e529a1..f79ce05ef 100644 --- a/helper/tests/helper-tests-19/src/test/java/com/linkedin/avroutil1/compatibility/avro19/Avro19FieldBuilderTest.java +++ b/helper/tests/helper-tests-19/src/test/java/com/linkedin/avroutil1/compatibility/avro19/Avro19FieldBuilderTest.java @@ -167,6 +167,8 @@ public void testArrayOfEnumDefaultValue() throws IOException { Object defaultValue = AvroCompatibilityHelper.getGenericDefaultValue(field); FieldBuilder builder = AvroCompatibilityHelper.newField(field); builder.setDefault(defaultValue); + + // Test that .build() should not throw an exception. Schema.Field resField = builder.build(); Assert.assertNotNull(resField.defaultVal()); }