From 511f48f0f8b4480945932a7cc0346de924612575 Mon Sep 17 00:00:00 2001 From: Uttam Kumar Date: Wed, 6 Dec 2023 13:45:36 -0800 Subject: [PATCH 01/13] Updated unit tests --- .../src/main/avro/vs110/TestCollections.avsc | 7 +++++ .../src/main/avro/vs111/TestCollections.avsc | 7 +++++ .../src/main/avro/vs14/TestCollections.avsc | 7 +++++ .../src/main/avro/vs15/TestCollections.avsc | 7 +++++ .../src/main/avro/vs16/TestCollections.avsc | 7 +++++ .../src/main/avro/vs17/TestCollections.avsc | 7 +++++ .../src/main/avro/vs18/TestCollections.avsc | 7 +++++ .../src/main/avro/vs19/TestCollections.avsc | 7 +++++ .../avro/charseqmethod/TestCollections.avsc | 7 +++++ .../avro/charseqmethod/TestCollections.avsc | 7 +++++ .../avroutil1/builder/SpecificRecordTest.java | 29 ++++++++++++++++++- 11 files changed, 98 insertions(+), 1 deletion(-) diff --git a/avro-builder/tests/codegen-110/src/main/avro/vs110/TestCollections.avsc b/avro-builder/tests/codegen-110/src/main/avro/vs110/TestCollections.avsc index a23f1b58a..c3a91c1f8 100644 --- a/avro-builder/tests/codegen-110/src/main/avro/vs110/TestCollections.avsc +++ b/avro-builder/tests/codegen-110/src/main/avro/vs110/TestCollections.avsc @@ -6,6 +6,13 @@ "name": "str", "type": "string" }, + { + "name": "intAr", + "type": { + "type": "array", + "items": "int" + } + }, { "name": "strAr", "type": { diff --git a/avro-builder/tests/codegen-111/src/main/avro/vs111/TestCollections.avsc b/avro-builder/tests/codegen-111/src/main/avro/vs111/TestCollections.avsc index 4e522d370..aa3942c55 100644 --- a/avro-builder/tests/codegen-111/src/main/avro/vs111/TestCollections.avsc +++ b/avro-builder/tests/codegen-111/src/main/avro/vs111/TestCollections.avsc @@ -6,6 +6,13 @@ "name": "str", "type": "string" }, + { + "name": "intAr", + "type": { + "type": "array", + "items": "int" + } + }, { "name": "strAr", "type": { diff --git a/avro-builder/tests/codegen-14/src/main/avro/vs14/TestCollections.avsc b/avro-builder/tests/codegen-14/src/main/avro/vs14/TestCollections.avsc index 304a52933..df6c6fe2d 100644 --- a/avro-builder/tests/codegen-14/src/main/avro/vs14/TestCollections.avsc +++ b/avro-builder/tests/codegen-14/src/main/avro/vs14/TestCollections.avsc @@ -6,6 +6,13 @@ "name": "str", "type": "string" }, + { + "name": "intAr", + "type": { + "type": "array", + "items": "int" + } + }, { "name": "strAr", "type": { diff --git a/avro-builder/tests/codegen-15/src/main/avro/vs15/TestCollections.avsc b/avro-builder/tests/codegen-15/src/main/avro/vs15/TestCollections.avsc index 63e766f07..0951ddcb1 100644 --- a/avro-builder/tests/codegen-15/src/main/avro/vs15/TestCollections.avsc +++ b/avro-builder/tests/codegen-15/src/main/avro/vs15/TestCollections.avsc @@ -6,6 +6,13 @@ "name": "str", "type": "string" }, + { + "name": "intAr", + "type": { + "type": "array", + "items": "int" + } + }, { "name": "strAr", "type": { diff --git a/avro-builder/tests/codegen-16/src/main/avro/vs16/TestCollections.avsc b/avro-builder/tests/codegen-16/src/main/avro/vs16/TestCollections.avsc index 1a95ae7d2..f6765a994 100644 --- a/avro-builder/tests/codegen-16/src/main/avro/vs16/TestCollections.avsc +++ b/avro-builder/tests/codegen-16/src/main/avro/vs16/TestCollections.avsc @@ -6,6 +6,13 @@ "name": "str", "type": "string" }, + { + "name": "intAr", + "type": { + "type": "array", + "items": "int" + } + }, { "name": "strAr", "type": { diff --git a/avro-builder/tests/codegen-17/src/main/avro/vs17/TestCollections.avsc b/avro-builder/tests/codegen-17/src/main/avro/vs17/TestCollections.avsc index 3611e3a79..3ab77a2a1 100644 --- a/avro-builder/tests/codegen-17/src/main/avro/vs17/TestCollections.avsc +++ b/avro-builder/tests/codegen-17/src/main/avro/vs17/TestCollections.avsc @@ -6,6 +6,13 @@ "name": "str", "type": "string" }, + { + "name": "intAr", + "type": { + "type": "array", + "items": "int" + } + }, { "name": "strAr", "type": { diff --git a/avro-builder/tests/codegen-18/src/main/avro/vs18/TestCollections.avsc b/avro-builder/tests/codegen-18/src/main/avro/vs18/TestCollections.avsc index 1027af4f8..03ba0de96 100644 --- a/avro-builder/tests/codegen-18/src/main/avro/vs18/TestCollections.avsc +++ b/avro-builder/tests/codegen-18/src/main/avro/vs18/TestCollections.avsc @@ -6,6 +6,13 @@ "name": "str", "type": "string" }, + { + "name": "intAr", + "type": { + "type": "array", + "items": "int" + } + }, { "name": "strAr", "type": { diff --git a/avro-builder/tests/codegen-19/src/main/avro/vs19/TestCollections.avsc b/avro-builder/tests/codegen-19/src/main/avro/vs19/TestCollections.avsc index ffcc2f949..8657a99ed 100644 --- a/avro-builder/tests/codegen-19/src/main/avro/vs19/TestCollections.avsc +++ b/avro-builder/tests/codegen-19/src/main/avro/vs19/TestCollections.avsc @@ -6,6 +6,13 @@ "name": "str", "type": "string" }, + { + "name": "intAr", + "type": { + "type": "array", + "items": "int" + } + }, { "name": "strAr", "type": { diff --git a/avro-builder/tests/codegen-charseq-method/src/main/avro/charseqmethod/TestCollections.avsc b/avro-builder/tests/codegen-charseq-method/src/main/avro/charseqmethod/TestCollections.avsc index d2bd29a5a..d6976ece2 100644 --- a/avro-builder/tests/codegen-charseq-method/src/main/avro/charseqmethod/TestCollections.avsc +++ b/avro-builder/tests/codegen-charseq-method/src/main/avro/charseqmethod/TestCollections.avsc @@ -6,6 +6,13 @@ "name": "str", "type": "string" }, + { + "name": "intAr", + "type": { + "type": "array", + "items": "int" + } + }, { "name": "strAr", "type": { diff --git a/avro-builder/tests/codegen-no-utf8-in-putbyindex/src/main/avro/charseqmethod/TestCollections.avsc b/avro-builder/tests/codegen-no-utf8-in-putbyindex/src/main/avro/charseqmethod/TestCollections.avsc index c70bd44cd..74ef23ea4 100644 --- a/avro-builder/tests/codegen-no-utf8-in-putbyindex/src/main/avro/charseqmethod/TestCollections.avsc +++ b/avro-builder/tests/codegen-no-utf8-in-putbyindex/src/main/avro/charseqmethod/TestCollections.avsc @@ -6,6 +6,13 @@ "name": "str", "type": "string" }, + { + "name": "intAr", + "type": { + "type": "array", + "items": "int" + } + }, { "name": "strAr", "type": { diff --git a/avro-builder/tests/tests-allavro/src/test/java/com/linkedin/avroutil1/builder/SpecificRecordTest.java b/avro-builder/tests/tests-allavro/src/test/java/com/linkedin/avroutil1/builder/SpecificRecordTest.java index d16abf716..307e0ae9f 100644 --- a/avro-builder/tests/tests-allavro/src/test/java/com/linkedin/avroutil1/builder/SpecificRecordTest.java +++ b/avro-builder/tests/tests-allavro/src/test/java/com/linkedin/avroutil1/builder/SpecificRecordTest.java @@ -1834,7 +1834,7 @@ public void testNewBuilder() throws Exception { public static Object[][] testPrivateModifierOnChunkMethodProvider() { return new Object[][]{{vs14.ThousandField.class}, {vs19.ThousandField.class}}; } - + @Test(dataProvider = "testPrivateModifierOnChunkMethodProvider") public void testPrivateModifierOnChunkMethod(Class clazz) { @@ -1854,6 +1854,33 @@ public void testPrivateModifierOnChunkMethod(Class chunkMethodNames.forEach(method -> Assert.assertTrue(Modifier.isPrivate(method.getModifiers()))); } + @Test + public void modifiablePrimitiveCollectionTest() { + String tba = "NewElement"; + RandomRecordGenerator generator = new RandomRecordGenerator(); + TestCollections instance = generator.randomSpecific(TestCollections.class, RecordGenerationConfig.newConfig().withAvoidNulls(true)); + + // array of string + instance.getStrAr().add(tba); + Assert.assertTrue(instance.getStrAr().contains(tba)); + + // union[null, List] + instance.getUnionOfArray().add(tba); + Assert.assertTrue(instance.getUnionOfArray().contains(tba)); + + // array (union[null, string]) + instance.getArOfUnionOfStr().add(tba); + Assert.assertTrue(instance.getArOfUnionOfStr().contains(tba)); + + + // Union (null, Map) + instance.getUnionOfMap().put("key1", tba); + Assert.assertEquals(tba, instance.getUnionOfMap().get("key1")); + + instance.getIntAr().add(Integer.MAX_VALUE); + Assert.assertEquals((int) instance.getIntAr().get(instance.getIntAr().size() - 1), Integer.MAX_VALUE); + } + @BeforeClass public void setup() { System.setProperty("org.apache.avro.specific.use_custom_coders", "true"); From 5ce94d15eb5e197296fd9d40c107ae591eacdb62 Mon Sep 17 00:00:00 2001 From: Uttam Kumar Date: Wed, 6 Dec 2023 13:52:53 -0800 Subject: [PATCH 02/13] List and Map transformers return views for primitiveCollection flag --- .../CollectionTransformerUtil.java | 299 ++++++++++++++++++ .../ListTransformer.java | 37 ++- .../collectiontransformer/MapTransformer.java | 26 +- 3 files changed, 351 insertions(+), 11 deletions(-) diff --git a/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/CollectionTransformerUtil.java b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/CollectionTransformerUtil.java index e717ce133..55145ed57 100644 --- a/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/CollectionTransformerUtil.java +++ b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/CollectionTransformerUtil.java @@ -7,6 +7,15 @@ package com.linkedin.avroutil1.compatibility.collectiontransformer; import com.linkedin.avroutil1.compatibility.StringUtils; +import java.util.AbstractList; +import java.util.AbstractMap; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.avro.util.Utf8; public class CollectionTransformerUtil { @@ -17,4 +26,294 @@ public static String getErrorMessageForInstance(Object obj) { return String.valueOf(obj) + ((obj == null) ? StringUtils.EMPTY_STRING : " (an instance of " + obj.getClass().getName() + ")"); } + + public static List createStringListView(List utf8List) { + return new AbstractList() { + @Override + public String get(int index) { + return String.valueOf(utf8List.get(index)); + } + + @Override + public int size() { + return utf8List.size(); + } + + @Override + public String set(int index, String element) { + String previousValue = String.valueOf(utf8List.get(index)); + utf8List.set(index, new Utf8(element)); + return previousValue; + } + + @Override + public void add(int index, String element) { + utf8List.add(index, new Utf8(element)); + } + + @Override + public boolean add(String element) { + return utf8List.add(new Utf8(element)); + } + + @Override + public boolean addAll(int index, java.util.Collection c) { + boolean modified = false; + for (String element : c) { + utf8List.add(index++, new Utf8(element)); + modified = true; + } + return modified; + } + }; + } + + public static List createCharSequenceListView(List utf8List) { + return new AbstractList() { + @Override + public CharSequence get(int index) { + return String.valueOf(utf8List.get(index)); + } + + @Override + public int size() { + return utf8List.size(); + } + + @Override + public CharSequence set(int index, CharSequence element) { + CharSequence previousValue = String.valueOf(utf8List.get(index)); + utf8List.set(index, new Utf8(element.toString())); + return previousValue; + } + + @Override + public void add(int index, CharSequence element) { + utf8List.add(index, new Utf8(element.toString())); + } + + @Override + public boolean add(CharSequence element) { + return utf8List.add(new Utf8(element.toString())); + } + + @Override + public boolean addAll(int index, java.util.Collection c) { + boolean modified = false; + for (CharSequence element : c) { + utf8List.add(index++, new Utf8(element.toString())); + modified = true; + } + return modified; + } + + }; + } + + public static List createUtf8ListView(List utf8List) { + return new AbstractList() { + @Override + public Utf8 get(int index) { + return utf8List.get(index); + } + + @Override + public int size() { + return utf8List.size(); + } + + @Override + public Utf8 set(int index, Utf8 element) { + Utf8 previousValue = utf8List.get(index); + utf8List.set(index, element); + return previousValue; + } + + @Override + public void add(int index, Utf8 element) { + utf8List.add(index, element); + } + + @Override + public boolean add(Utf8 element) { + return utf8List.add(element); + } + + @Override + public boolean addAll(int index, java.util.Collection c) { + return utf8List.addAll(index, c); + } + }; + } + + + public static Map createStringMapView(Map utf8Map) { + if (utf8Map == null) { + return null; + } + + return new AbstractMap() { + @Override + public Set> entrySet() { + return Collections.unmodifiableSet(utf8Map.entrySet().stream() + .collect(Collectors.toMap( + entry -> String.valueOf(entry.getKey()), + entry -> String.valueOf(entry.getValue()) + )) + .entrySet()); + } + + @Override + public String put(String key, String value) { + Utf8 utf8Key = new Utf8(key); + Utf8 utf8Value = new Utf8(value); + Utf8 previousValue = utf8Map.put(utf8Key, utf8Value); + return previousValue != null ? String.valueOf(previousValue) : null; + } + + @Override + public Set keySet() { + return Collections.unmodifiableSet(utf8Map.keySet().stream() + .map(String::valueOf) + .collect(Collectors.toSet())); + } + + @Override + public Collection values() { + return Collections.unmodifiableCollection(utf8Map.values().stream() + .map(String::valueOf) + .collect(Collectors.toList())); + } + + @Override + public int size() { + return utf8Map.size(); + } + + @Override + public boolean isEmpty() { + return utf8Map.isEmpty(); + } + + @Override + public boolean containsKey(Object key) { + return utf8Map.containsKey(new Utf8(String.valueOf(key))); + } + + @Override + public boolean containsValue(Object value) { + return utf8Map.containsValue(new Utf8(String.valueOf(value))); + } + + @Override + public String get(Object key) { + Utf8 utf8Key = new Utf8(String.valueOf(key)); + Utf8 utf8Value = utf8Map.get(utf8Key); + return utf8Value != null ? String.valueOf(utf8Value) : null; + } + + @Override + public String remove(Object key) { + Utf8 utf8Key = new Utf8(String.valueOf(key)); + Utf8 previousValue = utf8Map.remove(utf8Key); + return previousValue != null ? String.valueOf(previousValue) : null; + } + + @Override + public void putAll(Map m) { + m.forEach((key, value) -> { + Utf8 utf8Key = new Utf8(key); + Utf8 utf8Value = new Utf8(value); + utf8Map.put(utf8Key, utf8Value); + }); + } + }; + } + + public static Map createUtf8MapView(Map utf8Map) { + return utf8Map; + } + + public static Map createCharSequenceMapView(Map utf8Map) { + if (utf8Map == null) { + return null; + } + + return new AbstractMap() { + @Override + public Set> entrySet() { + return Collections.unmodifiableSet(utf8Map.entrySet().stream() + .collect(Collectors.toMap( + entry -> (CharSequence) String.valueOf(entry.getKey()), + entry -> (CharSequence) String.valueOf(entry.getValue()) + )) + .entrySet()); + } + + @Override + public CharSequence put(CharSequence key, CharSequence value) { + Utf8 utf8Key = new Utf8(key.toString()); + Utf8 utf8Value = new Utf8(value.toString()); + Utf8 previousValue = utf8Map.put(utf8Key, utf8Value); + return previousValue != null ? (CharSequence) String.valueOf(previousValue) : null; + } + + @Override + public Set keySet() { + return Collections.unmodifiableSet(utf8Map.keySet().stream() + .map(CharSequence::toString) + .collect(Collectors.toSet())); + } + + @Override + public Collection values() { + return Collections.unmodifiableCollection(utf8Map.values().stream() + .map(CharSequence::toString) + .collect(Collectors.toList())); + } + + @Override + public int size() { + return utf8Map.size(); + } + + @Override + public boolean isEmpty() { + return utf8Map.isEmpty(); + } + + @Override + public boolean containsKey(Object key) { + return utf8Map.containsKey(new Utf8(String.valueOf(key))); + } + + @Override + public boolean containsValue(Object value) { + return utf8Map.containsValue(new Utf8(String.valueOf(value))); + } + + @Override + public CharSequence get(Object key) { + Utf8 utf8Key = new Utf8(String.valueOf(key)); + Utf8 utf8Value = utf8Map.get(utf8Key); + return utf8Value != null ? (CharSequence) String.valueOf(utf8Value) : null; + } + + @Override + public CharSequence remove(Object key) { + Utf8 utf8Key = new Utf8(String.valueOf(key)); + Utf8 previousValue = utf8Map.remove(utf8Key); + return previousValue != null ? (CharSequence) String.valueOf(previousValue) : null; + } + + @Override + public void putAll(Map m) { + m.forEach((key, value) -> { + Utf8 utf8Key = new Utf8(String.valueOf(key)); + Utf8 utf8Value = new Utf8(String.valueOf(value)); + utf8Map.put(utf8Key, utf8Value); + }); + } + }; + } } diff --git a/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/ListTransformer.java b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/ListTransformer.java index bcae9fd6d..5601f3aa5 100644 --- a/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/ListTransformer.java +++ b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/ListTransformer.java @@ -11,6 +11,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import org.apache.avro.util.Utf8; public class ListTransformer { @@ -18,27 +19,43 @@ public class ListTransformer { public static List getUtf8List(Object listObj) { flushCharSeqFlag(); - return getUtf8(listObj); + return getUtf8(listObj, false); } public static List getStringList(Object listObj) { flushCharSeqFlag(); - return getString(listObj); + return getString(listObj, false); } public static List getCharSequenceList(Object listObj) { flushCharSeqFlag(); - return getCharSequence(listObj); + return getCharSequence(listObj, false); } - private static List getUtf8(Object listObj) { + public static List getUtf8List(Object listObj, boolean isPrimitiveCollection) { + flushCharSeqFlag(); + return getUtf8(listObj, isPrimitiveCollection); + } + + public static List getStringList(Object listObj, boolean isPrimitiveCollection) { + flushCharSeqFlag(); + return getString(listObj, isPrimitiveCollection); + } + + public static List getCharSequenceList(Object listObj, boolean isPrimitiveCollection) { + flushCharSeqFlag(); + return getCharSequence(listObj, isPrimitiveCollection); + } + + private static List getUtf8(Object listObj, boolean isPrimitive) { if(listObj == null) return null; + if(isPrimitive) return CollectionTransformerUtil.createUtf8ListView((List) listObj); if (listObj instanceof List) { List list = (List) listObj; List ret = new ArrayList(list.size()); for (Object item : list) { if (item instanceof List) { - ret.add(ListTransformer.getUtf8((List) item)); + ret.add(ListTransformer.getUtf8((List) item, false)); } else if (item instanceof Map) { hasCharSeq.set(true); ret.add(MapTransformer.getUtf8Map((Map) item)); @@ -60,15 +77,16 @@ private static void flushCharSeqFlag() { hasCharSeq.set(false); } - private static List getString(Object listObj) { + private static List getString(Object listObj, boolean isPrimitive) { if(listObj == null) return null; + if(isPrimitive) return CollectionTransformerUtil.createStringListView((List) listObj); List ret; if(listObj instanceof List) { List list = (List) listObj; ret = new ArrayList(list.size()); for (Object item : list) { if (item instanceof List) { - ret.add(ListTransformer.getString((List) item)); + ret.add(ListTransformer.getString((List) item, false)); } else if (item instanceof Map) { hasCharSeq.set(true); ret.add(MapTransformer.getStringMap((Map) item)); @@ -86,15 +104,16 @@ private static List getString(Object listObj) { } } - private static List getCharSequence(Object listObj) { + private static List getCharSequence(Object listObj, boolean isPrimitive) { if(listObj == null) return null; + if(isPrimitive) return CollectionTransformerUtil.createCharSequenceListView((List) listObj); List ret; if(listObj instanceof List) { List list = (List) listObj; ret = new ArrayList(list.size()); for (Object item : list) { if (item instanceof List) { - ret.add(ListTransformer.getString((List) item)); + ret.add(ListTransformer.getString((List) item, false)); } else if (item instanceof Map) { hasCharSeq.set(true); ret.add(MapTransformer.getStringMap((Map) item)); diff --git a/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/MapTransformer.java b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/MapTransformer.java index 6ce945af2..fb63bee25 100644 --- a/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/MapTransformer.java +++ b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/MapTransformer.java @@ -11,11 +11,27 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.avro.util.Utf8; public class MapTransformer { public static Map getUtf8Map(Object mapObj) { + return getUtf8Map(mapObj, false); + } + + public static Map getStringMap(Object mapObj) { + return getStringMap(mapObj, false); + } + + public static Map getCharSequenceMap(Object mapObj) { + return getCharSequenceMap(mapObj, false); + } + + public static Map getUtf8Map(Object mapObj, boolean isPrimitiveCollection) { + if(isPrimitiveCollection) { + return CollectionTransformerUtil.createUtf8MapView((Map) mapObj); + } if (mapObj == null) { return null; } @@ -44,7 +60,10 @@ public static Map getUtf8Map(Object mapObj) { return Collections.unmodifiableMap(ret); } - public static Map getStringMap(Object mapObj) { + public static Map getStringMap(Object mapObj, boolean isPrimitiveCollection) { + if(isPrimitiveCollection) { + return CollectionTransformerUtil.createStringMapView((Map) mapObj); + } if (mapObj == null) { return null; } @@ -72,7 +91,10 @@ public static Map getStringMap(Object mapObj) { return Collections.unmodifiableMap(ret); } - public static Map getCharSequenceMap(Object mapObj) { + public static Map getCharSequenceMap(Object mapObj, boolean isPrimitiveCollection) { + if(isPrimitiveCollection) { + return CollectionTransformerUtil.createCharSequenceMapView((Map) mapObj); + } if (mapObj == null) { return null; } From 8976b64466227987cb1cd374ee0b25865f23055e Mon Sep 17 00:00:00 2001 From: Uttam Kumar Date: Wed, 6 Dec 2023 13:53:16 -0800 Subject: [PATCH 03/13] Updated codegen to provide isPrimitiveFlag for List and Map transformer --- .../codegen/SpecificRecordClassGenerator.java | 58 ++++++++++--------- .../codegen/SpecificRecordGeneratorUtil.java | 48 +++++++++++++++ 2 files changed, 79 insertions(+), 27 deletions(-) diff --git a/avro-codegen/src/main/java/com/linkedin/avroutil1/codegen/SpecificRecordClassGenerator.java b/avro-codegen/src/main/java/com/linkedin/avroutil1/codegen/SpecificRecordClassGenerator.java index 27d40fac6..11b084544 100644 --- a/avro-codegen/src/main/java/com/linkedin/avroutil1/codegen/SpecificRecordClassGenerator.java +++ b/avro-codegen/src/main/java/com/linkedin/avroutil1/codegen/SpecificRecordClassGenerator.java @@ -1701,12 +1701,14 @@ private void addGetByIndexMethod(TypeSpec.Builder classBuilder, AvroRecordSchema switchBuilder.addStatement("case $L: return com.linkedin.avroutil1.compatibility.StringConverterUtil.get$L(this.$L)", fieldIndex++, fieldClass.getSimpleName(), escapedFieldName); } else if (SpecificRecordGeneratorUtil.isListTransformerApplicableForSchema(field.getSchema())) { switchBuilder.addStatement( - "case $L: return com.linkedin.avroutil1.compatibility.collectiontransformer.ListTransformer.get$LList(this.$L)", - fieldIndex++, config.getDefaultMethodStringRepresentation().getJsonValue(), escapedFieldName); + "case $L: return com.linkedin.avroutil1.compatibility.collectiontransformer.ListTransformer.get$LList(this.$L, $L)", + fieldIndex++, config.getDefaultMethodStringRepresentation().getJsonValue(), escapedFieldName, + SpecificRecordGeneratorUtil.isCollectionSchemaValuePrimitive(field.getSchema())); } else if (SpecificRecordGeneratorUtil.isMapTransformerApplicable(field.getSchema())) { switchBuilder.addStatement( - "case $L: return com.linkedin.avroutil1.compatibility.collectiontransformer.MapTransformer.get$LMap(this.$L)", - fieldIndex++, config.getDefaultMethodStringRepresentation().getJsonValue(), escapedFieldName); + "case $L: return com.linkedin.avroutil1.compatibility.collectiontransformer.MapTransformer.get$LMap(this.$L, $L)", + fieldIndex++, config.getDefaultMethodStringRepresentation().getJsonValue(), escapedFieldName, + SpecificRecordGeneratorUtil.isCollectionSchemaValuePrimitive(field.getSchema())); } else if (field.getSchema() != null && AvroType.UNION.equals(field.getSchema().type())) { switchBuilder.addStatement("case $L:", fieldIndex++); @@ -1726,16 +1728,16 @@ private void addGetByIndexMethod(TypeSpec.Builder classBuilder, AvroRecordSchema unionMemberSchema.getSchema())) { switchBuilder.beginControlFlow("else if($1L instanceof $2T)", escapedFieldName, List.class) .addStatement( - "return com.linkedin.avroutil1.compatibility.collectiontransformer.ListTransformer.get$1LList($2L)", - config.getDefaultMethodStringRepresentation().getJsonValue(), - escapedFieldName) + "return com.linkedin.avroutil1.compatibility.collectiontransformer.ListTransformer.get$1LList($2L, $3L)", + config.getDefaultMethodStringRepresentation().getJsonValue(), escapedFieldName, + SpecificRecordGeneratorUtil.isCollectionSchemaValuePrimitive(field.getSchema())) .endControlFlow(); } else if (SpecificRecordGeneratorUtil.isMapTransformerApplicable(unionMemberSchema.getSchema())) { switchBuilder.beginControlFlow("else if($1L instanceof $2T)", escapedFieldName, Map.class) .addStatement( - "return com.linkedin.avroutil1.compatibility.collectiontransformer.MapTransformer.get$1LMap($2L)", - config.getDefaultMethodStringRepresentation().getJsonValue(), - escapedFieldName) + "return com.linkedin.avroutil1.compatibility.collectiontransformer.MapTransformer.get$1LMap($2L, $3L)", + config.getDefaultMethodStringRepresentation().getJsonValue(), escapedFieldName, + SpecificRecordGeneratorUtil.isCollectionSchemaValuePrimitive(field.getSchema())) .endControlFlow(); } } @@ -1920,12 +1922,12 @@ private MethodSpec getSetterMethodSpec(AvroSchemaField field, SpecificRecordGene methodSpecBuilder.beginControlFlow("else if($1L instanceof $2T)", escapedFieldName, List.class); if (config.getDefaultFieldStringRepresentation().equals(AvroJavaStringRepresentation.STRING)) { methodSpecBuilder.addStatement( - "this.$1L = com.linkedin.avroutil1.compatibility.collectiontransformer.ListTransformer.getStringList($1L)", - escapedFieldName); + "this.$1L = com.linkedin.avroutil1.compatibility.collectiontransformer.ListTransformer.getStringList($1L, $2L)", + escapedFieldName, SpecificRecordGeneratorUtil.isCollectionSchemaValuePrimitive(field.getSchema())); } else { methodSpecBuilder.addStatement( - "this.$1L = com.linkedin.avroutil1.compatibility.collectiontransformer.ListTransformer.getUtf8List($1L)", - escapedFieldName); + "this.$1L = com.linkedin.avroutil1.compatibility.collectiontransformer.ListTransformer.getUtf8List($1L, $2L)", + escapedFieldName, SpecificRecordGeneratorUtil.isCollectionSchemaValuePrimitive(field.getSchema())); } methodSpecBuilder.endControlFlow(); @@ -1933,12 +1935,12 @@ private MethodSpec getSetterMethodSpec(AvroSchemaField field, SpecificRecordGene methodSpecBuilder.beginControlFlow("else if($1L instanceof $2T)", escapedFieldName, Map.class); if (config.getDefaultFieldStringRepresentation().equals(AvroJavaStringRepresentation.STRING)) { methodSpecBuilder.addStatement( - "this.$1L = com.linkedin.avroutil1.compatibility.collectiontransformer.MapTransformer.getStringMap($1L)", - escapedFieldName); + "this.$1L = com.linkedin.avroutil1.compatibility.collectiontransformer.MapTransformer.getStringMap($1L, $2L)", + escapedFieldName, SpecificRecordGeneratorUtil.isCollectionSchemaValuePrimitive(field.getSchema())); } else { methodSpecBuilder.addStatement( - "this.$1L = com.linkedin.avroutil1.compatibility.collectiontransformer.MapTransformer.getUtf8Map($1L)", - escapedFieldName); + "this.$1L = com.linkedin.avroutil1.compatibility.collectiontransformer.MapTransformer.getUtf8Map($1L, $2L)", + escapedFieldName, SpecificRecordGeneratorUtil.isCollectionSchemaValuePrimitive(field.getSchema())); } methodSpecBuilder.endControlFlow(); } @@ -1983,14 +1985,14 @@ private MethodSpec getGetterMethodSpec(AvroSchemaField field, SpecificRecordGene escapedFieldName, config.getDefaultMethodStringRepresentation().getJsonValue()); } else if (SpecificRecordGeneratorUtil.isListTransformerApplicableForSchema(field.getSchema())) { methodSpecBuilder.addStatement( - "return com.linkedin.avroutil1.compatibility.collectiontransformer.ListTransformer.get$LList(this.$L)", - config.getDefaultMethodStringRepresentation().getJsonValue(), - escapedFieldName); + "return com.linkedin.avroutil1.compatibility.collectiontransformer.ListTransformer.get$LList(this.$L, $L)", + config.getDefaultMethodStringRepresentation().getJsonValue(), escapedFieldName, + SpecificRecordGeneratorUtil.isCollectionSchemaValuePrimitive(field.getSchema())); } else if (SpecificRecordGeneratorUtil.isMapTransformerApplicable(field.getSchema())) { methodSpecBuilder.addStatement( - "return com.linkedin.avroutil1.compatibility.collectiontransformer.MapTransformer.get$LMap(this.$L)", + "return com.linkedin.avroutil1.compatibility.collectiontransformer.MapTransformer.get$LMap(this.$L, $L)", config.getDefaultMethodStringRepresentation().getJsonValue(), - escapedFieldName); + escapedFieldName, SpecificRecordGeneratorUtil.isCollectionSchemaValuePrimitive(field.getSchema())); } else if (field.getSchema() != null && AvroType.UNION.equals(field.getSchema().type())) { methodSpecBuilder.beginControlFlow("if (this.$1L == null)", escapedFieldName) @@ -2007,14 +2009,16 @@ private MethodSpec getGetterMethodSpec(AvroSchemaField field, SpecificRecordGene } else if (SpecificRecordGeneratorUtil.isListTransformerApplicableForSchema(unionMemberSchema.getSchema())) { methodSpecBuilder.beginControlFlow("else if($1L instanceof $2T)", escapedFieldName, List.class) .addStatement( - "return com.linkedin.avroutil1.compatibility.collectiontransformer.ListTransformer.get$2LList($1L)", - escapedFieldName, config.getDefaultMethodStringRepresentation().getJsonValue()) + "return com.linkedin.avroutil1.compatibility.collectiontransformer.ListTransformer.get$2LList($1L, $3L)", + escapedFieldName, config.getDefaultMethodStringRepresentation().getJsonValue(), + SpecificRecordGeneratorUtil.isCollectionSchemaValuePrimitive(field.getSchema())) .endControlFlow(); } else if (SpecificRecordGeneratorUtil.isMapTransformerApplicable(unionMemberSchema.getSchema())) { methodSpecBuilder.beginControlFlow("else if($1L instanceof $2T)", escapedFieldName, Map.class) .addStatement( - "return com.linkedin.avroutil1.compatibility.collectiontransformer.MapTransformer.get$2LMap($1L)", - escapedFieldName, config.getDefaultMethodStringRepresentation().getJsonValue()) + "return com.linkedin.avroutil1.compatibility.collectiontransformer.MapTransformer.get$2LMap($1L, $3L)", + escapedFieldName, config.getDefaultMethodStringRepresentation().getJsonValue(), + SpecificRecordGeneratorUtil.isCollectionSchemaValuePrimitive(field.getSchema())) .endControlFlow(); } } diff --git a/avro-codegen/src/main/java/com/linkedin/avroutil1/codegen/SpecificRecordGeneratorUtil.java b/avro-codegen/src/main/java/com/linkedin/avroutil1/codegen/SpecificRecordGeneratorUtil.java index b6040203e..7c5ecaf62 100644 --- a/avro-codegen/src/main/java/com/linkedin/avroutil1/codegen/SpecificRecordGeneratorUtil.java +++ b/avro-codegen/src/main/java/com/linkedin/avroutil1/codegen/SpecificRecordGeneratorUtil.java @@ -288,6 +288,54 @@ public static boolean isMapTransformerApplicable(AvroSchema schema) { return isNullUnionOf(AvroType.MAP, schema); } + /** + * If a schema is primitive type or a null union of primitive type, it can be handled as a primitive type + * @param schema schema to be validated + * @return true if schema can be handled as primitive, else false + */ + private static boolean canBeHandledAsPrimitiveType(AvroSchema schema) { + if (schema instanceof AvroUnionSchema) { + if (((AvroUnionSchema) schema).getTypes().size() != 2) { + return false; + } + + return ((AvroUnionSchema) schema).getTypes().get(0).getSchema().type().equals(AvroType.NULL) + && ((AvroUnionSchema) schema).getTypes().get(1).getSchema().type().isPrimitive() + || ((AvroUnionSchema) schema).getTypes().get(1).getSchema().type().equals(AvroType.NULL) + && ((AvroUnionSchema) schema).getTypes().get(0).getSchema().type().isPrimitive(); + } + + return schema.type().isPrimitive(); + } + + /** + * schema type must be either a List of Map (or a null union of List/Map) + * @return true if schema value is primitive + */ + public static boolean isCollectionSchemaValuePrimitive(AvroSchema schema) { + if(!isListTransformerApplicableForSchema(schema) && !isMapTransformerApplicable(schema)) { + return false; + } + if (schema.type().equals(AvroType.UNION)) { + if (((AvroUnionSchema) schema).getTypes().get(0).getSchema().type().equals(AvroType.NULL)) { + schema = ((AvroUnionSchema) schema).getTypes().get(1).getSchema(); + } else if (((AvroUnionSchema) schema).getTypes().get(1).getSchema().type().equals(AvroType.NULL)) { + schema = ((AvroUnionSchema) schema).getTypes().get(0).getSchema(); + } else { + // checks with isListTransformerApplicable and isMapTransformerApplicable should prevent this from happening + throw new IllegalArgumentException("schema type must be either a List of Map (or a null union of List/Map)"); + } + } + + if(AvroType.ARRAY.equals(schema.type())) { + return canBeHandledAsPrimitiveType(((AvroArraySchema) schema).getValueSchema()); + } else if (AvroType.MAP.equals(schema.type())) { + return canBeHandledAsPrimitiveType(((AvroMapSchema) schema).getValueSchema()); + } else { + // checks with isListTransformerApplicable and isMapTransformerApplicable should prevent this from happening + throw new IllegalArgumentException("schema type must be either a List of Map (or a null union of List/Map)"); + } + } public static boolean schemaContainsString(AvroSchema schema) { if (schema == null) { From 9d1f66dcfb80b5ed2186f545fc48569845435d75 Mon Sep 17 00:00:00 2001 From: Uttam Kumar Date: Sun, 11 Feb 2024 21:05:45 -0800 Subject: [PATCH 04/13] UT --- .../avroutil1/builder/SpecificRecordTest.java | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/avro-builder/tests/tests-allavro/src/test/java/com/linkedin/avroutil1/builder/SpecificRecordTest.java b/avro-builder/tests/tests-allavro/src/test/java/com/linkedin/avroutil1/builder/SpecificRecordTest.java index 808df3e86..3c4ce8bbe 100644 --- a/avro-builder/tests/tests-allavro/src/test/java/com/linkedin/avroutil1/builder/SpecificRecordTest.java +++ b/avro-builder/tests/tests-allavro/src/test/java/com/linkedin/avroutil1/builder/SpecificRecordTest.java @@ -1829,6 +1829,32 @@ public void testNewBuilder() throws Exception { compareIndexedRecords(instance, builder.build()); } + public void modifiablePrimitiveCollectionTest() { + String tba = "NewElement"; + RandomRecordGenerator generator = new RandomRecordGenerator(); + TestCollections instance = generator.randomSpecific(TestCollections.class, RecordGenerationConfig.newConfig().withAvoidNulls(true)); + + // array of string + instance.getStrAr().add(tba); + Assert.assertTrue(instance.getStrAr().contains(tba)); + + // union[null, List] + instance.getUnionOfArray().add(tba); + Assert.assertTrue(instance.getUnionOfArray().contains(tba)); + + // array (union[null, string]) + instance.getArOfUnionOfStr().add(tba); + Assert.assertTrue(instance.getArOfUnionOfStr().contains(tba)); + + + // Union (null, Map) + instance.getUnionOfMap().put("key1", tba); + Assert.assertEquals(tba, instance.getUnionOfMap().get("key1")); + + instance.getIntAr().add(Integer.MAX_VALUE); + Assert.assertEquals((int) instance.getIntAr().get(instance.getIntAr().size() - 1), Integer.MAX_VALUE); + } + @BeforeClass public void setup() { System.setProperty("org.apache.avro.specific.use_custom_coders", "true"); From a1b4fec90023287aa64423d6ef253ccc0e5c5f05 Mon Sep 17 00:00:00 2001 From: Uttam Kumar Date: Sun, 11 Feb 2024 21:33:32 -0800 Subject: [PATCH 05/13] Updated tests --- .../src/main/avro/vs110/TestCollections.avsc | 13 ++++++------- .../src/main/avro/vs111/TestCollections.avsc | 14 +++++++------- .../src/main/avro/vs14/TestCollections.avsc | 14 +++++++------- .../src/main/avro/vs15/TestCollections.avsc | 14 +++++++------- .../src/main/avro/vs16/TestCollections.avsc | 14 +++++++------- .../src/main/avro/vs17/TestCollections.avsc | 14 +++++++------- .../src/main/avro/vs18/TestCollections.avsc | 14 +++++++------- .../src/main/avro/vs19/TestCollections.avsc | 14 +++++++------- .../main/avro/charseqmethod/TestCollections.avsc | 14 +++++++------- .../main/avro/charseqmethod/TestCollections.avsc | 14 +++++++------- .../avroutil1/builder/SpecificRecordTest.java | 5 ++++- 11 files changed, 73 insertions(+), 71 deletions(-) diff --git a/avro-builder/tests/codegen-110/src/main/avro/vs110/TestCollections.avsc b/avro-builder/tests/codegen-110/src/main/avro/vs110/TestCollections.avsc index c3a91c1f8..6e3b6f58f 100644 --- a/avro-builder/tests/codegen-110/src/main/avro/vs110/TestCollections.avsc +++ b/avro-builder/tests/codegen-110/src/main/avro/vs110/TestCollections.avsc @@ -6,13 +6,6 @@ "name": "str", "type": "string" }, - { - "name": "intAr", - "type": { - "type": "array", - "items": "int" - } - }, { "name": "strAr", "type": { @@ -79,6 +72,12 @@ }] } } + },{ + "name": "intAr", + "type": { + "type": "array", + "items": "int" + } } ], "type": "record" diff --git a/avro-builder/tests/codegen-111/src/main/avro/vs111/TestCollections.avsc b/avro-builder/tests/codegen-111/src/main/avro/vs111/TestCollections.avsc index aa3942c55..ebbb6de31 100644 --- a/avro-builder/tests/codegen-111/src/main/avro/vs111/TestCollections.avsc +++ b/avro-builder/tests/codegen-111/src/main/avro/vs111/TestCollections.avsc @@ -6,13 +6,6 @@ "name": "str", "type": "string" }, - { - "name": "intAr", - "type": { - "type": "array", - "items": "int" - } - }, { "name": "strAr", "type": { @@ -79,6 +72,13 @@ }] } } + }, + { + "name": "intAr", + "type": { + "type": "array", + "items": "int" + } } ], "type": "record" diff --git a/avro-builder/tests/codegen-14/src/main/avro/vs14/TestCollections.avsc b/avro-builder/tests/codegen-14/src/main/avro/vs14/TestCollections.avsc index df6c6fe2d..c8cb8cbeb 100644 --- a/avro-builder/tests/codegen-14/src/main/avro/vs14/TestCollections.avsc +++ b/avro-builder/tests/codegen-14/src/main/avro/vs14/TestCollections.avsc @@ -6,13 +6,6 @@ "name": "str", "type": "string" }, - { - "name": "intAr", - "type": { - "type": "array", - "items": "int" - } - }, { "name": "strAr", "type": { @@ -79,6 +72,13 @@ }] } } + }, + { + "name": "intAr", + "type": { + "type": "array", + "items": "int" + } } ], "type": "record" diff --git a/avro-builder/tests/codegen-15/src/main/avro/vs15/TestCollections.avsc b/avro-builder/tests/codegen-15/src/main/avro/vs15/TestCollections.avsc index 0951ddcb1..e4d4fe3a9 100644 --- a/avro-builder/tests/codegen-15/src/main/avro/vs15/TestCollections.avsc +++ b/avro-builder/tests/codegen-15/src/main/avro/vs15/TestCollections.avsc @@ -6,13 +6,6 @@ "name": "str", "type": "string" }, - { - "name": "intAr", - "type": { - "type": "array", - "items": "int" - } - }, { "name": "strAr", "type": { @@ -79,6 +72,13 @@ }] } } + }, + { + "name": "intAr", + "type": { + "type": "array", + "items": "int" + } } ], "type": "record" diff --git a/avro-builder/tests/codegen-16/src/main/avro/vs16/TestCollections.avsc b/avro-builder/tests/codegen-16/src/main/avro/vs16/TestCollections.avsc index f6765a994..3e5832ad0 100644 --- a/avro-builder/tests/codegen-16/src/main/avro/vs16/TestCollections.avsc +++ b/avro-builder/tests/codegen-16/src/main/avro/vs16/TestCollections.avsc @@ -6,13 +6,6 @@ "name": "str", "type": "string" }, - { - "name": "intAr", - "type": { - "type": "array", - "items": "int" - } - }, { "name": "strAr", "type": { @@ -79,6 +72,13 @@ }] } } + }, + { + "name": "intAr", + "type": { + "type": "array", + "items": "int" + } } ], "type": "record" diff --git a/avro-builder/tests/codegen-17/src/main/avro/vs17/TestCollections.avsc b/avro-builder/tests/codegen-17/src/main/avro/vs17/TestCollections.avsc index 3ab77a2a1..ac9430f16 100644 --- a/avro-builder/tests/codegen-17/src/main/avro/vs17/TestCollections.avsc +++ b/avro-builder/tests/codegen-17/src/main/avro/vs17/TestCollections.avsc @@ -6,13 +6,6 @@ "name": "str", "type": "string" }, - { - "name": "intAr", - "type": { - "type": "array", - "items": "int" - } - }, { "name": "strAr", "type": { @@ -79,6 +72,13 @@ }] } } + }, + { + "name": "intAr", + "type": { + "type": "array", + "items": "int" + } } ], "type": "record" diff --git a/avro-builder/tests/codegen-18/src/main/avro/vs18/TestCollections.avsc b/avro-builder/tests/codegen-18/src/main/avro/vs18/TestCollections.avsc index 03ba0de96..42fa056fd 100644 --- a/avro-builder/tests/codegen-18/src/main/avro/vs18/TestCollections.avsc +++ b/avro-builder/tests/codegen-18/src/main/avro/vs18/TestCollections.avsc @@ -6,13 +6,6 @@ "name": "str", "type": "string" }, - { - "name": "intAr", - "type": { - "type": "array", - "items": "int" - } - }, { "name": "strAr", "type": { @@ -79,6 +72,13 @@ }] } } + }, + { + "name": "intAr", + "type": { + "type": "array", + "items": "int" + } } ], "type": "record" diff --git a/avro-builder/tests/codegen-19/src/main/avro/vs19/TestCollections.avsc b/avro-builder/tests/codegen-19/src/main/avro/vs19/TestCollections.avsc index 8657a99ed..2651f92e9 100644 --- a/avro-builder/tests/codegen-19/src/main/avro/vs19/TestCollections.avsc +++ b/avro-builder/tests/codegen-19/src/main/avro/vs19/TestCollections.avsc @@ -6,13 +6,6 @@ "name": "str", "type": "string" }, - { - "name": "intAr", - "type": { - "type": "array", - "items": "int" - } - }, { "name": "strAr", "type": { @@ -79,6 +72,13 @@ }] } } + }, + { + "name": "intAr", + "type": { + "type": "array", + "items": "int" + } } ], "type": "record" diff --git a/avro-builder/tests/codegen-charseq-method/src/main/avro/charseqmethod/TestCollections.avsc b/avro-builder/tests/codegen-charseq-method/src/main/avro/charseqmethod/TestCollections.avsc index d6976ece2..9dcaac88d 100644 --- a/avro-builder/tests/codegen-charseq-method/src/main/avro/charseqmethod/TestCollections.avsc +++ b/avro-builder/tests/codegen-charseq-method/src/main/avro/charseqmethod/TestCollections.avsc @@ -6,13 +6,6 @@ "name": "str", "type": "string" }, - { - "name": "intAr", - "type": { - "type": "array", - "items": "int" - } - }, { "name": "strAr", "type": { @@ -79,6 +72,13 @@ }] } } + }, + { + "name": "intAr", + "type": { + "type": "array", + "items": "int" + } } ], "type": "record" diff --git a/avro-builder/tests/codegen-no-utf8-in-putbyindex/src/main/avro/charseqmethod/TestCollections.avsc b/avro-builder/tests/codegen-no-utf8-in-putbyindex/src/main/avro/charseqmethod/TestCollections.avsc index 74ef23ea4..e8b8e730d 100644 --- a/avro-builder/tests/codegen-no-utf8-in-putbyindex/src/main/avro/charseqmethod/TestCollections.avsc +++ b/avro-builder/tests/codegen-no-utf8-in-putbyindex/src/main/avro/charseqmethod/TestCollections.avsc @@ -6,13 +6,6 @@ "name": "str", "type": "string" }, - { - "name": "intAr", - "type": { - "type": "array", - "items": "int" - } - }, { "name": "strAr", "type": { @@ -79,6 +72,13 @@ }] } } + }, + { + "name": "intAr", + "type": { + "type": "array", + "items": "int" + } } ], "type": "record" diff --git a/avro-builder/tests/tests-allavro/src/test/java/com/linkedin/avroutil1/builder/SpecificRecordTest.java b/avro-builder/tests/tests-allavro/src/test/java/com/linkedin/avroutil1/builder/SpecificRecordTest.java index 3c4ce8bbe..4d8c4b96b 100644 --- a/avro-builder/tests/tests-allavro/src/test/java/com/linkedin/avroutil1/builder/SpecificRecordTest.java +++ b/avro-builder/tests/tests-allavro/src/test/java/com/linkedin/avroutil1/builder/SpecificRecordTest.java @@ -1358,6 +1358,7 @@ private Object[][] testStringTypeParamsProvider() { put("unionOfMap", "java.util.Map"); put("arOfUnionOfStr", "java.util.List"); put("arOfMapOfUnionOfArray", "java.util.List>>"); + put("intAr", "java.util.List"); }}; Map vs14TestCollectionsCharSeqFieldToType = new LinkedHashMap() {{ @@ -1369,6 +1370,7 @@ private Object[][] testStringTypeParamsProvider() { put("unionOfMap", "java.util.Map"); put("arOfUnionOfStr", "java.util.List"); put("arOfMapOfUnionOfArray", "java.util.List>>"); + put("intAr", "java.util.List"); }}; return new Object[][]{ @@ -1484,7 +1486,7 @@ public void testRecordWithCharSeqStringTypeForMethods() throws Exception { .setArOfMap(Arrays.asList(mapCharSeq)) .setUnionOfMap(mapCharSeq) .setArOfUnionOfStr(Arrays.asList(str)) - .setArOfMapOfUnionOfArray(Arrays.asList(mapOfList)); + .setArOfMapOfUnionOfArray(Arrays.asList(mapOfList)).setIntAr(Arrays.asList(1, 2, 3)); charseqmethod.TestCollections testCollections = testCollectionsBuilder.build(); @@ -1815,6 +1817,7 @@ public void testNewBuilder() throws Exception { RandomRecordGenerator generator = new RandomRecordGenerator(); TestCollections instance = generator.randomSpecific(TestCollections.class, RecordGenerationConfig.newConfig().withAvoidNulls(true)); TestCollections.Builder builder = TestCollections.newBuilder() + .setIntAr(instance.getIntAr()) .setStr(instance.getStr()) .setStrAr(instance.getStrAr()) .setStrArAr(instance.getStrArAr()) From e9f3fea45eca485a4ea2ef9e86fbe4f9fcede20a Mon Sep 17 00:00:00 2001 From: Uttam Kumar Date: Tue, 13 Feb 2024 11:36:57 -0800 Subject: [PATCH 06/13] Moved View classes to their own declarations from inline --- .../avroutil1/builder/SpecificRecordTest.java | 6 + .../CharSequenceListView.java | 58 ++++ .../CharSequenceMapView.java | 102 ++++++ .../CollectionTransformerUtil.java | 306 ++---------------- .../collectiontransformer/StringListView.java | 59 ++++ .../collectiontransformer/StringMapView.java | 102 ++++++ .../collectiontransformer/Utf8ListView.java | 53 +++ 7 files changed, 415 insertions(+), 271 deletions(-) create mode 100644 helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/CharSequenceListView.java create mode 100644 helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/CharSequenceMapView.java create mode 100644 helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/StringListView.java create mode 100644 helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/StringMapView.java create mode 100644 helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/Utf8ListView.java diff --git a/avro-builder/tests/tests-allavro/src/test/java/com/linkedin/avroutil1/builder/SpecificRecordTest.java b/avro-builder/tests/tests-allavro/src/test/java/com/linkedin/avroutil1/builder/SpecificRecordTest.java index 4d8c4b96b..1e62f352c 100644 --- a/avro-builder/tests/tests-allavro/src/test/java/com/linkedin/avroutil1/builder/SpecificRecordTest.java +++ b/avro-builder/tests/tests-allavro/src/test/java/com/linkedin/avroutil1/builder/SpecificRecordTest.java @@ -1832,6 +1832,7 @@ public void testNewBuilder() throws Exception { compareIndexedRecords(instance, builder.build()); } + @Test public void modifiablePrimitiveCollectionTest() { String tba = "NewElement"; RandomRecordGenerator generator = new RandomRecordGenerator(); @@ -1840,22 +1841,27 @@ public void modifiablePrimitiveCollectionTest() { // array of string instance.getStrAr().add(tba); Assert.assertTrue(instance.getStrAr().contains(tba)); + Assert.assertTrue(instance.strAr.contains(new Utf8(tba))); // union[null, List] instance.getUnionOfArray().add(tba); Assert.assertTrue(instance.getUnionOfArray().contains(tba)); + Assert.assertTrue(instance.unionOfArray.contains(new Utf8(tba))); // array (union[null, string]) instance.getArOfUnionOfStr().add(tba); Assert.assertTrue(instance.getArOfUnionOfStr().contains(tba)); + Assert.assertTrue(instance.arOfUnionOfStr.contains(new Utf8(tba))); // Union (null, Map) instance.getUnionOfMap().put("key1", tba); Assert.assertEquals(tba, instance.getUnionOfMap().get("key1")); + Assert.assertEquals(new Utf8(tba), instance.unionOfMap.get(new Utf8("key1"))); instance.getIntAr().add(Integer.MAX_VALUE); Assert.assertEquals((int) instance.getIntAr().get(instance.getIntAr().size() - 1), Integer.MAX_VALUE); + Assert.assertEquals((int) instance.intAr.get(instance.getIntAr().size() - 1), Integer.MAX_VALUE); } @BeforeClass diff --git a/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/CharSequenceListView.java b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/CharSequenceListView.java new file mode 100644 index 000000000..01eb81704 --- /dev/null +++ b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/CharSequenceListView.java @@ -0,0 +1,58 @@ +/* + * Copyright 2024 LinkedIn Corp. + * Licensed under the BSD 2-Clause License (the "License"). + * See License in the project root for license information. + */ +package com.linkedin.avroutil1.compatibility.collectiontransformer; + +import java.util.AbstractList; +import org.apache.avro.util.Utf8; + + +/** + * View of List to allow get as CharSequence while still allowing put to reflect on the original object. + */ +public class CharSequenceListView extends AbstractList { + private java.util.List utf8List; + + public CharSequenceListView(java.util.List utf8List) { + this.utf8List = utf8List; + } + + @Override + public CharSequence get(int index) { + return String.valueOf(utf8List.get(index)); + } + + @Override + public int size() { + return utf8List.size(); + } + + @Override + public CharSequence set(int index, CharSequence element) { + CharSequence previousValue = String.valueOf(utf8List.get(index)); + utf8List.set(index, new Utf8(element.toString())); + return previousValue; + } + + @Override + public void add(int index, CharSequence element) { + utf8List.add(index, new Utf8(element.toString())); + } + + @Override + public boolean add(CharSequence element) { + return utf8List.add(new Utf8(element.toString())); + } + + @Override + public boolean addAll(int index, java.util.Collection c) { + boolean modified = false; + for (CharSequence element : c) { + utf8List.add(index++, new Utf8(element.toString())); + modified = true; + } + return modified; + } +} diff --git a/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/CharSequenceMapView.java b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/CharSequenceMapView.java new file mode 100644 index 000000000..e6d874afb --- /dev/null +++ b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/CharSequenceMapView.java @@ -0,0 +1,102 @@ +/* + * Copyright 2024 LinkedIn Corp. + * Licensed under the BSD 2-Clause License (the "License"). + * See License in the project root for license information. + */ +package com.linkedin.avroutil1.compatibility.collectiontransformer; + +import java.util.AbstractMap; +import java.util.Collection; +import java.util.Collections; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.avro.util.Utf8; + + +/** + * View of Map to allow get as String while still allowing put to reflect on the original object. + */ +public class CharSequenceMapView extends AbstractMap { + + private Map utf8Map; + + public CharSequenceMapView(Map utf8Map) { + this.utf8Map = utf8Map; + } + + @Override + public Set> entrySet() { + return Collections.unmodifiableSet(utf8Map.entrySet().stream() + .collect(Collectors.toMap( + entry -> (CharSequence) String.valueOf(entry.getKey()), + entry -> (CharSequence) String.valueOf(entry.getValue()) + )) + .entrySet()); + } + + @Override + public CharSequence put(CharSequence key, CharSequence value) { + Utf8 utf8Key = new Utf8(key.toString()); + Utf8 utf8Value = new Utf8(value.toString()); + Utf8 previousValue = utf8Map.put(utf8Key, utf8Value); + return previousValue != null ? (CharSequence) String.valueOf(previousValue) : null; + } + + @Override + public Set keySet() { + return Collections.unmodifiableSet(utf8Map.keySet().stream() + .map(CharSequence::toString) + .collect(Collectors.toSet())); + } + + @Override + public Collection values() { + return Collections.unmodifiableCollection(utf8Map.values().stream() + .map(CharSequence::toString) + .collect(Collectors.toList())); + } + + @Override + public int size() { + return utf8Map.size(); + } + + @Override + public boolean isEmpty() { + return utf8Map.isEmpty(); + } + + @Override + public boolean containsKey(Object key) { + return utf8Map.containsKey(new Utf8(String.valueOf(key))); + } + + @Override + public boolean containsValue(Object value) { + return utf8Map.containsValue(new Utf8(String.valueOf(value))); + } + + @Override + public CharSequence get(Object key) { + Utf8 utf8Key = new Utf8(String.valueOf(key)); + Utf8 utf8Value = utf8Map.get(utf8Key); + return utf8Value != null ? (CharSequence) String.valueOf(utf8Value) : null; + } + + @Override + public CharSequence remove(Object key) { + Utf8 utf8Key = new Utf8(String.valueOf(key)); + Utf8 previousValue = utf8Map.remove(utf8Key); + return previousValue != null ? (CharSequence) String.valueOf(previousValue) : null; + } + + @Override + public void putAll(Map m) { + m.forEach((key, value) -> { + Utf8 utf8Key = new Utf8(String.valueOf(key)); + Utf8 utf8Value = new Utf8(String.valueOf(value)); + utf8Map.put(utf8Key, utf8Value); + }); + } +} diff --git a/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/CollectionTransformerUtil.java b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/CollectionTransformerUtil.java index 55145ed57..3c08aef70 100644 --- a/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/CollectionTransformerUtil.java +++ b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/CollectionTransformerUtil.java @@ -7,14 +7,8 @@ package com.linkedin.avroutil1.compatibility.collectiontransformer; import com.linkedin.avroutil1.compatibility.StringUtils; -import java.util.AbstractList; -import java.util.AbstractMap; -import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; import org.apache.avro.util.Utf8; @@ -27,293 +21,63 @@ public static String getErrorMessageForInstance(Object obj) { : " (an instance of " + obj.getClass().getName() + ")"); } + /** + * Returns a {@StringListView} for the given list of {@Utf8} objects. + * @param utf8List list of {@Utf8} objects + * @return a {@StringListView} for the given list of {@Utf8} objects + */ public static List createStringListView(List utf8List) { - return new AbstractList() { - @Override - public String get(int index) { - return String.valueOf(utf8List.get(index)); - } - - @Override - public int size() { - return utf8List.size(); - } - - @Override - public String set(int index, String element) { - String previousValue = String.valueOf(utf8List.get(index)); - utf8List.set(index, new Utf8(element)); - return previousValue; - } - - @Override - public void add(int index, String element) { - utf8List.add(index, new Utf8(element)); - } - - @Override - public boolean add(String element) { - return utf8List.add(new Utf8(element)); - } - - @Override - public boolean addAll(int index, java.util.Collection c) { - boolean modified = false; - for (String element : c) { - utf8List.add(index++, new Utf8(element)); - modified = true; - } - return modified; - } - }; + return new StringListView(utf8List); } + /** + * Returns a {@CharSequenceListView} for the given list of {@Utf8} objects. + * @param utf8List list of {@Utf8} objects + * @return a {@CharSequenceListView} for the given list of {@Utf8} objects + */ public static List createCharSequenceListView(List utf8List) { - return new AbstractList() { - @Override - public CharSequence get(int index) { - return String.valueOf(utf8List.get(index)); - } - - @Override - public int size() { - return utf8List.size(); - } - - @Override - public CharSequence set(int index, CharSequence element) { - CharSequence previousValue = String.valueOf(utf8List.get(index)); - utf8List.set(index, new Utf8(element.toString())); - return previousValue; - } - - @Override - public void add(int index, CharSequence element) { - utf8List.add(index, new Utf8(element.toString())); - } - - @Override - public boolean add(CharSequence element) { - return utf8List.add(new Utf8(element.toString())); - } - - @Override - public boolean addAll(int index, java.util.Collection c) { - boolean modified = false; - for (CharSequence element : c) { - utf8List.add(index++, new Utf8(element.toString())); - modified = true; - } - return modified; - } - - }; + return new CharSequenceListView(utf8List); } + /** + * Returns a {@Utf8ListView} for the given list of {@Utf8} objects. + * @param utf8List list of {@Utf8} objects + * @return a {@Utf8ListView} for the given list of {@Utf8} objects + */ public static List createUtf8ListView(List utf8List) { - return new AbstractList() { - @Override - public Utf8 get(int index) { - return utf8List.get(index); - } - - @Override - public int size() { - return utf8List.size(); - } - - @Override - public Utf8 set(int index, Utf8 element) { - Utf8 previousValue = utf8List.get(index); - utf8List.set(index, element); - return previousValue; - } - - @Override - public void add(int index, Utf8 element) { - utf8List.add(index, element); - } - - @Override - public boolean add(Utf8 element) { - return utf8List.add(element); - } - - @Override - public boolean addAll(int index, java.util.Collection c) { - return utf8List.addAll(index, c); - } - }; + return new Utf8ListView(utf8List); } - + /** + * Returns a {@StringMapView} for the given map of {@Utf8} objects. + * @param utf8Map map of {@Utf8} objects + * @return a {@StringMapView} for the given map of {@Utf8} objects + */ public static Map createStringMapView(Map utf8Map) { if (utf8Map == null) { return null; } - - return new AbstractMap() { - @Override - public Set> entrySet() { - return Collections.unmodifiableSet(utf8Map.entrySet().stream() - .collect(Collectors.toMap( - entry -> String.valueOf(entry.getKey()), - entry -> String.valueOf(entry.getValue()) - )) - .entrySet()); - } - - @Override - public String put(String key, String value) { - Utf8 utf8Key = new Utf8(key); - Utf8 utf8Value = new Utf8(value); - Utf8 previousValue = utf8Map.put(utf8Key, utf8Value); - return previousValue != null ? String.valueOf(previousValue) : null; - } - - @Override - public Set keySet() { - return Collections.unmodifiableSet(utf8Map.keySet().stream() - .map(String::valueOf) - .collect(Collectors.toSet())); - } - - @Override - public Collection values() { - return Collections.unmodifiableCollection(utf8Map.values().stream() - .map(String::valueOf) - .collect(Collectors.toList())); - } - - @Override - public int size() { - return utf8Map.size(); - } - - @Override - public boolean isEmpty() { - return utf8Map.isEmpty(); - } - - @Override - public boolean containsKey(Object key) { - return utf8Map.containsKey(new Utf8(String.valueOf(key))); - } - - @Override - public boolean containsValue(Object value) { - return utf8Map.containsValue(new Utf8(String.valueOf(value))); - } - - @Override - public String get(Object key) { - Utf8 utf8Key = new Utf8(String.valueOf(key)); - Utf8 utf8Value = utf8Map.get(utf8Key); - return utf8Value != null ? String.valueOf(utf8Value) : null; - } - - @Override - public String remove(Object key) { - Utf8 utf8Key = new Utf8(String.valueOf(key)); - Utf8 previousValue = utf8Map.remove(utf8Key); - return previousValue != null ? String.valueOf(previousValue) : null; - } - - @Override - public void putAll(Map m) { - m.forEach((key, value) -> { - Utf8 utf8Key = new Utf8(key); - Utf8 utf8Value = new Utf8(value); - utf8Map.put(utf8Key, utf8Value); - }); - } - }; + return new StringMapView(utf8Map); } + /** + * Returns a {@CharSequenceMapView} for the given map of {@Utf8} objects. + * @param utf8Map map of {@Utf8} objects + * @return a {@CharSequenceMapView} for the given map of {@Utf8} objects + */ public static Map createUtf8MapView(Map utf8Map) { return utf8Map; } + /** + * Returns a {@CharSequenceMapView} for the given map of {@Utf8} objects. + * @param utf8Map map of {@Utf8} objects + * @return a {@CharSequenceMapView} for the given map of {@Utf8} objects + */ public static Map createCharSequenceMapView(Map utf8Map) { if (utf8Map == null) { return null; } - - return new AbstractMap() { - @Override - public Set> entrySet() { - return Collections.unmodifiableSet(utf8Map.entrySet().stream() - .collect(Collectors.toMap( - entry -> (CharSequence) String.valueOf(entry.getKey()), - entry -> (CharSequence) String.valueOf(entry.getValue()) - )) - .entrySet()); - } - - @Override - public CharSequence put(CharSequence key, CharSequence value) { - Utf8 utf8Key = new Utf8(key.toString()); - Utf8 utf8Value = new Utf8(value.toString()); - Utf8 previousValue = utf8Map.put(utf8Key, utf8Value); - return previousValue != null ? (CharSequence) String.valueOf(previousValue) : null; - } - - @Override - public Set keySet() { - return Collections.unmodifiableSet(utf8Map.keySet().stream() - .map(CharSequence::toString) - .collect(Collectors.toSet())); - } - - @Override - public Collection values() { - return Collections.unmodifiableCollection(utf8Map.values().stream() - .map(CharSequence::toString) - .collect(Collectors.toList())); - } - - @Override - public int size() { - return utf8Map.size(); - } - - @Override - public boolean isEmpty() { - return utf8Map.isEmpty(); - } - - @Override - public boolean containsKey(Object key) { - return utf8Map.containsKey(new Utf8(String.valueOf(key))); - } - - @Override - public boolean containsValue(Object value) { - return utf8Map.containsValue(new Utf8(String.valueOf(value))); - } - - @Override - public CharSequence get(Object key) { - Utf8 utf8Key = new Utf8(String.valueOf(key)); - Utf8 utf8Value = utf8Map.get(utf8Key); - return utf8Value != null ? (CharSequence) String.valueOf(utf8Value) : null; - } - - @Override - public CharSequence remove(Object key) { - Utf8 utf8Key = new Utf8(String.valueOf(key)); - Utf8 previousValue = utf8Map.remove(utf8Key); - return previousValue != null ? (CharSequence) String.valueOf(previousValue) : null; - } - - @Override - public void putAll(Map m) { - m.forEach((key, value) -> { - Utf8 utf8Key = new Utf8(String.valueOf(key)); - Utf8 utf8Value = new Utf8(String.valueOf(value)); - utf8Map.put(utf8Key, utf8Value); - }); - } - }; + return new CharSequenceMapView(utf8Map); } } diff --git a/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/StringListView.java b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/StringListView.java new file mode 100644 index 000000000..b0ee15154 --- /dev/null +++ b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/StringListView.java @@ -0,0 +1,59 @@ +/* + * Copyright 2024 LinkedIn Corp. + * Licensed under the BSD 2-Clause License (the "License"). + * See License in the project root for license information. + */ +package com.linkedin.avroutil1.compatibility.collectiontransformer; + +import java.util.AbstractList; +import org.apache.avro.util.Utf8; + + +/** + * View of List to allow get as String while still allowing set to reflect on the original object. + */ +public class StringListView extends AbstractList { + // Not final to allow addition + private java.util.List _utf8List; + + public StringListView(java.util.List utf8List) { + this._utf8List = utf8List; + } + + @Override + public String get(int index) { + return String.valueOf(_utf8List.get(index)); + } + + @Override + public int size() { + return _utf8List.size(); + } + + @Override + public String set(int index, String element) { + String previousValue = String.valueOf(_utf8List.get(index)); + _utf8List.set(index, new Utf8(element)); + return previousValue; + } + + @Override + public void add(int index, String element) { + _utf8List.add(index, new Utf8(element)); + } + + @Override + public boolean add(String element) { + return _utf8List.add(new Utf8(element)); + } + + @Override + public boolean addAll(int index, java.util.Collection c) { + boolean modified = false; + for (String element : c) { + _utf8List.add(index++, new Utf8(element)); + modified = true; + } + return modified; + } +} diff --git a/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/StringMapView.java b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/StringMapView.java new file mode 100644 index 000000000..f2838d829 --- /dev/null +++ b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/StringMapView.java @@ -0,0 +1,102 @@ +/* + * Copyright 2024 LinkedIn Corp. + * Licensed under the BSD 2-Clause License (the "License"). + * See License in the project root for license information. + */ +package com.linkedin.avroutil1.compatibility.collectiontransformer; + +import java.util.AbstractMap; +import java.util.Collection; +import java.util.Collections; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.avro.util.Utf8; + + +/** + * View of Map to allow get as String while still allowing put to reflect on the original object. + */ +public class StringMapView extends AbstractMap { + + private Map utf8Map; + + public StringMapView(Map utf8Map) { + this.utf8Map = utf8Map; + } + + @Override + public Set> entrySet() { + return Collections.unmodifiableSet(utf8Map.entrySet().stream() + .collect(Collectors.toMap( + entry -> String.valueOf(entry.getKey()), + entry -> String.valueOf(entry.getValue()) + )) + .entrySet()); + } + + @Override + public String put(String key, String value) { + Utf8 utf8Key = new Utf8(key); + Utf8 utf8Value = new Utf8(value); + Utf8 previousValue = utf8Map.put(utf8Key, utf8Value); + return previousValue != null ? String.valueOf(previousValue) : null; + } + + @Override + public Set keySet() { + return Collections.unmodifiableSet(utf8Map.keySet().stream() + .map(String::valueOf) + .collect(Collectors.toSet())); + } + + @Override + public Collection values() { + return Collections.unmodifiableCollection(utf8Map.values().stream() + .map(String::valueOf) + .collect(Collectors.toList())); + } + + @Override + public int size() { + return utf8Map.size(); + } + + @Override + public boolean isEmpty() { + return utf8Map.isEmpty(); + } + + @Override + public boolean containsKey(Object key) { + return utf8Map.containsKey(new Utf8(String.valueOf(key))); + } + + @Override + public boolean containsValue(Object value) { + return utf8Map.containsValue(new Utf8(String.valueOf(value))); + } + + @Override + public String get(Object key) { + Utf8 utf8Key = new Utf8(String.valueOf(key)); + Utf8 utf8Value = utf8Map.get(utf8Key); + return utf8Value != null ? String.valueOf(utf8Value) : null; + } + + @Override + public String remove(Object key) { + Utf8 utf8Key = new Utf8(String.valueOf(key)); + Utf8 previousValue = utf8Map.remove(utf8Key); + return previousValue != null ? String.valueOf(previousValue) : null; + } + + @Override + public void putAll(Map m) { + m.forEach((key, value) -> { + Utf8 utf8Key = new Utf8(key); + Utf8 utf8Value = new Utf8(value); + utf8Map.put(utf8Key, utf8Value); + }); + } +} diff --git a/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/Utf8ListView.java b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/Utf8ListView.java new file mode 100644 index 000000000..e7c1fd47c --- /dev/null +++ b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/Utf8ListView.java @@ -0,0 +1,53 @@ +/* + * Copyright 2024 LinkedIn Corp. + * Licensed under the BSD 2-Clause License (the "License"). + * See License in the project root for license information. + */ +package com.linkedin.avroutil1.compatibility.collectiontransformer; + +import java.util.AbstractList; +import org.apache.avro.util.Utf8; + + +/** + * View of List to allow get as Utf8 while still allowing set to reflect on the original object. + */ +public class Utf8ListView extends AbstractList { + private java.util.List utf8List; + + public Utf8ListView(java.util.List utf8List) { + this.utf8List = utf8List; + } + + @Override + public Utf8 get(int index) { + return utf8List.get(index); + } + + @Override + public int size() { + return utf8List.size(); + } + + @Override + public Utf8 set(int index, Utf8 element) { + Utf8 previousValue = utf8List.get(index); + utf8List.set(index, element); + return previousValue; + } + + @Override + public void add(int index, Utf8 element) { + utf8List.add(index, element); + } + + @Override + public boolean add(Utf8 element) { + return utf8List.add(element); + } + + @Override + public boolean addAll(int index, java.util.Collection c) { + return utf8List.addAll(index, c); + } +} From c4bb0e21a5c3c4b34e131907a1940cc02013c72a Mon Sep 17 00:00:00 2001 From: Uttam Kumar Date: Tue, 13 Feb 2024 11:44:01 -0800 Subject: [PATCH 07/13] Update javadoc --- .../CharSequenceListView.java | 2 +- .../CharSequenceMapView.java | 2 +- .../CollectionTransformerUtil.java | 36 +++++++++---------- .../collectiontransformer/StringListView.java | 2 +- .../collectiontransformer/StringMapView.java | 2 +- .../collectiontransformer/Utf8ListView.java | 2 +- 6 files changed, 23 insertions(+), 23 deletions(-) diff --git a/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/CharSequenceListView.java b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/CharSequenceListView.java index 01eb81704..dfc1e9208 100644 --- a/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/CharSequenceListView.java +++ b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/CharSequenceListView.java @@ -10,7 +10,7 @@ /** - * View of List to allow get as CharSequence while still allowing put to reflect on the original object. + * View of Utf8 List to allow get as CharSequence while still allowing put to reflect on the original object. */ public class CharSequenceListView extends AbstractList { private java.util.List utf8List; diff --git a/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/CharSequenceMapView.java b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/CharSequenceMapView.java index e6d874afb..eb73cb13a 100644 --- a/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/CharSequenceMapView.java +++ b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/CharSequenceMapView.java @@ -15,7 +15,7 @@ /** - * View of Map to allow get as String while still allowing put to reflect on the original object. + * View of Utf8 Map to allow get as String while still allowing put to reflect on the original object. */ public class CharSequenceMapView extends AbstractMap { diff --git a/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/CollectionTransformerUtil.java b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/CollectionTransformerUtil.java index 3c08aef70..39b022478 100644 --- a/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/CollectionTransformerUtil.java +++ b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/CollectionTransformerUtil.java @@ -22,36 +22,36 @@ public static String getErrorMessageForInstance(Object obj) { } /** - * Returns a {@StringListView} for the given list of {@Utf8} objects. - * @param utf8List list of {@Utf8} objects - * @return a {@StringListView} for the given list of {@Utf8} objects + * Returns a {@link StringListView} for the given list of {@link Utf8} objects. + * @param utf8List list of {@link Utf8} objects + * @return a {@link StringListView} for the given list of {@link Utf8} objects */ public static List createStringListView(List utf8List) { return new StringListView(utf8List); } /** - * Returns a {@CharSequenceListView} for the given list of {@Utf8} objects. - * @param utf8List list of {@Utf8} objects - * @return a {@CharSequenceListView} for the given list of {@Utf8} objects + * Returns a {@link CharSequenceListView} for the given list of {@link Utf8} objects. + * @param utf8List list of {@link Utf8} objects + * @return a {@link CharSequenceListView} for the given list of {@link Utf8} objects */ public static List createCharSequenceListView(List utf8List) { return new CharSequenceListView(utf8List); } /** - * Returns a {@Utf8ListView} for the given list of {@Utf8} objects. - * @param utf8List list of {@Utf8} objects - * @return a {@Utf8ListView} for the given list of {@Utf8} objects + * Returns a {@link Utf8ListView} for the given list of {@link Utf8} objects. + * @param utf8List list of {@link Utf8} objects + * @return a {@link Utf8ListView} for the given list of {@link Utf8} objects */ public static List createUtf8ListView(List utf8List) { return new Utf8ListView(utf8List); } /** - * Returns a {@StringMapView} for the given map of {@Utf8} objects. - * @param utf8Map map of {@Utf8} objects - * @return a {@StringMapView} for the given map of {@Utf8} objects + * Returns a {@link StringMapView} for the given map of {@link Utf8} objects. + * @param utf8Map map of {@link Utf8} objects + * @return a {@link StringMapView} for the given map of {@link Utf8} objects */ public static Map createStringMapView(Map utf8Map) { if (utf8Map == null) { @@ -61,18 +61,18 @@ public static Map createStringMapView(Map utf8Map) { } /** - * Returns a {@CharSequenceMapView} for the given map of {@Utf8} objects. - * @param utf8Map map of {@Utf8} objects - * @return a {@CharSequenceMapView} for the given map of {@Utf8} objects + * Returns a {@link CharSequenceMapView} for the given map of {@link Utf8} objects. + * @param utf8Map map of {@link Utf8} objects + * @return a {@link CharSequenceMapView} for the given map of {@link Utf8} objects */ public static Map createUtf8MapView(Map utf8Map) { return utf8Map; } /** - * Returns a {@CharSequenceMapView} for the given map of {@Utf8} objects. - * @param utf8Map map of {@Utf8} objects - * @return a {@CharSequenceMapView} for the given map of {@Utf8} objects + * Returns a {@link CharSequenceMapView} for the given map of {@link Utf8} objects. + * @param utf8Map map of {@link Utf8} objects + * @return a {@link CharSequenceMapView} for the given map of {@link Utf8} objects */ public static Map createCharSequenceMapView(Map utf8Map) { if (utf8Map == null) { diff --git a/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/StringListView.java b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/StringListView.java index b0ee15154..0c6301312 100644 --- a/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/StringListView.java +++ b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/StringListView.java @@ -10,7 +10,7 @@ /** - * View of List to allow get as String while still allowing set to reflect on the original object. + * View of Utf8 List to allow get as String while still allowing set to reflect on the original object. */ public class StringListView extends AbstractList { // Not final to allow addition diff --git a/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/StringMapView.java b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/StringMapView.java index f2838d829..19c0a500c 100644 --- a/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/StringMapView.java +++ b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/StringMapView.java @@ -15,7 +15,7 @@ /** - * View of Map to allow get as String while still allowing put to reflect on the original object. + * View of Utf8 Map to allow get as String while still allowing put to reflect on the original object. */ public class StringMapView extends AbstractMap { diff --git a/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/Utf8ListView.java b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/Utf8ListView.java index e7c1fd47c..f822a753e 100644 --- a/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/Utf8ListView.java +++ b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/Utf8ListView.java @@ -10,7 +10,7 @@ /** - * View of List to allow get as Utf8 while still allowing set to reflect on the original object. + * View of Utf8 List to allow get as Utf8 while still allowing set to reflect on the original object. */ public class Utf8ListView extends AbstractList { private java.util.List utf8List; From 9b7ae89c1875a842a9a5b26b7072e85915b329a8 Mon Sep 17 00:00:00 2001 From: Uttam Kumar Date: Tue, 13 Feb 2024 15:23:48 -0800 Subject: [PATCH 08/13] Added more unit tests --- .../avroutil1/builder/SpecificRecordTest.java | 64 +++++ .../CollectionViewTest.java | 240 ++++++++++++++++++ 2 files changed, 304 insertions(+) create mode 100644 avro-builder/tests/tests-allavro/src/test/java/com/linkedin/avroutil1/compatibility/collectiontransformer/CollectionViewTest.java diff --git a/avro-builder/tests/tests-allavro/src/test/java/com/linkedin/avroutil1/builder/SpecificRecordTest.java b/avro-builder/tests/tests-allavro/src/test/java/com/linkedin/avroutil1/builder/SpecificRecordTest.java index 1e62f352c..ebb0d3d97 100644 --- a/avro-builder/tests/tests-allavro/src/test/java/com/linkedin/avroutil1/builder/SpecificRecordTest.java +++ b/avro-builder/tests/tests-allavro/src/test/java/com/linkedin/avroutil1/builder/SpecificRecordTest.java @@ -1864,6 +1864,70 @@ public void modifiablePrimitiveCollectionTest() { Assert.assertEquals((int) instance.intAr.get(instance.getIntAr().size() - 1), Integer.MAX_VALUE); } + @Test + public void modifiablePrimitiveCollectionTestForCharSeq() { + String tba = "NewElement"; + RandomRecordGenerator generator = new RandomRecordGenerator(); + charseqmethod.TestCollections instance = generator.randomSpecific(charseqmethod.TestCollections.class, RecordGenerationConfig.newConfig().withAvoidNulls(true)); + + // array of string + instance.getStrAr().add(tba); + Assert.assertTrue(instance.getStrAr().contains(tba)); + Assert.assertTrue(instance.strAr.contains(new Utf8(tba))); + + // union[null, List] + instance.getUnionOfArray().add(tba); + Assert.assertTrue(instance.getUnionOfArray().contains(tba)); + Assert.assertTrue(instance.unionOfArray.contains(new Utf8(tba))); + + // array (union[null, string]) + instance.getArOfUnionOfStr().add(tba); + Assert.assertTrue(instance.getArOfUnionOfStr().contains(tba)); + Assert.assertTrue(instance.arOfUnionOfStr.contains(new Utf8(tba))); + + + // Union (null, Map) + instance.getUnionOfMap().put("key1", tba); + Assert.assertEquals(tba, instance.getUnionOfMap().get("key1")); + Assert.assertEquals(new Utf8(tba), instance.unionOfMap.get(new Utf8("key1"))); + + instance.getIntAr().add(Integer.MAX_VALUE); + Assert.assertEquals((int) instance.getIntAr().get(instance.getIntAr().size() - 1), Integer.MAX_VALUE); + Assert.assertEquals((int) instance.intAr.get(instance.getIntAr().size() - 1), Integer.MAX_VALUE); + } + + @Test + public void testCharSeqAccessorForNoUtf8() { + String tba = "NewElement"; + RandomRecordGenerator generator = new RandomRecordGenerator(); + noutf8.TestCollections instance = generator.randomSpecific(noutf8.TestCollections.class, RecordGenerationConfig.newConfig().withAvoidNulls(true)); + + // array of string + instance.getStrAr().add(tba); + Assert.assertTrue(instance.getStrAr().contains(tba)); + Assert.assertTrue(instance.strAr.contains(new Utf8(tba))); + + // union[null, List] + instance.getUnionOfArray().add(tba); + Assert.assertTrue(instance.getUnionOfArray().contains(tba)); + Assert.assertTrue(instance.unionOfArray.contains(new Utf8(tba))); + + // array (union[null, string]) + instance.getArOfUnionOfStr().add(tba); + Assert.assertTrue(instance.getArOfUnionOfStr().contains(tba)); + Assert.assertTrue(instance.arOfUnionOfStr.contains(new Utf8(tba))); + + + // Union (null, Map) + instance.getUnionOfMap().put("key1", tba); + Assert.assertEquals(tba, instance.getUnionOfMap().get("key1")); + Assert.assertEquals(new Utf8(tba), instance.unionOfMap.get(new Utf8("key1"))); + + instance.getIntAr().add(Integer.MAX_VALUE); + Assert.assertEquals((int) instance.getIntAr().get(instance.getIntAr().size() - 1), Integer.MAX_VALUE); + Assert.assertEquals((int) instance.intAr.get(instance.getIntAr().size() - 1), Integer.MAX_VALUE); + } + @BeforeClass public void setup() { System.setProperty("org.apache.avro.specific.use_custom_coders", "true"); diff --git a/avro-builder/tests/tests-allavro/src/test/java/com/linkedin/avroutil1/compatibility/collectiontransformer/CollectionViewTest.java b/avro-builder/tests/tests-allavro/src/test/java/com/linkedin/avroutil1/compatibility/collectiontransformer/CollectionViewTest.java new file mode 100644 index 000000000..d87860142 --- /dev/null +++ b/avro-builder/tests/tests-allavro/src/test/java/com/linkedin/avroutil1/compatibility/collectiontransformer/CollectionViewTest.java @@ -0,0 +1,240 @@ +package com.linkedin.avroutil1.compatibility.collectiontransformer; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.avro.util.Utf8; +import org.testng.Assert; +import org.testng.annotations.BeforeTest; +import org.testng.annotations.Test; + + +public class CollectionViewTest { + + List utf8List; + List recordList; + + Map utf8Map; + Map recordMap; + + @BeforeTest + public void testCollectionView(Class clazz) { + + utf8List = new ArrayList<>(); + recordList = new ArrayList<>(); + + utf8Map = new HashMap<>(); + recordMap = new HashMap<>(); + } + + @Test + public void testStringListView() { + String element = "test"; + List listOfElements = Arrays.asList("test", "test2", "test3"); + List view = CollectionTransformerUtil.createStringListView(utf8List); + // utf8 list is empty + Assert.assertEquals(utf8List.size(), 0); + // view should be empty + Assert.assertEquals(view.size(), 0); + + // add a string to the view + view.add(element); + // view should have 1 element + Assert.assertTrue(view.contains(element)); + // utf8 list should contain the same element + Assert.assertTrue(utf8List.contains(new Utf8(element))); + + // remove the element from the view + view.remove(element); + // view should be empty + Assert.assertEquals(view.size(), 0); + // utf8 list should be empty + Assert.assertEquals(utf8List.size(), 0); + + // add a list of elements to the view + view.addAll(listOfElements); + // view should have 3 elements + for (String s : listOfElements) { + Assert.assertTrue(view.contains(s)); + } + // utf8 list should contain the same 3 elements + for (String s : listOfElements) { + Assert.assertTrue(utf8List.contains(new Utf8(s))); + } + } + + @Test + public void testUtf8ListView() { + Utf8 element = new Utf8("test"); + List listOfElements = Arrays.asList(new Utf8("test"), new Utf8("test2"), new Utf8("test3")); + List view = CollectionTransformerUtil.createUtf8ListView(utf8List); + // utf8 list is empty + Assert.assertEquals(utf8List.size(), 0); + // view should be empty + Assert.assertEquals(view.size(), 0); + + // add a utf8 to the view + view.add(element); + // view should have 1 element + Assert.assertTrue(view.contains(element)); + // utf8 list should contain the same element + Assert.assertTrue(utf8List.contains(element)); + + // remove the element from the view + view.remove(element); + // view should be empty + Assert.assertEquals(view.size(), 0); + // utf8 list should be empty + Assert.assertEquals(utf8List.size(), 0); + + // add a list of elements to the view + view.addAll(listOfElements); + // view should have 3 elements + for (Utf8 u : listOfElements) { + Assert.assertTrue(view.contains(u)); + } + // utf8 list should contain the same 3 elements + for (Utf8 u : listOfElements) { + Assert.assertTrue(utf8List.contains(u)); + } + } + + @Test + public void testCharSequenceListView() { + Utf8 element = new Utf8("test"); + List listOfElements = Arrays.asList(new Utf8("test"), new Utf8("test2"), new Utf8("test3")); + List view = CollectionTransformerUtil.createCharSequenceListView(utf8List); + // utf8 list is empty + Assert.assertEquals(utf8List.size(), 0); + // view should be empty + Assert.assertEquals(view.size(), 0); + + // add a utf8 to the view + view.add(element); + // view should have 1 element + Assert.assertTrue(view.contains(element)); + // utf8 list should contain the same element + Assert.assertTrue(utf8List.contains(element)); + + // remove the element from the view + view.remove(element); + // view should be empty + Assert.assertEquals(view.size(), 0); + // utf8 list should be empty + Assert.assertEquals(utf8List.size(), 0); + + // add a list of elements to the view + view.addAll(listOfElements); + // view should have 3 elements + for (Utf8 u : listOfElements) { + Assert.assertTrue(view.contains(u)); + } + // utf8 list should contain the same 3 elements + for (Utf8 u : listOfElements) { + Assert.assertTrue(utf8List.contains(u)); + } + } + + @Test + public void testStringMapView() { + List keys = Arrays.asList("key1", "key2", "key3"); + String val = "value"; + + Map map = CollectionTransformerUtil.createStringMapView(utf8Map); + + // utf8 map is empty + Assert.assertEquals(utf8Map.size(), 0); + // view should be empty + Assert.assertEquals(map.size(), 0); + + // insert in view + for (String key : keys) { + map.put(key, key + val); + } + + // view should have 3 elements + for (String key : keys) { + Assert.assertTrue(map.containsKey(key)); + } + // utf8 map should contain the same 3 elements + for (String key : keys) { + Assert.assertTrue(utf8Map.containsKey(new Utf8(key))); + } + + // remove from view + for (String key : keys) { + map.remove(key); + } + // view should be empty + Assert.assertEquals(map.size(), 0); + } + + @Test + public void testUtf8MapView() { + List keys = Arrays.asList(new Utf8("key1"), new Utf8("key2"), new Utf8("key3")); + Utf8 val = new Utf8("value"); + + Map map = CollectionTransformerUtil.createUtf8MapView(utf8Map); + // utf8 map is empty + Assert.assertEquals(utf8Map.size(), 0); + // view should be empty + Assert.assertEquals(map.size(), 0); + + // insert in view + for (Utf8 key : keys) { + map.put(key, val); + } + + // view should have 3 elements + for (Utf8 key : keys) { + Assert.assertTrue(map.containsKey(key)); + } + // utf8 map should contain the same 3 elements + for (Utf8 key : keys) { + Assert.assertTrue(utf8Map.containsKey(key)); + } + + // remove from view + for (Utf8 key : keys) { + map.remove(key); + } + // view should be empty + Assert.assertEquals(map.size(), 0); + } + + @Test + public void testCharSequenceMapView() { + List keys = Arrays.asList("key1", "key2", "key3"); + String val = "value"; + + Map map = CollectionTransformerUtil.createCharSequenceMapView(utf8Map); + + // utf8 map is empty + Assert.assertEquals(utf8Map.size(), 0); + // view should be empty + Assert.assertEquals(map.size(), 0); + + // insert in view + for (CharSequence key : keys) { + map.put(key, key + val); + } + + // view should have 3 elements + for (CharSequence key : keys) { + Assert.assertTrue(map.containsKey(key)); + } + // utf8 map should contain the same 3 elements + for (CharSequence key : keys) { + Assert.assertTrue(utf8Map.containsKey(new Utf8(key))); + } + + // remove from view + for (CharSequence key : keys) { + map.remove(key); + } + // view should be empty + Assert.assertEquals(map.size(), 0); + } +} From 243ced707a90ca9d57b80fa7fe017d68ab9e6353 Mon Sep 17 00:00:00 2001 From: Uttam Kumar Date: Tue, 13 Feb 2024 15:24:19 -0800 Subject: [PATCH 09/13] During set operations, do not return unmodifiablw --- .../codegen/SpecificRecordClassGenerator.java | 64 +++++++++---------- .../ListTransformer.java | 58 +++++++++++++++++ .../collectiontransformer/MapTransformer.java | 58 +++++++++++++++++ 3 files changed, 148 insertions(+), 32 deletions(-) diff --git a/avro-codegen/src/main/java/com/linkedin/avroutil1/codegen/SpecificRecordClassGenerator.java b/avro-codegen/src/main/java/com/linkedin/avroutil1/codegen/SpecificRecordClassGenerator.java index 22177cfe4..f5fc58451 100644 --- a/avro-codegen/src/main/java/com/linkedin/avroutil1/codegen/SpecificRecordClassGenerator.java +++ b/avro-codegen/src/main/java/com/linkedin/avroutil1/codegen/SpecificRecordClassGenerator.java @@ -555,11 +555,11 @@ private void addAllArgsConstructor(AvroRecordSchema recordSchema, escapedFieldName); } else if (SpecificRecordGeneratorUtil.isListTransformerApplicableForSchema(field.getSchema())) { allArgsConstructorBuilder.addStatement( - "this.$1L = com.linkedin.avroutil1.compatibility.collectiontransformer.ListTransformer.getUtf8List($1L)", + "this.$1L = com.linkedin.avroutil1.compatibility.collectiontransformer.ListTransformer.convertToUtf8($1L)", escapedFieldName); } else if (SpecificRecordGeneratorUtil.isMapTransformerApplicable(field.getSchema())) { allArgsConstructorBuilder.addStatement( - "this.$1L = com.linkedin.avroutil1.compatibility.collectiontransformer.MapTransformer.getUtf8Map($1L)", + "this.$1L = com.linkedin.avroutil1.compatibility.collectiontransformer.MapTransformer.convertToUtf8($1L)", escapedFieldName); } else if (field.getSchema() != null && AvroType.UNION.equals(field.getSchema().type()) && !SpecificRecordGeneratorUtil.isSingleTypeNullableUnionSchema(field.getSchema())) { @@ -580,13 +580,13 @@ private void addAllArgsConstructor(AvroRecordSchema recordSchema, unionMemberSchema.getSchema())) { allArgsConstructorBuilder.beginControlFlow("else if($1L instanceof $2T)", escapedFieldName, List.class) .addStatement( - "this.$1L = com.linkedin.avroutil1.compatibility.collectiontransformer.ListTransformer.getUtf8List($1L)", + "this.$1L = com.linkedin.avroutil1.compatibility.collectiontransformer.ListTransformer.convertToUtf8($1L)", escapedFieldName) .endControlFlow(); } else if (SpecificRecordGeneratorUtil.isMapTransformerApplicable(unionMemberSchema.getSchema())) { allArgsConstructorBuilder.beginControlFlow("else if($1L instanceof $2T)", escapedFieldName, Map.class) .addStatement( - "this.$1L = com.linkedin.avroutil1.compatibility.collectiontransformer.MapTransformer.getUtf8Map($1L)", + "this.$1L = com.linkedin.avroutil1.compatibility.collectiontransformer.MapTransformer.convertToUtf8($1L)", escapedFieldName) .endControlFlow(); } @@ -664,15 +664,15 @@ private void populateBuilderClassBuilder(TypeSpec.Builder recordBuilder, AvroRec if (AvroType.ARRAY.equals(fieldSchema.type()) || SpecificRecordGeneratorUtil.isNullUnionOf(AvroType.ARRAY, field.getSchema())) { buildMethodCodeBlockBuilder.addStatement( "record.$1L = fieldSetFlags()[$2L] ? " - + "com.linkedin.avroutil1.compatibility.collectiontransformer.ListTransformer.getUtf8List(this.$1L) : " - + "($3L) com.linkedin.avroutil1.compatibility.collectiontransformer.ListTransformer.getUtf8List(com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper.getSpecificDefaultValue(fields()[$2L]))", + + "com.linkedin.avroutil1.compatibility.collectiontransformer.ListTransformer.convertToUtf8(this.$1L) : " + + "($3L) com.linkedin.avroutil1.compatibility.collectiontransformer.ListTransformer.convertToUtf8(com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper.getSpecificDefaultValue(fields()[$2L]))", escapedFieldName, fieldIndex, SpecificRecordGeneratorUtil.getTypeName(field.getSchema(), fieldAvroType, true, config.getDefaultFieldStringRepresentation())); } else if (AvroType.MAP.equals(fieldSchema.type()) || SpecificRecordGeneratorUtil.isNullUnionOf(AvroType.MAP, field.getSchema())) { buildMethodCodeBlockBuilder.addStatement( "record.$1L = fieldSetFlags()[$2L] ? " - + "com.linkedin.avroutil1.compatibility.collectiontransformer.MapTransformer.getUtf8Map(this.$1L) : " - + "($3L) com.linkedin.avroutil1.compatibility.collectiontransformer.MapTransformer.getUtf8Map(com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper.getSpecificDefaultValue(fields()[$2L]))", + + "com.linkedin.avroutil1.compatibility.collectiontransformer.MapTransformer.convertToUtf8(this.$1L) : " + + "($3L) com.linkedin.avroutil1.compatibility.collectiontransformer.MapTransformer.convertToUtf8(com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper.getSpecificDefaultValue(fields()[$2L]))", escapedFieldName, fieldIndex, SpecificRecordGeneratorUtil.getTypeName(field.getSchema(), fieldAvroType, true, config.getDefaultFieldStringRepresentation())); } else if (AvroType.UNION.equals(fieldSchema.type())) { @@ -695,8 +695,8 @@ private void populateBuilderClassBuilder(TypeSpec.Builder recordBuilder, AvroRec buildMethodCodeBlockBuilder.beginControlFlow("else if($1L instanceof $2T)", escapedFieldName, List.class) .addStatement( "record.$1L = fieldSetFlags()[$2L] ? " - + "com.linkedin.avroutil1.compatibility.collectiontransformer.ListTransformer.getUtf8List(this.$1L) : " - + "($3L) com.linkedin.avroutil1.compatibility.collectiontransformer.ListTransformer.getUtf8List(com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper.getSpecificDefaultValue(fields()[$2L]))", + + "com.linkedin.avroutil1.compatibility.collectiontransformer.ListTransformer.convertToUtf8(this.$1L) : " + + "($3L) com.linkedin.avroutil1.compatibility.collectiontransformer.ListTransformer.convertToUtf8(com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper.getSpecificDefaultValue(fields()[$2L]))", escapedFieldName, fieldIndex, SpecificRecordGeneratorUtil.getTypeName(field.getSchema(), fieldAvroType, true, config.getDefaultFieldStringRepresentation())) .endControlFlow(); @@ -704,8 +704,8 @@ private void populateBuilderClassBuilder(TypeSpec.Builder recordBuilder, AvroRec buildMethodCodeBlockBuilder.beginControlFlow("else if($1L instanceof $2T)", escapedFieldName, Map.class) .addStatement( "record.$1L = fieldSetFlags()[$2L] ? " - + "com.linkedin.avroutil1.compatibility.collectiontransformer.MapTransformer.getUtf8Map(this.$1L) : " - + "($3L) com.linkedin.avroutil1.compatibility.collectiontransformer.MapTransformer.getUtf8Map(com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper.getSpecificDefaultValue(fields()[$2L]))", + + "com.linkedin.avroutil1.compatibility.collectiontransformer.MapTransformer.convertToUtf8(this.$1L) : " + + "($3L) com.linkedin.avroutil1.compatibility.collectiontransformer.MapTransformer.convertToUtf8(com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper.getSpecificDefaultValue(fields()[$2L]))", escapedFieldName, fieldIndex, SpecificRecordGeneratorUtil.getTypeName(field.getSchema(), fieldAvroType, true, config.getDefaultFieldStringRepresentation())) .endControlFlow(); @@ -1463,13 +1463,13 @@ private void addPutByIndexMethod(TypeSpec.Builder classBuilder, AvroRecordSchema } else if (config.isUtf8EncodingInPutByIndexEnabled() && SpecificRecordGeneratorUtil.isListTransformerApplicableForSchema(field.getSchema())) { if (config.getDefaultFieldStringRepresentation().equals(AvroJavaStringRepresentation.STRING)) { switchBuilder.addStatement( - "case $1L: this.$2L = ($3T) com.linkedin.avroutil1.compatibility.collectiontransformer.ListTransformer.getStringList(value); break", + "case $1L: this.$2L = ($3T) com.linkedin.avroutil1.compatibility.collectiontransformer.ListTransformer.convertToString(value); break", fieldIndex++, escapedFieldName, SpecificRecordGeneratorUtil.getTypeName(field.getSchemaOrRef().getSchema(), field.getSchemaOrRef().getSchema().type(), true, config.getDefaultFieldStringRepresentation())); } else { switchBuilder.addStatement( - "case $1L: this.$2L = ($3T) com.linkedin.avroutil1.compatibility.collectiontransformer.ListTransformer.getUtf8List(value); break", + "case $1L: this.$2L = ($3T) com.linkedin.avroutil1.compatibility.collectiontransformer.ListTransformer.convertToUtf8(value); break", fieldIndex++, escapedFieldName, SpecificRecordGeneratorUtil.getTypeName(field.getSchemaOrRef().getSchema(), field.getSchemaOrRef().getSchema().type(), true, config.getDefaultFieldStringRepresentation())); @@ -1477,12 +1477,12 @@ private void addPutByIndexMethod(TypeSpec.Builder classBuilder, AvroRecordSchema } else if (config.isUtf8EncodingInPutByIndexEnabled() && SpecificRecordGeneratorUtil.isMapTransformerApplicable(field.getSchema())) { if (config.getDefaultFieldStringRepresentation().equals(AvroJavaStringRepresentation.STRING)) { switchBuilder.addStatement( - "case $1L: this.$2L = ($3T) com.linkedin.avroutil1.compatibility.collectiontransformer.MapTransformer.getStringMap(value); break", + "case $1L: this.$2L = ($3T) com.linkedin.avroutil1.compatibility.collectiontransformer.MapTransformer.convertToString(value); break", fieldIndex++, escapedFieldName, SpecificRecordGeneratorUtil.getTypeName(field.getSchemaOrRef().getSchema(), field.getSchemaOrRef().getSchema().type(), true, config.getDefaultFieldStringRepresentation())); } else { switchBuilder.addStatement( - "case $1L: this.$2L = ($3T) com.linkedin.avroutil1.compatibility.collectiontransformer.MapTransformer.getUtf8Map(value); break", + "case $1L: this.$2L = ($3T) com.linkedin.avroutil1.compatibility.collectiontransformer.MapTransformer.convertToUtf8(value); break", fieldIndex++, escapedFieldName, SpecificRecordGeneratorUtil.getTypeName(field.getSchemaOrRef().getSchema(), field.getSchemaOrRef().getSchema().type(), true, config.getDefaultFieldStringRepresentation())); } @@ -1511,12 +1511,12 @@ private void addPutByIndexMethod(TypeSpec.Builder classBuilder, AvroRecordSchema switchBuilder.beginControlFlow("else if($1L instanceof $2T)", escapedFieldName, List.class); if (config.getDefaultFieldStringRepresentation().equals(AvroJavaStringRepresentation.STRING)) { switchBuilder.addStatement( - "this.$1L = ($2T) com.linkedin.avroutil1.compatibility.collectiontransformer.ListTransformer.getStringList(value); break", + "this.$1L = ($2T) com.linkedin.avroutil1.compatibility.collectiontransformer.ListTransformer.convertToString(value); break", escapedFieldName, SpecificRecordGeneratorUtil.getTypeName(field.getSchemaOrRef().getSchema(), field.getSchemaOrRef().getSchema().type(), true, config.getDefaultFieldStringRepresentation())); } else { switchBuilder.addStatement( - "this.$1L = ($2T) com.linkedin.avroutil1.compatibility.collectiontransformer.ListTransformer.getUtf8List(value); break", + "this.$1L = ($2T) com.linkedin.avroutil1.compatibility.collectiontransformer.ListTransformer.convertToUtf8(value); break", escapedFieldName, SpecificRecordGeneratorUtil.getTypeName(field.getSchemaOrRef().getSchema(), field.getSchemaOrRef().getSchema().type(), true, config.getDefaultFieldStringRepresentation())); } @@ -1526,12 +1526,12 @@ private void addPutByIndexMethod(TypeSpec.Builder classBuilder, AvroRecordSchema switchBuilder.beginControlFlow("else if($1L instanceof $2T)", escapedFieldName, Map.class); if (config.getDefaultFieldStringRepresentation().equals(AvroJavaStringRepresentation.STRING)) { switchBuilder.addStatement( - "this.$1L = ($2T) com.linkedin.avroutil1.compatibility.collectiontransformer.MapTransformer.getStringMap(value); break", + "this.$1L = ($2T) com.linkedin.avroutil1.compatibility.collectiontransformer.MapTransformer.convertToString(value); break", escapedFieldName, SpecificRecordGeneratorUtil.getTypeName(field.getSchemaOrRef().getSchema(), field.getSchemaOrRef().getSchema().type(), true, config.getDefaultFieldStringRepresentation())); } else { switchBuilder.addStatement( - "this.$1L = ($2T) com.linkedin.avroutil1.compatibility.collectiontransformer.MapTransformer.getUtf8Map(value); break", + "this.$1L = ($2T) com.linkedin.avroutil1.compatibility.collectiontransformer.MapTransformer.convertToUtf8(value); break", escapedFieldName, SpecificRecordGeneratorUtil.getTypeName(field.getSchemaOrRef().getSchema(), field.getSchemaOrRef().getSchema().type(), true, config.getDefaultFieldStringRepresentation())); } @@ -1717,24 +1717,24 @@ private MethodSpec getSetterMethodSpec(AvroSchemaField field, SpecificRecordGene } else if (SpecificRecordGeneratorUtil.isListTransformerApplicableForSchema(field.getSchema())) { if (config.getDefaultFieldStringRepresentation().equals(AvroJavaStringRepresentation.STRING)) { methodSpecBuilder.addStatement( - "this.$1L = ($2T) com.linkedin.avroutil1.compatibility.collectiontransformer.ListTransformer.getStringList($1L)", + "this.$1L = ($2T) com.linkedin.avroutil1.compatibility.collectiontransformer.ListTransformer.convertToString($1L)", escapedFieldName, SpecificRecordGeneratorUtil.getTypeName(field.getSchemaOrRef().getSchema(), field.getSchemaOrRef().getSchema().type(), true, config.getDefaultFieldStringRepresentation())); } else { methodSpecBuilder.addStatement( - "this.$1L = ($2T) com.linkedin.avroutil1.compatibility.collectiontransformer.ListTransformer.getUtf8List($1L)", + "this.$1L = ($2T) com.linkedin.avroutil1.compatibility.collectiontransformer.ListTransformer.convertToUtf8($1L)", escapedFieldName, SpecificRecordGeneratorUtil.getTypeName(field.getSchemaOrRef().getSchema(), field.getSchemaOrRef().getSchema().type(), true, config.getDefaultFieldStringRepresentation())); } } else if (SpecificRecordGeneratorUtil.isMapTransformerApplicable(field.getSchema())) { if (config.getDefaultFieldStringRepresentation().equals(AvroJavaStringRepresentation.STRING)) { methodSpecBuilder.addStatement( - "this.$1L = ($2T) com.linkedin.avroutil1.compatibility.collectiontransformer.MapTransformer.getStringMap($1L)", + "this.$1L = ($2T) com.linkedin.avroutil1.compatibility.collectiontransformer.MapTransformer.convertToString($1L)", escapedFieldName, SpecificRecordGeneratorUtil.getTypeName(field.getSchemaOrRef().getSchema(), field.getSchemaOrRef().getSchema().type(), true, config.getDefaultFieldStringRepresentation())); } else { methodSpecBuilder.addStatement( - "this.$1L = ($2T) com.linkedin.avroutil1.compatibility.collectiontransformer.MapTransformer.getUtf8Map($1L)", + "this.$1L = ($2T) com.linkedin.avroutil1.compatibility.collectiontransformer.MapTransformer.convertToUtf8($1L)", escapedFieldName, SpecificRecordGeneratorUtil.getTypeName(field.getSchemaOrRef().getSchema(), field.getSchemaOrRef().getSchema().type(), true, config.getDefaultFieldStringRepresentation())); } @@ -1760,12 +1760,12 @@ private MethodSpec getSetterMethodSpec(AvroSchemaField field, SpecificRecordGene methodSpecBuilder.beginControlFlow("else if($1L instanceof $2T)", escapedFieldName, List.class); if (config.getDefaultFieldStringRepresentation().equals(AvroJavaStringRepresentation.STRING)) { methodSpecBuilder.addStatement( - "this.$1L = com.linkedin.avroutil1.compatibility.collectiontransformer.ListTransformer.getStringList($1L, $2L)", - escapedFieldName, SpecificRecordGeneratorUtil.isCollectionSchemaValuePrimitive(field.getSchema())); + "this.$1L = com.linkedin.avroutil1.compatibility.collectiontransformer.ListTransformer.convertToString($1L)", + escapedFieldName); } else { methodSpecBuilder.addStatement( - "this.$1L = com.linkedin.avroutil1.compatibility.collectiontransformer.ListTransformer.getUtf8List($1L, $2L)", - escapedFieldName, SpecificRecordGeneratorUtil.isCollectionSchemaValuePrimitive(field.getSchema())); + "this.$1L = com.linkedin.avroutil1.compatibility.collectiontransformer.ListTransformer.convertToUtf8($1L)", + escapedFieldName); } methodSpecBuilder.endControlFlow(); @@ -1773,12 +1773,12 @@ private MethodSpec getSetterMethodSpec(AvroSchemaField field, SpecificRecordGene methodSpecBuilder.beginControlFlow("else if($1L instanceof $2T)", escapedFieldName, Map.class); if (config.getDefaultFieldStringRepresentation().equals(AvroJavaStringRepresentation.STRING)) { methodSpecBuilder.addStatement( - "this.$1L = com.linkedin.avroutil1.compatibility.collectiontransformer.MapTransformer.getStringMap($1L, $2L)", - escapedFieldName, SpecificRecordGeneratorUtil.isCollectionSchemaValuePrimitive(field.getSchema())); + "this.$1L = com.linkedin.avroutil1.compatibility.collectiontransformer.MapTransformer.convertToString($1L)", + escapedFieldName); } else { methodSpecBuilder.addStatement( - "this.$1L = com.linkedin.avroutil1.compatibility.collectiontransformer.MapTransformer.getUtf8Map($1L, $2L)", - escapedFieldName, SpecificRecordGeneratorUtil.isCollectionSchemaValuePrimitive(field.getSchema())); + "this.$1L = com.linkedin.avroutil1.compatibility.collectiontransformer.MapTransformer.convertToUtf8($1L)", + escapedFieldName); } methodSpecBuilder.endControlFlow(); } diff --git a/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/ListTransformer.java b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/ListTransformer.java index 5601f3aa5..c7e298419 100644 --- a/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/ListTransformer.java +++ b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/ListTransformer.java @@ -73,6 +73,64 @@ private static List getUtf8(Object listObj, boolean isPrimitive) { } } + public static List convertToUtf8(Object listObj) { + if (listObj == null) { + return null; + } + if (listObj instanceof List) { + List list = (List) listObj; + List ret = new ArrayList(list.size()); + // for all items in the list + for (Object item : list) { + // recursively convert to Utf8 if the item is a list or a map + if (item instanceof List) { + ret.add(convertToUtf8((List) item)); + } else if (item instanceof Map) { + ret.add(MapTransformer.convertToUtf8((Map) item)); + // if the item is a CharSequence, convert it to Utf8 + } else if (item instanceof CharSequence) { + ret.add(StringConverterUtil.getUtf8(item)); + } else { + // otherwise, add the item as is + ret.add(item); + } + } + return ret; + } else { + throw new UnsupportedOperationException( + "Supports only Lists. Received" + CollectionTransformerUtil.getErrorMessageForInstance(listObj)); + } + } + + public static List convertToString(Object listObj) { + if (listObj == null) { + return null; + } + if (listObj instanceof List) { + List list = (List) listObj; + List ret = new ArrayList(list.size()); + // for all items in the list + for (Object item : list) { + // recursively convert to String if the item is a list or a map + if (item instanceof List) { + ret.add(convertToString((List) item)); + } else if (item instanceof Map) { + ret.add(MapTransformer.convertToString((Map) item)); + // if the item is a CharSequence, convert it to String + } else if (item instanceof CharSequence) { + ret.add(StringConverterUtil.getString(item)); + } else { + // otherwise, add the item as is + ret.add(item); + } + } + return ret; + } else { + throw new UnsupportedOperationException( + "Supports only Lists. Received" + CollectionTransformerUtil.getErrorMessageForInstance(listObj)); + } + } + private static void flushCharSeqFlag() { hasCharSeq.set(false); } diff --git a/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/MapTransformer.java b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/MapTransformer.java index fb63bee25..ad732d1b7 100644 --- a/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/MapTransformer.java +++ b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/MapTransformer.java @@ -121,4 +121,62 @@ public static Map getCharSequenceMap(Object mapObj, boolean isPrimitiveCollectio } return Collections.unmodifiableMap(ret); } + + public static Map convertToUtf8(Object mapObj) { + if (mapObj == null) { + return null; + } + if (mapObj instanceof Map) { + Map map = (Map) mapObj; + Map ret = new HashMap(map.size()); + // for all elements in the map + for(Object entry : map.entrySet()) { + Object key = ((Map.Entry) entry).getKey(); + Object val = ((Map.Entry) entry).getValue(); + // recursively convert to Utf8 if applicable + if (val instanceof List) { + ret.put(StringConverterUtil.getUtf8(key), ListTransformer.convertToUtf8((List) val)); + } else if (val instanceof Map) { + ret.put(StringConverterUtil.getUtf8(key), MapTransformer.convertToUtf8((Map) val)); + } else if (val instanceof CharSequence) { + ret.put(StringConverterUtil.getUtf8(key), StringConverterUtil.getUtf8(val)); + } else { + ret.put(StringConverterUtil.getUtf8(key), val); + } + } + return ret; + } else { + throw new UnsupportedOperationException( + "Supports only Map. Received" + CollectionTransformerUtil.getErrorMessageForInstance(mapObj)); + } + } + + public static Map convertToString(Object mapObj) { + if (mapObj == null) { + return null; + } + if (mapObj instanceof Map) { + Map map = (Map) mapObj; + Map ret = new HashMap(map.size()); + // for all elements in the map + for(Object entry : map.entrySet()) { + Object key = ((Map.Entry) entry).getKey(); + Object val = ((Map.Entry) entry).getValue(); + // recursively convert to String if applicable + if (val instanceof List) { + ret.put(StringConverterUtil.getString(key), ListTransformer.convertToString((List) val)); + } else if (val instanceof Map) { + ret.put(StringConverterUtil.getString(key), MapTransformer.convertToString((Map) val)); + } else if (val instanceof CharSequence) { + ret.put(StringConverterUtil.getString(key), StringConverterUtil.getString(val)); + } else { + ret.put(StringConverterUtil.getString(key), val); + } + } + return ret; + } else { + throw new UnsupportedOperationException( + "Supports only Map. Received" + CollectionTransformerUtil.getErrorMessageForInstance(mapObj)); + } + } } From 5d7363bab2dc2719fae37d0e6e9309613500a8dd Mon Sep 17 00:00:00 2001 From: Uttam Kumar Date: Wed, 14 Feb 2024 17:46:58 -0800 Subject: [PATCH 10/13] fix typo --- .../compatibility/collectiontransformer/CollectionViewTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/avro-builder/tests/tests-allavro/src/test/java/com/linkedin/avroutil1/compatibility/collectiontransformer/CollectionViewTest.java b/avro-builder/tests/tests-allavro/src/test/java/com/linkedin/avroutil1/compatibility/collectiontransformer/CollectionViewTest.java index d87860142..b3e19c0f5 100644 --- a/avro-builder/tests/tests-allavro/src/test/java/com/linkedin/avroutil1/compatibility/collectiontransformer/CollectionViewTest.java +++ b/avro-builder/tests/tests-allavro/src/test/java/com/linkedin/avroutil1/compatibility/collectiontransformer/CollectionViewTest.java @@ -227,7 +227,7 @@ public void testCharSequenceMapView() { } // utf8 map should contain the same 3 elements for (CharSequence key : keys) { - Assert.assertTrue(utf8Map.containsKey(new Utf8(key))); + Assert.assertTrue(utf8Map.containsKey(new Utf8(String.valueOf(key)))); } // remove from view From 547fc37fe21157d4d60f274553b32ddff221d7b0 Mon Sep 17 00:00:00 2001 From: Uttam Kumar Date: Wed, 14 Feb 2024 18:12:46 -0800 Subject: [PATCH 11/13] test updates --- .../CollectionViewTest.java | 41 ++++++++----------- .../CharSequenceListView.java | 5 +++ .../collectiontransformer/StringListView.java | 5 +++ .../collectiontransformer/Utf8ListView.java | 5 +++ 4 files changed, 32 insertions(+), 24 deletions(-) diff --git a/avro-builder/tests/tests-allavro/src/test/java/com/linkedin/avroutil1/compatibility/collectiontransformer/CollectionViewTest.java b/avro-builder/tests/tests-allavro/src/test/java/com/linkedin/avroutil1/compatibility/collectiontransformer/CollectionViewTest.java index b3e19c0f5..fde87bd20 100644 --- a/avro-builder/tests/tests-allavro/src/test/java/com/linkedin/avroutil1/compatibility/collectiontransformer/CollectionViewTest.java +++ b/avro-builder/tests/tests-allavro/src/test/java/com/linkedin/avroutil1/compatibility/collectiontransformer/CollectionViewTest.java @@ -1,3 +1,8 @@ +/* + * Copyright 2024 LinkedIn Corp. + * Licensed under the BSD 2-Clause License (the "License"). + * See License in the project root for license information. + */ package com.linkedin.avroutil1.compatibility.collectiontransformer; import java.util.ArrayList; @@ -7,30 +12,13 @@ import java.util.Map; import org.apache.avro.util.Utf8; import org.testng.Assert; -import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; public class CollectionViewTest { - - List utf8List; - List recordList; - - Map utf8Map; - Map recordMap; - - @BeforeTest - public void testCollectionView(Class clazz) { - - utf8List = new ArrayList<>(); - recordList = new ArrayList<>(); - - utf8Map = new HashMap<>(); - recordMap = new HashMap<>(); - } - @Test public void testStringListView() { + List utf8List = new ArrayList<>(); String element = "test"; List listOfElements = Arrays.asList("test", "test2", "test3"); List view = CollectionTransformerUtil.createStringListView(utf8List); @@ -67,6 +55,7 @@ public void testStringListView() { @Test public void testUtf8ListView() { + List utf8List = new ArrayList<>(); Utf8 element = new Utf8("test"); List listOfElements = Arrays.asList(new Utf8("test"), new Utf8("test2"), new Utf8("test3")); List view = CollectionTransformerUtil.createUtf8ListView(utf8List); @@ -103,7 +92,8 @@ public void testUtf8ListView() { @Test public void testCharSequenceListView() { - Utf8 element = new Utf8("test"); + List utf8List = new ArrayList<>(); + String element = "test"; List listOfElements = Arrays.asList(new Utf8("test"), new Utf8("test2"), new Utf8("test3")); List view = CollectionTransformerUtil.createCharSequenceListView(utf8List); // utf8 list is empty @@ -111,12 +101,12 @@ public void testCharSequenceListView() { // view should be empty Assert.assertEquals(view.size(), 0); - // add a utf8 to the view + // add to the view view.add(element); // view should have 1 element Assert.assertTrue(view.contains(element)); - // utf8 list should contain the same element - Assert.assertTrue(utf8List.contains(element)); + // utf8 list should contain the same element in Utf8 form + Assert.assertTrue(utf8List.contains(new Utf8(element))); // remove the element from the view view.remove(element); @@ -129,7 +119,7 @@ public void testCharSequenceListView() { view.addAll(listOfElements); // view should have 3 elements for (Utf8 u : listOfElements) { - Assert.assertTrue(view.contains(u)); + Assert.assertTrue(view.contains(String.valueOf(u))); } // utf8 list should contain the same 3 elements for (Utf8 u : listOfElements) { @@ -139,6 +129,7 @@ public void testCharSequenceListView() { @Test public void testStringMapView() { + Map utf8Map = new HashMap<>(); List keys = Arrays.asList("key1", "key2", "key3"); String val = "value"; @@ -163,7 +154,7 @@ public void testStringMapView() { Assert.assertTrue(utf8Map.containsKey(new Utf8(key))); } - // remove from view + // remove from map for (String key : keys) { map.remove(key); } @@ -173,6 +164,7 @@ public void testStringMapView() { @Test public void testUtf8MapView() { + Map utf8Map = new HashMap<>(); List keys = Arrays.asList(new Utf8("key1"), new Utf8("key2"), new Utf8("key3")); Utf8 val = new Utf8("value"); @@ -206,6 +198,7 @@ public void testUtf8MapView() { @Test public void testCharSequenceMapView() { + Map utf8Map = new HashMap<>(); List keys = Arrays.asList("key1", "key2", "key3"); String val = "value"; diff --git a/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/CharSequenceListView.java b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/CharSequenceListView.java index dfc1e9208..000c7dfb1 100644 --- a/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/CharSequenceListView.java +++ b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/CharSequenceListView.java @@ -55,4 +55,9 @@ public boolean addAll(int index, java.util.Collection c) } return modified; } + + @Override + public boolean remove(Object o) { + return utf8List.remove(new Utf8(o.toString())); + } } diff --git a/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/StringListView.java b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/StringListView.java index 0c6301312..8fba353ff 100644 --- a/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/StringListView.java +++ b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/StringListView.java @@ -56,4 +56,9 @@ public boolean addAll(int index, java.util.Collection c) { } return modified; } + + @Override + public boolean remove(Object o) { + return _utf8List.remove(new Utf8(o.toString())); + } } diff --git a/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/Utf8ListView.java b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/Utf8ListView.java index f822a753e..c3724f6bc 100644 --- a/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/Utf8ListView.java +++ b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/collectiontransformer/Utf8ListView.java @@ -50,4 +50,9 @@ public boolean add(Utf8 element) { public boolean addAll(int index, java.util.Collection c) { return utf8List.addAll(index, c); } + + @Override + public boolean remove(Object o) { + return utf8List.remove(o); + } } From e63d984566e4af3f51a8627acfe777635d08b454 Mon Sep 17 00:00:00 2001 From: Uttam Kumar Date: Thu, 15 Feb 2024 14:33:32 -0800 Subject: [PATCH 12/13] fix Unit test --- .../com/linkedin/avroutil1/builder/SpecificRecordTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/avro-builder/tests/tests-allavro/src/test/java/com/linkedin/avroutil1/builder/SpecificRecordTest.java b/avro-builder/tests/tests-allavro/src/test/java/com/linkedin/avroutil1/builder/SpecificRecordTest.java index ebb0d3d97..99ad21b6e 100644 --- a/avro-builder/tests/tests-allavro/src/test/java/com/linkedin/avroutil1/builder/SpecificRecordTest.java +++ b/avro-builder/tests/tests-allavro/src/test/java/com/linkedin/avroutil1/builder/SpecificRecordTest.java @@ -1087,13 +1087,14 @@ public void testSpecificRecordBuilder110(String stringField, String package$, Fl Assert.assertSame(builderTester.get(0), stringField); Assert.assertSame(builderTester.get(1), package$); Assert.assertSame(builderTester.get(6), min); - Assert.assertSame(builderTester.get(7), arrayOfRecord); Assert.assertSame(builderTester.get(10), simpleUnion); Assert.assertSame(builderTester.get(11), fixedType); Assert.assertEquals(builderTester.get(2), exception); Assert.assertEquals(builderTester.get(3), dbl); Assert.assertEquals(builderTester.get(4), isTrue); + // Equal, not same as the Record has string fields + Assert.assertEquals(builderTester.get(7), arrayOfRecord); // Use transformers to return a copy of data assertNotSameIfNotNull(builderTester.get(5), arrayOfStrings); Assert.assertEquals(builderTester.get(5), arrayOfStrings); From eac532635202f6340c695638a4257854b8ea51ec Mon Sep 17 00:00:00 2001 From: Uttam Kumar Date: Thu, 15 Feb 2024 15:46:40 -0800 Subject: [PATCH 13/13] Updated test --- .../avroutil1/builder/SpecificRecordTest.java | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/avro-builder/tests/tests-allavro/src/test/java/com/linkedin/avroutil1/builder/SpecificRecordTest.java b/avro-builder/tests/tests-allavro/src/test/java/com/linkedin/avroutil1/builder/SpecificRecordTest.java index 99ad21b6e..302c7b40c 100644 --- a/avro-builder/tests/tests-allavro/src/test/java/com/linkedin/avroutil1/builder/SpecificRecordTest.java +++ b/avro-builder/tests/tests-allavro/src/test/java/com/linkedin/avroutil1/builder/SpecificRecordTest.java @@ -255,7 +255,9 @@ public void testSpecificRecordBuilder14(String stringField, String package$, Flo Assert.assertSame(builderTester.get(0), stringField); Assert.assertSame(builderTester.get(1), package$); Assert.assertSame(builderTester.get(6), min); - Assert.assertSame(builderTester.get(7), arrayOfRecord); + + // Equal, not same as the Record has string fields + Assert.assertEquals(builderTester.get(7), arrayOfRecord); Assert.assertSame(builderTester.get(10), simpleUnion); Assert.assertSame(builderTester.get(11), fixedType); Assert.assertEquals(builderTester.get(2), exception); @@ -391,7 +393,9 @@ public void testSpecificRecordBuilder15(String stringField, String package$, Flo Assert.assertSame(builderTester.get(0), stringField); Assert.assertSame(builderTester.get(1), package$); Assert.assertSame(builderTester.get(6), min); - Assert.assertSame(builderTester.get(7), arrayOfRecord); + + // Equal, not same as the Record has string fields + Assert.assertEquals(builderTester.get(7), arrayOfRecord); Assert.assertSame(builderTester.get(10), simpleUnion); Assert.assertSame(builderTester.get(11), fixedType); Assert.assertEquals(builderTester.get(2), exception); @@ -528,7 +532,9 @@ public void testSpecificRecordBuilder16(String stringField, String package$, Flo Assert.assertSame(builderTester.get(0), stringField); Assert.assertSame(builderTester.get(1), package$); Assert.assertSame(builderTester.get(6), min); - Assert.assertSame(builderTester.get(7), arrayOfRecord); + + // Equal, not same as the Record has string fields + Assert.assertEquals(builderTester.get(7), arrayOfRecord); Assert.assertSame(builderTester.get(10), simpleUnion); Assert.assertSame(builderTester.get(11), fixedType); Assert.assertEquals(builderTester.get(2), exception); @@ -664,7 +670,9 @@ public void testSpecificRecordBuilder17(String stringField, String package$, Flo Assert.assertSame(builderTester.get(0), stringField); Assert.assertSame(builderTester.get(1), package$); Assert.assertSame(builderTester.get(6), min); - Assert.assertSame(builderTester.get(7), arrayOfRecord); + + // Equal, not same as the Record has string fields + Assert.assertEquals(builderTester.get(7), arrayOfRecord); Assert.assertSame(builderTester.get(10), simpleUnion); Assert.assertSame(builderTester.get(11), fixedType); Assert.assertEquals(builderTester.get(2), exception); @@ -804,7 +812,9 @@ public void testSpecificRecordBuilder18(String stringField, String package$, Flo Assert.assertSame(builderTester.get(0), stringField); Assert.assertSame(builderTester.get(1), package$); Assert.assertSame(builderTester.get(6), min); - Assert.assertSame(builderTester.get(7), arrayOfRecord); + + // Equal, not same as the Record has string fields + Assert.assertEquals(builderTester.get(7), arrayOfRecord); Assert.assertSame(builderTester.get(10), simpleUnion); Assert.assertSame(builderTester.get(11), fixedType); Assert.assertEquals(builderTester.get(2), exception); @@ -946,7 +956,9 @@ public void testSpecificRecordBuilder19(String stringField, String package$, Flo Assert.assertSame(builderTester.get(0), stringField); Assert.assertSame(builderTester.get(1), package$); Assert.assertSame(builderTester.get(6), min); - Assert.assertSame(builderTester.get(7), arrayOfRecord); + + // Equal, not same as the Record has string fields + Assert.assertEquals(builderTester.get(7), arrayOfRecord); Assert.assertSame(builderTester.get(10), simpleUnion); Assert.assertSame(builderTester.get(11), fixedType); Assert.assertEquals(builderTester.get(2), exception); @@ -1224,7 +1236,9 @@ public void testSpecificRecordBuilder111(String stringField, String package$, Fl Assert.assertSame(builderTester.get(0), stringField); Assert.assertSame(builderTester.get(1), package$); Assert.assertSame(builderTester.get(6), min); - Assert.assertSame(builderTester.get(7), arrayOfRecord); + + // Equal, not same as the Record has string fields + Assert.assertEquals(builderTester.get(7), arrayOfRecord); Assert.assertSame(builderTester.get(10), simpleUnion); Assert.assertSame(builderTester.get(11), fixedType); Assert.assertEquals(builderTester.get(2), exception);