support up to avro 1.11.3, align behaviour on content-beyond-end-of-schema with vanilla 1.11.2+

avro-builder/tests/codegen-111/build.gradle

@@ -54,7 +54,7 @@ compileJava.dependsOn runOwnCodegen
 dependencies {
     codegen project(":avro-builder:builder")
 
-    implementation ("org.apache.avro:avro:1.11.1") {
+    implementation ("org.apache.avro:avro:1.11.3") {
         exclude group: "org.mortbay.jetty"
         exclude group: "org.apache.velocity"
         exclude group: "commons-lang"

avro-builder/tests/tests-allavro/build.gradle

@@ -138,10 +138,10 @@ dependencies {
     exclude group: "org.slf4j"
   }
 
-  avro111 ("org.apache.avro:avro:1.11.1") {
+  avro111 ("org.apache.avro:avro:1.11.3") {
     exclude group: "org.slf4j"
   }
-  avro111 ("org.apache.avro:avro-compiler:1.11.1") {
+  avro111 ("org.apache.avro:avro-compiler:1.11.3") {
     exclude group: "org.slf4j"
   }
 }

fastserde/avro-fastserde-tests111/build.gradle

@@ -31,7 +31,7 @@ dependencies {
   testImplementation "org.apache.commons:commons-lang3:3.4"
   testImplementation "com.sun.codemodel:codemodel:2.6"
 
-  testImplementation ("org.apache.avro:avro:1.11.1") {
+  testImplementation ("org.apache.avro:avro:1.11.3") {
     exclude group: "org.slf4j"
   }
 
@@ -43,8 +43,8 @@ dependencies {
   testImplementation 'org.slf4j:slf4j-simple:1.7.14'
 
   codegen project(":helper:helper")
-  codegen "org.apache.avro:avro-tools:1.11.1"
-  codegen "org.apache.avro:avro-compiler:1.11.1"
+  codegen "org.apache.avro:avro-tools:1.11.3"
+  codegen "org.apache.avro:avro-compiler:1.11.3"
 }
 
 test {

helper/helper-common/src/main/java/com/linkedin/avroutil1/compatibility/Jackson2Utils.java

@@ -22,6 +22,7 @@
 import java.util.Map;
 import java.util.Objects;
 import org.apache.avro.Schema;
+import org.apache.avro.SchemaParseException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -228,7 +229,8 @@ public static void assertNoTrailingContent(String json) {
       throw new IllegalStateException("error parsing json out of " + json, e);
     }
     if (!dangling.isEmpty()) {
-      throw new IllegalArgumentException("dangling content beyond the end of a schema at line: "
+      //match behaviour of newer 1.11 and throw SchemaParseException
+      throw new SchemaParseException("dangling content beyond the end of a schema at line: "
           + endOfSchemaLocation.getLineNr() + " column: " + endOfSchemaLocation.getColumnNr() + ": " + dangling);
     }
   }

helper/impls/helper-impl-111/build.gradle

@@ -12,7 +12,7 @@ plugins {
 
 dependencies {
   implementation (project(":helper:helper-common"))
-  compileOnly ("org.apache.avro:avro:1.11.1")
+  compileOnly ("org.apache.avro:avro:1.11.3")
   testImplementation project(":test-common")
   testImplementation("org.mockito:mockito-inline:4.11.0")
 }

helper/impls/helper-impl-111/src/main/java/com/linkedin/avroutil1/compatibility/avro111/Avro111Utils.java

@@ -0,0 +1,75 @@
+/*
+ * Copyright 2023 LinkedIn Corp.
+ * Licensed under the BSD 2-Clause License (the "License").
+ * See License in the project root for license information.
+ */
+
+package com.linkedin.avroutil1.compatibility.avro111;
+
+import java.lang.reflect.Field;
+import org.apache.avro.generic.GenericDatumReader;
+import org.apache.avro.specific.SpecificData;
+
+
+/**
+ * utility code specific to avro 1.11
+ */
+public class Avro111Utils {
+
+  private final static boolean IS_AT_LEAST_1_11_1;
+  private final static boolean IS_AT_LEAST_1_11_2;
+  private final static boolean IS_AT_LEAST_1_11_3;
+
+  static {
+    Class<?>[] inners = GenericDatumReader.class.getDeclaredClasses(); //never null
+    boolean found = false;
+    for (Class<?> innerClass : inners) {
+      //inner class ReaderCache added in https://issues.apache.org/jira/browse/AVRO-3531
+      //see https://github.com/apache/avro/pull/1719/files#diff-eeb81a490688bfaa5b1dfdc5dfb97d6a1c6af057f55bddc40859150b167b9336R531
+      if ("ReaderCache".equals(innerClass.getSimpleName())) {
+        found = true;
+        break;
+      }
+    }
+    IS_AT_LEAST_1_11_1 = found;
+
+    found = false;
+    Field[] fields = SpecificData.class.getFields(); //never null
+    for (Field field : fields) {
+      //introduced in https://issues.apache.org/jira/browse/AVRO-3698
+      //see https://github.com/apache/avro/pull/2048/files#diff-f490a56ccb90aa34a1d14bf04baa055e10f78ba070873e6af47984de62552c20R94
+      if ("RESERVED_WORD_ESCAPE_CHAR".equals(field.getName())) {
+        found = true;
+        break;
+      }
+    }
+    IS_AT_LEAST_1_11_2 = found;
+
+    found = false;
+    try {
+      //added in https://issues.apache.org/jira/browse/AVRO-3819
+      //see https://github.com/apache/avro/pull/2432/files#diff-3d9d3ee7e05c1ae3060af2e14efdc0ef8e19b266807fc14ba41908807e258cd3R41
+      Class.forName("org.apache.avro.SystemLimitException");
+      found = true;
+    } catch (ClassNotFoundException ignored) {
+      //expected in avro < 1.11.3
+    }
+    IS_AT_LEAST_1_11_3 = found;
+  }
+
+  private Avro111Utils() {
+    //util class
+  }
+
+  public static boolean isAtLeast1111() {
+    return IS_AT_LEAST_1_11_1;
+  }
+
+  public static boolean isAtLeast1112() {
+    return IS_AT_LEAST_1_11_2;
+  }
+
+  public static boolean isAtLeast1113() {
+    return IS_AT_LEAST_1_11_3;
+  }
+}

helper/tests/codegen-111/build.gradle

@@ -161,15 +161,15 @@ compileJava.dependsOn runVanillaAvroCodegen, runCompatAvroCodegen, runCompatAvro
 jar.dependsOn runResourceGeneration
 
 dependencies {
-  codegen "org.apache.avro:avro-tools:1.11.1"
-  codegen "org.apache.avro:avro-compiler:1.11.1"
+  codegen "org.apache.avro:avro-tools:1.11.3"
+  codegen "org.apache.avro:avro-compiler:1.11.3"
   codegen files('../codegenClasspath')
   codegen project(":helper:tests:helper-tests-common")
   codegen project(":helper:helper")
 
   //required because generated code depends on the helper
   implementation project(":helper:helper")
-  implementation "org.apache.avro:avro:1.11.1"
+  implementation "org.apache.avro:avro:1.11.3"
 
   //this block required for resource generation code
   implementation project(":helper:tests:helper-tests-common")

helper/tests/helper-tests-111/build.gradle

@@ -13,6 +13,7 @@ plugins {
 
 dependencies {
   testImplementation project(":helper:helper")
+  testImplementation project(":helper:impls:helper-impl-111")
   testImplementation project(":helper:tests:helper-tests-common")
   testImplementation (project(":helper:tests:codegen-14")) {
     exclude group: "org.apache.avro"
@@ -39,8 +40,8 @@ dependencies {
     exclude group: "org.apache.avro"
   }
 
-  testImplementation "org.apache.avro:avro:1.11.1"
-  testImplementation "org.apache.avro:avro-compiler:1.11.1"
+  testImplementation "org.apache.avro:avro:1.11.3"
+  testImplementation "org.apache.avro:avro-compiler:1.11.3"
   testImplementation "com.google.guava:guava:28.2-jre"
   testImplementation "org.mockito:mockito-core:3.2.4"
 }

helper/tests/helper-tests-111/src/test/java/com/linkedin/avroutil1/compatibility/avro111/Avro111UtilsTest.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright 2023 LinkedIn Corp.
+ * Licensed under the BSD 2-Clause License (the "License").
+ * See License in the project root for license information.
+ */
+
+package com.linkedin.avroutil1.compatibility.avro111;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+
+public class Avro111UtilsTest {
+
+  @Test
+  public void testAvro1111Detection() throws Exception {
+    //at time of writing we use 1.11.3
+    Assert.assertTrue(Avro111Utils.isAtLeast1111());
+  }
+
+  @Test
+  public void testAvro1112Detection() throws Exception {
+    //at time of writing we use 1.11.3
+    Assert.assertTrue(Avro111Utils.isAtLeast1112());
+  }
+
+  @Test
+  public void testAvro1113Detection() throws Exception {
+    //at time of writing we use 1.11.3
+    Assert.assertTrue(Avro111Utils.isAtLeast1113());
+  }
+}

helper/tests/helper-tests-111_0/build.gradle

@@ -13,6 +13,7 @@ plugins {
 
 dependencies {
     testImplementation project(":helper:helper")
+    testImplementation project(":helper:impls:helper-impl-111")
     testImplementation project(":helper:tests:helper-tests-common")
     testImplementation (project(":helper:tests:codegen-14")) {
         exclude group: "org.apache.avro"

helper/tests/helper-tests-111_0/src/test/java/com.linkedin.avroutil1.compatibility.avro111/Avro111UtilsUnder1110Test.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright 2023 LinkedIn Corp.
+ * Licensed under the BSD 2-Clause License (the "License").
+ * See License in the project root for license information.
+ */
+
+package com.linkedin.avroutil1.compatibility.avro111;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+
+public class Avro111UtilsUnder1110Test {
+
+  @Test
+  public void testAvro1111Detection() throws Exception {
+    //this module depends on 1.11.0
+    Assert.assertFalse(Avro111Utils.isAtLeast1111());
+  }
+
+  @Test
+  public void testAvro1112Detection() throws Exception {
+    //this module depends on 1.11.0
+    Assert.assertFalse(Avro111Utils.isAtLeast1112());
+  }
+
+  @Test
+  public void testAvro1113Detection() throws Exception {
+    //this module depends on 1.11.0
+    Assert.assertFalse(Avro111Utils.isAtLeast1113());
+  }
+}

helper/tests/helper-tests-allavro/build.gradle

@@ -156,13 +156,13 @@ dependencies {
     exclude group: "org.slf4j"
   }
 
-  avro111 ("org.apache.avro:avro:1.11.1") {
+  avro111 ("org.apache.avro:avro:1.11.3") {
     exclude group: "org.slf4j"
   }
-  avro111 ("org.apache.avro:avro-compiler:1.11.1") {
+  avro111 ("org.apache.avro:avro-compiler:1.11.3") {
     exclude group: "org.slf4j"
   }
-  avro111NoCompiler ("org.apache.avro:avro:1.11.1") {
+  avro111NoCompiler ("org.apache.avro:avro:1.11.3") {
     exclude group: "org.slf4j"
   }
 }

helper/tests/helper-tests-allavro/src/test/java/com/linkedin/avroutil1/compatibility/AvroCompatibilityHelperParsingTest.java

@@ -238,21 +238,26 @@ public void testParseTrailingContent() throws Exception {
     String innocent = "{\"type\": \"string\"}";
     Assert.assertNotNull(AvroCompatibilityHelper.parse(innocent, SchemaParseConfiguration.STRICT, null).getMainSchema());
     Assert.assertNotNull(AvroCompatibilityHelper.parse(innocent + "  \n \t \r  ", SchemaParseConfiguration.STRICT, null).getMainSchema());
+    String badAvsc1 = innocent + "?";
+    String badAvsc2 = innocent + "; DROP TABLE STUDENTS";
     try {
-      AvroCompatibilityHelper.parse(innocent + "?", SchemaParseConfiguration.STRICT, null);
+      AvroCompatibilityHelper.parse(badAvsc1, SchemaParseConfiguration.STRICT, null);
       Assert.fail("expected to fail");
-    } catch (IllegalArgumentException expected) {
+    } catch (SchemaParseException | IllegalArgumentException expected) {
+      //1.11.2+ throws SchemaParseException. for older 1.11 helper fills in the gap
       Assert.assertTrue(expected.getMessage().contains("?"));
     }
-    Assert.assertNotNull(AvroCompatibilityHelper.parse(innocent + "?", SchemaParseConfiguration.LOOSE, null).getMainSchema());
     try {
-      AvroCompatibilityHelper.parse(innocent + "; DROP TABLE STUDENTS", SchemaParseConfiguration.STRICT, null);
+      AvroCompatibilityHelper.parse(badAvsc2, SchemaParseConfiguration.STRICT, null);
       Assert.fail("expected to fail");
-    } catch (IllegalArgumentException expected) {
+    } catch (SchemaParseException | IllegalArgumentException expected) {
+      //1.11.2+ throws SchemaParseException. for older 1.11 helper fills in the gap
       Assert.assertTrue(expected.getMessage().contains("DROP TABLE STUDENTS"));
     }
-    Assert.assertNotNull(
-        AvroCompatibilityHelper.parse(innocent + "; DROP TABLE STUDENTS", SchemaParseConfiguration.LOOSE, null).getMainSchema()
-    );
+    //stops being possible under 1.11.2+
+    if (AvroCompatibilityHelper.getRuntimeAvroVersion().earlierThan(AvroVersion.AVRO_1_11)) {
+      Assert.assertNotNull(AvroCompatibilityHelper.parse(badAvsc1, SchemaParseConfiguration.LOOSE, null).getMainSchema());
+      Assert.assertNotNull(AvroCompatibilityHelper.parse(badAvsc2, SchemaParseConfiguration.LOOSE, null).getMainSchema());
+    }
   }
 }