Re-apply avro update to 1.9.2 (#751) (#762)

Co-authored-by: Junchuan Wang <[email protected]>

CHANGELOG.md

@@ -14,8 +14,10 @@ and what APIs have changed, if applicable.
 
 ## [Unreleased]
 
-## [29.29.2] - 2022-02-17
+## [29.30.0] - 2022-02-28
+- Re-apply avro update to 1.9.2
 
+## [29.29.2] - 2022-02-17
 - Generalize avro --> pegasus translation code to accept any CharSequence value for strings
 
 ## [29.29.1] - 2022-02-10
@@ -28,7 +30,7 @@ and what APIs have changed, if applicable.
 - Fix weight double-count in D2 SimpleLoadBalancer
 
 ## [29.27.0] - 2022-01-25
-- Update Avro version to 1.92
+- Update Avro version to 1.9.2
 
 ## [29.26.4] - 2022-01-24
 - Map local variant of service ZNodes to cluster without colo suffix
@@ -5186,7 +5188,8 @@ patch operations can re-use these classes for generating patch messages.
 
 ## [0.14.1]
 
-[Unreleased]: https://github.com/linkedin/rest.li/compare/v29.29.2...master
+[Unreleased]: https://github.com/linkedin/rest.li/compare/v29.30.0...master
+[29.30.0]: https://github.com/linkedin/rest.li/compare/v29.29.2...v29.30.0
 [29.29.2]: https://github.com/linkedin/rest.li/compare/v29.29.1...v29.29.2
 [29.29.1]: https://github.com/linkedin/rest.li/compare/v29.29.0...v29.29.1
 [29.29.0]: https://github.com/linkedin/rest.li/compare/v29.28.0...v29.29.0

build.gradle

@@ -45,7 +45,7 @@ apply from: "${buildScriptDirPath}/configBuildScript.gradle"
 project.ext.externalDependency = [
   'antlr': 'org.antlr:antlr4:4.5',
   'antlrRuntime': 'org.antlr:antlr4-runtime:4.5',
-  'avro': 'org.apache.avro:avro:1.4.0',
+  'avro': 'org.apache.avro:avro:1.9.2',
   'avro_1_6': 'org.apache.avro:avro:1.6.3',
    // avro compatibility layer
   'avroUtil': 'com.linkedin.avroutil1:helper-all:0.2.17',
@@ -65,9 +65,9 @@ project.ext.externalDependency = [
   'guava': 'com.google.guava:guava:18.0',
   'httpclient': 'org.apache.httpcomponents:httpclient:4.3.1',
   'httpcore': 'org.apache.httpcomponents:httpcore:4.3.1',
-  'jacksonCore': 'com.fasterxml.jackson.core:jackson-core:2.9.7',
-  'jacksonSmile': 'com.fasterxml.jackson.dataformat:jackson-dataformat-smile:2.9.7',
-  'jacksonDataBind': 'com.fasterxml.jackson.core:jackson-databind:2.9.7',
+  'jacksonCore': 'com.fasterxml.jackson.core:jackson-core:2.10.2',
+  'jacksonSmile': 'com.fasterxml.jackson.dataformat:jackson-dataformat-smile:2.10.2',
+  'jacksonDataBind': 'com.fasterxml.jackson.core:jackson-databind:2.10.2',
   'jacksonCoreAsl_1_4': 'org.codehaus.jackson:jackson-core-asl:1.4.2',
   'jacksonCoreAsl_1_8': 'org.codehaus.jackson:jackson-core-asl:1.8.8',
   'javaparser': 'com.github.javaparser:javaparser-symbol-solver-core:3.15.11',

data-avro-1_6/src/main/java/com/linkedin/data/avro/AvroAdapter_1_6.java

@@ -16,14 +16,13 @@
 
 package com.linkedin.data.avro;
 
+import com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper;
 import java.io.IOException;
 import java.io.OutputStream;
 import org.apache.avro.Schema;
 import org.apache.avro.generic.GenericData;
 import org.apache.avro.io.Decoder;
-import org.apache.avro.io.DecoderFactory;
 import org.apache.avro.io.Encoder;
-import org.apache.avro.io.EncoderFactory;
 
 /**
  * Adapter for Avro 1.6
@@ -32,9 +31,6 @@
 @Deprecated
 public class AvroAdapter_1_6 implements AvroAdapter
 {
-  private final DecoderFactory _decoderFactory = DecoderFactory.get();
-  private final EncoderFactory _encoderFactory = EncoderFactory.get();
-
   @Override
   public boolean jsonUnionMemberHasFullName()
   {
@@ -44,36 +40,36 @@ public boolean jsonUnionMemberHasFullName()
   @Override
   public GenericData.EnumSymbol createEnumSymbol(Schema avroSchema, String enumValue)
   {
-    return new GenericData.EnumSymbol(avroSchema, enumValue);
+    return AvroCompatibilityHelper.newEnumSymbol(avroSchema, enumValue);
   }
 
   @Override
   public Schema stringToAvroSchema(String avroSchemaJson)
   {
-    return new Schema.Parser().parse(avroSchemaJson);
+    return Schema.parse(avroSchemaJson);
   }
 
   @Override
   public Decoder createBinaryDecoder(byte[] bytes) throws IOException
   {
-    return _decoderFactory.binaryDecoder(bytes, null);
+    return AvroCompatibilityHelper.newBinaryDecoder(bytes);
   }
 
   @Override
   public Encoder createBinaryEncoder(OutputStream outputStream) throws IOException
   {
-    return _encoderFactory.binaryEncoder(outputStream, null);
+    return AvroCompatibilityHelper.newBinaryEncoder(outputStream);
   }
 
   @Override
   public Decoder createJsonDecoder(Schema schema, String json) throws IOException
   {
-    return _decoderFactory.jsonDecoder(schema, json);
+    return AvroCompatibilityHelper.newJsonDecoder(schema, json);
   }
 
   @Override
   public Encoder createJsonEncoder(Schema schema, OutputStream outputStream) throws IOException
   {
-    return _encoderFactory.jsonEncoder(schema, outputStream);
+    return AvroCompatibilityHelper.newJsonEncoder(schema, outputStream, true);
   }
 }

data-avro-generator/src/test/java/com/linkedin/data/avro/generator/TestAvroSchemaGenerator.java

@@ -16,17 +16,24 @@
 
 package com.linkedin.data.avro.generator;
 
+import com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper;
 import com.linkedin.data.DataMap;
 import com.linkedin.data.TestUtil;
+import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 import org.apache.avro.Schema;
+import org.apache.avro.Schema.Parser;
+import org.apache.commons.compress.utils.IOUtils;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.DataProvider;
@@ -338,7 +345,12 @@ private void run(String[] args, Map.Entry<File, Map.Entry<String, String>> entry
       Schema avroSchema;
       try
       {
-        avroSchema = Schema.parse(avroSchemaInputStream);
+        avroSchema = AvroCompatibilityHelper.parse(
+            new BufferedReader(
+                new InputStreamReader(avroSchemaInputStream, StandardCharsets.UTF_8))
+                .lines()
+                .collect(Collectors.joining("\n"))
+        );
       }
       finally
       {

data-avro/src/main/java/com/linkedin/data/avro/AvroAdapter_1_4.java

@@ -17,16 +17,13 @@
 package com.linkedin.data.avro;
 
 
+import com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper;
 import java.io.IOException;
 import java.io.OutputStream;
 import org.apache.avro.Schema;
 import org.apache.avro.generic.GenericData;
-import org.apache.avro.io.BinaryEncoder;
 import org.apache.avro.io.Decoder;
-import org.apache.avro.io.DecoderFactory;
 import org.apache.avro.io.Encoder;
-import org.apache.avro.io.JsonDecoder;
-import org.apache.avro.io.JsonEncoder;
 
 
 /**
@@ -44,7 +41,7 @@ public boolean jsonUnionMemberHasFullName()
   @Override
   public GenericData.EnumSymbol createEnumSymbol(Schema avroSchema, String enumValue)
   {
-    return new GenericData.EnumSymbol(enumValue);
+    return AvroCompatibilityHelper.newEnumSymbol(avroSchema, enumValue);
   }
 
   @Override
@@ -56,29 +53,25 @@ public Schema stringToAvroSchema(String avroSchemaJson)
   @Override
   public Decoder createBinaryDecoder(byte[] bytes) throws IOException
   {
-    Decoder binaryDecoder = DecoderFactory.defaultFactory().createBinaryDecoder(bytes, null);
-    return binaryDecoder;
+    return AvroCompatibilityHelper.newBinaryDecoder(bytes);
   }
 
   @Override
   public Encoder createBinaryEncoder(OutputStream outputStream) throws IOException
   {
-    Encoder binaryEncoder = new BinaryEncoder(outputStream);
-    return binaryEncoder;
+    return AvroCompatibilityHelper.newBinaryEncoder(outputStream);
   }
 
   @Override
   public Decoder createJsonDecoder(Schema schema, String json) throws IOException
   {
-    Decoder jsonDecoder = new JsonDecoder(schema, json);
-    return jsonDecoder;
+    return AvroCompatibilityHelper.newJsonDecoder(schema, json);
   }
 
   @Override
   public Encoder createJsonEncoder(Schema schema, OutputStream outputStream) throws IOException
   {
-    Encoder jsonEncoder = new JsonEncoder(schema, outputStream);
-    return jsonEncoder;
+    return AvroCompatibilityHelper.newJsonEncoder(schema, outputStream, true);
   }
 }
 

data-avro/src/main/java/com/linkedin/data/avro/SchemaTranslator.java

@@ -18,6 +18,7 @@
 
 
 import com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper;
+import com.linkedin.avroutil1.compatibility.SchemaParseConfiguration;
 import com.linkedin.data.DataMap;
 import com.linkedin.data.DataMapBuilder;
 import com.linkedin.data.schema.DataSchema;
@@ -159,7 +160,12 @@ public static DataSchema avroToDataSchema(String avroSchemaInJson, AvroToDataSch
             Object optionalDefaultModeProperty = ((DataMap) dataProperty).get(SchemaTranslator.OPTIONAL_DEFAULT_MODE_PROPERTY);
             dataToAvroSchemaOptions.setOptionalDefaultMode(OptionalDefaultMode.valueOf(optionalDefaultModeProperty.toString()));
             Schema avroSchemaFromEmbedded = dataToAvroSchema(resultDataSchema, dataToAvroSchemaOptions);
-            Schema avroSchemaFromJson = Schema.parse(avroSchemaInJson);
+            Schema avroSchemaFromJson = AvroCompatibilityHelper.parse(avroSchemaInJson, SchemaParseConfiguration.STRICT, null).getMainSchema();
+            Object embededSchemaPropertyVal = avroSchemaFromJson.getObjectProp(DATA_PROPERTY);
+            if (embededSchemaPropertyVal != null)
+            {
+              avroSchemaFromEmbedded.addProp(DATA_PROPERTY, embededSchemaPropertyVal);
+            }
             if (!avroSchemaFromEmbedded.equals(avroSchemaFromJson))
             {
               throw new IllegalArgumentException("Embedded schema does not translate to input Avro schema: " + avroSchemaInJson);

data-avro/src/main/java/com/linkedin/data/avro/util/AvroUtil.java

@@ -17,6 +17,7 @@
 package com.linkedin.data.avro.util;
 
 import com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper;
+import com.linkedin.avroutil1.compatibility.AvroVersion;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
@@ -31,9 +32,32 @@ public class AvroUtil
 {
   public static String jsonFromGenericRecord(GenericRecord record) throws IOException
   {
-    GenericDatumWriter<GenericRecord> writer = new GenericDatumWriter<>();
+    return jsonFromGenericRecord(record, true);
+  }
+
+  public static String jsonFromGenericRecord(GenericRecord record, boolean pretty) throws IOException
+  {
     ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
-    Encoder jsonEncoder = AvroCompatibilityHelper.newJsonEncoder(record.getSchema(), outputStream, true);
+    return jsonFromGenericRecord(record,
+        outputStream,
+        AvroCompatibilityHelper.newJsonEncoder(record.getSchema(), outputStream, pretty));
+
+  }
+
+  public static String jsonFromGenericRecord(GenericRecord record, boolean pretty, AvroVersion version) throws IOException
+  {
+    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+    return jsonFromGenericRecord(record,
+        outputStream,
+        AvroCompatibilityHelper.newJsonEncoder(record.getSchema(), outputStream, pretty, version));
+  }
+
+  private static String jsonFromGenericRecord(
+      GenericRecord record,
+      ByteArrayOutputStream outputStream,
+      Encoder jsonEncoder) throws IOException
+  {
+    GenericDatumWriter<GenericRecord> writer = new GenericDatumWriter<>();
     writer.setSchema(record.getSchema());
     writer.write(record, jsonEncoder);
     jsonEncoder.flush();