From cdf7f906171d939a1c422ea2d4a6be9d38552844 Mon Sep 17 00:00:00 2001 From: Karthik Ramgopal Date: Fri, 12 Jan 2024 06:34:42 -0800 Subject: [PATCH] Decouple schema parsing from code generation (#532) Remove dead code Co-authored-by: Karthik Ramgopal --- .../builder/operations/OperationContext.java | 39 ++-- .../builder/operations}/SchemaSet.java | 2 +- .../builder/plugins/BuilderPluginContext.java | 18 +- .../avroutil1/builder/SchemaBuilder.java | 20 ++- .../codegen/OperationContextBuilder.java | 21 +++ .../codegen/own/AvroUtilCodeGenPlugin.java | 168 ++++++++++++++++++ ...a => AvroUtilOperationContextBuilder.java} | 165 +++-------------- .../codegen/vanilla/ClasspathSchemaSet.java | 1 + .../vanilla/FileSystemSchemaSetProvider.java | 1 + .../vanilla/ResolverPathSchemaSet.java | 1 + .../codegen/vanilla/SchemaSetProvider.java | 3 + .../codegen/vanilla/SimpleSchemaSet.java | 1 + .../vanilla/VanillaProcessedCodeGenOp.java | 18 +- .../codegen/SpecificRecordClassGenerator.java | 100 ++++------- .../codegen/SpecificRecordGeneratorUtil.java | 5 - 15 files changed, 297 insertions(+), 266 deletions(-) rename avro-builder/{builder/src/main/java/com/linkedin/avroutil1/builder/operations/codegen/vanilla => builder-spi/src/main/java/com/linkedin/avroutil1/builder/operations}/SchemaSet.java (90%) create mode 100644 avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/operations/codegen/OperationContextBuilder.java create mode 100644 avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/operations/codegen/own/AvroUtilCodeGenPlugin.java rename avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/operations/codegen/own/{AvroUtilCodeGenOp.java => AvroUtilOperationContextBuilder.java} (59%) diff --git a/avro-builder/builder-spi/src/main/java/com/linkedin/avroutil1/builder/operations/OperationContext.java b/avro-builder/builder-spi/src/main/java/com/linkedin/avroutil1/builder/operations/OperationContext.java index 104439e08..b779cd2ff 100644 --- a/avro-builder/builder-spi/src/main/java/com/linkedin/avroutil1/builder/operations/OperationContext.java +++ b/avro-builder/builder-spi/src/main/java/com/linkedin/avroutil1/builder/operations/OperationContext.java @@ -10,8 +10,8 @@ import java.io.File; import java.util.Collections; import java.util.HashMap; +import java.util.Objects; import java.util.Set; -import org.apache.avro.Schema; /** @@ -19,33 +19,21 @@ */ public class OperationContext { private final HashMap context; - private Set avroFiles; - private Set avroSchemas; + private final Set avroFiles; + private final Set avroSchemas; + private final SchemaSet lookupSchemaSet; - public OperationContext() { + public OperationContext(Set avroSchemas, Set avroFiles, SchemaSet lookupSchemaSet) { this.context = new HashMap<>(); - } - - public void addParsedSchemas(Set avroSchemas, Set avroFiles) { - if (this.avroFiles != null || this.avroSchemas != null) { - throw new IllegalStateException("Cannot initialize avro files twice"); - } - - this.avroFiles = Collections.unmodifiableSet(avroFiles); - this.avroSchemas = Collections.unmodifiableSet(avroSchemas); - } - - public void addVanillaSchemas(Set schemas, Set avroFiles) { - addParsedSchemas(AvroSchemaUtils.schemasToAvroSchemas(schemas), avroFiles); + this.avroSchemas = Collections.unmodifiableSet(Objects.requireNonNull(avroSchemas)); + this.avroFiles = Collections.unmodifiableSet(Objects.requireNonNull(avroFiles)); + this.lookupSchemaSet = lookupSchemaSet; } /** * Returns an unmodifiable set of all input avro files. */ public Set getAvroFiles() { - if (this.avroFiles == null) { - throw new IllegalStateException("Avro files haven't been added yet."); - } return this.avroFiles; } @@ -53,13 +41,16 @@ public Set getAvroFiles() { * Returns an unmodifiable set of all parsed avro schemas. */ public Set getAvroSchemas() { - if (this.avroSchemas == null) { - throw new IllegalStateException("Avro schemas haven't been added yet."); - } - return this.avroSchemas; } + /** + * Returns the lookup schema set. + */ + public SchemaSet getLookupSchemaSet() { + return this.lookupSchemaSet; + } + public void setConfigValue(String key, Object value) { if (this.context.containsKey(key)) { throw new IllegalStateException("Cannot initialize an already-initialized config key."); diff --git a/avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/operations/codegen/vanilla/SchemaSet.java b/avro-builder/builder-spi/src/main/java/com/linkedin/avroutil1/builder/operations/SchemaSet.java similarity index 90% rename from avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/operations/codegen/vanilla/SchemaSet.java rename to avro-builder/builder-spi/src/main/java/com/linkedin/avroutil1/builder/operations/SchemaSet.java index 685ddc5b7..0a4557f60 100644 --- a/avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/operations/codegen/vanilla/SchemaSet.java +++ b/avro-builder/builder-spi/src/main/java/com/linkedin/avroutil1/builder/operations/SchemaSet.java @@ -4,7 +4,7 @@ * See License in the project root for license information. */ -package com.linkedin.avroutil1.builder.operations.codegen.vanilla; +package com.linkedin.avroutil1.builder.operations; import java.util.List; diff --git a/avro-builder/builder-spi/src/main/java/com/linkedin/avroutil1/builder/plugins/BuilderPluginContext.java b/avro-builder/builder-spi/src/main/java/com/linkedin/avroutil1/builder/plugins/BuilderPluginContext.java index 78f661078..df9dcf93f 100644 --- a/avro-builder/builder-spi/src/main/java/com/linkedin/avroutil1/builder/plugins/BuilderPluginContext.java +++ b/avro-builder/builder-spi/src/main/java/com/linkedin/avroutil1/builder/plugins/BuilderPluginContext.java @@ -17,9 +17,13 @@ */ public class BuilderPluginContext { - private List operations = new ArrayList<>(1); - private OperationContext _operationContext = new OperationContext(); + private final List operations = new ArrayList<>(1); private volatile boolean sealed = false; + private final OperationContext operationContext; + + public BuilderPluginContext(OperationContext operationContext) { + this.operationContext = operationContext; + } public void add(Operation op) { if (sealed) { @@ -39,8 +43,12 @@ public void run() throws Exception { //"seal" any internal state to prevent plugins from trying to do weird things during execution sealed = true; - for (Operation op : operations) { - op.run(_operationContext); - } + operations.parallelStream().forEach(op -> { + try { + op.run(operationContext); + } catch (Exception e) { + throw new IllegalStateException("Exception running operation", e); + } + }); } } diff --git a/avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/SchemaBuilder.java b/avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/SchemaBuilder.java index 8840e7059..edb8b5afe 100644 --- a/avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/SchemaBuilder.java +++ b/avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/SchemaBuilder.java @@ -6,10 +6,12 @@ package com.linkedin.avroutil1.builder; +import com.linkedin.avroutil1.builder.operations.OperationContext; import com.linkedin.avroutil1.builder.operations.codegen.CodeGenerator; -import com.linkedin.avroutil1.builder.operations.codegen.own.AvroUtilCodeGenOp; +import com.linkedin.avroutil1.builder.operations.codegen.OperationContextBuilder; +import com.linkedin.avroutil1.builder.operations.codegen.own.AvroUtilCodeGenPlugin; import com.linkedin.avroutil1.builder.operations.codegen.CodeGenOpConfig; -import com.linkedin.avroutil1.builder.operations.Operation; +import com.linkedin.avroutil1.builder.operations.codegen.own.AvroUtilOperationContextBuilder; import com.linkedin.avroutil1.builder.operations.codegen.vanilla.VanillaProcessedCodeGenOp; import com.linkedin.avroutil1.builder.plugins.BuilderPlugin; import com.linkedin.avroutil1.builder.plugins.BuilderPluginContext; @@ -232,8 +234,6 @@ public static void main(String[] args) throws Exception { plugin.parseAndValidateOptions(options); } - BuilderPluginContext context = new BuilderPluginContext(); - CodeGenOpConfig opConfig = new CodeGenOpConfig( inputs, nonImportableSources, @@ -254,20 +254,22 @@ public static void main(String[] args) throws Exception { opConfig.validateParameters(); - Operation op; + OperationContextBuilder operationContextBuilder; switch (opConfig.getGeneratorType()) { case AVRO_UTIL: - op = new AvroUtilCodeGenOp(opConfig); + operationContextBuilder = new AvroUtilOperationContextBuilder(); + plugins.add(new AvroUtilCodeGenPlugin(opConfig)); break; case VANILLA: - op = new VanillaProcessedCodeGenOp(opConfig); + operationContextBuilder = new VanillaProcessedCodeGenOp(); break; default: throw new IllegalStateException("unhandled: " + opConfig.getGeneratorType()); } - context.add(op); + OperationContext opContext = operationContextBuilder.buildOperationContext(opConfig); + BuilderPluginContext context = new BuilderPluginContext(opContext); - //allow plugins to add operations + // Allow other plugins to add operations for (BuilderPlugin plugin : plugins) { plugin.createOperations(context); } diff --git a/avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/operations/codegen/OperationContextBuilder.java b/avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/operations/codegen/OperationContextBuilder.java new file mode 100644 index 000000000..127c4611b --- /dev/null +++ b/avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/operations/codegen/OperationContextBuilder.java @@ -0,0 +1,21 @@ +/* + * Copyright 2024 LinkedIn Corp. + * Licensed under the BSD 2-Clause License (the "License"). + * See License in the project root for license information. + */ + +package com.linkedin.avroutil1.builder.operations.codegen; + +import com.linkedin.avroutil1.builder.operations.OperationContext; + + +/** + * Builds operation context. + */ +public interface OperationContextBuilder { + + /** + * Builds and returns the {@link OperationContext}. + */ + OperationContext buildOperationContext(CodeGenOpConfig opConfig) throws Exception; +} diff --git a/avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/operations/codegen/own/AvroUtilCodeGenPlugin.java b/avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/operations/codegen/own/AvroUtilCodeGenPlugin.java new file mode 100644 index 000000000..e741e9a84 --- /dev/null +++ b/avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/operations/codegen/own/AvroUtilCodeGenPlugin.java @@ -0,0 +1,168 @@ +/* + * Copyright 2022 LinkedIn Corp. + * Licensed under the BSD 2-Clause License (the "License"). + * See License in the project root for license information. + */ + +package com.linkedin.avroutil1.builder.operations.codegen.own; + +import com.linkedin.avroutil1.builder.operations.OperationContext; +import com.linkedin.avroutil1.builder.operations.SchemaSet; +import com.linkedin.avroutil1.builder.operations.codegen.CodeGenOpConfig; +import com.linkedin.avroutil1.builder.plugins.BuilderPlugin; +import com.linkedin.avroutil1.builder.plugins.BuilderPluginContext; +import com.linkedin.avroutil1.codegen.SpecificRecordClassGenerator; +import com.linkedin.avroutil1.codegen.SpecificRecordGenerationConfig; +import com.linkedin.avroutil1.codegen.SpecificRecordGeneratorUtil; +import com.linkedin.avroutil1.model.AvroJavaStringRepresentation; +import com.linkedin.avroutil1.model.AvroNamedSchema; +import com.linkedin.avroutil1.model.AvroType; +import com.linkedin.avroutil1.model.AvroUnionSchema; +import com.squareup.javapoet.JavaFile; +import java.io.File; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * A code generation plugin using the avro-codegen module of avro-util + */ +public class AvroUtilCodeGenPlugin implements BuilderPlugin { + private static final Logger LOGGER = LoggerFactory.getLogger(AvroUtilCodeGenPlugin.class); + + private final CodeGenOpConfig config; + + public AvroUtilCodeGenPlugin(CodeGenOpConfig config) { + this.config = config; + } + + @Override + public Set supportedApiVersions() { + return Collections.singleton(1); + } + + @Override + public void createOperations(BuilderPluginContext context) { + context.add(this::generateCode); + } + + private void generateCode(OperationContext opContext) { + //mkdir any output folders that don't exist + if (!config.getOutputSpecificRecordClassesRoot().exists() && !config.getOutputSpecificRecordClassesRoot() + .mkdirs()) { + throw new IllegalStateException( + "unable to create destination folder " + config.getOutputSpecificRecordClassesRoot()); + } + + final AtomicInteger schemaCounter = new AtomicInteger(0); + final int schemaChunkSize = 500; + Collection> allNamedSchemaList = opContext.getAvroSchemas().stream().flatMap(schema -> { + if (schema instanceof AvroNamedSchema) { + return Stream.of((AvroNamedSchema) schema); + } else if (AvroType.UNION.equals(schema.type())) { + return ((AvroUnionSchema) schema).getTypes() + .stream() + .map(schemaOrRef -> (AvroNamedSchema) schemaOrRef.getSchema()); + } else { + return Stream.empty(); + } + }).collect(Collectors.groupingBy(it -> schemaCounter.getAndIncrement() / schemaChunkSize)).values(); + + long genStart = System.currentTimeMillis(); + + final SpecificRecordGenerationConfig generationConfig = + SpecificRecordGenerationConfig.getBroadCompatibilitySpecificRecordGenerationConfig( + AvroJavaStringRepresentation.fromJson(config.getStringRepresentation().toString()), + AvroJavaStringRepresentation.fromJson(config.getMethodStringRepresentation().toString()), + config.getMinAvroVersion(), config.isUtf8EncodingPutByIndexEnabled()); + + // Make sure the output folder exists + File outputFolder = config.getOutputSpecificRecordClassesRoot(); + if (!outputFolder.exists() && !outputFolder.mkdirs()) { + throw new IllegalStateException("unable to create output folder " + outputFolder); + } + final Path outputDirectoryPath = outputFolder.toPath(); + final SpecificRecordClassGenerator generator = new SpecificRecordClassGenerator(); + + int totalGeneratedClasses = allNamedSchemaList.parallelStream().map(allNamedSchemas -> { + HashSet alreadyGeneratedSchemaNames = new HashSet<>(); + List generatedSpecificClasses = new ArrayList<>(allNamedSchemas.size()); + for (AvroNamedSchema namedSchema : allNamedSchemas) { + try { + if (!alreadyGeneratedSchemaNames.contains(namedSchema.getFullName())) { + // skip codegen if schema is on classpath and config says to skip + if (config.shouldSkipCodegenIfSchemaOnClasspath() && + doesSchemaExistOnClasspath(namedSchema, opContext.getLookupSchemaSet())) { + continue; + } + + //top level schema + alreadyGeneratedSchemaNames.add(namedSchema.getFullName()); + generatedSpecificClasses.add(generator.generateSpecificClass(namedSchema, generationConfig)); + + // generate internal schemas if not already present + List internalSchemaList = + SpecificRecordGeneratorUtil.getNestedInternalSchemaList(namedSchema); + for (AvroNamedSchema namedInternalSchema : internalSchemaList) { + if (!alreadyGeneratedSchemaNames.contains(namedInternalSchema.getFullName())) { + // skip codegen for nested schemas if schema is on classpath and config says to skip + if (config.shouldSkipCodegenIfSchemaOnClasspath() && + doesSchemaExistOnClasspath(namedInternalSchema, opContext.getLookupSchemaSet())) { + continue; + } + + generatedSpecificClasses.add(generator.generateSpecificClass(namedInternalSchema, generationConfig)); + alreadyGeneratedSchemaNames.add(namedInternalSchema.getFullName()); + } + } + } + } catch (Exception e) { + throw new RuntimeException("failed to generate class for " + namedSchema.getFullName(), e); + } + } + writeJavaFilesToDisk(generatedSpecificClasses, outputDirectoryPath); + return generatedSpecificClasses.size(); + }).reduce(0, Integer::sum); + + long genEnd = System.currentTimeMillis(); + LOGGER.info("Generated {} java source files in {} millis", totalGeneratedClasses, genEnd - genStart); + } + + private boolean doesSchemaExistOnClasspath(AvroNamedSchema schema, SchemaSet schemaSet) { + if (schemaSet == null) { + return false; + } + + return schemaSet.getByName(schema.getFullName()) != null; + } + + private void writeJavaFilesToDisk(Collection javaFiles, Path outputFolderPath) { + + long writeStart = System.currentTimeMillis(); + + // write out the files we generated + int filesWritten = javaFiles.parallelStream().map(javaFile -> { + try { + javaFile.writeToPath(outputFolderPath); + } catch (Exception e) { + throw new IllegalStateException("while writing file " + javaFile.typeSpec.name, e); + } + + return 1; + }).reduce(0, Integer::sum); + + long writeEnd = System.currentTimeMillis(); + LOGGER.info("wrote out {} generated java source files under {} in {} millis", filesWritten, outputFolderPath, + writeEnd - writeStart); + } +} diff --git a/avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/operations/codegen/own/AvroUtilCodeGenOp.java b/avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/operations/codegen/own/AvroUtilOperationContextBuilder.java similarity index 59% rename from avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/operations/codegen/own/AvroUtilCodeGenOp.java rename to avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/operations/codegen/own/AvroUtilOperationContextBuilder.java index 001b14833..3ecb29e41 100644 --- a/avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/operations/codegen/own/AvroUtilCodeGenOp.java +++ b/avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/operations/codegen/own/AvroUtilOperationContextBuilder.java @@ -1,29 +1,23 @@ /* - * Copyright 2022 LinkedIn Corp. + * Copyright 2024 LinkedIn Corp. * Licensed under the BSD 2-Clause License (the "License"). * See License in the project root for license information. */ package com.linkedin.avroutil1.builder.operations.codegen.own; -import com.linkedin.avroutil1.builder.operations.Operation; import com.linkedin.avroutil1.builder.operations.OperationContext; import com.linkedin.avroutil1.builder.operations.codegen.CodeGenOpConfig; +import com.linkedin.avroutil1.builder.operations.codegen.OperationContextBuilder; import com.linkedin.avroutil1.builder.operations.codegen.util.AvscFileFinderUtil; import com.linkedin.avroutil1.builder.operations.codegen.vanilla.ClasspathSchemaSet; import com.linkedin.avroutil1.builder.operations.codegen.vanilla.ResolverPathSchemaSet; -import com.linkedin.avroutil1.builder.operations.codegen.vanilla.SchemaSet; -import com.linkedin.avroutil1.codegen.SpecificRecordClassGenerator; -import com.linkedin.avroutil1.codegen.SpecificRecordGenerationConfig; -import com.linkedin.avroutil1.codegen.SpecificRecordGeneratorUtil; +import com.linkedin.avroutil1.builder.operations.SchemaSet; import com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper; import com.linkedin.avroutil1.compatibility.AvscGenerationConfig; import com.linkedin.avroutil1.compatibility.SchemaComparisonConfiguration; -import com.linkedin.avroutil1.model.AvroJavaStringRepresentation; import com.linkedin.avroutil1.model.AvroNamedSchema; import com.linkedin.avroutil1.model.AvroSchema; -import com.linkedin.avroutil1.model.AvroType; -import com.linkedin.avroutil1.model.AvroUnionSchema; import com.linkedin.avroutil1.model.SchemaOrRef; import com.linkedin.avroutil1.parser.avsc.AvroParseContext; import com.linkedin.avroutil1.parser.avsc.AvscParseResult; @@ -31,17 +25,13 @@ import com.linkedin.avroutil1.util.ConfigurableAvroSchemaComparator; import com.linkedin.avroutil1.writer.avsc.AvscSchemaWriter; import com.linkedin.avroutil1.writer.avsc.AvscWriterConfig; -import com.squareup.javapoet.JavaFile; import java.io.File; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.StringJoiner; -import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.avro.Schema; @@ -49,29 +39,14 @@ import org.slf4j.LoggerFactory; -/** - * a code generation operation using the avro-codegen module of avro-util - */ -public class AvroUtilCodeGenOp implements Operation { - private static final Logger LOGGER = LoggerFactory.getLogger(AvroUtilCodeGenOp.class); - - private final CodeGenOpConfig config; +public class AvroUtilOperationContextBuilder implements OperationContextBuilder { - public AvroUtilCodeGenOp(CodeGenOpConfig config) { - this.config = config; - } + private static final Logger LOGGER = LoggerFactory.getLogger(AvroUtilOperationContextBuilder.class); @Override - public void run(OperationContext opContext) throws Exception { - //mkdir any output folders that don't exist - if (!config.getOutputSpecificRecordClassesRoot().exists() && !config.getOutputSpecificRecordClassesRoot() - .mkdirs()) { - throw new IllegalStateException( - "unable to create destination folder " + config.getOutputSpecificRecordClassesRoot()); - } - + public OperationContext buildOperationContext(CodeGenOpConfig config) throws Exception { if (!config.isAvro702Handling()) { - LOGGER.warn("Avro-702 handling was disabled, however Avro-702 handling cannot be disabled in AvroUtilCodeGenOp."); + LOGGER.warn("Avro-702 handling was disabled, however Avro-702 handling cannot be disabled."); } Set avscFiles = new HashSet<>(); @@ -93,7 +68,7 @@ public void run(OperationContext opContext) throws Exception { int numFiles = avscFiles.size() + nonImportableFiles.size(); if (numFiles == 0) { LOGGER.warn("no input schema files were found under roots " + config.getInputRoots()); - return; + return new OperationContext(Collections.emptySet(), Collections.emptySet(), null); } LOGGER.info("found " + numFiles + " avsc schema files in " + (scanEnd - scanStart) + " millis"); @@ -163,21 +138,6 @@ public void run(OperationContext opContext) throws Exception { long parseEnd = System.currentTimeMillis(); - final AtomicInteger schemaCounter = new AtomicInteger(0); - final int schemaChunkSize = 500; - Collection> allNamedSchemaList = parsedFiles.stream().flatMap(parseResult -> { - AvroSchema schema = parseResult.getTopLevelSchema(); - if (schema instanceof AvroNamedSchema) { - return Stream.of((AvroNamedSchema) parseResult.getTopLevelSchema()); - } else if (AvroType.UNION.equals(schema.type())) { - return ((AvroUnionSchema) schema).getTypes() - .stream() - .map(schemaOrRef -> (AvroNamedSchema) schemaOrRef.getSchema()); - } else { - return Stream.empty(); - } - }).collect(Collectors.groupingBy(it -> schemaCounter.getAndIncrement() / schemaChunkSize)).values(); - // Handle duplicate schemas Map> duplicates = context.getDuplicates(); for (Map.Entry> duplicateEntry : duplicates.entrySet()) { @@ -226,9 +186,6 @@ public void run(OperationContext opContext) throws Exception { } } - LOGGER.info("parsed {} named schemas in {} millis, {} of which have duplicates", schemaCounter.get(), - parseEnd - scanEnd, duplicates.size()); - //TODO fail if any errors or dups (depending on config) are found if (context.hasExternalReferences()) { //TODO - better formatting @@ -236,71 +193,19 @@ public void run(OperationContext opContext) throws Exception { "unresolved referenced to external schemas: " + context.getExternalReferences()); } - long genStart = System.currentTimeMillis(); - - final SpecificRecordGenerationConfig generationConfig = - SpecificRecordGenerationConfig.getBroadCompatibilitySpecificRecordGenerationConfig( - AvroJavaStringRepresentation.fromJson(config.getStringRepresentation().toString()), - AvroJavaStringRepresentation.fromJson(config.getMethodStringRepresentation().toString()), - config.getMinAvroVersion(), config.isUtf8EncodingPutByIndexEnabled()); - - // Make sure the output folder exists - File outputFolder = config.getOutputSpecificRecordClassesRoot(); - if (!outputFolder.exists() && !outputFolder.mkdirs()) { - throw new IllegalStateException("unable to create output folder " + outputFolder); - } - final Path outputDirectoryPath = outputFolder.toPath(); - final SpecificRecordClassGenerator generator = new SpecificRecordClassGenerator(); - - int totalGeneratedClasses = allNamedSchemaList.parallelStream().map(allNamedSchemas -> { - HashSet alreadyGeneratedSchemaNames = new HashSet<>(); - List generatedSpecificClasses = new ArrayList<>(allNamedSchemas.size()); - for (AvroNamedSchema namedSchema : allNamedSchemas) { - try { - if (!alreadyGeneratedSchemaNames.contains(namedSchema.getFullName())) { - // skip codegen if schema is on classpath and config says to skip - if (config.shouldSkipCodegenIfSchemaOnClasspath() && doesSchemaExistOnClasspath(namedSchema, lookupSchemaSet)) { - continue; - } - - //top level schema - alreadyGeneratedSchemaNames.add(namedSchema.getFullName()); - generatedSpecificClasses.add(generator.generateSpecificClass(namedSchema, generationConfig)); - - // generate internal schemas if not already present - List internalSchemaList = - SpecificRecordGeneratorUtil.getNestedInternalSchemaList(namedSchema); - for (AvroNamedSchema namedInternalSchema : internalSchemaList) { - if (!alreadyGeneratedSchemaNames.contains(namedInternalSchema.getFullName())) { - // skip codegen for nested schemas if schema is on classpath and config says to skip - if (config.shouldSkipCodegenIfSchemaOnClasspath() && doesSchemaExistOnClasspath(namedInternalSchema, - lookupSchemaSet)) { - continue; - } - - generatedSpecificClasses.add(generator.generateSpecificClass(namedInternalSchema, generationConfig)); - alreadyGeneratedSchemaNames.add(namedInternalSchema.getFullName()); - } - } - } - } catch (Exception e) { - throw new RuntimeException("failed to generate class for " + namedSchema.getFullName(), e); - } - } - writeJavaFilesToDisk(generatedSpecificClasses, outputDirectoryPath); - return generatedSpecificClasses.size(); - }).reduce(0, Integer::sum); + LOGGER.info("Parsed {} avsc files in {} millis, {} of which have duplicates", parsedFiles.size(), + parseEnd - scanEnd, duplicates.size()); - long genEnd = System.currentTimeMillis(); - LOGGER.info("generated {} java source files in {} millis", totalGeneratedClasses, genEnd - genStart); + long contextStart = System.currentTimeMillis(); + Set allAvroFiles = Stream.concat(avscFiles.stream(), nonImportableFiles.stream()).collect(Collectors.toSet()); + Set allTopLevelSchemas = + parsedFiles.stream().map(AvscParseResult::getTopLevelSchema).collect(Collectors.toSet()); + OperationContext operationContext = new OperationContext(allTopLevelSchemas, allAvroFiles, lookupSchemaSet); + long contextEnd = System.currentTimeMillis(); + LOGGER.info("Added {} top-level schemas across {} files to context in {} millis", allTopLevelSchemas.size(), + allAvroFiles.size(), contextEnd - contextStart); - Set allAvroFiles = new HashSet<>(avscFiles); - allAvroFiles.addAll(nonImportableFiles); - Set allTopLevelSchemas = new HashSet<>(); - for (AvscParseResult parsedFile : parsedFiles) { - allTopLevelSchemas.add(parsedFile.getTopLevelSchema()); - } - opContext.addParsedSchemas(allTopLevelSchemas, allAvroFiles); + return operationContext; } /** @@ -322,37 +227,9 @@ private Set parseAvscFiles(Set avscFiles, boolean areFile } context.add(fileParseResult, areFilesImportable); - // We skipp adding the referenced parse results to `parsedFiles` + // We skip adding the referenced parse results to `parsedFiles` parsedFiles.add(fileParseResult); } return parsedFiles; } - - private boolean doesSchemaExistOnClasspath(AvroNamedSchema schema, SchemaSet schemaSet) { - if (schemaSet == null) { - return false; - } - - return schemaSet.getByName(schema.getFullName()) != null; - } - - private void writeJavaFilesToDisk(Collection javaFiles, Path outputFolderPath) { - - long writeStart = System.currentTimeMillis(); - - // write out the files we generated - int filesWritten = javaFiles.parallelStream().map(javaFile -> { - try { - javaFile.writeToPath(outputFolderPath); - } catch (Exception e) { - throw new IllegalStateException("while writing file " + javaFile.typeSpec.name, e); - } - - return 1; - }).reduce(0, Integer::sum); - - long writeEnd = System.currentTimeMillis(); - LOGGER.info("wrote out {} generated java source files under {} in {} millis", filesWritten, outputFolderPath, - writeEnd - writeStart); - } } diff --git a/avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/operations/codegen/vanilla/ClasspathSchemaSet.java b/avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/operations/codegen/vanilla/ClasspathSchemaSet.java index f41684483..747757ccf 100644 --- a/avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/operations/codegen/vanilla/ClasspathSchemaSet.java +++ b/avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/operations/codegen/vanilla/ClasspathSchemaSet.java @@ -6,6 +6,7 @@ package com.linkedin.avroutil1.builder.operations.codegen.vanilla; +import com.linkedin.avroutil1.builder.operations.SchemaSet; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.Modifier; diff --git a/avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/operations/codegen/vanilla/FileSystemSchemaSetProvider.java b/avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/operations/codegen/vanilla/FileSystemSchemaSetProvider.java index a4b9deb1a..83eb2cc54 100644 --- a/avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/operations/codegen/vanilla/FileSystemSchemaSetProvider.java +++ b/avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/operations/codegen/vanilla/FileSystemSchemaSetProvider.java @@ -6,6 +6,7 @@ package com.linkedin.avroutil1.builder.operations.codegen.vanilla; +import com.linkedin.avroutil1.builder.operations.SchemaSet; import com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper; import com.linkedin.avroutil1.compatibility.AvroSchemaUtil; import com.linkedin.avroutil1.compatibility.SchemaParseConfiguration; diff --git a/avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/operations/codegen/vanilla/ResolverPathSchemaSet.java b/avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/operations/codegen/vanilla/ResolverPathSchemaSet.java index 9b7229810..84ce14ee5 100644 --- a/avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/operations/codegen/vanilla/ResolverPathSchemaSet.java +++ b/avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/operations/codegen/vanilla/ResolverPathSchemaSet.java @@ -6,6 +6,7 @@ package com.linkedin.avroutil1.builder.operations.codegen.vanilla; +import com.linkedin.avroutil1.builder.operations.SchemaSet; import com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper; import com.linkedin.avroutil1.compatibility.SchemaParseConfiguration; import java.io.File; diff --git a/avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/operations/codegen/vanilla/SchemaSetProvider.java b/avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/operations/codegen/vanilla/SchemaSetProvider.java index 75f8bf6be..4025c2417 100644 --- a/avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/operations/codegen/vanilla/SchemaSetProvider.java +++ b/avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/operations/codegen/vanilla/SchemaSetProvider.java @@ -6,6 +6,9 @@ package com.linkedin.avroutil1.builder.operations.codegen.vanilla; +import com.linkedin.avroutil1.builder.operations.SchemaSet; + + public interface SchemaSetProvider { @Deprecated //for backwards compatibility diff --git a/avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/operations/codegen/vanilla/SimpleSchemaSet.java b/avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/operations/codegen/vanilla/SimpleSchemaSet.java index 2cb2f1e35..0b51b135b 100644 --- a/avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/operations/codegen/vanilla/SimpleSchemaSet.java +++ b/avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/operations/codegen/vanilla/SimpleSchemaSet.java @@ -6,6 +6,7 @@ package com.linkedin.avroutil1.builder.operations.codegen.vanilla; +import com.linkedin.avroutil1.builder.operations.SchemaSet; import java.util.ArrayList; import java.util.HashMap; import java.util.List; diff --git a/avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/operations/codegen/vanilla/VanillaProcessedCodeGenOp.java b/avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/operations/codegen/vanilla/VanillaProcessedCodeGenOp.java index 4bde1ac8e..54a5bf889 100644 --- a/avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/operations/codegen/vanilla/VanillaProcessedCodeGenOp.java +++ b/avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/operations/codegen/vanilla/VanillaProcessedCodeGenOp.java @@ -6,9 +6,10 @@ package com.linkedin.avroutil1.builder.operations.codegen.vanilla; +import com.linkedin.avroutil1.builder.operations.AvroSchemaUtils; import com.linkedin.avroutil1.builder.operations.OperationContext; import com.linkedin.avroutil1.builder.operations.codegen.CodeGenOpConfig; -import com.linkedin.avroutil1.builder.operations.Operation; +import com.linkedin.avroutil1.builder.operations.codegen.OperationContextBuilder; import com.linkedin.avroutil1.builder.operations.codegen.util.AvscFileFinderUtil; import com.linkedin.avroutil1.compatibility.Avro702Checker; import com.linkedin.avroutil1.compatibility.AvscGenerationConfig; @@ -37,18 +38,12 @@ /** * a code generation operation using vanilla avro-compiler + helper post-processing */ -public class VanillaProcessedCodeGenOp implements Operation { +public class VanillaProcessedCodeGenOp implements OperationContextBuilder { private static final Logger LOGGER = LoggerFactory.getLogger(VanillaProcessedCodeGenOp.class); - private final CodeGenOpConfig config; - - public VanillaProcessedCodeGenOp(CodeGenOpConfig config) { - this.config = config; - } - @Override - public void run(OperationContext opContext) throws Exception { - //mkdir any output folders that dont exist + public OperationContext buildOperationContext(CodeGenOpConfig config) throws Exception { + //mkdir any output folders that don't exist if (!config.getOutputSpecificRecordClassesRoot().exists() && !config.getOutputSpecificRecordClassesRoot().mkdirs()) { throw new IllegalStateException("unable to create destination folder " + config.getOutputSpecificRecordClassesRoot()); } @@ -173,9 +168,8 @@ public void run(OperationContext opContext) throws Exception { } } - opContext.addVanillaSchemas(new HashSet<>(schemas), avroFiles); - LOGGER.info("Compilation complete."); + return new OperationContext(AvroSchemaUtils.schemasToAvroSchemas(new HashSet<>(schemas)), avroFiles, null); } private static String fqcnFromSchema(JSONObject schemaJson) throws JSONException { //works for both avsc and pdsc diff --git a/avro-codegen/src/main/java/com/linkedin/avroutil1/codegen/SpecificRecordClassGenerator.java b/avro-codegen/src/main/java/com/linkedin/avroutil1/codegen/SpecificRecordClassGenerator.java index ac7b7f64d..e16070bd6 100644 --- a/avro-codegen/src/main/java/com/linkedin/avroutil1/codegen/SpecificRecordClassGenerator.java +++ b/avro-codegen/src/main/java/com/linkedin/avroutil1/codegen/SpecificRecordClassGenerator.java @@ -83,9 +83,8 @@ void reset() { /*** * Generates Java class for top level schema. * - * @param topLevelSchema - * @param config * @return Java class of top level schema + * * @throws ClassNotFoundException */ public JavaFile generateSpecificClass(AvroNamedSchema topLevelSchema, @@ -106,13 +105,10 @@ public JavaFile generateSpecificClass(AvroNamedSchema topLevelSchema, } } - /*** + /** * Generates Java classes for top level schemas and all internally defined types, excludes references. - * * Not used during codegen. Uncomment for local testing * - * @param topLevelSchema - * @param config * @return List of Java files * @throws ClassNotFoundException */ @@ -150,7 +146,7 @@ private void populateJavaFilesOfInnerNamedSchemasFromRecord(AvroRecordSchema rec while (!schemaQueue.isEmpty()) { AvroSchema fieldSchema = schemaQueue.poll(); - if(fieldSchema instanceof AvroNamedSchema) { + if (fieldSchema instanceof AvroNamedSchema) { String fieldFullName = ((AvroNamedSchema) fieldSchema).getFullName(); if (visitedSchemasFullNames.contains(fieldFullName)) { continue; @@ -272,7 +268,7 @@ protected JavaFile generateSpecificFixed(AvroFixedSchema fixedSchema, SpecificRe .build() ); - addCommonClassComponents(config, classBuilder); + addCommonClassComponents(classBuilder); //add size annotation to class addAndInitializeSizeFieldToClass(classBuilder, fixedSchema); @@ -288,10 +284,8 @@ protected JavaFile generateSpecificFixed(AvroFixedSchema fixedSchema, SpecificRe /*** * Adds getClassSchema, getSchema, DatumReader, DatumWriter - * @param config - * @param classBuilder */ - private void addCommonClassComponents(SpecificRecordGenerationConfig config, TypeSpec.Builder classBuilder) { + private void addCommonClassComponents(TypeSpec.Builder classBuilder) { //add getClassSchema method classBuilder.addMethod(MethodSpec.methodBuilder("getClassSchema") .addModifiers(Modifier.PUBLIC, Modifier.STATIC) @@ -466,13 +460,12 @@ protected JavaFile generateSpecificRecord(AvroRecordSchema recordSchema, Specifi if (!recordSchema.getFields().isEmpty()) { // add all arg constructor if #args < 254 - addAllArgsConstructor(recordSchema, config.getDefaultFieldStringRepresentation(), - config.getDefaultMethodStringRepresentation(), classBuilder); + addAllArgsConstructor(recordSchema, config.getDefaultMethodStringRepresentation(), classBuilder); if (SpecificRecordGeneratorUtil.recordHasSimpleStringField(recordSchema)) { - addAllArgsConstructor(recordSchema, config.getDefaultFieldStringRepresentation(), + addAllArgsConstructor(recordSchema, config.getDefaultMethodStringRepresentation().equals(AvroJavaStringRepresentation.STRING) - ? AvroJavaStringRepresentation.CHAR_SEQUENCE : AvroJavaStringRepresentation.STRING, + ? AvroJavaStringRepresentation.CHAR_SEQUENCE : AvroJavaStringRepresentation.STRING, classBuilder); } @@ -508,30 +501,26 @@ protected JavaFile generateSpecificRecord(AvroRecordSchema recordSchema, Specifi MethodSpec.methodBuilder("hasCustomCoders") .addModifiers(Modifier.PROTECTED) .returns(boolean.class) - .addStatement("return $L", hasCustomCoders(recordSchema)) + .addStatement("return $L", true) .build()); - //customCoders - if (hasCustomCoders(recordSchema)) { - - // customEncode - MethodSpec.Builder customEncodeBuilder = MethodSpec - .methodBuilder("customEncode") - .addParameter(SpecificRecordGeneratorUtil.CLASSNAME_ENCODER, "out") - .addException(IOException.class) - .addModifiers(Modifier.PUBLIC); - addCustomEncodeMethod(customEncodeBuilder, recordSchema, config, sizeValCounter); - classBuilder.addMethod(customEncodeBuilder.build()); + // customEncode + MethodSpec.Builder customEncodeBuilder = MethodSpec + .methodBuilder("customEncode") + .addParameter(SpecificRecordGeneratorUtil.CLASSNAME_ENCODER, "out") + .addException(IOException.class) + .addModifiers(Modifier.PUBLIC); + addCustomEncodeMethod(customEncodeBuilder, recordSchema, config, sizeValCounter); + classBuilder.addMethod(customEncodeBuilder.build()); - //customDecode - MethodSpec.Builder customDecodeBuilder = MethodSpec - .methodBuilder("customDecode") - .addParameter(SpecificRecordGeneratorUtil.CLASSNAME_RESOLVING_DECODER, "in") - .addException(IOException.class) - .addModifiers(Modifier.PUBLIC); - addCustomDecodeMethod(customDecodeBuilder, recordSchema, config, classBuilder, sizeValCounter); - classBuilder.addMethod(customDecodeBuilder.build()); - } + //customDecode + MethodSpec.Builder customDecodeBuilder = MethodSpec + .methodBuilder("customDecode") + .addParameter(SpecificRecordGeneratorUtil.CLASSNAME_RESOLVING_DECODER, "in") + .addException(IOException.class) + .addModifiers(Modifier.PUBLIC); + addCustomDecodeMethod(customDecodeBuilder, recordSchema, config, classBuilder, sizeValCounter); + classBuilder.addMethod(customDecodeBuilder.build()); // Builder TypeSpec.Builder recordBuilder = TypeSpec.classBuilder("Builder"); @@ -553,14 +542,13 @@ protected JavaFile generateSpecificRecord(AvroRecordSchema recordSchema, Specifi } private void addAllArgsConstructor(AvroRecordSchema recordSchema, - AvroJavaStringRepresentation defaultFieldStringRepresentation, AvroJavaStringRepresentation defaultMethodStringRepresentation, TypeSpec.Builder classBuilder) { if(recordSchema.getFields().size() < 254) { MethodSpec.Builder allArgsConstructorBuilder = MethodSpec.constructorBuilder().addModifiers(Modifier.PUBLIC); for (AvroSchemaField field : recordSchema.getFields()) { //if declared schema, use fully qualified class (no import) String escapedFieldName = getFieldNameWithSuffix(field); - allArgsConstructorBuilder.addParameter(getParameterSpecForField(field, defaultMethodStringRepresentation, true)); + allArgsConstructorBuilder.addParameter(getParameterSpecForField(field, defaultMethodStringRepresentation)); if(SpecificRecordGeneratorUtil.isNullUnionOf(AvroType.STRING, field.getSchema())) { allArgsConstructorBuilder.addStatement( "this.$1L = com.linkedin.avroutil1.compatibility.StringConverterUtil.getUtf8($1L)", @@ -623,7 +611,7 @@ private void addAllArgsConstructor(AvroRecordSchema recordSchema, } private String replaceSingleDollarSignWithDouble(String str) { - if(str != null && !str.isEmpty() && str.contains("$")) { + if (str != null && str.contains("$")) { str = SpecificRecordGeneratorUtil.SINGLE_DOLLAR_SIGN_REGEX.matcher(str).replaceAll("\\$\\$"); } return str; @@ -631,7 +619,6 @@ private String replaceSingleDollarSignWithDouble(String str) { private void populateBuilderClassBuilder(TypeSpec.Builder recordBuilder, AvroRecordSchema recordSchema, SpecificRecordGenerationConfig config) throws ClassNotFoundException { - boolean canThrowMissingFieldException = false; recordBuilder.superclass(ClassName.get(CompatibleSpecificRecordBuilderBase.class)); CodeBlock.Builder otherBuilderConstructorFromRecordBlockBuilder = CodeBlock.builder(); CodeBlock.Builder otherBuilderConstructorFromOtherBuilderBlockBuilder = CodeBlock.builder(); @@ -806,13 +793,6 @@ private void populateBuilderClassBuilder(TypeSpec.Builder recordBuilder, AvroRec .addStatement("return record") .endControlFlow(); - if(canThrowMissingFieldException) { - buildMethodCodeBlockBuilder.beginControlFlow( - "catch (com.linkedin.avroutil1.compatibility.exception.AvroUtilMissingFieldException e)") - .addStatement("throw e") - .endControlFlow(); - } - buildMethodCodeBlockBuilder .beginControlFlow("catch ($T e)", Exception.class) .addStatement("throw new com.linkedin.avroutil1.compatibility.exception.AvroUtilException(e)") @@ -948,13 +928,13 @@ private String getFieldNameWithSuffix(AvroSchemaField field) { } private String getMethodNameForFieldWithPrefix(String prefix, String fieldName) { - if (fieldName.length() < 1) { + if (fieldName.isEmpty()) { throw new IllegalArgumentException("FieldName must be longer than 1"); } // Converts a snake_case name to a PascalCase name String pascalCasedField = Arrays.stream(fieldName.split("_")).map(s -> { - if (s.length() < 1) { + if (s.isEmpty()) { return s; } else { return s.substring(0, 1).toUpperCase() + s.substring(1); @@ -1233,11 +1213,6 @@ private String getSerializedCustomDecodeBlock(SpecificRecordGenerationConfig con return SpecificRecordGeneratorUtil.SINGLE_DOLLAR_SIGN_REGEX.matcher(serializedCodeBlock).replaceAll("\\$\\$"); } - - private boolean hasCustomCoders(AvroRecordSchema recordSchema) { - return true; - } - private void addCustomEncodeMethod(MethodSpec.Builder customEncodeBuilder, AvroRecordSchema recordSchema, SpecificRecordGenerationConfig config, Counter sizeValCounter) { for(AvroSchemaField field : recordSchema.getFields()) { @@ -1474,7 +1449,6 @@ private void addPutByIndexMethod(TypeSpec.Builder classBuilder, AvroRecordSchema SpecificRecordGeneratorUtil.getJavaClassForAvroTypeIfApplicable(field.getSchemaOrRef().getSchema().type(), config.getDefaultFieldStringRepresentation(), false); - AvroType fieldType = field.getSchemaOrRef().getSchema().type(); if (config.isUtf8EncodingInPutByIndexEnabled() && SpecificRecordGeneratorUtil.isNullUnionOf(AvroType.STRING, field.getSchema())) { // Default during transition, stores Utf8 in runtime for string fields if (config.getDefaultFieldStringRepresentation().equals(AvroJavaStringRepresentation.STRING)) { @@ -1596,7 +1570,6 @@ private void addGetByIndexMethod(TypeSpec.Builder classBuilder, AvroRecordSchema switchBuilder.beginControlFlow("switch (field)"); for (AvroSchemaField field : recordSchema.getFields()) { String escapedFieldName = getFieldNameWithSuffix(field); - AvroType fieldType = field.getSchema().type(); if (SpecificRecordGeneratorUtil.isNullUnionOf(AvroType.STRING, field.getSchema())) { Class fieldClass = SpecificRecordGeneratorUtil.getJavaClassForAvroTypeIfApplicable(AvroType.STRING, config.getDefaultMethodStringRepresentation(), false); @@ -1701,7 +1674,6 @@ private MethodSpec getOverloadedSetterSpecIfStringField(AvroSchemaField field, } private MethodSpec getSetterMethodSpec(AvroSchemaField field, SpecificRecordGenerationConfig config) { - AvroType fieldType; String escapedFieldName = getFieldNameWithSuffix(field); MethodSpec.Builder methodSpecBuilder = MethodSpec @@ -1709,7 +1681,6 @@ private MethodSpec getSetterMethodSpec(AvroSchemaField field, SpecificRecordGene .addModifiers(Modifier.PUBLIC); if(field.getSchemaOrRef().getSchema() != null) { - fieldType = field.getSchemaOrRef().getSchema().type(); Class fieldClass = SpecificRecordGeneratorUtil.getJavaClassForAvroTypeIfApplicable(field.getSchemaOrRef().getSchema().type(), config.getDefaultMethodStringRepresentation(), false); if (fieldClass != null) { if(fieldClass.equals(CharSequence.class)) { @@ -1729,7 +1700,6 @@ private MethodSpec getSetterMethodSpec(AvroSchemaField field, SpecificRecordGene methodSpecBuilder.addParameter( ClassName.get(field.getSchemaOrRef().getParentNamespace(), field.getSchemaOrRef().getRef()), escapedFieldName); - fieldType = null; } // false if field type is reference @@ -1824,13 +1794,12 @@ private MethodSpec getSetterMethodSpec(AvroSchemaField field, SpecificRecordGene } private MethodSpec getGetterMethodSpec(AvroSchemaField field, SpecificRecordGenerationConfig config) { - AvroType fieldType; String escapedFieldName = getFieldNameWithSuffix(field); MethodSpec.Builder methodSpecBuilder = MethodSpec .methodBuilder(getMethodNameForFieldWithPrefix("get", field.getName())).addModifiers(Modifier.PUBLIC); TypeName typeName = null; - if(field.getSchemaOrRef().getSchema() != null) { - fieldType = field.getSchemaOrRef().getSchema().type(); + if (field.getSchemaOrRef().getSchema() != null) { + AvroType fieldType = field.getSchemaOrRef().getSchema().type(); Class fieldClass = SpecificRecordGeneratorUtil.getJavaClassForAvroTypeIfApplicable(fieldType, config.getDefaultMethodStringRepresentation(), false); if (fieldClass != null) { methodSpecBuilder.returns(fieldClass); @@ -1841,7 +1810,6 @@ private MethodSpec getGetterMethodSpec(AvroSchemaField field, SpecificRecordGene } else { ClassName className = ClassName.get(field.getSchemaOrRef().getParentNamespace(), field.getSchemaOrRef().getRef()); methodSpecBuilder.returns(className); - fieldType = null; } // if fieldRepresentation != methodRepresentation for String field // false if field type is reference @@ -1920,14 +1888,14 @@ private FieldSpec.Builder getFieldSpecBuilder(AvroSchemaField field, SpecificRec return fieldSpecBuilder; } - private ParameterSpec getParameterSpecForField(AvroSchemaField field, AvroJavaStringRepresentation stringRepresentation, boolean useBoxedTypes) { - ParameterSpec.Builder parameterSpecBuilder = null; + private ParameterSpec getParameterSpecForField(AvroSchemaField field, AvroJavaStringRepresentation stringRepresentation) { + final ParameterSpec.Builder parameterSpecBuilder; String escapedFieldName = getFieldNameWithSuffix(field); if(field.getSchemaOrRef().getSchema() != null) { //TODO : Input validation for non-nullable fields since Boxed types are used for Constructors Class fieldClass = SpecificRecordGeneratorUtil.getJavaClassForAvroTypeIfApplicable(field.getSchemaOrRef().getSchema().type(), - stringRepresentation, useBoxedTypes); + stringRepresentation, true); if (fieldClass != null) { parameterSpecBuilder = ParameterSpec.builder(fieldClass, escapedFieldName); } else { diff --git a/avro-codegen/src/main/java/com/linkedin/avroutil1/codegen/SpecificRecordGeneratorUtil.java b/avro-codegen/src/main/java/com/linkedin/avroutil1/codegen/SpecificRecordGeneratorUtil.java index be5a975b1..ad8782948 100644 --- a/avro-codegen/src/main/java/com/linkedin/avroutil1/codegen/SpecificRecordGeneratorUtil.java +++ b/avro-codegen/src/main/java/com/linkedin/avroutil1/codegen/SpecificRecordGeneratorUtil.java @@ -199,8 +199,6 @@ public static TypeName getTypeName(AvroSchema fieldSchema, AvroType avroType, bo /*** * Get nested internal Named schemas for a given topLevel Record schema * Traverses through fields and catches their de-duped - * @param topLevelSchema - * @return */ public static List getNestedInternalSchemaList(AvroNamedSchema topLevelSchema) { @@ -219,8 +217,6 @@ public static List getNestedInternalSchemaList(AvroNamedSchema /*** * Checks if field type can be treated as null union of the given type * - * @param type - * @param schema * @return True if type [null, type] or [type, null] */ public static boolean isNullUnionOf(AvroType type, AvroSchema schema) { @@ -241,7 +237,6 @@ public static boolean isNullUnionOf(AvroType type, AvroSchema schema) { /*** * Handles list , union of list - * @param schema * @return true for List of String and List of Union of String */ public static boolean isListTransformerApplicableForSchema(AvroSchema schema) {