Skip to content

Commit

Permalink
Consolidate unmarshalling and parsing in a single pass (#5643)
Browse files Browse the repository at this point in the history 
* Consolidate unmarshalling and parsing in a single pass

* Address PR comments

* Address PR comments

* Add a test to assert that a non-object literal throws

* Add more testing

* Use Jackson's built-in fast float parsing

* Add customizaion option to enable the fast unmarshalling path

* Fix checkstyle issues

* Account for tests with unset clientConfiguration

* Fix codegen checkstyle
  • Loading branch information
@sugmanue
sugmanue authored Nov 4, 2024
1 parent 440cc9e commit f3e9dc5
Show file tree
Hide file tree
Showing 77 changed files with 5,182 additions and 1,568 deletions.
6 changes: 6 additions & 0 deletions .changes/next-release/feature-AWSSDKforJavav2-70d110c.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
{
"category": "AWS SDK for Java v2",
"contributor": "",
"type": "feature",
"description": "Improve unmarshalling performance of all JSON protocols by unifying parsing with unmarshalling instead of doing one after the other."
}
12 changes: 12 additions & 0 deletions ...n/java/software/amazon/awssdk/codegen/model/config/customization/CustomizationConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -347,6 +347,11 @@ public class CustomizationConfig {
*/
private boolean batchManagerSupported;

/**
* A boolean flag to indicate if the fast unmarshaller code path is enabled.
*/
private boolean enableFastUnmarshaller;

private CustomizationConfig() {
}

Expand Down Expand Up @@ -914,4 +919,11 @@ public void setBatchManagerSupported(boolean batchManagerSupported) {
this.batchManagerSupported = batchManagerSupported;
}

public boolean getEnableFastUnmarshaller() {
return enableFastUnmarshaller;
}

public void setEnableFastUnmarshaller(boolean enableFastUnmarshaller) {
this.enableFastUnmarshaller = enableFastUnmarshaller;
}
}
8 changes: 8 additions & 0 deletions ...gen/src/main/java/software/amazon/awssdk/codegen/poet/builder/BaseClientBuilderClass.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@
import software.amazon.awssdk.identity.spi.IdentityProvider;
import software.amazon.awssdk.identity.spi.IdentityProviders;
import software.amazon.awssdk.identity.spi.TokenIdentity;
import software.amazon.awssdk.protocols.json.internal.unmarshall.SdkClientJsonProtocolAdvancedOption;
import software.amazon.awssdk.regions.ServiceMetadataAdvancedOption;
import software.amazon.awssdk.utils.AttributeMap;
import software.amazon.awssdk.utils.CollectionUtils;
Expand Down Expand Up @@ -479,6 +480,13 @@ private MethodSpec finalizeServiceConfigurationMethod() {
.addCode(" .fipsEnabled(c.get($T.FIPS_ENDPOINT_ENABLED))", AwsClientOption.class)
.addCode(" .build());");

if (model.getMetadata().isJsonProtocol()) {
if (model.getCustomizationConfig().getEnableFastUnmarshaller()) {
builder.addStatement("builder.option($1T.ENABLE_FAST_UNMARSHALLER, true)",
SdkClientJsonProtocolAdvancedOption.class);
}
}

builder.addStatement("return builder.build()");
return builder.build();
}
Expand Down
15 changes: 13 additions & 2 deletions codegen/src/main/java/software/amazon/awssdk/codegen/poet/model/AwsServiceModel.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
Expand Down Expand Up @@ -98,7 +99,6 @@ public TypeSpec poetSpec() {
if (shapeModel.isEventStream()) {
return eventStreamInterfaceSpec();
}

List<FieldSpec> fields = shapeModelSpec.fields();

TypeSpec.Builder specBuilder = TypeSpec.classBuilder(className())
Expand All @@ -110,9 +110,9 @@ public TypeSpec poetSpec() {
.addFields(fields)
.addFields(shapeModelSpec.staticFields())
.addMethod(addModifier(sdkFieldsMethod(), FINAL))
.addMethod(addModifier(sdkFieldNameToFieldMethod(), FINAL))
.addTypes(nestedModelClassTypes());


if (shapeModel.isUnion()) {
specBuilder.addField(unionTypeField());
}
Expand Down Expand Up @@ -316,6 +316,17 @@ private MethodSpec sdkFieldsMethod() {
.build();
}

private MethodSpec sdkFieldNameToFieldMethod() {
ParameterizedTypeName sdkFieldType = ParameterizedTypeName.get(ClassName.get(SdkField.class),
WildcardTypeName.subtypeOf(ClassName.get(Object.class)));
return MethodSpec.methodBuilder("sdkFieldNameToField")
.addModifiers(PUBLIC)
.addAnnotation(Override.class)
.returns(ParameterizedTypeName.get(ClassName.get(Map.class), ClassName.get(String.class), sdkFieldType))
.addCode("return SDK_NAME_TO_FIELD;")
.build();
}

private MethodSpec getterCreator() {
TypeVariableName t = TypeVariableName.get("T");
return MethodSpec.methodBuilder("getter")
Expand Down
13 changes: 13 additions & 0 deletions codegen/src/main/java/software/amazon/awssdk/codegen/poet/model/ModelBuilderSpecs.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Consumer;
import javax.lang.model.element.Modifier;
Expand Down Expand Up @@ -141,6 +142,7 @@ public TypeSpec beanStyleBuilder() {
builderClassBuilder.addMethods(accessors());
builderClassBuilder.addMethod(buildMethod());
builderClassBuilder.addMethod(sdkFieldsMethod());
builderClassBuilder.addMethod(sdkFieldNameToFieldMethod());

if (shapeModel.isUnion()) {
builderClassBuilder.addMethod(handleUnionValueChangeMethod());
Expand Down Expand Up @@ -169,6 +171,17 @@ private MethodSpec sdkFieldsMethod() {
.build();
}

private MethodSpec sdkFieldNameToFieldMethod() {
ParameterizedTypeName sdkFieldType = ParameterizedTypeName.get(ClassName.get(SdkField.class),
WildcardTypeName.subtypeOf(ClassName.get(Object.class)));
return MethodSpec.methodBuilder("sdkFieldNameToField")
.addModifiers(PUBLIC)
.addAnnotation(Override.class)
.returns(ParameterizedTypeName.get(ClassName.get(Map.class), ClassName.get(String.class), sdkFieldType))
.addCode("return SDK_NAME_TO_FIELD;")
.build();
}

private TypeName builderImplSuperClass() {
if (isRequest()) {
return new AwsServiceBaseRequestSpec(intermediateModel).className().nestedClass("BuilderImpl");
Expand Down
49 changes: 43 additions & 6 deletions codegen/src/main/java/software/amazon/awssdk/codegen/poet/model/ShapeModelSpec.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,10 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
import javax.lang.model.element.Modifier;
Expand Down Expand Up @@ -95,6 +98,7 @@ public List<FieldSpec> fields(Modifier... modifiers) {

public Iterable<FieldSpec> staticFields(Modifier... modifiers) {
List<FieldSpec> fields = new ArrayList<>();
Map<String, String> nameToField = new LinkedHashMap<>();
shapeModel.getNonStreamingMembers().stream()
// Exceptions can be members of event stream shapes, need to filter those out of the models
.filter(m -> m.getShape() == null || m.getShape().getShapeType() != ShapeType.Exception)
Expand All @@ -107,6 +111,8 @@ public Iterable<FieldSpec> staticFields(Modifier... modifiers) {
Modifier.PRIVATE, Modifier.STATIC, Modifier.FINAL)
.initializer(sdkFieldInitializer(m))
.build());
String name = m.getHttp().getMarshallLocationName();
nameToField.put(name, namingStrategy.getSdkFieldFieldName(m));
});

ParameterizedTypeName sdkFieldType = ParameterizedTypeName.get(ClassName.get(SdkField.class),
Expand All @@ -115,13 +121,15 @@ public Iterable<FieldSpec> staticFields(Modifier... modifiers) {
fields.add(FieldSpec.builder(ParameterizedTypeName.get(ClassName.get(List.class),
sdkFieldType), "SDK_FIELDS",
Modifier.PRIVATE, Modifier.STATIC, Modifier.FINAL)
.initializer("$T.unmodifiableList($T.asList($L))",
ClassName.get(Collections.class),
ClassName.get(Arrays.class),
fields.stream()
.map(f -> f.name)
.collect(Collectors.joining(",")))
.initializer(sdkFieldsInitializer(fields))
.build());
fields.add(FieldSpec.builder(ParameterizedTypeName.get(ClassName.get(Map.class),
ClassName.get(String.class),
sdkFieldType),
"SDK_NAME_TO_FIELD",
Modifier.PRIVATE, Modifier.STATIC, Modifier.FINAL)
.initializer(memberNameToFieldInitializer(nameToField))
.build());
return fields;
}

Expand Down Expand Up @@ -369,4 +377,33 @@ private CodeBlock constructor(MemberModel m) {
}
}

private CodeBlock sdkFieldsInitializer(List<FieldSpec> fields) {
CodeBlock.Builder builder = CodeBlock.builder();
if (fields.isEmpty()) {
builder.add("$T.emptyList()", Collections.class);
return builder.build();
}
builder.add("$T.unmodifiableList($T.asList($L))",
ClassName.get(Collections.class),
ClassName.get(Arrays.class),
fields.stream()
.map(f -> f.name)
.collect(Collectors.joining(",")));
return builder.build();
}

private CodeBlock memberNameToFieldInitializer(Map<String, String> nameToField) {
CodeBlock.Builder builder = CodeBlock.builder();
if (nameToField.isEmpty()) {
builder.add("$T.emptyMap()", Collections.class);
return builder.build();
}
builder.add("$T.unmodifiableMap(", Collections.class);
builder.add("new $T<$T, $T<?>>() {{\n", HashMap.class, String.class, SdkField.class);
nameToField.forEach((name, field) -> builder.add("put($S, $L);\n", name, field));
builder.add("}}");
builder.add(")");
return builder.build();
}

}
53 changes: 53 additions & 0 deletions codegen/src/test/resources/software/amazon/awssdk/codegen/poet/model/alltypesrequest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
Expand Down Expand Up @@ -450,6 +451,48 @@ SdkField.<SdkBytes> builder(MarshallingType.SDK_BYTES)
POLYMORPHIC_TYPE_WITHOUT_SUB_TYPES_FIELD, ENUM_TYPE_FIELD, UNDERSCORE_NAME_TYPE_FIELD, MY_DOCUMENT_FIELD,
ALL_TYPES_UNION_STRUCTURE_FIELD));

private static final Map<String, SdkField<?>> SDK_NAME_TO_FIELD = Collections
.unmodifiableMap(new HashMap<String, SdkField<?>>() {
{
put("StringMember", STRING_MEMBER_FIELD);
put("IntegerMember", INTEGER_MEMBER_FIELD);
put("BooleanMember", BOOLEAN_MEMBER_FIELD);
put("FloatMember", FLOAT_MEMBER_FIELD);
put("DoubleMember", DOUBLE_MEMBER_FIELD);
put("LongMember", LONG_MEMBER_FIELD);
put("ShortMember", SHORT_MEMBER_FIELD);
put("ByteMember", BYTE_MEMBER_FIELD);
put("SimpleList", SIMPLE_LIST_FIELD);
put("ListOfEnums", LIST_OF_ENUMS_FIELD);
put("ListOfMaps", LIST_OF_MAPS_FIELD);
put("ListOfStructs", LIST_OF_STRUCTS_FIELD);
put("ListOfMapOfEnumToString", LIST_OF_MAP_OF_ENUM_TO_STRING_FIELD);
put("ListOfMapOfStringToStruct", LIST_OF_MAP_OF_STRING_TO_STRUCT_FIELD);
put("MapOfStringToIntegerList", MAP_OF_STRING_TO_INTEGER_LIST_FIELD);
put("MapOfStringToString", MAP_OF_STRING_TO_STRING_FIELD);
put("MapOfStringToSimpleStruct", MAP_OF_STRING_TO_SIMPLE_STRUCT_FIELD);
put("MapOfEnumToEnum", MAP_OF_ENUM_TO_ENUM_FIELD);
put("MapOfEnumToString", MAP_OF_ENUM_TO_STRING_FIELD);
put("MapOfStringToEnum", MAP_OF_STRING_TO_ENUM_FIELD);
put("MapOfEnumToSimpleStruct", MAP_OF_ENUM_TO_SIMPLE_STRUCT_FIELD);
put("MapOfEnumToListOfEnums", MAP_OF_ENUM_TO_LIST_OF_ENUMS_FIELD);
put("MapOfEnumToMapOfStringToEnum", MAP_OF_ENUM_TO_MAP_OF_STRING_TO_ENUM_FIELD);
put("TimestampMember", TIMESTAMP_MEMBER_FIELD);
put("StructWithNestedTimestampMember", STRUCT_WITH_NESTED_TIMESTAMP_MEMBER_FIELD);
put("BlobArg", BLOB_ARG_FIELD);
put("StructWithNestedBlob", STRUCT_WITH_NESTED_BLOB_FIELD);
put("BlobMap", BLOB_MAP_FIELD);
put("ListOfBlobs", LIST_OF_BLOBS_FIELD);
put("RecursiveStruct", RECURSIVE_STRUCT_FIELD);
put("PolymorphicTypeWithSubTypes", POLYMORPHIC_TYPE_WITH_SUB_TYPES_FIELD);
put("PolymorphicTypeWithoutSubTypes", POLYMORPHIC_TYPE_WITHOUT_SUB_TYPES_FIELD);
put("EnumType", ENUM_TYPE_FIELD);
put("Underscore_Name_Type", UNDERSCORE_NAME_TYPE_FIELD);
put("MyDocument", MY_DOCUMENT_FIELD);
put("AllTypesUnionStructure", ALL_TYPES_UNION_STRUCTURE_FIELD);
}
});

private final String stringMember;

private final Integer integerMember;
Expand Down Expand Up @@ -1604,6 +1647,11 @@ public final List<SdkField<?>> sdkFields() {
return SDK_FIELDS;
}

@Override
public final Map<String, SdkField<?>> sdkFieldNameToField() {
return SDK_NAME_TO_FIELD;
}

private static <T> Function<Object, T> getter(Function<AllTypesRequest, T> g) {
return obj -> g.apply((AllTypesRequest) obj);
}
Expand Down Expand Up @@ -3065,5 +3113,10 @@ public AllTypesRequest build() {
public List<SdkField<?>> sdkFields() {
return SDK_FIELDS;
}

@Override
public Map<String, SdkField<?>> sdkFieldNameToField() {
return SDK_NAME_TO_FIELD;
}
}
}
53 changes: 53 additions & 0 deletions codegen/src/test/resources/software/amazon/awssdk/codegen/poet/model/alltypesresponse.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
Expand Down Expand Up @@ -449,6 +450,48 @@ SdkField.<SdkBytes> builder(MarshallingType.SDK_BYTES)
POLYMORPHIC_TYPE_WITHOUT_SUB_TYPES_FIELD, ENUM_TYPE_FIELD, UNDERSCORE_NAME_TYPE_FIELD, MY_DOCUMENT_FIELD,
ALL_TYPES_UNION_STRUCTURE_FIELD));

private static final Map<String, SdkField<?>> SDK_NAME_TO_FIELD = Collections
.unmodifiableMap(new HashMap<String, SdkField<?>>() {
{
put("StringMember", STRING_MEMBER_FIELD);
put("IntegerMember", INTEGER_MEMBER_FIELD);
put("BooleanMember", BOOLEAN_MEMBER_FIELD);
put("FloatMember", FLOAT_MEMBER_FIELD);
put("DoubleMember", DOUBLE_MEMBER_FIELD);
put("LongMember", LONG_MEMBER_FIELD);
put("ShortMember", SHORT_MEMBER_FIELD);
put("ByteMember", BYTE_MEMBER_FIELD);
put("SimpleList", SIMPLE_LIST_FIELD);
put("ListOfEnums", LIST_OF_ENUMS_FIELD);
put("ListOfMaps", LIST_OF_MAPS_FIELD);
put("ListOfStructs", LIST_OF_STRUCTS_FIELD);
put("ListOfMapOfEnumToString", LIST_OF_MAP_OF_ENUM_TO_STRING_FIELD);
put("ListOfMapOfStringToStruct", LIST_OF_MAP_OF_STRING_TO_STRUCT_FIELD);
put("MapOfStringToIntegerList", MAP_OF_STRING_TO_INTEGER_LIST_FIELD);
put("MapOfStringToString", MAP_OF_STRING_TO_STRING_FIELD);
put("MapOfStringToSimpleStruct", MAP_OF_STRING_TO_SIMPLE_STRUCT_FIELD);
put("MapOfEnumToEnum", MAP_OF_ENUM_TO_ENUM_FIELD);
put("MapOfEnumToString", MAP_OF_ENUM_TO_STRING_FIELD);
put("MapOfStringToEnum", MAP_OF_STRING_TO_ENUM_FIELD);
put("MapOfEnumToSimpleStruct", MAP_OF_ENUM_TO_SIMPLE_STRUCT_FIELD);
put("MapOfEnumToListOfEnums", MAP_OF_ENUM_TO_LIST_OF_ENUMS_FIELD);
put("MapOfEnumToMapOfStringToEnum", MAP_OF_ENUM_TO_MAP_OF_STRING_TO_ENUM_FIELD);
put("TimestampMember", TIMESTAMP_MEMBER_FIELD);
put("StructWithNestedTimestampMember", STRUCT_WITH_NESTED_TIMESTAMP_MEMBER_FIELD);
put("BlobArg", BLOB_ARG_FIELD);
put("StructWithNestedBlob", STRUCT_WITH_NESTED_BLOB_FIELD);
put("BlobMap", BLOB_MAP_FIELD);
put("ListOfBlobs", LIST_OF_BLOBS_FIELD);
put("RecursiveStruct", RECURSIVE_STRUCT_FIELD);
put("PolymorphicTypeWithSubTypes", POLYMORPHIC_TYPE_WITH_SUB_TYPES_FIELD);
put("PolymorphicTypeWithoutSubTypes", POLYMORPHIC_TYPE_WITHOUT_SUB_TYPES_FIELD);
put("EnumType", ENUM_TYPE_FIELD);
put("Underscore_Name_Type", UNDERSCORE_NAME_TYPE_FIELD);
put("MyDocument", MY_DOCUMENT_FIELD);
put("AllTypesUnionStructure", ALL_TYPES_UNION_STRUCTURE_FIELD);
}
});

private final String stringMember;

private final Integer integerMember;
Expand Down Expand Up @@ -1603,6 +1646,11 @@ public final List<SdkField<?>> sdkFields() {
return SDK_FIELDS;
}

@Override
public final Map<String, SdkField<?>> sdkFieldNameToField() {
return SDK_NAME_TO_FIELD;
}

private static <T> Function<Object, T> getter(Function<AllTypesResponse, T> g) {
return obj -> g.apply((AllTypesResponse) obj);
}
Expand Down Expand Up @@ -3046,5 +3094,10 @@ public AllTypesResponse build() {
public List<SdkField<?>> sdkFields() {
return SDK_FIELDS;
}

@Override
public Map<String, SdkField<?>> sdkFieldNameToField() {
return SDK_NAME_TO_FIELD;
}
}
}
Loading

0 comments on commit f3e9dc5

Please sign in to comment.