diff --git a/docs/src/main/asciidoc/_configprops.adoc b/docs/src/main/asciidoc/_configprops.adoc index 339570a12..525751d15 100644 --- a/docs/src/main/asciidoc/_configprops.adoc +++ b/docs/src/main/asciidoc/_configprops.adoc @@ -46,14 +46,23 @@ |spring.cloud.aws.s3.accelerate-mode-enabled | | Option to enable using the accelerate endpoint when accessing S3. Accelerate endpoints allow faster transfer of objects by using Amazon CloudFront's globally distributed edge locations. |spring.cloud.aws.s3.checksum-validation-enabled | | Option to disable doing a validation of the checksum of an object stored in S3. |spring.cloud.aws.s3.chunked-encoding-enabled | | Option to enable using chunked encoding when signing the request payload for {@link software.amazon.awssdk.services.s3.model.PutObjectRequest} and {@link software.amazon.awssdk.services.s3.model.UploadPartRequest}. +|spring.cloud.aws.s3.config.enabled | `+++true+++` | Enables S3 Config File import integration. +|spring.cloud.aws.s3.config.reload.max-wait-for-restart | `+++2s+++` | If {@link ReloadStrategy#RESTART_CONTEXT} is configured, maximum waiting time for server restart. +|spring.cloud.aws.s3.config.reload.period | `+++1m+++` | Refresh period for {@link PollingAwsPropertySourceChangeDetector}. +|spring.cloud.aws.s3.config.reload.strategy | | Reload strategy to run when properties change. |spring.cloud.aws.s3.cross-region-enabled | | Enables cross-region bucket access. |spring.cloud.aws.s3.crt.initial-read-buffer-size-in-bytes | | Configure the starting buffer size the client will use to buffer the parts downloaded from S3. Maintain a larger window to keep up a high download throughput; parts cannot download in parallel unless the window is large enough to hold multiple parts. Maintain a smaller window to limit the amount of data buffered in memory. |spring.cloud.aws.s3.crt.max-concurrency | | Specifies the maximum number of S3 connections that should be established during transfer. |spring.cloud.aws.s3.crt.minimum-part-size-in-bytes | | Sets the minimum part size for transfer parts. Decreasing the minimum part size causes multipart transfer to be split into a larger number of smaller parts. Setting this value too low has a negative effect on transfer speeds, causing extra latency and network communication for each part. |spring.cloud.aws.s3.crt.target-throughput-in-gbps | | The target throughput for transfer requests. Higher value means more S3 connections will be opened. Whether the transfer manager can achieve the configured target throughput depends on various factors such as the network bandwidth of the environment and the configured `max-concurrency`. |spring.cloud.aws.s3.enabled | `+++true+++` | Enables S3 integration. +|spring.cloud.aws.s3.encryption.enable-delayed-authentication-mode | `+++false+++` | +|spring.cloud.aws.s3.encryption.enable-legacy-unauthenticated-modes | `+++false+++` | +|spring.cloud.aws.s3.encryption.enable-multipart-put-object | `+++false+++` | +|spring.cloud.aws.s3.encryption.key-id | | |spring.cloud.aws.s3.endpoint | | Overrides the default endpoint. |spring.cloud.aws.s3.path-style-access-enabled | | Option to enable using path style access for accessing S3 objects instead of DNS style access. DNS style access is preferred as it will result in better load balancing when accessing S3. +|spring.cloud.aws.s3.plugin.enable-fallback | `+++false+++` | If set to false if Access Grants does not find/return permissions, S3Client won't try to determine if policies grant access If set to true fallback policies S3/IAM will be evaluated. |spring.cloud.aws.s3.region | | Overrides the default region. |spring.cloud.aws.s3.transfer-manager.follow-symbolic-links | | Specifies whether to follow symbolic links when traversing the file tree in `S3TransferManager#uploadDirectory` operation. |spring.cloud.aws.s3.transfer-manager.max-depth | | Specifies the maximum number of levels of directories to visit in `S3TransferManager#uploadDirectory` operation. diff --git a/docs/src/main/asciidoc/s3.adoc b/docs/src/main/asciidoc/s3.adoc index 006955c11..5eb7101b0 100644 --- a/docs/src/main/asciidoc/s3.adoc +++ b/docs/src/main/asciidoc/s3.adoc @@ -378,6 +378,231 @@ There can be multiple `S3ClientCustomizer` beans present in single application c Note that `S3ClientCustomizer` beans are applied **after** `AwsSyncClientCustomizer` beans and therefore can overwrite previously set configurations. +=== Loading External Configuration + +Just like Spring Boot supports configuring application through `application.properties` stored in the file system, Spring Cloud AWS S3 integration extends this capability with fetching application configuration the S3 bucket through `spring.config.import` property. + +For example, assuming that there is a file `config.properties` in a bucket named `bucket-name`, to include it as Spring Boot configuration, add a following property to `application.properties` or `application.yml`: + +[source,properties] +---- +spring.config.import=aws-s3:/bucket-name/config.properties +---- + +If a file with given name does not exist in S3, application will fail to start. If file configuration is not required for the application, and it should continue to startup even when file configuration is missing, add `optional` before prefix: + +[source,properties] +---- +spring.config.import=optional:aws-s3:/bucket-name/config.properties +---- + +To load multiple files, separate their names with `;`: + +[source,properties] +---- +spring.config.import=aws-s3:/bucket-name/config.properties;/another-name/config.yml +---- + +If some files are required, and other ones are optional, list them as separate entries in `spring.config.import` property: + +[source,properties] +---- +spring.config.import[0]=optional:bucket-name/config.properties +spring.config.import[1]=aws-s3=/another-name/config.yml +---- + +Fetched files configuration can be referenced with `@Value`, bound to `@ConfigurationProperties` classes, or referenced in `application.properties` file. + +`JSON`, Java Properties and `YAML` configuration file formats are supported. + +File resolved with `spring.config.import` can be also referenced in `application.properties`. +For example, with a file `config.json` containing following JSON structure: + +[source,json] +---- +{ + "url": "someUrl" +} +---- + + +`spring.config.import` entry is added to `application.properties`: + +[source, properties] +---- +spring.config.import=aws-s3:/bucket-name/config.json +---- + +File configuration values can be referenced by JSON key names: + +[source, java] +---- +@Value("${url}" +private String url; +---- + +=== Customizing S3Client + +To use custom `S3Client` in `spring.config.import`, provide an implementation of `BootstrapRegistryInitializer`. For example: + +[source,java] +---- +package com.app; + +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; + +import org.springframework.boot.BootstrapRegistry; +import org.springframework.boot.BootstrapRegistryInitializer; + +public class S3ClientBootstrapConfiguration implements BootstrapRegistryInitializer { + + @Override + public void initialize(BootstrapRegistry registry) { + registry.register(S3Client.class, context -> { + AwsCredentialsProvider awsCredentialsProvider = StaticCredentialsProvider.create(AwsBasicCredentials.create("yourAccessKey", "yourSecretKey")); + return S3Client.builder().credentialsProvider(awsCredentialsProvider).region(Region.EU_WEST_2).build(); + }); + } +} +---- + +Note that this class must be listed under `org.springframework.boot.BootstrapRegistryInitializer` key in `META-INF/spring.factories`: + +[source, properties] +---- +org.springframework.boot.BootstrapRegistryInitializer=com.app.S3ClientBootstrapConfiguration +---- + +If you want to use autoconfigured `S3Client` but change underlying SDKClient or `ClientOverrideConfiguration` you will need to register bean of type `S3ClientCustomizer`: +Autoconfiguration will configure `S3Client` Bean with provided values after that, for example: + +[source,java] +---- +package com.app; + +import io.awspring.cloud.autoconfigure.s3.S3ClientCustomizer; +import java.time.Duration; +import org.springframework.boot.BootstrapRegistry; +import org.springframework.boot.BootstrapRegistryInitializer; +import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; +import software.amazon.awssdk.http.SdkHttpClient; +import software.amazon.awssdk.http.apache.ApacheHttpClient; +import software.amazon.awssdk.services.s3.S3Client; + +class S3ClientBootstrapConfiguration implements BootstrapRegistryInitializer { + + @Override + public void initialize(BootstrapRegistry registry) { + registry.register(S3ClientCustomizer.class, context -> (builder -> { + builder.overrideConfiguration(builder.overrideConfiguration().copy(c -> { + c.apiCallTimeout(Duration.ofMillis(2001)); + })); + })); + } +} +---- + +=== `PropertySource` Reload + +Some applications may need to detect changes on external property sources and update their internal status to reflect the new configuration. +The reload feature of Spring Cloud AWS S3 config import integration is able to trigger an application reload when a related file value changes. + +By default, this feature is disabled. You can enable it by using the `spring.cloud.aws.s3.config.reload.strategy` configuration property (for example, in the `application.properties` file) and adding following dependencies. + +[source,xml] +---- + + org.springframework.boot + spring-boot-starter-actuator + + + org.springframework.cloud + spring-cloud-commons + + + org.springframework.cloud + spring-cloud-context + +---- + +The following levels of reload are supported (by setting the `spring.cloud.aws.s3.config.reload.strategy` property): + +* `refresh` (default): Only configuration beans annotated with `@ConfigurationProperties` or `@RefreshScope` are reloaded. +This reload level leverages the refresh feature of Spring Cloud Context. + +* `restart_context`: the whole Spring `ApplicationContext` is gracefully restarted. Beans are recreated with the new configuration. +In order for the restart context functionality to work properly you must enable and expose the restart actuator endpoint +[source,yaml] +==== +---- +management: + endpoint: + restart: + enabled: true + endpoints: + web: + exposure: + include: restart +---- +==== + +Assuming that the reload feature is enabled with default settings (`refresh` mode), the following bean is refreshed when the file changes: + +==== +[java, source] +---- +@Configuration +@ConfigurationProperties(prefix = "bean") +public class MyConfig { + + private String message = "a message that can be changed live"; + + // getter and setters + +} +---- +==== + +To see that changes effectively happen, you can create another bean that prints the message periodically, as follows + +==== +[source,java] +---- +@Component +public class MyBean { + + @Autowired + private MyConfig config; + + @Scheduled(fixedDelay = 5000) + public void hello() { + System.out.println("The message is: " + config.getMessage()); + } +} +---- +==== + +The reload feature periodically re-creates the configuration from S3 file to see if it has changed. +You can configure the polling period by using the `spring.cloud.aws.s3.config.reload.period` (default value is 1 minute). + +=== Configuration + +The Spring Boot Starter for S3 provides the following configuration options: + +[cols="2,3,1,1"] +|=== +| Name | Description | Required | Default value +| `spring.cloud.aws.s3.config.enabled` | Enables the S3 config import integration. | No | `true` +| `spring.cloud.aws.s3.config.reload.strategy` | `Enum` | `refresh` | The strategy to use when firing a reload (`refresh`, `restart_context`) +| `spring.cloud.aws.s3.config.reload.period` | `Duration`| `15s` | The period for verifying changes +| `spring.cloud.aws.s3.config.reload.max-wait-time-for-restart` | `Duration`| `2s` | The maximum time between the detection of changes in property source and the application context restart when `restart_context` strategy is used. +|=== + + === IAM Permissions Following IAM permissions are required by Spring Cloud AWS: diff --git a/pom.xml b/pom.xml index 7d3d2c54f..ce3bf6fc9 100644 --- a/pom.xml +++ b/pom.xml @@ -58,7 +58,7 @@ spring-cloud-aws-test spring-cloud-aws-modulith docs - + diff --git a/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/config/s3/S3ConfigDataLoader.java b/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/config/s3/S3ConfigDataLoader.java new file mode 100644 index 000000000..470db3f1b --- /dev/null +++ b/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/config/s3/S3ConfigDataLoader.java @@ -0,0 +1,72 @@ +/* + * Copyright 2013-2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.awspring.cloud.autoconfigure.config.s3; + +import io.awspring.cloud.autoconfigure.config.BootstrapLoggingHelper; +import io.awspring.cloud.s3.config.S3PropertySource; +import java.util.Collections; +import java.util.Map; +import org.springframework.boot.context.config.ConfigData; +import org.springframework.boot.context.config.ConfigDataLoader; +import org.springframework.boot.context.config.ConfigDataLoaderContext; +import org.springframework.boot.context.config.ConfigDataResourceNotFoundException; +import org.springframework.boot.logging.DeferredLogFactory; +import org.springframework.core.env.MapPropertySource; +import org.springframework.lang.Nullable; +import software.amazon.awssdk.services.s3.S3Client; + +/** + * Loads config data from AWS S3. + * + * @author Kunal Varpe + * @since 3.3.0 + */ +public class S3ConfigDataLoader implements ConfigDataLoader { + + public S3ConfigDataLoader(DeferredLogFactory logFactory) { + BootstrapLoggingHelper.reconfigureLoggers(logFactory, "io.awspring.cloud.s3.config.S3PropertySource", + "io.awspring.cloud.autoconfigure.config.s3.S3PropertySources"); + } + + @Override + @Nullable + public ConfigData load(ConfigDataLoaderContext context, S3ConfigDataResource resource) { + try { + // resource is disabled if s3 integration is disabled via + // spring.cloud.aws.s3.config.enabled=false + if (resource.isEnabled()) { + S3Client s3Client = context.getBootstrapContext().get(S3Client.class); + S3PropertySource propertySource = resource.getPropertySources() + .createPropertySource(resource.getContext(), resource.isOptional(), s3Client); + if (propertySource != null) { + return new ConfigData(Collections.singletonList(propertySource)); + } + else { + return null; + } + } + else { + // create dummy empty config data + return new ConfigData(Collections.singletonList(new MapPropertySource("aws-s3:" + context, Map.of()))); + } + } + catch (Exception e) { + throw new ConfigDataResourceNotFoundException(resource, e); + } + + } + +} diff --git a/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/config/s3/S3ConfigDataLocationResolver.java b/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/config/s3/S3ConfigDataLocationResolver.java new file mode 100644 index 000000000..ba6106ca9 --- /dev/null +++ b/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/config/s3/S3ConfigDataLocationResolver.java @@ -0,0 +1,126 @@ +/* + * Copyright 2013-2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.awspring.cloud.autoconfigure.config.s3; + +import io.awspring.cloud.autoconfigure.AwsSyncClientCustomizer; +import io.awspring.cloud.autoconfigure.config.AbstractAwsConfigDataLocationResolver; +import io.awspring.cloud.autoconfigure.core.*; +import io.awspring.cloud.autoconfigure.s3.S3ClientCustomizer; +import io.awspring.cloud.autoconfigure.s3.properties.S3Properties; +import java.util.ArrayList; +import java.util.List; +import org.apache.commons.logging.Log; +import org.springframework.boot.BootstrapContext; +import org.springframework.boot.context.config.ConfigDataLocation; +import org.springframework.boot.context.config.ConfigDataLocationNotFoundException; +import org.springframework.boot.context.config.ConfigDataLocationResolverContext; +import org.springframework.boot.context.config.Profiles; +import org.springframework.boot.context.properties.bind.Bindable; +import org.springframework.boot.context.properties.bind.Binder; +import org.springframework.boot.logging.DeferredLogFactory; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3ClientBuilder; + +/** + * Resolves config data locations in AWS S3. + * + * @author Kunal Varpe + * @author Matej Nedic + * @since 3.3.0 + */ +public class S3ConfigDataLocationResolver extends AbstractAwsConfigDataLocationResolver { + + /** + * AWS S3 Config Data prefix. + */ + public static final String PREFIX = "aws-s3:"; + + private final Log log; + + public S3ConfigDataLocationResolver(DeferredLogFactory deferredLogFactory) { + this.log = deferredLogFactory.getLog(S3ConfigDataLocationResolver.class); + } + + @Override + protected String getPrefix() { + return PREFIX; + } + + @Override + public List resolveProfileSpecific(ConfigDataLocationResolverContext resolverContext, + ConfigDataLocation location, Profiles profiles) throws ConfigDataLocationNotFoundException { + + S3Properties s3Properties = loadProperties(resolverContext.getBinder()); + List locations = new ArrayList<>(); + S3PropertySources propertySources = new S3PropertySources(); + List contexts = getCustomContexts(location.getNonPrefixedValue(PREFIX)); + + if (s3Properties.getConfig().isEnabled()) { + registerBean(resolverContext, AwsProperties.class, loadAwsProperties(resolverContext.getBinder())); + registerBean(resolverContext, S3Properties.class, s3Properties); + registerBean(resolverContext, CredentialsProperties.class, + loadCredentialsProperties(resolverContext.getBinder())); + registerBean(resolverContext, RegionProperties.class, loadRegionProperties(resolverContext.getBinder())); + + registerAndPromoteBean(resolverContext, S3Client.class, this::createS3Client); + + contexts.forEach(propertySourceContext -> locations + .add(new S3ConfigDataResource(propertySourceContext, location.isOptional(), propertySources))); + + if (!location.isOptional() && locations.isEmpty()) { + throw new S3KeysMissingException( + "No S3 keys provided in `spring.config.import=aws-s3:` configuration."); + } + } + else { + // create dummy resources with enabled flag set to false, + // because returned locations cannot be empty + contexts.forEach(propertySourceContext -> locations.add( + new S3ConfigDataResource(propertySourceContext, location.isOptional(), false, propertySources))); + } + return locations; + } + + private S3Client createS3Client(BootstrapContext context) { + S3ClientBuilder builder = configure(S3Client.builder(), context.get(S3Properties.class), context); + + try { + S3ClientCustomizer s3ClientCustomizer = context.get(S3ClientCustomizer.class); + if (s3ClientCustomizer != null) { + s3ClientCustomizer.customize(builder); + } + } + catch (IllegalStateException e) { + log.debug("Bean of type AwsSyncClientCustomizer is not registered: " + e.getMessage()); + } + + try { + AwsSyncClientCustomizer awsSyncClientCustomizer = context.get(AwsSyncClientCustomizer.class); + if (awsSyncClientCustomizer != null) { + awsSyncClientCustomizer.customize(builder); + } + } + catch (IllegalStateException e) { + log.debug("Bean of type AwsSyncClientCustomizer is not registered: " + e.getMessage()); + } + + return builder.build(); + } + + protected S3Properties loadProperties(Binder binder) { + return binder.bind(S3Properties.PREFIX, Bindable.of(S3Properties.class)).orElseGet(S3Properties::new); + } +} diff --git a/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/config/s3/S3ConfigDataResource.java b/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/config/s3/S3ConfigDataResource.java new file mode 100644 index 000000000..74a6c391b --- /dev/null +++ b/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/config/s3/S3ConfigDataResource.java @@ -0,0 +1,96 @@ +/* + * Copyright 2013-2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.awspring.cloud.autoconfigure.config.s3; + +import java.util.Objects; +import org.springframework.boot.context.config.ConfigDataResource; +import org.springframework.core.style.ToStringCreator; + +/** + * Config data resource for AWS S3 integration. + * + * @author Kunal Varpe + * @since 3.3.0 + */ +public class S3ConfigDataResource extends ConfigDataResource { + + private final String context; + + private final boolean enabled; + + private final boolean optional; + + private final S3PropertySources propertySources; + + public S3ConfigDataResource(String context, boolean optional, S3PropertySources propertySources) { + this(context, optional, true, propertySources); + } + + public S3ConfigDataResource(String context, boolean optional, boolean enabled, S3PropertySources propertySources) { + this.context = context; + this.optional = optional; + this.propertySources = propertySources; + this.enabled = enabled; + } + + /** + * Returns context which is equal to S3 bucket and property file. + * @return the context + */ + public String getContext() { + return this.context; + } + + /** + * If application startup should fail when secret cannot be loaded or does not exist. + * @return is optional + */ + public boolean isOptional() { + return this.optional; + } + + public boolean isEnabled() { + return enabled; + } + + public S3PropertySources getPropertySources() { + return this.propertySources; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + S3ConfigDataResource that = (S3ConfigDataResource) o; + return this.optional == that.optional && this.context.equals(that.context); + } + + @Override + public int hashCode() { + return Objects.hash(this.optional, this.context); + } + + @Override + public String toString() { + return new ToStringCreator(this).append("context", context).append("optional", optional).toString(); + + } + +} diff --git a/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/config/s3/S3KeysMissingException.java b/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/config/s3/S3KeysMissingException.java new file mode 100644 index 000000000..54bcc6cb1 --- /dev/null +++ b/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/config/s3/S3KeysMissingException.java @@ -0,0 +1,31 @@ +/* + * Copyright 2013-2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.awspring.cloud.autoconfigure.config.s3; + +/** + * Thrown when configuration provided to ConfigDataLoader is missing S3 keys, for example + * `spring.config.import=aws-s3:`. + * + * @author Kunal Varpe + * @since 3.3.0 + */ +public class S3KeysMissingException extends RuntimeException { + + public S3KeysMissingException(String message) { + super(message); + } + +} diff --git a/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/config/s3/S3MissingKeysFailureAnalyzer.java b/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/config/s3/S3MissingKeysFailureAnalyzer.java new file mode 100644 index 000000000..2c6f78418 --- /dev/null +++ b/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/config/s3/S3MissingKeysFailureAnalyzer.java @@ -0,0 +1,37 @@ +/* + * Copyright 2013-2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.awspring.cloud.autoconfigure.config.s3; + +import org.springframework.boot.diagnostics.AbstractFailureAnalyzer; +import org.springframework.boot.diagnostics.FailureAnalysis; + +/** + * An {@link AbstractFailureAnalyzer} that performs analysis of a S3 configuration failure caused by not providing a S3 + * key to `spring.config.import` property. + * + * @author Kunal Varpe + * @since 3.3.0 + */ +public class S3MissingKeysFailureAnalyzer extends AbstractFailureAnalyzer { + + @Override + protected FailureAnalysis analyze(Throwable rootFailure, S3KeysMissingException cause) { + return new FailureAnalysis("Could not import properties from AWS S3: " + cause.getMessage(), + "Consider providing keys, for example `spring.config.import=aws-s3:bucket/application.properties`", + cause); + } + +} diff --git a/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/config/s3/S3PropertySources.java b/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/config/s3/S3PropertySources.java new file mode 100644 index 000000000..9fd3ed4e6 --- /dev/null +++ b/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/config/s3/S3PropertySources.java @@ -0,0 +1,70 @@ +/* + * Copyright 2013-2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.awspring.cloud.autoconfigure.config.s3; + +import io.awspring.cloud.s3.config.S3PropertySource; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; +import software.amazon.awssdk.services.s3.S3Client; + +/** + * Provides prefix config import support. + * + * @author Kunal Varpe + * @since 3.3.0 + */ +public class S3PropertySources { + + private static Log LOG = LogFactory.getLog(S3PropertySources.class); + + /** + * Creates property source for given context. + * @param context property source context equivalent to s3 bucket file + * @param optional if creating context should fail with exception if s3 bucket file cannot be loaded + * @param client S3 client + * @return a property source or null if s3 bucket file could not be loaded and optional is set to true + */ + @Nullable + public S3PropertySource createPropertySource(String context, boolean optional, S3Client client) { + Assert.notNull(context, "context is required"); + Assert.notNull(client, "S3Client is required"); + + LOG.info("Loading properties from AWS S3 object: " + context + ", optional: " + optional); + try { + S3PropertySource propertySource = new S3PropertySource(context, client); + propertySource.init(); + return propertySource; + } + catch (Exception e) { + LOG.warn("Unable to load AWS S3 object from " + context + ". " + e.getMessage()); + if (!optional) { + throw new AwsS3PropertySourceNotFoundException(e); + } + } + return null; + } + + static class AwsS3PropertySourceNotFoundException extends RuntimeException { + + AwsS3PropertySourceNotFoundException(Exception source) { + super(source); + } + + } + +} diff --git a/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/config/s3/S3ReloadAutoConfiguration.java b/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/config/s3/S3ReloadAutoConfiguration.java new file mode 100644 index 000000000..21362f3d6 --- /dev/null +++ b/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/config/s3/S3ReloadAutoConfiguration.java @@ -0,0 +1,87 @@ +/* + * Copyright 2013-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.awspring.cloud.autoconfigure.config.s3; + +import io.awspring.cloud.autoconfigure.config.reload.ConfigurationChangeDetector; +import io.awspring.cloud.autoconfigure.config.reload.ConfigurationUpdateStrategy; +import io.awspring.cloud.autoconfigure.config.reload.PollingAwsPropertySourceChangeDetector; +import io.awspring.cloud.autoconfigure.s3.properties.S3Properties; +import io.awspring.cloud.s3.config.S3PropertySource; +import java.util.Optional; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.actuate.autoconfigure.endpoint.EndpointAutoConfiguration; +import org.springframework.boot.actuate.autoconfigure.info.InfoEndpointAutoConfiguration; +import org.springframework.boot.autoconfigure.AutoConfiguration; +import org.springframework.boot.autoconfigure.AutoConfigureAfter; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.cloud.autoconfigure.RefreshAutoConfiguration; +import org.springframework.cloud.autoconfigure.RefreshEndpointAutoConfiguration; +import org.springframework.cloud.commons.util.TaskSchedulerWrapper; +import org.springframework.cloud.context.refresh.ContextRefresher; +import org.springframework.cloud.context.restart.RestartEndpoint; +import org.springframework.context.annotation.Bean; +import org.springframework.core.env.ConfigurableEnvironment; +import org.springframework.scheduling.TaskScheduler; +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; + +/** + * {@link AutoConfiguration Auto-Configuration} for reloading properties from S3. + * + * @author Matej Nedic + * @since 3.3.0 + */ +@AutoConfiguration +@EnableConfigurationProperties(S3Properties.class) +@ConditionalOnClass({ EndpointAutoConfiguration.class, RestartEndpoint.class, ContextRefresher.class }) +@AutoConfigureAfter({ InfoEndpointAutoConfiguration.class, RefreshEndpointAutoConfiguration.class, + RefreshAutoConfiguration.class }) +@ConditionalOnProperty(value = S3Properties.PREFIX + ".config.reload.strategy") +@ConditionalOnBean(ContextRefresher.class) +public class S3ReloadAutoConfiguration { + + @Bean("s3TaskScheduler") + @ConditionalOnMissingBean + public TaskSchedulerWrapper taskScheduler() { + ThreadPoolTaskScheduler threadPoolTaskScheduler = new ThreadPoolTaskScheduler(); + + threadPoolTaskScheduler.setThreadNamePrefix("spring-cloud-aws-s3-ThreadPoolTaskScheduler-"); + threadPoolTaskScheduler.setDaemon(true); + + return new TaskSchedulerWrapper<>(threadPoolTaskScheduler); + } + + @Bean("s3ConfigurationUpdateStrategy") + @ConditionalOnMissingBean(name = "s3ConfigurationUpdateStrategy") + public ConfigurationUpdateStrategy s3ConfigurationUpdateStrategy(S3Properties properties, + Optional restarter, ContextRefresher refresher) { + return ConfigurationUpdateStrategy.create(properties.getConfig().getReload(), refresher, restarter); + } + + @Bean + @ConditionalOnBean(ConfigurationUpdateStrategy.class) + public ConfigurationChangeDetector s3PollingAwsPropertySourceChangeDetector( + S3Properties properties, @Qualifier("s3ConfigurationUpdateStrategy") ConfigurationUpdateStrategy strategy, + @Qualifier("s3TaskScheduler") TaskSchedulerWrapper taskScheduler, + ConfigurableEnvironment environment) { + + return new PollingAwsPropertySourceChangeDetector<>(properties.getConfig().getReload(), S3PropertySource.class, + strategy, taskScheduler.getTaskScheduler(), environment); + } +} diff --git a/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/config/s3/package-info.java b/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/config/s3/package-info.java new file mode 100644 index 000000000..84df00339 --- /dev/null +++ b/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/config/s3/package-info.java @@ -0,0 +1,22 @@ +/* + * Copyright 2013-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * {@link org.springframework.boot.context.config.ConfigDataLoader} implementation for AWS S3. + */ +@org.springframework.lang.NonNullApi +@org.springframework.lang.NonNullFields +package io.awspring.cloud.autoconfigure.config.s3; diff --git a/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/s3/properties/S3ConfigProperties.java b/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/s3/properties/S3ConfigProperties.java new file mode 100644 index 000000000..67ac8a3f7 --- /dev/null +++ b/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/s3/properties/S3ConfigProperties.java @@ -0,0 +1,49 @@ +/* + * Copyright 2013-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.awspring.cloud.autoconfigure.s3.properties; + +import io.awspring.cloud.autoconfigure.config.reload.ReloadProperties; +import org.springframework.boot.context.properties.NestedConfigurationProperty; + +public class S3ConfigProperties { + + /** + * Properties related to configuration reload. + */ + @NestedConfigurationProperty + private ReloadProperties reload = new ReloadProperties(); + + /** + * Enables S3 Config File import integration. + */ + private boolean enabled = true; + + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public ReloadProperties getReload() { + return reload; + } + + public void setReload(ReloadProperties reload) { + this.reload = reload; + } +} diff --git a/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/s3/properties/S3Properties.java b/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/s3/properties/S3Properties.java index 3f528ca51..8d45c6270 100644 --- a/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/s3/properties/S3Properties.java +++ b/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/s3/properties/S3Properties.java @@ -112,6 +112,21 @@ public void setEncryption(S3EncryptionProperties encryption) { this.encryption = encryption; } + /** + * Properties related to configuration reload. + */ + + @NestedConfigurationProperty + private S3ConfigProperties config = new S3ConfigProperties(); + + public S3ConfigProperties getConfig() { + return config; + } + + public void setConfig(S3ConfigProperties config) { + this.config = config; + } + @Nullable public Boolean getAccelerateModeEnabled() { return this.accelerateModeEnabled; diff --git a/spring-cloud-aws-autoconfigure/src/main/resources/META-INF/spring.factories b/spring-cloud-aws-autoconfigure/src/main/resources/META-INF/spring.factories index 598fbf44f..171dee2ad 100644 --- a/spring-cloud-aws-autoconfigure/src/main/resources/META-INF/spring.factories +++ b/spring-cloud-aws-autoconfigure/src/main/resources/META-INF/spring.factories @@ -1,14 +1,17 @@ # ConfigData Location Resolvers org.springframework.boot.context.config.ConfigDataLocationResolver=\ io.awspring.cloud.autoconfigure.config.parameterstore.ParameterStoreConfigDataLocationResolver,\ -io.awspring.cloud.autoconfigure.config.secretsmanager.SecretsManagerConfigDataLocationResolver +io.awspring.cloud.autoconfigure.config.secretsmanager.SecretsManagerConfigDataLocationResolver,\ +io.awspring.cloud.autoconfigure.config.s3.S3ConfigDataLocationResolver # ConfigData Loaders org.springframework.boot.context.config.ConfigDataLoader=\ io.awspring.cloud.autoconfigure.config.parameterstore.ParameterStoreConfigDataLoader,\ -io.awspring.cloud.autoconfigure.config.secretsmanager.SecretsManagerConfigDataLoader +io.awspring.cloud.autoconfigure.config.secretsmanager.SecretsManagerConfigDataLoader,\ +io.awspring.cloud.autoconfigure.config.s3.S3ConfigDataLoader # Failure Analyzers org.springframework.boot.diagnostics.FailureAnalyzer=\ io.awspring.cloud.autoconfigure.config.parameterstore.ParameterStoreMissingKeysFailureAnalyzer, \ -io.awspring.cloud.autoconfigure.config.secretsmanager.SecretsManagerMissingKeysFailureAnalyzer +io.awspring.cloud.autoconfigure.config.secretsmanager.SecretsManagerMissingKeysFailureAnalyzer,\ +io.awspring.cloud.autoconfigure.config.s3.S3MissingKeysFailureAnalyzer diff --git a/spring-cloud-aws-autoconfigure/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/spring-cloud-aws-autoconfigure/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports index a52f7b419..657cbf245 100644 --- a/spring-cloud-aws-autoconfigure/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports +++ b/spring-cloud-aws-autoconfigure/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports @@ -14,3 +14,4 @@ io.awspring.cloud.autoconfigure.config.secretsmanager.SecretsManagerReloadAutoCo io.awspring.cloud.autoconfigure.config.secretsmanager.SecretsManagerAutoConfiguration io.awspring.cloud.autoconfigure.config.parameterstore.ParameterStoreReloadAutoConfiguration io.awspring.cloud.autoconfigure.config.parameterstore.ParameterStoreAutoConfiguration +io.awspring.cloud.autoconfigure.config.s3.S3ReloadAutoConfiguration diff --git a/spring-cloud-aws-autoconfigure/src/test/java/io/awspring/cloud/autoconfigure/config/s3/S3ConfigDataLoaderIntegrationTests.java b/spring-cloud-aws-autoconfigure/src/test/java/io/awspring/cloud/autoconfigure/config/s3/S3ConfigDataLoaderIntegrationTests.java new file mode 100644 index 000000000..99890c2ef --- /dev/null +++ b/spring-cloud-aws-autoconfigure/src/test/java/io/awspring/cloud/autoconfigure/config/s3/S3ConfigDataLoaderIntegrationTests.java @@ -0,0 +1,237 @@ +/* + * Copyright 2013-2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.awspring.cloud.autoconfigure.config.s3; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.*; +import static org.testcontainers.containers.localstack.LocalStackContainer.Service.S3; +import static org.testcontainers.shaded.org.awaitility.Awaitility.await; + +import io.awspring.cloud.autoconfigure.AwsSyncClientCustomizer; +import io.awspring.cloud.autoconfigure.ConfiguredAwsClient; +import io.awspring.cloud.autoconfigure.s3.S3ClientCustomizer; +import java.io.IOException; +import java.time.Duration; +import java.util.Map; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.springframework.boot.BootstrapRegistry; +import org.springframework.boot.BootstrapRegistryInitializer; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.ConfigurableApplicationContext; +import org.testcontainers.containers.localstack.LocalStackContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.shaded.com.fasterxml.jackson.core.JsonProcessingException; +import org.testcontainers.shaded.com.fasterxml.jackson.databind.ObjectMapper; +import org.testcontainers.utility.DockerImageName; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.client.builder.SdkDefaultClientBuilder; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.http.SdkHttpClient; +import software.amazon.awssdk.http.apache.ApacheHttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; + +/** + * Integration tests for loading configuration properties from AWS S3. + * + * @author Kunal Varpe + * @author Matej Nedic + */ + +@Testcontainers +public class S3ConfigDataLoaderIntegrationTests { + private static final String YAML_TYPE = "application/x-yaml"; + private static final String YAML_TYPE_ALTERNATIVE = "text/yaml"; + private static final String TEXT_TYPE = "text/plain"; + private static final String JSON_TYPE = "application/json"; + private static String BUCKET = "test-bucket"; + @Container + static LocalStackContainer localstack = new LocalStackContainer( + DockerImageName.parse("localstack/localstack:1.4.0")).withServices(S3).withReuse(true); + + @BeforeAll + static void beforeAll() { + createBucket(); + } + + @Test + void resolvesPropertyFromS3() { + SpringApplication application = new SpringApplication(S3ConfigDataLoaderIntegrationTests.App.class); + application.setWebApplicationType(WebApplicationType.NONE); + uploadFileToBucket("key1=value1", "application.properties", TEXT_TYPE); + + try (ConfigurableApplicationContext context = runApplication(application, + "aws-s3:test-bucket/application.properties")) { + assertThat(context.getEnvironment().getProperty("key1")).isEqualTo("value1"); + } + } + + @Test + void resolvesPropertyFromS3ComplexPath() { + SpringApplication application = new SpringApplication(S3ConfigDataLoaderIntegrationTests.App.class); + application.setWebApplicationType(WebApplicationType.NONE); + uploadFileToBucket("key1=value1", "myPath/unusual/application.properties", TEXT_TYPE); + + try (ConfigurableApplicationContext context = runApplication(application, + "aws-s3:test-bucket/myPath/unusual/application.properties")) { + assertThat(context.getEnvironment().getProperty("key1")).isEqualTo("value1"); + } + } + + @Test + void resolvesYamlFromS3() { + SpringApplication application = new SpringApplication(S3ConfigDataLoaderIntegrationTests.App.class); + application.setWebApplicationType(WebApplicationType.NONE); + uploadFileToBucket("key1: value1", "application.yaml", YAML_TYPE); + + try (ConfigurableApplicationContext context = runApplication(application, + "aws-s3:test-bucket/application.yaml")) { + assertThat(context.getEnvironment().getProperty("key1")).isEqualTo("value1"); + } + } + + @Test + void resolvesYamlAlternative() { + SpringApplication application = new SpringApplication(S3ConfigDataLoaderIntegrationTests.App.class); + application.setWebApplicationType(WebApplicationType.NONE); + uploadFileToBucket("key1: value1", "test.yaml", YAML_TYPE_ALTERNATIVE); + + try (ConfigurableApplicationContext context = runApplication(application, "aws-s3:test-bucket/test.yaml")) { + assertThat(context.getEnvironment().getProperty("key1")).isEqualTo("value1"); + } + } + + @Test + void resolveJson() throws JsonProcessingException { + SpringApplication application = new SpringApplication(S3ConfigDataLoaderIntegrationTests.App.class); + application.setWebApplicationType(WebApplicationType.NONE); + uploadFileToBucket(new ObjectMapper().writeValueAsString(Map.of("key1", "value1")), "application.json", + JSON_TYPE); + + try (ConfigurableApplicationContext context = runApplication(application, + "aws-s3:test-bucket/application.json")) { + assertThat(context.getEnvironment().getProperty("key1")).isEqualTo("value1"); + } + } + + @Test + void clientIsConfiguredWithCustomizerProvidedToBootstrapRegistry() throws JsonProcessingException { + SpringApplication application = new SpringApplication(S3ConfigDataLoaderIntegrationTests.App.class); + application.setWebApplicationType(WebApplicationType.NONE); + application.addBootstrapRegistryInitializer(new S3ConfigDataLoaderIntegrationTests.CustomizerConfiguration()); + uploadFileToBucket(new ObjectMapper().writeValueAsString(Map.of("key1", "value1")), "application.json", + JSON_TYPE); + + try (ConfigurableApplicationContext context = runApplication(application, + "aws-s3:test-bucket/application.json")) { + ConfiguredAwsClient client = new ConfiguredAwsClient(context.getBean(S3Client.class)); + assertThat(client.getApiCallTimeout()).isEqualTo(Duration.ofMillis(2001)); + assertThat(client.getSyncHttpClient()).isInstanceOf(SdkDefaultClientBuilder.NonManagedSdkHttpClient.class); + assertThat(client.getSyncHttpClient().clientName()).isEqualTo("mock-client"); + } + } + + @Test + void reloadPropertiesFromS3() { + SpringApplication application = new SpringApplication(S3ConfigDataLoaderIntegrationTests.App.class); + application.setWebApplicationType(WebApplicationType.NONE); + uploadFileToBucket("key1=value1", "reload.properties", TEXT_TYPE); + + try (ConfigurableApplicationContext context = application.run( + "--spring.config.import=aws-s3:test-bucket/reload.properties", + "--spring.cloud.aws.s3.config.reload.strategy=refresh", + "--spring.cloud.aws.s3.config.reload.period=PT1S", + "--spring.cloud.aws.s3.region=" + localstack.getRegion(), + "--spring.cloud.aws.endpoint=" + localstack.getEndpoint(), + "--spring.cloud.aws.credentials.access-key=noop", "--spring.cloud.aws.credentials.secret-key=noop", + "--spring.cloud.aws.region.static=" + localstack.getRegion())) { + + assertThat(context.getEnvironment().getProperty("key1")).isEqualTo("value1"); + + uploadFileToBucket("key1=value2", "reload.properties", TEXT_TYPE); + + await().atMost(Duration.ofSeconds(10)).untilAsserted(() -> { + assertThat(context.getEnvironment().getProperty("key1")).isEqualTo("value2"); + }); + } + } + + private static void createBucket() { + try { + localstack.execInContainer("awslocal", "s3", "mb", "s3://" + BUCKET, "--region", localstack.getRegion()); + } + catch (IOException | InterruptedException e) { + throw new RuntimeException(e); + } + } + + private static void uploadFileToBucket(String content, String key, String contentType) { + S3Client s3Client = S3Client.builder().region(Region.of(localstack.getRegion())) + .endpointOverride(localstack.getEndpoint()) + .credentialsProvider(StaticCredentialsProvider + .create(AwsBasicCredentials.create(localstack.getAccessKey(), localstack.getSecretKey()))) + .build(); + s3Client.putObject(PutObjectRequest.builder().bucket(BUCKET).contentType(contentType).key(key).build(), + RequestBody.fromBytes(content.getBytes())); + } + + private ConfigurableApplicationContext runApplication(SpringApplication application, String springConfigImport) { + return runApplication(application, springConfigImport, "spring.cloud.aws.s3.endpoint"); + } + + private ConfigurableApplicationContext runApplication(SpringApplication application, String springConfigImport, + String endpointProperty) { + return application.run("--spring.config.import=" + springConfigImport, + "--spring.cloud.aws.s3.region=" + localstack.getRegion(), + "--" + endpointProperty + "=" + localstack.getEndpoint(), + "--spring.cloud.aws.credentials.access-key=noop", "--spring.cloud.aws.credentials.secret-key=noop", + "--spring.cloud.aws.region.static=eu-west-1", "--logging.level.io.awspring.cloud.s3=debug"); + } + + static class CustomizerConfiguration implements BootstrapRegistryInitializer { + + @Override + public void initialize(BootstrapRegistry registry) { + registry.register(S3ClientCustomizer.class, context -> (builder -> { + builder.overrideConfiguration(builder.overrideConfiguration().copy(c -> { + c.apiCallTimeout(Duration.ofMillis(2001)); + })); + })); + + SdkHttpClient mock = spy(ApacheHttpClient.builder().build()); + when(mock.clientName()).thenReturn("mock-client"); + + registry.register(SdkHttpClient.class, context -> mock); + + registry.register(AwsSyncClientCustomizer.class, context -> (builder -> { + builder.httpClient(mock); + })); + } + } + + @SpringBootApplication + @EnableAutoConfiguration + static class App { + + } +} diff --git a/spring-cloud-aws-autoconfigure/src/test/java/io/awspring/cloud/autoconfigure/config/s3/S3ReloadAutoConfigurationTests.java b/spring-cloud-aws-autoconfigure/src/test/java/io/awspring/cloud/autoconfigure/config/s3/S3ReloadAutoConfigurationTests.java new file mode 100644 index 000000000..11a5e8e37 --- /dev/null +++ b/spring-cloud-aws-autoconfigure/src/test/java/io/awspring/cloud/autoconfigure/config/s3/S3ReloadAutoConfigurationTests.java @@ -0,0 +1,124 @@ +/* + * Copyright 2013-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.awspring.cloud.autoconfigure.config.s3; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockingDetails; + +import io.awspring.cloud.autoconfigure.config.reload.ConfigurationUpdateStrategy; +import io.awspring.cloud.autoconfigure.config.reload.PollingAwsPropertySourceChangeDetector; +import io.awspring.cloud.s3.config.S3PropertySource; +import org.junit.jupiter.api.Test; +import org.mockito.Answers; +import org.springframework.boot.actuate.autoconfigure.endpoint.EndpointAutoConfiguration; +import org.springframework.boot.autoconfigure.AutoConfigurations; +import org.springframework.boot.test.context.assertj.AssertableApplicationContext; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.cloud.autoconfigure.ConfigurationPropertiesRebinderAutoConfiguration; +import org.springframework.cloud.autoconfigure.RefreshAutoConfiguration; +import org.springframework.cloud.autoconfigure.RefreshEndpointAutoConfiguration; +import org.springframework.cloud.commons.util.TaskSchedulerWrapper; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.TaskScheduler; + +public class S3ReloadAutoConfigurationTests { + private final ApplicationContextRunner contextRunner = new ApplicationContextRunner() + .withPropertyValues("spring.cloud.aws.region.static:eu-west-1") + .withConfiguration(AutoConfigurations.of(S3ReloadAutoConfiguration.class, RefreshAutoConfiguration.class)); + + @Test + void createsBeansForRefreshStrategy() { + this.contextRunner.withPropertyValues("spring.cloud.aws.s3.config.reload.strategy:refresh") + .run(this::createsBeans); + } + + @Test + void createsBeansForRestartStrategy() { + this.contextRunner + .withConfiguration(AutoConfigurations.of(EndpointAutoConfiguration.class, + RefreshEndpointAutoConfiguration.class, ConfigurationPropertiesRebinderAutoConfiguration.class)) + .withPropertyValues("spring.cloud.aws.s3.config.reload.strategy:restart_context", + "management.endpoint.restart.enabled:true", "management.endpoints.web.exposure.include:restart") + .run(this::createsBeans); + } + + @Test + void doesntCreateBeansWhenStrategyNotSet() { + this.contextRunner.run(this::doesNotCreateBeans); + } + + @Test + void usesCustomTaskScheduler() { + this.contextRunner.withPropertyValues("spring.cloud.aws.s3.config.reload.strategy:refresh") + .withUserConfiguration(S3ReloadAutoConfigurationTests.CustomTaskSchedulerConfig.class).run(ctx -> { + Object taskScheduler = ctx.getBean("s3TaskScheduler"); + assertThat(mockingDetails(taskScheduler).isMock()).isTrue(); + }); + } + + @Test + void usesCustomConfigurationUpdateStrategy() { + this.contextRunner.withPropertyValues("spring.cloud.aws.s3.config.reload.strategy:refresh") + .withUserConfiguration(S3ReloadAutoConfigurationTests.CustomConfigurationUpdateStrategyConfig.class) + .run(ctx -> { + ConfigurationUpdateStrategy configurationUpdateStrategy = ctx + .getBean(ConfigurationUpdateStrategy.class); + assertThat(mockingDetails(configurationUpdateStrategy).isMock()).isTrue(); + }); + } + + private void doesNotCreateBeans(AssertableApplicationContext ctx) { + assertThat(ctx).doesNotHaveBean("s3ConfigurationUpdateStrategy"); + assertThat(ctx).doesNotHaveBean("s3PollingAwsPropertySourceChangeDetector"); + assertThat(ctx).doesNotHaveBean("s3TaskScheduler"); + } + + private void createsBeans(AssertableApplicationContext ctx) { + assertThat(ctx).hasBean("s3ConfigurationUpdateStrategy"); + assertThat(ctx.getBean("s3ConfigurationUpdateStrategy")).isInstanceOf(ConfigurationUpdateStrategy.class); + + assertThat(ctx).hasBean("s3PollingAwsPropertySourceChangeDetector"); + assertThat(ctx).getBean("s3PollingAwsPropertySourceChangeDetector") + .isInstanceOf(PollingAwsPropertySourceChangeDetector.class); + + PollingAwsPropertySourceChangeDetector changeDetector = ctx + .getBean(PollingAwsPropertySourceChangeDetector.class); + assertThat(changeDetector.getPropertySourceClass()).isEqualTo(S3PropertySource.class); + + assertThat(ctx).getBean("s3TaskScheduler").isInstanceOf(TaskSchedulerWrapper.class); + } + + @Configuration(proxyBeanMethods = false) + static class CustomTaskSchedulerConfig { + + @Bean("s3TaskScheduler") + public TaskSchedulerWrapper customTaskScheduler() { + return mock(TaskSchedulerWrapper.class, Answers.RETURNS_DEEP_STUBS); + } + } + + @Configuration(proxyBeanMethods = false) + static class CustomConfigurationUpdateStrategyConfig { + + @Bean("s3ConfigurationUpdateStrategy") + public ConfigurationUpdateStrategy configurationUpdateStrategy() { + return mock(ConfigurationUpdateStrategy.class); + } + } + +} diff --git a/spring-cloud-aws-modulith/spring-cloud-aws-modulith-events-sns/src/test/java/io/awspring/cloud/modulith/events/sns/SnsEventPublicationIntegrationTests.java b/spring-cloud-aws-modulith/spring-cloud-aws-modulith-events-sns/src/test/java/io/awspring/cloud/modulith/events/sns/SnsEventPublicationIntegrationTests.java index 9836034a7..0a3344da2 100644 --- a/spring-cloud-aws-modulith/spring-cloud-aws-modulith-events-sns/src/test/java/io/awspring/cloud/modulith/events/sns/SnsEventPublicationIntegrationTests.java +++ b/spring-cloud-aws-modulith/spring-cloud-aws-modulith-events-sns/src/test/java/io/awspring/cloud/modulith/events/sns/SnsEventPublicationIntegrationTests.java @@ -28,7 +28,6 @@ import org.springframework.modulith.events.ApplicationModuleListener; import org.springframework.modulith.events.Externalized; import org.springframework.test.context.DynamicPropertyRegistrar; -import org.springframework.test.context.DynamicPropertyRegistry; import org.springframework.transaction.annotation.Transactional; import org.testcontainers.containers.localstack.LocalStackContainer; import org.testcontainers.utility.DockerImageName; @@ -64,6 +63,7 @@ DynamicPropertyRegistrar dynamicPropertyRegistrar(LocalStackContainer localstack registry.add("spring.cloud.aws.region.static", localstack::getRegion); }; } + @Bean LocalStackContainer localStackContainer() { return new LocalStackContainer(DockerImageName.parse("localstack/localstack:3.8.1")); diff --git a/spring-cloud-aws-modulith/spring-cloud-aws-modulith-events-sqs/src/test/java/io/awspring/cloud/modulith/events/sqs/SqsEventPublicationIntegrationTests.java b/spring-cloud-aws-modulith/spring-cloud-aws-modulith-events-sqs/src/test/java/io/awspring/cloud/modulith/events/sqs/SqsEventPublicationIntegrationTests.java index 92d849db1..56948e372 100644 --- a/spring-cloud-aws-modulith/spring-cloud-aws-modulith-events-sqs/src/test/java/io/awspring/cloud/modulith/events/sqs/SqsEventPublicationIntegrationTests.java +++ b/spring-cloud-aws-modulith/spring-cloud-aws-modulith-events-sqs/src/test/java/io/awspring/cloud/modulith/events/sqs/SqsEventPublicationIntegrationTests.java @@ -28,7 +28,6 @@ import org.springframework.modulith.events.ApplicationModuleListener; import org.springframework.modulith.events.Externalized; import org.springframework.test.context.DynamicPropertyRegistrar; -import org.springframework.test.context.DynamicPropertyRegistry; import org.springframework.transaction.annotation.Transactional; import org.testcontainers.containers.localstack.LocalStackContainer; import org.testcontainers.utility.DockerImageName; @@ -62,6 +61,7 @@ DynamicPropertyRegistrar dynamicPropertyRegistrar(LocalStackContainer localstack registry.add("spring.cloud.aws.region.static", localstack::getRegion); }; } + @Bean LocalStackContainer localStackContainer() { return new LocalStackContainer(DockerImageName.parse("localstack/localstack:3.8.1")); diff --git a/spring-cloud-aws-s3/pom.xml b/spring-cloud-aws-s3/pom.xml index bf2c89bce..ed5f00a32 100644 --- a/spring-cloud-aws-s3/pom.xml +++ b/spring-cloud-aws-s3/pom.xml @@ -26,6 +26,10 @@ software.amazon.awssdk s3 + + io.awspring.cloud + spring-cloud-aws-core + software.amazon.awssdk s3-transfer-manager diff --git a/spring-cloud-aws-s3/src/main/java/io/awspring/cloud/s3/config/S3PropertySource.java b/spring-cloud-aws-s3/src/main/java/io/awspring/cloud/s3/config/S3PropertySource.java new file mode 100644 index 000000000..5c6b42d6d --- /dev/null +++ b/spring-cloud-aws-s3/src/main/java/io/awspring/cloud/s3/config/S3PropertySource.java @@ -0,0 +1,147 @@ +/* + * Copyright 2013-2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.awspring.cloud.s3.config; + +import io.awspring.cloud.core.config.AwsPropertySource; +import java.io.IOException; +import java.io.InputStream; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Properties; +import org.springframework.beans.factory.config.YamlPropertiesFactoryBean; +import org.springframework.core.io.InputStreamResource; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; + +/** + * Retrieves property sources path from the AWS S3 using the provided S3 client. + * + * @author Kunal Varpe + * @author Matej Nedic + * @since 3.3.0 + */ +public class S3PropertySource extends AwsPropertySource { + + private static final String YAML_TYPE = "application/x-yaml"; + private static final String YAML_TYPE_ALTERNATIVE = "text/yaml"; + private static final String TEXT_TYPE = "text/plain"; + private static final String JSON_TYPE = "application/json"; + + /** + * Path contains bucket name and properties files. + */ + private final String context; + + /** + * S3 Bucket which stores the property files. + */ + private final String bucket; + + private final String key; + + private final Map properties = new LinkedHashMap<>(); + + public S3PropertySource(String context, S3Client s3Client) { + super("aws-s3:" + context, s3Client); + Assert.notNull(context, "context is required"); + this.context = context; + this.bucket = resolveBucket(context); + this.key = resolveKey(context); + } + + /** + * Loads the properties from the S3. + */ + @Override + public void init() { + readPropertySourcesFromS3(GetObjectRequest.builder().bucket(bucket).key(key).build()); + } + + @Override + public S3PropertySource copy() { + return new S3PropertySource(this.context, this.source); + } + + @Override + public String[] getPropertyNames() { + return properties.keySet().toArray(String[]::new); + } + + @Override + public Object getProperty(String name) { + return properties.get(name); + } + + private void readPropertySourcesFromS3(GetObjectRequest getObjectRequest) { + try (ResponseInputStream s3PropertyFileResponse = source.getObject(getObjectRequest)) { + if (s3PropertyFileResponse != null) { + Properties props = switch (s3PropertyFileResponse.response().contentType()) { + case TEXT_TYPE -> readProperties(s3PropertyFileResponse); + case YAML_TYPE, YAML_TYPE_ALTERNATIVE, JSON_TYPE -> readYaml(s3PropertyFileResponse); + default -> throw new IllegalStateException( + "Cannot parse unknown content type: " + s3PropertyFileResponse.response().contentType()); + }; + for (Map.Entry entry : props.entrySet()) { + properties.put(String.valueOf(entry.getKey()), entry.getValue()); + } + } + } + catch (IOException e) { + logger.error("Exception has happened while trying to close S3InputStream!", e); + throw new RuntimeException(e); + } + } + + private Properties readProperties(InputStream inputStream) { + Properties properties = new Properties(); + try (InputStream in = inputStream) { + properties.load(in); + } + catch (Exception e) { + throw new IllegalStateException("Cannot load environment", e); + } + return properties; + } + + private Properties readYaml(InputStream inputStream) { + YamlPropertiesFactoryBean yaml = new YamlPropertiesFactoryBean(); + try (InputStream in = inputStream) { + yaml.setResources(new InputStreamResource(in)); + return yaml.getObject(); + } + catch (Exception e) { + throw new IllegalStateException("Cannot load environment", e); + } + } + + @Nullable + private String resolveBucket(String context) { + int delimitedIndex = context.indexOf("/"); + if (delimitedIndex != -1) { + return context.substring(0, delimitedIndex); + } + return null; + } + + private String resolveKey(String context) { + int delimitedIndex = context.indexOf("/") + 1; + return context.substring(delimitedIndex); + } +} diff --git a/spring-cloud-aws-s3/src/test/java/io/awspring/cloud/s3/config/S3PropertySourceTest.java b/spring-cloud-aws-s3/src/test/java/io/awspring/cloud/s3/config/S3PropertySourceTest.java new file mode 100644 index 000000000..14a5c0bc2 --- /dev/null +++ b/spring-cloud-aws-s3/src/test/java/io/awspring/cloud/s3/config/S3PropertySourceTest.java @@ -0,0 +1,101 @@ +/* + * Copyright 2013-2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.awspring.cloud.s3.config; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.Mockito; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.S3Exception; + +/** + * Unit tests for {@link S3PropertySource}. + * + * @author Kunal Varpe + */ +class S3PropertySourceTest { + + private static final String YAML_TYPE = "application/x-yaml"; + private static final String YAML_TYPE_ALTERNATIVE = "text/yaml"; + private static final String TEXT_TYPE = "text/plain"; + private static final String JSON_TYPE = "application/json"; + private S3Client s3Client = Mockito.mock(S3Client.class); + + @ParameterizedTest + @MethodSource("propertyFileContentSource") + void shouldParseDifferentFilesFromBucket(String sourceFileName, String contents, String contentType) { + S3PropertySource propertySource = new S3PropertySource("test-bucket/" + sourceFileName, s3Client); + InputStream is = new ByteArrayInputStream(contents.getBytes(StandardCharsets.UTF_8)); + ResponseInputStream response = new ResponseInputStream<>( + GetObjectResponse.builder().contentType(contentType).build(), is); + + when(s3Client.getObject(any(GetObjectRequest.class))).thenReturn(response); + + propertySource.init(); + + assertThat(propertySource.getName()).isEqualTo("aws-s3:test-bucket/" + sourceFileName); + assertThat(propertySource.getPropertyNames()).containsExactly("key1", "key2"); + assertThat(propertySource.getProperty("key2")).isEqualTo("value2"); + } + + private static Stream propertyFileContentSource() { + String props = """ + key1=value1 + key2=value2"""; + String ymlProps = """ + key1: value1 + key2: value2"""; + String jsonProps = """ + { "key1": "value1", "key2": "value2"}"""; + return Stream.of(Arguments.arguments("application.properties", props, TEXT_TYPE), + Arguments.arguments("application.yml", ymlProps, YAML_TYPE), + Arguments.arguments("application.yaml", ymlProps, YAML_TYPE_ALTERNATIVE), + Arguments.arguments("application.json", jsonProps, JSON_TYPE)); + } + + @Test + void shouldThrowNoSuchKeyExceptionWhenKeyDoesNotExists() { + S3PropertySource propertySource = new S3PropertySource("test-bucket;", s3Client); + + when(s3Client.getObject(any(GetObjectRequest.class))).thenThrow(NoSuchKeyException.class); + + assertThatThrownBy(propertySource::init, "You must provide key", NoSuchKeyException.class); + } + + @Test + void shouldThrowS3ExceptionWhenBucketDoesNotExists() { + S3PropertySource propertySource = new S3PropertySource("", s3Client); + + when(s3Client.getObject(any(GetObjectRequest.class))).thenThrow(S3Exception.class); + + assertThatThrownBy(propertySource::init, "You must provide key", S3Exception.class); + } +}