feat: smoke tests (#1427)

awslabs · Oct 4, 2024 · 1750f5b · 1750f5b
1 parent 36a1843
commit 1750f5b
Show file tree

Hide file tree

Showing 14 changed files with 597 additions and 1 deletion.
diff --git a/.gitignore b/.gitignore
@@ -12,8 +12,11 @@ build/
 .idea/
 __pycache__/
 local.properties
+
 # ignore generated files
 services/*/generated-src
 services/*/build.gradle.kts
 .kotest/
-*.klib
+*.klib
+tests/codegen/smoke-tests/services/*/generated-src
+tests/codegen/smoke-tests/services/*/build.gradle.kts
diff --git a/codegen/aws-sdk-codegen/build.gradle.kts b/codegen/aws-sdk-codegen/build.gradle.kts
@@ -29,6 +29,7 @@ dependencies {
     api(libs.smithy.aws.cloudformation.traits)
     api(libs.smithy.protocol.test.traits)
     implementation(libs.smithy.aws.endpoints)
+    implementation(libs.smithy.smoke.test.traits)
 
     testImplementation(libs.junit.jupiter)
     testImplementation(libs.junit.jupiter.params)

diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/GradleGenerator.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/GradleGenerator.kt
@@ -4,9 +4,17 @@
  */
 package aws.sdk.kotlin.codegen
 
+import aws.sdk.kotlin.codegen.model.traits.testing.TestFailedResponseTrait
+import aws.sdk.kotlin.codegen.model.traits.testing.TestSuccessResponseTrait
+import aws.sdk.kotlin.codegen.smoketests.smokeTestDenyList
 import software.amazon.smithy.kotlin.codegen.core.*
 import software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration
+import software.amazon.smithy.kotlin.codegen.model.expectShape
+import software.amazon.smithy.kotlin.codegen.model.hasTrait
 import software.amazon.smithy.kotlin.codegen.rendering.GradleWriter
+import software.amazon.smithy.kotlin.codegen.utils.topDownOperations
+import software.amazon.smithy.model.shapes.ServiceShape
+import software.amazon.smithy.smoketests.traits.SmokeTestsTrait
 
 // TODO - would be nice to allow integrations to define custom settings in the plugin
 // e.g. we could then more consistently apply this integration if we could define a property like: `build.isAwsSdk: true`
@@ -64,9 +72,72 @@ class GradleGenerator : KotlinIntegration {
                             }
                         }
                 }
+                if (ctx.model.topDownOperations(ctx.settings.service).any { it.hasTrait<SmokeTestsTrait>() } && ctx.settings.sdkId !in smokeTestDenyList) {
+                    write("")
+                    generateSmokeTestConfig(writer, ctx)
+                }
             }
 
         val contents = writer.toString()
         delegator.fileManifest.writeFile("build.gradle.kts", contents)
     }
+
+    private fun generateSmokeTestConfig(writer: GradleWriter, ctx: CodegenContext) {
+        generateSmokeTestJarTask(writer, ctx)
+        writer.write("")
+        generateSmokeTestTask(writer, ctx)
+    }
+
+    /**
+     * Generates a gradle task to create smoke test runner JARs
+     */
+    private fun generateSmokeTestJarTask(writer: GradleWriter, ctx: CodegenContext) {
+        writer.withBlock("jvm {", "}") {
+            withBlock("compilations {", "}") {
+                write("val mainPath = getByName(#S).output.classesDirs", "main")
+                write("val testPath = getByName(#S).output.classesDirs", "test")
+                withBlock("tasks {", "}") {
+                    withBlock("register<Jar>(#S) {", "}", "smokeTestJar") {
+                        write("description = #S", "Creates smoke tests jar")
+                        write("group = #S", "application")
+                        write("dependsOn(build)")
+                        write("mustRunAfter(build)")
+                        withBlock("manifest {", "}") {
+                            write("attributes[#S] = #S", "Main-Class", "${ctx.settings.pkg.name}.smoketests.SmokeTestsKt")
+                        }
+                        write("val runtimePath = configurations.getByName(#S).map { if (it.isDirectory) it else zipTree(it) }", "jvmRuntimeClasspath")
+                        write("duplicatesStrategy = DuplicatesStrategy.EXCLUDE")
+                        write("from(runtimePath, mainPath, testPath)")
+                        write("archiveBaseName.set(#S)", "\${project.name}-smoketests")
+                    }
+                }
+            }
+        }
+    }
+
+    /**
+     * Generates a gradle task to run smoke tests
+     */
+    private fun generateSmokeTestTask(writer: GradleWriter, ctx: CodegenContext) {
+        val hasSuccessResponseTrait = ctx.model.expectShape<ServiceShape>(ctx.settings.service).hasTrait(TestSuccessResponseTrait.ID)
+        val hasFailedResponseTrait = ctx.model.expectShape<ServiceShape>(ctx.settings.service).hasTrait(TestFailedResponseTrait.ID)
+        val inTestingEnvironment = hasFailedResponseTrait || hasSuccessResponseTrait
+
+        /**
+         * E2E tests don't have sdkVersion in jar names. They're added later for publishing.
+         * @see SmokeTestE2ETest
+         */
+        val jarName = if (inTestingEnvironment) "\${project.name}-smoketests.jar" else "\${project.name}-smoketests-\$sdkVersion.jar"
+
+        writer.withBlock("tasks.register<JavaExec>(#S) {", "}", "smokeTest") {
+            write("description = #S", "Runs smoke tests jar")
+            write("group = #S", "verification")
+            write("dependsOn(tasks.getByName(#S))", "smokeTestJar")
+            write("mustRunAfter(tasks.getByName(#S))", "smokeTestJar")
+            write("")
+            write("val sdkVersion: String by project")
+            write("val jarFile = file(#S)", "build/libs/$jarName")
+            write("classpath = files(jarFile)")
+        }
+    }
 }
diff --git a/...dk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/model/traits/testing/SmokeTestTraits.kt b/...dk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/model/traits/testing/SmokeTestTraits.kt
@@ -0,0 +1,27 @@
+package aws.sdk.kotlin.codegen.model.traits.testing
+
+import software.amazon.smithy.model.node.ObjectNode
+import software.amazon.smithy.model.shapes.ShapeId
+import software.amazon.smithy.model.traits.AnnotationTrait
+
+/**
+ * Indicates the annotated service should always return a failed response.
+ * IMPORTANT: This trait is intended for use in integration or E2E tests only, not in real-life smoke tests that run
+ * against a service endpoint.
+ */
+class TestFailedResponseTrait(node: ObjectNode) : AnnotationTrait(ID, node) {
+    companion object {
+        val ID: ShapeId = ShapeId.from("smithy.kotlin.traits#failedResponseTrait")
+    }
+}
+
+/**
+ * Indicates the annotated service should always return a success response.
+ * IMPORTANT: This trait is intended for use in integration or E2E tests only, not in real-life smoke tests that run
+ * against a service endpoint.
+ */
+class TestSuccessResponseTrait(node: ObjectNode) : AnnotationTrait(ID, node) {
+    companion object {
+        val ID: ShapeId = ShapeId.from("smithy.kotlin.traits#successResponseTrait")
+    }
+}
diff --git a/...n/src/main/kotlin/aws/sdk/kotlin/codegen/smoketests/SmokeTestsCodegenRegionIntegration.kt b/...n/src/main/kotlin/aws/sdk/kotlin/codegen/smoketests/SmokeTestsCodegenRegionIntegration.kt
@@ -0,0 +1,45 @@
+package aws.sdk.kotlin.codegen.smoketests
+
+import software.amazon.smithy.kotlin.codegen.KotlinSettings
+import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes
+import software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration
+import software.amazon.smithy.kotlin.codegen.integration.SectionWriter
+import software.amazon.smithy.kotlin.codegen.integration.SectionWriterBinding
+import software.amazon.smithy.kotlin.codegen.model.hasTrait
+import software.amazon.smithy.kotlin.codegen.rendering.smoketests.SmokeTestAdditionalEnvVars
+import software.amazon.smithy.kotlin.codegen.rendering.smoketests.SmokeTestDefaultConfig
+import software.amazon.smithy.kotlin.codegen.rendering.smoketests.SmokeTestRegionDefault
+import software.amazon.smithy.kotlin.codegen.utils.topDownOperations
+import software.amazon.smithy.model.Model
+import software.amazon.smithy.smoketests.traits.SmokeTestsTrait
+
+/**
+ * Adds AWS region support to smoke tests
+ */
+class SmokeTestsCodegenRegionIntegration : KotlinIntegration {
+    override fun enabledForService(model: Model, settings: KotlinSettings): Boolean =
+        model.topDownOperations(settings.service).any { it.hasTrait<SmokeTestsTrait>() }
+
+    override val sectionWriters: List<SectionWriterBinding>
+        get() = listOf(
+            SectionWriterBinding(SmokeTestAdditionalEnvVars, envVars),
+            SectionWriterBinding(SmokeTestDefaultConfig, region),
+            SectionWriterBinding(SmokeTestRegionDefault, regionDefault),
+        )
+
+    private val envVars = SectionWriter { writer, _ ->
+        writer.write(
+            "private val regionOverride = #T.System.getenv(#S)",
+            RuntimeTypes.Core.Utils.PlatformProvider,
+            "AWS_SMOKE_TEST_REGION",
+        )
+    }
+
+    private val region = SectionWriter { writer, _ ->
+        writer.write("region = regionOverride")
+    }
+
+    private val regionDefault = SectionWriter { writer, _ ->
+        writer.write("regionOverride ?:")
+    }
+}
diff --git a/...in/kotlin/aws/sdk/kotlin/codegen/smoketests/testing/SmokeTestFailHttpEngineIntegration.kt b/...in/kotlin/aws/sdk/kotlin/codegen/smoketests/testing/SmokeTestFailHttpEngineIntegration.kt
@@ -0,0 +1,50 @@
+package aws.sdk.kotlin.codegen.smoketests.testing
+
+import aws.sdk.kotlin.codegen.model.traits.testing.TestFailedResponseTrait
+import aws.sdk.kotlin.codegen.model.traits.testing.TestSuccessResponseTrait
+import software.amazon.smithy.kotlin.codegen.KotlinSettings
+import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes
+import software.amazon.smithy.kotlin.codegen.core.withBlock
+import software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration
+import software.amazon.smithy.kotlin.codegen.integration.SectionWriter
+import software.amazon.smithy.kotlin.codegen.integration.SectionWriterBinding
+import software.amazon.smithy.kotlin.codegen.model.expectShape
+import software.amazon.smithy.kotlin.codegen.model.hasTrait
+import software.amazon.smithy.kotlin.codegen.rendering.smoketests.SmokeTestHttpEngineOverride
+import software.amazon.smithy.kotlin.codegen.utils.topDownOperations
+import software.amazon.smithy.model.Model
+import software.amazon.smithy.model.shapes.ServiceShape
+import software.amazon.smithy.smoketests.traits.SmokeTestsTrait
+
+/**
+ * Adds [TestFailedResponseTrait] support to smoke tests
+ * IMPORTANT: This integration is intended for use in integration or E2E tests only, not in real-life smoke tests that run
+ * against a service endpoint.
+ */
+class SmokeTestFailHttpEngineIntegration : KotlinIntegration {
+    override fun enabledForService(model: Model, settings: KotlinSettings): Boolean =
+        model.topDownOperations(settings.service).any { it.hasTrait<SmokeTestsTrait>() } &&
+            !model.expectShape<ServiceShape>(settings.service).hasTrait(TestSuccessResponseTrait.ID) &&
+            model.expectShape<ServiceShape>(settings.service).hasTrait(TestFailedResponseTrait.ID)
+
+    override val sectionWriters: List<SectionWriterBinding>
+        get() = listOf(
+            SectionWriterBinding(SmokeTestHttpEngineOverride, httpClientOverride),
+        )
+
+    private val httpClientOverride = SectionWriter { writer, _ ->
+        writer.withBlock("httpClient = #T(", ")", RuntimeTypes.HttpTest.TestEngine) {
+            withBlock("roundTripImpl = { _, request ->", "}") {
+                write(
+                    "val resp = #T(#T.BadRequest, #T.Empty, #T.Empty)",
+                    RuntimeTypes.Http.Response.HttpResponse,
+                    RuntimeTypes.Http.StatusCode,
+                    RuntimeTypes.Http.Headers,
+                    RuntimeTypes.Http.HttpBody,
+                )
+                write("val now = #T.now()", RuntimeTypes.Core.Instant)
+                write("#T(request, resp, now, now)", RuntimeTypes.Http.HttpCall)
+            }
+        }
+    }
+}
diff --git a/...kotlin/aws/sdk/kotlin/codegen/smoketests/testing/SmokeTestSuccessHttpEngineIntegration.kt b/...kotlin/aws/sdk/kotlin/codegen/smoketests/testing/SmokeTestSuccessHttpEngineIntegration.kt
@@ -0,0 +1,37 @@
+package aws.sdk.kotlin.codegen.smoketests.testing
+
+import aws.sdk.kotlin.codegen.model.traits.testing.TestFailedResponseTrait
+import aws.sdk.kotlin.codegen.model.traits.testing.TestSuccessResponseTrait
+import software.amazon.smithy.kotlin.codegen.KotlinSettings
+import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes
+import software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration
+import software.amazon.smithy.kotlin.codegen.integration.SectionWriter
+import software.amazon.smithy.kotlin.codegen.integration.SectionWriterBinding
+import software.amazon.smithy.kotlin.codegen.model.expectShape
+import software.amazon.smithy.kotlin.codegen.model.hasTrait
+import software.amazon.smithy.kotlin.codegen.rendering.smoketests.SmokeTestHttpEngineOverride
+import software.amazon.smithy.kotlin.codegen.utils.topDownOperations
+import software.amazon.smithy.model.Model
+import software.amazon.smithy.model.shapes.ServiceShape
+import software.amazon.smithy.smoketests.traits.SmokeTestsTrait
+
+/**
+ * Adds [TestSuccessResponseTrait] support to smoke tests
+ * IMPORTANT: This integration is intended for use in integration or E2E tests only, not in real-life smoke tests that run
+ * against a service endpoint.
+ */
+class SmokeTestSuccessHttpEngineIntegration : KotlinIntegration {
+    override fun enabledForService(model: Model, settings: KotlinSettings): Boolean =
+        model.topDownOperations(settings.service).any { it.hasTrait<SmokeTestsTrait>() } &&
+            model.expectShape<ServiceShape>(settings.service).hasTrait(TestSuccessResponseTrait.ID) &&
+            !model.expectShape<ServiceShape>(settings.service).hasTrait(TestFailedResponseTrait.ID)
+
+    override val sectionWriters: List<SectionWriterBinding>
+        get() = listOf(
+            SectionWriterBinding(SmokeTestHttpEngineOverride, httpClientOverride),
+        )
+
+    private val httpClientOverride = SectionWriter { writer, _ ->
+        writer.write("httpClient = #T()", RuntimeTypes.HttpTest.TestEngine)
+    }
+}
diff --git a/...ces/META-INF/services/software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration b/...ces/META-INF/services/software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration
@@ -44,3 +44,6 @@ aws.sdk.kotlin.codegen.customization.s3.express.S3ExpressIntegration
 aws.sdk.kotlin.codegen.customization.s3.S3ExpiresIntegration
 aws.sdk.kotlin.codegen.BusinessMetricsIntegration
 aws.sdk.kotlin.codegen.smoketests.SmokeTestsDenyListIntegration
+aws.sdk.kotlin.codegen.smoketests.SmokeTestsCodegenRegionIntegration
+aws.sdk.kotlin.codegen.smoketests.testing.SmokeTestSuccessHttpEngineIntegration
+aws.sdk.kotlin.codegen.smoketests.testing.SmokeTestFailHttpEngineIntegration
diff --git a/settings.gradle.kts b/settings.gradle.kts
@@ -43,6 +43,8 @@ include(":services")
 include(":tests")
 include(":tests:codegen:event-stream")
 include(":tests:e2e-test-util")
+include(":tests:codegen:smoke-tests")
+include(":tests:codegen:smoke-tests:services")
 
 // generated services
 val File.isServiceDir: Boolean
@@ -57,6 +59,13 @@ file("services").listFiles().forEach {
     }
 }
 
+// generated services by smoke tests test suite
+file("tests/codegen/smoke-tests/services").listFiles().forEach {
+    if (it.isServiceDir) {
+        include(":tests:codegen:smoke-tests:services:${it.name}")
+    }
+}
+
 // Service benchmarks project
 val benchmarkServices = listOf(
     // keep this list in sync with tests/benchmarks/service-benchmarks/build.gradle.kts