feat: smoke tests trait

.changes/756754c3-f6e1-4ff2-ae31-08b3b67b6750.json

@@ -1,5 +1,5 @@
 {
     "id": "756754c3-f6e1-4ff2-ae31-08b3b67b6750",
     "type": "feature",
-    "description": "Add support for smoke tests"
+    "description": "Add support for [smoke tests](https://smithy.io/2.0/additional-specs/smoke-tests.html)"
 }

codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/core/KotlinDependency.kt

@@ -104,6 +104,7 @@ data class KotlinDependency(
         val CORE = KotlinDependency(GradleConfiguration.Api, RUNTIME_ROOT_NS, RUNTIME_GROUP, "runtime-core", RUNTIME_VERSION)
         val HTTP = KotlinDependency(GradleConfiguration.Implementation, "$RUNTIME_ROOT_NS.http", RUNTIME_GROUP, "http", RUNTIME_VERSION)
         val HTTP_CLIENT = KotlinDependency(GradleConfiguration.Api, "$RUNTIME_ROOT_NS.http", RUNTIME_GROUP, "http-client", RUNTIME_VERSION)
+        val HTTP_TEST = KotlinDependency(GradleConfiguration.Api, "$RUNTIME_ROOT_NS.httptest", RUNTIME_GROUP, "http-test", RUNTIME_VERSION)
         val SERDE = KotlinDependency(GradleConfiguration.Implementation, "$RUNTIME_ROOT_NS.serde", RUNTIME_GROUP, "serde", RUNTIME_VERSION)
         val SERDE_JSON = KotlinDependency(GradleConfiguration.Implementation, "$RUNTIME_ROOT_NS.serde.json", RUNTIME_GROUP, "serde-json", RUNTIME_VERSION)
         val SERDE_XML = KotlinDependency(GradleConfiguration.Implementation, "$RUNTIME_ROOT_NS.serde.xml", RUNTIME_GROUP, "serde-xml", RUNTIME_VERSION)

codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/core/RuntimeTypes.kt

@@ -89,10 +89,13 @@ object RuntimeTypes {
             val SmokeTestsInterceptor = symbol("SmokeTestsInterceptor")
             val SmokeTestsFailureException = symbol("SmokeTestsFailureException")
             val SmokeTestsSuccessException = symbol("SmokeTestsSuccessException")
-            val SmokeTestsUnexpectedException = symbol("SmokeTestsUnexpectedException")
         }
     }
 
+    object HttpTest : RuntimeTypePackage(KotlinDependency.HTTP_TEST) {
+        val TestEngine = symbol("TestEngine")
+    }
+
     object Core : RuntimeTypePackage(KotlinDependency.CORE) {
         val Clock = symbol("Clock", "time")
         val ExecutionContext = symbol("ExecutionContext", "operation")

codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/model/traits/FailedResponseTrait.kt

@@ -0,0 +1,14 @@
+package software.amazon.smithy.kotlin.codegen.model.traits
+
+import software.amazon.smithy.model.node.ObjectNode
+import software.amazon.smithy.model.shapes.ShapeId
+import software.amazon.smithy.model.traits.AnnotationTrait
+
+/**
+ * Indicates the annotated service should always return a failed response.
+ */
+class FailedResponseTrait(node: ObjectNode) : AnnotationTrait(ID, node) {
+    companion object {
+        val ID: ShapeId = ShapeId.from("com.test#failedResponseTrait")
+    }
+}

codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/model/traits/SuccessResponseTrait.kt

@@ -0,0 +1,14 @@
+package software.amazon.smithy.kotlin.codegen.model.traits
+
+import software.amazon.smithy.model.node.ObjectNode
+import software.amazon.smithy.model.shapes.ShapeId
+import software.amazon.smithy.model.traits.AnnotationTrait
+
+/**
+ * Indicates the annotated service should always return a success response.
+ */
+class SuccessResponseTrait(node: ObjectNode) : AnnotationTrait(ID, node) {
+    companion object {
+        val ID: ShapeId = ShapeId.from("com.test#successResponseTrait")
+    }
+}

codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/smoketests/SmokeTestsIntegration.kt

@@ -10,30 +10,21 @@ import software.amazon.smithy.model.Model
 import software.amazon.smithy.smoketests.traits.SmokeTestsTrait
 
 /**
- * Renders smoke test runner for a service if any of the operations has the smoke test trait.
+ * Renders smoke test runner for a service if any of the operations have the [SmokeTestsTrait].
  */
 class SmokeTestsIntegration : KotlinIntegration {
     override fun enabledForService(model: Model, settings: KotlinSettings): Boolean =
-        model.operations(settings.service).any { it.hasTrait<SmokeTestsTrait>() } && !smokeTestDenyList.contains(settings.sdkId)
+        model.operations(settings.service).any { it.hasTrait<SmokeTestsTrait>() }
 
     override fun writeAdditionalFiles(ctx: CodegenContext, delegator: KotlinDelegator) =
-        delegator.useFileWriter("SmokeTests.kt", "smoketests", "./jvm-src/main/java/") { writer ->
+        delegator.useFileWriter(
+            "SmokeTests.kt",
+            "${ctx.settings.pkg.name}.smoketests",
+            "./jvm-src/test/java/",
+        ) { writer ->
             SmokeTestsRunnerGenerator(
                 writer,
-                ctx.symbolProvider.toSymbol(ctx.model.expectShape(ctx.settings.service)),
-                ctx.model.operations(ctx.settings.service).filter { it.hasTrait<SmokeTestsTrait>() },
-                ctx.model,
-                ctx.symbolProvider,
-                ctx.settings.sdkId,
+                ctx,
             ).render()
         }
 }
-
-/**
- * SDK ID's of services that model smoke tests incorrectly
- */
-val smokeTestDenyList = setOf(
-    "Application Auto Scaling",
-    "SWF",
-    "WAFV2",
-)

codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/smoketests/SmokeTestsRunnerGenerator.kt

@@ -1,49 +1,74 @@
 package software.amazon.smithy.kotlin.codegen.rendering.smoketests
 
 import software.amazon.smithy.codegen.core.Symbol
-import software.amazon.smithy.codegen.core.SymbolProvider
 import software.amazon.smithy.kotlin.codegen.core.*
+import software.amazon.smithy.kotlin.codegen.integration.SectionId
+import software.amazon.smithy.kotlin.codegen.model.expectShape
 import software.amazon.smithy.kotlin.codegen.model.getTrait
-import software.amazon.smithy.kotlin.codegen.rendering.util.render
+import software.amazon.smithy.kotlin.codegen.model.hasTrait
+import software.amazon.smithy.kotlin.codegen.model.traits.FailedResponseTrait
+import software.amazon.smithy.kotlin.codegen.model.traits.SuccessResponseTrait
+import software.amazon.smithy.kotlin.codegen.rendering.util.format
 import software.amazon.smithy.kotlin.codegen.utils.dq
-import software.amazon.smithy.kotlin.codegen.utils.getOrNull
+import software.amazon.smithy.kotlin.codegen.utils.operations
 import software.amazon.smithy.kotlin.codegen.utils.toCamelCase
-import software.amazon.smithy.model.Model
 import software.amazon.smithy.model.shapes.OperationShape
+import software.amazon.smithy.model.shapes.ServiceShape
 import software.amazon.smithy.smoketests.traits.SmokeTestCase
 import software.amazon.smithy.smoketests.traits.SmokeTestsTrait
 import kotlin.jvm.optionals.getOrNull
 
+object SmokeTestsRunner : SectionId
+
+// Env var constants
+const val SKIP_TAGS = "AWS_SMOKE_TEST_SKIP_TAGS"
+const val SERVICE_FILTER = "AWS_SMOKE_TEST_SERVICE_IDS"
+
 /**
  * Renders smoke tests runner for a service
  */
 class SmokeTestsRunnerGenerator(
     private val writer: KotlinWriter,
-    private val service: Symbol,
-    private val operations: List<OperationShape>,
-    private val model: Model,
-    private val symbolProvider: SymbolProvider,
-    private val sdkId: String,
+    ctx: CodegenContext,
 ) {
+    // Generator config
+    private val model = ctx.model
+    private val sdkId = ctx.settings.sdkId
+    private val symbolProvider = ctx.symbolProvider
+    private val service = symbolProvider.toSymbol(model.expectShape(ctx.settings.service))
+    private val operations = ctx.model.operations(ctx.settings.service).filter { it.hasTrait<SmokeTestsTrait>() }
+
+    // Test config
+    private val hasSuccessResponseTrait = ctx.model.expectShape<ServiceShape>(ctx.settings.service).hasTrait(SuccessResponseTrait.ID)
+    private val hasFailedResponseTrait = ctx.model.expectShape<ServiceShape>(ctx.settings.service).hasTrait(FailedResponseTrait.ID)
+    init {
+        check(!(hasSuccessResponseTrait && hasFailedResponseTrait)) {
+            "A service can't have both the success response trait and the failed response trait."
+        }
+    }
+
     internal fun render() {
-        writer.write("import kotlin.system.exitProcess")
-        writer.emptyLine()
-        writer.write("private var exitCode = 0")
-        writer.write("private val regionOverride = System.getenv(\"AWS_SMOKE_TEST_REGION\")")
-        writer.write("private val skipTags = System.getenv(\"AWS_SMOKE_TEST_SKIP_TAGS\")?.let { it.split(\",\").map { it.trim() }.toSet() }")
-        writer.emptyLine()
-        writer.withBlock("public suspend fun main() {", "}") {
-            renderFunctionCalls()
-            write("exitProcess(exitCode)")
+        writer.declareSection(SmokeTestsRunner) {
+            writer.write("import kotlin.system.exitProcess")
+            writer.emptyLine()
+            writer.write("private var exitCode = 0")
+            writer.write("private val regionOverride = System.getenv(#S)", "AWS_SMOKE_TEST_REGION")
+            writer.write("private val skipTags = System.getenv(#S)?.let { it.split(\",\").map { it.trim() }.toSet() } ?: emptySet()", SKIP_TAGS)
+            writer.write("private val serviceFilter = System.getenv(#S)?.let { it.split(\",\").map { it.trim() }.toSet() }", SERVICE_FILTER)
+            writer.emptyLine()
+            writer.withBlock("public suspend fun main() {", "}") {
+                renderFunctionCalls()
+                write("exitProcess(exitCode)")
+            }
+            writer.emptyLine()
+            renderFunctions()
         }
-        writer.emptyLine()
-        renderFunctions()
     }
 
     private fun renderFunctionCalls() {
         operations.forEach { operation ->
             operation.getTrait<SmokeTestsTrait>()?.testCases?.forEach { testCase ->
-                writer.write("${testCase.id.toCamelCase()}()")
+                writer.write("${testCase.functionName}()")
             }
         }
     }
@@ -58,9 +83,19 @@ class SmokeTestsRunnerGenerator(
     }
 
     private fun renderFunction(operation: OperationShape, testCase: SmokeTestCase) {
-        writer.withBlock("private suspend fun ${testCase.id.toCamelCase()}() {", "}") {
+        writer.withBlock("private suspend fun ${testCase.functionName}() {", "}") {
             write("val tags = setOf<String>(${testCase.tags.joinToString(",") { it.dq()} })")
-            write("if (skipTags != null && tags.any { it in skipTags }) return")
+            writer.withBlock("if ((serviceFilter != null && #S !in serviceFilter) || tags.any { it in skipTags }) {", "}", sdkId) {
+                printTestResult(
+                    sdkId.filter { !it.isWhitespace() },
+                    testCase.id,
+                    testCase.expectation.isFailure,
+                    writer,
+                    "ok",
+                    "# skip",
+                )
+                writer.write("return")
+            }
             emptyLine()
             withInlineBlock("try {", "} ") {
                 renderClient(testCase)
@@ -74,35 +109,40 @@ class SmokeTestsRunnerGenerator(
 
     private fun renderClient(testCase: SmokeTestCase) {
         writer.withInlineBlock("#L {", "}", service) {
+            // Client config
             if (testCase.vendorParams.isPresent) {
                 testCase.vendorParams.get().members.forEach { vendorParam ->
                     if (vendorParam.key.value == "region") {
-                        write("region = regionOverride ?: #L", vendorParam.value.render())
+                        write("region = regionOverride ?: #L", vendorParam.value.format())
                     } else {
-                        write("#L = #L", vendorParam.key.value.toCamelCase(), vendorParam.value.render())
+                        write("#L = #L", vendorParam.key.value.toCamelCase(), vendorParam.value.format())
                     }
                 }
             } else {
                 write("region = regionOverride")
             }
             val expectingSpecificError = testCase.expectation.failure.getOrNull()?.errorId?.getOrNull() != null
-            write("interceptors.add(#T($expectingSpecificError))", RuntimeTypes.HttpClient.Interceptors.SmokeTestsInterceptor)
-        }
-        checkVendorParamsAreCorrect(testCase)
-    }
+            if (!expectingSpecificError) {
+                write("interceptors.add(#T())", RuntimeTypes.HttpClient.Interceptors.SmokeTestsInterceptor)
+            }
 
-    /**
-     * Smithy IDL doesn't check that vendor params are found in vendor params shape so we have to check here.
-     */
-    private fun checkVendorParamsAreCorrect(testCase: SmokeTestCase) {
-        if (testCase.vendorParamsShape.isPresent && testCase.vendorParams.isPresent) {
-            val vendorParamsShape = model.getShape(testCase.vendorParamsShape.get()).get()
-            val validVendorParams = vendorParamsShape.members().map { it.memberName }
-            val vendorParams = testCase.vendorParams.get().members.map { it.key.value }
-
-            vendorParams.forEach { vendorParam ->
-                check(validVendorParams.contains(vendorParam)) {
-                    "Smithy smoke test \"${testCase.id}\" contains invalid vendor param \"$vendorParam\", it was not found in vendor params shape \"${testCase.vendorParamsShape}\""
+            // Test config
+            if (hasSuccessResponseTrait) {
+                write("httpClient = #T()", RuntimeTypes.HttpTest.TestEngine)
+            }
+            if (hasFailedResponseTrait) {
+                withBlock("httpClient = #T(", ")", RuntimeTypes.HttpTest.TestEngine) {
+                    withBlock("roundTripImpl = { _, request ->", "}") {
+                        write(
+                            "val resp = #T(#T.BadRequest, #T.Empty, #T.Empty)",
+                            RuntimeTypes.Http.Response.HttpResponse,
+                            RuntimeTypes.Http.StatusCode,
+                            RuntimeTypes.Http.Headers,
+                            RuntimeTypes.Http.HttpBody,
+                        )
+                        write("val now = #T.now()", RuntimeTypes.Core.Instant)
+                        write("#T(request, resp, now, now)", RuntimeTypes.Http.HttpCall)
+                    }
                 }
             }
         }
@@ -111,12 +151,11 @@ class SmokeTestsRunnerGenerator(
     private fun renderOperation(operation: OperationShape, testCase: SmokeTestCase) {
         val operationSymbol = symbolProvider.toSymbol(model.getShape(operation.input.get()).get())
 
-        writer.addImport(operationSymbol)
         writer.withBlock(".use { client ->", "}") {
             withBlock("client.#L(", ")", operation.defaultName()) {
-                withBlock("#L {", "}", operationSymbol.name) {
+                withBlock("#L {", "}", operationSymbol) {
                     testCase.params.get().members.forEach { member ->
-                        write("#L = #L", member.key.value.toCamelCase(), member.value.render())
+                        write("#L = #L", member.key.value.toCamelCase(), member.value.format())
                     }
                 }
             }
@@ -146,17 +185,11 @@ class SmokeTestsRunnerGenerator(
     /**
      * Tries to get the specific exception required in the failure criterion of a test.
      * If no specific exception is required we default to the generic smoke tests failure exception.
-     *
-     * Some smoke tests model exceptions not found in the model, in that case we default to the generic smoke tests
-     * failure exception.
      */
-    private fun getFailureCriterion(testCase: SmokeTestCase): Symbol = testCase.expectation.failure.getOrNull()?.errorId?.let {
-        try {
-            symbolProvider.toSymbol(model.getShape(it.get()).get())
-        } catch (e: Exception) {
-            RuntimeTypes.HttpClient.Interceptors.SmokeTestsFailureException
-        }
-    } ?: RuntimeTypes.HttpClient.Interceptors.SmokeTestsFailureException
+    private fun getFailureCriterion(testCase: SmokeTestCase): Symbol =
+        testCase.expectation.failure.getOrNull()?.errorId?.getOrNull()?.let {
+            symbolProvider.toSymbol(model.getShape(it).get())
+        } ?: RuntimeTypes.HttpClient.Interceptors.SmokeTestsFailureException
 
     /**
      * Renders print statement for smoke test result in accordance to design doc & test anything protocol (TAP)
@@ -166,9 +199,18 @@ class SmokeTestsRunnerGenerator(
         testCase: String,
         errorExpected: Boolean,
         writer: KotlinWriter,
+        statusOverride: String? = null,
+        directive: String? = "",
     ) {
         val expectation = if (errorExpected) "error expected from service" else "no error expected from service"
-        val testResult = "\$status $service $testCase - $expectation"
+        val status = statusOverride ?: "\$status"
+        val testResult = "$status $service $testCase - $expectation $directive"
         writer.write("println(#S)", testResult)
     }
 }
+
+/**
+ * Derives a function name for a [SmokeTestCase]
+ */
+private val SmokeTestCase.functionName: String
+    get() = this.id.toCamelCase()

codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/util/Node.kt

@@ -10,20 +10,20 @@ import software.amazon.smithy.model.node.ObjectNode
 import software.amazon.smithy.model.node.StringNode
 
 /**
- * Renders a [Node] into String format for codegen.
+ * Formats a [Node] into a String for codegen.
  */
-fun Node.render(): String = when (this) {
+fun Node.format(): String = when (this) {
     is NullNode -> "null"
     is StringNode -> value.dq()
     is BooleanNode -> value.toString()
     is NumberNode -> value.toString()
     is ArrayNode -> elements.joinToString(",", "listOf(", ")") { element ->
-        element.render()
+        element.format()
     }
     is ObjectNode -> buildString {
         append("mapOf(")
         stringMap.forEach { (key, value) ->
-            append("\t${key.dq()} to ${value.render()}")
+            append("\t${key.dq()} to ${value.format()}")
         }
         append(")")
     }