refactor!: track upstream observability API changes (#974)

Refactor observability (tracing) API and introduce an OTeL backed provider.

.changes/5b1a8324-02da-4281-bd23-81c565eed973.json

@@ -0,0 +1,5 @@
+{
+    "id": "5b1a8324-02da-4281-bd23-81c565eed973",
+    "type": "misc",
+    "description": "**BREAKING**: Refactor observability API and configuration. See the [discussion](https://github.com/awslabs/aws-sdk-kotlin/discussions/981) for more information."
+}

aws-runtime/aws-config/build.gradle.kts

@@ -36,10 +36,9 @@ kotlin {
             dependencies {
                 api(project(":aws-runtime:aws-core"))
                 api("aws.smithy.kotlin:aws-credentials:$smithyKotlinVersion")
-                implementation("aws.smithy.kotlin:logging:$smithyKotlinVersion")
                 implementation("aws.smithy.kotlin:http:$smithyKotlinVersion")
                 implementation("aws.smithy.kotlin:http-auth:$smithyKotlinVersion")
-                implementation("aws.smithy.kotlin:tracing-core:$smithyKotlinVersion")
+                implementation("aws.smithy.kotlin:telemetry-api:$smithyKotlinVersion")
                 implementation("aws.smithy.kotlin:http-client-engine-default:$smithyKotlinVersion")
                 implementation(project(":aws-runtime:aws-http"))
 
@@ -56,6 +55,7 @@ kotlin {
                 implementation("aws.smithy.kotlin:aws-signing-common:$smithyKotlinVersion")
                 implementation("aws.smithy.kotlin:aws-signing-default:$smithyKotlinVersion")
                 implementation("aws.smithy.kotlin:http-auth-aws:$smithyKotlinVersion")
+                implementation("aws.smithy.kotlin:telemetry-defaults:$smithyKotlinVersion")
 
                 // additional dependencies required by generated sso provider(s)
                 implementation("aws.smithy.kotlin:aws-json-protocols:$smithyKotlinVersion")

aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/credentials/EcsCredentialsProvider.kt

@@ -29,6 +29,7 @@ import aws.smithy.kotlin.runtime.retries.policy.RetryDirective
 import aws.smithy.kotlin.runtime.retries.policy.RetryErrorType
 import aws.smithy.kotlin.runtime.retries.policy.RetryPolicy
 import aws.smithy.kotlin.runtime.serde.json.JsonDeserializer
+import aws.smithy.kotlin.runtime.telemetry.logging.logger
 import aws.smithy.kotlin.runtime.time.TimestampFormat
 import aws.smithy.kotlin.runtime.util.Attributes
 import aws.smithy.kotlin.runtime.util.PlatformEnvironProvider
@@ -68,7 +69,7 @@ public class EcsCredentialsProvider internal constructor(
     private val manageEngine = httpClient == null
     private val httpClient = httpClient ?: DefaultHttpEngine()
     override suspend fun resolve(attributes: Attributes): Credentials {
-        val logger = coroutineContext.getLogger<EcsCredentialsProvider>()
+        val logger = coroutineContext.logger<EcsCredentialsProvider>()
         val authToken = AwsSdkSetting.AwsContainerAuthorizationToken.resolve(platformProvider)
         val relativeUri = AwsSdkSetting.AwsContainerCredentialsRelativeUri.resolve(platformProvider)
         val fullUri = AwsSdkSetting.AwsContainerCredentialsFullUri.resolve(platformProvider)
@@ -84,6 +85,7 @@ public class EcsCredentialsProvider internal constructor(
             deserializer = EcsCredentialsDeserializer()
             context {
                 operationName = "EcsCredentialsProvider"
+                serviceName = "EcsContainerMetadata"
             }
             execution.endpointResolver = EndpointResolver { Endpoint(url) }
         }

aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/credentials/EnvironmentCredentialsProvider.kt

@@ -8,7 +8,7 @@ package aws.sdk.kotlin.runtime.auth.credentials
 import aws.sdk.kotlin.runtime.config.AwsSdkSetting
 import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials
 import aws.smithy.kotlin.runtime.auth.awscredentials.CredentialsProvider
-import aws.smithy.kotlin.runtime.tracing.trace
+import aws.smithy.kotlin.runtime.telemetry.logging.trace
 import aws.smithy.kotlin.runtime.util.Attributes
 import aws.smithy.kotlin.runtime.util.PlatformProvider
 import kotlin.coroutines.coroutineContext

aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/credentials/ImdsCredentialsProvider.kt

@@ -17,10 +17,10 @@ import aws.smithy.kotlin.runtime.auth.awscredentials.DEFAULT_CREDENTIALS_REFRESH
 import aws.smithy.kotlin.runtime.config.resolve
 import aws.smithy.kotlin.runtime.http.HttpStatusCode
 import aws.smithy.kotlin.runtime.serde.json.JsonDeserializer
+import aws.smithy.kotlin.runtime.telemetry.logging.info
+import aws.smithy.kotlin.runtime.telemetry.logging.warn
 import aws.smithy.kotlin.runtime.time.Clock
 import aws.smithy.kotlin.runtime.time.Instant
-import aws.smithy.kotlin.runtime.tracing.info
-import aws.smithy.kotlin.runtime.tracing.warn
 import aws.smithy.kotlin.runtime.util.Attributes
 import aws.smithy.kotlin.runtime.util.PlatformEnvironProvider
 import aws.smithy.kotlin.runtime.util.PlatformProvider

aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/credentials/ProcessCredentialsProvider.kt

@@ -7,12 +7,13 @@ package aws.sdk.kotlin.runtime.auth.credentials
 import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials
 import aws.smithy.kotlin.runtime.auth.awscredentials.CredentialsProvider
 import aws.smithy.kotlin.runtime.auth.awscredentials.CredentialsProviderException
-import aws.smithy.kotlin.runtime.logging.Logger
 import aws.smithy.kotlin.runtime.serde.json.JsonDeserializer
+import aws.smithy.kotlin.runtime.telemetry.logging.logger
 import aws.smithy.kotlin.runtime.time.Clock
 import aws.smithy.kotlin.runtime.time.Instant
 import aws.smithy.kotlin.runtime.util.Attributes
 import aws.smithy.kotlin.runtime.util.PlatformProvider
+import kotlin.coroutines.coroutineContext
 
 internal expect suspend fun executeCommand(
     command: String,
@@ -48,7 +49,7 @@ public class ProcessCredentialsProvider(
     private val timeoutMillis: Long = 60_000,
 ) : CredentialsProvider {
     override suspend fun resolve(attributes: Attributes): Credentials {
-        val logger = Logger.getLogger<ProcessCredentialsProvider>()
+        val logger = coroutineContext.logger<ProcessCredentialsProvider>()
 
         val (exitCode, output) = try {
             executeCommand(credentialProcess, platformProvider, maxOutputLengthBytes, timeoutMillis)

aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/credentials/ProfileBearerTokenProvider.kt

@@ -14,7 +14,7 @@ import aws.sdk.kotlin.runtime.config.profile.loadAwsSharedConfig
 import aws.smithy.kotlin.runtime.http.auth.BearerToken
 import aws.smithy.kotlin.runtime.http.auth.BearerTokenProvider
 import aws.smithy.kotlin.runtime.http.engine.HttpClientEngine
-import aws.smithy.kotlin.runtime.http.operation.getLogger
+import aws.smithy.kotlin.runtime.telemetry.logging.logger
 import aws.smithy.kotlin.runtime.time.Clock
 import aws.smithy.kotlin.runtime.util.Attributes
 import aws.smithy.kotlin.runtime.util.PlatformProvider
@@ -64,7 +64,7 @@ internal class ProfileBearerTokenProvider(
     private val sharedConfig = asyncLazy { loadAwsSharedConfig(platformProvider, profileName) }
 
     override suspend fun resolve(attributes: Attributes): BearerToken {
-        val logger = coroutineContext.getLogger<ProfileBearerTokenProvider>()
+        val logger = coroutineContext.logger<ProfileBearerTokenProvider>()
         val config = sharedConfig.get()
         logger.debug { "Loading bearer token from profile `${config.activeProfile.name}`" }
 

aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/credentials/ProfileCredentialsProvider.kt

@@ -16,8 +16,8 @@ import aws.smithy.kotlin.runtime.auth.awscredentials.CloseableCredentialsProvide
 import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials
 import aws.smithy.kotlin.runtime.auth.awscredentials.CredentialsProvider
 import aws.smithy.kotlin.runtime.http.engine.HttpClientEngine
-import aws.smithy.kotlin.runtime.http.operation.getLogger
 import aws.smithy.kotlin.runtime.io.closeIfCloseable
+import aws.smithy.kotlin.runtime.telemetry.logging.logger
 import aws.smithy.kotlin.runtime.time.TimestampFormat
 import aws.smithy.kotlin.runtime.util.Attributes
 import aws.smithy.kotlin.runtime.util.LazyAsyncValue
@@ -95,7 +95,7 @@ public class ProfileCredentialsProvider(
     )
 
     override suspend fun resolve(attributes: Attributes): Credentials {
-        val logger = coroutineContext.getLogger<ProfileCredentialsProvider>()
+        val logger = coroutineContext.logger<ProfileCredentialsProvider>()
         val sharedConfig = loadAwsSharedConfig(platformProvider, profileName)
         logger.debug { "Loading credentials from profile `${sharedConfig.activeProfile.name}`" }
         val chain = ProfileChain.resolve(sharedConfig)

aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/credentials/SsoCredentialsProvider.kt

@@ -12,10 +12,11 @@ import aws.smithy.kotlin.runtime.auth.awscredentials.CredentialsProvider
 import aws.smithy.kotlin.runtime.auth.awscredentials.CredentialsProviderException
 import aws.smithy.kotlin.runtime.http.engine.HttpClientEngine
 import aws.smithy.kotlin.runtime.serde.json.*
+import aws.smithy.kotlin.runtime.telemetry.logging.logger
+import aws.smithy.kotlin.runtime.telemetry.telemetryProvider
 import aws.smithy.kotlin.runtime.time.Clock
 import aws.smithy.kotlin.runtime.time.Instant
 import aws.smithy.kotlin.runtime.time.fromEpochMilliseconds
-import aws.smithy.kotlin.runtime.tracing.*
 import aws.smithy.kotlin.runtime.util.*
 import kotlin.coroutines.coroutineContext
 
@@ -106,8 +107,7 @@ public class SsoCredentialsProvider public constructor(
     }
 
     override suspend fun resolve(attributes: Attributes): Credentials {
-        val traceSpan = coroutineContext.traceSpan
-        val logger = traceSpan.logger<SsoCredentialsProvider>()
+        val logger = coroutineContext.logger<SsoCredentialsProvider>()
 
         val token = if (ssoTokenProvider != null) {
             logger.trace { "Attempting to load token using token provider for sso-session: `$ssoSessionName`" }
@@ -117,10 +117,11 @@ public class SsoCredentialsProvider public constructor(
             legacyLoadTokenFile()
         }
 
+        val telemetry = coroutineContext.telemetryProvider
         val client = SsoClient {
             region = ssoRegion
             httpClient = this@SsoCredentialsProvider.httpClient
-            tracer = traceSpan.asNestedTracer("SSO-")
+            telemetryProvider = telemetry
             // FIXME - create an anonymous credential provider to explicitly avoid default chain creation (technically the transform should remove need for sigv4 cred provider since it's all anon auth)
         }
 

aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/credentials/SsoTokenProvider.kt

@@ -16,11 +16,12 @@ import aws.smithy.kotlin.runtime.http.auth.BearerTokenProvider
 import aws.smithy.kotlin.runtime.http.engine.HttpClientEngine
 import aws.smithy.kotlin.runtime.io.use
 import aws.smithy.kotlin.runtime.serde.json.*
+import aws.smithy.kotlin.runtime.telemetry.logging.debug
+import aws.smithy.kotlin.runtime.telemetry.logging.error
+import aws.smithy.kotlin.runtime.telemetry.telemetryProvider
 import aws.smithy.kotlin.runtime.time.Clock
 import aws.smithy.kotlin.runtime.time.Instant
 import aws.smithy.kotlin.runtime.time.TimestampFormat
-import aws.smithy.kotlin.runtime.tracing.debug
-import aws.smithy.kotlin.runtime.tracing.error
 import aws.smithy.kotlin.runtime.util.*
 import kotlin.coroutines.coroutineContext
 import kotlin.time.Duration
@@ -118,9 +119,11 @@ public class SsoTokenProvider(
     }
 
     private suspend fun refreshToken(oldToken: SsoToken): SsoToken {
+        val telemetry = coroutineContext.telemetryProvider
         SsoOidcClient {
             region = ssoRegion
             httpClient = this@SsoTokenProvider.httpClient
+            telemetryProvider = telemetry
         }.use { client ->
             val resp = client.createToken {
                 clientId = oldToken.clientId

aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/credentials/StsAssumeRoleCredentialsProvider.kt

@@ -15,10 +15,11 @@ import aws.smithy.kotlin.runtime.auth.awscredentials.CredentialsProviderExceptio
 import aws.smithy.kotlin.runtime.auth.awscredentials.DEFAULT_CREDENTIALS_REFRESH_SECONDS
 import aws.smithy.kotlin.runtime.config.resolve
 import aws.smithy.kotlin.runtime.http.engine.HttpClientEngine
+import aws.smithy.kotlin.runtime.telemetry.logging.logger
+import aws.smithy.kotlin.runtime.telemetry.telemetryProvider
 import aws.smithy.kotlin.runtime.time.Instant
 import aws.smithy.kotlin.runtime.time.TimestampFormat
 import aws.smithy.kotlin.runtime.time.epochMilliseconds
-import aws.smithy.kotlin.runtime.tracing.*
 import aws.smithy.kotlin.runtime.util.Attributes
 import aws.smithy.kotlin.runtime.util.PlatformEnvironProvider
 import aws.smithy.kotlin.runtime.util.PlatformProvider
@@ -60,17 +61,17 @@ public class StsAssumeRoleCredentialsProvider(
 ) : CredentialsProvider {
 
     override suspend fun resolve(attributes: Attributes): Credentials {
-        val traceSpan = coroutineContext.traceSpan
-        val logger = traceSpan.logger<StsAssumeRoleCredentialsProvider>()
+        val logger = coroutineContext.logger<StsAssumeRoleCredentialsProvider>()
         logger.debug { "retrieving assumed credentials" }
 
         // NOTE: multi region access points require regional STS endpoints
         val provider = this
+        val telemetry = coroutineContext.telemetryProvider
         val client = StsClient {
             region = provider.region ?: GLOBAL_STS_PARTITION_ENDPOINT
             credentialsProvider = provider.credentialsProvider
             httpClient = provider.httpClient
-            tracer = traceSpan.asNestedTracer("STS-")
+            telemetryProvider = telemetry
         }
 
         val resp = try {

aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/credentials/StsWebIdentityCredentialsProvider.kt

@@ -15,8 +15,9 @@ import aws.smithy.kotlin.runtime.auth.awscredentials.DEFAULT_CREDENTIALS_REFRESH
 import aws.smithy.kotlin.runtime.config.EnvironmentSetting
 import aws.smithy.kotlin.runtime.config.resolve
 import aws.smithy.kotlin.runtime.http.engine.HttpClientEngine
+import aws.smithy.kotlin.runtime.telemetry.logging.logger
+import aws.smithy.kotlin.runtime.telemetry.telemetryProvider
 import aws.smithy.kotlin.runtime.time.TimestampFormat
-import aws.smithy.kotlin.runtime.tracing.*
 import aws.smithy.kotlin.runtime.util.Attributes
 import aws.smithy.kotlin.runtime.util.PlatformProvider
 import kotlin.coroutines.coroutineContext
@@ -73,20 +74,21 @@ public class StsWebIdentityCredentialsProvider(
     }
 
     override suspend fun resolve(attributes: Attributes): Credentials {
-        val traceSpan = coroutineContext.traceSpan
-        val logger = traceSpan.logger<StsAssumeRoleCredentialsProvider>()
+        val logger = coroutineContext.logger<StsAssumeRoleCredentialsProvider>()
         logger.debug { "retrieving assumed credentials via web identity" }
         val provider = this
 
         val token = platformProvider
             .readFileOrNull(webIdentityTokenFilePath)
             ?.decodeToString() ?: throw CredentialsProviderException("failed to read webIdentityToken from $webIdentityTokenFilePath")
 
+        val telemetry = coroutineContext.telemetryProvider
+
         val client = StsClient {
             region = provider.region
             httpClient = provider.httpClient
             // NOTE: credentials provider not needed for this operation
-            tracer = traceSpan.asNestedTracer("STS-")
+            telemetryProvider = telemetry
         }
 
         val resp = try {

aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory.kt

@@ -12,17 +12,19 @@ import aws.sdk.kotlin.runtime.config.profile.AwsProfile
 import aws.sdk.kotlin.runtime.config.profile.loadAwsSharedConfig
 import aws.sdk.kotlin.runtime.config.retries.resolveRetryStrategy
 import aws.sdk.kotlin.runtime.region.resolveRegion
+import aws.smithy.kotlin.runtime.ExperimentalApi
 import aws.smithy.kotlin.runtime.client.RetryStrategyClientConfig
 import aws.smithy.kotlin.runtime.client.SdkClient
 import aws.smithy.kotlin.runtime.client.SdkClientConfig
 import aws.smithy.kotlin.runtime.client.SdkClientFactory
 import aws.smithy.kotlin.runtime.client.config.ClientSettings
 import aws.smithy.kotlin.runtime.config.resolve
-import aws.smithy.kotlin.runtime.tracing.*
+import aws.smithy.kotlin.runtime.telemetry.TelemetryConfig
+import aws.smithy.kotlin.runtime.telemetry.TelemetryProvider
+import aws.smithy.kotlin.runtime.telemetry.trace.withSpan
 import aws.smithy.kotlin.runtime.util.LazyAsyncValue
 import aws.smithy.kotlin.runtime.util.PlatformProvider
 import aws.smithy.kotlin.runtime.util.asyncLazy
-import kotlin.coroutines.coroutineContext
 
 /**
  * Abstract base class all AWS client companion objects inherit from
@@ -45,18 +47,16 @@ public abstract class AbstractAwsSdkClientFactory<
     /**
      * Construct a [TClient] by resolving the configuration from the current environment.
      */
+    @OptIn(ExperimentalApi::class)
     public suspend fun fromEnvironment(block: (TConfigBuilder.() -> Unit)? = null): TClient {
         val builder = builder()
         val config = builder.config
 
-        val tracer = if (config is TracingClientConfig.Builder) {
-            if (config.tracer == null) config.tracer = defaultTracer(config.clientName)
-            config.tracer!!
-        } else {
-            defaultTracer(config.clientName)
-        }
+        // FIXME - use a default telemetry provider that at least wires up SLF4j logging
+        val telemetryProvider = (builder as? TelemetryConfig.Builder)?.telemetryProvider ?: TelemetryProvider.None
+        val tracer = telemetryProvider.tracerProvider.getOrCreateTracer("AwsSdkClientFactory")
 
-        coroutineContext.withRootTraceSpan(tracer.createRootSpan("Config resolution")) {
+        tracer.withSpan("fromEnvironment") {
             val profile = asyncLazy { loadAwsSharedConfig(PlatformProvider.System).activeProfile }
 
             // As a DslBuilderProperty, the value of retryStrategy cannot be checked for nullability because it may have
@@ -80,6 +80,4 @@ public abstract class AbstractAwsSdkClientFactory<
      * Inject any client-specific config.
      */
     protected open suspend fun finalizeConfig(builder: TClientBuilder, profile: LazyAsyncValue<AwsProfile>) { }
-
-    private fun defaultTracer(clientName: String): Tracer = DefaultTracer(LoggingTraceProbe, clientName)
 }

aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/ImdsClient.kt

@@ -23,6 +23,7 @@ import aws.smithy.kotlin.runtime.io.middleware.Phase
 import aws.smithy.kotlin.runtime.operation.ExecutionContext
 import aws.smithy.kotlin.runtime.time.Clock
 import aws.smithy.kotlin.runtime.util.PlatformProvider
+import kotlin.coroutines.coroutineContext
 import kotlin.time.Duration
 import kotlin.time.Duration.Companion.seconds
 
@@ -118,14 +119,15 @@ public class ImdsClient private constructor(builder: Builder) : InstanceMetadata
             }
             context {
                 operationName = path
+                serviceName = "IMDS"
 
                 // artifact of re-using ServiceEndpointResolver middleware
                 set(SdkClientOption.LogMode, logMode)
             }
 
             execution.endpointResolver = imdsEndpointProvider
         }
-        op.execution.retryPolicy = ImdsRetryPolicy()
+        op.execution.retryPolicy = ImdsRetryPolicy(coroutineContext)
 
         op.install(userAgentMiddleware)
         op.install(tokenMiddleware)

aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/ImdsRetryPolicy.kt

@@ -7,12 +7,15 @@ package aws.sdk.kotlin.runtime.config.imds
 
 import aws.smithy.kotlin.runtime.http.HttpStatusCode
 import aws.smithy.kotlin.runtime.http.category
-import aws.smithy.kotlin.runtime.logging.Logger
 import aws.smithy.kotlin.runtime.retries.policy.RetryDirective
 import aws.smithy.kotlin.runtime.retries.policy.RetryErrorType
 import aws.smithy.kotlin.runtime.retries.policy.RetryPolicy
+import aws.smithy.kotlin.runtime.telemetry.logging.logger
+import kotlin.coroutines.CoroutineContext
 
-internal class ImdsRetryPolicy : RetryPolicy<Any?> {
+internal class ImdsRetryPolicy(
+    private val callContext: CoroutineContext,
+) : RetryPolicy<Any?> {
     override fun evaluate(result: Result<Any?>): RetryDirective = when {
         result.isSuccess -> RetryDirective.TerminateAndSucceed
         else -> evaluate(result.exceptionOrNull()!!)
@@ -26,7 +29,7 @@ internal class ImdsRetryPolicy : RetryPolicy<Any?> {
                 // 401 indicates the token has expired, this is retryable
                 status == HttpStatusCode.Unauthorized -> RetryDirective.RetryError(RetryErrorType.ServerSide)
                 else -> {
-                    val logger = Logger.getLogger<ImdsRetryPolicy>()
+                    val logger = callContext.logger<ImdsRetryPolicy>()
                     logger.debug { "Non retryable IMDS error: statusCode=${throwable.statusCode}; ${throwable.message}" }
                     RetryDirective.TerminateAndFail
                 }