feat: or-2583 add observability for kbo sync lambda

Informatievlaanderen · Jan 24, 2025 · 66b1e8d · 66b1e8d
1 parent 1827971
commit 66b1e8d
Show file tree

Hide file tree

Showing 5 changed files with 284 additions and 3 deletions.
diff --git a/paket.dependencies b/paket.dependencies
@@ -57,6 +57,7 @@ nuget OpenTelemetry.Extensions.Hosting
 nuget OpenTelemetry.Instrumentation.AspNetCore
 nuget OpenTelemetry.Instrumentation.Http
 nuget OpenTelemetry.Instrumentation.Runtime
+nuget OpenTelemetry.Contrib.Instrumentation.AWS
 
 nuget Confluent.Kafka
 nuget Polly

diff --git a/paket.lock b/paket.lock
@@ -680,6 +680,13 @@ NUGET
     OpenTelemetry.Api.ProviderBuilderExtensions (1.9)
       Microsoft.Extensions.DependencyInjection.Abstractions (>= 8.0)
       OpenTelemetry.Api (>= 1.9)
+    OpenTelemetry.Contrib.Extensions.AWSXRay (1.2)
+      Newtonsoft.Json (>= 11.0.1)
+      OpenTelemetry (>= 1.1)
+      System.Net.Http (>= 4.3.4)
+    OpenTelemetry.Contrib.Instrumentation.AWS (1.0.2)
+      AWSSDK.Core (>= 3.5.1.24)
+      OpenTelemetry.Contrib.Extensions.AWSXRay (>= 1.0.1)
     OpenTelemetry.Exporter.Console (1.8.1)
       OpenTelemetry (>= 1.8.1)
       System.Text.Encodings.Web (>= 4.7.2)
@@ -708,9 +715,46 @@ NUGET
       Polly.Core (>= 8.4)
     Polly.Core (8.4)
     Result.Net (1.4)
+    runtime.debian.8-x64.runtime.native.System.Security.Cryptography.OpenSsl (4.3.3)
+    runtime.debian.9-x64.runtime.native.System.Security.Cryptography.OpenSsl (4.3.3)
+    runtime.fedora.23-x64.runtime.native.System.Security.Cryptography.OpenSsl (4.3.3)
+    runtime.fedora.24-x64.runtime.native.System.Security.Cryptography.OpenSsl (4.3.3)
+    runtime.fedora.27-x64.runtime.native.System.Security.Cryptography.OpenSsl (4.3.3)
+    runtime.fedora.28-x64.runtime.native.System.Security.Cryptography.OpenSsl (4.3.3)
     runtime.native.System (4.3.1)
       Microsoft.NETCore.Platforms (>= 1.1.1)
       Microsoft.NETCore.Targets (>= 1.1.3)
+    runtime.native.System.Net.Http (4.3.1)
+      Microsoft.NETCore.Platforms (>= 1.1.1)
+      Microsoft.NETCore.Targets (>= 1.1.3)
+    runtime.native.System.Security.Cryptography.Apple (4.3.1)
+      runtime.osx.10.10-x64.runtime.native.System.Security.Cryptography.Apple (>= 4.3.1)
+    runtime.native.System.Security.Cryptography.OpenSsl (4.3.3)
+      runtime.debian.8-x64.runtime.native.System.Security.Cryptography.OpenSsl (>= 4.3.3)
+      runtime.debian.9-x64.runtime.native.System.Security.Cryptography.OpenSsl (>= 4.3.3)
+      runtime.fedora.23-x64.runtime.native.System.Security.Cryptography.OpenSsl (>= 4.3.3)
+      runtime.fedora.24-x64.runtime.native.System.Security.Cryptography.OpenSsl (>= 4.3.3)
+      runtime.fedora.27-x64.runtime.native.System.Security.Cryptography.OpenSsl (>= 4.3.3)
+      runtime.fedora.28-x64.runtime.native.System.Security.Cryptography.OpenSsl (>= 4.3.3)
+      runtime.opensuse.13.2-x64.runtime.native.System.Security.Cryptography.OpenSsl (>= 4.3.3)
+      runtime.opensuse.42.1-x64.runtime.native.System.Security.Cryptography.OpenSsl (>= 4.3.3)
+      runtime.opensuse.42.3-x64.runtime.native.System.Security.Cryptography.OpenSsl (>= 4.3.3)
+      runtime.osx.10.10-x64.runtime.native.System.Security.Cryptography.OpenSsl (>= 4.3.3)
+      runtime.rhel.7-x64.runtime.native.System.Security.Cryptography.OpenSsl (>= 4.3.3)
+      runtime.ubuntu.14.04-x64.runtime.native.System.Security.Cryptography.OpenSsl (>= 4.3.3)
+      runtime.ubuntu.16.04-x64.runtime.native.System.Security.Cryptography.OpenSsl (>= 4.3.3)
+      runtime.ubuntu.16.10-x64.runtime.native.System.Security.Cryptography.OpenSsl (>= 4.3.3)
+      runtime.ubuntu.18.04-x64.runtime.native.System.Security.Cryptography.OpenSsl (>= 4.3.3)
+    runtime.opensuse.13.2-x64.runtime.native.System.Security.Cryptography.OpenSsl (4.3.3)
+    runtime.opensuse.42.1-x64.runtime.native.System.Security.Cryptography.OpenSsl (4.3.3)
+    runtime.opensuse.42.3-x64.runtime.native.System.Security.Cryptography.OpenSsl (4.3.3)
+    runtime.osx.10.10-x64.runtime.native.System.Security.Cryptography.Apple (4.3.1)
+    runtime.osx.10.10-x64.runtime.native.System.Security.Cryptography.OpenSsl (4.3.3)
+    runtime.rhel.7-x64.runtime.native.System.Security.Cryptography.OpenSsl (4.3.3)
+    runtime.ubuntu.14.04-x64.runtime.native.System.Security.Cryptography.OpenSsl (4.3.3)
+    runtime.ubuntu.16.04-x64.runtime.native.System.Security.Cryptography.OpenSsl (4.3.3)
+    runtime.ubuntu.16.10-x64.runtime.native.System.Security.Cryptography.OpenSsl (4.3.3)
+    runtime.ubuntu.18.04-x64.runtime.native.System.Security.Cryptography.OpenSsl (4.3.3)
     Scriban (5.10)
     Scrutor (4.2.2)
       Microsoft.Extensions.DependencyInjection.Abstractions (>= 6.0)
@@ -782,6 +826,17 @@ NUGET
       Microsoft.NETCore.Platforms (>= 1.1)
       Microsoft.NETCore.Targets (>= 1.1)
       System.Runtime (>= 4.3)
+    System.Collections.Concurrent (4.3)
+      System.Collections (>= 4.3)
+      System.Diagnostics.Debug (>= 4.3)
+      System.Diagnostics.Tracing (>= 4.3)
+      System.Globalization (>= 4.3)
+      System.Reflection (>= 4.3)
+      System.Resources.ResourceManager (>= 4.3)
+      System.Runtime (>= 4.3)
+      System.Runtime.Extensions (>= 4.3)
+      System.Threading (>= 4.3)
+      System.Threading.Tasks (>= 4.3)
     System.Collections.Immutable (8.0)
     System.ComponentModel (4.3)
       System.Runtime (>= 4.3)
@@ -832,6 +887,18 @@ NUGET
       Microsoft.NETCore.Platforms (>= 1.1)
       Microsoft.NETCore.Targets (>= 1.1)
       System.Runtime (>= 4.3)
+    System.Globalization.Calendars (4.3)
+      Microsoft.NETCore.Platforms (>= 1.1)
+      Microsoft.NETCore.Targets (>= 1.1)
+      System.Globalization (>= 4.3)
+      System.Runtime (>= 4.3)
+    System.Globalization.Extensions (4.3)
+      Microsoft.NETCore.Platforms (>= 1.1)
+      System.Globalization (>= 4.3)
+      System.Resources.ResourceManager (>= 4.3)
+      System.Runtime (>= 4.3)
+      System.Runtime.Extensions (>= 4.3)
+      System.Runtime.InteropServices (>= 4.3)
     System.IdentityModel.Tokens.Jwt (7.5.2)
       Microsoft.IdentityModel.JsonWebTokens (>= 7.5.2)
       Microsoft.IdentityModel.Tokens (>= 7.5.2)
@@ -862,6 +929,33 @@ NUGET
     System.Memory (4.5.5)
     System.Memory.Data (8.0)
       System.Text.Json (>= 8.0)
+    System.Net.Http (4.3.4)
+      Microsoft.NETCore.Platforms (>= 1.1.1)
+      runtime.native.System (>= 4.3)
+      runtime.native.System.Net.Http (>= 4.3)
+      runtime.native.System.Security.Cryptography.OpenSsl (>= 4.3.2)
+      System.Collections (>= 4.3)
+      System.Diagnostics.Debug (>= 4.3)
+      System.Diagnostics.DiagnosticSource (>= 4.3)
+      System.Diagnostics.Tracing (>= 4.3)
+      System.Globalization (>= 4.3)
+      System.Globalization.Extensions (>= 4.3)
+      System.IO (>= 4.3)
+      System.IO.FileSystem (>= 4.3)
+      System.Net.Primitives (>= 4.3)
+      System.Resources.ResourceManager (>= 4.3)
+      System.Runtime (>= 4.3)
+      System.Runtime.Extensions (>= 4.3)
+      System.Runtime.Handles (>= 4.3)
+      System.Runtime.InteropServices (>= 4.3)
+      System.Security.Cryptography.Algorithms (>= 4.3)
+      System.Security.Cryptography.Encoding (>= 4.3)
+      System.Security.Cryptography.OpenSsl (>= 4.3)
+      System.Security.Cryptography.Primitives (>= 4.3)
+      System.Security.Cryptography.X509Certificates (>= 4.3)
+      System.Text.Encoding (>= 4.3)
+      System.Threading (>= 4.3)
+      System.Threading.Tasks (>= 4.3)
     System.Net.NameResolution (4.3)
       Microsoft.NETCore.Platforms (>= 1.1)
       runtime.native.System (>= 4.3)
@@ -926,9 +1020,94 @@ NUGET
       System.IO (>= 4.3)
       System.Reflection (>= 4.3)
       System.Runtime (>= 4.3)
+    System.Runtime.Numerics (4.3)
+      System.Globalization (>= 4.3)
+      System.Resources.ResourceManager (>= 4.3)
+      System.Runtime (>= 4.3)
+      System.Runtime.Extensions (>= 4.3)
+    System.Security.Cryptography.Algorithms (4.3.1)
+      Microsoft.NETCore.Platforms (>= 1.1)
+      runtime.native.System.Security.Cryptography.Apple (>= 4.3.1)
+      runtime.native.System.Security.Cryptography.OpenSsl (>= 4.3.2)
+      System.Collections (>= 4.3)
+      System.IO (>= 4.3)
+      System.Resources.ResourceManager (>= 4.3)
+      System.Runtime (>= 4.3)
+      System.Runtime.Extensions (>= 4.3)
+      System.Runtime.Handles (>= 4.3)
+      System.Runtime.InteropServices (>= 4.3)
+      System.Runtime.Numerics (>= 4.3)
+      System.Security.Cryptography.Encoding (>= 4.3)
+      System.Security.Cryptography.Primitives (>= 4.3)
+      System.Text.Encoding (>= 4.3)
+    System.Security.Cryptography.Cng (5.0)
+      System.Formats.Asn1 (>= 5.0)
+    System.Security.Cryptography.Csp (4.3)
+      Microsoft.NETCore.Platforms (>= 1.1)
+      System.IO (>= 4.3)
+      System.Reflection (>= 4.3)
+      System.Resources.ResourceManager (>= 4.3)
+      System.Runtime (>= 4.3)
+      System.Runtime.Extensions (>= 4.3)
+      System.Runtime.Handles (>= 4.3)
+      System.Runtime.InteropServices (>= 4.3)
+      System.Security.Cryptography.Algorithms (>= 4.3)
+      System.Security.Cryptography.Encoding (>= 4.3)
+      System.Security.Cryptography.Primitives (>= 4.3)
+      System.Text.Encoding (>= 4.3)
+      System.Threading (>= 4.3)
+    System.Security.Cryptography.Encoding (4.3)
+      Microsoft.NETCore.Platforms (>= 1.1)
+      runtime.native.System.Security.Cryptography.OpenSsl (>= 4.3)
+      System.Collections (>= 4.3)
+      System.Collections.Concurrent (>= 4.3)
+      System.Linq (>= 4.3)
+      System.Resources.ResourceManager (>= 4.3)
+      System.Runtime (>= 4.3)
+      System.Runtime.Extensions (>= 4.3)
+      System.Runtime.Handles (>= 4.3)
+      System.Runtime.InteropServices (>= 4.3)
+      System.Security.Cryptography.Primitives (>= 4.3)
+      System.Text.Encoding (>= 4.3)
+    System.Security.Cryptography.OpenSsl (5.0)
+      System.Formats.Asn1 (>= 5.0)
     System.Security.Cryptography.Pkcs (8.0)
       System.Formats.Asn1 (>= 8.0)
+    System.Security.Cryptography.Primitives (4.3)
+      System.Diagnostics.Debug (>= 4.3)
+      System.Globalization (>= 4.3)
+      System.IO (>= 4.3)
+      System.Resources.ResourceManager (>= 4.3)
+      System.Runtime (>= 4.3)
+      System.Threading (>= 4.3)
+      System.Threading.Tasks (>= 4.3)
     System.Security.Cryptography.ProtectedData (8.0)
+    System.Security.Cryptography.X509Certificates (4.3.2)
+      Microsoft.NETCore.Platforms (>= 1.1)
+      runtime.native.System (>= 4.3)
+      runtime.native.System.Net.Http (>= 4.3)
+      runtime.native.System.Security.Cryptography.OpenSsl (>= 4.3.2)
+      System.Collections (>= 4.3)
+      System.Diagnostics.Debug (>= 4.3)
+      System.Globalization (>= 4.3)
+      System.Globalization.Calendars (>= 4.3)
+      System.IO (>= 4.3)
+      System.IO.FileSystem (>= 4.3)
+      System.IO.FileSystem.Primitives (>= 4.3)
+      System.Resources.ResourceManager (>= 4.3)
+      System.Runtime (>= 4.3)
+      System.Runtime.Extensions (>= 4.3)
+      System.Runtime.Handles (>= 4.3)
+      System.Runtime.InteropServices (>= 4.3)
+      System.Runtime.Numerics (>= 4.3)
+      System.Security.Cryptography.Algorithms (>= 4.3)
+      System.Security.Cryptography.Cng (>= 4.3)
+      System.Security.Cryptography.Csp (>= 4.3)
+      System.Security.Cryptography.Encoding (>= 4.3)
+      System.Security.Cryptography.OpenSsl (>= 4.3)
+      System.Security.Cryptography.Primitives (>= 4.3)
+      System.Text.Encoding (>= 4.3)
+      System.Threading (>= 4.3)
     System.Security.Cryptography.Xml (8.0.1)
       System.Security.Cryptography.Pkcs (>= 8.0)
     System.Security.Principal.Windows (5.0)

diff --git a/src/AssociationRegistry.KboMutations.SyncLambda/Function.cs b/src/AssociationRegistry.KboMutations.SyncLambda/Function.cs
@@ -24,13 +24,33 @@
 
 namespace AssociationRegistry.KboMutations.SyncLambda;
 
+using global::OpenTelemetry.Metrics;
+using global::OpenTelemetry.Trace;
 using KboMutations.Configuration;
 using Notifications;
+using OpenTelemetry.Metrics;
+using OpenTelemetry.Trace;
 
 public class Function
 {
+    private const string MetricsUri = "OTLP_METRICS_URI";
+    private const string TracingUri = "OTLP_TRACING_URI";
+    private const string OtlpLogsUri = "OTLP_LOGS_URI";
+
+    private static TracerProvider _tracerProvider;
+    private static MeterProvider _meterProvider;
+
     private static async Task Main()
     {
+        var otlpMetricsUri = Environment.GetEnvironmentVariable(MetricsUri);
+        var otlpTracingUri = Environment.GetEnvironmentVariable(TracingUri);
+
+        if (!string.IsNullOrEmpty(otlpMetricsUri))
+            _meterProvider = OpenTelemetrySetup.SetupMeter(otlpMetricsUri);
+
+        if (!string.IsNullOrEmpty(otlpTracingUri))
+            _tracerProvider = OpenTelemetrySetup.SetUpTracing(otlpTracingUri);
+
         var handler = FunctionHandler;
         await LambdaBootstrapBuilder.Create(handler, new SourceGeneratorLambdaJsonSerializer<LambdaFunctionJsonSerializerContext>())
             .Build()
@@ -69,6 +89,9 @@ private static async Task FunctionHandler(SQSEvent @event, ILambdaContext contex
         var loggerFactory = LoggerFactory.Create(builder =>
         {
             builder.AddProvider(new LambdaLoggerProvider(context.Logger));
+            var otlpLogssUri = Environment.GetEnvironmentVariable(OtlpLogsUri);
+
+            OpenTelemetrySetup.SetUpLogging(otlpLogssUri, builder);
         });
 
         var repository = new VerenigingsRepository(new EventStore.EventStore(store, eventConflictResolver, loggerFactory.CreateLogger<EventStore.EventStore>()));
@@ -99,9 +122,9 @@ private static async Task FunctionHandler(SQSEvent @event, ILambdaContext contex
         context.Logger.LogInformation($"{@event.Records.Count} RECORDS PROCESSED BY THE MESSAGE PROCESSOR");
     }
 
-    private static async Task<MagdaOptionsSection> GetMagdaOptions(IConfiguration config,
-        SsmClientWrapper ssmClient,
-        ParamNamesConfiguration? paramNamesConfiguration)
+    public static async Task<MagdaOptionsSection> GetMagdaOptions(IConfiguration config,
+                                                                  SsmClientWrapper ssmClient,
+                                                                  ParamNamesConfiguration? paramNamesConfiguration)
     {
         var magdaOptions = config.GetSection(MagdaOptionsSection.SectionName)
             .Get<MagdaOptionsSection>();

diff --git a/src/AssociationRegistry.KboMutations.SyncLambda/OpenTelemetrySetup.cs b/src/AssociationRegistry.KboMutations.SyncLambda/OpenTelemetrySetup.cs
@@ -0,0 +1,69 @@
+namespace AssociationRegistry.KboMutations.SyncLambda;
+
+using Microsoft.Extensions.Logging;
+using OpenTelemetry;
+using OpenTelemetry.Exporter;
+using OpenTelemetry.Logs;
+using OpenTelemetry.Metrics;
+using OpenTelemetry.Trace;
+
+public class OpenTelemetrySetup
+{
+    private const string OtelAuthHeader = "OTEL_AUTH_HEADER";
+
+    public static MeterProvider SetupMeter(string otlpMetricsUri)
+    {
+        return Sdk.CreateMeterProviderBuilder()
+                  .AddMeter("KboMutations.SyncLambda.Metrics")
+                  .AddRuntimeInstrumentation()
+                  .AddHttpClientInstrumentation()
+                  .AddConsoleExporter()
+                  .AddOtlpExporter(options =>
+                   {
+                       options.Endpoint =
+                           new Uri(otlpMetricsUri);
+
+                       AddAuth(options);
+                   })
+                  .Build();
+    }
+
+    private static void AddAuth(OtlpExporterOptions options)
+    {
+        var authHeader = Environment.GetEnvironmentVariable(OtelAuthHeader);
+
+        if (!string.IsNullOrEmpty(authHeader))
+            options.Headers = $"Authorization={authHeader}";
+    }
+
+    public static TracerProvider SetUpTracing(string otlpTracingUri)
+    {
+        return Sdk.CreateTracerProviderBuilder()
+                  .AddSource("KboMutations.SyncLambda.Tracing")
+                  .AddAWSInstrumentation()// Optional: if Lambda is triggered by HTTP
+                  .AddConsoleExporter()          // Replace with a production exporter
+                  .AddOtlpExporter(options =>
+                   {
+                       options.Endpoint =
+                           new Uri(otlpTracingUri);
+
+                       AddAuth(options);
+                   })
+                  .Build();
+    }
+
+    public static void SetUpLogging(string? otlpLogssUri, ILoggingBuilder builder)
+    {
+        if (!string.IsNullOrEmpty(otlpLogssUri))
+            builder.AddOpenTelemetry(options =>
+            {
+                options.AddConsoleExporter()
+                       .AddOtlpExporter(options =>
+                        {
+                            options.Endpoint = new Uri(otlpLogssUri);
+
+                            AddAuth(options);
+                        });
+            });
+    }
+}
diff --git a/src/AssociationRegistry.KboMutations.SyncLambda/paket.references b/src/AssociationRegistry.KboMutations.SyncLambda/paket.references
@@ -1,3 +1,12 @@
 Amazon.Lambda.SQSEvents
 AWSSDK.Core
 NodaTime
+
+OpenTelemetry
+OpenTelemetry.Extensions.Hosting
+OpenTelemetry.Instrumentation.Runtime
+OpenTelemetry.Contrib.Instrumentation.AWS
+OpenTelemetry.Exporter.OpenTelemetryProtocol
+OpenTelemetry.Exporter.Console
+OpenTelemetry.Instrumentation.Http
+Npgsql.OpenTelemetry