From eb6e9dce74cf26d486dbff7f5ec5a036b95c4a4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=96hlund?= Date: Fri, 31 Jan 2025 14:15:29 +0100 Subject: [PATCH 01/15] Emit metrics via otel instead of custom format --- .../ServiceControlComponentRunner.cs | 8 ++- src/ServiceControl.Audit/App.config | 2 +- .../Auditing/AuditIngestion.cs | 29 ++++---- .../Auditing/AuditIngestor.cs | 23 +----- .../Auditing/AuditMetrics.cs | 9 +++ .../Auditing/AuditPersister.cs | 71 +++++++------------ .../HostApplicationBuilderExtensions.cs | 38 ++++++++-- .../Metrics/MetricsReporterHostedService.cs | 30 -------- .../MetricsServiceCollectionExtensions.cs | 14 ---- .../Infrastructure/Settings/Settings.cs | 8 ++- .../ServiceControl.Audit.csproj | 5 +- 11 files changed, 97 insertions(+), 140 deletions(-) create mode 100644 src/ServiceControl.Audit/Auditing/AuditMetrics.cs delete mode 100644 src/ServiceControl.Audit/Infrastructure/Metrics/MetricsReporterHostedService.cs delete mode 100644 src/ServiceControl.Audit/Infrastructure/Metrics/MetricsServiceCollectionExtensions.cs diff --git a/src/ServiceControl.Audit.AcceptanceTests/TestSupport/ServiceControlComponentRunner.cs b/src/ServiceControl.Audit.AcceptanceTests/TestSupport/ServiceControlComponentRunner.cs index 35990ee062..12c10ddab0 100644 --- a/src/ServiceControl.Audit.AcceptanceTests/TestSupport/ServiceControlComponentRunner.cs +++ b/src/ServiceControl.Audit.AcceptanceTests/TestSupport/ServiceControlComponentRunner.cs @@ -51,17 +51,19 @@ async Task InitializeServiceControl(ScenarioContext context) TransportConnectionString = transportToUse.ConnectionString, MaximumConcurrencyLevel = 2, ServiceControlQueueAddress = "SHOULDNOTBEUSED", + OtelMetricsUrl = "http://localhost:4317", MessageFilter = messageContext => { var id = messageContext.NativeMessageId; var headers = messageContext.Headers; - var log = NServiceBus.Logging.LogManager.GetLogger(); - headers.TryGetValue(Headers.MessageId, out var originalMessageId); + headers.TryGetValue(Headers.MessageId, + out var originalMessageId); log.Debug($"OnMessage for message '{id}'({originalMessageId ?? string.Empty})."); //Do not filter out CC, SA and HB messages as they can't be stamped - if (headers.TryGetValue(Headers.EnclosedMessageTypes, out var messageTypes) + if (headers.TryGetValue(Headers.EnclosedMessageTypes, + out var messageTypes) && (messageTypes.StartsWith("ServiceControl.Contracts") || messageTypes.StartsWith("ServiceControl.EndpointPlugin"))) { return false; diff --git a/src/ServiceControl.Audit/App.config b/src/ServiceControl.Audit/App.config index 83610fa6ee..7cf230c9f0 100644 --- a/src/ServiceControl.Audit/App.config +++ b/src/ServiceControl.Audit/App.config @@ -8,7 +8,7 @@ These settings are only here so that we can debug ServiceControl while developin - + diff --git a/src/ServiceControl.Audit/Auditing/AuditIngestion.cs b/src/ServiceControl.Audit/Auditing/AuditIngestion.cs index 56d4244e84..676930e539 100644 --- a/src/ServiceControl.Audit/Auditing/AuditIngestion.cs +++ b/src/ServiceControl.Audit/Auditing/AuditIngestion.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; using System.Diagnostics; + using System.Diagnostics.Metrics; using System.Threading; using System.Threading.Channels; using System.Threading.Tasks; @@ -14,18 +15,14 @@ using Persistence; using Persistence.UnitOfWork; using ServiceControl.Infrastructure; - using ServiceControl.Infrastructure.Metrics; using Transports; class AuditIngestion : IHostedService { - static readonly long FrequencyInMilliseconds = Stopwatch.Frequency / 1000; - public AuditIngestion( Settings settings, ITransportCustomization transportCustomization, TransportSettings transportSettings, - Metrics metrics, IFailedAuditStorage failedImportsStorage, AuditIngestionCustomCheck.State ingestionState, AuditIngestor auditIngestor, @@ -40,10 +37,6 @@ public AuditIngestion( this.settings = settings; this.applicationLifetime = applicationLifetime; - batchSizeMeter = metrics.GetMeter("Audit ingestion - batch size"); - batchDurationMeter = metrics.GetMeter("Audit ingestion - batch processing duration", FrequencyInMilliseconds); - receivedMeter = metrics.GetCounter("Audit ingestion - received"); - if (!transportSettings.MaxConcurrency.HasValue) { throw new ArgumentException("MaxConcurrency is not set in TransportSettings"); @@ -102,6 +95,7 @@ async Task EnsureStarted(CancellationToken cancellationToken = default) await stoppable.StopReceive(cancellationToken); logger.Info("Shutting down due to failed persistence health check. Infrastructure shut down completed"); } + return; } @@ -168,6 +162,7 @@ async Task EnsureStopped(CancellationToken cancellationToken = default) logger.Info("Shutting down. Already stopped, skipping shut down"); return; //Already stopped } + var stoppable = queueIngestor; queueIngestor = null; logger.Info("Shutting down. Infrastructure shut down commencing"); @@ -196,7 +191,7 @@ async Task OnMessage(MessageContext messageContext, CancellationToken cancellati var taskCompletionSource = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); messageContext.SetTaskCompletionSource(taskCompletionSource); - receivedMeter.Mark(); + receivedMeter.Add(1); await channel.Writer.WriteAsync(messageContext, cancellationToken); await taskCompletionSource.Task; @@ -217,11 +212,11 @@ async Task Loop() contexts.Add(context); } - batchSizeMeter.Mark(contexts.Count); - using (batchDurationMeter.Measure()) - { - await auditIngestor.Ingest(contexts); - } + batchSizeMeter.Record(contexts.Count); + var sw = Stopwatch.StartNew(); + + await auditIngestor.Ingest(contexts); + batchDurationMeter.Record(sw.ElapsedMilliseconds); } catch (OperationCanceledException) { @@ -261,9 +256,9 @@ async Task Loop() readonly IAuditIngestionUnitOfWorkFactory unitOfWorkFactory; readonly Settings settings; readonly Channel channel; - readonly Meter batchSizeMeter; - readonly Meter batchDurationMeter; - readonly Counter receivedMeter; + readonly Histogram batchSizeMeter = AuditMetrics.Meter.CreateHistogram($"{AuditMetrics.Prefix}.batch_size"); + readonly Histogram batchDurationMeter = AuditMetrics.Meter.CreateHistogram($"{AuditMetrics.Prefix}.batch_duration_ms"); + readonly Counter receivedMeter = AuditMetrics.Meter.CreateCounter($"{AuditMetrics.Prefix}.received"); readonly Watchdog watchdog; readonly Task ingestionWorker; readonly IHostApplicationLifetime applicationLifetime; diff --git a/src/ServiceControl.Audit/Auditing/AuditIngestor.cs b/src/ServiceControl.Audit/Auditing/AuditIngestor.cs index 84d110a1f2..c2ec2623d1 100644 --- a/src/ServiceControl.Audit/Auditing/AuditIngestor.cs +++ b/src/ServiceControl.Audit/Auditing/AuditIngestor.cs @@ -14,13 +14,11 @@ using Persistence.UnitOfWork; using Recoverability; using SagaAudit; - using ServiceControl.Infrastructure.Metrics; using ServiceControl.Transports; public class AuditIngestor { public AuditIngestor( - Metrics metrics, Settings settings, IAuditIngestionUnitOfWorkFactory unitOfWorkFactory, EndpointInstanceMonitoring endpointInstanceMonitoring, @@ -32,26 +30,11 @@ ITransportCustomization transportCustomization { this.settings = settings; this.messageDispatcher = messageDispatcher; - - var ingestedAuditMeter = metrics.GetCounter("Audit ingestion - ingested audit"); - var ingestedSagaAuditMeter = metrics.GetCounter("Audit ingestion - ingested saga audit"); - var auditBulkInsertDurationMeter = metrics.GetMeter("Audit ingestion - audit bulk insert duration", FrequencyInMilliseconds); - var sagaAuditBulkInsertDurationMeter = metrics.GetMeter("Audit ingestion - saga audit bulk insert duration", FrequencyInMilliseconds); - var bulkInsertCommitDurationMeter = metrics.GetMeter("Audit ingestion - bulk insert commit duration", FrequencyInMilliseconds); - - var enrichers = new IEnrichImportedAuditMessages[] - { - new MessageTypeEnricher(), - new EnrichWithTrackingIds(), - new ProcessingStatisticsEnricher(), - new DetectNewEndpointsFromAuditImportsEnricher(endpointInstanceMonitoring), - new DetectSuccessfulRetriesEnricher(), - new SagaRelationshipsEnricher() - }.Concat(auditEnrichers).ToArray(); + var enrichers = new IEnrichImportedAuditMessages[] { new MessageTypeEnricher(), new EnrichWithTrackingIds(), new ProcessingStatisticsEnricher(), new DetectNewEndpointsFromAuditImportsEnricher(endpointInstanceMonitoring), new DetectSuccessfulRetriesEnricher(), new SagaRelationshipsEnricher() }.Concat(auditEnrichers).ToArray(); logQueueAddress = transportCustomization.ToTransportQualifiedQueueName(settings.AuditLogQueue); - auditPersister = new AuditPersister(unitOfWorkFactory, enrichers, ingestedAuditMeter, ingestedSagaAuditMeter, auditBulkInsertDurationMeter, sagaAuditBulkInsertDurationMeter, bulkInsertCommitDurationMeter, messageSession, messageDispatcher); + auditPersister = new AuditPersister(unitOfWorkFactory, enrichers, messageSession, messageDispatcher); } public async Task Ingest(List contexts) @@ -71,6 +54,7 @@ public async Task Ingest(List contexts) { Log.Debug($"Forwarding {stored.Count} messages"); } + await Forward(stored, logQueueAddress); if (Log.IsDebugEnabled) { @@ -159,7 +143,6 @@ public async Task VerifyCanReachForwardingAddress() readonly Lazy messageDispatcher; readonly string logQueueAddress; - static readonly long FrequencyInMilliseconds = Stopwatch.Frequency / 1000; static readonly ILog Log = LogManager.GetLogger(); } } \ No newline at end of file diff --git a/src/ServiceControl.Audit/Auditing/AuditMetrics.cs b/src/ServiceControl.Audit/Auditing/AuditMetrics.cs new file mode 100644 index 0000000000..3e02448fa1 --- /dev/null +++ b/src/ServiceControl.Audit/Auditing/AuditMetrics.cs @@ -0,0 +1,9 @@ +namespace ServiceControl.Audit.Auditing; + +using System.Diagnostics.Metrics; + +static class AuditMetrics +{ + public static readonly Meter Meter = new("ServiceControl", "0.1.0"); + public static readonly string Prefix = "particular.servicecontrol.audit"; +} \ No newline at end of file diff --git a/src/ServiceControl.Audit/Auditing/AuditPersister.cs b/src/ServiceControl.Audit/Auditing/AuditPersister.cs index 0a5d0d9938..54e7f0c2da 100644 --- a/src/ServiceControl.Audit/Auditing/AuditPersister.cs +++ b/src/ServiceControl.Audit/Auditing/AuditPersister.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; using System.Diagnostics; + using System.Diagnostics.Metrics; using System.Text.Json; using System.Threading.Tasks; using Infrastructure; @@ -15,29 +16,13 @@ using ServiceControl.Audit.Persistence.Monitoring; using ServiceControl.EndpointPlugin.Messages.SagaState; using ServiceControl.Infrastructure; - using ServiceControl.Infrastructure.Metrics; using ServiceControl.SagaAudit; - class AuditPersister + class AuditPersister(IAuditIngestionUnitOfWorkFactory unitOfWorkFactory, + IEnrichImportedAuditMessages[] enrichers, + IMessageSession messageSession, + Lazy messageDispatcher) { - public AuditPersister(IAuditIngestionUnitOfWorkFactory unitOfWorkFactory, - IEnrichImportedAuditMessages[] enrichers, - Counter ingestedAuditMeter, Counter ingestedSagaAuditMeter, Meter auditBulkInsertDurationMeter, - Meter sagaAuditBulkInsertDurationMeter, Meter bulkInsertCommitDurationMeter, IMessageSession messageSession, - Lazy messageDispatcher) - { - this.unitOfWorkFactory = unitOfWorkFactory; - this.enrichers = enrichers; - - this.ingestedAuditMeter = ingestedAuditMeter; - this.ingestedSagaAuditMeter = ingestedSagaAuditMeter; - this.auditBulkInsertDurationMeter = auditBulkInsertDurationMeter; - this.sagaAuditBulkInsertDurationMeter = sagaAuditBulkInsertDurationMeter; - this.bulkInsertCommitDurationMeter = bulkInsertCommitDurationMeter; - this.messageSession = messageSession; - this.messageDispatcher = messageDispatcher; - } - public async Task> Persist(IReadOnlyList contexts) { var stopwatch = Stopwatch.StartNew(); @@ -51,7 +36,6 @@ public async Task> Persist(IReadOnlyList(contexts.Count); @@ -89,12 +73,13 @@ public async Task> Persist(IReadOnlyList> Persist(IReadOnlyList> Persist(IReadOnlyList messageDispatcher; + readonly Counter ingestedAuditMeter = AuditMetrics.Meter.CreateCounter($"{AuditMetrics.Prefix}.ingested_audit_messages"); // metrics.GetCounter("Audit ingestion - ingested audit"); + readonly Counter ingestedSagaAuditMeter = AuditMetrics.Meter.CreateCounter($"{AuditMetrics.Prefix}.ingested_saga_audits"); // metrics.GetCounter("Audit ingestion - ingested audit"); + readonly Histogram auditBulkInsertDurationMeter = AuditMetrics.Meter.CreateHistogram($"{AuditMetrics.Prefix}.audit_bulk_insert_duration_ms"); // metrics.GetCounter("Audit ingestion - ingested audit"); + readonly Histogram sagaAuditBulkInsertDurationMeter = AuditMetrics.Meter.CreateHistogram($"{AuditMetrics.Prefix}.saga_bulk_insert_duration_ms"); // metrics.GetCounter("Audit ingestion - ingested audit"); + readonly Histogram bulkInsertCommitDurationMeter = AuditMetrics.Meter.CreateHistogram($"{AuditMetrics.Prefix}.audit_commit_duration_ms"); // metrics.GetCounter("Audit ingestion - ingested audit"); - readonly IEnrichImportedAuditMessages[] enrichers; - readonly IAuditIngestionUnitOfWorkFactory unitOfWorkFactory; static readonly ILog Logger = LogManager.GetLogger(); } } \ No newline at end of file diff --git a/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs b/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs index 8968565a50..1840ba6908 100644 --- a/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs +++ b/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs @@ -5,8 +5,8 @@ namespace ServiceControl.Audit; using System.Threading; using System.Threading.Tasks; using Auditing; +using Azure.Monitor.OpenTelemetry.Exporter; using Infrastructure; -using Infrastructure.Metrics; using Infrastructure.Settings; using Microsoft.AspNetCore.HttpLogging; using Microsoft.Extensions.DependencyInjection; @@ -20,6 +20,8 @@ namespace ServiceControl.Audit; using NServiceBus.Transport; using Persistence; using Transports; +using OpenTelemetry.Metrics; +using OpenTelemetry.Resources; static class HostApplicationBuilderExtensions { @@ -61,13 +63,38 @@ public static void AddServiceControlAudit(this IHostApplicationBuilder builder, // directly and to make things more complex of course the order of registration still matters ;) services.AddSingleton(provider => new Lazy(provider.GetRequiredService)); - services.AddMetrics(settings.PrintMetrics); - services.AddPersistence(persistenceSettings, persistenceConfiguration); NServiceBusFactory.Configure(settings, transportCustomization, transportSettings, onCriticalError, configuration); builder.UseNServiceBus(configuration); + if (!string.IsNullOrEmpty(settings.OtelMetricsUrl)) + { + builder.Services.AddOpenTelemetry() + .ConfigureResource(b => b.AddService(serviceName: settings.InstanceName)) + .WithMetrics(b => + { + b.AddMeter("ServiceControl"); + + if (Uri.TryCreate(settings.OtelMetricsUrl, UriKind.Absolute, out var uri)) + { + b.AddOtlpExporter(e => + { + e.Endpoint = uri; + }); + } + else + { + b.AddAzureMonitorMetricExporter(o => + { + o.ConnectionString = settings.OtelMetricsUrl; + }); + } + + b.AddConsoleExporter(); + }); + } + // Configure after the NServiceBus hosted service to ensure NServiceBus is already started if (settings.IngestAuditMessages) { @@ -101,9 +128,6 @@ static void RecordStartup(Settings settings, EndpointConfiguration endpointConfi var logger = LogManager.GetLogger(typeof(HostApplicationBuilderExtensions)); logger.Info(startupMessage); - endpointConfiguration.GetSettings().AddStartupDiagnosticsSection("Startup", new - { - Settings = settings - }); + endpointConfiguration.GetSettings().AddStartupDiagnosticsSection("Startup", new { Settings = settings }); } } \ No newline at end of file diff --git a/src/ServiceControl.Audit/Infrastructure/Metrics/MetricsReporterHostedService.cs b/src/ServiceControl.Audit/Infrastructure/Metrics/MetricsReporterHostedService.cs deleted file mode 100644 index c378a8f4d6..0000000000 --- a/src/ServiceControl.Audit/Infrastructure/Metrics/MetricsReporterHostedService.cs +++ /dev/null @@ -1,30 +0,0 @@ -namespace ServiceControl.Audit.Infrastructure.Metrics -{ - using System; - using System.Threading; - using System.Threading.Tasks; - using Microsoft.Extensions.Hosting; - using NServiceBus.Logging; - using ServiceControl.Infrastructure.Metrics; - - class MetricsReporterHostedService : IHostedService - { - readonly Metrics metrics; - MetricsReporter reporter; - - public MetricsReporterHostedService(Metrics metrics) => this.metrics = metrics; - - public Task StartAsync(CancellationToken cancellationToken) - { - var metricsLog = LogManager.GetLogger("Metrics"); - - reporter = new MetricsReporter(metrics, x => metricsLog.Info(x), TimeSpan.FromSeconds(5)); - - reporter.Start(); - - return Task.CompletedTask; - } - - public Task StopAsync(CancellationToken cancellationToken) => reporter.Stop(); - } -} \ No newline at end of file diff --git a/src/ServiceControl.Audit/Infrastructure/Metrics/MetricsServiceCollectionExtensions.cs b/src/ServiceControl.Audit/Infrastructure/Metrics/MetricsServiceCollectionExtensions.cs deleted file mode 100644 index a7bc6b1dd5..0000000000 --- a/src/ServiceControl.Audit/Infrastructure/Metrics/MetricsServiceCollectionExtensions.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace ServiceControl.Audit.Infrastructure.Metrics -{ - using Microsoft.Extensions.DependencyInjection; - using ServiceControl.Infrastructure.Metrics; - - static class MetricsServiceCollectionExtensions - { - public static void AddMetrics(this IServiceCollection services, bool printMetrics) - { - services.AddSingleton(new Metrics { Enabled = printMetrics }); - services.AddHostedService(); - } - } -} \ No newline at end of file diff --git a/src/ServiceControl.Audit/Infrastructure/Settings/Settings.cs b/src/ServiceControl.Audit/Infrastructure/Settings/Settings.cs index 183b695555..b77fcc295d 100644 --- a/src/ServiceControl.Audit/Infrastructure/Settings/Settings.cs +++ b/src/ServiceControl.Audit/Infrastructure/Settings/Settings.cs @@ -43,7 +43,9 @@ public Settings(string transportType = null, string persisterType = null, Loggin { Hostname = SettingsReader.Read(SettingsRootNamespace, "Hostname", "localhost"); Port = SettingsReader.Read(SettingsRootNamespace, "Port", 44444); - }; + } + + ; MaximumConcurrencyLevel = SettingsReader.Read(SettingsRootNamespace, "MaximumConcurrencyLevel"); ServiceControlQueueAddress = SettingsReader.Read(SettingsRootNamespace, "ServiceControlQueueAddress"); @@ -79,8 +81,7 @@ void LoadAuditQueueInformation() } } - [JsonIgnore] - public Func AssemblyLoadContextResolver { get; set; } + [JsonIgnore] public Func AssemblyLoadContextResolver { get; set; } public LoggingSettings LoggingSettings { get; } @@ -109,6 +110,7 @@ public string RootUrl public int Port { get; set; } public bool PrintMetrics => SettingsReader.Read(SettingsRootNamespace, "PrintMetrics"); + public string OtelMetricsUrl { get; set; } = SettingsReader.Read(SettingsRootNamespace, nameof(OtelMetricsUrl)); public string Hostname { get; private set; } public string VirtualDirectory => SettingsReader.Read(SettingsRootNamespace, "VirtualDirectory", string.Empty); diff --git a/src/ServiceControl.Audit/ServiceControl.Audit.csproj b/src/ServiceControl.Audit/ServiceControl.Audit.csproj index 3303782e3f..c1c7ee7075 100644 --- a/src/ServiceControl.Audit/ServiceControl.Audit.csproj +++ b/src/ServiceControl.Audit/ServiceControl.Audit.csproj @@ -18,17 +18,20 @@ - + + + + From 2d0880b3111f13fca91e4be46d627c5bb753a16a Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Fri, 31 Jan 2025 14:50:56 +0100 Subject: [PATCH 02/15] Minimize diff by reverting whitespace changes --- .../ServiceControlComponentRunner.cs | 6 ++---- .../Auditing/AuditIngestion.cs | 2 -- .../Auditing/AuditIngestor.cs | 18 +++++++++++++++--- .../HostApplicationBuilderExtensions.cs | 5 ++++- .../Infrastructure/Settings/Settings.cs | 7 +++---- 5 files changed, 24 insertions(+), 14 deletions(-) diff --git a/src/ServiceControl.Audit.AcceptanceTests/TestSupport/ServiceControlComponentRunner.cs b/src/ServiceControl.Audit.AcceptanceTests/TestSupport/ServiceControlComponentRunner.cs index 12c10ddab0..ba6f54ad66 100644 --- a/src/ServiceControl.Audit.AcceptanceTests/TestSupport/ServiceControlComponentRunner.cs +++ b/src/ServiceControl.Audit.AcceptanceTests/TestSupport/ServiceControlComponentRunner.cs @@ -57,13 +57,11 @@ async Task InitializeServiceControl(ScenarioContext context) var id = messageContext.NativeMessageId; var headers = messageContext.Headers; var log = NServiceBus.Logging.LogManager.GetLogger(); - headers.TryGetValue(Headers.MessageId, - out var originalMessageId); + headers.TryGetValue(Headers.MessageId, out var originalMessageId); log.Debug($"OnMessage for message '{id}'({originalMessageId ?? string.Empty})."); //Do not filter out CC, SA and HB messages as they can't be stamped - if (headers.TryGetValue(Headers.EnclosedMessageTypes, - out var messageTypes) + if (headers.TryGetValue(Headers.EnclosedMessageTypes, out var messageTypes) && (messageTypes.StartsWith("ServiceControl.Contracts") || messageTypes.StartsWith("ServiceControl.EndpointPlugin"))) { return false; diff --git a/src/ServiceControl.Audit/Auditing/AuditIngestion.cs b/src/ServiceControl.Audit/Auditing/AuditIngestion.cs index 676930e539..3c5b47a288 100644 --- a/src/ServiceControl.Audit/Auditing/AuditIngestion.cs +++ b/src/ServiceControl.Audit/Auditing/AuditIngestion.cs @@ -95,7 +95,6 @@ async Task EnsureStarted(CancellationToken cancellationToken = default) await stoppable.StopReceive(cancellationToken); logger.Info("Shutting down due to failed persistence health check. Infrastructure shut down completed"); } - return; } @@ -162,7 +161,6 @@ async Task EnsureStopped(CancellationToken cancellationToken = default) logger.Info("Shutting down. Already stopped, skipping shut down"); return; //Already stopped } - var stoppable = queueIngestor; queueIngestor = null; logger.Info("Shutting down. Infrastructure shut down commencing"); diff --git a/src/ServiceControl.Audit/Auditing/AuditIngestor.cs b/src/ServiceControl.Audit/Auditing/AuditIngestor.cs index c2ec2623d1..487b7e0e1a 100644 --- a/src/ServiceControl.Audit/Auditing/AuditIngestor.cs +++ b/src/ServiceControl.Audit/Auditing/AuditIngestor.cs @@ -30,11 +30,24 @@ ITransportCustomization transportCustomization { this.settings = settings; this.messageDispatcher = messageDispatcher; - var enrichers = new IEnrichImportedAuditMessages[] { new MessageTypeEnricher(), new EnrichWithTrackingIds(), new ProcessingStatisticsEnricher(), new DetectNewEndpointsFromAuditImportsEnricher(endpointInstanceMonitoring), new DetectSuccessfulRetriesEnricher(), new SagaRelationshipsEnricher() }.Concat(auditEnrichers).ToArray(); + var enrichers = new IEnrichImportedAuditMessages[] + { + new MessageTypeEnricher(), + new EnrichWithTrackingIds(), + new ProcessingStatisticsEnricher(), + new DetectNewEndpointsFromAuditImportsEnricher(endpointInstanceMonitoring), + new DetectSuccessfulRetriesEnricher(), + new SagaRelationshipsEnricher() + }.Concat(auditEnrichers).ToArray(); logQueueAddress = transportCustomization.ToTransportQualifiedQueueName(settings.AuditLogQueue); - auditPersister = new AuditPersister(unitOfWorkFactory, enrichers, messageSession, messageDispatcher); + auditPersister = new AuditPersister( + unitOfWorkFactory, + enrichers, + messageSession, + messageDispatcher + ); } public async Task Ingest(List contexts) @@ -54,7 +67,6 @@ public async Task Ingest(List contexts) { Log.Debug($"Forwarding {stored.Count} messages"); } - await Forward(stored, logQueueAddress); if (Log.IsDebugEnabled) { diff --git a/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs b/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs index 1840ba6908..f90fe923f9 100644 --- a/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs +++ b/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs @@ -128,6 +128,9 @@ static void RecordStartup(Settings settings, EndpointConfiguration endpointConfi var logger = LogManager.GetLogger(typeof(HostApplicationBuilderExtensions)); logger.Info(startupMessage); - endpointConfiguration.GetSettings().AddStartupDiagnosticsSection("Startup", new { Settings = settings }); + endpointConfiguration.GetSettings().AddStartupDiagnosticsSection("Startup", new + { + Settings = settings + }); } } \ No newline at end of file diff --git a/src/ServiceControl.Audit/Infrastructure/Settings/Settings.cs b/src/ServiceControl.Audit/Infrastructure/Settings/Settings.cs index b77fcc295d..d6a84f0662 100644 --- a/src/ServiceControl.Audit/Infrastructure/Settings/Settings.cs +++ b/src/ServiceControl.Audit/Infrastructure/Settings/Settings.cs @@ -43,9 +43,7 @@ public Settings(string transportType = null, string persisterType = null, Loggin { Hostname = SettingsReader.Read(SettingsRootNamespace, "Hostname", "localhost"); Port = SettingsReader.Read(SettingsRootNamespace, "Port", 44444); - } - - ; + }; MaximumConcurrencyLevel = SettingsReader.Read(SettingsRootNamespace, "MaximumConcurrencyLevel"); ServiceControlQueueAddress = SettingsReader.Read(SettingsRootNamespace, "ServiceControlQueueAddress"); @@ -81,7 +79,8 @@ void LoadAuditQueueInformation() } } - [JsonIgnore] public Func AssemblyLoadContextResolver { get; set; } + [JsonIgnore] + public Func AssemblyLoadContextResolver { get; set; } public LoggingSettings LoggingSettings { get; } From 9652aa03c587f6b288b53ef4e10b90f4b19fe7b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=96hlund?= Date: Fri, 31 Jan 2025 14:52:20 +0100 Subject: [PATCH 03/15] Approvals --- .../APIApprovals.PlatformSampleSettings.approved.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ServiceControl.Audit.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt b/src/ServiceControl.Audit.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt index c1dbbb4ec1..9862349774 100644 --- a/src/ServiceControl.Audit.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt +++ b/src/ServiceControl.Audit.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt @@ -12,6 +12,7 @@ "ApiUrl": "http://localhost:8888/api", "Port": 8888, "PrintMetrics": false, + "OtelMetricsUrl": null, "Hostname": "localhost", "VirtualDirectory": "", "TransportType": "LearningTransport", From 77b2c2c08d220db1ddeb1246608f15c1371d80bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=96hlund?= Date: Fri, 31 Jan 2025 15:05:51 +0100 Subject: [PATCH 04/15] Fix formatting --- src/ServiceControl.Audit/Auditing/AuditIngestor.cs | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/ServiceControl.Audit/Auditing/AuditIngestor.cs b/src/ServiceControl.Audit/Auditing/AuditIngestor.cs index 487b7e0e1a..fb6f2a7240 100644 --- a/src/ServiceControl.Audit/Auditing/AuditIngestor.cs +++ b/src/ServiceControl.Audit/Auditing/AuditIngestor.cs @@ -2,7 +2,6 @@ { using System; using System.Collections.Generic; - using System.Diagnostics; using System.Linq; using System.Threading.Tasks; using Infrastructure.Settings; @@ -30,15 +29,7 @@ ITransportCustomization transportCustomization { this.settings = settings; this.messageDispatcher = messageDispatcher; - var enrichers = new IEnrichImportedAuditMessages[] - { - new MessageTypeEnricher(), - new EnrichWithTrackingIds(), - new ProcessingStatisticsEnricher(), - new DetectNewEndpointsFromAuditImportsEnricher(endpointInstanceMonitoring), - new DetectSuccessfulRetriesEnricher(), - new SagaRelationshipsEnricher() - }.Concat(auditEnrichers).ToArray(); + var enrichers = new IEnrichImportedAuditMessages[] { new MessageTypeEnricher(), new EnrichWithTrackingIds(), new ProcessingStatisticsEnricher(), new DetectNewEndpointsFromAuditImportsEnricher(endpointInstanceMonitoring), new DetectSuccessfulRetriesEnricher(), new SagaRelationshipsEnricher() }.Concat(auditEnrichers).ToArray(); logQueueAddress = transportCustomization.ToTransportQualifiedQueueName(settings.AuditLogQueue); @@ -67,6 +58,7 @@ public async Task Ingest(List contexts) { Log.Debug($"Forwarding {stored.Count} messages"); } + await Forward(stored, logQueueAddress); if (Log.IsDebugEnabled) { From 03690a08672dadd7e70ffc8a439edd0bb6e3b4e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=96hlund?= Date: Fri, 31 Jan 2025 15:07:09 +0100 Subject: [PATCH 05/15] Apply suggestions from code review Co-authored-by: Ramon Smits --- src/ServiceControl.Audit/Auditing/AuditPersister.cs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/ServiceControl.Audit/Auditing/AuditPersister.cs b/src/ServiceControl.Audit/Auditing/AuditPersister.cs index 54e7f0c2da..adae02c6b0 100644 --- a/src/ServiceControl.Audit/Auditing/AuditPersister.cs +++ b/src/ServiceControl.Audit/Auditing/AuditPersister.cs @@ -75,8 +75,6 @@ public async Task> Persist(IReadOnlyList> Persist(IReadOnlyList> Persist(IReadOnlyList Date: Fri, 31 Jan 2025 15:21:50 +0100 Subject: [PATCH 06/15] fix formatting --- src/ServiceControl.Audit/Auditing/AuditPersister.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ServiceControl.Audit/Auditing/AuditPersister.cs b/src/ServiceControl.Audit/Auditing/AuditPersister.cs index adae02c6b0..16eae2f401 100644 --- a/src/ServiceControl.Audit/Auditing/AuditPersister.cs +++ b/src/ServiceControl.Audit/Auditing/AuditPersister.cs @@ -130,7 +130,7 @@ public async Task> Persist(IReadOnlyList Date: Sun, 2 Feb 2025 17:50:13 +0100 Subject: [PATCH 07/15] Only use standar otel --- src/Directory.Packages.props | 3 +++ .../HostApplicationBuilderExtensions.cs | 22 ++++--------------- .../ServiceControl.Audit.csproj | 1 - 3 files changed, 7 insertions(+), 19 deletions(-) diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index 37eaf6fc3b..c1132dbdf8 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -50,6 +50,9 @@ + + + diff --git a/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs b/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs index f90fe923f9..d981b55d3b 100644 --- a/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs +++ b/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs @@ -5,7 +5,6 @@ namespace ServiceControl.Audit; using System.Threading; using System.Threading.Tasks; using Auditing; -using Azure.Monitor.OpenTelemetry.Exporter; using Infrastructure; using Infrastructure.Settings; using Microsoft.AspNetCore.HttpLogging; @@ -76,20 +75,10 @@ public static void AddServiceControlAudit(this IHostApplicationBuilder builder, { b.AddMeter("ServiceControl"); - if (Uri.TryCreate(settings.OtelMetricsUrl, UriKind.Absolute, out var uri)) + b.AddOtlpExporter(e => { - b.AddOtlpExporter(e => - { - e.Endpoint = uri; - }); - } - else - { - b.AddAzureMonitorMetricExporter(o => - { - o.ConnectionString = settings.OtelMetricsUrl; - }); - } + e.Endpoint = new Uri(settings.OtelMetricsUrl); + }); b.AddConsoleExporter(); }); @@ -128,9 +117,6 @@ static void RecordStartup(Settings settings, EndpointConfiguration endpointConfi var logger = LogManager.GetLogger(typeof(HostApplicationBuilderExtensions)); logger.Info(startupMessage); - endpointConfiguration.GetSettings().AddStartupDiagnosticsSection("Startup", new - { - Settings = settings - }); + endpointConfiguration.GetSettings().AddStartupDiagnosticsSection("Startup", new { Settings = settings }); } } \ No newline at end of file diff --git a/src/ServiceControl.Audit/ServiceControl.Audit.csproj b/src/ServiceControl.Audit/ServiceControl.Audit.csproj index c1c7ee7075..084b5a48ce 100644 --- a/src/ServiceControl.Audit/ServiceControl.Audit.csproj +++ b/src/ServiceControl.Audit/ServiceControl.Audit.csproj @@ -23,7 +23,6 @@ - From 7616151d636aa3cacf889ab6fda0049aa1c3c6f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=96hlund?= Date: Sun, 2 Feb 2025 20:05:41 +0100 Subject: [PATCH 08/15] Set instance id --- src/Directory.Packages.props | 1 - .../HostApplicationBuilderExtensions.cs | 21 +++++++++++-------- .../ServiceControl.Audit.csproj | 1 - 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index c1132dbdf8..044ee33716 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -50,7 +50,6 @@ - diff --git a/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs b/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs index d981b55d3b..d1f08569dc 100644 --- a/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs +++ b/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs @@ -29,10 +29,11 @@ public static void AddServiceControlAudit(this IHostApplicationBuilder builder, Settings settings, EndpointConfiguration configuration) { + var version = FileVersionInfo.GetVersionInfo(typeof(HostApplicationBuilderExtensions).Assembly.Location).ProductVersion; var persistenceConfiguration = PersistenceConfigurationFactory.LoadPersistenceConfiguration(settings); var persistenceSettings = persistenceConfiguration.BuildPersistenceSettings(settings); - RecordStartup(settings, configuration, persistenceConfiguration); + RecordStartup(version, settings, configuration, persistenceConfiguration); builder.Logging.ClearProviders(); builder.Logging.AddNLog(); @@ -69,18 +70,22 @@ public static void AddServiceControlAudit(this IHostApplicationBuilder builder, if (!string.IsNullOrEmpty(settings.OtelMetricsUrl)) { + if (!Uri.TryCreate(settings.OtelMetricsUrl, UriKind.Absolute, out var otelMetricsUri)) + { + throw new UriFormatException($"Invalid OtelMetricsUrl: {settings.OtelMetricsUrl}"); + } builder.Services.AddOpenTelemetry() - .ConfigureResource(b => b.AddService(serviceName: settings.InstanceName)) + .ConfigureResource(b => b.AddService( + serviceName: "Particular.ServiceControl.Audit", + serviceVersion: version, + serviceInstanceId: settings.InstanceName)) .WithMetrics(b => { b.AddMeter("ServiceControl"); - b.AddOtlpExporter(e => { - e.Endpoint = new Uri(settings.OtelMetricsUrl); + e.Endpoint = otelMetricsUri; }); - - b.AddConsoleExporter(); }); } @@ -100,10 +105,8 @@ public static void AddServiceControlAuditInstallers(this IHostApplicationBuilder builder.Services.AddInstaller(persistenceSettings, persistenceConfiguration); } - static void RecordStartup(Settings settings, EndpointConfiguration endpointConfiguration, IPersistenceConfiguration persistenceConfiguration) + static void RecordStartup(string version, Settings settings, EndpointConfiguration endpointConfiguration, IPersistenceConfiguration persistenceConfiguration) { - var version = FileVersionInfo.GetVersionInfo(typeof(HostApplicationBuilderExtensions).Assembly.Location).ProductVersion; - var startupMessage = $@" ------------------------------------------------------------- ServiceControl Audit Version: {version} diff --git a/src/ServiceControl.Audit/ServiceControl.Audit.csproj b/src/ServiceControl.Audit/ServiceControl.Audit.csproj index 084b5a48ce..8f41ba97b1 100644 --- a/src/ServiceControl.Audit/ServiceControl.Audit.csproj +++ b/src/ServiceControl.Audit/ServiceControl.Audit.csproj @@ -28,7 +28,6 @@ - From e6d1f40219228c155d6c7438befcfc3e1158e5b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=96hlund?= Date: Mon, 3 Feb 2025 08:29:51 +0100 Subject: [PATCH 09/15] Set unit --- .../TestSupport/ServiceControlComponentRunner.cs | 1 - src/ServiceControl.Audit/Auditing/AuditIngestion.cs | 4 +++- src/ServiceControl.Audit/Auditing/AuditPersister.cs | 10 +++++----- .../{Auditing => Infrastructure}/AuditMetrics.cs | 4 ++-- 4 files changed, 10 insertions(+), 9 deletions(-) rename src/ServiceControl.Audit/{Auditing => Infrastructure}/AuditMetrics.cs (55%) diff --git a/src/ServiceControl.Audit.AcceptanceTests/TestSupport/ServiceControlComponentRunner.cs b/src/ServiceControl.Audit.AcceptanceTests/TestSupport/ServiceControlComponentRunner.cs index ba6f54ad66..0866b504db 100644 --- a/src/ServiceControl.Audit.AcceptanceTests/TestSupport/ServiceControlComponentRunner.cs +++ b/src/ServiceControl.Audit.AcceptanceTests/TestSupport/ServiceControlComponentRunner.cs @@ -51,7 +51,6 @@ async Task InitializeServiceControl(ScenarioContext context) TransportConnectionString = transportToUse.ConnectionString, MaximumConcurrencyLevel = 2, ServiceControlQueueAddress = "SHOULDNOTBEUSED", - OtelMetricsUrl = "http://localhost:4317", MessageFilter = messageContext => { var id = messageContext.NativeMessageId; diff --git a/src/ServiceControl.Audit/Auditing/AuditIngestion.cs b/src/ServiceControl.Audit/Auditing/AuditIngestion.cs index 3c5b47a288..93f93ca665 100644 --- a/src/ServiceControl.Audit/Auditing/AuditIngestion.cs +++ b/src/ServiceControl.Audit/Auditing/AuditIngestion.cs @@ -95,6 +95,7 @@ async Task EnsureStarted(CancellationToken cancellationToken = default) await stoppable.StopReceive(cancellationToken); logger.Info("Shutting down due to failed persistence health check. Infrastructure shut down completed"); } + return; } @@ -161,6 +162,7 @@ async Task EnsureStopped(CancellationToken cancellationToken = default) logger.Info("Shutting down. Already stopped, skipping shut down"); return; //Already stopped } + var stoppable = queueIngestor; queueIngestor = null; logger.Info("Shutting down. Infrastructure shut down commencing"); @@ -255,7 +257,7 @@ async Task Loop() readonly Settings settings; readonly Channel channel; readonly Histogram batchSizeMeter = AuditMetrics.Meter.CreateHistogram($"{AuditMetrics.Prefix}.batch_size"); - readonly Histogram batchDurationMeter = AuditMetrics.Meter.CreateHistogram($"{AuditMetrics.Prefix}.batch_duration_ms"); + readonly Histogram batchDurationMeter = AuditMetrics.Meter.CreateHistogram($"{AuditMetrics.Prefix}.batch_duration", unit: "ms"); readonly Counter receivedMeter = AuditMetrics.Meter.CreateCounter($"{AuditMetrics.Prefix}.received"); readonly Watchdog watchdog; readonly Task ingestionWorker; diff --git a/src/ServiceControl.Audit/Auditing/AuditPersister.cs b/src/ServiceControl.Audit/Auditing/AuditPersister.cs index 16eae2f401..1afceeab00 100644 --- a/src/ServiceControl.Audit/Auditing/AuditPersister.cs +++ b/src/ServiceControl.Audit/Auditing/AuditPersister.cs @@ -283,11 +283,11 @@ await messageDispatcher.Value.Dispatch(new TransportOperations(messagesToEmit.To } } - readonly Counter ingestedAuditMeter = AuditMetrics.Meter.CreateCounter($"{AuditMetrics.Prefix}.ingested_audit_messages"); // metrics.GetCounter("Audit ingestion - ingested audit"); - readonly Counter ingestedSagaAuditMeter = AuditMetrics.Meter.CreateCounter($"{AuditMetrics.Prefix}.ingested_saga_audits"); // metrics.GetCounter("Audit ingestion - ingested audit"); - readonly Histogram auditBulkInsertDurationMeter = AuditMetrics.Meter.CreateHistogram($"{AuditMetrics.Prefix}.audit_bulk_insert_duration_ms"); // metrics.GetCounter("Audit ingestion - ingested audit"); - readonly Histogram sagaAuditBulkInsertDurationMeter = AuditMetrics.Meter.CreateHistogram($"{AuditMetrics.Prefix}.saga_bulk_insert_duration_ms"); // metrics.GetCounter("Audit ingestion - ingested audit"); - readonly Histogram bulkInsertCommitDurationMeter = AuditMetrics.Meter.CreateHistogram($"{AuditMetrics.Prefix}.audit_commit_duration_ms"); // metrics.GetCounter("Audit ingestion - ingested audit"); + readonly Counter ingestedAuditMeter = AuditMetrics.Meter.CreateCounter($"{AuditMetrics.Prefix}.stored_audit_messages"); + readonly Counter ingestedSagaAuditMeter = AuditMetrics.Meter.CreateCounter($"{AuditMetrics.Prefix}.stored_saga_audits"); + readonly Histogram auditBulkInsertDurationMeter = AuditMetrics.Meter.CreateHistogram($"{AuditMetrics.Prefix}.bulk_insert_duration_audit", unit: "ms"); + readonly Histogram sagaAuditBulkInsertDurationMeter = AuditMetrics.Meter.CreateHistogram($"{AuditMetrics.Prefix}.bulk_insert_duration_sagas", unit: "ms"); + readonly Histogram bulkInsertCommitDurationMeter = AuditMetrics.Meter.CreateHistogram($"{AuditMetrics.Prefix}.commit_duration_audit", unit: "ms"); static readonly ILog Logger = LogManager.GetLogger(); } diff --git a/src/ServiceControl.Audit/Auditing/AuditMetrics.cs b/src/ServiceControl.Audit/Infrastructure/AuditMetrics.cs similarity index 55% rename from src/ServiceControl.Audit/Auditing/AuditMetrics.cs rename to src/ServiceControl.Audit/Infrastructure/AuditMetrics.cs index 3e02448fa1..25a9fa658f 100644 --- a/src/ServiceControl.Audit/Auditing/AuditMetrics.cs +++ b/src/ServiceControl.Audit/Infrastructure/AuditMetrics.cs @@ -1,9 +1,9 @@ -namespace ServiceControl.Audit.Auditing; +namespace ServiceControl.Audit; using System.Diagnostics.Metrics; static class AuditMetrics { - public static readonly Meter Meter = new("ServiceControl", "0.1.0"); + public static readonly Meter Meter = new("Particular.ServiceControl", "0.1.0"); public static readonly string Prefix = "particular.servicecontrol.audit"; } \ No newline at end of file From 88d2e7165a559b85079e20644581d7107ed71666 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=96hlund?= Date: Mon, 3 Feb 2025 08:35:02 +0100 Subject: [PATCH 10/15] Better metrics names --- .../Auditing/AuditIngestion.cs | 14 ++++++------- .../Auditing/AuditPersister.cs | 20 +++++++++---------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/ServiceControl.Audit/Auditing/AuditIngestion.cs b/src/ServiceControl.Audit/Auditing/AuditIngestion.cs index 93f93ca665..8d68570636 100644 --- a/src/ServiceControl.Audit/Auditing/AuditIngestion.cs +++ b/src/ServiceControl.Audit/Auditing/AuditIngestion.cs @@ -191,7 +191,7 @@ async Task OnMessage(MessageContext messageContext, CancellationToken cancellati var taskCompletionSource = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); messageContext.SetTaskCompletionSource(taskCompletionSource); - receivedMeter.Add(1); + receivedAudits.Add(1); await channel.Writer.WriteAsync(messageContext, cancellationToken); await taskCompletionSource.Task; @@ -212,11 +212,11 @@ async Task Loop() contexts.Add(context); } - batchSizeMeter.Record(contexts.Count); + auditBatchSize.Record(contexts.Count); var sw = Stopwatch.StartNew(); await auditIngestor.Ingest(contexts); - batchDurationMeter.Record(sw.ElapsedMilliseconds); + auditBatchDuration.Record(sw.ElapsedMilliseconds); } catch (OperationCanceledException) { @@ -247,7 +247,7 @@ async Task Loop() TransportInfrastructure transportInfrastructure; IMessageReceiver queueIngestor; - readonly SemaphoreSlim startStopSemaphore = new SemaphoreSlim(1); + readonly SemaphoreSlim startStopSemaphore = new(1); readonly string inputEndpoint; readonly ITransportCustomization transportCustomization; readonly TransportSettings transportSettings; @@ -256,9 +256,9 @@ async Task Loop() readonly IAuditIngestionUnitOfWorkFactory unitOfWorkFactory; readonly Settings settings; readonly Channel channel; - readonly Histogram batchSizeMeter = AuditMetrics.Meter.CreateHistogram($"{AuditMetrics.Prefix}.batch_size"); - readonly Histogram batchDurationMeter = AuditMetrics.Meter.CreateHistogram($"{AuditMetrics.Prefix}.batch_duration", unit: "ms"); - readonly Counter receivedMeter = AuditMetrics.Meter.CreateCounter($"{AuditMetrics.Prefix}.received"); + readonly Histogram auditBatchSize = AuditMetrics.Meter.CreateHistogram($"{AuditMetrics.Prefix}.batch_size_audits"); + readonly Histogram auditBatchDuration = AuditMetrics.Meter.CreateHistogram($"{AuditMetrics.Prefix}.batch_duration_audits", unit: "ms"); + readonly Counter receivedAudits = AuditMetrics.Meter.CreateCounter($"{AuditMetrics.Prefix}.received_audits"); readonly Watchdog watchdog; readonly Task ingestionWorker; readonly IHostApplicationLifetime applicationLifetime; diff --git a/src/ServiceControl.Audit/Auditing/AuditPersister.cs b/src/ServiceControl.Audit/Auditing/AuditPersister.cs index 1afceeab00..a11e534f98 100644 --- a/src/ServiceControl.Audit/Auditing/AuditPersister.cs +++ b/src/ServiceControl.Audit/Auditing/AuditPersister.cs @@ -75,9 +75,9 @@ public async Task> Persist(IReadOnlyList> Persist(IReadOnlyList> Persist(IReadOnlyList ingestedAuditMeter = AuditMetrics.Meter.CreateCounter($"{AuditMetrics.Prefix}.stored_audit_messages"); - readonly Counter ingestedSagaAuditMeter = AuditMetrics.Meter.CreateCounter($"{AuditMetrics.Prefix}.stored_saga_audits"); - readonly Histogram auditBulkInsertDurationMeter = AuditMetrics.Meter.CreateHistogram($"{AuditMetrics.Prefix}.bulk_insert_duration_audit", unit: "ms"); - readonly Histogram sagaAuditBulkInsertDurationMeter = AuditMetrics.Meter.CreateHistogram($"{AuditMetrics.Prefix}.bulk_insert_duration_sagas", unit: "ms"); - readonly Histogram bulkInsertCommitDurationMeter = AuditMetrics.Meter.CreateHistogram($"{AuditMetrics.Prefix}.commit_duration_audit", unit: "ms"); + readonly Counter storedAudits = AuditMetrics.Meter.CreateCounter($"{AuditMetrics.Prefix}.stored_audit_messages"); + readonly Counter storedSagas = AuditMetrics.Meter.CreateCounter($"{AuditMetrics.Prefix}.stored_saga_audits"); + readonly Histogram auditBulkInsertDuration = AuditMetrics.Meter.CreateHistogram($"{AuditMetrics.Prefix}.bulk_insert_duration_audits", unit: "ms"); + readonly Histogram sagaAuditBulkInsertDuration = AuditMetrics.Meter.CreateHistogram($"{AuditMetrics.Prefix}.bulk_insert_duration_sagas", unit: "ms"); + readonly Histogram auditCommitDuration = AuditMetrics.Meter.CreateHistogram($"{AuditMetrics.Prefix}.commit_duration_audits", unit: "ms"); static readonly ILog Logger = LogManager.GetLogger(); } From da4c35baae74e99ed8ad876b8f58705b1a542289 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=96hlund?= Date: Mon, 3 Feb 2025 08:50:16 +0100 Subject: [PATCH 11/15] Emit body size --- src/ServiceControl.Audit/Auditing/AuditIngestion.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ServiceControl.Audit/Auditing/AuditIngestion.cs b/src/ServiceControl.Audit/Auditing/AuditIngestion.cs index 8d68570636..6e5593d516 100644 --- a/src/ServiceControl.Audit/Auditing/AuditIngestion.cs +++ b/src/ServiceControl.Audit/Auditing/AuditIngestion.cs @@ -210,6 +210,7 @@ async Task Loop() while (channel.Reader.TryRead(out var context)) { contexts.Add(context); + auditMessageSize.Record(context.Body.Length / 1024.0); } auditBatchSize.Record(contexts.Count); @@ -258,6 +259,7 @@ async Task Loop() readonly Channel channel; readonly Histogram auditBatchSize = AuditMetrics.Meter.CreateHistogram($"{AuditMetrics.Prefix}.batch_size_audits"); readonly Histogram auditBatchDuration = AuditMetrics.Meter.CreateHistogram($"{AuditMetrics.Prefix}.batch_duration_audits", unit: "ms"); + readonly Histogram auditMessageSize = AuditMetrics.Meter.CreateHistogram($"{AuditMetrics.Prefix}.audit_message_size", unit: "kilobytes"); readonly Counter receivedAudits = AuditMetrics.Meter.CreateCounter($"{AuditMetrics.Prefix}.received_audits"); readonly Watchdog watchdog; readonly Task ingestionWorker; From 786559b1e71d6ed731d6207ba983eecb2c64ac27 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Mon, 3 Feb 2025 16:09:12 +0100 Subject: [PATCH 12/15] Fix meter name --- src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs | 2 +- src/ServiceControl.Audit/Infrastructure/AuditMetrics.cs | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs b/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs index d1f08569dc..318cd1ba11 100644 --- a/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs +++ b/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs @@ -81,7 +81,7 @@ public static void AddServiceControlAudit(this IHostApplicationBuilder builder, serviceInstanceId: settings.InstanceName)) .WithMetrics(b => { - b.AddMeter("ServiceControl"); + b.AddMeter(AuditMetrics.MeterName); b.AddOtlpExporter(e => { e.Endpoint = otelMetricsUri; diff --git a/src/ServiceControl.Audit/Infrastructure/AuditMetrics.cs b/src/ServiceControl.Audit/Infrastructure/AuditMetrics.cs index 25a9fa658f..4e07ae3453 100644 --- a/src/ServiceControl.Audit/Infrastructure/AuditMetrics.cs +++ b/src/ServiceControl.Audit/Infrastructure/AuditMetrics.cs @@ -4,6 +4,7 @@ namespace ServiceControl.Audit; static class AuditMetrics { - public static readonly Meter Meter = new("Particular.ServiceControl", "0.1.0"); + public const string MeterName = "Particular.ServiceControl"; + public static readonly Meter Meter = new(MeterName, "0.1.0"); public static readonly string Prefix = "particular.servicecontrol.audit"; } \ No newline at end of file From a2011fcec2f82a0c139aacb9065993b1b04fcec4 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Mon, 3 Feb 2025 16:09:51 +0100 Subject: [PATCH 13/15] Log that OpenTelemetry metrics exporter is enabled --- src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs b/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs index 318cd1ba11..47ac4e2976 100644 --- a/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs +++ b/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs @@ -87,6 +87,9 @@ public static void AddServiceControlAudit(this IHostApplicationBuilder builder, e.Endpoint = otelMetricsUri; }); }); + var logger = LogManager.GetLogger(typeof(HostApplicationBuilderExtensions)); + logger.InfoFormat("OpenTelemetry metrics exporter enabled: {0}", settings.OtelMetricsUrl); + } // Configure after the NServiceBus hosted service to ensure NServiceBus is already started From f9d1986f103ce8a67603dd1e4d3157aec05b554a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=96hlund?= Date: Tue, 4 Feb 2025 19:16:23 +0100 Subject: [PATCH 14/15] Stop using prefixes --- src/ServiceControl.Audit/Auditing/AuditIngestion.cs | 8 ++++---- src/ServiceControl.Audit/Auditing/AuditPersister.cs | 10 +++++----- .../HostApplicationBuilderExtensions.cs | 5 +++-- .../Infrastructure/AuditMetrics.cs | 10 ---------- src/ServiceControl.Audit/Infrastructure/Telemetry.cs | 9 +++++++++ 5 files changed, 21 insertions(+), 21 deletions(-) delete mode 100644 src/ServiceControl.Audit/Infrastructure/AuditMetrics.cs create mode 100644 src/ServiceControl.Audit/Infrastructure/Telemetry.cs diff --git a/src/ServiceControl.Audit/Auditing/AuditIngestion.cs b/src/ServiceControl.Audit/Auditing/AuditIngestion.cs index 6e5593d516..dbbe5a2ad4 100644 --- a/src/ServiceControl.Audit/Auditing/AuditIngestion.cs +++ b/src/ServiceControl.Audit/Auditing/AuditIngestion.cs @@ -257,10 +257,10 @@ async Task Loop() readonly IAuditIngestionUnitOfWorkFactory unitOfWorkFactory; readonly Settings settings; readonly Channel channel; - readonly Histogram auditBatchSize = AuditMetrics.Meter.CreateHistogram($"{AuditMetrics.Prefix}.batch_size_audits"); - readonly Histogram auditBatchDuration = AuditMetrics.Meter.CreateHistogram($"{AuditMetrics.Prefix}.batch_duration_audits", unit: "ms"); - readonly Histogram auditMessageSize = AuditMetrics.Meter.CreateHistogram($"{AuditMetrics.Prefix}.audit_message_size", unit: "kilobytes"); - readonly Counter receivedAudits = AuditMetrics.Meter.CreateCounter($"{AuditMetrics.Prefix}.received_audits"); + readonly Histogram auditBatchSize = Telemetry.Meter.CreateHistogram("messages_batch_size"); + readonly Histogram auditBatchDuration = Telemetry.Meter.CreateHistogram("messages_batch_duration", unit: "ms"); + readonly Histogram auditMessageSize = Telemetry.Meter.CreateHistogram("messages_size", unit: "kilobytes"); + readonly Counter receivedAudits = Telemetry.Meter.CreateCounter("messages_received"); readonly Watchdog watchdog; readonly Task ingestionWorker; readonly IHostApplicationLifetime applicationLifetime; diff --git a/src/ServiceControl.Audit/Auditing/AuditPersister.cs b/src/ServiceControl.Audit/Auditing/AuditPersister.cs index a11e534f98..4548a1cd3b 100644 --- a/src/ServiceControl.Audit/Auditing/AuditPersister.cs +++ b/src/ServiceControl.Audit/Auditing/AuditPersister.cs @@ -283,11 +283,11 @@ await messageDispatcher.Value.Dispatch(new TransportOperations(messagesToEmit.To } } - readonly Counter storedAudits = AuditMetrics.Meter.CreateCounter($"{AuditMetrics.Prefix}.stored_audit_messages"); - readonly Counter storedSagas = AuditMetrics.Meter.CreateCounter($"{AuditMetrics.Prefix}.stored_saga_audits"); - readonly Histogram auditBulkInsertDuration = AuditMetrics.Meter.CreateHistogram($"{AuditMetrics.Prefix}.bulk_insert_duration_audits", unit: "ms"); - readonly Histogram sagaAuditBulkInsertDuration = AuditMetrics.Meter.CreateHistogram($"{AuditMetrics.Prefix}.bulk_insert_duration_sagas", unit: "ms"); - readonly Histogram auditCommitDuration = AuditMetrics.Meter.CreateHistogram($"{AuditMetrics.Prefix}.commit_duration_audits", unit: "ms"); + readonly Counter storedAudits = Telemetry.Meter.CreateCounter("messages_stored"); + readonly Counter storedSagas = Telemetry.Meter.CreateCounter("sagas_stored"); + readonly Histogram auditBulkInsertDuration = Telemetry.Meter.CreateHistogram("messages_bulk_insert_duration", unit: "ms"); + readonly Histogram sagaAuditBulkInsertDuration = Telemetry.Meter.CreateHistogram("sagas_bulk_insert_duration", unit: "ms"); + readonly Histogram auditCommitDuration = Telemetry.Meter.CreateHistogram("messages_commit_duration", unit: "ms"); static readonly ILog Logger = LogManager.GetLogger(); } diff --git a/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs b/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs index 47ac4e2976..2b1784d437 100644 --- a/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs +++ b/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs @@ -74,6 +74,7 @@ public static void AddServiceControlAudit(this IHostApplicationBuilder builder, { throw new UriFormatException($"Invalid OtelMetricsUrl: {settings.OtelMetricsUrl}"); } + builder.Services.AddOpenTelemetry() .ConfigureResource(b => b.AddService( serviceName: "Particular.ServiceControl.Audit", @@ -81,15 +82,15 @@ public static void AddServiceControlAudit(this IHostApplicationBuilder builder, serviceInstanceId: settings.InstanceName)) .WithMetrics(b => { - b.AddMeter(AuditMetrics.MeterName); + b.AddMeter(Telemetry.MeterName); b.AddOtlpExporter(e => { e.Endpoint = otelMetricsUri; }); }); + var logger = LogManager.GetLogger(typeof(HostApplicationBuilderExtensions)); logger.InfoFormat("OpenTelemetry metrics exporter enabled: {0}", settings.OtelMetricsUrl); - } // Configure after the NServiceBus hosted service to ensure NServiceBus is already started diff --git a/src/ServiceControl.Audit/Infrastructure/AuditMetrics.cs b/src/ServiceControl.Audit/Infrastructure/AuditMetrics.cs deleted file mode 100644 index 4e07ae3453..0000000000 --- a/src/ServiceControl.Audit/Infrastructure/AuditMetrics.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace ServiceControl.Audit; - -using System.Diagnostics.Metrics; - -static class AuditMetrics -{ - public const string MeterName = "Particular.ServiceControl"; - public static readonly Meter Meter = new(MeterName, "0.1.0"); - public static readonly string Prefix = "particular.servicecontrol.audit"; -} \ No newline at end of file diff --git a/src/ServiceControl.Audit/Infrastructure/Telemetry.cs b/src/ServiceControl.Audit/Infrastructure/Telemetry.cs new file mode 100644 index 0000000000..519c0ce1c5 --- /dev/null +++ b/src/ServiceControl.Audit/Infrastructure/Telemetry.cs @@ -0,0 +1,9 @@ +namespace ServiceControl.Audit; + +using System.Diagnostics.Metrics; + +static class Telemetry +{ + public const string MeterName = "Particular.ServiceControl.Audit"; + public static readonly Meter Meter = new(MeterName, "0.1.0"); +} \ No newline at end of file From c9f9e745967fbc66c1f8ea1a0954e459e4f2cd29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=96hlund?= Date: Tue, 4 Feb 2025 19:19:48 +0100 Subject: [PATCH 15/15] Better name --- src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs | 2 +- src/ServiceControl.Audit/Infrastructure/Telemetry.cs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs b/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs index 2b1784d437..a9835669ba 100644 --- a/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs +++ b/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs @@ -77,7 +77,7 @@ public static void AddServiceControlAudit(this IHostApplicationBuilder builder, builder.Services.AddOpenTelemetry() .ConfigureResource(b => b.AddService( - serviceName: "Particular.ServiceControl.Audit", + serviceName: Telemetry.ServiceName, serviceVersion: version, serviceInstanceId: settings.InstanceName)) .WithMetrics(b => diff --git a/src/ServiceControl.Audit/Infrastructure/Telemetry.cs b/src/ServiceControl.Audit/Infrastructure/Telemetry.cs index 519c0ce1c5..b1b85a0b06 100644 --- a/src/ServiceControl.Audit/Infrastructure/Telemetry.cs +++ b/src/ServiceControl.Audit/Infrastructure/Telemetry.cs @@ -5,5 +5,6 @@ namespace ServiceControl.Audit; static class Telemetry { public const string MeterName = "Particular.ServiceControl.Audit"; + public static string ServiceName = MeterName; public static readonly Meter Meter = new(MeterName, "0.1.0"); } \ No newline at end of file