From ea6cf0eaae7335ac9026e140d7871ef0d77da561 Mon Sep 17 00:00:00 2001 From: John Simons Date: Mon, 3 Feb 2025 16:16:11 +1000 Subject: [PATCH] Adding health checker command --- .../AdapterAmazonSqsConfiguration.cs | 1 + .../AmazonSqsHelper.cs | 17 ++++++- .../AdapterAzureServiceBusConfiguration.cs | 6 ++- .../AzureServiceBusHelper.cs | 27 ++++++++++- .../Commands/HealthCheckCommand.cs | 39 ++++++++++++++++ .../Dockerfile | 1 + .../Program.cs | 1 + .../AdapterRabbitMqConfiguration.cs | 2 + .../RabbitMQHealthChecker.cs | 46 +++++++++++++++++++ .../RabbitMQHelper.cs | 20 ++++++++ .../IHealthCheckerProvider.cs | 4 ++ 11 files changed, 160 insertions(+), 4 deletions(-) create mode 100644 src/ServiceControl.Connector.MassTransit.Host/Commands/HealthCheckCommand.cs create mode 100644 src/ServiceControl.Connector.MassTransit.RabbitMQ/RabbitMQHealthChecker.cs create mode 100644 src/ServiceControl.Connector.MassTransit/IHealthCheckerProvider.cs diff --git a/src/ServiceControl.Connector.MassTransit.AmazonSQS/AdapterAmazonSqsConfiguration.cs b/src/ServiceControl.Connector.MassTransit.AmazonSQS/AdapterAmazonSqsConfiguration.cs index a00ba299..d97c595c 100644 --- a/src/ServiceControl.Connector.MassTransit.AmazonSQS/AdapterAmazonSqsConfiguration.cs +++ b/src/ServiceControl.Connector.MassTransit.AmazonSQS/AdapterAmazonSqsConfiguration.cs @@ -10,6 +10,7 @@ public static void UsingAmazonSqs(this IServiceCollection services, Action new AmazonSqsHelper(provider.GetRequiredService(), provider.GetRequiredService(), string.Empty)); services.AddSingleton(provider => provider.GetRequiredService()); services.AddSingleton(provider => provider.GetRequiredService()); + services.AddSingleton(provider => provider.GetRequiredService()); services.AddSingleton(); services.AddSingleton(); services.AddTransient(sp => diff --git a/src/ServiceControl.Connector.MassTransit.AmazonSQS/AmazonSqsHelper.cs b/src/ServiceControl.Connector.MassTransit.AmazonSQS/AmazonSqsHelper.cs index 025e067a..578d9874 100644 --- a/src/ServiceControl.Connector.MassTransit.AmazonSQS/AmazonSqsHelper.cs +++ b/src/ServiceControl.Connector.MassTransit.AmazonSQS/AmazonSqsHelper.cs @@ -3,7 +3,7 @@ using Amazon.SQS; using Amazon.SQS.Model; -sealed class AmazonSqsHelper(IAmazonSQS client, SqsTransport transportDefinition, string? queueNamePrefix = null) : IQueueInformationProvider, IQueueLengthProvider +sealed class AmazonSqsHelper(IAmazonSQS client, SqsTransport transportDefinition, string? queueNamePrefix = null) : IQueueInformationProvider, IQueueLengthProvider, IHealthCheckerProvider { readonly ConcurrentDictionary> queueUrlCache = new(); @@ -62,4 +62,19 @@ public async Task GetQueueLength(string name, CancellationToken cancellati return value; } + + public async Task<(bool Success, string ErrorMessage)> TryCheck(CancellationToken cancellationToken) + { + try + { + await GetQueues(cancellationToken).GetAsyncEnumerator(cancellationToken).MoveNextAsync(); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { } + catch (AmazonSQSException e) + { + return (false, e.Message); + } + + return (true, string.Empty); + } } \ No newline at end of file diff --git a/src/ServiceControl.Connector.MassTransit.AzureServiceBus/AdapterAzureServiceBusConfiguration.cs b/src/ServiceControl.Connector.MassTransit.AzureServiceBus/AdapterAzureServiceBusConfiguration.cs index d0bc4c4c..63431be7 100644 --- a/src/ServiceControl.Connector.MassTransit.AzureServiceBus/AdapterAzureServiceBusConfiguration.cs +++ b/src/ServiceControl.Connector.MassTransit.AzureServiceBus/AdapterAzureServiceBusConfiguration.cs @@ -10,8 +10,10 @@ public static void UsingAzureServiceBus(this IServiceCollection services, IConfi { var receiveMode = useDeadLetterQueue ? SubQueue.DeadLetter : SubQueue.None; - services.AddSingleton(b => new AzureServiceBusHelper(b.GetRequiredService>(), connectionString)); - services.AddSingleton(b => new AzureServiceBusHelper(b.GetRequiredService>(), connectionString)); + services.AddSingleton(b => new AzureServiceBusHelper(b.GetRequiredService>(), connectionString)); + services.AddSingleton(b => b.GetRequiredService()); + services.AddSingleton(b => b.GetRequiredService()); + services.AddSingleton(b => b.GetRequiredService()); services.AddTransient(_ => new AzureServiceBusTransport(connectionString) { TransportTransactionMode = TransportTransactionMode.ReceiveOnly, diff --git a/src/ServiceControl.Connector.MassTransit.AzureServiceBus/AzureServiceBusHelper.cs b/src/ServiceControl.Connector.MassTransit.AzureServiceBus/AzureServiceBusHelper.cs index ec2cdf3a..b4a3bfbd 100644 --- a/src/ServiceControl.Connector.MassTransit.AzureServiceBus/AzureServiceBusHelper.cs +++ b/src/ServiceControl.Connector.MassTransit.AzureServiceBus/AzureServiceBusHelper.cs @@ -2,7 +2,7 @@ using Azure.Messaging.ServiceBus.Administration; using Microsoft.Extensions.Logging; -class AzureServiceBusHelper(ILogger logger, string connectionstring) : IQueueInformationProvider, IQueueLengthProvider +class AzureServiceBusHelper(ILogger logger, string connectionstring) : IQueueInformationProvider, IQueueLengthProvider, IHealthCheckerProvider { readonly ServiceBusAdministrationClient client = new(connectionstring); @@ -30,4 +30,29 @@ public async Task GetQueueLength(string name, CancellationToken cancellati var queuesRuntimeProperties = await client.GetQueueRuntimePropertiesAsync(name, cancellationToken); return queuesRuntimeProperties.Value.ActiveMessageCount; } + + public async Task<(bool Success, string ErrorMessage)> TryCheck(CancellationToken cancellationToken) + { + try + { + var clientNoRetries = new ServiceBusAdministrationClient(connectionstring, new ServiceBusAdministrationClientOptions { Retry = { MaxRetries = 0 } }); + var result = clientNoRetries.GetQueuesAsync(cancellationToken); + + await foreach (var queueProperties in result) + { + break; + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { } + catch (UnauthorizedAccessException) + { + return (false, "The token has an invalid signature."); + } + catch (Azure.RequestFailedException e) + { + return (false, e.Message); + } + + return (true, string.Empty); + } } \ No newline at end of file diff --git a/src/ServiceControl.Connector.MassTransit.Host/Commands/HealthCheckCommand.cs b/src/ServiceControl.Connector.MassTransit.Host/Commands/HealthCheckCommand.cs new file mode 100644 index 00000000..26cb5d19 --- /dev/null +++ b/src/ServiceControl.Connector.MassTransit.Host/Commands/HealthCheckCommand.cs @@ -0,0 +1,39 @@ +namespace ServiceControl.Connector.MassTransit.Host.Commands; + +using System.CommandLine; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; + +public class HealthCheckCommand : Command +{ + public HealthCheckCommand() : base("health-check", "Performs a validation that the connector is able to connect to the broker.") + { + this.SetHandler(async context => + { + context.ExitCode = await InternalHandler(context.GetCancellationToken()); + }); + } + + async Task InternalHandler(CancellationToken cancellationToken) + { + var builder = Host.CreateEmptyApplicationBuilder(null); + builder.Configuration.AddEnvironmentVariables(); + builder.UseMassTransitConnector(true); + + var host = builder.Build(); + + var queueInformationProvider = host.Services.GetRequiredService(); + var (success, errorMessage) = await queueInformationProvider.TryCheck(cancellationToken); + + if (!success) + { + Console.WriteLine(errorMessage); + return 1; + } + + Console.WriteLine("Success"); + + return 0; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Connector.MassTransit.Host/Dockerfile b/src/ServiceControl.Connector.MassTransit.Host/Dockerfile index cb72bafb..f2f692f8 100644 --- a/src/ServiceControl.Connector.MassTransit.Host/Dockerfile +++ b/src/ServiceControl.Connector.MassTransit.Host/Dockerfile @@ -30,6 +30,7 @@ COPY --from=build ./build /app ENV QUEUES_FILE="/app/queues.txt" ENV DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1 +HEALTHCHECK --start-period=20s CMD ["/app/ServiceControl.Connector.MassTransit.Host", "health-check"] USER $APP_UID ENTRYPOINT ["/app/ServiceControl.Connector.MassTransit.Host"] \ No newline at end of file diff --git a/src/ServiceControl.Connector.MassTransit.Host/Program.cs b/src/ServiceControl.Connector.MassTransit.Host/Program.cs index 275539a0..60b0f00d 100644 --- a/src/ServiceControl.Connector.MassTransit.Host/Program.cs +++ b/src/ServiceControl.Connector.MassTransit.Host/Program.cs @@ -5,6 +5,7 @@ var startupCommand = new StartupCommand(args); startupCommand.AddCommand(new QueuesCommand()); +startupCommand.AddCommand(new HealthCheckCommand()); var commandLineBuilder = new CommandLineBuilder(startupCommand); diff --git a/src/ServiceControl.Connector.MassTransit.RabbitMQ/AdapterRabbitMqConfiguration.cs b/src/ServiceControl.Connector.MassTransit.RabbitMQ/AdapterRabbitMqConfiguration.cs index 3207b21e..9a1817b6 100644 --- a/src/ServiceControl.Connector.MassTransit.RabbitMQ/AdapterRabbitMqConfiguration.cs +++ b/src/ServiceControl.Connector.MassTransit.RabbitMQ/AdapterRabbitMqConfiguration.cs @@ -14,8 +14,10 @@ public static void UsingRabbitMQ(this IServiceCollection services, string connec var defaultCredential = new NetworkCredential(username ?? connectionConfiguration.UserName, password ?? connectionConfiguration.Password); var rabbitMqHelper = new RabbitMQHelper(connectionConfiguration.VirtualHost, managementApi, defaultCredential); + services.AddSingleton(rabbitMqHelper); services.AddSingleton(rabbitMqHelper); services.AddSingleton(rabbitMqHelper); + services.AddSingleton(); services.AddTransient(_ => new RabbitMQTransport( RoutingTopology.Conventional(QueueType.Quorum), connectionString, diff --git a/src/ServiceControl.Connector.MassTransit.RabbitMQ/RabbitMQHealthChecker.cs b/src/ServiceControl.Connector.MassTransit.RabbitMQ/RabbitMQHealthChecker.cs new file mode 100644 index 00000000..a930ef92 --- /dev/null +++ b/src/ServiceControl.Connector.MassTransit.RabbitMQ/RabbitMQHealthChecker.cs @@ -0,0 +1,46 @@ +using NServiceBus.Transport; +using RabbitMQ.Client.Exceptions; + +class RabbitMQHealthChecker(RabbitMQHelper helper, Configuration configuration, TransportInfrastructureFactory transportInfrastructureFactory) : IHealthCheckerProvider +{ + public async Task<(bool, string)> TryCheck(CancellationToken cancellationToken) + { + var result = await helper.TryCheck(cancellationToken); + + if (!result.Success) + { + return (false, result.ErrorMessage); + } + + var hostSettings = new HostSettings( + configuration.ReturnQueue, + $"Queue creator for {configuration.ReturnQueue}", + new StartupDiagnosticEntries(), + (_, __, ___) => + { + }, + false); + + var receiverSettings = new[]{ + new ReceiveSettings( + id: "Return", + receiveAddress: new QueueAddress(configuration.ReturnQueue), + usePublishSubscribe: false, + purgeOnStartup: false, + errorQueue: configuration.PoisonQueue)}; + + + try + { + var infrastructure = await transportInfrastructureFactory.CreateTransportInfrastructure(hostSettings, + receiverSettings, [configuration.PoisonQueue, configuration.ServiceControlQueue], cancellationToken); + await infrastructure.Shutdown(cancellationToken); + } + catch (BrokerUnreachableException e) + { + return (false, e.Message); + } + + return (true, string.Empty); + } +} \ No newline at end of file diff --git a/src/ServiceControl.Connector.MassTransit.RabbitMQ/RabbitMQHelper.cs b/src/ServiceControl.Connector.MassTransit.RabbitMQ/RabbitMQHelper.cs index 5e5fd5ce..0a82da13 100644 --- a/src/ServiceControl.Connector.MassTransit.RabbitMQ/RabbitMQHelper.cs +++ b/src/ServiceControl.Connector.MassTransit.RabbitMQ/RabbitMQHelper.cs @@ -105,4 +105,24 @@ public async Task GetQueueLength(string queueName, CancellationToken cance throw new Exception($"Failed to check the length of the queue {queueName} via URL {url}.", e); } } + + public async Task<(bool Success, string ErrorMessage)> TryCheck(CancellationToken cancellationToken) + { + var url = $"/api/queues/{HttpUtility.UrlEncode(vhost)}"; + + try + { + using var response = await httpClient.GetAsync(url, cancellationToken); + + if (response.IsSuccessStatusCode) + { + return (true, string.Empty); + } + return (false, response.ReasonPhrase ?? "Connection failed"); + } + catch (HttpRequestException e) + { + return (false, e.Message); + } + } } \ No newline at end of file diff --git a/src/ServiceControl.Connector.MassTransit/IHealthCheckerProvider.cs b/src/ServiceControl.Connector.MassTransit/IHealthCheckerProvider.cs new file mode 100644 index 00000000..64ea0f95 --- /dev/null +++ b/src/ServiceControl.Connector.MassTransit/IHealthCheckerProvider.cs @@ -0,0 +1,4 @@ +public interface IHealthCheckerProvider +{ + Task<(bool Success, string ErrorMessage)> TryCheck(CancellationToken cancellationToken); +} \ No newline at end of file