From 9cfd5303aae42fb58c59db1baeb2a3e0b8bbcdd0 Mon Sep 17 00:00:00 2001 From: Mikel Blanchard Date: Thu, 22 Feb 2024 15:50:40 -0800 Subject: [PATCH] Stress test improvements. --- .../OpenTelemetry.Tests.Stress.Logs.csproj | 2 + .../Program.cs | 35 +++-- .../OpenTelemetry.Tests.Stress.Metrics.csproj | 3 + .../Program.cs | 136 +++++++++++++----- .../OpenTelemetry.Tests.Stress.Traces.csproj | 2 + .../Program.cs | 23 ++- test/OpenTelemetry.Tests.Stress/Meat.cs | 10 +- .../OpenTelemetry.Tests.Stress.csproj | 2 + test/OpenTelemetry.Tests.Stress/Skeleton.cs | 87 ++++++++--- 9 files changed, 230 insertions(+), 70 deletions(-) diff --git a/test/OpenTelemetry.Tests.Stress.Logs/OpenTelemetry.Tests.Stress.Logs.csproj b/test/OpenTelemetry.Tests.Stress.Logs/OpenTelemetry.Tests.Stress.Logs.csproj index 1f1225d551a..1e3eb044dc1 100644 --- a/test/OpenTelemetry.Tests.Stress.Logs/OpenTelemetry.Tests.Stress.Logs.csproj +++ b/test/OpenTelemetry.Tests.Stress.Logs/OpenTelemetry.Tests.Stress.Logs.csproj @@ -12,8 +12,10 @@ + + diff --git a/test/OpenTelemetry.Tests.Stress.Logs/Program.cs b/test/OpenTelemetry.Tests.Stress.Logs/Program.cs index 6d2cb88fad0..b8d994c2343 100644 --- a/test/OpenTelemetry.Tests.Stress.Logs/Program.cs +++ b/test/OpenTelemetry.Tests.Stress.Logs/Program.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 using System.Runtime.CompilerServices; +using CommandLine; using Microsoft.Extensions.Logging; namespace OpenTelemetry.Tests.Stress; @@ -11,19 +12,10 @@ public partial class Program private static ILogger logger; private static Payload payload = new Payload(); - public static void Main() + public static void Main(string[] args) { - using var loggerFactory = LoggerFactory.Create(builder => - { - builder.AddOpenTelemetry(options => - { - options.AddProcessor(new DummyProcessor()); - }); - }); - - logger = loggerFactory.CreateLogger(); - - Stress(prometheusPort: 9464); + Parser.Default.ParseArguments(args) + .WithParsed(LaunchStressTest); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -36,4 +28,23 @@ protected static void Run() exception: null, formatter: (state, ex) => string.Empty); } + + protected static void WriteRunInformationToConsole(StressTestOptions options) + { + } + + private static void LaunchStressTest(StressTestOptions options) + { + using var loggerFactory = LoggerFactory.Create(builder => + { + builder.AddOpenTelemetry(options => + { + options.AddProcessor(new DummyProcessor()); + }); + }); + + logger = loggerFactory.CreateLogger(); + + RunStressTest(options); + } } diff --git a/test/OpenTelemetry.Tests.Stress.Metrics/OpenTelemetry.Tests.Stress.Metrics.csproj b/test/OpenTelemetry.Tests.Stress.Metrics/OpenTelemetry.Tests.Stress.Metrics.csproj index a783ed18d71..1b095ccb421 100644 --- a/test/OpenTelemetry.Tests.Stress.Metrics/OpenTelemetry.Tests.Stress.Metrics.csproj +++ b/test/OpenTelemetry.Tests.Stress.Metrics/OpenTelemetry.Tests.Stress.Metrics.csproj @@ -8,13 +8,16 @@ + + + diff --git a/test/OpenTelemetry.Tests.Stress.Metrics/Program.cs b/test/OpenTelemetry.Tests.Stress.Metrics/Program.cs index 102ad6d1df8..7c883527726 100644 --- a/test/OpenTelemetry.Tests.Stress.Metrics/Program.cs +++ b/test/OpenTelemetry.Tests.Stress.Metrics/Program.cs @@ -3,6 +3,8 @@ using System.Diagnostics.Metrics; using System.Runtime.CompilerServices; +using System.Text.Json.Serialization; +using CommandLine; using OpenTelemetry.Metrics; namespace OpenTelemetry.Tests.Stress; @@ -10,57 +12,125 @@ namespace OpenTelemetry.Tests.Stress; public partial class Program { private const int ArraySize = 10; - - // Note: Uncomment the below line if you want to run Histogram stress test private const int MaxHistogramMeasurement = 1000; private static readonly Meter TestMeter = new(Utils.GetCurrentMethodName()); + private static readonly Histogram TestHistogram = TestMeter.CreateHistogram("TestHistogram"); private static readonly Counter TestCounter = TestMeter.CreateCounter("TestCounter"); private static readonly string[] DimensionValues = new string[ArraySize]; private static readonly ThreadLocal ThreadLocalRandom = new(() => new Random()); + private static TestType testType; - // Note: Uncomment the below line if you want to run Histogram stress test - private static readonly Histogram TestHistogram = TestMeter.CreateHistogram("TestHistogram"); + protected enum TestType + { + /// Histogram. + Histogram, + + /// Counter. + Counter, + } + + public static void Main(string[] args) + { + Parser.Default.ParseArguments(args) + .WithParsed(LaunchStressTest); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + protected static void Run() + { + var random = ThreadLocalRandom.Value; + if (testType == TestType.Histogram) + { + TestHistogram.Record( + random.Next(MaxHistogramMeasurement), + new("DimName1", DimensionValues[random.Next(0, ArraySize)]), + new("DimName2", DimensionValues[random.Next(0, ArraySize)]), + new("DimName3", DimensionValues[random.Next(0, ArraySize)])); + } + else if (testType == TestType.Counter) + { + TestCounter.Add( + 100, + new("DimName1", DimensionValues[random.Next(0, ArraySize)]), + new("DimName2", DimensionValues[random.Next(0, ArraySize)]), + new("DimName3", DimensionValues[random.Next(0, ArraySize)])); + } + } + + protected static void WriteRunInformationToConsole(StressTestOptions options) + { + if (options.PrometheusTestMetricsPort != 0) + { + Console.Write($", testPrometheusEndpoint = http://localhost:{options.PrometheusTestMetricsPort}/metrics/"); + } + } - public static void Main() + private static void LaunchStressTest(StressTestOptions options) { for (int i = 0; i < ArraySize; i++) { DimensionValues[i] = $"DimValue{i}"; } - using var meterProvider = Sdk.CreateMeterProviderBuilder() - .AddMeter(TestMeter.Name) + var builder = Sdk.CreateMeterProviderBuilder() + .AddMeter(TestMeter.Name); - // .SetExemplarFilter(new AlwaysOnExemplarFilter()) - .AddPrometheusHttpListener( - options => options.UriPrefixes = new string[] { $"http://localhost:9185/" }) - .Build(); + if (options.PrometheusTestMetricsPort != 0) + { + builder.AddPrometheusHttpListener(o => o.UriPrefixes = new string[] { $"http://localhost:{options.PrometheusTestMetricsPort}/" }); + } + + if (options.EnableExemplars) + { + builder.SetExemplarFilter(new AlwaysOnExemplarFilter()); + } + + if (options.AddViewToFilterTags) + { + builder + .AddView("TestCounter", new MetricStreamConfiguration { TagKeys = new string[] { "DimName1" } }) + .AddView("TestHistogram", new MetricStreamConfiguration { TagKeys = new string[] { "DimName1" } }); + } + + if (options.AddOtlpExporter) + { + builder.AddOtlpExporter((exporterOptions, readerOptions) => + { + readerOptions.PeriodicExportingMetricReaderOptions.ExportIntervalMilliseconds = options.OtlpExporterExportIntervalMilliseconds; + }); + } + + using var meterProvider = builder.Build(); - Stress(prometheusPort: 9464); + testType = options.TestType; + + RunStressTest(options); } - // Note: Uncomment the below lines if you want to run Counter stress test - // [MethodImpl(MethodImplOptions.AggressiveInlining)] - // protected static void Run() - // { - // var random = ThreadLocalRandom.Value; - // TestCounter.Add( - // 100, - // new("DimName1", DimensionValues[random.Next(0, ArraySize)]), - // new("DimName2", DimensionValues[random.Next(0, ArraySize)]), - // new("DimName3", DimensionValues[random.Next(0, ArraySize)])); - // } - - // Note: Uncomment the below lines if you want to run Histogram stress test - [MethodImpl(MethodImplOptions.AggressiveInlining)] - protected static void Run() + protected partial class StressTestOptions + { + [JsonConverter(typeof(JsonStringEnumConverter))] + [Option('t', "type", HelpText = "The metrics stress test type to run. Valid values: [Histogram, Counter]. Default value: Histogram.", Required = false)] + public TestType TestType { get; set; } = TestType.Histogram; + + [Option('m', "metrics_port", HelpText = "The prometheus http listener port where prometheus will be exposed for retrieving test metrics while the stress test is running. Set to '0' to disable. Default value: 9185.", Required = false)] + public int PrometheusTestMetricsPort { get; set; } = 9185; + + [Option('v', "view", HelpText = "Whether or not a view should be configured to filter tags for the stress test. Default value: False.", Required = false)] + public bool AddViewToFilterTags { get; set; } + + [Option('o', "otlp", HelpText = "Whether or not an OTLP exporter should be added for the stress test. Default value: False.", Required = false)] + public bool AddOtlpExporter { get; set; } + + [Option('i', "interval", HelpText = "The OTLP exporter export interval in milliseconds. Default value: 5000.", Required = false)] + public int OtlpExporterExportIntervalMilliseconds { get; set; } = 5000; + + [Option('e', "exemplars", HelpText = "Whether or not to enable exemplars for the stress test. Default value: False.", Required = false)] + public bool EnableExemplars { get; set; } + } + + private sealed class NoOptions { - var random = ThreadLocalRandom.Value; - TestHistogram.Record( - random.Next(MaxHistogramMeasurement), - new("DimName1", DimensionValues[random.Next(0, ArraySize)]), - new("DimName2", DimensionValues[random.Next(0, ArraySize)]), - new("DimName3", DimensionValues[random.Next(0, ArraySize)])); } } diff --git a/test/OpenTelemetry.Tests.Stress.Traces/OpenTelemetry.Tests.Stress.Traces.csproj b/test/OpenTelemetry.Tests.Stress.Traces/OpenTelemetry.Tests.Stress.Traces.csproj index 41f6d28bc55..1229f8701f0 100644 --- a/test/OpenTelemetry.Tests.Stress.Traces/OpenTelemetry.Tests.Stress.Traces.csproj +++ b/test/OpenTelemetry.Tests.Stress.Traces/OpenTelemetry.Tests.Stress.Traces.csproj @@ -11,8 +11,10 @@ + + diff --git a/test/OpenTelemetry.Tests.Stress.Traces/Program.cs b/test/OpenTelemetry.Tests.Stress.Traces/Program.cs index 743da46b638..c73975dd0da 100644 --- a/test/OpenTelemetry.Tests.Stress.Traces/Program.cs +++ b/test/OpenTelemetry.Tests.Stress.Traces/Program.cs @@ -3,6 +3,7 @@ using System.Diagnostics; using System.Runtime.CompilerServices; +using CommandLine; using OpenTelemetry.Metrics; using OpenTelemetry.Trace; @@ -12,13 +13,10 @@ public partial class Program { private static readonly ActivitySource ActivitySource = new ActivitySource("OpenTelemetry.Tests.Stress"); - public static void Main() + public static void Main(string[] args) { - using var tracerProvider = Sdk.CreateTracerProviderBuilder() - .AddSource(ActivitySource.Name) - .Build(); - - Stress(prometheusPort: 9464); + Parser.Default.ParseArguments(args) + .WithParsed(LaunchStressTest); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -29,4 +27,17 @@ protected static void Run() activity?.SetTag("foo", "value"); } } + + protected static void WriteRunInformationToConsole(StressTestOptions options) + { + } + + private static void LaunchStressTest(StressTestOptions options) + { + using var tracerProvider = Sdk.CreateTracerProviderBuilder() + .AddSource(ActivitySource.Name) + .Build(); + + RunStressTest(options); + } } diff --git a/test/OpenTelemetry.Tests.Stress/Meat.cs b/test/OpenTelemetry.Tests.Stress/Meat.cs index 65e66535349..1a9fcabc39e 100644 --- a/test/OpenTelemetry.Tests.Stress/Meat.cs +++ b/test/OpenTelemetry.Tests.Stress/Meat.cs @@ -9,11 +9,19 @@ public partial class Program { public static void Main() { - Stress(concurrency: 1, prometheusPort: 9464); + RunStressTest(new() + { + Concurrency = 1, + PrometheusInternalMetricsPort = 9464, + }); } [MethodImpl(MethodImplOptions.AggressiveInlining)] protected static void Run() { } + + protected static void WriteRunInformationToConsole(StressTestOptions options) + { + } } diff --git a/test/OpenTelemetry.Tests.Stress/OpenTelemetry.Tests.Stress.csproj b/test/OpenTelemetry.Tests.Stress/OpenTelemetry.Tests.Stress.csproj index 60e3c917910..01af1c993ae 100644 --- a/test/OpenTelemetry.Tests.Stress/OpenTelemetry.Tests.Stress.csproj +++ b/test/OpenTelemetry.Tests.Stress/OpenTelemetry.Tests.Stress.csproj @@ -5,8 +5,10 @@ + + diff --git a/test/OpenTelemetry.Tests.Stress/Skeleton.cs b/test/OpenTelemetry.Tests.Stress/Skeleton.cs index cd3e5af7a8a..92350685d9d 100644 --- a/test/OpenTelemetry.Tests.Stress/Skeleton.cs +++ b/test/OpenTelemetry.Tests.Stress/Skeleton.cs @@ -4,6 +4,8 @@ using System.Diagnostics; using System.Diagnostics.Metrics; using System.Runtime.InteropServices; +using System.Text.Json; +using CommandLine; using OpenTelemetry.Metrics; namespace OpenTelemetry.Tests.Stress; @@ -17,21 +19,26 @@ static Program() { } - public static void Stress(int concurrency = 0, int prometheusPort = 0) + protected static void RunStressTest(StressTestOptions options) { + if (options == null) + { + throw new ArgumentNullException(nameof(options)); + } + #if DEBUG Console.WriteLine("***WARNING*** The current build is DEBUG which may affect timing!"); Console.WriteLine(); #endif - if (concurrency < 0) + if (options.Concurrency < 0) { - throw new ArgumentOutOfRangeException(nameof(concurrency), "concurrency level should be a non-negative number."); + throw new ArgumentOutOfRangeException(nameof(options.Concurrency), "Concurrency level should be a non-negative number."); } - if (concurrency == 0) + if (options.Concurrency == 0) { - concurrency = Environment.ProcessorCount; + options.Concurrency = Environment.ProcessorCount; } using var meter = new Meter("OpenTelemetry.Tests.Stress." + Guid.NewGuid().ToString("D")); @@ -55,29 +62,39 @@ public static void Stress(int concurrency = 0, int prometheusPort = 0) description: "The average CPU cycles for each `Run()` invocation, based on a small sliding window of few hundreds of milliseconds."); } - using var meterProvider = prometheusPort != 0 ? Sdk.CreateMeterProviderBuilder() + using var meterProvider = options.PrometheusInternalMetricsPort != 0 ? Sdk.CreateMeterProviderBuilder() .AddMeter(meter.Name) .AddRuntimeInstrumentation() - .AddPrometheusHttpListener( - options => options.UriPrefixes = new string[] { $"http://localhost:{prometheusPort}/" }) + .AddPrometheusHttpListener(o => o.UriPrefixes = new string[] { $"http://localhost:{options.PrometheusInternalMetricsPort}/" }) .Build() : null; - var statistics = new long[concurrency]; + var statistics = new long[options.Concurrency]; var watchForTotal = Stopwatch.StartNew(); + TimeSpan? duration = options.DurationSeconds > 0 + ? TimeSpan.FromSeconds(options.DurationSeconds) + : null; + Parallel.Invoke( () => { - Console.Write($"Running (concurrency = {concurrency}"); + Console.WriteLine($"Options: {JsonSerializer.Serialize(options)}"); + Console.WriteLine($"Run {Process.GetCurrentProcess().ProcessName}.exe --help to see available options."); + Console.Write($"Running (concurrency = {options.Concurrency}"); - if (prometheusPort != 0) + if (options.PrometheusInternalMetricsPort != 0) { - Console.Write($", prometheusEndpoint = http://localhost:{prometheusPort}/metrics/"); + Console.Write($", internalPrometheusEndpoint = http://localhost:{options.PrometheusInternalMetricsPort}/metrics/"); } - Console.WriteLine("), press to stop..."); + WriteRunInformationToConsole(options); + + Console.WriteLine("), press to stop, press to toggle statistics in the console..."); + Console.WriteLine(output); - var bOutput = false; + var outputCursorTop = Console.CursorTop - 1; + + var bOutput = true; var watch = new Stopwatch(); while (true) { @@ -103,7 +120,11 @@ public static void Stress(int concurrency = 0, int prometheusPort = 0) if (bOutput) { - Console.WriteLine(string.Format("{0} {1}", DateTime.UtcNow.ToString("O"), output)); + var tempCursorLeft = Console.CursorLeft; + var tempCursorTop = Console.CursorTop; + Console.SetCursorPosition(0, outputCursorTop); + Console.WriteLine(output.PadRight(Console.BufferWidth)); + Console.SetCursorPosition(tempCursorLeft, tempCursorTop); } var cntLoopsOld = (ulong)statistics.Sum(); @@ -122,13 +143,28 @@ public static void Stress(int concurrency = 0, int prometheusPort = 0) dLoopsPerSecond = (double)nLoops / ((double)watch.ElapsedMilliseconds / 1000.0); dCpuCyclesPerLoop = nLoops == 0 ? 0 : nCpuCycles / nLoops; - output = $"Loops: {cntLoopsTotal:n0}, Loops/Second: {dLoopsPerSecond:n0}, CPU Cycles/Loop: {dCpuCyclesPerLoop:n0}, RunwayTime (Seconds): {watchForTotal.Elapsed.TotalSeconds:n0} "; + var totalElapsedTime = watchForTotal.Elapsed; + + if (duration.HasValue) + { + output = $"Loops: {cntLoopsTotal:n0}, Loops/Second: {dLoopsPerSecond:n0}, CPU Cycles/Loop: {dCpuCyclesPerLoop:n0}, RemainingTime (Seconds): {(duration.Value - totalElapsedTime).TotalSeconds:n0}"; + if (totalElapsedTime > duration) + { + bContinue = false; + return; + } + } + else + { + output = $"Loops: {cntLoopsTotal:n0}, Loops/Second: {dLoopsPerSecond:n0}, CPU Cycles/Loop: {dCpuCyclesPerLoop:n0}, RunwayTime (Seconds): {totalElapsedTime.TotalSeconds:n0}"; + } + Console.Title = output; } }, () => { - Parallel.For(0, concurrency, (i) => + Parallel.For(0, options.Concurrency, (i) => { statistics[i] = 0; while (bContinue) @@ -145,10 +181,13 @@ public static void Stress(int concurrency = 0, int prometheusPort = 0) var cntCpuCyclesTotal = GetCpuCycles(); var cpuCyclesPerLoopTotal = cntLoopsTotal == 0 ? 0 : cntCpuCyclesTotal / cntLoopsTotal; Console.WriteLine("Stopping the stress test..."); - Console.WriteLine($"* Total Runaway Time (seconds) {watchForTotal.Elapsed.TotalSeconds:n0}"); + Console.WriteLine($"* Total Runway Time (Seconds) {watchForTotal.Elapsed.TotalSeconds:n0}"); Console.WriteLine($"* Total Loops: {cntLoopsTotal:n0}"); Console.WriteLine($"* Average Loops/Second: {totalLoopsPerSecond:n0}"); Console.WriteLine($"* Average CPU Cycles/Loop: {cpuCyclesPerLoopTotal:n0}"); +#if !NETFRAMEWORK + Console.WriteLine($"* GC Total Allocated Bytes: {GC.GetTotalAllocatedBytes()}"); +#endif } [DllImport("kernel32.dll")] @@ -169,4 +208,16 @@ private static ulong GetCpuCycles() return cycles; } + + protected partial class StressTestOptions + { + [Option('c', "concurrency", HelpText = "The concurrency (number of executing threads) for the stress test. Default value: Environment.ProcessorCount.", Required = false)] + public int Concurrency { get; set; } + + [Option('p', "internal_port", HelpText = "The prometheus http listener port where prometheus will be exposed for retrieving internal metrics while the stress test is running. Set to '0' to disable. Default value: 9464.", Required = false)] + public int PrometheusInternalMetricsPort { get; set; } = 9464; + + [Option('d', "duration", HelpText = "The duration for the stress test to run in seconds. If set to '0' or a negative value the stress test will run until canceled. Default value: 0.", Required = false)] + public int DurationSeconds { get; set; } + } }