Fix pattern for tests that use HttpClient (#589)

This change introduces a better API for us to test the DaprClient or
other HttpClient based APIs.

This will resolve the flakiness problems that we're seeing with some of
the actors tests.

Fixes: #573
Fixes: #588

Additionally fixed an issue where DaprHttpInteractor was misuing
HttpClientHandler. This would result in a new handler being created when
it isn't needed.

properties/IsExternalInit.cs

@@ -0,0 +1,18 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.ComponentModel;
+
+namespace System.Runtime.CompilerServices
+{
+    /// <summary>
+    /// Reserved to be used by the compiler for tracking metadata.
+    /// This class should not be used by developers in source code.
+    /// </summary>
+    [EditorBrowsable(EditorBrowsableState.Never)]
+    internal static class IsExternalInit
+    {
+    }
+
+    // This is a polyfill for init only properties in netcoreapp3.1
+}

properties/dapr_managed_netcore.props

@@ -3,11 +3,16 @@
   <Import Project="dapr_common.props" />
   <PropertyGroup>
     <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <LangVersion>9.0</LangVersion>
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
     <WarningLevel>4</WarningLevel>
     <WarnOnPackingNonPackableProject>false</WarnOnPackingNonPackableProject>
    </PropertyGroup>
 
+   <ItemGroup>
+    <Compile Include="$(MSBuildThisFileDirectory)\IsExternalInit.cs" />
+   </ItemGroup>
+
   <!-- Cls Compliant -->
   <PropertyGroup>
     <AssemblyClsCompliant>true</AssemblyClsCompliant>

src/Dapr.Actors/Client/ActorProxyFactory.cs

@@ -16,7 +16,7 @@ namespace Dapr.Actors.Client
     public class ActorProxyFactory : IActorProxyFactory
     {
         private ActorProxyOptions defaultOptions;
-        private readonly HttpClientHandler handler;
+        private readonly HttpMessageHandler handler;
 
         /// <inheritdoc/>
         public ActorProxyOptions DefaultOptions
@@ -32,7 +32,16 @@ public ActorProxyOptions DefaultOptions
         /// <summary>
         /// Initializes a new instance of the <see cref="ActorProxyFactory"/> class.
         /// </summary>
-        public ActorProxyFactory(ActorProxyOptions options = null, HttpClientHandler handler = null)
+        [Obsolete("Use the constructor that accepts HttpMessageHandler. This will be removed in the future.")]
+        public ActorProxyFactory(ActorProxyOptions options, HttpClientHandler handler)
+            : this(options, (HttpMessageHandler)handler)
+        {
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="ActorProxyFactory"/> class.
+        /// </summary>
+        public ActorProxyFactory(ActorProxyOptions options = null, HttpMessageHandler handler = null)
         {
             this.defaultOptions = options ?? new ActorProxyOptions();
             this.handler = handler;

src/Dapr.Actors/DaprHttpInteractor.cs

@@ -28,19 +28,20 @@ internal class DaprHttpInteractor : IDaprInteractor
         private readonly JsonSerializerOptions jsonSerializerOptions = JsonSerializerDefaults.Web;
         private const string DaprEndpoint = Constants.DaprDefaultEndpoint;
         private readonly string daprPort;
-        private static HttpClientHandler innerHandler = new HttpClientHandler();
-        private HttpClient httpClient = null;
-        private bool disposed = false;
+        private readonly static HttpMessageHandler defaultHandler = new HttpClientHandler();
+        private readonly HttpMessageHandler handler;
+        private HttpClient httpClient;
+        private bool disposed;
         private string daprApiToken;
 
         public DaprHttpInteractor(
-            HttpClientHandler clientHandler = null,
+            HttpMessageHandler clientHandler = null,
             string apiToken = null)
         {
             // Get Dapr port from Environment Variable if it has been overridden.
             this.daprPort = Environment.GetEnvironmentVariable("DAPR_HTTP_PORT") ?? Constants.DaprDefaultPort;
 
-            innerHandler = clientHandler ?? new HttpClientHandler();
+            this.handler = clientHandler ?? defaultHandler;
             this.daprApiToken = apiToken;
             this.httpClient = this.CreateHttpClient();
         }
@@ -433,7 +434,7 @@ private async Task<HttpResponseMessage> SendAsyncHandleSecurityExceptions(
 
         private HttpClient CreateHttpClient()
         {
-            return new HttpClient(innerHandler, false);
+            return new HttpClient(this.handler, false);
         }
 
         private void AddDaprApiTokenHeader(HttpRequestMessage request)

src/Dapr.Client/DaprClientBuilder.cs

@@ -48,6 +48,8 @@ public DaprClientBuilder()
         // property exposed for testing purposes
         internal string HttpEndpoint { get; private set; }
 
+        private Func<HttpClient> HttpClientFactory { get; set; }
+
         // property exposed for testing purposes
         internal JsonSerializerOptions JsonSerializerOptions { get; private set; }
 
@@ -71,6 +73,13 @@ public DaprClientBuilder UseHttpEndpoint(string httpEndpoint)
             return this;
         }
 
+        // Internal for testing of DaprClient
+        internal DaprClientBuilder UseHttpClientFactory(Func<HttpClient> factory)
+        {
+            this.HttpClientFactory = factory;
+            return this;
+        }
+
         /// <summary>
         /// Overrides the gRPC endpoint used by <see cref="DaprClient" /> for communicating with the Dapr runtime.
         /// </summary>
@@ -153,7 +162,8 @@ public DaprClient Build()
             var client = new Autogenerated.Dapr.DaprClient(channel);
 
             var apiTokenHeader = DaprClient.GetDaprApiTokenHeader(this.DaprApiToken);
-            return new DaprClientGrpc(channel, client, new HttpClient(), httpEndpoint, this.JsonSerializerOptions, apiTokenHeader);
+            var httpClient = HttpClientFactory is object ? HttpClientFactory() : new HttpClient();
+            return new DaprClientGrpc(channel, client, httpClient, httpEndpoint, this.JsonSerializerOptions, apiTokenHeader);
         }
     }
 }

src/Dapr.Client/properties/AssemblyInfo.cs

@@ -3,6 +3,7 @@
 // Licensed under the MIT License.
 // ------------------------------------------------------------
 
+[assembly: System.Runtime.CompilerServices.InternalsVisibleTo("Dapr.Actors.Test, PublicKey=0024000004800000940000000602000000240000525341310004000001000100b1f597635c44597fcecb493e2b1327033b29b1a98ac956a1a538664b68f87d45fbaada0438a15a6265e62864947cc067d8da3a7d93c5eb2fcbb850e396c8684dba74ea477d82a1bbb18932c0efb30b64ff1677f85ae833818707ac8b49ad8062ca01d2c89d8ab1843ae73e8ba9649cd28666b539444dcdee3639f95e2a099bb2")]
 [assembly: System.Runtime.CompilerServices.InternalsVisibleTo("Dapr.Client.Test, PublicKey=0024000004800000940000000602000000240000525341310004000001000100b1f597635c44597fcecb493e2b1327033b29b1a98ac956a1a538664b68f87d45fbaada0438a15a6265e62864947cc067d8da3a7d93c5eb2fcbb850e396c8684dba74ea477d82a1bbb18932c0efb30b64ff1677f85ae833818707ac8b49ad8062ca01d2c89d8ab1843ae73e8ba9649cd28666b539444dcdee3639f95e2a099bb2")]
 [assembly: System.Runtime.CompilerServices.InternalsVisibleTo("Dapr.AspNetCore.IntegrationTest, PublicKey=0024000004800000940000000602000000240000525341310004000001000100b1f597635c44597fcecb493e2b1327033b29b1a98ac956a1a538664b68f87d45fbaada0438a15a6265e62864947cc067d8da3a7d93c5eb2fcbb850e396c8684dba74ea477d82a1bbb18932c0efb30b64ff1677f85ae833818707ac8b49ad8062ca01d2c89d8ab1843ae73e8ba9649cd28666b539444dcdee3639f95e2a099bb2")]
 [assembly: System.Runtime.CompilerServices.InternalsVisibleTo("Dapr.AspNetCore.Test, PublicKey=0024000004800000940000000602000000240000525341310004000001000100b1f597635c44597fcecb493e2b1327033b29b1a98ac956a1a538664b68f87d45fbaada0438a15a6265e62864947cc067d8da3a7d93c5eb2fcbb850e396c8684dba74ea477d82a1bbb18932c0efb30b64ff1677f85ae833818707ac8b49ad8062ca01d2c89d8ab1843ae73e8ba9649cd28666b539444dcdee3639f95e2a099bb2")]

test/Dapr.Actors.Test/ApiTokenTests.cs

@@ -4,8 +4,6 @@
 // ------------------------------------------------------------
 
 using System;
-using System.Collections.Concurrent;
-using System.Net.Http;
 using System.Threading;
 using System.Threading.Tasks;
 using Dapr.Actors.Client;
@@ -16,107 +14,96 @@ namespace Dapr.Actors.Test
 {
     public class ApiTokenTests
     {
-        [Fact(Skip = "Failing due to #573")]
-        public void CreateProxyWithRemoting_WithApiToken()
+        [Fact]
+        public async Task CreateProxyWithRemoting_WithApiToken()
         {
+            await using var client = TestClient.CreateForMessageHandler();
+
             var actorId = new ActorId("abc");
-            var handler = new TestHttpClientHandler();
             var options = new ActorProxyOptions
             {
                 DaprApiToken = "test_token",
             };
-            var factory = new ActorProxyFactory(options, handler);
-            var proxy = factory.CreateActorProxy<ITestActor>(actorId, "TestActor");
-            var task = proxy.SetCountAsync(1, new CancellationToken());
 
-            handler.Requests.TryDequeue(out var entry).Should().BeTrue();
-            var headerValues = entry.Request.Headers.GetValues("dapr-api-token");
+            var request = await client.CaptureHttpRequestAsync(async handler =>
+            {
+                var factory = new ActorProxyFactory(options, handler);
+                var proxy = factory.CreateActorProxy<ITestActor>(actorId, "TestActor");
+                await proxy.SetCountAsync(1, new CancellationToken());
+            });
+
+            request.Dismiss();
+
+            var headerValues = request.Request.Headers.GetValues("dapr-api-token");
             headerValues.Should().Contain("test_token");
         }
 
-        [Fact(Skip = "Failing due to #573")]
-        public void CreateProxyWithRemoting_WithNoApiToken()
+        [Fact]
+        public async Task CreateProxyWithRemoting_WithNoApiToken()
         {
+            await using var client = TestClient.CreateForMessageHandler();
+
             var actorId = new ActorId("abc");
-            var handler = new TestHttpClientHandler();
-            var factory = new ActorProxyFactory(null, handler);
-            var proxy = factory.CreateActorProxy<ITestActor>(actorId, "TestActor");
-            var task = proxy.SetCountAsync(1, new CancellationToken());
-
-            handler.Requests.TryDequeue(out var entry).Should().BeTrue();
-            Action action = () => entry.Request.Headers.GetValues("dapr-api-token");
-            action.Should().Throw<InvalidOperationException>();
+
+            var request = await client.CaptureHttpRequestAsync(async handler =>
+            {
+                var factory = new ActorProxyFactory(null, handler);
+                var proxy = factory.CreateActorProxy<ITestActor>(actorId, "TestActor");
+                await proxy.SetCountAsync(1, new CancellationToken());
+            });
+
+            request.Dismiss();
+
+            Assert.Throws<InvalidOperationException>(() =>
+            {
+                request.Request.Headers.GetValues("dapr-api-token");
+            });
         }
 
-        [Fact(Skip = "Failing due to #573")]
-        public void CreateProxyWithNoRemoting_WithApiToken()
+        [Fact]
+        public async Task CreateProxyWithNoRemoting_WithApiToken()
         {
+            await using var client = TestClient.CreateForMessageHandler();
+
             var actorId = new ActorId("abc");
-            var handler = new TestHttpClientHandler();
             var options = new ActorProxyOptions
             {
                 DaprApiToken = "test_token",
             };
-            var factory = new ActorProxyFactory(options, handler);
-            var proxy = factory.Create(actorId, "TestActor");
-            var task = proxy.InvokeMethodAsync("SetCountAsync", 1, new CancellationToken());
-
-            handler.Requests.TryDequeue(out var entry).Should().BeTrue();
-            var headerValues = entry.Request.Headers.GetValues("dapr-api-token");
-            headerValues.Should().Contain("test_token");
-        }
 
-        [Fact(Skip = "Failing due to #573")]
-        public void CreateProxyWithNoRemoting_WithNoApiToken()
-        {
-            var actorId = new ActorId("abc");
-            var handler = new TestHttpClientHandler();
-            var factory = new ActorProxyFactory(null, handler);
-            var proxy = factory.Create(actorId, "TestActor");
-            var task = proxy.InvokeMethodAsync("SetCountAsync", 1, new CancellationToken());
-
-            handler.Requests.TryDequeue(out var entry).Should().BeTrue();
-            Action action = () => entry.Request.Headers.GetValues("dapr-api-token");
-            action.Should().Throw<InvalidOperationException>();
-        }
-
-
-        public class Entry
-        {
-            public Entry(HttpRequestMessage request)
+            var request = await client.CaptureHttpRequestAsync(async handler =>
             {
-                this.Request = request;
+                var factory = new ActorProxyFactory(options, handler);
+                var proxy = factory.Create(actorId, "TestActor");
+                await proxy.InvokeMethodAsync("SetCountAsync", 1, new CancellationToken());
+            });
 
-                this.Completion = new TaskCompletionSource<HttpResponseMessage>(TaskCreationOptions.RunContinuationsAsynchronously);
-            }
+            request.Dismiss();
 
-            public TaskCompletionSource<HttpResponseMessage> Completion { get; }
-
-            public HttpRequestMessage Request { get; }
+            var headerValues = request.Request.Headers.GetValues("dapr-api-token");
+            headerValues.Should().Contain("test_token");
         }
 
-        private class TestHttpClientHandler : HttpClientHandler
+        [Fact]
+        public async Task CreateProxyWithNoRemoting_WithNoApiToken()
         {
-            public TestHttpClientHandler()
-            {
-                this.Requests = new ConcurrentQueue<Entry>();
-            }
+            await using var client = TestClient.CreateForMessageHandler();
 
-            public ConcurrentQueue<Entry> Requests { get; }
+            var actorId = new ActorId("abc");
+
+            var request = await client.CaptureHttpRequestAsync(async handler =>
+            {
+                var factory = new ActorProxyFactory(null, handler);
+                var proxy = factory.Create(actorId, "TestActor");
+                await proxy.InvokeMethodAsync("SetCountAsync", 1, new CancellationToken());
+            });
 
-            public Action<Entry> Handler { get; set; }
+            request.Dismiss();
 
-            protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
+            Assert.Throws<InvalidOperationException>(() =>
             {
-                var entry = new Entry(request);
-                this.Handler?.Invoke(entry);
-                this.Requests.Enqueue(entry);
-
-                using (cancellationToken.Register(() => entry.Completion.TrySetCanceled()))
-                {
-                    return await entry.Completion.Task.ConfigureAwait(false);
-                }
-            }
+                request.Request.Headers.GetValues("dapr-api-token");
+            });
         }
     }
 }

test/Dapr.Actors.Test/Dapr.Actors.Test.csproj

@@ -2,6 +2,7 @@
   <PropertyGroup>
     <TargetFrameworks>netcoreapp3.1;net5</TargetFrameworks>
     <RootNamespace>Dapr.Actors</RootNamespace>
+    <DefineConstants>$(DefineConstants);ACTORS</DefineConstants>
   </PropertyGroup>
 
   <ItemGroup>
@@ -20,6 +21,11 @@
     </PackageReference>
   </ItemGroup>
 
+  <ItemGroup>
+    <Compile Include="..\Shared\GrpcUtils.cs" />
+    <Compile Include="..\Shared\TestClient.cs" />
+  </ItemGroup>
+
   <ItemGroup>
     <ProjectReference Include="..\..\src\Dapr.Actors\Dapr.Actors.csproj" />
   </ItemGroup>