Handle 429s gracefully

g3rv4 · Dec 15, 2023 · dc7b18f · dc7b18f
1 parent 3577a60
commit dc7b18f
Showing 1 changed file with 15 additions and 0 deletions.
diff --git a/src/Program.cs b/src/Program.cs
@@ -1,4 +1,5 @@
 using System.Collections.Concurrent;
+using System.Net;
 using System.Text.Json;
 using GetMoarFediverse;
 using GetMoarFediverse.Configuration;
@@ -153,6 +154,7 @@ await Parallel.ForEachAsync(Context.Configuration.Sites,
 
 var statusesToLoad = statusesToLoadBag.ToHashSet();
 Console.WriteLine($"Originally retrieved {statusesToLoadBag.Count} statuses. After removing duplicates, I got {statusesToLoad.Count} really unique ones");
+var rateLimited = false;
 foreach (var statusLink in statusesToLoad)
 {
     Console.WriteLine($"Bringing in {statusLink}");
@@ -164,8 +166,21 @@ await Parallel.ForEachAsync(Context.Configuration.Sites,
         };
 
         var res = await authClient.PostAsync("index", new FormUrlEncodedContent(content));
+        if (res.StatusCode == HttpStatusCode.TooManyRequests)
+        {
+            // fakerelay.gervas.io has a token bucket rate limit of 30 tokens per minute, allowing bursts of up to
+            // 60 requests per minute. Once we hit a 429, we should do a request every 2 seconds. 
+            Console.WriteLine("Got a 429 from FakeRelay, waiting 2 second and retrying");
+            rateLimited = true;
+            await Task.Delay(TimeSpan.FromSeconds(2));
+            res = await authClient.PostAsync("index", new FormUrlEncodedContent(content));
+        }
         res.EnsureSuccessStatusCode();
         importedList.Add(statusLink);
+        if (rateLimited)
+        {
+            await Task.Delay(TimeSpan.FromSeconds(2));
+        }
     }
     catch (Exception e)
     {