If we can't pull the robots information, don't break completely (#37)

g3rv4 · Feb 4, 2023 · 91ac0e2 · 91ac0e2
1 parent 32280c3
commit 91ac0e2
Showing 1 changed file with 13 additions and 1 deletion.
diff --git a/src/Program.cs b/src/Program.cs
@@ -41,7 +41,14 @@ await Parallel.ForEachAsync(Config.Instance.Sites,
     new ParallelOptions { MaxDegreeOfParallelism = Config.Instance.Sites.Length },
     async (site, _) =>
     {
-        sitesRobotFile[site.Host] = await robotsFileParser.FromUriAsync(new Uri($"http://{site.Host}/robots.txt"));
+        try
+        {
+            sitesRobotFile[site.Host] = await robotsFileParser.FromUriAsync(new Uri($"http://{site.Host}/robots.txt"));
+        }
+        catch
+        {
+            Console.WriteLine($"Ignoring {site.Host} because had issues fetching its robots data (is the site down?)");
+        }
     }
 );
 
@@ -88,6 +95,11 @@ await Parallel.ForEachAsync(Config.Instance.Sites,
             return;
         }
     }
+    else
+    {
+        Console.WriteLine($"Not scraping {url} because I couldn't fetch robots data.");
+        return;
+    }
 
     HttpResponseMessage? response = null;
     try