Version 0.2 is ready

DevelopersTree · Oct 7, 2017 · bbc6cbf · bbc6cbf
1 parent e5c1290
commit bbc6cbf
Show file tree

Hide file tree

Showing 4 changed files with 42 additions and 5 deletions.
diff --git a/src/DevTree.Crawler/Crawler.cs b/src/DevTree.Crawler/Crawler.cs
@@ -18,7 +18,7 @@ public class Crawler
         private int _delay;
         private int _maxPages;
         private List<WebPage> _webPages;
-
+        private const string StatsFileName = "$Stats.txt";
         public Crawler(string[] args)
         {
             _uriToCrawl = GetSiteToCrawl(args);
@@ -50,6 +50,8 @@ private IWebCrawler GetDefaultWebCrawler(int maxPagesToCrawl, int delayInMillise
             config.IsExternalPageLinksCrawlingEnabled = false;
             config.IsRespectRobotsDotTextEnabled = true;
             config.IsUriRecrawlingEnabled = false;
+            config.IsHttpRequestAutoRedirectsEnabled = true;
+            config.UserAgentString = "DevTree Crawler";
             config.MaxConcurrentThreads = 10;
             config.MaxPagesToCrawl = maxPagesToCrawl;
             config.MinCrawlDelayPerDomainMilliSeconds = delayInMilliseconds;
@@ -89,7 +91,7 @@ void crawler_ProcessPageCrawlCompleted(object sender, PageCrawlCompletedArgs e)
 
             _webPages.Add(page);
 
-            File.WriteAllText(page.FileName, contents);
+            IOHelper.SaveFile(page.FileName, contents);
 
             Console.WriteLine("Pages crowled: " + _webPages.Count);
             Console.WriteLine($"Page Crawled: {page.Url}, Saved to: {page.FileName}.");
@@ -98,7 +100,7 @@ void crawler_ProcessPageCrawlCompleted(object sender, PageCrawlCompletedArgs e)
         public void SaveStats()
         {
             var statistics = _webPages.Select(w => $"{w.Url}, {w.FileName}").ToArray();
-            File.WriteAllLines(ParameterHelper.GetPath(_savePath, "Stats.txt"), statistics);
+            IOHelper.SaveFile(ParameterHelper.GetPath(_savePath, StatsFileName), statistics);
         }
 
     }

diff --git a/src/DevTree.Crawler/DevTree.Crawler.csproj b/src/DevTree.Crawler/DevTree.Crawler.csproj
@@ -63,6 +63,7 @@
   </ItemGroup>
   <ItemGroup>
     <Compile Include="Crawler.cs" />
+    <Compile Include="IOHelper.cs" />
     <Compile Include="Kurdish.cs" />
     <Compile Include="KurdishStringComparer.cs" />
     <Compile Include="Merger.cs" />

diff --git a/src/DevTree.Crawler/IOHelper.cs b/src/DevTree.Crawler/IOHelper.cs
@@ -0,0 +1,34 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace DevTree.Crawler
+{
+    public static class IOHelper
+    {
+        public static void SaveFile(string path, string contents)
+        {
+            EnsureDirectoryExists(path);
+
+            File.WriteAllText(path, contents);
+        }
+
+        public static void SaveFile(string path, string[] lines)
+        {
+            EnsureDirectoryExists(path);
+
+            File.WriteAllLines(path, lines);
+        }
+
+        public static void EnsureDirectoryExists(string path)
+        {
+            var directory = Path.GetDirectoryName(path);
+
+            if (!Directory.Exists(directory))
+                Directory.CreateDirectory(directory);
+        }
+    }
+}
diff --git a/src/DevTree.Crawler/Properties/AssemblyInfo.cs b/src/DevTree.Crawler/Properties/AssemblyInfo.cs
@@ -32,5 +32,5 @@
 // You can specify all the values or you can default the Build and Revision Numbers
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("0.1.0.0")]
-[assembly: AssemblyFileVersion("0.1.0.0")]
+[assembly: AssemblyVersion("0.2.0.0")]
+[assembly: AssemblyFileVersion("0.2.0.0")]