roofman2008 · roofman2008 · Nov 23, 2021 · Nov 22, 2021 · Nov 22, 2021 · Nov 23, 2021
diff --git a/PaheScrapper/App.config b/PaheScrapper/App.config
@@ -1,12 +1,12 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 <configuration>
     <configSections>
-        <sectionGroup name="userSettings" type="System.Configuration.UserSettingsGroup, System, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089" >
+        <sectionGroup name="userSettings" type="System.Configuration.UserSettingsGroup, System, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
             <section name="PaheScrapper.Properties.Configuration" type="System.Configuration.ClientSettingsSection, System, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089" allowExeDefinition="MachineToLocalUser" requirePermission="false" />
         </sectionGroup>
     </configSections>
     <startup> 
-        <supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5"/>
+        <supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5" />
     </startup>
     <userSettings>
         <PaheScrapper.Properties.Configuration>
@@ -26,10 +26,10 @@
                 <value>8</value>
             </setting>
             <setting name="HTMLSaveStateThershold" serializeAs="String">
-                <value>50</value>
+                <value>5</value>
             </setting>
             <setting name="WebDriveSaveStateThershold" serializeAs="String">
-                <value>10</value>
+                <value>1</value>
             </setting>
             <setting name="WebDriveRestartOnError" serializeAs="String">
                 <value>True</value>
@@ -51,4 +51,12 @@
             </setting>
         </PaheScrapper.Properties.Configuration>
     </userSettings>
+  <runtime>
+    <assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
+      <dependentAssembly>
+        <assemblyIdentity name="System.Buffers" publicKeyToken="cc7b13ffcd2ddd51" culture="neutral" />
+        <bindingRedirect oldVersion="0.0.0.0-4.0.3.0" newVersion="4.0.3.0" />
+      </dependentAssembly>
+    </assemblyBinding>
+  </runtime>
 </configuration>
diff --git a/PaheScrapper/Helpers/ConsoleHelper.cs b/PaheScrapper/Helpers/ConsoleHelper.cs
@@ -28,6 +28,14 @@ public static void LogBranch(string text)
             LogWriter lw = new LogWriter("Branch: " + text);
         }
 
+        public static void LogTime(TimeSpan timeSpan, TimeSpan totalTimeSpan)
+        {
+            Console.BackgroundColor = ConsoleColor.DarkMagenta;
+            Console.ForegroundColor = ConsoleColor.Yellow;
+            Console.WriteLine($"{(int)timeSpan.TotalMilliseconds} ms - {totalTimeSpan:G}");
+            LogWriter lw = new LogWriter("Time: " + $"{(int)timeSpan.TotalMilliseconds} ms - {totalTimeSpan:G}");
+        }
+
         public static void LogStats(string text)
         {
             Console.BackgroundColor = ConsoleColor.DarkBlue;

diff --git a/PaheScrapper/Helpers/StringCompressor.cs b/PaheScrapper/Helpers/StringCompressor.cs
@@ -0,0 +1,60 @@
+using System;
+using System.IO;
+using System.IO.Compression;
+using System.Text;
+
+namespace PaheScrapper.Helpers
+{
+    internal static class StringCompressor
+    {
+        /// <summary>
+        /// Compresses the string.
+        /// </summary>
+        /// <param name="text">The text.</param>
+        /// <returns></returns>
+        public static string CompressString(string text)
+        {
+            byte[] buffer = Encoding.UTF8.GetBytes(text);
+            var memoryStream = new MemoryStream();
+            using (var gZipStream = new GZipStream(memoryStream, CompressionMode.Compress, true))
+            {
+                gZipStream.Write(buffer, 0, buffer.Length);
+            }
+
+            memoryStream.Position = 0;
+
+            var compressedData = new byte[memoryStream.Length];
+            memoryStream.Read(compressedData, 0, compressedData.Length);
+
+            var gZipBuffer = new byte[compressedData.Length + 4];
+            Buffer.BlockCopy(compressedData, 0, gZipBuffer, 4, compressedData.Length);
+            Buffer.BlockCopy(BitConverter.GetBytes(buffer.Length), 0, gZipBuffer, 0, 4);
+            return Convert.ToBase64String(gZipBuffer);
+        }
+
+        /// <summary>
+        /// Decompresses the string.
+        /// </summary>
+        /// <param name="compressedText">The compressed text.</param>
+        /// <returns></returns>
+        public static string DecompressString(string compressedText)
+        {
+            byte[] gZipBuffer = Convert.FromBase64String(compressedText);
+            using (var memoryStream = new MemoryStream())
+            {
+                int dataLength = BitConverter.ToInt32(gZipBuffer, 0);
+                memoryStream.Write(gZipBuffer, 4, gZipBuffer.Length - 4);
+
+                var buffer = new byte[dataLength];
+
+                memoryStream.Position = 0;
+                using (var gZipStream = new GZipStream(memoryStream, CompressionMode.Decompress))
+                {
+                    gZipStream.Read(buffer, 0, buffer.Length);
+                }
+
+                return Encoding.UTF8.GetString(buffer);
+            }
+        }
+    }
+}
diff --git a/PaheScrapper/Helpers/WebErrorReporter.cs b/PaheScrapper/Helpers/WebErrorReporter.cs
@@ -0,0 +1,19 @@
+using System.Net;
+using HtmlAgilityPack;
+using Newtonsoft.Json;
+
+namespace PaheScrapper.Helpers
+{
+    public static class WebErrorReporter
+    {
+        public static void HttpError(WebResponse webResponse)
+        {
+            LogWriter lw = new LogWriter("Http Dump: \n" + JsonConvert.SerializeObject(webResponse));
+        }
+
+        public static void HtmlError(HtmlDocument htmlDocument)
+        {
+            LogWriter lw = new LogWriter("Html Dump: \n" + StringCompressor.CompressString(htmlDocument.DocumentNode.InnerHtml));
+        }
+    }
+}
diff --git a/PaheScrapper/PaheScrapper.csproj b/PaheScrapper/PaheScrapper.csproj
@@ -68,9 +68,18 @@
       <HintPath>..\packages\DotNetSeleniumExtras.WaitHelpers.3.11.0\lib\net45\SeleniumExtras.WaitHelpers.dll</HintPath>
     </Reference>
     <Reference Include="System" />
+    <Reference Include="System.Buffers, Version=4.0.3.0, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=MSIL">
+      <HintPath>..\packages\System.Buffers.4.5.1\lib\netstandard1.1\System.Buffers.dll</HintPath>
+    </Reference>
     <Reference Include="System.Core" />
     <Reference Include="System.Drawing" />
     <Reference Include="System.Management" />
+    <Reference Include="System.Memory, Version=4.0.1.1, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=MSIL">
+      <HintPath>..\packages\System.Memory.4.5.4\lib\netstandard1.1\System.Memory.dll</HintPath>
+    </Reference>
+    <Reference Include="System.Runtime.CompilerServices.Unsafe, Version=4.0.4.1, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
+      <HintPath>..\packages\System.Runtime.CompilerServices.Unsafe.4.5.3\lib\netstandard1.0\System.Runtime.CompilerServices.Unsafe.dll</HintPath>
+    </Reference>
     <Reference Include="System.Xml.Linq" />
     <Reference Include="System.Data.DataSetExtensions" />
     <Reference Include="Microsoft.CSharp" />
@@ -91,8 +100,10 @@
     <Compile Include="Helpers\LogWriter.cs" />
     <Compile Include="Helpers\MyContractResolver.cs" />
     <Compile Include="Helpers\NetworkHelper.cs" />
+    <Compile Include="Helpers\WebErrorReporter.cs" />
     <Compile Include="Helpers\ProcessHelper.cs" />
     <Compile Include="Helpers\RutimeParser.cs" />
+    <Compile Include="Helpers\StringCompressor.cs" />
     <Compile Include="Helpers\UriHelper.cs" />
     <Compile Include="Helpers\VMDecoder.cs" />
     <Compile Include="Helpers\WebDriverHelper.cs" />

diff --git a/PaheScrapper/Program.cs b/PaheScrapper/Program.cs
@@ -84,20 +84,20 @@ static void FullScrape(string command = null)
                 if (command == "continue")
                 {
                     f = File.OpenText(Path.Combine(_folderPath, Configuration.Default.ManagerStateFilename));
-                    _manager = ScrapperManager.Deserialize(f.ReadToEnd());
+                    _manager = ScrapperManager.Deserialize(StringCompressor.DecompressString(f.ReadToEnd()));
                     f.Close();
                 }
                 else if (command == "resync")
                 {
                     f = File.OpenText(Path.Combine(_folderPath, Configuration.Default.ManagerStateFilename));
-                    _manager = ScrapperManager.Deserialize(f.ReadToEnd());
+                    _manager = ScrapperManager.Deserialize(StringCompressor.DecompressString(f.ReadToEnd()));
                     _manager.ResetState();
                     f.Close();
                 }
                 else if (command == "loop")
                 {
                     f = File.OpenText(Path.Combine(_folderPath, Configuration.Default.ManagerStateFilename));
-                    _manager = ScrapperManager.Deserialize(f.ReadToEnd());
+                    _manager = ScrapperManager.Deserialize(StringCompressor.DecompressString(f.ReadToEnd()));
                     f.Close();
                     isLoop = true;
                 }
@@ -108,9 +108,9 @@ static void FullScrape(string command = null)
                 else if (command == "state")
                 {
                     f = File.OpenText(Path.Combine(_folderPath, Configuration.Default.ManagerStateFilename));
-                    _manager = ScrapperManager.Deserialize(f.ReadToEnd());
+                    _manager = ScrapperManager.Deserialize(StringCompressor.DecompressString(f.ReadToEnd()));
                     f.Close();
-                    ConsoleHelper.LogCommandHandled($"Current State = {_manager.State.ToString()}");
+                    ConsoleHelper.LogCommandHandled($"Current State = {_manager.State}");
                     _manager = null;
                     command = null;
                     goto recommand;
@@ -164,7 +164,7 @@ static void FullScrape(string command = null)
 
                             using (var file = File.CreateText(Path.Combine(_folderPath, Configuration.Default.ManagerStateFilename)))
                             {
-                                file.WriteAsync(_manager.Serialize()).Wait();
+                                file.WriteAsync(StringCompressor.CompressString(_manager.Serialize())).Wait();
                                 file.Close();
                             }
 
@@ -209,7 +209,7 @@ static void FullScrape(string command = null)
                             using (var file =
                                 File.CreateText(Path.Combine(_folderPath, Configuration.Default.ManagerStateFilename)))
                             {
-                                file.WriteAsync(_manager.Serialize()).Wait();
+                                file.WriteAsync(StringCompressor.CompressString(_manager.Serialize())).Wait();
                                 file.Close();
                             }
 
@@ -232,7 +232,7 @@ static void FullScrape(string command = null)
                 ConsoleHelper.LogStorage("Final Save Scraper State");
                 using (var file = File.CreateText(Path.Combine(_folderPath, Configuration.Default.ManagerStateFilename)))
                 {
-                    file.WriteAsync(_manager.Serialize()).Wait();
+                    file.WriteAsync(StringCompressor.CompressString(_manager.Serialize())).Wait();
                     file.Close();
                 }
 

diff --git a/PaheScrapper/Properties/Configuration.Designer.cs b/PaheScrapper/Properties/Configuration.Designer.cs
diff --git a/PaheScrapper/Properties/Configuration.settings b/PaheScrapper/Properties/Configuration.settings
@@ -18,10 +18,10 @@
       <Value Profile="(Default)">8</Value>
     </Setting>
     <Setting Name="HTMLSaveStateThershold" Type="System.Int32" Scope="User">
-      <Value Profile="(Default)">50</Value>
+      <Value Profile="(Default)">5</Value>
     </Setting>
     <Setting Name="WebDriveSaveStateThershold" Type="System.Int32" Scope="User">
-      <Value Profile="(Default)">10</Value>
+      <Value Profile="(Default)">1</Value>
     </Setting>
     <Setting Name="WebDriveRestartOnError" Type="System.Boolean" Scope="User">
       <Value Profile="(Default)">True</Value>

diff --git a/PaheScrapper/ScrapperConstants.cs b/PaheScrapper/ScrapperConstants.cs
@@ -3,7 +3,7 @@
     public static class ScrapperConstants
     {
         public static string WebsiteLanding() => "https://pahe.ph/";
-        public static string WebsiteLandingPaging(int page) => page <= 1 ? WebsiteLanding() : $"https://pahe.ph/page/{page}/";
+        public static string WebsiteLandingPaging(int page) => page < 1 ? WebsiteLanding() : $"https://pahe.ph/page/{page + 1}/";
         public static int HttpRequestTimeout() => 15 * 1000;
     }
 }