Merge pull request #10 from roofman2008/dev

Alpha-2.5
roofman2008 · Nov 26, 2021 · 2faa309 · 2faa309
2 parents e1b9937 + 7c54d9e
commit 2faa309
Show file tree

Hide file tree

Showing 6 changed files with 59 additions and 17 deletions.
diff --git a/PaheScrapper/App.config b/PaheScrapper/App.config
@@ -26,10 +26,10 @@
                 <value>8</value>
             </setting>
             <setting name="HTMLSaveStateThershold" serializeAs="String">
-                <value>5</value>
+                <value>500</value>
             </setting>
             <setting name="WebDriveSaveStateThershold" serializeAs="String">
-                <value>1</value>
+                <value>100</value>
             </setting>
             <setting name="WebDriveRestartOnError" serializeAs="String">
                 <value>True</value>
@@ -44,7 +44,7 @@
                 <value>100</value>
             </setting>
             <setting name="FailsafeStateThershold" serializeAs="String">
-                <value>50</value>
+                <value>10</value>
             </setting>
             <setting name="OutputScrapeFilename" serializeAs="String">
                 <value>output.json</value>

diff --git a/PaheScrapper/Program.cs b/PaheScrapper/Program.cs
@@ -12,7 +12,7 @@ class Program
     {
         static void Main(string[] args)
         {
-            ConsoleHelper.LogInfo("Pahe Scraper - Alpha 2.4");
+            ConsoleHelper.LogInfo("Pahe Scraper - Alpha 2.5");
 
             if (args.Length == 3 && args[0] == "-d")
             {

diff --git a/PaheScrapper/Properties/Configuration.Designer.cs b/PaheScrapper/Properties/Configuration.Designer.cs
diff --git a/PaheScrapper/Properties/Configuration.settings b/PaheScrapper/Properties/Configuration.settings
@@ -18,10 +18,10 @@
       <Value Profile="(Default)">8</Value>
     </Setting>
     <Setting Name="HTMLSaveStateThershold" Type="System.Int32" Scope="User">
-      <Value Profile="(Default)">5</Value>
+      <Value Profile="(Default)">500</Value>
     </Setting>
     <Setting Name="WebDriveSaveStateThershold" Type="System.Int32" Scope="User">
-      <Value Profile="(Default)">1</Value>
+      <Value Profile="(Default)">100</Value>
     </Setting>
     <Setting Name="WebDriveRestartOnError" Type="System.Boolean" Scope="User">
       <Value Profile="(Default)">True</Value>
@@ -36,7 +36,7 @@
       <Value Profile="(Default)">100</Value>
     </Setting>
     <Setting Name="FailsafeStateThershold" Type="System.Int32" Scope="User">
-      <Value Profile="(Default)">50</Value>
+      <Value Profile="(Default)">10</Value>
     </Setting>
     <Setting Name="OutputScrapeFilename" Type="System.String" Scope="User">
       <Value Profile="(Default)">output.json</Value>

diff --git a/PaheScrapper/ScrapperManager.cs b/PaheScrapper/ScrapperManager.cs
@@ -51,22 +51,35 @@ void BypassSurcuriRoutine()
 
             void PersistHtmlState(bool transition)
             {
-                if ((_currentPage + 1) % Configuration.Default.HTMLSaveStateThershold == 0)
+                if (!transition)
                 {
-                    if (!transition)
+                    if ((_currentPage + 1) % Configuration.Default.HTMLSaveStateThershold == 0)
+                    {
                         _currentPage++;
 
+                        saveState(_scrapperState);
+                    }
+                }
+                else
+                {
                     saveState(_scrapperState);
                 }
             }
 
             void PersistWebDriveState(bool transition)
             {
-                if ((_currentPage + 1) % Configuration.Default.WebDriveSaveStateThershold == 0)
+                if (!transition)
                 {
-                    if (!transition)
+                    if ((_currentPage + 1) % Configuration.Default.WebDriveSaveStateThershold == 0)
+                    {
+
                         _currentPage++;
 
+                        saveState(_scrapperState);
+                    }
+                }
+                else
+                {
                     saveState(_scrapperState);
                 }
             }

diff --git a/PaheScrapper/ScrapperMethods.cs b/PaheScrapper/ScrapperMethods.cs
@@ -475,14 +475,43 @@ public static VMMovieLookup DecodeDetailsVM(HtmlDocument document)
             startIndex = documentHtml.IndexOf(startPattern, StringComparison.Ordinal);
             documentHtml = documentHtml.Substring(startIndex + startPattern.Length, documentHtml.Length - startPattern.Length - startIndex);
             endIndex = documentHtml.IndexOf(endPattern, StringComparison.Ordinal) + 1;
+
+            /*Array Case*/
+            bool arrayOfString = false;
+            if (endIndex == 0)
+            {
+                endPattern = "]; function";
+                endIndex = documentHtml.IndexOf(endPattern, StringComparison.Ordinal) + 1;
+                arrayOfString = true;
+            }
+
             documentHtml = documentHtml.Substring(0, endIndex);
 
             if (string.IsNullOrEmpty(documentHtml))
                 return new VMMovieLookup();
 
-            JObject linksObject = JObject.Parse(documentHtml);
-            IEnumerable<JToken> linksTokens = linksObject.Properties().Select(l => l.Value).ToArray();
-            string[] linksArray = linksTokens.Select(l => l.Value<string>()).ToArray();
+            string[] linksArray = null;
+
+            if (arrayOfString)
+            {
+                /*Array Case*/
+                if (documentHtml != "[]")
+                {
+                    JArray linksInArrayStruct = new JArray(documentHtml);
+                    linksArray = linksInArrayStruct.Children().Select(l => l.Value<string>()).ToArray();
+                }
+            }
+            else
+            {
+                /*Object Case*/
+                JObject linksObject = JObject.Parse(documentHtml);
+                IEnumerable<JToken> linksTokens = linksObject.Properties().Select(l => l.Value).ToArray();
+                linksArray = linksTokens.Select(l => l.Value<string>()).ToArray();
+            }
+
+            /*Empty Array => Empty Lookup*/
+            if (linksArray == null)
+                return new VMMovieLookup();
 
             //Movie Page Links Buttons
             documentHtml = decodedHtml;