Merge pull request #511 from hazzik/csv_load-detect-encoding

Automatically detect encoding in csv_load script function
Serg-Norseman · Nov 7, 2023 · 3a50d77 · 3a50d77
2 parents 0fe355c + 224834f
commit 3a50d77
Show file tree

Hide file tree

Showing 2 changed files with 28 additions and 13 deletions.
diff --git a/projects/GKCore/GKCore/GKUtils.cs b/projects/GKCore/GKCore/GKUtils.cs
@@ -44,7 +44,7 @@
 namespace GKCore
 {
     /// <summary>
-    /// 
+    ///
     /// </summary>
     public static class GKUtils
     {
@@ -369,9 +369,9 @@ public static TaskGoalRet GetTaskGoal(GDMTree tree, GDMTaskRecord taskRec)
         public static string GetTaskGoalStr(GDMTree tree, GDMTaskRecord taskRec)
         {
             if (tree == null || taskRec == null) return string.Empty;
-            
+
             string result = "";
-            
+
             var goal = GetTaskGoal(tree, taskRec);
 
             switch (goal.GoalType) {
@@ -482,7 +482,13 @@ public static CharsetResult DetectCharset(Stream stream, int bufferSize = 32768)
             return result;
         }
 
-        public static StreamReader GetDetectedStreamReader(Stream stream)
+        public static Encoding DetectEncoding(string fileName)
+        {
+            using (var file = File.OpenRead(fileName))
+                return DetectEncoding(file);
+        }
+
+        public static Encoding DetectEncoding(Stream stream)
         {
             Encoding defaultEncoding;
             try {
@@ -492,6 +498,13 @@ public static StreamReader GetDetectedStreamReader(Stream stream)
                 defaultEncoding = Encoding.UTF8;
             }
 
+            return defaultEncoding;
+        }
+
+        public static StreamReader GetDetectedStreamReader(Stream stream)
+        {
+            var defaultEncoding = DetectEncoding(stream);
+
             StreamReader reader = new StreamReader(stream, defaultEncoding);
             return reader;
         }
@@ -821,8 +834,8 @@ public static string GetShortDatePattern()
         }
 
         /// <summary>
-        /// The result of the function is a "normalized date", delimited by '.' and fixed order of parts: "dd.mm.yyyy". 
-        /// The pattern and regional date contain the delimiter '/'. 
+        /// The result of the function is a "normalized date", delimited by '.' and fixed order of parts: "dd.mm.yyyy".
+        /// The pattern and regional date contain the delimiter '/'.
         /// The pattern defines the position of the parts in a regional date format.
         /// </summary>
         /// <param name="regionalDate">date similar "01/20/1970"</param>
@@ -863,8 +876,8 @@ public static string GetNormalizeDate(string regionalDate, string pattern)
         }
 
         /// <summary>
-        /// The result of the function is a "regional date", delimited by '/' and regional order of parts: "mm/dd/yyyy" 
-        /// or any other. The pattern and regional date contain the delimiter '/'. 
+        /// The result of the function is a "regional date", delimited by '/' and regional order of parts: "mm/dd/yyyy"
+        /// or any other. The pattern and regional date contain the delimiter '/'.
         /// The pattern defines the position of the parts in a regional date format.
         /// </summary>
         /// <param name="normalizeDate">date with format "dd.mm.yyyy"</param>
@@ -1235,7 +1248,7 @@ public static int GetDaysForBirth(GDMIndividualRecord iRec)
                                     double curN = curY + curM / 12.0 + curD / 12.0 / 31.0;
                                     bdY = (bdN < curN) ? (curY + 1) : curY;
 
-                                    // There are valid birthdays on February 29th in leap years. 
+                                    // There are valid birthdays on February 29th in leap years.
                                     // For other years, we need a correction for an acceptable day.
                                     if (bdD == 29 && bdM == 2 && !DateTime.IsLeapYear(bdY)) {
                                         bdD -= 1;
@@ -1287,7 +1300,7 @@ public static int GetDaysForBirthAnniversary(GDMIndividualRecord iRec, out int y
                                     if (anniversary) {
                                         bdY = (bdN < curN) ? (curY + 1) : curY;
 
-                                        // There are valid birthdays on February 29th in leap years. 
+                                        // There are valid birthdays on February 29th in leap years.
                                         // For other years, we need a correction for an acceptable day.
                                         if (bdD == 29 && bdM == 2 && !DateTime.IsLeapYear(bdY)) {
                                             bdD -= 1;
@@ -2080,7 +2093,7 @@ private static void ShowPersonNamesakes(GDMTree tree, GDMIndividualRecord iRec,
                         int num4 = namesakes.Count;
                         for (int i = 0; i < num4; i++) {
                             GDMIndividualRecord relPerson = (GDMIndividualRecord)namesakes.GetObject(i);
-                            
+
                             summary.Add("    " + HyperLink(relPerson.XRef, namesakes[i], 0));
                         }
                     }

diff --git a/projects/GKCore/GKCore/ScriptEngine.cs b/projects/GKCore/GKCore/ScriptEngine.cs
@@ -714,9 +714,11 @@ public bool csv_load(string fileName, bool hasHeader)
             if (!File.Exists(fileName)) return result;
 
             try {
-                fCSVData = CSVReader.ReadCSVFile(fileName, Encoding.Unicode, hasHeader);
+                fCSVData = CSVReader.ReadCSVFile(fileName, GKUtils.DetectEncoding(fileName), hasHeader);
                 result = true;
-            } catch {
+            } catch (Exception e) {
+                Logger.WriteError("ScriptEngine.csv_load(" + fileName + "): fail", e);
+
                 result = false;
             }