Merge pull request #63 from aarondandy/release-4

Release 4.0

.github/workflows/ci.yml

@@ -2,9 +2,13 @@ name: CI
 
 on:
   push:
-    branches: [main]
+    branches:
+      - main
+      - release-*
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - release-*
   workflow_dispatch:
 
 env:

.gitignore

@@ -2,6 +2,8 @@
 ## files generated by popular Visual Studio add-ons.
 
 test/WeCantSpell.Hunspell.Benchmarking.MicroSuites/BenchmarkDotNet.Artifacts/**/*
+temp-data/
+
 
 # User-specific files
 *.suo

TestHarness/CheckTest.cs

@@ -0,0 +1,38 @@
+using System.Text;
+
+namespace WeCantSpell.Hunspell.TestHarness;
+
+public class CheckTest
+{
+    public static void Run(string dicFilePath, string wordFilePath)
+    {
+        var wordList = WordListReader.ReadFile(dicFilePath);
+        var checkWords = new List<string>();
+
+        using var reader = new StreamReader(new FileStream(wordFilePath, FileMode.Open, FileAccess.Read, FileShare.Read), Encoding.UTF8, true);
+
+        var lineSplitChars = " \t,".ToCharArray();
+        string? line;
+        while ((line = reader.ReadLine()?.Trim()) is not null)
+        {
+            if (line.Length == 0 || line.StartsWith('#') || line.StartsWith('['))
+            {
+                continue;
+            }
+
+            checkWords.AddRange(line.Split(lineSplitChars, StringSplitOptions.RemoveEmptyEntries));
+        }
+
+        const int numLoops = 500;
+
+        Console.WriteLine($"Checking {checkWords.Count} words {numLoops} times");
+
+        for (var i = 0; i < numLoops; i++)
+        {
+            foreach (var word in checkWords)
+            {
+                _ = wordList.CheckDetails(word);
+            }
+        }
+    }
+}

TestHarness/LoadTest.cs

@@ -0,0 +1,21 @@
+namespace WeCantSpell.Hunspell.TestHarness;
+
+public class LoadTest
+{
+    public static void LoadDictionary(string filePath)
+    {
+        var wordList = WordListReader.ReadFile(filePath);
+        Console.WriteLine($"Loaded {wordList.RootWords.Count()} roots");
+    }
+
+    public static void LoadAllDictionaries(string path)
+    {
+        var paths = Directory.GetFiles(path, "*.dic");
+
+        foreach (var filePath in paths)
+        {
+            var wordList = WordListReader.ReadFile(filePath);
+            Console.WriteLine($"Loaded {wordList.RootWords.Count()} roots from {filePath}");
+        }
+    }
+}

TestHarness/Program.cs

@@ -0,0 +1,13 @@
+using Cocona;
+
+using WeCantSpell.Hunspell.TestHarness;
+
+var app = CoconaApp.Create();
+
+app.AddCommand("load", (string file) => LoadTest.LoadDictionary(file));
+app.AddCommand("load-all", (string path) => LoadTest.LoadAllDictionaries(path));
+app.AddCommand("check", (string dicFile, string wordFile) => CheckTest.Run(dicFile, wordFile));
+app.AddCommand("suggest", (string dicFile, string wordFile) => SuggestTest.Run(dicFile, wordFile));
+app.AddCommand("suggest-word", (string dicFile, string word) => SuggestWordTest.Run(dicFile, word));
+
+app.Run();

TestHarness/Properties/launchSettings.json

@@ -0,0 +1,32 @@
+{
+  "profiles": {
+    "TestHarness": {
+      "commandName": "Project"
+    },
+    "Load en-US": {
+      "commandName": "Project",
+      "commandLineArgs": "load --file \"../samples/English (American).dic\"",
+      "workingDirectory": "./"
+    },
+    "Load All": {
+      "commandName": "Project",
+      "commandLineArgs": "load-all --path \"../samples/\"",
+      "workingDirectory": "./"
+    },
+    "Suggest en-US": {
+      "commandName": "Project",
+      "commandLineArgs": "suggest --dic-file \"../samples/English (American).dic\" --word-file \"../hunspell-origin/tests/suggestiontest/List_of_common_misspellings.txt\"",
+      "workingDirectory": "./"
+    },
+    "Check en-US": {
+      "commandName": "Project",
+      "commandLineArgs": "check --dic-file \"../samples/English (American).dic\" --word-file \"../hunspell-origin/tests/suggestiontest/List_of_common_misspellings.txt\"",
+      "workingDirectory": "./"
+    },
+    "Issue #40": {
+      "commandName": "Project",
+      "commandLineArgs": "suggest-word --dic-file \"../temp-data/fr-toutesvariantes.dic\" --word Systemes",
+      "workingDirectory": "./"
+    }
+  }
+}

TestHarness/SuggestTest.cs

@@ -0,0 +1,40 @@
+using System.Text;
+
+namespace WeCantSpell.Hunspell.TestHarness;
+
+public class SuggestTest
+{
+    public static void Run(string dicFilePath, string wordFilePath)
+    {
+        var wordList = WordListReader.ReadFile(dicFilePath);
+        var checkWords = new List<string>();
+
+        using var reader = new StreamReader(new FileStream(wordFilePath, FileMode.Open, FileAccess.Read, FileShare.Read), Encoding.UTF8, true);
+
+        var lineSplitChars = " \t,".ToCharArray();
+        string? line;
+        while ((line = reader.ReadLine()?.Trim()) is not null)
+        {
+            if (line.Length == 0 || line.StartsWith('#') || line.StartsWith('['))
+            {
+                continue;
+            }
+
+            checkWords.AddRange(line.Split(lineSplitChars, StringSplitOptions.RemoveEmptyEntries));
+        }
+
+        const int wordLimit = 2000;
+
+        if (checkWords.Count > wordLimit)
+        {
+            checkWords.RemoveRange(wordLimit, checkWords.Count - wordLimit);
+        }
+
+        Console.WriteLine($"Suggesting for {checkWords.Count} words");
+
+        foreach (var word in checkWords)
+        {
+            _ = wordList.Suggest(word);
+        }
+    }
+}

TestHarness/SuggestWordTest.cs

@@ -0,0 +1,43 @@
+namespace WeCantSpell.Hunspell.TestHarness;
+
+public class SuggestWordTest
+{
+    public static void Run(string dicFilePath, string word)
+    {
+        var wordList = WordListReader.ReadFile(dicFilePath);
+        const int wordLimit = 20;
+
+        var results = new List<int>(wordLimit);
+        var allSuggestions = new HashSet<string>();
+
+        Console.WriteLine($"Suggesting for word \"{word}\" {wordLimit} times");
+
+        var options = new QueryOptions
+        {
+            TimeLimitCompoundSuggest = TimeSpan.FromSeconds(10),
+            TimeLimitCompoundCheck = TimeSpan.FromSeconds(10),
+            TimeLimitSuggestGlobal = TimeSpan.FromSeconds(10),
+            TimeLimitSuggestStep = TimeSpan.FromSeconds(10),
+            MaxSuggestions = 10
+        };
+
+        for (var i = 0; i < wordLimit; i++)
+        {
+            var suggestions = wordList.Suggest(word, options);
+            allSuggestions.UnionWith(suggestions);
+            results.Add(suggestions.Count());
+        }
+
+        Console.WriteLine("Results:");
+        foreach (var r in results)
+        {
+            Console.WriteLine($"{r}");
+        }
+
+        Console.WriteLine("Suggestions:");
+        foreach (var s in allSuggestions)
+        {
+            Console.WriteLine(s);
+        }
+    }
+}

TestHarness/TestHarness.csproj

@@ -0,0 +1,23 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net6.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <RootNamespace>WeCantSpell.Hunspell.TestHarness</RootNamespace>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Cocona" Version="2.0.3" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <None Remove="WeCantSpell.Hunspell.TestHarness.v3.ncrunchproject" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\WeCantSpell.Hunspell\WeCantSpell.Hunspell.csproj" />
+  </ItemGroup>
+
+</Project>

TestHarness/TestHarness.v3.ncrunchproject

@@ -0,0 +1,5 @@
+<ProjectConfiguration>
+  <Settings>
+    <IgnoreThisComponentCompletely>True</IgnoreThisComponentCompletely>
+  </Settings>
+</ProjectConfiguration>

WeCantSpell.Hunspell.Benchmarking.LongRunning/AffixFileLoadPerfSpecs.cs

@@ -28,9 +28,7 @@ public void Setup(BenchmarkContext context)
         TestMode = TestMode.Measurement)]
     [MemoryMeasurement(MemoryMetric.TotalBytesAllocated)]
     [GcMeasurement(GcMetric.TotalCollections, GcGeneration.AllGc)]
-    [TimingMeasurement]
-    [CounterMeasurement(nameof(AffixFilesLoaded))]
-    [CounterThroughputAssertion(nameof(AffixFilesLoaded), MustBe.GreaterThanOrEqualTo, 5)]
+    [CounterThroughputAssertion(nameof(AffixFilesLoaded), MustBe.GreaterThanOrEqualTo, 10)]
     public void Benchmark(BenchmarkContext context)
     {
         foreach (var filePath in AffixFilePaths)

WeCantSpell.Hunspell.Benchmarking.LongRunning/DictionaryFileLoadPerfSpec.cs

@@ -38,8 +38,6 @@ public void Setup(BenchmarkContext context)
         TestMode = TestMode.Measurement)]
     [MemoryMeasurement(MemoryMetric.TotalBytesAllocated)]
     [GcMeasurement(GcMetric.TotalCollections, GcGeneration.AllGc)]
-    [TimingMeasurement]
-    [CounterMeasurement(nameof(DictionaryFilesLoaded))]
     [CounterThroughputAssertion(nameof(DictionaryFilesLoaded), MustBe.GreaterThanOrEqualTo, 2)]
     public void BenchmarkSync(BenchmarkContext context)
     {
@@ -58,8 +56,6 @@ public void BenchmarkSync(BenchmarkContext context)
         Skip = "All the perf improvements so far will impact sync")]
     [MemoryMeasurement(MemoryMetric.TotalBytesAllocated)]
     [GcMeasurement(GcMetric.TotalCollections, GcGeneration.AllGc)]
-    [TimingMeasurement]
-    [CounterMeasurement(nameof(DictionaryFilesLoaded))]
     [CounterThroughputAssertion(nameof(DictionaryFilesLoaded), MustBe.GreaterThanOrEqualTo, 2)]
     public void BenchmarkAsync(BenchmarkContext context)
     {

WeCantSpell.Hunspell.Benchmarking.LongRunning/EnWordCheckPerfSpecs.cs

@@ -20,9 +20,7 @@ public override void Setup(BenchmarkContext context)
         TestMode = TestMode.Measurement)]
     [MemoryMeasurement(MemoryMetric.TotalBytesAllocated)]
     [GcMeasurement(GcMetric.TotalCollections, GcGeneration.AllGc)]
-    [TimingMeasurement]
-    [CounterMeasurement(nameof(WordsChecked))]
-    [CounterThroughputAssertion(nameof(WordsChecked), MustBe.GreaterThanOrEqualTo, 400000)]
+    [CounterThroughputAssertion(nameof(WordsChecked), MustBe.GreaterThanOrEqualTo, 1_000_000)]
     public void Benchmark(BenchmarkContext context)
     {
         foreach (var word in Words)

WeCantSpell.Hunspell.Benchmarking.LongRunning/EnWordPerfBase.cs

@@ -36,10 +36,9 @@ void LoadChecker()
         void LoadWords()
         {
             Words = new List<string>();
-            using var stram = new FileStream(Path.Combine(filesDirectory, "List_of_common_misspellings.txt"), FileMode.Open, FileAccess.Read, FileShare.Read, 4096, true);
-            using var reader = new StreamReader(stram, Encoding.UTF8, true);
-            string line;
+            using var reader = new StreamReader(Path.Combine(filesDirectory, "List_of_common_misspellings.txt"), Encoding.UTF8, true);
 
+            string line;
             while ((line = reader.ReadLine()) is not null)
             {
                 line = line.Trim();

WeCantSpell.Hunspell.Benchmarking.LongRunning/EnWordSuggestPerfSpec.cs

@@ -17,17 +17,15 @@ public override void Setup(BenchmarkContext context)
 
     [PerfBenchmark(
         Description = "Ensure that words can be suggested quickly.",
-        NumberOfIterations = 3,
+        NumberOfIterations = 1,
         RunMode = RunMode.Throughput,
         TestMode = TestMode.Measurement)]
     [MemoryMeasurement(MemoryMetric.TotalBytesAllocated)]
     [GcMeasurement(GcMetric.TotalCollections, GcGeneration.AllGc)]
-    [TimingMeasurement]
-    [CounterMeasurement(nameof(SuggestionQueries))]
     [CounterThroughputAssertion(nameof(SuggestionQueries), MustBe.GreaterThanOrEqualTo, 30)]
     public void Benchmark(BenchmarkContext context)
     {
-        foreach (var word in Words.Take(100)) // TODO: remove the limit to allow the entire list when performance allows
+        foreach (var word in Words.Take(1000)) // TODO: remove the limit to allow the entire list when performance allows
         {
             _ = Checker.Suggest(word);
             SuggestionQueries.Increment();