re-order features in basic normalization

snapp-incubator · Feb 12, 2025 · 1205ce3 · 1205ce3
1 parent f387ca0
commit 1205ce3
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 3 deletions.
diff --git a/internal/normalization.go b/internal/normalization.go
@@ -109,9 +109,6 @@ func (n Normalize) BasicNormalizer(input string) string {
 	s = strings.ReplaceAll(s, nullString, emptyString)
 	stringInput := strings.ToLower(s)
 
-	if n.spaceCombiner {
-		stringInput = replaceMultiSpace(stringInput)
-	}
 	if n.urlRemover {
 		stringInput = removeURLs(stringInput)
 	}
@@ -121,6 +118,9 @@ func (n Normalize) BasicNormalizer(input string) string {
 	if n.endsWithEndOfLineChar {
 		stringInput = normalizeEndsWithEndOfLineChar(stringInput)
 	}
+	if n.spaceCombiner {
+		stringInput = replaceMultiSpace(stringInput)
+	}
 	if n.outerSpaceRemover { // should be last normalization step
 		stringInput = removeOuterSpace(stringInput)
 	}

diff --git a/internal/normalization_test.go b/internal/normalization_test.go
@@ -111,6 +111,15 @@ func TestNormalize_BasicNormalizer(t *testing.T) {
 			},
 			want: "بیست یک هزار",
 		},
+		{
+			name: "Should combine space after remove special characters",
+			args: args{
+				input:              "مدرسه - ابتدایی",
+				spaceCombiner:      true,
+				removeSpecialChars: true,
+			},
+			want: "مدرسه ابتدایی",
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {