Merge pull request #15 from Mdperez19/LissL/FirstUnitTests

LissL/FirstUnitTests

Author: Aliss55

pytest.ini

@@ -0,0 +1,6 @@
+[pytest]
+markers=
+    SentenceTokenizerNlkt
+    WordTokenizerNlkt
+    NormalizerPython
+testpaths = tests

requirements.txt

@@ -3,5 +3,7 @@
 
 class Normalizer(metaclass=ABCMeta):
     @abstractmethod
-    def normalize_sentences_tokens(self, sentences_tokens: list) -> list:
+    def normalize_sentences_tokens(self,
+                                   sentences_tokens: list[list[str]]
+                                   ) -> list[list[str]]:
         pass

spellchecker/natural_language_processing_tools/text_preprocessing/normalizer/Normalizer.py

@@ -5,7 +5,9 @@ class NormalizerPython(Normalizer):
     def __init__(self):
         pass
 
-    def normalize_sentences_tokens(self, sentences_tokens: list) -> list:
+    def normalize_sentences_tokens(self,
+                                   sentences_tokens: list[list[str]]
+                                   ) -> list[list[str]]:
         normalized_sentences_tokens = []
         for tokens_by_sentence in sentences_tokens:
             normalized_tokens_by_sentence = []

spellchecker/natural_language_processing_tools/text_preprocessing/normalizer/NormalizerPython.py

@@ -0,0 +1,28 @@
+import pytest
+
+from spellchecker.natural_language_processing_tools.text_preprocessing.normalizer.NormalizerPython import NormalizerPython
+
+class TestNormalizerPython:
+
+    @pytest.fixture
+    def normalizer(self):
+        return NormalizerPython()
+
+    @pytest.mark.NormalizerPython
+    def test_normalize_sentences_tokens(self,normalizer):
+        #Given
+        tokens = [["¿Cómo", "estás&?"],["Bien", "gracias"]]
+        expected_normalized_tokens = [["cómo","estás"],["bien","gracias"]]
+        #When
+        normalized_tokens = normalizer.normalize_sentences_tokens(tokens)
+        #Then
+        assert normalized_tokens == expected_normalized_tokens
+    @pytest.mark.NormalizerPython
+    def test_build_normalized_token(self, normalizer):
+        #Given
+        token="Hola%_)*+áéÍÓú"
+        expected_normalized_token="holaáéíóú"
+        #When
+        normalized_token = normalizer.build_normalized_token(token)
+        #Then
+        assert normalized_token == expected_normalized_token

tests/natural_language_provessing_tools/text_preprocessing/normalizer/test_NormalizerPython.py

@@ -0,0 +1,22 @@
+import pytest
+
+from spellchecker.natural_language_processing_tools.text_preprocessing.tokenizer.sentence.SentenceTokenizerNltk import \
+    SentenceTokenizerNltk
+
+class TestSentenceTokenizerNlkt:
+
+    @pytest.fixture
+    def sentence_tokenizer(self):
+        return SentenceTokenizerNltk()
+
+    @pytest.mark.SentenceTokenizerNlkt
+    def test_tokenize_text_by_sentence(self, sentence_tokenizer):
+        #Given
+        text = "Esto es una oración de prueba. Esta es otra oración de prueba."
+        expected_tokens = ["Esto es una oración de prueba.", "Esta es otra oración de prueba."]
+        #When
+        tokens = sentence_tokenizer.tokenize_text_by_sentence(text)
+        #Then
+        assert len(tokens) == 2
+        assert isinstance(tokens, list)
+        assert tokens == expected_tokens

tests/natural_language_provessing_tools/text_preprocessing/tokenizer/sentence/test_SentenceTokenizerNltk.py

@@ -0,0 +1,19 @@
+import pytest
+
+from spellchecker.natural_language_processing_tools.text_preprocessing.tokenizer.word.WordTokenizerNltk import \
+    WordTokenizerNltk
+
+
+class TestWordTokenizerNlkt:
+
+    @pytest.fixture
+    def word_tokenizer(self):
+        return WordTokenizerNltk()
+
+    @pytest.mark.WordTokenizerNlkt
+    def test_tokenize_sentence_by_words(self, word_tokenizer):
+        sentence = ["Esto es una oración de prueba que debe mostrar 10 tokens",
+                    "Esta es otra oración que tendrá 8 tokens"]
+        tokens = word_tokenizer.tokenize_sentence_by_words(sentence)
+        assert len(tokens) == 2
+        assert isinstance(tokens, list)