Skip to content

Commit

Permalink
fix: ruff format
Browse files Browse the repository at this point in the history
  • Loading branch information
himkt committed Jan 12, 2024
1 parent a0c0013 commit 34f6aa4
Show file tree
Hide file tree
Showing 8 changed files with 9 additions and 8 deletions.
4 changes: 1 addition & 3 deletions konoha/api/server.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,7 @@


FORMAT = "%(message)s"
logging.basicConfig(
level="NOTSET", format=FORMAT, datefmt="[%X]", handlers=[RichHandler()]
)
logging.basicConfig(level="NOTSET", format=FORMAT, datefmt="[%X]", handlers=[RichHandler()])


def create_app() -> FastAPI:
Expand Down
2 changes: 0 additions & 2 deletions konoha/sentence_tokenizer.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@


class SentenceTokenizer:

PERIOD = "。"
PERIOD_SPECIAL = "__PERIOD__"

Expand All @@ -20,7 +19,6 @@ def __init__(
period: Optional[str] = None,
patterns: Optional[List[Pattern[str]]] = None,
) -> None:

self._period = period or self.PERIOD
self._patterns = patterns or self.PATTERNS

Expand Down
2 changes: 2 additions & 0 deletions konoha/word_tokenizers/janome_tokenizer.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,13 @@

from konoha.data.token import Token
from konoha.word_tokenizers.tokenizer import BaseTokenizer
import requests


class JanomeTokenizer(BaseTokenizer):
def __init__(self, user_dictionary_path: Optional[str] = None) -> None:
from janome.tokenizer import Tokenizer

super().__init__(name="janome")
self._tokenizer = Tokenizer(udic=user_dictionary_path)

Expand Down
1 change: 1 addition & 0 deletions konoha/word_tokenizers/kytea_tokenizer.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
class KyTeaTokenizer(BaseTokenizer):
def __init__(self, model_path: Optional[str] = None) -> None:
from Mykytea import Mykytea

super().__init__(name="kytea")

kytea_option = ""
Expand Down
1 change: 1 addition & 0 deletions konoha/word_tokenizers/mecab_tokenizer.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ def __init__(
dictionary_format: Optional[str] = None,
) -> None:
from natto import MeCab

super().__init__(name="mecab")
options = []
if isinstance(user_dictionary_path, str):
Expand Down
5 changes: 2 additions & 3 deletions konoha/word_tokenizers/nagisa_tokenizer.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,11 @@
class NagisaTokenizer(BaseTokenizer):
def __init__(self) -> None:
from nagisa import Tagger

super().__init__(name="nagisa")
self._tokenizer = Tagger()

def tokenize(self, text: str) -> List[Token]:
response = self._tokenizer.tagging(text)
tokens = [
Token(surface=surface, postag=postag) for (surface, postag) in zip(response.words, response.postags)
]
tokens = [Token(surface=surface, postag=postag) for (surface, postag) in zip(response.words, response.postags)]
return tokens
1 change: 1 addition & 0 deletions konoha/word_tokenizers/sentencepiece_tokenizer.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
class SentencepieceTokenizer(BaseTokenizer):
def __init__(self, model_path: str) -> None:
from sentencepiece import SentencePieceProcessor

super().__init__(name="sentencepiece")
self._tokenizer = SentencePieceProcessor()
self._tokenizer.load(model_path)
Expand Down
1 change: 1 addition & 0 deletions konoha/word_tokenizers/sudachi_tokenizer.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ class SudachiTokenizer(BaseTokenizer):
def __init__(self, mode: str) -> None:
from sudachipy import dictionary
from sudachipy import tokenizer

super().__init__(name="sudachi ({})".format(mode))

try:
Expand Down

0 comments on commit 34f6aa4

Please sign in to comment.