Skip to content

Commit

Permalink
🎨 refactor(ft_detect): prepend module name to log messages
Browse files Browse the repository at this point in the history
🔥 remove(feature_test): delete redundant test case for strict mode

🐛 fix(ft_detect): raise original exception as cause in strict mode

🎨 style(feature_test): adjust test case structure for clarity
  • Loading branch information
sudoskys committed Jan 9, 2025
1 parent 70500aa commit 2de37bb
Show file tree
Hide file tree
Showing 2 changed files with 14 additions and 12 deletions.
8 changes: 5 additions & 3 deletions feature_test/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,6 @@

# 测试繁体,简体,日文,英文,韩文,法文,德文,西班牙文
print(detect_multilingual("Hello, world!你好世界!Привет, мир!", low_memory=False))
print(
detect_multilingual("Hello, world!你好世界!Привет, мир!", low_memory=True, use_strict_mode=True)
)
# [{'lang': 'ja', 'score': 0.32009604573249817}, {'lang': 'uk', 'score': 0.27781224250793457}, {'lang': 'zh', 'score': 0.17542070150375366}, {'lang': 'sr', 'score': 0.08751443773508072}, {'lang': 'bg', 'score': 0.05222449079155922}]
print(detect("hello world"))
print(detect("你好世界"))
Expand All @@ -19,3 +16,8 @@
print(detect_language("Hallo Welt"))
print(detect_language("Hola mundo"))
print(detect_language("這些機構主辦的課程,多以基本電腦使用為主,例如文書處理、中文輸入、互聯網應用等"))

# When offline, its raise error
print(
detect_multilingual("Hello, world!你好世界!Привет, мир!", low_memory=False, use_strict_mode=True)
)
18 changes: 9 additions & 9 deletions src/fast_langdetect/ft_detect/infer.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,10 +52,10 @@ def download_model(
:raises DetectError: If download fails
"""
if save_path.exists():
logger.info(f"Model already exists at {save_path}. Skipping download.")
logger.info(f"fast-langdetect:Model already exists at {save_path}. Skipping download.")
return

logger.info(f"Downloading FastText model from {download_url} to {save_path}")
logger.info(f"fast-langdetect:Downloading FastText model from {download_url} to {save_path}")
try:
download(
url=download_url,
Expand All @@ -66,7 +66,7 @@ def download_model(
timeout=30,
)
except Exception as e:
logger.error(f"Failed to download FastText model from {download_url}: {e}")
logger.error(f"fast-langdetect:Failed to download FastText model from {download_url}: {e}")
raise DetectError(f"Unable to download model from {download_url}")


Expand Down Expand Up @@ -94,7 +94,7 @@ def load_fasttext_model(
# Load FastText model
return fasttext.load_model(str(model_path))
except Exception as e:
logger.error(f"Failed to load FastText model from {model_path}: {e}")
logger.error(f"fast-langdetect:Failed to load FastText model from {model_path}: {e}")
raise DetectError(f"Failed to load FastText model: {e}")


Expand Down Expand Up @@ -131,13 +131,13 @@ def load_model(
_model_cache.cache_model(cache_key, model)
return model
except Exception as e:
logger.error(f"Failed to load model ({'low' if low_memory else 'high'} memory): {e}")
logger.error(f"fast-langdetect:Failed to load model ({'low' if low_memory else 'high'} memory): {e}")
if use_strict_mode:
raise DetectError("Failed to load FastText model.")
raise DetectError("Failed to load FastText model.") from e
elif not low_memory:
logger.info("Falling back to low-memory model...")
return load_model(low_memory=True, use_strict_mode=True)
raise
raise e


def detect(
Expand Down Expand Up @@ -169,7 +169,7 @@ def detect(
confidence_score = min(float(scores[0]), 1.0)
return {"lang": language_label, "score": confidence_score}
except Exception as e:
logger.error(f"Error during language detection: {e}")
logger.error(f"fast-langdetect:Error during language detection: {e}")
raise DetectError("Language detection failed.")


Expand Down Expand Up @@ -199,5 +199,5 @@ def detect_multilingual(
]
return sorted(results, key=lambda x: x["score"], reverse=True)
except Exception as e:
logger.error(f"Error during multilingual detection: {e}")
logger.error(f"fast-langdetect:Error during multilingual detection: {e}")
raise DetectError("Multilingual detection failed.")

0 comments on commit 2de37bb

Please sign in to comment.