remove old files and update poetry lock

VikParuchuri · Dec 10, 2024 · f9561a7 · f9561a7
1 parent 7fe4827
commit f9561a7
Show file tree

Hide file tree

Showing 7 changed files with 719 additions and 883 deletions.
diff --git a/benchmark.py b/benchmark.py
@@ -13,10 +13,8 @@
 from rapidfuzz import fuzz
 import tabulate
 from tqdm import tqdm
-import pypdfium2 as pdfium
 
 from pdftext.extraction import paginated_plain_text_output
-from pdftext.model import get_model
 from pdftext.settings import settings
 
 
@@ -94,6 +92,8 @@ def main():
                 pages = inference_func(pdf_path)
                 times[tool].append(time.time() - start)
                 tool_pages[tool] = pages
+                open(f"{row['__key__']}.{tool}.md", "w").write("\n".join(pages))
+                open(f"{row['__key__']}.pdf", "wb").write(pdf)
 
             for tool in alignment_tools:
                 alignments[tool].append(

diff --git a/models/dt.joblib b/models/dt.joblib
diff --git a/models/dt.onnx b/models/dt.onnx
diff --git a/pdftext/model.py b/pdftext/model.py
diff --git a/pdftext/settings.py b/pdftext/settings.py
@@ -4,9 +4,6 @@
 
 
 class Settings(BaseSettings):
-    BASE_PATH: str = os.path.dirname(os.path.dirname(__file__))
-    MODEL_PATH: str = os.path.join(BASE_PATH, "models", "dt.onnx")
-
     # Inference
     BLOCK_THRESHOLD: float = 0.8  # Confidence threshold for block detection
     WORKER_PAGE_THRESHOLD: int = 10  # Min number of pages per worker in parallel

diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -11,17 +11,14 @@ packages = [
     {include = "pdftext"}
 ]
 include = [
-    "extract_text.py",
-    "models/dt.joblib",
-    "models/dt.onnx"
+    "extract_text.py"
 ]
 
 [tool.poetry.dependencies]
 python = "^3.10"
 pypdfium2 = "^4.29.0"
 pydantic = "^2.7.1"
 pydantic-settings = "^2.2.1"
-onnxruntime = "^1.19.2"
 
 [tool.poetry.group.dev.dependencies]
 pymupdf = "^1.24.2"