From 870ed7a86f05a136bc77b4bbba2b27c447b65242 Mon Sep 17 00:00:00 2001 From: Vinyzu <50874994+Vinyzu@users.noreply.github.com> Date: Sat, 9 Dec 2023 17:55:09 +0100 Subject: [PATCH] Appendix | Update v1.1 - Added Proper Sync Threaded Loading - Improved Code Style of Async Wait Blocking - Added Tests to check proper threading --- chrome_fingerprints/fingerprints.py | 21 +++++++++++++++------ tests/test_async_fingerprints.py | 17 +++++++++++++++++ tests/test_fingerprints.py | 16 ++++++++++++++++ 3 files changed, 48 insertions(+), 6 deletions(-) diff --git a/chrome_fingerprints/fingerprints.py b/chrome_fingerprints/fingerprints.py index 25c5b24..182951d 100644 --- a/chrome_fingerprints/fingerprints.py +++ b/chrome_fingerprints/fingerprints.py @@ -62,8 +62,13 @@ def __init__(self) -> None: self.fingerprints: Optional[List[Dict[str, Any]]] = None # Preloading: Fingerprint Decompression & Json Loading takes ~2 seconds - with ThreadPoolExecutor(max_workers=1) as executor: - self.fingerprint_loading_feature = executor.submit(self.initialize_fingerprints) + self.executor = ThreadPoolExecutor(max_workers=1) + + try: + self.fingerprint_loading_feature = self.executor.submit(self.initialize_fingerprints) + except Exception as e: + self.executor.shutdown(wait=True, cancel_futures=True) + raise e def initialize_fingerprints(self) -> None: dir_path = os.path.dirname(os.path.realpath(__file__)) @@ -74,10 +79,14 @@ def initialize_fingerprints(self) -> None: self.fingerprints = orjson.loads(json_data) def get_fingerprint(self, fingerprint_index: Optional[int] = None) -> ChromeFingerprint: - if not self.fingerprint_loading_feature.done(): - self.fingerprint_loading_feature.result(timeout=10) + try: + if not self.fingerprint_loading_feature.done(): + self.fingerprint_loading_feature.result(timeout=10) - assert self.fingerprints + assert self.fingerprints + except Exception as e: + self.executor.shutdown(wait=True, cancel_futures=True) + raise e if fingerprint_index: fingerprint = self.fingerprints[fingerprint_index] @@ -109,7 +118,7 @@ async def initialize_fingerprints(self) -> None: async def get_fingerprint(self, fingerprint_index: Optional[int] = None) -> ChromeFingerprint: if not self.fingerprint_loading_feature.done(): - await asyncio.wait([self.fingerprint_loading_feature]) + await self.fingerprint_loading_feature assert self.fingerprints diff --git a/tests/test_async_fingerprints.py b/tests/test_async_fingerprints.py index 30fa45e..148fbb0 100644 --- a/tests/test_async_fingerprints.py +++ b/tests/test_async_fingerprints.py @@ -1,4 +1,5 @@ import pytest +import time from chrome_fingerprints import AsyncFingerprintGenerator, ChromeFingerprint @@ -20,3 +21,19 @@ async def test_all_fingerprints(async_fingerprint_generator: AsyncFingerprintGen for index in range(10000): chrome_fp = await async_fingerprint_generator.get_fingerprint(fingerprint_index=index) assert isinstance(chrome_fp, ChromeFingerprint) + + +@pytest.mark.asyncio +async def test_threaded_loading(): + start_time = time.time() + # Load the FingerprintGenerator + fingerprint_generator = AsyncFingerprintGenerator() + init_time = time.time() - start_time + # Load a Fingerprint + await fingerprint_generator.get_fingerprint() + load_time = time.time() - start_time + + # Check if the Fingerprint Load Time is bigger than the Fingerprint Init Time + # Because the DataLoading is threaded, the Loading should Not Block until a fingerprint is requested + # Therefore, we can check the threaded loading by checking the Load Times. + assert load_time > init_time diff --git a/tests/test_fingerprints.py b/tests/test_fingerprints.py index 5977b19..75f710f 100644 --- a/tests/test_fingerprints.py +++ b/tests/test_fingerprints.py @@ -1,3 +1,4 @@ +import time from chrome_fingerprints import FingerprintGenerator, ChromeFingerprint @@ -15,3 +16,18 @@ def test_all_fingerprints(fingerprint_generator: FingerprintGenerator): for index in range(10000): chrome_fp = fingerprint_generator.get_fingerprint(fingerprint_index=index) assert isinstance(chrome_fp, ChromeFingerprint) + + +def test_threaded_loading(): + start_time = time.time() + # Load the FingerprintGenerator + fingerprint_generator = FingerprintGenerator() + init_time = time.time() - start_time + # Load a Fingerprint + fingerprint_generator.get_fingerprint() + load_time = time.time() - start_time + + # Check if the Fingerprint Load Time is bigger than the Fingerprint Init Time + # Because the DataLoading is threaded, the Loading should Not Block until a fingerprint is requested + # Therefore, we can check the threaded loading by checking the Load Times. + assert load_time > init_time