From 1ce19743e1adae23442cf12b2fcf7c304d26e5cb Mon Sep 17 00:00:00 2001 From: Bardi Harborow Date: Tue, 28 Jun 2022 23:17:12 +1000 Subject: [PATCH 1/2] Add support for nested context manager use This commit adds support for nested use of the UsePrimaryDB context manager. If a new instance of the context manager was created at each level we could use instance variables to store the pinning state but the default instance is generally used directly. Instead, we store a stack of pinning values locally in the thread and pop each value as we unwind the stack. --- multidb/pinning.py | 7 ++++--- multidb/tests.py | 52 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/multidb/pinning.py b/multidb/pinning.py index aa529c2..8f554e9 100644 --- a/multidb/pinning.py +++ b/multidb/pinning.py @@ -34,7 +34,7 @@ def unpin_this_thread(): class UsePrimaryDB(object): - """A contextmanager/decorator to use the master database.""" + """A contextmanager/decorator to use the primary database.""" def __call__(self, func): @wraps(func) def decorator(*args, **kw): @@ -43,11 +43,12 @@ def decorator(*args, **kw): return decorator def __enter__(self): - _locals.old = this_thread_is_pinned() + _locals.old = getattr(_locals, 'old', []) + _locals.old.append(this_thread_is_pinned()) pin_this_thread() def __exit__(self, type, value, tb): - if not _locals.old: + if not _locals.old.pop(): unpin_this_thread() diff --git a/multidb/tests.py b/multidb/tests.py index 125cff4..136e63a 100644 --- a/multidb/tests.py +++ b/multidb/tests.py @@ -202,6 +202,22 @@ def check(): check() assert not this_thread_is_pinned() + def test_decorator_nested(self): + @use_primary_db + def check_inner(): + assert this_thread_is_pinned() + + @use_primary_db + def check_outer(): + assert this_thread_is_pinned() + check_inner() + assert this_thread_is_pinned() + + unpin_this_thread() + assert not this_thread_is_pinned() + check_outer() + assert not this_thread_is_pinned() + def test_decorator_resets(self): @use_primary_db def check(): @@ -211,6 +227,22 @@ def check(): check() assert this_thread_is_pinned() + def test_decorator_resets_nested(self): + @use_primary_db + def check_inner(): + assert this_thread_is_pinned() + + @use_primary_db + def check_outer(): + assert this_thread_is_pinned() + check_inner() + assert this_thread_is_pinned() + + pin_this_thread() + assert this_thread_is_pinned() + check_outer() + assert this_thread_is_pinned() + def test_context_manager(self): unpin_this_thread() assert not this_thread_is_pinned() @@ -218,6 +250,16 @@ def test_context_manager(self): assert this_thread_is_pinned() assert not this_thread_is_pinned() + def test_context_manager_nested(self): + unpin_this_thread() + assert not this_thread_is_pinned() + with use_primary_db: + assert this_thread_is_pinned() + with use_primary_db: + assert this_thread_is_pinned() + assert this_thread_is_pinned() + assert not this_thread_is_pinned() + def test_context_manager_resets(self): pin_this_thread() assert this_thread_is_pinned() @@ -225,6 +267,16 @@ def test_context_manager_resets(self): assert this_thread_is_pinned() assert this_thread_is_pinned() + def test_context_manager_resets_nested(self): + pin_this_thread() + assert this_thread_is_pinned() + with use_primary_db: + assert this_thread_is_pinned() + with use_primary_db: + assert this_thread_is_pinned() + assert this_thread_is_pinned() + assert this_thread_is_pinned() + def test_context_manager_exception(self): unpin_this_thread() assert not this_thread_is_pinned() From b40d9aa36018eb840e21a7c5e360ff9d389090bd Mon Sep 17 00:00:00 2001 From: Bardi Harborow Date: Tue, 28 Jun 2022 23:55:05 +1000 Subject: [PATCH 2/2] Add context manager for unpinning This commit adds a UseSecondaryDB context manager which unpins within a block. --- multidb/pinning.py | 27 ++++- multidb/tests.py | 250 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 271 insertions(+), 6 deletions(-) diff --git a/multidb/pinning.py b/multidb/pinning.py index 8f554e9..ed2412d 100644 --- a/multidb/pinning.py +++ b/multidb/pinning.py @@ -7,7 +7,7 @@ __all__ = ['this_thread_is_pinned', 'pin_this_thread', 'unpin_this_thread', - 'use_primary_db', 'use_master', 'db_write'] + 'use_primary_db', 'use_secondary_db', 'use_master', 'db_write'] _locals = threading.local() @@ -33,8 +33,8 @@ def unpin_this_thread(): _locals.pinned = False -class UsePrimaryDB(object): - """A contextmanager/decorator to use the primary database.""" +class _UseDB(object): + """A contextmanager/decorator to use the specified database.""" def __call__(self, func): @wraps(func) def decorator(*args, **kw): @@ -45,13 +45,29 @@ def decorator(*args, **kw): def __enter__(self): _locals.old = getattr(_locals, 'old', []) _locals.old.append(this_thread_is_pinned()) - pin_this_thread() def __exit__(self, type, value, tb): - if not _locals.old.pop(): + previous_state = _locals.old.pop() + if previous_state: + pin_this_thread() + else: unpin_this_thread() +class UsePrimaryDB(_UseDB): + """A contextmanager/decorator to use the primary database.""" + def __enter__(self): + super(UsePrimaryDB, self).__enter__() + pin_this_thread() + + +class UseSecondaryDB(_UseDB): + """A contextmanager/decorator to use the secondary database.""" + def __enter__(self): + super(UseSecondaryDB, self).__enter__() + unpin_this_thread() + + class DeprecatedUseMaster(UsePrimaryDB): def __enter__(self): warnings.warn( @@ -63,6 +79,7 @@ def __enter__(self): use_primary_db = UsePrimaryDB() +use_secondary_db = UseSecondaryDB() use_master = DeprecatedUseMaster() diff --git a/multidb/tests.py b/multidb/tests.py index 136e63a..36c1e09 100644 --- a/multidb/tests.py +++ b/multidb/tests.py @@ -20,7 +20,8 @@ pinning_cookie_samesite, pinning_cookie_secure, pinning_seconds, PinningRouterMiddleware) from multidb.pinning import (this_thread_is_pinned, pin_this_thread, - unpin_this_thread, use_primary_db, db_write) + unpin_this_thread, use_primary_db, + use_secondary_db, db_write) class UnpinningTestCase(TestCase): @@ -218,6 +219,22 @@ def check_outer(): check_outer() assert not this_thread_is_pinned() + def test_decorator_nested_mixed(self): + @use_primary_db + def check_inner(): + assert this_thread_is_pinned() + + @use_secondary_db + def check_outer(): + assert not this_thread_is_pinned() + check_inner() + assert not this_thread_is_pinned() + + unpin_this_thread() + assert not this_thread_is_pinned() + check_outer() + assert not this_thread_is_pinned() + def test_decorator_resets(self): @use_primary_db def check(): @@ -243,6 +260,22 @@ def check_outer(): check_outer() assert this_thread_is_pinned() + def test_decorator_resets_nested_mixed(self): + @use_primary_db + def check_inner(): + assert this_thread_is_pinned() + + @use_secondary_db + def check_outer(): + assert not this_thread_is_pinned() + check_inner() + assert not this_thread_is_pinned() + + pin_this_thread() + assert this_thread_is_pinned() + check_outer() + assert this_thread_is_pinned() + def test_context_manager(self): unpin_this_thread() assert not this_thread_is_pinned() @@ -260,6 +293,16 @@ def test_context_manager_nested(self): assert this_thread_is_pinned() assert not this_thread_is_pinned() + def test_context_manager_nested_mixed(self): + unpin_this_thread() + assert not this_thread_is_pinned() + with use_secondary_db: + assert not this_thread_is_pinned() + with use_primary_db: + assert this_thread_is_pinned() + assert not this_thread_is_pinned() + assert not this_thread_is_pinned() + def test_context_manager_resets(self): pin_this_thread() assert this_thread_is_pinned() @@ -277,6 +320,16 @@ def test_context_manager_resets_nested(self): assert this_thread_is_pinned() assert this_thread_is_pinned() + def test_context_manager_resets_nested_mixed(self): + pin_this_thread() + assert this_thread_is_pinned() + with use_secondary_db: + assert not this_thread_is_pinned() + with use_primary_db: + assert this_thread_is_pinned() + assert not this_thread_is_pinned() + assert this_thread_is_pinned() + def test_context_manager_exception(self): unpin_this_thread() assert not this_thread_is_pinned() @@ -334,6 +387,201 @@ def thread2_worker(): self.assertEqual(pinned[1], False) +class UseSecondaryDBTests(TestCase): + def test_decorator(self): + @use_secondary_db + def check(): + assert not this_thread_is_pinned() + pin_this_thread() + assert this_thread_is_pinned() + check() + assert this_thread_is_pinned() + + def test_decorator_nested(self): + @use_secondary_db + def check_inner(): + assert not this_thread_is_pinned() + + @use_secondary_db + def check_outer(): + assert not this_thread_is_pinned() + check_inner() + assert not this_thread_is_pinned() + + pin_this_thread() + assert this_thread_is_pinned() + check_outer() + assert this_thread_is_pinned() + + def test_decorator_nested_mixed(self): + @use_secondary_db + def check_inner(): + assert not this_thread_is_pinned() + + @use_primary_db + def check_outer(): + assert this_thread_is_pinned() + check_inner() + assert this_thread_is_pinned() + + pin_this_thread() + assert this_thread_is_pinned() + check_outer() + assert this_thread_is_pinned() + + def test_decorator_resets(self): + @use_secondary_db + def check(): + assert not this_thread_is_pinned() + unpin_this_thread() + assert not this_thread_is_pinned() + check() + assert not this_thread_is_pinned() + + def test_decorator_resets_nested(self): + @use_secondary_db + def check_inner(): + assert not this_thread_is_pinned() + + @use_secondary_db + def check_outer(): + assert not this_thread_is_pinned() + check_inner() + assert not this_thread_is_pinned() + + unpin_this_thread() + assert not this_thread_is_pinned() + check_outer() + assert not this_thread_is_pinned() + + def test_decorator_resets_nested_mixed(self): + @use_secondary_db + def check_inner(): + assert not this_thread_is_pinned() + + @use_primary_db + def check_outer(): + assert this_thread_is_pinned() + check_inner() + assert this_thread_is_pinned() + + unpin_this_thread() + assert not this_thread_is_pinned() + check_outer() + assert not this_thread_is_pinned() + + def test_context_manager(self): + pin_this_thread() + assert this_thread_is_pinned() + with use_secondary_db: + assert not this_thread_is_pinned() + assert this_thread_is_pinned() + + def test_context_manager_nested(self): + pin_this_thread() + assert this_thread_is_pinned() + with use_secondary_db: + assert not this_thread_is_pinned() + with use_secondary_db: + assert not this_thread_is_pinned() + assert not this_thread_is_pinned() + assert this_thread_is_pinned() + + def test_context_manager_nested_mixed(self): + pin_this_thread() + assert this_thread_is_pinned() + with use_primary_db: + assert this_thread_is_pinned() + with use_secondary_db: + assert not this_thread_is_pinned() + assert this_thread_is_pinned() + assert this_thread_is_pinned() + + def test_context_manager_resets(self): + unpin_this_thread() + assert not this_thread_is_pinned() + with use_secondary_db: + assert not this_thread_is_pinned() + assert not this_thread_is_pinned() + + def test_context_manager_resets_nested(self): + unpin_this_thread() + assert not this_thread_is_pinned() + with use_secondary_db: + assert not this_thread_is_pinned() + with use_secondary_db: + assert not this_thread_is_pinned() + assert not this_thread_is_pinned() + assert not this_thread_is_pinned() + + def test_context_manager_resets_nested_mixed(self): + unpin_this_thread() + assert not this_thread_is_pinned() + with use_primary_db: + assert this_thread_is_pinned() + with use_secondary_db: + assert not this_thread_is_pinned() + assert this_thread_is_pinned() + assert not this_thread_is_pinned() + + def test_context_manager_exception(self): + pin_this_thread() + assert this_thread_is_pinned() + with self.assertRaises(ValueError): + with use_secondary_db: + assert not this_thread_is_pinned() + raise ValueError + assert this_thread_is_pinned() + + def test_multithreaded_unpinning(self): + thread1_lock = Lock() + thread2_lock = Lock() + thread1_lock.acquire() + thread2_lock.acquire() + orchestrator = Lock() + orchestrator.acquire() + + pinned = {} + + def thread1_worker(): + pin_this_thread() + with use_secondary_db: + orchestrator.release() + thread1_lock.acquire() + + pinned[1] = this_thread_is_pinned() + + def thread2_worker(): + unpin_this_thread() + with use_secondary_db: + orchestrator.release() + thread2_lock.acquire() + + pinned[2] = this_thread_is_pinned() + orchestrator.release() + + thread1 = Thread(target=thread1_worker) + thread2 = Thread(target=thread2_worker) + + # thread1 starts, entering `use_primary_db` from a pinned state + thread1.start() + orchestrator.acquire() + + # thread2 starts, entering `use_primary_db` from an unpinned state + thread2.start() + orchestrator.acquire() + + # thread2 finishes, returning to an unpinned state + thread2_lock.release() + thread2.join() + self.assertEqual(pinned[2], False) + + # thread1 finishes, returning to a pinned state + thread1_lock.release() + thread1.join() + self.assertEqual(pinned[1], True) + + class DeprecationTestCase(TestCase): def test_masterslaverouter(self): with warnings.catch_warnings(record=True) as w: