From 254b0d3db846225e2a8bb64d2a588f5067f17c9b Mon Sep 17 00:00:00 2001 From: Thomas Heartman Date: Thu, 7 Mar 2024 13:00:00 +0100 Subject: [PATCH] refactor: cut test execution time (#305) This PR decreases test execution time for the test_client test suite by about 2 whole minutes (132 seconds, I think). It does that by extracting the `aggregate_metrics` function and, instead of waiting for the client to actually send metrics, just checking what the metrics would be aggregated to. In doing so, it might be worth adding another test to ensure that metrics **are** actually sent, but we may or may not already have this elsewhere. ## Type of change Please delete options that are not relevant. - [x] Refactor of tests # How Has This Been Tested? Please describe the tests that you ran to verify your changes. - [x] Unit tests # Checklist: - [x] My code follows the style guidelines of this project - [x] I have performed a self-review of my own code - [ ] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [ ] New and existing unit tests pass locally with my changes - [x] Any dependent changes have been merged and published in downstream modules --- UnleashClient/periodic_tasks/__init__.py | 2 +- UnleashClient/periodic_tasks/send_metrics.py | 34 +++++++---- tests/unit_tests/test_client.py | 62 ++++++++------------ tests/utilities/testing_constants.py | 4 +- 4 files changed, 50 insertions(+), 52 deletions(-) diff --git a/UnleashClient/periodic_tasks/__init__.py b/UnleashClient/periodic_tasks/__init__.py index a4391c50..13b71529 100644 --- a/UnleashClient/periodic_tasks/__init__.py +++ b/UnleashClient/periodic_tasks/__init__.py @@ -1,3 +1,3 @@ # ruff: noqa: F401 from .fetch_and_load import fetch_and_load_features -from .send_metrics import aggregate_and_send_metrics +from .send_metrics import aggregate_and_send_metrics, aggregate_metrics diff --git a/UnleashClient/periodic_tasks/send_metrics.py b/UnleashClient/periodic_tasks/send_metrics.py index 3df7ec7d..b1634e79 100644 --- a/UnleashClient/periodic_tasks/send_metrics.py +++ b/UnleashClient/periodic_tasks/send_metrics.py @@ -7,16 +7,9 @@ from UnleashClient.utils import LOGGER -def aggregate_and_send_metrics( - url: str, - app_name: str, - instance_id: str, - custom_headers: dict, - custom_options: dict, +def aggregate_metrics( features: dict, - cache: BaseCache, - request_timeout: int, -) -> None: +) -> dict: feature_stats_list = [] for feature_name in features.keys(): @@ -31,20 +24,37 @@ def aggregate_and_send_metrics( } } - features[feature_name].reset_stats() feature_stats_list.append(feature_stats) + return dict(ChainMap(*feature_stats_list)) + + +def aggregate_and_send_metrics( + url: str, + app_name: str, + instance_id: str, + custom_headers: dict, + custom_options: dict, + features: dict, + cache: BaseCache, + request_timeout: int, +) -> None: + feature_stats_dict = aggregate_metrics(features) + + for feature_name in features.keys(): + features[feature_name].reset_stats() + metrics_request = { "appName": app_name, "instanceId": instance_id, "bucket": { "start": cache.get(METRIC_LAST_SENT_TIME).isoformat(), "stop": datetime.now(timezone.utc).isoformat(), - "toggles": dict(ChainMap(*feature_stats_list)), + "toggles": feature_stats_dict, }, } - if feature_stats_list: + if feature_stats_dict: send_metrics( url, metrics_request, custom_headers, custom_options, request_timeout ) diff --git a/tests/unit_tests/test_client.py b/tests/unit_tests/test_client.py index e0a78d67..50a7b981 100644 --- a/tests/unit_tests/test_client.py +++ b/tests/unit_tests/test_client.py @@ -1,4 +1,3 @@ -import json import time import warnings from pathlib import Path @@ -39,6 +38,7 @@ from UnleashClient.cache import FileCache from UnleashClient.constants import FEATURES_URL, METRICS_URL, REGISTER_URL from UnleashClient.events import UnleashEvent, UnleashEventType +from UnleashClient.periodic_tasks import aggregate_metrics from UnleashClient.strategies import Strategy from UnleashClient.utils import InstanceAllowType @@ -210,7 +210,7 @@ def test_uc_lifecycle(unleash_client): status=304, headers={"etag": ETAG_VALUE}, ) - time.sleep(16) + time.sleep(REFRESH_INTERVAL + 1) # Simulate server provisioning change responses.add( @@ -220,7 +220,7 @@ def test_uc_lifecycle(unleash_client): status=200, headers={"etag": "W/somethingelse"}, ) - time.sleep(30) + time.sleep(REFRESH_INTERVAL * 2) assert len(unleash_client.features) >= 9 @@ -329,13 +329,13 @@ def test_uc_dirty_cache(unleash_client_nodestroy): # Create Unleash client and check initial load unleash_client.initialize_client() - time.sleep(5) + time.sleep(1) assert unleash_client.is_enabled("testFlag") unleash_client.unleash_scheduler.shutdown() # Check that everything works if previous cache exists. unleash_client.initialize_client() - time.sleep(5) + time.sleep(1) assert unleash_client.is_enabled("testFlag") @@ -484,9 +484,8 @@ def test_uc_metrics(unleash_client): time.sleep(1) assert unleash_client.is_enabled("testFlag") - time.sleep(12) - request = json.loads(responses.calls[-1].request.body) - assert request["bucket"]["toggles"]["testFlag"]["yes"] == 1 + metrics = aggregate_metrics(unleash_client.features) + assert metrics["testFlag"]["yes"] == 1 @responses.activate @@ -496,7 +495,6 @@ def test_uc_registers_metrics_for_nonexistent_features(unleash_client): responses.add( responses.GET, URL + FEATURES_URL, json=MOCK_FEATURE_RESPONSE, status=200 ) - responses.add(responses.POST, URL + METRICS_URL, json={}, status=202) # Create Unleash client and check initial load unleash_client.initialize_client() @@ -506,9 +504,8 @@ def test_uc_registers_metrics_for_nonexistent_features(unleash_client): unleash_client.is_enabled("nonexistent-flag") # Verify that the metrics are serialized - time.sleep(12) - request = json.loads(responses.calls[-1].request.body) - assert request["bucket"]["toggles"]["nonexistent-flag"]["no"] == 1 + metrics = aggregate_metrics(unleash_client.features) + assert metrics["nonexistent-flag"]["no"] == 1 @responses.activate @@ -520,16 +517,14 @@ def test_uc_metrics_dependencies(unleash_client): json=MOCK_FEATURE_WITH_DEPENDENCIES_RESPONSE, status=200, ) - responses.add(responses.POST, URL + METRICS_URL, json={}, status=202) unleash_client.initialize_client() time.sleep(1) assert unleash_client.is_enabled("Child") - time.sleep(12) - request = json.loads(responses.calls[-1].request.body) - assert request["bucket"]["toggles"]["Child"]["yes"] == 1 - assert "Parent" not in request["bucket"]["toggles"] + metrics = aggregate_metrics(unleash_client.features) + assert metrics["Child"]["yes"] == 1 + assert "Parent" not in metrics @responses.activate @@ -539,7 +534,6 @@ def test_uc_registers_variant_metrics_for_nonexistent_features(unleash_client): responses.add( responses.GET, URL + FEATURES_URL, json=MOCK_FEATURE_RESPONSE, status=200 ) - responses.add(responses.POST, URL + METRICS_URL, json={}, status=202) # Create Unleash client and check initial load unleash_client.initialize_client() @@ -548,11 +542,9 @@ def test_uc_registers_variant_metrics_for_nonexistent_features(unleash_client): # Check a flag that doesn't exist unleash_client.get_variant("nonexistent-flag") - # Verify that the metrics are serialized - time.sleep(12) - request = json.loads(responses.calls[-1].request.body) - assert request["bucket"]["toggles"]["nonexistent-flag"]["no"] == 1 - assert request["bucket"]["toggles"]["nonexistent-flag"]["variants"]["disabled"] == 1 + metrics = aggregate_metrics(unleash_client.features) + assert metrics["nonexistent-flag"]["no"] == 1 + assert metrics["nonexistent-flag"]["variants"]["disabled"] == 1 @responses.activate @@ -565,7 +557,6 @@ def test_uc_doesnt_count_metrics_for_dependency_parents(unleash_client): json=MOCK_FEATURE_WITH_DEPENDENCIES_RESPONSE, status=200, ) - responses.add(responses.POST, URL + METRICS_URL, json={}, status=202) # Create Unleash client and check initial load unleash_client.initialize_client() @@ -578,11 +569,10 @@ def test_uc_doesnt_count_metrics_for_dependency_parents(unleash_client): unleash_client.get_variant(child) # Verify that the parent doesn't have any metrics registered - time.sleep(12) - request = json.loads(responses.calls[-1].request.body) - assert request["bucket"]["toggles"][child]["yes"] == 2 - assert request["bucket"]["toggles"][child]["variants"]["childVariant"] == 1 - assert parent not in request["bucket"]["toggles"] + metrics = aggregate_metrics(unleash_client.features) + assert metrics[child]["yes"] == 2 + assert metrics[child]["variants"]["childVariant"] == 1 + assert parent not in metrics @responses.activate @@ -595,7 +585,6 @@ def test_uc_counts_metrics_for_child_even_if_parent_is_disabled(unleash_client): json=MOCK_FEATURE_WITH_DEPENDENCIES_RESPONSE, status=200, ) - responses.add(responses.POST, URL + METRICS_URL, json={}, status=202) # Create Unleash client and check initial load unleash_client.initialize_client() @@ -608,11 +597,10 @@ def test_uc_counts_metrics_for_child_even_if_parent_is_disabled(unleash_client): unleash_client.get_variant(child) # Verify that the parent doesn't have any metrics registered - time.sleep(12) - request = json.loads(responses.calls[-1].request.body) - assert request["bucket"]["toggles"][child]["no"] == 2 - assert request["bucket"]["toggles"][child]["variants"]["disabled"] == 1 - assert parent not in request["bucket"]["toggles"] + metrics = aggregate_metrics(unleash_client.features) + assert metrics[child]["no"] == 2 + assert metrics[child]["variants"]["disabled"] == 1 + assert parent not in metrics @responses.activate @@ -627,7 +615,7 @@ def test_uc_disabled_registration(unleash_client_toggle_only): unleash_client.initialize_client() unleash_client.is_enabled("testFlag") - time.sleep(20) + time.sleep(REFRESH_INTERVAL * 2) assert unleash_client.is_enabled("testFlag") for api_call in responses.calls: @@ -650,7 +638,7 @@ def test_uc_server_error(unleash_client): responses.add( responses.GET, URL + FEATURES_URL, json=MOCK_FEATURE_RESPONSE, status=200 ) - time.sleep(20) + time.sleep(REFRESH_INTERVAL * 2) assert unleash_client.is_enabled("testFlag") diff --git a/tests/utilities/testing_constants.py b/tests/utilities/testing_constants.py index ef32d0da..7f0311c2 100644 --- a/tests/utilities/testing_constants.py +++ b/tests/utilities/testing_constants.py @@ -14,9 +14,9 @@ APP_NAME = "pytest" ENVIRONMENT = "unit" INSTANCE_ID = "123" -REFRESH_INTERVAL = 15 +REFRESH_INTERVAL = 3 REFRESH_JITTER = None -METRICS_INTERVAL = 10 +METRICS_INTERVAL = 2 METRICS_JITTER = None DISABLE_METRICS = True DISABLE_REGISTRATION = True