Optimize fetching data from DB (redirects, mappings) fix #516

vaas/vaas/cluster/admin.py

@@ -217,7 +217,7 @@ class LogicalClusterAdmin(admin.ModelAdmin):
 
     def get_changelist_instance(self, request):
         # refresh provider on each changelist view
-        self.provider = MappingProvider(DomainMapping.objects.all())
+        self.provider = MappingProvider(list(DomainMapping.objects.all().prefetch_related('clusters')))
         return super().get_changelist_instance(request)
 
     def get_tags(self, obj: LogicalCluster) -> str:

vaas/vaas/cluster/mapping.py

@@ -10,11 +10,17 @@ def __init__(self, mappings: List[DomainMapping]):
             'dynamic': [],
         }
         for m in mappings:
-            self.mappings[m.type].append(m)
+            self.mappings[m.type].append(
+                {
+                    'cluster_keys': [c.pk for c in m.clusters.all()],
+                    'mapping': m
+                }
+            )
 
     def provide_related_domains(self, cluster: LogicalCluster) -> List[str]:
-        result = {m.domain for m in cluster.domainmapping_set.filter(type='static')}
+        result = {m['mapping'].domain for m in self.mappings['static'] if cluster.pk in m['cluster_keys']}
+        cluster_labels = set(list(cluster.parsed_labels().keys()))
         for m in self.mappings['dynamic']:
-            if m.is_cluster_related_by_labels(cluster):
-                result.add(m.domain)
+            if m['mapping'].has_matching_labels(cluster_labels):
+                result.add(m['mapping'].domain)
         return sorted(list(result))

vaas/vaas/cluster/models.py

@@ -146,6 +146,13 @@ def mapped_domains(self, cluster: LogicalCluster) -> List[str]:
         # sort domains in order to enforce stable rendering of vcl content for the same input
         return sorted(result)
 
+    def has_matching_labels(self, labels: Set[str]) -> bool:
+        result = False
+        # check if all needed placeholders of any mapping are satisfied by cluster labels
+        for required_labels in self.__parse_placeholders().values():
+            result = result or not required_labels.difference(labels)
+        return result
+
     def is_cluster_related_by_labels(self, cluster: LogicalCluster) -> bool:
         result = False
         cluster_labels = set(list(cluster.parsed_labels().keys()))

vaas/vaas/cluster/tests/test_mapping.py

@@ -8,15 +8,15 @@
 from vaas.cluster.models import LogicalCluster, DomainMapping
 
 
-@patch('vaas.cluster.models.LogicalCluster.domainmapping_set')
-def test_should_return_sorted_static_domains_connected_by_db_relation(domain_mapping_set_mock):
+@patch('vaas.cluster.models.DomainMapping.clusters')
+def test_should_return_sorted_static_domains_connected_by_db_relation(clusters_mock):
     # given static mapping related with logical cluster
     static_mappings = [
-        DomainMapping(type="static", domain="b-static.example.com"),
-        DomainMapping(type="static", domain="a-static.example.com"),
+        DomainMapping(id=1, type="static", domain="b-static.example.com"),
+        DomainMapping(id=2, type="static", domain="a-static.example.com"),
     ]
-    domain_mapping_set_mock.filter = Mock(return_value=static_mappings)
-    cluster = LogicalCluster()
+    cluster = LogicalCluster(id=100)
+    clusters_mock.all = Mock(return_value=[cluster])
 
     # when looking for related mappings for cluster
     provider = MappingProvider(static_mappings)
@@ -30,9 +30,15 @@ def test_should_return_sorted_static_domains_connected_by_db_relation(domain_map
 def test_should_return_dynamic_domains_matched_by_common_labels():
     # given dynamic mapping unrelated with logical cluster
     dynamic_mappings = [
-        DomainMapping(type="dynamic", domain="b-dynamic.example.com", mappings_list='["{label-one}.example.com"]'),
-        DomainMapping(type="dynamic", domain="a-dynamic.example.com", mappings_list='["{label-two}.example.com"]'),
-        DomainMapping(type="dynamic", domain="c-dynamic.example.com", mappings_list='["{no-label}.example.com"]'),
+        DomainMapping(
+            id=1, type="dynamic", domain="b-dynamic.example.com", mappings_list='["{label-one}.example.com"]'
+        ),
+        DomainMapping(
+            id=2, type="dynamic", domain="a-dynamic.example.com", mappings_list='["{label-two}.example.com"]'
+        ),
+        DomainMapping(
+            id=3, type="dynamic", domain="c-dynamic.example.com", mappings_list='["{no-label}.example.com"]'
+        ),
     ]
     # and cluster that has labels used as a placeholders in above mappings
     cluster = LogicalCluster(labels_list=json.dumps(["label-one:b", "label-two:a", "label-whatever:c"]))
@@ -50,6 +56,7 @@ def test_should_return_single_dynamic_domain_even_if_more_than_one_mapping_is_sa
     # given dynamic mapping unrelated with logical cluster
     dynamic_mappings = [
         DomainMapping(
+            id=1,
             type="dynamic",
             domain="multiple-dynamic.example.com",
             mappings_list='["{label-one}.example.com", "{label-two}.example.com"]'

vaas/vaas/vcl/renderer.py

@@ -399,19 +399,21 @@ def fetch_render_data(self):
         self.dcs = list(Dc.objects.all())
         self.template_blocks = list(VclTemplateBlock.objects.all().prefetch_related('template'))
         self.vcl_variables = list(VclVariable.objects.all())
-        backends = list(Backend.objects.filter(enabled=True).prefetch_related('director', 'dc'))
+        backends = list(
+            Backend.objects.filter(enabled=True).prefetch_related('director', 'dc', 'director__time_profile')
+        )
         canary_backend_ids = list(
             Backend.objects.values_list('id', flat=True).filter(tags__name='canary').prefetch_related('director', 'dc')
         )
         self.distributed_backends = self.distribute_backends(backends)
         self.distributed_canary_backends = self.prepare_canary_backends(canary_backend_ids, backends)
-        self.domain_mappings = list(DomainMapping.objects.all())
+        self.domain_mappings = list(DomainMapping.objects.all().prefetch_related('clusters'))
         self.mapping_provider = MappingProvider(self.domain_mappings)
 
     @collect_processing
     def assemble_redirects(self) -> dict[str, list[Redirect]]:
         redirects = {}
-        for redirect in Redirect.objects.all().order_by('src_domain', 'priority'):
+        for redirect in Redirect.objects.all().select_related('src_domain').order_by('src_domain', 'priority'):
             if redirect.src_domain.domain not in redirects.keys():
                 redirects[redirect.src_domain.domain] = []
             redirects[redirect.src_domain.domain].append(redirect)