From 087ba901d9b398af098173d24fd916d88d8beffe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Szulc?= Date: Mon, 21 Oct 2024 14:43:36 +0200 Subject: [PATCH 01/20] Fix data not JSON-serializable --- src/ralph/assets/invoice_report.py | 6 +++++- src/ralph/reports/helpers.py | 18 ++++++++++++------ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/src/ralph/assets/invoice_report.py b/src/ralph/assets/invoice_report.py index 9b44935b1f..ee828377d1 100644 --- a/src/ralph/assets/invoice_report.py +++ b/src/ralph/assets/invoice_report.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- import datetime +import json import logging from django.contrib import messages @@ -116,7 +117,7 @@ def _get_report_data(self, request, queryset): 'model': queryset.model._meta.model_name, 'base_info': { 'invoice_no': first_item.invoice_no, - 'invoice_date': first_item.invoice_date, + 'invoice_date': str(first_item.invoice_date), 'provider': first_item.provider, 'datetime': datetime.datetime.now().strftime( self._invoice_report_datetime_format @@ -148,6 +149,9 @@ def _get_pdf_content(self, data): template_content = f.read() service_pdf = ExternalService('PDF') + # Make sure data is JSON-serializable + # Will throw otherwise + data = json.loads(json.dumps(data)) result = service_pdf.run( template=template_content, data=data, diff --git a/src/ralph/reports/helpers.py b/src/ralph/reports/helpers.py index 798d7b043c..0c112ba350 100644 --- a/src/ralph/reports/helpers.py +++ b/src/ralph/reports/helpers.py @@ -1,4 +1,5 @@ import datetime +import json import os import tempfile @@ -25,17 +26,22 @@ def generate_report(name, requester, instances, language, context): service_pdf = ExternalService('PDF') for n in range(0, len(context), items_per_attachment): - result = service_pdf.run( - template=template_content, - data={ + # Make sure data is JSON-serializable + # Will throw otherwise + data = json.loads(json.dumps( + { 'id': ', '.join([str(obj.id) for obj in - instances[n:n+items_per_attachment]]), - 'now': datetime.datetime.now(), + instances[n:n + items_per_attachment]]), + 'now': str(datetime.datetime.now()), 'logged_user': obj_to_dict(requester), 'affected_user': obj_to_dict(instances[0].user), 'owner': obj_to_dict(instances[0].owner), - 'assets': context[n:n+items_per_attachment], + 'assets': context[n:n + items_per_attachment], } + )) + result = service_pdf.run( + template=template_content, + data=data ) filename = "_".join([ From c33ead2c5c99808e6e027af5b411a9c7bb160aad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Szulc?= Date: Wed, 23 Oct 2024 12:29:55 +0200 Subject: [PATCH 02/20] Updated changelog for 20241023.1 version. --- debian/changelog | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/debian/changelog b/debian/changelog index c21f3bf97c..4d140c8df5 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +ralph (20241023.1) bionic; urgency=medium + + * Updated changelog for 20241011.1 version. + * Fix data not JSON-serializable + + -- Paweł Szulc Wed, 23 Oct 2024 10:29:45 +0000 + ralph (20241011.1) bionic; urgency=medium [ Paweł Szulc ] From ba59dd03b9220949e8f545c71d6f5b671c86a374 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Szulc?= Date: Mon, 28 Oct 2024 12:13:45 +0100 Subject: [PATCH 03/20] Fix virtual server - hypervisor --- src/ralph/data_center/tests/factories.py | 2 +- src/ralph/virtual/admin.py | 15 ++++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/ralph/data_center/tests/factories.py b/src/ralph/data_center/tests/factories.py index cbbcfc1ef0..39f0586651 100644 --- a/src/ralph/data_center/tests/factories.py +++ b/src/ralph/data_center/tests/factories.py @@ -148,7 +148,7 @@ class DataCenterAssetFactory(DjangoModelFactory): force_depreciation = False model = factory.SubFactory(DataCenterAssetModelFactory) sn = factory.Faker('ssn') - barcode = factory.Sequence(lambda n: 'dc' + str(n + 10**8)) + barcode = None # factory.Sequence(lambda n: 'dc' + str(n + 10**8)) hostname = factory.Sequence(lambda n: 'ralph{}.allegro.pl'.format(n)) order_no = factory.Sequence(lambda n: 'Order number ' + str(n)) budget_info = factory.SubFactory(BudgetInfoFactory) diff --git a/src/ralph/virtual/admin.py b/src/ralph/virtual/admin.py index cea1675e49..1786417177 100644 --- a/src/ralph/virtual/admin.py +++ b/src/ralph/virtual/admin.py @@ -130,7 +130,7 @@ class VirtualServerAdmin( ) list_display = [ 'hostname', 'type', 'sn', 'service_env', 'configuration_path', - 'get_parent', 'scan_status', 'scm_status_check' + 'parent_', 'scan_status', 'scm_status_check' ] raw_id_fields = ['parent', 'service_env', 'configuration_path'] fields = [ @@ -174,13 +174,14 @@ def get_queryset(self, request): ) @mark_safe - def get_parent(self, obj): - if not obj.parent_id: + def parent_(self, obj): + try: + parent = obj.polymorphic_parent + return '{}'.format( + parent.get_absolute_url(), parent.hostname + ) + except: # this happens when no parent or parent doesn't have a hostname return '-' - return '{}'.format( - obj.parent.get_absolute_url(), obj.parent.hostname - ) - class CloudHostTabularInline(RalphTabularInline): can_delete = False From 67392e4c5c3e44e95bf0ead67c0a9d8eb2c56d49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Szulc?= Date: Mon, 28 Oct 2024 12:53:13 +0100 Subject: [PATCH 04/20] Revert remove barcode from factory --- src/ralph/data_center/tests/factories.py | 2 +- src/ralph/virtual/admin.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/ralph/data_center/tests/factories.py b/src/ralph/data_center/tests/factories.py index 39f0586651..cbbcfc1ef0 100644 --- a/src/ralph/data_center/tests/factories.py +++ b/src/ralph/data_center/tests/factories.py @@ -148,7 +148,7 @@ class DataCenterAssetFactory(DjangoModelFactory): force_depreciation = False model = factory.SubFactory(DataCenterAssetModelFactory) sn = factory.Faker('ssn') - barcode = None # factory.Sequence(lambda n: 'dc' + str(n + 10**8)) + barcode = factory.Sequence(lambda n: 'dc' + str(n + 10**8)) hostname = factory.Sequence(lambda n: 'ralph{}.allegro.pl'.format(n)) order_no = factory.Sequence(lambda n: 'Order number ' + str(n)) budget_info = factory.SubFactory(BudgetInfoFactory) diff --git a/src/ralph/virtual/admin.py b/src/ralph/virtual/admin.py index 1786417177..4ae4f6aebc 100644 --- a/src/ralph/virtual/admin.py +++ b/src/ralph/virtual/admin.py @@ -180,9 +180,10 @@ def parent_(self, obj): return '{}'.format( parent.get_absolute_url(), parent.hostname ) - except: # this happens when no parent or parent doesn't have a hostname + except: # noqa # this happens when no parent or parent doesn't have a hostname return '-' + class CloudHostTabularInline(RalphTabularInline): can_delete = False model = CloudHost From ada3e67ca12c099e179283b343eb0d669226d273 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Szulc?= Date: Mon, 28 Oct 2024 15:35:28 +0100 Subject: [PATCH 05/20] Fix virtual server - service env optional --- src/ralph/assets/api/serializers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ralph/assets/api/serializers.py b/src/ralph/assets/api/serializers.py index 1abe37d43d..9d026d3345 100644 --- a/src/ralph/assets/api/serializers.py +++ b/src/ralph/assets/api/serializers.py @@ -53,9 +53,9 @@ def get_object_type(self, instance): class OwnersFromServiceEnvSerializerMixin(RalphAPISerializer): business_owners = SimpleRalphUserSerializer( - many=True, source='service_env.service.business_owners') + many=True, source='service_env.service.business_owners', required=False) technical_owners = SimpleRalphUserSerializer( - many=True, source='service_env.service.technical_owners') + many=True, source='service_env.service.technical_owners', required=False) class BusinessSegmentSerializer(RalphAPISerializer): From d85290295967990246b469e1704ebccd02a370c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Szulc?= Date: Mon, 28 Oct 2024 16:24:00 +0100 Subject: [PATCH 06/20] Updated changelog for 20241028.1 version. --- debian/changelog | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/debian/changelog b/debian/changelog index 4d140c8df5..3cbabcd775 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,11 @@ +ralph (20241028.1) bionic; urgency=medium + + * Fix virtual server - hypervisor + * Revert remove barcode from factory + * Fix virtual server - service env optional + + -- Paweł Szulc Mon, 28 Oct 2024 15:23:53 +0000 + ralph (20241023.1) bionic; urgency=medium * Updated changelog for 20241011.1 version. From 46667975ea5848e27660fdb6ebc4c009f079cc76 Mon Sep 17 00:00:00 2001 From: awieckowski Date: Tue, 29 Oct 2024 08:57:50 +0100 Subject: [PATCH 07/20] Fix LDAP Groups sync --- src/ralph/accounts/ldap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ralph/accounts/ldap.py b/src/ralph/accounts/ldap.py index d8cdabe5bd..5dfb36b3ab 100644 --- a/src/ralph/accounts/ldap.py +++ b/src/ralph/accounts/ldap.py @@ -66,7 +66,7 @@ def mirror_groups(self): new_groups = [Group.objects.get_or_create(name=name)[0] for name in target_group_names if name not in existing_group_names] - self._user.groups = existing_groups + new_groups + self._user.groups.set(existing_groups + new_groups) _LDAPUser._mirror_groups_original = _LDAPUser._mirror_groups _LDAPUser._mirror_groups = mirror_groups From 49f2b06d71c2b95791bb5640bbe277a55a754485 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Szulc?= Date: Tue, 29 Oct 2024 11:46:29 +0100 Subject: [PATCH 08/20] Fix bulk edit for some models --- src/ralph/admin/mixins.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/ralph/admin/mixins.py b/src/ralph/admin/mixins.py index 3af0668dca..c0496a115a 100644 --- a/src/ralph/admin/mixins.py +++ b/src/ralph/admin/mixins.py @@ -541,8 +541,9 @@ def get_queryset(self, request): qs = super().get_queryset(request) id_list = request.GET.getlist(BULK_EDIT_VAR_IDS, []) if id_list: - qs = qs.filter(pk__in=id_list) - return qs + return self.model.objects.filter(id__in=id_list) + else: + return qs def get_list_display(self, request): """ From 0aab661a7a2cacd7393753d7cef752c441210269 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Szulc?= Date: Tue, 29 Oct 2024 14:00:09 +0100 Subject: [PATCH 09/20] Updated changelog for 20241029.1 version. --- debian/changelog | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/debian/changelog b/debian/changelog index 3cbabcd775..4088d203ae 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,13 @@ +ralph (20241029.1) bionic; urgency=medium + + [ awieckowski ] + * Fix LDAP Groups sync + + [ Paweł Szulc ] + * Fix bulk edit for some models + + -- Paweł Szulc Tue, 29 Oct 2024 13:00:03 +0000 + ralph (20241028.1) bionic; urgency=medium * Fix virtual server - hypervisor From c4351025320598bfc99c35b5514c6a42039e6d67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Szulc?= Date: Tue, 29 Oct 2024 15:13:32 +0100 Subject: [PATCH 10/20] Fix method to get limit choices --- src/ralph/lib/mixins/fields.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ralph/lib/mixins/fields.py b/src/ralph/lib/mixins/fields.py index b78a207e6f..be3fbfeca3 100644 --- a/src/ralph/lib/mixins/fields.py +++ b/src/ralph/lib/mixins/fields.py @@ -232,6 +232,9 @@ def deconstruct(self): def limit_choices_to(self): return self.limit_choices() + def get_limit_choices_to(self): + return self.limit_choices() + def limit_choices(self): """ Add limit_choices_to search by content_type for models From 29bddfad14658ca2c7584b315cfdbc8a9aea6377 Mon Sep 17 00:00:00 2001 From: awieckowski Date: Wed, 30 Oct 2024 12:36:08 +0100 Subject: [PATCH 11/20] Increase max user department field length --- src/ralph/accounts/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ralph/accounts/models.py b/src/ralph/accounts/models.py index d936cc4e35..bff526468b 100644 --- a/src/ralph/accounts/models.py +++ b/src/ralph/accounts/models.py @@ -102,7 +102,7 @@ class RalphUser( ) department = models.CharField( verbose_name=_('department'), - max_length=64, + max_length=128, blank=True, ) manager = models.CharField( From 09ec85286359f2e79ac78a678deab3c10072b49f Mon Sep 17 00:00:00 2001 From: awieckowski Date: Wed, 30 Oct 2024 12:36:55 +0100 Subject: [PATCH 12/20] Add missing migration --- .../migrations/0010_auto_20241030_1235.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 src/ralph/accounts/migrations/0010_auto_20241030_1235.py diff --git a/src/ralph/accounts/migrations/0010_auto_20241030_1235.py b/src/ralph/accounts/migrations/0010_auto_20241030_1235.py new file mode 100644 index 0000000000..41c9d3b7ca --- /dev/null +++ b/src/ralph/accounts/migrations/0010_auto_20241030_1235.py @@ -0,0 +1,18 @@ +# Generated by Django 2.0.13 on 2024-10-30 12:35 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('accounts', '0009_auto_20240621_1217'), + ] + + operations = [ + migrations.AlterField( + model_name='ralphuser', + name='department', + field=models.CharField(blank=True, max_length=128, verbose_name='department'), + ), + ] From 03da535881795dd111860169cbc81a19f94cbd12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Szulc?= Date: Thu, 31 Oct 2024 13:25:04 +0100 Subject: [PATCH 13/20] Fix data center asset export not getting filter from request --- src/ralph/admin/mixins.py | 3 --- src/ralph/data_center/admin.py | 7 ------- src/ralph/data_importer/resources.py | 8 ++++++++ 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/ralph/admin/mixins.py b/src/ralph/admin/mixins.py index c0496a115a..f359061786 100644 --- a/src/ralph/admin/mixins.py +++ b/src/ralph/admin/mixins.py @@ -421,9 +421,6 @@ def get_export_queryset(self, request): ) if resource_prefetch_related: queryset = queryset.prefetch_related(*resource_prefetch_related) - # cast to list to consider all prefetch_related (django-import-export - # use queryset.iterator() to "save memory", but then for every row - # sql queries are made to fetch all m2m relations) return list(queryset) def get_export_resource_class(self): diff --git a/src/ralph/data_center/admin.py b/src/ralph/data_center/admin.py index 32625631ae..f2d72b5e9c 100644 --- a/src/ralph/data_center/admin.py +++ b/src/ralph/data_center/admin.py @@ -490,13 +490,6 @@ class DataCenterAssetAdmin( }), ) - def get_export_queryset(self, request): - return DataCenterAsset.polymorphic_objects.select_related( - *self.list_select_related - ).polymorphic_prefetch_related( - DataCenterAsset=['tags', 'ethernet_set__ipaddress', 'parent__ethernet_set__ipaddress'], - ) - def get_multiadd_fields(self, obj=None): multiadd_fields = [ {'field': 'sn', 'allow_duplicates': False}, diff --git a/src/ralph/data_importer/resources.py b/src/ralph/data_importer/resources.py index 6f4faf5671..4e9634fab7 100644 --- a/src/ralph/data_importer/resources.py +++ b/src/ralph/data_importer/resources.py @@ -306,8 +306,16 @@ class DataCenterAssetResource(ResourceWithPrice, RalphModelResource): class Meta: model = physical.DataCenterAsset select_related = ( + 'model__manufacturer', 'model__category', 'service_env__service', 'service_env__environment', 'rack__server_room__data_center', + 'configuration_path', + 'property_of', + 'parent', + 'budget_info', + ) + prefetch_related = ( + 'tags', 'ethernet_set__ipaddress', 'parent__ethernet_set__ipaddress', ) exclude = ('content_type', 'asset_ptr', 'baseobject_ptr', 'connections') From d52c3aa35e40918e98ded4869a7e76e34cb40719 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Szulc?= Date: Thu, 31 Oct 2024 13:29:58 +0100 Subject: [PATCH 14/20] Fix format to work with newer flake --- src/ralph/__init__.py | 2 +- src/ralph/accounts/helpers.py | 28 ++++++++++++++-------------- src/ralph/accounts/ldap.py | 1 + src/ralph/api/routers.py | 1 + src/ralph/assets/api/serializers.py | 1 + src/ralph/helpers.py | 1 + src/ralph/lib/metrics/collector.py | 1 + src/ralph/licences/models.py | 1 + src/ralph/networks/filters.py | 2 ++ src/ralph/settings/dev.py | 1 + 10 files changed, 24 insertions(+), 15 deletions(-) diff --git a/src/ralph/__init__.py b/src/ralph/__init__.py index df6f6264bf..5adeaeba19 100644 --- a/src/ralph/__init__.py +++ b/src/ralph/__init__.py @@ -7,7 +7,7 @@ def monkey_options_init(self, meta, app_label): self._old__init__(meta, app_label) self.default_permissions = ('add', 'change', 'delete', 'view') -# TODO: create PR to Django - default_permissions from settings + Options._old__init__ = Options.__init__ Options.__init__ = lambda self, meta, app_label=None: monkey_options_init( self, meta, app_label diff --git a/src/ralph/accounts/helpers.py b/src/ralph/accounts/helpers.py index a04b923983..7cbc9a9cb4 100644 --- a/src/ralph/accounts/helpers.py +++ b/src/ralph/accounts/helpers.py @@ -10,20 +10,20 @@ from ralph.lib.transitions.models import Transition from ralph.sim_cards.models import SIMCard -ACCEPTANCE_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['TRANSITION_ID'] # noqa: E509 -ACCEPTANCE_SIM_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['TRANSITION_SIM_ID'] # noqa: E509 -ACCEPTANCE_BACK_OFFICE_ACCEPT_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['BACK_OFFICE_ACCEPT_STATUS'] # noqa: E509 -ACCEPTANCE_SIMCARD_ACCEPT_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['SIMCARD_ACCEPT_STATUS'] # noqa: E509 -ACCEPTANCE_LOAN_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['LOAN_TRANSITION_ID'] # noqa: E509 -ACCEPTANCE_BACK_OFFICE_ACCEPT_LOAN_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['BACK_OFFICE_ACCEPT_LOAN_STATUS'] # noqa: E509 -ACCEPTANCE_RETURN_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['RETURN_TRANSITION_ID'] # noqa: E509 -ACCEPTANCE_BACK_OFFICE_RETURN_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['BACK_OFFICE_ACCEPT_RETURN_STATUS'] # noqa: E509 -ACCEPTANCE_ACCESS_CARD_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['TRANSITION_ACCESS_CARD_ID'] # noqa: E509 -ACCEPTANCE_ACCESS_CARD_ACCEPT_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['ACCESS_CARD_ACCEPT_ACCEPT_STATUS'] # noqa: E509 -ACCEPTANCE_BACK_OFFICE_TEAM_ACCEPT_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['BACK_OFFICE_TEAM_ACCEPT_STATUS'] # noqa: E509 -ACCEPTANCE_TEAM_ACCEPT_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['TRANSITION_TEAM_ACCEPT_ID'] # noqa: E509 -ACCEPTANCE_BACK_OFFICE_TEST_ACCEPT_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['BACK_OFFICE_TEST_ACCEPT_STATUS'] # noqa: E509 -ACCEPTANCE_TEST_ACCEPT_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['TRANSITION_TEST_ACCEPT_ID'] # noqa: E509 +ACCEPTANCE_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['TRANSITION_ID'] # noqa +ACCEPTANCE_SIM_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['TRANSITION_SIM_ID'] # noqa +ACCEPTANCE_BACK_OFFICE_ACCEPT_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['BACK_OFFICE_ACCEPT_STATUS'] # noqa +ACCEPTANCE_SIMCARD_ACCEPT_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['SIMCARD_ACCEPT_STATUS'] # noqa +ACCEPTANCE_LOAN_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['LOAN_TRANSITION_ID'] # noqa +ACCEPTANCE_BACK_OFFICE_ACCEPT_LOAN_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['BACK_OFFICE_ACCEPT_LOAN_STATUS'] # noqa +ACCEPTANCE_RETURN_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['RETURN_TRANSITION_ID'] # noqa +ACCEPTANCE_BACK_OFFICE_RETURN_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['BACK_OFFICE_ACCEPT_RETURN_STATUS'] # noqa +ACCEPTANCE_ACCESS_CARD_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['TRANSITION_ACCESS_CARD_ID'] # noqa +ACCEPTANCE_ACCESS_CARD_ACCEPT_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['ACCESS_CARD_ACCEPT_ACCEPT_STATUS'] # noqa +ACCEPTANCE_BACK_OFFICE_TEAM_ACCEPT_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['BACK_OFFICE_TEAM_ACCEPT_STATUS'] # noqa +ACCEPTANCE_TEAM_ACCEPT_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['TRANSITION_TEAM_ACCEPT_ID'] # noqa +ACCEPTANCE_BACK_OFFICE_TEST_ACCEPT_STATUS = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['BACK_OFFICE_TEST_ACCEPT_STATUS'] # noqa +ACCEPTANCE_TEST_ACCEPT_TRANSITION_ID = settings.ACCEPT_ASSETS_FOR_CURRENT_USER_CONFIG['TRANSITION_TEST_ACCEPT_ID'] # noqa def transition_exists(transition_id): diff --git a/src/ralph/accounts/ldap.py b/src/ralph/accounts/ldap.py index 5dfb36b3ab..caf3380247 100644 --- a/src/ralph/accounts/ldap.py +++ b/src/ralph/accounts/ldap.py @@ -68,6 +68,7 @@ def mirror_groups(self): self._user.groups.set(existing_groups + new_groups) + _LDAPUser._mirror_groups_original = _LDAPUser._mirror_groups _LDAPUser._mirror_groups = mirror_groups diff --git a/src/ralph/api/routers.py b/src/ralph/api/routers.py index e7448e1468..54684c7086 100644 --- a/src/ralph/api/routers.py +++ b/src/ralph/api/routers.py @@ -58,4 +58,5 @@ def get(self, request, *args, **kwargs): return APIRoot.as_view() + router = RalphRouter() diff --git a/src/ralph/assets/api/serializers.py b/src/ralph/assets/api/serializers.py index 9d026d3345..c5e5ba397a 100644 --- a/src/ralph/assets/api/serializers.py +++ b/src/ralph/assets/api/serializers.py @@ -317,6 +317,7 @@ class Meta: 'parent' ) + # TODO: Is there a better way to make it work since drf 3.5? del ConfigurationClassSimpleSerializer._declared_fields['tags'] diff --git a/src/ralph/helpers.py b/src/ralph/helpers.py index 6df2114820..f5572599fd 100644 --- a/src/ralph/helpers.py +++ b/src/ralph/helpers.py @@ -41,6 +41,7 @@ def generate_pdf_response(pdf_data, file_name): ) return response + CACHE_DEFAULT = object() diff --git a/src/ralph/lib/metrics/collector.py b/src/ralph/lib/metrics/collector.py index 3ed7fd766b..366c2cb308 100644 --- a/src/ralph/lib/metrics/collector.py +++ b/src/ralph/lib/metrics/collector.py @@ -25,6 +25,7 @@ def build_statsd_client( ipv6=ipv6 ) + if settings.COLLECT_METRICS and statsd is None: statsd = build_statsd_client() diff --git a/src/ralph/licences/models.py b/src/ralph/licences/models.py index 9193a4ed5f..c6af79886e 100644 --- a/src/ralph/licences/models.py +++ b/src/ralph/licences/models.py @@ -111,6 +111,7 @@ def get_queryset(self): } ) + LICENCES_RELATED_OBJECTS_PREFETCH_RELATED = [ 'users', # prefetch all baseobjects related with licence; this allows to call diff --git a/src/ralph/networks/filters.py b/src/ralph/networks/filters.py index 588ae74be2..f0ddb28eb1 100644 --- a/src/ralph/networks/filters.py +++ b/src/ralph/networks/filters.py @@ -24,6 +24,8 @@ def get_private_network_filter(): max_ip = int(network.broadcast_address) filter_ |= Q(min_ip__gte=min_ip, max_ip__lte=max_ip) return filter_ + + PRIVATE_NETWORK_FILTER = get_private_network_filter() diff --git a/src/ralph/settings/dev.py b/src/ralph/settings/dev.py index e819d2db2f..5a768d93cb 100644 --- a/src/ralph/settings/dev.py +++ b/src/ralph/settings/dev.py @@ -5,6 +5,7 @@ def only_true(request): '''For django debug toolbar.''' return True + DEBUG = True INSTALLED_APPS = INSTALLED_APPS + ( From b19881779248bd57ff79f92ace059627d008322e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Szulc?= Date: Thu, 31 Oct 2024 14:06:30 +0100 Subject: [PATCH 15/20] Fix prefetches --- src/ralph/data_center/admin.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/ralph/data_center/admin.py b/src/ralph/data_center/admin.py index f2d72b5e9c..a1691b0979 100644 --- a/src/ralph/data_center/admin.py +++ b/src/ralph/data_center/admin.py @@ -28,6 +28,7 @@ from ralph.admin.mixins import ( BulkEditChangeListMixin, RalphAdmin, + RalphAdminImportExportMixin, RalphTabularInline ) from ralph.admin.views.extra import RalphDetailViewAdmin @@ -490,6 +491,15 @@ class DataCenterAssetAdmin( }), ) + def get_export_queryset(self, request): + # we skip method in RalphAdminImportExportMixin because it doesn't return a queryset + # sad :( + return super(RalphAdminImportExportMixin, self).get_export_queryset(request).select_related( + *self.list_select_related + ).polymorphic_prefetch_related( + DataCenterAsset=['tags', 'ethernet_set__ipaddress', 'parent__ethernet_set__ipaddress'], + ) + def get_multiadd_fields(self, obj=None): multiadd_fields = [ {'field': 'sn', 'allow_duplicates': False}, From 04002d96f0c3d72c9f0003d1bcaf2f156926d41d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Szulc?= Date: Thu, 31 Oct 2024 14:53:41 +0100 Subject: [PATCH 16/20] Make prefetches work for filtered and not filtered export --- src/ralph/data_center/admin.py | 26 ++++++++++++++------ src/ralph/data_importer/tests/test_export.py | 10 ++++++++ 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/src/ralph/data_center/admin.py b/src/ralph/data_center/admin.py index a1691b0979..24af936b21 100644 --- a/src/ralph/data_center/admin.py +++ b/src/ralph/data_center/admin.py @@ -35,7 +35,7 @@ from ralph.admin.views.main import RalphChangeList from ralph.admin.views.multiadd import MulitiAddAdminMixin from ralph.assets.invoice_report import AssetInvoiceReportMixin -from ralph.assets.models.base import BaseObject +from ralph.assets.models.base import BaseObject, BaseObjectPolymorphicQuerySet from ralph.assets.models.components import Ethernet from ralph.assets.views import ComponentsAdminView from ralph.attachments.admin import AttachmentsMixin @@ -492,13 +492,25 @@ class DataCenterAssetAdmin( ) def get_export_queryset(self, request): - # we skip method in RalphAdminImportExportMixin because it doesn't return a queryset - # sad :( - return super(RalphAdminImportExportMixin, self).get_export_queryset(request).select_related( - *self.list_select_related - ).polymorphic_prefetch_related( - DataCenterAsset=['tags', 'ethernet_set__ipaddress', 'parent__ethernet_set__ipaddress'], + qs = ( + super(RalphAdminImportExportMixin, self) + .get_export_queryset(request) + .select_related( + *self.list_select_related + ) ) + if isinstance(qs, BaseObjectPolymorphicQuerySet): + return qs.polymorphic_prefetch_related( + DataCenterAsset=[ + 'tags', + 'ethernet_set__ipaddress', + 'parent__ethernet_set__ipaddress' + ] + ) + else: + return qs.prefetch_related( + 'tags', 'ethernet_set__ipaddress', 'parent__ethernet_set__ipaddress' + ) def get_multiadd_fields(self, obj=None): multiadd_fields = [ diff --git a/src/ralph/data_importer/tests/test_export.py b/src/ralph/data_importer/tests/test_export.py index 61b623bfb7..1a0b8cc4b6 100644 --- a/src/ralph/data_importer/tests/test_export.py +++ b/src/ralph/data_importer/tests/test_export.py @@ -114,6 +114,16 @@ def test_data_center_asset_export_queries_count(self): DataCenterAsset ), max_queries=12) + def test_data_center_asset_export_filtered(self): + self._init(10) + first_id = next(iter(self.data_center_assets_map.keys())) + with CaptureQueriesContext(connections['default']) as cqc: + export_data = self._export( + DataCenterAsset, filters={'id': first_id} + ) + queries = len(cqc) + self.assertEqual(len(export_data.dict), 1) + self.assertLessEqual(queries, 10) class DataCenterAssetExporterTestCaseWithParent(DataCenterAssetExporterTestCase): def _init(self, num=10): From 9d3e255e67beaefe9154e277cbe8d91a30bc4996 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Szulc?= Date: Thu, 31 Oct 2024 15:26:23 +0100 Subject: [PATCH 17/20] loosen queries count constraint for export --- src/ralph/data_importer/tests/test_export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ralph/data_importer/tests/test_export.py b/src/ralph/data_importer/tests/test_export.py index 1a0b8cc4b6..814c903ace 100644 --- a/src/ralph/data_importer/tests/test_export.py +++ b/src/ralph/data_importer/tests/test_export.py @@ -123,7 +123,7 @@ def test_data_center_asset_export_filtered(self): ) queries = len(cqc) self.assertEqual(len(export_data.dict), 1) - self.assertLessEqual(queries, 10) + self.assertLessEqual(queries, 12) class DataCenterAssetExporterTestCaseWithParent(DataCenterAssetExporterTestCase): def _init(self, num=10): From 021b3e9a92b5bcfeb2a7732b437a8183954c78f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Szulc?= Date: Mon, 4 Nov 2024 11:17:57 +0100 Subject: [PATCH 18/20] Updated changelog for 20241104.1 version. --- debian/changelog | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/debian/changelog b/debian/changelog index 4088d203ae..d46dd621b5 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,14 @@ +ralph (20241104.1) bionic; urgency=medium + + * Fix method to get limit choices + * Fix data center asset export not getting filter from request + * Fix format to work with newer flake + * Fix prefetches + * Make prefetches work for filtered and not filtered export + * loosen queries count constraint for export + + -- Paweł Szulc Mon, 04 Nov 2024 10:17:53 +0000 + ralph (20241029.1) bionic; urgency=medium [ awieckowski ] From 4927a0c1ef68776900ba28db87fea67fc81bd273 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Szulc?= Date: Mon, 4 Nov 2024 13:14:12 +0100 Subject: [PATCH 19/20] Fix ldap sync when bytes received --- requirements/prod_ldap.txt | 2 +- .../accounts/management/commands/ldap_sync.py | 26 ++++++++++++------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/requirements/prod_ldap.txt b/requirements/prod_ldap.txt index 7aca55e51f..db5ba301a8 100644 --- a/requirements/prod_ldap.txt +++ b/requirements/prod_ldap.txt @@ -1,2 +1,2 @@ -r prod.txt -django-auth-ldap==1.2.7 +django-auth-ldap==1.6.1 diff --git a/src/ralph/accounts/management/commands/ldap_sync.py b/src/ralph/accounts/management/commands/ldap_sync.py index 1e619aac65..c592eeb207 100644 --- a/src/ralph/accounts/management/commands/ldap_sync.py +++ b/src/ralph/accounts/management/commands/ldap_sync.py @@ -24,6 +24,20 @@ ldap_module_exists = False +def decode_nested_dict(data): + if isinstance(data, dict): + return {key: decode_nested_dict(value) for key, value in data.items()} + elif isinstance(data, list): + return [decode_nested_dict(element) for element in data] + elif isinstance(data, bytes): + try: + return data.decode('utf-8') + except UnicodeDecodeError: + return data + else: + return data + + def _truncate(field_key, field_name, ldap_dict): """ Truncate user's field when it's longer then default django value, which is @@ -274,16 +288,8 @@ def populate_users(self): """Load users from ldap and populate them. Returns number of users.""" synced = 0 for user_dn, ldap_dict in self._get_users(): - if ldap_dict.get('c'): - try: - ldap_dict['c'] = [v.decode('utf-8') for v in ldap_dict['c']] - except UnicodeDecodeError: - logger.error( - "Can't decode country %s for user %s", - ldap_dict['c'], - user_dn - ) - continue + # decode bytes to str + ldap_dict = decode_nested_dict(ldap_dict) _truncate('sn', 'last_name', ldap_dict) user = self._create_or_update_user(user_dn, ldap_dict) self.nested_groups.handle(user) From 4565808184fd317fbd462b2fea9219fb0e68bb39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Szulc?= Date: Mon, 4 Nov 2024 14:06:30 +0100 Subject: [PATCH 20/20] Updated changelog for 20241104.2 version. --- debian/changelog | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/debian/changelog b/debian/changelog index d46dd621b5..03d60072cb 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,14 @@ +ralph (20241104.2) bionic; urgency=medium + + [ awieckowski ] + * Increase max user department field length + * Add missing migration + + [ Paweł Szulc ] + * Fix ldap sync when bytes received + + -- Paweł Szulc Mon, 04 Nov 2024 13:06:26 +0000 + ralph (20241104.1) bionic; urgency=medium * Fix method to get limit choices