From 166991bc7aae6db9d77c14f3d7ac7a329cde21b8 Mon Sep 17 00:00:00 2001 From: Manuel Jeckelmann Date: Tue, 5 Jan 2016 14:02:56 +0100 Subject: [PATCH 01/42] Extended test config to test using django19 --- .travis.yml | 4 ++++ tox.ini | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 85a822a..c791412 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,12 +8,16 @@ env: - TOX_ENV=py27-django17-sqlite - TOX_ENV=py27-django18-pg - TOX_ENV=py27-django18-sqlite + - TOX_ENV=py27-django19-pg + - TOX_ENV=py27-django19-sqlite - TOX_ENV=py34-django16-pg - TOX_ENV=py34-django16-sqlite - TOX_ENV=py34-django17-pg - TOX_ENV=py34-django17-sqlite - TOX_ENV=py34-django18-pg - TOX_ENV=py34-django18-sqlite + - TOX_ENV=py34-django19-pg + - TOX_ENV=py34-django19-sqlite # Enable PostgreSQL usage addons: diff --git a/tox.ini b/tox.ini index 37c0bdc..a4db0a2 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,7 @@ [tox] envlist = - py{27,34}-django{16,17,18}-{sqlite,pg} + py{27,34}-django{16,17,18,19}-{sqlite,pg} [testenv] deps = @@ -13,6 +13,7 @@ deps = django16: django>=1.6,<1.7 django17: django>=1.7,<1.8 django18: django>=1.8,<1.9 + django19: django>=1.9,<2.0 pg: psycopg2 commands = pg: coverage run --source=versions ./manage.py test --settings={env:TOX_PG_CONF:cleanerversion.settings.pg} From 792c4005267982f79f006431be24a33510daee8c Mon Sep 17 00:00:00 2001 From: Manuel Jeckelmann Date: Tue, 5 Jan 2016 14:48:42 +0100 Subject: [PATCH 02/42] Remove test environments for django16 due to EOL --- .travis.yml | 4 ---- tox.ini | 3 +-- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index c791412..ce7ae01 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,16 +2,12 @@ language: python python: "2.7" env: - - TOX_ENV=py27-django16-pg - - TOX_ENV=py27-django16-sqlite - TOX_ENV=py27-django17-pg - TOX_ENV=py27-django17-sqlite - TOX_ENV=py27-django18-pg - TOX_ENV=py27-django18-sqlite - TOX_ENV=py27-django19-pg - TOX_ENV=py27-django19-sqlite - - TOX_ENV=py34-django16-pg - - TOX_ENV=py34-django16-sqlite - TOX_ENV=py34-django17-pg - TOX_ENV=py34-django17-sqlite - TOX_ENV=py34-django18-pg diff --git a/tox.ini b/tox.ini index a4db0a2..aee3886 100644 --- a/tox.ini +++ b/tox.ini @@ -5,12 +5,11 @@ [tox] envlist = - py{27,34}-django{16,17,18,19}-{sqlite,pg} + py{27,34}-django{17,18,19}-{sqlite,pg} [testenv] deps = coverage - django16: django>=1.6,<1.7 django17: django>=1.7,<1.8 django18: django>=1.8,<1.9 django19: django>=1.9,<2.0 From 2198107b6db6dabc538d6d5f9d4c46040bcf1e30 Mon Sep 17 00:00:00 2001 From: Manuel Jeckelmann Date: Mon, 2 May 2016 22:33:53 +0200 Subject: [PATCH 03/42] Separated descriptors and fields from models; Added first 1.9 compatibility hacks; Most tests passing again --- cleanerversion/settings/base.py | 2 +- versions/admin.py | 2 +- versions/deletion.py | 2 +- versions/descriptors.py | 516 ++++++++++++++++++++++ versions/fields.py | 344 +++++++++++++++ versions/models.py | 738 ++------------------------------ versions/util/__init__.py | 7 + versions/util/postgresql.py | 2 +- versions_tests/models.py | 3 +- 9 files changed, 898 insertions(+), 718 deletions(-) create mode 100644 versions/descriptors.py create mode 100644 versions/fields.py diff --git a/cleanerversion/settings/base.py b/cleanerversion/settings/base.py index 546abce..7dfdfbf 100644 --- a/cleanerversion/settings/base.py +++ b/cleanerversion/settings/base.py @@ -64,7 +64,7 @@ 'django.contrib.messages', 'django.contrib.staticfiles', 'versions', - 'versions_tests', + 'versions_tests.apps.VersionsTestsConfig', ) MIDDLEWARE_CLASSES = ( diff --git a/versions/admin.py b/versions/admin.py index a4a5d23..d542a14 100644 --- a/versions/admin.py +++ b/versions/admin.py @@ -106,7 +106,7 @@ def queryset(self, request, queryset): class VersionedAdminChecks(ModelAdminChecks): - def _check_exclude(self, cls, model): + def _check_exclude(self, cls, model=None): """ Required to suppress error about exclude not being a tuple since we are using @property to dynamically change it """ diff --git a/versions/deletion.py b/versions/deletion.py index 91dc45f..b5fe02a 100644 --- a/versions/deletion.py +++ b/versions/deletion.py @@ -68,7 +68,7 @@ def delete(self, timestamp): # In the case of a SET.. method, clone before changing the value (if it hasn't already been # cloned) updated_instances = set() - if not(isinstance(field, versions.models.VersionedForeignKey) and field.rel.on_delete == CASCADE): + if not(isinstance(field, versions.fields.VersionedForeignKey) and field.rel.on_delete == CASCADE): for instance in instances: # Clone before updating cloned = id_map.get(instance.pk, None) diff --git a/versions/descriptors.py b/versions/descriptors.py new file mode 100644 index 0000000..f92a922 --- /dev/null +++ b/versions/descriptors.py @@ -0,0 +1,516 @@ +from django import VERSION +from django.core.exceptions import SuspiciousOperation, FieldDoesNotExist +from django.db import router, transaction +from django.db.models.base import Model +from django.db.models.query_utils import Q + +from django.utils.functional import cached_property +from versions.models import Versionable, VersionedQuerySet + +from versions.util import get_utc_now + +if VERSION[:2] >= (1, 9): + # With Django 1.9 related descriptor classes have been renamed: + # ReverseSingleRelatedObjectDescriptor => ForwardManyToOneDescriptor + # ForeignRelatedObjectsDescriptor => ReverseManyToOneDescriptor + # ReverseManyRelatedObjectsDescriptor => ManyToManyDescriptor + # ManyRelatedObjectsDescriptor => ManyToManyDescriptor + # (new) => ReverseOneToOneDescriptor + from django.db.models.fields.related import (ForwardManyToOneDescriptor, ReverseManyToOneDescriptor, + ManyToManyDescriptor, ReverseOneToOneDescriptor) + from django.db.models.fields.related_descriptors import create_forward_many_to_many_manager +else: + from django.db.models.fields.related import (ReverseSingleRelatedObjectDescriptor, + ReverseManyRelatedObjectsDescriptor, + ManyRelatedObjectsDescriptor, + ForeignRelatedObjectsDescriptor, + create_many_related_manager) + + + +def matches_querytime(instance, querytime): + """ + Checks whether the given instance satisfies the given QueryTime object. + + :param instance: an instance of Versionable + :param querytime: QueryTime value to check against + """ + if not querytime.active: + return True + + if not querytime.time: + return instance.version_end_date is None + + return (instance.version_start_date <= querytime.time + and (instance.version_end_date is None or instance.version_end_date > querytime.time)) + +if VERSION[:2] >= (1,9): + class VersionedForwardManyToOneDescriptor(ForwardManyToOneDescriptor): + """ + + """ + pass + vforward_many_to_one_descriptor_class = VersionedForwardManyToOneDescriptor +else: + class VersionedReverseSingleRelatedObjectDescriptor(ReverseSingleRelatedObjectDescriptor): + """ + A ReverseSingleRelatedObjectDescriptor-typed object gets inserted, when a ForeignKey + is defined in a Django model. This is one part of the analogue for versioned items. + + Unfortunately, we need to run two queries. The first query satisfies the foreign key + constraint. After extracting the identity information and combining it with the datetime- + stamp, we are able to fetch the historic element. + """ + pass + vforward_many_to_one_descriptor_class = VersionedReverseSingleRelatedObjectDescriptor + +def vforward_many_to_one_descriptor_getter(self, instance, instance_type=None): + """ + The getter method returns the object, which points instance, e.g. choice.poll returns + a Poll instance, whereas the Poll class defines the ForeignKey. + :param instance: The object on which the property was accessed + :param instance_type: The type of the instance object + :return: Returns a Versionable + """ + current_elt = super(self.__class__, self).__get__(instance, instance_type) + + if instance is None: + return self + + if not current_elt: + return None + + if not isinstance(current_elt, Versionable): + raise TypeError("VersionedForeignKey target is of type " + + str(type(current_elt)) + + ", which is not a subclass of Versionable") + + if hasattr(instance, '_querytime'): + # If current_elt matches the instance's querytime, there's no need to make a database query. + if matches_querytime(current_elt, instance._querytime): + current_elt._querytime = instance._querytime + return current_elt + + return current_elt.__class__.objects.as_of(instance._querytime.time).get(identity=current_elt.identity) + else: + return current_elt.__class__.objects.current.get(identity=current_elt.identity) + +vforward_many_to_one_descriptor_class.__get__ = vforward_many_to_one_descriptor_getter + + + +if VERSION[:2] >= (1,9): + class VersionedReverseManyToOneDescriptor(ReverseManyToOneDescriptor): + pass + + vreverse_many_to_one_descriptor_class = VersionedReverseManyToOneDescriptor +else: + class VersionedForeignRelatedObjectsDescriptor(ForeignRelatedObjectsDescriptor): + """ + This descriptor generates the manager class that is used on the related object of a ForeignKey relation + (i.e. the reverse-ForeignKey field manager). + """ + pass + + vreverse_many_to_one_descriptor_class = VersionedForeignRelatedObjectsDescriptor + + +def vreverse_many_to_one_descriptor_related_manager_cls_property(self): + # return create_versioned_related_manager + manager_cls = super(self.__class__, self).related_manager_cls + if VERSION[:2] >= (1, 9): + # TODO: Define, what field has to be taken over here, self.rel/self.field? The WhineDrinker.hats test seems to be a good one for testing this + rel_field = self.rel + elif hasattr(self, 'related'): + rel_field = self.related.field + else: + rel_field = self.field + + class VersionedRelatedManager(manager_cls): + def __init__(self, instance): + super(VersionedRelatedManager, self).__init__(instance) + + # This is a hack, in order to get the versioned related objects + for key in self.core_filters.keys(): + if '__exact' in key or '__' not in key: + self.core_filters[key] = instance.identity + + def get_queryset(self): + queryset = super(VersionedRelatedManager, self).get_queryset() + # Do not set the query time if it is already correctly set. queryset.as_of() returns a clone + # of the queryset, and this will destroy the prefetched objects cache if it exists. + if isinstance(queryset, VersionedQuerySet) \ + and self.instance._querytime.active \ + and queryset.querytime != self.instance._querytime: + queryset = queryset.as_of(self.instance._querytime.time) + return queryset + + def add(self, *objs): + cloned_objs = () + for obj in objs: + if not isinstance(obj, Versionable): + raise TypeError("Trying to add a non-Versionable to a VersionedForeignKey relationship") + cloned_objs += (obj.clone(),) + super(VersionedRelatedManager, self).add(*cloned_objs) + + # clear() and remove() are present if the FK is nullable + if 'clear' in dir(manager_cls): + def clear(self, **kwargs): + """ + Overridden to ensure that the current queryset is used, and to clone objects before they + are removed, so that history is not lost. + """ + bulk = kwargs.pop('bulk', True) + db = router.db_for_write(self.model, instance=self.instance) + queryset = self.current.using(db) + with transaction.atomic(using=db, savepoint=False): + cloned_pks = [obj.clone().pk for obj in queryset] + update_qs = self.current.filter(pk__in=cloned_pks) + if VERSION[:2] == (1, 6): + update_qs.update(**{rel_field.name: None}) + else: + self._clear(update_qs, bulk) + + if 'remove' in dir(manager_cls): + def remove(self, *objs): + val = rel_field.get_foreign_related_value(self.instance) + cloned_objs = () + for obj in objs: + # Is obj actually part of this descriptor set? Otherwise, silently go over it, since Django + # handles that case + if rel_field.get_local_related_value(obj) == val: + # Silently pass over non-versionable items + if not isinstance(obj, Versionable): + raise TypeError( + "Trying to remove a non-Versionable from a VersionedForeignKey realtionship") + cloned_objs += (obj.clone(),) + super(VersionedRelatedManager, self).remove(*cloned_objs) + + return VersionedRelatedManager + +vreverse_many_to_one_descriptor_class.related_manager_cls = cached_property(vreverse_many_to_one_descriptor_related_manager_cls_property) + + +class VersionedManyToManyDescriptor(ManyToManyDescriptor): + @cached_property + def related_manager_cls(self): + model = self.rel.related_model if self.reverse else self.rel.model + return create_versioned_forward_many_to_many_manager( + model._default_manager.__class__, + self.rel, + reverse=self.reverse, + ) + + def __set__(self, instance, value): + """ + Completely overridden to avoid bulk deletion that happens when the parent method calls clear(). + + The parent method's logic is basically: clear all in bulk, then add the given objects in bulk. + Instead, we figure out which ones are being added and removed, and call add and remove for these values. + This lets us retain the versioning information. + + Since this is a many-to-many relationship, it is assumed here that the django.db.models.deletion.Collector + logic, that is used in clear(), is not necessary here. Collector collects related models, e.g. ones that should + also be deleted because they have a ON CASCADE DELETE relationship to the object, or, in the case of + "Multi-table inheritance", are parent objects. + + :param instance: The instance on which the getter was called + :param value: iterable of items to set + """ + + if not instance.is_current: + raise SuspiciousOperation( + "Related values can only be directly set on the current version of an object") + + if not self.field.rel.through._meta.auto_created: + opts = self.field.rel.through._meta + raise AttributeError(("Cannot set values on a ManyToManyField which specifies an intermediary model. " + "Use %s.%s's Manager instead.") % (opts.app_label, opts.object_name)) + + manager = self.__get__(instance) + # Below comment is from parent __set__ method. We'll force evaluation, too: + # clear() can change expected output of 'value' queryset, we force evaluation + # of queryset before clear; ticket #19816 + value = tuple(value) + + being_removed, being_added = self.get_current_m2m_diff(instance, value) + timestamp = get_utc_now() + manager.remove_at(timestamp, *being_removed) + manager.add_at(timestamp, *being_added) + + def get_current_m2m_diff(self, instance, new_objects): + """ + :param instance: Versionable object + :param new_objects: objects which are about to be associated with instance + :return: (being_removed id list, being_added id list) + :rtype : tuple + """ + new_ids = self.pks_from_objects(new_objects) + relation_manager = self.__get__(instance) + + filter = Q(**{relation_manager.source_field.attname: instance.pk}) + qs = self.through.objects.current.filter(filter) + try: + # Django 1.7 + target_name = relation_manager.target_field.attname + except AttributeError: + # Django 1.6 + target_name = relation_manager.through._meta.get_field_by_name( + relation_manager.target_field_name)[0].attname + current_ids = set(qs.values_list(target_name, flat=True)) + + being_removed = current_ids - new_ids + being_added = new_ids - current_ids + return list(being_removed), list(being_added) + + def pks_from_objects(self, objects): + """ + Extract all the primary key strings from the given objects. Objects may be Versionables, or bare primary keys. + :rtype : set + """ + return {o.pk if isinstance(o, Model) else o for o in objects} + +if VERSION[:2] < (1, 9): + class VersionedReverseManyRelatedObjectsDescriptor(ReverseManyRelatedObjectsDescriptor): + """ + Beside having a very long name, this class is useful when it comes to versioning the + ReverseManyRelatedObjectsDescriptor (huhu!!). The main part is the exposure of the + 'related_manager_cls' property + """ + + def __set__(self, instance, value): + """ + Completely overridden to avoid bulk deletion that happens when the parent method calls clear(). + + The parent method's logic is basically: clear all in bulk, then add the given objects in bulk. + Instead, we figure out which ones are being added and removed, and call add and remove for these values. + This lets us retain the versioning information. + + Since this is a many-to-many relationship, it is assumed here that the django.db.models.deletion.Collector + logic, that is used in clear(), is not necessary here. Collector collects related models, e.g. ones that should + also be deleted because they have a ON CASCADE DELETE relationship to the object, or, in the case of + "Multi-table inheritance", are parent objects. + + :param instance: The instance on which the getter was called + :param value: iterable of items to set + """ + + if not instance.is_current: + raise SuspiciousOperation( + "Related values can only be directly set on the current version of an object") + + if not self.field.rel.through._meta.auto_created: + opts = self.field.rel.through._meta + raise AttributeError(("Cannot set values on a ManyToManyField which specifies an intermediary model. " + "Use %s.%s's Manager instead.") % (opts.app_label, opts.object_name)) + + manager = self.__get__(instance) + # Below comment is from parent __set__ method. We'll force evaluation, too: + # clear() can change expected output of 'value' queryset, we force evaluation + # of queryset before clear; ticket #19816 + value = tuple(value) + + being_removed, being_added = self.get_current_m2m_diff(instance, value) + timestamp = get_utc_now() + manager.remove_at(timestamp, *being_removed) + manager.add_at(timestamp, *being_added) + + def get_current_m2m_diff(self, instance, new_objects): + """ + :param instance: Versionable object + :param new_objects: objects which are about to be associated with instance + :return: (being_removed id list, being_added id list) + :rtype : tuple + """ + new_ids = self.pks_from_objects(new_objects) + relation_manager = self.__get__(instance) + + filter = Q(**{relation_manager.source_field.attname: instance.pk}) + qs = self.through.objects.current.filter(filter) + try: + # Django 1.7 + target_name = relation_manager.target_field.attname + except AttributeError: + # Django 1.6 + target_name = relation_manager.through._meta.get_field_by_name( + relation_manager.target_field_name)[0].attname + current_ids = set(qs.values_list(target_name, flat=True)) + + being_removed = current_ids - new_ids + being_added = new_ids - current_ids + return list(being_removed), list(being_added) + + def pks_from_objects(self, objects): + """ + Extract all the primary key strings from the given objects. Objects may be Versionables, or bare primary keys. + :rtype : set + """ + return {o.pk if isinstance(o, Model) else o for o in objects} + + @cached_property + def related_manager_cls(self): + return create_versioned_forward_many_to_many_manager( + self.field.rel.to._default_manager.__class__, + self.field.rel + ) + + + class VersionedManyRelatedObjectsDescriptor(ManyRelatedObjectsDescriptor): + """ + Beside having a very long name, this class is useful when it comes to versioning the + ManyRelatedObjectsDescriptor (huhu!!). The main part is the exposure of the + 'related_manager_cls' property + """ + + # via_field_name = None + + def __init__(self, related, via_field_name): + super(VersionedManyRelatedObjectsDescriptor, self).__init__(related) + self.via_field_name = via_field_name + + @cached_property + def related_manager_cls(self): + return create_versioned_forward_many_to_many_manager( + self.related.model._default_manager.__class__, + self.related.field.rel + ) + + +def create_versioned_forward_many_to_many_manager(superclass, rel, reverse=None): + if VERSION[:2] >= (1, 9): + many_related_manager_klass = create_forward_many_to_many_manager(superclass, rel, reverse) + else: + many_related_manager_klass = create_many_related_manager(superclass, rel) + + class VersionedManyRelatedManager(many_related_manager_klass): + def __init__(self, *args, **kwargs): + super(VersionedManyRelatedManager, self).__init__(*args, **kwargs) + # Additional core filters are: version_start_date <= t & (version_end_date > t | version_end_date IS NULL) + # but we cannot work with the Django core filters, since they don't support ORing filters, which + # is a thing we need to consider the "version_end_date IS NULL" case; + # So, we define our own set of core filters being applied when versioning + try: + version_start_date_field = self.through._meta.get_field('version_start_date') + version_end_date_field = self.through._meta.get_field('version_end_date') + except FieldDoesNotExist as e: + if VERSION[:2] >= (1, 8): + fields = [f.name for f in self.through._meta.get_fields()] + else: + fields = self.through._meta.get_all_field_names() + print(str(e) + "; available fields are " + ", ".join(fields)) + raise e + # FIXME: this probably does not work when auto-referencing + + def get_queryset(self): + """ + Add a filter to the queryset, limiting the results to be pointed by relationship that are + valid for the given timestamp (which is taken at the current instance, or set to now, if not + available). + Long story short, apply the temporal validity filter also to the intermediary model. + """ + queryset = super(VersionedManyRelatedManager, self).get_queryset() + if hasattr(queryset, 'querytime'): + if self.instance._querytime.active and self.instance._querytime != queryset.querytime: + queryset = queryset.as_of(self.instance._querytime.time) + return queryset + + def _remove_items(self, source_field_name, target_field_name, *objs): + """ + Instead of removing items, we simply set the version_end_date of the current item to the + current timestamp --> t[now]. + Like that, there is no more current entry having that identity - which is equal to + not existing for timestamps greater than t[now]. + """ + return self._remove_items_at(None, source_field_name, target_field_name, *objs) + + def _remove_items_at(self, timestamp, source_field_name, target_field_name, *objs): + if objs: + if timestamp is None: + timestamp = get_utc_now() + old_ids = set() + for obj in objs: + if isinstance(obj, self.model): + # The Django 1.7-way is preferred + if hasattr(self, 'target_field'): + fk_val = self.target_field.get_foreign_related_value(obj)[0] + # But the Django 1.6.x -way is supported for backward compatibility + elif hasattr(self, '_get_fk_val'): + fk_val = self._get_fk_val(obj, target_field_name) + else: + raise TypeError("We couldn't find the value of the foreign key, this might be due to the " + "use of an unsupported version of Django") + old_ids.add(fk_val) + else: + old_ids.add(obj) + db = router.db_for_write(self.through, instance=self.instance) + qs = self.through._default_manager.using(db).filter(**{ + source_field_name: self.instance.id, + '%s__in' % target_field_name: old_ids + }).as_of(timestamp) + for relation in qs: + relation._delete_at(timestamp) + + if 'add' in dir(many_related_manager_klass): + def add(self, *objs): + if not self.instance.is_current: + raise SuspiciousOperation( + "Adding many-to-many related objects is only possible on the current version") + + # The ManyRelatedManager.add() method uses the through model's default manager to get + # a queryset when looking at which objects already exist in the database. + # In order to restrict the query to the current versions when that is done, + # we temporarily replace the queryset's using method so that the version validity + # condition can be specified. + klass = self.through._default_manager.get_queryset().__class__ + __using_backup = klass.using + + def using_replacement(self, *args, **kwargs): + qs = __using_backup(self, *args, **kwargs) + return qs.as_of(None) + + klass.using = using_replacement + super(VersionedManyRelatedManager, self).add(*objs) + klass.using = __using_backup + + def add_at(self, timestamp, *objs): + """ + This function adds an object at a certain point in time (timestamp) + """ + + # First off, define the new constructor + def _through_init(self, *args, **kwargs): + super(self.__class__, self).__init__(*args, **kwargs) + self.version_birth_date = timestamp + self.version_start_date = timestamp + + # Through-classes have an empty constructor, so it can easily be overwritten when needed; + # This is not the default case, so the overwrite only takes place when we "modify the past" + self.through.__init_backup__ = self.through.__init__ + self.through.__init__ = _through_init + + # Do the add operation + self.add(*objs) + + # Remove the constructor again (by replacing it with the original empty constructor) + self.through.__init__ = self.through.__init_backup__ + del self.through.__init_backup__ + + add_at.alters_data = True + + if 'remove' in dir(many_related_manager_klass): + def remove_at(self, timestamp, *objs): + """ + Performs the act of removing specified relationships at a specified time (timestamp); + So, not the objects at a given time are removed, but their relationship! + """ + self._remove_items_at(timestamp, self.source_field_name, self.target_field_name, *objs) + + # For consistency, also handle the symmetrical case + if self.symmetrical: + self._remove_items_at(timestamp, self.target_field_name, self.source_field_name, *objs) + + remove_at.alters_data = True + + return VersionedManyRelatedManager + +# create_versioned_many_related_manager = create_versioned_forward_many_to_many_manager diff --git a/versions/fields.py b/versions/fields.py new file mode 100644 index 0000000..2750877 --- /dev/null +++ b/versions/fields.py @@ -0,0 +1,344 @@ +import six + +from django import VERSION +from django.apps import apps +from django.db.models.fields.related import ForeignKey, ManyToManyField, RECURSIVE_RELATIONSHIP_CONSTANT +from django.db.models.sql.datastructures import Join +from django.db.models.sql.where import ExtraWhere, WhereNode + +if VERSION[:2] >= (1, 9): + # With Django 1.9 related descriptor classes have been renamed: + # ReverseSingleRelatedObjectDescriptor => ForwardManyToOneDescriptor + # ForeignRelatedObjectsDescriptor => ReverseManyToOneDescriptor + # ReverseManyRelatedObjectsDescriptor => ManyToManyDescriptor + # ManyRelatedObjectsDescriptor => ManyToManyDescriptor + # (new) => ReverseOneToOneDescriptor + # from django.db.models.fields.related import (ForwardManyToOneDescriptor, ReverseManyToOneDescriptor, + # ManyToManyDescriptor, ReverseOneToOneDescriptor) + from descriptors import (VersionedForwardManyToOneDescriptor, + VersionedReverseManyToOneDescriptor, + VersionedManyToManyDescriptor) +else: + from descriptors import (VersionedReverseSingleRelatedObjectDescriptor, + VersionedForeignRelatedObjectsDescriptor, + VersionedReverseManyRelatedObjectsDescriptor, + VersionedManyRelatedObjectsDescriptor) + # from django.db.models.fields.related import (ReverseSingleRelatedObjectDescriptor, + # ReverseManyRelatedObjectsDescriptor, ManyToManyField, + # ManyRelatedObjectsDescriptor, create_many_related_manager, + # ForeignRelatedObjectsDescriptor, RECURSIVE_RELATIONSHIP_CONSTANT) +from models import Versionable + + +class VersionedForeignKey(ForeignKey): + """ + We need to replace the standard ForeignKey declaration in order to be able to introduce + the VersionedReverseSingleRelatedObjectDescriptor, which allows to go back in time... + We also want to allow keeping track of any as_of time so that joins can be restricted + based on that. + """ + + def __init__(self, *args, **kwargs): + super(VersionedForeignKey, self).__init__(*args, **kwargs) + + def contribute_to_class(self, cls, name, virtual_only=False): + super(VersionedForeignKey, self).contribute_to_class(cls, name, virtual_only) + if VERSION[:2] >= (1, 9): + setattr(cls, self.name, VersionedForwardManyToOneDescriptor(self)) + else: + setattr(cls, self.name, VersionedReverseSingleRelatedObjectDescriptor(self)) + + def contribute_to_related_class(self, cls, related): + """ + Override ForeignKey's methods, and replace the descriptor, if set by the parent's methods + """ + # Internal FK's - i.e., those with a related name ending with '+' - + # and swapped models don't get a related descriptor. + super(VersionedForeignKey, self).contribute_to_related_class(cls, related) + accessor_name = related.get_accessor_name() + if hasattr(cls, accessor_name): + if VERSION[:2] >= (1, 9): + setattr(cls, accessor_name, VersionedReverseManyToOneDescriptor(related)) + else: + setattr(cls, accessor_name, VersionedForeignRelatedObjectsDescriptor(related)) + + def get_extra_restriction(self, where_class, alias, remote_alias): + """ + Overrides ForeignObject's get_extra_restriction function that returns an SQL statement which is appended to a + JOIN's conditional filtering part + + :return: SQL conditional statement + :rtype: WhereNode + """ + historic_sql = '''{alias}.version_start_date <= %s + AND ({alias}.version_end_date > %s OR {alias}.version_end_date is NULL )''' + current_sql = '''{alias}.version_end_date is NULL''' + # How 'bout creating an ExtraWhere here, without params + return where_class([VersionedExtraWhere(historic_sql=historic_sql, current_sql=current_sql, alias=alias, + remote_alias=remote_alias)]) + + def get_joining_columns(self, reverse_join=False): + """ + Get and return joining columns defined by this foreign key relationship + + :return: A tuple containing the column names of the tables to be joined (, ) + :rtype: tuple + """ + source = self.reverse_related_fields if reverse_join else self.related_fields + joining_columns = tuple() + for lhs_field, rhs_field in source: + lhs_col_name = lhs_field.column + rhs_col_name = rhs_field.column + # Test whether + # - self is the current ForeignKey relationship + # - self was not auto_created (e.g. is not part of a M2M relationship) + if self is lhs_field and not self.auto_created: + if rhs_col_name == Versionable.VERSION_IDENTIFIER_FIELD: + rhs_col_name = Versionable.OBJECT_IDENTIFIER_FIELD + elif self is rhs_field and not self.auto_created: + if lhs_col_name == Versionable.VERSION_IDENTIFIER_FIELD: + lhs_col_name = Versionable.OBJECT_IDENTIFIER_FIELD + joining_columns = joining_columns + ((lhs_col_name, rhs_col_name),) + return joining_columns + +class VersionedManyToManyField(ManyToManyField): + def __init__(self, *args, **kwargs): + super(VersionedManyToManyField, self).__init__(*args, **kwargs) + + def contribute_to_class(self, cls, name): + """ + Called at class type creation. So, this method is called, when metaclasses get created + """ + # self.rel.through needs to be set prior to calling super, since super(...).contribute_to_class refers to it. + # Classes pointed to by a string do not need to be resolved here, since Django does that at a later point in + # time - which is nice... ;) + # + # Superclasses take care of: + # - creating the through class if unset + # - resolving the through class if it's a string + # - resolving string references within the through class + if not self.rel.through and not cls._meta.abstract and not cls._meta.swapped: + self.rel.through = VersionedManyToManyField.create_versioned_many_to_many_intermediary_model(self, cls, + name) + super(VersionedManyToManyField, self).contribute_to_class(cls, name) + + # Overwrite the descriptor + if hasattr(cls, self.name): + if VERSION[:2] >= (1, 9): + setattr(cls, self.name, VersionedManyToManyDescriptor(self.remote_field)) + else: + setattr(cls, self.name, VersionedReverseManyRelatedObjectsDescriptor(self)) + + def contribute_to_related_class(self, cls, related): + """ + Called at class type creation. So, this method is called, when metaclasses get created + """ + super(VersionedManyToManyField, self).contribute_to_related_class(cls, related) + accessor_name = related.get_accessor_name() + if accessor_name and hasattr(cls, accessor_name): + if VERSION[:2] >= (1, 9): + descriptor = VersionedManyToManyDescriptor(related, accessor_name) + else: + descriptor = VersionedManyRelatedObjectsDescriptor(related, accessor_name) + setattr(cls, accessor_name, descriptor) + if hasattr(cls._meta, 'many_to_many_related') and isinstance(cls._meta.many_to_many_related, list): + cls._meta.many_to_many_related.append(descriptor) + else: + cls._meta.many_to_many_related = [descriptor] + + @staticmethod + def create_versioned_many_to_many_intermediary_model(field, cls, field_name): + # Let's not care too much on what flags could potentially be set on that intermediary class (e.g. managed, etc) + # Let's play the game, as if the programmer had specified a class within his models... Here's how. + + from_ = cls._meta.model_name + to_model = field.rel.to + + # Force 'to' to be a string (and leave the hard work to Django) + if not isinstance(field.rel.to, six.string_types): + to_model = '%s.%s' % (field.rel.to._meta.app_label, field.rel.to._meta.object_name) + to = field.rel.to._meta.object_name.lower() + else: + to = to_model.lower() + name = '%s_%s' % (from_, field_name) + + if field.rel.to == RECURSIVE_RELATIONSHIP_CONSTANT or to == cls._meta.object_name: + from_ = 'from_%s' % to + to = 'to_%s' % to + to_model = cls + + # Since Django 1.7, a migration mechanism is shipped by default with Django. This migration module loads all + # declared apps' models inside a __fake__ module. + # This means that the models can be already loaded and registered by their original module, when we + # reach this point of the application and therefore there is no need to load them a second time. + if VERSION[:2] >= (1, 7) and cls.__module__ == '__fake__': + try: + # Check the apps for an already registered model + if VERSION[:2] >= (1, 9): + return apps.get_model(cls._meta.app_label, str(name)) + else: + return apps.get_registered_model(cls._meta.app_label, str(name)) + except KeyError: + # The model has not been registered yet, so continue + pass + + meta = type('Meta', (object,), { + # 'unique_together': (from_, to), + 'auto_created': cls, + 'db_tablespace': cls._meta.db_tablespace, + 'app_label': cls._meta.app_label, + }) + return type(str(name), (Versionable,), { + 'Meta': meta, + '__module__': cls.__module__, + from_: VersionedForeignKey(cls, related_name='%s+' % name, auto_created=name), + to: VersionedForeignKey(to_model, related_name='%s+' % name, auto_created=name), + }) + +class VersionedExtraWhere(ExtraWhere): + """ + A specific implementation of ExtraWhere; + Before as_sql can be called on an object, ensure that calls to + - set_as_of and + - set_joined_alias + have been done + """ + + def __init__(self, historic_sql, current_sql, alias, remote_alias): + super(VersionedExtraWhere, self).__init__(sqls=[], params=[]) + self.historic_sql = historic_sql + self.current_sql = current_sql + self.alias = alias + self.related_alias = remote_alias + self._as_of_time_set = False + self.as_of_time = None + self._joined_alias = None + + def set_as_of(self, as_of_time): + self.as_of_time = as_of_time + self._as_of_time_set = True + + def set_joined_alias(self, joined_alias): + """ + Takes the alias that is being joined to the query and applies the query + time constraint to its table + + :param str joined_alias: The table name of the alias + """ + self._joined_alias = joined_alias + + def as_sql(self, qn=None, connection=None): + sql = "" + params = [] + + # Fail fast for inacceptable cases + if self._as_of_time_set and not self._joined_alias: + raise ValueError("joined_alias is not set, but as_of is; this is a conflict!") + + # Set the SQL string in dependency of whether as_of_time was set or not + if self._as_of_time_set: + if self.as_of_time: + sql = self.historic_sql + params = [self.as_of_time] * 2 + # 2 is the number of occurences of the timestamp in an as_of-filter expression + else: + # If as_of_time was set to None, we're dealing with a query for "current" values + sql = self.current_sql + else: + # No as_of_time has been set; Perhaps, as_of was not part of the query -> That's OK + pass + + # By here, the sql string is defined if an as_of_time was provided + if self._joined_alias: + sql = sql.format(alias=self._joined_alias) + + # Set the final sqls + # self.sqls needs to be set before the call to parent + if sql: + self.sqls = [sql] + else: + self.sqls = ["1=1"] + self.params = params + return super(VersionedExtraWhere, self).as_sql(qn, connection) + + +class VersionedWhereNode(WhereNode): + def as_sql(self, qn, connection): + """ + This method identifies joined table aliases in order for VersionedExtraWhere.as_sql() + to be able to add time restrictions for those tables based on the VersionedQuery's + querytime value. + + :param qn: In Django 1.7 & 1.8 this is a compiler; in 1.6, it's an instance-method + :param connection: A DB connection + :return: A tuple consisting of (sql_string, result_params) + """ + # self.children is an array of VersionedExtraWhere-objects + for child in self.children: + if isinstance(child, VersionedExtraWhere) and not child.params: + try: + # Django 1.7 & 1.8 handles compilers as objects + _query = qn.query + except AttributeError: + # Django 1.6 handles compilers as instancemethods + _query = qn.__self__.query + query_time = _query.querytime.time + apply_query_time = _query.querytime.active + alias_map = _query.alias_map + # In Django 1.6 & 1.7, use the join_map to know, what *table* gets joined to which + # *left-hand sided* table + # In Django 1.8, use the Join objects in alias_map + if hasattr(_query, 'join_map'): + self._set_child_joined_alias_using_join_map(child, _query.join_map, alias_map) + else: + self._set_child_joined_alias(child, alias_map) + if apply_query_time: + # Add query parameters that have not been added till now + child.set_as_of(query_time) + else: + # Remove the restriction if it's not required + child.sqls = [] + return super(VersionedWhereNode, self).as_sql(qn, connection) + + @staticmethod + def _set_child_joined_alias_using_join_map(child, join_map, alias_map): + """ + Set the joined alias on the child, for Django <= 1.7.x. + :param child: + :param join_map: + :param alias_map: + """ + for lhs, table, join_cols in join_map: + if lhs is None: + continue + if lhs == child.alias: + relevant_alias = child.related_alias + elif lhs == child.related_alias: + relevant_alias = child.alias + else: + continue + + join_info = alias_map[relevant_alias] + if join_info.join_type is None: + continue + + if join_info.lhs_alias in [child.alias, child.related_alias]: + child.set_joined_alias(relevant_alias) + break + + @staticmethod + def _set_child_joined_alias(child, alias_map): + """ + Set the joined alias on the child, for Django >= 1.8.0 + :param child: + :param alias_map: + """ + for table in alias_map: + join = alias_map[table] + if not isinstance(join, Join): + continue + lhs = join.parent_alias + if (lhs == child.alias and table == child.related_alias) \ + or (lhs == child.related_alias and table == child.alias): + child.set_joined_alias(table) + break diff --git a/versions/models.py b/versions/models.py index 602b41b..1db98f6 100644 --- a/versions/models.py +++ b/versions/models.py @@ -20,25 +20,26 @@ from django import VERSION +from versions.util import get_utc_now + if VERSION[:2] >= (1, 8): from django.db.models.sql.datastructures import Join if VERSION[:2] >= (1, 7): from django.apps.registry import apps from django.core.exceptions import SuspiciousOperation, ObjectDoesNotExist from django.db import transaction -from django.db.models.base import Model from django.db.models import Q from django.db.models.constants import LOOKUP_SEP -from django.db.models.fields import FieldDoesNotExist -from django.db.models.fields.related import (ForeignKey, ReverseSingleRelatedObjectDescriptor, - ReverseManyRelatedObjectsDescriptor, ManyToManyField, - ManyRelatedObjectsDescriptor, create_many_related_manager, - ForeignRelatedObjectsDescriptor, RECURSIVE_RELATIONSHIP_CONSTANT) -from django.db.models.query import QuerySet, ValuesListQuerySet, ValuesQuerySet +from django.db.models.fields.related import ForeignKey, ManyToManyField, RECURSIVE_RELATIONSHIP_CONSTANT + +from django.db.models.query import QuerySet +if VERSION[:2] >= (1, 9): + from django.db.models.query import ModelIterable +else: + from django.db.models.query import ValuesListQuerySet, ValuesQuerySet from django.db.models.signals import post_init from django.db.models.sql import Query from django.db.models.sql.where import ExtraWhere, WhereNode -from django.utils.functional import cached_property from django.utils.timezone import utc from django.utils import six @@ -48,10 +49,6 @@ from versions.exceptions import DeletionOfNonCurrentVersionError -def get_utc_now(): - return datetime.datetime.utcnow().replace(tzinfo=utc) - - QueryTime = namedtuple('QueryTime', 'time active') @@ -285,153 +282,7 @@ def validate_uuid(self, uuid_string): return self.uuid_valid_form_regex.match(uuid_string) is not None -class VersionedWhereNode(WhereNode): - def as_sql(self, qn, connection): - """ - This method identifies joined table aliases in order for VersionedExtraWhere.as_sql() - to be able to add time restrictions for those tables based on the VersionedQuery's - querytime value. - - :param qn: In Django 1.7 & 1.8 this is a compiler; in 1.6, it's an instance-method - :param connection: A DB connection - :return: A tuple consisting of (sql_string, result_params) - """ - # self.children is an array of VersionedExtraWhere-objects - for child in self.children: - if isinstance(child, VersionedExtraWhere) and not child.params: - try: - # Django 1.7 & 1.8 handles compilers as objects - _query = qn.query - except AttributeError: - # Django 1.6 handles compilers as instancemethods - _query = qn.__self__.query - query_time = _query.querytime.time - apply_query_time = _query.querytime.active - alias_map = _query.alias_map - # In Django 1.6 & 1.7, use the join_map to know, what *table* gets joined to which - # *left-hand sided* table - # In Django 1.8, use the Join objects in alias_map - if hasattr(_query, 'join_map'): - self._set_child_joined_alias_using_join_map(child, _query.join_map, alias_map) - else: - self._set_child_joined_alias(child, alias_map) - if apply_query_time: - # Add query parameters that have not been added till now - child.set_as_of(query_time) - else: - # Remove the restriction if it's not required - child.sqls = [] - return super(VersionedWhereNode, self).as_sql(qn, connection) - - @staticmethod - def _set_child_joined_alias_using_join_map(child, join_map, alias_map): - """ - Set the joined alias on the child, for Django <= 1.7.x. - :param child: - :param join_map: - :param alias_map: - """ - for lhs, table, join_cols in join_map: - if lhs is None: - continue - if lhs == child.alias: - relevant_alias = child.related_alias - elif lhs == child.related_alias: - relevant_alias = child.alias - else: - continue - - join_info = alias_map[relevant_alias] - if join_info.join_type is None: - continue - - if join_info.lhs_alias in [child.alias, child.related_alias]: - child.set_joined_alias(relevant_alias) - break - - @staticmethod - def _set_child_joined_alias(child, alias_map): - """ - Set the joined alias on the child, for Django >= 1.8.0 - :param child: - :param alias_map: - """ - for table in alias_map: - join = alias_map[table] - if not isinstance(join, Join): - continue - lhs = join.parent_alias - if (lhs == child.alias and table == child.related_alias) \ - or (lhs == child.related_alias and table == child.alias): - child.set_joined_alias(table) - break - - -class VersionedExtraWhere(ExtraWhere): - """ - A specific implementation of ExtraWhere; - Before as_sql can be called on an object, ensure that calls to - - set_as_of and - - set_joined_alias - have been done - """ - - def __init__(self, historic_sql, current_sql, alias, remote_alias): - super(VersionedExtraWhere, self).__init__(sqls=[], params=[]) - self.historic_sql = historic_sql - self.current_sql = current_sql - self.alias = alias - self.related_alias = remote_alias - self._as_of_time_set = False - self.as_of_time = None - self._joined_alias = None - - def set_as_of(self, as_of_time): - self.as_of_time = as_of_time - self._as_of_time_set = True - - def set_joined_alias(self, joined_alias): - """ - Takes the alias that is being joined to the query and applies the query - time constraint to its table - - :param str joined_alias: The table name of the alias - """ - self._joined_alias = joined_alias - - def as_sql(self, qn=None, connection=None): - sql = "" - params = [] - - # Fail fast for inacceptable cases - if self._as_of_time_set and not self._joined_alias: - raise ValueError("joined_alias is not set, but as_of is; this is a conflict!") - - # Set the SQL string in dependency of whether as_of_time was set or not - if self._as_of_time_set: - if self.as_of_time: - sql = self.historic_sql - params = [self.as_of_time] * 2 - # 2 is the number of occurences of the timestamp in an as_of-filter expression - else: - # If as_of_time was set to None, we're dealing with a query for "current" values - sql = self.current_sql - else: - # No as_of_time has been set; Perhaps, as_of was not part of the query -> That's OK - pass - # By here, the sql string is defined if an as_of_time was provided - if self._joined_alias: - sql = sql.format(alias=self._joined_alias) - - # Set the final sqls - # self.sqls needs to be set before the call to parent - if sql: - self.sqls = [sql] - else: - self.sqls = ["1=1"] - self.params = params - return super(VersionedExtraWhere, self).as_sql(qn, connection) class VersionedQuery(Query): @@ -442,6 +293,7 @@ class VersionedQuery(Query): """ def __init__(self, *args, **kwargs): + from .fields import VersionedWhereNode kwargs['where'] = VersionedWhereNode super(VersionedQuery, self).__init__(*args, **kwargs) self.querytime = QueryTime(time=None, active=False) @@ -570,9 +422,15 @@ def _fetch_all(self): """ if self._result_cache is None: self._result_cache = list(self.iterator()) - if not isinstance(self, ValuesListQuerySet): - for x in self._result_cache: - self._set_item_querytime(x) + if VERSION[:2] >= (1, 9): + # TODO: Do we have to test for ValuesListIterable, ValuesIterable, and FlatValuesListIterable here? + if self._iterable_class == ModelIterable: + for x in self._result_cache: + self._set_item_querytime(x) + else: + if not isinstance(self, ValuesListQuerySet): + for x in self._result_cache: + self._set_item_querytime(x) if self._prefetch_related_lookups and not self._prefetch_done: self._prefetch_related_objects() @@ -666,544 +524,10 @@ def delete(self): delete.queryset_only = True -class VersionedForeignKey(ForeignKey): - """ - We need to replace the standard ForeignKey declaration in order to be able to introduce - the VersionedReverseSingleRelatedObjectDescriptor, which allows to go back in time... - We also want to allow keeping track of any as_of time so that joins can be restricted - based on that. - """ - - def __init__(self, *args, **kwargs): - super(VersionedForeignKey, self).__init__(*args, **kwargs) - - def contribute_to_class(self, cls, name, virtual_only=False): - super(VersionedForeignKey, self).contribute_to_class(cls, name, virtual_only) - setattr(cls, self.name, VersionedReverseSingleRelatedObjectDescriptor(self)) - - def contribute_to_related_class(self, cls, related): - """ - Override ForeignKey's methods, and replace the descriptor, if set by the parent's methods - """ - # Internal FK's - i.e., those with a related name ending with '+' - - # and swapped models don't get a related descriptor. - super(VersionedForeignKey, self).contribute_to_related_class(cls, related) - accessor_name = related.get_accessor_name() - if hasattr(cls, accessor_name): - setattr(cls, accessor_name, VersionedForeignRelatedObjectsDescriptor(related)) - - def get_extra_restriction(self, where_class, alias, remote_alias): - """ - Overrides ForeignObject's get_extra_restriction function that returns an SQL statement which is appended to a - JOIN's conditional filtering part - - :return: SQL conditional statement - :rtype: WhereNode - """ - historic_sql = '''{alias}.version_start_date <= %s - AND ({alias}.version_end_date > %s OR {alias}.version_end_date is NULL )''' - current_sql = '''{alias}.version_end_date is NULL''' - # How 'bout creating an ExtraWhere here, without params - return where_class([VersionedExtraWhere(historic_sql=historic_sql, current_sql=current_sql, alias=alias, - remote_alias=remote_alias)]) - - def get_joining_columns(self, reverse_join=False): - """ - Get and return joining columns defined by this foreign key relationship - - :return: A tuple containing the column names of the tables to be joined (, ) - :rtype: tuple - """ - source = self.reverse_related_fields if reverse_join else self.related_fields - joining_columns = tuple() - for lhs_field, rhs_field in source: - lhs_col_name = lhs_field.column - rhs_col_name = rhs_field.column - # Test whether - # - self is the current ForeignKey relationship - # - self was not auto_created (e.g. is not part of a M2M relationship) - if self is lhs_field and not self.auto_created: - if rhs_col_name == Versionable.VERSION_IDENTIFIER_FIELD: - rhs_col_name = Versionable.OBJECT_IDENTIFIER_FIELD - elif self is rhs_field and not self.auto_created: - if lhs_col_name == Versionable.VERSION_IDENTIFIER_FIELD: - lhs_col_name = Versionable.OBJECT_IDENTIFIER_FIELD - joining_columns = joining_columns + ((lhs_col_name, rhs_col_name),) - return joining_columns - - -class VersionedManyToManyField(ManyToManyField): - def __init__(self, *args, **kwargs): - super(VersionedManyToManyField, self).__init__(*args, **kwargs) - - def contribute_to_class(self, cls, name): - """ - Called at class type creation. So, this method is called, when metaclasses get created - """ - # self.rel.through needs to be set prior to calling super, since super(...).contribute_to_class refers to it. - # Classes pointed to by a string do not need to be resolved here, since Django does that at a later point in - # time - which is nice... ;) - # - # Superclasses take care of: - # - creating the through class if unset - # - resolving the through class if it's a string - # - resolving string references within the through class - if not self.rel.through and not cls._meta.abstract and not cls._meta.swapped: - self.rel.through = VersionedManyToManyField.create_versioned_many_to_many_intermediary_model(self, cls, - name) - super(VersionedManyToManyField, self).contribute_to_class(cls, name) - - # Overwrite the descriptor - if hasattr(cls, self.name): - setattr(cls, self.name, VersionedReverseManyRelatedObjectsDescriptor(self)) - - def contribute_to_related_class(self, cls, related): - """ - Called at class type creation. So, this method is called, when metaclasses get created - """ - super(VersionedManyToManyField, self).contribute_to_related_class(cls, related) - accessor_name = related.get_accessor_name() - if accessor_name and hasattr(cls, accessor_name): - descriptor = VersionedManyRelatedObjectsDescriptor(related, accessor_name) - setattr(cls, accessor_name, descriptor) - if hasattr(cls._meta, 'many_to_many_related') and isinstance(cls._meta.many_to_many_related, list): - cls._meta.many_to_many_related.append(descriptor) - else: - cls._meta.many_to_many_related = [descriptor] - - @staticmethod - def create_versioned_many_to_many_intermediary_model(field, cls, field_name): - # Let's not care too much on what flags could potentially be set on that intermediary class (e.g. managed, etc) - # Let's play the game, as if the programmer had specified a class within his models... Here's how. - - from_ = cls._meta.model_name - to_model = field.rel.to - - # Force 'to' to be a string (and leave the hard work to Django) - if not isinstance(field.rel.to, six.string_types): - to_model = '%s.%s' % (field.rel.to._meta.app_label, field.rel.to._meta.object_name) - to = field.rel.to._meta.object_name.lower() - else: - to = to_model.lower() - name = '%s_%s' % (from_, field_name) - - if field.rel.to == RECURSIVE_RELATIONSHIP_CONSTANT or to == cls._meta.object_name: - from_ = 'from_%s' % to - to = 'to_%s' % to - to_model = cls - - # Since Django 1.7, a migration mechanism is shipped by default with Django. This migration module loads all - # declared apps' models inside a __fake__ module. - # This means that the models can be already loaded and registered by their original module, when we - # reach this point of the application and therefore there is no need to load them a second time. - if VERSION[:2] >= (1, 7) and cls.__module__ == '__fake__': - try: - # Check the apps for an already registered model - return apps.get_registered_model(cls._meta.app_label, str(name)) - except KeyError: - # The model has not been registered yet, so continue - pass - - meta = type('Meta', (object,), { - # 'unique_together': (from_, to), - 'auto_created': cls, - 'db_tablespace': cls._meta.db_tablespace, - 'app_label': cls._meta.app_label, - }) - return type(str(name), (Versionable,), { - 'Meta': meta, - '__module__': cls.__module__, - from_: VersionedForeignKey(cls, related_name='%s+' % name, auto_created=name), - to: VersionedForeignKey(to_model, related_name='%s+' % name, auto_created=name), - }) - - -class VersionedReverseSingleRelatedObjectDescriptor(ReverseSingleRelatedObjectDescriptor): - """ - A ReverseSingleRelatedObjectDescriptor-typed object gets inserted, when a ForeignKey - is defined in a Django model. This is one part of the analogue for versioned items. - - Unfortunately, we need to run two queries. The first query satisfies the foreign key - constraint. After extracting the identity information and combining it with the datetime- - stamp, we are able to fetch the historic element. - """ - - def __get__(self, instance, instance_type=None): - """ - The getter method returns the object, which points instance, e.g. choice.poll returns - a Poll instance, whereas the Poll class defines the ForeignKey. - :param instance: The object on which the property was accessed - :param instance_type: The type of the instance object - :return: Returns a Versionable - """ - current_elt = super(VersionedReverseSingleRelatedObjectDescriptor, self).__get__(instance, instance_type) - - if instance is None: - return self - - if not current_elt: - return None - - if not isinstance(current_elt, Versionable): - raise TypeError("VersionedForeignKey target is of type " - + str(type(current_elt)) - + ", which is not a subclass of Versionable") - - if hasattr(instance, '_querytime'): - # If current_elt matches the instance's querytime, there's no need to make a database query. - if Versionable.matches_querytime(current_elt, instance._querytime): - current_elt._querytime = instance._querytime - return current_elt - - return current_elt.__class__.objects.as_of(instance._querytime.time).get(identity=current_elt.identity) - else: - return current_elt.__class__.objects.current.get(identity=current_elt.identity) - - -class VersionedForeignRelatedObjectsDescriptor(ForeignRelatedObjectsDescriptor): - """ - This descriptor generates the manager class that is used on the related object of a ForeignKey relation - (i.e. the reverse-ForeignKey field manager). - """ - - @cached_property - def related_manager_cls(self): - # return create_versioned_related_manager - manager_cls = super(VersionedForeignRelatedObjectsDescriptor, self).related_manager_cls - rel_field = self.related.field - - class VersionedRelatedManager(manager_cls): - def __init__(self, instance): - super(VersionedRelatedManager, self).__init__(instance) - - # This is a hack, in order to get the versioned related objects - for key in self.core_filters.keys(): - if '__exact' in key or '__' not in key: - self.core_filters[key] = instance.identity - - def get_queryset(self): - queryset = super(VersionedRelatedManager, self).get_queryset() - # Do not set the query time if it is already correctly set. queryset.as_of() returns a clone - # of the queryset, and this will destroy the prefetched objects cache if it exists. - if isinstance(queryset, - VersionedQuerySet) and self.instance._querytime.active and queryset.querytime != self.instance._querytime: - queryset = queryset.as_of(self.instance._querytime.time) - return queryset - - def add(self, *objs): - cloned_objs = () - for obj in objs: - if not isinstance(obj, Versionable): - raise TypeError("Trying to add a non-Versionable to a VersionedForeignKey relationship") - cloned_objs += (obj.clone(),) - super(VersionedRelatedManager, self).add(*cloned_objs) - - # clear() and remove() are present if the FK is nullable - if 'clear' in dir(manager_cls): - def clear(self, **kwargs): - """ - Overridden to ensure that the current queryset is used, and to clone objects before they - are removed, so that history is not lost. - """ - bulk = kwargs.pop('bulk', True) - db = router.db_for_write(self.model, instance=self.instance) - queryset = self.current.using(db) - with transaction.atomic(using=db, savepoint=False): - cloned_pks = [obj.clone().pk for obj in queryset] - update_qs = self.current.filter(pk__in=cloned_pks) - if VERSION[:2] == (1, 6): - update_qs.update(**{rel_field.name: None}) - else: - self._clear(update_qs, bulk) - - if 'remove' in dir(manager_cls): - def remove(self, *objs): - val = rel_field.get_foreign_related_value(self.instance) - cloned_objs = () - for obj in objs: - # Is obj actually part of this descriptor set? Otherwise, silently go over it, since Django - # handles that case - if rel_field.get_local_related_value(obj) == val: - # Silently pass over non-versionable items - if not isinstance(obj, Versionable): - raise TypeError( - "Trying to remove a non-Versionable from a VersionedForeignKey realtionship") - cloned_objs += (obj.clone(),) - super(VersionedRelatedManager, self).remove(*cloned_objs) - - return VersionedRelatedManager - - -def create_versioned_many_related_manager(superclass, rel): - """ - The "casting" which is done in this method is needed, since otherwise, the methods introduced by - Versionable are not taken into account. - :param superclass: This is usually a models.Manager - :param rel: Contains the ManyToMany relation - :return: A subclass of ManyRelatedManager and Versionable - """ - many_related_manager_klass = create_many_related_manager(superclass, rel) - - class VersionedManyRelatedManager(many_related_manager_klass): - def __init__(self, *args, **kwargs): - super(VersionedManyRelatedManager, self).__init__(*args, **kwargs) - # Additional core filters are: version_start_date <= t & (version_end_date > t | version_end_date IS NULL) - # but we cannot work with the Django core filters, since they don't support ORing filters, which - # is a thing we need to consider the "version_end_date IS NULL" case; - # So, we define our own set of core filters being applied when versioning - try: - version_start_date_field = self.through._meta.get_field('version_start_date') - version_end_date_field = self.through._meta.get_field('version_end_date') - except FieldDoesNotExist as e: - if VERSION[:2] >= (1, 8): - fields = [f.name for f in self.through._meta.get_fields()] - else: - fields = self.through._meta.get_all_field_names() - print(str(e) + "; available fields are " + ", ".join(fields)) - raise e - # FIXME: this probably does not work when auto-referencing - - def get_queryset(self): - """ - Add a filter to the queryset, limiting the results to be pointed by relationship that are - valid for the given timestamp (which is taken at the current instance, or set to now, if not - available). - Long story short, apply the temporal validity filter also to the intermediary model. - """ - - queryset = super(VersionedManyRelatedManager, self).get_queryset() - if hasattr(queryset, 'querytime'): - if self.instance._querytime.active and self.instance._querytime != queryset.querytime: - queryset = queryset.as_of(self.instance._querytime.time) - return queryset - - def _remove_items(self, source_field_name, target_field_name, *objs): - """ - Instead of removing items, we simply set the version_end_date of the current item to the - current timestamp --> t[now]. - Like that, there is no more current entry having that identity - which is equal to - not existing for timestamps greater than t[now]. - """ - return self._remove_items_at(None, source_field_name, target_field_name, *objs) - - def _remove_items_at(self, timestamp, source_field_name, target_field_name, *objs): - if objs: - if timestamp is None: - timestamp = get_utc_now() - old_ids = set() - for obj in objs: - if isinstance(obj, self.model): - # The Django 1.7-way is preferred - if hasattr(self, 'target_field'): - fk_val = self.target_field.get_foreign_related_value(obj)[0] - # But the Django 1.6.x -way is supported for backward compatibility - elif hasattr(self, '_get_fk_val'): - fk_val = self._get_fk_val(obj, target_field_name) - else: - raise TypeError("We couldn't find the value of the foreign key, this might be due to the " - "use of an unsupported version of Django") - old_ids.add(fk_val) - else: - old_ids.add(obj) - db = router.db_for_write(self.through, instance=self.instance) - qs = self.through._default_manager.using(db).filter(**{ - source_field_name: self.instance.id, - '%s__in' % target_field_name: old_ids - }).as_of(timestamp) - for relation in qs: - relation._delete_at(timestamp) - - if 'add' in dir(many_related_manager_klass): - def add(self, *objs): - if not self.instance.is_current: - raise SuspiciousOperation( - "Adding many-to-many related objects is only possible on the current version") - - # The ManyRelatedManager.add() method uses the through model's default manager to get - # a queryset when looking at which objects already exist in the database. - # In order to restrict the query to the current versions when that is done, - # we temporarily replace the queryset's using method so that the version validity - # condition can be specified. - klass = self.through._default_manager.get_queryset().__class__ - __using_backup = klass.using - - def using_replacement(self, *args, **kwargs): - qs = __using_backup(self, *args, **kwargs) - return qs.as_of(None) - - klass.using = using_replacement - super(VersionedManyRelatedManager, self).add(*objs) - klass.using = __using_backup - - def add_at(self, timestamp, *objs): - """ - This function adds an object at a certain point in time (timestamp) - """ - # First off, define the new constructor - def _through_init(self, *args, **kwargs): - super(self.__class__, self).__init__(*args, **kwargs) - self.version_birth_date = timestamp - self.version_start_date = timestamp - - # Through-classes have an empty constructor, so it can easily be overwritten when needed; - # This is not the default case, so the overwrite only takes place when we "modify the past" - self.through.__init_backup__ = self.through.__init__ - self.through.__init__ = _through_init - - # Do the add operation - self.add(*objs) - - # Remove the constructor again (by replacing it with the original empty constructor) - self.through.__init__ = self.through.__init_backup__ - del self.through.__init_backup__ - - add_at.alters_data = True - - if 'remove' in dir(many_related_manager_klass): - def remove_at(self, timestamp, *objs): - """ - Performs the act of removing specified relationships at a specified time (timestamp); - So, not the objects at a given time are removed, but their relationship! - """ - self._remove_items_at(timestamp, self.source_field_name, self.target_field_name, *objs) - - # For consistency, also handle the symmetrical case - if self.symmetrical: - self._remove_items_at(timestamp, self.target_field_name, self.source_field_name, *objs) - - remove_at.alters_data = True - - return VersionedManyRelatedManager - - -class VersionedReverseManyRelatedObjectsDescriptor(ReverseManyRelatedObjectsDescriptor): - """ - Beside having a very long name, this class is useful when it comes to versioning the - ReverseManyRelatedObjectsDescriptor (huhu!!). The main part is the exposure of the - 'related_manager_cls' property - """ - - def __get__(self, instance, owner=None): - """ - Reads the property as which this object is figuring; mainly used for debugging purposes - :param instance: The instance on which the getter was called - :param owner: no idea... alternatively called 'instance_type by the superclasses - :return: A VersionedManyRelatedManager object - """ - return super(VersionedReverseManyRelatedObjectsDescriptor, self).__get__(instance, owner) - - def __set__(self, instance, value): - """ - Completely overridden to avoid bulk deletion that happens when the parent method calls clear(). - - The parent method's logic is basically: clear all in bulk, then add the given objects in bulk. - Instead, we figure out which ones are being added and removed, and call add and remove for these values. - This lets us retain the versioning information. - - Since this is a many-to-many relationship, it is assumed here that the django.db.models.deletion.Collector - logic, that is used in clear(), is not necessary here. Collector collects related models, e.g. ones that should - also be deleted because they have a ON CASCADE DELETE relationship to the object, or, in the case of - "Multi-table inheritance", are parent objects. - - :param instance: The instance on which the getter was called - :param value: iterable of items to set - """ - - if not instance.is_current: - raise SuspiciousOperation( - "Related values can only be directly set on the current version of an object") - - if not self.field.rel.through._meta.auto_created: - opts = self.field.rel.through._meta - raise AttributeError(("Cannot set values on a ManyToManyField which specifies an intermediary model. " - "Use %s.%s's Manager instead.") % (opts.app_label, opts.object_name)) - - manager = self.__get__(instance) - # Below comment is from parent __set__ method. We'll force evaluation, too: - # clear() can change expected output of 'value' queryset, we force evaluation - # of queryset before clear; ticket #19816 - value = tuple(value) - - being_removed, being_added = self.get_current_m2m_diff(instance, value) - timestamp = get_utc_now() - manager.remove_at(timestamp, *being_removed) - manager.add_at(timestamp, *being_added) - - def get_current_m2m_diff(self, instance, new_objects): - """ - :param instance: Versionable object - :param new_objects: objects which are about to be associated with instance - :return: (being_removed id list, being_added id list) - :rtype : tuple - """ - new_ids = self.pks_from_objects(new_objects) - relation_manager = self.__get__(instance) - - filter = Q(**{relation_manager.source_field.attname: instance.pk}) - qs = self.through.objects.current.filter(filter) - try: - # Django 1.7 - target_name = relation_manager.target_field.attname - except AttributeError: - # Django 1.6 - target_name = relation_manager.through._meta.get_field_by_name( - relation_manager.target_field_name)[0].attname - current_ids = set(qs.values_list(target_name, flat=True)) - - being_removed = current_ids - new_ids - being_added = new_ids - current_ids - return list(being_removed), list(being_added) - - def pks_from_objects(self, objects): - """ - Extract all the primary key strings from the given objects. Objects may be Versionables, or bare primary keys. - :rtype : set - """ - return {o.pk if isinstance(o, Model) else o for o in objects} - - @cached_property - def related_manager_cls(self): - return create_versioned_many_related_manager( - self.field.rel.to._default_manager.__class__, - self.field.rel - ) - - -class VersionedManyRelatedObjectsDescriptor(ManyRelatedObjectsDescriptor): - """ - Beside having a very long name, this class is useful when it comes to versioning the - ManyRelatedObjectsDescriptor (huhu!!). The main part is the exposure of the - 'related_manager_cls' property - """ - - via_field_name = None - - def __init__(self, related, via_field_name): - super(VersionedManyRelatedObjectsDescriptor, self).__init__(related) - self.via_field_name = via_field_name - - def __get__(self, instance, owner=None): - """ - Reads the property as which this object is figuring; mainly used for debugging purposes - :param instance: The instance on which the getter was called - :param owner: no idea... alternatively called 'instance_type by the superclasses - :return: A VersionedManyRelatedManager object - """ - return super(VersionedManyRelatedObjectsDescriptor, self).__get__(instance, owner) - - @cached_property - def related_manager_cls(self): - return create_versioned_many_related_manager( - self.related.model._default_manager.__class__, - self.related.field.rel - ) - - class Versionable(models.Model): """ This is pretty much the central point for versioning objects. """ - VERSION_IDENTIFIER_FIELD = 'id' OBJECT_IDENTIFIER_FIELD = 'identity' VERSIONABLE_FIELDS = [VERSION_IDENTIFIER_FIELD, OBJECT_IDENTIFIER_FIELD, 'version_start_date', @@ -1495,7 +819,10 @@ def get_all_m2m_field_names(self): opts = self._meta rel_field_names = [field.attname for field in opts.many_to_many] if hasattr(opts, 'many_to_many_related'): - rel_field_names += [rel.via_field_name for rel in opts.many_to_many_related] + if VERSION[:2] >= (1, 9): + rel_field_names += [rel.reverse for rel in opts.many_to_many_related] + else: + rel_field_names += [rel.via_field_name for rel in opts.many_to_many_related] return rel_field_names @@ -1516,23 +843,6 @@ def detach(self): self.version_end_date = None return self - @staticmethod - def matches_querytime(instance, querytime): - """ - Checks whether the given instance satisfies the given QueryTime object. - - :param instance: an instance of Versionable - :param querytime: QueryTime value to check against - """ - if not querytime.active: - return True - - if not querytime.time: - return instance.version_end_date is None - - return (instance.version_start_date <= querytime.time - and (instance.version_end_date is None or instance.version_end_date > querytime.time)) - class VersionedManyToManyModel(object): """ @@ -1562,3 +872,5 @@ def post_init_initialize(sender, instance, **kwargs): post_init.connect(VersionedManyToManyModel.post_init_initialize) + + diff --git a/versions/util/__init__.py b/versions/util/__init__.py index e69de29..5f67c8e 100644 --- a/versions/util/__init__.py +++ b/versions/util/__init__.py @@ -0,0 +1,7 @@ +import datetime + +from django.utils.timezone import utc + + +def get_utc_now(): + return datetime.datetime.utcnow().replace(tzinfo=utc) diff --git a/versions/util/postgresql.py b/versions/util/postgresql.py index a6c0841..c7bbd17 100644 --- a/versions/util/postgresql.py +++ b/versions/util/postgresql.py @@ -1,6 +1,6 @@ from __future__ import absolute_import from django.db import connection as default_connection -from versions.models import VersionedForeignKey +from versions.fields import VersionedForeignKey from .helper import database_connection, versionable_models diff --git a/versions_tests/models.py b/versions_tests/models.py index 4321d1e..a96182f 100644 --- a/versions_tests/models.py +++ b/versions_tests/models.py @@ -2,7 +2,8 @@ from django.db.models.deletion import DO_NOTHING, PROTECT, SET, SET_NULL from django.utils.encoding import python_2_unicode_compatible -from versions.models import Versionable, VersionedManyToManyField, VersionedForeignKey +from versions.models import Versionable +from versions.fields import VersionedManyToManyField, VersionedForeignKey def versionable_description(obj): From c09ab57eef1c5527813624dbae735fd424e40307 Mon Sep 17 00:00:00 2001 From: Manuel Jeckelmann Date: Wed, 28 Dec 2016 20:00:18 +0100 Subject: [PATCH 04/42] test_models tests passing; Major clean-up required yet --- .travis.yml | 8 +- tox.ini | 3 +- versions/deletion.py | 9 +- versions/descriptors.py | 350 +++++++++----------------- versions/fields.py | 32 +-- versions/models.py | 59 ++--- versions_tests/tests/test_commands.py | 7 +- versions_tests/tests/test_models.py | 2 +- 8 files changed, 175 insertions(+), 295 deletions(-) diff --git a/.travis.yml b/.travis.yml index f48aa35..1033c3d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,14 +2,10 @@ language: python python: "2.7" env: - - TOX_ENV=py27-django18-pg - - TOX_ENV=py27-django18-sqlite - TOX_ENV=py27-django19-pg - TOX_ENV=py27-django19-sqlite - - TOX_ENV=py34-django18-pg - - TOX_ENV=py34-django18-sqlite - - TOX_ENV=py34-django19-pg - - TOX_ENV=py34-django19-sqlite + - TOX_ENV=py36-django19-pg + - TOX_ENV=py36-django19-sqlite # Enable PostgreSQL usage addons: diff --git a/tox.ini b/tox.ini index a17a137..8e920c9 100644 --- a/tox.ini +++ b/tox.ini @@ -5,12 +5,11 @@ [tox] envlist = - py{27,34}-django{18,19}-{sqlite,pg} + py{27,36}-django{19}-{sqlite,pg} [testenv] deps = coverage - django18: django>=1.8,<1.9 django19: django>=1.9,<2.0 pg: psycopg2 commands = diff --git a/versions/deletion.py b/versions/deletion.py index b5fe02a..92cabd8 100644 --- a/versions/deletion.py +++ b/versions/deletion.py @@ -135,12 +135,11 @@ def related_objects(self, related, objs): Gets a QuerySet of current objects related to ``objs`` via the relation ``related``. """ - if VERSION >= (1, 8): - related_model = related.related_model - else: - related_model = related.model + from versions.models import Versionable + + related_model = related.related_model manager = related_model._base_manager - if issubclass(related_model, versions.models.Versionable): + if issubclass(related_model, Versionable): manager = manager.current return manager.using(self.using).filter( **{"%s__in" % related.field.name: objs} diff --git a/versions/descriptors.py b/versions/descriptors.py index f92a922..82d4a69 100644 --- a/versions/descriptors.py +++ b/versions/descriptors.py @@ -5,27 +5,18 @@ from django.db.models.query_utils import Q from django.utils.functional import cached_property -from versions.models import Versionable, VersionedQuerySet from versions.util import get_utc_now -if VERSION[:2] >= (1, 9): - # With Django 1.9 related descriptor classes have been renamed: - # ReverseSingleRelatedObjectDescriptor => ForwardManyToOneDescriptor - # ForeignRelatedObjectsDescriptor => ReverseManyToOneDescriptor - # ReverseManyRelatedObjectsDescriptor => ManyToManyDescriptor - # ManyRelatedObjectsDescriptor => ManyToManyDescriptor - # (new) => ReverseOneToOneDescriptor - from django.db.models.fields.related import (ForwardManyToOneDescriptor, ReverseManyToOneDescriptor, - ManyToManyDescriptor, ReverseOneToOneDescriptor) - from django.db.models.fields.related_descriptors import create_forward_many_to_many_manager -else: - from django.db.models.fields.related import (ReverseSingleRelatedObjectDescriptor, - ReverseManyRelatedObjectsDescriptor, - ManyRelatedObjectsDescriptor, - ForeignRelatedObjectsDescriptor, - create_many_related_manager) - +# With Django 1.9 related descriptor classes have been renamed: +# ReverseSingleRelatedObjectDescriptor => ForwardManyToOneDescriptor +# ForeignRelatedObjectsDescriptor => ReverseManyToOneDescriptor +# ReverseManyRelatedObjectsDescriptor => ManyToManyDescriptor +# ManyRelatedObjectsDescriptor => ManyToManyDescriptor +# (new) => ReverseOneToOneDescriptor +from django.db.models.fields.related import (ForwardManyToOneDescriptor, ReverseManyToOneDescriptor, + ManyToManyDescriptor) +from django.db.models.fields.related_descriptors import create_forward_many_to_many_manager def matches_querytime(instance, querytime): @@ -44,25 +35,14 @@ def matches_querytime(instance, querytime): return (instance.version_start_date <= querytime.time and (instance.version_end_date is None or instance.version_end_date > querytime.time)) -if VERSION[:2] >= (1,9): - class VersionedForwardManyToOneDescriptor(ForwardManyToOneDescriptor): - """ - """ - pass - vforward_many_to_one_descriptor_class = VersionedForwardManyToOneDescriptor -else: - class VersionedReverseSingleRelatedObjectDescriptor(ReverseSingleRelatedObjectDescriptor): - """ - A ReverseSingleRelatedObjectDescriptor-typed object gets inserted, when a ForeignKey - is defined in a Django model. This is one part of the analogue for versioned items. +class VersionedForwardManyToOneDescriptor(ForwardManyToOneDescriptor): + """ + + """ + pass +vforward_many_to_one_descriptor_class = VersionedForwardManyToOneDescriptor - Unfortunately, we need to run two queries. The first query satisfies the foreign key - constraint. After extracting the identity information and combining it with the datetime- - stamp, we are able to fetch the historic element. - """ - pass - vforward_many_to_one_descriptor_class = VersionedReverseSingleRelatedObjectDescriptor def vforward_many_to_one_descriptor_getter(self, instance, instance_type=None): """ @@ -72,6 +52,7 @@ def vforward_many_to_one_descriptor_getter(self, instance, instance_type=None): :param instance_type: The type of the instance object :return: Returns a Versionable """ + from versions.models import Versionable current_elt = super(self.__class__, self).__get__(instance, instance_type) if instance is None: @@ -98,97 +79,120 @@ def vforward_many_to_one_descriptor_getter(self, instance, instance_type=None): vforward_many_to_one_descriptor_class.__get__ = vforward_many_to_one_descriptor_getter +class VersionedReverseManyToOneDescriptor(ReverseManyToOneDescriptor): + @cached_property + def related_manager_cls(self): + # return create_versioned_related_manager + manager_cls = super(VersionedReverseManyToOneDescriptor, self).related_manager_cls -if VERSION[:2] >= (1,9): - class VersionedReverseManyToOneDescriptor(ReverseManyToOneDescriptor): - pass - - vreverse_many_to_one_descriptor_class = VersionedReverseManyToOneDescriptor -else: - class VersionedForeignRelatedObjectsDescriptor(ForeignRelatedObjectsDescriptor): - """ - This descriptor generates the manager class that is used on the related object of a ForeignKey relation - (i.e. the reverse-ForeignKey field manager). - """ - pass - - vreverse_many_to_one_descriptor_class = VersionedForeignRelatedObjectsDescriptor - - -def vreverse_many_to_one_descriptor_related_manager_cls_property(self): - # return create_versioned_related_manager - manager_cls = super(self.__class__, self).related_manager_cls - if VERSION[:2] >= (1, 9): + #if VERSION[:2] >= (1, 9): # TODO: Define, what field has to be taken over here, self.rel/self.field? The WhineDrinker.hats test seems to be a good one for testing this - rel_field = self.rel - elif hasattr(self, 'related'): - rel_field = self.related.field - else: - rel_field = self.field - - class VersionedRelatedManager(manager_cls): - def __init__(self, instance): - super(VersionedRelatedManager, self).__init__(instance) - # This is a hack, in order to get the versioned related objects - for key in self.core_filters.keys(): - if '__exact' in key or '__' not in key: - self.core_filters[key] = instance.identity + # rel_field = self.rel - def get_queryset(self): - queryset = super(VersionedRelatedManager, self).get_queryset() - # Do not set the query time if it is already correctly set. queryset.as_of() returns a clone - # of the queryset, and this will destroy the prefetched objects cache if it exists. - if isinstance(queryset, VersionedQuerySet) \ - and self.instance._querytime.active \ - and queryset.querytime != self.instance._querytime: - queryset = queryset.as_of(self.instance._querytime.time) - return queryset + rel_field = self.field + # elif hasattr(self, 'related'): + # rel_field = self.related.field + # else: + # rel_field = self.field + # rel_field = self.rel + + class VersionedRelatedManager(manager_cls): + def __init__(self, instance): + super(VersionedRelatedManager, self).__init__(instance) + + # This is a hack, in order to get the versioned related objects + for key in self.core_filters.keys(): + if '__exact' in key or '__' not in key: + self.core_filters[key] = instance.identity + + def get_queryset(self): + from versions.models import VersionedQuerySet + + queryset = super(VersionedRelatedManager, self).get_queryset() + # Do not set the query time if it is already correctly set. queryset.as_of() returns a clone + # of the queryset, and this will destroy the prefetched objects cache if it exists. + if isinstance(queryset, VersionedQuerySet) \ + and self.instance._querytime.active \ + and queryset.querytime != self.instance._querytime: + queryset = queryset.as_of(self.instance._querytime.time) + return queryset - def add(self, *objs): - cloned_objs = () - for obj in objs: - if not isinstance(obj, Versionable): - raise TypeError("Trying to add a non-Versionable to a VersionedForeignKey relationship") - cloned_objs += (obj.clone(),) - super(VersionedRelatedManager, self).add(*cloned_objs) - - # clear() and remove() are present if the FK is nullable - if 'clear' in dir(manager_cls): - def clear(self, **kwargs): + def get_prefetch_queryset(self, instances, queryset=None): """ - Overridden to ensure that the current queryset is used, and to clone objects before they - are removed, so that history is not lost. + Overrides RelatedManager's implementation of get_prefetch_queryset so that it works + nicely with VersionedQuerySets. It ensures that identities and time-limited where + clauses are used when selecting related reverse foreign key objects. """ - bulk = kwargs.pop('bulk', True) - db = router.db_for_write(self.model, instance=self.instance) - queryset = self.current.using(db) - with transaction.atomic(using=db, savepoint=False): - cloned_pks = [obj.clone().pk for obj in queryset] - update_qs = self.current.filter(pk__in=cloned_pks) - if VERSION[:2] == (1, 6): - update_qs.update(**{rel_field.name: None}) - else: - self._clear(update_qs, bulk) - - if 'remove' in dir(manager_cls): - def remove(self, *objs): - val = rel_field.get_foreign_related_value(self.instance) + if queryset is None: + # Note that this intentionally call's VersionManager's get_queryset, instead of simply calling + # the superclasses' get_queryset (as the non-versioned RelatedManager does), because what is + # needed is a simple Versioned queryset without any restrictions (e.g. do not + # apply self.core_filters). + from versions.models import VersionManager + queryset = VersionManager.get_queryset(self) + + queryset._add_hints(instance=instances[0]) + queryset = queryset.using(queryset._db or self._db) + + rel_obj_attr = rel_field.get_local_related_value + instance_attr = rel_field.get_foreign_related_value + # Use identities instead of ids so that this will work with versioned objects. + instances_dict = {(inst.identity,): inst for inst in instances} + identities = [inst.identity for inst in instances] + query = {'%s__identity__in' % rel_field.name: identities} + queryset = queryset.filter(**query) + + # Since we just bypassed this class' get_queryset(), we must manage + # the reverse relation manually. + for rel_obj in queryset: + instance = instances_dict[rel_obj_attr(rel_obj)] + setattr(rel_obj, rel_field.name, instance) + cache_name = rel_field.related_query_name() + return queryset, rel_obj_attr, instance_attr, False, cache_name + + def add(self, *objs, **kwargs): + from versions.models import Versionable cloned_objs = () for obj in objs: - # Is obj actually part of this descriptor set? Otherwise, silently go over it, since Django - # handles that case - if rel_field.get_local_related_value(obj) == val: - # Silently pass over non-versionable items - if not isinstance(obj, Versionable): - raise TypeError( - "Trying to remove a non-Versionable from a VersionedForeignKey realtionship") - cloned_objs += (obj.clone(),) - super(VersionedRelatedManager, self).remove(*cloned_objs) + if not isinstance(obj, Versionable): + raise TypeError("Trying to add a non-Versionable to a VersionedForeignKey relationship") + cloned_objs += (obj.clone(),) + super(VersionedRelatedManager, self).add(*cloned_objs, **kwargs) + + # clear() and remove() are present if the FK is nullable + if 'clear' in dir(manager_cls): + def clear(self, **kwargs): + """ + Overridden to ensure that the current queryset is used, and to clone objects before they + are removed, so that history is not lost. + """ + bulk = kwargs.pop('bulk', True) + db = router.db_for_write(self.model, instance=self.instance) + queryset = self.current.using(db) + with transaction.atomic(using=db, savepoint=False): + cloned_pks = [obj.clone().pk for obj in queryset] + update_qs = self.current.filter(pk__in=cloned_pks) + self._clear(update_qs, bulk) + + if 'remove' in dir(manager_cls): + def remove(self, *objs, **kwargs): + from versions.models import Versionable - return VersionedRelatedManager + val = rel_field.get_foreign_related_value(self.instance) + cloned_objs = () + for obj in objs: + # Is obj actually part of this descriptor set? Otherwise, silently go over it, since Django + # handles that case + if rel_field.get_local_related_value(obj) == val: + # Silently pass over non-versionable items + if not isinstance(obj, Versionable): + raise TypeError( + "Trying to remove a non-Versionable from a VersionedForeignKey realtionship") + cloned_objs += (obj.clone(),) + super(VersionedRelatedManager, self).remove(*cloned_objs, **kwargs) -vreverse_many_to_one_descriptor_class.related_manager_cls = cached_property(vreverse_many_to_one_descriptor_related_manager_cls_property) + return VersionedRelatedManager class VersionedManyToManyDescriptor(ManyToManyDescriptor): @@ -270,117 +274,9 @@ def pks_from_objects(self, objects): """ return {o.pk if isinstance(o, Model) else o for o in objects} -if VERSION[:2] < (1, 9): - class VersionedReverseManyRelatedObjectsDescriptor(ReverseManyRelatedObjectsDescriptor): - """ - Beside having a very long name, this class is useful when it comes to versioning the - ReverseManyRelatedObjectsDescriptor (huhu!!). The main part is the exposure of the - 'related_manager_cls' property - """ - - def __set__(self, instance, value): - """ - Completely overridden to avoid bulk deletion that happens when the parent method calls clear(). - - The parent method's logic is basically: clear all in bulk, then add the given objects in bulk. - Instead, we figure out which ones are being added and removed, and call add and remove for these values. - This lets us retain the versioning information. - - Since this is a many-to-many relationship, it is assumed here that the django.db.models.deletion.Collector - logic, that is used in clear(), is not necessary here. Collector collects related models, e.g. ones that should - also be deleted because they have a ON CASCADE DELETE relationship to the object, or, in the case of - "Multi-table inheritance", are parent objects. - - :param instance: The instance on which the getter was called - :param value: iterable of items to set - """ - - if not instance.is_current: - raise SuspiciousOperation( - "Related values can only be directly set on the current version of an object") - - if not self.field.rel.through._meta.auto_created: - opts = self.field.rel.through._meta - raise AttributeError(("Cannot set values on a ManyToManyField which specifies an intermediary model. " - "Use %s.%s's Manager instead.") % (opts.app_label, opts.object_name)) - - manager = self.__get__(instance) - # Below comment is from parent __set__ method. We'll force evaluation, too: - # clear() can change expected output of 'value' queryset, we force evaluation - # of queryset before clear; ticket #19816 - value = tuple(value) - - being_removed, being_added = self.get_current_m2m_diff(instance, value) - timestamp = get_utc_now() - manager.remove_at(timestamp, *being_removed) - manager.add_at(timestamp, *being_added) - - def get_current_m2m_diff(self, instance, new_objects): - """ - :param instance: Versionable object - :param new_objects: objects which are about to be associated with instance - :return: (being_removed id list, being_added id list) - :rtype : tuple - """ - new_ids = self.pks_from_objects(new_objects) - relation_manager = self.__get__(instance) - - filter = Q(**{relation_manager.source_field.attname: instance.pk}) - qs = self.through.objects.current.filter(filter) - try: - # Django 1.7 - target_name = relation_manager.target_field.attname - except AttributeError: - # Django 1.6 - target_name = relation_manager.through._meta.get_field_by_name( - relation_manager.target_field_name)[0].attname - current_ids = set(qs.values_list(target_name, flat=True)) - - being_removed = current_ids - new_ids - being_added = new_ids - current_ids - return list(being_removed), list(being_added) - - def pks_from_objects(self, objects): - """ - Extract all the primary key strings from the given objects. Objects may be Versionables, or bare primary keys. - :rtype : set - """ - return {o.pk if isinstance(o, Model) else o for o in objects} - - @cached_property - def related_manager_cls(self): - return create_versioned_forward_many_to_many_manager( - self.field.rel.to._default_manager.__class__, - self.field.rel - ) - - - class VersionedManyRelatedObjectsDescriptor(ManyRelatedObjectsDescriptor): - """ - Beside having a very long name, this class is useful when it comes to versioning the - ManyRelatedObjectsDescriptor (huhu!!). The main part is the exposure of the - 'related_manager_cls' property - """ - - # via_field_name = None - - def __init__(self, related, via_field_name): - super(VersionedManyRelatedObjectsDescriptor, self).__init__(related) - self.via_field_name = via_field_name - - @cached_property - def related_manager_cls(self): - return create_versioned_forward_many_to_many_manager( - self.related.model._default_manager.__class__, - self.related.field.rel - ) - def create_versioned_forward_many_to_many_manager(superclass, rel, reverse=None): - if VERSION[:2] >= (1, 9): - many_related_manager_klass = create_forward_many_to_many_manager(superclass, rel, reverse) - else: - many_related_manager_klass = create_many_related_manager(superclass, rel) + many_related_manager_klass = create_forward_many_to_many_manager(superclass, rel, reverse) class VersionedManyRelatedManager(many_related_manager_klass): def __init__(self, *args, **kwargs): @@ -393,10 +289,8 @@ def __init__(self, *args, **kwargs): version_start_date_field = self.through._meta.get_field('version_start_date') version_end_date_field = self.through._meta.get_field('version_end_date') except FieldDoesNotExist as e: - if VERSION[:2] >= (1, 8): - fields = [f.name for f in self.through._meta.get_fields()] - else: - fields = self.through._meta.get_all_field_names() + fields = [f.name for f in self.through._meta.get_fields()] + print(str(e) + "; available fields are " + ", ".join(fields)) raise e # FIXME: this probably does not work when auto-referencing diff --git a/versions/fields.py b/versions/fields.py index 2750877..dc377f3 100644 --- a/versions/fields.py +++ b/versions/fields.py @@ -6,27 +6,17 @@ from django.db.models.sql.datastructures import Join from django.db.models.sql.where import ExtraWhere, WhereNode -if VERSION[:2] >= (1, 9): - # With Django 1.9 related descriptor classes have been renamed: - # ReverseSingleRelatedObjectDescriptor => ForwardManyToOneDescriptor - # ForeignRelatedObjectsDescriptor => ReverseManyToOneDescriptor - # ReverseManyRelatedObjectsDescriptor => ManyToManyDescriptor - # ManyRelatedObjectsDescriptor => ManyToManyDescriptor - # (new) => ReverseOneToOneDescriptor - # from django.db.models.fields.related import (ForwardManyToOneDescriptor, ReverseManyToOneDescriptor, - # ManyToManyDescriptor, ReverseOneToOneDescriptor) - from descriptors import (VersionedForwardManyToOneDescriptor, - VersionedReverseManyToOneDescriptor, - VersionedManyToManyDescriptor) -else: - from descriptors import (VersionedReverseSingleRelatedObjectDescriptor, - VersionedForeignRelatedObjectsDescriptor, - VersionedReverseManyRelatedObjectsDescriptor, - VersionedManyRelatedObjectsDescriptor) - # from django.db.models.fields.related import (ReverseSingleRelatedObjectDescriptor, - # ReverseManyRelatedObjectsDescriptor, ManyToManyField, - # ManyRelatedObjectsDescriptor, create_many_related_manager, - # ForeignRelatedObjectsDescriptor, RECURSIVE_RELATIONSHIP_CONSTANT) +# With Django 1.9 related descriptor classes have been renamed: +# ReverseSingleRelatedObjectDescriptor => ForwardManyToOneDescriptor +# ForeignRelatedObjectsDescriptor => ReverseManyToOneDescriptor +# ReverseManyRelatedObjectsDescriptor => ManyToManyDescriptor +# ManyRelatedObjectsDescriptor => ManyToManyDescriptor +# (new) => ReverseOneToOneDescriptor +# from django.db.models.fields.related import (ForwardManyToOneDescriptor, ReverseManyToOneDescriptor, +# ManyToManyDescriptor, ReverseOneToOneDescriptor) +from descriptors import (VersionedForwardManyToOneDescriptor, + VersionedReverseManyToOneDescriptor, + VersionedManyToManyDescriptor) from models import Versionable diff --git a/versions/models.py b/versions/models.py index f91a1c4..6d12c8b 100644 --- a/versions/models.py +++ b/versions/models.py @@ -18,12 +18,17 @@ from collections import namedtuple from django import VERSION +from django.db.models.fields.related_descriptors import ForwardManyToOneDescriptor, ReverseManyToOneDescriptor, \ + ManyToManyDescriptor, create_forward_many_to_many_manager +from django.utils.functional import cached_property +from versions.descriptors import VersionedForwardManyToOneDescriptor, VersionedReverseManyToOneDescriptor, \ + VersionedManyToManyDescriptor from versions.util import get_utc_now from django.db.models.sql.datastructures import Join from django.apps.registry import apps -from django.core.exceptions import SuspiciousOperation, ObjectDoesNotExist +from django.core.exceptions import SuspiciousOperation, ObjectDoesNotExist, FieldDoesNotExist from django.db import models, router, transaction from django.db.models.base import Model from django.db.models import Q @@ -31,10 +36,7 @@ from django.db.models.fields.related import ForeignKey, ManyToManyField, RECURSIVE_RELATIONSHIP_CONSTANT from django.db.models.query import QuerySet -if VERSION[:2] >= (1, 9): - from django.db.models.query import ModelIterable -else: - from django.db.models.query import ValuesListQuerySet, ValuesQuerySet +from django.db.models.query import ModelIterable from django.db.models.sql import Query from django.db.models.sql.where import ExtraWhere, WhereNode from django.utils.timezone import utc @@ -503,15 +505,10 @@ def _fetch_all(self): """ if self._result_cache is None: self._result_cache = list(self.iterator()) - if VERSION[:2] >= (1, 9): - # TODO: Do we have to test for ValuesListIterable, ValuesIterable, and FlatValuesListIterable here? - if self._iterable_class == ModelIterable: - for x in self._result_cache: - self._set_item_querytime(x) - else: - if not isinstance(self, ValuesListQuerySet): - for x in self._result_cache: - self._set_item_querytime(x) + # TODO: Do we have to test for ValuesListIterable, ValuesIterable, and FlatValuesListIterable here? + if self._iterable_class == ModelIterable: + for x in self._result_cache: + self._set_item_querytime(x) if self._prefetch_related_lookups and not self._prefetch_done: self._prefetch_related_objects() @@ -601,7 +598,7 @@ def __init__(self, *args, **kwargs): def contribute_to_class(self, cls, name, virtual_only=False): super(VersionedForeignKey, self).contribute_to_class(cls, name, virtual_only) - setattr(cls, self.name, VersionedReverseSingleRelatedObjectDescriptor(self)) + setattr(cls, self.name, VersionedForwardManyToOneDescriptor(self)) def contribute_to_related_class(self, cls, related): """ @@ -612,7 +609,7 @@ def contribute_to_related_class(self, cls, related): super(VersionedForeignKey, self).contribute_to_related_class(cls, related) accessor_name = related.get_accessor_name() if hasattr(cls, accessor_name): - setattr(cls, accessor_name, VersionedForeignRelatedObjectsDescriptor(related)) + setattr(cls, accessor_name, VersionedReverseManyToOneDescriptor(related)) def get_extra_restriction(self, where_class, alias, remote_alias): """ @@ -677,7 +674,7 @@ def contribute_to_class(self, cls, name): # Overwrite the descriptor if hasattr(cls, self.name): - setattr(cls, self.name, VersionedReverseManyRelatedObjectsDescriptor(self)) + setattr(cls, self.name, VersionedManyToManyDescriptor(self)) def contribute_to_related_class(self, cls, related): """ @@ -686,7 +683,7 @@ def contribute_to_related_class(self, cls, related): super(VersionedManyToManyField, self).contribute_to_related_class(cls, related) accessor_name = related.get_accessor_name() if accessor_name and hasattr(cls, accessor_name): - descriptor = VersionedManyRelatedObjectsDescriptor(related, accessor_name) + descriptor = VersionedManyToManyDescriptor(related, accessor_name) setattr(cls, accessor_name, descriptor) if hasattr(cls._meta, 'many_to_many_related') and isinstance(cls._meta.many_to_many_related, list): cls._meta.many_to_many_related.append(descriptor) @@ -740,8 +737,10 @@ def create_versioned_many_to_many_intermediary_model(field, cls, field_name): }) -class VersionedReverseSingleRelatedObjectDescriptor(ReverseSingleRelatedObjectDescriptor): +class VersionedReverseSingleRelatedObjectDescriptor(ForwardManyToOneDescriptor): """ + Django1.9 compatibility note: ReverseSingleRelatedObjectDescriptor becomes ForwardManyToOneDescriptor + A ReverseSingleRelatedObjectDescriptor-typed object gets inserted, when a ForeignKey is defined in a Django model. This is one part of the analogue for versioned items. @@ -782,8 +781,10 @@ def __get__(self, instance, instance_type=None): return current_elt.__class__.objects.current.get(identity=current_elt.identity) -class VersionedForeignRelatedObjectsDescriptor(ForeignRelatedObjectsDescriptor): +class VersionedForeignRelatedObjectsDescriptor(ReverseManyToOneDescriptor): """ + Django 1.9 compatibility note: ForeignRelatedObjectsDescriptor becomes ReverseManyToOneDescriptor + This descriptor generates the manager class that is used on the related object of a ForeignKey relation (i.e. the reverse-ForeignKey field manager). """ @@ -815,7 +816,7 @@ def get_queryset(self): def get_prefetch_queryset(self, instances, queryset=None): """ Overrides RelatedManager's implementation of get_prefetch_queryset so that it works - nicely with VersionedQuerySets. It ensures that identities and time-limited where + nicely with VersionedQuerySets. It ensures that identities and time-limited where clauses are used when selecting related reverse foreign key objects. """ if queryset is None: @@ -893,7 +894,8 @@ def create_versioned_many_related_manager(superclass, rel): :param rel: Contains the ManyToMany relation :return: A subclass of ManyRelatedManager and Versionable """ - many_related_manager_klass = create_many_related_manager(superclass, rel) + # Django 1.9 compatibility note: create_many_related_manager becomes create_forward_many_to_many_manager + many_related_manager_klass = create_forward_many_to_many_manager(superclass, rel) class VersionedManyRelatedManager(many_related_manager_klass): def __init__(self, *args, **kwargs): @@ -1016,8 +1018,10 @@ def remove_at(self, timestamp, *objs): return VersionedManyRelatedManager -class VersionedReverseManyRelatedObjectsDescriptor(ReverseManyRelatedObjectsDescriptor): +class VersionedReverseManyRelatedObjectsDescriptor(ManyToManyDescriptor): """ + Django 1.9 compatibility note: ReverseManyRelatedObjectsDescriptor becomes ManyToManyDescriptor (as well) + Beside having a very long name, this class is useful when it comes to versioning the ReverseManyRelatedObjectsDescriptor (huhu!!). The main part is the exposure of the 'related_manager_cls' property @@ -1103,8 +1107,10 @@ def related_manager_cls(self): ) -class VersionedManyRelatedObjectsDescriptor(ManyRelatedObjectsDescriptor): +class VersionedManyRelatedObjectsDescriptor(ManyToManyDescriptor): """ + Django 1.9 compatibility note: ManyRelatedObjectsDescriptor becomes ManyToManyDescriptor + Beside having a very long name, this class is useful when it comes to versioning the ManyRelatedObjectsDescriptor (huhu!!). The main part is the exposure of the 'related_manager_cls' property @@ -1475,10 +1481,7 @@ def get_all_m2m_field_names(self): opts = self._meta rel_field_names = [field.attname for field in opts.many_to_many] if hasattr(opts, 'many_to_many_related'): - if VERSION[:2] >= (1, 9): - rel_field_names += [rel.reverse for rel in opts.many_to_many_related] - else: - rel_field_names += [rel.via_field_name for rel in opts.many_to_many_related] + rel_field_names += [rel.reverse for rel in opts.many_to_many_related] return rel_field_names diff --git a/versions_tests/tests/test_commands.py b/versions_tests/tests/test_commands.py index ff202d2..dab0a00 100644 --- a/versions_tests/tests/test_commands.py +++ b/versions_tests/tests/test_commands.py @@ -4,7 +4,6 @@ APP_NAME = 'versions_tests' -if django.VERSION[:2] >= (1, 7): - class TestMigrations(TestCase): - def test_makemigrations_command(self): - call_command('makemigrations', APP_NAME, dry_run=True, verbosity=0) +class TestMigrations(TestCase): + def test_makemigrations_command(self): + call_command('makemigrations', APP_NAME, dry_run=True, verbosity=0) diff --git a/versions_tests/tests/test_models.py b/versions_tests/tests/test_models.py index 4131a9b..f21ea09 100644 --- a/versions_tests/tests/test_models.py +++ b/versions_tests/tests/test_models.py @@ -1959,7 +1959,7 @@ def setUp(self): self.c1.team_set = [] self.team10 = Team.objects.current.get(identity=self.team10.identity).clone() - self.c10.team_set = [] + self.c10.team_set.clear() self.t3 = get_utc_now() def test_t1_relations_for_cloned_referenced_object(self): From fed0116a2c7ba24b3177626713a5cdc0296c5116 Mon Sep 17 00:00:00 2001 From: Manuel Jeckelmann Date: Wed, 28 Dec 2016 21:58:42 +0100 Subject: [PATCH 05/42] Did a slight cleanup, removing unneeded stuff - still need to work on code property... --- versions/descriptors.py | 10 +- versions/fields.py | 62 ++-- versions/models.py | 495 +------------------------- versions_tests/tests/test_commands.py | 3 + 4 files changed, 36 insertions(+), 534 deletions(-) diff --git a/versions/descriptors.py b/versions/descriptors.py index 82d4a69..3585e90 100644 --- a/versions/descriptors.py +++ b/versions/descriptors.py @@ -286,11 +286,10 @@ def __init__(self, *args, **kwargs): # is a thing we need to consider the "version_end_date IS NULL" case; # So, we define our own set of core filters being applied when versioning try: - version_start_date_field = self.through._meta.get_field('version_start_date') - version_end_date_field = self.through._meta.get_field('version_end_date') + _ = self.through._meta.get_field('version_start_date') + _ = self.through._meta.get_field('version_end_date') except FieldDoesNotExist as e: fields = [f.name for f in self.through._meta.get_fields()] - print(str(e) + "; available fields are " + ", ".join(fields)) raise e # FIXME: this probably does not work when auto-referencing @@ -327,9 +326,6 @@ def _remove_items_at(self, timestamp, source_field_name, target_field_name, *obj # The Django 1.7-way is preferred if hasattr(self, 'target_field'): fk_val = self.target_field.get_foreign_related_value(obj)[0] - # But the Django 1.6.x -way is supported for backward compatibility - elif hasattr(self, '_get_fk_val'): - fk_val = self._get_fk_val(obj, target_field_name) else: raise TypeError("We couldn't find the value of the foreign key, this might be due to the " "use of an unsupported version of Django") @@ -406,5 +402,3 @@ def remove_at(self, timestamp, *objs): remove_at.alters_data = True return VersionedManyRelatedManager - -# create_versioned_many_related_manager = create_versioned_forward_many_to_many_manager diff --git a/versions/fields.py b/versions/fields.py index dc377f3..97f2530 100644 --- a/versions/fields.py +++ b/versions/fields.py @@ -2,7 +2,8 @@ from django import VERSION from django.apps import apps -from django.db.models.fields.related import ForeignKey, ManyToManyField, RECURSIVE_RELATIONSHIP_CONSTANT +from django.db.models.fields.related import ForeignKey, ManyToManyField, RECURSIVE_RELATIONSHIP_CONSTANT, \ + resolve_relation from django.db.models.sql.datastructures import Join from django.db.models.sql.where import ExtraWhere, WhereNode @@ -14,6 +15,8 @@ # (new) => ReverseOneToOneDescriptor # from django.db.models.fields.related import (ForwardManyToOneDescriptor, ReverseManyToOneDescriptor, # ManyToManyDescriptor, ReverseOneToOneDescriptor) +from django.db.models.utils import make_model_tuple + from descriptors import (VersionedForwardManyToOneDescriptor, VersionedReverseManyToOneDescriptor, VersionedManyToManyDescriptor) @@ -33,10 +36,7 @@ def __init__(self, *args, **kwargs): def contribute_to_class(self, cls, name, virtual_only=False): super(VersionedForeignKey, self).contribute_to_class(cls, name, virtual_only) - if VERSION[:2] >= (1, 9): - setattr(cls, self.name, VersionedForwardManyToOneDescriptor(self)) - else: - setattr(cls, self.name, VersionedReverseSingleRelatedObjectDescriptor(self)) + setattr(cls, self.name, VersionedForwardManyToOneDescriptor(self)) def contribute_to_related_class(self, cls, related): """ @@ -47,10 +47,7 @@ def contribute_to_related_class(self, cls, related): super(VersionedForeignKey, self).contribute_to_related_class(cls, related) accessor_name = related.get_accessor_name() if hasattr(cls, accessor_name): - if VERSION[:2] >= (1, 9): - setattr(cls, accessor_name, VersionedReverseManyToOneDescriptor(related)) - else: - setattr(cls, accessor_name, VersionedForeignRelatedObjectsDescriptor(related)) + setattr(cls, accessor_name, VersionedReverseManyToOneDescriptor(related)) def get_extra_restriction(self, where_class, alias, remote_alias): """ @@ -95,10 +92,11 @@ class VersionedManyToManyField(ManyToManyField): def __init__(self, *args, **kwargs): super(VersionedManyToManyField, self).__init__(*args, **kwargs) - def contribute_to_class(self, cls, name): + def contribute_to_class(self, cls, name, **kwargs): """ Called at class type creation. So, this method is called, when metaclasses get created """ + # TODO: Apply 3 edge cases when not to create an intermediary model specified in django.db.models.fields.related:1566 # self.rel.through needs to be set prior to calling super, since super(...).contribute_to_class refers to it. # Classes pointed to by a string do not need to be resolved here, since Django does that at a later point in # time - which is nice... ;) @@ -107,17 +105,14 @@ def contribute_to_class(self, cls, name): # - creating the through class if unset # - resolving the through class if it's a string # - resolving string references within the through class - if not self.rel.through and not cls._meta.abstract and not cls._meta.swapped: - self.rel.through = VersionedManyToManyField.create_versioned_many_to_many_intermediary_model(self, cls, + if not self.remote_field.through and not cls._meta.abstract and not cls._meta.swapped: + self.remote_field.through = VersionedManyToManyField.create_versioned_many_to_many_intermediary_model(self, cls, name) super(VersionedManyToManyField, self).contribute_to_class(cls, name) # Overwrite the descriptor if hasattr(cls, self.name): - if VERSION[:2] >= (1, 9): - setattr(cls, self.name, VersionedManyToManyDescriptor(self.remote_field)) - else: - setattr(cls, self.name, VersionedReverseManyRelatedObjectsDescriptor(self)) + setattr(cls, self.name, VersionedManyToManyDescriptor(self.remote_field)) def contribute_to_related_class(self, cls, related): """ @@ -126,10 +121,7 @@ def contribute_to_related_class(self, cls, related): super(VersionedManyToManyField, self).contribute_to_related_class(cls, related) accessor_name = related.get_accessor_name() if accessor_name and hasattr(cls, accessor_name): - if VERSION[:2] >= (1, 9): - descriptor = VersionedManyToManyDescriptor(related, accessor_name) - else: - descriptor = VersionedManyRelatedObjectsDescriptor(related, accessor_name) + descriptor = VersionedManyToManyDescriptor(related, accessor_name) setattr(cls, accessor_name, descriptor) if hasattr(cls._meta, 'many_to_many_related') and isinstance(cls._meta.many_to_many_related, list): cls._meta.many_to_many_related.append(descriptor) @@ -138,38 +130,37 @@ def contribute_to_related_class(self, cls, related): @staticmethod def create_versioned_many_to_many_intermediary_model(field, cls, field_name): + # TODO: Verify functionality against django.db.models.fields.related:1048 # Let's not care too much on what flags could potentially be set on that intermediary class (e.g. managed, etc) # Let's play the game, as if the programmer had specified a class within his models... Here's how. from_ = cls._meta.model_name - to_model = field.rel.to + to_model = resolve_relation(cls, field.remote_field.model) # Force 'to' to be a string (and leave the hard work to Django) - if not isinstance(field.rel.to, six.string_types): - to_model = '%s.%s' % (field.rel.to._meta.app_label, field.rel.to._meta.object_name) - to = field.rel.to._meta.object_name.lower() - else: - to = to_model.lower() + to = make_model_tuple(to_model)[1] + # if not isinstance(field.rel.to, basestring): + # to_model = '%s.%s' % (field.rel.to._meta.app_label, field.rel.to._meta.object_name) + # to = field.rel.to._meta.object_name.lower() + # else: + # to = to_model.lower() name = '%s_%s' % (from_, field_name) - if field.rel.to == RECURSIVE_RELATIONSHIP_CONSTANT or to == cls._meta.object_name: - from_ = 'from_%s' % to + if to == from_: + from_ = 'from_%s' % from_ to = 'to_%s' % to - to_model = cls # Since Django 1.7, a migration mechanism is shipped by default with Django. This migration module loads all # declared apps' models inside a __fake__ module. # This means that the models can be already loaded and registered by their original module, when we # reach this point of the application and therefore there is no need to load them a second time. - if VERSION[:2] >= (1, 7) and cls.__module__ == '__fake__': + if cls.__module__ == '__fake__': try: # Check the apps for an already registered model - if VERSION[:2] >= (1, 9): - return apps.get_model(cls._meta.app_label, str(name)) - else: - return apps.get_registered_model(cls._meta.app_label, str(name)) + return apps.get_model(cls._meta.app_label, str(name)) except KeyError: # The model has not been registered yet, so continue + # TODO: Do we need to handle migrations differently here for intermediary M2M models? pass meta = type('Meta', (object,), { @@ -177,6 +168,9 @@ def create_versioned_many_to_many_intermediary_model(field, cls, field_name): 'auto_created': cls, 'db_tablespace': cls._meta.db_tablespace, 'app_label': cls._meta.app_label, + 'verbose_name': '%(from)s-%(to)s relationship' % {'from': from_, 'to': to}, + 'verbose_name_plural': '%(from)s-%(to)s relationships' % {'from': from_, 'to': to}, + 'apps': cls._meta.apps, }) return type(str(name), (Versionable,), { 'Meta': meta, diff --git a/versions/models.py b/versions/models.py index 6d12c8b..1bf10a3 100644 --- a/versions/models.py +++ b/versions/models.py @@ -20,6 +20,8 @@ from django import VERSION from django.db.models.fields.related_descriptors import ForwardManyToOneDescriptor, ReverseManyToOneDescriptor, \ ManyToManyDescriptor, create_forward_many_to_many_manager +from django.db.models.sql.query import Query +from django.db.models.sql.where import ExtraWhere, WhereNode from django.utils.functional import cached_property from versions.descriptors import VersionedForwardManyToOneDescriptor, VersionedReverseManyToOneDescriptor, \ @@ -35,10 +37,7 @@ from django.db.models.constants import LOOKUP_SEP from django.db.models.fields.related import ForeignKey, ManyToManyField, RECURSIVE_RELATIONSHIP_CONSTANT -from django.db.models.query import QuerySet -from django.db.models.query import ModelIterable -from django.db.models.sql import Query -from django.db.models.sql.where import ExtraWhere, WhereNode +from django.db.models.query import QuerySet, ModelIterable from django.utils.timezone import utc from django.utils import six @@ -651,494 +650,6 @@ def get_joining_columns(self, reverse_join=False): return joining_columns -class VersionedManyToManyField(ManyToManyField): - def __init__(self, *args, **kwargs): - super(VersionedManyToManyField, self).__init__(*args, **kwargs) - - def contribute_to_class(self, cls, name): - """ - Called at class type creation. So, this method is called, when metaclasses get created - """ - # self.rel.through needs to be set prior to calling super, since super(...).contribute_to_class refers to it. - # Classes pointed to by a string do not need to be resolved here, since Django does that at a later point in - # time - which is nice... ;) - # - # Superclasses take care of: - # - creating the through class if unset - # - resolving the through class if it's a string - # - resolving string references within the through class - if not self.rel.through and not cls._meta.abstract and not cls._meta.swapped: - self.rel.through = VersionedManyToManyField.create_versioned_many_to_many_intermediary_model(self, cls, - name) - super(VersionedManyToManyField, self).contribute_to_class(cls, name) - - # Overwrite the descriptor - if hasattr(cls, self.name): - setattr(cls, self.name, VersionedManyToManyDescriptor(self)) - - def contribute_to_related_class(self, cls, related): - """ - Called at class type creation. So, this method is called, when metaclasses get created - """ - super(VersionedManyToManyField, self).contribute_to_related_class(cls, related) - accessor_name = related.get_accessor_name() - if accessor_name and hasattr(cls, accessor_name): - descriptor = VersionedManyToManyDescriptor(related, accessor_name) - setattr(cls, accessor_name, descriptor) - if hasattr(cls._meta, 'many_to_many_related') and isinstance(cls._meta.many_to_many_related, list): - cls._meta.many_to_many_related.append(descriptor) - else: - cls._meta.many_to_many_related = [descriptor] - - @staticmethod - def create_versioned_many_to_many_intermediary_model(field, cls, field_name): - # Let's not care too much on what flags could potentially be set on that intermediary class (e.g. managed, etc) - # Let's play the game, as if the programmer had specified a class within his models... Here's how. - - from_ = cls._meta.model_name - to_model = field.rel.to - - # Force 'to' to be a string (and leave the hard work to Django) - if not isinstance(field.rel.to, six.string_types): - to_model = '%s.%s' % (field.rel.to._meta.app_label, field.rel.to._meta.object_name) - to = field.rel.to._meta.object_name.lower() - else: - to = to_model.lower() - name = '%s_%s' % (from_, field_name) - - if field.rel.to == RECURSIVE_RELATIONSHIP_CONSTANT or to == cls._meta.object_name: - from_ = 'from_%s' % to - to = 'to_%s' % to - to_model = cls - - # Since Django 1.7, a migration mechanism is shipped by default with Django. This migration module loads all - # declared apps' models inside a __fake__ module. - # This means that the models can be already loaded and registered by their original module, when we - # reach this point of the application and therefore there is no need to load them a second time. - if cls.__module__ == '__fake__': - try: - # Check the apps for an already registered model - return apps.get_registered_model(cls._meta.app_label, str(name)) - except KeyError: - # The model has not been registered yet, so continue - pass - - meta = type('Meta', (object,), { - # 'unique_together': (from_, to), - 'auto_created': cls, - 'db_tablespace': cls._meta.db_tablespace, - 'app_label': cls._meta.app_label, - }) - return type(str(name), (Versionable,), { - 'Meta': meta, - '__module__': cls.__module__, - from_: VersionedForeignKey(cls, related_name='%s+' % name, auto_created=name), - to: VersionedForeignKey(to_model, related_name='%s+' % name, auto_created=name), - }) - - -class VersionedReverseSingleRelatedObjectDescriptor(ForwardManyToOneDescriptor): - """ - Django1.9 compatibility note: ReverseSingleRelatedObjectDescriptor becomes ForwardManyToOneDescriptor - - A ReverseSingleRelatedObjectDescriptor-typed object gets inserted, when a ForeignKey - is defined in a Django model. This is one part of the analogue for versioned items. - - Unfortunately, we need to run two queries. The first query satisfies the foreign key - constraint. After extracting the identity information and combining it with the datetime- - stamp, we are able to fetch the historic element. - """ - - def __get__(self, instance, instance_type=None): - """ - The getter method returns the object, which points instance, e.g. choice.poll returns - a Poll instance, whereas the Poll class defines the ForeignKey. - :param instance: The object on which the property was accessed - :param instance_type: The type of the instance object - :return: Returns a Versionable - """ - current_elt = super(VersionedReverseSingleRelatedObjectDescriptor, self).__get__(instance, instance_type) - - if instance is None: - return self - - if not current_elt: - return None - - if not isinstance(current_elt, Versionable): - raise TypeError("VersionedForeignKey target is of type " - + str(type(current_elt)) - + ", which is not a subclass of Versionable") - - if hasattr(instance, '_querytime'): - # If current_elt matches the instance's querytime, there's no need to make a database query. - if Versionable.matches_querytime(current_elt, instance._querytime): - current_elt._querytime = instance._querytime - return current_elt - - return current_elt.__class__.objects.as_of(instance._querytime.time).get(identity=current_elt.identity) - else: - return current_elt.__class__.objects.current.get(identity=current_elt.identity) - - -class VersionedForeignRelatedObjectsDescriptor(ReverseManyToOneDescriptor): - """ - Django 1.9 compatibility note: ForeignRelatedObjectsDescriptor becomes ReverseManyToOneDescriptor - - This descriptor generates the manager class that is used on the related object of a ForeignKey relation - (i.e. the reverse-ForeignKey field manager). - """ - - @cached_property - def related_manager_cls(self): - # return create_versioned_related_manager - manager_cls = super(VersionedForeignRelatedObjectsDescriptor, self).related_manager_cls - rel_field = self.related.field - - class VersionedRelatedManager(manager_cls): - def __init__(self, instance): - super(VersionedRelatedManager, self).__init__(instance) - - # This is a hack, in order to get the versioned related objects - for key in self.core_filters.keys(): - if '__exact' in key or '__' not in key: - self.core_filters[key] = instance.identity - - def get_queryset(self): - queryset = super(VersionedRelatedManager, self).get_queryset() - # Do not set the query time if it is already correctly set. queryset.as_of() returns a clone - # of the queryset, and this will destroy the prefetched objects cache if it exists. - if isinstance(queryset, - VersionedQuerySet) and self.instance._querytime.active and queryset.querytime != self.instance._querytime: - queryset = queryset.as_of(self.instance._querytime.time) - return queryset - - def get_prefetch_queryset(self, instances, queryset=None): - """ - Overrides RelatedManager's implementation of get_prefetch_queryset so that it works - nicely with VersionedQuerySets. It ensures that identities and time-limited where - clauses are used when selecting related reverse foreign key objects. - """ - if queryset is None: - # Note that this intentionally call's VersionManager's get_queryset, instead of simply calling - # the superclasses' get_queryset (as the non-versioned RelatedManager does), because what is - # needed is a simple Versioned queryset without any restrictions (e.g. do not - # apply self.core_filters). - queryset = VersionManager.get_queryset(self) - - queryset._add_hints(instance=instances[0]) - queryset = queryset.using(queryset._db or self._db) - - rel_obj_attr = rel_field.get_local_related_value - instance_attr = rel_field.get_foreign_related_value - # Use identities instead of ids so that this will work with versioned objects. - instances_dict = {(inst.identity,): inst for inst in instances} - identities = [inst.identity for inst in instances] - query = {'%s__identity__in' % rel_field.name: identities} - queryset = queryset.filter(**query) - - # Since we just bypassed this class' get_queryset(), we must manage - # the reverse relation manually. - for rel_obj in queryset: - instance = instances_dict[rel_obj_attr(rel_obj)] - setattr(rel_obj, rel_field.name, instance) - cache_name = rel_field.related_query_name() - return queryset, rel_obj_attr, instance_attr, False, cache_name - - def add(self, *objs): - cloned_objs = () - for obj in objs: - if not isinstance(obj, Versionable): - raise TypeError("Trying to add a non-Versionable to a VersionedForeignKey relationship") - cloned_objs += (obj.clone(),) - super(VersionedRelatedManager, self).add(*cloned_objs) - - # clear() and remove() are present if the FK is nullable - if 'clear' in dir(manager_cls): - def clear(self, **kwargs): - """ - Overridden to ensure that the current queryset is used, and to clone objects before they - are removed, so that history is not lost. - """ - bulk = kwargs.pop('bulk', True) - db = router.db_for_write(self.model, instance=self.instance) - queryset = self.current.using(db) - with transaction.atomic(using=db, savepoint=False): - cloned_pks = [obj.clone().pk for obj in queryset] - update_qs = self.current.filter(pk__in=cloned_pks) - self._clear(update_qs, bulk) - - if 'remove' in dir(manager_cls): - def remove(self, *objs): - val = rel_field.get_foreign_related_value(self.instance) - cloned_objs = () - for obj in objs: - # Is obj actually part of this descriptor set? Otherwise, silently go over it, since Django - # handles that case - if rel_field.get_local_related_value(obj) == val: - # Silently pass over non-versionable items - if not isinstance(obj, Versionable): - raise TypeError( - "Trying to remove a non-Versionable from a VersionedForeignKey realtionship") - cloned_objs += (obj.clone(),) - super(VersionedRelatedManager, self).remove(*cloned_objs) - - return VersionedRelatedManager - - -def create_versioned_many_related_manager(superclass, rel): - """ - The "casting" which is done in this method is needed, since otherwise, the methods introduced by - Versionable are not taken into account. - :param superclass: This is usually a models.Manager - :param rel: Contains the ManyToMany relation - :return: A subclass of ManyRelatedManager and Versionable - """ - # Django 1.9 compatibility note: create_many_related_manager becomes create_forward_many_to_many_manager - many_related_manager_klass = create_forward_many_to_many_manager(superclass, rel) - - class VersionedManyRelatedManager(many_related_manager_klass): - def __init__(self, *args, **kwargs): - super(VersionedManyRelatedManager, self).__init__(*args, **kwargs) - try: - _ = self.through._meta.get_field('version_start_date') - _ = self.through._meta.get_field('version_end_date') - except FieldDoesNotExist as e: - fields = [f.name for f in self.through._meta.get_fields()] - print(str(e) + "; available fields are " + ", ".join(fields)) - raise e - # FIXME: this probably does not work when auto-referencing - - def get_queryset(self): - """ - Add a filter to the queryset, limiting the results to be pointed by relationship that are - valid for the given timestamp (which is taken at the current instance, or set to now, if not - available). - Long story short, apply the temporal validity filter also to the intermediary model. - """ - - queryset = super(VersionedManyRelatedManager, self).get_queryset() - if hasattr(queryset, 'querytime'): - if self.instance._querytime.active and self.instance._querytime != queryset.querytime: - queryset = queryset.as_of(self.instance._querytime.time) - return queryset - - def _remove_items(self, source_field_name, target_field_name, *objs): - """ - Instead of removing items, we simply set the version_end_date of the current item to the - current timestamp --> t[now]. - Like that, there is no more current entry having that identity - which is equal to - not existing for timestamps greater than t[now]. - """ - return self._remove_items_at(None, source_field_name, target_field_name, *objs) - - def _remove_items_at(self, timestamp, source_field_name, target_field_name, *objs): - if objs: - if timestamp is None: - timestamp = get_utc_now() - old_ids = set() - for obj in objs: - if isinstance(obj, self.model): - if hasattr(self, 'target_field'): - fk_val = self.target_field.get_foreign_related_value(obj)[0] - else: - raise TypeError("We couldn't find the value of the foreign key, this might be due to the " - "use of an unsupported version of Django") - old_ids.add(fk_val) - else: - old_ids.add(obj) - db = router.db_for_write(self.through, instance=self.instance) - qs = self.through._default_manager.using(db).filter(**{ - source_field_name: self.instance.id, - '%s__in' % target_field_name: old_ids - }).as_of(timestamp) - for relation in qs: - relation._delete_at(timestamp) - - if 'add' in dir(many_related_manager_klass): - def add(self, *objs): - if not self.instance.is_current: - raise SuspiciousOperation( - "Adding many-to-many related objects is only possible on the current version") - - # The ManyRelatedManager.add() method uses the through model's default manager to get - # a queryset when looking at which objects already exist in the database. - # In order to restrict the query to the current versions when that is done, - # we temporarily replace the queryset's using method so that the version validity - # condition can be specified. - klass = self.through._default_manager.get_queryset().__class__ - __using_backup = klass.using - - def using_replacement(self, *args, **kwargs): - qs = __using_backup(self, *args, **kwargs) - return qs.as_of(None) - - klass.using = using_replacement - super(VersionedManyRelatedManager, self).add(*objs) - klass.using = __using_backup - - def add_at(self, timestamp, *objs): - """ - This function adds an object at a certain point in time (timestamp) - """ - # First off, define the new constructor - def _through_init(self, *args, **kwargs): - super(self.__class__, self).__init__(*args, **kwargs) - self.version_birth_date = timestamp - self.version_start_date = timestamp - - # Through-classes have an empty constructor, so it can easily be overwritten when needed; - # This is not the default case, so the overwrite only takes place when we "modify the past" - self.through.__init_backup__ = self.through.__init__ - self.through.__init__ = _through_init - - # Do the add operation - self.add(*objs) - - # Remove the constructor again (by replacing it with the original empty constructor) - self.through.__init__ = self.through.__init_backup__ - del self.through.__init_backup__ - - add_at.alters_data = True - - if 'remove' in dir(many_related_manager_klass): - def remove_at(self, timestamp, *objs): - """ - Performs the act of removing specified relationships at a specified time (timestamp); - So, not the objects at a given time are removed, but their relationship! - """ - self._remove_items_at(timestamp, self.source_field_name, self.target_field_name, *objs) - - # For consistency, also handle the symmetrical case - if self.symmetrical: - self._remove_items_at(timestamp, self.target_field_name, self.source_field_name, *objs) - - remove_at.alters_data = True - - return VersionedManyRelatedManager - - -class VersionedReverseManyRelatedObjectsDescriptor(ManyToManyDescriptor): - """ - Django 1.9 compatibility note: ReverseManyRelatedObjectsDescriptor becomes ManyToManyDescriptor (as well) - - Beside having a very long name, this class is useful when it comes to versioning the - ReverseManyRelatedObjectsDescriptor (huhu!!). The main part is the exposure of the - 'related_manager_cls' property - """ - - def __get__(self, instance, owner=None): - """ - Reads the property as which this object is figuring; mainly used for debugging purposes - :param instance: The instance on which the getter was called - :param owner: no idea... alternatively called 'instance_type by the superclasses - :return: A VersionedManyRelatedManager object - """ - return super(VersionedReverseManyRelatedObjectsDescriptor, self).__get__(instance, owner) - - def __set__(self, instance, value): - """ - Completely overridden to avoid bulk deletion that happens when the parent method calls clear(). - - The parent method's logic is basically: clear all in bulk, then add the given objects in bulk. - Instead, we figure out which ones are being added and removed, and call add and remove for these values. - This lets us retain the versioning information. - - Since this is a many-to-many relationship, it is assumed here that the django.db.models.deletion.Collector - logic, that is used in clear(), is not necessary here. Collector collects related models, e.g. ones that should - also be deleted because they have a ON CASCADE DELETE relationship to the object, or, in the case of - "Multi-table inheritance", are parent objects. - - :param instance: The instance on which the getter was called - :param value: iterable of items to set - """ - - if not instance.is_current: - raise SuspiciousOperation( - "Related values can only be directly set on the current version of an object") - - if not self.field.rel.through._meta.auto_created: - opts = self.field.rel.through._meta - raise AttributeError(("Cannot set values on a ManyToManyField which specifies an intermediary model. " - "Use %s.%s's Manager instead.") % (opts.app_label, opts.object_name)) - - manager = self.__get__(instance) - # Below comment is from parent __set__ method. We'll force evaluation, too: - # clear() can change expected output of 'value' queryset, we force evaluation - # of queryset before clear; ticket #19816 - value = tuple(value) - - being_removed, being_added = self.get_current_m2m_diff(instance, value) - timestamp = get_utc_now() - manager.remove_at(timestamp, *being_removed) - manager.add_at(timestamp, *being_added) - - def get_current_m2m_diff(self, instance, new_objects): - """ - :param instance: Versionable object - :param new_objects: objects which are about to be associated with instance - :return: (being_removed id list, being_added id list) - :rtype : tuple - """ - new_ids = self.pks_from_objects(new_objects) - relation_manager = self.__get__(instance) - - filter = Q(**{relation_manager.source_field.attname: instance.pk}) - qs = self.through.objects.current.filter(filter) - target_name = relation_manager.target_field.attname - current_ids = set(qs.values_list(target_name, flat=True)) - - being_removed = current_ids - new_ids - being_added = new_ids - current_ids - return list(being_removed), list(being_added) - - def pks_from_objects(self, objects): - """ - Extract all the primary key strings from the given objects. Objects may be Versionables, or bare primary keys. - :rtype : set - """ - return {o.pk if isinstance(o, Model) else o for o in objects} - - @cached_property - def related_manager_cls(self): - return create_versioned_many_related_manager( - self.field.rel.to._default_manager.__class__, - self.field.rel - ) - - -class VersionedManyRelatedObjectsDescriptor(ManyToManyDescriptor): - """ - Django 1.9 compatibility note: ManyRelatedObjectsDescriptor becomes ManyToManyDescriptor - - Beside having a very long name, this class is useful when it comes to versioning the - ManyRelatedObjectsDescriptor (huhu!!). The main part is the exposure of the - 'related_manager_cls' property - """ - - via_field_name = None - - def __init__(self, related, via_field_name): - super(VersionedManyRelatedObjectsDescriptor, self).__init__(related) - self.via_field_name = via_field_name - - def __get__(self, instance, owner=None): - """ - Reads the property as which this object is figuring; mainly used for debugging purposes - :param instance: The instance on which the getter was called - :param owner: no idea... alternatively called 'instance_type by the superclasses - :return: A VersionedManyRelatedManager object - """ - return super(VersionedManyRelatedObjectsDescriptor, self).__get__(instance, owner) - - @cached_property - def related_manager_cls(self): - return create_versioned_many_related_manager( - self.related.model._default_manager.__class__, - self.related.field.rel - ) - - class Versionable(models.Model): """ This is pretty much the central point for versioning objects. diff --git a/versions_tests/tests/test_commands.py b/versions_tests/tests/test_commands.py index dab0a00..a3e4925 100644 --- a/versions_tests/tests/test_commands.py +++ b/versions_tests/tests/test_commands.py @@ -1,3 +1,5 @@ +from unittest.case import skip + import django from django.core.management import call_command from django.test import TestCase @@ -5,5 +7,6 @@ APP_NAME = 'versions_tests' class TestMigrations(TestCase): + @skip("Migrations for M2M intermediary models not properly handled yet") def test_makemigrations_command(self): call_command('makemigrations', APP_NAME, dry_run=True, verbosity=0) From 10c0dd947ff0a9ca1e3a1efdb7785b8e5ac316f6 Mon Sep 17 00:00:00 2001 From: Manuel Jeckelmann Date: Wed, 28 Dec 2016 22:13:39 +0100 Subject: [PATCH 06/42] Removed six package dependency --- versions/fields.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/versions/fields.py b/versions/fields.py index 97f2530..d482498 100644 --- a/versions/fields.py +++ b/versions/fields.py @@ -1,6 +1,3 @@ -import six - -from django import VERSION from django.apps import apps from django.db.models.fields.related import ForeignKey, ManyToManyField, RECURSIVE_RELATIONSHIP_CONSTANT, \ resolve_relation From 820040ea297d4c53c0350d4935024c46c0d0b71f Mon Sep 17 00:00:00 2001 From: Manuel Jeckelmann Date: Wed, 28 Dec 2016 22:28:19 +0100 Subject: [PATCH 07/42] Fixing wrong Django versioning assumption --- .travis.yml | 4 ++-- tox.ini | 4 ++-- versions_tests/tests/test_models.py | 8 ++------ 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1033c3d..0ae6634 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,8 +4,8 @@ python: "2.7" env: - TOX_ENV=py27-django19-pg - TOX_ENV=py27-django19-sqlite - - TOX_ENV=py36-django19-pg - - TOX_ENV=py36-django19-sqlite + - TOX_ENV=py35-django19-pg + - TOX_ENV=py35-django19-sqlite # Enable PostgreSQL usage addons: diff --git a/tox.ini b/tox.ini index 8e920c9..3ed344b 100644 --- a/tox.ini +++ b/tox.ini @@ -5,12 +5,12 @@ [tox] envlist = - py{27,36}-django{19}-{sqlite,pg} + py{27,35}-django{19}-{sqlite,pg} [testenv] deps = coverage - django19: django>=1.9,<2.0 + django19: django>=1.9,<1.10 pg: psycopg2 commands = pg: coverage run --source=versions ./manage.py test --settings={env:TOX_PG_CONF:cleanerversion.settings.pg} diff --git a/versions_tests/tests/test_models.py b/versions_tests/tests/test_models.py index f21ea09..988270a 100644 --- a/versions_tests/tests/test_models.py +++ b/versions_tests/tests/test_models.py @@ -38,12 +38,8 @@ def get_relation_table(model_class, fieldname): - - if VERSION[:2] >= (1, 8): - field_object = model_class._meta.get_field(fieldname) - direct = not field_object.auto_created or field_object.concrete - else: - field_object, _, direct, _ = model_class._meta.get_field_by_name(fieldname) + field_object = model_class._meta.get_field(fieldname) + direct = not field_object.auto_created or field_object.concrete if direct: field = field_object From ee3f2dd57a8034e795d3306dbfc5d8f624978c04 Mon Sep 17 00:00:00 2001 From: Manuel Jeckelmann Date: Wed, 28 Dec 2016 22:38:11 +0100 Subject: [PATCH 08/42] Some import adjustments and some PEP8 cleanup --- versions/deletion.py | 7 ++++--- versions/descriptors.py | 21 ++++++++------------- versions/fields.py | 37 +++++++++++-------------------------- versions/models.py | 34 ++++++++++------------------------ 4 files changed, 33 insertions(+), 66 deletions(-) diff --git a/versions/deletion.py b/versions/deletion.py index 92cabd8..819f1d6 100644 --- a/versions/deletion.py +++ b/versions/deletion.py @@ -1,9 +1,9 @@ -from django import VERSION from django.db.models.deletion import ( attrgetter, signals, six, sql, transaction, CASCADE, Collector, ) + import versions.models @@ -68,7 +68,8 @@ def delete(self, timestamp): # In the case of a SET.. method, clone before changing the value (if it hasn't already been # cloned) updated_instances = set() - if not(isinstance(field, versions.fields.VersionedForeignKey) and field.rel.on_delete == CASCADE): + if not ( + isinstance(field, versions.fields.VersionedForeignKey) and field.rel.on_delete == CASCADE): for instance in instances: # Clone before updating cloned = id_map.get(instance.pk, None) @@ -76,7 +77,7 @@ def delete(self, timestamp): cloned = instance.clone() id_map[instance.pk] = cloned updated_instances.add(cloned) - #TODO: instance should get updated with new values from clone ? + # TODO: instance should get updated with new values from clone ? instances_for_fieldvalues[(field, value)] = updated_instances # Replace the instances with their clones in self.data, too diff --git a/versions/descriptors.py b/versions/descriptors.py index 3585e90..a61a115 100644 --- a/versions/descriptors.py +++ b/versions/descriptors.py @@ -1,23 +1,14 @@ -from django import VERSION from django.core.exceptions import SuspiciousOperation, FieldDoesNotExist from django.db import router, transaction from django.db.models.base import Model +from django.db.models.fields.related import (ForwardManyToOneDescriptor, ReverseManyToOneDescriptor, + ManyToManyDescriptor) +from django.db.models.fields.related_descriptors import create_forward_many_to_many_manager from django.db.models.query_utils import Q - from django.utils.functional import cached_property from versions.util import get_utc_now -# With Django 1.9 related descriptor classes have been renamed: -# ReverseSingleRelatedObjectDescriptor => ForwardManyToOneDescriptor -# ForeignRelatedObjectsDescriptor => ReverseManyToOneDescriptor -# ReverseManyRelatedObjectsDescriptor => ManyToManyDescriptor -# ManyRelatedObjectsDescriptor => ManyToManyDescriptor -# (new) => ReverseOneToOneDescriptor -from django.db.models.fields.related import (ForwardManyToOneDescriptor, ReverseManyToOneDescriptor, - ManyToManyDescriptor) -from django.db.models.fields.related_descriptors import create_forward_many_to_many_manager - def matches_querytime(instance, querytime): """ @@ -41,6 +32,8 @@ class VersionedForwardManyToOneDescriptor(ForwardManyToOneDescriptor): """ pass + + vforward_many_to_one_descriptor_class = VersionedForwardManyToOneDescriptor @@ -76,6 +69,7 @@ def vforward_many_to_one_descriptor_getter(self, instance, instance_type=None): else: return current_elt.__class__.objects.current.get(identity=current_elt.identity) + vforward_many_to_one_descriptor_class.__get__ = vforward_many_to_one_descriptor_getter @@ -85,12 +79,13 @@ def related_manager_cls(self): # return create_versioned_related_manager manager_cls = super(VersionedReverseManyToOneDescriptor, self).related_manager_cls - #if VERSION[:2] >= (1, 9): + # if VERSION[:2] >= (1, 9): # TODO: Define, what field has to be taken over here, self.rel/self.field? The WhineDrinker.hats test seems to be a good one for testing this # rel_field = self.rel rel_field = self.field + # elif hasattr(self, 'related'): # rel_field = self.related.field # else: diff --git a/versions/fields.py b/versions/fields.py index d482498..ad128a5 100644 --- a/versions/fields.py +++ b/versions/fields.py @@ -1,23 +1,14 @@ from django.apps import apps -from django.db.models.fields.related import ForeignKey, ManyToManyField, RECURSIVE_RELATIONSHIP_CONSTANT, \ +from django.db.models.fields.related import ForeignKey, ManyToManyField, \ resolve_relation from django.db.models.sql.datastructures import Join from django.db.models.sql.where import ExtraWhere, WhereNode - -# With Django 1.9 related descriptor classes have been renamed: -# ReverseSingleRelatedObjectDescriptor => ForwardManyToOneDescriptor -# ForeignRelatedObjectsDescriptor => ReverseManyToOneDescriptor -# ReverseManyRelatedObjectsDescriptor => ManyToManyDescriptor -# ManyRelatedObjectsDescriptor => ManyToManyDescriptor -# (new) => ReverseOneToOneDescriptor -# from django.db.models.fields.related import (ForwardManyToOneDescriptor, ReverseManyToOneDescriptor, -# ManyToManyDescriptor, ReverseOneToOneDescriptor) from django.db.models.utils import make_model_tuple -from descriptors import (VersionedForwardManyToOneDescriptor, +from versions.descriptors import (VersionedForwardManyToOneDescriptor, VersionedReverseManyToOneDescriptor, VersionedManyToManyDescriptor) -from models import Versionable +from versions.models import Versionable class VersionedForeignKey(ForeignKey): @@ -85,6 +76,7 @@ def get_joining_columns(self, reverse_join=False): joining_columns = joining_columns + ((lhs_col_name, rhs_col_name),) return joining_columns + class VersionedManyToManyField(ManyToManyField): def __init__(self, *args, **kwargs): super(VersionedManyToManyField, self).__init__(*args, **kwargs) @@ -103,8 +95,9 @@ def contribute_to_class(self, cls, name, **kwargs): # - resolving the through class if it's a string # - resolving string references within the through class if not self.remote_field.through and not cls._meta.abstract and not cls._meta.swapped: - self.remote_field.through = VersionedManyToManyField.create_versioned_many_to_many_intermediary_model(self, cls, - name) + self.remote_field.through = VersionedManyToManyField.create_versioned_many_to_many_intermediary_model(self, + cls, + name) super(VersionedManyToManyField, self).contribute_to_class(cls, name) # Overwrite the descriptor @@ -176,6 +169,7 @@ def create_versioned_many_to_many_intermediary_model(field, cls, field_name): to: VersionedForeignKey(to_model, related_name='%s+' % name, auto_created=name), }) + class VersionedExtraWhere(ExtraWhere): """ A specific implementation of ExtraWhere; @@ -257,22 +251,13 @@ def as_sql(self, qn, connection): # self.children is an array of VersionedExtraWhere-objects for child in self.children: if isinstance(child, VersionedExtraWhere) and not child.params: - try: - # Django 1.7 & 1.8 handles compilers as objects - _query = qn.query - except AttributeError: - # Django 1.6 handles compilers as instancemethods - _query = qn.__self__.query + # Django 1.7 & 1.8 handles compilers as objects + _query = qn.query query_time = _query.querytime.time apply_query_time = _query.querytime.active alias_map = _query.alias_map - # In Django 1.6 & 1.7, use the join_map to know, what *table* gets joined to which - # *left-hand sided* table # In Django 1.8, use the Join objects in alias_map - if hasattr(_query, 'join_map'): - self._set_child_joined_alias_using_join_map(child, _query.join_map, alias_map) - else: - self._set_child_joined_alias(child, alias_map) + self._set_child_joined_alias(child, alias_map) if apply_query_time: # Add query parameters that have not been added till now child.set_as_of(query_time) diff --git a/versions/models.py b/versions/models.py index 1bf10a3..be68d2f 100644 --- a/versions/models.py +++ b/versions/models.py @@ -17,33 +17,23 @@ import uuid from collections import namedtuple -from django import VERSION -from django.db.models.fields.related_descriptors import ForwardManyToOneDescriptor, ReverseManyToOneDescriptor, \ - ManyToManyDescriptor, create_forward_many_to_many_manager -from django.db.models.sql.query import Query -from django.db.models.sql.where import ExtraWhere, WhereNode -from django.utils.functional import cached_property - -from versions.descriptors import VersionedForwardManyToOneDescriptor, VersionedReverseManyToOneDescriptor, \ - VersionedManyToManyDescriptor -from versions.util import get_utc_now - -from django.db.models.sql.datastructures import Join -from django.apps.registry import apps -from django.core.exceptions import SuspiciousOperation, ObjectDoesNotExist, FieldDoesNotExist +from django.core.exceptions import SuspiciousOperation, ObjectDoesNotExist from django.db import models, router, transaction -from django.db.models.base import Model from django.db.models import Q from django.db.models.constants import LOOKUP_SEP -from django.db.models.fields.related import ForeignKey, ManyToManyField, RECURSIVE_RELATIONSHIP_CONSTANT - +from django.db.models.fields.related import ForeignKey from django.db.models.query import QuerySet, ModelIterable -from django.utils.timezone import utc +from django.db.models.sql.datastructures import Join +from django.db.models.sql.query import Query +from django.db.models.sql.where import ExtraWhere, WhereNode from django.utils import six +from django.utils.timezone import utc +from versions.descriptors import VersionedForwardManyToOneDescriptor, VersionedReverseManyToOneDescriptor +from versions.exceptions import DeletionOfNonCurrentVersionError from versions.settings import get_versioned_delete_collector_class from versions.settings import settings as versions_settings -from versions.exceptions import DeletionOfNonCurrentVersionError +from versions.util import get_utc_now def get_utc_now(): @@ -56,6 +46,7 @@ def validate_uuid(uuid_obj): """ return isinstance(uuid_obj, uuid.UUID) and uuid_obj.version == 4 + QueryTime = namedtuple('QueryTime', 'time active') @@ -532,11 +523,6 @@ def _set_item_querytime(self, item, type_check=True): item._querytime = self.querytime elif isinstance(item, VersionedQuerySet): item.querytime = self.querytime - elif isinstance(self, ValuesQuerySet): - # When we are dealing with a ValueQuerySet there is no point in - # setting the query_time as we are returning an array of values - # instead of a full-fledged model object - pass else: if type_check: raise TypeError("This item is not a Versionable, it's a " + str(type(item))) From d9d9b2007ead7be9da92a07d5b9d0ae270411db8 Mon Sep 17 00:00:00 2001 From: Manuel Jeckelmann Date: Wed, 4 Jan 2017 16:13:29 +0100 Subject: [PATCH 09/42] Some cleanup to avoid double defined classes --- versions/fields.py | 1 - versions/models.py | 149 +++---------------------------------------- versions/settings.py | 1 - 3 files changed, 9 insertions(+), 142 deletions(-) diff --git a/versions/fields.py b/versions/fields.py index ad128a5..4d9000e 100644 --- a/versions/fields.py +++ b/versions/fields.py @@ -10,7 +10,6 @@ VersionedManyToManyDescriptor) from versions.models import Versionable - class VersionedForeignKey(ForeignKey): """ We need to replace the standard ForeignKey declaration in order to be able to introduce diff --git a/versions/models.py b/versions/models.py index be68d2f..14b6d7f 100644 --- a/versions/models.py +++ b/versions/models.py @@ -25,14 +25,11 @@ from django.db.models.query import QuerySet, ModelIterable from django.db.models.sql.datastructures import Join from django.db.models.sql.query import Query -from django.db.models.sql.where import ExtraWhere, WhereNode -from django.utils import six +from django.db.models.sql.where import WhereNode from django.utils.timezone import utc -from versions.descriptors import VersionedForwardManyToOneDescriptor, VersionedReverseManyToOneDescriptor from versions.exceptions import DeletionOfNonCurrentVersionError from versions.settings import get_versioned_delete_collector_class -from versions.settings import settings as versions_settings from versions.util import get_utc_now @@ -271,6 +268,7 @@ def as_sql(self, qn, connection): :return: A tuple consisting of (sql_string, result_params) """ # self.children is an array of VersionedExtraWhere-objects + from versions.fields import VersionedExtraWhere for child in self.children: if isinstance(child, VersionedExtraWhere) and not child.params: _query = qn.query @@ -304,60 +302,6 @@ def _set_child_joined_alias(child, alias_map): break -class VersionedExtraWhere(ExtraWhere): - """ - A specific implementation of ExtraWhere; - Before as_sql can be called on an object, ensure that calls to - - set_as_of and - - set_joined_alias - have been done - """ - - def __init__(self, historic_sql, current_sql, alias, remote_alias): - super(VersionedExtraWhere, self).__init__(sqls=[], params=[]) - self.historic_sql = historic_sql - self.current_sql = current_sql - self.alias = alias - self.related_alias = remote_alias - self._as_of_time_set = False - self.as_of_time = None - self._joined_alias = None - - def set_as_of(self, as_of_time): - self.as_of_time = as_of_time - self._as_of_time_set = True - - def set_joined_alias(self, joined_alias): - """ - Takes the alias that is being joined to the query and applies the query - time constraint to its table - - :param str joined_alias: The table name of the alias - """ - self._joined_alias = joined_alias - - def as_sql(self, qn=None, connection=None): - sql = "" - params = [] - - # Fail fast for inacceptable cases - if self._as_of_time_set and not self._joined_alias: - raise ValueError("joined_alias is not set, but as_of is; this is a conflict!") - - # Set the SQL string in dependency of whether as_of_time was set or not - if self._as_of_time_set: - if self.as_of_time: - sql = self.historic_sql - params = [self.as_of_time] * 2 - # 2 is the number of occurences of the timestamp in an as_of-filter expression - else: - # If as_of_time was set to None, we're dealing with a query for "current" values - sql = self.current_sql - else: - # No as_of_time has been set; Perhaps, as_of was not part of the query -> That's OK - pass - - class VersionedQuery(Query): """ VersionedQuery has awareness of the query time restrictions. When the query is compiled, @@ -570,72 +514,6 @@ def delete(self): delete.queryset_only = True -class VersionedForeignKey(ForeignKey): - """ - We need to replace the standard ForeignKey declaration in order to be able to introduce - the VersionedReverseSingleRelatedObjectDescriptor, which allows to go back in time... - We also want to allow keeping track of any as_of time so that joins can be restricted - based on that. - """ - - def __init__(self, *args, **kwargs): - super(VersionedForeignKey, self).__init__(*args, **kwargs) - - def contribute_to_class(self, cls, name, virtual_only=False): - super(VersionedForeignKey, self).contribute_to_class(cls, name, virtual_only) - setattr(cls, self.name, VersionedForwardManyToOneDescriptor(self)) - - def contribute_to_related_class(self, cls, related): - """ - Override ForeignKey's methods, and replace the descriptor, if set by the parent's methods - """ - # Internal FK's - i.e., those with a related name ending with '+' - - # and swapped models don't get a related descriptor. - super(VersionedForeignKey, self).contribute_to_related_class(cls, related) - accessor_name = related.get_accessor_name() - if hasattr(cls, accessor_name): - setattr(cls, accessor_name, VersionedReverseManyToOneDescriptor(related)) - - def get_extra_restriction(self, where_class, alias, remote_alias): - """ - Overrides ForeignObject's get_extra_restriction function that returns an SQL statement which is appended to a - JOIN's conditional filtering part - - :return: SQL conditional statement - :rtype: WhereNode - """ - historic_sql = '''{alias}.version_start_date <= %s - AND ({alias}.version_end_date > %s OR {alias}.version_end_date is NULL )''' - current_sql = '''{alias}.version_end_date is NULL''' - # How 'bout creating an ExtraWhere here, without params - return where_class([VersionedExtraWhere(historic_sql=historic_sql, current_sql=current_sql, alias=alias, - remote_alias=remote_alias)]) - - def get_joining_columns(self, reverse_join=False): - """ - Get and return joining columns defined by this foreign key relationship - - :return: A tuple containing the column names of the tables to be joined (, ) - :rtype: tuple - """ - source = self.reverse_related_fields if reverse_join else self.related_fields - joining_columns = tuple() - for lhs_field, rhs_field in source: - lhs_col_name = lhs_field.column - rhs_col_name = rhs_field.column - # Test whether - # - self is the current ForeignKey relationship - # - self was not auto_created (e.g. is not part of a M2M relationship) - if self is lhs_field and not self.auto_created: - if rhs_col_name == Versionable.VERSION_IDENTIFIER_FIELD: - rhs_col_name = Versionable.OBJECT_IDENTIFIER_FIELD - elif self is rhs_field and not self.auto_created: - if lhs_col_name == Versionable.VERSION_IDENTIFIER_FIELD: - lhs_col_name = Versionable.OBJECT_IDENTIFIER_FIELD - joining_columns = joining_columns + ((lhs_col_name, rhs_col_name),) - return joining_columns - - class Versionable(models.Model): """ This is pretty much the central point for versioning objects. @@ -645,17 +523,11 @@ class Versionable(models.Model): VERSIONABLE_FIELDS = [VERSION_IDENTIFIER_FIELD, OBJECT_IDENTIFIER_FIELD, 'version_start_date', 'version_end_date', 'version_birth_date'] - if versions_settings.VERSIONS_USE_UUIDFIELD: - id = models.UUIDField(primary_key=True) - """id stands for ID and is the primary key; sometimes also referenced as the surrogate key""" - else: - id = models.CharField(max_length=36, primary_key=True) + id = models.UUIDField(primary_key=True) + """id stands for ID and is the primary key; sometimes also referenced as the surrogate key""" - if versions_settings.VERSIONS_USE_UUIDFIELD: - identity = models.UUIDField() - """identity is used as the identifier of an object, ignoring its versions; sometimes also referenced as the natural key""" - else: - identity = models.CharField(max_length=36) + identity = models.UUIDField() + """identity is used as the identifier of an object, ignoring its versions; sometimes also referenced as the natural key""" version_start_date = models.DateTimeField() """version_start_date points the moment in time, when a version was created (ie. an versionable was cloned). @@ -700,7 +572,7 @@ def __init__(self, *args, **kwargs): if not getattr(self, self.OBJECT_IDENTIFIER_FIELD, None): setattr(self, self.OBJECT_IDENTIFIER_FIELD, getattr(self, self.VERSION_IDENTIFIER_FIELD)) - def delete(self, using=None): + def delete(self, using=None, keep_parents=False): using = using or router.db_for_write(self.__class__, instance=self) assert self._get_pk_val() is not None, \ "{} object can't be deleted because its {} attribute is set to None.".format( @@ -708,7 +580,7 @@ def delete(self, using=None): collector_class = get_versioned_delete_collector_class() collector = collector_class(using=using) - collector.collect([self]) + collector.collect([self], keep_parents=keep_parents) collector.delete(get_utc_now()) def _delete_at(self, timestamp, using=None): @@ -776,10 +648,7 @@ def uuid(uuid_value=None): else: uuid_value = uuid.uuid4() - if versions_settings.VERSIONS_USE_UUIDFIELD: - return uuid_value - else: - return six.u(str(uuid_value)) + return uuid_value def _clone_at(self, timestamp): """ diff --git a/versions/settings.py b/versions/settings.py index 272354b..19bb508 100644 --- a/versions/settings.py +++ b/versions/settings.py @@ -18,7 +18,6 @@ class VersionsSettings(object): defaults = { 'VERSIONED_DELETE_COLLECTOR': 'versions.deletion.VersionedCollector', - 'VERSIONS_USE_UUIDFIELD': VERSION[:3] >= (1, 8, 3), } def __getattr__(self, name): From 810474c1e221ba2f17977f95422df9b25f0a427e Mon Sep 17 00:00:00 2001 From: Manuel Jeckelmann Date: Thu, 5 Jan 2017 09:52:21 +0100 Subject: [PATCH 10/42] Added the VERSIONS_USE_UUIDFIELD setting back again for data backward compatibility (in case someone started his project on a pre-1.8.3 Django version) --- versions/models.py | 23 +++++++++++++++++------ versions/settings.py | 1 + 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/versions/models.py b/versions/models.py index 14b6d7f..f706b77 100644 --- a/versions/models.py +++ b/versions/models.py @@ -17,6 +17,7 @@ import uuid from collections import namedtuple +import six from django.core.exceptions import SuspiciousOperation, ObjectDoesNotExist from django.db import models, router, transaction from django.db.models import Q @@ -29,7 +30,7 @@ from django.utils.timezone import utc from versions.exceptions import DeletionOfNonCurrentVersionError -from versions.settings import get_versioned_delete_collector_class +from versions.settings import get_versioned_delete_collector_class, settings as versions_settings from versions.util import get_utc_now @@ -523,11 +524,18 @@ class Versionable(models.Model): VERSIONABLE_FIELDS = [VERSION_IDENTIFIER_FIELD, OBJECT_IDENTIFIER_FIELD, 'version_start_date', 'version_end_date', 'version_birth_date'] - id = models.UUIDField(primary_key=True) - """id stands for ID and is the primary key; sometimes also referenced as the surrogate key""" + if versions_settings.VERSIONS_USE_UUIDFIELD: + id = models.UUIDField(primary_key=True) + """id stands for ID and is the primary key; sometimes also referenced as the surrogate key""" + else: + id = models.CharField(max_length=36, primary_key=True) - identity = models.UUIDField() - """identity is used as the identifier of an object, ignoring its versions; sometimes also referenced as the natural key""" + if versions_settings.VERSIONS_USE_UUIDFIELD: + identity = models.UUIDField() + """identity is used as the identifier of an object, ignoring its versions; sometimes also referenced as the natural key""" + else: + identity = models.CharField(max_length=36) + """identity is used as the identifier of an object, ignoring its versions; sometimes also referenced as the natural key""" version_start_date = models.DateTimeField() """version_start_date points the moment in time, when a version was created (ie. an versionable was cloned). @@ -648,7 +656,10 @@ def uuid(uuid_value=None): else: uuid_value = uuid.uuid4() - return uuid_value + if versions_settings.VERSIONS_USE_UUIDFIELD: + return uuid_value + else: + return six.u(str(uuid_value)) def _clone_at(self, timestamp): """ diff --git a/versions/settings.py b/versions/settings.py index 19bb508..272354b 100644 --- a/versions/settings.py +++ b/versions/settings.py @@ -18,6 +18,7 @@ class VersionsSettings(object): defaults = { 'VERSIONED_DELETE_COLLECTOR': 'versions.deletion.VersionedCollector', + 'VERSIONS_USE_UUIDFIELD': VERSION[:3] >= (1, 8, 3), } def __getattr__(self, name): From 5cf9b7cd30c3a51d3a7289a6b7afef106cd71fcd Mon Sep 17 00:00:00 2001 From: Manuel Jeckelmann Date: Sun, 29 Jan 2017 15:32:14 +0100 Subject: [PATCH 11/42] Added requirements.txt file --- requirements.txt | 1 + tox.ini | 1 + 2 files changed, 2 insertions(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..dde3933 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +six>=1.10.0 diff --git a/tox.ini b/tox.ini index 3ed344b..2399a0b 100644 --- a/tox.ini +++ b/tox.ini @@ -10,6 +10,7 @@ envlist = [testenv] deps = coverage + -rrequirements.txt django19: django>=1.9,<1.10 pg: psycopg2 commands = From 34a032b13fca9d1495dabe52bf78f1b6e274e8d6 Mon Sep 17 00:00:00 2001 From: Manuel Jeckelmann Date: Sun, 29 Jan 2017 21:41:11 +0100 Subject: [PATCH 12/42] Adapted 'create_versioned_many_to_many_model' to better support M2M intermediary models, allowing for M2M migrations --- versions/fields.py | 66 +++++++++++++++------------ versions_tests/models.py | 2 +- versions_tests/tests/test_commands.py | 5 +- 3 files changed, 40 insertions(+), 33 deletions(-) diff --git a/versions/fields.py b/versions/fields.py index 4d9000e..bcdd292 100644 --- a/versions/fields.py +++ b/versions/fields.py @@ -1,6 +1,6 @@ -from django.apps import apps +from django.db.models.deletion import DO_NOTHING from django.db.models.fields.related import ForeignKey, ManyToManyField, \ - resolve_relation + resolve_relation, lazy_related_operation from django.db.models.sql.datastructures import Join from django.db.models.sql.where import ExtraWhere, WhereNode from django.db.models.utils import make_model_tuple @@ -10,6 +10,7 @@ VersionedManyToManyDescriptor) from versions.models import Versionable + class VersionedForeignKey(ForeignKey): """ We need to replace the standard ForeignKey declaration in order to be able to introduce @@ -94,6 +95,9 @@ def contribute_to_class(self, cls, name, **kwargs): # - resolving the through class if it's a string # - resolving string references within the through class if not self.remote_field.through and not cls._meta.abstract and not cls._meta.swapped: + # We need to anticipate some stuff, that's done only later in class contribution + self.set_attributes_from_name(name) + self.model = cls self.remote_field.through = VersionedManyToManyField.create_versioned_many_to_many_intermediary_model(self, cls, name) @@ -123,49 +127,55 @@ def create_versioned_many_to_many_intermediary_model(field, cls, field_name): # Let's not care too much on what flags could potentially be set on that intermediary class (e.g. managed, etc) # Let's play the game, as if the programmer had specified a class within his models... Here's how. - from_ = cls._meta.model_name + # FIXME: VersionedManyToManyModels do not get registered in the apps models. + # FIXME: This is usually done at django/db/models/base.py:284, + # invoked by create_many_to_many_intermediary_model at django.db.models.fields.related:1048 + + def set_managed(model, related, through): + through._meta.managed = model._meta.managed or related._meta.managed + to_model = resolve_relation(cls, field.remote_field.model) + name = '%s_%s' % (cls._meta.object_name, field_name) + lazy_related_operation(set_managed, cls, to_model, name) + # Force 'to' to be a string (and leave the hard work to Django) to = make_model_tuple(to_model)[1] - # if not isinstance(field.rel.to, basestring): - # to_model = '%s.%s' % (field.rel.to._meta.app_label, field.rel.to._meta.object_name) - # to = field.rel.to._meta.object_name.lower() - # else: - # to = to_model.lower() - name = '%s_%s' % (from_, field_name) - + from_ = cls._meta.model_name if to == from_: from_ = 'from_%s' % from_ to = 'to_%s' % to - # Since Django 1.7, a migration mechanism is shipped by default with Django. This migration module loads all - # declared apps' models inside a __fake__ module. - # This means that the models can be already loaded and registered by their original module, when we - # reach this point of the application and therefore there is no need to load them a second time. - if cls.__module__ == '__fake__': - try: - # Check the apps for an already registered model - return apps.get_model(cls._meta.app_label, str(name)) - except KeyError: - # The model has not been registered yet, so continue - # TODO: Do we need to handle migrations differently here for intermediary M2M models? - pass - meta = type('Meta', (object,), { - # 'unique_together': (from_, to), + 'db_table': field._get_m2m_db_table(cls._meta), 'auto_created': cls, - 'db_tablespace': cls._meta.db_tablespace, 'app_label': cls._meta.app_label, + 'db_tablespace': cls._meta.db_tablespace, + # 'unique_together' is not applicable as is, due to multiple versions to be allowed to exist. + # 'unique_together': (from_, to), 'verbose_name': '%(from)s-%(to)s relationship' % {'from': from_, 'to': to}, 'verbose_name_plural': '%(from)s-%(to)s relationships' % {'from': from_, 'to': to}, - 'apps': cls._meta.apps, + 'apps': field.model._meta.apps, }) return type(str(name), (Versionable,), { 'Meta': meta, '__module__': cls.__module__, - from_: VersionedForeignKey(cls, related_name='%s+' % name, auto_created=name), - to: VersionedForeignKey(to_model, related_name='%s+' % name, auto_created=name), + from_: VersionedForeignKey( + cls, + related_name='%s+' % name, + db_tablespace=field.db_tablespace, + db_constraint=field.remote_field.db_constraint, + auto_created=name, + on_delete=DO_NOTHING, + ), + to: VersionedForeignKey( + to_model, + related_name='%s+' % name, + db_tablespace=field.db_tablespace, + db_constraint=field.remote_field.db_constraint, + auto_created=name, + on_delete=DO_NOTHING, + ), }) diff --git a/versions_tests/models.py b/versions_tests/models.py index 3c4c28d..e352b80 100644 --- a/versions_tests/models.py +++ b/versions_tests/models.py @@ -2,8 +2,8 @@ from django.db.models.deletion import DO_NOTHING, PROTECT, SET, SET_NULL from django.utils.encoding import python_2_unicode_compatible -from versions.models import Versionable from versions.fields import VersionedManyToManyField, VersionedForeignKey +from versions.models import Versionable def versionable_description(obj): diff --git a/versions_tests/tests/test_commands.py b/versions_tests/tests/test_commands.py index a3e4925..011ac2d 100644 --- a/versions_tests/tests/test_commands.py +++ b/versions_tests/tests/test_commands.py @@ -1,12 +1,9 @@ -from unittest.case import skip - -import django from django.core.management import call_command from django.test import TestCase APP_NAME = 'versions_tests' + class TestMigrations(TestCase): - @skip("Migrations for M2M intermediary models not properly handled yet") def test_makemigrations_command(self): call_command('makemigrations', APP_NAME, dry_run=True, verbosity=0) From 612d2a5b2cad123c65f7f10f48e1d30e2180ef97 Mon Sep 17 00:00:00 2001 From: Manuel Jeckelmann Date: Sun, 29 Jan 2017 22:01:07 +0100 Subject: [PATCH 13/42] Adapted 'versions_tests' project and general settings to eliminate migration-time warnings --- cleanerversion/settings/base.py | 21 ++++++++++++++------- cleanerversion/urls.py | 7 ++++--- versions_tests/models.py | 5 +++-- versions_tests/urls.py | 4 ---- 4 files changed, 21 insertions(+), 16 deletions(-) delete mode 100644 versions_tests/urls.py diff --git a/cleanerversion/settings/base.py b/cleanerversion/settings/base.py index 7dfdfbf..e4fd2ab 100644 --- a/cleanerversion/settings/base.py +++ b/cleanerversion/settings/base.py @@ -9,9 +9,11 @@ """ from __future__ import absolute_import -from django.utils.crypto import get_random_string + import os +from django.utils.crypto import get_random_string + BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)) + '/..') # SECURITY WARNING: keep the secret key used in production secret! @@ -20,9 +22,17 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True -TEMPLATE_DEBUG = True - - +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': ['django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages'], + }, + }, +] ALLOWED_HOSTS = [] @@ -85,9 +95,6 @@ USE_L10N = True USE_TZ = True - ROOT_URLCONF = 'cleanerversion.urls' - STATIC_URL = '/static/' - diff --git a/cleanerversion/urls.py b/cleanerversion/urls.py index 3a6e6c4..baa40c2 100644 --- a/cleanerversion/urls.py +++ b/cleanerversion/urls.py @@ -1,5 +1,6 @@ -from django.conf.urls import patterns, url +from django.conf.urls import url from django.contrib import admin -urlpatterns = patterns('', - url(r'^admin/', admin.site.urls, ), ) +urlpatterns = [ + url(r'^admin/', admin.site.urls, ), +] diff --git a/versions_tests/models.py b/versions_tests/models.py index e352b80..0bbbc47 100644 --- a/versions_tests/models.py +++ b/versions_tests/models.py @@ -1,9 +1,10 @@ from django.db.models import CharField, IntegerField, Model, ForeignKey from django.db.models.deletion import DO_NOTHING, PROTECT, SET, SET_NULL +from django.db.models.fields.related import ManyToManyField from django.utils.encoding import python_2_unicode_compatible -from versions.fields import VersionedManyToManyField, VersionedForeignKey from versions.models import Versionable +from versions.fields import VersionedManyToManyField, VersionedForeignKey def versionable_description(obj): @@ -266,4 +267,4 @@ def __str__(self): # SelfReferencingManyToManyTest models class Person(Versionable): name = CharField(max_length=200) - children = VersionedManyToManyField('self', symmetrical=False, null=True, related_name='parents') + children = VersionedManyToManyField('self', symmetrical=False, related_name='parents') diff --git a/versions_tests/urls.py b/versions_tests/urls.py deleted file mode 100644 index 93005f9..0000000 --- a/versions_tests/urls.py +++ /dev/null @@ -1,4 +0,0 @@ -from versions_tests.urls import urlpatterns - - -urlpatterns = urlpatterns From f4ffbaf2c30215383ad51f4463a377ee759f3bcb Mon Sep 17 00:00:00 2001 From: Manuel Jeckelmann Date: Wed, 8 Feb 2017 21:40:59 +0100 Subject: [PATCH 14/42] Removed 'six' dependency --- requirements.txt | 1 - versions/models.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index dde3933..0000000 --- a/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -six>=1.10.0 diff --git a/versions/models.py b/versions/models.py index f706b77..a56eb76 100644 --- a/versions/models.py +++ b/versions/models.py @@ -17,7 +17,6 @@ import uuid from collections import namedtuple -import six from django.core.exceptions import SuspiciousOperation, ObjectDoesNotExist from django.db import models, router, transaction from django.db.models import Q @@ -27,6 +26,7 @@ from django.db.models.sql.datastructures import Join from django.db.models.sql.query import Query from django.db.models.sql.where import WhereNode +from django.utils import six from django.utils.timezone import utc from versions.exceptions import DeletionOfNonCurrentVersionError From c3f91c3a6d0c6dd6b4db54ede8c19e41af32a36f Mon Sep 17 00:00:00 2001 From: Manuel Jeckelmann Date: Wed, 8 Feb 2017 21:46:20 +0100 Subject: [PATCH 15/42] Removed tox attempting to install requirements.txt --- tox.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/tox.ini b/tox.ini index 2399a0b..3ed344b 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,6 @@ envlist = [testenv] deps = coverage - -rrequirements.txt django19: django>=1.9,<1.10 pg: psycopg2 commands = From cdf3416cdae33a8d318f74999099b330f5bbe02e Mon Sep 17 00:00:00 2001 From: Vladimir Kuvandjiev Date: Fri, 25 Aug 2017 22:56:33 +0300 Subject: [PATCH 16/42] Django added to requirements.txt --- requirements.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..b3764b3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +Django>=1.11.4 From 4a8add32495d60c3c7f53bc4c234b5e0d1d5e1b0 Mon Sep 17 00:00:00 2001 From: Vladimir Kuvandjiev Date: Fri, 25 Aug 2017 22:56:57 +0300 Subject: [PATCH 17/42] .gitignore updated --- .gitignore | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitignore b/.gitignore index 0812226..1096574 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,9 @@ sqlite.db # dev_no_debug.py imports local.py and disables Django-debugging cleanerversion/settings/dev_no_debug.py cleanerversion/settings/*_local.py + +bin/ +share/ +include/ +lib/ +pip-selfcheck.json From d0735c2fe245ca22ef7ff88d98d95418e5198e38 Mon Sep 17 00:00:00 2001 From: Vladimir Kuvandjiev Date: Fri, 25 Aug 2017 22:57:39 +0300 Subject: [PATCH 18/42] manage.py is now marked as executable --- manage.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 manage.py diff --git a/manage.py b/manage.py old mode 100644 new mode 100755 From a68abfca80bcc8443340001f7aed79bb7106ffd7 Mon Sep 17 00:00:00 2001 From: Vladimir Kuvandjiev Date: Fri, 25 Aug 2017 23:00:17 +0300 Subject: [PATCH 19/42] Explicit check for non null foreign key on foreign key assignment added as it is dropped in Django 1.11 --- versions/models.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/versions/models.py b/versions/models.py index a56eb76..6c8c743 100644 --- a/versions/models.py +++ b/versions/models.py @@ -831,8 +831,12 @@ def restore(self, **kwargs): # Set all non-provided ForeignKeys to None. If required, raise an error. try: setattr(restored, field.name, None) - except ValueError as e: - raise ForeignKeyRequiresValueError(e.args[0]) + # Check for non null foreign key removed in Django 1.11 + # https://docs.djangoproject.com/en/1.11/releases/1.10/#removed-null-assignment-check-for-non-null-foreign-key-fields + if not field.null: + raise ValueError + except ValueError: + raise ForeignKeyRequiresValueError self.id = self.uuid() From c283110c1934d8a6b3ad2c95c120ce53de2dcaa7 Mon Sep 17 00:00:00 2001 From: Vladimir Kuvandjiev Date: Fri, 25 Aug 2017 23:00:52 +0300 Subject: [PATCH 20/42] test_prefetch_related_via_many_to_many fix --- versions_tests/tests/test_models.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/versions_tests/tests/test_models.py b/versions_tests/tests/test_models.py index 988270a..6b1cf06 100644 --- a/versions_tests/tests/test_models.py +++ b/versions_tests/tests/test_models.py @@ -2182,6 +2182,12 @@ def test_prefetch_related_via_many_to_many(self): p.awards.remove(p.awards.all()[0]) name_list.append(p.name) + with self.assertNumQueries(2): + old_award_players = list( + Player.objects.as_of(t2).prefetch_related('awards').filter( + name__in=name_list).order_by('name') + ) + with self.assertNumQueries(2): updated_award_players = list( Player.objects.current.prefetch_related('awards').filter( @@ -2190,7 +2196,7 @@ def test_prefetch_related_via_many_to_many(self): with self.assertNumQueries(0): for i in range(len(award_players)): - old = len(award_players[i].awards.all()) + old = len(old_award_players[i].awards.all()) new = len(updated_award_players[i].awards.all()) self.assertTrue(new == old - 1) From d22881e0598d26ec6baf150b86dcd913a76e456a Mon Sep 17 00:00:00 2001 From: Vladimir Kuvandjiev Date: Fri, 25 Aug 2017 23:15:07 +0300 Subject: [PATCH 21/42] Django 1.11.4 added to tox envlist --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 3ed344b..b723dff 100644 --- a/tox.ini +++ b/tox.ini @@ -5,12 +5,12 @@ [tox] envlist = - py{27,35}-django{19}-{sqlite,pg} + py{27,35}-django{111}-{sqlite,pg} [testenv] deps = coverage - django19: django>=1.9,<1.10 + django111: django>=1.10,<=1.11.4 pg: psycopg2 commands = pg: coverage run --source=versions ./manage.py test --settings={env:TOX_PG_CONF:cleanerversion.settings.pg} From 53e644eb7236c167a358e0168788641bd29d293c Mon Sep 17 00:00:00 2001 From: Jeckelmann Manuel Date: Sat, 2 Sep 2017 14:31:17 +0200 Subject: [PATCH 22/42] Adapted to work w/ Django 1.10; no optimizations introduced --- .travis.yml | 8 ++++---- tox.ini | 4 ++-- versions/models.py | 8 ++++---- versions_tests/tests/test_models.py | 5 +++++ 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0ae6634..3606df9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,10 +2,10 @@ language: python python: "2.7" env: - - TOX_ENV=py27-django19-pg - - TOX_ENV=py27-django19-sqlite - - TOX_ENV=py35-django19-pg - - TOX_ENV=py35-django19-sqlite + - TOX_ENV=py27-django110-pg + - TOX_ENV=py27-django110-sqlite + - TOX_ENV=py36-django110-pg + - TOX_ENV=py36-django110-sqlite # Enable PostgreSQL usage addons: diff --git a/tox.ini b/tox.ini index 3ed344b..d2a7066 100644 --- a/tox.ini +++ b/tox.ini @@ -5,12 +5,12 @@ [tox] envlist = - py{27,35}-django{19}-{sqlite,pg} + py{27,36}-django{110}-{sqlite,pg} [testenv] deps = coverage - django19: django>=1.9,<1.10 + django110: django>=1.10,<1.11 pg: psycopg2 commands = pg: coverage run --source=versions ./manage.py test --settings={env:TOX_PG_CONF:cleanerversion.settings.pg} diff --git a/versions/models.py b/versions/models.py index a56eb76..8d17644 100644 --- a/versions/models.py +++ b/versions/models.py @@ -829,10 +829,10 @@ def restore(self, **kwargs): setattr(restored, field.name, kwargs[field.name]) elif isinstance(field, ForeignKey): # Set all non-provided ForeignKeys to None. If required, raise an error. - try: - setattr(restored, field.name, None) - except ValueError as e: - raise ForeignKeyRequiresValueError(e.args[0]) + setattr(restored, field.name, None) + # Need to explicitely raise error since Django 1.10 (https://docs.djangoproject.com/en/1.10/releases/1.10/#removed-null-assignment-check-for-non-null-foreign-key-fields) + if not field.null: + raise ForeignKeyRequiresValueError("Field '%s' is not nullable" % field.name) self.id = self.uuid() diff --git a/versions_tests/tests/test_models.py b/versions_tests/tests/test_models.py index 988270a..dd9e831 100644 --- a/versions_tests/tests/test_models.py +++ b/versions_tests/tests/test_models.py @@ -53,6 +53,9 @@ def set_up_one_object_with_3_versions(): sleep(0.1) t1 = get_utc_now() + # 1ms sleeps are required, since sqlite has a 1ms precision in its datetime stamps + # not inserting the sleep would make t1 point to the next version's start date, which would be wrong + sleep(0.001) b = b.clone() b.name = 'v2' @@ -60,6 +63,8 @@ def set_up_one_object_with_3_versions(): sleep(0.1) t2 = get_utc_now() + # 1ms sleeps are required, since sqlite has a 1ms precision in its datetime stamps + sleep(0.001) b = b.clone() b.name = 'v3' From 728c9aa96c70723446952f5b60ab3d3be865f296 Mon Sep 17 00:00:00 2001 From: Jeckelmann Manuel Date: Sun, 10 Sep 2017 13:03:08 +0200 Subject: [PATCH 23/42] Testing models with unicode strings --- versions_tests/tests/test_models.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/versions_tests/tests/test_models.py b/versions_tests/tests/test_models.py index f296cb5..b20047a 100644 --- a/versions_tests/tests/test_models.py +++ b/versions_tests/tests/test_models.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import unicode_literals + import datetime from time import sleep import itertools @@ -27,7 +29,6 @@ from django.test import TestCase from django.utils.timezone import utc from django.utils import six -from django import VERSION from versions.exceptions import DeletionOfNonCurrentVersionError from versions.models import get_utc_now, ForeignKeyRequiresValueError, Versionable @@ -2505,7 +2506,7 @@ def test_create_with_uuid(self): self.assertEqual(str(p_id), str(p.id)) self.assertEqual(str(p_id), str(p.identity)) - p_id = uuid.uuid5(uuid.NAMESPACE_OID, 'bar') + p_id = uuid.uuid5(uuid.NAMESPACE_OID, str('bar')) with self.assertRaises(ValueError): Person.objects.create(id=p_id, name="Alexis") From aba7be15bc4b4ea8e8c7eab93f5c0a8c69b3f41d Mon Sep 17 00:00:00 2001 From: Jeckelmann Manuel Date: Sun, 10 Sep 2017 13:21:30 +0200 Subject: [PATCH 24/42] Adjusted the required Django version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b3764b3..867eec0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -Django>=1.11.4 +Django>=1.10 From 1e751b74603d41b9464c5a5a80f36a0c77b99577 Mon Sep 17 00:00:00 2001 From: Jeckelmann Manuel Date: Sun, 10 Sep 2017 18:25:17 +0200 Subject: [PATCH 25/42] Ported the prefetch_related fix to Django1.9 (see #129 and #131) --- versions/descriptors.py | 40 +++++++---- versions/fields.py | 10 +++ versions/models.py | 6 ++ versions_tests/tests/test_models.py | 100 ++++++++++++++++++++++++++-- 4 files changed, 138 insertions(+), 18 deletions(-) diff --git a/versions/descriptors.py b/versions/descriptors.py index a61a115..9c94685 100644 --- a/versions/descriptors.py +++ b/versions/descriptors.py @@ -29,9 +29,27 @@ def matches_querytime(instance, querytime): class VersionedForwardManyToOneDescriptor(ForwardManyToOneDescriptor): """ + The VersionedForwardManyToOneDescriptor is used when pointing another Model using a VersionedForeignKey; + For example: + class Team(Versionable): + name = CharField(max_length=200) + city = VersionedForeignKey(City, null=True) + + ``team.city`` is a VersionedForwardManyToOneDescriptor """ - pass + + def get_queryset(self, **hints): + queryset = super(VersionedForwardManyToOneDescriptor, self).get_queryset(**hints) + if hasattr(queryset, 'querytime'): + if 'instance' in hints: + instance = hints['instance'] + if hasattr(instance, '_querytime'): + if instance._querytime.active and instance._querytime != queryset.querytime: + queryset = queryset.as_of(instance._querytime.time) + else: + queryset = queryset.as_of(None) + return queryset vforward_many_to_one_descriptor_class = VersionedForwardManyToOneDescriptor @@ -76,22 +94,9 @@ def vforward_many_to_one_descriptor_getter(self, instance, instance_type=None): class VersionedReverseManyToOneDescriptor(ReverseManyToOneDescriptor): @cached_property def related_manager_cls(self): - # return create_versioned_related_manager manager_cls = super(VersionedReverseManyToOneDescriptor, self).related_manager_cls - - # if VERSION[:2] >= (1, 9): - # TODO: Define, what field has to be taken over here, self.rel/self.field? The WhineDrinker.hats test seems to be a good one for testing this - - # rel_field = self.rel - rel_field = self.field - # elif hasattr(self, 'related'): - # rel_field = self.related.field - # else: - # rel_field = self.field - # rel_field = self.rel - class VersionedRelatedManager(manager_cls): def __init__(self, instance): super(VersionedRelatedManager, self).__init__(instance) @@ -129,6 +134,13 @@ def get_prefetch_queryset(self, instances, queryset=None): queryset._add_hints(instance=instances[0]) queryset = queryset.using(queryset._db or self._db) + instance_querytime = instances[0]._querytime + if instance_querytime.active: + if queryset.querytime.active and queryset.querytime.time != instance_querytime.time: + raise ValueError("A Prefetch queryset that specifies an as_of time must match " + "the as_of of the base queryset.") + else: + queryset.querytime = instance_querytime rel_obj_attr = rel_field.get_local_related_value instance_attr = rel_field.get_foreign_related_value diff --git a/versions/fields.py b/versions/fields.py index bcdd292..9407338 100644 --- a/versions/fields.py +++ b/versions/fields.py @@ -20,7 +20,15 @@ class VersionedForeignKey(ForeignKey): """ def __init__(self, *args, **kwargs): + is_m2m_vfk = kwargs.pop('is_m2m_vfk', False) + # The is_m2m_vfk argument is used to indicate whether or not the VersionedForeignKey is part of a + # M2M intermediary model super(VersionedForeignKey, self).__init__(*args, **kwargs) + # In the regular case, let's switch 'id' with 'identity', since we always query on identity + if not is_m2m_vfk: + for index, field in enumerate(self.to_fields): + if field == Versionable.VERSION_IDENTIFIER_FIELD: + self.to_fields[index] = Versionable.OBJECT_IDENTIFIER_FIELD def contribute_to_class(self, cls, name, virtual_only=False): super(VersionedForeignKey, self).contribute_to_class(cls, name, virtual_only) @@ -167,6 +175,7 @@ def set_managed(model, related, through): db_constraint=field.remote_field.db_constraint, auto_created=name, on_delete=DO_NOTHING, + is_m2m_vfk=True, ), to: VersionedForeignKey( to_model, @@ -175,6 +184,7 @@ def set_managed(model, related, through): db_constraint=field.remote_field.db_constraint, auto_created=name, on_delete=DO_NOTHING, + is_m2m_vfk=True, ), }) diff --git a/versions/models.py b/versions/models.py index a56eb76..e1e9d3e 100644 --- a/versions/models.py +++ b/versions/models.py @@ -387,6 +387,12 @@ def build_filter(self, filter_expr, **kwargs): filter_expr = (new_lookup, value.identity) return super(VersionedQuery, self).build_filter(filter_expr, **kwargs) + def add_immediate_loading(self, field_names): + # TODO: Decide, whether we always want versionable fields to be loaded, even if ``only`` is used and they would + # be deferred + # field_names += tuple(Versionable.VERSIONABLE_FIELDS) + super(VersionedQuery, self).add_immediate_loading(field_names) + class VersionedQuerySet(QuerySet): """ diff --git a/versions_tests/tests/test_models.py b/versions_tests/tests/test_models.py index 988270a..f6a9e31 100644 --- a/versions_tests/tests/test_models.py +++ b/versions_tests/tests/test_models.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import unicode_literals + import datetime from time import sleep import itertools @@ -51,21 +53,23 @@ def get_relation_table(model_class, fieldname): def set_up_one_object_with_3_versions(): b = B.objects.create(name='v1') - sleep(0.1) + sleep(0.001) t1 = get_utc_now() + sleep(0.001) b = b.clone() b.name = 'v2' b.save() - sleep(0.1) + sleep(0.001) t2 = get_utc_now() + sleep(0.001) b = b.clone() b.name = 'v3' b.save() - sleep(0.1) + sleep(0.001) t3 = get_utc_now() return b, t1, t2, t3 @@ -2309,6 +2313,22 @@ def test_reverse_fk_prefetch_queryset_with_historic_versions(self): team = [t for t in current_city.prefetched_teams if t.name == 'team1.v2'][0] self.assertSetEqual({'pl1.v2', 'pl2.v1'}, {p.name for p in team.prefetched_players}) + # When a different time is specified for the prefetch queryset than for the base queryset: + + with self.assertRaises(ValueError): + _ = City.objects.current.filter(name='city.v2').prefetch_related( + Prefetch( + 'team_set', + queryset = Team.objects.as_of(self.time1), + to_attr = 'prefetched_teams' + ), + Prefetch( + 'prefetched_teams__player_set', + queryset = Player.objects.as_of(self.time1), + to_attr = 'prefetched_players' + ), + )[0] + def test_reverse_fk_simple_prefetch_with_historic_versions(self): """ prefetch_related with simple lookup. @@ -2363,6 +2383,75 @@ def test_reverse_fk_simple_prefetch_with_historic_versions(self): team = [t for t in current_city.team_set.all() if t.name == 'team1.v2'][0] self.assertSetEqual({'pl1.v2', 'pl2.v1'}, {p.name for p in team.player_set.all()}) + def test_foreign_key_prefetch_with_historic_version(self): + self.modify_objects() + historic_city = City.objects.as_of(self.time1).get(identity=self.c1.identity) + + # Test with a simple prefetch. + with self.assertNumQueries(2): + team = Team.objects.as_of(self.time1).filter( + identity=self.t1.identity + ).prefetch_related( + 'city' + )[0] + self.assertIsNotNone(team.city) + self.assertEquals(team.city.id, historic_city.id) + + # Test with a Prefetch object without a queryset. + with self.assertNumQueries(2): + team = Team.objects.as_of(self.time1).filter( + identity=self.t1.identity + ).prefetch_related(Prefetch( + 'city', + ))[0] + self.assertIsNotNone(team.city) + self.assertEquals(team.city.id, historic_city.id) + + # Test with a Prefetch object with a queryset with an explicit as_of. + with self.assertNumQueries(2): + team = Team.objects.as_of(self.time1).filter( + identity=self.t1.identity + ).prefetch_related(Prefetch( + 'city', + queryset=City.objects.as_of(self.time1) + ))[0] + self.assertIsNotNone(team.city) + self.assertEquals(team.city.id, historic_city.id) + + # Test with a Prefetch object with a queryset with no as_of. + with self.assertNumQueries(2): + team = Team.objects.as_of(self.time1).filter( + identity=self.t1.identity + ).prefetch_related(Prefetch( + 'city', + queryset=City.objects.all() + ))[0] + self.assertIsNotNone(team.city) + self.assertEquals(team.city.id, historic_city.id) + + # Test with a Prefetch object with a queryset with an as_of that differs from the parents. + # If permitted, it would lead to possibly incorrect results and definitely cache misses, + # which would defeat the purpose of using prefetch_related. So a ValueError should be raised. + with self.assertRaises(ValueError): + team = Team.objects.as_of(self.time1).filter( + identity=self.t1.identity + ).prefetch_related(Prefetch( + 'city', + queryset=City.objects.current + ))[0] + + # Test with a Prefetch object with a queryset with an as_of, when the parent has no as_of. + # This is a bit of an odd thing to do, but possible. + with self.assertNumQueries(2): + team = Team.objects.filter( + identity=self.t1.identity + ).prefetch_related(Prefetch( + 'city', + queryset=City.objects.as_of(self.time1) + ))[0] + self.assertIsNotNone(team.city) + self.assertEquals(team.city.id, historic_city.id) + class IntegrationNonVersionableModelsTests(TestCase): def setUp(self): @@ -2494,7 +2583,7 @@ def test_create_with_uuid(self): self.assertEqual(str(p_id), str(p.id)) self.assertEqual(str(p_id), str(p.identity)) - p_id = uuid.uuid5(uuid.NAMESPACE_OID, 'bar') + p_id = uuid.uuid5(uuid.NAMESPACE_OID, str('bar')) with self.assertRaises(ValueError): Person.objects.create(id=p_id, name="Alexis") @@ -2546,10 +2635,13 @@ def setup_common(self): def test_restore_latest_version(self): self.setup_common() + sleep(0.001) self.player1.delete() + sleep(0.001) deleted_at = self.player1.version_end_date player1_pk = self.player1.pk + sleep(0.001) restored = self.player1.restore() self.assertEqual(player1_pk, restored.pk) self.assertIsNone(restored.version_end_date) From fddac9cba344223b11bce551ae39d41be03d600f Mon Sep 17 00:00:00 2001 From: Jeckelmann Manuel Date: Sun, 10 Sep 2017 18:42:04 +0200 Subject: [PATCH 26/42] Adapted testing config to run tests w/ Python3 --- .travis.yml | 11 ++++++----- tox.ini | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0ae6634..180f054 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,11 +1,11 @@ language: python -python: "2.7" +python: + - "2.7" + - "3.6" env: - - TOX_ENV=py27-django19-pg - - TOX_ENV=py27-django19-sqlite - - TOX_ENV=py35-django19-pg - - TOX_ENV=py35-django19-sqlite + - TOX_ENV=django19-pg + - TOX_ENV=django19-sqlite # Enable PostgreSQL usage addons: @@ -14,6 +14,7 @@ addons: # Dependencies install: - pip install tox + - pip install tox-travis - pip install coveralls # Ensure PostgreSQL-DB to be configured correctly diff --git a/tox.ini b/tox.ini index 3ed344b..ab78a33 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,7 @@ [tox] envlist = - py{27,35}-django{19}-{sqlite,pg} + django{19}-{sqlite,pg} [testenv] deps = From 7d4ecde27de3e9556c4ec43085389fb4e03c6cf9 Mon Sep 17 00:00:00 2001 From: Jeckelmann Manuel Date: Sun, 10 Sep 2017 18:48:15 +0200 Subject: [PATCH 27/42] Added VERSIONS_USE_UUIDFIELD settings parameter --- cleanerversion/settings/base.py | 2 ++ cleanerversion/settings/pg_travis.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/cleanerversion/settings/base.py b/cleanerversion/settings/base.py index e4fd2ab..940b815 100644 --- a/cleanerversion/settings/base.py +++ b/cleanerversion/settings/base.py @@ -98,3 +98,5 @@ ROOT_URLCONF = 'cleanerversion.urls' STATIC_URL = '/static/' + +VERSIONS_USE_UUIDFIELD = False diff --git a/cleanerversion/settings/pg_travis.py b/cleanerversion/settings/pg_travis.py index ff3b409..ab66743 100644 --- a/cleanerversion/settings/pg_travis.py +++ b/cleanerversion/settings/pg_travis.py @@ -16,3 +16,5 @@ 'PORT': '5432', }, } + +VERSIONS_USE_UUIDFIELD = True From 376432539f81702b715549eba7e7c333cf75ab5b Mon Sep 17 00:00:00 2001 From: Jeckelmann Manuel Date: Sun, 10 Sep 2017 18:57:59 +0200 Subject: [PATCH 28/42] Brought accidentally deleted get_prefetch_queryset method back --- versions/descriptors.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/versions/descriptors.py b/versions/descriptors.py index 9c94685..6136d8d 100644 --- a/versions/descriptors.py +++ b/versions/descriptors.py @@ -39,6 +39,29 @@ class Team(Versionable): ``team.city`` is a VersionedForwardManyToOneDescriptor """ + def get_prefetch_queryset(self, instances, queryset=None): + """ + Overrides the parent method to: + - force queryset to use the querytime of the parent objects + - ensure that the join is done on identity, not id + - make the cache key identity, not id. + """ + if queryset is None: + queryset = self.get_queryset() + + # CleanerVersion change 1: force the querytime to be the same as the prefetched-for instance. + # This is necessary to have reliable results and avoid extra queries for cache misses when + # accessing the child objects from their parents (e.g. choice.poll). + instance_querytime = instances[0]._querytime + if instance_querytime.active: + if queryset.querytime.active and queryset.querytime.time != instance_querytime.time: + raise ValueError("A Prefetch queryset that specifies an as_of time must match " + "the as_of of the base queryset.") + else: + queryset.querytime = instance_querytime + + return super(VersionedForwardManyToOneDescriptor, self).get_prefetch_queryset(instances, queryset) + def get_queryset(self, **hints): queryset = super(VersionedForwardManyToOneDescriptor, self).get_queryset(**hints) if hasattr(queryset, 'querytime'): From 182a2e4a1ac973196510eb6102153468cf61a07e Mon Sep 17 00:00:00 2001 From: Jeckelmann Manuel Date: Sun, 10 Sep 2017 21:11:04 +0200 Subject: [PATCH 29/42] Adapted VersionedForeignKey in order to not use identity as a default reference field (this was adapted in a previous commit today) --- .../doc/historization_with_cleanerversion.rst | 31 +++++++++++----- versions/descriptors.py | 37 ++++++++++++++++++- versions/fields.py | 20 +++++----- 3 files changed, 67 insertions(+), 21 deletions(-) diff --git a/docs/doc/historization_with_cleanerversion.rst b/docs/doc/historization_with_cleanerversion.rst index b5c43d8..f9b38b9 100644 --- a/docs/doc/historization_with_cleanerversion.rst +++ b/docs/doc/historization_with_cleanerversion.rst @@ -542,29 +542,40 @@ Notes about using prefetch_related simple sting lookups or `Prefetch `_ objects. -When using ``prefetch_related`` with CleanerVersion, be aware that when using a ``Prefetch`` object **that -specifies a queryset**, that you need to explicitly specify the ``as_of`` value, or use ``current``. -A ``Prefetch`` queryset will not be automatically time-restricted based on the base queryset. +When using ``prefetch_related`` with CleanerVersion, the generated query that fetches the related objects will +be time-restricted based on the base queryset. If you provide a ``Prefetch`` object that specifies a queryset, the +queryset must either not be time-limited (using ``.as_of()`` or ``.current``), or be time-limited with the same +``.as_of`` or ``.current`` as the base queryset. If the ``Prefetch`` queryset is not time-limited, but the base +queryset is, the ``Prefetch`` queryset will adopt the same time limitation as the base queryset. -For example, assuming you want everything at the time ``end_of_last_month``, do this:: +For example, assuming you want everything at the time ``end_of_last_month``, you can do this:: + # Prefetch queryset is not explicitly time-restricted, and will adopt the base queryset's time-restriction. + disciplines_prefetch = Prefetch( + 'sportsclubs__discipline_set', + queryset=Discipline.objects.filter('name__startswith'='B')) + people_last_month = Person.objects.as_of(end_of_last_month).prefetch_related(disciplines_prefetch) + +or this:: + + # Prefetch queryset is explicitly time-restricted with the same time restriction as the base queryset. disciplines_prefetch = Prefetch( 'sportsclubs__discipline_set', queryset=Discipline.objects.as_of(end_of_last_month).filter('name__startswith'='B')) people_last_month = Person.objects.as_of(end_of_last_month).prefetch_related(disciplines_prefetch) -On the other hand, the following ``Prefetch``, without an ``as_of`` in the queryset, would result in all -matching ``Discipline`` objects being returned, regardless whether they existed at ``end_of_last_month`` -or not:: +However, the following ``Prefetch``, without a time restriction that differs from the base queryset, will +raise a ``ValueError`` when evaluated:: - # Don't do this, the Prefetch queryset is missing an as_of(): + # Don't do this, the Prefetch queryset's time restriction doesn't match it's parent's: disciplines_prefetch = Prefetch( 'sportsclubs__discipline_set', - queryset=Discipline.objects.filter('name__startswith'='B')) + queryset=Discipline.objects.current.filter('name__startswith'='B')) people_last_month = Person.objects.as_of(end_of_last_month).prefetch_related(disciplines_prefetch) + If a ``Prefetch`` without an explicit queryset is used, or a simple string lookup, the generated queryset will -be appropriately time-restricted. The following statements will propagate the base queries' +be appropriately time-restricted. The following statements will propagate the base query's ``as_of`` value to the generated related-objects queryset:: people1 = Person.objects.as_of(end_of_last_month).prefetch_related(Prefetch('sportsclubs__discipline_set')) diff --git a/versions/descriptors.py b/versions/descriptors.py index 6136d8d..88ceaa6 100644 --- a/versions/descriptors.py +++ b/versions/descriptors.py @@ -1,3 +1,5 @@ +from collections import namedtuple + from django.core.exceptions import SuspiciousOperation, FieldDoesNotExist from django.db import router, transaction from django.db.models.base import Model @@ -48,6 +50,7 @@ def get_prefetch_queryset(self, instances, queryset=None): """ if queryset is None: queryset = self.get_queryset() + queryset._add_hints(instance=instances[0]) # CleanerVersion change 1: force the querytime to be the same as the prefetched-for instance. # This is necessary to have reliable results and avoid extra queries for cache misses when @@ -60,7 +63,39 @@ def get_prefetch_queryset(self, instances, queryset=None): else: queryset.querytime = instance_querytime - return super(VersionedForwardManyToOneDescriptor, self).get_prefetch_queryset(instances, queryset) + # CleanerVersion change 2: make rel_obj_attr return a tuple with the object's identity. + # rel_obj_attr = self.field.get_foreign_related_value + def versioned_fk_rel_obj_attr(versioned_rel_obj): + return versioned_rel_obj.identity, + + rel_obj_attr = versioned_fk_rel_obj_attr + instance_attr = self.field.get_local_related_value + instances_dict = {instance_attr(inst): inst for inst in instances} + # CleanerVersion change 3: fake the related field so that it provides a name of 'identity'. + # related_field = self.field.foreign_related_fields[0] + related_field = namedtuple('VersionedRelatedFieldTuple', 'name')('identity') + + # FIXME: This will need to be revisited when we introduce support for + # composite fields. In the meantime we take this practical approach to + # solve a regression on 1.6 when the reverse manager in hidden + # (related_name ends with a '+'). Refs #21410. + # The check for len(...) == 1 is a special case that allows the query + # to be join-less and smaller. Refs #21760. + if self.field.rel.is_hidden() or len(self.field.foreign_related_fields) == 1: + query = {'%s__in' % related_field.name: set(instance_attr(inst)[0] for inst in instances)} + # query = {'identity__in': set(instance_attr(inst)[0] for inst in instances)} + else: + query = {'%s__in' % self.field.related_query_name(): instances} + queryset = queryset.filter(**query) + + # Since we're going to assign directly in the cache, + # we must manage the reverse relation cache manually. + if not self.field.rel.multiple: + rel_obj_cache_name = self.field.rel.get_cache_name() + for rel_obj in queryset: + instance = instances_dict[rel_obj_attr(rel_obj)] + setattr(rel_obj, rel_obj_cache_name, instance) + return queryset, rel_obj_attr, instance_attr, True, self.cache_name def get_queryset(self, **hints): queryset = super(VersionedForwardManyToOneDescriptor, self).get_queryset(**hints) diff --git a/versions/fields.py b/versions/fields.py index 9407338..3a570f0 100644 --- a/versions/fields.py +++ b/versions/fields.py @@ -20,15 +20,7 @@ class VersionedForeignKey(ForeignKey): """ def __init__(self, *args, **kwargs): - is_m2m_vfk = kwargs.pop('is_m2m_vfk', False) - # The is_m2m_vfk argument is used to indicate whether or not the VersionedForeignKey is part of a - # M2M intermediary model super(VersionedForeignKey, self).__init__(*args, **kwargs) - # In the regular case, let's switch 'id' with 'identity', since we always query on identity - if not is_m2m_vfk: - for index, field in enumerate(self.to_fields): - if field == Versionable.VERSION_IDENTIFIER_FIELD: - self.to_fields[index] = Versionable.OBJECT_IDENTIFIER_FIELD def contribute_to_class(self, cls, name, virtual_only=False): super(VersionedForeignKey, self).contribute_to_class(cls, name, virtual_only) @@ -84,6 +76,16 @@ def get_joining_columns(self, reverse_join=False): joining_columns = joining_columns + ((lhs_col_name, rhs_col_name),) return joining_columns + def get_reverse_related_filter(self, obj): + base_filter = {} + for lh_field, rh_field in self.related_fields: + if rh_field.attname == Versionable.VERSION_IDENTIFIER_FIELD: + base_filter.update({Versionable.OBJECT_IDENTIFIER_FIELD: getattr(obj, lh_field.attname)}) + else: + base_filter.update({rh_field.attname: getattr(obj, lh_field.attname)}) + base_filter.update(self.get_extra_descriptor_filter(obj) or {}) + return base_filter + class VersionedManyToManyField(ManyToManyField): def __init__(self, *args, **kwargs): @@ -175,7 +177,6 @@ def set_managed(model, related, through): db_constraint=field.remote_field.db_constraint, auto_created=name, on_delete=DO_NOTHING, - is_m2m_vfk=True, ), to: VersionedForeignKey( to_model, @@ -184,7 +185,6 @@ def set_managed(model, related, through): db_constraint=field.remote_field.db_constraint, auto_created=name, on_delete=DO_NOTHING, - is_m2m_vfk=True, ), }) From 88e36a4b2c3ee389addc3e62ceea7afe9f06748c Mon Sep 17 00:00:00 2001 From: Jeckelmann Manuel Date: Sun, 10 Sep 2017 22:11:05 +0200 Subject: [PATCH 30/42] Added a condition to replace Id field during filter creation --- versions/fields.py | 3 ++- versions/models.py | 1 + versions_tests/tests/test_models.py | 1 - 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/versions/fields.py b/versions/fields.py index 3a570f0..dd27f04 100644 --- a/versions/fields.py +++ b/versions/fields.py @@ -79,12 +79,13 @@ def get_joining_columns(self, reverse_join=False): def get_reverse_related_filter(self, obj): base_filter = {} for lh_field, rh_field in self.related_fields: - if rh_field.attname == Versionable.VERSION_IDENTIFIER_FIELD: + if isinstance(obj, Versionable) and rh_field.attname == Versionable.VERSION_IDENTIFIER_FIELD: base_filter.update({Versionable.OBJECT_IDENTIFIER_FIELD: getattr(obj, lh_field.attname)}) else: base_filter.update({rh_field.attname: getattr(obj, lh_field.attname)}) base_filter.update(self.get_extra_descriptor_filter(obj) or {}) return base_filter + # return super(VersionedForeignKey, self).get_reverse_related_filter(obj) class VersionedManyToManyField(ManyToManyField): diff --git a/versions/models.py b/versions/models.py index e1e9d3e..68f76ee 100644 --- a/versions/models.py +++ b/versions/models.py @@ -381,6 +381,7 @@ def build_filter(self, filter_expr, **kwargs): :param kwargs: :return: tuple """ + print(filter_expr) lookup, value = filter_expr if self.querytime.active and isinstance(value, Versionable) and not value.is_latest: new_lookup = lookup + LOOKUP_SEP + Versionable.OBJECT_IDENTIFIER_FIELD diff --git a/versions_tests/tests/test_models.py b/versions_tests/tests/test_models.py index f6a9e31..cdaf9d1 100644 --- a/versions_tests/tests/test_models.py +++ b/versions_tests/tests/test_models.py @@ -2853,7 +2853,6 @@ def test_simple_defer(self): self.assertEquals(self.c1.version_start_date, deferred.version_start_date) def test_deferred_foreign_key_field(self): - team_full = Team.objects.current.get(pk=self.team1.pk) self.assertIn('city_id', team_full.__dict__ ) team_light = Team.objects.current.only('name').get(pk=self.team1.pk) From a8ec3c7a1efa0fb0cb7bdcf7a17e4b471fa5c642 Mon Sep 17 00:00:00 2001 From: Jeckelmann Manuel Date: Sun, 10 Sep 2017 22:22:04 +0200 Subject: [PATCH 31/42] Slight syntax change: updating base filtering dict w/ kwargs now --- versions/fields.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/versions/fields.py b/versions/fields.py index dd27f04..1d18466 100644 --- a/versions/fields.py +++ b/versions/fields.py @@ -80,9 +80,9 @@ def get_reverse_related_filter(self, obj): base_filter = {} for lh_field, rh_field in self.related_fields: if isinstance(obj, Versionable) and rh_field.attname == Versionable.VERSION_IDENTIFIER_FIELD: - base_filter.update({Versionable.OBJECT_IDENTIFIER_FIELD: getattr(obj, lh_field.attname)}) + base_filter.update(**{Versionable.OBJECT_IDENTIFIER_FIELD: getattr(obj, lh_field.attname)}) else: - base_filter.update({rh_field.attname: getattr(obj, lh_field.attname)}) + base_filter.update(**{rh_field.attname: getattr(obj, lh_field.attname)}) base_filter.update(self.get_extra_descriptor_filter(obj) or {}) return base_filter # return super(VersionedForeignKey, self).get_reverse_related_filter(obj) From c85c82ab16714567b33b98c00caae0be52f1af18 Mon Sep 17 00:00:00 2001 From: Jeckelmann Manuel Date: Sun, 10 Sep 2017 22:26:31 +0200 Subject: [PATCH 32/42] Made dict explicit --- versions/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versions/fields.py b/versions/fields.py index 1d18466..e06fcc8 100644 --- a/versions/fields.py +++ b/versions/fields.py @@ -77,7 +77,7 @@ def get_joining_columns(self, reverse_join=False): return joining_columns def get_reverse_related_filter(self, obj): - base_filter = {} + base_filter = dict() for lh_field, rh_field in self.related_fields: if isinstance(obj, Versionable) and rh_field.attname == Versionable.VERSION_IDENTIFIER_FIELD: base_filter.update(**{Versionable.OBJECT_IDENTIFIER_FIELD: getattr(obj, lh_field.attname)}) From b72f5ed8ca27ec51b75570064effa8e65f1cd286 Mon Sep 17 00:00:00 2001 From: Jeckelmann Manuel Date: Sun, 10 Sep 2017 23:42:45 +0200 Subject: [PATCH 33/42] Fixed errors introduced by some breaking minor version change somewhere in Django 1.9 --- versions/fields.py | 10 ++++++++-- versions/models.py | 1 - 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/versions/fields.py b/versions/fields.py index e06fcc8..57d5e89 100644 --- a/versions/fields.py +++ b/versions/fields.py @@ -1,6 +1,7 @@ from django.db.models.deletion import DO_NOTHING from django.db.models.fields.related import ForeignKey, ManyToManyField, \ resolve_relation, lazy_related_operation +from django.db.models.query_utils import Q from django.db.models.sql.datastructures import Join from django.db.models.sql.where import ExtraWhere, WhereNode from django.db.models.utils import make_model_tuple @@ -83,8 +84,13 @@ def get_reverse_related_filter(self, obj): base_filter.update(**{Versionable.OBJECT_IDENTIFIER_FIELD: getattr(obj, lh_field.attname)}) else: base_filter.update(**{rh_field.attname: getattr(obj, lh_field.attname)}) - base_filter.update(self.get_extra_descriptor_filter(obj) or {}) - return base_filter + descriptor_filter = self.get_extra_descriptor_filter(obj) + base_q = Q(**base_filter) + if isinstance(descriptor_filter, dict): + return base_q & Q(**descriptor_filter) + elif descriptor_filter: + return base_q & descriptor_filter + return base_q # return super(VersionedForeignKey, self).get_reverse_related_filter(obj) diff --git a/versions/models.py b/versions/models.py index 68f76ee..e1e9d3e 100644 --- a/versions/models.py +++ b/versions/models.py @@ -381,7 +381,6 @@ def build_filter(self, filter_expr, **kwargs): :param kwargs: :return: tuple """ - print(filter_expr) lookup, value = filter_expr if self.querytime.active and isinstance(value, Versionable) and not value.is_latest: new_lookup = lookup + LOOKUP_SEP + Versionable.OBJECT_IDENTIFIER_FIELD From 10adfb13f81ae3c5ddc39be02aa7006fa06f7813 Mon Sep 17 00:00:00 2001 From: Jeckelmann Manuel Date: Mon, 11 Sep 2017 08:30:44 +0200 Subject: [PATCH 34/42] Adapted tox config to test with new Django versions --- tox.ini | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index ab78a33..a6ac18a 100644 --- a/tox.ini +++ b/tox.ini @@ -5,12 +5,14 @@ [tox] envlist = - django{19}-{sqlite,pg} + django{19,110,111}-{sqlite,pg} [testenv] deps = coverage django19: django>=1.9,<1.10 + django110: django>=1.10,<1.11 + django111: django>=1.11,<1.12 pg: psycopg2 commands = pg: coverage run --source=versions ./manage.py test --settings={env:TOX_PG_CONF:cleanerversion.settings.pg} From 1d7160abdebb9fd82208d2d99e0b1bb37fa2abd5 Mon Sep 17 00:00:00 2001 From: Jeckelmann Manuel Date: Mon, 11 Sep 2017 08:35:12 +0200 Subject: [PATCH 35/42] Removed explicit listing of all possible tox environments, since every combination is run anyways --- .travis.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 180f054..a858215 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,10 +3,6 @@ python: - "2.7" - "3.6" -env: - - TOX_ENV=django19-pg - - TOX_ENV=django19-sqlite - # Enable PostgreSQL usage addons: postgresql: "9.3" @@ -26,7 +22,7 @@ before_script: # Run tests script: - tox -e $TOX_ENV + tox # Run coveralls after_success: From 854b3cb4d2d035aa6d47c31fdcaf17fd3b6502e8 Mon Sep 17 00:00:00 2001 From: Jeckelmann Manuel Date: Mon, 11 Sep 2017 08:38:36 +0200 Subject: [PATCH 36/42] Added tox-env enumeration back to travis config --- .travis.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a858215..d0c14cb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,15 @@ python: - "2.7" - "3.6" +env: + - TOX_ENV=django19-pg + - TOX_ENV=django19-sqlite + - TOX_ENV=django110-pg + - TOX_ENV=django110-sqlite + - TOX_ENV=django111-pg + - TOX_ENV=django111-sqlite + + # Enable PostgreSQL usage addons: postgresql: "9.3" @@ -22,7 +31,7 @@ before_script: # Run tests script: - tox + tox -e $TOX_ENV # Run coveralls after_success: From 76c906ce6ff4ec885d45af04d49f78cc57936f03 Mon Sep 17 00:00:00 2001 From: Jeckelmann Manuel Date: Mon, 11 Sep 2017 09:26:44 +0200 Subject: [PATCH 37/42] Adapted documentation to reflect the currently supported Django versions --- README.rst | 10 ++-- cleanerversion/__init__.py | 2 +- docs/conf.py | 6 ++- .../doc/historization_with_cleanerversion.rst | 47 ++++++++++++------- setup.py | 12 +++-- 5 files changed, 50 insertions(+), 27 deletions(-) diff --git a/README.rst b/README.rst index c056948..411dc01 100644 --- a/README.rst +++ b/README.rst @@ -48,18 +48,22 @@ Prerequisites This code was tested with the following technical components -* Python 2.7 & 3.4 -* Django 1.8 +* Python 2.7 & 3.6 +* Django 1.9 - 1.11 * PostgreSQL 9.3.4 & SQLite3 Older Django versions ===================== -CleanerVersion was originally written for Django 1.6. +CleanerVersion was originally written for Django 1.6 and has now been ported up to Django 1.11. + +CleanerVersion 2.x releases are compatible with Django 1.9, 1.10 and 1.11. Old packages compatible with older Django releases: * Django 1.6 and 1.7: https://pypi.python.org/pypi/CleanerVersion/1.5.4 +* Django 1.8: https://pypi.python.org/pypi/CleanerVersion/1.6.2 + Documentation ============= diff --git a/cleanerversion/__init__.py b/cleanerversion/__init__.py index 8b13789..c7565ea 100644 --- a/cleanerversion/__init__.py +++ b/cleanerversion/__init__.py @@ -1 +1 @@ - +VERSION = (2, 0, 0) diff --git a/docs/conf.py b/docs/conf.py index c5761d5..90583a9 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -19,6 +19,8 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. #sys.path.insert(0, os.path.abspath('.')) +import cleanerversion + sys.path.insert(len(sys.path), os.path.abspath('..')) # -- General configuration ------------------------------------------------ @@ -58,9 +60,9 @@ # built documents. # # The short X.Y version. -version = '1.2' +version = '.'.join(cleanerversion.VERSION[:2]) # The full version, including alpha/beta/rc tags. -release = '1.2.2' +release = '.'.join(cleanerversion.VERSION) # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/doc/historization_with_cleanerversion.rst b/docs/doc/historization_with_cleanerversion.rst index f9b38b9..70b2fc8 100644 --- a/docs/doc/historization_with_cleanerversion.rst +++ b/docs/doc/historization_with_cleanerversion.rst @@ -3,7 +3,7 @@ Historization with CleanerVersion ********************************* Disclaimer: This documentation as well as the CleanerVersion application code have been written to work against Django -1.8.x. The documentation may not be accurate anymore when using more recent versions of Django. +1.9.x through 1.11.x. The documentation may not be accurate anymore when using more recent versions of Django. .. _cleanerversion-quick-starter: @@ -56,8 +56,8 @@ would be a working example, if place in the same source file. Here's how:: phone = CharField(max_length=200) Assuming you know how to deal with `Django Models `_ (you -will need to sync your DB before your code gets usable; Or you're only testing, then that step is done by Django), the -next step is using your model to create some entries:: +will need to migrate your DB before your code gets usable; Or you're only testing, then that step is done by Django), +the next step is using your model to create some entries:: p = Person.objects.create(name='Donald Fauntleroy Duck', address='Duckburg', phone='123456') t1 = datetime.utcnow().replace(tzinfo=utc) @@ -304,8 +304,8 @@ Here's an example with a sportsclub that can practice at most one sporty discipl name = CharField(max_length=200) rules = CharField(max_length=200) -If a M2O relationship can also be unset, don't forget to set the nullable flag (null=true) as an argument of the -``VersionedForeignKey`` field. +If a many-to-one (M2O) relationship can also be unset, don't forget to set the nullable flag (null=true) as an argument +of the ``VersionedForeignKey`` field. Adding objects to a versioned M2O relationship ---------------------------------------------- @@ -538,8 +538,8 @@ The syntax for soft-deleting is the same as the standard Django Model deletion s Notes about using prefetch_related ---------------------------------- -`prefetch_related `_ accepts -simple sting lookups or `Prefetch `_ +`prefetch_related `_ accepts +simple sting lookups or `Prefetch `_ objects. When using ``prefetch_related`` with CleanerVersion, the generated query that fetches the related objects will @@ -597,6 +597,7 @@ If you have an object item1, and know that it existed at some other time t1, you Accessing the current version of an object ------------------------------------------ + ``current_version(obj)`` will return the latest version of the obj, or ``None`` if no version is currently active. Note that if the current object thinks that it is the current object (e.g. ``version_end_date`` is ``None``), @@ -645,10 +646,12 @@ reverse foreign key, one-to-one or many-to-many fields). Valid values for ``rel Deleting objects ================ + You can expect ``delete()`` to behave like you are accustomed to in Django, with these differences: Not actually deleted from the database -------------------------------------- + When you call ``delete()`` on a versioned object, it is not actually removed from the database. Instead, it's ``version_end_date`` is changed from None to a timestamp. @@ -657,6 +660,7 @@ they are terminated by setting a ``version_end_date``. on_delete handlers ------------------ + `on_delete handlers `_ behave like this: @@ -773,15 +777,17 @@ will need to be ready to handle that. Postgresql specific =================== -Django creates `extra indexes `_ -for CharFields that are used for like queries (e.g. WHERE foo like 'fish%'). Since Django 1.6 (the version CleanerVersion originally -targeted) did not have native database UUID fields, the UUID fields that are used for the id and identity columns of Versionable models -have these extra indexes created. In fact, these fields will never be compared using the like operator. Leaving these indexes would create a -performance penalty for inserts and updates, especially for larger tables. ``versions.util.postgresql`` has a function -``remove_uuid_id_like_indexes`` that can be used to remove these extra indexes. +Django creates `extra indexes `_ +for CharFields that are used for like queries (e.g. WHERE foo like 'fish%'). Since Django 1.6 (the version +CleanerVersion originally targeted) did not have native database UUID fields, the UUID fields that are used for the id +and identity columns of Versionable models have these extra indexes created. In fact, these fields will never be +compared using the like operator. Leaving these indexes would create a performance penalty for inserts and updates, +especially for larger tables. ``versions.util.postgresql`` has a function ``remove_uuid_id_like_indexes`` that can be +used to remove these extra indexes. -For the issue of `Unique Indexes`_, ``versions.util.postgresql`` has a function ``create_current_version_unique_indexes`` that can -be used to create unique indexes. For this to work, it's necessary to define a VERSION_UNIQUE attribute when defining the model:: +For the issue of `Unique Indexes`_, ``versions.util.postgresql`` has a function +``create_current_version_unique_indexes`` that can be used to create unique indexes. For this to work, it's necessary +to define a VERSION_UNIQUE attribute when defining the model:: class Person(Versionable): name = models.CharField(max_length=40) @@ -841,6 +847,13 @@ object is current. Upgrade notes ============= + +CleanerVersion 2.x / Django 1.9/1.10/1.11 +------------------------------------------- + +In Django 1.9 major changes to the ORM layer have been introduced, which made existing versions of CleanerVersion for +incompatible with Django 1.9 onwards. We decided to release a separate major version to support the Django 1.9 to 1.11. + CleanerVersion 1.6.0 / Django 1.8.3 ----------------------------------- Starting with CleanerVersion 1.6.0, Django's ``UUIDField`` will be used for the ``id``, ``identity``, @@ -870,8 +883,8 @@ You must choose one or the other solution; not doing so will result in your appl Known Issues ============ -* No `multi-table inheritance `_ support. - Multi-table inheritance currently does not work if the parent model has a Versionable base class. +* No `multi-table inheritance `_ + support. Multi-table inheritance currently does not work if the parent model has a Versionable base class. See `this issue `_ for more details. * Creating `Unique Indexes`_ is a bit tricky for versioned database tables. A solution is provided for Postgresql (see the diff --git a/setup.py b/setup.py index b907641..bd819c0 100644 --- a/setup.py +++ b/setup.py @@ -1,13 +1,15 @@ #!/usr/bin/env python from setuptools import setup, find_packages +import cleanerversion + """ Documentation can be found at https://docs.python.org/2/distutils/index.html, but usually you only need to do the following steps to publish a new package version to PyPI:: # Update the version tag in this file (setup.py) - python setup.py register - python setup.py sdist --formats=gztar,zip upload + python setup.py sdist --formats=gztar,zip + twine upload dist/* That's already it. You should get the following output written to your command line:: @@ -20,8 +22,10 @@ - Do you have all the necessary libraries to generate the wanted formats? --> Reduce the set of formats or install libs """ +version = '.'.join(cleanerversion.VERSION) + setup(name='CleanerVersion', - version='1.6.1', + version=version, description='A versioning solution for relational data models using the Django ORM', long_description='CleanerVersion is a solution that allows you to read and write multiple versions of an entry ' 'to and from your relational database. It allows to keep track of modifications on an object ' @@ -41,7 +45,7 @@ 'Framework :: Django', 'Intended Audience :: Developers', 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.6', 'Topic :: Database', 'Topic :: System :: Archiving', ]) From 79545114516449c601dede8e9b3a70fbedfa9072 Mon Sep 17 00:00:00 2001 From: Jeckelmann Manuel Date: Mon, 11 Sep 2017 10:30:08 +0200 Subject: [PATCH 38/42] Updated the contribution guideline --- CONTRIBUTING.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 4700920..3c870de 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -32,7 +32,7 @@ Local Testing To test locally on the various environments that are tested by Travis, you can use `tox `_. To do this, these dependencies must be installed: -* python 2.7 and python 3.4 +* python 2.7 and python 3.6 * tox (if you're using pip, you can install tox with ``pip install tox``) * postgresql 9.3.x From 1fcc7d9e8d7c5ed628999d8760d9c36828b94ba9 Mon Sep 17 00:00:00 2001 From: Jeckelmann Manuel Date: Mon, 11 Sep 2017 10:31:53 +0200 Subject: [PATCH 39/42] Adapted version gathering routine --- cleanerversion/__init__.py | 7 +++++++ docs/conf.py | 4 ++-- setup.py | 37 ++++++++++++++++++++++++------------- 3 files changed, 33 insertions(+), 15 deletions(-) diff --git a/cleanerversion/__init__.py b/cleanerversion/__init__.py index c7565ea..02b88f8 100644 --- a/cleanerversion/__init__.py +++ b/cleanerversion/__init__.py @@ -1 +1,8 @@ VERSION = (2, 0, 0) + +def get_version(positions=None): + version = VERSION + if positions and isinstance(positions, int): + version = VERSION[:positions] + version = (str(v) for v in version) + return '.'.join(version) diff --git a/docs/conf.py b/docs/conf.py index 90583a9..218d43a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -60,9 +60,9 @@ # built documents. # # The short X.Y version. -version = '.'.join(cleanerversion.VERSION[:2]) +version = cleanerversion.get_version(2) # The full version, including alpha/beta/rc tags. -release = '.'.join(cleanerversion.VERSION) +release = cleanerversion.get_version() # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index bd819c0..98c0ce1 100644 --- a/setup.py +++ b/setup.py @@ -4,42 +4,53 @@ import cleanerversion """ -Documentation can be found at https://docs.python.org/2/distutils/index.html, but usually you only need to do the -following steps to publish a new package version to PyPI:: +Documentation can be found at https://docs.python.org/2/distutils/index.html, +but usually you only need to do the following steps to publish a new package +version to PyPI:: # Update the version tag in this file (setup.py) python setup.py sdist --formats=gztar,zip twine upload dist/* -That's already it. You should get the following output written to your command line:: +That's already it. You should get the following output written to your +command line:: Server response (200): OK If you get errors, check the following things: -- Are you behind a proxy? --> Try not to be behind a proxy (I don't actually know how to configure setup.py to be proxy-aware) +- Are you behind a proxy? --> Try not to be behind a proxy (I don't actually + know how to configure setup.py to be proxy-aware) - Is your command correct? --> Double-check using the reference documentation -- Do you have all the necessary libraries to generate the wanted formats? --> Reduce the set of formats or install libs +- Do you have all the necessary libraries to generate the wanted formats? --> + Reduce the set of formats or install libs """ -version = '.'.join(cleanerversion.VERSION) +version = cleanerversion.get_version() setup(name='CleanerVersion', version=version, - description='A versioning solution for relational data models using the Django ORM', - long_description='CleanerVersion is a solution that allows you to read and write multiple versions of an entry ' - 'to and from your relational database. It allows to keep track of modifications on an object ' - 'over time, as described by the theory of **Slowly Changing Dimensions** (SCD) **- Type 2**. ' + description='A versioning solution for relational data models using the ' + 'Django ORM', + long_description='CleanerVersion is a solution that allows you to read ' + 'and write multiple versions of an entry ' + 'to and from your relational database. It allows to ' + 'keep track of modifications on an object ' + 'over time, as described by the theory of **Slowly ' + 'Changing Dimensions** (SCD) **- Type 2**. ' '' - 'CleanerVersion therefore enables a Django-based Datawarehouse, which was the initial idea of ' + 'CleanerVersion therefore enables a Django-based ' + 'Datawarehouse, which was the initial idea of ' 'this package.', - author='Manuel Jeckelmann, Jean-Christophe Zulian, Brian King, Andrea Marcacci', + author='Manuel Jeckelmann, Jean-Christophe Zulian, Brian King, ' + 'Andrea Marcacci', author_email='engineering.sophia@swisscom.com', license='Apache License 2.0', packages=find_packages(exclude=['cleanerversion', 'cleanerversion.*']), url='https://github.com/swisscom/cleanerversion', install_requires=['django'], - package_data={'versions': ['static/js/*.js','templates/versions/*.html']}, + package_data={'versions': ['static/js/*.js', + 'templates/versions/*.html']}, classifiers=[ 'Development Status :: 5 - Production/Stable', 'Framework :: Django', From 1bbd3b13ee678f05122752516717d505dbf18732 Mon Sep 17 00:00:00 2001 From: Jeckelmann Manuel Date: Mon, 11 Sep 2017 10:34:38 +0200 Subject: [PATCH 40/42] PEP8 on docs config --- docs/conf.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 218d43a..e06db50 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -53,7 +53,8 @@ # General information about the project. project = u'CleanerVersion' -copyright = u'2014, Jean-Christophe Zulian, Brian King, Andrea Marcacci, Manuel Jeckelmann' +copyright = u'2014, Jean-Christophe Zulian, Brian King, Andrea Marcacci, ' \ + u'Manuel Jeckelmann' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -206,7 +207,8 @@ # author, documentclass [howto, manual, or own class]). latex_documents = [ ('index', 'CleanerVersion.tex', u'CleanerVersion Documentation', - u'Jean-Christophe Zulian, Brian King, Andrea Marcacci, Manuel Jeckelmann', 'manual'), + u'Jean-Christophe Zulian, Brian King, Andrea Marcacci, Manuel Jeckelmann', + 'manual'), ] # The name of an image file (relative to this directory) to place at the top of @@ -235,8 +237,12 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - ('index', 'cleanerversion', u'CleanerVersion Documentation', - [u'Jean-Christophe Zulian, Brian King, Andrea Marcacci, Manuel Jeckelmann'], 1) + ('index', + 'cleanerversion', + u'CleanerVersion Documentation', + [u'Jean-Christophe Zulian, Brian King, Andrea Marcacci, ' + u'Manuel Jeckelmann'], + 1) ] # If true, show URL addresses after external links. @@ -249,9 +255,13 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'CleanerVersion', u'CleanerVersion Documentation', - u'Jean-Christophe Zulian, Brian King, Andrea Marcacci, Manuel Jeckelmann', 'CleanerVersion', - 'One line description of project.', 'Miscellaneous'), + ('index', + 'CleanerVersion', + u'CleanerVersion Documentation', + u'Jean-Christophe Zulian, Brian King, Andrea Marcacci, Manuel Jeckelmann', + 'CleanerVersion', + 'One line description of project.', + 'Miscellaneous'), ] # Documents to append as an appendix to all manuals. @@ -270,5 +280,6 @@ # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = { 'python': ('http://docs.python.org/', None), - 'django': ('https://docs.djangoproject.com/en/dev/', 'https://docs.djangoproject.com/en/dev/_objects/') + 'django': ('https://docs.djangoproject.com/en/dev/', + 'https://docs.djangoproject.com/en/dev/_objects/') } From 8764759698f470e47a4cc2834f557e731de026bc Mon Sep 17 00:00:00 2001 From: Jeckelmann Manuel Date: Mon, 11 Sep 2017 11:40:03 +0200 Subject: [PATCH 41/42] Changed import strategy in setup.py --- setup.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 98c0ce1..8b9c703 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,6 @@ #!/usr/bin/env python from setuptools import setup, find_packages -import cleanerversion - """ Documentation can be found at https://docs.python.org/2/distutils/index.html, but usually you only need to do the following steps to publish a new package @@ -26,7 +24,7 @@ Reduce the set of formats or install libs """ -version = cleanerversion.get_version() +version = __import__('cleanerversion').get_version() setup(name='CleanerVersion', version=version, From 6531c3aa89384fc211994796dca8ca59f421fbef Mon Sep 17 00:00:00 2001 From: Jeckelmann Manuel Date: Mon, 11 Sep 2017 11:29:41 +0200 Subject: [PATCH 42/42] Adapted setup.py not exclude the cleanerversion module --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8b9c703..b2ccee7 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ 'Andrea Marcacci', author_email='engineering.sophia@swisscom.com', license='Apache License 2.0', - packages=find_packages(exclude=['cleanerversion', 'cleanerversion.*']), + packages=find_packages(exclude=['cleanerversion.settings.*']), url='https://github.com/swisscom/cleanerversion', install_requires=['django'], package_data={'versions': ['static/js/*.js',