diff --git a/.gitignore b/.gitignore
index 0812226..1096574 100644
--- a/.gitignore
+++ b/.gitignore
@@ -29,3 +29,9 @@ sqlite.db
# dev_no_debug.py imports local.py and disables Django-debugging
cleanerversion/settings/dev_no_debug.py
cleanerversion/settings/*_local.py
+
+bin/
+share/
+include/
+lib/
+pip-selfcheck.json
diff --git a/.travis.yml b/.travis.yml
index cefb568..30d29b7 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -4,10 +4,13 @@ python:
- "3.6"
env:
- - TOX_ENV=py27-django18-pg
- - TOX_ENV=py27-django18-sqlite
- - TOX_ENV=py36-django18-pg
- - TOX_ENV=py36-django18-sqlite
+ - TOX_ENV=django19-pg
+ - TOX_ENV=django19-sqlite
+ - TOX_ENV=django110-pg
+ - TOX_ENV=django110-sqlite
+ - TOX_ENV=django111-pg
+ - TOX_ENV=django111-sqlite
+
# Enable PostgreSQL usage
addons:
diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
index 4700920..3c870de 100644
--- a/CONTRIBUTING.rst
+++ b/CONTRIBUTING.rst
@@ -32,7 +32,7 @@ Local Testing
To test locally on the various environments that are tested by Travis, you can use `tox `_.
To do this, these dependencies must be installed:
-* python 2.7 and python 3.4
+* python 2.7 and python 3.6
* tox (if you're using pip, you can install tox with ``pip install tox``)
* postgresql 9.3.x
diff --git a/README.rst b/README.rst
index c056948..411dc01 100644
--- a/README.rst
+++ b/README.rst
@@ -48,18 +48,22 @@ Prerequisites
This code was tested with the following technical components
-* Python 2.7 & 3.4
-* Django 1.8
+* Python 2.7 & 3.6
+* Django 1.9 - 1.11
* PostgreSQL 9.3.4 & SQLite3
Older Django versions
=====================
-CleanerVersion was originally written for Django 1.6.
+CleanerVersion was originally written for Django 1.6 and has now been ported up to Django 1.11.
+
+CleanerVersion 2.x releases are compatible with Django 1.9, 1.10 and 1.11.
Old packages compatible with older Django releases:
* Django 1.6 and 1.7: https://pypi.python.org/pypi/CleanerVersion/1.5.4
+* Django 1.8: https://pypi.python.org/pypi/CleanerVersion/1.6.2
+
Documentation
=============
diff --git a/cleanerversion/__init__.py b/cleanerversion/__init__.py
index 8b13789..02b88f8 100644
--- a/cleanerversion/__init__.py
+++ b/cleanerversion/__init__.py
@@ -1 +1,8 @@
+VERSION = (2, 0, 0)
+def get_version(positions=None):
+ version = VERSION
+ if positions and isinstance(positions, int):
+ version = VERSION[:positions]
+ version = (str(v) for v in version)
+ return '.'.join(version)
diff --git a/cleanerversion/settings/base.py b/cleanerversion/settings/base.py
index 546abce..940b815 100644
--- a/cleanerversion/settings/base.py
+++ b/cleanerversion/settings/base.py
@@ -9,9 +9,11 @@
"""
from __future__ import absolute_import
-from django.utils.crypto import get_random_string
+
import os
+from django.utils.crypto import get_random_string
+
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)) + '/..')
# SECURITY WARNING: keep the secret key used in production secret!
@@ -20,9 +22,17 @@
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
-TEMPLATE_DEBUG = True
-
-
+TEMPLATES = [
+ {
+ 'BACKEND': 'django.template.backends.django.DjangoTemplates',
+ 'DIRS': [],
+ 'APP_DIRS': True,
+ 'OPTIONS': {
+ 'context_processors': ['django.contrib.auth.context_processors.auth',
+ 'django.contrib.messages.context_processors.messages'],
+ },
+ },
+]
ALLOWED_HOSTS = []
@@ -64,7 +74,7 @@
'django.contrib.messages',
'django.contrib.staticfiles',
'versions',
- 'versions_tests',
+ 'versions_tests.apps.VersionsTestsConfig',
)
MIDDLEWARE_CLASSES = (
@@ -85,9 +95,8 @@
USE_L10N = True
USE_TZ = True
-
ROOT_URLCONF = 'cleanerversion.urls'
-
STATIC_URL = '/static/'
+VERSIONS_USE_UUIDFIELD = False
diff --git a/cleanerversion/settings/pg_travis.py b/cleanerversion/settings/pg_travis.py
index ff3b409..ab66743 100644
--- a/cleanerversion/settings/pg_travis.py
+++ b/cleanerversion/settings/pg_travis.py
@@ -16,3 +16,5 @@
'PORT': '5432',
},
}
+
+VERSIONS_USE_UUIDFIELD = True
diff --git a/cleanerversion/urls.py b/cleanerversion/urls.py
index 3a6e6c4..baa40c2 100644
--- a/cleanerversion/urls.py
+++ b/cleanerversion/urls.py
@@ -1,5 +1,6 @@
-from django.conf.urls import patterns, url
+from django.conf.urls import url
from django.contrib import admin
-urlpatterns = patterns('',
- url(r'^admin/', admin.site.urls, ), )
+urlpatterns = [
+ url(r'^admin/', admin.site.urls, ),
+]
diff --git a/docs/conf.py b/docs/conf.py
index c5761d5..e06db50 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -19,6 +19,8 @@
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))
+import cleanerversion
+
sys.path.insert(len(sys.path), os.path.abspath('..'))
# -- General configuration ------------------------------------------------
@@ -51,16 +53,17 @@
# General information about the project.
project = u'CleanerVersion'
-copyright = u'2014, Jean-Christophe Zulian, Brian King, Andrea Marcacci, Manuel Jeckelmann'
+copyright = u'2014, Jean-Christophe Zulian, Brian King, Andrea Marcacci, ' \
+ u'Manuel Jeckelmann'
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
-version = '1.2'
+version = cleanerversion.get_version(2)
# The full version, including alpha/beta/rc tags.
-release = '1.2.2'
+release = cleanerversion.get_version()
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
@@ -204,7 +207,8 @@
# author, documentclass [howto, manual, or own class]).
latex_documents = [
('index', 'CleanerVersion.tex', u'CleanerVersion Documentation',
- u'Jean-Christophe Zulian, Brian King, Andrea Marcacci, Manuel Jeckelmann', 'manual'),
+ u'Jean-Christophe Zulian, Brian King, Andrea Marcacci, Manuel Jeckelmann',
+ 'manual'),
]
# The name of an image file (relative to this directory) to place at the top of
@@ -233,8 +237,12 @@
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
- ('index', 'cleanerversion', u'CleanerVersion Documentation',
- [u'Jean-Christophe Zulian, Brian King, Andrea Marcacci, Manuel Jeckelmann'], 1)
+ ('index',
+ 'cleanerversion',
+ u'CleanerVersion Documentation',
+ [u'Jean-Christophe Zulian, Brian King, Andrea Marcacci, '
+ u'Manuel Jeckelmann'],
+ 1)
]
# If true, show URL addresses after external links.
@@ -247,9 +255,13 @@
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
- ('index', 'CleanerVersion', u'CleanerVersion Documentation',
- u'Jean-Christophe Zulian, Brian King, Andrea Marcacci, Manuel Jeckelmann', 'CleanerVersion',
- 'One line description of project.', 'Miscellaneous'),
+ ('index',
+ 'CleanerVersion',
+ u'CleanerVersion Documentation',
+ u'Jean-Christophe Zulian, Brian King, Andrea Marcacci, Manuel Jeckelmann',
+ 'CleanerVersion',
+ 'One line description of project.',
+ 'Miscellaneous'),
]
# Documents to append as an appendix to all manuals.
@@ -268,5 +280,6 @@
# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {
'python': ('http://docs.python.org/', None),
- 'django': ('https://docs.djangoproject.com/en/dev/', 'https://docs.djangoproject.com/en/dev/_objects/')
+ 'django': ('https://docs.djangoproject.com/en/dev/',
+ 'https://docs.djangoproject.com/en/dev/_objects/')
}
diff --git a/docs/doc/historization_with_cleanerversion.rst b/docs/doc/historization_with_cleanerversion.rst
index 54d80ce..53ab610 100644
--- a/docs/doc/historization_with_cleanerversion.rst
+++ b/docs/doc/historization_with_cleanerversion.rst
@@ -3,7 +3,7 @@ Historization with CleanerVersion
*********************************
Disclaimer: This documentation as well as the CleanerVersion application code have been written to work against Django
-1.8.x. The documentation may not be accurate anymore when using more recent versions of Django.
+1.9.x through 1.11.x. The documentation may not be accurate anymore when using more recent versions of Django.
.. _cleanerversion-quick-starter:
@@ -56,8 +56,8 @@ would be a working example, if place in the same source file. Here's how::
phone = CharField(max_length=200)
Assuming you know how to deal with `Django Models `_ (you
-will need to sync your DB before your code gets usable; Or you're only testing, then that step is done by Django), the
-next step is using your model to create some entries::
+will need to migrate your DB before your code gets usable; Or you're only testing, then that step is done by Django),
+the next step is using your model to create some entries::
p = Person.objects.create(name='Donald Fauntleroy Duck', address='Duckburg', phone='123456')
t1 = datetime.utcnow().replace(tzinfo=utc)
@@ -304,8 +304,8 @@ Here's an example with a sportsclub that can practice at most one sporty discipl
name = CharField(max_length=200)
rules = CharField(max_length=200)
-If a M2O relationship can also be unset, don't forget to set the nullable flag (null=true) as an argument of the
-``VersionedForeignKey`` field.
+If a many-to-one (M2O) relationship can also be unset, don't forget to set the nullable flag (null=true) as an argument
+of the ``VersionedForeignKey`` field.
Adding objects to a versioned M2O relationship
----------------------------------------------
@@ -544,8 +544,8 @@ The syntax for soft-deleting is the same as the standard Django Model deletion s
Notes about using prefetch_related
----------------------------------
-`prefetch_related `_ accepts
-simple sting lookups or `Prefetch `_
+`prefetch_related `_ accepts
+simple sting lookups or `Prefetch `_
objects.
When using ``prefetch_related`` with CleanerVersion, the generated query that fetches the related objects will
@@ -603,6 +603,7 @@ If you have an object item1, and know that it existed at some other time t1, you
Accessing the current version of an object
------------------------------------------
+
``current_version(obj)`` will return the latest version of the obj, or ``None`` if no version is currently active.
Note that if the current object thinks that it is the current object (e.g. ``version_end_date`` is ``None``),
@@ -651,10 +652,12 @@ reverse foreign key, one-to-one or many-to-many fields). Valid values for ``rel
Deleting objects
================
+
You can expect ``delete()`` to behave like you are accustomed to in Django, with these differences:
Not actually deleted from the database
--------------------------------------
+
When you call ``delete()`` on a versioned object, it is not actually removed from the database. Instead, it's
``version_end_date`` is changed from None to a timestamp.
@@ -663,6 +666,7 @@ they are terminated by setting a ``version_end_date``.
on_delete handlers
------------------
+
`on_delete handlers `_
behave like this:
@@ -779,15 +783,17 @@ will need to be ready to handle that.
Postgresql specific
===================
-Django creates `extra indexes `_
-for CharFields that are used for like queries (e.g. WHERE foo like 'fish%'). Since Django 1.6 (the version CleanerVersion originally
-targeted) did not have native database UUID fields, the UUID fields that are used for the id and identity columns of Versionable models
-have these extra indexes created. In fact, these fields will never be compared using the like operator. Leaving these indexes would create a
-performance penalty for inserts and updates, especially for larger tables. ``versions.util.postgresql`` has a function
-``remove_uuid_id_like_indexes`` that can be used to remove these extra indexes.
+Django creates `extra indexes `_
+for CharFields that are used for like queries (e.g. WHERE foo like 'fish%'). Since Django 1.6 (the version
+CleanerVersion originally targeted) did not have native database UUID fields, the UUID fields that are used for the id
+and identity columns of Versionable models have these extra indexes created. In fact, these fields will never be
+compared using the like operator. Leaving these indexes would create a performance penalty for inserts and updates,
+especially for larger tables. ``versions.util.postgresql`` has a function ``remove_uuid_id_like_indexes`` that can be
+used to remove these extra indexes.
-For the issue of `Unique Indexes`_, ``versions.util.postgresql`` has a function ``create_current_version_unique_indexes`` that can
-be used to create unique indexes. For this to work, it's necessary to define a VERSION_UNIQUE attribute when defining the model::
+For the issue of `Unique Indexes`_, ``versions.util.postgresql`` has a function
+``create_current_version_unique_indexes`` that can be used to create unique indexes. For this to work, it's necessary
+to define a VERSION_UNIQUE attribute when defining the model::
class Person(Versionable):
name = models.CharField(max_length=40)
@@ -847,6 +853,13 @@ object is current.
Upgrade notes
=============
+
+CleanerVersion 2.x / Django 1.9/1.10/1.11
+-------------------------------------------
+
+In Django 1.9 major changes to the ORM layer have been introduced, which made existing versions of CleanerVersion for
+incompatible with Django 1.9 onwards. We decided to release a separate major version to support the Django 1.9 to 1.11.
+
CleanerVersion 1.6.0 / Django 1.8.3
-----------------------------------
Starting with CleanerVersion 1.6.0, Django's ``UUIDField`` will be used for the ``id``, ``identity``,
@@ -876,8 +889,8 @@ You must choose one or the other solution; not doing so will result in your appl
Known Issues
============
-* No `multi-table inheritance `_ support.
- Multi-table inheritance currently does not work if the parent model has a Versionable base class.
+* No `multi-table inheritance `_
+ support. Multi-table inheritance currently does not work if the parent model has a Versionable base class.
See `this issue `_ for more details.
* Creating `Unique Indexes`_ is a bit tricky for versioned database tables. A solution is provided for Postgresql (see the
diff --git a/manage.py b/manage.py
old mode 100644
new mode 100755
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..867eec0
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1 @@
+Django>=1.10
diff --git a/setup.py b/setup.py
index b907641..b2ccee7 100644
--- a/setup.py
+++ b/setup.py
@@ -2,46 +2,59 @@
from setuptools import setup, find_packages
"""
-Documentation can be found at https://docs.python.org/2/distutils/index.html, but usually you only need to do the
-following steps to publish a new package version to PyPI::
+Documentation can be found at https://docs.python.org/2/distutils/index.html,
+but usually you only need to do the following steps to publish a new package
+version to PyPI::
# Update the version tag in this file (setup.py)
- python setup.py register
- python setup.py sdist --formats=gztar,zip upload
+ python setup.py sdist --formats=gztar,zip
+ twine upload dist/*
-That's already it. You should get the following output written to your command line::
+That's already it. You should get the following output written to your
+command line::
Server response (200): OK
If you get errors, check the following things:
-- Are you behind a proxy? --> Try not to be behind a proxy (I don't actually know how to configure setup.py to be proxy-aware)
+- Are you behind a proxy? --> Try not to be behind a proxy (I don't actually
+ know how to configure setup.py to be proxy-aware)
- Is your command correct? --> Double-check using the reference documentation
-- Do you have all the necessary libraries to generate the wanted formats? --> Reduce the set of formats or install libs
+- Do you have all the necessary libraries to generate the wanted formats? -->
+ Reduce the set of formats or install libs
"""
+version = __import__('cleanerversion').get_version()
+
setup(name='CleanerVersion',
- version='1.6.1',
- description='A versioning solution for relational data models using the Django ORM',
- long_description='CleanerVersion is a solution that allows you to read and write multiple versions of an entry '
- 'to and from your relational database. It allows to keep track of modifications on an object '
- 'over time, as described by the theory of **Slowly Changing Dimensions** (SCD) **- Type 2**. '
+ version=version,
+ description='A versioning solution for relational data models using the '
+ 'Django ORM',
+ long_description='CleanerVersion is a solution that allows you to read '
+ 'and write multiple versions of an entry '
+ 'to and from your relational database. It allows to '
+ 'keep track of modifications on an object '
+ 'over time, as described by the theory of **Slowly '
+ 'Changing Dimensions** (SCD) **- Type 2**. '
''
- 'CleanerVersion therefore enables a Django-based Datawarehouse, which was the initial idea of '
+ 'CleanerVersion therefore enables a Django-based '
+ 'Datawarehouse, which was the initial idea of '
'this package.',
- author='Manuel Jeckelmann, Jean-Christophe Zulian, Brian King, Andrea Marcacci',
+ author='Manuel Jeckelmann, Jean-Christophe Zulian, Brian King, '
+ 'Andrea Marcacci',
author_email='engineering.sophia@swisscom.com',
license='Apache License 2.0',
- packages=find_packages(exclude=['cleanerversion', 'cleanerversion.*']),
+ packages=find_packages(exclude=['cleanerversion.settings.*']),
url='https://github.com/swisscom/cleanerversion',
install_requires=['django'],
- package_data={'versions': ['static/js/*.js','templates/versions/*.html']},
+ package_data={'versions': ['static/js/*.js',
+ 'templates/versions/*.html']},
classifiers=[
'Development Status :: 5 - Production/Stable',
'Framework :: Django',
'Intended Audience :: Developers',
'Programming Language :: Python :: 2.7',
- 'Programming Language :: Python :: 3.4',
+ 'Programming Language :: Python :: 3.6',
'Topic :: Database',
'Topic :: System :: Archiving',
])
diff --git a/tox.ini b/tox.ini
index e67add8..a6ac18a 100644
--- a/tox.ini
+++ b/tox.ini
@@ -5,14 +5,15 @@
[tox]
envlist =
- py{27,36}-django{18}-{sqlite,pg}
+ django{19,110,111}-{sqlite,pg}
[testenv]
deps =
coverage
- django18: django>=1.8,<1.9
+ django19: django>=1.9,<1.10
+ django110: django>=1.10,<1.11
+ django111: django>=1.11,<1.12
pg: psycopg2
commands =
pg: coverage run --source=versions ./manage.py test --settings={env:TOX_PG_CONF:cleanerversion.settings.pg}
sqlite: coverage run --source=versions ./manage.py test --settings=cleanerversion.settings.sqlite
-
diff --git a/versions/admin.py b/versions/admin.py
index 2fbc637..e33b85a 100644
--- a/versions/admin.py
+++ b/versions/admin.py
@@ -105,7 +105,7 @@ def queryset(self, request, queryset):
class VersionedAdminChecks(ModelAdminChecks):
- def _check_exclude(self, cls, model):
+ def _check_exclude(self, cls, model=None):
"""
Required to suppress error about exclude not being a tuple since we are using @property to dynamically change it
"""
diff --git a/versions/deletion.py b/versions/deletion.py
index 91dc45f..819f1d6 100644
--- a/versions/deletion.py
+++ b/versions/deletion.py
@@ -1,9 +1,9 @@
-from django import VERSION
from django.db.models.deletion import (
attrgetter, signals, six, sql, transaction,
CASCADE,
Collector,
)
+
import versions.models
@@ -68,7 +68,8 @@ def delete(self, timestamp):
# In the case of a SET.. method, clone before changing the value (if it hasn't already been
# cloned)
updated_instances = set()
- if not(isinstance(field, versions.models.VersionedForeignKey) and field.rel.on_delete == CASCADE):
+ if not (
+ isinstance(field, versions.fields.VersionedForeignKey) and field.rel.on_delete == CASCADE):
for instance in instances:
# Clone before updating
cloned = id_map.get(instance.pk, None)
@@ -76,7 +77,7 @@ def delete(self, timestamp):
cloned = instance.clone()
id_map[instance.pk] = cloned
updated_instances.add(cloned)
- #TODO: instance should get updated with new values from clone ?
+ # TODO: instance should get updated with new values from clone ?
instances_for_fieldvalues[(field, value)] = updated_instances
# Replace the instances with their clones in self.data, too
@@ -135,12 +136,11 @@ def related_objects(self, related, objs):
Gets a QuerySet of current objects related to ``objs`` via the relation ``related``.
"""
- if VERSION >= (1, 8):
- related_model = related.related_model
- else:
- related_model = related.model
+ from versions.models import Versionable
+
+ related_model = related.related_model
manager = related_model._base_manager
- if issubclass(related_model, versions.models.Versionable):
+ if issubclass(related_model, Versionable):
manager = manager.current
return manager.using(self.using).filter(
**{"%s__in" % related.field.name: objs}
diff --git a/versions/descriptors.py b/versions/descriptors.py
new file mode 100644
index 0000000..88ceaa6
--- /dev/null
+++ b/versions/descriptors.py
@@ -0,0 +1,469 @@
+from collections import namedtuple
+
+from django.core.exceptions import SuspiciousOperation, FieldDoesNotExist
+from django.db import router, transaction
+from django.db.models.base import Model
+from django.db.models.fields.related import (ForwardManyToOneDescriptor, ReverseManyToOneDescriptor,
+ ManyToManyDescriptor)
+from django.db.models.fields.related_descriptors import create_forward_many_to_many_manager
+from django.db.models.query_utils import Q
+from django.utils.functional import cached_property
+
+from versions.util import get_utc_now
+
+
+def matches_querytime(instance, querytime):
+ """
+ Checks whether the given instance satisfies the given QueryTime object.
+
+ :param instance: an instance of Versionable
+ :param querytime: QueryTime value to check against
+ """
+ if not querytime.active:
+ return True
+
+ if not querytime.time:
+ return instance.version_end_date is None
+
+ return (instance.version_start_date <= querytime.time
+ and (instance.version_end_date is None or instance.version_end_date > querytime.time))
+
+
+class VersionedForwardManyToOneDescriptor(ForwardManyToOneDescriptor):
+ """
+ The VersionedForwardManyToOneDescriptor is used when pointing another Model using a VersionedForeignKey;
+ For example:
+
+ class Team(Versionable):
+ name = CharField(max_length=200)
+ city = VersionedForeignKey(City, null=True)
+
+ ``team.city`` is a VersionedForwardManyToOneDescriptor
+ """
+
+ def get_prefetch_queryset(self, instances, queryset=None):
+ """
+ Overrides the parent method to:
+ - force queryset to use the querytime of the parent objects
+ - ensure that the join is done on identity, not id
+ - make the cache key identity, not id.
+ """
+ if queryset is None:
+ queryset = self.get_queryset()
+ queryset._add_hints(instance=instances[0])
+
+ # CleanerVersion change 1: force the querytime to be the same as the prefetched-for instance.
+ # This is necessary to have reliable results and avoid extra queries for cache misses when
+ # accessing the child objects from their parents (e.g. choice.poll).
+ instance_querytime = instances[0]._querytime
+ if instance_querytime.active:
+ if queryset.querytime.active and queryset.querytime.time != instance_querytime.time:
+ raise ValueError("A Prefetch queryset that specifies an as_of time must match "
+ "the as_of of the base queryset.")
+ else:
+ queryset.querytime = instance_querytime
+
+ # CleanerVersion change 2: make rel_obj_attr return a tuple with the object's identity.
+ # rel_obj_attr = self.field.get_foreign_related_value
+ def versioned_fk_rel_obj_attr(versioned_rel_obj):
+ return versioned_rel_obj.identity,
+
+ rel_obj_attr = versioned_fk_rel_obj_attr
+ instance_attr = self.field.get_local_related_value
+ instances_dict = {instance_attr(inst): inst for inst in instances}
+ # CleanerVersion change 3: fake the related field so that it provides a name of 'identity'.
+ # related_field = self.field.foreign_related_fields[0]
+ related_field = namedtuple('VersionedRelatedFieldTuple', 'name')('identity')
+
+ # FIXME: This will need to be revisited when we introduce support for
+ # composite fields. In the meantime we take this practical approach to
+ # solve a regression on 1.6 when the reverse manager in hidden
+ # (related_name ends with a '+'). Refs #21410.
+ # The check for len(...) == 1 is a special case that allows the query
+ # to be join-less and smaller. Refs #21760.
+ if self.field.rel.is_hidden() or len(self.field.foreign_related_fields) == 1:
+ query = {'%s__in' % related_field.name: set(instance_attr(inst)[0] for inst in instances)}
+ # query = {'identity__in': set(instance_attr(inst)[0] for inst in instances)}
+ else:
+ query = {'%s__in' % self.field.related_query_name(): instances}
+ queryset = queryset.filter(**query)
+
+ # Since we're going to assign directly in the cache,
+ # we must manage the reverse relation cache manually.
+ if not self.field.rel.multiple:
+ rel_obj_cache_name = self.field.rel.get_cache_name()
+ for rel_obj in queryset:
+ instance = instances_dict[rel_obj_attr(rel_obj)]
+ setattr(rel_obj, rel_obj_cache_name, instance)
+ return queryset, rel_obj_attr, instance_attr, True, self.cache_name
+
+ def get_queryset(self, **hints):
+ queryset = super(VersionedForwardManyToOneDescriptor, self).get_queryset(**hints)
+ if hasattr(queryset, 'querytime'):
+ if 'instance' in hints:
+ instance = hints['instance']
+ if hasattr(instance, '_querytime'):
+ if instance._querytime.active and instance._querytime != queryset.querytime:
+ queryset = queryset.as_of(instance._querytime.time)
+ else:
+ queryset = queryset.as_of(None)
+ return queryset
+
+
+vforward_many_to_one_descriptor_class = VersionedForwardManyToOneDescriptor
+
+
+def vforward_many_to_one_descriptor_getter(self, instance, instance_type=None):
+ """
+ The getter method returns the object, which points instance, e.g. choice.poll returns
+ a Poll instance, whereas the Poll class defines the ForeignKey.
+ :param instance: The object on which the property was accessed
+ :param instance_type: The type of the instance object
+ :return: Returns a Versionable
+ """
+ from versions.models import Versionable
+ current_elt = super(self.__class__, self).__get__(instance, instance_type)
+
+ if instance is None:
+ return self
+
+ if not current_elt:
+ return None
+
+ if not isinstance(current_elt, Versionable):
+ raise TypeError("VersionedForeignKey target is of type "
+ + str(type(current_elt))
+ + ", which is not a subclass of Versionable")
+
+ if hasattr(instance, '_querytime'):
+ # If current_elt matches the instance's querytime, there's no need to make a database query.
+ if matches_querytime(current_elt, instance._querytime):
+ current_elt._querytime = instance._querytime
+ return current_elt
+
+ return current_elt.__class__.objects.as_of(instance._querytime.time).get(identity=current_elt.identity)
+ else:
+ return current_elt.__class__.objects.current.get(identity=current_elt.identity)
+
+
+vforward_many_to_one_descriptor_class.__get__ = vforward_many_to_one_descriptor_getter
+
+
+class VersionedReverseManyToOneDescriptor(ReverseManyToOneDescriptor):
+ @cached_property
+ def related_manager_cls(self):
+ manager_cls = super(VersionedReverseManyToOneDescriptor, self).related_manager_cls
+ rel_field = self.field
+
+ class VersionedRelatedManager(manager_cls):
+ def __init__(self, instance):
+ super(VersionedRelatedManager, self).__init__(instance)
+
+ # This is a hack, in order to get the versioned related objects
+ for key in self.core_filters.keys():
+ if '__exact' in key or '__' not in key:
+ self.core_filters[key] = instance.identity
+
+ def get_queryset(self):
+ from versions.models import VersionedQuerySet
+
+ queryset = super(VersionedRelatedManager, self).get_queryset()
+ # Do not set the query time if it is already correctly set. queryset.as_of() returns a clone
+ # of the queryset, and this will destroy the prefetched objects cache if it exists.
+ if isinstance(queryset, VersionedQuerySet) \
+ and self.instance._querytime.active \
+ and queryset.querytime != self.instance._querytime:
+ queryset = queryset.as_of(self.instance._querytime.time)
+ return queryset
+
+ def get_prefetch_queryset(self, instances, queryset=None):
+ """
+ Overrides RelatedManager's implementation of get_prefetch_queryset so that it works
+ nicely with VersionedQuerySets. It ensures that identities and time-limited where
+ clauses are used when selecting related reverse foreign key objects.
+ """
+ if queryset is None:
+ # Note that this intentionally call's VersionManager's get_queryset, instead of simply calling
+ # the superclasses' get_queryset (as the non-versioned RelatedManager does), because what is
+ # needed is a simple Versioned queryset without any restrictions (e.g. do not
+ # apply self.core_filters).
+ from versions.models import VersionManager
+ queryset = VersionManager.get_queryset(self)
+
+ queryset._add_hints(instance=instances[0])
+ queryset = queryset.using(queryset._db or self._db)
+ instance_querytime = instances[0]._querytime
+ if instance_querytime.active:
+ if queryset.querytime.active and queryset.querytime.time != instance_querytime.time:
+ raise ValueError("A Prefetch queryset that specifies an as_of time must match "
+ "the as_of of the base queryset.")
+ else:
+ queryset.querytime = instance_querytime
+
+ rel_obj_attr = rel_field.get_local_related_value
+ instance_attr = rel_field.get_foreign_related_value
+ # Use identities instead of ids so that this will work with versioned objects.
+ instances_dict = {(inst.identity,): inst for inst in instances}
+ identities = [inst.identity for inst in instances]
+ query = {'%s__identity__in' % rel_field.name: identities}
+ queryset = queryset.filter(**query)
+
+ # Since we just bypassed this class' get_queryset(), we must manage
+ # the reverse relation manually.
+ for rel_obj in queryset:
+ instance = instances_dict[rel_obj_attr(rel_obj)]
+ setattr(rel_obj, rel_field.name, instance)
+ cache_name = rel_field.related_query_name()
+ return queryset, rel_obj_attr, instance_attr, False, cache_name
+
+ def add(self, *objs, **kwargs):
+ from versions.models import Versionable
+ cloned_objs = ()
+ for obj in objs:
+ if not isinstance(obj, Versionable):
+ raise TypeError("Trying to add a non-Versionable to a VersionedForeignKey relationship")
+ cloned_objs += (obj.clone(),)
+ super(VersionedRelatedManager, self).add(*cloned_objs, **kwargs)
+
+ # clear() and remove() are present if the FK is nullable
+ if 'clear' in dir(manager_cls):
+ def clear(self, **kwargs):
+ """
+ Overridden to ensure that the current queryset is used, and to clone objects before they
+ are removed, so that history is not lost.
+ """
+ bulk = kwargs.pop('bulk', True)
+ db = router.db_for_write(self.model, instance=self.instance)
+ queryset = self.current.using(db)
+ with transaction.atomic(using=db, savepoint=False):
+ cloned_pks = [obj.clone().pk for obj in queryset]
+ update_qs = self.current.filter(pk__in=cloned_pks)
+ self._clear(update_qs, bulk)
+
+ if 'remove' in dir(manager_cls):
+ def remove(self, *objs, **kwargs):
+ from versions.models import Versionable
+
+ val = rel_field.get_foreign_related_value(self.instance)
+ cloned_objs = ()
+ for obj in objs:
+ # Is obj actually part of this descriptor set? Otherwise, silently go over it, since Django
+ # handles that case
+ if rel_field.get_local_related_value(obj) == val:
+ # Silently pass over non-versionable items
+ if not isinstance(obj, Versionable):
+ raise TypeError(
+ "Trying to remove a non-Versionable from a VersionedForeignKey realtionship")
+ cloned_objs += (obj.clone(),)
+ super(VersionedRelatedManager, self).remove(*cloned_objs, **kwargs)
+
+ return VersionedRelatedManager
+
+
+class VersionedManyToManyDescriptor(ManyToManyDescriptor):
+ @cached_property
+ def related_manager_cls(self):
+ model = self.rel.related_model if self.reverse else self.rel.model
+ return create_versioned_forward_many_to_many_manager(
+ model._default_manager.__class__,
+ self.rel,
+ reverse=self.reverse,
+ )
+
+ def __set__(self, instance, value):
+ """
+ Completely overridden to avoid bulk deletion that happens when the parent method calls clear().
+
+ The parent method's logic is basically: clear all in bulk, then add the given objects in bulk.
+ Instead, we figure out which ones are being added and removed, and call add and remove for these values.
+ This lets us retain the versioning information.
+
+ Since this is a many-to-many relationship, it is assumed here that the django.db.models.deletion.Collector
+ logic, that is used in clear(), is not necessary here. Collector collects related models, e.g. ones that should
+ also be deleted because they have a ON CASCADE DELETE relationship to the object, or, in the case of
+ "Multi-table inheritance", are parent objects.
+
+ :param instance: The instance on which the getter was called
+ :param value: iterable of items to set
+ """
+
+ if not instance.is_current:
+ raise SuspiciousOperation(
+ "Related values can only be directly set on the current version of an object")
+
+ if not self.field.rel.through._meta.auto_created:
+ opts = self.field.rel.through._meta
+ raise AttributeError(("Cannot set values on a ManyToManyField which specifies an intermediary model. "
+ "Use %s.%s's Manager instead.") % (opts.app_label, opts.object_name))
+
+ manager = self.__get__(instance)
+ # Below comment is from parent __set__ method. We'll force evaluation, too:
+ # clear() can change expected output of 'value' queryset, we force evaluation
+ # of queryset before clear; ticket #19816
+ value = tuple(value)
+
+ being_removed, being_added = self.get_current_m2m_diff(instance, value)
+ timestamp = get_utc_now()
+ manager.remove_at(timestamp, *being_removed)
+ manager.add_at(timestamp, *being_added)
+
+ def get_current_m2m_diff(self, instance, new_objects):
+ """
+ :param instance: Versionable object
+ :param new_objects: objects which are about to be associated with instance
+ :return: (being_removed id list, being_added id list)
+ :rtype : tuple
+ """
+ new_ids = self.pks_from_objects(new_objects)
+ relation_manager = self.__get__(instance)
+
+ filter = Q(**{relation_manager.source_field.attname: instance.pk})
+ qs = self.through.objects.current.filter(filter)
+ try:
+ # Django 1.7
+ target_name = relation_manager.target_field.attname
+ except AttributeError:
+ # Django 1.6
+ target_name = relation_manager.through._meta.get_field_by_name(
+ relation_manager.target_field_name)[0].attname
+ current_ids = set(qs.values_list(target_name, flat=True))
+
+ being_removed = current_ids - new_ids
+ being_added = new_ids - current_ids
+ return list(being_removed), list(being_added)
+
+ def pks_from_objects(self, objects):
+ """
+ Extract all the primary key strings from the given objects. Objects may be Versionables, or bare primary keys.
+ :rtype : set
+ """
+ return {o.pk if isinstance(o, Model) else o for o in objects}
+
+
+def create_versioned_forward_many_to_many_manager(superclass, rel, reverse=None):
+ many_related_manager_klass = create_forward_many_to_many_manager(superclass, rel, reverse)
+
+ class VersionedManyRelatedManager(many_related_manager_klass):
+ def __init__(self, *args, **kwargs):
+ super(VersionedManyRelatedManager, self).__init__(*args, **kwargs)
+ # Additional core filters are: version_start_date <= t & (version_end_date > t | version_end_date IS NULL)
+ # but we cannot work with the Django core filters, since they don't support ORing filters, which
+ # is a thing we need to consider the "version_end_date IS NULL" case;
+ # So, we define our own set of core filters being applied when versioning
+ try:
+ _ = self.through._meta.get_field('version_start_date')
+ _ = self.through._meta.get_field('version_end_date')
+ except FieldDoesNotExist as e:
+ fields = [f.name for f in self.through._meta.get_fields()]
+ print(str(e) + "; available fields are " + ", ".join(fields))
+ raise e
+ # FIXME: this probably does not work when auto-referencing
+
+ def get_queryset(self):
+ """
+ Add a filter to the queryset, limiting the results to be pointed by relationship that are
+ valid for the given timestamp (which is taken at the current instance, or set to now, if not
+ available).
+ Long story short, apply the temporal validity filter also to the intermediary model.
+ """
+ queryset = super(VersionedManyRelatedManager, self).get_queryset()
+ if hasattr(queryset, 'querytime'):
+ if self.instance._querytime.active and self.instance._querytime != queryset.querytime:
+ queryset = queryset.as_of(self.instance._querytime.time)
+ return queryset
+
+ def _remove_items(self, source_field_name, target_field_name, *objs):
+ """
+ Instead of removing items, we simply set the version_end_date of the current item to the
+ current timestamp --> t[now].
+ Like that, there is no more current entry having that identity - which is equal to
+ not existing for timestamps greater than t[now].
+ """
+ return self._remove_items_at(None, source_field_name, target_field_name, *objs)
+
+ def _remove_items_at(self, timestamp, source_field_name, target_field_name, *objs):
+ if objs:
+ if timestamp is None:
+ timestamp = get_utc_now()
+ old_ids = set()
+ for obj in objs:
+ if isinstance(obj, self.model):
+ # The Django 1.7-way is preferred
+ if hasattr(self, 'target_field'):
+ fk_val = self.target_field.get_foreign_related_value(obj)[0]
+ else:
+ raise TypeError("We couldn't find the value of the foreign key, this might be due to the "
+ "use of an unsupported version of Django")
+ old_ids.add(fk_val)
+ else:
+ old_ids.add(obj)
+ db = router.db_for_write(self.through, instance=self.instance)
+ qs = self.through._default_manager.using(db).filter(**{
+ source_field_name: self.instance.id,
+ '%s__in' % target_field_name: old_ids
+ }).as_of(timestamp)
+ for relation in qs:
+ relation._delete_at(timestamp)
+
+ if 'add' in dir(many_related_manager_klass):
+ def add(self, *objs):
+ if not self.instance.is_current:
+ raise SuspiciousOperation(
+ "Adding many-to-many related objects is only possible on the current version")
+
+ # The ManyRelatedManager.add() method uses the through model's default manager to get
+ # a queryset when looking at which objects already exist in the database.
+ # In order to restrict the query to the current versions when that is done,
+ # we temporarily replace the queryset's using method so that the version validity
+ # condition can be specified.
+ klass = self.through._default_manager.get_queryset().__class__
+ __using_backup = klass.using
+
+ def using_replacement(self, *args, **kwargs):
+ qs = __using_backup(self, *args, **kwargs)
+ return qs.as_of(None)
+
+ klass.using = using_replacement
+ super(VersionedManyRelatedManager, self).add(*objs)
+ klass.using = __using_backup
+
+ def add_at(self, timestamp, *objs):
+ """
+ This function adds an object at a certain point in time (timestamp)
+ """
+
+ # First off, define the new constructor
+ def _through_init(self, *args, **kwargs):
+ super(self.__class__, self).__init__(*args, **kwargs)
+ self.version_birth_date = timestamp
+ self.version_start_date = timestamp
+
+ # Through-classes have an empty constructor, so it can easily be overwritten when needed;
+ # This is not the default case, so the overwrite only takes place when we "modify the past"
+ self.through.__init_backup__ = self.through.__init__
+ self.through.__init__ = _through_init
+
+ # Do the add operation
+ self.add(*objs)
+
+ # Remove the constructor again (by replacing it with the original empty constructor)
+ self.through.__init__ = self.through.__init_backup__
+ del self.through.__init_backup__
+
+ add_at.alters_data = True
+
+ if 'remove' in dir(many_related_manager_klass):
+ def remove_at(self, timestamp, *objs):
+ """
+ Performs the act of removing specified relationships at a specified time (timestamp);
+ So, not the objects at a given time are removed, but their relationship!
+ """
+ self._remove_items_at(timestamp, self.source_field_name, self.target_field_name, *objs)
+
+ # For consistency, also handle the symmetrical case
+ if self.symmetrical:
+ self._remove_items_at(timestamp, self.target_field_name, self.source_field_name, *objs)
+
+ remove_at.alters_data = True
+
+ return VersionedManyRelatedManager
diff --git a/versions/fields.py b/versions/fields.py
new file mode 100644
index 0000000..57d5e89
--- /dev/null
+++ b/versions/fields.py
@@ -0,0 +1,336 @@
+from django.db.models.deletion import DO_NOTHING
+from django.db.models.fields.related import ForeignKey, ManyToManyField, \
+ resolve_relation, lazy_related_operation
+from django.db.models.query_utils import Q
+from django.db.models.sql.datastructures import Join
+from django.db.models.sql.where import ExtraWhere, WhereNode
+from django.db.models.utils import make_model_tuple
+
+from versions.descriptors import (VersionedForwardManyToOneDescriptor,
+ VersionedReverseManyToOneDescriptor,
+ VersionedManyToManyDescriptor)
+from versions.models import Versionable
+
+
+class VersionedForeignKey(ForeignKey):
+ """
+ We need to replace the standard ForeignKey declaration in order to be able to introduce
+ the VersionedReverseSingleRelatedObjectDescriptor, which allows to go back in time...
+ We also want to allow keeping track of any as_of time so that joins can be restricted
+ based on that.
+ """
+
+ def __init__(self, *args, **kwargs):
+ super(VersionedForeignKey, self).__init__(*args, **kwargs)
+
+ def contribute_to_class(self, cls, name, virtual_only=False):
+ super(VersionedForeignKey, self).contribute_to_class(cls, name, virtual_only)
+ setattr(cls, self.name, VersionedForwardManyToOneDescriptor(self))
+
+ def contribute_to_related_class(self, cls, related):
+ """
+ Override ForeignKey's methods, and replace the descriptor, if set by the parent's methods
+ """
+ # Internal FK's - i.e., those with a related name ending with '+' -
+ # and swapped models don't get a related descriptor.
+ super(VersionedForeignKey, self).contribute_to_related_class(cls, related)
+ accessor_name = related.get_accessor_name()
+ if hasattr(cls, accessor_name):
+ setattr(cls, accessor_name, VersionedReverseManyToOneDescriptor(related))
+
+ def get_extra_restriction(self, where_class, alias, remote_alias):
+ """
+ Overrides ForeignObject's get_extra_restriction function that returns an SQL statement which is appended to a
+ JOIN's conditional filtering part
+
+ :return: SQL conditional statement
+ :rtype: WhereNode
+ """
+ historic_sql = '''{alias}.version_start_date <= %s
+ AND ({alias}.version_end_date > %s OR {alias}.version_end_date is NULL )'''
+ current_sql = '''{alias}.version_end_date is NULL'''
+ # How 'bout creating an ExtraWhere here, without params
+ return where_class([VersionedExtraWhere(historic_sql=historic_sql, current_sql=current_sql, alias=alias,
+ remote_alias=remote_alias)])
+
+ def get_joining_columns(self, reverse_join=False):
+ """
+ Get and return joining columns defined by this foreign key relationship
+
+ :return: A tuple containing the column names of the tables to be joined (, )
+ :rtype: tuple
+ """
+ source = self.reverse_related_fields if reverse_join else self.related_fields
+ joining_columns = tuple()
+ for lhs_field, rhs_field in source:
+ lhs_col_name = lhs_field.column
+ rhs_col_name = rhs_field.column
+ # Test whether
+ # - self is the current ForeignKey relationship
+ # - self was not auto_created (e.g. is not part of a M2M relationship)
+ if self is lhs_field and not self.auto_created:
+ if rhs_col_name == Versionable.VERSION_IDENTIFIER_FIELD:
+ rhs_col_name = Versionable.OBJECT_IDENTIFIER_FIELD
+ elif self is rhs_field and not self.auto_created:
+ if lhs_col_name == Versionable.VERSION_IDENTIFIER_FIELD:
+ lhs_col_name = Versionable.OBJECT_IDENTIFIER_FIELD
+ joining_columns = joining_columns + ((lhs_col_name, rhs_col_name),)
+ return joining_columns
+
+ def get_reverse_related_filter(self, obj):
+ base_filter = dict()
+ for lh_field, rh_field in self.related_fields:
+ if isinstance(obj, Versionable) and rh_field.attname == Versionable.VERSION_IDENTIFIER_FIELD:
+ base_filter.update(**{Versionable.OBJECT_IDENTIFIER_FIELD: getattr(obj, lh_field.attname)})
+ else:
+ base_filter.update(**{rh_field.attname: getattr(obj, lh_field.attname)})
+ descriptor_filter = self.get_extra_descriptor_filter(obj)
+ base_q = Q(**base_filter)
+ if isinstance(descriptor_filter, dict):
+ return base_q & Q(**descriptor_filter)
+ elif descriptor_filter:
+ return base_q & descriptor_filter
+ return base_q
+ # return super(VersionedForeignKey, self).get_reverse_related_filter(obj)
+
+
+class VersionedManyToManyField(ManyToManyField):
+ def __init__(self, *args, **kwargs):
+ super(VersionedManyToManyField, self).__init__(*args, **kwargs)
+
+ def contribute_to_class(self, cls, name, **kwargs):
+ """
+ Called at class type creation. So, this method is called, when metaclasses get created
+ """
+ # TODO: Apply 3 edge cases when not to create an intermediary model specified in django.db.models.fields.related:1566
+ # self.rel.through needs to be set prior to calling super, since super(...).contribute_to_class refers to it.
+ # Classes pointed to by a string do not need to be resolved here, since Django does that at a later point in
+ # time - which is nice... ;)
+ #
+ # Superclasses take care of:
+ # - creating the through class if unset
+ # - resolving the through class if it's a string
+ # - resolving string references within the through class
+ if not self.remote_field.through and not cls._meta.abstract and not cls._meta.swapped:
+ # We need to anticipate some stuff, that's done only later in class contribution
+ self.set_attributes_from_name(name)
+ self.model = cls
+ self.remote_field.through = VersionedManyToManyField.create_versioned_many_to_many_intermediary_model(self,
+ cls,
+ name)
+ super(VersionedManyToManyField, self).contribute_to_class(cls, name)
+
+ # Overwrite the descriptor
+ if hasattr(cls, self.name):
+ setattr(cls, self.name, VersionedManyToManyDescriptor(self.remote_field))
+
+ def contribute_to_related_class(self, cls, related):
+ """
+ Called at class type creation. So, this method is called, when metaclasses get created
+ """
+ super(VersionedManyToManyField, self).contribute_to_related_class(cls, related)
+ accessor_name = related.get_accessor_name()
+ if accessor_name and hasattr(cls, accessor_name):
+ descriptor = VersionedManyToManyDescriptor(related, accessor_name)
+ setattr(cls, accessor_name, descriptor)
+ if hasattr(cls._meta, 'many_to_many_related') and isinstance(cls._meta.many_to_many_related, list):
+ cls._meta.many_to_many_related.append(descriptor)
+ else:
+ cls._meta.many_to_many_related = [descriptor]
+
+ @staticmethod
+ def create_versioned_many_to_many_intermediary_model(field, cls, field_name):
+ # TODO: Verify functionality against django.db.models.fields.related:1048
+ # Let's not care too much on what flags could potentially be set on that intermediary class (e.g. managed, etc)
+ # Let's play the game, as if the programmer had specified a class within his models... Here's how.
+
+ # FIXME: VersionedManyToManyModels do not get registered in the apps models.
+ # FIXME: This is usually done at django/db/models/base.py:284,
+ # invoked by create_many_to_many_intermediary_model at django.db.models.fields.related:1048
+
+ def set_managed(model, related, through):
+ through._meta.managed = model._meta.managed or related._meta.managed
+
+ to_model = resolve_relation(cls, field.remote_field.model)
+
+ name = '%s_%s' % (cls._meta.object_name, field_name)
+ lazy_related_operation(set_managed, cls, to_model, name)
+
+ # Force 'to' to be a string (and leave the hard work to Django)
+ to = make_model_tuple(to_model)[1]
+ from_ = cls._meta.model_name
+ if to == from_:
+ from_ = 'from_%s' % from_
+ to = 'to_%s' % to
+
+ meta = type('Meta', (object,), {
+ 'db_table': field._get_m2m_db_table(cls._meta),
+ 'auto_created': cls,
+ 'app_label': cls._meta.app_label,
+ 'db_tablespace': cls._meta.db_tablespace,
+ # 'unique_together' is not applicable as is, due to multiple versions to be allowed to exist.
+ # 'unique_together': (from_, to),
+ 'verbose_name': '%(from)s-%(to)s relationship' % {'from': from_, 'to': to},
+ 'verbose_name_plural': '%(from)s-%(to)s relationships' % {'from': from_, 'to': to},
+ 'apps': field.model._meta.apps,
+ })
+ return type(str(name), (Versionable,), {
+ 'Meta': meta,
+ '__module__': cls.__module__,
+ from_: VersionedForeignKey(
+ cls,
+ related_name='%s+' % name,
+ db_tablespace=field.db_tablespace,
+ db_constraint=field.remote_field.db_constraint,
+ auto_created=name,
+ on_delete=DO_NOTHING,
+ ),
+ to: VersionedForeignKey(
+ to_model,
+ related_name='%s+' % name,
+ db_tablespace=field.db_tablespace,
+ db_constraint=field.remote_field.db_constraint,
+ auto_created=name,
+ on_delete=DO_NOTHING,
+ ),
+ })
+
+
+class VersionedExtraWhere(ExtraWhere):
+ """
+ A specific implementation of ExtraWhere;
+ Before as_sql can be called on an object, ensure that calls to
+ - set_as_of and
+ - set_joined_alias
+ have been done
+ """
+
+ def __init__(self, historic_sql, current_sql, alias, remote_alias):
+ super(VersionedExtraWhere, self).__init__(sqls=[], params=[])
+ self.historic_sql = historic_sql
+ self.current_sql = current_sql
+ self.alias = alias
+ self.related_alias = remote_alias
+ self._as_of_time_set = False
+ self.as_of_time = None
+ self._joined_alias = None
+
+ def set_as_of(self, as_of_time):
+ self.as_of_time = as_of_time
+ self._as_of_time_set = True
+
+ def set_joined_alias(self, joined_alias):
+ """
+ Takes the alias that is being joined to the query and applies the query
+ time constraint to its table
+
+ :param str joined_alias: The table name of the alias
+ """
+ self._joined_alias = joined_alias
+
+ def as_sql(self, qn=None, connection=None):
+ sql = ""
+ params = []
+
+ # Fail fast for inacceptable cases
+ if self._as_of_time_set and not self._joined_alias:
+ raise ValueError("joined_alias is not set, but as_of is; this is a conflict!")
+
+ # Set the SQL string in dependency of whether as_of_time was set or not
+ if self._as_of_time_set:
+ if self.as_of_time:
+ sql = self.historic_sql
+ params = [self.as_of_time] * 2
+ # 2 is the number of occurences of the timestamp in an as_of-filter expression
+ else:
+ # If as_of_time was set to None, we're dealing with a query for "current" values
+ sql = self.current_sql
+ else:
+ # No as_of_time has been set; Perhaps, as_of was not part of the query -> That's OK
+ pass
+
+ # By here, the sql string is defined if an as_of_time was provided
+ if self._joined_alias:
+ sql = sql.format(alias=self._joined_alias)
+
+ # Set the final sqls
+ # self.sqls needs to be set before the call to parent
+ if sql:
+ self.sqls = [sql]
+ else:
+ self.sqls = ["1=1"]
+ self.params = params
+ return super(VersionedExtraWhere, self).as_sql(qn, connection)
+
+
+class VersionedWhereNode(WhereNode):
+ def as_sql(self, qn, connection):
+ """
+ This method identifies joined table aliases in order for VersionedExtraWhere.as_sql()
+ to be able to add time restrictions for those tables based on the VersionedQuery's
+ querytime value.
+
+ :param qn: In Django 1.7 & 1.8 this is a compiler; in 1.6, it's an instance-method
+ :param connection: A DB connection
+ :return: A tuple consisting of (sql_string, result_params)
+ """
+ # self.children is an array of VersionedExtraWhere-objects
+ for child in self.children:
+ if isinstance(child, VersionedExtraWhere) and not child.params:
+ # Django 1.7 & 1.8 handles compilers as objects
+ _query = qn.query
+ query_time = _query.querytime.time
+ apply_query_time = _query.querytime.active
+ alias_map = _query.alias_map
+ # In Django 1.8, use the Join objects in alias_map
+ self._set_child_joined_alias(child, alias_map)
+ if apply_query_time:
+ # Add query parameters that have not been added till now
+ child.set_as_of(query_time)
+ else:
+ # Remove the restriction if it's not required
+ child.sqls = []
+ return super(VersionedWhereNode, self).as_sql(qn, connection)
+
+ @staticmethod
+ def _set_child_joined_alias_using_join_map(child, join_map, alias_map):
+ """
+ Set the joined alias on the child, for Django <= 1.7.x.
+ :param child:
+ :param join_map:
+ :param alias_map:
+ """
+ for lhs, table, join_cols in join_map:
+ if lhs is None:
+ continue
+ if lhs == child.alias:
+ relevant_alias = child.related_alias
+ elif lhs == child.related_alias:
+ relevant_alias = child.alias
+ else:
+ continue
+
+ join_info = alias_map[relevant_alias]
+ if join_info.join_type is None:
+ continue
+
+ if join_info.lhs_alias in [child.alias, child.related_alias]:
+ child.set_joined_alias(relevant_alias)
+ break
+
+ @staticmethod
+ def _set_child_joined_alias(child, alias_map):
+ """
+ Set the joined alias on the child, for Django >= 1.8.0
+ :param child:
+ :param alias_map:
+ """
+ for table in alias_map:
+ join = alias_map[table]
+ if not isinstance(join, Join):
+ continue
+ lhs = join.parent_alias
+ if (lhs == child.alias and table == child.related_alias) \
+ or (lhs == child.related_alias and table == child.alias):
+ child.set_joined_alias(table)
+ break
diff --git a/versions/models.py b/versions/models.py
index 490ab5f..8f0ee45 100644
--- a/versions/models.py
+++ b/versions/models.py
@@ -17,29 +17,21 @@
import uuid
from collections import namedtuple
-from django.db.models.sql.datastructures import Join
-from django.apps.registry import apps
from django.core.exceptions import SuspiciousOperation, ObjectDoesNotExist
from django.db import models, router, transaction
-from django.db.models.base import Model
from django.db.models import Q
from django.db.models.constants import LOOKUP_SEP
-from django.db.models.fields import FieldDoesNotExist
-from django.db.models.fields.related import (ForeignKey, ReverseSingleRelatedObjectDescriptor,
- ReverseManyRelatedObjectsDescriptor, ManyToManyField,
- ManyRelatedObjectsDescriptor, create_many_related_manager,
- ForeignRelatedObjectsDescriptor, RECURSIVE_RELATIONSHIP_CONSTANT)
-from django.db.models.query import QuerySet, ValuesListQuerySet, ValuesQuerySet
-from django.db.models.signals import post_init
-from django.db.models.sql import Query
-from django.db.models.sql.where import ExtraWhere, WhereNode
-from django.utils.functional import cached_property
-from django.utils.timezone import utc
+from django.db.models.fields.related import ForeignKey
+from django.db.models.query import QuerySet, ModelIterable
+from django.db.models.sql.datastructures import Join
+from django.db.models.sql.query import Query
+from django.db.models.sql.where import WhereNode
from django.utils import six
+from django.utils.timezone import utc
-from versions.settings import get_versioned_delete_collector_class
-from versions.settings import settings as versions_settings
from versions.exceptions import DeletionOfNonCurrentVersionError
+from versions.settings import get_versioned_delete_collector_class, settings as versions_settings
+from versions.util import get_utc_now
def get_utc_now():
@@ -52,6 +44,7 @@ def validate_uuid(uuid_obj):
"""
return isinstance(uuid_obj, uuid.UUID) and uuid_obj.version == 4
+
QueryTime = namedtuple('QueryTime', 'time active')
@@ -276,6 +269,7 @@ def as_sql(self, qn, connection):
:return: A tuple consisting of (sql_string, result_params)
"""
# self.children is an array of VersionedExtraWhere-objects
+ from versions.fields import VersionedExtraWhere
for child in self.children:
if isinstance(child, VersionedExtraWhere) and not child.params:
_query = qn.query
@@ -309,73 +303,6 @@ def _set_child_joined_alias(child, alias_map):
break
-class VersionedExtraWhere(ExtraWhere):
- """
- A specific implementation of ExtraWhere;
- Before as_sql can be called on an object, ensure that calls to
- - set_as_of and
- - set_joined_alias
- have been done
- """
-
- def __init__(self, historic_sql, current_sql, alias, remote_alias):
- super(VersionedExtraWhere, self).__init__(sqls=[], params=[])
- self.historic_sql = historic_sql
- self.current_sql = current_sql
- self.alias = alias
- self.related_alias = remote_alias
- self._as_of_time_set = False
- self.as_of_time = None
- self._joined_alias = None
-
- def set_as_of(self, as_of_time):
- self.as_of_time = as_of_time
- self._as_of_time_set = True
-
- def set_joined_alias(self, joined_alias):
- """
- Takes the alias that is being joined to the query and applies the query
- time constraint to its table
-
- :param str joined_alias: The table name of the alias
- """
- self._joined_alias = joined_alias
-
- def as_sql(self, qn=None, connection=None):
- sql = ""
- params = []
-
- # Fail fast for inacceptable cases
- if self._as_of_time_set and not self._joined_alias:
- raise ValueError("joined_alias is not set, but as_of is; this is a conflict!")
-
- # Set the SQL string in dependency of whether as_of_time was set or not
- if self._as_of_time_set:
- if self.as_of_time:
- sql = self.historic_sql
- params = [self.as_of_time] * 2
- # 2 is the number of occurences of the timestamp in an as_of-filter expression
- else:
- # If as_of_time was set to None, we're dealing with a query for "current" values
- sql = self.current_sql
- else:
- # No as_of_time has been set; Perhaps, as_of was not part of the query -> That's OK
- pass
-
- # By here, the sql string is defined if an as_of_time was provided
- if self._joined_alias:
- sql = sql.format(alias=self._joined_alias)
-
- # Set the final sqls
- # self.sqls needs to be set before the call to parent
- if sql:
- self.sqls = [sql]
- else:
- self.sqls = ["1=1"]
- self.params = params
- return super(VersionedExtraWhere, self).as_sql(qn, connection)
-
-
class VersionedQuery(Query):
"""
VersionedQuery has awareness of the query time restrictions. When the query is compiled,
@@ -384,6 +311,7 @@ class VersionedQuery(Query):
"""
def __init__(self, *args, **kwargs):
+ from .fields import VersionedWhereNode
kwargs['where'] = VersionedWhereNode
super(VersionedQuery, self).__init__(*args, **kwargs)
self.querytime = QueryTime(time=None, active=False)
@@ -459,6 +387,12 @@ def build_filter(self, filter_expr, **kwargs):
filter_expr = (new_lookup, value.identity)
return super(VersionedQuery, self).build_filter(filter_expr, **kwargs)
+ def add_immediate_loading(self, field_names):
+ # TODO: Decide, whether we always want versionable fields to be loaded, even if ``only`` is used and they would
+ # be deferred
+ # field_names += tuple(Versionable.VERSIONABLE_FIELDS)
+ super(VersionedQuery, self).add_immediate_loading(field_names)
+
class VersionedQuerySet(QuerySet):
"""
@@ -512,7 +446,8 @@ def _fetch_all(self):
"""
if self._result_cache is None:
self._result_cache = list(self.iterator())
- if not isinstance(self, ValuesListQuerySet):
+ # TODO: Do we have to test for ValuesListIterable, ValuesIterable, and FlatValuesListIterable here?
+ if self._iterable_class == ModelIterable:
for x in self._result_cache:
self._set_item_querytime(x)
if self._prefetch_related_lookups and not self._prefetch_done:
@@ -539,11 +474,6 @@ def _set_item_querytime(self, item, type_check=True):
item._querytime = self.querytime
elif isinstance(item, VersionedQuerySet):
item.querytime = self.querytime
- elif isinstance(self, ValuesQuerySet):
- # When we are dealing with a ValueQuerySet there is no point in
- # setting the query_time as we are returning an array of values
- # instead of a full-fledged model object
- pass
else:
if type_check:
raise TypeError("This item is not a Versionable, it's a " + str(type(item)))
@@ -591,618 +521,10 @@ def delete(self):
delete.queryset_only = True
-class VersionedForeignKey(ForeignKey):
- """
- We need to replace the standard ForeignKey declaration in order to be able to introduce
- the VersionedReverseSingleRelatedObjectDescriptor, which allows to go back in time...
- We also want to allow keeping track of any as_of time so that joins can be restricted
- based on that.
- """
-
- def __init__(self, *args, **kwargs):
- super(VersionedForeignKey, self).__init__(*args, **kwargs)
-
- def contribute_to_class(self, cls, name, virtual_only=False):
- super(VersionedForeignKey, self).contribute_to_class(cls, name, virtual_only)
- setattr(cls, self.name, VersionedReverseSingleRelatedObjectDescriptor(self))
-
- def contribute_to_related_class(self, cls, related):
- """
- Override ForeignKey's methods, and replace the descriptor, if set by the parent's methods
- """
- # Internal FK's - i.e., those with a related name ending with '+' -
- # and swapped models don't get a related descriptor.
- super(VersionedForeignKey, self).contribute_to_related_class(cls, related)
- accessor_name = related.get_accessor_name()
- if hasattr(cls, accessor_name):
- setattr(cls, accessor_name, VersionedForeignRelatedObjectsDescriptor(related))
-
- def get_extra_restriction(self, where_class, alias, remote_alias):
- """
- Overrides ForeignObject's get_extra_restriction function that returns an SQL statement which is appended to a
- JOIN's conditional filtering part
-
- :return: SQL conditional statement
- :rtype: WhereNode
- """
- historic_sql = '''{alias}.version_start_date <= %s
- AND ({alias}.version_end_date > %s OR {alias}.version_end_date is NULL )'''
- current_sql = '''{alias}.version_end_date is NULL'''
- # How 'bout creating an ExtraWhere here, without params
- return where_class([VersionedExtraWhere(historic_sql=historic_sql, current_sql=current_sql, alias=alias,
- remote_alias=remote_alias)])
-
- def get_joining_columns(self, reverse_join=False):
- """
- Get and return joining columns defined by this foreign key relationship
-
- :return: A tuple containing the column names of the tables to be joined (, )
- :rtype: tuple
- """
- source = self.reverse_related_fields if reverse_join else self.related_fields
- joining_columns = tuple()
- for lhs_field, rhs_field in source:
- lhs_col_name = lhs_field.column
- rhs_col_name = rhs_field.column
- # Test whether
- # - self is the current ForeignKey relationship
- # - self was not auto_created (e.g. is not part of a M2M relationship)
- if self is lhs_field and not self.auto_created:
- if rhs_col_name == Versionable.VERSION_IDENTIFIER_FIELD:
- rhs_col_name = Versionable.OBJECT_IDENTIFIER_FIELD
- elif self is rhs_field and not self.auto_created:
- if lhs_col_name == Versionable.VERSION_IDENTIFIER_FIELD:
- lhs_col_name = Versionable.OBJECT_IDENTIFIER_FIELD
- joining_columns = joining_columns + ((lhs_col_name, rhs_col_name),)
- return joining_columns
-
-
-class VersionedManyToManyField(ManyToManyField):
- def __init__(self, *args, **kwargs):
- super(VersionedManyToManyField, self).__init__(*args, **kwargs)
-
- def contribute_to_class(self, cls, name):
- """
- Called at class type creation. So, this method is called, when metaclasses get created
- """
- # self.rel.through needs to be set prior to calling super, since super(...).contribute_to_class refers to it.
- # Classes pointed to by a string do not need to be resolved here, since Django does that at a later point in
- # time - which is nice... ;)
- #
- # Superclasses take care of:
- # - creating the through class if unset
- # - resolving the through class if it's a string
- # - resolving string references within the through class
- if not self.rel.through and not cls._meta.abstract and not cls._meta.swapped:
- self.rel.through = VersionedManyToManyField.create_versioned_many_to_many_intermediary_model(self, cls,
- name)
- super(VersionedManyToManyField, self).contribute_to_class(cls, name)
-
- # Overwrite the descriptor
- if hasattr(cls, self.name):
- setattr(cls, self.name, VersionedReverseManyRelatedObjectsDescriptor(self))
-
- def contribute_to_related_class(self, cls, related):
- """
- Called at class type creation. So, this method is called, when metaclasses get created
- """
- super(VersionedManyToManyField, self).contribute_to_related_class(cls, related)
- accessor_name = related.get_accessor_name()
- if accessor_name and hasattr(cls, accessor_name):
- descriptor = VersionedManyRelatedObjectsDescriptor(related, accessor_name)
- setattr(cls, accessor_name, descriptor)
- if hasattr(cls._meta, 'many_to_many_related') and isinstance(cls._meta.many_to_many_related, list):
- cls._meta.many_to_many_related.append(descriptor)
- else:
- cls._meta.many_to_many_related = [descriptor]
-
- @staticmethod
- def create_versioned_many_to_many_intermediary_model(field, cls, field_name):
- # Let's not care too much on what flags could potentially be set on that intermediary class (e.g. managed, etc)
- # Let's play the game, as if the programmer had specified a class within his models... Here's how.
-
- from_ = cls._meta.model_name
- to_model = field.rel.to
-
- # Force 'to' to be a string (and leave the hard work to Django)
- if not isinstance(field.rel.to, six.string_types):
- to_model = '%s.%s' % (field.rel.to._meta.app_label, field.rel.to._meta.object_name)
- to = field.rel.to._meta.object_name.lower()
- else:
- to = to_model.lower()
- name = '%s_%s' % (from_, field_name)
-
- if field.rel.to == RECURSIVE_RELATIONSHIP_CONSTANT or to == cls._meta.object_name:
- from_ = 'from_%s' % to
- to = 'to_%s' % to
- to_model = cls
-
- # Since Django 1.7, a migration mechanism is shipped by default with Django. This migration module loads all
- # declared apps' models inside a __fake__ module.
- # This means that the models can be already loaded and registered by their original module, when we
- # reach this point of the application and therefore there is no need to load them a second time.
- if cls.__module__ == '__fake__':
- try:
- # Check the apps for an already registered model
- return apps.get_registered_model(cls._meta.app_label, str(name))
- except KeyError:
- # The model has not been registered yet, so continue
- pass
-
- meta = type('Meta', (object,), {
- # 'unique_together': (from_, to),
- 'auto_created': cls,
- 'db_tablespace': cls._meta.db_tablespace,
- 'app_label': cls._meta.app_label,
- })
- return type(str(name), (Versionable,), {
- 'Meta': meta,
- '__module__': cls.__module__,
- from_: VersionedForeignKey(cls, related_name='%s+' % name, auto_created=name),
- to: VersionedForeignKey(to_model, related_name='%s+' % name, auto_created=name),
- })
-
-
-class VersionedReverseSingleRelatedObjectDescriptor(ReverseSingleRelatedObjectDescriptor):
- """
- A ReverseSingleRelatedObjectDescriptor-typed object gets inserted, when a ForeignKey
- is defined in a Django model. This is one part of the analogue for versioned items.
-
- Unfortunately, we need to run two queries. The first query satisfies the foreign key
- constraint. After extracting the identity information and combining it with the datetime-
- stamp, we are able to fetch the historic element.
- """
-
- def __get__(self, instance, instance_type=None):
- """
- The getter method returns the object, which points instance, e.g. choice.poll returns
- a Poll instance, whereas the Poll class defines the ForeignKey.
- :param instance: The object on which the property was accessed
- :param instance_type: The type of the instance object
- :return: Returns a Versionable
- """
- current_elt = super(VersionedReverseSingleRelatedObjectDescriptor, self).__get__(instance, instance_type)
-
- if instance is None:
- return self
-
- if not current_elt:
- return None
-
- if not isinstance(current_elt, Versionable):
- raise TypeError("VersionedForeignKey target is of type "
- + str(type(current_elt))
- + ", which is not a subclass of Versionable")
-
- if hasattr(instance, '_querytime'):
- # If current_elt matches the instance's querytime, there's no need to make a database query.
- if Versionable.matches_querytime(current_elt, instance._querytime):
- current_elt._querytime = instance._querytime
- return current_elt
-
- return current_elt.__class__.objects.as_of(instance._querytime.time).get(identity=current_elt.identity)
- else:
- return current_elt.__class__.objects.current.get(identity=current_elt.identity)
-
- def get_prefetch_queryset(self, instances, queryset=None):
- """
- Overrides the parent method to:
- - force queryset to use the querytime of the parent objects
- - ensure that the join is done on identity, not id
- - make the cache key identity, not id.
- """
- if queryset is None:
- queryset = self.get_queryset()
- queryset._add_hints(instance=instances[0])
-
- # CleanerVersion change 1: force the querytime to be the same as the prefetched-for instance.
- # This is necessary to have reliable results and avoid extra queries for cache misses when
- # accessing the child objects from their parents (e.g. choice.poll).
- instance_querytime = instances[0]._querytime
- if instance_querytime.active:
- if queryset.querytime.active and queryset.querytime.time != instance_querytime.time:
- raise ValueError("A Prefetch queryset that specifies an as_of time must match "
- "the as_of of the base queryset.")
- else:
- queryset.querytime = instance_querytime
-
- # CleanerVersion change 2: make rel_obj_attr return a tuple with the object's identity.
- # rel_obj_attr = self.field.get_foreign_related_value
- def versioned_fk_rel_obj_attr(versioned_rel_obj):
- return versioned_rel_obj.identity,
- rel_obj_attr = versioned_fk_rel_obj_attr
- instance_attr = self.field.get_local_related_value
- instances_dict = {instance_attr(inst): inst for inst in instances}
- # CleanerVersion change 3: fake the related field so that it provides a name of 'identity'.
- # related_field = self.field.foreign_related_fields[0]
- related_field = namedtuple('VersionedRelatedFieldTuple', 'name')('identity')
-
- # FIXME: This will need to be revisited when we introduce support for
- # composite fields. In the meantime we take this practical approach to
- # solve a regression on 1.6 when the reverse manager in hidden
- # (related_name ends with a '+'). Refs #21410.
- # The check for len(...) == 1 is a special case that allows the query
- # to be join-less and smaller. Refs #21760.
- if self.field.rel.is_hidden() or len(self.field.foreign_related_fields) == 1:
- query = {'%s__in' % related_field.name: set(instance_attr(inst)[0] for inst in instances)}
- # query = {'identity__in': set(instance_attr(inst)[0] for inst in instances)}
- else:
- query = {'%s__in' % self.field.related_query_name(): instances}
- queryset = queryset.filter(**query)
-
- # Since we're going to assign directly in the cache,
- # we must manage the reverse relation cache manually.
- if not self.field.rel.multiple:
- rel_obj_cache_name = self.field.rel.get_cache_name()
- for rel_obj in queryset:
- instance = instances_dict[rel_obj_attr(rel_obj)]
- setattr(rel_obj, rel_obj_cache_name, instance)
- return queryset, rel_obj_attr, instance_attr, True, self.cache_name
-
-
-class VersionedForeignRelatedObjectsDescriptor(ForeignRelatedObjectsDescriptor):
- """
- This descriptor generates the manager class that is used on the related object of a ForeignKey relation
- (i.e. the reverse-ForeignKey field manager).
- """
-
- @cached_property
- def related_manager_cls(self):
- # return create_versioned_related_manager
- manager_cls = super(VersionedForeignRelatedObjectsDescriptor, self).related_manager_cls
- rel_field = self.related.field
-
- class VersionedRelatedManager(manager_cls):
- def __init__(self, instance):
- super(VersionedRelatedManager, self).__init__(instance)
-
- # This is a hack, in order to get the versioned related objects
- for key in self.core_filters.keys():
- if '__exact' in key or '__' not in key:
- self.core_filters[key] = instance.identity
-
- def get_queryset(self):
- queryset = super(VersionedRelatedManager, self).get_queryset()
- # Do not set the query time if it is already correctly set. queryset.as_of() returns a clone
- # of the queryset, and this will destroy the prefetched objects cache if it exists.
- if isinstance(queryset,
- VersionedQuerySet) and self.instance._querytime.active and queryset.querytime != self.instance._querytime:
- queryset = queryset.as_of(self.instance._querytime.time)
- return queryset
-
- def get_prefetch_queryset(self, instances, queryset=None):
- """
- Overrides RelatedManager's implementation of get_prefetch_queryset so that it works
- nicely with VersionedQuerySets. It ensures that identities and time-limited where
- clauses are used when selecting related reverse foreign key objects.
- """
- if queryset is None:
- # Note that this intentionally call's VersionManager's get_queryset, instead of simply calling
- # the superclasses' get_queryset (as the non-versioned RelatedManager does), because what is
- # needed is a simple Versioned queryset without any restrictions (e.g. do not
- # apply self.core_filters).
- queryset = VersionManager.get_queryset(self)
-
- queryset._add_hints(instance=instances[0])
- queryset = queryset.using(queryset._db or self._db)
- instance_querytime = instances[0]._querytime
- if instance_querytime.active:
- if queryset.querytime.active and queryset.querytime.time != instance_querytime.time:
- raise ValueError("A Prefetch queryset that specifies an as_of time must match "
- "the as_of of the base queryset.")
- else:
- queryset.querytime = instance_querytime
-
- rel_obj_attr = rel_field.get_local_related_value
- instance_attr = rel_field.get_foreign_related_value
- # Use identities instead of ids so that this will work with versioned objects.
- instances_dict = {(inst.identity,): inst for inst in instances}
- identities = [inst.identity for inst in instances]
- query = {'%s__identity__in' % rel_field.name: identities}
- queryset = queryset.filter(**query)
-
- # Since we just bypassed this class' get_queryset(), we must manage
- # the reverse relation manually.
- for rel_obj in queryset:
- instance = instances_dict[rel_obj_attr(rel_obj)]
- setattr(rel_obj, rel_field.name, instance)
- cache_name = rel_field.related_query_name()
- return queryset, rel_obj_attr, instance_attr, False, cache_name
-
- def add(self, *objs):
- cloned_objs = ()
- for obj in objs:
- if not isinstance(obj, Versionable):
- raise TypeError("Trying to add a non-Versionable to a VersionedForeignKey relationship")
- cloned_objs += (obj.clone(),)
- super(VersionedRelatedManager, self).add(*cloned_objs)
-
- # clear() and remove() are present if the FK is nullable
- if 'clear' in dir(manager_cls):
- def clear(self, **kwargs):
- """
- Overridden to ensure that the current queryset is used, and to clone objects before they
- are removed, so that history is not lost.
- """
- bulk = kwargs.pop('bulk', True)
- db = router.db_for_write(self.model, instance=self.instance)
- queryset = self.current.using(db)
- with transaction.atomic(using=db, savepoint=False):
- cloned_pks = [obj.clone().pk for obj in queryset]
- update_qs = self.current.filter(pk__in=cloned_pks)
- self._clear(update_qs, bulk)
-
- if 'remove' in dir(manager_cls):
- def remove(self, *objs):
- val = rel_field.get_foreign_related_value(self.instance)
- cloned_objs = ()
- for obj in objs:
- # Is obj actually part of this descriptor set? Otherwise, silently go over it, since Django
- # handles that case
- if rel_field.get_local_related_value(obj) == val:
- # Silently pass over non-versionable items
- if not isinstance(obj, Versionable):
- raise TypeError(
- "Trying to remove a non-Versionable from a VersionedForeignKey realtionship")
- cloned_objs += (obj.clone(),)
- super(VersionedRelatedManager, self).remove(*cloned_objs)
-
- return VersionedRelatedManager
-
-
-def create_versioned_many_related_manager(superclass, rel):
- """
- The "casting" which is done in this method is needed, since otherwise, the methods introduced by
- Versionable are not taken into account.
- :param superclass: This is usually a models.Manager
- :param rel: Contains the ManyToMany relation
- :return: A subclass of ManyRelatedManager and Versionable
- """
- many_related_manager_klass = create_many_related_manager(superclass, rel)
-
- class VersionedManyRelatedManager(many_related_manager_klass):
- def __init__(self, *args, **kwargs):
- super(VersionedManyRelatedManager, self).__init__(*args, **kwargs)
- try:
- _ = self.through._meta.get_field('version_start_date')
- _ = self.through._meta.get_field('version_end_date')
- except FieldDoesNotExist as e:
- fields = [f.name for f in self.through._meta.get_fields()]
- print(str(e) + "; available fields are " + ", ".join(fields))
- raise e
- # FIXME: this probably does not work when auto-referencing
-
- def get_queryset(self):
- """
- Add a filter to the queryset, limiting the results to be pointed by relationship that are
- valid for the given timestamp (which is taken at the current instance, or set to now, if not
- available).
- Long story short, apply the temporal validity filter also to the intermediary model.
- """
-
- queryset = super(VersionedManyRelatedManager, self).get_queryset()
- if hasattr(queryset, 'querytime'):
- if self.instance._querytime.active and self.instance._querytime != queryset.querytime:
- queryset = queryset.as_of(self.instance._querytime.time)
- return queryset
-
- def _remove_items(self, source_field_name, target_field_name, *objs):
- """
- Instead of removing items, we simply set the version_end_date of the current item to the
- current timestamp --> t[now].
- Like that, there is no more current entry having that identity - which is equal to
- not existing for timestamps greater than t[now].
- """
- return self._remove_items_at(None, source_field_name, target_field_name, *objs)
-
- def _remove_items_at(self, timestamp, source_field_name, target_field_name, *objs):
- if objs:
- if timestamp is None:
- timestamp = get_utc_now()
- old_ids = set()
- for obj in objs:
- if isinstance(obj, self.model):
- if hasattr(self, 'target_field'):
- fk_val = self.target_field.get_foreign_related_value(obj)[0]
- else:
- raise TypeError("We couldn't find the value of the foreign key, this might be due to the "
- "use of an unsupported version of Django")
- old_ids.add(fk_val)
- else:
- old_ids.add(obj)
- db = router.db_for_write(self.through, instance=self.instance)
- qs = self.through._default_manager.using(db).filter(**{
- source_field_name: self.instance.id,
- '%s__in' % target_field_name: old_ids
- }).as_of(timestamp)
- for relation in qs:
- relation._delete_at(timestamp)
-
- if 'add' in dir(many_related_manager_klass):
- def add(self, *objs):
- if not self.instance.is_current:
- raise SuspiciousOperation(
- "Adding many-to-many related objects is only possible on the current version")
-
- # The ManyRelatedManager.add() method uses the through model's default manager to get
- # a queryset when looking at which objects already exist in the database.
- # In order to restrict the query to the current versions when that is done,
- # we temporarily replace the queryset's using method so that the version validity
- # condition can be specified.
- klass = self.through._default_manager.get_queryset().__class__
- __using_backup = klass.using
-
- def using_replacement(self, *args, **kwargs):
- qs = __using_backup(self, *args, **kwargs)
- return qs.as_of(None)
-
- klass.using = using_replacement
- super(VersionedManyRelatedManager, self).add(*objs)
- klass.using = __using_backup
-
- def add_at(self, timestamp, *objs):
- """
- This function adds an object at a certain point in time (timestamp)
- """
- # First off, define the new constructor
- def _through_init(self, *args, **kwargs):
- super(self.__class__, self).__init__(*args, **kwargs)
- self.version_birth_date = timestamp
- self.version_start_date = timestamp
-
- # Through-classes have an empty constructor, so it can easily be overwritten when needed;
- # This is not the default case, so the overwrite only takes place when we "modify the past"
- self.through.__init_backup__ = self.through.__init__
- self.through.__init__ = _through_init
-
- # Do the add operation
- self.add(*objs)
-
- # Remove the constructor again (by replacing it with the original empty constructor)
- self.through.__init__ = self.through.__init_backup__
- del self.through.__init_backup__
-
- add_at.alters_data = True
-
- if 'remove' in dir(many_related_manager_klass):
- def remove_at(self, timestamp, *objs):
- """
- Performs the act of removing specified relationships at a specified time (timestamp);
- So, not the objects at a given time are removed, but their relationship!
- """
- self._remove_items_at(timestamp, self.source_field_name, self.target_field_name, *objs)
-
- # For consistency, also handle the symmetrical case
- if self.symmetrical:
- self._remove_items_at(timestamp, self.target_field_name, self.source_field_name, *objs)
-
- remove_at.alters_data = True
-
- return VersionedManyRelatedManager
-
-
-class VersionedReverseManyRelatedObjectsDescriptor(ReverseManyRelatedObjectsDescriptor):
- """
- Beside having a very long name, this class is useful when it comes to versioning the
- ReverseManyRelatedObjectsDescriptor (huhu!!). The main part is the exposure of the
- 'related_manager_cls' property
- """
-
- def __get__(self, instance, owner=None):
- """
- Reads the property as which this object is figuring; mainly used for debugging purposes
- :param instance: The instance on which the getter was called
- :param owner: no idea... alternatively called 'instance_type by the superclasses
- :return: A VersionedManyRelatedManager object
- """
- return super(VersionedReverseManyRelatedObjectsDescriptor, self).__get__(instance, owner)
-
- def __set__(self, instance, value):
- """
- Completely overridden to avoid bulk deletion that happens when the parent method calls clear().
-
- The parent method's logic is basically: clear all in bulk, then add the given objects in bulk.
- Instead, we figure out which ones are being added and removed, and call add and remove for these values.
- This lets us retain the versioning information.
-
- Since this is a many-to-many relationship, it is assumed here that the django.db.models.deletion.Collector
- logic, that is used in clear(), is not necessary here. Collector collects related models, e.g. ones that should
- also be deleted because they have a ON CASCADE DELETE relationship to the object, or, in the case of
- "Multi-table inheritance", are parent objects.
-
- :param instance: The instance on which the getter was called
- :param value: iterable of items to set
- """
-
- if not instance.is_current:
- raise SuspiciousOperation(
- "Related values can only be directly set on the current version of an object")
-
- if not self.field.rel.through._meta.auto_created:
- opts = self.field.rel.through._meta
- raise AttributeError(("Cannot set values on a ManyToManyField which specifies an intermediary model. "
- "Use %s.%s's Manager instead.") % (opts.app_label, opts.object_name))
-
- manager = self.__get__(instance)
- # Below comment is from parent __set__ method. We'll force evaluation, too:
- # clear() can change expected output of 'value' queryset, we force evaluation
- # of queryset before clear; ticket #19816
- value = tuple(value)
-
- being_removed, being_added = self.get_current_m2m_diff(instance, value)
- timestamp = get_utc_now()
- manager.remove_at(timestamp, *being_removed)
- manager.add_at(timestamp, *being_added)
-
- def get_current_m2m_diff(self, instance, new_objects):
- """
- :param instance: Versionable object
- :param new_objects: objects which are about to be associated with instance
- :return: (being_removed id list, being_added id list)
- :rtype : tuple
- """
- new_ids = self.pks_from_objects(new_objects)
- relation_manager = self.__get__(instance)
-
- filter = Q(**{relation_manager.source_field.attname: instance.pk})
- qs = self.through.objects.current.filter(filter)
- target_name = relation_manager.target_field.attname
- current_ids = set(qs.values_list(target_name, flat=True))
-
- being_removed = current_ids - new_ids
- being_added = new_ids - current_ids
- return list(being_removed), list(being_added)
-
- def pks_from_objects(self, objects):
- """
- Extract all the primary key strings from the given objects. Objects may be Versionables, or bare primary keys.
- :rtype : set
- """
- return {o.pk if isinstance(o, Model) else o for o in objects}
-
- @cached_property
- def related_manager_cls(self):
- return create_versioned_many_related_manager(
- self.field.rel.to._default_manager.__class__,
- self.field.rel
- )
-
-
-class VersionedManyRelatedObjectsDescriptor(ManyRelatedObjectsDescriptor):
- """
- Beside having a very long name, this class is useful when it comes to versioning the
- ManyRelatedObjectsDescriptor (huhu!!). The main part is the exposure of the
- 'related_manager_cls' property
- """
-
- via_field_name = None
-
- def __init__(self, related, via_field_name):
- super(VersionedManyRelatedObjectsDescriptor, self).__init__(related)
- self.via_field_name = via_field_name
-
- def __get__(self, instance, owner=None):
- """
- Reads the property as which this object is figuring; mainly used for debugging purposes
- :param instance: The instance on which the getter was called
- :param owner: no idea... alternatively called 'instance_type by the superclasses
- :return: A VersionedManyRelatedManager object
- """
- return super(VersionedManyRelatedObjectsDescriptor, self).__get__(instance, owner)
-
- @cached_property
- def related_manager_cls(self):
- return create_versioned_many_related_manager(
- self.related.model._default_manager.__class__,
- self.related.field.rel
- )
-
-
class Versionable(models.Model):
"""
This is pretty much the central point for versioning objects.
"""
-
VERSION_IDENTIFIER_FIELD = 'id'
OBJECT_IDENTIFIER_FIELD = 'identity'
VERSIONABLE_FIELDS = [VERSION_IDENTIFIER_FIELD, OBJECT_IDENTIFIER_FIELD, 'version_start_date',
@@ -1219,6 +541,7 @@ class Versionable(models.Model):
"""identity is used as the identifier of an object, ignoring its versions; sometimes also referenced as the natural key"""
else:
identity = models.CharField(max_length=36)
+ """identity is used as the identifier of an object, ignoring its versions; sometimes also referenced as the natural key"""
version_start_date = models.DateTimeField()
"""version_start_date points the moment in time, when a version was created (ie. an versionable was cloned).
@@ -1263,7 +586,7 @@ def __init__(self, *args, **kwargs):
if not getattr(self, self.OBJECT_IDENTIFIER_FIELD, None):
setattr(self, self.OBJECT_IDENTIFIER_FIELD, getattr(self, self.VERSION_IDENTIFIER_FIELD))
- def delete(self, using=None):
+ def delete(self, using=None, keep_parents=False):
using = using or router.db_for_write(self.__class__, instance=self)
assert self._get_pk_val() is not None, \
"{} object can't be deleted because its {} attribute is set to None.".format(
@@ -1271,7 +594,7 @@ def delete(self, using=None):
collector_class = get_versioned_delete_collector_class()
collector = collector_class(using=using)
- collector.collect([self])
+ collector.collect([self], keep_parents=keep_parents)
collector.delete(get_utc_now())
def _delete_at(self, timestamp, using=None):
@@ -1514,8 +837,12 @@ def restore(self, **kwargs):
# Set all non-provided ForeignKeys to None. If required, raise an error.
try:
setattr(restored, field.name, None)
- except ValueError as e:
- raise ForeignKeyRequiresValueError(e.args[0])
+ # Check for non null foreign key removed since Django 1.10
+ # https://docs.djangoproject.com/en/1.10/releases/1.10/#removed-null-assignment-check-for-non-null-foreign-key-fields
+ if not field.null:
+ raise ValueError
+ except ValueError:
+ raise ForeignKeyRequiresValueError
self.id = self.uuid()
@@ -1541,7 +868,7 @@ def get_all_m2m_field_names(self):
opts = self._meta
rel_field_names = [field.attname for field in opts.many_to_many]
if hasattr(opts, 'many_to_many_related'):
- rel_field_names += [rel.via_field_name for rel in opts.many_to_many_related]
+ rel_field_names += [rel.reverse for rel in opts.many_to_many_related]
return rel_field_names
diff --git a/versions/util/__init__.py b/versions/util/__init__.py
index e69de29..5f67c8e 100644
--- a/versions/util/__init__.py
+++ b/versions/util/__init__.py
@@ -0,0 +1,7 @@
+import datetime
+
+from django.utils.timezone import utc
+
+
+def get_utc_now():
+ return datetime.datetime.utcnow().replace(tzinfo=utc)
diff --git a/versions/util/postgresql.py b/versions/util/postgresql.py
index 9236124..66b9c77 100644
--- a/versions/util/postgresql.py
+++ b/versions/util/postgresql.py
@@ -1,6 +1,6 @@
from __future__ import absolute_import
from django.db import connection as default_connection
-from versions.models import VersionedForeignKey
+from versions.fields import VersionedForeignKey
from .helper import database_connection, versionable_models
diff --git a/versions_tests/models.py b/versions_tests/models.py
index b4c3f2b..0bbbc47 100644
--- a/versions_tests/models.py
+++ b/versions_tests/models.py
@@ -1,8 +1,10 @@
from django.db.models import CharField, IntegerField, Model, ForeignKey
from django.db.models.deletion import DO_NOTHING, PROTECT, SET, SET_NULL
+from django.db.models.fields.related import ManyToManyField
from django.utils.encoding import python_2_unicode_compatible
-from versions.models import Versionable, VersionedManyToManyField, VersionedForeignKey
+from versions.models import Versionable
+from versions.fields import VersionedManyToManyField, VersionedForeignKey
def versionable_description(obj):
@@ -265,4 +267,4 @@ def __str__(self):
# SelfReferencingManyToManyTest models
class Person(Versionable):
name = CharField(max_length=200)
- children = VersionedManyToManyField('self', symmetrical=False, null=True, related_name='parents')
+ children = VersionedManyToManyField('self', symmetrical=False, related_name='parents')
diff --git a/versions_tests/tests/test_commands.py b/versions_tests/tests/test_commands.py
index ff202d2..011ac2d 100644
--- a/versions_tests/tests/test_commands.py
+++ b/versions_tests/tests/test_commands.py
@@ -1,10 +1,9 @@
-import django
from django.core.management import call_command
from django.test import TestCase
APP_NAME = 'versions_tests'
-if django.VERSION[:2] >= (1, 7):
- class TestMigrations(TestCase):
- def test_makemigrations_command(self):
- call_command('makemigrations', APP_NAME, dry_run=True, verbosity=0)
+
+class TestMigrations(TestCase):
+ def test_makemigrations_command(self):
+ call_command('makemigrations', APP_NAME, dry_run=True, verbosity=0)
diff --git a/versions_tests/tests/test_models.py b/versions_tests/tests/test_models.py
index 79becef..dfe7181 100644
--- a/versions_tests/tests/test_models.py
+++ b/versions_tests/tests/test_models.py
@@ -12,6 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import unicode_literals
+
import datetime
from time import sleep
import itertools
@@ -27,7 +29,6 @@
from django.test import TestCase
from django.utils.timezone import utc
from django.utils import six
-from django import VERSION
from versions.exceptions import DeletionOfNonCurrentVersionError
from versions.models import get_utc_now, ForeignKeyRequiresValueError, Versionable
@@ -38,12 +39,8 @@
def get_relation_table(model_class, fieldname):
-
- if VERSION[:2] >= (1, 8):
- field_object = model_class._meta.get_field(fieldname)
- direct = not field_object.auto_created or field_object.concrete
- else:
- field_object, _, direct, _ = model_class._meta.get_field_by_name(fieldname)
+ field_object = model_class._meta.get_field(fieldname)
+ direct = not field_object.auto_created or field_object.concrete
if direct:
field = field_object
@@ -55,21 +52,26 @@ def get_relation_table(model_class, fieldname):
def set_up_one_object_with_3_versions():
b = B.objects.create(name='v1')
- sleep(0.1)
+ sleep(0.001)
t1 = get_utc_now()
+ # 1ms sleeps are required, since sqlite has a 1ms precision in its datetime stamps
+ # not inserting the sleep would make t1 point to the next version's start date, which would be wrong
+ sleep(0.001)
b = b.clone()
b.name = 'v2'
b.save()
- sleep(0.1)
+ sleep(0.001)
t2 = get_utc_now()
+ # 1ms sleeps are required, since sqlite has a 1ms precision in its datetime stamps
+ sleep(0.001)
b = b.clone()
b.name = 'v3'
b.save()
- sleep(0.1)
+ sleep(0.001)
t3 = get_utc_now()
return b, t1, t2, t3
@@ -1959,7 +1961,7 @@ def setUp(self):
self.c1.team_set = []
self.team10 = Team.objects.current.get(identity=self.team10.identity).clone()
- self.c10.team_set = []
+ self.c10.team_set.clear()
self.t3 = get_utc_now()
def test_t1_relations_for_cloned_referenced_object(self):
@@ -2186,6 +2188,12 @@ def test_prefetch_related_via_many_to_many(self):
p.awards.remove(p.awards.all()[0])
name_list.append(p.name)
+ with self.assertNumQueries(2):
+ old_award_players = list(
+ Player.objects.as_of(t2).prefetch_related('awards').filter(
+ name__in=name_list).order_by('name')
+ )
+
with self.assertNumQueries(2):
updated_award_players = list(
Player.objects.current.prefetch_related('awards').filter(
@@ -2194,7 +2202,7 @@ def test_prefetch_related_via_many_to_many(self):
with self.assertNumQueries(0):
for i in range(len(award_players)):
- old = len(award_players[i].awards.all())
+ old = len(old_award_players[i].awards.all())
new = len(updated_award_players[i].awards.all())
self.assertTrue(new == old - 1)
@@ -2318,13 +2326,13 @@ def test_reverse_fk_prefetch_queryset_with_historic_versions(self):
_ = City.objects.current.filter(name='city.v2').prefetch_related(
Prefetch(
'team_set',
- queryset=Team.objects.as_of(self.time1),
- to_attr='prefetched_teams'
+ queryset = Team.objects.as_of(self.time1),
+ to_attr = 'prefetched_teams'
),
Prefetch(
'prefetched_teams__player_set',
- queryset=Player.objects.as_of(self.time1),
- to_attr='prefetched_players'
+ queryset = Player.objects.as_of(self.time1),
+ to_attr = 'prefetched_players'
),
)[0]
@@ -2582,7 +2590,7 @@ def test_create_with_uuid(self):
self.assertEqual(str(p_id), str(p.id))
self.assertEqual(str(p_id), str(p.identity))
- p_id = uuid.uuid5(uuid.NAMESPACE_OID, 'bar')
+ p_id = uuid.uuid5(uuid.NAMESPACE_OID, str('bar'))
with self.assertRaises(ValueError):
Person.objects.create(id=p_id, name="Alexis")
@@ -2634,10 +2642,13 @@ def setup_common(self):
def test_restore_latest_version(self):
self.setup_common()
+ sleep(0.001)
self.player1.delete()
+ sleep(0.001)
deleted_at = self.player1.version_end_date
player1_pk = self.player1.pk
+ sleep(0.001)
restored = self.player1.restore()
self.assertEqual(player1_pk, restored.pk)
self.assertIsNone(restored.version_end_date)
@@ -2849,7 +2860,6 @@ def test_simple_defer(self):
self.assertEquals(self.c1.version_start_date, deferred.version_start_date)
def test_deferred_foreign_key_field(self):
-
team_full = Team.objects.current.get(pk=self.team1.pk)
self.assertIn('city_id', team_full.__dict__ )
team_light = Team.objects.current.only('name').get(pk=self.team1.pk)
diff --git a/versions_tests/urls.py b/versions_tests/urls.py
deleted file mode 100644
index 93005f9..0000000
--- a/versions_tests/urls.py
+++ /dev/null
@@ -1,4 +0,0 @@
-from versions_tests.urls import urlpatterns
-
-
-urlpatterns = urlpatterns