From fa490205416a400453621d0606c139b6e4033d97 Mon Sep 17 00:00:00 2001 From: Cryptophobia Date: Sat, 20 Mar 2021 13:53:51 -0400 Subject: [PATCH] feat(django): upgrade django framework and deps - upgrading django to 2.2.19 and djangorestframework to 3.12.2 - resolving all failing tests due to above upgrade - upgrade models to default `on_delete=models.CASCADE` if app or user are deleted - upgrade all other dependencies to latest versions - update Makefile to use hephy/shell-dev for tests - update Dockerfile to use `hephy/base:latest` for building - update Dockefile to use python3 -m pip commands - upgrade pip, setuptools, wheel using python3 command - resolves #140 - resolves #80 Signed-off-by: Cryptophobia --- Makefile | 2 +- rootfs/Dockerfile | 9 ++++-- rootfs/Dockerfile.test | 19 +++++++----- rootfs/api/migrations/0001_initial.py | 36 +++++++++++------------ rootfs/api/models/certificate.py | 5 ++-- rootfs/dev_requirements.txt | 8 ++--- rootfs/requirements.txt | 42 +++++++++++++-------------- 7 files changed, 65 insertions(+), 56 deletions(-) diff --git a/Makefile b/Makefile index e8b8c6f8d..6ddd64a6b 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ SHORT_NAME ?= $(COMPONENT) include versioning.mk -SHELLCHECK_PREFIX := docker run -v ${CURDIR}:/workdir -w /workdir quay.io/deis/shell-dev shellcheck +SHELLCHECK_PREFIX := docker run -v ${CURDIR}:/workdir -w /workdir hephy/shell-dev shellcheck SHELL_SCRIPTS = $(wildcard rootfs/bin/*) $(shell find "rootfs" -name '*.sh') $(wildcard _scripts/*.sh) # Test processes used in quick unit testing diff --git a/rootfs/Dockerfile b/rootfs/Dockerfile index 3fa6d5a3e..92fe53001 100644 --- a/rootfs/Dockerfile +++ b/rootfs/Dockerfile @@ -1,4 +1,4 @@ -FROM hephy/base:v0.4.1 +FROM hephy/base:latest RUN adduser --system \ --shell /bin/bash \ @@ -21,8 +21,13 @@ RUN buildDeps='gcc libffi-dev libpq-dev libldap2-dev libsasl2-dev python3-dev py # cryptography package needs pkg_resources python3-pkg-resources && \ ln -s /usr/bin/python3 /usr/bin/python && \ + # use python3 to upgrade pip and tools + python3 -m pip install --upgrade --no-cache-dir \ + pip \ + setuptools \ + wheel && \ mkdir -p /configs && chown -R deis:deis /configs && \ - pip3 install --disable-pip-version-check --no-cache-dir -r /app/requirements.txt && \ + python3 -m pip install --disable-pip-version-check --no-cache-dir -r /app/requirements.txt && \ # cleanup apt-get purge -y --auto-remove $buildDeps && \ apt-get autoremove -y && \ diff --git a/rootfs/Dockerfile.test b/rootfs/Dockerfile.test index c29478f23..3d51af560 100644 --- a/rootfs/Dockerfile.test +++ b/rootfs/Dockerfile.test @@ -1,4 +1,4 @@ -FROM hephy/base:v0.4.1 +FROM hephy/base:latest RUN adduser --system \ --shell /bin/bash \ @@ -18,11 +18,17 @@ RUN buildDeps='gcc libffi-dev libpq-dev libldap2-dev libsasl2-dev python3-dev py libpq5 \ libldap-2.4 \ python3-minimal \ - # cryptography package needs pkg_resources - python3-pkg-resources && \ + python3-distutils \ + # cryptography package needs pkg_resources + python3-pkg-resources && \ ln -s /usr/bin/python3 /usr/bin/python && \ + # use python3 to upgrade pip and tools + python3 -m pip install --upgrade --no-cache-dir \ + pip \ + setuptools \ + wheel && \ mkdir -p /configs && chown -R deis:deis /configs && \ - pip3 install --disable-pip-version-check --no-cache-dir -r /app/requirements.txt && \ + python3 -m pip install --disable-pip-version-check --no-cache-dir -r /app/requirements.txt && \ # cleanup apt-get purge -y --auto-remove $buildDeps && \ apt-get autoremove -y && \ @@ -54,10 +60,7 @@ RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ git \ postgresql \ - postgresql-contrib \ - python3-pip \ - python3-setuptools \ - python3-wheel && \ + postgresql-contrib && \ pip3 install --disable-pip-version-check --no-cache-dir -r dev_requirements.txt && \ sudo -u postgres -E $PGBIN/initdb diff --git a/rootfs/api/migrations/0001_initial.py b/rootfs/api/migrations/0001_initial.py index 1ae2dd8fd..2aaad50b4 100644 --- a/rootfs/api/migrations/0001_initial.py +++ b/rootfs/api/migrations/0001_initial.py @@ -22,7 +22,7 @@ class Migration(migrations.Migration): ('updated', models.DateTimeField(auto_now=True)), ('id', models.SlugField(max_length=24, unique=True, null=True, validators=[api.models.validate_app_id, api.models.validate_reserved_names])), ('structure', jsonfield.fields.JSONField(default={}, blank=True, validators=[api.models.validate_app_structure])), - ('owner', models.ForeignKey(to=settings.AUTH_USER_MODEL)), + ('owner', models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)), ], options={ 'permissions': (('use_app', 'Can use app'),), @@ -39,8 +39,8 @@ class Migration(migrations.Migration): ('sha', models.CharField(max_length=40, blank=True)), ('procfile', jsonfield.fields.JSONField(default={}, blank=True)), ('dockerfile', models.TextField(blank=True)), - ('app', models.ForeignKey(to='api.App')), - ('owner', models.ForeignKey(to=settings.AUTH_USER_MODEL)), + ('app', models.ForeignKey(to='api.App', on_delete=models.CASCADE)), + ('owner', models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)), ], options={ 'ordering': ['-created'], @@ -58,7 +58,7 @@ class Migration(migrations.Migration): ('key', models.TextField()), ('common_name', models.TextField(unique=True)), ('expires', models.DateTimeField()), - ('owner', models.ForeignKey(to=settings.AUTH_USER_MODEL)), + ('owner', models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)), ], options={ 'abstract': False, @@ -75,8 +75,8 @@ class Migration(migrations.Migration): ('memory', jsonfield.fields.JSONField(default={}, blank=True)), ('cpu', jsonfield.fields.JSONField(default={}, blank=True)), ('tags', jsonfield.fields.JSONField(default={}, blank=True)), - ('app', models.ForeignKey(to='api.App')), - ('owner', models.ForeignKey(to=settings.AUTH_USER_MODEL)), + ('app', models.ForeignKey(to='api.App', on_delete=models.CASCADE)), + ('owner', models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)), ], options={ 'ordering': ['-created'], @@ -92,8 +92,8 @@ class Migration(migrations.Migration): ('updated', models.DateTimeField(auto_now=True)), ('type', models.CharField(max_length=128)), ('num', models.PositiveIntegerField()), - ('app', models.ForeignKey(to='api.App')), - ('owner', models.ForeignKey(to=settings.AUTH_USER_MODEL)), + ('app', models.ForeignKey(to='api.App', on_delete=models.CASCADE)), + ('owner', models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)), ], options={ 'ordering': ['created'], @@ -108,8 +108,8 @@ class Migration(migrations.Migration): ('created', models.DateTimeField(auto_now_add=True)), ('updated', models.DateTimeField(auto_now=True)), ('domain', models.TextField(unique=True)), - ('app', models.ForeignKey(to='api.App')), - ('owner', models.ForeignKey(to=settings.AUTH_USER_MODEL)), + ('app', models.ForeignKey(to='api.App', on_delete=models.CASCADE)), + ('owner', models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)), ], options={ 'abstract': False, @@ -125,7 +125,7 @@ class Migration(migrations.Migration): ('id', models.CharField(max_length=128)), ('public', models.TextField(unique=True, validators=[api.models.validate_base64])), ('fingerprint', models.CharField(max_length=128)), - ('owner', models.ForeignKey(to=settings.AUTH_USER_MODEL)), + ('owner', models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)), ], options={ 'verbose_name': 'SSH Key', @@ -144,8 +144,8 @@ class Migration(migrations.Migration): ('receive_repo', models.CharField(max_length=255)), ('ssh_connection', models.CharField(max_length=255)), ('ssh_original_command', models.CharField(max_length=255)), - ('app', models.ForeignKey(to='api.App')), - ('owner', models.ForeignKey(to=settings.AUTH_USER_MODEL)), + ('app', models.ForeignKey(to='api.App', on_delete=models.CASCADE)), + ('owner', models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)), ], options={ 'ordering': ['-created'], @@ -161,10 +161,10 @@ class Migration(migrations.Migration): ('updated', models.DateTimeField(auto_now=True)), ('version', models.PositiveIntegerField()), ('summary', models.TextField(null=True, blank=True)), - ('app', models.ForeignKey(to='api.App')), - ('build', models.ForeignKey(to='api.Build', null=True)), - ('config', models.ForeignKey(to='api.Config')), - ('owner', models.ForeignKey(to=settings.AUTH_USER_MODEL)), + ('app', models.ForeignKey(to='api.App', on_delete=models.CASCADE)), + ('build', models.ForeignKey(to='api.Build', null=True, on_delete=models.CASCADE)), + ('config', models.ForeignKey(to='api.Config', on_delete=models.CASCADE)), + ('owner', models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)), ], options={ 'ordering': ['-created'], @@ -187,7 +187,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='container', name='release', - field=models.ForeignKey(to='api.Release'), + field=models.ForeignKey(to='api.Release', on_delete=models.CASCADE), preserve_default=True, ), migrations.AlterUniqueTogether( diff --git a/rootfs/api/models/certificate.py b/rootfs/api/models/certificate.py index 596633fab..b7b2768a1 100644 --- a/rootfs/api/models/certificate.py +++ b/rootfs/api/models/certificate.py @@ -148,8 +148,9 @@ def save(self, *args, **kwargs): subject = certificate.get_subject().get_components() self.subject = '/' + '/'.join('%s=%s' % (key.decode(encoding='UTF-8'), value.decode(encoding='UTF-8')) for key, value in subject) # noqa - # public fingerprint of certificate - self.fingerprint = certificate.digest('sha256') + # public fingerprint of certificate - need to decode to UTF-8 in pyOpenSSL + # Convert bytes to string + self.fingerprint = certificate.digest('sha256').decode(encoding='UTF-8') # SubjectAltName from the certificate - return a list self.san = get_subj_alt_name(certificate) diff --git a/rootfs/dev_requirements.txt b/rootfs/dev_requirements.txt index 0281c4cf9..f6594207d 100644 --- a/rootfs/dev_requirements.txt +++ b/rootfs/dev_requirements.txt @@ -1,15 +1,15 @@ # Run "make test-unit" for the % of code exercised during tests -coverage==4.4.1 +coverage==4.5.4 # Run "make test-style" to check python syntax and style flake8==3.4.1 -# code coverage report at https://codecov.io/github/deis/controller -codecov==2.0.9 +# code coverage report at https://codecov.io/github/teamhephy/controller +codecov==2.1.11 # mock out python-requests, mostly k8s # requests-mock==1.3.0 -git+https://github.com/deisthree/requests-mock.git@class_adapter#egg=request_mock +git+https://github.com/teamhephy/requests-mock.git@class_adapter#egg=requests_mock # tail a log and pipe into tbgrep to find all tracebacks tbgrep==0.3.0 diff --git a/rootfs/requirements.txt b/rootfs/requirements.txt index f8cd59dba..d1af0a86d 100644 --- a/rootfs/requirements.txt +++ b/rootfs/requirements.txt @@ -1,23 +1,23 @@ # Deis controller requirements -backoff==1.4.3 -django==1.11.29 -django-auth-ldap==1.2.15 -django-cors-middleware==1.3.1 -django-guardian==1.4.9 -djangorestframework==3.9.4 +backoff==1.8.0 +django==2.2.19,<3.0 +django-auth-ldap==2.3.0,<3.0 +django-cors-middleware==1.5.0 +django-guardian==2.3.0 +djangorestframework==3.12.2,<4.0 docker-py==1.10.6 -gunicorn==19.7.1 -idna==2.6 -jmespath==0.9.3 -jsonfield==2.0.2 -jsonschema==2.6.0 -morph==0.1.2 -ndg-httpsclient==0.4.2 -packaging==16.8 -pyasn1==0.3.2 -psycopg2-binary==2.7.5 -pyldap==2.4.37 -pyOpenSSL==17.5.0 -pytz==2017.2 -requests==2.20.0 -requests-toolbelt==0.8.0 +gunicorn>=20.0.4 +idna>=2.6 +jmespath>=0.10.0 +jsonfield>=3.1.0 +jsonschema>=3.2.0 +morph==0.1.4 +ndg-httpsclient==0.5.1 +packaging==20.0 +pyasn1==0.3.7 +psycopg2-binary==2.8.6 +pyldap==3.0.0 +pyOpenSSL==20.0.1 +pytz>=2021.1 +requests>=2.25.1 +requests-toolbelt>=0.9.1