feat(django): upgrade django framework and deps

  - upgrading django to 2.2.19 and djangorestframework to 3.12.2
  - resolving all failing tests due to above upgrade
  - upgrade models to default `on_delete=models.CASCADE` if app or user are deleted
  - upgrade all other dependencies to latest versions
  - update Makefile to use hephy/shell-dev for tests
  - update Dockerfile to use `hephy/base:latest` for building
  - update Dockefile to use python3 -m pip commands
  - upgrade pip, setuptools, wheel using python3 command

Signed-off-by: Cryptophobia <aouzounov@gmail.com>

Makefile

@@ -6,7 +6,7 @@ SHORT_NAME ?= $(COMPONENT)
 
 include versioning.mk
 
-SHELLCHECK_PREFIX := docker run -v ${CURDIR}:/workdir -w /workdir quay.io/deis/shell-dev shellcheck
+SHELLCHECK_PREFIX := docker run -v ${CURDIR}:/workdir -w /workdir hephy/shell-dev shellcheck
 SHELL_SCRIPTS = $(wildcard rootfs/bin/*) $(shell find "rootfs" -name '*.sh') $(wildcard _scripts/*.sh)
 
 # Test processes used in quick unit testing

rootfs/Dockerfile

@@ -1,4 +1,4 @@
-FROM hephy/base:v0.4.1
+FROM hephy/base:latest
 
 RUN adduser --system \
 	--shell /bin/bash \
@@ -21,8 +21,13 @@ RUN buildDeps='gcc libffi-dev libpq-dev libldap2-dev libsasl2-dev python3-dev py
         # cryptography package needs pkg_resources
         python3-pkg-resources && \
     ln -s /usr/bin/python3 /usr/bin/python && \
+		# use python3 to upgrade pip and tools
+		python3 -m pip install --upgrade --no-cache-dir \
+				pip \
+				setuptools \
+				wheel && \
     mkdir -p /configs && chown -R deis:deis /configs && \
-    pip3 install --disable-pip-version-check --no-cache-dir -r /app/requirements.txt && \
+    python3 -m pip install --disable-pip-version-check --no-cache-dir -r /app/requirements.txt && \
     # cleanup
     apt-get purge -y --auto-remove $buildDeps && \
     apt-get autoremove -y && \

rootfs/Dockerfile.test

@@ -1,4 +1,4 @@
-FROM hephy/base:v0.4.1
+FROM hephy/base:latest
 
 RUN adduser --system \
 	--shell /bin/bash \
@@ -18,11 +18,17 @@ RUN buildDeps='gcc libffi-dev libpq-dev libldap2-dev libsasl2-dev python3-dev py
         libpq5 \
         libldap-2.4 \
         python3-minimal \
-        # cryptography package needs pkg_resources
-        python3-pkg-resources && \
+				python3-distutils \
+				# cryptography package needs pkg_resources
+				python3-pkg-resources && \
     ln -s /usr/bin/python3 /usr/bin/python && \
+		# use python3 to upgrade pip and tools
+		python3 -m pip install --upgrade --no-cache-dir \
+				pip \
+				setuptools \
+				wheel && \
     mkdir -p /configs && chown -R deis:deis /configs && \
-    pip3 install --disable-pip-version-check --no-cache-dir -r /app/requirements.txt && \
+    python3 -m pip install --disable-pip-version-check --no-cache-dir -r /app/requirements.txt && \
     # cleanup
     apt-get purge -y --auto-remove $buildDeps && \
     apt-get autoremove -y && \
@@ -54,10 +60,7 @@ RUN apt-get update && \
     DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
         git \
         postgresql \
-        postgresql-contrib \
-        python3-pip \
-        python3-setuptools \
-        python3-wheel && \
+        postgresql-contrib && \
     pip3 install --disable-pip-version-check --no-cache-dir -r dev_requirements.txt && \
     sudo -u postgres -E $PGBIN/initdb
 

rootfs/api/migrations/0001_initial.py

@@ -22,7 +22,7 @@ class Migration(migrations.Migration):
                 ('updated', models.DateTimeField(auto_now=True)),
                 ('id', models.SlugField(max_length=24, unique=True, null=True, validators=[api.models.validate_app_id, api.models.validate_reserved_names])),
                 ('structure', jsonfield.fields.JSONField(default={}, blank=True, validators=[api.models.validate_app_structure])),
-                ('owner', models.ForeignKey(to=settings.AUTH_USER_MODEL)),
+                ('owner', models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)),
             ],
             options={
                 'permissions': (('use_app', 'Can use app'),),
@@ -39,8 +39,8 @@ class Migration(migrations.Migration):
                 ('sha', models.CharField(max_length=40, blank=True)),
                 ('procfile', jsonfield.fields.JSONField(default={}, blank=True)),
                 ('dockerfile', models.TextField(blank=True)),
-                ('app', models.ForeignKey(to='api.App')),
-                ('owner', models.ForeignKey(to=settings.AUTH_USER_MODEL)),
+                ('app', models.ForeignKey(to='api.App', on_delete=models.CASCADE)),
+                ('owner', models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)),
             ],
             options={
                 'ordering': ['-created'],
@@ -58,7 +58,7 @@ class Migration(migrations.Migration):
                 ('key', models.TextField()),
                 ('common_name', models.TextField(unique=True)),
                 ('expires', models.DateTimeField()),
-                ('owner', models.ForeignKey(to=settings.AUTH_USER_MODEL)),
+                ('owner', models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)),
             ],
             options={
                 'abstract': False,
@@ -75,8 +75,8 @@ class Migration(migrations.Migration):
                 ('memory', jsonfield.fields.JSONField(default={}, blank=True)),
                 ('cpu', jsonfield.fields.JSONField(default={}, blank=True)),
                 ('tags', jsonfield.fields.JSONField(default={}, blank=True)),
-                ('app', models.ForeignKey(to='api.App')),
-                ('owner', models.ForeignKey(to=settings.AUTH_USER_MODEL)),
+                ('app', models.ForeignKey(to='api.App', on_delete=models.CASCADE)),
+                ('owner', models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)),
             ],
             options={
                 'ordering': ['-created'],
@@ -92,8 +92,8 @@ class Migration(migrations.Migration):
                 ('updated', models.DateTimeField(auto_now=True)),
                 ('type', models.CharField(max_length=128)),
                 ('num', models.PositiveIntegerField()),
-                ('app', models.ForeignKey(to='api.App')),
-                ('owner', models.ForeignKey(to=settings.AUTH_USER_MODEL)),
+                ('app', models.ForeignKey(to='api.App', on_delete=models.CASCADE)),
+                ('owner', models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)),
             ],
             options={
                 'ordering': ['created'],
@@ -108,8 +108,8 @@ class Migration(migrations.Migration):
                 ('created', models.DateTimeField(auto_now_add=True)),
                 ('updated', models.DateTimeField(auto_now=True)),
                 ('domain', models.TextField(unique=True)),
-                ('app', models.ForeignKey(to='api.App')),
-                ('owner', models.ForeignKey(to=settings.AUTH_USER_MODEL)),
+                ('app', models.ForeignKey(to='api.App', on_delete=models.CASCADE)),
+                ('owner', models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)),
             ],
             options={
                 'abstract': False,
@@ -125,7 +125,7 @@ class Migration(migrations.Migration):
                 ('id', models.CharField(max_length=128)),
                 ('public', models.TextField(unique=True, validators=[api.models.validate_base64])),
                 ('fingerprint', models.CharField(max_length=128)),
-                ('owner', models.ForeignKey(to=settings.AUTH_USER_MODEL)),
+                ('owner', models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)),
             ],
             options={
                 'verbose_name': 'SSH Key',
@@ -144,8 +144,8 @@ class Migration(migrations.Migration):
                 ('receive_repo', models.CharField(max_length=255)),
                 ('ssh_connection', models.CharField(max_length=255)),
                 ('ssh_original_command', models.CharField(max_length=255)),
-                ('app', models.ForeignKey(to='api.App')),
-                ('owner', models.ForeignKey(to=settings.AUTH_USER_MODEL)),
+                ('app', models.ForeignKey(to='api.App', on_delete=models.CASCADE)),
+                ('owner', models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)),
             ],
             options={
                 'ordering': ['-created'],
@@ -161,10 +161,10 @@ class Migration(migrations.Migration):
                 ('updated', models.DateTimeField(auto_now=True)),
                 ('version', models.PositiveIntegerField()),
                 ('summary', models.TextField(null=True, blank=True)),
-                ('app', models.ForeignKey(to='api.App')),
-                ('build', models.ForeignKey(to='api.Build', null=True)),
-                ('config', models.ForeignKey(to='api.Config')),
-                ('owner', models.ForeignKey(to=settings.AUTH_USER_MODEL)),
+                ('app', models.ForeignKey(to='api.App', on_delete=models.CASCADE)),
+                ('build', models.ForeignKey(to='api.Build', null=True, on_delete=models.CASCADE)),
+                ('config', models.ForeignKey(to='api.Config', on_delete=models.CASCADE)),
+                ('owner', models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)),
             ],
             options={
                 'ordering': ['-created'],
@@ -187,7 +187,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='container',
             name='release',
-            field=models.ForeignKey(to='api.Release'),
+            field=models.ForeignKey(to='api.Release', on_delete=models.CASCADE),
             preserve_default=True,
         ),
         migrations.AlterUniqueTogether(

rootfs/api/models/certificate.py

@@ -148,8 +148,8 @@ def save(self, *args, **kwargs):
         subject = certificate.get_subject().get_components()
         self.subject = '/' + '/'.join('%s=%s' % (key.decode(encoding='UTF-8'), value.decode(encoding='UTF-8')) for key, value in subject)  # noqa
 
-        # public fingerprint of certificate
-        self.fingerprint = certificate.digest('sha256')
+        # public fingerprint of certificate - need to decode to UTF-8 in django 2.2
+        self.fingerprint = certificate.digest('sha256').decode(encoding='UTF-8')
 
         # SubjectAltName from the certificate - return a list
         self.san = get_subj_alt_name(certificate)

rootfs/dev_requirements.txt

@@ -1,15 +1,15 @@
 # Run "make test-unit" for the % of code exercised during tests
-coverage==4.4.1
+coverage==4.5.4
 
 # Run "make test-style" to check python syntax and style
 flake8==3.4.1
 
-# code coverage report at https://codecov.io/github/deis/controller
-codecov==2.0.9
+# code coverage report at https://codecov.io/github/teamhephy/controller
+codecov==2.1.11
 
 # mock out python-requests, mostly k8s
 # requests-mock==1.3.0
-git+https://github.com/deisthree/requests-mock.git@class_adapter#egg=request_mock
+git+https://github.com/teamhephy/requests-mock.git@class_adapter#egg=requests_mock
 
 # tail a log and pipe into tbgrep to find all tracebacks
 tbgrep==0.3.0

rootfs/requirements.txt

@@ -1,23 +1,23 @@
 # Deis controller requirements
-backoff==1.4.3
-django==1.11.29
-django-auth-ldap==1.2.15
-django-cors-middleware==1.3.1
-django-guardian==1.4.9
-djangorestframework==3.9.4
+backoff>=1.8.0
+django>=2.2.19,<3.0
+django-auth-ldap>=2.3.0,<3.0
+django-cors-middleware>=1.5.0
+django-guardian>=2.3.0
+djangorestframework>=3.12.2,<4.0
 docker-py==1.10.6
-gunicorn==19.7.1
-idna==2.6
-jmespath==0.9.3
-jsonfield==2.0.2
-jsonschema==2.6.0
-morph==0.1.2
-ndg-httpsclient==0.4.2
-packaging==16.8
-pyasn1==0.3.2
-psycopg2-binary==2.7.5
-pyldap==2.4.37
-pyOpenSSL==17.5.0
-pytz==2017.2
-requests==2.20.0
-requests-toolbelt==0.8.0
+gunicorn>=20.0.4
+idna>=2.6
+jmespath>=0.10.0
+jsonfield>=3.1.0
+jsonschema>=3.2.0
+morph==0.1.4
+ndg-httpsclient==0.5.1
+packaging==20.0
+pyasn1==0.3.7
+psycopg2-binary==2.8.6
+pyldap==3.0.0
+pyOpenSSL>=20.0.1
+pytz>=2021.1
+requests>=2.25.1
+requests-toolbelt>=0.9.1