Add task to clean up orphaned uploads (#3793)

website/photos/tasks.py

@@ -1,10 +1,13 @@
 import logging
+import os
 
 from django.db import transaction
 from django.dispatch import Signal
+from django.utils import timezone
 
 from celery import shared_task
-from django_drf_filepond.models import TemporaryUpload
+from django_drf_filepond.models import TemporaryUpload, TemporaryUploadChunked
+from django_drf_filepond.models import storage as filepond_storage
 
 from mailinglists.services import get_member_email_addresses
 from members.models.member import Member
@@ -65,3 +68,26 @@ def process_album_upload(
     finally:
         archive.delete()
         upload.delete()
+
+
+@shared_task
+def clean_broken_uploads():
+    # Cancel and remove completed uploads that are older than 12 hours.
+    for upload in TemporaryUpload.objects.filter(
+        uploaded__lte=timezone.now() - timezone.timedelta(hours=12)
+    ):
+        logger.info(f"Removing old upload {upload.upload_id}")
+        upload.file.delete()
+        upload.delete()
+
+    # Cancel and remove uploads that have not received new chunks in the last hour.
+    for tuc in TemporaryUploadChunked.objects.filter(
+        last_upload_time__lt=timezone.now() - timezone.timedelta(hours=1),
+    ).exclude(upload_id__in=TemporaryUpload.objects.values("upload_id")):
+        logger.info(f"Removing incomplete chunked upload {tuc.upload_id}")
+        # Store the chunk and check if we've now completed the upload.
+        upload_file = os.path.join(
+            tuc.upload_dir, f"{tuc.file_id}_{tuc.last_chunk + 1}"
+        )
+        filepond_storage.delete(upload_file)
+        tuc.delete()

website/thaliawebsite/settings.py

@@ -437,6 +437,10 @@ def show_toolbar(request):
         "task": "promotion.tasks.promo_update_overview_daily",
         "schedule": crontab(minute=0, hour=8),
     },
+    "cleanupfileponduploads": {
+        "task": "photos.tasks.clean_broken_uploads",
+        "schedule": crontab(minute=45),
+    },
 }
 
 ###############################################################################