From 7089a0fa7efdf6de934e6bd22a27a5f64c27eba7 Mon Sep 17 00:00:00 2001 From: Benjamin Dauvergne Date: Fri, 23 Mar 2018 16:39:52 +0100 Subject: [PATCH 2/2] add a fargo-cleanup command (#22682) --- debian/fargo.cron.hourly | 1 + fargo/fargo/management/__init__.py | 0 fargo/fargo/management/commands/__init__.py | 0 fargo/fargo/management/commands/fargo-cleanup.py | 11 +++++ fargo/fargo/managers.py | 15 ++++-- .../migrations/0015_document_creation_date.py | 20 ++++++++ fargo/fargo/models.py | 3 ++ fargo/fargo/utils.py | 12 +++++ fargo/oauth2/migrations/0001_initial.py | 2 +- fargo/oauth2/models.py | 27 ++++++++++- fargo/settings.py | 1 + tests/test_commands.py | 54 ++++++++++++++++++++++ tox.ini | 39 ++++++++-------- 13 files changed, 160 insertions(+), 25 deletions(-) create mode 100644 fargo/fargo/management/__init__.py create mode 100644 fargo/fargo/management/commands/__init__.py create mode 100644 fargo/fargo/management/commands/fargo-cleanup.py create mode 100644 fargo/fargo/migrations/0015_document_creation_date.py create mode 100644 tests/test_commands.py diff --git a/debian/fargo.cron.hourly b/debian/fargo.cron.hourly index 99a6804..171b443 100644 --- a/debian/fargo.cron.hourly +++ b/debian/fargo.cron.hourly @@ -1,3 +1,4 @@ #!/bin/sh sudo -u fargo /usr/bin/fargo-manage tenant_command clearsessions --all +sudo -u fargo /usr/bin/fargo-manage tenant_command fargo-cleanup --all diff --git a/fargo/fargo/management/__init__.py b/fargo/fargo/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/fargo/fargo/management/commands/__init__.py b/fargo/fargo/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/fargo/fargo/management/commands/fargo-cleanup.py b/fargo/fargo/management/commands/fargo-cleanup.py new file mode 100644 index 0000000..6cc73ad --- /dev/null +++ b/fargo/fargo/management/commands/fargo-cleanup.py @@ -0,0 +1,11 @@ +from django.core.management.base import NoArgsCommand + +from fargo.fargo.utils import cleanup + + +class Command(NoArgsCommand): + help = 'Clean expired models' + + def handle_noargs(self, **options): + cleanup() + diff --git a/fargo/fargo/managers.py b/fargo/fargo/managers.py index b177682..70dfe0b 100644 --- a/fargo/fargo/managers.py +++ b/fargo/fargo/managers.py @@ -1,14 +1,21 @@ +import datetime + from django.db import models +from django.utils.timezone import now from . import utils class DocumentManager(models.Manager): - def clean(self): - '''Remove all documents not linked to an user''' - qs = self.filter(user_documents__isnull=True) + def cleanup(self, n=None): + '''Delete all orphaned documents''' + n = n or now() + # use a window of 60 seconds to be sure this document will never be used + qs = self.filter(creation_date__lt=n - datetime.timedelta(seconds=60)) + qs = qs.filter(user_documents__isnull=True, + oauth2_tempfiles__isnull=True) for document in qs: - qs.content.delete(False) + document.content.delete(False) qs.delete() def get_by_file(self, f): diff --git a/fargo/fargo/migrations/0015_document_creation_date.py b/fargo/fargo/migrations/0015_document_creation_date.py new file mode 100644 index 0000000..efee72d --- /dev/null +++ b/fargo/fargo/migrations/0015_document_creation_date.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.11 on 2018-03-23 15:30 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('fargo', '0014_auto_20171016_0854'), + ] + + operations = [ + migrations.AddField( + model_name='document', + name='creation_date', + field=models.DateTimeField(auto_now_add=True), + ), + ] diff --git a/fargo/fargo/models.py b/fargo/fargo/models.py index 1774359..6f0f17b 100644 --- a/fargo/fargo/models.py +++ b/fargo/fargo/models.py @@ -96,6 +96,7 @@ class UserDocument(models.Model): self.document.mime_type.split('/')[0], re.sub('[/\.+-]', '-', self.document.mime_type)) + class Validation(models.Model): '''Validation of a document as special kind for an user, the data field contains metadata extracted from the document. @@ -169,6 +170,7 @@ class Document(models.Model): mime_type = models.CharField( max_length=256, blank=True) + creation_date = models.DateTimeField(auto_now_add=True) objects = managers.DocumentManager() @@ -222,6 +224,7 @@ def create_thumbnail(sender, instance, created, **kwargs): instance.thumbnail_full_path]) threading.Thread(target=do).start() + @receiver(post_delete, sender=Document) def delete_file(sender, instance, **kwargs): if instance.content: diff --git a/fargo/fargo/utils.py b/fargo/fargo/utils.py index 320a375..8ee8a42 100644 --- a/fargo/fargo/utils.py +++ b/fargo/fargo/utils.py @@ -2,6 +2,7 @@ import hashlib from django.utils.timezone import utc from django.utils.encoding import smart_bytes +from django.db import models try: import magic @@ -40,3 +41,14 @@ def get_mime_type(path): if mime_type.startswith('cannot open'): mime_type = None return mime_type + +def cleanup_model(model, n=None): + manager = getattr(model, 'objects', None) + if hasattr(manager, 'cleanup'): + manager.cleanup(n=n) + + +def cleanup(n=None): + for app in models.get_apps(): + for model in models.get_models(app): + cleanup_model(model, n=n) diff --git a/fargo/oauth2/migrations/0001_initial.py b/fargo/oauth2/migrations/0001_initial.py index d6e5f92..2179119 100644 --- a/fargo/oauth2/migrations/0001_initial.py +++ b/fargo/oauth2/migrations/0001_initial.py @@ -37,7 +37,7 @@ class Migration(migrations.Migration): fields=[ ('hash_key', models.CharField(max_length=128, serialize=False, primary_key=True)), ('filename', models.CharField(max_length=512)), - ('document', models.ForeignKey(to='fargo.Document')), + ('document', models.ForeignKey(to='fargo.Document', related_name='oauth2_tempfiles')), ], ), ] diff --git a/fargo/oauth2/models.py b/fargo/oauth2/models.py index d330643..6165ab4 100644 --- a/fargo/oauth2/models.py +++ b/fargo/oauth2/models.py @@ -15,11 +15,15 @@ # along with this program. If not, see . import uuid +import datetime +from django.conf import settings from django.core.exceptions import ValidationError from django.core.validators import URLValidator from django.db import models from django.utils.translation import ugettext_lazy as _ +from django.db.models.query import QuerySet +from django.utils.timezone import now from fargo.fargo.models import Document, UserDocument @@ -63,6 +67,13 @@ class OAuth2Client(models.Model): return self.client_name +class CleanupQuerySet(QuerySet): + def cleanup(self, n=None): + n = n or now() + threshold = n - datetime.timedelta(seconds=2 * self.model.get_lifetime()) + self.filter(creation_date__lt=threshold).delete() + + class OAuth2Authorize(models.Model): client = models.ForeignKey(OAuth2Client) user_document = models.ForeignKey(UserDocument) @@ -70,6 +81,14 @@ class OAuth2Authorize(models.Model): code = models.CharField(max_length=255, default=generate_uuid) creation_date = models.DateTimeField(auto_now_add=True) + objects = CleanupQuerySet.as_manager() + + @classmethod + def get_lifetime(cls): + return max( + settings.FARGO_CODE_LIFETIME, + settings.FARGO_ACCESS_TOKEN_LIFETIME) + def __repr__(self): return 'OAuth2Authorize for document %r' % self.user_document @@ -77,6 +96,12 @@ class OAuth2Authorize(models.Model): class OAuth2TempFile(models.Model): uuid = models.CharField(max_length=32, default=generate_uuid, primary_key=True) client = models.ForeignKey(OAuth2Client) - document = models.ForeignKey(Document) + document = models.ForeignKey(Document, related_name='oauth2_tempfiles') filename = models.CharField(max_length=512) creation_date = models.DateTimeField(auto_now_add=True) + + objects = CleanupQuerySet.as_manager() + + @classmethod + def get_lifetime(cls): + return settings.FARGO_OAUTH2_TEMPFILE_LIFETIME diff --git a/fargo/settings.py b/fargo/settings.py index d33f429..cdc92ae 100644 --- a/fargo/settings.py +++ b/fargo/settings.py @@ -228,6 +228,7 @@ INCLUDE_EDIT_LINK = False FARGO_CODE_LIFETIME = 300 FARGO_ACCESS_TOKEN_LIFETIME = 3600 +FARGO_OAUTH2_TEMPFILE_LIFETIME = 86400 local_settings_file = os.environ.get('FARGO_SETTINGS_FILE', os.path.join( diff --git a/tests/test_commands.py b/tests/test_commands.py new file mode 100644 index 0000000..bf0835b --- /dev/null +++ b/tests/test_commands.py @@ -0,0 +1,54 @@ +import datetime + +from django.core.management import call_command + +from django.contrib.auth.models import User +from fargo.fargo.models import UserDocument, Document +from fargo.oauth2.models import OAuth2TempFile, OAuth2Client +from django.core.files.base import ContentFile + + +def test_cleanup(freezer, john_doe): + start = freezer() + + client = OAuth2Client.objects.create(client_name='c', redirect_uris='') + + foo = Document.objects.create(content=ContentFile('foo', name='foo.txt')) + bar = Document.objects.create(content=ContentFile('bar', name='bar.txt')) + UserDocument.objects.create(user=john_doe, + document=foo, + filename='foo.txt', + title='', + description='') + OAuth2TempFile.objects.create(document=bar, client=client, filename='bar.txt') + + call_command('fargo-cleanup') + + assert UserDocument.objects.all().count() + assert OAuth2TempFile.objects.all().count() + assert Document.objects.all().count() == 2 + + User.objects.all().delete() + + assert not UserDocument.objects.all().count() + assert Document.objects.all().count() == 2 + + call_command('fargo-cleanup') + + assert Document.objects.all().count() == 2 + + freezer.move_to(start + datetime.timedelta(seconds=120)) + call_command('fargo-cleanup') + + assert Document.objects.all().count() == 1 + + freezer.move_to(start + datetime.timedelta(days=3)) + + call_command('fargo-cleanup') + + assert not OAuth2TempFile.objects.count() + assert Document.objects.count() + + call_command('fargo-cleanup') + + assert not Document.objects.count() diff --git a/tox.ini b/tox.ini index 897793a..9e4c8ce 100644 --- a/tox.ini +++ b/tox.ini @@ -7,25 +7,26 @@ usedevelop = True whitelist_externals = /bin/mv setenv = - sqlite: DB_ENGINE=sqlite3 - pg: DB_ENGINE=postgresql_psycopg2 - DJANGO_SETTINGS_MODULE=fargo.settings - FARGO_SETTINGS_FILE=tests/settings.py - coverage: COVERAGE=--cov=fargo --cov-report xml + sqlite: DB_ENGINE=sqlite3 + pg: DB_ENGINE=postgresql_psycopg2 + DJANGO_SETTINGS_MODULE=fargo.settings + FARGO_SETTINGS_FILE=tests/settings.py + coverage: COVERAGE=--cov=fargo --cov-report xml deps = - dj18: django>=1.8,<1.9 - dj18: django-tables2<1.1 - dj111: django>=1.11,<1.12 - dj111: django-tables2>=1.5 - pytest>=3.3.0 - pytest-cov - pytest-random - pytest-mock - pytest-django - django-webtest - WebTest - djangorestframework>=3.3,<3.4 + dj18: django>=1.8,<1.9 + dj18: django-tables2<1.1 + dj111: django>=1.11,<1.12 + dj111: django-tables2>=1.5 + pytest>=3.3.0 + pytest-cov + pytest-random + pytest-mock + pytest-django + pytest-freezegun + django-webtest + WebTest + djangorestframework>=3.3,<3.4 commands = - py.test {env:COVERAGE:} {posargs:--random --junit-xml=junit-{envname}.xml tests} - coverage: mv coverage.xml coverage-{envname}.xml + py.test {env:COVERAGE:} {posargs:--random --junit-xml=junit-{envname}.xml tests} + coverage: mv coverage.xml coverage-{envname}.xml -- 2.14.2