Project

General

Profile

0001-add-a-fargo-cleanup-command-22682.patch

Benjamin Dauvergne, 23 Mar 2018 05:02 PM

Download (11.3 KB)

View differences:

Subject: [PATCH] add a fargo-cleanup command (#22682)

 debian/fargo.cron.hourly                           |  1 +
 fargo/fargo/management/__init__.py                 |  0
 fargo/fargo/management/commands/__init__.py        |  0
 fargo/fargo/management/commands/fargo-cleanup.py   | 11 +++++
 fargo/fargo/managers.py                            | 15 ++++--
 .../migrations/0015_document_creation_date.py      | 20 ++++++++
 fargo/fargo/models.py                              |  3 ++
 fargo/fargo/utils.py                               | 14 ++++++
 fargo/oauth2/migrations/0004_auto_20180323_1530.py | 21 +++++++++
 fargo/oauth2/models.py                             | 27 ++++++++++-
 fargo/settings.py                                  |  1 +
 tests/test_commands.py                             | 54 ++++++++++++++++++++++
 tox.ini                                            |  1 +
 13 files changed, 163 insertions(+), 5 deletions(-)
 create mode 100644 fargo/fargo/management/__init__.py
 create mode 100644 fargo/fargo/management/commands/__init__.py
 create mode 100644 fargo/fargo/management/commands/fargo-cleanup.py
 create mode 100644 fargo/fargo/migrations/0015_document_creation_date.py
 create mode 100644 fargo/oauth2/migrations/0004_auto_20180323_1530.py
 create mode 100644 tests/test_commands.py
debian/fargo.cron.hourly
1 1
#!/bin/sh
2 2

  
3 3
sudo -u fargo /usr/bin/fargo-manage tenant_command clearsessions --all
4
sudo -u fargo /usr/bin/fargo-manage tenant_command fargo-cleanup --all
fargo/fargo/management/commands/fargo-cleanup.py
1
from django.core.management.base import NoArgsCommand
2

  
3
from fargo.fargo.utils import cleanup
4

  
5

  
6
class Command(NoArgsCommand):
7
    help = 'Clean expired models of authentic2.'
8

  
9
    def handle_noargs(self, **options):
10
        cleanup()
11

  
fargo/fargo/managers.py
1
import datetime
2

  
1 3
from django.db import models
4
from django.utils.timezone import now
2 5

  
3 6
from . import utils
4 7

  
5 8

  
6 9
class DocumentManager(models.Manager):
7
    def clean(self):
8
        '''Remove all documents not linked to an user'''
9
        qs = self.filter(user_documents__isnull=True)
10
    def cleanup(self, n=None):
11
        '''Delete all orphaned documents'''
12
        n = n or now()
13
        # use a window of 60 seconds to be sure this document will never be used
14
        qs = self.filter(creation_date__lt=n - datetime.timedelta(seconds=60))
15
        qs = qs.filter(user_documents__isnull=True,
16
                       oauth2_tempfiles__isnull=True)
10 17
        for document in qs:
11
            qs.content.delete(False)
18
            document.content.delete(False)
12 19
        qs.delete()
13 20

  
14 21
    def get_by_file(self, f):
fargo/fargo/migrations/0015_document_creation_date.py
1
# -*- coding: utf-8 -*-
2
# Generated by Django 1.11.11 on 2018-03-23 15:30
3
from __future__ import unicode_literals
4

  
5
from django.db import migrations, models
6

  
7

  
8
class Migration(migrations.Migration):
9

  
10
    dependencies = [
11
        ('fargo', '0014_auto_20171016_0854'),
12
    ]
13

  
14
    operations = [
15
        migrations.AddField(
16
            model_name='document',
17
            name='creation_date',
18
            field=models.DateTimeField(auto_now=True),
19
        ),
20
    ]
fargo/fargo/models.py
101 101
                self.document.mime_type.split('/')[0],
102 102
                re.sub('[/\.+-]', '-', self.document.mime_type))
103 103

  
104

  
104 105
class Validation(models.Model):
105 106
    '''Validation of a document as special kind for an user,
106 107
       the data field contains metadata extracted from the document.
......
174 175
    mime_type = models.CharField(
175 176
        max_length=256,
176 177
        blank=True)
178
    creation_date = models.DateTimeField(auto_now=True)
177 179

  
178 180
    objects = managers.DocumentManager()
179 181

  
......
230 232
                         instance.thumbnail_full_path])
231 233
    threading.Thread(target=do).start()
232 234

  
235

  
233 236
@receiver(post_delete, sender=Document)
234 237
def delete_file(sender, instance, **kwargs):
235 238
    if instance.content:
fargo/fargo/utils.py
1 1
import hashlib
2

  
2 3
from django.utils.timezone import utc
4
from django.db import models
3 5

  
4 6

  
5 7
def to_isodate(dt):
......
18 20
    for chunk in f.chunks():
19 21
        hasher.update(chunk)
20 22
    return hasher.hexdigest()
23

  
24

  
25
def cleanup_model(model, n=None):
26
    manager = getattr(model, 'objects', None)
27
    if hasattr(manager, 'cleanup'):
28
        manager.cleanup(n=n)
29

  
30

  
31
def cleanup(n=None):
32
    for app in models.get_apps():
33
        for model in models.get_models(app):
34
            cleanup_model(model, n=n)
fargo/oauth2/migrations/0004_auto_20180323_1530.py
1
# -*- coding: utf-8 -*-
2
# Generated by Django 1.11.11 on 2018-03-23 15:30
3
from __future__ import unicode_literals
4

  
5
from django.db import migrations, models
6
import django.db.models.deletion
7

  
8

  
9
class Migration(migrations.Migration):
10

  
11
    dependencies = [
12
        ('oauth2', '0003_auto_20180322_1016'),
13
    ]
14

  
15
    operations = [
16
        migrations.AlterField(
17
            model_name='oauth2tempfile',
18
            name='document',
19
            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='oauth2_tempfiles', to='fargo.Document'),
20
        ),
21
    ]
fargo/oauth2/models.py
15 15
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
16 16

  
17 17
import uuid
18
import datetime
18 19

  
20
from django.conf import settings
19 21
from django.core.exceptions import ValidationError
20 22
from django.core.validators import URLValidator
21 23
from django.db import models
22 24
from django.utils.translation import ugettext_lazy as _
25
from django.db.models.query import QuerySet
26
from django.utils.timezone import now
23 27

  
24 28
from fargo.fargo.models import Document, UserDocument
25 29

  
......
63 67
        return self.client_name
64 68

  
65 69

  
70
class CleanupQuerySet(QuerySet):
71
    def cleanup(self, n=None):
72
        n = n or now()
73
        threshold = n - datetime.timedelta(seconds=2 * self.model.get_lifetime())
74
        self.filter(creation_date__lt=threshold).delete()
75

  
76

  
66 77
class OAuth2Authorize(models.Model):
67 78
    client = models.ForeignKey(OAuth2Client)
68 79
    user_document = models.ForeignKey(UserDocument)
......
70 81
    code = models.CharField(max_length=255, default=generate_uuid)
71 82
    creation_date = models.DateTimeField(auto_now=True)
72 83

  
84
    objects = CleanupQuerySet.as_manager()
85

  
86
    @classmethod
87
    def get_lifetime(cls):
88
        return max(
89
            settings.FARGO_CODE_LIFETIME,
90
            settings.FARGO_ACCESS_TOKEN_LIFETIME)
91

  
73 92
    def __repr__(self):
74 93
        return 'OAuth2Authorize for document %r' % self.user_document
75 94

  
......
77 96
class OAuth2TempFile(models.Model):
78 97
    uuid = models.CharField(max_length=16, default=generate_uuid, primary_key=True)
79 98
    client = models.ForeignKey(OAuth2Client)
80
    document = models.ForeignKey(Document)
99
    document = models.ForeignKey(Document, related_name='oauth2_tempfiles')
81 100
    filename = models.CharField(max_length=512)
82 101
    creation_date = models.DateTimeField(auto_now=True)
102

  
103
    objects = CleanupQuerySet.as_manager()
104

  
105
    @classmethod
106
    def get_lifetime(cls):
107
        return settings.FARGO_OAUTH2_TEMPFILE_LIFETIME
fargo/settings.py
228 228

  
229 229
FARGO_CODE_LIFETIME = 300
230 230
FARGO_ACCESS_TOKEN_LIFETIME = 3600
231
FARGO_OAUTH2_TEMPFILE_LIFETIME = 86400
231 232

  
232 233
local_settings_file = os.environ.get('FARGO_SETTINGS_FILE',
233 234
                                     os.path.join(
tests/test_commands.py
1
import datetime
2

  
3
from django.core.management import call_command
4

  
5
from django.contrib.auth.models import User
6
from fargo.fargo.models import UserDocument, Document
7
from fargo.oauth2.models import OAuth2TempFile, OAuth2Client
8
from django.core.files.base import ContentFile
9

  
10

  
11
def test_cleanup(freezer, john_doe):
12
    start = freezer()
13

  
14
    client = OAuth2Client.objects.create(client_name='c', redirect_uris='')
15

  
16
    foo = Document.objects.create(content=ContentFile('foo', name='foo.txt'))
17
    bar = Document.objects.create(content=ContentFile('bar', name='bar.txt'))
18
    UserDocument.objects.create(user=john_doe,
19
                                document=foo,
20
                                filename='foo.txt',
21
                                title='',
22
                                description='')
23
    OAuth2TempFile.objects.create(document=bar, client=client, filename='bar.txt')
24

  
25
    call_command('fargo-cleanup')
26

  
27
    assert UserDocument.objects.all().count()
28
    assert OAuth2TempFile.objects.all().count()
29
    assert Document.objects.all().count() == 2
30

  
31
    User.objects.all().delete()
32

  
33
    assert not UserDocument.objects.all().count()
34
    assert Document.objects.all().count() == 2
35

  
36
    call_command('fargo-cleanup')
37

  
38
    assert Document.objects.all().count() == 2
39

  
40
    freezer.move_to(start + datetime.timedelta(seconds=120))
41
    call_command('fargo-cleanup')
42

  
43
    assert Document.objects.all().count() == 1
44

  
45
    freezer.move_to(start + datetime.timedelta(days=3))
46

  
47
    call_command('fargo-cleanup')
48

  
49
    assert not OAuth2TempFile.objects.count()
50
    assert Document.objects.count()
51

  
52
    call_command('fargo-cleanup')
53

  
54
    assert not Document.objects.count()
tox.ini
22 22
	pytest-random
23 23
	pytest-mock
24 24
	pytest-django
25
        pytest-freezegun
25 26
	django-webtest
26 27
	WebTest
27 28
	djangorestframework>=3.3,<3.4
28
-