Projet

Général

Profil

0002-add-a-fargo-cleanup-command-22682.patch

Benjamin Dauvergne, 26 mars 2018 15:37

Télécharger (12,4 ko)

Voir les différences:

Subject: [PATCH 2/2] add a fargo-cleanup command (#22682)

 debian/fargo.cron.hourly                           |  1 +
 fargo/fargo/management/__init__.py                 |  0
 fargo/fargo/management/commands/__init__.py        |  0
 fargo/fargo/management/commands/fargo-cleanup.py   | 11 +++++
 fargo/fargo/managers.py                            | 15 ++++--
 .../migrations/0015_document_creation_date.py      | 20 ++++++++
 fargo/fargo/models.py                              |  3 ++
 fargo/fargo/utils.py                               | 12 +++++
 fargo/oauth2/migrations/0001_initial.py            |  2 +-
 fargo/oauth2/models.py                             | 27 ++++++++++-
 fargo/settings.py                                  |  1 +
 tests/test_commands.py                             | 54 ++++++++++++++++++++++
 tox.ini                                            | 39 ++++++++--------
 13 files changed, 160 insertions(+), 25 deletions(-)
 create mode 100644 fargo/fargo/management/__init__.py
 create mode 100644 fargo/fargo/management/commands/__init__.py
 create mode 100644 fargo/fargo/management/commands/fargo-cleanup.py
 create mode 100644 fargo/fargo/migrations/0015_document_creation_date.py
 create mode 100644 tests/test_commands.py
debian/fargo.cron.hourly
1 1
#!/bin/sh
2 2

  
3 3
sudo -u fargo /usr/bin/fargo-manage tenant_command clearsessions --all
4
sudo -u fargo /usr/bin/fargo-manage tenant_command fargo-cleanup --all
fargo/fargo/management/commands/fargo-cleanup.py
1
from django.core.management.base import NoArgsCommand
2

  
3
from fargo.fargo.utils import cleanup
4

  
5

  
6
class Command(NoArgsCommand):
7
    help = 'Clean expired models of authentic2.'
8

  
9
    def handle_noargs(self, **options):
10
        cleanup()
11

  
fargo/fargo/managers.py
1
import datetime
2

  
1 3
from django.db import models
4
from django.utils.timezone import now
2 5

  
3 6
from . import utils
4 7

  
5 8

  
6 9
class DocumentManager(models.Manager):
7
    def clean(self):
8
        '''Remove all documents not linked to an user'''
9
        qs = self.filter(user_documents__isnull=True)
10
    def cleanup(self, n=None):
11
        '''Delete all orphaned documents'''
12
        n = n or now()
13
        # use a window of 60 seconds to be sure this document will never be used
14
        qs = self.filter(creation_date__lt=n - datetime.timedelta(seconds=60))
15
        qs = qs.filter(user_documents__isnull=True,
16
                       oauth2_tempfiles__isnull=True)
10 17
        for document in qs:
11
            qs.content.delete(False)
18
            document.content.delete(False)
12 19
        qs.delete()
13 20

  
14 21
    def get_by_file(self, f):
fargo/fargo/migrations/0015_document_creation_date.py
1
# -*- coding: utf-8 -*-
2
# Generated by Django 1.11.11 on 2018-03-23 15:30
3
from __future__ import unicode_literals
4

  
5
from django.db import migrations, models
6

  
7

  
8
class Migration(migrations.Migration):
9

  
10
    dependencies = [
11
        ('fargo', '0014_auto_20171016_0854'),
12
    ]
13

  
14
    operations = [
15
        migrations.AddField(
16
            model_name='document',
17
            name='creation_date',
18
            field=models.DateTimeField(auto_now_add=True),
19
        ),
20
    ]
fargo/fargo/models.py
96 96
                self.document.mime_type.split('/')[0],
97 97
                re.sub('[/\.+-]', '-', self.document.mime_type))
98 98

  
99

  
99 100
class Validation(models.Model):
100 101
    '''Validation of a document as special kind for an user,
101 102
       the data field contains metadata extracted from the document.
......
169 170
    mime_type = models.CharField(
170 171
        max_length=256,
171 172
        blank=True)
173
    creation_date = models.DateTimeField(auto_now_add=True)
172 174

  
173 175
    objects = managers.DocumentManager()
174 176

  
......
222 224
                         instance.thumbnail_full_path])
223 225
    threading.Thread(target=do).start()
224 226

  
227

  
225 228
@receiver(post_delete, sender=Document)
226 229
def delete_file(sender, instance, **kwargs):
227 230
    if instance.content:
fargo/fargo/utils.py
2 2

  
3 3
from django.utils.timezone import utc
4 4
from django.utils.encoding import smart_bytes
5
from django.db import models
5 6

  
6 7
try:
7 8
    import magic
......
40 41
        if mime_type.startswith('cannot open'):
41 42
            mime_type = None
42 43
    return mime_type
44

  
45
def cleanup_model(model, n=None):
46
    manager = getattr(model, 'objects', None)
47
    if hasattr(manager, 'cleanup'):
48
        manager.cleanup(n=n)
49

  
50

  
51
def cleanup(n=None):
52
    for app in models.get_apps():
53
        for model in models.get_models(app):
54
            cleanup_model(model, n=n)
fargo/oauth2/migrations/0001_initial.py
37 37
            fields=[
38 38
                ('hash_key', models.CharField(max_length=128, serialize=False, primary_key=True)),
39 39
                ('filename', models.CharField(max_length=512)),
40
                ('document', models.ForeignKey(to='fargo.Document')),
40
                ('document', models.ForeignKey(to='fargo.Document', related_name='oauth2_tempfiles')),
41 41
            ],
42 42
        ),
43 43
    ]
fargo/oauth2/models.py
15 15
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
16 16

  
17 17
import uuid
18
import datetime
18 19

  
20
from django.conf import settings
19 21
from django.core.exceptions import ValidationError
20 22
from django.core.validators import URLValidator
21 23
from django.db import models
22 24
from django.utils.translation import ugettext_lazy as _
25
from django.db.models.query import QuerySet
26
from django.utils.timezone import now
23 27

  
24 28
from fargo.fargo.models import Document, UserDocument
25 29

  
......
63 67
        return self.client_name
64 68

  
65 69

  
70
class CleanupQuerySet(QuerySet):
71
    def cleanup(self, n=None):
72
        n = n or now()
73
        threshold = n - datetime.timedelta(seconds=2 * self.model.get_lifetime())
74
        self.filter(creation_date__lt=threshold).delete()
75

  
76

  
66 77
class OAuth2Authorize(models.Model):
67 78
    client = models.ForeignKey(OAuth2Client)
68 79
    user_document = models.ForeignKey(UserDocument)
......
70 81
    code = models.CharField(max_length=255, default=generate_uuid)
71 82
    creation_date = models.DateTimeField(auto_now_add=True)
72 83

  
84
    objects = CleanupQuerySet.as_manager()
85

  
86
    @classmethod
87
    def get_lifetime(cls):
88
        return max(
89
            settings.FARGO_CODE_LIFETIME,
90
            settings.FARGO_ACCESS_TOKEN_LIFETIME)
91

  
73 92
    def __repr__(self):
74 93
        return 'OAuth2Authorize for document %r' % self.user_document
75 94

  
......
77 96
class OAuth2TempFile(models.Model):
78 97
    uuid = models.CharField(max_length=32, default=generate_uuid, primary_key=True)
79 98
    client = models.ForeignKey(OAuth2Client)
80
    document = models.ForeignKey(Document)
99
    document = models.ForeignKey(Document, related_name='oauth2_tempfiles')
81 100
    filename = models.CharField(max_length=512)
82 101
    creation_date = models.DateTimeField(auto_now_add=True)
102

  
103
    objects = CleanupQuerySet.as_manager()
104

  
105
    @classmethod
106
    def get_lifetime(cls):
107
        return settings.FARGO_OAUTH2_TEMPFILE_LIFETIME
fargo/settings.py
228 228

  
229 229
FARGO_CODE_LIFETIME = 300
230 230
FARGO_ACCESS_TOKEN_LIFETIME = 3600
231
FARGO_OAUTH2_TEMPFILE_LIFETIME = 86400
231 232

  
232 233
local_settings_file = os.environ.get('FARGO_SETTINGS_FILE',
233 234
                                     os.path.join(
tests/test_commands.py
1
import datetime
2

  
3
from django.core.management import call_command
4

  
5
from django.contrib.auth.models import User
6
from fargo.fargo.models import UserDocument, Document
7
from fargo.oauth2.models import OAuth2TempFile, OAuth2Client
8
from django.core.files.base import ContentFile
9

  
10

  
11
def test_cleanup(freezer, john_doe):
12
    start = freezer()
13

  
14
    client = OAuth2Client.objects.create(client_name='c', redirect_uris='')
15

  
16
    foo = Document.objects.create(content=ContentFile('foo', name='foo.txt'))
17
    bar = Document.objects.create(content=ContentFile('bar', name='bar.txt'))
18
    UserDocument.objects.create(user=john_doe,
19
                                document=foo,
20
                                filename='foo.txt',
21
                                title='',
22
                                description='')
23
    OAuth2TempFile.objects.create(document=bar, client=client, filename='bar.txt')
24

  
25
    call_command('fargo-cleanup')
26

  
27
    assert UserDocument.objects.all().count()
28
    assert OAuth2TempFile.objects.all().count()
29
    assert Document.objects.all().count() == 2
30

  
31
    User.objects.all().delete()
32

  
33
    assert not UserDocument.objects.all().count()
34
    assert Document.objects.all().count() == 2
35

  
36
    call_command('fargo-cleanup')
37

  
38
    assert Document.objects.all().count() == 2
39

  
40
    freezer.move_to(start + datetime.timedelta(seconds=120))
41
    call_command('fargo-cleanup')
42

  
43
    assert Document.objects.all().count() == 1
44

  
45
    freezer.move_to(start + datetime.timedelta(days=3))
46

  
47
    call_command('fargo-cleanup')
48

  
49
    assert not OAuth2TempFile.objects.count()
50
    assert Document.objects.count()
51

  
52
    call_command('fargo-cleanup')
53

  
54
    assert not Document.objects.count()
tox.ini
7 7
whitelist_externals =
8 8
  /bin/mv
9 9
setenv =
10
	sqlite: DB_ENGINE=sqlite3
11
	pg: DB_ENGINE=postgresql_psycopg2
12
	DJANGO_SETTINGS_MODULE=fargo.settings
13
	FARGO_SETTINGS_FILE=tests/settings.py
14
	coverage: COVERAGE=--cov=fargo --cov-report xml
10
        sqlite: DB_ENGINE=sqlite3
11
        pg: DB_ENGINE=postgresql_psycopg2
12
        DJANGO_SETTINGS_MODULE=fargo.settings
13
        FARGO_SETTINGS_FILE=tests/settings.py
14
        coverage: COVERAGE=--cov=fargo --cov-report xml
15 15
deps =
16
	dj18: django>=1.8,<1.9
17
	dj18: django-tables2<1.1
18
	dj111: django>=1.11,<1.12
19
	dj111: django-tables2>=1.5
20
	pytest>=3.3.0
21
	pytest-cov
22
	pytest-random
23
	pytest-mock
24
	pytest-django
25
	django-webtest
26
	WebTest
27
	djangorestframework>=3.3,<3.4
16
        dj18: django>=1.8,<1.9
17
        dj18: django-tables2<1.1
18
        dj111: django>=1.11,<1.12
19
        dj111: django-tables2>=1.5
20
        pytest>=3.3.0
21
        pytest-cov
22
        pytest-random
23
        pytest-mock
24
        pytest-django
25
        pytest-freezegun
26
        django-webtest
27
        WebTest
28
        djangorestframework>=3.3,<3.4
28 29
commands =
29
	py.test {env:COVERAGE:} {posargs:--random --junit-xml=junit-{envname}.xml tests}
30
	coverage: mv coverage.xml coverage-{envname}.xml
30
        py.test {env:COVERAGE:} {posargs:--random --junit-xml=junit-{envname}.xml tests}
31
        coverage: mv coverage.xml coverage-{envname}.xml
31 32

  
32
-