0002-add-a-fargo-cleanup-command-22682.patch
debian/fargo.cron.hourly | ||
---|---|---|
1 | 1 |
#!/bin/sh |
2 | 2 | |
3 | 3 |
sudo -u fargo /usr/bin/fargo-manage tenant_command clearsessions --all |
4 |
sudo -u fargo /usr/bin/fargo-manage tenant_command fargo-cleanup --all |
fargo/fargo/management/commands/fargo-cleanup.py | ||
---|---|---|
1 |
from django.core.management.base import NoArgsCommand |
|
2 | ||
3 |
from fargo.fargo.utils import cleanup |
|
4 | ||
5 | ||
6 |
class Command(NoArgsCommand): |
|
7 |
help = 'Clean expired models of authentic2.' |
|
8 | ||
9 |
def handle_noargs(self, **options): |
|
10 |
cleanup() |
|
11 |
fargo/fargo/managers.py | ||
---|---|---|
1 |
import datetime |
|
2 | ||
1 | 3 |
from django.db import models |
4 |
from django.utils.timezone import now |
|
2 | 5 | |
3 | 6 |
from . import utils |
4 | 7 | |
5 | 8 | |
6 | 9 |
class DocumentManager(models.Manager): |
7 |
def clean(self): |
|
8 |
'''Remove all documents not linked to an user''' |
|
9 |
qs = self.filter(user_documents__isnull=True) |
|
10 |
def cleanup(self, n=None): |
|
11 |
'''Delete all orphaned documents''' |
|
12 |
n = n or now() |
|
13 |
# use a window of 60 seconds to be sure this document will never be used |
|
14 |
qs = self.filter(creation_date__lt=n - datetime.timedelta(seconds=60)) |
|
15 |
qs = qs.filter(user_documents__isnull=True, |
|
16 |
oauth2_tempfiles__isnull=True) |
|
10 | 17 |
for document in qs: |
11 |
qs.content.delete(False)
|
|
18 |
document.content.delete(False)
|
|
12 | 19 |
qs.delete() |
13 | 20 | |
14 | 21 |
def get_by_file(self, f): |
fargo/fargo/migrations/0015_document_creation_date.py | ||
---|---|---|
1 |
# -*- coding: utf-8 -*- |
|
2 |
# Generated by Django 1.11.11 on 2018-03-23 15:30 |
|
3 |
from __future__ import unicode_literals |
|
4 | ||
5 |
from django.db import migrations, models |
|
6 | ||
7 | ||
8 |
class Migration(migrations.Migration): |
|
9 | ||
10 |
dependencies = [ |
|
11 |
('fargo', '0014_auto_20171016_0854'), |
|
12 |
] |
|
13 | ||
14 |
operations = [ |
|
15 |
migrations.AddField( |
|
16 |
model_name='document', |
|
17 |
name='creation_date', |
|
18 |
field=models.DateTimeField(auto_now_add=True), |
|
19 |
), |
|
20 |
] |
fargo/fargo/models.py | ||
---|---|---|
96 | 96 |
self.document.mime_type.split('/')[0], |
97 | 97 |
re.sub('[/\.+-]', '-', self.document.mime_type)) |
98 | 98 | |
99 | ||
99 | 100 |
class Validation(models.Model): |
100 | 101 |
'''Validation of a document as special kind for an user, |
101 | 102 |
the data field contains metadata extracted from the document. |
... | ... | |
169 | 170 |
mime_type = models.CharField( |
170 | 171 |
max_length=256, |
171 | 172 |
blank=True) |
173 |
creation_date = models.DateTimeField(auto_now_add=True) |
|
172 | 174 | |
173 | 175 |
objects = managers.DocumentManager() |
174 | 176 | |
... | ... | |
222 | 224 |
instance.thumbnail_full_path]) |
223 | 225 |
threading.Thread(target=do).start() |
224 | 226 | |
227 | ||
225 | 228 |
@receiver(post_delete, sender=Document) |
226 | 229 |
def delete_file(sender, instance, **kwargs): |
227 | 230 |
if instance.content: |
fargo/fargo/utils.py | ||
---|---|---|
2 | 2 | |
3 | 3 |
from django.utils.timezone import utc |
4 | 4 |
from django.utils.encoding import smart_bytes |
5 |
from django.db import models |
|
5 | 6 | |
6 | 7 |
try: |
7 | 8 |
import magic |
... | ... | |
40 | 41 |
if mime_type.startswith('cannot open'): |
41 | 42 |
mime_type = None |
42 | 43 |
return mime_type |
44 | ||
45 |
def cleanup_model(model, n=None): |
|
46 |
manager = getattr(model, 'objects', None) |
|
47 |
if hasattr(manager, 'cleanup'): |
|
48 |
manager.cleanup(n=n) |
|
49 | ||
50 | ||
51 |
def cleanup(n=None): |
|
52 |
for app in models.get_apps(): |
|
53 |
for model in models.get_models(app): |
|
54 |
cleanup_model(model, n=n) |
fargo/oauth2/migrations/0001_initial.py | ||
---|---|---|
37 | 37 |
fields=[ |
38 | 38 |
('hash_key', models.CharField(max_length=128, serialize=False, primary_key=True)), |
39 | 39 |
('filename', models.CharField(max_length=512)), |
40 |
('document', models.ForeignKey(to='fargo.Document')), |
|
40 |
('document', models.ForeignKey(to='fargo.Document', related_name='oauth2_tempfiles')),
|
|
41 | 41 |
], |
42 | 42 |
), |
43 | 43 |
] |
fargo/oauth2/models.py | ||
---|---|---|
15 | 15 |
# along with this program. If not, see <http://www.gnu.org/licenses/>. |
16 | 16 | |
17 | 17 |
import uuid |
18 |
import datetime |
|
18 | 19 | |
20 |
from django.conf import settings |
|
19 | 21 |
from django.core.exceptions import ValidationError |
20 | 22 |
from django.core.validators import URLValidator |
21 | 23 |
from django.db import models |
22 | 24 |
from django.utils.translation import ugettext_lazy as _ |
25 |
from django.db.models.query import QuerySet |
|
26 |
from django.utils.timezone import now |
|
23 | 27 | |
24 | 28 |
from fargo.fargo.models import Document, UserDocument |
25 | 29 | |
... | ... | |
63 | 67 |
return self.client_name |
64 | 68 | |
65 | 69 | |
70 |
class CleanupQuerySet(QuerySet): |
|
71 |
def cleanup(self, n=None): |
|
72 |
n = n or now() |
|
73 |
threshold = n - datetime.timedelta(seconds=2 * self.model.get_lifetime()) |
|
74 |
self.filter(creation_date__lt=threshold).delete() |
|
75 | ||
76 | ||
66 | 77 |
class OAuth2Authorize(models.Model): |
67 | 78 |
client = models.ForeignKey(OAuth2Client) |
68 | 79 |
user_document = models.ForeignKey(UserDocument) |
... | ... | |
70 | 81 |
code = models.CharField(max_length=255, default=generate_uuid) |
71 | 82 |
creation_date = models.DateTimeField(auto_now_add=True) |
72 | 83 | |
84 |
objects = CleanupQuerySet.as_manager() |
|
85 | ||
86 |
@classmethod |
|
87 |
def get_lifetime(cls): |
|
88 |
return max( |
|
89 |
settings.FARGO_CODE_LIFETIME, |
|
90 |
settings.FARGO_ACCESS_TOKEN_LIFETIME) |
|
91 | ||
73 | 92 |
def __repr__(self): |
74 | 93 |
return 'OAuth2Authorize for document %r' % self.user_document |
75 | 94 | |
... | ... | |
77 | 96 |
class OAuth2TempFile(models.Model): |
78 | 97 |
uuid = models.CharField(max_length=32, default=generate_uuid, primary_key=True) |
79 | 98 |
client = models.ForeignKey(OAuth2Client) |
80 |
document = models.ForeignKey(Document) |
|
99 |
document = models.ForeignKey(Document, related_name='oauth2_tempfiles')
|
|
81 | 100 |
filename = models.CharField(max_length=512) |
82 | 101 |
creation_date = models.DateTimeField(auto_now_add=True) |
102 | ||
103 |
objects = CleanupQuerySet.as_manager() |
|
104 | ||
105 |
@classmethod |
|
106 |
def get_lifetime(cls): |
|
107 |
return settings.FARGO_OAUTH2_TEMPFILE_LIFETIME |
fargo/settings.py | ||
---|---|---|
228 | 228 | |
229 | 229 |
FARGO_CODE_LIFETIME = 300 |
230 | 230 |
FARGO_ACCESS_TOKEN_LIFETIME = 3600 |
231 |
FARGO_OAUTH2_TEMPFILE_LIFETIME = 86400 |
|
231 | 232 | |
232 | 233 |
local_settings_file = os.environ.get('FARGO_SETTINGS_FILE', |
233 | 234 |
os.path.join( |
tests/test_commands.py | ||
---|---|---|
1 |
import datetime |
|
2 | ||
3 |
from django.core.management import call_command |
|
4 | ||
5 |
from django.contrib.auth.models import User |
|
6 |
from fargo.fargo.models import UserDocument, Document |
|
7 |
from fargo.oauth2.models import OAuth2TempFile, OAuth2Client |
|
8 |
from django.core.files.base import ContentFile |
|
9 | ||
10 | ||
11 |
def test_cleanup(freezer, john_doe): |
|
12 |
start = freezer() |
|
13 | ||
14 |
client = OAuth2Client.objects.create(client_name='c', redirect_uris='') |
|
15 | ||
16 |
foo = Document.objects.create(content=ContentFile('foo', name='foo.txt')) |
|
17 |
bar = Document.objects.create(content=ContentFile('bar', name='bar.txt')) |
|
18 |
UserDocument.objects.create(user=john_doe, |
|
19 |
document=foo, |
|
20 |
filename='foo.txt', |
|
21 |
title='', |
|
22 |
description='') |
|
23 |
OAuth2TempFile.objects.create(document=bar, client=client, filename='bar.txt') |
|
24 | ||
25 |
call_command('fargo-cleanup') |
|
26 | ||
27 |
assert UserDocument.objects.all().count() |
|
28 |
assert OAuth2TempFile.objects.all().count() |
|
29 |
assert Document.objects.all().count() == 2 |
|
30 | ||
31 |
User.objects.all().delete() |
|
32 | ||
33 |
assert not UserDocument.objects.all().count() |
|
34 |
assert Document.objects.all().count() == 2 |
|
35 | ||
36 |
call_command('fargo-cleanup') |
|
37 | ||
38 |
assert Document.objects.all().count() == 2 |
|
39 | ||
40 |
freezer.move_to(start + datetime.timedelta(seconds=120)) |
|
41 |
call_command('fargo-cleanup') |
|
42 | ||
43 |
assert Document.objects.all().count() == 1 |
|
44 | ||
45 |
freezer.move_to(start + datetime.timedelta(days=3)) |
|
46 | ||
47 |
call_command('fargo-cleanup') |
|
48 | ||
49 |
assert not OAuth2TempFile.objects.count() |
|
50 |
assert Document.objects.count() |
|
51 | ||
52 |
call_command('fargo-cleanup') |
|
53 | ||
54 |
assert not Document.objects.count() |
tox.ini | ||
---|---|---|
7 | 7 |
whitelist_externals = |
8 | 8 |
/bin/mv |
9 | 9 |
setenv = |
10 |
sqlite: DB_ENGINE=sqlite3
|
|
11 |
pg: DB_ENGINE=postgresql_psycopg2
|
|
12 |
DJANGO_SETTINGS_MODULE=fargo.settings
|
|
13 |
FARGO_SETTINGS_FILE=tests/settings.py
|
|
14 |
coverage: COVERAGE=--cov=fargo --cov-report xml
|
|
10 |
sqlite: DB_ENGINE=sqlite3
|
|
11 |
pg: DB_ENGINE=postgresql_psycopg2
|
|
12 |
DJANGO_SETTINGS_MODULE=fargo.settings
|
|
13 |
FARGO_SETTINGS_FILE=tests/settings.py
|
|
14 |
coverage: COVERAGE=--cov=fargo --cov-report xml
|
|
15 | 15 |
deps = |
16 |
dj18: django>=1.8,<1.9 |
|
17 |
dj18: django-tables2<1.1 |
|
18 |
dj111: django>=1.11,<1.12 |
|
19 |
dj111: django-tables2>=1.5 |
|
20 |
pytest>=3.3.0 |
|
21 |
pytest-cov |
|
22 |
pytest-random |
|
23 |
pytest-mock |
|
24 |
pytest-django |
|
25 |
django-webtest |
|
26 |
WebTest |
|
27 |
djangorestframework>=3.3,<3.4 |
|
16 |
dj18: django>=1.8,<1.9 |
|
17 |
dj18: django-tables2<1.1 |
|
18 |
dj111: django>=1.11,<1.12 |
|
19 |
dj111: django-tables2>=1.5 |
|
20 |
pytest>=3.3.0 |
|
21 |
pytest-cov |
|
22 |
pytest-random |
|
23 |
pytest-mock |
|
24 |
pytest-django |
|
25 |
pytest-freezegun |
|
26 |
django-webtest |
|
27 |
WebTest |
|
28 |
djangorestframework>=3.3,<3.4 |
|
28 | 29 |
commands = |
29 |
py.test {env:COVERAGE:} {posargs:--random --junit-xml=junit-{envname}.xml tests}
|
|
30 |
coverage: mv coverage.xml coverage-{envname}.xml
|
|
30 |
py.test {env:COVERAGE:} {posargs:--random --junit-xml=junit-{envname}.xml tests}
|
|
31 |
coverage: mv coverage.xml coverage-{envname}.xml
|
|
31 | 32 | |
32 |
- |