Projet

Général

Profil

0001-general-remove-unused-document-types-validation-code.patch

Frédéric Péters, 10 août 2022 14:11

Télécharger (14,5 ko)

Voir les différences:

Subject: [PATCH] general: remove unused document types & validation code
 (#14148)

 fargo/fargo/admin.py                          |  7 --
 fargo/fargo/api_views.py                      | 79 +------------------
 .../migrations/0004_auto_20220810_1352.py     | 18 +++++
 fargo/fargo/models.py                         | 56 -------------
 fargo/fargo/utils.py                          |  6 --
 fargo/fargo/views.py                          | 13 ---
 fargo/settings.py                             | 22 ------
 fargo/urls.py                                 |  5 +-
 tests/test_api.py                             | 40 ----------
 9 files changed, 21 insertions(+), 225 deletions(-)
 create mode 100644 fargo/fargo/migrations/0004_auto_20220810_1352.py
fargo/fargo/admin.py
88 88
    thumbnail.short_description = _('thumbnail')
89 89

  
90 90

  
91
class ValidationAdmin(admin.ModelAdmin):
92
    fields = ['user', 'content_hash', 'document_type', 'start', 'end', 'data', 'creator', 'created', 'origin']
93
    readonly_fields = ['created']
94
    list_display = ['user', 'display', 'document_type', 'start', 'end', 'creator', 'created', 'origin']
95

  
96

  
97 91
class OriginAdmin(admin.ModelAdmin):
98 92
    fields = ['label', 'slug']
99 93
    list_display = ['label', 'slug']
......
101 95

  
102 96
admin.site.register(models.UserDocument, UserDocumentAdmin)
103 97
admin.site.register(models.Document, DocumentAdmin)
104
admin.site.register(models.Validation, ValidationAdmin)
105 98
admin.site.register(models.Origin, OriginAdmin)
fargo/fargo/api_views.py
18 18

  
19 19
from django.conf import settings
20 20
from django.contrib.auth.models import User
21
from django.urls import reverse
22 21
from django.utils.text import slugify
23
from django.utils.timezone import now
24
from rest_framework import exceptions, filters, mixins, routers, serializers, status, viewsets
22
from rest_framework import exceptions, filters, serializers, status
25 23
from rest_framework.generics import GenericAPIView, ListAPIView
26 24
from rest_framework.permissions import IsAdminUser
27 25
from rest_framework.response import Response
28 26

  
29 27
from . import api_errors, api_fields, utils
30
from .models import Document, Origin, UserDocument, Validation
28
from .models import Document, Origin, UserDocument
31 29

  
32 30
try:
33 31
    from mellon.models import UserSAMLIdentifier
......
166 164
recent_documents = RecentDocuments.as_view()
167 165

  
168 166

  
169
class ValidationSerializer(UserSerializerMixin, serializers.ModelSerializer):
170
    origin = api_fields.SlugCreatedRelatedField(slug_field='label', queryset=Origin.objects.all())
171
    url = serializers.SerializerMethodField()
172
    display = serializers.CharField(read_only=True)
173

  
174
    def __init__(self, *args, **kwargs):
175
        schema = kwargs.pop('schema')
176
        super().__init__(*args, **kwargs)
177
        self.document_type = schema['name']
178
        self.document_type_schema = schema
179
        for field in schema['metadata']:
180
            name = field['varname']
181
            required = field.get('required', True)
182
            self.fields[name] = serializers.CharField(
183
                source='data.%s' % name, required=required, allow_blank=True
184
            )
185

  
186
    def get_url(self, instance):
187
        url = reverse(
188
            'fargo-api-validation-detail', kwargs={'document_type': instance.document_type, 'pk': instance.pk}
189
        )
190
        if 'request' in self.context:
191
            url = self.context['request'].build_absolute_uri(url)
192
        return url
193

  
194
    def validate(self, data):
195
        data = super().validate(data)
196
        data['document_type'] = self.document_type
197
        data['created'] = now().replace(microsecond=0)
198
        data['start'] = data['created'].date()
199
        data['end'] = data['start'] + datetime.timedelta(seconds=settings.FARGO_VALIDATION_LIFETIME)
200
        data['creator'] = data['creator'][:256]
201
        return data
202

  
203
    class Meta:
204
        model = Validation
205
        exclude = ('data', 'user', 'document_type')
206
        read_only_fields = ('created', 'start', 'end')
207

  
208

  
209 167
class FilterByUser(filters.BaseFilterBackend):
210 168
    def filter_queryset(self, request, queryset, view):
211 169
        if 'user_email' in request.GET:
......
213 171
        elif 'user_nameid' in request.GET:
214 172
            return queryset.filter(user__saml_identifiers__name_id=request.GET['user_nameid'])
215 173
        return queryset
216

  
217

  
218
class ValidationAPI(
219
    CommonAPIMixin,
220
    mixins.CreateModelMixin,
221
    mixins.RetrieveModelMixin,
222
    mixins.ListModelMixin,
223
    viewsets.GenericViewSet,
224
):
225
    serializer_class = ValidationSerializer
226
    permission_classes = (IsAdminUser,)
227
    filter_backends = [FilterByUser]
228
    queryset = Validation.objects.all()
229

  
230
    def get_queryset(self):
231
        return super().get_queryset().filter(document_type=self.document_type)
232

  
233
    def initial(self, request, document_type, *args, **kwargs):
234
        self.document_type_schema = utils.get_document_type_schema(settings, document_type)
235
        if not self.document_type_schema:
236
            error = serializers.ValidationError('unknown document type')
237
            error.status_code = status.HTTP_404_NOT_FOUND
238
            raise error
239
        self.document_type = document_type
240
        super().initial(request, document_type, *args, **kwargs)
241

  
242
    def get_serializer(self, *args, **kwargs):
243
        # pass schema to serializer class
244
        return super().get_serializer(schema=self.document_type_schema, *args, **kwargs)
245

  
246

  
247
router = routers.SimpleRouter()
248
router.register(r'validation/(?P<document_type>[^/]*)', ValidationAPI, basename='fargo-api-validation')
fargo/fargo/migrations/0004_auto_20220810_1352.py
1
# Generated by Django 2.2.28 on 2022-08-10 11:52
2

  
3
from django.conf import settings
4
from django.db import migrations, models
5

  
6

  
7
class Migration(migrations.Migration):
8

  
9
    dependencies = [
10
        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
11
        ('fargo', '0003_text_to_jsonb'),
12
    ]
13

  
14
    operations = [
15
        migrations.DeleteModel(
16
            name='Validation',
17
        ),
18
    ]
fargo/fargo/models.py
115 115
        )
116 116

  
117 117

  
118
@python_2_unicode_compatible
119
class Validation(models.Model):
120
    """Validation of a document as special kind for an user,
121
    the data field contains metadata extracted from the document.
122
    """
123

  
124
    user = models.ForeignKey(settings.AUTH_USER_MODEL, verbose_name=_('user'), on_delete=models.CASCADE)
125
    content_hash = models.CharField(max_length=128, verbose_name=_('content hash'), blank=True, null=True)
126
    origin = models.ForeignKey(Origin, verbose_name=_('origin'), null=True, on_delete=models.CASCADE)
127
    document_type = models.CharField(max_length=256, verbose_name=_('document type'))
128
    data = JSONField(null=True, verbose_name=_('data'), default=dict)
129
    start = models.DateField(verbose_name=_('start date'))
130
    end = models.DateField(verbose_name=_('end date'))
131
    creator = models.CharField(max_length=256, verbose_name=_('creator'))
132
    created = models.DateTimeField(verbose_name=_('creation date'))
133

  
134
    @property
135
    def document_type_schema(self):
136
        return utils.get_document_type_schema(settings, self.document_type) or {}
137

  
138
    @property
139
    def metadata(self):
140
        return self.document_type_schema.get('metadata', [])
141

  
142
    def display(self):
143
        template = self.document_type_schema.get('display_template', '')
144
        if template:
145
            try:
146
                return force_text(template.format(**self.data))
147
            except KeyError:
148
                pass
149
        parts = []
150
        for meta_field in self.metadata:
151
            parts.append(
152
                _('%(label)s: %(value)s')
153
                % {
154
                    'label': meta_field['label'],
155
                    'value': self.data.get(meta_field['varname'], ''),
156
                }
157
            )
158
        return force_text('; '.join(parts))
159

  
160
    display.short_description = _('description')
161

  
162
    @property
163
    def user_document(self):
164
        if self.content_hash:
165
            try:
166
                return UserDocument.objects.get(document__content_hash=self.content_hash)
167
            except UserDocument.DoesNotExist:
168
                pass
169

  
170
    def __str__(self):
171
        return self.display()
172

  
173

  
174 118
class Document(models.Model):
175 119
    '''Content indexed documents'''
176 120

  
fargo/fargo/utils.py
30 30
    return dt.astimezone(utc).isoformat('T').split('.')[0] + 'Z'
31 31

  
32 32

  
33
def get_document_type_schema(settings, document_type):
34
    for schema in settings.FARGO_DOCUMENT_TYPES:
35
        if schema.get('name') == document_type:
36
            return schema
37

  
38

  
39 33
def sha256_of_file(f):
40 34
    '''Compute SHA-256 hash of Django File object'''
41 35
    hasher = hashlib.sha256()
fargo/fargo/views.py
329 329
logout = LogoutView.as_view()
330 330

  
331 331

  
332
class DocumentTypes(View):
333
    def get(self, request):
334
        document_types = deepcopy(settings.FARGO_DOCUMENT_TYPES)
335
        for document_type in document_types:
336
            document_type.pop('display_template', None)
337
        data = {
338
            'err': 0,
339
            'data': document_types,
340
        }
341
        return HttpResponse(dumps(data), content_type='application/json')
342

  
343

  
344 332
home = login_required(Homepage.as_view())
345 333
download = login_required(Download.as_view())
346 334
thumbnail = login_required(Thumbnail.as_view())
......
352 340
jsonp = login_required(JSONP.as_view())
353 341
json = login_required(JSON.as_view())
354 342
pick_list = xframe_options_exempt(login_required(PickList.as_view()))
355
document_types = DocumentTypes.as_view()
fargo/settings.py
179 179

  
180 180
FARGO_VALIDATION_LIFETIME = 3600 * 24 * 31 * 6  # nearly 6 months
181 181

  
182
FARGO_DOCUMENT_TYPES = [
183
    {
184
        'name': 'avis-d-imposition',
185
        'label': 'Avis d\'imposition',
186
        'metadata': [
187
            {'label': 'Personne-s concernée-s', 'varname': 'personnes_concernees', 'type': 'string'},
188
            {
189
                'label': 'Année',
190
                'varname': 'annee',
191
                'type': 'string',
192
                'validation': ' *[0-9]{4} *',
193
            },
194
            {
195
                'label': 'Revenu fiscal de référence',
196
                'varname': 'revenu_fiscal_de_reference',
197
                'type': 'string',
198
                'validation': ' *[0-9]+ *',
199
            },
200
        ],
201
    },
202
]
203

  
204 182
LOGIN_REDIRECT_URL = 'home'
205 183

  
206 184
REST_FRAMEWORK = {
fargo/urls.py
18 18
from django.conf.urls import include, url
19 19
from django.contrib import admin
20 20

  
21
from .fargo.api_views import push_document, recent_documents, router
21
from .fargo.api_views import push_document, recent_documents
22 22
from .fargo.views import (
23 23
    delete,
24
    document_types,
25 24
    download,
26 25
    edit,
27 26
    home,
......
51 50
    url(r'^admin/', admin.site.urls),
52 51
    url(r'^login/$', login, name='auth_login'),
53 52
    url(r'^logout/$', logout, name='auth_logout'),
54
    url(r'^document-types/$', document_types, name='document_types'),
55 53
    url(r'^api/documents/push/$', push_document, name='fargo-api-push-document'),
56 54
    url(r'^api/documents/recently-added/$', recent_documents),
57
    url(r'^api/', include(router.urls)),
58 55
]
59 56

  
60 57
if settings.DEBUG and 'debug_toolbar' in settings.INSTALLED_APPS:
tests/test_api.py
25 25
pytestmark = pytest.mark.django_db
26 26

  
27 27

  
28
def test_create_validation(settings, app, admin_user, john_doe, jane_doe):
29
    login(app)
30
    data = {
31
        'user_email': john_doe.email,
32
    }
33
    schema = utils.get_document_type_schema(settings, 'avis-d-imposition')
34
    url = '/api/validation/avis-d-imposition/'
35
    assert models.Validation.objects.count() == 0
36
    response = app.post_json(url, params=data, status=400)
37
    assert response.json['result'] == 0
38
    assert set(response.json['errors'].keys()) == set(
39
        [field['varname'] for field in schema['metadata']] + ['creator', 'origin']
40
    )
41
    assert models.Validation.objects.count() == 0
42
    data.update(
43
        {
44
            'personnes_concernees': 'John and Lisa Doe',
45
            'annee': '2016',
46
            'revenu_fiscal_de_reference': '32455',
47
            'creator': 'FooBar',
48
            'origin': 'wcs.example.com',
49
        }
50
    )
51
    response1 = app.post_json(url, params=data, status=201)
52
    assert set(response1.json.keys()) == {'result', 'data'}
53
    assert response1.json['result'] == 1
54
    assert set(data.keys()) - {'user_email'} < set(response1.json['data'].keys())
55
    assert models.Validation.objects.count() == 1
56
    data['user_email'] = jane_doe.email
57
    response2 = app.post_json(url, params=data, status=201)
58
    assert models.Validation.objects.count() == 2
59
    response3 = app.get(url)
60
    assert response3.json['data']['count'] == 2
61
    response4 = app.get(url + '?%s' % urlencode({'user_email': john_doe.email}))
62
    assert response4.json['data']['count'] == 1
63
    response5 = app.get(response2.json['data']['url'], status=200)
64
    assert response5.json['result'] == 1
65
    assert response5.json['data'] == response2.json['data']
66

  
67

  
68 28
def test_push_document(app, admin_user, john_doe):
69 29
    login(app)
70 30
    data = {
71
-