Projet

Général

Profil

0004-cells-w.c.s.-cells-invalid-report-38009.patch

Lauréline Guérin, 02 mars 2020 16:08

Télécharger (23,7 ko)

Voir les différences:

Subject: [PATCH 4/6] cells: w.c.s. cells invalid report (#38009)

 combo/apps/wcs/__init__.py |   9 ++
 combo/apps/wcs/models.py   | 168 +++++++++++++++++++++------
 tests/test_wcs.py          | 227 ++++++++++++++++++++++++++++++++++++-
 3 files changed, 370 insertions(+), 34 deletions(-)
combo/apps/wcs/__init__.py
65 65
        return urls.urlpatterns
66 66

  
67 67
    def hourly(self):
68
        from combo.data.library import get_cell_classes
69
        from combo.data.models import CellBase
70

  
71
        cell_classes = [c for c in self.get_models() if c in get_cell_classes()]
72
        for cell in CellBase.get_cells(cell_filter=lambda x: x in cell_classes, page__snapshot__isnull=True):
73
            if hasattr(cell, 'check_validity'):
74
                cell.check_validity()
75

  
68 76
        self.update_db_cache()
69 77

  
70 78
    def update_db_cache(self):
......
74 82
                cell_filter=lambda x: x in (WcsFormCell, WcsCategoryCell, WcsFormsOfCategoryCell)):
75 83
            cell.save()
76 84

  
85

  
77 86
default_app_config = 'combo.apps.wcs.AppConfig'
combo/apps/wcs/models.py
19 19
import copy
20 20
import logging
21 21

  
22
from django import template
23 22
from django.conf import settings
24 23
from django.db import models
25 24
from django.forms import models as model_forms
......
34 33

  
35 34
from .utils import get_wcs_json, is_wcs_enabled, get_wcs_services
36 35

  
36

  
37
invalid_reason_codes = {
38
    'wcs_form_not_defined': _('No form set'),
39
    'wcs_form_not_found': _('Invalid form'),
40
    'wcs_category_not_defined': _('No category set'),
41
    'wcs_category_not_found': _('Invalid category'),
42
    'wcs_data_failure': _('Failed to get data'),
43
}
44

  
45

  
37 46
@register_cell_class
38 47
class WcsFormCell(CellBase):
39 48
    formdef_reference = models.CharField(_('Form'), max_length=150)
......
47 56
    add_link_label = _('New form link')
48 57
    edit_link_label = _('Edit form link')
49 58
    add_as_link_code = 'form-link'
59
    invalid_reason_codes = invalid_reason_codes
50 60

  
51 61
    is_enabled = classmethod(is_wcs_enabled)
52 62

  
......
58 68
        return WcsFormCellForm
59 69

  
60 70
    def save(self, *args, **kwargs):
61
        if self.formdef_reference:
62
            wcs_key, form_slug = self.formdef_reference.split(':')
63
            wcs_site = get_wcs_services().get(wcs_key)
64
            forms_response_json = get_wcs_json(wcs_site, 'api/formdefs/')
65
            if isinstance(forms_response_json, dict):
66
                # forward compability with future w.c.s. API
67
                forms_response_json = forms_response_json.get('data')
68
            for form in forms_response_json:
69
                slug = form.get('slug')
70
                if slug == form_slug:
71
                    self.cached_title = form.get('title')
72
                    self.cached_url = form.get('url')
73
                    self.cached_json = form
74
        return super(WcsFormCell, self).save(*args, **kwargs)
71
        if 'update_fields' in kwargs:
72
            # don't populate the cache
73
            return super(WcsFormCell, self).save(*args, **kwargs)
74

  
75
        def populate_cache():
76
            if self.formdef_reference:
77
                wcs_key, form_slug = self.formdef_reference.split(':')
78
                wcs_site = get_wcs_services().get(wcs_key)
79
                forms_response_json = get_wcs_json(wcs_site, 'api/formdefs/')
80

  
81
                if not forms_response_json:
82
                    # can not retrieve data, don't report cell as invalid
83
                    self.mark_as_valid()
84
                    return
85

  
86
                if isinstance(forms_response_json, dict):
87
                    # forward compability with future w.c.s. API
88
                    forms_response_json = forms_response_json.get('data')
89
                form_found = False
90
                for form in forms_response_json:
91
                    slug = form.get('slug')
92
                    if slug == form_slug:
93
                        self.cached_title = form.get('title')
94
                        self.cached_url = form.get('url')
95
                        self.cached_json = form
96
                        form_found = True
97
                        break
98
                if form_found:
99
                    self.mark_as_valid()
100
                    return
101
                else:
102
                    return self.mark_as_invalid('wcs_form_not_found', save=False)
103
            else:
104
                return self.mark_as_invalid('wcs_form_not_defined', save=False)
105

  
106
        validity_info = populate_cache()
107
        result = super(WcsFormCell, self).save(*args, **kwargs)
108
        if validity_info:
109
            # save validity_info after Cell insert
110
            validity_info.content_object = self
111
            validity_info.save()
112
        return result
75 113

  
76 114
    def get_cell_extra_context(self, context):
77 115
        context = super(WcsFormCell, self).get_cell_extra_context(context)
......
129 167
    cached_description = models.TextField(_('Description'), blank=True)
130 168
    cached_url = models.URLField(_('Cached URL'))
131 169

  
170
    invalid_reason_codes = invalid_reason_codes
171

  
132 172
    class Meta:
133 173
        abstract = True
134 174

  
135 175
    def save(self, *args, **kwargs):
136
        if self.category_reference:
137
            wcs_key, category_slug = self.category_reference.split(':')
138
            wcs_site = get_wcs_services().get(wcs_key)
139
            categories_response_json = get_wcs_json(wcs_site, 'api/categories/')
140
            for category in categories_response_json.get('data'):
141
                slug = category.get('slug')
142
                if slug == category_slug:
143
                    self.cached_title = category.get('title')
144
                    self.cached_description = category.get('description') or ''
145
                    self.cached_url = category.get('url')
146
        return super(WcsCommonCategoryCell, self).save(*args, **kwargs)
176
        if 'update_fields' in kwargs:
177
            # don't populate the cache
178
            return super(WcsCommonCategoryCell, self).save(*args, **kwargs)
179

  
180
        def populate_cache():
181
            if self.category_reference:
182
                wcs_key, category_slug = self.category_reference.split(':')
183
                wcs_site = get_wcs_services().get(wcs_key)
184
                categories_response_json = get_wcs_json(wcs_site, 'api/categories/')
185

  
186
                if not categories_response_json:
187
                    # can not retrieve data, don't report cell as invalid
188
                    self.mark_as_valid()
189
                    return
190

  
191
                category_found = False
192
                for category in categories_response_json.get('data', []):
193
                    slug = category.get('slug')
194
                    if slug == category_slug:
195
                        self.cached_title = category.get('title')
196
                        self.cached_description = category.get('description') or ''
197
                        self.cached_url = category.get('url')
198
                        category_found = True
199
                        break
200
                if category_found:
201
                    self.mark_as_valid()
202
                    return
203
                else:
204
                    return self.mark_as_invalid('wcs_category_not_found', save=False)
205
            else:
206
                return self.mark_as_invalid('wcs_category_not_defined', save=False)
207

  
208
        validity_info = populate_cache()
209
        result = super(WcsCommonCategoryCell, self).save(*args, **kwargs)
210
        if validity_info:
211
            # save validity_info after Cell insert
212
            validity_info.content_object = self
213
            validity_info.save()
214
        return result
147 215

  
148 216
    def get_additional_label(self):
149 217
        if not self.cached_title:
......
192 260
    is_enabled = classmethod(is_wcs_enabled)
193 261
    cache_duration = 5
194 262
    api_url = None
195
    warn_on_404 = True
263
    warn_on_4xx = True
264
    invalid_reason_codes = invalid_reason_codes
196 265

  
197 266
    def get_api_url(self, context):
198 267
        return self.api_url
......
242 311
                for item in wcs_site['data']:
243 312
                    item['site_slug'] = wcs_slug
244 313

  
245
        if not 200 in returns: # not a single valid answer
246
            if returns != set([404]) or self.warn_on_404:
247
                logging.error('failed to get data from any %s (%r)', api_url, returns)
314
        if 200 not in returns:  # not a single valid answer
315
            logging.debug('failed to get data from any %s (%r)', api_url, returns)
316
            if all([400 <= r < 500 for r in returns]) and self.warn_on_4xx:
317
                # only 4xx errors, not a user cell, report the cell as invalid
318
                self.mark_as_invalid('wcs_data_failure', force=False)
319
            else:
320
                self.mark_as_valid()
321
        else:
322
            self.mark_as_valid()
248 323

  
249 324
        return wcs_sites
250 325

  
......
293 368

  
294 369

  
295 370
class WcsUserDataBaseCell(WcsDataBaseCell):
296
    warn_on_404 = False
371
    warn_on_4xx = False
297 372
    user_dependant = True
298 373

  
299 374
    class Meta:
......
305 380
        return super(WcsUserDataBaseCell, self).is_visible(user)
306 381

  
307 382

  
383
class CategoriesValidityMixin(object):
384
    def check_validity(self):
385
        categories = self.categories.get('data', [])
386

  
387
        if not categories:
388
            return
389

  
390
        for category_reference in categories:
391
            wcs_key, category_slug = category_reference.split(':')
392
            wcs_site = get_wcs_services().get(wcs_key)
393
            categories_response_json = get_wcs_json(wcs_site, 'api/categories/')
394

  
395
            if not categories_response_json:
396
                # can not retrieve data, don't report cell as invalid
397
                continue
398

  
399
            category_found = any(
400
                [c.get('slug') == category_slug
401
                 for c in categories_response_json.get('data', [])])
402
            if not category_found:
403
                self.mark_as_invalid('wcs_category_not_found')
404
                return
405

  
406
        self.mark_as_valid()
407

  
408

  
308 409
@register_cell_class
309
class WcsCurrentFormsCell(WcsUserDataBaseCell):
410
class WcsCurrentFormsCell(WcsUserDataBaseCell, CategoriesValidityMixin):
310 411
    variable_name = 'user_forms'
311 412

  
312 413
    categories = JSONField(_('Categories'), blank=True)
......
400 501

  
401 502
        return context
402 503

  
504

  
403 505
@register_cell_class
404
class WcsCurrentDraftsCell(WcsUserDataBaseCell):
506
class WcsCurrentDraftsCell(WcsUserDataBaseCell, CategoriesValidityMixin):
405 507
    variable_name = 'current_drafts'
406 508
    template_name = 'combo/wcs/current_drafts.html'
407 509

  
tests/test_wcs.py
22 22

  
23 23
import mock
24 24

  
25
from combo.data.models import CellBase, LinkListCell, Page
25
from combo.data.library import get_cell_classes
26
from combo.data.models import CellBase, LinkListCell, Page, ValidityInfo
26 27
from combo.apps.search.engines import engines
27 28
from combo.apps.wcs.models import (WcsFormCell, WcsCurrentFormsCell,
28 29
        WcsFormsOfCategoryCell, WcsCurrentDraftsCell, WcsCategoryCell,
......
299 300
    appconfig.update_db_cache()
300 301
    assert WcsFormCell.objects.get(id=cell.id).cached_title == 'form title'
301 302

  
303

  
304
@wcs_present
305
def test_form_cell_validity():
306
    page = Page.objects.create(title='xxx', slug='test_form_cell_save_cache', template_name='standard')
307
    cell = WcsFormCell.objects.create(page=page, placeholder='content', order=0)
308
    validity_info = ValidityInfo.objects.latest('pk')
309
    assert validity_info.invalid_reason_code == 'wcs_form_not_defined'
310
    assert validity_info.invalid_since is not None
311

  
312
    cell.formdef_reference = u'default:form-title'
313
    cell.save()
314
    assert ValidityInfo.objects.exists() is False
315

  
316
    # can not retrieve formdefs, don't set cell as invalid
317
    with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
318
        mock_json = mock.Mock(status_code=404)
319
        requests_get.return_value = mock_json
320
        cell.save()
321
    assert ValidityInfo.objects.exists() is False
322

  
323
    cell.formdef_reference = u'default:foobar'
324
    cell.save()
325
    validity_info = ValidityInfo.objects.latest('pk')
326
    assert validity_info.invalid_reason_code == 'wcs_form_not_found'
327
    assert validity_info.invalid_since is not None
328

  
329

  
302 330
@wcs_present
303 331
def test_form_cell_load():
304 332
    page = Page(title='xxx', slug='test_form_cell_save_cache', template_name='standard')
......
328 356
    assert cell.cached_title == 'Test 3'
329 357
    assert cell.get_additional_label() == 'Test 3'
330 358

  
359

  
360
@wcs_present
361
def test_category_cell_validity():
362
    page = Page.objects.create(title='xxx', slug='test_category_cell_save_cache', template_name='standard')
363
    cell = WcsCategoryCell.objects.create(page=page, placeholder='content', order=0)
364
    validity_info = ValidityInfo.objects.latest('pk')
365
    assert validity_info.invalid_reason_code == 'wcs_category_not_defined'
366
    assert validity_info.invalid_since is not None
367

  
368
    cell.category_reference = u'default:test-3'
369
    cell.save()
370
    assert ValidityInfo.objects.exists() is False
371

  
372
    # can not retrieve categories, don't set cell as invalid
373
    with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
374
        mock_json = mock.Mock(status_code=404)
375
        requests_get.return_value = mock_json
376
        cell.save()
377
    assert ValidityInfo.objects.exists() is False
378

  
379
    cell.category_reference = 'default:foobar'
380
    cell.save()
381
    validity_info = ValidityInfo.objects.latest('pk')
382
    assert validity_info.invalid_reason_code == 'wcs_category_not_found'
383
    assert validity_info.invalid_since is not None
384

  
385

  
331 386
@wcs_present
332 387
def test_form_cell_render():
333 388
    page = Page(title='xxx', slug='test_form_cell_render', template_name='standard')
......
426 481
    # check include drafts
427 482
    cell.include_drafts = False
428 483
    with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
484
        mock_json = mock.Mock(status_code=200)
485
        requests_get.return_value = mock_json
429 486
        cell.get_cell_extra_context(context)
430 487
    assert requests_get.call_args_list[0][0][0] == '/api/user/forms?limit=100&sort=desc'
431 488

  
432 489
    cell.include_drafts = True
433 490
    with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
491
        mock_json = mock.Mock(status_code=200)
492
        requests_get.return_value = mock_json
434 493
        cell.get_cell_extra_context(context)
435 494
    assert requests_get.call_args_list[0][0][0] == '/api/user/forms?limit=100&sort=desc&include-drafts=on'
436 495

  
437 496

  
497
@wcs_present
498
def test_current_forms_cell_validity(context):
499
    page = Page.objects.create(title='xxx', slug='test_current_forms_cell_render', template_name='standard')
500
    cell = WcsCurrentFormsCell.objects.create(page=page, placeholder='content', order=0)
501
    context['request'].user = MockUser()
502
    context['synchronous'] = True  # to get fresh content
503

  
504
    cell.get_data(context)
505
    assert ValidityInfo.objects.exists() is False
506

  
507
    with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
508
        mock_json = mock.Mock(status_code=404)
509
        requests_get.return_value = mock_json
510
        cell.get_data(context)
511
    assert ValidityInfo.objects.exists() is False
512

  
513

  
514
@wcs_present
515
def test_current_forms_cell_check_validity(context):
516
    page = Page.objects.create(title='xxx', slug='test_current_forms_cell_render', template_name='standard')
517
    cell = WcsCurrentFormsCell.objects.create(page=page, placeholder='content', order=0)
518

  
519
    # no category
520
    cell.check_validity()
521
    assert ValidityInfo.objects.exists() is False
522

  
523
    # valid categories
524
    cell.categories = {'data': ['default:test-3', 'default:test-9']}
525
    cell.save()
526
    cell.check_validity()
527
    assert ValidityInfo.objects.exists() is False
528

  
529
    # can not retrieve categories, don't set cell as invalid
530
    with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
531
        mock_json = mock.Mock(status_code=404)
532
        requests_get.return_value = mock_json
533
        cell.check_validity()
534
    assert ValidityInfo.objects.exists() is False
535

  
536
    # invalid category
537
    cell.categories = {'data': ['default:foobar', 'default:test-9']}
538
    cell.save()
539
    cell.check_validity()
540
    validity_info = ValidityInfo.objects.latest('pk')
541
    assert validity_info.invalid_reason_code == 'wcs_category_not_found'
542
    assert validity_info.invalid_since is not None
543

  
544

  
438 545
@wcs_present
439 546
def test_current_forms_cell_render_single_site(context):
440 547
    page = Page(title='xxx', slug='test_current_forms_cell_render', template_name='standard')
......
542 649
    assert 'other' in data
543 650

  
544 651

  
652
@wcs_present
653
def test_care_forms_cell_validity(context):
654
    page = Page.objects.create(title='xxx', slug='test_care_forms_cell_render', template_name='standard')
655
    cell = WcsCareFormsCell.objects.create(page=page, placeholder='content', order=0)
656
    context['synchronous'] = True  # to get fresh content
657
    with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
658
        mock_json = mock.Mock(status_code=200)
659
        requests_get.return_value = mock_json
660
        cell.get_data(context)
661
    assert ValidityInfo.objects.exists() is False
662

  
663
    # can not retrieve data, don't set cell as invalid
664
    with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
665
        mock_json = mock.Mock(status_code=500)
666
        requests_get.return_value = mock_json
667
        cell.get_data(context)
668
    assert ValidityInfo.objects.exists() is False
669

  
670
    with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
671
        mock_json = mock.Mock(status_code=404)
672
        requests_get.return_value = mock_json
673
        cell.get_data(context)
674
    validity_info = ValidityInfo.objects.latest('pk')
675
    assert validity_info.invalid_reason_code == 'wcs_data_failure'
676
    assert validity_info.invalid_since is not None
677

  
678

  
545 679
@wcs_present
546 680
def test_care_forms_cell_render_single_site(context):
547 681
    page = Page(title='xxx', slug='test_care_forms_cell_render', template_name='standard')
......
579 713
            (u'other:test-3', u'test2 : Test 3'),
580 714
            (u'other:test-9', u'test2 : Test 9')]
581 715

  
716

  
582 717
@wcs_present
583 718
def test_forms_of_category_cell_render(context):
584 719
    page = Page(title='xxx', slug='test_forms_of_category_cell_render', template_name='standard')
......
625 760
    assert cell.render_for_search() == ''
626 761
    assert len(list(cell.get_external_links_data())) == 2
627 762

  
763

  
764
@wcs_present
765
def test_forms_of_category_cell_validity(context):
766
    page = Page.objects.create(title='xxx', slug='test_forms_of_category_cell_render', template_name='standard')
767
    cell = WcsFormsOfCategoryCell.objects.create(page=page, placeholder='content', order=0)
768
    validity_info = ValidityInfo.objects.latest('pk')
769
    assert validity_info.invalid_reason_code == 'wcs_category_not_defined'
770
    assert validity_info.invalid_since is not None
771
    cell.category_reference = 'default:test-9'
772
    cell.save()
773
    assert ValidityInfo.objects.exists() is False
774

  
775
    with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
776
        mock_json = mock.Mock(status_code=404)
777
        requests_get.return_value = mock_json
778
        cell.render(context)
779
    validity_info = ValidityInfo.objects.latest('pk')
780
    assert validity_info.invalid_reason_code == 'wcs_data_failure'
781
    assert validity_info.invalid_since is not None
782
    cell.mark_as_valid()
783

  
784
    # can not retrieve data, don't set cell as invalid
785
    with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
786
        mock_json = mock.Mock(status_code=500)
787
        requests_get.return_value = mock_json
788
        cell.render(context)
789
    assert ValidityInfo.objects.exists() is False
790

  
791
    # can not retrieve categories, don't set cell as invalid
792
    with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
793
        mock_json = mock.Mock(status_code=404)
794
        requests_get.return_value = mock_json
795
        cell.save()
796
    assert ValidityInfo.objects.exists() is False
797

  
798
    cell.category_reference = 'default:foobar'
799
    cell.save()
800
    validity_info = ValidityInfo.objects.latest('pk')
801
    assert validity_info.invalid_reason_code == 'wcs_category_not_found'
802
    assert validity_info.invalid_since is not None
803

  
804

  
628 805
@wcs_present
629 806
def test_current_drafts_cell_render_unlogged(context):
630 807
    page = Page(title='xxx', slug='test_current_drafts_cell_render', template_name='standard')
......
666 843
    assert len(extra_context['drafts']) == 0
667 844

  
668 845

  
846
@wcs_present
847
def test_current_drafts_cell_check_validity(context):
848
    page = Page.objects.create(title='xxx', slug='test_current_forms_cell_render', template_name='standard')
849
    cell = WcsCurrentDraftsCell.objects.create(page=page, placeholder='content', order=0)
850

  
851
    # no category
852
    cell.check_validity()
853
    assert ValidityInfo.objects.exists() is False
854

  
855
    # valid categories
856
    cell.categories = {'data': ['default:test-3', 'default:test-9']}
857
    cell.save()
858
    cell.check_validity()
859
    assert ValidityInfo.objects.exists() is False
860

  
861
    # can not retrieve vategories, don't set cell as invalid
862
    with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
863
        mock_json = mock.Mock(status_code=404)
864
        requests_get.return_value = mock_json
865
        cell.check_validity()
866
    assert ValidityInfo.objects.exists() is False
867

  
868
    # invalid category
869
    cell.categories = {'data': ['default:foobar', 'default:test-9']}
870
    cell.save()
871
    cell.check_validity()
872
    validity_info = ValidityInfo.objects.latest('pk')
873
    assert validity_info.invalid_reason_code == 'wcs_category_not_found'
874
    assert validity_info.invalid_since is not None
875

  
876

  
669 877
@wcs_present
670 878
def test_manager_forms_of_category_cell(app, admin_user):
671 879
    Page.objects.all().delete()
......
971 1179
    assert WcsFormCell.objects.count() == 0
972 1180

  
973 1181

  
1182
@wcs_present
974 1183
def test_import_export_pages_with_links():
975 1184
    page = Page(title=u'bar', slug='bar', order=1)
976 1185
    page.save()
......
998 1207
    assert new_item.cached_title == item.cached_title
999 1208
    assert new_item.cached_url == item.cached_url
1000 1209
    assert new_item.cached_json == item.cached_json
1210

  
1211

  
1212
@wcs_present
1213
def test_hourly():
1214
    appconfig = apps.get_app_config('wcs')
1215
    page = Page.objects.create(title='xxx', slug='test_current_forms_cell_render', template_name='standard')
1216
    cell_classes = [c for c in appconfig.get_models() if c in get_cell_classes()]
1217
    for klass in cell_classes:
1218
        klass.objects.create(page=page, placeholder='content', order=0)
1219
    for klass in cell_classes:
1220
        if klass in [WcsCurrentFormsCell, WcsCurrentDraftsCell]:
1221
            with mock.patch('combo.apps.wcs.models.%s.check_validity' % klass.__name__) as check_validity:
1222
                appconfig.hourly()
1223
            assert check_validity.call_args_list == [mock.call()]
1224
        else:
1225
            assert hasattr(klass, 'check_validity') is False
1001
-