Projet

Général

Profil

0004-cells-w.c.s.-cells-invalid-report-38009.patch

Lauréline Guérin, 03 mars 2020 14:49

Télécharger (23,7 ko)

Voir les différences:

Subject: [PATCH 4/6] cells: w.c.s. cells invalid report (#38009)

 combo/apps/wcs/__init__.py |   9 ++
 combo/apps/wcs/models.py   | 169 +++++++++++++++++++++------
 tests/test_wcs.py          | 227 ++++++++++++++++++++++++++++++++++++-
 3 files changed, 371 insertions(+), 34 deletions(-)
combo/apps/wcs/__init__.py
65 65
        return urls.urlpatterns
66 66

  
67 67
    def hourly(self):
68
        from combo.data.library import get_cell_classes
69
        from combo.data.models import CellBase
70

  
71
        cell_classes = [c for c in self.get_models() if c in get_cell_classes()]
72
        for cell in CellBase.get_cells(cell_filter=lambda x: x in cell_classes, page__snapshot__isnull=True):
73
            if hasattr(cell, 'check_validity'):
74
                cell.check_validity()
75

  
68 76
        self.update_db_cache()
69 77

  
70 78
    def update_db_cache(self):
......
74 82
                cell_filter=lambda x: x in (WcsFormCell, WcsCategoryCell, WcsFormsOfCategoryCell)):
75 83
            cell.save()
76 84

  
85

  
77 86
default_app_config = 'combo.apps.wcs.AppConfig'
combo/apps/wcs/models.py
19 19
import copy
20 20
import logging
21 21

  
22
from django import template
23 22
from django.conf import settings
24 23
from django.db import models
25 24
from django.forms import models as model_forms
......
34 33

  
35 34
from .utils import get_wcs_json, is_wcs_enabled, get_wcs_services
36 35

  
36

  
37
invalid_reason_codes = {
38
    'wcs_form_not_defined': _('No form set'),
39
    'wcs_form_not_found': _('Invalid form'),
40
    'wcs_category_not_defined': _('No category set'),
41
    'wcs_category_not_found': _('Invalid category'),
42
    'wcs_data_failure': _('Failed to get data'),
43
}
44

  
45

  
37 46
@register_cell_class
38 47
class WcsFormCell(CellBase):
39 48
    formdef_reference = models.CharField(_('Form'), max_length=150)
......
47 56
    add_link_label = _('New form link')
48 57
    edit_link_label = _('Edit form link')
49 58
    add_as_link_code = 'form-link'
59
    invalid_reason_codes = invalid_reason_codes
50 60

  
51 61
    is_enabled = classmethod(is_wcs_enabled)
52 62

  
......
58 68
        return WcsFormCellForm
59 69

  
60 70
    def save(self, *args, **kwargs):
61
        if self.formdef_reference:
62
            wcs_key, form_slug = self.formdef_reference.split(':')
63
            wcs_site = get_wcs_services().get(wcs_key)
64
            forms_response_json = get_wcs_json(wcs_site, 'api/formdefs/')
65
            if isinstance(forms_response_json, dict):
66
                # forward compability with future w.c.s. API
67
                forms_response_json = forms_response_json.get('data')
68
            for form in forms_response_json:
69
                slug = form.get('slug')
70
                if slug == form_slug:
71
                    self.cached_title = form.get('title')
72
                    self.cached_url = form.get('url')
73
                    self.cached_json = form
74
        return super(WcsFormCell, self).save(*args, **kwargs)
71
        if 'update_fields' in kwargs:
72
            # don't populate the cache
73
            return super(WcsFormCell, self).save(*args, **kwargs)
74

  
75
        def populate_cache():
76
            if self.formdef_reference:
77
                wcs_key, form_slug = self.formdef_reference.split(':')
78
                wcs_site = get_wcs_services().get(wcs_key)
79
                forms_response_json = get_wcs_json(wcs_site, 'api/formdefs/')
80

  
81
                if not forms_response_json:
82
                    # can not retrieve data, don't report cell as invalid
83
                    self.mark_as_valid()
84
                    return
85

  
86
                if isinstance(forms_response_json, dict):
87
                    # forward compability with future w.c.s. API
88
                    forms_response_json = forms_response_json.get('data')
89

  
90
                form_found = False
91
                for form in forms_response_json or []:
92
                    slug = form.get('slug')
93
                    if slug == form_slug:
94
                        self.cached_title = form.get('title')
95
                        self.cached_url = form.get('url')
96
                        self.cached_json = form
97
                        form_found = True
98
                        break
99
                if form_found:
100
                    self.mark_as_valid()
101
                    return
102
                else:
103
                    return self.mark_as_invalid('wcs_form_not_found', save=False)
104
            else:
105
                return self.mark_as_invalid('wcs_form_not_defined', save=False)
106

  
107
        validity_info = populate_cache()
108
        result = super(WcsFormCell, self).save(*args, **kwargs)
109
        if validity_info:
110
            # save validity_info after Cell insert
111
            validity_info.content_object = self
112
            validity_info.save()
113
        return result
75 114

  
76 115
    def get_cell_extra_context(self, context):
77 116
        context = super(WcsFormCell, self).get_cell_extra_context(context)
......
129 168
    cached_description = models.TextField(_('Description'), blank=True)
130 169
    cached_url = models.URLField(_('Cached URL'))
131 170

  
171
    invalid_reason_codes = invalid_reason_codes
172

  
132 173
    class Meta:
133 174
        abstract = True
134 175

  
135 176
    def save(self, *args, **kwargs):
136
        if self.category_reference:
137
            wcs_key, category_slug = self.category_reference.split(':')
138
            wcs_site = get_wcs_services().get(wcs_key)
139
            categories_response_json = get_wcs_json(wcs_site, 'api/categories/')
140
            for category in categories_response_json.get('data'):
141
                slug = category.get('slug')
142
                if slug == category_slug:
143
                    self.cached_title = category.get('title')
144
                    self.cached_description = category.get('description') or ''
145
                    self.cached_url = category.get('url')
146
        return super(WcsCommonCategoryCell, self).save(*args, **kwargs)
177
        if 'update_fields' in kwargs:
178
            # don't populate the cache
179
            return super(WcsCommonCategoryCell, self).save(*args, **kwargs)
180

  
181
        def populate_cache():
182
            if self.category_reference:
183
                wcs_key, category_slug = self.category_reference.split(':')
184
                wcs_site = get_wcs_services().get(wcs_key)
185
                categories_response_json = get_wcs_json(wcs_site, 'api/categories/')
186

  
187
                if not categories_response_json:
188
                    # can not retrieve data, don't report cell as invalid
189
                    self.mark_as_valid()
190
                    return
191

  
192
                category_found = False
193
                for category in categories_response_json.get('data', []):
194
                    slug = category.get('slug')
195
                    if slug == category_slug:
196
                        self.cached_title = category.get('title')
197
                        self.cached_description = category.get('description') or ''
198
                        self.cached_url = category.get('url')
199
                        category_found = True
200
                        break
201
                if category_found:
202
                    self.mark_as_valid()
203
                    return
204
                else:
205
                    return self.mark_as_invalid('wcs_category_not_found', save=False)
206
            else:
207
                return self.mark_as_invalid('wcs_category_not_defined', save=False)
208

  
209
        validity_info = populate_cache()
210
        result = super(WcsCommonCategoryCell, self).save(*args, **kwargs)
211
        if validity_info:
212
            # save validity_info after Cell insert
213
            validity_info.content_object = self
214
            validity_info.save()
215
        return result
147 216

  
148 217
    def get_additional_label(self):
149 218
        if not self.cached_title:
......
192 261
    is_enabled = classmethod(is_wcs_enabled)
193 262
    cache_duration = 5
194 263
    api_url = None
195
    warn_on_404 = True
264
    warn_on_4xx = True
265
    invalid_reason_codes = invalid_reason_codes
196 266

  
197 267
    def get_api_url(self, context):
198 268
        return self.api_url
......
242 312
                for item in wcs_site['data']:
243 313
                    item['site_slug'] = wcs_slug
244 314

  
245
        if not 200 in returns: # not a single valid answer
246
            if returns != set([404]) or self.warn_on_404:
247
                logging.error('failed to get data from any %s (%r)', api_url, returns)
315
        if 200 not in returns:  # not a single valid answer
316
            logging.debug('failed to get data from any %s (%r)', api_url, returns)
317
            if all([400 <= r < 500 for r in returns]) and self.warn_on_4xx:
318
                # only 4xx errors, not a user cell, report the cell as invalid
319
                self.mark_as_invalid('wcs_data_failure', force=False)
320
            else:
321
                self.mark_as_valid()
322
        else:
323
            self.mark_as_valid()
248 324

  
249 325
        return wcs_sites
250 326

  
......
293 369

  
294 370

  
295 371
class WcsUserDataBaseCell(WcsDataBaseCell):
296
    warn_on_404 = False
372
    warn_on_4xx = False
297 373
    user_dependant = True
298 374

  
299 375
    class Meta:
......
305 381
        return super(WcsUserDataBaseCell, self).is_visible(user)
306 382

  
307 383

  
384
class CategoriesValidityMixin(object):
385
    def check_validity(self):
386
        categories = self.categories.get('data', [])
387

  
388
        if not categories:
389
            return
390

  
391
        for category_reference in categories:
392
            wcs_key, category_slug = category_reference.split(':')
393
            wcs_site = get_wcs_services().get(wcs_key)
394
            categories_response_json = get_wcs_json(wcs_site, 'api/categories/')
395

  
396
            if not categories_response_json:
397
                # can not retrieve data, don't report cell as invalid
398
                continue
399

  
400
            category_found = any(
401
                [c.get('slug') == category_slug
402
                 for c in categories_response_json.get('data', [])])
403
            if not category_found:
404
                self.mark_as_invalid('wcs_category_not_found')
405
                return
406

  
407
        self.mark_as_valid()
408

  
409

  
308 410
@register_cell_class
309
class WcsCurrentFormsCell(WcsUserDataBaseCell):
411
class WcsCurrentFormsCell(WcsUserDataBaseCell, CategoriesValidityMixin):
310 412
    variable_name = 'user_forms'
311 413

  
312 414
    categories = JSONField(_('Categories'), blank=True)
......
400 502

  
401 503
        return context
402 504

  
505

  
403 506
@register_cell_class
404
class WcsCurrentDraftsCell(WcsUserDataBaseCell):
507
class WcsCurrentDraftsCell(WcsUserDataBaseCell, CategoriesValidityMixin):
405 508
    variable_name = 'current_drafts'
406 509
    template_name = 'combo/wcs/current_drafts.html'
407 510

  
tests/test_wcs.py
22 22

  
23 23
import mock
24 24

  
25
from combo.data.models import CellBase, LinkListCell, Page
25
from combo.data.library import get_cell_classes
26
from combo.data.models import CellBase, LinkListCell, Page, ValidityInfo
26 27
from combo.apps.search.engines import engines
27 28
from combo.apps.wcs.models import (WcsFormCell, WcsCurrentFormsCell,
28 29
        WcsFormsOfCategoryCell, WcsCurrentDraftsCell, WcsCategoryCell,
......
299 300
    appconfig.update_db_cache()
300 301
    assert WcsFormCell.objects.get(id=cell.id).cached_title == 'form title'
301 302

  
303

  
304
@wcs_present
305
def test_form_cell_validity():
306
    page = Page.objects.create(title='xxx', slug='test_form_cell_save_cache', template_name='standard')
307
    cell = WcsFormCell.objects.create(page=page, placeholder='content', order=0)
308
    validity_info = ValidityInfo.objects.latest('pk')
309
    assert validity_info.invalid_reason_code == 'wcs_form_not_defined'
310
    assert validity_info.invalid_since is not None
311

  
312
    cell.formdef_reference = u'default:form-title'
313
    cell.save()
314
    assert ValidityInfo.objects.exists() is False
315

  
316
    # can not retrieve formdefs, don't set cell as invalid
317
    with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
318
        mock_json = mock.Mock(status_code=404)
319
        requests_get.return_value = mock_json
320
        cell.save()
321
    assert ValidityInfo.objects.exists() is False
322

  
323
    cell.formdef_reference = u'default:foobar'
324
    cell.save()
325
    validity_info = ValidityInfo.objects.latest('pk')
326
    assert validity_info.invalid_reason_code == 'wcs_form_not_found'
327
    assert validity_info.invalid_since is not None
328

  
329

  
302 330
@wcs_present
303 331
def test_form_cell_load():
304 332
    page = Page(title='xxx', slug='test_form_cell_save_cache', template_name='standard')
......
328 356
    assert cell.cached_title == 'Test 3'
329 357
    assert cell.get_additional_label() == 'Test 3'
330 358

  
359

  
360
@wcs_present
361
def test_category_cell_validity():
362
    page = Page.objects.create(title='xxx', slug='test_category_cell_save_cache', template_name='standard')
363
    cell = WcsCategoryCell.objects.create(page=page, placeholder='content', order=0)
364
    validity_info = ValidityInfo.objects.latest('pk')
365
    assert validity_info.invalid_reason_code == 'wcs_category_not_defined'
366
    assert validity_info.invalid_since is not None
367

  
368
    cell.category_reference = u'default:test-3'
369
    cell.save()
370
    assert ValidityInfo.objects.exists() is False
371

  
372
    # can not retrieve categories, don't set cell as invalid
373
    with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
374
        mock_json = mock.Mock(status_code=404)
375
        requests_get.return_value = mock_json
376
        cell.save()
377
    assert ValidityInfo.objects.exists() is False
378

  
379
    cell.category_reference = 'default:foobar'
380
    cell.save()
381
    validity_info = ValidityInfo.objects.latest('pk')
382
    assert validity_info.invalid_reason_code == 'wcs_category_not_found'
383
    assert validity_info.invalid_since is not None
384

  
385

  
331 386
@wcs_present
332 387
def test_form_cell_render():
333 388
    page = Page(title='xxx', slug='test_form_cell_render', template_name='standard')
......
426 481
    # check include drafts
427 482
    cell.include_drafts = False
428 483
    with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
484
        mock_json = mock.Mock(status_code=200)
485
        requests_get.return_value = mock_json
429 486
        cell.get_cell_extra_context(context)
430 487
    assert requests_get.call_args_list[0][0][0] == '/api/user/forms?limit=100&sort=desc'
431 488

  
432 489
    cell.include_drafts = True
433 490
    with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
491
        mock_json = mock.Mock(status_code=200)
492
        requests_get.return_value = mock_json
434 493
        cell.get_cell_extra_context(context)
435 494
    assert requests_get.call_args_list[0][0][0] == '/api/user/forms?limit=100&sort=desc&include-drafts=on'
436 495

  
437 496

  
497
@wcs_present
498
def test_current_forms_cell_validity(context):
499
    page = Page.objects.create(title='xxx', slug='test_current_forms_cell_render', template_name='standard')
500
    cell = WcsCurrentFormsCell.objects.create(page=page, placeholder='content', order=0)
501
    context['request'].user = MockUser()
502
    context['synchronous'] = True  # to get fresh content
503

  
504
    cell.get_data(context)
505
    assert ValidityInfo.objects.exists() is False
506

  
507
    with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
508
        mock_json = mock.Mock(status_code=404)
509
        requests_get.return_value = mock_json
510
        cell.get_data(context)
511
    assert ValidityInfo.objects.exists() is False
512

  
513

  
514
@wcs_present
515
def test_current_forms_cell_check_validity(context):
516
    page = Page.objects.create(title='xxx', slug='test_current_forms_cell_render', template_name='standard')
517
    cell = WcsCurrentFormsCell.objects.create(page=page, placeholder='content', order=0)
518

  
519
    # no category
520
    cell.check_validity()
521
    assert ValidityInfo.objects.exists() is False
522

  
523
    # valid categories
524
    cell.categories = {'data': ['default:test-3', 'default:test-9']}
525
    cell.save()
526
    cell.check_validity()
527
    assert ValidityInfo.objects.exists() is False
528

  
529
    # can not retrieve categories, don't set cell as invalid
530
    with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
531
        mock_json = mock.Mock(status_code=404)
532
        requests_get.return_value = mock_json
533
        cell.check_validity()
534
    assert ValidityInfo.objects.exists() is False
535

  
536
    # invalid category
537
    cell.categories = {'data': ['default:foobar', 'default:test-9']}
538
    cell.save()
539
    cell.check_validity()
540
    validity_info = ValidityInfo.objects.latest('pk')
541
    assert validity_info.invalid_reason_code == 'wcs_category_not_found'
542
    assert validity_info.invalid_since is not None
543

  
544

  
438 545
@wcs_present
439 546
def test_current_forms_cell_render_single_site(context):
440 547
    page = Page(title='xxx', slug='test_current_forms_cell_render', template_name='standard')
......
542 649
    assert 'other' in data
543 650

  
544 651

  
652
@wcs_present
653
def test_care_forms_cell_validity(context):
654
    page = Page.objects.create(title='xxx', slug='test_care_forms_cell_render', template_name='standard')
655
    cell = WcsCareFormsCell.objects.create(page=page, placeholder='content', order=0)
656
    context['synchronous'] = True  # to get fresh content
657
    with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
658
        mock_json = mock.Mock(status_code=200)
659
        requests_get.return_value = mock_json
660
        cell.get_data(context)
661
    assert ValidityInfo.objects.exists() is False
662

  
663
    # can not retrieve data, don't set cell as invalid
664
    with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
665
        mock_json = mock.Mock(status_code=500)
666
        requests_get.return_value = mock_json
667
        cell.get_data(context)
668
    assert ValidityInfo.objects.exists() is False
669

  
670
    with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
671
        mock_json = mock.Mock(status_code=404)
672
        requests_get.return_value = mock_json
673
        cell.get_data(context)
674
    validity_info = ValidityInfo.objects.latest('pk')
675
    assert validity_info.invalid_reason_code == 'wcs_data_failure'
676
    assert validity_info.invalid_since is not None
677

  
678

  
545 679
@wcs_present
546 680
def test_care_forms_cell_render_single_site(context):
547 681
    page = Page(title='xxx', slug='test_care_forms_cell_render', template_name='standard')
......
579 713
            (u'other:test-3', u'test2 : Test 3'),
580 714
            (u'other:test-9', u'test2 : Test 9')]
581 715

  
716

  
582 717
@wcs_present
583 718
def test_forms_of_category_cell_render(context):
584 719
    page = Page(title='xxx', slug='test_forms_of_category_cell_render', template_name='standard')
......
625 760
    assert cell.render_for_search() == ''
626 761
    assert len(list(cell.get_external_links_data())) == 2
627 762

  
763

  
764
@wcs_present
765
def test_forms_of_category_cell_validity(context):
766
    page = Page.objects.create(title='xxx', slug='test_forms_of_category_cell_render', template_name='standard')
767
    cell = WcsFormsOfCategoryCell.objects.create(page=page, placeholder='content', order=0)
768
    validity_info = ValidityInfo.objects.latest('pk')
769
    assert validity_info.invalid_reason_code == 'wcs_category_not_defined'
770
    assert validity_info.invalid_since is not None
771
    cell.category_reference = 'default:test-9'
772
    cell.save()
773
    assert ValidityInfo.objects.exists() is False
774

  
775
    with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
776
        mock_json = mock.Mock(status_code=404)
777
        requests_get.return_value = mock_json
778
        cell.render(context)
779
    validity_info = ValidityInfo.objects.latest('pk')
780
    assert validity_info.invalid_reason_code == 'wcs_data_failure'
781
    assert validity_info.invalid_since is not None
782
    cell.mark_as_valid()
783

  
784
    # can not retrieve data, don't set cell as invalid
785
    with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
786
        mock_json = mock.Mock(status_code=500)
787
        requests_get.return_value = mock_json
788
        cell.render(context)
789
    assert ValidityInfo.objects.exists() is False
790

  
791
    # can not retrieve categories, don't set cell as invalid
792
    with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
793
        mock_json = mock.Mock(status_code=404)
794
        requests_get.return_value = mock_json
795
        cell.save()
796
    assert ValidityInfo.objects.exists() is False
797

  
798
    cell.category_reference = 'default:foobar'
799
    cell.save()
800
    validity_info = ValidityInfo.objects.latest('pk')
801
    assert validity_info.invalid_reason_code == 'wcs_category_not_found'
802
    assert validity_info.invalid_since is not None
803

  
804

  
628 805
@wcs_present
629 806
def test_current_drafts_cell_render_unlogged(context):
630 807
    page = Page(title='xxx', slug='test_current_drafts_cell_render', template_name='standard')
......
666 843
    assert len(extra_context['drafts']) == 0
667 844

  
668 845

  
846
@wcs_present
847
def test_current_drafts_cell_check_validity(context):
848
    page = Page.objects.create(title='xxx', slug='test_current_forms_cell_render', template_name='standard')
849
    cell = WcsCurrentDraftsCell.objects.create(page=page, placeholder='content', order=0)
850

  
851
    # no category
852
    cell.check_validity()
853
    assert ValidityInfo.objects.exists() is False
854

  
855
    # valid categories
856
    cell.categories = {'data': ['default:test-3', 'default:test-9']}
857
    cell.save()
858
    cell.check_validity()
859
    assert ValidityInfo.objects.exists() is False
860

  
861
    # can not retrieve vategories, don't set cell as invalid
862
    with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
863
        mock_json = mock.Mock(status_code=404)
864
        requests_get.return_value = mock_json
865
        cell.check_validity()
866
    assert ValidityInfo.objects.exists() is False
867

  
868
    # invalid category
869
    cell.categories = {'data': ['default:foobar', 'default:test-9']}
870
    cell.save()
871
    cell.check_validity()
872
    validity_info = ValidityInfo.objects.latest('pk')
873
    assert validity_info.invalid_reason_code == 'wcs_category_not_found'
874
    assert validity_info.invalid_since is not None
875

  
876

  
669 877
@wcs_present
670 878
def test_manager_forms_of_category_cell(app, admin_user):
671 879
    Page.objects.all().delete()
......
971 1179
    assert WcsFormCell.objects.count() == 0
972 1180

  
973 1181

  
1182
@wcs_present
974 1183
def test_import_export_pages_with_links():
975 1184
    page = Page(title=u'bar', slug='bar', order=1)
976 1185
    page.save()
......
998 1207
    assert new_item.cached_title == item.cached_title
999 1208
    assert new_item.cached_url == item.cached_url
1000 1209
    assert new_item.cached_json == item.cached_json
1210

  
1211

  
1212
@wcs_present
1213
def test_hourly():
1214
    appconfig = apps.get_app_config('wcs')
1215
    page = Page.objects.create(title='xxx', slug='test_current_forms_cell_render', template_name='standard')
1216
    cell_classes = [c for c in appconfig.get_models() if c in get_cell_classes()]
1217
    for klass in cell_classes:
1218
        klass.objects.create(page=page, placeholder='content', order=0)
1219
    for klass in cell_classes:
1220
        if klass in [WcsCurrentFormsCell, WcsCurrentDraftsCell]:
1221
            with mock.patch('combo.apps.wcs.models.%s.check_validity' % klass.__name__) as check_validity:
1222
                appconfig.hourly()
1223
            assert check_validity.call_args_list == [mock.call()]
1224
        else:
1225
            assert hasattr(klass, 'check_validity') is False
1001
-