0004-cells-w.c.s.-cells-invalid-report-38009.patch

Lauréline Guérin, 21 février 2020 11:14

Voir les différences: en ligne côte à côte

Subject: [PATCH 4/6] cells: w.c.s. cells invalid report (#38009)

 combo/apps/wcs/__init__.py |   9 ++
 combo/apps/wcs/models.py   | 151 +++++++++++++++++++-----
 tests/test_wcs.py          | 232 +++++++++++++++++++++++++++++++++++++
 3 files changed, 361 insertions(+), 31 deletions(-)

             return urls.urlpatterns
         def hourly(self):
             from combo.data.library import get_cell_classes
             from combo.data.models import CellBase
             cell_classes = [c for c in self.get_models() if c in get_cell_classes()]
             for cell in CellBase.get_cells(cell_filter=lambda x: x in cell_classes, page__snapshot__isnull=True):
                 if hasattr(cell, 'check_validity'):
                     cell.check_validity()
             self.update_db_cache()
         def update_db_cache(self):
-...
                     cell_filter=lambda x: x in (WcsFormCell, WcsCategoryCell, WcsFormsOfCategoryCell)):
                 cell.save()
     default_app_config = 'combo.apps.wcs.AppConfig'

     import copy
     import logging
     from django import template
     from django.conf import settings
     from django.db import models
     from django.forms import models as model_forms
-...
     from .utils import get_wcs_json, is_wcs_enabled, get_wcs_services
     invalid_reason_codes = {
         'wcs_form_not_defined': _('No form set'),
         'wcs_form_not_found': _('Invalid form'),
         'wcs_category_not_defined': _('No category set'),
         'wcs_category_not_found': _('Invalid category'),
         'wcs_data_failure': _('Failed to get data'),
+    }
     @register_cell_class
     class WcsFormCell(CellBase):
         formdef_reference = models.CharField(_('Form'), max_length=150)
-...
         add_link_label = _('New form link')
         edit_link_label = _('Edit form link')
         add_as_link_code = 'form-link'
         invalid_reason_codes = invalid_reason_codes
         is_enabled = classmethod(is_wcs_enabled)
-...
             return WcsFormCellForm
         def save(self, *args, **kwargs):
             if self.formdef_reference:
                 wcs_key, form_slug = self.formdef_reference.split(':')
                 wcs_site = get_wcs_services().get(wcs_key)
                 forms_response_json = get_wcs_json(wcs_site, 'api/formdefs/')
                 if isinstance(forms_response_json, dict):
                     # forward compability with future w.c.s. API
                     forms_response_json = forms_response_json.get('data')
                 for form in forms_response_json:
                     slug = form.get('slug')
                     if slug == form_slug:
                         self.cached_title = form.get('title')
                         self.cached_url = form.get('url')
                         self.cached_json = form
             if 'update_fields' in kwargs:
                 # don't populate the cache
                 return super(WcsFormCell, self).save(*args, **kwargs)
             def populate_cache():
                 if self.formdef_reference:
                     wcs_key, form_slug = self.formdef_reference.split(':')
                     wcs_site = get_wcs_services().get(wcs_key)
                     forms_response_json = get_wcs_json(wcs_site, 'api/formdefs/')
                     if not forms_response_json:
                         # can not retrieve data, don't report cell as invalid
                         self.mark_as_valid(save=False)
                         return
                     if isinstance(forms_response_json, dict):
                         # forward compability with future w.c.s. API
                         forms_response_json = forms_response_json.get('data')
                     form_found = False
                     for form in forms_response_json:
                         slug = form.get('slug')
                         if slug == form_slug:
                             self.cached_title = form.get('title')
                             self.cached_url = form.get('url')
                             self.cached_json = form
                             form_found = True
                             break
                     if form_found:
                         self.mark_as_valid(save=False)
                     else:
                         self.mark_as_invalid('wcs_form_not_found', save=False)
                 else:
                     self.mark_as_invalid('wcs_form_not_defined', save=False)
             populate_cache()
             return super(WcsFormCell, self).save(*args, **kwargs)
         def get_cell_extra_context(self, context):
-...
         cached_description = models.TextField(_('Description'), blank=True)
         cached_url = models.URLField(_('Cached URL'))
         invalid_reason_codes = invalid_reason_codes
         class Meta:
             abstract = True
         def save(self, *args, **kwargs):
             if self.category_reference:
                 wcs_key, category_slug = self.category_reference.split(':')
                 wcs_site = get_wcs_services().get(wcs_key)
                 categories_response_json = get_wcs_json(wcs_site, 'api/categories/')
                 for category in categories_response_json.get('data'):
                     slug = category.get('slug')
                     if slug == category_slug:
                         self.cached_title = category.get('title')
                         self.cached_description = category.get('description') or ''
                         self.cached_url = category.get('url')
             if 'update_fields' in kwargs:
                 # don't populate the cache
                 return super(WcsCommonCategoryCell, self).save(*args, **kwargs)
             def populate_cache():
                 if self.category_reference:
                     wcs_key, category_slug = self.category_reference.split(':')
                     wcs_site = get_wcs_services().get(wcs_key)
                     categories_response_json = get_wcs_json(wcs_site, 'api/categories/')
                     if not categories_response_json:
                         # can not retrieve data, don't report cell as invalid
                         return
                     category_found = False
                     for category in categories_response_json.get('data', []):
                         slug = category.get('slug')
                         if slug == category_slug:
                             self.cached_title = category.get('title')
                             self.cached_description = category.get('description') or ''
                             self.cached_url = category.get('url')
                             category_found = True
                             break
                     if category_found:
                         self.mark_as_valid(save=False)
                     else:
                         self.mark_as_invalid('wcs_category_not_found', save=False)
                 else:
                     self.mark_as_invalid('wcs_category_not_defined', save=False)
             populate_cache()
             return super(WcsCommonCategoryCell, self).save(*args, **kwargs)
         def get_additional_label(self):
-...
         is_enabled = classmethod(is_wcs_enabled)
         cache_duration = 5
         api_url = None
         warn_on_404 = True
         warn_on_4xx = True
         invalid_reason_codes = invalid_reason_codes
         def get_api_url(self, context):
             return self.api_url
-...
                     for item in wcs_site['data']:
                         item['site_slug'] = wcs_slug
             if not 200 in returns: # not a single valid answer
                 if returns != set([404]) or self.warn_on_404:
                     logging.error('failed to get data from any %s (%r)', api_url, returns)
             if 200 not in returns:  # not a single valid answer
                 logging.debug('failed to get data from any %s (%r)', api_url, returns)
                 if all([400 <= r < 500 for r in returns]) and self.warn_on_4xx:
                     # only 4xx errors, not a user cell, report the cell as invalid
                     self.mark_as_invalid('wcs_data_failure', force=False)
                 else:
                     self.mark_as_valid()
             else:
                 self.mark_as_valid()
             return wcs_sites
-...
     class WcsUserDataBaseCell(WcsDataBaseCell):
         warn_on_404 = False
         warn_on_4xx = False
         user_dependant = True
         class Meta:
-...
             return super(WcsUserDataBaseCell, self).is_visible(user)
     class CategoriesValidityMixin(object):
         def check_validity(self):
             categories = self.categories.get('data', [])
             if not categories:
                 return
             for category_reference in categories:
                 wcs_key, category_slug = category_reference.split(':')
                 wcs_site = get_wcs_services().get(wcs_key)
                 categories_response_json = get_wcs_json(wcs_site, 'api/categories/')
                 if not categories_response_json:
                     # can not retrieve data, don't report cell as invalid
                     continue
                 category_found = any(
                     [c.get('slug') == category_slug
                      for c in categories_response_json.get('data', [])])
                 if not category_found:
                     self.mark_as_invalid('wcs_category_not_found')
                     return
             self.mark_as_valid()
     @register_cell_class
     class WcsCurrentFormsCell(WcsUserDataBaseCell):
     class WcsCurrentFormsCell(WcsUserDataBaseCell, CategoriesValidityMixin):
         variable_name = 'user_forms'
         categories = JSONField(_('Categories'), blank=True)
-...
             return context
     @register_cell_class
     class WcsCurrentDraftsCell(WcsUserDataBaseCell):
     class WcsCurrentDraftsCell(WcsUserDataBaseCell, CategoriesValidityMixin):
         variable_name = 'current_drafts'
         template_name = 'combo/wcs/current_drafts.html'

     import mock
     from combo.data.library import get_cell_classes
     from combo.data.models import CellBase, LinkListCell, Page
     from combo.apps.search.engines import engines
     from combo.apps.wcs.models import (WcsFormCell, WcsCurrentFormsCell,
-...
         appconfig.update_db_cache()
         assert WcsFormCell.objects.get(id=cell.id).cached_title == 'form title'
     @wcs_present
     def test_form_cell_validity():
         page = Page.objects.create(title='xxx', slug='test_form_cell_save_cache', template_name='standard')
         cell = WcsFormCell.objects.create(page=page, placeholder='content', order=0)
         assert cell.invalid_reason_code == 'wcs_form_not_defined'
         assert cell.invalid_since is not None
         cell.formdef_reference = u'default:form-title'
         cell.save()
         assert cell.invalid_reason_code is None
         assert cell.invalid_since is None
         # can not retrieve formdefs, don't set cell as invalid
         with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
             mock_json = mock.Mock(status_code=404)
             requests_get.return_value = mock_json
             cell.save()
         assert cell.invalid_reason_code is None
         assert cell.invalid_since is None
         cell.formdef_reference = u'default:foobar'
         cell.save()
         assert cell.invalid_reason_code == 'wcs_form_not_found'
         assert cell.invalid_since is not None
     @wcs_present
     def test_form_cell_load():
         page = Page(title='xxx', slug='test_form_cell_save_cache', template_name='standard')
-...
         assert cell.cached_title == 'Test 3'
         assert cell.get_additional_label() == 'Test 3'
     @wcs_present
     def test_category_cell_validity():
         page = Page.objects.create(title='xxx', slug='test_category_cell_save_cache', template_name='standard')
         cell = WcsCategoryCell.objects.create(page=page, placeholder='content', order=0)
         assert cell.invalid_reason_code == 'wcs_category_not_defined'
         assert cell.invalid_since is not None
         cell.category_reference = u'default:test-3'
         cell.save()
         assert cell.invalid_reason_code is None
         assert cell.invalid_since is None
         # can not retrieve categories, don't set cell as invalid
         with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
             mock_json = mock.Mock(status_code=404)
             requests_get.return_value = mock_json
             cell.save()
         assert cell.invalid_reason_code is None
         assert cell.invalid_since is None
         cell.category_reference = 'default:foobar'
         cell.save()
         assert cell.invalid_reason_code == 'wcs_category_not_found'
         assert cell.invalid_since is not None
     @wcs_present
     def test_form_cell_render():
         page = Page(title='xxx', slug='test_form_cell_render', template_name='standard')
-...
         # check include drafts
         cell.include_drafts = False
         with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
             mock_json = mock.Mock(status_code=200)
             requests_get.return_value = mock_json
             cell.get_cell_extra_context(context)
         assert requests_get.call_args_list[0][0][0] == '/api/user/forms?limit=100&sort=desc'
         cell.include_drafts = True
         with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
             mock_json = mock.Mock(status_code=200)
             requests_get.return_value = mock_json
             cell.get_cell_extra_context(context)
         assert requests_get.call_args_list[0][0][0] == '/api/user/forms?limit=100&sort=desc&include-drafts=on'
     @wcs_present
     def test_current_forms_cell_validity(context):
         page = Page.objects.create(title='xxx', slug='test_current_forms_cell_render', template_name='standard')
         cell = WcsCurrentFormsCell.objects.create(page=page, placeholder='content', order=0)
         context['request'].user = MockUser()
         context['synchronous'] = True  # to get fresh content
         cell.get_data(context)
         assert cell.invalid_reason_code is None
         assert cell.invalid_since is None
         with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
             mock_json = mock.Mock(status_code=404)
             requests_get.return_value = mock_json
             cell.get_data(context)
         assert cell.invalid_reason_code is None  # error not reported
         assert cell.invalid_since is None
     @wcs_present
     def test_current_forms_cell_check_validity(context):
         page = Page.objects.create(title='xxx', slug='test_current_forms_cell_render', template_name='standard')
         cell = WcsCurrentFormsCell.objects.create(page=page, placeholder='content', order=0)
         # no category
         cell.check_validity()
         assert cell.invalid_reason_code is None
         assert cell.invalid_since is None
         # valid categories
         cell.categories = {'data': ['default:test-3', 'default:test-9']}
         cell.save()
         cell.check_validity()
         assert cell.invalid_reason_code is None
         assert cell.invalid_since is None
         # can not retrieve categories, don't set cell as invalid
         with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
             mock_json = mock.Mock(status_code=404)
             requests_get.return_value = mock_json
             cell.check_validity()
         assert cell.invalid_reason_code is None
         assert cell.invalid_since is None
         # invalid category
         cell.categories = {'data': ['default:foobar', 'default:test-9']}
         cell.save()
         cell.check_validity()
         assert cell.invalid_reason_code == 'wcs_category_not_found'
         assert cell.invalid_since is not None
     @wcs_present
     def test_current_forms_cell_render_single_site(context):
         page = Page(title='xxx', slug='test_current_forms_cell_render', template_name='standard')
-...
         assert 'other' in data
     @wcs_present
     def test_care_forms_cell_validity(context):
         page = Page.objects.create(title='xxx', slug='test_care_forms_cell_render', template_name='standard')
         cell = WcsCareFormsCell.objects.create(page=page, placeholder='content', order=0)
         context['synchronous'] = True  # to get fresh content
         with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
             mock_json = mock.Mock(status_code=200)
             requests_get.return_value = mock_json
             cell.get_data(context)
         assert cell.invalid_reason_code is None
         assert cell.invalid_since is None
         # can not retrieve data, don't set cell as invalid
         with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
             mock_json = mock.Mock(status_code=500)
             requests_get.return_value = mock_json
             cell.get_data(context)
         assert cell.invalid_reason_code is None
         assert cell.invalid_since is None
         with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
             mock_json = mock.Mock(status_code=404)
             requests_get.return_value = mock_json
             cell.get_data(context)
         assert cell.invalid_reason_code == 'wcs_data_failure'
         assert cell.invalid_since is not None
     @wcs_present
     def test_care_forms_cell_render_single_site(context):
         page = Page(title='xxx', slug='test_care_forms_cell_render', template_name='standard')
-...
                 (u'other:test-3', u'test2 : Test 3'),
                 (u'other:test-9', u'test2 : Test 9')]
     @wcs_present
     def test_forms_of_category_cell_render(context):
         page = Page(title='xxx', slug='test_forms_of_category_cell_render', template_name='standard')
-...
         assert cell.render_for_search() == ''
         assert len(list(cell.get_external_links_data())) == 2
     @wcs_present
     def test_forms_of_category_cell_validity(context):
         page = Page.objects.create(title='xxx', slug='test_forms_of_category_cell_render', template_name='standard')
         cell = WcsFormsOfCategoryCell.objects.create(page=page, placeholder='content', order=0)
         assert cell.invalid_reason_code == 'wcs_category_not_defined'
         assert cell.invalid_since is not None
         cell.category_reference = 'default:test-9'
         cell.save()
         assert cell.invalid_reason_code is None
         assert cell.invalid_since is None
         with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
             mock_json = mock.Mock(status_code=404)
             requests_get.return_value = mock_json
             cell.render(context)
         assert cell.invalid_reason_code == 'wcs_data_failure'
         assert cell.invalid_since is not None
         cell.mark_as_valid()
         # can not retrieve data, don't set cell as invalid
         with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
             mock_json = mock.Mock(status_code=500)
             requests_get.return_value = mock_json
             cell.render(context)
         assert cell.invalid_reason_code is None
         assert cell.invalid_since is None
         # can not retrieve categories, don't set cell as invalid
         with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
             mock_json = mock.Mock(status_code=404)
             requests_get.return_value = mock_json
             cell.save()
         assert cell.invalid_reason_code is None
         assert cell.invalid_since is None
         cell.category_reference = 'default:foobar'
         cell.save()
         assert cell.invalid_reason_code == 'wcs_category_not_found'
         assert cell.invalid_since is not None
     @wcs_present
     def test_current_drafts_cell_render_unlogged(context):
         page = Page(title='xxx', slug='test_current_drafts_cell_render', template_name='standard')
-...
         assert len(extra_context['drafts']) == 0
     @wcs_present
     def test_current_drafts_cell_check_validity(context):
         page = Page.objects.create(title='xxx', slug='test_current_forms_cell_render', template_name='standard')
         cell = WcsCurrentDraftsCell.objects.create(page=page, placeholder='content', order=0)
         # no category
         cell.check_validity()
         assert cell.invalid_reason_code is None
         assert cell.invalid_since is None
         # valid categories
         cell.categories = {'data': ['default:test-3', 'default:test-9']}
         cell.save()
         cell.check_validity()
         assert cell.invalid_reason_code is None
         assert cell.invalid_since is None
         # can not retrieve vategories, don't set cell as invalid
         with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
             mock_json = mock.Mock(status_code=404)
             requests_get.return_value = mock_json
             cell.check_validity()
         assert cell.invalid_reason_code is None
         assert cell.invalid_since is None
         # invalid category
         cell.categories = {'data': ['default:foobar', 'default:test-9']}
         cell.save()
         cell.check_validity()
         assert cell.invalid_reason_code == 'wcs_category_not_found'
         assert cell.invalid_since is not None
     @wcs_present
     def test_manager_forms_of_category_cell(app, admin_user):
         Page.objects.all().delete()
-...
         assert WcsFormCell.objects.count() == 0
     @wcs_present
     def test_import_export_pages_with_links():
         page = Page(title=u'bar', slug='bar', order=1)
         page.save()
-...
         assert new_item.cached_title == item.cached_title
         assert new_item.cached_url == item.cached_url
         assert new_item.cached_json == item.cached_json
     @wcs_present
     def test_hourly():
         appconfig = apps.get_app_config('wcs')
         page = Page.objects.create(title='xxx', slug='test_current_forms_cell_render', template_name='standard')
         cell_classes = [c for c in appconfig.get_models() if c in get_cell_classes()]
         for klass in cell_classes:
             klass.objects.create(page=page, placeholder='content', order=0)
         for klass in cell_classes:
             if klass in [WcsCurrentFormsCell, WcsCurrentDraftsCell]:
                 with mock.patch('combo.apps.wcs.models.%s.check_validity' % klass.__name__) as check_validity:
                     appconfig.hourly()
                 assert check_validity.call_args_list == [mock.call()]
             else:
                 assert hasattr(klass, 'check_validity') is False
+    -

Projet

Général

Profil

Produits Entr'ouvert » Combo

0004-cells-w.c.s.-cells-invalid-report-38009.patch