0001-wcs-handle-network-error-on-requests-31438.patch

Lauréline Guérin, 05 octobre 2020 10:07

Voir les différences: en ligne côte à côte

Subject: [PATCH] wcs: handle network error on requests (#31438)

 combo/apps/wcs/models.py |  90 +++++++++++-----------
 combo/apps/wcs/utils.py  |  22 ++++--
 tests/test_wcs.py        | 161 ++++++++++++++++++++++++++++++++++-----
 3 files changed, 203 insertions(+), 70 deletions(-)

     from django.utils.translation import ugettext_lazy as _
     from jsonfield import JSONField
     from requests.exceptions import RequestException
     from combo.data.models import CellBase, Page
     from combo.data.library import register_cell_class
-...
                     wcs_site = get_wcs_services().get(wcs_key)
                     forms_response_json = get_wcs_json(wcs_site, 'api/formdefs/')
                     if not forms_response_json:
                     if not forms_response_json or forms_response_json.get('err') == 1:
                         # can not retrieve data, don't report cell as invalid
                         self.mark_as_valid()
                         return
                     if isinstance(forms_response_json, dict):
                         # forward compability with future w.c.s. API
                         forms_response_json = forms_response_json.get('data')
                     form_found = False
                     for form in forms_response_json or []:
                     for form in forms_response_json.get('data') or []:
                         slug = form.get('slug')
                         if slug == form_slug:
                             self.cached_title = form.get('title')
-...
                     wcs_site = get_wcs_services().get(wcs_key)
                     categories_response_json = get_wcs_json(wcs_site, 'api/categories/')
                     if not categories_response_json:
                     if not categories_response_json or categories_response_json.get('err') == 1:
                         # can not retrieve data, don't report cell as invalid
                         self.mark_as_valid()
                         return
                     category_found = False
                     for category in categories_response_json.get('data', []):
                     for category in categories_response_json.get('data') or []:
                         slug = category.get('slug')
                         if slug == category_slug:
                             self.cached_title = category.get('title')
-...
                 wcs_site['base_url'] = url
                 wcs_site['slug'] = wcs_slug
                 response = requests.get(
                         api_url,
                         remote_service=wcs_site,
                         user=getattr(context.get('request'), 'user', None),
                         cache_duration=self.cache_duration,
                         raise_if_not_cached=not(context.get('synchronous')),
                         log_errors=False)
                 returns.add(response.status_code)
                 if response.status_code == 200:
                     json_response = response.json()
                     if isinstance(json_response, list):
                         # backward compat with older w.c.s.
                         wcs_site['data'] = json_response
                     elif json_response.get('err', 0) == 0:
                         wcs_site['data'] = json_response['data']
                     else:
                         # skip errors
                         continue
                     # and mark all items with the site info
                     for item in wcs_site['data']:
                         item['site_slug'] = wcs_slug
                 try:
                     response = requests.get(
                             api_url,
                             remote_service=wcs_site,
                             user=getattr(context.get('request'), 'user', None),
                             cache_duration=self.cache_duration,
                             raise_if_not_cached=not(context.get('synchronous')),
                             log_errors=False)
                     returns.add(response.status_code)
                     response.raise_for_status()
                 except RequestException:
                     continue
                 json_response = response.json()
                 if json_response.get('err', 0) == 0:
                     wcs_site['data'] = json_response['data']
                 else:
                     # skip errors
                     continue
                 # and mark all items with the site info
                 for item in wcs_site['data']:
                     item['site_slug'] = wcs_slug
             if 200 not in returns:  # not a single valid answer
             if returns and 200 not in returns:  # not a single valid answer
                 logging.debug('failed to get data from any %s (%r)', api_url, returns)
                 if all([400 <= r < 500 for r in returns]) and self.warn_on_4xx:
                     # only 4xx errors, not a user cell, report the cell as invalid
-...
                 wcs_site = get_wcs_services().get(wcs_key)
                 categories_response_json = get_wcs_json(wcs_site, 'api/categories/')
                 if not categories_response_json:
                 if not categories_response_json or categories_response_json.get('err') == 1:
                     # can not retrieve data, don't report cell as invalid
                     continue
                 category_found = any(
                     [c.get('slug') == category_slug
                      for c in categories_response_json.get('data', [])])
                      for c in categories_response_json.get('data') or []])
                 if not category_found:
                     self.mark_as_invalid('wcs_category_not_found')
                     return
-...
             wcs_site = get_wcs_services().get(wcs_key)
             categories_response_json = get_wcs_json(wcs_site, 'api/categories/')
             if not categories_response_json:
             if not categories_response_json or categories_response_json.get('err') == 1:
                 # can not retrieve data, don't report cell as invalid
                 return
             category_found = any(
                 [c.get('slug') == category_slug
                  for c in categories_response_json.get('data', [])])
                  for c in categories_response_json.get('data') or []])
             if not category_found:
                 self.mark_as_invalid('wcs_category_not_found')
                 return
-...
                     wcs_site = get_wcs_services().get(wcs_key)
                     card_models = get_wcs_json(wcs_site, 'api/cards/@list')
                     if not card_models:
                     if not card_models or card_models.get('err') == 1:
                         # can not retrieve data, don't report cell as invalid
                         self.mark_as_valid()
                         return
                     card_found = False
                     for card in card_models.get('data', []):
                     for card in card_models.get('data') or []:
                         slug = card['slug']
                         if slug == card_slug:
                             self.cached_title = card['title']
-...
                     if card_schema.get('err') == 1:
                         if card_schema.get('err_class') == 'Page not found':
                             self.mark_as_invalid('wcs_card_not_found')
                         else:
                             self.mark_as_valid()
                         return
                     self.cached_title = card_schema['name']
-...
             wcs_site = get_wcs_services().get(self.wcs_site)
             response = requests.get(
                     api_url,
                     remote_service=wcs_site,
                     user=getattr(context.get('request'), 'user', None),
                     cache_duration=5,
                     raise_if_not_cached=not(context.get('synchronous')),
                     log_errors=False)
             try:
                 response = requests.get(
                         api_url,
                         remote_service=wcs_site,
                         user=getattr(context.get('request'), 'user', None),
                         cache_duration=5,
                         raise_if_not_cached=not(context.get('synchronous')),
                         log_errors=False)
                 response.raise_for_status()
             except RequestException:
                 return extra_context
             if response.status_code == 200:
                 extra_context['card'] = response.json()

     from django.conf import settings
     from requests.exceptions import RequestException
     from combo.utils import requests
     def is_wcs_enabled(cls):
         return hasattr(settings, 'KNOWN_SERVICES') and settings.KNOWN_SERVICES.get('wcs')
     def get_wcs_services():
         if not is_wcs_enabled(None):
             return {}
-...
             # no site specified (probably an import referencing a not yet deployed
             # site)
             return {'err': 1, 'err_desc': 'no-wcs-site'}
         response = requests.get(path, remote_service=wcs_site, without_user=True,
         try:
             response = requests.get(
                 path, remote_service=wcs_site, without_user=True,
                 headers={'accept': 'application/json'})
         if response.status_code != 200:
             # the exact error will have been logged during request.
             return {}
             response.raise_for_status()
         except RequestException:
             return {'err': 1, 'data': None}
         return response.json()
     def get_wcs_options(url, include_category_slug=False):
         references = []
         for wcs_key, wcs_site in sorted(get_wcs_services().items(), key=lambda x: x[1]['title']):
             site_title = wcs_site.get('title')
             response_json = get_wcs_json(wcs_site, url)
             if type(response_json) is dict:
                 if response_json.get('err') == 1:
                     continue
                 response_json = response_json.get('data')
             if response_json.get('err') == 1:
                 continue
             response_json = response_json.get('data')
             if not response_json:
                 continue
             for element in response_json:

     from django.utils.six.moves.urllib import parse as urlparse
     import mock
     from requests.exceptions import ConnectionError
     from requests.models import Response
     from combo.data.library import get_cell_classes
     from combo.data.models import CellBase, LinkListCell, Page, ValidityInfo
-...
         cell.save()
         assert ValidityInfo.objects.exists() is False
         # can not retrieve data, don't set cell as invalid
         with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
             mock_resp = Response()
             mock_resp.status_code = 500
             requests_get.return_value = mock_resp
             cell.save()
         assert ValidityInfo.objects.exists() is False
         with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
             requests_get.side_effect = ConnectionError()
             cell.save()
         assert ValidityInfo.objects.exists() is False
         # can not retrieve formdefs, don't set cell as invalid
         with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
             mock_json = mock.Mock(status_code=404)
             requests_get.return_value = mock_json
             mock_resp = Response()
             mock_resp.status_code = 404
             requests_get.return_value = mock_resp
             cell.save()
         assert ValidityInfo.objects.exists() is False
-...
         cell.save()
         assert ValidityInfo.objects.exists() is False
         # can not retrieve data, don't set cell as invalid
         with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
             mock_resp = Response()
             mock_resp.status_code = 500
             requests_get.return_value = mock_resp
             cell.save()
         assert ValidityInfo.objects.exists() is False
         with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
             requests_get.side_effect = ConnectionError()
             cell.save()
         assert ValidityInfo.objects.exists() is False
         # can not retrieve categories, don't set cell as invalid
         with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
             mock_json = mock.Mock(status_code=404)
             requests_get.return_value = mock_json
             mock_resp = Response()
             mock_resp.status_code = 404
             requests_get.return_value = mock_resp
             cell.save()
         assert ValidityInfo.objects.exists() is False
-...
         cell.get_data(context)
         assert ValidityInfo.objects.exists() is False
         # can not retrieve data, don't set cell as invalid
         with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
             mock_json = mock.Mock(status_code=404)
             requests_get.return_value = mock_json
             mock_resp = Response()
             mock_resp.status_code = 500
             requests_get.return_value = mock_resp
             cell.get_data(context)
         assert ValidityInfo.objects.exists() is False
         with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
             requests_get.side_effect = ConnectionError()
             cell.get_data(context)
         assert ValidityInfo.objects.exists() is False
         with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
             mock_resp = Response()
             mock_resp.status_code = 404
             requests_get.return_value = mock_resp
             cell.get_data(context)
         assert ValidityInfo.objects.exists() is False
-...
         cell.check_validity()
         assert ValidityInfo.objects.exists() is False
         # can not retrieve data, don't set cell as invalid
         with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
             mock_resp = Response()
             mock_resp.status_code = 500
             requests_get.return_value = mock_resp
             cell.check_validity()
         assert ValidityInfo.objects.exists() is False
         with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
             requests_get.side_effect = ConnectionError()
             cell.check_validity()
         assert ValidityInfo.objects.exists() is False
         # can not retrieve categories, don't set cell as invalid
         with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
             mock_json = mock.Mock(status_code=404)
             requests_get.return_value = mock_json
             mock_resp = Response()
             mock_resp.status_code = 404
             requests_get.return_value = mock_resp
             cell.check_validity()
         assert ValidityInfo.objects.exists() is False
-...
             cell.get_data(context)
         assert ValidityInfo.objects.exists() is False
         # can not retrieve data, don't set cell as invalid
         with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
             mock_resp = Response()
             mock_resp.status_code = 500
             requests_get.return_value = mock_resp
             cell.get_data(context)
         assert ValidityInfo.objects.exists() is False
         with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
             requests_get.side_effect = ConnectionError()
             cell.get_data(context)
         assert ValidityInfo.objects.exists() is False
         # can not retrieve data, don't set cell as invalid
         with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
             mock_json = mock.Mock(status_code=500)
-...
             context.pop('combo_display_even_empty_categories')
     def test_forms_of_category_cell_validity(context):
     @mock.patch('combo.apps.wcs.utils.requests.send', side_effect=mocked_requests_send)
     def test_forms_of_category_cell_validity(mock_send, context):
         page = Page.objects.create(title='xxx', slug='test_forms_of_category_cell_render', template_name='standard')
         cell = WcsFormsOfCategoryCell.objects.create(page=page, placeholder='content', order=0)
         validity_info = ValidityInfo.objects.latest('pk')
-...
             requests_get.return_value = mock_json
             cell.render(context)
         assert ValidityInfo.objects.exists() is False
         with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
             requests_get.side_effect = ConnectionError()
             cell.render(context)
         assert ValidityInfo.objects.exists() is False
         # can not retrieve categories, don't set cell as invalid
         with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
             mock_json = mock.Mock(status_code=404)
             requests_get.return_value = mock_json
             mock_resp = Response()
             mock_resp.status_code = 404
             requests_get.return_value = mock_resp
             cell.save()
         assert ValidityInfo.objects.exists() is False
-...
         assert validity_info.invalid_since is not None
     def test_forms_of_category_cell_check_validity(context):
     @mock.patch('combo.apps.wcs.utils.requests.send', side_effect=mocked_requests_send)
     def test_forms_of_category_cell_check_validity(mock_send, context):
         page = Page.objects.create(title='xxx', slug='test_forms_of_category_cell_render', template_name='standard')
         cell = WcsFormsOfCategoryCell.objects.create(page=page, placeholder='content', order=0)
-...
         cell.check_validity()
         assert ValidityInfo.objects.exists() is False
         # can not retrieve vategories, don't set cell as invalid
         # can not retrieve data, don't set cell as invalid
         with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
             mock_json = mock.Mock(status_code=404)
             requests_get.return_value = mock_json
             mock_resp = Response()
             mock_resp.status_code = 500
             requests_get.return_value = mock_resp
             cell.check_validity()
         assert ValidityInfo.objects.exists() is False
         with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
             requests_get.side_effect = ConnectionError()
             cell.check_validity()
         assert ValidityInfo.objects.exists() is False
         # can not retrieve categories, don't set cell as invalid
         with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
             mock_resp = Response()
             mock_resp.status_code = 404
             requests_get.return_value = mock_resp
             cell.check_validity()
         assert ValidityInfo.objects.exists() is False
-...
         cell.save()
         assert ValidityInfo.objects.exists() is False
         # can not retrieve data, don't set cell as invalid
         with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
             mock_resp = Response()
             mock_resp.status_code = 500
             requests_get.return_value = mock_resp
             cell.save()
         assert ValidityInfo.objects.exists() is False
         with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
             requests_get.side_effect = ConnectionError()
             cell.save()
         assert ValidityInfo.objects.exists() is False
         # can not retrieve carddefs, don't set cell as invalid
         with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
             mock_json = mock.Mock(status_code=404)
             requests_get.return_value = mock_json
             mock_resp = Response()
             mock_resp.status_code = 404
             requests_get.return_value = mock_resp
             cell.save()
         assert ValidityInfo.objects.exists() is False
-...
         cell.save()
         assert ValidityInfo.objects.exists() is False
         # can not retrieve data, don't set cell as invalid
         with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
             mock_resp = Response()
             mock_resp.status_code = 500
             requests_get.return_value = mock_resp
             cell.save()
         assert ValidityInfo.objects.exists() is False
         with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
             requests_get.side_effect = ConnectionError()
             cell.save()
         assert ValidityInfo.objects.exists() is False
         # can not retrieve carddefs, don't set cell as invalid
         with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
             mock_json = mock.Mock(status_code=404)
             requests_get.return_value = mock_json
             mock_resp = Response()
             mock_resp.status_code = 404
             requests_get.return_value = mock_resp
             cell.save()
         assert ValidityInfo.objects.exists() is False
-...
         context['synchronous'] = True  # to get fresh content
         with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
             mock_resp = Response()
             mock_resp.status_code = 500
             requests_get.return_value = mock_resp
             result = cell.render(context)
         assert '<p>Unknown Card</p>' in result
         with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
             requests_get.side_effect = ConnectionError()
             result = cell.render(context)
         assert '<p>Unknown Card</p>' in result
         result = cell.render(context)
         assert '<h2>Card Model 1 - aa</h2>' in result
         assert '<span class="label">Field A</span>\n    \n    <span class="value">a</span>' in result
+    -

Projet

Général

Profil

Produits Entr'ouvert » Combo

0001-wcs-handle-network-error-on-requests-31438.patch