Projet

Général

Profil

0001-json-cell-add-timeout-parameter-17920.patch

Thomas Noël, 07 août 2017 16:49

Télécharger (7,33 ko)

Voir les différences: 

Subject: [PATCH] json cell: add timeout parameter (#17920)

... and handle requests errors, consequently

 combo/data/models.py | 40 ++++++++++++++++++++++++++++++----------
 tests/test_cells.py  | 16 +++++++++++++++-
 2 files changed, 45 insertions(+), 11 deletions(-)
combo/data/models.py
832 832 
    varnames = None
833 833 
    force_async = False
834 834 
    log_errors = True
835 
    timeout = 28  # combo wsgi service timeout - 2 seconds
835 836 
    actions = {}
836 837 
    additional_data = None
837 838 
        # [
......
859 860 
        self._json_content = None
860 861 

  		




  861
  862
  
    
        data_urls = [{'key': 'json', 'url': self.url, 'cache_duration': self.cache_duration,

  		




  862
  
  
    
                      'log_errors': self.log_errors}]

  		




  
  863
  
    
                      'log_errors': self.log_errors, 'timeout': self.timeout}]

  		




  863
  864
  
    
        data_urls.extend(self.additional_data or [])

  		




  864
  865
  
    

  		




  865
  866
  
    
        for data_url_dict in data_urls:

  		




  ......




  867
  868
  
    

  		




  868
  869
  
    
        for data_url_dict in data_urls:

  		




  869
  870
  
    
            data_key = data_url_dict['key']

  		




  
  871
  
    
            log_errors = data_url_dict.get('log_errors', self.log_errors)

  		




  870
  872
  
    
            try:

  		




  871
  873
  
    
                url = utils.get_templated_url(data_url_dict['url'], context)

  		




  872
  874
  
    
            except utils.UnknownTemplateVariableError:

  		




  873
  875
  
    
                logger = logging.getLogger(__name__)

  		




  874
  876
  
    
                logger.warning('unknown variable in template URL (%s)', self.url)

  		




  875
  877
  
    
                continue

  		




  876
  
  
    
            json_response = utils.requests.get(url,

  		




  877
  
  
    
                    headers={'Accept': 'application/json'},

  		




  878
  
  
    
                    remote_service='auto',

  		




  879
  
  
    
                    cache_duration=data_url_dict.get('cache_duration', self.cache_duration),

  		




  880
  
  
    
                    without_user=True,

  		




  881
  
  
    
                    raise_if_not_cached=not(context.get('synchronous')),

  		




  882
  
  
    
                    invalidate_cache=invalidate_cache,

  		




  883
  
  
    
                    log_errors=data_url_dict.get('log_errors', self.log_errors),

  		




  884
  
  
    
                    )

  		




  885
  878
  
    
            extra_context[data_key + '_url'] = url

  		




  
  879
  
    
            try:

  		




  
  880
  
    
                json_response = utils.requests.get(url,

  		




  
  881
  
    
                        headers={'Accept': 'application/json'},

  		




  
  882
  
    
                        remote_service='auto',

  		




  
  883
  
    
                        cache_duration=data_url_dict.get('cache_duration', self.cache_duration),

  		




  
  884
  
    
                        without_user=True,

  		




  
  885
  
    
                        raise_if_not_cached=not(context.get('synchronous')),

  		




  
  886
  
    
                        invalidate_cache=invalidate_cache,

  		




  
  887
  
    
                        log_errors=log_errors,

  		




  
  888
  
    
                        timeout=data_url_dict.get('timeout', self.timeout),

  		




  
  889
  
    
                        )

  		




  
  890
  
    
            except requests.RequestException as e:

  		




  
  891
  
    
                extra_context[data_key + '_status'] = -1

  		




  
  892
  
    
                extra_context[data_key + '_error'] = unicode(e)

  		




  
  893
  
    
                extra_context[data_key + '_exception'] = e

  		




  
  894
  
    
                logger = logging.getLogger(__name__)

  		




  
  895
  
    
                if log_errors:

  		




  
  896
  
    
                    logger.warning(u'error on request %r: %', url, unicode(e))

  		




  
  897
  
    
                else:

  		




  
  898
  
    
                    logger.debug(u'error on request %r: %', url, unicode(e))

  		




  
  899
  
    
                continue

  		




  886
  900
  
    
            extra_context[data_key + '_status'] = json_response.status_code

  		




  887
  901
  
    
            if json_response.status_code // 100 == 2:

  		




  888
  902
  
    
                if json_response.status_code != 204:  # 204 = No Content

  		




  ......




  930
  944
  
    
            raise PermissionDenied()

  		




  931
  945
  
    

  		




  932
  946
  
    
        error_message = self.actions[action].get('error-message')

  		




  
  947
  
    
        timeout = self.actions[action].get('timeout', self.timeout)

  		




  933
  948
  
    
        logger = logging.getLogger(__name__)

  		




  934
  949
  
    

  		




  935
  950
  
    
        content = {}

  		




  ......




  984
  999
  
    
    varnames_str = models.CharField(_('Variable names'), max_length=200, blank=True,

  		




  985
  1000
  
    
            help_text=_('Comma separated list of query-string variables '

  		




  986
  1001
  
    
                        'to be copied in template context'))

  		




  
  1002
  
    
    timeout = models.PositiveIntegerField(_('Request timeout'), default=28)

  		




  987
  1003
  
    

  		




  988
  1004
  
    
    class Meta:

  		




  989
  1005
  
    
        verbose_name = _('JSON Feed')

  		




  ......




  1043
  1059
  
    
    def log_errors(self):

  		




  1044
  1060
  
    
        return settings.JSON_CELL_TYPES[self.key].get('log_errors',

  		




  1045
  1061
  
    
                JsonCellBase.log_errors)

  		




  
  1062
  
    
    @property

  		




  
  1063
  
    
    def timeout(self):

  		




  
  1064
  
    
        return settings.JSON_CELL_TYPES[self.key].get('timeout',

  		




  
  1065
  
    
                JsonCellBase.timeout)

  		




  1046
  1066
  
    

  		




  1047
  1067
  
    
    @property

  		




  1048
  1068
  
    
    def actions(self):

  		












  

    
      tests/test_cells.py
    
  





  186
  186
  
    
        assert context['json_status'] == 404

  		




  187
  187
  
    
        assert context['json_error'] == data

  		




  188
  188
  
    

  		




  
  189
  
    
        def mocked_requests_connection_error(*args, **kwargs):

  		




  
  190
  
    
            raise requests.ConnectionError('boom')

  		




  
  191
  
    
        requests_get.side_effect = mocked_requests_connection_error

  		




  
  192
  
    
        context = cell.get_cell_extra_context({})

  		




  
  193
  
    
        assert context['json'] is None

  		




  
  194
  
    
        assert context['json_status'] == -1

  		




  
  195
  
    
        assert context['json_url'] == 'http://test2'

  		




  
  196
  
    
        assert context['json_error'] == 'boom'

  		




  
  197
  
    
        assert isinstance(context['json_exception'], requests.ConnectionError)

  		




  
  198
  
    

  		




  189
  199
  
    
    with pytest.raises(NothingInCacheException):

  		




  190
  200
  
    
        cell.url = 'http://test3'

  		




  191
  201
  
    
        cell.render({})

  		




  ......




  342
  352
  
    
                'name': 'Foobar',

  		




  343
  353
  
    
                'url': 'http://foo?var=[identifier]',

  		




  344
  354
  
    
                'log_errors': False,

  		




  
  355
  
    
                'timeout': 42,

  		




  345
  356
  
    
                'form': [

  		




  346
  357
  
    
                   {

  		




  347
  358
  
    
                     "varname": "identifier",

  		




  ......




  368
  379
  
    
            assert requests_get.call_count == 1

  		




  369
  380
  
    
            assert requests_get.call_args[0][0] == 'http://foo?var=plop'

  		




  370
  381
  
    
            assert requests_get.call_args[-1]['log_errors'] == False

  		




  
  382
  
    
            assert requests_get.call_args[-1]['timeout'] == 42

  		




  371
  383
  
    

  		




  372
  384
  
    
def test_json_force_async():

  		




  373
  385
  
    
    cell = JsonCellBase()

  		




  ......




  409
  421
  
    
                'name': 'Foobar',

  		




  410
  422
  
    
                'url': 'http://foo',

  		




  411
  423
  
    
                'additional-data': [

  		




  412
  
  
    
                    {'key': 'plop', 'url': 'http://bar', 'log_errors': False},

  		




  
  424
  
    
                    {'key': 'plop', 'url': 'http://bar', 'log_errors': False, 'timeout': 42},

  		




  413
  425
  
    
                ]

  		




  414
  426
  
    
            }},

  		




  415
  427
  
    
            TEMPLATE_DIRS=['%s/templates-1' % os.path.abspath(os.path.dirname(__file__))]):

  		




  ......




  430
  442
  
    
            assert len(requests_get.mock_calls) == 2

  		




  431
  443
  
    
            assert requests_get.mock_calls[0][1][0] == 'http://foo'

  		




  432
  444
  
    
            assert requests_get.mock_calls[0][-1]['log_errors'] == True

  		




  
  445
  
    
            assert requests_get.mock_calls[0][-1]['timeout'] == 28

  		




  433
  446
  
    
            assert requests_get.mock_calls[1][1][0] == 'http://bar'

  		




  434
  447
  
    
            assert requests_get.mock_calls[1][-1]['log_errors'] == False

  		




  
  448
  
    
            assert requests_get.mock_calls[1][-1]['timeout'] == 42

  		




  435
  449
  
    

  		




  436
  450
  
    
        with mock.patch('combo.utils.requests.get') as requests_get:

  		




  437
  451
  
    
            data = {'data': 'toto'}

  		




  438
  
  
    
-