Projet

Général

Profil

0001-misc-find-logs-corresponding-to-the-same-API-call-38.patch

Valentin Deniaud, 18 février 2020 16:19

Télécharger (8,38 ko)

Voir les différences:

Subject: [PATCH] misc: find logs corresponding to the same API call (#38157)

 passerelle/base/models.py                       |  9 +++++++--
 passerelle/templates/passerelle/manage/log.html |  4 +++-
 passerelle/views.py                             | 16 +++++++---------
 tests/test_generic_endpoint.py                  | 13 +++++++++++++
 tests/test_manager.py                           |  4 ++++
 tests/utils.py                                  |  1 +
 6 files changed, 35 insertions(+), 12 deletions(-)
passerelle/base/models.py
9 9
import traceback
10 10
import base64
11 11
import itertools
12
import uuid
12 13

  
13 14
from django.apps import apps
14 15
from django.conf import settings
......
131 132

  
132 133
    def __init__(self, *args, **kwargs):
133 134
        super(BaseResource, self).__init__(*args, **kwargs)
134
        self.logger = ProxyLogger(connector=self)
135
        self.logger = ProxyLogger(connector=self, instance_id=str(uuid.uuid4()))
135 136

  
136 137
    def __str__(self):
137 138
        return self.title
......
762 763

  
763 764

  
764 765
class ProxyLogger(object):
765
    def __init__(self, connector, extra=None):
766
    def __init__(self, connector, extra=None, instance_id=None):
766 767
        self.connector = connector
767 768
        self.appname = connector.get_connector_slug()
768 769
        self.slug = connector.slug
769 770
        self.extra = extra or {}
770 771
        logger_name = 'passerelle.resource.%s.%s' % (self.appname, self.slug)
772
        self.instance_id = instance_id
771 773
        self._logger = logging.getLogger(logger_name)
772 774
        self._logger.setLevel(connector.log_level)
773 775

  
......
823 825
                return isinstance(value, (list, dict, bool) + six.integer_types + six.string_types)
824 826
            attr['extra'] = {key: value for key, value in extra.items() if is_json_serializable(value)}
825 827

  
828
            if self.instance_id:
829
                attr['extra']['request_id'] = self.instance_id
830

  
826 831
            if getattr(request, 'META', None):
827 832
                if 'HTTP_X_FORWARDED_FOR' in request.META:
828 833
                    sourceip = request.META.get('HTTP_X_FORWARDED_FOR', '').split(",")[0].strip()
passerelle/templates/passerelle/manage/log.html
1
{% load passerelle %}
1
{% load passerelle i18n %}
2 2
<div class="log-dialog">
3 3
<table>
4 4
{% for key, value in logline.extra.items %}
......
10 10
      {% with val=value|censor %}
11 11
      {% if key == 'connector_endpoint_url' %}
12 12
      <a href={{val}}>{{val}}</a>
13
      {% elif key == 'request_id' %}
14
      {{val}} (<a href="{% url 'view-logs-connector' connector=object.get_connector_slug slug=object.slug %}?q={{val}}">{% trans "search for logs from the same call" %}</a>)
13 15
      {% else %}
14 16
      {{val|linebreaksbr|urlize}}
15 17
      {% endif %}
passerelle/views.py
378 378
    @csrf_exempt
379 379
    def dispatch(self, request, *args, **kwargs):
380 380
        self.init_stuff(request, *args, **kwargs)
381
        connector = self.get_object()
381
        self.connector = self.get_object()
382 382
        self.endpoint = None
383
        for name, method in inspect.getmembers(connector, inspect.ismethod):
383
        for name, method in inspect.getmembers(self.connector, inspect.ismethod):
384 384
            if not hasattr(method, 'endpoint_info'):
385 385
                continue
386 386
            if not method.endpoint_info.name == kwargs.get('endpoint'):
......
395 395
                self.endpoint = method
396 396
        if not self.endpoint:
397 397
            raise Http404()
398
        if kwargs.get('endpoint') == 'up' and hasattr(connector.check_status, 'not_implemented'):
398
        if kwargs.get('endpoint') == 'up' and hasattr(self.connector.check_status, 'not_implemented'):
399 399
            # hide automatic up endpoint if check_status method is not implemented
400 400
            raise Http404()
401 401
        return super(GenericEndpointView, self).dispatch(request, *args, **kwargs)
......
407 407
        perm = self.endpoint.endpoint_info.perm
408 408
        if not perm:
409 409
            return True
410
        return is_authorized(request, self.get_object(), perm)
410
        return is_authorized(request, self.connector, perm)
411 411

  
412 412
    def perform(self, request, *args, **kwargs):
413 413
        if request.method.lower() not in self.endpoint.endpoint_info.methods:
......
437 437

  
438 438
        # auto log request's inputs
439 439
        connector_name, endpoint_name = kwargs['connector'], kwargs['endpoint']
440
        connector = self.get_object()
441 440
        url = request.get_full_path()
442
        payload = request.body[:connector.logging_parameters.requests_max_size]
441
        payload = request.body[:self.connector.logging_parameters.requests_max_size]
443 442
        try:
444 443
            payload = payload.decode('utf-8')
445 444
        except UnicodeDecodeError:
446 445
            payload = '<BINARY PAYLOAD>'
447
        connector.logger.info('endpoint %s %s (%r) ' %
446
        self.connector.logger.info('endpoint %s %s (%r) ' %
448 447
                              (request.method, url, payload),
449 448
                              extra={
450 449
                                  'request': request,
......
478 477
            kwargs['other_params'] = match.kwargs
479 478
        elif kwargs.get('rest'):
480 479
            raise Http404()
481
        connector = self.get_object()
482
        return to_json(logger=connector.logger)(self.perform)(request, *args, **kwargs)
480
        return to_json(logger=self.connector.logger)(self.perform)(request, *args, **kwargs)
483 481

  
484 482
    def post(self, request, *args, **kwargs):
485 483
        return self.get(request, *args, **kwargs)
tests/test_generic_endpoint.py
147 147
    assert ResourceLog.objects.last().levelno == 30
148 148

  
149 149

  
150
@mock.patch('requests.Session.send')
151
def test_proxy_logger_request_id(mocked_get, caplog, app, arcgis):
152
    payload = open(os.path.join(os.path.dirname(__file__), 'data', 'nancy_arcgis', 'sigresponse.json')).read()
153
    mocked_get.return_value = utils.FakedResponse(content=payload, status_code=200)
154
    arcgis.log_evel = 'DEBUG'
155
    arcgis.base_url = 'https://example.net/'
156
    arcgis.save()
157
    resp = app.get('/arcgis/test/district', params={'lon': 6.172122, 'lat': 48.673836}, status=200)
158

  
159
    log1, log2 = ResourceLog.objects.filter(appname='arcgis', slug='test').all()
160
    assert log1.extra['request_id'] == log2.extra['request_id']
161

  
162

  
150 163
class FakeConnectorBase(object):
151 164
    slug = 'connector'
152 165

  
tests/test_manager.py
188 188
    resp.form['q'] = ''
189 189
    resp = resp.form.submit()
190 190
    assert resp.text.count('<td class="timestamp">') == 4
191

  
191 192
    log_pk = re.findall(r'data-pk="(.*)"', resp.text)[0]
192 193
    base_url = re.findall(r'data-log-base-url="(.*)"', resp.text)[0]
193 194
    resp = app.get(base_url + log_pk + '/')
195
    resp = resp.click('search for logs from the same call')
196
    assert resp.text.count('<td class="timestamp">') == 2
197

  
194 198
    resp = app.get(base_url + '12345' + '/', status=404)
195 199

  
196 200
def test_logging_parameters(app, admin_user):
tests/utils.py
26 26

  
27 27

  
28 28
class FakedResponse(mock.Mock):
29
    headers = {}
29 30

  
30 31
    def json(self):
31 32
        return json_loads(self.content)
32
-