From 3f57cc4ec689515e30b9a3392905b3ab7167ebbe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Laur=C3=A9line=20Gu=C3=A9rin?= <zebuline@entrouvert.com>
Date: Mon, 9 Nov 2020 17:55:32 +0100
Subject: [PATCH 1/2] logs: better queryset to search for logs (#48074)

---
 .../base/migrations/0025_transaction_id.py    | 19 +++++++++
 .../base/migrations/0026_transaction_id.py    | 18 ++++++++
 .../base/migrations/0027_transaction_id.py    | 19 +++++++++
 passerelle/base/models.py                     |  2 +
 .../templates/passerelle/manage/log.html      |  4 +-
 passerelle/views.py                           | 16 ++++---
 tests/test_manager.py                         | 42 +++++++++++++++----
 7 files changed, 106 insertions(+), 14 deletions(-)
 create mode 100644 passerelle/base/migrations/0025_transaction_id.py
 create mode 100644 passerelle/base/migrations/0026_transaction_id.py
 create mode 100644 passerelle/base/migrations/0027_transaction_id.py

diff --git a/passerelle/base/migrations/0025_transaction_id.py b/passerelle/base/migrations/0025_transaction_id.py
new file mode 100644
index 00000000..4eb71aa2
--- /dev/null
+++ b/passerelle/base/migrations/0025_transaction_id.py
@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('base', '0024_auto_20201103_1256'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='resourcelog',
+            name='transaction_id',
+            field=models.UUIDField(null=True),
+        ),
+    ]
diff --git a/passerelle/base/migrations/0026_transaction_id.py b/passerelle/base/migrations/0026_transaction_id.py
new file mode 100644
index 00000000..3443b833
--- /dev/null
+++ b/passerelle/base/migrations/0026_transaction_id.py
@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('base', '0025_transaction_id'),
+    ]
+
+    operations = [
+        migrations.RunSQL(
+            ["UPDATE base_resourcelog SET transaction_id=(extra->>'transaction_id')::uuid"],
+            reverse_sql=[]
+        ),
+    ]
diff --git a/passerelle/base/migrations/0027_transaction_id.py b/passerelle/base/migrations/0027_transaction_id.py
new file mode 100644
index 00000000..ee8eb5f2
--- /dev/null
+++ b/passerelle/base/migrations/0027_transaction_id.py
@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('base', '0026_transaction_id'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='resourcelog',
+            name='transaction_id',
+            field=models.UUIDField(db_index=True, null=True),
+        ),
+    ]
diff --git a/passerelle/base/models.py b/passerelle/base/models.py
index 6828322f..b20644de 100644
--- a/passerelle/base/models.py
+++ b/passerelle/base/models.py
@@ -799,6 +799,7 @@ class ResourceLog(models.Model):
     sourceip = models.GenericIPAddressField(blank=True, null=True, verbose_name=_('Source IP'))
     message = models.TextField(verbose_name='message')
     extra = JSONField(verbose_name='extras', default={})
+    transaction_id = models.UUIDField(null=True, db_index=True)
 
     class Meta:
         ordering = ('id',)
@@ -910,6 +911,7 @@ class ProxyLogger(object):
             attr['extra'] = {key: value for key, value in extra.items() if is_json_serializable(value)}
 
             if self.transaction_id:
+                attr['transaction_id'] = self.transaction_id
                 attr['extra']['transaction_id'] = self.transaction_id
 
             if getattr(request, 'META', None):
diff --git a/passerelle/templates/passerelle/manage/log.html b/passerelle/templates/passerelle/manage/log.html
index cb621664..90f75e18 100644
--- a/passerelle/templates/passerelle/manage/log.html
+++ b/passerelle/templates/passerelle/manage/log.html
@@ -29,9 +29,9 @@
 {% endfor %}
 </table>
 
-{% if logline.extra.transaction_id %}
+{% if logline.transaction_id %}
 <div class="buttons">
-<a class="button" href="{% url 'view-logs-connector' connector=object.get_connector_slug slug=object.slug %}?q={{logline.extra.transaction_id}}">{% trans "Search for logs from the same call" %}</a>
+<a class="button" href="{% url 'view-logs-connector' connector=object.get_connector_slug slug=object.slug %}?q={{logline.transaction_id}}">{% trans "Search for logs from the same call" %}</a>
 </div>
 {% endif %}
 
diff --git a/passerelle/views.py b/passerelle/views.py
index caa54334..cf01fb83 100644
--- a/passerelle/views.py
+++ b/passerelle/views.py
@@ -19,6 +19,7 @@ import hashlib
 import inspect
 import json
 import logging
+import uuid
 
 from django.apps import apps
 from django.conf.urls import url
@@ -27,8 +28,7 @@ from django.core.exceptions import PermissionDenied
 from django.contrib.auth import logout as auth_logout
 from django.contrib.auth import views as auth_views
 from django.db import transaction
-from django.db.models import TextField, Q
-from django.db.models.functions import Cast
+from django.db.models import Q
 from django.http import HttpResponse, HttpResponseRedirect, Http404
 from django.views.decorators.csrf import csrf_exempt
 from django.views.generic import (
@@ -249,9 +249,15 @@ class GenericViewLogsConnectorView(GenericConnectorMixin, ListView):
             try:
                 date = date_parser.parse(query, dayfirst=True)
             except Exception:
-                qs = qs.annotate(
-                    text_extra=Cast('extra', TextField())
-                ).filter(Q(text_extra__icontains=query) | Q(message__icontains=query))
+                query_uuid = None
+                try:
+                    query_uuid = uuid.UUID(query)
+                except ValueError:
+                    pass
+                else:
+                    qs = qs.filter(transaction_id=query_uuid)
+                if query_uuid is None or query_uuid is not None and not qs.exists():
+                    qs = qs.filter(Q(message__icontains=query) | Q(extra__icontains=query))
             else:
                 date = make_aware(date)
                 if date.hour == 0 and date.minute == 0 and date.second == 0:
diff --git a/tests/test_manager.py b/tests/test_manager.py
index f1785882..8adb34e9 100644
--- a/tests/test_manager.py
+++ b/tests/test_manager.py
@@ -1,5 +1,6 @@
 import datetime
 import re
+import uuid
 
 from webtest import Upload
 
@@ -247,22 +248,49 @@ def test_logs(app, admin_user):
     assert 'title="Level 42 - Oct. 6, 2020 14:08:12"' in resp.text
 
 
-def test_logs_search_in_message(app, admin_user):
+def test_logs_search(app, admin_user):
     csv = CsvDataSource.objects.create(csv_file=File(StringIO('1;t\n'), 't.csv'), slug='t', title='t')
     app = login(app)
 
-    ResourceLog.objects.create(appname=csv.get_connector_slug(), slug=csv.slug, levelno=1, message='hop')
-    resp = app.get(csv.get_absolute_url())
-    resp = resp.click('full page')
-    assert resp.text.count('<td class="timestamp">') == 1
+    transaction_id = str(uuid.uuid4())
+    log1 = ResourceLog.objects.create(
+        appname=csv.get_connector_slug(), slug=csv.slug, levelno=1, message='hop',
+        transaction_id=transaction_id,
+        extra={'transaction_id': transaction_id, 'foo': 'plop'})
+    log2 = ResourceLog.objects.create(
+        appname=csv.get_connector_slug(), slug=csv.slug, levelno=1, message='plop',
+        transaction_id=transaction_id,
+        extra={'bar': 'hop'})
+    log3 = ResourceLog.objects.create(
+        appname=csv.get_connector_slug(), slug=csv.slug, levelno=1, message='foo',
+        extra={'bar': 'hop'})
+
+    resp = app.get('/manage/csvdatasource/t/logs/')
+    assert list(resp.context['page_obj'].object_list) == [log3, log2, log1]
+
+    resp.form['q'] = transaction_id
+    resp = resp.form.submit()
+    assert list(resp.context['page_obj'].object_list) == [log2, log1]
+
+    resp.form['q'] = str(uuid.uuid4())
+    resp = resp.form.submit()
+    assert list(resp.context['page_obj'].object_list) == []
 
     resp.form['q'] = 'not there'
     resp = resp.form.submit()
-    assert resp.text.count('<td class="timestamp">') == 0
+    assert list(resp.context['page_obj'].object_list) == []
 
     resp.form['q'] = 'hop'
     resp = resp.form.submit()
-    assert resp.text.count('<td class="timestamp">') == 1
+    assert list(resp.context['page_obj'].object_list) == [log3, log2, log1]
+
+    resp.form['q'] = 'plop'
+    resp = resp.form.submit()
+    assert list(resp.context['page_obj'].object_list) == [log2, log1]
+
+    resp.form['q'] = 'foo'
+    resp = resp.form.submit()
+    assert list(resp.context['page_obj'].object_list) == [log3, log1]
 
 
 def test_logging_parameters(app, admin_user):
-- 
2.28.0

