From a68d669bb060905577d4fbb0940ebc5bbd4797b5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20P=C3=A9ters?= <fpeters@entrouvert.com>
Date: Mon, 31 Dec 2018 14:31:25 +0100
Subject: [PATCH] misc: add evolution messages to fulltext search (#22651)

---
 tests/test_sql.py  | 20 +++++++++++++++++++-
 wcs/qommon/misc.py |  7 +++++++
 wcs/sql.py         |  8 +++++++-
 3 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/tests/test_sql.py b/tests/test_sql.py
index 892b9badd..49e4d004c 100644
--- a/tests/test_sql.py
+++ b/tests/test_sql.py
@@ -18,6 +18,7 @@ from wcs.formdata import Evolution
 from wcs.roles import Role
 from wcs.workflows import Workflow, CommentableWorkflowStatusItem, WorkflowCriticalityLevel
 from wcs.wf.jump import JumpWorkflowStatusItem
+from wcs.wf.register_comment import RegisterCommenterWorkflowStatusItem
 from wcs import sql
 import wcs.qommon.storage as st
 
@@ -912,14 +913,31 @@ def test_sql_criteria_fts():
             t.data = {'3': 'foo'}
         else:
             t.data = {'3': 'bar'}
+        t.just_created()
         t.store()
-    t.store()
 
     assert data_class.count() == 50
     assert len(data_class.select()) == 50
 
     assert [x.id for x in data_class.select([st.FtsMatch('BAR')], order_by='id')] == range(21, 51)
 
+    # check fts against data in history
+    assert len(data_class.select([st.FtsMatch('XXX')])) == 0
+    formdata1 = data_class.select([st.FtsMatch('BAR')])[0]
+    formdata1.evolution[0].comment = 'XXX'
+    formdata1.store()
+    assert len(data_class.select([st.FtsMatch('XXX')])) == 1
+    assert data_class.select([st.FtsMatch('XXX')])[0].id == formdata1.id
+
+    assert len(data_class.select([st.FtsMatch('yyy')])) == 0
+    item = RegisterCommenterWorkflowStatusItem()
+    item.comment = '<span>ÿÿÿ</span>'
+    item.perform(formdata1)
+    assert formdata1.evolution[-1].display_parts()[-1] == '<span>ÿÿÿ</span>'
+    formdata1.store()
+    assert len(data_class.select([st.FtsMatch('yyy')])) == 1
+    assert len(data_class.select([st.FtsMatch('span')])) == 0
+
     assert data_class.count([st.FtsMatch('Pierre')]) == 0
     sql.SqlUser.wipe()
     user = sql.SqlUser()
diff --git a/wcs/qommon/misc.py b/wcs/qommon/misc.py
index 51fea2745..a4016ca20 100644
--- a/wcs/qommon/misc.py
+++ b/wcs/qommon/misc.py
@@ -37,7 +37,9 @@ except ImportError:
 
 from django.conf import settings
 from django.utils import datetime_safe
+from django.utils.html import strip_tags
 from django.template import engines, TemplateSyntaxError, VariableDoesNotExist
+from django.utils.six.moves.html_parser import HTMLParser
 
 from quixote import get_publisher, get_response, get_request
 from quixote.html import htmltext
@@ -611,3 +613,8 @@ def normalize_geolocation(lat_lon):
     lat = wrap(lat, decimal.Decimal('-90.0'), decimal.Decimal('90.0'))
     lon = wrap(lon, decimal.Decimal('-180.0'), decimal.Decimal('180.0'))
     return {'lat': float(lat), 'lon': float(lon)}
+
+def html2text(text):
+    if isinstance(text, (htmltext, str)):
+        text = unicode(str(text), get_publisher().site_charset)
+    return site_encode(HTMLParser().unescape(strip_tags(text)))
diff --git a/wcs/sql.py b/wcs/sql.py
index 9c1d78b58..4424d0a57 100644
--- a/wcs/sql.py
+++ b/wcs/sql.py
@@ -1424,6 +1424,11 @@ class SqlFormData(SqlMixin, wcs.formdata.FormData):
             for evo in self._evolution:
                 if evo.comment:
                     fts_strings.append(evo.comment)
+                for part in evo.parts or []:
+                    if hasattr(part, 'view'):
+                        html_part = part.view()
+                        if html_part:
+                            fts_strings.append(qommon.misc.html2text(html_part))
         user = self.get_user()
         if user:
             fts_strings.append(user.get_display_name())
@@ -2155,7 +2160,7 @@ def get_yearly_totals(period_start=None, period_end=None, criterias=None):
     return result
 
 
-SQL_LEVEL = 28
+SQL_LEVEL = 29
 
 def migrate_global_views(conn, cur):
     cur.execute('''SELECT COUNT(*) FROM information_schema.tables
@@ -2263,6 +2268,7 @@ def migrate():
         # 21: (third part), add user ascii_names to full-text index
         # 23: (second part) use misc.simplify() over full text queries
         # 28: add display id and formdef name to full-text index
+        # 29: add evolution parts to full-text index
         set_reindex('formdata', 'needed', conn=conn, cur=cur)
     if sql_level < 24:
         from wcs.formdef import FormDef
-- 
2.20.1

