From 9443e8e2e61b3cebfde25ca8821912c569779586 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Laur=C3=A9line=20Gu=C3=A9rin?= <zebuline@entrouvert.com>
Date: Fri, 16 Sep 2022 16:30:12 +0200
Subject: [PATCH 1/2] fields: new field RichTextField (#36498)

---
 tests/api/test_formdata.py        |  5 ++-
 tests/test_fields.py              |  7 +++-
 tests/test_formdef_import.py      |  4 +--
 tests/test_templates.py           | 24 +++++++++++++
 wcs/fields.py                     | 53 ++++++++++++++++++++++++---
 wcs/formdef.py                    |  2 ++
 wcs/qommon/form.py                |  4 +++
 wcs/qommon/misc.py                | 60 +++++++++++++++++++++++++++++++
 wcs/qommon/templatetags/qommon.py | 11 +++++-
 wcs/wf/export_to_model.py         |  2 ++
 10 files changed, 162 insertions(+), 10 deletions(-)

diff --git a/tests/api/test_formdata.py b/tests/api/test_formdata.py
index 5cf8f4350..25822df61 100644
--- a/tests/api/test_formdata.py
+++ b/tests/api/test_formdata.py
@@ -139,6 +139,7 @@ def test_formdata(pub, local_user):
         fields.FileField(id='3', label='foobar4', varname='file'),
         fields.ItemField(id='4', label='foobar5', varname='item', data_source={'type': 'foobar'}),
         fields.BlockField(id='5', label='test', varname='blockdata', type='block:foobar', max_items=3),
+        fields.TextField(id='6', label='rich text', varname='richtext', type='text', display_mode='rich'),
     ]
     Workflow.wipe()
     workflow = Workflow(name='foo')
@@ -168,6 +169,7 @@ def test_formdata(pub, local_user):
             'schema': {},  # not important here
         },
         '5_display': 'hello',
+        '6': '<script></script><p>foo</p>',
     }
     formdata.data['4_display'] = item_field.store_display_value(formdata.data, item_field.id)
     formdata.data['4_structured'] = item_field.store_structured_value(formdata.data, item_field.id)
@@ -186,7 +188,7 @@ def test_formdata(pub, local_user):
 
     assert datetime.datetime.strptime(resp.json['last_update_time'], '%Y-%m-%dT%H:%M:%S')
     assert datetime.datetime.strptime(resp.json['receipt_time'], '%Y-%m-%dT%H:%M:%S')
-    assert len(resp.json['fields']) == 8
+    assert len(resp.json['fields']) == 9
     assert 'foobar' in resp.json['fields']
     assert 'foobar2' not in resp.json['fields']  # foobar2 has no varname, not in json
     assert resp.json['user']['name'] == local_user.name
@@ -204,6 +206,7 @@ def test_formdata(pub, local_user):
     assert resp.json['fields']['blockdata_raw'] == [
         {'foo': 'plop', 'bar': 'foo', 'bar_raw': '1', 'bar_structured': 'XXX'}
     ]
+    assert resp.json['fields']['richtext'] == '<p>foo</p>'  # only allowed tags
     assert resp.json['workflow']['status']['name'] == 'New'
     assert resp.json['workflow']['real_status']['name'] == 'New'
     assert resp.json['submission'] == {'backoffice': False, 'channel': 'web'}
diff --git a/tests/test_fields.py b/tests/test_fields.py
index 1617cafbb..e15e40757 100644
--- a/tests/test_fields.py
+++ b/tests/test_fields.py
@@ -105,7 +105,12 @@ def test_text():
     assert fields.TextField().get_view_short_value('foo' * 15) == ('foo' * 10)[:27] + '(…)'
     assert fields.TextField().get_view_value('foo') == '<p>foo</p>'
     assert fields.TextField().get_view_value('foo\n\nfoo') == '<p>foo\n</p><p>\nfoo</p>'
-    assert fields.TextField(pre=True).get_view_value('foo') == '<pre>foo</pre>'
+    assert fields.TextField(display_mode='pre').get_view_value('foo') == '<pre>foo</pre>'
+    assert (
+        fields.TextField(display_mode='rich').get_view_short_value('<p>foo</p>' * 15)
+        == ('foo' * 10)[:27] + '(…)'
+    )
+    assert fields.TextField(display_mode='rich').get_view_value('<script></script><p>foo</p>') == '<p>foo</p>'
 
     form = Form(use_tokens=False)
     fields.TextField().add_to_form(form)
diff --git a/tests/test_formdef_import.py b/tests/test_formdef_import.py
index 658140ca7..c694249d1 100644
--- a/tests/test_formdef_import.py
+++ b/tests/test_formdef_import.py
@@ -148,7 +148,7 @@ def test_more_fields(pub):
     formdef = FormDef()
     formdef.name = 'Blah'
     formdef.fields = [
-        fields.TextField(type='text', label='Bar', pre=True),
+        fields.TextField(type='text', label='Bar', display_mode='pre'),
         fields.EmailField(type='email', label='Bar'),
         fields.BoolField(type='bool', label='Bar'),
         fields.DateField(type='date', label='Bar', minimum_date='2014-01-01'),
@@ -204,7 +204,7 @@ def test_include_id(pub):
     formdef = FormDef()
     formdef.name = 'Blah'
     formdef.fields = [
-        fields.TextField(type='text', label='Bar', pre=True),
+        fields.TextField(type='text', label='Bar', display_mode='pre'),
         fields.EmailField(type='email', label='Bar'),
         fields.BoolField(type='bool', label='Bar'),
         fields.DateField(type='date', label='Bar', minimum_date='2014-01-01'),
diff --git a/tests/test_templates.py b/tests/test_templates.py
index af08e055a..96c29770f 100644
--- a/tests/test_templates.py
+++ b/tests/test_templates.py
@@ -1477,3 +1477,27 @@ def test_as_template(pub):
     context = {'template': 'Hello {{ form_var_foo }}'}
     assert Template('{{ template }}').render(context) == 'Hello {{ form_var_foo }}'
     assert Template('{{ template|as_template }}').render(context) == 'Hello Foo Bar'
+
+
+def test_stripsometags(pub):
+    context = {
+        'value': (
+            '<h1>title 1</h1>'
+            '<script>my-script</script><p><em>foo</em></p><a href="link" other-attr="foobar">link</a><br />'
+            '<strong>strong</strong>'
+            '<ul><li>li 1</li><li>li 2</li></ul>'
+        )
+    }
+
+    assert Template('{{ value|stripsometags }}').render(context) == 'title 1my-scriptfoolinkstrongli 1li 2'
+    assert (
+        Template('{{ value|stripsometags:"p,br" }}').render(context)
+        == 'title 1my-script<p>foo</p>link<br />strongli 1li 2'
+    )
+    assert (
+        Template('{{ value|stripsometags:"strong,em" }}').render(context)
+        == 'title 1my-script<em>foo</em>link<strong>strong</strong>li 1li 2'
+    )
+    assert Template('{{ value|stripsometags:"p,br,h1,ul,li" }}').render(context) == (
+        '<h1>title 1</h1>my-script<p>foo</p>link<br />strong<ul><li>li 1</li><li>li 2</li></ul>'
+    )
diff --git a/wcs/fields.py b/wcs/fields.py
index 782d328db..43e63e51b 100644
--- a/wcs/fields.py
+++ b/wcs/fields.py
@@ -28,7 +28,7 @@ import xml.etree.ElementTree as ET
 
 from django.utils.encoding import force_bytes, force_text, smart_text
 from django.utils.formats import date_format as django_date_format
-from django.utils.html import urlize
+from django.utils.html import strip_tags, urlize
 from quixote import get_publisher, get_request, get_session
 from quixote.html import TemplateIO, htmlescape, htmltag, htmltext
 
@@ -59,6 +59,7 @@ from .qommon.form import (
     PasswordEntryWidget,
     RadiobuttonsWidget,
     RankedItemsWidget,
+    RichTextWidget,
     SingleSelectHintWidget,
     SingleSelectTableWidget,
     SingleSelectWidget,
@@ -80,6 +81,7 @@ from .qommon.misc import (
     get_document_type_value_options,
     get_document_types,
     strftime,
+    strip_some_tags,
     xml_node_text,
 )
 from .qommon.ods import NS as OD_NS
@@ -1322,10 +1324,26 @@ class TextField(WidgetField):
     widget_class = TextWidget
     cols = None
     rows = None
-    pre = False
+    pre = None
+    display_mode = 'plain'
     maxlength = None
     extra_attributes = ['cols', 'rows', 'maxlength']
 
+    def migrate(self):
+        changed = super().migrate()
+        if isinstance(getattr(self, 'pre', None), bool):
+            if self.pre:
+                self.display_mode = 'pre'
+            else:
+                self.display_mode = 'plain'
+            self.pre = None
+            changed = True
+        return changed
+
+    def perform_more_widget_changes(self, *args, **kwargs):
+        if self.display_mode == 'rich':
+            self.widget_class = RichTextWidget
+
     def fill_admin_form(self, form):
         WidgetField.fill_admin_form(self, form)
         if self.cols:
@@ -1343,17 +1361,33 @@ class TextField(WidgetField):
             form.add(HiddenWidget, 'cols', value=None)
         form.add(StringWidget, 'rows', title=_('Number of rows'), value=self.rows)
         form.add(StringWidget, 'maxlength', title=_('Maximum number of characters'), value=self.maxlength)
-        form.add(CheckboxWidget, 'pre', title=_('Preformatted Text'), value=self.pre, advanced=True)
+        display_options = [
+            ('plain', _('Plain Text (paragraphs)')),
+            ('pre', _('Preformatted Text')),
+            ('rich', _('Rich Text')),
+        ]
+        form.add(
+            RadiobuttonsWidget,
+            'display_mode',
+            title=_('Text display'),
+            value=self.display_mode,
+            default_value='plain',
+            options=display_options,
+            extra_css_class='widget-inline-radio no-bottom-margin',
+            advanced=True,
+        )
 
     def get_admin_attributes(self):
-        return WidgetField.get_admin_attributes(self) + ['cols', 'rows', 'pre', 'maxlength']
+        return WidgetField.get_admin_attributes(self) + ['cols', 'rows', 'display_mode', 'maxlength']
 
     def convert_value_from_str(self, value):
         return value
 
     def get_view_value(self, value, **kwargs):
-        if self.pre:
+        if self.display_mode == 'pre':
             return htmltext('<pre>') + value + htmltext('</pre>')
+        elif self.display_mode == 'rich':
+            return htmltext(strip_some_tags(value, RichTextWidget.ALL_TAGS))
         else:
             try:
                 return (
@@ -1365,6 +1399,8 @@ class TextField(WidgetField):
                 return ''
 
     def get_opendocument_node_value(self, value, formdata=None, **kwargs):
+        if self.display_mode == 'rich':
+            return
         paragraphs = []
         for paragraph in value.splitlines():
             if paragraph.strip():
@@ -1374,8 +1410,15 @@ class TextField(WidgetField):
         return paragraphs
 
     def get_view_short_value(self, value, max_len=30):
+        if self.display_mode == 'rich':
+            return ellipsize(str(strip_tags(value)), max_len)
         return ellipsize(str(value), max_len)
 
+    def get_json_value(self, value, **kwargs):
+        if self.display_mode == 'rich':
+            return str(self.get_view_value(value))
+        return value
+
 
 register_field_class(TextField)
 
diff --git a/wcs/formdef.py b/wcs/formdef.py
index 00a244aa4..eb9f513ca 100644
--- a/wcs/formdef.py
+++ b/wcs/formdef.py
@@ -1666,6 +1666,8 @@ class FormDef(StorableObject):
                 ),
             ):
                 continue
+            if isinstance(field, fields.TextField) and field.display_mode == 'rich':
+                continue
             if data is None:
                 continue
             if data.get(field.id) is None:
diff --git a/wcs/qommon/form.py b/wcs/qommon/form.py
index d3ef2b9e2..7e629e03a 100644
--- a/wcs/qommon/form.py
+++ b/wcs/qommon/form.py
@@ -2440,6 +2440,10 @@ class MiniRichTextWidget(WysiwygTextWidget):
         get_response().add_css_include('../xstatic/css/godo.css')
 
 
+class RichTextWidget(MiniRichTextWidget):
+    ALL_TAGS = ['p', 'b', 'strong', 'i', 'em', 'br', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'ul', 'li']
+
+
 class TableWidget(CompositeWidget):
     readonly = False
 
diff --git a/wcs/qommon/misc.py b/wcs/qommon/misc.py
index b6254e98e..9f51253db 100644
--- a/wcs/qommon/misc.py
+++ b/wcs/qommon/misc.py
@@ -52,7 +52,10 @@ from django.template import TemplateSyntaxError, VariableDoesNotExist
 from django.utils import datetime_safe
 from django.utils.encoding import force_bytes, force_text
 from django.utils.formats import localize
+from django.utils.functional import keep_lazy_text
+from django.utils.html import MLStripper as DjangoMLStripper
 from django.utils.html import strip_tags
+from django.utils.safestring import mark_safe
 from django.utils.text import Truncator
 from django.utils.timezone import is_aware, make_naive
 from quixote import get_publisher, get_request, get_response, redirect
@@ -1147,3 +1150,60 @@ def eval_python(expression, *args, **kwargs):
         raise UnauthorizedPythonUsage()
     # noqa pylint: disable=eval-used
     return eval(expression, *args, **kwargs)
+
+
+class MLStripper(DjangoMLStripper):
+    def __init__(self, allowed_tags):
+        super().__init__()
+        self.reset()
+        self.fed = []
+        self.allowed_tags = allowed_tags
+
+    def handle_starttag(self, tag, attrs):
+        if tag not in self.allowed_tags:
+            return
+
+        if tag == 'a':
+            for attr in attrs:
+                if attr[0] == 'href':
+                    self.fed.append('<a href="%s">' % attr[1])
+                    return
+        if tag == 'br':
+            self.fed.append('<br />')
+            return
+
+        self.fed.append('<%s>' % tag)
+
+    def handle_endtag(self, tag):
+        if tag not in self.allowed_tags:
+            return
+
+        if tag == 'br':
+            return
+
+        self.fed.append('</%s>' % tag)
+
+
+def _strip_once(value, allowed_tags):
+    """
+    Internal tag stripping utility used by strip_some_tags.
+    """
+    s = MLStripper(allowed_tags)
+    s.feed(value)
+    s.close()
+    return s.get_data()
+
+
+@keep_lazy_text
+def strip_some_tags(value, allowed_tags):
+    """Return the given HTML with all tags stripped except allowed_tags."""
+    # Note: in typical case this loop executes _strip_once once. Loop condition
+    # is redundant, but helps to reduce number of executions of _strip_once.
+    value = str(value)
+    while '<' in value and '>' in value:
+        new_value = _strip_once(value, allowed_tags)
+        if value.count('<') == new_value.count('<'):
+            # _strip_once wasn't able to detect more tags.
+            break
+        value = new_value
+    return mark_safe(value)
diff --git a/wcs/qommon/templatetags/qommon.py b/wcs/qommon/templatetags/qommon.py
index 39ade1e45..3908438b3 100644
--- a/wcs/qommon/templatetags/qommon.py
+++ b/wcs/qommon/templatetags/qommon.py
@@ -53,7 +53,7 @@ from django.utils.timezone import is_naive, make_aware
 from wcs.qommon import calendar, evalutils, upload_storage
 from wcs.qommon.admin.texts import TextsDirectory
 from wcs.qommon.humantime import seconds2humanduration
-from wcs.qommon.misc import validate_phone_fr
+from wcs.qommon.misc import strip_some_tags, validate_phone_fr
 
 register = template.Library()
 
@@ -1022,3 +1022,12 @@ def as_template(value):
     from wcs.workflows import WorkflowStatusItem
 
     return WorkflowStatusItem.compute(unlazy(value))
+
+
+@register.filter(is_safe=True)
+def stripsometags(value, arg=None):
+    arg = arg or ''
+    allowed_tags = arg.split(',')
+    allowed_tags = [t.strip() for t in allowed_tags]
+    allowed_tags = [t for t in allowed_tags if t]
+    return strip_some_tags(unlazy(value), allowed_tags)
diff --git a/wcs/wf/export_to_model.py b/wcs/wf/export_to_model.py
index 4242d376d..6ae84a3e3 100644
--- a/wcs/wf/export_to_model.py
+++ b/wcs/wf/export_to_model.py
@@ -729,6 +729,8 @@ class ExportToModel(WorkflowStatusItem):
                 unset_value_i.text = _('Not set')
             else:
                 node_value = f.get_opendocument_node_value(value, formdata)
+                if node_value is None:
+                    continue
                 if isinstance(node_value, list):
                     for node in node_value:
                         section_node.append(node)
-- 
2.35.1

