From 1f3a90e0ea6d96418178f62ddbe31e2462b49426 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Laur=C3=A9line=20Gu=C3=A9rin?= <zebuline@entrouvert.com>
Date: Fri, 16 Sep 2022 16:30:12 +0200
Subject: [PATCH 1/2] fields: new field RichTextField (#36498)

---
 tests/admin_pages/test_form.py    | 28 +++++++++++++++
 tests/api/test_formdata.py        |  5 ++-
 tests/test_fields.py              | 32 +++++++++++++++++
 tests/test_templates.py           | 24 +++++++++++++
 wcs/fields.py                     | 26 +++++++++++++-
 wcs/formdef.py                    |  1 +
 wcs/qommon/form.py                |  4 +++
 wcs/qommon/misc.py                | 60 +++++++++++++++++++++++++++++++
 wcs/qommon/templatetags/qommon.py | 11 +++++-
 wcs/sql.py                        |  1 +
 10 files changed, 189 insertions(+), 3 deletions(-)

diff --git a/tests/admin_pages/test_form.py b/tests/admin_pages/test_form.py
index f01eccb3b..9ebcdb6cc 100644
--- a/tests/admin_pages/test_form.py
+++ b/tests/admin_pages/test_form.py
@@ -3274,6 +3274,34 @@ def test_form_new_computed_field(pub):
     assert resp.form['data_source$type'].options == [('None', True, 'None'), ('carddef:baz', False, 'Baz')]
 
 
+def test_form_new_richtext_field(pub):
+    create_superuser(pub)
+    create_role(pub)
+
+    pub.site_options.set('options', 'enable-richtext-field', 'true')
+    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
+        pub.site_options.write(fd)
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form title'
+    formdef.fields = []
+    formdef.store()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/forms/1/')
+    resp = resp.click(href='fields/')
+
+    resp.forms[0]['label'] = 'foobar'
+    resp.forms[0]['type'] = 'richtext'
+    resp = resp.forms[0].submit().follow()
+
+    assert len(FormDef.get(1).fields) == 1
+    field = FormDef.get(1).fields[0]
+    assert field.key == 'richtext'
+    assert field.label == 'foobar'
+
+
 def test_form_category_management_roles(pub, backoffice_user, backoffice_role):
     app = login(get_app(pub), username='backoffice-user', password='backoffice-user')
     app.get('/backoffice/forms/', status=403)
diff --git a/tests/api/test_formdata.py b/tests/api/test_formdata.py
index 5cf8f4350..8abf8f505 100644
--- a/tests/api/test_formdata.py
+++ b/tests/api/test_formdata.py
@@ -139,6 +139,7 @@ def test_formdata(pub, local_user):
         fields.FileField(id='3', label='foobar4', varname='file'),
         fields.ItemField(id='4', label='foobar5', varname='item', data_source={'type': 'foobar'}),
         fields.BlockField(id='5', label='test', varname='blockdata', type='block:foobar', max_items=3),
+        fields.RichTextField(id='6', label='rich text', varname='richtext', type='richtext'),
     ]
     Workflow.wipe()
     workflow = Workflow(name='foo')
@@ -168,6 +169,7 @@ def test_formdata(pub, local_user):
             'schema': {},  # not important here
         },
         '5_display': 'hello',
+        '6': '<script></script><p>foo</p>',
     }
     formdata.data['4_display'] = item_field.store_display_value(formdata.data, item_field.id)
     formdata.data['4_structured'] = item_field.store_structured_value(formdata.data, item_field.id)
@@ -186,7 +188,7 @@ def test_formdata(pub, local_user):
 
     assert datetime.datetime.strptime(resp.json['last_update_time'], '%Y-%m-%dT%H:%M:%S')
     assert datetime.datetime.strptime(resp.json['receipt_time'], '%Y-%m-%dT%H:%M:%S')
-    assert len(resp.json['fields']) == 8
+    assert len(resp.json['fields']) == 9
     assert 'foobar' in resp.json['fields']
     assert 'foobar2' not in resp.json['fields']  # foobar2 has no varname, not in json
     assert resp.json['user']['name'] == local_user.name
@@ -204,6 +206,7 @@ def test_formdata(pub, local_user):
     assert resp.json['fields']['blockdata_raw'] == [
         {'foo': 'plop', 'bar': 'foo', 'bar_raw': '1', 'bar_structured': 'XXX'}
     ]
+    assert resp.json['fields']['richtext'] == '<p>foo</p>'  # only allowed tags
     assert resp.json['workflow']['status']['name'] == 'New'
     assert resp.json['workflow']['real_status']['name'] == 'New'
     assert resp.json['submission'] == {'backoffice': False, 'channel': 'web'}
diff --git a/tests/test_fields.py b/tests/test_fields.py
index 1617cafbb..2155bd4c5 100644
--- a/tests/test_fields.py
+++ b/tests/test_fields.py
@@ -120,6 +120,11 @@ def test_text():
     assert 'rows="12"' in str(form.render())
 
 
+def test_richtext(pub):
+    assert fields.RichTextField().get_view_short_value('<p>foo</p>' * 15) == ('foo' * 10)[:27] + '(…)'
+    assert fields.RichTextField().get_view_value('<script></script><p>foo</p>') == '<p>foo</p>'
+
+
 def test_email():
     assert (
         fields.EmailField().get_view_value('foo@localhost')
@@ -787,6 +792,33 @@ def test_new_field_type_options(pub):
         ('computed', 'Computed Data', 'computed'),
     ]
 
+    pub.site_options.set('options', 'enable-richtext-field', 'true')
+    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
+        pub.site_options.write(fd)
+
+    assert fields.get_field_options(blacklisted_types=[]) == [
+        ('string', 'Text (line)', 'string'),
+        ('text', 'Long Text', 'text'),
+        ('richtext', 'Long Text (Rich)', 'richtext'),
+        ('email', 'Email', 'email'),
+        ('bool', 'Check Box (single choice)', 'bool'),
+        ('file', 'File Upload', 'file'),
+        ('date', 'Date', 'date'),
+        ('item', 'List', 'item'),
+        ('items', 'Multiple choice list', 'items'),
+        ('table-select', 'Table of Lists', 'table-select'),
+        ('tablerows', 'Table with rows', 'tablerows'),
+        ('map', 'Map', 'map'),
+        ('ranked-items', 'Ranked Items', 'ranked-items'),
+        ('', '—', ''),
+        ('title', 'Title', 'title'),
+        ('subtitle', 'Subtitle', 'subtitle'),
+        ('comment', 'Comment', 'comment'),
+        ('page', 'Page', 'page'),
+        ('', '—', ''),
+        ('computed', 'Computed Data', 'computed'),
+    ]
+
 
 def test_block_do_not_pickle_cache(pub):
     FormDef.wipe()
diff --git a/tests/test_templates.py b/tests/test_templates.py
index af08e055a..96c29770f 100644
--- a/tests/test_templates.py
+++ b/tests/test_templates.py
@@ -1477,3 +1477,27 @@ def test_as_template(pub):
     context = {'template': 'Hello {{ form_var_foo }}'}
     assert Template('{{ template }}').render(context) == 'Hello {{ form_var_foo }}'
     assert Template('{{ template|as_template }}').render(context) == 'Hello Foo Bar'
+
+
+def test_stripsometags(pub):
+    context = {
+        'value': (
+            '<h1>title 1</h1>'
+            '<script>my-script</script><p><em>foo</em></p><a href="link" other-attr="foobar">link</a><br />'
+            '<strong>strong</strong>'
+            '<ul><li>li 1</li><li>li 2</li></ul>'
+        )
+    }
+
+    assert Template('{{ value|stripsometags }}').render(context) == 'title 1my-scriptfoolinkstrongli 1li 2'
+    assert (
+        Template('{{ value|stripsometags:"p,br" }}').render(context)
+        == 'title 1my-script<p>foo</p>link<br />strongli 1li 2'
+    )
+    assert (
+        Template('{{ value|stripsometags:"strong,em" }}').render(context)
+        == 'title 1my-script<em>foo</em>link<strong>strong</strong>li 1li 2'
+    )
+    assert Template('{{ value|stripsometags:"p,br,h1,ul,li" }}').render(context) == (
+        '<h1>title 1</h1>my-script<p>foo</p>link<br />strong<ul><li>li 1</li><li>li 2</li></ul>'
+    )
diff --git a/wcs/fields.py b/wcs/fields.py
index 782d328db..935fbafd3 100644
--- a/wcs/fields.py
+++ b/wcs/fields.py
@@ -28,7 +28,7 @@ import xml.etree.ElementTree as ET
 
 from django.utils.encoding import force_bytes, force_text, smart_text
 from django.utils.formats import date_format as django_date_format
-from django.utils.html import urlize
+from django.utils.html import strip_tags, urlize
 from quixote import get_publisher, get_request, get_session
 from quixote.html import TemplateIO, htmlescape, htmltag, htmltext
 
@@ -59,6 +59,7 @@ from .qommon.form import (
     PasswordEntryWidget,
     RadiobuttonsWidget,
     RankedItemsWidget,
+    RichTextWidget,
     SingleSelectHintWidget,
     SingleSelectTableWidget,
     SingleSelectWidget,
@@ -80,6 +81,7 @@ from .qommon.misc import (
     get_document_type_value_options,
     get_document_types,
     strftime,
+    strip_some_tags,
     xml_node_text,
 )
 from .qommon.ods import NS as OD_NS
@@ -1380,6 +1382,26 @@ class TextField(WidgetField):
 register_field_class(TextField)
 
 
+class RichTextField(TextField):
+    key = 'richtext'
+    description = _('Long Text (Rich)')
+
+    widget_class = RichTextWidget
+    get_opendocument_node_value = None
+
+    def get_view_value(self, value, **kwargs):
+        return htmltext(strip_some_tags(value, self.widget_class.ALL_TAGS))
+
+    def get_view_short_value(self, value, max_len=30):
+        return ellipsize(str(strip_tags(value)), max_len)
+
+    def get_json_value(self, value, **kwargs):
+        return str(self.get_view_value(value))
+
+
+register_field_class(RichTextField)
+
+
 class EmailField(WidgetField):
     key = 'email'
     description = _('Email')
@@ -3992,6 +4014,8 @@ def get_field_options(blacklisted_types):
     widgets, non_widgets = [], []
     disabled_fields = (get_publisher().get_site_option('disabled-fields') or '').split(',')
     disabled_fields = [f.strip() for f in disabled_fields if f.strip()]
+    if not get_publisher().get_site_option('enable-richtext-field'):
+        disabled_fields += ['richtext']
     for klass in field_classes:
         if klass is ComputedField:
             continue
diff --git a/wcs/formdef.py b/wcs/formdef.py
index 00a244aa4..787cbc52b 100644
--- a/wcs/formdef.py
+++ b/wcs/formdef.py
@@ -1663,6 +1663,7 @@ class FormDef(StorableObject):
                     fields.CommentField,
                     fields.PageField,
                     fields.ComputedField,
+                    fields.RichTextField,
                 ),
             ):
                 continue
diff --git a/wcs/qommon/form.py b/wcs/qommon/form.py
index d3ef2b9e2..7e629e03a 100644
--- a/wcs/qommon/form.py
+++ b/wcs/qommon/form.py
@@ -2440,6 +2440,10 @@ class MiniRichTextWidget(WysiwygTextWidget):
         get_response().add_css_include('../xstatic/css/godo.css')
 
 
+class RichTextWidget(MiniRichTextWidget):
+    ALL_TAGS = ['p', 'b', 'strong', 'i', 'em', 'br', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'ul', 'li']
+
+
 class TableWidget(CompositeWidget):
     readonly = False
 
diff --git a/wcs/qommon/misc.py b/wcs/qommon/misc.py
index b6254e98e..9f51253db 100644
--- a/wcs/qommon/misc.py
+++ b/wcs/qommon/misc.py
@@ -52,7 +52,10 @@ from django.template import TemplateSyntaxError, VariableDoesNotExist
 from django.utils import datetime_safe
 from django.utils.encoding import force_bytes, force_text
 from django.utils.formats import localize
+from django.utils.functional import keep_lazy_text
+from django.utils.html import MLStripper as DjangoMLStripper
 from django.utils.html import strip_tags
+from django.utils.safestring import mark_safe
 from django.utils.text import Truncator
 from django.utils.timezone import is_aware, make_naive
 from quixote import get_publisher, get_request, get_response, redirect
@@ -1147,3 +1150,60 @@ def eval_python(expression, *args, **kwargs):
         raise UnauthorizedPythonUsage()
     # noqa pylint: disable=eval-used
     return eval(expression, *args, **kwargs)
+
+
+class MLStripper(DjangoMLStripper):
+    def __init__(self, allowed_tags):
+        super().__init__()
+        self.reset()
+        self.fed = []
+        self.allowed_tags = allowed_tags
+
+    def handle_starttag(self, tag, attrs):
+        if tag not in self.allowed_tags:
+            return
+
+        if tag == 'a':
+            for attr in attrs:
+                if attr[0] == 'href':
+                    self.fed.append('<a href="%s">' % attr[1])
+                    return
+        if tag == 'br':
+            self.fed.append('<br />')
+            return
+
+        self.fed.append('<%s>' % tag)
+
+    def handle_endtag(self, tag):
+        if tag not in self.allowed_tags:
+            return
+
+        if tag == 'br':
+            return
+
+        self.fed.append('</%s>' % tag)
+
+
+def _strip_once(value, allowed_tags):
+    """
+    Internal tag stripping utility used by strip_some_tags.
+    """
+    s = MLStripper(allowed_tags)
+    s.feed(value)
+    s.close()
+    return s.get_data()
+
+
+@keep_lazy_text
+def strip_some_tags(value, allowed_tags):
+    """Return the given HTML with all tags stripped except allowed_tags."""
+    # Note: in typical case this loop executes _strip_once once. Loop condition
+    # is redundant, but helps to reduce number of executions of _strip_once.
+    value = str(value)
+    while '<' in value and '>' in value:
+        new_value = _strip_once(value, allowed_tags)
+        if value.count('<') == new_value.count('<'):
+            # _strip_once wasn't able to detect more tags.
+            break
+        value = new_value
+    return mark_safe(value)
diff --git a/wcs/qommon/templatetags/qommon.py b/wcs/qommon/templatetags/qommon.py
index 39ade1e45..3908438b3 100644
--- a/wcs/qommon/templatetags/qommon.py
+++ b/wcs/qommon/templatetags/qommon.py
@@ -53,7 +53,7 @@ from django.utils.timezone import is_naive, make_aware
 from wcs.qommon import calendar, evalutils, upload_storage
 from wcs.qommon.admin.texts import TextsDirectory
 from wcs.qommon.humantime import seconds2humanduration
-from wcs.qommon.misc import validate_phone_fr
+from wcs.qommon.misc import strip_some_tags, validate_phone_fr
 
 register = template.Library()
 
@@ -1022,3 +1022,12 @@ def as_template(value):
     from wcs.workflows import WorkflowStatusItem
 
     return WorkflowStatusItem.compute(unlazy(value))
+
+
+@register.filter(is_safe=True)
+def stripsometags(value, arg=None):
+    arg = arg or ''
+    allowed_tags = arg.split(',')
+    allowed_tags = [t.strip() for t in allowed_tags]
+    allowed_tags = [t for t in allowed_tags if t]
+    return strip_some_tags(unlazy(value), allowed_tags)
diff --git a/wcs/sql.py b/wcs/sql.py
index 03fe92ec3..a0f1fc036 100644
--- a/wcs/sql.py
+++ b/wcs/sql.py
@@ -78,6 +78,7 @@ SQL_TYPE_MAPPING = {
     'comment': None,
     'page': None,
     'text': 'text',
+    'richtext': 'text',
     'bool': 'boolean',
     'file': 'bytea',
     'date': 'date',
-- 
2.35.1

