From c659d900b8f968290ffbad0cf84dfdba2a9ffd64 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20P=C3=A9ters?= <fpeters@entrouvert.com>
Date: Wed, 10 Oct 2018 09:50:16 +0200
Subject: [PATCH] forms: add support for live list contents (#27173)

---
 tests/test_form_pages.py             | 53 +++++++++++++++++++++++++
 tests/utilities.py                   |  1 +
 wcs/data_sources.py                  | 10 ++---
 wcs/fields.py                        |  4 +-
 wcs/formdef.py                       | 58 ++++++++++++++++++----------
 wcs/forms/root.py                    | 45 ++++++++++++++++++---
 wcs/qommon/static/js/qommon.forms.js | 13 +++++++
 wcs/qommon/substitution.py           |  9 ++++-
 8 files changed, 158 insertions(+), 35 deletions(-)

diff --git a/tests/test_form_pages.py b/tests/test_form_pages.py
index a1c653696..755b977cd 100644
--- a/tests/test_form_pages.py
+++ b/tests/test_form_pages.py
@@ -5480,3 +5480,56 @@ def test_field_live_condition_multipages(pub):
     assert 'name="f2"' in resp.body
     assert 'name="f4"' in resp.body
     resp = resp.form.submit('submit')
+
+def test_field_live_select_content(pub, http_requests):
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'Foo'
+    formdef.fields = [
+        fields.StringField(type='string', id='1', label='Bar', size='40',
+            required=True, varname='bar'),
+        fields.StringField(type='string', id='2', label='Bar2', size='40',
+            required=True, varname='bar2'),
+        fields.ItemField(type='item', id='3', label='Foo',
+            data_source={
+                'type': 'json',
+                'value': '{% if form_var_bar2 %}http://remote.example.net/json-list?plop={{form_var_bar2}}{% endif %}'
+            }),
+    ]
+    formdef.store()
+
+    app = get_app(pub)
+    resp = app.get('/foo/')
+    assert 'f1' in resp.form.fields
+    assert 'f2' in resp.form.fields
+    assert resp.html.find('div', {'data-field-id': '2'}).attrs['data-live-source'] == 'true'
+    assert resp.html.find('div', {'data-field-id': '3'}).find('select')
+    resp.form['f1'] = 'hello'
+    live_resp = app.post('/foo/live', params=resp.form.submit_fields())
+    assert live_resp.json['result']['1']['visible']
+    assert live_resp.json['result']['2']['visible']
+    assert live_resp.json['result']['3']['visible']
+    assert not 'items' in live_resp.json['result']['3']
+    resp.form['f2'] = 'plop'
+    live_resp = app.post('/foo/live?modified_field_id=2', params=resp.form.submit_fields())
+    assert live_resp.json['result']['1']['visible']
+    assert live_resp.json['result']['2']['visible']
+    assert live_resp.json['result']['3']['visible']
+    assert 'items' in live_resp.json['result']['3']
+    resp.form['f3'].options = []
+    for item in live_resp.json['result']['3']['items']:
+        # simulate javascript filling the <select>
+        resp.form['f3'].options.append((item['id'], False, item['text']))
+    resp.form['f3'] = 'a'
+    resp = resp.form.submit('submit')
+    assert 'Check values then click submit.' in resp.body
+    assert 'name="f1"' in resp.body
+    assert 'name="f2"' in resp.body
+    assert 'name="f3"' in resp.body
+    resp = resp.form.submit('submit')
+    resp = resp.follow()
+    formdata = formdef.data_class().select()[0]
+    assert formdata.data['1'] == 'hello'
+    assert formdata.data['2'] == 'plop'
+    assert formdata.data['3'] == 'a'
+    assert formdata.data['3_display'] == 'b'
diff --git a/tests/utilities.py b/tests/utilities.py
index 884dea46e..790c148c0 100644
--- a/tests/utilities.py
+++ b/tests/utilities.py
@@ -327,6 +327,7 @@ class HttpRequestsMocking(object):
             'http://remote.example.net/404-json': (404, '{"err": 1}', None),
             'http://remote.example.net/500': (500, 'internal server error', None),
             'http://remote.example.net/json': (200, '{"foo": "bar"}', None),
+            'http://remote.example.net/json-list': (200, '{"data": [{"id": "a", "text": "b"}]}', None),
             'http://remote.example.net/json-err0': (200, '{"data": "foo", "err": 0}', None),
             'http://remote.example.net/json-err1': (200, '{"data": "", "err": 1}', None),
             'http://remote.example.net/json-errstr': (200, '{"data": "", "err": "bug"}', None),
diff --git a/wcs/data_sources.py b/wcs/data_sources.py
index e889120d6..ce1e1fcb7 100644
--- a/wcs/data_sources.py
+++ b/wcs/data_sources.py
@@ -95,8 +95,8 @@ class DataSourceSelectionWidget(CompositeWidget):
         return r.getvalue()
 
 
-def get_items(data_source, include_disabled=False):
-    structured_items = get_structured_items(data_source)
+def get_items(data_source, include_disabled=False, mode=None):
+    structured_items = get_structured_items(data_source, mode=mode)
     tupled_items = []
     for item in structured_items:
         if item.get('disabled') and not include_disabled:
@@ -108,7 +108,7 @@ def get_items(data_source, include_disabled=False):
     return tupled_items
 
 
-def get_structured_items(data_source):
+def get_structured_items(data_source, mode=None):
     cache_duration = 0
     if data_source.get('type') not in ('json', 'jsonp', 'formula'):
         # named data source
@@ -126,7 +126,7 @@ def get_structured_items(data_source):
         #   - three elements, (id, text, key)
         #   - two elements, (id, text)
         #   - a single element, (id,)
-        variables = get_publisher().substitutions.get_context_variables()
+        variables = get_publisher().substitutions.get_context_variables(mode=mode)
         global_eval_dict = get_publisher().get_global_eval_dict()
         global_eval_dict.update(data_source_functions)
         try:
@@ -162,7 +162,7 @@ def get_structured_items(data_source):
             return []
         url = url.strip()
         if Template.is_template_string(url):
-            vars = get_publisher().substitutions.get_context_variables()
+            vars = get_publisher().substitutions.get_context_variables(mode=mode)
             url = get_variadic_url(url, vars)
 
         request = get_request()
diff --git a/wcs/fields.py b/wcs/fields.py
index dea7cdbfc..d3f93ebd3 100644
--- a/wcs/fields.py
+++ b/wcs/fields.py
@@ -1183,9 +1183,9 @@ class ItemField(WidgetField):
         self.items = []
         WidgetField.__init__(self, **kwargs)
 
-    def get_options(self):
+    def get_options(self, mode=None):
         if self.data_source:
-            return [x[:3] for x in data_sources.get_items(self.data_source)]
+            return [x[:3] for x in data_sources.get_items(self.data_source, mode=mode)]
         if self.items:
             return [(x, x) for x in self.items]
         return []
diff --git a/wcs/formdef.py b/wcs/formdef.py
index 757c6048a..256ebb2e5 100644
--- a/wcs/formdef.py
+++ b/wcs/formdef.py
@@ -512,7 +512,7 @@ class FormDef(StorableObject):
     def get_display_id_format(self):
         return '[formdef_id]-[form_number_raw]'
 
-    def create_form(self, page=None, displayed_fields=None):
+    def create_form(self, page=None, displayed_fields=None, transient_formdata=None):
         form = Form(enctype="multipart/form-data", use_tokens=False)
         if self.appearance_keywords:
             form.attrs['class'] = 'quixote %s' % self.appearance_keywords
@@ -521,10 +521,18 @@ class FormDef(StorableObject):
         form.ERROR_NOTICE = _('There were errors processing the form and '
                                'you cannot go to the next page. Do '
                                'check below that you filled all fields correctly.')
-        self.add_fields_to_form(form, page=page, displayed_fields=displayed_fields)
+        self.add_fields_to_form(form,
+                page=page,
+                displayed_fields=displayed_fields,
+                transient_formdata=transient_formdata)
         return form
 
-    def add_fields_to_form(self, form, page=None, displayed_fields=None, form_data=None):
+    def add_fields_to_form(self,
+            form,
+            page=None,
+            displayed_fields=None,
+            form_data=None,  # a dictionary, to fill fields
+            transient_formdata=None):  # a FormData
         current_page = 0
         on_page = (page is None)
         for field in self.fields:
@@ -549,6 +557,10 @@ class FormDef(StorableObject):
             widget = field.add_to_form(form, value)
             widget.is_hidden = not(visible)
             widget.field = field
+            if transient_formdata and not widget.is_hidden:
+                transient_formdata.data.update(self.get_field_data(field, widget))
+                widget._parsed = False
+                widget.error = None
 
     def get_page(self, page_no):
         return [x for x in self.fields if x.type == 'page'][page_no]
@@ -611,29 +623,33 @@ class FormDef(StorableObject):
 
         return form
 
+    def get_field_data(self, field, widget):
+        d = {}
+        d[field.id] = widget.parse()
+        if d.get(field.id) and field.convert_value_from_str:
+            d[field.id] = field.convert_value_from_str(d[field.id])
+        if d.get(field.id) and field.store_display_value:
+            display_value = field.store_display_value(d, field.id)
+            if display_value:
+                d['%s_display' % field.id] = display_value
+            elif d.has_key('%s_display' % field.id):
+                del d['%s_display' % field.id]
+        if d.get(field.id) and field.store_structured_value:
+            structured_value = field.store_structured_value(d, field.id)
+            if structured_value:
+                d['%s_structured' % field.id] = structured_value
+            elif '%s_structured' % field.id in d:
+                del d['%s_structured' % field.id]
+        if getattr(widget, 'cleanup', None):
+            widget.cleanup()
+        return d
+
     def get_data(self, form):
         d = {}
         for field in self.fields:
             widget = form.get_widget('f%s' % field.id)
             if widget:
-                d[field.id] = widget.parse()
-            if d.get(field.id) and field.convert_value_from_str:
-                d[field.id] = field.convert_value_from_str(d[field.id])
-            if d.get(field.id) and field.store_display_value:
-                display_value = field.store_display_value(d, field.id)
-                if display_value:
-                    d['%s_display' % field.id] = display_value
-                elif d.has_key('%s_display' % field.id):
-                    del d['%s_display' % field.id]
-            if d.get(field.id) and field.store_structured_value:
-                structured_value = field.store_structured_value(d, field.id)
-                if structured_value:
-                    d['%s_structured' % field.id] = structured_value
-                elif '%s_structured' % field.id in d:
-                    del d['%s_structured' % field.id]
-            if widget and widget.cleanup:
-                widget.cleanup()
-
+                d.update(self.get_field_data(field, widget))
         return d
 
     def export_to_json(self, include_id=False, indent=None):
diff --git a/wcs/forms/root.py b/wcs/forms/root.py
index 30c603888..64a8ebe05 100644
--- a/wcs/forms/root.py
+++ b/wcs/forms/root.py
@@ -45,6 +45,7 @@ from qommon.form import *
 from qommon.logger import BotFilter
 from qommon import emails
 
+from wcs import data_sources
 from wcs.categories import Category
 from wcs.formdef import FormDef
 from wcs.formdata import FormData
@@ -384,6 +385,16 @@ class FormPage(Directory, FormTemplateMixin):
                     if not varname in live_condition_fields:
                         live_condition_fields[varname] = []
                     live_condition_fields[varname].append(field)
+            if field.key == 'item' and field.data_source:
+                real_data_source = data_sources.get_real(field.data_source)
+                if real_data_source.get('type') != 'json':
+                    continue
+                varnames = re.findall(r'\bform[_\.]var[_\.]([a-zA-Z0-9_]+?)(?:_raw|\b)',
+                        real_data_source.get('value'))
+                for varname in varnames:
+                    if not varname in live_condition_fields:
+                        live_condition_fields[varname] = []
+                    live_condition_fields[varname].append(field)
 
         for field in displayed_fields:
             if field.varname in live_condition_fields:
@@ -699,7 +710,12 @@ class FormPage(Directory, FormTemplateMixin):
             self.feed_current_data(magictoken)
 
             submitted_fields = []
-            form = self.create_form(page=page, displayed_fields=submitted_fields)
+            transient_formdata = self.get_transient_formdata()
+            with get_publisher().substitutions.temporary_feed(
+                    transient_formdata, force_mode='lazy'):
+                form = self.create_form(page=page,
+                        displayed_fields=submitted_fields,
+                        transient_formdata=transient_formdata)
             form.add_submit('previous')
             if self.has_draft_support():
                 form.add_submit('removedraft')
@@ -1005,9 +1021,6 @@ class FormPage(Directory, FormTemplateMixin):
         if not session:
             return result_error('missing session')
 
-        formdata = self.get_transient_formdata()
-        get_publisher().substitutions.feed(formdata)
-
         page_id = get_request().form.get('page_id')
         if page_id:
             for field in self.formdef.fields:
@@ -1017,14 +1030,36 @@ class FormPage(Directory, FormTemplateMixin):
         else:
             page = None
 
+        formdata = self.get_transient_formdata()
+        get_publisher().substitutions.feed(formdata)
         displayed_fields = []
-        form = self.create_form(page=page, displayed_fields=displayed_fields)
+        with get_publisher().substitutions.temporary_feed(formdata, force_mode='lazy'):
+            form = self.create_form(
+                    page=page,
+                    displayed_fields=displayed_fields,
+                    transient_formdata=formdata)
         formdata.data.update(self.formdef.get_data(form))
 
         result = {}
         for field in displayed_fields:
             result[field.id] = {'visible': field.is_visible(formdata.data, self.formdef)}
 
+        modified_field_varname = None
+        for field in displayed_fields:
+            if field.id == get_request().form.get('modified_field_id'):
+                modified_field_varname = field.varname
+
+        for field in displayed_fields:
+            if field.key == 'item' and field.data_source:
+                real_data_source = data_sources.get_real(field.data_source)
+                if real_data_source.get('type') != 'json':
+                    continue
+                varnames = re.findall(r'\bform[_\.]var[_\.]([a-zA-Z0-9_]+?)(?:_raw|\b)',
+                        real_data_source.get('value'))
+                if modified_field_varname in varnames:
+                    result[field.id]['items'] = [
+                            {'id': x[2], 'text': x[1]} for x in field.get_options(mode='lazy')]
+
         return json.dumps({'result': result})
 
     def submitted(self, form, existing_formdata = None):
diff --git a/wcs/qommon/static/js/qommon.forms.js b/wcs/qommon/static/js/qommon.forms.js
index dab37b519..d8f1f06f9 100644
--- a/wcs/qommon/static/js/qommon.forms.js
+++ b/wcs/qommon/static/js/qommon.forms.js
@@ -91,6 +91,19 @@ $(function() {
           } else {
             $widget.hide();
           }
+          if (value.items) {
+            // replace <select> contents
+            var $select = $widget.find('select');
+            var current_value = $select.val();
+            $select.empty();
+            for (var i=0; i<value.items.length; i++) {
+              var $option = $('<option></option>', {value: value.items[i].id, text: value.items[i].text});
+              if (value.items[i].id == current_value) {
+                $option.attr('selected', 'selected');
+              }
+              $option.appendTo($select);
+            }
+          }
         });
       }
     });
diff --git a/wcs/qommon/substitution.py b/wcs/qommon/substitution.py
index 7441ba0b1..7a1a74b15 100644
--- a/wcs/qommon/substitution.py
+++ b/wcs/qommon/substitution.py
@@ -32,9 +32,10 @@ def invalidate_substitution_cache(func):
 class Substitutions(object):
     substitutions_dict = {}
     dynamic_sources = []
-
     sources = None
 
+    _forced_mode = None
+
     def __init__(self):
         self.sources = []
 
@@ -77,7 +78,7 @@ class Substitutions(object):
         self.invalidate_cache()
 
     @contextmanager
-    def temporary_feed(self, source):
+    def temporary_feed(self, source, force_mode=None):
         if source is None or source in self.sources:
             yield
             return
@@ -85,7 +86,9 @@ class Substitutions(object):
         orig_sources, self.sources = self.sources, self.sources[:]
         self.sources.append(source)
         self.invalidate_cache()
+        old_mode, self._forced_mode = self._forced_mode, force_mode
         yield
+        self._forced_mode = old_mode
         self.sources = orig_sources
         self.invalidate_cache()
 
@@ -95,6 +98,8 @@ class Substitutions(object):
                 delattr(self, '_cache_context_variables%r' % value)
 
     def get_context_variables(self, mode=None):
+        if self._forced_mode:
+            mode = self._forced_mode
         lazy = mode in get_publisher().get_lazy_variables_modes() if mode else False
         d = getattr(self, '_cache_context_variables%r' % lazy, None)
         if d is not None:
-- 
2.19.1