From e1fa613ca9c99c36c8fe1b8d965ec8e25dfc8d31 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20P=C3=A9ters?= <fpeters@entrouvert.com>
Date: Mon, 14 Sep 2020 09:44:52 +0200
Subject: [PATCH] fields: add marker selection on map field (#47066)

---
 tests/test_form_pages.py                      |  54 ++++++++++-
 tests/test_formdata.py                        |  41 ++++++++-
 tests/test_workflows.py                       |   4 +
 tests/utilities.py                            |  13 +++
 wcs/api.py                                    |  14 ++-
 wcs/data_sources.py                           |  84 +++++++++++++++---
 wcs/fields.py                                 |  78 +++++++++++++---
 wcs/formdata.py                               |   6 +-
 wcs/formdef.py                                |   5 +-
 wcs/forms/common.py                           |   2 +-
 wcs/qommon/form.py                            |   8 ++
 .../static/images/blank-marker-icon.png       | Bin 0 -> 1783 bytes
 wcs/qommon/static/js/qommon.map.js            |  55 +++++++++++-
 .../templates/qommon/forms/widgets/map.html   |  37 ++++++++
 wcs/variables.py                              |  14 ++-
 15 files changed, 375 insertions(+), 40 deletions(-)
 create mode 100644 wcs/qommon/static/images/blank-marker-icon.png

diff --git a/tests/test_form_pages.py b/tests/test_form_pages.py
index 4584aaf89..26cd1ffcb 100644
--- a/tests/test_form_pages.py
+++ b/tests/test_form_pages.py
@@ -10,7 +10,7 @@ import re
 import time
 import zipfile
 import base64
-from webtest import Upload, Hidden
+from webtest import Upload, Hidden, Radio
 import mock
 import xml.etree.ElementTree as ET
 
@@ -4427,7 +4427,7 @@ def test_form_map_field_back_and_submit(pub):
     assert formdef.data_class().count() == 1
     data_id = formdef.data_class().select()[0].id
     data = formdef.data_class().get(data_id)
-    assert data.data == {'0': {'lat': 1.234, 'lon': -1.234}, '1': 'bla'}
+    assert data.data == {'0': {'lat': 1.234, 'lon': -1.234}, '0_display': '1.234;-1.234', '1': 'bla'}
 
 
 def test_map_field_migration(pub):
@@ -4497,7 +4497,55 @@ def test_form_map_multi_page(pub):
     assert formdef.data_class().count() == 1
     data_id = formdef.data_class().select()[0].id
     data = formdef.data_class().get(data_id)
-    assert data.data == {'1': {'lat': 1.234, 'lon': -1.234}, '3': 'bar'}
+    assert data.data == {'1': {'lat': 1.234, 'lon': -1.234}, '1_display': '1.234;-1.234', '3': 'bar'}
+
+
+def test_form_map_data_source(pub, http_requests):
+    NamedDataSource.wipe()
+    data_source = NamedDataSource(name='foobar')
+    data_source.data_source = {
+        'type': 'geojson',
+        'value': 'http://remote.example.net/geojson',
+    }
+    data_source.id_property = 'id'
+    data_source.label_template_property = '{{ text }}'
+    data_source.store()
+
+    formdef = create_formdef()
+    formdef.fields = [
+        fields.MapField(id='1', label='map', data_source={'type': 'foobar'}),
+    ]
+    formdef.store()
+    formdef.data_class().wipe()
+    resp = get_app(pub).get('/test/')
+    assert resp.pyquery('div[data-markers-radio-name]')[0].attrib['data-markers-url'] == '/api/geojson/foobar'
+    assert resp.pyquery('div[data-markers-radio-name]')[0].attrib['data-markers-radio-name'] == 'f1$marker_id'
+    # simulate qommon.map.js that will create radio inputs
+    resp.form.fields['f1$marker_id'] = [Radio(form=resp.form, tag='input', name='f1$marker_id', pos=5)]
+    resp.form.fields['f1$marker_id'][0].options.append(('1', False, None))
+    resp.form.fields['f1$marker_id'][0].options.append(('2', False, None))
+    resp.form.fields['f1$marker_id'][0].optionPositions.append(5)
+    resp.form.fields['f1$marker_id'][0].optionPositions.append(6)
+    resp.form.field_order.append(('f1$marker_id', resp.form.fields['f1$marker_id'][0]))
+    resp.form['f1$marker_id'].value = '1'  # click on marker
+    resp.form['f1$latlng'] = '1;2'  # set via js
+    resp = resp.form.submit('submit')
+    assert 'Check values then click submit.' in resp
+    # selected option is displayed as readonly:
+    assert resp.pyquery('input[type=text][value=foo][readonly]')
+    assert resp.pyquery('input[type=hidden][name="f1$marker_id"][value="1"]')
+    assert int(float(resp.pyquery('.qommon-map')[0].attrib['data-init-lat'])) == 1
+    assert int(float(resp.pyquery('.qommon-map')[0].attrib['data-init-lng'])) == 2
+    resp = resp.form.submit('submit')
+    resp = resp.follow()
+    assert 'The form has been recorded' in resp
+    assert resp.pyquery('input[type=text][value=foo][readonly]')
+    assert int(float(resp.pyquery('.qommon-map')[0].attrib['data-init-lat'])) == 1
+    assert int(float(resp.pyquery('.qommon-map')[0].attrib['data-init-lng'])) == 2
+    assert formdef.data_class().count() == 1
+    data_id = formdef.data_class().select()[0].id
+    data = formdef.data_class().get(data_id)
+    assert data.data == {'1': {'lat': 1.0, 'lon': 2.0, 'marker_id': '1'}, '1_display': 'foo'}
 
 
 def test_form_middle_session_change(pub):
diff --git a/tests/test_formdata.py b/tests/test_formdata.py
index 7bcbcfcbd..7e2ea9272 100644
--- a/tests/test_formdata.py
+++ b/tests/test_formdata.py
@@ -20,6 +20,7 @@ from wcs.blocks import BlockDef
 from wcs.carddef import CardDef
 from wcs.categories import Category
 from wcs.conditions import Condition
+from wcs.data_sources import NamedDataSource
 from wcs.formdef import FormDef
 from wcs.formdata import Evolution
 from wcs.roles import Role
@@ -628,6 +629,7 @@ def variable_test_data(pub):
         },
     }
     formdata.data['5'].receive([b'hello world'])
+    formdata.data['7_display'] = formdef.fields[7].store_display_value(formdata.data, '7')
     formdata.geolocations = {'base': {'lat': 1, 'lon': 2}}
     formdata.store()
     pub.substitutions.feed(pub)
@@ -762,6 +764,43 @@ def test_lazy_formdata(pub, variable_test_data):
     assert lazy_formdata.tracking_code == tracking_code.id
 
 
+def test_lazy_formdata_map_item_field(pub, http_requests):
+    NamedDataSource.wipe()
+    data_source = NamedDataSource(name='foobar')
+    data_source.data_source = {
+        'type': 'geojson',
+        'value': 'http://remote.example.net/geojson',
+    }
+    data_source.id_property = 'id'
+    data_source.label_template_property = '{{ text }}'
+    data_source.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'foobar map item'
+    formdef.fields = [
+        fields.MapField(id='14', label='map-item', varname='map_item', data_source={'type': 'foobar'}),
+    ]
+    formdef.store()
+    formdef.data_class().wipe()
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.data = {
+        '14': {'lat': 1, 'lon': 2, 'marker_id': '1'},  # map-item
+    }
+    formdata.data['14_display'] = formdef.fields[0].store_display_value(formdata.data, '14')
+    formdata.store()
+    pub.substitutions.feed(pub)
+    pub.substitutions.feed(formdef)
+    pub.substitutions.feed(formdata)
+
+    context = pub.substitutions.get_context_variables(mode='lazy')
+    assert Template('{{form_var_map_item}}').render(context) == 'foo'
+    assert Template('{{form_var_map_item_lat}}').render(context) == '1'
+    assert Template('{{form_var_map_item_lon}}').render(context) == '2'
+    assert Template('{{form_var_map_item_marker_id}}').render(context) == '1'
+
+
 def test_lazy_formdata_duplicated_varname(pub, variable_test_data):
     formdef = FormDef.select()[0]
     formdata = FormDef.select()[0].data_class().select()[0]
@@ -1148,7 +1187,6 @@ def test_lazy_map_variable(pub, variable_test_data):
         pub.substitutions.reset()
         pub.substitutions.feed(formdef)
         with pub.substitutions.temporary_feed(formdata, force_mode=mode):
-            assert WorkflowStatusItem.compute('=form_var_map["lat"]', raises=True) == 2
             assert WorkflowStatusItem.compute('{{ form_var_map }}', raises=True) == '2;4'
             assert WorkflowStatusItem.compute('{{ form_var_map|split:";"|first }}', raises=True) == '2'
             assert WorkflowStatusItem.compute('=form_var_map_lat', raises=True) == 2
@@ -1166,6 +1204,7 @@ def test_lazy_map_variable(pub, variable_test_data):
             assert WorkflowStatusItem.compute('{{ form_var_map|distance:form|floatformat }}', raises=True) == '248515.5'
 
     formdata.data['7'] = None
+    formdata.data['7_display'] = None
     formdata.store()
     pub.substitutions.reset()
     pub.substitutions.feed(formdef)
diff --git a/tests/test_workflows.py b/tests/test_workflows.py
index beb1f5877..683d04274 100644
--- a/tests/test_workflows.py
+++ b/tests/test_workflows.py
@@ -2965,6 +2965,7 @@ def test_geolocate_map(pub):
 
     formdata = formdef.data_class()()
     formdata.data = {'2': {'lat': 48.8337085, 'lon': 2.3233693}}
+    formdata.data['2_display'] = formdef.fields[0].store_display_value(formdata.data, '2')
     formdata.just_created()
     formdata.store()
     pub.substitutions.feed(formdata)
@@ -2995,6 +2996,7 @@ def test_geolocate_overwrite(pub):
 
     formdata = formdef.data_class()()
     formdata.data = {'2': {'lat': 48.8337085, 'lon': 2.3233693}}
+    formdata.data['2_display'] = formdef.fields[0].store_display_value(formdata.data, '2')
     formdata.just_created()
     formdata.store()
     pub.substitutions.feed(formdata)
@@ -3008,11 +3010,13 @@ def test_geolocate_overwrite(pub):
     assert int(formdata.geolocations['base']['lon']) == 2
 
     formdata.data = {'2': {'lat': 48.8337085, 'lon': 3.3233693}}
+    formdata.data['2_display'] = formdef.fields[0].store_display_value(formdata.data, '2')
     item.perform(formdata)
     assert int(formdata.geolocations['base']['lat']) == 48
     assert int(formdata.geolocations['base']['lon']) == 3
 
     formdata.data = {'2': {'lat': 48.8337085, 'lon': 4.3233693}}
+    formdata.data['2_display'] = formdef.fields[0].store_display_value(formdata.data, '2')
     item.overwrite = False
     item.perform(formdata)
     assert int(formdata.geolocations['base']['lat']) == 48
diff --git a/tests/utilities.py b/tests/utilities.py
index 9ed741564..c1afb42b2 100644
--- a/tests/utilities.py
+++ b/tests/utilities.py
@@ -1,5 +1,6 @@
 import email.header
 import email.parser
+import json
 import os
 import tempfile
 import random
@@ -326,6 +327,17 @@ class HttpRequestsMocking(object):
 
         with open(os.path.join(os.path.dirname(__file__), 'idp_metadata.xml')) as fd:
             metadata = fd.read()
+        geojson = {
+            'features': [
+                {'properties': {'id': '1', 'text': 'foo'},
+                 'geometry': {'type': 'Point', 'coordinates': [1, 2]}
+                },
+                {'properties': {'id': '2', 'text': 'bar'},
+                 'geometry': {'type': 'Point', 'coordinates': [3, 4]}
+                }
+            ]
+        }
+
         status, data, headers = {
             'http://remote.example.net/204': (204, None, None),
             'http://remote.example.net/400': (400, 'bad request', None),
@@ -339,6 +351,7 @@ class HttpRequestsMocking(object):
             'http://remote.example.net/json-err0': (200, '{"data": "foo", "err": 0}', None),
             'http://remote.example.net/json-err1': (200, '{"data": "", "err": 1}', None),
             'http://remote.example.net/json-list-err1': (200, '{"data": [{"id": "a", "text": "b"}], "err": 1}', None),
+            'http://remote.example.net/geojson': (200, json.dumps(geojson), None),
             'http://remote.example.net/json-errstr': (200, '{"data": "", "err": "bug"}', None),
             'http://remote.example.net/json-errheader0': (200, '{"foo": "bar"}',
                                               {'x-error-code': '0'}),
diff --git a/wcs/api.py b/wcs/api.py
index 4e018e78f..d0bc4dc76 100644
--- a/wcs/api.py
+++ b/wcs/api.py
@@ -37,6 +37,7 @@ from wcs.categories import Category
 from wcs.conditions import Condition, ValidationError
 from wcs.carddef import CardDef
 from wcs.formdef import FormDef
+from wcs.data_sources import get_object as get_data_source_object
 from wcs.roles import Role, logged_users_role
 from wcs.forms.common import FormStatusPage
 import wcs.qommon.storage as st
@@ -839,10 +840,20 @@ class AutocompleteDirectory(Directory):
         return misc.urlopen(url).read()
 
 
+class GeoJsonDirectory(Directory):
+    def _q_lookup(self, component):
+        try:
+            data_source = get_data_source_object({'type': component})
+        except KeyError:
+            raise TraversalError()
+        get_response().set_content_type('application/json')
+        return json.dumps(data_source.get_geojson_data())
+
+
 class ApiDirectory(Directory):
     _q_exports = ['forms', 'roles', ('reverse-geocoding', 'reverse_geocoding'),
             'formdefs', 'categories', 'user', 'users', 'code', 'autocomplete',
-            'cards']
+            'cards', 'geojson']
 
     cards = ApiCardsDirectory()
     forms = ApiFormsDirectory()
@@ -852,6 +863,7 @@ class ApiDirectory(Directory):
     users = ApiUsersDirectory()
     code = ApiTrackingCodeDirectory()
     autocomplete = AutocompleteDirectory()
+    geojson = GeoJsonDirectory()
 
     def roles(self):
         get_response().set_content_type('application/json')
diff --git a/wcs/data_sources.py b/wcs/data_sources.py
index 73ab1b716..4d9d49ba2 100644
--- a/wcs/data_sources.py
+++ b/wcs/data_sources.py
@@ -52,7 +52,8 @@ def register_data_source_function(function, function_name=None):
 
 class DataSourceSelectionWidget(CompositeWidget):
     def __init__(self, name, value=None, allow_jsonp=True,
-            allow_geojson=False, allow_named_sources=True, **kwargs):
+            allow_geojson=False, allow_named_sources=True,
+            require_configured_geographic_source=False, **kwargs):
         CompositeWidget.__init__(self, name, value, **kwargs)
 
         if not value:
@@ -60,18 +61,23 @@ class DataSourceSelectionWidget(CompositeWidget):
 
         options = []
         if allow_named_sources:
-            options.extend([(x.slug, x.name, x.slug) for x in NamedDataSource.select()])
-            from wcs.carddef import CardDef
-            options.extend(list(CardDef.get_as_data_source_options()))
+            selected_sources = NamedDataSource.select()
+            if require_configured_geographic_source:
+                selected_sources = [x for x in selected_sources if x.type == 'geojson']
+            options.extend([(x.slug, x.name, x.slug) for x in selected_sources])
+            if not require_configured_geographic_source:
+                from wcs.carddef import CardDef
+                options.extend(list(CardDef.get_as_data_source_options()))
             options.sort(key=lambda x: misc.simplify(x[1]))
 
         options.insert(0, (None, _('None'), None))
-        options.append(('json', _('JSON URL'), 'json'))
-        if allow_jsonp:
-            options.append(('jsonp', _('JSONP URL'), 'jsonp'))
-        if allow_geojson:
-            options.append(('geojson', _('GeoJSON URL'), 'geojson'))
-        options.append(('formula', _('Python Expression'), 'python'))
+        if not require_configured_geographic_source:
+            options.append(('json', _('JSON URL'), 'json'))
+            if allow_jsonp:
+                options.append(('jsonp', _('JSONP URL'), 'jsonp'))
+            if allow_geojson:
+                options.append(('geojson', _('GeoJSON URL'), 'geojson'))
+            options.append(('formula', _('Python Expression'), 'python'))
 
         self.add(SingleSelectWidget, 'type', options=options, value=value.get('type'),
                  attrs={'data-dynamic-display-parent': 'true'})
@@ -116,7 +122,7 @@ def get_items(data_source, include_disabled=False, mode=None):
     return tupled_items
 
 
-def request_json_items(url, data_source):
+def get_json_from_url(url, data_source):
     url = sign_url_auto_orig(url)
     geojson = data_source.get('type') == 'geojson'
     try:
@@ -141,6 +147,14 @@ def request_json_items(url, data_source):
         else:
             get_logger().warning('Error reading JSON data source output (%s)' % str(e))
         return None
+    return entries
+
+
+def request_json_items(url, data_source):
+    geojson = data_source.get('type') == 'geojson'
+    entries = get_json_from_url(url, data_source)
+    if entries is None:
+        return None
     items = []
     if geojson:
         id_property = data_source.get('id_property') or 'id'
@@ -419,6 +433,52 @@ class NamedDataSource(XmlStorableObject):
                     get_session().get_data_source_query_url_token(self.get_json_query_url()))
         return None
 
+    def get_geojson_url(self):
+        assert self.type == 'geojson'
+        return '/api/geojson/%s' % self.slug
+
+    def get_geojson_data(self):
+        url = self.data_source.get('value').strip()
+        if Template.is_template_string(url):
+            vars = get_publisher().substitutions.get_context_variables(mode='lazy')
+            url = get_variadic_url(url, vars)
+
+        request = get_request()
+        if hasattr(request, 'datasources_cache') and url in request.datasources_cache:
+            return request.datasources_cache[url]
+
+        cache_duration = 0
+        if self.cache_duration:
+            cache_duration = int(self.cache_duration)
+
+        if cache_duration:
+            cache_key = 'geojson-data-source-%s' % force_str(hashlib.md5(force_bytes(url)).hexdigest())
+            from django.core.cache import cache
+            data = cache.get(cache_key)
+            if data is not None:
+                return data
+
+        data = get_json_from_url(url, self.data_source)
+        id_property = self.id_property or 'id'
+        label_template_property = self.label_template_property or '{{ text }}'
+
+        for feature in data['features']:
+            feature['properties']['_id'] = feature['properties'][id_property]
+            try:
+                feature['properties']['_text'] = Template(
+                        label_template_property).render(feature['properties'])
+            except (TemplateSyntaxError, VariableDoesNotExist):
+                pass
+            if not feature['properties'].get('_text'):
+                feature['properties']['_text'] = feature['properties']['_id']
+
+        if hasattr(request, 'datasources_cache'):
+            request.datasources_cache[url] = data
+        if cache_duration:
+            cache.set(cache_key, data, cache_duration)
+
+        return data
+
     def get_value_by_id(self, param_name, param_value):
         url = self.data_source.get('value').strip()
         if Template.is_template_string(url):
@@ -465,7 +525,7 @@ class NamedDataSource(XmlStorableObject):
         if self.type == 'json' and self.id_parameter:
             value = self.get_value_by_id(self.id_parameter, option_id)
         else:
-            structured_items = get_structured_items(self.data_source, mode='lazy')
+            structured_items = get_structured_items(self.extended_data_source, mode='lazy')
             for item in structured_items:
                 if str(item['id']) == str(option_id):
                     value = item
diff --git a/wcs/fields.py b/wcs/fields.py
index 072287120..152ecd3c2 100644
--- a/wcs/fields.py
+++ b/wcs/fields.py
@@ -544,24 +544,26 @@ class WidgetField(Field):
     def perform_more_widget_changes(self, form, kwargs, edit = True):
         pass
 
-
-    def add_to_view_form(self, form, value = None):
-        kwargs = {'render_br': False}
+    def add_to_view_form(self, form, value=None, display_value=None, **kwargs):
+        widget_kwargs = {'render_br': False}
 
         self.field_key = 'f%s' % self.id
-        self.perform_more_widget_changes(form, kwargs, False)
+        self.perform_more_widget_changes(form, widget_kwargs, False)
 
         for k in self.extra_attributes:
             if hasattr(self, k):
-                kwargs[k] = getattr(self, k)
+                widget_kwargs[k] = getattr(self, k)
 
-        if self.widget_class is StringWidget and not 'size' in kwargs and value:
+        if self.widget_class is StringWidget and not 'size' in widget_kwargs and value:
             # set a size if there is not one already defined, this will be for
             # example the case with ItemField
-            kwargs['size'] = len(value)
+            widget_kwargs['size'] = len(value)
 
-        form.add(self.widget_class, self.field_key, title = self.label,
-                value = value, readonly = 'readonly', **kwargs)
+        if display_value and getattr(self.widget_class, 'allow_display_value', True):
+            widget_kwargs['display_value'] = display_value
+
+        form.add(self.widget_class, self.field_key, title=self.label,
+                value=value, readonly='readonly', **widget_kwargs)
         widget = form.get_widget(self.field_key)
         widget.transfer_form_value(get_request())
         widget.field = self
@@ -570,6 +572,7 @@ class WidgetField(Field):
                 widget.extra_css_class = '%s %s' % (widget.extra_css_class, self.extra_css_class)
             else:
                 widget.extra_css_class = self.extra_css_class
+        return widget
 
     def get_display_locations_options(self):
         return [('validation', _('Validation Page')),
@@ -1370,9 +1373,9 @@ class DateField(WidgetField):
             value = self.convert_value_to_str(value)
         return WidgetField.add_to_form(self, form, value=value)
 
-    def add_to_view_form(self, form, value=None):
+    def add_to_view_form(self, form, value=None, **kwargs):
         value = strftime(misc.date_format(), value)
-        return super().add_to_view_form(form, value=value)
+        return super().add_to_view_form(form, value=value, **kwargs)
 
     def get_view_value(self, value, **kwargs):
         try:
@@ -1582,7 +1585,7 @@ class ItemField(WidgetField):
         span.text = od_clean_text(force_text(value))
         return span
 
-    def add_to_view_form(self, form, value = None):
+    def add_to_view_form(self, form, value=None, **kwargs):
         real_value = value
         label_value = ''
         if value is not None:
@@ -2406,6 +2409,7 @@ class MapField(WidgetField):
     max_zoom = None
     default_position = None
     init_with_geoloc = False
+    data_source = {}
 
     widget_class = MapWidget
     extra_attributes = ['initial_zoom', 'min_zoom', 'max_zoom',
@@ -2434,6 +2438,13 @@ class MapField(WidgetField):
         form.add(CheckboxWidget, 'init_with_geoloc',
                 title=_('Initialize position using device geolocation'),
                 value=self.init_with_geoloc, required=False)
+        form.add(data_sources.DataSourceSelectionWidget, 'data_source',
+                 value=self.data_source,
+                 title=_('Markers data source'),
+                 hint=_('This will fill the map with markers from an external source.'),
+                 required=False,
+                 require_configured_geographic_source=True,
+                 advanced=False)
 
     def check_admin_form(self, form):
         initial_zoom = form.get_widget('initial_zoom').parse()
@@ -2451,7 +2462,21 @@ class MapField(WidgetField):
     def get_admin_attributes(self):
         return WidgetField.get_admin_attributes(self) + ['initial_zoom',
                 'min_zoom', 'max_zoom', 'default_position',
-                'init_with_geoloc']
+                'init_with_geoloc', 'data_source']
+
+    def add_to_view_form(self, form, value=None, **kwargs):
+        widget = super().add_to_view_form(form, value=value, **kwargs)
+        if self.data_source:
+            data_source = data_sources.get_object(self.data_source)
+            widget.add_geojson_markers(data_source.get_geojson_url())
+        return widget
+
+    def add_to_form(self, form, value=None):
+        widget = super().add_to_form(form, value=value)
+        if self.data_source:
+            data_source = data_sources.get_object(self.data_source)
+            widget.add_geojson_markers(data_source.get_geojson_url())
+        return widget
 
     def get_prefill_value(self, user=None, force_string=True):
         if self.prefill.get('type') != 'string' or not self.prefill.get('value'):
@@ -2466,9 +2491,19 @@ class MapField(WidgetField):
         return (coords, False)
 
     def get_view_value(self, value, **kwargs):
-        widget = self.widget_class('x%s' % random.random(), value, readonly=True)
+        widget_kwargs = {}
+        if 'value_id' in kwargs:  # get raw value
+            widget_kwargs['display_value'] = value
+            value = kwargs.get('value_id')
+        widget = self.widget_class('x%s' % random.random(), value, readonly=True, **widget_kwargs)
         return widget.render_widget_content()
 
+    def get_view_short_value(self, value, max_len=30, raw_value=None, **kwargs):
+        if self.data_source:
+            # value will be _display value
+            return value or ''
+        return self.get_view_value(value, value_id=raw_value)
+
     def get_rst_view_value(self, value, indent=''):
         if isinstance(value, str):  # compatiblity with old pickled data
             return indent + value
@@ -2490,6 +2525,21 @@ class MapField(WidgetField):
             return self.convert_value_from_str(value)
         return value
 
+    def get_display_value(self, value):
+        if not self.data_source:
+            if value:
+                # backward compatibility
+                return '%(lat)s;%(lon)s' % value
+            return ''
+        data_source = data_sources.get_object(self.data_source)
+        if data_source is None or value is None:
+            return ''
+        return data_source.get_display_value(value.get('marker_id'))
+
+    def store_display_value(self, data, field_id):
+        value = data.get(field_id)
+        return self.get_display_value(value)
+
     def get_structured_value(self, data):
         return self.get_json_value(data.get(self.id))
 
diff --git a/wcs/formdata.py b/wcs/formdata.py
index ea13e4ffd..b45b62482 100644
--- a/wcs/formdata.py
+++ b/wcs/formdata.py
@@ -676,8 +676,8 @@ class FormData(StorableObject):
         if max_length is not None:
             # if max_length is set the target is a backoffice listing/table,
             # return an html value, appropriately shortened.
-            field_value = self.data.get('%s_display' % field.id, field_value)
-            return field.get_view_short_value(field_value, max_length)
+            display_value = self.data.get('%s_display' % field.id, field_value)
+            return field.get_view_short_value(display_value, max_length, raw_value=field_value)
         else:
             # otherwise return the actual "raw" field value
             return field_value
@@ -1172,7 +1172,7 @@ class FormData(StorableObject):
                 continue
 
             value, value_details = f.get_value_info(self.data)
-            if value is None and not (f.required and include_unset_required_fields):
+            if (value is None and not value_details) and not (f.required and include_unset_required_fields):
                 continue
 
             current_page_fields.append({'field': f, 'value': value, 'value_details': value_details})
diff --git a/wcs/formdef.py b/wcs/formdef.py
index 167af8ece..6e42a5536 100644
--- a/wcs/formdef.py
+++ b/wcs/formdef.py
@@ -665,7 +665,10 @@ class FormDef(StorableObject):
                     field.add_to_view_form(form, value)
                     form.widgets.append(HtmlWidget(htmltext('</div>')))
                 else:
-                    field.add_to_view_form(form, value)
+                    kwargs = {}
+                    if field.store_display_value:
+                        kwargs['display_value'] = dict.get(field.id + '_display')
+                    field.add_to_view_form(form, value, **kwargs)
 
             if visible_contents:
                 form.widgets.append(HtmlWidget(htmltext('</div></div>')))
diff --git a/wcs/forms/common.py b/wcs/forms/common.py
index dfdc09bae..1e36644ac 100644
--- a/wcs/forms/common.py
+++ b/wcs/forms/common.py
@@ -491,7 +491,7 @@ class FormStatusPage(Directory, FormTemplateMixin):
             r += htmltext('<div class="%s">' % ' '.join(css_classes))
             r += htmltext('<span class="label">%s</span> ') % f.label
             value, value_details = field_value_info['value'], field_value_info['value_details']
-            if value is None:
+            if value is None and not value_details:
                 r += htmltext('<div class="value"><i>%s</i></div>') % _('Not set')
             else:
                 r += htmltext('<div class="value">')
diff --git a/wcs/qommon/form.py b/wcs/qommon/form.py
index bdb4854fe..fa8845268 100644
--- a/wcs/qommon/form.py
+++ b/wcs/qommon/form.py
@@ -2361,8 +2361,10 @@ $(function() {
 
 class MapWidget(CompositeWidget):
     template_name = 'qommon/forms/widgets/map.html'
+    geojson_markers_url = None
 
     def __init__(self, name, value=None, **kwargs):
+        self.display_value = kwargs.pop('display_value', None)
         CompositeWidget.__init__(self, name, value, **kwargs)
         latlng_value = None
         if isinstance(value, str):  # legacy data type
@@ -2390,6 +2392,10 @@ class MapWidget(CompositeWidget):
     def transfer_form_value(self, request):
         request.form[self.get_widget('latlng').name] = self.point2str(self.value)
 
+    def add_geojson_markers(self, url):
+        self.geojson_markers_url = url
+        self.add(HiddenWidget, 'marker_id')
+
     def initial_position(self):
         if isinstance(self.value, str):
             return {'lat': self.value.split(';')[0],
@@ -2408,6 +2414,8 @@ class MapWidget(CompositeWidget):
         if self.value:
             lat, lon = self.value.split(';')
             self.value = misc.normalize_geolocation({'lat': lat, 'lon': lon})
+            if self.geojson_markers_url:
+                self.value['marker_id'] = self.get('marker_id')
 
 
 class HiddenErrorWidget(HiddenWidget):
diff --git a/wcs/qommon/static/images/blank-marker-icon.png b/wcs/qommon/static/images/blank-marker-icon.png
new file mode 100644
index 0000000000000000000000000000000000000000..830438ddb4889d415b35d93aca44572b3a2b7596
GIT binary patch
diff --git a/wcs/qommon/static/js/qommon.map.js b/wcs/qommon/static/js/qommon.map.js
index 595e239df..75812bfa0 100644
--- a/wcs/qommon/static/js/qommon.map.js
+++ b/wcs/qommon/static/js/qommon.map.js
@@ -46,10 +46,14 @@ $(window).on('load', function() {
      var hidden = $(this).prev();
      map.marker = null;
      var latlng;
+     var initial_marker_id = $map_widget.data('markers-initial-id');
      if ($map_widget.data('init-lat')) {
        latlng = [$map_widget.data('init-lat'), $map_widget.data('init-lng')]
-       map.marker = L.marker(latlng);
-       map.marker.addTo(map);
+       if (typeof initial_marker_id == 'undefined') {
+         // if markers are used they will appear via their input widget
+         map.marker = L.marker(latlng);
+         map.marker.addTo(map);
+       }
      } else if ($map_widget.data('def-lat')) {
        latlng = [$map_widget.data('def-lat'), $map_widget.data('def-lng')]
      } else {
@@ -67,7 +71,52 @@ $(window).on('load', function() {
      } else {
        map.addControl(gps_control);
      }
-     if (! $map_widget.data('readonly')) {
+     if ($map_widget.data('markers-url')) {
+       var radio_name = $map_widget.data('markers-radio-name');
+       $map_widget.on('change', 'input[name="' + radio_name + '"]', function() {
+         var $radio = $(this);
+         if ($radio.is(':checked')) {
+           hidden.val($radio.data('lat') + ';' + $radio.data('lng'));
+           hidden.trigger('change');
+         }
+       });
+       $.getJSON($map_widget.data('markers-url')).done(
+         function(data) {
+           var geo_json = L.geoJson(data, {
+             pointToLayer: function (feature, latlng) {
+               var $label = $('<label>', {
+                 title: feature.properties._text
+               });
+               var $radio = $('<input>', {
+                 value: feature.properties._id,
+                 name: radio_name,
+                 type: 'radio',
+                 'data-lat': latlng.lat,
+                 'data-lng': latlng.lng
+               });
+               if (typeof initial_marker_id !== 'undefined' && feature.properties._id == initial_marker_id) {
+                 $radio.attr('checked', 'checked');
+               }
+               $label.append($radio);
+               $label.append($('<span></span>'));
+               var div_marker = L.divIcon({
+                 className: 'item-marker',
+                 html: '<div>' + $label.prop('outerHTML') + '</div>'
+               });
+               return L.marker(latlng, {icon: div_marker});
+             }
+           });
+           geo_json.addTo(map);
+         }
+       );
+     } else if ($map_widget.data('readonly') && initial_marker_id) {
+       // readonly and marker
+       console.log('plop');
+         map.marker = L.marker(latlng);
+         map.marker.addTo(map);
+     }
+
+     if (! $map_widget.data('readonly') && ! $map_widget.data('markers-url')) {
        map.on('click', function(e) {
          $map_widget.trigger('set-geolocation', e.latlng);
        });
diff --git a/wcs/qommon/templates/qommon/forms/widgets/map.html b/wcs/qommon/templates/qommon/forms/widgets/map.html
index f8375a006..dcfa46bd1 100644
--- a/wcs/qommon/templates/qommon/forms/widgets/map.html
+++ b/wcs/qommon/templates/qommon/forms/widgets/map.html
@@ -2,6 +2,10 @@
 
 {% block widget-control %}
 {% localize off %}
+{% if widget.display_value and widget.readonly %}
+<div class="display-value"><input type="text"
+  value="{{widget.display_value}}" readonly size="{{widget.display_value|length}}"></div>
+{% endif %}
 <input type="hidden" name="{{widget.name}}$latlng" {% if widget.value %}value="{{widget.value.lat}};{{widget.value.lon}}"{% endif %}>
 <div id="map-{{widget.name}}" class="qommon-map"
   {% if widget.readonly %}data-readonly="true"{% endif %}
@@ -11,6 +15,39 @@
     data-init-lat="{{ widget.initial_position.lat }}"
     data-init-lng="{{ widget.initial_position.lng }}"
   {% endif %}
+  {% if widget.geojson_markers_url %}
+    {% if not widget.readonly %}
+    data-markers-url="{{ widget.geojson_markers_url }}"
+    data-markers-radio-name="{{widget.name}}$marker_id"
+    {% endif %}
+    {% if widget.value.marker_id %}
+    data-markers-initial-id="{{ widget.value.marker_id }}"
+    {% endif %}
+  {% endif %}
 ></div>
+{% if widget.readonly and widget.geojson_markers_url and widget.value.marker_id %}
+<input type="hidden" name="{{widget.name}}$marker_id" value="{{ widget.value.marker_id }}">
+{% endif %}
 {% endlocalize %}
+
+{% if widget.geojson_markers_url %}
+<style>
+.item-marker input + span {
+  position: relative;
+  z-index: 1000;
+  margin-top: -55px;
+  margin-left: -5px;
+  display: block;
+  width: 25px;
+  height: 41px;
+  background: url(/static/images/blank-marker-icon.png);
+}
+.item-marker input:checked + span {
+  background: url(/static/xstatic/images/marker-icon.png);
+}
+.MapWidget div.display-value {
+  max-width: 100%;
+}
+</style>
+{% endif %}
 {% endblock %}
diff --git a/wcs/variables.py b/wcs/variables.py
index 128e8d262..65b20ffc5 100644
--- a/wcs/variables.py
+++ b/wcs/variables.py
@@ -840,10 +840,22 @@ class LazyFieldVarMap(LazyFieldVarStructured):
         return self._data.get(self._field.id).split(*args, **kwargs)
 
     def inspect_keys(self):
+        if self._field.data_source:
+            return ['lat', 'lon', 'marker_id']
         return ['lat', 'lon']
 
+    @property
+    def marker_id(self):
+        value = self._data.get(self._field.id)
+        if value:
+            return value.get('marker_id')
+
     def __str__(self):
-        # backward compatibility
+        if self._field.data_source:
+            value = self._data.get(self._field.id + '_display')
+            if value:
+                return value
+        # "lat;lon" for backward compatibility
         value = self._data.get(self._field.id)
         if not value:
             return ''
-- 
2.28.0

