From bac6b5a1044bc9cac7f2e705ab538d40093c75f8 Mon Sep 17 00:00:00 2001
From: Nicolas ROCHE <nroche@entrouvert.com>
Date: Wed, 9 Oct 2019 16:42:44 +0200
Subject: [PATCH] datasources: avoid error 500 on lost datasource (#18431)

---
 tests/test_form_pages.py | 20 ++++++++++++++++++
 wcs/fields.py            | 45 ++++++++++++++++++++++++++++------------
 wcs/formdef.py           |  5 ++++-
 wcs/forms/common.py      |  5 ++++-
 4 files changed, 60 insertions(+), 15 deletions(-)

diff --git a/tests/test_form_pages.py b/tests/test_form_pages.py
index 1f9a122d..d2e7dc80 100644
--- a/tests/test_form_pages.py
+++ b/tests/test_form_pages.py
@@ -4238,6 +4238,26 @@ def test_form_text_field_submit(pub):
     resp = resp.forms[0].submit('submit')
     assert 'Check values then click submit.' in resp.body
 
+def test_unknown_datasource(pub):
+    formdef = create_formdef()
+    formdef.fields = [
+        fields.StringField(id='1', label='string',
+                varname='string', required=False, data_source={'type': 'foobar'}),
+        fields.ItemField(id='2', label='item',
+                varname='item', required=False, data_source={'type': 'foobar'}),
+        fields.ItemsField(id='3', label='items',
+                varname='items', required=False, data_source={'type': 'foobar'}),
+    ]
+    formdef.store()
+
+    data_class = formdef.data_class()
+    data_class.wipe()
+
+    resp = get_app(pub).get('/test/')
+    formdef.data_class().wipe()
+    resp = resp.forms[0].submit('submit') # should go straight to validation
+    assert 'Check values then click submit.' in resp.body
+
 def test_form_items_datasource(pub):
     formdef = create_formdef()
     formdef.fields = [fields.ItemsField(id='1', label='items',
diff --git a/wcs/fields.py b/wcs/fields.py
index bdf3f1bb..baa72c3b 100644
--- a/wcs/fields.py
+++ b/wcs/fields.py
@@ -574,6 +574,22 @@ class WidgetField(Field):
             return [self.convert_value_to_str(element)]
         return [element]
 
+    def call_data_source(self, fonction, **kwargs):
+        try:
+            return fonction(self.data_source, **kwargs)
+        except KeyError as e:
+            get_logger().warn('%s from %s field on %s form'
+                              % (e, self.label, self.formdef.name))
+            return None
+
+    def get_data_source_iter(self, fonction, **kwargs):
+        try:
+            return fonction(self.data_source, **kwargs)
+        except KeyError as e:
+            get_logger().warn('%s from %s field on %s form'
+                              % (e, self.label, self.formdef.name))
+            return []
+
 field_classes = []
 field_types = []
 
@@ -724,7 +740,9 @@ class StringField(WidgetField):
 
     def perform_more_widget_changes(self, form, kwargs, edit=True):
         if self.data_source:
-            real_data_source = data_sources.get_real(self.data_source)
+            real_data_source = self.call_data_source(data_sources.get_real)
+            if not real_data_source:
+                return
             if real_data_source.get('type') == 'jsonp':
                 kwargs['url'] = real_data_source.get('value')
                 self.widget_class = AutocompleteStringWidget
@@ -1304,13 +1322,13 @@ class ItemField(WidgetField):
 
     def get_options(self, mode=None):
         if self.data_source:
-            return [x[:3] for x in data_sources.get_items(self.data_source, mode=mode)]
+            return [x[:3] for x in self.get_data_source_iter(data_sources.get_items, mode=mode)]
         if self.items:
             return [(x, x) for x in self.items]
         return []
 
     def perform_more_widget_changes(self, form, kwargs, edit=True):
-        data_source = data_sources.get_object(self.data_source)
+        data_source = self.call_data_source(data_sources.get_object)
 
         if data_source and data_source.type == 'jsonp':
             # a source defined as JSONP can only be used in autocomplete mode
@@ -1324,7 +1342,7 @@ class ItemField(WidgetField):
         if self.items:
             kwargs['options'] = self.get_options()
         elif self.data_source:
-            items = data_sources.get_items(self.data_source,
+            items = self.get_data_source_iter(data_sources.get_items,
                     include_disabled=self.display_disabled_items)
             kwargs['options'] = [x[:3] for x in items if not x[-1].get('disabled')]
             kwargs['options_with_attributes'] = items[:]
@@ -1346,7 +1364,7 @@ class ItemField(WidgetField):
             kwargs['select2'] = True
 
     def get_display_value(self, value):
-        data_source = data_sources.get_object(self.data_source)
+        data_source = self.call_data_source(data_sources.get_object)
         if data_source is None:
             return value or ''
 
@@ -1378,7 +1396,7 @@ class ItemField(WidgetField):
 
     def store_display_value(self, data, field_id):
         value = data.get(field_id)
-        data_source = data_sources.get_object(self.data_source)
+        data_source = self.call_data_source(data_sources.get_object)
         if data_source and data_source.type == 'jsonp':
             if get_request():
                 display_value = get_request().form.get('f%s_display' % field_id)
@@ -1397,7 +1415,7 @@ class ItemField(WidgetField):
         return self.get_display_value(value)
 
     def store_structured_value(self, data, field_id):
-        data_source = data_sources.get_object(self.data_source)
+        data_source = self.call_data_source(data_sources.get_object)
         if data_source is None:
             return
 
@@ -1454,7 +1472,7 @@ class ItemField(WidgetField):
         return item_items_stats(self, values)
 
     def feed_session(self, value, display_value):
-        real_data_source = data_sources.get_real(self.data_source)
+        real_data_source = self.call_data_source(data_sources.get_real)
         if real_data_source and real_data_source.get('type') == 'jsonp':
             if not get_session().jsonp_display_values:
                 get_session().jsonp_display_values = {}
@@ -1476,7 +1494,7 @@ class ItemField(WidgetField):
     def export_to_json(self, include_id=False, anonymise=True):
         field = super(ItemField, self).export_to_json(include_id=include_id, anonymise=anonymise)
         if self.data_source and not anonymise:
-            structured_items = data_sources.get_structured_items(self.data_source)
+            structured_items = self.call_data_source(data_sources.get_structured_items)
             if structured_items:
                 field['structured_items'] = structured_items
                 if not field.get('items'):
@@ -1509,7 +1527,8 @@ class ItemsField(WidgetField):
         if self.data_source:
             if self._cached_data_source:
                 return self._cached_data_source
-            self._cached_data_source = [x[:3] for x in data_sources.get_items(self.data_source)]
+            self._cached_data_source = [x[:3]
+                    for x in self.get_data_source_iter(data_sources.get_items)]
             return self._cached_data_source[:]
         elif self.items:
             return [(x, x) for x in self.items]
@@ -1520,7 +1539,7 @@ class ItemsField(WidgetField):
         kwargs['options'] = self.get_options()
         kwargs['max_choices'] = self.max_choices
         if self.data_source:
-            items = data_sources.get_items(self.data_source,
+            items = self.get_data_source_iter(data_sources.get_items,
                     include_disabled=self.display_disabled_items)
             kwargs['options'] = [x[:3] for x in items if not x[-1].get('disabled')]
             kwargs['options_with_attributes'] = items[:]
@@ -1634,7 +1653,7 @@ class ItemsField(WidgetField):
         value = data.get(field_id)
         if not self.data_source:
             return
-        structured_options = data_sources.get_structured_items(self.data_source)
+        structured_options = self.call_data_source(data_sources.get_structured_items)
         if not structured_options:
             return
         if not set(structured_options[0].keys()) != set(['id', 'text']):
@@ -1649,7 +1668,7 @@ class ItemsField(WidgetField):
     def export_to_json(self, include_id=False, anonymise=True):
         field = super(ItemsField, self).export_to_json(include_id=include_id, anonymise=True)
         if self.data_source and not anonymise:
-            structured_items = data_sources.get_structured_items(self.data_source)
+            structured_items = self.call_data_source(data_sources.get_structured_items)
             if structured_items:
                 field['structured_items'] = structured_items
                 if not field.get('items'):
diff --git a/wcs/formdef.py b/wcs/formdef.py
index 473cb7a4..0f29bc12 100644
--- a/wcs/formdef.py
+++ b/wcs/formdef.py
@@ -667,7 +667,10 @@ class FormDef(StorableObject):
                         live_condition_fields[varname] = []
                     live_condition_fields[varname].append(field)
             if field.key == 'item' and field.data_source:
-                real_data_source = data_sources.get_real(field.data_source)
+                try:
+                    real_data_source = data_sources.get_real(field.data_source)
+                except KeyError as e:
+                    continue
                 if real_data_source.get('type') != 'json':
                     continue
                 varnames = field.get_referenced_varnames(
diff --git a/wcs/forms/common.py b/wcs/forms/common.py
index e4c60adb..24d5ef44 100644
--- a/wcs/forms/common.py
+++ b/wcs/forms/common.py
@@ -651,7 +651,10 @@ class FormStatusPage(Directory, FormTemplateMixin):
 
         for field in displayed_fields:
             if field.key == 'item' and field.data_source:
-                data_source = data_sources.get_object(field.data_source)
+                try:
+                    data_source = data_sources.get_object(field.data_source)
+                except KeyError as e:
+                    continue
                 if data_source.type != 'json':
                     continue
                 varnames = field.get_referenced_varnames(
-- 
2.23.0

