From 460751999eaa12ec884f8d6548d885c2a3f7ec7a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20P=C3=A9ters?= <fpeters@entrouvert.com>
Date: Mon, 24 Feb 2020 09:56:02 +0100
Subject: [PATCH] general: use lazy variables to generate inspect page (#39849)

---
 tests/test_backoffice_pages.py | 25 ++++++++++++
 wcs/backoffice/management.py   | 40 ++++++++++---------
 wcs/data_sources.py            |  3 ++
 wcs/qommon/substitution.py     | 21 ++++++++++
 wcs/variables.py               | 70 +++++++++++++++++++++++++++++-----
 wcs/wf/create_formdata.py      |  5 +++
 6 files changed, 135 insertions(+), 29 deletions(-)

diff --git a/tests/test_backoffice_pages.py b/tests/test_backoffice_pages.py
index 4ebced497..44e87f160 100644
--- a/tests/test_backoffice_pages.py
+++ b/tests/test_backoffice_pages.py
@@ -32,6 +32,7 @@ from wcs.qommon import errors, sessions
 from wcs.qommon.form import PicklableUpload
 from wcs.qommon.ident.password_accounts import PasswordAccount
 from wcs.qommon.http_request import HTTPRequest
+from wcs.qommon.substitution import CompatibilityNamesDict
 from wcs.roles import Role
 from wcs.workflows import (Workflow, CommentableWorkflowStatusItem,
         ChoiceWorkflowStatusItem, EditableWorkflowStatusItem,
@@ -5923,6 +5924,30 @@ def test_backoffice_create_formdata_backoffice_submission(create_formdata):
     assert pq('.field-type-file .value').text() == 'bar'
 
 
+def test_linked_forms_variables(create_formdata):
+    # create source formdata
+    formdata = create_formdata['source_formdef'].data_class()()
+    upload = PicklableUpload('/foo/bar', content_type='text/plain')
+    upload.receive([b'hello world'])
+    formdata.data = {
+        '0': 'coucou',
+        '1': upload,
+    }
+    formdata.just_created()
+    formdata.store()
+    formdata.perform_workflow()
+    formdata.store()
+    formdata.jump_status('2')
+    formdata.perform_workflow()
+    formdata.store()
+
+    get_publisher().substitutions.reset()
+    get_publisher().substitutions.feed(formdata)
+    substvars = get_publisher().substitutions.get_context_variables(mode='lazy')
+    assert str(substvars['form_links_resubmitted_form_var_foo_string']) == 'coucou'
+    assert 'form_links_resubmitted_form_var_foo_string' in substvars.get_flat_keys()
+
+
 def test_backoffice_create_formdata_map_fields_by_varname(create_formdata):
     create_formdata['create_formdata'].map_fields_by_varname = True
     create_formdata['create_formdata'].mappings  = []
diff --git a/wcs/backoffice/management.py b/wcs/backoffice/management.py
index 7f50dfb9c..d5014f4ec 100644
--- a/wcs/backoffice/management.py
+++ b/wcs/backoffice/management.py
@@ -19,6 +19,7 @@ import datetime
 import json
 import re
 import time
+import types
 import vobject
 
 try:
@@ -44,6 +45,7 @@ from ..qommon import misc, get_logger
 from ..qommon.evalutils import make_datetime
 from ..qommon.misc import C_, ellipsize
 from ..qommon.afterjobs import AfterJob
+from ..qommon.substitution import CompatibilityNamesDict
 from ..qommon import emails
 from ..qommon import sms
 from ..qommon import errors
@@ -62,6 +64,7 @@ from wcs.categories import Category
 from wcs.formdata import FormData
 from wcs.formdef import FormDef
 from wcs.roles import logged_users_role, Role
+from wcs.variables import LazyFieldVar
 from wcs.workflows import get_role_translation, template_on_formdata
 
 from .submission import FormFillPage
@@ -2619,8 +2622,10 @@ class FormBackOfficeStatusPage(FormStatusPage):
         r += htmltext('<div id="inspect-variables" class="section">')
         r += htmltext('<h2>%s</h2>') % _('Variables')
         r += htmltext('<ul class="form-inspector biglist">')
-        substvars = self.filled.get_static_substitution_variables()
-        substvars.update(self.filled.formdef.get_static_substitution_variables())
+
+        substvars = CompatibilityNamesDict()
+        substvars.update(self.filled.get_substitution_variables())
+        static_substvars = {}
 
         def safe(v):
             if isinstance(v, str):
@@ -2635,25 +2640,22 @@ class FormBackOfficeStatusPage(FormStatusPage):
                     v = repr(v)
             return v
 
-        backward_compatibility_varnames = re.compile('^(attachments|form_field_.*|'
-                                                 'form_f[0-9]+|form_fbo[0-9]+|form_user_f[0-9]+|'
-                                                 'form_f[a-f0-9]{8}_.*|'
-                                                 'form_fbo[a-f0-9]{8}_.*|'
-                                                 'form_user_f[a-f0-9]{8}_.*|'
-                                                 'form_user_field_.*|form_user_f_.*)$')
-        blacklisted_varnames = ['form_objects']
-
-        for k, v in sorted(substvars.items()):
-            if k in blacklisted_varnames:
+        for k in sorted(substvars.get_flat_keys()):
+            k = safe(k)
+            v = substvars[k]
+            if isinstance(v, LazyFieldVar):
+                r += htmltext('<li><code title="%s">%s</code>') % (k, k)
+                r += htmltext('  <div class="value"><span>%s</span>') % v
+            elif isinstance(v, (types.FunctionType, types.MethodType)):
                 continue
-            if backward_compatibility_varnames.search(k):
+            elif hasattr(v, 'inspect_keys') or isinstance(v, dict):
+                # skip expanded
                 continue
-
-            k = safe(k)
-            r += htmltext('<li><code title="%s">%s</code>') % (k, k)
-            r += htmltext('  <div class="value"><span>%s</span>') % ellipsize(safe(v), 10000)
-            if not isinstance(v, six.string_types):
-                r += htmltext(' <span class="type">(%r)</span>') % type(v)
+            else:
+                r += htmltext('<li><code title="%s">%s</code>') % (k, k)
+                r += htmltext('  <div class="value"><span>%s</span>') % ellipsize(safe(v), 10000)
+                if not isinstance(v, six.string_types):
+                    r += htmltext(' <span class="type">(%r)</span>') % type(v)
             r += htmltext('</div></li>')
         r += htmltext('</div>')
 
diff --git a/wcs/data_sources.py b/wcs/data_sources.py
index 434768082..67a15b46f 100644
--- a/wcs/data_sources.py
+++ b/wcs/data_sources.py
@@ -451,3 +451,6 @@ class StubNamedDataSource(NamedDataSource):
 class DataSourcesSubstitutionProxy(object):
     def __getattr__(self, attr):
         return get_structured_items(NamedDataSource.get_by_slug(attr).data_source)
+
+    def inspect_keys(self):
+        return []
diff --git a/wcs/qommon/substitution.py b/wcs/qommon/substitution.py
index 64145541c..eb601f176 100644
--- a/wcs/qommon/substitution.py
+++ b/wcs/qommon/substitution.py
@@ -154,6 +154,27 @@ class CompatibilityNamesDict(dict):
         except KeyError:
             return default
 
+    def get_flat_keys(self):
+        flat_keys = {}
+
+        def flatten(base, depth=10):
+            item = self[base]
+            if hasattr(item, 'inspect_keys'):
+                sub_keys = item.inspect_keys()
+            elif isinstance(item, dict):
+                sub_keys = item.keys()
+            else:
+                return
+            for sub_key in sub_keys:
+                new_base = '%s_%s' % (base, sub_key)
+                flat_keys[new_base] = None
+                flatten(new_base, depth=depth - 1)
+
+        for key in self.keys():
+            flatten(key)
+
+        return flat_keys.keys()
+
     def __getitem__(self, key):
         try:
             val = super(CompatibilityNamesDict, self).__getitem__(key)
diff --git a/wcs/variables.py b/wcs/variables.py
index 8c50bc635..586344447 100644
--- a/wcs/variables.py
+++ b/wcs/variables.py
@@ -168,6 +168,9 @@ class LazyFormDefOption(object):
     def __init__(self, formdef):
         self._formdef = formdef
 
+    def inspect_keys(self):
+        return self.options.keys()
+
     _options = None
     @property
     def options(self):
@@ -202,6 +205,13 @@ class LazyFormData(LazyFormDef):
         super(LazyFormData, self).__init__(formdata.formdef)
         self._formdata = formdata
 
+    def inspect_keys(self):
+        blacklist = ('field', 'inspect_keys', 'page_no', 'role', 'formdef', 'objects')
+        for key in dir(self):
+            if key[0] == '_' or key in blacklist:
+                continue
+            yield key
+
     @property
     def objects(self):
         return LazyFormDefObjectsManager(self._formdef, formdata=self._formdata)
@@ -442,6 +452,9 @@ class LazyFormDataVar(object):
         self._data = data or {}
         self._formdata = formdata
 
+    def inspect_keys(self):
+        return self.varnames.keys()
+
     _varnames = None
     @property
     def varnames(self):
@@ -488,6 +501,8 @@ class LazyFormDataVar(object):
             return LazyFieldVarMap(self._data, field, self._formdata)
         if field.key == 'password':
             return LazyFieldVarPassword(self._data, field, self._formdata)
+        if field.store_structured_value:
+            return LazyFieldVarStructured(self._data, field, self._formdata)
 
         return LazyFieldVar(self._data, field, self._formdata)
 
@@ -560,15 +575,6 @@ class LazyFieldVar(object):
             return getattr(self, key)
         except AttributeError:
             pass
-        structured_value = self._field.get_structured_value(self._data)
-        if not structured_value:
-            raise KeyError(key)
-        if isinstance(structured_value, dict):
-            return structured_value[key]
-        if isinstance(structured_value, list):
-            for i, struct_value in enumerate(structured_value):
-                if str(key) == str(i):
-                    return struct_value
         raise KeyError(key)
 
     def __iter__(self):
@@ -590,6 +596,41 @@ class LazyFieldVar(object):
         raise AssertionError('lazy cannot be pickled')
 
 
+class LazyFieldVarStructured(LazyFieldVar):
+    def inspect_keys(self):
+        structured_value = self._field.get_structured_value(self._data)
+        if not structured_value:
+            return []
+
+        keys = ['raw']
+
+        def walk(base, value):
+            if isinstance(value, dict):
+                for k, v in value.items():
+                    walk(k if not base else base + '_' + k, v)
+            else:
+                keys.append(base)
+
+        walk('', structured_value)
+        return keys
+
+    def __getitem__(self, key):
+        try:
+            return super(LazyFieldVarStructured, self).__getitem__(key)
+        except KeyError:
+            pass
+        structured_value = self._field.get_structured_value(self._data)
+        if not structured_value:
+            raise KeyError(key)
+        if isinstance(structured_value, dict):
+            return structured_value[key]
+        if isinstance(structured_value, list):
+            for i, struct_value in enumerate(structured_value):
+                if str(key) == str(i):
+                    return struct_value
+        raise KeyError(key)
+
+
 class DateOperatorsMixin(object):
     def __eq__(self, other):
         if hasattr(other, 'timetuple'):
@@ -643,6 +684,9 @@ def lazy_date(value):
 
 
 class LazyFieldVarDate(DateOperatorsMixin, LazyFieldVar):
+    def inspect_keys(self):
+        return ['year', 'month', 'day']
+
     def get_raw(self):
         return self._data.get(self._field.id)
 
@@ -688,12 +732,15 @@ class LazyFieldVarDate(DateOperatorsMixin, LazyFieldVar):
     day = tm_mday
 
 
-class LazyFieldVarMap(LazyFieldVar):
+class LazyFieldVarMap(LazyFieldVarStructured):
     def split(self, *args, **kwargs):
         # Compatibility with usage of map variable as a string. It is
         # recommended to use lat/lon properties instead.
         return self._data.get(self._field.id).split(*args, **kwargs)
 
+    def inspect_keys(self):
+        return ['lat', 'lon']
+
 
 class LazyFieldVarPassword(LazyFieldVar):
     def __getitem__(self, key):
@@ -708,6 +755,9 @@ class LazyUser(object):
     def __init__(self, user):
         self._user = user
 
+    def inspect_keys(self):
+        return ['display_name', 'email', 'var']
+
     @property
     def display_name(self):
         return self._user.display_name
diff --git a/wcs/wf/create_formdata.py b/wcs/wf/create_formdata.py
index e70678f39..0d644ec31 100644
--- a/wcs/wf/create_formdata.py
+++ b/wcs/wf/create_formdata.py
@@ -177,6 +177,11 @@ class LazyFormDataLinks(object):
                 return part.formdata.get_substitution_variables()
         raise AttributeError(varname)
 
+    def inspect_keys(self):
+        for part in self._formdata.iter_evolution_parts():
+            if isinstance(part, LinkedFormdataEvolutionPart) and part.varname:
+                yield part.varname
+
 
 class CreateFormdataWorkflowStatusItem(WorkflowStatusItem):
     description = N_('New Form Creation')
-- 
2.25.1

