From 9f974f15e4b3452a595ed04a15583bde8fa8b7d8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20P=C3=A9ters?= <fpeters@entrouvert.com>
Date: Tue, 19 Oct 2021 15:18:42 +0200
Subject: [PATCH] misc: extend |filter_by to work with fields with non-unique
 varname (#56614)

---
 tests/test_formdata.py | 55 ++++++++++++++++++++++++++++++++++++++----
 wcs/variables.py       | 46 +++++++++++++++++++++++------------
 2 files changed, 81 insertions(+), 20 deletions(-)

diff --git a/tests/test_formdata.py b/tests/test_formdata.py
index d66b4d872..2c394e6f2 100644
--- a/tests/test_formdata.py
+++ b/tests/test_formdata.py
@@ -1277,17 +1277,17 @@ def test_lazy_formdata_queryset_filter(pub, variable_test_data):
         queryset = lazy_formdata.objects.filter_by('datefield').apply_exclude_value(
             datetime.date(2018, 7, 31).timetuple()
         )
-        assert queryset.count == 1
+        assert queryset.count == 6  # 1 + 5 null
         queryset = lazy_formdata.objects.filter_by('datefield').apply_exclude_value(
             datetime.date(2018, 7, 31)
         )
-        assert queryset.count == 1
+        assert queryset.count == 6
         queryset = lazy_formdata.objects.filter_by('datefield').apply_exclude_value(
             datetime.datetime(2018, 7, 31)
         )
-        assert queryset.count == 1
+        assert queryset.count == 6
         queryset = lazy_formdata.objects.filter_by('datefield').apply_exclude_value('2018-07-31')
-        assert queryset.count == 1
+        assert queryset.count == 6
         queryset = lazy_formdata.objects.filter_by('datefield').apply_exclude_value('still not a date')
         assert queryset.count == 0
         assert pub.loggederror_class.count() == 3
@@ -1394,7 +1394,7 @@ def test_lazy_formdata_queryset_filter(pub, variable_test_data):
         tmpl = Template('{{form_objects|filter_by:"foo_foo"|exclude_value:form_var_foo_foo|count}}')
         assert tmpl.render(context) == '4'
         tmpl = Template('{{form_objects|filter_by:"datefield"|exclude_value:form_var_datefield|count}}')
-        assert tmpl.render(context) == '1'
+        assert tmpl.render(context) == '6'
         tmpl = Template('{{form_objects|filter_by:"boolfield"|exclude_value:form_var_boolfield|count}}')
         assert tmpl.render(context) == '6'
         tmpl = Template('{{form_objects|filter_by:"term1"|exclude_value:form_var_term1|count}}')
@@ -1547,6 +1547,51 @@ def test_lazy_formdata_queryset_filter(pub, variable_test_data):
     assert 'not a date' not in LazyFormData(formdata).objects.getlist('datefield')
 
 
+def test_lazy_formdata_queryset_filter_non_unique_varname(pub, variable_test_data):
+    lazy_formdata = variable_test_data
+    formdef = lazy_formdata._formdef
+    # modify fields to have foo_foo as varname for both fields[0] and fields[7]
+    assert formdef.fields[7].label == 'string2'
+    formdef.fields[7].varname = 'foo_foo'
+    formdef.store()
+
+    data_class = lazy_formdata._formdef.data_class()
+    for i in range(6):
+        formdata = data_class()
+        formdata.data = {'0': 'bar', '6': 'baz'}
+        if i == 5:
+            formdata.data['3'] = datetime.date(2018, 8, 31).timetuple()
+        formdata.just_created()
+        formdata.store()
+    formdatas = []
+    for _ in range(4):
+        formdata = data_class()
+        formdata.data = {
+            '0': 'foo',
+        }
+        formdata.just_created()
+        formdata.jump_status('finished')
+        formdata.store()
+        formdatas.append(formdata)
+
+    context = pub.substitutions.get_context_variables(mode='lazy')
+    tmpl = Template('{{form_objects|filter_by:"foo_foo"|filter_value:"bar"|count}}')
+    assert tmpl.render(context) == '7'
+    tmpl = Template('{{form_objects|filter_by:"foo_foo"|filter_value:"other"|count}}')
+    assert tmpl.render(context) == '1'
+    tmpl = Template('{{form_objects|filter_by:"foo_foo"|filter_value:"baz"|count}}')
+    assert tmpl.render(context) == '6'
+    tmpl = Template('{{form_objects|filter_by:"foo_foo"|exclude_value:"bar"|count}}')
+    assert tmpl.render(context) == '4'  # 11 - 7
+
+    for formdata in formdatas[:-1]:
+        formdata.data['6'] = 'bar'
+        formdata.store()
+
+    tmpl = Template('{{form_objects|filter_by:"foo_foo"|exclude_value:"bar"|count}}')
+    assert tmpl.render(context) == '1'
+
+
 def test_lazy_formdata_queryset_get_from_first(pub, variable_test_data):
     context = pub.substitutions.get_context_variables(mode='lazy')
     del context['form']  # remove actual form from context
diff --git a/wcs/variables.py b/wcs/variables.py
index 9eaae0873..b732df957 100644
--- a/wcs/variables.py
+++ b/wcs/variables.py
@@ -26,7 +26,7 @@ from .formdata import get_workflow_roles_substitution_variables
 from .formdef import FormDef
 from .qommon import _, force_str, misc
 from .qommon.evalutils import make_datetime
-from .qommon.storage import Equal, Intersects, Not, NotEqual, Null, Or
+from .qommon.storage import And, Equal, Intersects, Not, NotEqual, Null, Or
 from .qommon.substitution import CompatibilityNamesDict
 from .qommon.templatetags.qommon import parse_datetime
 
@@ -175,24 +175,30 @@ class LazyFormDefObjectsManager:
     def filter_by_number(self, value):
         return self._clone(self._criterias + [Equal('id_display', str(value))])
 
-    def get_field(self, key):
+    def get_fields(self, key):
         for field in self._formdef.get_all_fields():
             if getattr(field, 'varname', None) == key:
-                return field
+                yield field
+
+    def get_field(self, key):
+        for field in self.get_fields(key):
+            return field
 
     def apply_filter_value(self, value, exclude=False):
         assert self.pending_attr
 
-        field = self.get_field(self.pending_attr)
-        if field is None:
+        fields = list(self.get_fields(self.pending_attr))
+        if not fields:
             get_publisher().record_error(
                 _('Invalid filter "%s"') % self.pending_attr, formdata=self._formdata
             )
             return self.none()
 
-        if field.convert_value_from_anything:
+        if fields[0].convert_value_from_anything:
+            # consider all fields with same varname are of the same type
+            # (it should definitely be)
             try:
-                value = field.convert_value_from_anything(value)
+                value = fields[0].convert_value_from_anything(value)
             except (ValueError, AttributeError):
                 get_publisher().record_error(
                     _('Invalid value "%s" for filter "%s"') % (value, self.pending_attr),
@@ -202,16 +208,26 @@ class LazyFormDefObjectsManager:
 
         from wcs import sql
 
-        field_id = sql.get_field_id(field)
-        if isinstance(value, list):
-            criterias = Intersects(field_id, value, field=field)
-        else:
-            criterias = Equal(field_id, value, field=field)
+        criterias = []
+        for field in fields:
+            field_id = sql.get_field_id(field)
+            if isinstance(value, list):
+                criteria = Intersects(field_id, value, field=field)
+                if exclude:
+                    criteria = Not(criteria)
+            elif exclude:
+                criteria = Or([NotEqual(field_id, value, field=field), Null(field_id)])
+            else:
+                criteria = Equal(field_id, value, field=field)
+            criterias.append(criteria)
 
-        if exclude:
-            criterias = Not(criterias)
+        if len(criterias) > 1:
+            if exclude:
+                criterias = [And(criterias)]
+            else:
+                criterias = [Or(criterias)]
 
-        return self._clone(self._criterias + [criterias])
+        return self._clone(self._criterias + criterias)
 
     def apply_exclude_value(self, value):
         return self.apply_filter_value(value, exclude=True)
-- 
2.33.0

