From 77b8f83cc8c3bc32063a452bd76e4d61eee2d504 Mon Sep 17 00:00:00 2001
From: Nicolas ROCHE <nroche@entrouvert.com>
Date: Fri, 27 May 2022 12:09:37 +0200
Subject: [PATCH] opendatasoft: remove operators form query parameter (#43235)

---
 passerelle/apps/opendatasoft/models.py |  6 +++++-
 tests/test_opendatasoft.py             | 16 ++++++++++++++--
 2 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/passerelle/apps/opendatasoft/models.py b/passerelle/apps/opendatasoft/models.py
index 508b4bb5..f59ee364 100644
--- a/passerelle/apps/opendatasoft/models.py
+++ b/passerelle/apps/opendatasoft/models.py
@@ -9,16 +9,17 @@
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU Affero General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import re
 from urllib import parse as urlparse
 
 from django.db import models
 from django.shortcuts import get_object_or_404
 from django.urls import reverse
 from django.utils.translation import ugettext_lazy as _
 from requests import RequestException
 
@@ -72,17 +73,20 @@ class OpenDataSoft(BaseResource):
         scheme, netloc, path, params, query, fragment = urlparse.urlparse(self.service_url)
         path = urlparse.urljoin(path, 'api/records/1.0/search/')
         url = urlparse.urlunparse((scheme, netloc, path, params, query, fragment))
 
         params = {'dataset': dataset}
         if id is not None:
             params['q'] = 'recordid:%s' % id
         elif q is not None:
-            params['q'] = q
+            # remove query language operators
+            terms = re.split('[^\w]', q)
+            terms = [term for term in terms if len(term) > 1 and term.lower() not in ['and', 'or', 'not']]
+            params['q'] = ' '.join(terms)
         elif sort:
             params['sort'] = sort
         if self.api_key:
             params['apikey'] = self.api_key
         if limit:
             params['rows'] = limit
         params.update(urlparse.parse_qs(filter_expression))
 
diff --git a/tests/test_opendatasoft.py b/tests/test_opendatasoft.py
index c304e033..e897143f 100644
--- a/tests/test_opendatasoft.py
+++ b/tests/test_opendatasoft.py
@@ -239,17 +239,17 @@ def test_search_using_q(mocked_get, app, connector):
         'q': "rue de l'aubepine",
     }
     mocked_get.return_value = tests.utils.FakedResponse(content=FAKED_CONTENT_Q_SEARCH, status_code=200)
     resp = app.get(endpoint, params=params, status=200)
     assert mocked_get.call_args[1]['params'] == {
         'apikey': 'my_secret',
         'dataset': 'referentiel-adresse-test',
         'rows': '3',
-        'q': "rue de l'aubepine",
+        'q': "rue de aubepine",
     }
     assert not resp.json['err']
     assert len(resp.json['data']) == 3
     # check order is kept
     assert [x['id'] for x in resp.json['data']] == [
         'e00cf6161e52a4c8fe510b2b74d4952036cb3473',
         '7cafcd5c692773e8b863587b2d38d6be82e023d8',
         '0984a5e1745701f71c91af73ce764e1f7132e0ff',
@@ -258,16 +258,28 @@ def test_search_using_q(mocked_get, app, connector):
     assert [x['text'] for x in resp.json['data']] == [
         "33 RUE DE L'AUBEPINE Strasbourg",
         "19 RUE DE L'AUBEPINE Lipsheim",
         "29 RUE DE L'AUBEPINE Strasbourg",
     ]
     # check additional attributes
     assert [x['numero'] for x in resp.json['data']] == ['33', '19', '29']
 
+    # check operators are removed
+    params['q'] = 'please, do NOT send boolean operators like And, OR and nOt'
+    resp = app.get(endpoint, params=params, status=200)
+    assert mocked_get.call_args[1]['params']['q'] == 'please do send boolean operators like'
+
+    params['q'] = 'field operators are almost ignored too:, -, ==, >, <, >=, <=, [start_date TO end_date]'
+    resp = app.get(endpoint, params=params, status=200)
+    assert (
+        mocked_get.call_args[1]['params']['q']
+        == 'field operators are almost ignored too start_date TO end_date'
+    )
+
 
 @mock.patch('passerelle.utils.Request.get')
 def test_search_using_id(mocked_get, app, connector):
     endpoint = tests.utils.generic_endpoint_url('opendatasoft', 'search', slug=connector.slug)
     assert endpoint == '/opendatasoft/my_connector/search'
     params = {
         'dataset': 'referentiel-adresse-test',
         'text_template': '{{numero}} {{nom_rue}} {{nom_commun}}',
@@ -327,17 +339,17 @@ def test_query_q_using_q(mocked_get, app, query):
     mocked_get.return_value = tests.utils.FakedResponse(content=FAKED_CONTENT_Q_SEARCH, status_code=200)
     resp = app.get(endpoint, params=params, status=200)
     assert mocked_get.call_args[1]['params'] == {
         'apikey': 'my_secret',
         'dataset': 'referentiel-adresse-test',
         'refine.source': ['Ville et Eurométropole de Strasbourg'],
         'exclude.numero': ['42', '43'],
         'rows': 3,
-        'q': "rue de l'aubepine",
+        'q': "rue de aubepine",
     }
     assert not resp.json['err']
     assert len(resp.json['data']) == 3
     # check order is kept
     assert [x['id'] for x in resp.json['data']] == [
         'e00cf6161e52a4c8fe510b2b74d4952036cb3473',
         '7cafcd5c692773e8b863587b2d38d6be82e023d8',
         '0984a5e1745701f71c91af73ce764e1f7132e0ff',
-- 
2.30.2

