From e3fe9947ecdfb055a742e9ee0ebc6df567f315d1 Mon Sep 17 00:00:00 2001
From: Nicolas ROCHE <nroche@entrouvert.com>
Date: Tue, 20 Jul 2021 12:25:31 +0200
Subject: [PATCH] opendatasoft: add limit parameter to queries (#55698)

---
 .../migrations/0004_query_limit.py            | 22 +++++++++++++++++++
 passerelle/apps/opendatasoft/models.py        | 15 ++++++++-----
 tests/test_opendatasoft.py                    | 13 ++++++-----
 3 files changed, 39 insertions(+), 11 deletions(-)
 create mode 100644 passerelle/apps/opendatasoft/migrations/0004_query_limit.py

diff --git a/passerelle/apps/opendatasoft/migrations/0004_query_limit.py b/passerelle/apps/opendatasoft/migrations/0004_query_limit.py
new file mode 100644
index 00000000..28ba5298
--- /dev/null
+++ b/passerelle/apps/opendatasoft/migrations/0004_query_limit.py
@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.29 on 2021-07-20 10:13
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('opendatasoft', '0003_query_sort'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='query',
+            name='limit',
+            field=models.PositiveIntegerField(
+                default=10, help_text='Number of results to return in a single call', verbose_name='Limit'
+            ),
+        ),
+    ]
diff --git a/passerelle/apps/opendatasoft/models.py b/passerelle/apps/opendatasoft/models.py
index 82e9db3f..8f98b111 100644
--- a/passerelle/apps/opendatasoft/models.py
+++ b/passerelle/apps/opendatasoft/models.py
@@ -60,17 +60,17 @@ class OpenDataSoft(BaseResource):
         for data_query in data_queries:
             query = Query.import_json(data_query)
             query.resource = instance
             queries.append(query)
         Query.objects.bulk_create(queries)
         return instance
 
     def call_search(
-        self, dataset=None, text_template='', filter_expression='', sort=None, id=None, q=None, limit=None
+        self, dataset=None, text_template='', filter_expression='', sort=None, limit=None, id=None, q=None
     ):
         scheme, netloc, path, params, query, fragment = urlparse.urlparse(self.service_url)
         path = urlparse.urljoin(path, 'api/records/1.0/search/')
         url = urlparse.urlunparse((scheme, netloc, path, params, query, fragment))
 
         params = {'dataset': dataset}
         if id is not None:
             params['q'] = 'recordid:%s' % id
@@ -104,25 +104,25 @@ class OpenDataSoft(BaseResource):
 
     @endpoint(
         perm='can_access',
         description=_('Search'),
         parameters={
             'dataset': {'description': _('Dataset')},
             'text_template': {'description': _('Text template')},
             'sort': {'description': _('Sort field')},
+            'limit': {'description': _('Maximum items')},
             'id': {'description': _('Record identifier')},
             'q': {'description': _('Full text query')},
-            'limit': {'description': _('Maximum items')},
         },
     )
     def search(
-        self, request, dataset=None, text_template='', sort=None, id=None, q=None, limit=None, **kwargs
+        self, request, dataset=None, text_template='', sort=None, limit=None, id=None, q=None, **kwargs
     ):
-        result = self.call_search(dataset, text_template, '', sort, id, q, limit)
+        result = self.call_search(dataset, text_template, '', sort, limit, id, q)
         return {'data': result}
 
     @endpoint(
         name='q',
         description=_('Query'),
         pattern=r'^(?P<query_slug>[\w:_-]+)/$',
         perm='can_access',
         show=False,
@@ -160,31 +160,36 @@ class Query(BaseQuery):
     sort = models.CharField(
         verbose_name=_('Sort field'),
         help_text=_(
             "Sorts results by the specified field. A minus sign - may be used to perform an ascending sort."
         ),
         max_length=256,
         blank=True,
     )
+    limit = models.PositiveIntegerField(
+        default=10,
+        verbose_name='Limit',
+        help_text=_('Number of results to return in a single call'),
+    )
 
     delete_view = 'opendatasoft-query-delete'
     edit_view = 'opendatasoft-query-edit'
 
     def q(self, request, **kwargs):
         return self.resource.call_search(
             dataset=self.dataset,
             text_template=self.text_template,
             filter_expression='&'.join(
                 [x.strip() for x in str(self.filter_expression).splitlines() if x.strip()]
             ),
             sort=self.sort,
+            limit=self.limit,
             id=kwargs.get('id'),
             q=kwargs.get('q'),
-            limit=kwargs.get('limit'),
         )
 
     def as_endpoint(self):
         endpoint = super(Query, self).as_endpoint(path=self.resource.q.endpoint_info.name)
 
         search_endpoint = self.resource.search.endpoint_info
         endpoint.func = search_endpoint.func
         endpoint.show_undocumented_params = False
diff --git a/tests/test_opendatasoft.py b/tests/test_opendatasoft.py
index 00dd0afa..570efb31 100644
--- a/tests/test_opendatasoft.py
+++ b/tests/test_opendatasoft.py
@@ -138,22 +138,23 @@ def connector():
 def query(connector):
     return Query.objects.create(
         resource=connector,
         name='Référenciel adresses de test',
         slug='my_query',
         description='Rechercher une adresse',
         dataset='referentiel-adresse-test',
         text_template='{{numero}} {{nom_rue}} {{nom_commun}}',
-        sort='-nom_rue',
         filter_expression='''
 refine.source=Ville et Eurométropole de Strasbourg
 exclude.numero=42
 exclude.numero=43
 ''',
+        sort='-nom_rue',
+        limit=3,
     )
 
 
 def test_views(db, admin_user, app, connector):
     app = login(app)
     resp = app.get('/opendatasoft/my_connector/', status=200)
     resp = resp.click('New Query')
     resp.form['name'] = 'my query'
@@ -228,26 +229,26 @@ def test_search(mocked_get, app, connector):
 @mock.patch('passerelle.utils.Request.get')
 def test_search_using_q(mocked_get, app, connector):
     endpoint = utils.generic_endpoint_url('opendatasoft', 'search', slug=connector.slug)
     assert endpoint == '/opendatasoft/my_connector/search'
     params = {
         'dataset': 'referentiel-adresse-test',
         'text_template': '{{numero}} {{nom_rue}} {{nom_commun}}',
         'sort': '-nom_rue',
+        'limit': '3',
         'q': "rue de l'aubepine",
-        'limit': 3,
     }
     mocked_get.return_value = utils.FakedResponse(content=FAKED_CONTENT_Q_SEARCH, status_code=200)
     resp = app.get(endpoint, params=params, status=200)
     assert mocked_get.call_args[1]['params'] == {
         'apikey': 'my_secret',
         'dataset': 'referentiel-adresse-test',
-        'q': "rue de l'aubepine",
         'rows': '3',
+        'q': "rue de l'aubepine",
     }
     assert not resp.json['err']
     assert len(resp.json['data']) == 3
     # check order is kept
     assert [x['id'] for x in resp.json['data']] == [
         'e00cf6161e52a4c8fe510b2b74d4952036cb3473',
         '7cafcd5c692773e8b863587b2d38d6be82e023d8',
         '0984a5e1745701f71c91af73ce764e1f7132e0ff',
@@ -291,17 +292,17 @@ def test_query_q(mocked_get, app, query):
     mocked_get.return_value = utils.FakedResponse(content=FAKED_CONTENT_Q_SEARCH, status_code=200)
     resp = app.get(endpoint, params=params, status=200)
     assert mocked_get.call_args[1]['params'] == {
         'apikey': 'my_secret',
         'dataset': 'referentiel-adresse-test',
         'refine.source': ['Ville et Eurométropole de Strasbourg'],
         'exclude.numero': ['42', '43'],
         'sort': '-nom_rue',
-        'rows': '3',
+        'rows': 3,
     }
     assert not resp.json['err']
     assert len(resp.json['data']) == 3
     # check order is kept
     assert [x['id'] for x in resp.json['data']] == [
         'e00cf6161e52a4c8fe510b2b74d4952036cb3473',
         '7cafcd5c692773e8b863587b2d38d6be82e023d8',
         '0984a5e1745701f71c91af73ce764e1f7132e0ff',
@@ -316,27 +317,26 @@ def test_query_q(mocked_get, app, query):
     assert [x['numero'] for x in resp.json['data']] == ['33', '19', '29']
 
 
 @mock.patch('passerelle.utils.Request.get')
 def test_query_q_using_q(mocked_get, app, query):
     endpoint = '/opendatasoft/my_connector/q/my_query/'
     params = {
         'q': "rue de l'aubepine",
-        'limit': 3,
     }
     mocked_get.return_value = utils.FakedResponse(content=FAKED_CONTENT_Q_SEARCH, status_code=200)
     resp = app.get(endpoint, params=params, status=200)
     assert mocked_get.call_args[1]['params'] == {
         'apikey': 'my_secret',
         'dataset': 'referentiel-adresse-test',
         'refine.source': ['Ville et Eurométropole de Strasbourg'],
         'exclude.numero': ['42', '43'],
+        'rows': 3,
         'q': "rue de l'aubepine",
-        'rows': '3',
     }
     assert not resp.json['err']
     assert len(resp.json['data']) == 3
     # check order is kept
     assert [x['id'] for x in resp.json['data']] == [
         'e00cf6161e52a4c8fe510b2b74d4952036cb3473',
         '7cafcd5c692773e8b863587b2d38d6be82e023d8',
         '0984a5e1745701f71c91af73ce764e1f7132e0ff',
@@ -359,16 +359,17 @@ def test_query_q_using_id(mocked_get, app, query):
     }
     mocked_get.return_value = utils.FakedResponse(content=FAKED_CONTENT_ID_SEARCH, status_code=200)
     resp = app.get(endpoint, params=params, status=200)
     assert mocked_get.call_args[1]['params'] == {
         'apikey': 'my_secret',
         'dataset': 'referentiel-adresse-test',
         'refine.source': ['Ville et Eurométropole de Strasbourg'],
         'exclude.numero': ['42', '43'],
+        'rows': 3,
         'q': 'recordid:7cafcd5c692773e8b863587b2d38d6be82e023d8',
     }
     assert len(resp.json['data']) == 1
     assert resp.json['data'][0]['text'] == "19 RUE DE L'AUBEPINE Lipsheim"
 
 
 def test_opendatasoft_query_unicity(admin_user, app, connector, query):
     connector2 = OpenDataSoft.objects.create(
-- 
2.30.2

