Projet

Général

Profil

0001-opendatasoft-add-limit-parameter-to-queries-55698.patch

Nicolas Roche (absent jusqu'au 3 avril), 20 juillet 2021 12:43

Télécharger (9,19 ko)

Voir les différences:

Subject: [PATCH] opendatasoft: add limit parameter to queries (#55698)

 .../migrations/0004_query_limit.py            | 22 +++++++++++++++++++
 passerelle/apps/opendatasoft/models.py        | 15 ++++++++-----
 tests/test_opendatasoft.py                    | 13 ++++++-----
 3 files changed, 39 insertions(+), 11 deletions(-)
 create mode 100644 passerelle/apps/opendatasoft/migrations/0004_query_limit.py
passerelle/apps/opendatasoft/migrations/0004_query_limit.py
1
# -*- coding: utf-8 -*-
2
# Generated by Django 1.11.29 on 2021-07-20 10:13
3
from __future__ import unicode_literals
4

  
5
from django.db import migrations, models
6

  
7

  
8
class Migration(migrations.Migration):
9

  
10
    dependencies = [
11
        ('opendatasoft', '0003_query_sort'),
12
    ]
13

  
14
    operations = [
15
        migrations.AddField(
16
            model_name='query',
17
            name='limit',
18
            field=models.PositiveIntegerField(
19
                default=10, help_text='Number of results to return in a single call', verbose_name='Limit'
20
            ),
21
        ),
22
    ]
passerelle/apps/opendatasoft/models.py
60 60
        for data_query in data_queries:
61 61
            query = Query.import_json(data_query)
62 62
            query.resource = instance
63 63
            queries.append(query)
64 64
        Query.objects.bulk_create(queries)
65 65
        return instance
66 66

  
67 67
    def call_search(
68
        self, dataset=None, text_template='', filter_expression='', sort='', id=None, q=None, limit=None
68
        self, dataset=None, text_template='', filter_expression='', sort='', limit=None, id=None, q=None
69 69
    ):
70 70
        scheme, netloc, path, params, query, fragment = urlparse.urlparse(self.service_url)
71 71
        path = urlparse.urljoin(path, 'api/records/1.0/search/')
72 72
        url = urlparse.urlunparse((scheme, netloc, path, params, query, fragment))
73 73

  
74 74
        if id is not None:
75 75
            query = 'recordid:%s' % id
76 76
        else:
......
107 107

  
108 108
    @endpoint(
109 109
        perm='can_access',
110 110
        description=_('Search'),
111 111
        parameters={
112 112
            'dataset': {'description': _('Dataset')},
113 113
            'text_template': {'description': _('Text template')},
114 114
            'sort': {'description': _('Sort field')},
115
            'limit': {'description': _('Maximum items')},
115 116
            'id': {'description': _('Record identifier')},
116 117
            'q': {'description': _('Full text query')},
117
            'limit': {'description': _('Maximum items')},
118 118
        },
119 119
    )
120
    def search(self, request, dataset=None, text_template='', sort='', id=None, q=None, limit=None, **kwargs):
121
        result = self.call_search(dataset, text_template, '', sort, id, q, limit)
120
    def search(self, request, dataset=None, text_template='', sort='', limit=None, id=None, q=None, **kwargs):
121
        result = self.call_search(dataset, text_template, '', sort, limit, id, q)
122 122
        return {'data': result}
123 123

  
124 124
    @endpoint(
125 125
        name='q',
126 126
        description=_('Query'),
127 127
        pattern=r'^(?P<query_slug>[\w:_-]+)/$',
128 128
        perm='can_access',
129 129
        show=False,
......
161 161
    sort = models.CharField(
162 162
        verbose_name=_('Sort field'),
163 163
        help_text=_(
164 164
            "Sorts results by the specified field. A minus sign - may be used to perform an ascending sort."
165 165
        ),
166 166
        max_length=256,
167 167
        blank=True,
168 168
    )
169
    limit = models.PositiveIntegerField(
170
        default=10,
171
        verbose_name='Limit',
172
        help_text=_('Number of results to return in a single call'),
173
    )
169 174

  
170 175
    delete_view = 'opendatasoft-query-delete'
171 176
    edit_view = 'opendatasoft-query-edit'
172 177

  
173 178
    def q(self, request, **kwargs):
174 179
        return self.resource.call_search(
175 180
            dataset=self.dataset,
176 181
            text_template=self.text_template,
177 182
            filter_expression='&'.join(
178 183
                [x.strip() for x in str(self.filter_expression).splitlines() if x.strip()]
179 184
            ),
180 185
            sort=self.sort,
186
            limit=self.limit,
181 187
            id=kwargs.get('id'),
182 188
            q=kwargs.get('q'),
183
            limit=kwargs.get('limit'),
184 189
        )
185 190

  
186 191
    def as_endpoint(self):
187 192
        endpoint = super(Query, self).as_endpoint(path=self.resource.q.endpoint_info.name)
188 193

  
189 194
        search_endpoint = self.resource.search.endpoint_info
190 195
        endpoint.func = search_endpoint.func
191 196
        endpoint.show_undocumented_params = False
tests/test_opendatasoft.py
138 138
def query(connector):
139 139
    return Query.objects.create(
140 140
        resource=connector,
141 141
        name='Référenciel adresses de test',
142 142
        slug='my_query',
143 143
        description='Rechercher une adresse',
144 144
        dataset='referentiel-adresse-test',
145 145
        text_template='{{numero}} {{nom_rue}} {{nom_commun}}',
146
        sort='-nom_rue',
147 146
        filter_expression='''
148 147
refine.source=Ville et Eurométropole de Strasbourg
149 148
exclude.numero=42
150 149
exclude.numero=43
151 150
''',
151
        sort='-nom_rue',
152
        limit=3,
152 153
    )
153 154

  
154 155

  
155 156
def test_views(db, admin_user, app, connector):
156 157
    app = login(app)
157 158
    resp = app.get('/opendatasoft/my_connector/', status=200)
158 159
    resp = resp.click('New Query')
159 160
    resp.form['name'] = 'my query'
......
192 193
@mock.patch('passerelle.utils.Request.get')
193 194
def test_search_using_q(mocked_get, app, connector):
194 195
    endpoint = utils.generic_endpoint_url('opendatasoft', 'search', slug=connector.slug)
195 196
    assert endpoint == '/opendatasoft/my_connector/search'
196 197
    params = {
197 198
        'dataset': 'referentiel-adresse-test',
198 199
        'text_template': '{{numero}} {{nom_rue}} {{nom_commun}}',
199 200
        'sort': '-nom_rue',
201
        'limit': '3',
200 202
        'q': "rue de l'aubepine",
201
        'limit': 3,
202 203
    }
203 204
    mocked_get.return_value = utils.FakedResponse(content=FAKED_CONTENT_Q_SEARCH, status_code=200)
204 205
    resp = app.get(endpoint, params=params, status=200)
205 206
    assert mocked_get.call_args[1]['params'] == {
206 207
        'apikey': 'my_secret',
207 208
        'dataset': 'referentiel-adresse-test',
208 209
        'sort': '-nom_rue',
209
        'q': "rue de l'aubepine",
210 210
        'rows': '3',
211
        'q': "rue de l'aubepine",
211 212
    }
212 213
    assert not resp.json['err']
213 214
    assert len(resp.json['data']) == 3
214 215
    # check order is kept
215 216
    assert [x['id'] for x in resp.json['data']] == [
216 217
        'e00cf6161e52a4c8fe510b2b74d4952036cb3473',
217 218
        '7cafcd5c692773e8b863587b2d38d6be82e023d8',
218 219
        '0984a5e1745701f71c91af73ce764e1f7132e0ff',
......
242 243
    assert resp.json['data'][0]['text'] == "19 RUE DE L'AUBEPINE Lipsheim"
243 244

  
244 245

  
245 246
@mock.patch('passerelle.utils.Request.get')
246 247
def test_query_q_using_q(mocked_get, app, query):
247 248
    endpoint = '/opendatasoft/my_connector/q/my_query/'
248 249
    params = {
249 250
        'q': "rue de l'aubepine",
250
        'limit': 3,
251 251
    }
252 252
    mocked_get.return_value = utils.FakedResponse(content=FAKED_CONTENT_Q_SEARCH, status_code=200)
253 253
    resp = app.get(endpoint, params=params, status=200)
254 254
    assert mocked_get.call_args[1]['params'] == {
255 255
        'apikey': 'my_secret',
256 256
        'dataset': 'referentiel-adresse-test',
257 257
        'refine.source': ['Ville et Eurométropole de Strasbourg'],
258 258
        'exclude.numero': ['42', '43'],
259 259
        'sort': '-nom_rue',
260
        'rows': 3,
260 261
        'q': "rue de l'aubepine",
261
        'rows': '3',
262 262
    }
263 263
    assert not resp.json['err']
264 264
    assert len(resp.json['data']) == 3
265 265
    # check order is kept
266 266
    assert [x['id'] for x in resp.json['data']] == [
267 267
        'e00cf6161e52a4c8fe510b2b74d4952036cb3473',
268 268
        '7cafcd5c692773e8b863587b2d38d6be82e023d8',
269 269
        '0984a5e1745701f71c91af73ce764e1f7132e0ff',
......
286 286
    }
287 287
    mocked_get.return_value = utils.FakedResponse(content=FAKED_CONTENT_ID_SEARCH, status_code=200)
288 288
    resp = app.get(endpoint, params=params, status=200)
289 289
    assert mocked_get.call_args[1]['params'] == {
290 290
        'dataset': 'referentiel-adresse-test',
291 291
        'q': 'recordid:7cafcd5c692773e8b863587b2d38d6be82e023d8',
292 292
        'apikey': 'my_secret',
293 293
        'refine.source': ['Ville et Eurométropole de Strasbourg'],
294
        'exclude.numero': ['42', '43'],
295 294
        'sort': '-nom_rue',
295
        'rows': 3,
296
        'exclude.numero': ['42', '43'],
296 297
    }
297 298
    assert len(resp.json['data']) == 1
298 299
    assert resp.json['data'][0]['text'] == "19 RUE DE L'AUBEPINE Lipsheim"
299 300

  
300 301

  
301 302
def test_opendatasoft_query_unicity(admin_user, app, connector, query):
302 303
    connector2 = OpenDataSoft.objects.create(
303 304
        slug='my_connector2',
304
-