0001-opendatasoft-add-limit-parameter-to-queries-55698.patch
| passerelle/apps/opendatasoft/migrations/0004_query_limit.py | ||
|---|---|---|
| 1 |
# -*- coding: utf-8 -*- |
|
| 2 |
# Generated by Django 1.11.29 on 2021-07-20 10:13 |
|
| 3 |
from __future__ import unicode_literals |
|
| 4 | ||
| 5 |
from django.db import migrations, models |
|
| 6 | ||
| 7 | ||
| 8 |
class Migration(migrations.Migration): |
|
| 9 | ||
| 10 |
dependencies = [ |
|
| 11 |
('opendatasoft', '0003_query_sort'),
|
|
| 12 |
] |
|
| 13 | ||
| 14 |
operations = [ |
|
| 15 |
migrations.AddField( |
|
| 16 |
model_name='query', |
|
| 17 |
name='limit', |
|
| 18 |
field=models.PositiveIntegerField( |
|
| 19 |
default=10, help_text='Number of results to return in a single call', verbose_name='Limit' |
|
| 20 |
), |
|
| 21 |
), |
|
| 22 |
] |
|
| passerelle/apps/opendatasoft/models.py | ||
|---|---|---|
| 60 | 60 |
for data_query in data_queries: |
| 61 | 61 |
query = Query.import_json(data_query) |
| 62 | 62 |
query.resource = instance |
| 63 | 63 |
queries.append(query) |
| 64 | 64 |
Query.objects.bulk_create(queries) |
| 65 | 65 |
return instance |
| 66 | 66 | |
| 67 | 67 |
def call_search( |
| 68 |
self, dataset=None, text_template='', filter_expression='', sort=None, id=None, q=None, limit=None
|
|
| 68 |
self, dataset=None, text_template='', filter_expression='', sort=None, limit=None, id=None, q=None
|
|
| 69 | 69 |
): |
| 70 | 70 |
scheme, netloc, path, params, query, fragment = urlparse.urlparse(self.service_url) |
| 71 | 71 |
path = urlparse.urljoin(path, 'api/records/1.0/search/') |
| 72 | 72 |
url = urlparse.urlunparse((scheme, netloc, path, params, query, fragment)) |
| 73 | 73 | |
| 74 | 74 |
params = {'dataset': dataset}
|
| 75 | 75 |
if id is not None: |
| 76 | 76 |
params['q'] = 'recordid:%s' % id |
| ... | ... | |
| 104 | 104 | |
| 105 | 105 |
@endpoint( |
| 106 | 106 |
perm='can_access', |
| 107 | 107 |
description=_('Search'),
|
| 108 | 108 |
parameters={
|
| 109 | 109 |
'dataset': {'description': _('Dataset')},
|
| 110 | 110 |
'text_template': {'description': _('Text template')},
|
| 111 | 111 |
'sort': {'description': _('Sort field')},
|
| 112 |
'limit': {'description': _('Maximum items')},
|
|
| 112 | 113 |
'id': {'description': _('Record identifier')},
|
| 113 | 114 |
'q': {'description': _('Full text query')},
|
| 114 |
'limit': {'description': _('Maximum items')},
|
|
| 115 | 115 |
}, |
| 116 | 116 |
) |
| 117 | 117 |
def search( |
| 118 |
self, request, dataset=None, text_template='', sort=None, id=None, q=None, limit=None, **kwargs
|
|
| 118 |
self, request, dataset=None, text_template='', sort=None, limit=None, id=None, q=None, **kwargs
|
|
| 119 | 119 |
): |
| 120 |
result = self.call_search(dataset, text_template, '', sort, id, q, limit)
|
|
| 120 |
result = self.call_search(dataset, text_template, '', sort, limit, id, q)
|
|
| 121 | 121 |
return {'data': result}
|
| 122 | 122 | |
| 123 | 123 |
@endpoint( |
| 124 | 124 |
name='q', |
| 125 | 125 |
description=_('Query'),
|
| 126 | 126 |
pattern=r'^(?P<query_slug>[\w:_-]+)/$', |
| 127 | 127 |
perm='can_access', |
| 128 | 128 |
show=False, |
| ... | ... | |
| 160 | 160 |
sort = models.CharField( |
| 161 | 161 |
verbose_name=_('Sort field'),
|
| 162 | 162 |
help_text=_( |
| 163 | 163 |
"Sorts results by the specified field. A minus sign - may be used to perform an ascending sort." |
| 164 | 164 |
), |
| 165 | 165 |
max_length=256, |
| 166 | 166 |
blank=True, |
| 167 | 167 |
) |
| 168 |
limit = models.PositiveIntegerField( |
|
| 169 |
default=10, |
|
| 170 |
verbose_name='Limit', |
|
| 171 |
help_text=_('Number of results to return in a single call'),
|
|
| 172 |
) |
|
| 168 | 173 | |
| 169 | 174 |
delete_view = 'opendatasoft-query-delete' |
| 170 | 175 |
edit_view = 'opendatasoft-query-edit' |
| 171 | 176 | |
| 172 | 177 |
def q(self, request, **kwargs): |
| 173 | 178 |
return self.resource.call_search( |
| 174 | 179 |
dataset=self.dataset, |
| 175 | 180 |
text_template=self.text_template, |
| 176 | 181 |
filter_expression='&'.join( |
| 177 | 182 |
[x.strip() for x in str(self.filter_expression).splitlines() if x.strip()] |
| 178 | 183 |
), |
| 179 | 184 |
sort=self.sort, |
| 185 |
limit=self.limit, |
|
| 180 | 186 |
id=kwargs.get('id'),
|
| 181 | 187 |
q=kwargs.get('q'),
|
| 182 |
limit=kwargs.get('limit'),
|
|
| 183 | 188 |
) |
| 184 | 189 | |
| 185 | 190 |
def as_endpoint(self): |
| 186 | 191 |
endpoint = super(Query, self).as_endpoint(path=self.resource.q.endpoint_info.name) |
| 187 | 192 | |
| 188 | 193 |
search_endpoint = self.resource.search.endpoint_info |
| 189 | 194 |
endpoint.func = search_endpoint.func |
| 190 | 195 |
endpoint.show_undocumented_params = False |
| tests/test_opendatasoft.py | ||
|---|---|---|
| 138 | 138 |
def query(connector): |
| 139 | 139 |
return Query.objects.create( |
| 140 | 140 |
resource=connector, |
| 141 | 141 |
name='Référenciel adresses de test', |
| 142 | 142 |
slug='my_query', |
| 143 | 143 |
description='Rechercher une adresse', |
| 144 | 144 |
dataset='referentiel-adresse-test', |
| 145 | 145 |
text_template='{{numero}} {{nom_rue}} {{nom_commun}}',
|
| 146 |
sort='-nom_rue', |
|
| 147 | 146 |
filter_expression=''' |
| 148 | 147 |
refine.source=Ville et Eurométropole de Strasbourg |
| 149 | 148 |
exclude.numero=42 |
| 150 | 149 |
exclude.numero=43 |
| 151 | 150 |
''', |
| 151 |
sort='-nom_rue', |
|
| 152 |
limit=3, |
|
| 152 | 153 |
) |
| 153 | 154 | |
| 154 | 155 | |
| 155 | 156 |
def test_views(db, admin_user, app, connector): |
| 156 | 157 |
app = login(app) |
| 157 | 158 |
resp = app.get('/opendatasoft/my_connector/', status=200)
|
| 158 | 159 |
resp = resp.click('New Query')
|
| 159 | 160 |
resp.form['name'] = 'my query' |
| ... | ... | |
| 228 | 229 |
@mock.patch('passerelle.utils.Request.get')
|
| 229 | 230 |
def test_search_using_q(mocked_get, app, connector): |
| 230 | 231 |
endpoint = utils.generic_endpoint_url('opendatasoft', 'search', slug=connector.slug)
|
| 231 | 232 |
assert endpoint == '/opendatasoft/my_connector/search' |
| 232 | 233 |
params = {
|
| 233 | 234 |
'dataset': 'referentiel-adresse-test', |
| 234 | 235 |
'text_template': '{{numero}} {{nom_rue}} {{nom_commun}}',
|
| 235 | 236 |
'sort': '-nom_rue', |
| 237 |
'limit': '3', |
|
| 236 | 238 |
'q': "rue de l'aubepine", |
| 237 |
'limit': 3, |
|
| 238 | 239 |
} |
| 239 | 240 |
mocked_get.return_value = utils.FakedResponse(content=FAKED_CONTENT_Q_SEARCH, status_code=200) |
| 240 | 241 |
resp = app.get(endpoint, params=params, status=200) |
| 241 | 242 |
assert mocked_get.call_args[1]['params'] == {
|
| 242 | 243 |
'apikey': 'my_secret', |
| 243 | 244 |
'dataset': 'referentiel-adresse-test', |
| 244 |
'q': "rue de l'aubepine", |
|
| 245 | 245 |
'rows': '3', |
| 246 |
'q': "rue de l'aubepine", |
|
| 246 | 247 |
} |
| 247 | 248 |
assert not resp.json['err'] |
| 248 | 249 |
assert len(resp.json['data']) == 3 |
| 249 | 250 |
# check order is kept |
| 250 | 251 |
assert [x['id'] for x in resp.json['data']] == [ |
| 251 | 252 |
'e00cf6161e52a4c8fe510b2b74d4952036cb3473', |
| 252 | 253 |
'7cafcd5c692773e8b863587b2d38d6be82e023d8', |
| 253 | 254 |
'0984a5e1745701f71c91af73ce764e1f7132e0ff', |
| ... | ... | |
| 291 | 292 |
mocked_get.return_value = utils.FakedResponse(content=FAKED_CONTENT_Q_SEARCH, status_code=200) |
| 292 | 293 |
resp = app.get(endpoint, params=params, status=200) |
| 293 | 294 |
assert mocked_get.call_args[1]['params'] == {
|
| 294 | 295 |
'apikey': 'my_secret', |
| 295 | 296 |
'dataset': 'referentiel-adresse-test', |
| 296 | 297 |
'refine.source': ['Ville et Eurométropole de Strasbourg'], |
| 297 | 298 |
'exclude.numero': ['42', '43'], |
| 298 | 299 |
'sort': '-nom_rue', |
| 299 |
'rows': '3',
|
|
| 300 |
'rows': 3,
|
|
| 300 | 301 |
} |
| 301 | 302 |
assert not resp.json['err'] |
| 302 | 303 |
assert len(resp.json['data']) == 3 |
| 303 | 304 |
# check order is kept |
| 304 | 305 |
assert [x['id'] for x in resp.json['data']] == [ |
| 305 | 306 |
'e00cf6161e52a4c8fe510b2b74d4952036cb3473', |
| 306 | 307 |
'7cafcd5c692773e8b863587b2d38d6be82e023d8', |
| 307 | 308 |
'0984a5e1745701f71c91af73ce764e1f7132e0ff', |
| ... | ... | |
| 316 | 317 |
assert [x['numero'] for x in resp.json['data']] == ['33', '19', '29'] |
| 317 | 318 | |
| 318 | 319 | |
| 319 | 320 |
@mock.patch('passerelle.utils.Request.get')
|
| 320 | 321 |
def test_query_q_using_q(mocked_get, app, query): |
| 321 | 322 |
endpoint = '/opendatasoft/my_connector/q/my_query/' |
| 322 | 323 |
params = {
|
| 323 | 324 |
'q': "rue de l'aubepine", |
| 324 |
'limit': 3, |
|
| 325 | 325 |
} |
| 326 | 326 |
mocked_get.return_value = utils.FakedResponse(content=FAKED_CONTENT_Q_SEARCH, status_code=200) |
| 327 | 327 |
resp = app.get(endpoint, params=params, status=200) |
| 328 | 328 |
assert mocked_get.call_args[1]['params'] == {
|
| 329 | 329 |
'apikey': 'my_secret', |
| 330 | 330 |
'dataset': 'referentiel-adresse-test', |
| 331 | 331 |
'refine.source': ['Ville et Eurométropole de Strasbourg'], |
| 332 | 332 |
'exclude.numero': ['42', '43'], |
| 333 |
'rows': 3, |
|
| 333 | 334 |
'q': "rue de l'aubepine", |
| 334 |
'rows': '3', |
|
| 335 | 335 |
} |
| 336 | 336 |
assert not resp.json['err'] |
| 337 | 337 |
assert len(resp.json['data']) == 3 |
| 338 | 338 |
# check order is kept |
| 339 | 339 |
assert [x['id'] for x in resp.json['data']] == [ |
| 340 | 340 |
'e00cf6161e52a4c8fe510b2b74d4952036cb3473', |
| 341 | 341 |
'7cafcd5c692773e8b863587b2d38d6be82e023d8', |
| 342 | 342 |
'0984a5e1745701f71c91af73ce764e1f7132e0ff', |
| ... | ... | |
| 359 | 359 |
} |
| 360 | 360 |
mocked_get.return_value = utils.FakedResponse(content=FAKED_CONTENT_ID_SEARCH, status_code=200) |
| 361 | 361 |
resp = app.get(endpoint, params=params, status=200) |
| 362 | 362 |
assert mocked_get.call_args[1]['params'] == {
|
| 363 | 363 |
'apikey': 'my_secret', |
| 364 | 364 |
'dataset': 'referentiel-adresse-test', |
| 365 | 365 |
'refine.source': ['Ville et Eurométropole de Strasbourg'], |
| 366 | 366 |
'exclude.numero': ['42', '43'], |
| 367 |
'rows': 3, |
|
| 367 | 368 |
'q': 'recordid:7cafcd5c692773e8b863587b2d38d6be82e023d8', |
| 368 | 369 |
} |
| 369 | 370 |
assert len(resp.json['data']) == 1 |
| 370 | 371 |
assert resp.json['data'][0]['text'] == "19 RUE DE L'AUBEPINE Lipsheim" |
| 371 | 372 | |
| 372 | 373 | |
| 373 | 374 |
def test_opendatasoft_query_unicity(admin_user, app, connector, query): |
| 374 | 375 |
connector2 = OpenDataSoft.objects.create( |
| 375 |
- |
|