0001-opendatasoft-add-limit-parameter-to-queries-55698.patch
passerelle/apps/opendatasoft/migrations/0004_query_limit.py | ||
---|---|---|
1 |
# -*- coding: utf-8 -*- |
|
2 |
# Generated by Django 1.11.29 on 2021-07-20 10:13 |
|
3 |
from __future__ import unicode_literals |
|
4 | ||
5 |
from django.db import migrations, models |
|
6 | ||
7 | ||
8 |
class Migration(migrations.Migration): |
|
9 | ||
10 |
dependencies = [ |
|
11 |
('opendatasoft', '0003_query_sort'), |
|
12 |
] |
|
13 | ||
14 |
operations = [ |
|
15 |
migrations.AddField( |
|
16 |
model_name='query', |
|
17 |
name='limit', |
|
18 |
field=models.PositiveIntegerField( |
|
19 |
default=10, help_text='Number of results to return in a single call', verbose_name='Limit' |
|
20 |
), |
|
21 |
), |
|
22 |
] |
passerelle/apps/opendatasoft/models.py | ||
---|---|---|
60 | 60 |
for data_query in data_queries: |
61 | 61 |
query = Query.import_json(data_query) |
62 | 62 |
query.resource = instance |
63 | 63 |
queries.append(query) |
64 | 64 |
Query.objects.bulk_create(queries) |
65 | 65 |
return instance |
66 | 66 | |
67 | 67 |
def call_search( |
68 |
self, dataset=None, text_template='', filter_expression='', sort='', id=None, q=None, limit=None
|
|
68 |
self, dataset=None, text_template='', filter_expression='', sort='', limit=None, id=None, q=None
|
|
69 | 69 |
): |
70 | 70 |
scheme, netloc, path, params, query, fragment = urlparse.urlparse(self.service_url) |
71 | 71 |
path = urlparse.urljoin(path, 'api/records/1.0/search/') |
72 | 72 |
url = urlparse.urlunparse((scheme, netloc, path, params, query, fragment)) |
73 | 73 | |
74 | 74 |
if id is not None: |
75 | 75 |
query = 'recordid:%s' % id |
76 | 76 |
else: |
... | ... | |
107 | 107 | |
108 | 108 |
@endpoint( |
109 | 109 |
perm='can_access', |
110 | 110 |
description=_('Search'), |
111 | 111 |
parameters={ |
112 | 112 |
'dataset': {'description': _('Dataset')}, |
113 | 113 |
'text_template': {'description': _('Text template')}, |
114 | 114 |
'sort': {'description': _('Sort field')}, |
115 |
'limit': {'description': _('Maximum items')}, |
|
115 | 116 |
'id': {'description': _('Record identifier')}, |
116 | 117 |
'q': {'description': _('Full text query')}, |
117 |
'limit': {'description': _('Maximum items')}, |
|
118 | 118 |
}, |
119 | 119 |
) |
120 |
def search(self, request, dataset=None, text_template='', sort='', id=None, q=None, limit=None, **kwargs):
|
|
121 |
result = self.call_search(dataset, text_template, '', sort, id, q, limit)
|
|
120 |
def search(self, request, dataset=None, text_template='', sort='', limit=None, id=None, q=None, **kwargs):
|
|
121 |
result = self.call_search(dataset, text_template, '', sort, limit, id, q)
|
|
122 | 122 |
return {'data': result} |
123 | 123 | |
124 | 124 |
@endpoint( |
125 | 125 |
name='q', |
126 | 126 |
description=_('Query'), |
127 | 127 |
pattern=r'^(?P<query_slug>[\w:_-]+)/$', |
128 | 128 |
perm='can_access', |
129 | 129 |
show=False, |
... | ... | |
161 | 161 |
sort = models.CharField( |
162 | 162 |
verbose_name=_('Sort field'), |
163 | 163 |
help_text=_( |
164 | 164 |
"Sorts results by the specified field. A minus sign - may be used to perform an ascending sort." |
165 | 165 |
), |
166 | 166 |
max_length=256, |
167 | 167 |
blank=True, |
168 | 168 |
) |
169 |
limit = models.PositiveIntegerField( |
|
170 |
default=10, |
|
171 |
verbose_name='Limit', |
|
172 |
help_text=_('Number of results to return in a single call'), |
|
173 |
) |
|
169 | 174 | |
170 | 175 |
delete_view = 'opendatasoft-query-delete' |
171 | 176 |
edit_view = 'opendatasoft-query-edit' |
172 | 177 | |
173 | 178 |
def q(self, request, **kwargs): |
174 | 179 |
return self.resource.call_search( |
175 | 180 |
dataset=self.dataset, |
176 | 181 |
text_template=self.text_template, |
177 | 182 |
filter_expression='&'.join( |
178 | 183 |
[x.strip() for x in str(self.filter_expression).splitlines() if x.strip()] |
179 | 184 |
), |
180 | 185 |
sort=self.sort, |
186 |
limit=self.limit, |
|
181 | 187 |
id=kwargs.get('id'), |
182 | 188 |
q=kwargs.get('q'), |
183 |
limit=kwargs.get('limit'), |
|
184 | 189 |
) |
185 | 190 | |
186 | 191 |
def as_endpoint(self): |
187 | 192 |
endpoint = super(Query, self).as_endpoint(path=self.resource.q.endpoint_info.name) |
188 | 193 | |
189 | 194 |
search_endpoint = self.resource.search.endpoint_info |
190 | 195 |
endpoint.func = search_endpoint.func |
191 | 196 |
endpoint.show_undocumented_params = False |
tests/test_opendatasoft.py | ||
---|---|---|
138 | 138 |
def query(connector): |
139 | 139 |
return Query.objects.create( |
140 | 140 |
resource=connector, |
141 | 141 |
name='Référenciel adresses de test', |
142 | 142 |
slug='my_query', |
143 | 143 |
description='Rechercher une adresse', |
144 | 144 |
dataset='referentiel-adresse-test', |
145 | 145 |
text_template='{{numero}} {{nom_rue}} {{nom_commun}}', |
146 |
sort='-nom_rue', |
|
147 | 146 |
filter_expression=''' |
148 | 147 |
refine.source=Ville et Eurométropole de Strasbourg |
149 | 148 |
exclude.numero=42 |
150 | 149 |
exclude.numero=43 |
151 | 150 |
''', |
151 |
sort='-nom_rue', |
|
152 |
limit=3, |
|
152 | 153 |
) |
153 | 154 | |
154 | 155 | |
155 | 156 |
def test_views(db, admin_user, app, connector): |
156 | 157 |
app = login(app) |
157 | 158 |
resp = app.get('/opendatasoft/my_connector/', status=200) |
158 | 159 |
resp = resp.click('New Query') |
159 | 160 |
resp.form['name'] = 'my query' |
... | ... | |
192 | 193 |
@mock.patch('passerelle.utils.Request.get') |
193 | 194 |
def test_search_using_q(mocked_get, app, connector): |
194 | 195 |
endpoint = utils.generic_endpoint_url('opendatasoft', 'search', slug=connector.slug) |
195 | 196 |
assert endpoint == '/opendatasoft/my_connector/search' |
196 | 197 |
params = { |
197 | 198 |
'dataset': 'referentiel-adresse-test', |
198 | 199 |
'text_template': '{{numero}} {{nom_rue}} {{nom_commun}}', |
199 | 200 |
'sort': '-nom_rue', |
201 |
'limit': '3', |
|
200 | 202 |
'q': "rue de l'aubepine", |
201 |
'limit': 3, |
|
202 | 203 |
} |
203 | 204 |
mocked_get.return_value = utils.FakedResponse(content=FAKED_CONTENT_Q_SEARCH, status_code=200) |
204 | 205 |
resp = app.get(endpoint, params=params, status=200) |
205 | 206 |
assert mocked_get.call_args[1]['params'] == { |
206 | 207 |
'apikey': 'my_secret', |
207 | 208 |
'dataset': 'referentiel-adresse-test', |
208 | 209 |
'sort': '-nom_rue', |
209 |
'q': "rue de l'aubepine", |
|
210 | 210 |
'rows': '3', |
211 |
'q': "rue de l'aubepine", |
|
211 | 212 |
} |
212 | 213 |
assert not resp.json['err'] |
213 | 214 |
assert len(resp.json['data']) == 3 |
214 | 215 |
# check order is kept |
215 | 216 |
assert [x['id'] for x in resp.json['data']] == [ |
216 | 217 |
'e00cf6161e52a4c8fe510b2b74d4952036cb3473', |
217 | 218 |
'7cafcd5c692773e8b863587b2d38d6be82e023d8', |
218 | 219 |
'0984a5e1745701f71c91af73ce764e1f7132e0ff', |
... | ... | |
242 | 243 |
assert resp.json['data'][0]['text'] == "19 RUE DE L'AUBEPINE Lipsheim" |
243 | 244 | |
244 | 245 | |
245 | 246 |
@mock.patch('passerelle.utils.Request.get') |
246 | 247 |
def test_query_q_using_q(mocked_get, app, query): |
247 | 248 |
endpoint = '/opendatasoft/my_connector/q/my_query/' |
248 | 249 |
params = { |
249 | 250 |
'q': "rue de l'aubepine", |
250 |
'limit': 3, |
|
251 | 251 |
} |
252 | 252 |
mocked_get.return_value = utils.FakedResponse(content=FAKED_CONTENT_Q_SEARCH, status_code=200) |
253 | 253 |
resp = app.get(endpoint, params=params, status=200) |
254 | 254 |
assert mocked_get.call_args[1]['params'] == { |
255 | 255 |
'apikey': 'my_secret', |
256 | 256 |
'dataset': 'referentiel-adresse-test', |
257 | 257 |
'refine.source': ['Ville et Eurométropole de Strasbourg'], |
258 | 258 |
'exclude.numero': ['42', '43'], |
259 | 259 |
'sort': '-nom_rue', |
260 |
'rows': 3, |
|
260 | 261 |
'q': "rue de l'aubepine", |
261 |
'rows': '3', |
|
262 | 262 |
} |
263 | 263 |
assert not resp.json['err'] |
264 | 264 |
assert len(resp.json['data']) == 3 |
265 | 265 |
# check order is kept |
266 | 266 |
assert [x['id'] for x in resp.json['data']] == [ |
267 | 267 |
'e00cf6161e52a4c8fe510b2b74d4952036cb3473', |
268 | 268 |
'7cafcd5c692773e8b863587b2d38d6be82e023d8', |
269 | 269 |
'0984a5e1745701f71c91af73ce764e1f7132e0ff', |
... | ... | |
286 | 286 |
} |
287 | 287 |
mocked_get.return_value = utils.FakedResponse(content=FAKED_CONTENT_ID_SEARCH, status_code=200) |
288 | 288 |
resp = app.get(endpoint, params=params, status=200) |
289 | 289 |
assert mocked_get.call_args[1]['params'] == { |
290 | 290 |
'dataset': 'referentiel-adresse-test', |
291 | 291 |
'q': 'recordid:7cafcd5c692773e8b863587b2d38d6be82e023d8', |
292 | 292 |
'apikey': 'my_secret', |
293 | 293 |
'refine.source': ['Ville et Eurométropole de Strasbourg'], |
294 |
'exclude.numero': ['42', '43'], |
|
295 | 294 |
'sort': '-nom_rue', |
295 |
'rows': 3, |
|
296 |
'exclude.numero': ['42', '43'], |
|
296 | 297 |
} |
297 | 298 |
assert len(resp.json['data']) == 1 |
298 | 299 |
assert resp.json['data'][0]['text'] == "19 RUE DE L'AUBEPINE Lipsheim" |
299 | 300 | |
300 | 301 | |
301 | 302 |
def test_opendatasoft_query_unicity(admin_user, app, connector, query): |
302 | 303 |
connector2 = OpenDataSoft.objects.create( |
303 | 304 |
slug='my_connector2', |
304 |
- |