0003-opendatasoft-add-facet-filters-50212.patch
passerelle/apps/opendatasoft/migrations/0002_auto_20210409_1143.py | ||
---|---|---|
1 |
# -*- coding: utf-8 -*- |
|
2 |
# Generated by Django 1.11.18 on 2021-04-09 09:43 |
|
3 |
from __future__ import unicode_literals |
|
4 | ||
5 |
from django.db import migrations, models |
|
6 | ||
7 | ||
8 |
class Migration(migrations.Migration): |
|
9 | ||
10 |
dependencies = [ |
|
11 |
('opendatasoft', '0001_initial'), |
|
12 |
] |
|
13 | ||
14 |
operations = [ |
|
15 |
migrations.AddField( |
|
16 |
model_name='query', |
|
17 |
name='filter_expression', |
|
18 |
field=models.TextField( |
|
19 |
blank=True, |
|
20 |
help_text='Specify refine and exclude facet expressions separated by &', |
|
21 |
verbose_name='filter', |
|
22 |
), |
|
23 |
), |
|
24 |
migrations.AlterField( |
|
25 |
model_name='opendatasoft', |
|
26 |
name='service_url', |
|
27 |
field=models.CharField( |
|
28 |
help_text='URL without ending "api/records/1.0/search/"', |
|
29 |
max_length=256, |
|
30 |
verbose_name='Site URL', |
|
31 |
), |
|
32 |
), |
|
33 |
] |
passerelle/apps/opendatasoft/models.py | ||
---|---|---|
59 | 59 |
Query.objects.filter(resource=instance).delete() |
60 | 60 |
for data_query in data_queries: |
61 | 61 |
query = Query.import_json(data_query) |
62 | 62 |
query.resource = instance |
63 | 63 |
queries.append(query) |
64 | 64 |
Query.objects.bulk_create(queries) |
65 | 65 |
return instance |
66 | 66 | |
67 |
@endpoint( |
|
68 |
perm='can_access', |
|
69 |
description=_('Search'), |
|
70 |
parameters={ |
|
71 |
'dataset': {'description': _('Dataset')}, |
|
72 |
'text_template': {'description': _('Text template')}, |
|
73 |
'id': {'description': _('Record identifier')}, |
|
74 |
'q': {'description': _('Full text query')}, |
|
75 |
'limit': {'description': _('Maximum items')}, |
|
76 |
}, |
|
77 |
) |
|
78 |
def search(self, request, dataset=None, text_template='', id=None, q=None, limit=None, **kwargs): |
|
67 |
def call_search( |
|
68 |
self, dataset=None, text_template='', filter_expression='', id=None, q=None, limit=None, **kwargs |
|
69 |
): |
|
79 | 70 |
scheme, netloc, path, params, query, fragment = urlparse.urlparse(self.service_url) |
80 | 71 |
path = urlparse.urljoin(path, 'api/records/1.0/search/') |
81 | 72 |
url = urlparse.urlunparse((scheme, netloc, path, params, query, fragment)) |
82 | 73 | |
83 | 74 |
if id is not None: |
84 | 75 |
query = 'recordid:%s' % id |
85 | 76 |
else: |
86 | 77 |
query = q |
87 | 78 |
params = { |
88 | 79 |
'dataset': dataset, |
89 | 80 |
'q': query, |
90 | 81 |
} |
91 | 82 |
if self.api_key: |
92 | 83 |
params.update({'apikey': self.api_key}) |
93 | 84 |
if limit: |
94 | 85 |
params.update({'rows': limit}) |
86 |
params.update(urlparse.parse_qs(filter_expression)) |
|
95 | 87 | |
96 | 88 |
result_response = self.requests.get(url, params=params) |
97 | 89 |
err_desc = result_response.json().get('error') |
98 | 90 |
if err_desc: |
99 | 91 |
raise APIError(err_desc, http_status=200) |
100 | 92 | |
101 | 93 |
result = [] |
102 | 94 |
for record in result_response.json().get('records'): |
103 | 95 |
data = {} |
104 | 96 |
for key, value in record.get('fields').items(): |
105 | 97 |
data[key] = value |
106 | 98 |
data['id'] = record.get('recordid') |
107 | 99 |
data['text'] = render_to_string(text_template, data).strip() |
108 | 100 |
result.append(data) |
109 | 101 | |
102 |
return result |
|
103 | ||
104 |
@endpoint( |
|
105 |
perm='can_access', |
|
106 |
description=_('Search'), |
|
107 |
parameters={ |
|
108 |
'dataset': {'description': _('Dataset')}, |
|
109 |
'text_template': {'description': _('Text template')}, |
|
110 |
'id': {'description': _('Record identifier')}, |
|
111 |
'q': {'description': _('Full text query')}, |
|
112 |
'limit': {'description': _('Maximum items')}, |
|
113 |
}, |
|
114 |
) |
|
115 |
def search(self, request, dataset=None, text_template='', id=None, q=None, limit=None, **kwargs): |
|
116 |
result = self.call_search(dataset, text_template, '', id, q, limit) |
|
110 | 117 |
return {'data': result} |
111 | 118 | |
112 | 119 |
@endpoint( |
113 | 120 |
name='q', |
114 | 121 |
description=_('Query'), |
115 | 122 |
pattern=r'^(?P<query_slug>[\w:_-]+)/$', |
116 | 123 |
perm='can_access', |
117 | 124 |
show=False, |
118 | 125 |
) |
119 | 126 |
def q(self, request, query_slug, **kwargs): |
120 | 127 |
query = get_object_or_404(Query, resource=self, slug=query_slug) |
121 |
return query.q(request, **kwargs) |
|
128 |
result = query.q(**kwargs) |
|
129 |
return {'data': result} |
|
122 | 130 | |
123 | 131 |
def create_query_url(self): |
124 | 132 |
return reverse('opendatasoft-query-new', kwargs={'slug': self.slug}) |
125 | 133 | |
126 | 134 | |
127 | 135 |
class Query(BaseQuery): |
128 | 136 |
resource = models.ForeignKey( |
129 | 137 |
to=OpenDataSoft, related_name='queries', verbose_name=_('Resource'), on_delete=models.CASCADE |
... | ... | |
135 | 143 |
help_text=_('dataset to query'), |
136 | 144 |
) |
137 | 145 |
text_template = models.TextField( |
138 | 146 |
verbose_name=_('Text template'), |
139 | 147 |
help_text=_("Use Django's template syntax. Attributes can be accessed through {{ attributes.name }}"), |
140 | 148 |
validators=[validate_template], |
141 | 149 |
blank=True, |
142 | 150 |
) |
151 |
filter_expression = models.TextField( |
|
152 |
verbose_name=_('filter'), |
|
153 |
help_text=_('Specify refine and exclude facet expressions separated by &'), |
|
154 |
blank=True, |
|
155 |
) |
|
143 | 156 | |
144 | 157 |
delete_view = 'opendatasoft-query-delete' |
145 | 158 |
edit_view = 'opendatasoft-query-edit' |
146 | 159 | |
147 |
def q(self, request, **kwargs): |
|
148 |
return self.resource.search(request, dataset=self.dataset, text_template=self.text_template, **kwargs) |
|
160 |
def q(self, **kwargs): |
|
161 |
return self.resource.call_search( |
|
162 |
self.dataset, |
|
163 |
self.text_template, |
|
164 |
self.filter_expression, |
|
165 |
kwargs.get('id'), |
|
166 |
kwargs.get('q'), |
|
167 |
kwargs.get('limit'), |
|
168 |
) |
|
149 | 169 | |
150 | 170 |
def as_endpoint(self): |
151 | 171 |
endpoint = super(Query, self).as_endpoint(path=self.resource.q.endpoint_info.name) |
152 | 172 | |
153 | 173 |
search_endpoint = self.resource.search.endpoint_info |
154 | 174 |
endpoint.func = search_endpoint.func |
155 | 175 |
endpoint.show_undocumented_params = False |
156 | 176 |
tests/test_opendatasoft.py | ||
---|---|---|
140 | 140 |
def query(connector): |
141 | 141 |
return Query.objects.create( |
142 | 142 |
resource=connector, |
143 | 143 |
name='Référenciel adresses de test', |
144 | 144 |
slug='my_query', |
145 | 145 |
description='Rechercher une adresse', |
146 | 146 |
dataset='referentiel-adresse-test', |
147 | 147 |
text_template='{{numero}} {{nom_rue}} {{nom_commun}}', |
148 |
filter_expression='refine.source=Ville et Eurométropole de Strasbourg&exclude.numero=42&exclude.numero=43', |
|
148 | 149 |
) |
149 | 150 | |
150 | 151 | |
151 | 152 |
def test_views(db, admin_user, app, connector): |
152 | 153 |
app = login(app) |
153 | 154 |
resp = app.get('/opendatasoft/my_connector/', status=200) |
154 | 155 |
resp = resp.click('New Query') |
155 | 156 |
resp.form['name'] = 'my query' |
... | ... | |
234 | 235 |
def test_query_q_using_q(mocked_get, app, query): |
235 | 236 |
endpoint = '/opendatasoft/my_connector/q/my_query/' |
236 | 237 |
params = { |
237 | 238 |
'q': "rue de l'aubepine", |
238 | 239 |
'limit': 3, |
239 | 240 |
} |
240 | 241 |
mocked_get.return_value = utils.FakedResponse(content=FAKED_CONTENT_Q_SEARCH, status_code=200) |
241 | 242 |
resp = app.get(endpoint, params=params, status=200) |
243 |
assert mocked_get.call_args[1]['params'] == { |
|
244 |
'dataset': 'referentiel-adresse-test', |
|
245 |
'q': "rue de l'aubepine", |
|
246 |
'apikey': 'my_secret', |
|
247 |
'rows': '3', |
|
248 |
'refine.source': ['Ville et Eurométropole de Strasbourg'], |
|
249 |
'exclude.numero': ['42', '43'], |
|
250 |
} |
|
242 | 251 |
assert not resp.json['err'] |
243 | 252 |
assert len(resp.json['data']) == 3 |
244 | 253 |
# check order is kept |
245 | 254 |
assert [x['id'] for x in resp.json['data']] == [ |
246 | 255 |
'e00cf6161e52a4c8fe510b2b74d4952036cb3473', |
247 | 256 |
'7cafcd5c692773e8b863587b2d38d6be82e023d8', |
248 | 257 |
'0984a5e1745701f71c91af73ce764e1f7132e0ff', |
249 | 258 |
] |
... | ... | |
260 | 269 |
@mock.patch('passerelle.utils.Request.get') |
261 | 270 |
def test_query_q_using_id(mocked_get, app, query): |
262 | 271 |
endpoint = '/opendatasoft/my_connector/q/my_query/' |
263 | 272 |
params = { |
264 | 273 |
'id': '7cafcd5c692773e8b863587b2d38d6be82e023d8', |
265 | 274 |
} |
266 | 275 |
mocked_get.return_value = utils.FakedResponse(content=FAKED_CONTENT_ID_SEARCH, status_code=200) |
267 | 276 |
resp = app.get(endpoint, params=params, status=200) |
277 |
assert mocked_get.call_args[1]['params'] == { |
|
278 |
'dataset': 'referentiel-adresse-test', |
|
279 |
'q': 'recordid:7cafcd5c692773e8b863587b2d38d6be82e023d8', |
|
280 |
'apikey': 'my_secret', |
|
281 |
'refine.source': ['Ville et Eurométropole de Strasbourg'], |
|
282 |
'exclude.numero': ['42', '43'], |
|
283 |
} |
|
268 | 284 |
assert len(resp.json['data']) == 1 |
269 | 285 |
assert resp.json['data'][0]['text'] == "19 RUE DE L'AUBEPINE Lipsheim" |
270 | 286 | |
271 | 287 | |
272 | 288 |
def test_opendatasoft_query_unicity(admin_user, app, connector, query): |
273 | 289 |
connector2 = OpenDataSoft.objects.create( |
274 | 290 |
slug='my_connector2', |
275 | 291 |
api_key='my_secret', |
276 |
- |