Projet

Général

Profil

0003-opendatasoft-add-facet-filters-50212.patch

Nicolas Roche, 12 avril 2021 18:16

Télécharger (9,64 ko)

Voir les différences:

Subject: [PATCH 3/3] opendatasoft: add facet filters (#50212)

 .../migrations/0002_auto_20210409_1143.py     | 33 +++++++++++++
 passerelle/apps/opendatasoft/models.py        | 46 +++++++++++++------
 tests/test_opendatasoft.py                    | 16 +++++++
 3 files changed, 81 insertions(+), 14 deletions(-)
 create mode 100644 passerelle/apps/opendatasoft/migrations/0002_auto_20210409_1143.py
passerelle/apps/opendatasoft/migrations/0002_auto_20210409_1143.py
1
# -*- coding: utf-8 -*-
2
# Generated by Django 1.11.18 on 2021-04-09 09:43
3
from __future__ import unicode_literals
4

  
5
from django.db import migrations, models
6

  
7

  
8
class Migration(migrations.Migration):
9

  
10
    dependencies = [
11
        ('opendatasoft', '0001_initial'),
12
    ]
13

  
14
    operations = [
15
        migrations.AddField(
16
            model_name='query',
17
            name='filter_expression',
18
            field=models.TextField(
19
                blank=True,
20
                help_text='Specify refine and exclude facet expressions separated by &',
21
                verbose_name='filter',
22
            ),
23
        ),
24
        migrations.AlterField(
25
            model_name='opendatasoft',
26
            name='service_url',
27
            field=models.CharField(
28
                help_text='URL without ending "api/records/1.0/search/"',
29
                max_length=256,
30
                verbose_name='Site URL',
31
            ),
32
        ),
33
    ]
passerelle/apps/opendatasoft/models.py
59 59
            Query.objects.filter(resource=instance).delete()
60 60
        for data_query in data_queries:
61 61
            query = Query.import_json(data_query)
62 62
            query.resource = instance
63 63
            queries.append(query)
64 64
        Query.objects.bulk_create(queries)
65 65
        return instance
66 66

  
67
    @endpoint(
68
        perm='can_access',
69
        description=_('Search'),
70
        parameters={
71
            'dataset': {'description': _('Dataset')},
72
            'text_template': {'description': _('Text template')},
73
            'id': {'description': _('Record identifier')},
74
            'q': {'description': _('Full text query')},
75
            'limit': {'description': _('Maximum items')},
76
        },
77
    )
78
    def search(self, request, dataset=None, text_template='', id=None, q=None, limit=None, **kwargs):
67
    def call_search(
68
        self, dataset=None, text_template='', filter_expression='', id=None, q=None, limit=None, **kwargs
69
    ):
79 70
        scheme, netloc, path, params, query, fragment = urlparse.urlparse(self.service_url)
80 71
        path = urlparse.urljoin(path, 'api/records/1.0/search/')
81 72
        url = urlparse.urlunparse((scheme, netloc, path, params, query, fragment))
82 73

  
83 74
        if id is not None:
84 75
            query = 'recordid:%s' % id
85 76
        else:
86 77
            query = q
87 78
        params = {
88 79
            'dataset': dataset,
89 80
            'q': query,
90 81
        }
91 82
        if self.api_key:
92 83
            params.update({'apikey': self.api_key})
93 84
        if limit:
94 85
            params.update({'rows': limit})
86
        params.update(urlparse.parse_qs(filter_expression))
95 87

  
96 88
        result_response = self.requests.get(url, params=params)
97 89
        err_desc = result_response.json().get('error')
98 90
        if err_desc:
99 91
            raise APIError(err_desc, http_status=200)
100 92

  
101 93
        result = []
102 94
        for record in result_response.json().get('records'):
103 95
            data = {}
104 96
            for key, value in record.get('fields').items():
105 97
                data[key] = value
106 98
            data['id'] = record.get('recordid')
107 99
            data['text'] = render_to_string(text_template, data).strip()
108 100
            result.append(data)
109 101

  
102
        return result
103

  
104
    @endpoint(
105
        perm='can_access',
106
        description=_('Search'),
107
        parameters={
108
            'dataset': {'description': _('Dataset')},
109
            'text_template': {'description': _('Text template')},
110
            'id': {'description': _('Record identifier')},
111
            'q': {'description': _('Full text query')},
112
            'limit': {'description': _('Maximum items')},
113
        },
114
    )
115
    def search(self, request, dataset=None, text_template='', id=None, q=None, limit=None, **kwargs):
116
        result = self.call_search(dataset, text_template, id, q, limit)
110 117
        return {'data': result}
111 118

  
112 119
    @endpoint(
113 120
        name='q',
114 121
        description=_('Query'),
115 122
        pattern=r'^(?P<query_slug>[\w:_-]+)/$',
116 123
        perm='can_access',
117 124
        show=False,
118 125
    )
119 126
    def q(self, request, query_slug, **kwargs):
120 127
        query = get_object_or_404(Query, resource=self, slug=query_slug)
121
        return query.q(request, **kwargs)
128
        result = query.q(request, **kwargs)
129
        return {'data': result}
122 130

  
123 131
    def create_query_url(self):
124 132
        return reverse('opendatasoft-query-new', kwargs={'slug': self.slug})
125 133

  
126 134

  
127 135
class Query(BaseQuery):
128 136
    resource = models.ForeignKey(
129 137
        to=OpenDataSoft, related_name='queries', verbose_name=_('Resource'), on_delete=models.CASCADE
......
135 143
        help_text=_('dataset to query'),
136 144
    )
137 145
    text_template = models.TextField(
138 146
        verbose_name=_('Text template'),
139 147
        help_text=_("Use Django's template syntax. Attributes can be accessed through {{ attributes.name }}"),
140 148
        validators=[validate_template],
141 149
        blank=True,
142 150
    )
151
    filter_expression = models.TextField(
152
        verbose_name=_('filter'),
153
        help_text=_('Specify refine and exclude facet expressions separated by &'),
154
        blank=True,
155
    )
143 156

  
144 157
    delete_view = 'opendatasoft-query-delete'
145 158
    edit_view = 'opendatasoft-query-edit'
146 159

  
147 160
    def q(self, request, **kwargs):
148
        return self.resource.search(request, dataset=self.dataset, text_template=self.text_template, **kwargs)
161
        return self.resource.call_search(
162
            dataset=self.dataset,
163
            text_template=self.text_template,
164
            filter_expression=self.filter_expression,
165
            **kwargs,
166
        )
149 167

  
150 168
    def as_endpoint(self):
151 169
        endpoint = super(Query, self).as_endpoint(path=self.resource.q.endpoint_info.name)
152 170

  
153 171
        search_endpoint = self.resource.search.endpoint_info
154 172
        endpoint.func = search_endpoint.func
155 173
        endpoint.show_undocumented_params = False
156 174

  
tests/test_opendatasoft.py
140 140
def query(connector):
141 141
    return Query.objects.create(
142 142
        resource=connector,
143 143
        name='Référenciel adresses de test',
144 144
        slug='my_query',
145 145
        description='Rechercher une adresse',
146 146
        dataset='referentiel-adresse-test',
147 147
        text_template='{{numero}} {{nom_rue}} {{nom_commun}}',
148
        filter_expression='refine.source=Ville et Eurométropole de Strasbourg&exclude.numero=42&exclude.numero=43',
148 149
    )
149 150

  
150 151

  
151 152
def test_views(db, admin_user, app, connector):
152 153
    app = login(app)
153 154
    resp = app.get('/opendatasoft/my_connector/', status=200)
154 155
    resp = resp.click('New Query')
155 156
    resp.form['name'] = 'my query'
......
234 235
def test_query_q_using_q(mocked_get, app, query):
235 236
    endpoint = '/opendatasoft/my_connector/q/my_query/'
236 237
    params = {
237 238
        'q': "rue de l'aubepine",
238 239
        'limit': 3,
239 240
    }
240 241
    mocked_get.return_value = utils.FakedResponse(content=FAKED_CONTENT_Q_SEARCH, status_code=200)
241 242
    resp = app.get(endpoint, params=params, status=200)
243
    assert mocked_get.call_args[1]['params'] == {
244
        'dataset': 'referentiel-adresse-test',
245
        'q': "rue de l'aubepine",
246
        'apikey': 'my_secret',
247
        'rows': '3',
248
        'refine.source': ['Ville et Eurométropole de Strasbourg'],
249
        'exclude.numero': ['42', '43'],
250
    }
242 251
    assert not resp.json['err']
243 252
    assert len(resp.json['data']) == 3
244 253
    # check order is kept
245 254
    assert [x['id'] for x in resp.json['data']] == [
246 255
        'e00cf6161e52a4c8fe510b2b74d4952036cb3473',
247 256
        '7cafcd5c692773e8b863587b2d38d6be82e023d8',
248 257
        '0984a5e1745701f71c91af73ce764e1f7132e0ff',
249 258
    ]
......
260 269
@mock.patch('passerelle.utils.Request.get')
261 270
def test_query_q_using_id(mocked_get, app, query):
262 271
    endpoint = '/opendatasoft/my_connector/q/my_query/'
263 272
    params = {
264 273
        'id': '7cafcd5c692773e8b863587b2d38d6be82e023d8',
265 274
    }
266 275
    mocked_get.return_value = utils.FakedResponse(content=FAKED_CONTENT_ID_SEARCH, status_code=200)
267 276
    resp = app.get(endpoint, params=params, status=200)
277
    assert mocked_get.call_args[1]['params'] == {
278
        'dataset': 'referentiel-adresse-test',
279
        'q': 'recordid:7cafcd5c692773e8b863587b2d38d6be82e023d8',
280
        'apikey': 'my_secret',
281
        'refine.source': ['Ville et Eurométropole de Strasbourg'],
282
        'exclude.numero': ['42', '43'],
283
    }
268 284
    assert len(resp.json['data']) == 1
269 285
    assert resp.json['data'][0]['text'] == "19 RUE DE L'AUBEPINE Lipsheim"
270 286

  
271 287

  
272 288
def test_opendatasoft_query_unicity(admin_user, app, connector, query):
273 289
    connector2 = OpenDataSoft.objects.create(
274 290
        slug='my_connector2',
275 291
        api_key='my_secret',
276
-