Projet

Général

Profil

0002-opengis-handle-XML-format-for-WFS-features-too-38865.patch

Nicolas Roche (absent jusqu'au 3 avril), 24 janvier 2020 16:50

Télécharger (20,9 ko)

Voir les différences:

Subject: [PATCH 2/2] opengis: handle XML format for WFS features too (#38865)

 passerelle/apps/opengis/models.py | 102 +++++++++++++++++++------
 tests/test_opengis.py             | 122 ++++++++++++++++++++++++------
 2 files changed, 176 insertions(+), 48 deletions(-)
passerelle/apps/opengis/models.py
45 45
            d[attribute_name] = build_dict_from_xml(child)
46 46
    return d
47 47

  
48
def parse_xml_features(element):
49
    features = []
50
    ns = {'gml': 'http://www.opengis.net/gml'}
51
    for featureMember in element.findall(".//gml:featureMember", ns):
52
        feature = {}
53
        properties = {}
54
        for elem in featureMember[0]:
55
            if (elem.tag.endswith('geo_shape')
56
                and elem[0].tag == '{%s}Point' % ns['gml']
57
                and elem[0][0].tag == '{%s}pos' % ns['gml']):
58
                    coordinates = [float(x) for x in elem[0][0].text.split(' ')]
59
                    feature['geometry'] = {
60
                        'coordinates': coordinates,
61
                        'type': 'Point',
62
                    }
63
            elif elem.text != '':
64
                index = elem.tag.find('}')
65
                properties[elem.tag[index+1:]] = elem.text
66
        feature['properties'] = properties
67
        feature['type'] = "Feature"
68
        features.append(feature)
69
    return features
70

  
71

  
48 72
PROJECTIONS = (
49 73
    ('EPSG:2154', _('EPSG:2154 (Lambert-93)')),
50 74
    ('EPSG:3857', _('EPSG:3857 (WGS 84 / Pseudo-Mercator)')),
......
72 96
    class Meta:
73 97
        verbose_name = _('OpenGIS')
74 98

  
75
    def get_service_version(self, service_type, service_url, renew=False):
99
    def get_service_options(self, service_type, service_url, renew=False):
76 100
        if not service_url:
77 101
            raise APIError('no %s URL declared' % service_type)
78
        cache_key = 'opengis-%s-%s-version' % (service_type, self.id)
102
        cache_key = 'opengis-%s-%s-options' % (service_type, self.id)
79 103
        if not renew:
80
            service_version = cache.get(cache_key)
81
            if service_version:
82
                return service_version
104
            service_options = cache.get(cache_key)
105
            if service_options:
106
                return service_options
83 107
        response = self.requests.get(service_url, params={'request': 'GetCapabilities'})
84 108
        element = ET.fromstring(response.content)
85 109
        service_version = element.attrib.get('version')
86
        # cache version number for an hour
87
        cache.set(cache_key, service_version, 3600)
88
        return service_version
110
        service_outputformat = None
111
        if service_type == 'wfs':
112
            service_outputformat = 'xml'
113
            ns = {'ows': 'http://www.opengis.net/ows/1.1'}
114
            op = element.find(".//ows:Operation[@name='GetFeature']", ns)
115
            if op is not None:
116
                values = [x.text for x in op.iterfind(".//ows:Value", ns)]
117
                if values and len([x for x in values if 'json' in x]) > 0:
118
                    service_outputformat = 'json'
119
        service_options = (service_version, service_outputformat)
120
        # cache version number and outputFormat for an hour
121
        cache.set(cache_key, service_options, 3600)
122
        return service_options
89 123

  
90 124
    def get_wms_service_version(self, renew=False):
91
        return self.get_service_version('wms', self.wms_service_url, renew=renew)
125
        return self.get_service_options('wms', self.wms_service_url, renew=renew)[0]
92 126

  
93 127
    def get_wfs_service_version(self, renew=False):
94
        return self.get_service_version('wfs', self.wfs_service_url, renew=renew)
128
        return self.get_service_options('wfs', self.wfs_service_url, renew=renew)[0]
95 129

  
96
    def get_typename_label(self):
97
        version_str = self.get_wfs_service_version()
130
    def get_wfs_typename_label(self):
131
        version_str = self.get_wfs_service_version()[0]
98 132
        version_tuple = tuple(int(x) for x in version_str.split('.'))
99 133
        if version_tuple <= (1, 1, 0):
100 134
            return 'TYPENAME'
101 135
        else:
102 136
            return 'TYPENAMES'
103 137

  
138
    def get_wfs_outputformat(self):
139
        return self.get_service_options('wfs', self.wfs_service_url)[1]
140

  
104 141
    def check_status(self):
105 142
        if self.wms_service_url:
106 143
            response = self.requests.get(
......
148 185
              })
149 186
    def features(self, request, type_names, property_name, cql_filter=None,
150 187
                 filter_property_name=None, q=None, **kwargs):
188
        output_format = self.get_wfs_outputformat()
151 189
        params = {
152 190
            'VERSION': self.get_wfs_service_version(),
153 191
            'SERVICE': 'WFS',
154 192
            'REQUEST': 'GetFeature',
155
            self.get_typename_label(): type_names,
193
            self.get_wfs_typename_label(): type_names,
156 194
            'PROPERTYNAME': property_name,
157
            'OUTPUTFORMAT': 'json',
195
            'OUTPUTFORMAT': output_format,
158 196
        }
159 197
        if cql_filter:
160 198
            params.update({'CQL_FILTER': cql_filter})
......
165 203
                    operator = 'LIKE'
166 204
                params['CQL_FILTER'] += ' AND %s %s \'%%%s%%\'' % (filter_property_name, operator, q)
167 205
        response = self.requests.get(self.wfs_service_url, params=params)
206
        self.handle_opengis_error(response)
207
        # continue if handle_opengis_error did not raise an error
208
        if output_format == 'json':
209
            try:
210
                features = response.json()['features']
211
            except ValueError:
212
                raise APIError(u'OpenGIS Error: unparsable error',
213
                               data={'content': repr(response.content[:1024])})
214
        else:
215
            try:
216
                element = ET.fromstring(response.content.encode(encoding='UTF-8'))
217
            except ET.ParseError as exception:
218
                raise APIError(u'OpenGIS Error: %s' % exception,
219
                               data={'content': repr(response.content[:1024])})
220
            features = parse_xml_features(element)
168 221
        data = []
169
        try:
170
            response.json()
171
        except ValueError:
172
            self.handle_opengis_error(response)
173
            # if handle_opengis_error did not raise an error, we raise a generic one
174
            raise APIError(u'OpenGIS Error: unparsable error',
175
                           data={'content': repr(response.content[:1024])})
176
        for feature in response.json()['features']:
222
        for feature in features:
177 223
            feature['text'] = feature['properties'].get(property_name)
178 224
            data.append(feature)
179 225
        return {'data': data}
......
301 347
        lon, lat = self.convert_coordinates(lon, lat)
302 348

  
303 349
        cql_filter = 'DWITHIN(the_geom,Point(%s %s),%s,meters)' % (lon, lat, self.search_radius)
350
        output_format = self.get_wfs_outputformat()
304 351
        params = {
305 352
            'VERSION': self.get_wfs_service_version(),
306 353
            'SERVICE': 'WFS',
307 354
            'REQUEST': 'GetFeature',
308
            self.get_typename_label(): self.query_layer,
309
            'OUTPUTFORMAT': 'json',
355
            self.get_wfs_typename_label(): self.query_layer,
356
            'OUTPUTFORMAT': output_format,
310 357
            'CQL_FILTER': cql_filter
311 358
        }
312 359
        response = self.requests.get(self.wfs_service_url, params=params)
313 360
        if not response.ok:
314 361
            raise APIError('Webservice returned status code %s' % response.status_code)
362
        if output_format == 'json':
363
            features = response.json().get('features')
364
        else:
365
            element = ET.fromstring(response.text.encode(encoding='UTF-8'))
366
            features = parse_xml_features(element)
315 367
        closest_feature = {}
316 368
        min_delta = None
317
        for feature in response.json().get('features'):
369
        for feature in features:
318 370
            if not feature['geometry']['type'] == 'Point':
319 371
                continue  # skip unknown
320 372
            lon_diff = abs(float(lon) - float(feature['geometry']['coordinates'][0]))
tests/test_opengis.py
1
# -*- coding: utf-8 -*-
1 2
import mock
2 3
import pytest
3 4

  
......
39 40
        <ows:ServiceType>WFS</ows:ServiceType><ows:ServiceTypeVersion>2.0.0</ows:ServiceTypeVersion><ows:Fees/>
40 41
        <ows:AccessConstraints/>
41 42
    </ows:ServiceIdentification>
43
    <ows:ServiceProvider/>
44
    <ows:OperationsMetadata>
45
      <ows:Operation name="GetFeature">
46
        <ows:DCP/>
47
        <ows:Parameter name="outputFormat">
48
          <ows:AllowedValues>
49
            <ows:Value>text/xml; subtype=gml/3.2</ows:Value>
50
            <ows:Value>application/json</ows:Value>
51
            <ows:Value>json</ows:Value>
52
          </ows:AllowedValues>
53
        </ows:Parameter>
54
      </ows:Operation>
55
    </ows:OperationsMetadata>
42 56
</wfs:WFS_Capabilities>'''
43 57

  
44 58
FAKE_SERVICE_CAPABILITIES_V1_0_0 = '''<?xml version="1.0" encoding="UTF-8"?>
......
69 83
            "properties": {
70 84
                "nom": "Champagnier"
71 85
            },
72
"type": "Feature"
86
            "type": "Feature"
73 87
        },
74 88
        {
75 89
            "geometry": null,
......
115 129
    "type": "FeatureCollection"
116 130
}'''
117 131

  
132
FAKE_FEATURES_XML = u'''<?xml version="1.0" encoding="UTF-8"?>
133
<wfs:FeatureCollection xmlns:gml="http://www.opengis.net/gml"
134
xmlns:wfs="http://www.opengis.net/wfs" xmlns:xxx="http://www.somewhere.net/xxx">
135
  <gml:boundedBy/>
136
  <gml:featureMember>
137
    <xxx:points_apport_volontaire_dmt gml:id="points_apport_volontaire_dmt.e097e421cc609bcc752c43cd4e4c9d992cb0ee3d">
138
      <xxx:geo_shape>
139
        <gml:Point srsName="urn:x-ogc:def:crs:EPSG:4326">
140
          <gml:pos>1914018.64 4224644.61</gml:pos>
141
        </gml:Point>
142
      </xxx:geo_shape>
143
      <numero>4</numero>
144
      <xxx:nom_voie>place victor hugo</xxx:nom_voie>
145
      <xxx:code_post>38000</xxx:code_post>
146
      <xxx:nom_commune>Grenoble</xxx:nom_commune>
147
    </xxx:points_apport_volontaire_dmt>
148
  </gml:featureMember>
149
</wfs:FeatureCollection>'''
150

  
118 151
FAKE_ERROR = u'''<ows:ExceptionReport
119 152
     xmlns:xs="http://www.w3.org/2001/XMLSchema"
120 153
     xmlns:ows="http://www.opengis.net/ows/1.1"
......
241 274
def geoserver_responses_v1_0_0(url, **kwargs):
242 275
    if kwargs['params'].get('request') == 'GetCapabilities':
243 276
        return utils.FakedResponse(status_code=200, content=FAKE_SERVICE_CAPABILITIES_V1_0_0)
244
    return utils.FakedResponse(status_code=200, content=FAKE_FEATURES_JSON)
277
    return utils.FakedResponse(status_code=200, content=FAKE_FEATURES_XML)
278

  
245 279

  
246 280
def geoserver_responses_errors(url, **kwargs):
247 281
    if kwargs['params'].get('request') == 'GetCapabilities':
248 282
        return utils.FakedResponse(status_code=200, content=FAKE_SERVICE_CAPABILITIES)
249 283
    return utils.FakedResponse(status_code=200, content=FAKE_ERROR)
250 284

  
285
def geoserver_responses_errors_v1_0_0(url, **kwargs):
286
    if kwargs['params'].get('request') == 'GetCapabilities':
287
        return utils.FakedResponse(status_code=200, content=FAKE_SERVICE_CAPABILITIES_V1_0_0)
288
    return utils.FakedResponse(status_code=200, content=FAKE_ERROR)
289

  
251 290

  
252 291
def geoserver_responses_errors_unparsable(url, **kwargs):
253 292
    if kwargs['params'].get('request') == 'GetCapabilities':
254 293
        return utils.FakedResponse(status_code=200, content=FAKE_SERVICE_CAPABILITIES)
255 294
    return utils.FakedResponse(status_code=200, content=FAKE_ERROR[:10])
256 295

  
296
def geoserver_responses_errors_unparsable_v1_0_0(url, **kwargs):
297
    if kwargs['params'].get('request') == 'GetCapabilities':
298
        return utils.FakedResponse(status_code=200, content=FAKE_SERVICE_CAPABILITIES_V1_0_0)
299
    return utils.FakedResponse(status_code=200, content=FAKE_ERROR[:10])
300

  
257 301

  
258 302
@mock.patch('passerelle.utils.Request.get')
259 303
def test_feature_info(mocked_get, app, connector):
......
318 362
    assert resp.json['err_desc'] == 'no wfs URL declared'
319 363

  
320 364

  
365
@pytest.mark.parametrize("server_responses, nb_res", [
366
    (geoserver_responses, 7),
367
    (geoserver_responses_v1_0_0, 1),
368
])
321 369
@mock.patch('passerelle.utils.Request.get')
322
def test_get_feature(mocked_get, app, connector):
370
def test_get_feature(mocked_get, server_responses, nb_res, app, connector):
323 371
    endpoint = utils.generic_endpoint_url('opengis', 'features', slug=connector.slug)
324 372
    assert endpoint == '/opengis/test/features'
325
    mocked_get.side_effect = geoserver_responses
373
    mocked_get.side_effect = server_responses
326 374
    resp = app.get(endpoint, params={'type_names': 'ref_metro_limites_communales', 'property_name': 'nom'})
327 375
    assert mocked_get.call_args[1]['params']['REQUEST'] == 'GetFeature'
328 376
    assert mocked_get.call_args[1]['params']['PROPERTYNAME'] == 'nom'
329
    assert mocked_get.call_args[1]['params']['TYPENAMES'] == 'ref_metro_limites_communales'
330
    assert mocked_get.call_args[1]['params']['OUTPUTFORMAT'] == 'json'
377
    assert mocked_get.call_args[1]['params'][connector.get_wfs_typename_label()] == 'ref_metro_limites_communales'
378
    assert mocked_get.call_args[1]['params']['OUTPUTFORMAT'] == connector.get_wfs_outputformat()
331 379
    assert mocked_get.call_args[1]['params']['SERVICE'] == 'WFS'
332 380
    assert mocked_get.call_args[1]['params']['VERSION'] == connector.get_wfs_service_version()
333
    assert len(resp.json['data']) == 7
381
    assert len(resp.json['data']) == nb_res
334 382
    for item in resp.json['data']:
335 383
        assert 'text' in item
336 384

  
......
368 416
    assert 'CQL_FILTER' not in mocked_get.call_args[1]['params']
369 417

  
370 418

  
419
@pytest.mark.parametrize("server_responses", [
420
    (geoserver_responses_errors),
421
    (geoserver_responses_errors_v1_0_0),
422
])
371 423
@mock.patch('passerelle.utils.Request.get')
372
def test_get_feature_error(mocked_get, app, connector):
424
def test_get_feature_error(mocked_get, server_responses, app, connector):
373 425
    endpoint = utils.generic_endpoint_url('opengis', 'features', slug=connector.slug)
374 426
    assert endpoint == '/opengis/test/features'
375
    mocked_get.side_effect = geoserver_responses_errors
427
    mocked_get.side_effect = server_responses
376 428
    resp = app.get(endpoint, params={
377 429
        'type_names': 'ref_metro_limites_communales',
378 430
        'property_name': 'nom'
379 431
    })
380 432
    assert mocked_get.call_args[1]['params']['REQUEST'] == 'GetFeature'
381 433
    assert mocked_get.call_args[1]['params']['PROPERTYNAME'] == 'nom'
382
    assert mocked_get.call_args[1]['params']['TYPENAMES'] == 'ref_metro_limites_communales'
383
    assert mocked_get.call_args[1]['params']['OUTPUTFORMAT'] == 'json'
434
    assert mocked_get.call_args[1]['params'][connector.get_wfs_typename_label()] == 'ref_metro_limites_communales'
435
    assert mocked_get.call_args[1]['params']['OUTPUTFORMAT'] == connector.get_wfs_outputformat()
384 436
    assert mocked_get.call_args[1]['params']['SERVICE'] == 'WFS'
385 437
    assert mocked_get.call_args[1]['params']['VERSION'] == connector.get_wfs_service_version()
386 438
    result = resp.json
......
389 441
    assert 'Could not parse' in result['data']['text']
390 442

  
391 443

  
444
@pytest.mark.parametrize("server_responses, error_msg", [
445
    (geoserver_responses_errors_unparsable, 'unparsable error'),
446
    (geoserver_responses_errors_unparsable_v1_0_0, 'unclosed token: line 1, column 0'),
447
])
392 448
@mock.patch('passerelle.utils.Request.get')
393
def test_get_feature_error2(mocked_get, app, connector):
449
def test_get_feature_error2(mocked_get, server_responses, error_msg, app, connector):
394 450
    endpoint = utils.generic_endpoint_url('opengis', 'features', slug=connector.slug)
395 451
    assert endpoint == '/opengis/test/features'
396
    mocked_get.side_effect = geoserver_responses_errors_unparsable
452
    mocked_get.side_effect = server_responses
397 453
    resp = app.get(endpoint, params={
398 454
        'type_names': 'ref_metro_limites_communales',
399 455
        'property_name': 'nom'
400 456
    })
401 457
    assert mocked_get.call_args[1]['params']['REQUEST'] == 'GetFeature'
402 458
    assert mocked_get.call_args[1]['params']['PROPERTYNAME'] == 'nom'
403
    assert mocked_get.call_args[1]['params']['TYPENAMES'] == 'ref_metro_limites_communales'
404
    assert mocked_get.call_args[1]['params']['OUTPUTFORMAT'] == 'json'
459
    assert mocked_get.call_args[1]['params'][connector.get_wfs_typename_label()] == 'ref_metro_limites_communales'
460
    assert mocked_get.call_args[1]['params']['OUTPUTFORMAT'] == connector.get_wfs_outputformat()
405 461
    assert mocked_get.call_args[1]['params']['SERVICE'] == 'WFS'
406 462
    assert mocked_get.call_args[1]['params']['VERSION'] == connector.get_wfs_service_version()
407 463
    result = resp.json
408 464
    assert result['err'] == 1
409
    assert result['err_desc'] == 'OpenGIS Error: unparsable error'
465
    assert result['err_desc'] == 'OpenGIS Error: %s' % error_msg
410 466
    assert '<ows:' in result['data']['content']
411 467

  
412 468

  
413 469
@pytest.mark.parametrize("server_responses, version, typename_label", [
470
    (geoserver_responses, '2.0.0', 'TYPENAMES'),
414 471
    (geoserver_responses_v1_0_0, '1.0.0', 'TYPENAME'),
415
    (geoserver_responses, '2.0.0', 'TYPENAMES')])
472
])
416 473
@mock.patch('passerelle.utils.Request.get')
417 474
def test_typename_parameter_upgrade(mocked_get, server_responses, version, typename_label, app, connector):
418 475
    endpoint = utils.generic_endpoint_url('opengis', 'features', slug=connector.slug)
419 476
    assert endpoint == '/opengis/test/features'
420 477
    mocked_get.side_effect = server_responses
421
    resp = app.get(endpoint, params={'type_names': '...', 'property_name': '...'})
478
    resp = app.get(endpoint, params={'type_names': 'ref_metro_limites_communales', 'property_name': 'nom'})
422 479
    assert mocked_get.call_args[1]['params']['REQUEST'] == 'GetFeature'
423 480
    assert mocked_get.call_args[1]['params']['VERSION'] == version
424 481
    assert typename_label in mocked_get.call_args[1]['params'].keys()
482
    assert mocked_get.call_args[1]['params'][connector.get_wfs_typename_label()] == 'ref_metro_limites_communales'
483

  
484

  
485
@pytest.mark.parametrize("server_responses, service_format", [
486
    (geoserver_responses, 'json'),
487
    (geoserver_responses_v1_0_0, 'xml'),
488
])
489
@mock.patch('passerelle.utils.Request.get')
490
def test_outputFormat_possible_values(mocked_get, server_responses, service_format, app, connector):
491
    endpoint = utils.generic_endpoint_url('opengis', 'features', slug=connector.slug)
492
    assert endpoint == '/opengis/test/features'
493
    mocked_get.side_effect = server_responses
494
    resp = app.get(endpoint, params={'type_names': '...', 'property_name': '...'})
495
    assert mocked_get.call_args[1]['params']['REQUEST'] == 'GetFeature'
496
    assert mocked_get.call_args[1]['params']['OUTPUTFORMAT'] == service_format
425 497

  
426 498

  
499
@pytest.mark.parametrize("resp1, resp2, resp3, resp4", [
500
    (FAKE_SERVICE_CAPABILITIES, FAKE_GEOLOCATED_FEATURE, '{"features": []}', '{}'),
501
    (FAKE_SERVICE_CAPABILITIES_V1_0_0, FAKE_FEATURES_XML, '<?xml version="1.0"?><empty/>', ''),
502
])
427 503
@mock.patch('passerelle.utils.Request.get')
428
def test_reverse_geocoding(mocked_get, app, connector):
504
def test_reverse_geocoding(mocked_get, resp1, resp2, resp3, resp4, app, connector):
429 505
    connector.search_radius = 45
430 506
    connector.projection = 'EPSG:3945'
431 507
    connector.save()
......
434 510

  
435 511
    def side_effect(url, **kwargs):
436 512
        if kwargs['params'].get('request') == 'GetCapabilities':
437
            return utils.FakedResponse(status_code=200, content=FAKE_SERVICE_CAPABILITIES)
513
            return utils.FakedResponse(status_code=200, content=resp1)
438 514
        return mock.DEFAULT
439 515
    mocked_get.side_effect = side_effect
440
    mocked_get.return_value = utils.FakedResponse(content=FAKE_GEOLOCATED_FEATURE, status_code=200)
516
    mocked_get.return_value = utils.FakedResponse(content=resp2, status_code=200)
441 517
    resp = app.get(endpoint,
442 518
                   params={
443 519
                       'lat': '45.1893469606986',
......
455 531
    connector.projection = 'EPSG:4326'
456 532
    connector.search_radius = 10
457 533
    connector.save()
458
    mocked_get.return_value = utils.FakedResponse(content='{"features": []}', status_code=200)
534
    mocked_get.return_value = utils.FakedResponse(content=resp3, status_code=200)
459 535
    resp = app.get(
460 536
        endpoint,
461 537
        params={
......
466 542
    assert resp.json['err'] == 1
467 543
    assert resp.json['err_desc'] == 'Unable to geocode'
468 544

  
469
    mocked_get.return_value = utils.FakedResponse(status_code=404, content='{}', ok=False)
545
    mocked_get.return_value = utils.FakedResponse(status_code=404, content=resp4, ok=False)
470 546
    resp = app.get(endpoint,
471 547
                   params={
472 548
                       'lat': '45.183784',
473
-