From bd85ecd6dc29a7d3d38a46bcd889dc3827134f09 Mon Sep 17 00:00:00 2001
From: Thomas NOEL <tnoel@entrouvert.com>
Date: Thu, 12 May 2022 15:30:04 +0200
Subject: [PATCH 2/3] arcgis: add feature server query endpoint (#63825)

---
 passerelle/apps/arcgis/models.py |  91 ++++++++++-
 tests/test_arcgis.py             | 270 +++++++++++++++++++++++++++++++
 2 files changed, 355 insertions(+), 6 deletions(-)

diff --git a/passerelle/apps/arcgis/models.py b/passerelle/apps/arcgis/models.py
index 15f74c55..531c0e3e 100644
--- a/passerelle/apps/arcgis/models.py
+++ b/passerelle/apps/arcgis/models.py
@@ -76,7 +76,7 @@ class ArcGIS(BaseResource, HTTPResource):
             params['units'] = 'esriSRUnit_Meter'
         return params
 
-    def get_query_response(self, uri, params, id_template, text_template, full):
+    def get_query_response(self, uri, params, id_template, text_template, full, text_fieldname=None):
         url = urlparse.urljoin(self.base_url, uri)
         response = self.requests.get(url, params=params)
 
@@ -92,11 +92,14 @@ class ArcGIS(BaseResource, HTTPResource):
 
         features = infos.pop('features', [])
         id_fieldname = infos.get('objectIdFieldName') or 'OBJECTID'
-        text_fieldname = infos.get('displayFieldName')
+        text_fieldname = text_fieldname or infos.get('displayFieldName')
+        aliases = {}
+        for field in infos.get('fields') or []:
+            if field.get('alias') and field.get('name'):
+                aliases[field['alias']] = field['name']
         if infos.get('fieldAliases'):
-            aliases = {v: k for k, v in infos['fieldAliases'].items()}
-        else:
-            aliases = {}
+            for name, alias in infos['fieldAliases'].items():
+                aliases[alias] = name
 
         data = []
 
@@ -108,7 +111,9 @@ class ArcGIS(BaseResource, HTTPResource):
         for n, feature in enumerate(features):
             if 'attributes' in feature:
                 feature['id'] = '%s' % get_feature_attribute(feature, id_fieldname)
-                feature['text'] = '%s' % get_feature_attribute(feature, text_fieldname)
+                feature['text'] = (
+                    '%s' % get_feature_attribute(feature, text_fieldname) if text_fieldname else feature['id']
+                )
             else:
                 feature['id'] = feature['text'] = '%d' % (n + 1)
             if id_template:
@@ -193,6 +198,80 @@ class ArcGIS(BaseResource, HTTPResource):
             uri, params, id_template=id_template, text_template=template, full=full
         )
 
+    @endpoint(
+        name='featureservice-query',
+        description=_('Feature Service Query'),
+        perm='can_access',
+        parameters={
+            'folder': {
+                'description': _('Folder name'),
+                'example_value': 'Specialty',
+            },
+            'service': {
+                'description': _('Service name'),
+                'example_value': 'ESRI_StateCityHighway_USA',
+            },
+            'layer': {
+                'description': _('Layer or table name'),
+                'example_value': '1',
+            },
+            'lat': {'description': _('Latitude')},
+            'lon': {'description': _('Longitude')},
+            'latmin': {'description': _('Minimal latitude (envelope)')},
+            'lonmin': {'description': _('Minimal longitude (envelope)')},
+            'latmax': {'description': _('Maximal latitude (envelope)')},
+            'lonmax': {'description': _('Maximal longitude (envelope)')},
+            'text_fieldname': {
+                'description': _('Field name for text attribute'),
+                'example_value': 'STATE_NAME',
+            },
+            'template': {
+                'description': _('Django template for text attribute'),
+                'example_value': '{{ attributes.STATE_NAME }} ({{ attributes.STATE_ABBR }})',
+            },
+            'id_template': {
+                'description': _('Django template for id attribute'),
+            },
+            'full': {
+                'description': _('Returns all ArcGIS informations (geometry, metadata)'),
+                'type': 'bool',
+            },
+        },
+    )
+    def featureservice_query(
+        self,
+        request,
+        service,
+        layer='0',
+        folder='',
+        lat=None,
+        lon=None,
+        latmin=None,
+        lonmin=None,
+        latmax=None,
+        lonmax=None,
+        text_fieldname=None,
+        template=None,
+        id_template=None,
+        full=False,
+        **kwargs,
+    ):
+        uri = 'services/'
+        if folder:
+            uri += folder + '/'
+        uri = uri + service + '/FeatureServer/' + layer + '/query'
+
+        params = self.build_common_params(lat, lon, latmin, lonmin, latmax, lonmax, **kwargs)
+
+        return self.get_query_response(
+            uri,
+            params,
+            id_template=id_template,
+            text_template=template,
+            full=full,
+            text_fieldname=text_fieldname,
+        )
+
     @endpoint(
         name='tile',
         description=_('Tiles layer'),
diff --git a/tests/test_arcgis.py b/tests/test_arcgis.py
index e7922041..d5a552b0 100644
--- a/tests/test_arcgis.py
+++ b/tests/test_arcgis.py
@@ -81,6 +81,66 @@ STATES = '''{
    "displayFieldName" : "STATE_NAME"
 }'''
 
+FEATURES = '''{
+    "features": [
+        {
+            "attributes": {
+                "nom": "Le Nom",
+                "id": "onze",
+                "objectid": 11
+            },
+            "geometry" : {
+               "rings" : [
+                  [
+                     [-111.475425113078, 44.7021622250113],
+                     [-111.480804007084, 44.6914159859524]
+                  ]
+               ]
+            }
+        },
+        {
+            "attributes": {
+                "nom": "Autre Nom",
+                "id": "dix",
+                "objectid": 10
+            },
+            "geometry" : {
+               "rings" : [
+                  [
+                     [-111.475425113078, 44.7021622250113],
+                     [-111.480804007084, 44.6914159859524]
+                  ]
+               ]
+            }
+        }
+    ],
+    "fields": [
+        {
+            "alias": "OBJECTID",
+            "name": "objectid",
+            "type": "esriFieldTypeOID"
+        },
+        {
+            "alias": "quartier",
+            "name": "nom",
+            "type": "esriFieldTypeString"
+        },
+        {
+            "alias": "code",
+            "length": 5,
+            "name": "id",
+            "type": "esriFieldTypeString"
+        }
+    ],
+    "geometryType": "esriGeometryPolygon",
+    "globalIdFieldName": "",
+    "objectIdFieldName": "objectid",
+    "spatialReference": {
+        "latestWkid": 3947,
+        "wkid": 3947
+    }
+}'''
+
 
 @pytest.fixture
 def arcgis():
@@ -264,6 +324,216 @@ def test_arcgis_mapservice_query(app, arcgis):
         assert resp.json['err_desc'] == '<lonmin> <latmin> <lonmax> and <latmax> must be floats'
 
 
+def test_arcgis_featureservice_query(app, arcgis):
+    endpoint = tests.utils.generic_endpoint_url('arcgis', 'featureservice-query', slug=arcgis.slug)
+    assert endpoint == '/arcgis/test/featureservice-query'
+    params = {'folder': 'fold', 'service': 'serv', 'layer': '42'}
+
+    with mock.patch('passerelle.utils.Request.get') as requests_get:
+        requests_get.return_value = tests.utils.FakedResponse(content=FEATURES, status_code=200)
+
+        resp = app.get(endpoint, params=params, status=403)
+        assert requests_get.call_count == 0
+        assert resp.json['err'] == 1
+        assert resp.json['err_class'] == 'django.core.exceptions.PermissionDenied'
+
+        # open access
+        api = ApiUser.objects.create(username='all', keytype='', key='')
+        obj_type = ContentType.objects.get_for_model(arcgis)
+        AccessRight.objects.create(
+            codename='can_access', apiuser=api, resource_type=obj_type, resource_pk=arcgis.pk
+        )
+        resp = app.get(endpoint, params=params, status=200)
+        assert requests_get.call_count == 1
+        assert (
+            requests_get.call_args[0][0]
+            == 'https://arcgis.example.net/services/fold/serv/FeatureServer/42/query'
+        )
+        args = requests_get.call_args[1]['params']
+        assert args['f'] == 'json'
+        assert args['outFields'] == '*'
+        assert 'data' in resp.json
+        assert resp.json['err'] == 0
+        assert len(resp.json['data']) == 2
+        assert resp.json['data'][0]['id'] == '11'
+        assert resp.json['data'][0]['text'] == '11'
+        assert 'geometry' not in resp.json['data'][0]
+        assert 'metadata' not in resp.json
+
+        params['full'] = 'on'
+        resp = app.get(endpoint, params=params, status=200)
+        assert requests_get.call_count == 2
+        assert (
+            requests_get.call_args[0][0]
+            == 'https://arcgis.example.net/services/fold/serv/FeatureServer/42/query'
+        )
+        args = requests_get.call_args[1]['params']
+        assert args['f'] == 'json'
+        assert args['outFields'] == '*'
+        assert 'data' in resp.json
+        assert resp.json['err'] == 0
+        assert len(resp.json['data']) == 2
+        assert resp.json['data'][0]['id'] == '11'
+        assert resp.json['data'][0]['text'] == '11'
+        assert resp.json['data'][0]['geometry']
+        assert resp.json['metadata']
+
+        params['lat'] = '9.87654'
+        params['lon'] = '1.12345'
+        resp = app.get(endpoint, params=params, status=200)
+        assert requests_get.call_count == 3
+        assert (
+            requests_get.call_args[0][0]
+            == 'https://arcgis.example.net/services/fold/serv/FeatureServer/42/query'
+        )
+        args = requests_get.call_args[1]['params']
+        assert args['geometry'] == '1.12345,9.87654'
+        assert args['geometryType'] == 'esriGeometryPoint'
+
+        del params['lat']  # missing lat, do not search by geometry
+        resp = app.get(endpoint, params=params, status=200)
+        assert requests_get.call_count == 4
+        assert (
+            requests_get.call_args[0][0]
+            == 'https://arcgis.example.net/services/fold/serv/FeatureServer/42/query'
+        )
+        args = requests_get.call_args[1]['params']
+        assert 'geometry' not in args
+        assert 'geometryType' not in args
+
+        params.update({'latmin': '1', 'lonmin': '2', 'latmax': '3', 'lonmax': '4'})
+        resp = app.get(endpoint, params=params, status=200)
+        assert requests_get.call_count == 5
+        assert (
+            requests_get.call_args[0][0]
+            == 'https://arcgis.example.net/services/fold/serv/FeatureServer/42/query'
+        )
+        args = requests_get.call_args[1]['params']
+        assert args['geometry'] == '2.0,1.0,4.0,3.0'
+        assert args['geometryType'] == 'esriGeometryEnvelope'
+
+        del params['latmin']  # incomplete box, do not search by geometry
+        resp = app.get(endpoint, params=params, status=200)
+        assert requests_get.call_count == 6
+        assert (
+            requests_get.call_args[0][0]
+            == 'https://arcgis.example.net/services/fold/serv/FeatureServer/42/query'
+        )
+        args = requests_get.call_args[1]['params']
+        assert 'geometry' not in args
+        assert 'geometryType' not in args
+
+        # others params are directly sent to ArcGIS
+        params['spatialRel'] = 'esriSpatialRelContains'
+        params.update({'latmin': '1', 'lonmin': '2', 'latmax': '3', 'lonmax': '4'})
+        resp = app.get(endpoint, params=params, status=200)
+        assert requests_get.call_count == 7
+        assert (
+            requests_get.call_args[0][0]
+            == 'https://arcgis.example.net/services/fold/serv/FeatureServer/42/query'
+        )
+        args = requests_get.call_args[1]['params']
+        assert args['geometry'] == '2.0,1.0,4.0,3.0'
+        assert args['geometryType'] == 'esriGeometryEnvelope'
+        assert args['spatialRel'] == 'esriSpatialRelContains'
+
+        # folder
+        params['folder'] = 'foo/bar'
+        resp = app.get(endpoint, params=params, status=200)
+        assert (
+            requests_get.call_args[0][0]
+            == 'https://arcgis.example.net/services/foo/bar/serv/FeatureServer/42/query'
+        )
+        del params['folder']
+        resp = app.get(endpoint, params=params, status=200)
+        assert (
+            requests_get.call_args[0][0] == 'https://arcgis.example.net/services/serv/FeatureServer/42/query'
+        )
+
+        # minimal call
+        resp = app.get(endpoint, params={'service': 'srv'}, status=200)
+        assert requests_get.call_args[0][0] == 'https://arcgis.example.net/services/srv/FeatureServer/0/query'
+        args = requests_get.call_args[1]['params']
+        assert args == {'f': 'json', 'inSR': '4326', 'outSR': '4326', 'outFields': '*'}
+
+        # distance
+        resp = app.get(endpoint, params={'service': 'srv', 'distance': '100'}, status=200)
+        assert requests_get.call_args[0][0] == 'https://arcgis.example.net/services/srv/FeatureServer/0/query'
+        args = requests_get.call_args[1]['params']
+        assert args['distance'] == '100'
+        assert args['units'] == 'esriSRUnit_Meter'  # default unit
+        resp = app.get(
+            endpoint,
+            params={'service': 'srv', 'distance': '5', 'units': 'esriSRUnit_NauticalMile'},
+            status=200,
+        )
+        assert requests_get.call_args[0][0] == 'https://arcgis.example.net/services/srv/FeatureServer/0/query'
+        args = requests_get.call_args[1]['params']
+        assert args['distance'] == '5'
+        assert args['units'] == 'esriSRUnit_NauticalMile'
+
+        # text_fieldname
+        for text_fieldname in ('nom', 'quartier'):  # 'nom': real name, 'quartier': its alias
+            resp = app.get(
+                endpoint,
+                params={
+                    'service': 'srv',
+                    'text_fieldname': text_fieldname,
+                },
+                status=200,
+            )
+            assert 'data' in resp.json
+            assert resp.json['err'] == 0
+            assert len(resp.json['data']) == 2
+            assert resp.json['data'][0]['id'] == '11'
+            assert resp.json['data'][0]['text'] == 'Le Nom'
+            assert resp.json['data'][1]['id'] == '10'
+            assert resp.json['data'][1]['text'] == 'Autre Nom'
+
+        # templates
+        resp = app.get(
+            endpoint,
+            params={
+                'service': 'srv',
+                'template': '{{ attributes.nom }}',
+                'id_template': '{{ attributes.id }}',
+            },
+            status=200,
+        )
+        assert 'data' in resp.json
+        assert resp.json['err'] == 0
+        assert len(resp.json['data']) == 2
+        assert resp.json['data'][0]['id'] == 'onze'
+        assert resp.json['data'][0]['text'] == 'Le Nom'
+        assert resp.json['data'][1]['id'] == 'dix'
+        assert resp.json['data'][1]['text'] == 'Autre Nom'
+
+    # call errors
+    with mock.patch('passerelle.utils.Request.get') as requests_get:
+        requests_get.return_value = tests.utils.FakedResponse(content=FEATURES, status_code=200)
+        resp = app.get(endpoint, params={}, status=400)
+        assert requests_get.call_count == 0
+        assert resp.json['err'] == 1
+        assert resp.json['err_class'] == 'passerelle.views.WrongParameter'
+        assert resp.json['err_desc'] == "missing parameters: 'service'."
+
+        resp = app.get(endpoint, params={'service': 'src', 'lat': '0', 'lon': 'y'}, status=400)
+        assert requests_get.call_count == 0
+        assert resp.json['err'] == 1
+        assert resp.json['err_class'] == 'passerelle.utils.jsonresponse.APIError'
+        assert resp.json['err_desc'] == '<lon> and <lat> must be floats'
+
+        resp = app.get(
+            endpoint,
+            params={'service': 'src', 'latmin': '0', 'lonmin': 'y', 'latmax': '0', 'lonmax': '1'},
+            status=400,
+        )
+        assert requests_get.call_count == 0
+        assert resp.json['err'] == 1
+        assert resp.json['err_class'] == 'passerelle.utils.jsonresponse.APIError'
+        assert resp.json['err_desc'] == '<lonmin> <latmin> <lonmax> and <latmax> must be floats'
+
+
 @pytest.mark.parametrize(
     'format_string,fail',
     [
-- 
2.35.1

