Produits Entr'ouvert » Passerelle

# -*- coding: utf-8 -*-

# Generated by Django 1.11.16 on 2018-11-02 14:50

from __future__ import unicode_literals

from django.db import migrations, models

class Migration(migrations.Migration):

    dependencies = [

        ('arcgis', '0002_auto_20170920_0951'),

    ]

    operations = [

        migrations.AlterModelOptions(

            name='arcgis',

            options={'verbose_name': 'ArcGIS REST API'},

        ),

        migrations.AddField(

            model_name='arcgis',

            name='basic_auth_password',

            field=models.CharField(blank=True, max_length=128, verbose_name='Basic authentication password'),

        ),

        migrations.AddField(

            model_name='arcgis',

            name='basic_auth_username',

            field=models.CharField(blank=True, max_length=128, verbose_name='Basic authentication username'),

        ),

        migrations.AddField(

            model_name='arcgis',

            name='client_certificate',

            field=models.FileField(blank=True, null=True, upload_to=b'', verbose_name='TLS client certificate'),

        ),

        migrations.AddField(

            model_name='arcgis',

            name='http_proxy',

            field=models.CharField(blank=True, max_length=128, verbose_name='HTTP and HTTPS proxy'),

        ),

        migrations.AddField(

            model_name='arcgis',

            name='trusted_certificate_authorities',

            field=models.FileField(blank=True, null=True, upload_to=b'', verbose_name='TLS trusted CAs'),

        ),

        migrations.AddField(

            model_name='arcgis',

            name='verify_cert',

            field=models.BooleanField(default=True, verbose_name='TLS verify certificates'),

        ),

        migrations.AlterField(

            model_name='arcgis',

            name='base_url',

            field=models.URLField(verbose_name='Webservice Base URL'),

        ),

        migrations.AlterField(

            model_name='arcgis',

            name='log_level',

            field=models.CharField(choices=[(b'NOTSET', b'NOTSET'), (b'DEBUG', b'DEBUG'), (b'INFO', b'INFO'), (b'WARNING', b'WARNING'), (b'ERROR', b'ERROR'), (b'CRITICAL', b'CRITICAL')], default=b'INFO', max_length=10, verbose_name='Log Level'),

        ),

    ]

# passerelle - uniform access to multiple data sources and services

# Copyright (C) 2018 Entr'ouvert

import urlparse

from django.template import Template, Context

from passerelle.utils.api import endpoint

from passerelle.base.models import BaseResource, HTTPResource

class ArcGISError(APIError):

    pass

class ArcGIS(BaseResource, HTTPResource):

    base_url = models.URLField(_('Webservice Base URL'))

        verbose_name = _('ArcGIS REST API')

    @endpoint(name='mapservice-query',

              description=_('Map Service Query'),

              perm='can_access',

              parameters={

                  'folder': {

                      'description': _('Folder name'),

                      'example_value': 'Specialty',

                  },

                  'service': {

                      'description': _('Service name'),

                      'example_value': 'ESRI_StateCityHighway_USA',

                  },

                  'layer': {

                      'description': _('Layer or table name'),

                      'example_value': '1',

                  },

                  'lat': {'description': _('Latitude')},

                  'lon': {'description': _('Longitude')},

                  'latmin': {'description': _('Minimal latitude (envelope)')},

                  'lonmin': {'description': _('Minimal longitude (envelope)')},

                  'latmax': {'description': _('Minimal atitude (envelope)')},

                  'lonmax': {'description': _('Minimal Longitude (envelope)')},

                  'q': {'description': _('Search text in display field')},

                  'template': {

                      'description': _('Django template for text attribute'),

                      'example_value': '{{ attributes.STATE_NAME }} ({{ attributes.STATE_ABBR }})',

                  },

                  'id_template': {

                      'description': _('Django template for id attribute'),

                  },

                  'full': {

                      'description': _('Returns all ArcGIS informations (geometry, metadata)'),

                      'type': 'bool',

                  },

              })

    def mapservice_query(self, request, service, layer='0', folder='', lat=None, lon=None,

                         latmin=None, lonmin=None, latmax=None, lonmax=None, q=None,

                         template=None, id_template=None, full=False, **kwargs):

        url = urlparse.urljoin(self.base_url, 'services/')

        if folder:

            url = urlparse.urljoin(url, folder + '/')

        url = urlparse.urljoin(url, service + '/MapServer/' + layer + '/query')

        # build query params

        # cf https://developers.arcgis.com/rest/services-reference/query-map-service-layer-.htm

            'f': 'json',

            'outFields': '*',

        if lat and lon:

            try:

                lon, lat = float(lon), float(lat)

            except (ValueError,):

                raise APIError('<lon> and <lat> must be floats', http_status=400)

            params['geometry'] = '{},{}'.format(lon, lat)

            params['geometryType'] = 'esriGeometryPoint'

        elif latmin and lonmin and latmax and lonmax:

            try:

                lonmin, latmin = float(lonmin), float(latmin)

                lonmax, latmax = float(lonmax), float(latmax)

            except (ValueError,):

                raise APIError('<lonmin> <latmin> <lonmax> and <latmax> must be floats',

                               http_status=400)

            params['geometry'] = '{},{},{},{}'.format(lonmin, latmin, lonmax, latmax)

            params['geometryType'] = 'esriGeometryEnvelope'

        if q is not None:

            params['text'] = q

        # consider all remaining parameters as ArcGIS ones

        params.update(kwargs)

        if 'where' not in params and 'text' not in params:

            params['where'] = '1=1'

        if 'distance' in params and 'units' not in params:

            params['units'] = 'esriSRUnit_Meter'

        # errors

        if response.status_code // 100 != 2:

            raise ArcGISError('ArcGIS returned status code %s' % response.status_code)

        try:

            infos = response.json()

        except (ValueError,):

            raise ArcGISError('ArcGIS returned invalid JSON content: %r' % response.content)

        if 'error' in infos:

            err_desc = infos['error'].get('message') or 'unknown ArcGIS error'

            raise ArcGISError(err_desc, data=infos)

        features = infos.pop('features', [])

        id_fieldname = infos.get('objectIdFieldName') or 'OBJECTID'

        text_fieldname = infos.get('displayFieldName')

        if infos.get('fieldAliases'):

            aliases = {v: k for k, v in infos['fieldAliases'].items()}

        else:

            aliases = {}

        # data is the features list, with 'id' and 'text' entries

        data = []

        def get_feature_attribute(feature, attribute):

            if attribute in feature['attributes']:

                return feature['attributes'][attribute]

            return feature['attributes'].get(aliases.get(attribute))

        if template:

            template = Template(template)

        if id_template:

            id_template = Template(id_template)

        for n, feature in enumerate(features):

            if 'attributes' in feature:

                feature['id'] = '%s' % get_feature_attribute(feature, id_fieldname)

                feature['text'] = '%s' % get_feature_attribute(feature, text_fieldname)

            else:

                feature['id'] = feature['text'] = '%d' % (n+1)

            if template:

                feature['text'] = template.render(Context(feature))

            if id_template:

                feature['id'] = id_template.render(Context(feature))

            if not full and 'geometry' in feature:

                del feature['geometry']

            data.append(feature)

        if full:

            return {'data': data, 'metadata': infos}

    @endpoint(name='district',

              description=_('Districts in Nancy Town (deprecated)'),

              parameters={

                  'lat': {'description': _('Latitude')},

                  'lon': {'description': _('Longitude')},

              })

    def district(self, request, lon=None, lat=None):

        if 'NANCY_Grc' in self.base_url:

            # Nancy URL used to contains folder, service and layer, remove them

            self.base_url = 'https://geoservices.grand-nancy.org/arcgis/rest/'

        features = self.mapservice_query(request, folder='public', service='NANCY_Grc', layer='0',

                                         template='{{ attributes.NOM }}',

                                         id_template='{{ attributes.NUMERO }}',

                                         lon=lon, lat=lat)['data']

        if not features:

            raise APIError('No features found.')

        for feature in features:

            del feature['attributes']

        feature['id'] = int(feature['id'])

        if len(features) == 1:

            return {'data': features[0]}

        return {'data': features}

# -*- coding: utf-8 -*-

import utils

from passerelle.contrib.arcgis.models import ArcGIS

# from http://sampleserver1.arcgisonline.com/ArcGIS/rest/services/fold/serv/MapServer/1

STATES = '''{

   "fieldAliases" : {

      "OBJECTID" : "OBJECTID",

      "STATE_NAME" : "STATE_NAME",

      "STATE_ABBR" : "STATE_ABBR"

   },

   "features" : [

      {

         "attributes" : {

            "STATE_NAME" : "Texas",

            "STATE_ABBR" : "TX",

            "OBJECTID" : 40

         },

         "geometry" : {

            "rings" : [

               [

                  [-105.998886788462, 31.3939400524361],

                  [-106.21328556164, 31.4782464373727]

               ]

            ]

         }

      },

      {

         "geometry" : {

            "rings" : [

               [

                  [-111.475425113078, 44.7021622250113],

                  [-111.480804007084, 44.6914159859524]

               ]

            ]

         },

         "attributes" : {

            "STATE_NAME" : "Montana",

            "STATE_ABBR" : "MT",

            "OBJECTID" : 2

         }

      }

   ],

   "spatialReference" : {

      "wkid" : 4326

   },

   "fields" : [

      {

         "alias" : "OBJECTID",

         "type" : "esriFieldTypeOID",

         "name" : "OBJECTID"

      },

      {

         "type" : "esriFieldTypeString",

         "alias" : "STATE_NAME",

         "length" : 25,

         "name" : "STATE_NAME"

      },

      {

         "length" : 2,

         "alias" : "STATE_ABBR",

         "type" : "esriFieldTypeString",

         "name" : "STATE_ABBR"

      }

   ],

   "geometryType" : "esriGeometryPolygon",

   "displayFieldName" : "STATE_NAME"

}'''

def arcgis(db):

    return ArcGIS.objects.create(slug='test',

                                 base_url='https://arcgis.example.net/')

def test_arcgis_mapservice_query(app, arcgis):

    endpoint = utils.generic_endpoint_url('arcgis', 'mapservice-query', slug=arcgis.slug)

    assert endpoint == '/arcgis/test/mapservice-query'

    params = {

        'folder': 'fold',

        'service': 'serv',

        'layer': '1'

    }

    with mock.patch('passerelle.utils.Request.get') as requests_get:

        requests_get.return_value = utils.FakedResponse(content=STATES,

                                                        status_code=200)

        resp = app.get(endpoint, params=params, status=403)

        assert requests_get.call_count == 0

        assert resp.json['err'] == 1

        assert resp.json['err_class'] == 'django.core.exceptions.PermissionDenied'

        # open access

        api = ApiUser.objects.create(username='all', keytype='', key='')

        obj_type = ContentType.objects.get_for_model(arcgis)

        AccessRight.objects.create(codename='can_access', apiuser=api, resource_type=obj_type,

                                   resource_pk=arcgis.pk)

        resp = app.get(endpoint, params=params, status=200)

        assert requests_get.call_count == 1

        assert requests_get.call_args[0][0] == 'https://arcgis.example.net/services/fold/serv/MapServer/1/query'

        args = requests_get.call_args[1]['params']

        assert args['f'] == 'json'

        assert args['outFields'] == '*'

        assert args['where'] == '1=1'

        assert 'data' in resp.json

        assert resp.json['err'] == 0

        assert len(resp.json['data']) == 2

        assert resp.json['data'][0]['id'] == '40'

        assert resp.json['data'][0]['text'] == 'Texas'

        assert 'geometry' not in resp.json['data'][0]

        assert 'metadata' not in resp.json

        params['full'] = 'on'

        resp = app.get(endpoint, params=params, status=200)

        assert requests_get.call_count == 2

        assert requests_get.call_args[0][0] == 'https://arcgis.example.net/services/fold/serv/MapServer/1/query'

        args = requests_get.call_args[1]['params']

        assert args['f'] == 'json'

        assert args['outFields'] == '*'

        assert args['where'] == '1=1'

        assert 'data' in resp.json

        assert resp.json['err'] == 0

        assert len(resp.json['data']) == 2

        assert resp.json['data'][0]['id'] == '40'

        assert resp.json['data'][0]['text'] == 'Texas'

        assert resp.json['data'][0]['geometry']

        assert resp.json['metadata']

        params['q'] = 'Texas'

        resp = app.get(endpoint, params=params, status=200)

        assert requests_get.call_count == 3

        assert requests_get.call_args[0][0] == 'https://arcgis.example.net/services/fold/serv/MapServer/1/query'

        args = requests_get.call_args[1]['params']

        assert args['text'] == 'Texas'

        assert 'where' not in args

        params['lat'] = '9.87654'

        params['lon'] = '1.12345'

        resp = app.get(endpoint, params=params, status=200)

        assert requests_get.call_count == 4

        assert requests_get.call_args[0][0] == 'https://arcgis.example.net/services/fold/serv/MapServer/1/query'

        args = requests_get.call_args[1]['params']

        assert args['geometry'] == '1.12345,9.87654'

        assert args['geometryType'] == 'esriGeometryPoint'

        del params['lat']  # missing lat, do not search by geometry

        resp = app.get(endpoint, params=params, status=200)

        assert requests_get.call_count == 5

        assert requests_get.call_args[0][0] == 'https://arcgis.example.net/services/fold/serv/MapServer/1/query'

        args = requests_get.call_args[1]['params']

        assert 'geometry' not in args

        assert 'geometryType' not in args

        params.update({'latmin': '1', 'lonmin': '2', 'latmax': '3', 'lonmax': '4'})

        resp = app.get(endpoint, params=params, status=200)

        assert requests_get.call_count == 6

        assert requests_get.call_args[0][0] == 'https://arcgis.example.net/services/fold/serv/MapServer/1/query'

        args = requests_get.call_args[1]['params']

        assert args['geometry'] == '2.0,1.0,4.0,3.0'

        assert args['geometryType'] == 'esriGeometryEnvelope'

        del params['latmin']  # incomplete box, do not search by geometry

        resp = app.get(endpoint, params=params, status=200)

        assert requests_get.call_count == 7

        assert requests_get.call_args[0][0] == 'https://arcgis.example.net/services/fold/serv/MapServer/1/query'

        args = requests_get.call_args[1]['params']

        assert 'geometry' not in args

        assert 'geometryType' not in args

        # others params are directly sent to ArcGIS

        params['spatialRel'] = 'esriSpatialRelContains'

        params.update({'latmin': '1', 'lonmin': '2', 'latmax': '3', 'lonmax': '4'})

        resp = app.get(endpoint, params=params, status=200)

        assert requests_get.call_count == 8

        assert requests_get.call_args[0][0] == 'https://arcgis.example.net/services/fold/serv/MapServer/1/query'

        args = requests_get.call_args[1]['params']

        assert args['geometry'] == '2.0,1.0,4.0,3.0'

        assert args['geometryType'] == 'esriGeometryEnvelope'

        assert args['spatialRel'] == 'esriSpatialRelContains'

        # folder

        params['folder'] = 'foo/bar'

        resp = app.get(endpoint, params=params, status=200)

        assert requests_get.call_args[0][0] == 'https://arcgis.example.net/services/foo/bar/serv/MapServer/1/query'

        del params['folder']

        resp = app.get(endpoint, params=params, status=200)

        assert requests_get.call_args[0][0] == 'https://arcgis.example.net/services/serv/MapServer/1/query'

        # minimal call

        resp = app.get(endpoint, params={'service': 'srv'}, status=200)

        assert requests_get.call_args[0][0] == 'https://arcgis.example.net/services/srv/MapServer/0/query'

        args = requests_get.call_args[1]['params']

        assert args == {'f': 'json', 'inSR': '4326', 'outSR': '4326',

                        'outFields': '*', 'where': '1=1'}

        # distance

        resp = app.get(endpoint, params={'service': 'srv', 'distance': '100'}, status=200)

        assert requests_get.call_args[0][0] == 'https://arcgis.example.net/services/srv/MapServer/0/query'

        args = requests_get.call_args[1]['params']

        assert args['distance'] == '100'

        assert args['units'] == 'esriSRUnit_Meter'  # default unit

        resp = app.get(endpoint, params={'service': 'srv', 'distance': '5', 'units':

                                         'esriSRUnit_NauticalMile'}, status=200)

        assert requests_get.call_args[0][0] == 'https://arcgis.example.net/services/srv/MapServer/0/query'

        args = requests_get.call_args[1]['params']

        assert args['distance'] == '5'

        assert args['units'] == 'esriSRUnit_NauticalMile'

    # call errors

    with mock.patch('passerelle.utils.Request.get') as requests_get:

        requests_get.return_value = utils.FakedResponse(content=STATES,

                                                        status_code=200)

        resp = app.get(endpoint, params={}, status=400)

        assert requests_get.call_count == 0

        assert resp.json['err'] == 1

        assert resp.json['err_class'] == 'passerelle.views.WrongParameter'

        assert resp.json['err_desc'] == "missing parameters: 'service'."

        resp = app.get(endpoint, params={'service': 'src', 'lat': '0', 'lon': 'y'}, status=400)

        assert requests_get.call_count == 0

        assert resp.json['err'] == 1

        assert resp.json['err_class'] == 'passerelle.utils.jsonresponse.APIError'

        assert resp.json['err_desc'] == '<lon> and <lat> must be floats'

        resp = app.get(endpoint, params={'service': 'src', 'latmin': '0', 'lonmin': 'y',

                                         'latmax': '0', 'lonmax': '1'}, status=400)

        assert requests_get.call_count == 0

        assert resp.json['err'] == 1

        assert resp.json['err_class'] == 'passerelle.utils.jsonresponse.APIError'

        assert resp.json['err_desc'] == '<lonmin> <latmin> <lonmax> and <latmax> must be floats'

import os

import json

import pytest

import mock

from django.core.urlresolvers import reverse

from django.contrib.contenttypes.models import ContentType

from passerelle.base.models import ApiUser, AccessRight

from passerelle.contrib.arcgis.models import ArcGIS

TEST_BASE_DIR = os.path.join(os.path.dirname(__file__), 'data', 'nancy_arcgis')

def get_file_content(filename):

    with open(os.path.join(TEST_BASE_DIR, filename), 'rb') as fd:

        return fd.read()

class MockedRequestsResponse(mock.Mock):

    def json(self):

        return json.loads(self.content)

@pytest.fixture

def setup(db):

    api = ApiUser.objects.create(username='all',

                                 keytype='', key='')

    arcgis = ArcGIS.objects.create(base_url='https://example.net/layer/0', slug='test')

    obj_type = ContentType.objects.get_for_model(arcgis)

    AccessRight.objects.create(codename='can_access', apiuser=api,

                               resource_type=obj_type, resource_pk=arcgis.pk)

    return arcgis

@pytest.fixture

def url():

    return reverse('generic-endpoint', kwargs={

        'connector': 'arcgis', 'slug': 'test', 'endpoint': 'district'})

def test_get_district_parameters_error(app, setup, url):

    resp = app.get(url, params={'lon': 'lon', 'lat': 'lat'}, status=400)

    assert resp.json['err_desc'] == '<lon> and <lat> must be floats'

@mock.patch('passerelle.utils.Request.get')

def test_get_district(mocked_get, app, setup, url):

    mocked_get.return_value = MockedRequestsResponse(

        content=get_file_content('sigresponse.json'),

        status_code=200)

    resp = app.get(url, params={'lon': 6.172122, 'lat': 48.673836}, status=200)

    data = resp.json['data']

    assert data['id'] == 4

    assert data['text'] == 'HAUSSONVILLE / BLANDAN / MON DESERT / SAURUPT'

@mock.patch('passerelle.utils.Request.get')

def test_get_all_district(mocked_get, app, setup, url):

    mocked_get.return_value = MockedRequestsResponse(

        content=get_file_content('all_districts.json'),

        status_code=200)

    resp = app.get(url, status=200)

    data = resp.json['data']

    assert len(data) == 7

@mock.patch('passerelle.utils.Request.get')

def test_no_district(mocked_get, app, setup, url):

    mocked_get.return_value = MockedRequestsResponse(

        content='{"features": []}',

        status_code=200)

    resp = app.get(url, status=200)

    assert resp.json['err'] == 1

    assert resp.json['err_class'] == 'passerelle.utils.jsonresponse.APIError'

    assert resp.json['err_desc'] == 'No features found.'

from passerelle.contrib.arcgis.models import ArcGIS

    return utils.setup_access_rights(ArcGIS.objects.create(slug='test', log_level='DEBUG'))