From 461e0cb81741e9f5cccc461f1695689859994b47 Mon Sep 17 00:00:00 2001
From: Valentin Deniaud <vdeniaud@entrouvert.com>
Date: Mon, 25 Nov 2019 18:00:33 +0100
Subject: [PATCH 4/6] =?UTF-8?q?base=5Fadresse:=20add=20API=20G=C3=A9o=20en?=
 =?UTF-8?q?dpoints=20(#11497)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Namely /cities/, /departements/ and /regions/.
---
 .../migrations/0015_auto_20191206_1244.py     |  95 +++++
 passerelle/apps/base_adresse/models.py        | 258 +++++++++++++-
 .../base_adresse/baseadresse_detail.html      |   5 +
 tests/test_base_adresse.py                    | 334 +++++++++++++++++-
 4 files changed, 679 insertions(+), 13 deletions(-)
 create mode 100644 passerelle/apps/base_adresse/migrations/0015_auto_20191206_1244.py

diff --git a/passerelle/apps/base_adresse/migrations/0015_auto_20191206_1244.py b/passerelle/apps/base_adresse/migrations/0015_auto_20191206_1244.py
new file mode 100644
index 00000000..8061b976
--- /dev/null
+++ b/passerelle/apps/base_adresse/migrations/0015_auto_20191206_1244.py
@@ -0,0 +1,95 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.18 on 2019-12-06 11:44
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+import passerelle.apps.base_adresse.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('base_adresse', '0014_auto_20190207_0456'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='CityModel',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=150, verbose_name='City name')),
+                ('unaccent_name', models.CharField(max_length=150, null=True, verbose_name='City name ascii char')),
+                ('code', models.CharField(max_length=5, verbose_name='INSEE code')),
+                ('zipcode', models.CharField(max_length=5, verbose_name='Postal code')),
+                ('population', models.PositiveIntegerField(verbose_name='Population')),
+                ('last_update', models.DateTimeField(auto_now=True, null=True, verbose_name='Last update')),
+            ],
+            options={
+                'ordering': ['-population', 'zipcode', 'unaccent_name', 'name'],
+            },
+            bases=(passerelle.apps.base_adresse.models.UnaccentNameMixin, models.Model),
+        ),
+        migrations.CreateModel(
+            name='DepartmentModel',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=100, verbose_name='Department name')),
+                ('unaccent_name', models.CharField(max_length=150, null=True, verbose_name='Department name ascii char')),
+                ('code', models.CharField(max_length=3, unique=True, verbose_name='Department code')),
+                ('last_update', models.DateTimeField(auto_now=True, null=True, verbose_name='Last update')),
+            ],
+            options={
+                'ordering': ['code'],
+            },
+            bases=(passerelle.apps.base_adresse.models.UnaccentNameMixin, models.Model),
+        ),
+        migrations.CreateModel(
+            name='RegionModel',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=150, verbose_name='Region name')),
+                ('unaccent_name', models.CharField(max_length=150, null=True, verbose_name='Region name ascii char')),
+                ('code', models.CharField(max_length=2, unique=True, verbose_name='Region code')),
+                ('last_update', models.DateTimeField(auto_now=True, null=True, verbose_name='Last update')),
+            ],
+            options={
+                'ordering': ['code'],
+            },
+            bases=(passerelle.apps.base_adresse.models.UnaccentNameMixin, models.Model),
+        ),
+        migrations.AddField(
+            model_name='baseadresse',
+            name='api_geo_url',
+            field=models.CharField(default=b'https://geo.api.gouv.fr/', help_text='Base Adresse API Geo URL', max_length=128, verbose_name='API Geo URL'),
+        ),
+        migrations.AlterField(
+            model_name='baseadresse',
+            name='zipcode',
+            field=models.CharField(blank=True, max_length=600, verbose_name='Postal codes or department number to get streets, separated with commas'),
+        ),
+        migrations.AlterField(
+            model_name='streetmodel',
+            name='city',
+            field=models.CharField(max_length=150, verbose_name='City'),
+        ),
+        migrations.AddField(
+            model_name='departmentmodel',
+            name='region',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='base_adresse.RegionModel'),
+        ),
+        migrations.AddField(
+            model_name='citymodel',
+            name='department',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='base_adresse.DepartmentModel'),
+        ),
+        migrations.AddField(
+            model_name='citymodel',
+            name='region',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='base_adresse.RegionModel'),
+        ),
+        migrations.AlterUniqueTogether(
+            name='citymodel',
+            unique_together=set([('code', 'zipcode')]),
+        ),
+    ]
diff --git a/passerelle/apps/base_adresse/models.py b/passerelle/apps/base_adresse/models.py
index 30068f62..65454266 100644
--- a/passerelle/apps/base_adresse/models.py
+++ b/passerelle/apps/base_adresse/models.py
@@ -4,6 +4,7 @@ import os
 import urlparse
 import unicodedata
 
+from requests.exceptions import ConnectionError
 
 from django.db import connection, models
 from django.db.models import Q
@@ -15,6 +16,8 @@ from django.utils.six.moves.urllib.parse import urljoin
 
 from passerelle.base.models import BaseResource
 from passerelle.utils.api import endpoint
+from passerelle.utils.conversion import simplify
+from passerelle.utils.jsonresponse import APIError
 
 
 class BaseAdresse(BaseResource):
@@ -24,16 +27,23 @@ class BaseAdresse(BaseResource):
         verbose_name=_('Service URL'),
         help_text=_('Base Adresse Web Service URL'))
 
+    api_geo_url = models.CharField(
+        max_length=128, blank=False,
+        default='https://geo.api.gouv.fr/',
+        verbose_name=_('API Geo URL'),
+        help_text=_('Base Adresse API Geo URL'))
+
     category = _('Geographic information system')
 
-    api_description = _("The API is a partial view of OpenStreetMap's Nominatim "
-                        "own API; it currently doesn't support all parameters and "
-                        "is limited to the JSON format.")
+    api_description = _("The geocoding endpoints are a partial view of OpenStreetMap's "
+                        "Nominatim own API; it currently doesn't support all parameters and "
+                        "is limited to the JSON format. The cities, departments and regions "
+                        "endpoints source data from French API Geo.")
 
     zipcode = models.CharField(
         max_length=600,
         blank=True,
-        verbose_name=_('Postal codes or county number to get streets, separated with commas'))
+        verbose_name=_('Postal codes or department number to get streets, separated with commas'))
 
     class Meta:
         verbose_name = _('Base Adresse Web Service')
@@ -135,8 +145,7 @@ class BaseAdresse(BaseResource):
         else:
             streets = StreetModel.objects.all()
             if q:
-                unaccented_q = unicodedata.normalize('NFKD', q).encode('ascii', 'ignore').lower()
-                streets = streets.filter(unaccent_name__icontains=unaccented_q)
+                streets = streets.filter(unaccent_name__icontains=simplify(q))
 
         if zipcode:
             streets = streets.filter(zipcode__startswith=zipcode)
@@ -158,6 +167,82 @@ class BaseAdresse(BaseResource):
 
         return {'data': result}
 
+    @endpoint(description=_('Cities list'),
+              parameters={
+                  'id': {'description': _('Get exactly one city using its code and postal code '
+                                          'separated with a dot'),
+                         'example_value': '75056.75014'},
+                  'q': {'description': _("Search text in name or postal code"),
+                        'example_value': 'Paris'},
+                  'code': {'description': _('INSEE code'), 'example_value': '75056'},
+                  'region_code': {'description': _('Region code'), 'example_value': '11'},
+                  'department_code': {'description': _('Department code'), 'example_value': '75'},
+              })
+    def cities(self, request, id=None, q=None, code=None, region_code=None,
+               department_code=None):
+        cities = CityModel.objects.all()
+
+        if id is not None:
+            try:
+                code, zipcode = id.split('.')
+            except ValueError as e:
+                raise APIError('Invalid id')
+            cities = cities.filter(code=code, zipcode=zipcode)
+        if q:
+            unaccented_q = simplify(q)
+            cities = cities.filter(Q(unaccent_name__istartswith=unaccented_q) |
+                                   Q(zipcode__istartswith=unaccented_q))
+        if code:
+            cities = cities.filter(code=code)
+        if region_code:
+            cities = cities.filter(region__code=region_code)
+        if department_code:
+            cities = cities.filter(department__code=department_code)
+
+        cities = cities.select_related('department', 'region')
+        return {'data': [city.to_json() for city in cities]}
+
+    @endpoint(description=_('Departments list'),
+              parameters={
+                  'id': {'description': _('Get exactly one department using its code'),
+                         'example_value': '59'},
+                  'q': {'description': _('Search text in name or code'), 'example_value': 'Nord'},
+                  'region_code': {'description': _('Region code'), 'example_value': '32'},
+              })
+    def departments(self, request, id=None, q=None, region_code=None):
+        departments = DepartmentModel.objects.all()
+
+        if id is not None:
+            departments = departments.filter(code=id)
+        if q:
+            unaccented_q = simplify(q)
+            departments = departments.filter(Q(unaccent_name__istartswith=unaccented_q) |
+                                             Q(code__istartswith=unaccented_q))
+        if region_code:
+            departments = departments.filter(region__code=region_code)
+
+        departments = departments.select_related('region')
+        return {'data': [department.to_json() for department in departments]}
+
+    @endpoint(description=_('Regions list'),
+              parameters={
+                  'id': {'description': _('Get exactly one region using its code'),
+                         'example_value': '32'},
+                  'q': {'description': _('Search text in name or code'),
+                        'example_value': 'Hauts-de-France'},
+              })
+    def regions(self, request, id=None, q=None):
+        regions = RegionModel.objects.all()
+
+        if id is not None:
+            regions = regions.filter(code=id)
+        if q:
+            unaccented_q = simplify(q)
+            regions = regions.filter(Q(unaccent_name__istartswith=unaccented_q) |
+                                     Q(code__istartswith=unaccented_q))
+
+        return {'data': [region.to_json() for region in regions]}
+
     def check_status(self):
         if self.service_url == 'https://api-adresse.data.gouv.fr/':
             result = self.search(None, '169 rue du chateau, paris')
@@ -174,6 +259,9 @@ class BaseAdresse(BaseResource):
             criteria |= Q(zipcode__startswith=zipcode)
         return StreetModel.objects.filter(criteria)
 
+    def cities_exist(self):
+        return CityModel.objects.exists()
+
     def update_streets_data(self):
         if not self.get_zipcodes():
             return
@@ -218,20 +306,89 @@ class BaseAdresse(BaseResource):
 
         self.get_streets_queryset().filter(last_update__lt=start_update).delete()
 
+    def get_api_geo_endpoint(self, endpoint):
+        error = None
+        try:
+            response = self.requests.get(urljoin(self.api_geo_url, endpoint))
+        except ConnectionError as e:
+            error = e
+        if response.status_code == 200:
+            try:
+                result = response.json()
+            except ValueError:
+                error = 'invalid json, got: %s' % response.text
+            else:
+                if not result:
+                    raise Exception('api geo returns empty json')
+        else:
+            error = 'bad status code (%s)' % response.status_code
+        if error:
+            self.logger.error('failed to update api geo data for endpoint %s: %s',
+                              endpoint, error)
+            return
+        return result
+
+    def update_api_geo_data(self):
+        regions_json = self.get_api_geo_endpoint('regions')
+        departments_json = self.get_api_geo_endpoint('departements')
+        cities_json = self.get_api_geo_endpoint('communes')
+        if not (regions_json and departments_json and cities_json):
+            return
+        start_update = timezone.now()
+
+        for data in regions_json:
+            defaults = {
+                'name': data['nom'],
+            }
+            RegionModel.objects.update_or_create(code=data['code'], defaults=defaults)
+        RegionModel.objects.filter(last_update__lt=start_update).delete()
+
+        for data in departments_json:
+            defaults = {
+                'name': data['nom'],
+                'region': RegionModel.objects.get(code=data['codeRegion']),
+            }
+            DepartmentModel.objects.update_or_create(code=data['code'], defaults=defaults)
+        DepartmentModel.objects.filter(last_update__lt=start_update).delete()
+
+        for data in cities_json:
+            for zipcode in data['codesPostaux']:
+                defaults = {
+                    'name': data['nom'],
+                    'population': data.get('population', 0),
+                }
+                if data.get('codeDepartement'):
+                    defaults['department'] = DepartmentModel.objects.get(code=data['codeDepartement'])
+                if data.get('codeRegion'):
+                    defaults['region'] = RegionModel.objects.get(code=data['codeRegion'])
+                CityModel.objects.update_or_create(
+                    code=data['code'], zipcode=zipcode, defaults=defaults)
+        CityModel.objects.filter(last_update__lt=start_update).delete()
+
     def hourly(self):
         super(BaseAdresse, self).hourly()
+        # don't wait for daily job to grab data
         if self.get_zipcodes() and not self.get_streets_queryset().exists():
-            # don't wait for daily job to grab streets
             self.update_streets_data()
+        if not CityModel.objects.exists():
+            self.update_api_geo_data()
 
     def daily(self):
         super(BaseAdresse, self).daily()
         self.update_streets_data()
+        self.update_api_geo_data()
 
 
-class StreetModel(models.Model):
+class UnaccentNameMixin(object):
 
-    city = models.CharField(_('City'), max_length=100)
+    def save(self, *args, **kwargs):
+        self.unaccent_name = unicodedata.normalize('NFKD', self.name).encode('ascii', 'ignore').lower()
+        super(UnaccentNameMixin, self).save(*args, **kwargs)
+
+
+class StreetModel(UnaccentNameMixin, models.Model):
+
+    city = models.CharField(_('City'), max_length=150)
     name = models.CharField(_('Street name'), max_length=150)
     unaccent_name = models.CharField(_('Street name ascii char'), max_length=150, null=True)
     zipcode = models.CharField(_('Postal code'), max_length=5)
@@ -245,6 +402,83 @@ class StreetModel(models.Model):
     def __unicode__(self):
         return self.name
 
-    def save(self, *args, **kwargs):
-        self.unaccent_name = unicodedata.normalize('NFKD', self.name).encode('ascii', 'ignore')
-        super(StreetModel, self).save(*args, **kwargs)
+
+@six.python_2_unicode_compatible
+class RegionModel(UnaccentNameMixin, models.Model):
+
+    name = models.CharField(_('Region name'), max_length=150)
+    unaccent_name = models.CharField(_('Region name ascii char'), max_length=150, null=True)
+    code = models.CharField(_('Region code'), max_length=2, unique=True)
+    last_update = models.DateTimeField(_('Last update'), null=True, auto_now=True)
+
+    def to_json(self):
+        return {
+            'text': str(self),
+            'id': self.code,
+            'code': self.code,
+            'name': self.name,
+        }
+
+    class Meta:
+        ordering = ['code']
+
+    def __str__(self):
+        return '%s %s' % (self.code, self.name)
+
+
+@six.python_2_unicode_compatible
+class DepartmentModel(UnaccentNameMixin, models.Model):
+
+    name = models.CharField(_('Department name'), max_length=100)
+    unaccent_name = models.CharField(_('Department name ascii char'), max_length=150, null=True)
+    code = models.CharField(_('Department code'), max_length=3, unique=True)
+    region = models.ForeignKey(RegionModel, on_delete=models.CASCADE)
+    last_update = models.DateTimeField(_('Last update'), null=True, auto_now=True)
+
+    def to_json(self):
+        return {
+            'text': str(self),
+            'id': self.code,
+            'code': self.code,
+            'name': self.name,
+            'region_code': self.region.code,
+        }
+
+    class Meta:
+        ordering = ['code']
+
+    def __str__(self):
+        return '%s %s' % (self.code, self.name)
+
+
+@six.python_2_unicode_compatible
+class CityModel(UnaccentNameMixin, models.Model):
+
+    name = models.CharField(_('City name'), max_length=150)
+    unaccent_name = models.CharField(_('City name ascii char'), max_length=150, null=True)
+    code = models.CharField(_('INSEE code'), max_length=5)
+    zipcode = models.CharField(_('Postal code'), max_length=5)
+    population = models.PositiveIntegerField(_('Population'))
+    department = models.ForeignKey(DepartmentModel, on_delete=models.CASCADE, blank=True, null=True)
+    region = models.ForeignKey(RegionModel, on_delete=models.CASCADE, blank=True, null=True)
+    last_update = models.DateTimeField(_('Last update'), null=True, auto_now=True)
+
+    def to_json(self):
+        data = {
+            'text': str(self),
+            'id': '%s.%s' % (self.code, self.zipcode),
+            'code': self.code,
+            'name': self.name,
+            'zipcode': self.zipcode,
+            'population': self.population,
+            'department_code': self.department.code if self.department else None,
+            'region_code': self.region.code if self.region else None,
+        }
+        return data
+
+    class Meta:
+        ordering = ['-population', 'zipcode', 'unaccent_name', 'name']
+        unique_together = ('code', 'zipcode')
+
+    def __str__(self):
+        return '%s %s' % (self.zipcode, self.name)
diff --git a/passerelle/apps/base_adresse/templates/base_adresse/baseadresse_detail.html b/passerelle/apps/base_adresse/templates/base_adresse/baseadresse_detail.html
index 1a36bfda..a2e63c23 100644
--- a/passerelle/apps/base_adresse/templates/base_adresse/baseadresse_detail.html
+++ b/passerelle/apps/base_adresse/templates/base_adresse/baseadresse_detail.html
@@ -8,6 +8,11 @@
 {% trans "Street data is not available yet, it should soon be downloaded." %}
 </div>
 {% endif %}
+{% if not object.cities_exist %}
+<div class="infonotice">
+{% trans "API Géo data is not available yet, it should soon be downloaded." %}
+</div>
+{% endif %}
 {% endblock %}
 
 {% block security %}
diff --git a/tests/test_base_adresse.py b/tests/test_base_adresse.py
index 0deced34..56800fc5 100644
--- a/tests/test_base_adresse.py
+++ b/tests/test_base_adresse.py
@@ -6,9 +6,13 @@ import mock
 import utils
 import json
 
+from six.moves.urllib.parse import urljoin
+
 from django.core.management import call_command
+from django.core.management.base import CommandError
 
-from passerelle.apps.base_adresse.models import BaseAdresse, StreetModel
+from passerelle.apps.base_adresse.models import (BaseAdresse, StreetModel, CityModel,
+                                                 DepartmentModel, RegionModel)
 
 FAKED_CONTENT = json.dumps({
     "limit": 1,
@@ -41,6 +45,54 @@ FAKED_CONTENT = json.dumps({
 
 FAKE_DATA = ''
 
+FAKE_API_GEO_LIST = [
+    {
+        "code": "75056",
+        "codeDepartement": "75",
+        "codeRegion": "11",
+        "codesPostaux": [
+            "75001",
+            "75002",
+        ],
+        "nom": "Paris",
+        "population": 2190327,
+    },
+    {
+        "code": "97501",
+        "codesPostaux": [
+            "97500"
+        ],
+        "nom": "Miquelon-Langlade",
+        "population": 596
+    }
+]
+
+FAKE_API_GEO = json.dumps(FAKE_API_GEO_LIST)
+
+FAKE_API_GEO_DEPARTMENTS = json.dumps([
+    {
+        "code": "75",
+        "codeRegion": "11",
+        "nom": "Paris"
+    },
+    {
+        "code": "58",
+        "codeRegion": "27",
+        "nom": "Nièvre",
+    }
+])
+
+FAKE_API_GEO_REGIONS = json.dumps([
+    {
+        "code": "11",
+        "nom": "Île-de-France"
+    },
+    {
+        "code": "27",
+        "nom": "Bourgogne-Franche-Comté"
+    }
+])
+
 
 @pytest.fixture
 def base_adresse(db):
@@ -74,6 +126,42 @@ def street(db):
                                       citycode=u'73001')
 
 
+@pytest.fixture
+def region(db):
+    return RegionModel.objects.create(name=u'Auvergne-Rhône-Alpes', code='84')
+
+
+@pytest.fixture
+def department(db, region):
+    return DepartmentModel.objects.create(name=u'Savoie', code='73', region=region)
+
+
+@pytest.fixture
+def city(db, region, department):
+    return CityModel.objects.create(name=u'Chambéry', code='73065', zipcode='73000',
+                                    population=42000, region=region, department=department)
+
+
+@pytest.fixture
+def miquelon(db):
+    return CityModel.objects.create(name=u'Miquelon-Langlade', code='97501', zipcode='97500',
+                                    population=42)
+
+
+@pytest.fixture
+def mock_update_api_geo():
+    with mock.patch('passerelle.apps.base_adresse.models.BaseAdresse.update_api_geo_data',
+                    new=lambda x: None) as _fixture:
+        yield _fixture
+
+
+@pytest.fixture
+def mock_update_streets():
+    with mock.patch('passerelle.apps.base_adresse.models.BaseAdresse.update_streets_data',
+                    new=lambda x: None) as _fixture:
+        yield _fixture
+
+
 @mock.patch('passerelle.utils.Request.get')
 def test_base_adresse_search(mocked_get, app, base_adresse):
     endpoint = utils.generic_endpoint_url('base-adresse', 'search', slug=base_adresse.slug)
@@ -195,6 +283,7 @@ def test_base_adresse_streets_get_by_id(app, base_adresse, street):
     assert len(resp.json['data']) == 0
 
 
+@pytest.mark.usefixtures('mock_update_api_geo')
 @mock.patch('passerelle.utils.Request.get')
 def test_base_adresse_command_update(mocked_get, db, base_adresse):
     filepath = os.path.join(os.path.dirname(__file__), 'data', 'update_streets_test.bz2')
@@ -214,8 +303,10 @@ def test_base_adresse_command_update(mocked_get, db, base_adresse):
     assert mocked_get.call_count == 2
 
 
+@pytest.mark.usefixtures('mock_update_api_geo')
 @mock.patch('passerelle.utils.Request.get')
 def test_base_adresse_command_hourly_update(mocked_get, db, base_adresse):
+    base_adresse.update_api_geo_data = lambda: None
     filepath = os.path.join(os.path.dirname(__file__), 'data', 'update_streets_test.bz2')
     mocked_get.return_value = utils.FakedResponse(content=open(filepath).read(), status_code=200)
     # check the first hourly job downloads streets
@@ -227,8 +318,10 @@ def test_base_adresse_command_hourly_update(mocked_get, db, base_adresse):
     assert mocked_get.call_count == 1
 
 
+@pytest.mark.usefixtures('mock_update_api_geo')
 @mock.patch('passerelle.utils.Request.get')
 def test_base_adresse_command_update_97x(mocked_get, db, base_adresse_97x):
+    base_adresse_97x.update_api_geo_data = lambda: None
     filepath = os.path.join(os.path.dirname(__file__), 'data', 'update_streets_test.bz2')
     mocked_get.return_value = utils.FakedResponse(content=open(filepath).read(), status_code=200)
     call_command('cron', 'daily')
@@ -236,8 +329,10 @@ def test_base_adresse_command_update_97x(mocked_get, db, base_adresse_97x):
     assert StreetModel.objects.count() == 2
 
 
+@pytest.mark.usefixtures('mock_update_api_geo')
 @mock.patch('passerelle.utils.Request.get')
 def test_base_adresse_command_update_corsica(mocked_get, db, base_adresse_corsica):
+    base_adresse_corsica.update_api_geo_data = lambda: None
     filepath = os.path.join(os.path.dirname(__file__), 'data', 'update_streets_test.bz2')
     mocked_get.return_value = utils.FakedResponse(content=open(filepath).read(), status_code=200)
     call_command('cron', 'daily')
@@ -247,8 +342,10 @@ def test_base_adresse_command_update_corsica(mocked_get, db, base_adresse_corsic
     assert StreetModel.objects.count() == 0
 
 
+@pytest.mark.usefixtures('mock_update_api_geo')
 @mock.patch('passerelle.utils.Request.get')
 def test_base_adresse_command_update_multiple(mocked_get, db, base_adresse_multiple):
+    base_adresse_multiple.update_api_geo_data = lambda: None
     filepath = os.path.join(os.path.dirname(__file__), 'data', 'update_streets_test.bz2')
     mocked_get.return_value = utils.FakedResponse(content=open(filepath).read(), status_code=200)
     call_command('cron', 'daily')
@@ -258,3 +355,238 @@ def test_base_adresse_command_update_multiple(mocked_get, db, base_adresse_multi
     mocked_get.assert_any_call('http://bano.openstreetmap.fr/BAN_odbl/BAN_odbl_2A-json.bz2')
     mocked_get.assert_any_call('http://bano.openstreetmap.fr/BAN_odbl/BAN_odbl_2B-json.bz2')
     assert StreetModel.objects.count() == 5
+
+
+def test_base_adresse_cities(app, base_adresse, city, department, region):
+    resp = app.get('/base-adresse/%s/cities?q=chambe' % base_adresse.slug)
+    result = resp.json['data'][0]
+    assert result['name'] == city.name
+    assert result['text'] == '%s %s' % (city.zipcode, city.name)
+    assert result['code'] == city.code
+    assert result['zipcode'] == city.zipcode
+    assert result['id'] == '%s.%s' % (city.code, city.zipcode)
+    assert result['population'] == city.population
+    assert result['region_code'] == city.region.code
+    assert result['department_code'] == city.department.code
+
+    resp = app.get('/base-adresse/%s/cities?q=73' % base_adresse.slug)
+    assert resp.json['data'][0] == result
+
+    resp = app.get('/base-adresse/%s/cities?code=73065' % base_adresse.slug)
+    assert resp.json['data'][0] == result
+
+
+def test_base_adresse_cities_missing_region_and_department(app, base_adresse, miquelon):
+    resp = app.get('/base-adresse/%s/cities?q=miqu' % base_adresse.slug)
+    result = resp.json['data'][0]
+    assert result['name'] == miquelon.name
+    assert not result['department_code']
+    assert not result['region_code']
+
+
+def test_base_adresse_cities_region_department(app, base_adresse, miquelon, city):
+    reg = RegionModel.objects.create(name=u'IdF', code='11')
+    dep = DepartmentModel.objects.create(name=u'Paris', code='75', region=reg)
+    paris = CityModel.objects.create(name=u'Paris', code='75056', zipcode='75014',
+                                        population=2000000, region=reg, department=dep)
+
+    resp = app.get('/base-adresse/%s/cities?department_code=73' % base_adresse.slug)
+    result = resp.json['data']
+    assert len(result) == 1
+    assert result[0]['name'] == city.name
+
+    resp = app.get('/base-adresse/%s/cities?region_code=84' % base_adresse.slug)
+    result = resp.json['data']
+    assert len(result) == 1
+    assert result[0]['name'] == city.name
+
+    resp = app.get('/base-adresse/%s/cities?region_code=84&department_code=75' % base_adresse.slug)
+    result = resp.json['data']
+    assert not result
+
+
+def test_base_adresse_cities_sort_order(app, base_adresse, miquelon, city):
+    assert miquelon.population < city.population
+    resp = app.get('/base-adresse/%s/cities' % base_adresse.slug)
+    result = resp.json['data']
+    assert result[0]['name'] == city.name
+    assert result[1]['name'] == miquelon.name
+
+
+def test_base_adresse_cities_get_by_id(app, base_adresse, city):
+    for i in range(1, 10):
+        # create additional cities
+        city.pk = None
+        city.zipcode = int(city.zipcode) + i
+        city.save()
+
+    resp = app.get('/base-adresse/%s/cities?q=cham' % base_adresse.slug)
+    result = resp.json['data'][0]
+    assert len(resp.json['data']) == 10
+    city_id = result['id']
+
+    resp = app.get('/base-adresse/%s/cities?id=%s' % (base_adresse.slug, city_id))
+    assert len(resp.json['data']) == 1
+    result2 = resp.json['data'][0]
+    assert result2['text'] == result['text']
+
+    # non integer id.
+    resp = app.get('/base-adresse/%s/cities?id=%s' % (base_adresse.slug, 'XXX'))
+    assert resp.json['err'] == 1
+
+    # integer but without match.
+    resp = app.get('/base-adresse/%s/cities?id=%s' % (base_adresse.slug, '1.1'))
+    assert len(resp.json['data']) == 0
+
+
+def test_base_adresse_departments(app, base_adresse, department, region):
+    resp = app.get('/base-adresse/%s/departments?q=sav' % base_adresse.slug)
+    result = resp.json['data'][0]
+    assert result['name'] == department.name
+    assert result['code'] == department.code
+    assert result['id'] == department.code
+    assert result['text'] == '%s %s' % (department.code, department.name)
+    assert result['region_code'] == region.code
+
+    resp = app.get('/base-adresse/%s/departments?q=73' % base_adresse.slug)
+    result = resp.json['data'][0]
+    assert result['name'] == department.name
+
+    resp = app.get('/base-adresse/%s/departments?id=%s' % (base_adresse.slug, department.code))
+    result = resp.json['data'][0]
+    assert result['name'] == department.name
+
+
+def test_base_adresse_departments_region(app, base_adresse, department):
+    reg = RegionModel.objects.create(name=u'IdF', code='11')
+    paris = DepartmentModel.objects.create(name=u'Paris', code='75', region=reg)
+
+    resp = app.get('/base-adresse/%s/departments?region_code=84' % base_adresse.slug)
+    result = resp.json['data']
+    assert len(result) == 1
+    assert result[0]['name'] == department.name
+
+
+def test_base_adresse_regions(app, base_adresse, region):
+    resp = app.get('/base-adresse/%s/regions?q=au' % base_adresse.slug)
+    result = resp.json['data'][0]
+    assert result['name'] == region.name
+    assert result['code'] == region.code
+    assert result['id'] == region.code
+    assert result['text'] == '%s %s' % (region.code, region.name)
+
+    resp = app.get('/base-adresse/%s/regions?id=%s' % (base_adresse.slug, region.code))
+    result = resp.json['data'][0]
+    assert result['name'] == region.name
+
+
+@pytest.mark.usefixtures('mock_update_streets')
+@mock.patch('passerelle.utils.Request.get')
+def test_base_adresse_command_update_geo(mocked_get, db, base_adresse):
+    return_values = [utils.FakedResponse(content=content, status_code=200)
+                     for content in (FAKE_API_GEO_REGIONS, FAKE_API_GEO_DEPARTMENTS, FAKE_API_GEO)]
+    mocked_get.side_effect = return_values
+    call_command('cron', 'daily')
+    assert mocked_get.call_count == 3
+    mocked_get.assert_any_call(urljoin(base_adresse.api_geo_url, 'communes'))
+    mocked_get.assert_any_call(urljoin(base_adresse.api_geo_url, 'regions'))
+    mocked_get.assert_any_call(urljoin(base_adresse.api_geo_url, 'departements'))
+
+    regions = RegionModel.objects.all()
+    assert regions.count() == 2
+    idf = regions.get(name='Île-de-France')
+    assert idf.code == '11'
+    centre = regions.get(name='Bourgogne-Franche-Comté')
+    assert centre.code == '27'
+
+    departments = DepartmentModel.objects.all()
+    assert departments.count() == 2
+    paris_dep = departments.get(name='Paris')
+    assert paris_dep.code == '75'
+    assert paris_dep.region == idf
+    nievre = departments.get(name='Nièvre')
+    assert nievre.code == '58'
+    assert nievre.region == centre
+
+    cities = CityModel.objects.all()
+    assert cities.count() == 3
+    paris = cities.get(zipcode='75001')
+    assert paris.name == 'Paris'
+    assert paris.code == '75056'
+    assert paris.population == 2190327
+    assert paris.department.code == '75'
+    assert paris.region.code == '11'
+
+    paris2 = cities.get(zipcode='75002')
+    paris_json = paris.to_json()
+    for key, value in paris2.to_json().items():
+        if not key in ['id', 'text', 'zipcode']:
+            assert paris_json[key] == value
+
+    miquelon = cities.get(zipcode='97500')
+    assert miquelon.name == 'Miquelon-Langlade'
+    assert miquelon.code == '97501'
+    assert miquelon.population == 596
+    assert not miquelon.department
+    assert not miquelon.region
+
+    # check a new call downloads again
+    mocked_get.side_effect = return_values
+    call_command('cron', 'daily')
+    assert mocked_get.call_count == 6
+    # and doesn't delete anything
+    assert CityModel.objects.count() == 3
+    assert DepartmentModel.objects.count() == 2
+    assert RegionModel.objects.count() == 2
+
+
+@pytest.mark.usefixtures('mock_update_streets')
+@mock.patch('passerelle.utils.Request.get')
+def test_base_adresse_command_update_geo_delete(mocked_get, db, base_adresse):
+    return_values = [utils.FakedResponse(content=content, status_code=200)
+                     for content in (FAKE_API_GEO_REGIONS, FAKE_API_GEO_DEPARTMENTS, FAKE_API_GEO)]
+    mocked_get.side_effect = return_values
+    call_command('cron', 'daily')
+    assert CityModel.objects.count() == 3
+
+    new_fake_api_geo = json.dumps([FAKE_API_GEO_LIST[1]])
+    return_values = [utils.FakedResponse(content=content, status_code=200)
+                     for content in (FAKE_API_GEO_REGIONS, FAKE_API_GEO_DEPARTMENTS, new_fake_api_geo)]
+    mocked_get.side_effect = return_values
+    call_command('cron', 'daily')
+    assert CityModel.objects.count() == 1
+
+
+@pytest.mark.usefixtures('mock_update_streets')
+@mock.patch('passerelle.utils.Request.get')
+def test_base_adresse_command_hourly_update_geo(mocked_get, db, base_adresse):
+    return_values = [utils.FakedResponse(content=content, status_code=200)
+                     for content in (FAKE_API_GEO_REGIONS, FAKE_API_GEO_DEPARTMENTS, FAKE_API_GEO)]
+    mocked_get.side_effect = return_values
+    # check the first hourly job downloads data
+    call_command('cron', 'hourly')
+    assert mocked_get.call_count == 3
+    assert CityModel.objects.count() == 3
+    # check a second call doesn't download anything
+    call_command('cron', 'hourly')
+    assert mocked_get.call_count == 3
+
+
+@pytest.mark.usefixtures('mock_update_streets')
+@mock.patch('passerelle.utils.Request.get')
+def test_base_adresse_command_update_geo_invalid(mocked_get, db, base_adresse):
+    mocked_get.return_value = utils.FakedResponse(content='{}', status_code=200)
+    with pytest.raises(CommandError):
+        call_command('cron', 'daily')
+    assert mocked_get.call_count == 1
+    assert not RegionModel.objects.exists()
+
+    mocked_get.return_value = utils.FakedResponse(content=FAKE_API_GEO, status_code=500)
+    call_command('cron', 'daily')
+    assert mocked_get.call_count == 4
+    assert not RegionModel.objects.exists()
+
+    mocked_get.return_value = utils.FakedResponse(content='not-json', status_code=200)
+    call_command('cron', 'daily')
+    assert mocked_get.call_count == 7
+    assert not RegionModel.objects.exists()
-- 
2.20.1

