From 8336548fe8f5cb40c5d2accafd619c2ae929b0c9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20NO=C3=8BL?= <tnoel@entrouvert.com>
Date: Sun, 15 Aug 2021 22:28:58 +0200
Subject: [PATCH 2/2] add sectorization system (#56001)

---
 passerelle/address/__init__.py                |   0
 passerelle/address/models.py                  | 120 +++++++++
 .../migrations/0025_baseadresse_sectors.py    |  19 ++
 passerelle/apps/base_adresse/models.py        |  16 +-
 passerelle/settings.py                        |   1 +
 passerelle/utils/__init__.py                  |   8 +-
 tests/test_address.py                         | 236 ++++++++++++++++++
 tests/test_cron.py                            |   2 +-
 tests/test_import_export.py                   |  43 +++-
 9 files changed, 434 insertions(+), 11 deletions(-)
 create mode 100644 passerelle/address/__init__.py
 create mode 100644 passerelle/address/models.py
 create mode 100644 passerelle/apps/base_adresse/migrations/0025_baseadresse_sectors.py
 create mode 100644 tests/test_address.py

diff --git a/passerelle/address/__init__.py b/passerelle/address/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/passerelle/address/models.py b/passerelle/address/models.py
new file mode 100644
index 00000000..32465f7d
--- /dev/null
+++ b/passerelle/address/models.py
@@ -0,0 +1,120 @@
+# passerelle - uniform access to multiple data sources and services
+# Copyright (C) 2021  Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from urllib.parse import urljoin
+
+from django.conf import settings
+from django.db import models
+from django.db.models import Q
+from django.utils.translation import ugettext_lazy as _
+
+from passerelle.apps.sector.models import (
+    MAX_HOUSENUMBER,
+    PARITY_ALL,
+    PARITY_EVEN,
+    PARITY_ODD,
+    Sectorization,
+    SectorResource,
+)
+from passerelle.base.models import BaseResource
+from passerelle.utils.api import endpoint
+
+
+class AddressResource(BaseResource):
+    sectors = models.ManyToManyField(SectorResource, blank=True, related_name='resources')
+
+    category = _('Geographic information system')
+
+    class Meta:
+        abstract = True
+
+    def export_json(self):
+        d = super().export_json()
+        d['sectors'] = [sector.slug for sector in self.sectors.all()]
+        return d
+
+    @classmethod
+    def import_json_real(cls, overwrite, instance, d, **kwargs):
+        sectors = d.pop('sectors', [])
+        instance = super().import_json_real(overwrite, instance, d, **kwargs)
+        if instance and overwrite:
+            instance.sectors.clear()
+        for slug in sectors:
+            sector = SectorResource.objects.filter(slug=slug).first()
+            if sector:
+                instance.sectors.add(sector)
+        return instance
+
+    def sectorize(self, address):
+        """
+        add 'sectors' entry in address.
+        address is a Nominatim formatted dict and should contain 'street_id' and
+        'house_number' in its 'address' dict.
+        """
+        if not self.sectors.exists():
+            return
+        address['sectors'] = {}
+
+        street_id = address['address'].get('street_id')
+        if not street_id:
+            address['sectorization_error'] = 'missing street_id in address'
+            return
+
+        try:
+            house_number = int(address['address'].get('house_number'))
+        except (TypeError, ValueError):
+            house_number = None
+
+        query = Sectorization.objects.filter(sector__resource__in=self.sectors.all(), street_id=street_id)
+        if house_number is not None:
+            query = query.filter(min_housenumber__lte=house_number, max_housenumber__gte=house_number)
+            parity = PARITY_ODD if house_number % 2 else PARITY_EVEN
+            query = query.filter(Q(parity=PARITY_ALL) | Q(parity=parity))
+        else:
+            query = query.filter(parity=PARITY_ALL, min_housenumber=0, max_housenumber=MAX_HOUSENUMBER)
+        for sectorization in query.reverse():
+            address['sectors'][sectorization.sector.resource.slug] = {
+                'id': sectorization.sector.slug,
+                'text': sectorization.sector.title,
+            }
+
+    @endpoint(
+        name='sectors',
+        description=_('List related Sectorizations'),
+        parameters={
+            'id': {'description': _('Sector Identifier (slug)')},
+            'q': {'description': _('Filter by Sector Title ou Identifier')},
+        },
+    )
+    def sectors_(self, request, q=None, id=None):
+        query = self.sectors.all()
+        if id is not None:
+            query = query.filter(slug=id)
+        elif q is not None:
+            query = query.filter(Q(slug__icontains=q) | Q(title__icontains=q))
+        return {
+            'data': [
+                {
+                    'id': resource.slug,
+                    'text': resource.title,
+                    'description': resource.description,
+                    'sectors_url': urljoin(
+                        settings.SITE_BASE_URL, '%ssectors/' % resource.get_absolute_url()
+                    ),
+                }
+                for resource in query
+            ]
+        }
diff --git a/passerelle/apps/base_adresse/migrations/0025_baseadresse_sectors.py b/passerelle/apps/base_adresse/migrations/0025_baseadresse_sectors.py
new file mode 100644
index 00000000..8144fd0e
--- /dev/null
+++ b/passerelle/apps/base_adresse/migrations/0025_baseadresse_sectors.py
@@ -0,0 +1,19 @@
+# Generated by Django 2.2.19 on 2021-08-15 20:24
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sector', '0001_initial'),
+        ('base_adresse', '0024_resource_in_models_alter'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='baseadresse',
+            name='sectors',
+            field=models.ManyToManyField(blank=True, related_name='resources', to='sector.SectorResource'),
+        ),
+    ]
diff --git a/passerelle/apps/base_adresse/models.py b/passerelle/apps/base_adresse/models.py
index b1ec1709..ee9e79f3 100644
--- a/passerelle/apps/base_adresse/models.py
+++ b/passerelle/apps/base_adresse/models.py
@@ -12,14 +12,14 @@ from django.utils.translation import ugettext_lazy as _
 from requests import RequestException
 from requests.exceptions import ConnectionError
 
-from passerelle.base.models import BaseResource
+from passerelle.address.models import AddressResource
 from passerelle.compat import json_loads
 from passerelle.utils.api import endpoint
 from passerelle.utils.conversion import simplify
 from passerelle.utils.jsonresponse import APIError
 
 
-class BaseAdresse(BaseResource):
+class BaseAdresse(AddressResource):
     service_url = models.CharField(
         max_length=128,
         blank=False,
@@ -36,8 +36,6 @@ class BaseAdresse(BaseResource):
         help_text=_('Base Adresse API Geo URL'),
     )
 
-    category = _('Geographic information system')
-
     api_description = _(
         'The geocoding endpoints are a partial view of '
         '<a href="https://wiki.openstreetmap.org/wiki/Nominatim">Nominatim</a> own API; '
@@ -68,8 +66,12 @@ class BaseAdresse(BaseResource):
     class Meta:
         verbose_name = _('Base Adresse Web Service')
 
-    @staticmethod
-    def format_address_data(data):
+    def sectorize(self, address):
+        ban_id = address.get('ban_id') or ''
+        address['address']['street_id'] = '_'.join(ban_id.split('_', 2)[0:2])
+        super().sectorize(address)
+
+    def format_address_data(self, data):
         result = {}
         result['lon'] = str(data['geometry']['coordinates'][0])
         result['lat'] = str(data['geometry']['coordinates'][1])
@@ -90,6 +92,7 @@ class BaseAdresse(BaseResource):
                 result['ban_id'] = value
                 result['id'] = '%s~%s~%s' % (value, result['lat'], result['lon'])
         result['id'] = '%s~%s' % (result['id'], result['text'])
+        self.sectorize(result)
         return result
 
     @endpoint(
@@ -176,6 +179,7 @@ class BaseAdresse(BaseResource):
         except AddressCacheModel.DoesNotExist:
             pass
         else:
+            self.sectorize(address.data)  # if sectors have been updated since caching
             address.update_timestamp()
             return {'data': [address.data]}
         # Use search with label as q and lat/lon as geographic hint
diff --git a/passerelle/settings.py b/passerelle/settings.py
index b1519089..f98c5fd0 100644
--- a/passerelle/settings.py
+++ b/passerelle/settings.py
@@ -117,6 +117,7 @@ INSTALLED_APPS = (
     'django.contrib.postgres',
     # base app
     'passerelle.base',
+    'passerelle.address',
     'passerelle.sms',
     # connectors
     'passerelle.apps.actesweb',
diff --git a/passerelle/utils/__init__.py b/passerelle/utils/__init__.py
index 0be6e671..ca12e85a 100644
--- a/passerelle/utils/__init__.py
+++ b/passerelle/utils/__init__.py
@@ -384,8 +384,12 @@ def import_site(d, if_empty=False, clean=False, overwrite=False, import_users=Fa
             for apiuser in d.get('apiusers', []):
                 ApiUser.import_json(apiuser, overwrite=overwrite)
 
-        for resource in d.get('resources', []):
-            BaseResource.import_json(resource, overwrite=overwrite, import_users=import_users)
+        resources = d.get('resources', [])
+        # import SectorResource first, as AddressResource may need them
+        for res in [r for r in resources if r['resource_type'] == 'sector.sectorresource']:
+            BaseResource.import_json(res, overwrite=overwrite, import_users=import_users)
+        for res in [r for r in resources if r['resource_type'] != 'sector.sectorresource']:
+            BaseResource.import_json(res, overwrite=overwrite, import_users=import_users)
 
 
 def batch(iterable, size):
diff --git a/tests/test_address.py b/tests/test_address.py
new file mode 100644
index 00000000..72922cda
--- /dev/null
+++ b/tests/test_address.py
@@ -0,0 +1,236 @@
+# passerelle - uniform access to multiple data sources and services
+# Copyright (C) 2021 Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import json
+
+import mock
+import pytest
+import utils
+from django.core.files import File
+from django.utils.six import StringIO
+
+from passerelle.apps.base_adresse.models import BaseAdresse
+from passerelle.apps.sector.models import SectorResource
+
+BAN = {
+    "attribution": "BAN",
+    "features": [
+        {
+            "geometry": {"coordinates": [2.323365, 48.833702], "type": "Point"},
+            "properties": {
+                "city": "Paris",
+                "citycode": "75114",
+                "context": "75, Paris, Île-de-France",
+                "district": "Paris 14e Arrondissement",
+                "housenumber": "169",
+                "id": "75114_1912_00169",
+                "importance": 0.77751,
+                "label": "169 Rue du Château 75014 Paris",
+                "name": "169 Rue du Château",
+                "postcode": "75014",
+                "score": 0.9797736363636363,
+                "street": "Rue du Château",
+                "type": "housenumber",
+                "x": 650331.55,
+                "y": 6859506.98,
+            },
+            "type": "Feature",
+        },
+        {
+            "geometry": {"coordinates": [2.323365, 48.833702], "type": "Point"},
+            "properties": {
+                "city": "Paris",
+                "citycode": "75114",
+                "context": "75, Paris, Île-de-France",
+                "district": "Paris 14e Arrondissement",
+                "housenumber": "167",
+                "id": "75114_1912_00167",
+                "importance": 0.77751,
+                "label": "167 Rue du Château 75014 Paris",
+                "name": "167 Rue du Château",
+                "postcode": "75014",
+                "score": 0.9797736363636363,
+                "street": "Rue du Château",
+                "type": "housenumber",
+                "x": 650331.55,
+                "y": 6859506.98,
+            },
+            "type": "Feature",
+        },
+        {
+            "geometry": {"coordinates": [2.323365, 48.833702], "type": "Point"},
+            "properties": {
+                "city": "Paris",
+                "citycode": "75114",
+                "context": "75, Paris, Île-de-France",
+                "district": "Paris 14e Arrondissement",
+                "housenumber": "170",
+                "id": "75114_1912_00169",
+                "importance": 0.77751,
+                "label": "170 Rue du Château 75014 Paris",
+                "name": "170 Rue du Château",
+                "postcode": "75014",
+                "score": 0.9797736363636363,
+                "street": "Rue du Château",
+                "type": "housenumber",
+                "x": 650331.55,
+                "y": 6859506.98,
+            },
+            "type": "Feature",
+        },
+        # no house number
+        {
+            "geometry": {"coordinates": [2.323365, 48.833702], "type": "Point"},
+            "properties": {
+                "city": "Paris",
+                "citycode": "75114",
+                "context": "75, Paris, Île-de-France",
+                "district": "Paris 14e Arrondissement",
+                "id": "75114_1912_00169",
+                "importance": 0.77751,
+                "label": "XX Rue du Château 75014 Paris",
+                "name": "Rue du Château",
+                "postcode": "75014",
+                "score": 0.9797736363636363,
+                "street": "Rue du Château",
+                "type": "housenumber",
+                "x": 650331.55,
+                "y": 6859506.98,
+            },
+            "type": "Feature",
+        },
+        {
+            "geometry": {"coordinates": [2.323365, 48.833702], "type": "Point"},
+            "properties": {
+                "city": "Paris",
+                "citycode": "75114",
+                "context": "75, Paris, Île-de-France",
+                "district": "Paris 14e Arrondissement",
+                "id": "75114_1913_00000",
+                "importance": 0.77751,
+                "label": "YY Rue du Château 75014 Paris",
+                "name": "Rue du Château",
+                "postcode": "75014",
+                "score": 0.9797736363636363,
+                "street": "Rue du Château",
+                "type": "housenumber",
+                "x": 650331.55,
+                "y": 6859506.98,
+            },
+            "type": "Feature",
+        },
+        # empty id (no street_id)
+        {
+            "geometry": {"coordinates": [2.323365, 48.833702], "type": "Point"},
+            "properties": {
+                "city": "Paris",
+                "citycode": "75114",
+                "context": "75, Paris, Île-de-France",
+                "district": "Paris 14e Arrondissement",
+                "housenumber": "169",
+                "id": "",  # empty id => no street_id
+                "importance": 0.77751,
+                "label": "169 Rue du Château 75014 Paris",
+                "name": "169 Rue du Château",
+                "postcode": "75014",
+                "score": 0.9797736363636363,
+                "street": "Rue du Château",
+                "type": "housenumber",
+                "x": 650331.55,
+                "y": 6859506.98,
+            },
+            "type": "Feature",
+        },
+    ],
+    "licence": "ETALAB-2.0",
+    "limit": 5,
+    "query": "169 rue du chateau, 75014 Paris",
+    "type": "FeatureCollection",
+    "version": "draft",
+}
+
+CSV = """street_id,parity,min_housenumber,max_housenumber,sector_id,sector_name
+75114_1912,P,,,gs-moulin,Groupe Scolaire Moulin
+75114_1912,I,0,167,gs-zola,Groupe Scolaire Zola
+75114_1912,I,168,999999,ecole-hugo,École Hugo
+75114_1913,,,,ecole-pascal,École Pascal
+"""
+
+
+@pytest.fixture
+def sector(db):
+    return utils.setup_access_rights(
+        SectorResource.objects.create(
+            slug='ecole',
+            title='Secteur scolaire',
+            description='desc',
+            csv_file=File(StringIO(CSV), 'sectorization.csv'),
+        )
+    )
+
+
+@pytest.fixture
+def base_adresse(db):
+    return utils.setup_access_rights(BaseAdresse.objects.create(slug='base-adresse', zipcode='75114'))
+
+
+@mock.patch('passerelle.utils.Request.get')
+def test_sectorization(mocked_get, app, base_adresse, sector):
+    assert base_adresse.sectors.count() == 0
+    mocked_get.return_value = utils.FakedResponse(content=json.dumps(BAN), status_code=200)
+    url = utils.generic_endpoint_url('base-adresse', 'search', slug=base_adresse.slug)
+    addresses = app.get(url, params={'q': 'foo'}, status=200).json
+    assert len(addresses) == 6
+    for address in addresses:
+        assert 'sectors' not in address
+
+    base_adresse.sectors.add(sector)
+
+    # empty sector
+    sector.sector_set.all().delete()
+    addresses = app.get(url, params={'q': 'foo'}, status=200).json
+    assert len(addresses) == 6
+    for address in addresses:
+        assert address['sectors'] == {}  # sector is empty
+
+    sector.save()  # reload CSV data in sector
+    addresses = app.get(url, params={'q': 'foo'}, status=200).json
+    assert addresses[0]['sectors'] == {'ecole': {'id': 'ecole-hugo', 'text': 'École Hugo'}}
+    assert addresses[1]['sectors'] == {'ecole': {'id': 'gs-zola', 'text': 'Groupe Scolaire Zola'}}
+    assert addresses[2]['sectors'] == {'ecole': {'id': 'gs-moulin', 'text': 'Groupe Scolaire Moulin'}}
+    # addresses without housenumber
+    assert addresses[3]['sectors'] == {}
+    assert addresses[4]['sectors'] == {'ecole': {'id': 'ecole-pascal', 'text': 'École Pascal'}}
+    # bad address format, no street_id
+    assert addresses[5]['sectors'] == {}
+    assert addresses[5]['sectorization_error'] == 'missing street_id in address'
+
+    url = utils.generic_endpoint_url('base-adresse', 'sectors', slug=base_adresse.slug)
+    for params in (None, {'id': 'ecole'}, {'q': 'scolaire'}):
+        sectors = app.get(url, params=params, status=200).json
+        assert sectors['err'] == 0
+        assert sectors['data'] == [
+            {
+                'id': 'ecole',
+                'text': 'Secteur scolaire',
+                'sectors_url': 'http://localhost/sector/ecole/sectors/',
+                'description': 'desc',
+            }
+        ]
+    for params in ({'id': 'foo'}, {'q': 'foo'}):
+        sectors = app.get(url, params=params, status=200).json
+        assert sectors['err'] == 0
+        assert sectors['data'] == []
diff --git a/tests/test_cron.py b/tests/test_cron.py
index 5c9af22b..f8abbef3 100644
--- a/tests/test_cron.py
+++ b/tests/test_cron.py
@@ -17,7 +17,7 @@ def test_cron_error(db, caplog):
     connector = BaseAdresse.objects.create(slug='base-adresse')
     excep = Exception('hello')
     with mock.patch(
-        'passerelle.apps.base_adresse.models.BaseResource.hourly', new=mock.Mock(side_effect=excep)
+        'passerelle.apps.base_adresse.models.AddressResource.hourly', new=mock.Mock(side_effect=excep)
     ):
         with pytest.raises(CommandError):
             call_command('cron', 'hourly')
diff --git a/tests/test_import_export.py b/tests/test_import_export.py
index 2ceae812..17fcd505 100644
--- a/tests/test_import_export.py
+++ b/tests/test_import_export.py
@@ -15,7 +15,10 @@ from django.utils.encoding import force_bytes, force_text
 from django.utils.six import BytesIO, StringIO
 
 from passerelle.apps.base_adresse.models import BaseAdresse
+from passerelle.apps.bdp.models import Bdp
+from passerelle.apps.csvdatasource.models import CsvDataSource, Query
 from passerelle.apps.ovh.models import OVHSMSGateway
+from passerelle.apps.sector.models import SectorResource
 from passerelle.base.models import AccessRight, ApiUser
 from passerelle.compat import json_loads
 from passerelle.utils import export_site, import_site
@@ -44,8 +47,8 @@ data = """121;69981;DELANOUE;Eliot;H
 
 data_bom = force_text(data, 'utf-8').encode('utf-8-sig')
 
-from passerelle.apps.bdp.models import Bdp
-from passerelle.apps.csvdatasource.models import CsvDataSource, Query
+SECTOR_CSV = """street_id,parity,min_housenumber,max_housenumber,sector_id,sector_name
+38 75114_1912,P,,, gs-moulin, Groupe Scolaire Moulin"""
 
 pytestmark = pytest.mark.django_db
 
@@ -113,6 +116,8 @@ def clear():
     Query.objects.all().delete()
     CsvDataSource.objects.all().delete()
     ApiUser.objects.all().delete()
+    BaseAdresse.objects.all().delete()
+    SectorResource.objects.all().delete()
 
 
 def test_export_csvdatasource(app, setup, filetype):
@@ -238,6 +243,7 @@ def test_export_by_slug():
 
 def test_export_base_adresse():
     ba = BaseAdresse.objects.create(
+        slug='ba',
         service_url='https://api-adresse.data.gouv.fr/',
         api_geo_url='https://geo.api.gouv.fr/',
         zipcode='75013',
@@ -248,6 +254,39 @@ def test_export_base_adresse():
     new_ba = BaseAdresse.import_json(ba_export, overwrite=True)
     assert ba == new_ba
 
+    # test sectors relationships
+
+    assert new_ba.sectors.count() == 0
+    assert ba_export['sectors'] == []
+
+    sector = SectorResource.objects.create(
+        slug='sector1',
+        title='sector 1',
+        csv_file=File(StringIO(SECTOR_CSV), 'sectorization.csv'),
+    )
+    ba.sectors.add(sector)
+    ba_export = ba.export_json()
+    assert ba_export['sectors'] == ['sector1']
+    new_ba = BaseAdresse.import_json(ba_export, overwrite=True)
+    assert new_ba.sectors.count() == 1
+    assert new_ba.sectors.get() == sector
+
+    ba_export['sectors'] == ['sector1', 'sec-no-exist']
+    new_ba = BaseAdresse.import_json(ba_export, overwrite=True)
+    assert new_ba.sectors.count() == 1
+    assert new_ba.sectors.get() == sector
+
+    # export all site: handle dependencies
+    export = export_site()
+    clear()
+    assert SectorResource.objects.count() == BaseAdresse.objects.count() == 0
+    ba = [d for d in export['resources'] if d['slug'] == 'ba'][0]
+    sector1 = [d for d in export['resources'] if d['slug'] == 'sector1'][0]
+    export['resources'] = [ba, sector1]  # import must be clever: create sector1 before ba
+    import_site(export, import_users=True)
+    ba = BaseAdresse.objects.get()
+    assert ba.sectors.get() == SectorResource.objects.get()
+
 
 def test_export_ovh():
     ovh = OVHSMSGateway.objects.create(
-- 
2.30.2

