From f204df2df67731150bde677c445d905477db3fbd Mon Sep 17 00:00:00 2001
From: Thomas NOEL <tnoel@entrouvert.com>
Date: Tue, 3 Aug 2021 10:44:05 +0200
Subject: [PATCH] add address sectorization system base (#56001)

---
 passerelle/address/__init__.py                |   0
 passerelle/address/admin.py                   |  30 +++
 passerelle/address/migrations/0001_initial.py |  78 ++++++++
 .../migrations/0002_auto_20210804_1601.py     |  25 +++
 passerelle/address/migrations/__init__.py     |   0
 passerelle/address/models.py                  | 176 ++++++++++++++++++
 .../0025_baseadresse_sector_types.py          |  21 +++
 passerelle/apps/base_adresse/models.py        |  16 +-
 passerelle/settings.py                        |   1 +
 tests/test_cron.py                            |   2 +-
 10 files changed, 342 insertions(+), 7 deletions(-)
 create mode 100644 passerelle/address/__init__.py
 create mode 100644 passerelle/address/admin.py
 create mode 100644 passerelle/address/migrations/0001_initial.py
 create mode 100644 passerelle/address/migrations/0002_auto_20210804_1601.py
 create mode 100644 passerelle/address/migrations/__init__.py
 create mode 100644 passerelle/address/models.py
 create mode 100644 passerelle/apps/base_adresse/migrations/0025_baseadresse_sector_types.py

diff --git a/passerelle/address/__init__.py b/passerelle/address/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/passerelle/address/admin.py b/passerelle/address/admin.py
new file mode 100644
index 00000000..e8e86f53
--- /dev/null
+++ b/passerelle/address/admin.py
@@ -0,0 +1,30 @@
+from django.contrib import admin
+
+from passerelle.address.models import Sector, Sectorization, SectorType
+
+
+class SectorTypeAdmin(admin.ModelAdmin):
+    prepopulated_fields = {'slug': ('name',)}
+
+
+class SectorAdmin(admin.ModelAdmin):
+    prepopulated_fields = {'slug': ('name',)}
+    list_display = ('name', 'sector_type', 'slug')
+
+
+class SectorizationAdmin(admin.ModelAdmin):
+    list_display = (
+        'id',
+        'street_id',
+        'parity',
+        'min_housenumber',
+        'max_housenumber',
+        'sector',
+        'sector_type',
+    )
+    list_filter = ('sector__sector_type',)
+
+
+admin.site.register(SectorType, SectorTypeAdmin)
+admin.site.register(Sector, SectorAdmin)
+admin.site.register(Sectorization, SectorizationAdmin)
diff --git a/passerelle/address/migrations/0001_initial.py b/passerelle/address/migrations/0001_initial.py
new file mode 100644
index 00000000..5d1fc117
--- /dev/null
+++ b/passerelle/address/migrations/0001_initial.py
@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.18 on 2021-08-03 10:07
+from __future__ import unicode_literals
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = []
+
+    operations = [
+        migrations.CreateModel(
+            name='Sector',
+            fields=[
+                (
+                    'id',
+                    models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+                ),
+                ('name', models.CharField(max_length=256, verbose_name='Name')),
+                ('slug', models.CharField(max_length=128, verbose_name='slug')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Sectorization',
+            fields=[
+                (
+                    'id',
+                    models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+                ),
+                ('street_id', models.CharField(max_length=64, verbose_name='Street Identifier')),
+                (
+                    'parity',
+                    models.CharField(
+                        choices=[('P', 'even numbers'), ('I', 'odd numbers'), ('N', 'all numbers')],
+                        default='N',
+                        max_length=1,
+                        verbose_name='Parity of numbers',
+                    ),
+                ),
+                (
+                    'min_housenumber',
+                    models.PositiveIntegerField(default=0, verbose_name='Minimal house number'),
+                ),
+                (
+                    'max_housenumber',
+                    models.PositiveIntegerField(default=99999999, verbose_name='Maximal house number'),
+                ),
+                (
+                    'sector',
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to='address.Sector',
+                        verbose_name='Sector',
+                    ),
+                ),
+            ],
+        ),
+        migrations.CreateModel(
+            name='SectorType',
+            fields=[
+                (
+                    'id',
+                    models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+                ),
+                ('name', models.CharField(max_length=256, verbose_name='Name')),
+                ('slug', models.CharField(max_length=128, verbose_name='slug')),
+            ],
+        ),
+        migrations.AddField(
+            model_name='sector',
+            name='sector_type',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='address.SectorType'),
+        ),
+    ]
diff --git a/passerelle/address/migrations/0002_auto_20210804_1601.py b/passerelle/address/migrations/0002_auto_20210804_1601.py
new file mode 100644
index 00000000..ac1013b8
--- /dev/null
+++ b/passerelle/address/migrations/0002_auto_20210804_1601.py
@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.18 on 2021-08-04 14:01
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('address', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='sectorization',
+            name='parity',
+            field=models.CharField(
+                choices=[('P', 'even'), ('I', 'odd'), ('N', 'all')],
+                default='N',
+                max_length=1,
+                verbose_name='Parity of numbers',
+            ),
+        ),
+    ]
diff --git a/passerelle/address/migrations/__init__.py b/passerelle/address/migrations/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/passerelle/address/models.py b/passerelle/address/models.py
new file mode 100644
index 00000000..f8072ea2
--- /dev/null
+++ b/passerelle/address/models.py
@@ -0,0 +1,176 @@
+# passerelle - uniform access to multiple data sources and services
+# Copyright (C) 2021  Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from django.core.exceptions import ValidationError
+from django.db import models
+from django.db.models import Q
+from django.utils.translation import ugettext_lazy as _
+
+from passerelle.base.models import BaseResource
+from passerelle.utils.api import endpoint
+
+PARITY_CHOICES = (
+    # stored in french: Pair / Impair / Neutre
+    ('P', _('even')),
+    ('I', _('odd')),
+    ('N', _('all')),
+)
+
+# Sectorization system, street by street
+
+
+class SectorType(models.Model):
+    name = models.CharField(max_length=256, verbose_name=_('Name'))
+    slug = models.CharField(max_length=128, verbose_name=_('slug'))
+
+    def __str__(self):
+        return '%s [%s]' % (self.name, self.slug)
+
+
+class Sector(models.Model):
+    sector_type = models.ForeignKey(SectorType, models.CASCADE)
+    name = models.CharField(max_length=256, verbose_name=_('Name'))
+    slug = models.CharField(max_length=128, verbose_name=_('slug'))
+
+    def __str__(self):
+        return '%s > %s [%s]' % (self.sector_type, self.name, self.slug)
+
+
+class Sectorization(models.Model):
+    MAX_HOUSENUMBER = 999_999
+
+    sector = models.ForeignKey(Sector, on_delete=models.CASCADE, verbose_name=_('Sector'))
+    street_id = models.CharField(max_length=64, verbose_name=_('Street Identifier'))
+    parity = models.CharField(
+        max_length=1, choices=PARITY_CHOICES, default='N', verbose_name=_('Parity of numbers')
+    )
+    min_housenumber = models.PositiveIntegerField(default=0, verbose_name=_('Minimal house number'))
+    max_housenumber = models.PositiveIntegerField(
+        default=MAX_HOUSENUMBER, verbose_name=_('Maximal house number')
+    )
+
+    class Meta:
+        ordering = ['street_id', 'min_housenumber', 'parity']
+
+    def __str__(self):
+        return '%s, parity:%s, min:%s, max:%s → %s' % (
+            self.street_id,
+            self.parity,
+            self.min_housenumber,
+            self.max_housenumber,
+            self.sector,
+        )
+
+    def clean(self):
+        if not self.max_housenumber or self.max_housenumber > self.MAX_HOUSENUMBER:
+            self.max_housenumber = self.MAX_HOUSENUMBER
+        if self.min_housenumber > self.max_housenumber:
+            raise ValidationError(_('Minimal house number may not be lesser than maximal house number.'))
+
+    @property
+    def sector_type(self):
+        return self.sector.sector_type
+
+
+class AddressResource(BaseResource):
+    sector_types = models.ManyToManyField(SectorType, blank=True, related_name='sector_types')
+
+    category = _('Geographic information system')
+
+    class Meta:
+        abstract = True
+
+    def sectorize(self, address):
+        """
+        add 'sectors' entry in address.
+        address is a Nominatim formatted dict and should contain 'street_id' and
+        'house_number' in its 'address' dict.
+        """
+        if not self.sector_types.exists():
+            return
+        address['sectors'] = {}
+
+        street_id = address['address'].get('street_id')
+        if not street_id:
+            address['sectorization_error'] = 'missing street_id in address'
+            return
+
+        try:
+            house_number = int(address['address'].get('house_number'))
+        except (TypeError, ValueError):
+            house_number = None
+
+        query = Sectorization.objects.filter(
+            sector__sector_type__in=self.sector_types.all(), street_id=street_id
+        )
+        if house_number is not None:
+            query = query.filter(min_housenumber__lte=house_number, max_housenumber__gte=house_number)
+            parity = 'I' if house_number % 2 else 'P'
+            query = query.filter(Q(parity='N') | Q(parity=parity))
+        else:
+            query = query.filter(parity='N', min_housenumber=0, max_housenumber=self.MAX_HOUSENUMBER)
+        for sectorization in query.reverse():
+            address['sectors'][sectorization.sector.sector_type.slug] = {
+                'id': sectorization.sector.slug,
+                'text': sectorization.sector.name,
+            }
+
+    # data sources endpoint
+    @endpoint(
+        name='sector-types',
+        description=_('Sector types'),
+        parameters={
+            'id': {'description': _('Sector Type identifier (slug)')},
+            'q': {'description': _("Sector Type name")},
+        },
+    )
+    def sector_types_(self, request, q=None, id=None):
+        query = self.sector_types
+        if id is not None:
+            query = query.filter(slug=id)
+        elif q is not None:
+            query = query.filter(Q(slug__icontains=q) | Q(name__icontains=q))
+        else:
+            query = query.all()
+        return {'data': [{'id': st.slug, 'text': st.name} for st in query]}
+
+    @endpoint(
+        name='sectors',
+        description=_('Sectors'),
+        parameters={
+            'id': {'description': _('Sector identifier (slug)')},
+            'q': {'description': _("Sector name")},
+            'sector_type': {'description': _('Filter by Sector Type identifier (slug)')},
+        },
+    )
+    def sectors(self, request, q=None, id=None, sector_type=None):
+        query = Sector.objects.filter(sector_type__in=self.sector_types.all())
+        if id is not None:
+            query = query.filter(slug=id)
+        elif q is not None:
+            query = query.filter(Q(slug__icontains=q) | Q(name__icontains=q))
+        else:
+            query = query.all()
+        return {
+            'data': [
+                {
+                    'id': sect.slug,
+                    'text': sect.name,
+                    'sector_type': {'id': sect.sector_type.slug, 'text': sect.sector_type.name},
+                }
+                for sect in query
+            ]
+        }
diff --git a/passerelle/apps/base_adresse/migrations/0025_baseadresse_sector_types.py b/passerelle/apps/base_adresse/migrations/0025_baseadresse_sector_types.py
new file mode 100644
index 00000000..1597f44c
--- /dev/null
+++ b/passerelle/apps/base_adresse/migrations/0025_baseadresse_sector_types.py
@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.18 on 2021-08-04 14:02
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('address', '0002_auto_20210804_1601'),
+        ('base_adresse', '0024_resource_in_models_alter'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='baseadresse',
+            name='sector_types',
+            field=models.ManyToManyField(blank=True, related_name='sector_types', to='address.SectorType'),
+        ),
+    ]
diff --git a/passerelle/apps/base_adresse/models.py b/passerelle/apps/base_adresse/models.py
index b1ec1709..ee9e79f3 100644
--- a/passerelle/apps/base_adresse/models.py
+++ b/passerelle/apps/base_adresse/models.py
@@ -12,14 +12,14 @@ from django.utils.translation import ugettext_lazy as _
 from requests import RequestException
 from requests.exceptions import ConnectionError
 
-from passerelle.base.models import BaseResource
+from passerelle.address.models import AddressResource
 from passerelle.compat import json_loads
 from passerelle.utils.api import endpoint
 from passerelle.utils.conversion import simplify
 from passerelle.utils.jsonresponse import APIError
 
 
-class BaseAdresse(BaseResource):
+class BaseAdresse(AddressResource):
     service_url = models.CharField(
         max_length=128,
         blank=False,
@@ -36,8 +36,6 @@ class BaseAdresse(BaseResource):
         help_text=_('Base Adresse API Geo URL'),
     )
 
-    category = _('Geographic information system')
-
     api_description = _(
         'The geocoding endpoints are a partial view of '
         '<a href="https://wiki.openstreetmap.org/wiki/Nominatim">Nominatim</a> own API; '
@@ -68,8 +66,12 @@ class BaseAdresse(BaseResource):
     class Meta:
         verbose_name = _('Base Adresse Web Service')
 
-    @staticmethod
-    def format_address_data(data):
+    def sectorize(self, address):
+        ban_id = address.get('ban_id') or ''
+        address['address']['street_id'] = '_'.join(ban_id.split('_', 2)[0:2])
+        super().sectorize(address)
+
+    def format_address_data(self, data):
         result = {}
         result['lon'] = str(data['geometry']['coordinates'][0])
         result['lat'] = str(data['geometry']['coordinates'][1])
@@ -90,6 +92,7 @@ class BaseAdresse(BaseResource):
                 result['ban_id'] = value
                 result['id'] = '%s~%s~%s' % (value, result['lat'], result['lon'])
         result['id'] = '%s~%s' % (result['id'], result['text'])
+        self.sectorize(result)
         return result
 
     @endpoint(
@@ -176,6 +179,7 @@ class BaseAdresse(BaseResource):
         except AddressCacheModel.DoesNotExist:
             pass
         else:
+            self.sectorize(address.data)  # if sectors have been updated since caching
             address.update_timestamp()
             return {'data': [address.data]}
         # Use search with label as q and lat/lon as geographic hint
diff --git a/passerelle/settings.py b/passerelle/settings.py
index 3750c8f0..de05e2d6 100644
--- a/passerelle/settings.py
+++ b/passerelle/settings.py
@@ -117,6 +117,7 @@ INSTALLED_APPS = (
     'django.contrib.postgres',
     # base app
     'passerelle.base',
+    'passerelle.address',
     'passerelle.sms',
     # connectors
     'passerelle.apps.actesweb',
diff --git a/tests/test_cron.py b/tests/test_cron.py
index 5c9af22b..f8abbef3 100644
--- a/tests/test_cron.py
+++ b/tests/test_cron.py
@@ -17,7 +17,7 @@ def test_cron_error(db, caplog):
     connector = BaseAdresse.objects.create(slug='base-adresse')
     excep = Exception('hello')
     with mock.patch(
-        'passerelle.apps.base_adresse.models.BaseResource.hourly', new=mock.Mock(side_effect=excep)
+        'passerelle.apps.base_adresse.models.AddressResource.hourly', new=mock.Mock(side_effect=excep)
     ):
         with pytest.raises(CommandError):
             call_command('cron', 'hourly')
-- 
2.30.2

