From 8acec22a1f73f0fed9b1960fec54cb6088c1bfe7 Mon Sep 17 00:00:00 2001
From: Thomas NOEL <tnoel@entrouvert.com>
Date: Tue, 3 Aug 2021 10:44:05 +0200
Subject: [PATCH] add address sectorization system base (#56001)

---
 passerelle/address/__init__.py                |   0
 passerelle/address/models.py                  |  96 +++++++++++++++
 .../migrations/0025_baseadresse_sectors.py    |  19 +++
 passerelle/apps/base_adresse/models.py        |  16 ++-
 passerelle/apps/sector/__init__.py            |   0
 passerelle/apps/sector/admin.py               |  42 +++++++
 .../apps/sector/migrations/0001_initial.py    |  99 +++++++++++++++
 passerelle/apps/sector/migrations/__init__.py |   0
 passerelle/apps/sector/models.py              | 115 ++++++++++++++++++
 passerelle/settings.py                        |   2 +
 tests/test_cron.py                            |   2 +-
 11 files changed, 384 insertions(+), 7 deletions(-)
 create mode 100644 passerelle/address/__init__.py
 create mode 100644 passerelle/address/models.py
 create mode 100644 passerelle/apps/base_adresse/migrations/0025_baseadresse_sectors.py
 create mode 100644 passerelle/apps/sector/__init__.py
 create mode 100644 passerelle/apps/sector/admin.py
 create mode 100644 passerelle/apps/sector/migrations/0001_initial.py
 create mode 100644 passerelle/apps/sector/migrations/__init__.py
 create mode 100644 passerelle/apps/sector/models.py

diff --git a/passerelle/address/__init__.py b/passerelle/address/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/passerelle/address/models.py b/passerelle/address/models.py
new file mode 100644
index 00000000..96e411ad
--- /dev/null
+++ b/passerelle/address/models.py
@@ -0,0 +1,96 @@
+# passerelle - uniform access to multiple data sources and services
+# Copyright (C) 2021  Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from urllib.parse import urljoin
+
+from django.conf import settings
+from django.db import models
+from django.db.models import Q
+from django.utils.translation import ugettext_lazy as _
+
+from passerelle.apps.sector.models import Sectorization, SectorResource
+from passerelle.base.models import BaseResource
+from passerelle.utils.api import endpoint
+
+
+class AddressResource(BaseResource):
+    sectors = models.ManyToManyField(SectorResource, blank=True, related_name='resources')
+
+    category = _('Geographic information system')
+
+    class Meta:
+        abstract = True
+
+    def sectorize(self, address):
+        """
+        add 'sectors' entry in address.
+        address is a Nominatim formatted dict and should contain 'street_id' and
+        'house_number' in its 'address' dict.
+        """
+        if not self.sectors.exists():
+            return
+        address['sectors'] = {}
+
+        street_id = address['address'].get('street_id')
+        if not street_id:
+            address['sectorization_error'] = 'missing street_id in address'
+            return
+
+        try:
+            house_number = int(address['address'].get('house_number'))
+        except (TypeError, ValueError):
+            house_number = None
+
+        query = Sectorization.objects.filter(sector__resource__in=self.sectors.all(), street_id=street_id)
+        if house_number is not None:
+            query = query.filter(min_housenumber__lte=house_number, max_housenumber__gte=house_number)
+            parity = 'I' if house_number % 2 else 'P'
+            query = query.filter(Q(parity='N') | Q(parity=parity))
+        else:
+            query = query.filter(parity='N', min_housenumber=0, max_housenumber=self.MAX_HOUSENUMBER)
+        for sectorization in query.reverse():
+            address['sectors'][sectorization.sector.resource.slug] = {
+                'id': sectorization.sector.slug,
+                'text': sectorization.sector.name,
+            }
+
+    @endpoint(
+        name='sectors',
+        description=_('List related Sectorizations'),
+        parameters={
+            'id': {'description': _('Sector Identifier (slug)')},
+            'q': {'description': _('Filter by Sector Title ou Identifier')},
+        },
+    )
+    def sectors_(self, request, q=None, id=None):
+        query = self.sectors.all()
+        if id is not None:
+            query = query.filter(slug=id)
+        elif q is not None:
+            query = query.filter(Q(slug__icontains=q) | Q(title__icontains=q))
+        return {
+            'data': [
+                {
+                    'id': resource.slug,
+                    'text': resource.title,
+                    'description': resource.description,
+                    'sectors_url': urljoin(
+                        settings.SITE_BASE_URL, '%ssectors/' % resource.get_absolute_url()
+                    ),
+                }
+                for resource in query
+            ]
+        }
diff --git a/passerelle/apps/base_adresse/migrations/0025_baseadresse_sectors.py b/passerelle/apps/base_adresse/migrations/0025_baseadresse_sectors.py
new file mode 100644
index 00000000..8e3de949
--- /dev/null
+++ b/passerelle/apps/base_adresse/migrations/0025_baseadresse_sectors.py
@@ -0,0 +1,19 @@
+# Generated by Django 2.2.19 on 2021-08-06 13:36
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sector', '0001_initial'),
+        ('base_adresse', '0024_resource_in_models_alter'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='baseadresse',
+            name='sectors',
+            field=models.ManyToManyField(blank=True, related_name='resources', to='sector.SectorResource'),
+        ),
+    ]
diff --git a/passerelle/apps/base_adresse/models.py b/passerelle/apps/base_adresse/models.py
index b1ec1709..ee9e79f3 100644
--- a/passerelle/apps/base_adresse/models.py
+++ b/passerelle/apps/base_adresse/models.py
@@ -12,14 +12,14 @@ from django.utils.translation import ugettext_lazy as _
 from requests import RequestException
 from requests.exceptions import ConnectionError
 
-from passerelle.base.models import BaseResource
+from passerelle.address.models import AddressResource
 from passerelle.compat import json_loads
 from passerelle.utils.api import endpoint
 from passerelle.utils.conversion import simplify
 from passerelle.utils.jsonresponse import APIError
 
 
-class BaseAdresse(BaseResource):
+class BaseAdresse(AddressResource):
     service_url = models.CharField(
         max_length=128,
         blank=False,
@@ -36,8 +36,6 @@ class BaseAdresse(BaseResource):
         help_text=_('Base Adresse API Geo URL'),
     )
 
-    category = _('Geographic information system')
-
     api_description = _(
         'The geocoding endpoints are a partial view of '
         '<a href="https://wiki.openstreetmap.org/wiki/Nominatim">Nominatim</a> own API; '
@@ -68,8 +66,12 @@ class BaseAdresse(BaseResource):
     class Meta:
         verbose_name = _('Base Adresse Web Service')
 
-    @staticmethod
-    def format_address_data(data):
+    def sectorize(self, address):
+        ban_id = address.get('ban_id') or ''
+        address['address']['street_id'] = '_'.join(ban_id.split('_', 2)[0:2])
+        super().sectorize(address)
+
+    def format_address_data(self, data):
         result = {}
         result['lon'] = str(data['geometry']['coordinates'][0])
         result['lat'] = str(data['geometry']['coordinates'][1])
@@ -90,6 +92,7 @@ class BaseAdresse(BaseResource):
                 result['ban_id'] = value
                 result['id'] = '%s~%s~%s' % (value, result['lat'], result['lon'])
         result['id'] = '%s~%s' % (result['id'], result['text'])
+        self.sectorize(result)
         return result
 
     @endpoint(
@@ -176,6 +179,7 @@ class BaseAdresse(BaseResource):
         except AddressCacheModel.DoesNotExist:
             pass
         else:
+            self.sectorize(address.data)  # if sectors have been updated since caching
             address.update_timestamp()
             return {'data': [address.data]}
         # Use search with label as q and lat/lon as geographic hint
diff --git a/passerelle/apps/sector/__init__.py b/passerelle/apps/sector/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/passerelle/apps/sector/admin.py b/passerelle/apps/sector/admin.py
new file mode 100644
index 00000000..893973aa
--- /dev/null
+++ b/passerelle/apps/sector/admin.py
@@ -0,0 +1,42 @@
+# passerelle - uniform access to multiple data sources and services
+# Copyright (C) 2021  Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from django.contrib import admin
+
+from passerelle.apps.sector.models import Sector, Sectorization
+
+
+class SectorAdmin(admin.ModelAdmin):
+    prepopulated_fields = {'slug': ('title',)}
+    list_display = ('title', 'slug', 'resource')
+    list_filter = ('resource',)
+
+
+class SectorizationAdmin(admin.ModelAdmin):
+    list_display = (
+        'id',
+        'street_id',
+        'parity',
+        'min_housenumber',
+        'max_housenumber',
+        'sector',
+        'resource',
+    )
+    list_filter = ('sector__resource',)
+
+
+admin.site.register(Sector, SectorAdmin)
+admin.site.register(Sectorization, SectorizationAdmin)
diff --git a/passerelle/apps/sector/migrations/0001_initial.py b/passerelle/apps/sector/migrations/0001_initial.py
new file mode 100644
index 00000000..d1ee95ca
--- /dev/null
+++ b/passerelle/apps/sector/migrations/0001_initial.py
@@ -0,0 +1,99 @@
+# Generated by Django 2.2.19 on 2021-08-06 13:31
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('base', '0029_auto_20210202_1627'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Sector',
+            fields=[
+                (
+                    'id',
+                    models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+                ),
+                ('title', models.CharField(max_length=256, verbose_name='Title')),
+                ('slug', models.CharField(max_length=128, verbose_name='Identifier')),
+            ],
+            options={
+                'ordering': ['resource', 'slug'],
+            },
+        ),
+        migrations.CreateModel(
+            name='SectorResource',
+            fields=[
+                (
+                    'id',
+                    models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+                ),
+                ('title', models.CharField(max_length=50, verbose_name='Title')),
+                ('slug', models.SlugField(unique=True, verbose_name='Identifier')),
+                ('description', models.TextField(verbose_name='Description')),
+                (
+                    'users',
+                    models.ManyToManyField(
+                        blank=True,
+                        related_name='_sectorresource_users_+',
+                        related_query_name='+',
+                        to='base.ApiUser',
+                    ),
+                ),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='Sectorization',
+            fields=[
+                (
+                    'id',
+                    models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+                ),
+                ('street_id', models.CharField(max_length=64, verbose_name='Street Identifier')),
+                (
+                    'parity',
+                    models.CharField(
+                        choices=[('P', 'even'), ('I', 'odd'), ('N', 'all')],
+                        default='N',
+                        max_length=1,
+                        verbose_name='Parity of numbers',
+                    ),
+                ),
+                (
+                    'min_housenumber',
+                    models.PositiveIntegerField(default=0, verbose_name='Minimal house number'),
+                ),
+                (
+                    'max_housenumber',
+                    models.PositiveIntegerField(default=999999, verbose_name='Maximal house number'),
+                ),
+                (
+                    'sector',
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to='sector.Sector', verbose_name='Sector'
+                    ),
+                ),
+            ],
+            options={
+                'ordering': ['street_id', 'min_housenumber', 'parity'],
+            },
+        ),
+        migrations.AddField(
+            model_name='sector',
+            name='resource',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='sector.SectorResource'),
+        ),
+        migrations.AlterUniqueTogether(
+            name='sector',
+            unique_together={('resource', 'slug')},
+        ),
+    ]
diff --git a/passerelle/apps/sector/migrations/__init__.py b/passerelle/apps/sector/migrations/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/passerelle/apps/sector/models.py b/passerelle/apps/sector/models.py
new file mode 100644
index 00000000..4aeab886
--- /dev/null
+++ b/passerelle/apps/sector/models.py
@@ -0,0 +1,115 @@
+# passerelle - uniform access to multiple data sources and services
+# Copyright (C) 2021  Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from django.core.exceptions import ValidationError
+from django.db import models
+from django.db.models import Q
+from django.utils.translation import ugettext_lazy as _
+
+from passerelle.base.models import BaseResource
+from passerelle.utils.api import endpoint
+
+PARITY_CHOICES = (
+    # stored in french: Pair / Impair / Neutre
+    ('P', _('even')),
+    ('I', _('odd')),
+    ('N', _('all')),
+)
+
+
+class SectorResource(BaseResource):
+    category = _('Geographic information system')
+
+    def __str__(self):
+        return '%s [%s]' % (self.title, self.slug)
+
+    @endpoint(
+        name='sectors',
+        description=_('List of Sectors'),
+        parameters={
+            'id': {'description': _('Sector identifier (slug)')},
+            'q': {'description': _('Filter by Sector Title or Identifier')},
+            'street_id': {'description': _('Get sectors for this Street identifier')},
+            'house_number': {'description': _('Get sectors by this House Number. Requires a street_id')},
+        },
+    )
+    def sectors(self, request, q=None, id=None):
+        query = self.sector_set.all()
+        if id is not None:
+            query = query.filter(slug=id)
+        elif q is not None:
+            query = query.filter(Q(slug__icontains=q) | Q(title__icontains=q))
+        return {
+            'data': [
+                {
+                    'id': sector.slug,
+                    'text': sector.title,
+                }
+                for sector in query
+            ]
+        }
+
+    # TODO endpoints:
+    # csv export/import of all Sectorization
+
+
+class Sector(models.Model):
+    resource = models.ForeignKey(SectorResource, on_delete=models.CASCADE)
+    title = models.CharField(max_length=256, verbose_name=_('Title'))
+    slug = models.CharField(max_length=128, verbose_name=_('Identifier'))
+
+    class Meta:
+        ordering = ['resource', 'slug']
+        unique_together = ('resource', 'slug')
+
+    def __str__(self):
+        return '%s > %s [%s]' % (self.resource, self.title, self.slug)
+
+
+class Sectorization(models.Model):
+    MAX_HOUSENUMBER = 999_999
+
+    sector = models.ForeignKey(Sector, on_delete=models.CASCADE, verbose_name=_('Sector'))
+    street_id = models.CharField(max_length=64, verbose_name=_('Street Identifier'))
+    parity = models.CharField(
+        max_length=1, choices=PARITY_CHOICES, default='N', verbose_name=_('Parity of numbers')
+    )
+    min_housenumber = models.PositiveIntegerField(default=0, verbose_name=_('Minimal house number'))
+    max_housenumber = models.PositiveIntegerField(
+        default=MAX_HOUSENUMBER, verbose_name=_('Maximal house number')
+    )
+
+    class Meta:
+        ordering = ['street_id', 'min_housenumber', 'parity']
+
+    def __str__(self):
+        return '%s, parity:%s, min:%s, max:%s → %s' % (
+            self.street_id,
+            self.parity,
+            self.min_housenumber,
+            self.max_housenumber,
+            self.sector,
+        )
+
+    @property
+    def resource(self):
+        return self.sector.resource
+
+    def clean(self):
+        if not self.max_housenumber or self.max_housenumber > self.MAX_HOUSENUMBER:
+            self.max_housenumber = self.MAX_HOUSENUMBER
+        if self.min_housenumber > self.max_housenumber:
+            raise ValidationError(_('Minimal house number may not be lesser than maximal house number.'))
diff --git a/passerelle/settings.py b/passerelle/settings.py
index 3750c8f0..f98c5fd0 100644
--- a/passerelle/settings.py
+++ b/passerelle/settings.py
@@ -117,6 +117,7 @@ INSTALLED_APPS = (
     'django.contrib.postgres',
     # base app
     'passerelle.base',
+    'passerelle.address',
     'passerelle.sms',
     # connectors
     'passerelle.apps.actesweb',
@@ -158,6 +159,7 @@ INSTALLED_APPS = (
     'passerelle.apps.oxyd',
     'passerelle.apps.phonecalls',
     'passerelle.apps.photon',
+    'passerelle.apps.sector',
     'passerelle.apps.solis',
     'passerelle.apps.twilio',
     'passerelle.apps.vivaticket',
diff --git a/tests/test_cron.py b/tests/test_cron.py
index 5c9af22b..f8abbef3 100644
--- a/tests/test_cron.py
+++ b/tests/test_cron.py
@@ -17,7 +17,7 @@ def test_cron_error(db, caplog):
     connector = BaseAdresse.objects.create(slug='base-adresse')
     excep = Exception('hello')
     with mock.patch(
-        'passerelle.apps.base_adresse.models.BaseResource.hourly', new=mock.Mock(side_effect=excep)
+        'passerelle.apps.base_adresse.models.AddressResource.hourly', new=mock.Mock(side_effect=excep)
     ):
         with pytest.raises(CommandError):
             call_command('cron', 'hourly')
-- 
2.30.2

