From 2343041ea1bffb712d2c8f2a1bf233eca9bc5f52 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20NO=C3=8BL?= <tnoel@entrouvert.com>
Date: Sun, 15 Aug 2021 22:27:57 +0200
Subject: [PATCH 1/2] add sector connector (#56001)

---
 passerelle/apps/sector/__init__.py            |   0
 passerelle/apps/sector/admin.py               |  42 ++
 .../apps/sector/migrations/0001_initial.py    | 116 ++++++
 passerelle/apps/sector/migrations/__init__.py |   0
 passerelle/apps/sector/models.py              | 371 +++++++++++++++++
 passerelle/settings.py                        |   1 +
 tests/test_sector.py                          | 383 ++++++++++++++++++
 7 files changed, 913 insertions(+)
 create mode 100644 passerelle/apps/sector/__init__.py
 create mode 100644 passerelle/apps/sector/admin.py
 create mode 100644 passerelle/apps/sector/migrations/0001_initial.py
 create mode 100644 passerelle/apps/sector/migrations/__init__.py
 create mode 100644 passerelle/apps/sector/models.py
 create mode 100644 tests/test_sector.py

diff --git a/passerelle/apps/sector/__init__.py b/passerelle/apps/sector/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/passerelle/apps/sector/admin.py b/passerelle/apps/sector/admin.py
new file mode 100644
index 00000000..893973aa
--- /dev/null
+++ b/passerelle/apps/sector/admin.py
@@ -0,0 +1,42 @@
+# passerelle - uniform access to multiple data sources and services
+# Copyright (C) 2021  Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from django.contrib import admin
+
+from passerelle.apps.sector.models import Sector, Sectorization
+
+
+class SectorAdmin(admin.ModelAdmin):
+    prepopulated_fields = {'slug': ('title',)}
+    list_display = ('title', 'slug', 'resource')
+    list_filter = ('resource',)
+
+
+class SectorizationAdmin(admin.ModelAdmin):
+    list_display = (
+        'id',
+        'street_id',
+        'parity',
+        'min_housenumber',
+        'max_housenumber',
+        'sector',
+        'resource',
+    )
+    list_filter = ('sector__resource',)
+
+
+admin.site.register(Sector, SectorAdmin)
+admin.site.register(Sectorization, SectorizationAdmin)
diff --git a/passerelle/apps/sector/migrations/0001_initial.py b/passerelle/apps/sector/migrations/0001_initial.py
new file mode 100644
index 00000000..63ee4065
--- /dev/null
+++ b/passerelle/apps/sector/migrations/0001_initial.py
@@ -0,0 +1,116 @@
+# Generated by Django 2.2.19 on 2021-08-15 19:40
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+import passerelle.apps.sector.models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('base', '0029_auto_20210202_1627'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Sector',
+            fields=[
+                (
+                    'id',
+                    models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+                ),
+                ('title', models.CharField(max_length=256, verbose_name='Title')),
+                ('slug', models.CharField(max_length=128, verbose_name='Identifier')),
+            ],
+            options={
+                'ordering': ['resource', 'slug'],
+            },
+        ),
+        migrations.CreateModel(
+            name='SectorResource',
+            fields=[
+                (
+                    'id',
+                    models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+                ),
+                ('title', models.CharField(max_length=50, verbose_name='Title')),
+                ('slug', models.SlugField(unique=True, verbose_name='Identifier')),
+                ('description', models.TextField(verbose_name='Description')),
+                (
+                    'csv_file',
+                    models.FileField(
+                        help_text='CSV file',
+                        upload_to=passerelle.apps.sector.models.upload_to,
+                        verbose_name='Sectorization file',
+                    ),
+                ),
+                (
+                    'titles_in_first_line',
+                    models.BooleanField(
+                        default=True,
+                        help_text='If not, column titles are: street_id,parity,min_housenumber,max_housenumber,sector_id,sector_name, …',
+                        verbose_name='First line defines column titles',
+                    ),
+                ),
+                (
+                    'users',
+                    models.ManyToManyField(
+                        blank=True,
+                        related_name='_sectorresource_users_+',
+                        related_query_name='+',
+                        to='base.ApiUser',
+                    ),
+                ),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='Sectorization',
+            fields=[
+                (
+                    'id',
+                    models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+                ),
+                ('street_id', models.CharField(max_length=64, verbose_name='Street Identifier')),
+                (
+                    'parity',
+                    models.SmallIntegerField(
+                        choices=[(0, 'all'), (1, 'odd'), (2, 'even')],
+                        default=0,
+                        verbose_name='Parity of numbers',
+                    ),
+                ),
+                (
+                    'min_housenumber',
+                    models.PositiveIntegerField(default=0, verbose_name='Minimal house number'),
+                ),
+                (
+                    'max_housenumber',
+                    models.PositiveIntegerField(default=999999, verbose_name='Maximal house number'),
+                ),
+                (
+                    'sector',
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to='sector.Sector', verbose_name='Sector'
+                    ),
+                ),
+            ],
+            options={
+                'ordering': ['street_id', 'min_housenumber', 'parity'],
+            },
+        ),
+        migrations.AddField(
+            model_name='sector',
+            name='resource',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='sector.SectorResource'),
+        ),
+        migrations.AlterUniqueTogether(
+            name='sector',
+            unique_together={('resource', 'slug')},
+        ),
+    ]
diff --git a/passerelle/apps/sector/migrations/__init__.py b/passerelle/apps/sector/migrations/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/passerelle/apps/sector/models.py b/passerelle/apps/sector/models.py
new file mode 100644
index 00000000..262b7427
--- /dev/null
+++ b/passerelle/apps/sector/models.py
@@ -0,0 +1,371 @@
+# passerelle - uniform access to multiple data sources and services
+# Copyright (C) 2021  Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import csv
+import datetime
+import mimetypes
+import os
+
+from django.conf import settings
+from django.core.exceptions import ValidationError
+from django.core.files.base import ContentFile
+from django.db import models, transaction
+from django.db.models import Q
+from django.http import HttpResponse
+from django.utils.encoding import force_bytes, force_str, smart_text
+from django.utils.timezone import now
+from django.utils.translation import ugettext_lazy as _
+
+from passerelle.base.models import BaseResource
+from passerelle.utils.api import endpoint
+from passerelle.utils.jsonresponse import APIError
+
+PARITY_ALL = 0
+PARITY_ODD = 1
+PARITY_EVEN = 2
+
+PARITY_CHOICES = (
+    (PARITY_ALL, _('all')),
+    (PARITY_ODD, _('odd')),
+    (PARITY_EVEN, _('even')),
+)
+
+CSV_TITLES = ['street_id', 'parity', 'min_housenumber', 'max_housenumber', 'sector_id', 'sector_name']
+
+MAX_HOUSENUMBER = 999_999
+
+
+def upload_to(instance, filename):
+    return '%s/%s/%s' % (instance.get_connector_slug(), instance.slug, filename)
+
+
+class SectorResource(BaseResource):
+    csv_file = models.FileField(
+        _('Sectorization file'),
+        upload_to=upload_to,
+        help_text=_('CSV file'),
+    )
+    titles_in_first_line = models.BooleanField(
+        _('First line defines column titles'),
+        default=True,
+        help_text=_('If not, column titles are: %s, …') % ','.join(CSV_TITLES),
+    )
+
+    category = _('Geographic information system')
+    hide_description_fields = ['csv_file']
+
+    def __str__(self):
+        return '%s [%s]' % (self.title, self.slug)
+
+    def daily(self):
+        super().daily()
+        self.clean_old_csv_files()
+
+    def clean_old_csv_files(self):
+        if not os.path.exists(self.csv_file.path):
+            return
+        base_dir = os.path.dirname(self.csv_file.path)
+        if os.path.dirname(self.csv_file.name) != os.path.join(self.get_connector_slug(), self.slug):
+            # path is not compliant with upload_to, do nothing
+            return
+
+        for filename in os.listdir(base_dir):
+            filepath = os.path.join(base_dir, filename)
+            if not os.path.isfile(filepath):
+                continue
+            if os.path.basename(self.csv_file.name) == filename:
+                # current file
+                continue
+            mtime = os.stat(filepath).st_mtime
+            if mtime > (now() + datetime.timedelta(days=-7)).timestamp():
+                # too young
+                continue
+
+            if getattr(settings, 'SECTOR_REMOVE_ON_CLEAN', False) is True:
+                # remove
+                os.unlink(filepath)
+            else:
+                # move file in unused-files dir
+                unused_dir = os.path.join(base_dir, 'unused-files')
+                os.makedirs(unused_dir, exist_ok=True)
+                os.rename(filepath, os.path.join(unused_dir, filename))
+
+    def clean(self, *args, **kwargs):
+        try:
+            self.import_csv()
+        except Exception as e:
+            raise ValidationError(_('Invalid CSV file: %s') % e)
+        return super().clean(*args, **kwargs)
+
+    @transaction.atomic
+    def import_csv(self):
+        try:
+            self.csv_file.seek(0)
+            content = force_bytes(self.csv_file.read())
+            content = force_str(content.decode('utf-8-sig', 'ignore').encode('utf-8'))  # handle BOM
+            dialect = csv.Sniffer().sniff(content)
+            reader = csv.reader(content.splitlines(), dialect)
+        except Exception as e:
+            raise ValidationError(_('failed to read CSV (%s)') % e)
+        if self.titles_in_first_line:
+            first_line = next(reader)
+            titles = [name.strip().lower() for name in first_line]
+            if not set(CSV_TITLES).issubset(titles):
+                raise ValidationError(
+                    _('missing column(s) in header: %s.') % ', '.join(set(CSV_TITLES) - set(titles))
+                )
+        else:
+            titles = CSV_TITLES
+        indexes = [titles.index(t) for t in titles if t]
+        captions = [titles[i] for i in indexes]
+
+        # now ready to import data, first delete all sectors
+        # sectorization will also be delete (cascade)
+        # (will be cancelled by transaction on any import error)
+        self.sector_set.all().delete()
+
+        def get_cell(row, index):
+            try:
+                return row[index]
+            except IndexError:
+                return ''
+
+        sector_id = sector_name = None
+        sectors = {}
+        for row in reader:
+            if not row:
+                continue  # do not consider empty lines
+            row = [smart_text(x).strip() for x in row]
+            row = {caption: get_cell(row, index) for caption, index in zip(captions, indexes)}
+
+            if row['sector_id']:
+                sector_id = row['sector_id']
+                sector_name = row['sector_name'] or row['sector_id']
+            if not sector_id:
+                raise ValidationError(_('missing sector_id, line %s') % reader.line_num)
+            if not row['street_id']:
+                raise ValidationError(_('missing street_id, line %s') % reader.line_num)
+            if not row['parity']:
+                parity = PARITY_ALL
+            elif row['parity'].lower()[0] in (str(PARITY_EVEN), 'e', 'p'):  # p = pair (even, in french)
+                parity = PARITY_EVEN
+            elif row['parity'].lower()[0] in (str(PARITY_ODD), 'o', 'i'):  # i = impair (odd, in french)
+                parity = PARITY_ODD
+            else:
+                parity = PARITY_ALL
+            try:
+                min_housenumber = int(row['min_housenumber'])
+            except (ValueError, TypeError):
+                min_housenumber = 0
+            try:
+                max_housenumber = int(row['max_housenumber'])
+            except (ValueError, TypeError):
+                max_housenumber = MAX_HOUSENUMBER
+
+            if sector_id not in sectors:
+                sectors[sector_id] = self.sector_set.create(resource=self, slug=sector_id, title=sector_name)
+            sectors[sector_id].sectorization_set.create(
+                sector=sectors[sector_id],
+                street_id=row['street_id'],
+                parity=parity,
+                min_housenumber=min_housenumber,
+                max_housenumber=max_housenumber,
+            )
+
+    @endpoint(
+        description=_('Update sectorization with a CSV file'),
+        display_category=_('Management'),
+        perm='can_access',
+        methods=['put'],
+    )
+    def update(self, request):
+        ext = mimetypes.guess_extension(request.content_type)
+        if not ext:
+            raise APIError(
+                "can't guess filename extension for '%s' content type" % request.content_type, http_status=400
+            )
+        name = self.csv_file.storage.get_available_name('api-uploaded-file' + ext)
+        self.csv_file = ContentFile(content=request.body, name=name)
+        try:
+            self.clean()
+        except ValidationError as e:
+            raise APIError(e, http_status=400)
+        self.save()
+        return {
+            'updated': self.csv_file.name,
+            'data': [
+                {
+                    'id': sector.slug,
+                    'text': sector.title,
+                }
+                for sector in self.sector_set.all()
+            ],
+        }
+
+    @endpoint(
+        description=_('Get sectorization as a CSV file'),
+        display_category=_('Management'),
+        perm='can_access',
+        parameters={
+            'even': {
+                'description': _('Even numbers indicator (default: %s)') % PARITY_EVEN,
+                'example_value': 'P',
+            },
+            'odd': {
+                'description': _('Odd numbers indicator (default: %s)') % PARITY_ODD,
+                'example_value': 'I',
+            },
+            'mix': {
+                'description': _('Even or odd numbers indicator (default: %s)') % PARITY_ALL,
+                'example_value': '',
+            },
+            'limits': {'description': _('Show housenumber min/max (0/%s)') % MAX_HOUSENUMBER, 'type': 'bool'},
+            'repeat': {'description': _('Repeat sector id and name on all lines'), 'type': 'bool'},
+        },
+    )
+    def export(self, request, even=None, odd=None, mix=None, limits=False, repeat=False):
+        response = HttpResponse(content_type='text/csv')
+        date = now().strftime('%Y-%m-%d_%H:%M')
+        response['Content-Disposition'] = 'attachment; filename="sector-%s-%s.csv"' % (self.slug, date)
+        writer = csv.writer(response, delimiter=',', quotechar='"', quoting=csv.QUOTE_ALL)
+        if self.titles_in_first_line:
+            writer.writerow(CSV_TITLES)
+        parity = dict(PARITY_CHOICES)
+        if even is not None:
+            parity[PARITY_EVEN] = even
+        if odd is not None:
+            parity[PARITY_ODD] = odd
+        if mix is not None:
+            parity[PARITY_ALL] = mix
+        for sector in self.sector_set.all().order_by('slug'):
+            first = True
+            for sectorization in sector.sectorization_set.all().order_by('street_id'):
+                writer.writerow(
+                    [
+                        sectorization.street_id,
+                        parity.get(sectorization.parity, mix),
+                        sectorization.min_housenumber if (limits or sectorization.min_housenumber) else '',
+                        sectorization.max_housenumber
+                        if (limits or sectorization.max_housenumber < MAX_HOUSENUMBER)
+                        else '',
+                        sector.slug if (repeat or first) else '',
+                        sector.title if (repeat or first) else '',
+                    ]
+                )
+                first = False
+        return response
+
+    @endpoint(
+        name='sectors',
+        description=_('List of Sectors'),
+        perm='can_access',
+        display_category=_('Data sources'),
+        parameters={
+            'id': {'description': _('Sector identifier (slug)')},
+            'q': {'description': _('Filter by Sector Title or Identifier')},
+            'street_id': {'description': _('Get sectors for this Street identifier')},
+            'house_number': {
+                'description': _('Get sectors by this House Number (requires a street_id)'),
+                'type': 'integer',
+            },
+        },
+    )
+    def sectors(self, request, q=None, id=None, street_id=None, house_number=None):
+        if house_number and not street_id:
+            raise APIError('house_number requires a street_id', http_status=400)
+
+        # search by street and house number
+        if street_id:
+            query = Sectorization.objects.filter(sector__resource=self, street_id=street_id)
+            if house_number is not None:
+                house_number = int(house_number)
+                query = query.filter(min_housenumber__lte=house_number, max_housenumber__gte=house_number)
+                parity = PARITY_ODD if house_number % 2 else PARITY_EVEN
+                query = query.filter(Q(parity=PARITY_ALL) | Q(parity=parity))
+            else:
+                query = query.filter(parity=PARITY_ALL, min_housenumber=0, max_housenumber=MAX_HOUSENUMBER)
+            return {
+                'data': [
+                    {
+                        'id': sectorization.sector.slug,
+                        'text': sectorization.sector.title,
+                    }
+                ]
+                for sectorization in query.reverse()
+            }
+
+        # list of sectors
+        query = self.sector_set.all()
+        if id is not None:
+            query = query.filter(slug=id)
+        elif q is not None:
+            query = query.filter(Q(slug__icontains=q) | Q(title__icontains=q))
+        return {
+            'data': [
+                {
+                    'id': sector.slug,
+                    'text': sector.title,
+                }
+                for sector in query
+            ]
+        }
+
+
+class Sector(models.Model):
+    resource = models.ForeignKey(SectorResource, on_delete=models.CASCADE)
+    title = models.CharField(max_length=256, verbose_name=_('Title'))
+    slug = models.CharField(max_length=128, verbose_name=_('Identifier'))
+
+    class Meta:
+        ordering = ['resource', 'slug']
+        unique_together = ('resource', 'slug')
+
+    def __str__(self):
+        return '%s > %s [%s]' % (self.resource, self.title, self.slug)
+
+
+class Sectorization(models.Model):
+    sector = models.ForeignKey(Sector, on_delete=models.CASCADE, verbose_name=_('Sector'))
+    street_id = models.CharField(max_length=64, verbose_name=_('Street Identifier'))
+    parity = models.SmallIntegerField(
+        choices=PARITY_CHOICES, default=PARITY_ALL, verbose_name=_('Parity of numbers')
+    )
+    min_housenumber = models.PositiveIntegerField(default=0, verbose_name=_('Minimal house number'))
+    max_housenumber = models.PositiveIntegerField(
+        default=MAX_HOUSENUMBER, verbose_name=_('Maximal house number')
+    )
+
+    class Meta:
+        ordering = ['street_id', 'min_housenumber', 'parity']
+
+    def __str__(self):
+        return '%s, parity:%s, min:%s, max:%s → %s' % (
+            self.street_id,
+            dict(PARITY_CHOICES).get(self.parity),
+            self.min_housenumber,
+            self.max_housenumber,
+            self.sector,
+        )
+
+    @property
+    def resource(self):
+        return self.sector.resource
+
+    def clean(self):
+        if not self.max_housenumber or self.max_housenumber > MAX_HOUSENUMBER:
+            self.max_housenumber = MAX_HOUSENUMBER
+        if self.min_housenumber > self.max_housenumber:
+            raise ValidationError(_('Minimal house number may not be lesser than maximal house number.'))
diff --git a/passerelle/settings.py b/passerelle/settings.py
index 3750c8f0..b1519089 100644
--- a/passerelle/settings.py
+++ b/passerelle/settings.py
@@ -158,6 +158,7 @@ INSTALLED_APPS = (
     'passerelle.apps.oxyd',
     'passerelle.apps.phonecalls',
     'passerelle.apps.photon',
+    'passerelle.apps.sector',
     'passerelle.apps.solis',
     'passerelle.apps.twilio',
     'passerelle.apps.vivaticket',
diff --git a/tests/test_sector.py b/tests/test_sector.py
new file mode 100644
index 00000000..c2baef3b
--- /dev/null
+++ b/tests/test_sector.py
@@ -0,0 +1,383 @@
+# passerelle - uniform access to multiple data sources and services
+# Copyright (C) 2021 Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import datetime
+import os
+from posix import stat_result
+from stat import ST_MTIME
+
+import pytest
+import utils
+from django.core.exceptions import ValidationError
+from django.core.files import File
+from django.core.management import call_command
+from django.urls import reverse
+from django.utils.encoding import force_str, force_text
+from django.utils.six import StringIO
+from django.utils.timezone import now
+
+from passerelle.apps.sector.models import Sectorization, SectorResource
+from passerelle.base.models import AccessRight
+
+CSV = """street_id,parity,min_housenumber,max_housenumber,sector_id,sector_name
+75114_1912,P,,, gs-moulin, Groupe Scolaire Moulin
+75114_1912,I,0,999999,gs-zola,Groupe Scolaire Zola
+75114_1913,N,0,999999,ecole-hugo,École Hugo
+75114_1914,,,10,ecole-hugo, École Hugo
+
+75114_1914,,11,, ecole-hugo2, École Hugo 2
+75114_1915,,,,ecole-hugo2 , École Hugo 2
+"""
+
+CSV_BOM = force_str(force_text(CSV, 'utf-8').encode('utf-8-sig'))
+
+CSV_NO_FIRST_LINE = """75114_1912,P,,, gs-moulin, Groupe Scolaire Moulin
+75114_1912,I,0,999999,gs-zola,Groupe Scolaire Zola
+75114_1913,N,0,999999,ecole-hugo,École Hugo
+75114_1914,,,10,ecole-hugo, École Hugo
+
+75114_1914,,11,, ecole-hugo, École Hugo"""
+
+CSV_REORDERED = """sector_id,sector_name,street_id,parity,min_housenumber,max_housenumber,foo,bar
+gs-moulin, Groupe Scolaire Moulin, 75114_1912,P,,,aaa,bbb
+gs-zola,Groupe Scolaire Zola,75114_1912,I,0,999999,xxx,yyy
+ecole-hugo,École Hugo,75114_1913,N,0,999999,000,1
+,,75114_1999,N,0,999999,,
+"""
+
+CSV_MISSING_COLUMN = """street_id,min_housenumber,max_housenumber,sector_id,sector_name
+75114_1912,,,foo,
+,0,999999,gs-zola,Groupe Scolaire Zola"""
+
+CSV_MISSING_SECTOR = """street_id,parity,min_housenumber,max_housenumber,sector_id,sector_name
+75114_1912,P,,, ,
+75114_1912,I,0,999999,gs-zola,Groupe Scolaire Zola"""
+
+CSV_MISSING_STREET = """street_id,parity,min_housenumber,max_housenumber,sector_id,sector_name
+75114_1912,P,,,foo,
+,I,0,999999,gs-zola,Groupe Scolaire Zola"""
+
+
+@pytest.fixture
+def sector(db):
+    return utils.setup_access_rights(
+        SectorResource.objects.create(
+            slug='test',
+            title='title',
+            csv_file=File(StringIO(CSV), 'sectorization.csv'),
+        )
+    )
+
+
+def test_sector_creation(sector):
+    assert sector.sector_set.count() == 0
+    assert '%s' % sector == 'title [test]'
+
+    sector.clean()
+    assert sector.sector_set.count() == 4
+    hugo = sector.sector_set.get(slug='ecole-hugo')
+    assert Sectorization.objects.filter(sector=hugo).count() == 2
+    hugo2 = sector.sector_set.get(slug='gs-zola')
+    assert Sectorization.objects.filter(sector=hugo2).count() == 1
+
+
+def test_sector_creation_bom(sector):
+    sector.clean()
+    assert sector.sector_set.count() == 4
+
+
+def test_sector_creation_nofirstline(sector):
+    sector.csv_file = File(StringIO(CSV_NO_FIRST_LINE), 'sectorization.csv')
+    with pytest.raises(ValidationError, match='Invalid CSV file:.*missing column'):
+        sector.clean()
+    assert sector.sector_set.count() == 0
+    sector.titles_in_first_line = False
+    sector.clean()
+    assert sector.sector_set.count() == 3
+
+
+def test_sector_reordered(sector):
+    sector.csv_file = File(StringIO(CSV_REORDERED), 'sectorization.csv')
+    sector.clean()
+    assert sector.sector_set.count() == 3
+
+
+def test_sector_empty_file(sector):
+    sector.csv_file = File(StringIO(''), 'sectorization.csv')
+    with pytest.raises(ValidationError, match='Invalid CSV file:.*failed to read CSV'):
+        sector.clean()
+
+
+def test_sector_missing_sector(sector):
+    sector.csv_file = File(StringIO(CSV_MISSING_SECTOR), 'sectorization.csv')
+    with pytest.raises(ValidationError, match='Invalid CSV file:.*missing sector_id, line 2'):
+        sector.clean()
+
+
+def test_sector_missing_street(sector):
+    sector.csv_file = File(StringIO(CSV_MISSING_STREET), 'sectorization.csv')
+    with pytest.raises(ValidationError, match='Invalid CSV file:.*missing street_id, line 3'):
+        sector.clean()
+
+
+def test_sector_missing_column(sector):
+    sector.csv_file = File(StringIO(CSV_MISSING_COLUMN), 'sectorization.csv')
+    with pytest.raises(ValidationError, match='Invalid CSV file:.*missing column.*: parity\.'):
+        sector.clean()
+
+
+def test_sector_endpoint_sectors(app, sector):
+    url = reverse(
+        'generic-endpoint',
+        kwargs={
+            'connector': 'sector',
+            'slug': sector.slug,
+            'endpoint': 'sectors',
+        },
+    )
+    result = app.get(url).json
+    assert result['err'] == 0
+    assert len(result['data']) == 0
+    sector.clean()
+    result = app.get(url).json
+    assert result['err'] == 0
+    assert len(result['data']) == 4
+    assert {'id': 'ecole-hugo', 'text': 'École Hugo'} in result['data']
+
+    result = app.get(url, params={'id': 'ecole-hugo'}).json
+    assert result['err'] == 0
+    assert len(result['data']) == 1
+    assert {'id': 'ecole-hugo', 'text': 'École Hugo'} in result['data']
+
+    result = app.get(url, params={'q': 'hugo'}).json
+    assert result['err'] == 0
+    assert len(result['data']) == 2
+    assert {'id': 'ecole-hugo', 'text': 'École Hugo'} in result['data']
+    assert {'id': 'ecole-hugo2', 'text': 'École Hugo 2'} in result['data']
+
+    result = app.get(url, params={'q': 'foobar'}).json
+    assert result['err'] == 0
+    assert len(result['data']) == 0
+
+    # search a sector by street and house number
+    result = app.get(url, params={'street_id': '75114_1915'}).json
+    assert result['err'] == 0
+    assert result['data'] == [{'id': 'ecole-hugo2', 'text': 'École Hugo 2'}]
+    result = app.get(url, params={'street_id': '75114_1915', 'house_number': '123'}).json
+    assert result['err'] == 0
+    assert result['data'] == [{'id': 'ecole-hugo2', 'text': 'École Hugo 2'}]
+    result = app.get(url, params={'street_id': '75114_1912', 'house_number': '12'}).json  # even
+    assert result['err'] == 0
+    assert result['data'] == [{'id': 'gs-moulin', 'text': 'Groupe Scolaire Moulin'}]
+    result = app.get(url, params={'street_id': '75114_1912', 'house_number': '13'}).json  # odd
+    assert result['err'] == 0
+    assert result['data'] == [{'id': 'gs-zola', 'text': 'Groupe Scolaire Zola'}]
+    result = app.get(url, params={'street_id': '75114_1914', 'house_number': '5'}).json  # 5 <= 10
+    assert result['err'] == 0
+    assert result['data'] == [{'id': 'ecole-hugo', 'text': 'École Hugo'}]
+    result = app.get(url, params={'street_id': '75114_1914', 'house_number': '20'}).json  # 20 >= 11
+    assert result['err'] == 0
+    assert result['data'] == [{'id': 'ecole-hugo2', 'text': 'École Hugo 2'}]
+    # bad searches
+    result = app.get(url, params={'street_id': '75114_1915', 'house_number': 'abc'}, status=400).json
+    assert result['err'] == 1
+    assert result['err_desc'] == 'invalid value for parameter "house_number"'  # not an integer
+    result = app.get(url, params={'house_number': '123'}, status=400).json
+    assert result['err'] == 1
+    assert result['err_desc'] == 'house_number requires a street_id'
+
+    # access right is needed
+    AccessRight.objects.all().delete()
+    result = app.get(url, status=403).json
+    assert result['err'] == 1
+    assert 'PermissionDenied' in result['err_class']
+    assert result['data'] is None
+
+
+def test_sector_endpoint_export(app, sector):
+    url = reverse(
+        'generic-endpoint',
+        kwargs={
+            'connector': 'sector',
+            'slug': sector.slug,
+            'endpoint': 'export',
+        },
+    )
+    resp = app.get(url)
+    assert resp.headers['content-type'] == 'text/csv'
+    assert resp.text.startswith(
+        '"street_id","parity","min_housenumber","max_housenumber","sector_id","sector_name"'
+    )
+    assert len(resp.text.splitlines()) == 1
+    sector.titles_in_first_line = False
+    sector.save()
+    resp = app.get(url)
+    assert resp.headers['content-type'] == 'text/csv'
+    assert resp.text == ''
+    sector.titles_in_first_line = True
+    sector.save()
+
+    sector.clean()
+    resp = app.get(url)
+    assert resp.headers['content-type'] == 'text/csv'
+    # "street_id","parity","min_housenumber","max_housenumber","sector_id","sector_name"
+    # "75114_1913","all","","","ecole-hugo","École Hugo"
+    # "75114_1914","all","","10","",""
+    # "75114_1915","all","","","ecole-hugo2","École Hugo 2"
+    # "75114_1914","all","11","","",""
+    # "75114_1912","odd","","","gs-zola","Groupe Scolaire Zola"
+    # "75114_1912","even","","","gs-moulin","Groupe Scolaire Moulin"
+    assert len(resp.text.splitlines()) == 7
+    assert resp.text.count('"all"') == 4
+    assert resp.text.count('"odd"') == 1
+    assert resp.text.count('"even"') == 1
+    assert resp.text.count('"ecole-hugo"') == 1
+    assert resp.text.count('"0"') == 0
+    assert resp.text.count('"999999"') == 0
+
+    # import -> export again
+    initial_export = resp.text
+    sector.csv_file = File(StringIO(initial_export), 'data.csv')
+    sector.clean()
+    resp = app.get(url)
+    assert resp.text == initial_export
+
+    # modify export format
+    resp = app.get(
+        url, params={'odd': 'IMPAIRS', 'even': 'PAIRS', 'mix': 'TOUS', 'repeat': 'true', 'limits': 'true'}
+    )
+    assert len(resp.text.splitlines()) == 7
+    assert resp.text.count('"TOUS"') == 4
+    assert resp.text.count('"IMPAIRS"') == 1
+    assert resp.text.count('"PAIRS"') == 1
+    assert resp.text.count('"ecole-hugo"') == 2  # repeat
+    assert resp.text.count('"0"') == 5  # limits
+    assert resp.text.count('"999999"') == 5
+
+    # access right is needed
+    AccessRight.objects.all().delete()
+    result = app.get(url, status=403).json
+    assert result['err'] == 1
+    assert 'PermissionDenied' in result['err_class']
+    assert result['data'] is None
+
+
+def test_sector_endpoint_update(app, sector):
+    url = reverse(
+        'generic-endpoint',
+        kwargs={
+            'connector': 'sector',
+            'slug': sector.slug,
+            'endpoint': 'update',
+        },
+    )
+    sector.clean()
+    assert sector.sector_set.count() == 4
+    result = app.put(url, params=CSV_REORDERED, headers={'Content-Type': 'text/csv'}).json
+    assert sector.sector_set.count() == 3
+    assert result['err'] == 0
+    assert len(result['data']) == 3
+    assert result['updated'] == 'sector/test/api-uploaded-file.csv'
+
+    result = app.put(url, params=CSV_MISSING_COLUMN, headers={'Content-Type': 'text/csv'}, status=400).json
+    assert result['err'] == 1
+    assert "missing column" in result['err_desc']
+
+    result = app.put(url, params=CSV_REORDERED, headers={}, status=400).json
+    assert result['err'] == 1
+    assert "can't guess filename extension" in result['err_desc']
+
+    # access right is needed
+    AccessRight.objects.all().delete()
+    result = app.put(url, params=CSV_REORDERED, headers={'Content-Type': 'text/csv'}, status=403).json
+    assert result['err'] == 1
+    assert 'PermissionDenied' in result['err_class']
+
+
+@pytest.mark.parametrize('remove_files', [False, True])
+def test_daily_clean(settings, remove_files, sector):
+    settings.SECTOR_REMOVE_ON_CLEAN = remove_files
+
+    sectordata_dir = os.path.dirname(sector.csv_file.path)
+    other_dir = os.path.join(settings.MEDIA_ROOT, 'foo', sector.slug)
+    os.makedirs(other_dir)
+
+    # create additional file in sector dir
+    with open(os.path.join(sectordata_dir, 'csv-file.csv'), 'w'):
+        pass
+    os.makedirs(os.path.join(sectordata_dir, 'not-a-file'))
+    # create additional file in other dir
+    with open(os.path.join(other_dir, 'csv-file.csv'), 'w'):
+        pass
+
+    call_command('cron', 'daily')
+
+    # not changed
+    assert os.listdir(other_dir) == ['csv-file.csv']
+    # too soon to be removed
+    dir_list = os.listdir(sectordata_dir)
+    dir_list.sort()
+    assert dir_list == ['csv-file.csv', 'not-a-file', 'sectorization.csv']
+
+    orig_os_stat = os.stat
+
+    def _fake_stat(arg, delta):
+        faked = list(orig_os_stat(arg))
+        faked[ST_MTIME] = (now() + delta).timestamp()
+        return stat_result(faked)
+
+    try:
+        # 1 week ago but one minute too soon
+        os.stat = lambda arg: _fake_stat(arg, datetime.timedelta(days=-7, minutes=1))
+        call_command('cron', 'daily')
+
+        # not changed
+        assert os.listdir(other_dir) == ['csv-file.csv']
+        # still too soon to be removed
+        dir_list = os.listdir(sectordata_dir)
+        dir_list.sort()
+        assert dir_list == ['csv-file.csv', 'not-a-file', 'sectorization.csv']
+
+        # 1 week ago
+        os.stat = lambda arg: _fake_stat(arg, datetime.timedelta(days=-7))
+        call_command('cron', 'daily')
+
+        # not changed
+        assert os.listdir(other_dir) == ['csv-file.csv']
+        # removed or moved
+        dir_list = os.listdir(sectordata_dir)
+        dir_list.sort()
+        if remove_files:
+            assert dir_list == ['not-a-file', 'sectorization.csv']
+        else:
+            assert dir_list == ['not-a-file', 'sectorization.csv', 'unused-files']
+            assert os.listdir(os.path.join(sectordata_dir, 'unused-files')) == ['csv-file.csv']
+
+        # wrong storage directory, do nothing
+        with open(os.path.join(other_dir, 'bar.csv'), 'w'):
+            pass
+        sector.csv_file.name = 'foo/%s/csv-file.csv' % sector.slug
+        sector.save()
+        assert sorted(os.listdir(other_dir)) == ['bar.csv', 'csv-file.csv']
+        call_command('cron', 'daily')
+        assert sorted(os.listdir(other_dir)) == ['bar.csv', 'csv-file.csv']
+
+        # unknown file
+        sector.csv_file.name = 'sector/%s/bar.csv' % sector.slug
+        sector.save()
+        call_command('cron', 'daily')
+    finally:
+        os.stat = orig_os_stat
-- 
2.30.2

