From da9b818451dc39ab528e98349dccfb09e366387a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Laur=C3=A9line=20Gu=C3=A9rin?= <zebuline@entrouvert.com>
Date: Mon, 20 Jun 2022 16:24:28 +0200
Subject: [PATCH] ban import: revert last optimizations (#66342)

This reverts commit 7479f1d1430713de4304f1801590c80d5981b408.
---
 passerelle/apps/base_adresse/models.py | 108 ++++++++-----------------
 tests/test_base_adresse.py             |   1 -
 2 files changed, 32 insertions(+), 77 deletions(-)

diff --git a/passerelle/apps/base_adresse/models.py b/passerelle/apps/base_adresse/models.py
index fe55071a..496028a3 100644
--- a/passerelle/apps/base_adresse/models.py
+++ b/passerelle/apps/base_adresse/models.py
@@ -4,9 +4,7 @@ import json
 from io import StringIO
 from urllib import parse as urlparse
 
-from dateutil.parser import parse as parsedate
 from django.contrib.postgres.fields import JSONField
-from django.core.exceptions import ObjectDoesNotExist
 from django.db import connection, models
 from django.db.models import Q
 from django.utils import timezone
@@ -425,6 +423,8 @@ class BaseAdresse(AddressResource):
         if not self.get_zipcodes():
             return
 
+        start_update = timezone.now()
+
         zipcodes = self.get_zipcodes()
         departments = set()
         for zipcode in zipcodes:
@@ -436,7 +436,6 @@ class BaseAdresse(AddressResource):
             else:
                 departments.add(zipcode[:2])
 
-        seen_citycodes = set()
         for department in departments:
             try:
                 ban_gz = self.requests.get(
@@ -448,14 +447,6 @@ class BaseAdresse(AddressResource):
                 continue
             if ban_gz.status_code != 200:
                 continue
-            if "Last-Modified" in ban_gz.headers:
-                # import file only if it is less than 36 hours old...
-                # 36 hours: 1 day + 12 hours, since 12 hours is what it takes nowadays (2022-06-01) to import the BAN data...
-                last_modified = parsedate(ban_gz.headers['Last-Modified'])
-                if datetime.datetime.now(tz=last_modified.tzinfo) - last_modified > datetime.timedelta(
-                    hours=36
-                ):
-                    continue
 
             ban_file = StringIO(gzip.decompress(ban_gz.content).decode('utf-8'))
             line = _not_found = object()
@@ -477,37 +468,21 @@ class BaseAdresse(AddressResource):
                         continue
                 elif citycode != street_info['citycode']:
                     continue
-                seen_citycodes.add(citycode)
-                try:
-                    current_street = self.streetmodel_set.get(ban_id=ban_id)
-                    changed = False
-                    values = {
-                        "name": street_info['name'][:150],
-                        "city": street_info['city'],
-                        "zipcode": street_info['postcode'],
-                        "citycode": citycode,
-                        "type": street_info['type'],
-                    }
-                    for (k, v) in values.items():
-                        if getattr(current_street, k) != v:
-                            setattr(current_street, k, v)
-                            changed = True
-                    if changed:
-                        current_street.save()
-                except ObjectDoesNotExist:
-                    self.streetmodel_set.create(
-                        resource=self,
-                        citycode=citycode,
-                        name=street_info['name'][:150],
-                        ban_id=ban_id,
-                        city=street_info['city'],
-                        zipcode=street_info['postcode'],
-                        type=street_info['type'],
-                    )
+                self.streetmodel_set.update_or_create(
+                    resource=self,
+                    citycode=citycode,
+                    name=street_info['name'][:150],
+                    defaults={
+                        'ban_id': ban_id,
+                        'city': street_info['city'],
+                        'zipcode': street_info['postcode'],
+                        'type': street_info['type'],
+                    },
+                )
             if line is _not_found:
                 raise Exception('bano file is empty')
 
-        self.get_streets_queryset().exclude(citycode__in=seen_citycodes).delete()
+        self.get_streets_queryset().filter(last_update__lt=start_update).delete()
 
     def get_api_geo_endpoint(self, endpoint):
         if not self.api_geo_url:
@@ -538,54 +513,35 @@ class BaseAdresse(AddressResource):
         cities_json = self.get_api_geo_endpoint('communes')
         if not (regions_json and departments_json and cities_json):
             return
+        start_update = timezone.now()
 
-        regions_to_keep = set()
         for data in regions_json:
-            regions_to_keep.add(data['code'])
-            keys = {"code": data["code"]}
-            attributes = {"name": data["nom"]}
-            try:
-                self.regionmodel_set.get(**keys, **attributes)
-            except ObjectDoesNotExist:
-                self.regionmodel_set.update_or_create(**keys, defaults=attributes)
-        self.regionmodel_set.exclude(code__in=regions_to_keep).delete()
+            defaults = {
+                'name': data['nom'],
+            }
+            self.regionmodel_set.update_or_create(code=data['code'], defaults=defaults)
+        self.regionmodel_set.filter(last_update__lt=start_update).delete()
 
-        departments_to_keep = set()
         for data in departments_json:
-            departments_to_keep.add(data['code'])
-            keys = {"code": data["code"]}
-            attributes = {
-                "name": data['nom'],
-                "region": self.regionmodel_set.get(code=data["codeRegion"]),
+            defaults = {
+                'name': data['nom'],
+                'region': self.regionmodel_set.get(code=data['codeRegion']),
             }
-            try:
-                self.departmentmodel_set.get(**keys, **attributes)
-            except ObjectDoesNotExist:
-                self.departmentmodel_set.update_or_create(**keys, defaults=attributes)
-        self.departmentmodel_set.exclude(code__in=departments_to_keep).delete()
+            self.departmentmodel_set.update_or_create(code=data['code'], defaults=defaults)
+        self.departmentmodel_set.filter(last_update__lt=start_update).delete()
 
-        cities_to_keep = set()
         for data in cities_json:
             for zipcode in data['codesPostaux']:
-                keys = {"code": data["code"], "zipcode": zipcode}
-                attributes = {
-                    "population": data.get('population', 0),
-                    "name": data["nom"],
+                defaults = {
+                    'name': data['nom'],
+                    'population': data.get('population', 0),
                 }
                 if data.get('codeDepartement'):
-                    attributes["department"] = self.departmentmodel_set.get(code=data['codeDepartement'])
+                    defaults['department'] = self.departmentmodel_set.get(code=data['codeDepartement'])
                 if data.get('codeRegion'):
-                    attributes["region"] = self.regionmodel_set.get(code=data['codeRegion'])
-
-                try:
-                    current_city = self.citymodel_set.get(**keys, **attributes)
-                    cities_to_keep.add(current_city.id)
-                except ObjectDoesNotExist:
-                    new_city, _ = self.citymodel_set.update_or_create(**keys, defaults=attributes)
-
-                    cities_to_keep.add(new_city.id)
-
-        self.citymodel_set.exclude(id__in=cities_to_keep).delete()
+                    defaults['region'] = self.regionmodel_set.get(code=data['codeRegion'])
+                self.citymodel_set.update_or_create(code=data['code'], zipcode=zipcode, defaults=defaults)
+        self.citymodel_set.filter(last_update__lt=start_update).delete()
 
     def clean_addresses_cache(self):
         old_addresses = self.addresscachemodel_set.filter(
diff --git a/tests/test_base_adresse.py b/tests/test_base_adresse.py
index 458b0167..43b5c563 100644
--- a/tests/test_base_adresse.py
+++ b/tests/test_base_adresse.py
@@ -394,7 +394,6 @@ def test_base_adresse_streets_get_by_id(app, base_adresse, street):
 def test_base_adresse_streets_get_by_codes(app, base_adresse, street):
     for i in range(20):
         StreetModel.objects.create(
-            ban_id="%s_%s" % (75000 + i, i),
             city='Paris %d' % i,
             name='La rue %d' % i,
             zipcode=str(75000 + i * 10),
-- 
2.35.1

