From 79ce3d6a70a5dbe053f28fabf57094da892f978d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Laur=C3=A9line=20Gu=C3=A9rin?= <zebuline@entrouvert.com>
Date: Mon, 27 Apr 2020 09:19:28 +0200
Subject: [PATCH] base_adresse: use new ban data (#38204)

---
 passerelle/apps/base_adresse/models.py |  51 ++++++++++++++-----------
 tests/data/update_streets_test.bz2     | Bin 956 -> 0 bytes
 tests/data/update_streets_test.csv.gz  | Bin 0 -> 694 bytes
 tests/test_base_adresse.py             |  46 ++++++++++++----------
 tests/test_jobs.py                     |   7 ++--
 5 files changed, 59 insertions(+), 45 deletions(-)
 delete mode 100644 tests/data/update_streets_test.bz2
 create mode 100644 tests/data/update_streets_test.csv.gz

diff --git a/passerelle/apps/base_adresse/models.py b/passerelle/apps/base_adresse/models.py
index 09d9d942..f282cddd 100644
--- a/passerelle/apps/base_adresse/models.py
+++ b/passerelle/apps/base_adresse/models.py
@@ -1,5 +1,6 @@
-import bz2
+import csv
 import datetime
+import gzip
 
 from jsonfield import JSONField
 from requests import RequestException
@@ -10,10 +11,10 @@ from django.utils.http import urlencode
 from django.utils.translation import ugettext_lazy as _
 from django.utils import timezone
 from django.utils import six
+from django.utils.six import StringIO
 from django.utils.six.moves.urllib import parse as urlparse
 
 from passerelle.base.models import BaseResource
-from passerelle.compat import json_loads
 from passerelle.utils.api import endpoint
 from passerelle.utils.conversion import simplify
 from passerelle.utils.jsonresponse import APIError
@@ -354,28 +355,34 @@ class BaseAdresse(BaseResource):
                 departments.add(zipcode[:2])
 
         for department in departments:
-            ban_file = self.requests.get(
-                'http://bano.openstreetmap.fr/BAN_odbl/BAN_odbl_{}-json.bz2'.format(department))
-            if ban_file.status_code != 200:
+            ban_gz = self.requests.get(
+                'https://adresse.data.gouv.fr/data/ban/adresses/latest/csv/adresses-{}.csv.gz'.format(department))
+            if ban_gz.status_code != 200:
                 continue
 
-            line = _not_found = object()
-            for line in bz2.decompress(ban_file.content).splitlines():
-                street_info = json_loads(line)
-                if street_info['type'] == 'street' and street_info['postcode'].startswith(zipcodes):
-                    if type(street_info['citycode']) is list:
-                        street_info['citycode'] = six.text_type(street_info['citycode'][0])
-                    if type(street_info['name']) is list:
-                        street_info['name'] = six.text_type(street_info['name'][0])
-                    street = StreetModel.objects.get_or_create(citycode=street_info['citycode'],
-                                                               name=street_info['name'][:150])
-                    street[0].city = street_info['city']
-                    street[0].name = street_info['name'][:150]
-                    street[0].zipcode = street_info['postcode']
-                    street[0].type = street_info['type']
-                    street[0].citycode = street_info['citycode']
-                    street[0].save()
-            if line is _not_found:
+            street_info = _not_found = object()
+            if six.PY3:
+                ban_file = StringIO(gzip.decompress(ban_gz.content).decode('utf-8'))
+            else:
+                ban_file = gzip.GzipFile(fileobj=StringIO(ban_gz.content))
+            reader = csv.DictReader(ban_file, delimiter=';')
+            for street_info in reader:
+                if street_info['code_postal'].startswith(zipcodes):
+                    defaults = {
+                        'city': street_info['nom_commune'],
+                        'zipcode': street_info['code_postal'],
+                        'type': 'street',
+                        'citycode': street_info['code_insee']
+                    }
+                    street, created = StreetModel.objects.get_or_create(
+                        citycode=street_info['code_insee'],
+                        name=street_info['nom_voie'][:150],
+                        defaults=defaults)
+                    if not created:
+                        for k, v in defaults.items():
+                            setattr(street, k, v)
+                        street.save()
+            if street_info is _not_found:
                 raise Exception('bano file is empty')
 
         self.get_streets_queryset().filter(last_update__lt=start_update).delete()
diff --git a/tests/data/update_streets_test.bz2 b/tests/data/update_streets_test.bz2
deleted file mode 100644
index bf8e6c80a9327f9f24d18fef5026010926b7823d..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/tests/data/update_streets_test.csv.gz b/tests/data/update_streets_test.csv.gz
new file mode 100644
index 0000000000000000000000000000000000000000..5637d8f86c775de9a6cea9c4d40536b31200a2f0
GIT binary patch
diff --git a/tests/test_base_adresse.py b/tests/test_base_adresse.py
index 186b1a21..88a262ad 100644
--- a/tests/test_base_adresse.py
+++ b/tests/test_base_adresse.py
@@ -348,17 +348,19 @@ def test_base_adresse_streets_get_by_codes(app, base_adresse, street):
     assert resp.json['err'] == 0
     assert len(resp.json['data']) == 0
 
+
 @pytest.mark.usefixtures('mock_update_api_geo')
 @mock.patch('passerelle.utils.Request.get')
 def test_base_adresse_command_update(mocked_get, db, base_adresse):
-    filepath = os.path.join(os.path.dirname(__file__), 'data', 'update_streets_test.bz2')
-    mocked_get.return_value = utils.FakedResponse(content=open(filepath, 'rb').read(), status_code=200)
+    filepath = os.path.join(os.path.dirname(__file__), 'data', 'update_streets_test.csv.gz')
+    with open(filepath, 'rb') as ban_file:
+        mocked_get.return_value = utils.FakedResponse(content=ban_file.read(), status_code=200)
     call_command('cron', 'daily')
-    mocked_get.assert_called_once_with('http://bano.openstreetmap.fr/BAN_odbl/BAN_odbl_73-json.bz2')
+    mocked_get.assert_called_once_with('https://adresse.data.gouv.fr/data/ban/adresses/latest/csv/adresses-73.csv.gz')
     streets = StreetModel.objects.all()
     assert len(streets) == 3
     street = StreetModel.objects.order_by('id').first()
-    assert street.name == 'Chemin de la Vie, LA GRANGE DU TRIEU'
+    assert street.name == 'Chemin de la Vie'
     assert street.zipcode == '73610'
     assert street.type == 'street'
     assert street.city == 'Aiguebelette-le-Lac'
@@ -372,11 +374,12 @@ def test_base_adresse_command_update(mocked_get, db, base_adresse):
 @mock.patch('passerelle.utils.Request.get')
 def test_base_adresse_command_hourly_update(mocked_get, db, base_adresse):
     base_adresse.update_api_geo_data = lambda: None
-    filepath = os.path.join(os.path.dirname(__file__), 'data', 'update_streets_test.bz2')
-    mocked_get.return_value = utils.FakedResponse(content=open(filepath, 'rb').read(), status_code=200)
+    filepath = os.path.join(os.path.dirname(__file__), 'data', 'update_streets_test.csv.gz')
+    with open(filepath, 'rb') as ban_file:
+        mocked_get.return_value = utils.FakedResponse(content=ban_file.read(), status_code=200)
     # check the first hourly job downloads streets
     call_command('cron', 'hourly')
-    mocked_get.assert_called_once_with('http://bano.openstreetmap.fr/BAN_odbl/BAN_odbl_73-json.bz2')
+    mocked_get.assert_called_once_with('https://adresse.data.gouv.fr/data/ban/adresses/latest/csv/adresses-73.csv.gz')
     assert StreetModel.objects.all().count() == 3
     # check a second call doesn't download anything
     call_command('cron', 'hourly')
@@ -387,10 +390,11 @@ def test_base_adresse_command_hourly_update(mocked_get, db, base_adresse):
 @mock.patch('passerelle.utils.Request.get')
 def test_base_adresse_command_update_97x(mocked_get, db, base_adresse_97x):
     base_adresse_97x.update_api_geo_data = lambda: None
-    filepath = os.path.join(os.path.dirname(__file__), 'data', 'update_streets_test.bz2')
-    mocked_get.return_value = utils.FakedResponse(content=open(filepath, 'rb').read(), status_code=200)
+    filepath = os.path.join(os.path.dirname(__file__), 'data', 'update_streets_test.csv.gz')
+    with open(filepath, 'rb') as ban_file:
+        mocked_get.return_value = utils.FakedResponse(content=ban_file.read(), status_code=200)
     call_command('cron', 'daily')
-    mocked_get.assert_called_once_with('http://bano.openstreetmap.fr/BAN_odbl/BAN_odbl_974-json.bz2')
+    mocked_get.assert_called_once_with('https://adresse.data.gouv.fr/data/ban/adresses/latest/csv/adresses-974.csv.gz')
     assert StreetModel.objects.count() == 2
 
 
@@ -398,12 +402,13 @@ def test_base_adresse_command_update_97x(mocked_get, db, base_adresse_97x):
 @mock.patch('passerelle.utils.Request.get')
 def test_base_adresse_command_update_corsica(mocked_get, db, base_adresse_corsica):
     base_adresse_corsica.update_api_geo_data = lambda: None
-    filepath = os.path.join(os.path.dirname(__file__), 'data', 'update_streets_test.bz2')
-    mocked_get.return_value = utils.FakedResponse(content=open(filepath, 'rb').read(), status_code=200)
+    filepath = os.path.join(os.path.dirname(__file__), 'data', 'update_streets_test.csv.gz')
+    with open(filepath, 'rb') as ban_file:
+        mocked_get.return_value = utils.FakedResponse(content=ban_file.read(), status_code=200)
     call_command('cron', 'daily')
     assert mocked_get.call_count == 2
-    mocked_get.assert_any_call('http://bano.openstreetmap.fr/BAN_odbl/BAN_odbl_2A-json.bz2')
-    mocked_get.assert_any_call('http://bano.openstreetmap.fr/BAN_odbl/BAN_odbl_2B-json.bz2')
+    mocked_get.assert_any_call('https://adresse.data.gouv.fr/data/ban/adresses/latest/csv/adresses-2A.csv.gz')
+    mocked_get.assert_any_call('https://adresse.data.gouv.fr/data/ban/adresses/latest/csv/adresses-2B.csv.gz')
     assert StreetModel.objects.count() == 0
 
 
@@ -411,14 +416,15 @@ def test_base_adresse_command_update_corsica(mocked_get, db, base_adresse_corsic
 @mock.patch('passerelle.utils.Request.get')
 def test_base_adresse_command_update_multiple(mocked_get, db, base_adresse_multiple):
     base_adresse_multiple.update_api_geo_data = lambda: None
-    filepath = os.path.join(os.path.dirname(__file__), 'data', 'update_streets_test.bz2')
-    mocked_get.return_value = utils.FakedResponse(content=open(filepath, 'rb').read(), status_code=200)
+    filepath = os.path.join(os.path.dirname(__file__), 'data', 'update_streets_test.csv.gz')
+    with open(filepath, 'rb') as ban_file:
+        mocked_get.return_value = utils.FakedResponse(content=ban_file.read(), status_code=200)
     call_command('cron', 'daily')
     assert mocked_get.call_count == 4
-    mocked_get.assert_any_call('http://bano.openstreetmap.fr/BAN_odbl/BAN_odbl_73-json.bz2')
-    mocked_get.assert_any_call('http://bano.openstreetmap.fr/BAN_odbl/BAN_odbl_974-json.bz2')
-    mocked_get.assert_any_call('http://bano.openstreetmap.fr/BAN_odbl/BAN_odbl_2A-json.bz2')
-    mocked_get.assert_any_call('http://bano.openstreetmap.fr/BAN_odbl/BAN_odbl_2B-json.bz2')
+    mocked_get.assert_any_call('https://adresse.data.gouv.fr/data/ban/adresses/latest/csv/adresses-73.csv.gz')
+    mocked_get.assert_any_call('https://adresse.data.gouv.fr/data/ban/adresses/latest/csv/adresses-974.csv.gz')
+    mocked_get.assert_any_call('https://adresse.data.gouv.fr/data/ban/adresses/latest/csv/adresses-2A.csv.gz')
+    mocked_get.assert_any_call('https://adresse.data.gouv.fr/data/ban/adresses/latest/csv/adresses-2B.csv.gz')
     assert StreetModel.objects.count() == 5
 
 
diff --git a/tests/test_jobs.py b/tests/test_jobs.py
index ac8438d9..bae1dda5 100644
--- a/tests/test_jobs.py
+++ b/tests/test_jobs.py
@@ -18,8 +18,9 @@ from test_base_adresse import base_adresse, StreetModel
 
 @mock.patch('passerelle.utils.Request.get')
 def test_jobs(mocked_get, app, base_adresse, freezer):
-    filepath = os.path.join(os.path.dirname(__file__), 'data', 'update_streets_test.bz2')
-    mocked_get.return_value = utils.FakedResponse(content=open(filepath, 'rb').read(), status_code=200)
+    filepath = os.path.join(os.path.dirname(__file__), 'data', 'update_streets_test.csv.gz')
+    with open(filepath, 'rb') as ban_file:
+        mocked_get.return_value = utils.FakedResponse(content=ban_file.read(), status_code=200)
 
     freezer.move_to('2019-01-01 00:00:00')
     job = base_adresse.add_job('update_streets_data')
@@ -30,7 +31,7 @@ def test_jobs(mocked_get, app, base_adresse, freezer):
     assert StreetModel.objects.count() == 3
 
     # don't delete streets if bano file is empty
-    mocked_get.return_value = utils.FakedResponse(content='', status_code=200)
+    mocked_get.return_value = utils.FakedResponse(content=b'', status_code=200)
     freezer.move_to('2019-01-01 12:00:00')
     job = base_adresse.add_job('update_streets_data')
     assert job.status == 'registered'
-- 
2.20.1

