Projet

Général

Profil

0001-base_adresse-use-new-ban-data-38204.patch

Lauréline Guérin, 28 avril 2020 10:48

Télécharger (15,2 ko)

Voir les différences: 

Subject: [PATCH] base_adresse: use new ban data (#38204)


 passerelle/apps/base_adresse/models.py |  31 +++++++++--------
 tests/data/update_streets_test.bz2     | Bin 956 -> 0 bytes
 tests/data/update_streets_test.gz      | Bin 0 -> 867 bytes
 tests/test_base_adresse.py             |  44 ++++++++++++++-----------
 tests/test_jobs.py                     |  13 +++++---
 5 files changed, 51 insertions(+), 37 deletions(-)
 delete mode 100644 tests/data/update_streets_test.bz2
 create mode 100644 tests/data/update_streets_test.gz
passerelle/apps/base_adresse/models.py
1 
import bz2
2 1 
import datetime
2 
import gzip
3 3 

  		




  4
  4
  
    
from jsonfield import JSONField

  		




  5
  5
  
    
from requests import RequestException

  		




  ......




  10
  10
  
    
from django.utils.translation import ugettext_lazy as _

  		




  11
  11
  
    
from django.utils import timezone

  		




  12
  12
  
    
from django.utils import six

  		




  
  13
  
    
from django.utils.six import StringIO

  		




  13
  14
  
    
from django.utils.six.moves.urllib import parse as urlparse

  		




  14
  15
  
    

  		




  15
  16
  
    
from passerelle.base.models import BaseResource

  		




  ......




  354
  355
  
    
                departments.add(zipcode[:2])

  		




  355
  356
  
    

  		




  356
  357
  
    
        for department in departments:

  		




  357
  
  
    
            ban_file = self.requests.get(

  		




  358
  
  
    
                'http://bano.openstreetmap.fr/BAN_odbl/BAN_odbl_{}-json.bz2'.format(department))

  		




  359
  
  
    
            if ban_file.status_code != 200:

  		




  
  358
  
    
            ban_gz = self.requests.get(

  		




  
  359
  
    
                'https://adresse.data.gouv.fr/data/ban/adresses/latest/addok/adresses-addok-{}.ndjson.gz'.format(department))

  		




  
  360
  
    
            if ban_gz.status_code != 200:

  		




  360
  361
  
    
                continue

  		




  361
  362
  
    

  		




  
  363
  
    
            if six.PY3:

  		




  
  364
  
    
                ban_file = StringIO(gzip.decompress(ban_gz.content).decode('utf-8'))

  		




  
  365
  
    
            else:

  		




  
  366
  
    
                ban_file = gzip.GzipFile(fileobj=StringIO(ban_gz.content))

  		




  362
  367
  
    
            line = _not_found = object()

  		




  363
  
  
    
            for line in bz2.decompress(ban_file.content).splitlines():

  		




  
  368
  
    
            for line in ban_file:

  		




  364
  369
  
    
                street_info = json_loads(line)

  		




  365
  370
  
    
                if street_info['type'] == 'street' and street_info['postcode'].startswith(zipcodes):

  		




  366
  371
  
    
                    if type(street_info['citycode']) is list:

  		




  367
  372
  
    
                        street_info['citycode'] = six.text_type(street_info['citycode'][0])

  		




  368
  373
  
    
                    if type(street_info['name']) is list:

  		




  369
  374
  
    
                        street_info['name'] = six.text_type(street_info['name'][0])

  		




  370
  
  
    
                    street = StreetModel.objects.get_or_create(citycode=street_info['citycode'],

  		




  371
  
  
    
                                                               name=street_info['name'][:150])

  		




  372
  
  
    
                    street[0].city = street_info['city']

  		




  373
  
  
    
                    street[0].name = street_info['name'][:150]

  		




  374
  
  
    
                    street[0].zipcode = street_info['postcode']

  		




  375
  
  
    
                    street[0].type = street_info['type']

  		




  376
  
  
    
                    street[0].citycode = street_info['citycode']

  		




  377
  
  
    
                    street[0].save()

  		




  
  375
  
    
                    StreetModel.objects.update_or_create(

  		




  
  376
  
    
                        citycode=street_info['citycode'],

  		




  
  377
  
    
                        name=street_info['name'][:150],

  		




  
  378
  
    
                        defaults={

  		




  
  379
  
    
                            'city': street_info['city'],

  		




  
  380
  
    
                            'zipcode': street_info['postcode'],

  		




  
  381
  
    
                            'type': street_info['type'],

  		




  
  382
  
    
                        })

  		




  378
  383
  
    
            if line is _not_found:

  		




  379
  384
  
    
                raise Exception('bano file is empty')

  		




  380
  385
  
    

  		












  

    
      tests/test_base_adresse.py
    
  





  348
  348
  
    
    assert resp.json['err'] == 0

  		




  349
  349
  
    
    assert len(resp.json['data']) == 0

  		




  350
  350
  
    

  		




  
  351
  
    

  		




  351
  352
  
    
@pytest.mark.usefixtures('mock_update_api_geo')

  		




  352
  353
  
    
@mock.patch('passerelle.utils.Request.get')

  		




  353
  354
  
    
def test_base_adresse_command_update(mocked_get, db, base_adresse):

  		




  354
  
  
    
    filepath = os.path.join(os.path.dirname(__file__), 'data', 'update_streets_test.bz2')

  		




  355
  
  
    
    mocked_get.return_value = utils.FakedResponse(content=open(filepath, 'rb').read(), status_code=200)

  		




  
  355
  
    
    filepath = os.path.join(os.path.dirname(__file__), 'data', 'update_streets_test.gz')

  		




  
  356
  
    
    with open(filepath, 'rb') as ban_file:

  		




  
  357
  
    
        mocked_get.return_value = utils.FakedResponse(content=ban_file.read(), status_code=200)

  		




  356
  358
  
    
    call_command('cron', 'daily')

  		




  357
  
  
    
    mocked_get.assert_called_once_with('http://bano.openstreetmap.fr/BAN_odbl/BAN_odbl_73-json.bz2')

  		




  
  359
  
    
    mocked_get.assert_called_once_with('https://adresse.data.gouv.fr/data/ban/adresses/latest/addok/adresses-addok-73.ndjson.gz')

  		




  358
  360
  
    
    streets = StreetModel.objects.all()

  		




  359
  361
  
    
    assert len(streets) == 3

  		




  360
  362
  
    
    street = StreetModel.objects.order_by('id').first()

  		




  ......




  372
  374
  
    
@mock.patch('passerelle.utils.Request.get')

  		




  373
  375
  
    
def test_base_adresse_command_hourly_update(mocked_get, db, base_adresse):

  		




  374
  376
  
    
    base_adresse.update_api_geo_data = lambda: None

  		




  375
  
  
    
    filepath = os.path.join(os.path.dirname(__file__), 'data', 'update_streets_test.bz2')

  		




  376
  
  
    
    mocked_get.return_value = utils.FakedResponse(content=open(filepath, 'rb').read(), status_code=200)

  		




  
  377
  
    
    filepath = os.path.join(os.path.dirname(__file__), 'data', 'update_streets_test.gz')

  		




  
  378
  
    
    with open(filepath, 'rb') as ban_file:

  		




  
  379
  
    
        mocked_get.return_value = utils.FakedResponse(content=ban_file.read(), status_code=200)

  		




  377
  380
  
    
    # check the first hourly job downloads streets

  		




  378
  381
  
    
    call_command('cron', 'hourly')

  		




  379
  
  
    
    mocked_get.assert_called_once_with('http://bano.openstreetmap.fr/BAN_odbl/BAN_odbl_73-json.bz2')

  		




  
  382
  
    
    mocked_get.assert_called_once_with('https://adresse.data.gouv.fr/data/ban/adresses/latest/addok/adresses-addok-73.ndjson.gz')

  		




  380
  383
  
    
    assert StreetModel.objects.all().count() == 3

  		




  381
  384
  
    
    # check a second call doesn't download anything

  		




  382
  385
  
    
    call_command('cron', 'hourly')

  		




  ......




  387
  390
  
    
@mock.patch('passerelle.utils.Request.get')

  		




  388
  391
  
    
def test_base_adresse_command_update_97x(mocked_get, db, base_adresse_97x):

  		




  389
  392
  
    
    base_adresse_97x.update_api_geo_data = lambda: None

  		




  390
  
  
    
    filepath = os.path.join(os.path.dirname(__file__), 'data', 'update_streets_test.bz2')

  		




  391
  
  
    
    mocked_get.return_value = utils.FakedResponse(content=open(filepath, 'rb').read(), status_code=200)

  		




  
  393
  
    
    filepath = os.path.join(os.path.dirname(__file__), 'data', 'update_streets_test.gz')

  		




  
  394
  
    
    with open(filepath, 'rb') as ban_file:

  		




  
  395
  
    
        mocked_get.return_value = utils.FakedResponse(content=ban_file.read(), status_code=200)

  		




  392
  396
  
    
    call_command('cron', 'daily')

  		




  393
  
  
    
    mocked_get.assert_called_once_with('http://bano.openstreetmap.fr/BAN_odbl/BAN_odbl_974-json.bz2')

  		




  
  397
  
    
    mocked_get.assert_called_once_with('https://adresse.data.gouv.fr/data/ban/adresses/latest/addok/adresses-addok-974.ndjson.gz')

  		




  394
  398
  
    
    assert StreetModel.objects.count() == 2

  		




  395
  399
  
    

  		




  396
  400
  
    

  		




  ......




  398
  402
  
    
@mock.patch('passerelle.utils.Request.get')

  		




  399
  403
  
    
def test_base_adresse_command_update_corsica(mocked_get, db, base_adresse_corsica):

  		




  400
  404
  
    
    base_adresse_corsica.update_api_geo_data = lambda: None

  		




  401
  
  
    
    filepath = os.path.join(os.path.dirname(__file__), 'data', 'update_streets_test.bz2')

  		




  402
  
  
    
    mocked_get.return_value = utils.FakedResponse(content=open(filepath, 'rb').read(), status_code=200)

  		




  
  405
  
    
    filepath = os.path.join(os.path.dirname(__file__), 'data', 'update_streets_test.gz')

  		




  
  406
  
    
    with open(filepath, 'rb') as ban_file:

  		




  
  407
  
    
        mocked_get.return_value = utils.FakedResponse(content=ban_file.read(), status_code=200)

  		




  403
  408
  
    
    call_command('cron', 'daily')

  		




  404
  409
  
    
    assert mocked_get.call_count == 2

  		




  405
  
  
    
    mocked_get.assert_any_call('http://bano.openstreetmap.fr/BAN_odbl/BAN_odbl_2A-json.bz2')

  		




  406
  
  
    
    mocked_get.assert_any_call('http://bano.openstreetmap.fr/BAN_odbl/BAN_odbl_2B-json.bz2')

  		




  
  410
  
    
    mocked_get.assert_any_call('https://adresse.data.gouv.fr/data/ban/adresses/latest/addok/adresses-addok-2A.ndjson.gz')

  		




  
  411
  
    
    mocked_get.assert_any_call('https://adresse.data.gouv.fr/data/ban/adresses/latest/addok/adresses-addok-2B.ndjson.gz')

  		




  407
  412
  
    
    assert StreetModel.objects.count() == 0

  		




  408
  413
  
    

  		




  409
  414
  
    

  		




  ......




  411
  416
  
    
@mock.patch('passerelle.utils.Request.get')

  		




  412
  417
  
    
def test_base_adresse_command_update_multiple(mocked_get, db, base_adresse_multiple):

  		




  413
  418
  
    
    base_adresse_multiple.update_api_geo_data = lambda: None

  		




  414
  
  
    
    filepath = os.path.join(os.path.dirname(__file__), 'data', 'update_streets_test.bz2')

  		




  415
  
  
    
    mocked_get.return_value = utils.FakedResponse(content=open(filepath, 'rb').read(), status_code=200)

  		




  
  419
  
    
    filepath = os.path.join(os.path.dirname(__file__), 'data', 'update_streets_test.gz')

  		




  
  420
  
    
    with open(filepath, 'rb') as ban_file:

  		




  
  421
  
    
        mocked_get.return_value = utils.FakedResponse(content=ban_file.read(), status_code=200)

  		




  416
  422
  
    
    call_command('cron', 'daily')

  		




  417
  423
  
    
    assert mocked_get.call_count == 4

  		




  418
  
  
    
    mocked_get.assert_any_call('http://bano.openstreetmap.fr/BAN_odbl/BAN_odbl_73-json.bz2')

  		




  419
  
  
    
    mocked_get.assert_any_call('http://bano.openstreetmap.fr/BAN_odbl/BAN_odbl_974-json.bz2')

  		




  420
  
  
    
    mocked_get.assert_any_call('http://bano.openstreetmap.fr/BAN_odbl/BAN_odbl_2A-json.bz2')

  		




  421
  
  
    
    mocked_get.assert_any_call('http://bano.openstreetmap.fr/BAN_odbl/BAN_odbl_2B-json.bz2')

  		




  
  424
  
    
    mocked_get.assert_any_call('https://adresse.data.gouv.fr/data/ban/adresses/latest/addok/adresses-addok-73.ndjson.gz')

  		




  
  425
  
    
    mocked_get.assert_any_call('https://adresse.data.gouv.fr/data/ban/adresses/latest/addok/adresses-addok-974.ndjson.gz')

  		




  
  426
  
    
    mocked_get.assert_any_call('https://adresse.data.gouv.fr/data/ban/adresses/latest/addok/adresses-addok-2A.ndjson.gz')

  		




  
  427
  
    
    mocked_get.assert_any_call('https://adresse.data.gouv.fr/data/ban/adresses/latest/addok/adresses-addok-2B.ndjson.gz')

  		




  422
  428
  
    
    assert StreetModel.objects.count() == 5

  		




  423
  429
  
    

  		




  424
  430
  
    

  		












  

    
      tests/test_jobs.py
    
  





  18
  18
  
    

  		




  19
  19
  
    
@mock.patch('passerelle.utils.Request.get')

  		




  20
  20
  
    
def test_jobs(mocked_get, app, base_adresse, freezer):

  		




  21
  
  
    
    filepath = os.path.join(os.path.dirname(__file__), 'data', 'update_streets_test.bz2')

  		




  22
  
  
    
    mocked_get.return_value = utils.FakedResponse(content=open(filepath, 'rb').read(), status_code=200)

  		




  
  21
  
    
    filepath = os.path.join(os.path.dirname(__file__), 'data', 'update_streets_test.gz')

  		




  
  22
  
    
    with open(filepath, 'rb') as ban_file:

  		




  
  23
  
    
        mocked_get.return_value = utils.FakedResponse(content=ban_file.read(), status_code=200)

  		




  23
  24
  
    

  		




  24
  25
  
    
    freezer.move_to('2019-01-01 00:00:00')

  		




  25
  26
  
    
    job = base_adresse.add_job('update_streets_data')

  		




  ......




  30
  31
  
    
    assert StreetModel.objects.count() == 3

  		




  31
  32
  
    

  		




  32
  33
  
    
    # don't delete streets if bano file is empty

  		




  33
  
  
    
    mocked_get.return_value = utils.FakedResponse(content='', status_code=200)

  		




  
  34
  
    
    mocked_get.return_value = utils.FakedResponse(content=b'', status_code=200)

  		




  34
  35
  
    
    freezer.move_to('2019-01-01 12:00:00')

  		




  35
  36
  
    
    job = base_adresse.add_job('update_streets_data')

  		




  36
  37
  
    
    assert job.status == 'registered'

  		




  ......




  43
  44
  
    
        'error running update_streets_data job (bano file is empty)')

  		




  44
  45
  
    
    assert StreetModel.objects.count() == 3

  		




  45
  46
  
    

  		




  46
  
  
    
    mocked_get.return_value = utils.FakedResponse(content=open(filepath, 'rb').read(), status_code=200)

  		




  
  47
  
    
    with open(filepath, 'rb') as ban_file:

  		




  
  48
  
    
        mocked_get.return_value = utils.FakedResponse(content=ban_file.read(), status_code=200)

  		




  47
  49
  
    

  		




  48
  50
  
    
    StreetModel.objects.all().delete()

  		




  49
  51
  
    

  		




  ......




  133
  135
  
    
    StreetModel.objects.all().delete()

  		




  134
  136
  
    
    with mock.patch('passerelle.apps.base_adresse.models.BaseAdresse.down') as down:

  		




  135
  137
  
    
        down.side_effect = lambda: True

  		




  136
  
  
    
        mocked_get.return_value = utils.FakedResponse(content=open(filepath, 'rb').read(), status_code=200)

  		




  
  138
  
    
        with open(filepath, 'rb') as ban_file:

  		




  
  139
  
    
            mocked_get.return_value = utils.FakedResponse(content=ban_file.read(), status_code=200)

  		




  137
  140
  
    
        job = base_adresse.add_job('update_streets_data')

  		




  138
  141
  
    
        assert job.status == 'registered'

  		




  139
  142
  
    

  		




  140
  
  
    
-