0001-base_adresse-add-addresses-endpoint-39387.patch
passerelle/apps/base_adresse/migrations/0016_addresscachemodel.py | ||
---|---|---|
1 |
# -*- coding: utf-8 -*- |
|
2 |
# Generated by Django 1.11.18 on 2020-01-30 11:34 |
|
3 |
from __future__ import unicode_literals |
|
4 | ||
5 |
from django.db import migrations, models |
|
6 |
import jsonfield.fields |
|
7 | ||
8 | ||
9 |
class Migration(migrations.Migration): |
|
10 | ||
11 |
dependencies = [ |
|
12 |
('base_adresse', '0015_auto_20191206_1244'), |
|
13 |
] |
|
14 | ||
15 |
operations = [ |
|
16 |
migrations.CreateModel( |
|
17 |
name='AddressCacheModel', |
|
18 |
fields=[ |
|
19 |
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), |
|
20 |
('api_id', models.CharField(max_length=30, unique=True)), |
|
21 |
('data', jsonfield.fields.JSONField(default=dict)), |
|
22 |
('timestamp', models.DateTimeField(auto_now=True)), |
|
23 |
], |
|
24 |
), |
|
25 |
] |
passerelle/apps/base_adresse/models.py | ||
---|---|---|
1 | 1 |
import bz2 |
2 |
import datetime |
|
2 | 3 |
import unicodedata |
3 | 4 | |
5 |
from jsonfield import JSONField |
|
4 | 6 |
from requests import RequestException |
5 | 7 | |
6 | 8 |
from django.db import connection, models |
... | ... | |
49 | 51 |
class Meta: |
50 | 52 |
verbose_name = _('Base Adresse Web Service') |
51 | 53 | |
54 |
@staticmethod |
|
55 |
def format_address_data(data): |
|
56 |
result = {} |
|
57 |
result['lon'] = str(data['geometry']['coordinates'][0]) |
|
58 |
result['lat'] = str(data['geometry']['coordinates'][1]) |
|
59 |
result['address'] = {'country': 'France'} |
|
60 |
for prop, value in data['properties'].items(): |
|
61 |
if prop in ('city', 'postcode', 'citycode'): |
|
62 |
result['address'][prop] = value |
|
63 |
elif prop == 'housenumber': |
|
64 |
result['address']['house_number'] = value |
|
65 |
elif prop == 'label': |
|
66 |
result['text'] = result['display_name'] = value |
|
67 |
elif prop == 'name': |
|
68 |
house_number = data['properties'].get('housenumber') |
|
69 |
if house_number and value.startswith(house_number): |
|
70 |
value = value[len(house_number):].strip() |
|
71 |
result['address']['road'] = value |
|
72 |
elif prop == 'id': |
|
73 |
result['id'] = value |
|
74 |
return result |
|
75 | ||
52 | 76 |
@endpoint(pattern='(?P<q>.+)?$', |
53 |
description=_('Geocoding'),
|
|
77 |
description=_('Addresses list'),
|
|
54 | 78 |
parameters={ |
55 |
'q': {'description': _('Address'), 'example_value': '169 rue du chateau, paris'} |
|
79 |
'id': {'description': _('Address identifier')}, |
|
80 |
'q': {'description': _('Address'), 'example_value': '169 rue du chateau, paris'}, |
|
81 |
'page_limit': {'description': _('Maximum number of results to return. Must be ' |
|
82 |
'lower than 20.')}, |
|
83 |
'zipcode': {'description': _('Zipcode'), 'example_value': '75014'}, |
|
84 |
'lat': {'description': _('Prioritize results according to coordinates. "lat" ' |
|
85 |
'parameter must be present.')}, |
|
86 |
'lon': {'description': _('Prioritize results according to coordinates. "lon" ' |
|
87 |
'parameter must be present.')}, |
|
56 | 88 |
}) |
57 |
def search(self, request, q, zipcode='', lat=None, lon=None, **kwargs): |
|
58 |
if kwargs.get('format', 'json') != 'json': |
|
59 |
raise NotImplementedError() |
|
89 |
def addresses(self, request, id=None, q=None, zipcode='', lat=None, lon=None, page_limit=5): |
|
90 |
if id is not None: |
|
91 |
try: |
|
92 |
address = AddressCacheModel.objects.get(api_id=id) |
|
93 |
except AddressCacheModel.DoesNotExist: |
|
94 |
return {'err': _('Address ID not found')} |
|
95 |
address.update_timestamp() |
|
96 |
return {'data': [address.data]} |
|
60 | 97 | |
61 | 98 |
if not q: |
62 |
return [] |
|
99 |
return {'data': []} |
|
100 | ||
101 |
try: |
|
102 |
if int(page_limit) > 20: |
|
103 |
page_limit = 20 |
|
104 |
except ValueError: |
|
105 |
page_limit = 5 |
|
63 | 106 | |
64 | 107 |
scheme, netloc, path, params, query, fragment = urlparse.urlparse(self.service_url) |
65 | 108 |
path = urlparse.urljoin(path, 'search/') |
66 |
query_args = {'q': q, 'limit': 1}
|
|
109 |
query_args = {'q': q, 'limit': page_limit}
|
|
67 | 110 |
if zipcode: |
68 | 111 |
query_args['postcode'] = zipcode |
69 | 112 |
if lat and lon: |
... | ... | |
78 | 121 |
for feature in result_response.json().get('features'): |
79 | 122 |
if not feature['geometry']['type'] == 'Point': |
80 | 123 |
continue # skip unknown |
81 |
result.append({ |
|
82 |
'lon': str(feature['geometry']['coordinates'][0]), |
|
83 |
'lat': str(feature['geometry']['coordinates'][1]), |
|
84 |
'display_name': feature['properties']['label'], |
|
85 |
}) |
|
86 |
break |
|
124 |
data = self.format_address_data(feature) |
|
125 |
result.append(data) |
|
126 |
AddressCacheModel.objects.update_or_create(api_id=data['id'], data=data) |
|
87 | 127 | |
88 |
return result |
|
128 |
return {'data': result} |
|
129 | ||
130 |
@endpoint(pattern='(?P<q>.+)?$', description=_('Geocoding (Nominatim API)'), |
|
131 |
parameters={ |
|
132 |
'q': {'description': _('Address'), 'example_value': '169 rue du chateau, paris'}, |
|
133 |
}) |
|
134 |
def search(self, request, q, zipcode='', lat=None, lon=None, **kwargs): |
|
135 |
if kwargs.get('format', 'json') != 'json': |
|
136 |
raise NotImplementedError() |
|
137 |
result = self.addresses(request, q=q, zipcode=zipcode, lat=lat, lon=lon, page_limit=1) |
|
138 |
return result['data'] |
|
89 | 139 | |
90 | 140 |
@endpoint(description=_('Reverse geocoding'), |
91 | 141 |
parameters={ |
... | ... | |
107 | 157 |
for feature in result_response.json().get('features'): |
108 | 158 |
if not feature['geometry']['type'] == 'Point': |
109 | 159 |
continue # skip unknown |
110 |
result = {} |
|
111 |
result['lon'] = str(feature['geometry']['coordinates'][0]) |
|
112 |
result['lat'] = str(feature['geometry']['coordinates'][1]) |
|
113 |
result['address'] = {'country': 'France'} |
|
114 |
for prop in feature['properties']: |
|
115 |
if prop in ('city', 'postcode', 'citycode'): |
|
116 |
result['address'][prop] = feature['properties'][prop] |
|
117 |
elif prop == 'housenumber': |
|
118 |
result['address']['house_number'] = feature['properties'][prop] |
|
119 |
elif prop == 'label': |
|
120 |
result['display_name'] = feature['properties'][prop] |
|
121 |
elif prop == 'name': |
|
122 |
house_number = feature['properties'].get('housenumber') |
|
123 |
value = feature['properties'][prop] |
|
124 |
if house_number and value.startswith(house_number): |
|
125 |
value = value[len(house_number):].strip() |
|
126 |
result['address']['road'] = value |
|
160 |
result = self.format_address_data(feature) |
|
127 | 161 |
return result |
128 | 162 | |
129 | 163 |
@endpoint(description=_('Streets from zipcode'), |
... | ... | |
368 | 402 |
code=data['code'], zipcode=zipcode, defaults=defaults) |
369 | 403 |
CityModel.objects.filter(last_update__lt=start_update).delete() |
370 | 404 | |
405 |
def clean_addresses_cache(self): |
|
406 |
old_addresses = AddressCacheModel.objects.filter( |
|
407 |
timestamp__lt=timezone.now() - datetime.timedelta(hours=1) |
|
408 |
) |
|
409 |
old_addresses.delete() |
|
410 | ||
371 | 411 |
def hourly(self): |
372 | 412 |
super(BaseAdresse, self).hourly() |
413 |
self.clean_addresses_cache() |
|
373 | 414 |
# don't wait for daily job to grab data |
374 | 415 |
if self.get_zipcodes() and not self.get_streets_queryset().exists(): |
375 | 416 |
self.update_streets_data() |
... | ... | |
489 | 530 | |
490 | 531 |
def __str__(self): |
491 | 532 |
return '%s %s' % (self.zipcode, self.name) |
533 | ||
534 | ||
535 |
class AddressCacheModel(models.Model): |
|
536 |
api_id = models.CharField(max_length=30, unique=True) |
|
537 |
data = JSONField() |
|
538 |
timestamp = models.DateTimeField(auto_now=True) |
|
539 | ||
540 |
def update_timestamp(self): |
|
541 |
self.save() |
tests/conftest.py | ||
---|---|---|
1 | 1 |
import pytest |
2 |
from httmock import urlmatch, HTTMock, response |
|
2 |
from httmock import urlmatch, HTTMock, response, remember_called
|
|
3 | 3 | |
4 | 4 |
import django_webtest |
5 | 5 | |
... | ... | |
25 | 25 | |
26 | 26 | |
27 | 27 |
@urlmatch(netloc='^api-adresse.data.gouv.fr$', path='^/search/$') |
28 |
@remember_called |
|
28 | 29 |
def api_adresse_data_gouv_fr_search(url, request): |
29 | 30 |
return response(200, { |
30 | 31 |
"limit": 1, |
... | ... | |
95 | 96 |
@pytest.yield_fixture |
96 | 97 |
def mock_api_adresse_data_gouv_fr_search(): |
97 | 98 |
with HTTMock(api_adresse_data_gouv_fr_search): |
98 |
yield None
|
|
99 |
yield api_adresse_data_gouv_fr_search
|
|
99 | 100 | |
100 | 101 | |
101 | 102 |
@pytest.yield_fixture |
tests/test_base_adresse.py | ||
---|---|---|
1 | 1 |
# -*- coding: utf-8 -*- |
2 | 2 | |
3 |
import datetime |
|
3 | 4 |
import os |
4 | 5 |
import pytest |
5 | 6 |
import mock |
... | ... | |
13 | 14 |
from django.utils.six.moves.urllib.parse import urljoin |
14 | 15 | |
15 | 16 |
from passerelle.apps.base_adresse.models import (BaseAdresse, StreetModel, CityModel, |
16 |
DepartmentModel, RegionModel) |
|
17 |
DepartmentModel, RegionModel, AddressCacheModel)
|
|
17 | 18 | |
18 | 19 |
FAKED_CONTENT = json.dumps({ |
19 | 20 |
"limit": 1, |
... | ... | |
604 | 605 |
call_command('cron', 'daily') |
605 | 606 |
assert mocked_get.call_count == 3 |
606 | 607 |
assert not RegionModel.objects.exists() |
608 | ||
609 | ||
610 |
@mock.patch('passerelle.utils.Request.get') |
|
611 |
def test_base_adresse_addresses(mocked_get, app, base_adresse): |
|
612 |
endpoint = utils.generic_endpoint_url('base-adresse', 'addresses', slug=base_adresse.slug) |
|
613 |
assert endpoint == '/base-adresse/base-adresse/addresses' |
|
614 |
mocked_get.return_value = utils.FakedResponse(content=FAKED_CONTENT, status_code=200) |
|
615 |
resp = app.get(endpoint, params={'q': 'plop'}, status=200) |
|
616 |
data = resp.json['data'][0] |
|
617 |
assert data['lat'] == '47.474633' |
|
618 |
assert data['lon'] == '-0.593775' |
|
619 |
assert data['display_name'] == 'Rue Roger Halope 49000 Angers' |
|
620 |
assert data['text'] == 'Rue Roger Halope 49000 Angers' |
|
621 |
assert data['id'] == '49007_6950_be54bd' |
|
622 |
assert data['address']['city'] == 'Angers' |
|
623 |
assert data['address']['postcode'] == '49000' |
|
624 |
assert data['address']['citycode'] == '49007' |
|
625 |
assert data['address']['road'] == 'Rue Roger Halope' |
|
626 | ||
627 | ||
628 |
@mock.patch('passerelle.utils.Request.get') |
|
629 |
def test_base_adresse_addresses_qs_page_limit(mocked_get, app, base_adresse): |
|
630 |
resp = app.get('/base-adresse/%s/addresses?q=plop&page_limit=1' % base_adresse.slug) |
|
631 |
assert 'limit=1' in mocked_get.call_args[0][0] |
|
632 | ||
633 |
resp = app.get('/base-adresse/%s/addresses?q=plop&page_limit=100' % base_adresse.slug) |
|
634 |
assert 'limit=20' in mocked_get.call_args[0][0] |
|
635 | ||
636 |
resp = app.get('/base-adresse/%s/addresses?q=plop&page_limit=blabla' % base_adresse.slug) |
|
637 |
assert 'limit=5' in mocked_get.call_args[0][0] |
|
638 | ||
639 | ||
640 |
def test_base_adresse_addresses_cache(app, base_adresse, mock_api_adresse_data_gouv_fr_search): |
|
641 |
resp = app.get('/base-adresse/%s/addresses?q=plop' % base_adresse.slug) |
|
642 |
assert mock_api_adresse_data_gouv_fr_search.call['count'] == 1 |
|
643 | ||
644 |
data = resp.json['data'][0] |
|
645 |
assert data['text'] == 'Rue Roger Halope 49000 Angers' |
|
646 | ||
647 |
api_id = data['id'] |
|
648 |
assert AddressCacheModel.objects.filter(api_id=api_id).exists() |
|
649 |
assert AddressCacheModel.objects.count() == 1 |
|
650 | ||
651 |
resp = app.get('/base-adresse/%s/addresses?id=%s' % (base_adresse.slug, api_id)) |
|
652 |
assert mock_api_adresse_data_gouv_fr_search.call['count'] == 1 # no new call |
|
653 |
assert data['text'] == 'Rue Roger Halope 49000 Angers' |
|
654 |
assert 'address' in data |
|
655 | ||
656 |
resp = app.get('/base-adresse/%s/addresses?q=plop' % base_adresse.slug) |
|
657 |
assert AddressCacheModel.objects.count() == 1 # no new object has been created |
|
658 | ||
659 | ||
660 |
def test_base_adresse_addresses_cache_err(app, base_adresse, |
|
661 |
mock_api_adresse_data_gouv_fr_search): |
|
662 |
resp = app.get('/base-adresse/%s/addresses?id=%s' % (base_adresse.slug, 'wrong_id')) |
|
663 |
assert mock_api_adresse_data_gouv_fr_search.call['count'] == 0 |
|
664 |
assert 'err' in resp.json |
|
665 | ||
666 | ||
667 |
@pytest.mark.usefixtures('mock_update_api_geo', 'mock_update_streets') |
|
668 |
def test_base_adresse_addresses_clean_cache(app, base_adresse, freezer, |
|
669 |
mock_api_adresse_data_gouv_fr_search): |
|
670 |
resp = app.get('/base-adresse/%s/addresses?q=plop' % base_adresse.slug) |
|
671 |
assert AddressCacheModel.objects.count() == 1 |
|
672 | ||
673 |
freezer.move_to(datetime.timedelta(minutes=30)) |
|
674 |
call_command('cron', 'hourly') |
|
675 |
assert AddressCacheModel.objects.count() == 1 |
|
676 | ||
677 |
freezer.move_to(datetime.timedelta(minutes=30, seconds=1)) |
|
678 |
call_command('cron', 'hourly') |
|
679 |
assert AddressCacheModel.objects.count() == 0 |
|
680 | ||
681 |
resp = app.get('/base-adresse/%s/addresses?q=plop' % base_adresse.slug) |
|
682 |
assert AddressCacheModel.objects.count() == 1 |
|
683 | ||
684 |
# asking for the address again resets the timestamp |
|
685 |
freezer.move_to(datetime.timedelta(hours=1, seconds=1)) |
|
686 |
resp = app.get('/base-adresse/%s/addresses?q=plop' % base_adresse.slug) |
|
687 |
call_command('cron', 'hourly') |
|
688 |
assert AddressCacheModel.objects.count() == 1 |
|
689 | ||
690 |
freezer.move_to(datetime.timedelta(hours=1, seconds=1)) |
|
691 |
resp = app.get('/base-adresse/%s/addresses?id=%s' % (base_adresse.slug, '49007_6950_be54bd')) |
|
692 |
call_command('cron', 'hourly') |
|
693 |
assert AddressCacheModel.objects.count() == 1 |
|
607 |
- |