0001-base_adresse-add-addresses-endpoint-39387.patch
passerelle/apps/base_adresse/migrations/0016_auto_20200130_1604.py | ||
---|---|---|
1 |
# -*- coding: utf-8 -*- |
|
2 |
# Generated by Django 1.11.18 on 2020-01-30 15:04 |
|
3 |
from __future__ import unicode_literals |
|
4 | ||
5 |
from django.db import migrations, models |
|
6 |
import jsonfield.fields |
|
7 | ||
8 | ||
9 |
class Migration(migrations.Migration): |
|
10 | ||
11 |
dependencies = [ |
|
12 |
('base_adresse', '0015_auto_20191206_1244'), |
|
13 |
] |
|
14 | ||
15 |
operations = [ |
|
16 |
migrations.CreateModel( |
|
17 |
name='AddressCacheModel', |
|
18 |
fields=[ |
|
19 |
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), |
|
20 |
('api_id', models.CharField(max_length=30, unique=True)), |
|
21 |
('data', jsonfield.fields.JSONField(default=dict)), |
|
22 |
('timestamp', models.DateTimeField(auto_now=True)), |
|
23 |
], |
|
24 |
), |
|
25 |
migrations.AddField( |
|
26 |
model_name='baseadresse', |
|
27 |
name='latitude', |
|
28 |
field=models.FloatField(blank=True, help_text='Geographic priority for /addresses/ endpoint.', null=True, verbose_name='Latitude'), |
|
29 |
), |
|
30 |
migrations.AddField( |
|
31 |
model_name='baseadresse', |
|
32 |
name='longitude', |
|
33 |
field=models.FloatField(blank=True, help_text='Geographic priority for /addresses/ endpoint.', null=True, verbose_name='Longitude'), |
|
34 |
), |
|
35 |
] |
passerelle/apps/base_adresse/models.py | ||
---|---|---|
1 | 1 |
import bz2 |
2 |
import datetime |
|
2 | 3 |
import unicodedata |
3 | 4 | |
5 |
from jsonfield import JSONField |
|
4 | 6 |
from requests import RequestException |
5 | 7 | |
6 | 8 |
from django.db import connection, models |
... | ... | |
46 | 48 |
blank=True, |
47 | 49 |
verbose_name=_('Postal codes or department number to get streets, separated with commas')) |
48 | 50 | |
51 |
latitude = models.FloatField( |
|
52 |
null=True, blank=True, |
|
53 |
verbose_name=_('Latitude'), |
|
54 |
help_text=_('Geographic priority for /addresses/ endpoint.'), |
|
55 |
) |
|
56 |
longitude = models.FloatField( |
|
57 |
null=True, blank=True, |
|
58 |
verbose_name=_('Longitude'), |
|
59 |
help_text=_('Geographic priority for /addresses/ endpoint.'), |
|
60 |
) |
|
61 | ||
49 | 62 |
class Meta: |
50 | 63 |
verbose_name = _('Base Adresse Web Service') |
51 | 64 | |
65 |
@staticmethod |
|
66 |
def format_address_data(data): |
|
67 |
result = {} |
|
68 |
result['lon'] = str(data['geometry']['coordinates'][0]) |
|
69 |
result['lat'] = str(data['geometry']['coordinates'][1]) |
|
70 |
result['address'] = {'country': 'France'} |
|
71 |
for prop, value in data['properties'].items(): |
|
72 |
if prop in ('city', 'postcode', 'citycode'): |
|
73 |
result['address'][prop] = value |
|
74 |
elif prop == 'housenumber': |
|
75 |
result['address']['house_number'] = value |
|
76 |
elif prop == 'label': |
|
77 |
result['text'] = result['display_name'] = value |
|
78 |
elif prop == 'name': |
|
79 |
house_number = data['properties'].get('housenumber') |
|
80 |
if house_number and value.startswith(house_number): |
|
81 |
value = value[len(house_number):].strip() |
|
82 |
result['address']['road'] = value |
|
83 |
elif prop == 'id': |
|
84 |
result['id'] = value |
|
85 |
return result |
|
86 | ||
52 | 87 |
@endpoint(pattern='(?P<q>.+)?$', |
53 |
description=_('Geocoding'),
|
|
88 |
description=_('Addresses list'),
|
|
54 | 89 |
parameters={ |
55 |
'q': {'description': _('Address'), 'example_value': '169 rue du chateau, paris'} |
|
90 |
'id': {'description': _('Address identifier')}, |
|
91 |
'q': {'description': _('Address'), 'example_value': '169 rue du chateau, paris'}, |
|
92 |
'page_limit': {'description': _('Maximum number of results to return. Must be ' |
|
93 |
'lower than 20.')}, |
|
94 |
'zipcode': {'description': _('Zipcode'), 'example_value': '75014'}, |
|
95 |
'lat': {'description': _('Prioritize results according to coordinates. "lat" ' |
|
96 |
'parameter must be present.')}, |
|
97 |
'lon': {'description': _('Prioritize results according to coordinates. "lon" ' |
|
98 |
'parameter must be present.')}, |
|
56 | 99 |
}) |
57 |
def search(self, request, q, zipcode='', lat=None, lon=None, **kwargs): |
|
58 |
if kwargs.get('format', 'json') != 'json': |
|
59 |
raise NotImplementedError() |
|
100 |
def addresses(self, request, id=None, q=None, zipcode='', lat=None, lon=None, page_limit=5): |
|
101 |
if id is not None: |
|
102 |
try: |
|
103 |
address = AddressCacheModel.objects.get(api_id=id) |
|
104 |
except AddressCacheModel.DoesNotExist: |
|
105 |
return {'err': _('Address ID not found')} |
|
106 |
address.update_timestamp() |
|
107 |
return {'data': [address.data]} |
|
60 | 108 | |
61 | 109 |
if not q: |
62 |
return [] |
|
110 |
return {'data': []} |
|
111 | ||
112 |
try: |
|
113 |
if int(page_limit) > 20: |
|
114 |
page_limit = 20 |
|
115 |
except ValueError: |
|
116 |
page_limit = 5 |
|
63 | 117 | |
64 | 118 |
scheme, netloc, path, params, query, fragment = urlparse.urlparse(self.service_url) |
65 | 119 |
path = urlparse.urljoin(path, 'search/') |
66 |
query_args = {'q': q, 'limit': 1}
|
|
120 |
query_args = {'q': q, 'limit': page_limit}
|
|
67 | 121 |
if zipcode: |
68 | 122 |
query_args['postcode'] = zipcode |
69 |
if lat and lon: |
|
70 |
query_args['lat'] = lat |
|
71 |
query_args['lon'] = lon |
|
123 |
if self.latitude and self.longitude or lat and lon:
|
|
124 |
query_args['lat'] = lat or self.latitude
|
|
125 |
query_args['lon'] = lon or self.longitude
|
|
72 | 126 |
query = urlencode(query_args) |
73 | 127 |
url = urlparse.urlunparse((scheme, netloc, path, params, query, fragment)) |
74 | 128 | |
... | ... | |
78 | 132 |
for feature in result_response.json().get('features'): |
79 | 133 |
if not feature['geometry']['type'] == 'Point': |
80 | 134 |
continue # skip unknown |
81 |
result.append({ |
|
82 |
'lon': str(feature['geometry']['coordinates'][0]), |
|
83 |
'lat': str(feature['geometry']['coordinates'][1]), |
|
84 |
'display_name': feature['properties']['label'], |
|
85 |
}) |
|
86 |
break |
|
135 |
data = self.format_address_data(feature) |
|
136 |
result.append(data) |
|
137 |
AddressCacheModel.objects.update_or_create(api_id=data['id'], data=data) |
|
87 | 138 | |
88 |
return result |
|
139 |
return {'data': result} |
|
140 | ||
141 |
@endpoint(pattern='(?P<q>.+)?$', description=_('Geocoding (Nominatim API)'), |
|
142 |
parameters={ |
|
143 |
'q': {'description': _('Address'), 'example_value': '169 rue du chateau, paris'}, |
|
144 |
}) |
|
145 |
def search(self, request, q, zipcode='', lat=None, lon=None, **kwargs): |
|
146 |
if kwargs.get('format', 'json') != 'json': |
|
147 |
raise NotImplementedError() |
|
148 |
result = self.addresses(request, q=q, zipcode=zipcode, lat=lat, lon=lon, page_limit=1) |
|
149 |
return result['data'] |
|
89 | 150 | |
90 | 151 |
@endpoint(description=_('Reverse geocoding'), |
91 | 152 |
parameters={ |
... | ... | |
107 | 168 |
for feature in result_response.json().get('features'): |
108 | 169 |
if not feature['geometry']['type'] == 'Point': |
109 | 170 |
continue # skip unknown |
110 |
result = {} |
|
111 |
result['lon'] = str(feature['geometry']['coordinates'][0]) |
|
112 |
result['lat'] = str(feature['geometry']['coordinates'][1]) |
|
113 |
result['address'] = {'country': 'France'} |
|
114 |
for prop in feature['properties']: |
|
115 |
if prop in ('city', 'postcode', 'citycode'): |
|
116 |
result['address'][prop] = feature['properties'][prop] |
|
117 |
elif prop == 'housenumber': |
|
118 |
result['address']['house_number'] = feature['properties'][prop] |
|
119 |
elif prop == 'label': |
|
120 |
result['display_name'] = feature['properties'][prop] |
|
121 |
elif prop == 'name': |
|
122 |
house_number = feature['properties'].get('housenumber') |
|
123 |
value = feature['properties'][prop] |
|
124 |
if house_number and value.startswith(house_number): |
|
125 |
value = value[len(house_number):].strip() |
|
126 |
result['address']['road'] = value |
|
171 |
result = self.format_address_data(feature) |
|
127 | 172 |
return result |
128 | 173 | |
129 | 174 |
@endpoint(description=_('Streets from zipcode'), |
... | ... | |
368 | 413 |
code=data['code'], zipcode=zipcode, defaults=defaults) |
369 | 414 |
CityModel.objects.filter(last_update__lt=start_update).delete() |
370 | 415 | |
416 |
def clean_addresses_cache(self): |
|
417 |
old_addresses = AddressCacheModel.objects.filter( |
|
418 |
timestamp__lt=timezone.now() - datetime.timedelta(hours=1) |
|
419 |
) |
|
420 |
old_addresses.delete() |
|
421 | ||
371 | 422 |
def hourly(self): |
372 | 423 |
super(BaseAdresse, self).hourly() |
424 |
self.clean_addresses_cache() |
|
373 | 425 |
# don't wait for daily job to grab data |
374 | 426 |
if self.get_zipcodes() and not self.get_streets_queryset().exists(): |
375 | 427 |
self.update_streets_data() |
... | ... | |
489 | 541 | |
490 | 542 |
def __str__(self): |
491 | 543 |
return '%s %s' % (self.zipcode, self.name) |
544 | ||
545 | ||
546 |
class AddressCacheModel(models.Model): |
|
547 |
api_id = models.CharField(max_length=30, unique=True) |
|
548 |
data = JSONField() |
|
549 |
timestamp = models.DateTimeField(auto_now=True) |
|
550 | ||
551 |
def update_timestamp(self): |
|
552 |
self.save() |
tests/conftest.py | ||
---|---|---|
1 | 1 |
import pytest |
2 |
from httmock import urlmatch, HTTMock, response |
|
2 |
from httmock import urlmatch, HTTMock, response, remember_called
|
|
3 | 3 | |
4 | 4 |
import django_webtest |
5 | 5 | |
... | ... | |
25 | 25 | |
26 | 26 | |
27 | 27 |
@urlmatch(netloc='^api-adresse.data.gouv.fr$', path='^/search/$') |
28 |
@remember_called |
|
28 | 29 |
def api_adresse_data_gouv_fr_search(url, request): |
29 | 30 |
return response(200, { |
30 | 31 |
"limit": 1, |
... | ... | |
95 | 96 |
@pytest.yield_fixture |
96 | 97 |
def mock_api_adresse_data_gouv_fr_search(): |
97 | 98 |
with HTTMock(api_adresse_data_gouv_fr_search): |
98 |
yield None
|
|
99 |
yield api_adresse_data_gouv_fr_search
|
|
99 | 100 | |
100 | 101 | |
101 | 102 |
@pytest.yield_fixture |
tests/test_base_adresse.py | ||
---|---|---|
1 | 1 |
# -*- coding: utf-8 -*- |
2 | 2 | |
3 |
import datetime |
|
3 | 4 |
import os |
4 | 5 |
import pytest |
5 | 6 |
import mock |
... | ... | |
13 | 14 |
from django.utils.six.moves.urllib.parse import urljoin |
14 | 15 | |
15 | 16 |
from passerelle.apps.base_adresse.models import (BaseAdresse, StreetModel, CityModel, |
16 |
DepartmentModel, RegionModel) |
|
17 |
DepartmentModel, RegionModel, AddressCacheModel)
|
|
17 | 18 | |
18 | 19 |
FAKED_CONTENT = json.dumps({ |
19 | 20 |
"limit": 1, |
... | ... | |
118 | 119 |
zipcode='73, 73100, 97425,20000 ')) |
119 | 120 | |
120 | 121 | |
122 |
@pytest.fixture |
|
123 |
def base_adresse_coordinates(db): |
|
124 |
return utils.setup_access_rights(BaseAdresse.objects.create(slug='base-adresse', |
|
125 |
latitude=1.2, longitude=2.1)) |
|
126 | ||
127 | ||
121 | 128 |
@pytest.fixture |
122 | 129 |
def street(db): |
123 | 130 |
return StreetModel.objects.create(city=u'Chambéry', |
... | ... | |
604 | 611 |
call_command('cron', 'daily') |
605 | 612 |
assert mocked_get.call_count == 3 |
606 | 613 |
assert not RegionModel.objects.exists() |
614 | ||
615 | ||
616 |
@mock.patch('passerelle.utils.Request.get') |
|
617 |
def test_base_adresse_addresses(mocked_get, app, base_adresse): |
|
618 |
endpoint = utils.generic_endpoint_url('base-adresse', 'addresses', slug=base_adresse.slug) |
|
619 |
assert endpoint == '/base-adresse/base-adresse/addresses' |
|
620 |
mocked_get.return_value = utils.FakedResponse(content=FAKED_CONTENT, status_code=200) |
|
621 |
resp = app.get(endpoint, params={'q': 'plop'}, status=200) |
|
622 |
data = resp.json['data'][0] |
|
623 |
assert data['lat'] == '47.474633' |
|
624 |
assert data['lon'] == '-0.593775' |
|
625 |
assert data['display_name'] == 'Rue Roger Halope 49000 Angers' |
|
626 |
assert data['text'] == 'Rue Roger Halope 49000 Angers' |
|
627 |
assert data['id'] == '49007_6950_be54bd' |
|
628 |
assert data['address']['city'] == 'Angers' |
|
629 |
assert data['address']['postcode'] == '49000' |
|
630 |
assert data['address']['citycode'] == '49007' |
|
631 |
assert data['address']['road'] == 'Rue Roger Halope' |
|
632 | ||
633 | ||
634 |
@mock.patch('passerelle.utils.Request.get') |
|
635 |
def test_base_adresse_addresses_qs_page_limit(mocked_get, app, base_adresse): |
|
636 |
resp = app.get('/base-adresse/%s/addresses?q=plop&page_limit=1' % base_adresse.slug) |
|
637 |
assert 'limit=1' in mocked_get.call_args[0][0] |
|
638 | ||
639 |
resp = app.get('/base-adresse/%s/addresses?q=plop&page_limit=100' % base_adresse.slug) |
|
640 |
assert 'limit=20' in mocked_get.call_args[0][0] |
|
641 | ||
642 |
resp = app.get('/base-adresse/%s/addresses?q=plop&page_limit=blabla' % base_adresse.slug) |
|
643 |
assert 'limit=5' in mocked_get.call_args[0][0] |
|
644 | ||
645 | ||
646 |
@mock.patch('passerelle.utils.Request.get') |
|
647 |
def test_base_adresse_addresses_qs_coordinates(mocked_get, app, base_adresse_coordinates): |
|
648 |
resp = app.get('/base-adresse/%s/addresses?q=plop' % base_adresse_coordinates.slug) |
|
649 |
assert 'lat=%s' % base_adresse_coordinates.latitude in mocked_get.call_args[0][0] |
|
650 |
assert 'lon=%s' % base_adresse_coordinates.longitude in mocked_get.call_args[0][0] |
|
651 | ||
652 |
resp = app.get('/base-adresse/%s/addresses?q=plop&lat=42&lon=43' % base_adresse_coordinates.slug) |
|
653 |
assert 'lat=42' in mocked_get.call_args[0][0] |
|
654 |
assert 'lon=43' in mocked_get.call_args[0][0] |
|
655 | ||
656 | ||
657 |
def test_base_adresse_addresses_cache(app, base_adresse, mock_api_adresse_data_gouv_fr_search): |
|
658 |
resp = app.get('/base-adresse/%s/addresses?q=plop' % base_adresse.slug) |
|
659 |
assert mock_api_adresse_data_gouv_fr_search.call['count'] == 1 |
|
660 | ||
661 |
data = resp.json['data'][0] |
|
662 |
assert data['text'] == 'Rue Roger Halope 49000 Angers' |
|
663 | ||
664 |
api_id = data['id'] |
|
665 |
assert AddressCacheModel.objects.filter(api_id=api_id).exists() |
|
666 |
assert AddressCacheModel.objects.count() == 1 |
|
667 | ||
668 |
resp = app.get('/base-adresse/%s/addresses?id=%s' % (base_adresse.slug, api_id)) |
|
669 |
assert mock_api_adresse_data_gouv_fr_search.call['count'] == 1 # no new call |
|
670 |
assert data['text'] == 'Rue Roger Halope 49000 Angers' |
|
671 |
assert 'address' in data |
|
672 | ||
673 |
resp = app.get('/base-adresse/%s/addresses?q=plop' % base_adresse.slug) |
|
674 |
assert AddressCacheModel.objects.count() == 1 # no new object has been created |
|
675 | ||
676 | ||
677 |
def test_base_adresse_addresses_cache_err(app, base_adresse, |
|
678 |
mock_api_adresse_data_gouv_fr_search): |
|
679 |
resp = app.get('/base-adresse/%s/addresses?id=%s' % (base_adresse.slug, 'wrong_id')) |
|
680 |
assert mock_api_adresse_data_gouv_fr_search.call['count'] == 0 |
|
681 |
assert 'err' in resp.json |
|
682 | ||
683 | ||
684 |
@pytest.mark.usefixtures('mock_update_api_geo', 'mock_update_streets') |
|
685 |
def test_base_adresse_addresses_clean_cache(app, base_adresse, freezer, |
|
686 |
mock_api_adresse_data_gouv_fr_search): |
|
687 |
resp = app.get('/base-adresse/%s/addresses?q=plop' % base_adresse.slug) |
|
688 |
assert AddressCacheModel.objects.count() == 1 |
|
689 | ||
690 |
freezer.move_to(datetime.timedelta(minutes=30)) |
|
691 |
call_command('cron', 'hourly') |
|
692 |
assert AddressCacheModel.objects.count() == 1 |
|
693 | ||
694 |
freezer.move_to(datetime.timedelta(minutes=30, seconds=1)) |
|
695 |
call_command('cron', 'hourly') |
|
696 |
assert AddressCacheModel.objects.count() == 0 |
|
697 | ||
698 |
resp = app.get('/base-adresse/%s/addresses?q=plop' % base_adresse.slug) |
|
699 |
assert AddressCacheModel.objects.count() == 1 |
|
700 | ||
701 |
# asking for the address again resets the timestamp |
|
702 |
freezer.move_to(datetime.timedelta(hours=1, seconds=1)) |
|
703 |
resp = app.get('/base-adresse/%s/addresses?q=plop' % base_adresse.slug) |
|
704 |
call_command('cron', 'hourly') |
|
705 |
assert AddressCacheModel.objects.count() == 1 |
|
706 | ||
707 |
freezer.move_to(datetime.timedelta(hours=1, seconds=1)) |
|
708 |
resp = app.get('/base-adresse/%s/addresses?id=%s' % (base_adresse.slug, '49007_6950_be54bd')) |
|
709 |
call_command('cron', 'hourly') |
|
710 |
assert AddressCacheModel.objects.count() == 1 |
|
607 |
- |