| 4 |
4 |
from io import StringIO
|
| 5 |
5 |
from urllib import parse as urlparse
|
| 6 |
6 |
|
| 7 |
|
from dateutil.parser import parse as parsedate
|
| 8 |
7 |
from django.contrib.postgres.fields import JSONField
|
| 9 |
|
from django.core.exceptions import ObjectDoesNotExist
|
| 10 |
8 |
from django.db import connection, models
|
| 11 |
9 |
from django.db.models import Q
|
| 12 |
10 |
from django.utils import timezone
|
| ... | ... | |
| 425 |
423 |
if not self.get_zipcodes():
|
| 426 |
424 |
return
|
| 427 |
425 |
|
|
426 |
start_update = timezone.now()
|
|
427 |
|
| 428 |
428 |
zipcodes = self.get_zipcodes()
|
| 429 |
429 |
departments = set()
|
| 430 |
430 |
for zipcode in zipcodes:
|
| ... | ... | |
| 436 |
436 |
else:
|
| 437 |
437 |
departments.add(zipcode[:2])
|
| 438 |
438 |
|
| 439 |
|
seen_citycodes = set()
|
| 440 |
439 |
for department in departments:
|
| 441 |
440 |
try:
|
| 442 |
441 |
ban_gz = self.requests.get(
|
| ... | ... | |
| 448 |
447 |
continue
|
| 449 |
448 |
if ban_gz.status_code != 200:
|
| 450 |
449 |
continue
|
| 451 |
|
if "Last-Modified" in ban_gz.headers:
|
| 452 |
|
# import file only if it is less than 36 hours old...
|
| 453 |
|
# 36 hours: 1 day + 12 hours, since 12 hours is what it takes nowadays (2022-06-01) to import the BAN data...
|
| 454 |
|
last_modified = parsedate(ban_gz.headers['Last-Modified'])
|
| 455 |
|
if datetime.datetime.now(tz=last_modified.tzinfo) - last_modified > datetime.timedelta(
|
| 456 |
|
hours=36
|
| 457 |
|
):
|
| 458 |
|
continue
|
| 459 |
450 |
|
| 460 |
451 |
ban_file = StringIO(gzip.decompress(ban_gz.content).decode('utf-8'))
|
| 461 |
452 |
line = _not_found = object()
|
| ... | ... | |
| 477 |
468 |
continue
|
| 478 |
469 |
elif citycode != street_info['citycode']:
|
| 479 |
470 |
continue
|
| 480 |
|
seen_citycodes.add(citycode)
|
| 481 |
|
try:
|
| 482 |
|
current_street = self.streetmodel_set.get(ban_id=ban_id)
|
| 483 |
|
changed = False
|
| 484 |
|
values = {
|
| 485 |
|
"name": street_info['name'][:150],
|
| 486 |
|
"city": street_info['city'],
|
| 487 |
|
"zipcode": street_info['postcode'],
|
| 488 |
|
"citycode": citycode,
|
| 489 |
|
"type": street_info['type'],
|
| 490 |
|
}
|
| 491 |
|
for (k, v) in values.items():
|
| 492 |
|
if getattr(current_street, k) != v:
|
| 493 |
|
setattr(current_street, k, v)
|
| 494 |
|
changed = True
|
| 495 |
|
if changed:
|
| 496 |
|
current_street.save()
|
| 497 |
|
except ObjectDoesNotExist:
|
| 498 |
|
self.streetmodel_set.create(
|
| 499 |
|
resource=self,
|
| 500 |
|
citycode=citycode,
|
| 501 |
|
name=street_info['name'][:150],
|
| 502 |
|
ban_id=ban_id,
|
| 503 |
|
city=street_info['city'],
|
| 504 |
|
zipcode=street_info['postcode'],
|
| 505 |
|
type=street_info['type'],
|
| 506 |
|
)
|
|
471 |
self.streetmodel_set.update_or_create(
|
|
472 |
resource=self,
|
|
473 |
citycode=citycode,
|
|
474 |
name=street_info['name'][:150],
|
|
475 |
defaults={
|
|
476 |
'ban_id': ban_id,
|
|
477 |
'city': street_info['city'],
|
|
478 |
'zipcode': street_info['postcode'],
|
|
479 |
'type': street_info['type'],
|
|
480 |
},
|
|
481 |
)
|
| 507 |
482 |
if line is _not_found:
|
| 508 |
483 |
raise Exception('bano file is empty')
|
| 509 |
484 |
|
| 510 |
|
self.get_streets_queryset().exclude(citycode__in=seen_citycodes).delete()
|
|
485 |
self.get_streets_queryset().filter(last_update__lt=start_update).delete()
|
| 511 |
486 |
|
| 512 |
487 |
def get_api_geo_endpoint(self, endpoint):
|
| 513 |
488 |
if not self.api_geo_url:
|
| ... | ... | |
| 538 |
513 |
cities_json = self.get_api_geo_endpoint('communes')
|
| 539 |
514 |
if not (regions_json and departments_json and cities_json):
|
| 540 |
515 |
return
|
|
516 |
start_update = timezone.now()
|
| 541 |
517 |
|
| 542 |
|
regions_to_keep = set()
|
| 543 |
518 |
for data in regions_json:
|
| 544 |
|
regions_to_keep.add(data['code'])
|
| 545 |
|
keys = {"code": data["code"]}
|
| 546 |
|
attributes = {"name": data["nom"]}
|
| 547 |
|
try:
|
| 548 |
|
self.regionmodel_set.get(**keys, **attributes)
|
| 549 |
|
except ObjectDoesNotExist:
|
| 550 |
|
self.regionmodel_set.update_or_create(**keys, defaults=attributes)
|
| 551 |
|
self.regionmodel_set.exclude(code__in=regions_to_keep).delete()
|
|
519 |
defaults = {
|
|
520 |
'name': data['nom'],
|
|
521 |
}
|
|
522 |
self.regionmodel_set.update_or_create(code=data['code'], defaults=defaults)
|
|
523 |
self.regionmodel_set.filter(last_update__lt=start_update).delete()
|
| 552 |
524 |
|
| 553 |
|
departments_to_keep = set()
|
| 554 |
525 |
for data in departments_json:
|
| 555 |
|
departments_to_keep.add(data['code'])
|
| 556 |
|
keys = {"code": data["code"]}
|
| 557 |
|
attributes = {
|
| 558 |
|
"name": data['nom'],
|
| 559 |
|
"region": self.regionmodel_set.get(code=data["codeRegion"]),
|
|
526 |
defaults = {
|
|
527 |
'name': data['nom'],
|
|
528 |
'region': self.regionmodel_set.get(code=data['codeRegion']),
|
| 560 |
529 |
}
|
| 561 |
|
try:
|
| 562 |
|
self.departmentmodel_set.get(**keys, **attributes)
|
| 563 |
|
except ObjectDoesNotExist:
|
| 564 |
|
self.departmentmodel_set.update_or_create(**keys, defaults=attributes)
|
| 565 |
|
self.departmentmodel_set.exclude(code__in=departments_to_keep).delete()
|
|
530 |
self.departmentmodel_set.update_or_create(code=data['code'], defaults=defaults)
|
|
531 |
self.departmentmodel_set.filter(last_update__lt=start_update).delete()
|
| 566 |
532 |
|
| 567 |
|
cities_to_keep = set()
|
| 568 |
533 |
for data in cities_json:
|
| 569 |
534 |
for zipcode in data['codesPostaux']:
|
| 570 |
|
keys = {"code": data["code"], "zipcode": zipcode}
|
| 571 |
|
attributes = {
|
| 572 |
|
"population": data.get('population', 0),
|
| 573 |
|
"name": data["nom"],
|
|
535 |
defaults = {
|
|
536 |
'name': data['nom'],
|
|
537 |
'population': data.get('population', 0),
|
| 574 |
538 |
}
|
| 575 |
539 |
if data.get('codeDepartement'):
|
| 576 |
|
attributes["department"] = self.departmentmodel_set.get(code=data['codeDepartement'])
|
|
540 |
defaults['department'] = self.departmentmodel_set.get(code=data['codeDepartement'])
|
| 577 |
541 |
if data.get('codeRegion'):
|
| 578 |
|
attributes["region"] = self.regionmodel_set.get(code=data['codeRegion'])
|
| 579 |
|
|
| 580 |
|
try:
|
| 581 |
|
current_city = self.citymodel_set.get(**keys, **attributes)
|
| 582 |
|
cities_to_keep.add(current_city.id)
|
| 583 |
|
except ObjectDoesNotExist:
|
| 584 |
|
new_city, _ = self.citymodel_set.update_or_create(**keys, defaults=attributes)
|
| 585 |
|
|
| 586 |
|
cities_to_keep.add(new_city.id)
|
| 587 |
|
|
| 588 |
|
self.citymodel_set.exclude(id__in=cities_to_keep).delete()
|
|
542 |
defaults['region'] = self.regionmodel_set.get(code=data['codeRegion'])
|
|
543 |
self.citymodel_set.update_or_create(code=data['code'], zipcode=zipcode, defaults=defaults)
|
|
544 |
self.citymodel_set.filter(last_update__lt=start_update).delete()
|
| 589 |
545 |
|
| 590 |
546 |
def clean_addresses_cache(self):
|
| 591 |
547 |
old_addresses = self.addresscachemodel_set.filter(
|