From 38316b3a3ab1472bd65ae445313be848ed9b21e0 Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Wed, 10 Feb 2021 15:56:45 +0100 Subject: [PATCH 2/3] dataviz: load statistics list dynamically (#50891) --- combo/apps/dataviz/__init__.py | 51 ++-------------------------------- combo/apps/dataviz/forms.py | 14 ++++++++-- combo/apps/dataviz/utils.py | 51 ++++++++++++++++++++++++++++++++++ tests/test_dataviz.py | 7 +++-- 4 files changed, 70 insertions(+), 53 deletions(-) create mode 100644 combo/apps/dataviz/utils.py diff --git a/combo/apps/dataviz/__init__.py b/combo/apps/dataviz/__init__.py index 2e90348a..41ba6c84 100644 --- a/combo/apps/dataviz/__init__.py +++ b/combo/apps/dataviz/__init__.py @@ -15,13 +15,8 @@ # along with this program. If not, see . import django.apps -from django.core import checks -from django.conf import settings -from django.utils import timezone from django.utils.translation import ugettext_lazy as _ -from combo.utils import requests - class AppConfig(django.apps.AppConfig): name = 'combo.apps.dataviz' @@ -32,50 +27,10 @@ class AppConfig(django.apps.AppConfig): return urls.urlpatterns def hourly(self): - self.update_available_statistics() - - def update_available_statistics(self): - from .models import Statistic, ChartNgCell - if not settings.KNOWN_SERVICES: - return - - start_update = timezone.now() - for provider in settings.STATISTICS_PROVIDERS: - if isinstance(provider, dict): - url = provider['url'] - sites = {provider['id']: {'title': provider['name']}} - provider = provider['id'] - else: - sites = settings.KNOWN_SERVICES.get(provider, {}) - url = '/visualization/json/' if provider == 'bijoe' else '/api/statistics/' - - for site_key, site_dict in sites.items(): - site_title = site_dict.pop('title', '') - response = requests.get( - url, remote_service=site_dict, without_user=True, headers={'accept': 'application/json'} - ) - if response.status_code != 200: - continue - - result = response.json() - if isinstance(result, dict): - result = result['data'] # detect new api - - for stat in result: - Statistic.objects.update_or_create( - slug=stat.get('slug') or stat['id'], - site_slug=site_key, - service_slug=provider, - defaults={ - 'label': stat['name'], - 'url': stat.get('data-url') or stat['url'], - 'site_title': site_title, - 'filters': stat.get('filters', []), - 'available': True, - } - ) - Statistic.objects.filter(last_update__lt=start_update).update(available=False) + from .models import ChartNgCell + from .utils import update_available_statistics + update_available_statistics() for cell in ChartNgCell.objects.all(): cell.check_validity() diff --git a/combo/apps/dataviz/forms.py b/combo/apps/dataviz/forms.py index 24278f11..5041be57 100644 --- a/combo/apps/dataviz/forms.py +++ b/combo/apps/dataviz/forms.py @@ -20,9 +20,10 @@ from django import forms from django.conf import settings from django.db.models import Q -from combo.utils import requests +from combo.utils import requests, cache_during_request -from .models import ChartCell, ChartNgCell +from .models import ChartCell, ChartNgCell, Statistic +from .utils import update_available_statistics class ChartForm(forms.ModelForm): @@ -42,6 +43,12 @@ class ChartForm(forms.ModelForm): self.fields['url'].widget = forms.Select(choices=available_charts) +@cache_during_request +def get_statistic_choices(): + update_available_statistics() + return Statistic.objects.all() + + class ChartNgForm(forms.ModelForm): blank_choice = ('', '---------') @@ -56,6 +63,8 @@ class ChartNgForm(forms.ModelForm): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + stat_field = self.fields['statistic'] + stat_field.queryset = get_statistic_choices() field_ids = list(self._meta.fields) if not self.instance.statistic or self.instance.statistic.service_slug == 'bijoe': @@ -63,7 +72,6 @@ class ChartNgForm(forms.ModelForm): x for x in field_ids if x not in ('time_range', 'time_range_start', 'time_range_end') ] - stat_field = self.fields['statistic'] if not self.instance.statistic: stat_field.queryset = stat_field.queryset.filter(available=True) else: diff --git a/combo/apps/dataviz/utils.py b/combo/apps/dataviz/utils.py new file mode 100644 index 00000000..c8f6981f --- /dev/null +++ b/combo/apps/dataviz/utils.py @@ -0,0 +1,51 @@ +from django.conf import settings +from django.utils import timezone + +from combo.utils import requests + +from .models import Statistic + + +def update_available_statistics(): + if not settings.KNOWN_SERVICES: + return + + start_update = timezone.now() + for provider in settings.STATISTICS_PROVIDERS: + if isinstance(provider, dict): + url = provider['url'] + sites = {provider['id']: {'title': provider['name']}} + provider = provider['id'] + else: + sites = settings.KNOWN_SERVICES.get(provider, {}) + url = '/visualization/json/' if provider == 'bijoe' else '/api/statistics/' + + for site_key, site_dict in sites.items(): + response = requests.get( + url, + timeout=5, + remote_service=site_dict if provider in settings.KNOWN_SERVICES else {}, + without_user=True, + headers={'accept': 'application/json'}, + ) + if response.status_code != 200: + continue + + result = response.json() + if isinstance(result, dict): + result = result['data'] # detect new api + + for stat in result: + Statistic.objects.update_or_create( + slug=stat.get('slug') or stat['id'], + site_slug=site_key, + service_slug=provider, + defaults={ + 'label': stat['name'], + 'url': stat.get('data-url') or stat['url'], + 'site_title': site_dict.get('title', ''), + 'filters': stat.get('filters', []), + 'available': True, + }, + ) + Statistic.objects.filter(last_update__lt=start_update).update(available=False) diff --git a/tests/test_dataviz.py b/tests/test_dataviz.py index ab1519c8..b9b41077 100644 --- a/tests/test_dataviz.py +++ b/tests/test_dataviz.py @@ -1041,6 +1041,7 @@ def test_chartng_cell_manager_new_api_dynamic_fields(app, admin_user, new_api_st assert 'time_interval' in resp.text +@with_httmock(new_api_mock) def test_chartng_cell_manager_multiple_cells(app, admin_user, new_api_statistics, nocache): page = Page.objects.create(title='One', slug='index') cell = ChartNgCell.objects.create(page=page, order=1, placeholder='content') @@ -1048,13 +1049,15 @@ def test_chartng_cell_manager_multiple_cells(app, admin_user, new_api_statistics app = login(app) with CaptureQueriesContext(connection) as ctx: resp = app.get('/manage/pages/%s/' % page.id) - assert len(ctx.captured_queries) == 18 + assert len(ctx.captured_queries) == 35 + assert new_api_mock.call['count'] == 1 cell = ChartNgCell.objects.create(page=page, order=2, placeholder='content') cell = ChartNgCell.objects.create(page=page, order=3, placeholder='content') with CaptureQueriesContext(connection) as ctx: resp = app.get('/manage/pages/%s/' % page.id) - assert len(ctx.captured_queries) == 20 + assert len(ctx.captured_queries) == 37 + assert new_api_mock.call['count'] == 2 @with_httmock(bijoe_mock) -- 2.20.1