From a166e9f1dee95ab03c07755f684dc14b1bee1eee Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Tue, 9 Feb 2021 18:00:52 +0100 Subject: [PATCH] dataviz: update statistics list every 5 minutes (#50892) --- combo/apps/dataviz/__init__.py | 49 +------------- .../commands/update_available_statistics.py | 67 +++++++++++++++++++ debian/combo.cron.d | 1 + tests/test_dataviz.py | 13 ++-- 4 files changed, 76 insertions(+), 54 deletions(-) create mode 100644 combo/apps/dataviz/management/commands/update_available_statistics.py diff --git a/combo/apps/dataviz/__init__.py b/combo/apps/dataviz/__init__.py index 2e90348a..356d0ad9 100644 --- a/combo/apps/dataviz/__init__.py +++ b/combo/apps/dataviz/__init__.py @@ -15,12 +15,8 @@ # along with this program. If not, see . import django.apps -from django.core import checks -from django.conf import settings -from django.utils import timezone from django.utils.translation import ugettext_lazy as _ -from combo.utils import requests class AppConfig(django.apps.AppConfig): @@ -32,50 +28,7 @@ class AppConfig(django.apps.AppConfig): return urls.urlpatterns def hourly(self): - self.update_available_statistics() - - def update_available_statistics(self): - from .models import Statistic, ChartNgCell - if not settings.KNOWN_SERVICES: - return - - start_update = timezone.now() - for provider in settings.STATISTICS_PROVIDERS: - if isinstance(provider, dict): - url = provider['url'] - sites = {provider['id']: {'title': provider['name']}} - provider = provider['id'] - else: - sites = settings.KNOWN_SERVICES.get(provider, {}) - url = '/visualization/json/' if provider == 'bijoe' else '/api/statistics/' - - for site_key, site_dict in sites.items(): - site_title = site_dict.pop('title', '') - response = requests.get( - url, remote_service=site_dict, without_user=True, headers={'accept': 'application/json'} - ) - if response.status_code != 200: - continue - - result = response.json() - if isinstance(result, dict): - result = result['data'] # detect new api - - for stat in result: - Statistic.objects.update_or_create( - slug=stat.get('slug') or stat['id'], - site_slug=site_key, - service_slug=provider, - defaults={ - 'label': stat['name'], - 'url': stat.get('data-url') or stat['url'], - 'site_title': site_title, - 'filters': stat.get('filters', []), - 'available': True, - } - ) - Statistic.objects.filter(last_update__lt=start_update).update(available=False) - + from combo.apps.dataviz.models import ChartNgCell for cell in ChartNgCell.objects.all(): cell.check_validity() diff --git a/combo/apps/dataviz/management/commands/update_available_statistics.py b/combo/apps/dataviz/management/commands/update_available_statistics.py new file mode 100644 index 00000000..ae83d645 --- /dev/null +++ b/combo/apps/dataviz/management/commands/update_available_statistics.py @@ -0,0 +1,67 @@ +# combo - content management system +# Copyright (C) 2021 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from django.conf import settings +from django.core.management.base import BaseCommand +from django.utils import timezone, translation + +from combo.apps.dataviz.models import Statistic +from combo.utils import requests + + +class Command(BaseCommand): + + def handle(self, *args, **kwargs): + translation.activate(settings.LANGUAGE_CODE) + if not settings.KNOWN_SERVICES: + return + + start_update = timezone.now() + for provider in settings.STATISTICS_PROVIDERS: + if isinstance(provider, dict): + url = provider['url'] + sites = {provider['id']: {'title': provider['name']}} + provider = provider['id'] + else: + sites = settings.KNOWN_SERVICES.get(provider, {}) + url = '/visualization/json/' if provider == 'bijoe' else '/api/statistics/' + + for site_key, site_dict in sites.items(): + site_title = site_dict.pop('title', '') + response = requests.get( + url, remote_service=site_dict, without_user=True, headers={'accept': 'application/json'} + ) + if response.status_code != 200: + continue + + result = response.json() + if isinstance(result, dict): + result = result['data'] # detect new api + + for stat in result: + Statistic.objects.update_or_create( + slug=stat.get('slug') or stat['id'], + site_slug=site_key, + service_slug=provider, + defaults={ + 'label': stat['name'], + 'url': stat.get('data-url') or stat['url'], + 'site_title': site_title, + 'filters': stat.get('filters', []), + 'available': True, + } + ) + Statistic.objects.filter(last_update__lt=start_update).update(available=False) diff --git a/debian/combo.cron.d b/debian/combo.cron.d index 2ed4108c..34501b3f 100644 --- a/debian/combo.cron.d +++ b/debian/combo.cron.d @@ -1,3 +1,4 @@ MAILTO=root 0 8 * * * combo /usr/bin/combo-manage tenant_command notify_new_remote_invoices --all-tenants -v0 +*/5 * * * * combo /usr/bin/combo-manage tenant_command update_available_statistics --all-tenants -v0 diff --git a/tests/test_dataviz.py b/tests/test_dataviz.py index 205d26e2..681390c1 100644 --- a/tests/test_dataviz.py +++ b/tests/test_dataviz.py @@ -9,6 +9,7 @@ from requests.exceptions import HTTPError from django.apps import apps from django.contrib.auth.models import User, Group +from django.core.management import call_command from django.db import connection from django.db.migrations.executor import MigrationExecutor from django.test import override_settings @@ -323,7 +324,7 @@ def statistics(settings): } settings.STATISTICS_PROVIDERS = ['bijoe'] appconfig = apps.get_app_config('dataviz') - appconfig.hourly() + call_command('update_available_statistics') assert Statistic.objects.count() == len(VISUALIZATION_JSON) @@ -342,7 +343,7 @@ def new_api_statistics(settings): } settings.STATISTICS_PROVIDERS = ['authentic'] appconfig = apps.get_app_config('dataviz') - appconfig.hourly() + call_command('update_available_statistics') assert Statistic.objects.count() == len(STATISTICS_LIST['data']) @@ -1112,7 +1113,7 @@ def test_dataviz_hourly_unavailable_statistic(statistics, nocache): appconfig = apps.get_app_config('dataviz') with HTTMock(bijoe_mock_unavailable): - appconfig.hourly() + call_command('update_available_statistics') assert Statistic.objects.filter(available=True).count() == all_stats_count - 2 @@ -1165,7 +1166,7 @@ def test_dataviz_api_list_statistics(new_api_statistics, settings): appconfig = apps.get_app_config('dataviz') with HTTMock(server_error): - appconfig.hourly() + call_command('update_available_statistics') assert Statistic.objects.count() == statistics_count @urlmatch(scheme='https', netloc=r'stat.com', path='/stats/') @@ -1173,7 +1174,7 @@ def test_dataviz_api_list_statistics(new_api_statistics, settings): return {'content': json.dumps(catalog), 'status_code': 200} with HTTMock(success): - appconfig.hourly() + call_command('update_available_statistics') assert Statistic.objects.count() == statistics_count + 1 statistic = Statistic.objects.get(slug='test') @@ -1187,7 +1188,7 @@ def test_dataviz_api_list_statistics(new_api_statistics, settings): settings.STATISTICS_PROVIDERS.append('unknown') appconfig = apps.get_app_config('dataviz') with HTTMock(success): - appconfig.hourly() # unknown provider is ignored + call_command('update_available_statistics') # unknown provider is ignored @with_httmock(new_api_mock) -- 2.20.1