From b1616edd2b7d4e37588327ae1043d8240f2e663b Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Thu, 26 Nov 2020 16:09:26 +0100 Subject: [PATCH 3/4] dataviz: save available visualizations in db (#48865) --- combo/apps/dataviz/__init__.py | 32 +++ combo/apps/dataviz/forms.py | 27 +- .../migrations/0012_auto_20201126_1557.py | 42 ++++ .../migrations/0013_update_chartng_cells.py | 40 +++ .../migrations/0014_auto_20201130_1534.py | 23 ++ combo/apps/dataviz/models.py | 60 +++-- .../dataviz/templates/combo/chartngcell.html | 4 +- .../templates/combo/chartngcell_form.html | 2 +- combo/apps/dataviz/views.py | 2 +- tests/test_dataviz.py | 238 ++++++++++++------ 10 files changed, 344 insertions(+), 126 deletions(-) create mode 100644 combo/apps/dataviz/migrations/0012_auto_20201126_1557.py create mode 100644 combo/apps/dataviz/migrations/0013_update_chartng_cells.py create mode 100644 combo/apps/dataviz/migrations/0014_auto_20201130_1534.py diff --git a/combo/apps/dataviz/__init__.py b/combo/apps/dataviz/__init__.py index 8fe74c4c..c6b7e5e2 100644 --- a/combo/apps/dataviz/__init__.py +++ b/combo/apps/dataviz/__init__.py @@ -19,8 +19,11 @@ import re import django.apps from django.core import checks from django.conf import settings +from django.utils import timezone from django.utils.translation import ugettext_lazy as _ +from combo.utils import requests + class AppConfig(django.apps.AppConfig): name = 'combo.apps.dataviz' @@ -30,4 +33,33 @@ class AppConfig(django.apps.AppConfig): from . import urls return urls.urlpatterns + def hourly(self): + self.update_available_statistics() + + def update_available_statistics(self): + from .models import Statistic + if not settings.KNOWN_SERVICES: + return + + start_update = timezone.now() + bijoe_sites = settings.KNOWN_SERVICES.get('bijoe').items() + for site_key, site_dict in bijoe_sites: + result = requests.get('/visualization/json/', + remote_service=site_dict, without_user=True, + headers={'accept': 'application/json'}).json() + for stat in result: + Statistic.objects.update_or_create( + slug=stat['slug'], + site_slug=site_key, + service_slug='bijoe', + defaults={ + 'label': stat['name'], + 'url': stat['data-url'], + 'site_title': site_dict.get('title', ''), + 'available': True, + } + ) + Statistic.objects.filter(last_update__lt=start_update).update(available=False) + + default_app_config = 'combo.apps.dataviz.AppConfig' diff --git a/combo/apps/dataviz/forms.py b/combo/apps/dataviz/forms.py index 857528ed..117f366f 100644 --- a/combo/apps/dataviz/forms.py +++ b/combo/apps/dataviz/forms.py @@ -16,11 +16,12 @@ from django import forms from django.conf import settings +from django.db.models import Q from django.utils.translation import ugettext_lazy as _ from combo.utils import requests -from .models import ChartCell, ChartNgCell +from .models import ChartCell, ChartNgCell, Statistic class ChartForm(forms.ModelForm): @@ -43,24 +44,12 @@ class ChartForm(forms.ModelForm): class ChartNgForm(forms.ModelForm): class Meta: model = ChartNgCell - fields = ('title', 'data_reference', 'chart_type', 'height', 'sort_order', + fields = ('title', 'statistic', 'chart_type', 'height', 'sort_order', 'hide_null_values') def __init__(self, *args, **kwargs): - super(ChartNgForm, self).__init__(*args, **kwargs) - data_references = [] - bijoe_sites = settings.KNOWN_SERVICES.get('bijoe').items() - for site_key, site_dict in bijoe_sites: - result = requests.get('/visualization/json/', - remote_service=site_dict, without_user=True, - headers={'accept': 'application/json'}).json() - if len(bijoe_sites) > 1: - label_prefix = _('%s: ') % site_dict.get('title') - else: - label_prefix = '' - data_references.extend([ - ('%s:%s' % (site_key, x['slug']), '%s%s' % (label_prefix, x['name'])) - for x in result]) - - data_references.sort(key=lambda x: x[1]) - self.fields['data_reference'].widget = forms.Select(choices=data_references) + super().__init__(*args, **kwargs) + q_filters = Q(available=True) + if self.instance.statistic: + q_filters |= Q(pk=self.instance.statistic.pk) + self.fields['statistic'].queryset = self.fields['statistic'].queryset.filter(q_filters) diff --git a/combo/apps/dataviz/migrations/0012_auto_20201126_1557.py b/combo/apps/dataviz/migrations/0012_auto_20201126_1557.py new file mode 100644 index 00000000..de77c435 --- /dev/null +++ b/combo/apps/dataviz/migrations/0012_auto_20201126_1557.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2020-11-26 14:57 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('dataviz', '0011_auto_20200813_1100'), + ] + + operations = [ + migrations.CreateModel( + name='Statistic', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('slug', models.SlugField(max_length=256, verbose_name='Slug')), + ('label', models.CharField(max_length=256, verbose_name='Label')), + ('site_slug', models.SlugField(max_length=256, verbose_name='Site slug')), + ('service_slug', models.SlugField(max_length=256, verbose_name='Service slug')), + ('site_title', models.CharField(max_length=256, verbose_name='Site title')), + ('url', models.URLField(verbose_name='Data URL')), + ('last_update', models.DateTimeField(auto_now=True, null=True, verbose_name='Last update')), + ('available', models.BooleanField(default=True, verbose_name='Available data')), + ], + options={ + 'ordering': ['-available', 'site_title', 'label'], + }, + ), + migrations.AlterUniqueTogether( + name='statistic', + unique_together=set([('slug', 'site_slug', 'service_slug')]), + ), + migrations.AddField( + model_name='chartngcell', + name='statistic', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='cells', to='dataviz.Statistic', verbose_name='Data'), + ), + ] diff --git a/combo/apps/dataviz/migrations/0013_update_chartng_cells.py b/combo/apps/dataviz/migrations/0013_update_chartng_cells.py new file mode 100644 index 00000000..a2fc1c46 --- /dev/null +++ b/combo/apps/dataviz/migrations/0013_update_chartng_cells.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2020-11-30 14:26 +from __future__ import unicode_literals + +from django.conf import settings +from django.db import migrations + + +def update_cells(apps, schema_editor): + Statistic = apps.get_model('dataviz', 'Statistic') + ChartNgCell = apps.get_model('dataviz', 'ChartNgCell') + bijoe_sites = settings.KNOWN_SERVICES.get('bijoe', {}) + + for cell in ChartNgCell.objects.filter(statistic__isnull=True): + if not cell.data_reference or not cell.cached_json: + continue + site_slug, slug = cell.data_reference.split(':') + statistic, _ = Statistic.objects.get_or_create( + slug=slug, + site_slug=site_slug, + service_slug='bijoe', + defaults={ + 'label': cell.cached_json['name'], + 'url': cell.cached_json['data-url'], + 'site_title': bijoe_sites.get(site_slug, {}).get('title'), + } + ) + cell.statistic = statistic + cell.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ('dataviz', '0012_auto_20201126_1557'), + ] + + operations = [ + migrations.RunPython(update_cells, migrations.RunPython.noop), + ] diff --git a/combo/apps/dataviz/migrations/0014_auto_20201130_1534.py b/combo/apps/dataviz/migrations/0014_auto_20201130_1534.py new file mode 100644 index 00000000..918b78cd --- /dev/null +++ b/combo/apps/dataviz/migrations/0014_auto_20201130_1534.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2020-11-30 14:34 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('dataviz', '0013_update_chartng_cells'), + ] + + operations = [ + migrations.RemoveField( + model_name='chartngcell', + name='cached_json', + ), + migrations.RemoveField( + model_name='chartngcell', + name='data_reference', + ), + ] diff --git a/combo/apps/dataviz/models.py b/combo/apps/dataviz/models.py index e063c319..8629de38 100644 --- a/combo/apps/dataviz/models.py +++ b/combo/apps/dataviz/models.py @@ -103,11 +103,43 @@ class ChartCell(CellBase): return context +class StatisticManager(models.Manager): + def get_by_natural_key(self, slug, site_slug, service_slug): + return self.get_or_create(slug=slug, site_slug=site_slug, service_slug=service_slug)[0] + + +class Statistic(models.Model): + slug = models.SlugField(_('Slug'), max_length=256) + label = models.CharField(_('Label'), max_length=256) + site_slug = models.SlugField(_('Site slug'), max_length=256) + service_slug = models.SlugField(_('Service slug'), max_length=256) + site_title = models.CharField(_('Site title'), max_length=256) + url = models.URLField(_('Data URL')) + available = models.BooleanField(_('Available data'), default=True) + last_update = models.DateTimeField(_('Last update'), null=True, auto_now=True) + + objects = StatisticManager() + + class Meta: + ordering = ['-available', 'site_title', 'label'] + unique_together = ['slug', 'site_slug', 'service_slug'] + + def __str__(self): + name = _('%s: %s') % (self.site_title, self.label) + if not self.available: + name = _('%s (unavailable)') % name + return name + + def natural_key(self): + return (self.slug, self.site_slug, self.service_slug) + + @register_cell_class class ChartNgCell(CellBase): - data_reference = models.CharField(_('Data'), max_length=150) + statistic = models.ForeignKey( + verbose_name=_('Data'), to=Statistic, blank=False, null=True, on_delete=models.SET_NULL, related_name='cells' + ) title = models.CharField(_('Title'), max_length=150, blank=True) - cached_json = JSONField(blank=True) chart_type = models.CharField(_('Chart Type'), max_length=20, default='bar', choices=( ('bar', _('Bar')), @@ -154,28 +186,12 @@ class ChartNgCell(CellBase): def get_additional_label(self): return self.title - def is_relevant(self, context=None): - return bool(self.data_reference) - - def save(self, *args, **kwargs): - if self.data_reference: - site_key, visualization_slug = self.data_reference.split(':') - site_dict = settings.KNOWN_SERVICES['bijoe'][site_key] - response_json = requests.get('/visualization/json/', - remote_service=site_dict, without_user=True, - headers={'accept': 'application/json'}).json() - if isinstance(response_json, dict): - # forward compatibility with possible API change - response_json = response_json.get('data') - for visualization in response_json: - slug = visualization.get('slug') - if slug == visualization_slug: - self.cached_json = visualization - return super(ChartNgCell, self).save(*args, **kwargs) + def is_relevant(self, context): + return bool(self.statistic) def get_cell_extra_context(self, context): ctx = super(ChartNgCell, self).get_cell_extra_context(context) - if self.chart_type == 'table' and self.cached_json: + if self.chart_type == 'table' and self.statistic: try: chart = self.get_chart(raise_if_not_cached=not(context.get('synchronous'))) except UnsupportedDataSet: @@ -193,7 +209,7 @@ class ChartNgCell(CellBase): def get_chart(self, width=None, height=None, raise_if_not_cached=False): response = requests.get( - self.cached_json['data-url'], + self.statistic.url, cache_duration=300, raise_if_not_cached=raise_if_not_cached) response.raise_for_status() diff --git a/combo/apps/dataviz/templates/combo/chartngcell.html b/combo/apps/dataviz/templates/combo/chartngcell.html index 7a7bf34c..0986ab09 100644 --- a/combo/apps/dataviz/templates/combo/chartngcell.html +++ b/combo/apps/dataviz/templates/combo/chartngcell.html @@ -1,8 +1,6 @@ {% load i18n %} {% if cell.title %}

{{cell.title}}

{% endif %} -{% if not cell.cached_json %} -
{% trans "Unavailable data." %}
-{% elif cell.chart_type == "table" %} +{% if cell.chart_type == "table" %} {{table|safe}} {% else %}
diff --git a/combo/apps/dataviz/templates/combo/chartngcell_form.html b/combo/apps/dataviz/templates/combo/chartngcell_form.html index b0717869..33d7b8f0 100644 --- a/combo/apps/dataviz/templates/combo/chartngcell_form.html +++ b/combo/apps/dataviz/templates/combo/chartngcell_form.html @@ -1,6 +1,6 @@
{{ form.as_p }} -{% if cell.cached_json and cell.chart_type != "table" and cell.is_relevant %} +{% if cell.statistic and cell.chart_type != "table" %}
diff --git a/combo/apps/dataviz/views.py b/combo/apps/dataviz/views.py index cf319e08..c37d4a23 100644 --- a/combo/apps/dataviz/views.py +++ b/combo/apps/dataviz/views.py @@ -36,7 +36,7 @@ def dataviz_graph(request, *args, **kwargs): raise PermissionDenied() if not cell.is_visible(user=request.user): raise PermissionDenied() - if not cell.cached_json: + if not cell.statistic: raise Http404('misconfigured cell') error_text = None try: diff --git a/tests/test_dataviz.py b/tests/test_dataviz.py index 66a5765a..7036ce53 100644 --- a/tests/test_dataviz.py +++ b/tests/test_dataviz.py @@ -2,15 +2,19 @@ import json import mock import pytest +from datetime import timedelta from httmock import HTTMock, with_httmock from requests.exceptions import HTTPError from django.apps import apps from django.contrib.auth.models import User, Group +from django.db import connection +from django.db.migrations.executor import MigrationExecutor from django.test import override_settings +from django.utils import timezone from combo.data.models import Page -from combo.apps.dataviz.models import Gauge, ChartNgCell, UnsupportedDataSet +from combo.apps.dataviz.models import Gauge, ChartNgCell, UnsupportedDataSet, Statistic from .test_public import login, normal_user @@ -224,15 +228,21 @@ def bijoe_mock(url, request): return {'content': json.dumps(response), 'request': request, 'status_code': 404} +@pytest.fixture @with_httmock(bijoe_mock) -def test_chartng_cell(app): +def statistics(): + appconfig = apps.get_app_config('dataviz') + appconfig.hourly() + + +@with_httmock(bijoe_mock) +def test_chartng_cell(app, statistics): page = Page(title='One', slug='index') page.save() cell = ChartNgCell(page=page, order=1) - cell.data_reference = 'plop:example' + cell.statistic = Statistic.objects.get(slug='example') cell.save() - assert cell.cached_json == VISUALIZATION_JSON[0] # bar chart = cell.get_chart() @@ -260,9 +270,8 @@ def test_chartng_cell(app): # data in Y cell.chart_type = 'bar' - cell.data_reference = 'plop:second' + cell.statistic = Statistic.objects.get(slug='second') cell.save() - assert cell.cached_json == VISUALIZATION_JSON[1] chart = cell.get_chart() assert chart.x_labels == ['web', 'mail', 'phone', 'email'] @@ -270,9 +279,8 @@ def test_chartng_cell(app): # data in X/Y cell.chart_type = 'bar' - cell.data_reference = 'plop:third' + cell.statistic = Statistic.objects.get(slug='third') cell.save() - assert cell.cached_json == VISUALIZATION_JSON[2] chart = cell.get_chart() assert chart.x_labels == ['web', 'mail', 'phone', 'email'] @@ -283,16 +291,15 @@ def test_chartng_cell(app): # single data point cell.chart_type = 'bar' - cell.data_reference = 'plop:fourth' + cell.statistic = Statistic.objects.get(slug='fourth') cell.save() - assert cell.cached_json == VISUALIZATION_JSON[3] chart = cell.get_chart() assert chart.x_labels == [''] assert chart.raw_series == [([222], {'title': ''})] # loop/X - cell.data_reference = 'plop:fifth' + cell.statistic = Statistic.objects.get(slug='fifth') cell.save() chart = cell.get_chart() assert chart.x_labels == ['web', 'mail', 'phone', 'email'] @@ -302,7 +309,7 @@ def test_chartng_cell(app): ] # loop/Y - cell.data_reference = 'plop:sixth' + cell.statistic = Statistic.objects.get(slug='sixth') cell.save() chart = cell.get_chart() assert chart.x_labels == ['web', 'mail', 'phone', 'email'] @@ -312,39 +319,38 @@ def test_chartng_cell(app): ] # loop/X/Y - cell.data_reference = 'plop:seventh' + cell.statistic = Statistic.objects.get(slug='seventh') cell.save() with pytest.raises(UnsupportedDataSet): chart = cell.get_chart() # duration - cell.data_reference = 'plop:eighth' + cell.statistic = Statistic.objects.get(slug='eighth') cell.save() chart = cell.get_chart() # loop/X/Y - cell.data_reference = 'plop:nineth' + cell.statistic = Statistic.objects.get(slug='nineth') cell.save() with pytest.raises(UnsupportedDataSet): chart = cell.get_chart() # deleted visualization - cell.data_reference = 'plop:eleventh' + cell.statistic = Statistic.objects.get(slug='eleventh') cell.save() with pytest.raises(HTTPError): chart = cell.get_chart() @with_httmock(bijoe_mock) -def test_chartng_cell_hide_null_values(app): +def test_chartng_cell_hide_null_values(app, statistics): page = Page(title='One', slug='index') page.save() cell = ChartNgCell(page=page, order=1) - cell.data_reference = 'plop:example' + cell.statistic = Statistic.objects.get(slug='example') cell.hide_null_values = True cell.save() - assert cell.cached_json == VISUALIZATION_JSON[0] # bar chart = cell.get_chart() @@ -372,9 +378,8 @@ def test_chartng_cell_hide_null_values(app): # data in Y cell.chart_type = 'bar' - cell.data_reference = 'plop:second' + cell.statistic = Statistic.objects.get(slug='second') cell.save() - assert cell.cached_json == VISUALIZATION_JSON[1] chart = cell.get_chart() assert chart.x_labels == ['web', 'mail', 'email'] @@ -382,9 +387,8 @@ def test_chartng_cell_hide_null_values(app): # data in X/Y cell.chart_type = 'bar' - cell.data_reference = 'plop:third' + cell.statistic = Statistic.objects.get(slug='third') cell.save() - assert cell.cached_json == VISUALIZATION_JSON[2] chart = cell.get_chart() assert chart.x_labels == ['web', 'mail', 'phone', 'email'] @@ -395,16 +399,15 @@ def test_chartng_cell_hide_null_values(app): # single data point cell.chart_type = 'bar' - cell.data_reference = 'plop:fourth' + cell.statistic = Statistic.objects.get(slug='fourth') cell.save() - assert cell.cached_json == VISUALIZATION_JSON[3] chart = cell.get_chart() assert chart.x_labels == [''] assert chart.raw_series == [([222], {'title': ''})] # loop/X - cell.data_reference = 'plop:fifth' + cell.statistic = Statistic.objects.get(slug='fifth') cell.save() chart = cell.get_chart() assert chart.x_labels == ['web', 'mail', 'phone', 'email'] @@ -414,7 +417,7 @@ def test_chartng_cell_hide_null_values(app): ] # loop/Y - cell.data_reference = 'plop:sixth' + cell.statistic = Statistic.objects.get(slug='sixth') cell.save() chart = cell.get_chart() assert chart.x_labels == ['web', 'mail', 'phone', 'email'] @@ -425,15 +428,14 @@ def test_chartng_cell_hide_null_values(app): @with_httmock(bijoe_mock) -def test_chartng_cell_sort_order_alpha(app): +def test_chartng_cell_sort_order_alpha(app, statistics): page = Page(title='One', slug='index') page.save() cell = ChartNgCell(page=page, order=1) - cell.data_reference = 'plop:example' + cell.statistic = Statistic.objects.get(slug='example') cell.sort_order = 'alpha' cell.save() - assert cell.cached_json == VISUALIZATION_JSON[0] # bar chart = cell.get_chart() @@ -461,9 +463,8 @@ def test_chartng_cell_sort_order_alpha(app): # data in Y cell.chart_type = 'bar' - cell.data_reference = 'plop:second' + cell.statistic = Statistic.objects.get(slug='second') cell.save() - assert cell.cached_json == VISUALIZATION_JSON[1] chart = cell.get_chart() assert chart.x_labels == ['email', 'mail', 'phone', 'web'] @@ -471,9 +472,8 @@ def test_chartng_cell_sort_order_alpha(app): # data in X/Y cell.chart_type = 'bar' - cell.data_reference = 'plop:third' + cell.statistic = Statistic.objects.get(slug='third') cell.save() - assert cell.cached_json == VISUALIZATION_JSON[2] chart = cell.get_chart() assert chart.x_labels == ['web', 'mail', 'phone', 'email'] @@ -484,16 +484,15 @@ def test_chartng_cell_sort_order_alpha(app): # single data point cell.chart_type = 'bar' - cell.data_reference = 'plop:fourth' + cell.statistic = Statistic.objects.get(slug='fourth') cell.save() - assert cell.cached_json == VISUALIZATION_JSON[3] chart = cell.get_chart() assert chart.x_labels == [''] assert chart.raw_series == [([222], {'title': ''})] # loop/X - cell.data_reference = 'plop:fifth' + cell.statistic = Statistic.objects.get(slug='fifth') cell.save() chart = cell.get_chart() assert chart.x_labels == ['web', 'mail', 'phone', 'email'] @@ -503,7 +502,7 @@ def test_chartng_cell_sort_order_alpha(app): ] # loop/Y - cell.data_reference = 'plop:sixth' + cell.statistic = Statistic.objects.get(slug='sixth') cell.save() chart = cell.get_chart() assert chart.x_labels == ['web', 'mail', 'phone', 'email'] @@ -514,15 +513,14 @@ def test_chartng_cell_sort_order_alpha(app): @with_httmock(bijoe_mock) -def test_chartng_cell_sort_order_desc(app): +def test_chartng_cell_sort_order_desc(app, statistics): page = Page(title='One', slug='index') page.save() cell = ChartNgCell(page=page, order=1) - cell.data_reference = 'plop:example' + cell.statistic = Statistic.objects.get(slug='example') cell.sort_order = 'desc' cell.save() - assert cell.cached_json == VISUALIZATION_JSON[0] # bar chart = cell.get_chart() @@ -550,9 +548,8 @@ def test_chartng_cell_sort_order_desc(app): # data in Y cell.chart_type = 'bar' - cell.data_reference = 'plop:second' + cell.statistic = Statistic.objects.get(slug='second') cell.save() - assert cell.cached_json == VISUALIZATION_JSON[1] chart = cell.get_chart() assert chart.x_labels == ['web', 'mail', 'email', 'phone'] @@ -560,9 +557,8 @@ def test_chartng_cell_sort_order_desc(app): # data in X/Y cell.chart_type = 'bar' - cell.data_reference = 'plop:third' + cell.statistic = Statistic.objects.get(slug='third') cell.save() - assert cell.cached_json == VISUALIZATION_JSON[2] chart = cell.get_chart() assert chart.x_labels == ['web', 'mail', 'phone', 'email'] @@ -573,16 +569,15 @@ def test_chartng_cell_sort_order_desc(app): # single data point cell.chart_type = 'bar' - cell.data_reference = 'plop:fourth' + cell.statistic = Statistic.objects.get(slug='fourth') cell.save() - assert cell.cached_json == VISUALIZATION_JSON[3] chart = cell.get_chart() assert chart.x_labels == [''] assert chart.raw_series == [([222], {'title': ''})] # loop/X - cell.data_reference = 'plop:fifth' + cell.statistic = Statistic.objects.get(slug='fifth') cell.save() chart = cell.get_chart() assert chart.x_labels == ['web', 'mail', 'phone', 'email'] @@ -592,7 +587,7 @@ def test_chartng_cell_sort_order_desc(app): ] # loop/Y - cell.data_reference = 'plop:sixth' + cell.statistic = Statistic.objects.get(slug='sixth') cell.save() chart = cell.get_chart() assert chart.x_labels == ['web', 'mail', 'phone', 'email'] @@ -603,12 +598,12 @@ def test_chartng_cell_sort_order_desc(app): @with_httmock(bijoe_mock) -def test_chartng_cell_view(app, normal_user): +def test_chartng_cell_view(app, normal_user, statistics): page = Page(title='One', slug='index') page.save() cell = ChartNgCell(page=page, order=1, placeholder='content') - cell.data_reference = 'plop:example' + cell.statistic = Statistic.objects.get(slug='example') cell.save() location = '/api/dataviz/graph/%s/' % cell.id resp = app.get(location) # get data in cache @@ -649,7 +644,7 @@ def test_chartng_cell_view(app, normal_user): assert '222' in resp.text # unsupported dataset - cell.data_reference = 'plop:seventh' + cell.statistic = Statistic.objects.get(slug='seventh') cell.save() resp = app.get(location) # get data in cache resp = app.get('/') @@ -661,7 +656,7 @@ def test_chartng_cell_view(app, normal_user): assert 'Unsupported dataset' in resp.text # durations - cell.data_reference = 'plop:eighth' + cell.statistic = Statistic.objects.get(slug='eighth') cell.chart_type = 'table' cell.save() resp = app.get(location) # get data in cache @@ -680,7 +675,7 @@ def test_chartng_cell_view(app, normal_user): assert '>1 day<' in resp.text # percents - cell.data_reference = 'plop:tenth' + cell.statistic = Statistic.objects.get(slug='tenth') cell.chart_type = 'table' cell.save() resp = app.get(location) # get data in cache @@ -693,55 +688,57 @@ def test_chartng_cell_view(app, normal_user): assert '>10.0%<' in resp.text # deleted visualization - cell.data_reference = 'plop:eleventh' + cell.statistic = Statistic.objects.get(slug='eleventh') cell.save() resp = app.get(location) assert 'not found' in resp.text - # cell with missing cached_json (probably after import and missing - # bijoe visualisation) + # cell with no statistic chosen cell.chart_type = 'table' + cell.statistic = None cell.save() - ChartNgCell.objects.filter(id=cell.id).update(cached_json={}) resp = app.get('/') - assert 'warningnotice' in resp.text + assert not 'cell' in resp.text @with_httmock(bijoe_mock) -def test_chartng_cell_manager(app, admin_user): +def test_chartng_cell_manager(app, admin_user, statistics): page = Page(title='One', slug='index') page.save() + Statistic.objects.create( + slug='unavailable-stat', label='Unavailable Stat', site_slug='plop', available=False + ) app = login(app) cell = ChartNgCell(page=page, order=1, placeholder='content') - cell.data_reference = 'plop:example' + cell.statistic = Statistic.objects.get(slug='example') cell.save() resp = app.get('/manage/pages/%s/' % page.id) - assert resp.form['cdataviz_chartngcell-%s-data_reference' % cell.id].options == [ - (u'plop:eighth', False, u'eighth visualization (duration)'), - (u'plop:eleventh', False, u'eleventh visualization (not found)'), - (u'plop:example', True, u'example visualization (X)'), - (u'plop:fifth', False, u'fifth visualization (loop/X)'), - (u'plop:fourth', False, u'fourth visualization (no axis)'), - (u'plop:nineth', False, u'nineth visualization (loop over varying dimensions)'), - (u'plop:second', False, u'second visualization (Y)'), - (u'plop:seventh', False, u'seventh visualization (loop/X/Y)'), - (u'plop:sixth', False, u'sixth visualization (loop/Y)'), - (u'plop:tenth', False, u'tenth visualization (percents)'), - (u'plop:third', False, u'third visualization (X/Y)'), - ] + statistics_field = resp.form['cdataviz_chartngcell-%s-statistic' % cell.id] + assert len(statistics_field.options) == 12 + assert statistics_field.value == str(cell.statistic.pk) + assert statistics_field.options[1][2] == 'test: eighth visualization (duration)' + assert not 'Unavailable Stat' in resp.text + + cell.statistic = Statistic.objects.get(slug='unavailable-stat') + cell.save() + resp = app.get('/manage/pages/%s/' % page.id) + statistics_field = resp.form['cdataviz_chartngcell-%s-statistic' % cell.id] + assert len(statistics_field.options) == 13 + assert 'Unavailable Stat' in resp.text @with_httmock(bijoe_mock) -def test_table_cell(app, admin_user): +def test_table_cell(app, admin_user, statistics): page = Page(title='One', slug='index') page.save() app = login(app) cell = ChartNgCell(page=page, order=1, placeholder='content') - cell.data_reference = 'plop:example' + cell.statistic = Statistic.objects.get(slug='example') + cell.statistic = Statistic.objects.get(slug='example') cell.chart_type = 'table' cell.save() location = '/api/dataviz/graph/%s/' % cell.id @@ -749,28 +746,109 @@ def test_table_cell(app, admin_user): resp = app.get('/') assert resp.text.count('Total') == 1 - cell.data_reference = 'plop:second' + cell.statistic = Statistic.objects.get(slug='second') cell.save() resp = app.get(location) resp = app.get('/') assert resp.text.count('Total') == 1 - cell.data_reference = 'plop:third' + cell.statistic = Statistic.objects.get(slug='third') cell.save() resp = app.get(location) resp = app.get('/') assert '114' in resp.text assert resp.text.count('Total') == 2 - cell.data_reference = 'plop:fourth' + cell.statistic = Statistic.objects.get(slug='fourth') cell.save() resp = app.get(location) resp = app.get('/') assert resp.text.count('Total') == 0 # total of durations is not computed - cell.data_reference = 'plop:eigth' + cell.statistic = Statistic.objects.get(slug='eighth') cell.save() resp = app.get(location) resp = app.get('/') assert resp.text.count('Total') == 0 + + +def test_dataviz_hourly_unavailable_statistic(freezer, statistics): + all_stats_count = Statistic.objects.count() + assert Statistic.objects.filter(available=True).count() == all_stats_count + + def bijoe_mock_unavailable(url, request): + visualization_json = VISUALIZATION_JSON[2:] + return {'content': json.dumps(visualization_json), 'request': request, 'status_code': 200} + + freezer.move_to(timezone.now() + timedelta(hours=1)) # clear requests cache + appconfig = apps.get_app_config('dataviz') + with HTTMock(bijoe_mock_unavailable): + appconfig.hourly() + assert Statistic.objects.filter(available=True).count() == all_stats_count - 2 + + +def test_dataviz_import_cell(): + page = Page.objects.create(title='One', slug='index') + cell = ChartNgCell.objects.create(page=page, order=1, slug='test', placeholder='content') + statistic = Statistic.objects.create( + slug='example', site_slug='plop', service_slug='bijoe', url='https://example.org' + ) + cell.statistic = statistic + cell.save() + + site_export = [page.get_serialized_page()] + cell.delete() + + Page.load_serialized_pages(site_export) + cell = ChartNgCell.objects.get(slug='test') + assert cell.statistic.pk == statistic.pk + + cell.delete() + statistic.delete() + + Page.load_serialized_pages(site_export) + cell = ChartNgCell.objects.get(slug='test') + assert cell.statistic.slug == statistic.slug + assert cell.statistic.site_slug == statistic.site_slug + assert cell.statistic.service_slug == statistic.service_slug + + +def test_dataviz_cell_migration(): + page = Page.objects.create(title='One', slug='index') + app = 'dataviz' + + migrate_from = [(app, '0012_auto_20201126_1557')] + migrate_to = [(app, '0013_update_chartng_cells')] + executor = MigrationExecutor(connection) + old_apps = executor.loader.project_state(migrate_from).apps + executor.migrate(migrate_from) + + Pagee = old_apps.get_model('data', 'Page') + page = Pagee.objects.get(pk=page.pk) + ChartNgCell = old_apps.get_model(app, 'ChartNgCell') + cell = ChartNgCell.objects.create( + page=page, + order=1, + data_reference='plop:example', + cached_json={ + 'data-url': 'https://bijoe.example.com/visualization/1/json/', + 'path': 'https://bijoe.example.com/visualization/1/iframe/?signature=123', + 'name': 'example visualization (X)', + 'slug': 'example', + }, + ) + + executor = MigrationExecutor(connection) + executor.migrate(migrate_to) + executor.loader.build_graph() + + apps = executor.loader.project_state(migrate_to).apps + ChartNgCell = apps.get_model(app, 'ChartNgCell') + cell = ChartNgCell.objects.get(pk=cell.pk) + assert cell.statistic.slug == 'example' + assert cell.statistic.label == 'example visualization (X)' + assert cell.statistic.url == 'https://bijoe.example.com/visualization/1/json/' + assert cell.statistic.site_slug == 'plop' + assert cell.statistic.service_slug == 'bijoe' + assert cell.statistic.site_title == 'test' -- 2.20.1