From f72a20ab933b6aa5a6ad702d46001091b205fefa Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Wed, 2 Dec 2020 14:23:14 +0100 Subject: [PATCH 2/3] dataviz: handle api filters (#49175) --- combo/apps/dataviz/__init__.py | 1 + combo/apps/dataviz/forms.py | 47 +++++++++- .../migrations/0015_auto_20201202_1424.py | 26 ++++++ combo/apps/dataviz/models.py | 3 + tests/test_dataviz.py | 86 ++++++++++++++++++- 5 files changed, 157 insertions(+), 6 deletions(-) create mode 100644 combo/apps/dataviz/migrations/0015_auto_20201202_1424.py diff --git a/combo/apps/dataviz/__init__.py b/combo/apps/dataviz/__init__.py index 123d1acb..4a6bc947 100644 --- a/combo/apps/dataviz/__init__.py +++ b/combo/apps/dataviz/__init__.py @@ -64,6 +64,7 @@ class AppConfig(django.apps.AppConfig): 'label': stat['name'], 'url': stat.get('data-url') or stat['url'], 'site_title': site_dict.get('title', ''), + 'filters': stat.get('filters', []), 'available': True, } ) diff --git a/combo/apps/dataviz/forms.py b/combo/apps/dataviz/forms.py index 117f366f..9ff2879b 100644 --- a/combo/apps/dataviz/forms.py +++ b/combo/apps/dataviz/forms.py @@ -14,6 +14,8 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from collections import OrderedDict + from django import forms from django.conf import settings from django.db.models import Q @@ -42,6 +44,8 @@ class ChartForm(forms.ModelForm): class ChartNgForm(forms.ModelForm): + blank_choice = ('', '---------') + class Meta: model = ChartNgCell fields = ('title', 'statistic', 'chart_type', 'height', 'sort_order', @@ -49,7 +53,42 @@ class ChartNgForm(forms.ModelForm): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - q_filters = Q(available=True) - if self.instance.statistic: - q_filters |= Q(pk=self.instance.statistic.pk) - self.fields['statistic'].queryset = self.fields['statistic'].queryset.filter(q_filters) + stat_field = self.fields['statistic'] + if not self.instance.statistic: + stat_field.queryset = stat_field.queryset.filter(available=True) + return + + # display current statistic in choices even if unavailable + stat_field.queryset = stat_field.queryset.filter(Q(available=True) | Q(pk=self.instance.statistic.pk)) + + field_ids = list(self._meta.fields) + field_insert_index = field_ids.index('statistic') + 1 + for filter_ in reversed(self.instance.statistic.filters): + filter_id = filter_['id'] + choices = [(option['id'], option['label']) for option in filter_['options']] + initial = self.instance.filter_params.get(filter_id) or filter_.get('default') + + required = filter_.get('required', False) + if not required: + choices.insert(0, self.blank_choice) + + self.fields[filter_id] = forms.ChoiceField( + label=filter_['label'], choices=choices, required=required, initial=initial + ) + field_ids.insert(field_insert_index, filter_id) + + # reorder so that filter fields appear after 'statistic' field + self.fields = OrderedDict((field_id, self.fields[field_id]) for field_id in field_ids) + + def save(self, *args, **kwargs): + if 'statistic' in self.changed_data: + self.instance.filter_params.clear() + else: + for filter_ in self.instance.statistic.filters: + field = filter_['id'] + value = self.cleaned_data.get(field) + if value: + self.instance.filter_params[field] = value + else: + self.instance.filter_params.pop(field, None) + return super().save(*args, **kwargs) diff --git a/combo/apps/dataviz/migrations/0015_auto_20201202_1424.py b/combo/apps/dataviz/migrations/0015_auto_20201202_1424.py new file mode 100644 index 00000000..7537d0fe --- /dev/null +++ b/combo/apps/dataviz/migrations/0015_auto_20201202_1424.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2020-12-02 13:24 +from __future__ import unicode_literals + +from django.db import migrations +import jsonfield.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('dataviz', '0014_auto_20201130_1534'), + ] + + operations = [ + migrations.AddField( + model_name='chartngcell', + name='filter_params', + field=jsonfield.fields.JSONField(default=dict), + ), + migrations.AddField( + model_name='statistic', + name='filters', + field=jsonfield.fields.JSONField(default=list), + ), + ] diff --git a/combo/apps/dataviz/models.py b/combo/apps/dataviz/models.py index de87fbe1..e416b8cd 100644 --- a/combo/apps/dataviz/models.py +++ b/combo/apps/dataviz/models.py @@ -115,6 +115,7 @@ class Statistic(models.Model): service_slug = models.SlugField(_('Service slug'), max_length=256) site_title = models.CharField(_('Site title'), max_length=256) url = models.URLField(_('Data URL')) + filters = JSONField(default=list) available = models.BooleanField(_('Available data'), default=True) last_update = models.DateTimeField(_('Last update'), null=True, auto_now=True) @@ -139,6 +140,7 @@ class ChartNgCell(CellBase): statistic = models.ForeignKey( verbose_name=_('Data'), to=Statistic, blank=False, null=True, on_delete=models.SET_NULL, related_name='cells' ) + filter_params = JSONField(default=dict) title = models.CharField(_('Title'), max_length=150, blank=True) chart_type = models.CharField(_('Chart Type'), max_length=20, default='bar', choices=( @@ -213,6 +215,7 @@ class ChartNgCell(CellBase): def get_chart(self, width=None, height=None, raise_if_not_cached=False): response = requests.get( self.statistic.url, + params=self.filter_params, cache_duration=300, remote_service='auto', without_user=True, diff --git a/tests/test_dataviz.py b/tests/test_dataviz.py index 4fa068e6..b8c292c2 100644 --- a/tests/test_dataviz.py +++ b/tests/test_dataviz.py @@ -3,7 +3,7 @@ import json import mock import pytest from datetime import timedelta -from httmock import HTTMock, with_httmock +from httmock import HTTMock, with_httmock, remember_called from requests.exceptions import HTTPError from django.apps import apps @@ -225,6 +225,27 @@ STATISTICS_LIST = { 'url': 'https://authentic.example.com/api/statistics/one-serie/', 'name': 'One serie stat', 'id': 'one-serie', + "filters": [ + { + "default": "month", + "id": "time_interval", + "label": "Time interval", + "options": [ + {"id": "day", "label": "Day"}, + {"id": "month", "label": "Month"}, + {"id": "year", "label": "Year"}, + ], + "required": True, + }, + { + "id": "ou", + "label": "Organizational Unit", + "options": [ + {"id": "default", "label": "Default OU"}, + {"id": "other", "label": "Other OU"}, + ], + }, + ], }, { 'url': 'https://authentic.example.com/api/statistics/two-series/', @@ -245,6 +266,7 @@ STATISTICS_LIST = { } +@remember_called def new_api_mock(url, request): if url.path == '/visualization/json/': # nothing from bijoe return {'content': b'{}', 'request': request, 'status_code': 200} @@ -905,11 +927,51 @@ def test_chartng_cell_manager_new_api(app, admin_user, new_api_statistics): app = login(app) resp = app.get('/manage/pages/%s/' % page.id) - statistics_field = resp.form['cdataviz_chartngcell-%s-statistic' % cell.id] + field_prefix = 'cdataviz_chartngcell-%s-' % cell.id + statistics_field = resp.form[field_prefix + 'statistic'] assert len(statistics_field.options) == 5 assert statistics_field.value == str(cell.statistic.pk) assert statistics_field.options[3][2] == 'Connection: One serie stat' + time_interval_field = resp.form[field_prefix + 'time_interval'] + assert time_interval_field.pos == statistics_field.pos + 1 + assert time_interval_field.value == 'month' + assert time_interval_field.options == [ + ('day', False, 'Day'), + ('month', True, 'Month'), + ('year', False, 'Year'), + ] + + ou_field = resp.form[field_prefix + 'ou'] + assert ou_field.pos == statistics_field.pos + 2 + assert ou_field.value == '' + assert ou_field.options == [ + ('', True, '---------'), + ('default', False, 'Default OU'), + ('other', False, 'Other OU'), + ] + resp.form[field_prefix + 'ou'] = 'default' + + resp = resp.form.submit().follow() + assert resp.form[field_prefix + 'ou'].value == 'default' + cell.refresh_from_db() + assert cell.filter_params == {'ou': 'default', 'time_interval': 'month'} + resp.form[field_prefix + 'ou'] = '' + + resp = resp.form.submit().follow() + assert resp.form[field_prefix + 'ou'].value == '' + cell.refresh_from_db() + assert cell.filter_params == {'time_interval': 'month'} + + no_filters_stat = Statistic.objects.get(slug='two-series') + resp.form[field_prefix + 'statistic'] = no_filters_stat.pk + resp = resp.form.submit().follow() + assert resp.form[field_prefix + 'statistic'].value == str(no_filters_stat.pk) + assert field_prefix + 'time_interval' not in resp.form.fields + assert field_prefix + 'ou' not in resp.form.fields + cell.refresh_from_db() + assert cell.filter_params == {} + @with_httmock(bijoe_mock) def test_table_cell(app, admin_user, statistics): @@ -1072,3 +1134,23 @@ def test_dataviz_api_list_statistics(new_api_statistics): assert statistic.site_title == 'Connection' assert statistic.url == 'https://authentic.example.com/api/statistics/one-serie/' assert statistic.available + + +@with_httmock(new_api_mock) +def test_chartng_cell_new_api_filter_params(new_api_statistics, nocache): + page = Page.objects.create(title='One', slug='index') + cell = ChartNgCell(page=page, order=1, placeholder='content') + cell.statistic = Statistic.objects.get(slug='one-serie') + cell.save() + + chart = cell.get_chart() + request = new_api_mock.call['requests'][0] + assert 'time_interval' not in request.url + assert 'ou' not in request.url + + cell.filter_params = {'time_interval': 'day', 'ou': 'default'} + cell.save() + chart = cell.get_chart() + request = new_api_mock.call['requests'][1] + assert 'time_interval=day' in request.url + assert 'ou=default' in request.url -- 2.20.1