From 48af0cf4086b9c9ed874123c788806e35dcb497f Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Wed, 12 Jan 2022 14:40:56 +0100 Subject: [PATCH 5/5] dataviz: add new filters cell (#60547) --- combo/apps/dataviz/forms.py | 79 +++++++- .../migrations/0016_auto_20201215_1624.py | 4 +- .../migrations/0021_chartfilterscell.py | 49 +++++ combo/apps/dataviz/models.py | 29 ++- .../templates/combo/chart-filters.html | 43 +++++ .../dataviz/templates/combo/chartngcell.html | 8 +- combo/apps/dataviz/views.py | 7 +- tests/test_dataviz.py | 181 +++++++++++++++++- tests/test_manager.py | 8 +- tests/test_search.py | 2 +- 10 files changed, 395 insertions(+), 15 deletions(-) create mode 100644 combo/apps/dataviz/migrations/0021_chartfilterscell.py create mode 100644 combo/apps/dataviz/templates/combo/chart-filters.html diff --git a/combo/apps/dataviz/forms.py b/combo/apps/dataviz/forms.py index 59eb6112..28b5b3b7 100644 --- a/combo/apps/dataviz/forms.py +++ b/combo/apps/dataviz/forms.py @@ -27,7 +27,7 @@ from django.utils.translation import ugettext_lazy as _ from combo.utils import cache_during_request, requests, spooler -from .models import ChartCell, ChartNgCell +from .models import TIME_FILTERS, ChartCell, ChartNgCell class ChartForm(forms.ModelForm): @@ -169,3 +169,80 @@ class ChartNgForm(ChartFiltersMixin, forms.ModelForm): else: if not date: self.add_error(template_field, _('Template does not evaluate to a valid date.')) + + +class ChartNgPartialForm(ChartFiltersMixin, forms.ModelForm): + class Meta: + model = ChartNgCell + fields = ( + 'time_range', + 'time_range_start', + 'time_range_end', + ) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fields.update(self.get_filter_fields(self.instance)) + for field in self.fields.values(): + field.required = False + + def clean(self): + for filter_ in self.instance.statistic.filters: + if filter_['id'] in self.data: + self.instance.filter_params[filter_['id']] = self.cleaned_data.get(filter_['id']) + + +class ChartFiltersForm(ChartFiltersMixin, forms.ModelForm): + class Meta: + model = ChartNgCell + fields = ( + 'time_range', + 'time_range_start', + 'time_range_end', + ) + widgets = { + 'time_range_start': forms.DateInput(attrs={'type': 'date'}, format='%Y-%m-%d'), + 'time_range_end': forms.DateInput(attrs={'type': 'date'}, format='%Y-%m-%d'), + } + + def __init__(self, *args, **kwargs): + page = kwargs.pop('page') + super().__init__(*args, **kwargs) + self.fields['time_range'].choices = BLANK_CHOICE_DASH + TIME_FILTERS + + chart_cells = list(ChartNgCell.objects.filter(page=page, statistic__isnull=False).order_by('order')) + if not chart_cells: + self.fields.clear() + return + + first_cell = chart_cells[0] + for field in self._meta.fields: + self.fields[field].initial = getattr(first_cell, field) + dynamic_fields = self.get_filter_fields(first_cell) + dynamic_fields_values = {k: v for k, v in first_cell.filter_params.items()} + + for cell in chart_cells[1:]: + cell_filter_fields = self.get_filter_fields(cell) + + # keep only common fields + dynamic_fields = {k: v for k, v in dynamic_fields.items() if k in cell_filter_fields} + + # keep only same value fields + for field, value in cell.filter_params.items(): + if field in dynamic_fields and value != dynamic_fields_values.get(field): + del dynamic_fields[field] + + if cell.time_range != first_cell.time_range: + for field in self._meta.fields: + self.fields.pop(field, None) + + # ensure compatible choices lists + for field_name, field in cell_filter_fields.items(): + if field_name in dynamic_fields: + dynamic_fields[field_name].choices = [ + x for x in dynamic_fields[field_name].choices if x in field.choices + ] + if dynamic_fields[field_name].choices == []: + del dynamic_fields[field_name] + + self.fields.update(dynamic_fields) diff --git a/combo/apps/dataviz/migrations/0016_auto_20201215_1624.py b/combo/apps/dataviz/migrations/0016_auto_20201215_1624.py index a001e80f..fc11190f 100644 --- a/combo/apps/dataviz/migrations/0016_auto_20201215_1624.py +++ b/combo/apps/dataviz/migrations/0016_auto_20201215_1624.py @@ -2,7 +2,7 @@ from django.db import migrations, models -from combo.apps.dataviz.models import TIME_FILTERS +from combo.apps.dataviz.models import TIME_FILTERS, TIME_FILTERS_TEMPLATE class Migration(migrations.Migration): @@ -17,7 +17,7 @@ class Migration(migrations.Migration): name='time_range', field=models.CharField( blank=True, - choices=TIME_FILTERS, + choices=TIME_FILTERS + TIME_FILTERS_TEMPLATE, max_length=20, verbose_name='Filtering (time)', ), diff --git a/combo/apps/dataviz/migrations/0021_chartfilterscell.py b/combo/apps/dataviz/migrations/0021_chartfilterscell.py new file mode 100644 index 00000000..83dc2679 --- /dev/null +++ b/combo/apps/dataviz/migrations/0021_chartfilterscell.py @@ -0,0 +1,49 @@ +# Generated by Django 2.2.19 on 2022-01-18 10:17 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('data', '0051_link_cell_max_length'), + ('auth', '0011_update_proxy_permissions'), + ('dataviz', '0020_auto_20220118_1103'), + ] + + operations = [ + migrations.CreateModel( + name='ChartFiltersCell', + fields=[ + ( + 'id', + models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + ('placeholder', models.CharField(max_length=20)), + ('order', models.PositiveIntegerField()), + ('slug', models.SlugField(blank=True, verbose_name='Slug')), + ( + 'extra_css_class', + models.CharField( + blank=True, max_length=100, verbose_name='Extra classes for CSS styling' + ), + ), + ( + 'template_name', + models.CharField(blank=True, max_length=50, null=True, verbose_name='Cell Template'), + ), + ('public', models.BooleanField(default=True, verbose_name='Public')), + ( + 'restricted_to_unlogged', + models.BooleanField(default=False, verbose_name='Restrict to unlogged users'), + ), + ('last_update_timestamp', models.DateTimeField(auto_now=True)), + ('groups', models.ManyToManyField(blank=True, to='auth.Group', verbose_name='Groups')), + ('page', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='data.Page')), + ], + options={ + 'verbose_name': 'Filters', + }, + ), + ] diff --git a/combo/apps/dataviz/models.py b/combo/apps/dataviz/models.py index 3a8b2f5a..361b239c 100644 --- a/combo/apps/dataviz/models.py +++ b/combo/apps/dataviz/models.py @@ -158,7 +158,7 @@ class Statistic(models.Model): ) -TIME_FILTERS = ( +TIME_FILTERS = [ ('previous-year', _('Previous year')), ('current-year', _('Current year')), ('next-year', _('Next year')), @@ -169,8 +169,8 @@ TIME_FILTERS = ( ('current-week', _('Current week')), ('next-week', _('Next week')), ('range', _('Free range (date)')), - ('range-template', _('Free range (template)')), -) +] +TIME_FILTERS_TEMPLATE = [('range-template', _('Free range (template)'))] @register_cell_class @@ -192,7 +192,7 @@ class ChartNgCell(CellBase): _('Filtering (time)'), max_length=20, blank=True, - choices=TIME_FILTERS, + choices=TIME_FILTERS + TIME_FILTERS_TEMPLATE, ) time_range_start = models.DateField(_('From'), null=True, blank=True) time_range_end = models.DateField(_('To'), null=True, blank=True) @@ -678,3 +678,24 @@ class ChartNgCell(CellBase): data['x_labels'] = x_labels for i, serie in enumerate(data['series']): serie['data'] = [values[i] for values in aggregates.values()] + + +@register_cell_class +class ChartFiltersCell(CellBase): + title = _('Filters') + default_template_name = 'combo/chart-filters.html' + max_one_by_page = True + + class Meta: + verbose_name = _('Filters') + + @classmethod + def is_enabled(cls): + return settings.STATISTICS_PROVIDERS + + def get_cell_extra_context(self, context): + from .forms import ChartFiltersForm + + ctx = super().get_cell_extra_context(context) + ctx['form'] = ChartFiltersForm(page=self.page) + return ctx diff --git a/combo/apps/dataviz/templates/combo/chart-filters.html b/combo/apps/dataviz/templates/combo/chart-filters.html new file mode 100644 index 00000000..496dbb1b --- /dev/null +++ b/combo/apps/dataviz/templates/combo/chart-filters.html @@ -0,0 +1,43 @@ +{% load i18n %} + +{% block cell-content %} +

{{ cell.title }}

+ +
+ {% if form.fields %} +
+ {{ form.as_p }} +
+ +
+
+ {% else %} +

+ {% blocktrans trimmed %} + No filters are available. Note that only filters that are shared between all chart cells will appear. Furthermore, in case they have a value, it must be the same accross all cells. + {% endblocktrans %} +

+ {% endif %} +
+ + + +{% endblock %} diff --git a/combo/apps/dataviz/templates/combo/chartngcell.html b/combo/apps/dataviz/templates/combo/chartngcell.html index 10df161c..732cac6d 100644 --- a/combo/apps/dataviz/templates/combo/chartngcell.html +++ b/combo/apps/dataviz/templates/combo/chartngcell.html @@ -13,12 +13,18 @@ $(function() { var chart_cell = $('#chart-{{cell.id}}').parent(); var new_width = Math.floor($(chart_cell).width()); var ratio = new_width / last_width; + var filter_params = $('#chart-filters').serialize(); if (ratio > 1.2 || ratio < 0.8) { $('#chart-{{cell.id}}').attr('src', - "{% url 'combo-dataviz-graph' cell=cell.id %}?width=" + new_width); + "{% url 'combo-dataviz-graph' cell=cell.id %}?width=" + new_width + '&' + filter_params); last_width = new_width; } }).trigger('combo:resize-graphs'); + $(window).on('combo:refresh-graphs', function() { + var filter_params = $('#chart-filters').serialize(); + $('#chart-{{cell.id}}').attr('src', + "{% url 'combo-dataviz-graph' cell=cell.id %}?width=" + last_width + '&' + filter_params); + }); }); {% endif %} diff --git a/combo/apps/dataviz/views.py b/combo/apps/dataviz/views.py index 8a5661b0..9d6c54b5 100644 --- a/combo/apps/dataviz/views.py +++ b/combo/apps/dataviz/views.py @@ -23,6 +23,7 @@ from requests.exceptions import HTTPError from combo.utils import get_templated_url, requests +from .forms import ChartNgPartialForm from .models import ChartNgCell, Gauge, UnsupportedDataSet @@ -49,8 +50,12 @@ class DatavizGraphView(DetailView): return super().dispatch(request, *args, **kwargs) def get(self, request, *args, **kwargs): + form = ChartNgPartialForm(request.GET, instance=self.cell) + if not form.is_valid(): + return self.svg_error(_('Wrong parameters.')) + try: - chart = self.cell.get_chart( + chart = form.instance.get_chart( width=int(request.GET['width']) if request.GET.get('width') else None, height=int(request.GET['height']) if request.GET.get('height') else int(self.cell.height), ) diff --git a/tests/test_dataviz.py b/tests/test_dataviz.py index d5d02c6a..79f3dac6 100644 --- a/tests/test_dataviz.py +++ b/tests/test_dataviz.py @@ -10,7 +10,7 @@ from django.test import override_settings from httmock import HTTMock, remember_called, urlmatch, with_httmock from requests.exceptions import HTTPError -from combo.apps.dataviz.models import ChartNgCell, Gauge, Statistic, UnsupportedDataSet +from combo.apps.dataviz.models import ChartFiltersCell, ChartNgCell, Gauge, Statistic, UnsupportedDataSet from combo.data.models import Page, ValidityInfo from .test_public import login @@ -1743,3 +1743,182 @@ def test_chartng_cell_new_api_aggregation(new_api_statistics, app, admin_user, n assert len(chart.x_labels) == 1 assert chart.x_labels == ['W53-2020'] assert chart.raw_series == [([19], {'title': 'Serie 1'})] + + +@with_httmock(new_api_mock) +def test_chart_filters_cell(new_api_statistics, app, admin_user, nocache): + page = Page.objects.create(title='One', slug='index') + ChartFiltersCell.objects.create(page=page, order=1, placeholder='content') + app = login(app) + resp = app.get('/') + assert 'No filters are available' in resp.text + + # add unconfigured chart + first_cell = ChartNgCell.objects.create(page=page, order=2, placeholder='content') + resp = app.get('/') + assert 'No filters are available' in resp.text + + # add statistics to chart + first_cell.statistic = Statistic.objects.get(slug='one-serie') + first_cell.save() + resp = app.get('/') + assert len(resp.form.fields) == 7 + assert 'time_range_start' in resp.form.fields + assert 'time_range_end' in resp.form.fields + + time_range_field = resp.form['time_range'] + assert time_range_field.value == '' + assert time_range_field.options == [ + ('', True, '---------'), + ('previous-year', False, 'Previous year'), + ('current-year', False, 'Current year'), + ('next-year', False, 'Next year'), + ('previous-month', False, 'Previous month'), + ('current-month', False, 'Current month'), + ('next-month', False, 'Next month'), + ('previous-week', False, 'Previous week'), + ('current-week', False, 'Current week'), + ('next-week', False, 'Next week'), + ('range', False, 'Free range (date)'), + ] + + time_interval_field = resp.form['time_interval'] + assert time_interval_field.value == 'month' + assert time_interval_field.options == [ + ('day', False, 'Day'), + ('month', True, 'Month'), + ('year', False, 'Year'), + ('week', False, 'Week'), + ('weekday', False, 'Week day'), + ] + + service_field = resp.form['service'] + assert service_field.value == 'chrono' + assert service_field.options == [ + ('', False, '---------'), + ('chrono', True, 'Chrono'), + ('combo', False, 'Combo'), + ] + + ou_field = resp.form['ou'] + assert ou_field.value == '' + assert ou_field.options == [ + ('', True, '---------'), + ('default', False, 'Default OU'), + ('other', False, 'Other OU'), + ] + + # adding new cell with same statistics changes nothing + cell = ChartNgCell(page=page, order=3, placeholder='content') + cell.statistic = Statistic.objects.get(slug='one-serie') + cell.save() + old_resp = resp + resp = app.get('/') + for field in ('time_range', 'time_interval', 'service', 'ou'): + assert resp.form[field].options == old_resp.form[field].options + + # changing one filter value makes it disappear + cell.filter_params = {'ou': 'default'} + cell.save() + resp = app.get('/') + assert 'ou' not in resp.form.fields + for field in ('time_range', 'time_interval', 'service'): + assert resp.form[field].options == old_resp.form[field].options + + # setting the same value for the other cell makes it appear again + first_cell.filter_params = {'ou': 'default'} + first_cell.save() + resp = app.get('/') + assert resp.form['ou'].value == 'default' + + # changing statistics type of cell remove some fields + cell.statistic = Statistic.objects.get(slug='daily') + cell.save() + resp = app.get('/') + assert 'ou' not in resp.form.fields + assert 'service' not in resp.form.fields + for field in ('time_range', 'time_interval'): + assert resp.form[field].options == old_resp.form[field].options + + # changing time_interval value makes interval fields disappear + cell.time_range = 'next-year' + cell.save() + old_resp = resp + resp = app.get('/') + assert 'time_range' not in resp.form.fields + assert 'time_range_start' not in resp.form.fields + assert 'time_range_end' not in resp.form.fields + assert resp.form['time_interval'].options == old_resp.form['time_interval'].options + + # setting the same value for the other cell makes it appear again + first_cell.time_range = 'next-year' + first_cell.save() + resp = app.get('/') + assert resp.form['time_range'].value == 'next-year' + assert resp.form['time_interval'].options == old_resp.form['time_interval'].options + + # only common choices are shown + first_cell.statistic.filters[0]['options'].remove({'id': 'day', 'label': 'Day'}) + first_cell.statistic.save() + resp = app.get('/') + assert resp.form['time_interval'].options == [ + ('month', True, 'Month'), + ('year', False, 'Year'), + ] + + # if no common choices exist, field is removed + first_cell.statistic.filters[0]['options'] = [{'id': 'random', 'label': 'Random'}] + first_cell.statistic.save() + resp = app.get('/') + assert 'time_interval' not in resp.form.fields + assert resp.form['time_range'].value == 'next-year' + + # form is not shown if no common filters exist + first_cell.time_range = 'current-year' + first_cell.save() + resp = app.get('/') + assert 'No filters are available' in resp.text + + +@with_httmock(new_api_mock) +def test_chartng_cell_api_view_get_parameters(app, normal_user, new_api_statistics, nocache): + page = Page.objects.create(title='One', slug='index') + cell = ChartNgCell(page=page, order=1, placeholder='content') + cell.statistic = Statistic.objects.get(slug='one-serie') + cell.save() + + location = '/api/dataviz/graph/%s/' % cell.id + app.get(location) + request = new_api_mock.call['requests'][0] + assert 'time_interval=' not in request.url + assert 'ou=' not in request.url + + cell.filter_params = {'time_interval': 'month', 'ou': 'default'} + cell.save() + app.get(location) + request = new_api_mock.call['requests'][1] + assert 'time_interval=month' in request.url + assert 'ou=default' in request.url + + app.get(location + '?time_interval=year') + request = new_api_mock.call['requests'][2] + assert 'time_interval=year' in request.url + assert 'ou=default' in request.url + + cell.filter_params.clear() + cell.statistic = Statistic.objects.get(slug='filter-multiple') + cell.save() + app.get(location + '?color=green&color=blue') + request = new_api_mock.call['requests'][3] + assert 'color=green&color=blue' in request.url + + # unknown params + app.get(location + '?time_interval=month&ou=default') + request = new_api_mock.call['requests'][4] + assert 'time_interval=' not in request.url + assert 'ou=' not in request.url + + # wrong params + resp = app.get(location + '?time_range_start=xxx') + assert 'Wrong parameters' in resp.text + assert len(new_api_mock.call['requests']) == 5 diff --git a/tests/test_manager.py b/tests/test_manager.py index 9226c29f..056e07f8 100644 --- a/tests/test_manager.py +++ b/tests/test_manager.py @@ -926,7 +926,7 @@ def test_site_export_import_json(app, admin_user): resp.form['site_file'] = Upload('site-export.json', site_export, 'application/json') with CaptureQueriesContext(connection) as ctx: resp = resp.form.submit() - assert len(ctx.captured_queries) in [303, 304] + assert len(ctx.captured_queries) in [308, 309] assert Page.objects.count() == 4 assert PageSnapshot.objects.all().count() == 4 @@ -937,7 +937,7 @@ def test_site_export_import_json(app, admin_user): resp.form['site_file'] = Upload('site-export.json', site_export, 'application/json') with CaptureQueriesContext(connection) as ctx: resp = resp.form.submit() - assert len(ctx.captured_queries) == 273 + assert len(ctx.captured_queries) == 277 assert set(Page.objects.get(slug='one').related_cells['cell_types']) == {'data_textcell', 'data_linkcell'} assert Page.objects.count() == 4 assert LinkCell.objects.count() == 2 @@ -2276,7 +2276,7 @@ def test_page_versionning(app, admin_user): with CaptureQueriesContext(connection) as ctx: resp2 = resp.click('view', index=1) - assert len(ctx.captured_queries) == 70 + assert len(ctx.captured_queries) == 71 assert Page.snapshots.latest('pk').related_cells == {'cell_types': ['data_textcell']} assert resp2.text.index('Hello world') < resp2.text.index('Foobar3') @@ -2337,7 +2337,7 @@ def test_page_versionning(app, admin_user): resp = resp.click('restore', index=6) with CaptureQueriesContext(connection) as ctx: resp = resp.form.submit().follow() - assert len(ctx.captured_queries) == 144 + assert len(ctx.captured_queries) == 146 resp2 = resp.click('See online') assert resp2.text.index('Foobar1') < resp2.text.index('Foobar2') < resp2.text.index('Foobar3') diff --git a/tests/test_search.py b/tests/test_search.py index e556699e..308ecbf4 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -1420,7 +1420,7 @@ def test_index_site_num_queries(settings, app): assert IndexedCell.objects.count() == 50 with CaptureQueriesContext(connection) as ctx: index_site() - assert len(ctx.captured_queries) == 224 + assert len(ctx.captured_queries) == 225 SearchCell.objects.create( page=page, placeholder='content', order=0, _search_services={'data': ['search1']} -- 2.30.2