From e3e4e12331d2eff8abeb09ab73efe10509cd5e27 Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Thu, 10 Feb 2022 15:00:08 +0100 Subject: [PATCH] dataviz: allow page variable as filter value (#57616) --- combo/apps/dataviz/forms.py | 22 +- combo/apps/dataviz/models.py | 42 +++- .../dataviz/templates/combo/chartngcell.html | 20 +- combo/apps/dataviz/views.py | 20 +- combo/utils/spooler.py | 7 +- tests/test_dataviz.py | 189 ++++++++++++++++++ 6 files changed, 283 insertions(+), 17 deletions(-) diff --git a/combo/apps/dataviz/forms.py b/combo/apps/dataviz/forms.py index 1153d6d1..67fd3169 100644 --- a/combo/apps/dataviz/forms.py +++ b/combo/apps/dataviz/forms.py @@ -70,16 +70,28 @@ class ChartFiltersMixin: choices = [(option['id'], option['label']) for option in filter_['options']] initial = cell.filter_params.get(filter_id, filter_.get('default')) - possible_choices = {choice[0] for choice in choices} - for choice in initial if isinstance(initial, list) else [initial]: - if choice and choice not in possible_choices: - choices.append((choice, _('%s (unavailable)') % choice)) - required = filter_.get('required', False) multiple = filter_.get('multiple') if not required and not multiple: choices = BLANK_CHOICE_DASH + choices + extra_variables = cell.page.get_extra_variables_keys() + variable_choices = [('variable:' + key, key) for key in extra_variables] + + possible_choices = {choice[0] for choice in choices} + for choice in initial if isinstance(initial, list) else [initial]: + if not choice: + continue + if choice.startswith('variable:'): + variable = choice.replace('variable:', '') + if not variable in extra_variables: + variable_choices.append((choice, _('%s (unavailable)') % variable)) + elif choice not in possible_choices: + choices.append((choice, _('%s (unavailable)') % choice)) + + if variable_choices and not multiple and filter_id != 'time_interval': + choices.append((_('Page variables'), variable_choices)) + field_class = forms.MultipleChoiceField if multiple else forms.ChoiceField fields[filter_id] = field_class( label=filter_['label'], choices=choices, required=required, initial=initial diff --git a/combo/apps/dataviz/models.py b/combo/apps/dataviz/models.py index d75c0585..d077dfe8 100644 --- a/combo/apps/dataviz/models.py +++ b/combo/apps/dataviz/models.py @@ -25,12 +25,13 @@ from dateutil.relativedelta import MO, relativedelta from django.conf import settings from django.contrib.postgres.fields import JSONField from django.db import models, transaction -from django.template import Context, Template +from django.template import Context, RequestContext, Template, TemplateSyntaxError, VariableDoesNotExist from django.template.defaultfilters import date as format_date from django.urls import reverse from django.utils import timezone from django.utils.dates import WEEKDAYS from django.utils.encoding import force_text +from django.utils.functional import cached_property from django.utils.translation import gettext from django.utils.translation import ugettext_lazy as _ from django.utils.translation import ungettext @@ -45,6 +46,14 @@ class UnsupportedDataSet(Exception): pass +class MissingRequest(Exception): + pass + + +class MissingVariable(Exception): + pass + + @register_cell_class class Gauge(CellBase): title = models.CharField(_('Title'), max_length=150, blank=True, null=True) @@ -298,10 +307,17 @@ class ChartNgCell(CellBase): def get_cell_extra_context(self, context): ctx = super().get_cell_extra_context(context) if self.chart_type == 'table' and self.statistic and self.statistic.url: + self._context = context try: chart = self.get_chart(raise_if_not_cached=not (context.get('synchronous'))) except UnsupportedDataSet: ctx['table'] = '

%s

' % _('Unsupported dataset.') + except MissingVariable: + ctx['table'] = '

%s

' % _('Page variable not found.') + except TemplateSyntaxError: + ctx['table'] = '

%s

' % _('Syntax error in page variable.') + except VariableDoesNotExist: + ctx['table'] = '

%s

' % _('Cannot evaluate page variable.') except HTTPError as e: if e.response.status_code == 404: ctx['table'] = '

%s

' % _('Visualization not found.') @@ -390,7 +406,8 @@ class ChartNgCell(CellBase): return chart def get_filter_params(self): - params = {k: v for k, v in self.filter_params.items() if v} + params = {k: self.evaluate_filter_value(v) for k, v in self.filter_params.items() if v} + now = timezone.now().date() if self.time_range == 'current-year': params['start'] = date(year=now.year, month=1, day=1) @@ -440,6 +457,27 @@ class ChartNgCell(CellBase): params['time_interval'] = 'day' return params + def evaluate_filter_value(self, value): + if isinstance(value, list) or not value.startswith('variable:'): + return value + + try: + variable = self.page.extra_variables[value.replace('variable:', '')] + except KeyError: + raise MissingVariable + + return Template(variable).render(self.request_context) + + @cached_property + def request_context(self): + if hasattr(self, '_context'): + return Context(self._context) + + if not hasattr(self, '_request'): + raise MissingRequest + + return RequestContext(self._request, self._request.extra_context) + def parse_response(self, response, chart): # normalize axis to have a fake axis when there are no dimensions and # always a x axis when there is a single dimension. diff --git a/combo/apps/dataviz/templates/combo/chartngcell.html b/combo/apps/dataviz/templates/combo/chartngcell.html index 732cac6d..fb6b31f8 100644 --- a/combo/apps/dataviz/templates/combo/chartngcell.html +++ b/combo/apps/dataviz/templates/combo/chartngcell.html @@ -9,21 +9,29 @@ diff --git a/combo/apps/dataviz/views.py b/combo/apps/dataviz/views.py index 9d6c54b5..5720f780 100644 --- a/combo/apps/dataviz/views.py +++ b/combo/apps/dataviz/views.py @@ -14,9 +14,11 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from django.core import signing from django.core.exceptions import PermissionDenied -from django.http import Http404, HttpResponse +from django.http import Http404, HttpResponse, HttpResponseBadRequest from django.shortcuts import render +from django.template import TemplateSyntaxError, VariableDoesNotExist from django.utils.translation import ugettext_lazy as _ from django.views.generic import DetailView from requests.exceptions import HTTPError @@ -24,7 +26,7 @@ from requests.exceptions import HTTPError from combo.utils import get_templated_url, requests from .forms import ChartNgPartialForm -from .models import ChartNgCell, Gauge, UnsupportedDataSet +from .models import ChartNgCell, Gauge, MissingVariable, UnsupportedDataSet def ajax_gauge_count(request, *args, **kwargs): @@ -54,6 +56,14 @@ class DatavizGraphView(DetailView): if not form.is_valid(): return self.svg_error(_('Wrong parameters.')) + request.extra_context = {} + if request.GET.get('ctx'): + try: + request.extra_context = signing.loads(request.GET['ctx']) + except signing.BadSignature: + return HttpResponseBadRequest('bad signature') + + form.instance._request = request try: chart = form.instance.get_chart( width=int(request.GET['width']) if request.GET.get('width') else None, @@ -61,6 +71,12 @@ class DatavizGraphView(DetailView): ) except UnsupportedDataSet: return self.svg_error(_('Unsupported dataset.')) + except MissingVariable: + return self.svg_error(_('Page variable not found.')) + except TemplateSyntaxError: + return self.svg_error(_('Syntax error in page variable.')) + except VariableDoesNotExist: + return self.svg_error(_('Cannot evaluate page variable.')) except HTTPError as e: if e.response.status_code == 404: return self.svg_error(_('Visualization not found.')) diff --git a/combo/utils/spooler.py b/combo/utils/spooler.py index 347cdc58..86786935 100644 --- a/combo/utils/spooler.py +++ b/combo/utils/spooler.py @@ -95,10 +95,13 @@ def refresh_statistics_list(): @tenantspool def refresh_statistics_data(cell_pk): - from combo.apps.dataviz.models import ChartNgCell + from combo.apps.dataviz.models import ChartNgCell, MissingRequest, MissingVariable try: cell = ChartNgCell.objects.get(pk=cell_pk) except ChartNgCell.DoesNotExist: return - cell.get_statistic_data(invalidate_cache=True) + try: + cell.get_statistic_data(invalidate_cache=True) + except (MissingRequest, MissingVariable): + pass diff --git a/tests/test_dataviz.py b/tests/test_dataviz.py index 4a9b388b..1d4fb76e 100644 --- a/tests/test_dataviz.py +++ b/tests/test_dataviz.py @@ -1377,6 +1377,68 @@ def test_chartng_cell_manager_new_api_dynamic_fields(app, admin_user, new_api_st assert 'time_interval' in resp.text +@with_httmock(new_api_mock) +def test_chartng_cell_manager_new_api_page_variables(app, admin_user, new_api_statistics): + page = Page.objects.create(title='One', slug='index') + cell = ChartNgCell(page=page, order=1, placeholder='content') + cell.statistic = Statistic.objects.get(slug='one-serie') + cell.save() + + app = login(app) + resp = app.get('/manage/pages/%s/' % page.id) + assert '' not in resp.text + + page.extra_variables = {'foo': 'bar', 'bar_id': '{{ 40|add:2 }}'} + page.save() + resp = app.get('/manage/pages/%s/' % page.id) + assert '' in resp.text + + field_prefix = 'cdataviz_chartngcell-%s-' % cell.id + assert resp.form[field_prefix + 'ou'].options == [ + ('', True, '---------'), + ('default', False, 'Default OU'), + ('other', False, 'Other OU'), + ('variable:bar_id', False, 'bar_id'), + ('variable:foo', False, 'foo'), + ] + assert resp.form[field_prefix + 'service'].options == [ + ('', False, '---------'), + ('chrono', True, 'Chrono'), + ('combo', False, 'Combo'), + ('variable:bar_id', False, 'bar_id'), + ('variable:foo', False, 'foo'), + ] + + resp.form[field_prefix + 'ou'] = 'variable:foo' + resp = resp.form.submit().follow() + assert resp.form[field_prefix + 'ou'].value == 'variable:foo' + cell.refresh_from_db() + assert cell.filter_params['ou'] == 'variable:foo' + + del page.extra_variables['foo'] + page.save() + resp = app.get('/manage/pages/%s/' % page.id) + assert resp.form[field_prefix + 'ou'].options == [ + ('', False, '---------'), + ('default', False, 'Default OU'), + ('other', False, 'Other OU'), + ('variable:bar_id', False, 'bar_id'), + ('variable:foo', True, 'foo (unavailable)'), + ] + + # no variables allowed for time_interval + time_interval_field = resp.form[field_prefix + 'time_interval'] + assert [x[0] for x in time_interval_field.options] == ['day', 'month', 'year', 'week', 'weekday'] + + # no variables allowed for multiple choice field + cell.statistic = Statistic.objects.get(slug='filter-multiple') + cell.save() + resp = app.get('/manage/pages/%s/' % page.id) + + color_field = resp.form[field_prefix + 'color'] + assert [x[0] for x in color_field.options] == ['red', 'green', 'blue'] + + @with_httmock(bijoe_mock) def test_table_cell(app, admin_user, statistics): page = Page(title='One', slug='index') @@ -1650,6 +1712,125 @@ def test_chartng_cell_new_api_filter_params_month(new_api_statistics, nocache, f assert 'start=2021-12-01' in request.url and 'end=2022-01-01' in request.url +@with_httmock(new_api_mock) +def test_chartng_cell_new_api_filter_params_page_variables(app, admin_user, new_api_statistics, nocache): + Page.objects.create(title='One', slug='index') + page = Page.objects.create( + title='One', + slug='cards', + sub_slug='card_id', + extra_variables={ + 'foo': 'bar', + 'bar_id': '{{ 40|add:2 }}', + 'syntax_error': '{% for %}', + 'subslug_dependant': '{{ 40|add:card_id }}', + }, + ) + cell = ChartNgCell(page=page, order=1, placeholder='content') + cell.statistic = Statistic.objects.get(slug='one-serie') + cell.filter_params = {'service': 'chrono', 'ou': 'variable:foo'} + cell.save() + + location = '/api/dataviz/graph/%s/' % cell.pk + app.get(location) + request = new_api_mock.call['requests'][0] + assert 'service=chrono' in request.url + assert 'ou=bar' in request.url + + cell.filter_params = {'service': 'chrono', 'ou': 'variable:bar_id'} + cell.save() + + app.get(location) + request = new_api_mock.call['requests'][1] + assert 'service=chrono' in request.url + assert 'ou=42' in request.url + + # unknown variable + cell.filter_params = {'service': 'chrono', 'ou': 'variable:unknown'} + cell.save() + + resp = app.get(location) + assert len(new_api_mock.call['requests']) == 2 + assert 'Page variable not found.' in resp.text + + # variable with invalid syntax + cell.filter_params = {'service': 'chrono', 'ou': 'variable:syntax_error'} + cell.save() + + resp = app.get(location) + assert len(new_api_mock.call['requests']) == 2 + assert 'Syntax error in page variable.' in resp.text + + # variable with missing context + cell.filter_params = {'service': 'chrono', 'ou': 'variable:subslug_dependant'} + cell.save() + + resp = app.get(location) + assert len(new_api_mock.call['requests']) == 2 + assert 'Cannot evaluate page variable.' in resp.text + + # simulate call from page view + app = login(app) + resp = app.get('/cards/2/') + ctx = resp.pyquery('.chartngcell').attr('data-extra-context') + + app.get(location + '?ctx=%s' % ctx) + request = new_api_mock.call['requests'][2] + assert 'service=chrono' in request.url + assert 'ou=42' in request.url + + # reste à tester missing variable + # et avec display table + + +@with_httmock(new_api_mock) +def test_chartng_cell_new_api_filter_params_page_variables_table(new_api_statistics, nocache): + Page.objects.create(title='One', slug='index') + page = Page.objects.create( + title='One', + slug='cards', + sub_slug='card_id', + extra_variables={ + 'foo': 'bar', + 'syntax_error': '{% for %}', + 'subslug_dependant': '{{ 40|add:card_id }}', + }, + ) + cell = ChartNgCell(page=page, order=1, placeholder='content', chart_type='table') + cell.statistic = Statistic.objects.get(slug='one-serie') + cell.filter_params = {'service': 'chrono', 'ou': 'variable:foo'} + cell.save() + + cell.render({**page.extra_variables, 'synchronous': True}) + request = new_api_mock.call['requests'][0] + assert 'service=chrono' in request.url + assert 'ou=bar' in request.url + + # unknown variable + cell.filter_params = {'service': 'chrono', 'ou': 'variable:unknown'} + cell.save() + + content = cell.render({'synchronous': True}) + assert len(new_api_mock.call['requests']) == 1 + assert 'Page variable not found.' in content + + # variable with invalid syntax + cell.filter_params = {'service': 'chrono', 'ou': 'variable:syntax_error'} + cell.save() + + content = cell.render({**page.extra_variables, 'synchronous': True}) + assert len(new_api_mock.call['requests']) == 1 + assert 'Syntax error in page variable.' in content + + # variable with missing context + cell.filter_params = {'service': 'chrono', 'ou': 'variable:subslug_dependant'} + cell.save() + + content = cell.render({**page.extra_variables, 'synchronous': True}) + assert len(new_api_mock.call['requests']) == 1 + assert 'Cannot evaluate page variable.' in content + + def test_dataviz_check_validity(nocache): page = Page.objects.create(title='One', slug='index') stat = Statistic.objects.create(url='https://stat.com/stats/1/') @@ -1962,6 +2143,14 @@ def test_spooler_refresh_statistics_data(new_api_statistics): refresh_statistics_data(cell.pk) assert len(new_api_mock.call['requests']) == 2 + # variables cannot be evaluated in spooler + page.extra_variables = {'test': 'test'} + page.save() + cell.filter_params = {'ou': 'variable:test'} + cell.save() + refresh_statistics_data(cell.pk) + assert len(new_api_mock.call['requests']) == 2 + ChartNgCell.objects.all().delete() refresh_statistics_data(cell.pk) assert len(new_api_mock.call['requests']) == 2 -- 2.30.2