From 5033cc320db5497ea3e07adb7448028e77e6c616 Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Mon, 19 Apr 2021 17:25:45 +0200 Subject: [PATCH] dataviz: aggregate received data by time intervals (#53180) --- combo/apps/dataviz/forms.py | 17 +++++++ combo/apps/dataviz/models.py | 45 ++++++++++++++++- tests/test_dataviz.py | 94 ++++++++++++++++++++++++++++++++++++ 3 files changed, 155 insertions(+), 1 deletion(-) diff --git a/combo/apps/dataviz/forms.py b/combo/apps/dataviz/forms.py index 18761856..fdf71c62 100644 --- a/combo/apps/dataviz/forms.py +++ b/combo/apps/dataviz/forms.py @@ -21,6 +21,7 @@ from django import forms from django.conf import settings from django.db import transaction from django.db.models import Q +from django.utils.translation import ugettext_lazy as _ from combo.utils import cache_during_request, requests, spooler @@ -54,6 +55,11 @@ def trigger_statistics_list_refresh(): class ChartNgForm(forms.ModelForm): blank_choice = ('', '---------') + time_intervals = ( + ('_month', _('Month')), + ('_year', _('Year')), + ('_weekday', _('Week day')), + ) class Meta: model = ChartNgCell @@ -109,6 +115,9 @@ class ChartNgForm(forms.ModelForm): self.fields = OrderedDict((field_id, self.fields[field_id]) for field_id in field_ids) + if 'time_interval' in self.fields: + self.extend_time_interval_choices() + def save(self, *args, **kwargs): if 'statistic' in self.changed_data: self.instance.filter_params.clear() @@ -122,3 +131,11 @@ class ChartNgForm(forms.ModelForm): else: self.instance.filter_params.pop(field, None) return super().save(*args, **kwargs) + + def extend_time_interval_choices(self): + choice_ids = {choice_id for choice_id, _ in self.fields['time_interval'].choices} + if 'day' not in choice_ids: + return + for choice in self.time_intervals: + if choice[0].strip('_') not in choice_ids: + self.fields['time_interval'].choices.append(choice) diff --git a/combo/apps/dataviz/models.py b/combo/apps/dataviz/models.py index 13524584..abfd51f8 100644 --- a/combo/apps/dataviz/models.py +++ b/combo/apps/dataviz/models.py @@ -17,14 +17,18 @@ import copy import os import sys -from datetime import date +from collections import OrderedDict +from datetime import date, datetime, timedelta import pygal import pygal.util +from dateutil.relativedelta import relativedelta from django.conf import settings from django.db import models, transaction +from django.template.defaultfilters import date as format_date from django.urls import reverse from django.utils import timezone +from django.utils.dates import WEEKDAYS from django.utils.encoding import force_text from django.utils.translation import gettext from django.utils.translation import ugettext_lazy as _ @@ -324,6 +328,11 @@ class ChartNgCell(CellBase): self.add_data_to_chart(chart, data, y_labels) else: data = response['data'] + + interval = self.filter_params.get('time_interval', '') + if interval == 'day' or interval.startswith('_'): + self.aggregate_data(data, interval) + chart.x_labels = data['x_labels'] chart.axis_count = min(len(data['series']), 2) self.prepare_chart(chart, width, height) @@ -347,6 +356,38 @@ class ChartNgCell(CellBase): return chart + @staticmethod + def aggregate_data(data, interval): + series_data = [serie['data'] for serie in data['series']] + dates = [datetime.strptime(label, '%Y-%m-%d') for label in data['x_labels']] + min_date, max_date = min(dates), max(dates) + + if interval == 'day': + x_labels = [ + (min_date + timedelta(days=i)).strftime('%d-%m-%Y') + for i in range((max_date - min_date).days + 1) + ] + elif interval == '_month': + month_difference = max_date.month - min_date.month + (max_date.year - min_date.year) * 12 + x_labels = [ + (min_date + relativedelta(months=i)).strftime('%m-%Y') for i in range(month_difference + 1) + ] + elif interval == '_year': + x_labels = [str(year) for year in range(min_date.year, max_date.year + 1)] + elif interval == '_weekday': + x_labels = [str(label) for label in WEEKDAYS.values()] + + aggregates = OrderedDict((label, [0] * len(series_data)) for label in x_labels) + date_formats = {'day': 'd-m-Y', '_month': 'm-Y', '_year': 'Y', '_weekday': 'l'} + for i, date in enumerate(dates): + key = format_date(date, date_formats[interval]) + for j in range(len(series_data)): + aggregates[key][j] += series_data[j][i] or 0 + + data['x_labels'] = x_labels + for i, serie in enumerate(data['series']): + serie['data'] = [values[i] for values in aggregates.values()] + def get_filter_params(self): params = self.filter_params.copy() now = timezone.now().date() @@ -365,6 +406,8 @@ class ChartNgCell(CellBase): params['start'] = self.time_range_start if self.time_range_end: params['end'] = self.time_range_end + if 'time_interval' in params and params['time_interval'].startswith('_'): + params['time_interval'] = 'day' return params def parse_response(self, response, chart): diff --git a/tests/test_dataviz.py b/tests/test_dataviz.py index 9b5ffab1..16d0d3df 100644 --- a/tests/test_dataviz.py +++ b/tests/test_dataviz.py @@ -318,6 +318,22 @@ STATISTICS_LIST = { 'name': '404 not found stat', 'id': 'not-found', }, + { + 'url': 'https://authentic.example.com/api/statistics/daily/', + 'name': 'daily discontinuous serie', + 'id': 'daily', + "filters": [ + { + "default": "day", + "id": "time_interval", + "label": "Time interval", + "options": [ + {"id": "day", "label": "Day"}, + ], + "required": True, + } + ], + }, ] } @@ -352,6 +368,17 @@ def new_api_mock(url, request): return {'content': json.dumps(response), 'request': request, 'status_code': 200} if url.path == '/api/statistics/not-found/': return {'content': b'', 'request': request, 'status_code': 404} + if url.path == '/api/statistics/daily/': + response = { + 'data': { + 'series': [ + {'data': [None, 1, 16, 2], 'label': 'Serie 1'}, + {'data': [2, 2, 1, None], 'label': 'Serie 2'}, + ], + 'x_labels': ['2020-10-06', '2020-10-13', '2020-11-30', '2022-02-01'], + }, + } + return {'content': json.dumps(response), 'request': request, 'status_code': 200} @pytest.fixture @@ -1057,6 +1084,7 @@ def test_chartng_cell_manager_new_api(app, admin_user, new_api_statistics): ('day', False, 'Day'), ('month', True, 'Month'), ('year', False, 'Year'), + ('_weekday', False, 'Week day'), ] ou_field = resp.form[field_prefix + 'ou'] @@ -1350,3 +1378,69 @@ def test_dataviz_check_validity(nocache): cell.check_validity() validity_info = ValidityInfo.objects.latest('pk') assert validity_info.invalid_reason_code == 'statistic_data_not_found' + + +@with_httmock(new_api_mock) +def test_chartng_cell_new_api_aggregation(new_api_statistics, app, admin_user, nocache): + page = Page.objects.create(title='One', slug='index') + cell = ChartNgCell(page=page, order=1, placeholder='content') + cell.statistic = Statistic.objects.get(slug='daily') + cell.save() + + app = login(app) + resp = app.get('/manage/pages/%s/' % page.id) + time_interval_field = resp.form['cdataviz_chartngcell-%s-time_interval' % cell.id] + assert time_interval_field.value == 'day' + assert time_interval_field.options == [ + ('day', True, 'Day'), + ('_month', False, 'Month'), + ('_year', False, 'Year'), + ('_weekday', False, 'Week day'), + ] + resp.form.submit() + cell.refresh_from_db() + + chart = cell.get_chart() + assert len(chart.x_labels) == 484 + assert chart.x_labels[:3] == ['06-10-2020', '07-10-2020', '08-10-2020'] + assert chart.x_labels[-3:] == ['30-01-2022', '31-01-2022', '01-02-2022'] + assert chart.raw_series[0][0][:8] == [0, 0, 0, 0, 0, 0, 0, 1] + assert chart.raw_series[1][0][:8] == [2, 0, 0, 0, 0, 0, 0, 2] + + time_interval_field.value = '_month' + resp.form.submit() + cell.refresh_from_db() + + chart = cell.get_chart() + assert 'time_interval=day' in new_api_mock.call['requests'][1].url + assert len(chart.x_labels) == 17 + assert chart.x_labels[:3] == ['10-2020', '11-2020', '12-2020'] + assert chart.x_labels[-3:] == ['12-2021', '01-2022', '02-2022'] + assert chart.raw_series == [ + ([1, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], {'title': 'Serie 1'}), + ([4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], {'title': 'Serie 2'}), + ] + + time_interval_field.value = '_year' + resp.form.submit() + cell.refresh_from_db() + + chart = cell.get_chart() + assert 'time_interval=day' in new_api_mock.call['requests'][2].url + assert chart.x_labels == ['2020', '2021', '2022'] + assert chart.raw_series == [ + ([17, 0, 2], {'title': 'Serie 1'}), + ([5, 0, 0], {'title': 'Serie 2'}), + ] + + time_interval_field.value = '_weekday' + resp.form.submit() + cell.refresh_from_db() + + chart = cell.get_chart() + assert 'time_interval=day' in new_api_mock.call['requests'][3].url + assert chart.x_labels == ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday'] + assert chart.raw_series == [ + ([16, 3, 0, 0, 0, 0, 0], {'title': 'Serie 1'}), + ([1, 4, 0, 0, 0, 0, 0], {'title': 'Serie 2'}), + ] -- 2.20.1