0001-dataviz-aggregate-received-data-by-time-intervals-53.patch

Valentin Deniaud, 20 avril 2021 12:34

Voir les différences: en ligne côte à côte

Subject: [PATCH] dataviz: aggregate received data by time intervals (#53180)

 combo/apps/dataviz/forms.py  | 17 +++++++
 combo/apps/dataviz/models.py | 45 ++++++++++++++++-
 tests/test_dataviz.py        | 94 ++++++++++++++++++++++++++++++++++++
 3 files changed, 155 insertions(+), 1 deletion(-)

     from django.conf import settings
     from django.db import transaction
     from django.db.models import Q
     from django.utils.translation import ugettext_lazy as _
     from combo.utils import cache_during_request, requests, spooler
-...
     class ChartNgForm(forms.ModelForm):
         blank_choice = ('', '---------')
         time_intervals = (
             ('_month', _('Month')),
             ('_year', _('Year')),
             ('_weekday', _('Week day')),
+        )
         class Meta:
             model = ChartNgCell
-...
             self.fields = OrderedDict((field_id, self.fields[field_id]) for field_id in field_ids)
             if 'time_interval' in self.fields:
                 self.extend_time_interval_choices()
         def save(self, *args, **kwargs):
             if 'statistic' in self.changed_data:
                 self.instance.filter_params.clear()
-...
                     else:
                         self.instance.filter_params.pop(field, None)
             return super().save(*args, **kwargs)
         def extend_time_interval_choices(self):
             choice_ids = {choice_id for choice_id, _ in self.fields['time_interval'].choices}
             if 'day' not in choice_ids:
                 return
             for choice in self.time_intervals:
                 if choice[0].strip('_') not in choice_ids:
                     self.fields['time_interval'].choices.append(choice)

     import copy
     import os
     import sys
     from datetime import date
     from collections import OrderedDict
     from datetime import date, datetime, timedelta
     import pygal
     import pygal.util
     from dateutil.relativedelta import relativedelta
     from django.conf import settings
     from django.db import models, transaction
     from django.template.defaultfilters import date as format_date
     from django.urls import reverse
     from django.utils import timezone
     from django.utils.dates import WEEKDAYS
     from django.utils.encoding import force_text
     from django.utils.translation import gettext
     from django.utils.translation import ugettext_lazy as _
-...
                 self.add_data_to_chart(chart, data, y_labels)
             else:
                 data = response['data']
                 interval = self.filter_params.get('time_interval', '')
                 if interval == 'day' or interval.startswith('_'):
                     self.aggregate_data(data, interval)
                 chart.x_labels = data['x_labels']
                 chart.axis_count = min(len(data['series']), 2)
                 self.prepare_chart(chart, width, height)
-...
             return chart
         @staticmethod
         def aggregate_data(data, interval):
             series_data = [serie['data'] for serie in data['series']]
             dates = [datetime.strptime(label, '%Y-%m-%d') for label in data['x_labels']]
             min_date, max_date = min(dates), max(dates)
             if interval == 'day':
                 x_labels = [
                     (min_date + timedelta(days=i)).strftime('%d-%m-%Y')
                     for i in range((max_date - min_date).days + 1)
+                ]
             elif interval == '_month':
                 month_difference = max_date.month - min_date.month + (max_date.year - min_date.year) * 12
                 x_labels = [
                     (min_date + relativedelta(months=i)).strftime('%m-%Y') for i in range(month_difference + 1)
+                ]
             elif interval == '_year':
                 x_labels = [str(year) for year in range(min_date.year, max_date.year + 1)]
             elif interval == '_weekday':
                 x_labels = [str(label) for label in WEEKDAYS.values()]
             aggregates = OrderedDict((label, [0] * len(series_data)) for label in x_labels)
             date_formats = {'day': 'd-m-Y', '_month': 'm-Y', '_year': 'Y', '_weekday': 'l'}
             for i, date in enumerate(dates):
                 key = format_date(date, date_formats[interval])
                 for j in range(len(series_data)):
                     aggregates[key][j] += series_data[j][i] or 0
             data['x_labels'] = x_labels
             for i, serie in enumerate(data['series']):
                 serie['data'] = [values[i] for values in aggregates.values()]
         def get_filter_params(self):
             params = self.filter_params.copy()
             now = timezone.now().date()
-...
                     params['start'] = self.time_range_start
                 if self.time_range_end:
                     params['end'] = self.time_range_end
             if 'time_interval' in params and params['time_interval'].startswith('_'):
                 params['time_interval'] = 'day'
             return params
         def parse_response(self, response, chart):

                 'name': '404 not found stat',
                 'id': 'not-found',
             },
+            {
                 'url': 'https://authentic.example.com/api/statistics/daily/',
                 'name': 'daily discontinuous serie',
                 'id': 'daily',
                 "filters": [
+                    {
                         "default": "day",
                         "id": "time_interval",
                         "label": "Time interval",
                         "options": [
                             {"id": "day", "label": "Day"},
                         ],
                         "required": True,
+                    }
                 ],
             },
+        ]
+    }
-...
             return {'content': json.dumps(response), 'request': request, 'status_code': 200}
         if url.path == '/api/statistics/not-found/':
             return {'content': b'', 'request': request, 'status_code': 404}
         if url.path == '/api/statistics/daily/':
             response = {
                 'data': {
                     'series': [
                         {'data': [None, 1, 16, 2], 'label': 'Serie 1'},
                         {'data': [2, 2, 1, None], 'label': 'Serie 2'},
                     ],
                     'x_labels': ['2020-10-06', '2020-10-13', '2020-11-30', '2022-02-01'],
                 },
+            }
             return {'content': json.dumps(response), 'request': request, 'status_code': 200}
     @pytest.fixture
-...
             ('day', False, 'Day'),
             ('month', True, 'Month'),
             ('year', False, 'Year'),
             ('_weekday', False, 'Week day'),
+        ]
         ou_field = resp.form[field_prefix + 'ou']
-...
             cell.check_validity()
         validity_info = ValidityInfo.objects.latest('pk')
         assert validity_info.invalid_reason_code == 'statistic_data_not_found'
     @with_httmock(new_api_mock)
     def test_chartng_cell_new_api_aggregation(new_api_statistics, app, admin_user, nocache):
         page = Page.objects.create(title='One', slug='index')
         cell = ChartNgCell(page=page, order=1, placeholder='content')
         cell.statistic = Statistic.objects.get(slug='daily')
         cell.save()
         app = login(app)
         resp = app.get('/manage/pages/%s/' % page.id)
         time_interval_field = resp.form['cdataviz_chartngcell-%s-time_interval' % cell.id]
         assert time_interval_field.value == 'day'
         assert time_interval_field.options == [
             ('day', True, 'Day'),
             ('_month', False, 'Month'),
             ('_year', False, 'Year'),
             ('_weekday', False, 'Week day'),
+        ]
         resp.form.submit()
         cell.refresh_from_db()
         chart = cell.get_chart()
         assert len(chart.x_labels) == 484
         assert chart.x_labels[:3] == ['06-10-2020', '07-10-2020', '08-10-2020']
         assert chart.x_labels[-3:] == ['30-01-2022', '31-01-2022', '01-02-2022']
         assert chart.raw_series[0][0][:8] == [0, 0, 0, 0, 0, 0, 0, 1]
         assert chart.raw_series[1][0][:8] == [2, 0, 0, 0, 0, 0, 0, 2]
         time_interval_field.value = '_month'
         resp.form.submit()
         cell.refresh_from_db()
         chart = cell.get_chart()
         assert 'time_interval=day' in new_api_mock.call['requests'][1].url
         assert len(chart.x_labels) == 17
         assert chart.x_labels[:3] == ['10-2020', '11-2020', '12-2020']
         assert chart.x_labels[-3:] == ['12-2021', '01-2022', '02-2022']
         assert chart.raw_series == [
             ([1, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], {'title': 'Serie 1'}),
             ([4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], {'title': 'Serie 2'}),
+        ]
         time_interval_field.value = '_year'
         resp.form.submit()
         cell.refresh_from_db()
         chart = cell.get_chart()
         assert 'time_interval=day' in new_api_mock.call['requests'][2].url
         assert chart.x_labels == ['2020', '2021', '2022']
         assert chart.raw_series == [
             ([17, 0, 2], {'title': 'Serie 1'}),
             ([5, 0, 0], {'title': 'Serie 2'}),
+        ]
         time_interval_field.value = '_weekday'
         resp.form.submit()
         cell.refresh_from_db()
         chart = cell.get_chart()
         assert 'time_interval=day' in new_api_mock.call['requests'][3].url
         assert chart.x_labels == ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday']
         assert chart.raw_series == [
             ([16, 3, 0, 0, 0, 0, 0], {'title': 'Serie 1'}),
             ([1, 4, 0, 0, 0, 0, 0], {'title': 'Serie 2'}),
+        ]
+    -

Projet

Général

Profil

Produits Entr'ouvert » Combo

0001-dataviz-aggregate-received-data-by-time-intervals-53.patch