From 3d9243534232308c7768822c5cac9862341fd2ee Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Mon, 30 Nov 2020 14:10:39 +0100 Subject: [PATCH 4/4] dataviz: handle new api to get statistics from elsewhere (#48865) --- combo/apps/dataviz/__init__.py | 42 +++++---- combo/apps/dataviz/forms.py | 5 +- combo/apps/dataviz/models.py | 152 +++++++++++++++++++++------------ combo/settings.py | 3 + 4 files changed, 130 insertions(+), 72 deletions(-) diff --git a/combo/apps/dataviz/__init__.py b/combo/apps/dataviz/__init__.py index c6b7e5e2..123d1acb 100644 --- a/combo/apps/dataviz/__init__.py +++ b/combo/apps/dataviz/__init__.py @@ -41,24 +41,32 @@ class AppConfig(django.apps.AppConfig): if not settings.KNOWN_SERVICES: return + statistics_providers = settings.STATISTICS_PROVIDERS + ['bijoe'] start_update = timezone.now() - bijoe_sites = settings.KNOWN_SERVICES.get('bijoe').items() - for site_key, site_dict in bijoe_sites: - result = requests.get('/visualization/json/', - remote_service=site_dict, without_user=True, - headers={'accept': 'application/json'}).json() - for stat in result: - Statistic.objects.update_or_create( - slug=stat['slug'], - site_slug=site_key, - service_slug='bijoe', - defaults={ - 'label': stat['name'], - 'url': stat['data-url'], - 'site_title': site_dict.get('title', ''), - 'available': True, - } - ) + for service in statistics_providers: + sites = settings.KNOWN_SERVICES.get(service, {}).items() + for site_key, site_dict in sites: + if service == 'bijoe': + result = requests.get('/visualization/json/', + remote_service=site_dict, without_user=True, + headers={'accept': 'application/json'}).json() + else: + result = requests.get('/api/statistics/', + remote_service=site_dict, without_user=True, + headers={'accept': 'application/json'}).json()['data'] + + for stat in result: + Statistic.objects.update_or_create( + slug=stat.get('slug') or stat['id'], + site_slug=site_key, + service_slug=service, + defaults={ + 'label': stat['name'], + 'url': stat.get('data-url') or stat['url'], + 'site_title': site_dict.get('title', ''), + 'available': True, + } + ) Statistic.objects.filter(last_update__lt=start_update).update(available=False) diff --git a/combo/apps/dataviz/forms.py b/combo/apps/dataviz/forms.py index 117f366f..dab978b1 100644 --- a/combo/apps/dataviz/forms.py +++ b/combo/apps/dataviz/forms.py @@ -44,7 +44,7 @@ class ChartForm(forms.ModelForm): class ChartNgForm(forms.ModelForm): class Meta: model = ChartNgCell - fields = ('title', 'statistic', 'chart_type', 'height', 'sort_order', + fields = ('title', 'statistic', 'time_interval', 'chart_type', 'height', 'sort_order', 'hide_null_values') def __init__(self, *args, **kwargs): @@ -53,3 +53,6 @@ class ChartNgForm(forms.ModelForm): if self.instance.statistic: q_filters |= Q(pk=self.instance.statistic.pk) self.fields['statistic'].queryset = self.fields['statistic'].queryset.filter(q_filters) + + if self.instance.statistic and self.instance.statistic.service_slug == 'bijoe': + self.fields.pop('time_interval') diff --git a/combo/apps/dataviz/models.py b/combo/apps/dataviz/models.py index 8629de38..a5be1934 100644 --- a/combo/apps/dataviz/models.py +++ b/combo/apps/dataviz/models.py @@ -136,9 +136,21 @@ class Statistic(models.Model): @register_cell_class class ChartNgCell(CellBase): + DAY = 'day' + MONTH = 'month' + YEAR = 'year' + TIME_INTERVAL_CHOICES = { + (DAY, _('Day')), + (MONTH, _('Month')), + (YEAR, _('Year')), + } + statistic = models.ForeignKey( verbose_name=_('Data'), to=Statistic, blank=False, null=True, on_delete=models.SET_NULL, related_name='cells' ) + time_interval = models.CharField( + _('Time interval (if applicable)'), max_length=16, choices=TIME_INTERVAL_CHOICES, default=MONTH + ) title = models.CharField(_('Title'), max_length=150, blank=True) chart_type = models.CharField(_('Chart Type'), max_length=20, default='bar', choices=( @@ -177,7 +189,7 @@ class ChartNgCell(CellBase): @classmethod def is_enabled(self): - return hasattr(settings, 'KNOWN_SERVICES') and settings.KNOWN_SERVICES.get('bijoe') + return settings.KNOWN_SERVICES.get('bijoe') or settings.STATISTICS_PROVIDERS def get_default_form_class(self): from .forms import ChartNgForm @@ -202,15 +214,19 @@ class ChartNgCell(CellBase): else: ctx['table'] = chart.render_table( transpose=bool(chart.axis_count == 2), - total=chart.compute_sum, + total=getattr(chart, 'compute_sum', True), ) ctx['table'] = ctx['table'].replace('', '
') return ctx def get_chart(self, width=None, height=None, raise_if_not_cached=False): + params = {'time_interval': self.time_interval} response = requests.get( self.statistic.url, + params=params, cache_duration=300, + remote_service='auto', + without_user=True, raise_if_not_cached=raise_if_not_cached) response.raise_for_status() response = response.json() @@ -229,30 +245,86 @@ class ChartNgCell(CellBase): 'table': pygal.Bar, }[self.chart_type](config=pygal.Config(style=copy.copy(style))) + chart.config.margin = 0 + if width: + chart.config.width = width + if height: + chart.config.height = height + if width or height: + chart.config.explicit_size = True + chart.config.js = [os.path.join(settings.STATIC_URL, 'js/pygal-tooltips.js')] + + chart.truncate_legend = 30 + # matplotlib tab10 palette + chart.config.style.colors = ( + '#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', + '#9467bd', '#8c564b', '#e377c2', '#7f7f7f', + '#bcbd22', '#17becf') + + if self.statistic.service_slug == 'bijoe': + x_labels = response['axis'].get('x_labels') or [] + chart.show_legend = bool(len(response['axis']) > 1) + else: + x_labels = response['data']['x_labels'] + chart.show_legend = bool(len(response['data']['series']) > 1) + + chart.x_labels = x_labels + + if self.chart_type == 'dot': + chart.show_legend = False + # use a single colour for dots + chart.config.style.colors = ('#1f77b4',) * len(x_labels) + + if self.chart_type != 'pie': + if chart.config.width and chart.config.width < 500: + chart.legend_at_bottom = True + if self.chart_type == 'horizontal-bar': + # truncate labels + chart.x_labels = [pygal.util.truncate(x, 15) for x in chart.x_labels] + else: + chart.show_legend = True + if chart.config.width and chart.config.width < 500: + chart.truncate_legend = 15 + + if self.statistic.service_slug == 'bijoe': + return self.add_bijoe_data_to_chart(chart, response) + else: + return self.add_data_to_chart(chart, response['data']) + + def add_data_to_chart(self, chart, data): + chart.axis_count = 1 if len(data['series']) == 1 else 2 + if self.chart_type == 'table' and chart.axis_count == 1: + self.add_total_to_line_table(chart, data['series'][0]['data']) + + for serie in data['series']: + chart.add(serie['label'], serie['data']) + + return chart + + def add_bijoe_data_to_chart(self, chart, response): # normalize axis to have a fake axis when there are no dimensions and # always a x axis when there is a single dimension. data = response['data'] loop_labels = response['axis'].get('loop') or [] - x_labels = response['axis'].get('x_labels') or [] y_labels = response['axis'].get('y_labels') or [] if loop_labels: - if x_labels and y_labels: + if chart.x_labels and y_labels: # no support for three dimensions raise UnsupportedDataSet() if not y_labels: y_labels = loop_labels else: - x_labels, y_labels = y_labels, loop_labels - if len(y_labels) != len(data) or not all([len(x) == len(x_labels) for x in data]): + chart.x_labels, y_labels = y_labels, loop_labels + if len(y_labels) != len(data) or not all([len(x) == len(chart.x_labels) for x in data]): # varying dimensions raise UnsupportedDataSet() - if not x_labels and not y_labels: # unidata - x_labels = [''] + if not chart.x_labels and not y_labels: # unidata + chart.x_labels = [''] y_labels = [''] data = [data] chart.axis_count = 0 - elif not x_labels: - x_labels = y_labels + elif not chart.x_labels: + chart.x_labels = y_labels y_labels = [''] chart.axis_count = 1 elif not y_labels: @@ -264,13 +336,13 @@ class ChartNgCell(CellBase): # hide/sort values if chart.axis_count == 1 and (self.sort_order != 'none' or self.hide_null_values): if self.sort_order == 'alpha': - tmp_items = sorted(zip(x_labels, data), key=lambda x: x[0]) + tmp_items = sorted(zip(chart.x_labels, data), key=lambda x: x[0]) elif self.sort_order == 'asc': - tmp_items = sorted(zip(x_labels, data), key=lambda x: (x[1] or 0)) + tmp_items = sorted(zip(chart.x_labels, data), key=lambda x: (x[1] or 0)) elif self.sort_order == 'desc': - tmp_items = sorted(zip(x_labels, data), key=lambda x: (x[1] or 0), reverse=True) + tmp_items = sorted(zip(chart.x_labels, data), key=lambda x: (x[1] or 0), reverse=True) else: - tmp_items = zip(x_labels, data) + tmp_items = zip(chart.x_labels, data) tmp_x_labels = [] tmp_data = [] for label, value in tmp_items: @@ -278,62 +350,27 @@ class ChartNgCell(CellBase): continue tmp_x_labels.append(label) tmp_data.append(value) - x_labels = tmp_x_labels + chart.x_labels = tmp_x_labels data = tmp_data - chart.config.margin = 0 - if width: - chart.config.width = width - if height: - chart.config.height = height - if width or height: - chart.config.explicit_size = True - chart.config.js = [os.path.join(settings.STATIC_URL, 'js/pygal-tooltips.js')] - chart.x_labels = x_labels - - chart.show_legend = bool(len(response['axis']) > 1) - chart.truncate_legend = 30 - # matplotlib tab10 palette - chart.config.style.colors = ( - '#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', - '#9467bd', '#8c564b', '#e377c2', '#7f7f7f', - '#bcbd22', '#17becf') - - if self.chart_type == 'dot': - chart.show_legend = False - # use a single colour for dots - chart.config.style.colors = ('#1f77b4',) * len(x_labels) - chart.compute_sum = bool(response.get('measure') == 'integer') - if chart.compute_sum and self.chart_type == 'table': - if chart.axis_count < 2: # workaround pygal - chart.compute_sum = False - if chart.axis_count == 1: - data.append(sum(data)) - x_labels.append(gettext('Total')) + if chart.compute_sum and self.chart_type == 'table' and chart.axis_count < 2: # workaround pygal + self.add_total_to_line_table(chart, data) if self.chart_type != 'pie': for i, serie_label in enumerate(y_labels): if chart.axis_count < 2: values = data else: - values = [data[i][j] for j in range(len(x_labels))] + values = [data[i][j] for j in range(len(chart.x_labels))] chart.add(serie_label, values) - if width and width < 500: - chart.legend_at_bottom = True - if self.chart_type == 'horizontal-bar': - # truncate labels - chart.x_labels = [pygal.util.truncate(x, 15) for x in chart.x_labels] else: # pie, create a serie by data, to get different colours values = data - for label, value in zip(x_labels, values): + for label, value in zip(chart.x_labels, values): if not value: continue chart.add(label, value) - chart.show_legend = True - if width and width < 500: - chart.truncate_legend = 15 if response.get('unit') == 'seconds' or response.get('measure') == 'duration': def format_duration(value): @@ -363,3 +400,10 @@ class ChartNgCell(CellBase): chart.config.value_formatter = percent_formatter return chart + + @staticmethod + def add_total_to_line_table(chart, data): + chart.compute_sum = False + if chart.axis_count == 1: + data.append(sum(data)) + chart.x_labels.append(gettext('Total')) diff --git a/combo/settings.py b/combo/settings.py index beab8c32..214e6f7a 100644 --- a/combo/settings.py +++ b/combo/settings.py @@ -348,6 +348,9 @@ COMBO_CELL_ASSET_SLOTS = { # known services KNOWN_SERVICES = {} +# known services exposing statistics +STATISTICS_PROVIDERS = [] + # PWA Settings PWA_VAPID_PUBLIK_KEY = None PWA_VAPID_PRIVATE_KEY = None -- 2.20.1