From 55cf841696f5b977b6c3857a9618a085d89748a8 Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Mon, 30 Nov 2020 14:10:39 +0100 Subject: [PATCH 2/3] dataviz: handle new api to get statistics from elsewhere (#48865) --- combo/apps/dataviz/__init__.py | 42 ++++--- combo/apps/dataviz/models.py | 56 ++++++--- combo/settings.py | 3 + tests/test_dataviz.py | 221 +++++++++++++++++++++++++++++++++ 4 files changed, 290 insertions(+), 32 deletions(-) diff --git a/combo/apps/dataviz/__init__.py b/combo/apps/dataviz/__init__.py index c6b7e5e2..123d1acb 100644 --- a/combo/apps/dataviz/__init__.py +++ b/combo/apps/dataviz/__init__.py @@ -41,24 +41,32 @@ class AppConfig(django.apps.AppConfig): if not settings.KNOWN_SERVICES: return + statistics_providers = settings.STATISTICS_PROVIDERS + ['bijoe'] start_update = timezone.now() - bijoe_sites = settings.KNOWN_SERVICES.get('bijoe').items() - for site_key, site_dict in bijoe_sites: - result = requests.get('/visualization/json/', - remote_service=site_dict, without_user=True, - headers={'accept': 'application/json'}).json() - for stat in result: - Statistic.objects.update_or_create( - slug=stat['slug'], - site_slug=site_key, - service_slug='bijoe', - defaults={ - 'label': stat['name'], - 'url': stat['data-url'], - 'site_title': site_dict.get('title', ''), - 'available': True, - } - ) + for service in statistics_providers: + sites = settings.KNOWN_SERVICES.get(service, {}).items() + for site_key, site_dict in sites: + if service == 'bijoe': + result = requests.get('/visualization/json/', + remote_service=site_dict, without_user=True, + headers={'accept': 'application/json'}).json() + else: + result = requests.get('/api/statistics/', + remote_service=site_dict, without_user=True, + headers={'accept': 'application/json'}).json()['data'] + + for stat in result: + Statistic.objects.update_or_create( + slug=stat.get('slug') or stat['id'], + site_slug=site_key, + service_slug=service, + defaults={ + 'label': stat['name'], + 'url': stat.get('data-url') or stat['url'], + 'site_title': site_dict.get('title', ''), + 'available': True, + } + ) Statistic.objects.filter(last_update__lt=start_update).update(available=False) diff --git a/combo/apps/dataviz/models.py b/combo/apps/dataviz/models.py index 7da16722..afab1779 100644 --- a/combo/apps/dataviz/models.py +++ b/combo/apps/dataviz/models.py @@ -177,7 +177,7 @@ class ChartNgCell(CellBase): @classmethod def is_enabled(self): - return hasattr(settings, 'KNOWN_SERVICES') and settings.KNOWN_SERVICES.get('bijoe') + return settings.KNOWN_SERVICES.get('bijoe') or settings.STATISTICS_PROVIDERS def get_default_form_class(self): from .forms import ChartNgForm @@ -202,7 +202,7 @@ class ChartNgCell(CellBase): else: ctx['table'] = chart.render_table( transpose=bool(chart.axis_count == 2), - total=chart.compute_sum, + total=getattr(chart, 'compute_sum', True), ) ctx['table'] = ctx['table'].replace('', '
') return ctx @@ -211,6 +211,8 @@ class ChartNgCell(CellBase): response = requests.get( self.statistic.url, cache_duration=300, + remote_service='auto', + without_user=True, raise_if_not_cached=raise_if_not_cached) response.raise_for_status() response = response.json() @@ -229,18 +231,33 @@ class ChartNgCell(CellBase): 'table': pygal.Bar, }[self.chart_type](config=pygal.Config(style=copy.copy(style))) - x_labels, y_labels, data = self.parse_response(response, chart) - chart.x_labels = x_labels - self.prepare_chart(chart, width, height) + if self.statistic.service_slug == 'bijoe': + x_labels, y_labels, data = self.parse_response(response, chart) + chart.x_labels = x_labels + self.prepare_chart(chart, width, height) - if chart.axis_count == 1: - if self.hide_null_values: - data = self.hide_values(chart, data) - if self.sort_order != 'none': - data = self.sort_values(chart, data) - if chart.compute_sum and self.chart_type == 'table': - data = self.add_total_to_line_table(chart, data) - self.add_data_to_chart(chart, data, y_labels) + if chart.axis_count == 1: + data = self.process_one_dimensional_data(chart, data) + self.add_data_to_chart(chart, data, y_labels) + else: + data = response['data'] + chart.x_labels = data['x_labels'] + chart.axis_count = min(len(data['series']), 2) + self.prepare_chart(chart, width, height) + + if chart.axis_count == 1: + data['series'][0]['data'] = self.process_one_dimensional_data( + chart, data['series'][0]['data'] + ) + if self.chart_type == 'pie': + data["series"] = [ + {"label": label, "data": [data]} + for label, data in zip(chart.x_labels, data["series"][0]["data"]) + if data + ] + + for serie in data['series']: + chart.add(serie['label'], serie['data']) return chart @@ -277,7 +294,6 @@ class ChartNgCell(CellBase): else: chart.axis_count = 2 - chart.show_legend = bool(len(response['axis']) > 1) chart.compute_sum = bool(response.get('measure') == 'integer' and chart.axis_count > 0) formatter = self.get_value_formatter(response.get('unit'), response.get('measure')) @@ -296,6 +312,7 @@ class ChartNgCell(CellBase): chart.config.explicit_size = True chart.config.js = [os.path.join(settings.STATIC_URL, 'js/pygal-tooltips.js')] + chart.show_legend = bool(chart.axis_count > 1) chart.truncate_legend = 30 # matplotlib tab10 palette chart.config.style.colors = ( @@ -319,6 +336,15 @@ class ChartNgCell(CellBase): if width and width < 500: chart.truncate_legend = 15 + def process_one_dimensional_data(self, chart, data): + if self.hide_null_values: + data = self.hide_values(chart, data) + if self.sort_order != 'none': + data = self.sort_values(chart, data) + if getattr(chart, 'compute_sum', True) and self.chart_type == 'table': + data = self.add_total_to_line_table(chart, data) + return data + @staticmethod def hide_values(chart, data): x_labels, new_data = zip(*[(label, value) for label, value in zip(chart.x_labels, data) if value]) @@ -340,7 +366,7 @@ class ChartNgCell(CellBase): def add_total_to_line_table(chart, data): # workaround pygal chart.compute_sum = False - data.append(sum(data)) + data.append(sum(x for x in data if x is not None)) chart.x_labels.append(gettext('Total')) return data diff --git a/combo/settings.py b/combo/settings.py index ad5578b0..9b92f461 100644 --- a/combo/settings.py +++ b/combo/settings.py @@ -350,6 +350,9 @@ COMBO_MAP_LAYER_ASSET_SLOTS = {} # known services KNOWN_SERVICES = {} +# known services exposing statistics +STATISTICS_PROVIDERS = [] + # PWA Settings PWA_VAPID_PUBLIK_KEY = None PWA_VAPID_PRIVATE_KEY = None diff --git a/tests/test_dataviz.py b/tests/test_dataviz.py index aeb4a470..25ee920b 100644 --- a/tests/test_dataviz.py +++ b/tests/test_dataviz.py @@ -219,6 +219,71 @@ def bijoe_mock(url, request): return {'content': json.dumps(response), 'request': request, 'status_code': 404} +STATISTICS_LIST = { + 'data': [ + { + 'url': 'https://authentic.example.com/api/statistics/one-serie/', + 'name': 'One serie stat', + 'id': 'one-serie', + }, + { + 'url': 'https://authentic.example.com/api/statistics/two-series/', + 'name': 'Two series stat', + 'id': 'two-series', + }, + { + 'url': 'https://authentic.example.com/api/statistics/no-data/', + 'name': 'No data stat', + 'id': 'no-data', + }, + { + 'url': 'https://authentic.example.com/api/statistics/not-found/', + 'name': '404 not found stat', + 'id': 'not-found', + }, + ] +} + + +def new_api_mock(url, request): + if url.path == '/visualization/json/': # nothing from bijoe + return {'content': b'{}', 'request': request, 'status_code': 200} + if url.path == '/api/statistics/': + return {'content': json.dumps(STATISTICS_LIST), 'request': request, 'status_code': 200} + if url.path == '/api/statistics/one-serie/': + response = { + 'data': { + 'series': [ + {'data': [None, 16, 2], 'label': 'Serie 1'}, + ], + 'x_labels': ['2020-10', '2020-11', '2020-12'], + }, + } + return {'content': json.dumps(response), 'request': request, 'status_code': 200} + if url.path == '/api/statistics/two-series/': + response = { + 'data': { + 'series': [ + {'data': [None, 16, 2], 'label': 'Serie 1'}, + {'data': [2, 1, None], 'label': 'Serie 2'}, + ], + 'x_labels': ['2020-10', '2020-11', '2020-12'], + }, + } + return {'content': json.dumps(response), 'request': request, 'status_code': 200} + if url.path == '/api/statistics/no-data/': + response = { + 'data': { + 'series': [ + ], + 'x_labels': [], + }, + } + return {'content': json.dumps(response), 'request': request, 'status_code': 200} + if url.path == '/api/statistics/not-found/': + return {'content': b'', 'request': request, 'status_code': 404} + + @pytest.fixture @with_httmock(bijoe_mock) def statistics(settings): @@ -232,6 +297,25 @@ def statistics(settings): assert Statistic.objects.count() == 11 +@pytest.fixture +@with_httmock(new_api_mock) +def new_api_statistics(settings): + settings.KNOWN_SERVICES = { + 'authentic': { + 'connection': { + 'title': 'Connection', + 'url': 'https://authentic.example.com', + 'secret': 'combo', + 'orig': 'combo', + } + } + } + settings.STATISTICS_PROVIDERS = ['authentic'] + appconfig = apps.get_app_config('dataviz') + appconfig.hourly() + assert Statistic.objects.count() == 4 + + @with_httmock(bijoe_mock) def test_chartng_cell(app, statistics): page = Page(title='One', slug='index') @@ -339,6 +423,47 @@ def test_chartng_cell(app, statistics): chart = cell.get_chart() +@with_httmock(new_api_mock) +def test_chartng_cell_new_api(app, new_api_statistics): + page = Page.objects.create(title='One', slug='index') + cell = ChartNgCell(page=page, order=1) + cell.statistic = Statistic.objects.get(slug='one-serie') + cell.save() + + chart = cell.get_chart() + assert chart.__class__.__name__ == 'Bar' + assert chart.x_labels == ['2020-10', '2020-11', '2020-12'] + assert chart.raw_series == [([None, 16, 2], {'title': 'Serie 1'},)] + + cell.chart_type = 'pie' + chart = cell.get_chart() + assert chart.__class__.__name__ == 'Pie' + assert chart.x_labels == ['2020-10', '2020-11', '2020-12'] + assert chart.raw_series == [ + ([16], {'title': '2020-11'}), + ([2], {'title': '2020-12'}), + ] + + cell.statistic = Statistic.objects.get(slug='two-series') + cell.save() + + chart = cell.get_chart() + assert chart.x_labels == ['2020-10', '2020-11', '2020-12'] + assert chart.raw_series == [([None, 16, 2], {'title': 'Serie 1'}), ([2, 1, None], {'title': 'Serie 2'})] + + cell.statistic = Statistic.objects.get(slug='no-data') + cell.save() + + chart = cell.get_chart() + assert chart.x_labels == [] + assert chart.raw_series == [] + + cell.statistic = Statistic.objects.get(slug='not-found') + cell.save() + with pytest.raises(HTTPError): + chart = cell.get_chart() + + @with_httmock(bijoe_mock) def test_chartng_cell_hide_null_values(app, statistics): page = Page(title='One', slug='index') @@ -424,6 +549,19 @@ def test_chartng_cell_hide_null_values(app, statistics): ] +@with_httmock(new_api_mock) +def test_chartng_cell_hide_null_values_new_api(app, new_api_statistics): + page = Page.objects.create(title='One', slug='index') + cell = ChartNgCell(page=page, order=1, hide_null_values=True) + cell.statistic = Statistic.objects.get(slug='one-serie') + cell.hide_null_values = True + cell.save() + + chart = cell.get_chart() + assert chart.x_labels == ['2020-11', '2020-12'] + assert chart.raw_series == [([16, 2], {'title': 'Serie 1'},)] + + @with_httmock(bijoe_mock) def test_chartng_cell_sort_order_alpha(app, statistics): page = Page(title='One', slug='index') @@ -594,6 +732,19 @@ def test_chartng_cell_sort_order_desc(app, statistics): ] +@with_httmock(new_api_mock) +def test_chartng_cell_sort_order_new_api(app, new_api_statistics): + page = Page.objects.create(title='One', slug='index') + cell = ChartNgCell(page=page, order=1) + cell.statistic = Statistic.objects.get(slug='one-serie') + cell.sort_order = 'desc' + cell.save() + + chart = cell.get_chart() + assert chart.x_labels == ['2020-11', '2020-12', '2020-10'] + assert chart.raw_series == [([16, 2, None], {'title': 'Serie 1'},)] + + @with_httmock(bijoe_mock) def test_chartng_cell_view(app, normal_user, statistics): page = Page(title='One', slug='index') @@ -698,6 +849,31 @@ def test_chartng_cell_view(app, normal_user, statistics): assert not 'cell' in resp.text +@with_httmock(new_api_mock) +def test_chartng_cell_view_new_api(app, normal_user, new_api_statistics): + page = Page.objects.create(title='One', slug='index') + cell = ChartNgCell(page=page, order=1, placeholder='content') + cell.statistic = Statistic.objects.get(slug='one-serie') + cell.save() + + location = '/api/dataviz/graph/%s/' % cell.id + resp = app.get('/') + assert 'min-height: 250px' in resp.text + assert location in resp.text + + # table visualization + cell.chart_type = 'table' + cell.save() + resp = app.get('/') + assert '' in resp.text + + # deleted visualization + cell.statistic = Statistic.objects.get(slug='not-found') + cell.save() + resp = app.get(location) + assert 'not found' in resp.text + + @with_httmock(bijoe_mock) def test_chartng_cell_manager(app, admin_user, statistics): page = Page(title='One', slug='index') @@ -726,6 +902,21 @@ def test_chartng_cell_manager(app, admin_user, statistics): assert 'Unavailable Stat' in resp.text +@with_httmock(new_api_mock) +def test_chartng_cell_manager_new_api(app, admin_user, new_api_statistics): + page = Page.objects.create(title='One', slug='index') + cell = ChartNgCell(page=page, order=1, placeholder='content') + cell.statistic = Statistic.objects.get(slug='one-serie') + cell.save() + + app = login(app) + resp = app.get('/manage/pages/%s/' % page.id) + statistics_field = resp.form['cdataviz_chartngcell-%s-statistic' % cell.id] + assert len(statistics_field.options) == 5 + assert statistics_field.value == str(cell.statistic.pk) + assert statistics_field.options[3][2] == 'Connection: One serie stat' + + @with_httmock(bijoe_mock) def test_table_cell(app, admin_user, statistics): page = Page(title='One', slug='index') @@ -769,6 +960,25 @@ def test_table_cell(app, admin_user, statistics): assert resp.text.count('Total') == 0 +@with_httmock(new_api_mock) +def test_table_cell_new_api(app, admin_user, new_api_statistics): + page = Page.objects.create(title='One', slug='index') + cell = ChartNgCell(page=page, order=1, placeholder='content') + cell.statistic = Statistic.objects.get(slug='one-serie') + cell.chart_type = 'table' + cell.save() + + app = login(app) + resp = app.get('/') + assert resp.text.count('Total') == 1 + + cell.statistic = Statistic.objects.get(slug='two-series') + cell.save() + resp = app.get('/') + assert '21' in resp.text + assert resp.text.count('Total') == 2 + + def test_dataviz_hourly_unavailable_statistic(freezer, statistics): all_stats_count = Statistic.objects.count() assert Statistic.objects.filter(available=True).count() == all_stats_count @@ -853,3 +1063,14 @@ def test_dataviz_cell_migration(settings): assert cell.statistic.site_slug == 'plop' assert cell.statistic.service_slug == 'bijoe' assert cell.statistic.site_title == 'test' + + +@with_httmock(new_api_mock) +def test_dataviz_api_list_statistics(new_api_statistics): + statistic = Statistic.objects.get(slug='one-serie') + assert statistic.label == 'One serie stat' + assert statistic.site_slug == 'connection' + assert statistic.service_slug == 'authentic' + assert statistic.site_title == 'Connection' + assert statistic.url == 'https://authentic.example.com/api/statistics/one-serie/' + assert statistic.available -- 2.20.1
18