0002-dataviz-handle-new-api-to-get-statistics-from-elsewh.patch

Valentin Deniaud, 07 décembre 2020 16:40

Voir les différences: en ligne côte à côte

Subject: [PATCH 2/3] dataviz: handle new api to get statistics from elsewhere
 (#48865)

 combo/apps/dataviz/__init__.py |  42 ++++---
 combo/apps/dataviz/models.py   |  56 ++++++---
 combo/settings.py              |   3 +
 tests/test_dataviz.py          | 221 +++++++++++++++++++++++++++++++++
 4 files changed, 290 insertions(+), 32 deletions(-)

             if not settings.KNOWN_SERVICES:
                 return
             statistics_providers = settings.STATISTICS_PROVIDERS + ['bijoe']
             start_update = timezone.now()
             bijoe_sites = settings.KNOWN_SERVICES.get('bijoe').items()
             for site_key, site_dict in bijoe_sites:
                 result = requests.get('/visualization/json/',
                         remote_service=site_dict, without_user=True,
                         headers={'accept': 'application/json'}).json()
                 for stat in result:
                     Statistic.objects.update_or_create(
                         slug=stat['slug'],
                         site_slug=site_key,
                         service_slug='bijoe',
                         defaults={
                             'label': stat['name'],
                             'url': stat['data-url'],
                             'site_title': site_dict.get('title', ''),
                             'available': True,
+                        }
+                    )
             for service in statistics_providers:
                 sites = settings.KNOWN_SERVICES.get(service, {}).items()
                 for site_key, site_dict in sites:
                     if service == 'bijoe':
                         result = requests.get('/visualization/json/',
                                 remote_service=site_dict, without_user=True,
                                 headers={'accept': 'application/json'}).json()
                     else:
                         result = requests.get('/api/statistics/',
                                 remote_service=site_dict, without_user=True,
                                 headers={'accept': 'application/json'}).json()['data']
                     for stat in result:
                         Statistic.objects.update_or_create(
                             slug=stat.get('slug') or stat['id'],
                             site_slug=site_key,
                             service_slug=service,
                             defaults={
                                 'label': stat['name'],
                                 'url': stat.get('data-url') or stat['url'],
                                 'site_title': site_dict.get('title', ''),
                                 'available': True,
+                            }
+                        )
             Statistic.objects.filter(last_update__lt=start_update).update(available=False)

         @classmethod
         def is_enabled(self):
             return hasattr(settings, 'KNOWN_SERVICES') and settings.KNOWN_SERVICES.get('bijoe')
             return settings.KNOWN_SERVICES.get('bijoe') or settings.STATISTICS_PROVIDERS
         def get_default_form_class(self):
             from .forms import ChartNgForm
-...
                 else:
                     ctx['table'] = chart.render_table(
                         transpose=bool(chart.axis_count == 2),
                         total=chart.compute_sum,
                         total=getattr(chart, 'compute_sum', True),
+                    )
                     ctx['table'] = ctx['table'].replace('<table>', '<table class="main">')
             return ctx
-...
             response = requests.get(
                     self.statistic.url,
                     cache_duration=300,
                     remote_service='auto',
                     without_user=True,
                     raise_if_not_cached=raise_if_not_cached)
             response.raise_for_status()
             response = response.json()
-...
                 'table': pygal.Bar,
                 }[self.chart_type](config=pygal.Config(style=copy.copy(style)))
             x_labels, y_labels, data = self.parse_response(response, chart)
             chart.x_labels = x_labels
             self.prepare_chart(chart, width, height)
             if self.statistic.service_slug == 'bijoe':
                 x_labels, y_labels, data = self.parse_response(response, chart)
                 chart.x_labels = x_labels
                 self.prepare_chart(chart, width, height)
             if chart.axis_count == 1:
                 if self.hide_null_values:
                     data = self.hide_values(chart, data)
                 if self.sort_order != 'none':
                     data = self.sort_values(chart, data)
                 if chart.compute_sum and self.chart_type == 'table':
                     data = self.add_total_to_line_table(chart, data)
             self.add_data_to_chart(chart, data, y_labels)
                 if chart.axis_count == 1:
                     data = self.process_one_dimensional_data(chart, data)
                 self.add_data_to_chart(chart, data, y_labels)
             else:
                 data = response['data']
                 chart.x_labels = data['x_labels']
                 chart.axis_count = min(len(data['series']), 2)
                 self.prepare_chart(chart, width, height)
                 if chart.axis_count == 1:
                     data['series'][0]['data'] = self.process_one_dimensional_data(
                         chart, data['series'][0]['data']
+                    )
                     if self.chart_type == 'pie':
                         data["series"] = [
                             {"label": label, "data": [data]}
                             for label, data in zip(chart.x_labels, data["series"][0]["data"])
                             if data
+                        ]
                 for serie in data['series']:
                     chart.add(serie['label'], serie['data'])
             return chart
-...
             else:
                 chart.axis_count = 2
             chart.show_legend = bool(len(response['axis']) > 1)
             chart.compute_sum = bool(response.get('measure') == 'integer' and chart.axis_count > 0)
             formatter = self.get_value_formatter(response.get('unit'), response.get('measure'))
-...
                 chart.config.explicit_size = True
             chart.config.js = [os.path.join(settings.STATIC_URL, 'js/pygal-tooltips.js')]
             chart.show_legend = bool(chart.axis_count > 1)
             chart.truncate_legend = 30
             # matplotlib tab10 palette
             chart.config.style.colors = (
-...
                 if width and width < 500:
                     chart.truncate_legend = 15
         def process_one_dimensional_data(self, chart, data):
             if self.hide_null_values:
                 data = self.hide_values(chart, data)
             if self.sort_order != 'none':
                 data = self.sort_values(chart, data)
             if getattr(chart, 'compute_sum', True) and self.chart_type == 'table':
                 data = self.add_total_to_line_table(chart, data)
             return data
         @staticmethod
         def hide_values(chart, data):
             x_labels, new_data = zip(*[(label, value) for label, value in zip(chart.x_labels, data) if value])
-...
         def add_total_to_line_table(chart, data):
             # workaround pygal
             chart.compute_sum = False
             data.append(sum(data))
             data.append(sum(x for x in data if x is not None))
             chart.x_labels.append(gettext('Total'))
             return data

     # known services
     KNOWN_SERVICES = {}
     # known services exposing statistics
     STATISTICS_PROVIDERS = []
     # PWA Settings
     PWA_VAPID_PUBLIK_KEY = None
     PWA_VAPID_PRIVATE_KEY = None

             return {'content': json.dumps(response), 'request': request, 'status_code': 404}
     STATISTICS_LIST = {
         'data': [
+            {
                 'url': 'https://authentic.example.com/api/statistics/one-serie/',
                 'name': 'One serie stat',
                 'id': 'one-serie',
             },
+            {
                 'url': 'https://authentic.example.com/api/statistics/two-series/',
                 'name': 'Two series stat',
                 'id': 'two-series',
             },
+            {
                 'url': 'https://authentic.example.com/api/statistics/no-data/',
                 'name': 'No data stat',
                 'id': 'no-data',
             },
+            {
                 'url': 'https://authentic.example.com/api/statistics/not-found/',
                 'name': '404 not found stat',
                 'id': 'not-found',
             },
+        ]
+    }
     def new_api_mock(url, request):
         if url.path == '/visualization/json/':  # nothing from bijoe
             return {'content': b'{}', 'request': request, 'status_code': 200}
         if url.path == '/api/statistics/':
             return {'content': json.dumps(STATISTICS_LIST), 'request': request, 'status_code': 200}
         if url.path == '/api/statistics/one-serie/':
             response = {
                 'data': {
                     'series': [
                         {'data': [None, 16, 2], 'label': 'Serie 1'},
                     ],
                     'x_labels': ['2020-10', '2020-11', '2020-12'],
                 },
+            }
             return {'content': json.dumps(response), 'request': request, 'status_code': 200}
         if url.path == '/api/statistics/two-series/':
             response = {
                 'data': {
                     'series': [
                         {'data': [None, 16, 2], 'label': 'Serie 1'},
                         {'data': [2, 1, None], 'label': 'Serie 2'},
                     ],
                     'x_labels': ['2020-10', '2020-11', '2020-12'],
                 },
+            }
             return {'content': json.dumps(response), 'request': request, 'status_code': 200}
         if url.path == '/api/statistics/no-data/':
             response = {
                 'data': {
                     'series': [
                     ],
                     'x_labels': [],
                 },
+            }
             return {'content': json.dumps(response), 'request': request, 'status_code': 200}
         if url.path == '/api/statistics/not-found/':
             return {'content': b'', 'request': request, 'status_code': 404}
     @pytest.fixture
     @with_httmock(bijoe_mock)
     def statistics(settings):
-...
         assert Statistic.objects.count() == 11
     @pytest.fixture
     @with_httmock(new_api_mock)
     def new_api_statistics(settings):
         settings.KNOWN_SERVICES = {
             'authentic': {
                 'connection': {
                     'title': 'Connection',
                     'url': 'https://authentic.example.com',
                     'secret': 'combo',
                     'orig': 'combo',
+                }
+            }
+        }
         settings.STATISTICS_PROVIDERS = ['authentic']
         appconfig = apps.get_app_config('dataviz')
         appconfig.hourly()
         assert Statistic.objects.count() == 4
     @with_httmock(bijoe_mock)
     def test_chartng_cell(app, statistics):
         page = Page(title='One', slug='index')
-...
             chart = cell.get_chart()
     @with_httmock(new_api_mock)
     def test_chartng_cell_new_api(app, new_api_statistics):
         page = Page.objects.create(title='One', slug='index')
         cell = ChartNgCell(page=page, order=1)
         cell.statistic = Statistic.objects.get(slug='one-serie')
         cell.save()
         chart = cell.get_chart()
         assert chart.__class__.__name__ == 'Bar'
         assert chart.x_labels == ['2020-10', '2020-11', '2020-12']
         assert chart.raw_series == [([None, 16, 2], {'title': 'Serie 1'},)]
         cell.chart_type = 'pie'
         chart = cell.get_chart()
         assert chart.__class__.__name__ == 'Pie'
         assert chart.x_labels == ['2020-10', '2020-11', '2020-12']
         assert chart.raw_series == [
             ([16], {'title': '2020-11'}),
             ([2], {'title': '2020-12'}),
+        ]
         cell.statistic = Statistic.objects.get(slug='two-series')
         cell.save()
         chart = cell.get_chart()
         assert chart.x_labels == ['2020-10', '2020-11', '2020-12']
         assert chart.raw_series == [([None, 16, 2], {'title': 'Serie 1'}), ([2, 1, None], {'title': 'Serie 2'})]
         cell.statistic = Statistic.objects.get(slug='no-data')
         cell.save()
         chart = cell.get_chart()
         assert chart.x_labels == []
         assert chart.raw_series == []
         cell.statistic = Statistic.objects.get(slug='not-found')
         cell.save()
         with pytest.raises(HTTPError):
             chart = cell.get_chart()
     @with_httmock(bijoe_mock)
     def test_chartng_cell_hide_null_values(app, statistics):
         page = Page(title='One', slug='index')
-...
+        ]
     @with_httmock(new_api_mock)
     def test_chartng_cell_hide_null_values_new_api(app, new_api_statistics):
         page = Page.objects.create(title='One', slug='index')
         cell = ChartNgCell(page=page, order=1, hide_null_values=True)
         cell.statistic = Statistic.objects.get(slug='one-serie')
         cell.hide_null_values = True
         cell.save()
         chart = cell.get_chart()
         assert chart.x_labels == ['2020-11', '2020-12']
         assert chart.raw_series == [([16, 2], {'title': 'Serie 1'},)]
     @with_httmock(bijoe_mock)
     def test_chartng_cell_sort_order_alpha(app, statistics):
         page = Page(title='One', slug='index')
-...
+        ]
     @with_httmock(new_api_mock)
     def test_chartng_cell_sort_order_new_api(app, new_api_statistics):
         page = Page.objects.create(title='One', slug='index')
         cell = ChartNgCell(page=page, order=1)
         cell.statistic = Statistic.objects.get(slug='one-serie')
         cell.sort_order = 'desc'
         cell.save()
         chart = cell.get_chart()
         assert chart.x_labels == ['2020-11', '2020-12', '2020-10']
         assert chart.raw_series == [([16, 2, None], {'title': 'Serie 1'},)]
     @with_httmock(bijoe_mock)
     def test_chartng_cell_view(app, normal_user, statistics):
         page = Page(title='One', slug='index')
-...
         assert not 'cell' in resp.text
     @with_httmock(new_api_mock)
     def test_chartng_cell_view_new_api(app, normal_user, new_api_statistics):
         page = Page.objects.create(title='One', slug='index')
         cell = ChartNgCell(page=page, order=1, placeholder='content')
         cell.statistic = Statistic.objects.get(slug='one-serie')
         cell.save()
         location = '/api/dataviz/graph/%s/' % cell.id
         resp = app.get('/')
         assert 'min-height: 250px' in resp.text
         assert location in resp.text
         # table visualization
         cell.chart_type = 'table'
         cell.save()
         resp = app.get('/')
         assert '<td>18</td>' in resp.text
         # deleted visualization
         cell.statistic = Statistic.objects.get(slug='not-found')
         cell.save()
         resp = app.get(location)
         assert 'not found' in resp.text
     @with_httmock(bijoe_mock)
     def test_chartng_cell_manager(app, admin_user, statistics):
         page = Page(title='One', slug='index')
-...
         assert 'Unavailable Stat' in resp.text
     @with_httmock(new_api_mock)
     def test_chartng_cell_manager_new_api(app, admin_user, new_api_statistics):
         page = Page.objects.create(title='One', slug='index')
         cell = ChartNgCell(page=page, order=1, placeholder='content')
         cell.statistic = Statistic.objects.get(slug='one-serie')
         cell.save()
         app = login(app)
         resp = app.get('/manage/pages/%s/' % page.id)
         statistics_field = resp.form['cdataviz_chartngcell-%s-statistic' % cell.id]
         assert len(statistics_field.options) == 5
         assert statistics_field.value == str(cell.statistic.pk)
         assert statistics_field.options[3][2] == 'Connection: One serie stat'
     @with_httmock(bijoe_mock)
     def test_table_cell(app, admin_user, statistics):
         page = Page(title='One', slug='index')
-...
         assert resp.text.count('Total') == 0
     @with_httmock(new_api_mock)
     def test_table_cell_new_api(app, admin_user, new_api_statistics):
         page = Page.objects.create(title='One', slug='index')
         cell = ChartNgCell(page=page, order=1, placeholder='content')
         cell.statistic = Statistic.objects.get(slug='one-serie')
         cell.chart_type = 'table'
         cell.save()
         app = login(app)
         resp = app.get('/')
         assert resp.text.count('Total') == 1
         cell.statistic = Statistic.objects.get(slug='two-series')
         cell.save()
         resp = app.get('/')
         assert '21' in resp.text
         assert resp.text.count('Total') == 2
     def test_dataviz_hourly_unavailable_statistic(freezer, statistics):
         all_stats_count = Statistic.objects.count()
         assert Statistic.objects.filter(available=True).count() == all_stats_count
-...
         assert cell.statistic.site_slug == 'plop'
         assert cell.statistic.service_slug == 'bijoe'
         assert cell.statistic.site_title == 'test'
     @with_httmock(new_api_mock)
     def test_dataviz_api_list_statistics(new_api_statistics):
         statistic = Statistic.objects.get(slug='one-serie')
         assert statistic.label == 'One serie stat'
         assert statistic.site_slug == 'connection'
         assert statistic.service_slug == 'authentic'
         assert statistic.site_title == 'Connection'
         assert statistic.url == 'https://authentic.example.com/api/statistics/one-serie/'
         assert statistic.available
+    -

Projet

Général

Profil

Produits Entr'ouvert » Combo

0002-dataviz-handle-new-api-to-get-statistics-from-elsewh.patch