From 7510e7637f6381869ad682756fdc8a249a928fa7 Mon Sep 17 00:00:00 2001
From: Valentin Deniaud <vdeniaud@entrouvert.com>
Date: Thu, 11 Feb 2021 11:59:00 +0100
Subject: [PATCH 3/3] dataviz: reduce queries number on statistics list update
 (#50891)

---
 combo/apps/dataviz/utils.py | 58 +++++++++++++++++++++++++++++--------
 tests/test_dataviz.py       | 17 +++++++++--
 2 files changed, 60 insertions(+), 15 deletions(-)

diff --git a/combo/apps/dataviz/utils.py b/combo/apps/dataviz/utils.py
index c8f6981f..986eac99 100644
--- a/combo/apps/dataviz/utils.py
+++ b/combo/apps/dataviz/utils.py
@@ -1,3 +1,6 @@
+from collections import OrderedDict
+
+import django
 from django.conf import settings
 from django.utils import timezone
 
@@ -10,6 +13,7 @@ def update_available_statistics():
     if not settings.KNOWN_SERVICES:
         return
 
+    results = []
     start_update = timezone.now()
     for provider in settings.STATISTICS_PROVIDERS:
         if isinstance(provider, dict):
@@ -36,16 +40,46 @@ def update_available_statistics():
                 result = result['data']  # detect new api
 
             for stat in result:
-                Statistic.objects.update_or_create(
-                    slug=stat.get('slug') or stat['id'],
-                    site_slug=site_key,
-                    service_slug=provider,
-                    defaults={
-                        'label': stat['name'],
-                        'url': stat.get('data-url') or stat['url'],
-                        'site_title': site_dict.get('title', ''),
-                        'filters': stat.get('filters', []),
-                        'available': True,
-                    },
+                results.append(
+                    Statistic(
+                        slug=stat.get('slug') or stat['id'],
+                        site_slug=site_key,
+                        service_slug=provider,
+                        label=stat['name'],
+                        url=stat.get('data-url') or stat['url'],
+                        site_title=site_dict.get('title', ''),
+                        filters=stat.get('filters', []),
+                        available=True,
+                    )
                 )
-    Statistic.objects.filter(last_update__lt=start_update).update(available=False)
+
+    update_fields = ('label', 'url', 'site_title', 'filters', 'available')
+    all_statistics = {stat.natural_key(): stat for stat in Statistic.objects.all()}
+    statistics_to_create = []
+    statistics_to_update = {}
+    for stat in results:
+        existing_stat = all_statistics.get(stat.natural_key())
+        if existing_stat:
+            for field in update_fields:
+                new_value = getattr(stat, field)
+                if getattr(existing_stat, field) != new_value:
+                    setattr(existing_stat, field, new_value)
+                    statistics_to_update[existing_stat.pk] = existing_stat
+        else:
+            statistics_to_create.append(stat)
+
+    Statistic.objects.bulk_create(statistics_to_create)
+    if django.VERSION < (2, 2, 0):
+        for statistic in statistics_to_update.values():
+            Statistic.objects.filter(pk=statistic.pk).update(
+                **{field: getattr(statistic, field) for field in update_fields}
+            )
+    else:
+        Statistic.objects.bulk_update(statistics_to_update.values(), update_fields)
+
+    available_stats = Statistic.objects.filter(available=True)
+    for stat in results:
+        available_stats = available_stats.exclude(
+            slug=stat.slug, site_slug=stat.site_slug, service_slug=stat.service_slug
+        )
+    available_stats.update(available=False)
diff --git a/tests/test_dataviz.py b/tests/test_dataviz.py
index b9b41077..8cdaff46 100644
--- a/tests/test_dataviz.py
+++ b/tests/test_dataviz.py
@@ -1049,14 +1049,14 @@ def test_chartng_cell_manager_multiple_cells(app, admin_user, new_api_statistics
     app = login(app)
     with CaptureQueriesContext(connection) as ctx:
         resp = app.get('/manage/pages/%s/' % page.id)
-        assert len(ctx.captured_queries) == 35
+        assert len(ctx.captured_queries) == 20
         assert new_api_mock.call['count'] == 1
 
     cell = ChartNgCell.objects.create(page=page, order=2, placeholder='content')
     cell = ChartNgCell.objects.create(page=page, order=3, placeholder='content')
     with CaptureQueriesContext(connection) as ctx:
         resp = app.get('/manage/pages/%s/' % page.id)
-        assert len(ctx.captured_queries) == 37
+        assert len(ctx.captured_queries) == 22
         assert new_api_mock.call['count'] == 2
 
 
@@ -1168,7 +1168,7 @@ def test_dataviz_import_cell():
 
 
 @with_httmock(new_api_mock)
-def test_dataviz_api_list_statistics(new_api_statistics, settings):
+def test_dataviz_api_list_statistics(new_api_statistics, settings, nocache):
     statistic = Statistic.objects.get(slug='one-serie')
     assert statistic.label == 'One serie stat'
     assert statistic.site_slug == 'connection'
@@ -1209,6 +1209,17 @@ def test_dataviz_api_list_statistics(new_api_statistics, settings):
     assert statistic.url == 'https://stat.com/stats/1/'
     assert statistic.available
 
+    # update statistic attribute
+    catalog['data'] = [{'url': 'https://stat.com/stats/2/', 'name': 'Test 2', 'id': 'test'}]
+
+    with HTTMock(success):
+        appconfig.hourly()
+
+    statistic.refresh_from_db()
+    assert statistic.label == 'Test 2'
+    assert statistic.url == 'https://stat.com/stats/2/'
+    assert statistic.available
+
     settings.STATISTICS_PROVIDERS.append('unknown')
     appconfig = apps.get_app_config('dataviz')
     with HTTMock(success):
-- 
2.20.1