0003-dataviz-reduce-queries-number-on-statistics-list-upd.patch
| combo/apps/dataviz/utils.py | ||
|---|---|---|
| 1 |
from collections import OrderedDict |
|
| 2 | ||
| 3 |
import django |
|
| 1 | 4 |
from django.conf import settings |
| 2 | 5 |
from django.utils import timezone |
| 3 | 6 | |
| ... | ... | |
| 10 | 13 |
if not settings.KNOWN_SERVICES: |
| 11 | 14 |
return |
| 12 | 15 | |
| 16 |
results = [] |
|
| 13 | 17 |
start_update = timezone.now() |
| 14 | 18 |
for provider in settings.STATISTICS_PROVIDERS: |
| 15 | 19 |
if isinstance(provider, dict): |
| ... | ... | |
| 36 | 40 |
result = result['data'] # detect new api |
| 37 | 41 | |
| 38 | 42 |
for stat in result: |
| 39 |
Statistic.objects.update_or_create(
|
|
| 40 |
slug=stat.get('slug') or stat['id'],
|
|
| 41 |
site_slug=site_key,
|
|
| 42 |
service_slug=provider,
|
|
| 43 |
defaults={
|
|
| 44 |
'label': stat['name'],
|
|
| 45 |
'url': stat.get('data-url') or stat['url'],
|
|
| 46 |
'site_title': site_dict.get('title', ''),
|
|
| 47 |
'filters': stat.get('filters', []),
|
|
| 48 |
'available': True,
|
|
| 49 |
},
|
|
| 43 |
results.append(
|
|
| 44 |
Statistic(
|
|
| 45 |
slug=stat.get('slug') or stat['id'],
|
|
| 46 |
site_slug=site_key,
|
|
| 47 |
service_slug=provider,
|
|
| 48 |
label=stat['name'],
|
|
| 49 |
url=stat.get('data-url') or stat['url'],
|
|
| 50 |
site_title=site_dict.get('title', ''),
|
|
| 51 |
filters=stat.get('filters', []),
|
|
| 52 |
available=True,
|
|
| 53 |
)
|
|
| 50 | 54 |
) |
| 51 |
Statistic.objects.filter(last_update__lt=start_update).update(available=False) |
|
| 55 | ||
| 56 |
update_fields = ('label', 'url', 'site_title', 'filters', 'available')
|
|
| 57 |
all_statistics = {stat.natural_key(): stat for stat in Statistic.objects.all()}
|
|
| 58 |
statistics_to_create = [] |
|
| 59 |
statistics_to_update = {}
|
|
| 60 |
for stat in results: |
|
| 61 |
existing_stat = all_statistics.get(stat.natural_key()) |
|
| 62 |
if existing_stat: |
|
| 63 |
for field in update_fields: |
|
| 64 |
new_value = getattr(stat, field) |
|
| 65 |
if getattr(existing_stat, field) != new_value: |
|
| 66 |
setattr(existing_stat, field, new_value) |
|
| 67 |
statistics_to_update[existing_stat.pk] = existing_stat |
|
| 68 |
else: |
|
| 69 |
statistics_to_create.append(stat) |
|
| 70 | ||
| 71 |
Statistic.objects.bulk_create(statistics_to_create) |
|
| 72 |
if django.VERSION < (2, 2, 0): |
|
| 73 |
for statistic in statistics_to_update.values(): |
|
| 74 |
Statistic.objects.filter(pk=statistic.pk).update( |
|
| 75 |
**{field: getattr(statistic, field) for field in update_fields}
|
|
| 76 |
) |
|
| 77 |
else: |
|
| 78 |
Statistic.objects.bulk_update(statistics_to_update.values(), update_fields) |
|
| 79 | ||
| 80 |
available_stats = Statistic.objects.filter(available=True) |
|
| 81 |
for stat in results: |
|
| 82 |
available_stats = available_stats.exclude( |
|
| 83 |
slug=stat.slug, site_slug=stat.site_slug, service_slug=stat.service_slug |
|
| 84 |
) |
|
| 85 |
available_stats.update(available=False) |
|
| tests/test_dataviz.py | ||
|---|---|---|
| 1049 | 1049 |
app = login(app) |
| 1050 | 1050 |
with CaptureQueriesContext(connection) as ctx: |
| 1051 | 1051 |
resp = app.get('/manage/pages/%s/' % page.id)
|
| 1052 |
assert len(ctx.captured_queries) == 35
|
|
| 1052 |
assert len(ctx.captured_queries) == 20
|
|
| 1053 | 1053 |
assert new_api_mock.call['count'] == 1 |
| 1054 | 1054 | |
| 1055 | 1055 |
cell = ChartNgCell.objects.create(page=page, order=2, placeholder='content') |
| 1056 | 1056 |
cell = ChartNgCell.objects.create(page=page, order=3, placeholder='content') |
| 1057 | 1057 |
with CaptureQueriesContext(connection) as ctx: |
| 1058 | 1058 |
resp = app.get('/manage/pages/%s/' % page.id)
|
| 1059 |
assert len(ctx.captured_queries) == 37
|
|
| 1059 |
assert len(ctx.captured_queries) == 22
|
|
| 1060 | 1060 |
assert new_api_mock.call['count'] == 2 |
| 1061 | 1061 | |
| 1062 | 1062 | |
| ... | ... | |
| 1168 | 1168 | |
| 1169 | 1169 | |
| 1170 | 1170 |
@with_httmock(new_api_mock) |
| 1171 |
def test_dataviz_api_list_statistics(new_api_statistics, settings): |
|
| 1171 |
def test_dataviz_api_list_statistics(new_api_statistics, settings, nocache):
|
|
| 1172 | 1172 |
statistic = Statistic.objects.get(slug='one-serie') |
| 1173 | 1173 |
assert statistic.label == 'One serie stat' |
| 1174 | 1174 |
assert statistic.site_slug == 'connection' |
| ... | ... | |
| 1209 | 1209 |
assert statistic.url == 'https://stat.com/stats/1/' |
| 1210 | 1210 |
assert statistic.available |
| 1211 | 1211 | |
| 1212 |
# update statistic attribute |
|
| 1213 |
catalog['data'] = [{'url': 'https://stat.com/stats/2/', 'name': 'Test 2', 'id': 'test'}]
|
|
| 1214 | ||
| 1215 |
with HTTMock(success): |
|
| 1216 |
appconfig.hourly() |
|
| 1217 | ||
| 1218 |
statistic.refresh_from_db() |
|
| 1219 |
assert statistic.label == 'Test 2' |
|
| 1220 |
assert statistic.url == 'https://stat.com/stats/2/' |
|
| 1221 |
assert statistic.available |
|
| 1222 | ||
| 1212 | 1223 |
settings.STATISTICS_PROVIDERS.append('unknown')
|
| 1213 | 1224 |
appconfig = apps.get_app_config('dataviz')
|
| 1214 | 1225 |
with HTTMock(success): |
| 1215 |
- |
|