0001-dataviz-update-statistics-list-every-5-minutes-50892.patch
combo/apps/dataviz/__init__.py | ||
---|---|---|
15 | 15 |
# along with this program. If not, see <http://www.gnu.org/licenses/>. |
16 | 16 | |
17 | 17 |
import django.apps |
18 |
from django.core import checks |
|
19 |
from django.conf import settings |
|
20 |
from django.utils import timezone |
|
21 | 18 |
from django.utils.translation import ugettext_lazy as _ |
22 | 19 | |
23 |
from combo.utils import requests |
|
24 | 20 | |
25 | 21 | |
26 | 22 |
class AppConfig(django.apps.AppConfig): |
... | ... | |
32 | 28 |
return urls.urlpatterns |
33 | 29 | |
34 | 30 |
def hourly(self): |
35 |
self.update_available_statistics() |
|
36 | ||
37 |
def update_available_statistics(self): |
|
38 |
from .models import Statistic, ChartNgCell |
|
39 |
if not settings.KNOWN_SERVICES: |
|
40 |
return |
|
41 | ||
42 |
start_update = timezone.now() |
|
43 |
for provider in settings.STATISTICS_PROVIDERS: |
|
44 |
if isinstance(provider, dict): |
|
45 |
url = provider['url'] |
|
46 |
sites = {provider['id']: {'title': provider['name']}} |
|
47 |
provider = provider['id'] |
|
48 |
else: |
|
49 |
sites = settings.KNOWN_SERVICES.get(provider, {}) |
|
50 |
url = '/visualization/json/' if provider == 'bijoe' else '/api/statistics/' |
|
51 | ||
52 |
for site_key, site_dict in sites.items(): |
|
53 |
site_title = site_dict.pop('title', '') |
|
54 |
response = requests.get( |
|
55 |
url, remote_service=site_dict, without_user=True, headers={'accept': 'application/json'} |
|
56 |
) |
|
57 |
if response.status_code != 200: |
|
58 |
continue |
|
59 | ||
60 |
result = response.json() |
|
61 |
if isinstance(result, dict): |
|
62 |
result = result['data'] # detect new api |
|
63 | ||
64 |
for stat in result: |
|
65 |
Statistic.objects.update_or_create( |
|
66 |
slug=stat.get('slug') or stat['id'], |
|
67 |
site_slug=site_key, |
|
68 |
service_slug=provider, |
|
69 |
defaults={ |
|
70 |
'label': stat['name'], |
|
71 |
'url': stat.get('data-url') or stat['url'], |
|
72 |
'site_title': site_title, |
|
73 |
'filters': stat.get('filters', []), |
|
74 |
'available': True, |
|
75 |
} |
|
76 |
) |
|
77 |
Statistic.objects.filter(last_update__lt=start_update).update(available=False) |
|
78 | ||
31 |
from combo.apps.dataviz.models import ChartNgCell |
|
79 | 32 |
for cell in ChartNgCell.objects.all(): |
80 | 33 |
cell.check_validity() |
81 | 34 |
combo/apps/dataviz/management/commands/update_available_statistics.py | ||
---|---|---|
1 |
# combo - content management system |
|
2 |
# Copyright (C) 2021 Entr'ouvert |
|
3 |
# |
|
4 |
# This program is free software: you can redistribute it and/or modify it |
|
5 |
# under the terms of the GNU Affero General Public License as published |
|
6 |
# by the Free Software Foundation, either version 3 of the License, or |
|
7 |
# (at your option) any later version. |
|
8 |
# |
|
9 |
# This program is distributed in the hope that it will be useful, |
|
10 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
11 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
12 |
# GNU Affero General Public License for more details. |
|
13 |
# |
|
14 |
# You should have received a copy of the GNU Affero General Public License |
|
15 |
# along with this program. If not, see <http://www.gnu.org/licenses/>. |
|
16 | ||
17 |
from django.conf import settings |
|
18 |
from django.core.management.base import BaseCommand |
|
19 |
from django.utils import timezone, translation |
|
20 | ||
21 |
from combo.apps.dataviz.models import Statistic |
|
22 |
from combo.utils import requests |
|
23 | ||
24 | ||
25 |
class Command(BaseCommand): |
|
26 | ||
27 |
def handle(self, *args, **kwargs): |
|
28 |
translation.activate(settings.LANGUAGE_CODE) |
|
29 |
if not settings.KNOWN_SERVICES: |
|
30 |
return |
|
31 | ||
32 |
start_update = timezone.now() |
|
33 |
for provider in settings.STATISTICS_PROVIDERS: |
|
34 |
if isinstance(provider, dict): |
|
35 |
url = provider['url'] |
|
36 |
sites = {provider['id']: {'title': provider['name']}} |
|
37 |
provider = provider['id'] |
|
38 |
else: |
|
39 |
sites = settings.KNOWN_SERVICES.get(provider, {}) |
|
40 |
url = '/visualization/json/' if provider == 'bijoe' else '/api/statistics/' |
|
41 | ||
42 |
for site_key, site_dict in sites.items(): |
|
43 |
site_title = site_dict.pop('title', '') |
|
44 |
response = requests.get( |
|
45 |
url, remote_service=site_dict, without_user=True, headers={'accept': 'application/json'} |
|
46 |
) |
|
47 |
if response.status_code != 200: |
|
48 |
continue |
|
49 | ||
50 |
result = response.json() |
|
51 |
if isinstance(result, dict): |
|
52 |
result = result['data'] # detect new api |
|
53 | ||
54 |
for stat in result: |
|
55 |
Statistic.objects.update_or_create( |
|
56 |
slug=stat.get('slug') or stat['id'], |
|
57 |
site_slug=site_key, |
|
58 |
service_slug=provider, |
|
59 |
defaults={ |
|
60 |
'label': stat['name'], |
|
61 |
'url': stat.get('data-url') or stat['url'], |
|
62 |
'site_title': site_title, |
|
63 |
'filters': stat.get('filters', []), |
|
64 |
'available': True, |
|
65 |
} |
|
66 |
) |
|
67 |
Statistic.objects.filter(last_update__lt=start_update).update(available=False) |
debian/combo.cron.d | ||
---|---|---|
1 | 1 |
MAILTO=root |
2 | 2 | |
3 | 3 |
0 8 * * * combo /usr/bin/combo-manage tenant_command notify_new_remote_invoices --all-tenants -v0 |
4 |
*/5 * * * * combo /usr/bin/combo-manage tenant_command update_available_statistics --all-tenants -v0 |
tests/test_dataviz.py | ||
---|---|---|
9 | 9 | |
10 | 10 |
from django.apps import apps |
11 | 11 |
from django.contrib.auth.models import User, Group |
12 |
from django.core.management import call_command |
|
12 | 13 |
from django.db import connection |
13 | 14 |
from django.db.migrations.executor import MigrationExecutor |
14 | 15 |
from django.test import override_settings |
... | ... | |
323 | 324 |
} |
324 | 325 |
settings.STATISTICS_PROVIDERS = ['bijoe'] |
325 | 326 |
appconfig = apps.get_app_config('dataviz') |
326 |
appconfig.hourly()
|
|
327 |
call_command('update_available_statistics')
|
|
327 | 328 |
assert Statistic.objects.count() == len(VISUALIZATION_JSON) |
328 | 329 | |
329 | 330 | |
... | ... | |
342 | 343 |
} |
343 | 344 |
settings.STATISTICS_PROVIDERS = ['authentic'] |
344 | 345 |
appconfig = apps.get_app_config('dataviz') |
345 |
appconfig.hourly()
|
|
346 |
call_command('update_available_statistics')
|
|
346 | 347 |
assert Statistic.objects.count() == len(STATISTICS_LIST['data']) |
347 | 348 | |
348 | 349 | |
... | ... | |
1112 | 1113 | |
1113 | 1114 |
appconfig = apps.get_app_config('dataviz') |
1114 | 1115 |
with HTTMock(bijoe_mock_unavailable): |
1115 |
appconfig.hourly()
|
|
1116 |
call_command('update_available_statistics')
|
|
1116 | 1117 |
assert Statistic.objects.filter(available=True).count() == all_stats_count - 2 |
1117 | 1118 | |
1118 | 1119 | |
... | ... | |
1165 | 1166 | |
1166 | 1167 |
appconfig = apps.get_app_config('dataviz') |
1167 | 1168 |
with HTTMock(server_error): |
1168 |
appconfig.hourly()
|
|
1169 |
call_command('update_available_statistics')
|
|
1169 | 1170 |
assert Statistic.objects.count() == statistics_count |
1170 | 1171 | |
1171 | 1172 |
@urlmatch(scheme='https', netloc=r'stat.com', path='/stats/') |
... | ... | |
1173 | 1174 |
return {'content': json.dumps(catalog), 'status_code': 200} |
1174 | 1175 | |
1175 | 1176 |
with HTTMock(success): |
1176 |
appconfig.hourly()
|
|
1177 |
call_command('update_available_statistics')
|
|
1177 | 1178 | |
1178 | 1179 |
assert Statistic.objects.count() == statistics_count + 1 |
1179 | 1180 |
statistic = Statistic.objects.get(slug='test') |
... | ... | |
1187 | 1188 |
settings.STATISTICS_PROVIDERS.append('unknown') |
1188 | 1189 |
appconfig = apps.get_app_config('dataviz') |
1189 | 1190 |
with HTTMock(success): |
1190 |
appconfig.hourly() # unknown provider is ignored
|
|
1191 |
call_command('update_available_statistics') # unknown provider is ignored
|
|
1191 | 1192 | |
1192 | 1193 | |
1193 | 1194 |
@with_httmock(new_api_mock) |
1194 |
- |