0002-dataviz-load-statistics-list-dynamically-50891.patch
| combo/apps/dataviz/__init__.py | ||
|---|---|---|
| 15 | 15 |
# along with this program. If not, see <http://www.gnu.org/licenses/>. |
| 16 | 16 | |
| 17 | 17 |
import django.apps |
| 18 |
from django.core import checks |
|
| 19 |
from django.conf import settings |
|
| 20 |
from django.utils import timezone |
|
| 21 | 18 |
from django.utils.translation import ugettext_lazy as _ |
| 22 | 19 | |
| 23 |
from combo.utils import requests |
|
| 24 | ||
| 25 | 20 | |
| 26 | 21 |
class AppConfig(django.apps.AppConfig): |
| 27 | 22 |
name = 'combo.apps.dataviz' |
| ... | ... | |
| 32 | 27 |
return urls.urlpatterns |
| 33 | 28 | |
| 34 | 29 |
def hourly(self): |
| 35 |
self.update_available_statistics() |
|
| 36 | ||
| 37 |
def update_available_statistics(self): |
|
| 38 |
from .models import Statistic, ChartNgCell |
|
| 39 |
if not settings.KNOWN_SERVICES: |
|
| 40 |
return |
|
| 41 | ||
| 42 |
start_update = timezone.now() |
|
| 43 |
for provider in settings.STATISTICS_PROVIDERS: |
|
| 44 |
if isinstance(provider, dict): |
|
| 45 |
url = provider['url'] |
|
| 46 |
sites = {provider['id']: {'title': provider['name']}}
|
|
| 47 |
provider = provider['id'] |
|
| 48 |
else: |
|
| 49 |
sites = settings.KNOWN_SERVICES.get(provider, {})
|
|
| 50 |
url = '/visualization/json/' if provider == 'bijoe' else '/api/statistics/' |
|
| 51 | ||
| 52 |
for site_key, site_dict in sites.items(): |
|
| 53 |
site_title = site_dict.pop('title', '')
|
|
| 54 |
response = requests.get( |
|
| 55 |
url, remote_service=site_dict, without_user=True, headers={'accept': 'application/json'}
|
|
| 56 |
) |
|
| 57 |
if response.status_code != 200: |
|
| 58 |
continue |
|
| 59 | ||
| 60 |
result = response.json() |
|
| 61 |
if isinstance(result, dict): |
|
| 62 |
result = result['data'] # detect new api |
|
| 63 | ||
| 64 |
for stat in result: |
|
| 65 |
Statistic.objects.update_or_create( |
|
| 66 |
slug=stat.get('slug') or stat['id'],
|
|
| 67 |
site_slug=site_key, |
|
| 68 |
service_slug=provider, |
|
| 69 |
defaults={
|
|
| 70 |
'label': stat['name'], |
|
| 71 |
'url': stat.get('data-url') or stat['url'],
|
|
| 72 |
'site_title': site_title, |
|
| 73 |
'filters': stat.get('filters', []),
|
|
| 74 |
'available': True, |
|
| 75 |
} |
|
| 76 |
) |
|
| 77 |
Statistic.objects.filter(last_update__lt=start_update).update(available=False) |
|
| 30 |
from .models import ChartNgCell |
|
| 31 |
from .utils import update_available_statistics |
|
| 78 | 32 | |
| 33 |
update_available_statistics() |
|
| 79 | 34 |
for cell in ChartNgCell.objects.all(): |
| 80 | 35 |
cell.check_validity() |
| 81 | 36 | |
| combo/apps/dataviz/forms.py | ||
|---|---|---|
| 20 | 20 |
from django.conf import settings |
| 21 | 21 |
from django.db.models import Q |
| 22 | 22 | |
| 23 |
from combo.utils import requests |
|
| 23 |
from combo.utils import requests, cache_during_request
|
|
| 24 | 24 | |
| 25 |
from .models import ChartCell, ChartNgCell |
|
| 25 |
from .models import ChartCell, ChartNgCell, Statistic |
|
| 26 |
from .utils import update_available_statistics |
|
| 26 | 27 | |
| 27 | 28 | |
| 28 | 29 |
class ChartForm(forms.ModelForm): |
| ... | ... | |
| 42 | 43 |
self.fields['url'].widget = forms.Select(choices=available_charts) |
| 43 | 44 | |
| 44 | 45 | |
| 46 |
@cache_during_request |
|
| 47 |
def get_statistic_choices(): |
|
| 48 |
update_available_statistics() |
|
| 49 |
return Statistic.objects.all() |
|
| 50 | ||
| 51 | ||
| 45 | 52 |
class ChartNgForm(forms.ModelForm): |
| 46 | 53 |
blank_choice = ('', '---------')
|
| 47 | 54 | |
| ... | ... | |
| 56 | 63 | |
| 57 | 64 |
def __init__(self, *args, **kwargs): |
| 58 | 65 |
super().__init__(*args, **kwargs) |
| 66 |
stat_field = self.fields['statistic'] |
|
| 67 |
stat_field.queryset = get_statistic_choices() |
|
| 59 | 68 | |
| 60 | 69 |
field_ids = list(self._meta.fields) |
| 61 | 70 |
if not self.instance.statistic or self.instance.statistic.service_slug == 'bijoe': |
| ... | ... | |
| 63 | 72 |
x for x in field_ids if x not in ('time_range', 'time_range_start', 'time_range_end')
|
| 64 | 73 |
] |
| 65 | 74 | |
| 66 |
stat_field = self.fields['statistic'] |
|
| 67 | 75 |
if not self.instance.statistic: |
| 68 | 76 |
stat_field.queryset = stat_field.queryset.filter(available=True) |
| 69 | 77 |
else: |
| combo/apps/dataviz/utils.py | ||
|---|---|---|
| 1 |
from django.conf import settings |
|
| 2 |
from django.utils import timezone |
|
| 3 | ||
| 4 |
from combo.utils import requests |
|
| 5 | ||
| 6 |
from .models import Statistic |
|
| 7 | ||
| 8 | ||
| 9 |
def update_available_statistics(): |
|
| 10 |
if not settings.KNOWN_SERVICES: |
|
| 11 |
return |
|
| 12 | ||
| 13 |
start_update = timezone.now() |
|
| 14 |
for provider in settings.STATISTICS_PROVIDERS: |
|
| 15 |
if isinstance(provider, dict): |
|
| 16 |
url = provider['url'] |
|
| 17 |
sites = {provider['id']: {'title': provider['name']}}
|
|
| 18 |
provider = provider['id'] |
|
| 19 |
else: |
|
| 20 |
sites = settings.KNOWN_SERVICES.get(provider, {})
|
|
| 21 |
url = '/visualization/json/' if provider == 'bijoe' else '/api/statistics/' |
|
| 22 | ||
| 23 |
for site_key, site_dict in sites.items(): |
|
| 24 |
response = requests.get( |
|
| 25 |
url, |
|
| 26 |
timeout=5, |
|
| 27 |
remote_service=site_dict if provider in settings.KNOWN_SERVICES else {},
|
|
| 28 |
without_user=True, |
|
| 29 |
headers={'accept': 'application/json'},
|
|
| 30 |
) |
|
| 31 |
if response.status_code != 200: |
|
| 32 |
continue |
|
| 33 | ||
| 34 |
result = response.json() |
|
| 35 |
if isinstance(result, dict): |
|
| 36 |
result = result['data'] # detect new api |
|
| 37 | ||
| 38 |
for stat in result: |
|
| 39 |
Statistic.objects.update_or_create( |
|
| 40 |
slug=stat.get('slug') or stat['id'],
|
|
| 41 |
site_slug=site_key, |
|
| 42 |
service_slug=provider, |
|
| 43 |
defaults={
|
|
| 44 |
'label': stat['name'], |
|
| 45 |
'url': stat.get('data-url') or stat['url'],
|
|
| 46 |
'site_title': site_dict.get('title', ''),
|
|
| 47 |
'filters': stat.get('filters', []),
|
|
| 48 |
'available': True, |
|
| 49 |
}, |
|
| 50 |
) |
|
| 51 |
Statistic.objects.filter(last_update__lt=start_update).update(available=False) |
|
| tests/test_dataviz.py | ||
|---|---|---|
| 1041 | 1041 |
assert 'time_interval' in resp.text |
| 1042 | 1042 | |
| 1043 | 1043 | |
| 1044 |
@with_httmock(new_api_mock) |
|
| 1044 | 1045 |
def test_chartng_cell_manager_multiple_cells(app, admin_user, new_api_statistics, nocache): |
| 1045 | 1046 |
page = Page.objects.create(title='One', slug='index') |
| 1046 | 1047 |
cell = ChartNgCell.objects.create(page=page, order=1, placeholder='content') |
| ... | ... | |
| 1048 | 1049 |
app = login(app) |
| 1049 | 1050 |
with CaptureQueriesContext(connection) as ctx: |
| 1050 | 1051 |
resp = app.get('/manage/pages/%s/' % page.id)
|
| 1051 |
assert len(ctx.captured_queries) == 18 |
|
| 1052 |
assert len(ctx.captured_queries) == 35 |
|
| 1053 |
assert new_api_mock.call['count'] == 1 |
|
| 1052 | 1054 | |
| 1053 | 1055 |
cell = ChartNgCell.objects.create(page=page, order=2, placeholder='content') |
| 1054 | 1056 |
cell = ChartNgCell.objects.create(page=page, order=3, placeholder='content') |
| 1055 | 1057 |
with CaptureQueriesContext(connection) as ctx: |
| 1056 | 1058 |
resp = app.get('/manage/pages/%s/' % page.id)
|
| 1057 |
assert len(ctx.captured_queries) == 20 |
|
| 1059 |
assert len(ctx.captured_queries) == 37 |
|
| 1060 |
assert new_api_mock.call['count'] == 2 |
|
| 1058 | 1061 | |
| 1059 | 1062 | |
| 1060 | 1063 |
@with_httmock(bijoe_mock) |
| 1061 |
- |
|