0001-dataviz-move-statistic-list-update-code-50891.patch
| combo/apps/dataviz/__init__.py | ||
|---|---|---|
| 15 | 15 |
# along with this program. If not, see <http://www.gnu.org/licenses/>. |
| 16 | 16 | |
| 17 | 17 |
import django.apps |
| 18 |
from django.core import checks |
|
| 19 |
from django.conf import settings |
|
| 20 |
from django.utils import timezone |
|
| 21 | 18 |
from django.utils.translation import ugettext_lazy as _ |
| 22 | 19 | |
| 23 |
from combo.utils import requests |
|
| 24 | ||
| 25 | 20 | |
| 26 | 21 |
class AppConfig(django.apps.AppConfig): |
| 27 | 22 |
name = 'combo.apps.dataviz' |
| ... | ... | |
| 33 | 28 |
return urls.urlpatterns |
| 34 | 29 | |
| 35 | 30 |
def hourly(self): |
| 36 |
self.update_available_statistics() |
|
| 37 | ||
| 38 |
def update_available_statistics(self): |
|
| 39 |
from .models import Statistic, ChartNgCell |
|
| 40 | ||
| 41 |
if not settings.KNOWN_SERVICES: |
|
| 42 |
return |
|
| 43 | ||
| 44 |
start_update = timezone.now() |
|
| 45 |
for provider in settings.STATISTICS_PROVIDERS: |
|
| 46 |
if isinstance(provider, dict): |
|
| 47 |
url = provider['url'] |
|
| 48 |
sites = {provider['id']: {'title': provider['name']}}
|
|
| 49 |
provider = provider['id'] |
|
| 50 |
else: |
|
| 51 |
sites = settings.KNOWN_SERVICES.get(provider, {})
|
|
| 52 |
url = '/visualization/json/' if provider == 'bijoe' else '/api/statistics/' |
|
| 53 | ||
| 54 |
for site_key, site_dict in sites.items(): |
|
| 55 |
site_title = site_dict.pop('title', '')
|
|
| 56 |
response = requests.get( |
|
| 57 |
url, remote_service=site_dict, without_user=True, headers={'accept': 'application/json'}
|
|
| 58 |
) |
|
| 59 |
if response.status_code != 200: |
|
| 60 |
continue |
|
| 61 | ||
| 62 |
result = response.json() |
|
| 63 |
if isinstance(result, dict): |
|
| 64 |
result = result['data'] # detect new api |
|
| 65 | ||
| 66 |
for stat in result: |
|
| 67 |
Statistic.objects.update_or_create( |
|
| 68 |
slug=stat.get('slug') or stat['id'],
|
|
| 69 |
site_slug=site_key, |
|
| 70 |
service_slug=provider, |
|
| 71 |
defaults={
|
|
| 72 |
'label': stat['name'], |
|
| 73 |
'url': stat.get('data-url') or stat['url'],
|
|
| 74 |
'site_title': site_title, |
|
| 75 |
'filters': stat.get('filters', []),
|
|
| 76 |
'available': True, |
|
| 77 |
}, |
|
| 78 |
) |
|
| 79 |
Statistic.objects.filter(last_update__lt=start_update).update(available=False) |
|
| 31 |
from .models import ChartNgCell |
|
| 32 |
from .utils import update_available_statistics |
|
| 80 | 33 | |
| 34 |
update_available_statistics() |
|
| 81 | 35 |
for cell in ChartNgCell.objects.all(): |
| 82 | 36 |
cell.check_validity() |
| 83 | 37 | |
| combo/apps/dataviz/utils.py | ||
|---|---|---|
| 1 |
from django.conf import settings |
|
| 2 |
from django.utils import timezone |
|
| 3 | ||
| 4 |
from combo.utils import requests |
|
| 5 | ||
| 6 |
from .models import Statistic |
|
| 7 | ||
| 8 | ||
| 9 |
def update_available_statistics(): |
|
| 10 |
if not settings.KNOWN_SERVICES: |
|
| 11 |
return |
|
| 12 | ||
| 13 |
start_update = timezone.now() |
|
| 14 |
for provider in settings.STATISTICS_PROVIDERS: |
|
| 15 |
if isinstance(provider, dict): |
|
| 16 |
url = provider['url'] |
|
| 17 |
sites = {provider['id']: {'title': provider['name']}}
|
|
| 18 |
provider = provider['id'] |
|
| 19 |
else: |
|
| 20 |
sites = settings.KNOWN_SERVICES.get(provider, {})
|
|
| 21 |
url = '/visualization/json/' if provider == 'bijoe' else '/api/statistics/' |
|
| 22 | ||
| 23 |
for site_key, site_dict in sites.items(): |
|
| 24 |
response = requests.get( |
|
| 25 |
url, |
|
| 26 |
timeout=5, |
|
| 27 |
remote_service=site_dict if provider in settings.KNOWN_SERVICES else {},
|
|
| 28 |
without_user=True, |
|
| 29 |
headers={'accept': 'application/json'},
|
|
| 30 |
) |
|
| 31 |
if response.status_code != 200: |
|
| 32 |
continue |
|
| 33 | ||
| 34 |
result = response.json() |
|
| 35 |
if isinstance(result, dict): |
|
| 36 |
result = result['data'] # detect new api |
|
| 37 | ||
| 38 |
for stat in result: |
|
| 39 |
Statistic.objects.update_or_create( |
|
| 40 |
slug=stat.get('slug') or stat['id'],
|
|
| 41 |
site_slug=site_key, |
|
| 42 |
service_slug=provider, |
|
| 43 |
defaults={
|
|
| 44 |
'label': stat['name'], |
|
| 45 |
'url': stat.get('data-url') or stat['url'],
|
|
| 46 |
'site_title': site_dict.get('title', ''),
|
|
| 47 |
'filters': stat.get('filters', []),
|
|
| 48 |
'available': True, |
|
| 49 |
}, |
|
| 50 |
) |
|
| 51 |
Statistic.objects.filter(last_update__lt=start_update).update(available=False) |
|
| 0 |
- |
|