0001-dataviz-update-statistics-list-every-5-minutes-50892.patch

Valentin Deniaud, 10 février 2021 10:24

Voir les différences: en ligne côte à côte

Subject: [PATCH] dataviz: update statistics list every 5 minutes (#50892)

 combo/apps/dataviz/__init__.py                | 49 +-------------
 .../commands/update_available_statistics.py   | 66 +++++++++++++++++++
 debian/combo.cron.d                           |  1 +
 tests/test_dataviz.py                         | 13 ++--
 4 files changed, 75 insertions(+), 54 deletions(-)
 create mode 100644 combo/apps/dataviz/management/commands/update_available_statistics.py

     # along with this program.  If not, see <http://www.gnu.org/licenses/>.
     import django.apps
     from django.core import checks
     from django.conf import settings
     from django.utils import timezone
     from django.utils.translation import ugettext_lazy as _
     from combo.utils import requests
     class AppConfig(django.apps.AppConfig):
-...
             return urls.urlpatterns
         def hourly(self):
             self.update_available_statistics()
         def update_available_statistics(self):
             from .models import Statistic, ChartNgCell
             if not settings.KNOWN_SERVICES:
                 return
             start_update = timezone.now()
             for provider in settings.STATISTICS_PROVIDERS:
                 if isinstance(provider, dict):
                     url = provider['url']
                     sites = {provider['id']: {'title': provider['name']}}
                     provider = provider['id']
                 else:
                     sites = settings.KNOWN_SERVICES.get(provider, {})
                     url = '/visualization/json/' if provider == 'bijoe' else '/api/statistics/'
                 for site_key, site_dict in sites.items():
                     site_title = site_dict.pop('title', '')
                     response = requests.get(
                         url, remote_service=site_dict, without_user=True, headers={'accept': 'application/json'}
+                    )
                     if response.status_code != 200:
                         continue
                     result = response.json()
                     if isinstance(result, dict):
                         result = result['data']  # detect new api
                     for stat in result:
                         Statistic.objects.update_or_create(
                             slug=stat.get('slug') or stat['id'],
                             site_slug=site_key,
                             service_slug=provider,
                             defaults={
                                 'label': stat['name'],
                                 'url': stat.get('data-url') or stat['url'],
                                 'site_title': site_title,
                                 'filters': stat.get('filters', []),
                                 'available': True,
+                            }
+                        )
             Statistic.objects.filter(last_update__lt=start_update).update(available=False)
             from combo.apps.dataviz.models import ChartNgCell
             for cell in ChartNgCell.objects.all():
                 cell.check_validity()

     # combo - content management system
     # Copyright (C) 2021  Entr'ouvert
+    #
     # This program is free software: you can redistribute it and/or modify it
     # under the terms of the GNU Affero General Public License as published
     # by the Free Software Foundation, either version 3 of the License, or
     # (at your option) any later version.
+    #
     # This program is distributed in the hope that it will be useful,
     # but WITHOUT ANY WARRANTY; without even the implied warranty of
     # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     # GNU Affero General Public License for more details.
+    #
     # You should have received a copy of the GNU Affero General Public License
     # along with this program.  If not, see <http://www.gnu.org/licenses/>.
     from django.conf import settings
     from django.core.management.base import BaseCommand
     from django.utils import timezone
     from combo.apps.dataviz.models import Statistic
     from combo.utils import requests
     class Command(BaseCommand):
         def handle(self, *args, **kwargs):
             if not settings.KNOWN_SERVICES:
                 return
             start_update = timezone.now()
             for provider in settings.STATISTICS_PROVIDERS:
                 if isinstance(provider, dict):
                     url = provider['url']
                     sites = {provider['id']: {'title': provider['name']}}
                     provider = provider['id']
                 else:
                     sites = settings.KNOWN_SERVICES.get(provider, {})
                     url = '/visualization/json/' if provider == 'bijoe' else '/api/statistics/'
                 for site_key, site_dict in sites.items():
                     site_title = site_dict.pop('title', '')
                     response = requests.get(
                         url, remote_service=site_dict, timeout=5, without_user=True, headers={'accept': 'application/json'}
+                    )
                     if response.status_code != 200:
                         continue
                     result = response.json()
                     if isinstance(result, dict):
                         result = result['data']  # detect new api
                     for stat in result:
                         Statistic.objects.update_or_create(
                             slug=stat.get('slug') or stat['id'],
                             site_slug=site_key,
                             service_slug=provider,
                             defaults={
                                 'label': stat['name'],
                                 'url': stat.get('data-url') or stat['url'],
                                 'site_title': site_title,
                                 'filters': stat.get('filters', []),
                                 'available': True,
+                            }
+                        )
             Statistic.objects.filter(last_update__lt=start_update).update(available=False)

debian/combo.cron.d
1	1	MAILTO=root
2	2
3	3	0 8 * * * combo /usr/bin/combo-manage tenant_command notify_new_remote_invoices --all-tenants -v0
	4	/5 * * * combo /usr/bin/combo-manage tenant_command update_available_statistics --all-tenants -v0

     from django.apps import apps
     from django.contrib.auth.models import User, Group
     from django.core.management import call_command
     from django.db import connection
     from django.db.migrations.executor import MigrationExecutor
     from django.test import override_settings
-...
+        }
         settings.STATISTICS_PROVIDERS = ['bijoe']
         appconfig = apps.get_app_config('dataviz')
         appconfig.hourly()
         call_command('update_available_statistics')
         assert Statistic.objects.count() == len(VISUALIZATION_JSON)
-...
+        }
         settings.STATISTICS_PROVIDERS = ['authentic']
         appconfig = apps.get_app_config('dataviz')
         appconfig.hourly()
         call_command('update_available_statistics')
         assert Statistic.objects.count() == len(STATISTICS_LIST['data'])
-...
         appconfig = apps.get_app_config('dataviz')
         with HTTMock(bijoe_mock_unavailable):
             appconfig.hourly()
             call_command('update_available_statistics')
         assert Statistic.objects.filter(available=True).count() == all_stats_count - 2
-...
         appconfig = apps.get_app_config('dataviz')
         with HTTMock(server_error):
             appconfig.hourly()
             call_command('update_available_statistics')
         assert Statistic.objects.count() == statistics_count
         @urlmatch(scheme='https', netloc=r'stat.com', path='/stats/')
-...
             return {'content': json.dumps(catalog), 'status_code': 200}
         with HTTMock(success):
             appconfig.hourly()
             call_command('update_available_statistics')
         assert Statistic.objects.count() == statistics_count + 1
         statistic = Statistic.objects.get(slug='test')
-...
         settings.STATISTICS_PROVIDERS.append('unknown')
         appconfig = apps.get_app_config('dataviz')
         with HTTMock(success):
             appconfig.hourly()  # unknown provider is ignored
             call_command('update_available_statistics')  # unknown provider is ignored
     @with_httmock(new_api_mock)
+    -

Projet

Général

Profil

Produits Entr'ouvert » Combo

0001-dataviz-update-statistics-list-every-5-minutes-50892.patch