0001-journal-add-event-type-statistics-47467.patch

Valentin Deniaud, 23 novembre 2020 18:18

Voir les différences: en ligne côte à côte

Subject: [PATCH] journal: add event type statistics (#47467)

 src/authentic2/apps/journal/models.py |  33 ++++++-
 src/authentic2/apps/journal/utils.py  |  19 ++++
 src/authentic2/journal_event_types.py | 102 +++++++++++++++++----
 tests/test_journal.py                 | 124 ++++++++++++++++++++++++--
 4 files changed, 250 insertions(+), 28 deletions(-)

     from django.conf import settings
     from django.contrib.auth import get_user_model
     from django.contrib.postgres.fields import ArrayField, JSONField
     from django.contrib.postgres.fields.jsonb import KeyTextTransform
     from django.contrib.contenttypes.models import ContentType
     from django.core.exceptions import ObjectDoesNotExist
     from django.db import models
     from django.db.models import QuerySet, Q, F, Value
     from django.db.models import QuerySet, Q, F, Value, Count
     from django.db.models.functions import Trunc
     from django.utils.translation import ugettext_lazy as _
     from django.utils.timezone import utc, now
-...
         def get_message(self, event, context=None):
             return self.label
         @classmethod
         def get_statistics(cls, group_by_time, group_by_field, group_by_references=False, start=None, end=None):
             if group_by_time not in ('timestamp', 'day', 'week', 'month', 'year'):
                 raise ValueError('Usupported value for group_by_time: %s' % time_group_by)
             event_type = EventType.objects.get_for_name(cls.name)
             qs = Event.objects.filter(type=event_type)
             if start:
                 qs = qs.filter(timestamp__gte=start)
             if end:
                 qs = qs.filter(timestamp__lte=end)
             value_fields = []
             if group_by_time != 'timestamp':
                 qs = qs.annotate(**{group_by_time: Trunc('timestamp', kind=group_by_time)})
             if not group_by_field.startswith('user') and not group_by_field.startswith('session'):
                 # get field from JSONField
                 qs = qs.annotate(**{group_by_field: KeyTextTransform(group_by_field, 'data')})
             values = [group_by_time, group_by_field]
             if group_by_references:
                 values.append('reference_ids')
             qs = qs.values(*values)
             qs = qs.annotate(count=Count('id'))
             return qs.order_by(group_by_time)
         def __repr__(self):
             return '<EventTypeDefinition %r %s>' % (self.name, self.label)

                 old[key] = _json_value(old_value)
             new[key] = _json_value(form.cleaned_data.get(key))
         return {'old': old, 'new': new}
     class Statistics:
         def __init__(self, qs, x_label_field):
             self.x_labels = list(qs.distinct().values_list(x_label_field, flat=True))
             self._x_labels_indexes = {label: i for i, label in enumerate(self.x_labels)}
             self.series = {}
         def add(self, x_label, y_label, value, **kwargs):
             serie = self.get_serie(y_label, **kwargs)
             index = self.x_index(x_label)
             serie[index] = (serie[index] or 0) + value
         def get_serie(self, label, loop_label=None):
             series = self.series if not loop_label else self.series.setdefault(loop_label, {})
             return series.setdefault(label, [None] * len(self.x_labels))
         def x_index(self, x_label):
             return self._x_labels_indexes[x_label]

     # You should have received a copy of the GNU Affero General Public License
     # along with this program.  If not, see <http://www.gnu.org/licenses/>.
     from django.contrib.contenttypes.models import ContentType
     from django.utils.translation import ugettext_lazy as _
     from authentic2.custom_user.models import get_attributes_map
     from authentic2.apps.journal.models import EventTypeDefinition
     from authentic2.apps.journal.utils import form_to_old_new
     from authentic2.apps.journal.models import EventTypeDefinition, n_2_pairing_rev
     from authentic2.apps.journal.utils import form_to_old_new, Statistics
     from authentic2.custom_user.models import User
     from . import models
     from .models import Service
     class EventTypeWithService(EventTypeDefinition):
-...
         @classmethod
         def get_service_name(self, event):
             (service,) = event.get_typed_references(models.Service)
             (service,) = event.get_typed_references(Service)
             if service is not None:
                 return str(service)
             if 'service_name' in event.data:
-...
             return ''
     class EventTypeWithMethod(EventTypeWithService):
         @classmethod
         def record(cls, user, session, service, how):
             super().record(user=user, session=session, service=service, data={'how': how})
         @classmethod
         def get_method_statistics(cls, group_by_time='timestamp', start=None, end=None):
             qs = cls.get_statistics(group_by_time=group_by_time, group_by_field='how', start=start, end=end)
             stats = Statistics(qs, x_label_field=group_by_time)
             for stat in qs:
                 stats.add(x_label=stat[group_by_time], y_label=stat['how'], value=stat['count'])
             series = []
             for how, data in stats.series.items():
                 label = _(login_method_label(how or ''))
                 series.append({'label': label, 'data': data})
             return {
                 'x_labels': [label.isoformat() for label in stats.x_labels],
                 'series': series,
+            }
         @classmethod
         def _get_method_statistics_by_reference(cls, group_by_time, reference, **kwargs):
             qs = cls.get_statistics(group_by_time, 'how', group_by_references=True, **kwargs)
             stats = Statistics(qs, x_label_field=group_by_time)
             def get_reference_label(instance_pk, labels_cache={}):
                 label = labels_cache.get(instance_pk)
                 if not label:
                     service = Service.objects.get(pk=instance_pk)
                     if reference == 'service':
                         label = str(service)
                     elif reference == 'ou':
                         label = str(service.ou)
                 return label
             service_ct_id = ContentType.objects.get_for_model(Service).pk
             for stat in qs:
                 for reference_id in stat['reference_ids'] or []:
                     content_type_id, instance_pk = n_2_pairing_rev(reference_id)
                     if content_type_id == service_ct_id:
                         reference_label = get_reference_label(instance_pk)
                         break
                 else:
                     reference_label = _('None')
                 stats.add(
                     x_label=stat[group_by_time],
                     y_label=stat['how'],
                     value=stat['count'],
                     loop_label=reference_label,
+                )
             series = {}
             for loop_label, y_data in stats.series.items():
                 for how, data in y_data.items():
                     y_label = _(login_method_label(how or ''))
                     service_series = series.setdefault(loop_label, [])
                     service_series.append({'label': y_label, 'data': data})
             return {
                 'x_labels': [label.isoformat() for label in stats.x_labels],
                 'series': series,
+            }
         @classmethod
         def get_service_statistics(cls, group_by_time='timestamp', start=None, end=None, **kwargs):
             return cls._get_method_statistics_by_reference(group_by_time, 'service', start=start, end=end)
         @classmethod
         def get_service_ou_statistics(cls, group_by_time='timestamp', start=None, end=None, **kwargs):
             return cls._get_method_statistics_by_reference(group_by_time, 'ou', start=start, end=end)
     def login_method_label(how):
         if how.startswith('password'):
             return _('password')
-...
                     yield name
     class UserLogin(EventTypeWithService):
     class UserLogin(EventTypeWithMethod):
         name = 'user.login'
         label = _('login')
         @classmethod
         def record(cls, user, session, service, how):
             super().record(user=user, session=session, service=service, data={'how': how})
         @classmethod
         def get_message(cls, event, context):
             how = event.get_data('how')
-...
             return _('registration request with email "%s"') % email
     class UserRegistration(EventTypeWithService):
     class UserRegistration(EventTypeWithMethod):
         name = 'user.registration'
         label = _('registration')
         @classmethod
         def record(cls, user, session, service, how):
             super().record(user=user, session=session, service=service, data={'how': how})
         @classmethod
         def get_message(cls, event, context):
             how = event.get_data('how')
-...
             super().record(user=user, session=session, service=service)
     class UserServiceSSO(EventTypeWithService):
     class UserServiceSSO(EventTypeWithMethod):
         name = 'user.service.sso'
         label = _('service single sign on')
         @classmethod
         def record(cls, user, session, service, how):
             super().record(user=user, session=session, service=service, data={'how': how})
         @classmethod
         def get_message(cls, event, context):
             service_name = cls.get_service_name(event)

     import mock
     import pytest
     import pytz
     from django.contrib.auth import get_user_model
     from django.core.management import call_command
-...
         assert Event.objects.which_references(service).count() == 6
         assert Event.objects.which_references(Service).count() == 11
         assert Event.objects.from_cursor(ev1.cursor).count() == 12
         assert list(Event.objects.all()[ev2.cursor:2]) == events[6:8]
         assert list(Event.objects.all()[-4:ev2.cursor]) == events[3:7]
         assert list(Event.objects.all()[ev2.cursor : 2]) == events[6:8]
         assert list(Event.objects.all()[-4 : ev2.cursor]) == events[3:7]
         assert set(Event.objects.which_references(service)[0].references) == set([service])
         # verify type, user and service are prefetched
-...
         assert EventTypeDefinition.get_for_name('user.sso') is SSO
         with pytest.raises(AssertionError, match='already registered'):
             class SSO2(UserEventTypes):
                 name = 'user.sso'
                 label = 'Single Sign On'
-...
         assert page.is_last_page
         assert not page.next_page_url
         assert page.previous_page_url
         assert page.events == random_events[-page.limit:]
         assert page.events == random_events[-page.limit :]
         request = rf.get('/' + page.previous_page_url)
         page = JournalForm(data=request.GET).page
-...
         assert not page.is_last_page
         assert page.next_page_url
         assert page.previous_page_url
         assert page.events == random_events[-2 * page.limit:-page.limit]
         assert page.events == random_events[-2 * page.limit : -page.limit]
         request = rf.get('/' + page.previous_page_url)
         page = JournalForm(data=request.GET).page
-...
         assert not page.is_last_page
         assert page.next_page_url
         assert page.previous_page_url
         assert page.events == random_events[-3 * page.limit:-2 * page.limit]
         assert page.events == random_events[-3 * page.limit : -2 * page.limit]
         request = rf.get('/' + page.next_page_url)
         form = JournalForm(data=request.GET)
-...
         assert not page.is_last_page
         assert page.next_page_url
         assert page.previous_page_url
         assert page.events == random_events[-2 * page.limit:-page.limit]
         assert page.events == random_events[-2 * page.limit : -page.limit]
         event_after_the_first_page = random_events[page.limit]
         request = rf.get('/' + form.make_url('before_cursor', event_after_the_first_page.cursor))
-...
         journal = Journal()
         journal.record('user.registration.request', email='john.doe@example.com')
         assert len(caplog.records) == 0
         with mock.patch.object(some_event_types['UserRegistrationRequest'], 'record', side_effect=Exception('boum')):
         with mock.patch.object(
             some_event_types['UserRegistrationRequest'], 'record', side_effect=Exception('boum')
         ):
             journal.record('user.registration.request', email='john.doe@example.com')
         assert len(caplog.records) == 1
         assert caplog.records[0].levelname == 'ERROR'
-...
         event = Event.objects.get()
         event.message
         assert not(caplog.records)
         assert not (caplog.records)
         caplog.clear()
         with mock.patch.object(some_event_types['UserLogin'], 'get_message', side_effect=Exception('boum')):
-...
         assert len(caplog.records) == 1
         assert caplog.records[0].levelname == 'ERROR'
         assert caplog.records[0].message == 'could not render message of event type "user.login"'
     @pytest.mark.parametrize('event_type_name', ['user.login', 'user.registration'])
     def test_statistics(db, event_type_name, freezer):
         from authentic2.a2_rbac.utils import get_default_ou
         from authentic2.a2_rbac.models import OrganizationalUnit as OU
         user = User.objects.create(username='john.doe', email='john.doe@example.com')
         user2 = User.objects.create(username='jane.doe', email='jane.doe@example.com')
         ou = OU.objects.create(name='Second OU')
         portal = Service.objects.create(name='portal', slug='portal', ou=ou)
         agendas = Service.objects.create(name='agendas', slug='agendas', ou=get_default_ou())
         forms = Service.objects.create(name='forms', slug='forms', ou=get_default_ou())
         method = {'how': 'password-on-https'}
         method2 = {'how': 'fc'}
         event_type = EventType.objects.get_for_name(event_type_name)
         freezer.move_to('2020-02-03 12:00')
         event = Event.objects.create(type=event_type, references=[user, portal], user=user, data=method)
         event = Event.objects.create(type=event_type, references=[user2, portal], user=user2, data=method)
         freezer.move_to('2020-02-03 13:00')
         event = Event.objects.create(type=event_type, references=[user, portal], user=user, data=method2)
         event = Event.objects.create(type=event_type, references=[user2, portal], user=user2, data=method2)
         freezer.move_to('2020-03-03 12:00')
         event = Event.objects.create(type=event_type, references=[user, portal], user=user, data=method)
         event = Event.objects.create(type=event_type, references=[user, agendas], user=user, data=method)
         event = Event.objects.create(type=event_type, references=[user, forms], user=user, data=method)
         event = Event.objects.create(type=event_type, user=user)
         event_type_definition = event_type.definition
         stats = event_type_definition.get_method_statistics()
         stats['series'].sort(key=lambda x: x['label'])
         assert stats == {
             'x_labels': ['2020-02-03T12:00:00+00:00', '2020-02-03T13:00:00+00:00', '2020-03-03T12:00:00+00:00'],
             'series': [
                 {'label': 'FranceConnect', 'data': [None, 2, None]},
                 {'label': 'none', 'data': [None, None, 1]},
                 {'label': 'password', 'data': [2, None, 3]},
             ],
+        }
         start = datetime(year=2020, month=2, day=3, hour=12, minute=30, tzinfo=pytz.UTC)
         end = datetime(year=2020, month=2, day=3, hour=13, minute=30, tzinfo=pytz.UTC)
         stats = event_type_definition.get_method_statistics(start=start, end=end)
         assert stats == {
             'x_labels': ['2020-02-03T13:00:00+00:00'],
             'series': [
                 {'label': 'FranceConnect', 'data': [2]},
             ],
+        }
         stats = event_type_definition.get_method_statistics('month')
         stats['series'].sort(key=lambda x: x['label'])
         assert stats == {
             'x_labels': ['2020-02-01T00:00:00+01:00', '2020-03-01T00:00:00+01:00'],
             'series': [
                 {'label': 'FranceConnect', 'data': [2, None]},
                 {'label': 'none', 'data': [None, 1]},
                 {'label': 'password', 'data': [2, 3]},
             ],
+        }
         stats = event_type_definition.get_method_statistics('year')
         stats['series'].sort(key=lambda x: x['label'])
         assert stats == {
             'x_labels': ['2020-01-01T00:00:00+01:00'],
             'series': [
                 {'label': 'FranceConnect', 'data': [2]},
                 {'label': 'none', 'data': [1]},
                 {'label': 'password', 'data': [5]},
             ],
+        }
         stats = event_type_definition.get_service_statistics('month')
         stats['series']['portal'].sort(key=lambda x: x['label'])
         assert stats == {
             'x_labels': ['2020-02-01T00:00:00+01:00', '2020-03-01T00:00:00+01:00'],
             'series': {
                 'portal': [{'label': 'FranceConnect', 'data': [2, None]}, {'label': 'password', 'data': [2, 1]}],
                 'agendas': [{'label': 'password', 'data': [None, 1]}],
                 'forms': [{'label': 'password', 'data': [None, 1]}],
                 'None': [{'label': 'none', 'data': [None, 1]}],
             },
+        }
         stats = event_type_definition.get_service_ou_statistics('month')
         stats['series']['Second OU'].sort(key=lambda x: x['label'])
         assert stats == {
             'x_labels': ['2020-02-01T00:00:00+01:00', '2020-03-01T00:00:00+01:00'],
             'series': {
                 'Second OU': [
                     {'label': 'FranceConnect', 'data': [2, None]},
                     {'label': 'password', 'data': [2, 1]},
                 ],
                 'Default organizational unit': [{'label': 'password', 'data': [None, 2]}],
                 'None': [{'label': 'none', 'data': [None, 1]}],
             },
+        }
+    -

Projet

Général

Profil

Produits Entr'ouvert » Authentic 2

0001-journal-add-event-type-statistics-47467.patch