From ae4084b3a60343b096d4d7f020478c0f3802de28 Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Thu, 5 Aug 2021 17:17:34 +0200 Subject: [PATCH 2/2] api: prefetch events in multiple agendas datetimes (#55370) --- chrono/agendas/models.py | 58 ++++++++++++++++++++++++++++++++++- chrono/api/views.py | 61 +++++++------------------------------ tests/api/test_datetimes.py | 2 +- 3 files changed, 69 insertions(+), 52 deletions(-) diff --git a/chrono/agendas/models.py b/chrono/agendas/models.py index 68afb929..fe7316f4 100644 --- a/chrono/agendas/models.py +++ b/chrono/agendas/models.py @@ -32,7 +32,7 @@ from django.contrib.postgres.fields import ArrayField, JSONField from django.core.exceptions import FieldDoesNotExist, ValidationError from django.core.validators import MaxValueValidator, MinValueValidator from django.db import IntegrityError, connection, models, transaction -from django.db.models import Count, IntegerField, Max, OuterRef, Q, Subquery, Value +from django.db.models import Count, IntegerField, Max, OuterRef, Prefetch, Q, Subquery, Value from django.db.models.functions import Coalesce from django.template import Context, Template, TemplateSyntaxError, VariableDoesNotExist, engines from django.urls import reverse @@ -877,6 +877,62 @@ class Agenda(models.Model): if e.desk_id == desk.pk or e.unavailability_calendar_id in uc_ids ] + @staticmethod + def prefetch_events_and_exceptions(qs, annotate_events=False, user_external_id=None): + event_queryset = Event.objects.filter( + Q(publication_date__isnull=True) | Q(publication_date__lte=localtime(now()).date()), + cancelled=False, + start_datetime__gte=localtime(now()), + ).order_by() + + if user_external_id: + event_queryset = Event.annotate_queryset_for_user(event_queryset, user_external_id) + if annotate_events: + event_queryset = Event.annotate_queryset(event_queryset) + + recurring_event_queryset = Event.objects.filter( + Q(publication_date__isnull=True) | Q(publication_date__lte=localtime(now()).date()), + recurrence_days__isnull=False, + ) + exceptions_desk = Desk.objects.filter(slug='_exceptions_holder').prefetch_related( + 'unavailability_calendars' + ) + qs = qs.filter(kind='events').prefetch_related( + Prefetch( + 'event_set', + queryset=event_queryset, + to_attr='prefetched_events', + ), + Prefetch( + 'event_set', + queryset=recurring_event_queryset, + to_attr='prefetched_recurring_events', + ), + Prefetch( + 'desk_set', + queryset=exceptions_desk, + to_attr='prefetched_desks', + ), + ) + agendas_exceptions = TimePeriodException.objects.filter( + Q(desk__slug='_exceptions_holder', desk__agenda__in=qs) + | Q( + unavailability_calendar__desks__slug='_exceptions_holder', + unavailability_calendar__desks__agenda__in=qs, + ), + start_datetime__gte=localtime(now()), + ) + agendas = list(qs) + for agenda in agendas: + desk = agenda.prefetched_desks[0] + uc_ids = [uc.pk for uc in desk.unavailability_calendars.all()] + agenda.prefetched_exceptions = [ + e + for e in agendas_exceptions + if e.desk_id == desk.pk or e.unavailability_calendar_id in uc_ids + ] + return agendas + def is_available_for_simple_management(self): if self.kind != 'meetings': return False diff --git a/chrono/api/views.py b/chrono/api/views.py index 7077cd3c..680517b2 100644 --- a/chrono/api/views.py +++ b/chrono/api/views.py @@ -681,57 +681,15 @@ class Agendas(APIView): with_open_events = request.GET.get('with_open_events') in ['1', 'true'] if with_open_events: # return only events agenda - event_queryset = Event.objects.filter( - Q(publication_date__isnull=True) | Q(publication_date__lte=localtime(now()).date()), - cancelled=False, - start_datetime__gte=localtime(now()), - ).order_by() - recurring_event_queryset = Event.objects.filter( - Q(publication_date__isnull=True) | Q(publication_date__lte=localtime(now()).date()), - recurrence_days__isnull=False, - ) - exceptions_desk = Desk.objects.filter(slug='_exceptions_holder').prefetch_related( - 'unavailability_calendars' - ) - agendas_queryset = agendas_queryset.filter(kind='events').prefetch_related( - Prefetch( - 'event_set', - queryset=event_queryset, - to_attr='prefetched_events', - ), - Prefetch( - 'event_set', - queryset=recurring_event_queryset, - to_attr='prefetched_recurring_events', - ), - Prefetch( - 'desk_set', - queryset=exceptions_desk, - to_attr='prefetched_desks', - ), - ) - agendas_exceptions = TimePeriodException.objects.filter( - Q(desk__slug='_exceptions_holder', desk__agenda__in=agendas_queryset) - | Q( - unavailability_calendar__desks__slug='_exceptions_holder', - unavailability_calendar__desks__agenda__in=agendas_queryset, - ), - start_datetime__gte=localtime(now()), - ) + agendas_queryset = Agenda.prefetch_events_and_exceptions(agendas_queryset) agendas = [] for agenda in agendas_queryset: - if with_open_events: - desk = agenda.prefetched_desks[0] - uc_ids = [uc.pk for uc in desk.unavailability_calendars.all()] - agenda.prefetched_exceptions = [ - e - for e in agendas_exceptions - if e.desk_id == desk.pk or e.unavailability_calendar_id in uc_ids - ] - if not any(not e.full for e in agenda.get_open_events(prefetched_queryset=True)): - # exclude agendas without open events - continue + if with_open_events and not any( + not e.full for e in agenda.get_open_events(prefetched_queryset=True) + ): + # exclude agendas without open events + continue agendas.append(get_agenda_detail(request, agenda)) return Response({'data': agendas}) @@ -861,14 +819,17 @@ class MultipleAgendasDatetimes(APIView): ) user_external_id = payload.get('user_external_id') or payload.get('exclude_user_external_id') + agendas = Agenda.prefetch_events_and_exceptions( + agendas, annotate_events=True, user_external_id=user_external_id + ) + entries = [] for agenda in agendas: entries.extend( agenda.get_open_events( - annotate_queryset=True, + prefetched_queryset=True, min_start=payload.get('date_start'), max_start=payload.get('date_end'), - user_external_id=user_external_id, ) ) diff --git a/tests/api/test_datetimes.py b/tests/api/test_datetimes.py index 5a07941d..b27eb0ea 100644 --- a/tests/api/test_datetimes.py +++ b/tests/api/test_datetimes.py @@ -1451,4 +1451,4 @@ def test_datetimes_multiple_agendas_queries(app): with CaptureQueriesContext(connection) as ctx: resp = app.get('/api/agenda/datetimes/', params={'agendas': ','.join(str(i) for i in range(10))}) assert len(resp.json['data']) == 20 - assert len(ctx.captured_queries) == 21 + assert len(ctx.captured_queries) == 7 -- 2.20.1