From 1ddc763947be389ae07f0d6035bb25836091dbef Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Mon, 5 Dec 2022 18:26:56 +0100 Subject: [PATCH] statistics: add time between two statuses (#71661) --- tests/api/test_statistics.py | 261 +++++++++++++++++++++++++++++++++++ wcs/statistics/views.py | 128 ++++++++++++++++- wcs/urls.py | 5 + 3 files changed, 393 insertions(+), 1 deletion(-) diff --git a/tests/api/test_statistics.py b/tests/api/test_statistics.py index 3debc63af..0b71201e4 100644 --- a/tests/api/test_statistics.py +++ b/tests/api/test_statistics.py @@ -4,6 +4,7 @@ import os import pytest from wcs import fields +from wcs.backoffice.management import format_time from wcs.blocks import BlockDef from wcs.carddef import CardDef from wcs.categories import CardDefCategory, Category @@ -15,6 +16,10 @@ from ..utilities import clean_temporary_pub, create_temporary_pub, get_app from .utils import sign_uri +def get_humanized_duration_serie(json_resp): + return [format_time(x) for x in json_resp['data']['series'][0]['data']] + + @pytest.fixture def pub(): pub = create_temporary_pub() @@ -246,6 +251,19 @@ def test_statistics_index_cards(pub): ] +def test_statistics_index_resolution_time(pub): + formdef = FormDef() + formdef.name = 'test 1' + formdef.fields = [] + formdef.store() + formdef.data_class().wipe() + + resp = get_app(pub).get(sign_uri('/api/statistics/')) + resolution_time_stat = [x for x in resp.json['data'] if x['id'] == 'resolution_time'][0] + form_filter = [x for x in resolution_time_stat['filters'] if x['id'] == 'form'][0] + assert form_filter['options'] == [{'id': 'test-1', 'label': 'test 1'}] + + def test_statistics_forms_count(pub): category_a = Category(name='Category A') category_a.store() @@ -762,3 +780,246 @@ def test_statistics_cards_count(pub): resp = get_app(pub).get(sign_uri('/api/statistics/cards/count/?card=%s' % 'invalid'), status=400) assert resp.text == 'invalid form' + + +def test_statistics_resolution_time(pub, freezer): + workflow = Workflow(name='Workflow One') + new_status = workflow.add_status(name='New status') + middle_status = workflow.add_status(name='Middle status') + workflow.add_status(name='End status') + workflow.add_status(name='End status 2') + + # add jump from new to end + jump = new_status.add_action('jump', id='_jump') + jump.status = '3' + + # add jump form new to middle and from middle to end 2 + jump = new_status.add_action('jump', id='_jump') + jump.status = '2' + jump = middle_status.add_action('jump', id='_jump') + jump.status = '4' + + workflow.store() + + formdef = FormDef() + formdef.name = 'test' + formdef.workflow_id = workflow.id + formdef.store() + + freezer.move_to(datetime.date(2021, 1, 1)) + formdata_list = [] + for i in range(3): + formdata = formdef.data_class()() + formdata.just_created() + formdata_list.append(formdata) + + # one formdata resolved in one day + freezer.move_to(datetime.date(2021, 1, 2)) + formdata_list[0].jump_status('3') + formdata_list[0].store() + + # one formdata resolved in two days, passing by middle status + formdata_list[1].jump_status('2') + freezer.move_to(datetime.date(2021, 1, 3)) + formdata_list[1].jump_status('4') + formdata_list[1].store() + + # one formdata blocked in middle status for three days + freezer.move_to(datetime.date(2021, 1, 4)) + formdata_list[2].jump_status('2') + formdata_list[2].store() + + # by default, count forms between initial status and final statuses + resp = get_app(pub).get(sign_uri('/api/statistics/resolution-time/?form=test')) + assert resp.json['data'] == { + 'series': [ + { + 'data': [86400.0, 172800.0, 129600.0, 129600.0], + 'label': 'Time between two statuses', + } + ], + 'subfilters': [ + { + 'id': 'start_status', + 'label': 'Start status', + 'options': [ + {'id': '1', 'label': 'New status'}, + {'id': '2', 'label': 'Middle status'}, + {'id': '3', 'label': 'End status'}, + {'id': '4', 'label': 'End status 2'}, + ], + 'required': True, + 'default': '1', + }, + { + 'default': 'done', + 'id': 'end_status', + 'label': 'End status', + 'options': [ + {'id': 'done', 'label': 'Any final status'}, + {'id': '2', 'label': 'Middle status'}, + {'id': '3', 'label': 'End status'}, + {'id': '4', 'label': 'End status 2'}, + ], + 'required': True, + }, + ], + 'x_labels': ['Minimum time', 'Maximum time', 'Mean', 'Median'], + } + + assert get_humanized_duration_serie(resp.json) == [ + '1 day(s) and 0 hour(s)', + '2 day(s) and 0 hour(s)', + '1 day(s) and 12 hour(s)', + '1 day(s) and 12 hour(s)', + ] + + # specify end status + resp = get_app(pub).get(sign_uri('/api/statistics/resolution-time/?form=test&end_status=3')) + assert get_humanized_duration_serie(resp.json) == [ + '1 day(s) and 0 hour(s)', + '1 day(s) and 0 hour(s)', + '1 day(s) and 0 hour(s)', + '1 day(s) and 0 hour(s)', + ] + + # specify start status + resp = get_app(pub).get(sign_uri('/api/statistics/resolution-time/?form=test&start_status=2')) + assert get_humanized_duration_serie(resp.json) == [ + '1 day(s) and 0 hour(s)', + '1 day(s) and 0 hour(s)', + '1 day(s) and 0 hour(s)', + '1 day(s) and 0 hour(s)', + ] + + # specify start and end statuses + resp = get_app(pub).get( + sign_uri('/api/statistics/resolution-time/?form=test&start_status=2&end_status=4') + ) + assert get_humanized_duration_serie(resp.json) == [ + '1 day(s) and 0 hour(s)', + '1 day(s) and 0 hour(s)', + '1 day(s) and 0 hour(s)', + '1 day(s) and 0 hour(s)', + ] + + resp = get_app(pub).get( + sign_uri('/api/statistics/resolution-time/?form=test&start_status=1&end_status=2') + ) + assert get_humanized_duration_serie(resp.json) == [ + '1 day(s) and 0 hour(s)', + '3 day(s) and 0 hour(s)', + '2 day(s) and 0 hour(s)', + '2 day(s) and 0 hour(s)', + ] + + # unknown statuses + default_resp = get_app(pub).get(sign_uri('/api/statistics/resolution-time/?form=test')) + resp = get_app(pub).get(sign_uri('/api/statistics/resolution-time/?form=test&start_status=42')) + assert resp.json == default_resp.json + + resp = get_app(pub).get(sign_uri('/api/statistics/resolution-time/?form=test&end_status=42')) + assert resp.json == default_resp.json + + # specify start and end statuses which does not match any formdata + resp = get_app(pub).get( + sign_uri('/api/statistics/resolution-time/?form=test&start_status=2&end_status=3') + ) + assert resp.json['data']['series'][0]['data'] == [] + + # unknown form + resp = get_app(pub).get(sign_uri('/api/statistics/resolution-time/?form=xxx'), status=400) + + +def test_statistics_resolution_time_median(pub, freezer): + workflow = Workflow(name='Workflow One') + new_status = workflow.add_status(name='New status') + workflow.add_status(name='End status') + jump = new_status.add_action('jump', id='_jump') + jump.status = '2' + workflow.store() + + formdef = FormDef() + formdef.name = 'test' + formdef.workflow_id = workflow.id + formdef.store() + + for i in range(2, 11): + formdata = formdef.data_class()() + freezer.move_to(datetime.date(2021, 1, 1)) + formdata.just_created() + + if i != 10: + # add lots of formdata resolved in a few days + freezer.move_to(datetime.date(2021, 1, i)) + else: + # one formdata took 3 months + freezer.move_to(datetime.date(2021, 4, 1)) + + formdata.jump_status('2') + formdata.store() + + resp = get_app(pub).get(sign_uri('/api/statistics/resolution-time/?form=test')) + assert get_humanized_duration_serie(resp.json) == [ + '1 day(s) and 0 hour(s)', # min + '89 day(s) and 22 hour(s)', # max + '13 day(s) and 23 hour(s)', # mean + '5 day(s) and 0 hour(s)', # median + ] + + +def test_statistics_resolution_time_start_end_filter(pub, freezer): + workflow = Workflow(name='Workflow One') + new_status = workflow.add_status(name='New status') + workflow.add_status(name='End status') + jump = new_status.add_action('jump', id='_jump') + jump.status = '2' + workflow.store() + + formdef = FormDef() + formdef.name = 'test' + formdef.workflow_id = workflow.id + formdef.store() + + # create formdata, the latest being the longest to resolve + for i in range(1, 10): + formdata = formdef.data_class()() + freezer.move_to(datetime.date(2021, 1, i)) + formdata.just_created() + freezer.move_to(datetime.date(2021, 1, i * 2)) + formdata.jump_status('2') + formdata.store() + + resp = get_app(pub).get(sign_uri('/api/statistics/resolution-time/?form=test')) + assert get_humanized_duration_serie(resp.json) == [ + '1 day(s) and 0 hour(s)', # min + '9 day(s) and 0 hour(s)', # max + '5 day(s) and 0 hour(s)', # mean + '5 day(s) and 0 hour(s)', # median + ] + + resp = get_app(pub).get(sign_uri('/api/statistics/resolution-time/?form=test&start=2021-01-05')) + assert get_humanized_duration_serie(resp.json) == [ + '5 day(s) and 0 hour(s)', # min + '9 day(s) and 0 hour(s)', # max + '7 day(s) and 0 hour(s)', # mean + '7 day(s) and 0 hour(s)', # median + ] + + resp = get_app(pub).get(sign_uri('/api/statistics/resolution-time/?form=test&end=2021-01-05')) + assert get_humanized_duration_serie(resp.json) == [ + '1 day(s) and 0 hour(s)', # min + '4 day(s) and 0 hour(s)', # max + '2 day(s) and 12 hour(s)', # mean + '2 day(s) and 12 hour(s)', # median + ] + + resp = get_app(pub).get( + sign_uri('/api/statistics/resolution-time/?form=test&start=2021-01-04&end=2021-01-05') + ) + assert get_humanized_duration_serie(resp.json) == [ + '4 day(s) and 0 hour(s)', # min + '4 day(s) and 0 hour(s)', # max + '4 day(s) and 0 hour(s)', # mean + '4 day(s) and 0 hour(s)', # median + ] diff --git a/wcs/statistics/views.py b/wcs/statistics/views.py index 4cde97a3d..4610691cb 100644 --- a/wcs/statistics/views.py +++ b/wcs/statistics/views.py @@ -15,6 +15,7 @@ # along with this program; if not, see . import collections +import time from django.http import HttpResponseBadRequest, HttpResponseForbidden, JsonResponse from django.urls import reverse @@ -29,7 +30,7 @@ from wcs.categories import Category from wcs.formdata import FormData from wcs.formdef import FormDef from wcs.qommon import _, misc, pgettext_lazy -from wcs.qommon.storage import Contains, Equal, Null, Or, StrictNotEqual +from wcs.qommon.storage import Contains, Equal, GreaterOrEqual, Less, Null, Or, StrictNotEqual class RestrictedView(View): @@ -155,6 +156,21 @@ class IndexView(RestrictedView): }, ], }, + { + 'name': _('Time between two statuses'), + 'url': request.build_absolute_uri(reverse('api-statistics-resolution-time')), + 'id': 'resolution_time', + 'data_type': 'seconds', + 'filters': [ + { + 'id': 'form', + 'label': _('Form'), + 'options': self.get_form_options(FormDef, include_all_option=False), + 'required': True, + 'has_subfilters': True, + }, + ], + }, ] } ) @@ -466,3 +482,113 @@ class CardsCountView(FormsCountView): formpage_class = CardPage has_global_count_support = False label = _('Cards Count') + + +class ResolutionTimeView(RestrictedView): + label = _('Time between two statuses') + + def get(self, request, *args, **kwargs): + formdef_slug = request.GET.get('form', '_nothing') + try: + formdef = FormDef.get_by_urlname(formdef_slug, ignore_migration=True) + except KeyError: + return HttpResponseBadRequest('invalid form') + + results = self.get_statistics(formdef) + return JsonResponse( + { + 'data': { + 'x_labels': [x[0] for x in results], + 'series': [{'label': _('Time between two statuses'), 'data': [x[1] for x in results]}], + 'subfilters': self.get_subfilters(formdef), + }, + 'err': 0, + } + ) + + @staticmethod + def get_subfilters(formdef): + status_options = [ + {'id': status.id, 'label': status.name} for status in formdef.workflow.possible_status + ] + + return [ + { + 'id': 'start_status', + 'label': _('Start status'), + 'options': status_options, + 'required': True, + 'default': status_options[0]['id'], + }, + { + 'id': 'end_status', + 'label': _('End status'), + 'options': [{'id': 'done', 'label': _('Any final status')}] + status_options[1:], + 'required': True, + 'default': 'done', + }, + ] + + def get_statistics(self, formdef): + criterias = [StrictNotEqual('status', 'draft')] + if self.request.GET.get('start'): + criterias.append(GreaterOrEqual('receipt_time', self.request.GET['start'])) + if self.request.GET.get('end'): + criterias.append(Less('receipt_time', self.request.GET['end'])) + + values = formdef.data_class().select(criterias) + # load all evolutions in a single batch, to avoid as many query as + # there are formdata when computing resolution times statistics. + formdef.data_class().load_all_evolutions(values) + + start_status = self.request.GET.get('start_status', formdef.workflow.possible_status[0].id) + end_status = self.request.GET.get('end_status', 'done') + + try: + start_status = formdef.workflow.get_status(start_status) + except KeyError: + start_status = formdef.workflow.possible_status[0] + + end_statuses = None + if end_status != 'done': + try: + end_statuses = {'wf-%s' % formdef.workflow.get_status(end_status).id} + except KeyError: + pass + + if not end_statuses: + end_statuses = {'wf-%s' % status.id for status in formdef.workflow.get_endpoint_status()} + + res_time_forms = [] + for filled in values: + start_time = None + for evo in filled.evolution or []: + if start_status and evo.status == 'wf-%s' % start_status.id: + start_time = time.mktime(evo.time) + elif evo.status in end_statuses: + if start_status and not start_time: + break + start_time = start_time or time.mktime(filled.receipt_time) + res_time_forms.append(time.mktime(evo.time) - start_time) + break + + if not res_time_forms: + return [] + res_time_forms.sort() + + sum_times = sum(res_time_forms) + len_times = len(res_time_forms) + mean = sum_times // len_times + + if len_times % 2: + median = res_time_forms[len_times // 2] + else: + midpt = len_times // 2 + median = (res_time_forms[midpt - 1] + res_time_forms[midpt]) // 2 + + return [ + (_('Minimum time'), res_time_forms[0]), + (_('Maximum time'), res_time_forms[-1]), + (_('Mean'), mean), + (_('Median'), median), + ] diff --git a/wcs/urls.py b/wcs/urls.py index c6617c829..b7615a567 100644 --- a/wcs/urls.py +++ b/wcs/urls.py @@ -61,6 +61,11 @@ urlpatterns = [ statistics_views.CardsCountView.as_view(), name='api-statistics-cards-count', ), + path( + 'api/statistics/resolution-time/', + statistics_views.ResolutionTimeView.as_view(), + name='api-statistics-resolution-time', + ), # provide django.contrib.auth view names for compatibility with # templates created for classic django applications. path('login/', compat.quixote, name='auth_login'), -- 2.35.1