From 94871812ba5a435f257dcae01354ee96c4c8ed54 Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Thu, 3 Nov 2022 18:12:44 +0100 Subject: [PATCH] statistics: add submission channel filter (#63376) --- tests/api/test_statistics.py | 55 +++++++++++++++++++++++++++++++++++- wcs/sql.py | 38 +++++++++++++++++-------- wcs/statistics/views.py | 39 +++++++++++++++++++++++-- 3 files changed, 118 insertions(+), 14 deletions(-) diff --git a/tests/api/test_statistics.py b/tests/api/test_statistics.py index 64379e3f3..b8dbda60e 100644 --- a/tests/api/test_statistics.py +++ b/tests/api/test_statistics.py @@ -190,6 +190,19 @@ def test_statistics_index_forms(pub): ], ] + resp = get_app(pub).get(sign_uri('/api/statistics/')) + form_filter = [x for x in resp.json['data'][0]['filters'] if x['id'] == 'channel'][0] + assert form_filter['options'] == [ + {'id': '_all', 'label': 'All'}, + {'id': 'mail', 'label': 'Mail'}, + {'id': 'email', 'label': 'Email'}, + {'id': 'phone', 'label': 'Phone'}, + {'id': 'counter', 'label': 'Counter'}, + {'id': 'fax', 'label': 'Fax'}, + {'id': 'web', 'label': 'Web'}, + {'id': 'social-network', 'label': 'Social Network'}, + ] + def test_statistics_index_cards(pub): carddef = CardDef() @@ -253,16 +266,24 @@ def test_statistics_forms_count(pub): formdef2.store() formdef2.data_class().wipe() - for _i in range(20): + for i in range(20): formdata = formdef.data_class()() formdata.just_created() formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple() + # "Web" channel has three equivalent values + if i == 0: + formdata.submission_channel = 'web' + elif i == 1: + formdata.submission_channel = '' + else: + formdata.submission_channel = None formdata.store() for _i in range(30): formdata = formdef2.data_class()() formdata.just_created() formdata.receipt_time = datetime.datetime(2021, 3, 1, 2, 0).timetuple() + formdata.submission_channel = 'mail' formdata.store() # draft should not be counted @@ -363,6 +384,16 @@ def test_statistics_forms_count(pub): 'err': 0, } + # apply channel filter + resp = get_app(pub).get(sign_uri('/api/statistics/forms/count/?channel=mail')) + assert resp.json['data']['series'] == [{'data': [30], 'label': 'Forms Count'}] + + resp = get_app(pub).get(sign_uri('/api/statistics/forms/count/?channel=web')) + assert resp.json['data']['series'] == [{'data': [20], 'label': 'Forms Count'}] + + resp = get_app(pub).get(sign_uri('/api/statistics/forms/count/?channel=_all')) + assert resp.json['data']['series'] == [{'data': [20, 0, 30], 'label': 'Forms Count'}] + def test_statistics_forms_count_subfilters(pub, formdef): for i in range(2): @@ -382,6 +413,7 @@ def test_statistics_forms_count_subfilters(pub, formdef): 'id': 'group-by', 'label': 'Group by', 'options': [ + {'id': 'channel', 'label': 'Channel'}, {'id': 'test-item', 'label': 'Test item'}, {'id': 'test-items', 'label': 'Test items'}, {'id': 'checkbox', 'label': 'Checkbox'}, @@ -600,11 +632,19 @@ def test_statistics_forms_count_group_by(pub, formdef, anonymise): formdata.data['2_display'] = 'Foo' formdata.data['3'] = ['bar', 'baz'] formdata.data['3_display'] = 'Bar, Baz' + # "Web" channel has three equivalent values + if i == 1: + formdata.submission_channel = 'web' + elif i == 2: + formdata.submission_channel = '' + else: + formdata.submission_channel = None elif i % 2: formdata.data['1'] = False formdata.data['2'] = 'baz' formdata.data['3'] = ['baz'] formdata.jump_status('2') + formdata.submission_channel = 'mail' else: formdata.receipt_time = datetime.datetime(2021, 3, 1, 2, 0).timetuple() formdata.store() @@ -670,11 +710,24 @@ def test_statistics_forms_count_group_by(pub, formdef, anonymise): {'data': [13, None, 4], 'label': 'New status'}, ] + # group by channel + resp = get_app(pub).get(sign_uri(url + '&group-by=channel')) + assert resp.json['data']['x_labels'] == ['2021-01', '2021-02', '2021-03'] + assert resp.json['data']['series'] == [ + {'data': [3, None, None], 'label': 'Mail'}, + {'data': [13, None, 4], 'label': 'Web'}, + ] + # group by item field without time interval resp = get_app(pub).get(sign_uri(url + '&group-by=test-item&time_interval=none')) assert resp.json['data']['x_labels'] == ['baz', 'Foo', 'None'] assert resp.json['data']['series'] == [{'data': [3, 13, 4], 'label': 'Forms Count'}] + # group by submission channel without time interval + resp = get_app(pub).get(sign_uri(url + '&group-by=channel&time_interval=none')) + assert resp.json['data']['x_labels'] == ['Mail', 'Web'] + assert resp.json['data']['series'] == [{'data': [3, 17], 'label': 'Forms Count'}] + # group by on block field is not supported resp = get_app(pub).get(sign_uri(url + '&group-by=blockdata_bool')) assert resp.json['data']['series'] == [{'data': [16, 0, 4], 'label': 'Forms Count'}] diff --git a/wcs/sql.py b/wcs/sql.py index f4f26d3aa..e68f4fac7 100644 --- a/wcs/sql.py +++ b/wcs/sql.py @@ -4162,10 +4162,15 @@ def get_period_query( return statement -def get_time_aggregate_query(time_interval, query, group_by, function='DATE_TRUNC'): +def get_time_aggregate_query(time_interval, query, group_by, function='DATE_TRUNC', null_values=None): statement = f"SELECT {function}('{time_interval}', receipt_time) AS {time_interval}, " if group_by: - statement += '%s, ' % group_by + if null_values: + statement += ( + f'CASE WHEN {group_by} IN {null_values} THEN null ELSE {group_by} END as {group_by}, ' + ) + else: + statement += '%s, ' % group_by statement += 'COUNT(*) ' statement += query @@ -4220,13 +4225,15 @@ def get_total_counts(user_roles): @guard_postgres -def get_weekday_totals(period_start=None, period_end=None, criterias=None, group_by=None): +def get_weekday_totals(period_start=None, period_end=None, criterias=None, group_by=None, null_values=None): conn, cur = get_connection_and_cursor() parameters = {} statement = get_period_query( period_start=period_start, period_end=period_end, criterias=criterias, parameters=parameters ) - statement = get_time_aggregate_query('dow', statement, group_by, function='DATE_PART') + statement = get_time_aggregate_query( + 'dow', statement, group_by, function='DATE_PART', null_values=null_values + ) cur.execute(statement, parameters) result = cur.fetchall() @@ -4278,11 +4285,17 @@ def get_formdef_totals(period_start=None, period_end=None, criterias=None): @guard_postgres -def get_global_totals(period_start=None, period_end=None, criterias=None, group_by=None): +def get_global_totals(period_start=None, period_end=None, criterias=None, group_by=None, null_values=None): conn, cur = get_connection_and_cursor() statement = 'SELECT ' if group_by: - statement += f'{group_by}, ' + if null_values: + statement += ( + f'CASE WHEN {group_by} IN {null_values} THEN null ELSE {group_by} END as {group_by}_new, ' + ) + group_by += '_new' + else: + statement += f'{group_by}, ' statement += 'COUNT(*) ' parameters = {} @@ -4304,13 +4317,15 @@ def get_global_totals(period_start=None, period_end=None, criterias=None, group_ @guard_postgres -def get_hour_totals(period_start=None, period_end=None, criterias=None, group_by=None): +def get_hour_totals(period_start=None, period_end=None, criterias=None, group_by=None, null_values=None): conn, cur = get_connection_and_cursor() parameters = {} statement = get_period_query( period_start=period_start, period_end=period_end, criterias=criterias, parameters=parameters ) - statement = get_time_aggregate_query('hour', statement, group_by, function='DATE_PART') + statement = get_time_aggregate_query( + 'hour', statement, group_by, function='DATE_PART', null_values=null_values + ) cur.execute(statement, parameters) result = cur.fetchall() @@ -4334,13 +4349,14 @@ def get_monthly_totals( period_end=None, criterias=None, group_by=None, + null_values=None, ): conn, cur = get_connection_and_cursor() parameters = {} statement = get_period_query( period_start=period_start, period_end=period_end, criterias=criterias, parameters=parameters ) - statement = get_time_aggregate_query('month', statement, group_by) + statement = get_time_aggregate_query('month', statement, group_by, null_values=null_values) cur.execute(statement, parameters) raw_result = cur.fetchall() @@ -4364,13 +4380,13 @@ def get_monthly_totals( @guard_postgres -def get_yearly_totals(period_start=None, period_end=None, criterias=None, group_by=None): +def get_yearly_totals(period_start=None, period_end=None, criterias=None, group_by=None, null_values=None): conn, cur = get_connection_and_cursor() parameters = {} statement = get_period_query( period_start=period_start, period_end=period_end, criterias=criterias, parameters=parameters ) - statement = get_time_aggregate_query('year', statement, group_by) + statement = get_time_aggregate_query('year', statement, group_by, null_values=null_values) cur.execute(statement, parameters) raw_result = cur.fetchall() diff --git a/wcs/statistics/views.py b/wcs/statistics/views.py index c018d8a63..00dc08833 100644 --- a/wcs/statistics/views.py +++ b/wcs/statistics/views.py @@ -26,9 +26,10 @@ from wcs.backoffice.data_management import CardPage from wcs.backoffice.management import FormPage from wcs.carddef import CardDef from wcs.categories import Category +from wcs.formdata import FormData from wcs.formdef import FormDef from wcs.qommon import _, misc, pgettext_lazy -from wcs.qommon.storage import Contains, Equal, StrictNotEqual +from wcs.qommon.storage import Contains, Equal, Null, Or, StrictNotEqual class RestrictedView(View): @@ -45,6 +46,9 @@ class IndexView(RestrictedView): category_options = [{'id': '_all', 'label': pgettext_lazy('categories', 'All')}] + [ {'id': x.url_name, 'label': x.name} for x in categories ] + channel_options = [{'id': '_all', 'label': pgettext_lazy('channel', 'All')}] + [ + {'id': key, 'label': label} for key, label in FormData.get_submission_channels().items() + ] return JsonResponse( { 'data': [ @@ -81,6 +85,13 @@ class IndexView(RestrictedView): 'required': True, 'default': 'month', }, + { + 'id': 'channel', + 'label': _('Channel'), + 'options': channel_options, + 'required': True, + 'default': '_all', + }, { 'id': 'category', 'label': _('Category'), @@ -234,6 +245,20 @@ class FormsCountView(RestrictedView): else: totals_kwargs['criterias'].append(Equal('category_id', category.id)) + channel = request.GET.get('channel', '_all') + if channel == 'web': + totals_kwargs['criterias'].append( + Or( + [ + Equal('submission_channel', 'web'), + Equal('submission_channel', ''), + Null('submission_channel'), + ] + ) + ) + elif channel != '_all': + totals_kwargs['criterias'].append(Equal('submission_channel', channel)) + time_interval_methods = { 'month': sql.get_monthly_totals, 'year': sql.get_yearly_totals, @@ -333,7 +358,8 @@ class FormsCountView(RestrictedView): { 'id': 'group-by', 'label': _('Group by'), - 'options': [{'id': x[0], 'label': x[1]} for x in field_choices], + 'options': [{'id': 'channel', 'label': _('Channel')}] + + [{'id': x[0], 'label': x[1]} for x in field_choices], }, ) @@ -366,6 +392,15 @@ class FormsCountView(RestrictedView): if not group_by: return + if group_by == 'channel': + totals_kwargs['group_by'] = 'submission_channel' + totals_kwargs['null_values'] = ('web', '') + + group_labels.update(FormData.get_submission_channels()) + group_labels[None] = _('Web') + group_labels[''] = _('Web') + return + group_by_field = self.get_group_by_field(form_page, group_by) if not group_by_field: return -- 2.35.1