From adc0effbba9a15480416bdcf8bd785d438270c77 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20P=C3=A9ters?= <fpeters@entrouvert.com>
Date: Sun, 4 Apr 2021 15:11:08 +0200
Subject: [PATCH] api: add statistics API for direct combo usage (#52731)

---
 tests/api/test_statistics.py | 155 +++++++++++++++++++++++++++++++++++
 wcs/backoffice/management.py |  18 +---
 wcs/sql.py                   |  16 +++-
 wcs/statistics/__init__.py   |   0
 wcs/statistics/views.py      | 128 +++++++++++++++++++++++++++++
 wcs/urls.py                  |   7 ++
 6 files changed, 306 insertions(+), 18 deletions(-)
 create mode 100644 tests/api/test_statistics.py
 create mode 100644 wcs/statistics/__init__.py
 create mode 100644 wcs/statistics/views.py

diff --git a/tests/api/test_statistics.py b/tests/api/test_statistics.py
new file mode 100644
index 000000000..ff10472c9
--- /dev/null
+++ b/tests/api/test_statistics.py
@@ -0,0 +1,155 @@
+import datetime
+import os
+
+import pytest
+
+from wcs.categories import Category
+from wcs.formdef import FormDef
+from wcs.qommon.http_request import HTTPRequest
+
+from ..utilities import clean_temporary_pub, create_temporary_pub, get_app
+from .utils import sign_uri
+
+
+@pytest.fixture
+def pub():
+    pub = create_temporary_pub(sql_mode=True)
+    Category.wipe()
+    FormDef.wipe()
+
+    req = HTTPRequest(None, {'SCRIPT_NAME': '/', 'SERVER_NAME': 'example.net'})
+    pub.set_app_dir(req)
+    pub.cfg['identification'] = {'methods': ['password']}
+    pub.cfg['language'] = {'language': 'en'}
+    pub.write_cfg()
+
+    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
+        fd.write(
+            '''\
+[api-secrets]
+coucou = 1234
+'''
+        )
+
+    return pub
+
+
+def teardown_module(module):
+    clean_temporary_pub()
+
+
+def test_statistics_index(pub):
+    get_app(pub).get('/api/statistics/', status=403)
+    resp = get_app(pub).get(sign_uri('/api/statistics/'))
+    assert resp.json['data'][0]['name'] == 'Forms Count'
+    assert resp.json['data'][0]['url'] == 'http://example.net/api/statistics/forms/count/'
+
+
+def test_statistics_index_no_sql(pub):
+    pub.is_using_postgresql = lambda: False
+    assert get_app(pub).get(sign_uri('/api/statistics/')).json == {'data': [], 'err': 0}
+
+
+def test_statistics_index_categories(pub):
+    Category(name='Category A').store()
+    Category(name='Category B').store()
+    resp = get_app(pub).get(sign_uri('/api/statistics/'))
+    category_filter = [x for x in resp.json['data'][0]['filters'] if x['id'] == 'category'][0]
+    assert len(category_filter['options']) == 3
+
+
+def test_statistics_forms_count(pub):
+    category_a = Category(name='Category A')
+    category_a.store()
+    category_b = Category(name='Category B')
+    category_b.store()
+
+    formdef = FormDef()
+    formdef.name = 'test 1'
+    formdef.category_id = category_a.id
+    formdef.fields = []
+    formdef.store()
+    formdef.data_class().wipe()
+
+    formdef2 = FormDef()
+    formdef2.name = 'test 2'
+    formdef2.category_id = category_b.id
+    formdef2.fields = []
+    formdef2.store()
+    formdef2.data_class().wipe()
+
+    for _i in range(20):
+        formdata = formdef.data_class()()
+        formdata.just_created()
+        formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+        formdata.store()
+
+    for _i in range(30):
+        formdata = formdef2.data_class()()
+        formdata.just_created()
+        formdata.receipt_time = datetime.datetime(2021, 3, 1, 2, 0).timetuple()
+        formdata.store()
+
+    resp = get_app(pub).get(sign_uri('/api/statistics/forms/count/'))
+    assert resp.json == {
+        'data': {
+            'series': [{'data': [20, 0, 30], 'label': 'Forms Count'}],
+            'x_labels': ['2021-01', '2021-02', '2021-03'],
+        },
+        'err': 0,
+    }
+
+    resp = get_app(pub).get(sign_uri('/api/statistics/forms/count/?time_interval=year'))
+    assert resp.json == {
+        'data': {
+            'series': [{'data': [50], 'label': 'Forms Count'}],
+            'x_labels': ['2021'],
+        },
+        'err': 0,
+    }
+
+    resp = get_app(pub).get(sign_uri('/api/statistics/forms/count/?time_interval=weekday'))
+    assert resp.json == {
+        'data': {
+            'series': [{'data': [30, 0, 0, 0, 20, 0, 0], 'label': 'Forms Count'}],
+            'x_labels': ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday'],
+        },
+        'err': 0,
+    }
+
+    resp = get_app(pub).get(sign_uri('/api/statistics/forms/count/?time_interval=hour'))
+    assert resp.json == {
+        'data': {
+            'series': [
+                {
+                    'label': 'Forms Count',
+                    'data': [20, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+                }
+            ],
+            'x_labels': list(range(24)),
+        },
+        'err': 0,
+    }
+
+    # time_interval=day is not supported
+    get_app(pub).get(sign_uri('/api/statistics/forms/count/?time_interval=day'), status=400)
+
+    # apply category filter
+    resp = get_app(pub).get(sign_uri('/api/statistics/forms/count/?category=%s' % category_a.id))
+    assert resp.json == {
+        'data': {
+            'series': [{'data': [20], 'label': 'Forms Count'}],
+            'x_labels': ['2021-01'],
+        },
+        'err': 0,
+    }
+
+    # apply period filter
+    resp = get_app(pub).get(sign_uri('/api/statistics/forms/count/?end=2021-02-01'))
+    assert resp.json == {
+        'data': {
+            'series': [{'data': [20], 'label': 'Forms Count'}],
+            'x_labels': ['2021-01'],
+        },
+        'err': 0,
+    }
diff --git a/wcs/backoffice/management.py b/wcs/backoffice/management.py
index 198bc5bac..29493dd58 100644
--- a/wcs/backoffice/management.py
+++ b/wcs/backoffice/management.py
@@ -3455,22 +3455,6 @@ def do_graphs_section(period_start=None, period_end=None, criterias=None):
         yearly_totals = [(datetime.date.today().year, 0)]
 
     weekday_totals = sql.get_weekday_totals(period_start, period_end, criterias)
-    weekday_line = []
-    weekday_names = [
-        _('Sunday'),
-        _('Monday'),
-        _('Tuesday'),
-        _('Wednesday'),
-        _('Thursday'),
-        _('Friday'),
-        _('Saturday'),
-    ]
-    for weekday, total in weekday_totals:
-        label = weekday_names[weekday]
-        weekday_line.append((label, total))
-    # move Sunday to the last place
-    weekday_line = weekday_line[1:] + [weekday_line[0]]
-
     hour_totals = sql.get_hour_totals(period_start, period_end, criterias)
 
     r += htmltext(
@@ -3481,7 +3465,7 @@ var month_line = %(month_line)s;
 var year_line = %(year_line)s;
 </script>'''
         % {
-            'weekday_line': json.dumps(weekday_line),
+            'weekday_line': json.dumps(weekday_totals),
             'hour_line': json.dumps(hour_totals),
             'month_line': json.dumps(monthly_totals),
             'year_line': json.dumps(yearly_totals),
diff --git a/wcs/sql.py b/wcs/sql.py
index 83f66b6f8..dc0c98abd 100644
--- a/wcs/sql.py
+++ b/wcs/sql.py
@@ -46,7 +46,7 @@ from wcs.qommon import PICKLE_KWARGS, force_str
 
 from . import qommon
 from .publisher import UnpicklerClass
-from .qommon import get_cfg
+from .qommon import _, get_cfg
 from .qommon.misc import strftime
 from .qommon.storage import _take, deep_bytes2str
 from .qommon.storage import parse_clause as parse_storage_clause
@@ -3178,6 +3178,20 @@ def get_weekday_totals(period_start=None, period_end=None, criterias=None):
             result.append((weekday, 0))
     result.sort()
 
+    # add labels,
+    weekday_names = [
+        _('Sunday'),
+        _('Monday'),
+        _('Tuesday'),
+        _('Wednesday'),
+        _('Thursday'),
+        _('Friday'),
+        _('Saturday'),
+    ]
+    result = [(weekday_names[x], y) for (x, y) in result]
+    # and move Sunday last
+    result = result[1:] + [result[0]]
+
     conn.commit()
     cur.close()
 
diff --git a/wcs/statistics/__init__.py b/wcs/statistics/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/wcs/statistics/views.py b/wcs/statistics/views.py
new file mode 100644
index 000000000..a0fa5e22e
--- /dev/null
+++ b/wcs/statistics/views.py
@@ -0,0 +1,128 @@
+# w.c.s. - web application for online forms
+# Copyright (C) 2005-2021  Entr'ouvert
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <http://www.gnu.org/licenses/>.
+
+from django.http import HttpResponseBadRequest, HttpResponseForbidden, JsonResponse
+from django.urls import reverse
+from django.views.generic import View
+from quixote import get_publisher
+
+from wcs.api_utils import is_url_signed
+from wcs.categories import Category
+from wcs.qommon import _, misc
+from wcs.qommon.misc import C_
+from wcs.qommon.storage import Equal
+
+
+class RestrictedView(View):
+    def dispatch(self, *args, **kwargs):
+        if not is_url_signed():
+            return HttpResponseForbidden()
+        return super().dispatch(*args, **kwargs)
+
+
+class IndexView(RestrictedView):
+    def get(self, request, *args, **kwargs):
+        if not get_publisher().is_using_postgresql():
+            return JsonResponse({'data': [], 'err': 0})
+        categories = Category.select()
+        categories.sort(key=lambda x: misc.simplify(x.name))
+        category_options = [{'id': '_all', 'label': C_('categories|All')}] + [
+            {'id': x.id, 'label': x.name} for x in categories
+        ]
+        return JsonResponse(
+            {
+                'data': [
+                    {
+                        'name': _('Forms Count'),
+                        'url': request.build_absolute_uri(reverse('api-statistics-forms-count')),
+                        'id': 'forms_counts',
+                        'filters': [
+                            {
+                                'id': 'time_interval',
+                                'label': _('Interval'),
+                                'options': [
+                                    {
+                                        'id': 'month',
+                                        'label': _('Month'),
+                                    },
+                                    {
+                                        'id': 'year',
+                                        'label': _('Year'),
+                                    },
+                                    {
+                                        'id': 'weekday',
+                                        'label': _('Week day'),
+                                    },
+                                    {
+                                        'id': 'hour',
+                                        'label': _('Hour'),
+                                    },
+                                ],
+                                'required': True,
+                                'default': 'month',
+                            },
+                            {
+                                'id': 'category',
+                                'label': _('Category'),
+                                'options': category_options,
+                                'required': False,
+                                'default': '_all',
+                            },
+                        ],
+                    }
+                ]
+            }
+        )
+
+
+class FormsCountView(RestrictedView):
+    def get(self, request, *args, **kwargs):
+        from wcs import sql
+
+        time_interval = request.GET.get('time_interval', 'month')
+        totals_kwargs = {
+            'period_start': request.GET.get('start'),
+            'period_end': request.GET.get('end'),
+            'criterias': [],
+        }
+        category_id = request.GET.get('category')
+        if category_id and category_id != '_all':
+            totals_kwargs['criterias'].append(Equal('category_id', category_id))
+        time_interval_methods = {
+            'month': sql.get_monthly_totals,
+            'year': sql.get_yearly_totals,
+            'weekday': sql.get_weekday_totals,
+            'hour': sql.get_hour_totals,
+        }
+        if time_interval in time_interval_methods:
+            totals = time_interval_methods[time_interval](**totals_kwargs)
+        else:
+            return HttpResponseBadRequest('invalid time_interval parameter')
+
+        return JsonResponse(
+            {
+                'data': {
+                    'x_labels': [x[0] for x in totals],
+                    'series': [
+                        {
+                            'label': _('Forms Count'),
+                            'data': [x[1] for x in totals],
+                        }
+                    ],
+                },
+                'err': 0,
+            }
+        )
diff --git a/wcs/urls.py b/wcs/urls.py
index 7e7528d24..7bb10c78b 100644
--- a/wcs/urls.py
+++ b/wcs/urls.py
@@ -17,6 +17,7 @@
 from django.conf.urls import url
 
 from . import api, compat, views
+from .statistics import views as statistics_views
 
 urlpatterns = [
     url(r'^robots.txt$', views.robots_txt),
@@ -26,6 +27,12 @@ urlpatterns = [
     url(r'^api/validate-expression$', api.validate_expression, name='api-validate-expression'),
     url(r'^api/reverse-geocoding$', api.reverse_geocoding, name='api-reverse-geocoding'),
     url(r'^api/geocoding$', api.geocoding, name='api-geocoding'),
+    url(r'^api/statistics/$', statistics_views.IndexView.as_view()),
+    url(
+        r'^api/statistics/forms/count/$',
+        statistics_views.FormsCountView.as_view(),
+        name='api-statistics-forms-count',
+    ),
     # provide django.contrib.auth view names for compatibility with
     # templates created for classic django applications.
     url(r'^login/$', compat.quixote, name='auth_login'),
-- 
2.31.0

