From 279cdc0d21ba8dff344e2bfbc5a38a6b9eccd354 Mon Sep 17 00:00:00 2001
From: Nicolas ROCHE <nroche@entrouvert.com>
Date: Mon, 18 Mar 2019 20:02:26 +0100
Subject: [PATCH 3/4] matomo: views form manual and automatic configuration
 (#31778)

---
 hobo/matomo/__init__.py                       |   0
 hobo/matomo/forms.py                          |  37 +++++
 .../matomo/templates/hobo/matomo_disable.html |  20 +++
 .../templates/hobo/matomo_enable_auto.html    |  20 +++
 .../templates/hobo/matomo_enable_manual.html  |  20 +++
 hobo/matomo/templates/hobo/matomo_home.html   | 114 ++++++++++++++
 hobo/matomo/urls.py                           |  26 ++++
 hobo/matomo/views.py                          | 125 ++++++++++++++++
 hobo/settings.py                              |   1 +
 hobo/templates/hobo/home.html                 |   1 +
 hobo/urls.py                                  |   3 +
 tests/test_matomo_views.py                    | 141 ++++++++++++++++++
 12 files changed, 508 insertions(+)
 create mode 100644 hobo/matomo/__init__.py
 create mode 100644 hobo/matomo/forms.py
 create mode 100644 hobo/matomo/templates/hobo/matomo_disable.html
 create mode 100644 hobo/matomo/templates/hobo/matomo_enable_auto.html
 create mode 100644 hobo/matomo/templates/hobo/matomo_enable_manual.html
 create mode 100644 hobo/matomo/templates/hobo/matomo_home.html
 create mode 100644 hobo/matomo/urls.py
 create mode 100644 hobo/matomo/views.py
 create mode 100644 tests/test_matomo_views.py

diff --git a/hobo/matomo/__init__.py b/hobo/matomo/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/hobo/matomo/forms.py b/hobo/matomo/forms.py
new file mode 100644
index 0000000..0f45b25
--- /dev/null
+++ b/hobo/matomo/forms.py
@@ -0,0 +1,37 @@
+# hobo - portal to configure and deploy applications
+# Copyright (C) 2015-2019  Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from django import forms
+from django.utils.translation import ugettext_lazy as _
+
+
+class SettingsForm(forms.Form):
+    """
+    According to publik-base-theme/templates/includes/tracking.html,
+    the 2 tracking_js variables are merged into the unique below field.
+    If JS code added is compliant to CNIL policy, we store it into
+    'cnil_compliant_visits_tracking_js' else into 'visits_tracking_js'.
+    The goal is to display a banner advertising users about intrusive JS.
+    """
+    tracking_js = forms.CharField(
+        label=_('Tracking Javascript'),
+        help_text=_('See <a href="https://developer.matomo.org/guides/tracking-javascript-guide">Matomo partners site</a> for getting your Javascript\'s tracking code.'),
+        required=False,
+        widget=forms.Textarea(attrs={'size': 1024}))
+
+
+class EnableForm(forms.Form):
+    pass
diff --git a/hobo/matomo/templates/hobo/matomo_disable.html b/hobo/matomo/templates/hobo/matomo_disable.html
new file mode 100644
index 0000000..f35a0d4
--- /dev/null
+++ b/hobo/matomo/templates/hobo/matomo_disable.html
@@ -0,0 +1,20 @@
+{% extends "hobo/matomo_home.html" %}
+{% load i18n %}
+
+{% block appbar %}
+  <h2>{% trans "User tracking" %}</h2>
+{% endblock %}
+
+{% block content %}
+<form method="post">
+{% csrf_token %}
+<p>
+{% trans "Are you sure you want to disable user tracking support?" %}
+{{ form.as_p }}
+<div class="buttons">
+  <button class="submit-button">{% trans "Disable" %}</button>
+  <a class="cancel" href="{% url 'matomo-home' %}">{% trans "Cancel" %}</a>
+</div>
+</form>
+
+{% endblock %}
diff --git a/hobo/matomo/templates/hobo/matomo_enable_auto.html b/hobo/matomo/templates/hobo/matomo_enable_auto.html
new file mode 100644
index 0000000..f0ba07a
--- /dev/null
+++ b/hobo/matomo/templates/hobo/matomo_enable_auto.html
@@ -0,0 +1,20 @@
+{% extends "hobo/matomo_home.html" %}
+{% load i18n %}
+
+{% block appbar %}
+  <h2>{% trans "User tracking" %}</h2>
+{% endblock %}
+
+{% block content %}
+<form method="post">
+{% csrf_token %}
+<p>
+{% trans "Are you sure you want to enable automatic user tracking support?" %}
+{{ form.as_p }}
+<div class="buttons">
+<button class="submit-button">{% trans "Enable" %}</button>
+<a class="cancel" href="{% url 'matomo-home' %}">{% trans "Cancel" %}</a>
+</div>
+</form>
+
+{% endblock %}
diff --git a/hobo/matomo/templates/hobo/matomo_enable_manual.html b/hobo/matomo/templates/hobo/matomo_enable_manual.html
new file mode 100644
index 0000000..7b2f7dc
--- /dev/null
+++ b/hobo/matomo/templates/hobo/matomo_enable_manual.html
@@ -0,0 +1,20 @@
+{% extends "hobo/matomo_home.html" %}
+{% load i18n %}
+
+{% block appbar %}
+  <h2>{% trans "User tracking" %}</h2>
+{% endblock %}
+
+{% block content %}
+<form method="post">
+{% csrf_token %}
+<p>
+{% trans "Are you sure you want to enable manual user tracking support?" %}
+{{ form.as_p }}
+<div class="buttons">
+<button class="submit-button">{% trans "Enable" %}</button>
+<a class="cancel" href="{% url 'matomo-home' %}">{% trans "Cancel" %}</a>
+</div>
+</form>
+
+{% endblock %}
diff --git a/hobo/matomo/templates/hobo/matomo_home.html b/hobo/matomo/templates/hobo/matomo_home.html
new file mode 100644
index 0000000..aa95949
--- /dev/null
+++ b/hobo/matomo/templates/hobo/matomo_home.html
@@ -0,0 +1,114 @@
+{% extends "hobo/base.html" %}
+{% load i18n %}
+
+{% block breadcrumb %}
+{{ block.super }}
+<a href="{% url 'matomo-home' %}">Matomo</a>
+{% endblock %}
+
+{% block appbar %}
+<h2>{% trans "User tracking" %}</h2>
+{% if enabled %}
+<span class="actions">
+  <a rel="popup" href="{% url 'matomo-disable' %}">
+    {% trans 'Disable' %}
+  </a>
+  {% if mode == 'auto' %}
+  <a href={{matomo_url}}/index.php?module=Login&action=logme&login={{matomo_user_login}}&password={{matomo_password}}>
+    {% trans "Open visits tracking dashboard" %}
+  </a>
+  {% endif %}
+</span>
+{% endif %}
+{% endblock %}
+
+{% block content %}
+
+<div class="infonotice">
+  {% if not enabled %}
+  {% blocktrans %}
+  <p>
+    Matomo is the tracker solution recommended by the National Commission for Data Protection and Liberties (<a href=https://www.cnil.fr/fr/solutions-pour-les-cookies-de-mesure-daudience>CNIL-France</a>).
+  </p>
+  <p>
+    The audience measurement tools are deployed to obtain information about visitor navigation. They help to understand where users come from on a site and reconstruct their browsing activity. These tools use technologies that permit to trace users on your site and associate a \"referrer\" or campaign with a unique identifier.
+  </p>
+  The Matomo tool requires little setting to be exempt from legal consent:
+  {% endblocktrans %}
+  <ul>
+    <li>{% trans 'No life time extension on cookies' %}
+      <a href=https://www.cnil.fr/sites/default/files/typo/document/Configuration_piwik.pdf>
+	...
+      </a>
+    </li>
+    <li>{% trans 'Anonymize the IP address' %}
+      {% if matomo_ws_available %}
+      <a href={{matomo_url}}/index.php?module=PrivacyManager&action=privacySettings>
+	...
+      </a>
+      {% endif %}
+    </li>
+
+    <li>{% trans 'Offer a solution against the deposit of cookies' %}
+      {% if matomo_ws_available %}
+      <a href={{matomo_url}}/index.php?module=PrivacyManager&action=usersOptOut>
+	...
+      </a>
+      {% endif %}
+    </li>
+  </ul>
+  {% else %}
+  {% if mode == 'manual' %}
+  {% trans "Manual mode" %}
+  {% elif mode == 'auto' %}
+  {% trans "Automatic mode" %}
+  {% endif %}
+  {% endif %}
+</div>
+
+{% if not enabled %}
+<p>
+  {% if error != '' %}
+  <div class="errornotice">
+    {{ error }}
+  </div>
+  {% endif %}
+  {% trans "Support is currently disabled." %}
+</p>
+<p>
+  <a class="button" rel="popup" href="{% url 'matomo-enable-manual' %}">{% trans 'Manual' %}</a>
+  {% if matomo_ws_available %}
+  <a class="button" rel="popup" href="{% url 'matomo-enable-auto' %}">{% trans 'Automatic' %}</a>
+  {% endif %}
+</p>
+{% else %}
+
+{% if tracking_js != '' %}
+{% if cnil_ack_level == 'success' %}
+<div class="successnotice">
+  {% trans "Excellent respect of user rights" %}
+</div>
+{% elif cnil_ack_level == 'warning' %}
+<div class="warningnotice">
+  {% trans "Good respect of user rights" %}
+</div>
+{% elif cnil_ack_level == 'error' %}
+<div class="errornotice">
+  {% trans "No respect of user rights" %}
+</div>
+{% endif %}
+{% endif %}
+
+{% if mode == 'manual' %}
+<form method="post">
+  {% csrf_token %}
+  {{ form.as_p }}
+
+  <div class="buttons">
+    <button class="submit-button">{% trans "Save" %}</button>
+  </div>
+</form>
+{% endif %}
+
+{% endif %}
+{% endblock %}
diff --git a/hobo/matomo/urls.py b/hobo/matomo/urls.py
new file mode 100644
index 0000000..e070c8d
--- /dev/null
+++ b/hobo/matomo/urls.py
@@ -0,0 +1,26 @@
+# hobo - portal to configure and deploy applications
+# Copyright (C) 2015-2019  Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from django.conf.urls import url
+
+from . import views
+
+urlpatterns = [
+    url(r'^$', views.home, name='matomo-home'),
+    url(r'^enable-manual$', views.enable_manual, name='matomo-enable-manual'),
+    url(r'^enable-auto$', views.enable_auto, name='matomo-enable-auto'),
+    url(r'^disable$', views.disable, name='matomo-disable'),
+]
diff --git a/hobo/matomo/views.py b/hobo/matomo/views.py
new file mode 100644
index 0000000..9509a71
--- /dev/null
+++ b/hobo/matomo/views.py
@@ -0,0 +1,125 @@
+# hobo - portal to configure and deploy applications
+# Copyright (C) 2015-2019  Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import hashlib
+
+from django.core.urlresolvers import reverse_lazy
+from django.conf import settings
+from django.views.generic import RedirectView, FormView
+
+from .forms import SettingsForm, EnableForm
+from .utils import get_variable, get_tracking_js, put_tracking_js, \
+    MatomoError, MatomoWS, \
+    get_tenant_name_and_public_urls, compute_cnil_acknowledgment_level, \
+    auto_configure_matomo
+
+
+class HomeView(FormView):
+    template_name = 'hobo/matomo_home.html'
+    form_class = SettingsForm
+    success_url = reverse_lazy('matomo-home')
+
+    def get_initial(self):
+        initial = super(HomeView, self).get_initial()
+        initial['tracking_js'] = get_tracking_js()
+        return initial
+
+    def form_valid(self, form):
+        if get_variable('matomo_mode').json == 'manual':
+            tracking_js = form.cleaned_data['tracking_js']
+            put_tracking_js(tracking_js)
+        return super(HomeView, self).form_valid(form)
+
+    def get_context_data(self, **kwargs):
+        context = super(HomeView, self).get_context_data(**kwargs)
+        context['enabled'] = bool(get_variable('matomo_enable').json)
+        context['mode'] = get_variable('matomo_mode').json
+        context['error'] = get_variable('matomo_error').value
+        tracking_js = get_tracking_js()
+        context['cnil_ack_level'] = compute_cnil_acknowledgment_level(tracking_js)
+        context['tracking_js'] = tracking_js
+
+        try:
+            matomo = MatomoWS()
+        except MatomoError:
+            context['matomo_ws_available'] = False
+        else:
+            context['matomo_ws_available'] = True
+            context['matomo_url'] = matomo.url_ws_base
+
+        # automatic mode is using variables on DB
+        if context['mode'] == 'auto':
+            tenant_name, unused = get_tenant_name_and_public_urls()
+            password = get_variable('matomo_password').value
+            context['matomo_user_login'] = tenant_name
+            context['matomo_password'] = hashlib.md5(password).hexdigest()
+            context['id_site'] = get_variable('matomo_id_site').value
+        return context
+
+home = HomeView.as_view()
+
+
+class EnableManualView(FormView):
+    form_class = EnableForm
+    template_name = 'hobo/matomo_enable_manual.html'
+    success_url = reverse_lazy('matomo-home')
+
+    def form_valid(self, form):
+        variable = get_variable('matomo_enable')
+        variable.value = 'true'
+        variable.save()
+        variable = get_variable('matomo_mode')
+        variable.value = 'manual'
+        variable.save()
+        return super(EnableManualView, self).form_valid(form)
+
+enable_manual = EnableManualView.as_view()
+
+
+class EnableAutoView(FormView):
+    form_class = EnableForm
+    template_name = 'hobo/matomo_enable_auto.html'
+    success_url = reverse_lazy('matomo-home')
+
+    def form_valid(self, form):
+        variable = get_variable('matomo_mode')
+        variable.value = 'auto'
+        variable.save()
+        if auto_configure_matomo():
+            variable = get_variable('matomo_enable')
+            variable.value = 'true'
+            variable.save()
+        return super(EnableAutoView, self).form_valid(form)
+
+enable_auto = EnableAutoView.as_view()
+
+
+class DisableView(FormView):
+    form_class = EnableForm
+    template_name = 'hobo/matomo_disable.html'
+    success_url = reverse_lazy('matomo-home')
+
+    def form_valid(self, form):
+        put_tracking_js('')
+        for key, val in [('matomo_enable', 'false'),
+                         ('matomo_mode', ''),
+                         ('matomo_error', '')]:
+            variable = get_variable(key)
+            variable.value = val
+            variable.save()
+        return super(DisableView, self).form_valid(form)
+
+disable = DisableView.as_view()
diff --git a/hobo/settings.py b/hobo/settings.py
index 3678eef..b8b0353 100644
--- a/hobo/settings.py
+++ b/hobo/settings.py
@@ -41,6 +41,7 @@ INSTALLED_APPS = (
     'gadjo',
     'hobo.environment',
     'hobo.franceconnect',
+    'hobo.matomo',
     'hobo.profile',
     'hobo.theme',
     'hobo.emails',
diff --git a/hobo/templates/hobo/home.html b/hobo/templates/hobo/home.html
index 3e957f8..14b6e46 100644
--- a/hobo/templates/hobo/home.html
+++ b/hobo/templates/hobo/home.html
@@ -10,6 +10,7 @@
     <li><a href="{% url 'theme-home' %}">{% trans 'Theme' %}</a></li>
     <li><a href="{% url 'emails-home' %}">{% trans 'Emails' %}</a></li>
     <li><a href="{% url 'franceconnect-home' %}">FranceConnect</a></li>
+    <li><a href="{% url 'matomo-home' %}">{% trans 'User tracking' %}</a></li>
     <li><a href="{% url 'environment-home' %}">{% trans 'Services' %}</a></li>
     <li><a href="{% url 'environment-variables' %}">{% trans 'Variables' %}</a></li>
   </ul>
diff --git a/hobo/urls.py b/hobo/urls.py
index d0f0d52..da16e21 100644
--- a/hobo/urls.py
+++ b/hobo/urls.py
@@ -8,6 +8,7 @@ from .views import admin_required, login, login_local, logout, home, health_json
 from .urls_utils import decorated_includes
 from .environment.urls import urlpatterns as environment_urls
 from .franceconnect.urls import urlpatterns as franceconnect_urls
+from .matomo.urls import urlpatterns as matomo_urls
 from .profile.urls import urlpatterns as profile_urls
 from .theme.urls import urlpatterns as theme_urls
 from .emails.urls import urlpatterns as emails_urls
@@ -20,6 +21,8 @@ urlpatterns = [
                                              include(profile_urls))),
     url(r'^franceconnect/',
         decorated_includes(admin_required, include(franceconnect_urls))),
+    url(r'^matomo/',
+        decorated_includes(admin_required, include(matomo_urls))),
     url(r'^theme/', decorated_includes(admin_required,
                                              include(theme_urls))),
     url(r'^emails/', decorated_includes(admin_required, include(emails_urls))),
diff --git a/tests/test_matomo_views.py b/tests/test_matomo_views.py
new file mode 100644
index 0000000..617ae55
--- /dev/null
+++ b/tests/test_matomo_views.py
@@ -0,0 +1,141 @@
+# -*- coding: utf-8 -*-
+
+import mock
+import pytest
+from requests import Response
+from webtest import TestApp
+
+from django.conf import settings
+from django.contrib.auth.models import User
+from django.test import override_settings
+
+from hobo.environment.models import Variable, Wcs, Combo, Fargo
+from hobo.wsgi import application
+
+pytestmark = pytest.mark.django_db
+
+CONFIG = {'URL': 'https://matomo.test',
+          'TOKEN_AUTH': '1234',
+          'EMAIL_TEMPLATE': 'noreply+%s@entrouvert.test'}
+
+GET_SITE_NOT_ALREADY_THERE = """<?xml version="1.0" encoding="utf-8" ?>
+<result>
+    <error message="An unexpected website was found in the request: website id was set to '42' ." />
+</result>
+"""
+
+ADD_SITE_SUCCESS = """<?xml version="1.0" encoding="utf-8" ?>
+<result>42</result>
+"""
+
+ADD_USER_SUCCESS = """<?xml version="1.0" encoding="utf-8" ?>
+<result>
+    <success message="ok" />
+</result>
+"""
+
+JAVASCRIPT_TAG = """<?xml version="1.0" encoding="utf-8" ?>
+<result>&lt;!-- Matomo --&gt;
+&lt;script type=&quot;text/javascript&quot;&gt;
+  var _paq = window._paq || [];
+  /* tracker methods like &quot;setCustomDimension&quot; should be called before &quot;trackPageView&quot; */
+  _paq.push(['trackPageView']);
+  _paq.push(['enableLinkTracking']);
+  (function() {
+    var u=&quot;//matomo-test.entrouvert.org/&quot;;
+    _paq.push(['setTrackerUrl', u+'piwik.php']);
+    _paq.push(['setSiteId', '7']);
+    var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
+    g.type='text/javascript'; g.async=true; g.defer=true; g.src=u+'piwik.js'; s.parentNode.insertBefore(g,s);
+  })();
+&lt;/script&gt;
+&lt;!-- End Matomo Code --&gt;
+</result>
+"""
+
+@pytest.fixture
+def admin_user():
+    try:
+        user = User.objects.get(username='admin')
+    except User.DoesNotExist:
+        user = User.objects.create_superuser('admin', email=None, password='admin')
+    return user
+
+def login(app, username='admin', password='admin'):
+    login_page = app.get('/login/')
+    login_form = login_page.forms[0]
+    login_form['username'] = username
+    login_form['password'] = password
+    resp = login_form.submit()
+    assert resp.status_int == 302
+    return app
+
+def test_unlogged_access():
+    # connect while not being logged in
+    app = TestApp(application)
+    resp = app.get('/matomo/', status=302)
+    assert resp.location.endswith('/login/?next=/matomo/')
+
+def test_access(admin_user):
+    app = login(TestApp(application))
+    assert app.get('/matomo/', status=200)
+
+def test_disable(admin_user):
+    app = login(TestApp(application))
+    resp1 = app.get('/matomo/disable', status=200)
+    resp2 = resp1.form.submit()
+    assert resp2.location.endswith('/matomo/')
+
+def test_enable_manual(admin_user):
+    """scenario where user manually paste a javascript code"""
+    app = login(TestApp(application))
+    resp1 = app.get('/matomo/enable-manual', status=200)
+    resp2 = resp1.form.submit().follow()
+    resp2.form['tracking_js'] = '...js_code...'
+    resp3 = resp2.form.submit().follow()
+    assert resp3.body.find('Good respect of user rights') != -1
+
+def auto_conf_mocked_post(url, **kwargs):
+    contents = [GET_SITE_NOT_ALREADY_THERE,
+                ADD_SITE_SUCCESS,
+                ADD_USER_SUCCESS,
+                JAVASCRIPT_TAG]
+    response = Response()
+    response._content = contents[auto_conf_mocked_post.cpt]
+    response.status_code = 200
+
+    auto_conf_mocked_post.cpt += 1
+    return response
+
+def test_available_options(admin_user):
+    """check available buttons (manual/automatic configurations)"""
+    with override_settings(MATOMO_SERVER=CONFIG):
+        app = login(TestApp(application))
+        resp = app.get('/matomo/', status=200)
+        assert str(resp).find('href="/matomo/enable-manual"') != -1
+        assert str(resp).find('href="/matomo/enable-auto"') != -1
+
+    # without configuration: no automatic configuration available
+    app = login(TestApp(application))
+    resp = app.get('/matomo/', status=200)
+    assert str(resp).find('href="/matomo/enable-manual"') != -1
+    assert str(resp).find('href="/matomo/enable-auto"') == -1
+
+@mock.patch('requests.post', side_effect=auto_conf_mocked_post)
+def test_enable_auto(mocked_post, admin_user):
+    """automatic scenario"""
+    Combo.objects.create(base_url='https://combo.dev.publik.love',
+                         template_name='portal-user')
+    Wcs.objects.create(base_url='https://wcs.dev.publik.love')
+    Fargo.objects.create(base_url='https://fargo.dev.publik.love')
+
+    auto_conf_mocked_post.cpt = 0
+    with override_settings(MATOMO_SERVER=CONFIG):
+        app = login(TestApp(application))
+        resp1 = app.get('/matomo/enable-auto', status=200)
+        resp2 = resp1.form.submit()
+
+        # call utils.py::auto_configure_matomo()
+        resp3 = resp2.follow()
+        print resp3.body
+        assert resp3.body.find('Excellent respect of user rights') != -1
-- 
2.20.1

